fix(dns): dnsimple expects FQDNs in CNAMEs

DNSimple doesn't appear to follow the typical behaviour of appending the
domain unless the CNAME is terminated with `.`

To avoid further problems, let's just explicilty use the FQDN for all
CNAMEs.

https://support.dnsimple.com/articles/cname-record/

For comparison:
```
;; ANSWER SECTION:
alerts.forkos.org.	300	IN	CNAME	meta01.infra.p.
```

```
;; ANSWER SECTION:
alerts.forkos.org.	181	IN	CNAME	meta01.infra.p.forkos.org.
meta01.infra.p.forkos.org. 181	IN	A	163.172.69.160
```

common/base-server.nix

@@ -55,4 +55,19 @@
     "en_US.UTF-8/UTF-8"
     "fr_FR.UTF-8/UTF-8"
   ];
+
+  time.timeZone = "UTC";
+
+  security.acme.acceptTerms = true;
+  security.acme.defaults.email = "infra@forkos.org";
+
+  # Enable system diffs.
+  system.activationScripts.system-diff = {
+    supportsDryActivation = true; # safe: only outputs to stdout
+    text = ''
+      if [ -e /run/current-system ]; then
+        PATH=$PATH:${pkgs.nix}/bin ${pkgs.nvd}/bin/nvd diff /run/current-system $systemConfig
+      fi
+    '';
+  };
 }

common/channels.nix

@@ -0,0 +1,32 @@
+# Taken from https://github.com/NixOS/infra/blob/master/channels.nix
+{
+  # "Channel name" = {
+  #   # This should be the <value> part of
+  #   # https://hydra.forkos.org/job/<value>/latest-finished
+  #   job = "project/jobset/jobname";
+  #
+  #   # When adding a new version, determine if it needs to be tagged as a
+  #   # variant -- for example:
+  #   # nixos-xx.xx         => primary
+  #   # nixos-xx.xx-small   => small
+  #   # nixos-xx.xx-darwin  => darwin
+  #   # nixos-xx.xx-aarch64 => aarch64
+  #   variant = "primary";
+  #
+  #   # Channel Status:
+  #   # '*-unstable' channels are always "rolling"
+  #   # Otherwise a release generally progresses through the following phases:
+  #   #
+  #   #  - Directly after branch off                   => "beta"
+  #   #  - Once the channel is released                => "stable"
+  #   #  - Once the next channel is released           => "deprecated"
+  #   #  - N months after the next channel is released => "unmaintained"
+  #   #    (check the release notes for when this should happen)
+  #   status = "beta";
+  # };
+  "forkos-unstable" = {
+    job = "forkos/nixos-main/tested";
+    variant = "primary";
+    status = "rolling";
+  };
+}

common/nix.nix

@@ -14,7 +14,7 @@
 
   # Use our cache and trust its signing key. Still use cache.nixos.org as
   # fallback.
-  nix.settings.substituters = [ "https://bagel-cache.s3-web.delroth.net/" ];
+  nix.settings.substituters = [ "https://cache.forkos.org/" ];
   nix.settings.trusted-public-keys = [
     "cache.forkos.org:xfXIUJO1yiEITJmYsVmNDa9BFSlgTh/YqZ+4ei1EhQg="
   ];

common/raito-proxy-aware-nginx.nix

@@ -1,9 +1,10 @@
 # This enables an IPv6-only server which is proxied by kurisu.lahfa.xyz to have proper IPv4 logs via PROXY protocol.
 { config, lib, ... }:
 let
-  inherit (lib) mkEnableOption mkIf;
+  inherit (lib) mkEnableOption mkIf concatStringsSep;
   cfg = config.bagel.raito.v6-proxy-awareness;
-  allowedUpstream = "2001:bc8:38ee:99::1/128";
+  #                      outside of raito infra    inside of raito infra
+  allowedUpstreams = [ "2001:bc8:38ee::1/128"     "2001:bc8:38ee:99::1/128" ];
 in
   {
     options.bagel.raito.v6-proxy-awareness.enable = mkEnableOption "the kurisu.lahfa.xyz's sniproxy awareness for NGINX";
@@ -20,8 +21,8 @@ in
         ];
 
         appendHttpConfig = ''
-          # Kurisu node
-          set_real_ip_from ${allowedUpstream};
+          # Kurisu nodes
+          ${concatStringsSep "\n" (map (up: "set_real_ip_from ${up};") allowedUpstreams)}
           real_ip_header proxy_protocol;
         '';
       };
@@ -29,7 +30,7 @@ in
       # Move to nftables if firewall is enabled.
       networking.nftables.enable = true;
       networking.firewall.extraInputRules = ''
-        ip6 saddr ${allowedUpstream} tcp dport 444 accept
+        ${concatStringsSep "\n" (map (up: "ip6 saddr ${up} tcp dport 444 accept") allowedUpstreams)}
       '';
     };
   }

common/raito-vm.nix

@@ -30,8 +30,6 @@ in
   config = mkIf cfg.enable {
     services.qemuGuest.enable = true;
     systemd.network.enable = true;
-    security.acme.defaults.email = "bagel-acme@lahfa.xyz";
-    security.acme.acceptTerms = true;
     networking.useDHCP = lib.mkDefault false;
 
     systemd.network.networks."10-nat-lan" = {

common/ssh-keys.nix

@@ -2,10 +2,12 @@
   machines = {
     bagel-box = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAsO4bNqY04uG13Pg3ubHfRDssTphDLzZ4YUniE5/p+M";
     meta01 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM5t9gYorOWgpCFDJgb24pyCKIabGpeI2H/UfdvXODcT";
+    public01 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPBy8G8rfLA6E9i+t5kjVafxU1c2NXATXKxoXTH4Kgtm";
     gerrit01 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIA+eSZu+u9sCynrMlsmFzQHLIELQAuVg0Cs1pBvwb4+A";
     fodwatch = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFRyTNfvKl5FcSyzGzw+h+bNFNOxdhvI67WdUZ2iIJ1L";
     buildbot = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJgIu6ouagYqBeMLfmn1CbaDJMuZcPH9bnUhkht8GfuB";
     git = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEQJcpkCUOx8+5oukMX6lxrYcIX8FyHu8Mc/3+ieKMUn";
+    build-coord = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINpAEJP7F+XtJBpQP1jTzwXwQgJrFxwEJjPf/rnCXkJA";
     builder-0 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBHSNcDGctvlG6BHcJuYIzW9WsBJsts2vpwSketsbXoL";
     builder-1 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIQOGUjERK7Mx8UPM/rbOdMqVyn1sbWqYOG6CbOzH2wm";
     builder-2 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMKzXIqCoYElEKIYgjbSpqEcDeOvV+Wo3Agq3jba83cB";
@@ -17,7 +19,6 @@
     builder-8 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKGSWHNeqT0kF/e4yVy2ieW98X5QMyCYIYZh9WTmQDs1";
     builder-9 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOhws9zGgocVY36dMtOL+CXadpvRMffxoWMkfEcTBJm7";
     builder-10 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE7sgIuTSqZiZhp8TvObSbIEhcHHsL5hcmYA22uzwxth";
-    builder-11 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEAqFo1qJY7MSUkfB+zxXB8Lpt/Iqz/RR5A+zwhpRWhr";
     wob-vpn-gw = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINVytPPW8XnXf/rD5TFzsw//CZc2lBjQLmDzlVGPZsjh";
   };
 

dashboards/default.nix

@@ -0,0 +1,43 @@
+{ gerrit-dashboard, stdenv, symlinkJoin, jsonnet, fetchFromGitHub, lib, ... }:
+let
+  inherit (lib) concatMapStringsSep;
+  datasource-id = "mimir";
+in
+rec {
+  grafonnet = fetchFromGitHub {
+    owner = "grafana";
+    repo = "grafonnet-lib";
+    # TODO: figure out how to read the jsonnet lockfile
+    # and propagate this a bit cleverly.
+    rev = "a1d61cce1da59c71409b99b5c7568511fec661ea";
+    hash = "sha256-fs5JZJbcL6sQXBjYhp5eeRtjTFw0J1O/BcwBC8Vm9EM=";
+  };
+  buildJsonnetDashboards = dashboardSrc: targets: stdenv.mkDerivation {
+    name = "jsonnet-grafana-dashboards";
+    src = dashboardSrc;
+    buildInputs = [ jsonnet ];
+    buildPhase = ''
+      runHook preBuild
+      mkdir -p $out
+      ${concatMapStringsSep "\n" (target: "jsonnet -J ${grafonnet} --ext-str datasource=${datasource-id} --ext-code publish=true $src/${target} > $out/${baseNameOf target}.json") targets}
+      runHook postBuild
+    '';
+  };
+
+  allDashboards = symlinkJoin {
+    name = "all-jsonnet-dashboards";
+    paths = [
+      (buildJsonnetDashboards gerrit-dashboard [
+        "dashboards/gerrit/caches/gerrit-caches.jsonnet"
+        "dashboards/gerrit/fetch-clone/gerrit-fetch-clone.jsonnet"
+        "dashboards/gerrit/fetch-clone/gerrit-phases.jsonnet"
+        "dashboards/gerrit/healthcheck/gerrit-healthcheck.jsonnet"
+        "dashboards/gerrit/latency/gerrit-push-latency.jsonnet"
+        "dashboards/gerrit/latency/gerrit-ui-actions-latency.jsonnet"
+        "dashboards/gerrit/overview/gerrit-overview.jsonnet"
+        "dashboards/gerrit/process/gerrit-process.jsonnet"
+        "dashboards/gerrit/queues/gerrit-queues.jsonnet"
+      ])
+    ];
+  };
+}

flake.lock

@@ -10,11 +10,11 @@
         "systems": "systems"
       },
       "locked": {
-        "lastModified": 1720546205,
-        "narHash": "sha256-boCXsjYVxDviyzoEyAk624600f3ZBo/DKtUdvMTpbGY=",
+        "lastModified": 1723293904,
+        "narHash": "sha256-b+uqzj+Wa6xgMS9aNbX4I+sXeb5biPDi39VgvSFqFvU=",
         "owner": "ryantm",
         "repo": "agenix",
-        "rev": "de96bd907d5fbc3b14fc33ad37d1b9a3cb15edc6",
+        "rev": "f6291c5935fdc4e0bef208cfc0dcab7e3f7a1c41",
         "type": "github"
       },
       "original": {
@@ -23,6 +23,29 @@
         "type": "github"
       }
     },
+    "attic": {
+      "inputs": {
+        "crane": "crane",
+        "flake-compat": "flake-compat_2",
+        "flake-utils": "flake-utils_2",
+        "nixpkgs": "nixpkgs",
+        "nixpkgs-stable": "nixpkgs-stable"
+      },
+      "locked": {
+        "lastModified": 1711742460,
+        "narHash": "sha256-0O4v6e4a1toxXZ2gf5INhg4WPE5C5T+SVvsBt+45Mcc=",
+        "owner": "zhaofengli",
+        "repo": "attic",
+        "rev": "4dbdbee45728d8ce5788db6461aaaa89d98081f0",
+        "type": "github"
+      },
+      "original": {
+        "owner": "zhaofengli",
+        "ref": "main",
+        "repo": "attic",
+        "type": "github"
+      }
+    },
     "bats-assert": {
       "flake": false,
       "locked": {
@@ -64,11 +87,11 @@
         "treefmt-nix": "treefmt-nix"
       },
       "locked": {
-        "lastModified": 1722339507,
-        "narHash": "sha256-CdkQx6l0CuPtU9tMw3u73PAMhj+8YUAlTUVyrTCZas8=",
+        "lastModified": 1722939563,
+        "narHash": "sha256-lMe8aXgF550iQLRaoU+yn8yYQ4x2qiyqANgsFyjfWwA=",
         "ref": "refs/heads/non-flakes",
-        "rev": "15963fa0e687cb39660aadf147402c955d671029",
-        "revCount": 292,
+        "rev": "4a162a8aa5dad6cecdb33bd8534e67e0bdaeb13f",
+        "revCount": 295,
         "type": "git",
         "url": "https://git.lix.systems/lix-project/buildbot-nix.git"
       },
@@ -78,6 +101,26 @@
         "url": "https://git.lix.systems/lix-project/buildbot-nix.git"
       }
     },
+    "channel-scripts": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1725128016,
+        "narHash": "sha256-4TvaXELsl+1OcGNgqB/5HVXVxBvdIQkhJsY4FyiDcNU=",
+        "ref": "refs/heads/main",
+        "rev": "23b6c38ed7e11417bf624f6e4fb6cde0d2be6400",
+        "revCount": 261,
+        "type": "git",
+        "url": "https://git.lix.systems/the-distro/channel-scripts.git"
+      },
+      "original": {
+        "type": "git",
+        "url": "https://git.lix.systems/the-distro/channel-scripts.git"
+      }
+    },
     "colmena": {
       "inputs": {
         "flake-compat": "flake-compat",
@@ -101,6 +144,50 @@
         "type": "github"
       }
     },
+    "crane": {
+      "inputs": {
+        "nixpkgs": [
+          "grapevine",
+          "attic",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1702918879,
+        "narHash": "sha256-tWJqzajIvYcaRWxn+cLUB9L9Pv4dQ3Bfit/YjU5ze3g=",
+        "owner": "ipetkov",
+        "repo": "crane",
+        "rev": "7195c00c272fdd92fc74e7d5a0a2844b9fadb2fb",
+        "type": "github"
+      },
+      "original": {
+        "owner": "ipetkov",
+        "repo": "crane",
+        "type": "github"
+      }
+    },
+    "crane_2": {
+      "inputs": {
+        "nixpkgs": [
+          "grapevine",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1716569590,
+        "narHash": "sha256-5eDbq8TuXFGGO3mqJFzhUbt5zHVTf5zilQoyW5jnJwo=",
+        "owner": "ipetkov",
+        "repo": "crane",
+        "rev": "109987da061a1bf452f435f1653c47511587d919",
+        "type": "github"
+      },
+      "original": {
+        "owner": "ipetkov",
+        "ref": "master",
+        "repo": "crane",
+        "type": "github"
+      }
+    },
     "darwin": {
       "inputs": {
         "nixpkgs": [
@@ -123,6 +210,29 @@
         "type": "github"
       }
     },
+    "fenix": {
+      "inputs": {
+        "nixpkgs": [
+          "grapevine",
+          "nixpkgs"
+        ],
+        "rust-analyzer-src": "rust-analyzer-src"
+      },
+      "locked": {
+        "lastModified": 1716359173,
+        "narHash": "sha256-pYcjP6Gy7i6jPWrjiWAVV0BCQp+DdmGaI/k65lBb/kM=",
+        "owner": "nix-community",
+        "repo": "fenix",
+        "rev": "b6fc5035b28e36a98370d0eac44f4ef3fd323df6",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "ref": "main",
+        "repo": "fenix",
+        "type": "github"
+      }
+    },
     "flake-compat": {
       "flake": false,
       "locked": {
@@ -140,6 +250,39 @@
       }
     },
     "flake-compat_2": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1673956053,
+        "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "flake-compat_3": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1696426674,
+        "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "ref": "master",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "flake-compat_4": {
       "flake": false,
       "locked": {
         "lastModified": 1696426674,
@@ -185,11 +328,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1719994518,
-        "narHash": "sha256-pQMhCCHyQGRzdfAkdJ4cIWiw+JNuWsTX7f0ZYSyz0VY=",
+        "lastModified": 1722555600,
+        "narHash": "sha256-XOQkdLafnb/p9ij77byFQjDf5m5QYl9b2REiVClC+x4=",
         "owner": "hercules-ci",
         "repo": "flake-parts",
-        "rev": "9227223f6d922fee3c7b190b2cc238a99527bbb7",
+        "rev": "8471fe90ad337a8074e957b69ca4d0089218391d",
         "type": "github"
       },
       "original": {
@@ -214,6 +357,40 @@
       }
     },
     "flake-utils_2": {
+      "locked": {
+        "lastModified": 1667395993,
+        "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "flake-utils_3": {
+      "inputs": {
+        "systems": "systems_2"
+      },
+      "locked": {
+        "lastModified": 1710146030,
+        "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "ref": "main",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "flake-utils_4": {
       "locked": {
         "lastModified": 1634851050,
         "narHash": "sha256-N83GlSGPJJdcqhUxSCS/WwW5pksYf3VP1M13cDRTSVA=",
@@ -228,6 +405,51 @@
         "type": "github"
       }
     },
+    "gerrit-dashboard": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1724509518,
+        "narHash": "sha256-fwYXZVddxfzrlDa3QnFCwHqrbEX+3PrWy0QOlbO+8jk=",
+        "ref": "refs/heads/master",
+        "rev": "e544abac81c581558d68abb2a8dd583049073939",
+        "revCount": 75,
+        "type": "git",
+        "url": "https://git.lix.systems/the-distro/gerrit-monitoring.git"
+      },
+      "original": {
+        "type": "git",
+        "url": "https://git.lix.systems/the-distro/gerrit-monitoring.git"
+      }
+    },
+    "grapevine": {
+      "inputs": {
+        "attic": "attic",
+        "crane": "crane_2",
+        "fenix": "fenix",
+        "flake-compat": "flake-compat_3",
+        "flake-utils": "flake-utils_3",
+        "nix-filter": "nix-filter",
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "rust-manifest": "rust-manifest"
+      },
+      "locked": {
+        "host": "gitlab.computer.surgery",
+        "lastModified": 1723576377,
+        "narHash": "sha256-sTa4XT5xMQkhhLknOfVd433YS1TvkMrE45qAsI1ZB6U=",
+        "owner": "matrix",
+        "repo": "grapevine-fork",
+        "rev": "3b99032456700d06dd937db6a85976a8be9d4fa7",
+        "type": "gitlab"
+      },
+      "original": {
+        "host": "gitlab.computer.surgery",
+        "owner": "matrix",
+        "repo": "grapevine-fork",
+        "type": "gitlab"
+      }
+    },
     "home-manager": {
       "inputs": {
         "nixpkgs": [
@@ -258,11 +480,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1721682989,
-        "narHash": "sha256-kjJiZ7m4HKqbZ2mxNQiB32/goKFb8BRi8OqC4wIU0OI=",
+        "lastModified": 1724616313,
+        "narHash": "sha256-9syppf9Gm/6F4wQQAbsf7rGY1DooMsprnsEY/0eaewg=",
         "ref": "refs/heads/main",
-        "rev": "4b107e6ff36bd89958fba36e0fe0340903e7cd13",
-        "revCount": 4190,
+        "rev": "44b9a7b95d23e7a8587cb963f00382046707f2db",
+        "revCount": 4202,
         "type": "git",
         "url": "https://git.lix.systems/lix-project/hydra.git"
       },
@@ -273,7 +495,7 @@
     },
     "lix": {
       "inputs": {
-        "flake-compat": "flake-compat_2",
+        "flake-compat": "flake-compat_4",
         "nix2container": "nix2container",
         "nixpkgs": [
           "hydra",
@@ -283,11 +505,11 @@
         "pre-commit-hooks": "pre-commit-hooks"
       },
       "locked": {
-        "lastModified": 1721091462,
-        "narHash": "sha256-0cmEeoOiB91BviTJHzIyxkY+Gxv3O8ZnnExVAoXEFGI=",
+        "lastModified": 1723919517,
+        "narHash": "sha256-D6+zmRXzr85p7riphuIrJQqangoJe70XM5jHhMWwXws=",
         "ref": "refs/heads/main",
-        "rev": "6b4d46e9e0e1dd80e0977684ab20d14bcd1a6bc3",
-        "revCount": 15967,
+        "rev": "278fddc317cf0cf4d3602d0ec0f24d1dd281fadb",
+        "revCount": 16138,
         "type": "git",
         "url": "https://git.lix.systems/lix-project/lix"
       },
@@ -311,11 +533,11 @@
         "treefmt-nix": "treefmt-nix_2"
       },
       "locked": {
-        "lastModified": 1721195872,
-        "narHash": "sha256-TlvRq634MSl22BWLmpTy2vdtKntbZlsUwdMq8Mp9AWs=",
+        "lastModified": 1723579251,
+        "narHash": "sha256-xnHtfw0gRhV+2S9U7hQwvp2klTy1Iv7FlMMO0/WiMVc=",
         "ref": "refs/heads/main",
-        "rev": "c057494450f2d1420726ddb0bab145a5ff4ddfdd",
-        "revCount": 608,
+        "rev": "42a160bce2fd9ffebc3809746bc80cc7208f9b08",
+        "revCount": 609,
         "type": "git",
         "url": "https://git.lix.systems/lix-project/nix-eval-jobs"
       },
@@ -324,6 +546,22 @@
         "url": "https://git.lix.systems/lix-project/nix-eval-jobs"
       }
     },
+    "nix-filter": {
+      "locked": {
+        "lastModified": 1710156097,
+        "narHash": "sha256-1Wvk8UP7PXdf8bCCaEoMnOT1qe5/Duqgj+rL8sRQsSM=",
+        "owner": "numtide",
+        "repo": "nix-filter",
+        "rev": "3342559a24e85fc164b295c3444e8a139924675b",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "ref": "main",
+        "repo": "nix-filter",
+        "type": "github"
+      }
+    },
     "nix-gerrit": {
       "inputs": {
         "nixpkgs": [
@@ -369,11 +607,11 @@
     "nix2container": {
       "flake": false,
       "locked": {
-        "lastModified": 1712990762,
-        "narHash": "sha256-hO9W3w7NcnYeX8u8cleHiSpK2YJo7ecarFTUlbybl7k=",
+        "lastModified": 1720642556,
+        "narHash": "sha256-qsnqk13UmREKmRT7c8hEnz26X3GFFyIQrqx4EaRc1Is=",
         "owner": "nlewo",
         "repo": "nix2container",
-        "rev": "20aad300c925639d5d6cbe30013c8357ce9f2a2e",
+        "rev": "3853e5caf9ad24103b13aa6e0e8bcebb47649fe4",
         "type": "github"
       },
       "original": {
@@ -384,11 +622,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1721116560,
-        "narHash": "sha256-++TYlGMAJM1Q+0nMVaWBSEvEUjRs7ZGiNQOpqbQApCU=",
+        "lastModified": 1711401922,
+        "narHash": "sha256-QoQqXoj8ClGo0sqD/qWKFWezgEwUL0SUh37/vY2jNhc=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "9355fa86e6f27422963132c2c9aeedb0fb963d93",
+        "rev": "07262b18b97000d16a4bdb003418bd2fb067a932",
         "type": "github"
       },
       "original": {
@@ -414,17 +652,34 @@
         "type": "github"
       }
     },
-    "nixpkgs_2": {
+    "nixpkgs-stable": {
       "locked": {
-        "lastModified": 1636823747,
-        "narHash": "sha256-oWo1nElRAOZqEf90Yek2ixdHyjD+gqtS/pAgwaQ9UhQ=",
-        "owner": "nixos",
+        "lastModified": 1711460390,
+        "narHash": "sha256-akSgjDZL6pVHEfSE6sz1DNSXuYX6hq+P/1Z5IoYWs7E=",
+        "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "f6a2ed2082d9a51668c86ba27d0b5496f7a2ea93",
+        "rev": "44733514b72e732bd49f5511bd0203dea9b9a434",
         "type": "github"
       },
       "original": {
-        "owner": "nixos",
+        "owner": "NixOS",
+        "ref": "nixos-23.11",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs_2": {
+      "locked": {
+        "lastModified": 1723221148,
+        "narHash": "sha256-7pjpeQlZUNQ4eeVntytU3jkw9dFK3k1Htgk2iuXjaD8=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "154bcb95ad51bc257c2ce4043a725de6ca700ef6",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixpkgs-unstable",
         "repo": "nixpkgs",
         "type": "github"
       }
@@ -432,11 +687,11 @@
     "pre-commit-hooks": {
       "flake": false,
       "locked": {
-        "lastModified": 1712055707,
-        "narHash": "sha256-4XLvuSIDZJGS17xEwSrNuJLL7UjDYKGJSbK1WWX2AK8=",
+        "lastModified": 1721042469,
+        "narHash": "sha256-6FPUl7HVtvRHCCBQne7Ylp4p+dpP3P/OYuzjztZ4s70=",
         "owner": "cachix",
         "repo": "git-hooks.nix",
-        "rev": "e35aed5fda3cc79f88ed7f1795021e559582093a",
+        "rev": "f451c19376071a90d8c58ab1a953c6e9840527fd",
         "type": "github"
       },
       "original": {
@@ -449,17 +704,49 @@
       "inputs": {
         "agenix": "agenix",
         "buildbot-nix": "buildbot-nix",
+        "channel-scripts": "channel-scripts",
         "colmena": "colmena",
+        "gerrit-dashboard": "gerrit-dashboard",
+        "grapevine": "grapevine",
         "hydra": "hydra",
         "lix": [
           "hydra",
           "lix"
         ],
         "nix-gerrit": "nix-gerrit",
-        "nixpkgs": "nixpkgs",
+        "nixpkgs": "nixpkgs_2",
         "terranix": "terranix"
       }
     },
+    "rust-analyzer-src": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1716107283,
+        "narHash": "sha256-NJgrwLiLGHDrCia5AeIvZUHUY7xYGVryee0/9D3Ir1I=",
+        "owner": "rust-lang",
+        "repo": "rust-analyzer",
+        "rev": "21ec8f523812b88418b2bfc64240c62b3dd967bd",
+        "type": "github"
+      },
+      "original": {
+        "owner": "rust-lang",
+        "ref": "nightly",
+        "repo": "rust-analyzer",
+        "type": "github"
+      }
+    },
+    "rust-manifest": {
+      "flake": false,
+      "locked": {
+        "narHash": "sha256-aZFye4UrtlcvLHrISldx4g9uGt3thDbVlLMK5keBSj0=",
+        "type": "file",
+        "url": "https://static.rust-lang.org/dist/channel-rust-1.78.0.toml"
+      },
+      "original": {
+        "type": "file",
+        "url": "https://static.rust-lang.org/dist/channel-rust-1.78.0.toml"
+      }
+    },
     "stable": {
       "locked": {
         "lastModified": 1696039360,
@@ -491,12 +778,29 @@
         "type": "github"
       }
     },
+    "systems_2": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
     "terranix": {
       "inputs": {
         "bats-assert": "bats-assert",
         "bats-support": "bats-support",
-        "flake-utils": "flake-utils_2",
-        "nixpkgs": "nixpkgs_2",
+        "flake-utils": "flake-utils_4",
+        "nixpkgs": [
+          "nixpkgs"
+        ],
         "terranix-examples": "terranix-examples"
       },
       "locked": {
@@ -558,11 +862,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1721059077,
-        "narHash": "sha256-gCICMMX7VMSKKt99giDDtRLkHJ0cwSgBtDijJAqTlto=",
+        "lastModified": 1723454642,
+        "narHash": "sha256-S0Gvsenh0II7EAaoc9158ZB4vYyuycvMGKGxIbERNAM=",
         "owner": "numtide",
         "repo": "treefmt-nix",
-        "rev": "0fb28f237f83295b4dd05e342f333b447c097398",
+        "rev": "349de7bc435bdff37785c2466f054ed1766173be",
         "type": "github"
       },
       "original": {

flake.nix

@@ -3,7 +3,9 @@
 
   inputs = {
     nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
+
     terranix.url = "github:terranix/terranix";
+    terranix.inputs.nixpkgs.follows = "nixpkgs";
 
     agenix.url = "github:ryantm/agenix";
     agenix.inputs.nixpkgs.follows = "nixpkgs";
@@ -17,10 +19,24 @@
     nix-gerrit.url = "git+https://git.lix.systems/the-distro/nix-gerrit.git";
     nix-gerrit.inputs.nixpkgs.follows = "nixpkgs";
 
+    gerrit-dashboard.url = "git+https://git.lix.systems/the-distro/gerrit-monitoring.git";
+    gerrit-dashboard.flake = false;
+
     buildbot-nix.url = "git+https://git.lix.systems/lix-project/buildbot-nix.git?ref=refs/heads/non-flakes";
     buildbot-nix.inputs.nixpkgs.follows = "nixpkgs";
 
+    channel-scripts.url = "git+https://git.lix.systems/the-distro/channel-scripts.git";
+    channel-scripts.inputs.nixpkgs.follows = "nixpkgs";
+
     lix.follows = "hydra/lix";
+
+    grapevine = {
+      type = "gitlab";
+      host = "gitlab.computer.surgery";
+      owner = "matrix";
+      repo = "grapevine-fork";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
   };
 
   outputs = { self, nixpkgs, terranix, colmena, ... } @ inputs:
@@ -38,6 +54,7 @@
           inputs.hydra.overlays.default
           inputs.lix.overlays.default
           inputs.nix-gerrit.overlays.default
+          inputs.channel-scripts.overlays.default
         ];
       };
       terraform = pkgs.opentofu;
@@ -46,6 +63,7 @@
         modules = [
           ./terraform
           {
+            bagel.dnsimple.enable = true;
             bagel.gandi.enable = true;
             bagel.hydra.enable = true;
           }
@@ -99,7 +117,7 @@
         bagel.baremetal.builders = { enable = true; num = i; netboot = i >= 6; };
       };
 
-      builders = lib.listToAttrs (lib.genList makeBuilder 12);
+      builders = lib.listToAttrs (lib.genList makeBuilder 11);
     in {
       meta.nixpkgs = systemBits.x86_64-linux.pkgs;
       meta.specialArgs.inputs = inputs;
@@ -112,6 +130,7 @@
       wob-vpn-gw.imports = commonModules ++ [ ./hosts/wob-vpn-gw ];
       buildbot.imports = commonModules ++ [ ./hosts/buildbot ];
       public01.imports = commonModules ++ [ ./hosts/public01 ];
+      build-coord.imports = commonModules ++ [ ./hosts/build-coord ];
     } // builders;
 
     hydraJobs = builtins.mapAttrs (n: v: v.config.system.build.netbootDir or v.config.system.build.toplevel) self.nixosConfigurations;

hosts/bagel-box/default.nix

@@ -37,20 +37,11 @@
 
   bagel.services = {
     postgres.enable = true;
-
-    hydra.enable = true;
-    hydra.dbi = "dbi:Pg:dbname=hydra;user=hydra";
-    # Takes 10 builders (0 → 9).
-    hydra.builders = lib.genList (i: "builder-${builtins.toString i}") 10;
-
     ofborg.enable = true;
   };
 
   bagel.sysadmin.enable = true;
 
-  security.acme.acceptTerms = true;
-  security.acme.defaults.email = "infra@forkos.org";
-
   services.openssh.enable = true;
 
   system.stateVersion = "24.11";

hosts/build-coord/default.nix

@@ -0,0 +1,21 @@
+{ lib, ... }:
+{
+  imports = [ ./hardware.nix ];
+
+  networking.hostName = "build-coord";
+  networking.domain = "wob01.infra.forkos.org";
+
+  bagel.sysadmin.enable = true;
+
+  bagel.services = {
+    hydra.enable = true;
+    # Takes 10 builders (0 → 9).
+    hydra.builders = lib.genList (i: "builder-${builtins.toString i}") 10;
+  };
+
+  # Hydra is proxied.
+  bagel.raito.v6-proxy-awareness.enable = true;
+
+  system.stateVersion = "24.05";
+  deployment.targetHost = "build-coord.wob01.infra.forkos.org";
+}

hosts/build-coord/hardware.nix

@@ -0,0 +1,87 @@
+{
+  boot.initrd.availableKernelModules = [ "ahci" "ehci_pci" "usb_storage" "usbhid" "sd_mod" ];
+  boot.initrd.kernelModules = [ "dm-snapshot" ];
+
+  nixpkgs.hostPlatform = "x86_64-linux";
+  hardware.cpu.intel.updateMicrocode = true;
+
+  boot.loader.systemd-boot.enable = true;
+  boot.loader.efi.canTouchEfiVariables = true;
+  boot.initrd.systemd.enable = true;
+
+  boot.initrd.services.lvm.enable = true;
+
+  boot.kernelParams = [
+    "console=tty1"
+    "console=ttyS0,115200"
+  ];
+
+  fileSystems = {
+    "/" = {
+      device = "/dev/disk/by-label/root";
+      fsType = "ext4";
+    };
+
+    "/boot" = {
+      device = "/dev/disk/by-label/BOOT";
+      fsType = "vfat";
+      options = [ "fmask=0022" "dmask=0022" ];
+    };
+  };
+
+  swapDevices = [
+    {
+      device = "/swapfile";
+      size = 20 * 1024;  # 50GiB
+    }
+  ];
+
+  zramSwap = {
+    enable = true;
+    memoryPercent = 25;
+  };
+
+  networking.useNetworkd = true;
+
+  systemd.network = {
+    netdevs = {
+      "40-uplink" = {
+        netdevConfig = {
+          Kind = "bond";
+          Name = "uplink";
+        };
+        bondConfig = {
+          Mode = "802.3ad";
+          TransmitHashPolicy = "layer3+4";
+        };
+      };
+    };
+    networks = {
+      "40-eno1" = {
+        name = "eno1";
+        bond = [ "uplink" ];
+      };
+      "40-eno2" = {
+        name = "eno2";
+        bond = [ "uplink" ];
+      };
+    };
+  };
+  networking.interfaces.uplink.ipv6.addresses = [
+    { address = "2a01:584:11::1:11"; prefixLength = 64; }
+  ];
+  networking.defaultGateway6 = { interface = "uplink"; address = "2a01:584:11::1"; };
+
+  services.coredns = {
+    enable = true;
+    config = ''
+      . {
+        bind lo
+        forward . 2001:4860:4860::6464
+        template ANY A { rcode NOERROR }
+      }
+    '';
+  };
+  services.resolved.enable = false;
+  networking.resolvconf.useLocalResolver = true;
+}

hosts/buildbot/default.nix

@@ -9,8 +9,6 @@
   # TODO: make it the default
   networking.domain = "infra.forkos.org";
 
-  time.timeZone = "Europe/Paris";
-
   bagel.sysadmin.enable = true;
   # Buildbot is proxied.
   bagel.raito.v6-proxy-awareness.enable = true;
@@ -28,7 +26,7 @@
   bagel.services.buildbot = {
     enable = true;
     domain = "buildbot.forkos.org";
-    builders = [ "builder-11" ];
+    builders = [ "builder-10" ];
   };
 
   i18n.defaultLocale = "en_US.UTF-8";

hosts/fodwatch/default.nix

@@ -8,8 +8,6 @@
   networking.hostName = "fodwatch";
   networking.domain = "infra.forkos.org";
 
-  time.timeZone = "Europe/Paris";
-
   bagel.sysadmin.enable = true;
   # Fodwatch will be proxied.
   bagel.raito.v6-proxy-awareness.enable = true;

hosts/gerrit01/default.nix

@@ -9,8 +9,6 @@
   # TODO: make it the default
   networking.domain = "infra.forkos.org";
 
-  time.timeZone = "Europe/Paris";
-
   bagel.sysadmin.enable = true;
   # Gerrit is proxied.
   bagel.raito.v6-proxy-awareness.enable = true;
@@ -32,6 +30,7 @@
 
   bagel.services.gerrit = {
     enable = true;
+    pyroscope.enable = true;
     domains = [
       "cl.forkos.org"
     ];
@@ -47,12 +46,13 @@
   };
   bagel.nixpkgs.one-way-sync =
   let
-    mkNixpkgsJob = { timer, branchName, localRefspec ? null }: {
-      name = "nixpkgs-${branchName}";
+    mkNixpkgsJob = { timer, fromRefspec, localRefspec ? fromRefspec }: {
       fromUri = "https://github.com/NixOS/nixpkgs";
-      fromRefspec = branchName;
-      localRefspec = if localRefspec != null then localRefspec else branchName;
-      inherit timer;
+      inherit fromRefspec localRefspec timer;
+    };
+    mkLocalJob = { timer, fromRefspec, localRefspec }: {
+      fromUri = "https://cl.forkos.org/nixpkgs";
+      inherit fromRefspec localRefspec timer;
     };
     in
   {
@@ -61,48 +61,59 @@
     pushUrl = "ssh://ows_bot@cl.forkos.org:29418/nixpkgs";
     deployKeyPath = config.age.secrets.ows-deploy-key.path;
 
-    branches."refs/heads/main" = mkNixpkgsJob {
+    # Sync main -> staging-next -> staging
+    branches."main-to-staging-next" = mkLocalJob {
+      timer = "00/8:20:00"; # every 8 hours, 20 minutes past the full hour
+      fromRefspec = "main";
+      localRefspec = "staging-next";
+    };
+    branches."staging-next-to-staging" = mkLocalJob {
+      timer = "00/8:40:00"; # every 8 hours, 40 minutes past the full hour
+      fromRefspec = "staging-next";
+      localRefspec = "staging";
+    };
+
+    # Sync nixpkgs -> fork
+    branches."nixpkgs-master" = mkNixpkgsJob {
       timer = "hourly";
-      branchName = "master";
+      fromRefspec = "master";
       localRefspec = "main";
     };
 
-    branches."refs/heads/staging" = mkNixpkgsJob {
+    branches."nixpkgs-staging" = mkNixpkgsJob {
       timer = "hourly";
-      branchName = "staging";
+      fromRefspec = "staging";
     };
 
-    branches."refs/heads/release-24.05" = mkNixpkgsJob {
+    branches."nixpkgs-release-24.05" = mkNixpkgsJob {
       timer = "hourly";
-      branchName = "release-24.05";
+      fromRefspec = "release-24.05";
     };
 
-    branches."refs/heads/staging-24.05" = mkNixpkgsJob {
+    branches."nixpkgs-staging-24.05" = mkNixpkgsJob {
       timer = "hourly";
-      branchName = "staging-24.05";
+      fromRefspec = "staging-24.05";
     };
 
-    branches."refs/heads/release-23.11" = mkNixpkgsJob {
+    branches."nixpkgs-release-23.11" = mkNixpkgsJob {
       timer = "hourly";
-      branchName = "release-23.11";
+      fromRefspec = "release-23.11";
     };
 
-    branches."refs/heads/staging-23.11" = mkNixpkgsJob {
+    branches."nixpkgs-staging-23.11" = mkNixpkgsJob {
       timer = "hourly";
-      branchName = "staging-23.11";
+      fromRefspec = "staging-23.11";
     };
 
     # Testing jobs for personal sandbox branches
-    branches."refs/heads/sandbox/raito/raito-unstable-small" = {
-      name = "raito-unstable-sync";
+    branches."raito-unstable-sync" = {
       fromUri = "https://github.com/NixOS/nixpkgs";
       fromRefspec = "nixos-unstable-small";
       localRefspec = "sandbox/raito/raito-unstable-small";
       timer = "*-*-* 12:00:00";
     };
 
-    branches."refs/heads/sandbox/raito/raito-nixos-24.05" = {
-      name = "raito-release-sync";
+    branches."raito-release-sync" = {
       fromUri = "https://github.com/NixOS/nixpkgs";
       fromRefspec = "nixos-24.05";
       localRefspec = "sandbox/raito/raito-nixos-24.05";
@@ -110,6 +121,26 @@
     };
   };
 
+  age.secrets.s3-channel-staging-keys.file = ../../secrets/s3-channel-staging-keys.age;
+  bagel.nixpkgs.channel-scripts = {
+    enable = true;
+    otlp.enable = true;
+    nixpkgsUrl = "https://cl.forkos.org/nixpkgs.git";
+    hydraUrl = "https://hydra.forkos.org";
+    binaryCacheUrl = "https://cache.forkos.org";
+    baseUriForGitRevisions = "https://cl.forkos.org/plugins/gitiles/nixpkgs/+";
+    s3 = {
+      release = "bagel-channel-scripts-test";
+      channel = "bagel-channel-scripts-test";
+    };
+    releaseBucketCredentialsFile = config.age.secrets.s3-channel-staging-keys.path;
+    deployKeyFile = config.age.secrets.priv-ssh-key.path;
+    extraArgs = [
+      "--bypass-preflight-checks"
+    ];
+    channels = import ../../common/channels.nix;
+  };
+
   i18n.defaultLocale = "fr_FR.UTF-8";
 
   system.stateVersion = "24.05";

hosts/git/default.nix

@@ -8,8 +8,6 @@ in
   networking.hostName = "git";
   networking.domain = "infra.forkos.org";
 
-  time.timeZone = "Europe/Paris";
-
   bagel.sysadmin.enable = true;
   # Forgejo will be proxied.
   bagel.raito.v6-proxy-awareness.enable = true;

hosts/meta01/default.nix

@@ -2,8 +2,6 @@
   networking.hostName = "meta01";
   networking.domain = "infra.forkos.org";
 
-  time.timeZone = "Europe/Paris";
-
   bagel.sysadmin.enable = true;
   # netbox is proxied.
   bagel.raito.v6-proxy-awareness.enable = true;
@@ -24,6 +22,15 @@
   bagel.services.prometheus.enable = true;
   bagel.services.loki.enable = true;
   bagel.services.grafana.enable = true;
+  bagel.services.grapevine.enable = true;
+  bagel.services.pyroscope.enable = true;
+  bagel.services.tempo.enable = true;
+  bagel.services.hookshot = {
+    enable = true;
+    admins = [
+      "@k900:0upti.me"
+    ];
+  };
 
   i18n.defaultLocale = "fr_FR.UTF-8";
 

hosts/public01/default.nix

@@ -9,11 +9,13 @@
   # TODO: make it the default
   networking.domain = "infra.forkos.org";
 
-  time.timeZone = "Europe/Paris";
-
   bagel.sysadmin.enable = true;
-  # Buildbot is proxied.
+  # Newsletter is proxied.
   bagel.raito.v6-proxy-awareness.enable = true;
+  bagel.newsletter = {
+    enable = true;
+    domain = "news.forkos.org";
+  };
   bagel.hardware.raito-vm = {
     enable = true;
     networking = {
@@ -25,6 +27,17 @@
     };
   };
 
+  bagel.services.s3-revproxy = {
+    enable = true;
+    domain = "forkos.org";
+    s3.apiUrl = "s3.delroth.net";
+    targets = {
+      channels = "bagel-channels";
+      releases = "bagel-releases";
+      channel-scripts-test = "bagel-channel-scripts-test";
+    };
+  };
+
   i18n.defaultLocale = "en_US.UTF-8";
 
   system.stateVersion = "24.05";

hosts/wob-vpn-gw/netboot.nix

@@ -1,56 +1,61 @@
-{ lib, pkgs, nodes, config, ... }:
+{ config, lib, pkgs, nodes, modulesPath, ... }:
 
 # The way the connection is established is specific to the wob01 site and the Intel S2600KPR blades.
 # Proper netboot is not possible, because while the blades and the APU board (which is the netboot
 # server here) are in the same L2 network, the uplink connection of each blade is an LACP LAG,
 # meaning that the switch on the other side will only enable the port if it sees valid LACP packets.
-# IPXE sends out these LACP packets while it is probing the ports, however the NICs of the blades
-# do not have a flash which IPXE could be written to.
 # We work around this by presenting a virtual floppy drive using the "IUSB" protocol of the BMC.
-# This virtual floppy drive contains an per-blade customized IPXE script which will initialize the
-# network connection including IP configuration and chainload the actual script off the netboot
-# server.
+# This virtual floppy drive contains an per-blade customized initramfs which will initialize the
+# network connection including IP configuration and load the actual image off hydra.
 
 let
   netboot-server-ip = "2a01:584:11::2";
+  netbootNodes = lib.filterAttrs (_: node: node.config.bagel.baremetal.builders.enable && node.config.bagel.baremetal.builders.netboot) nodes;
 in {
-  networking.firewall.allowedTCPPorts = [ 80 ];
+
+  assertions = [
+    {
+      assertion = !(lib.elem 443 config.networking.firewall.allowedTCPPorts);
+      message = ''
+        Port 443 is in networking.firewalls.allowedTCPPorts, but should be only manually
+        allowed for specific IPs and source ports in ${builtins.toJSON __curPos}
+      '';
+    }
+  ];
 
   systemd.services = lib.mapAttrs' (nodename: node: let
-    ip = "2a01:584:11::1:${toString node.config.bagel.baremetal.builders.num}";
     bmcIp = "192.168.1.${toString (node.config.bagel.baremetal.builders.num * 4 + 2)}";
-    gw = "2a01:584:11::1";
-    dns = "2a01:580:6000::ff01";
-    ipxe = node.pkgs.ipxe.override {
-      embedScript = builtins.toFile "bootstrap-${node.config.networking.hostName}.ipxe" ''
-        #!ipxe
-        ifopen net0
-
-        echo ip ${ip}/64
-        set net0/ip6:ipv6 ${ip}
-        set net0/len6:int8 64
-        echo gw ${gw}
-        set net0/gateway6:ipv6 ${gw}
-        echo dns ${dns}
-        set net0/dns6:ipv6 ${dns}
-
-        # wait for the lacp link to come up
-        ping --count 20 ${gw}
-
-        chain https://hydra.forkos.org/job/infra/main/${node.config.networking.hostName}/latest/download-by-type/file/ipxe
-
-        # if it fails, show a shell 
-        shell
-      '';
-    };
+    notipxe = node.config.system.build.notipxe.config.system.build.usbImage;
   in lib.nameValuePair "iusb-spoof-${nodename}" {
     wantedBy = [ "multi-user.target" ];
     serviceConfig = {
-      Restart = "on-failure";
+      Restart = "always";
     };
     script = ''
       AUTH_TOKEN=$(${pkgs.iusb-spoof}/bin/make-token ${bmcIp})
-      exec ${pkgs.iusb-spoof}/bin/iusb-spoof -r ${bmcIp} 5123 $AUTH_TOKEN ${ipxe}/ipxe-efi.usb
+      exec ${pkgs.iusb-spoof}/bin/iusb-spoof -r ${bmcIp} 5123 $AUTH_TOKEN ${notipxe}
     '';
-  }) (lib.filterAttrs (_: node: node.config.bagel.baremetal.builders.enable && node.config.bagel.baremetal.builders.netboot) nodes);
+  }) netbootNodes;
+
+  # Since the builders are stateless, they can not store their ssh hostkeys
+  networking.firewall.allowedTCPPorts = [ 80 ]; # for ACME
+  networking.firewall.extraInputRules = ''
+    ip6 saddr 2a01:584:11::/64 tcp sport < 1024 tcp dport 443 accept;
+  '';
+  services.nginx = {
+    enable = true;
+    virtualHosts."vpn-gw.wob01.infra.forkos.org" = {
+      enableACME = true;
+      forceSSL = true;
+      locations = lib.mapAttrs' (nodename: node: let
+        ip = "2a01:584:11::1:${toString node.config.bagel.baremetal.builders.num}";
+      in lib.nameValuePair "/${nodename}/" {
+        root = "/var/www";
+        extraConfig = ''
+          allow ${ip};
+          deny all;
+        '';
+      }) netbootNodes;
+    };
+  };
 }

overlays/default.nix

@@ -1,5 +1,9 @@
 [
   (final: prev: {
     iusb-spoof = final.callPackage ./iusb-spoof.nix {};
+    u-root = final.callPackage ./u-root {};
+    pyroscope = final.callPackage ./pyroscope {};
+    s3-revproxy = final.callPackage ./s3-revproxy {};
+    git-gc-preserve = final.callPackage ./git-gc-preserve {};
   })
 ]

overlays/git-gc-preserve/default.nix

@@ -0,0 +1,9 @@
+{ writeShellApplication, git, nettools }:
+
+writeShellApplication {
+  name = "git-gc-preserve";
+
+  runtimeInputs = [ git nettools ];
+
+  text = (builtins.readFile ./script.sh);
+}

overlays/git-gc-preserve/script.sh

@@ -0,0 +1,132 @@
+#!/usr/bin/env bash
+
+set +o errexit
+# Copyright (C) 2022 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+usage() { # exit code
+  cat <<-EOF
+NAME
+    git-gc-preserve - Run git gc and preserve old packs to avoid races for JGit
+SYNOPSIS
+    git gc-preserve
+DESCRIPTION
+    Runs git gc and can preserve old packs to avoid races with concurrently
+    executed commands in JGit.
+    This command uses custom git config options to configure if preserved packs
+    from the last run of git gc should be pruned and if packs should be preserved.
+    This is similar to the implementation in JGit [1] which is used by
+    JGit to avoid errors [2] in such situations.
+    The command prevents concurrent runs of the command on the same repository
+    by acquiring an exclusive file lock on the file
+      "\$repopath/gc-preserve.pid"
+    If it cannot acquire the lock it fails immediately with exit code 3.
+    Failure Exit Codes
+        1: General failure
+        2: Couldn't determine repository path. If the current working directory
+           is outside of the working tree of the git repository use git option
+           --git-dir to pass the root path of the repository.
+           E.g.
+              $ git --git-dir ~/git/foo gc-preserve
+        3: Another process already runs $0 on the same repository
+    [1] https://git.eclipse.org/r/c/jgit/jgit/+/87969
+    [2] https://git.eclipse.org/r/c/jgit/jgit/+/122288
+CONFIGURATION
+    "pack.prunepreserved": if set to "true" preserved packs from the last gc run
+      are pruned before current packs are preserved.
+    "pack.preserveoldpacks": if set to "true" current packs will be hard linked
+      to objects/pack/preserved before git gc is executed. JGit will
+      fallback to the preserved packs in this directory in case it comes
+      across missing objects which might be caused by a concurrent run of
+      git gc.
+EOF
+  exit "$1"
+}
+# acquire file lock, unlock when the script exits
+lock() { # repo
+  readonly LOCKFILE="$1/gc-preserve.pid"
+  test -f "$LOCKFILE" || touch "$LOCKFILE"
+  exec 9> "$LOCKFILE"
+  if flock -nx 9; then
+    echo -n "$$ $USER@$(hostname)" >&9
+    trap unlock EXIT
+  else
+    echo "$0 is already running"
+    exit 3
+  fi
+}
+unlock() {
+  # only delete if the file descriptor 9 is open
+  if { : >&9 ; } &> /dev/null; then
+    rm -f "$LOCKFILE"
+  fi
+  # close the file handle to release file lock
+  exec 9>&-
+}
+# prune preserved packs if pack.prunepreserved == true
+prune_preserved() { # repo
+  configured=$(git --git-dir="$1" config --get pack.prunepreserved)
+  if [ "$configured" != "true" ]; then
+    return 0
+  fi
+  local preserved=$1/objects/pack/preserved
+  if [ -d "$preserved" ]; then
+    printf "Pruning old preserved packs: "
+    count=$(find "$preserved" -name "*.old-pack" | wc -l)
+    rm -rf "$preserved"
+    echo "$count, done."
+  fi
+}
+# preserve packs if pack.preserveoldpacks == true
+preserve_packs() { # repo
+  configured=$(git --git-dir="$1" config --get pack.preserveoldpacks)
+  if [ "$configured" != "true" ]; then
+    return 0
+  fi
+  local packdir=$1/objects/pack
+  pushd "$packdir" >/dev/null || exit 1
+  mkdir -p preserved
+  printf "Preserving packs: "
+  count=0
+  for file in pack-*{.pack,.idx} ; do
+    ln -f "$file" preserved/"$(get_preserved_packfile_name "$file")"
+    if [[ "$file" == pack-*.pack ]]; then
+      ((count++))
+    fi
+  done
+  echo "$count, done."
+  popd >/dev/null || exit 1
+}
+# pack-0...2.pack to pack-0...2.old-pack
+# pack-0...2.idx to pack-0...2.old-idx
+get_preserved_packfile_name() { # packfile > preserved_packfile
+  local old=${1/%\.pack/.old-pack}
+  old=${old/%\.idx/.old-idx}
+  echo "$old"
+}
+# main
+while [ $# -gt 0 ] ; do
+    case "$1" in
+        -u|-h)  usage 0 ;;
+    esac
+    shift
+done
+args=$(git rev-parse --sq-quote "$@")
+repopath=$(git rev-parse --git-dir)
+if [ -z "$repopath" ]; then
+  usage 2
+fi
+lock "$repopath"
+prune_preserved "$repopath"
+preserve_packs "$repopath"
+git gc ${args:+"$args"} || { EXIT_CODE="$?"; echo "git gc failed"; exit "$EXIT_CODE"; }

overlays/iusb-spoof.nix

@@ -9,7 +9,7 @@ rustPlatform.buildRustPackage rec {
 
   src = builtins.fetchGit {
     url = "https://git.lix.systems/the-distro/iusb-spoof/";
-    rev = "a1ec0384e724f609bb8e391512a8fa76d9894e55";
+    rev = "fafd47986239cc2f4dfbbae74b17555608806581";
   };
 
   cargoLock.lockFile = src + "/Cargo.lock";

overlays/pyroscope/default.nix

@@ -0,0 +1,42 @@
+{ lib
+, buildGoModule
+, fetchFromGitHub
+}:
+
+buildGoModule rec {
+  pname = "pyroscope";
+  version = "1.7.1";
+
+  src = fetchFromGitHub {
+    owner = "grafana";
+    repo = "pyroscope";
+    rev = "v${version}";
+    hash = "sha256-iMP67J0Q8Cgo52iImMzAM3PEkk6uLF7r6v9TyXZVaIE=";
+  };
+
+  env.GOWORK = "off";
+
+  vendorHash = "sha256-ggntpnU9s2rpkv6S0LnZNexrdkBsdsUrGPc93SVrK4M=";
+
+  subPackages = [ "cmd/profilecli" "cmd/pyroscope" ];
+
+  ldflags = [
+    "-extldflags"
+    "-static"
+    "-s"
+    "-w"
+    "-X=github.com/grafana/pyroscope/pkg/util/build.Branch=${src.rev}"
+    "-X=github.com/grafana/pyroscope/pkg/util/build.Version=${version}"
+    "-X=github.com/grafana/pyroscope/pkg/util/build.Revision=${src.rev}"
+    "-X=github.com/grafana/pyroscope/pkg/util/build.BuildDate=1970-01-01T00:00:00Z"
+  ];
+
+  meta = with lib; {
+    description = "Continuous profiling platform";
+    homepage = "https://github.com/grafana/pyroscope";
+    changelog = "https://github.com/grafana/pyroscope/blob/${src.rev}/CHANGELOG.md";
+    license = licenses.agpl3Only;
+    maintainers = with maintainers; [ raitobezarius ];
+    mainProgram = "pyroscope";
+  };
+}

overlays/s3-revproxy/default.nix

@@ -0,0 +1,39 @@
+# Originally written by Jade Lovelace for Lix.
+{ lib, buildGoModule, fetchFromGitHub }:
+buildGoModule rec {
+  pname = "s3-revproxy";
+  version = "4.15.0";
+
+  src = fetchFromGitHub {
+    owner = "oxyno-zeta";
+    repo = "s3-proxy";
+    rev = "v${version}";
+    hash = "sha256-q0cfAo8Uz7wtKljmSDaJ320bjg2yXydvvxubAsMKzbc=";
+  };
+
+  vendorHash = "sha256-dOwNQtTfOCQcjgNBV/FeWdwbW9xi1OK5YD7PBPPDKOQ=";
+
+  ldflags = [
+    "-X github.com/oxyno-zeta/s3-proxy/pkg/s3-proxy/version.Version=${version}"
+    "-X github.com/oxyno-zeta/s3-proxy/pkg/s3-proxy/version.Metadata="
+  ];
+
+  postPatch = ''
+    # Refer to the included templates in the package instead of cwd-relative
+    sed -i "s#Path = \"templates/#Path = \"$out/share/s3-revproxy/templates/#" pkg/s3-proxy/config/config.go
+  '';
+
+  postInstall = ''
+    mkdir -p $out/share/s3-revproxy
+    cp -r templates/ $out/share/s3-revproxy/templates
+  '';
+
+  meta = {
+    description = "S3 Reverse Proxy with GET, PUT and DELETE methods and authentication (OpenID Connect and Basic Auth)";
+    homepage = "https://oxyno-zeta.github.io/s3-proxy";
+    # hm, not having a maintainers entry is kind of inconvenient
+    maintainers = [ ];
+    licenses = lib.licenses.asl20;
+    mainProgram = "s3-proxy";
+  };
+}

overlays/u-root/default.nix

@@ -0,0 +1,20 @@
+{ buildGoModule, fetchFromGitHub }:
+
+buildGoModule rec {
+  pname = "u-root";
+  version = "0.14.0";
+
+  src = fetchFromGitHub {
+    owner = "u-root";
+    repo = "u-root";
+    rev = "v${version}";
+    hash = "sha256-8zA3pHf45MdUcq/MA/mf0KCTxB1viHieU/oigYwIPgo=";
+  };
+
+  patches = [
+    ./u-root-allow-https.patch
+  ];
+
+  vendorHash = null;
+  doCheck = false;
+}

overlays/u-root/u-root-allow-https.patch

@@ -0,0 +1,12 @@
+diff --git a/pkg/curl/schemes.go b/pkg/curl/schemes.go
+index 8bac3bc0..cd396cbc 100644
+--- a/pkg/curl/schemes.go
++++ b/pkg/curl/schemes.go
+@@ -81,6 +81,7 @@ var (
+ 	DefaultSchemes = Schemes{
+ 		"tftp": DefaultTFTPClient,
+ 		"http": DefaultHTTPClient,
++		"https": DefaultHTTPClient,
+ 		"file": &LocalFileClient{},
+ 	}
+ )

pkgs/forgejo/default.nix

@@ -1,6 +1,6 @@
-{ forgejo }:
+{ forgejo-lts }:
 
-forgejo.overrideAttrs (prev: {
+forgejo-lts.overrideAttrs (prev: {
   patches = [
     # Branch divergence calculations for a single branch may take 100-200ms on something as big
     # as nixpkgs. The branch view defaults to 20 branches for each page, taking roughtly 3s to

secrets.nix

@@ -4,14 +4,19 @@ let
   commonKeys = keys.users.delroth ++ keys.users.raito;
 
   secrets = with keys; {
-    hydra-s3-credentials = [ machines.bagel-box ];
-    hydra-signing-priv = [ machines.bagel-box ];
-    hydra-ssh-key-priv = [ machines.bagel-box ];
+    hydra-postgres-key = [ machines.build-coord ];
+    hydra-s3-credentials = [ machines.build-coord ];
+    hydra-signing-priv = [ machines.build-coord ];
+    hydra-ssh-key-priv = [ machines.build-coord ];
+
     netbox-environment = [ machines.meta01 ];
     mimir-environment = [ machines.meta01 ];
+    mimir-webhook-url = [ machines.meta01 ];
     grafana-oauth-secret = [ machines.meta01 ];
     loki-environment = [ machines.meta01 ];
     gerrit-prometheus-bearer-token = [ machines.gerrit01 machines.meta01 ];
+    pyroscope-secrets = [ machines.meta01 ];
+    tempo-environment = [ machines.meta01 ];
 
     buildbot-worker-password = [ machines.buildbot ];
     buildbot-oauth-secret = [ machines.buildbot ];
@@ -28,6 +33,14 @@ let
     metrics-push-password = builtins.attrValues machines;
 
     ows-deploy-key = [ machines.gerrit01 ];
+    s3-channel-staging-keys = [ machines.gerrit01 ];
+    s3-channel-keys = [ machines.gerrit01 ];
+
+    postgres-ca-priv = [ machines.bagel-box ];
+    postgres-tls-priv = [ machines.bagel-box ];
+
+    newsletter-secrets = [ machines.public01 ];
+    s3-revproxy-api-keys = [ machines.public01 ];
   };
 in
   builtins.listToAttrs (

secrets/buildbot-oauth-secret.age

@@ -1,20 +1,20 @@
 age-encryption.org/v1
--> ssh-ed25519 87T2Ig g15A5EWi9IhaxPFS6SD6YYm/aFnC0Dum7zK8/ZUtW0s
-791D6C8mAy2dhDAlqRQ+q41FlQTJX2WfZQPjuwetP2A
--> ssh-ed25519 K3b7BA cJY9qIFVmucmMJLTFffkRCNYeudZl+8Yrm5SkxQ4eSI
-97nXyKffZGoGJ6252UKUEJHiFgdk8XUkAAkXy2PLepM
--> ssh-ed25519 +qVung HMBSUjfmaFLVx64epj0djkqNMe3CdKN1fxAVuu+Dtmg
-AxT62n2p/pP9WZmmuHClSKKgXhr4FjEQpEs0HfdNGfw
+-> ssh-ed25519 87T2Ig tzPD1x6XKuDfgJ8jkQnwW/ALp2pkANCeNoO8xdUqq30
+QSsuO6Dwc8QJuY92gXRnWB5aJ2SU9X2uFh01GmLVaQE
+-> ssh-ed25519 K3b7BA 9G9Uw1xY8hq//xphNWrPn5y7vG2o8/kwkC8cJGuf/mI
+Ip0019OUaFq2ZDFI3i77hdsp9IqFV2qqYIB/TnDSXgo
+-> ssh-ed25519 +qVung dx22ef+x9X5mr73L8NUzxYQa640M2XViELjJcpgF3go
+CXyit7pk8SPNHBgULlMQUAasGAn4C36zcwOBDI46nU4
 -> ssh-rsa krWCLQ
-N0Duz2bONcCUZ76QhPsCJ4BHHWqzFdZLqFdl+6GeW+tgIp2Nb4la8eNfgzYGSwTy
-53bRePNMIBTkChXFYt/4fUdqaiiVYg25swMeVLQBJnjJkcAks0Gf44FXLIaoPr1M
-56rtixpSX31WDKwHbUF/40G6Xut8KNlI8BdwiOl9ibgnuEf4mYQbwFbRQbLMK5IK
-Rf/7SEmAqqfY/HG1RqqgCs4kEpvFTKqEEDpgjOoyS2tyKN2351jya91YzotLja4I
-sLoMg/G3UNtxfdaCgK7TP4IxV9blkVMDPAbyR622VbS0sEa7uJGzb86jDDsZXaKX
-9iWK9n4hMKZDv9gBbhTIWg
--> ssh-ed25519 /vwQcQ hMkCrUcLGxdZMYgi1D1Kr5qUdGNfza2UTvRJKiHObgM
-7Lz70zSMPk/tsU1CZGOk/BPA7NSSnSJgFbG5TjyOXvA
--> ssh-ed25519 0R97PA OQjDTknVmrYVclcqlT31YjZx+3a/0GxfjuVQFmPJ7UQ
-KMGTMfO/mO5EAYacyz1hmHnQgzunRqkDeglhbGVNWe4
---- ScDZvSiVSjNXm8TSoLSAM+KpcFORnCXiemYbCBcz2jQ
-™ŸÄhÜ}E¹ÊœËíUÌùᢌƒÿ…<C3BF>é™k¢ág[<5B>ñCƒ"<22>–NÛj•u5«<0C>ÄCXÕöÈGt¡TOmñ
+NlGh0hM10NOuek7MbrFo0iul0kQQtDFmZIhgpyqaATMdCDRBXJOyhASHU5N0zDDJ
+MLaJUV0l2o1ghBF9RhSKdoUPVEn8Cce/nfQepYzMlfc4UG3qWXwabwR6EtqqCZCJ
+jAEWZ8taTKDmzoXwuygCW+bRBuoMMrcfzu7V90N+mQpZWtOScatb6E7d5VRqjlar
+st1ZQu5ccghufyQSUmOC7GpojOyutX5EvbMGn84X4ouZRHRX/8fTgaqicV+aeAIb
+QyXisOrO6C+Jle5qfxzMSe8c/TCyF2574kD6F1BQ9Kpkinn8v7OWcIXtkNmZ5hzK
+vs0Bej8yZVsoBkj1vWAM0A
+-> ssh-ed25519 /vwQcQ n+hr1cV1zRs1S86YnA+0oRB8SCaPKtkoMNe15ZsVVwM
+fdFtUqno07ik6FpW5zMImIjd8wM8dMgwU+RqjeT2PiI
+-> ssh-ed25519 0R97PA ddPILw57gkuKvAqlmpa+MnV/LSEdyQzQaAarCUqQ1xE
+ozK5a6uXZDc17OrX0OZun9hmZwP3H3rYQiNuKnukqsg
+--- f7yGgKQpCPj64Ps0HfMcToYircGH5SPqMzVZrUMB8ZI
+føv[iY\ÅšMP,¯Ùh°Èxb—Ðÿ«J<C2AB>*ºË"”+¬ÒA0T˜?KmˆPÈË2¹'2±‚µ³Ø=¯êÚÏŸj”

secrets/buildbot-worker-password.age

@@ -1,20 +1,20 @@
 age-encryption.org/v1
--> ssh-ed25519 87T2Ig df+IMqWM/HNjaY74zibFQIdUdC3K7uQlm3U9R9NUtFY
-hPSbCuWvqy/7FEj7YScYztyt5GVx4Y7tgGuKKkSKoRg
--> ssh-ed25519 K3b7BA xN8wzUKHqjOb/tqA+EI+0H0MSQRihRfydchwVqYWAVU
-maLMpZe8orvTT6Av+YkhT8FcG4dc7bzDgOW339nSw1g
--> ssh-ed25519 +qVung oM1uphTbjI54t4U9jNd1zORqpjBG17MwDf2eNDmOlkg
-oUHVuQt2SHIwtV82pgnKJ7g2jcVBAHWOzPK46otoh34
+-> ssh-ed25519 87T2Ig p8lEB5da4fIfLH/HKBsghzq5mvQLB69UB4+uAi3DGCw
+NeZ3jPTUKa7MiqjrFPrYuP4VneytQPdBNqf+omPZJYM
+-> ssh-ed25519 K3b7BA uP2K1hU7uLmiHXmmoUdsB7CHQq61ZkEAjG/aK863RDw
+0chTczEMXASdYiwqNxDQ+vMXXhjOf64oIQ2ULZmQI8Y
+-> ssh-ed25519 +qVung jUgEqz3+ypL7mwJ1R7lfeOMhkon/aRrNSJUJT3X7vmU
+pgOiwrp9JiA20yw9bsxi8eiQ9/23CYXKRBGF1pea9eI
 -> ssh-rsa krWCLQ
-eYspf5hUKdFQl1RxPaNTj0viAPd+kzp8Xbwn+q6fSITMacmyTY5J8FckLx2YXDxy
-Qm/OsEK0ZOvxnHMrL0oAJjKSy/MamE+9heT3QO+LUN30QxbOIOqHMrl3waadWZdx
-ZGOWK+r+dKGYNsxFv+t1Y/4DBKKzlXFWhJ0aL7nMOqq9+Ca+UZuE41j7eWGGPPLy
-fuW/iOVVxQ+EEeCDpatQSrFPKaeWCCVP9oIDFtE4dsKxubMa4EpUoag0UvEIW182
-UGS8BvMqYgx+obqJDkhXXBK9apmJS2ojcfdtCbNOCV9Ett72Nm/iY5NjLprFMLde
-8wWGA6s3hBOP39lq0eiSxw
--> ssh-ed25519 /vwQcQ 3zLcLDaDVhIn2knezexYM5Fqu/O9wwORnJIhsXHqgj0
-HchGikQMgkDj0qQgtDdsdKokV+nMjdv6t0uVISeU7Q8
--> ssh-ed25519 0R97PA 6lm6B6B3dzSdhdcf5rjyTu+7cCtWRxVpWeapJX3nbQo
-x/w4dEfFyxPi4lbNEqgjEblPVfQyj+q1JjeQHiVFhDw
---- oo5BK1pG+43amUg803Uv511RNtdQ/PDwlXUrV/AbOAA
-…ÙUqÆçïµ[f7ƒêŒë¼¨FìˆY<13>™Ùm¶ØLS?Úℶ‡÷ƒöæ<Kø©F¤z¥V^³U¨N»¯ôƒ)zÔ<7A>¥ž@<40>SÀF€Y‡ËG2^žƒ˜à„»N|
+snCHrLHzkjimwIxKO90IjnHwOArlozO9kd/aCdZZnYNgh/QG3rUSceSn9yTHbtMV
+izv0SU51LrRU+JyE+a524AxKhyPBvGDig20j7hMy5fVxZqeunztqtlha5gaYYaQg
+Tbfs9tDP+pCIgzMVNqYf6EJ4MK7qjNf9DE5I490Eta5YZxAi/3To3BmZmIYtCz6l
+1kNRiSmWCbZqE25keFgPCgRMFXAFK9W6NmL+HamqCUhjPoJg/Gd4sf39EONT0PYg
+7BpCOAnwwfECHPxpM3qv0h2kJXTb4DZ715cFReSVyQe5fvKv8hoWhl/S+++pEYT8
+u/LKBx/o7e3Kd7cm2RGnBw
+-> ssh-ed25519 /vwQcQ 4+IQPRsMMHmuSGL7T7IbRkTTuL+TTqgdQp5FSbyt8Dw
+KOI0LKQ0oA5XtxaW7wftlEJB0BGVnx41HUJMG92SRUA
+-> ssh-ed25519 0R97PA l1aWUEv8nLEtYnpY1gjTJqk5UYm51NDqOjYmL83rZ10
+B7qDZwCpolkIajqCXeOepwmF6ciJfKvr+AN7VouMUvA
+--- lz/IMMPxBpD3Bzuv9Wl23+swBQHlblhlAO/ZXAgN0hU
+µoÍüÌ<EFBFBD>²-‚Īr °eó|Í?ït
èìÎZ<C38E>¬sÒì!ŸƒÁ<>@Ï'–ìèz6UöÎg
JøÑOµ–s13<31>š‹8<î’%-·Ô‡Eÿ}–Šdm9¿å¢Óoæ

secrets/grafana-oauth-secret.age

@@ -1,20 +1,20 @@
 age-encryption.org/v1
--> ssh-ed25519 j2r2qQ JSveX4zYEjb4jJH4eg4oXA6r3oc0jBx8NgjhN9JrjlQ
-1ZIr/XFClbwJHn0ppJnolpb4QlgZOA8JX5OjjY4x6pU
--> ssh-ed25519 K3b7BA sXUjuZFK0PL/KndxRCJCM5Kg8OmVseRZNWG8mL1alRc
-U9MMgDtqtmsS1W5i04Pa/b4JBTSjK6FffZxgYI3phtg
--> ssh-ed25519 +qVung FNSElbiw0frYcsO0xoyPQgRGqAe/aVX21dTB6yk+GQg
-zHT/xU+yfXYSBO2HLwoHrGf5ns6BDVb8MlhVVQCBlOc
+-> ssh-ed25519 j2r2qQ vwcaLpvGJ9swXnV8idDwi9jdRPSj38As9p2QFkIJ1Xc
+FLnZeblHDQQcWjFm1iaghbvuFgOG3miwtkRE5sz1+X0
+-> ssh-ed25519 K3b7BA 9VRe2rBwg3G9lxxfxL/yLob2NZmLJTBMxzx0Ew8VwmY
+/I2W80UykNvll5o98OPeMpIsddOel9B7uQlio0X3gcs
+-> ssh-ed25519 +qVung VsqKzMD85aps4PIx2zqae2Dj7YWibiaKYb5z7ws8ggM
+Y9dRd/hOz8h4avlutBQ1YZgHIAf/AuTr5WaByKlFbLE
 -> ssh-rsa krWCLQ
-ye0mLiYeyvlp4EZX7mZ3F7B9V9JSeoiCodzccS+5qIEd6gr+RTHSnKYqwf/nwf8F
-qKLwbxWjpmkIzBWeswy8AJ8159aucGEmB+3/tTSwd+QlRkru4Z/7jtfU64KQttgt
-vaRfc9J/85AJJ2V6Sw/xG8SgxyLBbp/XIN2+tmb0g3kAWiuLcrLk3H/MsfmxDVXg
-RQjugP5K2+fEZc77dHQTrMI58K9TrSw1zYA1ee8J/fl9IJ7J77qi5UgizY+YfX8T
-SmR9DeYUe+hKgCB2k/KgAxp4WOQNgUOFBTsE5FW+kQQpfGx5aqR6vCYU+CPsA3Zb
-FwV0l+g4FUVy+xAtqaGSAQ
--> ssh-ed25519 /vwQcQ fbnK1jYiUwUsgD8sSTboJCBfcuwJXKNCaJaWYuIfmVk
-Uj2+uBABMTxq1MBsiHXgkdFMOpIN7gfxoJVKOQff1Pw
--> ssh-ed25519 0R97PA yYOb6AYAFWvm7W2KYT5v9zznkF4Di/vatH48Xgx0x2E
-yUm+MKj9496BkdX2FpLyhML7budUyqT1hL9hpghxSnI
---- ogCPBrmdbeDorj3t5BL05ge6VngXBpUEDW4qaaKIa0U
-%¨šÚlD]Ϫ?©ßŠÑ(ÿ†E/Wu穉T¶îç[}ž$ÁÍS„Šˆ^[:¸]
he0XUœp¸äq<C3A4>`0A
+gjyaUFrIIbZnFTGVw4XEZzkTIP/+qXV6/q0W8Wb4EtqQXDRISFT+bwxQU/S2p5hf
+7+JGcn4BZg6puOJ5BBABWtpn6gcX5OFfga5azIdioF/R19XByT+0SK5njw8g1VPS
+R7o8kQt2yvKWayoq9Cis5XRg+4KANkwOQaNTO8AdiCwgq9nc0Cd9avk8QhaFoR74
+D5cf8jPsufp744rQqwhWDoG533LS1WUUuYZqRmtp2Vz+r583RhSscaNyA7ddr7o6
+e9ZQJyL5bKiN8qe3Xm76lLypf/wg7+aGn8HHnO6GA65g+VYfjLMODEqCN/+uDJtB
+g8v2wzKIGYlZiV1hEjH8nw
+-> ssh-ed25519 /vwQcQ 4pU5JGK5vpZbFgq01a9YY8VmSJvPSHPSZD50TLJwKHc
+L46UA/p+bNSR8cLmL8G7VpmAcZ+sy5AROc4yj2ABOWg
+-> ssh-ed25519 0R97PA Tk00kYLhsEy1HJcmKLgaLWTdNP8XV/cdKHMLzyK6glk
+kwyQZr/h6MutROJmjVfPWGcf9xN5Uc5w5mVyuKcK64g
+--- E0vVtBqbjNkZY0/1dFJ53uVAR7IGPO+OMmXkpJcKmlw
+{ÐQê%è‹õY•B,isr¥1‘<31>|¼yLÕ'7?¶iŠM…¶MU]×ê/d2¸I1u¶2hZjHåh&¥

secrets/hydra-signing-priv.age

@@ -1,20 +1,20 @@
 age-encryption.org/v1
--> ssh-ed25519 +HUDfA NMyTM3c++HKU2klLjAbUUFS81k21LUwEoqR1OUBuLjU
-OrKxpksxoay93URtmN9HhnK43QrM/Gs0qRuENZvHWJI
--> ssh-ed25519 K3b7BA LloEGN8cbVvGraHs5cPIZRJJyTPFrmmeGwZyqov9m2U
-XPvNpQT3aFVoidOhmePGgiTyytIWtd4rs59Qq9xl/I4
--> ssh-ed25519 +qVung 3hat0gKIl1WjXnkP6p+/8RyTxZkaVnLgV9B8plICPlY
-jmRKWCUCDpDExmq4SEq8WpqQheBSRD4uqrTgxy2u6PM
+-> ssh-ed25519 +uvEmw TNYFQxSUv5yMmlTWoIxCOlv6UR+RA50cb5aJbo0yEE0
+Yw3sTPqYf7A33RI87CqoPWe2gh0FuvdBGGKqHV55Atc
+-> ssh-ed25519 K3b7BA vMlHenY6jSIfnxQD6xh09cwwV+YVBkLuSMHcyKD+dCk
+heXkAEqRawBlHqcr6ldmhWmk7qPtGLMDFC3QT79vdMM
+-> ssh-ed25519 +qVung fgimLW5X0z4Eh2u3fIr5bgR5/c1SKam9CKW/2mqtTik
+8VKJJr+FRE0j5YvjfdMXugNA4UwUebKrkeAe+9LYBnQ
 -> ssh-rsa krWCLQ
-QPOl96dmoxY5YtMmL68+6MQpGwZc68ajaRkcEKmYYu4/XB+mffRKsNtyiKJQwEi7
-szvAced8C4RMNrCf3xyF77Sm1UV8YCyaHyplb2/yjv5YDvCDwTp2GnadDoAaLrXU
-jf6ocI8409XWQHEEEofHZRjmfmIBUx1lTwbGFMt48V7MZdadFjXmSmUMvxsu/Rj5
-NLjoPNRBzqPIw6U7nTSmkG2HOeHlA9Z5a33MsXYs8NPH22Spjjy+VvxrLv8VAjnf
-7kGjviW4ZcdEQ7Aox+9V+6qArrIy7lJ9lOIZA2LueCtKhQAmKnInFxRyyN0Nk4ls
-tjlBFJQEG2v14iaHENwRAg
--> ssh-ed25519 /vwQcQ o7qseMFb4ViV7ylSl2ug7xFZn7GZGqCapWRCq2vyVVs
-hKqzk9BcK5l0VhLfPONKKv6SRnDCw2n+RoaeQbOnT8Y
--> ssh-ed25519 0R97PA DTEowwoCXTMGxfQIXOnwn5fjlih0UmQJCKs2II4gCVY
-8BpVhUV8qg3zcCJe7OwHoJrfYIpBtOquqhFyfZx7mRQ
---- r2NmEyV0/Goas5lXMHeFoafcrbMHvs4ob0Zg4cVil3w
-=‚TÏ`Ô}%xÖ¸œêFtÆêlãµ)•ò1Ð]Œ<DvwFøè:Qf@nÇÀU=~FžË-Ò!©ò÷þÌ«i6wXtü'k®ç`)A›äûÌtI1Tî£U+Ú~×’aÒÉПONƒÄèLþI£p@œT)¤'_ÞoDö0»,JfGVó­k’/
+sa3fA4GglovY8H6jimpTvQPW/axun8WADPlIXzpX/Zeshkzem+pQoQqptzDlnmH8
+8AngqXgFYrmHgNNAylavgcrxbjNrtlJU24ldF1YIubz7VsU1678F27LCd9B0c2dn
+X+0CccH19lM8Q+zVI2Wrq9R83MEP/5uOOc+eXXnvNSGqfKgZ2OplG/HUllFS13j6
+uiQy5zwJJKkII7KUThcGteux7NONoLeUqRE8CW2uSeY9fXBWKgxeENKgiT7PEAAo
+nvwWa+GatEYf6eUz8Lph8lETorgP+7JS2VQRAkmhDbjQLTYzfFmiJGE/mzyobslf
+ZEq6Oj5UNgnzdWmK5ZYKPg
+-> ssh-ed25519 /vwQcQ 9EG/cydlzlLd6cFed7DzmwzubzJUXvD9mX3WKDyFD1s
+3Emj+tVZmnsC/YZdChvyaxeObbBsri347vZl0ff9kH4
+-> ssh-ed25519 0R97PA kcIYyWKxpJmjcrel+YodZQiR2zGPqfjzMyJXsz2XOzM
+SUlgGGs2BVRzTHT/ULNo1AiN5SY1BETFtJRY6LDr4JI
+--- l87sO6IuwSeCeQ8ktvYFI0xr4Utcl8KfpAV7WePc1y4
+÷ÚÖ
~÷J3¦Œ§Í‘1íÇè²ù<º?%×ý<0E>›Á‡ÉÒ\—\Ï7\»Ú¨åU-&W'd”{իɼ½u	"Û#Ž}¯õ…x–ìšz¥®Nj„!éfUqDG‘<é<7F>Ñca<63>‚'´+	¸Ï±]Pó»ö€D
Õ¸’´þ<C2B4>£¼­ŽÿÂçÌ\¦<>

secrets/loki-environment.age

@@ -1,21 +1,20 @@
 age-encryption.org/v1
--> ssh-ed25519 j2r2qQ 6qyr94uky6B36UOY0jd5NXgF2rJ3RWBUzZ32c5iOTmY
-fjlI3fjYjwyNQBs4K4pq/5c7oBkf5XUXoGlBOBpmPu4
--> ssh-ed25519 K3b7BA N9VYT/ZslG07KldzO8sPE5TiYYwxJqpYU87ED4PuBXw
-P1s9L57prPqM4fjcYHv+g0rgP/NvFr13CgCxthVHZ4c
--> ssh-ed25519 +qVung Ry8uUFsmYmP+Urw46lhAsCc3S+QiWu1mn8J3rIy+KFQ
-iB7xAfdpHwOzAnLvosJb+F50QKsOYWr7CHC3srsS6ME
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
 -> ssh-rsa krWCLQ
-w0xIVFtUghdAO7SxZD10rBMtdQESEvYUEKxnWzLh0cjcRhaVT/BXSZQsKV2Rupoo
-nDL5uy0k+tPXm0HroZ6VkZ0fH/lOpeUR69ZvJmClKql3Fnf1385+5BvT719cbbaq
-yll49gx0+ms/oB9jS3SPwbOg+UJgnkZCeu9138h3MG7yWNtVuA9l5hsJioVvOVlS
-Z5EXbjdQR9xYjSwR+b8MYZ97ej5fXpuULEopbx2wXt84u1e67vTETqflitR7lrzy
-A6F65g35aagPJZGHzfrKVToy3pfXm9ky/30DolWLD0DpG7G6o/8afy8O4yBAGlv3
-ZLTaUbrdILSz2ff1Njx4Nw
--> ssh-ed25519 /vwQcQ YqqmX/f4whOk97kCgSPo6oj/274eYlBWtS+OahAAQ34
-hoCbhupzSTx+wNIorzYGHyGvU/L8unKEyD7Bqq23YP0
--> ssh-ed25519 0R97PA 17SDtfT9GzAsIsQB24AmYXpW8v4+LEakup+tdFroHTk
-HIvBhAGA2GMVWFBP3OTFEn+XpPFBJDOJDK3SQ94mNKM
---- CD1QrxYGAhhy+l7U5kOXn1shCwz8pYJNuGRugPxmzJw
-ñY¾Æ‹N	Ï<>x
™êÿrR^z[¤ã¸è…•ªa”z
-óæÔÉ¿Ïžu0c¯c;y<>Ÿ¢›&{ñèxA]‚þ†¨Q¨¼_:̱ í€öUoiDl (‹Å
ëwÝKi,j.oFyÌ°$}•Y§@1”È™„Y£²è¶u Ò*¡ÏþÅ<C3BE>¥™0…
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>

secrets/mimir-webhook-url.age

@@ -0,0 +1,20 @@
+age-encryption.org/v1
+-> ssh-ed25519 j2r2qQ uYhcMpOER5j/SWUX1mNvkOU9Rumr0CgVBuGv9EHGpFY
+6kAgrwjgB7C1cMd410EpUegcxxGRcNOwCMJPXppepvE
+-> ssh-ed25519 K3b7BA 57GDNt5nwxgzCV5bnMPEPUeyZNG1U+zajCIjeoHjLAE
+rFCbfodjXHZ0aVLtW6xtoh6e/VH/HwFdFzjnQ2QEEXQ
+-> ssh-ed25519 +qVung DnLKAJPnUDpZ2+wXDZWpxwZkvv8oDyu3xxObTMT9W1I
+vh59DYoQLpiro5eBjwgNH2YHRsGY/i6TB7zPfQicOEU
+-> ssh-rsa krWCLQ
+ekvGooB5sCmAniHU7hlk+iCkYMQ7Rw2SJx8tp4FnpfAWJbRMH8CpTFYFiDvlHfFy
+Ce1OpkNkkipzBge0OCrfn6Y5iVz2CZHYHf8Ul5ueHwmb5fS7seT3yMoWhhSw/zE/
+G3snrBORT9S9+KTRnVnKiy+O3CaMZY+q41RR35Fs3mmVc/of2ILc/Jj3a3t+uBTX
+axkOMU6z6R6i3Ps5SbwJTaB9q2kMPvZFOO9Nmku1wohjetz64wvm+fDx0XVRPe4A
+jDQRPKAMIZK68SYHk/9azmlBtJSJnvxcxyj3IaU9MBskUCldWi8CQ9jQ+1XAIuHX
+0Etcsx7MhzBpuhx2xZ+dyg
+-> ssh-ed25519 /vwQcQ uW41w2RAtfMaOm1wJktMcbVporqKgdGA5SY03OcPmlM
+WgL8DWPU735Ysowq0HtvbrT6Tc3XEpwws3AycqpBgtM
+-> ssh-ed25519 0R97PA 59AFQx8ngDwQUdmfOeOFUARQQqaAdLA5WH67Wsld4yM
+o6jSWtlidZssWsJsI8xAaASi8p1sirLJFJwizzPXIBM
+--- scUnldbU89ICZYlniDbGEqeUF7QUoO1kcZLl8abyttk
+öR{p@IµþOlKKõŒ§!
<01>œWÎÅœ[R<-A‚ÐbÔ<¯·÷0õÐu¹øµñU’gBÏF~µ«=ÊõeQò}î4Ø:ô²¢5ƯŠtaØ™û”<C3BB>æ·<C3A6>§°x±M
ë?Ew0<77>8.

secrets/newsletter-secrets.age

@@ -0,0 +1,21 @@
+age-encryption.org/v1
+-> ssh-ed25519 CyxfgQ fWH/o2+Uf0i/JFIuVjCnkhDIfYndtL8EeDcxSxhKVH8
+ShSPmdwnxzDuUe/kCx8e61JJAoHMwguNydn+5OIGuAg
+-> ssh-ed25519 K3b7BA p+wXAGvPqTX63dlZNCTIq3F4QFMWEJH1R+Ex4SJ5UTk
+1sFqFqnUM8YvZy7BEBArg3eLxCCsLXq2jNI7XLKq/Ww
+-> ssh-ed25519 +qVung rcpgzVQ1PmoNF2i0K0nAknzZwPXICBggzqhIZwO+8xY
+9rjsTwLm5u1GOJmnJYriXXAY1unG7y+WJ4G2ltxX34U
+-> ssh-rsa krWCLQ
+seXsQjs62kxn/agyKda2l19PI4xzDl1gM7rEnaEBV8UNLOPNxh41HTnP2etgDXSc
+4eyS3ntHXIOHmN4+JBn+Q/wuhzMGQmAcoFWbjqVVPOrpPYjgCG7q/iUD8kULxLB9
+UpF0gLsg1TnvrkTwlpxr8rP/PM+ZgyQAA84S96j9TW0coyTUoH/ZX1wWGtS4aalm
+aTrOMZGScZu7onTg+tYvR+aBKlFL28h08I5nqbA39srnCNuU68+OUhLgLUfiTscl
+umwNh/C4BP2Tmc6gxQiY8o3tGqGBssGH5+WqKzbK151vJjq80RKAS1HCaSSfmxkP
+vWkXWN3NQkJyqCBpuPYilg
+-> ssh-ed25519 /vwQcQ eUH0B+cCoUubIKbG+bA25kRj0TnZabB6t8jVK04NrFs
+ovkI0C4W5CJXMZIZdpaTtQNc+TGkQ3Yq87Dei3BMUsA
+-> ssh-ed25519 0R97PA u/I45pxH3Bnja/Jw/6IukINRuC0e1IKu8UVygVgIomc
+xyHuiHf1/nJirnhXbGHJnextGQa95tDo/RPRRnDCkIg
+--- LGqO4Bsa8bofD1W5YrQp75SlGLNg1XaFZ0rPUuvLPTo
+Êçã ‹ÜmlW£{@I3…*¹ŒÇ™@ÞªL7Wª
¤ÝŒY
+n
õö~Tb\V‘•ÜvPÙpPô
ïoÌS"ôm/Ûµ/bÝp’Äžêq¸£¦šeDj6–ÆþTì)

secrets/tempo-environment.age

@@ -0,0 +1,20 @@
+age-encryption.org/v1
+-> ssh-ed25519 j2r2qQ kbi4mciOrjd7/X86xfmkDaMZhvZakoSJ6qjqLF3ljkE
+Q2BsgMLJ8AmjhnggRi+wkICj18NCA2HW1t8clemReUw
+-> ssh-ed25519 K3b7BA wNGmX9S9bJgd2JDte9QoNDfyycgmq4JMu2bc5nyYYik
+uUiutxAI3nI0M51W97aPRVE/l4dV2PEjph8eWOMLHIE
+-> ssh-ed25519 +qVung raYJ5vwMP9JopSdfa+ofkLY/gc0zcW4wTNBFTca+MXw
+sa/rWGSYrI4y6rn4JSboldWKUGvx6HbtsYo78AFOkBo
+-> ssh-rsa krWCLQ
+FLq8NwkiGw2gXptVVY393f0p9hFom57xHWPxtAlzOcRT8gvWu/uwgV+0raOcOcJa
+xxr5Sib+2D3UnUhprVPmH5Os9bI2seFAiej1MVVWLqvMtQHLFwnrzZTyZpxsXpQq
+5qQhNEADuQc4uD/ELVjGHKt6nF1Cl/GbgNLIOF/ITZ0pm1O1MjtT6MYJhQJhc6sb
+sno/wQyTXjj7rC06nyLX/rgOWrJSOeaz9eVp0A8k8/I0TXu/vRCW9gqWtv2m8sbh
+1uUHIm0l8f3z+zrL6OlZnpMFw4jpiiGoCYKPzD17I0onDYIjtdVS5iO9BsckxV/a
+wQWbyONUwbGCfeNSVAzZbg
+-> ssh-ed25519 /vwQcQ jwf7fwy4wKz7q761DNu8SyFHGgFlwq4P/Pn44Nido3E
+1q/jvt/vtD4ziY3eCDqk1XwMPpNUd80POTV2VVsumCE
+-> ssh-ed25519 0R97PA XeuziQ+wsoh0KSHXk5Qkl1kQOsAu1Ax1zTg13+XWd3M
+B1KHKm3tx/EsnE6hY+w7ya1ilhYiUs9AbwARHNkJi90
+--- JgQA6gCYZu8xcbXEl9VypccEIBO6uAJIdhBefr4doRQ
+V3ZðõÚ<EFBFBD>ç-·Ý.ê«sòÀ³3 ÎiS‰a5#¿Ð{åÔÈ®Dý˜YêNèãëù«ýoL+ÔÝ#–M<sws P»¢+í¢Ó‰ïBDoÊξÆÏuFí”Ç^Â¥•<C2A5>—ÝG@ÍM×ÛãÐØìq¦ºG^Qb s<
;ÂÒnC+ÖÊxª_­Úì]S<16>Ð

services/baremetal-builder/assignments.nix

@@ -20,7 +20,7 @@ in builtins.listToAttrs (
   }; }
   ++
   # These are not currently used for hydra
-  genBuilders { offset = 10; count = 2; f = name: {
+  genBuilders { offset = 10; count = 1; f = name: {
     cores = 8;
     max-jobs = 8;
     supported-features = [ "kvm" "nixos-test" "big-parallel" ];

services/baremetal-builder/default.nix

@@ -1,8 +1,10 @@
-{ pkgs, lib, config, extendModules, ... }:
+{ pkgs, lib, config, ... }:
 let
   cfg = config.bagel.baremetal.builders;
 in
 {
+  imports = [ ./netboot.nix ];
+
   options = {
 
     bagel.baremetal.builders = {
@@ -55,31 +57,36 @@ in
 
     boot.initrd.services.lvm.enable = true;
 
-    fileSystems."/" = {
-      device = "/dev/disk/by-label/root";
-      fsType = "xfs";
-    };
-
-    fileSystems."/mnt" = {
-      device = "/dev/disk/by-label/hydra";
-      fsType = "xfs";
-      options = ["logbsize=256k"];
-    };
     boot.kernel.sysctl."fs.xfs.xfssyncd_centisecs" = "12000";
+    fileSystems = lib.mkMerge [
+      (lib.mkIf (!cfg.netboot) {
+        "/" = {
+          device = "/dev/disk/by-label/root";
+          fsType = "xfs";
+        };
 
-    # We want the tmp filesystem on the same filesystem as the hydra store, so that builds can use reflinks
-    fileSystems."/tmp" = {
-      device = "/mnt/tmp";
-      options = [ "bind" ];
-    };
+        "/boot" = {
+          device = "/dev/disk/by-label/BOOT";
+          fsType = "vfat";
+          options = [ "fmask=0022" "dmask=0022" ];
+        };
+      })
+      {
+        "/mnt" = {
+          device = "/dev/disk/by-label/hydra";
+          fsType = "xfs";
+          options = ["logbsize=256k"];
+        };
 
-    fileSystems."/boot" = {
-      device = "/dev/disk/by-label/BOOT";
-      fsType = "vfat";
-      options = [ "fmask=0022" "dmask=0022" ];
-    };
+        # We want the tmp filesystem on the same filesystem as the hydra store, so that builds can use reflinks
+        "/tmp" = {
+          device = "/mnt/tmp";
+          options = [ "bind" ];
+        };
+      }
+    ];
 
-    swapDevices = [
+    swapDevices = lib.optionals (!cfg.netboot) [
       {
         device = "/swapfile";
         size = 50 * 1024;  # 50GiB
@@ -92,8 +99,8 @@ in
     };
 
     boot.kernelParams = [
-      "console=ttyS0,115200"
       "console=tty1"
+      "console=ttyS0,115200"
     ];
 
     networking.useNetworkd = true;
@@ -157,8 +164,8 @@ in
       script = ''
         while : ; do
           percent_filled=$(($(stat -f --format="100-(100*%a/%b)" /mnt)))
-          if [ "$percent_filled" -gt "54" ]; then
-            ${config.nix.package.out}/bin/nix-store --gc --max-freed 50G --store /mnt
+          if [ "$percent_filled" -gt "85" ]; then
+            ${config.nix.package.out}/bin/nix-store --gc --max-freed 80G --store /mnt
           else
             break
           fi
@@ -177,39 +184,6 @@ in
 
     environment.systemPackages = [ pkgs.ipmitool ];
 
-    system.build = lib.mkIf cfg.netboot {
-      netbootVariant = extendModules {
-        modules = [
-          (
-            { modulesPath, ... }:
-
-            {
-              imports = [ (modulesPath + "/installer/netboot/netboot.nix") ];
-            }
-          )
-        ];
-      };
-      netbootDir = let
-        kernelTarget = pkgs.stdenv.hostPlatform.linux-kernel.target;
-        build = config.system.build.netbootVariant.config.system.build;
-      in
-        pkgs.symlinkJoin {
-          name = "netboot";
-          paths = [
-            build.netbootRamdisk
-            build.kernel
-            build.netbootIpxeScript
-          ];
-          postBuild = ''
-            mkdir -p $out/nix-support
-            echo "file ${kernelTarget} ${build.kernel}/${kernelTarget}" >> $out/nix-support/hydra-build-products
-            echo "file initrd ${build.netbootRamdisk}/initrd" >> $out/nix-support/hydra-build-products
-            echo "file ipxe ${build.netbootIpxeScript}/netboot.ipxe" >> $out/nix-support/hydra-build-products
-          '';
-          preferLocalBuild = true;
-        };
-    };
-
     system.stateVersion = "24.05";
   };
 }

services/baremetal-builder/netboot.nix

@@ -0,0 +1,169 @@
+{ modulesPath, pkgs, lib, config, extendModules, ... }@node:
+let
+  cfg = config.bagel.baremetal.builders;
+in
+{
+  config = lib.mkIf (cfg.enable && cfg.netboot) {
+    systemd.services.sshd.after = [ "provision-ssh-hostkey.service" ];
+    systemd.services.provision-ssh-hostkey = {
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = true;
+      };
+      script = ''
+        mkdir -p /etc/ssh
+        umask 0077
+        until ${pkgs.iputils}/bin/ping -c 1 vpn-gw.wob01.infra.forkos.org; do sleep 1; done
+        ${pkgs.curl}/bin/curl --local-port 25-1024 https://vpn-gw.wob01.infra.forkos.org/${config.networking.hostName}/ssh_host_ed25519_key > /etc/ssh/ssh_host_ed25519_key
+        # Run the activation script again to trigger agenix decryption
+        /run/current-system/activate
+      '';
+    };
+
+    system.build = {
+
+      # Build a kernel and initramfs which will download the IPXE script from hydra using
+      # u-root pxeboot tool and kexec into the final netbooted system.
+      notipxe = import (modulesPath + "/..") {
+        system = "x86_64-linux";
+        configuration = 
+          { pkgs, config, ... }:
+
+          {
+            system.stateVersion = "24.11";
+            boot.initrd.availableKernelModules = [ "ahci" "ehci_pci" "usb_storage" "usbhid" "sd_mod" "igb" "bonding" ];
+            boot.kernelParams = [ "console=ttyS0,115200" "panic=1" "boot.panic_on_fail" ];
+            #boot.initrd.systemd.emergencyAccess = true;
+            networking.hostName = "${node.config.networking.hostName}-boot";
+            nixpkgs.overlays = import ../../overlays;
+            boot.loader.grub.enable = false;
+            fileSystems."/".device = "bogus"; # this config will never be booted
+            boot.initrd.systemd.enable = true;
+            boot.initrd.systemd.network = {
+              enable = true;
+              networks = node.config.systemd.network.networks;
+              netdevs = node.config.systemd.network.netdevs;
+            };
+            boot.initrd.systemd.storePaths = [
+              "${pkgs.u-root}/bin/pxeboot"
+              "${pkgs.iputils}/bin/ping"
+            ];
+            boot.initrd.systemd.services.kexec = {
+              serviceConfig.Restart = "on-failure";
+              serviceConfig.Type = "oneshot";
+              wantedBy = [ "initrd-root-fs.target" ];
+              before = [ "sysroot.mount" ];
+              script = ''
+                ln -sf /dev/console /dev/tty
+                until ${pkgs.iputils}/bin/ping -c 1 hydra.forkos.org; do sleep 1; done
+                ${pkgs.u-root}/bin/pxeboot -v -ipv4=false -file https://hydra.forkos.org/job/infra/main/${node.config.networking.hostName}/latest/download-by-type/file/ipxe 
+              '';
+            };
+            boot.initrd.systemd.contents."/etc/ssl/certs/ca-certificates.crt".source = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
+            boot.initrd.services.resolved.enable = false;
+            boot.initrd.systemd.contents."/etc/resolv.conf".text = ''
+              nameserver 2001:4860:4860::6464
+            '';
+            boot.initrd.systemd.contents."/etc/systemd/journald.conf".text = ''
+              [Journal]
+              ForwardToConsole=yes
+              MaxLevelConsole=debug
+            '';
+
+            # Provide a bootable USB drive image
+            system.build.usbImage = pkgs.callPackage ({ stdenv, runCommand, dosfstools, e2fsprogs, mtools, libfaketime, util-linux, nukeReferences }:
+              runCommand "boot-img-${node.config.networking.hostName}" {
+                nativeBuildInputs = [ dosfstools e2fsprogs libfaketime mtools util-linux ];
+                outputs = [ "out" "firmware_part" ];
+              } ''
+                export img=$out
+                truncate -s 40M $img
+
+                sfdisk $img <<EOF
+                    label: gpt
+                    label-id: F222513B-DED1-49FA-B591-20CE86A2FE7F
+
+                    type=C12A7328-F81F-11D2-BA4B-00A0C93EC93B, bootable
+                EOF
+
+                # Create a FAT32 /boot/firmware partition of suitable size into firmware_part.img
+                eval $(partx $img -o START,SECTORS --nr 1 --pairs)
+                truncate -s $((2081 * 512 + SECTORS * 512)) firmware_part.img
+
+                mkfs.vfat --invariant -i 2e24ec82 -n BOOT firmware_part.img
+
+                # Populate the files intended for /boot/firmware
+                mkdir -p firmware/EFI/BOOT firmware/loader/entries
+                cp ${pkgs.systemd}/lib/systemd/boot/efi/systemd-boot*.efi firmware/EFI/BOOT/BOOT${lib.toUpper stdenv.hostPlatform.efiArch}.EFI
+
+                cat > firmware/loader/loader.conf << EOF
+                default foo
+                EOF
+                cat > firmware/loader/entries/default.conf << EOF
+                title Default
+                linux /EFI/${pkgs.stdenv.hostPlatform.linux-kernel.target}
+                initrd /EFI/initrd
+                options init=${config.system.build.toplevel}/init ${toString config.boot.kernelParams}
+                EOF
+                cp ${config.system.build.kernel}/${pkgs.stdenv.hostPlatform.linux-kernel.target} firmware/EFI/${pkgs.stdenv.hostPlatform.linux-kernel.target}
+                cp ${config.system.build.initialRamdisk}/${config.system.boot.loader.initrdFile} firmware/EFI/initrd
+
+                find firmware -exec touch --date=2000-01-01 {} +
+                # Copy the populated /boot/firmware into the SD image
+                cd firmware
+                # Force a fixed order in mcopy for better determinism, and avoid file globbing
+                for d in $(find . -type d -mindepth 1 | sort); do
+                  faketime "2000-01-01 00:00:00" mmd -i ../firmware_part.img "::/$d"
+                done
+                for f in $(find . -type f | sort); do
+                  mcopy -pvm -i ../firmware_part.img "$f" "::/$f"
+                done
+                cd ..
+
+                # Verify the FAT partition before copying it.
+                fsck.vfat -vn firmware_part.img
+                dd conv=notrunc if=firmware_part.img of=$img seek=$START count=$SECTORS
+
+                cp firmware_part.img $firmware_part
+              ''
+            ) {};
+          }
+        ;
+      };
+
+      # This is the config which will actually be booted
+      netbootVariant = extendModules {
+        modules = [
+          (
+            { modulesPath, ... }:
+
+            {
+              imports = [ (modulesPath + "/installer/netboot/netboot.nix") ];
+            }
+          )
+        ];
+      };
+      # A derivation combining all the artifacts required for netbooting for the hydra job
+      netbootDir = let
+        kernelTarget = pkgs.stdenv.hostPlatform.linux-kernel.target;
+        build = config.system.build.netbootVariant.config.system.build;
+      in
+        pkgs.symlinkJoin {
+          name = "netboot";
+          paths = [
+            build.netbootRamdisk
+            build.kernel
+            build.netbootIpxeScript
+          ];
+          postBuild = ''
+            mkdir -p $out/nix-support
+            echo "file ${kernelTarget} $out/${kernelTarget}" >> $out/nix-support/hydra-build-products
+            echo "file initrd $out/initrd" >> $out/nix-support/hydra-build-products
+            echo "file ipxe $out/netboot.ipxe" >> $out/nix-support/hydra-build-products
+          '';
+          preferLocalBuild = true;
+        };
+    };
+  };
+}

services/channel-scripts/default.nix

@@ -0,0 +1,192 @@
+{ lib, config, pkgs, ... }:
+# FIXME(raito): I'm really really really not happy with this design of NixOS module, clean up all of this someday.
+let
+  inherit (lib) mkEnableOption mkOption types mkIf mapAttrsToList mkPackageOption concatStringsSep mkMerge;
+  cfg = config.bagel.nixpkgs.channel-scripts;
+  toml = pkgs.formats.toml { };
+  configFile = toml.generate "forkos.toml" cfg.settings;
+  orderLib = import ./service-order.nix { inherit lib; };
+  makeUpdateJob = channelName: mainJob: {
+    name = "update-${channelName}";
+    value = {
+      description = "Update channel ${channelName}";
+      path = with pkgs; [ git ];
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = false;
+        User = "channel-scripts";
+        DynamicUser = true;
+        StateDirectory = "channel-scripts";
+        MemoryHigh = "80%";
+        EnvironmentFile = [
+          cfg.releaseBucketCredentialsFile
+        ];
+        Environment = cfg.extraEnvironment;
+        # TODO: we should have our own secret for this.
+        LoadCredential = [ "password:${config.age.secrets.alloy-push-password.path}" ];
+      };
+      unitConfig.After = [ "networking.target" ];
+      script =
+      ''
+        # A stateful copy of nixpkgs
+        dir=/var/lib/channel-scripts/nixpkgs
+        if ! [[ -e $dir ]]; then
+          git clone --bare ${cfg.nixpkgsUrl} $dir
+        fi
+        GIT_DIR=$dir git config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*'
+
+        CREDENTIAL=$(echo -en "promtail:$(cat $CREDENTIALS_DIRECTORY/password)" | base64)
+        export OTEL_EXPORTER_OTLP_HEADERS="Authorization=Basic $CREDENTIAL"
+        # TODO: use escapeShellArgs
+        exec ${cfg.package}/bin/mirror-forkos -c ${configFile} ${concatStringsSep " " cfg.extraArgs} apply ${channelName} ${mainJob}
+      '';
+    };
+  };
+  updateJobs = orderLib.mkOrderedChain (mapAttrsToList (n: { job, ... }: makeUpdateJob n job) cfg.channels);
+  channelOpts = { ... }: {
+    options = {
+      job = mkOption {
+        type = types.str;
+        example = "nixos/trunk-combined/tested";
+      };
+
+      variant = mkOption {
+        type = types.enum [ "primary" "small" "darwin" "aarch64" ];
+        example = "primary";
+      };
+
+      status = mkOption {
+        type = types.enum [ "beta" "stable" "deprecated" "unmaintained" "rolling" ];
+        example = "rolling";
+      };
+    };
+  };
+in
+{
+  options.bagel.nixpkgs.channel-scripts = {
+    enable = mkEnableOption ''the channel scripts.
+      Fast forwarding channel branches which are read-only except for this privileged bot
+      based on our Hydra acceptance tests.
+    '';
+
+    otlp.enable = mkEnableOption "the OTLP export process";
+
+    s3 = {
+      release = mkOption {
+        type = types.str;
+      };
+
+      channel = mkOption {
+        type = types.str;
+      };
+    };
+
+    package = mkPackageOption pkgs "mirror-forkos" { };
+
+    settings = mkOption {
+      type = types.attrsOf types.anything;
+    };
+
+    nixpkgsUrl = mkOption {
+      type = types.str;
+      default = "https://cl.forkos.org/nixpkgs.git";
+      description = "URL to the nixpkgs repository to clone and to push to";
+    };
+
+    binaryCacheUrl = mkOption {
+      type = types.str;
+      default = "https://cache.forkos.org";
+      description = "URL to the binary cache";
+    };
+
+    baseUriForGitRevisions = mkOption {
+      type = types.str;
+      description = "Base URI to generate link to a certain revision";
+    };
+
+    extraArgs = mkOption {
+      type = types.listOf types.str;
+      default = [ ];
+      description = "Extra arguments passed to the mirroring program";
+    };
+
+    releaseBucketCredentialsFile = mkOption {
+      type = types.path;
+      description = ''Path to the release bucket credentials file exporting S3-style environment variables.
+        For example, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` for the S3 operations to work.
+      '';
+    };
+
+    deployKeyFile = mkOption {
+      type = types.path;
+      description = ''Path to the private SSH key which is allowed to deploy things to the protected channel references on the Git repository.
+      '';
+    };
+
+    hydraUrl = mkOption {
+      type = types.str;
+      default = "https://hydra.forkos.org";
+      description = "URL to the Hydra instance";
+    };
+
+    channels = mkOption {
+      type = types.attrsOf (types.submodule channelOpts);
+      description = "List of channels to mirror";
+    };
+
+    extraEnvironment = mkOption {
+      type = types.listOf types.str;
+    };
+  };
+
+  config = mkIf cfg.enable {
+    bagel.nixpkgs.channel-scripts.extraEnvironment = mkMerge [
+      ([
+        "RUST_LOG=info"
+      ])
+      (mkIf cfg.otlp.enable [
+        ''OTEL_EXPORTER_OTLP_TRACES_ENDPOINT="https://tempo.forkos.org/v1/traces"''
+      ])
+    ];
+    bagel.nixpkgs.channel-scripts.settings = {
+      hydra_uri = cfg.hydraUrl;
+      binary_cache_uri = cfg.binaryCacheUrl;
+      base_git_uri_for_revision = cfg.baseUriForGitRevisions;
+      nixpkgs_dir = "/var/lib/channel-scripts/nixpkgs";
+      s3_release_bucket_name = cfg.s3.release;
+      s3_channel_bucket_name = cfg.s3.channel;
+    };
+
+    users.users.channel-scripts = {
+      description = "Channel scripts user";
+      isSystemUser = true;
+      group = "channel-scripts";
+    };
+    users.groups.channel-scripts = {};
+
+    systemd.services = (lib.listToAttrs updateJobs) // {
+      "update-all-channels" = {
+        description = "Start all channel updates.";
+        unitConfig = {
+          After = map
+            (service: "${service.name}.service")
+            updateJobs;
+          Wants = map
+            (service: "${service.name}.service")
+            updateJobs;
+        };
+        script = "true";
+      };
+    };
+
+    systemd.timers."update-all-channels" = {
+      description = "Start all channel updates.";
+      wantedBy = [ "timers.target" ];
+      timerConfig = {
+        OnUnitInactiveSec = 600;
+        OnBootSec = 900;
+        AccuracySec = 300;
+      };
+    };
+  };
+}

services/channel-scripts/service-order.nix

@@ -0,0 +1,63 @@
+# Vendored from https://raw.githubusercontent.com/NixOS/infra/master/lib/service-order.nix
+# TODO: get rid of me?
+# Ordering Services
+#
+# Given a set of services, make them run one at a time in a specific
+# order, on a timer.
+{ lib }:
+{
+  # Given a list of systemd service, give each one an After
+  # attribute, so they start in a specific order. The returned
+  # list can be converted in to a systemd.services attrset with
+  # `lib.listToAttrs`.
+  #
+  # Example:
+  #
+  #  mkOrderedChain [
+  #    { name = "foo"; value = { script = "true"; }; }
+  #    { name = "bar"; value = { script = "true"; }; }
+  #  ]
+  #
+  # => [
+  #  {
+  #    name = "foo";
+  #    value = {
+  #      script = "true";
+  #      unitConfig = { After = []; };
+  #    };
+  #  }
+  #  {
+  #    name = "bar";
+  #    value = {
+  #      script = "true";
+  #      unitConfig = { After = [ "bar" ]; };
+  #    };
+  #  }
+  #
+  mkOrderedChain = jobs: let
+    unitConfigFrom = job: job.unitConfig or {};
+    afterFrom = job: (unitConfigFrom job).After or [];
+    previousFrom = collector:
+      if collector ? previous
+      then [collector.previous]
+      else [];
+
+      ordered = builtins.foldl'
+        (collector: item: {
+          services = collector.services
+          ++ [{
+            inherit (item) name;
+            value = item.value // {
+              unitConfig = (unitConfigFrom item.value) //
+              {
+                After = (afterFrom item.value) ++
+                  (previousFrom collector);
+              };
+            };
+          }];
+          previous = "${item.name}.service";
+        })
+        { services = []; }
+        jobs;
+  in ordered.services;
+}

services/default.nix

@@ -1,7 +1,9 @@
 {
   imports = [
     ./gerrit
+    ./channel-scripts
     ./hydra
+    ./matrix
     ./monitoring
     ./netbox
     ./ofborg
@@ -9,5 +11,7 @@
     ./forgejo
     ./baremetal-builder
     ./buildbot
+    ./newsletter
+    ./s3-revproxy
   ];
 }

services/gerrit/default.nix

@@ -12,6 +12,10 @@ in
 {
   options.bagel.services.gerrit = {
     enable = mkEnableOption "Gerrit";
+    pyroscope.enable = mkEnableOption ''Pyroscope client,
+      this will send profiling of all Java processes on the current host
+      to our Pyroscope instance.
+    '';
     domains = mkOption {
       type = types.listOf types.str;
       description = "List of domains that Gerrit will answer to";
@@ -41,8 +45,13 @@ in
 
   config = mkIf cfg.enable {
     networking.firewall.allowedTCPPorts = [ cfg.port ];
+    age.secrets.alloy-push-password.file = ../../secrets/metrics-push-password.age;
 
-    environment.systemPackages = [ jdk ];
+    environment.systemPackages = [ jdk
+      pkgs.git
+      # https://gerrit.googlesource.com/gerrit/+/refs/heads/master/contrib/git-gc-preserve
+      pkgs.git-gc-preserve
+    ];
 
     fileSystems."/var/lib/gerrit" = mkIf (cfg.data != "/var/lib/gerrit") {
       device = cfg.data;
@@ -55,6 +64,64 @@ in
     };
     users.groups.git = {};
 
+    services.alloy = {
+      enable = cfg.pyroscope.enable;
+      extraFlags = [
+        # Debugging interface.
+        "--server.http.listen-addr=127.0.0.1:15555"
+      ];
+    };
+    systemd.services.alloy.serviceConfig = {
+      User = lib.mkForce "root";
+      Group = lib.mkForce "root";
+      DynamicUser = lib.mkForce false;
+    };
+    systemd.services.alloy.serviceConfig.LoadCredential = [ "password:${config.age.secrets.alloy-push-password.path}" ];
+    environment.etc."alloy/config.alloy".text = ''
+      pyroscope.write "production" {
+        endpoint {
+          url = "https://pyroscope.forkos.org"
+          basic_auth {
+            username = "promtail"
+            password_file = "/run/credentials/alloy.service/password"
+          }
+        }
+      }
+
+      discovery.process "all" {
+        refresh_interval = "60s"
+        discover_config {
+          cwd = true
+          exe = true
+          commandline = true
+          username = true
+          uid = true
+          container_id = true
+        }
+      }
+
+      discovery.relabel "java" {
+        targets = discovery.process.all.targets
+        rule {
+          action = "keep"
+          regex = ".*/java$"
+          source_labels = ["__meta_process_exe"]
+        }
+      }
+
+      pyroscope.java "java" {
+        targets = discovery.relabel.java.output
+        forward_to = [pyroscope.write.production.receiver]
+        profiling_config {
+          interval = "60s"
+          alloc = "512k"
+          cpu = true
+          sample_rate = 100
+          lock = "1ms"
+        }
+      }
+    '';
+
     services.gerrit = {
       enable = true;
       listenAddress = "[::]:4778"; # 4778 - grrt
@@ -111,14 +178,16 @@ in
 
         receive.timeout = "4min";
         # Default is 0, infinite.
-        transfer.timeout = "30min";
+        # When system is under pressure and there's too many packfiles
+        # the search for reuse can take a stupid amount of time.
+        transfer.timeout = "60min";
 
         # We may overshoot but it's OK.
         core.packedGitWindowSize = "256k";
         # Sum of all current packfiles is ~1.2G
         # Largest packfile is 906MB.
         # Average packfile is ~5-10MB.
-        core.packedGitLimit = "1g";
+        core.packedGitLimit = "2g";
         # We have plenty of memory, let's avoid file system cache → Gerrit needless copies.
         core.packedGitUseStrongRefs = true;
         core.packedGitOpenFiles = 4096;

services/gerrit/one-way-sync.nix

@@ -3,7 +3,7 @@ let
   cfg = config.bagel.nixpkgs.one-way-sync;
   inherit (lib) mkIf mkOption mkEnableOption types mapAttrs';
 
-  mkSyncTimer = { name, timer, ... }: {
+  mkSyncTimer = name: { timer, ... }: {
     wantedBy = [ "timers.target" ];
 
     timerConfig = {
@@ -12,7 +12,7 @@ let
       Unit = "ows-${name}.service";
     };
   };
-  mkSyncService = targetRef: { name, fromUri, fromRefspec, localRefspec, ... }: {
+  mkSyncService = name: { fromUri, fromRefspec, localRefspec, ... }: {
     path = [ pkgs.gitFull pkgs.openssh pkgs.lix ];
     script = ''
       set -xe
@@ -25,11 +25,11 @@ let
       fi
 
       cd /var/lib/onewaysync/nixpkgs
-      echo "Syncing ${fromUri}:${fromRefspec} to /var/lib/onewaysync/nixpkgs:${targetRef}"
+      echo "Syncing ${fromUri}:${fromRefspec} to ${cfg.pushUrl}:refs/heads/${localRefspec}"
       echo "Current ref: $EXPECTED_REF"
       git worktree add -f "$RUNTIME_DIRECTORY"/${name} refs/remotes/origin/${localRefspec}
       cd "$RUNTIME_DIRECTORY"/${name}
-      git pull origin ${localRefspec}
+      git pull origin ${localRefspec} --no-rebase
       EXPECTED_REF=$(git rev-list refs/remotes/origin/${localRefspec} | head -1)
       git config user.name Fork-o-Tron
       git config user.email noreply@forkos.org
@@ -43,7 +43,7 @@ let
       # Do not allow auto-merging a staging iteration
       test "$OLD_STDENV" = "$NEW_STDENV"
     '' + ''
-      GIT_SSH_COMMAND='ssh -i ${cfg.deployKeyPath}' git push ${cfg.pushUrl} HEAD:${targetRef}
+      GIT_SSH_COMMAND='ssh -i ${cfg.deployKeyPath}' git push ${cfg.pushUrl} HEAD:refs/heads/${localRefspec}
     '';
     serviceConfig = {
       User = "git";
@@ -120,12 +120,12 @@ in
 
   config = mkIf cfg.enable {
     systemd.timers = mapAttrs' (name: value: {
-      name = "ows-${value.name}";
-      value = mkSyncTimer value;
+      name = "ows-${name}";
+      value = mkSyncTimer name value;
     }) cfg.branches;
 
     systemd.services = mapAttrs' (name: value: {
-      name = "ows-${value.name}";
+      name = "ows-${name}";
       value = mkSyncService name value;
     }) cfg.branches;
   };

services/hydra/default.nix

@@ -9,7 +9,11 @@ let
 
   mkCacheSettings = settings: builtins.concatStringsSep "&" (
     lib.mapAttrsToList (k: v: "${k}=${v}") settings
-    );
+  );
+
+  mkPgConnString = options: builtins.concatStringsSep ";" (
+    lib.mapAttrsToList (k: v: "${k}=${v}") options
+  );
 
   # XXX: to support Nix's dumb public host key syntax (base64'd), this outputs
   # a string with shell-style command interpolations: $(...).
@@ -50,11 +54,6 @@ in {
   options.bagel.services.hydra = with lib; {
     enable = mkEnableOption "Hydra coordinator";
 
-    dbi = mkOption {
-      type = types.str;
-      description = "DBI connection string for the Hydra postgres database";
-    };
-
     builders = mkOption {
       type = types.listOf types.str;
       description = "List of builders to configure for Hydra";
@@ -69,6 +68,10 @@ in {
 
     age.secrets.hydra-s3-credentials.file = ../../secrets/hydra-s3-credentials.age;
 
+    age.secrets.hydra-postgres-key.group = "hydra";
+    age.secrets.hydra-postgres-key.mode = "0440";
+    age.secrets.hydra-postgres-key.file = ../../secrets/hydra-postgres-key.age;
+
     age.secrets.hydra-signing-priv.owner = "hydra-queue-runner";
     age.secrets.hydra-signing-priv.file = ../../secrets/hydra-signing-priv.age;
 
@@ -97,9 +100,18 @@ in {
     services.hydra-dev = {
       enable = true;
 
-      listenHost = "localhost";
+      listenHost = "127.0.0.1";
       port = port;
-      dbi = cfg.dbi;
+
+      dbi = "dbi:Pg:${mkPgConnString {
+        host = "postgres.forkos.org";
+        dbname = "hydra";
+        user = "hydra";
+        sslmode = "verify-full";
+        sslcert = "${./postgres.crt}";
+        sslkey = config.age.secrets.hydra-postgres-key.path;
+        sslrootcert = "${../postgres/ca.crt}";
+      }}";
 
       hydraURL = "https://hydra.forkos.org";
       useSubstitutes = false;
@@ -134,7 +146,7 @@ in {
 
         server_store_uri = https://bagel-cache.s3-web.delroth.net?local-nar-cache=${narCacheDir}
         binary_cache_public_url = https://bagel-cache.s3-web.delroth.net
-        log_prefix = https://bagel-cache.s3-web.delroth.net
+        log_prefix = https://bagel-cache.s3-web.delroth.net/
 
         upload_logs_to_binary_cache = true
 
@@ -146,6 +158,10 @@ in {
 
         max_output_size = ${builtins.toString (3 * 1024 * 1024 * 1024)}
         max_db_connections = 100
+
+        <git-input>
+          timeout = 1800
+        </git-input>
       '';
     };
 

services/hydra/postgres.crt

@@ -0,0 +1,12 @@
+-----BEGIN CERTIFICATE-----
+MIIBtDCCAVugAwIBAgIQTU55o4gtG8EFZArwsuj4NjAKBggqhkjOPQQDAjAiMSAw
+HgYDVQQDExdGb3JrT1MgUG9zdGdyZXMgUm9vdCBDQTAeFw0yNDA4MTYwNTU0MTda
+Fw0zNDA4MTYxNzU0MTdaMBAxDjAMBgNVBAMTBWh5ZHJhMFkwEwYHKoZIzj0CAQYI
+KoZIzj0DAQcDQgAEnTgiFZOXBrcPlWDxJPXUFgxIi7/T7LmwLtpGPK/G6R8KA9cS
+4UXF5Ifz2dCgozTlhqLROKb81yhNsSy1tOcFyKOBhDCBgTAOBgNVHQ8BAf8EBAMC
+B4AwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMB0GA1UdDgQWBBQGh6HM
+jl41qw/F0vpBdmOWQ2IfGzAfBgNVHSMEGDAWgBQy4WX/hUExQ/i1h7MvF6Ow2irN
+izAQBgNVHREECTAHggVoeWRyYTAKBggqhkjOPQQDAgNHADBEAiAEypqfyMOGbEJv
+dKI1tyj890uq5Osr5+9wxGBvJDMJNwIgefyOdFcvJTzbfHgLmORpBOVtnpbkwj5y
+rMnjT8gYjEA=
+-----END CERTIFICATE-----

services/matrix/default.nix

@@ -0,0 +1,68 @@
+{
+  config,
+  lib,
+  inputs,
+  ...
+}:
+
+let
+  cfg = config.bagel.services.grapevine;
+  inherit (lib) mkEnableOption mkIf;
+in
+
+{
+  imports = [
+    inputs.grapevine.nixosModules.default
+    ./hookshot.nix
+  ];
+
+  options.bagel.services.grapevine.enable = mkEnableOption "Grapevine";
+
+  config = mkIf cfg.enable {
+    services = {
+      grapevine = {
+        enable = true;
+        settings = {
+          listen = [
+            {
+              type = "tcp";
+              address = "127.0.0.1";
+              port = 6167;
+            }
+          ];
+          server_name = "forkos.org";
+          database.backend = "rocksdb";
+        };
+      };
+
+      nginx = {
+        upstreams.grapevine.servers."127.0.0.1:6167" = { };
+
+        virtualHosts = {
+          "matrix.forkos.org" = {
+            forceSSL = true;
+            enableACME = true;
+            locations."/".proxyPass = "http://grapevine";
+          };
+
+          "forkos.org" = {
+            forceSSL = true;
+            enableACME = true;
+            locations = {
+              "= /.well-known/matrix/server".extraConfig = ''
+                add_header Content-Type application/json;
+                add_header Access-Control-Allow-Origin *;
+                return 200 '{"m.server": "matrix.forkos.org:443"}';
+              '';
+              "= /.well-known/matrix/client".extraConfig = ''
+                add_header Content-Type application/json;
+                add_header Access-Control-Allow-Origin *;
+                return 200 '{"m.homeserver": {"base_url": "https://matrix.forkos.org/"}, "m.identity_server": {"base_url": "https://matrix.org/"}, "org.matrix.msc3575.proxy": {"url": "https://matrix.forkos.org"}}';
+              '';
+            };
+          };
+        };
+      };
+    };
+  };
+}

services/matrix/hookshot.nix

@@ -0,0 +1,77 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.bagel.services.hookshot;
+  inherit (lib) mkEnableOption mkIf mkOption types;
+  keyPath = "/var/lib/matrix-hookshot/key.pem";
+in
+{
+  options.bagel.services.hookshot = {
+    enable = mkEnableOption "matrix-hookshot";
+    settings = mkOption {
+      description = "Settings";
+      type = (pkgs.formats.yaml { }).type;
+    };
+    admins = mkOption {
+      description = "List of admin MXIDs";
+      type = types.listOf types.str;
+    };
+  };
+
+  config = mkIf cfg.enable {
+    systemd.services.matrix-hookshot = {
+      wantedBy = ["multi-user.target"];
+      wants = ["network-online.target"];
+      after = ["network-online.target"];
+      serviceConfig = {
+        ExecStart = "${lib.getExe pkgs.matrix-hookshot} ${pkgs.writers.writeYAML "config.yaml" cfg.settings}";
+        ExecStartPre = pkgs.writeShellScript "hookshot-generate-key" ''
+          if [ ! -f ${keyPath} ]; then
+            mkdir -p $(dirname ${keyPath})
+            ${lib.getExe pkgs.openssl} genpkey -out ${keyPath} -outform PEM -algorithm RSA -pkeyopt rsa_keygen_bits:4096
+          fi
+        '';
+        DynamicUser = true;
+        StateDirectory = "matrix-hookshot";
+        WorkingDirectory = "/var/lib/matrix-hookshot";
+      };
+    };
+
+    bagel.services.hookshot.settings = {
+      bridge = {
+        domain = "forkos.org";
+        url = "https://matrix.forkos.org";
+        mediaUrl = "https://forkos.org";
+        port = 9993;
+        bindAddress = "127.0.0.1";
+      };
+      passFile = keyPath;
+      listeners = [{
+        port = 9994;
+        bindAddress = "127.0.0.1";
+        resources = [ "webhooks" ];
+      }];
+      generic = {
+        enabled = true;
+        urlPrefix = "https://alerts.forkos.org/webhook";
+      };
+      permissions = map (mxid: {
+        actor = mxid;
+        services = [{
+          service = "*";
+          level = "admin";
+        }];
+      }) cfg.admins;
+    };
+
+    services.nginx.virtualHosts."alerts.forkos.org" = {
+      forceSSL = true;
+      enableACME = true;
+      locations."/".proxyPass = "http://127.0.0.1:9994";
+    };
+  };
+}

services/monitoring/default.nix

@@ -3,5 +3,7 @@
     ./exporters
     ./lgtm
     ./agent.nix
+    ./hookshot-adapter
+    ./pyroscope
   ];
-}
+}

services/monitoring/hookshot-adapter/default.nix

@@ -0,0 +1,30 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.bagel.services.alertmanager-hookshot-adapter;
+  inherit (lib) mkEnableOption mkIf;
+  package = pkgs.callPackage ./package.nix {};
+in
+{
+  options.bagel.services.alertmanager-hookshot-adapter.enable = mkEnableOption "alertmanager to matrix-hookshot adapter";
+
+  config = mkIf cfg.enable {
+    systemd.services.alertmanager-hookshot-adapter = {
+      wantedBy = ["multi-user.target"];
+      wants = ["network-online.target"];
+      after = ["network-online.target"];
+      environment = {
+        PORT = "9100";
+        UPSTREAM = "https://alerts.forkos.org/webhook";
+      };
+      serviceConfig = {
+        ExecStart = lib.getExe package;
+        DynamicUser = true;
+      };
+    };
+  };
+}

services/monitoring/hookshot-adapter/package.json

@@ -0,0 +1,23 @@
+{
+  "name": "alertmanager-hookshot-adapter",
+  "version": "1.0.0",
+  "description": "Adapter between alertmanager webhooks and the Matrix Hookshot Apapter",
+  "main": "index.ts",
+  "license": "Apache-2.0",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/hm-edu/alertmanager-hookshot-adapter"
+  },
+  "dependencies": {
+    "@types/express": "^4.17.21",
+    "@types/node": "^20.11.20",
+    "dotenv": "^16.4.5",
+    "express": "^4.18.2",
+    "node-fetch": "^3.3.2",
+    "typescript": "^5.3.3",
+    "winston": "^3.13.0"
+  },
+  "scripts": {
+    "build": "npx tsc"
+  }
+}

services/monitoring/hookshot-adapter/package.nix

@@ -0,0 +1,40 @@
+{
+  lib,
+  mkYarnPackage,
+  fetchFromGitHub,
+  fetchYarnDeps,
+  makeWrapper,
+  nodejs,
+}:
+
+mkYarnPackage rec {
+  pname = "alertmanager-hookshot-adapter";
+  version = "1.9.1";
+
+  src = fetchFromGitHub {
+    owner = "hm-edu";
+    repo = "alertmanager-hookshot-adapter";
+    rev = "v${version}";
+    hash = "sha256-KTk70zFA1tymmR8AYrAl2XIyA+SPs5Uksd6Z3kvUb+o=";
+  };
+
+  packageJSON = ./package.json;
+
+  offlineCache = fetchYarnDeps {
+    yarnLock = "${src}/yarn.lock";
+    hash = "sha256-LU25cXB+0DdcHRzKQ1hjQIVntarqPOUXZTgcw6lvLRM=";
+  };
+
+  buildPhase = ''
+    yarn build
+  '';
+
+  nativeBuildInputs = [ makeWrapper ];
+
+  postInstall = ''
+    makeWrapper ${lib.getExe nodejs} $out/bin/alertmanager-hookshot-adapter \
+      --add-flags $out/libexec/alertmanager-hookshot-adapter/deps/alertmanager-hookshot-adapter/dist/index.js
+  '';
+
+  meta.mainProgram = "alertmanager-hookshot-adapter";
+}

services/monitoring/lgtm/alerts/demo.yaml

@@ -1,5 +0,0 @@
-groups:
-  - name: Demo alerts
-    rules:
-      - alert: Demo alert
-        expr: 1

services/monitoring/lgtm/alerts/forkos.yaml

@@ -0,0 +1,11 @@
+groups:
+  - name: ForkOS automation
+    rules:
+      - alert: SyncFailedTooOften
+        expr: 'changes(node_systemd_unit_state{name=~"ows.*.service",state="failed"}[24h]) > 2'
+        for: 30m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Synchronization job {{ $labels.name }} has failed more than twice in the last 24 hours"
+          description: "On {{ $labels.instance }}, the synchronization job has failed more than twice in the last 24 hours, check if there's a conflict or a stdenv change."

services/monitoring/lgtm/alerts/postgres.yml

@@ -0,0 +1,102 @@
+groups:
+- name: PostgreSQL
+  rules:
+
+    - alert: PostgresqlTableNotAutoVacuumed
+      expr: '(pg_stat_user_tables_last_autovacuum > 0) and (time() - pg_stat_user_tables_last_autovacuum) > 60 * 60 * 24 * 10'
+      for: 0m
+      labels:
+        severity: warning
+      annotations:
+        summary: Postgresql table not auto vacuumed (instance {{ $labels.instance }})
+        description: "Table {{ $labels.relname }} has not been auto vacuumed for 10 days\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+    - alert: PostgresqlTableNotAutoAnalyzed
+      expr: '(pg_stat_user_tables_last_autoanalyze > 0) and (time() - pg_stat_user_tables_last_autoanalyze) > 24 * 60 * 60 * 10'
+      for: 0m
+      labels:
+        severity: warning
+      annotations:
+        summary: Postgresql table not auto analyzed (instance {{ $labels.instance }})
+        description: "Table {{ $labels.relname }} has not been auto analyzed for 10 days\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+    - alert: PostgresqlDeadLocks
+      expr: 'increase(pg_stat_database_deadlocks{datname!~"template.*|postgres"}[1m]) > 5'
+      for: 0m
+      labels:
+        severity: warning
+      annotations:
+        summary: Postgresql dead locks (instance {{ $labels.instance }})
+        description: "PostgreSQL has dead-locks\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+    - alert: PostgresqlHighRollbackRate
+      expr: 'sum by (namespace,datname) ((rate(pg_stat_database_xact_rollback{datname!~"template.*|postgres",datid!="0"}[3m])) / ((rate(pg_stat_database_xact_rollback{datname!~"template.*|postgres",datid!="0"}[3m])) + (rate(pg_stat_database_xact_commit{datname!~"template.*|postgres",datid!="0"}[3m])))) > 0.02'
+      for: 0m
+      labels:
+        severity: warning
+      annotations:
+        summary: Postgresql high rollback rate (instance {{ $labels.instance }})
+        description: "Ratio of transactions being aborted compared to committed is > 2 %\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+    - alert: PostgresqlHighRateStatementTimeout
+      expr: 'rate(postgresql_errors_total{type="statement_timeout"}[1m]) > 3'
+      for: 0m
+      labels:
+        severity: critical
+      annotations:
+        summary: Postgresql high rate statement timeout (instance {{ $labels.instance }})
+        description: "Postgres transactions showing high rate of statement timeouts\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+    - alert: PostgresqlHighRateDeadlock
+      expr: 'increase(postgresql_errors_total{type="deadlock_detected"}[1m]) > 1'
+      for: 0m
+      labels:
+        severity: critical
+      annotations:
+        summary: Postgresql high rate deadlock (instance {{ $labels.instance }})
+        description: "Postgres detected deadlocks\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+    - alert: PostgresqlTooManyDeadTuples
+      expr: '((pg_stat_user_tables_n_dead_tup > 10000) / (pg_stat_user_tables_n_live_tup + pg_stat_user_tables_n_dead_tup)) >= 0.1'
+      for: 2m
+      labels:
+        severity: warning
+      annotations:
+        summary: Postgresql too many dead tuples (instance {{ $labels.instance }})
+        description: "PostgreSQL dead tuples is too large\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+    - alert: PostgresqlTooManyLocksAcquired
+      expr: '((sum (pg_locks_count)) / (pg_settings_max_locks_per_transaction * pg_settings_max_connections)) > 0.20'
+      for: 2m
+      labels:
+        severity: critical
+      annotations:
+        summary: Postgresql too many locks acquired (instance {{ $labels.instance }})
+        description: "Too many locks acquired on the database. If this alert happens frequently, we may need to increase the postgres setting max_locks_per_transaction.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+    - alert: PostgresqlBloatIndexHigh(>80%)
+      expr: 'pg_bloat_btree_bloat_pct > 80 and on (idxname) (pg_bloat_btree_real_size > 100000000)'
+      for: 1h
+      labels:
+        severity: warning
+      annotations:
+        summary: Postgresql bloat index high (> 80%) (instance {{ $labels.instance }})
+        description: "The index {{ $labels.idxname }} is bloated. You should execute `REINDEX INDEX CONCURRENTLY {{ $labels.idxname }};`\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+    - alert: PostgresqlBloatTableHigh(>80%)
+      expr: 'pg_bloat_table_bloat_pct > 80 and on (relname) (pg_bloat_table_real_size > 200000000)'
+      for: 1h
+      labels:
+        severity: warning
+      annotations:
+        summary: Postgresql bloat table high (> 80%) (instance {{ $labels.instance }})
+        description: "The table {{ $labels.relname }} is bloated. You should execute `VACUUM {{ $labels.relname }};`\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+    - alert: PostgresqlInvalidIndex
+      expr: 'pg_genaral_index_info_pg_relation_size{indexrelname=~".*ccnew.*"}'
+      for: 6h
+      labels:
+        severity: warning
+      annotations:
+        summary: Postgresql invalid index (instance {{ $labels.instance }})
+        description: "The table {{ $labels.relname }} has an invalid index: {{ $labels.indexrelname }}. You should execute `DROP INDEX {{ $labels.indexrelname }};`\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

services/monitoring/lgtm/alerts/resources.yaml

@@ -0,0 +1,76 @@
+groups:
+  - name: Host & hardware
+    rules:
+      - alert: HostOutOfMemory
+        expr: (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}
+        for: 2m
+        labels:
+          severity: warning
+        annotations:
+          summary: Host out of memory (instance {{ $labels.instance }})
+          description: "Node memory is filling up (< 10% left)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+      - alert: HostMemoryUnderMemoryPressure
+        expr: (rate(node_vmstat_pgmajfault[1m]) > 1000) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}
+        for: 2m
+        labels:
+          severity: warning
+        annotations:
+          summary: Host memory under memory pressure (instance {{ $labels.instance }})
+          description: "The node is under heavy memory pressure. High rate of major page faults\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+      - alert: HostMemoryIsUnderutilized
+        expr: (100 - (avg_over_time(node_memory_MemAvailable_bytes[30m]) / node_memory_MemTotal_bytes * 100) < 20) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}
+        for: 1w
+        labels:
+          severity: info
+        annotations:
+          summary: Host Memory is underutilized (instance {{ $labels.instance }})
+          description: "Node memory is < 20% for 1 week. Consider reducing memory space. (instance {{ $labels.instance }})\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+      - alert: HostOutOfDiskSpace
+        expr: ((node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and ON (instance, device, mountpoint) node_filesystem_readonly == 0) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}
+        for: 2m
+        labels:
+          severity: warning
+        annotations:
+          summary: Host out of disk space (instance {{ $labels.instance }})
+          description: "Disk is almost full (< 10% left)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+      - alert: HostDiskWillFillIn24Hours
+        expr: ((node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and ON (instance, device, mountpoint) predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs"}[1h], 24 * 3600) < 0 and ON (instance, device, mountpoint) node_filesystem_readonly == 0) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}
+        for: 2m
+        labels:
+          severity: warning
+        annotations:
+          summary: Host disk will fill in 24 hours (instance {{ $labels.instance }})
+          description: "Filesystem is predicted to run out of space within the next 24 hours at current write rate\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+      - alert: HostCpuIsUnderutilized
+        expr: (100 - (rate(node_cpu_seconds_total{mode="idle"}[30m]) * 100) < 20) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}
+        for: 1w
+        labels:
+          severity: info
+        annotations:
+          summary: Host CPU is underutilized (instance {{ $labels.instance }})
+          description: "CPU load is < 20% for 1 week. Consider reducing the number of CPUs.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+      - alert: HostCpuStealNoisyNeighbor
+        expr: (avg by(instance) (rate(node_cpu_seconds_total{mode="steal"}[5m])) * 100 > 10) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}
+        for: 0m
+        labels:
+          severity: warning
+        annotations:
+          summary: Host CPU steal noisy neighbor (instance {{ $labels.instance }})
+          description: "CPU steal is > 10%. A noisy neighbor is killing VM performances or a spot instance may be out of credit.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+      - alert: HostOomKillDetected
+        expr: (increase(node_vmstat_oom_kill[1m]) > 0) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}
+        for: 0m
+        labels:
+          severity: warning
+        annotations:
+          summary: Host OOM kill detected (instance {{ $labels.instance }})
+          description: "OOM kill detected\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+      - alert: HostNetworkInterfaceSaturated
+        expr: ((rate(node_network_receive_bytes_total{device!~"^tap.*|^vnet.*|^veth.*|^tun.*"}[1m]) + rate(node_network_transmit_bytes_total{device!~"^tap.*|^vnet.*|^veth.*|^tun.*"}[1m])) / node_network_speed_bytes{device!~"^tap.*|^vnet.*|^veth.*|^tun.*"} > 0.8 < 10000) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}
+        for: 1m
+        labels:
+          severity: warning
+        annotations:
+          summary: Host Network Interface Saturated (instance {{ $labels.instance }})
+          description: "The network interface \"{{ $labels.device }}\" on \"{{ $labels.instance }}\" is getting overloaded.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+

services/monitoring/lgtm/default.nix

@@ -3,5 +3,6 @@
     ./grafana.nix
     ./loki.nix
     ./mimir.nix
+    ./tempo.nix
   ];
-}
+}

services/monitoring/lgtm/grafana.nix

@@ -1,11 +1,16 @@
 {
   config,
   lib,
+  pkgs,
+  inputs,
   ...
 }:
 let
   cfg = config.bagel.services.grafana;
   inherit (lib) mkEnableOption mkIf;
+  generatedJsonnetDashboards = (pkgs.callPackage ../../../dashboards {
+    inherit (inputs) gerrit-dashboard;
+  }).allDashboards;
 in
 {
   options.bagel.services.grafana.enable = mkEnableOption "Grafana frontend";
@@ -16,8 +21,6 @@ in
       owner = "grafana";
     };
 
-    bagel.services.postgres.enable = true;
-
     services = {
       grafana = {
         enable = true;
@@ -86,6 +89,10 @@ in
                 name = "default";
                 options.path = ./dashboards;
               }
+              {
+                name = "jsonnet";
+                options.path = generatedJsonnetDashboards;
+              }
             ];
           };
 
@@ -118,6 +125,21 @@ in
                   implementation = "mimir";
                 };
               }
+              {
+                name = "Pyroscope";
+                type = "grafana-pyroscope-datasource";
+                uid = "pyroscope";
+                access = "proxy";
+                url = "http://127.0.0.1:4040";
+              }
+              {
+                name = "Tempo";
+                type = "tempo";
+                uid = "tempo";
+                access = "proxy";
+                url = "http://127.0.0.1:9190";
+                jsonData.streamingEnabled.search = true;
+              }
             ];
           };
         };
@@ -147,7 +169,7 @@ in
         };
       };
     };
-    
+
     bagel.monitoring.grafana-agent.exporters.grafana.port = 2342;
   };
 }

services/monitoring/lgtm/mimir.nix

@@ -20,6 +20,7 @@ in
         owner = "nginx";
       };
       mimir-environment.file = ../../../secrets/mimir-environment.age;
+      mimir-webhook-url.file = ../../../secrets/mimir-webhook-url.age;
     };
 
     services.mimir = {
@@ -75,6 +76,11 @@ in
             receivers = [
               {
                 name = "matrix";
+                webhook_configs = [{
+                  # Mimir can't expand environment variables in external config files,
+                  # so work around it.
+                  url_file = "/run/credentials/mimir.service/webhook-url";
+                }];
               }
             ];
           };
@@ -91,7 +97,10 @@ in
       # Avoid that by ensuring it starts after the network is set up.
       wants = [ "network-online.target" ];
       after = ["network-online.target"];
-      serviceConfig.EnvironmentFile = [ config.age.secrets.mimir-environment.path ];
+      serviceConfig = {
+        EnvironmentFile = [ config.age.secrets.mimir-environment.path ];
+        LoadCredential = [ "webhook-url:${config.age.secrets.mimir-webhook-url.path}" ];
+      };
     };
 
     services.nginx = {
@@ -111,5 +120,6 @@ in
     };
 
     bagel.monitoring.grafana-agent.exporters.mimir.port = 9009;
+    bagel.services.alertmanager-hookshot-adapter.enable = true;
   };
 }

services/monitoring/lgtm/tempo.nix

@@ -0,0 +1,79 @@
+{
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = config.bagel.services.tempo;
+  inherit (lib) mkEnableOption mkIf;
+in
+{
+  options.bagel.services.tempo.enable = mkEnableOption "Tempo trace store";
+
+  config = mkIf cfg.enable {
+    age.secrets = {
+      metrics-push-htpasswd = {
+        file = ../../../secrets/metrics-push-htpasswd.age;
+        owner = "nginx";
+      };
+      tempo-environment.file = ../../../secrets/tempo-environment.age;
+    };
+
+    services.tempo = {
+      enable = true;
+      extraFlags = ["--config.expand-env=true"];
+      settings = {
+        multitenancy_enabled = false;
+        stream_over_http_enabled = true;
+
+        server = {
+          http_listen_port = 9190;
+          grpc_listen_port = 9195;
+        };
+        distributor.receivers.otlp.protocols.http.endpoint = "127.0.0.1:4138";
+
+        storage.trace = {
+          backend = "s3";
+          s3 = {
+            endpoint = "s3.delroth.net";
+            bucket = "bagel-tempo";
+            secret_key = "\${S3_KEY}"; # This is a secret injected via an environment variable
+            access_key = "\${S3_KEY_ID}";
+          };
+          wal.path = "/var/lib/tempo/traces-wal";
+        };
+
+        metrics_generator.storage = {
+          path = "/var/lib/tempo/metrics-wal";
+          remote_write = [
+            {
+              url = "http://127.0.0.1:9009/api/v1/push";
+            }
+          ];
+        };
+
+        overrides.defaults.metrics_generator.processors = [ "span-metrics" ];
+      };
+    };
+
+    systemd.services.tempo.serviceConfig.EnvironmentFile = [ config.age.secrets.tempo-environment.path ];
+
+    services.nginx = {
+      upstreams.tempo = {
+        servers."${config.services.tempo.settings.distributor.receivers.otlp.protocols.http.endpoint}" = {};
+        extraConfig = "keepalive 16;";
+      };
+
+      virtualHosts."tempo.forkos.org" = {
+        enableACME = true;
+        forceSSL = true;
+        locations."/" = {
+          proxyPass = "http://tempo";
+          basicAuthFile = config.age.secrets.metrics-push-htpasswd.path;
+        };
+      };
+    };
+
+    bagel.monitoring.grafana-agent.exporters.tempo.port = 9190;
+  };
+}

services/monitoring/pyroscope/default.nix

@@ -0,0 +1,74 @@
+{ lib, config, ... }:
+let
+  inherit (lib) mkEnableOption mkIf;
+  cfg = config.bagel.services.pyroscope;
+  pyroscopePort = config.services.pyroscope.settings.server.http_listen_port;
+in
+{
+  options.bagel.services.pyroscope = {
+    enable = mkEnableOption "pyroscope server";
+  };
+
+  # TODO: send me to nixpkgs
+  imports = [
+    ./module.nix
+  ];
+
+  config = mkIf cfg.enable {
+    age.secrets.pyroscope-secrets.file = ../../../secrets/pyroscope-secrets.age;
+    services.nginx = {
+      upstreams.pyroscope = {
+        servers."127.0.0.1:${toString pyroscopePort}" = {};
+        extraConfig = "keepalive 16;";
+      };
+
+      virtualHosts."pyroscope.forkos.org" = {
+        enableACME = true;
+        forceSSL = true;
+        locations."/ingest" = {
+          proxyPass = "http://pyroscope";
+          basicAuthFile = config.age.secrets.metrics-push-htpasswd.path;
+        };
+        locations."/push.v1.PusherService/Push" = {
+          proxyPass = "http://pyroscope";
+          basicAuthFile = config.age.secrets.metrics-push-htpasswd.path;
+        };
+      };
+    };
+    services.pyroscope = {
+      enable = true;
+      secretFile = config.age.secrets.pyroscope-secrets.path;
+      settings = {
+        target = "all";
+        multitenancy_enabled = false;
+
+        api.base-url = "https://pyroscope.forkos.org";
+        analytics.reporting_enabled = false;
+
+        storage = {
+          backend = "s3";
+          s3 = {
+            endpoint = "s3.delroth.net";
+            region = "garage";
+            bucket_name = "bagel-pyroscope";
+            access_key_id = "\${S3_KEY_ID}";
+            secret_access_key = "\${S3_KEY}";
+            force_path_style = true;
+          };
+        };
+        server = {
+          http_listen_port = 4040;
+          grpc_listen_port = 9097;
+          grpc_server_max_recv_msg_size = 104857600;
+          grpc_server_max_send_msg_size = 104857600;
+          grpc_server_max_concurrent_streams = 1000;
+        };
+
+        memberlist = {
+          advertise_port = 7948;
+          bind_port = 7948;
+        };
+      };
+    };
+  };
+}

services/monitoring/pyroscope/module.nix

@@ -0,0 +1,42 @@
+{ pkgs, lib, config, ... }:
+let
+  inherit (lib) mkEnableOption mkPackageOption mkOption types mkIf;
+  settingsFormatYaml = pkgs.formats.yaml { };
+  cfg = config.services.pyroscope;
+  configFile = settingsFormatYaml.generate "settings.yaml" cfg.settings;
+in
+{
+  options.services.pyroscope = {
+    enable = mkEnableOption "pyroscope, a continuous profiling platform";
+    package = mkPackageOption pkgs "pyroscope" { };
+    secretFile = mkOption {
+      type = types.path;
+    };
+    settings = mkOption {
+      description = "Pyroscope settings. See <>";
+
+      type = types.submodule {
+        freeformType = settingsFormatYaml.type;
+      };
+    };
+  };
+
+  config = mkIf cfg.enable {
+    systemd.services.pyroscope = {
+      description = "Pyroscope server - a continuous profiling platform";
+      wantedBy = [ "multi-user.target" ];
+      wants = [ "network-online.target" ];
+      after = [ "network-online.target" ];
+      serviceConfig = {
+        ExecStart = "${cfg.package}/bin/pyroscope -config.file ${configFile} -config.expand-env";
+        WorkingDirectory = "/var/lib/pyroscope";
+        User = "pyroscope";
+        DynamicUser = true;
+        Restart = "on-failure";
+        RuntimeDirectory = "pyroscope";
+        StateDirectory = "pyroscope";
+        EnvironmentFile = [ cfg.secretFile ];
+      };
+    };
+  };
+}

services/newsletter/default.nix

@@ -0,0 +1,43 @@
+{ config, lib, ... }:
+let
+  cfg = config.bagel.newsletter;
+  inherit (lib) mkIf mkOption mkEnableOption types;
+  port = 18999;
+  address = "127.0.0.1:${toString port}";
+in
+{
+  options.bagel.newsletter = {
+    enable = mkEnableOption "the newsletter web service (listmonk)";
+    domain = mkOption {
+      type = types.str;
+    };
+  };
+
+  config = mkIf cfg.enable {
+    age.secrets.newsletter-secrets.file = ../../secrets/newsletter-secrets.age;
+    services.listmonk = {
+      enable = true;
+      secretFile = config.age.secrets.newsletter-secrets.path;
+      settings."app" = {
+        inherit address;
+        admin_username = "admin";
+      };
+      database.createLocally = true;
+    };
+
+    services.nginx.enable = true;
+    services.nginx.virtualHosts."${cfg.domain}" = {
+      enableACME = true;
+      forceSSL = true;
+      locations."/".proxyPass = "http://${address}";
+    };
+
+    users.users.listmonk = {
+      isSystemUser = true;
+      group = "listmonk";
+    };
+    users.groups.listmonk = {};
+
+    networking.firewall.allowedTCPPorts = [ 80 443 ];
+  };
+}

services/postgres/ca.crt

@@ -0,0 +1,11 @@
+-----BEGIN CERTIFICATE-----
+MIIBiDCCAS6gAwIBAgIRAIX2H5zZS7SY7/uXfA59emswCgYIKoZIzj0EAwIwIjEg
+MB4GA1UEAxMXRm9ya09TIFBvc3RncmVzIFJvb3QgQ0EwHhcNMjQwODE2MDUyNzIx
+WhcNMzQwODE0MDUyNzIxWjAiMSAwHgYDVQQDExdGb3JrT1MgUG9zdGdyZXMgUm9v
+dCBDQTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABP7WPT1ooa4Irv1V58Bgqg7t
+S4hXymbps7CyFMwy1gILOazDqh1YmgacofWST1gf0qm9Uo4YgKtWyCdZndWjLqmj
+RTBDMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAGAQH/AgEBMB0GA1UdDgQW
+BBQy4WX/hUExQ/i1h7MvF6Ow2irNizAKBggqhkjOPQQDAgNIADBFAiBnpvX9a4N9
+f0pnG58f8GG/Yu91N2s0eESiPcMjzRB3vwIhAPo6YMFzNrB6IWxiUGtlOni1eY06
+iCsMoQ7B0zTfwBGW
+-----END CERTIFICATE-----

services/postgres/default.nix

@@ -10,6 +10,9 @@ in {
   };
 
   config = lib.mkIf cfg.enable {
+    age.secrets.postgresql-tls-priv.owner = "postgres";
+    age.secrets.postgresql-tls-priv.file = ../../secrets/postgres-tls-priv.age;
+
     systemd.tmpfiles.rules = [
       "d /var/db 0755 root root - -"
       "d /var/db/postgresql 0750 postgres postgres - -"
@@ -21,6 +24,8 @@ in {
       package = pkgs.postgresql_16;
       dataDir = dataDir;
 
+      enableTCPIP = true;
+
       # TODO: Where to put this to properly couple things? It doesn't belong
       # here, but using it in services/hydra would require running on
       # localhost. Probably needs to be replaced with some different way of
@@ -41,12 +46,60 @@ in {
         hydra-users postgres postgres
       '';
       authentication = ''
-        local hydra all ident map=hydra-users
+        local hydra all peer map=hydra-users
+
+        # Allow any connection over TLS with a valid client certificate: signed
+        # by our CA, and with username = cert CN.
+        hostssl all all all cert clientcert=verify-full
       '';
 
       settings = {
         max_connections = 500;
+
+        ssl = true;
+        ssl_ca_file = "${./ca.crt}";
+        ssl_cert_file = "${./server.crt}";
+        ssl_key_file = config.age.secrets.postgresql-tls-priv.path;
       };
     };
+
+    networking.firewall.allowedTCPPorts = [ config.services.postgresql.settings.port ];
+
+    # Provisioned on the server so that CA operations can be done there.
+    age.secrets.postgresql-ca-priv.owner = "postgres";
+    age.secrets.postgresql-ca-priv.file = ../../secrets/postgres-ca-priv.age;
+
+    users.users.postgres.packages = [
+      (pkgs.writeShellScriptBin "postgres-mint-new-client" ''
+        #! ${pkgs.runtimeShell}
+
+        set -eo pipefail
+
+        if [ $# -eq 0 ]; then
+          echo "usage: $0 <pg-username>" >&2
+          exit 1
+        fi
+
+        user=$1
+        step=${pkgs.lib.getExe pkgs.step-cli}
+
+        tmpdir=$(mktemp -d)
+        trap "rm -rf $tmpdir" EXIT
+
+        $step certificate create "$user" "$tmpdir/client.crt" "$tmpdir/client.key" \
+          --profile leaf \
+          --ca ${./ca.crt} \
+          --ca-key ${config.age.secrets.postgresql-ca-priv.path} \
+          --not-after 87660h \
+          --no-password --insecure
+
+        echo "Client certificate:"
+        cat "$tmpdir/client.crt"
+        echo
+
+        echo "Client private key:"
+        cat "$tmpdir/client.key"
+      '')
+    ];
   };
 }

services/postgres/server.crt

@@ -0,0 +1,12 @@
+-----BEGIN CERTIFICATE-----
+MIIB0DCCAXegAwIBAgIQTayv82V9dp0ptyMbDcN70zAKBggqhkjOPQQDAjAiMSAw
+HgYDVQQDExdGb3JrT1MgUG9zdGdyZXMgUm9vdCBDQTAeFw0yNDA4MTYwNTMwMjBa
+Fw0zNDA4MTYxNzMwMjBaMB4xHDAaBgNVBAMTE3Bvc3RncmVzLmZvcmtvcy5vcmcw
+WTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAR7St4dklWkvYbCi+3xPY5YFfBwgErz
+8TLtT5F2l5aKN0+I0sBo+ktiTNl8BzaVzXJmLa2xzRt2jQgB2R0IZgyko4GSMIGP
+MA4GA1UdDwEB/wQEAwIHgDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIw
+HQYDVR0OBBYEFCELKmGcbzVZf6Qx1cWCeXVC3XXJMB8GA1UdIwQYMBaAFDLhZf+F
+QTFD+LWHsy8Xo7DaKs2LMB4GA1UdEQQXMBWCE3Bvc3RncmVzLmZvcmtvcy5vcmcw
+CgYIKoZIzj0EAwIDRwAwRAIgJTM0nO2UaJnlGNIOJ1oT7HClNBxmH5oQu2DwVMuS
+MB8CIDbg/nrYjnRmFGGtbWvLbdvHIZ7t0A4kjkLwJ2QCr6Ni
+-----END CERTIFICATE-----

services/s3-revproxy/default.nix

@@ -0,0 +1,141 @@
+# Originally written by Jade Lovelace for Lix.
+# Adapted for ForkOS.
+{ lib, config, ... }:
+let
+  inherit (lib) mkEnableOption mkIf types mkOption mapAttrs mapAttrs' nameValuePair;
+  cfg = config.bagel.services.s3-revproxy;
+  s3RevproxyPort = 10652;
+  mkTarget = { name, bucket ? name }: {
+    mount = {
+      host = "${name}.${cfg.domain}";
+      path = [ "/" ];
+    };
+    actions.GET = {
+      enabled = true;
+      config = {
+        # e.g. /2.90 will 404, so it will redirect to /2.90/ if it is a directory
+        redirectWithTrailingSlashForNotFoundFile = true;
+        indexDocument = "index.html";
+      };
+    };
+
+    bucket = {
+      name = bucket;
+      region = "garage";
+      s3Endpoint = "https://${cfg.s3.apiUrl}";
+      credentials = {
+        accessKey.env = "AWS_ACCESS_KEY_ID";
+        secretKey.env = "AWS_SECRET_KEY";
+      };
+    };
+  };
+  # Makes a subdomain that gets proxied through s3-proxy to provide directory
+  # listings and reasonable 404 pages.
+  # This is not used on cache, since there a directory listing for cache is a
+  # liability at best.
+  mkProxiedSubdomain = subdomain: {
+    enableACME = true;
+    forceSSL = true;
+    locations."/" = {
+      recommendedProxySettings = true;
+      proxyPass = "http://127.0.0.1:${toString s3RevproxyPort}/";
+    };
+  };
+in
+{
+  options.bagel.services.s3-revproxy = {
+    enable = mkEnableOption "a S3 reverse proxy";
+
+    domain = mkOption {
+      type = types.str;
+    };
+
+    s3 = {
+      apiUrl = mkOption {
+        type = types.str;
+        description = "An URL to the S3 API";
+      };
+    };
+
+    targets = mkOption {
+      type = types.attrsOf types.str;
+      description = ''
+        A mapping between a nice name and a bucket name.
+      '';
+    };
+  };
+
+  imports = [
+    ./module.nix
+  ];
+
+  config = mkIf cfg.enable {
+    age.secrets.s3-revproxy-api-keys.file = ../../secrets/s3-revproxy-api-keys.age;
+    # For each target, generate an entry that passes it to the s3-revproxy.
+    services.nginx.virtualHosts = mapAttrs' (subdomain: _: nameValuePair "${subdomain}.${cfg.domain}" (mkProxiedSubdomain subdomain)) cfg.targets;
+    # this solves garage supporting neither anonymous access nor automatic
+    # directory indexing by simply ignoring garage's web server and replacing it
+    # with overengineered golang instead.
+    services.s3-revproxy = {
+      enable = true;
+      settings = {
+        templates = {
+          helpers = [ ./s3-revproxy-templates/_helpers.tpl ];
+          notFoundError = {
+            headers = {
+              "Content-Type" = "{{ template \"main.headers.contentType\" . }}";
+            };
+            status = "404";
+          };
+          folderList = {
+            path = ./s3-revproxy-templates/folder-list.tpl;
+            headers = {
+              "Content-Type" = "{{ template \"main.headers.contentType\" . }}";
+            };
+            # empty s3 directories are not real and cannot hurt you.
+            # due to redirectWithTrailingSlashForNotFoundFile, garbage file names
+            # get redirected as folders, which then appear as empty, yielding
+            # poor UX.
+            status = ''
+              {{- if eq (len .Entries) 0 -}}
+                404
+              {{- else -}}
+                200
+              {{- end -}}
+            '';
+          };
+        };
+        /* For metrics and debugging (e.g. pulling the config)
+        internalServer = {
+          listenAddr = "127.0.0.1";
+          port = 1337;
+        };
+        */
+        server = {
+          listenAddr = "127.0.0.1";
+          port = s3RevproxyPort;
+
+          # it's going right into nginx, so no point
+          compress.enabled = false;
+          cors = {
+            enabled = true;
+            allowMethods = [ "GET" ];
+            allowOrigins = [ "*" ];
+          };
+
+          cache = {
+            noCacheEnabled = false;
+            # Taken from the original Perl script.
+            # maxage=600: Serve from cache for 5 minutes.
+            # stale-while-revaliadate=1800: Serve from cache while updating in the background for 30 minutes.
+            # https://web.dev/stale-while-revalidate/
+            # https://developer.fastly.com/learning/concepts/cache-freshness/
+            cacheControl = "maxage=600,stale-while-revalidate=1800,public";
+          };
+        };
+        targets = mapAttrs (name: bucket: mkTarget { inherit name bucket; }) cfg.targets;
+      };
+      environmentFile = config.age.secrets.s3-revproxy-api-keys.path;
+    };
+  };
+}

services/s3-revproxy/module.nix

@@ -0,0 +1,69 @@
+# Originally, written by Jade Lovelace for Lix.
+{ config, pkgs, lib, ... }:
+let cfg = config.services.s3-revproxy;
+  settingsGenerator = pkgs.formats.yaml {};
+  # Needs to be in a directory, so we might as well implement autoreload, why not!
+  configFile = settingsGenerator.generate "config.yaml" cfg.settings;
+
+  inherit (lib) types;
+in
+{
+  options.services.s3-revproxy = {
+    enable = lib.mkEnableOption "s3 reverse proxy";
+    package = lib.mkPackageOption pkgs "s3-revproxy" {};
+    settings = lib.mkOption {
+      default = { };
+      type = settingsGenerator.type;
+      description = ''
+        Settings to use for the service. See the documentation at https://oxyno-zeta.github.io/s3-proxy/configuration/structure/
+      '';
+    };
+
+    environmentFile = lib.mkOption {
+      type = types.nullOr types.path;
+      default = null;
+      description = ''
+        Environment file to use for s3-revproxy.
+      '';
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    environment.etc."s3-revproxy/config.yaml".source = configFile;
+    systemd.services.s3-revproxy = {
+      wantedBy = [ "multi-user.target" ];
+
+      serviceConfig = {
+        ExecStart = "${lib.getExe cfg.package} --config /etc/s3-revproxy";
+
+        DynamicUser = true;
+        CapabilityBoundingSet = "";
+        NoNewPrivileges = true;
+        PrivateTmp = true;
+        PrivateUsers = true;
+        PrivateDevices = true;
+        ProtectHome = true;
+        ProtectClock = true;
+        ProtectProc = "noaccess";
+        ProcSubset = "pid";
+        UMask = "0077";
+        ProtectKernelLogs = true;
+        ProtectKernelModules = true;
+        ProtectKernelTunables = true;
+        ProtectControlGroups = true;
+        ProtectHostname = true;
+        RestrictSUIDSGID = true;
+        RestrictRealtime = true;
+        RestrictNamespaces = true;
+        LockPersonality = true;
+        RemoveIPC = true;
+        SystemCallFilter = [ "@system-service" "~@privileged" ];
+        RestrictAddressFamilies = [ "AF_INET" "AF_INET6" ];
+        MemoryDenyWriteExecute = true;
+        SystemCallArchitectures = "native";
+
+        EnvironmentFile = lib.optionals (cfg.environmentFile != null) [ cfg.environmentFile ];
+      };
+    };
+  };
+}

services/s3-revproxy/s3-revproxy-templates/_helpers.tpl

@@ -0,0 +1,37 @@
+{{/* SPDX-License-Identifier: Apache-2.0 */}}
+{{/* SPDX-FileCopyrightText: 2024 s3-proxy contributors */}}
+
+{{- /* This function will allow to get user identifier. */ -}}
+{{- define "main.userIdentifier" -}}
+{{- if .User -}}
+{{- .User.GetIdentifier -}}
+{{- end -}}
+{{- end -}}
+
+
+{{- /* This function will allow to get the content type header from "Accept" header */ -}}
+{{- define "main.headers.contentType" -}}
+{{- if contains "application/json" (.Request.Header.Get "Accept") -}}
+application/json; charset=utf-8
+{{- else -}}
+text/html; charset=utf-8
+{{- end -}}
+{{- end -}}
+
+{{- /* This will forge the json output of an error */ -}}
+{{- define "main.body.errorJsonBody" -}}
+{"error": {{ .Error.Error | toJson }}}
+{{- end -}}
+
+{{- define "notFoundErrorBody" -}}
+  {{- if contains "application/json" (.Request.Header.Get "Accept") -}}
+  {{ template "main.body.errorJsonBody" . }}
+  {{- else -}}
+  <!DOCTYPE html>
+  <html>
+    <body>
+      <h1>Not Found {{ .Request.URL.Path }}</h1>
+    </body>
+  </html>
+  {{- end -}}
+{{- end -}}

services/s3-revproxy/s3-revproxy-templates/folder-list.tpl

@@ -0,0 +1,52 @@
+{{/* SPDX-License-Identifier: Apache-2.0 */}}
+{{/* SPDX-FileCopyrightText: 2024 s3-proxy contributors */}}
+{{- $root := . -}}
+{{- if eq (len .Entries) 0 -}}
+  {{- include "notFoundErrorBody" . -}}
+{{- else -}}
+
+{{- if contains "application/json" (.Request.Header.Get "Accept") -}}
+[
+  {{- $maxLen := len $root.Entries -}}
+  {{- range $index, $entry := $root.Entries -}}
+  {"name": {{ $entry.Name | toJson -}}
+    ,"etag": {{ $entry.ETag | toJson -}}
+    ,"type": {{ $entry.Type | toJson -}}
+    ,"size": {{ $entry.Size | toJson -}}
+    ,"path": {{ $entry.Path | toJson -}}
+    ,"lastModified": {{ $entry.LastModified | date "2006-01-02T15:04:05Z07:00" | toJson -}}
+  }{{- if ne $index (sub $maxLen 1) -}},{{- end -}}
+  {{- end -}}
+]
+{{- else -}}
+  <!DOCTYPE html>
+  <html>
+    <body>
+      <h1>Index of {{ .Request.URL.Path }}</h1>
+      <table style="width:100%">
+          <thead>
+              <tr>
+                  <th style="border-right:1px solid black;text-align:start">Entry</th>
+                  <th style="border-right:1px solid black;text-align:start">Size</th>
+                  <th style="border-right:1px solid black;text-align:start">Last modified</th>
+              </tr>
+          </thead>
+          <tbody style="border-top:1px solid black">
+            <tr>
+              <td style="border-right:1px solid black;padding: 0 5px"><a href="..">..</a></td>
+              <td style="border-right:1px solid black;padding: 0 5px"> - </td>
+              <td style="padding: 0 5px"> - </td>
+            </tr>
+          {{- range .Entries }}
+            <tr>
+                <td style="border-right:1px solid black;padding: 0 5px"><a href="{{ .Path }}">{{ .Name }}</a></td>
+                <td style="border-right:1px solid black;padding: 0 5px">{{- if eq .Type "FOLDER" -}} - {{- else -}}{{ .Size | humanSize }}{{- end -}}</td>
+                <td style="padding: 0 5px">{{ .LastModified }}</td>
+            </tr>
+          {{- end }}
+          </tbody>
+      </table>
+    </body>
+  </html>
+  {{- end -}}
+{{- end -}}

services/s3-revproxy/s3-revproxy-templates/not-found-error.tpl

@@ -0,0 +1,3 @@
+{{/* SPDX-License-Identifier: Apache-2.0 */}}
+{{/* SPDX-FileCopyrightText: 2024 s3-proxy contributors */}}
+{{- include "notFoundErrorBody" . -}}

terraform/default.nix

@@ -2,6 +2,7 @@
   imports = [
     ./common.nix
     ./gandi.nix
+    ./dnsimple.nix
     ./hydra.nix
     ./state.nix
   ];

terraform/dnsimple.nix

@@ -0,0 +1,147 @@
+{ lib, config, ... }:
+let
+  inherit (lib) mkEnableOption mkIf tf genList;
+  cfg = config.bagel.dnsimple;
+in
+{
+  options.bagel.dnsimple = {
+    enable = mkEnableOption "the DNSimple configuration";
+  };
+
+  config = mkIf cfg.enable {
+    terraform.required_providers.dnsimple = {
+      version = "~> 1.7.0";
+      source = "dnsimple/dnsimple";
+    };
+
+    resource.secret_resource.dnsimple_token.lifecycle.prevent_destroy = true;
+    resource.secret_resource.dnsimple_account.lifecycle.prevent_destroy = true;
+
+    provider.dnsimple = {
+      token = tf.ref "resource.secret_resource.dnsimple_token.value";
+      account = tf.ref "resource.secret_resource.dnsimple_account.value";
+    };
+
+    resource.dnsimple_zone.forkos_org = {
+      name = "forkos.org";
+    };
+
+    resource.dnsimple_zone.fleurixos_org = {
+      name = "fleurixos.org";
+    };
+
+    resource.dnsimple_zone.floral_systems = {
+      name = "floral.systems";
+    };
+
+    resource.dnsimple_zone.flowery_systems = {
+      name = "flowery.systems";
+    };
+
+    resource.dnsimple_zone.petalpkgs_org = {
+      name = "petalpkgs.org";
+    };
+
+    resource.dnsimple_zone.vzfdfp_de = {
+      name = "vzfdfp.de";
+    };
+
+    resource.dnsimple_zone_record = let
+      # https://registry.terraform.io/providers/dnsimple/dnsimple/latest/docs/resources/zone_record
+      canonicalName = zoneName: record: let
+        # TODO: make less fragile and have actual unique and stable names
+        normalize = builtins.replaceStrings ["." "@"] ["_" "_root_"];
+        zone = normalize zoneName;
+        name = normalize record.name;
+      in "${zone}_${record.type}_${name}";
+
+      record = name: ttl: type: value: {
+        inherit name ttl type value;
+      };
+
+      proxyRecords = name: ttl: type: value: [
+        # kurisu.lahfa.xyz running a sniproxy:
+        (record name ttl "A" "163.172.69.160")
+        (record name ttl type value)
+      ];
+
+      # Creates a extra *.p record pointing to the sniproxy
+      dualProxyRecords = name: ttl: type: value: lib.flatten [
+        (record name ttl type value)
+        (proxyRecords "${name}.p" ttl type value)
+      ];
+
+      domain = zoneName: records:
+        builtins.listToAttrs (map (record: {
+          name = canonicalName zoneName record;
+          value = record // {
+            zone_name = zoneName;
+          };
+        }
+      ) (lib.flatten records));
+      zones = domains: lib.zipAttrs (lib.mapAttrsToList (zoneName: records: domain zoneName records) domains);
+    in zones {
+        "forkos.org" = ([
+          # (record "@" 300 "A" "163.172.69.160")
+          (record "@" 300 "AAAA" "2001:bc8:38ee:100:1000::20")
+
+          (dualProxyRecords "bagel-box.infra" 300 "AAAA" "2001:bc8:38ee:100:100::1")
+          (dualProxyRecords "gerrit01.infra" 300 "AAAA" "2001:bc8:38ee:100:1000::10")
+          (dualProxyRecords "meta01.infra" 300 "AAAA" "2001:bc8:38ee:100:1000::20")
+          (dualProxyRecords "fodwatch.infra" 300 "AAAA" "2001:bc8:38ee:100:1000::30")
+          # git.infra.forkos.org exposes opensshd
+          (dualProxyRecords "git.infra" 300 "AAAA" "2001:bc8:38ee:100:1000::41")
+          # git.p.forkos.org exposes forgejo ssh server.
+          (proxyRecords "git.p" 300 "AAAA" "2001:bc8:38ee:100:1000::40")
+          (dualProxyRecords "buildbot.infra" 300 "AAAA" "2001:bc8:38ee:100:1000::50")
+          (dualProxyRecords "public01.infra" 300 "AAAA" "2001:bc8:38ee:100:1000::60")
+
+          (record "cl" 300 "CNAME" "gerrit01.infra.p.forkos.org")
+          (record "fodwatch" 300 "CNAME" "fodwatch.infra.p.forkos.org")
+          # git.p.forkos.org is the proxy variant of the Forgejo server.
+          (record "git" 300 "CNAME" "git.p.forkos.org")
+          (record "netbox" 300 "CNAME" "meta01.infra.p.forkos.org")
+          (record "amqp" 300 "CNAME" "bagel-box.infra.p.forkos.org")
+          (record "grafana" 300 "CNAME" "meta01.infra.p.forkos.org")
+          (record "hydra" 300 "CNAME" "build-coord.wob01.infra.p.forkos.org")
+          (record "loki" 300 "CNAME" "meta01.infra.p.forkos.org")
+          (record "mimir" 300 "CNAME" "meta01.infra.p.forkos.org")
+          (record "pyroscope" 300 "CNAME" "meta01.infra.p.forkos.org")
+          (record "tempo" 300 "CNAME" "meta01.infra.p.forkos.org")
+          (record "matrix" 300 "CNAME" "meta01.infra.p.forkos.org")
+          (record "alerts" 300 "CNAME" "meta01.infra.p.forkos.org")
+          (record "buildbot" 300 "CNAME" "buildbot.infra.p.forkos.org")
+          (record "b" 300 "CNAME" "public01.infra.p.forkos.org")
+          (record "postgres" 300 "CNAME" "bagel-box.infra.p.forkos.org")
+          (record "news" 3600 "CNAME" "public01.infra.p.forkos.org")
+
+          # S3 in delroth's basement
+          (record "cache" 300 "AAAA" "2a02:168:6426::12") # smol.delroth.net
+          (record "cache" 300 "A" "195.39.247.161") # sni proxy
+
+          (record "vpn-gw.wob01.infra" 300 "AAAA" "2a01:584:11::2")
+
+          (dualProxyRecords "build-coord.wob01.infra" 300 "AAAA" "2a01:584:11::1:11")
+          # TODO: do not hardcode, just reuse the Colmena hive module outputs to generate all the required details.
+        ]
+        ++ (map (index: record "builder-${toString index}.wob01.infra" 300 "AAAA" "2a01:584:11::1:${toString index}") (genList lib.id 11))
+        ++ (
+          let
+            # FIXME: figure out a way to poke `config.services.s3-revproxy` and
+            # automate the DNS part away?
+            buckets = [
+              "channels"
+              "releases"
+              "channel-scripts-test"
+            ];
+          in
+          map (bucket: record "${bucket}" 300 "CNAME" "public01.infra.p.forkos.org") buckets
+        ));
+        "flowery.systems" = [
+          (record "" 300 "ALIAS" "news.forkos.org")
+        ];
+        "vzfdfp.de" = [
+        ];
+    };
+  };
+}

terraform/gandi.nix

@@ -56,39 +56,60 @@ in
         }) (lib.flatten records));
 
     in forkosRecords ([
-      # (record "@" 3600 "A" ["163.172.69.160"])
-      (record "@" 3600 "AAAA" ["2001:bc8:38ee:100:1000::20"])
+      # (record "@" 300 "A" ["163.172.69.160"])
+      (record "@" 300 "AAAA" ["2001:bc8:38ee:100:1000::20"])
 
-      (dualProxyRecords "bagel-box.infra" 3600 "AAAA" ["2001:bc8:38ee:100:100::1"])
-      (dualProxyRecords "gerrit01.infra" 3600 "AAAA" ["2001:bc8:38ee:100:1000::10"])
-      (dualProxyRecords "meta01.infra" 3600 "AAAA" ["2001:bc8:38ee:100:1000::20"])
-      (dualProxyRecords "fodwatch.infra" 3600 "AAAA" ["2001:bc8:38ee:100:1000::30"])
+      (dualProxyRecords "bagel-box.infra" 300 "AAAA" ["2001:bc8:38ee:100:100::1"])
+      (dualProxyRecords "gerrit01.infra" 300 "AAAA" ["2001:bc8:38ee:100:1000::10"])
+      (dualProxyRecords "meta01.infra" 300 "AAAA" ["2001:bc8:38ee:100:1000::20"])
+      (dualProxyRecords "fodwatch.infra" 300 "AAAA" ["2001:bc8:38ee:100:1000::30"])
       # git.infra.forkos.org exposes opensshd
-      (dualProxyRecords "git.infra" 3600 "AAAA" ["2001:bc8:38ee:100:1000::41"])
+      (dualProxyRecords "git.infra" 300 "AAAA" ["2001:bc8:38ee:100:1000::41"])
       # git.p.forkos.org exposes forgejo ssh server.
-      (proxyRecords "git.p" 3600 "AAAA" ["2001:bc8:38ee:100:1000::40"])
-      (dualProxyRecords "buildbot.infra" 3600 "AAAA" ["2001:bc8:38ee:100:1000::50"])
-      (dualProxyRecords "public01.infra" 3600 "AAAA" ["2001:bc8:38ee:100:1000::60"])
+      (proxyRecords "git.p" 300 "AAAA" ["2001:bc8:38ee:100:1000::40"])
+      (dualProxyRecords "buildbot.infra" 300 "AAAA" ["2001:bc8:38ee:100:1000::50"])
+      (dualProxyRecords "public01.infra" 300 "AAAA" ["2001:bc8:38ee:100:1000::60"])
 
-      (record "cl" 3600 "CNAME" ["gerrit01.infra.p"])
-      (record "fodwatch" 3600 "CNAME" ["fodwatch.infra.p"])
+      (record "cl" 300 "CNAME" ["gerrit01.infra.p"])
+      (record "fodwatch" 300 "CNAME" ["fodwatch.infra.p"])
       # git.p.forkos.org is the proxy variant of the Forgejo server.
-      (record "git" 3600 "CNAME" ["git.p"])
-      (record "netbox" 3600 "CNAME" ["meta01.infra.p"])
-      (record "amqp" 3600 "CNAME" ["bagel-box.infra.p"])
-      (record "grafana" 3600 "CNAME" ["meta01.infra.p"])
-      (record "hydra" 3600 "CNAME" ["bagel-box.infra.p"])
-      (record "loki" 3600 "CNAME" ["meta01.infra.p"])
-      (record "mimir" 3600 "CNAME" ["meta01.infra.p"])
-      (record "matrix" 3600 "CNAME" ["meta01.infra.p"])
-      (record "buildbot" 3600 "CNAME" ["buildbot.infra.p"])
-      (record "b" 3600 "CNAME" ["public01.infra.p"])
+      (record "git" 300 "CNAME" ["git.p"])
+      (record "netbox" 300 "CNAME" ["meta01.infra.p"])
+      (record "amqp" 300 "CNAME" ["bagel-box.infra.p"])
+      (record "grafana" 300 "CNAME" ["meta01.infra.p"])
+      (record "hydra" 300 "CNAME" ["build-coord.wob01.infra.p"])
+      (record "loki" 300 "CNAME" ["meta01.infra.p"])
+      (record "mimir" 300 "CNAME" ["meta01.infra.p"])
+      (record "pyroscope" 300 "CNAME" ["meta01.infra.p"])
+      (record "tempo" 300 "CNAME" ["meta01.infra.p"])
+      (record "matrix" 300 "CNAME" ["meta01.infra.p"])
+      (record "alerts" 300 "CNAME" ["meta01.infra.p"])
+      (record "buildbot" 300 "CNAME" ["buildbot.infra.p"])
+      (record "b" 300 "CNAME" ["public01.infra.p"])
+      (record "postgres" 300 "CNAME" ["bagel-box.infra.p"])
+      (record "news" 3600 "CNAME" ["public01.infra.p"])
 
       # S3 in delroth's basement
-      (record "cache" 3600 "CNAME" ["smol.delroth.net."])
+      (record "cache" 300 "AAAA" ["2a02:168:6426::12"]) # smol.delroth.net
+      (record "cache" 300 "A" ["195.39.247.161"]) # sni proxy
 
-      (record "vpn-gw.wob01.infra" 3600 "AAAA" [ "2a01:584:11::2" ])
+      (record "vpn-gw.wob01.infra" 300 "AAAA" [ "2a01:584:11::2" ])
+
+      (dualProxyRecords "build-coord.wob01.infra" 300 "AAAA" [ "2a01:584:11::1:11" ])
       # TODO: do not hardcode, just reuse the Colmena hive module outputs to generate all the required details.
-    ] ++ map (index: record "builder-${toString index}.wob01.infra" 3600 "AAAA" [ "2a01:584:11::1:${toString index}" ]) (genList lib.id 12));
+    ]
+    ++ (map (index: record "builder-${toString index}.wob01.infra" 300 "AAAA" [ "2a01:584:11::1:${toString index}" ]) (genList lib.id 11))
+    ++ (
+      let
+        # FIXME: figure out a way to poke `config.services.s3-revproxy` and
+        # automate the DNS part away?
+        buckets = [
+          "channels"
+          "releases"
+          "channel-scripts-test"
+        ];
+      in
+      map (bucket: record "${bucket}" 300 "CNAME" [ "public01.infra.p" ]) buckets
+    ));
   };
 }

terraform/hydra.nix

@@ -33,7 +33,7 @@ in
     };
 
     resource.hydra_jobset.k900-experiments = {
-      project = config.resource.hydra_project.forkos.name;
+      project = tf.ref "resource.hydra_project.forkos.name";
       state = "enabled";
       visible = true;
       name = "nixpkgs-experiments";
@@ -55,7 +55,7 @@ in
         {
           name = "nixpkgs";
           type = "git";
-          value = "https://github.com/nixos/nixpkgs 03ff49192b044786362c8c94d8501eac5c6eada4";
+          value = "https://github.com/K900/nixpkgs 467efb1055fa3c0f49d6b6a4505a4c662a836f68";
           notify_committers = false;
         }
         {
@@ -72,7 +72,7 @@ in
     };
 
     resource.hydra_jobset.raito-nixos-rolling-small = {
-      project = config.resource.hydra_project.forkos.name;
+      project = tf.ref "resource.hydra_project.forkos.name";
       state = "enabled";
       visible = true;
       name = "raito-nixos-rolling-small";
@@ -112,7 +112,7 @@ in
     };
 
     resource.hydra_jobset.delroth-nixpkgs-staging-small = {
-      project = config.resource.hydra_project.forkos.name;
+      project = tf.ref "resource.hydra_project.forkos.name";
       state = "enabled";
       visible = true;
       name = "delroth-nixpkgs-staging-small";
@@ -162,7 +162,7 @@ in
     };
 
     resource.hydra_jobset.infra_main = {
-      project = config.resource.hydra_project.infra.name;
+      project = tf.ref "resource.hydra_project.infra.name";
       state = "enabled";
       visible = true;
       name = "main";
@@ -189,7 +189,7 @@ in
     };
 
     resource.hydra_jobset.hydra_main = {
-      project = config.resource.hydra_project.hydra.name;
+      project = tf.ref "resource.hydra_project.hydra.name";
       state = "enabled";
       visible = true;
       name = "main";
@@ -206,7 +206,7 @@ in
     };
 
     resource.hydra_jobset.nixos-staging-next-small = {
-      project = config.resource.hydra_project.forkos.name;
+      project = tf.ref "resource.hydra_project.forkos.name";
       state = "enabled";
       visible = true;
       name = "nixos-staging-next-small";
@@ -246,7 +246,7 @@ in
     };
 
     resource.hydra_jobset.nixpkgs-staging-next = {
-      project = config.resource.hydra_project.forkos.name;
+      project = tf.ref "resource.hydra_project.forkos.name";
       state = "enabled";
       visible = true;
       name = "nixpkgs-staging-next";
@@ -286,7 +286,7 @@ in
     };
 
     resource.hydra_jobset.nixos-main = {
-      project = config.resource.hydra_project.forkos.name;
+      project = tf.ref "resource.hydra_project.forkos.name";
       state = "enabled";
       visible = true;
       name = "nixos-main";