fix(forgejo): lower cache.last_commit TTL to limit size of the cache

We really don't want to cache them for a year, which is the default.

Yes, computing them may be expensive, but not worth a multi-gigabyte
redis database that takes minutes to load into RAM on service (re)start.

.envrc

@@ -1,2 +1,11 @@
+#!/usr/bin/env bash
+# the shebang is ignored, but nice for editors
+
 # shellcheck shell=bash
-use flake
+if type -P lorri &>/dev/null; then
+  eval "$(lorri direnv --flake .)"
+else
+  echo 'while direnv evaluated .envrc, could not find the command "lorri" [https://github.com/nix-community/lorri]'
+  use flake
+fi
+

README.md

@@ -1 +1,32 @@
-Infrastructure for the donut shaped thing that is absolutely not a donut.
+# Infrastructure for the donut shaped thing that is absolutely not a donut.
+
+## Quick start
+
+### Build the infrastructure
+
+```
+$ colmena build --on @localboot
+```
+
+Notice that `@localboot` is load-bearing as we have some machines that _cannot be_ deployed with vanilla Colmena. Fixing this is welcome.
+
+### Recommended deploy process
+
+```
+$ colmena apply dry-activate $machine # Verify that the nvd log is reasonable.
+$ colmena apply $machine
+```
+
+### Recommended upgrade process
+
+```
+$ nix flake update
+$ colmena apply dry-activate --on @localboot # Verify that the nvd log is reasonable. Run it twice to get only NVD logs shown.
+$ colmena apply --on @localboot
+```
+
+## Troubleshooting
+
+### I failed to deploy `gerrit01`
+
+Our Gerrit source build is known to have some hiccups sometimes, we are always interested in build logs, feel free to attach information in a new issue so we can make it more reliable.

common/admins.nix

@@ -1,13 +1,47 @@
+{ lib, ... }:
 let
-  keys = import ./ssh-keys.nix;
-in {
-  users.users.root.openssh.authorizedKeys.keys = 
-    keys.users.delroth ++
-    keys.users.k900 ++ 
-    keys.users.raito ++
-    keys.users.maxine ++ 
-    keys.users.jade ++
-    keys.users.janik ++
-    keys.users.lukegb ++
-    keys.users.yuka;
+  inherit (lib) genAttrs;
+in
+# Note: to add somefew in this list.
+# Ensure their SSH key is already in common/ssh-keys.nix with
+# the same username for here, so that the keys is automatically added.
+{
+  bagel.groups = {
+    floral-infra.members = [
+      "delroth"
+      "emilylange"
+      "hexchen"
+      "jade"
+      "janik"
+      "k900"
+      "maxine"
+      "raito"
+      "thubrecht"
+      "winter"
+      "yuka"
+      "ckie"
+    ];
+
+    lix-infra.members = [
+      "raito"
+      "hexchen"
+      "jade"
+      "pennae"
+    ];
+  };
+  bagel.users = genAttrs [
+    "delroth"
+    "emilylange"
+    "hexchen"
+    "jade"
+    "janik"
+    "k900"
+    "maxine"
+    "raito"
+    "thubrecht"
+    "winter"
+    "yuka"
+    "ckie"
+    "pennae"
+  ] (name: {});
 }

common/base-server.nix

@@ -1,7 +1,13 @@
 { lib, pkgs, ... }: {
+  imports = [
+    ./known-ssh-keys.nix
+    ./cgroups.nix
+  ];
+
   nixpkgs.overlays = import ../overlays;
 
   nix.package = lib.mkDefault pkgs.lix;
+  system.tools.nixos-option.enable = false;
   services.openssh.enable = lib.mkForce true;
 
   networking.nftables.enable = true;
@@ -25,8 +31,8 @@
   nix.gc = {
     automatic = true;
     persistent = true;
-    dates = "daily";
-    options = "--delete-older-than 30d";
+    dates = lib.mkDefault "daily";
+    options = lib.mkDefault "--delete-older-than 30d";
   };
 
   services.journald.extraConfig = "SystemMaxUse=512M";
@@ -51,4 +57,19 @@
     "en_US.UTF-8/UTF-8"
     "fr_FR.UTF-8/UTF-8"
   ];
+
+  time.timeZone = "UTC";
+
+  security.acme.acceptTerms = true;
+  security.acme.defaults.email = "infra@forkos.org";
+
+  # Enable system diffs.
+  system.activationScripts.system-diff = {
+    supportsDryActivation = true; # safe: only outputs to stdout
+    text = ''
+      if [ -e /run/current-system ]; then
+        PATH=$PATH:${pkgs.nix}/bin ${pkgs.nvd}/bin/nvd diff /run/current-system $systemConfig
+      fi
+    '';
+  };
 }

common/cgroups.nix

@@ -0,0 +1,83 @@
+# Relatively inspired by fbtax2:
+# https://facebookmicrosites.github.io/cgroup2/docs/fbtax-results.html
+#
+# See also the Chris Down talk at LISA'21:
+# https://www.usenix.org/conference/lisa21/presentation/down
+{ ... }:
+let
+  systemCriticalSliceConfig = {
+    ManagedOOMMemoryPressure = "kill";
+
+    # guarantee availability of memory
+    MemoryMin = "192M";
+    # default 100
+    IOWeight = 1000;
+    # default 100
+    CPUWeight = 1000;
+  };
+in
+{
+  systemd.oomd = {
+    enable = true;
+    # why not, we have cgroups at user level now so it'll just kill the
+    # terminal
+    enableRootSlice = true;
+    enableSystemSlice = true;
+    enableUserSlices = true;
+  };
+
+  systemd.enableCgroupAccounting = true;
+
+  systemd.services.nix-daemon = {
+    serviceConfig = {
+      # FIXME: how do i deprioritize this for memory
+      CPUWeight = 10;
+      IOWeight = 10;
+    };
+  };
+
+  systemd.slices.hostcritical = {
+    description = "Ensures that services to keep the system alive remain alive";
+
+    unitConfig = {
+      # required to avoid a dependency cycle on systemd-oomd. systemd will
+      # actually guess this right but we should fix it anyway.
+      DefaultDependencies = false;
+    };
+
+    sliceConfig = systemCriticalSliceConfig;
+  };
+
+  # make root logins higher priority for resources
+  systemd.slices."user-0" = {
+    sliceConfig = systemCriticalSliceConfig;
+  };
+
+
+  systemd.slices.system = {
+    sliceConfig = {
+      ManagedOOMMemoryPressure = "kill";
+      ManagedOOMMemoryPressureLimit = "50%";
+
+      IOWeight = 100;
+    };
+  };
+
+  systemd.services.sshd = {
+    serviceConfig = {
+      Slice = "hostcritical.slice";
+    };
+  };
+
+  systemd.services.systemd-oomd = {
+    serviceConfig = {
+      Slice = "hostcritical.slice";
+    };
+  };
+
+  systemd.services.systemd-journald = {
+    serviceConfig = {
+      Slice = "hostcritical.slice";
+    };
+  };
+}

common/channels.nix

@@ -0,0 +1,32 @@
+# Taken from https://github.com/NixOS/infra/blob/master/channels.nix
+{
+  # "Channel name" = {
+  #   # This should be the <value> part of
+  #   # https://hydra.forkos.org/job/<value>/latest-finished
+  #   job = "project/jobset/jobname";
+  #
+  #   # When adding a new version, determine if it needs to be tagged as a
+  #   # variant -- for example:
+  #   # nixos-xx.xx         => primary
+  #   # nixos-xx.xx-small   => small
+  #   # nixos-xx.xx-darwin  => darwin
+  #   # nixos-xx.xx-aarch64 => aarch64
+  #   variant = "primary";
+  #
+  #   # Channel Status:
+  #   # '*-unstable' channels are always "rolling"
+  #   # Otherwise a release generally progresses through the following phases:
+  #   #
+  #   #  - Directly after branch off                   => "beta"
+  #   #  - Once the channel is released                => "stable"
+  #   #  - Once the next channel is released           => "deprecated"
+  #   #  - N months after the next channel is released => "unmaintained"
+  #   #    (check the release notes for when this should happen)
+  #   status = "beta";
+  # };
+  "forkos-unstable" = {
+    job = "forkos/nixos-main/tested";
+    variant = "primary";
+    status = "rolling";
+  };
+}

common/default.nix

@@ -1,11 +1,14 @@
 {
   imports = [
     ./admins.nix
+    ./server-acl.nix
     ./base-server.nix
     ./hardening.nix
     ./nix.nix
     ./raito-proxy-aware-nginx.nix
-    ./raito-vm.nix
     ./sysadmin
+    ./hardware
+    ./zsh.nix
+    ./secrets.nix
   ];
 }

common/hardware/default.nix

@@ -0,0 +1,7 @@
+{ ... }: {
+  imports = [
+    ./raito-vm.nix
+    ./oracle-vm.nix
+    ./hetzner.nix
+  ];
+}

common/hardware/hetzner.nix

@@ -0,0 +1,76 @@
+
+{ lib, config, ... }:
+let
+  cfg = config.bagel.hardware.hetzner;
+  inherit (lib) mkEnableOption mkIf mkOption types;
+in
+{
+  options.bagel.hardware.hetzner = {
+    enable = mkEnableOption "Hetzner's hardware defaults";
+
+    platformType = mkOption {
+      # Only VMs are supported.
+      type = types.enum [ "virtual-machine" ];
+    };
+
+    system = mkOption {
+      # Only the aarch64-linux VM Hetzner is supported.
+      type = types.enum [ "aarch64-linux" ];
+    };
+
+    networking.wan = {
+      mac = mkOption {
+        type = types.str;
+        description = "MAC address of the WAN interface in the Hetzner machine";
+      };
+      address = mkOption {
+        type = types.listOf types.str;
+        description = "List of static addresses attached to the WAN interface";
+      };
+    };
+  };
+
+  config = mkIf cfg.enable {
+    # A bunch of stuff is virtio.
+    boot.initrd.availableKernelModules = [
+      "xhci_pci"
+      "usbhid"
+      "sr_mod"
+      "virtio_gpu"
+      "virtio_scsi"
+      "virtio_rng"
+      "virtio_pci"
+    ];
+
+    boot.loader.systemd-boot.enable = true;
+    boot.loader.efi.canTouchEfiVariables = true;
+
+    networking.useDHCP = lib.mkDefault false;
+
+    # Stolen from the netplan provided by aarch64 Ubuntu images.
+    systemd.network.enable = true;
+    systemd.network.links."10-wan" = {
+      linkConfig.Name = "wan";
+      matchConfig.MACAddress = cfg.networking.mac;
+    };
+    systemd.network.networks."10-wan" = {
+      matchConfig.Name = "wan";
+      networkingConfig.Address = cfg.networking.address;
+      linkConfig.RequiredForOnline = true;
+      DHCP = "ipv4";
+      routes = [
+        {
+          routeConfig = {
+            Destination = "::/0";
+            GatewayOnLink = true;
+            Gateway = "fe80::1";
+          };
+        }
+      ];
+      dhcpV4Config = {
+        RouteMetric = 100;
+        UseMTU = true;
+      };
+    };
+  };
+}

common/hardware/oracle-vm.nix

@@ -0,0 +1,52 @@
+
+{ lib, config, modulesPath, ... }:
+let
+  cfg = config.bagel.hardware.oracle-vm;
+  inherit (lib) mkEnableOption mkIf mkOption types;
+in
+{
+  options.bagel.hardware.oracle-vm = {
+    enable = mkEnableOption "Oracle's VM hardware defaults";
+
+    system = mkOption {
+      # Only the free Oracle VMs are supported.
+      type = types.enum [ "aarch64-linux" ];
+    };
+  };
+
+  # Imports a bunch of virtio modules.
+  imports = [
+    "${modulesPath}/profiles/qemu-guest.nix"
+  ];
+
+  config = mkIf cfg.enable {
+    boot.loader.systemd-boot.enable = true;
+    boot.loader.efi.canTouchEfiVariables = true;
+    boot.initrd.systemd.enable = true;
+
+    boot.initrd.availableKernelModules = [
+      "xhci_pci" "virtio_pci" "usbhid" "sr_mod"
+    ];
+    boot.initrd.kernelModules = [ ];
+    boot.kernelModules = [ ];
+    boot.extraModulePackages = [ ];
+
+    nixpkgs.hostPlatform = cfg.system;
+
+    # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+    # (the default) this is the recommended approach. When using systemd-networkd it's
+    # still possible to use this option, but it's recommended to use it in conjunction
+    # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+    networking.useDHCP = lib.mkDefault false;
+    # Examples:
+    # 2: enp0s3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
+    #  link/ether 02:00:17:00:91:6e brd ff:ff:ff:ff:ff:ff
+    #  inet 10.0.0.94/24 brd 10.0.0.255 scope global dynamic noprefixroute enp0s3
+    #     valid_lft 44162sec preferred_lft 33362sec
+    #  inet6 fe80::17ff:fe00:916e/64 scope link
+    #     valid_lft forever preferred_lft forever
+    # [root@build02-aarch64-lahfa:~]# ip r
+    # default via 10.0.0.1 dev enp0s3 proto dhcp src 10.0.0.94 metric 1002 mtu 9000
+    networking.interfaces.enp0s3.useDHCP = lib.mkDefault true;
+  };
+}

common/hardware/raito-vm.nix

@@ -1,7 +1,7 @@
-{ lib, config, ... }: 
+{ lib, config, ... }:
 let
   cfg = config.bagel.hardware.raito-vm;
-  inherit (lib) mkEnableOption mkIf mkOption types;
+  inherit (lib) mkEnableOption mkIf mkOption types split toIntBase10;
 in
 {
   options.bagel.hardware.raito-vm = {
@@ -30,8 +30,6 @@ in
   config = mkIf cfg.enable {
     services.qemuGuest.enable = true;
     systemd.network.enable = true;
-    security.acme.defaults.email = "bagel-acme@lahfa.xyz";
-    security.acme.acceptTerms = true;
     networking.useDHCP = lib.mkDefault false;
 
     systemd.network.networks."10-nat-lan" = {
@@ -56,6 +54,17 @@ in
       linkConfig.Name = "wan";
     };
 
+    bagel.infra.self.wan =
+    let
+      parts = split "/" cfg.networking.wan.address;
+      address = builtins.elemAt parts 0;
+      prefixLength = toIntBase10 (builtins.elemAt 1 parts);
+    in
+    {
+      family = "inet6";
+      inherit address prefixLength;
+    };
+
     boot.loader.systemd-boot.enable = true;
 
     boot.initrd.kernelModules = [

common/known-ssh-keys.nix

@@ -0,0 +1,7 @@
+{ ... }:
+{
+  programs.ssh.knownHosts = {
+    "[cl.forkos.org]:29418".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM82mJ259C8Nc+BHHNBeRWXWhL3dfirQhmFbDAwHMle3";
+    "[gerrit.lix.systems]:2022".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICC/S6Z56uhv7zBMutkV0nU8eDuRcl3trykGWBch4L/l";
+  };
+}

common/nix.nix

@@ -14,7 +14,7 @@
 
   # Use our cache and trust its signing key. Still use cache.nixos.org as
   # fallback.
-  nix.settings.substituters = [ "https://bagel-cache.s3-web.delroth.net/" ];
+  nix.settings.substituters = [ "https://cache.forkos.org/" ];
   nix.settings.trusted-public-keys = [
     "cache.forkos.org:xfXIUJO1yiEITJmYsVmNDa9BFSlgTh/YqZ+4ei1EhQg="
   ];

common/raito-proxy-aware-nginx.nix

@@ -1,9 +1,10 @@
 # This enables an IPv6-only server which is proxied by kurisu.lahfa.xyz to have proper IPv4 logs via PROXY protocol.
 { config, lib, ... }:
 let
-  inherit (lib) mkEnableOption mkIf;
+  inherit (lib) mkEnableOption mkIf concatStringsSep;
   cfg = config.bagel.raito.v6-proxy-awareness;
-  allowedUpstream = "2001:bc8:38ee:99::1/128";
+  #                      outside of raito infra    inside of raito infra
+  allowedUpstreams = [ "2001:bc8:38ee::1/128"     "2001:bc8:38ee:99::1/128" ];
 in
   {
     options.bagel.raito.v6-proxy-awareness.enable = mkEnableOption "the kurisu.lahfa.xyz's sniproxy awareness for NGINX";
@@ -20,8 +21,8 @@ in
         ];
 
         appendHttpConfig = ''
-          # Kurisu node
-          set_real_ip_from ${allowedUpstream};
+          # Kurisu nodes
+          ${concatStringsSep "\n" (map (up: "set_real_ip_from ${up};") allowedUpstreams)}
           real_ip_header proxy_protocol;
         '';
       };
@@ -29,7 +30,7 @@ in
       # Move to nftables if firewall is enabled.
       networking.nftables.enable = true;
       networking.firewall.extraInputRules = ''
-        ip6 saddr ${allowedUpstream} tcp dport 444 accept
+        ${concatStringsSep "\n" (map (up: "ip6 saddr ${up} tcp dport 444 accept") allowedUpstreams)}
       '';
     };
   }

common/secrets.nix

@@ -0,0 +1,22 @@
+## This is a simple secret abstraction with multi-tenancy awareness.
+{ config, lib, ... }:
+let
+  cfg = config.bagel.secrets;
+  inherit (lib) mkOption types genAttrs;
+in
+{
+  options.bagel.secrets = {
+    tenant = mkOption {
+      type = types.enum [ "lix" "floral" ];
+    };
+
+    files = mkOption {
+      type = types.listOf types.str;
+      default = [ ];
+    };
+  };
+
+  config.age.secrets = genAttrs cfg.files (secretFile: {
+      file = ../secrets/${cfg.tenant}/${secretFile}.age;
+    });
+}

common/server-acl.nix

@@ -0,0 +1,69 @@
+{ lib, config, ... }:
+let
+  keys = import ./ssh-keys.nix;
+  inherit (lib) mkOption types length concatMap listToAttrs catAttrs attrValues;
+  cfgAdmins = config.bagel.admins;
+  cfgGroups = config.bagel.groups;
+  cfgUsers = config.bagel.users;
+
+  userOpts = { name, ... }: {
+    options = {
+      sshKeys = mkOption {
+        type = types.listOf types.str;
+        description = "List of SSH keys associated to this user, defaults to `ssh-keys.nix` entries.";
+        default = keys.users.${name} or [ ];
+      };
+    };
+  };
+  groupOpts = { name, ... }: {
+    options = {
+      members = mkOption {
+        type = types.listOf types.str;
+        description = "List of users member of this group";
+        example = [ "raito" ];
+        default = [ ];
+      };
+    };
+  };
+
+  # There might be duplicate in that list. We will turn it into an attribute set.
+  allowedMembers = listToAttrs (
+    map (member: {
+      name = member;
+      value = cfgUsers.${member};
+    }) (concatMap (allowedGroup: cfgGroups.${allowedGroup}.members) cfgAdmins.allowedGroups));
+
+  rootKeys = concatMap ({ sshKeys, ... }: sshKeys) (attrValues allowedMembers);
+in
+{
+  options.bagel.users = mkOption {
+    type = types.attrsOf (types.submodule userOpts);
+    description = "User configuration for server ACLs";
+  };
+
+  options.bagel.groups = mkOption {
+    type = types.attrsOf (types.submodule groupOpts);
+    description = "Group configuration for server ACLs";
+  };
+
+  options.bagel.admins = {
+    allowedGroups = mkOption {
+      type = types.listOf types.str;
+      default = [ "catch-all" ];
+      description = "List of groups which are allowed to admin this machine.";
+      example = [ "lix" "build-infra" ];
+    };
+  };
+
+  config = {
+    assertions = [
+      { assertion = length config.users.users.root.openssh.authorizedKeys.keys > 0;
+        # TODO: you can add printing of `concatStringsSep ", " cfg.allowedGroups` to diagnose
+        # which are the allowed groups and existing admins.
+        message = "root@${config.networking.fqdnOrHostName} has no SSH key attached, this machine will lose its access if you deploy it successfully! Set a valid `bagel.admins.allowedGroups` or ensure you have at least one administrator of the relevant group registered";
+      }
+    ];
+
+    users.users.root.openssh.authorizedKeys.keys = rootKeys;
+  };
+}

common/ssh-keys.nix

@@ -1,9 +1,14 @@
 {
   machines = {
+    # Floral
     bagel-box = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAsO4bNqY04uG13Pg3ubHfRDssTphDLzZ4YUniE5/p+M";
     meta01 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM5t9gYorOWgpCFDJgb24pyCKIabGpeI2H/UfdvXODcT";
+    public01 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPBy8G8rfLA6E9i+t5kjVafxU1c2NXATXKxoXTH4Kgtm";
     gerrit01 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIA+eSZu+u9sCynrMlsmFzQHLIELQAuVg0Cs1pBvwb4+A";
     fodwatch = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFRyTNfvKl5FcSyzGzw+h+bNFNOxdhvI67WdUZ2iIJ1L";
+    buildbot = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJgIu6ouagYqBeMLfmn1CbaDJMuZcPH9bnUhkht8GfuB";
+    git = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEQJcpkCUOx8+5oukMX6lxrYcIX8FyHu8Mc/3+ieKMUn";
+    build-coord = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINpAEJP7F+XtJBpQP1jTzwXwQgJrFxwEJjPf/rnCXkJA";
     builder-0 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBHSNcDGctvlG6BHcJuYIzW9WsBJsts2vpwSketsbXoL";
     builder-1 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIQOGUjERK7Mx8UPM/rbOdMqVyn1sbWqYOG6CbOzH2wm";
     builder-2 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMKzXIqCoYElEKIYgjbSpqEcDeOvV+Wo3Agq3jba83cB";
@@ -15,20 +20,27 @@
     builder-8 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKGSWHNeqT0kF/e4yVy2ieW98X5QMyCYIYZh9WTmQDs1";
     builder-9 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOhws9zGgocVY36dMtOL+CXadpvRMffxoWMkfEcTBJm7";
     builder-10 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE7sgIuTSqZiZhp8TvObSbIEhcHHsL5hcmYA22uzwxth";
-    builder-11 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEAqFo1qJY7MSUkfB+zxXB8Lpt/Iqz/RR5A+zwhpRWhr";
     wob-vpn-gw = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINVytPPW8XnXf/rD5TFzsw//CZc2lBjQLmDzlVGPZsjh";
+
+    # Lix
+    build01-aarch64-lix = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICC69NZD/zhIB/wUb5odg46bss5g8hH2fDl22bk4qeSW";
+    build02-aarch64-lix = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGdJE375pe58RJbhKwXRp3D//+SJ3ssiVZrLsM9CLHn0";
+    build01-aarch64-darwin-lix = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMVf1uO0lv5UBti/naW/+amqLxvWZg+StXk9aM+lJ7e4";
+
+    buildbot-lix = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFoVSh35UqNQZ6ZZ1c6CzqERC40ovQ/KDXz8pC7nNlkR";
+
+    # Raito infrastructure
+    epyc-newtype-fr = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOXT9Init1MhKt4rjBANLq0t0bPww/WQZ96uB4AEDrml";
   };
 
   users = {
     delroth = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII3tjB4KYDok3KlWxdBp/yEmqhhmybd+w0VO4xUwLKKV" ];
-    raito = [
-      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICaw9ihTG7ucB8P38XdalEWev8+q96e2yNm4B+/I9IJp"
-      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDcEkYM1r8QVNM/G5CxJInEdoBCWjEHHDdHlzDYNSUIdHHsn04QY+XI67AdMCm8w30GZnLUIj5RiJEWXREUApby0GrfxGGcy8otforygfgtmuUKAUEHdU2MMwrQI7RtTZ8oQ0USRGuqvmegxz3l5caVU7qGvBllJ4NUHXrkZSja2/51vq80RF4MKkDGiz7xUTixI2UcBwQBCA/kQedKV9G28EH+1XfvePqmMivZjl+7VyHsgUVj9eRGA1XWFw59UPZG8a7VkxO/Eb3K9NF297HUAcFMcbY6cPFi9AaBgu3VC4eetDnoN/+xT1owiHi7BReQhGAy/6cdf7C/my5ehZwD"
-      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE0xMwWedkKosax9+7D2OlnMxFL/eV4CvFZLsbLptpXr"
-      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiXXYkhRh+s7ixZ8rvG8ntIqd6FELQ9hh7HoaHQJRPU"
+    emilylange = [ "no-touch-required sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIL7jgq3i+N3gVJhs4shm7Kmw6dIocs2OuR0GBMG1RxfKAAAABHNzaDo=" ];
+    hexchen = [
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINJ0tCxsEilAzV6LaNpUpcjzyEn4ptw8kFz3R+Z3YjEF hexchen@backup"
+      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDI3T1eFS77URHZ/HVWkMOqx7W1U54zJtn9C7QWsHOtyH72i/4EVj8SxYqLllElh1kuKUXSUipPeEzVsipFVvfH0wEuTDgFffiSQ3a8lfUgdEBuoySwceEoPgc5deapkOmiDIDeeWlrRe3nqspLRrSWU1DirMxoFPbwqJXRvpl6qJPxRg+2IolDcXlZ6yxB4Vv48vzRfVzZNUz7Pjmy2ebU8PbDoFWL/S3m7yOzQpv3L7KYBz7+rkjuF3AU2vy6CAfIySkVpspZZLtkTGCIJF228ev0e8NvhuN6ZnjzXxVTQOy32HCdPdbBbicu0uHfZ5O7JX9DjGd8kk1r2dnZwwy/ hexchen@yubi5"
+      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC4CLJ+mFfq5XiBXROKewmN9WYmj+79bj/AoaR6Iud2pirulot3tkrrLe2cMjiNWFX8CGVqrsAELKUA8EyUTJfStlcTE0/QNESTRmdDaC+lZL41pWUO9KOiD6/0axAhHXrSJ0ScvbqtD0CtpnCKKxtuOflVPoUGZsH9cLKJNRKfEka0H0GgeKb5Tp618R/WNAQOwaCcXzg/nG4Bgv3gJW4Nm9IKy/MwRZqtILi8Mtd+2diTqpMwyNRmbenmRHCQ1vRw46joYkledVqrmSlfSMFgIHI1zRSBXb/JkG2IvIyB5TGbTkC4N2fqJNpH8wnCKuOvs46xmgdiRA26P48C2em3 hexchen@yubi5c"
     ];
-    k900 = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOi9vgVGs+S5kEsUqHPvyMMh1Q9gqL4TcbHoe5d73tun" ];
-    maxine = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILpWQfhNFdrxMTP/1DwBVuk49f3df9iH7Tbdu8ltIKjr" ];
     jade = [
       "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDNldAg4t13/i69TD786The+U3wbiNUdW2Kc9KNWvEhgpf4y4x4Sft0oYfkPw5cjX4H3APqfD+b7ItAG0GCbwHw6KMYPoVMNK08zBMJUqt1XExbqGeFLqBaeqDsmEAYXJRbjMTAorpOCtgQdoCKK/DvZ51zUWXxT8UBNHSl19Ryv5Ry5VVdbAE35rqs57DQ9+ma6htXnsBEmmnC+1Zv1FE956m/OpBTId50mor7nS2FguAtPZnDPpTd5zl9kZmJEuWCrmy6iinw5V4Uy1mLeZkQv+/FtozbyifCRCvps9nHpv4mBSU5ABLgnRRvXs+D41Jx7xloNADr1nNgpsNrYaTh hed-bot-ssh-tpm-rsa"
       "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIKYljH8iPMrH00lOb3ETxRrZimdKzPPEdsJQ5D5ovtOwAAAACnNzaDpzc2hrZXk= ssh:sshkey"
@@ -39,7 +51,22 @@
       "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIOYg513QZsVzoyVycXZjg4F3T3+OwtcY3WAhrlfyLgLTAAAABHNzaDo="
       "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBLZxVITpJ8xbiCa/u2gjSSIupeiqOnRh+8tFIoVhCON"
     ];
+    k900 = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOi9vgVGs+S5kEsUqHPvyMMh1Q9gqL4TcbHoe5d73tun" ];
     lukegb = [ ''cert-authority,principals="lukegb" ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEqNOwlR7Qa8cbGpDfSCOweDPbAGQOZIcoRgh6s/J8DR'' ];
-    yuka = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKath4/fDnlv/4fzxkPrQN1ttmoPRNu/m9bEtdPJBDfY cardno:16_933_242" ];
+    maxine = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILpWQfhNFdrxMTP/1DwBVuk49f3df9iH7Tbdu8ltIKjr" ];
+    raito = [
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICaw9ihTG7ucB8P38XdalEWev8+q96e2yNm4B+/I9IJp"
+      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDcEkYM1r8QVNM/G5CxJInEdoBCWjEHHDdHlzDYNSUIdHHsn04QY+XI67AdMCm8w30GZnLUIj5RiJEWXREUApby0GrfxGGcy8otforygfgtmuUKAUEHdU2MMwrQI7RtTZ8oQ0USRGuqvmegxz3l5caVU7qGvBllJ4NUHXrkZSja2/51vq80RF4MKkDGiz7xUTixI2UcBwQBCA/kQedKV9G28EH+1XfvePqmMivZjl+7VyHsgUVj9eRGA1XWFw59UPZG8a7VkxO/Eb3K9NF297HUAcFMcbY6cPFi9AaBgu3VC4eetDnoN/+xT1owiHi7BReQhGAy/6cdf7C/my5ehZwD"
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE0xMwWedkKosax9+7D2OlnMxFL/eV4CvFZLsbLptpXr"
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiXXYkhRh+s7ixZ8rvG8ntIqd6FELQ9hh7HoaHQJRPU"
+    ];
+    thubrecht = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPM1jpXR7BWQa7Sed7ii3SbvIPRRlKb3G91qC0vOwfJn" ];
+    yuka = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIxQ3NYBi8v1f/vhxLKDcA6upmX0pctRDbnK6SER5OUR yureka" ];
+    winter = [ "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIH/LDRUG+U+++UmlxvA2kspioTjktQZ8taDcHq8gVlkfAAAABHNzaDo=" ];
+    ckie = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIH3uTwzSSMAPg84fwbNp2cq9+BdLFeA1VzDGth4zCAbz https://mei.puppycat.house" ];
+    pennae = [
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIC5Wf5/IbyFpdziWfwxkQqxOf3r1L9pYn6xQBEKFwmMY"
+      "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIK8icXjHkb4XzbIVN3djH4CE7RvgGd+3xbG4cgh0Yls5AAAABHNzaDo="
+    ];
   };
 }

common/sysadmin/default.nix

@@ -13,7 +13,11 @@ in
         tmux
         rsync
         fd
+        eza
+        grc
         ripgrep
+        delta
+        tshark
         pv
         kitty.terminfo
         config.boot.kernelPackages.perf

common/zsh.nix

@@ -0,0 +1,15 @@
+{ lib, pkgs, config, ... }: {
+  programs.zsh = {
+    enable = true;
+    enableCompletion = true;
+    autosuggestions.enable = true;
+    interactiveShellInit = ''
+      ${lib.getExe pkgs.nix-your-shell} zsh | source /dev/stdin
+    '';
+    promptInit = ''
+      # https://grml.org/zsh/grml-zsh-refcard.pdf
+      source ${pkgs.grml-zsh-config}/etc/zsh/zshrc
+      PS1='%n@${config.networking.fqdn} %/ \$ '
+    '';
+  };
+}

dashboards/default.nix

@@ -0,0 +1,43 @@
+{ gerrit-dashboard, stdenv, symlinkJoin, jsonnet, fetchFromGitHub, lib, ... }:
+let
+  inherit (lib) concatMapStringsSep;
+  datasource-id = "mimir";
+in
+rec {
+  grafonnet = fetchFromGitHub {
+    owner = "grafana";
+    repo = "grafonnet-lib";
+    # TODO: figure out how to read the jsonnet lockfile
+    # and propagate this a bit cleverly.
+    rev = "a1d61cce1da59c71409b99b5c7568511fec661ea";
+    hash = "sha256-fs5JZJbcL6sQXBjYhp5eeRtjTFw0J1O/BcwBC8Vm9EM=";
+  };
+  buildJsonnetDashboards = dashboardSrc: targets: stdenv.mkDerivation {
+    name = "jsonnet-grafana-dashboards";
+    src = dashboardSrc;
+    buildInputs = [ jsonnet ];
+    buildPhase = ''
+      runHook preBuild
+      mkdir -p $out
+      ${concatMapStringsSep "\n" (target: "jsonnet -J ${grafonnet} --ext-str datasource=${datasource-id} --ext-code publish=true $src/${target} > $out/${baseNameOf target}.json") targets}
+      runHook postBuild
+    '';
+  };
+
+  allDashboards = symlinkJoin {
+    name = "all-jsonnet-dashboards";
+    paths = [
+      (buildJsonnetDashboards gerrit-dashboard [
+        "dashboards/gerrit/caches/gerrit-caches.jsonnet"
+        "dashboards/gerrit/fetch-clone/gerrit-fetch-clone.jsonnet"
+        "dashboards/gerrit/fetch-clone/gerrit-phases.jsonnet"
+        "dashboards/gerrit/healthcheck/gerrit-healthcheck.jsonnet"
+        "dashboards/gerrit/latency/gerrit-push-latency.jsonnet"
+        "dashboards/gerrit/latency/gerrit-ui-actions-latency.jsonnet"
+        "dashboards/gerrit/overview/gerrit-overview.jsonnet"
+        "dashboards/gerrit/process/gerrit-process.jsonnet"
+        "dashboards/gerrit/queues/gerrit-queues.jsonnet"
+      ])
+    ];
+  };
+}

flake.lock

@@ -10,11 +10,11 @@
         "systems": "systems"
       },
       "locked": {
-        "lastModified": 1720546205,
-        "narHash": "sha256-boCXsjYVxDviyzoEyAk624600f3ZBo/DKtUdvMTpbGY=",
+        "lastModified": 1723293904,
+        "narHash": "sha256-b+uqzj+Wa6xgMS9aNbX4I+sXeb5biPDi39VgvSFqFvU=",
         "owner": "ryantm",
         "repo": "agenix",
-        "rev": "de96bd907d5fbc3b14fc33ad37d1b9a3cb15edc6",
+        "rev": "f6291c5935fdc4e0bef208cfc0dcab7e3f7a1c41",
         "type": "github"
       },
       "original": {
@@ -23,14 +23,38 @@
         "type": "github"
       }
     },
+    "attic": {
+      "inputs": {
+        "crane": "crane",
+        "flake-compat": "flake-compat_2",
+        "flake-parts": "flake-parts_2",
+        "nix-github-actions": "nix-github-actions_2",
+        "nixpkgs": "nixpkgs",
+        "nixpkgs-stable": "nixpkgs-stable"
+      },
+      "locked": {
+        "lastModified": 1731270564,
+        "narHash": "sha256-6KMC/NH/VWP5Eb+hA56hz0urel3jP6Y6cF2PX6xaTkk=",
+        "owner": "zhaofengli",
+        "repo": "attic",
+        "rev": "47752427561f1c34debb16728a210d378f0ece36",
+        "type": "github"
+      },
+      "original": {
+        "owner": "zhaofengli",
+        "ref": "main",
+        "repo": "attic",
+        "type": "github"
+      }
+    },
     "bats-assert": {
       "flake": false,
       "locked": {
-        "lastModified": 1636059754,
-        "narHash": "sha256-ewME0l27ZqfmAwJO4h5biTALc9bDLv7Bl3ftBzBuZwk=",
+        "lastModified": 1692829535,
+        "narHash": "sha256-oDqhUQ6Xg7a3xx537SWLGRzqP3oKKeyY4UYGCdz9z/Y=",
         "owner": "bats-core",
         "repo": "bats-assert",
-        "rev": "34551b1d7f8c7b677c1a66fc0ac140d6223409e5",
+        "rev": "e2d855bc78619ee15b0c702b5c30fb074101159f",
         "type": "github"
       },
       "original": {
@@ -42,11 +66,11 @@
     "bats-support": {
       "flake": false,
       "locked": {
-        "lastModified": 1548869839,
-        "narHash": "sha256-Gr4ntadr42F2Ks8Pte2D4wNDbijhujuoJi4OPZnTAZU=",
+        "lastModified": 1693050811,
+        "narHash": "sha256-PxJaH16+QrsfZqtkWVt5K6TwJB5gjIXnbGo+MB84WIU=",
         "owner": "bats-core",
         "repo": "bats-support",
-        "rev": "d140a65044b2d6810381935ae7f0c94c7023c8c3",
+        "rev": "9bf10e876dd6b624fe44423f0b35e064225f7556",
         "type": "github"
       },
       "original": {
@@ -55,21 +79,65 @@
         "type": "github"
       }
     },
+    "buildbot-nix": {
+      "inputs": {
+        "flake-parts": "flake-parts",
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "treefmt-nix": "treefmt-nix"
+      },
+      "locked": {
+        "lastModified": 1730064416,
+        "narHash": "sha256-Opbtu9hKijGkEx+GYbSu3MJms3lFxZmAGTFyckguWMM=",
+        "ref": "refs/heads/forkos",
+        "rev": "79137b14f3cb376204f739f44b05aebfc288ca89",
+        "revCount": 310,
+        "type": "git",
+        "url": "https://git.lix.systems/lix-project/buildbot-nix.git"
+      },
+      "original": {
+        "ref": "refs/heads/forkos",
+        "type": "git",
+        "url": "https://git.lix.systems/lix-project/buildbot-nix.git"
+      }
+    },
+    "channel-scripts": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1734197525,
+        "narHash": "sha256-rb/+iJBNsfXnz+PJSdlsCViodtEHrgfz/Fixq2NXUFI=",
+        "ref": "refs/heads/main",
+        "rev": "6e4ae567a3f872bdb90a62d588bb5cc4b3596258",
+        "revCount": 265,
+        "type": "git",
+        "url": "https://git.lix.systems/the-distro/channel-scripts.git"
+      },
+      "original": {
+        "type": "git",
+        "url": "https://git.lix.systems/the-distro/channel-scripts.git"
+      }
+    },
     "colmena": {
       "inputs": {
         "flake-compat": "flake-compat",
         "flake-utils": "flake-utils",
+        "nix-github-actions": "nix-github-actions",
         "nixpkgs": [
           "nixpkgs"
         ],
         "stable": "stable"
       },
       "locked": {
-        "lastModified": 1711386353,
-        "narHash": "sha256-gWEpb8Hybnoqb4O4tmpohGZk6+aerAbJpywKcFIiMlg=",
+        "lastModified": 1731527002,
+        "narHash": "sha256-dI9I6suECoIAmbS4xcrqF8r2pbmed8WWm5LIF1yWPw8=",
         "owner": "zhaofengli",
         "repo": "colmena",
-        "rev": "cd65ef7a25cdc75052fbd04b120aeb066c3881db",
+        "rev": "e3ad42138015fcdf2524518dd564a13145c72ea1",
         "type": "github"
       },
       "original": {
@@ -78,6 +146,44 @@
         "type": "github"
       }
     },
+    "crane": {
+      "inputs": {
+        "nixpkgs": [
+          "grapevine",
+          "attic",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1722960479,
+        "narHash": "sha256-NhCkJJQhD5GUib8zN9JrmYGMwt4lCRp6ZVNzIiYCl0Y=",
+        "owner": "ipetkov",
+        "repo": "crane",
+        "rev": "4c6c77920b8d44cd6660c1621dea6b3fc4b4c4f4",
+        "type": "github"
+      },
+      "original": {
+        "owner": "ipetkov",
+        "repo": "crane",
+        "type": "github"
+      }
+    },
+    "crane_2": {
+      "locked": {
+        "lastModified": 1731098351,
+        "narHash": "sha256-HQkYvKvaLQqNa10KEFGgWHfMAbWBfFp+4cAgkut+NNE=",
+        "owner": "ipetkov",
+        "repo": "crane",
+        "rev": "ef80ead953c1b28316cc3f8613904edc2eb90c28",
+        "type": "github"
+      },
+      "original": {
+        "owner": "ipetkov",
+        "ref": "master",
+        "repo": "crane",
+        "type": "github"
+      }
+    },
     "darwin": {
       "inputs": {
         "nixpkgs": [
@@ -100,6 +206,29 @@
         "type": "github"
       }
     },
+    "fenix": {
+      "inputs": {
+        "nixpkgs": [
+          "grapevine",
+          "nixpkgs"
+        ],
+        "rust-analyzer-src": "rust-analyzer-src"
+      },
+      "locked": {
+        "lastModified": 1731738660,
+        "narHash": "sha256-tIXhc9lX1b030v812yVJanSR37OnpTb/OY5rU3TbShA=",
+        "owner": "nix-community",
+        "repo": "fenix",
+        "rev": "e10ba121773f754a30d31b6163919a3e404a434f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "ref": "main",
+        "repo": "fenix",
+        "type": "github"
+      }
+    },
     "flake-compat": {
       "flake": false,
       "locked": {
@@ -132,6 +261,121 @@
         "type": "github"
       }
     },
+    "flake-compat_3": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1696426674,
+        "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "ref": "master",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "flake-compat_4": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1696426674,
+        "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "flake-parts": {
+      "inputs": {
+        "nixpkgs-lib": [
+          "buildbot-nix",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1706830856,
+        "narHash": "sha256-a0NYyp+h9hlb7ddVz4LUn1vT/PLwqfrWYcHMvFB1xYg=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "b253292d9c0a5ead9bc98c4e9a26c6312e27d69f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
+    "flake-parts_2": {
+      "inputs": {
+        "nixpkgs-lib": [
+          "grapevine",
+          "attic",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1722555600,
+        "narHash": "sha256-XOQkdLafnb/p9ij77byFQjDf5m5QYl9b2REiVClC+x4=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "8471fe90ad337a8074e957b69ca4d0089218391d",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
+    "flake-parts_3": {
+      "inputs": {
+        "nixpkgs-lib": [
+          "hydra",
+          "nix-eval-jobs",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1730504689,
+        "narHash": "sha256-hgmguH29K2fvs9szpq2r3pz2/8cJd2LPS+b4tfNFCwE=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "506278e768c2a08bec68eb62932193e341f55c90",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
+    "flake-parts_4": {
+      "inputs": {
+        "nixpkgs-lib": "nixpkgs-lib"
+      },
+      "locked": {
+        "lastModified": 1727826117,
+        "narHash": "sha256-K5ZLCyfO/Zj9mPFldf3iwS6oZStJcU4tSpiXTMYaaL0=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "3d04084d54bedc3d6b8b736c70ef449225c361b1",
+        "type": "github"
+      },
+      "original": {
+        "id": "flake-parts",
+        "type": "indirect"
+      }
+    },
     "flake-utils": {
       "locked": {
         "lastModified": 1659877975,
@@ -148,20 +392,70 @@
       }
     },
     "flake-utils_2": {
+      "inputs": {
+        "systems": "systems_2"
+      },
       "locked": {
-        "lastModified": 1634851050,
-        "narHash": "sha256-N83GlSGPJJdcqhUxSCS/WwW5pksYf3VP1M13cDRTSVA=",
+        "lastModified": 1731533236,
+        "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
         "owner": "numtide",
         "repo": "flake-utils",
-        "rev": "c91f3de5adaf1de973b797ef7485e441a65b8935",
+        "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
         "type": "github"
       },
       "original": {
         "owner": "numtide",
+        "ref": "main",
         "repo": "flake-utils",
         "type": "github"
       }
     },
+    "gerrit-dashboard": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1724509518,
+        "narHash": "sha256-fwYXZVddxfzrlDa3QnFCwHqrbEX+3PrWy0QOlbO+8jk=",
+        "ref": "refs/heads/master",
+        "rev": "e544abac81c581558d68abb2a8dd583049073939",
+        "revCount": 75,
+        "type": "git",
+        "url": "https://git.lix.systems/the-distro/gerrit-monitoring.git"
+      },
+      "original": {
+        "type": "git",
+        "url": "https://git.lix.systems/the-distro/gerrit-monitoring.git"
+      }
+    },
+    "grapevine": {
+      "inputs": {
+        "attic": "attic",
+        "crane": "crane_2",
+        "fenix": "fenix",
+        "flake-compat": "flake-compat_3",
+        "flake-utils": "flake-utils_2",
+        "nix-filter": "nix-filter",
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "rocksdb": "rocksdb",
+        "rust-manifest": "rust-manifest"
+      },
+      "locked": {
+        "host": "gitlab.computer.surgery",
+        "lastModified": 1734138037,
+        "narHash": "sha256-pN/nJ9tR6ewnpVUUzcF+Z9L/0R0WmtBVePJOqx9rzTk=",
+        "owner": "matrix",
+        "repo": "grapevine-fork",
+        "rev": "8537c0e8ac3eb388500587b035008e5f98204a4b",
+        "type": "gitlab"
+      },
+      "original": {
+        "host": "gitlab.computer.surgery",
+        "owner": "matrix",
+        "repo": "grapevine-fork",
+        "type": "gitlab"
+      }
+    },
     "home-manager": {
       "inputs": {
         "nixpkgs": [
@@ -185,17 +479,18 @@
     },
     "hydra": {
       "inputs": {
-        "nix": "nix",
+        "lix": "lix",
+        "nix-eval-jobs": "nix-eval-jobs",
         "nixpkgs": [
           "nixpkgs"
         ]
       },
       "locked": {
-        "lastModified": 1720843955,
-        "narHash": "sha256-GpkZ7OorcArMaFVZMPHkXHKQVJAWjMSCaPmS8hw0PB0=",
+        "lastModified": 1733503045,
+        "narHash": "sha256-VoMam8Zzbk+X6dIYwH2f9NqItL6g9YDhQvGybzSl8xQ=",
         "ref": "refs/heads/main",
-        "rev": "fb9e29d4d0f2f591cd1d706fd3b7334af7d34b84",
-        "revCount": 4174,
+        "rev": "eccf01d4fef67f87b6383f96c73781bd08b686ac",
+        "revCount": 4230,
         "type": "git",
         "url": "https://git.lix.systems/lix-project/hydra.git"
       },
@@ -204,9 +499,9 @@
         "url": "https://git.lix.systems/lix-project/hydra.git"
       }
     },
-    "nix": {
+    "lix": {
       "inputs": {
-        "flake-compat": "flake-compat_2",
+        "flake-compat": "flake-compat_4",
         "nix2container": "nix2container",
         "nixpkgs": [
           "hydra",
@@ -216,17 +511,77 @@
         "pre-commit-hooks": "pre-commit-hooks"
       },
       "locked": {
-        "lastModified": 1720733512,
-        "narHash": "sha256-vq9CLDvqSSvH4L7YhDa0ihTOrAry4jntKiuoNb5n98M=",
+        "lastModified": 1732112222,
+        "narHash": "sha256-H7GN4++a4vE49SUNojZx+FSk4mmpb2ifJUtJMJHProI=",
         "ref": "refs/heads/main",
-        "rev": "4b109ec1a8fc4550150f56f0f46f2f41d844bda8",
-        "revCount": 15950,
+        "rev": "66f6dbda32959dd5cf3a9aaba15af72d037ab7ff",
+        "revCount": 16513,
         "type": "git",
-        "url": "https://git@git.lix.systems/lix-project/lix"
+        "url": "https://git.lix.systems/lix-project/lix"
       },
       "original": {
         "type": "git",
-        "url": "https://git@git.lix.systems/lix-project/lix"
+        "url": "https://git.lix.systems/lix-project/lix"
+      }
+    },
+    "nix-eval-jobs": {
+      "inputs": {
+        "flake-parts": "flake-parts_3",
+        "lix": [
+          "hydra",
+          "lix"
+        ],
+        "nix-github-actions": "nix-github-actions_3",
+        "nixpkgs": [
+          "hydra",
+          "nixpkgs"
+        ],
+        "treefmt-nix": "treefmt-nix_2"
+      },
+      "locked": {
+        "lastModified": 1732351635,
+        "narHash": "sha256-H94CcQ3yamG5+RMxtxXllR02YIlxQ5WD/8PcolO9yEA=",
+        "ref": "refs/heads/main",
+        "rev": "dfc286ca3dc49118c30d8d6205d6d6af76c62b7a",
+        "revCount": 617,
+        "type": "git",
+        "url": "https://git.lix.systems/lix-project/nix-eval-jobs"
+      },
+      "original": {
+        "type": "git",
+        "url": "https://git.lix.systems/lix-project/nix-eval-jobs"
+      }
+    },
+    "nix-filter": {
+      "locked": {
+        "lastModified": 1731533336,
+        "narHash": "sha256-oRam5PS1vcrr5UPgALW0eo1m/5/pls27Z/pabHNy2Ms=",
+        "owner": "numtide",
+        "repo": "nix-filter",
+        "rev": "f7653272fd234696ae94229839a99b73c9ab7de0",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "ref": "main",
+        "repo": "nix-filter",
+        "type": "github"
+      }
+    },
+    "nix-forgejo": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1734486094,
+        "narHash": "sha256-WZ2vtxCItlfev7um8lBFzXtp7WYVHsDNrCnk1i8yLAU=",
+        "ref": "refs/heads/main",
+        "rev": "403b7bab1b5c37275ab946a5944c2caaf12eca78",
+        "revCount": 1,
+        "type": "git",
+        "url": "https://git.lix.systems/the-distro/nix-forgejo.git"
+      },
+      "original": {
+        "type": "git",
+        "url": "https://git.lix.systems/the-distro/nix-forgejo.git"
       }
     },
     "nix-gerrit": {
@@ -236,27 +591,93 @@
         ]
       },
       "locked": {
-        "lastModified": 1720891381,
-        "narHash": "sha256-bdZRPgnkROSejmwMOrlcqHMWmuPIVIzjk6r5FbS+fqU=",
-        "ref": "refs/heads/main",
-        "rev": "23dd318e6741ff686d3069c53ecf475eac8a0565",
-        "revCount": 5,
+        "lastModified": 1734192622,
+        "narHash": "sha256-AkT4QHHneyWBL9UDhvrmPnQUOfN9ETP295y6TtuW6rU=",
+        "ref": "refs/heads/bump-minor-3_10",
+        "rev": "c011f670b335b52150af5c75f21e987d166ecec2",
+        "revCount": 8,
         "type": "git",
         "url": "https://git.lix.systems/the-distro/nix-gerrit.git"
       },
       "original": {
+        "ref": "refs/heads/bump-minor-3_10",
         "type": "git",
         "url": "https://git.lix.systems/the-distro/nix-gerrit.git"
       }
     },
+    "nix-github-actions": {
+      "inputs": {
+        "nixpkgs": [
+          "colmena",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1729742964,
+        "narHash": "sha256-B4mzTcQ0FZHdpeWcpDYPERtyjJd/NIuaQ9+BV1h+MpA=",
+        "owner": "nix-community",
+        "repo": "nix-github-actions",
+        "rev": "e04df33f62cdcf93d73e9a04142464753a16db67",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "nix-github-actions",
+        "type": "github"
+      }
+    },
+    "nix-github-actions_2": {
+      "inputs": {
+        "nixpkgs": [
+          "grapevine",
+          "attic",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1729742964,
+        "narHash": "sha256-B4mzTcQ0FZHdpeWcpDYPERtyjJd/NIuaQ9+BV1h+MpA=",
+        "owner": "nix-community",
+        "repo": "nix-github-actions",
+        "rev": "e04df33f62cdcf93d73e9a04142464753a16db67",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "nix-github-actions",
+        "type": "github"
+      }
+    },
+    "nix-github-actions_3": {
+      "inputs": {
+        "nixpkgs": [
+          "hydra",
+          "nix-eval-jobs",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1731952509,
+        "narHash": "sha256-p4gB3Rhw8R6Ak4eMl8pqjCPOLCZRqaehZxdZ/mbFClM=",
+        "owner": "nix-community",
+        "repo": "nix-github-actions",
+        "rev": "7b5f051df789b6b20d259924d349a9ba3319b226",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "nix-github-actions",
+        "type": "github"
+      }
+    },
     "nix2container": {
       "flake": false,
       "locked": {
-        "lastModified": 1712990762,
-        "narHash": "sha256-hO9W3w7NcnYeX8u8cleHiSpK2YJo7ecarFTUlbybl7k=",
+        "lastModified": 1724996935,
+        "narHash": "sha256-njRK9vvZ1JJsP8oV2OgkBrpJhgQezI03S7gzskCcHos=",
         "owner": "nlewo",
         "repo": "nix2container",
-        "rev": "20aad300c925639d5d6cbe30013c8357ce9f2a2e",
+        "rev": "fa6bb0a1159f55d071ba99331355955ae30b3401",
         "type": "github"
       },
       "original": {
@@ -267,11 +688,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1720750130,
-        "narHash": "sha256-y2wc7CdK0vVSIbx7MdVoZzuMcUoLvZXm+pQf2RIr1OU=",
+        "lastModified": 1726042813,
+        "narHash": "sha256-LnNKCCxnwgF+575y0pxUdlGZBO/ru1CtGHIqQVfvjlA=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "6794d064edc69918bb0fc0e0eda33ece324be17a",
+        "rev": "159be5db480d1df880a0135ca0bfed84c2f88353",
         "type": "github"
       },
       "original": {
@@ -281,6 +702,18 @@
         "type": "github"
       }
     },
+    "nixpkgs-lib": {
+      "locked": {
+        "lastModified": 1727825735,
+        "narHash": "sha256-0xHYkMkeLVQAMa7gvkddbPqpxph+hDzdu1XdGPJR+Os=",
+        "type": "tarball",
+        "url": "https://github.com/NixOS/nixpkgs/archive/fb192fec7cc7a4c26d51779e9bab07ce6fa5597a.tar.gz"
+      },
+      "original": {
+        "type": "tarball",
+        "url": "https://github.com/NixOS/nixpkgs/archive/fb192fec7cc7a4c26d51779e9bab07ce6fa5597a.tar.gz"
+      }
+    },
     "nixpkgs-regression": {
       "locked": {
         "lastModified": 1643052045,
@@ -297,29 +730,63 @@
         "type": "github"
       }
     },
-    "nixpkgs_2": {
+    "nixpkgs-stable": {
       "locked": {
-        "lastModified": 1636823747,
-        "narHash": "sha256-oWo1nElRAOZqEf90Yek2ixdHyjD+gqtS/pAgwaQ9UhQ=",
-        "owner": "nixos",
+        "lastModified": 1724316499,
+        "narHash": "sha256-Qb9MhKBUTCfWg/wqqaxt89Xfi6qTD3XpTzQ9eXi3JmE=",
+        "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "f6a2ed2082d9a51668c86ba27d0b5496f7a2ea93",
+        "rev": "797f7dc49e0bc7fab4b57c021cdf68f595e47841",
         "type": "github"
       },
       "original": {
-        "owner": "nixos",
+        "owner": "NixOS",
+        "ref": "nixos-24.05",
         "repo": "nixpkgs",
         "type": "github"
       }
     },
+    "nixpkgs_2": {
+      "locked": {
+        "lastModified": 1733940404,
+        "narHash": "sha256-Pj39hSoUA86ZePPF/UXiYHHM7hMIkios8TYG29kQT4g=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "5d67ea6b4b63378b9c13be21e2ec9d1afc921713",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "ofborg": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1734308727,
+        "narHash": "sha256-/bJhMZQ5VSblvgqAR9hSLwdm5pxenn/UMY8pDDVSquI=",
+        "ref": "refs/heads/vcs-generalization",
+        "rev": "7bcc8fa584c66f317923337658974c0525e5779f",
+        "revCount": 1495,
+        "type": "git",
+        "url": "https://git.lix.systems/the-distro/ofborg.git"
+      },
+      "original": {
+        "ref": "refs/heads/vcs-generalization",
+        "type": "git",
+        "url": "https://git.lix.systems/the-distro/ofborg.git"
+      }
+    },
     "pre-commit-hooks": {
       "flake": false,
       "locked": {
-        "lastModified": 1712055707,
-        "narHash": "sha256-4XLvuSIDZJGS17xEwSrNuJLL7UjDYKGJSbK1WWX2AK8=",
+        "lastModified": 1726745158,
+        "narHash": "sha256-D5AegvGoEjt4rkKedmxlSEmC+nNLMBPWFxvmYnVLhjk=",
         "owner": "cachix",
         "repo": "git-hooks.nix",
-        "rev": "e35aed5fda3cc79f88ed7f1795021e559582093a",
+        "rev": "4e743a6920eab45e8ba0fbe49dc459f1423a4b74",
         "type": "github"
       },
       "original": {
@@ -328,36 +795,105 @@
         "type": "github"
       }
     },
+    "rocksdb": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1730475155,
+        "narHash": "sha256-u5uuShM2SxHc9/zL4UU56IhCcR/ZQbzde0LgOYS44bM=",
+        "owner": "facebook",
+        "repo": "rocksdb",
+        "rev": "3c27a3dde0993210c5cc30d99717093f7537916f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "facebook",
+        "ref": "v9.7.4",
+        "repo": "rocksdb",
+        "type": "github"
+      }
+    },
     "root": {
       "inputs": {
         "agenix": "agenix",
+        "buildbot-nix": "buildbot-nix",
+        "channel-scripts": "channel-scripts",
         "colmena": "colmena",
+        "gerrit-dashboard": "gerrit-dashboard",
+        "grapevine": "grapevine",
         "hydra": "hydra",
         "lix": [
           "hydra",
-          "nix"
+          "lix"
         ],
+        "nix-forgejo": "nix-forgejo",
         "nix-gerrit": "nix-gerrit",
-        "nixpkgs": "nixpkgs",
+        "nixpkgs": "nixpkgs_2",
+        "ofborg": "ofborg",
+        "stateless-uptime-kuma": "stateless-uptime-kuma",
         "terranix": "terranix"
       }
     },
+    "rust-analyzer-src": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1731693936,
+        "narHash": "sha256-uHUUS1WPyW6ohp5Bt3dAZczUlQ22vOn7YZF8vaPKIEw=",
+        "owner": "rust-lang",
+        "repo": "rust-analyzer",
+        "rev": "1b90e979aeee8d1db7fe14603a00834052505497",
+        "type": "github"
+      },
+      "original": {
+        "owner": "rust-lang",
+        "ref": "nightly",
+        "repo": "rust-analyzer",
+        "type": "github"
+      }
+    },
+    "rust-manifest": {
+      "flake": false,
+      "locked": {
+        "narHash": "sha256-tB9BZB6nRHDk5ELIVlGYlIjViLKBjQl52nC1avhcCwA=",
+        "type": "file",
+        "url": "https://static.rust-lang.org/dist/channel-rust-1.81.0.toml"
+      },
+      "original": {
+        "type": "file",
+        "url": "https://static.rust-lang.org/dist/channel-rust-1.81.0.toml"
+      }
+    },
     "stable": {
       "locked": {
-        "lastModified": 1696039360,
-        "narHash": "sha256-g7nIUV4uq1TOVeVIDEZLb005suTWCUjSY0zYOlSBsyE=",
+        "lastModified": 1730883749,
+        "narHash": "sha256-mwrFF0vElHJP8X3pFCByJR365Q2463ATp2qGIrDUdlE=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "32dcb45f66c0487e92db8303a798ebc548cadedc",
+        "rev": "dba414932936fde69f0606b4f1d87c5bc0003ede",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-23.05",
+        "ref": "nixos-24.05",
         "repo": "nixpkgs",
         "type": "github"
       }
     },
+    "stateless-uptime-kuma": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1728243069,
+        "narHash": "sha256-l9fgwesnmFxasCaYUCD7L9bGGJXytLuwtx3CZMgpwJg=",
+        "ref": "refs/heads/master",
+        "rev": "880f444ff7862d6127b051cf1a993ad1585b1652",
+        "revCount": 25,
+        "type": "git",
+        "url": "https://git.dgnum.eu/DGNum/stateless-uptime-kuma.git"
+      },
+      "original": {
+        "type": "git",
+        "url": "https://git.dgnum.eu/DGNum/stateless-uptime-kuma.git"
+      }
+    },
     "systems": {
       "locked": {
         "lastModified": 1681028828,
@@ -373,20 +909,53 @@
         "type": "github"
       }
     },
+    "systems_2": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "systems_3": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
     "terranix": {
       "inputs": {
         "bats-assert": "bats-assert",
         "bats-support": "bats-support",
-        "flake-utils": "flake-utils_2",
-        "nixpkgs": "nixpkgs_2",
+        "flake-parts": "flake-parts_4",
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "systems": "systems_3",
         "terranix-examples": "terranix-examples"
       },
       "locked": {
-        "lastModified": 1695406838,
-        "narHash": "sha256-xiUfVD6rtsVWFotVtUW3Q1nQh4obKzgvpN1wqZuGXvM=",
+        "lastModified": 1728959489,
+        "narHash": "sha256-1Pu2j5xsBTuoyga08ZVf+rKp3FOMmJh/0fXen/idOrA=",
         "owner": "terranix",
         "repo": "terranix",
-        "rev": "fc9077ca02ab5681935dbf0ecd725c4d889b9275",
+        "rev": "7734e2ee6a1472807a33ce1e7da794bed2aaf91c",
         "type": "github"
       },
       "original": {
@@ -397,11 +966,11 @@
     },
     "terranix-examples": {
       "locked": {
-        "lastModified": 1636300201,
-        "narHash": "sha256-0n1je1WpiR6XfCsvi8ZK7GrpEnMl+DpwhWaO1949Vbc=",
+        "lastModified": 1637156952,
+        "narHash": "sha256-KqvXIe1yiKOEP9BRYqNQN+LOWPCsWojh0WjEgv5jfEI=",
         "owner": "terranix",
         "repo": "terranix-examples",
-        "rev": "a934aa1cf88f6bd6c6ddb4c77b77ec6e1660bd5e",
+        "rev": "921680efb8af0f332d8ad73718d53907f9483e24",
         "type": "github"
       },
       "original": {
@@ -409,6 +978,49 @@
         "repo": "terranix-examples",
         "type": "github"
       }
+    },
+    "treefmt-nix": {
+      "inputs": {
+        "nixpkgs": [
+          "buildbot-nix",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1708897213,
+        "narHash": "sha256-QECZB+Hgz/2F/8lWvHNk05N6NU/rD9bWzuNn6Cv8oUk=",
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "rev": "e497a9ddecff769c2a7cbab51e1ed7a8501e7a3a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "type": "github"
+      }
+    },
+    "treefmt-nix_2": {
+      "inputs": {
+        "nixpkgs": [
+          "hydra",
+          "nix-eval-jobs",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1732292307,
+        "narHash": "sha256-5WSng844vXt8uytT5djmqBCkopyle6ciFgteuA9bJpw=",
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "rev": "705df92694af7093dfbb27109ce16d828a79155f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "type": "github"
+      }
     }
   },
   "root": "root",

flake.nix

@@ -2,8 +2,10 @@
   description = "Bagel cooking infrastructure";
 
   inputs = {
-    nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
+
     terranix.url = "github:terranix/terranix";
+    terranix.inputs.nixpkgs.follows = "nixpkgs";
 
     agenix.url = "github:ryantm/agenix";
     agenix.inputs.nixpkgs.follows = "nixpkgs";
@@ -14,38 +16,77 @@
     hydra.url = "git+https://git.lix.systems/lix-project/hydra.git";
     hydra.inputs.nixpkgs.follows = "nixpkgs";
 
-    nix-gerrit.url = "git+https://git.lix.systems/the-distro/nix-gerrit.git";
+    nix-gerrit.url = "git+https://git.lix.systems/the-distro/nix-gerrit.git?ref=refs/heads/bump-minor-3_10";
     nix-gerrit.inputs.nixpkgs.follows = "nixpkgs";
 
-    lix.follows = "hydra/nix";
+    nix-forgejo.url = "git+https://git.lix.systems/the-distro/nix-forgejo.git";
+    nix-forgejo.flake = false;
+
+    ofborg.url = "git+https://git.lix.systems/the-distro/ofborg.git?ref=refs/heads/vcs-generalization";
+    ofborg.flake = false;
+
+    gerrit-dashboard.url = "git+https://git.lix.systems/the-distro/gerrit-monitoring.git";
+    gerrit-dashboard.flake = false;
+
+    buildbot-nix.url = "git+https://git.lix.systems/lix-project/buildbot-nix.git?ref=refs/heads/forkos";
+    buildbot-nix.inputs.nixpkgs.follows = "nixpkgs";
+
+    channel-scripts.url = "git+https://git.lix.systems/the-distro/channel-scripts.git";
+    channel-scripts.inputs.nixpkgs.follows = "nixpkgs";
+
+    stateless-uptime-kuma.url = "git+https://git.dgnum.eu/DGNum/stateless-uptime-kuma.git";
+    stateless-uptime-kuma.flake = false;
+
+    lix.follows = "hydra/lix";
+
+    grapevine = {
+      type = "gitlab";
+      host = "gitlab.computer.surgery";
+      owner = "matrix";
+      repo = "grapevine-fork";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
   };
 
-  outputs = { self, nixpkgs, terranix, colmena, ... } @ inputs:
+  outputs = { self, nixpkgs, terranix, colmena, ofborg, ... } @ inputs:
   let
-    system = "x86_64-linux";
-    pkgs = import nixpkgs {
-      localSystem = system;
-      overlays = [
-        inputs.hydra.overlays.default
-        inputs.lix.overlays.default
-        inputs.nix-gerrit.overlays.default
-      ];
-    };
-    lib = pkgs.lib;
-    terraform = pkgs.opentofu;
-    terraformCfg = terranix.lib.terranixConfiguration {
+    supportedSystems = [ "x86_64-linux" "aarch64-linux" ];
+    forEachSystem = f: builtins.listToAttrs (map (system: {
+      name = system;
+      value = f system;
+    }) supportedSystems);
+    systemBits = forEachSystem (system: rec {
       inherit system;
-      modules = [
-        ./terraform
-        {
-          bagel.gandi.enable = true;
-          bagel.hydra.enable = true;
-        }
-      ];
-    };
+      pkgs = import nixpkgs {
+        localSystem = system;
+        overlays = [
+          inputs.hydra.overlays.default
+          inputs.lix.overlays.default
+          inputs.nix-gerrit.overlays.default
+          inputs.channel-scripts.overlays.default
+          (import inputs.ofborg {
+            pkgs = import nixpkgs { localSystem = system; };
+          }).overlay
+          (import "${inputs.stateless-uptime-kuma}/overlay.nix")
+        ];
+      };
+      terraform = pkgs.opentofu;
+      terraformCfg = terranix.lib.terranixConfiguration {
+        inherit system;
+        modules = [
+          ./terraform
+          {
+            bagel.dnsimple.enable = true;
+            bagel.hydra.enable = true;
+          }
+        ];
+      };
+    });
+    forEachSystem' = f: forEachSystem (system: (f systemBits.${system}));
+    inherit (nixpkgs) lib;
   in
   {
-    apps.${system} = {
+    apps = forEachSystem' ({ system, pkgs, terraformCfg, terraform, ... }: {
       tf = {
         type = "app";
         program = toString (pkgs.writers.writeBash "tf" ''
@@ -56,16 +97,19 @@
       };
 
       default = self.apps.${system}.tf;
-    };
+    });
 
-    devShells.${system}.default = pkgs.mkShell {
-      packages = [
-        inputs.agenix.packages.${system}.agenix
+    devShells = forEachSystem' ({ system, pkgs, ... }: {
+      default = pkgs.mkShell {
+        packages = [
+          inputs.agenix.packages.${system}.agenix
 
-        pkgs.colmena
-        pkgs.opentofu
-      ];
-    };
+          pkgs.opentofu
+
+          (pkgs.callPackage ./lib/colmena-wrapper.nix { })
+        ];
+      };
+    });
 
     nixosConfigurations = (colmena.lib.makeHive self.outputs.colmena).nodes;
 
@@ -73,35 +117,100 @@
       commonModules = [
         inputs.agenix.nixosModules.default
         inputs.hydra.nixosModules.hydra
+        inputs.buildbot-nix.nixosModules.buildbot-coordinator
+        inputs.buildbot-nix.nixosModules.buildbot-worker
 
         ./services
         ./common
       ];
 
-      makeBuilder = i: lib.nameValuePair "builder-${toString i}" {
-        imports = commonModules;
-        bagel.baremetal.builders = { enable = true; num = i; };
+      floralInfraModules = commonModules ++ [
+        ({ config, lib, ... }: {
+          # This means that anyone with @floral-infra permissions
+          # can ssh on root of every machines handled here.
+          bagel.admins.allowedGroups = [
+            "floral-infra"
+          ];
+
+          # Tag all machines which have local boot as local bootables.
+          deployment.tags = lib.mkMerge [
+            [ "floral" ]
+            (lib.mkIf (config.bagel.baremetal.builders.enable -> !config.bagel.baremetal.builders.netboot)
+              [ "localboot" ]
+            )
+          ];
+
+          bagel.monitoring.grafana-agent.tenant = "floral";
+          bagel.secrets.tenant = "floral";
+          bagel.builders.extra-build-capacity.provider.tenant = "floral";
+          bagel.services.buildbot.tenant = "floral";
+        })
+      ];
+
+      # These are Floral baremetal builders.
+      makeBuilder = i:
+      let
+        enableNetboot = i >= 6;
+      in
+      lib.nameValuePair "builder-${toString i}" {
+        imports = floralInfraModules;
+        bagel.baremetal.builders = { enable = true; num = i; netboot = enableNetboot; };
       };
 
-      builders = lib.listToAttrs (lib.genList makeBuilder 12);
+      lixInfraModules = commonModules ++ [
+        {
+          # This means that anyone with @lix-infra permissions
+          # can ssh on root of every machines handled here.
+          bagel.admins.allowedGroups = [
+            "lix-infra"
+          ];
+
+          # Tag all machines which have local boot as local bootables.
+          # Lix has no netbootable machine.
+          deployment.tags = [ "localboot" "lix" ];
+
+          bagel.monitoring.grafana-agent.tenant = "lix";
+          bagel.secrets.tenant = "lix";
+          bagel.builders.extra-build-capacity.provider = {
+            tenant = "lix";
+            buildfarmPublicKeys = [
+              # buildbot.lix.systems SSH key
+              "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDu4cEqZzAI/1vZjSQkTJ4ijIg9nuloOuSKUrnkJIOFn"
+            ];
+          };
+          bagel.services.buildbot.tenant = "lix";
+        }
+      ];
+
+      builders = lib.listToAttrs (map makeBuilder [4 5 10 11]);
     in {
-      meta.nixpkgs = import nixpkgs {
-        localSystem = system;
-        overlays = [
-          inputs.hydra.overlays.default
-          inputs.lix.overlays.default
-          inputs.nix-gerrit.overlays.default
+      meta.nixpkgs = systemBits.x86_64-linux.pkgs;
+      # Add any non-x86_64 native systems here.
+      # Cross compilation is not supported yet.
+      meta.nodeNixpkgs =
+        let
+          aarch64-systems = systems: lib.genAttrs systems (system: systemBits.aarch64-linux.pkgs);
+        in
+        aarch64-systems [
+          "build01-aarch64-lix"
         ];
-      };
       meta.specialArgs.inputs = inputs;
 
-      bagel-box.imports = commonModules ++ [ ./hosts/bagel-box ];
-      meta01.imports = commonModules ++ [ ./hosts/meta01 ];
-      gerrit01.imports = commonModules ++ [ ./hosts/gerrit01 ];
-      fodwatch.imports = commonModules ++ [ ./hosts/fodwatch ];
-      wob-vpn-gw.imports = commonModules ++ [ ./hosts/wob-vpn-gw ];
+      bagel-box.imports = floralInfraModules ++ [ ./hosts/bagel-box ];
+      meta01.imports = floralInfraModules ++ [ ./hosts/meta01 ];
+      gerrit01.imports = floralInfraModules ++ [ ./hosts/gerrit01 ];
+      fodwatch.imports = floralInfraModules ++ [ ./hosts/fodwatch ];
+      git.imports = floralInfraModules ++ [ ./hosts/git ];
+      wob-vpn-gw.imports = floralInfraModules ++ [ ./hosts/wob-vpn-gw ];
+      buildbot.imports = floralInfraModules ++ [ ./hosts/buildbot ];
+      public01.imports = floralInfraModules ++ [ ./hosts/public01 ];
+      build-coord.imports = floralInfraModules ++ [ ./hosts/build-coord ];
+
+      build01-aarch64-lix.imports = lixInfraModules ++ [ ./hosts/build01-aarch64-lix ];
+      buildbot-lix.imports = lixInfraModules ++ [ ./hosts/buildbot-lix ];
     } // builders;
 
-    hydraJobs = builtins.mapAttrs (n: v: v.config.system.build.toplevel) self.nixosConfigurations;
+    hydraJobs = builtins.mapAttrs (n: v: v.config.system.build.netbootDir or v.config.system.build.toplevel) self.nixosConfigurations;
+    buildbotJobs = builtins.mapAttrs (_: v: v.config.system.build.toplevel) self.nixosConfigurations;
   };
 }

hosts/bagel-box/default.nix

@@ -20,6 +20,7 @@
     useHostResolvConf = false;
 
     hostName = "bagel-box";
+    domain = "infra.forkos.org";
     nameservers = [ "2001:4860:4860::8844" ];
 
     interfaces.host0.ipv6.addresses = [
@@ -36,21 +37,27 @@
 
   bagel.services = {
     postgres.enable = true;
+    ofborg = {
+      rabbitmq.enable = true;
+      pastebin.enable = true;
+      # TODO: statcheck.enable = true;
 
-    hydra.enable = true;
-    hydra.dbi = "dbi:Pg:dbname=hydra;user=hydra";
-    # Takes 4 builders (0 → 3).
-    hydra.builders = lib.genList (i: "builder-${builtins.toString i}") 4;
+      mass-rebuilder.enable = true;
+      # TODO: enable once ready.
+      builder.enable = false;
 
-    ofborg.enable = true;
+      gerrit-event-streamer.enable = true;
+      gerrit-generic-vcs-filter.enable = true;
+
+      # FIXME: plug into our prometheus stack.
+      stats.enable = true;
+    };
   };
 
   bagel.sysadmin.enable = true;
 
-  security.acme.acceptTerms = true;
-  security.acme.defaults.email = "infra@forkos.org";
-
   services.openssh.enable = true;
 
+  system.stateVersion = "24.11";
   deployment.targetHost = "bagel-box.infra.forkos.org";
 }

hosts/build-coord/default.nix

@@ -0,0 +1,29 @@
+{ lib, ... }:
+{
+  imports = [ ./hardware.nix ];
+
+  networking.hostName = "build-coord";
+  networking.domain = "wob01.infra.forkos.org";
+
+  bagel.sysadmin.enable = true;
+
+  bagel.services = {
+    hydra.enable = true;
+    hydra.builders = map (i: "builder-${builtins.toString i}") [4 5 10];
+
+    # Arguably, the build-coordinator is the most sensitive piece of our own infrastructure.
+    # Henceforth, it can run as well another sensitive piece of the system: the Vault.
+    vault = {
+      enable = true;
+      domain = "vault.forkos.org";
+    };
+  };
+
+  bagel.monitoring.exporters.hydra.enable = true;
+
+  # Hydra is proxied.
+  bagel.raito.v6-proxy-awareness.enable = true;
+
+  system.stateVersion = "24.05";
+  deployment.targetHost = "build-coord.wob01.infra.forkos.org";
+}

hosts/build-coord/hardware.nix

@@ -0,0 +1,93 @@
+{
+  boot.initrd.availableKernelModules = [ "ahci" "ehci_pci" "usb_storage" "usbhid" "sd_mod" ];
+  boot.initrd.kernelModules = [ "dm-snapshot" ];
+
+  nixpkgs.hostPlatform = "x86_64-linux";
+  hardware.cpu.intel.updateMicrocode = true;
+
+  boot.loader.systemd-boot.enable = true;
+  boot.loader.efi.canTouchEfiVariables = true;
+  boot.initrd.systemd.enable = true;
+
+  boot.initrd.services.lvm.enable = true;
+
+  boot.kernelParams = [
+    "console=tty1"
+    "console=ttyS0,115200"
+  ];
+
+  fileSystems = {
+    "/" = {
+      device = "/dev/disk/by-label/root";
+      fsType = "ext4";
+    };
+
+    "/boot" = {
+      device = "/dev/disk/by-label/BOOT";
+      fsType = "vfat";
+      options = [ "fmask=0022" "dmask=0022" ];
+    };
+  };
+
+  swapDevices = [
+    {
+      device = "/swapfile";
+      size = 20 * 1024;  # 50GiB
+    }
+  ];
+
+  zramSwap = {
+    enable = true;
+    memoryPercent = 100;
+  };
+
+  networking.useNetworkd = true;
+
+  systemd.network = {
+    netdevs = {
+      "40-uplink" = {
+        netdevConfig = {
+          Kind = "bond";
+          Name = "uplink";
+        };
+        bondConfig = {
+          Mode = "802.3ad";
+          TransmitHashPolicy = "layer3+4";
+        };
+      };
+    };
+    networks = {
+      "40-eno1" = {
+        name = "eno1";
+        bond = [ "uplink" ];
+      };
+      "40-eno2" = {
+        name = "eno2";
+        bond = [ "uplink" ];
+      };
+    };
+  };
+  networking.interfaces.uplink.ipv6.addresses = [
+    { address = "2a01:584:11::1:11"; prefixLength = 64; }
+  ];
+  networking.defaultGateway6 = { interface = "uplink"; address = "2a01:584:11::1"; };
+
+  bagel.infra.self.wan = {
+    family = "inet6";
+    address = "2a01:584:11::1:11";
+    prefixLength = 64;
+  };
+
+  services.coredns = {
+    enable = true;
+    config = ''
+      . {
+        bind lo
+        forward . 2001:4860:4860::6464
+        template ANY A { rcode NOERROR }
+      }
+    '';
+  };
+  services.resolved.enable = false;
+  networking.resolvconf.useLocalResolver = true;
+}

hosts/build01-aarch64-lix/default.nix

@@ -0,0 +1,27 @@
+{ ... }: {
+  networking.hostName = "build01";
+  networking.domain = "aarch64.lix.systems";
+
+  # Those free sweet VMs.
+  bagel.hardware.oracle-vm = {
+    enable = true;
+    system = "aarch64-linux";
+  };
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/a333323c-99f0-4258-8f68-496858d56f71";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/3E74-C937";
+      fsType = "vfat";
+    };
+
+  swapDevices = [ ];
+
+  bagel.builders.extra-build-capacity.provider.enable = true;
+  i18n.defaultLocale = "en_US.UTF-8";
+  system.stateVersion = "24.05";
+  deployment.targetHost = "build01.aarch64.lix.systems";
+}

hosts/buildbot-lix/default.nix

@@ -0,0 +1,71 @@
+# Configuration for a virtual machine in Raito's micro-DC basement.
+# 32 vCPU (2014 grade Xeon though)
+# 32GB RAM
+# 30GB SSD
+# 500GB HDD
+# All specifications can be upgraded to a certain extent, just ask Raito.
+# Hosts the coordinator for Buildbot.
+#
+# vim: et:ts=2:sw=2:
+#
+{ lib, modulesPath, ... }: {
+  networking.hostName = "buildbot";
+  networking.domain = "lix.systems";
+
+  zramSwap.enable = true;
+
+  bagel.sysadmin.enable = true;
+  # Buildbot is proxied.
+  bagel.raito.v6-proxy-awareness.enable = true;
+  bagel.hardware.raito-vm = {
+    enable = true;
+    networking = {
+      nat-lan-mac = "BC:24:11:75:62:42";
+      wan = {
+        mac = "BC:24:11:B2:5F:2E";
+        address = "2001:bc8:38ee:100::200/56";
+      };
+    };
+  };
+  i18n.defaultLocale = "en_US.UTF-8";
+
+  bagel.services.buildbot = {
+    enable = true;
+    domain = "buildbot.lix.systems";
+    gerrit =
+    {
+      domain = "gerrit.lix.systems";
+      port = 2022;
+      username = "buildbot";
+    };
+    cors.allowedOrigins = [
+      "https://*.lix.systems"
+    ];
+    projects = [
+      "lix"
+      "lix-installer"
+    ];
+    buildSystems = [
+      "x86_64-linux"
+      "aarch64-linux"
+      "aarch64-darwin"
+      # Too slow.
+      /* "x86_64-darwin" */
+    ];
+    # Lix is not allowed to use yet Floral's x86_64 builders for now.
+    builders = [ ];
+  };
+
+  # This machine does not use /nix from btrfs, and instead uses a store on a bigger disk.
+  fileSystems."/nix" =
+    lib.mkForce
+    { device = "/dev/disk/by-uuid/1815ca49-d0b0-4b99-8aec-0d790498ba6f";
+      fsType = "xfs";
+      neededForBoot = true;
+      options = [ "relatime" ];
+    };
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+  system.stateVersion = "24.05";
+  deployment.targetHost = "buildbot.lix.systems";
+}

hosts/buildbot/default.nix

@@ -0,0 +1,54 @@
+{
+  config,
+  lib,
+  pkgs,
+  nodes,
+  ...
+}:
+{
+  networking.hostName = "buildbot";
+  # TODO: make it the default
+  networking.domain = "infra.forkos.org";
+
+  bagel.sysadmin.enable = true;
+  # Buildbot is proxied.
+  bagel.raito.v6-proxy-awareness.enable = true;
+  bagel.hardware.raito-vm = {
+    enable = true;
+    networking = {
+      nat-lan-mac = "BC:24:11:E7:42:8B";
+      wan = {
+        address = "2001:bc8:38ee:100:1000::50/64";
+        mac = "BC:24:11:C9:BA:6C";
+      };
+    };
+  };
+
+  bagel.services.buildbot = {
+    enable = true;
+    domain = "buildbot.forkos.org";
+    gerrit =
+      let
+        cfgGerrit = nodes.gerrit01.config.bagel.services.gerrit;
+      in
+      {
+        domain = cfgGerrit.canonicalDomain;
+        port = cfgGerrit.port;
+        username = "buildbot";
+      };
+    cors.allowedOrigins = [
+      "https://*.forkos.org"
+    ];
+    projects = [
+      "buildbot-test"
+      "nixpkgs"
+      "infra"
+    ];
+    builders = [ "builder-4" ];
+  };
+
+  i18n.defaultLocale = "en_US.UTF-8";
+
+  system.stateVersion = "24.05";
+  deployment.targetHost = "buildbot.infra.forkos.org";
+}

hosts/fodwatch/default.nix

@@ -8,8 +8,6 @@
   networking.hostName = "fodwatch";
   networking.domain = "infra.forkos.org";
 
-  time.timeZone = "Europe/Paris";
-
   bagel.sysadmin.enable = true;
   # Fodwatch will be proxied.
   bagel.raito.v6-proxy-awareness.enable = true;

hosts/gerrit01/default.nix

@@ -9,8 +9,6 @@
   # TODO: make it the default
   networking.domain = "infra.forkos.org";
 
-  time.timeZone = "Europe/Paris";
-
   bagel.sysadmin.enable = true;
   # Gerrit is proxied.
   bagel.raito.v6-proxy-awareness.enable = true;
@@ -25,6 +23,9 @@
     };
   };
 
+  # Block all these crawlers!!
+  bagel.services.nginx.crawler-blocker.enable = true;
+
   fileSystems."/gerrit-data" = {
     device = "/dev/disk/by-uuid/d1062305-0dea-4740-9a27-b6b1691862a4";
     fsType = "ext4";
@@ -32,12 +33,104 @@
 
   bagel.services.gerrit = {
     enable = true;
+    pyroscope.enable = true;
     domains = [
       "cl.forkos.org"
     ];
+    canonicalDomain = "cl.forkos.org";
     data = "/gerrit-data";
   };
 
+  age.secrets.ows-deploy-key = {
+    file = ../../secrets/floral/ows-deploy-key.age;
+    mode = "0600";
+    owner = "git";
+    group = "git";
+  };
+  bagel.nixpkgs.one-way-sync =
+  let
+    mkNixpkgsJob = { timer, fromRefspec, localRefspec ? fromRefspec }: {
+      fromUri = "https://github.com/NixOS/nixpkgs";
+      inherit fromRefspec localRefspec timer;
+    };
+    mkLocalJob = { timer, fromRefspec, localRefspec }: {
+      fromUri = "https://cl.forkos.org/nixpkgs";
+      inherit fromRefspec localRefspec timer;
+    };
+    in
+  {
+    enable = true;
+
+    stateDirectory = "/gerrit-data/ows";
+
+    pushUrl = "ssh://ows_bot@cl.forkos.org:29418/nixpkgs";
+    deployKeyPath = config.age.secrets.ows-deploy-key.path;
+
+    # Sync main -> staging-next -> staging
+    branches."main-to-staging-next" = mkLocalJob {
+      timer = "00/8:20:00"; # every 8 hours, 20 minutes past the full hour
+      fromRefspec = "main";
+      localRefspec = "staging-next";
+    };
+    branches."staging-next-to-staging" = mkLocalJob {
+      timer = "00/8:40:00"; # every 8 hours, 40 minutes past the full hour
+      fromRefspec = "staging-next";
+      localRefspec = "staging";
+    };
+
+    # Sync nixpkgs -> fork
+    branches."nixpkgs-master" = mkNixpkgsJob {
+      timer = "hourly";
+      fromRefspec = "master";
+      localRefspec = "main";
+    };
+
+    branches."nixpkgs-staging" = mkNixpkgsJob {
+      timer = "hourly";
+      fromRefspec = "staging";
+    };
+
+    branches."nixpkgs-release-24.05" = mkNixpkgsJob {
+      timer = "hourly";
+      fromRefspec = "release-24.05";
+    };
+
+    branches."nixpkgs-staging-24.05" = mkNixpkgsJob {
+      timer = "hourly";
+      fromRefspec = "staging-24.05";
+    };
+
+    branches."nixpkgs-release-23.11" = mkNixpkgsJob {
+      timer = "hourly";
+      fromRefspec = "release-23.11";
+    };
+
+    branches."nixpkgs-staging-23.11" = mkNixpkgsJob {
+      timer = "hourly";
+      fromRefspec = "staging-23.11";
+    };
+  };
+
+  age.secrets.s3-channel-staging-keys.file = ../../secrets/floral/s3-channel-staging-keys.age;
+  bagel.nixpkgs.channel-scripts = {
+    enable = true;
+    otlp.enable = true;
+    nixpkgsUrl = "https://cl.forkos.org/nixpkgs.git";
+    hydraUrl = "https://hydra.forkos.org";
+    binaryCacheUrl = "https://cache.forkos.org";
+    baseUriForGitRevisions = "https://cl.forkos.org/plugins/gitiles/nixpkgs/+";
+    s3 = {
+      release = "bagel-channel-scripts-test";
+      channel = "bagel-channel-scripts-test";
+    };
+    releaseBucketCredentialsFile = config.age.secrets.s3-channel-staging-keys.path;
+    deployKeyFile = config.age.secrets.priv-ssh-key.path;
+    extraArgs = [
+      "--bypass-preflight-checks"
+    ];
+    channels = import ../../common/channels.nix;
+  };
+
   i18n.defaultLocale = "fr_FR.UTF-8";
 
   system.stateVersion = "24.05";

hosts/git/default.nix

@@ -0,0 +1,47 @@
+let
+  ipv6 = {
+    openssh ="2001:bc8:38ee:100:1000::41";
+    forgejo = "2001:bc8:38ee:100:1000::40";
+  };
+in
+{
+  networking.hostName = "git";
+  networking.domain = "infra.forkos.org";
+
+  bagel.sysadmin.enable = true;
+  # Forgejo will be proxied.
+  bagel.raito.v6-proxy-awareness.enable = true;
+  bagel.hardware.raito-vm = {
+    enable = true;
+    networking = {
+      nat-lan-mac = "BC:24:11:83:71:56";
+      wan = {
+        address = "${ipv6.forgejo}/64";
+        mac = "BC:24:11:0B:8A:81";
+      };
+    };
+  };
+
+  # Add one additional IPv6, so we can have both OpenSSH and
+  # Forgejo's built-in server bind on port :22.
+  systemd.network.networks."10-wan".networkConfig.Address = [ "${ipv6.openssh}/64" ];
+  services.openssh.listenAddresses = [{
+    addr = "[${ipv6.openssh}]";
+  }];
+  # Defaults to network.target, but networkd may take a while to settle and set up
+  # the required (additional) IPv6 address, leading to sshd to not being able to
+  # bind to the requested IP, crashing 5 times and running into the default
+  # restart counter limit (5).
+  systemd.services.sshd.wants = [ "network-online.target" ];
+  systemd.services.sshd.after = [ "network-online.target" ];
+
+  bagel.services.forgejo = {
+    enable = true;
+    sshBindAddr = ipv6.forgejo;
+  };
+
+  i18n.defaultLocale = "en_US.UTF-8";
+
+  system.stateVersion = "24.05";
+  deployment.targetHost = "git.infra.forkos.org";
+}

hosts/meta01/default.nix

@@ -2,8 +2,6 @@
   networking.hostName = "meta01";
   networking.domain = "infra.forkos.org";
 
-  time.timeZone = "Europe/Paris";
-
   bagel.sysadmin.enable = true;
   # netbox is proxied.
   bagel.raito.v6-proxy-awareness.enable = true;
@@ -24,6 +22,15 @@
   bagel.services.prometheus.enable = true;
   bagel.services.loki.enable = true;
   bagel.services.grafana.enable = true;
+  bagel.services.grapevine.enable = true;
+  bagel.services.pyroscope.enable = true;
+  bagel.services.tempo.enable = true;
+  bagel.services.hookshot = {
+    enable = true;
+    admins = [
+      "@k900:0upti.me"
+    ];
+  };
 
   i18n.defaultLocale = "fr_FR.UTF-8";
 

hosts/public01/default.nix

@@ -0,0 +1,50 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+{
+  networking.hostName = "public01";
+  # TODO: make it the default
+  networking.domain = "infra.forkos.org";
+
+  bagel.status = {
+    enable = true;
+    domain = "status.forkos.org";
+  };
+
+  bagel.sysadmin.enable = true;
+  # Newsletter is proxied.
+  bagel.raito.v6-proxy-awareness.enable = true;
+  bagel.newsletter = {
+    enable = true;
+    domain = "news.forkos.org";
+  };
+  bagel.hardware.raito-vm = {
+    enable = true;
+    networking = {
+      nat-lan-mac = "BC:24:11:A4:F7:D3";
+      wan = {
+        address = "2001:bc8:38ee:100:1000::60/64";
+        mac = "BC:24:11:DB:B8:10";
+      };
+    };
+  };
+
+  bagel.services.s3-revproxy = {
+    enable = true;
+    domain = "forkos.org";
+    s3.apiUrl = "s3.delroth.net";
+    targets = {
+      channels = "bagel-channels";
+      releases = "bagel-releases";
+      channel-scripts-test = "bagel-channel-scripts-test";
+    };
+  };
+
+  i18n.defaultLocale = "en_US.UTF-8";
+
+  system.stateVersion = "24.05";
+  deployment.targetHost = "public01.infra.forkos.org";
+}

hosts/wob-vpn-gw/default.nix

@@ -1,6 +1,10 @@
 { pkgs, lib, ... }:
 
 {
+  imports = [
+    ./netboot.nix
+  ];
+
   ###### Hardware ######
   boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "ehci_pci" "sd_mod" "sdhci_pci" ];
   boot.kernelModules = [ "kvm-amd" ];

hosts/wob-vpn-gw/netboot.nix

@@ -0,0 +1,61 @@
+{ config, lib, pkgs, nodes, modulesPath, ... }:
+
+# The way the connection is established is specific to the wob01 site and the Intel S2600KPR blades.
+# Proper netboot is not possible, because while the blades and the APU board (which is the netboot
+# server here) are in the same L2 network, the uplink connection of each blade is an LACP LAG,
+# meaning that the switch on the other side will only enable the port if it sees valid LACP packets.
+# We work around this by presenting a virtual floppy drive using the "IUSB" protocol of the BMC.
+# This virtual floppy drive contains an per-blade customized initramfs which will initialize the
+# network connection including IP configuration and load the actual image off hydra.
+
+let
+  netboot-server-ip = "2a01:584:11::2";
+  netbootNodes = lib.filterAttrs (_: node: node.config.bagel.baremetal.builders.enable && node.config.bagel.baremetal.builders.netboot) nodes;
+in {
+
+  assertions = [
+    {
+      assertion = !(lib.elem 443 config.networking.firewall.allowedTCPPorts);
+      message = ''
+        Port 443 is in networking.firewalls.allowedTCPPorts, but should be only manually
+        allowed for specific IPs and source ports in ${builtins.toJSON __curPos}
+      '';
+    }
+  ];
+
+  systemd.services = lib.mapAttrs' (nodename: node: let
+    bmcIp = "192.168.1.${toString (node.config.bagel.baremetal.builders.num * 4 + 2)}";
+    notipxe = node.config.system.build.notipxe.config.system.build.usbImage;
+  in lib.nameValuePair "iusb-spoof-${nodename}" {
+    wantedBy = [ "multi-user.target" ];
+    serviceConfig = {
+      Restart = "always";
+    };
+    script = ''
+      AUTH_TOKEN=$(${pkgs.iusb-spoof}/bin/make-token ${bmcIp})
+      exec ${pkgs.iusb-spoof}/bin/iusb-spoof -r ${bmcIp} 5123 $AUTH_TOKEN ${notipxe}
+    '';
+  }) netbootNodes;
+
+  # Since the builders are stateless, they can not store their ssh hostkeys
+  networking.firewall.allowedTCPPorts = [ 80 ]; # for ACME
+  networking.firewall.extraInputRules = ''
+    ip6 saddr 2a01:584:11::/64 tcp sport < 1024 tcp dport 443 accept;
+  '';
+  services.nginx = {
+    enable = true;
+    virtualHosts."vpn-gw.wob01.infra.forkos.org" = {
+      enableACME = true;
+      forceSSL = true;
+      locations = lib.mapAttrs' (nodename: node: let
+        ip = "2a01:584:11::1:${toString node.config.bagel.baremetal.builders.num}";
+      in lib.nameValuePair "/${nodename}/" {
+        root = "/var/www";
+        extraConfig = ''
+          allow ${ip};
+          deny all;
+        '';
+      }) netbootNodes;
+    };
+  };
+}

lib/colmena-wrapper.nix

@@ -0,0 +1,14 @@
+# A wrapper for colmena that prevents accidentally deploying changes without
+# having pulled.
+{ colmena, runCommandNoCC }:
+runCommandNoCC "colmena-wrapper"
+{
+  env.colmena = "${colmena}/bin/colmena";
+} ''
+  mkdir -p $out
+  ln -s ${colmena}/share $out/share
+  mkdir $out/bin
+
+  substituteAll ${./colmena-wrapper.sh.in} $out/bin/colmena
+  chmod +x $out/bin/colmena
+''

lib/colmena-wrapper.sh.in

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+doChecks() {
+    # creates refs in the refs/prefetch/remotes/origin namespace
+    echo "Prefetching repo changes..." >&2
+    git fetch --quiet --prefetch --no-write-fetch-head origin
+
+    diffs=$(git rev-list --left-right --count HEAD...refs/prefetch/remotes/origin/main)
+    only_in_local=$(echo "$diffs" | cut -f1)
+    only_in_main=$(echo "$diffs" | cut -f2)
+
+    if [[ $only_in_main -gt 0 && ! -v $FOOTGUN_ME_UWU ]]; then
+        echo >&2
+        echo "Attempting to deploy when main has $only_in_main commits not in your branch!" >&2
+        echo "This will probably revert someone's changes. Consider merging them." >&2
+        echo "If you really mean it, set the environment variable FOOTGUN_ME_UWU" >&2
+        exit 1
+    fi
+
+    if [[ $only_in_local -gt 0 ]]; then
+        echo "You have $only_in_local commits not yet pushed to main. Reminder to push them after :)" >&2
+    fi
+}
+
+if [[ $1 == 'apply' ]]; then
+    doChecks
+fi
+
+exec @colmena@ "$@"

overlays/default.nix

@@ -1 +1,9 @@
-[]
+[
+  (final: prev: {
+    iusb-spoof = final.callPackage ./iusb-spoof.nix {};
+    u-root = final.callPackage ./u-root {};
+    pyroscope = final.callPackage ./pyroscope {};
+    s3-revproxy = final.callPackage ./s3-revproxy {};
+    git-gc-preserve = final.callPackage ./git-gc-preserve {};
+  })
+]

overlays/git-gc-preserve/default.nix

@@ -0,0 +1,9 @@
+{ writeShellApplication, git, nettools }:
+
+writeShellApplication {
+  name = "git-gc-preserve";
+
+  runtimeInputs = [ git nettools ];
+
+  text = (builtins.readFile ./script.sh);
+}

overlays/git-gc-preserve/script.sh

@@ -0,0 +1,132 @@
+#!/usr/bin/env bash
+
+set +o errexit
+# Copyright (C) 2022 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+usage() { # exit code
+  cat <<-EOF
+NAME
+    git-gc-preserve - Run git gc and preserve old packs to avoid races for JGit
+SYNOPSIS
+    git gc-preserve
+DESCRIPTION
+    Runs git gc and can preserve old packs to avoid races with concurrently
+    executed commands in JGit.
+    This command uses custom git config options to configure if preserved packs
+    from the last run of git gc should be pruned and if packs should be preserved.
+    This is similar to the implementation in JGit [1] which is used by
+    JGit to avoid errors [2] in such situations.
+    The command prevents concurrent runs of the command on the same repository
+    by acquiring an exclusive file lock on the file
+      "\$repopath/gc-preserve.pid"
+    If it cannot acquire the lock it fails immediately with exit code 3.
+    Failure Exit Codes
+        1: General failure
+        2: Couldn't determine repository path. If the current working directory
+           is outside of the working tree of the git repository use git option
+           --git-dir to pass the root path of the repository.
+           E.g.
+              $ git --git-dir ~/git/foo gc-preserve
+        3: Another process already runs $0 on the same repository
+    [1] https://git.eclipse.org/r/c/jgit/jgit/+/87969
+    [2] https://git.eclipse.org/r/c/jgit/jgit/+/122288
+CONFIGURATION
+    "pack.prunepreserved": if set to "true" preserved packs from the last gc run
+      are pruned before current packs are preserved.
+    "pack.preserveoldpacks": if set to "true" current packs will be hard linked
+      to objects/pack/preserved before git gc is executed. JGit will
+      fallback to the preserved packs in this directory in case it comes
+      across missing objects which might be caused by a concurrent run of
+      git gc.
+EOF
+  exit "$1"
+}
+# acquire file lock, unlock when the script exits
+lock() { # repo
+  readonly LOCKFILE="$1/gc-preserve.pid"
+  test -f "$LOCKFILE" || touch "$LOCKFILE"
+  exec 9> "$LOCKFILE"
+  if flock -nx 9; then
+    echo -n "$$ $USER@$(hostname)" >&9
+    trap unlock EXIT
+  else
+    echo "$0 is already running"
+    exit 3
+  fi
+}
+unlock() {
+  # only delete if the file descriptor 9 is open
+  if { : >&9 ; } &> /dev/null; then
+    rm -f "$LOCKFILE"
+  fi
+  # close the file handle to release file lock
+  exec 9>&-
+}
+# prune preserved packs if pack.prunepreserved == true
+prune_preserved() { # repo
+  configured=$(git --git-dir="$1" config --get pack.prunepreserved)
+  if [ "$configured" != "true" ]; then
+    return 0
+  fi
+  local preserved=$1/objects/pack/preserved
+  if [ -d "$preserved" ]; then
+    printf "Pruning old preserved packs: "
+    count=$(find "$preserved" -name "*.old-pack" | wc -l)
+    rm -rf "$preserved"
+    echo "$count, done."
+  fi
+}
+# preserve packs if pack.preserveoldpacks == true
+preserve_packs() { # repo
+  configured=$(git --git-dir="$1" config --get pack.preserveoldpacks)
+  if [ "$configured" != "true" ]; then
+    return 0
+  fi
+  local packdir=$1/objects/pack
+  pushd "$packdir" >/dev/null || exit 1
+  mkdir -p preserved
+  printf "Preserving packs: "
+  count=0
+  for file in pack-*{.pack,.idx} ; do
+    ln -f "$file" preserved/"$(get_preserved_packfile_name "$file")"
+    if [[ "$file" == pack-*.pack ]]; then
+      ((count++))
+    fi
+  done
+  echo "$count, done."
+  popd >/dev/null || exit 1
+}
+# pack-0...2.pack to pack-0...2.old-pack
+# pack-0...2.idx to pack-0...2.old-idx
+get_preserved_packfile_name() { # packfile > preserved_packfile
+  local old=${1/%\.pack/.old-pack}
+  old=${old/%\.idx/.old-idx}
+  echo "$old"
+}
+# main
+while [ $# -gt 0 ] ; do
+    case "$1" in
+        -u|-h)  usage 0 ;;
+    esac
+    shift
+done
+args=$(git rev-parse --sq-quote "$@")
+repopath=$(git rev-parse --git-dir)
+if [ -z "$repopath" ]; then
+  usage 2
+fi
+lock "$repopath"
+prune_preserved "$repopath"
+preserve_packs "$repopath"
+git gc ${args:+"$args"} || { EXIT_CODE="$?"; echo "git gc failed"; exit "$EXIT_CODE"; }

overlays/iusb-spoof.nix

@@ -0,0 +1,23 @@
+{ rustPlatform, python3, makeWrapper }:
+let
+  pythonEnv = python3.withPackages (p: with p; [ requests ]);
+in
+
+rustPlatform.buildRustPackage rec {
+  pname = "iusb-spoof";
+  version = "0.1.0";
+
+  src = builtins.fetchGit {
+    url = "https://git.lix.systems/the-distro/iusb-spoof/";
+    rev = "fafd47986239cc2f4dfbbae74b17555608806581";
+  };
+
+  cargoLock.lockFile = src + "/Cargo.lock";
+
+  nativeBuildInputs = [ makeWrapper ];
+
+  postInstall = ''
+    install -Dm644 $src/make-token.py $out/opt/make-token.py
+    makeWrapper ${pythonEnv.interpreter} $out/bin/make-token --add-flags "$out/opt/make-token.py"
+  '';
+}

overlays/pyroscope/default.nix

@@ -0,0 +1,43 @@
+{ lib
+, buildGo122Module
+, fetchFromGitHub
+}:
+
+# FIXME: update, remove this pin
+buildGo122Module rec {
+  pname = "pyroscope";
+  version = "1.7.1";
+
+  src = fetchFromGitHub {
+    owner = "grafana";
+    repo = "pyroscope";
+    rev = "v${version}";
+    hash = "sha256-iMP67J0Q8Cgo52iImMzAM3PEkk6uLF7r6v9TyXZVaIE=";
+  };
+
+  env.GOWORK = "off";
+
+  vendorHash = "sha256-ggntpnU9s2rpkv6S0LnZNexrdkBsdsUrGPc93SVrK4M=";
+
+  subPackages = [ "cmd/profilecli" "cmd/pyroscope" ];
+
+  ldflags = [
+    "-extldflags"
+    "-static"
+    "-s"
+    "-w"
+    "-X=github.com/grafana/pyroscope/pkg/util/build.Branch=${src.rev}"
+    "-X=github.com/grafana/pyroscope/pkg/util/build.Version=${version}"
+    "-X=github.com/grafana/pyroscope/pkg/util/build.Revision=${src.rev}"
+    "-X=github.com/grafana/pyroscope/pkg/util/build.BuildDate=1970-01-01T00:00:00Z"
+  ];
+
+  meta = with lib; {
+    description = "Continuous profiling platform";
+    homepage = "https://github.com/grafana/pyroscope";
+    changelog = "https://github.com/grafana/pyroscope/blob/${src.rev}/CHANGELOG.md";
+    license = licenses.agpl3Only;
+    maintainers = with maintainers; [ raitobezarius ];
+    mainProgram = "pyroscope";
+  };
+}

overlays/s3-revproxy/default.nix

@@ -0,0 +1,39 @@
+# Originally written by Jade Lovelace for Lix.
+{ lib, buildGoModule, fetchFromGitHub }:
+buildGoModule rec {
+  pname = "s3-revproxy";
+  version = "4.15.0";
+
+  src = fetchFromGitHub {
+    owner = "oxyno-zeta";
+    repo = "s3-proxy";
+    rev = "v${version}";
+    hash = "sha256-q0cfAo8Uz7wtKljmSDaJ320bjg2yXydvvxubAsMKzbc=";
+  };
+
+  vendorHash = "sha256-dOwNQtTfOCQcjgNBV/FeWdwbW9xi1OK5YD7PBPPDKOQ=";
+
+  ldflags = [
+    "-X github.com/oxyno-zeta/s3-proxy/pkg/s3-proxy/version.Version=${version}"
+    "-X github.com/oxyno-zeta/s3-proxy/pkg/s3-proxy/version.Metadata="
+  ];
+
+  postPatch = ''
+    # Refer to the included templates in the package instead of cwd-relative
+    sed -i "s#Path = \"templates/#Path = \"$out/share/s3-revproxy/templates/#" pkg/s3-proxy/config/config.go
+  '';
+
+  postInstall = ''
+    mkdir -p $out/share/s3-revproxy
+    cp -r templates/ $out/share/s3-revproxy/templates
+  '';
+
+  meta = {
+    description = "S3 Reverse Proxy with GET, PUT and DELETE methods and authentication (OpenID Connect and Basic Auth)";
+    homepage = "https://oxyno-zeta.github.io/s3-proxy";
+    # hm, not having a maintainers entry is kind of inconvenient
+    maintainers = [ ];
+    licenses = lib.licenses.asl20;
+    mainProgram = "s3-proxy";
+  };
+}

overlays/u-root/default.nix

@@ -0,0 +1,20 @@
+{ buildGoModule, fetchFromGitHub }:
+
+buildGoModule rec {
+  pname = "u-root";
+  version = "0.14.0";
+
+  src = fetchFromGitHub {
+    owner = "u-root";
+    repo = "u-root";
+    rev = "v${version}";
+    hash = "sha256-8zA3pHf45MdUcq/MA/mf0KCTxB1viHieU/oigYwIPgo=";
+  };
+
+  patches = [
+    ./u-root-allow-https.patch
+  ];
+
+  vendorHash = null;
+  doCheck = false;
+}

overlays/u-root/u-root-allow-https.patch

@@ -0,0 +1,12 @@
+diff --git a/pkg/curl/schemes.go b/pkg/curl/schemes.go
+index 8bac3bc0..cd396cbc 100644
+--- a/pkg/curl/schemes.go
++++ b/pkg/curl/schemes.go
+@@ -81,6 +81,7 @@ var (
+ 	DefaultSchemes = Schemes{
+ 		"tftp": DefaultTFTPClient,
+ 		"http": DefaultHTTPClient,
++		"https": DefaultHTTPClient,
+ 		"file": &LocalFileClient{},
+ 	}
+ )

secrets.nix

@@ -1,26 +1,78 @@
 let
   keys = import common/ssh-keys.nix;
 
-  commonKeys = keys.users.delroth ++ keys.users.raito;
+  commonKeys = {
+    # WARNING: `keys.users.*` are *lists*, so you need concatenate them, don't put them into lists!
+    # Otherwise, agenix will be confused!
+    global = keys.users.raito;
+    lix = keys.users.hexchen ++ keys.users.jade;
+    floral = keys.users.delroth;
+  };
 
   secrets = with keys; {
-    hydra-s3-credentials = [ machines.bagel-box ];
-    hydra-signing-priv = [ machines.bagel-box ];
-    hydra-ssh-key-priv = [ machines.bagel-box ];
-    netbox-environment = [ machines.meta01 ];
-    mimir-environment = [ machines.meta01 ];
-    grafana-oauth-secret = [ machines.meta01 ];
-    loki-environment = [ machines.meta01 ];
-    gerrit-prometheus-bearer-token = [ machines.gerrit01 machines.meta01 ];
+    floral = {
+      hydra-postgres-key = [ machines.build-coord ];
+      hydra-s3-credentials = [ machines.build-coord ];
+      hydra-signing-priv = [ machines.build-coord ];
+      hydra-ssh-key-priv = [ machines.build-coord ];
 
-    # These are the same password, but nginx wants it in htpasswd format
-    metrics-push-htpasswd = [ machines.meta01 ];
-    metrics-push-password = builtins.attrValues machines;
+      netbox-environment = [ machines.meta01 ];
+      mimir-environment = [ machines.meta01 ];
+      mimir-webhook-url = [ machines.meta01 ];
+      grafana-oauth-secret = [ machines.meta01 ];
+      loki-environment = [ machines.meta01 ];
+      gerrit-prometheus-bearer-token = [ machines.gerrit01 machines.meta01 ];
+      pyroscope-secrets = [ machines.meta01 ];
+      tempo-environment = [ machines.meta01 ];
+
+      buildbot-worker-password = [ machines.buildbot ];
+      buildbot-oauth-secret = [ machines.buildbot ];
+      buildbot-workers = [ machines.buildbot ];
+      # Private SSH key to Gerrit
+      # ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHx52RUPWzTa2rBA96xcnGjjzAboNN/hm6gW+Q6JiSos
+      buildbot-service-key = [ machines.buildbot ];
+      # Signing key for Buildbot's specific cache
+      buildbot-signing-key = [ machines.buildbot ];
+      buildbot-remote-builder-key = [ machines.buildbot ];
+
+      # These are the same password, but nginx wants it in htpasswd format
+      metrics-push-htpasswd = [ machines.meta01 ];
+      # Yes, even Lix machines are included in this monitoring infrastructure.
+      metrics-push-password = builtins.attrValues machines;
+
+      ows-deploy-key = [ machines.gerrit01 ];
+      s3-channel-staging-keys = [ machines.gerrit01 ];
+      s3-channel-keys = [ machines.gerrit01 ];
+
+      postgres-ca-priv = [ machines.bagel-box ];
+      postgres-tls-priv = [ machines.bagel-box ];
+      rabbitmq-password = [ machines.bagel-box ];
+      gerrit-event-listener-ssh-key = [ machines.bagel-box ];
+
+      newsletter-secrets = [ machines.public01 ];
+      s3-revproxy-api-keys = [ machines.public01 ];
+      stateless-uptime-kuma-password = [ machines.public01 ];
+    };
+
+    lix = {
+      buildbot-worker-password = [ machines.buildbot-lix ];
+      buildbot-oauth-secret = [ machines.buildbot-lix ];
+      buildbot-workers = [ machines.buildbot-lix ];
+      # Private SSH key to Gerrit
+      # ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHx52RUPWzTa2rBA96xcnGjjzAboNN/hm6gW+Q6JiSos
+      buildbot-service-key = [ machines.buildbot-lix ];
+      # Signing key for Buildbot's specific cache
+      buildbot-signing-key = [ machines.buildbot-lix ];
+      buildbot-remote-builder-key = [ machines.buildbot-lix ];
+    };
   };
+
+  mkSecretListFor = tenant:
+    map (secretName: {
+      name = "secrets/${tenant}/${secretName}.age";
+      value.publicKeys = secrets.${tenant}."${secretName}" ++ commonKeys.global ++ commonKeys.${tenant};
+    }) (builtins.attrNames secrets.${tenant});
 in
   builtins.listToAttrs (
-    map (secretName: {
-      name = "secrets/${secretName}.age";
-      value.publicKeys = secrets."${secretName}" ++ commonKeys;
-    }) (builtins.attrNames secrets)
+    (mkSecretListFor "floral") ++ (mkSecretListFor "lix")
   )

secrets/floral/buildbot-oauth-secret.age

@@ -0,0 +1,20 @@
+age-encryption.org/v1
+-> ssh-ed25519 87T2Ig tzPD1x6XKuDfgJ8jkQnwW/ALp2pkANCeNoO8xdUqq30
+QSsuO6Dwc8QJuY92gXRnWB5aJ2SU9X2uFh01GmLVaQE
+-> ssh-ed25519 K3b7BA 9G9Uw1xY8hq//xphNWrPn5y7vG2o8/kwkC8cJGuf/mI
+Ip0019OUaFq2ZDFI3i77hdsp9IqFV2qqYIB/TnDSXgo
+-> ssh-ed25519 +qVung dx22ef+x9X5mr73L8NUzxYQa640M2XViELjJcpgF3go
+CXyit7pk8SPNHBgULlMQUAasGAn4C36zcwOBDI46nU4
+-> ssh-rsa krWCLQ
+NlGh0hM10NOuek7MbrFo0iul0kQQtDFmZIhgpyqaATMdCDRBXJOyhASHU5N0zDDJ
+MLaJUV0l2o1ghBF9RhSKdoUPVEn8Cce/nfQepYzMlfc4UG3qWXwabwR6EtqqCZCJ
+jAEWZ8taTKDmzoXwuygCW+bRBuoMMrcfzu7V90N+mQpZWtOScatb6E7d5VRqjlar
+st1ZQu5ccghufyQSUmOC7GpojOyutX5EvbMGn84X4ouZRHRX/8fTgaqicV+aeAIb
+QyXisOrO6C+Jle5qfxzMSe8c/TCyF2574kD6F1BQ9Kpkinn8v7OWcIXtkNmZ5hzK
+vs0Bej8yZVsoBkj1vWAM0A
+-> ssh-ed25519 /vwQcQ n+hr1cV1zRs1S86YnA+0oRB8SCaPKtkoMNe15ZsVVwM
+fdFtUqno07ik6FpW5zMImIjd8wM8dMgwU+RqjeT2PiI
+-> ssh-ed25519 0R97PA ddPILw57gkuKvAqlmpa+MnV/LSEdyQzQaAarCUqQ1xE
+ozK5a6uXZDc17OrX0OZun9hmZwP3H3rYQiNuKnukqsg
+--- f7yGgKQpCPj64Ps0HfMcToYircGH5SPqMzVZrUMB8ZI
+føv[iY\ÅšMP,¯Ùh°Èxb—Ðÿ«J<C2AB>*ºË"”+¬ÒA0T˜?KmˆPÈË2¹'2±‚µ³Ø=¯êÚÏŸj”

secrets/floral/buildbot-worker-password.age

@@ -0,0 +1,20 @@
+age-encryption.org/v1
+-> ssh-ed25519 87T2Ig p8lEB5da4fIfLH/HKBsghzq5mvQLB69UB4+uAi3DGCw
+NeZ3jPTUKa7MiqjrFPrYuP4VneytQPdBNqf+omPZJYM
+-> ssh-ed25519 K3b7BA uP2K1hU7uLmiHXmmoUdsB7CHQq61ZkEAjG/aK863RDw
+0chTczEMXASdYiwqNxDQ+vMXXhjOf64oIQ2ULZmQI8Y
+-> ssh-ed25519 +qVung jUgEqz3+ypL7mwJ1R7lfeOMhkon/aRrNSJUJT3X7vmU
+pgOiwrp9JiA20yw9bsxi8eiQ9/23CYXKRBGF1pea9eI
+-> ssh-rsa krWCLQ
+snCHrLHzkjimwIxKO90IjnHwOArlozO9kd/aCdZZnYNgh/QG3rUSceSn9yTHbtMV
+izv0SU51LrRU+JyE+a524AxKhyPBvGDig20j7hMy5fVxZqeunztqtlha5gaYYaQg
+Tbfs9tDP+pCIgzMVNqYf6EJ4MK7qjNf9DE5I490Eta5YZxAi/3To3BmZmIYtCz6l
+1kNRiSmWCbZqE25keFgPCgRMFXAFK9W6NmL+HamqCUhjPoJg/Gd4sf39EONT0PYg
+7BpCOAnwwfECHPxpM3qv0h2kJXTb4DZ715cFReSVyQe5fvKv8hoWhl/S+++pEYT8
+u/LKBx/o7e3Kd7cm2RGnBw
+-> ssh-ed25519 /vwQcQ 4+IQPRsMMHmuSGL7T7IbRkTTuL+TTqgdQp5FSbyt8Dw
+KOI0LKQ0oA5XtxaW7wftlEJB0BGVnx41HUJMG92SRUA
+-> ssh-ed25519 0R97PA l1aWUEv8nLEtYnpY1gjTJqk5UYm51NDqOjYmL83rZ10
+B7qDZwCpolkIajqCXeOepwmF6ciJfKvr+AN7VouMUvA
+--- lz/IMMPxBpD3Bzuv9Wl23+swBQHlblhlAO/ZXAgN0hU
+µoÍüÌ<EFBFBD>²-‚Īr °eó|Í?ït
èìÎZ<C38E>¬sÒì!ŸƒÁ<>@Ï'–ìèz6UöÎg
JøÑOµ–s13<31>š‹8<î’%-·Ô‡Eÿ}–Šdm9¿å¢Óoæ

secrets/floral/grafana-oauth-secret.age

@@ -0,0 +1,20 @@
+age-encryption.org/v1
+-> ssh-ed25519 j2r2qQ vwcaLpvGJ9swXnV8idDwi9jdRPSj38As9p2QFkIJ1Xc
+FLnZeblHDQQcWjFm1iaghbvuFgOG3miwtkRE5sz1+X0
+-> ssh-ed25519 K3b7BA 9VRe2rBwg3G9lxxfxL/yLob2NZmLJTBMxzx0Ew8VwmY
+/I2W80UykNvll5o98OPeMpIsddOel9B7uQlio0X3gcs
+-> ssh-ed25519 +qVung VsqKzMD85aps4PIx2zqae2Dj7YWibiaKYb5z7ws8ggM
+Y9dRd/hOz8h4avlutBQ1YZgHIAf/AuTr5WaByKlFbLE
+-> ssh-rsa krWCLQ
+gjyaUFrIIbZnFTGVw4XEZzkTIP/+qXV6/q0W8Wb4EtqQXDRISFT+bwxQU/S2p5hf
+7+JGcn4BZg6puOJ5BBABWtpn6gcX5OFfga5azIdioF/R19XByT+0SK5njw8g1VPS
+R7o8kQt2yvKWayoq9Cis5XRg+4KANkwOQaNTO8AdiCwgq9nc0Cd9avk8QhaFoR74
+D5cf8jPsufp744rQqwhWDoG533LS1WUUuYZqRmtp2Vz+r583RhSscaNyA7ddr7o6
+e9ZQJyL5bKiN8qe3Xm76lLypf/wg7+aGn8HHnO6GA65g+VYfjLMODEqCN/+uDJtB
+g8v2wzKIGYlZiV1hEjH8nw
+-> ssh-ed25519 /vwQcQ 4pU5JGK5vpZbFgq01a9YY8VmSJvPSHPSZD50TLJwKHc
+L46UA/p+bNSR8cLmL8G7VpmAcZ+sy5AROc4yj2ABOWg
+-> ssh-ed25519 0R97PA Tk00kYLhsEy1HJcmKLgaLWTdNP8XV/cdKHMLzyK6glk
+kwyQZr/h6MutROJmjVfPWGcf9xN5Uc5w5mVyuKcK64g
+--- E0vVtBqbjNkZY0/1dFJ53uVAR7IGPO+OMmXkpJcKmlw
+{ÐQê%è‹õY•B,isr¥1‘<31>|¼yLÕ'7?¶iŠM…¶MU]×ê/d2¸I1u¶2hZjHåh&¥

secrets/floral/hydra-s3-credentials.age

@@ -0,0 +1,20 @@
+age-encryption.org/v1
+-> ssh-ed25519 +uvEmw Kuduppyhz98frjlV96R/WcmPdaWmHbNKZhQs76GGTx8
+7zpedlPflGOi6FqkpswAJflx77yde7M2XlTw/8mz2tU
+-> ssh-ed25519 K3b7BA TPNmUK38+TR58MpsgxIe91bY6E1j9HecIFn0AKdat20
+MjXh06xd3mkPcK+iEonx+itsHvEGHSknzO6Sgh5WdZ4
+-> ssh-ed25519 +qVung KereHQ3Kl0f7O0xzl2s4Yu4KX7OOA17R7p/w8uRx/AQ
+3aOqUoBCDurkh5jT2fq5MDPQKIDISQdXBhF+qeRppnA
+-> ssh-rsa krWCLQ
+TVlmM2LYR339Aglo2D4j/Itr0E+mg7UEaV9n+sUYyit0phWS1zMI2YMc78Xbmn73
+6U0VYi/3hpesD6/8uA5sywuueOMntlL32aECz/DJPC71feMjvHTxiJpqnFw6DQYJ
+FvERtvJ2U7QiStv6UeS1vOucP1/om0Qj4smTXBWYsDglTSLx56/bghCsM21RNZZb
+yd8JE5CEdtCHduj+uRHbnEYsnGYM7R/Gw9XAuajFLw6BxqEtHi5xOivQ2P0Tm+Bc
+SVHW48iF8S3q1tx5QU7oIMZcCobOeHb6w+C1GHiSeJy3R8hWkEwfNxCCc0rSItKd
+edqO4YPz/zT2DWoUx+n0Og
+-> ssh-ed25519 /vwQcQ TLa0Xty2LlqBiP9Lk0lC+S/BoVT+VbRhY3qPHIGf20E
+3mzqkwT8dvP11GAVJiVIc+MiN/pLP2b6KbC+1F86tg8
+-> ssh-ed25519 0R97PA pbGz7e6nU4M4cpJRmmxWxUV3O2rWytIP18M7OpMpa04
+doSBv72rqS5gNusMjKw8KwzXHbzoLlFUSdLqp/f5aRM
+--- beE6zNg+kY7jke/79FGZoNTq7Wbe3eqNWvLD3igQJdg
+bÓÿ¾ ¾ÿÔáïØÉ
mÿŠ$Ÿ–‡¸’Ý={6
eÿ¨SIîumT8”äÿÁ{òo<C3B2>Ë3Ý&¾12¡p¶	œZ÷³4zx¡8B˜¤a@â(_Þ7(‚Ñ^tµä±Á‘<C381>g“Ùà4Ö*&Á`*
øäNšB÷$ÉÆ	ò£§m<C2A7>"ÓtD$Î5‡k
u˜eÎ+X@LJüç£h2ü£Þ*H¥ ÂåaÚ¢

secrets/floral/hydra-signing-priv.age

@@ -0,0 +1,20 @@
+age-encryption.org/v1
+-> ssh-ed25519 +uvEmw TNYFQxSUv5yMmlTWoIxCOlv6UR+RA50cb5aJbo0yEE0
+Yw3sTPqYf7A33RI87CqoPWe2gh0FuvdBGGKqHV55Atc
+-> ssh-ed25519 K3b7BA vMlHenY6jSIfnxQD6xh09cwwV+YVBkLuSMHcyKD+dCk
+heXkAEqRawBlHqcr6ldmhWmk7qPtGLMDFC3QT79vdMM
+-> ssh-ed25519 +qVung fgimLW5X0z4Eh2u3fIr5bgR5/c1SKam9CKW/2mqtTik
+8VKJJr+FRE0j5YvjfdMXugNA4UwUebKrkeAe+9LYBnQ
+-> ssh-rsa krWCLQ
+sa3fA4GglovY8H6jimpTvQPW/axun8WADPlIXzpX/Zeshkzem+pQoQqptzDlnmH8
+8AngqXgFYrmHgNNAylavgcrxbjNrtlJU24ldF1YIubz7VsU1678F27LCd9B0c2dn
+X+0CccH19lM8Q+zVI2Wrq9R83MEP/5uOOc+eXXnvNSGqfKgZ2OplG/HUllFS13j6
+uiQy5zwJJKkII7KUThcGteux7NONoLeUqRE8CW2uSeY9fXBWKgxeENKgiT7PEAAo
+nvwWa+GatEYf6eUz8Lph8lETorgP+7JS2VQRAkmhDbjQLTYzfFmiJGE/mzyobslf
+ZEq6Oj5UNgnzdWmK5ZYKPg
+-> ssh-ed25519 /vwQcQ 9EG/cydlzlLd6cFed7DzmwzubzJUXvD9mX3WKDyFD1s
+3Emj+tVZmnsC/YZdChvyaxeObbBsri347vZl0ff9kH4
+-> ssh-ed25519 0R97PA kcIYyWKxpJmjcrel+YodZQiR2zGPqfjzMyJXsz2XOzM
+SUlgGGs2BVRzTHT/ULNo1AiN5SY1BETFtJRY6LDr4JI
+--- l87sO6IuwSeCeQ8ktvYFI0xr4Utcl8KfpAV7WePc1y4
+÷ÚÖ
~÷J3¦Œ§Í‘1íÇè²ù<º?%×ý<0E>›Á‡ÉÒ\—\Ï7\»Ú¨åU-&W'd”{իɼ½u	"Û#Ž}¯õ…x–ìšz¥®Nj„!éfUqDG‘<é<7F>Ñca<63>‚'´+	¸Ï±]Pó»ö€D
Õ¸’´þ<C2B4>£¼­ŽÿÂçÌ\¦<>

secrets/floral/hydra-ssh-key-priv.age

@@ -0,0 +1,22 @@
+age-encryption.org/v1
+-> ssh-ed25519 +uvEmw BoYFUISKrlypCBQW/fA9UNRSnxQ93FkQupWUWAeOd3c
+wQH9gNk8TtjOgrwYwCuedPBbmftd7JhJk53ga2qo2iE
+-> ssh-ed25519 K3b7BA vyQIsvbNrHI0Mui5UdLz2mWcYvnTQBupWiAfEP5NfXk
+WeHg0PyfuaSJVzuiBPa1Tanj4NdqHvnZFWhXhIgbWc0
+-> ssh-ed25519 +qVung LneEmMhBqJxN0bgM7/Z+jJ8U8MJmCgE2EghBmDJ/aT4
+nd3B7afUNX1ZLCjHdoJ8+tabXmi38lQkLlhthYjIplI
+-> ssh-rsa krWCLQ
+krJRF4AvwfEFro4uiLIBB2RQTwO9COSyAqkjOi6jgLzJFMaU62EZrgfSYu24a6zs
+JyAHQ5k78uh3EhhbSzu6U65fComCbRAo+NiN2BJU7jb28y64suJbezJ+LE4P35CX
+biVgycaSc+OCrb1F6e6QOREi7+YjK3VrI4ZVCu93hSQNNRi/U5bbigQXt7NwcSIJ
+bY93sset4wg9Zwjk7tFg1bHiyOK3ZvYYQGlMjUxiWGl4Qch1fpL2CJNR32mZybps
+GZc7x69E8EhoHvdI0u1AXwS2raLhyRBPxFzu0r7nPlSi17TnLnU3Ux3BkVEDa6bh
+eVHqAB4dudNCC4wOY/ZE+Q
+-> ssh-ed25519 /vwQcQ scOp+aVA3TfY269EzQ6E4YX0uAu7qVVVIDmBvFGaYk8
+AYqW8+A7oxH/0m8OUReWxto1xWcnOnZOkX45ejuFJiM
+-> ssh-ed25519 0R97PA YqfHMAAiMcH/efJ0K5URDJkdLqlJIlQ+pSnuGUOJzjI
+v6BujlFcBF71SzvlZzA+tWku/A8bZzLkRfHtoCdbCO4
+--- PeORL3PTxYsxaY6GliSm1dRAH+hxf1n5LNeRYDq+poM
+I«­Ñ_÷l꺓CqyƹÚ@‹Ýç¾Wï;äù¹¨<ñÛ5ÅF¼·Ee7þ11=ø‰øQ<‚pmƒ¬ì¼^²<>òŸµÉÑÃÙƒ¨[†GÞ½÷$j†©<E280A0>±¨m’±‡I×ñí¶\«•ÉO§ëÇBdÜÛ< O›ª$ª„5£ãÒ¼âá‰õw˜REŽ3Y×ë!Ïd4ŒBFõ‡>ªŠÈ˜\EKËhæg‹ôÙ^f;ˆ1>tk ™‹£{ª»ÓMð³ D§š³87®\uÄ,íSá›ÕIinÀ:ø¢z"“Æ<1E>-XBñíÀ<C3AD>‰u<E280B0>jš
C5Õ­‘Ðï,Âg“*\]ÒO†îÇye5ïQîÄ!‰$þ.‹†+ù¦²¶zÀF¨ÑŠ^¨·SµµL<11>äÍB+óÓ¼ËR“Hxö6ÿåʉ1f<31>=jú8›¾!o¿@‚Rzrü5p´(#‰w|Gd­×¼O@>²0ã{$ËhE‹°Ä<C2B0>ÕûS
ê_¦æš^#›Oÿžò– 2c¹ŽŸ#G<>ž'º?e9yÂ)ô
ªÅ–Ì~ÑÓ¤
+Ì’tl#­#„»I³>o½Å³°)ˆóu
+†

secrets/floral/loki-environment.age

@@ -0,0 +1,20 @@
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>

secrets/floral/metrics-push-password.age

@@ -0,0 +1,68 @@
+age-encryption.org/v1
+-> ssh-ed25519 +HUDfA d5f2ESneC0wsoc9rwTjNfNXMBjCbjAQ7euthH2Buq1E
+5CynaQ8zhDRBvcmifhCsiDtllztCVAqs8rU36DOxgPw
+-> ssh-ed25519 +uvEmw EtYRis2LP0jv1W8mx8vFYNzkgi8OoqnA8cM2huS6NBk
+ll1csFIO+hVYk+I0uSVJmlDKj9aTWvf4kaYI5LJcm7w
+-> ssh-ed25519 DMaM1w ex4QJN8CG99J15i+yvqGEiEZn9OlGIC+cmLHL4u8ZEI
+VXnOv4CGK68q5t6hUV3oKAtxGZ+4FVbrmE1yMn16A0Q
+-> ssh-ed25519 sixKXw drXN6+q1y7L7ZU4chTfHfelu5GcTdff+i/UMFV0+3RQ
++8jmgnMh2OpQ3vhAuyQYWslfx7KO84a8KsCpoRD3Yl8
+-> ssh-ed25519 aHbF7w Af7NgjZ/Nvh5FHrX2VlF5riTIhJ+fdxTo6OR+8PcNwA
+ktKpm/HnOnw2Ym7xee3N1rneEX7+/xDhcp71N1NNHAA
+-> ssh-ed25519 87T2Ig 8mEUxJ/5NUvV+qQCDQH2Tm6Ryr5hf4xgsQlqXGf03Fw
+EavMcnsg/3EYBLQEBHX+0oTDKq5ZL4vj+mZntPM8UMU
+-> ssh-ed25519 Ao+7Wg UphWbatIaa+R1oZbfHazFhrawf0vax/3ZZS7YuX03Hs
+dwBbwoV0jpjiKr+nj+CRfUDgDl7ISpsCintVAzHnIFQ
+-> ssh-ed25519 wIR2ZA ZM58Nq7eJX9JVeYkoJf+mw8hxhYGoTx042ow1u3mJkw
+UtEaf7e4xsPO0ISlIF9LF+GcwTBqw4AXdMO4MASfgLQ
+-> ssh-ed25519 oGiV/Q G5KX/Eox+9md0yFRUZvGIsio2gWM17soHsL6H6zEX2g
+vI8jPjBAoFF0xhvRRLPzCMSiQOQ0fKuRb3CYVu3KUUo
+-> ssh-ed25519 gO3aog p9nZtjzoA0zJM+7Y6R16mpdub3dhu67yOYTUNKRytgI
+YL9vAp1+CK7jgmXkB47ufZMz+/swngkdUvEGR1zFZwc
+-> ssh-ed25519 N/+Clw 6LzFdtNsWewuJK2r97ZXJbRazvK3raN78UGanR/zWVU
+WT0y+sfDP3ffVwRcbYw51ArFR3OzXnoyi9IXwZZKEL8
+-> ssh-ed25519 CtkSZw CV0jQ5dIbgFtMxGK1X9b1qJOKmske8VgIPW5NW9mAwc
+clv7P3de61nZmXrvbOgL7Llw8ZqBMm2WFqgpznDwKv8
+-> ssh-ed25519 keg2lg 3Nk40ByQj8RThj4QDY2BdAkw55mXAJprXQRGjQqGvz0
+f8OFszJ8p90crFd+awEE12CNd7b22zgpH2XRKmH/Hf0
+-> ssh-ed25519 H885DA GDiJYH+gaC++TSqfoPDOTFcsCZRhEl0EeTeab7tgcWU
+kMILmwNMnMS7rgC3kKsAksu4Txn5owPU2y09h4aHKY8
+-> ssh-ed25519 Rq7K4Q VCNxGtCSCD2OYSWWwl0+yf189xV3QwRiwo80h4NPTkE
+hHkgYHLbISdl/RRdlInp9ub854M9ZKFSXpLgKW2YkmQ
+-> ssh-ed25519 vvyRpw XSCCrqEOtvzQRssI0U1DHirKoPgbOUKJxNKnioHhT2Y
+HGey1j0Kxae5Qs0aw6eqFziQGiRmNA+lEwbRdf5hhbM
+-> ssh-ed25519 aSEktQ mXY70Lgl76J4O5dPdDcIqmJ40EinigDuZrUghpGOq2I
+U2qeVFxGCYCEFWU+7vHc5Mu9EuzScowrjnwUyoqPj5U
+-> ssh-ed25519 cD6JxA at89poimBZyeeM8CQrxDxN0yCNDT2k04++py1fFycj8
+cQV/K5zc5x/oYnJ4N0MX3sTboT4G4ZNvVUVdHuJRzbA
+-> ssh-ed25519 ec6XRQ spJtb/xy4k4dmwKz8R2CPhC1WcuNV/rnDT978GkjHHk
+KrGEVGts/AhzbRNreqQ/CVanXL3l/9oMWxnpBLj23qU
+-> ssh-ed25519 1qYEfw KRkTYlvvnsCIExKQNmCyU7YxnGZsiI03kzecXNpLzUQ
+h2YagV7BzlsF7banzwXbOudTdlFzT7LC8PvtxAsX36U
+-> ssh-ed25519 2D+APA 4hdYlOnNIT9Q6tyKwXzy+u66Ti2EJopK43Sipebd0As
+tuesc9/QcEu4q9bTFJ5zJr0qvgLcmpn4at4cYtHrtbE
+-> ssh-ed25519 eTSU6g i1qT6PtepHXnoLCqDbhk86QG+SR9luQaw34a34gy5mw
+YE9VBAT5SLW2ECHRU+dMg9na6OQNVRVGuhY8vOdmE/Q
+-> ssh-ed25519 j2r2qQ TTTbSB/8UIDmmI3C9+u24PYZNfjl9jGADKHNWIwLfGE
+SNDforwii/GFp82TpyOcVIVrZWCe2QQKrjzPA6XA7Jc
+-> ssh-ed25519 CyxfgQ P5EiJ54v65Sz1gHuI0s170Z7c1WjcZLlb7NYigElfVs
+iYJUGpoE9LBIlv+O1navSSsy3EJ8tusXXX+/QAQvjNI
+-> ssh-ed25519 C/bBAQ hlBDpQRkcVCr3B6TCrBjxauhUX6Ndpm0s6x8W4hU6gM
+OFG3EuGJkSoEEXhbJ/Tp2DBdnBcs+hzxjNRdvcOSpQs
+-> ssh-ed25519 +qVung cGEGpO8NJfpj9ixAH9lhYkPKPDdQWryVxSOhMGQdnWM
++MycbIEab3P/AOS9i/YmPBDXB76hp3xUcWI4VMihV2w
+-> ssh-rsa krWCLQ
+Zv3dPYERlX1MaVaJTBDwIcjt1yLmu4Z7MovPgjGg01p+XsdBXeepTyOl+gRBwGgo
+AW5CIuaChYxtSNJ6nOgSaUpqzILycUF1xE1jROe3MIX2MZ4KGD1qoqcHbiCAng+a
+RqYrwAKnNea9FQMVfhYZBkRoYE6ne1R+0G6BoFM/okz24pAAFPBx+sMMhfTkt0uV
+kHVx0dgRw1pxa7Na98WH/7E0zp9VuBvVHGXfk1rfW/UQlbIO5RP3nldFoa6OmOWS
+JZ022UvjyC1re0KCurka4y+qmaiRKnTBmpIXxJFMwNCAQ8O8SeAQ3DHKHmXNMOIL
+ZVICtRRk0uX36AVU8DWDog
+-> ssh-ed25519 /vwQcQ kF8+hsA+0Msjd3q0SL52cae5RDqx4ls5kPKnc3UZyms
+Q33kIKJL3Vjxu7LQ5l4M3tlEuj+OW4uGh1x+JxthW8A
+-> ssh-ed25519 0R97PA gWBH71l6w9upTE0DwqOMSvWXc5VyJiKFAQLaSpWQ43E
+IrOrvzEa0bABw6UOpP8pM8WhuRNMaWJ2khljJIKwOS8
+-> ssh-ed25519 K3b7BA oS14iav9pSioLecMkOanJz89OJygLugvrnnTs5pKzz8
+akupMSiqXussXJyHwFm/f0imKALjowJVqd8/LFcC/58
+--- bCJXTEDaKg4FF2dQp2zFMU4Zm0zACekCuDvthcAyncM
+&­Ÿ
€Waïãà›BD R(¯¥Ñ”ufj<>úVÁ8nÆ>‚ß›øëæðZúâ{Idƒ„©,³*„%Ç“È‚z«

secrets/floral/mimir-webhook-url.age

@@ -0,0 +1,20 @@
+age-encryption.org/v1
+-> ssh-ed25519 j2r2qQ uYhcMpOER5j/SWUX1mNvkOU9Rumr0CgVBuGv9EHGpFY
+6kAgrwjgB7C1cMd410EpUegcxxGRcNOwCMJPXppepvE
+-> ssh-ed25519 K3b7BA 57GDNt5nwxgzCV5bnMPEPUeyZNG1U+zajCIjeoHjLAE
+rFCbfodjXHZ0aVLtW6xtoh6e/VH/HwFdFzjnQ2QEEXQ
+-> ssh-ed25519 +qVung DnLKAJPnUDpZ2+wXDZWpxwZkvv8oDyu3xxObTMT9W1I
+vh59DYoQLpiro5eBjwgNH2YHRsGY/i6TB7zPfQicOEU
+-> ssh-rsa krWCLQ
+ekvGooB5sCmAniHU7hlk+iCkYMQ7Rw2SJx8tp4FnpfAWJbRMH8CpTFYFiDvlHfFy
+Ce1OpkNkkipzBge0OCrfn6Y5iVz2CZHYHf8Ul5ueHwmb5fS7seT3yMoWhhSw/zE/
+G3snrBORT9S9+KTRnVnKiy+O3CaMZY+q41RR35Fs3mmVc/of2ILc/Jj3a3t+uBTX
+axkOMU6z6R6i3Ps5SbwJTaB9q2kMPvZFOO9Nmku1wohjetz64wvm+fDx0XVRPe4A
+jDQRPKAMIZK68SYHk/9azmlBtJSJnvxcxyj3IaU9MBskUCldWi8CQ9jQ+1XAIuHX
+0Etcsx7MhzBpuhx2xZ+dyg
+-> ssh-ed25519 /vwQcQ uW41w2RAtfMaOm1wJktMcbVporqKgdGA5SY03OcPmlM
+WgL8DWPU735Ysowq0HtvbrT6Tc3XEpwws3AycqpBgtM
+-> ssh-ed25519 0R97PA 59AFQx8ngDwQUdmfOeOFUARQQqaAdLA5WH67Wsld4yM
+o6jSWtlidZssWsJsI8xAaASi8p1sirLJFJwizzPXIBM
+--- scUnldbU89ICZYlniDbGEqeUF7QUoO1kcZLl8abyttk
+öR{p@IµþOlKKõŒ§!
<01>œWÎÅœ[R<-A‚ÐbÔ<¯·÷0õÐu¹øµñU’gBÏF~µ«=ÊõeQò}î4Ø:ô²¢5ƯŠtaØ™û”<C3BB>æ·<C3A6>§°x±M
ë?Ew0<77>8.

secrets/floral/newsletter-secrets.age

@@ -0,0 +1,21 @@
+age-encryption.org/v1
+-> ssh-ed25519 CyxfgQ fWH/o2+Uf0i/JFIuVjCnkhDIfYndtL8EeDcxSxhKVH8
+ShSPmdwnxzDuUe/kCx8e61JJAoHMwguNydn+5OIGuAg
+-> ssh-ed25519 K3b7BA p+wXAGvPqTX63dlZNCTIq3F4QFMWEJH1R+Ex4SJ5UTk
+1sFqFqnUM8YvZy7BEBArg3eLxCCsLXq2jNI7XLKq/Ww
+-> ssh-ed25519 +qVung rcpgzVQ1PmoNF2i0K0nAknzZwPXICBggzqhIZwO+8xY
+9rjsTwLm5u1GOJmnJYriXXAY1unG7y+WJ4G2ltxX34U
+-> ssh-rsa krWCLQ
+seXsQjs62kxn/agyKda2l19PI4xzDl1gM7rEnaEBV8UNLOPNxh41HTnP2etgDXSc
+4eyS3ntHXIOHmN4+JBn+Q/wuhzMGQmAcoFWbjqVVPOrpPYjgCG7q/iUD8kULxLB9
+UpF0gLsg1TnvrkTwlpxr8rP/PM+ZgyQAA84S96j9TW0coyTUoH/ZX1wWGtS4aalm
+aTrOMZGScZu7onTg+tYvR+aBKlFL28h08I5nqbA39srnCNuU68+OUhLgLUfiTscl
+umwNh/C4BP2Tmc6gxQiY8o3tGqGBssGH5+WqKzbK151vJjq80RKAS1HCaSSfmxkP
+vWkXWN3NQkJyqCBpuPYilg
+-> ssh-ed25519 /vwQcQ eUH0B+cCoUubIKbG+bA25kRj0TnZabB6t8jVK04NrFs
+ovkI0C4W5CJXMZIZdpaTtQNc+TGkQ3Yq87Dei3BMUsA
+-> ssh-ed25519 0R97PA u/I45pxH3Bnja/Jw/6IukINRuC0e1IKu8UVygVgIomc
+xyHuiHf1/nJirnhXbGHJnextGQa95tDo/RPRRnDCkIg
+--- LGqO4Bsa8bofD1W5YrQp75SlGLNg1XaFZ0rPUuvLPTo
+Êçã ‹ÜmlW£{@I3…*¹ŒÇ™@ÞªL7Wª
¤ÝŒY
+n
õö~Tb\V‘•ÜvPÙpPô
ïoÌS"ôm/Ûµ/bÝp’Äžêq¸£¦šeDj6–ÆþTì)

secrets/floral/stateless-uptime-kuma-password.age

@@ -0,0 +1,20 @@
+age-encryption.org/v1
+-> ssh-ed25519 CyxfgQ D2o8bUccO13DKF4COLBQ9mJbACsE2XsRa5S+N71WnTk
+ZaldT7HhQxbxf2ptIwdMYkC60eGtzihc7uwcAkq7s00
+-> ssh-ed25519 K3b7BA AiUCG5CnNyv1DPu+iEwEgW9GqZ8zgpgxKJTAp350ADc
+cUVaDv7F1haQIF11/UhhDAR5DrfJlPttGfDjkv+z9vY
+-> ssh-ed25519 +qVung 1JXeXyea+2Pcwoln/NLRiR8IPPIiB3gaFCP4imyv4DA
+JWmAY6ZnyU46KxzhRrQigGmUPba9lJDDyRQ2GjQShqc
+-> ssh-rsa krWCLQ
+ciLu/+cXfQrB1ms8oTv+xi4eADyL4j0qwnY/6TE0wAXkQHuNXDmpF6ccWZoS2DqN
+NcnGXL6+WyWxmwlyBEq/rsBPvi1g0M6Md7Z4gXn2UvjJ+S7WyA8QEwkxoTDkJS7x
+k/NvtunmggVsWVK4Xdi5DKRw+f32qr/8GysDhIPrTt43iReBKNbyuYWmC5Ec85ep
+JU4JzCNZjJ07kixS5Y9BhaJbpEr47lCXE/KtJUvm3VAxS9IwfUn7KHHdFWynbExi
+F898j3zOR/kgYmeA0oTiexRD3Y2LCvjXIHQZ3MobbZ/PBrjWxe78Sw2vy2t5JLtB
+gFG0K8M1z8DT6a8TtvXEgg
+-> ssh-ed25519 /vwQcQ kUM21TO9iSa8oVXMlNxR7Kc+8TV4C/uTzyQ+t3xnARA
+oXt+egWWONsKT48H4vZ2CPdy3Zfb2QeQVe9l7dDyO/w
+-> ssh-ed25519 0R97PA e/piqf2RD5QgPaQs6jsJdzJgfZR9n1JDIWpbvLZErSs
+UTJH8POFdZ4+N9WkLoNESl1pvcVD0MS1qn7AdS/mg34
+--- 9aYEP0eHDKMacIf09h+OJqIYw+N99+FrW/x/do8Lbo4
+$ ÖëWÛ\zú—¾=s/à@.Ç,?ƒW6n^ù#–i!§Ã–ï¶1]±Nvù±Ž'Ï¥¹6?‚'mµpPÒqýŸº

secrets/floral/tempo-environment.age

@@ -0,0 +1,20 @@
+age-encryption.org/v1
+-> ssh-ed25519 j2r2qQ kbi4mciOrjd7/X86xfmkDaMZhvZakoSJ6qjqLF3ljkE
+Q2BsgMLJ8AmjhnggRi+wkICj18NCA2HW1t8clemReUw
+-> ssh-ed25519 K3b7BA wNGmX9S9bJgd2JDte9QoNDfyycgmq4JMu2bc5nyYYik
+uUiutxAI3nI0M51W97aPRVE/l4dV2PEjph8eWOMLHIE
+-> ssh-ed25519 +qVung raYJ5vwMP9JopSdfa+ofkLY/gc0zcW4wTNBFTca+MXw
+sa/rWGSYrI4y6rn4JSboldWKUGvx6HbtsYo78AFOkBo
+-> ssh-rsa krWCLQ
+FLq8NwkiGw2gXptVVY393f0p9hFom57xHWPxtAlzOcRT8gvWu/uwgV+0raOcOcJa
+xxr5Sib+2D3UnUhprVPmH5Os9bI2seFAiej1MVVWLqvMtQHLFwnrzZTyZpxsXpQq
+5qQhNEADuQc4uD/ELVjGHKt6nF1Cl/GbgNLIOF/ITZ0pm1O1MjtT6MYJhQJhc6sb
+sno/wQyTXjj7rC06nyLX/rgOWrJSOeaz9eVp0A8k8/I0TXu/vRCW9gqWtv2m8sbh
+1uUHIm0l8f3z+zrL6OlZnpMFw4jpiiGoCYKPzD17I0onDYIjtdVS5iO9BsckxV/a
+wQWbyONUwbGCfeNSVAzZbg
+-> ssh-ed25519 /vwQcQ jwf7fwy4wKz7q761DNu8SyFHGgFlwq4P/Pn44Nido3E
+1q/jvt/vtD4ziY3eCDqk1XwMPpNUd80POTV2VVsumCE
+-> ssh-ed25519 0R97PA XeuziQ+wsoh0KSHXk5Qkl1kQOsAu1Ax1zTg13+XWd3M
+B1KHKm3tx/EsnE6hY+w7ya1ilhYiUs9AbwARHNkJi90
+--- JgQA6gCYZu8xcbXEl9VypccEIBO6uAJIdhBefr4doRQ
+V3ZðõÚ<EFBFBD>ç-·Ý.ê«sòÀ³3 ÎiS‰a5#¿Ð{åÔÈ®Dý˜YêNèãëù«ýoL+ÔÝ#–M<sws P»¢+í¢Ó‰ïBDoÊξÆÏuFí”Ç^Â¥•<C2A5>—ÝG@ÍM×ÛãÐØìq¦ºG^Qb s<
;ÂÒnC+ÖÊxª_­Úì]S<16>Ð

secrets/gerrit-prometheus-bearer-token.age

@@ -1,22 +0,0 @@
-age-encryption.org/v1
--> ssh-ed25519 2D+APA Vh/FrR9oyO8V1pEMQkmGbHCePB6RU+dPm+Z4bgKenEg
-2G5eLlYe8IS7fsEBorFljUwQZ9sEk/FEr25S4p5hWLk
--> ssh-ed25519 j2r2qQ 9+NX0Guhux9QlAxx2MtSZH0OZpDk1CQZ4Blu1P9fpgQ
-PDUoAjBaIdKQAvRblvc0QEtrvp5MpE8HsCwKWwAn0uE
--> ssh-ed25519 K3b7BA wuOc6LGnjsC4Rb9D9QX3YVgMqWPvBK27Q0vqADLpsk8
-wRnoNzkyaU9SGlOtpqY2pAeIwD9lGWKrqNn3D3W7U6Y
--> ssh-ed25519 +qVung biXtZHmjJmsazEmp1iIGUqmuV1YP94bzrMjoZTmGPjg
-GDN4WZGTIP6b2nmjyhikHeOrZi9YEtiPOyaJLzUl138
--> ssh-rsa krWCLQ
-UkNySvhS5o6v6/7xGvn43hgD5y2D91oH4pjU3Oa83CW6ha80dnE+JkSTpTdz7Og0
-vtZJuisNpcH254zTt8OAUpWN/tVXlD34RyV1xo1eHEWgUzKactrhlACpSbzYBdVJ
-8cUj7jiE+qjIOtrU2sHWo09NKpf0J2YEPwajuBy1/fPrivlgXAzdAAnP4gll02x1
-Et8lUn6HVfYDGtrDo/PUUdgcGudVeCOJbvvrKYkuqe8vsNYgnFHM8dkTJmObL8dz
-zp4MEuIQ3WrrXActSnTs+QAGIFSskOIr1DQlJRYzQcYtd8wkfx9a+6oxBECZyDAZ
-T4yso7ctflKlr6OqpJYzeA
--> ssh-ed25519 /vwQcQ +jsCn0OlVpuyVA0XSvD3ZCDRTBq29UV9qsDvE4XaGk0
-p2qblImpl+G0pefJ0T/GjanIc7+bNuA0wRB4mUuFGXM
--> ssh-ed25519 0R97PA /bE6+eVlzeJKOOMqz4QjFdsu+5XDv9L8cZ94cPZ5WQk
-Xco24ijeQnaT7jcsfXLQPzGr1FE/zy9+qVoQ20DLP+Q
---- NDqgX11cTXR48vD9YmAIYx+og0n1OQj+bbkKwqv2BeE
-šÊ\”wÔ
ðä9Ì’7öcØšƒ’‹%}|k®?š×$9·lö&<13>=¸vñþܹ!Pã<50>Þ3b·<62>ù퀩

secrets/grafana-oauth-secret.age

@@ -1,20 +0,0 @@
-age-encryption.org/v1
--> ssh-ed25519 j2r2qQ qI/dlkHZYcNkCVgZbxpw5Ps2anl8pofaFPi4p6kOHAo
-KWL+H9at/p/AfCjfO8+SgMhn97F+DqLO2ymYUOHkWjQ
--> ssh-ed25519 K3b7BA URYQ0jFY5yHS+dodR1RqodNWrrXkMnzTp5OCSv1gbWI
-bnyrPvWnzDRNh4mI5HBPkNl3NSZE1ycMK3LLExMEYbo
--> ssh-ed25519 +qVung z8e56tCZ4TLkrX7BfH+5RrGxGoT3q9V1FB/ySsH3tg4
-jIpEEVF8jCp/ks5eYXh3O7+TLidvzYsnBRFd3LkgLXw
--> ssh-rsa krWCLQ
-XG8KKBT/hEvB+c1RDGUrDR4HrfAertfOIzQTquMQ+Z3Nde3Ybxf8W+rWGQDErbq4
-VlvC/wVVnGnqgE/tJMQP41sCMKSH61MPyiNZC63g4RW9e2H9YQfWWrnuBh668G+3
-3sE0FSdIAB+UlI2jlbMiG60QaT6zV0XyOrugLX/G2R+D4aXYIVvMtcwYq2oIHy58
-1DE5llUZHGsQ8APXZle7ZGyO48ELOQkVn8ozPlPFhvz2y9srgBZvNL/wadjvLstv
-2vBTBoRk8HnTLOiybAnGtOfK6kWUMdfSYMvhu0IM8UBSoxwxOHTfIttKDu2ZMB8g
-c/RnKbV2z0PBdXVrYuijPg
--> ssh-ed25519 /vwQcQ qinzScNz0IFoHUaCeGXne6ddllQ0dA/TJr5Z/nbfvTQ
-0YpTZ2Z2WwN0sJ1CIV8voPS298u9uHbRQMlV0GMrvFI
--> ssh-ed25519 0R97PA en5iGTQoH0/QJKl38HNe4xun/FxVBIun7Z23mBW+4XE
-Sjshx8hLyP4iY40y/Fehc0wZTBH0d1Lu+auX8L5n28s
---- i5+vCeWbFTRR2YbIX4lwbEORRhaI5NkCwqaMEJqrPEs
-ÿ\ìƒF·Ri±ñXa,.øÝoªâr›çhE0=$Ç‚uGa/oÑÑÆÂiíf¥•x¦Óš?Ðg¹CiÉ

secrets/lix/buildbot-oauth-secret.age

@@ -0,0 +1,7 @@
+age-encryption.org/v1
+-> ssh-ed25519 Ao+7Wg q7oRHUUlAvD8OUbpPT7d6eLMPWU0YS/verYTDE5BCkY
+/87/1uqOvRYeqel9IjnFmGyF9SXUQD8MTgfcj91b/Fs
+--- ulIeB91NJ7z/64h9BCLSD9/RW/zwv3m1Zo2ovNuInv8
+Îœç}³Óš#épÇ o>ä·*vµ÷ÄåŽs?[¦º´L
+<EFBFBD>þz™rý‰?R±Ñó7<Ê
+æi!€{X„¾òÓ

secrets/lix/buildbot-signing-key.age

@@ -0,0 +1,6 @@
+age-encryption.org/v1
+-> ssh-ed25519 Ao+7Wg EMpfs0EpWwaIKAoUBfEkyAHLIwi6JnGG6RvUWM5LjnU
+LKiwUBNc791U/GVRNlRPZE/TEMJjcFFrLruFJhiyiOI
+--- 0khp8u+4vHgGyQqP05m473Eo09eyOUZLI5+EK4olzoc
+N3(
+ª•û
xRq°<71>f<EFBFBD>Ó;ͼ3¬~RˆÓ
C^ñ+fœš1”®˜xˆ÷ÅëñSØ—hâ
£ÖË°GˆÓn–YIûµ:7¾!°u×Hþy/‰Øð‰™.¯¤á^¹lC™ôUÈËþ5cž:]ÿNž&'MÎè¶É-˜–ÆHF¦D0‘ cjô ‹Ð~

secrets/lix/buildbot-workers.age

@@ -0,0 +1,6 @@
+age-encryption.org/v1
+-> ssh-ed25519 Ao+7Wg RPKKoI5l5cYVdSvOxTHCUtwceac4vSi3+vlaqHr8kQg
+qbgTHCeQDNM30IJNZ/BU6wgryJwB316H5GWWaYB/wng
+--- GuFi3GSRdlBJ5YRjfAVyFDZ+4TH575kFJLnFp5847N0
+-èƒÞHÖÜ*x´M7¼t<,4ˆŠÑ^<5E>5@v°<>£€º,Z•MÒg=M
+» 3výJÄ«ÐÖê¿Nz­8'<^'4&WÂf"Êõ´À›ë\©º»ëêwmzúlAl|+„‘ÆKš~68ñE­Ýîk•8ø?S&òaM‹Ý~ž¹ê¿]Vfø ÝJxaõDù¥x

secrets/loki-environment.age

@@ -1,20 +0,0 @@
-age-encryption.org/v1
--> ssh-ed25519 j2r2qQ JzVKQt25f18L96aJWsJtFAR4mvMVCgYMKu/xtJ1BeDw
-vj+HpNQCNNxDRA+7HgjiD0XlGG/Yy+tk8KmszMkxdag
--> ssh-ed25519 K3b7BA judlH57lGOGmaTEG19gYiORJT9uXiAlxZrP+ISTHDT4
-MS7e24A6rEMUtUUl8DlYXPy9NhqAq4buOWT0iYKvbSY
--> ssh-ed25519 +qVung vglRR5LYFZw8v6zRhybGPBctwDgYoskbpGYiLNW9qxM
-VdjQTykQSVWubGimCHiekQX7EQdgOB3PYsRHiFnpPkg
--> ssh-rsa krWCLQ
-hLYT6U+dUVuicVO8hSw4KcfkM9bay4JR3TEWGlmmIxcQ67LNggzuyRvV6U2yfucg
-Xyxezdd9LArf8z1eV/y3iwsY0PvK9qwtgpgH/NxaF7djhTA8+c3c3a6w4sqdHn0m
-/RZU+eKSFeDWII7fn6o7JxzITFhF1FYH6PJYA2cb3PvbPw/JSja8EVZ7192ShqGW
-22TThbZmmKoOPbmDxmQIygZTxqyaXkoFOnTWqqTzOfNtBOBFXT+cIFh3ctGWLw79
-u7O5c2dmpXoE0bdndQ7GUSPrgRzOYHQ5hLg8WtC56EYjE11Bxj88fktzw4hZTbYQ
-jrS8Pa68UPhUmSfutlpd4A
--> ssh-ed25519 /vwQcQ MqdVxRlS+EMA3f6B0D6m2ylvCE7WVq1av/CvsNVAB24
-KX8RJ1bzUUhsYW6qN06FTzis5i13IIoIpUb5FkW9wkw
--> ssh-ed25519 0R97PA RHUvc9XQIxOW0GCyt0vRxPHyVXlpqM9gaUps4q/Grx8
-bxgFxtbtbvDi9knzasdR7u33Mb7x7LcBzqEB/g4Oc4A
---- Z175YCdbPBBSItxomyXPSo6xILLV4GT4gpA4Oxz9qgo
-EìVÀõ±ž™êÞ<EFBFBD>Ú¾¾Ó¦xYÊqšÑ84™6¦¯&Ö‘ï<13>·”ž„Ý!óZmëû°¤Ãd.à™46ÅÈ·ØËòø/<2F>´<EFBFBD>=°ß܈'hM³_ü£j
>ªÑ6ãR<>&Ú
·u²þŸøEùÜ^8c;×Ä›¶:Q1Ü)ú1L¹_~,<2C>K¥ÞÃîôµB¤7–

secrets/metrics-push-htpasswd.age

@@ -1,20 +0,0 @@
-age-encryption.org/v1
--> ssh-ed25519 j2r2qQ n1lfxDP73nfF/CYtE4gpUH6YgjAQbx/2TTuyfFUBiHQ
-LGzudpjsYA92pM0UpUT9CWZD+e+rzGFP4ndxPE0MByo
--> ssh-ed25519 K3b7BA NRnnKaOtdtIjkRdam5vAA9Yj1RUJRReugWKRglWAoQ4
-Xprx5TSU1rNH7NMl0X07K1KexCVXMEu7BFxbiPwxvBY
--> ssh-ed25519 +qVung qZsGi4JqgpHrjlg2VdY+OhXb0BzYTytBBqY3jNsrSgU
-GgvQG5iMd6XTZRCC3EBBvqF7nhkqAJmxdIkCFRV46Ok
--> ssh-rsa krWCLQ
-EkmY8uc79xWfKjlIozS4Yigorz9IdK8T8VjMnVcJN6+rhoRctQNVCj4JgogY4wa0
-V3ObjoRPZgVU3qPmkPgIKVa2Mvf6MrCMwvvE4j2Yyy6lmQEwFdvk4s2c6AD6T8Bf
-rktRYqOcFavuDr348e0ZzKniFTRcPMcY49mqBR/mWIfSEtLxBgpFUCn6f40PLndT
-3dse7kgRBlrKbzmf6JIsITHejqwDRq2bZqHWAmZhb6+ske7oDicAt90FDoDbrwvd
-YwXPRDCxgATlNz8n/xFUxd35X+zEftUUtANSGtihIE4LcdsO7IOwv/FCjdEn/3YW
-ZtQjphnxgDsY61PEFCMnYg
--> ssh-ed25519 /vwQcQ DKQuo5jVunUFTCbOxVV57Xl6q+DDOVDWXdon/lZlLi0
-doN6en8IK4Ju0uATp+IZAhYl1tvdnfyxHziSobb1ER4
--> ssh-ed25519 0R97PA I1GECXSPagJ5kD7CeVA21TQmpMEgLeaiB7XYEomUl2U
-d0kO+4SkAPC/ois39SZafEhTqvmDpCZbWTUU1aUZ47o
---- 555iE+C2kDLIdAJ5KARyKcBQZSDRWASuzcNiKZ9IbRI
-òeÕceV&˜ßà‰g˜óáÔÄýæ›6•=6!õC<C3B5>Cˆû^»âÕèí€zÕ§®(Ó<>!ÄB•B|ô<>ï°Ú'¿Rªîž†_a	UtI³3

secrets/metrics-push-password.age

@@ -1,52 +0,0 @@
-age-encryption.org/v1
--> ssh-ed25519 +HUDfA FOqd+I9DzoloOMK2InPz8yAGsk+ZgMKy0n542DmF5ig
-sui4rdOQcvjL6H9rPSbSAyIggaSbsIVrontrkFpPPC0
--> ssh-ed25519 wIR2ZA V4KPrGw2NKeOBWpjsRbhUJ/eLR8/hvExNMpcBvC7gCY
-Zjc+HtALqZbp+L8tUUgaFe9LR4NKptpFq/L7xhTItXM
--> ssh-ed25519 oGiV/Q kJS4DAPBTOgADY7LCZnIfORMM1RJez/5XGoKDfErHjM
-LN3XE7qM2SHqQwb+JjIq5tMvt77NI4+YOxYnZh82udA
--> ssh-ed25519 gO3aog gJFIrngWZp4ypA2IZwr+c0JkWgUu9VN5AzoyyhozlDE
-lezfokY1lgABSKNO+Fr+tTlIjC3gzc4Bw2YlGLy+WvI
--> ssh-ed25519 r/iJSw VzO6pblztwci/TMfha+dOc6Vg4DC/1oSNEt0aFaCYRE
-Mf0LjSjWJA2lMt1M1z+tGJ+9NVMxd8J5CSMvaLK8zB4
--> ssh-ed25519 N/+Clw uNBuYGWU+LLY856o15jLkJNk6pu42FnX55CoE98/ukA
-zh+sZ0nskVPUKd3Ajg1FHng7caKhkEHiRFcm8c53siw
--> ssh-ed25519 CtkSZw YP79uyNelg7+nbeois1vu64anUC0lhUhIie6EqUz2i0
-rb9zte3dN0+uwjyJLGaUfeEQcVtMerKEOVAocLGXUYs
--> ssh-ed25519 keg2lg +g5uYkOOyQABVmL+9t08aaMklNEbBO2j6vqKyrwYrhA
-U4FzATeou9spmYchqHPR/WR79Y+ILWpwhLwxjYQd7d4
--> ssh-ed25519 H885DA tAx+W9kfJkvERw9KPKZInC0s44QqQIu71MPUosasHy4
-5ks2qkZfkMLK4meVHTfWpR8qCeU3vKdPiWVRTyD6OhI
--> ssh-ed25519 Rq7K4Q xwSlrqIh+rZFv6w1iDcPyD0nEmESlmHleUHsVPrG2Bg
-OgrWCBqb7SAtQQSUnTQ1l9JRyDGS2DgzKRRbMCtKK7g
--> ssh-ed25519 vvyRpw wQB8wg6bGvb68pvEp+7khrNpZTUxSVzLIfubbYsX+34
-KZ2/Vnxg7Gpazc26lYddjNnMxpoteb5ysuTZUg00ZvE
--> ssh-ed25519 aSEktQ KdKSZuVH/v+gkZkL07YdUJ5vvH2+mcUR4x+mXHylhys
-MRGd8l+0X6XVq1KpLqYqUZD/4EkOKz3mpHsdQepc6kc
--> ssh-ed25519 cD6JxA FesXIZs/X+fWefYjP0sfkwz6bYLxOkuIzQppwZYXNTU
-hg+ZTdCGuQ66FIc+NZI023Aunnhz+Ds5cFKUwNj+MGU
--> ssh-ed25519 1qYEfw HRQdZ4u1UWpzwIF/0lbJ1NVDQ+/Rl913jk+BwLM0KCE
-CHlDCaov7TWme5YMBiV6Tby0IReB8pER/RbDkpI3TWM
--> ssh-ed25519 2D+APA BTVVWo3G0tZj/hUMH5cwByYf3LjAg2RNVMhYrkXxXjQ
-iKghO+M6xpp95xVrmydz9GJJIOK5JrIsoL+CSFD77uM
--> ssh-ed25519 j2r2qQ RC/2vV5yr1af4iyeouQwIBK/r8b4nD51WwxgbuMEgG0
-L+uqV7eeCNqnMTqCNmvLPZFNTdmlYu/i7+3NVwmpIxA
--> ssh-ed25519 C/bBAQ KO1owoeb7pbuXtDS+f/TziotgffL0Eg6qnjJ9W8Yp2c
-af4IhSiXlMPiNuM473dIeWQqNbRgb3ciHyoa6buolyU
--> ssh-ed25519 K3b7BA h4mC/hZ10ToaaYDRyBOyPpcvA28sY5FPCQPuaTTRIws
-VG4QtmEOnubhhjV3CS49aYOyVl/Dq+ryxfZENgFJZTo
--> ssh-ed25519 +qVung 6gs9DdduYx2twVsFED7HJnGFfKZynUctQIO4F3MXfj8
-gMmU2tXwR9K8Nb5gMKPbTexE58FOAK6QlVYzGvaX3hw
--> ssh-rsa krWCLQ
-vjNcmgDmmaNUSXIUgKf1digOgbohvyKkYSUalTOskvPo+9NRZbp0IJ7DoYLRrSBB
-DobCBM078iKOvIGGJCIbMS86/z/7lz6SSPcbfM1EG+hknVJLZaj+K3PYYSX6QTUC
-6rWSC+yg0gKehAhnYO3q+8mnismk7SERdyCZDNtPwHOhTAt6NZ6e+33VFxnbJPTz
-IvoNU/RTUhV+XuKbtosm55PqDkOuTM27jesZ0/SARYL+gVgaltacqt4kzbEMOP/W
-tv2kU6f1eNaX71c57DGI7rfcvLrPRAjTxUhsuKJPGQeaHtfiWz832gUMIJOEjoo0
-mvrAfyoykJRbPGNFl5pMmg
--> ssh-ed25519 /vwQcQ gpPktkJ57USbj7kn1qbeUQDbHHSCuzWM5OcmNooBMi8
-6JPXUJYQ1IjRVv90r1EJx3EUMDPmU9X1FK6j/6vT5hE
--> ssh-ed25519 0R97PA vzT774La7rcOMz7/KYjSUsY+D6V5bi5j3ghdDBLBoAU
-HAXfMmFuj3YJGCBR1U0btPlr9MdIBYnwT1ufbHaAxVk
---- /0DCLjy0dwjRGPnkNk/a9fZ1ox9+LVkwh9Y5jiyA8x4
-·1ë³KëB³|†Ü\<5C>öST¾¦i¸ð¦/<2F>ØÕ
hž9}%ä\÷Ÿþ,"g<>ì°Z³<5A>ÊšwÍþ0»ä5¸´
Rm’'

secrets/netbox-environment.age

@@ -1,21 +0,0 @@
-age-encryption.org/v1
--> ssh-ed25519 j2r2qQ 8qMRxnJL5p7M7Egtim/MZQTx0Z55dK1VKbR1drFkMRk
-q7AWFD4wg4eEIoPzPY3gmPNt9vSPv9s1TII2R0a4QoA
--> ssh-ed25519 K3b7BA tZtpUP6oDvY28vaLwzlLwlv/QQaDmbuwdPRvs2j3yxM
-gnJPbdLJML4MldoJTwsR/93ioOId53IvuSnpQwqmYoY
--> ssh-ed25519 +qVung fYyGsDgnf2wO4NZ+zOeiWWu3wLe001xHgZatXvVd60w
-Kxs9u1EZbP/abuBev0u9f8keraKibvoVDHqYvvbZJOA
--> ssh-rsa krWCLQ
-dGijkGbcpWqNgrsYSXGEYgLJadgf2imVRDZpMpR2SNqKeBgvIRSriwQQSUCnntZB
-pwul5dzZ9okr16xrghK66tCizBWwvfHtyACAFcI0xyCEf20Ydm1pbarSibK9RDb3
-JwJdvUor370sTkuWagBzM3+cfpeO8HhxEu46tNG1RP2EtEkdSXQ8056g7TrSUQt/
-XI385S5/WuurmBVlZuVTBXVsvGYU4OBAIlrYiym4loaSOGJMUCK8MZMfg+3w+RXW
-fScsZ0VS1eB4DkAiJEptJlesrpHPOegq+HyczxGAp0z7mcO0ffZBOrKzBQB7fsdV
-sn0R1gKpx9y9T6VE/uJ4Tg
--> ssh-ed25519 /vwQcQ mUHdSEXaTCrk2Nq/OPoo/3i8jXZfLbUBZewg6rwvdGQ
-7wxUPlQqkZXNo6zhqd/niQDUZWrKVzgWWkUPcW/ueds
--> ssh-ed25519 0R97PA nuM2B10VwPti+CBybZzBGLzo7SM9lHgKAM1CZj4U8iY
-3tfc6NC/D+lPPk5Fk6tDWbc15m4Eo/sI4WGTC33zQAc
---- efqcGqHksXsmGOFOwC+0UcYtUk+FuiGt4PHkHFzQ4OQ
-ˆ 3᫬ÒÊSâj´;…Dæ©æÄ6¾±R˜LݘƒmÄ+þÃc<07>˜ÞU¶WÅÐSü<53>yï!yÁËlwÅ ¼˜u3ÏõBb‰nloò	q
êÏTÑÙñ&é•.ªpš4‰¢‰¸!ó<>ËÎÔÞ€ÏÙ‚Ãò’ˆCì‚F¸¤$<13>§‰&
-<Ix¹LWy3çÔñqvbspêÐŽX²LÙob‡,-ÜJÌñ¯G’Ò³À6çéó“°û1¼s_rÆþgŒ7%×]÷»Œqž‡×›AôV¤*ï`«‹ê\Æ̽tLpöi®Þf

services/baremetal-builder/assignments.nix

@@ -0,0 +1,20 @@
+# This file contains information on which builder(s) are providing how many
+# job slots and providing which nix features
+let
+  genBuilders = { offset ? 0, count, f }: builtins.genList (x: rec { name = "builder-${toString (offset + x)}"; value = f name; }) count;
+in builtins.listToAttrs (
+  genBuilders { offset = 4; count = 2; f = name: {
+    cores = 8;
+    max-jobs = 8;
+    supported-features = [ "kvm" "nixos-test" ];
+    required-features = [ ];
+  }; }
+  ++
+  # This builder is exclusively for big-parallel
+  genBuilders { offset = 10; count = 1; f = name: {
+    cores = 20;
+    max-jobs = 1;
+    supported-features = [ "kvm" "nixos-test" "big-parallel" ];
+    required-features = [ "big-parallel" ];
+  }; }
+)

services/baremetal-builder/default.nix

@@ -3,10 +3,13 @@ let
   cfg = config.bagel.baremetal.builders;
 in
 {
+  imports = [ ./netboot.nix ];
+
   options = {
 
     bagel.baremetal.builders = {
       enable = lib.mkEnableOption "baremetal bagel oven";
+      netboot = lib.mkEnableOption "netboot";
       num = lib.mkOption {
         type = lib.types.int;
       };
@@ -28,8 +31,22 @@ in
         "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAvUT9YBig9LQPHgypIBHQuC32XqDKxlFZ2CfgDi0ZKx"
       ];
     };
-    nix.settings.trusted-users = [ "builder" ];
 
+    users.users.buildbot = {
+      isSystemUser = true;
+      group = "nogroup";
+      home = "/var/empty";
+      shell = "/bin/sh";
+      openssh.authorizedKeys.keys = [
+        # Do not hardcode Buildbot's public key, selectively
+        # add the keys of the coordinators that require us.
+        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGMnOLLX0vGTZbSJrUmF9ZFXt/NIId/MUrEpXmL2vxod"
+      ];
+    };
+    nix.settings = {
+      trusted-users = [ "builder" "buildbot" ];
+      inherit ((import ./assignments.nix).${config.networking.hostName}) max-jobs cores;
+    };
 
     nixpkgs.hostPlatform = "x86_64-linux";
     hardware.cpu.intel.updateMicrocode = true;
@@ -40,18 +57,36 @@ in
 
     boot.initrd.services.lvm.enable = true;
 
-    fileSystems."/" = {
-      device = "/dev/disk/by-label/root";
-      fsType = "xfs";
-    };
+    boot.kernel.sysctl."fs.xfs.xfssyncd_centisecs" = "12000";
+    fileSystems = lib.mkMerge [
+      (lib.mkIf (!cfg.netboot) {
+        "/" = {
+          device = "/dev/disk/by-label/root";
+          fsType = "xfs";
+        };
 
-    fileSystems."/boot" = {
-      device = "/dev/disk/by-label/BOOT";
-      fsType = "vfat";
-      options = [ "fmask=0022" "dmask=0022" ];
-    };
+        "/boot" = {
+          device = "/dev/disk/by-label/BOOT";
+          fsType = "vfat";
+          options = [ "fmask=0022" "dmask=0022" ];
+        };
+      })
+      {
+        "/mnt" = {
+          device = "/dev/disk/by-label/hydra";
+          fsType = "xfs";
+          options = ["logbsize=256k"];
+        };
 
-    swapDevices = [
+        # We want the tmp filesystem on the same filesystem as the hydra store, so that builds can use reflinks
+        "/tmp" = {
+          device = "/mnt/tmp";
+          options = [ "bind" ];
+        };
+      }
+    ];
+
+    swapDevices = lib.optionals (!cfg.netboot) [
       {
         device = "/swapfile";
         size = 50 * 1024;  # 50GiB
@@ -64,8 +99,8 @@ in
     };
 
     boot.kernelParams = [
-      "console=ttyS0,115200"
       "console=tty1"
+      "console=ttyS0,115200"
     ];
 
     networking.useNetworkd = true;
@@ -100,6 +135,11 @@ in
       { address = "2a01:584:11::1:${toString cfg.num}"; prefixLength = 64; }
     ];
     networking.defaultGateway6 = { interface = "uplink"; address = "2a01:584:11::1"; };
+    bagel.infra.self.wan = {
+      family = "inet6";
+      address = "2a01:584:11::1:${toString cfg.num}";
+      prefixLength = 64;
+    };
     deployment.targetHost = "2a01:584:11::1:${toString cfg.num}";
     deployment.tags = [ "builders" ];
 
@@ -123,6 +163,27 @@ in
       MaxStartups = "500:30:1000";
     };
 
+    systemd.services.hydra-gc = {
+      wantedBy = [ "multi-user.target" ];
+      description = "Nix Garbage Collector";
+      script = ''
+        while : ; do
+          percent_filled=$(($(stat -f --format="100-(100*%a/%b)" /mnt)))
+          if [ "$percent_filled" -gt "85" ]; then
+            ${config.nix.package.out}/bin/nix-store --gc --max-freed 80G --store /mnt
+          else
+            break
+          fi
+        done
+      '';
+      serviceConfig.Type = "oneshot";
+      serviceConfig.User = "builder";
+    };
+    systemd.timers.hydra-gc = {
+      timerConfig.OnUnitInactiveSec = "10min";
+      wantedBy = [ "timers.target" ];
+    };
+    systemd.timers.hydra-gc.timerConfig.Persistent = true;
 
     bagel.sysadmin.enable = true;
 

services/baremetal-builder/netboot.nix

@@ -0,0 +1,169 @@
+{ modulesPath, pkgs, lib, config, extendModules, ... }@node:
+let
+  cfg = config.bagel.baremetal.builders;
+in
+{
+  config = lib.mkIf (cfg.enable && cfg.netboot) {
+    systemd.services.sshd.after = [ "provision-ssh-hostkey.service" ];
+    systemd.services.provision-ssh-hostkey = {
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = true;
+      };
+      script = ''
+        mkdir -p /etc/ssh
+        umask 0077
+        until ${pkgs.iputils}/bin/ping -c 1 vpn-gw.wob01.infra.forkos.org; do sleep 1; done
+        ${pkgs.curl}/bin/curl --local-port 25-1024 https://vpn-gw.wob01.infra.forkos.org/${config.networking.hostName}/ssh_host_ed25519_key > /etc/ssh/ssh_host_ed25519_key
+        # Run the activation script again to trigger agenix decryption
+        /run/current-system/activate
+      '';
+    };
+
+    system.build = {
+
+      # Build a kernel and initramfs which will download the IPXE script from hydra using
+      # u-root pxeboot tool and kexec into the final netbooted system.
+      notipxe = import (modulesPath + "/..") {
+        system = "x86_64-linux";
+        configuration = 
+          { pkgs, config, ... }:
+
+          {
+            system.stateVersion = "24.11";
+            boot.initrd.availableKernelModules = [ "ahci" "ehci_pci" "usb_storage" "usbhid" "sd_mod" "igb" "bonding" ];
+            boot.kernelParams = [ "console=ttyS0,115200" "panic=1" "boot.panic_on_fail" ];
+            #boot.initrd.systemd.emergencyAccess = true;
+            networking.hostName = "${node.config.networking.hostName}-boot";
+            nixpkgs.overlays = import ../../overlays;
+            boot.loader.grub.enable = false;
+            fileSystems."/".device = "bogus"; # this config will never be booted
+            boot.initrd.systemd.enable = true;
+            boot.initrd.systemd.network = {
+              enable = true;
+              networks = node.config.systemd.network.networks;
+              netdevs = node.config.systemd.network.netdevs;
+            };
+            boot.initrd.systemd.storePaths = [
+              "${pkgs.u-root}/bin/pxeboot"
+              "${pkgs.iputils}/bin/ping"
+            ];
+            boot.initrd.systemd.services.kexec = {
+              serviceConfig.Restart = "on-failure";
+              serviceConfig.Type = "oneshot";
+              wantedBy = [ "initrd-root-fs.target" ];
+              before = [ "sysroot.mount" ];
+              script = ''
+                ln -sf /dev/console /dev/tty
+                until ${pkgs.iputils}/bin/ping -c 1 hydra.forkos.org; do sleep 1; done
+                ${pkgs.u-root}/bin/pxeboot -v -ipv4=false -file https://hydra.forkos.org/job/infra/main/${node.config.networking.hostName}/latest/download-by-type/file/ipxe 
+              '';
+            };
+            boot.initrd.systemd.contents."/etc/ssl/certs/ca-certificates.crt".source = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
+            boot.initrd.services.resolved.enable = false;
+            boot.initrd.systemd.contents."/etc/resolv.conf".text = ''
+              nameserver 2001:4860:4860::6464
+            '';
+            boot.initrd.systemd.contents."/etc/systemd/journald.conf".text = ''
+              [Journal]
+              ForwardToConsole=yes
+              MaxLevelConsole=debug
+            '';
+
+            # Provide a bootable USB drive image
+            system.build.usbImage = pkgs.callPackage ({ stdenv, runCommand, dosfstools, e2fsprogs, mtools, libfaketime, util-linux, nukeReferences }:
+              runCommand "boot-img-${node.config.networking.hostName}" {
+                nativeBuildInputs = [ dosfstools e2fsprogs libfaketime mtools util-linux ];
+                outputs = [ "out" "firmware_part" ];
+              } ''
+                export img=$out
+                truncate -s 40M $img
+
+                sfdisk $img <<EOF
+                    label: gpt
+                    label-id: F222513B-DED1-49FA-B591-20CE86A2FE7F
+
+                    type=C12A7328-F81F-11D2-BA4B-00A0C93EC93B, bootable
+                EOF
+
+                # Create a FAT32 /boot/firmware partition of suitable size into firmware_part.img
+                eval $(partx $img -o START,SECTORS --nr 1 --pairs)
+                truncate -s $((2081 * 512 + SECTORS * 512)) firmware_part.img
+
+                mkfs.vfat --invariant -i 2e24ec82 -n BOOT firmware_part.img
+
+                # Populate the files intended for /boot/firmware
+                mkdir -p firmware/EFI/BOOT firmware/loader/entries
+                cp ${pkgs.systemd}/lib/systemd/boot/efi/systemd-boot*.efi firmware/EFI/BOOT/BOOT${lib.toUpper stdenv.hostPlatform.efiArch}.EFI
+
+                cat > firmware/loader/loader.conf << EOF
+                default foo
+                EOF
+                cat > firmware/loader/entries/default.conf << EOF
+                title Default
+                linux /EFI/${pkgs.stdenv.hostPlatform.linux-kernel.target}
+                initrd /EFI/initrd
+                options init=${config.system.build.toplevel}/init ${toString config.boot.kernelParams}
+                EOF
+                cp ${config.system.build.kernel}/${pkgs.stdenv.hostPlatform.linux-kernel.target} firmware/EFI/${pkgs.stdenv.hostPlatform.linux-kernel.target}
+                cp ${config.system.build.initialRamdisk}/${config.system.boot.loader.initrdFile} firmware/EFI/initrd
+
+                find firmware -exec touch --date=2000-01-01 {} +
+                # Copy the populated /boot/firmware into the SD image
+                cd firmware
+                # Force a fixed order in mcopy for better determinism, and avoid file globbing
+                for d in $(find . -type d -mindepth 1 | sort); do
+                  faketime "2000-01-01 00:00:00" mmd -i ../firmware_part.img "::/$d"
+                done
+                for f in $(find . -type f | sort); do
+                  mcopy -pvm -i ../firmware_part.img "$f" "::/$f"
+                done
+                cd ..
+
+                # Verify the FAT partition before copying it.
+                fsck.vfat -vn firmware_part.img
+                dd conv=notrunc if=firmware_part.img of=$img seek=$START count=$SECTORS
+
+                cp firmware_part.img $firmware_part
+              ''
+            ) {};
+          }
+        ;
+      };
+
+      # This is the config which will actually be booted
+      netbootVariant = extendModules {
+        modules = [
+          (
+            { modulesPath, ... }:
+
+            {
+              imports = [ (modulesPath + "/installer/netboot/netboot.nix") ];
+            }
+          )
+        ];
+      };
+      # A derivation combining all the artifacts required for netbooting for the hydra job
+      netbootDir = let
+        kernelTarget = pkgs.stdenv.hostPlatform.linux-kernel.target;
+        build = config.system.build.netbootVariant.config.system.build;
+      in
+        pkgs.symlinkJoin {
+          name = "netboot";
+          paths = [
+            build.netbootRamdisk
+            build.kernel
+            build.netbootIpxeScript
+          ];
+          postBuild = ''
+            mkdir -p $out/nix-support
+            echo "file ${kernelTarget} $out/${kernelTarget}" >> $out/nix-support/hydra-build-products
+            echo "file initrd $out/initrd" >> $out/nix-support/hydra-build-products
+            echo "file ipxe $out/netboot.ipxe" >> $out/nix-support/hydra-build-products
+          '';
+          preferLocalBuild = true;
+        };
+    };
+  };
+}

services/block-crawlers/blocked-ua.txt

@@ -0,0 +1,40 @@
+AI2Bot
+Ai2Bot-Dolma
+Amazonbot
+anthropic-ai
+Applebot
+Applebot-Extended
+Bytespider
+CCBot
+ChatGPT-User
+Claude-Web
+ClaudeBot
+cohere-ai
+Diffbot
+FacebookBot
+facebookexternalhit
+FriendlyCrawler
+Google-Extended
+GoogleOther
+GoogleOther-Image
+GoogleOther-Video
+GPTBot
+iaskspider/2.0
+ICC-Crawler
+ImagesiftBot
+img2dataset
+ISSCyberRiskCrawler
+Kangaroo Bot
+Meta-ExternalAgent
+Meta-ExternalFetcher
+OAI-SearchBot
+omgili
+omgilibot
+PerplexityBot
+PetalBot
+Scrapy
+Sidetrade indexer bot
+Timpibot
+VelenPublicWebCrawler
+Webzio-Extended
+YouBot

services/block-crawlers/default.nix

@@ -0,0 +1,32 @@
+{ pkgs, config, lib, ... }:
+let
+  inherit (lib) mkEnableOption mkIf mkOption types concatStringsSep mkDefault splitString;
+  cfg = config.bagel.services.nginx.crawler-blocker;
+  mkRobotsFile = blockedUAs: pkgs.writeText "robots.txt" ''
+    ${concatStringsSep "\n" (map (ua: "User-agent: ${ua}") blockedUAs)}
+    Disallow: /
+  '';
+in
+{
+  options = {
+    bagel.services.nginx.crawler-blocker = {
+      enable = mkEnableOption "the crawler blocker";
+
+      userAgents = mkOption {
+        type = types.listOf types.str;
+        default = splitString "\n" (builtins.readFile ./blocked-ua.txt);
+      };
+    };
+
+    services.nginx.virtualHosts = mkOption {
+      type = types.attrsOf (types.submodule {
+        config = {
+          locations."= /robots.txt" = mkIf cfg.enable (mkDefault {
+            alias = mkRobotsFile cfg.userAgents;
+          });
+        };
+      });
+    };
+  };
+}
+

services/buildbot/default.nix

@@ -0,0 +1,233 @@
+{
+  nodes,
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.bagel.services.buildbot;
+  ssh-keys = import ../../common/ssh-keys.nix;
+  freeGbDiskSpace = 20;
+  extraTenantSpecificBuilders = {
+    lix = import ./lix.nix {
+      inherit config nodes;
+    };
+    floral = [ ];
+  }.${cfg.tenant or (throw "${cfg.tenant} is not a known tenant")};
+  clientId = {
+    lix = "buildbot";
+    floral = "forkos-buildbot";
+  }.${cfg.tenant or (throw "${cfg.tenant} is not a known tenant")};
+  inherit (lib) mkEnableOption mkOption mkIf types;
+in
+{
+  options.bagel.services.buildbot = {
+    enable = mkEnableOption "Buildbot";
+
+    tenant = mkOption {
+      type = types.enum [ "lix" "floral" ];
+      description = "Which buildbot tenant to enable";
+    };
+
+    domain = mkOption {
+      type = types.str;
+      description = "Domain name for this Buildbot";
+    };
+
+    gerrit = {
+      domain = mkOption {
+        type = types.str;
+        description = "Canonical domain of the Gerrit associated to this Buildbot";
+        example = [ "cl.forkos.org" ];
+      };
+
+      port = mkOption {
+        type = types.port;
+        description = "Gerrit SSH port for this Buildbot";
+      };
+
+      username = mkOption {
+        type = types.str;
+        description = "Gerrit service username for this Buildbot";
+      };
+    };
+
+    cors.allowedOrigins = mkOption {
+      type = types.listOf types.str;
+      example = [ "*.forkos.org" ];
+      description = "Allowed origin for Buildbot and NGINX for CORS without the protocol";
+    };
+
+    buildSystems = mkOption {
+      type = types.listOf (types.enum [ "x86_64-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" ]);
+      default = [ "x86_64-linux" ];
+      example = [ "x86_64-linux" "aarch64-linux" ];
+      description = "Supported build systems for this buildbot instance.";
+    };
+
+    projects = mkOption {
+      type = types.listOf types.str;
+      example = [ "nixpkgs" ];
+      description = "Static list of projects enabled for Buildbot CI";
+    };
+
+    builders = mkOption {
+      type = types.listOf types.str;
+      description = "List of builders to configure for Buildbot";
+      example = [ "builder-2" "builder-3" ];
+    };
+  };
+
+  config = mkIf cfg.enable {
+    networking.firewall.allowedTCPPorts = [ 80 443 ];
+    bagel.secrets.files = [
+      "buildbot-worker-password"
+      "buildbot-oauth-secret"
+      "buildbot-workers"
+      "buildbot-service-key"
+      "buildbot-signing-key"
+      "buildbot-remote-builder-key"
+    ];
+    age.secrets.buildbot-signing-key = {
+      owner = "buildbot-worker";
+      group = "buildbot-worker";
+    };
+    age.secrets.buildbot-remote-builder-key = {
+      file = ../../secrets/${cfg.tenant}/buildbot-remote-builder-key.age;
+      owner = "buildbot-worker";
+      group = "buildbot-worker";
+    };
+
+    services.nginx = {
+      recommendedProxySettings = true;
+      appendHttpConfig = ''
+        # Our session stuff is too big with the TWISTED_COOKIE in addition.
+        # Default is usually 4k or 8k.
+        large_client_header_buffers 4 16k;
+      '';
+      virtualHosts.${cfg.domain} = {
+        forceSSL = true;
+        enableACME = true;
+        extraConfig = ''
+          # This is needed so that logged-in users in Buildbot can include their credentials in their requests.
+          add_header Access-Control-Allow-Credentials 'true' always;
+        '';
+      };
+    };
+
+    services.buildbot-nix.worker = {
+      enable = true;
+      workerPasswordFile = config.age.secrets.buildbot-worker-password.path;
+      # All credits to eldritch horrors for this beauty.
+      workerArchitectures =
+      {
+        # nix-eval-jobs runs under a lock, error reports do not (but are cheap)
+        other = 8;
+      } // (
+        lib.filterAttrs
+          (n: v: lib.elem n config.services.buildbot-nix.coordinator.buildSystems)
+          (lib.zipAttrsWith
+            (_: lib.foldl' lib.add 0)
+            (lib.concatMap
+              (m: map (s: { ${s} = m.maxJobs; }) m.systems)
+              config.services.buildbot-nix.coordinator.buildMachines))
+        );
+    };
+
+    services.buildbot-nix.coordinator = {
+      enable = true;
+
+      inherit (cfg) domain;
+      # TODO(raito): is that really necessary when we can just collect buildMachines' systems?
+      inherit (cfg) buildSystems;
+
+      oauth2 = {
+        name = "Lix";
+        inherit clientId;
+        clientSecretFile = config.age.secrets.buildbot-oauth-secret.path;
+        resourceEndpoint = "https://identity.lix.systems";
+        authUri = "https://identity.lix.systems/realms/lix-project/protocol/openid-connect/auth";
+        tokenUri = "https://identity.lix.systems/realms/lix-project/protocol/openid-connect/token";
+        userinfoUri = "https://identity.lix.systems/realms/lix-project/protocol/openid-connect/userinfo";
+      };
+
+      # TODO(raito): this is not really necessary, we never have remote buildbot workers.
+      # we can replace all of this with automatic localworker generation on buildbot-nix side.
+      workersFile = config.age.secrets.buildbot-workers.path;
+
+      # We rely on NGINX to do the CORS dance.
+      allowedOrigins = cfg.cors.allowedOrigins;
+
+      buildMachines = map (n: {
+          hostName = nodes.${n}.config.networking.fqdn;
+          protocol = "ssh-ng";
+          # Follows Hydra.
+          maxJobs = 8;
+          sshKey = config.age.secrets.buildbot-remote-builder-key.path;
+          sshUser = "buildbot";
+          systems = [ "x86_64-linux" ];
+          supportedFeatures = nodes.${n}.config.nix.settings.system-features;
+          # Contrary to how Nix works, here we can specify non-base64 public host keys.
+          publicHostKey = ssh-keys.machines.${n};
+        }
+      ) cfg.builders ++ extraTenantSpecificBuilders;
+
+      gerrit = {
+        # Manually managed account…
+        # TODO: https://git.lix.systems/the-distro/infra/issues/69
+        inherit (cfg.gerrit) domain port username;
+        privateKeyFile = config.age.secrets.buildbot-service-key.path;
+        inherit (cfg) projects;
+      };
+
+      evalWorkerCount = 6;
+      evalMaxMemorySize = "4096";
+
+      signingKeyFile = config.age.secrets.buildbot-signing-key.path;
+    };
+
+    # Make PostgreSQL restart smoother.
+    systemd.services.postgresql.serviceConfig = {
+      Restart = "always";
+      RestartMaxDelaySec = "5m";
+      RestartSteps = 10;
+    };
+
+    services.postgresql.settings = {
+      # DB Version: 15
+      # OS Type: linux
+      # DB Type: web
+      # Total Memory (RAM): 64 GB
+      # CPUs num: 16
+      # Connections num: 100
+      # Data Storage: ssd
+      max_connections = 100;
+      shared_buffers = "16GB";
+      effective_cache_size = "48GB";
+      maintenance_work_mem = "2GB";
+      checkpoint_completion_target = 0.9;
+      wal_buffers = "16MB";
+      default_statistics_target = 100;
+      random_page_cost = 1.1;
+      effective_io_concurrency = 200;
+      work_mem = "41943kB";
+      huge_pages = "try";
+      min_wal_size = "1GB";
+      max_wal_size = "4GB";
+      max_worker_processes = 16;
+      max_parallel_workers_per_gather = 4;
+      max_parallel_workers = 16;
+      max_parallel_maintenance_workers = 4;
+    };
+
+    nix.settings.keep-derivations = true;
+    nix.gc = {
+      automatic = true;
+      dates = "hourly";
+      options = ''
+        --max-freed "$((${toString freeGbDiskSpace} * 1024**3 - 1024 * $(df -P -k /nix/store | tail -n 1 | ${pkgs.gawk}/bin/awk '{ print $4 }')))"
+      '';
+    };
+  };
+}

services/buildbot/lix.nix

@@ -0,0 +1,50 @@
+{ config, nodes, ... }:
+let
+  ssh-keys = import ../../common/ssh-keys.nix;
+in
+  [
+    {
+      hostName = "build01.aarch64.lix.systems";
+      maxJobs = 2;
+      protocol = "ssh-ng";
+      sshKey = config.age.secrets.buildbot-remote-builder-key.path;
+      sshUser = "nix";
+      systems = [ "aarch64-linux" ];
+      publicHostKey = ssh-keys.machines.build01-aarch64-lix;
+      supportedFeatures = nodes.build01-aarch64-lix.config.nix.settings.system-features;
+    }
+    {
+      hostName = "build02.aarch64.lix.systems";
+      maxJobs = 4;
+      protocol = "ssh-ng";
+      sshKey = config.age.secrets.buildbot-remote-builder-key.path;
+      sshUser = "nix";
+      systems = [ "aarch64-linux" ];
+      publicHostKey = ssh-keys.machines.build02-aarch64-lix;
+      # TODO: use build02 features.
+      supportedFeatures = nodes.build01-aarch64-lix.config.nix.settings.system-features;
+    }
+    {
+      hostName = "build01.aarch64-darwin.lix.systems";
+      maxJobs = 2;
+      protocol = "ssh-ng";
+      sshKey = config.age.secrets.buildbot-remote-builder-key.path;
+      sshUser = "m1";
+      systems = [ "aarch64-darwin" "x86_64-darwin" ];
+      publicHostKey = ssh-keys.machines.build01-aarch64-darwin-lix;
+      supportedFeatures = [ "big-parallel" ];
+    }
+    # a.k.a. https://git.newtype.fr/newtype/newtype-org-configurations/src/branch/main/docs/epyc.md
+    {
+      hostName = "epyc.infra.newtype.fr";
+      # at 256G this could run 64 builds but the machine is shared
+      # (and historically we used no more than 16 concurrent jobs)
+      maxJobs = 16;
+      protocol = "ssh-ng";
+      sshKey = config.age.secrets.buildbot-remote-builder-key.path;
+      sshUser = "nix";
+      systems = [ "x86_64-linux" "i686-linux" ];
+      publicHostKey = ssh-keys.machines.epyc-newtype-fr;
+      supportedFeatures = [ "benchmark" "big-parallel" "nixos-test" "kvm" ];
+    }
+  ]

services/channel-scripts/default.nix

@@ -0,0 +1,221 @@
+{ lib, config, pkgs, ... }:
+# FIXME(raito): I'm really really really not happy with this design of NixOS module, clean up all of this someday.
+let
+  inherit (lib) mkEnableOption mkOption types mkIf mapAttrsToList mkPackageOption concatStringsSep mkMerge;
+  cfg = config.bagel.nixpkgs.channel-scripts;
+  toml = pkgs.formats.toml { };
+  configFile = toml.generate "forkos.toml" cfg.settings;
+  orderLib = import ./service-order.nix { inherit lib; };
+  makeUpdateJob = channelName: mainJob: {
+    name = "update-${channelName}";
+    value = {
+      description = "Update channel ${channelName}";
+      path = with pkgs; [ git ];
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = false;
+        User = "channel-scripts";
+        DynamicUser = true;
+        StateDirectory = "channel-scripts";
+        MemoryHigh = "80%";
+        EnvironmentFile = [
+          cfg.releaseBucketCredentialsFile
+        ];
+        Environment = cfg.extraEnvironment;
+        # TODO: we should have our own secret for this.
+        LoadCredential = [ "password:${config.age.secrets.alloy-push-password.path}" ];
+      };
+      unitConfig.After = [ "networking.target" ];
+      script =
+      ''
+        # A stateful copy of nixpkgs
+        dir=/var/lib/channel-scripts/nixpkgs
+        if ! [[ -e $dir ]]; then
+          git clone --bare ${cfg.nixpkgsUrl} $dir
+        fi
+        GIT_DIR=$dir git config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*'
+
+        CREDENTIAL=$(echo -en "promtail:$(cat $CREDENTIALS_DIRECTORY/password)" | base64)
+        export OTEL_EXPORTER_OTLP_HEADERS="Authorization=Basic $CREDENTIAL"
+        # TODO: use escapeShellArgs
+        exec ${cfg.package}/bin/mirror-forkos -c ${configFile} ${concatStringsSep " " cfg.extraArgs} apply ${channelName} ${mainJob}
+      '';
+    };
+  };
+  updateJobs = orderLib.mkOrderedChain (mapAttrsToList (n: { job, ... }: makeUpdateJob n job) cfg.channels);
+  channelOpts = { ... }: {
+    options = {
+      job = mkOption {
+        type = types.str;
+        example = "nixos/trunk-combined/tested";
+      };
+
+      variant = mkOption {
+        type = types.enum [ "primary" "small" "darwin" "aarch64" ];
+        example = "primary";
+      };
+
+      status = mkOption {
+        type = types.enum [ "beta" "stable" "deprecated" "unmaintained" "rolling" ];
+        example = "rolling";
+      };
+    };
+  };
+in
+{
+  options.bagel.nixpkgs.channel-scripts = {
+    enable = mkEnableOption ''the channel scripts.
+      Fast forwarding channel branches which are read-only except for this privileged bot
+      based on our Hydra acceptance tests.
+    '';
+
+    otlp.enable = mkEnableOption "the OTLP export process";
+
+    s3 = {
+      release = mkOption {
+        type = types.str;
+      };
+
+      channel = mkOption {
+        type = types.str;
+      };
+    };
+
+    package = mkPackageOption pkgs "mirror-forkos" { };
+
+    settings = mkOption {
+      type = types.attrsOf types.anything;
+    };
+
+    nixpkgsUrl = mkOption {
+      type = types.str;
+      default = "https://cl.forkos.org/nixpkgs.git";
+      description = "URL to the nixpkgs repository to clone and to push to";
+    };
+
+    binaryCacheUrl = mkOption {
+      type = types.str;
+      default = "https://cache.forkos.org";
+      description = "URL to the binary cache";
+    };
+
+    baseUriForGitRevisions = mkOption {
+      type = types.str;
+      description = "Base URI to generate link to a certain revision";
+    };
+
+    extraArgs = mkOption {
+      type = types.listOf types.str;
+      default = [ ];
+      description = "Extra arguments passed to the mirroring program";
+    };
+
+    releaseBucketCredentialsFile = mkOption {
+      type = types.path;
+      description = ''Path to the release bucket credentials file exporting S3-style environment variables.
+        For example, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` for the S3 operations to work.
+      '';
+    };
+
+    deployKeyFile = mkOption {
+      type = types.path;
+      description = ''Path to the private SSH key which is allowed to deploy things to the protected channel references on the Git repository.
+      '';
+    };
+
+    hydraUrl = mkOption {
+      type = types.str;
+      default = "https://hydra.forkos.org";
+      description = "URL to the Hydra instance";
+    };
+
+    channels = mkOption {
+      type = types.attrsOf (types.submodule channelOpts);
+      description = "List of channels to mirror";
+    };
+
+    extraEnvironment = mkOption {
+      type = types.listOf types.str;
+    };
+  };
+
+  config = mkIf cfg.enable {
+    bagel.nixpkgs.channel-scripts.extraEnvironment = mkMerge [
+      ([
+        "RUST_LOG=info"
+      ])
+      (mkIf cfg.otlp.enable [
+        ''OTEL_EXPORTER_OTLP_TRACES_ENDPOINT="https://tempo.forkos.org/v1/traces"''
+      ])
+    ];
+    bagel.nixpkgs.channel-scripts.settings = {
+      hydra_uri = cfg.hydraUrl;
+      binary_cache_uri = cfg.binaryCacheUrl;
+      base_git_uri_for_revision = cfg.baseUriForGitRevisions;
+      # TODO: this leaks information about where channel-scripts are hosted.
+      # Cleanup this later with a proper module option.
+      repo_dir = "/gerrit-data/channel-scripts/nixpkgs";
+      s3_release_bucket_name = cfg.s3.release;
+      s3_channel_bucket_name = cfg.s3.channel;
+    };
+
+    users.users.channel-scripts = {
+      description = "Channel scripts user";
+      isSystemUser = true;
+      group = "channel-scripts";
+    };
+    users.groups.channel-scripts = {};
+
+    systemd.services = (lib.listToAttrs updateJobs) // {
+      "update-all-channels" = {
+        description = "Start all channel updates.";
+        unitConfig = {
+          After = map
+            (service: "${service.name}.service")
+            updateJobs;
+          Wants = map
+            (service: "${service.name}.service")
+            updateJobs;
+        };
+        script = "true";
+      };
+      "cleanup-failed-streaming-prefixes" = {
+        description = "Cleanup all failed streaming prefixes on the channel bucket (channel-scripts)";
+        conflicts = map (service: "${service.name}.service") updateJobs;
+        after = [ "networking.target" ];
+        serviceConfig = {
+          Type = "oneshot";
+          RemainAfterExit = false;
+          User = "channel-scripts";
+          DynamicUser = true;
+          StateDirectory = "channel-scripts";
+          EnvironmentFile = [
+            cfg.releaseBucketCredentialsFile
+          ];
+          Environment = cfg.extraEnvironment;
+          LoadCredential = [ "password:${config.age.secrets.alloy-push-password.path}" ];
+          ExecStart = "${cfg.package}/bin/mirror-forkos -c ${configFile} ${concatStringsSep " " cfg.extraArgs} cleanup-streamed-prefixes";
+        };
+      };
+    };
+
+    systemd.timers."update-all-channels" = {
+      description = "Start all channel updates.";
+      wantedBy = [ "timers.target" ];
+      timerConfig = {
+        OnUnitInactiveSec = 600;
+        OnBootSec = 900;
+        AccuracySec = 300;
+      };
+    };
+
+    systemd.timers."cleanup-failed-streaming-prefixes" = {
+      description = "Cleanup failed streaming prefixes for channel-scripts";
+      wantedBy = [ "timers.target" ];
+      timerConfig = {
+        OnCalendar = "daily";
+        RandomizedDelaySec = "1h";
+      };
+    };
+  };
+}