feat: block automatically crawlers if the blocker is enabled

This help us getting rid of useless traffic by crawlers.

It is enabled for gerrit01 which is suffering the most from this.

Signed-off-by: Raito Bezarius <masterancpp@gmail.com>

.gitignore

@@ -4,3 +4,5 @@ config.tf.json
 .direnv
 .terraform
 .terraform.lock.hcl
+secrets/*
+!secrets/*.age

README.md

@@ -1 +1,32 @@
-Infrastructure for the donut shaped thing that is absolutely not a donut.
+# Infrastructure for the donut shaped thing that is absolutely not a donut.
+
+## Quick start
+
+### Build the infrastructure
+
+```
+$ colmena build --on @localboot
+```
+
+Notice that `@localboot` is load-bearing as we have some machines that _cannot be_ deployed with vanilla Colmena. Fixing this is welcome.
+
+### Recommended deploy process
+
+```
+$ colmena apply dry-activate $machine # Verify that the nvd log is reasonable.
+$ colmena apply $machine
+```
+
+### Recommended upgrade process
+
+```
+$ nix flake update
+$ colmena apply dry-activate --on @localboot # Verify that the nvd log is reasonable. Run it twice to get only NVD logs shown.
+$ colmena apply --on @localboot
+```
+
+## Troubleshooting
+
+### I failed to deploy `gerrit01`
+
+Our Gerrit source build is known to have some hiccups sometimes, we are always interested in build logs, feel free to attach information in a new issue so we can make it more reliable.

common/admins.nix

@@ -1,13 +1,45 @@
+{ lib, ... }:
 let
-  keys = import ./ssh-keys.nix;
-in {
-  users.users.root.openssh.authorizedKeys.keys = 
-    keys.users.delroth ++
-    keys.users.k900 ++ 
-    keys.users.raito ++
-    keys.users.maxine ++ 
-    keys.users.jade ++
-    keys.users.janik ++
-    keys.users.lukegb ++
-    keys.users.yuka;
+  inherit (lib) genAttrs;
+in
+# Note: to add somefew in this list. 
+# Ensure their SSH key is already in common/ssh-keys.nix with
+# the same username for here, so that the keys is automatically added.
+{
+  bagel.groups = {
+    floral-infra.members = [
+      "delroth"
+      "emilylange"
+      "hexchen"
+      "jade"
+      "janik"
+      "k900"
+      "maxine"
+      "raito"
+      "thubrecht"
+      "winter"
+      "yuka"
+      "ckie"
+    ];
+
+    lix-infra.members = [
+      "raito"
+      "hexchen"
+      "jade"
+    ];
+  };
+  bagel.users = genAttrs [
+    "delroth"
+    "emilylange"
+    "hexchen"
+    "jade"
+    "janik"
+    "k900"
+    "maxine"
+    "raito"
+    "thubrecht"
+    "winter"
+    "yuka"
+    "ckie"
+  ] (name: {});
 }

common/base-server.nix

@@ -1,4 +1,9 @@
 { lib, pkgs, ... }: {
+  imports = [
+    ./known-ssh-keys.nix
+    ./cgroups.nix
+  ];
+
   nixpkgs.overlays = import ../overlays;
 
   nix.package = lib.mkDefault pkgs.lix;
@@ -25,8 +30,8 @@
   nix.gc = {
     automatic = true;
     persistent = true;
-    dates = "daily";
-    options = "--delete-older-than 30d";
+    dates = lib.mkDefault "daily";
+    options = lib.mkDefault "--delete-older-than 30d";
   };
 
   services.journald.extraConfig = "SystemMaxUse=512M";
@@ -51,4 +56,19 @@
     "en_US.UTF-8/UTF-8"
     "fr_FR.UTF-8/UTF-8"
   ];
+
+  time.timeZone = "UTC";
+
+  security.acme.acceptTerms = true;
+  security.acme.defaults.email = "infra@forkos.org";
+
+  # Enable system diffs.
+  system.activationScripts.system-diff = {
+    supportsDryActivation = true; # safe: only outputs to stdout
+    text = ''
+      if [ -e /run/current-system ]; then
+        PATH=$PATH:${pkgs.nix}/bin ${pkgs.nvd}/bin/nvd diff /run/current-system $systemConfig
+      fi
+    '';
+  };
 }

common/cgroups.nix

@@ -0,0 +1,83 @@
+# Relatively inspired by fbtax2:
+# https://facebookmicrosites.github.io/cgroup2/docs/fbtax-results.html
+#
+# See also the Chris Down talk at LISA'21:
+# https://www.usenix.org/conference/lisa21/presentation/down
+{ ... }:
+let
+  systemCriticalSliceConfig = {
+    ManagedOOMMemoryPressure = "kill";
+
+    # guarantee availability of memory
+    MemoryMin = "192M";
+    # default 100
+    IOWeight = 1000;
+    # default 100
+    CPUWeight = 1000;
+  };
+in
+{
+  systemd.oomd = {
+    enable = true;
+    # why not, we have cgroups at user level now so it'll just kill the
+    # terminal
+    enableRootSlice = true;
+    enableSystemSlice = true;
+    enableUserSlices = true;
+  };
+
+  systemd.enableCgroupAccounting = true;
+
+  systemd.services.nix-daemon = {
+    serviceConfig = {
+      # FIXME: how do i deprioritize this for memory
+      CPUWeight = 10;
+      IOWeight = 10;
+    };
+  };
+
+  systemd.slices.hostcritical = {
+    description = "Ensures that services to keep the system alive remain alive";
+
+    unitConfig = {
+      # required to avoid a dependency cycle on systemd-oomd. systemd will
+      # actually guess this right but we should fix it anyway.
+      DefaultDependencies = false;
+    };
+
+    sliceConfig = systemCriticalSliceConfig;
+  };
+
+  # make root logins higher priority for resources
+  systemd.slices."user-0" = {
+    sliceConfig = systemCriticalSliceConfig;
+  };
+
+
+  systemd.slices.system = {
+    sliceConfig = {
+      ManagedOOMMemoryPressure = "kill";
+      ManagedOOMMemoryPressureLimit = "50%";
+
+      IOWeight = 100;
+    };
+  };
+
+  systemd.services.sshd = {
+    serviceConfig = {
+      Slice = "hostcritical.slice";
+    };
+  };
+
+  systemd.services.systemd-oomd = {
+    serviceConfig = {
+      Slice = "hostcritical.slice";
+    };
+  };
+
+  systemd.services.systemd-journald = {
+    serviceConfig = {
+      Slice = "hostcritical.slice";
+    };
+  };
+}

common/channels.nix

@@ -0,0 +1,32 @@
+# Taken from https://github.com/NixOS/infra/blob/master/channels.nix
+{
+  # "Channel name" = {
+  #   # This should be the <value> part of
+  #   # https://hydra.forkos.org/job/<value>/latest-finished
+  #   job = "project/jobset/jobname";
+  #
+  #   # When adding a new version, determine if it needs to be tagged as a
+  #   # variant -- for example:
+  #   # nixos-xx.xx         => primary
+  #   # nixos-xx.xx-small   => small
+  #   # nixos-xx.xx-darwin  => darwin
+  #   # nixos-xx.xx-aarch64 => aarch64
+  #   variant = "primary";
+  #
+  #   # Channel Status:
+  #   # '*-unstable' channels are always "rolling"
+  #   # Otherwise a release generally progresses through the following phases:
+  #   #
+  #   #  - Directly after branch off                   => "beta"
+  #   #  - Once the channel is released                => "stable"
+  #   #  - Once the next channel is released           => "deprecated"
+  #   #  - N months after the next channel is released => "unmaintained"
+  #   #    (check the release notes for when this should happen)
+  #   status = "beta";
+  # };
+  "forkos-unstable" = {
+    job = "forkos/nixos-main/tested";
+    variant = "primary";
+    status = "rolling";
+  };
+}

common/default.nix

@@ -1,11 +1,14 @@
 {
   imports = [
     ./admins.nix
+    ./server-acl.nix
     ./base-server.nix
     ./hardening.nix
     ./nix.nix
     ./raito-proxy-aware-nginx.nix
-    ./raito-vm.nix
     ./sysadmin
+    ./hardware
+    ./zsh.nix
+    ./secrets.nix
   ];
 }

common/hardware/default.nix

@@ -0,0 +1,7 @@
+{ ... }: {
+  imports = [
+    ./raito-vm.nix
+    ./oracle-vm.nix
+    ./hetzner.nix
+  ];
+}

common/hardware/hetzner.nix

@@ -0,0 +1,76 @@
+
+{ lib, config, ... }:
+let
+  cfg = config.bagel.hardware.hetzner;
+  inherit (lib) mkEnableOption mkIf mkOption types;
+in
+{
+  options.bagel.hardware.hetzner = {
+    enable = mkEnableOption "Hetzner's hardware defaults";
+
+    platformType = mkOption {
+      # Only VMs are supported.
+      type = types.enum [ "virtual-machine" ];
+    };
+
+    system = mkOption {
+      # Only the aarch64-linux VM Hetzner is supported.
+      type = types.enum [ "aarch64-linux" ];
+    };
+
+    networking.wan = {
+      mac = mkOption {
+        type = types.str;
+        description = "MAC address of the WAN interface in the Hetzner machine";
+      };
+      address = mkOption {
+        type = types.listOf types.str;
+        description = "List of static addresses attached to the WAN interface";
+      };
+    };
+  };
+
+  config = mkIf cfg.enable {
+    # A bunch of stuff is virtio.
+    boot.initrd.availableKernelModules = [
+      "xhci_pci"
+      "usbhid"
+      "sr_mod"
+      "virtio_gpu"
+      "virtio_scsi"
+      "virtio_rng"
+      "virtio_pci"
+    ];
+
+    boot.loader.systemd-boot.enable = true;
+    boot.loader.efi.canTouchEfiVariables = true;
+
+    networking.useDHCP = lib.mkDefault false;
+
+    # Stolen from the netplan provided by aarch64 Ubuntu images.
+    systemd.network.enable = true;
+    systemd.network.links."10-wan" = {
+      linkConfig.Name = "wan";
+      matchConfig.MACAddress = cfg.networking.mac;
+    };
+    systemd.network.networks."10-wan" = {
+      matchConfig.Name = "wan";
+      networkingConfig.Address = cfg.networking.address;
+      linkConfig.RequiredForOnline = true;
+      DHCP = "ipv4";
+      routes = [
+        {
+          routeConfig = {
+            Destination = "::/0";
+            GatewayOnLink = true;
+            Gateway = "fe80::1";
+          };
+        }
+      ];
+      dhcpV4Config = {
+        RouteMetric = 100;
+        UseMTU = true;
+      };
+    };
+  };
+}

common/hardware/oracle-vm.nix

@@ -0,0 +1,52 @@
+
+{ lib, config, modulesPath, ... }:
+let
+  cfg = config.bagel.hardware.oracle-vm;
+  inherit (lib) mkEnableOption mkIf mkOption types;
+in
+{
+  options.bagel.hardware.oracle-vm = {
+    enable = mkEnableOption "Oracle's VM hardware defaults";
+
+    system = mkOption {
+      # Only the free Oracle VMs are supported.
+      type = types.enum [ "aarch64-linux" ];
+    };
+  };
+
+  # Imports a bunch of virtio modules.
+  imports = [
+    "${modulesPath}/profiles/qemu-guest.nix"
+  ];
+
+  config = mkIf cfg.enable {
+    boot.loader.systemd-boot.enable = true;
+    boot.loader.efi.canTouchEfiVariables = true;
+    boot.initrd.systemd.enable = true;
+
+    boot.initrd.availableKernelModules = [
+      "xhci_pci" "virtio_pci" "usbhid" "sr_mod"
+    ];
+    boot.initrd.kernelModules = [ ];
+    boot.kernelModules = [ ];
+    boot.extraModulePackages = [ ];
+
+    nixpkgs.hostPlatform = cfg.system;
+
+    # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+    # (the default) this is the recommended approach. When using systemd-networkd it's
+    # still possible to use this option, but it's recommended to use it in conjunction
+    # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+    networking.useDHCP = lib.mkDefault false;
+    # Examples:
+    # 2: enp0s3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
+    #  link/ether 02:00:17:00:91:6e brd ff:ff:ff:ff:ff:ff
+    #  inet 10.0.0.94/24 brd 10.0.0.255 scope global dynamic noprefixroute enp0s3
+    #     valid_lft 44162sec preferred_lft 33362sec
+    #  inet6 fe80::17ff:fe00:916e/64 scope link
+    #     valid_lft forever preferred_lft forever
+    # [root@build02-aarch64-lahfa:~]# ip r
+    # default via 10.0.0.1 dev enp0s3 proto dhcp src 10.0.0.94 metric 1002 mtu 9000
+    networking.interfaces.enp0s3.useDHCP = lib.mkDefault true;
+  };
+}

common/hardware/raito-vm.nix

@@ -30,8 +30,6 @@ in
   config = mkIf cfg.enable {
     services.qemuGuest.enable = true;
     systemd.network.enable = true;
-    security.acme.defaults.email = "bagel-acme@lahfa.xyz";
-    security.acme.acceptTerms = true;
     networking.useDHCP = lib.mkDefault false;
 
     systemd.network.networks."10-nat-lan" = {

common/known-ssh-keys.nix

@@ -0,0 +1,7 @@
+{ ... }:
+{
+  programs.ssh.knownHosts = {
+    "[cl.forkos.org]:29418".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM82mJ259C8Nc+BHHNBeRWXWhL3dfirQhmFbDAwHMle3";
+    "[gerrit.lix.systems]:2022".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICC/S6Z56uhv7zBMutkV0nU8eDuRcl3trykGWBch4L/l";
+  };
+}

common/nix.nix

@@ -14,7 +14,7 @@
 
   # Use our cache and trust its signing key. Still use cache.nixos.org as
   # fallback.
-  nix.settings.substituters = [ "https://bagel-cache.s3-web.delroth.net/" ];
+  nix.settings.substituters = [ "https://cache.forkos.org/" ];
   nix.settings.trusted-public-keys = [
     "cache.forkos.org:xfXIUJO1yiEITJmYsVmNDa9BFSlgTh/YqZ+4ei1EhQg="
   ];

common/raito-proxy-aware-nginx.nix

@@ -1,9 +1,10 @@
 # This enables an IPv6-only server which is proxied by kurisu.lahfa.xyz to have proper IPv4 logs via PROXY protocol.
 { config, lib, ... }:
 let
-  inherit (lib) mkEnableOption mkIf;
+  inherit (lib) mkEnableOption mkIf concatStringsSep;
   cfg = config.bagel.raito.v6-proxy-awareness;
-  allowedUpstream = "2001:bc8:38ee:99::1/128";
+  #                      outside of raito infra    inside of raito infra
+  allowedUpstreams = [ "2001:bc8:38ee::1/128"     "2001:bc8:38ee:99::1/128" ];
 in
   {
     options.bagel.raito.v6-proxy-awareness.enable = mkEnableOption "the kurisu.lahfa.xyz's sniproxy awareness for NGINX";
@@ -20,8 +21,8 @@ in
         ];
 
         appendHttpConfig = ''
-          # Kurisu node
-          set_real_ip_from ${allowedUpstream};
+          # Kurisu nodes
+          ${concatStringsSep "\n" (map (up: "set_real_ip_from ${up};") allowedUpstreams)}
           real_ip_header proxy_protocol;
         '';
       };
@@ -29,7 +30,7 @@ in
       # Move to nftables if firewall is enabled.
       networking.nftables.enable = true;
       networking.firewall.extraInputRules = ''
-        ip6 saddr ${allowedUpstream} tcp dport 444 accept
+        ${concatStringsSep "\n" (map (up: "ip6 saddr ${up} tcp dport 444 accept") allowedUpstreams)}
       '';
     };
   }

common/secrets.nix

@@ -0,0 +1,22 @@
+## This is a simple secret abstraction with multi-tenancy awareness.
+{ config, lib, ... }:
+let
+  cfg = config.bagel.secrets;
+  inherit (lib) mkOption types genAttrs;
+in
+{
+  options.bagel.secrets = {
+    tenant = mkOption {
+      type = types.enum [ "lix" "floral" ];
+    };
+
+    files = mkOption {
+      type = types.listOf types.str;
+      default = [ ];
+    };
+  };
+
+  config.age.secrets = genAttrs cfg.files (secretFile: {
+      file = ../secrets/${cfg.tenant}/${secretFile}.age;
+    });
+}

common/server-acl.nix

@@ -0,0 +1,69 @@
+{ lib, config, ... }:
+let
+  keys = import ./ssh-keys.nix;
+  inherit (lib) mkOption types length concatMap listToAttrs catAttrs attrValues;
+  cfgAdmins = config.bagel.admins;
+  cfgGroups = config.bagel.groups;
+  cfgUsers = config.bagel.users;
+
+  userOpts = { name, ... }: {
+    options = {
+      sshKeys = mkOption {
+        type = types.listOf types.str;
+        description = "List of SSH keys associated to this user, defaults to `ssh-keys.nix` entries.";
+        default = keys.users.${name} or [ ];
+      };
+    };
+  };
+  groupOpts = { name, ... }: {
+    options = {
+      members = mkOption {
+        type = types.listOf types.str;
+        description = "List of users member of this group";
+        example = [ "raito" ];
+        default = [ ];
+      };
+    };
+  };
+
+  # There might be duplicate in that list. We will turn it into an attribute set.
+  allowedMembers = listToAttrs (
+    map (member: {
+      name = member;
+      value = cfgUsers.${member};
+    }) (concatMap (allowedGroup: cfgGroups.${allowedGroup}.members) cfgAdmins.allowedGroups));
+
+  rootKeys = concatMap ({ sshKeys, ... }: sshKeys) (attrValues allowedMembers);
+in
+{
+  options.bagel.users = mkOption {
+    type = types.attrsOf (types.submodule userOpts);
+    description = "User configuration for server ACLs";
+  };
+
+  options.bagel.groups = mkOption {
+    type = types.attrsOf (types.submodule groupOpts);
+    description = "Group configuration for server ACLs";
+  };
+
+  options.bagel.admins = {
+    allowedGroups = mkOption {
+      type = types.listOf types.str;
+      default = [ "catch-all" ];
+      description = "List of groups which are allowed to admin this machine.";
+      example = [ "lix" "build-infra" ];
+    };
+  };
+
+  config = {
+    assertions = [
+      { assertion = length config.users.users.root.openssh.authorizedKeys.keys > 0;
+        # TODO: you can add printing of `concatStringsSep ", " cfg.allowedGroups` to diagnose
+        # which are the allowed groups and existing admins.
+        message = "root@${config.networking.fqdnOrHostName} has no SSH key attached, this machine will lose its access if you deploy it successfully! Set a valid `bagel.admins.allowedGroups` or ensure you have at least one administrator of the relevant group registered";
+      }
+    ];
+
+    users.users.root.openssh.authorizedKeys.keys = rootKeys;
+  };
+}

common/ssh-keys.nix

@@ -1,9 +1,14 @@
 {
   machines = {
+    # Floral
     bagel-box = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAsO4bNqY04uG13Pg3ubHfRDssTphDLzZ4YUniE5/p+M";
     meta01 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM5t9gYorOWgpCFDJgb24pyCKIabGpeI2H/UfdvXODcT";
+    public01 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPBy8G8rfLA6E9i+t5kjVafxU1c2NXATXKxoXTH4Kgtm";
     gerrit01 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIA+eSZu+u9sCynrMlsmFzQHLIELQAuVg0Cs1pBvwb4+A";
     fodwatch = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFRyTNfvKl5FcSyzGzw+h+bNFNOxdhvI67WdUZ2iIJ1L";
+    buildbot = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJgIu6ouagYqBeMLfmn1CbaDJMuZcPH9bnUhkht8GfuB";
+    git = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEQJcpkCUOx8+5oukMX6lxrYcIX8FyHu8Mc/3+ieKMUn";
+    build-coord = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINpAEJP7F+XtJBpQP1jTzwXwQgJrFxwEJjPf/rnCXkJA";
     builder-0 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBHSNcDGctvlG6BHcJuYIzW9WsBJsts2vpwSketsbXoL";
     builder-1 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIQOGUjERK7Mx8UPM/rbOdMqVyn1sbWqYOG6CbOzH2wm";
     builder-2 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMKzXIqCoYElEKIYgjbSpqEcDeOvV+Wo3Agq3jba83cB";
@@ -15,20 +20,27 @@
     builder-8 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKGSWHNeqT0kF/e4yVy2ieW98X5QMyCYIYZh9WTmQDs1";
     builder-9 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOhws9zGgocVY36dMtOL+CXadpvRMffxoWMkfEcTBJm7";
     builder-10 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE7sgIuTSqZiZhp8TvObSbIEhcHHsL5hcmYA22uzwxth";
-    builder-11 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEAqFo1qJY7MSUkfB+zxXB8Lpt/Iqz/RR5A+zwhpRWhr";
     wob-vpn-gw = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINVytPPW8XnXf/rD5TFzsw//CZc2lBjQLmDzlVGPZsjh";
+
+    # Lix
+    build01-aarch64-lix = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICC69NZD/zhIB/wUb5odg46bss5g8hH2fDl22bk4qeSW";
+    build02-aarch64-lix = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGdJE375pe58RJbhKwXRp3D//+SJ3ssiVZrLsM9CLHn0";
+    build01-aarch64-darwin-lix = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMVf1uO0lv5UBti/naW/+amqLxvWZg+StXk9aM+lJ7e4";
+
+    buildbot-lix = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFoVSh35UqNQZ6ZZ1c6CzqERC40ovQ/KDXz8pC7nNlkR";
+
+    # Raito infrastructure
+    epyc-newtype-fr = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOXT9Init1MhKt4rjBANLq0t0bPww/WQZ96uB4AEDrml";
   };
 
   users = {
     delroth = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII3tjB4KYDok3KlWxdBp/yEmqhhmybd+w0VO4xUwLKKV" ];
-    raito = [
-      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICaw9ihTG7ucB8P38XdalEWev8+q96e2yNm4B+/I9IJp"
-      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDcEkYM1r8QVNM/G5CxJInEdoBCWjEHHDdHlzDYNSUIdHHsn04QY+XI67AdMCm8w30GZnLUIj5RiJEWXREUApby0GrfxGGcy8otforygfgtmuUKAUEHdU2MMwrQI7RtTZ8oQ0USRGuqvmegxz3l5caVU7qGvBllJ4NUHXrkZSja2/51vq80RF4MKkDGiz7xUTixI2UcBwQBCA/kQedKV9G28EH+1XfvePqmMivZjl+7VyHsgUVj9eRGA1XWFw59UPZG8a7VkxO/Eb3K9NF297HUAcFMcbY6cPFi9AaBgu3VC4eetDnoN/+xT1owiHi7BReQhGAy/6cdf7C/my5ehZwD"
-      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE0xMwWedkKosax9+7D2OlnMxFL/eV4CvFZLsbLptpXr"
-      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiXXYkhRh+s7ixZ8rvG8ntIqd6FELQ9hh7HoaHQJRPU"
+    emilylange = [ "no-touch-required sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIL7jgq3i+N3gVJhs4shm7Kmw6dIocs2OuR0GBMG1RxfKAAAABHNzaDo=" ];
+    hexchen = [
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINJ0tCxsEilAzV6LaNpUpcjzyEn4ptw8kFz3R+Z3YjEF hexchen@backup"
+      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDI3T1eFS77URHZ/HVWkMOqx7W1U54zJtn9C7QWsHOtyH72i/4EVj8SxYqLllElh1kuKUXSUipPeEzVsipFVvfH0wEuTDgFffiSQ3a8lfUgdEBuoySwceEoPgc5deapkOmiDIDeeWlrRe3nqspLRrSWU1DirMxoFPbwqJXRvpl6qJPxRg+2IolDcXlZ6yxB4Vv48vzRfVzZNUz7Pjmy2ebU8PbDoFWL/S3m7yOzQpv3L7KYBz7+rkjuF3AU2vy6CAfIySkVpspZZLtkTGCIJF228ev0e8NvhuN6ZnjzXxVTQOy32HCdPdbBbicu0uHfZ5O7JX9DjGd8kk1r2dnZwwy/ hexchen@yubi5"
+      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC4CLJ+mFfq5XiBXROKewmN9WYmj+79bj/AoaR6Iud2pirulot3tkrrLe2cMjiNWFX8CGVqrsAELKUA8EyUTJfStlcTE0/QNESTRmdDaC+lZL41pWUO9KOiD6/0axAhHXrSJ0ScvbqtD0CtpnCKKxtuOflVPoUGZsH9cLKJNRKfEka0H0GgeKb5Tp618R/WNAQOwaCcXzg/nG4Bgv3gJW4Nm9IKy/MwRZqtILi8Mtd+2diTqpMwyNRmbenmRHCQ1vRw46joYkledVqrmSlfSMFgIHI1zRSBXb/JkG2IvIyB5TGbTkC4N2fqJNpH8wnCKuOvs46xmgdiRA26P48C2em3 hexchen@yubi5c"
     ];
-    k900 = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOi9vgVGs+S5kEsUqHPvyMMh1Q9gqL4TcbHoe5d73tun" ];
-    maxine = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILpWQfhNFdrxMTP/1DwBVuk49f3df9iH7Tbdu8ltIKjr" ];
     jade = [
       "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDNldAg4t13/i69TD786The+U3wbiNUdW2Kc9KNWvEhgpf4y4x4Sft0oYfkPw5cjX4H3APqfD+b7ItAG0GCbwHw6KMYPoVMNK08zBMJUqt1XExbqGeFLqBaeqDsmEAYXJRbjMTAorpOCtgQdoCKK/DvZ51zUWXxT8UBNHSl19Ryv5Ry5VVdbAE35rqs57DQ9+ma6htXnsBEmmnC+1Zv1FE956m/OpBTId50mor7nS2FguAtPZnDPpTd5zl9kZmJEuWCrmy6iinw5V4Uy1mLeZkQv+/FtozbyifCRCvps9nHpv4mBSU5ABLgnRRvXs+D41Jx7xloNADr1nNgpsNrYaTh hed-bot-ssh-tpm-rsa"
       "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIKYljH8iPMrH00lOb3ETxRrZimdKzPPEdsJQ5D5ovtOwAAAACnNzaDpzc2hrZXk= ssh:sshkey"
@@ -39,7 +51,18 @@
       "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIOYg513QZsVzoyVycXZjg4F3T3+OwtcY3WAhrlfyLgLTAAAABHNzaDo="
       "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBLZxVITpJ8xbiCa/u2gjSSIupeiqOnRh+8tFIoVhCON"
     ];
+    k900 = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOi9vgVGs+S5kEsUqHPvyMMh1Q9gqL4TcbHoe5d73tun" ];
     lukegb = [ ''cert-authority,principals="lukegb" ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEqNOwlR7Qa8cbGpDfSCOweDPbAGQOZIcoRgh6s/J8DR'' ];
-    yuka = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKath4/fDnlv/4fzxkPrQN1ttmoPRNu/m9bEtdPJBDfY cardno:16_933_242" ];
+    maxine = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILpWQfhNFdrxMTP/1DwBVuk49f3df9iH7Tbdu8ltIKjr" ];
+    raito = [
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICaw9ihTG7ucB8P38XdalEWev8+q96e2yNm4B+/I9IJp"
+      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDcEkYM1r8QVNM/G5CxJInEdoBCWjEHHDdHlzDYNSUIdHHsn04QY+XI67AdMCm8w30GZnLUIj5RiJEWXREUApby0GrfxGGcy8otforygfgtmuUKAUEHdU2MMwrQI7RtTZ8oQ0USRGuqvmegxz3l5caVU7qGvBllJ4NUHXrkZSja2/51vq80RF4MKkDGiz7xUTixI2UcBwQBCA/kQedKV9G28EH+1XfvePqmMivZjl+7VyHsgUVj9eRGA1XWFw59UPZG8a7VkxO/Eb3K9NF297HUAcFMcbY6cPFi9AaBgu3VC4eetDnoN/+xT1owiHi7BReQhGAy/6cdf7C/my5ehZwD"
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE0xMwWedkKosax9+7D2OlnMxFL/eV4CvFZLsbLptpXr"
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiXXYkhRh+s7ixZ8rvG8ntIqd6FELQ9hh7HoaHQJRPU"
+    ];
+    thubrecht = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPM1jpXR7BWQa7Sed7ii3SbvIPRRlKb3G91qC0vOwfJn" ];
+    yuka = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIxQ3NYBi8v1f/vhxLKDcA6upmX0pctRDbnK6SER5OUR yureka" ];
+    winter = [ "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIH/LDRUG+U+++UmlxvA2kspioTjktQZ8taDcHq8gVlkfAAAABHNzaDo=" ];
+    ckie = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIH3uTwzSSMAPg84fwbNp2cq9+BdLFeA1VzDGth4zCAbz https://mei.puppycat.house" ];
   };
 }

common/sysadmin/default.nix

@@ -13,7 +13,11 @@ in
         tmux
         rsync
         fd
+        eza
+        grc
         ripgrep
+        delta
+        tshark
         pv
         kitty.terminfo
         config.boot.kernelPackages.perf

common/zsh.nix

@@ -0,0 +1,16 @@
+{ lib, pkgs, config, ... }: {
+  users.defaultUserShell = pkgs.zsh;
+  programs.zsh = {
+    enable = true;
+    enableCompletion = true;
+    autosuggestions.enable = true;
+    interactiveShellInit = ''
+      ${lib.getExe pkgs.nix-your-shell} zsh | source /dev/stdin
+    '';
+    promptInit = ''
+      # https://grml.org/zsh/grml-zsh-refcard.pdf
+      source ${pkgs.grml-zsh-config}/etc/zsh/zshrc
+      PS1='%n@${config.networking.fqdn} %/ \$ '
+    '';
+  };
+}

dashboards/default.nix

@@ -0,0 +1,43 @@
+{ gerrit-dashboard, stdenv, symlinkJoin, jsonnet, fetchFromGitHub, lib, ... }:
+let
+  inherit (lib) concatMapStringsSep;
+  datasource-id = "mimir";
+in
+rec {
+  grafonnet = fetchFromGitHub {
+    owner = "grafana";
+    repo = "grafonnet-lib";
+    # TODO: figure out how to read the jsonnet lockfile
+    # and propagate this a bit cleverly.
+    rev = "a1d61cce1da59c71409b99b5c7568511fec661ea";
+    hash = "sha256-fs5JZJbcL6sQXBjYhp5eeRtjTFw0J1O/BcwBC8Vm9EM=";
+  };
+  buildJsonnetDashboards = dashboardSrc: targets: stdenv.mkDerivation {
+    name = "jsonnet-grafana-dashboards";
+    src = dashboardSrc;
+    buildInputs = [ jsonnet ];
+    buildPhase = ''
+      runHook preBuild
+      mkdir -p $out
+      ${concatMapStringsSep "\n" (target: "jsonnet -J ${grafonnet} --ext-str datasource=${datasource-id} --ext-code publish=true $src/${target} > $out/${baseNameOf target}.json") targets}
+      runHook postBuild
+    '';
+  };
+
+  allDashboards = symlinkJoin {
+    name = "all-jsonnet-dashboards";
+    paths = [
+      (buildJsonnetDashboards gerrit-dashboard [
+        "dashboards/gerrit/caches/gerrit-caches.jsonnet"
+        "dashboards/gerrit/fetch-clone/gerrit-fetch-clone.jsonnet"
+        "dashboards/gerrit/fetch-clone/gerrit-phases.jsonnet"
+        "dashboards/gerrit/healthcheck/gerrit-healthcheck.jsonnet"
+        "dashboards/gerrit/latency/gerrit-push-latency.jsonnet"
+        "dashboards/gerrit/latency/gerrit-ui-actions-latency.jsonnet"
+        "dashboards/gerrit/overview/gerrit-overview.jsonnet"
+        "dashboards/gerrit/process/gerrit-process.jsonnet"
+        "dashboards/gerrit/queues/gerrit-queues.jsonnet"
+      ])
+    ];
+  };
+}

flake.lock

@@ -10,11 +10,11 @@
         "systems": "systems"
       },
       "locked": {
-        "lastModified": 1720546205,
-        "narHash": "sha256-boCXsjYVxDviyzoEyAk624600f3ZBo/DKtUdvMTpbGY=",
+        "lastModified": 1723293904,
+        "narHash": "sha256-b+uqzj+Wa6xgMS9aNbX4I+sXeb5biPDi39VgvSFqFvU=",
         "owner": "ryantm",
         "repo": "agenix",
-        "rev": "de96bd907d5fbc3b14fc33ad37d1b9a3cb15edc6",
+        "rev": "f6291c5935fdc4e0bef208cfc0dcab7e3f7a1c41",
         "type": "github"
       },
       "original": {
@@ -23,6 +23,29 @@
         "type": "github"
       }
     },
+    "attic": {
+      "inputs": {
+        "crane": "crane",
+        "flake-compat": "flake-compat_2",
+        "flake-utils": "flake-utils_2",
+        "nixpkgs": "nixpkgs",
+        "nixpkgs-stable": "nixpkgs-stable"
+      },
+      "locked": {
+        "lastModified": 1711742460,
+        "narHash": "sha256-0O4v6e4a1toxXZ2gf5INhg4WPE5C5T+SVvsBt+45Mcc=",
+        "owner": "zhaofengli",
+        "repo": "attic",
+        "rev": "4dbdbee45728d8ce5788db6461aaaa89d98081f0",
+        "type": "github"
+      },
+      "original": {
+        "owner": "zhaofengli",
+        "ref": "main",
+        "repo": "attic",
+        "type": "github"
+      }
+    },
     "bats-assert": {
       "flake": false,
       "locked": {
@@ -55,6 +78,49 @@
         "type": "github"
       }
     },
+    "buildbot-nix": {
+      "inputs": {
+        "flake-parts": "flake-parts",
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "treefmt-nix": "treefmt-nix"
+      },
+      "locked": {
+        "lastModified": 1728837991,
+        "narHash": "sha256-+jXVHPmX9eUtH2JhMKye0Tm2KMQTmD8FlHHfbcaXMOI=",
+        "ref": "refs/heads/bring-back-old-gerrit-reporting",
+        "rev": "879e9cdcdf2d7e6566ee512d015acc4d23f35517",
+        "revCount": 302,
+        "type": "git",
+        "url": "https://git.lix.systems/lix-project/buildbot-nix.git"
+      },
+      "original": {
+        "ref": "refs/heads/bring-back-old-gerrit-reporting",
+        "type": "git",
+        "url": "https://git.lix.systems/lix-project/buildbot-nix.git"
+      }
+    },
+    "channel-scripts": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1725128016,
+        "narHash": "sha256-4TvaXELsl+1OcGNgqB/5HVXVxBvdIQkhJsY4FyiDcNU=",
+        "ref": "refs/heads/main",
+        "rev": "23b6c38ed7e11417bf624f6e4fb6cde0d2be6400",
+        "revCount": 261,
+        "type": "git",
+        "url": "https://git.lix.systems/the-distro/channel-scripts.git"
+      },
+      "original": {
+        "type": "git",
+        "url": "https://git.lix.systems/the-distro/channel-scripts.git"
+      }
+    },
     "colmena": {
       "inputs": {
         "flake-compat": "flake-compat",
@@ -78,6 +144,50 @@
         "type": "github"
       }
     },
+    "crane": {
+      "inputs": {
+        "nixpkgs": [
+          "grapevine",
+          "attic",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1702918879,
+        "narHash": "sha256-tWJqzajIvYcaRWxn+cLUB9L9Pv4dQ3Bfit/YjU5ze3g=",
+        "owner": "ipetkov",
+        "repo": "crane",
+        "rev": "7195c00c272fdd92fc74e7d5a0a2844b9fadb2fb",
+        "type": "github"
+      },
+      "original": {
+        "owner": "ipetkov",
+        "repo": "crane",
+        "type": "github"
+      }
+    },
+    "crane_2": {
+      "inputs": {
+        "nixpkgs": [
+          "grapevine",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1716569590,
+        "narHash": "sha256-5eDbq8TuXFGGO3mqJFzhUbt5zHVTf5zilQoyW5jnJwo=",
+        "owner": "ipetkov",
+        "repo": "crane",
+        "rev": "109987da061a1bf452f435f1653c47511587d919",
+        "type": "github"
+      },
+      "original": {
+        "owner": "ipetkov",
+        "ref": "master",
+        "repo": "crane",
+        "type": "github"
+      }
+    },
     "darwin": {
       "inputs": {
         "nixpkgs": [
@@ -100,6 +210,29 @@
         "type": "github"
       }
     },
+    "fenix": {
+      "inputs": {
+        "nixpkgs": [
+          "grapevine",
+          "nixpkgs"
+        ],
+        "rust-analyzer-src": "rust-analyzer-src"
+      },
+      "locked": {
+        "lastModified": 1716359173,
+        "narHash": "sha256-pYcjP6Gy7i6jPWrjiWAVV0BCQp+DdmGaI/k65lBb/kM=",
+        "owner": "nix-community",
+        "repo": "fenix",
+        "rev": "b6fc5035b28e36a98370d0eac44f4ef3fd323df6",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "ref": "main",
+        "repo": "fenix",
+        "type": "github"
+      }
+    },
     "flake-compat": {
       "flake": false,
       "locked": {
@@ -117,6 +250,39 @@
       }
     },
     "flake-compat_2": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1673956053,
+        "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "flake-compat_3": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1696426674,
+        "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "ref": "master",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "flake-compat_4": {
       "flake": false,
       "locked": {
         "lastModified": 1696426674,
@@ -132,6 +298,49 @@
         "type": "github"
       }
     },
+    "flake-parts": {
+      "inputs": {
+        "nixpkgs-lib": [
+          "buildbot-nix",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1706830856,
+        "narHash": "sha256-a0NYyp+h9hlb7ddVz4LUn1vT/PLwqfrWYcHMvFB1xYg=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "b253292d9c0a5ead9bc98c4e9a26c6312e27d69f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
+    "flake-parts_2": {
+      "inputs": {
+        "nixpkgs-lib": [
+          "hydra",
+          "nix-eval-jobs",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1722555600,
+        "narHash": "sha256-XOQkdLafnb/p9ij77byFQjDf5m5QYl9b2REiVClC+x4=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "8471fe90ad337a8074e957b69ca4d0089218391d",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
     "flake-utils": {
       "locked": {
         "lastModified": 1659877975,
@@ -148,6 +357,40 @@
       }
     },
     "flake-utils_2": {
+      "locked": {
+        "lastModified": 1667395993,
+        "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "flake-utils_3": {
+      "inputs": {
+        "systems": "systems_2"
+      },
+      "locked": {
+        "lastModified": 1710146030,
+        "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "ref": "main",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "flake-utils_4": {
       "locked": {
         "lastModified": 1634851050,
         "narHash": "sha256-N83GlSGPJJdcqhUxSCS/WwW5pksYf3VP1M13cDRTSVA=",
@@ -162,6 +405,51 @@
         "type": "github"
       }
     },
+    "gerrit-dashboard": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1724509518,
+        "narHash": "sha256-fwYXZVddxfzrlDa3QnFCwHqrbEX+3PrWy0QOlbO+8jk=",
+        "ref": "refs/heads/master",
+        "rev": "e544abac81c581558d68abb2a8dd583049073939",
+        "revCount": 75,
+        "type": "git",
+        "url": "https://git.lix.systems/the-distro/gerrit-monitoring.git"
+      },
+      "original": {
+        "type": "git",
+        "url": "https://git.lix.systems/the-distro/gerrit-monitoring.git"
+      }
+    },
+    "grapevine": {
+      "inputs": {
+        "attic": "attic",
+        "crane": "crane_2",
+        "fenix": "fenix",
+        "flake-compat": "flake-compat_3",
+        "flake-utils": "flake-utils_3",
+        "nix-filter": "nix-filter",
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "rust-manifest": "rust-manifest"
+      },
+      "locked": {
+        "host": "gitlab.computer.surgery",
+        "lastModified": 1727994504,
+        "narHash": "sha256-FC6M1KKX58HbU9LG+cG6EJRr02J9lE/o0iiDi6m1gv8=",
+        "owner": "matrix",
+        "repo": "grapevine-fork",
+        "rev": "5a490a4397f0c6a36dab1cb631dadc67a849deab",
+        "type": "gitlab"
+      },
+      "original": {
+        "host": "gitlab.computer.surgery",
+        "owner": "matrix",
+        "repo": "grapevine-fork",
+        "type": "gitlab"
+      }
+    },
     "home-manager": {
       "inputs": {
         "nixpkgs": [
@@ -185,17 +473,18 @@
     },
     "hydra": {
       "inputs": {
-        "nix": "nix",
+        "lix": "lix",
+        "nix-eval-jobs": "nix-eval-jobs",
         "nixpkgs": [
           "nixpkgs"
         ]
       },
       "locked": {
-        "lastModified": 1720843955,
-        "narHash": "sha256-GpkZ7OorcArMaFVZMPHkXHKQVJAWjMSCaPmS8hw0PB0=",
+        "lastModified": 1728321752,
+        "narHash": "sha256-GbBAoBF7ZObz0IP+g0LZKxMafpMvNKjTEu9haiZbV54=",
         "ref": "refs/heads/main",
-        "rev": "fb9e29d4d0f2f591cd1d706fd3b7334af7d34b84",
-        "revCount": 4174,
+        "rev": "ee1234c15cdcb427dbd4828e0add09d02cd606c9",
+        "revCount": 4220,
         "type": "git",
         "url": "https://git.lix.systems/lix-project/hydra.git"
       },
@@ -204,9 +493,9 @@
         "url": "https://git.lix.systems/lix-project/hydra.git"
       }
     },
-    "nix": {
+    "lix": {
       "inputs": {
-        "flake-compat": "flake-compat_2",
+        "flake-compat": "flake-compat_4",
         "nix2container": "nix2container",
         "nixpkgs": [
           "hydra",
@@ -216,17 +505,61 @@
         "pre-commit-hooks": "pre-commit-hooks"
       },
       "locked": {
-        "lastModified": 1720733512,
-        "narHash": "sha256-vq9CLDvqSSvH4L7YhDa0ihTOrAry4jntKiuoNb5n98M=",
+        "lastModified": 1728163191,
+        "narHash": "sha256-SW0IEBsPN1EysqzvfDT+8Kimtzy03O1BxQQm7ZB6fRY=",
         "ref": "refs/heads/main",
-        "rev": "4b109ec1a8fc4550150f56f0f46f2f41d844bda8",
-        "revCount": 15950,
+        "rev": "ed9b7f4f84fd60ad8618645cc1bae2d686ff0db6",
+        "revCount": 16323,
         "type": "git",
-        "url": "https://git@git.lix.systems/lix-project/lix"
+        "url": "https://git.lix.systems/lix-project/lix"
       },
       "original": {
         "type": "git",
-        "url": "https://git@git.lix.systems/lix-project/lix"
+        "url": "https://git.lix.systems/lix-project/lix"
+      }
+    },
+    "nix-eval-jobs": {
+      "inputs": {
+        "flake-parts": "flake-parts_2",
+        "lix": [
+          "hydra",
+          "lix"
+        ],
+        "nix-github-actions": "nix-github-actions",
+        "nixpkgs": [
+          "hydra",
+          "nixpkgs"
+        ],
+        "treefmt-nix": "treefmt-nix_2"
+      },
+      "locked": {
+        "lastModified": 1723579251,
+        "narHash": "sha256-xnHtfw0gRhV+2S9U7hQwvp2klTy1Iv7FlMMO0/WiMVc=",
+        "ref": "refs/heads/main",
+        "rev": "42a160bce2fd9ffebc3809746bc80cc7208f9b08",
+        "revCount": 609,
+        "type": "git",
+        "url": "https://git.lix.systems/lix-project/nix-eval-jobs"
+      },
+      "original": {
+        "type": "git",
+        "url": "https://git.lix.systems/lix-project/nix-eval-jobs"
+      }
+    },
+    "nix-filter": {
+      "locked": {
+        "lastModified": 1710156097,
+        "narHash": "sha256-1Wvk8UP7PXdf8bCCaEoMnOT1qe5/Duqgj+rL8sRQsSM=",
+        "owner": "numtide",
+        "repo": "nix-filter",
+        "rev": "3342559a24e85fc164b295c3444e8a139924675b",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "ref": "main",
+        "repo": "nix-filter",
+        "type": "github"
       }
     },
     "nix-gerrit": {
@@ -249,14 +582,36 @@
         "url": "https://git.lix.systems/the-distro/nix-gerrit.git"
       }
     },
+    "nix-github-actions": {
+      "inputs": {
+        "nixpkgs": [
+          "hydra",
+          "nix-eval-jobs",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1720066371,
+        "narHash": "sha256-uPlLYH2S0ACj0IcgaK9Lsf4spmJoGejR9DotXiXSBZQ=",
+        "owner": "nix-community",
+        "repo": "nix-github-actions",
+        "rev": "622f829f5fe69310a866c8a6cd07e747c44ef820",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "nix-github-actions",
+        "type": "github"
+      }
+    },
     "nix2container": {
       "flake": false,
       "locked": {
-        "lastModified": 1712990762,
-        "narHash": "sha256-hO9W3w7NcnYeX8u8cleHiSpK2YJo7ecarFTUlbybl7k=",
+        "lastModified": 1720642556,
+        "narHash": "sha256-qsnqk13UmREKmRT7c8hEnz26X3GFFyIQrqx4EaRc1Is=",
         "owner": "nlewo",
         "repo": "nix2container",
-        "rev": "20aad300c925639d5d6cbe30013c8357ce9f2a2e",
+        "rev": "3853e5caf9ad24103b13aa6e0e8bcebb47649fe4",
         "type": "github"
       },
       "original": {
@@ -267,11 +622,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1720750130,
-        "narHash": "sha256-y2wc7CdK0vVSIbx7MdVoZzuMcUoLvZXm+pQf2RIr1OU=",
+        "lastModified": 1711401922,
+        "narHash": "sha256-QoQqXoj8ClGo0sqD/qWKFWezgEwUL0SUh37/vY2jNhc=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "6794d064edc69918bb0fc0e0eda33ece324be17a",
+        "rev": "07262b18b97000d16a4bdb003418bd2fb067a932",
         "type": "github"
       },
       "original": {
@@ -297,17 +652,34 @@
         "type": "github"
       }
     },
-    "nixpkgs_2": {
+    "nixpkgs-stable": {
       "locked": {
-        "lastModified": 1636823747,
-        "narHash": "sha256-oWo1nElRAOZqEf90Yek2ixdHyjD+gqtS/pAgwaQ9UhQ=",
-        "owner": "nixos",
+        "lastModified": 1711460390,
+        "narHash": "sha256-akSgjDZL6pVHEfSE6sz1DNSXuYX6hq+P/1Z5IoYWs7E=",
+        "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "f6a2ed2082d9a51668c86ba27d0b5496f7a2ea93",
+        "rev": "44733514b72e732bd49f5511bd0203dea9b9a434",
         "type": "github"
       },
       "original": {
-        "owner": "nixos",
+        "owner": "NixOS",
+        "ref": "nixos-23.11",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs_2": {
+      "locked": {
+        "lastModified": 1728093190,
+        "narHash": "sha256-CAZF2NRuHmqTtRTNAruWpHA43Gg2UvuCNEIzabP0l6M=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "e2f08f4d8b3ecb5cf5c9fd9cb2d53bb3c71807da",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixpkgs-unstable",
         "repo": "nixpkgs",
         "type": "github"
       }
@@ -315,11 +687,11 @@
     "pre-commit-hooks": {
       "flake": false,
       "locked": {
-        "lastModified": 1712055707,
-        "narHash": "sha256-4XLvuSIDZJGS17xEwSrNuJLL7UjDYKGJSbK1WWX2AK8=",
+        "lastModified": 1721042469,
+        "narHash": "sha256-6FPUl7HVtvRHCCBQne7Ylp4p+dpP3P/OYuzjztZ4s70=",
         "owner": "cachix",
         "repo": "git-hooks.nix",
-        "rev": "e35aed5fda3cc79f88ed7f1795021e559582093a",
+        "rev": "f451c19376071a90d8c58ab1a953c6e9840527fd",
         "type": "github"
       },
       "original": {
@@ -331,17 +703,51 @@
     "root": {
       "inputs": {
         "agenix": "agenix",
+        "buildbot-nix": "buildbot-nix",
+        "channel-scripts": "channel-scripts",
         "colmena": "colmena",
+        "gerrit-dashboard": "gerrit-dashboard",
+        "grapevine": "grapevine",
         "hydra": "hydra",
         "lix": [
           "hydra",
-          "nix"
+          "lix"
         ],
         "nix-gerrit": "nix-gerrit",
-        "nixpkgs": "nixpkgs",
+        "nixpkgs": "nixpkgs_2",
+        "stateless-uptime-kuma": "stateless-uptime-kuma",
         "terranix": "terranix"
       }
     },
+    "rust-analyzer-src": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1716107283,
+        "narHash": "sha256-NJgrwLiLGHDrCia5AeIvZUHUY7xYGVryee0/9D3Ir1I=",
+        "owner": "rust-lang",
+        "repo": "rust-analyzer",
+        "rev": "21ec8f523812b88418b2bfc64240c62b3dd967bd",
+        "type": "github"
+      },
+      "original": {
+        "owner": "rust-lang",
+        "ref": "nightly",
+        "repo": "rust-analyzer",
+        "type": "github"
+      }
+    },
+    "rust-manifest": {
+      "flake": false,
+      "locked": {
+        "narHash": "sha256-tB9BZB6nRHDk5ELIVlGYlIjViLKBjQl52nC1avhcCwA=",
+        "type": "file",
+        "url": "https://static.rust-lang.org/dist/channel-rust-1.81.0.toml"
+      },
+      "original": {
+        "type": "file",
+        "url": "https://static.rust-lang.org/dist/channel-rust-1.81.0.toml"
+      }
+    },
     "stable": {
       "locked": {
         "lastModified": 1696039360,
@@ -358,6 +764,22 @@
         "type": "github"
       }
     },
+    "stateless-uptime-kuma": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1728243069,
+        "narHash": "sha256-l9fgwesnmFxasCaYUCD7L9bGGJXytLuwtx3CZMgpwJg=",
+        "ref": "refs/heads/master",
+        "rev": "880f444ff7862d6127b051cf1a993ad1585b1652",
+        "revCount": 25,
+        "type": "git",
+        "url": "https://git.dgnum.eu/DGNum/stateless-uptime-kuma.git"
+      },
+      "original": {
+        "type": "git",
+        "url": "https://git.dgnum.eu/DGNum/stateless-uptime-kuma.git"
+      }
+    },
     "systems": {
       "locked": {
         "lastModified": 1681028828,
@@ -373,12 +795,29 @@
         "type": "github"
       }
     },
+    "systems_2": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
     "terranix": {
       "inputs": {
         "bats-assert": "bats-assert",
         "bats-support": "bats-support",
-        "flake-utils": "flake-utils_2",
-        "nixpkgs": "nixpkgs_2",
+        "flake-utils": "flake-utils_4",
+        "nixpkgs": [
+          "nixpkgs"
+        ],
         "terranix-examples": "terranix-examples"
       },
       "locked": {
@@ -409,6 +848,49 @@
         "repo": "terranix-examples",
         "type": "github"
       }
+    },
+    "treefmt-nix": {
+      "inputs": {
+        "nixpkgs": [
+          "buildbot-nix",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1708897213,
+        "narHash": "sha256-QECZB+Hgz/2F/8lWvHNk05N6NU/rD9bWzuNn6Cv8oUk=",
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "rev": "e497a9ddecff769c2a7cbab51e1ed7a8501e7a3a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "type": "github"
+      }
+    },
+    "treefmt-nix_2": {
+      "inputs": {
+        "nixpkgs": [
+          "hydra",
+          "nix-eval-jobs",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1723454642,
+        "narHash": "sha256-S0Gvsenh0II7EAaoc9158ZB4vYyuycvMGKGxIbERNAM=",
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "rev": "349de7bc435bdff37785c2466f054ed1766173be",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "type": "github"
+      }
     }
   },
   "root": "root",

flake.nix

@@ -3,7 +3,9 @@
 
   inputs = {
     nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
+
     terranix.url = "github:terranix/terranix";
+    terranix.inputs.nixpkgs.follows = "nixpkgs";
 
     agenix.url = "github:ryantm/agenix";
     agenix.inputs.nixpkgs.follows = "nixpkgs";
@@ -17,35 +19,65 @@
     nix-gerrit.url = "git+https://git.lix.systems/the-distro/nix-gerrit.git";
     nix-gerrit.inputs.nixpkgs.follows = "nixpkgs";
 
-    lix.follows = "hydra/nix";
+    gerrit-dashboard.url = "git+https://git.lix.systems/the-distro/gerrit-monitoring.git";
+    gerrit-dashboard.flake = false;
+
+    buildbot-nix.url = "git+https://git.lix.systems/lix-project/buildbot-nix.git?ref=refs/heads/bring-back-old-gerrit-reporting";
+    buildbot-nix.inputs.nixpkgs.follows = "nixpkgs";
+
+    channel-scripts.url = "git+https://git.lix.systems/the-distro/channel-scripts.git";
+    channel-scripts.inputs.nixpkgs.follows = "nixpkgs";
+
+    stateless-uptime-kuma.url = "git+https://git.dgnum.eu/DGNum/stateless-uptime-kuma.git";
+    stateless-uptime-kuma.flake = false;
+
+    lix.follows = "hydra/lix";
+
+    grapevine = {
+      type = "gitlab";
+      host = "gitlab.computer.surgery";
+      owner = "matrix";
+      repo = "grapevine-fork";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
   };
 
   outputs = { self, nixpkgs, terranix, colmena, ... } @ inputs:
   let
-    system = "x86_64-linux";
-    pkgs = import nixpkgs {
-      localSystem = system;
-      overlays = [
-        inputs.hydra.overlays.default
-        inputs.lix.overlays.default
-        inputs.nix-gerrit.overlays.default
-      ];
-    };
-    lib = pkgs.lib;
-    terraform = pkgs.opentofu;
-    terraformCfg = terranix.lib.terranixConfiguration {
+    supportedSystems = [ "x86_64-linux" "aarch64-linux" ];
+    forEachSystem = f: builtins.listToAttrs (map (system: {
+      name = system;
+      value = f system;
+    }) supportedSystems);
+    systemBits = forEachSystem (system: rec {
       inherit system;
-      modules = [
-        ./terraform
-        {
-          bagel.gandi.enable = true;
-          bagel.hydra.enable = true;
-        }
-      ];
-    };
+      pkgs = import nixpkgs {
+        localSystem = system;
+        overlays = [
+          inputs.hydra.overlays.default
+          inputs.lix.overlays.default
+          inputs.nix-gerrit.overlays.default
+          inputs.channel-scripts.overlays.default
+          (import "${inputs.stateless-uptime-kuma}/overlay.nix")
+        ];
+      };
+      terraform = pkgs.opentofu;
+      terraformCfg = terranix.lib.terranixConfiguration {
+        inherit system;
+        modules = [
+          ./terraform
+          {
+            bagel.dnsimple.enable = true;
+            bagel.hydra.enable = true;
+          }
+        ];
+      };
+    });
+    forEachSystem' = f: forEachSystem (system: (f systemBits.${system}));
+    inherit (nixpkgs) lib;
   in
   {
-    apps.${system} = {
+    apps = forEachSystem' ({ system, pkgs, terraformCfg, terraform, ... }: {
       tf = {
         type = "app";
         program = toString (pkgs.writers.writeBash "tf" ''
@@ -56,16 +88,19 @@
       };
 
       default = self.apps.${system}.tf;
-    };
+    });
 
-    devShells.${system}.default = pkgs.mkShell {
-      packages = [
-        inputs.agenix.packages.${system}.agenix
+    devShells = forEachSystem' ({ system, pkgs, ... }: {
+      default = pkgs.mkShell {
+        packages = [
+          inputs.agenix.packages.${system}.agenix
 
-        pkgs.colmena
-        pkgs.opentofu
-      ];
-    };
+          pkgs.opentofu
+
+          (pkgs.callPackage ./lib/colmena-wrapper.nix { })
+        ];
+      };
+    });
 
     nixosConfigurations = (colmena.lib.makeHive self.outputs.colmena).nodes;
 
@@ -73,35 +108,100 @@
       commonModules = [
         inputs.agenix.nixosModules.default
         inputs.hydra.nixosModules.hydra
+        inputs.buildbot-nix.nixosModules.buildbot-coordinator
+        inputs.buildbot-nix.nixosModules.buildbot-worker
 
         ./services
         ./common
       ];
 
-      makeBuilder = i: lib.nameValuePair "builder-${toString i}" {
-        imports = commonModules;
-        bagel.baremetal.builders = { enable = true; num = i; };
+      floralInfraModules = commonModules ++ [
+        ({ config, lib, ... }: {
+          # This means that anyone with @floral-infra permissions
+          # can ssh on root of every machines handled here.
+          bagel.admins.allowedGroups = [
+            "floral-infra"
+          ];
+
+          # Tag all machines which have local boot as local bootables.
+          deployment.tags = lib.mkMerge [
+            [ "floral" ]
+            (lib.mkIf (config.bagel.baremetal.builders.enable -> !config.bagel.baremetal.builders.netboot)
+              [ "localboot" ]
+            )
+          ];
+
+          bagel.monitoring.grafana-agent.tenant = "floral";
+          bagel.secrets.tenant = "floral";
+          bagel.builders.extra-build-capacity.provider.tenant = "floral";
+          bagel.services.buildbot.tenant = "floral";
+        })
+      ];
+
+      # These are Floral baremetal builders.
+      makeBuilder = i:
+      let
+        enableNetboot = i >= 6;
+      in
+      lib.nameValuePair "builder-${toString i}" {
+        imports = floralInfraModules;
+        bagel.baremetal.builders = { enable = true; num = i; netboot = enableNetboot; };
       };
 
-      builders = lib.listToAttrs (lib.genList makeBuilder 12);
+      lixInfraModules = commonModules ++ [
+        {
+          # This means that anyone with @lix-infra permissions
+          # can ssh on root of every machines handled here.
+          bagel.admins.allowedGroups = [
+            "lix-infra"
+          ];
+
+          # Tag all machines which have local boot as local bootables.
+          # Lix has no netbootable machine.
+          deployment.tags = [ "localboot" "lix" ];
+
+          bagel.monitoring.grafana-agent.tenant = "lix";
+          bagel.secrets.tenant = "lix";
+          bagel.builders.extra-build-capacity.provider = {
+            tenant = "lix";
+            buildfarmPublicKeys = [
+              # buildbot.lix.systems SSH key
+              "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDu4cEqZzAI/1vZjSQkTJ4ijIg9nuloOuSKUrnkJIOFn"
+            ];
+          };
+          bagel.services.buildbot.tenant = "lix";
+        }
+      ];
+
+      builders = lib.listToAttrs (lib.genList makeBuilder 11);
     in {
-      meta.nixpkgs = import nixpkgs {
-        localSystem = system;
-        overlays = [
-          inputs.hydra.overlays.default
-          inputs.lix.overlays.default
-          inputs.nix-gerrit.overlays.default
+      meta.nixpkgs = systemBits.x86_64-linux.pkgs;
+      # Add any non-x86_64 native systems here.
+      # Cross compilation is not supported yet.
+      meta.nodeNixpkgs =
+        let
+          aarch64-systems = systems: lib.genAttrs systems (system: systemBits.aarch64-linux.pkgs);
+        in
+        aarch64-systems [
+          "build01-aarch64-lix"
         ];
-      };
       meta.specialArgs.inputs = inputs;
 
-      bagel-box.imports = commonModules ++ [ ./hosts/bagel-box ];
-      meta01.imports = commonModules ++ [ ./hosts/meta01 ];
-      gerrit01.imports = commonModules ++ [ ./hosts/gerrit01 ];
-      fodwatch.imports = commonModules ++ [ ./hosts/fodwatch ];
-      wob-vpn-gw.imports = commonModules ++ [ ./hosts/wob-vpn-gw ];
+      bagel-box.imports = floralInfraModules ++ [ ./hosts/bagel-box ];
+      meta01.imports = floralInfraModules ++ [ ./hosts/meta01 ];
+      gerrit01.imports = floralInfraModules ++ [ ./hosts/gerrit01 ];
+      fodwatch.imports = floralInfraModules ++ [ ./hosts/fodwatch ];
+      git.imports = floralInfraModules ++ [ ./hosts/git ];
+      wob-vpn-gw.imports = floralInfraModules ++ [ ./hosts/wob-vpn-gw ];
+      buildbot.imports = floralInfraModules ++ [ ./hosts/buildbot ];
+      public01.imports = floralInfraModules ++ [ ./hosts/public01 ];
+      build-coord.imports = floralInfraModules ++ [ ./hosts/build-coord ];
+
+      build01-aarch64-lix.imports = lixInfraModules ++ [ ./hosts/build01-aarch64-lix ];
+      buildbot-lix.imports = lixInfraModules ++ [ ./hosts/buildbot-lix ];
     } // builders;
 
-    hydraJobs = builtins.mapAttrs (n: v: v.config.system.build.toplevel) self.nixosConfigurations;
+    hydraJobs = builtins.mapAttrs (n: v: v.config.system.build.netbootDir or v.config.system.build.toplevel) self.nixosConfigurations;
+    buildbotJobs = builtins.mapAttrs (_: v: v.config.system.build.toplevel) self.nixosConfigurations;
   };
 }

hosts/bagel-box/default.nix

@@ -20,6 +20,7 @@
     useHostResolvConf = false;
 
     hostName = "bagel-box";
+    domain = "infra.forkos.org";
     nameservers = [ "2001:4860:4860::8844" ];
 
     interfaces.host0.ipv6.addresses = [
@@ -36,21 +37,13 @@
 
   bagel.services = {
     postgres.enable = true;
-
-    hydra.enable = true;
-    hydra.dbi = "dbi:Pg:dbname=hydra;user=hydra";
-    # Takes 4 builders (0 → 3).
-    hydra.builders = lib.genList (i: "builder-${builtins.toString i}") 4;
-
     ofborg.enable = true;
   };
 
   bagel.sysadmin.enable = true;
 
-  security.acme.acceptTerms = true;
-  security.acme.defaults.email = "infra@forkos.org";
-
   services.openssh.enable = true;
 
+  system.stateVersion = "24.11";
   deployment.targetHost = "bagel-box.infra.forkos.org";
 }

hosts/build-coord/default.nix

@@ -0,0 +1,23 @@
+{ lib, ... }:
+{
+  imports = [ ./hardware.nix ];
+
+  networking.hostName = "build-coord";
+  networking.domain = "wob01.infra.forkos.org";
+
+  bagel.sysadmin.enable = true;
+
+  bagel.services = {
+    hydra.enable = true;
+    # Takes 10 builders (0 → 9).
+    hydra.builders = lib.genList (i: "builder-${builtins.toString i}") 10;
+  };
+
+  bagel.monitoring.exporters.hydra.enable = true;
+
+  # Hydra is proxied.
+  bagel.raito.v6-proxy-awareness.enable = true;
+
+  system.stateVersion = "24.05";
+  deployment.targetHost = "build-coord.wob01.infra.forkos.org";
+}

hosts/build-coord/hardware.nix

@@ -0,0 +1,87 @@
+{
+  boot.initrd.availableKernelModules = [ "ahci" "ehci_pci" "usb_storage" "usbhid" "sd_mod" ];
+  boot.initrd.kernelModules = [ "dm-snapshot" ];
+
+  nixpkgs.hostPlatform = "x86_64-linux";
+  hardware.cpu.intel.updateMicrocode = true;
+
+  boot.loader.systemd-boot.enable = true;
+  boot.loader.efi.canTouchEfiVariables = true;
+  boot.initrd.systemd.enable = true;
+
+  boot.initrd.services.lvm.enable = true;
+
+  boot.kernelParams = [
+    "console=tty1"
+    "console=ttyS0,115200"
+  ];
+
+  fileSystems = {
+    "/" = {
+      device = "/dev/disk/by-label/root";
+      fsType = "ext4";
+    };
+
+    "/boot" = {
+      device = "/dev/disk/by-label/BOOT";
+      fsType = "vfat";
+      options = [ "fmask=0022" "dmask=0022" ];
+    };
+  };
+
+  swapDevices = [
+    {
+      device = "/swapfile";
+      size = 20 * 1024;  # 50GiB
+    }
+  ];
+
+  zramSwap = {
+    enable = true;
+    memoryPercent = 25;
+  };
+
+  networking.useNetworkd = true;
+
+  systemd.network = {
+    netdevs = {
+      "40-uplink" = {
+        netdevConfig = {
+          Kind = "bond";
+          Name = "uplink";
+        };
+        bondConfig = {
+          Mode = "802.3ad";
+          TransmitHashPolicy = "layer3+4";
+        };
+      };
+    };
+    networks = {
+      "40-eno1" = {
+        name = "eno1";
+        bond = [ "uplink" ];
+      };
+      "40-eno2" = {
+        name = "eno2";
+        bond = [ "uplink" ];
+      };
+    };
+  };
+  networking.interfaces.uplink.ipv6.addresses = [
+    { address = "2a01:584:11::1:11"; prefixLength = 64; }
+  ];
+  networking.defaultGateway6 = { interface = "uplink"; address = "2a01:584:11::1"; };
+
+  services.coredns = {
+    enable = true;
+    config = ''
+      . {
+        bind lo
+        forward . 2001:4860:4860::6464
+        template ANY A { rcode NOERROR }
+      }
+    '';
+  };
+  services.resolved.enable = false;
+  networking.resolvconf.useLocalResolver = true;
+}

hosts/build01-aarch64-lix/default.nix

@@ -0,0 +1,27 @@
+{ ... }: {
+  networking.hostName = "build01";
+  networking.domain = "aarch64.lix.systems";
+
+  # Those free sweet VMs.
+  bagel.hardware.oracle-vm = {
+    enable = true;
+    system = "aarch64-linux";
+  };
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/a333323c-99f0-4258-8f68-496858d56f71";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/3E74-C937";
+      fsType = "vfat";
+    };
+
+  swapDevices = [ ];
+
+  bagel.builders.extra-build-capacity.provider.enable = true;
+  i18n.defaultLocale = "en_US.UTF-8";
+  system.stateVersion = "24.05";
+  deployment.targetHost = "build01.aarch64.lix.systems";
+}

hosts/buildbot-lix/default.nix

@@ -0,0 +1,71 @@
+# Configuration for a virtual machine in Raito's micro-DC basement.
+# 32 vCPU (2014 grade Xeon though)
+# 32GB RAM
+# 30GB SSD
+# 500GB HDD
+# All specifications can be upgraded to a certain extent, just ask Raito.
+# Hosts the coordinator for Buildbot.
+#
+# vim: et:ts=2:sw=2:
+#
+{ lib, modulesPath, ... }: {
+  networking.hostName = "buildbot";
+  networking.domain = "lix.systems";
+
+  zramSwap.enable = true;
+
+  bagel.sysadmin.enable = true;
+  # Buildbot is proxied.
+  bagel.raito.v6-proxy-awareness.enable = true;
+  bagel.hardware.raito-vm = {
+    enable = true;
+    networking = {
+      nat-lan-mac = "BC:24:11:75:62:42";
+      wan = {
+        mac = "BC:24:11:B2:5F:2E";
+        address = "2001:bc8:38ee:100::200/56";
+      };
+    };
+  };
+  i18n.defaultLocale = "en_US.UTF-8";
+
+  bagel.services.buildbot = {
+    enable = true;
+    domain = "buildbot.lix.systems";
+    gerrit =
+    {
+      domain = "gerrit.lix.systems";
+      port = 2022;
+      username = "buildbot";
+    };
+    cors.allowedOrigins = [
+      "https://*.lix.systems"
+    ];
+    projects = [
+      "lix"
+      "lix-installer"
+    ];
+    buildSystems = [
+      "x86_64-linux"
+      "aarch64-linux"
+      "aarch64-darwin"
+      # Too slow.
+      /* "x86_64-darwin" */
+    ];
+    # Lix is not allowed to use yet Floral's x86_64 builders for now.
+    builders = [ ];
+  };
+
+  # This machine does not use /nix from btrfs, and instead uses a store on a bigger disk.
+  fileSystems."/nix" =
+    lib.mkForce
+    { device = "/dev/disk/by-uuid/1815ca49-d0b0-4b99-8aec-0d790498ba6f";
+      fsType = "xfs";
+      neededForBoot = true;
+      options = [ "relatime" ];
+    };
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+  system.stateVersion = "24.05";
+  deployment.targetHost = "buildbot.lix.systems";
+}

hosts/buildbot/default.nix

@@ -0,0 +1,54 @@
+{
+  config,
+  lib,
+  pkgs,
+  nodes,
+  ...
+}:
+{
+  networking.hostName = "buildbot";
+  # TODO: make it the default
+  networking.domain = "infra.forkos.org";
+
+  bagel.sysadmin.enable = true;
+  # Buildbot is proxied.
+  bagel.raito.v6-proxy-awareness.enable = true;
+  bagel.hardware.raito-vm = {
+    enable = true;
+    networking = {
+      nat-lan-mac = "BC:24:11:E7:42:8B";
+      wan = {
+        address = "2001:bc8:38ee:100:1000::50/64";
+        mac = "BC:24:11:C9:BA:6C";
+      };
+    };
+  };
+
+  bagel.services.buildbot = {
+    enable = true;
+    domain = "buildbot.forkos.org";
+    gerrit =
+      let
+        cfgGerrit = nodes.gerrit01.config.bagel.services.gerrit;
+      in
+      {
+        domain = cfgGerrit.canonicalDomain;
+        port = cfgGerrit.port;
+        username = "buildbot";
+      };
+    cors.allowedOrigins = [
+      "https://*.forkos.org"
+    ];
+    projects = [
+      "buildbot-test"
+      "nixpkgs"
+      "infra"
+    ];
+    builders = [ "builder-9" ];
+  };
+
+  i18n.defaultLocale = "en_US.UTF-8";
+
+  system.stateVersion = "24.05";
+  deployment.targetHost = "buildbot.infra.forkos.org";
+}

hosts/fodwatch/default.nix

@@ -8,8 +8,6 @@
   networking.hostName = "fodwatch";
   networking.domain = "infra.forkos.org";
 
-  time.timeZone = "Europe/Paris";
-
   bagel.sysadmin.enable = true;
   # Fodwatch will be proxied.
   bagel.raito.v6-proxy-awareness.enable = true;

hosts/gerrit01/default.nix

@@ -9,8 +9,6 @@
   # TODO: make it the default
   networking.domain = "infra.forkos.org";
 
-  time.timeZone = "Europe/Paris";
-
   bagel.sysadmin.enable = true;
   # Gerrit is proxied.
   bagel.raito.v6-proxy-awareness.enable = true;
@@ -25,6 +23,9 @@
     };
   };
 
+  # Block all these crawlers!!
+  bagel.services.nginx.crawler-blocker.enable = true;
+
   fileSystems."/gerrit-data" = {
     device = "/dev/disk/by-uuid/d1062305-0dea-4740-9a27-b6b1691862a4";
     fsType = "ext4";
@@ -32,12 +33,117 @@
 
   bagel.services.gerrit = {
     enable = true;
+    pyroscope.enable = true;
     domains = [
       "cl.forkos.org"
     ];
+    canonicalDomain = "cl.forkos.org";
     data = "/gerrit-data";
   };
 
+  age.secrets.ows-deploy-key = {
+    file = ../../secrets/floral/ows-deploy-key.age;
+    mode = "0600";
+    owner = "git";
+    group = "git";
+  };
+  bagel.nixpkgs.one-way-sync =
+  let
+    mkNixpkgsJob = { timer, fromRefspec, localRefspec ? fromRefspec }: {
+      fromUri = "https://github.com/NixOS/nixpkgs";
+      inherit fromRefspec localRefspec timer;
+    };
+    mkLocalJob = { timer, fromRefspec, localRefspec }: {
+      fromUri = "https://cl.forkos.org/nixpkgs";
+      inherit fromRefspec localRefspec timer;
+    };
+    in
+  {
+    enable = true;
+
+    pushUrl = "ssh://ows_bot@cl.forkos.org:29418/nixpkgs";
+    deployKeyPath = config.age.secrets.ows-deploy-key.path;
+
+    # Sync main -> staging-next -> staging
+    branches."main-to-staging-next" = mkLocalJob {
+      timer = "00/8:20:00"; # every 8 hours, 20 minutes past the full hour
+      fromRefspec = "main";
+      localRefspec = "staging-next";
+    };
+    branches."staging-next-to-staging" = mkLocalJob {
+      timer = "00/8:40:00"; # every 8 hours, 40 minutes past the full hour
+      fromRefspec = "staging-next";
+      localRefspec = "staging";
+    };
+
+    # Sync nixpkgs -> fork
+    branches."nixpkgs-master" = mkNixpkgsJob {
+      timer = "hourly";
+      fromRefspec = "master";
+      localRefspec = "main";
+    };
+
+    branches."nixpkgs-staging" = mkNixpkgsJob {
+      timer = "hourly";
+      fromRefspec = "staging";
+    };
+
+    branches."nixpkgs-release-24.05" = mkNixpkgsJob {
+      timer = "hourly";
+      fromRefspec = "release-24.05";
+    };
+
+    branches."nixpkgs-staging-24.05" = mkNixpkgsJob {
+      timer = "hourly";
+      fromRefspec = "staging-24.05";
+    };
+
+    branches."nixpkgs-release-23.11" = mkNixpkgsJob {
+      timer = "hourly";
+      fromRefspec = "release-23.11";
+    };
+
+    branches."nixpkgs-staging-23.11" = mkNixpkgsJob {
+      timer = "hourly";
+      fromRefspec = "staging-23.11";
+    };
+
+    # Testing jobs for personal sandbox branches
+    branches."raito-unstable-sync" = {
+      fromUri = "https://github.com/NixOS/nixpkgs";
+      fromRefspec = "nixos-unstable-small";
+      localRefspec = "sandbox/raito/raito-unstable-small";
+      timer = "*-*-* 12:00:00";
+    };
+
+    branches."raito-release-sync" = {
+      fromUri = "https://github.com/NixOS/nixpkgs";
+      fromRefspec = "nixos-24.05";
+      localRefspec = "sandbox/raito/raito-nixos-24.05";
+      timer = "daily";
+    };
+  };
+
+  age.secrets.s3-channel-staging-keys.file = ../../secrets/floral/s3-channel-staging-keys.age;
+  bagel.nixpkgs.channel-scripts = {
+    enable = true;
+    otlp.enable = true;
+    nixpkgsUrl = "https://cl.forkos.org/nixpkgs.git";
+    hydraUrl = "https://hydra.forkos.org";
+    binaryCacheUrl = "https://cache.forkos.org";
+    baseUriForGitRevisions = "https://cl.forkos.org/plugins/gitiles/nixpkgs/+";
+    s3 = {
+      release = "bagel-channel-scripts-test";
+      channel = "bagel-channel-scripts-test";
+    };
+    releaseBucketCredentialsFile = config.age.secrets.s3-channel-staging-keys.path;
+    deployKeyFile = config.age.secrets.priv-ssh-key.path;
+    extraArgs = [
+      "--bypass-preflight-checks"
+    ];
+    channels = import ../../common/channels.nix;
+  };
+
   i18n.defaultLocale = "fr_FR.UTF-8";
 
   system.stateVersion = "24.05";

hosts/git/default.nix

@@ -0,0 +1,47 @@
+let
+  ipv6 = {
+    openssh ="2001:bc8:38ee:100:1000::41";
+    forgejo = "2001:bc8:38ee:100:1000::40";
+  };
+in
+{
+  networking.hostName = "git";
+  networking.domain = "infra.forkos.org";
+
+  bagel.sysadmin.enable = true;
+  # Forgejo will be proxied.
+  bagel.raito.v6-proxy-awareness.enable = true;
+  bagel.hardware.raito-vm = {
+    enable = true;
+    networking = {
+      nat-lan-mac = "BC:24:11:83:71:56";
+      wan = {
+        address = "${ipv6.forgejo}/64";
+        mac = "BC:24:11:0B:8A:81";
+      };
+    };
+  };
+
+  # Add one additional IPv6, so we can have both OpenSSH and
+  # Forgejo's built-in server bind on port :22.
+  systemd.network.networks."10-wan".networkConfig.Address = [ "${ipv6.openssh}/64" ];
+  services.openssh.listenAddresses = [{
+    addr = "[${ipv6.openssh}]";
+  }];
+  # Defaults to network.target, but networkd may take a while to settle and set up
+  # the required (additional) IPv6 address, leading to sshd to not being able to
+  # bind to the requested IP, crashing 5 times and running into the default
+  # restart counter limit (5).
+  systemd.services.sshd.wants = [ "network-online.target" ];
+  systemd.services.sshd.after = [ "network-online.target" ];
+
+  bagel.services.forgejo = {
+    enable = true;
+    sshBindAddr = ipv6.forgejo;
+  };
+
+  i18n.defaultLocale = "en_US.UTF-8";
+
+  system.stateVersion = "24.05";
+  deployment.targetHost = "git.infra.forkos.org";
+}

hosts/meta01/default.nix

@@ -2,8 +2,6 @@
   networking.hostName = "meta01";
   networking.domain = "infra.forkos.org";
 
-  time.timeZone = "Europe/Paris";
-
   bagel.sysadmin.enable = true;
   # netbox is proxied.
   bagel.raito.v6-proxy-awareness.enable = true;
@@ -24,6 +22,15 @@
   bagel.services.prometheus.enable = true;
   bagel.services.loki.enable = true;
   bagel.services.grafana.enable = true;
+  bagel.services.grapevine.enable = true;
+  bagel.services.pyroscope.enable = true;
+  bagel.services.tempo.enable = true;
+  bagel.services.hookshot = {
+    enable = true;
+    admins = [
+      "@k900:0upti.me"
+    ];
+  };
 
   i18n.defaultLocale = "fr_FR.UTF-8";
 

hosts/public01/default.nix

@@ -0,0 +1,50 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+{
+  networking.hostName = "public01";
+  # TODO: make it the default
+  networking.domain = "infra.forkos.org";
+
+  bagel.status = {
+    enable = true;
+    domain = "status.forkos.org";
+  };
+
+  bagel.sysadmin.enable = true;
+  # Newsletter is proxied.
+  bagel.raito.v6-proxy-awareness.enable = true;
+  bagel.newsletter = {
+    enable = true;
+    domain = "news.forkos.org";
+  };
+  bagel.hardware.raito-vm = {
+    enable = true;
+    networking = {
+      nat-lan-mac = "BC:24:11:A4:F7:D3";
+      wan = {
+        address = "2001:bc8:38ee:100:1000::60/64";
+        mac = "BC:24:11:DB:B8:10";
+      };
+    };
+  };
+
+  bagel.services.s3-revproxy = {
+    enable = true;
+    domain = "forkos.org";
+    s3.apiUrl = "s3.delroth.net";
+    targets = {
+      channels = "bagel-channels";
+      releases = "bagel-releases";
+      channel-scripts-test = "bagel-channel-scripts-test";
+    };
+  };
+
+  i18n.defaultLocale = "en_US.UTF-8";
+
+  system.stateVersion = "24.05";
+  deployment.targetHost = "public01.infra.forkos.org";
+}

hosts/wob-vpn-gw/default.nix

@@ -1,6 +1,10 @@
 { pkgs, lib, ... }:
 
 {
+  imports = [
+    ./netboot.nix
+  ];
+
   ###### Hardware ######
   boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "ehci_pci" "sd_mod" "sdhci_pci" ];
   boot.kernelModules = [ "kvm-amd" ];

hosts/wob-vpn-gw/netboot.nix

@@ -0,0 +1,61 @@
+{ config, lib, pkgs, nodes, modulesPath, ... }:
+
+# The way the connection is established is specific to the wob01 site and the Intel S2600KPR blades.
+# Proper netboot is not possible, because while the blades and the APU board (which is the netboot
+# server here) are in the same L2 network, the uplink connection of each blade is an LACP LAG,
+# meaning that the switch on the other side will only enable the port if it sees valid LACP packets.
+# We work around this by presenting a virtual floppy drive using the "IUSB" protocol of the BMC.
+# This virtual floppy drive contains an per-blade customized initramfs which will initialize the
+# network connection including IP configuration and load the actual image off hydra.
+
+let
+  netboot-server-ip = "2a01:584:11::2";
+  netbootNodes = lib.filterAttrs (_: node: node.config.bagel.baremetal.builders.enable && node.config.bagel.baremetal.builders.netboot) nodes;
+in {
+
+  assertions = [
+    {
+      assertion = !(lib.elem 443 config.networking.firewall.allowedTCPPorts);
+      message = ''
+        Port 443 is in networking.firewalls.allowedTCPPorts, but should be only manually
+        allowed for specific IPs and source ports in ${builtins.toJSON __curPos}
+      '';
+    }
+  ];
+
+  systemd.services = lib.mapAttrs' (nodename: node: let
+    bmcIp = "192.168.1.${toString (node.config.bagel.baremetal.builders.num * 4 + 2)}";
+    notipxe = node.config.system.build.notipxe.config.system.build.usbImage;
+  in lib.nameValuePair "iusb-spoof-${nodename}" {
+    wantedBy = [ "multi-user.target" ];
+    serviceConfig = {
+      Restart = "always";
+    };
+    script = ''
+      AUTH_TOKEN=$(${pkgs.iusb-spoof}/bin/make-token ${bmcIp})
+      exec ${pkgs.iusb-spoof}/bin/iusb-spoof -r ${bmcIp} 5123 $AUTH_TOKEN ${notipxe}
+    '';
+  }) netbootNodes;
+
+  # Since the builders are stateless, they can not store their ssh hostkeys
+  networking.firewall.allowedTCPPorts = [ 80 ]; # for ACME
+  networking.firewall.extraInputRules = ''
+    ip6 saddr 2a01:584:11::/64 tcp sport < 1024 tcp dport 443 accept;
+  '';
+  services.nginx = {
+    enable = true;
+    virtualHosts."vpn-gw.wob01.infra.forkos.org" = {
+      enableACME = true;
+      forceSSL = true;
+      locations = lib.mapAttrs' (nodename: node: let
+        ip = "2a01:584:11::1:${toString node.config.bagel.baremetal.builders.num}";
+      in lib.nameValuePair "/${nodename}/" {
+        root = "/var/www";
+        extraConfig = ''
+          allow ${ip};
+          deny all;
+        '';
+      }) netbootNodes;
+    };
+  };
+}

lib/colmena-wrapper.nix

@@ -0,0 +1,14 @@
+# A wrapper for colmena that prevents accidentally deploying changes without
+# having pulled.
+{ colmena, runCommandNoCC }:
+runCommandNoCC "colmena-wrapper"
+{
+  env.colmena = "${colmena}/bin/colmena";
+} ''
+  mkdir -p $out
+  ln -s ${colmena}/share $out/share
+  mkdir $out/bin
+
+  substituteAll ${./colmena-wrapper.sh.in} $out/bin/colmena
+  chmod +x $out/bin/colmena
+''

lib/colmena-wrapper.sh.in

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+doChecks() {
+    # creates refs in the refs/prefetch/remotes/origin namespace
+    echo "Prefetching repo changes..." >&2
+    git fetch --quiet --prefetch --no-write-fetch-head origin
+
+    diffs=$(git rev-list --left-right --count HEAD...refs/prefetch/remotes/origin/main)
+    only_in_local=$(echo "$diffs" | cut -f1)
+    only_in_main=$(echo "$diffs" | cut -f2)
+
+    if [[ $only_in_main -gt 0 && ! -v $FOOTGUN_ME_UWU ]]; then
+        echo >&2
+        echo "Attempting to deploy when main has $only_in_main commits not in your branch!" >&2
+        echo "This will probably revert someone's changes. Consider merging them." >&2
+        echo "If you really mean it, set the environment variable FOOTGUN_ME_UWU" >&2
+        exit 1
+    fi
+
+    if [[ $only_in_local -gt 0 ]]; then
+        echo "You have $only_in_local commits not yet pushed to main. Reminder to push them after :)" >&2
+    fi
+}
+
+if [[ $1 == 'apply' ]]; then
+    doChecks
+fi
+
+exec @colmena@ "$@"

overlays/default.nix

@@ -1 +1,9 @@
-[]
+[
+  (final: prev: {
+    iusb-spoof = final.callPackage ./iusb-spoof.nix {};
+    u-root = final.callPackage ./u-root {};
+    pyroscope = final.callPackage ./pyroscope {};
+    s3-revproxy = final.callPackage ./s3-revproxy {};
+    git-gc-preserve = final.callPackage ./git-gc-preserve {};
+  })
+]

overlays/git-gc-preserve/default.nix

@@ -0,0 +1,9 @@
+{ writeShellApplication, git, nettools }:
+
+writeShellApplication {
+  name = "git-gc-preserve";
+
+  runtimeInputs = [ git nettools ];
+
+  text = (builtins.readFile ./script.sh);
+}

overlays/git-gc-preserve/script.sh

@@ -0,0 +1,132 @@
+#!/usr/bin/env bash
+
+set +o errexit
+# Copyright (C) 2022 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+usage() { # exit code
+  cat <<-EOF
+NAME
+    git-gc-preserve - Run git gc and preserve old packs to avoid races for JGit
+SYNOPSIS
+    git gc-preserve
+DESCRIPTION
+    Runs git gc and can preserve old packs to avoid races with concurrently
+    executed commands in JGit.
+    This command uses custom git config options to configure if preserved packs
+    from the last run of git gc should be pruned and if packs should be preserved.
+    This is similar to the implementation in JGit [1] which is used by
+    JGit to avoid errors [2] in such situations.
+    The command prevents concurrent runs of the command on the same repository
+    by acquiring an exclusive file lock on the file
+      "\$repopath/gc-preserve.pid"
+    If it cannot acquire the lock it fails immediately with exit code 3.
+    Failure Exit Codes
+        1: General failure
+        2: Couldn't determine repository path. If the current working directory
+           is outside of the working tree of the git repository use git option
+           --git-dir to pass the root path of the repository.
+           E.g.
+              $ git --git-dir ~/git/foo gc-preserve
+        3: Another process already runs $0 on the same repository
+    [1] https://git.eclipse.org/r/c/jgit/jgit/+/87969
+    [2] https://git.eclipse.org/r/c/jgit/jgit/+/122288
+CONFIGURATION
+    "pack.prunepreserved": if set to "true" preserved packs from the last gc run
+      are pruned before current packs are preserved.
+    "pack.preserveoldpacks": if set to "true" current packs will be hard linked
+      to objects/pack/preserved before git gc is executed. JGit will
+      fallback to the preserved packs in this directory in case it comes
+      across missing objects which might be caused by a concurrent run of
+      git gc.
+EOF
+  exit "$1"
+}
+# acquire file lock, unlock when the script exits
+lock() { # repo
+  readonly LOCKFILE="$1/gc-preserve.pid"
+  test -f "$LOCKFILE" || touch "$LOCKFILE"
+  exec 9> "$LOCKFILE"
+  if flock -nx 9; then
+    echo -n "$$ $USER@$(hostname)" >&9
+    trap unlock EXIT
+  else
+    echo "$0 is already running"
+    exit 3
+  fi
+}
+unlock() {
+  # only delete if the file descriptor 9 is open
+  if { : >&9 ; } &> /dev/null; then
+    rm -f "$LOCKFILE"
+  fi
+  # close the file handle to release file lock
+  exec 9>&-
+}
+# prune preserved packs if pack.prunepreserved == true
+prune_preserved() { # repo
+  configured=$(git --git-dir="$1" config --get pack.prunepreserved)
+  if [ "$configured" != "true" ]; then
+    return 0
+  fi
+  local preserved=$1/objects/pack/preserved
+  if [ -d "$preserved" ]; then
+    printf "Pruning old preserved packs: "
+    count=$(find "$preserved" -name "*.old-pack" | wc -l)
+    rm -rf "$preserved"
+    echo "$count, done."
+  fi
+}
+# preserve packs if pack.preserveoldpacks == true
+preserve_packs() { # repo
+  configured=$(git --git-dir="$1" config --get pack.preserveoldpacks)
+  if [ "$configured" != "true" ]; then
+    return 0
+  fi
+  local packdir=$1/objects/pack
+  pushd "$packdir" >/dev/null || exit 1
+  mkdir -p preserved
+  printf "Preserving packs: "
+  count=0
+  for file in pack-*{.pack,.idx} ; do
+    ln -f "$file" preserved/"$(get_preserved_packfile_name "$file")"
+    if [[ "$file" == pack-*.pack ]]; then
+      ((count++))
+    fi
+  done
+  echo "$count, done."
+  popd >/dev/null || exit 1
+}
+# pack-0...2.pack to pack-0...2.old-pack
+# pack-0...2.idx to pack-0...2.old-idx
+get_preserved_packfile_name() { # packfile > preserved_packfile
+  local old=${1/%\.pack/.old-pack}
+  old=${old/%\.idx/.old-idx}
+  echo "$old"
+}
+# main
+while [ $# -gt 0 ] ; do
+    case "$1" in
+        -u|-h)  usage 0 ;;
+    esac
+    shift
+done
+args=$(git rev-parse --sq-quote "$@")
+repopath=$(git rev-parse --git-dir)
+if [ -z "$repopath" ]; then
+  usage 2
+fi
+lock "$repopath"
+prune_preserved "$repopath"
+preserve_packs "$repopath"
+git gc ${args:+"$args"} || { EXIT_CODE="$?"; echo "git gc failed"; exit "$EXIT_CODE"; }

overlays/iusb-spoof.nix

@@ -0,0 +1,23 @@
+{ rustPlatform, python3, makeWrapper }:
+let
+  pythonEnv = python3.withPackages (p: with p; [ requests ]);
+in
+
+rustPlatform.buildRustPackage rec {
+  pname = "iusb-spoof";
+  version = "0.1.0";
+
+  src = builtins.fetchGit {
+    url = "https://git.lix.systems/the-distro/iusb-spoof/";
+    rev = "fafd47986239cc2f4dfbbae74b17555608806581";
+  };
+
+  cargoLock.lockFile = src + "/Cargo.lock";
+
+  nativeBuildInputs = [ makeWrapper ];
+
+  postInstall = ''
+    install -Dm644 $src/make-token.py $out/opt/make-token.py
+    makeWrapper ${pythonEnv.interpreter} $out/bin/make-token --add-flags "$out/opt/make-token.py"
+  '';
+}

overlays/pyroscope/default.nix

@@ -0,0 +1,42 @@
+{ lib
+, buildGoModule
+, fetchFromGitHub
+}:
+
+buildGoModule rec {
+  pname = "pyroscope";
+  version = "1.7.1";
+
+  src = fetchFromGitHub {
+    owner = "grafana";
+    repo = "pyroscope";
+    rev = "v${version}";
+    hash = "sha256-iMP67J0Q8Cgo52iImMzAM3PEkk6uLF7r6v9TyXZVaIE=";
+  };
+
+  env.GOWORK = "off";
+
+  vendorHash = "sha256-ggntpnU9s2rpkv6S0LnZNexrdkBsdsUrGPc93SVrK4M=";
+
+  subPackages = [ "cmd/profilecli" "cmd/pyroscope" ];
+
+  ldflags = [
+    "-extldflags"
+    "-static"
+    "-s"
+    "-w"
+    "-X=github.com/grafana/pyroscope/pkg/util/build.Branch=${src.rev}"
+    "-X=github.com/grafana/pyroscope/pkg/util/build.Version=${version}"
+    "-X=github.com/grafana/pyroscope/pkg/util/build.Revision=${src.rev}"
+    "-X=github.com/grafana/pyroscope/pkg/util/build.BuildDate=1970-01-01T00:00:00Z"
+  ];
+
+  meta = with lib; {
+    description = "Continuous profiling platform";
+    homepage = "https://github.com/grafana/pyroscope";
+    changelog = "https://github.com/grafana/pyroscope/blob/${src.rev}/CHANGELOG.md";
+    license = licenses.agpl3Only;
+    maintainers = with maintainers; [ raitobezarius ];
+    mainProgram = "pyroscope";
+  };
+}

overlays/s3-revproxy/default.nix

@@ -0,0 +1,39 @@
+# Originally written by Jade Lovelace for Lix.
+{ lib, buildGoModule, fetchFromGitHub }:
+buildGoModule rec {
+  pname = "s3-revproxy";
+  version = "4.15.0";
+
+  src = fetchFromGitHub {
+    owner = "oxyno-zeta";
+    repo = "s3-proxy";
+    rev = "v${version}";
+    hash = "sha256-q0cfAo8Uz7wtKljmSDaJ320bjg2yXydvvxubAsMKzbc=";
+  };
+
+  vendorHash = "sha256-dOwNQtTfOCQcjgNBV/FeWdwbW9xi1OK5YD7PBPPDKOQ=";
+
+  ldflags = [
+    "-X github.com/oxyno-zeta/s3-proxy/pkg/s3-proxy/version.Version=${version}"
+    "-X github.com/oxyno-zeta/s3-proxy/pkg/s3-proxy/version.Metadata="
+  ];
+
+  postPatch = ''
+    # Refer to the included templates in the package instead of cwd-relative
+    sed -i "s#Path = \"templates/#Path = \"$out/share/s3-revproxy/templates/#" pkg/s3-proxy/config/config.go
+  '';
+
+  postInstall = ''
+    mkdir -p $out/share/s3-revproxy
+    cp -r templates/ $out/share/s3-revproxy/templates
+  '';
+
+  meta = {
+    description = "S3 Reverse Proxy with GET, PUT and DELETE methods and authentication (OpenID Connect and Basic Auth)";
+    homepage = "https://oxyno-zeta.github.io/s3-proxy";
+    # hm, not having a maintainers entry is kind of inconvenient
+    maintainers = [ ];
+    licenses = lib.licenses.asl20;
+    mainProgram = "s3-proxy";
+  };
+}

overlays/u-root/default.nix

@@ -0,0 +1,20 @@
+{ buildGoModule, fetchFromGitHub }:
+
+buildGoModule rec {
+  pname = "u-root";
+  version = "0.14.0";
+
+  src = fetchFromGitHub {
+    owner = "u-root";
+    repo = "u-root";
+    rev = "v${version}";
+    hash = "sha256-8zA3pHf45MdUcq/MA/mf0KCTxB1viHieU/oigYwIPgo=";
+  };
+
+  patches = [
+    ./u-root-allow-https.patch
+  ];
+
+  vendorHash = null;
+  doCheck = false;
+}

overlays/u-root/u-root-allow-https.patch

@@ -0,0 +1,12 @@
+diff --git a/pkg/curl/schemes.go b/pkg/curl/schemes.go
+index 8bac3bc0..cd396cbc 100644
+--- a/pkg/curl/schemes.go
++++ b/pkg/curl/schemes.go
+@@ -81,6 +81,7 @@ var (
+ 	DefaultSchemes = Schemes{
+ 		"tftp": DefaultTFTPClient,
+ 		"http": DefaultHTTPClient,
++		"https": DefaultHTTPClient,
+ 		"file": &LocalFileClient{},
+ 	}
+ )

pkgs/forgejo/branch-view_remove-expensive-commit-divergence-metric.patch

@@ -0,0 +1,59 @@
+diff --git a/services/repository/branch.go b/services/repository/branch.go
+index e1a313749f..5a8d823eef 100644
+--- a/services/repository/branch.go
++++ b/services/repository/branch.go
+@@ -26,7 +26,6 @@ import (
+ 	"code.gitea.io/gitea/modules/timeutil"
+ 	webhook_module "code.gitea.io/gitea/modules/webhook"
+ 	notify_service "code.gitea.io/gitea/services/notify"
+-	files_service "code.gitea.io/gitea/services/repository/files"
+ 
+ 	"xorm.io/builder"
+ )
+@@ -129,21 +128,7 @@ func loadOneBranch(ctx context.Context, repo *repo_model.Repository, dbBranch *g
+ 	p := protectedBranches.GetFirstMatched(branchName)
+ 	isProtected := p != nil
+ 
+-	var divergence *git.DivergeObject
+-
+-	// it's not default branch
+-	if repo.DefaultBranch != dbBranch.Name && !dbBranch.IsDeleted {
+-		var err error
+-		divergence, err = files_service.CountDivergingCommits(ctx, repo, git.BranchPrefix+branchName)
+-		if err != nil {
+-			return nil, fmt.Errorf("CountDivergingCommits: %v", err)
+-		}
+-	}
+-
+-	if divergence == nil {
+-		// tolerate the error that we cannot get divergence
+-		divergence = &git.DivergeObject{Ahead: -1, Behind: -1}
+-	}
++	divergence := &git.DivergeObject{Ahead: -1, Behind: -1}
+ 
+ 	pr, err := issues_model.GetLatestPullRequestByHeadInfo(ctx, repo.ID, branchName)
+ 	if err != nil {
+diff --git a/templates/repo/branch/list.tmpl b/templates/repo/branch/list.tmpl
+index a577fed450..e102796315 100644
+--- a/templates/repo/branch/list.tmpl
++++ b/templates/repo/branch/list.tmpl
+@@ -102,19 +102,6 @@
+ 							{{end}}
+ 							</td>
+ 							<td class="two wide ui">
+-								{{if and (not .DBBranch.IsDeleted) $.DefaultBranchBranch}}
+-								<div class="commit-divergence">
+-									<div class="bar-group">
+-										<div class="count count-behind">{{.CommitsBehind}}</div>
+-										{{/* old code bears 0/0.0 = NaN output, so it might output invalid "width: NaNpx", it just works and doesn't caues any problem. */}}
+-										<div class="bar bar-behind" style="width: {{Eval 100 "*" .CommitsBehind "/" "(" .CommitsBehind "+" .CommitsAhead "+" 0.0 ")"}}%"></div>
+-									</div>
+-									<div class="bar-group">
+-										<div class="count count-ahead">{{.CommitsAhead}}</div>
+-										<div class="bar bar-ahead" style="width: {{Eval 100 "*" .CommitsAhead "/" "(" .CommitsBehind "+" .CommitsAhead "+" 0.0 ")"}}%"></div>
+-									</div>
+-								</div>
+-								{{end}}
+ 							</td>
+ 							<td class="two wide right aligned">
+ 								{{if not .LatestPullRequest}}

pkgs/forgejo/commit-view_fix-broken-and-expensive-cherry-pick-default-branch-selection.patch

@@ -0,0 +1,32 @@
+diff --git a/routers/web/repo/commit.go b/routers/web/repo/commit.go
+index 718454e063..8fa299710c 100644
+--- a/routers/web/repo/commit.go
++++ b/routers/web/repo/commit.go
+@@ -408,12 +408,6 @@ func Diff(ctx *context.Context) {
+ 		}
+ 	}
+ 
+-	ctx.Data["BranchName"], err = commit.GetBranchName()
+-	if err != nil {
+-		ctx.ServerError("commit.GetBranchName", err)
+-		return
+-	}
+-
+ 	ctx.HTML(http.StatusOK, tplCommitPage)
+ }
+ 
+diff --git a/templates/repo/commit_page.tmpl b/templates/repo/commit_page.tmpl
+index c37fb46975..18c9cf18f8 100644
+--- a/templates/repo/commit_page.tmpl
++++ b/templates/repo/commit_page.tmpl
+@@ -71,8 +71,8 @@
+ 												"branchForm" "branch-dropdown-form"
+ 												"branchURLPrefix" (printf "%s/_cherrypick/%s/" $.RepoLink .CommitID) "branchURLSuffix" ""
+ 												"setAction" true "submitForm" true}}
+-											<form method="get" action="{{$.RepoLink}}/_cherrypick/{{.CommitID}}/{{if $.BranchName}}{{PathEscapeSegments $.BranchName}}{{else}}{{PathEscapeSegments $.Repository.DefaultBranch}}{{end}}" id="branch-dropdown-form">
+-												<input type="hidden" name="ref" value="{{if $.BranchName}}{{$.BranchName}}{{else}}{{$.Repository.DefaultBranch}}{{end}}">
++											<form method="get" action="{{$.RepoLink}}/_cherrypick/{{.CommitID}}/{{PathEscapeSegments $.Repository.DefaultBranch}}" id="branch-dropdown-form">
++												<input type="hidden" name="ref" value="{{$.Repository.DefaultBranch}}">
+ 												<input type="hidden" name="refType" value="branch">
+ 												<input type="hidden" id="cherry-pick-type" name="cherry-pick-type"><br>
+ 												<button type="submit" id="cherry-pick-submit" class="ui primary button"></button>

pkgs/forgejo/default.nix

@@ -0,0 +1,40 @@
+{ forgejo-lts }:
+
+forgejo-lts.overrideAttrs (prev: {
+  patches = [
+    # Branch divergence calculations for a single branch may take 100-200ms on something as big
+    # as nixpkgs. The branch view defaults to 20 branches for each page, taking roughtly 3s to
+    # calculate each branch sequentially and render, while consuming a single core at 100%.
+    # The idea is to look into making this less expensive or async.
+    # But for now, to get this going, we will simply drop that metric.
+    ./branch-view_remove-expensive-commit-divergence-metric.patch
+
+    # This is literally broken and eats resources for nothing of value.
+    # We should upstream this.
+    # The tl;dr is: It calculates the nearest branch for the requested commit at
+    # /:owner/:repo/commit/:commit to use it as the default cherry-pick target branch
+    # selection in a drop-down only users with commit perms can actually view and use.
+    # It's expensive to calculate and happens on every request to /commit/:commit.
+    # To add insult to injury, it's hardly of any use: The nearest branch of a commit
+    # will almost always be a branch that already carries the commit. The branch you
+    # most likely don't want to cherry-pick to.
+    ./commit-view_fix-broken-and-expensive-cherry-pick-default-branch-selection.patch
+
+    # Disable various /:owner/:repo/activity/ sub-views. They are expensive, which is
+    # totally fine and expected. There is even proper caching in place.
+    # However, on a scale of nixpkgs, those calculations take ages, while, of course,
+    # pinning a single CPU core at 100%.
+    # For now, we will simply disable this feature.
+    # Due to the 501 status code it returns, the frontend prints a "Not implemented"
+    # error, saving us from patching the frontend while still providing a helpful
+    # user-facing error text.
+    # It should be noted that this particular status code has the downside of being
+    # in the 5xx range, meaning it will show up as such in our prometheus metrics.
+    ./disable-expensive-repository-activity-stats.patch
+
+    # Migrations and pull-mirrors are something easily abused to bring a public instance to a complete halt.
+    # Both features can be disabled via repository.DISABLE_MIGRATIONS and mirror.ENABLE, but we want to keep
+    # this functionality for admins.
+    ./limit-migrations-and-pull-mirrors-to-admins.patch
+  ];
+})

pkgs/forgejo/disable-expensive-repository-activity-stats.patch

@@ -0,0 +1,34 @@
+diff --git a/routers/web/web.go b/routers/web/web.go
+index ee9694f41c..f55b8d6f62 100644
+--- a/routers/web/web.go
++++ b/routers/web/web.go
+@@ -57,6 +57,10 @@ import (
+ 	"github.com/prometheus/client_golang/prometheus"
+ )
+ 
++func endpointNotImplemented(ctx *context.Context) {
++	ctx.JSON(http.StatusNotImplemented, "This endpoint has been removed due to performance issues with it and as such is not longer implemented.")
++}
++
+ // optionsCorsHandler return a http handler which sets CORS options if enabled by config, it blocks non-CORS OPTIONS requests.
+ func optionsCorsHandler() func(next http.Handler) http.Handler {
+ 	var corsHandler func(next http.Handler) http.Handler
+@@ -1425,15 +1429,15 @@ func registerRoutes(m *web.Route) {
+ 			m.Get("/{period}", repo.Activity)
+ 			m.Group("/contributors", func() {
+ 				m.Get("", repo.Contributors)
+-				m.Get("/data", repo.ContributorsData)
++				m.Get("/data", endpointNotImplemented)
+ 			}, repo.MustBeNotEmpty, context.RequireRepoReaderOr(unit.TypeCode))
+ 			m.Group("/code-frequency", func() {
+ 				m.Get("", repo.CodeFrequency)
+-				m.Get("/data", repo.CodeFrequencyData)
++				m.Get("/data", endpointNotImplemented)
+ 			}, repo.MustBeNotEmpty, context.RequireRepoReaderOr(unit.TypeCode))
+ 			m.Group("/recent-commits", func() {
+ 				m.Get("", repo.RecentCommits)
+-				m.Get("/data", repo.RecentCommitsData)
++				m.Get("/data", endpointNotImplemented)
+ 			}, repo.MustBeNotEmpty, context.RequireRepoReaderOr(unit.TypeCode))
+ 		}, context.RepoRef(), context.RequireRepoReaderOr(unit.TypeCode, unit.TypePullRequests, unit.TypeIssues, unit.TypeReleases))
+ 

pkgs/forgejo/limit-migrations-and-pull-mirrors-to-admins.patch

@@ -0,0 +1,53 @@
+diff --git a/routers/api/v1/repo/migrate.go b/routers/api/v1/repo/migrate.go
+index 2caaa130e8..455e89e93e 100644
+--- a/routers/api/v1/repo/migrate.go
++++ b/routers/api/v1/repo/migrate.go
+@@ -12,7 +12,6 @@ import (
+ 
+ 	"code.gitea.io/gitea/models"
+ 	"code.gitea.io/gitea/models/db"
+-	"code.gitea.io/gitea/models/organization"
+ 	"code.gitea.io/gitea/models/perm"
+ 	access_model "code.gitea.io/gitea/models/perm/access"
+ 	repo_model "code.gitea.io/gitea/models/repo"
+@@ -86,22 +85,7 @@ func Migrate(ctx *context.APIContext) {
+ 	}
+ 
+ 	if !ctx.Doer.IsAdmin {
+-		if !repoOwner.IsOrganization() && ctx.Doer.ID != repoOwner.ID {
+-			ctx.Error(http.StatusForbidden, "", "Given user is not an organization.")
+-			return
+-		}
+-
+-		if repoOwner.IsOrganization() {
+-			// Check ownership of organization.
+-			isOwner, err := organization.OrgFromUser(repoOwner).IsOwnedBy(ctx, ctx.Doer.ID)
+-			if err != nil {
+-				ctx.Error(http.StatusInternalServerError, "IsOwnedBy", err)
+-				return
+-			} else if !isOwner {
+-				ctx.Error(http.StatusForbidden, "", "Given user is not owner of organization.")
+-				return
+-			}
+-		}
++		ctx.Error(http.StatusForbidden, "", "You need to be administrator of this Forgejo instance to be able to create mirrors.")
+ 	}
+ 
+ 	remoteAddr, err := forms.ParseRemoteAddr(form.CloneAddr, form.AuthUsername, form.AuthPassword)
+diff --git a/routers/web/repo/migrate.go b/routers/web/repo/migrate.go
+index 97b0c425ea..554a470eab 100644
+--- a/routers/web/repo/migrate.go
++++ b/routers/web/repo/migrate.go
+@@ -150,6 +150,12 @@ func handleMigrateRemoteAddrError(ctx *context.Context, err error, tpl base.TplN
+ // MigratePost response for migrating from external git repository
+ func MigratePost(ctx *context.Context) {
+ 	form := web.GetForm(ctx).(*forms.MigrateRepoForm)
++
++	if !ctx.Doer.IsAdmin {
++		ctx.Error(http.StatusForbidden, "MigratePost: you need to be site administrator to use migrations and mirrors")
++		return
++	}
++
+ 	if setting.Repository.DisableMigrations {
+ 		ctx.Error(http.StatusForbidden, "MigratePost: the site administrator has disabled migrations")
+ 		return

secrets.nix

@@ -1,25 +1,76 @@
 let
   keys = import common/ssh-keys.nix;
 
-  commonKeys = keys.users.delroth ++ keys.users.raito;
+  commonKeys = {
+    # WARNING: `keys.users.*` are *lists*, so you need concatenate them, don't put them into lists!
+    # Otherwise, agenix will be confused!
+    global = keys.users.raito;
+    lix = keys.users.hexchen ++ keys.users.jade;
+    floral = keys.users.delroth;
+  };
 
   secrets = with keys; {
-    hydra-s3-credentials = [ machines.bagel-box ];
-    hydra-signing-priv = [ machines.bagel-box ];
-    hydra-ssh-key-priv = [ machines.bagel-box ];
-    netbox-environment = [ machines.meta01 ];
-    mimir-environment = [ machines.meta01 ];
-    grafana-oauth-secret = [ machines.meta01 ];
-    loki-environment = [ machines.meta01 ];
+    floral = {
+      hydra-postgres-key = [ machines.build-coord ];
+      hydra-s3-credentials = [ machines.build-coord ];
+      hydra-signing-priv = [ machines.build-coord ];
+      hydra-ssh-key-priv = [ machines.build-coord ];
 
-    # These are the same password, but nginx wants it in htpasswd format
-    metrics-push-htpasswd = [ machines.meta01 ];
-    metrics-push-password = builtins.attrValues machines;
+      netbox-environment = [ machines.meta01 ];
+      mimir-environment = [ machines.meta01 ];
+      mimir-webhook-url = [ machines.meta01 ];
+      grafana-oauth-secret = [ machines.meta01 ];
+      loki-environment = [ machines.meta01 ];
+      gerrit-prometheus-bearer-token = [ machines.gerrit01 machines.meta01 ];
+      pyroscope-secrets = [ machines.meta01 ];
+      tempo-environment = [ machines.meta01 ];
+
+      buildbot-worker-password = [ machines.buildbot ];
+      buildbot-oauth-secret = [ machines.buildbot ];
+      buildbot-workers = [ machines.buildbot ];
+      # Private SSH key to Gerrit
+      # ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHx52RUPWzTa2rBA96xcnGjjzAboNN/hm6gW+Q6JiSos
+      buildbot-service-key = [ machines.buildbot ];
+      # Signing key for Buildbot's specific cache
+      buildbot-signing-key = [ machines.buildbot ];
+      buildbot-remote-builder-key = [ machines.buildbot ];
+
+      # These are the same password, but nginx wants it in htpasswd format
+      metrics-push-htpasswd = [ machines.meta01 ];
+      # Yes, even Lix machines are included in this monitoring infrastructure.
+      metrics-push-password = builtins.attrValues machines;
+
+      ows-deploy-key = [ machines.gerrit01 ];
+      s3-channel-staging-keys = [ machines.gerrit01 ];
+      s3-channel-keys = [ machines.gerrit01 ];
+
+      postgres-ca-priv = [ machines.bagel-box ];
+      postgres-tls-priv = [ machines.bagel-box ];
+
+      newsletter-secrets = [ machines.public01 ];
+      s3-revproxy-api-keys = [ machines.public01 ];
+      stateless-uptime-kuma-password = [ machines.public01 ];
+    };
+
+    lix = {
+      buildbot-worker-password = [ machines.buildbot-lix ];
+      buildbot-oauth-secret = [ machines.buildbot-lix ];
+      buildbot-workers = [ machines.buildbot-lix ];
+      # Private SSH key to Gerrit
+      # ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHx52RUPWzTa2rBA96xcnGjjzAboNN/hm6gW+Q6JiSos
+      buildbot-service-key = [ machines.buildbot-lix ];
+      # Signing key for Buildbot's specific cache
+      buildbot-signing-key = [ machines.buildbot-lix ];
+      buildbot-remote-builder-key = [ machines.buildbot-lix ];
+    };
   };
+
+  mkSecretListFor = tenant:
+    map (secretName: {
+      name = "secrets/${tenant}/${secretName}.age";
+      value.publicKeys = secrets.${tenant}."${secretName}" ++ commonKeys.global ++ commonKeys.${tenant};
+    }) (builtins.attrNames secrets.${tenant});
 in
   builtins.listToAttrs (
-    map (secretName: {
-      name = "secrets/${secretName}.age";
-      value.publicKeys = secrets."${secretName}" ++ commonKeys;
-    }) (builtins.attrNames secrets)
+    (mkSecretListFor "floral") ++ (mkSecretListFor "lix")
   )

secrets/floral/buildbot-oauth-secret.age

@@ -0,0 +1,20 @@
+age-encryption.org/v1
+-> ssh-ed25519 87T2Ig tzPD1x6XKuDfgJ8jkQnwW/ALp2pkANCeNoO8xdUqq30
+QSsuO6Dwc8QJuY92gXRnWB5aJ2SU9X2uFh01GmLVaQE
+-> ssh-ed25519 K3b7BA 9G9Uw1xY8hq//xphNWrPn5y7vG2o8/kwkC8cJGuf/mI
+Ip0019OUaFq2ZDFI3i77hdsp9IqFV2qqYIB/TnDSXgo
+-> ssh-ed25519 +qVung dx22ef+x9X5mr73L8NUzxYQa640M2XViELjJcpgF3go
+CXyit7pk8SPNHBgULlMQUAasGAn4C36zcwOBDI46nU4
+-> ssh-rsa krWCLQ
+NlGh0hM10NOuek7MbrFo0iul0kQQtDFmZIhgpyqaATMdCDRBXJOyhASHU5N0zDDJ
+MLaJUV0l2o1ghBF9RhSKdoUPVEn8Cce/nfQepYzMlfc4UG3qWXwabwR6EtqqCZCJ
+jAEWZ8taTKDmzoXwuygCW+bRBuoMMrcfzu7V90N+mQpZWtOScatb6E7d5VRqjlar
+st1ZQu5ccghufyQSUmOC7GpojOyutX5EvbMGn84X4ouZRHRX/8fTgaqicV+aeAIb
+QyXisOrO6C+Jle5qfxzMSe8c/TCyF2574kD6F1BQ9Kpkinn8v7OWcIXtkNmZ5hzK
+vs0Bej8yZVsoBkj1vWAM0A
+-> ssh-ed25519 /vwQcQ n+hr1cV1zRs1S86YnA+0oRB8SCaPKtkoMNe15ZsVVwM
+fdFtUqno07ik6FpW5zMImIjd8wM8dMgwU+RqjeT2PiI
+-> ssh-ed25519 0R97PA ddPILw57gkuKvAqlmpa+MnV/LSEdyQzQaAarCUqQ1xE
+ozK5a6uXZDc17OrX0OZun9hmZwP3H3rYQiNuKnukqsg
+--- f7yGgKQpCPj64Ps0HfMcToYircGH5SPqMzVZrUMB8ZI
+føv[iY\ÅšMP,¯Ùh°Èxb—Ðÿ«J<C2AB>*ºË"”+¬ÒA0T˜?KmˆPÈË2¹'2±‚µ³Ø=¯êÚÏŸj”

secrets/floral/buildbot-worker-password.age

@@ -0,0 +1,20 @@
+age-encryption.org/v1
+-> ssh-ed25519 87T2Ig p8lEB5da4fIfLH/HKBsghzq5mvQLB69UB4+uAi3DGCw
+NeZ3jPTUKa7MiqjrFPrYuP4VneytQPdBNqf+omPZJYM
+-> ssh-ed25519 K3b7BA uP2K1hU7uLmiHXmmoUdsB7CHQq61ZkEAjG/aK863RDw
+0chTczEMXASdYiwqNxDQ+vMXXhjOf64oIQ2ULZmQI8Y
+-> ssh-ed25519 +qVung jUgEqz3+ypL7mwJ1R7lfeOMhkon/aRrNSJUJT3X7vmU
+pgOiwrp9JiA20yw9bsxi8eiQ9/23CYXKRBGF1pea9eI
+-> ssh-rsa krWCLQ
+snCHrLHzkjimwIxKO90IjnHwOArlozO9kd/aCdZZnYNgh/QG3rUSceSn9yTHbtMV
+izv0SU51LrRU+JyE+a524AxKhyPBvGDig20j7hMy5fVxZqeunztqtlha5gaYYaQg
+Tbfs9tDP+pCIgzMVNqYf6EJ4MK7qjNf9DE5I490Eta5YZxAi/3To3BmZmIYtCz6l
+1kNRiSmWCbZqE25keFgPCgRMFXAFK9W6NmL+HamqCUhjPoJg/Gd4sf39EONT0PYg
+7BpCOAnwwfECHPxpM3qv0h2kJXTb4DZ715cFReSVyQe5fvKv8hoWhl/S+++pEYT8
+u/LKBx/o7e3Kd7cm2RGnBw
+-> ssh-ed25519 /vwQcQ 4+IQPRsMMHmuSGL7T7IbRkTTuL+TTqgdQp5FSbyt8Dw
+KOI0LKQ0oA5XtxaW7wftlEJB0BGVnx41HUJMG92SRUA
+-> ssh-ed25519 0R97PA l1aWUEv8nLEtYnpY1gjTJqk5UYm51NDqOjYmL83rZ10
+B7qDZwCpolkIajqCXeOepwmF6ciJfKvr+AN7VouMUvA
+--- lz/IMMPxBpD3Bzuv9Wl23+swBQHlblhlAO/ZXAgN0hU
+µoÍüÌ<EFBFBD>²-‚Īr °eó|Í?ït
èìÎZ<C38E>¬sÒì!ŸƒÁ<>@Ï'–ìèz6UöÎg
JøÑOµ–s13<31>š‹8<î’%-·Ô‡Eÿ}–Šdm9¿å¢Óoæ

secrets/floral/grafana-oauth-secret.age

@@ -0,0 +1,20 @@
+age-encryption.org/v1
+-> ssh-ed25519 j2r2qQ vwcaLpvGJ9swXnV8idDwi9jdRPSj38As9p2QFkIJ1Xc
+FLnZeblHDQQcWjFm1iaghbvuFgOG3miwtkRE5sz1+X0
+-> ssh-ed25519 K3b7BA 9VRe2rBwg3G9lxxfxL/yLob2NZmLJTBMxzx0Ew8VwmY
+/I2W80UykNvll5o98OPeMpIsddOel9B7uQlio0X3gcs
+-> ssh-ed25519 +qVung VsqKzMD85aps4PIx2zqae2Dj7YWibiaKYb5z7ws8ggM
+Y9dRd/hOz8h4avlutBQ1YZgHIAf/AuTr5WaByKlFbLE
+-> ssh-rsa krWCLQ
+gjyaUFrIIbZnFTGVw4XEZzkTIP/+qXV6/q0W8Wb4EtqQXDRISFT+bwxQU/S2p5hf
+7+JGcn4BZg6puOJ5BBABWtpn6gcX5OFfga5azIdioF/R19XByT+0SK5njw8g1VPS
+R7o8kQt2yvKWayoq9Cis5XRg+4KANkwOQaNTO8AdiCwgq9nc0Cd9avk8QhaFoR74
+D5cf8jPsufp744rQqwhWDoG533LS1WUUuYZqRmtp2Vz+r583RhSscaNyA7ddr7o6
+e9ZQJyL5bKiN8qe3Xm76lLypf/wg7+aGn8HHnO6GA65g+VYfjLMODEqCN/+uDJtB
+g8v2wzKIGYlZiV1hEjH8nw
+-> ssh-ed25519 /vwQcQ 4pU5JGK5vpZbFgq01a9YY8VmSJvPSHPSZD50TLJwKHc
+L46UA/p+bNSR8cLmL8G7VpmAcZ+sy5AROc4yj2ABOWg
+-> ssh-ed25519 0R97PA Tk00kYLhsEy1HJcmKLgaLWTdNP8XV/cdKHMLzyK6glk
+kwyQZr/h6MutROJmjVfPWGcf9xN5Uc5w5mVyuKcK64g
+--- E0vVtBqbjNkZY0/1dFJ53uVAR7IGPO+OMmXkpJcKmlw
+{ÐQê%è‹õY•B,isr¥1‘<31>|¼yLÕ'7?¶iŠM…¶MU]×ê/d2¸I1u¶2hZjHåh&¥

secrets/floral/hydra-s3-credentials.age

@@ -0,0 +1,20 @@
+age-encryption.org/v1
+-> ssh-ed25519 +uvEmw Kuduppyhz98frjlV96R/WcmPdaWmHbNKZhQs76GGTx8
+7zpedlPflGOi6FqkpswAJflx77yde7M2XlTw/8mz2tU
+-> ssh-ed25519 K3b7BA TPNmUK38+TR58MpsgxIe91bY6E1j9HecIFn0AKdat20
+MjXh06xd3mkPcK+iEonx+itsHvEGHSknzO6Sgh5WdZ4
+-> ssh-ed25519 +qVung KereHQ3Kl0f7O0xzl2s4Yu4KX7OOA17R7p/w8uRx/AQ
+3aOqUoBCDurkh5jT2fq5MDPQKIDISQdXBhF+qeRppnA
+-> ssh-rsa krWCLQ
+TVlmM2LYR339Aglo2D4j/Itr0E+mg7UEaV9n+sUYyit0phWS1zMI2YMc78Xbmn73
+6U0VYi/3hpesD6/8uA5sywuueOMntlL32aECz/DJPC71feMjvHTxiJpqnFw6DQYJ
+FvERtvJ2U7QiStv6UeS1vOucP1/om0Qj4smTXBWYsDglTSLx56/bghCsM21RNZZb
+yd8JE5CEdtCHduj+uRHbnEYsnGYM7R/Gw9XAuajFLw6BxqEtHi5xOivQ2P0Tm+Bc
+SVHW48iF8S3q1tx5QU7oIMZcCobOeHb6w+C1GHiSeJy3R8hWkEwfNxCCc0rSItKd
+edqO4YPz/zT2DWoUx+n0Og
+-> ssh-ed25519 /vwQcQ TLa0Xty2LlqBiP9Lk0lC+S/BoVT+VbRhY3qPHIGf20E
+3mzqkwT8dvP11GAVJiVIc+MiN/pLP2b6KbC+1F86tg8
+-> ssh-ed25519 0R97PA pbGz7e6nU4M4cpJRmmxWxUV3O2rWytIP18M7OpMpa04
+doSBv72rqS5gNusMjKw8KwzXHbzoLlFUSdLqp/f5aRM
+--- beE6zNg+kY7jke/79FGZoNTq7Wbe3eqNWvLD3igQJdg
+bÓÿ¾ ¾ÿÔáïØÉ
mÿŠ$Ÿ–‡¸’Ý={6
eÿ¨SIîumT8”äÿÁ{òo<C3B2>Ë3Ý&¾12¡p¶	œZ÷³4zx¡8B˜¤a@â(_Þ7(‚Ñ^tµä±Á‘<C381>g“Ùà4Ö*&Á`*
øäNšB÷$ÉÆ	ò£§m<C2A7>"ÓtD$Î5‡k
u˜eÎ+X@LJüç£h2ü£Þ*H¥ ÂåaÚ¢

secrets/floral/hydra-signing-priv.age

@@ -0,0 +1,20 @@
+age-encryption.org/v1
+-> ssh-ed25519 +uvEmw TNYFQxSUv5yMmlTWoIxCOlv6UR+RA50cb5aJbo0yEE0
+Yw3sTPqYf7A33RI87CqoPWe2gh0FuvdBGGKqHV55Atc
+-> ssh-ed25519 K3b7BA vMlHenY6jSIfnxQD6xh09cwwV+YVBkLuSMHcyKD+dCk
+heXkAEqRawBlHqcr6ldmhWmk7qPtGLMDFC3QT79vdMM
+-> ssh-ed25519 +qVung fgimLW5X0z4Eh2u3fIr5bgR5/c1SKam9CKW/2mqtTik
+8VKJJr+FRE0j5YvjfdMXugNA4UwUebKrkeAe+9LYBnQ
+-> ssh-rsa krWCLQ
+sa3fA4GglovY8H6jimpTvQPW/axun8WADPlIXzpX/Zeshkzem+pQoQqptzDlnmH8
+8AngqXgFYrmHgNNAylavgcrxbjNrtlJU24ldF1YIubz7VsU1678F27LCd9B0c2dn
+X+0CccH19lM8Q+zVI2Wrq9R83MEP/5uOOc+eXXnvNSGqfKgZ2OplG/HUllFS13j6
+uiQy5zwJJKkII7KUThcGteux7NONoLeUqRE8CW2uSeY9fXBWKgxeENKgiT7PEAAo
+nvwWa+GatEYf6eUz8Lph8lETorgP+7JS2VQRAkmhDbjQLTYzfFmiJGE/mzyobslf
+ZEq6Oj5UNgnzdWmK5ZYKPg
+-> ssh-ed25519 /vwQcQ 9EG/cydlzlLd6cFed7DzmwzubzJUXvD9mX3WKDyFD1s
+3Emj+tVZmnsC/YZdChvyaxeObbBsri347vZl0ff9kH4
+-> ssh-ed25519 0R97PA kcIYyWKxpJmjcrel+YodZQiR2zGPqfjzMyJXsz2XOzM
+SUlgGGs2BVRzTHT/ULNo1AiN5SY1BETFtJRY6LDr4JI
+--- l87sO6IuwSeCeQ8ktvYFI0xr4Utcl8KfpAV7WePc1y4
+÷ÚÖ
~÷J3¦Œ§Í‘1íÇè²ù<º?%×ý<0E>›Á‡ÉÒ\—\Ï7\»Ú¨åU-&W'd”{իɼ½u	"Û#Ž}¯õ…x–ìšz¥®Nj„!éfUqDG‘<é<7F>Ñca<63>‚'´+	¸Ï±]Pó»ö€D
Õ¸’´þ<C2B4>£¼­ŽÿÂçÌ\¦<>

secrets/floral/hydra-ssh-key-priv.age

@@ -0,0 +1,22 @@
+age-encryption.org/v1
+-> ssh-ed25519 +uvEmw BoYFUISKrlypCBQW/fA9UNRSnxQ93FkQupWUWAeOd3c
+wQH9gNk8TtjOgrwYwCuedPBbmftd7JhJk53ga2qo2iE
+-> ssh-ed25519 K3b7BA vyQIsvbNrHI0Mui5UdLz2mWcYvnTQBupWiAfEP5NfXk
+WeHg0PyfuaSJVzuiBPa1Tanj4NdqHvnZFWhXhIgbWc0
+-> ssh-ed25519 +qVung LneEmMhBqJxN0bgM7/Z+jJ8U8MJmCgE2EghBmDJ/aT4
+nd3B7afUNX1ZLCjHdoJ8+tabXmi38lQkLlhthYjIplI
+-> ssh-rsa krWCLQ
+krJRF4AvwfEFro4uiLIBB2RQTwO9COSyAqkjOi6jgLzJFMaU62EZrgfSYu24a6zs
+JyAHQ5k78uh3EhhbSzu6U65fComCbRAo+NiN2BJU7jb28y64suJbezJ+LE4P35CX
+biVgycaSc+OCrb1F6e6QOREi7+YjK3VrI4ZVCu93hSQNNRi/U5bbigQXt7NwcSIJ
+bY93sset4wg9Zwjk7tFg1bHiyOK3ZvYYQGlMjUxiWGl4Qch1fpL2CJNR32mZybps
+GZc7x69E8EhoHvdI0u1AXwS2raLhyRBPxFzu0r7nPlSi17TnLnU3Ux3BkVEDa6bh
+eVHqAB4dudNCC4wOY/ZE+Q
+-> ssh-ed25519 /vwQcQ scOp+aVA3TfY269EzQ6E4YX0uAu7qVVVIDmBvFGaYk8
+AYqW8+A7oxH/0m8OUReWxto1xWcnOnZOkX45ejuFJiM
+-> ssh-ed25519 0R97PA YqfHMAAiMcH/efJ0K5URDJkdLqlJIlQ+pSnuGUOJzjI
+v6BujlFcBF71SzvlZzA+tWku/A8bZzLkRfHtoCdbCO4
+--- PeORL3PTxYsxaY6GliSm1dRAH+hxf1n5LNeRYDq+poM
+I«­Ñ_÷l꺓CqyƹÚ@‹Ýç¾Wï;äù¹¨<ñÛ5ÅF¼·Ee7þ11=ø‰øQ<‚pmƒ¬ì¼^²<>òŸµÉÑÃÙƒ¨[†GÞ½÷$j†©<E280A0>±¨m’±‡I×ñí¶\«•ÉO§ëÇBdÜÛ< O›ª$ª„5£ãÒ¼âá‰õw˜REŽ3Y×ë!Ïd4ŒBFõ‡>ªŠÈ˜\EKËhæg‹ôÙ^f;ˆ1>tk ™‹£{ª»ÓMð³ D§š³87®\uÄ,íSá›ÕIinÀ:ø¢z"“Æ<1E>-XBñíÀ<C3AD>‰u<E280B0>jš
C5Õ­‘Ðï,Âg“*\]ÒO†îÇye5ïQîÄ!‰$þ.‹†+ù¦²¶zÀF¨ÑŠ^¨·SµµL<11>äÍB+óÓ¼ËR“Hxö6ÿåʉ1f<31>=jú8›¾!o¿@‚Rzrü5p´(#‰w|Gd­×¼O@>²0ã{$ËhE‹°Ä<C2B0>ÕûS
ê_¦æš^#›Oÿžò– 2c¹ŽŸ#G<>ž'º?e9yÂ)ô
ªÅ–Ì~ÑÓ¤
+Ì’tl#­#„»I³>o½Å³°)ˆóu
+†

secrets/floral/loki-environment.age

@@ -0,0 +1,20 @@
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>

secrets/floral/metrics-push-password.age

@@ -0,0 +1,68 @@
+age-encryption.org/v1
+-> ssh-ed25519 +HUDfA d5f2ESneC0wsoc9rwTjNfNXMBjCbjAQ7euthH2Buq1E
+5CynaQ8zhDRBvcmifhCsiDtllztCVAqs8rU36DOxgPw
+-> ssh-ed25519 +uvEmw EtYRis2LP0jv1W8mx8vFYNzkgi8OoqnA8cM2huS6NBk
+ll1csFIO+hVYk+I0uSVJmlDKj9aTWvf4kaYI5LJcm7w
+-> ssh-ed25519 DMaM1w ex4QJN8CG99J15i+yvqGEiEZn9OlGIC+cmLHL4u8ZEI
+VXnOv4CGK68q5t6hUV3oKAtxGZ+4FVbrmE1yMn16A0Q
+-> ssh-ed25519 sixKXw drXN6+q1y7L7ZU4chTfHfelu5GcTdff+i/UMFV0+3RQ
++8jmgnMh2OpQ3vhAuyQYWslfx7KO84a8KsCpoRD3Yl8
+-> ssh-ed25519 aHbF7w Af7NgjZ/Nvh5FHrX2VlF5riTIhJ+fdxTo6OR+8PcNwA
+ktKpm/HnOnw2Ym7xee3N1rneEX7+/xDhcp71N1NNHAA
+-> ssh-ed25519 87T2Ig 8mEUxJ/5NUvV+qQCDQH2Tm6Ryr5hf4xgsQlqXGf03Fw
+EavMcnsg/3EYBLQEBHX+0oTDKq5ZL4vj+mZntPM8UMU
+-> ssh-ed25519 Ao+7Wg UphWbatIaa+R1oZbfHazFhrawf0vax/3ZZS7YuX03Hs
+dwBbwoV0jpjiKr+nj+CRfUDgDl7ISpsCintVAzHnIFQ
+-> ssh-ed25519 wIR2ZA ZM58Nq7eJX9JVeYkoJf+mw8hxhYGoTx042ow1u3mJkw
+UtEaf7e4xsPO0ISlIF9LF+GcwTBqw4AXdMO4MASfgLQ
+-> ssh-ed25519 oGiV/Q G5KX/Eox+9md0yFRUZvGIsio2gWM17soHsL6H6zEX2g
+vI8jPjBAoFF0xhvRRLPzCMSiQOQ0fKuRb3CYVu3KUUo
+-> ssh-ed25519 gO3aog p9nZtjzoA0zJM+7Y6R16mpdub3dhu67yOYTUNKRytgI
+YL9vAp1+CK7jgmXkB47ufZMz+/swngkdUvEGR1zFZwc
+-> ssh-ed25519 N/+Clw 6LzFdtNsWewuJK2r97ZXJbRazvK3raN78UGanR/zWVU
+WT0y+sfDP3ffVwRcbYw51ArFR3OzXnoyi9IXwZZKEL8
+-> ssh-ed25519 CtkSZw CV0jQ5dIbgFtMxGK1X9b1qJOKmske8VgIPW5NW9mAwc
+clv7P3de61nZmXrvbOgL7Llw8ZqBMm2WFqgpznDwKv8
+-> ssh-ed25519 keg2lg 3Nk40ByQj8RThj4QDY2BdAkw55mXAJprXQRGjQqGvz0
+f8OFszJ8p90crFd+awEE12CNd7b22zgpH2XRKmH/Hf0
+-> ssh-ed25519 H885DA GDiJYH+gaC++TSqfoPDOTFcsCZRhEl0EeTeab7tgcWU
+kMILmwNMnMS7rgC3kKsAksu4Txn5owPU2y09h4aHKY8
+-> ssh-ed25519 Rq7K4Q VCNxGtCSCD2OYSWWwl0+yf189xV3QwRiwo80h4NPTkE
+hHkgYHLbISdl/RRdlInp9ub854M9ZKFSXpLgKW2YkmQ
+-> ssh-ed25519 vvyRpw XSCCrqEOtvzQRssI0U1DHirKoPgbOUKJxNKnioHhT2Y
+HGey1j0Kxae5Qs0aw6eqFziQGiRmNA+lEwbRdf5hhbM
+-> ssh-ed25519 aSEktQ mXY70Lgl76J4O5dPdDcIqmJ40EinigDuZrUghpGOq2I
+U2qeVFxGCYCEFWU+7vHc5Mu9EuzScowrjnwUyoqPj5U
+-> ssh-ed25519 cD6JxA at89poimBZyeeM8CQrxDxN0yCNDT2k04++py1fFycj8
+cQV/K5zc5x/oYnJ4N0MX3sTboT4G4ZNvVUVdHuJRzbA
+-> ssh-ed25519 ec6XRQ spJtb/xy4k4dmwKz8R2CPhC1WcuNV/rnDT978GkjHHk
+KrGEVGts/AhzbRNreqQ/CVanXL3l/9oMWxnpBLj23qU
+-> ssh-ed25519 1qYEfw KRkTYlvvnsCIExKQNmCyU7YxnGZsiI03kzecXNpLzUQ
+h2YagV7BzlsF7banzwXbOudTdlFzT7LC8PvtxAsX36U
+-> ssh-ed25519 2D+APA 4hdYlOnNIT9Q6tyKwXzy+u66Ti2EJopK43Sipebd0As
+tuesc9/QcEu4q9bTFJ5zJr0qvgLcmpn4at4cYtHrtbE
+-> ssh-ed25519 eTSU6g i1qT6PtepHXnoLCqDbhk86QG+SR9luQaw34a34gy5mw
+YE9VBAT5SLW2ECHRU+dMg9na6OQNVRVGuhY8vOdmE/Q
+-> ssh-ed25519 j2r2qQ TTTbSB/8UIDmmI3C9+u24PYZNfjl9jGADKHNWIwLfGE
+SNDforwii/GFp82TpyOcVIVrZWCe2QQKrjzPA6XA7Jc
+-> ssh-ed25519 CyxfgQ P5EiJ54v65Sz1gHuI0s170Z7c1WjcZLlb7NYigElfVs
+iYJUGpoE9LBIlv+O1navSSsy3EJ8tusXXX+/QAQvjNI
+-> ssh-ed25519 C/bBAQ hlBDpQRkcVCr3B6TCrBjxauhUX6Ndpm0s6x8W4hU6gM
+OFG3EuGJkSoEEXhbJ/Tp2DBdnBcs+hzxjNRdvcOSpQs
+-> ssh-ed25519 +qVung cGEGpO8NJfpj9ixAH9lhYkPKPDdQWryVxSOhMGQdnWM
++MycbIEab3P/AOS9i/YmPBDXB76hp3xUcWI4VMihV2w
+-> ssh-rsa krWCLQ
+Zv3dPYERlX1MaVaJTBDwIcjt1yLmu4Z7MovPgjGg01p+XsdBXeepTyOl+gRBwGgo
+AW5CIuaChYxtSNJ6nOgSaUpqzILycUF1xE1jROe3MIX2MZ4KGD1qoqcHbiCAng+a
+RqYrwAKnNea9FQMVfhYZBkRoYE6ne1R+0G6BoFM/okz24pAAFPBx+sMMhfTkt0uV
+kHVx0dgRw1pxa7Na98WH/7E0zp9VuBvVHGXfk1rfW/UQlbIO5RP3nldFoa6OmOWS
+JZ022UvjyC1re0KCurka4y+qmaiRKnTBmpIXxJFMwNCAQ8O8SeAQ3DHKHmXNMOIL
+ZVICtRRk0uX36AVU8DWDog
+-> ssh-ed25519 /vwQcQ kF8+hsA+0Msjd3q0SL52cae5RDqx4ls5kPKnc3UZyms
+Q33kIKJL3Vjxu7LQ5l4M3tlEuj+OW4uGh1x+JxthW8A
+-> ssh-ed25519 0R97PA gWBH71l6w9upTE0DwqOMSvWXc5VyJiKFAQLaSpWQ43E
+IrOrvzEa0bABw6UOpP8pM8WhuRNMaWJ2khljJIKwOS8
+-> ssh-ed25519 K3b7BA oS14iav9pSioLecMkOanJz89OJygLugvrnnTs5pKzz8
+akupMSiqXussXJyHwFm/f0imKALjowJVqd8/LFcC/58
+--- bCJXTEDaKg4FF2dQp2zFMU4Zm0zACekCuDvthcAyncM
+&­Ÿ
€Waïãà›BD R(¯¥Ñ”ufj<>úVÁ8nÆ>‚ß›øëæðZúâ{Idƒ„©,³*„%Ç“È‚z«

secrets/floral/mimir-webhook-url.age

@@ -0,0 +1,20 @@
+age-encryption.org/v1
+-> ssh-ed25519 j2r2qQ uYhcMpOER5j/SWUX1mNvkOU9Rumr0CgVBuGv9EHGpFY
+6kAgrwjgB7C1cMd410EpUegcxxGRcNOwCMJPXppepvE
+-> ssh-ed25519 K3b7BA 57GDNt5nwxgzCV5bnMPEPUeyZNG1U+zajCIjeoHjLAE
+rFCbfodjXHZ0aVLtW6xtoh6e/VH/HwFdFzjnQ2QEEXQ
+-> ssh-ed25519 +qVung DnLKAJPnUDpZ2+wXDZWpxwZkvv8oDyu3xxObTMT9W1I
+vh59DYoQLpiro5eBjwgNH2YHRsGY/i6TB7zPfQicOEU
+-> ssh-rsa krWCLQ
+ekvGooB5sCmAniHU7hlk+iCkYMQ7Rw2SJx8tp4FnpfAWJbRMH8CpTFYFiDvlHfFy
+Ce1OpkNkkipzBge0OCrfn6Y5iVz2CZHYHf8Ul5ueHwmb5fS7seT3yMoWhhSw/zE/
+G3snrBORT9S9+KTRnVnKiy+O3CaMZY+q41RR35Fs3mmVc/of2ILc/Jj3a3t+uBTX
+axkOMU6z6R6i3Ps5SbwJTaB9q2kMPvZFOO9Nmku1wohjetz64wvm+fDx0XVRPe4A
+jDQRPKAMIZK68SYHk/9azmlBtJSJnvxcxyj3IaU9MBskUCldWi8CQ9jQ+1XAIuHX
+0Etcsx7MhzBpuhx2xZ+dyg
+-> ssh-ed25519 /vwQcQ uW41w2RAtfMaOm1wJktMcbVporqKgdGA5SY03OcPmlM
+WgL8DWPU735Ysowq0HtvbrT6Tc3XEpwws3AycqpBgtM
+-> ssh-ed25519 0R97PA 59AFQx8ngDwQUdmfOeOFUARQQqaAdLA5WH67Wsld4yM
+o6jSWtlidZssWsJsI8xAaASi8p1sirLJFJwizzPXIBM
+--- scUnldbU89ICZYlniDbGEqeUF7QUoO1kcZLl8abyttk
+öR{p@IµþOlKKõŒ§!
<01>œWÎÅœ[R<-A‚ÐbÔ<¯·÷0õÐu¹øµñU’gBÏF~µ«=ÊõeQò}î4Ø:ô²¢5ƯŠtaØ™û”<C3BB>æ·<C3A6>§°x±M
ë?Ew0<77>8.

secrets/floral/newsletter-secrets.age

@@ -0,0 +1,21 @@
+age-encryption.org/v1
+-> ssh-ed25519 CyxfgQ fWH/o2+Uf0i/JFIuVjCnkhDIfYndtL8EeDcxSxhKVH8
+ShSPmdwnxzDuUe/kCx8e61JJAoHMwguNydn+5OIGuAg
+-> ssh-ed25519 K3b7BA p+wXAGvPqTX63dlZNCTIq3F4QFMWEJH1R+Ex4SJ5UTk
+1sFqFqnUM8YvZy7BEBArg3eLxCCsLXq2jNI7XLKq/Ww
+-> ssh-ed25519 +qVung rcpgzVQ1PmoNF2i0K0nAknzZwPXICBggzqhIZwO+8xY
+9rjsTwLm5u1GOJmnJYriXXAY1unG7y+WJ4G2ltxX34U
+-> ssh-rsa krWCLQ
+seXsQjs62kxn/agyKda2l19PI4xzDl1gM7rEnaEBV8UNLOPNxh41HTnP2etgDXSc
+4eyS3ntHXIOHmN4+JBn+Q/wuhzMGQmAcoFWbjqVVPOrpPYjgCG7q/iUD8kULxLB9
+UpF0gLsg1TnvrkTwlpxr8rP/PM+ZgyQAA84S96j9TW0coyTUoH/ZX1wWGtS4aalm
+aTrOMZGScZu7onTg+tYvR+aBKlFL28h08I5nqbA39srnCNuU68+OUhLgLUfiTscl
+umwNh/C4BP2Tmc6gxQiY8o3tGqGBssGH5+WqKzbK151vJjq80RKAS1HCaSSfmxkP
+vWkXWN3NQkJyqCBpuPYilg
+-> ssh-ed25519 /vwQcQ eUH0B+cCoUubIKbG+bA25kRj0TnZabB6t8jVK04NrFs
+ovkI0C4W5CJXMZIZdpaTtQNc+TGkQ3Yq87Dei3BMUsA
+-> ssh-ed25519 0R97PA u/I45pxH3Bnja/Jw/6IukINRuC0e1IKu8UVygVgIomc
+xyHuiHf1/nJirnhXbGHJnextGQa95tDo/RPRRnDCkIg
+--- LGqO4Bsa8bofD1W5YrQp75SlGLNg1XaFZ0rPUuvLPTo
+Êçã ‹ÜmlW£{@I3…*¹ŒÇ™@ÞªL7Wª
¤ÝŒY
+n
õö~Tb\V‘•ÜvPÙpPô
ïoÌS"ôm/Ûµ/bÝp’Äžêq¸£¦šeDj6–ÆþTì)

secrets/floral/stateless-uptime-kuma-password.age

@@ -0,0 +1,20 @@
+age-encryption.org/v1
+-> ssh-ed25519 CyxfgQ D2o8bUccO13DKF4COLBQ9mJbACsE2XsRa5S+N71WnTk
+ZaldT7HhQxbxf2ptIwdMYkC60eGtzihc7uwcAkq7s00
+-> ssh-ed25519 K3b7BA AiUCG5CnNyv1DPu+iEwEgW9GqZ8zgpgxKJTAp350ADc
+cUVaDv7F1haQIF11/UhhDAR5DrfJlPttGfDjkv+z9vY
+-> ssh-ed25519 +qVung 1JXeXyea+2Pcwoln/NLRiR8IPPIiB3gaFCP4imyv4DA
+JWmAY6ZnyU46KxzhRrQigGmUPba9lJDDyRQ2GjQShqc
+-> ssh-rsa krWCLQ
+ciLu/+cXfQrB1ms8oTv+xi4eADyL4j0qwnY/6TE0wAXkQHuNXDmpF6ccWZoS2DqN
+NcnGXL6+WyWxmwlyBEq/rsBPvi1g0M6Md7Z4gXn2UvjJ+S7WyA8QEwkxoTDkJS7x
+k/NvtunmggVsWVK4Xdi5DKRw+f32qr/8GysDhIPrTt43iReBKNbyuYWmC5Ec85ep
+JU4JzCNZjJ07kixS5Y9BhaJbpEr47lCXE/KtJUvm3VAxS9IwfUn7KHHdFWynbExi
+F898j3zOR/kgYmeA0oTiexRD3Y2LCvjXIHQZ3MobbZ/PBrjWxe78Sw2vy2t5JLtB
+gFG0K8M1z8DT6a8TtvXEgg
+-> ssh-ed25519 /vwQcQ kUM21TO9iSa8oVXMlNxR7Kc+8TV4C/uTzyQ+t3xnARA
+oXt+egWWONsKT48H4vZ2CPdy3Zfb2QeQVe9l7dDyO/w
+-> ssh-ed25519 0R97PA e/piqf2RD5QgPaQs6jsJdzJgfZR9n1JDIWpbvLZErSs
+UTJH8POFdZ4+N9WkLoNESl1pvcVD0MS1qn7AdS/mg34
+--- 9aYEP0eHDKMacIf09h+OJqIYw+N99+FrW/x/do8Lbo4
+$ ÖëWÛ\zú—¾=s/à@.Ç,?ƒW6n^ù#–i!§Ã–ï¶1]±Nvù±Ž'Ï¥¹6?‚'mµpPÒqýŸº

secrets/floral/tempo-environment.age

@@ -0,0 +1,20 @@
+age-encryption.org/v1
+-> ssh-ed25519 j2r2qQ kbi4mciOrjd7/X86xfmkDaMZhvZakoSJ6qjqLF3ljkE
+Q2BsgMLJ8AmjhnggRi+wkICj18NCA2HW1t8clemReUw
+-> ssh-ed25519 K3b7BA wNGmX9S9bJgd2JDte9QoNDfyycgmq4JMu2bc5nyYYik
+uUiutxAI3nI0M51W97aPRVE/l4dV2PEjph8eWOMLHIE
+-> ssh-ed25519 +qVung raYJ5vwMP9JopSdfa+ofkLY/gc0zcW4wTNBFTca+MXw
+sa/rWGSYrI4y6rn4JSboldWKUGvx6HbtsYo78AFOkBo
+-> ssh-rsa krWCLQ
+FLq8NwkiGw2gXptVVY393f0p9hFom57xHWPxtAlzOcRT8gvWu/uwgV+0raOcOcJa
+xxr5Sib+2D3UnUhprVPmH5Os9bI2seFAiej1MVVWLqvMtQHLFwnrzZTyZpxsXpQq
+5qQhNEADuQc4uD/ELVjGHKt6nF1Cl/GbgNLIOF/ITZ0pm1O1MjtT6MYJhQJhc6sb
+sno/wQyTXjj7rC06nyLX/rgOWrJSOeaz9eVp0A8k8/I0TXu/vRCW9gqWtv2m8sbh
+1uUHIm0l8f3z+zrL6OlZnpMFw4jpiiGoCYKPzD17I0onDYIjtdVS5iO9BsckxV/a
+wQWbyONUwbGCfeNSVAzZbg
+-> ssh-ed25519 /vwQcQ jwf7fwy4wKz7q761DNu8SyFHGgFlwq4P/Pn44Nido3E
+1q/jvt/vtD4ziY3eCDqk1XwMPpNUd80POTV2VVsumCE
+-> ssh-ed25519 0R97PA XeuziQ+wsoh0KSHXk5Qkl1kQOsAu1Ax1zTg13+XWd3M
+B1KHKm3tx/EsnE6hY+w7ya1ilhYiUs9AbwARHNkJi90
+--- JgQA6gCYZu8xcbXEl9VypccEIBO6uAJIdhBefr4doRQ
+V3ZðõÚ<EFBFBD>ç-·Ý.ê«sòÀ³3 ÎiS‰a5#¿Ð{åÔÈ®Dý˜YêNèãëù«ýoL+ÔÝ#–M<sws P»¢+í¢Ó‰ïBDoÊξÆÏuFí”Ç^Â¥•<C2A5>—ÝG@ÍM×ÛãÐØìq¦ºG^Qb s<
;ÂÒnC+ÖÊxª_­Úì]S<16>Ð

secrets/grafana-oauth-secret.age

@@ -1,20 +0,0 @@
-age-encryption.org/v1
--> ssh-ed25519 j2r2qQ qI/dlkHZYcNkCVgZbxpw5Ps2anl8pofaFPi4p6kOHAo
-KWL+H9at/p/AfCjfO8+SgMhn97F+DqLO2ymYUOHkWjQ
--> ssh-ed25519 K3b7BA URYQ0jFY5yHS+dodR1RqodNWrrXkMnzTp5OCSv1gbWI
-bnyrPvWnzDRNh4mI5HBPkNl3NSZE1ycMK3LLExMEYbo
--> ssh-ed25519 +qVung z8e56tCZ4TLkrX7BfH+5RrGxGoT3q9V1FB/ySsH3tg4
-jIpEEVF8jCp/ks5eYXh3O7+TLidvzYsnBRFd3LkgLXw
--> ssh-rsa krWCLQ
-XG8KKBT/hEvB+c1RDGUrDR4HrfAertfOIzQTquMQ+Z3Nde3Ybxf8W+rWGQDErbq4
-VlvC/wVVnGnqgE/tJMQP41sCMKSH61MPyiNZC63g4RW9e2H9YQfWWrnuBh668G+3
-3sE0FSdIAB+UlI2jlbMiG60QaT6zV0XyOrugLX/G2R+D4aXYIVvMtcwYq2oIHy58
-1DE5llUZHGsQ8APXZle7ZGyO48ELOQkVn8ozPlPFhvz2y9srgBZvNL/wadjvLstv
-2vBTBoRk8HnTLOiybAnGtOfK6kWUMdfSYMvhu0IM8UBSoxwxOHTfIttKDu2ZMB8g
-c/RnKbV2z0PBdXVrYuijPg
--> ssh-ed25519 /vwQcQ qinzScNz0IFoHUaCeGXne6ddllQ0dA/TJr5Z/nbfvTQ
-0YpTZ2Z2WwN0sJ1CIV8voPS298u9uHbRQMlV0GMrvFI
--> ssh-ed25519 0R97PA en5iGTQoH0/QJKl38HNe4xun/FxVBIun7Z23mBW+4XE
-Sjshx8hLyP4iY40y/Fehc0wZTBH0d1Lu+auX8L5n28s
---- i5+vCeWbFTRR2YbIX4lwbEORRhaI5NkCwqaMEJqrPEs
-ÿ\ìƒF·Ri±ñXa,.øÝoªâr›çhE0=$Ç‚uGa/oÑÑÆÂiíf¥•x¦Óš?Ðg¹CiÉ

secrets/lix/buildbot-oauth-secret.age

@@ -0,0 +1,7 @@
+age-encryption.org/v1
+-> ssh-ed25519 Ao+7Wg q7oRHUUlAvD8OUbpPT7d6eLMPWU0YS/verYTDE5BCkY
+/87/1uqOvRYeqel9IjnFmGyF9SXUQD8MTgfcj91b/Fs
+--- ulIeB91NJ7z/64h9BCLSD9/RW/zwv3m1Zo2ovNuInv8
+Îœç}³Óš#épÇ o>ä·*vµ÷ÄåŽs?[¦º´L
+<EFBFBD>þz™rý‰?R±Ñó7<Ê
+æi!€{X„¾òÓ

secrets/lix/buildbot-signing-key.age

@@ -0,0 +1,6 @@
+age-encryption.org/v1
+-> ssh-ed25519 Ao+7Wg EMpfs0EpWwaIKAoUBfEkyAHLIwi6JnGG6RvUWM5LjnU
+LKiwUBNc791U/GVRNlRPZE/TEMJjcFFrLruFJhiyiOI
+--- 0khp8u+4vHgGyQqP05m473Eo09eyOUZLI5+EK4olzoc
+N3(
+ª•û
xRq°<71>f<EFBFBD>Ó;ͼ3¬~RˆÓ
C^ñ+fœš1”®˜xˆ÷ÅëñSØ—hâ
£ÖË°GˆÓn–YIûµ:7¾!°u×Hþy/‰Øð‰™.¯¤á^¹lC™ôUÈËþ5cž:]ÿNž&'MÎè¶É-˜–ÆHF¦D0‘ cjô ‹Ð~

secrets/lix/buildbot-workers.age

@@ -0,0 +1,6 @@
+age-encryption.org/v1
+-> ssh-ed25519 Ao+7Wg RPKKoI5l5cYVdSvOxTHCUtwceac4vSi3+vlaqHr8kQg
+qbgTHCeQDNM30IJNZ/BU6wgryJwB316H5GWWaYB/wng
+--- GuFi3GSRdlBJ5YRjfAVyFDZ+4TH575kFJLnFp5847N0
+-èƒÞHÖÜ*x´M7¼t<,4ˆŠÑ^<5E>5@v°<>£€º,Z•MÒg=M
+» 3výJÄ«ÐÖê¿Nz­8'<^'4&WÂf"Êõ´À›ë\©º»ëêwmzúlAl|+„‘ÆKš~68ñE­Ýîk•8ø?S&òaM‹Ý~ž¹ê¿]Vfø ÝJxaõDù¥x

secrets/loki-environment.age

@@ -1,20 +0,0 @@
-age-encryption.org/v1
--> ssh-ed25519 j2r2qQ JzVKQt25f18L96aJWsJtFAR4mvMVCgYMKu/xtJ1BeDw
-vj+HpNQCNNxDRA+7HgjiD0XlGG/Yy+tk8KmszMkxdag
--> ssh-ed25519 K3b7BA judlH57lGOGmaTEG19gYiORJT9uXiAlxZrP+ISTHDT4
-MS7e24A6rEMUtUUl8DlYXPy9NhqAq4buOWT0iYKvbSY
--> ssh-ed25519 +qVung vglRR5LYFZw8v6zRhybGPBctwDgYoskbpGYiLNW9qxM
-VdjQTykQSVWubGimCHiekQX7EQdgOB3PYsRHiFnpPkg
--> ssh-rsa krWCLQ
-hLYT6U+dUVuicVO8hSw4KcfkM9bay4JR3TEWGlmmIxcQ67LNggzuyRvV6U2yfucg
-Xyxezdd9LArf8z1eV/y3iwsY0PvK9qwtgpgH/NxaF7djhTA8+c3c3a6w4sqdHn0m
-/RZU+eKSFeDWII7fn6o7JxzITFhF1FYH6PJYA2cb3PvbPw/JSja8EVZ7192ShqGW
-22TThbZmmKoOPbmDxmQIygZTxqyaXkoFOnTWqqTzOfNtBOBFXT+cIFh3ctGWLw79
-u7O5c2dmpXoE0bdndQ7GUSPrgRzOYHQ5hLg8WtC56EYjE11Bxj88fktzw4hZTbYQ
-jrS8Pa68UPhUmSfutlpd4A
--> ssh-ed25519 /vwQcQ MqdVxRlS+EMA3f6B0D6m2ylvCE7WVq1av/CvsNVAB24
-KX8RJ1bzUUhsYW6qN06FTzis5i13IIoIpUb5FkW9wkw
--> ssh-ed25519 0R97PA RHUvc9XQIxOW0GCyt0vRxPHyVXlpqM9gaUps4q/Grx8
-bxgFxtbtbvDi9knzasdR7u33Mb7x7LcBzqEB/g4Oc4A
---- Z175YCdbPBBSItxomyXPSo6xILLV4GT4gpA4Oxz9qgo
-EìVÀõ±ž™êÞ<EFBFBD>Ú¾¾Ó¦xYÊqšÑ84™6¦¯&Ö‘ï<13>·”ž„Ý!óZmëû°¤Ãd.à™46ÅÈ·ØËòø/<2F>´<EFBFBD>=°ß܈'hM³_ü£j
>ªÑ6ãR<>&Ú
·u²þŸøEùÜ^8c;×Ä›¶:Q1Ü)ú1L¹_~,<2C>K¥ÞÃîôµB¤7–

secrets/metrics-push-htpasswd.age

@@ -1,20 +0,0 @@
-age-encryption.org/v1
--> ssh-ed25519 j2r2qQ n1lfxDP73nfF/CYtE4gpUH6YgjAQbx/2TTuyfFUBiHQ
-LGzudpjsYA92pM0UpUT9CWZD+e+rzGFP4ndxPE0MByo
--> ssh-ed25519 K3b7BA NRnnKaOtdtIjkRdam5vAA9Yj1RUJRReugWKRglWAoQ4
-Xprx5TSU1rNH7NMl0X07K1KexCVXMEu7BFxbiPwxvBY
--> ssh-ed25519 +qVung qZsGi4JqgpHrjlg2VdY+OhXb0BzYTytBBqY3jNsrSgU
-GgvQG5iMd6XTZRCC3EBBvqF7nhkqAJmxdIkCFRV46Ok
--> ssh-rsa krWCLQ
-EkmY8uc79xWfKjlIozS4Yigorz9IdK8T8VjMnVcJN6+rhoRctQNVCj4JgogY4wa0
-V3ObjoRPZgVU3qPmkPgIKVa2Mvf6MrCMwvvE4j2Yyy6lmQEwFdvk4s2c6AD6T8Bf
-rktRYqOcFavuDr348e0ZzKniFTRcPMcY49mqBR/mWIfSEtLxBgpFUCn6f40PLndT
-3dse7kgRBlrKbzmf6JIsITHejqwDRq2bZqHWAmZhb6+ske7oDicAt90FDoDbrwvd
-YwXPRDCxgATlNz8n/xFUxd35X+zEftUUtANSGtihIE4LcdsO7IOwv/FCjdEn/3YW
-ZtQjphnxgDsY61PEFCMnYg
--> ssh-ed25519 /vwQcQ DKQuo5jVunUFTCbOxVV57Xl6q+DDOVDWXdon/lZlLi0
-doN6en8IK4Ju0uATp+IZAhYl1tvdnfyxHziSobb1ER4
--> ssh-ed25519 0R97PA I1GECXSPagJ5kD7CeVA21TQmpMEgLeaiB7XYEomUl2U
-d0kO+4SkAPC/ois39SZafEhTqvmDpCZbWTUU1aUZ47o
---- 555iE+C2kDLIdAJ5KARyKcBQZSDRWASuzcNiKZ9IbRI
-òeÕceV&˜ßà‰g˜óáÔÄýæ›6•=6!õC<C3B5>Cˆû^»âÕèí€zÕ§®(Ó<>!ÄB•B|ô<>ï°Ú'¿Rªîž†_a	UtI³3

secrets/metrics-push-password.age

@@ -1,52 +0,0 @@
-age-encryption.org/v1
--> ssh-ed25519 +HUDfA FOqd+I9DzoloOMK2InPz8yAGsk+ZgMKy0n542DmF5ig
-sui4rdOQcvjL6H9rPSbSAyIggaSbsIVrontrkFpPPC0
--> ssh-ed25519 wIR2ZA V4KPrGw2NKeOBWpjsRbhUJ/eLR8/hvExNMpcBvC7gCY
-Zjc+HtALqZbp+L8tUUgaFe9LR4NKptpFq/L7xhTItXM
--> ssh-ed25519 oGiV/Q kJS4DAPBTOgADY7LCZnIfORMM1RJez/5XGoKDfErHjM
-LN3XE7qM2SHqQwb+JjIq5tMvt77NI4+YOxYnZh82udA
--> ssh-ed25519 gO3aog gJFIrngWZp4ypA2IZwr+c0JkWgUu9VN5AzoyyhozlDE
-lezfokY1lgABSKNO+Fr+tTlIjC3gzc4Bw2YlGLy+WvI
--> ssh-ed25519 r/iJSw VzO6pblztwci/TMfha+dOc6Vg4DC/1oSNEt0aFaCYRE
-Mf0LjSjWJA2lMt1M1z+tGJ+9NVMxd8J5CSMvaLK8zB4
--> ssh-ed25519 N/+Clw uNBuYGWU+LLY856o15jLkJNk6pu42FnX55CoE98/ukA
-zh+sZ0nskVPUKd3Ajg1FHng7caKhkEHiRFcm8c53siw
--> ssh-ed25519 CtkSZw YP79uyNelg7+nbeois1vu64anUC0lhUhIie6EqUz2i0
-rb9zte3dN0+uwjyJLGaUfeEQcVtMerKEOVAocLGXUYs
--> ssh-ed25519 keg2lg +g5uYkOOyQABVmL+9t08aaMklNEbBO2j6vqKyrwYrhA
-U4FzATeou9spmYchqHPR/WR79Y+ILWpwhLwxjYQd7d4
--> ssh-ed25519 H885DA tAx+W9kfJkvERw9KPKZInC0s44QqQIu71MPUosasHy4
-5ks2qkZfkMLK4meVHTfWpR8qCeU3vKdPiWVRTyD6OhI
--> ssh-ed25519 Rq7K4Q xwSlrqIh+rZFv6w1iDcPyD0nEmESlmHleUHsVPrG2Bg
-OgrWCBqb7SAtQQSUnTQ1l9JRyDGS2DgzKRRbMCtKK7g
--> ssh-ed25519 vvyRpw wQB8wg6bGvb68pvEp+7khrNpZTUxSVzLIfubbYsX+34
-KZ2/Vnxg7Gpazc26lYddjNnMxpoteb5ysuTZUg00ZvE
--> ssh-ed25519 aSEktQ KdKSZuVH/v+gkZkL07YdUJ5vvH2+mcUR4x+mXHylhys
-MRGd8l+0X6XVq1KpLqYqUZD/4EkOKz3mpHsdQepc6kc
--> ssh-ed25519 cD6JxA FesXIZs/X+fWefYjP0sfkwz6bYLxOkuIzQppwZYXNTU
-hg+ZTdCGuQ66FIc+NZI023Aunnhz+Ds5cFKUwNj+MGU
--> ssh-ed25519 1qYEfw HRQdZ4u1UWpzwIF/0lbJ1NVDQ+/Rl913jk+BwLM0KCE
-CHlDCaov7TWme5YMBiV6Tby0IReB8pER/RbDkpI3TWM
--> ssh-ed25519 2D+APA BTVVWo3G0tZj/hUMH5cwByYf3LjAg2RNVMhYrkXxXjQ
-iKghO+M6xpp95xVrmydz9GJJIOK5JrIsoL+CSFD77uM
--> ssh-ed25519 j2r2qQ RC/2vV5yr1af4iyeouQwIBK/r8b4nD51WwxgbuMEgG0
-L+uqV7eeCNqnMTqCNmvLPZFNTdmlYu/i7+3NVwmpIxA
--> ssh-ed25519 C/bBAQ KO1owoeb7pbuXtDS+f/TziotgffL0Eg6qnjJ9W8Yp2c
-af4IhSiXlMPiNuM473dIeWQqNbRgb3ciHyoa6buolyU
--> ssh-ed25519 K3b7BA h4mC/hZ10ToaaYDRyBOyPpcvA28sY5FPCQPuaTTRIws
-VG4QtmEOnubhhjV3CS49aYOyVl/Dq+ryxfZENgFJZTo
--> ssh-ed25519 +qVung 6gs9DdduYx2twVsFED7HJnGFfKZynUctQIO4F3MXfj8
-gMmU2tXwR9K8Nb5gMKPbTexE58FOAK6QlVYzGvaX3hw
--> ssh-rsa krWCLQ
-vjNcmgDmmaNUSXIUgKf1digOgbohvyKkYSUalTOskvPo+9NRZbp0IJ7DoYLRrSBB
-DobCBM078iKOvIGGJCIbMS86/z/7lz6SSPcbfM1EG+hknVJLZaj+K3PYYSX6QTUC
-6rWSC+yg0gKehAhnYO3q+8mnismk7SERdyCZDNtPwHOhTAt6NZ6e+33VFxnbJPTz
-IvoNU/RTUhV+XuKbtosm55PqDkOuTM27jesZ0/SARYL+gVgaltacqt4kzbEMOP/W
-tv2kU6f1eNaX71c57DGI7rfcvLrPRAjTxUhsuKJPGQeaHtfiWz832gUMIJOEjoo0
-mvrAfyoykJRbPGNFl5pMmg
--> ssh-ed25519 /vwQcQ gpPktkJ57USbj7kn1qbeUQDbHHSCuzWM5OcmNooBMi8
-6JPXUJYQ1IjRVv90r1EJx3EUMDPmU9X1FK6j/6vT5hE
--> ssh-ed25519 0R97PA vzT774La7rcOMz7/KYjSUsY+D6V5bi5j3ghdDBLBoAU
-HAXfMmFuj3YJGCBR1U0btPlr9MdIBYnwT1ufbHaAxVk
---- /0DCLjy0dwjRGPnkNk/a9fZ1ox9+LVkwh9Y5jiyA8x4
-·1ë³KëB³|†Ü\<5C>öST¾¦i¸ð¦/<2F>ØÕ
hž9}%ä\÷Ÿþ,"g<>ì°Z³<5A>ÊšwÍþ0»ä5¸´
Rm’'

secrets/netbox-environment.age

@@ -1,21 +0,0 @@
-age-encryption.org/v1
--> ssh-ed25519 j2r2qQ 8qMRxnJL5p7M7Egtim/MZQTx0Z55dK1VKbR1drFkMRk
-q7AWFD4wg4eEIoPzPY3gmPNt9vSPv9s1TII2R0a4QoA
--> ssh-ed25519 K3b7BA tZtpUP6oDvY28vaLwzlLwlv/QQaDmbuwdPRvs2j3yxM
-gnJPbdLJML4MldoJTwsR/93ioOId53IvuSnpQwqmYoY
--> ssh-ed25519 +qVung fYyGsDgnf2wO4NZ+zOeiWWu3wLe001xHgZatXvVd60w
-Kxs9u1EZbP/abuBev0u9f8keraKibvoVDHqYvvbZJOA
--> ssh-rsa krWCLQ
-dGijkGbcpWqNgrsYSXGEYgLJadgf2imVRDZpMpR2SNqKeBgvIRSriwQQSUCnntZB
-pwul5dzZ9okr16xrghK66tCizBWwvfHtyACAFcI0xyCEf20Ydm1pbarSibK9RDb3
-JwJdvUor370sTkuWagBzM3+cfpeO8HhxEu46tNG1RP2EtEkdSXQ8056g7TrSUQt/
-XI385S5/WuurmBVlZuVTBXVsvGYU4OBAIlrYiym4loaSOGJMUCK8MZMfg+3w+RXW
-fScsZ0VS1eB4DkAiJEptJlesrpHPOegq+HyczxGAp0z7mcO0ffZBOrKzBQB7fsdV
-sn0R1gKpx9y9T6VE/uJ4Tg
--> ssh-ed25519 /vwQcQ mUHdSEXaTCrk2Nq/OPoo/3i8jXZfLbUBZewg6rwvdGQ
-7wxUPlQqkZXNo6zhqd/niQDUZWrKVzgWWkUPcW/ueds
--> ssh-ed25519 0R97PA nuM2B10VwPti+CBybZzBGLzo7SM9lHgKAM1CZj4U8iY
-3tfc6NC/D+lPPk5Fk6tDWbc15m4Eo/sI4WGTC33zQAc
---- efqcGqHksXsmGOFOwC+0UcYtUk+FuiGt4PHkHFzQ4OQ
-ˆ 3᫬ÒÊSâj´;…Dæ©æÄ6¾±R˜LݘƒmÄ+þÃc<07>˜ÞU¶WÅÐSü<53>yï!yÁËlwÅ ¼˜u3ÏõBb‰nloò	q
êÏTÑÙñ&é•.ªpš4‰¢‰¸!ó<>ËÎÔÞ€ÏÙ‚Ãò’ˆCì‚F¸¤$<13>§‰&
-<Ix¹LWy3çÔñqvbspêÐŽX²LÙob‡,-ÜJÌñ¯G’Ò³À6çéó“°û1¼s_rÆþgŒ7%×]÷»Œqž‡×›AôV¤*ï`«‹ê\Æ̽tLpöi®Þf

services/baremetal-builder/assignments.nix

@@ -0,0 +1,29 @@
+# This file contains information on which builder(s) are providing how many
+# job slots and providing which nix features
+let
+  genBuilders = { offset ? 0, count, f }: builtins.genList (x: rec { name = "builder-${toString (offset + x)}"; value = f name; }) count;
+in builtins.listToAttrs (
+  # The first 8 builders are general purpose hydra builders
+  genBuilders { count = 8; f = name: {
+    cores = 8;
+    max-jobs = 8;
+    supported-features = [ "kvm" "nixos-test" ];
+    required-features = [ ];
+  }; }
+  ++
+  # The last 2 builders are exclusively for big-parallel
+  genBuilders { offset = 8; count = 2; f = name: {
+    cores = 20;
+    max-jobs = 1;
+    supported-features = [ "kvm" "nixos-test" "big-parallel" ];
+    required-features = [ "big-parallel" ];
+  }; }
+  ++
+  # These are not currently used for hydra
+  genBuilders { offset = 10; count = 1; f = name: {
+    cores = 8;
+    max-jobs = 8;
+    supported-features = [ "kvm" "nixos-test" "big-parallel" ];
+    required-features = [ ];
+  }; }
+)

services/baremetal-builder/default.nix

@@ -3,10 +3,13 @@ let
   cfg = config.bagel.baremetal.builders;
 in
 {
+  imports = [ ./netboot.nix ];
+
   options = {
 
     bagel.baremetal.builders = {
       enable = lib.mkEnableOption "baremetal bagel oven";
+      netboot = lib.mkEnableOption "netboot";
       num = lib.mkOption {
         type = lib.types.int;
       };
@@ -28,8 +31,22 @@ in
         "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAvUT9YBig9LQPHgypIBHQuC32XqDKxlFZ2CfgDi0ZKx"
       ];
     };
-    nix.settings.trusted-users = [ "builder" ];
 
+    users.users.buildbot = {
+      isSystemUser = true;
+      group = "nogroup";
+      home = "/var/empty";
+      shell = "/bin/sh";
+      openssh.authorizedKeys.keys = [
+        # Do not hardcode Buildbot's public key, selectively
+        # add the keys of the coordinators that require us.
+        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGMnOLLX0vGTZbSJrUmF9ZFXt/NIId/MUrEpXmL2vxod"
+      ];
+    };
+    nix.settings = {
+      trusted-users = [ "builder" "buildbot" ];
+      inherit ((import ./assignments.nix).${config.networking.hostName}) max-jobs cores;
+    };
 
     nixpkgs.hostPlatform = "x86_64-linux";
     hardware.cpu.intel.updateMicrocode = true;
@@ -40,18 +57,36 @@ in
 
     boot.initrd.services.lvm.enable = true;
 
-    fileSystems."/" = {
-      device = "/dev/disk/by-label/root";
-      fsType = "xfs";
-    };
+    boot.kernel.sysctl."fs.xfs.xfssyncd_centisecs" = "12000";
+    fileSystems = lib.mkMerge [
+      (lib.mkIf (!cfg.netboot) {
+        "/" = {
+          device = "/dev/disk/by-label/root";
+          fsType = "xfs";
+        };
 
-    fileSystems."/boot" = {
-      device = "/dev/disk/by-label/BOOT";
-      fsType = "vfat";
-      options = [ "fmask=0022" "dmask=0022" ];
-    };
+        "/boot" = {
+          device = "/dev/disk/by-label/BOOT";
+          fsType = "vfat";
+          options = [ "fmask=0022" "dmask=0022" ];
+        };
+      })
+      {
+        "/mnt" = {
+          device = "/dev/disk/by-label/hydra";
+          fsType = "xfs";
+          options = ["logbsize=256k"];
+        };
 
-    swapDevices = [
+        # We want the tmp filesystem on the same filesystem as the hydra store, so that builds can use reflinks
+        "/tmp" = {
+          device = "/mnt/tmp";
+          options = [ "bind" ];
+        };
+      }
+    ];
+
+    swapDevices = lib.optionals (!cfg.netboot) [
       {
         device = "/swapfile";
         size = 50 * 1024;  # 50GiB
@@ -64,8 +99,8 @@ in
     };
 
     boot.kernelParams = [
-      "console=ttyS0,115200"
       "console=tty1"
+      "console=ttyS0,115200"
     ];
 
     networking.useNetworkd = true;
@@ -123,6 +158,27 @@ in
       MaxStartups = "500:30:1000";
     };
 
+    systemd.services.hydra-gc = {
+      wantedBy = [ "multi-user.target" ];
+      description = "Nix Garbage Collector";
+      script = ''
+        while : ; do
+          percent_filled=$(($(stat -f --format="100-(100*%a/%b)" /mnt)))
+          if [ "$percent_filled" -gt "85" ]; then
+            ${config.nix.package.out}/bin/nix-store --gc --max-freed 80G --store /mnt
+          else
+            break
+          fi
+        done
+      '';
+      serviceConfig.Type = "oneshot";
+      serviceConfig.User = "builder";
+    };
+    systemd.timers.hydra-gc = {
+      timerConfig.OnUnitInactiveSec = "10min";
+      wantedBy = [ "timers.target" ];
+    };
+    systemd.timers.hydra-gc.timerConfig.Persistent = true;
 
     bagel.sysadmin.enable = true;
 

services/baremetal-builder/netboot.nix

@@ -0,0 +1,169 @@
+{ modulesPath, pkgs, lib, config, extendModules, ... }@node:
+let
+  cfg = config.bagel.baremetal.builders;
+in
+{
+  config = lib.mkIf (cfg.enable && cfg.netboot) {
+    systemd.services.sshd.after = [ "provision-ssh-hostkey.service" ];
+    systemd.services.provision-ssh-hostkey = {
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = true;
+      };
+      script = ''
+        mkdir -p /etc/ssh
+        umask 0077
+        until ${pkgs.iputils}/bin/ping -c 1 vpn-gw.wob01.infra.forkos.org; do sleep 1; done
+        ${pkgs.curl}/bin/curl --local-port 25-1024 https://vpn-gw.wob01.infra.forkos.org/${config.networking.hostName}/ssh_host_ed25519_key > /etc/ssh/ssh_host_ed25519_key
+        # Run the activation script again to trigger agenix decryption
+        /run/current-system/activate
+      '';
+    };
+
+    system.build = {
+
+      # Build a kernel and initramfs which will download the IPXE script from hydra using
+      # u-root pxeboot tool and kexec into the final netbooted system.
+      notipxe = import (modulesPath + "/..") {
+        system = "x86_64-linux";
+        configuration = 
+          { pkgs, config, ... }:
+
+          {
+            system.stateVersion = "24.11";
+            boot.initrd.availableKernelModules = [ "ahci" "ehci_pci" "usb_storage" "usbhid" "sd_mod" "igb" "bonding" ];
+            boot.kernelParams = [ "console=ttyS0,115200" "panic=1" "boot.panic_on_fail" ];
+            #boot.initrd.systemd.emergencyAccess = true;
+            networking.hostName = "${node.config.networking.hostName}-boot";
+            nixpkgs.overlays = import ../../overlays;
+            boot.loader.grub.enable = false;
+            fileSystems."/".device = "bogus"; # this config will never be booted
+            boot.initrd.systemd.enable = true;
+            boot.initrd.systemd.network = {
+              enable = true;
+              networks = node.config.systemd.network.networks;
+              netdevs = node.config.systemd.network.netdevs;
+            };
+            boot.initrd.systemd.storePaths = [
+              "${pkgs.u-root}/bin/pxeboot"
+              "${pkgs.iputils}/bin/ping"
+            ];
+            boot.initrd.systemd.services.kexec = {
+              serviceConfig.Restart = "on-failure";
+              serviceConfig.Type = "oneshot";
+              wantedBy = [ "initrd-root-fs.target" ];
+              before = [ "sysroot.mount" ];
+              script = ''
+                ln -sf /dev/console /dev/tty
+                until ${pkgs.iputils}/bin/ping -c 1 hydra.forkos.org; do sleep 1; done
+                ${pkgs.u-root}/bin/pxeboot -v -ipv4=false -file https://hydra.forkos.org/job/infra/main/${node.config.networking.hostName}/latest/download-by-type/file/ipxe 
+              '';
+            };
+            boot.initrd.systemd.contents."/etc/ssl/certs/ca-certificates.crt".source = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
+            boot.initrd.services.resolved.enable = false;
+            boot.initrd.systemd.contents."/etc/resolv.conf".text = ''
+              nameserver 2001:4860:4860::6464
+            '';
+            boot.initrd.systemd.contents."/etc/systemd/journald.conf".text = ''
+              [Journal]
+              ForwardToConsole=yes
+              MaxLevelConsole=debug
+            '';
+
+            # Provide a bootable USB drive image
+            system.build.usbImage = pkgs.callPackage ({ stdenv, runCommand, dosfstools, e2fsprogs, mtools, libfaketime, util-linux, nukeReferences }:
+              runCommand "boot-img-${node.config.networking.hostName}" {
+                nativeBuildInputs = [ dosfstools e2fsprogs libfaketime mtools util-linux ];
+                outputs = [ "out" "firmware_part" ];
+              } ''
+                export img=$out
+                truncate -s 40M $img
+
+                sfdisk $img <<EOF
+                    label: gpt
+                    label-id: F222513B-DED1-49FA-B591-20CE86A2FE7F
+
+                    type=C12A7328-F81F-11D2-BA4B-00A0C93EC93B, bootable
+                EOF
+
+                # Create a FAT32 /boot/firmware partition of suitable size into firmware_part.img
+                eval $(partx $img -o START,SECTORS --nr 1 --pairs)
+                truncate -s $((2081 * 512 + SECTORS * 512)) firmware_part.img
+
+                mkfs.vfat --invariant -i 2e24ec82 -n BOOT firmware_part.img
+
+                # Populate the files intended for /boot/firmware
+                mkdir -p firmware/EFI/BOOT firmware/loader/entries
+                cp ${pkgs.systemd}/lib/systemd/boot/efi/systemd-boot*.efi firmware/EFI/BOOT/BOOT${lib.toUpper stdenv.hostPlatform.efiArch}.EFI
+
+                cat > firmware/loader/loader.conf << EOF
+                default foo
+                EOF
+                cat > firmware/loader/entries/default.conf << EOF
+                title Default
+                linux /EFI/${pkgs.stdenv.hostPlatform.linux-kernel.target}
+                initrd /EFI/initrd
+                options init=${config.system.build.toplevel}/init ${toString config.boot.kernelParams}
+                EOF
+                cp ${config.system.build.kernel}/${pkgs.stdenv.hostPlatform.linux-kernel.target} firmware/EFI/${pkgs.stdenv.hostPlatform.linux-kernel.target}
+                cp ${config.system.build.initialRamdisk}/${config.system.boot.loader.initrdFile} firmware/EFI/initrd
+
+                find firmware -exec touch --date=2000-01-01 {} +
+                # Copy the populated /boot/firmware into the SD image
+                cd firmware
+                # Force a fixed order in mcopy for better determinism, and avoid file globbing
+                for d in $(find . -type d -mindepth 1 | sort); do
+                  faketime "2000-01-01 00:00:00" mmd -i ../firmware_part.img "::/$d"
+                done
+                for f in $(find . -type f | sort); do
+                  mcopy -pvm -i ../firmware_part.img "$f" "::/$f"
+                done
+                cd ..
+
+                # Verify the FAT partition before copying it.
+                fsck.vfat -vn firmware_part.img
+                dd conv=notrunc if=firmware_part.img of=$img seek=$START count=$SECTORS
+
+                cp firmware_part.img $firmware_part
+              ''
+            ) {};
+          }
+        ;
+      };
+
+      # This is the config which will actually be booted
+      netbootVariant = extendModules {
+        modules = [
+          (
+            { modulesPath, ... }:
+
+            {
+              imports = [ (modulesPath + "/installer/netboot/netboot.nix") ];
+            }
+          )
+        ];
+      };
+      # A derivation combining all the artifacts required for netbooting for the hydra job
+      netbootDir = let
+        kernelTarget = pkgs.stdenv.hostPlatform.linux-kernel.target;
+        build = config.system.build.netbootVariant.config.system.build;
+      in
+        pkgs.symlinkJoin {
+          name = "netboot";
+          paths = [
+            build.netbootRamdisk
+            build.kernel
+            build.netbootIpxeScript
+          ];
+          postBuild = ''
+            mkdir -p $out/nix-support
+            echo "file ${kernelTarget} $out/${kernelTarget}" >> $out/nix-support/hydra-build-products
+            echo "file initrd $out/initrd" >> $out/nix-support/hydra-build-products
+            echo "file ipxe $out/netboot.ipxe" >> $out/nix-support/hydra-build-products
+          '';
+          preferLocalBuild = true;
+        };
+    };
+  };
+}

services/block-crawlers/blocked-ua.txt

@@ -0,0 +1,40 @@
+AI2Bot
+Ai2Bot-Dolma
+Amazonbot
+anthropic-ai
+Applebot
+Applebot-Extended
+Bytespider
+CCBot
+ChatGPT-User
+Claude-Web
+ClaudeBot
+cohere-ai
+Diffbot
+FacebookBot
+facebookexternalhit
+FriendlyCrawler
+Google-Extended
+GoogleOther
+GoogleOther-Image
+GoogleOther-Video
+GPTBot
+iaskspider/2.0
+ICC-Crawler
+ImagesiftBot
+img2dataset
+ISSCyberRiskCrawler
+Kangaroo Bot
+Meta-ExternalAgent
+Meta-ExternalFetcher
+OAI-SearchBot
+omgili
+omgilibot
+PerplexityBot
+PetalBot
+Scrapy
+Sidetrade indexer bot
+Timpibot
+VelenPublicWebCrawler
+Webzio-Extended
+YouBot

services/block-crawlers/default.nix

@@ -0,0 +1,32 @@
+{ pkgs, config, lib, ... }:
+let
+  inherit (lib) mkEnableOption mkIf mkOption types concatStringsSep mkDefault splitString;
+  cfg = config.bagel.services.nginx.crawler-blocker;
+  mkRobotsFile = blockedUAs: pkgs.writeText "robots.txt" ''
+    ${concatStringsSep "\n" (map (ua: "User-agent: ${ua}") blockedUAs)}
+    Disallow: /
+  '';
+in
+{
+  options = {
+    bagel.services.nginx.crawler-blocker = {
+      enable = mkEnableOption "the crawler blocker";
+
+      userAgents = mkOption {
+        type = types.listOf types.str;
+        default = splitString "\n" (builtins.readFile ./blocked-ua.txt);
+      };
+    };
+
+    services.nginx.virtualHosts = mkOption {
+      type = types.attrsOf (types.submodule {
+        config = {
+          locations."= /robots.txt" = mkIf cfg.enable (mkDefault {
+            alias = mkRobotsFile cfg.userAgents;
+          });
+        };
+      });
+    };
+  };
+}
+

services/buildbot/default.nix

@@ -0,0 +1,207 @@
+{
+  nodes,
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.bagel.services.buildbot;
+  ssh-keys = import ../../common/ssh-keys.nix;
+  freeGbDiskSpace = 20;
+  extraTenantSpecificBuilders = {
+    lix = import ./lix.nix {
+      inherit config nodes;
+    };
+    floral = [ ];
+  }.${cfg.tenant or (throw "${cfg.tenant} is not a known tenant")};
+  clientId = {
+    lix = "buildbot";
+    floral = "forkos-buildbot";
+  }.${cfg.tenant or (throw "${cfg.tenant} is not a known tenant")};
+  inherit (lib) mkEnableOption mkOption mkIf types;
+in
+{
+  options.bagel.services.buildbot = {
+    enable = mkEnableOption "Buildbot";
+
+    tenant = mkOption {
+      type = types.enum [ "lix" "floral" ];
+      description = "Which buildbot tenant to enable";
+    };
+
+    domain = mkOption {
+      type = types.str;
+      description = "Domain name for this Buildbot";
+    };
+
+    gerrit = {
+      domain = mkOption {
+        type = types.str;
+        description = "Canonical domain of the Gerrit associated to this Buildbot";
+        example = [ "cl.forkos.org" ];
+      };
+
+      port = mkOption {
+        type = types.port;
+        description = "Gerrit SSH port for this Buildbot";
+      };
+
+      username = mkOption {
+        type = types.str;
+        description = "Gerrit service username for this Buildbot";
+      };
+    };
+
+    cors.allowedOrigins = mkOption {
+      type = types.listOf types.str;
+      example = [ "*.forkos.org" ];
+      description = "Allowed origin for Buildbot and NGINX for CORS without the protocol";
+    };
+
+    buildSystems = mkOption {
+      type = types.listOf (types.enum [ "x86_64-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" ]);
+      default = [ "x86_64-linux" ];
+      example = [ "x86_64-linux" "aarch64-linux" ];
+      description = "Supported build systems for this buildbot instance.";
+    };
+
+    projects = mkOption {
+      type = types.listOf types.str;
+      example = [ "nixpkgs" ];
+      description = "Static list of projects enabled for Buildbot CI";
+    };
+
+    builders = mkOption {
+      type = types.listOf types.str;
+      description = "List of builders to configure for Buildbot";
+      example = [ "builder-2" "builder-3" ];
+    };
+  };
+
+  config = mkIf cfg.enable {
+    networking.firewall.allowedTCPPorts = [ 80 443 ];
+    bagel.secrets.files = [
+      "buildbot-worker-password"
+      "buildbot-oauth-secret"
+      "buildbot-workers"
+      "buildbot-service-key"
+      "buildbot-signing-key"
+      "buildbot-remote-builder-key"
+    ];
+    age.secrets.buildbot-signing-key = {
+      owner = "buildbot-worker";
+      group = "buildbot-worker";
+    };
+    age.secrets.buildbot-remote-builder-key = {
+      file = ../../secrets/${cfg.tenant}/buildbot-remote-builder-key.age;
+      owner = "buildbot-worker";
+      group = "buildbot-worker";
+    };
+
+    services.nginx = {
+      recommendedProxySettings = true;
+      appendHttpConfig = ''
+        # Our session stuff is too big with the TWISTED_COOKIE in addition.
+        # Default is usually 4k or 8k.
+        large_client_header_buffers 4 16k;
+      '';
+      virtualHosts.${cfg.domain} = {
+        forceSSL = true;
+        enableACME = true;
+        extraConfig = ''
+          # This is needed so that logged-in users in Buildbot can include their credentials in their requests.
+          add_header Access-Control-Allow-Credentials 'true' always;
+        '';
+      };
+    };
+
+    services.buildbot-nix.worker = {
+      enable = true;
+      workerPasswordFile = config.age.secrets.buildbot-worker-password.path;
+      # All credits to eldritch horrors for this beauty.
+      workerArchitectures =
+      {
+        # nix-eval-jobs runs under a lock, error reports do not (but are cheap)
+        other = 8;
+      } // (
+        lib.filterAttrs
+          (n: v: lib.elem n config.services.buildbot-nix.coordinator.buildSystems)
+          (lib.zipAttrsWith
+            (_: lib.foldl' lib.add 0)
+            (lib.concatMap
+              (m: map (s: { ${s} = m.maxJobs; }) m.systems)
+              config.services.buildbot-nix.coordinator.buildMachines))
+        );
+    };
+
+    services.buildbot-nix.coordinator = {
+      enable = true;
+
+      inherit (cfg) domain;
+      # TODO(raito): is that really necessary when we can just collect buildMachines' systems?
+      inherit (cfg) buildSystems;
+
+      oauth2 = {
+        name = "Lix";
+        inherit clientId;
+        clientSecretFile = config.age.secrets.buildbot-oauth-secret.path;
+        resourceEndpoint = "https://identity.lix.systems";
+        authUri = "https://identity.lix.systems/realms/lix-project/protocol/openid-connect/auth";
+        tokenUri = "https://identity.lix.systems/realms/lix-project/protocol/openid-connect/token";
+        userinfoUri = "https://identity.lix.systems/realms/lix-project/protocol/openid-connect/userinfo";
+      };
+
+      # TODO(raito): this is not really necessary, we never have remote buildbot workers.
+      # we can replace all of this with automatic localworker generation on buildbot-nix side.
+      workersFile = config.age.secrets.buildbot-workers.path;
+
+      # We rely on NGINX to do the CORS dance.
+      allowedOrigins = cfg.cors.allowedOrigins;
+
+      buildMachines = map (n: {
+          hostName = nodes.${n}.config.networking.fqdn;
+          protocol = "ssh-ng";
+          # Follows Hydra.
+          maxJobs = 8;
+          sshKey = config.age.secrets.buildbot-remote-builder-key.path;
+          sshUser = "buildbot";
+          systems = [ "x86_64-linux" ];
+          supportedFeatures = nodes.${n}.config.nix.settings.system-features;
+          # Contrary to how Nix works, here we can specify non-base64 public host keys.
+          publicHostKey = ssh-keys.machines.${n};
+        }
+      ) cfg.builders ++ extraTenantSpecificBuilders;
+
+      gerrit = {
+        # Manually managed account…
+        # TODO: https://git.lix.systems/the-distro/infra/issues/69
+        inherit (cfg.gerrit) domain port username;
+        privateKeyFile = config.age.secrets.buildbot-service-key.path;
+        inherit (cfg) projects;
+      };
+
+      evalWorkerCount = 6;
+      evalMaxMemorySize = "4096";
+
+      signingKeyFile = config.age.secrets.buildbot-signing-key.path;
+    };
+
+    # Make PostgreSQL restart smoother.
+    systemd.services.postgresql.serviceConfig = {
+      Restart = "always";
+      RestartMaxDelaySec = "5m";
+      RestartSteps = 10;
+    };
+
+
+    nix.settings.keep-derivations = true;
+    nix.gc = {
+      automatic = true;
+      dates = "hourly";
+      options = ''
+        --max-freed "$((${toString freeGbDiskSpace} * 1024**3 - 1024 * $(df -P -k /nix/store | tail -n 1 | ${pkgs.gawk}/bin/awk '{ print $4 }')))"
+      '';
+    };
+  };
+}