ofborg: enable nginx for certs

Flake lock file updates:

• Updated input 'nixpkgs':
    'github:NixOS/nixpkgs/e2f08f4d8b3ecb5cf5c9fd9cb2d53bb3c71807da' (2024-10-05)
  → 'github:NixOS/nixpkgs/7ffd9ae656aec493492b44d0ddfb28e79a1ea25d' (2024-11-02)

.envrc

@@ -1,2 +1,11 @@
+#!/usr/bin/env bash
+# the shebang is ignored, but nice for editors
+
 # shellcheck shell=bash
-use flake
+if type -P lorri &>/dev/null; then
+  eval "$(lorri direnv --flake .)"
+else
+  echo 'while direnv evaluated .envrc, could not find the command "lorri" [https://github.com/nix-community/lorri]'
+  use flake
+fi
+

README.md

@@ -1 +1,32 @@
-Infrastructure for the donut shaped thing that is absolutely not a donut.
+# Infrastructure for the donut shaped thing that is absolutely not a donut.
+
+## Quick start
+
+### Build the infrastructure
+
+```
+$ colmena build --on @localboot
+```
+
+Notice that `@localboot` is load-bearing as we have some machines that _cannot be_ deployed with vanilla Colmena. Fixing this is welcome.
+
+### Recommended deploy process
+
+```
+$ colmena apply dry-activate $machine # Verify that the nvd log is reasonable.
+$ colmena apply $machine
+```
+
+### Recommended upgrade process
+
+```
+$ nix flake update
+$ colmena apply dry-activate --on @localboot # Verify that the nvd log is reasonable. Run it twice to get only NVD logs shown.
+$ colmena apply --on @localboot
+```
+
+## Troubleshooting
+
+### I failed to deploy `gerrit01`
+
+Our Gerrit source build is known to have some hiccups sometimes, we are always interested in build logs, feel free to attach information in a new issue so we can make it more reliable.

common/admins.nix

@@ -1,16 +1,47 @@
+{ lib, ... }:
 let
-  keys = import ./ssh-keys.nix;
-in {
-  users.users.root.openssh.authorizedKeys.keys = 
-    keys.users.delroth ++
-    keys.users.emilylange ++
-    keys.users.hexchen ++
-    keys.users.jade ++
-    keys.users.janik ++
-    keys.users.k900 ++
-    keys.users.lukegb ++
-    keys.users.maxine ++
-    keys.users.raito ++
-    keys.users.thubrecht ++
-    keys.users.yuka;
+  inherit (lib) genAttrs;
+in
+# Note: to add somefew in this list.
+# Ensure their SSH key is already in common/ssh-keys.nix with
+# the same username for here, so that the keys is automatically added.
+{
+  bagel.groups = {
+    floral-infra.members = [
+      "delroth"
+      "emilylange"
+      "hexchen"
+      "jade"
+      "janik"
+      "k900"
+      "maxine"
+      "raito"
+      "thubrecht"
+      "winter"
+      "yuka"
+      "ckie"
+    ];
+
+    lix-infra.members = [
+      "raito"
+      "hexchen"
+      "jade"
+      "pennae"
+    ];
+  };
+  bagel.users = genAttrs [
+    "delroth"
+    "emilylange"
+    "hexchen"
+    "jade"
+    "janik"
+    "k900"
+    "maxine"
+    "raito"
+    "thubrecht"
+    "winter"
+    "yuka"
+    "ckie"
+    "pennae"
+  ] (name: {});
 }

common/base-server.nix

@@ -1,11 +1,13 @@
 { lib, pkgs, ... }: {
   imports = [
     ./known-ssh-keys.nix
+    ./cgroups.nix
   ];
 
   nixpkgs.overlays = import ../overlays;
 
   nix.package = lib.mkDefault pkgs.lix;
+  system.tools.nixos-option.enable = false;
   services.openssh.enable = lib.mkForce true;
 
   networking.nftables.enable = true;
@@ -30,7 +32,7 @@
     automatic = true;
     persistent = true;
     dates = lib.mkDefault "daily";
-    options = "--delete-older-than 30d";
+    options = lib.mkDefault "--delete-older-than 30d";
   };
 
   services.journald.extraConfig = "SystemMaxUse=512M";
@@ -55,4 +57,19 @@
     "en_US.UTF-8/UTF-8"
     "fr_FR.UTF-8/UTF-8"
   ];
+
+  time.timeZone = "UTC";
+
+  security.acme.acceptTerms = true;
+  security.acme.defaults.email = "infra@forkos.org";
+
+  # Enable system diffs.
+  system.activationScripts.system-diff = {
+    supportsDryActivation = true; # safe: only outputs to stdout
+    text = ''
+      if [ -e /run/current-system ]; then
+        PATH=$PATH:${pkgs.nix}/bin ${pkgs.nvd}/bin/nvd diff /run/current-system $systemConfig
+      fi
+    '';
+  };
 }

common/cgroups.nix

@@ -0,0 +1,83 @@
+# Relatively inspired by fbtax2:
+# https://facebookmicrosites.github.io/cgroup2/docs/fbtax-results.html
+#
+# See also the Chris Down talk at LISA'21:
+# https://www.usenix.org/conference/lisa21/presentation/down
+{ ... }:
+let
+  systemCriticalSliceConfig = {
+    ManagedOOMMemoryPressure = "kill";
+
+    # guarantee availability of memory
+    MemoryMin = "192M";
+    # default 100
+    IOWeight = 1000;
+    # default 100
+    CPUWeight = 1000;
+  };
+in
+{
+  systemd.oomd = {
+    enable = true;
+    # why not, we have cgroups at user level now so it'll just kill the
+    # terminal
+    enableRootSlice = true;
+    enableSystemSlice = true;
+    enableUserSlices = true;
+  };
+
+  systemd.enableCgroupAccounting = true;
+
+  systemd.services.nix-daemon = {
+    serviceConfig = {
+      # FIXME: how do i deprioritize this for memory
+      CPUWeight = 10;
+      IOWeight = 10;
+    };
+  };
+
+  systemd.slices.hostcritical = {
+    description = "Ensures that services to keep the system alive remain alive";
+
+    unitConfig = {
+      # required to avoid a dependency cycle on systemd-oomd. systemd will
+      # actually guess this right but we should fix it anyway.
+      DefaultDependencies = false;
+    };
+
+    sliceConfig = systemCriticalSliceConfig;
+  };
+
+  # make root logins higher priority for resources
+  systemd.slices."user-0" = {
+    sliceConfig = systemCriticalSliceConfig;
+  };
+
+
+  systemd.slices.system = {
+    sliceConfig = {
+      ManagedOOMMemoryPressure = "kill";
+      ManagedOOMMemoryPressureLimit = "50%";
+
+      IOWeight = 100;
+    };
+  };
+
+  systemd.services.sshd = {
+    serviceConfig = {
+      Slice = "hostcritical.slice";
+    };
+  };
+
+  systemd.services.systemd-oomd = {
+    serviceConfig = {
+      Slice = "hostcritical.slice";
+    };
+  };
+
+  systemd.services.systemd-journald = {
+    serviceConfig = {
+      Slice = "hostcritical.slice";
+    };
+  };
+}

common/channels.nix

@@ -0,0 +1,32 @@
+# Taken from https://github.com/NixOS/infra/blob/master/channels.nix
+{
+  # "Channel name" = {
+  #   # This should be the <value> part of
+  #   # https://hydra.forkos.org/job/<value>/latest-finished
+  #   job = "project/jobset/jobname";
+  #
+  #   # When adding a new version, determine if it needs to be tagged as a
+  #   # variant -- for example:
+  #   # nixos-xx.xx         => primary
+  #   # nixos-xx.xx-small   => small
+  #   # nixos-xx.xx-darwin  => darwin
+  #   # nixos-xx.xx-aarch64 => aarch64
+  #   variant = "primary";
+  #
+  #   # Channel Status:
+  #   # '*-unstable' channels are always "rolling"
+  #   # Otherwise a release generally progresses through the following phases:
+  #   #
+  #   #  - Directly after branch off                   => "beta"
+  #   #  - Once the channel is released                => "stable"
+  #   #  - Once the next channel is released           => "deprecated"
+  #   #  - N months after the next channel is released => "unmaintained"
+  #   #    (check the release notes for when this should happen)
+  #   status = "beta";
+  # };
+  "forkos-unstable" = {
+    job = "forkos/nixos-main/tested";
+    variant = "primary";
+    status = "rolling";
+  };
+}

common/default.nix

@@ -1,12 +1,14 @@
 {
   imports = [
     ./admins.nix
+    ./server-acl.nix
     ./base-server.nix
     ./hardening.nix
     ./nix.nix
     ./raito-proxy-aware-nginx.nix
-    ./raito-vm.nix
     ./sysadmin
+    ./hardware
     ./zsh.nix
+    ./secrets.nix
   ];
 }

common/hardware/default.nix

@@ -0,0 +1,7 @@
+{ ... }: {
+  imports = [
+    ./raito-vm.nix
+    ./oracle-vm.nix
+    ./hetzner.nix
+  ];
+}

common/hardware/hetzner.nix

@@ -0,0 +1,76 @@
+
+{ lib, config, ... }:
+let
+  cfg = config.bagel.hardware.hetzner;
+  inherit (lib) mkEnableOption mkIf mkOption types;
+in
+{
+  options.bagel.hardware.hetzner = {
+    enable = mkEnableOption "Hetzner's hardware defaults";
+
+    platformType = mkOption {
+      # Only VMs are supported.
+      type = types.enum [ "virtual-machine" ];
+    };
+
+    system = mkOption {
+      # Only the aarch64-linux VM Hetzner is supported.
+      type = types.enum [ "aarch64-linux" ];
+    };
+
+    networking.wan = {
+      mac = mkOption {
+        type = types.str;
+        description = "MAC address of the WAN interface in the Hetzner machine";
+      };
+      address = mkOption {
+        type = types.listOf types.str;
+        description = "List of static addresses attached to the WAN interface";
+      };
+    };
+  };
+
+  config = mkIf cfg.enable {
+    # A bunch of stuff is virtio.
+    boot.initrd.availableKernelModules = [
+      "xhci_pci"
+      "usbhid"
+      "sr_mod"
+      "virtio_gpu"
+      "virtio_scsi"
+      "virtio_rng"
+      "virtio_pci"
+    ];
+
+    boot.loader.systemd-boot.enable = true;
+    boot.loader.efi.canTouchEfiVariables = true;
+
+    networking.useDHCP = lib.mkDefault false;
+
+    # Stolen from the netplan provided by aarch64 Ubuntu images.
+    systemd.network.enable = true;
+    systemd.network.links."10-wan" = {
+      linkConfig.Name = "wan";
+      matchConfig.MACAddress = cfg.networking.mac;
+    };
+    systemd.network.networks."10-wan" = {
+      matchConfig.Name = "wan";
+      networkingConfig.Address = cfg.networking.address;
+      linkConfig.RequiredForOnline = true;
+      DHCP = "ipv4";
+      routes = [
+        {
+          routeConfig = {
+            Destination = "::/0";
+            GatewayOnLink = true;
+            Gateway = "fe80::1";
+          };
+        }
+      ];
+      dhcpV4Config = {
+        RouteMetric = 100;
+        UseMTU = true;
+      };
+    };
+  };
+}

common/hardware/oracle-vm.nix

@@ -0,0 +1,52 @@
+
+{ lib, config, modulesPath, ... }:
+let
+  cfg = config.bagel.hardware.oracle-vm;
+  inherit (lib) mkEnableOption mkIf mkOption types;
+in
+{
+  options.bagel.hardware.oracle-vm = {
+    enable = mkEnableOption "Oracle's VM hardware defaults";
+
+    system = mkOption {
+      # Only the free Oracle VMs are supported.
+      type = types.enum [ "aarch64-linux" ];
+    };
+  };
+
+  # Imports a bunch of virtio modules.
+  imports = [
+    "${modulesPath}/profiles/qemu-guest.nix"
+  ];
+
+  config = mkIf cfg.enable {
+    boot.loader.systemd-boot.enable = true;
+    boot.loader.efi.canTouchEfiVariables = true;
+    boot.initrd.systemd.enable = true;
+
+    boot.initrd.availableKernelModules = [
+      "xhci_pci" "virtio_pci" "usbhid" "sr_mod"
+    ];
+    boot.initrd.kernelModules = [ ];
+    boot.kernelModules = [ ];
+    boot.extraModulePackages = [ ];
+
+    nixpkgs.hostPlatform = cfg.system;
+
+    # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+    # (the default) this is the recommended approach. When using systemd-networkd it's
+    # still possible to use this option, but it's recommended to use it in conjunction
+    # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+    networking.useDHCP = lib.mkDefault false;
+    # Examples:
+    # 2: enp0s3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
+    #  link/ether 02:00:17:00:91:6e brd ff:ff:ff:ff:ff:ff
+    #  inet 10.0.0.94/24 brd 10.0.0.255 scope global dynamic noprefixroute enp0s3
+    #     valid_lft 44162sec preferred_lft 33362sec
+    #  inet6 fe80::17ff:fe00:916e/64 scope link
+    #     valid_lft forever preferred_lft forever
+    # [root@build02-aarch64-lahfa:~]# ip r
+    # default via 10.0.0.1 dev enp0s3 proto dhcp src 10.0.0.94 metric 1002 mtu 9000
+    networking.interfaces.enp0s3.useDHCP = lib.mkDefault true;
+  };
+}

common/hardware/raito-vm.nix

@@ -30,8 +30,6 @@ in
   config = mkIf cfg.enable {
     services.qemuGuest.enable = true;
     systemd.network.enable = true;
-    security.acme.defaults.email = "bagel-acme@lahfa.xyz";
-    security.acme.acceptTerms = true;
     networking.useDHCP = lib.mkDefault false;
 
     systemd.network.networks."10-nat-lan" = {

common/known-ssh-keys.nix

@@ -2,5 +2,6 @@
 {
   programs.ssh.knownHosts = {
     "[cl.forkos.org]:29418".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM82mJ259C8Nc+BHHNBeRWXWhL3dfirQhmFbDAwHMle3";
+    "[gerrit.lix.systems]:2022".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICC/S6Z56uhv7zBMutkV0nU8eDuRcl3trykGWBch4L/l";
   };
 }

common/nix.nix

@@ -14,7 +14,7 @@
 
   # Use our cache and trust its signing key. Still use cache.nixos.org as
   # fallback.
-  nix.settings.substituters = [ "https://bagel-cache.s3-web.delroth.net/" ];
+  nix.settings.substituters = [ "https://cache.forkos.org/" ];
   nix.settings.trusted-public-keys = [
     "cache.forkos.org:xfXIUJO1yiEITJmYsVmNDa9BFSlgTh/YqZ+4ei1EhQg="
   ];

common/raito-proxy-aware-nginx.nix

@@ -1,9 +1,10 @@
 # This enables an IPv6-only server which is proxied by kurisu.lahfa.xyz to have proper IPv4 logs via PROXY protocol.
 { config, lib, ... }:
 let
-  inherit (lib) mkEnableOption mkIf;
+  inherit (lib) mkEnableOption mkIf concatStringsSep;
   cfg = config.bagel.raito.v6-proxy-awareness;
-  allowedUpstream = "2001:bc8:38ee:99::1/128";
+  #                      outside of raito infra    inside of raito infra
+  allowedUpstreams = [ "2001:bc8:38ee::1/128"     "2001:bc8:38ee:99::1/128" ];
 in
   {
     options.bagel.raito.v6-proxy-awareness.enable = mkEnableOption "the kurisu.lahfa.xyz's sniproxy awareness for NGINX";
@@ -20,8 +21,8 @@ in
         ];
 
         appendHttpConfig = ''
-          # Kurisu node
-          set_real_ip_from ${allowedUpstream};
+          # Kurisu nodes
+          ${concatStringsSep "\n" (map (up: "set_real_ip_from ${up};") allowedUpstreams)}
           real_ip_header proxy_protocol;
         '';
       };
@@ -29,7 +30,7 @@ in
       # Move to nftables if firewall is enabled.
       networking.nftables.enable = true;
       networking.firewall.extraInputRules = ''
-        ip6 saddr ${allowedUpstream} tcp dport 444 accept
+        ${concatStringsSep "\n" (map (up: "ip6 saddr ${up} tcp dport 444 accept") allowedUpstreams)}
       '';
     };
   }

common/secrets.nix

@@ -0,0 +1,22 @@
+## This is a simple secret abstraction with multi-tenancy awareness.
+{ config, lib, ... }:
+let
+  cfg = config.bagel.secrets;
+  inherit (lib) mkOption types genAttrs;
+in
+{
+  options.bagel.secrets = {
+    tenant = mkOption {
+      type = types.enum [ "lix" "floral" ];
+    };
+
+    files = mkOption {
+      type = types.listOf types.str;
+      default = [ ];
+    };
+  };
+
+  config.age.secrets = genAttrs cfg.files (secretFile: {
+      file = ../secrets/${cfg.tenant}/${secretFile}.age;
+    });
+}

common/server-acl.nix

@@ -0,0 +1,69 @@
+{ lib, config, ... }:
+let
+  keys = import ./ssh-keys.nix;
+  inherit (lib) mkOption types length concatMap listToAttrs catAttrs attrValues;
+  cfgAdmins = config.bagel.admins;
+  cfgGroups = config.bagel.groups;
+  cfgUsers = config.bagel.users;
+
+  userOpts = { name, ... }: {
+    options = {
+      sshKeys = mkOption {
+        type = types.listOf types.str;
+        description = "List of SSH keys associated to this user, defaults to `ssh-keys.nix` entries.";
+        default = keys.users.${name} or [ ];
+      };
+    };
+  };
+  groupOpts = { name, ... }: {
+    options = {
+      members = mkOption {
+        type = types.listOf types.str;
+        description = "List of users member of this group";
+        example = [ "raito" ];
+        default = [ ];
+      };
+    };
+  };
+
+  # There might be duplicate in that list. We will turn it into an attribute set.
+  allowedMembers = listToAttrs (
+    map (member: {
+      name = member;
+      value = cfgUsers.${member};
+    }) (concatMap (allowedGroup: cfgGroups.${allowedGroup}.members) cfgAdmins.allowedGroups));
+
+  rootKeys = concatMap ({ sshKeys, ... }: sshKeys) (attrValues allowedMembers);
+in
+{
+  options.bagel.users = mkOption {
+    type = types.attrsOf (types.submodule userOpts);
+    description = "User configuration for server ACLs";
+  };
+
+  options.bagel.groups = mkOption {
+    type = types.attrsOf (types.submodule groupOpts);
+    description = "Group configuration for server ACLs";
+  };
+
+  options.bagel.admins = {
+    allowedGroups = mkOption {
+      type = types.listOf types.str;
+      default = [ "catch-all" ];
+      description = "List of groups which are allowed to admin this machine.";
+      example = [ "lix" "build-infra" ];
+    };
+  };
+
+  config = {
+    assertions = [
+      { assertion = length config.users.users.root.openssh.authorizedKeys.keys > 0;
+        # TODO: you can add printing of `concatStringsSep ", " cfg.allowedGroups` to diagnose
+        # which are the allowed groups and existing admins.
+        message = "root@${config.networking.fqdnOrHostName} has no SSH key attached, this machine will lose its access if you deploy it successfully! Set a valid `bagel.admins.allowedGroups` or ensure you have at least one administrator of the relevant group registered";
+      }
+    ];
+
+    users.users.root.openssh.authorizedKeys.keys = rootKeys;
+  };
+}

common/ssh-keys.nix

@@ -1,11 +1,14 @@
 {
   machines = {
+    # Floral
     bagel-box = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAsO4bNqY04uG13Pg3ubHfRDssTphDLzZ4YUniE5/p+M";
     meta01 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM5t9gYorOWgpCFDJgb24pyCKIabGpeI2H/UfdvXODcT";
+    public01 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPBy8G8rfLA6E9i+t5kjVafxU1c2NXATXKxoXTH4Kgtm";
     gerrit01 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIA+eSZu+u9sCynrMlsmFzQHLIELQAuVg0Cs1pBvwb4+A";
     fodwatch = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFRyTNfvKl5FcSyzGzw+h+bNFNOxdhvI67WdUZ2iIJ1L";
     buildbot = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJgIu6ouagYqBeMLfmn1CbaDJMuZcPH9bnUhkht8GfuB";
     git = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEQJcpkCUOx8+5oukMX6lxrYcIX8FyHu8Mc/3+ieKMUn";
+    build-coord = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINpAEJP7F+XtJBpQP1jTzwXwQgJrFxwEJjPf/rnCXkJA";
     builder-0 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBHSNcDGctvlG6BHcJuYIzW9WsBJsts2vpwSketsbXoL";
     builder-1 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIQOGUjERK7Mx8UPM/rbOdMqVyn1sbWqYOG6CbOzH2wm";
     builder-2 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMKzXIqCoYElEKIYgjbSpqEcDeOvV+Wo3Agq3jba83cB";
@@ -17,8 +20,17 @@
     builder-8 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKGSWHNeqT0kF/e4yVy2ieW98X5QMyCYIYZh9WTmQDs1";
     builder-9 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOhws9zGgocVY36dMtOL+CXadpvRMffxoWMkfEcTBJm7";
     builder-10 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE7sgIuTSqZiZhp8TvObSbIEhcHHsL5hcmYA22uzwxth";
-    builder-11 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEAqFo1qJY7MSUkfB+zxXB8Lpt/Iqz/RR5A+zwhpRWhr";
     wob-vpn-gw = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINVytPPW8XnXf/rD5TFzsw//CZc2lBjQLmDzlVGPZsjh";
+
+    # Lix
+    build01-aarch64-lix = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICC69NZD/zhIB/wUb5odg46bss5g8hH2fDl22bk4qeSW";
+    build02-aarch64-lix = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGdJE375pe58RJbhKwXRp3D//+SJ3ssiVZrLsM9CLHn0";
+    build01-aarch64-darwin-lix = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMVf1uO0lv5UBti/naW/+amqLxvWZg+StXk9aM+lJ7e4";
+
+    buildbot-lix = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFoVSh35UqNQZ6ZZ1c6CzqERC40ovQ/KDXz8pC7nNlkR";
+
+    # Raito infrastructure
+    epyc-newtype-fr = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOXT9Init1MhKt4rjBANLq0t0bPww/WQZ96uB4AEDrml";
   };
 
   users = {
@@ -49,6 +61,12 @@
       "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiXXYkhRh+s7ixZ8rvG8ntIqd6FELQ9hh7HoaHQJRPU"
     ];
     thubrecht = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPM1jpXR7BWQa7Sed7ii3SbvIPRRlKb3G91qC0vOwfJn" ];
-    yuka = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKath4/fDnlv/4fzxkPrQN1ttmoPRNu/m9bEtdPJBDfY cardno:16_933_242" ];
+    yuka = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIxQ3NYBi8v1f/vhxLKDcA6upmX0pctRDbnK6SER5OUR yureka" ];
+    winter = [ "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIH/LDRUG+U+++UmlxvA2kspioTjktQZ8taDcHq8gVlkfAAAABHNzaDo=" ];
+    ckie = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIH3uTwzSSMAPg84fwbNp2cq9+BdLFeA1VzDGth4zCAbz https://mei.puppycat.house" ];
+    pennae = [
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIC5Wf5/IbyFpdziWfwxkQqxOf3r1L9pYn6xQBEKFwmMY"
+      "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIK8icXjHkb4XzbIVN3djH4CE7RvgGd+3xbG4cgh0Yls5AAAABHNzaDo="
+    ];
   };
 }

common/sysadmin/default.nix

@@ -13,7 +13,11 @@ in
         tmux
         rsync
         fd
+        eza
+        grc
         ripgrep
+        delta
+        tshark
         pv
         kitty.terminfo
         config.boot.kernelPackages.perf

common/zsh.nix

@@ -1,5 +1,4 @@
 { lib, pkgs, config, ... }: {
-  users.defaultUserShell = pkgs.zsh;
   programs.zsh = {
     enable = true;
     enableCompletion = true;

dashboards/default.nix

@@ -0,0 +1,43 @@
+{ gerrit-dashboard, stdenv, symlinkJoin, jsonnet, fetchFromGitHub, lib, ... }:
+let
+  inherit (lib) concatMapStringsSep;
+  datasource-id = "mimir";
+in
+rec {
+  grafonnet = fetchFromGitHub {
+    owner = "grafana";
+    repo = "grafonnet-lib";
+    # TODO: figure out how to read the jsonnet lockfile
+    # and propagate this a bit cleverly.
+    rev = "a1d61cce1da59c71409b99b5c7568511fec661ea";
+    hash = "sha256-fs5JZJbcL6sQXBjYhp5eeRtjTFw0J1O/BcwBC8Vm9EM=";
+  };
+  buildJsonnetDashboards = dashboardSrc: targets: stdenv.mkDerivation {
+    name = "jsonnet-grafana-dashboards";
+    src = dashboardSrc;
+    buildInputs = [ jsonnet ];
+    buildPhase = ''
+      runHook preBuild
+      mkdir -p $out
+      ${concatMapStringsSep "\n" (target: "jsonnet -J ${grafonnet} --ext-str datasource=${datasource-id} --ext-code publish=true $src/${target} > $out/${baseNameOf target}.json") targets}
+      runHook postBuild
+    '';
+  };
+
+  allDashboards = symlinkJoin {
+    name = "all-jsonnet-dashboards";
+    paths = [
+      (buildJsonnetDashboards gerrit-dashboard [
+        "dashboards/gerrit/caches/gerrit-caches.jsonnet"
+        "dashboards/gerrit/fetch-clone/gerrit-fetch-clone.jsonnet"
+        "dashboards/gerrit/fetch-clone/gerrit-phases.jsonnet"
+        "dashboards/gerrit/healthcheck/gerrit-healthcheck.jsonnet"
+        "dashboards/gerrit/latency/gerrit-push-latency.jsonnet"
+        "dashboards/gerrit/latency/gerrit-ui-actions-latency.jsonnet"
+        "dashboards/gerrit/overview/gerrit-overview.jsonnet"
+        "dashboards/gerrit/process/gerrit-process.jsonnet"
+        "dashboards/gerrit/queues/gerrit-queues.jsonnet"
+      ])
+    ];
+  };
+}

flake.lock

@@ -10,11 +10,11 @@
         "systems": "systems"
       },
       "locked": {
-        "lastModified": 1720546205,
-        "narHash": "sha256-boCXsjYVxDviyzoEyAk624600f3ZBo/DKtUdvMTpbGY=",
+        "lastModified": 1723293904,
+        "narHash": "sha256-b+uqzj+Wa6xgMS9aNbX4I+sXeb5biPDi39VgvSFqFvU=",
         "owner": "ryantm",
         "repo": "agenix",
-        "rev": "de96bd907d5fbc3b14fc33ad37d1b9a3cb15edc6",
+        "rev": "f6291c5935fdc4e0bef208cfc0dcab7e3f7a1c41",
         "type": "github"
       },
       "original": {
@@ -23,14 +23,38 @@
         "type": "github"
       }
     },
+    "attic": {
+      "inputs": {
+        "crane": "crane",
+        "flake-compat": "flake-compat_2",
+        "flake-parts": "flake-parts_2",
+        "nix-github-actions": "nix-github-actions_2",
+        "nixpkgs": "nixpkgs",
+        "nixpkgs-stable": "nixpkgs-stable"
+      },
+      "locked": {
+        "lastModified": 1730257295,
+        "narHash": "sha256-OQl+aAsKiyygvpzck1u0sZf/R4T9zM903CgNDFmmzA8=",
+        "owner": "zhaofengli",
+        "repo": "attic",
+        "rev": "48c8b395bfbc6b76c7eae74df6c74351255a095c",
+        "type": "github"
+      },
+      "original": {
+        "owner": "zhaofengli",
+        "ref": "main",
+        "repo": "attic",
+        "type": "github"
+      }
+    },
     "bats-assert": {
       "flake": false,
       "locked": {
-        "lastModified": 1636059754,
-        "narHash": "sha256-ewME0l27ZqfmAwJO4h5biTALc9bDLv7Bl3ftBzBuZwk=",
+        "lastModified": 1692829535,
+        "narHash": "sha256-oDqhUQ6Xg7a3xx537SWLGRzqP3oKKeyY4UYGCdz9z/Y=",
         "owner": "bats-core",
         "repo": "bats-assert",
-        "rev": "34551b1d7f8c7b677c1a66fc0ac140d6223409e5",
+        "rev": "e2d855bc78619ee15b0c702b5c30fb074101159f",
         "type": "github"
       },
       "original": {
@@ -42,11 +66,11 @@
     "bats-support": {
       "flake": false,
       "locked": {
-        "lastModified": 1548869839,
-        "narHash": "sha256-Gr4ntadr42F2Ks8Pte2D4wNDbijhujuoJi4OPZnTAZU=",
+        "lastModified": 1693050811,
+        "narHash": "sha256-PxJaH16+QrsfZqtkWVt5K6TwJB5gjIXnbGo+MB84WIU=",
         "owner": "bats-core",
         "repo": "bats-support",
-        "rev": "d140a65044b2d6810381935ae7f0c94c7023c8c3",
+        "rev": "9bf10e876dd6b624fe44423f0b35e064225f7556",
         "type": "github"
       },
       "original": {
@@ -64,35 +88,56 @@
         "treefmt-nix": "treefmt-nix"
       },
       "locked": {
-        "lastModified": 1721409873,
-        "narHash": "sha256-h0njWQRvtkjK0NJ/Kgj76sXBhWwq5HGJm7OMcigmNw4=",
-        "ref": "refs/heads/refactor",
-        "rev": "54bba654d4279dfd112345b6470547851feb1457",
-        "revCount": 267,
+        "lastModified": 1730064416,
+        "narHash": "sha256-Opbtu9hKijGkEx+GYbSu3MJms3lFxZmAGTFyckguWMM=",
+        "ref": "refs/heads/forkos",
+        "rev": "79137b14f3cb376204f739f44b05aebfc288ca89",
+        "revCount": 310,
         "type": "git",
         "url": "https://git.lix.systems/lix-project/buildbot-nix.git"
       },
       "original": {
-        "ref": "refs/heads/refactor",
+        "ref": "refs/heads/forkos",
         "type": "git",
         "url": "https://git.lix.systems/lix-project/buildbot-nix.git"
       }
     },
+    "channel-scripts": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1725128016,
+        "narHash": "sha256-4TvaXELsl+1OcGNgqB/5HVXVxBvdIQkhJsY4FyiDcNU=",
+        "ref": "refs/heads/main",
+        "rev": "23b6c38ed7e11417bf624f6e4fb6cde0d2be6400",
+        "revCount": 261,
+        "type": "git",
+        "url": "https://git.lix.systems/the-distro/channel-scripts.git"
+      },
+      "original": {
+        "type": "git",
+        "url": "https://git.lix.systems/the-distro/channel-scripts.git"
+      }
+    },
     "colmena": {
       "inputs": {
         "flake-compat": "flake-compat",
         "flake-utils": "flake-utils",
+        "nix-github-actions": "nix-github-actions",
         "nixpkgs": [
           "nixpkgs"
         ],
         "stable": "stable"
       },
       "locked": {
-        "lastModified": 1711386353,
-        "narHash": "sha256-gWEpb8Hybnoqb4O4tmpohGZk6+aerAbJpywKcFIiMlg=",
+        "lastModified": 1731527002,
+        "narHash": "sha256-dI9I6suECoIAmbS4xcrqF8r2pbmed8WWm5LIF1yWPw8=",
         "owner": "zhaofengli",
         "repo": "colmena",
-        "rev": "cd65ef7a25cdc75052fbd04b120aeb066c3881db",
+        "rev": "e3ad42138015fcdf2524518dd564a13145c72ea1",
         "type": "github"
       },
       "original": {
@@ -101,6 +146,44 @@
         "type": "github"
       }
     },
+    "crane": {
+      "inputs": {
+        "nixpkgs": [
+          "grapevine",
+          "attic",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1722960479,
+        "narHash": "sha256-NhCkJJQhD5GUib8zN9JrmYGMwt4lCRp6ZVNzIiYCl0Y=",
+        "owner": "ipetkov",
+        "repo": "crane",
+        "rev": "4c6c77920b8d44cd6660c1621dea6b3fc4b4c4f4",
+        "type": "github"
+      },
+      "original": {
+        "owner": "ipetkov",
+        "repo": "crane",
+        "type": "github"
+      }
+    },
+    "crane_2": {
+      "locked": {
+        "lastModified": 1730060262,
+        "narHash": "sha256-RMgSVkZ9H03sxC+Vh4jxtLTCzSjPq18UWpiM0gq6shQ=",
+        "owner": "ipetkov",
+        "repo": "crane",
+        "rev": "498d9f122c413ee1154e8131ace5a35a80d8fa76",
+        "type": "github"
+      },
+      "original": {
+        "owner": "ipetkov",
+        "ref": "master",
+        "repo": "crane",
+        "type": "github"
+      }
+    },
     "darwin": {
       "inputs": {
         "nixpkgs": [
@@ -123,6 +206,29 @@
         "type": "github"
       }
     },
+    "fenix": {
+      "inputs": {
+        "nixpkgs": [
+          "grapevine",
+          "nixpkgs"
+        ],
+        "rust-analyzer-src": "rust-analyzer-src"
+      },
+      "locked": {
+        "lastModified": 1730442928,
+        "narHash": "sha256-U1DWb5c3EfkA7pqx5V1H4AWRA+EaE6UJ0lIRvK1RxgM=",
+        "owner": "nix-community",
+        "repo": "fenix",
+        "rev": "87b4d20f896c99018dde4702a9c6157b516f2a76",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "ref": "main",
+        "repo": "fenix",
+        "type": "github"
+      }
+    },
     "flake-compat": {
       "flake": false,
       "locked": {
@@ -155,6 +261,39 @@
         "type": "github"
       }
     },
+    "flake-compat_3": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1696426674,
+        "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "ref": "master",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "flake-compat_4": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1696426674,
+        "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
     "flake-parts": {
       "inputs": {
         "nixpkgs-lib": [
@@ -179,17 +318,17 @@
     "flake-parts_2": {
       "inputs": {
         "nixpkgs-lib": [
-          "hydra",
-          "nix-eval-jobs",
+          "grapevine",
+          "attic",
           "nixpkgs"
         ]
       },
       "locked": {
-        "lastModified": 1719994518,
-        "narHash": "sha256-pQMhCCHyQGRzdfAkdJ4cIWiw+JNuWsTX7f0ZYSyz0VY=",
+        "lastModified": 1722555600,
+        "narHash": "sha256-XOQkdLafnb/p9ij77byFQjDf5m5QYl9b2REiVClC+x4=",
         "owner": "hercules-ci",
         "repo": "flake-parts",
-        "rev": "9227223f6d922fee3c7b190b2cc238a99527bbb7",
+        "rev": "8471fe90ad337a8074e957b69ca4d0089218391d",
         "type": "github"
       },
       "original": {
@@ -198,6 +337,45 @@
         "type": "github"
       }
     },
+    "flake-parts_3": {
+      "inputs": {
+        "nixpkgs-lib": [
+          "hydra",
+          "nix-eval-jobs",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1722555600,
+        "narHash": "sha256-XOQkdLafnb/p9ij77byFQjDf5m5QYl9b2REiVClC+x4=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "8471fe90ad337a8074e957b69ca4d0089218391d",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
+    "flake-parts_4": {
+      "inputs": {
+        "nixpkgs-lib": "nixpkgs-lib"
+      },
+      "locked": {
+        "lastModified": 1727826117,
+        "narHash": "sha256-K5ZLCyfO/Zj9mPFldf3iwS6oZStJcU4tSpiXTMYaaL0=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "3d04084d54bedc3d6b8b736c70ef449225c361b1",
+        "type": "github"
+      },
+      "original": {
+        "id": "flake-parts",
+        "type": "indirect"
+      }
+    },
     "flake-utils": {
       "locked": {
         "lastModified": 1659877975,
@@ -214,20 +392,70 @@
       }
     },
     "flake-utils_2": {
+      "inputs": {
+        "systems": "systems_2"
+      },
       "locked": {
-        "lastModified": 1634851050,
-        "narHash": "sha256-N83GlSGPJJdcqhUxSCS/WwW5pksYf3VP1M13cDRTSVA=",
+        "lastModified": 1726560853,
+        "narHash": "sha256-X6rJYSESBVr3hBoH0WbKE5KvhPU5bloyZ2L4K60/fPQ=",
         "owner": "numtide",
         "repo": "flake-utils",
-        "rev": "c91f3de5adaf1de973b797ef7485e441a65b8935",
+        "rev": "c1dfcf08411b08f6b8615f7d8971a2bfa81d5e8a",
         "type": "github"
       },
       "original": {
         "owner": "numtide",
+        "ref": "main",
         "repo": "flake-utils",
         "type": "github"
       }
     },
+    "gerrit-dashboard": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1724509518,
+        "narHash": "sha256-fwYXZVddxfzrlDa3QnFCwHqrbEX+3PrWy0QOlbO+8jk=",
+        "ref": "refs/heads/master",
+        "rev": "e544abac81c581558d68abb2a8dd583049073939",
+        "revCount": 75,
+        "type": "git",
+        "url": "https://git.lix.systems/the-distro/gerrit-monitoring.git"
+      },
+      "original": {
+        "type": "git",
+        "url": "https://git.lix.systems/the-distro/gerrit-monitoring.git"
+      }
+    },
+    "grapevine": {
+      "inputs": {
+        "attic": "attic",
+        "crane": "crane_2",
+        "fenix": "fenix",
+        "flake-compat": "flake-compat_3",
+        "flake-utils": "flake-utils_2",
+        "nix-filter": "nix-filter",
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "rocksdb": "rocksdb",
+        "rust-manifest": "rust-manifest"
+      },
+      "locked": {
+        "host": "gitlab.computer.surgery",
+        "lastModified": 1731820516,
+        "narHash": "sha256-BJiFq5meQMwOLhVS9/m2HCZoNUIN5u5weJXr94dkIRU=",
+        "owner": "matrix",
+        "repo": "grapevine-fork",
+        "rev": "887e59cf03efb70bfa68b56dda32c411ba6cd5fa",
+        "type": "gitlab"
+      },
+      "original": {
+        "host": "gitlab.computer.surgery",
+        "owner": "matrix",
+        "repo": "grapevine-fork",
+        "type": "gitlab"
+      }
+    },
     "home-manager": {
       "inputs": {
         "nixpkgs": [
@@ -258,11 +486,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1721682989,
-        "narHash": "sha256-kjJiZ7m4HKqbZ2mxNQiB32/goKFb8BRi8OqC4wIU0OI=",
+        "lastModified": 1728321752,
+        "narHash": "sha256-GbBAoBF7ZObz0IP+g0LZKxMafpMvNKjTEu9haiZbV54=",
         "ref": "refs/heads/main",
-        "rev": "4b107e6ff36bd89958fba36e0fe0340903e7cd13",
-        "revCount": 4190,
+        "rev": "ee1234c15cdcb427dbd4828e0add09d02cd606c9",
+        "revCount": 4220,
         "type": "git",
         "url": "https://git.lix.systems/lix-project/hydra.git"
       },
@@ -273,7 +501,7 @@
     },
     "lix": {
       "inputs": {
-        "flake-compat": "flake-compat_2",
+        "flake-compat": "flake-compat_4",
         "nix2container": "nix2container",
         "nixpkgs": [
           "hydra",
@@ -283,11 +511,11 @@
         "pre-commit-hooks": "pre-commit-hooks"
       },
       "locked": {
-        "lastModified": 1721091462,
-        "narHash": "sha256-0cmEeoOiB91BviTJHzIyxkY+Gxv3O8ZnnExVAoXEFGI=",
+        "lastModified": 1728163191,
+        "narHash": "sha256-SW0IEBsPN1EysqzvfDT+8Kimtzy03O1BxQQm7ZB6fRY=",
         "ref": "refs/heads/main",
-        "rev": "6b4d46e9e0e1dd80e0977684ab20d14bcd1a6bc3",
-        "revCount": 15967,
+        "rev": "ed9b7f4f84fd60ad8618645cc1bae2d686ff0db6",
+        "revCount": 16323,
         "type": "git",
         "url": "https://git.lix.systems/lix-project/lix"
       },
@@ -298,12 +526,12 @@
     },
     "nix-eval-jobs": {
       "inputs": {
-        "flake-parts": "flake-parts_2",
+        "flake-parts": "flake-parts_3",
         "lix": [
           "hydra",
           "lix"
         ],
-        "nix-github-actions": "nix-github-actions",
+        "nix-github-actions": "nix-github-actions_3",
         "nixpkgs": [
           "hydra",
           "nixpkgs"
@@ -311,11 +539,11 @@
         "treefmt-nix": "treefmt-nix_2"
       },
       "locked": {
-        "lastModified": 1721195872,
-        "narHash": "sha256-TlvRq634MSl22BWLmpTy2vdtKntbZlsUwdMq8Mp9AWs=",
+        "lastModified": 1723579251,
+        "narHash": "sha256-xnHtfw0gRhV+2S9U7hQwvp2klTy1Iv7FlMMO0/WiMVc=",
         "ref": "refs/heads/main",
-        "rev": "c057494450f2d1420726ddb0bab145a5ff4ddfdd",
-        "revCount": 608,
+        "rev": "42a160bce2fd9ffebc3809746bc80cc7208f9b08",
+        "revCount": 609,
         "type": "git",
         "url": "https://git.lix.systems/lix-project/nix-eval-jobs"
       },
@@ -324,6 +552,22 @@
         "url": "https://git.lix.systems/lix-project/nix-eval-jobs"
       }
     },
+    "nix-filter": {
+      "locked": {
+        "lastModified": 1730207686,
+        "narHash": "sha256-SCHiL+1f7q9TAnxpasriP6fMarWE5H43t25F5/9e28I=",
+        "owner": "numtide",
+        "repo": "nix-filter",
+        "rev": "776e68c1d014c3adde193a18db9d738458cd2ba4",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "ref": "main",
+        "repo": "nix-filter",
+        "type": "github"
+      }
+    },
     "nix-gerrit": {
       "inputs": {
         "nixpkgs": [
@@ -345,6 +589,49 @@
       }
     },
     "nix-github-actions": {
+      "inputs": {
+        "nixpkgs": [
+          "colmena",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1729742964,
+        "narHash": "sha256-B4mzTcQ0FZHdpeWcpDYPERtyjJd/NIuaQ9+BV1h+MpA=",
+        "owner": "nix-community",
+        "repo": "nix-github-actions",
+        "rev": "e04df33f62cdcf93d73e9a04142464753a16db67",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "nix-github-actions",
+        "type": "github"
+      }
+    },
+    "nix-github-actions_2": {
+      "inputs": {
+        "nixpkgs": [
+          "grapevine",
+          "attic",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1729742964,
+        "narHash": "sha256-B4mzTcQ0FZHdpeWcpDYPERtyjJd/NIuaQ9+BV1h+MpA=",
+        "owner": "nix-community",
+        "repo": "nix-github-actions",
+        "rev": "e04df33f62cdcf93d73e9a04142464753a16db67",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "nix-github-actions",
+        "type": "github"
+      }
+    },
+    "nix-github-actions_3": {
       "inputs": {
         "nixpkgs": [
           "hydra",
@@ -369,11 +656,11 @@
     "nix2container": {
       "flake": false,
       "locked": {
-        "lastModified": 1712990762,
-        "narHash": "sha256-hO9W3w7NcnYeX8u8cleHiSpK2YJo7ecarFTUlbybl7k=",
+        "lastModified": 1720642556,
+        "narHash": "sha256-qsnqk13UmREKmRT7c8hEnz26X3GFFyIQrqx4EaRc1Is=",
         "owner": "nlewo",
         "repo": "nix2container",
-        "rev": "20aad300c925639d5d6cbe30013c8357ce9f2a2e",
+        "rev": "3853e5caf9ad24103b13aa6e0e8bcebb47649fe4",
         "type": "github"
       },
       "original": {
@@ -384,11 +671,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1721116560,
-        "narHash": "sha256-++TYlGMAJM1Q+0nMVaWBSEvEUjRs7ZGiNQOpqbQApCU=",
+        "lastModified": 1726042813,
+        "narHash": "sha256-LnNKCCxnwgF+575y0pxUdlGZBO/ru1CtGHIqQVfvjlA=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "9355fa86e6f27422963132c2c9aeedb0fb963d93",
+        "rev": "159be5db480d1df880a0135ca0bfed84c2f88353",
         "type": "github"
       },
       "original": {
@@ -398,6 +685,18 @@
         "type": "github"
       }
     },
+    "nixpkgs-lib": {
+      "locked": {
+        "lastModified": 1727825735,
+        "narHash": "sha256-0xHYkMkeLVQAMa7gvkddbPqpxph+hDzdu1XdGPJR+Os=",
+        "type": "tarball",
+        "url": "https://github.com/NixOS/nixpkgs/archive/fb192fec7cc7a4c26d51779e9bab07ce6fa5597a.tar.gz"
+      },
+      "original": {
+        "type": "tarball",
+        "url": "https://github.com/NixOS/nixpkgs/archive/fb192fec7cc7a4c26d51779e9bab07ce6fa5597a.tar.gz"
+      }
+    },
     "nixpkgs-regression": {
       "locked": {
         "lastModified": 1643052045,
@@ -414,17 +713,34 @@
         "type": "github"
       }
     },
-    "nixpkgs_2": {
+    "nixpkgs-stable": {
       "locked": {
-        "lastModified": 1636823747,
-        "narHash": "sha256-oWo1nElRAOZqEf90Yek2ixdHyjD+gqtS/pAgwaQ9UhQ=",
-        "owner": "nixos",
+        "lastModified": 1724316499,
+        "narHash": "sha256-Qb9MhKBUTCfWg/wqqaxt89Xfi6qTD3XpTzQ9eXi3JmE=",
+        "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "f6a2ed2082d9a51668c86ba27d0b5496f7a2ea93",
+        "rev": "797f7dc49e0bc7fab4b57c021cdf68f595e47841",
         "type": "github"
       },
       "original": {
-        "owner": "nixos",
+        "owner": "NixOS",
+        "ref": "nixos-24.05",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs_2": {
+      "locked": {
+        "lastModified": 1731676054,
+        "narHash": "sha256-OZiZ3m8SCMfh3B6bfGC/Bm4x3qc1m2SVEAlkV6iY7Yg=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "5e4fbfb6b3de1aa2872b76d49fafc942626e2add",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-unstable",
         "repo": "nixpkgs",
         "type": "github"
       }
@@ -432,11 +748,11 @@
     "pre-commit-hooks": {
       "flake": false,
       "locked": {
-        "lastModified": 1712055707,
-        "narHash": "sha256-4XLvuSIDZJGS17xEwSrNuJLL7UjDYKGJSbK1WWX2AK8=",
+        "lastModified": 1721042469,
+        "narHash": "sha256-6FPUl7HVtvRHCCBQne7Ylp4p+dpP3P/OYuzjztZ4s70=",
         "owner": "cachix",
         "repo": "git-hooks.nix",
-        "rev": "e35aed5fda3cc79f88ed7f1795021e559582093a",
+        "rev": "f451c19376071a90d8c58ab1a953c6e9840527fd",
         "type": "github"
       },
       "original": {
@@ -445,37 +761,103 @@
         "type": "github"
       }
     },
+    "rocksdb": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1730475155,
+        "narHash": "sha256-u5uuShM2SxHc9/zL4UU56IhCcR/ZQbzde0LgOYS44bM=",
+        "owner": "facebook",
+        "repo": "rocksdb",
+        "rev": "3c27a3dde0993210c5cc30d99717093f7537916f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "facebook",
+        "ref": "v9.7.4",
+        "repo": "rocksdb",
+        "type": "github"
+      }
+    },
     "root": {
       "inputs": {
         "agenix": "agenix",
         "buildbot-nix": "buildbot-nix",
+        "channel-scripts": "channel-scripts",
         "colmena": "colmena",
+        "gerrit-dashboard": "gerrit-dashboard",
+        "grapevine": "grapevine",
         "hydra": "hydra",
         "lix": [
           "hydra",
           "lix"
         ],
         "nix-gerrit": "nix-gerrit",
-        "nixpkgs": "nixpkgs",
+        "nixpkgs": "nixpkgs_2",
+        "stateless-uptime-kuma": "stateless-uptime-kuma",
         "terranix": "terranix"
       }
     },
+    "rust-analyzer-src": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1730386175,
+        "narHash": "sha256-0Uq+/B8eu7pw8B8pxuGdFYKjcVLwNMcHfDxU9sXh7rg=",
+        "owner": "rust-lang",
+        "repo": "rust-analyzer",
+        "rev": "0ba893e1a00d92557ac91efb771d72eee36ca687",
+        "type": "github"
+      },
+      "original": {
+        "owner": "rust-lang",
+        "ref": "nightly",
+        "repo": "rust-analyzer",
+        "type": "github"
+      }
+    },
+    "rust-manifest": {
+      "flake": false,
+      "locked": {
+        "narHash": "sha256-tB9BZB6nRHDk5ELIVlGYlIjViLKBjQl52nC1avhcCwA=",
+        "type": "file",
+        "url": "https://static.rust-lang.org/dist/channel-rust-1.81.0.toml"
+      },
+      "original": {
+        "type": "file",
+        "url": "https://static.rust-lang.org/dist/channel-rust-1.81.0.toml"
+      }
+    },
     "stable": {
       "locked": {
-        "lastModified": 1696039360,
-        "narHash": "sha256-g7nIUV4uq1TOVeVIDEZLb005suTWCUjSY0zYOlSBsyE=",
+        "lastModified": 1730883749,
+        "narHash": "sha256-mwrFF0vElHJP8X3pFCByJR365Q2463ATp2qGIrDUdlE=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "32dcb45f66c0487e92db8303a798ebc548cadedc",
+        "rev": "dba414932936fde69f0606b4f1d87c5bc0003ede",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-23.05",
+        "ref": "nixos-24.05",
         "repo": "nixpkgs",
         "type": "github"
       }
     },
+    "stateless-uptime-kuma": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1728243069,
+        "narHash": "sha256-l9fgwesnmFxasCaYUCD7L9bGGJXytLuwtx3CZMgpwJg=",
+        "ref": "refs/heads/master",
+        "rev": "880f444ff7862d6127b051cf1a993ad1585b1652",
+        "revCount": 25,
+        "type": "git",
+        "url": "https://git.dgnum.eu/DGNum/stateless-uptime-kuma.git"
+      },
+      "original": {
+        "type": "git",
+        "url": "https://git.dgnum.eu/DGNum/stateless-uptime-kuma.git"
+      }
+    },
     "systems": {
       "locked": {
         "lastModified": 1681028828,
@@ -491,20 +873,53 @@
         "type": "github"
       }
     },
+    "systems_2": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "systems_3": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
     "terranix": {
       "inputs": {
         "bats-assert": "bats-assert",
         "bats-support": "bats-support",
-        "flake-utils": "flake-utils_2",
-        "nixpkgs": "nixpkgs_2",
+        "flake-parts": "flake-parts_4",
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "systems": "systems_3",
         "terranix-examples": "terranix-examples"
       },
       "locked": {
-        "lastModified": 1695406838,
-        "narHash": "sha256-xiUfVD6rtsVWFotVtUW3Q1nQh4obKzgvpN1wqZuGXvM=",
+        "lastModified": 1728959489,
+        "narHash": "sha256-1Pu2j5xsBTuoyga08ZVf+rKp3FOMmJh/0fXen/idOrA=",
         "owner": "terranix",
         "repo": "terranix",
-        "rev": "fc9077ca02ab5681935dbf0ecd725c4d889b9275",
+        "rev": "7734e2ee6a1472807a33ce1e7da794bed2aaf91c",
         "type": "github"
       },
       "original": {
@@ -515,11 +930,11 @@
     },
     "terranix-examples": {
       "locked": {
-        "lastModified": 1636300201,
-        "narHash": "sha256-0n1je1WpiR6XfCsvi8ZK7GrpEnMl+DpwhWaO1949Vbc=",
+        "lastModified": 1637156952,
+        "narHash": "sha256-KqvXIe1yiKOEP9BRYqNQN+LOWPCsWojh0WjEgv5jfEI=",
         "owner": "terranix",
         "repo": "terranix-examples",
-        "rev": "a934aa1cf88f6bd6c6ddb4c77b77ec6e1660bd5e",
+        "rev": "921680efb8af0f332d8ad73718d53907f9483e24",
         "type": "github"
       },
       "original": {
@@ -558,11 +973,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1721059077,
-        "narHash": "sha256-gCICMMX7VMSKKt99giDDtRLkHJ0cwSgBtDijJAqTlto=",
+        "lastModified": 1723454642,
+        "narHash": "sha256-S0Gvsenh0II7EAaoc9158ZB4vYyuycvMGKGxIbERNAM=",
         "owner": "numtide",
         "repo": "treefmt-nix",
-        "rev": "0fb28f237f83295b4dd05e342f333b447c097398",
+        "rev": "349de7bc435bdff37785c2466f054ed1766173be",
         "type": "github"
       },
       "original": {

flake.nix

@@ -2,8 +2,10 @@
   description = "Bagel cooking infrastructure";
 
   inputs = {
-    nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
+
     terranix.url = "github:terranix/terranix";
+    terranix.inputs.nixpkgs.follows = "nixpkgs";
 
     agenix.url = "github:ryantm/agenix";
     agenix.inputs.nixpkgs.follows = "nixpkgs";
@@ -17,38 +19,65 @@
     nix-gerrit.url = "git+https://git.lix.systems/the-distro/nix-gerrit.git";
     nix-gerrit.inputs.nixpkgs.follows = "nixpkgs";
 
-    buildbot-nix.url = "git+https://git.lix.systems/lix-project/buildbot-nix.git?ref=refs/heads/refactor";
+    gerrit-dashboard.url = "git+https://git.lix.systems/the-distro/gerrit-monitoring.git";
+    gerrit-dashboard.flake = false;
+
+    buildbot-nix.url = "git+https://git.lix.systems/lix-project/buildbot-nix.git?ref=refs/heads/forkos";
     buildbot-nix.inputs.nixpkgs.follows = "nixpkgs";
 
+    channel-scripts.url = "git+https://git.lix.systems/the-distro/channel-scripts.git";
+    channel-scripts.inputs.nixpkgs.follows = "nixpkgs";
+
+    stateless-uptime-kuma.url = "git+https://git.dgnum.eu/DGNum/stateless-uptime-kuma.git";
+    stateless-uptime-kuma.flake = false;
+
     lix.follows = "hydra/lix";
+
+    grapevine = {
+      type = "gitlab";
+      host = "gitlab.computer.surgery";
+      owner = "matrix";
+      repo = "grapevine-fork";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
   };
 
   outputs = { self, nixpkgs, terranix, colmena, ... } @ inputs:
   let
-    system = "x86_64-linux";
+    supportedSystems = [ "x86_64-linux" "aarch64-linux" ];
+    forEachSystem = f: builtins.listToAttrs (map (system: {
+      name = system;
+      value = f system;
+    }) supportedSystems);
+    systemBits = forEachSystem (system: rec {
+      inherit system;
       pkgs = import nixpkgs {
         localSystem = system;
         overlays = [
           inputs.hydra.overlays.default
           inputs.lix.overlays.default
           inputs.nix-gerrit.overlays.default
+          inputs.channel-scripts.overlays.default
+          (import "${inputs.stateless-uptime-kuma}/overlay.nix")
         ];
       };
-    lib = pkgs.lib;
       terraform = pkgs.opentofu;
       terraformCfg = terranix.lib.terranixConfiguration {
         inherit system;
         modules = [
           ./terraform
           {
-          bagel.gandi.enable = true;
+            bagel.dnsimple.enable = true;
             bagel.hydra.enable = true;
           }
         ];
       };
+    });
+    forEachSystem' = f: forEachSystem (system: (f systemBits.${system}));
+    inherit (nixpkgs) lib;
   in
   {
-    apps.${system} = {
+    apps = forEachSystem' ({ system, pkgs, terraformCfg, terraform, ... }: {
       tf = {
         type = "app";
         program = toString (pkgs.writers.writeBash "tf" ''
@@ -59,16 +88,19 @@
       };
 
       default = self.apps.${system}.tf;
-    };
+    });
 
-    devShells.${system}.default = pkgs.mkShell {
+    devShells = forEachSystem' ({ system, pkgs, ... }: {
+      default = pkgs.mkShell {
         packages = [
           inputs.agenix.packages.${system}.agenix
 
-        pkgs.colmena
           pkgs.opentofu
+
+          (pkgs.callPackage ./lib/colmena-wrapper.nix { })
         ];
       };
+    });
 
     nixosConfigurations = (colmena.lib.makeHive self.outputs.colmena).nodes;
 
@@ -83,33 +115,93 @@
         ./common
       ];
 
-      makeBuilder = i: lib.nameValuePair "builder-${toString i}" {
-        imports = commonModules;
-        bagel.baremetal.builders = { enable = true; num = i; };
+      floralInfraModules = commonModules ++ [
+        ({ config, lib, ... }: {
+          # This means that anyone with @floral-infra permissions
+          # can ssh on root of every machines handled here.
+          bagel.admins.allowedGroups = [
+            "floral-infra"
+          ];
+
+          # Tag all machines which have local boot as local bootables.
+          deployment.tags = lib.mkMerge [
+            [ "floral" ]
+            (lib.mkIf (config.bagel.baremetal.builders.enable -> !config.bagel.baremetal.builders.netboot)
+              [ "localboot" ]
+            )
+          ];
+
+          bagel.monitoring.grafana-agent.tenant = "floral";
+          bagel.secrets.tenant = "floral";
+          bagel.builders.extra-build-capacity.provider.tenant = "floral";
+          bagel.services.buildbot.tenant = "floral";
+        })
+      ];
+
+      # These are Floral baremetal builders.
+      makeBuilder = i:
+      let
+        enableNetboot = i >= 6;
+      in
+      lib.nameValuePair "builder-${toString i}" {
+        imports = floralInfraModules;
+        bagel.baremetal.builders = { enable = true; num = i; netboot = enableNetboot; };
       };
 
-      builders = lib.listToAttrs (lib.genList makeBuilder 12);
-    in {
-      meta.nixpkgs = import nixpkgs {
-        localSystem = system;
-        overlays = [
-          inputs.hydra.overlays.default
-          inputs.lix.overlays.default
-          inputs.nix-gerrit.overlays.default
+      lixInfraModules = commonModules ++ [
+        {
+          # This means that anyone with @lix-infra permissions
+          # can ssh on root of every machines handled here.
+          bagel.admins.allowedGroups = [
+            "lix-infra"
+          ];
+
+          # Tag all machines which have local boot as local bootables.
+          # Lix has no netbootable machine.
+          deployment.tags = [ "localboot" "lix" ];
+
+          bagel.monitoring.grafana-agent.tenant = "lix";
+          bagel.secrets.tenant = "lix";
+          bagel.builders.extra-build-capacity.provider = {
+            tenant = "lix";
+            buildfarmPublicKeys = [
+              # buildbot.lix.systems SSH key
+              "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDu4cEqZzAI/1vZjSQkTJ4ijIg9nuloOuSKUrnkJIOFn"
             ];
           };
+          bagel.services.buildbot.tenant = "lix";
+        }
+      ];
+
+      builders = lib.listToAttrs (lib.genList makeBuilder 11);
+    in {
+      meta.nixpkgs = systemBits.x86_64-linux.pkgs;
+      # Add any non-x86_64 native systems here.
+      # Cross compilation is not supported yet.
+      meta.nodeNixpkgs =
+        let
+          aarch64-systems = systems: lib.genAttrs systems (system: systemBits.aarch64-linux.pkgs);
+        in
+        aarch64-systems [
+          "build01-aarch64-lix"
+        ];
       meta.specialArgs.inputs = inputs;
 
-      bagel-box.imports = commonModules ++ [ ./hosts/bagel-box ];
-      meta01.imports = commonModules ++ [ ./hosts/meta01 ];
-      gerrit01.imports = commonModules ++ [ ./hosts/gerrit01 ];
-      fodwatch.imports = commonModules ++ [ ./hosts/fodwatch ];
-      git.imports = commonModules ++ [ ./hosts/git ];
-      wob-vpn-gw.imports = commonModules ++ [ ./hosts/wob-vpn-gw ];
-      buildbot.imports = commonModules ++ [ ./hosts/buildbot ];
+      bagel-box.imports = floralInfraModules ++ [ ./hosts/bagel-box ];
+      meta01.imports = floralInfraModules ++ [ ./hosts/meta01 ];
+      gerrit01.imports = floralInfraModules ++ [ ./hosts/gerrit01 ];
+      fodwatch.imports = floralInfraModules ++ [ ./hosts/fodwatch ];
+      git.imports = floralInfraModules ++ [ ./hosts/git ];
+      wob-vpn-gw.imports = floralInfraModules ++ [ ./hosts/wob-vpn-gw ];
+      buildbot.imports = floralInfraModules ++ [ ./hosts/buildbot ];
+      public01.imports = floralInfraModules ++ [ ./hosts/public01 ];
+      build-coord.imports = floralInfraModules ++ [ ./hosts/build-coord ];
+
+      build01-aarch64-lix.imports = lixInfraModules ++ [ ./hosts/build01-aarch64-lix ];
+      buildbot-lix.imports = lixInfraModules ++ [ ./hosts/buildbot-lix ];
     } // builders;
 
-    hydraJobs = builtins.mapAttrs (n: v: v.config.system.build.toplevel) self.nixosConfigurations;
+    hydraJobs = builtins.mapAttrs (n: v: v.config.system.build.netbootDir or v.config.system.build.toplevel) self.nixosConfigurations;
     buildbotJobs = builtins.mapAttrs (_: v: v.config.system.build.toplevel) self.nixosConfigurations;
   };
 }

hosts/bagel-box/default.nix

@@ -37,20 +37,11 @@
 
   bagel.services = {
     postgres.enable = true;
-
-    hydra.enable = true;
-    hydra.dbi = "dbi:Pg:dbname=hydra;user=hydra";
-    # Takes 10 builders (0 → 9).
-    hydra.builders = lib.genList (i: "builder-${builtins.toString i}") 10;
-
     ofborg.enable = true;
   };
 
   bagel.sysadmin.enable = true;
 
-  security.acme.acceptTerms = true;
-  security.acme.defaults.email = "infra@forkos.org";
-
   services.openssh.enable = true;
 
   system.stateVersion = "24.11";

hosts/build-coord/default.nix

@@ -0,0 +1,23 @@
+{ lib, ... }:
+{
+  imports = [ ./hardware.nix ];
+
+  networking.hostName = "build-coord";
+  networking.domain = "wob01.infra.forkos.org";
+
+  bagel.sysadmin.enable = true;
+
+  bagel.services = {
+    hydra.enable = true;
+    # Takes 10 builders (0 → 9).
+    hydra.builders = lib.genList (i: "builder-${builtins.toString i}") 10;
+  };
+
+  bagel.monitoring.exporters.hydra.enable = true;
+
+  # Hydra is proxied.
+  bagel.raito.v6-proxy-awareness.enable = true;
+
+  system.stateVersion = "24.05";
+  deployment.targetHost = "build-coord.wob01.infra.forkos.org";
+}

hosts/build-coord/hardware.nix

@@ -0,0 +1,87 @@
+{
+  boot.initrd.availableKernelModules = [ "ahci" "ehci_pci" "usb_storage" "usbhid" "sd_mod" ];
+  boot.initrd.kernelModules = [ "dm-snapshot" ];
+
+  nixpkgs.hostPlatform = "x86_64-linux";
+  hardware.cpu.intel.updateMicrocode = true;
+
+  boot.loader.systemd-boot.enable = true;
+  boot.loader.efi.canTouchEfiVariables = true;
+  boot.initrd.systemd.enable = true;
+
+  boot.initrd.services.lvm.enable = true;
+
+  boot.kernelParams = [
+    "console=tty1"
+    "console=ttyS0,115200"
+  ];
+
+  fileSystems = {
+    "/" = {
+      device = "/dev/disk/by-label/root";
+      fsType = "ext4";
+    };
+
+    "/boot" = {
+      device = "/dev/disk/by-label/BOOT";
+      fsType = "vfat";
+      options = [ "fmask=0022" "dmask=0022" ];
+    };
+  };
+
+  swapDevices = [
+    {
+      device = "/swapfile";
+      size = 20 * 1024;  # 50GiB
+    }
+  ];
+
+  zramSwap = {
+    enable = true;
+    memoryPercent = 100;
+  };
+
+  networking.useNetworkd = true;
+
+  systemd.network = {
+    netdevs = {
+      "40-uplink" = {
+        netdevConfig = {
+          Kind = "bond";
+          Name = "uplink";
+        };
+        bondConfig = {
+          Mode = "802.3ad";
+          TransmitHashPolicy = "layer3+4";
+        };
+      };
+    };
+    networks = {
+      "40-eno1" = {
+        name = "eno1";
+        bond = [ "uplink" ];
+      };
+      "40-eno2" = {
+        name = "eno2";
+        bond = [ "uplink" ];
+      };
+    };
+  };
+  networking.interfaces.uplink.ipv6.addresses = [
+    { address = "2a01:584:11::1:11"; prefixLength = 64; }
+  ];
+  networking.defaultGateway6 = { interface = "uplink"; address = "2a01:584:11::1"; };
+
+  services.coredns = {
+    enable = true;
+    config = ''
+      . {
+        bind lo
+        forward . 2001:4860:4860::6464
+        template ANY A { rcode NOERROR }
+      }
+    '';
+  };
+  services.resolved.enable = false;
+  networking.resolvconf.useLocalResolver = true;
+}

hosts/build01-aarch64-lix/default.nix

@@ -0,0 +1,27 @@
+{ ... }: {
+  networking.hostName = "build01";
+  networking.domain = "aarch64.lix.systems";
+
+  # Those free sweet VMs.
+  bagel.hardware.oracle-vm = {
+    enable = true;
+    system = "aarch64-linux";
+  };
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/a333323c-99f0-4258-8f68-496858d56f71";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/3E74-C937";
+      fsType = "vfat";
+    };
+
+  swapDevices = [ ];
+
+  bagel.builders.extra-build-capacity.provider.enable = true;
+  i18n.defaultLocale = "en_US.UTF-8";
+  system.stateVersion = "24.05";
+  deployment.targetHost = "build01.aarch64.lix.systems";
+}

hosts/buildbot-lix/default.nix

@@ -0,0 +1,71 @@
+# Configuration for a virtual machine in Raito's micro-DC basement.
+# 32 vCPU (2014 grade Xeon though)
+# 32GB RAM
+# 30GB SSD
+# 500GB HDD
+# All specifications can be upgraded to a certain extent, just ask Raito.
+# Hosts the coordinator for Buildbot.
+#
+# vim: et:ts=2:sw=2:
+#
+{ lib, modulesPath, ... }: {
+  networking.hostName = "buildbot";
+  networking.domain = "lix.systems";
+
+  zramSwap.enable = true;
+
+  bagel.sysadmin.enable = true;
+  # Buildbot is proxied.
+  bagel.raito.v6-proxy-awareness.enable = true;
+  bagel.hardware.raito-vm = {
+    enable = true;
+    networking = {
+      nat-lan-mac = "BC:24:11:75:62:42";
+      wan = {
+        mac = "BC:24:11:B2:5F:2E";
+        address = "2001:bc8:38ee:100::200/56";
+      };
+    };
+  };
+  i18n.defaultLocale = "en_US.UTF-8";
+
+  bagel.services.buildbot = {
+    enable = true;
+    domain = "buildbot.lix.systems";
+    gerrit =
+    {
+      domain = "gerrit.lix.systems";
+      port = 2022;
+      username = "buildbot";
+    };
+    cors.allowedOrigins = [
+      "https://*.lix.systems"
+    ];
+    projects = [
+      "lix"
+      "lix-installer"
+    ];
+    buildSystems = [
+      "x86_64-linux"
+      "aarch64-linux"
+      "aarch64-darwin"
+      # Too slow.
+      /* "x86_64-darwin" */
+    ];
+    # Lix is not allowed to use yet Floral's x86_64 builders for now.
+    builders = [ ];
+  };
+
+  # This machine does not use /nix from btrfs, and instead uses a store on a bigger disk.
+  fileSystems."/nix" =
+    lib.mkForce
+    { device = "/dev/disk/by-uuid/1815ca49-d0b0-4b99-8aec-0d790498ba6f";
+      fsType = "xfs";
+      neededForBoot = true;
+      options = [ "relatime" ];
+    };
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+  system.stateVersion = "24.05";
+  deployment.targetHost = "buildbot.lix.systems";
+}

hosts/buildbot/default.nix

@@ -2,6 +2,7 @@
   config,
   lib,
   pkgs,
+  nodes,
   ...
 }:
 {
@@ -9,8 +10,6 @@
   # TODO: make it the default
   networking.domain = "infra.forkos.org";
 
-  time.timeZone = "Europe/Paris";
-
   bagel.sysadmin.enable = true;
   # Buildbot is proxied.
   bagel.raito.v6-proxy-awareness.enable = true;
@@ -28,7 +27,24 @@
   bagel.services.buildbot = {
     enable = true;
     domain = "buildbot.forkos.org";
-    builders = [ "builder-3" ];
+    gerrit =
+      let
+        cfgGerrit = nodes.gerrit01.config.bagel.services.gerrit;
+      in
+      {
+        domain = cfgGerrit.canonicalDomain;
+        port = cfgGerrit.port;
+        username = "buildbot";
+      };
+    cors.allowedOrigins = [
+      "https://*.forkos.org"
+    ];
+    projects = [
+      "buildbot-test"
+      "nixpkgs"
+      "infra"
+    ];
+    builders = [ "builder-9" ];
   };
 
   i18n.defaultLocale = "en_US.UTF-8";

hosts/fodwatch/default.nix

@@ -8,8 +8,6 @@
   networking.hostName = "fodwatch";
   networking.domain = "infra.forkos.org";
 
-  time.timeZone = "Europe/Paris";
-
   bagel.sysadmin.enable = true;
   # Fodwatch will be proxied.
   bagel.raito.v6-proxy-awareness.enable = true;

hosts/gerrit01/default.nix

@@ -9,8 +9,6 @@
   # TODO: make it the default
   networking.domain = "infra.forkos.org";
 
-  time.timeZone = "Europe/Paris";
-
   bagel.sysadmin.enable = true;
   # Gerrit is proxied.
   bagel.raito.v6-proxy-awareness.enable = true;
@@ -25,6 +23,9 @@
     };
   };
 
+  # Block all these crawlers!!
+  bagel.services.nginx.crawler-blocker.enable = true;
+
   fileSystems."/gerrit-data" = {
     device = "/dev/disk/by-uuid/d1062305-0dea-4740-9a27-b6b1691862a4";
     fsType = "ext4";
@@ -32,6 +33,7 @@
 
   bagel.services.gerrit = {
     enable = true;
+    pyroscope.enable = true;
     domains = [
       "cl.forkos.org"
     ];
@@ -40,19 +42,20 @@
   };
 
   age.secrets.ows-deploy-key = {
-    file = ../../secrets/ows-deploy-key.age;
+    file = ../../secrets/floral/ows-deploy-key.age;
     mode = "0600";
     owner = "git";
     group = "git";
   };
   bagel.nixpkgs.one-way-sync =
   let
-    mkNixpkgsJob = { timer, branchName }: {
-      name = "nixpkgs-${branchName}";
+    mkNixpkgsJob = { timer, fromRefspec, localRefspec ? fromRefspec }: {
       fromUri = "https://github.com/NixOS/nixpkgs";
-      fromRefspec = branchName;
-      localRefspec = branchName;
-      inherit timer;
+      inherit fromRefspec localRefspec timer;
+    };
+    mkLocalJob = { timer, fromRefspec, localRefspec }: {
+      fromUri = "https://cl.forkos.org/nixpkgs";
+      inherit fromRefspec localRefspec timer;
     };
     in
   {
@@ -61,47 +64,59 @@
     pushUrl = "ssh://ows_bot@cl.forkos.org:29418/nixpkgs";
     deployKeyPath = config.age.secrets.ows-deploy-key.path;
 
-    branches."refs/heads/main" = mkNixpkgsJob {
-      timer = "hourly";
-      branchName = "main";
+    # Sync main -> staging-next -> staging
+    branches."main-to-staging-next" = mkLocalJob {
+      timer = "00/8:20:00"; # every 8 hours, 20 minutes past the full hour
+      fromRefspec = "main";
+      localRefspec = "staging-next";
+    };
+    branches."staging-next-to-staging" = mkLocalJob {
+      timer = "00/8:40:00"; # every 8 hours, 40 minutes past the full hour
+      fromRefspec = "staging-next";
+      localRefspec = "staging";
     };
 
-    branches."refs/heads/staging" = mkNixpkgsJob {
+    # Sync nixpkgs -> fork
+    branches."nixpkgs-master" = mkNixpkgsJob {
       timer = "hourly";
-      branchName = "staging";
+      fromRefspec = "master";
+      localRefspec = "main";
     };
 
-    branches."refs/heads/release-24.05" = mkNixpkgsJob {
+    branches."nixpkgs-staging" = mkNixpkgsJob {
       timer = "hourly";
-      branchName = "release-24.05";
+      fromRefspec = "staging";
     };
 
-    branches."refs/heads/staging-24.05" = mkNixpkgsJob {
+    branches."nixpkgs-release-24.05" = mkNixpkgsJob {
       timer = "hourly";
-      branchName = "staging-24.05";
+      fromRefspec = "release-24.05";
     };
 
-    branches."refs/heads/release-23.11" = mkNixpkgsJob {
+    branches."nixpkgs-staging-24.05" = mkNixpkgsJob {
       timer = "hourly";
-      branchName = "release-23.11";
+      fromRefspec = "staging-24.05";
     };
 
-    branches."refs/heads/staging-23.11" = mkNixpkgsJob {
+    branches."nixpkgs-release-23.11" = mkNixpkgsJob {
       timer = "hourly";
-      branchName = "staging-23.11";
+      fromRefspec = "release-23.11";
+    };
+
+    branches."nixpkgs-staging-23.11" = mkNixpkgsJob {
+      timer = "hourly";
+      fromRefspec = "staging-23.11";
     };
 
     # Testing jobs for personal sandbox branches
-    branches."refs/heads/sandbox/raito/raito-unstable-small" = {
-      name = "raito-unstable-sync";
+    branches."raito-unstable-sync" = {
       fromUri = "https://github.com/NixOS/nixpkgs";
       fromRefspec = "nixos-unstable-small";
       localRefspec = "sandbox/raito/raito-unstable-small";
       timer = "*-*-* 12:00:00";
     };
 
-    branches."refs/heads/sandbox/raito/raito-nixos-24.05" = {
-      name = "raito-release-sync";
+    branches."raito-release-sync" = {
       fromUri = "https://github.com/NixOS/nixpkgs";
       fromRefspec = "nixos-24.05";
       localRefspec = "sandbox/raito/raito-nixos-24.05";
@@ -109,6 +124,26 @@
     };
   };
 
+  age.secrets.s3-channel-staging-keys.file = ../../secrets/floral/s3-channel-staging-keys.age;
+  bagel.nixpkgs.channel-scripts = {
+    enable = true;
+    otlp.enable = true;
+    nixpkgsUrl = "https://cl.forkos.org/nixpkgs.git";
+    hydraUrl = "https://hydra.forkos.org";
+    binaryCacheUrl = "https://cache.forkos.org";
+    baseUriForGitRevisions = "https://cl.forkos.org/plugins/gitiles/nixpkgs/+";
+    s3 = {
+      release = "bagel-channel-scripts-test";
+      channel = "bagel-channel-scripts-test";
+    };
+    releaseBucketCredentialsFile = config.age.secrets.s3-channel-staging-keys.path;
+    deployKeyFile = config.age.secrets.priv-ssh-key.path;
+    extraArgs = [
+      "--bypass-preflight-checks"
+    ];
+    channels = import ../../common/channels.nix;
+  };
+
   i18n.defaultLocale = "fr_FR.UTF-8";
 
   system.stateVersion = "24.05";

hosts/git/default.nix

@@ -8,8 +8,6 @@ in
   networking.hostName = "git";
   networking.domain = "infra.forkos.org";
 
-  time.timeZone = "Europe/Paris";
-
   bagel.sysadmin.enable = true;
   # Forgejo will be proxied.
   bagel.raito.v6-proxy-awareness.enable = true;

hosts/meta01/default.nix

@@ -2,8 +2,6 @@
   networking.hostName = "meta01";
   networking.domain = "infra.forkos.org";
 
-  time.timeZone = "Europe/Paris";
-
   bagel.sysadmin.enable = true;
   # netbox is proxied.
   bagel.raito.v6-proxy-awareness.enable = true;
@@ -24,6 +22,15 @@
   bagel.services.prometheus.enable = true;
   bagel.services.loki.enable = true;
   bagel.services.grafana.enable = true;
+  bagel.services.grapevine.enable = true;
+  bagel.services.pyroscope.enable = true;
+  bagel.services.tempo.enable = true;
+  bagel.services.hookshot = {
+    enable = true;
+    admins = [
+      "@k900:0upti.me"
+    ];
+  };
 
   i18n.defaultLocale = "fr_FR.UTF-8";
 

hosts/public01/default.nix

@@ -0,0 +1,50 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+{
+  networking.hostName = "public01";
+  # TODO: make it the default
+  networking.domain = "infra.forkos.org";
+
+  bagel.status = {
+    enable = true;
+    domain = "status.forkos.org";
+  };
+
+  bagel.sysadmin.enable = true;
+  # Newsletter is proxied.
+  bagel.raito.v6-proxy-awareness.enable = true;
+  bagel.newsletter = {
+    enable = true;
+    domain = "news.forkos.org";
+  };
+  bagel.hardware.raito-vm = {
+    enable = true;
+    networking = {
+      nat-lan-mac = "BC:24:11:A4:F7:D3";
+      wan = {
+        address = "2001:bc8:38ee:100:1000::60/64";
+        mac = "BC:24:11:DB:B8:10";
+      };
+    };
+  };
+
+  bagel.services.s3-revproxy = {
+    enable = true;
+    domain = "forkos.org";
+    s3.apiUrl = "s3.delroth.net";
+    targets = {
+      channels = "bagel-channels";
+      releases = "bagel-releases";
+      channel-scripts-test = "bagel-channel-scripts-test";
+    };
+  };
+
+  i18n.defaultLocale = "en_US.UTF-8";
+
+  system.stateVersion = "24.05";
+  deployment.targetHost = "public01.infra.forkos.org";
+}

hosts/wob-vpn-gw/default.nix

@@ -1,6 +1,10 @@
 { pkgs, lib, ... }:
 
 {
+  imports = [
+    ./netboot.nix
+  ];
+
   ###### Hardware ######
   boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "ehci_pci" "sd_mod" "sdhci_pci" ];
   boot.kernelModules = [ "kvm-amd" ];

hosts/wob-vpn-gw/netboot.nix

@@ -0,0 +1,61 @@
+{ config, lib, pkgs, nodes, modulesPath, ... }:
+
+# The way the connection is established is specific to the wob01 site and the Intel S2600KPR blades.
+# Proper netboot is not possible, because while the blades and the APU board (which is the netboot
+# server here) are in the same L2 network, the uplink connection of each blade is an LACP LAG,
+# meaning that the switch on the other side will only enable the port if it sees valid LACP packets.
+# We work around this by presenting a virtual floppy drive using the "IUSB" protocol of the BMC.
+# This virtual floppy drive contains an per-blade customized initramfs which will initialize the
+# network connection including IP configuration and load the actual image off hydra.
+
+let
+  netboot-server-ip = "2a01:584:11::2";
+  netbootNodes = lib.filterAttrs (_: node: node.config.bagel.baremetal.builders.enable && node.config.bagel.baremetal.builders.netboot) nodes;
+in {
+
+  assertions = [
+    {
+      assertion = !(lib.elem 443 config.networking.firewall.allowedTCPPorts);
+      message = ''
+        Port 443 is in networking.firewalls.allowedTCPPorts, but should be only manually
+        allowed for specific IPs and source ports in ${builtins.toJSON __curPos}
+      '';
+    }
+  ];
+
+  systemd.services = lib.mapAttrs' (nodename: node: let
+    bmcIp = "192.168.1.${toString (node.config.bagel.baremetal.builders.num * 4 + 2)}";
+    notipxe = node.config.system.build.notipxe.config.system.build.usbImage;
+  in lib.nameValuePair "iusb-spoof-${nodename}" {
+    wantedBy = [ "multi-user.target" ];
+    serviceConfig = {
+      Restart = "always";
+    };
+    script = ''
+      AUTH_TOKEN=$(${pkgs.iusb-spoof}/bin/make-token ${bmcIp})
+      exec ${pkgs.iusb-spoof}/bin/iusb-spoof -r ${bmcIp} 5123 $AUTH_TOKEN ${notipxe}
+    '';
+  }) netbootNodes;
+
+  # Since the builders are stateless, they can not store their ssh hostkeys
+  networking.firewall.allowedTCPPorts = [ 80 ]; # for ACME
+  networking.firewall.extraInputRules = ''
+    ip6 saddr 2a01:584:11::/64 tcp sport < 1024 tcp dport 443 accept;
+  '';
+  services.nginx = {
+    enable = true;
+    virtualHosts."vpn-gw.wob01.infra.forkos.org" = {
+      enableACME = true;
+      forceSSL = true;
+      locations = lib.mapAttrs' (nodename: node: let
+        ip = "2a01:584:11::1:${toString node.config.bagel.baremetal.builders.num}";
+      in lib.nameValuePair "/${nodename}/" {
+        root = "/var/www";
+        extraConfig = ''
+          allow ${ip};
+          deny all;
+        '';
+      }) netbootNodes;
+    };
+  };
+}

lib/colmena-wrapper.nix

@@ -0,0 +1,14 @@
+# A wrapper for colmena that prevents accidentally deploying changes without
+# having pulled.
+{ colmena, runCommandNoCC }:
+runCommandNoCC "colmena-wrapper"
+{
+  env.colmena = "${colmena}/bin/colmena";
+} ''
+  mkdir -p $out
+  ln -s ${colmena}/share $out/share
+  mkdir $out/bin
+
+  substituteAll ${./colmena-wrapper.sh.in} $out/bin/colmena
+  chmod +x $out/bin/colmena
+''

lib/colmena-wrapper.sh.in

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+doChecks() {
+    # creates refs in the refs/prefetch/remotes/origin namespace
+    echo "Prefetching repo changes..." >&2
+    git fetch --quiet --prefetch --no-write-fetch-head origin
+
+    diffs=$(git rev-list --left-right --count HEAD...refs/prefetch/remotes/origin/main)
+    only_in_local=$(echo "$diffs" | cut -f1)
+    only_in_main=$(echo "$diffs" | cut -f2)
+
+    if [[ $only_in_main -gt 0 && ! -v $FOOTGUN_ME_UWU ]]; then
+        echo >&2
+        echo "Attempting to deploy when main has $only_in_main commits not in your branch!" >&2
+        echo "This will probably revert someone's changes. Consider merging them." >&2
+        echo "If you really mean it, set the environment variable FOOTGUN_ME_UWU" >&2
+        exit 1
+    fi
+
+    if [[ $only_in_local -gt 0 ]]; then
+        echo "You have $only_in_local commits not yet pushed to main. Reminder to push them after :)" >&2
+    fi
+}
+
+if [[ $1 == 'apply' ]]; then
+    doChecks
+fi
+
+exec @colmena@ "$@"

overlays/default.nix

@@ -1 +1,9 @@
-[]
+[
+  (final: prev: {
+    iusb-spoof = final.callPackage ./iusb-spoof.nix {};
+    u-root = final.callPackage ./u-root {};
+    pyroscope = final.callPackage ./pyroscope {};
+    s3-revproxy = final.callPackage ./s3-revproxy {};
+    git-gc-preserve = final.callPackage ./git-gc-preserve {};
+  })
+]

overlays/git-gc-preserve/default.nix

@@ -0,0 +1,9 @@
+{ writeShellApplication, git, nettools }:
+
+writeShellApplication {
+  name = "git-gc-preserve";
+
+  runtimeInputs = [ git nettools ];
+
+  text = (builtins.readFile ./script.sh);
+}

overlays/git-gc-preserve/script.sh

@@ -0,0 +1,132 @@
+#!/usr/bin/env bash
+
+set +o errexit
+# Copyright (C) 2022 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+usage() { # exit code
+  cat <<-EOF
+NAME
+    git-gc-preserve - Run git gc and preserve old packs to avoid races for JGit
+SYNOPSIS
+    git gc-preserve
+DESCRIPTION
+    Runs git gc and can preserve old packs to avoid races with concurrently
+    executed commands in JGit.
+    This command uses custom git config options to configure if preserved packs
+    from the last run of git gc should be pruned and if packs should be preserved.
+    This is similar to the implementation in JGit [1] which is used by
+    JGit to avoid errors [2] in such situations.
+    The command prevents concurrent runs of the command on the same repository
+    by acquiring an exclusive file lock on the file
+      "\$repopath/gc-preserve.pid"
+    If it cannot acquire the lock it fails immediately with exit code 3.
+    Failure Exit Codes
+        1: General failure
+        2: Couldn't determine repository path. If the current working directory
+           is outside of the working tree of the git repository use git option
+           --git-dir to pass the root path of the repository.
+           E.g.
+              $ git --git-dir ~/git/foo gc-preserve
+        3: Another process already runs $0 on the same repository
+    [1] https://git.eclipse.org/r/c/jgit/jgit/+/87969
+    [2] https://git.eclipse.org/r/c/jgit/jgit/+/122288
+CONFIGURATION
+    "pack.prunepreserved": if set to "true" preserved packs from the last gc run
+      are pruned before current packs are preserved.
+    "pack.preserveoldpacks": if set to "true" current packs will be hard linked
+      to objects/pack/preserved before git gc is executed. JGit will
+      fallback to the preserved packs in this directory in case it comes
+      across missing objects which might be caused by a concurrent run of
+      git gc.
+EOF
+  exit "$1"
+}
+# acquire file lock, unlock when the script exits
+lock() { # repo
+  readonly LOCKFILE="$1/gc-preserve.pid"
+  test -f "$LOCKFILE" || touch "$LOCKFILE"
+  exec 9> "$LOCKFILE"
+  if flock -nx 9; then
+    echo -n "$$ $USER@$(hostname)" >&9
+    trap unlock EXIT
+  else
+    echo "$0 is already running"
+    exit 3
+  fi
+}
+unlock() {
+  # only delete if the file descriptor 9 is open
+  if { : >&9 ; } &> /dev/null; then
+    rm -f "$LOCKFILE"
+  fi
+  # close the file handle to release file lock
+  exec 9>&-
+}
+# prune preserved packs if pack.prunepreserved == true
+prune_preserved() { # repo
+  configured=$(git --git-dir="$1" config --get pack.prunepreserved)
+  if [ "$configured" != "true" ]; then
+    return 0
+  fi
+  local preserved=$1/objects/pack/preserved
+  if [ -d "$preserved" ]; then
+    printf "Pruning old preserved packs: "
+    count=$(find "$preserved" -name "*.old-pack" | wc -l)
+    rm -rf "$preserved"
+    echo "$count, done."
+  fi
+}
+# preserve packs if pack.preserveoldpacks == true
+preserve_packs() { # repo
+  configured=$(git --git-dir="$1" config --get pack.preserveoldpacks)
+  if [ "$configured" != "true" ]; then
+    return 0
+  fi
+  local packdir=$1/objects/pack
+  pushd "$packdir" >/dev/null || exit 1
+  mkdir -p preserved
+  printf "Preserving packs: "
+  count=0
+  for file in pack-*{.pack,.idx} ; do
+    ln -f "$file" preserved/"$(get_preserved_packfile_name "$file")"
+    if [[ "$file" == pack-*.pack ]]; then
+      ((count++))
+    fi
+  done
+  echo "$count, done."
+  popd >/dev/null || exit 1
+}
+# pack-0...2.pack to pack-0...2.old-pack
+# pack-0...2.idx to pack-0...2.old-idx
+get_preserved_packfile_name() { # packfile > preserved_packfile
+  local old=${1/%\.pack/.old-pack}
+  old=${old/%\.idx/.old-idx}
+  echo "$old"
+}
+# main
+while [ $# -gt 0 ] ; do
+    case "$1" in
+        -u|-h)  usage 0 ;;
+    esac
+    shift
+done
+args=$(git rev-parse --sq-quote "$@")
+repopath=$(git rev-parse --git-dir)
+if [ -z "$repopath" ]; then
+  usage 2
+fi
+lock "$repopath"
+prune_preserved "$repopath"
+preserve_packs "$repopath"
+git gc ${args:+"$args"} || { EXIT_CODE="$?"; echo "git gc failed"; exit "$EXIT_CODE"; }

overlays/iusb-spoof.nix

@@ -0,0 +1,23 @@
+{ rustPlatform, python3, makeWrapper }:
+let
+  pythonEnv = python3.withPackages (p: with p; [ requests ]);
+in
+
+rustPlatform.buildRustPackage rec {
+  pname = "iusb-spoof";
+  version = "0.1.0";
+
+  src = builtins.fetchGit {
+    url = "https://git.lix.systems/the-distro/iusb-spoof/";
+    rev = "fafd47986239cc2f4dfbbae74b17555608806581";
+  };
+
+  cargoLock.lockFile = src + "/Cargo.lock";
+
+  nativeBuildInputs = [ makeWrapper ];
+
+  postInstall = ''
+    install -Dm644 $src/make-token.py $out/opt/make-token.py
+    makeWrapper ${pythonEnv.interpreter} $out/bin/make-token --add-flags "$out/opt/make-token.py"
+  '';
+}

overlays/pyroscope/default.nix

@@ -0,0 +1,43 @@
+{ lib
+, buildGo122Module
+, fetchFromGitHub
+}:
+
+# FIXME: update, remove this pin
+buildGo122Module rec {
+  pname = "pyroscope";
+  version = "1.7.1";
+
+  src = fetchFromGitHub {
+    owner = "grafana";
+    repo = "pyroscope";
+    rev = "v${version}";
+    hash = "sha256-iMP67J0Q8Cgo52iImMzAM3PEkk6uLF7r6v9TyXZVaIE=";
+  };
+
+  env.GOWORK = "off";
+
+  vendorHash = "sha256-ggntpnU9s2rpkv6S0LnZNexrdkBsdsUrGPc93SVrK4M=";
+
+  subPackages = [ "cmd/profilecli" "cmd/pyroscope" ];
+
+  ldflags = [
+    "-extldflags"
+    "-static"
+    "-s"
+    "-w"
+    "-X=github.com/grafana/pyroscope/pkg/util/build.Branch=${src.rev}"
+    "-X=github.com/grafana/pyroscope/pkg/util/build.Version=${version}"
+    "-X=github.com/grafana/pyroscope/pkg/util/build.Revision=${src.rev}"
+    "-X=github.com/grafana/pyroscope/pkg/util/build.BuildDate=1970-01-01T00:00:00Z"
+  ];
+
+  meta = with lib; {
+    description = "Continuous profiling platform";
+    homepage = "https://github.com/grafana/pyroscope";
+    changelog = "https://github.com/grafana/pyroscope/blob/${src.rev}/CHANGELOG.md";
+    license = licenses.agpl3Only;
+    maintainers = with maintainers; [ raitobezarius ];
+    mainProgram = "pyroscope";
+  };
+}

overlays/s3-revproxy/default.nix

@@ -0,0 +1,39 @@
+# Originally written by Jade Lovelace for Lix.
+{ lib, buildGoModule, fetchFromGitHub }:
+buildGoModule rec {
+  pname = "s3-revproxy";
+  version = "4.15.0";
+
+  src = fetchFromGitHub {
+    owner = "oxyno-zeta";
+    repo = "s3-proxy";
+    rev = "v${version}";
+    hash = "sha256-q0cfAo8Uz7wtKljmSDaJ320bjg2yXydvvxubAsMKzbc=";
+  };
+
+  vendorHash = "sha256-dOwNQtTfOCQcjgNBV/FeWdwbW9xi1OK5YD7PBPPDKOQ=";
+
+  ldflags = [
+    "-X github.com/oxyno-zeta/s3-proxy/pkg/s3-proxy/version.Version=${version}"
+    "-X github.com/oxyno-zeta/s3-proxy/pkg/s3-proxy/version.Metadata="
+  ];
+
+  postPatch = ''
+    # Refer to the included templates in the package instead of cwd-relative
+    sed -i "s#Path = \"templates/#Path = \"$out/share/s3-revproxy/templates/#" pkg/s3-proxy/config/config.go
+  '';
+
+  postInstall = ''
+    mkdir -p $out/share/s3-revproxy
+    cp -r templates/ $out/share/s3-revproxy/templates
+  '';
+
+  meta = {
+    description = "S3 Reverse Proxy with GET, PUT and DELETE methods and authentication (OpenID Connect and Basic Auth)";
+    homepage = "https://oxyno-zeta.github.io/s3-proxy";
+    # hm, not having a maintainers entry is kind of inconvenient
+    maintainers = [ ];
+    licenses = lib.licenses.asl20;
+    mainProgram = "s3-proxy";
+  };
+}

overlays/u-root/default.nix

@@ -0,0 +1,20 @@
+{ buildGoModule, fetchFromGitHub }:
+
+buildGoModule rec {
+  pname = "u-root";
+  version = "0.14.0";
+
+  src = fetchFromGitHub {
+    owner = "u-root";
+    repo = "u-root";
+    rev = "v${version}";
+    hash = "sha256-8zA3pHf45MdUcq/MA/mf0KCTxB1viHieU/oigYwIPgo=";
+  };
+
+  patches = [
+    ./u-root-allow-https.patch
+  ];
+
+  vendorHash = null;
+  doCheck = false;
+}

overlays/u-root/u-root-allow-https.patch

@@ -0,0 +1,12 @@
+diff --git a/pkg/curl/schemes.go b/pkg/curl/schemes.go
+index 8bac3bc0..cd396cbc 100644
+--- a/pkg/curl/schemes.go
++++ b/pkg/curl/schemes.go
+@@ -81,6 +81,7 @@ var (
+ 	DefaultSchemes = Schemes{
+ 		"tftp": DefaultTFTPClient,
+ 		"http": DefaultHTTPClient,
++		"https": DefaultHTTPClient,
+ 		"file": &LocalFileClient{},
+ 	}
+ )

pkgs/forgejo/default.nix

@@ -1,6 +1,6 @@
-{ forgejo }:
+{ forgejo-lts }:
 
-forgejo.overrideAttrs (prev: {
+forgejo-lts.overrideAttrs (prev: {
   patches = [
     # Branch divergence calculations for a single branch may take 100-200ms on something as big
     # as nixpkgs. The branch view defaults to 20 branches for each page, taking roughtly 3s to

secrets.nix

@@ -1,17 +1,29 @@
 let
   keys = import common/ssh-keys.nix;
 
-  commonKeys = keys.users.delroth ++ keys.users.raito;
+  commonKeys = {
+    # WARNING: `keys.users.*` are *lists*, so you need concatenate them, don't put them into lists!
+    # Otherwise, agenix will be confused!
+    global = keys.users.raito;
+    lix = keys.users.hexchen ++ keys.users.jade;
+    floral = keys.users.delroth;
+  };
 
   secrets = with keys; {
-    hydra-s3-credentials = [ machines.bagel-box ];
-    hydra-signing-priv = [ machines.bagel-box ];
-    hydra-ssh-key-priv = [ machines.bagel-box ];
+    floral = {
+      hydra-postgres-key = [ machines.build-coord ];
+      hydra-s3-credentials = [ machines.build-coord ];
+      hydra-signing-priv = [ machines.build-coord ];
+      hydra-ssh-key-priv = [ machines.build-coord ];
+
       netbox-environment = [ machines.meta01 ];
       mimir-environment = [ machines.meta01 ];
+      mimir-webhook-url = [ machines.meta01 ];
       grafana-oauth-secret = [ machines.meta01 ];
       loki-environment = [ machines.meta01 ];
       gerrit-prometheus-bearer-token = [ machines.gerrit01 machines.meta01 ];
+      pyroscope-secrets = [ machines.meta01 ];
+      tempo-environment = [ machines.meta01 ];
 
       buildbot-worker-password = [ machines.buildbot ];
       buildbot-oauth-secret = [ machines.buildbot ];
@@ -25,14 +37,40 @@ let
 
       # These are the same password, but nginx wants it in htpasswd format
       metrics-push-htpasswd = [ machines.meta01 ];
+      # Yes, even Lix machines are included in this monitoring infrastructure.
       metrics-push-password = builtins.attrValues machines;
 
       ows-deploy-key = [ machines.gerrit01 ];
+      s3-channel-staging-keys = [ machines.gerrit01 ];
+      s3-channel-keys = [ machines.gerrit01 ];
+
+      postgres-ca-priv = [ machines.bagel-box ];
+      postgres-tls-priv = [ machines.bagel-box ];
+
+      newsletter-secrets = [ machines.public01 ];
+      s3-revproxy-api-keys = [ machines.public01 ];
+      stateless-uptime-kuma-password = [ machines.public01 ];
     };
+
+    lix = {
+      buildbot-worker-password = [ machines.buildbot-lix ];
+      buildbot-oauth-secret = [ machines.buildbot-lix ];
+      buildbot-workers = [ machines.buildbot-lix ];
+      # Private SSH key to Gerrit
+      # ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHx52RUPWzTa2rBA96xcnGjjzAboNN/hm6gW+Q6JiSos
+      buildbot-service-key = [ machines.buildbot-lix ];
+      # Signing key for Buildbot's specific cache
+      buildbot-signing-key = [ machines.buildbot-lix ];
+      buildbot-remote-builder-key = [ machines.buildbot-lix ];
+    };
+  };
+
+  mkSecretListFor = tenant:
+    map (secretName: {
+      name = "secrets/${tenant}/${secretName}.age";
+      value.publicKeys = secrets.${tenant}."${secretName}" ++ commonKeys.global ++ commonKeys.${tenant};
+    }) (builtins.attrNames secrets.${tenant});
 in
   builtins.listToAttrs (
-    map (secretName: {
-      name = "secrets/${secretName}.age";
-      value.publicKeys = secrets."${secretName}" ++ commonKeys;
-    }) (builtins.attrNames secrets)
+    (mkSecretListFor "floral") ++ (mkSecretListFor "lix")
   )

secrets/buildbot-oauth-secret.age

@@ -1,20 +0,0 @@
-age-encryption.org/v1
--> ssh-ed25519 87T2Ig g15A5EWi9IhaxPFS6SD6YYm/aFnC0Dum7zK8/ZUtW0s
-791D6C8mAy2dhDAlqRQ+q41FlQTJX2WfZQPjuwetP2A
--> ssh-ed25519 K3b7BA cJY9qIFVmucmMJLTFffkRCNYeudZl+8Yrm5SkxQ4eSI
-97nXyKffZGoGJ6252UKUEJHiFgdk8XUkAAkXy2PLepM
--> ssh-ed25519 +qVung HMBSUjfmaFLVx64epj0djkqNMe3CdKN1fxAVuu+Dtmg
-AxT62n2p/pP9WZmmuHClSKKgXhr4FjEQpEs0HfdNGfw
--> ssh-rsa krWCLQ
-N0Duz2bONcCUZ76QhPsCJ4BHHWqzFdZLqFdl+6GeW+tgIp2Nb4la8eNfgzYGSwTy
-53bRePNMIBTkChXFYt/4fUdqaiiVYg25swMeVLQBJnjJkcAks0Gf44FXLIaoPr1M
-56rtixpSX31WDKwHbUF/40G6Xut8KNlI8BdwiOl9ibgnuEf4mYQbwFbRQbLMK5IK
-Rf/7SEmAqqfY/HG1RqqgCs4kEpvFTKqEEDpgjOoyS2tyKN2351jya91YzotLja4I
-sLoMg/G3UNtxfdaCgK7TP4IxV9blkVMDPAbyR622VbS0sEa7uJGzb86jDDsZXaKX
-9iWK9n4hMKZDv9gBbhTIWg
--> ssh-ed25519 /vwQcQ hMkCrUcLGxdZMYgi1D1Kr5qUdGNfza2UTvRJKiHObgM
-7Lz70zSMPk/tsU1CZGOk/BPA7NSSnSJgFbG5TjyOXvA
--> ssh-ed25519 0R97PA OQjDTknVmrYVclcqlT31YjZx+3a/0GxfjuVQFmPJ7UQ
-KMGTMfO/mO5EAYacyz1hmHnQgzunRqkDeglhbGVNWe4
---- ScDZvSiVSjNXm8TSoLSAM+KpcFORnCXiemYbCBcz2jQ
-™ŸÄhÜ}E¹ÊœËíUÌùᢌƒÿ…<C3BF>é™k¢ág[<5B>ñCƒ"<22>–NÛj•u5«<0C>ÄCXÕöÈGt¡TOmñ

secrets/buildbot-worker-password.age

@@ -1,20 +0,0 @@
-age-encryption.org/v1
--> ssh-ed25519 87T2Ig df+IMqWM/HNjaY74zibFQIdUdC3K7uQlm3U9R9NUtFY
-hPSbCuWvqy/7FEj7YScYztyt5GVx4Y7tgGuKKkSKoRg
--> ssh-ed25519 K3b7BA xN8wzUKHqjOb/tqA+EI+0H0MSQRihRfydchwVqYWAVU
-maLMpZe8orvTT6Av+YkhT8FcG4dc7bzDgOW339nSw1g
--> ssh-ed25519 +qVung oM1uphTbjI54t4U9jNd1zORqpjBG17MwDf2eNDmOlkg
-oUHVuQt2SHIwtV82pgnKJ7g2jcVBAHWOzPK46otoh34
--> ssh-rsa krWCLQ
-eYspf5hUKdFQl1RxPaNTj0viAPd+kzp8Xbwn+q6fSITMacmyTY5J8FckLx2YXDxy
-Qm/OsEK0ZOvxnHMrL0oAJjKSy/MamE+9heT3QO+LUN30QxbOIOqHMrl3waadWZdx
-ZGOWK+r+dKGYNsxFv+t1Y/4DBKKzlXFWhJ0aL7nMOqq9+Ca+UZuE41j7eWGGPPLy
-fuW/iOVVxQ+EEeCDpatQSrFPKaeWCCVP9oIDFtE4dsKxubMa4EpUoag0UvEIW182
-UGS8BvMqYgx+obqJDkhXXBK9apmJS2ojcfdtCbNOCV9Ett72Nm/iY5NjLprFMLde
-8wWGA6s3hBOP39lq0eiSxw
--> ssh-ed25519 /vwQcQ 3zLcLDaDVhIn2knezexYM5Fqu/O9wwORnJIhsXHqgj0
-HchGikQMgkDj0qQgtDdsdKokV+nMjdv6t0uVISeU7Q8
--> ssh-ed25519 0R97PA 6lm6B6B3dzSdhdcf5rjyTu+7cCtWRxVpWeapJX3nbQo
-x/w4dEfFyxPi4lbNEqgjEblPVfQyj+q1JjeQHiVFhDw
---- oo5BK1pG+43amUg803Uv511RNtdQ/PDwlXUrV/AbOAA
-…ÙUqÆçïµ[f7ƒêŒë¼¨FìˆY<13>™Ùm¶ØLS?Úℶ‡÷ƒöæ<Kø©F¤z¥V^³U¨N»¯ôƒ)zÔ<7A>¥ž@<40>SÀF€Y‡ËG2^žƒ˜à„»N|

secrets/floral/buildbot-oauth-secret.age

@@ -0,0 +1,20 @@
+age-encryption.org/v1
+-> ssh-ed25519 87T2Ig tzPD1x6XKuDfgJ8jkQnwW/ALp2pkANCeNoO8xdUqq30
+QSsuO6Dwc8QJuY92gXRnWB5aJ2SU9X2uFh01GmLVaQE
+-> ssh-ed25519 K3b7BA 9G9Uw1xY8hq//xphNWrPn5y7vG2o8/kwkC8cJGuf/mI
+Ip0019OUaFq2ZDFI3i77hdsp9IqFV2qqYIB/TnDSXgo
+-> ssh-ed25519 +qVung dx22ef+x9X5mr73L8NUzxYQa640M2XViELjJcpgF3go
+CXyit7pk8SPNHBgULlMQUAasGAn4C36zcwOBDI46nU4
+-> ssh-rsa krWCLQ
+NlGh0hM10NOuek7MbrFo0iul0kQQtDFmZIhgpyqaATMdCDRBXJOyhASHU5N0zDDJ
+MLaJUV0l2o1ghBF9RhSKdoUPVEn8Cce/nfQepYzMlfc4UG3qWXwabwR6EtqqCZCJ
+jAEWZ8taTKDmzoXwuygCW+bRBuoMMrcfzu7V90N+mQpZWtOScatb6E7d5VRqjlar
+st1ZQu5ccghufyQSUmOC7GpojOyutX5EvbMGn84X4ouZRHRX/8fTgaqicV+aeAIb
+QyXisOrO6C+Jle5qfxzMSe8c/TCyF2574kD6F1BQ9Kpkinn8v7OWcIXtkNmZ5hzK
+vs0Bej8yZVsoBkj1vWAM0A
+-> ssh-ed25519 /vwQcQ n+hr1cV1zRs1S86YnA+0oRB8SCaPKtkoMNe15ZsVVwM
+fdFtUqno07ik6FpW5zMImIjd8wM8dMgwU+RqjeT2PiI
+-> ssh-ed25519 0R97PA ddPILw57gkuKvAqlmpa+MnV/LSEdyQzQaAarCUqQ1xE
+ozK5a6uXZDc17OrX0OZun9hmZwP3H3rYQiNuKnukqsg
+--- f7yGgKQpCPj64Ps0HfMcToYircGH5SPqMzVZrUMB8ZI
+føv[iY\ÅšMP,¯Ùh°Èxb—Ðÿ«J<C2AB>*ºË"”+¬ÒA0T˜?KmˆPÈË2¹'2±‚µ³Ø=¯êÚÏŸj”

secrets/floral/buildbot-worker-password.age

@@ -0,0 +1,20 @@
+age-encryption.org/v1
+-> ssh-ed25519 87T2Ig p8lEB5da4fIfLH/HKBsghzq5mvQLB69UB4+uAi3DGCw
+NeZ3jPTUKa7MiqjrFPrYuP4VneytQPdBNqf+omPZJYM
+-> ssh-ed25519 K3b7BA uP2K1hU7uLmiHXmmoUdsB7CHQq61ZkEAjG/aK863RDw
+0chTczEMXASdYiwqNxDQ+vMXXhjOf64oIQ2ULZmQI8Y
+-> ssh-ed25519 +qVung jUgEqz3+ypL7mwJ1R7lfeOMhkon/aRrNSJUJT3X7vmU
+pgOiwrp9JiA20yw9bsxi8eiQ9/23CYXKRBGF1pea9eI
+-> ssh-rsa krWCLQ
+snCHrLHzkjimwIxKO90IjnHwOArlozO9kd/aCdZZnYNgh/QG3rUSceSn9yTHbtMV
+izv0SU51LrRU+JyE+a524AxKhyPBvGDig20j7hMy5fVxZqeunztqtlha5gaYYaQg
+Tbfs9tDP+pCIgzMVNqYf6EJ4MK7qjNf9DE5I490Eta5YZxAi/3To3BmZmIYtCz6l
+1kNRiSmWCbZqE25keFgPCgRMFXAFK9W6NmL+HamqCUhjPoJg/Gd4sf39EONT0PYg
+7BpCOAnwwfECHPxpM3qv0h2kJXTb4DZ715cFReSVyQe5fvKv8hoWhl/S+++pEYT8
+u/LKBx/o7e3Kd7cm2RGnBw
+-> ssh-ed25519 /vwQcQ 4+IQPRsMMHmuSGL7T7IbRkTTuL+TTqgdQp5FSbyt8Dw
+KOI0LKQ0oA5XtxaW7wftlEJB0BGVnx41HUJMG92SRUA
+-> ssh-ed25519 0R97PA l1aWUEv8nLEtYnpY1gjTJqk5UYm51NDqOjYmL83rZ10
+B7qDZwCpolkIajqCXeOepwmF6ciJfKvr+AN7VouMUvA
+--- lz/IMMPxBpD3Bzuv9Wl23+swBQHlblhlAO/ZXAgN0hU
+µoÍüÌ<EFBFBD>²-‚Īr °eó|Í?ït
èìÎZ<C38E>¬sÒì!ŸƒÁ<>@Ï'–ìèz6UöÎg
JøÑOµ–s13<31>š‹8<î’%-·Ô‡Eÿ}–Šdm9¿å¢Óoæ

secrets/floral/grafana-oauth-secret.age

@@ -0,0 +1,20 @@
+age-encryption.org/v1
+-> ssh-ed25519 j2r2qQ vwcaLpvGJ9swXnV8idDwi9jdRPSj38As9p2QFkIJ1Xc
+FLnZeblHDQQcWjFm1iaghbvuFgOG3miwtkRE5sz1+X0
+-> ssh-ed25519 K3b7BA 9VRe2rBwg3G9lxxfxL/yLob2NZmLJTBMxzx0Ew8VwmY
+/I2W80UykNvll5o98OPeMpIsddOel9B7uQlio0X3gcs
+-> ssh-ed25519 +qVung VsqKzMD85aps4PIx2zqae2Dj7YWibiaKYb5z7ws8ggM
+Y9dRd/hOz8h4avlutBQ1YZgHIAf/AuTr5WaByKlFbLE
+-> ssh-rsa krWCLQ
+gjyaUFrIIbZnFTGVw4XEZzkTIP/+qXV6/q0W8Wb4EtqQXDRISFT+bwxQU/S2p5hf
+7+JGcn4BZg6puOJ5BBABWtpn6gcX5OFfga5azIdioF/R19XByT+0SK5njw8g1VPS
+R7o8kQt2yvKWayoq9Cis5XRg+4KANkwOQaNTO8AdiCwgq9nc0Cd9avk8QhaFoR74
+D5cf8jPsufp744rQqwhWDoG533LS1WUUuYZqRmtp2Vz+r583RhSscaNyA7ddr7o6
+e9ZQJyL5bKiN8qe3Xm76lLypf/wg7+aGn8HHnO6GA65g+VYfjLMODEqCN/+uDJtB
+g8v2wzKIGYlZiV1hEjH8nw
+-> ssh-ed25519 /vwQcQ 4pU5JGK5vpZbFgq01a9YY8VmSJvPSHPSZD50TLJwKHc
+L46UA/p+bNSR8cLmL8G7VpmAcZ+sy5AROc4yj2ABOWg
+-> ssh-ed25519 0R97PA Tk00kYLhsEy1HJcmKLgaLWTdNP8XV/cdKHMLzyK6glk
+kwyQZr/h6MutROJmjVfPWGcf9xN5Uc5w5mVyuKcK64g
+--- E0vVtBqbjNkZY0/1dFJ53uVAR7IGPO+OMmXkpJcKmlw
+{ÐQê%è‹õY•B,isr¥1‘<31>|¼yLÕ'7?¶iŠM…¶MU]×ê/d2¸I1u¶2hZjHåh&¥

secrets/floral/hydra-s3-credentials.age

@@ -0,0 +1,20 @@
+age-encryption.org/v1
+-> ssh-ed25519 +uvEmw Kuduppyhz98frjlV96R/WcmPdaWmHbNKZhQs76GGTx8
+7zpedlPflGOi6FqkpswAJflx77yde7M2XlTw/8mz2tU
+-> ssh-ed25519 K3b7BA TPNmUK38+TR58MpsgxIe91bY6E1j9HecIFn0AKdat20
+MjXh06xd3mkPcK+iEonx+itsHvEGHSknzO6Sgh5WdZ4
+-> ssh-ed25519 +qVung KereHQ3Kl0f7O0xzl2s4Yu4KX7OOA17R7p/w8uRx/AQ
+3aOqUoBCDurkh5jT2fq5MDPQKIDISQdXBhF+qeRppnA
+-> ssh-rsa krWCLQ
+TVlmM2LYR339Aglo2D4j/Itr0E+mg7UEaV9n+sUYyit0phWS1zMI2YMc78Xbmn73
+6U0VYi/3hpesD6/8uA5sywuueOMntlL32aECz/DJPC71feMjvHTxiJpqnFw6DQYJ
+FvERtvJ2U7QiStv6UeS1vOucP1/om0Qj4smTXBWYsDglTSLx56/bghCsM21RNZZb
+yd8JE5CEdtCHduj+uRHbnEYsnGYM7R/Gw9XAuajFLw6BxqEtHi5xOivQ2P0Tm+Bc
+SVHW48iF8S3q1tx5QU7oIMZcCobOeHb6w+C1GHiSeJy3R8hWkEwfNxCCc0rSItKd
+edqO4YPz/zT2DWoUx+n0Og
+-> ssh-ed25519 /vwQcQ TLa0Xty2LlqBiP9Lk0lC+S/BoVT+VbRhY3qPHIGf20E
+3mzqkwT8dvP11GAVJiVIc+MiN/pLP2b6KbC+1F86tg8
+-> ssh-ed25519 0R97PA pbGz7e6nU4M4cpJRmmxWxUV3O2rWytIP18M7OpMpa04
+doSBv72rqS5gNusMjKw8KwzXHbzoLlFUSdLqp/f5aRM
+--- beE6zNg+kY7jke/79FGZoNTq7Wbe3eqNWvLD3igQJdg
+bÓÿ¾ ¾ÿÔáïØÉ
mÿŠ$Ÿ–‡¸’Ý={6
eÿ¨SIîumT8”äÿÁ{òo<C3B2>Ë3Ý&¾12¡p¶	œZ÷³4zx¡8B˜¤a@â(_Þ7(‚Ñ^tµä±Á‘<C381>g“Ùà4Ö*&Á`*
øäNšB÷$ÉÆ	ò£§m<C2A7>"ÓtD$Î5‡k
u˜eÎ+X@LJüç£h2ü£Þ*H¥ ÂåaÚ¢

secrets/floral/hydra-signing-priv.age

@@ -0,0 +1,20 @@
+age-encryption.org/v1
+-> ssh-ed25519 +uvEmw TNYFQxSUv5yMmlTWoIxCOlv6UR+RA50cb5aJbo0yEE0
+Yw3sTPqYf7A33RI87CqoPWe2gh0FuvdBGGKqHV55Atc
+-> ssh-ed25519 K3b7BA vMlHenY6jSIfnxQD6xh09cwwV+YVBkLuSMHcyKD+dCk
+heXkAEqRawBlHqcr6ldmhWmk7qPtGLMDFC3QT79vdMM
+-> ssh-ed25519 +qVung fgimLW5X0z4Eh2u3fIr5bgR5/c1SKam9CKW/2mqtTik
+8VKJJr+FRE0j5YvjfdMXugNA4UwUebKrkeAe+9LYBnQ
+-> ssh-rsa krWCLQ
+sa3fA4GglovY8H6jimpTvQPW/axun8WADPlIXzpX/Zeshkzem+pQoQqptzDlnmH8
+8AngqXgFYrmHgNNAylavgcrxbjNrtlJU24ldF1YIubz7VsU1678F27LCd9B0c2dn
+X+0CccH19lM8Q+zVI2Wrq9R83MEP/5uOOc+eXXnvNSGqfKgZ2OplG/HUllFS13j6
+uiQy5zwJJKkII7KUThcGteux7NONoLeUqRE8CW2uSeY9fXBWKgxeENKgiT7PEAAo
+nvwWa+GatEYf6eUz8Lph8lETorgP+7JS2VQRAkmhDbjQLTYzfFmiJGE/mzyobslf
+ZEq6Oj5UNgnzdWmK5ZYKPg
+-> ssh-ed25519 /vwQcQ 9EG/cydlzlLd6cFed7DzmwzubzJUXvD9mX3WKDyFD1s
+3Emj+tVZmnsC/YZdChvyaxeObbBsri347vZl0ff9kH4
+-> ssh-ed25519 0R97PA kcIYyWKxpJmjcrel+YodZQiR2zGPqfjzMyJXsz2XOzM
+SUlgGGs2BVRzTHT/ULNo1AiN5SY1BETFtJRY6LDr4JI
+--- l87sO6IuwSeCeQ8ktvYFI0xr4Utcl8KfpAV7WePc1y4
+÷ÚÖ
~÷J3¦Œ§Í‘1íÇè²ù<º?%×ý<0E>›Á‡ÉÒ\—\Ï7\»Ú¨åU-&W'd”{իɼ½u	"Û#Ž}¯õ…x–ìšz¥®Nj„!éfUqDG‘<é<7F>Ñca<63>‚'´+	¸Ï±]Pó»ö€D
Õ¸’´þ<C2B4>£¼­ŽÿÂçÌ\¦<>

secrets/floral/hydra-ssh-key-priv.age

@@ -0,0 +1,22 @@
+age-encryption.org/v1
+-> ssh-ed25519 +uvEmw BoYFUISKrlypCBQW/fA9UNRSnxQ93FkQupWUWAeOd3c
+wQH9gNk8TtjOgrwYwCuedPBbmftd7JhJk53ga2qo2iE
+-> ssh-ed25519 K3b7BA vyQIsvbNrHI0Mui5UdLz2mWcYvnTQBupWiAfEP5NfXk
+WeHg0PyfuaSJVzuiBPa1Tanj4NdqHvnZFWhXhIgbWc0
+-> ssh-ed25519 +qVung LneEmMhBqJxN0bgM7/Z+jJ8U8MJmCgE2EghBmDJ/aT4
+nd3B7afUNX1ZLCjHdoJ8+tabXmi38lQkLlhthYjIplI
+-> ssh-rsa krWCLQ
+krJRF4AvwfEFro4uiLIBB2RQTwO9COSyAqkjOi6jgLzJFMaU62EZrgfSYu24a6zs
+JyAHQ5k78uh3EhhbSzu6U65fComCbRAo+NiN2BJU7jb28y64suJbezJ+LE4P35CX
+biVgycaSc+OCrb1F6e6QOREi7+YjK3VrI4ZVCu93hSQNNRi/U5bbigQXt7NwcSIJ
+bY93sset4wg9Zwjk7tFg1bHiyOK3ZvYYQGlMjUxiWGl4Qch1fpL2CJNR32mZybps
+GZc7x69E8EhoHvdI0u1AXwS2raLhyRBPxFzu0r7nPlSi17TnLnU3Ux3BkVEDa6bh
+eVHqAB4dudNCC4wOY/ZE+Q
+-> ssh-ed25519 /vwQcQ scOp+aVA3TfY269EzQ6E4YX0uAu7qVVVIDmBvFGaYk8
+AYqW8+A7oxH/0m8OUReWxto1xWcnOnZOkX45ejuFJiM
+-> ssh-ed25519 0R97PA YqfHMAAiMcH/efJ0K5URDJkdLqlJIlQ+pSnuGUOJzjI
+v6BujlFcBF71SzvlZzA+tWku/A8bZzLkRfHtoCdbCO4
+--- PeORL3PTxYsxaY6GliSm1dRAH+hxf1n5LNeRYDq+poM
+I«­Ñ_÷l꺓CqyƹÚ@‹Ýç¾Wï;äù¹¨<ñÛ5ÅF¼·Ee7þ11=ø‰øQ<‚pmƒ¬ì¼^²<>òŸµÉÑÃÙƒ¨[†GÞ½÷$j†©<E280A0>±¨m’±‡I×ñí¶\«•ÉO§ëÇBdÜÛ< O›ª$ª„5£ãÒ¼âá‰õw˜REŽ3Y×ë!Ïd4ŒBFõ‡>ªŠÈ˜\EKËhæg‹ôÙ^f;ˆ1>tk ™‹£{ª»ÓMð³ D§š³87®\uÄ,íSá›ÕIinÀ:ø¢z"“Æ<1E>-XBñíÀ<C3AD>‰u<E280B0>jš
C5Õ­‘Ðï,Âg“*\]ÒO†îÇye5ïQîÄ!‰$þ.‹†+ù¦²¶zÀF¨ÑŠ^¨·SµµL<11>äÍB+óÓ¼ËR“Hxö6ÿåʉ1f<31>=jú8›¾!o¿@‚Rzrü5p´(#‰w|Gd­×¼O@>²0ã{$ËhE‹°Ä<C2B0>ÕûS
ê_¦æš^#›Oÿžò– 2c¹ŽŸ#G<>ž'º?e9yÂ)ô
ªÅ–Ì~ÑÓ¤
+Ì’tl#­#„»I³>o½Å³°)ˆóu
+†

secrets/floral/loki-environment.age

@@ -0,0 +1,20 @@
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>

secrets/floral/metrics-push-password.age

@@ -0,0 +1,68 @@
+age-encryption.org/v1
+-> ssh-ed25519 +HUDfA d5f2ESneC0wsoc9rwTjNfNXMBjCbjAQ7euthH2Buq1E
+5CynaQ8zhDRBvcmifhCsiDtllztCVAqs8rU36DOxgPw
+-> ssh-ed25519 +uvEmw EtYRis2LP0jv1W8mx8vFYNzkgi8OoqnA8cM2huS6NBk
+ll1csFIO+hVYk+I0uSVJmlDKj9aTWvf4kaYI5LJcm7w
+-> ssh-ed25519 DMaM1w ex4QJN8CG99J15i+yvqGEiEZn9OlGIC+cmLHL4u8ZEI
+VXnOv4CGK68q5t6hUV3oKAtxGZ+4FVbrmE1yMn16A0Q
+-> ssh-ed25519 sixKXw drXN6+q1y7L7ZU4chTfHfelu5GcTdff+i/UMFV0+3RQ
++8jmgnMh2OpQ3vhAuyQYWslfx7KO84a8KsCpoRD3Yl8
+-> ssh-ed25519 aHbF7w Af7NgjZ/Nvh5FHrX2VlF5riTIhJ+fdxTo6OR+8PcNwA
+ktKpm/HnOnw2Ym7xee3N1rneEX7+/xDhcp71N1NNHAA
+-> ssh-ed25519 87T2Ig 8mEUxJ/5NUvV+qQCDQH2Tm6Ryr5hf4xgsQlqXGf03Fw
+EavMcnsg/3EYBLQEBHX+0oTDKq5ZL4vj+mZntPM8UMU
+-> ssh-ed25519 Ao+7Wg UphWbatIaa+R1oZbfHazFhrawf0vax/3ZZS7YuX03Hs
+dwBbwoV0jpjiKr+nj+CRfUDgDl7ISpsCintVAzHnIFQ
+-> ssh-ed25519 wIR2ZA ZM58Nq7eJX9JVeYkoJf+mw8hxhYGoTx042ow1u3mJkw
+UtEaf7e4xsPO0ISlIF9LF+GcwTBqw4AXdMO4MASfgLQ
+-> ssh-ed25519 oGiV/Q G5KX/Eox+9md0yFRUZvGIsio2gWM17soHsL6H6zEX2g
+vI8jPjBAoFF0xhvRRLPzCMSiQOQ0fKuRb3CYVu3KUUo
+-> ssh-ed25519 gO3aog p9nZtjzoA0zJM+7Y6R16mpdub3dhu67yOYTUNKRytgI
+YL9vAp1+CK7jgmXkB47ufZMz+/swngkdUvEGR1zFZwc
+-> ssh-ed25519 N/+Clw 6LzFdtNsWewuJK2r97ZXJbRazvK3raN78UGanR/zWVU
+WT0y+sfDP3ffVwRcbYw51ArFR3OzXnoyi9IXwZZKEL8
+-> ssh-ed25519 CtkSZw CV0jQ5dIbgFtMxGK1X9b1qJOKmske8VgIPW5NW9mAwc
+clv7P3de61nZmXrvbOgL7Llw8ZqBMm2WFqgpznDwKv8
+-> ssh-ed25519 keg2lg 3Nk40ByQj8RThj4QDY2BdAkw55mXAJprXQRGjQqGvz0
+f8OFszJ8p90crFd+awEE12CNd7b22zgpH2XRKmH/Hf0
+-> ssh-ed25519 H885DA GDiJYH+gaC++TSqfoPDOTFcsCZRhEl0EeTeab7tgcWU
+kMILmwNMnMS7rgC3kKsAksu4Txn5owPU2y09h4aHKY8
+-> ssh-ed25519 Rq7K4Q VCNxGtCSCD2OYSWWwl0+yf189xV3QwRiwo80h4NPTkE
+hHkgYHLbISdl/RRdlInp9ub854M9ZKFSXpLgKW2YkmQ
+-> ssh-ed25519 vvyRpw XSCCrqEOtvzQRssI0U1DHirKoPgbOUKJxNKnioHhT2Y
+HGey1j0Kxae5Qs0aw6eqFziQGiRmNA+lEwbRdf5hhbM
+-> ssh-ed25519 aSEktQ mXY70Lgl76J4O5dPdDcIqmJ40EinigDuZrUghpGOq2I
+U2qeVFxGCYCEFWU+7vHc5Mu9EuzScowrjnwUyoqPj5U
+-> ssh-ed25519 cD6JxA at89poimBZyeeM8CQrxDxN0yCNDT2k04++py1fFycj8
+cQV/K5zc5x/oYnJ4N0MX3sTboT4G4ZNvVUVdHuJRzbA
+-> ssh-ed25519 ec6XRQ spJtb/xy4k4dmwKz8R2CPhC1WcuNV/rnDT978GkjHHk
+KrGEVGts/AhzbRNreqQ/CVanXL3l/9oMWxnpBLj23qU
+-> ssh-ed25519 1qYEfw KRkTYlvvnsCIExKQNmCyU7YxnGZsiI03kzecXNpLzUQ
+h2YagV7BzlsF7banzwXbOudTdlFzT7LC8PvtxAsX36U
+-> ssh-ed25519 2D+APA 4hdYlOnNIT9Q6tyKwXzy+u66Ti2EJopK43Sipebd0As
+tuesc9/QcEu4q9bTFJ5zJr0qvgLcmpn4at4cYtHrtbE
+-> ssh-ed25519 eTSU6g i1qT6PtepHXnoLCqDbhk86QG+SR9luQaw34a34gy5mw
+YE9VBAT5SLW2ECHRU+dMg9na6OQNVRVGuhY8vOdmE/Q
+-> ssh-ed25519 j2r2qQ TTTbSB/8UIDmmI3C9+u24PYZNfjl9jGADKHNWIwLfGE
+SNDforwii/GFp82TpyOcVIVrZWCe2QQKrjzPA6XA7Jc
+-> ssh-ed25519 CyxfgQ P5EiJ54v65Sz1gHuI0s170Z7c1WjcZLlb7NYigElfVs
+iYJUGpoE9LBIlv+O1navSSsy3EJ8tusXXX+/QAQvjNI
+-> ssh-ed25519 C/bBAQ hlBDpQRkcVCr3B6TCrBjxauhUX6Ndpm0s6x8W4hU6gM
+OFG3EuGJkSoEEXhbJ/Tp2DBdnBcs+hzxjNRdvcOSpQs
+-> ssh-ed25519 +qVung cGEGpO8NJfpj9ixAH9lhYkPKPDdQWryVxSOhMGQdnWM
++MycbIEab3P/AOS9i/YmPBDXB76hp3xUcWI4VMihV2w
+-> ssh-rsa krWCLQ
+Zv3dPYERlX1MaVaJTBDwIcjt1yLmu4Z7MovPgjGg01p+XsdBXeepTyOl+gRBwGgo
+AW5CIuaChYxtSNJ6nOgSaUpqzILycUF1xE1jROe3MIX2MZ4KGD1qoqcHbiCAng+a
+RqYrwAKnNea9FQMVfhYZBkRoYE6ne1R+0G6BoFM/okz24pAAFPBx+sMMhfTkt0uV
+kHVx0dgRw1pxa7Na98WH/7E0zp9VuBvVHGXfk1rfW/UQlbIO5RP3nldFoa6OmOWS
+JZ022UvjyC1re0KCurka4y+qmaiRKnTBmpIXxJFMwNCAQ8O8SeAQ3DHKHmXNMOIL
+ZVICtRRk0uX36AVU8DWDog
+-> ssh-ed25519 /vwQcQ kF8+hsA+0Msjd3q0SL52cae5RDqx4ls5kPKnc3UZyms
+Q33kIKJL3Vjxu7LQ5l4M3tlEuj+OW4uGh1x+JxthW8A
+-> ssh-ed25519 0R97PA gWBH71l6w9upTE0DwqOMSvWXc5VyJiKFAQLaSpWQ43E
+IrOrvzEa0bABw6UOpP8pM8WhuRNMaWJ2khljJIKwOS8
+-> ssh-ed25519 K3b7BA oS14iav9pSioLecMkOanJz89OJygLugvrnnTs5pKzz8
+akupMSiqXussXJyHwFm/f0imKALjowJVqd8/LFcC/58
+--- bCJXTEDaKg4FF2dQp2zFMU4Zm0zACekCuDvthcAyncM
+&­Ÿ
€Waïãà›BD R(¯¥Ñ”ufj<>úVÁ8nÆ>‚ß›øëæðZúâ{Idƒ„©,³*„%Ç“È‚z«

secrets/floral/mimir-webhook-url.age

@@ -0,0 +1,20 @@
+age-encryption.org/v1
+-> ssh-ed25519 j2r2qQ uYhcMpOER5j/SWUX1mNvkOU9Rumr0CgVBuGv9EHGpFY
+6kAgrwjgB7C1cMd410EpUegcxxGRcNOwCMJPXppepvE
+-> ssh-ed25519 K3b7BA 57GDNt5nwxgzCV5bnMPEPUeyZNG1U+zajCIjeoHjLAE
+rFCbfodjXHZ0aVLtW6xtoh6e/VH/HwFdFzjnQ2QEEXQ
+-> ssh-ed25519 +qVung DnLKAJPnUDpZ2+wXDZWpxwZkvv8oDyu3xxObTMT9W1I
+vh59DYoQLpiro5eBjwgNH2YHRsGY/i6TB7zPfQicOEU
+-> ssh-rsa krWCLQ
+ekvGooB5sCmAniHU7hlk+iCkYMQ7Rw2SJx8tp4FnpfAWJbRMH8CpTFYFiDvlHfFy
+Ce1OpkNkkipzBge0OCrfn6Y5iVz2CZHYHf8Ul5ueHwmb5fS7seT3yMoWhhSw/zE/
+G3snrBORT9S9+KTRnVnKiy+O3CaMZY+q41RR35Fs3mmVc/of2ILc/Jj3a3t+uBTX
+axkOMU6z6R6i3Ps5SbwJTaB9q2kMPvZFOO9Nmku1wohjetz64wvm+fDx0XVRPe4A
+jDQRPKAMIZK68SYHk/9azmlBtJSJnvxcxyj3IaU9MBskUCldWi8CQ9jQ+1XAIuHX
+0Etcsx7MhzBpuhx2xZ+dyg
+-> ssh-ed25519 /vwQcQ uW41w2RAtfMaOm1wJktMcbVporqKgdGA5SY03OcPmlM
+WgL8DWPU735Ysowq0HtvbrT6Tc3XEpwws3AycqpBgtM
+-> ssh-ed25519 0R97PA 59AFQx8ngDwQUdmfOeOFUARQQqaAdLA5WH67Wsld4yM
+o6jSWtlidZssWsJsI8xAaASi8p1sirLJFJwizzPXIBM
+--- scUnldbU89ICZYlniDbGEqeUF7QUoO1kcZLl8abyttk
+öR{p@IµþOlKKõŒ§!
<01>œWÎÅœ[R<-A‚ÐbÔ<¯·÷0õÐu¹øµñU’gBÏF~µ«=ÊõeQò}î4Ø:ô²¢5ƯŠtaØ™û”<C3BB>æ·<C3A6>§°x±M
ë?Ew0<77>8.

secrets/floral/newsletter-secrets.age

@@ -0,0 +1,21 @@
+age-encryption.org/v1
+-> ssh-ed25519 CyxfgQ fWH/o2+Uf0i/JFIuVjCnkhDIfYndtL8EeDcxSxhKVH8
+ShSPmdwnxzDuUe/kCx8e61JJAoHMwguNydn+5OIGuAg
+-> ssh-ed25519 K3b7BA p+wXAGvPqTX63dlZNCTIq3F4QFMWEJH1R+Ex4SJ5UTk
+1sFqFqnUM8YvZy7BEBArg3eLxCCsLXq2jNI7XLKq/Ww
+-> ssh-ed25519 +qVung rcpgzVQ1PmoNF2i0K0nAknzZwPXICBggzqhIZwO+8xY
+9rjsTwLm5u1GOJmnJYriXXAY1unG7y+WJ4G2ltxX34U
+-> ssh-rsa krWCLQ
+seXsQjs62kxn/agyKda2l19PI4xzDl1gM7rEnaEBV8UNLOPNxh41HTnP2etgDXSc
+4eyS3ntHXIOHmN4+JBn+Q/wuhzMGQmAcoFWbjqVVPOrpPYjgCG7q/iUD8kULxLB9
+UpF0gLsg1TnvrkTwlpxr8rP/PM+ZgyQAA84S96j9TW0coyTUoH/ZX1wWGtS4aalm
+aTrOMZGScZu7onTg+tYvR+aBKlFL28h08I5nqbA39srnCNuU68+OUhLgLUfiTscl
+umwNh/C4BP2Tmc6gxQiY8o3tGqGBssGH5+WqKzbK151vJjq80RKAS1HCaSSfmxkP
+vWkXWN3NQkJyqCBpuPYilg
+-> ssh-ed25519 /vwQcQ eUH0B+cCoUubIKbG+bA25kRj0TnZabB6t8jVK04NrFs
+ovkI0C4W5CJXMZIZdpaTtQNc+TGkQ3Yq87Dei3BMUsA
+-> ssh-ed25519 0R97PA u/I45pxH3Bnja/Jw/6IukINRuC0e1IKu8UVygVgIomc
+xyHuiHf1/nJirnhXbGHJnextGQa95tDo/RPRRnDCkIg
+--- LGqO4Bsa8bofD1W5YrQp75SlGLNg1XaFZ0rPUuvLPTo
+Êçã ‹ÜmlW£{@I3…*¹ŒÇ™@ÞªL7Wª
¤ÝŒY
+n
õö~Tb\V‘•ÜvPÙpPô
ïoÌS"ôm/Ûµ/bÝp’Äžêq¸£¦šeDj6–ÆþTì)

secrets/floral/stateless-uptime-kuma-password.age

@@ -0,0 +1,20 @@
+age-encryption.org/v1
+-> ssh-ed25519 CyxfgQ D2o8bUccO13DKF4COLBQ9mJbACsE2XsRa5S+N71WnTk
+ZaldT7HhQxbxf2ptIwdMYkC60eGtzihc7uwcAkq7s00
+-> ssh-ed25519 K3b7BA AiUCG5CnNyv1DPu+iEwEgW9GqZ8zgpgxKJTAp350ADc
+cUVaDv7F1haQIF11/UhhDAR5DrfJlPttGfDjkv+z9vY
+-> ssh-ed25519 +qVung 1JXeXyea+2Pcwoln/NLRiR8IPPIiB3gaFCP4imyv4DA
+JWmAY6ZnyU46KxzhRrQigGmUPba9lJDDyRQ2GjQShqc
+-> ssh-rsa krWCLQ
+ciLu/+cXfQrB1ms8oTv+xi4eADyL4j0qwnY/6TE0wAXkQHuNXDmpF6ccWZoS2DqN
+NcnGXL6+WyWxmwlyBEq/rsBPvi1g0M6Md7Z4gXn2UvjJ+S7WyA8QEwkxoTDkJS7x
+k/NvtunmggVsWVK4Xdi5DKRw+f32qr/8GysDhIPrTt43iReBKNbyuYWmC5Ec85ep
+JU4JzCNZjJ07kixS5Y9BhaJbpEr47lCXE/KtJUvm3VAxS9IwfUn7KHHdFWynbExi
+F898j3zOR/kgYmeA0oTiexRD3Y2LCvjXIHQZ3MobbZ/PBrjWxe78Sw2vy2t5JLtB
+gFG0K8M1z8DT6a8TtvXEgg
+-> ssh-ed25519 /vwQcQ kUM21TO9iSa8oVXMlNxR7Kc+8TV4C/uTzyQ+t3xnARA
+oXt+egWWONsKT48H4vZ2CPdy3Zfb2QeQVe9l7dDyO/w
+-> ssh-ed25519 0R97PA e/piqf2RD5QgPaQs6jsJdzJgfZR9n1JDIWpbvLZErSs
+UTJH8POFdZ4+N9WkLoNESl1pvcVD0MS1qn7AdS/mg34
+--- 9aYEP0eHDKMacIf09h+OJqIYw+N99+FrW/x/do8Lbo4
+$ ÖëWÛ\zú—¾=s/à@.Ç,?ƒW6n^ù#–i!§Ã–ï¶1]±Nvù±Ž'Ï¥¹6?‚'mµpPÒqýŸº

secrets/floral/tempo-environment.age

@@ -0,0 +1,20 @@
+age-encryption.org/v1
+-> ssh-ed25519 j2r2qQ kbi4mciOrjd7/X86xfmkDaMZhvZakoSJ6qjqLF3ljkE
+Q2BsgMLJ8AmjhnggRi+wkICj18NCA2HW1t8clemReUw
+-> ssh-ed25519 K3b7BA wNGmX9S9bJgd2JDte9QoNDfyycgmq4JMu2bc5nyYYik
+uUiutxAI3nI0M51W97aPRVE/l4dV2PEjph8eWOMLHIE
+-> ssh-ed25519 +qVung raYJ5vwMP9JopSdfa+ofkLY/gc0zcW4wTNBFTca+MXw
+sa/rWGSYrI4y6rn4JSboldWKUGvx6HbtsYo78AFOkBo
+-> ssh-rsa krWCLQ
+FLq8NwkiGw2gXptVVY393f0p9hFom57xHWPxtAlzOcRT8gvWu/uwgV+0raOcOcJa
+xxr5Sib+2D3UnUhprVPmH5Os9bI2seFAiej1MVVWLqvMtQHLFwnrzZTyZpxsXpQq
+5qQhNEADuQc4uD/ELVjGHKt6nF1Cl/GbgNLIOF/ITZ0pm1O1MjtT6MYJhQJhc6sb
+sno/wQyTXjj7rC06nyLX/rgOWrJSOeaz9eVp0A8k8/I0TXu/vRCW9gqWtv2m8sbh
+1uUHIm0l8f3z+zrL6OlZnpMFw4jpiiGoCYKPzD17I0onDYIjtdVS5iO9BsckxV/a
+wQWbyONUwbGCfeNSVAzZbg
+-> ssh-ed25519 /vwQcQ jwf7fwy4wKz7q761DNu8SyFHGgFlwq4P/Pn44Nido3E
+1q/jvt/vtD4ziY3eCDqk1XwMPpNUd80POTV2VVsumCE
+-> ssh-ed25519 0R97PA XeuziQ+wsoh0KSHXk5Qkl1kQOsAu1Ax1zTg13+XWd3M
+B1KHKm3tx/EsnE6hY+w7ya1ilhYiUs9AbwARHNkJi90
+--- JgQA6gCYZu8xcbXEl9VypccEIBO6uAJIdhBefr4doRQ
+V3ZðõÚ<EFBFBD>ç-·Ý.ê«sòÀ³3 ÎiS‰a5#¿Ð{åÔÈ®Dý˜YêNèãëù«ýoL+ÔÝ#–M<sws P»¢+í¢Ó‰ïBDoÊξÆÏuFí”Ç^Â¥•<C2A5>—ÝG@ÍM×ÛãÐØìq¦ºG^Qb s<
;ÂÒnC+ÖÊxª_­Úì]S<16>Ð

secrets/gerrit-prometheus-bearer-token.age

@@ -1,22 +0,0 @@
-age-encryption.org/v1
--> ssh-ed25519 2D+APA jiLDQ8JlYhaivXQQhjEfZrGWn7o6Wd2OMrLorEVSPns
-qRzHYcBhtGSm4RW7C4oW+VWSzHiDXkCN6bGeej2Gcpo
--> ssh-ed25519 j2r2qQ OcnIHB/vJoKuvhsT9dx1B+5lXguARtB9wSquW2KBB3M
-pgzC2KOFi3Yj1gCPemVK3a9Grv2SkwZ6AI1EFdh4hoc
--> ssh-ed25519 K3b7BA ibHY8wN3rNit1mO2dJZ44rwLylMaR39a7Oz3CGV561o
-4ElWORF/4lVEz33CJiuFG4rwUSIIOyi2L/W7Td7MX5M
--> ssh-ed25519 +qVung q4DDHS3M24kke2NCcpHEaUbUgoQB6QwnmDiwmdIOuBw
-Yfa6v23oezdDICE8I0UaVCShKlx9lN3DnBnSb63LU64
--> ssh-rsa krWCLQ
-gLBHP4Z8EBW1y7Yf9sfWMU+/fJ4WWp+NGRR7ebO5GwUeYobDYm/eYQ7rD3Q9k0rF
-kU51GYBaO7m5gLqc2Tq4+YjE2/EXDvjqkDSoyNrjQaaGTLqzvPYlCvKWyROjqJjX
-UwzPbQx5XVIKNgpsR9e6/hoJiJbDpavM+HQo+1zwoKAg5FvZZkE5UnIiSjuAxMgR
-+tmrhBfHEYkpbCCrXVE0jLCup8gPIci1PyXWkdhJy+HyHVkbYowGwNawNobNr1cF
-dJ5IU8P/DSSqZ1qWSl6ju7JKjzXU2Xq87/g7wJyrKGpe37pJmPIT86nCJTut+AK9
-iFED/y/p5NCtohyhztosgA
--> ssh-ed25519 /vwQcQ rzEjV56G+USMdpWklrGQSHuzG8d+S0zWhhwrmuyTyiA
-y+uMRG8NdAD0H4ipRN+sJPn1P0CGs4bk+U4qtetP3O0
--> ssh-ed25519 0R97PA ULWdDUjDg9oTEOqzCKUJl8yN+qwwmlSi1PFwRvr7aWM
-YWaE+STxKfQzxYMtP/cA20q0atXLdsjeA5nJyl2f8iI
---- Avs8hTgLwcBy8hyYWjR/Jbs5YaKozv2oBmGs51ckquA
-Wœ·Ü<C2B7>dŸ›ÝàÕò`@½Óµ3ž ‚¼½5è½b›Y%³A†Z=KiÐÑ76,¢w,1žŒèáÎôkØåRšAÄ‚FuÎÎ

secrets/grafana-oauth-secret.age

@@ -1,20 +0,0 @@
-age-encryption.org/v1
--> ssh-ed25519 j2r2qQ JSveX4zYEjb4jJH4eg4oXA6r3oc0jBx8NgjhN9JrjlQ
-1ZIr/XFClbwJHn0ppJnolpb4QlgZOA8JX5OjjY4x6pU
--> ssh-ed25519 K3b7BA sXUjuZFK0PL/KndxRCJCM5Kg8OmVseRZNWG8mL1alRc
-U9MMgDtqtmsS1W5i04Pa/b4JBTSjK6FffZxgYI3phtg
--> ssh-ed25519 +qVung FNSElbiw0frYcsO0xoyPQgRGqAe/aVX21dTB6yk+GQg
-zHT/xU+yfXYSBO2HLwoHrGf5ns6BDVb8MlhVVQCBlOc
--> ssh-rsa krWCLQ
-ye0mLiYeyvlp4EZX7mZ3F7B9V9JSeoiCodzccS+5qIEd6gr+RTHSnKYqwf/nwf8F
-qKLwbxWjpmkIzBWeswy8AJ8159aucGEmB+3/tTSwd+QlRkru4Z/7jtfU64KQttgt
-vaRfc9J/85AJJ2V6Sw/xG8SgxyLBbp/XIN2+tmb0g3kAWiuLcrLk3H/MsfmxDVXg
-RQjugP5K2+fEZc77dHQTrMI58K9TrSw1zYA1ee8J/fl9IJ7J77qi5UgizY+YfX8T
-SmR9DeYUe+hKgCB2k/KgAxp4WOQNgUOFBTsE5FW+kQQpfGx5aqR6vCYU+CPsA3Zb
-FwV0l+g4FUVy+xAtqaGSAQ
--> ssh-ed25519 /vwQcQ fbnK1jYiUwUsgD8sSTboJCBfcuwJXKNCaJaWYuIfmVk
-Uj2+uBABMTxq1MBsiHXgkdFMOpIN7gfxoJVKOQff1Pw
--> ssh-ed25519 0R97PA yYOb6AYAFWvm7W2KYT5v9zznkF4Di/vatH48Xgx0x2E
-yUm+MKj9496BkdX2FpLyhML7budUyqT1hL9hpghxSnI
---- ogCPBrmdbeDorj3t5BL05ge6VngXBpUEDW4qaaKIa0U
-%¨šÚlD]Ϫ?©ßŠÑ(ÿ†E/Wu穉T¶îç[}ž$ÁÍS„Šˆ^[:¸]
he0XUœp¸äq<C3A4>`0A

secrets/hydra-signing-priv.age

@@ -1,20 +0,0 @@
-age-encryption.org/v1
--> ssh-ed25519 +HUDfA NMyTM3c++HKU2klLjAbUUFS81k21LUwEoqR1OUBuLjU
-OrKxpksxoay93URtmN9HhnK43QrM/Gs0qRuENZvHWJI
--> ssh-ed25519 K3b7BA LloEGN8cbVvGraHs5cPIZRJJyTPFrmmeGwZyqov9m2U
-XPvNpQT3aFVoidOhmePGgiTyytIWtd4rs59Qq9xl/I4
--> ssh-ed25519 +qVung 3hat0gKIl1WjXnkP6p+/8RyTxZkaVnLgV9B8plICPlY
-jmRKWCUCDpDExmq4SEq8WpqQheBSRD4uqrTgxy2u6PM
--> ssh-rsa krWCLQ
-QPOl96dmoxY5YtMmL68+6MQpGwZc68ajaRkcEKmYYu4/XB+mffRKsNtyiKJQwEi7
-szvAced8C4RMNrCf3xyF77Sm1UV8YCyaHyplb2/yjv5YDvCDwTp2GnadDoAaLrXU
-jf6ocI8409XWQHEEEofHZRjmfmIBUx1lTwbGFMt48V7MZdadFjXmSmUMvxsu/Rj5
-NLjoPNRBzqPIw6U7nTSmkG2HOeHlA9Z5a33MsXYs8NPH22Spjjy+VvxrLv8VAjnf
-7kGjviW4ZcdEQ7Aox+9V+6qArrIy7lJ9lOIZA2LueCtKhQAmKnInFxRyyN0Nk4ls
-tjlBFJQEG2v14iaHENwRAg
--> ssh-ed25519 /vwQcQ o7qseMFb4ViV7ylSl2ug7xFZn7GZGqCapWRCq2vyVVs
-hKqzk9BcK5l0VhLfPONKKv6SRnDCw2n+RoaeQbOnT8Y
--> ssh-ed25519 0R97PA DTEowwoCXTMGxfQIXOnwn5fjlih0UmQJCKs2II4gCVY
-8BpVhUV8qg3zcCJe7OwHoJrfYIpBtOquqhFyfZx7mRQ
---- r2NmEyV0/Goas5lXMHeFoafcrbMHvs4ob0Zg4cVil3w
-=‚TÏ`Ô}%xÖ¸œêFtÆêlãµ)•ò1Ð]Œ<DvwFøè:Qf@nÇÀU=~FžË-Ò!©ò÷þÌ«i6wXtü'k®ç`)A›äûÌtI1Tî£U+Ú~×’aÒÉПONƒÄèLþI£p@œT)¤'_ÞoDö0»,JfGVó­k’/

secrets/lix/buildbot-oauth-secret.age

@@ -0,0 +1,7 @@
+age-encryption.org/v1
+-> ssh-ed25519 Ao+7Wg q7oRHUUlAvD8OUbpPT7d6eLMPWU0YS/verYTDE5BCkY
+/87/1uqOvRYeqel9IjnFmGyF9SXUQD8MTgfcj91b/Fs
+--- ulIeB91NJ7z/64h9BCLSD9/RW/zwv3m1Zo2ovNuInv8
+Îœç}³Óš#épÇ o>ä·*vµ÷ÄåŽs?[¦º´L
+<EFBFBD>þz™rý‰?R±Ñó7<Ê
+æi!€{X„¾òÓ

secrets/lix/buildbot-signing-key.age

@@ -0,0 +1,6 @@
+age-encryption.org/v1
+-> ssh-ed25519 Ao+7Wg EMpfs0EpWwaIKAoUBfEkyAHLIwi6JnGG6RvUWM5LjnU
+LKiwUBNc791U/GVRNlRPZE/TEMJjcFFrLruFJhiyiOI
+--- 0khp8u+4vHgGyQqP05m473Eo09eyOUZLI5+EK4olzoc
+N3(
+ª•û
xRq°<71>f<EFBFBD>Ó;ͼ3¬~RˆÓ
C^ñ+fœš1”®˜xˆ÷ÅëñSØ—hâ
£ÖË°GˆÓn–YIûµ:7¾!°u×Hþy/‰Øð‰™.¯¤á^¹lC™ôUÈËþ5cž:]ÿNž&'MÎè¶É-˜–ÆHF¦D0‘ cjô ‹Ð~

secrets/lix/buildbot-workers.age

@@ -0,0 +1,6 @@
+age-encryption.org/v1
+-> ssh-ed25519 Ao+7Wg RPKKoI5l5cYVdSvOxTHCUtwceac4vSi3+vlaqHr8kQg
+qbgTHCeQDNM30IJNZ/BU6wgryJwB316H5GWWaYB/wng
+--- GuFi3GSRdlBJ5YRjfAVyFDZ+4TH575kFJLnFp5847N0
+-èƒÞHÖÜ*x´M7¼t<,4ˆŠÑ^<5E>5@v°<>£€º,Z•MÒg=M
+» 3výJÄ«ÐÖê¿Nz­8'<^'4&WÂf"Êõ´À›ë\©º»ëêwmzúlAl|+„‘ÆKš~68ñE­Ýîk•8ø?S&òaM‹Ý~ž¹ê¿]Vfø ÝJxaõDù¥x

secrets/loki-environment.age

@@ -1,21 +0,0 @@
-age-encryption.org/v1
--> ssh-ed25519 j2r2qQ 6qyr94uky6B36UOY0jd5NXgF2rJ3RWBUzZ32c5iOTmY
-fjlI3fjYjwyNQBs4K4pq/5c7oBkf5XUXoGlBOBpmPu4
--> ssh-ed25519 K3b7BA N9VYT/ZslG07KldzO8sPE5TiYYwxJqpYU87ED4PuBXw
-P1s9L57prPqM4fjcYHv+g0rgP/NvFr13CgCxthVHZ4c
--> ssh-ed25519 +qVung Ry8uUFsmYmP+Urw46lhAsCc3S+QiWu1mn8J3rIy+KFQ
-iB7xAfdpHwOzAnLvosJb+F50QKsOYWr7CHC3srsS6ME
--> ssh-rsa krWCLQ
-w0xIVFtUghdAO7SxZD10rBMtdQESEvYUEKxnWzLh0cjcRhaVT/BXSZQsKV2Rupoo
-nDL5uy0k+tPXm0HroZ6VkZ0fH/lOpeUR69ZvJmClKql3Fnf1385+5BvT719cbbaq
-yll49gx0+ms/oB9jS3SPwbOg+UJgnkZCeu9138h3MG7yWNtVuA9l5hsJioVvOVlS
-Z5EXbjdQR9xYjSwR+b8MYZ97ej5fXpuULEopbx2wXt84u1e67vTETqflitR7lrzy
-A6F65g35aagPJZGHzfrKVToy3pfXm9ky/30DolWLD0DpG7G6o/8afy8O4yBAGlv3
-ZLTaUbrdILSz2ff1Njx4Nw
--> ssh-ed25519 /vwQcQ YqqmX/f4whOk97kCgSPo6oj/274eYlBWtS+OahAAQ34
-hoCbhupzSTx+wNIorzYGHyGvU/L8unKEyD7Bqq23YP0
--> ssh-ed25519 0R97PA 17SDtfT9GzAsIsQB24AmYXpW8v4+LEakup+tdFroHTk
-HIvBhAGA2GMVWFBP3OTFEn+XpPFBJDOJDK3SQ94mNKM
---- CD1QrxYGAhhy+l7U5kOXn1shCwz8pYJNuGRugPxmzJw
-ñY¾Æ‹N	Ï<>x
™êÿrR^z[¤ã¸è…•ªa”z
-óæÔÉ¿Ïžu0c¯c;y<>Ÿ¢›&{ñèxA]‚þ†¨Q¨¼_:̱ í€öUoiDl (‹Å
ëwÝKi,j.oFyÌ°$}•Y§@1”È™„Y£²è¶u Ò*¡ÏþÅ<C3BE>¥™0…

secrets/metrics-push-htpasswd.age

@@ -1,21 +0,0 @@
-age-encryption.org/v1
--> ssh-ed25519 j2r2qQ sIYTVOTWNToDSNa4qiIaSoac7zka54g/opQ70q1SAA8
-2Z1mlCWxjakHqRbArU2BkT7B/Dx0XKH7kCnBa+OYI+s
--> ssh-ed25519 K3b7BA PGyd27M/Hmk6qpRf8bcI4QWrS0vrPgjiZzaXvKQkJDQ
-ixrciiNR/th0FM9MxVx/omHdI61EmAhTA465SjxECF8
--> ssh-ed25519 +qVung Q7k74fDLKwCdzobz0b6ByS2LrhMOIC58Ofto0gpBLFE
-p4CIje+sO/nOaO1lzAY9n2HYLUKxEvKDbxeR6dOyM00
--> ssh-rsa krWCLQ
-ezrZTitn0/BRD0K7e2K53qz9AZCa0aHlzFSuyzqyVJLdAZUxBUnfBwmGuJgKTa4Q
-fWsXBs+L65hkcL6/VKS7oSGGyoEHmoPFKbb08B6FKLHt9V1td5xbHIoTYbvSavUA
-g3wpTUa4eG3ivcu96VjyyBKTAc7LN7h7dSMbvvP5tpWT5vL+WstCdFf7zzUL9HBS
-yI8dzEbCQIgAAaHj90MREgIIgIB27Dn1PvkEBGYky5ybBRa3DXVyqnX0dDtsXWpK
-ipRPDV7HC1+x2TlqQjD5ED737r/AP573IXbnRLSEWnGDjtd/JWQmfOO3JACoRjU6
-qfb5SSDT9QriuWSow7CDhQ
--> ssh-ed25519 /vwQcQ duuo3BGe4Q1MHMljgzmtpzvtiOvAHqKu2HS9SBxLuhE
-GCwccbE5lX5uPIri/7Vn6hzpfL7ouJBFU14bKjl6yTM
--> ssh-ed25519 0R97PA WIFf8tbMlmNrNFF5tRcL+mOJ40SvIdppAtItWtxzCk8
-miU7Z4poEVMZCeAEef1VS0jouCDxGro2xLEE3hnRJEQ
---- Iaff5rxl9r1qEnlpkOpGyBGtAvGMLyBlJQ45iInuAnw
-cýI±C«¤2ˆ7µ	½³Ú“nZMþ`œ{7È`¨½V@ñyzÀÅžª€)ÛY‰DÄßÇX—o“óä	~<PÙ›òš5Tpúx
-ÓRÏÜö

secrets/metrics-push-password.age

@@ -1,57 +0,0 @@
-age-encryption.org/v1
--> ssh-ed25519 +HUDfA SrjyocQ2U/mcmsVX3bhTDPiNfnRepZ+J//d4JkVrQ0w
-MELfJrKcLlC3rWKHdMZKZyXB0ztzmZUjWUcT8ibP8vE
--> ssh-ed25519 87T2Ig IN9MMxRNzgKHBmGwidVWIvq2xpNVkbioWjG0lf+B5zM
-sXIXfrTak7E8isigDDnrzvjJli5ma5f9fOJnWCdDRpU
--> ssh-ed25519 wIR2ZA 4DD/V3Xq1B2t8Zb11MnvtSZ3Oq5Glvka93g313dVSyU
-TrQiCJGOtitCCfNy0PdaRaPnk2mYCEPKtnOtdAzGolg
--> ssh-ed25519 oGiV/Q W67zxBlGYg3PhUbwBiGE2vVoIl455R+4g3EClZKwulI
-2sldkyyBUGxhXRCoa/vW5LrxbI0TqerOeOqrTtzY3Mo
--> ssh-ed25519 gO3aog YVF4hdjNYxOPE8v95BENIb6khsu0+tztaPNNCsXoWDE
-LLX/uofYt5/HQ7q5L35UK2t05rOlhCDnC4SIJx0bNtM
--> ssh-ed25519 r/iJSw RMwg0xLCOVA+wc08f67kkUVIgy6W3Ypd3jRkRHFA+l4
-KR5RElZHGzzLU9hjr3Qg3NwudDxMtHqcf2t6xjDMz+U
--> ssh-ed25519 N/+Clw BBYMWbIT8dXcD7SU+LrIuFeM+2RodGF2rW1ubx/W9mU
-yANEUWhFtNkx3VArOTTW+rREcxwzkN47CD2kK6JsMns
--> ssh-ed25519 CtkSZw wy5ZfWI6tqN3OZDqRZvb6lhj8Pt+GrP3YryqhjH0ugo
-OtY/WsGkJJghGGAh4cfZOxkg/WcYJ4w2gu4Hu9VHntc
--> ssh-ed25519 keg2lg lzE0HqDHBwDyuc5m5T9YSxxTgEk4mOQWY3l7a1+QKD0
-cn07YAocsIrSeWo1ZGyFzq3un8kdpEuS6zYpKs7G/iI
--> ssh-ed25519 H885DA eZJW1T2VPMhDs/ygauDFdd1Md3D830ysel1yUZkZoSI
-wpq1+ndzQWUUN2yYMKnEZrOcgCuqKIrDjaeX+XpkQgk
--> ssh-ed25519 Rq7K4Q CQ+Y2k5F8Q79GF5PQh8qDmxWgrKcqJHjAodVBqKqQkc
-SkcUl6dFoBQmPOOjTEopgcn5vzLH2oHICymAAS7nsAQ
--> ssh-ed25519 vvyRpw nW2eCEqQ6uCT9RgIJyCSpP4JHwQtKDSiBBp1wdVFtTE
-DQcHIBTNqvFVYV1fXbGhu0pCwa++knjLpCVFC3npaS0
--> ssh-ed25519 aSEktQ 7SEG8F8UyH0gR9uT+mFfBIXsAIUFnNd2bZgyJ8C/gVQ
-JTlr5eIhpepOoCxi54nrG7Wjxq9CXZYkb33kd2urdak
--> ssh-ed25519 cD6JxA QKVkY0MS3LeJf+YfwJT2yysuseg8tSAEGHOBgHFsVkc
-IpAAWCWxHNg1MOBjG+JNXcTE/xNrDW8+5Cz/hNWVYvU
--> ssh-ed25519 1qYEfw pA2G6CxFosIcXsBnTUfN1wsPs3Ue5aMzo7wameAacXM
-av7xGnRkh57JtgF37QtaF//eYS/pHqznHY4DJewRp5s
--> ssh-ed25519 2D+APA SOSVjgiiugDWg9HeFIlaLa+mo3q8AHhntl1tHEB6QUQ
-QINZr847DASGM32Si6t1mHH6fCkKnq/sa1+3IXhaSlE
--> ssh-ed25519 eTSU6g NuV8gm/Ijo6BpZptiYua2bnYNoxuHcOtce9zGNyi0yo
-E4zAIpZN5eTWJanPEwS7B6RfnnMRLDaOj+5l5L4GdCk
--> ssh-ed25519 j2r2qQ PpKKKAJikQKWAaYvDhIoiPeTkWtE1chw8lCpZ4O+LHs
-4kR0ZNRMt0fljaOu3UgqVrUFnc6v916IyKdYkvz/zfA
--> ssh-ed25519 C/bBAQ m7XsRBwlHgWXifCif/8H9TcSqs0so5hha2T4tCq6qn4
-QltQrR6Y3Im4xo8DtpzN5kMsHNfkpG0FE6Y2GnkrH5Y
--> ssh-ed25519 K3b7BA x91SNkgN6NSlw2FZnliA+c6zoTYyeuZh2iT+Rl+qtT4
-nKU6GcX4WLTRncStiW6BS7iK7zlCVhn55FPjRNniqSc
--> ssh-ed25519 +qVung opSEU5VaLZcm4GhcKlNtG/Ut0jU6oTYQuqvnDkuSGT4
-ny6Wfsi/PIj5A9q/fwL3vwnkft/yH6fqlPIXo0cklfY
--> ssh-rsa krWCLQ
-p5Y5fVwyG2s7m9ClsgbcVz/fSF2lJvbXxuN8O4b6sp+QiABmSGs0R3pZuf1v9xBr
-Jc0JWhl4vvvb9F9WUbJR50hIpdWo6iX4vrz3TnSvPFmnpUpRfe+a29ZJhp0vCA4a
-HVaOJGlnGZ5BdSkvPslGVCPu684OmO/veL5G1H7xmN6yg2b3n7SaGF7A4+rpVqgI
-6GZiFpnM6LpyKyoTyXRL0ghzjhwggQCCnBaN7GIUhvPacPdilAJWmnagQzx8aZpT
-LRe1WAeKH2Lbar4UNeot3MzWkZxUXyyWszTMe1ca94N3jY7MG8adzX3guMykP5qA
-eya7UOphIwkQKlVB3N5bfQ
--> ssh-ed25519 /vwQcQ xQFghc3LzwG82u+h80e3NdfbCh85OKdai32pwvS3uzs
-MdUPg9BHvPX85jWnV7evkNekPrzoJuT8FP0l/mhfZDk
--> ssh-ed25519 0R97PA 8cDQRKrujysaUiD5OxdrpmWn7ZZCJ9SNbLYtWuTSmXg
-HFa/6WbK4aMK3cKEMEycyiclTu8jOcCMcr1R7Ebh73c
---- wZAdkwtibHAVLCqtfmZ54ZtPwDPogkRwfKREBR2xOeY
-[‹‰Ã×÷ÂûȹÂÃökR‘î\Äub<75>çГRö
-‚†qþà«‘ÎHÈbe{Y
gÿ<67>mÀ¿¥Ûs®Æ¬„[-p¸

secrets/mimir-environment.age

@@ -1,20 +0,0 @@
-age-encryption.org/v1
--> ssh-ed25519 j2r2qQ UjtB95lmxn+XB8k/vX9QIfl80clhwmHaxqT4KTpkJ0w
-TwNCD7AyNNHPavVXUl9fLEqTvHdnu0Pmfr12D2Y0OGE
--> ssh-ed25519 K3b7BA YoRKtTbADH/AIVxGRrbXhX8/6Yjem9syG7MZRZpDCQk
-kNBh8xjdE/TfAcq9+cSoxjRqEwqjKimziofhkGngFK4
--> ssh-ed25519 +qVung YTRu96IpMbUVa2paF2o8fWqV0LcJDnH7CDc6WXPO/ig
-woDkR0Y4OmFb/jkjd3hDJaeOwLkZq/7AoI7orFAnMfQ
--> ssh-rsa krWCLQ
-sQgaw0aa8AtJ49gDNiB0olJLStO50XqroBVCTO0a7ttB+Z6XoIqLjm6ZZFfIJA2W
-eNJlkURP6oA8qFhC/lv6xTDFgQlg70ADb5PREIaZJ6SXxJGJoi9nXOlvNh77ZOS4
-JJlyg/1e6NHGXdlDAgRN3vzwBF4NPvFGGqgv/hYbxGw8X8JBSDMfNpS3lrZL2f3G
-aGGLfS9/OHdWFM8YAOhETJbU+FUOPtUvdcc+Hue/NCQUBr09UISR/NQ93zY5CW2z
-Tkk8sFEs2FHm7cb7YLLFyQ15fqbT1esQEp67/75jD84/v0Vfl5OPdaZhQB3r3M/s
-mTC8J3SCyKsEmYEB0uFgtA
--> ssh-ed25519 /vwQcQ 4OD7j4pmu0Tv0nRvDak0axNjkQcEtoOaTW3TIM/S12E
-2Z+tkmpdflsyiet5L1M9mLjW83mx/7LyJcZEHdmrKaE
--> ssh-ed25519 0R97PA zOX+3fvm5CQU+GlucS7DNk9N2Nf2kq7d2xLx12ZKDXU
-+biVbfNARvvuZTHdiUM5CTJto1LSDVXyeZPxryrj3gI
---- F6iUbsggHBdhccZdiHCZbPQZjyLS5qwxxxKNl16lhiI
-çHÓÅ<EFBFBD>ç£<EFBFBD>•*{`-Ådýî<03>cznᳩªÜtö™ÍŸv‘"W]h²3°_ÛG\6ì{ýÓi+®@7J¬ôâ`Ëk-ÑÆӔﲈå–^™Y6ï\¯Þ+4Û£…»aäÐ-Y$¬•Îð/5Ëóû×°óE3ÁKáoÕwcÔT¢D<C2A2>•ÈYh—5`>

services/baremetal-builder/assignments.nix

@@ -0,0 +1,29 @@
+# This file contains information on which builder(s) are providing how many
+# job slots and providing which nix features
+let
+  genBuilders = { offset ? 0, count, f }: builtins.genList (x: rec { name = "builder-${toString (offset + x)}"; value = f name; }) count;
+in builtins.listToAttrs (
+  # The first 8 builders are general purpose hydra builders
+  genBuilders { count = 8; f = name: {
+    cores = 8;
+    max-jobs = 8;
+    supported-features = [ "kvm" "nixos-test" ];
+    required-features = [ ];
+  }; }
+  ++
+  # The last 2 builders are exclusively for big-parallel
+  genBuilders { offset = 8; count = 2; f = name: {
+    cores = 20;
+    max-jobs = 1;
+    supported-features = [ "kvm" "nixos-test" "big-parallel" ];
+    required-features = [ "big-parallel" ];
+  }; }
+  ++
+  # These are not currently used for hydra
+  genBuilders { offset = 10; count = 1; f = name: {
+    cores = 8;
+    max-jobs = 8;
+    supported-features = [ "kvm" "nixos-test" "big-parallel" ];
+    required-features = [ ];
+  }; }
+)

services/baremetal-builder/default.nix

@@ -3,10 +3,13 @@ let
   cfg = config.bagel.baremetal.builders;
 in
 {
+  imports = [ ./netboot.nix ];
+
   options = {
 
     bagel.baremetal.builders = {
       enable = lib.mkEnableOption "baremetal bagel oven";
+      netboot = lib.mkEnableOption "netboot";
       num = lib.mkOption {
         type = lib.types.int;
       };
@@ -40,8 +43,10 @@ in
         "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGMnOLLX0vGTZbSJrUmF9ZFXt/NIId/MUrEpXmL2vxod"
       ];
     };
-    nix.settings.trusted-users = [ "builder" "buildbot" ];
-
+    nix.settings = {
+      trusted-users = [ "builder" "buildbot" ];
+      inherit ((import ./assignments.nix).${config.networking.hostName}) max-jobs cores;
+    };
 
     nixpkgs.hostPlatform = "x86_64-linux";
     hardware.cpu.intel.updateMicrocode = true;
@@ -52,28 +57,36 @@ in
 
     boot.initrd.services.lvm.enable = true;
 
-    fileSystems."/" = {
+    boot.kernel.sysctl."fs.xfs.xfssyncd_centisecs" = "12000";
+    fileSystems = lib.mkMerge [
+      (lib.mkIf (!cfg.netboot) {
+        "/" = {
           device = "/dev/disk/by-label/root";
           fsType = "xfs";
         };
 
-    fileSystems."/mnt" = {
-      device = "/dev/disk/by-label/hydra";
-      fsType = "xfs";
-    };
-    # We want the tmp filesystem on the same filesystem as the hydra store, so that builds can use reflinks
-    fileSystems."/tmp" = {
-      device = "/mnt/tmp";
-      options = [ "bind" ];
-    };
-
-    fileSystems."/boot" = {
+        "/boot" = {
           device = "/dev/disk/by-label/BOOT";
           fsType = "vfat";
           options = [ "fmask=0022" "dmask=0022" ];
         };
+      })
+      {
+        "/mnt" = {
+          device = "/dev/disk/by-label/hydra";
+          fsType = "xfs";
+          options = ["logbsize=256k"];
+        };
 
-    swapDevices = [
+        # We want the tmp filesystem on the same filesystem as the hydra store, so that builds can use reflinks
+        "/tmp" = {
+          device = "/mnt/tmp";
+          options = [ "bind" ];
+        };
+      }
+    ];
+
+    swapDevices = lib.optionals (!cfg.netboot) [
       {
         device = "/swapfile";
         size = 50 * 1024;  # 50GiB
@@ -86,8 +99,8 @@ in
     };
 
     boot.kernelParams = [
-      "console=ttyS0,115200"
       "console=tty1"
+      "console=ttyS0,115200"
     ];
 
     networking.useNetworkd = true;
@@ -146,11 +159,24 @@ in
     };
 
     systemd.services.hydra-gc = {
+      wantedBy = [ "multi-user.target" ];
       description = "Nix Garbage Collector";
-      script = "exec ${config.nix.package.out}/bin/nix-store --gc --store /mnt";
+      script = ''
+        while : ; do
+          percent_filled=$(($(stat -f --format="100-(100*%a/%b)" /mnt)))
+          if [ "$percent_filled" -gt "85" ]; then
+            ${config.nix.package.out}/bin/nix-store --gc --max-freed 80G --store /mnt
+          else
+            break
+          fi
+        done
+      '';
       serviceConfig.Type = "oneshot";
       serviceConfig.User = "builder";
-      startAt = "*-*-* 00/8:00:00";
+    };
+    systemd.timers.hydra-gc = {
+      timerConfig.OnUnitInactiveSec = "10min";
+      wantedBy = [ "timers.target" ];
     };
     systemd.timers.hydra-gc.timerConfig.Persistent = true;