infra/services/baremetal-builder/default.nix
Yureka 74e06ac6d0 hydra gc every 20h
metrics analysis has showed that this is unlikely to fill up the builders
2024-07-24 09:35:18 +02:00

168 lines
4.4 KiB
Nix

{ pkgs, lib, config, ... }:
let
cfg = config.bagel.baremetal.builders;
in
{
options = {
bagel.baremetal.builders = {
enable = lib.mkEnableOption "baremetal bagel oven";
num = lib.mkOption {
type = lib.types.int;
};
};
};
config = lib.mkIf cfg.enable {
boot.initrd.availableKernelModules = [ "ahci" "ehci_pci" "usb_storage" "usbhid" "sd_mod" ];
boot.initrd.kernelModules = [ "dm-snapshot" ];
users.users.builder = {
isSystemUser = true;
group = "nogroup";
home = "/var/empty";
shell = "/bin/sh";
openssh.authorizedKeys.keys = [
# Do not hardcode Hydra's public key, selectively
# add the keys of the coordinators that require us.
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAvUT9YBig9LQPHgypIBHQuC32XqDKxlFZ2CfgDi0ZKx"
];
};
users.users.buildbot = {
isSystemUser = true;
group = "nogroup";
home = "/var/empty";
shell = "/bin/sh";
openssh.authorizedKeys.keys = [
# Do not hardcode Buildbot's public key, selectively
# add the keys of the coordinators that require us.
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGMnOLLX0vGTZbSJrUmF9ZFXt/NIId/MUrEpXmL2vxod"
];
};
nix.settings.trusted-users = [ "builder" "buildbot" ];
nixpkgs.hostPlatform = "x86_64-linux";
hardware.cpu.intel.updateMicrocode = true;
boot.loader.systemd-boot.enable = true;
boot.loader.efi.canTouchEfiVariables = true;
boot.initrd.systemd.enable = true;
boot.initrd.services.lvm.enable = true;
fileSystems."/" = {
device = "/dev/disk/by-label/root";
fsType = "xfs";
};
fileSystems."/mnt" = {
device = "/dev/disk/by-label/hydra";
fsType = "xfs";
};
# We want the tmp filesystem on the same filesystem as the hydra store, so that builds can use reflinks
fileSystems."/tmp" = {
device = "/mnt/tmp";
options = [ "bind" ];
};
fileSystems."/boot" = {
device = "/dev/disk/by-label/BOOT";
fsType = "vfat";
options = [ "fmask=0022" "dmask=0022" ];
};
swapDevices = [
{
device = "/swapfile";
size = 50 * 1024; # 50GiB
}
];
zramSwap = {
enable = true;
memoryPercent = 25;
};
boot.kernelParams = [
"console=ttyS0,115200"
"console=tty1"
];
networking.useNetworkd = true;
networking.hostName = "builder-${toString cfg.num}";
networking.domain = "wob01.infra.forkos.org";
systemd.network = {
netdevs = {
"40-uplink" = {
netdevConfig = {
Kind = "bond";
Name = "uplink";
};
bondConfig = {
Mode = "802.3ad";
TransmitHashPolicy = "layer3+4";
};
};
};
networks = {
"40-eno1" = {
name = "eno1";
bond = [ "uplink" ];
};
"40-eno2" = {
name = "eno2";
bond = [ "uplink" ];
};
};
};
networking.interfaces.uplink.ipv6.addresses = [
{ address = "2a01:584:11::1:${toString cfg.num}"; prefixLength = 64; }
];
networking.defaultGateway6 = { interface = "uplink"; address = "2a01:584:11::1"; };
deployment.targetHost = "2a01:584:11::1:${toString cfg.num}";
deployment.tags = [ "builders" ];
# Why can't we have nice things? https://bugs.openjdk.org/browse/JDK-8170568
services.coredns = {
enable = true;
config = ''
. {
bind lo
forward . 2001:4860:4860::6464
template ANY A { rcode NOERROR }
}
'';
};
services.resolved.enable = false;
networking.resolvconf.useLocalResolver = true;
# Hydra blasts ssh connections and does not multiplex. Loosen some of the
# rate limiting.
services.openssh.settings = {
MaxStartups = "500:30:1000";
};
systemd.services.hydra-gc = {
description = "Nix Garbage Collector";
script = "exec ${config.nix.package.out}/bin/nix-store --gc --store /mnt";
serviceConfig.Type = "oneshot";
serviceConfig.User = "builder";
};
systemd.timers.hydra-gc = {
timerConfig.OnStartupSec = "4h";
timerConfig.OnUnitActiveSec = "20h";
wantedBy = [ "timers.target" ];
};
systemd.timers.hydra-gc.timerConfig.Persistent = true;
bagel.sysadmin.enable = true;
environment.systemPackages = [ pkgs.ipmitool ];
system.stateVersion = "24.05";
};
}