{ pkgs, lib, config, ... }: let cfg = config.bagel.baremetal.builders; in { imports = [ ./netboot.nix ]; options = { bagel.baremetal.builders = { enable = lib.mkEnableOption "baremetal bagel oven"; netboot = lib.mkEnableOption "netboot"; num = lib.mkOption { type = lib.types.int; }; }; }; config = lib.mkIf cfg.enable { boot.initrd.availableKernelModules = [ "ahci" "ehci_pci" "usb_storage" "usbhid" "sd_mod" ]; boot.initrd.kernelModules = [ "dm-snapshot" ]; users.users.builder = { isSystemUser = true; group = "nogroup"; home = "/var/empty"; shell = "/bin/sh"; openssh.authorizedKeys.keys = [ # Do not hardcode Hydra's public key, selectively # add the keys of the coordinators that require us. "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAvUT9YBig9LQPHgypIBHQuC32XqDKxlFZ2CfgDi0ZKx" ]; }; users.users.buildbot = { isSystemUser = true; group = "nogroup"; home = "/var/empty"; shell = "/bin/sh"; openssh.authorizedKeys.keys = [ # Do not hardcode Buildbot's public key, selectively # add the keys of the coordinators that require us. "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGMnOLLX0vGTZbSJrUmF9ZFXt/NIId/MUrEpXmL2vxod" ]; }; nix.settings = { trusted-users = [ "builder" "buildbot" ]; inherit ((import ./assignments.nix).${config.networking.hostName}) max-jobs cores; }; nixpkgs.hostPlatform = "x86_64-linux"; hardware.cpu.intel.updateMicrocode = true; boot.loader.systemd-boot.enable = true; boot.loader.efi.canTouchEfiVariables = true; boot.initrd.systemd.enable = true; boot.initrd.services.lvm.enable = true; boot.kernel.sysctl."fs.xfs.xfssyncd_centisecs" = "12000"; fileSystems = lib.mkMerge [ (lib.mkIf (!cfg.netboot) { "/" = { device = "/dev/disk/by-label/root"; fsType = "xfs"; }; "/boot" = { device = "/dev/disk/by-label/BOOT"; fsType = "vfat"; options = [ "fmask=0022" "dmask=0022" ]; }; }) { "/mnt" = { device = "/dev/disk/by-label/hydra"; fsType = "xfs"; options = [ "logbsize=256k" ]; }; # We want the tmp filesystem on the same filesystem as the hydra store, so that builds can use reflinks "/tmp" = { device = "/mnt/tmp"; options = [ "bind" ]; }; } ]; swapDevices = lib.optionals (!cfg.netboot) [ { device = "/swapfile"; size = 50 * 1024; # 50GiB } ]; zramSwap = { enable = true; memoryPercent = 25; }; boot.kernelParams = [ "console=tty1" "console=ttyS0,115200" ]; networking.useNetworkd = true; networking.hostName = "builder-${toString cfg.num}"; networking.domain = "wob01.infra.forkos.org"; systemd.network = { netdevs = { "40-uplink" = { netdevConfig = { Kind = "bond"; Name = "uplink"; }; bondConfig = { Mode = "802.3ad"; TransmitHashPolicy = "layer3+4"; }; }; }; networks = { "40-eno1" = { name = "eno1"; bond = [ "uplink" ]; }; "40-eno2" = { name = "eno2"; bond = [ "uplink" ]; }; }; }; networking.interfaces.uplink.ipv6.addresses = [ { address = "2a01:584:11::1:${toString cfg.num}"; prefixLength = 64; } ]; networking.defaultGateway6 = { interface = "uplink"; address = "2a01:584:11::1"; }; deployment.targetHost = "2a01:584:11::1:${toString cfg.num}"; deployment.tags = [ "builders" ]; # Why can't we have nice things? https://bugs.openjdk.org/browse/JDK-8170568 services.coredns = { enable = true; config = '' . { bind lo forward . 2001:4860:4860::6464 template ANY A { rcode NOERROR } } ''; }; services.resolved.enable = false; networking.resolvconf.useLocalResolver = true; # Hydra blasts ssh connections and does not multiplex. Loosen some of the # rate limiting. services.openssh.settings = { MaxStartups = "500:30:1000"; }; systemd.services.hydra-gc = { wantedBy = [ "multi-user.target" ]; description = "Nix Garbage Collector"; script = '' while : ; do percent_filled=$(($(stat -f --format="100-(100*%a/%b)" /mnt))) if [ "$percent_filled" -gt "54" ]; then ${config.nix.package.out}/bin/nix-store --gc --max-freed 50G --store /mnt else break fi done ''; serviceConfig.Type = "oneshot"; serviceConfig.User = "builder"; }; systemd.timers.hydra-gc = { timerConfig.OnUnitInactiveSec = "10min"; wantedBy = [ "timers.target" ]; }; systemd.timers.hydra-gc.timerConfig.Persistent = true; bagel.sysadmin.enable = true; environment.systemPackages = [ pkgs.ipmitool ]; system.stateVersion = "24.05"; }; }