infra/services/baremetal-builder/default.nix

{ pkgs, lib, config, extendModules, ... }:
let
  cfg = config.bagel.baremetal.builders;
in
{
  options = {

    bagel.baremetal.builders = {
      enable = lib.mkEnableOption "baremetal bagel oven";
      netboot = lib.mkEnableOption "netboot";
      num = lib.mkOption {
        type = lib.types.int;
      };
    };
  };

  config = lib.mkIf cfg.enable {
    boot.initrd.availableKernelModules = [ "ahci" "ehci_pci" "usb_storage" "usbhid" "sd_mod" ];
    boot.initrd.kernelModules = [ "dm-snapshot" ];

    users.users.builder = {
      isSystemUser = true;
      group = "nogroup";
      home = "/var/empty";
      shell = "/bin/sh";
      openssh.authorizedKeys.keys = [
        # Do not hardcode Hydra's public key, selectively
        # add the keys of the coordinators that require us.
        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAvUT9YBig9LQPHgypIBHQuC32XqDKxlFZ2CfgDi0ZKx"
      ];
    };

    users.users.buildbot = {
      isSystemUser = true;
      group = "nogroup";
      home = "/var/empty";
      shell = "/bin/sh";
      openssh.authorizedKeys.keys = [
        # Do not hardcode Buildbot's public key, selectively
        # add the keys of the coordinators that require us.
        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGMnOLLX0vGTZbSJrUmF9ZFXt/NIId/MUrEpXmL2vxod"
      ];
    };
    nix.settings = {
      trusted-users = [ "builder" "buildbot" ];
      inherit ((import ./assignments.nix).${config.networking.hostName}) max-jobs cores;
    };

    nixpkgs.hostPlatform = "x86_64-linux";
    hardware.cpu.intel.updateMicrocode = true;

    boot.loader.systemd-boot.enable = true;
    boot.loader.efi.canTouchEfiVariables = true;
    boot.initrd.systemd.enable = true;

    boot.initrd.services.lvm.enable = true;

    boot.kernel.sysctl."fs.xfs.xfssyncd_centisecs" = "12000";
    fileSystems = lib.mkMerge [
      (lib.mkIf (!cfg.netboot) {
        "/" = {
          device = "/dev/disk/by-label/root";
          fsType = "xfs";
        };

        "/boot" = {
          device = "/dev/disk/by-label/BOOT";
          fsType = "vfat";
          options = [ "fmask=0022" "dmask=0022" ];
        };
      })
      {
        "/mnt" = {
          device = "/dev/disk/by-label/hydra";
          fsType = "xfs";
          options = ["logbsize=256k"];
        };

        # We want the tmp filesystem on the same filesystem as the hydra store, so that builds can use reflinks
        "/tmp" = {
          device = "/mnt/tmp";
          options = [ "bind" ];
        };
      }
    ];

    swapDevices = lib.optionals (!cfg.netboot) [
      {
        device = "/swapfile";
        size = 50 * 1024;  # 50GiB
      }
    ];

    zramSwap = {
      enable = true;
      memoryPercent = 25;
    };

    boot.kernelParams = [
      "console=tty1"
      "console=ttyS0,115200"
    ];

    networking.useNetworkd = true;
    networking.hostName = "builder-${toString cfg.num}";
    networking.domain = "wob01.infra.forkos.org";

    systemd.network = {
      netdevs = {
        "40-uplink" = {
          netdevConfig = {
            Kind = "bond";
            Name = "uplink";
          };
          bondConfig = {
            Mode = "802.3ad";
            TransmitHashPolicy = "layer3+4";
          };
        };
      };
      networks = {
        "40-eno1" = {
          name = "eno1";
          bond = [ "uplink" ];
        };
        "40-eno2" = {
          name = "eno2";
          bond = [ "uplink" ];
        };
      };
    };
    networking.interfaces.uplink.ipv6.addresses = [
      { address = "2a01:584:11::1:${toString cfg.num}"; prefixLength = 64; }
    ];
    networking.defaultGateway6 = { interface = "uplink"; address = "2a01:584:11::1"; };
    deployment.targetHost = "2a01:584:11::1:${toString cfg.num}";
    deployment.tags = [ "builders" ];

    # Why can't we have nice things? https://bugs.openjdk.org/browse/JDK-8170568
    services.coredns = {
      enable = true;
      config = ''
        . {
          bind lo
          forward . 2001:4860:4860::6464
          template ANY A { rcode NOERROR }
        }
      '';
    };
    services.resolved.enable = false;
    networking.resolvconf.useLocalResolver = true;

    # Hydra blasts ssh connections and does not multiplex. Loosen some of the
    # rate limiting.
    services.openssh.settings = {
      MaxStartups = "500:30:1000";
    };

    systemd.services.hydra-gc = {
      wantedBy = [ "multi-user.target" ];
      description = "Nix Garbage Collector";
      script = ''
        while : ; do
          percent_filled=$(($(stat -f --format="100-(100*%a/%b)" /mnt)))
          if [ "$percent_filled" -gt "54" ]; then
            ${config.nix.package.out}/bin/nix-store --gc --max-freed 50G --store /mnt
          else
            break
          fi
        done
      '';
      serviceConfig.Type = "oneshot";
      serviceConfig.User = "builder";
    };
    systemd.timers.hydra-gc = {
      timerConfig.OnUnitInactiveSec = "10min";
      wantedBy = [ "timers.target" ];
    };
    systemd.timers.hydra-gc.timerConfig.Persistent = true;

    bagel.sysadmin.enable = true;

    environment.systemPackages = [ pkgs.ipmitool ];

    system.build = lib.mkIf cfg.netboot {
      netbootVariant = extendModules {
        modules = [
          (
            { modulesPath, ... }:

            {
              imports = [ (modulesPath + "/installer/netboot/netboot.nix") ];
            }
          )
        ];
      };
      netbootDir = let
        kernelTarget = pkgs.stdenv.hostPlatform.linux-kernel.target;
        build = config.system.build.netbootVariant.config.system.build;
      in
        pkgs.symlinkJoin {
          name = "netboot";
          paths = [
            build.netbootRamdisk
            build.kernel
            build.netbootIpxeScript
          ];
          postBuild = ''
            mkdir -p $out/nix-support
            echo "file ${kernelTarget} $out/${kernelTarget}" >> $out/nix-support/hydra-build-products
            echo "file initrd $out/initrd" >> $out/nix-support/hydra-build-products
            echo "file ipxe $out/netboot.ipxe" >> $out/nix-support/hydra-build-products
          '';
          preferLocalBuild = true;
        };
    };

    system.stateVersion = "24.05";
  };
}