infra/services/baremetal-builder/default.nix

{
  pkgs,
  lib,
  config,
  ...
}:
let
  cfg = config.bagel.baremetal.builders;
in
{
  imports = [ ./netboot.nix ];

  options = {

    bagel.baremetal.builders = {
      enable = lib.mkEnableOption "baremetal bagel oven";
      netboot = lib.mkEnableOption "netboot";
      num = lib.mkOption { type = lib.types.int; };
    };
  };

  config = lib.mkIf cfg.enable {
    boot.initrd.availableKernelModules = [
      "ahci"
      "ehci_pci"
      "usb_storage"
      "usbhid"
      "sd_mod"
    ];
    boot.initrd.kernelModules = [ "dm-snapshot" ];

    users.users.builder = {
      isSystemUser = true;
      group = "nogroup";
      home = "/var/empty";
      shell = "/bin/sh";
      openssh.authorizedKeys.keys = [
        # Do not hardcode Hydra's public key, selectively
        # add the keys of the coordinators that require us.
        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAvUT9YBig9LQPHgypIBHQuC32XqDKxlFZ2CfgDi0ZKx"
      ];
    };

    users.users.buildbot = {
      isSystemUser = true;
      group = "nogroup";
      home = "/var/empty";
      shell = "/bin/sh";
      openssh.authorizedKeys.keys = [
        # Do not hardcode Buildbot's public key, selectively
        # add the keys of the coordinators that require us.
        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGMnOLLX0vGTZbSJrUmF9ZFXt/NIId/MUrEpXmL2vxod"
      ];
    };
    nix.settings = {
      trusted-users = [
        "builder"
        "buildbot"
      ];
      inherit ((import ./assignments.nix).${config.networking.hostName}) max-jobs cores;
    };

    nixpkgs.hostPlatform = "x86_64-linux";
    hardware.cpu.intel.updateMicrocode = true;

    boot.loader.systemd-boot.enable = true;
    boot.loader.efi.canTouchEfiVariables = true;
    boot.initrd.systemd.enable = true;

    boot.initrd.services.lvm.enable = true;

    boot.kernel.sysctl."fs.xfs.xfssyncd_centisecs" = "12000";
    fileSystems = lib.mkMerge [
      (lib.mkIf (!cfg.netboot) {
        "/" = {
          device = "/dev/disk/by-label/root";
          fsType = "xfs";
        };

        "/boot" = {
          device = "/dev/disk/by-label/BOOT";
          fsType = "vfat";
          options = [
            "fmask=0022"
            "dmask=0022"
          ];
        };
      })
      {
        "/mnt" = {
          device = "/dev/disk/by-label/hydra";
          fsType = "xfs";
          options = [ "logbsize=256k" ];
        };

        # We want the tmp filesystem on the same filesystem as the hydra store, so that builds can use reflinks
        "/tmp" = {
          device = "/mnt/tmp";
          options = [ "bind" ];
        };
      }
    ];

    swapDevices = lib.optionals (!cfg.netboot) [
      {
        device = "/swapfile";
        size = 50 * 1024; # 50GiB
      }
    ];

    zramSwap = {
      enable = true;
      memoryPercent = 25;
    };

    boot.kernelParams = [
      "console=tty1"
      "console=ttyS0,115200"
    ];

    networking.useNetworkd = true;
    networking.hostName = "builder-${toString cfg.num}";
    networking.domain = "wob01.infra.forkos.org";

    systemd.network = {
      netdevs = {
        "40-uplink" = {
          netdevConfig = {
            Kind = "bond";
            Name = "uplink";
          };
          bondConfig = {
            Mode = "802.3ad";
            TransmitHashPolicy = "layer3+4";
          };
        };
      };
      networks = {
        "40-eno1" = {
          name = "eno1";
          bond = [ "uplink" ];
        };
        "40-eno2" = {
          name = "eno2";
          bond = [ "uplink" ];
        };
      };
    };
    networking.interfaces.uplink.ipv6.addresses = [
      {
        address = "2a01:584:11::1:${toString cfg.num}";
        prefixLength = 64;
      }
    ];
    networking.defaultGateway6 = {
      interface = "uplink";
      address = "2a01:584:11::1";
    };
    deployment.targetHost = "2a01:584:11::1:${toString cfg.num}";
    deployment.tags = [ "builders" ];

    # Why can't we have nice things? https://bugs.openjdk.org/browse/JDK-8170568
    services.coredns = {
      enable = true;
      config = ''
        . {
          bind lo
          forward . 2001:4860:4860::6464
          template ANY A { rcode NOERROR }
        }
      '';
    };
    services.resolved.enable = false;
    networking.resolvconf.useLocalResolver = true;

    # Hydra blasts ssh connections and does not multiplex. Loosen some of the
    # rate limiting.
    services.openssh.settings = {
      MaxStartups = "500:30:1000";
    };

    systemd.services.hydra-gc = {
      wantedBy = [ "multi-user.target" ];
      description = "Nix Garbage Collector";
      script = ''
        while : ; do
          percent_filled=$(($(stat -f --format="100-(100*%a/%b)" /mnt)))
          if [ "$percent_filled" -gt "54" ]; then
            ${config.nix.package.out}/bin/nix-store --gc --max-freed 50G --store /mnt
          else
            break
          fi
        done
      '';
      serviceConfig.Type = "oneshot";
      serviceConfig.User = "builder";
    };
    systemd.timers.hydra-gc = {
      timerConfig.OnUnitInactiveSec = "10min";
      wantedBy = [ "timers.target" ];
    };
    systemd.timers.hydra-gc.timerConfig.Persistent = true;

    bagel.sysadmin.enable = true;

    environment.systemPackages = [ pkgs.ipmitool ];

    system.stateVersion = "24.05";
  };
}