infra/services/baremetal-builder/netboot.nix

{ modulesPath, pkgs, lib, config, extendModules, ... }@node:
let
  cfg = config.bagel.baremetal.builders;
in
{
  config = lib.mkIf (cfg.enable && cfg.netboot) {
    systemd.services.sshd.after = [ "provision-ssh-hostkey.service" ];
    systemd.services.provision-ssh-hostkey = {
      wantedBy = [ "multi-user.target" ];
      serviceConfig = {
        Type = "oneshot";
        RemainAfterExit = true;
      };
      script = ''
        mkdir -p /etc/ssh
        umask 0077
        until ${pkgs.iputils}/bin/ping -c 1 vpn-gw.wob01.infra.forkos.org; do sleep 1; done
        ${pkgs.curl}/bin/curl --local-port 25-1024 https://vpn-gw.wob01.infra.forkos.org/${config.networking.hostName}/ssh_host_ed25519_key > /etc/ssh/ssh_host_ed25519_key
        # Run the activation script again to trigger agenix decryption
        /run/current-system/activate
      '';
    };

    system.build = {

      # Build a kernel and initramfs which will download the IPXE script from hydra using
      # u-root pxeboot tool and kexec into the final netbooted system.
      notipxe = import (modulesPath + "/..") {
        system = "x86_64-linux";
        configuration = 
          { pkgs, config, ... }:

          {
            system.stateVersion = "24.11";
            boot.initrd.availableKernelModules = [ "ahci" "ehci_pci" "usb_storage" "usbhid" "sd_mod" "igb" "bonding" ];
            boot.kernelParams = [ "console=ttyS0,115200" "panic=1" "boot.panic_on_fail" ];
            #boot.initrd.systemd.emergencyAccess = true;
            networking.hostName = "${node.config.networking.hostName}-boot";
            nixpkgs.overlays = import ../../overlays;
            boot.loader.grub.enable = false;
            fileSystems."/".device = "bogus"; # this config will never be booted
            boot.initrd.systemd.enable = true;
            boot.initrd.systemd.network = {
              enable = true;
              networks = node.config.systemd.network.networks;
              netdevs = node.config.systemd.network.netdevs;
            };
            boot.initrd.systemd.storePaths = [
              "${pkgs.u-root}/bin/pxeboot"
              "${pkgs.iputils}/bin/ping"
            ];
            boot.initrd.systemd.services.kexec = {
              serviceConfig.Restart = "on-failure";
              serviceConfig.Type = "oneshot";
              wantedBy = [ "initrd-root-fs.target" ];
              before = [ "sysroot.mount" ];
              script = ''
                ln -sf /dev/console /dev/tty
                until ${pkgs.iputils}/bin/ping -c 1 hydra.forkos.org; do sleep 1; done
                ${pkgs.u-root}/bin/pxeboot -v -ipv4=false -file https://hydra.forkos.org/job/infra/main/${node.config.networking.hostName}/latest/download-by-type/file/ipxe 
              '';
            };
            boot.initrd.systemd.contents."/etc/ssl/certs/ca-certificates.crt".source = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
            boot.initrd.services.resolved.enable = false;
            boot.initrd.systemd.contents."/etc/resolv.conf".text = ''
              nameserver 2001:4860:4860::6464
            '';
            boot.initrd.systemd.contents."/etc/systemd/journald.conf".text = ''
              [Journal]
              ForwardToConsole=yes
              MaxLevelConsole=debug
            '';

            # Provide a bootable USB drive image
            system.build.usbImage = pkgs.callPackage ({ stdenv, runCommand, dosfstools, e2fsprogs, mtools, libfaketime, util-linux, nukeReferences }:
              runCommand "boot-img-${node.config.networking.hostName}" {
                nativeBuildInputs = [ dosfstools e2fsprogs libfaketime mtools util-linux ];
                outputs = [ "out" "firmware_part" ];
              } ''
                export img=$out
                truncate -s 40M $img

                sfdisk $img <<EOF
                    label: gpt
                    label-id: F222513B-DED1-49FA-B591-20CE86A2FE7F

                    type=C12A7328-F81F-11D2-BA4B-00A0C93EC93B, bootable
                EOF

                # Create a FAT32 /boot/firmware partition of suitable size into firmware_part.img
                eval $(partx $img -o START,SECTORS --nr 1 --pairs)
                truncate -s $((2081 * 512 + SECTORS * 512)) firmware_part.img

                mkfs.vfat --invariant -i 2e24ec82 -n BOOT firmware_part.img

                # Populate the files intended for /boot/firmware
                mkdir -p firmware/EFI/BOOT firmware/loader/entries
                cp ${pkgs.systemd}/lib/systemd/boot/efi/systemd-boot*.efi firmware/EFI/BOOT/BOOT${lib.toUpper stdenv.hostPlatform.efiArch}.EFI

                cat > firmware/loader/loader.conf << EOF
                default foo
                EOF
                cat > firmware/loader/entries/default.conf << EOF
                title Default
                linux /EFI/${pkgs.stdenv.hostPlatform.linux-kernel.target}
                initrd /EFI/initrd
                options init=${config.system.build.toplevel}/init ${toString config.boot.kernelParams}
                EOF
                cp ${config.system.build.kernel}/${pkgs.stdenv.hostPlatform.linux-kernel.target} firmware/EFI/${pkgs.stdenv.hostPlatform.linux-kernel.target}
                cp ${config.system.build.initialRamdisk}/${config.system.boot.loader.initrdFile} firmware/EFI/initrd

                find firmware -exec touch --date=2000-01-01 {} +
                # Copy the populated /boot/firmware into the SD image
                cd firmware
                # Force a fixed order in mcopy for better determinism, and avoid file globbing
                for d in $(find . -type d -mindepth 1 | sort); do
                  faketime "2000-01-01 00:00:00" mmd -i ../firmware_part.img "::/$d"
                done
                for f in $(find . -type f | sort); do
                  mcopy -pvm -i ../firmware_part.img "$f" "::/$f"
                done
                cd ..

                # Verify the FAT partition before copying it.
                fsck.vfat -vn firmware_part.img
                dd conv=notrunc if=firmware_part.img of=$img seek=$START count=$SECTORS

                cp firmware_part.img $firmware_part
              ''
            ) {};
          }
        ;
      };

      # This is the config which will actually be booted
      netbootVariant = extendModules {
        modules = [
          (
            { modulesPath, ... }:

            {
              imports = [ (modulesPath + "/installer/netboot/netboot.nix") ];
            }
          )
        ];
      };
      # A derivation combining all the artifacts required for netbooting for the hydra job
      netbootDir = let
        kernelTarget = pkgs.stdenv.hostPlatform.linux-kernel.target;
        build = config.system.build.netbootVariant.config.system.build;
      in
        pkgs.symlinkJoin {
          name = "netboot";
          paths = [
            build.netbootRamdisk
            build.kernel
            build.netbootIpxeScript
          ];
          postBuild = ''
            mkdir -p $out/nix-support
            echo "file ${kernelTarget} $out/${kernelTarget}" >> $out/nix-support/hydra-build-products
            echo "file initrd $out/initrd" >> $out/nix-support/hydra-build-products
            echo "file ipxe $out/netboot.ipxe" >> $out/nix-support/hydra-build-products
          '';
          preferLocalBuild = true;
        };
    };
  };
}