build netboot files from hydra

flake.nix

@@ -96,7 +96,7 @@
 
       makeBuilder = i: lib.nameValuePair "builder-${toString i}" {
         imports = commonModules;
-        bagel.baremetal.builders = { enable = true; num = i; };
+        bagel.baremetal.builders = { enable = true; num = i; netboot = i >= 6; };
       };
 
       builders = lib.listToAttrs (lib.genList makeBuilder 12);
@@ -114,7 +114,7 @@
       public01.imports = commonModules ++ [ ./hosts/public01 ];
     } // builders;
 
-    hydraJobs = builtins.mapAttrs (n: v: v.config.system.build.toplevel) self.nixosConfigurations;
+    hydraJobs = builtins.mapAttrs (n: v: v.config.system.build.netbootDir or v.config.system.build.toplevel) self.nixosConfigurations;
     buildbotJobs = builtins.mapAttrs (_: v: v.config.system.build.toplevel) self.nixosConfigurations;
   };
 }

hosts/wob-vpn-gw/default.nix

@@ -1,6 +1,10 @@
 { pkgs, lib, ... }:
 
 {
+  imports = [
+    ./netboot.nix
+  ];
+
   ###### Hardware ######
   boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "ehci_pci" "sd_mod" "sdhci_pci" ];
   boot.kernelModules = [ "kvm-amd" ];

hosts/wob-vpn-gw/netboot.nix

@@ -0,0 +1,56 @@
+{ lib, pkgs, nodes, config, ... }:
+
+# The way the connection is established is specific to the wob01 site and the Intel S2600KPR blades.
+# Proper netboot is not possible, because while the blades and the APU board (which is the netboot
+# server here) are in the same L2 network, the uplink connection of each blade is an LACP LAG,
+# meaning that the switch on the other side will only enable the port if it sees valid LACP packets.
+# IPXE sends out these LACP packets while it is probing the ports, however the NICs of the blades
+# do not have a flash which IPXE could be written to.
+# We work around this by presenting a virtual floppy drive using the "IUSB" protocol of the BMC.
+# This virtual floppy drive contains an per-blade customized IPXE script which will initialize the
+# network connection including IP configuration and chainload the actual script off the netboot
+# server.
+
+let
+  netboot-server-ip = "2a01:584:11::2";
+in {
+  networking.firewall.allowedTCPPorts = [ 80 ];
+
+  systemd.services = lib.mapAttrs' (nodename: node: let
+    ip = "2a01:584:11::1:${toString node.config.bagel.baremetal.builders.num}";
+    bmcIp = "192.168.1.${toString (node.config.bagel.baremetal.builders.num * 4 + 2)}";
+    gw = "2a01:584:11::1";
+    dns = "2a01:580:6000::ff01";
+    ipxe = node.pkgs.ipxe.override {
+      embedScript = builtins.toFile "bootstrap-${node.config.networking.hostName}.ipxe" ''
+        #!ipxe
+        ifopen net0
+
+        echo ip ${ip}/64
+        set net0/ip6:ipv6 ${ip}
+        set net0/len6:int8 64
+        echo gw ${gw}
+        set net0/gateway6:ipv6 ${gw}
+        echo dns ${dns}
+        set net0/dns6:ipv6 ${dns}
+
+        # wait for the lacp link to come up
+        ping --count 20 ${gw}
+
+        chain https://hydra.forkos.org/job/infra/main/${node.config.networking.hostName}/latest/download-by-type/file/ipxe
+
+        # if it fails, show a shell 
+        shell
+      '';
+    };
+  in lib.nameValuePair "iusb-spoof-${nodename}" {
+    wantedBy = [ "multi-user.target" ];
+    serviceConfig = {
+      Restart = "on-failure";
+    };
+    script = ''
+      AUTH_TOKEN=$(${pkgs.iusb-spoof}/bin/make-token ${bmcIp})
+      exec ${pkgs.iusb-spoof}/bin/iusb-spoof -r ${bmcIp} 5123 $AUTH_TOKEN ${ipxe}/ipxe-efi.usb
+    '';
+  }) (lib.filterAttrs (_: node: node.config.bagel.baremetal.builders.enable && node.config.bagel.baremetal.builders.netboot) nodes);
+}

overlays/default.nix

@@ -1 +1,5 @@
-[]
+[
+  (final: prev: {
+    iusb-spoof = final.callPackage ./iusb-spoof.nix {};
+  })
+]

overlays/iusb-spoof.nix

@@ -0,0 +1,23 @@
+{ rustPlatform, python3, makeWrapper }:
+let
+  pythonEnv = python3.withPackages (p: with p; [ requests ]);
+in
+
+rustPlatform.buildRustPackage rec {
+  pname = "iusb-spoof";
+  version = "0.1.0";
+
+  src = builtins.fetchGit {
+    url = "https://git.lix.systems/the-distro/iusb-spoof/";
+    rev = "a1ec0384e724f609bb8e391512a8fa76d9894e55";
+  };
+
+  cargoLock.lockFile = src + "/Cargo.lock";
+
+  nativeBuildInputs = [ makeWrapper ];
+
+  postInstall = ''
+    install -Dm644 $src/make-token.py $out/opt/make-token.py
+    makeWrapper ${pythonEnv.interpreter} $out/bin/make-token --add-flags "$out/opt/make-token.py"
+  '';
+}

services/baremetal-builder/default.nix

@@ -1,4 +1,4 @@
-{ pkgs, lib, config, ... }:
+{ pkgs, lib, config, extendModules, ... }:
 let
   cfg = config.bagel.baremetal.builders;
 in
@@ -7,6 +7,7 @@ in
 
     bagel.baremetal.builders = {
       enable = lib.mkEnableOption "baremetal bagel oven";
+      netboot = lib.mkEnableOption "netboot";
       num = lib.mkOption {
         type = lib.types.int;
       };
@@ -176,6 +177,39 @@ in
 
     environment.systemPackages = [ pkgs.ipmitool ];
 
+    system.build = lib.mkIf cfg.netboot {
+      netbootVariant = extendModules {
+        modules = [
+          (
+            { modulesPath, ... }:
+
+            {
+              imports = [ (modulesPath + "/installer/netboot/netboot.nix") ];
+            }
+          )
+        ];
+      };
+      netbootDir = let
+        kernelTarget = pkgs.stdenv.hostPlatform.linux-kernel.target;
+        build = config.system.build.netbootVariant.config.system.build;
+      in
+        pkgs.symlinkJoin {
+          name = "netboot";
+          paths = [
+            build.netbootRamdisk
+            build.kernel
+            build.netbootIpxeScript
+          ];
+          postBuild = ''
+            mkdir -p $out/nix-support
+            echo "file ${kernelTarget} ${build.kernel}/${kernelTarget}" >> $out/nix-support/hydra-build-products
+            echo "file initrd ${build.netbootRamdisk}/initrd" >> $out/nix-support/hydra-build-products
+            echo "file ipxe ${build.netbootIpxeScript}/netboot.ipxe" >> $out/nix-support/hydra-build-products
+          '';
+          preferLocalBuild = true;
+        };
+    };
+
     system.stateVersion = "24.05";
   };
 }