{ lib, pkgs, nodes, config, ... }: # The way the connection is established is specific to the wob01 site and the Intel S2600KPR blades. # Proper netboot is not possible, because while the blades and the APU board (which is the netboot # server here) are in the same L2 network, the uplink connection of each blade is an LACP LAG, # meaning that the switch on the other side will only enable the port if it sees valid LACP packets. # IPXE sends out these LACP packets while it is probing the ports, however the NICs of the blades # do not have a flash which IPXE could be written to. # We work around this by presenting a virtual floppy drive using the "IUSB" protocol of the BMC. # This virtual floppy drive contains an per-blade customized IPXE script which will initialize the # network connection including IP configuration and chainload the actual script off the netboot # server. let netboot-server-ip = "2a01:584:11::2"; in { networking.firewall.allowedTCPPorts = [ 80 ]; systemd.services = lib.mapAttrs' (nodename: node: let ip = "2a01:584:11::1:${toString node.config.bagel.baremetal.builders.num}"; bmcIp = "192.168.1.${toString (node.config.bagel.baremetal.builders.num * 4 + 2)}"; gw = "2a01:584:11::1"; dns = "2a01:580:6000::ff01"; ipxe = node.pkgs.ipxe.override { embedScript = builtins.toFile "bootstrap-${node.config.networking.hostName}.ipxe" '' #!ipxe ifopen net0 echo ip ${ip}/64 set net0/ip6:ipv6 ${ip} set net0/len6:int8 64 echo gw ${gw} set net0/gateway6:ipv6 ${gw} echo dns ${dns} set net0/dns6:ipv6 ${dns} # wait for the lacp link to come up ping --count 20 ${gw} chain https://hydra.forkos.org/job/infra/main/${node.config.networking.hostName}/latest/download-by-type/file/ipxe # if it fails, show a shell shell ''; }; in lib.nameValuePair "iusb-spoof-${nodename}" { wantedBy = [ "multi-user.target" ]; serviceConfig = { Restart = "on-failure"; }; script = '' AUTH_TOKEN=$(${pkgs.iusb-spoof}/bin/make-token ${bmcIp}) exec ${pkgs.iusb-spoof}/bin/iusb-spoof -r ${bmcIp} 5123 $AUTH_TOKEN ${ipxe}/ipxe-efi.usb ''; }) (lib.filterAttrs (_: node: node.config.bagel.baremetal.builders.enable && node.config.bagel.baremetal.builders.netboot) nodes); }