diff --git a/flake.nix b/flake.nix index 76c7e3f..0174d9d 100644 --- a/flake.nix +++ b/flake.nix @@ -96,7 +96,7 @@ makeBuilder = i: lib.nameValuePair "builder-${toString i}" { imports = commonModules; - bagel.baremetal.builders = { enable = true; num = i; }; + bagel.baremetal.builders = { enable = true; num = i; netboot = i >= 6; }; }; builders = lib.listToAttrs (lib.genList makeBuilder 12); diff --git a/hosts/wob-vpn-gw/default.nix b/hosts/wob-vpn-gw/default.nix index e56232c..80a7655 100644 --- a/hosts/wob-vpn-gw/default.nix +++ b/hosts/wob-vpn-gw/default.nix @@ -1,6 +1,10 @@ { pkgs, lib, ... }: { + imports = [ + ./netboot.nix + ]; + ###### Hardware ###### boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "ehci_pci" "sd_mod" "sdhci_pci" ]; boot.kernelModules = [ "kvm-amd" ]; diff --git a/hosts/wob-vpn-gw/netboot.nix b/hosts/wob-vpn-gw/netboot.nix new file mode 100644 index 0000000..e30e822 --- /dev/null +++ b/hosts/wob-vpn-gw/netboot.nix @@ -0,0 +1,56 @@ +{ lib, pkgs, nodes, config, ... }: + +# The way the connection is established is specific to the wob01 site and the Intel S2600KPR blades. +# Proper netboot is not possible, because while the blades and the APU board (which is the netboot +# server here) are in the same L2 network, the uplink connection of each blade is an LACP LAG, +# meaning that the switch on the other side will only enable the port if it sees valid LACP packets. +# IPXE sends out these LACP packets while it is probing the ports, however the NICs of the blades +# do not have a flash which IPXE could be written to. +# We work around this by presenting a virtual floppy drive using the "IUSB" protocol of the BMC. +# This virtual floppy drive contains an per-blade customized IPXE script which will initialize the +# network connection including IP configuration and chainload the actual script off the netboot +# server. + +let + netboot-server-ip = "2a01:584:11::2"; +in { + networking.firewall.allowedTCPPorts = [ 80 ]; + + systemd.services = lib.mapAttrs' (nodename: node: let + ip = "2a01:584:11::1:${toString node.config.bagel.baremetal.builders.num}"; + bmcIp = "192.168.1.${toString (node.config.bagel.baremetal.builders.num * 4 + 2)}"; + gw = "2a01:584:11::1"; + dns = "2a01:580:6000::ff01"; + ipxe = node.pkgs.ipxe.override { + embedScript = builtins.toFile "bootstrap-${node.config.networking.hostName}.ipxe" '' + #!ipxe + ifopen net0 + + echo ip ${ip}/64 + set net0/ip6:ipv6 ${ip} + set net0/len6:int8 64 + echo gw ${gw} + set net0/gateway6:ipv6 ${gw} + echo dns ${dns} + set net0/dns6:ipv6 ${dns} + + # wait for the lacp link to come up + ping --count 20 ${gw} + + chain https://hydra.forkos.org/job/infra/main/${node.config.networking.hostName}/latest/download-by-type/file/ipxe + + # if it fails, show a shell + shell + ''; + }; + in lib.nameValuePair "iusb-spoof-${nodename}" { + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + Restart = "on-failure"; + }; + script = '' + AUTH_TOKEN=$(${pkgs.iusb-spoof}/bin/make-token ${bmcIp}) + exec ${pkgs.iusb-spoof}/bin/iusb-spoof -r ${bmcIp} 5123 $AUTH_TOKEN ${ipxe}/ipxe-efi.usb + ''; + }) (lib.filterAttrs (_: node: node.config.bagel.baremetal.builders.enable && node.config.bagel.baremetal.builders.netboot) nodes); +} diff --git a/overlays/default.nix b/overlays/default.nix index fe51488..6190c52 100644 --- a/overlays/default.nix +++ b/overlays/default.nix @@ -1 +1,5 @@ -[] +[ + (final: prev: { + iusb-spoof = final.callPackage ./iusb-spoof.nix {}; + }) +] diff --git a/overlays/iusb-spoof.nix b/overlays/iusb-spoof.nix new file mode 100644 index 0000000..2afe31a --- /dev/null +++ b/overlays/iusb-spoof.nix @@ -0,0 +1,23 @@ +{ rustPlatform, python3, makeWrapper }: +let + pythonEnv = python3.withPackages (p: with p; [ requests ]); +in + +rustPlatform.buildRustPackage rec { + pname = "iusb-spoof"; + version = "0.1.0"; + + src = builtins.fetchGit { + url = "https://git.lix.systems/the-distro/iusb-spoof/"; + rev = "a1ec0384e724f609bb8e391512a8fa76d9894e55"; + }; + + cargoLock.lockFile = src + "/Cargo.lock"; + + nativeBuildInputs = [ makeWrapper ]; + + postInstall = '' + install -Dm644 $src/make-token.py $out/opt/make-token.py + makeWrapper ${pythonEnv.interpreter} $out/bin/make-token --add-flags "$out/opt/make-token.py" + ''; +} diff --git a/services/baremetal-builder/default.nix b/services/baremetal-builder/default.nix index 74af67f..9c5e58d 100644 --- a/services/baremetal-builder/default.nix +++ b/services/baremetal-builder/default.nix @@ -7,6 +7,7 @@ in bagel.baremetal.builders = { enable = lib.mkEnableOption "baremetal bagel oven"; + netboot = lib.mkEnableOption "netboot"; num = lib.mkOption { type = lib.types.int; };