diff --git a/hosts/wob-vpn-gw/netboot.nix b/hosts/wob-vpn-gw/netboot.nix index e30e822..be57011 100644 --- a/hosts/wob-vpn-gw/netboot.nix +++ b/hosts/wob-vpn-gw/netboot.nix @@ -1,4 +1,4 @@ -{ lib, pkgs, nodes, config, ... }: +{ lib, pkgs, nodes, config, modulesPath, ... }: # The way the connection is established is specific to the wob01 site and the Intel S2600KPR blades. # Proper netboot is not possible, because while the blades and the APU board (which is the netboot @@ -21,36 +21,19 @@ in { bmcIp = "192.168.1.${toString (node.config.bagel.baremetal.builders.num * 4 + 2)}"; gw = "2a01:584:11::1"; dns = "2a01:580:6000::ff01"; - ipxe = node.pkgs.ipxe.override { - embedScript = builtins.toFile "bootstrap-${node.config.networking.hostName}.ipxe" '' - #!ipxe - ifopen net0 - - echo ip ${ip}/64 - set net0/ip6:ipv6 ${ip} - set net0/len6:int8 64 - echo gw ${gw} - set net0/gateway6:ipv6 ${gw} - echo dns ${dns} - set net0/dns6:ipv6 ${dns} - - # wait for the lacp link to come up - ping --count 20 ${gw} - - chain https://hydra.forkos.org/job/infra/main/${node.config.networking.hostName}/latest/download-by-type/file/ipxe - - # if it fails, show a shell - shell - ''; - }; + notipxe = node.config.system.build.notipxe.config.system.build.usbImage; in lib.nameValuePair "iusb-spoof-${nodename}" { - wantedBy = [ "multi-user.target" ]; + + # The iusb-spoof service is currently unreliable and may lock up the BMC + block the builder from booting + # Thus, it has to be started manually per builder when needed. + #wantedBy = [ "multi-user.target" ]; + serviceConfig = { - Restart = "on-failure"; + Restart = "always"; }; script = '' AUTH_TOKEN=$(${pkgs.iusb-spoof}/bin/make-token ${bmcIp}) - exec ${pkgs.iusb-spoof}/bin/iusb-spoof -r ${bmcIp} 5123 $AUTH_TOKEN ${ipxe}/ipxe-efi.usb + exec ${pkgs.iusb-spoof}/bin/iusb-spoof -r ${bmcIp} 5123 $AUTH_TOKEN ${notipxe} ''; }) (lib.filterAttrs (_: node: node.config.bagel.baremetal.builders.enable && node.config.bagel.baremetal.builders.netboot) nodes); } diff --git a/overlays/default.nix b/overlays/default.nix index 6190c52..5162ed7 100644 --- a/overlays/default.nix +++ b/overlays/default.nix @@ -1,5 +1,6 @@ [ (final: prev: { iusb-spoof = final.callPackage ./iusb-spoof.nix {}; + u-root = final.callPackage ./u-root {}; }) ] diff --git a/overlays/u-root/default.nix b/overlays/u-root/default.nix new file mode 100644 index 0000000..acddbfb --- /dev/null +++ b/overlays/u-root/default.nix @@ -0,0 +1,20 @@ +{ buildGoModule, fetchFromGitHub }: + +buildGoModule rec { + pname = "u-root"; + version = "0.14.0"; + + src = fetchFromGitHub { + owner = "u-root"; + repo = "u-root"; + rev = "v${version}"; + hash = "sha256-8zA3pHf45MdUcq/MA/mf0KCTxB1viHieU/oigYwIPgo="; + }; + + patches = [ + ./u-root-allow-https.patch + ]; + + vendorHash = null; + doCheck = false; +} diff --git a/overlays/u-root/u-root-allow-https.patch b/overlays/u-root/u-root-allow-https.patch new file mode 100644 index 0000000..c2a0e94 --- /dev/null +++ b/overlays/u-root/u-root-allow-https.patch @@ -0,0 +1,12 @@ +diff --git a/pkg/curl/schemes.go b/pkg/curl/schemes.go +index 8bac3bc0..cd396cbc 100644 +--- a/pkg/curl/schemes.go ++++ b/pkg/curl/schemes.go +@@ -81,6 +81,7 @@ var ( + DefaultSchemes = Schemes{ + "tftp": DefaultTFTPClient, + "http": DefaultHTTPClient, ++ "https": DefaultHTTPClient, + "file": &LocalFileClient{}, + } + ) diff --git a/services/baremetal-builder/default.nix b/services/baremetal-builder/default.nix index 52c2ab6..0b9de91 100644 --- a/services/baremetal-builder/default.nix +++ b/services/baremetal-builder/default.nix @@ -1,8 +1,10 @@ -{ pkgs, lib, config, extendModules, ... }: +{ pkgs, lib, config, ... }: let cfg = config.bagel.baremetal.builders; in { + imports = [ ./netboot.nix ]; + options = { bagel.baremetal.builders = { @@ -182,39 +184,6 @@ in environment.systemPackages = [ pkgs.ipmitool ]; - system.build = lib.mkIf cfg.netboot { - netbootVariant = extendModules { - modules = [ - ( - { modulesPath, ... }: - - { - imports = [ (modulesPath + "/installer/netboot/netboot.nix") ]; - } - ) - ]; - }; - netbootDir = let - kernelTarget = pkgs.stdenv.hostPlatform.linux-kernel.target; - build = config.system.build.netbootVariant.config.system.build; - in - pkgs.symlinkJoin { - name = "netboot"; - paths = [ - build.netbootRamdisk - build.kernel - build.netbootIpxeScript - ]; - postBuild = '' - mkdir -p $out/nix-support - echo "file ${kernelTarget} $out/${kernelTarget}" >> $out/nix-support/hydra-build-products - echo "file initrd $out/initrd" >> $out/nix-support/hydra-build-products - echo "file ipxe $out/netboot.ipxe" >> $out/nix-support/hydra-build-products - ''; - preferLocalBuild = true; - }; - }; - system.stateVersion = "24.05"; }; } diff --git a/services/baremetal-builder/netboot.nix b/services/baremetal-builder/netboot.nix new file mode 100644 index 0000000..06a46be --- /dev/null +++ b/services/baremetal-builder/netboot.nix @@ -0,0 +1,149 @@ +{ modulesPath, pkgs, lib, config, extendModules, ... }@node: +let + cfg = config.bagel.baremetal.builders; +in +{ + config = lib.mkIf (cfg.enable && cfg.netboot) { + system.build = { + + # Build a kernel and initramfs which will download the IPXE script from hydra using + # u-root pxeboot tool and kexec into the final netbooted system. + notipxe = import (modulesPath + "/..") { + system = "x86_64-linux"; + configuration = + { pkgs, config, ... }: + + { + system.stateVersion = "24.11"; + boot.initrd.availableKernelModules = [ "ahci" "ehci_pci" "usb_storage" "usbhid" "sd_mod" "igb" "bonding" ]; + boot.kernelParams = [ "console=ttyS0,115200" "panic=1" "boot.panic_on_fail" ]; + #boot.initrd.systemd.emergencyAccess = true; + networking.hostName = "${node.config.networking.hostName}-boot"; + nixpkgs.overlays = import ../../overlays; + boot.loader.grub.enable = false; + fileSystems."/".device = "bogus"; # this config will never be booted + boot.initrd.systemd.enable = true; + boot.initrd.systemd.network = { + enable = true; + networks = node.config.systemd.network.networks; + netdevs = node.config.systemd.network.netdevs; + }; + boot.initrd.systemd.storePaths = [ + "${pkgs.u-root}/bin/pxeboot" + "${pkgs.iputils}/bin/ping" + ]; + boot.initrd.systemd.services.kexec = { + serviceConfig.Restart = "on-failure"; + serviceConfig.Type = "oneshot"; + wantedBy = [ "initrd-root-fs.target" ]; + before = [ "sysroot.mount" ]; + script = '' + ln -sf /dev/console /dev/tty + until ${pkgs.iputils}/bin/ping -c 1 hydra.forkos.org; do sleep 1; done + ${pkgs.u-root}/bin/pxeboot -v -ipv4=false -file https://hydra.forkos.org/job/infra/main/${node.config.networking.hostName}/latest/download-by-type/file/ipxe + ''; + }; + boot.initrd.systemd.contents."/etc/ssl/certs/ca-certificates.crt".source = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"; + boot.initrd.services.resolved.enable = false; + boot.initrd.systemd.contents."/etc/resolv.conf".text = '' + nameserver 2001:4860:4860::6464 + ''; + boot.initrd.systemd.contents."/etc/systemd/journald.conf".text = '' + [Journal] + ForwardToConsole=yes + MaxLevelConsole=debug + ''; + + # Provide a bootable USB drive image + system.build.usbImage = pkgs.callPackage ({ stdenv, runCommand, dosfstools, e2fsprogs, mtools, libfaketime, util-linux, nukeReferences }: + runCommand "boot-img-${node.config.networking.hostName}" { + nativeBuildInputs = [ dosfstools e2fsprogs libfaketime mtools util-linux ]; + } '' + export img=$out + truncate -s 40M $img + + sfdisk $img < firmware/loader/loader.conf << EOF + default foo + EOF + cat > firmware/loader/entries/default.conf << EOF + title Default + linux /EFI/${pkgs.stdenv.hostPlatform.linux-kernel.target} + initrd /EFI/initrd + options init=${config.system.build.toplevel}/init ${toString config.boot.kernelParams} + EOF + cp ${config.system.build.kernel}/${pkgs.stdenv.hostPlatform.linux-kernel.target} firmware/EFI/${pkgs.stdenv.hostPlatform.linux-kernel.target} + cp ${config.system.build.initialRamdisk}/${config.system.boot.loader.initrdFile} firmware/EFI/initrd + + find firmware -exec touch --date=2000-01-01 {} + + # Copy the populated /boot/firmware into the SD image + cd firmware + # Force a fixed order in mcopy for better determinism, and avoid file globbing + for d in $(find . -type d -mindepth 1 | sort); do + faketime "2000-01-01 00:00:00" mmd -i ../firmware_part.img "::/$d" + done + for f in $(find . -type f | sort); do + mcopy -pvm -i ../firmware_part.img "$f" "::/$f" + done + cd .. + + # Verify the FAT partition before copying it. + fsck.vfat -vn firmware_part.img + dd conv=notrunc if=firmware_part.img of=$img seek=$START count=$SECTORS + '' + ) {}; + } + ; + }; + + # This is the config which will actually be booted + netbootVariant = extendModules { + modules = [ + ( + { modulesPath, ... }: + + { + imports = [ (modulesPath + "/installer/netboot/netboot.nix") ]; + } + ) + ]; + }; + # A derivation combining all the artifacts required for netbooting for the hydra job + netbootDir = let + kernelTarget = pkgs.stdenv.hostPlatform.linux-kernel.target; + build = config.system.build.netbootVariant.config.system.build; + in + pkgs.symlinkJoin { + name = "netboot"; + paths = [ + build.netbootRamdisk + build.kernel + build.netbootIpxeScript + ]; + postBuild = '' + mkdir -p $out/nix-support + echo "file ${kernelTarget} $out/${kernelTarget}" >> $out/nix-support/hydra-build-products + echo "file initrd $out/initrd" >> $out/nix-support/hydra-build-products + echo "file ipxe $out/netboot.ipxe" >> $out/nix-support/hydra-build-products + ''; + preferLocalBuild = true; + }; + }; + }; +}