builders/netboot: add separate firmware_part output

at first activation it does not yet have a working network setup

common/nix.nix

@@ -14,7 +14,7 @@
 
   # Use our cache and trust its signing key. Still use cache.nixos.org as
   # fallback.
-  nix.settings.substituters = [ "https://bagel-cache.s3-web.delroth.net/" ];
+  nix.settings.substituters = [ "https://cache.forkos.org/" ];
   nix.settings.trusted-public-keys = [
     "cache.forkos.org:xfXIUJO1yiEITJmYsVmNDa9BFSlgTh/YqZ+4ei1EhQg="
   ];

flake.lock

@@ -64,11 +64,11 @@
         "treefmt-nix": "treefmt-nix"
       },
       "locked": {
-        "lastModified": 1722339507,
-        "narHash": "sha256-CdkQx6l0CuPtU9tMw3u73PAMhj+8YUAlTUVyrTCZas8=",
+        "lastModified": 1722939563,
+        "narHash": "sha256-lMe8aXgF550iQLRaoU+yn8yYQ4x2qiyqANgsFyjfWwA=",
         "ref": "refs/heads/non-flakes",
-        "rev": "15963fa0e687cb39660aadf147402c955d671029",
-        "revCount": 292,
+        "rev": "4a162a8aa5dad6cecdb33bd8534e67e0bdaeb13f",
+        "revCount": 295,
         "type": "git",
         "url": "https://git.lix.systems/lix-project/buildbot-nix.git"
       },
@@ -258,17 +258,17 @@
         ]
       },
       "locked": {
-        "lastModified": 1721682989,
-        "narHash": "sha256-kjJiZ7m4HKqbZ2mxNQiB32/goKFb8BRi8OqC4wIU0OI=",
+        "lastModified": 1722688238,
+        "narHash": "sha256-x6BnYtArF6IDs7bS8ExokgAQBOlrxXxD0EOBIlASmfM=",
         "ref": "refs/heads/main",
-        "rev": "4b107e6ff36bd89958fba36e0fe0340903e7cd13",
-        "revCount": 4190,
+        "rev": "9b5ac87de73ea4646dbb2af979db91f096d29960",
+        "revCount": 4191,
         "type": "git",
-        "url": "https://git.lix.systems/lix-project/hydra.git"
+        "url": "https://git.lix.systems/the-distro/hydra.git"
       },
       "original": {
         "type": "git",
-        "url": "https://git.lix.systems/lix-project/hydra.git"
+        "url": "https://git.lix.systems/the-distro/hydra.git"
       }
     },
     "lix": {

flake.nix

@@ -11,7 +11,7 @@
     colmena.url = "github:zhaofengli/colmena";
     colmena.inputs.nixpkgs.follows = "nixpkgs";
 
-    hydra.url = "git+https://git.lix.systems/lix-project/hydra.git";
+    hydra.url = "git+https://git.lix.systems/the-distro/hydra.git";
     hydra.inputs.nixpkgs.follows = "nixpkgs";
 
     nix-gerrit.url = "git+https://git.lix.systems/the-distro/nix-gerrit.git";

hosts/wob-vpn-gw/netboot.nix

@@ -1,56 +1,63 @@
-{ lib, pkgs, nodes, config, ... }:
+{ config, lib, pkgs, nodes, modulesPath, ... }:
 
 # The way the connection is established is specific to the wob01 site and the Intel S2600KPR blades.
 # Proper netboot is not possible, because while the blades and the APU board (which is the netboot
 # server here) are in the same L2 network, the uplink connection of each blade is an LACP LAG,
 # meaning that the switch on the other side will only enable the port if it sees valid LACP packets.
-# IPXE sends out these LACP packets while it is probing the ports, however the NICs of the blades
-# do not have a flash which IPXE could be written to.
 # We work around this by presenting a virtual floppy drive using the "IUSB" protocol of the BMC.
-# This virtual floppy drive contains an per-blade customized IPXE script which will initialize the
-# network connection including IP configuration and chainload the actual script off the netboot
-# server.
+# This virtual floppy drive contains an per-blade customized initramfs which will initialize the
+# network connection including IP configuration and load the actual image off hydra.
 
 let
   netboot-server-ip = "2a01:584:11::2";
+  netbootNodes = lib.filterAttrs (_: node: node.config.bagel.baremetal.builders.enable && node.config.bagel.baremetal.builders.netboot) nodes;
 in {
-  networking.firewall.allowedTCPPorts = [ 80 ];
+
+  assertions = [
+    {
+      assertion = !(lib.elem 443 config.networking.firewall.allowedTCPPorts);
+      message = ''
+        Port 443 is in networking.firewalls.allowedTCPPorts, but should be only manually
+        allowed for specific IPs and source ports in ${builtins.toJSON __curPos}
+      '';
+    }
+  ];
 
   systemd.services = lib.mapAttrs' (nodename: node: let
-    ip = "2a01:584:11::1:${toString node.config.bagel.baremetal.builders.num}";
     bmcIp = "192.168.1.${toString (node.config.bagel.baremetal.builders.num * 4 + 2)}";
-    gw = "2a01:584:11::1";
-    dns = "2a01:580:6000::ff01";
-    ipxe = node.pkgs.ipxe.override {
-      embedScript = builtins.toFile "bootstrap-${node.config.networking.hostName}.ipxe" ''
-        #!ipxe
-        ifopen net0
-
-        echo ip ${ip}/64
-        set net0/ip6:ipv6 ${ip}
-        set net0/len6:int8 64
-        echo gw ${gw}
-        set net0/gateway6:ipv6 ${gw}
-        echo dns ${dns}
-        set net0/dns6:ipv6 ${dns}
-
-        # wait for the lacp link to come up
-        ping --count 20 ${gw}
-
-        chain https://hydra.forkos.org/job/infra/main/${node.config.networking.hostName}/latest/download-by-type/file/ipxe
-
-        # if it fails, show a shell 
-        shell
-      '';
-    };
+    notipxe = node.config.system.build.notipxe.config.system.build.usbImage;
   in lib.nameValuePair "iusb-spoof-${nodename}" {
     wantedBy = [ "multi-user.target" ];
     serviceConfig = {
-      Restart = "on-failure";
+      Restart = "always";
     };
     script = ''
       AUTH_TOKEN=$(${pkgs.iusb-spoof}/bin/make-token ${bmcIp})
-      exec ${pkgs.iusb-spoof}/bin/iusb-spoof -r ${bmcIp} 5123 $AUTH_TOKEN ${ipxe}/ipxe-efi.usb
+      exec ${pkgs.iusb-spoof}/bin/iusb-spoof -r ${bmcIp} 5123 $AUTH_TOKEN ${notipxe}
     '';
-  }) (lib.filterAttrs (_: node: node.config.bagel.baremetal.builders.enable && node.config.bagel.baremetal.builders.netboot) nodes);
+  }) netbootNodes;
+
+  # Since the builders are stateless, they can not store their ssh hostkeys
+  networking.firewall.allowedTCPPorts = [ 80 ]; # for ACME
+  networking.firewall.extraInputRules = ''
+    ip6 saddr 2a01:584:11::/64 tcp sport < 1024 tcp dport 443 accept;
+  '';
+  security.acme.acceptTerms = true;
+  security.acme.defaults.email = "infra@forkos.org";
+  services.nginx = {
+    enable = true;
+    virtualHosts."vpn-gw.wob01.infra.forkos.org" = {
+      enableACME = true;
+      forceSSL = true;
+      locations = lib.mapAttrs' (nodename: node: let
+        ip = "2a01:584:11::1:${toString node.config.bagel.baremetal.builders.num}";
+      in lib.nameValuePair "/${nodename}/" {
+        root = "/var/www";
+        extraConfig = ''
+          allow ${ip};
+          deny all;
+        '';
+      }) netbootNodes;
+    };
+  };
 }

overlays/default.nix

@@ -1,5 +1,6 @@
 [
   (final: prev: {
     iusb-spoof = final.callPackage ./iusb-spoof.nix {};
+    u-root = final.callPackage ./u-root {};
   })
 ]

overlays/iusb-spoof.nix

@@ -9,7 +9,7 @@ rustPlatform.buildRustPackage rec {
 
   src = builtins.fetchGit {
     url = "https://git.lix.systems/the-distro/iusb-spoof/";
-    rev = "a1ec0384e724f609bb8e391512a8fa76d9894e55";
+    rev = "fafd47986239cc2f4dfbbae74b17555608806581";
   };
 
   cargoLock.lockFile = src + "/Cargo.lock";

overlays/u-root/default.nix

@@ -0,0 +1,20 @@
+{ buildGoModule, fetchFromGitHub }:
+
+buildGoModule rec {
+  pname = "u-root";
+  version = "0.14.0";
+
+  src = fetchFromGitHub {
+    owner = "u-root";
+    repo = "u-root";
+    rev = "v${version}";
+    hash = "sha256-8zA3pHf45MdUcq/MA/mf0KCTxB1viHieU/oigYwIPgo=";
+  };
+
+  patches = [
+    ./u-root-allow-https.patch
+  ];
+
+  vendorHash = null;
+  doCheck = false;
+}

overlays/u-root/u-root-allow-https.patch

@@ -0,0 +1,12 @@
+diff --git a/pkg/curl/schemes.go b/pkg/curl/schemes.go
+index 8bac3bc0..cd396cbc 100644
+--- a/pkg/curl/schemes.go
++++ b/pkg/curl/schemes.go
+@@ -81,6 +81,7 @@ var (
+ 	DefaultSchemes = Schemes{
+ 		"tftp": DefaultTFTPClient,
+ 		"http": DefaultHTTPClient,
++		"https": DefaultHTTPClient,
+ 		"file": &LocalFileClient{},
+ 	}
+ )

services/baremetal-builder/default.nix

@@ -1,8 +1,10 @@
-{ pkgs, lib, config, extendModules, ... }:
+{ pkgs, lib, config, ... }:
 let
   cfg = config.bagel.baremetal.builders;
 in
 {
+  imports = [ ./netboot.nix ];
+
   options = {
 
     bagel.baremetal.builders = {
@@ -42,10 +44,20 @@ in
       ];
     };
     nix.settings = {
-      trusted-users = [ "builder" "buildbot" ];
       inherit ((import ./assignments.nix).${config.networking.hostName}) max-jobs cores;
     };
 
+    services.openssh.extraConfig = ''
+      Match User buildbot
+        AllowAgentForwarding no
+        AllowTcpForwarding no
+        PermitTTY no
+        PermitTunnel no
+        X11Forwarding no
+        ForceCommand ${config.nix.package.out}/bin/nix-daemon --store /mnt --stdio
+      Match All
+    '';
+
     nixpkgs.hostPlatform = "x86_64-linux";
     hardware.cpu.intel.updateMicrocode = true;
 
@@ -55,31 +67,36 @@ in
 
     boot.initrd.services.lvm.enable = true;
 
-    fileSystems."/" = {
-      device = "/dev/disk/by-label/root";
-      fsType = "xfs";
-    };
-
-    fileSystems."/mnt" = {
-      device = "/dev/disk/by-label/hydra";
-      fsType = "xfs";
-      options = ["logbsize=256k"];
-    };
     boot.kernel.sysctl."fs.xfs.xfssyncd_centisecs" = "12000";
+    fileSystems = lib.mkMerge [
+      (lib.mkIf (!cfg.netboot) {
+        "/" = {
+          device = "/dev/disk/by-label/root";
+          fsType = "xfs";
+        };
 
-    # We want the tmp filesystem on the same filesystem as the hydra store, so that builds can use reflinks
-    fileSystems."/tmp" = {
-      device = "/mnt/tmp";
-      options = [ "bind" ];
-    };
+        "/boot" = {
+          device = "/dev/disk/by-label/BOOT";
+          fsType = "vfat";
+          options = [ "fmask=0022" "dmask=0022" ];
+        };
+      })
+      {
+        "/mnt" = {
+          device = "/dev/disk/by-label/hydra";
+          fsType = "xfs";
+          options = ["logbsize=256k"];
+        };
 
-    fileSystems."/boot" = {
-      device = "/dev/disk/by-label/BOOT";
-      fsType = "vfat";
-      options = [ "fmask=0022" "dmask=0022" ];
-    };
+        # We want the tmp filesystem on the same filesystem as the hydra store, so that builds can use reflinks
+        "/tmp" = {
+          device = "/mnt/tmp";
+          options = [ "bind" ];
+        };
+      }
+    ];
 
-    swapDevices = [
+    swapDevices = lib.optionals (!cfg.netboot) [
       {
         device = "/swapfile";
         size = 50 * 1024;  # 50GiB
@@ -92,8 +109,8 @@ in
     };
 
     boot.kernelParams = [
-      "console=ttyS0,115200"
       "console=tty1"
+      "console=ttyS0,115200"
     ];
 
     networking.useNetworkd = true;
@@ -177,39 +194,6 @@ in
 
     environment.systemPackages = [ pkgs.ipmitool ];
 
-    system.build = lib.mkIf cfg.netboot {
-      netbootVariant = extendModules {
-        modules = [
-          (
-            { modulesPath, ... }:
-
-            {
-              imports = [ (modulesPath + "/installer/netboot/netboot.nix") ];
-            }
-          )
-        ];
-      };
-      netbootDir = let
-        kernelTarget = pkgs.stdenv.hostPlatform.linux-kernel.target;
-        build = config.system.build.netbootVariant.config.system.build;
-      in
-        pkgs.symlinkJoin {
-          name = "netboot";
-          paths = [
-            build.netbootRamdisk
-            build.kernel
-            build.netbootIpxeScript
-          ];
-          postBuild = ''
-            mkdir -p $out/nix-support
-            echo "file ${kernelTarget} ${build.kernel}/${kernelTarget}" >> $out/nix-support/hydra-build-products
-            echo "file initrd ${build.netbootRamdisk}/initrd" >> $out/nix-support/hydra-build-products
-            echo "file ipxe ${build.netbootIpxeScript}/netboot.ipxe" >> $out/nix-support/hydra-build-products
-          '';
-          preferLocalBuild = true;
-        };
-    };
-
     system.stateVersion = "24.05";
   };
 }

services/baremetal-builder/netboot.nix

@@ -0,0 +1,169 @@
+{ modulesPath, pkgs, lib, config, extendModules, ... }@node:
+let
+  cfg = config.bagel.baremetal.builders;
+in
+{
+  config = lib.mkIf (cfg.enable && cfg.netboot) {
+    systemd.services.sshd.after = [ "provision-ssh-hostkey.service" ];
+    systemd.services.provision-ssh-hostkey = {
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = true;
+      };
+      script = ''
+        mkdir -p /etc/ssh
+        umask 0077
+        until ${pkgs.iputils}/bin/ping -c 1 vpn-gw.wob01.infra.forkos.org; do sleep 1; done
+        ${pkgs.curl}/bin/curl --local-port 25-1024 https://vpn-gw.wob01.infra.forkos.org/${config.networking.hostName}/ssh_host_ed25519_key > /etc/ssh/ssh_host_ed25519_key
+        # Run the activation script again to trigger agenix decryption
+        /run/current-system/activate
+      '';
+    };
+
+    system.build = {
+
+      # Build a kernel and initramfs which will download the IPXE script from hydra using
+      # u-root pxeboot tool and kexec into the final netbooted system.
+      notipxe = import (modulesPath + "/..") {
+        system = "x86_64-linux";
+        configuration = 
+          { pkgs, config, ... }:
+
+          {
+            system.stateVersion = "24.11";
+            boot.initrd.availableKernelModules = [ "ahci" "ehci_pci" "usb_storage" "usbhid" "sd_mod" "igb" "bonding" ];
+            boot.kernelParams = [ "console=ttyS0,115200" "panic=1" "boot.panic_on_fail" ];
+            #boot.initrd.systemd.emergencyAccess = true;
+            networking.hostName = "${node.config.networking.hostName}-boot";
+            nixpkgs.overlays = import ../../overlays;
+            boot.loader.grub.enable = false;
+            fileSystems."/".device = "bogus"; # this config will never be booted
+            boot.initrd.systemd.enable = true;
+            boot.initrd.systemd.network = {
+              enable = true;
+              networks = node.config.systemd.network.networks;
+              netdevs = node.config.systemd.network.netdevs;
+            };
+            boot.initrd.systemd.storePaths = [
+              "${pkgs.u-root}/bin/pxeboot"
+              "${pkgs.iputils}/bin/ping"
+            ];
+            boot.initrd.systemd.services.kexec = {
+              serviceConfig.Restart = "on-failure";
+              serviceConfig.Type = "oneshot";
+              wantedBy = [ "initrd-root-fs.target" ];
+              before = [ "sysroot.mount" ];
+              script = ''
+                ln -sf /dev/console /dev/tty
+                until ${pkgs.iputils}/bin/ping -c 1 hydra.forkos.org; do sleep 1; done
+                ${pkgs.u-root}/bin/pxeboot -v -ipv4=false -file https://hydra.forkos.org/job/infra/main/${node.config.networking.hostName}/latest/download-by-type/file/ipxe 
+              '';
+            };
+            boot.initrd.systemd.contents."/etc/ssl/certs/ca-certificates.crt".source = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
+            boot.initrd.services.resolved.enable = false;
+            boot.initrd.systemd.contents."/etc/resolv.conf".text = ''
+              nameserver 2001:4860:4860::6464
+            '';
+            boot.initrd.systemd.contents."/etc/systemd/journald.conf".text = ''
+              [Journal]
+              ForwardToConsole=yes
+              MaxLevelConsole=debug
+            '';
+
+            # Provide a bootable USB drive image
+            system.build.usbImage = pkgs.callPackage ({ stdenv, runCommand, dosfstools, e2fsprogs, mtools, libfaketime, util-linux, nukeReferences }:
+              runCommand "boot-img-${node.config.networking.hostName}" {
+                nativeBuildInputs = [ dosfstools e2fsprogs libfaketime mtools util-linux ];
+                outputs = [ "out" "firmware_part" ];
+              } ''
+                export img=$out
+                truncate -s 40M $img
+
+                sfdisk $img <<EOF
+                    label: gpt
+                    label-id: F222513B-DED1-49FA-B591-20CE86A2FE7F
+
+                    type=C12A7328-F81F-11D2-BA4B-00A0C93EC93B, bootable
+                EOF
+
+                # Create a FAT32 /boot/firmware partition of suitable size into firmware_part.img
+                eval $(partx $img -o START,SECTORS --nr 1 --pairs)
+                truncate -s $((2081 * 512 + SECTORS * 512)) firmware_part.img
+
+                mkfs.vfat --invariant -i 2e24ec82 -n BOOT firmware_part.img
+
+                # Populate the files intended for /boot/firmware
+                mkdir -p firmware/EFI/BOOT firmware/loader/entries
+                cp ${pkgs.systemd}/lib/systemd/boot/efi/systemd-boot*.efi firmware/EFI/BOOT/BOOT${lib.toUpper stdenv.hostPlatform.efiArch}.EFI
+
+                cat > firmware/loader/loader.conf << EOF
+                default foo
+                EOF
+                cat > firmware/loader/entries/default.conf << EOF
+                title Default
+                linux /EFI/${pkgs.stdenv.hostPlatform.linux-kernel.target}
+                initrd /EFI/initrd
+                options init=${config.system.build.toplevel}/init ${toString config.boot.kernelParams}
+                EOF
+                cp ${config.system.build.kernel}/${pkgs.stdenv.hostPlatform.linux-kernel.target} firmware/EFI/${pkgs.stdenv.hostPlatform.linux-kernel.target}
+                cp ${config.system.build.initialRamdisk}/${config.system.boot.loader.initrdFile} firmware/EFI/initrd
+
+                find firmware -exec touch --date=2000-01-01 {} +
+                # Copy the populated /boot/firmware into the SD image
+                cd firmware
+                # Force a fixed order in mcopy for better determinism, and avoid file globbing
+                for d in $(find . -type d -mindepth 1 | sort); do
+                  faketime "2000-01-01 00:00:00" mmd -i ../firmware_part.img "::/$d"
+                done
+                for f in $(find . -type f | sort); do
+                  mcopy -pvm -i ../firmware_part.img "$f" "::/$f"
+                done
+                cd ..
+
+                # Verify the FAT partition before copying it.
+                fsck.vfat -vn firmware_part.img
+                dd conv=notrunc if=firmware_part.img of=$img seek=$START count=$SECTORS
+
+                cp firmware_part.img $firmware_part
+              ''
+            ) {};
+          }
+        ;
+      };
+
+      # This is the config which will actually be booted
+      netbootVariant = extendModules {
+        modules = [
+          (
+            { modulesPath, ... }:
+
+            {
+              imports = [ (modulesPath + "/installer/netboot/netboot.nix") ];
+            }
+          )
+        ];
+      };
+      # A derivation combining all the artifacts required for netbooting for the hydra job
+      netbootDir = let
+        kernelTarget = pkgs.stdenv.hostPlatform.linux-kernel.target;
+        build = config.system.build.netbootVariant.config.system.build;
+      in
+        pkgs.symlinkJoin {
+          name = "netboot";
+          paths = [
+            build.netbootRamdisk
+            build.kernel
+            build.netbootIpxeScript
+          ];
+          postBuild = ''
+            mkdir -p $out/nix-support
+            echo "file ${kernelTarget} $out/${kernelTarget}" >> $out/nix-support/hydra-build-products
+            echo "file initrd $out/initrd" >> $out/nix-support/hydra-build-products
+            echo "file ipxe $out/netboot.ipxe" >> $out/nix-support/hydra-build-products
+          '';
+          preferLocalBuild = true;
+        };
+    };
+  };
+}