From 6dc424dd43956a1f672394a366381de40807ab95 Mon Sep 17 00:00:00 2001
From: Yureka <yuka@yuka.dev>
Date: Thu, 1 Aug 2024 22:16:48 +0200
Subject: [PATCH] wob01: serve an ipxe over iusb-spoof

---
 flake.nix                              |  2 +-
 hosts/wob-vpn-gw/default.nix           |  4 ++
 hosts/wob-vpn-gw/netboot.nix           | 56 ++++++++++++++++++++++++++
 overlays/default.nix                   |  6 ++-
 overlays/iusb-spoof.nix                | 23 +++++++++++
 services/baremetal-builder/default.nix |  1 +
 6 files changed, 90 insertions(+), 2 deletions(-)
 create mode 100644 hosts/wob-vpn-gw/netboot.nix
 create mode 100644 overlays/iusb-spoof.nix

diff --git a/flake.nix b/flake.nix
index 76c7e3f..0174d9d 100644
--- a/flake.nix
+++ b/flake.nix
@@ -96,7 +96,7 @@
 
       makeBuilder = i: lib.nameValuePair "builder-${toString i}" {
         imports = commonModules;
-        bagel.baremetal.builders = { enable = true; num = i; };
+        bagel.baremetal.builders = { enable = true; num = i; netboot = i >= 6; };
       };
 
       builders = lib.listToAttrs (lib.genList makeBuilder 12);
diff --git a/hosts/wob-vpn-gw/default.nix b/hosts/wob-vpn-gw/default.nix
index e56232c..80a7655 100644
--- a/hosts/wob-vpn-gw/default.nix
+++ b/hosts/wob-vpn-gw/default.nix
@@ -1,6 +1,10 @@
 { pkgs, lib, ... }:
 
 {
+  imports = [
+    ./netboot.nix
+  ];
+
   ###### Hardware ######
   boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "ehci_pci" "sd_mod" "sdhci_pci" ];
   boot.kernelModules = [ "kvm-amd" ];
diff --git a/hosts/wob-vpn-gw/netboot.nix b/hosts/wob-vpn-gw/netboot.nix
new file mode 100644
index 0000000..e30e822
--- /dev/null
+++ b/hosts/wob-vpn-gw/netboot.nix
@@ -0,0 +1,56 @@
+{ lib, pkgs, nodes, config, ... }:
+
+# The way the connection is established is specific to the wob01 site and the Intel S2600KPR blades.
+# Proper netboot is not possible, because while the blades and the APU board (which is the netboot
+# server here) are in the same L2 network, the uplink connection of each blade is an LACP LAG,
+# meaning that the switch on the other side will only enable the port if it sees valid LACP packets.
+# IPXE sends out these LACP packets while it is probing the ports, however the NICs of the blades
+# do not have a flash which IPXE could be written to.
+# We work around this by presenting a virtual floppy drive using the "IUSB" protocol of the BMC.
+# This virtual floppy drive contains an per-blade customized IPXE script which will initialize the
+# network connection including IP configuration and chainload the actual script off the netboot
+# server.
+
+let
+  netboot-server-ip = "2a01:584:11::2";
+in {
+  networking.firewall.allowedTCPPorts = [ 80 ];
+
+  systemd.services = lib.mapAttrs' (nodename: node: let
+    ip = "2a01:584:11::1:${toString node.config.bagel.baremetal.builders.num}";
+    bmcIp = "192.168.1.${toString (node.config.bagel.baremetal.builders.num * 4 + 2)}";
+    gw = "2a01:584:11::1";
+    dns = "2a01:580:6000::ff01";
+    ipxe = node.pkgs.ipxe.override {
+      embedScript = builtins.toFile "bootstrap-${node.config.networking.hostName}.ipxe" ''
+        #!ipxe
+        ifopen net0
+
+        echo ip ${ip}/64
+        set net0/ip6:ipv6 ${ip}
+        set net0/len6:int8 64
+        echo gw ${gw}
+        set net0/gateway6:ipv6 ${gw}
+        echo dns ${dns}
+        set net0/dns6:ipv6 ${dns}
+
+        # wait for the lacp link to come up
+        ping --count 20 ${gw}
+
+        chain https://hydra.forkos.org/job/infra/main/${node.config.networking.hostName}/latest/download-by-type/file/ipxe
+
+        # if it fails, show a shell 
+        shell
+      '';
+    };
+  in lib.nameValuePair "iusb-spoof-${nodename}" {
+    wantedBy = [ "multi-user.target" ];
+    serviceConfig = {
+      Restart = "on-failure";
+    };
+    script = ''
+      AUTH_TOKEN=$(${pkgs.iusb-spoof}/bin/make-token ${bmcIp})
+      exec ${pkgs.iusb-spoof}/bin/iusb-spoof -r ${bmcIp} 5123 $AUTH_TOKEN ${ipxe}/ipxe-efi.usb
+    '';
+  }) (lib.filterAttrs (_: node: node.config.bagel.baremetal.builders.enable && node.config.bagel.baremetal.builders.netboot) nodes);
+}
diff --git a/overlays/default.nix b/overlays/default.nix
index fe51488..6190c52 100644
--- a/overlays/default.nix
+++ b/overlays/default.nix
@@ -1 +1,5 @@
-[]
+[
+  (final: prev: {
+    iusb-spoof = final.callPackage ./iusb-spoof.nix {};
+  })
+]
diff --git a/overlays/iusb-spoof.nix b/overlays/iusb-spoof.nix
new file mode 100644
index 0000000..2afe31a
--- /dev/null
+++ b/overlays/iusb-spoof.nix
@@ -0,0 +1,23 @@
+{ rustPlatform, python3, makeWrapper }:
+let
+  pythonEnv = python3.withPackages (p: with p; [ requests ]);
+in
+
+rustPlatform.buildRustPackage rec {
+  pname = "iusb-spoof";
+  version = "0.1.0";
+
+  src = builtins.fetchGit {
+    url = "https://git.lix.systems/the-distro/iusb-spoof/";
+    rev = "a1ec0384e724f609bb8e391512a8fa76d9894e55";
+  };
+
+  cargoLock.lockFile = src + "/Cargo.lock";
+
+  nativeBuildInputs = [ makeWrapper ];
+
+  postInstall = ''
+    install -Dm644 $src/make-token.py $out/opt/make-token.py
+    makeWrapper ${pythonEnv.interpreter} $out/bin/make-token --add-flags "$out/opt/make-token.py"
+  '';
+}
diff --git a/services/baremetal-builder/default.nix b/services/baremetal-builder/default.nix
index 74af67f..9c5e58d 100644
--- a/services/baremetal-builder/default.nix
+++ b/services/baremetal-builder/default.nix
@@ -7,6 +7,7 @@ in
 
     bagel.baremetal.builders = {
       enable = lib.mkEnableOption "baremetal bagel oven";
+      netboot = lib.mkEnableOption "netboot";
       num = lib.mkOption {
         type = lib.types.int;
       };