forked from the-distro/infra
add 'notipxe' netboot loader based on systemd-initrd + u-root
This commit is contained in:
parent
aeb8102ae4
commit
79dea0686b
|
|
@ -1,4 +1,4 @@
|
|||
{ lib, pkgs, nodes, config, ... }:
|
||||
{ lib, pkgs, nodes, config, modulesPath, ... }:
|
||||
|
||||
# The way the connection is established is specific to the wob01 site and the Intel S2600KPR blades.
|
||||
# Proper netboot is not possible, because while the blades and the APU board (which is the netboot
|
||||
|
|
@ -21,36 +21,19 @@ in {
|
|||
bmcIp = "192.168.1.${toString (node.config.bagel.baremetal.builders.num * 4 + 2)}";
|
||||
gw = "2a01:584:11::1";
|
||||
dns = "2a01:580:6000::ff01";
|
||||
ipxe = node.pkgs.ipxe.override {
|
||||
embedScript = builtins.toFile "bootstrap-${node.config.networking.hostName}.ipxe" ''
|
||||
#!ipxe
|
||||
ifopen net0
|
||||
|
||||
echo ip ${ip}/64
|
||||
set net0/ip6:ipv6 ${ip}
|
||||
set net0/len6:int8 64
|
||||
echo gw ${gw}
|
||||
set net0/gateway6:ipv6 ${gw}
|
||||
echo dns ${dns}
|
||||
set net0/dns6:ipv6 ${dns}
|
||||
|
||||
# wait for the lacp link to come up
|
||||
ping --count 20 ${gw}
|
||||
|
||||
chain https://hydra.forkos.org/job/infra/main/${node.config.networking.hostName}/latest/download-by-type/file/ipxe
|
||||
|
||||
# if it fails, show a shell
|
||||
shell
|
||||
'';
|
||||
};
|
||||
notipxe = node.config.system.build.notipxe.config.system.build.usbImage;
|
||||
in lib.nameValuePair "iusb-spoof-${nodename}" {
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
|
||||
# The iusb-spoof service is currently unreliable and may lock up the BMC + block the builder from booting
|
||||
# Thus, it has to be started manually per builder when needed.
|
||||
#wantedBy = [ "multi-user.target" ];
|
||||
|
||||
serviceConfig = {
|
||||
Restart = "on-failure";
|
||||
Restart = "always";
|
||||
};
|
||||
script = ''
|
||||
AUTH_TOKEN=$(${pkgs.iusb-spoof}/bin/make-token ${bmcIp})
|
||||
exec ${pkgs.iusb-spoof}/bin/iusb-spoof -r ${bmcIp} 5123 $AUTH_TOKEN ${ipxe}/ipxe-efi.usb
|
||||
exec ${pkgs.iusb-spoof}/bin/iusb-spoof -r ${bmcIp} 5123 $AUTH_TOKEN ${notipxe}
|
||||
'';
|
||||
}) (lib.filterAttrs (_: node: node.config.bagel.baremetal.builders.enable && node.config.bagel.baremetal.builders.netboot) nodes);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
[
|
||||
(final: prev: {
|
||||
iusb-spoof = final.callPackage ./iusb-spoof.nix {};
|
||||
u-root = final.callPackage ./u-root {};
|
||||
})
|
||||
]
|
||||
|
|
|
|||
20
overlays/u-root/default.nix
Normal file
20
overlays/u-root/default.nix
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
{ buildGoModule, fetchFromGitHub }:
|
||||
|
||||
buildGoModule rec {
|
||||
pname = "u-root";
|
||||
version = "0.14.0";
|
||||
|
||||
src = fetchFromGitHub {
|
||||
owner = "u-root";
|
||||
repo = "u-root";
|
||||
rev = "v${version}";
|
||||
hash = "sha256-8zA3pHf45MdUcq/MA/mf0KCTxB1viHieU/oigYwIPgo=";
|
||||
};
|
||||
|
||||
patches = [
|
||||
./u-root-allow-https.patch
|
||||
];
|
||||
|
||||
vendorHash = null;
|
||||
doCheck = false;
|
||||
}
|
||||
12
overlays/u-root/u-root-allow-https.patch
Normal file
12
overlays/u-root/u-root-allow-https.patch
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
diff --git a/pkg/curl/schemes.go b/pkg/curl/schemes.go
|
||||
index 8bac3bc0..cd396cbc 100644
|
||||
--- a/pkg/curl/schemes.go
|
||||
+++ b/pkg/curl/schemes.go
|
||||
@@ -81,6 +81,7 @@ var (
|
||||
DefaultSchemes = Schemes{
|
||||
"tftp": DefaultTFTPClient,
|
||||
"http": DefaultHTTPClient,
|
||||
+ "https": DefaultHTTPClient,
|
||||
"file": &LocalFileClient{},
|
||||
}
|
||||
)
|
||||
|
|
@ -1,8 +1,10 @@
|
|||
{ pkgs, lib, config, extendModules, ... }:
|
||||
{ pkgs, lib, config, ... }:
|
||||
let
|
||||
cfg = config.bagel.baremetal.builders;
|
||||
in
|
||||
{
|
||||
imports = [ ./netboot.nix ];
|
||||
|
||||
options = {
|
||||
|
||||
bagel.baremetal.builders = {
|
||||
|
|
@ -182,39 +184,6 @@ in
|
|||
|
||||
environment.systemPackages = [ pkgs.ipmitool ];
|
||||
|
||||
system.build = lib.mkIf cfg.netboot {
|
||||
netbootVariant = extendModules {
|
||||
modules = [
|
||||
(
|
||||
{ modulesPath, ... }:
|
||||
|
||||
{
|
||||
imports = [ (modulesPath + "/installer/netboot/netboot.nix") ];
|
||||
}
|
||||
)
|
||||
];
|
||||
};
|
||||
netbootDir = let
|
||||
kernelTarget = pkgs.stdenv.hostPlatform.linux-kernel.target;
|
||||
build = config.system.build.netbootVariant.config.system.build;
|
||||
in
|
||||
pkgs.symlinkJoin {
|
||||
name = "netboot";
|
||||
paths = [
|
||||
build.netbootRamdisk
|
||||
build.kernel
|
||||
build.netbootIpxeScript
|
||||
];
|
||||
postBuild = ''
|
||||
mkdir -p $out/nix-support
|
||||
echo "file ${kernelTarget} $out/${kernelTarget}" >> $out/nix-support/hydra-build-products
|
||||
echo "file initrd $out/initrd" >> $out/nix-support/hydra-build-products
|
||||
echo "file ipxe $out/netboot.ipxe" >> $out/nix-support/hydra-build-products
|
||||
'';
|
||||
preferLocalBuild = true;
|
||||
};
|
||||
};
|
||||
|
||||
system.stateVersion = "24.05";
|
||||
};
|
||||
}
|
||||
|
|
|
|||
149
services/baremetal-builder/netboot.nix
Normal file
149
services/baremetal-builder/netboot.nix
Normal file
|
|
@ -0,0 +1,149 @@
|
|||
{ modulesPath, pkgs, lib, config, extendModules, ... }@node:
|
||||
let
|
||||
cfg = config.bagel.baremetal.builders;
|
||||
in
|
||||
{
|
||||
config = lib.mkIf (cfg.enable && cfg.netboot) {
|
||||
system.build = {
|
||||
|
||||
# Build a kernel and initramfs which will download the IPXE script from hydra using
|
||||
# u-root pxeboot tool and kexec into the final netbooted system.
|
||||
notipxe = import (modulesPath + "/..") {
|
||||
system = "x86_64-linux";
|
||||
configuration =
|
||||
{ pkgs, config, ... }:
|
||||
|
||||
{
|
||||
system.stateVersion = "24.11";
|
||||
boot.initrd.availableKernelModules = [ "ahci" "ehci_pci" "usb_storage" "usbhid" "sd_mod" "igb" "bonding" ];
|
||||
boot.kernelParams = [ "console=ttyS0,115200" "panic=1" "boot.panic_on_fail" ];
|
||||
#boot.initrd.systemd.emergencyAccess = true;
|
||||
networking.hostName = "${node.config.networking.hostName}-boot";
|
||||
nixpkgs.overlays = import ../../overlays;
|
||||
boot.loader.grub.enable = false;
|
||||
fileSystems."/".device = "bogus"; # this config will never be booted
|
||||
boot.initrd.systemd.enable = true;
|
||||
boot.initrd.systemd.network = {
|
||||
enable = true;
|
||||
networks = node.config.systemd.network.networks;
|
||||
netdevs = node.config.systemd.network.netdevs;
|
||||
};
|
||||
boot.initrd.systemd.storePaths = [
|
||||
"${pkgs.u-root}/bin/pxeboot"
|
||||
"${pkgs.iputils}/bin/ping"
|
||||
];
|
||||
boot.initrd.systemd.services.kexec = {
|
||||
serviceConfig.Restart = "on-failure";
|
||||
serviceConfig.Type = "oneshot";
|
||||
wantedBy = [ "initrd-root-fs.target" ];
|
||||
before = [ "sysroot.mount" ];
|
||||
script = ''
|
||||
ln -sf /dev/console /dev/tty
|
||||
until ${pkgs.iputils}/bin/ping -c 1 hydra.forkos.org; do sleep 1; done
|
||||
${pkgs.u-root}/bin/pxeboot -v -ipv4=false -file https://hydra.forkos.org/job/infra/main/${node.config.networking.hostName}/latest/download-by-type/file/ipxe
|
||||
'';
|
||||
};
|
||||
boot.initrd.systemd.contents."/etc/ssl/certs/ca-certificates.crt".source = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
|
||||
boot.initrd.services.resolved.enable = false;
|
||||
boot.initrd.systemd.contents."/etc/resolv.conf".text = ''
|
||||
nameserver 2001:4860:4860::6464
|
||||
'';
|
||||
boot.initrd.systemd.contents."/etc/systemd/journald.conf".text = ''
|
||||
[Journal]
|
||||
ForwardToConsole=yes
|
||||
MaxLevelConsole=debug
|
||||
'';
|
||||
|
||||
# Provide a bootable USB drive image
|
||||
system.build.usbImage = pkgs.callPackage ({ stdenv, runCommand, dosfstools, e2fsprogs, mtools, libfaketime, util-linux, nukeReferences }:
|
||||
runCommand "boot-img-${node.config.networking.hostName}" {
|
||||
nativeBuildInputs = [ dosfstools e2fsprogs libfaketime mtools util-linux ];
|
||||
} ''
|
||||
export img=$out
|
||||
truncate -s 40M $img
|
||||
|
||||
sfdisk $img <<EOF
|
||||
label: gpt
|
||||
label-id: F222513B-DED1-49FA-B591-20CE86A2FE7F
|
||||
|
||||
type=C12A7328-F81F-11D2-BA4B-00A0C93EC93B, bootable
|
||||
EOF
|
||||
|
||||
# Create a FAT32 /boot/firmware partition of suitable size into firmware_part.img
|
||||
eval $(partx $img -o START,SECTORS --nr 1 --pairs)
|
||||
truncate -s $((2081 * 512 + SECTORS * 512)) firmware_part.img
|
||||
|
||||
mkfs.vfat --invariant -i 2e24ec82 -n BOOT firmware_part.img
|
||||
|
||||
# Populate the files intended for /boot/firmware
|
||||
mkdir -p firmware/EFI/BOOT firmware/loader/entries
|
||||
cp ${pkgs.systemd}/lib/systemd/boot/efi/systemd-boot*.efi firmware/EFI/BOOT/BOOT${lib.toUpper stdenv.hostPlatform.efiArch}.EFI
|
||||
|
||||
cat > firmware/loader/loader.conf << EOF
|
||||
default foo
|
||||
EOF
|
||||
cat > firmware/loader/entries/default.conf << EOF
|
||||
title Default
|
||||
linux /EFI/${pkgs.stdenv.hostPlatform.linux-kernel.target}
|
||||
initrd /EFI/initrd
|
||||
options init=${config.system.build.toplevel}/init ${toString config.boot.kernelParams}
|
||||
EOF
|
||||
cp ${config.system.build.kernel}/${pkgs.stdenv.hostPlatform.linux-kernel.target} firmware/EFI/${pkgs.stdenv.hostPlatform.linux-kernel.target}
|
||||
cp ${config.system.build.initialRamdisk}/${config.system.boot.loader.initrdFile} firmware/EFI/initrd
|
||||
|
||||
find firmware -exec touch --date=2000-01-01 {} +
|
||||
# Copy the populated /boot/firmware into the SD image
|
||||
cd firmware
|
||||
# Force a fixed order in mcopy for better determinism, and avoid file globbing
|
||||
for d in $(find . -type d -mindepth 1 | sort); do
|
||||
faketime "2000-01-01 00:00:00" mmd -i ../firmware_part.img "::/$d"
|
||||
done
|
||||
for f in $(find . -type f | sort); do
|
||||
mcopy -pvm -i ../firmware_part.img "$f" "::/$f"
|
||||
done
|
||||
cd ..
|
||||
|
||||
# Verify the FAT partition before copying it.
|
||||
fsck.vfat -vn firmware_part.img
|
||||
dd conv=notrunc if=firmware_part.img of=$img seek=$START count=$SECTORS
|
||||
''
|
||||
) {};
|
||||
}
|
||||
;
|
||||
};
|
||||
|
||||
# This is the config which will actually be booted
|
||||
netbootVariant = extendModules {
|
||||
modules = [
|
||||
(
|
||||
{ modulesPath, ... }:
|
||||
|
||||
{
|
||||
imports = [ (modulesPath + "/installer/netboot/netboot.nix") ];
|
||||
}
|
||||
)
|
||||
];
|
||||
};
|
||||
# A derivation combining all the artifacts required for netbooting for the hydra job
|
||||
netbootDir = let
|
||||
kernelTarget = pkgs.stdenv.hostPlatform.linux-kernel.target;
|
||||
build = config.system.build.netbootVariant.config.system.build;
|
||||
in
|
||||
pkgs.symlinkJoin {
|
||||
name = "netboot";
|
||||
paths = [
|
||||
build.netbootRamdisk
|
||||
build.kernel
|
||||
build.netbootIpxeScript
|
||||
];
|
||||
postBuild = ''
|
||||
mkdir -p $out/nix-support
|
||||
echo "file ${kernelTarget} $out/${kernelTarget}" >> $out/nix-support/hydra-build-products
|
||||
echo "file initrd $out/initrd" >> $out/nix-support/hydra-build-products
|
||||
echo "file ipxe $out/netboot.ipxe" >> $out/nix-support/hydra-build-products
|
||||
'';
|
||||
preferLocalBuild = true;
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
Loading…
Reference in a new issue