the-distro/infra
9
0
Fork 5
Code Issues 38 Pull requests 6 Packages Projects 1 Releases Wiki Activity

builders: provision ssh hostkeys on boot

Browse source
This commit is contained in:
Yureka 2024-08-04 18:12:02 +02:00
parent 27d66d390e
commit bce44930b1
2 changed files with 58 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

50
hosts/wob-vpn-gw/netboot.nix
View file

@ -1,26 +1,30 @@
{ lib, pkgs, nodes, config, modulesPath, ... }:
{ config, lib, pkgs, nodes, modulesPath, ... }:
# The way the connection is established is specific to the wob01 site and the Intel S2600KPR blades.
# Proper netboot is not possible, because while the blades and the APU board (which is the netboot
# server here) are in the same L2 network, the uplink connection of each blade is an LACP LAG,
# meaning that the switch on the other side will only enable the port if it sees valid LACP packets.
# IPXE sends out these LACP packets while it is probing the ports, however the NICs of the blades
# do not have a flash which IPXE could be written to.
# We work around this by presenting a virtual floppy drive using the "IUSB" protocol of the BMC.
# This virtual floppy drive contains an per-blade customized IPXE script which will initialize the
# network connection including IP configuration and chainload the actual script off the netboot
# server.
# This virtual floppy drive contains an per-blade customized initramfs which will initialize the
# network connection including IP configuration and load the actual image off hydra.
let
netboot-server-ip = "2a01:584:11::2";
netbootNodes = lib.filterAttrs (_: node: node.config.bagel.baremetal.builders.enable && node.config.bagel.baremetal.builders.netboot) nodes;
in {
networking.firewall.allowedTCPPorts = [ 80 ];
assertions = [
{
assertion = !(lib.elem 443 config.networking.firewall.allowedTCPPorts);
message = ''
Port 443 is in networking.firewalls.allowedTCPPorts, but should be only manually
allowed for specific IPs and source ports in ${builtins.toJSON __curPos}
'';
}
];
systemd.services = lib.mapAttrs' (nodename: node: let
ip = "2a01:584:11::1:${toString node.config.bagel.baremetal.builders.num}";
bmcIp = "192.168.1.${toString (node.config.bagel.baremetal.builders.num * 4 + 2)}";
gw = "2a01:584:11::1";
dns = "2a01:580:6000::ff01";
notipxe = node.config.system.build.notipxe.config.system.build.usbImage;
in lib.nameValuePair "iusb-spoof-${nodename}" {
wantedBy = [ "multi-user.target" ];
@ -31,5 +35,29 @@ in {
AUTH_TOKEN=$(${pkgs.iusb-spoof}/bin/make-token ${bmcIp})
exec ${pkgs.iusb-spoof}/bin/iusb-spoof -r ${bmcIp} 5123 $AUTH_TOKEN ${notipxe}
'';
}) (lib.filterAttrs (_: node: node.config.bagel.baremetal.builders.enable && node.config.bagel.baremetal.builders.netboot) nodes);
}) netbootNodes;
# Since the builders are stateless, they can not store their ssh hostkeys
networking.firewall.allowedTCPPorts = [ 80 ]; # for ACME
networking.firewall.extraInputRules = ''
ip6 saddr 2a01:584:11::/64 tcp sport < 1024 tcp dport 443 accept;
'';
security.acme.acceptTerms = true;
security.acme.defaults.email = "infra@forkos.org";
services.nginx = {
enable = true;
virtualHosts."vpn-gw.wob01.infra.forkos.org" = {
enableACME = true;
forceSSL = true;
locations = lib.mapAttrs' (nodename: node: let
ip = "2a01:584:11::1:${toString node.config.bagel.baremetal.builders.num}";
in lib.nameValuePair "/${nodename}/" {
root = "/var/www";
extraConfig = ''
allow ${ip};
deny all;
'';
}) netbootNodes;
};
};
}

19
services/baremetal-builder/netboot.nix
View file

@ -4,6 +4,25 @@ let
in
{
config = lib.mkIf (cfg.enable && cfg.netboot) {
system.activationScripts.agenixInstall.deps = ["provisionSshHostKey"];
system.activationScripts.provisionSshHostKey = {
text = ''
echo provisioning ssh hostkey
if [ ! -f /etc/ssh/ssh_host_ed25519_key ]
then
mkdir -p /etc/ssh
(
umask 0077
curl --local-port 25-1024 https://vpn-gw.wob01.infra.forkos.org/${config.networking.hostName}/ssh_host_ed25519_key > /etc/ssh/ssh_host_ed25519_key
)
fi
'';
deps = [
"specialfs"
];
};
system.build = {
# Build a kernel and initramfs which will download the IPXE script from hydra using