add baremetal builders

common/ssh-keys.nix

@@ -4,6 +4,19 @@
     meta01 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM5t9gYorOWgpCFDJgb24pyCKIabGpeI2H/UfdvXODcT";
     gerrit01 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIA+eSZu+u9sCynrMlsmFzQHLIELQAuVg0Cs1pBvwb4+A";
     fodwatch = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFRyTNfvKl5FcSyzGzw+h+bNFNOxdhvI67WdUZ2iIJ1L";
+    builder-0 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBHSNcDGctvlG6BHcJuYIzW9WsBJsts2vpwSketsbXoL";
+    builder-1 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIQOGUjERK7Mx8UPM/rbOdMqVyn1sbWqYOG6CbOzH2wm";
+    builder-2 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMKzXIqCoYElEKIYgjbSpqEcDeOvV+Wo3Agq3jba83cB";
+    builder-3 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGq0A5233XGt34T097KaEKBUqFvaa7a6nYZRsSO0166l";
+    builder-4 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIB9dVo2xZhgIMDgB1rUj5ApmppL39BtYu/+OFHeduvXr";
+    builder-5 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE7vZTBxrVHmHpv7slQ8A8XwjjbfN+ZJA0V5C3k0wNBD";
+    builder-6 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOt1qR/2BRtc6PABuSBulowwJVO6wBNDyEFzh0qsTeOF";
+    builder-7 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFinAAw1v8TJB8/wcmTVBbHHc4LCYh6z4TO6ViwUPkoh";
+    builder-8 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKGSWHNeqT0kF/e4yVy2ieW98X5QMyCYIYZh9WTmQDs1";
+    builder-9 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOhws9zGgocVY36dMtOL+CXadpvRMffxoWMkfEcTBJm7";
+    builder-10 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE7sgIuTSqZiZhp8TvObSbIEhcHHsL5hcmYA22uzwxth";
+    builder-11 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEAqFo1qJY7MSUkfB+zxXB8Lpt/Iqz/RR5A+zwhpRWhr";
+
   };
 
   users = {

flake.nix

@@ -123,6 +123,29 @@
           ./hosts/fodwatch.forkos.org
         ];
       };
-    };
+
+      wob-vpn-gw = {
+        imports = [
+          inputs.agenix.nixosModules.default
+          inputs.hydra.nixosModules.hydra
+
+          ./services
+          ./common
+          ./hosts/wob-vpn-gw.forkos.org
+        ];
+      };
+
+    } // (lib.listToAttrs (lib.genList (i: lib.nameValuePair "builder-${toString i}" {
+
+      imports = [
+        inputs.agenix.nixosModules.default
+        inputs.hydra.nixosModules.hydra
+        ./services
+        ./common
+        {
+          bagel.baremetal.builders = { enable = true; num = i; };
+        }
+      ];
+    }) 12));
   };
 }

hosts/wob-vpn-gw.forkos.org/default.nix

@@ -0,0 +1,112 @@
+{ lib, ... }:
+
+{
+  ###### Hardware ######
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "ehci_pci" "sd_mod" "sdhci_pci" ];
+  boot.kernelModules = [ "kvm-amd" ];
+
+  boot.loader.grub.device = "/dev/sda";
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/58688a5c-e3ce-4868-804b-4e34d1370f36";
+      fsType = "f2fs";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/38caa628-3b6d-4fb4-8767-beee09a196a6";
+      fsType = "ext2";
+    };
+
+  nixpkgs.hostPlatform = "x86_64-linux";
+
+  hardware.cpu.amd.updateMicrocode = true;
+
+  # Enable serial output
+  boot.loader.grub.extraConfig = ''
+    serial --speed=115200 --unit=0 --word=8 --parity=no --stop=1
+    terminal_input serial
+    terminal_output serial
+  '';
+  boot.kernelParams = [
+    "console=ttyS0,115200"
+    "console=tty1"
+  ];
+
+  ###### Config #######
+
+  boot.initrd.systemd.enable = true;
+
+  networking.useNetworkd = true;
+
+  systemd.network = {
+    netdevs = {
+      "40-uplink" = {
+        netdevConfig = {
+          Kind = "bond";
+          Name = "uplink";
+        };
+        bondConfig = {
+          Mode = "802.3ad";
+          TransmitHashPolicy = "layer3+4";
+        };
+      };
+      "40-oob" = {
+        netdevConfig = {
+          Kind = "bond";
+          Name = "oob";
+        };
+        bondConfig = {
+          Mode = "802.3ad";
+          TransmitHashPolicy = "layer3+4";
+        };
+      };
+    };
+    networks = {
+      "40-enp1s0" = {
+        name = "enp1s0";
+        bond = [ "uplink" ];
+      };
+      "40-enp2s0" = {
+        name = "enp2s0";
+        bond = [ "uplink" ];
+      };
+      "40-enp3s0" = {
+        name = "enp3s0";
+        bond = [ "oob" ];
+      };
+      "40-enp4s0" = {
+        name = "enp4s0";
+        bond = [ "oob" ];
+      };
+    } // lib.listToAttrs (map (x: lib.nameValuePair "40-bmc${toString x}" {
+      name = "bmc${toString x}";
+      address = [ "192.168.1.${toString (x*4 + 1)}/30" ];
+      #address = [ "192.168.${toString x}.1/24" ];
+      networkConfig.DHCPServer = true;
+    }) (lib.genList lib.id 12));
+  };
+
+  networking.vlans = lib.listToAttrs (map (x: lib.nameValuePair "bmc${toString x}" {
+    interface = "oob";
+    id = 101 + x;
+  }) (lib.genList lib.id 12));
+
+  networking.interfaces = {
+    uplink = {
+      ipv6.addresses = [
+        {
+          address = "2a01:584:11::2";
+          prefixLength = 64;
+        }
+      ];
+    };
+  };
+
+  networking.defaultGateway6 = { interface = "uplink"; address = "2a01:584:11::1"; };
+
+  deployment.targetHost = "2a01:584:11::2";
+
+  bagel.sysadmin.enable = true;
+
+  system.stateVersion = "24.05";
+}

services/baremetal-builder/default.nix

@@ -0,0 +1,85 @@
+{ lib, config, ... }:
+let
+  cfg = config.bagel.baremetal.builders;
+in
+{
+  options = {
+
+    bagel.baremetal.builders = {
+      enable = lib.mkEnableOption "baremetal bagel oven";
+      num = lib.mkOption {
+        type = lib.types.int;
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+
+    boot.initrd.availableKernelModules = [ "ahci" "ehci_pci" "usb_storage" "usbhid" "sd_mod" ];
+    boot.initrd.kernelModules = [ "dm-snapshot" ];
+
+    nixpkgs.hostPlatform = "x86_64-linux";
+    hardware.cpu.intel.updateMicrocode = true;
+
+    boot.loader.systemd-boot.enable = true;
+    boot.loader.efi.canTouchEfiVariables = true;
+    boot.initrd.systemd.enable = true;
+
+    boot.initrd.services.lvm.enable = true;
+
+    fileSystems."/" = {
+      device = "/dev/disk/by-label/root";
+      fsType = "xfs";
+    };
+
+    fileSystems."/boot" = {
+      device = "/dev/disk/by-label/BOOT";
+      fsType = "vfat";
+      options = [ "fmask=0022" "dmask=0022" ];
+    };
+
+    boot.kernelParams = [
+      "console=ttyS0,115200"
+      "console=tty1"
+    ];
+
+    networking.useNetworkd = true;
+    networking.hostName = "builder-${toString cfg.num}";
+
+    systemd.network = {
+      netdevs = {
+        "40-uplink" = {
+          netdevConfig = {
+            Kind = "bond";
+            Name = "uplink";
+          };
+          bondConfig = {
+            Mode = "802.3ad";
+            TransmitHashPolicy = "layer3+4";
+          };
+        };
+      };
+      networks = {
+        "40-eno1" = {
+          name = "eno1";
+          bond = [ "uplink" ];
+        };
+        "40-eno2" = {
+          name = "eno2";
+          bond = [ "uplink" ];
+        };
+      };
+    };
+    networking.interfaces.uplink.ipv6.addresses = [
+      { address = "2a01:584:11::1:${toString cfg.num}"; prefixLength = 64; }
+    ];
+    networking.defaultGateway6 = { interface = "uplink"; address = "2a01:584:11::1"; };
+    deployment.targetHost = "2a01:584:11::1:${toString cfg.num}";
+
+    networking.nameservers = lib.mkForce ["2001:4860:4860::6464"]; # todo: other dns64
+
+    bagel.sysadmin.enable = true;
+
+    system.stateVersion = "24.05";
+  };
+}

services/default.nix

@@ -6,5 +6,6 @@
     ./netbox
     ./ofborg
     ./postgres
+    ./baremetal-builder
   ];
 }