add baremetal builders

common/admins.nix

@@ -8,6 +8,7 @@ in {
     keys.users.maxine ++ 
     keys.users.jade ++
     keys.users.lukegb ++
+    keys.users.yuka ++
     [
       # more raito
       "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDcEkYM1r8QVNM/G5CxJInEdoBCWjEHHDdHlzDYNSUIdHHsn04QY+XI67AdMCm8w30GZnLUIj5RiJEWXREUApby0GrfxGGcy8otforygfgtmuUKAUEHdU2MMwrQI7RtTZ8oQ0USRGuqvmegxz3l5caVU7qGvBllJ4NUHXrkZSja2/51vq80RF4MKkDGiz7xUTixI2UcBwQBCA/kQedKV9G28EH+1XfvePqmMivZjl+7VyHsgUVj9eRGA1XWFw59UPZG8a7VkxO/Eb3K9NF297HUAcFMcbY6cPFi9AaBgu3VC4eetDnoN/+xT1owiHi7BReQhGAy/6cdf7C/my5ehZwD"

common/ssh-keys.nix

@@ -17,5 +17,6 @@
       "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBO4idMfdJxDJuBNOid60d4I+qxj09RHt+YkCYV2eXt6tGrEXg+S8hTQusy/SqooiXUH9pt4tea2RuBPN9+UwrH0= type-a yubikey slot 9a"
     ];
     lukegb = [ ''cert-authority,principals="lukegb" ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEqNOwlR7Qa8cbGpDfSCOweDPbAGQOZIcoRgh6s/J8DR'' ];
+    yuka = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKath4/fDnlv/4fzxkPrQN1ttmoPRNu/m9bEtdPJBDfY cardno:16_933_242" ];
   };
 }

flake.nix

@@ -20,7 +20,7 @@
     lix.follows = "hydra/nix";
   };
 
-  outputs = { self, nixpkgs, terranix, ... } @ inputs:
+  outputs = { self, nixpkgs, terranix, colmena, ... } @ inputs:
   let
     system = "x86_64-linux";
     pkgs = import nixpkgs {
@@ -67,6 +67,8 @@
       ];
     };
 
+    nixosConfigurations = (colmena.lib.makeHive self.outputs.colmena).nodes;
+
     colmena = {
       meta.nixpkgs = import nixpkgs {
         localSystem = system;
@@ -121,6 +123,29 @@
           ./hosts/fodwatch.forkos.org
         ];
       };
-    };
+
+      wob-vpn-gw = {
+        imports = [
+          inputs.agenix.nixosModules.default
+          inputs.hydra.nixosModules.hydra
+
+          ./services
+          ./common
+          ./hosts/wob-vpn-gw.forkos.org
+        ];
+      };
+
+    } // (lib.listToAttrs (lib.genList (i: lib.nameValuePair "builder-${toString i}" {
+
+      imports = [
+        inputs.agenix.nixosModules.default
+        inputs.hydra.nixosModules.hydra
+        ./services
+        ./common
+        {
+          bagel.baremetal.builders = { enable = true; num = i; };
+        }
+      ];
+    }) 12));
   };
 }

hosts/wob-vpn-gw.forkos.org/default.nix

@@ -0,0 +1,112 @@
+{ lib, ... }:
+
+{
+  ###### Hardware ######
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "ehci_pci" "sd_mod" "sdhci_pci" ];
+  boot.kernelModules = [ "kvm-amd" ];
+
+  boot.loader.grub.device = "/dev/sda";
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/58688a5c-e3ce-4868-804b-4e34d1370f36";
+      fsType = "f2fs";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/38caa628-3b6d-4fb4-8767-beee09a196a6";
+      fsType = "ext2";
+    };
+
+  nixpkgs.hostPlatform = "x86_64-linux";
+
+  hardware.cpu.amd.updateMicrocode = true;
+
+  # Enable serial output
+  boot.loader.grub.extraConfig = ''
+    serial --speed=115200 --unit=0 --word=8 --parity=no --stop=1
+    terminal_input serial
+    terminal_output serial
+  '';
+  boot.kernelParams = [
+    "console=ttyS0,115200"
+    "console=tty1"
+  ];
+
+  ###### Config #######
+
+  boot.initrd.systemd.enable = true;
+
+  networking.useNetworkd = true;
+
+  systemd.network = {
+    netdevs = {
+      "40-uplink" = {
+        netdevConfig = {
+          Kind = "bond";
+          Name = "uplink";
+        };
+        bondConfig = {
+          Mode = "802.3ad";
+          TransmitHashPolicy = "layer3+4";
+        };
+      };
+      "40-oob" = {
+        netdevConfig = {
+          Kind = "bond";
+          Name = "oob";
+        };
+        bondConfig = {
+          Mode = "802.3ad";
+          TransmitHashPolicy = "layer3+4";
+        };
+      };
+    };
+    networks = {
+      "40-enp1s0" = {
+        name = "enp1s0";
+        bond = [ "uplink" ];
+      };
+      "40-enp2s0" = {
+        name = "enp2s0";
+        bond = [ "uplink" ];
+      };
+      "40-enp3s0" = {
+        name = "enp3s0";
+        bond = [ "oob" ];
+      };
+      "40-enp4s0" = {
+        name = "enp4s0";
+        bond = [ "oob" ];
+      };
+    } // lib.listToAttrs (map (x: lib.nameValuePair "40-bmc${toString x}" {
+      name = "bmc${toString x}";
+      address = [ "192.168.1.${toString (x*4 + 1)}/30" ];
+      #address = [ "192.168.${toString x}.1/24" ];
+      networkConfig.DHCPServer = true;
+    }) (lib.genList lib.id 12));
+  };
+
+  networking.vlans = lib.listToAttrs (map (x: lib.nameValuePair "bmc${toString x}" {
+    interface = "oob";
+    id = 101 + x;
+  }) (lib.genList lib.id 12));
+
+  networking.interfaces = {
+    uplink = {
+      ipv6.addresses = [
+        {
+          address = "2a01:584:11::2";
+          prefixLength = 64;
+        }
+      ];
+    };
+  };
+
+  networking.defaultGateway6 = { interface = "uplink"; address = "2a01:584:11::1"; };
+
+  deployment.targetHost = "2a01:584:11::2";
+
+  bagel.sysadmin.enable = true;
+
+  system.stateVersion = "24.05";
+}

services/baremetal-builder/default.nix

@@ -0,0 +1,85 @@
+{ lib, config, ... }:
+let
+  cfg = config.bagel.baremetal.builders;
+in
+{
+  options = {
+
+    bagel.baremetal.builders = {
+      enable = lib.mkEnableOption "baremetal bagel oven";
+      num = lib.mkOption {
+        type = lib.types.int;
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+
+    boot.initrd.availableKernelModules = [ "ahci" "ehci_pci" "usb_storage" "usbhid" "sd_mod" ];
+    boot.initrd.kernelModules = [ "dm-snapshot" ];
+
+    nixpkgs.hostPlatform = "x86_64-linux";
+    hardware.cpu.intel.updateMicrocode = true;
+
+    boot.loader.systemd-boot.enable = true;
+    boot.loader.efi.canTouchEfiVariables = true;
+    boot.initrd.systemd.enable = true;
+
+    boot.initrd.services.lvm.enable = true;
+
+    fileSystems."/" = {
+      device = "/dev/disk/by-label/root";
+      fsType = "xfs";
+    };
+
+    fileSystems."/boot" = {
+      device = "/dev/disk/by-label/BOOT";
+      fsType = "vfat";
+      options = [ "fmask=0022" "dmask=0022" ];
+    };
+
+    boot.kernelParams = [
+      "console=ttyS0,115200"
+      "console=tty1"
+    ];
+
+    networking.useNetworkd = true;
+    networking.hostName = "builder-${toString cfg.num}";
+
+    systemd.network = {
+      netdevs = {
+        "40-uplink" = {
+          netdevConfig = {
+            Kind = "bond";
+            Name = "uplink";
+          };
+          bondConfig = {
+            Mode = "802.3ad";
+            TransmitHashPolicy = "layer3+4";
+          };
+        };
+      };
+      networks = {
+        "40-eno1" = {
+          name = "eno1";
+          bond = [ "uplink" ];
+        };
+        "40-eno2" = {
+          name = "eno2";
+          bond = [ "uplink" ];
+        };
+      };
+    };
+    networking.interfaces.uplink.ipv6.addresses = [
+      { address = "2a01:584:11::1:${toString cfg.num}"; prefixLength = 64; }
+    ];
+    networking.defaultGateway6 = { interface = "uplink"; address = "2a01:584:11::1"; };
+    deployment.targetHost = "2a01:584:11::1:${toString cfg.num}";
+
+    networking.nameservers = lib.mkForce ["2001:4860:4860::6464"]; # todo: other dns64
+
+    bagel.sysadmin.enable = true;
+
+    system.stateVersion = "24.05";
+  };
+}

services/default.nix

@@ -6,5 +6,6 @@
     ./netbox
     ./ofborg
     ./postgres
+    ./baremetal-builder
   ];
 }