Merge pull request 'hydra: wire up new builders' (#47) from hydra-wire-up into main

Reviewed-on: the-distro/infra#47

common/admins.nix

@@ -8,12 +8,5 @@ in {
     keys.users.maxine ++ 
     keys.users.jade ++
     keys.users.lukegb ++
-    keys.users.yuka ++
+    keys.users.yuka;
-    [
-      # more raito
-      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDcEkYM1r8QVNM/G5CxJInEdoBCWjEHHDdHlzDYNSUIdHHsn04QY+XI67AdMCm8w30GZnLUIj5RiJEWXREUApby0GrfxGGcy8otforygfgtmuUKAUEHdU2MMwrQI7RtTZ8oQ0USRGuqvmegxz3l5caVU7qGvBllJ4NUHXrkZSja2/51vq80RF4MKkDGiz7xUTixI2UcBwQBCA/kQedKV9G28EH+1XfvePqmMivZjl+7VyHsgUVj9eRGA1XWFw59UPZG8a7VkxO/Eb3K9NF297HUAcFMcbY6cPFi9AaBgu3VC4eetDnoN/+xT1owiHi7BReQhGAy/6cdf7C/my5ehZwD"
-      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE0xMwWedkKosax9+7D2OlnMxFL/eV4CvFZLsbLptpXr"
-      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiXXYkhRh+s7ixZ8rvG8ntIqd6FELQ9hh7HoaHQJRPU"
-      "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJFsZ7PMDt80tYXHyScQajNhqH4wuYg/o0OxfOHaZD4rXuT0VIKflKH1M9LslfHWIEH3XNeqhQOziH9r+Ny5JcM="
-    ];
 }

common/ssh-keys.nix

@@ -21,7 +21,13 @@
 
   users = {
     delroth = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII3tjB4KYDok3KlWxdBp/yEmqhhmybd+w0VO4xUwLKKV" ];
-    raito = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICaw9ihTG7ucB8P38XdalEWev8+q96e2yNm4B+/I9IJp" ];
+    raito = [
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICaw9ihTG7ucB8P38XdalEWev8+q96e2yNm4B+/I9IJp"
+      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDcEkYM1r8QVNM/G5CxJInEdoBCWjEHHDdHlzDYNSUIdHHsn04QY+XI67AdMCm8w30GZnLUIj5RiJEWXREUApby0GrfxGGcy8otforygfgtmuUKAUEHdU2MMwrQI7RtTZ8oQ0USRGuqvmegxz3l5caVU7qGvBllJ4NUHXrkZSja2/51vq80RF4MKkDGiz7xUTixI2UcBwQBCA/kQedKV9G28EH+1XfvePqmMivZjl+7VyHsgUVj9eRGA1XWFw59UPZG8a7VkxO/Eb3K9NF297HUAcFMcbY6cPFi9AaBgu3VC4eetDnoN/+xT1owiHi7BReQhGAy/6cdf7C/my5ehZwD"
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE0xMwWedkKosax9+7D2OlnMxFL/eV4CvFZLsbLptpXr"
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiXXYkhRh+s7ixZ8rvG8ntIqd6FELQ9hh7HoaHQJRPU"
+      "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJFsZ7PMDt80tYXHyScQajNhqH4wuYg/o0OxfOHaZD4rXuT0VIKflKH1M9LslfHWIEH3XNeqhQOziH9r+Ny5JcM="
+    ];
     k900 = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOi9vgVGs+S5kEsUqHPvyMMh1Q9gqL4TcbHoe5d73tun" ];
     maxine = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILpWQfhNFdrxMTP/1DwBVuk49f3df9iH7Tbdu8ltIKjr" ];
     jade = [

hosts/bagel-box/default.nix

@@ -40,6 +40,11 @@
     hydra.enable = true;
     hydra.dbi = "dbi:Pg:dbname=hydra;user=hydra";
 
+    hydra.builders = [
+      "builder-0"
+      "builder-1"
+    ];
+
     ofborg.enable = true;
   };
 

secrets.nix

@@ -1,7 +1,7 @@
 let
   keys = import common/ssh-keys.nix;
 
-  commonKeys = keys.users.delroth;
+  commonKeys = keys.users.delroth ++ keys.users.raito;
 
   secrets = with keys; {
     hydra-s3-credentials = [ machines.bagel-box ];

services/baremetal-builder/default.nix

@@ -14,10 +14,23 @@ in
   };
 
   config = lib.mkIf cfg.enable {
-
     boot.initrd.availableKernelModules = [ "ahci" "ehci_pci" "usb_storage" "usbhid" "sd_mod" ];
     boot.initrd.kernelModules = [ "dm-snapshot" ];
 
+    users.users.builder = {
+      isSystemUser = true;
+      group = "nogroup";
+      home = "/var/empty";
+      shell = "/bin/sh";
+      openssh.authorizedKeys.keys = [
+        # Do not hardcode Hydra's public key, selectively
+        # add the keys of the coordinators that require us.
+        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAvUT9YBig9LQPHgypIBHQuC32XqDKxlFZ2CfgDi0ZKx"
+      ];
+    };
+    nix.settings.trusted-users = [ "builder" ];
+
+
     nixpkgs.hostPlatform = "x86_64-linux";
     hardware.cpu.intel.updateMicrocode = true;
 
@@ -45,6 +58,7 @@ in
 
     networking.useNetworkd = true;
     networking.hostName = "builder-${toString cfg.num}";
+    networking.domain = "wob01.infra.forkos.org";
 
     systemd.network = {
       netdevs = {
@@ -75,6 +89,7 @@ in
     ];
     networking.defaultGateway6 = { interface = "uplink"; address = "2a01:584:11::1"; };
     deployment.targetHost = "2a01:584:11::1:${toString cfg.num}";
+    deployment.tags = [ "builders" ];
 
     networking.nameservers = lib.mkForce ["2001:4860:4860::6464"]; # todo: other dns64
 

services/hydra/default.nix

@@ -1,14 +1,28 @@
-{ config, lib, pkgs, ... }:
+{ nodes, config, lib, pkgs, ... }:
 
 let
   cfg = config.bagel.services.hydra;
+  ssh-keys = import ../../common/ssh-keys.nix;
 
   narCacheDir = "/var/cache/hydra/nar-cache";
   port = 3000;
 
   mkCacheSettings = settings: builtins.concatStringsSep "&" (
     lib.mapAttrsToList (k: v: "${k}=${v}") settings
-  );
+    );
+
+  mkBaremetalBuilder = { nrCores, publicHostKey, host, speedFactor ? 1, user ? "builder", supportedSystems ? [ "i686-linux" "x86_64-linux" ], supportedFeatures ? [ "big-parallel" "kvm" "nixos-test" ] }: 
+  "ssh://${user}@${host} ${lib.concatStringsSep "," supportedSystems} ${config.age.secrets.hydra-ssh-key-priv.path} ${toString nrCores} ${toString speedFactor} ${lib.concatStringsSep "," supportedFeatures} - ${publicHostKey}";
+
+  # TODO:
+  # - generalize to new architectures
+  # - generalize to new features
+  baremetalBuilders = lib.concatStringsSep "\n" 
+    (map (n: mkBaremetalBuilder {
+      nrCores = 40; # TODO: do not hardcode this, use the node's builder configuration.
+      publicHostKey = ssh-keys.machines.${n};
+      host = nodes.${n}.config.networking.fqdn;
+    }) cfg.builders);
 in {
   options.bagel.services.hydra = with lib; {
     enable = mkEnableOption "Hydra coordinator";
@@ -17,9 +31,19 @@ in {
       type = types.str;
       description = "DBI connection string for the Hydra postgres database";
     };
+
+    builders = mkOption {
+      type = types.listOf types.str;
+      description = "List of builders to configure for Hydra";
+      example = [ "builder-0" "builder-1" ];
+    };
   };
 
   config = lib.mkIf cfg.enable {
+    # TODO: we should assert or warn that the builders
+    # does indeed have our public SSH key and are *builders*
+    # as a simple evaluation preflight check.
+
     age.secrets.hydra-s3-credentials.file = ../../secrets/hydra-s3-credentials.age;
 
     age.secrets.hydra-ssh-key-priv.owner = "hydra-queue-runner";
@@ -54,7 +78,8 @@ in {
       buildMachinesFiles = [
         (pkgs.writeText "hydra-builders.conf" ''
           ssh://bagel-builder@epyc.infra.newtype.fr i686-linux,x86_64-linux ${config.age.secrets.hydra-ssh-key-priv.path} 8 1 big-parallel,kvm,nixos-test - c3NoLWVkMjU1MTkgQUFBQUMzTnphQzFsWkRJMU5URTVBQUFBSU9YVDlJbml0MU1oS3Q0cmpCQU5McTB0MGJQd3cvV1FaOTZ1QjRBRURybWwgcm9vdEBuaXhvcwo=
-        '')
+          ${baremetalBuilders}
+      '')
       ];
 
       extraConfig = ''

terraform/gandi.nix

@@ -1,6 +1,6 @@
 { lib, config, ... }:
 let
-  inherit (lib) mkEnableOption mkIf tf;
+  inherit (lib) mkEnableOption mkIf tf genList;
   cfg = config.bagel.gandi;
 in
 {
@@ -43,7 +43,7 @@ in
           };
         }) records);
 
-    in forkosRecords [
+    in forkosRecords ([
       # (record "@" 3600 "A" ["163.172.69.160"])
       (record "@" 3600 "AAAA" ["2001:bc8:38ee:100:1000::20"])
 
@@ -67,6 +67,9 @@ in
       (record "loki" 3600 "CNAME" ["meta01.infra"])
       (record "mimir" 3600 "CNAME" ["meta01.infra"])
       (record "matrix" 3600 "CNAME" ["meta01.infra"])
-    ];
+
+      (record "vpn-gw.wob01.infra" 3600 "AAAA" [ "2a01:584:11::2" ])
+      # TODO: do not hardcode, just reuse the Colmena hive module outputs to generate all the required details.
+    ] ++ map (index: record "builder-${toString index}.wob01.infra" 3600 "AAAA" [ "2a01:584:11::1:${toString index}" ]) (genList lib.id 12));
   };
 }