From a30c1f7d78c290bafd5dc352313cfbb87c99b190 Mon Sep 17 00:00:00 2001 From: Raito Bezarius Date: Wed, 10 Jul 2024 00:43:03 +0200 Subject: [PATCH 1/5] hydra: wire up new builders Signed-off-by: Raito Bezarius --- hosts/bagel-box/default.nix | 5 +++++ services/baremetal-builder/default.nix | 1 + services/hydra/default.nix | 31 +++++++++++++++++++++++--- 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/hosts/bagel-box/default.nix b/hosts/bagel-box/default.nix index d864346..1560c2d 100644 --- a/hosts/bagel-box/default.nix +++ b/hosts/bagel-box/default.nix @@ -40,6 +40,11 @@ hydra.enable = true; hydra.dbi = "dbi:Pg:dbname=hydra;user=hydra"; + hydra.builders = [ + "builder-0" + "builder-1" + ]; + ofborg.enable = true; }; diff --git a/services/baremetal-builder/default.nix b/services/baremetal-builder/default.nix index 95b9f03..b871482 100644 --- a/services/baremetal-builder/default.nix +++ b/services/baremetal-builder/default.nix @@ -45,6 +45,7 @@ in networking.useNetworkd = true; networking.hostName = "builder-${toString cfg.num}"; + networking.domain = "wob01.infra.forkos.org"; systemd.network = { netdevs = { diff --git a/services/hydra/default.nix b/services/hydra/default.nix index 40a2b30..6cbd367 100644 --- a/services/hydra/default.nix +++ b/services/hydra/default.nix @@ -1,14 +1,28 @@ -{ config, lib, pkgs, ... }: +{ nodes, config, lib, pkgs, ... }: let cfg = config.bagel.services.hydra; + ssh-keys = import ../../common/ssh-keys.nix; narCacheDir = "/var/cache/hydra/nar-cache"; port = 3000; mkCacheSettings = settings: builtins.concatStringsSep "&" ( lib.mapAttrsToList (k: v: "${k}=${v}") settings - ); + ); + + mkBaremetalBuilder = { nrCores, publicHostKey, host, speedFactor ? 1, user ? "builder", supportedSystems ? [ "i686-linux" "x86_64-linux" ], supportedFeatures ? [ "big-parallel" "kvm" "nixos-test" ] }: + "ssh://${user}@${host} ${lib.concatStringsSep "," supportedSystems} ${config.age.secrets.hydra-ssh-key-priv.path} ${toString nrCores} ${toString speedFactor} ${lib.concatStringsSep "," supportedFeatures} - ${publicHostKey}"; + + # TODO: + # - generalize to new architectures + # - generalize to new features + baremetalBuilders = lib.concatStringsSep "\n" + (map (n: mkBaremetalBuilder { + nrCores = 40; # TODO: do not hardcode this, use the node's builder configuration. + publicHostKey = ssh-keys.machines.${n}; + host = nodes.${n}.config.networking.fqdn; + }) cfg.builders); in { options.bagel.services.hydra = with lib; { enable = mkEnableOption "Hydra coordinator"; @@ -17,9 +31,19 @@ in { type = types.str; description = "DBI connection string for the Hydra postgres database"; }; + + builders = mkOption { + type = types.listOf types.str; + description = "List of builders to configure for Hydra"; + example = [ "builder-0" "builder-1" ]; + }; }; config = lib.mkIf cfg.enable { + # TODO: we should assert or warn that the builders + # does indeed have our public SSH key and are *builders* + # as a simple evaluation preflight check. + age.secrets.hydra-s3-credentials.file = ../../secrets/hydra-s3-credentials.age; age.secrets.hydra-ssh-key-priv.owner = "hydra-queue-runner"; @@ -54,7 +78,8 @@ in { buildMachinesFiles = [ (pkgs.writeText "hydra-builders.conf" '' ssh://bagel-builder@epyc.infra.newtype.fr i686-linux,x86_64-linux ${config.age.secrets.hydra-ssh-key-priv.path} 8 1 big-parallel,kvm,nixos-test - c3NoLWVkMjU1MTkgQUFBQUMzTnphQzFsWkRJMU5URTVBQUFBSU9YVDlJbml0MU1oS3Q0cmpCQU5McTB0MGJQd3cvV1FaOTZ1QjRBRURybWwgcm9vdEBuaXhvcwo= - '') + ${baremetalBuilders} + '') ]; extraConfig = '' From 645ad7d06228a5d7163ae74e7498f42f19d04712 Mon Sep 17 00:00:00 2001 From: Raito Bezarius Date: Wed, 10 Jul 2024 00:55:25 +0200 Subject: [PATCH 2/5] builders: add builder user currently hardcoded to hydra's coordinator public key Signed-off-by: Raito Bezarius --- services/baremetal-builder/default.nix | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/services/baremetal-builder/default.nix b/services/baremetal-builder/default.nix index b871482..5e4c51b 100644 --- a/services/baremetal-builder/default.nix +++ b/services/baremetal-builder/default.nix @@ -14,10 +14,23 @@ in }; config = lib.mkIf cfg.enable { - boot.initrd.availableKernelModules = [ "ahci" "ehci_pci" "usb_storage" "usbhid" "sd_mod" ]; boot.initrd.kernelModules = [ "dm-snapshot" ]; + users.users.builder = { + isSystemUser = true; + group = "nogroup"; + home = "/var/empty"; + shell = "/bin/sh"; + openssh.authorizedKeys.keys = [ + # Do not hardcode Hydra's public key, selectively + # add the keys of the coordinators that require us. + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAvUT9YBig9LQPHgypIBHQuC32XqDKxlFZ2CfgDi0ZKx" + ]; + }; + nix.settings.trusted-users = [ "builder" ]; + + nixpkgs.hostPlatform = "x86_64-linux"; hardware.cpu.intel.updateMicrocode = true; From 90e54d7292bde52e9666637843e6e816ce46722a Mon Sep 17 00:00:00 2001 From: Raito Bezarius Date: Wed, 10 Jul 2024 00:55:42 +0200 Subject: [PATCH 3/5] terraform: add DNS records for VPN-GW & builders Signed-off-by: Raito Bezarius --- terraform/gandi.nix | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/terraform/gandi.nix b/terraform/gandi.nix index 52c3640..82f7ffd 100644 --- a/terraform/gandi.nix +++ b/terraform/gandi.nix @@ -1,6 +1,6 @@ { lib, config, ... }: let - inherit (lib) mkEnableOption mkIf tf; + inherit (lib) mkEnableOption mkIf tf genList; cfg = config.bagel.gandi; in { @@ -43,7 +43,7 @@ in }; }) records); - in forkosRecords [ + in forkosRecords ([ # (record "@" 3600 "A" ["163.172.69.160"]) (record "@" 3600 "AAAA" ["2001:bc8:38ee:100:1000::20"]) @@ -67,6 +67,9 @@ in (record "loki" 3600 "CNAME" ["meta01.infra"]) (record "mimir" 3600 "CNAME" ["meta01.infra"]) (record "matrix" 3600 "CNAME" ["meta01.infra"]) - ]; + + (record "vpn-gw.wob01.infra" 3600 "AAAA" [ "2a01:584:11::2" ]) + # TODO: do not hardcode, just reuse the Colmena hive module outputs to generate all the required details. + ] ++ map (index: record "builder-${toString index}.wob01.infra" 3600 "AAAA" [ "2a01:584:11::1:${toString index}" ]) (genList lib.id 12)); }; } From f9f955214fd6a7042ae99331d0a29852a041e7b4 Mon Sep 17 00:00:00 2001 From: Raito Bezarius Date: Wed, 10 Jul 2024 00:59:17 +0200 Subject: [PATCH 4/5] ssh-keys: add raito to secrets set Signed-off-by: Raito Bezarius --- common/admins.nix | 9 +-------- common/ssh-keys.nix | 8 +++++++- secrets.nix | 2 +- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/common/admins.nix b/common/admins.nix index e22bba8..17ca2f6 100644 --- a/common/admins.nix +++ b/common/admins.nix @@ -8,12 +8,5 @@ in { keys.users.maxine ++ keys.users.jade ++ keys.users.lukegb ++ - keys.users.yuka ++ - [ - # more raito - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDcEkYM1r8QVNM/G5CxJInEdoBCWjEHHDdHlzDYNSUIdHHsn04QY+XI67AdMCm8w30GZnLUIj5RiJEWXREUApby0GrfxGGcy8otforygfgtmuUKAUEHdU2MMwrQI7RtTZ8oQ0USRGuqvmegxz3l5caVU7qGvBllJ4NUHXrkZSja2/51vq80RF4MKkDGiz7xUTixI2UcBwQBCA/kQedKV9G28EH+1XfvePqmMivZjl+7VyHsgUVj9eRGA1XWFw59UPZG8a7VkxO/Eb3K9NF297HUAcFMcbY6cPFi9AaBgu3VC4eetDnoN/+xT1owiHi7BReQhGAy/6cdf7C/my5ehZwD" - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE0xMwWedkKosax9+7D2OlnMxFL/eV4CvFZLsbLptpXr" - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiXXYkhRh+s7ixZ8rvG8ntIqd6FELQ9hh7HoaHQJRPU" - "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJFsZ7PMDt80tYXHyScQajNhqH4wuYg/o0OxfOHaZD4rXuT0VIKflKH1M9LslfHWIEH3XNeqhQOziH9r+Ny5JcM=" - ]; + keys.users.yuka; } diff --git a/common/ssh-keys.nix b/common/ssh-keys.nix index 54fb408..8a61a06 100644 --- a/common/ssh-keys.nix +++ b/common/ssh-keys.nix @@ -21,7 +21,13 @@ users = { delroth = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII3tjB4KYDok3KlWxdBp/yEmqhhmybd+w0VO4xUwLKKV" ]; - raito = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICaw9ihTG7ucB8P38XdalEWev8+q96e2yNm4B+/I9IJp" ]; + raito = [ + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICaw9ihTG7ucB8P38XdalEWev8+q96e2yNm4B+/I9IJp" + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDcEkYM1r8QVNM/G5CxJInEdoBCWjEHHDdHlzDYNSUIdHHsn04QY+XI67AdMCm8w30GZnLUIj5RiJEWXREUApby0GrfxGGcy8otforygfgtmuUKAUEHdU2MMwrQI7RtTZ8oQ0USRGuqvmegxz3l5caVU7qGvBllJ4NUHXrkZSja2/51vq80RF4MKkDGiz7xUTixI2UcBwQBCA/kQedKV9G28EH+1XfvePqmMivZjl+7VyHsgUVj9eRGA1XWFw59UPZG8a7VkxO/Eb3K9NF297HUAcFMcbY6cPFi9AaBgu3VC4eetDnoN/+xT1owiHi7BReQhGAy/6cdf7C/my5ehZwD" + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE0xMwWedkKosax9+7D2OlnMxFL/eV4CvFZLsbLptpXr" + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiXXYkhRh+s7ixZ8rvG8ntIqd6FELQ9hh7HoaHQJRPU" + "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJFsZ7PMDt80tYXHyScQajNhqH4wuYg/o0OxfOHaZD4rXuT0VIKflKH1M9LslfHWIEH3XNeqhQOziH9r+Ny5JcM=" + ]; k900 = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOi9vgVGs+S5kEsUqHPvyMMh1Q9gqL4TcbHoe5d73tun" ]; maxine = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILpWQfhNFdrxMTP/1DwBVuk49f3df9iH7Tbdu8ltIKjr" ]; jade = [ diff --git a/secrets.nix b/secrets.nix index 368ff9d..7bfd8a7 100644 --- a/secrets.nix +++ b/secrets.nix @@ -1,7 +1,7 @@ let keys = import common/ssh-keys.nix; - commonKeys = keys.users.delroth; + commonKeys = keys.users.delroth ++ keys.users.raito; secrets = with keys; { hydra-s3-credentials = [ machines.bagel-box ]; From 2308870aa580392d246b73186ac2ceb274571f8a Mon Sep 17 00:00:00 2001 From: Raito Bezarius Date: Wed, 10 Jul 2024 00:59:31 +0200 Subject: [PATCH 5/5] builders: add a nice tag to deploy all of them at once Signed-off-by: Raito Bezarius --- services/baremetal-builder/default.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/services/baremetal-builder/default.nix b/services/baremetal-builder/default.nix index 5e4c51b..522513e 100644 --- a/services/baremetal-builder/default.nix +++ b/services/baremetal-builder/default.nix @@ -89,6 +89,7 @@ in ]; networking.defaultGateway6 = { interface = "uplink"; address = "2a01:584:11::1"; }; deployment.targetHost = "2a01:584:11::1:${toString cfg.num}"; + deployment.tags = [ "builders" ]; networking.nameservers = lib.mkForce ["2001:4860:4860::6464"]; # todo: other dns64