the-distro/infra
9
0
Fork 5
Code Issues 39 Pull requests 8 Packages Projects 1 Releases Wiki Activity

hydra: wire up new builders #47

Merged
raito merged 5 commits from hydra-wire-up into main 2024-07-09 23:00:31 +00:00
Conversation 0 Commits 5 Files changed 7 +64 -17
7 changed files with 64 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
common/admins.nix
View file

@ -8,12 +8,5 @@ in {
keys.users.maxine ++
keys.users.jade ++
keys.users.lukegb ++
keys.users.yuka ++
[
# more raito
"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDcEkYM1r8QVNM/G5CxJInEdoBCWjEHHDdHlzDYNSUIdHHsn04QY+XI67AdMCm8w30GZnLUIj5RiJEWXREUApby0GrfxGGcy8otforygfgtmuUKAUEHdU2MMwrQI7RtTZ8oQ0USRGuqvmegxz3l5caVU7qGvBllJ4NUHXrkZSja2/51vq80RF4MKkDGiz7xUTixI2UcBwQBCA/kQedKV9G28EH+1XfvePqmMivZjl+7VyHsgUVj9eRGA1XWFw59UPZG8a7VkxO/Eb3K9NF297HUAcFMcbY6cPFi9AaBgu3VC4eetDnoN/+xT1owiHi7BReQhGAy/6cdf7C/my5ehZwD"
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE0xMwWedkKosax9+7D2OlnMxFL/eV4CvFZLsbLptpXr"
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiXXYkhRh+s7ixZ8rvG8ntIqd6FELQ9hh7HoaHQJRPU"
"ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJFsZ7PMDt80tYXHyScQajNhqH4wuYg/o0OxfOHaZD4rXuT0VIKflKH1M9LslfHWIEH3XNeqhQOziH9r+Ny5JcM="
];
keys.users.yuka;
}

8
common/ssh-keys.nix
View file

@ -21,7 +21,13 @@
users = {
delroth = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII3tjB4KYDok3KlWxdBp/yEmqhhmybd+w0VO4xUwLKKV" ];
raito = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICaw9ihTG7ucB8P38XdalEWev8+q96e2yNm4B+/I9IJp" ];
raito = [
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICaw9ihTG7ucB8P38XdalEWev8+q96e2yNm4B+/I9IJp"
"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDcEkYM1r8QVNM/G5CxJInEdoBCWjEHHDdHlzDYNSUIdHHsn04QY+XI67AdMCm8w30GZnLUIj5RiJEWXREUApby0GrfxGGcy8otforygfgtmuUKAUEHdU2MMwrQI7RtTZ8oQ0USRGuqvmegxz3l5caVU7qGvBllJ4NUHXrkZSja2/51vq80RF4MKkDGiz7xUTixI2UcBwQBCA/kQedKV9G28EH+1XfvePqmMivZjl+7VyHsgUVj9eRGA1XWFw59UPZG8a7VkxO/Eb3K9NF297HUAcFMcbY6cPFi9AaBgu3VC4eetDnoN/+xT1owiHi7BReQhGAy/6cdf7C/my5ehZwD"
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE0xMwWedkKosax9+7D2OlnMxFL/eV4CvFZLsbLptpXr"
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiXXYkhRh+s7ixZ8rvG8ntIqd6FELQ9hh7HoaHQJRPU"
"ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJFsZ7PMDt80tYXHyScQajNhqH4wuYg/o0OxfOHaZD4rXuT0VIKflKH1M9LslfHWIEH3XNeqhQOziH9r+Ny5JcM="
];
k900 = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOi9vgVGs+S5kEsUqHPvyMMh1Q9gqL4TcbHoe5d73tun" ];
maxine = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILpWQfhNFdrxMTP/1DwBVuk49f3df9iH7Tbdu8ltIKjr" ];
jade = [

5
hosts/bagel-box/default.nix
View file

@ -40,6 +40,11 @@
hydra.enable = true;
hydra.dbi = "dbi:Pg:dbname=hydra;user=hydra";
hydra.builders = [
"builder-0"
"builder-1"
];
ofborg.enable = true;
};

2
secrets.nix
View file

@ -1,7 +1,7 @@
let
keys = import common/ssh-keys.nix;
commonKeys = keys.users.delroth;
commonKeys = keys.users.delroth ++ keys.users.raito;
secrets = with keys; {
hydra-s3-credentials = [ machines.bagel-box ];

17
services/baremetal-builder/default.nix
View file

@ -14,10 +14,23 @@ in
};
config = lib.mkIf cfg.enable {
boot.initrd.availableKernelModules = [ "ahci" "ehci_pci" "usb_storage" "usbhid" "sd_mod" ];
boot.initrd.kernelModules = [ "dm-snapshot" ];
users.users.builder = {
isSystemUser = true;
group = "nogroup";
home = "/var/empty";
shell = "/bin/sh";
openssh.authorizedKeys.keys = [
# Do not hardcode Hydra's public key, selectively
# add the keys of the coordinators that require us.
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAvUT9YBig9LQPHgypIBHQuC32XqDKxlFZ2CfgDi0ZKx"
];
};
nix.settings.trusted-users = [ "builder" ];
nixpkgs.hostPlatform = "x86_64-linux";
hardware.cpu.intel.updateMicrocode = true;
@ -45,6 +58,7 @@ in
networking.useNetworkd = true;
networking.hostName = "builder-${toString cfg.num}";
networking.domain = "wob01.infra.forkos.org";
systemd.network = {
netdevs = {
@ -75,6 +89,7 @@ in
];
networking.defaultGateway6 = { interface = "uplink"; address = "2a01:584:11::1"; };
deployment.targetHost = "2a01:584:11::1:${toString cfg.num}";
deployment.tags = [ "builders" ];
networking.nameservers = lib.mkForce ["2001:4860:4860::6464"]; # todo: other dns64

27
services/hydra/default.nix
View file

@ -1,7 +1,8 @@
{ config, lib, pkgs, ... }:
{ nodes, config, lib, pkgs, ... }:
let
cfg = config.bagel.services.hydra;
ssh-keys = import ../../common/ssh-keys.nix;
narCacheDir = "/var/cache/hydra/nar-cache";
port = 3000;
@ -9,6 +10,19 @@ let
mkCacheSettings = settings: builtins.concatStringsSep "&" (
lib.mapAttrsToList (k: v: "${k}=${v}") settings
);
mkBaremetalBuilder = { nrCores, publicHostKey, host, speedFactor ? 1, user ? "builder", supportedSystems ? [ "i686-linux" "x86_64-linux" ], supportedFeatures ? [ "big-parallel" "kvm" "nixos-test" ] }:
"ssh://${user}@${host} ${lib.concatStringsSep "," supportedSystems} ${config.age.secrets.hydra-ssh-key-priv.path} ${toString nrCores} ${toString speedFactor} ${lib.concatStringsSep "," supportedFeatures} - ${publicHostKey}";
# TODO:
# - generalize to new architectures
# - generalize to new features
baremetalBuilders = lib.concatStringsSep "\n"
(map (n: mkBaremetalBuilder {
nrCores = 40; # TODO: do not hardcode this, use the node's builder configuration.
publicHostKey = ssh-keys.machines.${n};
host = nodes.${n}.config.networking.fqdn;
}) cfg.builders);
in {
options.bagel.services.hydra = with lib; {
enable = mkEnableOption "Hydra coordinator";
@ -17,9 +31,19 @@ in {
type = types.str;
description = "DBI connection string for the Hydra postgres database";
};
builders = mkOption {
type = types.listOf types.str;
description = "List of builders to configure for Hydra";
example = [ "builder-0" "builder-1" ];
};
};
config = lib.mkIf cfg.enable {
# TODO: we should assert or warn that the builders
# does indeed have our public SSH key and are *builders*
# as a simple evaluation preflight check.
age.secrets.hydra-s3-credentials.file = ../../secrets/hydra-s3-credentials.age;
age.secrets.hydra-ssh-key-priv.owner = "hydra-queue-runner";
@ -54,6 +78,7 @@ in {
buildMachinesFiles = [
(pkgs.writeText "hydra-builders.conf" ''
ssh://bagel-builder@epyc.infra.newtype.fr i686-linux,x86_64-linux ${config.age.secrets.hydra-ssh-key-priv.path} 8 1 big-parallel,kvm,nixos-test - c3NoLWVkMjU1MTkgQUFBQUMzTnphQzFsWkRJMU5URTVBQUFBSU9YVDlJbml0MU1oS3Q0cmpCQU5McTB0MGJQd3cvV1FaOTZ1QjRBRURybWwgcm9vdEBuaXhvcwo=
${baremetalBuilders}
'')
];

9
terraform/gandi.nix
View file

@ -1,6 +1,6 @@
{ lib, config, ... }:
let
inherit (lib) mkEnableOption mkIf tf;
inherit (lib) mkEnableOption mkIf tf genList;
cfg = config.bagel.gandi;
in
{
@ -43,7 +43,7 @@ in
};
}) records);
in forkosRecords [
in forkosRecords ([
# (record "@" 3600 "A" ["163.172.69.160"])
(record "@" 3600 "AAAA" ["2001:bc8:38ee:100:1000::20"])
@ -67,6 +67,9 @@ in
(record "loki" 3600 "CNAME" ["meta01.infra"])
(record "mimir" 3600 "CNAME" ["meta01.infra"])
(record "matrix" 3600 "CNAME" ["meta01.infra"])
];
(record "vpn-gw.wob01.infra" 3600 "AAAA" [ "2a01:584:11::2" ])
# TODO: do not hardcode, just reuse the Colmena hive module outputs to generate all the required details.
] ++ map (index: record "builder-${toString index}.wob01.infra" 3600 "AAAA" [ "2a01:584:11::1:${toString index}" ]) (genList lib.id 12));
};
}