forked from the-distro/infra
Merge pull request 'hydra: wire up new builders' (#47) from hydra-wire-up into main
Reviewed-on: the-distro/infra#47
This commit is contained in:
commit
664fa033aa
|
|
@ -8,12 +8,5 @@ in {
|
|||
keys.users.maxine ++
|
||||
keys.users.jade ++
|
||||
keys.users.lukegb ++
|
||||
keys.users.yuka ++
|
||||
[
|
||||
# more raito
|
||||
"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDcEkYM1r8QVNM/G5CxJInEdoBCWjEHHDdHlzDYNSUIdHHsn04QY+XI67AdMCm8w30GZnLUIj5RiJEWXREUApby0GrfxGGcy8otforygfgtmuUKAUEHdU2MMwrQI7RtTZ8oQ0USRGuqvmegxz3l5caVU7qGvBllJ4NUHXrkZSja2/51vq80RF4MKkDGiz7xUTixI2UcBwQBCA/kQedKV9G28EH+1XfvePqmMivZjl+7VyHsgUVj9eRGA1XWFw59UPZG8a7VkxO/Eb3K9NF297HUAcFMcbY6cPFi9AaBgu3VC4eetDnoN/+xT1owiHi7BReQhGAy/6cdf7C/my5ehZwD"
|
||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE0xMwWedkKosax9+7D2OlnMxFL/eV4CvFZLsbLptpXr"
|
||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiXXYkhRh+s7ixZ8rvG8ntIqd6FELQ9hh7HoaHQJRPU"
|
||||
"ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJFsZ7PMDt80tYXHyScQajNhqH4wuYg/o0OxfOHaZD4rXuT0VIKflKH1M9LslfHWIEH3XNeqhQOziH9r+Ny5JcM="
|
||||
];
|
||||
keys.users.yuka;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,7 +21,13 @@
|
|||
|
||||
users = {
|
||||
delroth = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII3tjB4KYDok3KlWxdBp/yEmqhhmybd+w0VO4xUwLKKV" ];
|
||||
raito = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICaw9ihTG7ucB8P38XdalEWev8+q96e2yNm4B+/I9IJp" ];
|
||||
raito = [
|
||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICaw9ihTG7ucB8P38XdalEWev8+q96e2yNm4B+/I9IJp"
|
||||
"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDcEkYM1r8QVNM/G5CxJInEdoBCWjEHHDdHlzDYNSUIdHHsn04QY+XI67AdMCm8w30GZnLUIj5RiJEWXREUApby0GrfxGGcy8otforygfgtmuUKAUEHdU2MMwrQI7RtTZ8oQ0USRGuqvmegxz3l5caVU7qGvBllJ4NUHXrkZSja2/51vq80RF4MKkDGiz7xUTixI2UcBwQBCA/kQedKV9G28EH+1XfvePqmMivZjl+7VyHsgUVj9eRGA1XWFw59UPZG8a7VkxO/Eb3K9NF297HUAcFMcbY6cPFi9AaBgu3VC4eetDnoN/+xT1owiHi7BReQhGAy/6cdf7C/my5ehZwD"
|
||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE0xMwWedkKosax9+7D2OlnMxFL/eV4CvFZLsbLptpXr"
|
||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiXXYkhRh+s7ixZ8rvG8ntIqd6FELQ9hh7HoaHQJRPU"
|
||||
"ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJFsZ7PMDt80tYXHyScQajNhqH4wuYg/o0OxfOHaZD4rXuT0VIKflKH1M9LslfHWIEH3XNeqhQOziH9r+Ny5JcM="
|
||||
];
|
||||
k900 = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOi9vgVGs+S5kEsUqHPvyMMh1Q9gqL4TcbHoe5d73tun" ];
|
||||
maxine = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILpWQfhNFdrxMTP/1DwBVuk49f3df9iH7Tbdu8ltIKjr" ];
|
||||
jade = [
|
||||
|
|
|
|||
|
|
@ -40,6 +40,11 @@
|
|||
hydra.enable = true;
|
||||
hydra.dbi = "dbi:Pg:dbname=hydra;user=hydra";
|
||||
|
||||
hydra.builders = [
|
||||
"builder-0"
|
||||
"builder-1"
|
||||
];
|
||||
|
||||
ofborg.enable = true;
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
let
|
||||
keys = import common/ssh-keys.nix;
|
||||
|
||||
commonKeys = keys.users.delroth;
|
||||
commonKeys = keys.users.delroth ++ keys.users.raito;
|
||||
|
||||
secrets = with keys; {
|
||||
hydra-s3-credentials = [ machines.bagel-box ];
|
||||
|
|
|
|||
|
|
@ -14,10 +14,23 @@ in
|
|||
};
|
||||
|
||||
config = lib.mkIf cfg.enable {
|
||||
|
||||
boot.initrd.availableKernelModules = [ "ahci" "ehci_pci" "usb_storage" "usbhid" "sd_mod" ];
|
||||
boot.initrd.kernelModules = [ "dm-snapshot" ];
|
||||
|
||||
users.users.builder = {
|
||||
isSystemUser = true;
|
||||
group = "nogroup";
|
||||
home = "/var/empty";
|
||||
shell = "/bin/sh";
|
||||
openssh.authorizedKeys.keys = [
|
||||
# Do not hardcode Hydra's public key, selectively
|
||||
# add the keys of the coordinators that require us.
|
||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAvUT9YBig9LQPHgypIBHQuC32XqDKxlFZ2CfgDi0ZKx"
|
||||
];
|
||||
};
|
||||
nix.settings.trusted-users = [ "builder" ];
|
||||
|
||||
|
||||
nixpkgs.hostPlatform = "x86_64-linux";
|
||||
hardware.cpu.intel.updateMicrocode = true;
|
||||
|
||||
|
|
@ -45,6 +58,7 @@ in
|
|||
|
||||
networking.useNetworkd = true;
|
||||
networking.hostName = "builder-${toString cfg.num}";
|
||||
networking.domain = "wob01.infra.forkos.org";
|
||||
|
||||
systemd.network = {
|
||||
netdevs = {
|
||||
|
|
@ -75,6 +89,7 @@ in
|
|||
];
|
||||
networking.defaultGateway6 = { interface = "uplink"; address = "2a01:584:11::1"; };
|
||||
deployment.targetHost = "2a01:584:11::1:${toString cfg.num}";
|
||||
deployment.tags = [ "builders" ];
|
||||
|
||||
networking.nameservers = lib.mkForce ["2001:4860:4860::6464"]; # todo: other dns64
|
||||
|
||||
|
|
|
|||
|
|
@ -1,14 +1,28 @@
|
|||
{ config, lib, pkgs, ... }:
|
||||
{ nodes, config, lib, pkgs, ... }:
|
||||
|
||||
let
|
||||
cfg = config.bagel.services.hydra;
|
||||
ssh-keys = import ../../common/ssh-keys.nix;
|
||||
|
||||
narCacheDir = "/var/cache/hydra/nar-cache";
|
||||
port = 3000;
|
||||
|
||||
mkCacheSettings = settings: builtins.concatStringsSep "&" (
|
||||
lib.mapAttrsToList (k: v: "${k}=${v}") settings
|
||||
);
|
||||
);
|
||||
|
||||
mkBaremetalBuilder = { nrCores, publicHostKey, host, speedFactor ? 1, user ? "builder", supportedSystems ? [ "i686-linux" "x86_64-linux" ], supportedFeatures ? [ "big-parallel" "kvm" "nixos-test" ] }:
|
||||
"ssh://${user}@${host} ${lib.concatStringsSep "," supportedSystems} ${config.age.secrets.hydra-ssh-key-priv.path} ${toString nrCores} ${toString speedFactor} ${lib.concatStringsSep "," supportedFeatures} - ${publicHostKey}";
|
||||
|
||||
# TODO:
|
||||
# - generalize to new architectures
|
||||
# - generalize to new features
|
||||
baremetalBuilders = lib.concatStringsSep "\n"
|
||||
(map (n: mkBaremetalBuilder {
|
||||
nrCores = 40; # TODO: do not hardcode this, use the node's builder configuration.
|
||||
publicHostKey = ssh-keys.machines.${n};
|
||||
host = nodes.${n}.config.networking.fqdn;
|
||||
}) cfg.builders);
|
||||
in {
|
||||
options.bagel.services.hydra = with lib; {
|
||||
enable = mkEnableOption "Hydra coordinator";
|
||||
|
|
@ -17,9 +31,19 @@ in {
|
|||
type = types.str;
|
||||
description = "DBI connection string for the Hydra postgres database";
|
||||
};
|
||||
|
||||
builders = mkOption {
|
||||
type = types.listOf types.str;
|
||||
description = "List of builders to configure for Hydra";
|
||||
example = [ "builder-0" "builder-1" ];
|
||||
};
|
||||
};
|
||||
|
||||
config = lib.mkIf cfg.enable {
|
||||
# TODO: we should assert or warn that the builders
|
||||
# does indeed have our public SSH key and are *builders*
|
||||
# as a simple evaluation preflight check.
|
||||
|
||||
age.secrets.hydra-s3-credentials.file = ../../secrets/hydra-s3-credentials.age;
|
||||
|
||||
age.secrets.hydra-ssh-key-priv.owner = "hydra-queue-runner";
|
||||
|
|
@ -54,7 +78,8 @@ in {
|
|||
buildMachinesFiles = [
|
||||
(pkgs.writeText "hydra-builders.conf" ''
|
||||
ssh://bagel-builder@epyc.infra.newtype.fr i686-linux,x86_64-linux ${config.age.secrets.hydra-ssh-key-priv.path} 8 1 big-parallel,kvm,nixos-test - c3NoLWVkMjU1MTkgQUFBQUMzTnphQzFsWkRJMU5URTVBQUFBSU9YVDlJbml0MU1oS3Q0cmpCQU5McTB0MGJQd3cvV1FaOTZ1QjRBRURybWwgcm9vdEBuaXhvcwo=
|
||||
'')
|
||||
${baremetalBuilders}
|
||||
'')
|
||||
];
|
||||
|
||||
extraConfig = ''
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{ lib, config, ... }:
|
||||
let
|
||||
inherit (lib) mkEnableOption mkIf tf;
|
||||
inherit (lib) mkEnableOption mkIf tf genList;
|
||||
cfg = config.bagel.gandi;
|
||||
in
|
||||
{
|
||||
|
|
@ -43,7 +43,7 @@ in
|
|||
};
|
||||
}) records);
|
||||
|
||||
in forkosRecords [
|
||||
in forkosRecords ([
|
||||
# (record "@" 3600 "A" ["163.172.69.160"])
|
||||
(record "@" 3600 "AAAA" ["2001:bc8:38ee:100:1000::20"])
|
||||
|
||||
|
|
@ -67,6 +67,9 @@ in
|
|||
(record "loki" 3600 "CNAME" ["meta01.infra"])
|
||||
(record "mimir" 3600 "CNAME" ["meta01.infra"])
|
||||
(record "matrix" 3600 "CNAME" ["meta01.infra"])
|
||||
];
|
||||
|
||||
(record "vpn-gw.wob01.infra" 3600 "AAAA" [ "2a01:584:11::2" ])
|
||||
# TODO: do not hardcode, just reuse the Colmena hive module outputs to generate all the required details.
|
||||
] ++ map (index: record "builder-${toString index}.wob01.infra" 3600 "AAAA" [ "2a01:584:11::1:${toString index}" ]) (genList lib.id 12));
|
||||
};
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue