forked from the-distro/infra
159 lines
4.9 KiB
Nix
159 lines
4.9 KiB
Nix
{ nodes, config, lib, pkgs, ... }:
|
|
|
|
let
|
|
cfg = config.bagel.services.hydra;
|
|
ssh-keys = import ../../common/ssh-keys.nix;
|
|
|
|
narCacheDir = "/var/cache/hydra/nar-cache";
|
|
port = 3000;
|
|
|
|
mkCacheSettings = settings: builtins.concatStringsSep "&" (
|
|
lib.mapAttrsToList (k: v: "${k}=${v}") settings
|
|
);
|
|
|
|
# XXX: to support Nix's dumb public host key syntax (base64'd), this outputs
|
|
# a string with shell-style command interpolations: $(...).
|
|
mkBaremetalBuilder = { nrCores, publicHostKey, host, speedFactor ? 1, user ? "builder", supportedSystems ? [ "i686-linux" "x86_64-linux" ], supportedFeatures ? [ "big-parallel" "kvm" "nixos-test" ] }:
|
|
"ssh://${user}@${host} ${lib.concatStringsSep "," supportedSystems} ${config.age.secrets.hydra-ssh-key-priv.path} ${toString nrCores} ${toString speedFactor} ${lib.concatStringsSep "," supportedFeatures} - $(echo -n '${publicHostKey}' | base64 -w0)";
|
|
|
|
# TODO:
|
|
# - generalize to new architectures
|
|
# - generalize to new features
|
|
baremetalBuilders = lib.concatStringsSep "\n"
|
|
(map (n: mkBaremetalBuilder {
|
|
nrCores = 40; # TODO: do not hardcode this, use the node's builder configuration.
|
|
publicHostKey = ssh-keys.machines.${n};
|
|
host = nodes.${n}.config.networking.fqdn;
|
|
}) cfg.builders);
|
|
in {
|
|
options.bagel.services.hydra = with lib; {
|
|
enable = mkEnableOption "Hydra coordinator";
|
|
|
|
dbi = mkOption {
|
|
type = types.str;
|
|
description = "DBI connection string for the Hydra postgres database";
|
|
};
|
|
|
|
builders = mkOption {
|
|
type = types.listOf types.str;
|
|
description = "List of builders to configure for Hydra";
|
|
example = [ "builder-0" "builder-1" ];
|
|
};
|
|
};
|
|
|
|
config = lib.mkIf cfg.enable {
|
|
# TODO: we should assert or warn that the builders
|
|
# does indeed have our public SSH key and are *builders*
|
|
# as a simple evaluation preflight check.
|
|
|
|
age.secrets.hydra-s3-credentials.file = ../../secrets/hydra-s3-credentials.age;
|
|
|
|
age.secrets.hydra-signing-priv.owner = "hydra-queue-runner";
|
|
age.secrets.hydra-signing-priv.file = ../../secrets/hydra-signing-priv.age;
|
|
|
|
age.secrets.hydra-ssh-key-priv.owner = "hydra-queue-runner";
|
|
age.secrets.hydra-ssh-key-priv.file = ../../secrets/hydra-ssh-key-priv.age;
|
|
|
|
systemd.tmpfiles.rules = [
|
|
"d /var/cache/hydra 0755 hydra hydra - -"
|
|
"d ${narCacheDir} 0755 hydra hydra 1d -"
|
|
];
|
|
|
|
# XXX: Otherwise services.hydra-dev overwrites it to only hydra-queue-runner...
|
|
#
|
|
# Can be removed once this is added to some common config template.
|
|
nix.settings.trusted-users = [ "root" "hydra" "@wheel" ];
|
|
|
|
services.hydra-dev = {
|
|
enable = true;
|
|
|
|
listenHost = "localhost";
|
|
port = port;
|
|
dbi = cfg.dbi;
|
|
|
|
hydraURL = "https://hydra.forkos.org";
|
|
useSubstitutes = false;
|
|
|
|
notificationSender = "hydra@forkos.org";
|
|
|
|
# XXX: hydra overlay sets pkgs.hydra, but hydra's nixos module uses
|
|
# pkgs.hydra_unstable...
|
|
package = pkgs.hydra;
|
|
|
|
buildMachinesFiles = [
|
|
(pkgs.runCommandNoCC "hydra-builders.conf" {} ''
|
|
cat >$out <<EOF
|
|
${baremetalBuilders}
|
|
EOF
|
|
'')
|
|
];
|
|
|
|
extraConfig = ''
|
|
store_uri = s3://bagel-cache?${mkCacheSettings {
|
|
endpoint = "s3.delroth.net";
|
|
region = "garage";
|
|
|
|
secret-key = config.age.secrets.hydra-signing-priv.path;
|
|
|
|
compression = "zstd";
|
|
log-compression = "br";
|
|
ls-compression = "br";
|
|
|
|
write-nar-listing = "1";
|
|
}}
|
|
|
|
server_store_uri = https://bagel-cache.s3-web.delroth.net?local-nar-cache=${narCacheDir}
|
|
binary_cache_public_url = https://bagel-cache.s3-web.delroth.net
|
|
log_prefix = https://bagel-cache.s3-web.delroth.net
|
|
|
|
upload_logs_to_binary_cache = true
|
|
|
|
evaluator_workers = 4
|
|
evaluator_max_memory_size = 4096
|
|
max_concurrent_evals = 1
|
|
|
|
allow_import_from_derivation = false
|
|
|
|
max_output_size = ${builtins.toString (3 * 1024 * 1024 * 1024)}
|
|
max_db_connections = 100
|
|
'';
|
|
};
|
|
|
|
systemd.services.hydra-queue-runner.serviceConfig.EnvironmentFile =
|
|
config.age.secrets.hydra-s3-credentials.path;
|
|
|
|
services.nginx = {
|
|
enable = true;
|
|
enableReload = true;
|
|
|
|
recommendedBrotliSettings = true;
|
|
recommendedGzipSettings = true;
|
|
recommendedOptimisation = true;
|
|
recommendedProxySettings = true;
|
|
recommendedTlsSettings = true;
|
|
recommendedZstdSettings = true;
|
|
|
|
proxyTimeout = "900s";
|
|
|
|
appendConfig = ''
|
|
worker_processes auto;
|
|
'';
|
|
|
|
virtualHosts."hydra.forkos.org" = {
|
|
forceSSL = true;
|
|
enableACME = true;
|
|
|
|
locations."/" = {
|
|
proxyPass = "http://127.0.0.1:${builtins.toString port}";
|
|
};
|
|
|
|
locations."/static/" = {
|
|
alias = "${config.services.hydra-dev.package}/libexec/hydra/root/static/";
|
|
};
|
|
};
|
|
};
|
|
|
|
networking.firewall.allowedTCPPorts = [ 80 443 ];
|
|
};
|
|
}
|