{ nodes, config, lib, pkgs, ... }: let cfg = config.bagel.services.hydra; ssh-keys = import ../../common/ssh-keys.nix; narCacheDir = "/var/cache/hydra/nar-cache"; port = 3000; mkCacheSettings = settings: builtins.concatStringsSep "&" ( lib.mapAttrsToList (k: v: "${k}=${v}") settings ); mkBaremetalBuilder = { nrCores, publicHostKey, host, speedFactor ? 1, user ? "builder", supportedSystems ? [ "i686-linux" "x86_64-linux" ], supportedFeatures ? [ "big-parallel" "kvm" "nixos-test" ] }: "ssh://${user}@${host} ${lib.concatStringsSep "," supportedSystems} ${config.age.secrets.hydra-ssh-key-priv.path} ${toString nrCores} ${toString speedFactor} ${lib.concatStringsSep "," supportedFeatures} - ${publicHostKey}"; # TODO: # - generalize to new architectures # - generalize to new features baremetalBuilders = lib.concatStringsSep "\n" (map (n: mkBaremetalBuilder { nrCores = 40; # TODO: do not hardcode this, use the node's builder configuration. publicHostKey = ssh-keys.machines.${n}; host = nodes.${n}.config.networking.fqdn; }) cfg.builders); in { options.bagel.services.hydra = with lib; { enable = mkEnableOption "Hydra coordinator"; dbi = mkOption { type = types.str; description = "DBI connection string for the Hydra postgres database"; }; builders = mkOption { type = types.listOf types.str; description = "List of builders to configure for Hydra"; example = [ "builder-0" "builder-1" ]; }; }; config = lib.mkIf cfg.enable { # TODO: we should assert or warn that the builders # does indeed have our public SSH key and are *builders* # as a simple evaluation preflight check. age.secrets.hydra-s3-credentials.file = ../../secrets/hydra-s3-credentials.age; age.secrets.hydra-ssh-key-priv.owner = "hydra-queue-runner"; age.secrets.hydra-ssh-key-priv.file = ../../secrets/hydra-ssh-key-priv.age; systemd.tmpfiles.rules = [ "d /var/cache/hydra 0755 hydra hydra - -" "d ${narCacheDir} 0755 hydra hydra 1d -" ]; # XXX: Otherwise services.hydra-dev overwrites it to only hydra-queue-runner... # # Can be removed once this is added to some common config template. nix.settings.trusted-users = [ "root" "hydra" "@wheel" ]; services.hydra-dev = { enable = true; listenHost = "localhost"; port = port; dbi = cfg.dbi; hydraURL = "https://hydra.forkos.org"; useSubstitutes = false; notificationSender = "hydra@forkos.org"; # XXX: hydra overlay sets pkgs.hydra, but hydra's nixos module uses # pkgs.hydra_unstable... package = pkgs.hydra; buildMachinesFiles = [ (pkgs.writeText "hydra-builders.conf" baremetalBuilders) ]; extraConfig = '' store_uri = s3://bagel-cache?${mkCacheSettings { endpoint = "s3.delroth.net"; region = "garage"; #secret-key = "TODO"; compression = "zstd"; log-compression = "br"; ls-compression = "br"; write-nar-listing = "1"; }} server_store_uri = https://bagel-cache.s3-web.delroth.net?local-nar-cache=${narCacheDir} binary_cache_public_url = https://bagel-cache.s3-web.delroth.net log_prefix = https://bagel-cache.s3-web.delroth.net upload_logs_to_binary_cache = true evaluator_workers = 4 evaluator_max_memory_size = 4096 max_concurrent_evals = 1 allow_import_from_derivation = false max_output_size = ${builtins.toString (3 * 1024 * 1024 * 1024)} max_db_connections = 100 ''; }; systemd.services.hydra-queue-runner.serviceConfig.EnvironmentFile = config.age.secrets.hydra-s3-credentials.path; services.nginx = { enable = true; enableReload = true; recommendedBrotliSettings = true; recommendedGzipSettings = true; recommendedOptimisation = true; recommendedProxySettings = true; recommendedTlsSettings = true; recommendedZstdSettings = true; proxyTimeout = "900s"; appendConfig = '' worker_processes auto; ''; virtualHosts."hydra.forkos.org" = { forceSSL = true; enableACME = true; locations."/" = { proxyPass = "http://127.0.0.1:${builtins.toString port}"; }; locations."/static/" = { alias = "${config.services.hydra-dev.package}/libexec/hydra/root/static/"; }; }; }; networking.firewall.allowedTCPPorts = [ 80 443 ]; }; }