the-infra/services/hydra/default.nix

{ nodes, config, lib, pkgs, ... }:

let
  cfg = config.bagel.services.hydra;
  ssh-keys = import ../../common/ssh-keys.nix;

  narCacheDir = "/var/cache/hydra/nar-cache";
  port = 3000;

  mkCacheSettings = settings: builtins.concatStringsSep "&" (
    lib.mapAttrsToList (k: v: "${k}=${v}") settings
    );

  # XXX: to support Nix's dumb public host key syntax (base64'd), this outputs
  # a string with shell-style command interpolations: $(...).
  mkBaremetalBuilder = { parallelBuilds, publicHostKey, host, speedFactor ? 1, user ? "builder", supportedSystems ? [ "i686-linux" "x86_64-linux" ], supportedFeatures ? [ "big-parallel" "kvm" "nixos-test" ] }:
  "ssh://${user}@${host}?remote-store=/mnt ${lib.concatStringsSep "," supportedSystems} ${config.age.secrets.hydra-ssh-key-priv.path} ${toString parallelBuilds} ${toString speedFactor} ${lib.concatStringsSep "," supportedFeatures} - $(echo -n '${publicHostKey}' | base64 -w0)";

  # TODO:
  # - generalize to new architectures
  # - generalize to new features
  baremetalBuilders = lib.concatStringsSep "\n"
    (map (n: mkBaremetalBuilder {
      parallelBuilds = 8; # TODO: do not hardcode this, use the node's builder configuration.
      publicHostKey = ssh-keys.machines.${n};
      host = nodes.${n}.config.networking.fqdn;
    }) cfg.builders);
in {
  options.bagel.services.hydra = with lib; {
    enable = mkEnableOption "Hydra coordinator";

    dbi = mkOption {
      type = types.str;
      description = "DBI connection string for the Hydra postgres database";
    };

    builders = mkOption {
      type = types.listOf types.str;
      description = "List of builders to configure for Hydra";
      example = [ "builder-0" "builder-1" ];
    };
  };

  config = lib.mkIf cfg.enable {
    # TODO: we should assert or warn that the builders
    # does indeed have our public SSH key and are *builders*
    # as a simple evaluation preflight check.

    age.secrets.hydra-s3-credentials.file = ../../secrets/hydra-s3-credentials.age;

    age.secrets.hydra-signing-priv.owner = "hydra-queue-runner";
    age.secrets.hydra-signing-priv.file = ../../secrets/hydra-signing-priv.age;

    age.secrets.hydra-ssh-key-priv.owner = "hydra-queue-runner";
    age.secrets.hydra-ssh-key-priv.file = ../../secrets/hydra-ssh-key-priv.age;

    systemd.tmpfiles.rules = [
      "d /var/cache/hydra 0755 hydra hydra -  -"
      "d ${narCacheDir}   0755 hydra hydra 1d -"
    ];

    # XXX: Otherwise services.hydra-dev overwrites it to only hydra-queue-runner...
    #
    # Can be removed once this is added to some common config template.
    nix.settings.trusted-users = [ "root" "hydra" "hydra-www" "@wheel" ];

    # Because Hydra can't fetch flake inputs otherwise... also yes, this
    # prefix-based matching is absurdly bad.
    nix.settings.allowed-uris = [
      "github:"
      "https://github.com/"
      "https://git.lix.systems/"
      "https://git@git.lix.systems/"
    ];

    services.hydra-dev = {
      enable = true;

      listenHost = "localhost";
      port = port;
      dbi = cfg.dbi;

      hydraURL = "https://hydra.forkos.org";
      useSubstitutes = false;

      notificationSender = "hydra@forkos.org";

      # XXX: hydra overlay sets pkgs.hydra, but hydra's nixos module uses
      # pkgs.hydra_unstable...
      package = pkgs.hydra;

      buildMachinesFiles = [
        (pkgs.runCommandNoCC "hydra-builders.conf" {} ''
          cat >$out <<EOF
          ${baremetalBuilders}
          EOF
        '')
      ];

      extraConfig = ''
        store_uri = s3://bagel-cache?${mkCacheSettings {
          endpoint = "s3.delroth.net";
          region = "garage";

          secret-key = config.age.secrets.hydra-signing-priv.path;

          compression = "zstd";
          log-compression = "br";
          ls-compression = "br";

          write-nar-listing = "1";
        }}

        server_store_uri = https://bagel-cache.s3-web.delroth.net?local-nar-cache=${narCacheDir}
        binary_cache_public_url = https://bagel-cache.s3-web.delroth.net
        log_prefix = https://bagel-cache.s3-web.delroth.net

        upload_logs_to_binary_cache = true

        evaluator_workers = 4
        evaluator_max_memory_size = 4096
        max_concurrent_evals = 1

        allow_import_from_derivation = false

        max_output_size = ${builtins.toString (3 * 1024 * 1024 * 1024)}
        max_db_connections = 100
      '';
    };

    systemd.services.hydra-queue-runner = {
      # FIXME: should probably be set in the upstream Hydra module?
      wants = [ "network-online.target" ];
      serviceConfig.EnvironmentFile = config.age.secrets.hydra-s3-credentials.path;
    };

    services.nginx = {
      enable = true;
      enableReload = true;

      recommendedBrotliSettings = true;
      recommendedGzipSettings = true;
      recommendedOptimisation = true;
      recommendedProxySettings = true;
      recommendedTlsSettings = true;
      recommendedZstdSettings = true;

      proxyTimeout = "900s";

      appendConfig = ''
        worker_processes auto;
      '';

      virtualHosts."hydra.forkos.org" = {
        forceSSL = true;
        enableACME = true;

        locations."/" = {
          proxyPass = "http://127.0.0.1:${builtins.toString port}";
        };

        locations."/static/" = {
          alias = "${config.services.hydra-dev.package}/libexec/hydra/root/static/";
        };
      };
    };

    networking.firewall.allowedTCPPorts = [ 80 443 ];
  };
}