infra/services/hydra/default.nix

{ nodes, config, lib, pkgs, ... }:

let
  cfg = config.bagel.services.hydra;
  ssh-keys = import ../../common/ssh-keys.nix;

  narCacheDir = "/var/cache/hydra/nar-cache";
  port = 3000;

  mkCacheSettings = settings: builtins.concatStringsSep "&" (
    lib.mapAttrsToList (k: v: "${k}=${v}") settings
  );

  mkPgConnString = options: builtins.concatStringsSep ";" (
    lib.mapAttrsToList (k: v: "${k}=${v}") options
  );

  # XXX: to support Nix's dumb public host key syntax (base64'd), this outputs
  # a string with shell-style command interpolations: $(...).
  mkBaremetalBuilder = {
    parallelBuilds,
    publicHostKey,
    host,
    speedFactor ? 1,
    user ? "builder",
    supportedSystems ? [ "i686-linux" "x86_64-linux" ],
    supportedFeatures ? [ "big-parallel" "kvm" "nixos-test" ],
    requiredFeatures ? [ ]
  }:
  let
    supportedFeatures_ = if (supportedFeatures != []) then lib.concatStringsSep "," supportedFeatures else "-";
    requiredFeatures_ = if (requiredFeatures != []) then lib.concatStringsSep "," requiredFeatures else "-";
  in
  "ssh://${user}@${host}?remote-store=/mnt ${lib.concatStringsSep "," supportedSystems} ${config.age.secrets.hydra-ssh-key-priv.path} ${toString parallelBuilds} ${toString speedFactor} ${supportedFeatures_} ${requiredFeatures_} $(echo -n '${publicHostKey}' | base64 -w0)";

  # TODO:
  # - generalize to new architectures
  # - generalize to new features
  baremetalBuilders = lib.concatStringsSep "\n"
    (map (n: let
      assignments = (import ../baremetal-builder/assignments.nix).${n} or {
        inherit (nodes.${n}.config.nix.settings) max-jobs;
        supported-features = [ "big-parallel" "kvm" "nixos-test" ];
        required-features = [];
      };
    in mkBaremetalBuilder {
      parallelBuilds = assignments.max-jobs;
      supportedFeatures = assignments.supported-features;
      requiredFeatures = assignments.required-features;
      publicHostKey = ssh-keys.machines.${n};
      host = nodes.${n}.config.networking.fqdn;
    }) cfg.builders);
in {
  options.bagel.services.hydra = with lib; {
    enable = mkEnableOption "Hydra coordinator";

    builders = mkOption {
      type = types.listOf types.str;
      description = "List of builders to configure for Hydra";
      example = [ "builder-0" "builder-1" ];
    };
  };

  config = lib.mkIf cfg.enable {
    # TODO: we should assert or warn that the builders
    # does indeed have our public SSH key and are *builders*
    # as a simple evaluation preflight check.

    bagel.secrets.files = [
      "hydra-s3-credentials"
      "hydra-postgres-key"
      "hydra-signing-priv"
      "hydra-ssh-key-priv"
    ];

    age.secrets.hydra-postgres-key.group = "hydra";
    age.secrets.hydra-postgres-key.mode = "0440";

    age.secrets.hydra-signing-priv.owner = "hydra-queue-runner";

    age.secrets.hydra-ssh-key-priv.owner = "hydra-queue-runner";

    systemd.tmpfiles.rules = [
      "d /var/cache/hydra 0755 hydra hydra -  -"
      "d ${narCacheDir}   0755 hydra hydra 1d -"
    ];

    # XXX: Otherwise services.hydra-dev overwrites it to only hydra-queue-runner...
    #
    # Can be removed once this is added to some common config template.
    nix.settings.trusted-users = [ "root" "hydra" "hydra-www" "@wheel" ];

    # Because Hydra can't fetch flake inputs otherwise... also yes, this
    # prefix-based matching is absurdly bad.
    nix.settings.allowed-uris = [
      "github:"
      "https://github.com/"
      "https://git.lix.systems/"
      "https://git@git.lix.systems/"
    ];

    services.hydra-dev = {
      enable = true;

      listenHost = "127.0.0.1";
      port = port;

      dbi = "dbi:Pg:${mkPgConnString {
        host = "postgres.forkos.org";
        dbname = "hydra";
        user = "hydra";
        sslmode = "verify-full";
        sslcert = "${./postgres.crt}";
        sslkey = config.age.secrets.hydra-postgres-key.path;
        sslrootcert = "${../postgres/ca.crt}";
      }}";

      hydraURL = "https://hydra.forkos.org";
      useSubstitutes = false;

      notificationSender = "hydra@forkos.org";

      # XXX: hydra overlay sets pkgs.hydra, but hydra's nixos module uses
      # pkgs.hydra_unstable...
      package = pkgs.hydra;

      buildMachinesFiles = [
        (pkgs.runCommandNoCC "hydra-builders.conf" {} ''
          cat >$out <<EOF
          ${baremetalBuilders}
          EOF
        '')
      ];

      extraConfig = ''
        store_uri = s3://bagel-cache?${mkCacheSettings {
          endpoint = "s3.delroth.net";
          region = "garage";

          secret-key = config.age.secrets.hydra-signing-priv.path;

          compression = "zstd";
          log-compression = "br";
          ls-compression = "br";

          write-nar-listing = "1";
        }}

        server_store_uri = https://bagel-cache.s3-web.delroth.net?local-nar-cache=${narCacheDir}
        binary_cache_public_url = https://bagel-cache.s3-web.delroth.net
        log_prefix = https://bagel-cache.s3-web.delroth.net/

        upload_logs_to_binary_cache = true

        evaluator_workers = 16
        evaluator_max_memory_size = 4096
        max_concurrent_evals = 1

        allow_import_from_derivation = false

        max_output_size = ${builtins.toString (3 * 1024 * 1024 * 1024)}
        max_db_connections = 100

        <git-input>
          timeout = 1800
        </git-input>
      '';
    };

    systemd.services.hydra-queue-runner = {
      # FIXME: should probably be set in the upstream Hydra module?
      wants = [ "network-online.target" ];
      serviceConfig.EnvironmentFile = config.age.secrets.hydra-s3-credentials.path;
    };

    services.nginx = {
      enable = true;
      enableReload = true;

      recommendedBrotliSettings = true;
      recommendedGzipSettings = true;
      recommendedOptimisation = true;
      recommendedProxySettings = true;
      recommendedTlsSettings = true;
      recommendedZstdSettings = true;

      proxyTimeout = "900s";

      appendConfig = ''
        worker_processes auto;
      '';

      virtualHosts."hydra.forkos.org" = {
        forceSSL = true;
        enableACME = true;

        locations."/" = {
          proxyPass = "http://127.0.0.1:${builtins.toString port}";
        };

        locations."/static/" = {
          alias = "${config.services.hydra-dev.package}/libexec/hydra/root/static/";
        };
      };
    };

    networking.firewall.allowedTCPPorts = [ 80 443 ];
  };
}