hydra: provide S3 and SSH credentials (via agenix)

common/ssh-keys.nix

@@ -0,0 +1,5 @@
+{
+  machines.bagel-box = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJW7jmkJ73tx9lsrz9UhqJIJdoqZGuhsHti55xny5/yp";
+
+  users.delroth = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII3tjB4KYDok3KlWxdBp/yEmqhhmybd+w0VO4xUwLKKV" ];
+}

flake.lock

@@ -1,5 +1,28 @@
 {
   "nodes": {
+    "agenix": {
+      "inputs": {
+        "darwin": "darwin",
+        "home-manager": "home-manager",
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1718371084,
+        "narHash": "sha256-abpBi61mg0g+lFFU0zY4C6oP6fBwPzbHPKBGw676xsA=",
+        "owner": "ryantm",
+        "repo": "agenix",
+        "rev": "3a56735779db467538fb2e577eda28a9daacaca6",
+        "type": "github"
+      },
+      "original": {
+        "owner": "ryantm",
+        "repo": "agenix",
+        "type": "github"
+      }
+    },
     "colmena": {
       "inputs": {
         "flake-compat": "flake-compat",
@@ -23,6 +46,28 @@
         "type": "github"
       }
     },
+    "darwin": {
+      "inputs": {
+        "nixpkgs": [
+          "agenix",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1700795494,
+        "narHash": "sha256-gzGLZSiOhf155FW7262kdHo2YDeugp3VuIFb4/GGng0=",
+        "owner": "lnl7",
+        "repo": "nix-darwin",
+        "rev": "4b9b83d5a92e8c1fbfd8eb27eda375908c11ec4d",
+        "type": "github"
+      },
+      "original": {
+        "owner": "lnl7",
+        "ref": "master",
+        "repo": "nix-darwin",
+        "type": "github"
+      }
+    },
     "flake-compat": {
       "flake": false,
       "locked": {
@@ -39,6 +84,22 @@
         "type": "github"
       }
     },
+    "flake-compat_2": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1696426674,
+        "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
     "flake-utils": {
       "locked": {
         "lastModified": 1659877975,
@@ -54,6 +115,89 @@
         "type": "github"
       }
     },
+    "home-manager": {
+      "inputs": {
+        "nixpkgs": [
+          "agenix",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1703113217,
+        "narHash": "sha256-7ulcXOk63TIT2lVDSExj7XzFx09LpdSAPtvgtM7yQPE=",
+        "owner": "nix-community",
+        "repo": "home-manager",
+        "rev": "3bfaacf46133c037bb356193bd2f1765d9dc82c1",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "home-manager",
+        "type": "github"
+      }
+    },
+    "hydra": {
+      "inputs": {
+        "nix": "nix",
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1719253535,
+        "narHash": "sha256-HE0QHbOEfmmiFRBtWy+7pTlaDolxDfo5mgPcV20KOGA=",
+        "ref": "refs/heads/main",
+        "rev": "e9d0a3a754d5a477126ecb3c0bac3bf91a5bb189",
+        "revCount": 4171,
+        "type": "git",
+        "url": "file:///home/delroth/work/hydra-lix"
+      },
+      "original": {
+        "type": "git",
+        "url": "file:///home/delroth/work/hydra-lix"
+      }
+    },
+    "nix": {
+      "inputs": {
+        "flake-compat": "flake-compat_2",
+        "nix2container": "nix2container",
+        "nixpkgs": [
+          "hydra",
+          "nixpkgs"
+        ],
+        "nixpkgs-regression": "nixpkgs-regression",
+        "pre-commit-hooks": "pre-commit-hooks"
+      },
+      "locked": {
+        "lastModified": 1719211568,
+        "narHash": "sha256-oIgmvhe3CV/36LC0KXgqWnKXma39wabks8U9JBMDfO4=",
+        "ref": "refs/heads/main",
+        "rev": "4c3d93611f2848c56ebc69c85f2b1e18001ed3c7",
+        "revCount": 15877,
+        "type": "git",
+        "url": "https://git@git.lix.systems/lix-project/lix"
+      },
+      "original": {
+        "type": "git",
+        "url": "https://git@git.lix.systems/lix-project/lix"
+      }
+    },
+    "nix2container": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1712990762,
+        "narHash": "sha256-hO9W3w7NcnYeX8u8cleHiSpK2YJo7ecarFTUlbybl7k=",
+        "owner": "nlewo",
+        "repo": "nix2container",
+        "rev": "20aad300c925639d5d6cbe30013c8357ce9f2a2e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nlewo",
+        "repo": "nix2container",
+        "type": "github"
+      }
+    },
     "nixpkgs": {
       "locked": {
         "lastModified": 1718870667,
@@ -70,9 +214,47 @@
         "type": "github"
       }
     },
+    "nixpkgs-regression": {
+      "locked": {
+        "lastModified": 1643052045,
+        "narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2",
+        "type": "github"
+      }
+    },
+    "pre-commit-hooks": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1712055707,
+        "narHash": "sha256-4XLvuSIDZJGS17xEwSrNuJLL7UjDYKGJSbK1WWX2AK8=",
+        "owner": "cachix",
+        "repo": "git-hooks.nix",
+        "rev": "e35aed5fda3cc79f88ed7f1795021e559582093a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "repo": "git-hooks.nix",
+        "type": "github"
+      }
+    },
     "root": {
       "inputs": {
+        "agenix": "agenix",
         "colmena": "colmena",
+        "hydra": "hydra",
+        "lix": [
+          "hydra",
+          "nix"
+        ],
         "nixpkgs": "nixpkgs"
       }
     },
@@ -91,6 +273,21 @@
         "repo": "nixpkgs",
         "type": "github"
       }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
     }
   },
   "root": "root",

flake.nix

@@ -4,17 +4,39 @@
   inputs = {
     nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
 
+    agenix.url = "github:ryantm/agenix";
+    agenix.inputs.nixpkgs.follows = "nixpkgs";
+
     colmena.url = "github:zhaofengli/colmena";
     colmena.inputs.nixpkgs.follows = "nixpkgs";
+
+    #hydra.url = "git+https://git.lix.systems/lix-project/hydra.git";
+    hydra.url = "/home/delroth/work/hydra-lix";
+    hydra.inputs.nixpkgs.follows = "nixpkgs";
+
+    lix.follows = "hydra/nix";
   };
 
   outputs = { nixpkgs, ... } @ inputs: {
     colmena = {
       meta.nixpkgs = import nixpkgs {
         system = "x86_64-linux";
+        overlays = [
+          inputs.hydra.overlays.default
+          inputs.lix.overlays.default
+        ];
       };
+      meta.specialArgs.inputs = inputs;
+
       bagel-box = {
-        imports = [ ./hosts/bagel-box ];
+        imports = [
+          inputs.agenix.nixosModules.default
+          inputs.hydra.nixosModules.hydra
+
+          ./services
+
+          ./hosts/bagel-box
+        ];
       };
     };
   };

hosts/bagel-box/default.nix

@@ -34,6 +34,16 @@
     firewall.allowPing = true;
   };
 
+  bagel.services = {
+    postgres.enable = true;
+
+    hydra.enable = true;
+    hydra.dbi = "dbi:Pg:dbname=hydra;user=hydra";
+  };
+
+  security.acme.acceptTerms = true;
+  security.acme.defaults.email = "bagel@delroth.net";
+
   services.openssh.enable = true;
   users.users.root.openssh.authorizedKeys.keys = [
     # delroth
@@ -46,5 +56,5 @@
 
   ];
 
-  deployment.targetHost = "2001:bc8:38ee:100:100::1";
+  deployment.targetHost = "bagel-box.delroth.net";
 }

secrets.nix

@@ -0,0 +1,16 @@
+let
+  keys = import common/ssh-keys.nix;
+
+  commonKeys = keys.users.delroth;
+
+  secrets = with keys; {
+    hydra-s3-credentials = [ machines.bagel-box ];
+    hydra-ssh-key-priv = [ machines.bagel-box ];
+  };
+in
+  builtins.listToAttrs (
+    map (secretName: {
+      name = "secrets/${secretName}.age";
+      value.publicKeys = secrets."${secretName}" ++ commonKeys;
+    }) (builtins.attrNames secrets)
+  )

secrets/hydra-s3-credentials.age

@@ -0,0 +1,8 @@
+age-encryption.org/v1
+-> ssh-ed25519 zI09CQ IfOmA+uPS3mNQHx/8XG6Hh+GLsfUUXQPA9x6+9Aw7jg
+5iNgA/ImRbbEYgMysQtj4sYpJfZMtj79Yj+41bckrj4
+-> ssh-ed25519 K3b7BA wtps2j28He4oR5d/rCTNy7INSq0xlm27YO6h5ANf7Xs
+YdiMBtKw6G+NiqwaN3jAugDT1Q0zo6Cvjiph6zkIUMg
+--- xAU32CtSvaWLKOKwh9dv97ZWCot4eeMO1+0RsQo8hIA
+ˆsÁ°¬Cw
+LYڹѩjС&‰õÙNøhÿÛNê”	 Ã>àÎ	kßâNÊÛO_ÿJòþ»Œªå<1E>›aˆ¢àÜ4IÑûºìtÓtÐÙK?RÚÆWX¾4Iþ–&)“<>Á^2Þ NÚüvGvFÀŒÐbDMªÄ˜(k(Aèº\V&kaF•'<27>´‰:a%Œk7Êíí!…9Q¾oÈ®k·Dïí’ñ

services/default.nix

@@ -0,0 +1,6 @@
+{
+  imports = [
+    ./hydra
+    ./postgres
+  ];
+}

services/hydra/default.nix

@@ -0,0 +1,127 @@
+{ config, lib, pkgs, ... }:
+
+let
+  cfg = config.bagel.services.hydra;
+
+  narCacheDir = "/var/cache/hydra/nar-cache";
+  port = 3000;
+
+  mkCacheSettings = settings: builtins.concatStringsSep "&" (
+    lib.mapAttrsToList (k: v: "${k}=${v}") settings
+  );
+in {
+  options.bagel.services.hydra = with lib; {
+    enable = mkEnableOption "Hydra coordinator";
+
+    dbi = mkOption {
+      type = types.str;
+      description = "DBI connection string for the Hydra postgres database";
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    age.secrets.hydra-s3-credentials.file = ../../secrets/hydra-s3-credentials.age;
+
+    age.secrets.hydra-ssh-key-priv.owner = "hydra-queue-runner";
+    age.secrets.hydra-ssh-key-priv.file = ../../secrets/hydra-ssh-key-priv.age;
+
+    systemd.tmpfiles.rules = [
+      "d /var/cache/hydra 0755 hydra hydra -  -"
+      "d ${narCacheDir}   0755 hydra hydra 1d -"
+    ];
+
+    # XXX: Otherwise services.hydra-dev overwrites it to only hydra-queue-runner...
+    #
+    # Can be removed once this is added to some common config template.
+    nix.settings.trusted-users = [ "root" "@wheel" ];
+
+    services.hydra-dev = {
+      enable = true;
+
+      listenHost = "localhost";
+      port = port;
+      dbi = cfg.dbi;
+
+      hydraURL = "https://hydra.bagel.delroth.net";
+      useSubstitutes = false;
+
+      notificationSender = "bagel@delroth.net";
+
+      # XXX: hydra overlay sets pkgs.hydra, but hydra's nixos module uses
+      # pkgs.hydra_unstable...
+      package = pkgs.hydra;
+
+      buildMachinesFiles = [
+        (pkgs.writeText "hydra-builders.conf" ''
+          ssh://bagel-builder@epyc.infra.newtype.fr x86_64-linux ${config.age.secrets.hydra-ssh-key-priv.path} 8 1 big-parallel,kvm,nixos-test - c3NoLWVkMjU1MTkgQUFBQUMzTnphQzFsWkRJMU5URTVBQUFBSUJwcFBwKzhsdDFSTDNodW5aaGlXRUUvY1laaHJXYjFzaVhKVWpiU2l6Rzggcm9vdEBlcHljCg==
+        '')
+      ];
+
+      extraConfig = ''
+        store_uri = s3://bagel-cache?${mkCacheSettings {
+          endpoint = "s3.delroth.net";
+          region = "garage";
+
+          #secret-key = "TODO";
+
+          compression = "zstd";
+          log-compression = "br";
+          ls-compression = "br";
+
+          write-nar-listing = "1";
+        }}
+
+        server_store_uri = https://bagel-cache.s3-web.delroth.net?local-nar-cache=${narCacheDir}
+        binary_cache_public_url = https://bagel-cache.s3-web.delroth.net
+        log_prefix = https://bagel-cache.s3-web.delroth.net
+
+        upload_logs_to_binary_cache = true
+
+        evaluator_workers = 4
+        evaluator_max_memory_size = 4096
+        max_concurrent_evals = 1
+
+        allow_import_from_derivation = false
+
+        max_output_size = ${builtins.toString (3 * 1024 * 1024 * 1024)}
+        max_db_connections = 100
+      '';
+    };
+
+    systemd.services.hydra-queue-runner.serviceConfig.EnvironmentFile =
+      config.age.secrets.hydra-s3-credentials.path;
+
+    services.nginx = {
+      enable = true;
+      enableReload = true;
+
+      recommendedBrotliSettings = true;
+      recommendedGzipSettings = true;
+      recommendedOptimisation = true;
+      recommendedProxySettings = true;
+      recommendedTlsSettings = true;
+      recommendedZstdSettings = true;
+
+      proxyTimeout = "900s";
+
+      appendConfig = ''
+        worker_processes auto;
+      '';
+
+      virtualHosts."hydra.bagel.delroth.net" = {
+        forceSSL = true;
+        enableACME = true;
+
+        locations."/" = {
+          proxyPass = "http://127.0.0.1:${builtins.toString port}";
+        };
+
+        locations."/static/" = {
+          alias = "${config.services.hydra-dev.package}/libexec/hydra/root/static/";
+        };
+      };
+    };
+
+    networking.firewall.allowedTCPPorts = [ 80 443 ];
+  };
+}

services/postgres/default.nix

@@ -0,0 +1,48 @@
+{ config, lib, pkgs, ... }:
+
+let
+  cfg = config.bagel.services.postgres;
+
+  dataDir = "/var/db/postgresql/16";
+in {
+  options.bagel.services.postgres = with lib; {
+    enable = mkEnableOption "PostgreSQL server";
+  };
+
+  config = lib.mkIf cfg.enable {
+    systemd.tmpfiles.rules = [
+      "d /var/db 0755 root root - -"
+      "d /var/db/postgresql 0770 postgres postgres - -"
+      "d ${dataDir} 0770 postgres postgres - -"
+    ];
+
+    services.postgresql = {
+      enable = true;
+      package = pkgs.postgresql_16;
+      dataDir = dataDir;
+
+      # TODO: Where to put this to properly couple things? It doesn't belong
+      # here, but using it in services/hydra would require running on
+      # localhost. Probably needs to be replaced with some different way of
+      # ensuring the DB/user exist.
+      ensureDatabases = [ "hydra" ];
+      ensureUsers = [
+        {
+          name = "hydra";
+          ensureDBOwnership = true;
+        }
+      ];
+      identMap = ''
+        hydra-users hydra hydra
+        hydra-users hydra-queue-runner hydra
+        hydra-users hydra-www hydra
+        hydra-users root hydra
+        # The postgres user is used to create the pg_trgm extension for the hydra database
+        hydra-users postgres postgres
+      '';
+      authentication = ''
+        local hydra all ident map=hydra-users
+      '';
+    };
+  };
+}