buildbot-nix/nix/coordinator.nix

{ config
, pkgs
, lib
, ...
}:
let
  cfg = config.services.buildbot-nix.coordinator;
in
{
  options = {
    services.buildbot-nix.coordinator = {
      enable = lib.mkEnableOption "buildbot-coordinator";
      dbUrl = lib.mkOption {
        type = lib.types.str;
        default = "postgresql://@/buildbot";
        description = "Postgresql database url";
      };
      cachix = {
        name = lib.mkOption {
          type = lib.types.nullOr lib.types.str;
          default = null;
          description = "Cachix name";
        };

        signingKeyFile = lib.mkOption {
          type = lib.types.nullOr lib.types.path;
          default = null;
          description = "Cachix signing key";
        };

        authTokenFile = lib.mkOption {
          type = lib.types.nullOr lib.types.str;
          default = null;
          description = "Cachix auth token";
        };
      };
      workersFile = lib.mkOption {
        type = lib.types.path;
        description = "File containing a list of nix workers";
      };
      oauth2SecretFile = lib.mkOption {
        type = lib.types.path;
        description = "File containing an OAuth 2 client secret";
      };
      buildSystems = lib.mkOption {
        type = lib.types.listOf lib.types.str;
        default = [ pkgs.hostPlatform.system ];
        description = "Systems that we will be build";
      };
      evalMaxMemorySize = lib.mkOption {
        type = lib.types.str;
        default = "2048";
        description = ''
          Maximum memory size for nix-eval-jobs (in MiB) per
          worker. After the limit is reached, the worker is
          restarted.
        '';
      };
      evalWorkerCount = lib.mkOption {
        type = lib.types.nullOr lib.types.int;
        default = null;
        description = ''
          Number of nix-eval-jobs worker processes. If null, the number of cores is used.
          If you experience memory issues (buildbot-workers going out-of-memory), you can reduce this number.
        '';
      };
      domain = lib.mkOption {
        type = lib.types.str;
        description = "Buildbot domain";
        example = "buildbot.numtide.com";
      };

      outputsPath = lib.mkOption {
        type = lib.types.nullOr lib.types.path;
        description = "Path where we store the latest build store paths names for nix attributes as text files. This path will be exposed via nginx at \${domain}/nix-outputs";
        default = null;
        example = "/var/www/buildbot/nix-outputs";
      };
    };
  };
  config = lib.mkIf cfg.enable {
    # By default buildbot uses a normal user, which is not a good default, because
    # we grant normal users potentially access to other resources. Also
    # we don't to be able to ssh into buildbot.

    users.users.buildbot = {
      isNormalUser = lib.mkForce false;
      isSystemUser = true;
    };

    assertions = [
      {
        assertion = cfg.cachix.name != null -> cfg.cachix.signingKeyFile != null || cfg.cachix.authTokenFile != null;
        message = "if cachix.name is provided, then cachix.signingKeyFile and cachix.authTokenFile must be set";
      }
    ];

    services.buildbot-master = {
      enable = true;

      # disable example workers from nixpkgs
      builders = [ ];
      schedulers = [ ];
      workers = [ ];

      home = "/var/lib/buildbot";
      extraImports = ''
        from datetime import timedelta
        from buildbot_nix import GerritNixConfigurator, CachixConfig
      '';
      configurators = [
        ''
          util.JanitorConfigurator(logHorizon=timedelta(weeks=4), hour=12, dayOfWeek=6)
        ''
        ''
          GerritNixConfigurator(
              "gerrit.lix.systems",
              "buildbot",
              2022,
              "/var/lib/buildbot/master/id_gerrit",
              url=${builtins.toJSON config.services.buildbot-master.buildbotUrl},
              nix_eval_max_memory_size=${builtins.toJSON cfg.evalMaxMemorySize},
              nix_eval_worker_count=${if cfg.evalWorkerCount == null then "None" else builtins.toString cfg.evalWorkerCount},
              nix_supported_systems=${builtins.toJSON cfg.buildSystems},
              outputs_path=${if cfg.outputsPath == null then "None" else builtins.toJSON cfg.outputsPath},
          )
        ''
      ];
      buildbotUrl =
        let
          host = config.services.nginx.virtualHosts.${cfg.domain};
          hasSSL = host.forceSSL || host.addSSL;
        in
        "${if hasSSL then "https" else "http"}://${cfg.domain}/";
      dbUrl = cfg.dbUrl;
      pythonPackages = ps: [
        ps.requests
        ps.treq
        ps.psycopg2
        (ps.toPythonModule pkgs.buildbot-worker)
        pkgs.buildbot-plugins.www-react
        (pkgs.python3.pkgs.callPackage ../default.nix { })
      ];
    };

    systemd.services.buildbot-master = {
      after = [ "postgresql.service" ];
      serviceConfig = {
        # in master.py we read secrets from $CREDENTIALS_DIRECTORY
        LoadCredential = [
          "buildbot-nix-workers:${cfg.workersFile}"
          "buildbot-oauth2-secret:${cfg.oauth2SecretFile}"
        ]
        ++ lib.optional (cfg.cachix.signingKeyFile != null)
          "cachix-signing-key:${builtins.toString cfg.cachix.signingKeyFile}"
        ++ lib.optional (cfg.cachix.authTokenFile != null)
          "cachix-auth-token:${builtins.toString cfg.cachix.authTokenFile}";
      };
    };

    services.postgresql = {
      enable = true;
      ensureDatabases = [ "buildbot" ];
      ensureUsers = [{
        name = "buildbot";
        ensureDBOwnership = true;
      }];
    };

    services.nginx.enable = true;
    services.nginx.virtualHosts.${cfg.domain} = 
    let
      port = config.services.buildbot-master.port;
    in
    {
      locations = {
        "/".proxyPass = "http://127.0.0.1:${builtins.toString port}/";
        "/sse" = {
          proxyPass = "http://127.0.0.1:${builtins.toString port}/sse";
          # proxy buffering will prevent sse to work
          extraConfig = "proxy_buffering off;";
        };
        "/ws" = {
          proxyPass = "http://127.0.0.1:${builtins.toString port}/ws";
          proxyWebsockets = true;
          # raise the proxy timeout for the websocket
          extraConfig = "proxy_read_timeout 6000s;";
        };
      } // lib.optionalAttrs (cfg.outputsPath != null) {
        "/nix-outputs".root = cfg.outputsPath;
      };
    };

    systemd.tmpfiles.rules = lib.optional (cfg.outputsPath != null)
      # Allow buildbot-coordinator to write to this directory
      "d ${cfg.outputsPath} 0755 buildbot buildbot - -";
  };
}