buildbot-nix/nix/coordinator.nix

{ config
, options
, pkgs
, lib
, ...
}:
let
  inherit (lib) filterAttrs;
  cfg = config.services.buildbot-nix.coordinator;
  debuggingManhole = if cfg.debugging.enable then
  "manhole.TelnetManhole(${toString cfg.debugging.port}, 'admin', 'admin')"
  else "None";
in
{
  options = {
    services.buildbot-nix.coordinator = {
      enable = lib.mkEnableOption "buildbot-coordinator";
      dbUrl = lib.mkOption {
        type = lib.types.str;
        default = "postgresql://@/buildbot";
        description = "Postgresql database url";
      };

      workersFile = lib.mkOption {
        type = lib.types.path;
        description = "File containing a list of nix workers";
      };

      buildMachines = lib.mkOption {
        type = options.nix.buildMachines.type;
        description = "List of local remote builders machines associated to that Buildbot instance";
      };

      debugging = {
        enable = lib.mkEnableOption "manhole's buildbot debugging on localhost using `admin:admin`";
        port = lib.mkOption {
          type = lib.types.port;
          default = 15000;
        };
      };

      oauth2 = {
        name = lib.mkOption {
          type = lib.types.str;
          description = "Name of the OAuth2 login method";
        };

        icon = lib.mkOption {
          type = lib.types.str;
          description = "FontAwesome string for the icon associated to the OAuth2 login";
          default = "fa-login";
          example = "fa-login";
        };

        clientId = lib.mkOption {
          type = lib.types.str;
          description = "Client ID for the OAuth2 authentication";
        };

        clientSecretFile = lib.mkOption {
          type = lib.types.path;
          description = "Path to a file containing an OAuth 2 client secret";
        };

        resourceEndpoint = lib.mkOption {
          type = lib.types.str;
          description = "URL to the OAuth 2 resource";
          example = "https://identity.lix.systems";
        };

        authUri = lib.mkOption {
          type = lib.types.str;
          description = "Authentication URI";
          example = "https://identity.lix.systems/realms/lix-project/protocol/openid-connect/auth";
        };

        tokenUri = lib.mkOption {
          type = lib.types.str;
          description = "Token URI";
          example = "https://identity.lix.systems/realms/lix-project/protocol/openid-connect/token";
        };
        userinfoUri = lib.mkOption {
          type = lib.types.str;
          description = "User info URI";
          example = "https://identity.lix.systems/realms/lix-project/protocol/openid-connect/token";
        };
      };
      buildSystems = lib.mkOption {
        type = lib.types.listOf lib.types.str;
        default = [ pkgs.hostPlatform.system ];
        description = "Systems that we will be build";
      };
      evalMaxMemorySize = lib.mkOption {
        type = lib.types.str;
        default = "2048";
        description = ''
          Maximum memory size for nix-eval-jobs (in MiB) per
          worker. After the limit is reached, the worker is
          restarted.
        '';
      };
      evalWorkerCount = lib.mkOption {
        type = lib.types.nullOr lib.types.int;
        default = null;
        description = ''
          Number of nix-eval-jobs worker processes. If null, the number of cores is used.
          If you experience memory issues (buildbot-workers going out-of-memory), you can reduce this number.
        '';
      };
      domain = lib.mkOption {
        type = lib.types.str;
        description = "Buildbot domain";
        example = "buildbot.numtide.com";
      };

      allowedOrigins = lib.mkOption {
        type = lib.types.listOf lib.types.str;
        description = "Allowed origins for buildbot";
        example = [ "*.mydomain.com" ];
      };

      signingKeyFile = lib.mkOption {
        type = lib.types.nullOr lib.types.path;
        description = "A path to a Nix signing key";
        default = null;
        example = "/run/agenix.d/signing-key";
      };

      prometheus = {
        enable = lib.mkEnableOption " the export of metrics in Prometheus format";
        address = lib.mkOption {
          type = lib.types.str;
          default = "";
          description = "The local IPv4 or IPv6 address to which to bind; defaults to '' represents all IPv4 addresses.";
        };
        port = lib.mkOption {
          type = lib.types.port;
          default = 9100;
          description = "A port on which the metrics endpoint will be available";
        };
      };

      gerrit = {
        domain = lib.mkOption {
          type = lib.types.str;
          description = "Domain to the Gerrit server";
          example = "gerrit.lix.systems";
        };

        username = lib.mkOption {
          type = lib.types.str;
          description = "Username to log in to the Gerrit API";
          example = "buildbot";
        };

        port = lib.mkOption {
          type = lib.types.port;
          description = "Port to log in to the Gerrit API";
          example = 2022;
        };

        privateKeyFile = lib.mkOption {
          type = lib.types.path;
          description = ''
            Path to the SSH private key to authenticate against the Gerrit API
          '';
          example = "/var/lib/buildbot/master/id_gerrit";
        };

        projects = lib.mkOption {
          type = lib.types.listOf lib.types.str;
          description = ''
            List of projects which are to check on Gerrit.
          '';
          example = [ "lix" ];
        };
      };

      binaryCache = {
        enable = lib.mkEnableOption " binary cache upload to a S3 bucket";
        profileCredentialsFile = lib.mkOption {
          type = lib.types.nullOr lib.types.path;
          description = "A path to the various AWS profile credentials related to the S3 bucket containing a profile named `default`";
          default = null;
          example = "/run/agenix.d/aws-profile";
        };
        bucket = lib.mkOption {
          type = lib.types.nullOr lib.types.str;
          description = "Bucket where to store the data";
          default = null;
          example = "lix-cache";
        };
        endpoint = lib.mkOption {
          type = lib.types.nullOr lib.types.str;
          description = "Endpoint for the S3 server";
          default = null;
          example = "s3.lix.systems";
        };
        region = lib.mkOption {
          type = lib.types.nullOr lib.types.str;
          description = "Region for the S3 bucket";
          default = null;
          example = "garage";
        };
      };
    };
  };
  config = lib.mkIf cfg.enable {
    # By default buildbot uses a normal user, which is not a good default, because
    # we grant normal users potentially access to other resources. Also
    # we don't to be able to ssh into buildbot.

    users.users.buildbot = {
      isNormalUser = lib.mkForce false;
      isSystemUser = true;
    };

    services.buildbot-master = {
      enable = true;

      # disable example workers from nixpkgs
      builders = [ ];
      schedulers = [ ];
      workers = [ ];

      home = "/var/lib/buildbot";
      extraImports = ''
        from datetime import timedelta
        from buildbot_nix import GerritNixConfigurator, read_secret_file, make_oauth2_method, OAuth2Config, assemble_secret_file_path
        from buildbot import manhole

        # TODO(raito): make me configurable from the NixOS module.
        # how?
        CustomOAuth2 = make_oauth2_method(OAuth2Config(
          name=${builtins.toJSON cfg.oauth2.name},
          faIcon=${builtins.toJSON cfg.oauth2.icon},
          resourceEndpoint=${builtins.toJSON cfg.oauth2.resourceEndpoint},
          authUri=${builtins.toJSON cfg.oauth2.authUri},
          tokenUri=${builtins.toJSON cfg.oauth2.tokenUri},
          userinfoUri=${builtins.toJSON cfg.oauth2.userinfoUri}
        ))
      '';
      configurators = [
        ''
          util.JanitorConfigurator(logHorizon=timedelta(weeks=4), hour=12, dayOfWeek=6)
        ''
        ''
          GerritNixConfigurator(
              "${cfg.gerrit.domain}",
              "${cfg.gerrit.username}",
              "${toString cfg.gerrit.port}",
              assemble_secret_file_path('buildbot-service-private-key'),
              projects=${builtins.toJSON cfg.gerrit.projects},
              allowed_origins=${builtins.toJSON cfg.allowedOrigins},
              url=${builtins.toJSON config.services.buildbot-master.buildbotUrl},
              nix_eval_max_memory_size=${builtins.toJSON cfg.evalMaxMemorySize},
              nix_eval_worker_count=${if cfg.evalWorkerCount == null then "None" else builtins.toString cfg.evalWorkerCount},
              nix_supported_systems=${builtins.toJSON cfg.buildSystems},
              prometheus_config=${if (!cfg.prometheus.enable) then "None" else builtins.toJSON {
                inherit (cfg.prometheus) address port;
              }},
              nix_builders=${builtins.toJSON (map (b: filterAttrs (n: _: n != "system") b) cfg.buildMachines)},
              # Signing key file must be available on the workers and readable.
              signing_keyfile=${if cfg.signingKeyFile == null then "None" else builtins.toJSON cfg.signingKeyFile},
              binary_cache_config=${if (!cfg.binaryCache.enable) then "None" else builtins.toJSON {
                inherit (cfg.binaryCache) bucket region endpoint;
                profile = "default";
              }},
              auth_method=CustomOAuth2(${builtins.toJSON cfg.oauth2.clientId},
                read_secret_file('buildbot-oauth2-secret'),
                autologin=True
              ),
              manhole=${debuggingManhole}
          )
        ''
      ];
      buildbotUrl =
        let
          host = config.services.nginx.virtualHosts.${cfg.domain};
          hasSSL = host.forceSSL || host.addSSL;
        in
        "${if hasSSL then "https" else "http"}://${cfg.domain}/";
      dbUrl = cfg.dbUrl;
      pythonPackages = ps: ([
        ps.requests
        ps.treq
        ps.psycopg2
        (ps.toPythonModule pkgs.buildbot-worker)
        pkgs.buildbot-plugins.www
        (pkgs.python3.pkgs.callPackage ../default.nix { })
      ] ++ lib.optional cfg.prometheus.enable (pkgs.python3.pkgs.callPackage ./prometheus-plugin.nix { })
      );
    };

    # TODO(raito): we assume worker runs on coordinator. please clean up this later.
    systemd.services.buildbot-worker.serviceConfig.Environment =
      lib.mkIf cfg.binaryCache.enable (
        let
          awsConfigFile = pkgs.writeText "config.ini"  ''
            [default]
            region = ${cfg.binaryCache.region}
            endpoint_url = ${cfg.binaryCache.endpoint}
          '';
        in
        [
          "AWS_CONFIG_FILE=${awsConfigFile}"
          "AWS_SHARED_CREDENTIALS_FILE=${cfg.binaryCache.profileCredentialsFile}"
        ]
      );

    systemd.services.buildbot-master = {
      after = [ "postgresql.service" ];
      path = [ pkgs.openssh ];
      serviceConfig = {
        # in master.py we read secrets from $CREDENTIALS_DIRECTORY
        LoadCredential = [
          "buildbot-nix-workers:${cfg.workersFile}"
          "buildbot-oauth2-secret:${cfg.oauth2.clientSecretFile}"
          "buildbot-service-private-key:${cfg.gerrit.privateKeyFile}"
        ];
      };
    };

    services.postgresql = {
      enable = true;
      ensureDatabases = [ "buildbot" ];
      ensureUsers = [{
        name = "buildbot";
        ensureDBOwnership = true;
      }];
    };

    services.nginx.enable = true;
    services.nginx.virtualHosts.${cfg.domain} = 
    let
      port = config.services.buildbot-master.port;
    in
    {
      locations = {
        "/".proxyPass = "http://127.0.0.1:${builtins.toString port}/";
        "/sse" = {
          proxyPass = "http://127.0.0.1:${builtins.toString port}/sse";
          # proxy buffering will prevent sse to work
          extraConfig = "proxy_buffering off;";
        };
        "/ws" = {
          proxyPass = "http://127.0.0.1:${builtins.toString port}/ws";
          proxyWebsockets = true;
          # raise the proxy timeout for the websocket
          extraConfig = "proxy_read_timeout 6000s;";
        };
      };
    };
  };
}