buildbot_nix/binary_cache.py

@@ -0,0 +1,12 @@
+from dataclasses import dataclass
+
+@dataclass
+class S3BinaryCacheConfig:
+    region: str
+    bucket: str
+    endpoint: str
+    profile: str
+
+class LocalSigner:
+    def __init__(self, keyfile: str):
+        self.keyfile = keyfile

buildbot_nix/worker.py

@@ -22,8 +22,14 @@ class WorkerConfig:
         .read_text()
         .rstrip("\r\n")
     )
-    worker_count: int = int(
+    worker_arch_list: dict[str, int] = field(
-        os.environ.get("WORKER_COUNT", str(multiprocessing.cpu_count())),
+        default_factory=lambda: dict(other=1) | {
+            arch: int(count)
+            for arch, count in (
+                e.split("=")
+                for e in os.environ.get("WORKER_ARCH_LIST", "").split(",")
+            )
+        },
     )
     buildbot_dir: Path = field(
         default_factory=lambda: Path(require_env("BUILDBOT_DIR"))
@@ -34,13 +40,14 @@ class WorkerConfig:
 def setup_worker(
     application: components.Componentized,
     builder_id: int,
+    arch: str,
     config: WorkerConfig,
 ) -> None:
-    basedir = config.buildbot_dir.parent / f"{config.buildbot_dir.name}-{builder_id:03}"
+    basedir = config.buildbot_dir.parent / f"{config.buildbot_dir.name}-{builder_id:03}/{arch}"
     basedir.mkdir(parents=True, exist_ok=True, mode=0o700)
 
     hostname = socket.gethostname()
-    workername = f"{hostname}-{builder_id:03}"
+    workername = f"{hostname}-{builder_id:03}-{arch}"
     keepalive = 600
     umask = None
     maxdelay = 300
@@ -66,8 +73,9 @@ def setup_worker(
 
 
 def setup_workers(application: components.Componentized, config: WorkerConfig) -> None:
-    for i in range(config.worker_count):
+    for arch, jobs in config.worker_arch_list.items():
-        setup_worker(application, i, config)
+        for i in range(jobs):
+            setup_worker(application, i, arch, config)
 
 
 # note: this line is matched against to check that this is a worker

examples/default.nix

@@ -46,14 +46,6 @@ in
           # optional nix-eval-jobs settings
           # evalWorkerCount = 8; # limit number of concurrent evaluations
           # evalMaxMemorySize = "2048"; # limit memory usage per evaluation
-
-          # optional cachix
-          #cachix = {
-          #  name = "my-cachix";
-          #  # One of the following is required:
-          #  signingKey = "/var/lib/secrets/cachix-key";
-          #  authToken = "/var/lib/secrets/cachix-token";
-          #};
         };
       })
       buildbot-nix.nixosModules.buildbot-master

flake.nix

@@ -20,7 +20,7 @@
         ] ++ inputs.nixpkgs.lib.optional (inputs.treefmt-nix ? flakeModule) ./nix/treefmt/flake-module.nix;
         systems = [ "x86_64-linux" ];
         flake = {
-          nixosModules.buildbot-master = ./nix/master.nix;
+          nixosModules.buildbot-coordinator = ./nix/coordinator.nix;
           nixosModules.buildbot-worker = ./nix/worker.nix;
 
           nixosConfigurations =

nix/coordinator.nix

@@ -4,82 +4,25 @@
 , ...
 }:
 let
-  cfg = config.services.buildbot-nix.master;
+  cfg = config.services.buildbot-nix.coordinator;
 in
 {
   options = {
-    services.buildbot-nix.master = {
+    services.buildbot-nix.coordinator = {
-      enable = lib.mkEnableOption "buildbot-master";
+      enable = lib.mkEnableOption "buildbot-coordinator";
       dbUrl = lib.mkOption {
         type = lib.types.str;
         default = "postgresql://@/buildbot";
         description = "Postgresql database url";
       };
-      cachix = {
-        name = lib.mkOption {
-          type = lib.types.nullOr lib.types.str;
-          default = null;
-          description = "Cachix name";
-        };
-
-        signingKeyFile = lib.mkOption {
-          type = lib.types.nullOr lib.types.path;
-          default = null;
-          description = "Cachix signing key";
-        };
-
-        authTokenFile = lib.mkOption {
-          type = lib.types.nullOr lib.types.str;
-          default = null;
-          description = "Cachix auth token";
-        };
-      };
-      github = {
-        tokenFile = lib.mkOption {
-          type = lib.types.path;
-          description = "Github token file";
-        };
-        webhookSecretFile = lib.mkOption {
-          type = lib.types.path;
-          description = "Github webhook secret file";
-        };
-        oauthSecretFile = lib.mkOption {
-          type = lib.types.path;
-          description = "Github oauth secret file";
-        };
-        # TODO: make this an option
-        # https://github.com/organizations/numtide/settings/applications
-        # Application name: BuildBot
-        # Homepage URL: https://buildbot.numtide.com
-        # Authorization callback URL: https://buildbot.numtide.com/auth/login
-        # oauth_token:  2516248ec6289e4d9818122cce0cbde39e4b788d
-        oauthId = lib.mkOption {
-          type = lib.types.str;
-          description = "Github oauth id. Used for the login button";
-        };
-        # Most likely you want to use the same user as for the buildbot
-        user = lib.mkOption {
-          type = lib.types.str;
-          description = "Github user that is used for the buildbot";
-        };
-        admins = lib.mkOption {
-          type = lib.types.listOf lib.types.str;
-          default = [ ];
-          description = "Users that are allowed to login to buildbot, trigger builds and change settings";
-        };
-        topic = lib.mkOption {
-          type = lib.types.nullOr lib.types.str;
-          default = "build-with-buildbot";
-          description = ''
-            Projects that have this topic will be built by buildbot.
-            If null, all projects that the buildbot github user has access to, are built.
-          '';
-        };
-      };
       workersFile = lib.mkOption {
         type = lib.types.path;
         description = "File containing a list of nix workers";
       };
+      oauth2SecretFile = lib.mkOption {
+        type = lib.types.path;
+        description = "File containing an OAuth 2 client secret";
+      };
       buildSystems = lib.mkOption {
         type = lib.types.listOf lib.types.str;
         default = [ pkgs.hostPlatform.system ];
@@ -114,6 +57,41 @@ in
         default = null;
         example = "/var/www/buildbot/nix-outputs";
       };
+
+      signingKeyFile = lib.mkOption {
+        type = lib.types.nullOr lib.types.path;
+        description = "A path to a Nix signing key";
+        default = null;
+        example = "/run/agenix.d/signing-key";
+      };
+
+      binaryCache = {
+        enable = lib.mkEnableOption " binary cache upload to a S3 bucket";
+        profileCredentialsFile = lib.mkOption {
+          type = lib.types.nullOr lib.types.path;
+          description = "A path to the various AWS profile credentials related to the S3 bucket containing a profile named `default`";
+          default = null;
+          example = "/run/agenix.d/aws-profile";
+        };
+        bucket = lib.mkOption {
+          type = lib.types.nullOr lib.types.str;
+          description = "Bucket where to store the data";
+          default = null;
+          example = "lix-cache";
+        };
+        endpoint = lib.mkOption {
+          type = lib.types.nullOr lib.types.str;
+          description = "Endpoint for the S3 server";
+          default = null;
+          example = "s3.lix.systems";
+        };
+        region = lib.mkOption {
+          type = lib.types.nullOr lib.types.str;
+          description = "Region for the S3 bucket";
+          default = null;
+          example = "garage";
+        };
+      };
     };
   };
   config = lib.mkIf cfg.enable {
@@ -126,13 +104,6 @@ in
       isSystemUser = true;
     };
 
-    assertions = [
-      {
-        assertion = cfg.cachix.name != null -> cfg.cachix.signingKeyFile != null || cfg.cachix.authTokenFile != null;
-        message = "if cachix.name is provided, then cachix.signingKeyFile and cachix.authTokenFile must be set";
-      }
-    ];
-
     services.buildbot-master = {
       enable = true;
 
@@ -144,30 +115,29 @@ in
       home = "/var/lib/buildbot";
       extraImports = ''
         from datetime import timedelta
-        from buildbot_nix import GithubConfig, NixConfigurator, CachixConfig
+        from buildbot_nix import GerritNixConfigurator
       '';
       configurators = [
         ''
           util.JanitorConfigurator(logHorizon=timedelta(weeks=4), hour=12, dayOfWeek=6)
         ''
         ''
-          NixConfigurator(
+          GerritNixConfigurator(
-              github=GithubConfig(
+              "gerrit.lix.systems",
-                  oauth_id=${builtins.toJSON cfg.github.oauthId},
+              "buildbot",
-                  admins=${builtins.toJSON cfg.github.admins},
+              2022,
-                  buildbot_user=${builtins.toJSON cfg.github.user},
+              "/var/lib/buildbot/master/id_gerrit",
-                  topic=${builtins.toJSON cfg.github.topic},
-              ),
-              cachix=${if cfg.cachix.name == null then "None" else "CachixConfig(
-                  name=${builtins.toJSON cfg.cachix.name},
-                  signing_key_secret_name=${if cfg.cachix.signingKeyFile != null then builtins.toJSON "cachix-signing-key" else "None"},
-                  auth_token_secret_name=${if cfg.cachix.authTokenFile != null then builtins.toJSON "cachix-auth-token" else "None"},
-              )"},
               url=${builtins.toJSON config.services.buildbot-master.buildbotUrl},
               nix_eval_max_memory_size=${builtins.toJSON cfg.evalMaxMemorySize},
               nix_eval_worker_count=${if cfg.evalWorkerCount == null then "None" else builtins.toString cfg.evalWorkerCount},
               nix_supported_systems=${builtins.toJSON cfg.buildSystems},
               outputs_path=${if cfg.outputsPath == null then "None" else builtins.toJSON cfg.outputsPath},
+              # Signing key file must be available on the workers and readable.
+              signing_keyfile=${if cfg.signingKeyFile == null then "None" else builtins.toJSON cfg.signingKeyFile},
+              binary_cache_config=${if (!cfg.binaryCache.enable) then "None" else builtins.toJSON {
+                inherit (cfg.binaryCache) bucket region endpoint;
+                profile = "default";
+              }}
           )
         ''
       ];
@@ -177,31 +147,41 @@ in
           hasSSL = host.forceSSL || host.addSSL;
         in
         "${if hasSSL then "https" else "http"}://${cfg.domain}/";
-      dbUrl = config.services.buildbot-nix.master.dbUrl;
+      dbUrl = cfg.dbUrl;
       pythonPackages = ps: [
         ps.requests
         ps.treq
         ps.psycopg2
         (ps.toPythonModule pkgs.buildbot-worker)
-        pkgs.buildbot-plugins.www-react
+        pkgs.buildbot-plugins.www
         (pkgs.python3.pkgs.callPackage ../default.nix { })
       ];
     };
 
+    # TODO(raito): we assume worker runs on coordinator. please clean up this later.
+    systemd.services.buildbot-worker.serviceConfig.Environment =
+      lib.mkIf cfg.binaryCache.enable (
+        let
+          awsConfigFile = pkgs.writeText "config.ini"  ''
+            [default]
+            region = ${cfg.binaryCache.region}
+            endpoint_url = ${cfg.binaryCache.endpoint}
+          '';
+        in
+        [
+          "AWS_CONFIG_FILE=${awsConfigFile}"
+          "AWS_SHARED_CREDENTIALS_FILE=${cfg.binaryCache.profileCredentialsFile}"
+        ]
+      );
+
     systemd.services.buildbot-master = {
       after = [ "postgresql.service" ];
       serviceConfig = {
         # in master.py we read secrets from $CREDENTIALS_DIRECTORY
         LoadCredential = [
-          "github-token:${cfg.github.tokenFile}"
-          "github-webhook-secret:${cfg.github.webhookSecretFile}"
-          "github-oauth-secret:${cfg.github.oauthSecretFile}"
           "buildbot-nix-workers:${cfg.workersFile}"
-        ]
+          "buildbot-oauth2-secret:${cfg.oauth2SecretFile}"
-        ++ lib.optional (cfg.cachix.signingKeyFile != null)
+        ];
-          "cachix-signing-key:${builtins.toString cfg.cachix.signingKeyFile}"
-        ++ lib.optional (cfg.cachix.authTokenFile != null)
-          "cachix-auth-token:${builtins.toString cfg.cachix.authTokenFile}";
       };
     };
 
@@ -215,16 +195,20 @@ in
     };
 
     services.nginx.enable = true;
-    services.nginx.virtualHosts.${cfg.domain} = {
+    services.nginx.virtualHosts.${cfg.domain} = 
+    let
+      port = config.services.buildbot-master.port;
+    in
+    {
       locations = {
-        "/".proxyPass = "http://127.0.0.1:${builtins.toString config.services.buildbot-master.port}/";
+        "/".proxyPass = "http://127.0.0.1:${builtins.toString port}/";
         "/sse" = {
-          proxyPass = "http://127.0.0.1:${builtins.toString config.services.buildbot-master.port}/sse";
+          proxyPass = "http://127.0.0.1:${builtins.toString port}/sse";
           # proxy buffering will prevent sse to work
           extraConfig = "proxy_buffering off;";
         };
         "/ws" = {
-          proxyPass = "http://127.0.0.1:${builtins.toString config.services.buildbot-master.port}/ws";
+          proxyPass = "http://127.0.0.1:${builtins.toString port}/ws";
           proxyWebsockets = true;
           # raise the proxy timeout for the websocket
           extraConfig = "proxy_read_timeout 6000s;";
@@ -234,11 +218,8 @@ in
       };
     };
 
-    systemd.tmpfiles.rules = [
+    systemd.tmpfiles.rules = lib.optional (cfg.outputsPath != null)
-      # delete legacy gcroot location, can be dropped after 2024-06-01
+      # Allow buildbot-coordinator to write to this directory
-      "R /var/lib/buildbot-worker/gcroot - - - - -"
-    ] ++ lib.optional (cfg.outputsPath != null)
-      # Allow buildbot-master to write to this directory
       "d ${cfg.outputsPath} 0755 buildbot buildbot - -";
   };
 }

nix/worker.nix

@@ -19,15 +19,19 @@ in
         defaultText = "pkgs.buildbot-worker";
         description = "The buildbot-worker package to use.";
       };
-      masterUrl = lib.mkOption {
+      coordinatorUrl = lib.mkOption {
         type = lib.types.str;
         default = "tcp:host=localhost:port=9989";
-        description = "The buildbot master url.";
+        description = "The buildbot coordinator url.";
       };
       workerPasswordFile = lib.mkOption {
         type = lib.types.path;
         description = "The buildbot worker password file.";
       };
+      workerArchitectures = lib.mkOption {
+        type = lib.types.attrsOf lib.types.int;
+        description = "Nix `system`s the worker should feel responsible for.";
+      };
     };
   };
   config = lib.mkIf cfg.enable {
@@ -54,15 +58,17 @@ in
       after = [ "network.target" "buildbot-master.service" ];
       wantedBy = [ "multi-user.target" ];
       path = [
-        pkgs.cachix
         pkgs.git
         pkgs.openssh
         pkgs.nix
         pkgs.nix-eval-jobs
       ];
       environment.PYTHONPATH = "${python.withPackages (_: [cfg.package])}/${python.sitePackages}";
-      environment.MASTER_URL = cfg.masterUrl;
+      environment.MASTER_URL = cfg.coordinatorUrl;
       environment.BUILDBOT_DIR = buildbotDir;
+      environment.WORKER_ARCH_LIST =
+        lib.concatStringsSep ","
+          (lib.mapAttrsToList (arch: jobs: "${arch}=${toString jobs}") cfg.workerArchitectures);
 
       serviceConfig = {
         # We rather want the CI job to fail on OOM than to have a broken buildbot worker.
@@ -70,7 +76,9 @@ in
         OOMPolicy = "continue";
 
         LoadCredential = [ "worker-password-file:${cfg.workerPasswordFile}" ];
-        Environment = [ "WORKER_PASSWORD_FILE=%d/worker-password-file" ];
+        Environment = [
+          "WORKER_PASSWORD_FILE=%d/worker-password-file"
+        ];
         Type = "simple";
         User = "buildbot-worker";
         Group = "buildbot-worker";