feat: binary cache

Signed-off-by: Raito Bezarius <raito@lix.systems>

buildbot_nix/__init__.py

@@ -26,6 +26,7 @@ from buildbot.process.buildstep import CANCELLED
 from buildbot.process.buildstep import EXCEPTION
 from buildbot.process.buildstep import SUCCESS
 from buildbot.process.results import worst_status
+from buildbot_nix.binary_cache import LocalSigner
 
 if TYPE_CHECKING:
     from buildbot.process.log import Log
@@ -34,6 +35,8 @@ from twisted.internet import defer, threads
 from twisted.logger import Logger
 from twisted.python.failure import Failure
 
+from .binary_cache import S3BinaryCacheConfig
+
 from .github_projects import (
     slugify_project_name,
 )
@@ -498,6 +501,8 @@ def nix_build_config(
     project: GerritProject,
     worker_names: list[str],
     outputs_path: Path | None = None,
+    signing_keyfile: str | None = None,
+    binary_cache_config: S3BinaryCacheConfig | None = None
 ) -> util.BuilderConfig:
     """Builds one nix flake attribute."""
     factory = util.BuildFactory()
@@ -528,6 +533,39 @@ def nix_build_config(
         ),
     )
 
+    if signing_keyfile is not None:
+        factory.addStep(
+            steps.ShellCommand(
+                name="Sign the store path",
+                command=[
+                    "nix",
+                    "store",
+                    "sign",
+                    "--keyfile",
+                    signing_keyfile,
+                    util.Interpolate(
+                        "%(prop:drv_path)s^*"
+                    )
+                ]
+            ),
+        )
+
+    if binary_cache_config is not None:
+        factory.addStep(
+            steps.ShellCommand(
+                name="Upload the store path to the cache",
+                command=[
+                    "nix",
+                    "copy",
+                    "--to",
+                    f"s3://{binary_cache_config.bucket}?profile={binary_cache_config.profile}&region={binary_cache_config.region}&endpoint={binary_cache_config.endpoint}",
+                    util.Property(
+                        "out_path"
+                    )
+                ]
+            )
+        )
+
     factory.addStep(
         steps.ShellCommand(
             name="Register gcroot",
@@ -586,6 +624,8 @@ def config_for_project(
     nix_eval_max_memory_size: int,
     eval_lock: util.MasterLock,
     outputs_path: Path | None = None,
+    signing_keyfile: str | None = None,
+    binary_cache_config: S3BinaryCacheConfig | None = None
 ) -> Project:
     config["projects"].append(Project(project.name))
     config["schedulers"].extend(
@@ -642,6 +682,8 @@ def config_for_project(
                 project,
                 worker_names,
                 outputs_path=outputs_path,
+                signing_keyfile=signing_keyfile,
+                binary_cache_config=binary_cache_config
             ),
         ],
     )
@@ -742,6 +784,8 @@ class GerritNixConfigurator(ConfiguratorBase):
         nix_eval_worker_count: int | None,
         nix_eval_max_memory_size: int,
         nix_workers_secret_name: str = "buildbot-nix-workers",  # noqa: S107
+        signing_keyfile: str | None = None,
+        binary_cache_config: dict[str, str] | None = None,
         outputs_path: str | None = None,
     ) -> None:
         super().__init__()
@@ -754,6 +798,11 @@ class GerritNixConfigurator(ConfiguratorBase):
         self.nix_supported_systems = nix_supported_systems
         self.gerrit_change_source = GerritChangeSource(gerrit_server, gerrit_user, gerritport=gerrit_port, identity_file=gerrit_sshkey_path)
         self.url = url
+        if binary_cache_config is not None:
+            self.binary_cache_config = S3BinaryCacheConfig(**binary_cache_config)
+        else:
+            self.binary_cache_config = None
+        self.signing_keyfile = signing_keyfile
         if outputs_path is None:
             self.outputs_path = None
         else:
@@ -786,6 +835,8 @@ class GerritNixConfigurator(ConfiguratorBase):
             self.nix_eval_max_memory_size,
             eval_lock,
             self.outputs_path,
+            signing_keyfile=self.signing_keyfile,
+            binary_cache_config=self.binary_cache_config
         )
 
         config["change_source"] = self.gerrit_change_source

buildbot_nix/binary_cache.py

@@ -0,0 +1,12 @@
+from dataclasses import dataclass
+
+@dataclass
+class S3BinaryCacheConfig:
+    region: str
+    bucket: str
+    endpoint: str
+    profile: str
+
+class LocalSigner:
+    def __init__(self, keyfile: str):
+        self.keyfile = keyfile

nix/coordinator.nix

@@ -57,6 +57,34 @@ in
         default = null;
         example = "/var/www/buildbot/nix-outputs";
       };
+
+      binaryCache = {
+        enable = lib.mkEnableOption " binary cache upload to a S3 bucket";
+        profileCredentialsFile = lib.mkOption {
+          type = lib.types.nullOr lib.types.path;
+          description = "A path to the various AWS profile credentials related to the S3 bucket containing a profile named `default`";
+          default = null;
+          example = "/run/agenix.d/aws-profile";
+        };
+        bucket = lib.mkOption {
+          type = lib.types.nullOr lib.types.str;
+          description = "Bucket where to store the data";
+          default = null;
+          example = "lix-cache";
+        };
+        endpoint = lib.mkOption {
+          type = lib.types.nullOr lib.types.str;
+          description = "Endpoint for the S3 server";
+          default = null;
+          example = "s3.lix.systems";
+        };
+        region = lib.mkOption {
+          type = lib.types.nullOr lib.types.str;
+          description = "Region for the S3 bucket";
+          default = null;
+          example = "garage";
+        };
+      };
     };
   };
   config = lib.mkIf cfg.enable {
@@ -97,6 +125,10 @@ in
               nix_eval_worker_count=${if cfg.evalWorkerCount == null then "None" else builtins.toString cfg.evalWorkerCount},
               nix_supported_systems=${builtins.toJSON cfg.buildSystems},
               outputs_path=${if cfg.outputsPath == null then "None" else builtins.toJSON cfg.outputsPath},
+              binary_cache_config=${if (!cfg.binaryCache.enable) then "None" else builtins.toJSON {
+                inherit (cfg.binaryCache) bucket region endpoint;
+                profile = "default";
+              }}
           )
         ''
       ];
@@ -117,6 +149,20 @@ in
       ];
     };
 
+    # TODO(raito): we assume worker runs on coordinator. please clean up this later.
+    systemd.services.buildbot-worker.serviceConfig.Environment =
+        let
+          awsConfigFile = pkgs.writeText "config.ini"  ''
+            [default]
+            region = ${cfg.binaryCache.region}
+            endpoint_url = ${cfg.binaryCache.endpoint}
+          '';
+        in
+        [
+          "AWS_CONFIG_FILE=${awsConfigFile}"
+          "AWS_SHARED_CREDENTIALS_FILE=${cfg.binaryCache.profileCredentialsFile}"
+        ];
+
     systemd.services.buildbot-master = {
       after = [ "postgresql.service" ];
       serviceConfig = {