From fe70af3ad68c907d9bb37f1c58792cd329a124ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 4 Nov 2023 09:19:56 +0100 Subject: [PATCH 1/3] nixos/master: make buildbot a system user --- nix/master.nix | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/nix/master.nix b/nix/master.nix index ce0756f..e79761a 100644 --- a/nix/master.nix +++ b/nix/master.nix @@ -83,8 +83,18 @@ in }; }; config = lib.mkIf cfg.enable { + # By default buildbot uses a normal user, which is not a good default, because + # we grant normal users potentially access to other resources. Also + # we don't to be able to ssh into buildbot. + + users.users.buildbot = { + isNormalUser = lib.mkForce false; + isSystemUser = true; + }; + services.buildbot-master = { enable = true; + home = "/var/lib/buildbot"; extraImports = '' from datetime import timedelta from buildbot_nix import GithubConfig, NixConfigurator From ecaafe61b4785ae1db6c28ca32091eafc5b52b0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 4 Nov 2023 09:50:23 +0100 Subject: [PATCH 2/3] make nginx output paths optional --- buildbot_nix/__init__.py | 25 ++++++++++++++++++----- nix/master.nix | 43 +++++++++++++++++++++++----------------- 2 files changed, 45 insertions(+), 23 deletions(-) diff --git a/buildbot_nix/__init__.py b/buildbot_nix/__init__.py index a7f33e8..20cbc94 100644 --- a/buildbot_nix/__init__.py +++ b/buildbot_nix/__init__.py @@ -235,8 +235,9 @@ class UpdateBuildOutput(steps.BuildStep): on the target machine. """ - def __init__(self, **kwargs: Any) -> None: + def __init__(self, path: Path, **kwargs: Any) -> None: super().__init__(**kwargs) + self.path = path def run(self) -> Generator[Any, object, Any]: props = self.build.getProperties() @@ -247,9 +248,8 @@ class UpdateBuildOutput(steps.BuildStep): attr = os.path.basename(props.getProperty("attr")) out_path = props.getProperty("out_path") # XXX don't hardcode this - p = Path("/var/www/buildbot/nix-outputs/") - os.makedirs(p, exist_ok=True) - (p / attr).write_text(out_path) + self.path.mkdir(parents=True, exist_ok=True) + (self.path / attr).write_text(out_path) return util.SUCCESS @@ -471,6 +471,7 @@ def nix_build_config( worker_names: list[str], has_cachix_auth_token: bool = False, has_cachix_signing_key: bool = False, + outputs_path: Path | None = None, ) -> util.BuilderConfig: """ Builds one nix flake attribute. @@ -536,7 +537,13 @@ def nix_build_config( command=["rm", "-f", util.Interpolate("result-%(prop:attr)s")], ) ) - factory.addStep(UpdateBuildOutput(name="Update build output")) + if outputs_path is not None: + factory.addStep( + UpdateBuildOutput( + name="Update build output", + path=outputs_path, + ) + ) return util.BuilderConfig( name=f"{project.name}/nix-build", project=project.name, @@ -578,6 +585,7 @@ def config_for_project( github: GithubConfig, nix_supported_systems: list[str], nix_eval_max_memory_size: int, + outputs_path: Path | None = None, ) -> Project: ## get a deterministic jitter for the project # random.seed(project.name) @@ -666,6 +674,7 @@ def config_for_project( worker_names, has_cachix_auth_token, has_cachix_signing_key, + outputs_path=outputs_path, ), nix_update_flake_config( project, @@ -689,6 +698,7 @@ class NixConfigurator(ConfiguratorBase): nix_supported_systems: list[str], nix_eval_max_memory_size: int = 4096, nix_workers_secret_name: str = "buildbot-nix-workers", + outputs_path: str | None = None, ) -> None: super().__init__() self.nix_workers_secret_name = nix_workers_secret_name @@ -697,6 +707,10 @@ class NixConfigurator(ConfiguratorBase): self.github = github self.url = url self.systemd_credentials_dir = os.environ["CREDENTIALS_DIRECTORY"] + if outputs_path is None: + self.outputs_path = None + else: + self.outputs_path = Path(outputs_path) def configure(self, config: dict[str, Any]) -> None: projects = load_projects(self.github.token(), self.github.project_cache_file) @@ -734,6 +748,7 @@ class NixConfigurator(ConfiguratorBase): self.github, self.nix_supported_systems, self.nix_eval_max_memory_size, + self.outputs_path, ) # Reload github projects diff --git a/nix/master.nix b/nix/master.nix index e79761a..8b808d3 100644 --- a/nix/master.nix +++ b/nix/master.nix @@ -80,6 +80,13 @@ in description = "Buildbot domain"; example = "buildbot.numtide.com"; }; + + outputsPath = lib.mkOption { + type = lib.types.nullOr lib.types.path; + description = "Path where we store the latest build store paths names for nix attributes as text files. This path will be exposed via nginx at \${domain}/nix-outputs"; + default = null; + example = "/var/www/buildbot/nix-outputs"; + }; }; }; config = lib.mkIf cfg.enable { @@ -120,6 +127,7 @@ in url=${builtins.toJSON config.services.buildbot-master.buildbotUrl}, nix_eval_max_memory_size=${builtins.toJSON cfg.evalMaxMemorySize}, nix_supported_systems=${builtins.toJSON cfg.buildSystems}, + outputs_path=${if cfg.outputsPath == null then "None" else builtins.toJSON cfg.outputsPath}, ) '' ]; @@ -175,27 +183,26 @@ in services.nginx.enable = true; services.nginx.virtualHosts.${cfg.domain} = { - locations."/".proxyPass = "http://127.0.0.1:${builtins.toString config.services.buildbot-master.port}/"; - locations."/sse" = { - proxyPass = "http://127.0.0.1:${builtins.toString config.services.buildbot-master.port}/sse"; - # proxy buffering will prevent sse to work - extraConfig = "proxy_buffering off;"; + locations = { + "/".proxyPass = "http://127.0.0.1:${builtins.toString config.services.buildbot-master.port}/"; + "/sse" = { + proxyPass = "http://127.0.0.1:${builtins.toString config.services.buildbot-master.port}/sse"; + # proxy buffering will prevent sse to work + extraConfig = "proxy_buffering off;"; + }; + "/ws" = { + proxyPass = "http://127.0.0.1:${builtins.toString config.services.buildbot-master.port}/ws"; + proxyWebsockets = true; + # raise the proxy timeout for the websocket + extraConfig = "proxy_read_timeout 6000s;"; + }; + } // lib.optionalAttrs (cfg.outputsPath != null) { + "/nix-outputs".root = cfg.outputsPath; }; - locations."/ws" = { - proxyPass = "http://127.0.0.1:${builtins.toString config.services.buildbot-master.port}/ws"; - proxyWebsockets = true; - # raise the proxy timeout for the websocket - extraConfig = "proxy_read_timeout 6000s;"; - }; - - # In this directory we store the lastest build store paths for nix attributes - locations."/nix-outputs".root = "/var/www/buildbot/"; }; # Allow buildbot-master to write to this directory - systemd.tmpfiles.rules = [ - "d /var/www/buildbot/nix-outputs 0755 buildbot buildbot - -" - ]; - + systemd.tmpfiles.rules = lib.optional (cfg.outputsPath != null) + "d ${cfg.outputPath} 0755 buildbot buildbot - -"; }; } From 333cd055c1b5bbdff0689f9f08cd684f0003a315 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 4 Nov 2023 11:25:33 +0100 Subject: [PATCH 3/3] disable default schedulers defined in nixos --- nix/master.nix | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/nix/master.nix b/nix/master.nix index 8b808d3..59b0d30 100644 --- a/nix/master.nix +++ b/nix/master.nix @@ -101,6 +101,12 @@ in services.buildbot-master = { enable = true; + + # disable example workers from nixpkgs + builders = [ ]; + schedulers = [ ]; + workers = [ ]; + home = "/var/lib/buildbot"; extraImports = '' from datetime import timedelta