From b514df614b27e35da44a46bef3b77248cadafa6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 26 Dec 2023 19:48:27 +0100 Subject: [PATCH 1/5] devshell: add mypy + ruff --- flake.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flake.nix b/flake.nix index 53c1036..e6f5dab 100644 --- a/flake.nix +++ b/flake.nix @@ -37,6 +37,8 @@ packages.default = pkgs.mkShell { packages = [ pkgs.bashInteractive + pkgs.mypy + pkgs.ruff ]; }; packages.buildbot-nix = pkgs.python3.pkgs.callPackage ./default.nix { }; From 35079f89e71ea353d8a90ffbcf4654df905ca09f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 26 Dec 2023 19:49:57 +0100 Subject: [PATCH 2/5] apply ruff --- buildbot_nix/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/buildbot_nix/__init__.py b/buildbot_nix/__init__.py index 97efefc..b3ace20 100644 --- a/buildbot_nix/__init__.py +++ b/buildbot_nix/__init__.py @@ -12,6 +12,10 @@ from dataclasses import dataclass from pathlib import Path from typing import Any +from twisted.internet import defer, threads +from twisted.logger import Logger +from twisted.python.failure import Failure + from buildbot.configurators import ConfiguratorBase from buildbot.plugins import reporters, schedulers, secrets, steps, util, worker from buildbot.process import buildstep, logobserver, remotecommand @@ -22,9 +26,6 @@ from buildbot.process.results import ALL_RESULTS, statusToString from buildbot.steps.trigger import Trigger from buildbot.util import asyncSleep from buildbot.www.authz.endpointmatchers import EndpointMatcherBase, Match -from twisted.internet import defer, threads -from twisted.logger import Logger -from twisted.python.failure import Failure from .github_projects import ( GithubProject, From 44cfc8253bbaddc8fe5f99dce7092c776cdfba11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 24 Dec 2023 08:04:05 +0100 Subject: [PATCH 3/5] add buildbot-effects This is an implementation of hercules-ci-effects in python. --- bin/buildbot-effects | 9 ++ buildbot_effects/__init__.py | 233 +++++++++++++++++++++++++++++++++++ buildbot_effects/cli.py | 88 +++++++++++++ buildbot_effects/options.py | 13 ++ pyproject.toml | 6 +- 5 files changed, 348 insertions(+), 1 deletion(-) create mode 100755 bin/buildbot-effects create mode 100644 buildbot_effects/__init__.py create mode 100644 buildbot_effects/cli.py create mode 100644 buildbot_effects/options.py diff --git a/bin/buildbot-effects b/bin/buildbot-effects new file mode 100755 index 0000000..e4fd8b9 --- /dev/null +++ b/bin/buildbot-effects @@ -0,0 +1,9 @@ +#!/usr/bin/env python +import sys +from pathlib import Path +sys.path.append(str(Path(__file__).parent.parent)) + +from hercules_effects.cli import main + +if __name__ == '__main__': + main() diff --git a/buildbot_effects/__init__.py b/buildbot_effects/__init__.py new file mode 100644 index 0000000..6f8352a --- /dev/null +++ b/buildbot_effects/__init__.py @@ -0,0 +1,233 @@ +import json +import os +import shlex +import shutil +import subprocess +import sys +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path +from tempfile import NamedTemporaryFile +from typing import IO, Any + +from .options import EffectsOptions + + +def run( + cmd: list[str], + stdin: int | IO[str] | None = None, + stdout: int | IO[str] | None = None, + stderr: int | IO[str] | None = None, + verbose: bool = True, +) -> subprocess.CompletedProcess[str]: + if verbose: + print("$", shlex.join(cmd), file=sys.stderr) + return subprocess.run( + cmd, + check=True, + text=True, + stdin=stdin, + stdout=stdout, + stderr=stderr, + ) + + +def git_command(args: list[str], path: Path) -> str: + cmd = ["git", "-C", str(path), *args] + proc = run(cmd, stdout=subprocess.PIPE) + return proc.stdout.strip() + + +def get_git_rev(path: Path) -> str: + return git_command(["rev-parse", "--verify", "HEAD"], path) + + +def get_git_branch(path: Path) -> str: + return git_command(["rev-parse", "--abbrev-ref", "HEAD"], path) + + +def get_git_remote_url(path: Path) -> str | None: + try: + return git_command(["remote", "get-url", "origin"], path) + except subprocess.CalledProcessError: + return None + + +def git_get_tag(path: Path, rev: str) -> str | None: + tags = git_command(["tag", "--points-at", rev], path) + if tags: + return tags.splitlines()[1] + return None + + +def effects_args(opts: EffectsOptions) -> dict[str, Any]: + rev = opts.rev or get_git_rev(opts.path) + short_rev = rev[:7] + branch = opts.branch or get_git_branch(opts.path) + repo = opts.repo or opts.path.name + tag = opts.tag or git_get_tag(opts.path, rev) + url = opts.url or get_git_remote_url(opts.path) + primary_repo = dict( + name=repo, + branch=branch, + # TODO: support ref + ref=None, + tag=tag, + rev=rev, + shortRev=short_rev, + remoteHttpUrl=url, + ) + return { + "primaryRepo": primary_repo, + **primary_repo, + } + + +def nix_command(*args: str) -> list[str]: + return ["nix", "--extra-experimental-features", "nix-command flakes", *args] + + +def effect_function(opts: EffectsOptions) -> str: + args = effects_args(opts) + rev = args["rev"] + escaped_args = json.dumps(json.dumps(args)) + url = json.dumps(f"git+file://{opts.path}?rev={rev}#") + return f"""(((builtins.getFlake {url}).outputs.herculesCI (builtins.fromJSON {escaped_args})).onPush.default.outputs.hci-effects)""" + + +def list_effects(opts: EffectsOptions) -> list[str]: + cmd = nix_command( + "eval", + "--json", + "--expr", + f"builtins.attrNames {effect_function(opts)}", + ) + proc = run(cmd, stdout=subprocess.PIPE) + return json.loads(proc.stdout) + + +def instantiate_effects(opts: EffectsOptions) -> str: + cmd = [ + "nix-instantiate", + "--expr", + f"{effect_function(opts)}.deploy.run", + ] + proc = run(cmd, stdout=subprocess.PIPE) + return proc.stdout.rstrip() + + +def parse_derivation(path: str) -> dict[str, Any]: + cmd = [ + "nix", + "--extra-experimental-features", + "nix-command flakes", + "derivation", + "show", + f"{path}^*", + ] + proc = run(cmd, stdout=subprocess.PIPE) + return json.loads(proc.stdout) + + +def env_args(env: dict[str, str]) -> list[str]: + result = [] + for k, v in env.items(): + result.append("--setenv") + result.append(f"{k}") + result.append(f"{v}") + return result + + +@contextmanager +def pipe() -> Iterator[tuple[IO[str], IO[str]]]: + r, w = os.pipe() + r_file = os.fdopen(r, "r") + w_file = os.fdopen(w, "w") + try: + yield r_file, w_file + finally: + r_file.close() + w_file.close() + + +def run_effects( + drv_path: str, drv: dict[str, Any], secrets: dict[str, Any] = {} +) -> None: + builder = drv["builder"] + args = drv["args"] + sandboxed_cmd = [ + builder, + *args, + ] + env = {} + env["IN_HERCULES_CI_EFFECT"] = "true" + env["HERCULES_CI_SECRETS_JSON"] = "/run/secrets.json" + env["NIX_BUILD_TOP"] = "/build" + bwrap = shutil.which("bwrap") + if bwrap is None: + raise Exception("bwrap not found") + + bubblewrap_cmd = [ + "nix", + "develop", + "-i", + f"{drv_path}^*", + "-c", + bwrap, + "--unshare-all", + "--share-net", + "--new-session", + "--die-with-parent", + "--dir", + "/build", + "--chdir", + "/build", + "--tmpfs", + "/tmp", + "--tmpfs", + "/build", + "--proc", + "/proc", + "--dev", + "/dev", + "--ro-bind", + "/etc/resolv.conf", + "/etc/resolv.conf", + "--ro-bind", + "/etc/hosts", + "/etc/hosts", + "--ro-bind", + "/nix/store", + "/nix/store", + ] + + with NamedTemporaryFile() as tmp: + secrets = secrets.copy() + secrets["hercules-ci"] = {"data": {"token": "dummy"}} + tmp.write(json.dumps(secrets).encode()) + bubblewrap_cmd.extend( + [ + "--ro-bind", + tmp.name, + "/run/secrets.json", + ] + ) + bubblewrap_cmd.extend(env_args(env)) + bubblewrap_cmd.append("--") + bubblewrap_cmd.extend(sandboxed_cmd) + with pipe() as (r_file, w_file): + print("$", shlex.join(bubblewrap_cmd), file=sys.stderr) + proc = subprocess.Popen( + bubblewrap_cmd, + text=True, + stdin=subprocess.DEVNULL, + stdout=w_file, + stderr=w_file, + ) + w_file.close() + with proc: + for line in r_file: + print(line, end="") + proc.wait() + if proc.returncode != 0: + raise Exception(f"command failed with exit code {proc.returncode}") diff --git a/buildbot_effects/cli.py b/buildbot_effects/cli.py new file mode 100644 index 0000000..eb2dd10 --- /dev/null +++ b/buildbot_effects/cli.py @@ -0,0 +1,88 @@ +import argparse +from collections.abc import Callable +from pathlib import Path +import json + +from .options import EffectsOptions +from . import list_effects, instantiate_effects, parse_derivation, run_effects + + +def list_command(options: EffectsOptions) -> None: + print(list_effects(options)) + + +def run_command(options: EffectsOptions) -> None: + drv_path = instantiate_effects(options) + drvs = parse_derivation(drv_path) + drv = next(iter(drvs.values())) + + if options.secrets: + secrets = json.loads(options.secrets.read_text()) + else: + secrets = {} + run_effects(drv_path, drv, secrets=secrets) + + +def run_all_command(options: EffectsOptions) -> None: + print("TODO") + + +def parse_args() -> tuple[Callable[[EffectsOptions], None], EffectsOptions]: + parser = argparse.ArgumentParser(description="Run effects from a hercules-ci flake") + parser.add_argument( + "--secrets", + type=Path, + help="Path to a json file with secrets", + ) + parser.add_argument( + "--rev", + type=str, + help="Git revision to use", + ) + parser.add_argument( + "--branch", + type=str, + help="Git branch to use", + ) + parser.add_argument( + "--repo", + type=str, + help="Git repo to prepend to be", + ) + parser.add_argument( + "--path", + type=str, + help="Path to the repository", + ) + subparser = parser.add_subparsers( + dest="command", + required=True, + help="Command to run", + ) + list_parser = subparser.add_parser( + "list", + help="List available effects", + ) + list_parser.set_defaults(command=list_command) + run_parser = subparser.add_parser( + "run", + help="Run an effect", + ) + run_parser.set_defaults(command=run_command) + run_parser.add_argument( + "effect", + help="Effect to run", + ) + run_all_parser = subparser.add_parser( + "run-all", + help="Run all effects", + ) + run_all_parser.set_defaults(command=run_all_command) + + args = parser.parse_args() + return args.command, EffectsOptions(secrets=args.secrets) + + +def main() -> None: + command, options = parse_args() + command(options) diff --git a/buildbot_effects/options.py b/buildbot_effects/options.py new file mode 100644 index 0000000..02abd87 --- /dev/null +++ b/buildbot_effects/options.py @@ -0,0 +1,13 @@ +from dataclasses import dataclass, field +from pathlib import Path + + +@dataclass +class EffectsOptions: + secrets: Path | None = None + path: Path = field(default_factory=lambda: Path.cwd()) + repo: str | None = "" + rev: str | None = None + branch: str | None = None + url: str | None = None + tag: str | None = None diff --git a/pyproject.toml b/pyproject.toml index 8969d53..492c346 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,9 +21,13 @@ classifiers = [ "Programming Language :: Python" ] version = "0.0.1" +scripts = { buildbot-effects = "hercules_effects.cli:main" } [tool.setuptools] -packages = ["buildbot_nix"] +packages = [ + "buildbot_nix", + "buildbot_effects" +] [tool.ruff] target-version = "py311" From 0fc24dd4ac76474621e737d134b5de151aadbd56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 26 Dec 2023 21:56:12 +0100 Subject: [PATCH 4/5] ruff: enable more linting --- pyproject.toml | 52 +++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 49 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 492c346..b683a00 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,11 +32,57 @@ packages = [ [tool.ruff] target-version = "py311" line-length = 88 -select = ["E", "F", "I", "U", "N"] -ignore = [ "E501" ] +select = ["ALL"] +ignore = [ + # pydocstyle + "D", + # todo comments + "TD", + # fixmes + "FIX", + + # Unused function argument + "ARG001", + "ARG002", + + # Missing type annotation for `self` in method + "ANN101", + # Dynamically typed expressions (typing.Any) + "ANN401", + # Trailing comma missing + "COM812", + # Unnecessary `dict` call (rewrite as a literal) + "C408", + # Boolean-typed positional argument in function definition + "FBT001", + # Logging statement uses f-string + "G004", + # disabled on ruff's recommendation as causes problems with the formatter + "ISC001", + # Use of `assert` detected + "S101", + # `subprocess` call: check for execution of untrusted input + "S603", + # Starting a process with a partial executable path + "S607", + # Boolean default positional argument in function definition + "FBT002", + + # Too many statements + "PLR0915", + # Too many arguments in function definition + "PLR0913", + "PLR0912", # Too many branches + # $X is too complex + "C901", + + "E501", # line too long + "T201", # `print` found + "PLR2004", # Magic value used in comparison +] [tool.mypy] -python_version = "3.10" +python_version = "3.11" pretty = true warn_redundant_casts = true disallow_untyped_calls = true From 8b9c060dbe094519c9628a090bab220778e637a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 26 Dec 2023 21:56:36 +0100 Subject: [PATCH 5/5] fix new linting errors --- buildbot_effects/__init__.py | 20 +++- buildbot_effects/cli.py | 9 +- buildbot_nix/__init__.py | 196 +++++++++++++++++--------------- buildbot_nix/github_projects.py | 56 ++++++--- buildbot_nix/worker.py | 28 +++-- 5 files changed, 175 insertions(+), 134 deletions(-) diff --git a/buildbot_effects/__init__.py b/buildbot_effects/__init__.py index 6f8352a..ea6da55 100644 --- a/buildbot_effects/__init__.py +++ b/buildbot_effects/__init__.py @@ -13,6 +13,10 @@ from typing import IO, Any from .options import EffectsOptions +class BuildbotEffectsError(Exception): + pass + + def run( cmd: list[str], stdin: int | IO[str] | None = None, @@ -151,8 +155,12 @@ def pipe() -> Iterator[tuple[IO[str], IO[str]]]: def run_effects( - drv_path: str, drv: dict[str, Any], secrets: dict[str, Any] = {} + drv_path: str, + drv: dict[str, Any], + secrets: dict[str, Any] | None = None, ) -> None: + if secrets is None: + secrets = {} builder = drv["builder"] args = drv["args"] sandboxed_cmd = [ @@ -165,7 +173,8 @@ def run_effects( env["NIX_BUILD_TOP"] = "/build" bwrap = shutil.which("bwrap") if bwrap is None: - raise Exception("bwrap not found") + msg = "bwrap' executable not found" + raise BuildbotEffectsError(msg) bubblewrap_cmd = [ "nix", @@ -183,7 +192,7 @@ def run_effects( "--chdir", "/build", "--tmpfs", - "/tmp", + "/tmp", # noqa: S108 "--tmpfs", "/build", "--proc", @@ -210,7 +219,7 @@ def run_effects( "--ro-bind", tmp.name, "/run/secrets.json", - ] + ], ) bubblewrap_cmd.extend(env_args(env)) bubblewrap_cmd.append("--") @@ -230,4 +239,5 @@ def run_effects( print(line, end="") proc.wait() if proc.returncode != 0: - raise Exception(f"command failed with exit code {proc.returncode}") + msg = f"command failed with exit code {proc.returncode}" + raise BuildbotEffectsError(msg) diff --git a/buildbot_effects/cli.py b/buildbot_effects/cli.py index eb2dd10..556e4cc 100644 --- a/buildbot_effects/cli.py +++ b/buildbot_effects/cli.py @@ -1,10 +1,10 @@ import argparse +import json from collections.abc import Callable from pathlib import Path -import json +from . import instantiate_effects, list_effects, parse_derivation, run_effects from .options import EffectsOptions -from . import list_effects, instantiate_effects, parse_derivation, run_effects def list_command(options: EffectsOptions) -> None: @@ -16,10 +16,7 @@ def run_command(options: EffectsOptions) -> None: drvs = parse_derivation(drv_path) drv = next(iter(drvs.values())) - if options.secrets: - secrets = json.loads(options.secrets.read_text()) - else: - secrets = {} + secrets = json.loads(options.secrets.read_text()) if options.secrets else {} run_effects(drv_path, drv, secrets=secrets) diff --git a/buildbot_nix/__init__.py b/buildbot_nix/__init__.py index b3ace20..391042c 100644 --- a/buildbot_nix/__init__.py +++ b/buildbot_nix/__init__.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 - import json import multiprocessing import os @@ -10,16 +8,11 @@ from collections import defaultdict from collections.abc import Generator from dataclasses import dataclass from pathlib import Path -from typing import Any - -from twisted.internet import defer, threads -from twisted.logger import Logger -from twisted.python.failure import Failure +from typing import TYPE_CHECKING, Any from buildbot.configurators import ConfiguratorBase from buildbot.plugins import reporters, schedulers, secrets, steps, util, worker from buildbot.process import buildstep, logobserver, remotecommand -from buildbot.process.log import Log from buildbot.process.project import Project from buildbot.process.properties import Interpolate, Properties from buildbot.process.results import ALL_RESULTS, statusToString @@ -27,9 +20,16 @@ from buildbot.steps.trigger import Trigger from buildbot.util import asyncSleep from buildbot.www.authz.endpointmatchers import EndpointMatcherBase, Match +if TYPE_CHECKING: + from buildbot.process.log import Log + +from twisted.internet import defer, threads +from twisted.logger import Logger +from twisted.python.failure import Failure + from .github_projects import ( GithubProject, - create_project_hook, # noqa: E402 + create_project_hook, load_projects, refresh_projects, slugify_project_name, @@ -40,10 +40,12 @@ SKIPPED_BUILDER_NAME = "skipped-builds" log = Logger() +class BuildbotNixError(Exception): + pass + + class BuildTrigger(Trigger): - """ - Dynamic trigger that creates a build for every attribute. - """ + """Dynamic trigger that creates a build for every attribute.""" def __init__( self, @@ -123,9 +125,7 @@ class BuildTrigger(Trigger): return triggered_schedulers def getCurrentSummary(self) -> dict[str, str]: # noqa: N802 - """ - The original build trigger will the generic builder name `nix-build` in this case, which is not helpful - """ + """The original build trigger will the generic builder name `nix-build` in this case, which is not helpful""" if not self.triggeredNames: return {"step": "running"} summary = [] @@ -134,14 +134,13 @@ class BuildTrigger(Trigger): count = self._result_list.count(status) if count: summary.append( - f"{self._result_list.count(status)} {statusToString(status, count)}" + f"{self._result_list.count(status)} {statusToString(status, count)}", ) return {"step": f"({', '.join(summary)})"} class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep): - """ - Parses the output of `nix-eval-jobs` and triggers a `nix-build` build for + """Parses the output of `nix-eval-jobs` and triggers a `nix-build` build for every attribute. """ @@ -169,7 +168,8 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep): try: job = json.loads(line) except json.JSONDecodeError as e: - raise Exception(f"Failed to parse line: {line}") from e + msg = f"Failed to parse line: {line}" + raise BuildbotNixError(msg) from e jobs.append(job) build_props = self.build.getProperties() repo_name = build_props.getProperty( @@ -180,9 +180,7 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep): filtered_jobs = [] for job in jobs: system = job.get("system") - if not system: # report eval errors - filtered_jobs.append(job) - elif system in self.supported_systems: + if not system or system in self.supported_systems: # report eval errors filtered_jobs.append(job) self.build.addStepsAfterCurrentStep( @@ -192,8 +190,8 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep): skipped_builds_scheduler=f"{project_id}-nix-skipped-build", name="build flake", jobs=filtered_jobs, - ) - ] + ), + ], ) return result @@ -204,13 +202,12 @@ class RetryCounter: def __init__(self, retries: int) -> None: self.builds: dict[uuid.UUID, int] = defaultdict(lambda: retries) - def retry_build(self, id: uuid.UUID) -> int: - retries = self.builds[id] + def retry_build(self, build_id: uuid.UUID) -> int: + retries = self.builds[build_id] if retries > 1: - self.builds[id] = retries - 1 + self.builds[build_id] = retries - 1 return retries - else: - return 0 + return 0 # For now we limit this to two. Often this allows us to make the error log @@ -219,9 +216,7 @@ RETRY_COUNTER = RetryCounter(retries=2) class EvalErrorStep(steps.BuildStep): - """ - Shows the error message of a failed evaluation. - """ + """Shows the error message of a failed evaluation.""" @defer.inlineCallbacks def run(self) -> Generator[Any, object, Any]: @@ -234,9 +229,7 @@ class EvalErrorStep(steps.BuildStep): class NixBuildCommand(buildstep.ShellMixin, steps.BuildStep): - """ - Builds a nix derivation. - """ + """Builds a nix derivation.""" def __init__(self, **kwargs: Any) -> None: kwargs = self.setupShellMixin(kwargs) @@ -257,8 +250,7 @@ class NixBuildCommand(buildstep.ShellMixin, steps.BuildStep): class UpdateBuildOutput(steps.BuildStep): - """ - Updates store paths in a public www directory. + """Updates store paths in a public www directory. This is useful to prefetch updates without having to evaluate on the target machine. """ @@ -270,11 +262,11 @@ class UpdateBuildOutput(steps.BuildStep): def run(self) -> Generator[Any, object, Any]: props = self.build.getProperties() if props.getProperty("branch") != props.getProperty( - "github.repository.default_branch" + "github.repository.default_branch", ): return util.SKIPPED - attr = os.path.basename(props.getProperty("attr")) + attr = Path(props.getProperty("attr")).name out_path = props.getProperty("out_path") # XXX don't hardcode this self.path.mkdir(parents=True, exist_ok=True) @@ -320,12 +312,12 @@ def reload_github_projects( github_token_secret: str, project_cache_file: Path, ) -> util.BuilderConfig: - """ - Updates the flake an opens a PR for it. - """ + """Updates the flake an opens a PR for it.""" factory = util.BuildFactory() factory.addStep( - ReloadGithubProjects(github_token_secret, project_cache_file=project_cache_file) + ReloadGithubProjects( + github_token_secret, project_cache_file=project_cache_file + ), ) return util.BuilderConfig( name="reload-github-projects", @@ -339,20 +331,25 @@ def reload_github_projects( class GitWithRetry(steps.Git): @defer.inlineCallbacks def run_vc( - self, branch: str, revision: str, patch: str + self, + branch: str, + revision: str, + patch: str, ) -> Generator[Any, object, Any]: retry_counter = 0 while True: try: res = yield super().run_vc(branch, revision, patch) - return res - except Exception as e: + except Exception as e: # noqa: BLE001 retry_counter += 1 if retry_counter == 3: - raise e + msg = "Failed to clone" + raise BuildbotNixError(msg) from e log: Log = yield self.addLog("log") yield log.addStderr(f"Retrying git clone (error: {e})\n") yield asyncSleep(2 << retry_counter) # 2, 4, 8 + else: + return res def nix_eval_config( @@ -364,14 +361,13 @@ def nix_eval_config( worker_count: int, max_memory_size: int, ) -> util.BuilderConfig: - """ - Uses nix-eval-jobs to evaluate hydraJobs from flake.nix in parallel. + """Uses nix-eval-jobs to evaluate hydraJobs from flake.nix in parallel. For each evaluated attribute a new build pipeline is started. """ factory = util.BuildFactory() # check out the source url_with_secret = util.Interpolate( - f"https://git:%(secret:{github_token_secret})s@github.com/%(prop:project)s" + f"https://git:%(secret:{github_token_secret})s@github.com/%(prop:project)s", ) factory.addStep( GitWithRetry( @@ -379,7 +375,7 @@ def nix_eval_config( method="clean", submodules=True, haltOnFailure=True, - ) + ), ) factory.addStep( @@ -406,7 +402,7 @@ def nix_eval_config( ], haltOnFailure=True, locks=[eval_lock.access("exclusive")], - ) + ), ) return util.BuilderConfig( @@ -439,9 +435,7 @@ def nix_build_config( cachix: CachixConfig | None = None, outputs_path: Path | None = None, ) -> util.BuilderConfig: - """ - Builds one nix flake attribute. - """ + """Builds one nix flake attribute.""" factory = util.BuildFactory() factory.addStep( NixBuildCommand( @@ -467,7 +461,7 @@ def nix_build_config( # We increase this over the default since the build output might end up in a different `nix build`. timeout=60 * 60 * 3, haltOnFailure=True, - ) + ), ) if cachix: factory.addStep( @@ -480,7 +474,7 @@ def nix_build_config( cachix.name, util.Interpolate("result-%(prop:attr)s"), ], - ) + ), ) factory.addStep( @@ -491,27 +485,27 @@ def nix_build_config( "--add-root", # FIXME: cleanup old build attributes util.Interpolate( - "/nix/var/nix/gcroots/per-user/buildbot-worker/%(prop:project)s/%(prop:attr)s" + "/nix/var/nix/gcroots/per-user/buildbot-worker/%(prop:project)s/%(prop:attr)s", ), "-r", util.Property("out_path"), ], doStepIf=lambda s: s.getProperty("branch") == s.getProperty("github.repository.default_branch"), - ) + ), ) factory.addStep( steps.ShellCommand( name="Delete temporary gcroots", command=["rm", "-f", util.Interpolate("result-%(prop:attr)s")], - ) + ), ) if outputs_path is not None: factory.addStep( UpdateBuildOutput( name="Update build output", path=outputs_path, - ) + ), ) return util.BuilderConfig( name=f"{project.name}/nix-build", @@ -524,18 +518,17 @@ def nix_build_config( def nix_skipped_build_config( - project: GithubProject, worker_names: list[str] + project: GithubProject, + worker_names: list[str], ) -> util.BuilderConfig: - """ - Dummy builder that is triggered when a build is skipped. - """ + """Dummy builder that is triggered when a build is skipped.""" factory = util.BuildFactory() factory.addStep( EvalErrorStep( name="Nix evaluation", doStepIf=lambda s: s.getProperty("error"), hideStepIf=lambda _, s: not s.getProperty("error"), - ) + ), ) # This is just a dummy step showing the cached build @@ -544,7 +537,7 @@ def nix_skipped_build_config( name="Nix build (cached)", doStepIf=lambda _: False, hideStepIf=lambda _, s: s.getProperty("error"), - ) + ), ) return util.BuilderConfig( name=f"{project.name}/nix-skipped-build", @@ -596,7 +589,7 @@ def config_for_project( config["schedulers"].extend( [ schedulers.SingleBranchScheduler( - name=f"{project.id}-default-branch", + name=f"{project.project_id}-default-branch", change_filter=util.ChangeFilter( repository=project.url, filter_fn=lambda c: c.branch @@ -607,7 +600,7 @@ def config_for_project( ), # this is compatible with bors or github's merge queue schedulers.SingleBranchScheduler( - name=f"{project.id}-merge-queue", + name=f"{project.project_id}-merge-queue", change_filter=util.ChangeFilter( repository=project.url, branch_re="(gh-readonly-queue/.*|staging|trying)", @@ -616,35 +609,36 @@ def config_for_project( ), # build all pull requests schedulers.SingleBranchScheduler( - name=f"{project.id}-prs", + name=f"{project.project_id}-prs", change_filter=util.ChangeFilter( - repository=project.url, category="pull" + repository=project.url, + category="pull", ), builderNames=[f"{project.name}/nix-eval"], ), # this is triggered from `nix-eval` schedulers.Triggerable( - name=f"{project.id}-nix-build", + name=f"{project.project_id}-nix-build", builderNames=[f"{project.name}/nix-build"], ), # this is triggered from `nix-eval` when the build is skipped schedulers.Triggerable( - name=f"{project.id}-nix-skipped-build", + name=f"{project.project_id}-nix-skipped-build", builderNames=[f"{project.name}/nix-skipped-build"], ), # allow to manually trigger a nix-build schedulers.ForceScheduler( - name=f"{project.id}-force", + name=f"{project.project_id}-force", builderNames=[f"{project.name}/nix-eval"], properties=[ util.StringParameter( name="project", label="Name of the GitHub repository.", default=project.name, - ) + ), ], ), - ] + ], ) config["builders"].extend( [ @@ -666,18 +660,23 @@ def config_for_project( outputs_path=outputs_path, ), nix_skipped_build_config(project, [SKIPPED_BUILDER_NAME]), - ] + ], ) class AnyProjectEndpointMatcher(EndpointMatcherBase): - def __init__(self, builders: set[str] = set(), **kwargs: Any) -> None: + def __init__(self, builders: set[str] | None = None, **kwargs: Any) -> None: + if builders is None: + builders = set() self.builders = builders super().__init__(**kwargs) @defer.inlineCallbacks def check_builder( - self, endpoint_object: Any, endpoint_dict: dict[str, Any], object_type: str + self, + endpoint_object: Any, + endpoint_dict: dict[str, Any], + object_type: str, ) -> Generator[Any, Any, Any]: res = yield endpoint_object.get({}, endpoint_dict) if res is None: @@ -685,7 +684,7 @@ class AnyProjectEndpointMatcher(EndpointMatcherBase): builder = yield self.master.data.get(("builders", res["builderid"])) if builder["name"] in self.builders: - log.warn( + log.warning( "Builder {builder} allowed by {role}: {builders}", builder=builder["name"], role=self.role, @@ -693,7 +692,7 @@ class AnyProjectEndpointMatcher(EndpointMatcherBase): ) return Match(self.master, **{object_type: res}) else: - log.warn( + log.warning( "Builder {builder} not allowed by {role}: {builders}", builder=builder["name"], role=self.role, @@ -701,17 +700,26 @@ class AnyProjectEndpointMatcher(EndpointMatcherBase): ) def match_BuildEndpoint_rebuild( # noqa: N802 - self, epobject: Any, epdict: dict[str, Any], options: dict[str, Any] + self, + epobject: Any, + epdict: dict[str, Any], + options: dict[str, Any], ) -> Generator[Any, Any, Any]: return self.check_builder(epobject, epdict, "build") def match_BuildEndpoint_stop( # noqa: N802 - self, epobject: Any, epdict: dict[str, Any], options: dict[str, Any] + self, + epobject: Any, + epdict: dict[str, Any], + options: dict[str, Any], ) -> Generator[Any, Any, Any]: return self.check_builder(epobject, epdict, "build") def match_BuildRequestEndpoint_stop( # noqa: N802 - self, epobject: Any, epdict: dict[str, Any], options: dict[str, Any] + self, + epobject: Any, + epdict: dict[str, Any], + options: dict[str, Any], ) -> Generator[Any, Any, Any]: return self.check_builder(epobject, epdict, "buildrequest") @@ -719,7 +727,7 @@ class AnyProjectEndpointMatcher(EndpointMatcherBase): def setup_authz(projects: list[GithubProject], admins: list[str]) -> util.Authz: allow_rules = [] allowed_builders_by_org: defaultdict[str, set[str]] = defaultdict( - lambda: {"reload-github-projects"} + lambda: {"reload-github-projects"}, ) for project in projects: @@ -752,14 +760,13 @@ class NixConfigurator(ConfiguratorBase): def __init__( self, - # Shape of this file: - # [ { "name": "", "pass": "", "cores": "" } ] + # Shape of this file: [ { "name": "", "pass": "", "cores": "" } ] github: GithubConfig, url: str, nix_supported_systems: list[str], nix_eval_worker_count: int | None, nix_eval_max_memory_size: int, - nix_workers_secret_name: str = "buildbot-nix-workers", + nix_workers_secret_name: str = "buildbot-nix-workers", # noqa: S107 cachix: CachixConfig | None = None, outputs_path: str | None = None, ) -> None: @@ -824,7 +831,7 @@ class NixConfigurator(ConfiguratorBase): [worker_names[0]], self.github.token(), self.github.project_cache_file, - ) + ), ) config["workers"].append(worker.LocalWorker(SKIPPED_BUILDER_NAME)) config["schedulers"].extend( @@ -840,7 +847,7 @@ class NixConfigurator(ConfiguratorBase): builderNames=["reload-github-projects"], periodicBuildTimer=12 * 60 * 60, ), - ] + ], ) config["services"].append( reporters.GitHubStatusPush( @@ -849,11 +856,11 @@ class NixConfigurator(ConfiguratorBase): # we use `virtual_builder_name` in the webinterface # so that we distinguish what has beeing build context=Interpolate("buildbot/%(prop:status_name)s"), - ) + ), ) systemd_secrets = secrets.SecretInAFile( - dirname=os.environ["CREDENTIALS_DIRECTORY"] + dirname=os.environ["CREDENTIALS_DIRECTORY"], ) config["secretsProviders"].append(systemd_secrets) @@ -871,7 +878,7 @@ class NixConfigurator(ConfiguratorBase): if "auth" not in config["www"]: config["www"].setdefault("avatar_methods", []) config["www"]["avatar_methods"].append( - util.AvatarGitHub(token=self.github.token()) + util.AvatarGitHub(token=self.github.token()), ) config["www"]["auth"] = util.GitHubAuth( self.github.oauth_id, @@ -880,5 +887,6 @@ class NixConfigurator(ConfiguratorBase): ) config["www"]["authz"] = setup_authz( - admins=self.github.admins, projects=projects + admins=self.github.admins, + projects=projects, ) diff --git a/buildbot_nix/github_projects.py b/buildbot_nix/github_projects.py index 047f1ab..2c703f6 100644 --- a/buildbot_nix/github_projects.py +++ b/buildbot_nix/github_projects.py @@ -1,6 +1,6 @@ +import contextlib import http.client import json -import os import urllib.request from pathlib import Path from tempfile import NamedTemporaryFile @@ -9,6 +9,10 @@ from typing import Any from twisted.python import log +class GithubError(Exception): + pass + + class HttpResponse: def __init__(self, raw: http.client.HTTPResponse) -> None: self.raw = raw @@ -23,26 +27,32 @@ class HttpResponse: def http_request( url: str, method: str = "GET", - headers: dict[str, str] = {}, + headers: dict[str, str] | None = None, data: dict[str, Any] | None = None, ) -> HttpResponse: body = None if data: body = json.dumps(data).encode("ascii") + if headers is None: + headers = {} headers = headers.copy() headers["User-Agent"] = "buildbot-nix" - req = urllib.request.Request(url, headers=headers, method=method, data=body) + + if not url.startswith("https:"): + msg = "url must be https: {url}" + raise GithubError(msg) + + req = urllib.request.Request( # noqa: S310 + url, headers=headers, method=method, data=body + ) try: - resp = urllib.request.urlopen(req) + resp = urllib.request.urlopen(req) # noqa: S310 except urllib.request.HTTPError as e: resp_body = "" - try: + with contextlib.suppress(OSError, UnicodeDecodeError): resp_body = e.fp.read().decode("utf-8", "replace") - except Exception: - pass - raise Exception( - f"Request for {method} {url} failed with {e.code} {e.reason}: {resp_body}" - ) from e + msg = f"Request for {method} {url} failed with {e.code} {e.reason}: {resp_body}" + raise GithubError(msg) from e return HttpResponse(resp) @@ -56,7 +66,8 @@ def paginated_github_request(url: str, token: str) -> list[dict[str, Any]]: headers={"Authorization": f"Bearer {token}"}, ) except OSError as e: - raise Exception(f"failed to fetch {next_url}: {e}") from e + msg = f"failed to fetch {next_url}: {e}" + raise GithubError(msg) from e next_url = None link = res.headers()["Link"] if link is not None: @@ -94,7 +105,7 @@ class GithubProject: return self.data["html_url"] @property - def id(self) -> str: + def project_id(self) -> str: return slugify_project_name(self.data["full_name"]) @property @@ -111,13 +122,21 @@ class GithubProject: def create_project_hook( - owner: str, repo: str, token: str, webhook_url: str, webhook_secret: str + owner: str, + repo: str, + token: str, + webhook_url: str, + webhook_secret: str, ) -> None: hooks = paginated_github_request( - f"https://api.github.com/repos/{owner}/{repo}/hooks?per_page=100", token + f"https://api.github.com/repos/{owner}/{repo}/hooks?per_page=100", + token, ) config = dict( - url=webhook_url, content_type="json", insecure_ssl="0", secret=webhook_secret + url=webhook_url, + content_type="json", + insecure_ssl="0", + secret=webhook_secret, ) data = dict(name="web", active=True, events=["push", "pull_request"], config=config) headers = { @@ -149,18 +168,19 @@ def refresh_projects(github_token: str, repo_cache_file: Path) -> None: if not repo["permissions"]["admin"]: name = repo["full_name"] log.msg( - f"skipping {name} because we do not have admin privileges, needed for hook management" + f"skipping {name} because we do not have admin privileges, needed for hook management", ) else: repos.append(repo) with NamedTemporaryFile("w", delete=False, dir=repo_cache_file.parent) as f: + path = Path(f.name) try: f.write(json.dumps(repos)) f.flush() - os.rename(f.name, repo_cache_file) + path.rename(repo_cache_file) except OSError: - os.unlink(f.name) + path.unlink() raise diff --git a/buildbot_nix/worker.py b/buildbot_nix/worker.py index 65d0f8b..5fc1fd7 100644 --- a/buildbot_nix/worker.py +++ b/buildbot_nix/worker.py @@ -1,9 +1,7 @@ -#!/usr/bin/env python3 - import multiprocessing import os import socket -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path from buildbot_worker.bot import Worker @@ -18,19 +16,27 @@ def require_env(key: str) -> str: @dataclass class WorkerConfig: - password: str = Path(require_env("WORKER_PASSWORD_FILE")).read_text().rstrip("\r\n") - worker_count: int = int( - os.environ.get("WORKER_COUNT", str(multiprocessing.cpu_count())) + password: str = field( + default_factory=lambda: Path(require_env("WORKER_PASSWORD_FILE")) + .read_text() + .rstrip("\r\n") ) - buildbot_dir: str = require_env("BUILDBOT_DIR") - master_url: str = require_env("MASTER_URL") + worker_count: int = int( + os.environ.get("WORKER_COUNT", str(multiprocessing.cpu_count())), + ) + buildbot_dir: Path = field( + default_factory=lambda: Path(require_env("BUILDBOT_DIR")) + ) + master_url: str = field(default_factory=lambda: require_env("MASTER_URL")) def setup_worker( - application: service.Application, id: int, config: WorkerConfig + application: service.Application, + builder_id: int, + config: WorkerConfig, ) -> None: - basedir = f"{config.buildbot_dir}-{id:03}" - os.makedirs(basedir, mode=0o700, exist_ok=True) + basedir = config.buildbot_dir.parent / f"{config.buildbot_dir.name}-{builder_id:03}" + basedir.mkdir(parents=True, exist_ok=True, mode=0o700) hostname = socket.gethostname() workername = f"{hostname}-{id:03}"