From 7b8d200475ba2b6f5952329566d8e600e99ffa27 Mon Sep 17 00:00:00 2001 From: Raito Bezarius Date: Fri, 19 Jul 2024 21:57:48 +0200 Subject: [PATCH] feat: support non-flake entrypoint The way to use Buildbot jobs without Flakes is via `.ci/buildbot.nix`. Signed-off-by: Raito Bezarius --- buildbot_nix/__init__.py | 160 ++++++++++++++++++++++++++++++++++----- 1 file changed, 139 insertions(+), 21 deletions(-) diff --git a/buildbot_nix/__init__.py b/buildbot_nix/__init__.py index 9e89d7f..81fb525 100644 --- a/buildbot_nix/__init__.py +++ b/buildbot_nix/__init__.py @@ -23,6 +23,7 @@ from buildbot.reporters.generators.build import BuildStatusGenerator from buildbot.reporters.message import MessageFormatterFunction from buildbot.process.buildstep import EXCEPTION from buildbot.process.buildstep import SUCCESS +from buildbot.process.buildstep import BuildStepFailed from buildbot.process.results import worst_status if TYPE_CHECKING: @@ -37,6 +38,14 @@ log = Logger() FLAKE_TARGET_ATTRIBUTE_FOR_JOBS = "buildbotJobs" +@dataclass +class EvaluatorSettings: + supported_systems: list[str] + worker_count: int + max_memory_size: int + gc_roots_dir: str + lock: util.MasterLock + @dataclass class NixBuilder: protocol: str @@ -406,6 +415,119 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep): return result +def make_job_evaluator(name: str, settings: EvaluatorSettings, flake: bool) -> NixEvalCommand: + actual_command = [] + + if flake: + actual_command += ["--flake", f".#{FLAKE_TARGET_ATTRIBUTE_FOR_JOBS}"] + else: + actual_command += [] + + return NixEvalCommand( + env={}, + name=name, + supported_systems=settings.supported_systems, + command=[ + "nix-eval-jobs", + "--workers", + str(settings.worker_count), + "--max-memory-size", + str(settings.max_memory_size), + "--gc-roots-dir", + settings.gc_roots_dir, + "--force-recurse", + "--check-cache-status", + ] + actual_command, + haltOnFailure=True, + locks=[settings.lock.access("exclusive")] + ) + + +class NixCommandMixin(buildstep.CommandMixin): + def isNixParseable(self, filename): + command = ['nix-instantiate', '--parse', filename] + cmd = yield self.makeRemoteShellCommand( + command=command, stdioLogName='nix-parse', collectStdout=True + ) + + yield self.runCommand(cmd) + log = yield self.getLog("nix-parse") + yield log.finish() + + return cmd.results() == SUCCESS + + def evalFlakeAttrNames(self, attribute, log=None): + command = ['nix', 'eval', f'.#{attribute}', '--apply', 'builtins.attrNames', '--json'] + cmd = yield self.makeRemoteShellCommand( + command=command, stdioLogName='nix-parse', collectStdout=True + ) + + yield self.runCommand(cmd) + log = yield self.getLog("nix-parse") + yield log.finish() + + return cmd.results() == SUCCESS + + +class NixConfigure(buildstep.CommandMixin, steps.BuildStep): + """ + Determine what `NixEvalCommand` step should be added after + based on the existence of: + + - flake.nix + - .ci/buildbot.nix + """ + + def __init__(self, eval_settings: EvaluatorSettings, **kwargs: Any) -> None: + self.evaluator_settings = eval_settings + kwargs = self.setupShellMixin(kwargs) + super().__init__(**kwargs) + self.addLogObserver("stdio", self.observer) + + @defer.inlineCallbacks + def run(self) -> Generator[Any, object, Any]: + configure_log: Log = yield self.getLog("stdio") + + # TODO: build `isNixParseable`, `evalFlakeAttr` + + # Takes precedence. + configure_log.addStdout("checking if there's a .ci/buildbot.nix...") + ci_buildbot_defn_exists = yield self.pathExists('.ci/buildbot.nix') + if ci_buildbot_defn_exists: + configure_log.addStdout(".ci/buildbot.nix found, configured for non-flake CI, checking parseability...") + parseable = yield self.isNixParseable('.ci/buildbot.nix') + if not parseable: + configure_log.addStderr(".ci/buildbot.nix is not parseable, cannot continue!") + raise BuildStepFailed() + + configure_log.addStdout(".ci/buildbot.nix parseable!") + self.build.addStepAfterCurrentStep( + make_job_evaluator( + "evaluate `.ci/buildbot.nix` jobs", + self.evaluator_settings, + False + ) + ) + + flake_exists = yield self.pathExists('flake.nix') + if flake_exists: + configure_log.addStdout(f"flake.nix found, checking if there's a `.#{FLAKE_TARGET_ATTRIBUTE_FOR_JOBS}` attribute set for CI...") + jobs: list[str] = yield self.evalFlakeAttrNames(FLAKE_TARGET_ATTRIBUTE_FOR_JOBS) # type: ignore + if not jobs: + configure_log.addStdout("flake.nix has no jobs, terminating early") + return + + configure_log.addStdout(f"flake.nix has {len(jobs)} top-level buildbot jobs") + self.build.addStepAfterCurrentStep( + make_job_evaluator( + "evaluate `flake.nix` jobs", + self.evaluator_settings, + True + ) + ) + return + + configure_log.addStdout("neither flake.nix found neither .ci/buildbot.nix, no CI to run!") class NixBuildCommand(buildstep.ShellMixin, steps.BuildStep): """Builds a nix derivation.""" @@ -445,7 +567,8 @@ def nix_eval_config( worker_count: int, max_memory_size: int, ) -> util.BuilderConfig: - """Uses nix-eval-jobs to evaluate $FLAKE_TARGET_ATTRIBUTE_FOR_JOBS (`.#hydraJobs` by default) from flake.nix in parallel. + """ + Uses nix-eval-jobs to evaluate the entrypoint of this project. For each evaluated attribute a new build pipeline is started. """ factory = util.BuildFactory() @@ -467,27 +590,22 @@ def nix_eval_config( "/nix/var/nix/gcroots/per-user/buildbot-worker/%(prop:project)s/drvs/%(prop:workername)s/", ) + eval_settings = EvaluatorSettings( + supported_systems=supported_systems, + worker_count=worker_count, + max_memory_size=max_memory_size, + gc_roots_dir=drv_gcroots_dir, + lock=eval_lock + ) + + # NixConfigure will choose + # how to add a NixEvalCommand job + # based on whether there's a flake.nix or + # a .ci/buildbot.nix. factory.addStep( - NixEvalCommand( - env={}, - name="evaluate flake", - supported_systems=supported_systems, - command=[ - "nix-eval-jobs", - "--workers", - str(worker_count), - "--max-memory-size", - str(max_memory_size), - "--gc-roots-dir", - drv_gcroots_dir, - "--force-recurse", - "--check-cache-status", - "--flake", - f".#{FLAKE_TARGET_ATTRIBUTE_FOR_JOBS}" - ], - haltOnFailure=True, - locks=[eval_lock.access("exclusive")], - ), + NixConfigure( + eval_settings + ) ) factory.addStep(