feat: support non-flake entrypoint
The way to use Buildbot jobs without Flakes is via `.ci/buildbot.nix`. Signed-off-by: Raito Bezarius <raito@lix.systems>
This commit is contained in:
parent
54bba654d4
commit
7b8d200475
|
|
@ -23,6 +23,7 @@ from buildbot.reporters.generators.build import BuildStatusGenerator
|
|||
from buildbot.reporters.message import MessageFormatterFunction
|
||||
from buildbot.process.buildstep import EXCEPTION
|
||||
from buildbot.process.buildstep import SUCCESS
|
||||
from buildbot.process.buildstep import BuildStepFailed
|
||||
from buildbot.process.results import worst_status
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -37,6 +38,14 @@ log = Logger()
|
|||
|
||||
FLAKE_TARGET_ATTRIBUTE_FOR_JOBS = "buildbotJobs"
|
||||
|
||||
@dataclass
|
||||
class EvaluatorSettings:
|
||||
supported_systems: list[str]
|
||||
worker_count: int
|
||||
max_memory_size: int
|
||||
gc_roots_dir: str
|
||||
lock: util.MasterLock
|
||||
|
||||
@dataclass
|
||||
class NixBuilder:
|
||||
protocol: str
|
||||
|
|
@ -406,6 +415,119 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep):
|
|||
|
||||
return result
|
||||
|
||||
def make_job_evaluator(name: str, settings: EvaluatorSettings, flake: bool) -> NixEvalCommand:
|
||||
actual_command = []
|
||||
|
||||
if flake:
|
||||
actual_command += ["--flake", f".#{FLAKE_TARGET_ATTRIBUTE_FOR_JOBS}"]
|
||||
else:
|
||||
actual_command += []
|
||||
|
||||
return NixEvalCommand(
|
||||
env={},
|
||||
name=name,
|
||||
supported_systems=settings.supported_systems,
|
||||
command=[
|
||||
"nix-eval-jobs",
|
||||
"--workers",
|
||||
str(settings.worker_count),
|
||||
"--max-memory-size",
|
||||
str(settings.max_memory_size),
|
||||
"--gc-roots-dir",
|
||||
settings.gc_roots_dir,
|
||||
"--force-recurse",
|
||||
"--check-cache-status",
|
||||
] + actual_command,
|
||||
haltOnFailure=True,
|
||||
locks=[settings.lock.access("exclusive")]
|
||||
)
|
||||
|
||||
|
||||
class NixCommandMixin(buildstep.CommandMixin):
|
||||
def isNixParseable(self, filename):
|
||||
command = ['nix-instantiate', '--parse', filename]
|
||||
cmd = yield self.makeRemoteShellCommand(
|
||||
command=command, stdioLogName='nix-parse', collectStdout=True
|
||||
)
|
||||
|
||||
yield self.runCommand(cmd)
|
||||
log = yield self.getLog("nix-parse")
|
||||
yield log.finish()
|
||||
|
||||
return cmd.results() == SUCCESS
|
||||
|
||||
def evalFlakeAttrNames(self, attribute, log=None):
|
||||
command = ['nix', 'eval', f'.#{attribute}', '--apply', 'builtins.attrNames', '--json']
|
||||
cmd = yield self.makeRemoteShellCommand(
|
||||
command=command, stdioLogName='nix-parse', collectStdout=True
|
||||
)
|
||||
|
||||
yield self.runCommand(cmd)
|
||||
log = yield self.getLog("nix-parse")
|
||||
yield log.finish()
|
||||
|
||||
return cmd.results() == SUCCESS
|
||||
|
||||
|
||||
class NixConfigure(buildstep.CommandMixin, steps.BuildStep):
|
||||
"""
|
||||
Determine what `NixEvalCommand` step should be added after
|
||||
based on the existence of:
|
||||
|
||||
- flake.nix
|
||||
- .ci/buildbot.nix
|
||||
"""
|
||||
|
||||
def __init__(self, eval_settings: EvaluatorSettings, **kwargs: Any) -> None:
|
||||
self.evaluator_settings = eval_settings
|
||||
kwargs = self.setupShellMixin(kwargs)
|
||||
super().__init__(**kwargs)
|
||||
self.addLogObserver("stdio", self.observer)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def run(self) -> Generator[Any, object, Any]:
|
||||
configure_log: Log = yield self.getLog("stdio")
|
||||
|
||||
# TODO: build `isNixParseable`, `evalFlakeAttr`
|
||||
|
||||
# Takes precedence.
|
||||
configure_log.addStdout("checking if there's a .ci/buildbot.nix...")
|
||||
ci_buildbot_defn_exists = yield self.pathExists('.ci/buildbot.nix')
|
||||
if ci_buildbot_defn_exists:
|
||||
configure_log.addStdout(".ci/buildbot.nix found, configured for non-flake CI, checking parseability...")
|
||||
parseable = yield self.isNixParseable('.ci/buildbot.nix')
|
||||
if not parseable:
|
||||
configure_log.addStderr(".ci/buildbot.nix is not parseable, cannot continue!")
|
||||
raise BuildStepFailed()
|
||||
|
||||
configure_log.addStdout(".ci/buildbot.nix parseable!")
|
||||
self.build.addStepAfterCurrentStep(
|
||||
make_job_evaluator(
|
||||
"evaluate `.ci/buildbot.nix` jobs",
|
||||
self.evaluator_settings,
|
||||
False
|
||||
)
|
||||
)
|
||||
|
||||
flake_exists = yield self.pathExists('flake.nix')
|
||||
if flake_exists:
|
||||
configure_log.addStdout(f"flake.nix found, checking if there's a `.#{FLAKE_TARGET_ATTRIBUTE_FOR_JOBS}` attribute set for CI...")
|
||||
jobs: list[str] = yield self.evalFlakeAttrNames(FLAKE_TARGET_ATTRIBUTE_FOR_JOBS) # type: ignore
|
||||
if not jobs:
|
||||
configure_log.addStdout("flake.nix has no jobs, terminating early")
|
||||
return
|
||||
|
||||
configure_log.addStdout(f"flake.nix has {len(jobs)} top-level buildbot jobs")
|
||||
self.build.addStepAfterCurrentStep(
|
||||
make_job_evaluator(
|
||||
"evaluate `flake.nix` jobs",
|
||||
self.evaluator_settings,
|
||||
True
|
||||
)
|
||||
)
|
||||
return
|
||||
|
||||
configure_log.addStdout("neither flake.nix found neither .ci/buildbot.nix, no CI to run!")
|
||||
|
||||
class NixBuildCommand(buildstep.ShellMixin, steps.BuildStep):
|
||||
"""Builds a nix derivation."""
|
||||
|
|
@ -445,7 +567,8 @@ def nix_eval_config(
|
|||
worker_count: int,
|
||||
max_memory_size: int,
|
||||
) -> util.BuilderConfig:
|
||||
"""Uses nix-eval-jobs to evaluate $FLAKE_TARGET_ATTRIBUTE_FOR_JOBS (`.#hydraJobs` by default) from flake.nix in parallel.
|
||||
"""
|
||||
Uses nix-eval-jobs to evaluate the entrypoint of this project.
|
||||
For each evaluated attribute a new build pipeline is started.
|
||||
"""
|
||||
factory = util.BuildFactory()
|
||||
|
|
@ -467,27 +590,22 @@ def nix_eval_config(
|
|||
"/nix/var/nix/gcroots/per-user/buildbot-worker/%(prop:project)s/drvs/%(prop:workername)s/",
|
||||
)
|
||||
|
||||
eval_settings = EvaluatorSettings(
|
||||
supported_systems=supported_systems,
|
||||
worker_count=worker_count,
|
||||
max_memory_size=max_memory_size,
|
||||
gc_roots_dir=drv_gcroots_dir,
|
||||
lock=eval_lock
|
||||
)
|
||||
|
||||
# NixConfigure will choose
|
||||
# how to add a NixEvalCommand job
|
||||
# based on whether there's a flake.nix or
|
||||
# a .ci/buildbot.nix.
|
||||
factory.addStep(
|
||||
NixEvalCommand(
|
||||
env={},
|
||||
name="evaluate flake",
|
||||
supported_systems=supported_systems,
|
||||
command=[
|
||||
"nix-eval-jobs",
|
||||
"--workers",
|
||||
str(worker_count),
|
||||
"--max-memory-size",
|
||||
str(max_memory_size),
|
||||
"--gc-roots-dir",
|
||||
drv_gcroots_dir,
|
||||
"--force-recurse",
|
||||
"--check-cache-status",
|
||||
"--flake",
|
||||
f".#{FLAKE_TARGET_ATTRIBUTE_FOR_JOBS}"
|
||||
],
|
||||
haltOnFailure=True,
|
||||
locks=[eval_lock.access("exclusive")],
|
||||
),
|
||||
NixConfigure(
|
||||
eval_settings
|
||||
)
|
||||
)
|
||||
|
||||
factory.addStep(
|
||||
|
|
|
|||
Loading…
Reference in a new issue