lix-project/buildbot-nix
15
0
Fork 0
Code Issues 13 Pull requests 7 Packages Projects Releases Wiki Activity

pre-filter drv_info into all_deps

Browse source 
otherwise failure reporting is *enormous* with the entirety of a full
derivation info dump in there
This commit is contained in:
eldritch horrors 2024-03-11 13:07:35 +01:00
parent 13a67b483a
commit 51f7b52149
1 changed files with 20 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

40
buildbot_nix/__init__.py
View file

@ -61,11 +61,11 @@ class BuildTrigger(steps.BuildStep):
self, self,
builds_scheduler: str, builds_scheduler: str,
jobs: list[dict[str, Any]], jobs: list[dict[str, Any]],
drv_info: dict[str, Any], all_deps: dict[str, Any],
**kwargs: Any, **kwargs: Any,
) -> None: ) -> None:
self.jobs = jobs self.jobs = jobs
self.drv_info = drv_info self.all_deps = all_deps
self.config = None self.config = None
self.builds_scheduler = builds_scheduler self.builds_scheduler = builds_scheduler
self._result_list = [] self._result_list = []
@ -159,20 +159,9 @@ class BuildTrigger(steps.BuildStep):
build_props = self.build.getProperties() build_props = self.build.getProperties()
source = f"nix-eval-lix" source = f"nix-eval-lix"
all_deps = dict()
for drv, info in self.drv_info.items():
all_deps[drv] = set(info.get("inputDrvs").keys())
def closure_of(key, deps):
r, size = set([key]), 0
while len(r) != size:
size = len(r)
r.update(*[ deps[k] for k in r ])
return r.difference([key])
job_set = set(( drv for drv in ( job.get("drvPath") for job in self.jobs ) if drv ))
all_deps = { k: list(closure_of(k, all_deps).intersection(job_set)) for k in job_set }
builds_to_schedule = list(self.jobs) builds_to_schedule = list(self.jobs)
build_schedule_order = [] build_schedule_order = []
sorter = graphlib.TopologicalSorter(all_deps) sorter = graphlib.TopologicalSorter(self.all_deps)
for item in sorter.static_order(): for item in sorter.static_order():
i = 0 i = 0
while i < len(builds_to_schedule): while i < len(builds_to_schedule):
@ -191,7 +180,7 @@ class BuildTrigger(steps.BuildStep):
print('Scheduling..') print('Scheduling..')
schedule_now = [] schedule_now = []
for build in list(build_schedule_order): for build in list(build_schedule_order):
if all_deps.get(build.get("drvPath"), []) == []: if self.all_deps.get(build.get("drvPath"), []) == []:
build_schedule_order.remove(build) build_schedule_order.remove(build)
schedule_now.append(build) schedule_now.append(build)
if len(schedule_now) == 0: if len(schedule_now) == 0:
@ -238,7 +227,7 @@ class BuildTrigger(steps.BuildStep):
while True: while True:
old_paths = list(failed_paths) old_paths = list(failed_paths)
for build in list(build_schedule_order): for build in list(build_schedule_order):
deps = all_deps.get(build.get("drvPath"), []) deps = self.all_deps.get(build.get("drvPath"), [])
for path in old_paths: for path in old_paths:
if path in deps: if path in deps:
failed_checks.append(build) failed_checks.append(build)
@ -252,9 +241,9 @@ class BuildTrigger(steps.BuildStep):
print(' Removed jobs: ' + ', '.join(removed)) print(' Removed jobs: ' + ', '.join(removed))
all_results = worst_status(result, all_results) all_results = worst_status(result, all_results)
print(f' New result: {util.Results[all_results].upper()}') print(f' New result: {util.Results[all_results].upper()}')
for dep in all_deps: for dep in self.all_deps:
if job.get("drvPath") in all_deps[dep]: if job.get("drvPath") in self.all_deps[dep]:
all_deps[dep].remove(job.get("drvPath")) self.all_deps[dep].remove(job.get("drvPath"))
print('Done!') print('Done!')
if self.ended: if self.ended:
return util.CANCELLED return util.CANCELLED
@ -328,6 +317,17 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep):
except json.JSONDecodeError as e: except json.JSONDecodeError as e:
msg = f"Failed to parse `nix derivation show` output for {cmd.command}" msg = f"Failed to parse `nix derivation show` output for {cmd.command}"
raise BuildbotNixError(msg) from e raise BuildbotNixError(msg) from e
all_deps = dict()
for drv, info in drv_info.items():
all_deps[drv] = set(info.get("inputDrvs").keys())
def closure_of(key, deps):
r, size = set([key]), 0
while len(r) != size:
size = len(r)
r.update(*[ deps[k] for k in r ])
return r.difference([key])
job_set = set(( drv for drv in ( job.get("drvPath") for job in filtered_jobs ) if drv ))
all_deps = { k: list(closure_of(k, all_deps).intersection(job_set)) for k in job_set }
self.build.addStepsAfterCurrentStep( self.build.addStepsAfterCurrentStep(
[ [
@ -335,7 +335,7 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep):
builds_scheduler=f"lix-nix-build", builds_scheduler=f"lix-nix-build",
name="build flake", name="build flake",
jobs=filtered_jobs, jobs=filtered_jobs,
drv_info=drv_info, all_deps=all_deps,
), ),
], ],
) )