lix-project/buildbot-nix
15
0
Fork 0
Code Issues 6 Pull requests 2 Packages Projects Releases Wiki Activity

Skip scheduling cached builds; improve reporter message

Browse source
This commit is contained in:
puck 2024-03-11 15:05:15 +00:00
parent de02c833d0
commit e9b3b38bbf
1 changed files with 31 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

49
buildbot_nix/__init__.py
View file

@ -158,6 +158,7 @@ class BuildTrigger(steps.BuildStep):
self.running = True self.running = True
build_props = self.build.getProperties() build_props = self.build.getProperties()
source = f"nix-eval-lix" source = f"nix-eval-lix"
logs: Log = yield self.addLog("build info")
builds_to_schedule = list(self.jobs) builds_to_schedule = list(self.jobs)
build_schedule_order = [] build_schedule_order = []
@ -173,20 +174,23 @@ class BuildTrigger(steps.BuildStep):
done = [] done = []
scheduled = [] scheduled = []
failed = {} failed = []
all_results = SUCCESS all_results = SUCCESS
ss_for_trigger = self.prepareSourcestampListForTrigger() ss_for_trigger = self.prepareSourcestampListForTrigger()
while not self.ended and (len(build_schedule_order) > 0 or len(scheduled) > 0): while not self.ended and (len(build_schedule_order) > 0 or len(scheduled) > 0):
print('Scheduling..')
schedule_now = [] schedule_now = []
for build in list(build_schedule_order): for build in list(build_schedule_order):
if self.all_deps.get(build.get("drvPath"), []) == []: if self.all_deps.get(build.get("drvPath"), []) == []:
build_schedule_order.remove(build) build_schedule_order.remove(build)
schedule_now.append(build) schedule_now.append(build)
if len(schedule_now) == 0:
print(' No builds to schedule found.')
for job in schedule_now: for job in schedule_now:
print(f" - {job.get('attr')}") if job.get('isCached'):
logs.addStdout(f"Cached {job.get('attr')} ({job.get('drvPath')}) - skipping\n")
for dep in self.all_deps:
if job.get("drvPath") in self.all_deps[dep]:
self.all_deps[dep].remove(job.get("drvPath"))
continue
logs.addStdout(f"Scheduling {job.get('attr')} ({job.get('drvPath')})\n")
(scheduler, props) = self.schedule_one(build_props, job) (scheduler, props) = self.schedule_one(build_props, job)
scheduler = self.getSchedulerByName(scheduler) scheduler = self.getSchedulerByName(scheduler)
@ -211,7 +215,11 @@ class BuildTrigger(steps.BuildStep):
yield self.addURL(f"{scheduler.name} #{brid}", url) yield self.addURL(f"{scheduler.name} #{brid}", url)
self._add_results(brid) self._add_results(brid)
self.brids.append(brid) self.brids.append(brid)
print('Waiting..') if len(scheduled) == 0:
if len(build_schedule_order) == 0:
logs.addStderr('Ran out of builds\n')
break
continue
wait_for_next = defer.DeferredList([results for _, _, results in scheduled], fireOnOneCallback = True, fireOnOneErrback=True) wait_for_next = defer.DeferredList([results for _, _, results in scheduled], fireOnOneCallback = True, fireOnOneErrback=True)
self.waitForFinishDeferred = wait_for_next self.waitForFinishDeferred = wait_for_next
results, index = yield wait_for_next results, index = yield wait_for_next
@ -219,11 +227,16 @@ class BuildTrigger(steps.BuildStep):
done.append((job, brids, results)) done.append((job, brids, results))
del scheduled[index] del scheduled[index]
result = results[0] result = results[0]
print(f' Found finished build {job.get("attr")}, result {util.Results[result].upper()}') logs.addStdout(f'Build {job.get("attr")} ({job.get("drvPath")}) finished, result {util.Results[result].upper()}\n')
if result != SUCCESS: if result != SUCCESS:
failed_checks = [] failed_checks = []
failed_paths = [job.get('drvPath')] failed_paths = [job.get('drvPath')]
removed = [] removed = []
failed.append((
job.get("attr"),
"failed",
[ getURLForBuildrequest(self.master, brid) for brid in brids.values() ]
))
while True: while True:
old_paths = list(failed_paths) old_paths = list(failed_paths)
for build in list(build_schedule_order): for build in list(build_schedule_order):
@ -234,22 +247,21 @@ class BuildTrigger(steps.BuildStep):
failed_paths.append(build.get("drvPath")) failed_paths.append(build.get("drvPath"))
build_schedule_order.remove(build) build_schedule_order.remove(build)
removed.append(build.get("attr")) removed.append(build.get("attr"))
failed[build.get("attr")] = (f"dependency {job.get('attr')} failed", []) failed.append((build.get("attr"), f"dependency {job.get('attr')} failed", []))
break break
if old_paths == failed_paths: if old_paths == failed_paths:
break break
failed[job.get("attr")] = ( if len(removed) > 3:
"failed", yield logs.addStdout(' Skipping jobs: ' + ', '.join(removed[:3]) + f', ... ({len(removed) - 3} more)\n')
[ getURLForBuildrequest(self.master, brid) for brid in brids.values() ] else:
) yield logs.addStdout(' Skipping jobs: ' + ', '.join(removed) + '\n')
print(' Removed jobs: ' + ', '.join(removed))
all_results = worst_status(result, all_results) all_results = worst_status(result, all_results)
print(f' New result: {util.Results[all_results].upper()}')
for dep in self.all_deps: for dep in self.all_deps:
if job.get("drvPath") in self.all_deps[dep]: if job.get("drvPath") in self.all_deps[dep]:
self.all_deps[dep].remove(job.get("drvPath")) self.all_deps[dep].remove(job.get("drvPath"))
print('Done!') yield logs.addHeader('Done!\n')
yield logs.finish()
build_props.setProperty("failed_builds", failed, "nix-eval-lix") build_props.setProperty("failed_builds", failed, "nix-eval-lix")
if self.ended: if self.ended:
return util.CANCELLED return util.CANCELLED
@ -653,7 +665,7 @@ def gerritReviewCB(builderName, build, result, master, arg):
if builderName != 'lix/nix-eval': if builderName != 'lix/nix-eval':
return dict() return dict()
failed = build['properties'].get('failed_builds', [{}])[0] failed = build['properties'].get('failed_builds', [[]])[0]
labels = { labels = {
'Verified': -1 if failed else 1, 'Verified': -1 if failed else 1,
@ -663,8 +675,9 @@ def gerritReviewCB(builderName, build, result, master, arg):
message += "The result is: %s\n" % util.Results[result].upper() message += "The result is: %s\n" % util.Results[result].upper()
if result != util.SUCCESS: if result != util.SUCCESS:
message += "\nFailed checks:\n" message += "\nFailed checks:\n"
for check, context in sorted(failed.items()): for check, how, urls in failed:
how, urls = context if not urls:
message += " "
message += f" - {check}: {how}" message += f" - {check}: {how}"
if urls: if urls:
message += f" (see {', '.join(urls)})" message += f" (see {', '.join(urls)})"