From 4057371c76c88df3e677ee5fc8016a2f8b37746e Mon Sep 17 00:00:00 2001 From: Puck Meerburg Date: Sun, 10 Mar 2024 22:55:38 +0000 Subject: [PATCH] Add build result tracking, schedule newly available builds --- buildbot_nix/__init__.py | 87 +++++++++++++++++++++++++++++++++++----- 1 file changed, 76 insertions(+), 11 deletions(-) diff --git a/buildbot_nix/__init__.py b/buildbot_nix/__init__.py index 55bc97d..2fd6ee2 100644 --- a/buildbot_nix/__init__.py +++ b/buildbot_nix/__init__.py @@ -26,6 +26,7 @@ from buildbot.reporters.utils import getURLForBuildrequest from buildbot.process.buildstep import CANCELLED from buildbot.process.buildstep import EXCEPTION from buildbot.process.buildstep import SUCCESS +from buildbot.process.results import worst_status if TYPE_CHECKING: from buildbot.process.log import Log @@ -72,6 +73,27 @@ class BuildTrigger(steps.BuildStep): self.config = None self.builds_scheduler = builds_scheduler self.skipped_builds_scheduler = skipped_builds_scheduler + self._result_list = [] + self.ended = False + self.waitForFinishDeferred = None + super().__init__(**kwargs) + + def interrupt(self, reason): + # We cancel the buildrequests, as the data api handles + # both cases: + # - build started: stop is sent, + # - build not created yet: related buildrequests are set to CANCELLED. + # Note that there is an identified race condition though (more details + # are available at buildbot.data.buildrequests). + for brid in self.brids: + self.master.data.control( + "cancel", {'reason': 'parent build was interrupted'}, ("buildrequests", brid) + ) + if self.running and not self.ended: + self.ended = True + # if we are interrupted because of a connection lost, we interrupt synchronously + if self.build.conn is None and self.waitForFinishDeferred is not None: + self.waitForFinishDeferred.cancel() def getSchedulerByName(self, name): schedulers = self.master.scheduler_manager.namedServices @@ -82,6 +104,7 @@ class BuildTrigger(steps.BuildStep): return sch def schedule_one(self, build_props, job): + source = f"nix-eval-lix" attr = job.get("attr", "eval-error") name = attr name = f"hydraJobs.{name}" @@ -115,7 +138,7 @@ class BuildTrigger(steps.BuildStep): return (self.builds_scheduler, props) @defer.inlineCallbacks - def _add_results(self, brid, results): + def _add_results(self, brid): @defer.inlineCallbacks def _is_buildrequest_complete(brid): buildrequest = yield self.master.db.buildrequests.getBuildRequest(brid) @@ -128,6 +151,15 @@ class BuildTrigger(steps.BuildStep): self._result_list.append(build["results"]) self.updateSummary() + def prepareSourcestampListForTrigger(self): + ss_for_trigger = {} + objs_from_build = self.build.getAllSourceStamps() + for ss in objs_from_build: + ss_for_trigger[ss.codebase] = ss.asDict() + + trigger_values = [ss_for_trigger[k] for k in sorted(ss_for_trigger.keys())] + return trigger_values + @defer.inlineCallbacks def run(self): build_props = self.build.getProperties() @@ -154,7 +186,7 @@ class BuildTrigger(steps.BuildStep): sorter = graphlib.TopologicalSorter(all_deps) for item in sorter.static_order(): i = 0 - while i < builds_to_schedule.len(): + while i < len(builds_to_schedule): if item == builds_to_schedule[i].get("drvPath"): build_schedule_order.append(builds_to_schedule[i]) del builds_to_schedule[i] @@ -163,14 +195,20 @@ class BuildTrigger(steps.BuildStep): done = [] scheduled = [] - while len(build_schedule_order) > 0 and len(scheduled) > 0: + failed = [] + all_results = SUCCESS + ss_for_trigger = self.prepareSourcestampListForTrigger() + while len(build_schedule_order) > 0 or len(scheduled) > 0: + print('Scheduling..') schedule_now = [] for build in list(build_schedule_order): if all_deps.get(build.get("drvPath"), []) == []: build_schedule_order.remove(build) schedule_now.append(build) - + if len(schedule_now) == 0: + print(' No builds to schedule found.') for job in schedule_now: + print(f" - {job.get('attr')}") (scheduler, props) = self.schedule_one(build_props, job) scheduler = self.getSchedulerByName(scheduler) @@ -192,21 +230,46 @@ class BuildTrigger(steps.BuildStep): for brid in brids.values(): url = getURLForBuildrequest(self.master, brid) - yield self.addURL(f"{sch.name} #{brid}", url) + yield self.addURL(f"{scheduler.name} #{brid}", url) self._add_results(brid) - + print('Waiting..') wait_for_next = defer.DeferredList([results for _, _, results in scheduled], fireOnOneCallback = True, fireOnOneErrback=True) + self.waitForFinishDeferred = wait_for_next results, index = yield wait_for_next job, brids, _ = scheduled[index] done.append((job, brids, results)) del scheduled[index] - # TODO: remove dep from all_deps - # TODO: calculate final result + result = results[0] + print(f' Found finished build {job.get("attr")}, result {util.Results[result].upper()}') + if result != SUCCESS: + failed_checks = [] + failed_paths = [] + removed = [] + while True: + old_paths = list(failed_paths) + print(failed_checks, old_paths) + for build in list(build_schedule_order): + deps = all_deps.get(build.get("drvPath"), []) + for path in old_paths: + if path in deps: + failed_checks.append(build) + failed_paths.append(build.get("drvPath")) + build_schedule_order.remove(build) + removed.append(build.get("attr")) + + break + if old_paths == failed_paths: + break + print(' Removed jobs: ' + ', '.join(removed)) + all_results = worst_status(result, all_results) + print(f' New result: {util.Results[all_results].upper()}') + for dep in all_deps: + if job.get("drvPath") in all_deps[dep]: + all_deps[dep].remove(job.get("drvPath")) + print('Done!') + return all_results def getCurrentSummary(self) -> dict[str, str]: # noqa: N802 - """The original build trigger will the generic builder name `nix-build` in this case, which is not helpful""" - if not self.triggeredNames: - return {"step": "running"} summary = [] if self._result_list: for status in ALL_RESULTS: @@ -850,6 +913,7 @@ class GerritNixConfigurator(ConfiguratorBase): config["change_source"] = self.gerrit_change_source config["workers"].append(worker.LocalWorker(SKIPPED_BUILDER_NAME)) + """ config["services"].append( reporters.GerritStatusPush(self.gerrit_server, self.gerrit_user, port=2022, @@ -865,6 +929,7 @@ class GerritNixConfigurator(ConfiguratorBase): # summaryArg=self.url) ) + """ systemd_secrets = secrets.SecretInAFile( dirname=os.environ["CREDENTIALS_DIRECTORY"],