From 29468995040ae21e0e1c14c1bdbb16ccb514caa8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 9 Aug 2019 19:11:38 +0200
Subject: [PATCH 01/11] Turn hydra-notify into a daemon

It now receives notifications about started/finished builds/steps via
PostgreSQL. This gets rid of the (substantial) overhead of starting
hydra-notify for every event. It also allows other programs (even on
other machines) to listen to Hydra notifications.
---
 src/hydra-queue-runner/builder.cc            |  30 +++---
 src/hydra-queue-runner/hydra-queue-runner.cc | 104 +++----------------
 src/hydra-queue-runner/queue-monitor.cc      |   6 +-
 src/hydra-queue-runner/state.hh              |  40 +------
 src/script/hydra-notify                      |  99 ++++++++++++------
 5 files changed, 105 insertions(+), 174 deletions(-)

diff --git a/src/hydra-queue-runner/builder.cc b/src/hydra-queue-runner/builder.cc
index d9a7cbbb..ce65f117 100644
--- a/src/hydra-queue-runner/builder.cc
+++ b/src/hydra-queue-runner/builder.cc
@@ -99,6 +99,8 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
     unsigned int maxSilentTime, buildTimeout;
     unsigned int repeats = step->isDeterministic ? 1 : 0;
 
+    auto conn(dbPool.get());
+
     {
         std::set<Build::ptr> dependents;
         std::set<Step::ptr> steps;
@@ -122,8 +124,10 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
 
         for (auto build2 : dependents) {
             if (build2->drvPath == step->drvPath) {
-              build = build2;
-              enqueueNotificationItem({NotificationItem::Type::BuildStarted, build->id});
+                build = build2;
+                pqxx::work txn(*conn);
+                notifyBuildStarted(txn, build->id);
+                txn.commit();
             }
             {
                 auto i = jobsetRepeats.find(std::make_pair(build2->projectName, build2->jobsetName));
@@ -144,8 +148,6 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
 
     bool quit = buildId == buildOne && step->drvPath == buildDrvPath;
 
-    auto conn(dbPool.get());
-
     RemoteResult result;
     BuildOutput res;
     unsigned int stepNr = 0;
@@ -170,11 +172,6 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
             } catch (...) {
                 ignoreException();
             }
-
-            /* Asynchronously run plugins. FIXME: if we're killed,
-               plugin actions might not be run. Need to ensure
-               at-least-once semantics. */
-            enqueueNotificationItem({NotificationItem::Type::StepFinished, buildId, {}, stepNr, result.logFile});
         }
     });
 
@@ -342,8 +339,12 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
 
         /* Send notification about the builds that have this step as
            the top-level. */
-        for (auto id : buildIDs)
-            enqueueNotificationItem({NotificationItem::Type::BuildFinished, id});
+        {
+            pqxx::work txn(*conn);
+            for (auto id : buildIDs)
+                notifyBuildFinished(txn, id, {});
+            txn.commit();
+        }
 
         /* Wake up any dependent steps that have no other
            dependencies. */
@@ -462,11 +463,10 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
 
         /* Send notification about this build and its dependents. */
         {
-            auto notificationSenderQueue_(notificationSenderQueue.lock());
-            notificationSenderQueue_->push(NotificationItem{NotificationItem::Type::BuildFinished, buildId, dependentIDs});
+            pqxx::work txn(*conn);
+            notifyBuildFinished(txn, buildId, dependentIDs);
+            txn.commit();
         }
-        notificationSenderWakeup.notify_one();
-
     }
 
     // FIXME: keep stats about aborted steps?
diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc
index 1c82cab3..27d5bdcf 100644
--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -299,6 +299,9 @@ void State::finishBuildStep(pqxx::work & txn, const RemoteResult & result,
         (result.timesBuilt, result.timesBuilt > 0)
         (result.isNonDeterministic, result.timesBuilt > 1)
         .exec();
+    assert(result.logFile.find('\'') == std::string::npos);
+    txn.exec(fmt("notify step_finished, '%d %d %s'", buildId, stepNr,
+            result.logFile.empty() ? "-" : result.logFile));
 }
 
 
@@ -450,74 +453,20 @@ bool State::checkCachedFailure(Step::ptr step, Connection & conn)
 }
 
 
-void State::notificationSender()
+void State::notifyBuildStarted(pqxx::work & txn, BuildID buildId)
 {
-    while (true) {
-        try {
+    txn.exec(fmt("notify build_started, '%s'", buildId));
+}
 
-            NotificationItem item;
-            {
-                auto notificationSenderQueue_(notificationSenderQueue.lock());
-                while (notificationSenderQueue_->empty())
-                    notificationSenderQueue_.wait(notificationSenderWakeup);
-                item = notificationSenderQueue_->front();
-                notificationSenderQueue_->pop();
-            }
 
-            MaintainCount<counter> mc(nrNotificationsInProgress);
-
-            printMsg(lvlChatty, format("sending notification about build %1%") % item.id);
-
-            auto now1 = std::chrono::steady_clock::now();
-
-            Pid pid = startProcess([&]() {
-                Strings argv;
-                switch (item.type) {
-                    case NotificationItem::Type::BuildStarted:
-                        argv = {"hydra-notify", "build-started", std::to_string(item.id)};
-                        for (auto id : item.dependentIds)
-                            argv.push_back(std::to_string(id));
-                        break;
-                    case NotificationItem::Type::BuildFinished:
-                        argv = {"hydra-notify", "build-finished", std::to_string(item.id)};
-                        for (auto id : item.dependentIds)
-                            argv.push_back(std::to_string(id));
-                        break;
-                    case NotificationItem::Type::StepFinished:
-                        argv = {"hydra-notify", "step-finished", std::to_string(item.id), std::to_string(item.stepNr), item.logPath};
-                        break;
-                };
-                printMsg(lvlChatty, "Executing hydra-notify " + concatStringsSep(" ", argv));
-                execvp("hydra-notify", (char * *) stringsToCharPtrs(argv).data()); // FIXME: remove cast
-                throw SysError("cannot start hydra-notify");
-            });
-
-            int res = pid.wait();
-
-            if (!statusOk(res))
-                throw Error("notification about build %d failed: %s", item.id, statusToString(res));
-
-            auto now2 = std::chrono::steady_clock::now();
-
-            if (item.type == NotificationItem::Type::BuildFinished) {
-                auto conn(dbPool.get());
-                pqxx::work txn(*conn);
-                txn.parameterized
-                    ("update Builds set notificationPendingSince = null where id = $1")
-                    (item.id)
-                    .exec();
-                txn.commit();
-            }
-
-            nrNotificationTimeMs += std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
-            nrNotificationsDone++;
-
-        } catch (std::exception & e) {
-            nrNotificationsFailed++;
-            printMsg(lvlError, format("notification sender: %1%") % e.what());
-            sleep(5);
-        }
-    }
+void State::notifyBuildFinished(pqxx::work & txn, BuildID buildId,
+    const std::vector<BuildID> & dependentIds)
+{
+    auto payload = fmt("%d ", buildId);
+    for (auto & d : dependentIds)
+        payload += fmt("%d ", d);
+    // FIXME: apparently parameterized() doesn't support NOTIFY.
+    txn.exec(fmt("notify build_finished, '%s'", payload));
 }
 
 
@@ -589,13 +538,6 @@ void State::dumpStatus(Connection & conn, bool log)
         root.attr("nrDbConnections", dbPool.count());
         root.attr("nrActiveDbUpdates", nrActiveDbUpdates);
         root.attr("memoryTokensInUse", memoryTokens.currentUse());
-        root.attr("nrNotificationsDone", nrNotificationsDone);
-        root.attr("nrNotificationsFailed", nrNotificationsFailed);
-        root.attr("nrNotificationsInProgress", nrNotificationsInProgress);
-        root.attr("nrNotificationsPending", notificationSenderQueue.lock()->size());
-        root.attr("nrNotificationTimeMs", nrNotificationTimeMs);
-        uint64_t nrNotificationsTotal = nrNotificationsDone + nrNotificationsFailed;
-        root.attr("nrNotificationTimeAvgMs", nrNotificationsTotal == 0 ? 0.0 : (float) nrNotificationTimeMs / nrNotificationsTotal);
 
         {
             auto nested = root.object("machines");
@@ -843,24 +785,6 @@ void State::run(BuildID buildOne)
 
     std::thread(&State::dispatcher, this).detach();
 
-    /* Idem for notification sending. */
-    auto maxConcurrentNotifications = config->getIntOption("max-concurrent-notifications", 2);
-    for (uint64_t i = 0; i < maxConcurrentNotifications; ++i)
-        std::thread(&State::notificationSender, this).detach();
-
-    /* Enqueue notification items for builds that were finished
-       previously, but for which we didn't manage to send
-       notifications. */
-    {
-        auto conn(dbPool.get());
-        pqxx::work txn(*conn);
-        auto res = txn.parameterized("select id from Builds where notificationPendingSince > 0").exec();
-        for (auto const & row : res) {
-            auto id = row["id"].as<BuildID>();
-            enqueueNotificationItem({NotificationItem::Type::BuildFinished, id});
-        }
-    }
-
     /* Periodically clean up orphaned busy steps in the database. */
     std::thread([&]() {
         while (true) {
diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc
index c10f895b..e657a4b8 100644
--- a/src/hydra-queue-runner/queue-monitor.cc
+++ b/src/hydra-queue-runner/queue-monitor.cc
@@ -193,13 +193,12 @@ bool State::getQueuedBuilds(Connection & conn,
                     (build->id)
                     ((int) (ex.step->drvPath == build->drvPath ? bsFailed : bsDepFailed))
                     (time(0)).exec();
+                notifyBuildFinished(txn, build->id, {});
                 txn.commit();
                 build->finishedInDB = true;
                 nrBuildsDone++;
             }
 
-            enqueueNotificationItem({NotificationItem::Type::BuildFinished, build->id});
-
             return;
         }
 
@@ -230,13 +229,12 @@ bool State::getQueuedBuilds(Connection & conn,
             time_t now = time(0);
             printMsg(lvlInfo, format("marking build %1% as succeeded (cached)") % build->id);
             markSucceededBuild(txn, build, res, true, now, now);
+            notifyBuildFinished(txn, build->id, {});
             txn.commit();
             }
 
             build->finishedInDB = true;
 
-            enqueueNotificationItem({NotificationItem::Type::BuildFinished, build->id});
-
             return;
         }
 
diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh
index 79bbe355..fedca088 100644
--- a/src/hydra-queue-runner/state.hh
+++ b/src/hydra-queue-runner/state.hh
@@ -347,39 +347,6 @@ private:
     counter bytesSent{0};
     counter bytesReceived{0};
     counter nrActiveDbUpdates{0};
-    counter nrNotificationsDone{0};
-    counter nrNotificationsFailed{0};
-    counter nrNotificationsInProgress{0};
-    counter nrNotificationTimeMs{0};
-
-    /* Notification sender work queue. FIXME: if hydra-queue-runner is
-       killed before it has finished sending notifications about a
-       build, then the notifications may be lost. It would be better
-       to mark builds with pending notification in the database. */
-    struct NotificationItem
-    {
-        enum class Type : char {
-           BuildStarted,
-           BuildFinished,
-           StepFinished,
-        };
-        Type type;
-        BuildID id;
-        std::vector<BuildID> dependentIds;
-        unsigned int stepNr;
-        nix::Path logPath;
-    };
-    nix::Sync<std::queue<NotificationItem>> notificationSenderQueue;
-    std::condition_variable notificationSenderWakeup;
-
-    void enqueueNotificationItem(const NotificationItem && item)
-    {
-        {
-            auto notificationSenderQueue_(notificationSenderQueue.lock());
-            notificationSenderQueue_->emplace(item);
-        }
-        notificationSenderWakeup.notify_one();
-    }
 
     /* Specific build to do for --build-one (testing only). */
     BuildID buildOne;
@@ -540,9 +507,10 @@ private:
 
     bool checkCachedFailure(Step::ptr step, Connection & conn);
 
-    /* Thread that asynchronously invokes hydra-notify to send build
-       notifications. */
-    void notificationSender();
+    void notifyBuildStarted(pqxx::work & txn, BuildID buildId);
+
+    void notifyBuildFinished(pqxx::work & txn, BuildID buildId,
+        const std::vector<BuildID> & dependentIds);
 
     /* Acquire the global queue runner lock, or null if somebody else
        has it. */
diff --git a/src/script/hydra-notify b/src/script/hydra-notify
index 83963731..4da648ba 100755
--- a/src/script/hydra-notify
+++ b/src/script/hydra-notify
@@ -5,6 +5,7 @@ use utf8;
 use Hydra::Plugin;
 use Hydra::Helper::Nix;
 use Hydra::Helper::AddBuilds;
+use IO::Select;
 
 STDERR->autoflush(1);
 binmode STDERR, ":encoding(utf8)";
@@ -15,20 +16,37 @@ my $db = Hydra::Model::DB->new();
 
 my @plugins = Hydra::Plugin->instantiate(db => $db, config => $config);
 
-my $cmd = shift @ARGV or die "Syntax: hydra-notify build-started BUILD | build-finished BUILD-ID [BUILD-IDs...] | step-finished BUILD-ID STEP-NR LOG-PATH\n";
+my $dbh = $db->storage->dbh;
 
-my $buildId = shift @ARGV or die;
-my $build = $db->resultset('Builds')->find($buildId)
-    or die "build $buildId does not exist\n";
+$dbh->do("listen build_started");
+$dbh->do("listen build_finished");
+$dbh->do("listen step_finished");
+
+sub buildStarted {
+    my ($buildId) = @_;
+
+    my $build = $db->resultset('Builds')->find($buildId)
+        or die "build $buildId does not exist\n";
+
+    foreach my $plugin (@plugins) {
+        eval { $plugin->buildStarted($build); };
+        if ($@) {
+            print STDERR "$plugin->buildStarted: $@\n";
+        }
+    }
+}
+
+sub buildFinished {
+    my ($build, @deps) = @_;
 
-if ($cmd eq "build-finished") {
     my $project = $build->project;
     my $jobset = $build->jobset;
     if (length($project->declfile) && $jobset->name eq ".jobsets" && $build->iscurrent) {
         handleDeclarativeJobsetBuild($db, $project, $build);
     }
+
     my @dependents;
-    foreach my $id (@ARGV) {
+    foreach my $id (@deps) {
         my $dep = $db->resultset('Builds')->find($id)
             or die "build $id does not exist\n";
         push @dependents, $dep;
@@ -40,33 +58,20 @@ if ($cmd eq "build-finished") {
             print STDERR "$plugin->buildFinished: $@\n";
         }
     }
+
+    $build->update({ notificationpendingsince => undef });
 }
 
-elsif ($cmd eq "build-queued") {
-    foreach my $plugin (@plugins) {
-        eval { $plugin->buildQueued($build); };
-        if ($@) {
-            print STDERR "$plugin->buildQueued: $@\n";
-        }
-    }
-}
+sub stepFinished {
+    my ($buildId, $stepNr, $logPath) = @_;
 
-elsif ($cmd eq "build-started") {
-    foreach my $plugin (@plugins) {
-        eval { $plugin->buildStarted($build); };
-        if ($@) {
-            print STDERR "$plugin->buildStarted: $@\n";
-        }
-    }
-}
+    my $build = $db->resultset('Builds')->find($buildId)
+        or die "build $buildId does not exist\n";
 
-elsif ($cmd eq "step-finished") {
-    die if scalar @ARGV < 2;
-    my $stepNr = shift @ARGV;
     my $step = $build->buildsteps->find({stepnr => $stepNr})
         or die "step $stepNr does not exist\n";
-    my $logPath = shift @ARGV;
-    $logPath = undef if $logPath eq "";
+
+    $logPath = undef if $logPath eq "-";
 
     foreach my $plugin (@plugins) {
         eval { $plugin->stepFinished($step, $logPath); };
@@ -76,6 +81,42 @@ elsif ($cmd eq "step-finished") {
     }
 }
 
-else {
-    die "unknown action ‘$cmd’";
+# Process builds that finished while hydra-notify wasn't running.
+for my $build ($db->resultset('Builds')->search(
+                   { notificationpendingsince => { '!=', undef } }))
+{
+    my $buildId = $build->id;
+    print STDERR "sending notifications for build ${\$buildId}...\n";
+    buildFinished($build);
+}
+
+# Process incoming notifications.
+my $fd = $dbh->func("getfd");
+my $sel = IO::Select->new($fd);
+
+while (1) {
+    $sel->can_read;
+    my $notify = $dbh->func("pg_notifies");
+    next if !$notify;
+
+    my ($channelName, $pid, $payload) = @$notify;
+    #print STDERR "got '$channelName' from $pid: $payload\n";
+
+    my @payload = split / /, $payload;
+
+    eval {
+        if ($channelName eq "build_started") {
+            buildStarted(int($payload[0]));
+        } elsif ($channelName eq "build_finished") {
+            my $buildId = int($payload[0]);
+            my $build = $db->resultset('Builds')->find($buildId)
+                or die "build $buildId does not exist\n";
+            buildFinished($build, @payload[1..$#payload]);
+        } elsif ($channelName eq "step_finished") {
+            stepFinished(int($payload[0]), int($payload[1]));
+        }
+    };
+    if ($@) {
+        print STDERR "error processing message '$payload' on channel '$channelName': $@\n";
+    }
 }

From c7861b85c4c3cc974b27147bbf3cc258b9fe9cc3 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 9 Aug 2019 19:26:44 +0200
Subject: [PATCH 02/11] Add hydra-notify service

---
 hydra-module.nix | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/hydra-module.nix b/hydra-module.nix
index 52eccd40..682967f9 100644
--- a/hydra-module.nix
+++ b/hydra-module.nix
@@ -379,6 +379,21 @@ in
           };
       };
 
+    systemd.services.hydra-notify =
+      { wantedBy = [ "multi-user.target" ];
+        requires = [ "hydra-init.service" ];
+        after = [ "hydra-init.service" ];
+        restartTriggers = [ hydraConf ];
+        environment = env // {
+          PGPASSFILE = "${baseDir}/pgpass-queue-runner"; # grrr
+        };
+        serviceConfig =
+          { ExecStart = "@${cfg.package}/bin/hydra-notify hydra-notify";
+            # FIXME: run this under a less privileged user?
+            User = "hydra-queue-runner";
+          };
+      };
+
     # If there is less than a certain amount of free disk space, stop
     # the queue/evaluator to prevent builds from failing or aborting.
     systemd.services.hydra-check-space =

From 7114d2aceb9d58b8cd9458f3e8f4f05c02f7a9bf Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 12 Aug 2019 15:26:12 +0200
Subject: [PATCH 03/11] Separate payload elements using \t

---
 src/hydra-queue-runner/hydra-queue-runner.cc | 6 +++---
 src/script/hydra-notify                      | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc
index 27d5bdcf..cd873d65 100644
--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -299,9 +299,9 @@ void State::finishBuildStep(pqxx::work & txn, const RemoteResult & result,
         (result.timesBuilt, result.timesBuilt > 0)
         (result.isNonDeterministic, result.timesBuilt > 1)
         .exec();
-    assert(result.logFile.find('\'') == std::string::npos);
-    txn.exec(fmt("notify step_finished, '%d %d %s'", buildId, stepNr,
-            result.logFile.empty() ? "-" : result.logFile));
+    assert(result.logFile.find('\t') == std::string::npos);
+    txn.exec(fmt("notify step_finished, '%d\t%d\t%s'",
+            buildId, stepNr, result.logFile));
 }
 
 
diff --git a/src/script/hydra-notify b/src/script/hydra-notify
index 4da648ba..81dd3c14 100755
--- a/src/script/hydra-notify
+++ b/src/script/hydra-notify
@@ -102,7 +102,7 @@ while (1) {
     my ($channelName, $pid, $payload) = @$notify;
     #print STDERR "got '$channelName' from $pid: $payload\n";
 
-    my @payload = split / /, $payload;
+    my @payload = split /\t/, $payload;
 
     eval {
         if ($channelName eq "build_started") {

From 976d88d6758b8420980bc83aafbb4ab5af1e4579 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 12 Aug 2019 15:26:35 +0200
Subject: [PATCH 04/11] Send notifications when evaluations start/finish/fail

* 'eval_started' has the format '<tmpId>\t<project>\t<jobset>'.

* 'eval_failed' has the format '<tmpId>'. (The cause of the error can
  be found in the database.)

* 'eval_added' has the format '<tmpId>:<evalId>'.
---
 src/script/hydra-eval-jobset | 21 +++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/src/script/hydra-eval-jobset b/src/script/hydra-eval-jobset
index 9d375c1b..5b31c735 100755
--- a/src/script/hydra-eval-jobset
+++ b/src/script/hydra-eval-jobset
@@ -568,7 +568,7 @@ sub permute {
 
 
 sub checkJobsetWrapped {
-    my ($jobset) = @_;
+    my ($jobset, $tmpId) = @_;
     my $project = $jobset->project;
     my $jobsetsJobset = length($project->declfile) && $jobset->name eq ".jobsets";
     my $inputInfo = {};
@@ -607,6 +607,7 @@ sub checkJobsetWrapped {
         print STDERR $fetchError;
         txn_do($db, sub {
             $jobset->update({ lastcheckedtime => time, fetcherrormsg => $fetchError }) if !$dryRun;
+            $db->storage->dbh->do("notify eval_failed, ?", undef, join('\t', $tmpId));
         });
         return;
     }
@@ -622,6 +623,7 @@ sub checkJobsetWrapped {
         Net::Statsd::increment("hydra.evaluator.unchanged_checkouts");
         txn_do($db, sub {
             $jobset->update({ lastcheckedtime => time, fetcherrormsg => undef });
+            $db->storage->dbh->do("notify eval_cached, ?", undef, join('\t', $tmpId));
         });
         return;
     }
@@ -690,6 +692,9 @@ sub checkJobsetWrapped {
             , nrbuilds => $jobsetChanged ? scalar(keys %buildMap) : undef
             });
 
+        $db->storage->dbh->do("notify eval_added, ?", undef,
+                              join('\t', $tmpId, $ev->id));
+
         if ($jobsetChanged) {
             # Create JobsetEvalMembers mappings.
             while (my ($id, $x) = each %buildMap) {
@@ -767,10 +772,6 @@ sub checkJobsetWrapped {
     Net::Statsd::increment("hydra.evaluator.evals");
     Net::Statsd::increment("hydra.evaluator.cached_evals") unless $jobsetChanged;
 
-    #while (my ($id, $x) = each %buildMap) {
-    #    system("hydra-notify build-queued $id") if $x->{new};
-    #}
-
     # Store the error messages for jobs that failed to evaluate.
     my $msg = "";
     foreach my $job (values %{$jobs}) {
@@ -788,8 +789,15 @@ sub checkJobset {
 
     my $startTime = clock_gettime(CLOCK_MONOTONIC);
 
+    # Add an ID to eval_* notifications so receivers can correlate
+    # them.
+    my $tmpId = "${startTime}.$$";
+
+    $db->storage->dbh->do("notify eval_started, ?", undef,
+                          join('\t', $tmpId, $jobset->get_column('project'), $jobset->name));
+
     eval {
-        checkJobsetWrapped($jobset);
+        checkJobsetWrapped($jobset, $tmpId);
     };
     my $checkError = $@;
 
@@ -802,6 +810,7 @@ sub checkJobset {
         txn_do($db, sub {
             $jobset->update({lastcheckedtime => time});
             setJobsetError($jobset, $checkError);
+            $db->storage->dbh->do("notify eval_failed, ?", undef, join('\t', $tmpId));
         }) if !$dryRun;
         $failed = 1;
     }

From 7c7cc8c059343ea0a27029b6af58075b63c432d4 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 12 Aug 2019 15:50:43 +0200
Subject: [PATCH 05/11] Fix duplicate step_finished notification

---
 src/hydra-queue-runner/builder.cc | 29 ++++++++++++-----------------
 1 file changed, 12 insertions(+), 17 deletions(-)

diff --git a/src/hydra-queue-runner/builder.cc b/src/hydra-queue-runner/builder.cc
index ce65f117..edd4b1f7 100644
--- a/src/hydra-queue-runner/builder.cc
+++ b/src/hydra-queue-runner/builder.cc
@@ -228,6 +228,11 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
     time_t stepStopTime = time(0);
     if (!result.stopTime) result.stopTime = stepStopTime;
 
+    /* For standard failures, we don't care about the error
+       message. */
+    if (result.stepStatus != bsAborted)
+        result.errorMsg = "";
+
     /* Account the time we spent building this step by dividing it
        among the jobsets that depend on it. */
     {
@@ -240,6 +245,13 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
         }
     }
 
+    /* Finish the step in the database. */
+    if (stepNr) {
+        pqxx::work txn(*conn);
+        finishBuildStep(txn, result, buildId, stepNr, machine->sshName);
+        txn.commit();
+    }
+
     /* The step had a hopefully temporary failure (e.g. network
        issue). Retry a number of times. */
     if (result.canRetry) {
@@ -253,11 +265,6 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
         }
         if (retry) {
             auto mc = startDbUpdate();
-            {
-            pqxx::work txn(*conn);
-            finishBuildStep(txn, result, buildId, stepNr, machine->sshName);
-            txn.commit();
-            }
             stepFinished = true;
             if (quit) exit(1);
             return sRetry;
@@ -312,8 +319,6 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
 
                 pqxx::work txn(*conn);
 
-                finishBuildStep(txn, result, buildId, stepNr, machine->sshName);
-
                 for (auto & b : direct) {
                     printMsg(lvlInfo, format("marking build %1% as succeeded") % b->id);
                     markSucceededBuild(txn, b, res, buildId != b->id || result.isCached,
@@ -370,11 +375,6 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
 
     } else {
 
-        /* For standard failures, we don't care about the error
-           message. */
-        if (result.stepStatus != bsAborted)
-            result.errorMsg = "";
-
         /* Register failure in the database for all Build objects that
            directly or indirectly depend on this step. */
 
@@ -420,11 +420,6 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
                         result.stepStatus, result.errorMsg, buildId == build2->id ? 0 : buildId);
                 }
 
-                if (result.stepStatus != bsCachedFailure && !stepFinished) {
-                    assert(stepNr);
-                    finishBuildStep(txn, result, buildId, stepNr, machine->sshName);
-                }
-
                 /* Mark all builds that depend on this derivation as failed. */
                 for (auto & build2 : indirect) {
                     if (build2->finishedInDB) continue;

From 72c36373bb952249eb4ba9e75873dc3d77b7011d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 12 Aug 2019 17:28:21 +0200
Subject: [PATCH 06/11] hydra-notify: Fix processing notifications

---
 src/script/hydra-notify | 37 +++++++++++++++++++------------------
 1 file changed, 19 insertions(+), 18 deletions(-)

diff --git a/src/script/hydra-notify b/src/script/hydra-notify
index 81dd3c14..f67fb44e 100755
--- a/src/script/hydra-notify
+++ b/src/script/hydra-notify
@@ -96,27 +96,28 @@ my $sel = IO::Select->new($fd);
 
 while (1) {
     $sel->can_read;
-    my $notify = $dbh->func("pg_notifies");
-    next if !$notify;
 
-    my ($channelName, $pid, $payload) = @$notify;
-    #print STDERR "got '$channelName' from $pid: $payload\n";
+    while (my $notify = $dbh->func("pg_notifies")) {
 
-    my @payload = split /\t/, $payload;
+        my ($channelName, $pid, $payload) = @$notify;
+        #print STDERR "got '$channelName' from $pid: $payload\n";
 
-    eval {
-        if ($channelName eq "build_started") {
-            buildStarted(int($payload[0]));
-        } elsif ($channelName eq "build_finished") {
-            my $buildId = int($payload[0]);
-            my $build = $db->resultset('Builds')->find($buildId)
-                or die "build $buildId does not exist\n";
-            buildFinished($build, @payload[1..$#payload]);
-        } elsif ($channelName eq "step_finished") {
-            stepFinished(int($payload[0]), int($payload[1]));
+        my @payload = split /\t/, $payload;
+
+        eval {
+            if ($channelName eq "build_started") {
+                buildStarted(int($payload[0]));
+            } elsif ($channelName eq "build_finished") {
+                my $buildId = int($payload[0]);
+                my $build = $db->resultset('Builds')->find($buildId)
+                    or die "build $buildId does not exist\n";
+                buildFinished($build, @payload[1..$#payload]);
+            } elsif ($channelName eq "step_finished") {
+                stepFinished(int($payload[0]), int($payload[1]));
+            }
+        };
+        if ($@) {
+            print STDERR "error processing message '$payload' on channel '$channelName': $@\n";
         }
-    };
-    if ($@) {
-        print STDERR "error processing message '$payload' on channel '$channelName': $@\n";
     }
 }

From d08cfa48d784dc75bfc59c444780a5a0487506aa Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 12 Aug 2019 17:34:01 +0200
Subject: [PATCH 07/11] Add a 'step_started' notification

---
 src/hydra-queue-runner/hydra-queue-runner.cc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc
index cd873d65..acc838c7 100644
--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -268,6 +268,9 @@ unsigned int State::createBuildStep(pqxx::work & txn, time_t startTime, BuildID
             ("insert into BuildStepOutputs (build, stepnr, name, path) values ($1, $2, $3, $4)")
             (buildId)(stepNr)(output.first)(output.second.path).exec();
 
+    if (status == bsBusy)
+        txn.exec(fmt("notify step_started, '%d\t%d'", buildId, stepNr));
+
     return stepNr;
 }
 

From e2537f741b96a546ab81928f2e11fbf2d4180cf8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 13 Aug 2019 11:19:10 +0200
Subject: [PATCH 08/11] Restart hydra-notify

---
 hydra-module.nix | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/hydra-module.nix b/hydra-module.nix
index 682967f9..d601ac37 100644
--- a/hydra-module.nix
+++ b/hydra-module.nix
@@ -391,6 +391,8 @@ in
           { ExecStart = "@${cfg.package}/bin/hydra-notify hydra-notify";
             # FIXME: run this under a less privileged user?
             User = "hydra-queue-runner";
+            Restart = "always";
+            RestartSec = 5;
           };
       };
 

From f49a089fc0c80ea9f39d15f1becb02053672d6eb Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 13 Aug 2019 17:19:40 +0200
Subject: [PATCH 09/11] hydra-notify: Don't do an unnecessary fetch of Jobsets

---
 src/script/hydra-notify | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/script/hydra-notify b/src/script/hydra-notify
index f67fb44e..c8b5b0de 100755
--- a/src/script/hydra-notify
+++ b/src/script/hydra-notify
@@ -40,8 +40,8 @@ sub buildFinished {
     my ($build, @deps) = @_;
 
     my $project = $build->project;
-    my $jobset = $build->jobset;
-    if (length($project->declfile) && $jobset->name eq ".jobsets" && $build->iscurrent) {
+    my $jobsetName = $build->get_column('jobset');
+    if (length($project->declfile) && $jobsetName eq ".jobsets" && $build->iscurrent) {
         handleDeclarativeJobsetBuild($db, $project, $build);
     }
 

From 16811d3e78255cec75f935cad94ca3b4d58109b7 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 13 Aug 2019 17:20:16 +0200
Subject: [PATCH 10/11] Plugins: Add isEnabled method

Plugins are now disabled at startup time unless there is some relevant
configuration in hydra.conf. This avoids hydra-notify having to do a
lot of redundant work (a lot of plugins did a lot of database queries
*before* deciding they were disabled).

Note: BitBucketStatus users will need to add 'enable_bitbucket_status
= 1' to hydra.conf.
---
 src/lib/Hydra/Plugin.pm                      | 6 +++++-
 src/lib/Hydra/Plugin/BitBucketStatus.pm      | 5 +++++
 src/lib/Hydra/Plugin/CircleCINotification.pm | 5 +++++
 src/lib/Hydra/Plugin/CoverityScan.pm         | 5 +++++
 src/lib/Hydra/Plugin/EmailNotification.pm    | 6 ++++--
 src/lib/Hydra/Plugin/GithubStatus.pm         | 5 +++++
 src/lib/Hydra/Plugin/GitlabStatus.pm         | 5 +++++
 src/lib/Hydra/Plugin/HipChatNotification.pm  | 5 +++++
 src/lib/Hydra/Plugin/InfluxDBNotification.pm | 5 +++++
 src/lib/Hydra/Plugin/RunCommand.pm           | 5 +++++
 src/lib/Hydra/Plugin/S3Backup.pm             | 5 +++++
 src/lib/Hydra/Plugin/SlackNotification.pm    | 5 +++++
 12 files changed, 59 insertions(+), 3 deletions(-)

diff --git a/src/lib/Hydra/Plugin.pm b/src/lib/Hydra/Plugin.pm
index 8177d9ce..119d2738 100644
--- a/src/lib/Hydra/Plugin.pm
+++ b/src/lib/Hydra/Plugin.pm
@@ -12,11 +12,15 @@ sub new {
     return $self;
 }
 
+sub isEnabled {
+    return 1;
+}
+
 sub instantiate {
     my ($class, %args) = @_;
     my $plugins = [];
     $args{plugins} = $plugins;
-    push @$plugins, $class->plugins(%args);
+    push @$plugins, grep { $_->isEnabled } $class->plugins(%args);
     return @$plugins;
 }
 
diff --git a/src/lib/Hydra/Plugin/BitBucketStatus.pm b/src/lib/Hydra/Plugin/BitBucketStatus.pm
index e02c6c71..f209c6f1 100644
--- a/src/lib/Hydra/Plugin/BitBucketStatus.pm
+++ b/src/lib/Hydra/Plugin/BitBucketStatus.pm
@@ -7,6 +7,11 @@ use JSON;
 use LWP::UserAgent;
 use Hydra::Helper::CatalystUtils;
 
+sub isEnabled {
+    my ($self) = @_;
+    return $self->{config}->{enable_bitbucket_status} == 1;
+}
+
 sub toBitBucketState {
     my ($buildStatus) = @_;
     if ($buildStatus == 0) {
diff --git a/src/lib/Hydra/Plugin/CircleCINotification.pm b/src/lib/Hydra/Plugin/CircleCINotification.pm
index e42fb565..e9e8623d 100644
--- a/src/lib/Hydra/Plugin/CircleCINotification.pm
+++ b/src/lib/Hydra/Plugin/CircleCINotification.pm
@@ -7,6 +7,11 @@ use LWP::UserAgent;
 use Hydra::Helper::CatalystUtils;
 use JSON;
 
+sub isEnabled {
+    my ($self) = @_;
+    return defined $self->{config}->{circleci};
+}
+
 sub buildFinished {
     my ($self, $build, $dependents) = @_;
     my $cfg = $self->{config}->{circleci};
diff --git a/src/lib/Hydra/Plugin/CoverityScan.pm b/src/lib/Hydra/Plugin/CoverityScan.pm
index 924111f4..1ee60843 100644
--- a/src/lib/Hydra/Plugin/CoverityScan.pm
+++ b/src/lib/Hydra/Plugin/CoverityScan.pm
@@ -6,6 +6,11 @@ use File::Basename;
 use LWP::UserAgent;
 use Hydra::Helper::CatalystUtils;
 
+sub isEnabled {
+    my ($self) = @_;
+    return defined $self->{config}->{coverityscan};
+}
+
 sub buildFinished {
     my ($self, $b, $dependents) = @_;
 
diff --git a/src/lib/Hydra/Plugin/EmailNotification.pm b/src/lib/Hydra/Plugin/EmailNotification.pm
index c2887631..9c2639d1 100644
--- a/src/lib/Hydra/Plugin/EmailNotification.pm
+++ b/src/lib/Hydra/Plugin/EmailNotification.pm
@@ -9,6 +9,10 @@ use Hydra::Helper::Nix;
 use Hydra::Helper::CatalystUtils;
 use Hydra::Helper::Email;
 
+sub isEnabled {
+    my ($self) = @_;
+    return $self->{config}->{email_notification} == 1;
+}
 
 my $template = <<EOF;
 Hi,
@@ -44,8 +48,6 @@ EOF
 sub buildFinished {
     my ($self, $build, $dependents) = @_;
 
-    return unless $self->{config}->{email_notification} // 0;
-
     die unless $build->finished;
 
     # Figure out to whom to send notification for each build.  For
diff --git a/src/lib/Hydra/Plugin/GithubStatus.pm b/src/lib/Hydra/Plugin/GithubStatus.pm
index 59b0dc06..7db6bb52 100644
--- a/src/lib/Hydra/Plugin/GithubStatus.pm
+++ b/src/lib/Hydra/Plugin/GithubStatus.pm
@@ -8,6 +8,11 @@ use LWP::UserAgent;
 use Hydra::Helper::CatalystUtils;
 use List::Util qw(max);
 
+sub isEnabled {
+    my ($self) = @_;
+    return defined $self->{config}->{githubstatus};
+}
+
 sub toGithubState {
     my ($buildStatus) = @_;
     if ($buildStatus == 0) {
diff --git a/src/lib/Hydra/Plugin/GitlabStatus.pm b/src/lib/Hydra/Plugin/GitlabStatus.pm
index 4aef2f09..55c01472 100644
--- a/src/lib/Hydra/Plugin/GitlabStatus.pm
+++ b/src/lib/Hydra/Plugin/GitlabStatus.pm
@@ -16,6 +16,11 @@ use List::Util qw(max);
 #   - gitlab_project_id => ID of the project in Gitlab, i.e. in the above
 #     case the ID in gitlab of "nixexprs"
 
+sub isEnabled {
+    my ($self) = @_;
+    return defined $self->{config}->{gitlab_authorization};
+}
+
 sub toGitlabState {
     my ($status, $buildStatus) = @_;
     if ($status == 0) {
diff --git a/src/lib/Hydra/Plugin/HipChatNotification.pm b/src/lib/Hydra/Plugin/HipChatNotification.pm
index d8e907f1..7df39593 100644
--- a/src/lib/Hydra/Plugin/HipChatNotification.pm
+++ b/src/lib/Hydra/Plugin/HipChatNotification.pm
@@ -5,6 +5,11 @@ use parent 'Hydra::Plugin';
 use LWP::UserAgent;
 use Hydra::Helper::CatalystUtils;
 
+sub isEnabled {
+    my ($self) = @_;
+    return defined $self->{config}->{hipchat};
+}
+
 sub buildFinished {
     my ($self, $build, $dependents) = @_;
 
diff --git a/src/lib/Hydra/Plugin/InfluxDBNotification.pm b/src/lib/Hydra/Plugin/InfluxDBNotification.pm
index bef83d1c..89732e3b 100644
--- a/src/lib/Hydra/Plugin/InfluxDBNotification.pm
+++ b/src/lib/Hydra/Plugin/InfluxDBNotification.pm
@@ -7,6 +7,11 @@ use HTTP::Request;
 use LWP::UserAgent;
 # use Hydra::Helper::CatalystUtils;
 
+sub isEnabled {
+    my ($self) = @_;
+    return defined $self->{config}->{influxdb};
+}
+
 sub toBuildStatusDetailed {
     my ($buildStatus) = @_;
     if ($buildStatus == 0) {
diff --git a/src/lib/Hydra/Plugin/RunCommand.pm b/src/lib/Hydra/Plugin/RunCommand.pm
index ca2140c5..f7ca45a9 100644
--- a/src/lib/Hydra/Plugin/RunCommand.pm
+++ b/src/lib/Hydra/Plugin/RunCommand.pm
@@ -5,6 +5,11 @@ use parent 'Hydra::Plugin';
 use experimental 'smartmatch';
 use JSON;
 
+sub isEnabled {
+    my ($self) = @_;
+    return defined $self->{config}->{runcommand};
+}
+
 sub configSectionMatches {
     my ($name, $project, $jobset, $job) = @_;
 
diff --git a/src/lib/Hydra/Plugin/S3Backup.pm b/src/lib/Hydra/Plugin/S3Backup.pm
index a6d3ac85..346cf239 100644
--- a/src/lib/Hydra/Plugin/S3Backup.pm
+++ b/src/lib/Hydra/Plugin/S3Backup.pm
@@ -14,6 +14,11 @@ use Nix::Store;
 use Hydra::Model::DB;
 use Hydra::Helper::CatalystUtils;
 
+sub isEnabled {
+    my ($self) = @_;
+    return defined $self->{config}->{s3backup};
+}
+
 my $client;
 my %compressors = (
     xz => "| $Nix::Config::xz",
diff --git a/src/lib/Hydra/Plugin/SlackNotification.pm b/src/lib/Hydra/Plugin/SlackNotification.pm
index cffb0d5f..6ee249c8 100644
--- a/src/lib/Hydra/Plugin/SlackNotification.pm
+++ b/src/lib/Hydra/Plugin/SlackNotification.pm
@@ -7,6 +7,11 @@ use LWP::UserAgent;
 use Hydra::Helper::CatalystUtils;
 use JSON;
 
+sub isEnabled {
+    my ($self) = @_;
+    return defined $self->{config}->{slack};
+}
+
 sub renderDuration {
     my ($build) = @_;
     my $duration = $build->stoptime - $build->starttime;

From 92d8d6baa5334466844fdc9aef7f8cd40a51d679 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 13 Aug 2019 17:42:19 +0200
Subject: [PATCH 11/11] Avoid fetching Projects/Jobsets just to get the name
 column

In particular, doing a 'select * from Jobsets where ...' must be
avoided, because the 'errormsg' column can be very big.
---
 src/lib/Hydra/Controller/API.pm              |  6 +++---
 src/lib/Hydra/Controller/JobsetEval.pm       |  2 +-
 src/lib/Hydra/Helper/CatalystUtils.pm        | 14 +++++++-------
 src/lib/Hydra/Plugin/EmailNotification.pm    |  8 ++++----
 src/lib/Hydra/Plugin/GitlabStatus.pm         |  2 +-
 src/lib/Hydra/Plugin/HipChatNotification.pm  |  2 +-
 src/lib/Hydra/Plugin/InfluxDBNotification.pm |  8 ++++----
 src/lib/Hydra/Plugin/SlackNotification.pm    |  2 +-
 src/script/hydra-eval-jobset                 |  6 +++---
 9 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/src/lib/Hydra/Controller/API.pm b/src/lib/Hydra/Controller/API.pm
index dfde7b30..45cecfea 100644
--- a/src/lib/Hydra/Controller/API.pm
+++ b/src/lib/Hydra/Controller/API.pm
@@ -76,7 +76,7 @@ sub latestbuilds : Chained('api') PathPart('latestbuilds') Args(0) {
 sub jobsetToHash {
     my ($jobset) = @_;
     return {
-        project => $jobset->project->name,
+        project => $jobset->get_column('project'),
         name => $jobset->name,
         nrscheduled => $jobset->get_column("nrscheduled"),
         nrsucceeded => $jobset->get_column("nrsucceeded"),
@@ -206,12 +206,12 @@ sub scmdiff : Path('/api/scmdiff') Args(0) {
 
 sub triggerJobset {
     my ($self, $c, $jobset, $force) = @_;
-    print STDERR "triggering jobset ", $jobset->project->name . ":" . $jobset->name, "\n";
+    print STDERR "triggering jobset ", $jobset->get_column('project') . ":" . $jobset->name, "\n";
     txn_do($c->model('DB')->schema, sub {
         $jobset->update({ triggertime => time });
         $jobset->update({ forceeval => 1 }) if $force;
     });
-    push @{$c->{stash}->{json}->{jobsetsTriggered}}, $jobset->project->name . ":" . $jobset->name;
+    push @{$c->{stash}->{json}->{jobsetsTriggered}}, $jobset->get_column('project') . ":" . $jobset->name;
 }
 
 
diff --git a/src/lib/Hydra/Controller/JobsetEval.pm b/src/lib/Hydra/Controller/JobsetEval.pm
index 12af29c2..77a4385f 100644
--- a/src/lib/Hydra/Controller/JobsetEval.pm
+++ b/src/lib/Hydra/Controller/JobsetEval.pm
@@ -142,7 +142,7 @@ sub release : Chained('evalChain') PathPart('release') Args(0) {
     $releaseName ||= $_->releasename foreach @builds;
 
     # If no release name has been defined by any of the builds, compose one of the project name and evaluation id
-    $releaseName = $eval->project->name."-".$eval->id unless defined $releaseName;
+    $releaseName = $eval->get_column('project') . "-" . $eval->id unless defined $releaseName;
 
     my $release;
 
diff --git a/src/lib/Hydra/Helper/CatalystUtils.pm b/src/lib/Hydra/Helper/CatalystUtils.pm
index a6401676..b9019638 100644
--- a/src/lib/Hydra/Helper/CatalystUtils.pm
+++ b/src/lib/Hydra/Helper/CatalystUtils.pm
@@ -60,9 +60,9 @@ sub getNextBuild {
     (my $nextBuild) = $c->model('DB::Builds')->search(
       { finished => 1
       , system => $build->system
-      , project => $build->project->name
-      , jobset => $build->jobset->name
-      , job => $build->job->name
+      , project => $build->get_column('project')
+      , jobset => $build->get_column('jobset')
+      , job => $build->get_column('job')
       , 'me.id' =>  { '>' => $build->id }
       }, {rows => 1, order_by => "me.id ASC"});
 
@@ -77,9 +77,9 @@ sub getPreviousSuccessfulBuild {
     (my $prevBuild) = $c->model('DB::Builds')->search(
       { finished => 1
       , system => $build->system
-      , project => $build->project->name
-      , jobset => $build->jobset->name
-      , job => $build->job->name
+      , project => $build->get_column('project')
+      , jobset => $build->get_column('jobset')
+      , job => $build->get_column('job')
       , buildstatus => 0
       , 'me.id' =>  { '<' => $build->id }
       }, {rows => 1, order_by => "me.id DESC"});
@@ -289,7 +289,7 @@ sub parseJobsetName {
 
 sub showJobName {
     my ($build) = @_;
-    return $build->project->name . ":" . $build->jobset->name . ":" . $build->job->name;
+    return $build->get_column('project') . ":" . $build->get_column('jobset') . ":" . $build->get_column('job');
 }
 
 
diff --git a/src/lib/Hydra/Plugin/EmailNotification.pm b/src/lib/Hydra/Plugin/EmailNotification.pm
index 9c2639d1..33935069 100644
--- a/src/lib/Hydra/Plugin/EmailNotification.pm
+++ b/src/lib/Hydra/Plugin/EmailNotification.pm
@@ -100,7 +100,7 @@ sub buildFinished {
             , dependents => [grep { $_->id != $build->id } @builds]
             , baseurl => getBaseUrl($self->{config})
             , showJobName => \&showJobName, showStatus => \&showStatus
-            , showSystem => index($build->job->name, $build->system) == -1
+            , showSystem => index($build->get_column('job'), $build->system) == -1
             , nrCommits => $nrCommits
             , authorList => $authorList
             };
@@ -119,9 +119,9 @@ sub buildFinished {
 
         sendEmail(
             $self->{config}, $to, $subject, $body,
-            [ 'X-Hydra-Project'  => $build->project->name,
-            , 'X-Hydra-Jobset'   => $build->jobset->name,
-            , 'X-Hydra-Job'      => $build->job->name,
+            [ 'X-Hydra-Project'  => $build->get_column('project'),
+            , 'X-Hydra-Jobset'   => $build->get_column('jobset'),
+            , 'X-Hydra-Job'      => $build->get_column('job'),
             , 'X-Hydra-System'   => $build->system
             ]);
     }
diff --git a/src/lib/Hydra/Plugin/GitlabStatus.pm b/src/lib/Hydra/Plugin/GitlabStatus.pm
index 55c01472..0c9c36ba 100644
--- a/src/lib/Hydra/Plugin/GitlabStatus.pm
+++ b/src/lib/Hydra/Plugin/GitlabStatus.pm
@@ -55,7 +55,7 @@ sub common {
                 state => $state,
                 target_url => "$baseurl/build/" . $b->id,
                 description => "Hydra build #" . $b->id . " of $jobName",
-                name => "Hydra " . $b->job->name,
+                name => "Hydra " . $b->get_column('job'),
             });
         while (my $eval = $evals->next) {
             my $gitlabstatusInput = $eval->jobsetevalinputs->find({ name => "gitlab_status_repo" });
diff --git a/src/lib/Hydra/Plugin/HipChatNotification.pm b/src/lib/Hydra/Plugin/HipChatNotification.pm
index 7df39593..e4d1b74f 100644
--- a/src/lib/Hydra/Plugin/HipChatNotification.pm
+++ b/src/lib/Hydra/Plugin/HipChatNotification.pm
@@ -59,7 +59,7 @@ sub buildFinished {
 
         my $msg = "";
         $msg .= "<img src='$img'/> ";
-        $msg .= "Job <a href='$baseurl/job/${\$build->project->name}/${\$build->jobset->name}/${\$build->job->name}'>${\showJobName($build)}</a>";
+        $msg .= "Job <a href='$baseurl/job/${\$build->get_column('project')}/${\$build->get_column('jobset')}/${\$build->get_column('job')}'>${\showJobName($build)}</a>";
         $msg .= " (and ${\scalar @deps} others)" if scalar @deps > 0;
         $msg .= ": <a href='$baseurl/build/${\$build->id}'>" . showStatus($build) . "</a>";
 
diff --git a/src/lib/Hydra/Plugin/InfluxDBNotification.pm b/src/lib/Hydra/Plugin/InfluxDBNotification.pm
index 89732e3b..3b653201 100644
--- a/src/lib/Hydra/Plugin/InfluxDBNotification.pm
+++ b/src/lib/Hydra/Plugin/InfluxDBNotification.pm
@@ -104,10 +104,10 @@ sub buildFinished {
         my $tagSet = {
             status  => toBuildStatusClass($b->buildstatus),
             result  => toBuildStatusDetailed($b->buildstatus),
-            project => $b->project->name,
-            jobset  => $b->jobset->name,
-            repo    => ($b->jobset->name =~ /^(.*)\.pr-/) ? $1 : $b->jobset->name,
-            job     => $b->job->name,
+            project => $b->get_column('project'),
+            jobset  => $b->get_column('jobset'),
+            repo    => ($b->get_column('jobset') =~ /^(.*)\.pr-/) ? $1 : $b->get_column('jobset'),
+            job     => $b->get_column('job'),
             system  => $b->system,
             cached  => $b->iscachedbuild ? "true" : "false",
         };
diff --git a/src/lib/Hydra/Plugin/SlackNotification.pm b/src/lib/Hydra/Plugin/SlackNotification.pm
index 6ee249c8..96933c51 100644
--- a/src/lib/Hydra/Plugin/SlackNotification.pm
+++ b/src/lib/Hydra/Plugin/SlackNotification.pm
@@ -81,7 +81,7 @@ sub buildFinished {
             "danger";
 
         my $text = "";
-        $text .= "Job <$baseurl/job/${\$build->project->name}/${\$build->jobset->name}/${\$build->job->name}|${\showJobName($build)}>";
+        $text .= "Job <$baseurl/job/${\$build->get_column('project')}/${\$build->get_column('jobset')}/${\$build->get_column('job')}|${\showJobName($build)}>";
         $text .= " (and ${\scalar @deps} others)" if scalar @deps > 0;
         $text .= ": <$baseurl/build/${\$build->id}|" . showStatus($build) . ">". " in " . renderDuration($build);
 
diff --git a/src/script/hydra-eval-jobset b/src/script/hydra-eval-jobset
index 5b31c735..2049fe24 100755
--- a/src/script/hydra-eval-jobset
+++ b/src/script/hydra-eval-jobset
@@ -438,7 +438,7 @@ sub checkBuild {
                 # semantically unnecessary (because they're implied by
                 # the eval), but they give a factor 1000 speedup on
                 # the Nixpkgs jobset with PostgreSQL.
-                { project => $jobset->project->name, jobset => $jobset->name, job => $jobName,
+                { project => $jobset->get_column('project'), jobset => $jobset->name, job => $jobName,
                   name => $firstOutputName, path => $firstOutputPath },
                 { rows => 1, columns => ['id'], join => ['buildoutputs'] });
             if (defined $prevBuild) {
@@ -489,7 +489,7 @@ sub checkBuild {
         $buildMap->{$build->id} = { id => $build->id, jobName => $jobName, new => 1, drvPath => $drvPath };
         $$jobOutPathMap{$jobName . "\t" . $firstOutputPath} = $build->id;
 
-        print STDERR "added build ${\$build->id} (${\$jobset->project->name}:${\$jobset->name}:$jobName)\n";
+        print STDERR "added build ${\$build->id} (${\$jobset->get_column('project')}:${\$jobset->name}:$jobName)\n";
     });
 
     return $build;
@@ -531,7 +531,7 @@ sub sendJobsetErrorNotification() {
     return if $jobset->project->owner->emailonerror == 0;
     return if $errorMsg eq "";
 
-    my $projectName = $jobset->project->name;
+    my $projectName = $jobset->get_column('project');
     my $jobsetName = $jobset->name;
     my $body = "Hi,\n"
         . "\n"