From f592ce0026e58e6c1d3f0ce268497c0c2ac0f340 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Fri, 11 Oct 2013 10:46:40 +0200
Subject: [PATCH] Fix extreme slowness in hydra-queue-runner

If there are builds in the queue that depend on another scheduled
build, then hydra-queue-runner will start the dependency first and
block the dependent builds.  This is implemented in
findBuildDependencyInQueue.  However, if there are tens of thousands
of such dependent builds, since each call to
findBuildDependencyInQueue may take a second or so, hydra-queue-runner
will spend hours just deciding which builds *not* to do.  Thus very
little progress is made.

So now, when a build is started, we immediately check which builds are
"blocked" by it (i.e. depend on it), and remove such builds from
consideration.
---
 src/script/hydra-queue-runner | 31 ++++++++++++++++++++++++++-----
 1 file changed, 26 insertions(+), 5 deletions(-)

diff --git a/src/script/hydra-queue-runner b/src/script/hydra-queue-runner
index 4dbb88db..f67d8cc2 100755
--- a/src/script/hydra-queue-runner
+++ b/src/script/hydra-queue-runner
@@ -9,6 +9,7 @@ use Hydra::Helper::Nix;
 use Hydra::Model::DB;
 use IO::Handle;
 use Nix::Store;
+use Set::Scalar;
 
 chdir Hydra::Model::DB::getHydraPath or die;
 my $db = Hydra::Model::DB->new();
@@ -52,14 +53,25 @@ sub findBuildDependencyInQueue {
     my @deps = grep { /\.drv$/ && $_ ne $build->drvpath } computeFSClosure(0, 0, $build->drvpath);
     return unless scalar @deps > 0;
     foreach my $d (@deps) {
-        my $b = $buildsByDrv->{$d};
-        next unless defined $b;
-        return $db->resultset('Builds')->find($b);
+        my $bs = $buildsByDrv->{$d};
+        next unless defined $bs;
+        return $db->resultset('Builds')->find((@$bs)[0]);
     }
     return undef;
 }
 
 
+sub blockBuilds {
+    my ($buildsByDrv, $blockedBuilds, $build) = @_;
+    my @rdeps = grep { /\.drv$/ && $_ ne $build->drvpath } computeFSClosure(1, 0, $build->drvpath);
+    foreach my $drv (@rdeps) {
+        my $bs = $buildsByDrv->{$drv};
+        next if !defined $bs;
+        $blockedBuilds->insert($_) foreach @$bs;
+    }
+}
+
+
 sub checkBuilds {
     # print "looking for runnable builds...\n";
 
@@ -80,8 +92,13 @@ sub checkBuilds {
         # Cache scheduled builds by derivation path to speed up
         # findBuildDependencyInQueue.
         my $buildsByDrv = {};
-        $buildsByDrv->{$_->drvpath} = $_->id
-            foreach $db->resultset('Builds')->search({ finished => 0 }, { join => ['project'] });
+        push @{$buildsByDrv->{$_->drvpath}}, $_->id
+            foreach $db->resultset('Builds')->search({ finished => 0 });
+
+        # Builds in the queue of which a dependency is already building.
+        my $blockedBuilds = Set::Scalar->new();
+        blockBuilds($buildsByDrv, $blockedBuilds, $_)
+            foreach $db->resultset('Builds')->search({ finished => 0, busy => 1 });
 
         # Get the system types for the runnable builds.
         my @systemTypes = $db->resultset('Builds')->search(
@@ -162,6 +179,8 @@ sub checkBuilds {
                         { order_by => ["priority DESC", "id"] });
 
                     foreach my $build (@builds) {
+                        next if $blockedBuilds->has($build->id);
+
                         # Find a dependency of $build that has no queued
                         # dependencies itself.  This isn't strictly necessary,
                         # but it ensures that Nix builds are done as part of
@@ -187,6 +206,8 @@ sub checkBuilds {
 
                         $timeSpentPerJobset->{$jobset->get_column('project')}->{$jobset->name} += $costPerBuild;
 
+                        blockBuilds($buildsByDrv, $blockedBuilds, $build);
+
                         next j;
                     }
                 }