Remove obsolete Builds columns and provide accurate "Running builds"

This removes the "busy", "locker" and "logfile" columns, which are no
longer used by the queue runner. The "Running builds" page now only
shows builds that have an active build step.

src/hydra-queue-runner/builder.cc

@@ -110,11 +110,10 @@ bool State::doBuildStep(std::shared_ptr<StoreAPI> store, Step::ptr step,
     else {
 
         /* Create a build step record indicating that we started
-           building. Also, mark the selected build as busy. */
+           building. */
         {
             pqxx::work txn(*conn);
             stepNr = createBuildStep(txn, result.startTime, build, step, machine->sshName, bssBusy);
-            txn.parameterized("update Builds set busy = 1 where id = $1")(build->id).exec();
             txn.commit();
         }
 
@@ -341,7 +340,7 @@ bool State::doBuildStep(std::shared_ptr<StoreAPI> store, Step::ptr step,
                     if (build2->finishedInDB) continue;
                     printMsg(lvlError, format("marking build %1% as failed") % build2->id);
                     txn.parameterized
-                        ("update Builds set finished = 1, busy = 0, buildStatus = $2, startTime = $3, stopTime = $4, isCachedBuild = $5 where id = $1 and finished = 0")
+                        ("update Builds set finished = 1, buildStatus = $2, startTime = $3, stopTime = $4, isCachedBuild = $5 where id = $1 and finished = 0")
                         (build2->id)
                         ((int) (build2->drvPath != step->drvPath && buildStatus == bsFailed ? bsDepFailed : buildStatus))
                         (result.startTime)

src/hydra-queue-runner/hydra-queue-runner.cc

@@ -159,7 +159,6 @@ void State::clearBusy(Connection & conn, time_t stopTime)
         ("update BuildSteps set busy = 0, status = $1, stopTime = $2 where busy = 1")
         ((int) bssAborted)
         (stopTime, stopTime != 0).exec();
-    txn.exec("update Builds set busy = 0 where finished = 0 and busy = 1");
     txn.commit();
 }
 
@@ -309,7 +308,7 @@ void State::markSucceededBuild(pqxx::work & txn, Build::ptr build,
     if (txn.parameterized("select 1 from Builds where id = $1 and finished = 0")(build->id).exec().empty()) return;
 
     txn.parameterized
-        ("update Builds set finished = 1, busy = 0, buildStatus = $2, startTime = $3, stopTime = $4, size = $5, closureSize = $6, releaseName = $7, isCachedBuild = $8 where id = $1")
+        ("update Builds set finished = 1, buildStatus = $2, startTime = $3, stopTime = $4, size = $5, closureSize = $6, releaseName = $7, isCachedBuild = $8 where id = $1")
         (build->id)
         ((int) (res.failed ? bsFailedWithOutput : bsSuccess))
         (startTime)

src/hydra-queue-runner/queue-monitor.cc

@@ -125,7 +125,7 @@ bool State::getQueuedBuilds(Connection & conn, std::shared_ptr<StoreAPI> store,
             if (!build->finishedInDB) {
                 pqxx::work txn(conn);
                 txn.parameterized
-                    ("update Builds set finished = 1, busy = 0, buildStatus = $2, startTime = $3, stopTime = $3, errorMsg = $4 where id = $1 and finished = 0")
+                    ("update Builds set finished = 1, buildStatus = $2, startTime = $3, stopTime = $3, errorMsg = $4 where id = $1 and finished = 0")
                     (build->id)
                     ((int) bsAborted)
                     (time(0))
@@ -201,7 +201,7 @@ bool State::getQueuedBuilds(Connection & conn, std::shared_ptr<StoreAPI> store,
 
                     createBuildStep(txn, 0, build, r, "", bssCachedFailure, "", propagatedFrom);
                     txn.parameterized
-                        ("update Builds set finished = 1, busy = 0, buildStatus = $2, startTime = $3, stopTime = $3, isCachedBuild = 1 where id = $1 and finished = 0")
+                        ("update Builds set finished = 1, buildStatus = $2, startTime = $3, stopTime = $3, isCachedBuild = 1 where id = $1 and finished = 0")
                         (build->id)
                         ((int) (step == r ? bsFailed : bsDepFailed))
                         (time(0)).exec();

src/lib/Hydra/Controller/API.pm

@@ -38,7 +38,6 @@ sub buildToHash {
     if($build->finished) {
         $result->{'buildstatus'} = $build->get_column("buildstatus");
     } else {
-        $result->{'busy'} = $build->get_column("busy");
         $result->{'priority'} = $build->get_column("priority");
     }
 
@@ -114,7 +113,7 @@ sub queue : Chained('api') PathPart('queue') Args(0) {
     my $nr = $c->request->params->{nr};
     error($c, "Parameter not defined!") if !defined $nr;
 
-    my @builds = $c->model('DB::Builds')->search({finished => 0}, {rows => $nr, order_by => ["busy DESC", "priority DESC", "id"]});
+    my @builds = $c->model('DB::Builds')->search({finished => 0}, {rows => $nr, order_by => ["priority DESC", "id"]});
 
     my @list;
     push @list, buildToHash($_) foreach @builds;
@@ -136,16 +135,6 @@ sub nrqueue : Chained('api') PathPart('nrqueue') Args(0) {
 }
 
 
-sub nrrunning : Chained('api') PathPart('nrrunning') Args(0) {
-    my ($self, $c) = @_;
-    my $nrRunningBuilds = $c->model('DB::Builds')->search({finished => 0, busy => 1 })->count();
-    $c->stash->{'plain'} = {
-        data => "$nrRunningBuilds"
-    };
-    $c->forward('Hydra::View::Plain');
-}
-
-
 sub nrbuilds : Chained('api') PathPart('nrbuilds') Args(0) {
     my ($self, $c) = @_;
     my $nr = $c->request->params->{nr};
@@ -212,44 +201,6 @@ sub scmdiff : Chained('api') PathPart('scmdiff') Args(0) {
 }
 
 
-sub readNormalizedLog {
-    my ($file) = @_;
-    my $pipe = (-f "$file.bz2" ? "cat $file.bz2 | bzip2 -d" : "cat $file");
-    my $res = `$pipe`;
-
-    $res =~ s/\/nix\/store\/[a-z0-9]*-/\/nix\/store\/...-/g;
-    $res =~ s/nix-build-[a-z0-9]*-/nix-build-...-/g;
-    $res =~ s/[0-9]{2}:[0-9]{2}:[0-9]{2}/00:00:00/g;
-    return $res;
-}
-
-
-sub logdiff : Chained('api') PathPart('logdiff') Args(2) {
-    my ($self, $c, $buildid1, $buildid2) = @_;
-
-    my $diff = "";
-
-    my $build1 = getBuild($c, $buildid1);
-    notFound($c, "Build with ID $buildid1 doesn't exist.")
-        if !defined $build1;
-    my $build2 = getBuild($c, $buildid2);
-    notFound($c, "Build with ID $buildid2 doesn't exist.")
-        if !defined $build2;
-
-    if (-f $build1->logfile && -f $build2->logfile) {
-        my $logtext1 = readNormalizedLog($build1->logfile);
-        my $logtext2 = readNormalizedLog($build2->logfile);
-        $diff = diff \$logtext1, \$logtext2;
-    } else {
-        $c->response->status(404);
-    }
-
-    $c->response->content_type('text/x-diff');
-    $c->stash->{'plain'} = { data => (scalar $diff) || " " };
-    $c->forward('Hydra::View::Plain');
-}
-
-
 sub triggerJobset {
     my ($self, $c, $jobset) = @_;
     print STDERR "triggering jobset ", $jobset->project->name . ":" . $jobset->name, "\n";

src/lib/Hydra/Controller/Build.pm

@@ -167,7 +167,7 @@ sub showLog {
         my $url = $c->request->uri->as_string;
         $url =~ s/tail-reload/tail/g;
         $c->stash->{url} = $url;
-        $c->stash->{reload} = !$c->stash->{build}->finished && $c->stash->{build}->busy;
+        $c->stash->{reload} = !$c->stash->{build}->finished;
         $c->stash->{title} = "";
         $c->stash->{contents} = (scalar logContents($logPath, 50)) || " ";
         $c->stash->{template} = 'plain-reload.tt';

src/lib/Hydra/Controller/Root.pm

@@ -44,7 +44,9 @@ sub begin :Private {
     }
 
     if (scalar(@args) == 0 || $args[0] ne "static") {
-        $c->stash->{nrRunningBuilds} = $c->model('DB::Builds')->search({ finished => 0, busy => 1 }, {})->count();
+        $c->stash->{nrRunningBuilds} = $c->model('DB::Builds')->search(
+            { finished => 0, 'buildsteps.busy' => 1 },
+            { join => 'buildsteps', select => ["id"], distinct => 1 })->count();
         $c->stash->{nrQueuedBuilds} = $c->model('DB::Builds')->search({ finished => 0 })->count();
     }
 
@@ -89,7 +91,11 @@ sub queue_GET {
     $c->stash->{flashMsg} //= $c->flash->{buildMsg};
     $self->status_ok(
         $c,
-        entity => [$c->model('DB::Builds')->search({finished => 0}, { order_by => ["globalpriority desc", "id"]})]
+        entity => [$c->model('DB::Builds')->search(
+            { finished => 0 },
+            { order_by => ["globalpriority desc", "id"],
+            , columns => [@buildListColumns]
+            })]
     );
 }
 
@@ -100,7 +106,12 @@ sub status_GET {
     my ($self, $c) = @_;
     $self->status_ok(
         $c,
-        entity => [$c->model('DB::Builds')->search({finished => 0, busy => 1}, { order_by => ["priority DESC", "id"]})]
+        entity => [$c->model('DB::Builds')->search(
+            { finished => 0, "buildsteps.busy" => 1 },
+            { order_by => ["globalpriority DESC", "id"],
+              join => "buildsteps",
+              columns => [@buildListColumns]
+            })]
     );
 }
 
@@ -126,7 +137,7 @@ sub machines :Local Args(0) {
 
     $c->stash->{machines} = $machines;
     $c->stash->{steps} = [ $c->model('DB::BuildSteps')->search(
-        { finished => 0, 'me.busy' => 1, 'build.busy' => 1, },
+        { finished => 0, 'me.busy' => 1, },
         { join => [ 'build' ]
         , order_by => [ 'machine', 'stepnr' ]
         } ) ];

src/lib/Hydra/Helper/AddBuilds.pm

@@ -450,8 +450,6 @@ sub checkBuild {
             , nixexprpath => $jobset->nixexprpath
             , priority => $buildInfo->{schedulingPriority}
             , finished => 0
-            , busy => 0
-            , locker => ""
             , iscurrent => 1
             , ischannel => $buildInfo->{isChannel}
             });

src/lib/Hydra/Helper/CatalystUtils.pm

@@ -29,7 +29,7 @@ our @EXPORT = qw(
 
 
 # Columns from the Builds table needed to render build lists.
-Readonly our @buildListColumns => ('id', 'finished', 'timestamp', 'stoptime', 'project', 'jobset', 'job', 'nixname', 'system', 'priority', 'busy', 'buildstatus', 'releasename');
+Readonly our @buildListColumns => ('id', 'finished', 'timestamp', 'stoptime', 'project', 'jobset', 'job', 'nixname', 'system', 'buildstatus', 'releasename');
 
 
 sub getBuild {

src/lib/Hydra/Helper/Nix.pm

@@ -455,8 +455,6 @@ sub restartBuilds($$) {
     txn_do($db, sub {
         $nrRestarted = $builds->update(
             { finished => 0
-            , busy => 0
-            , locker => ""
             , iscachedbuild => 0
             });
 

src/lib/Hydra/Schema/Builds.pm

@@ -120,7 +120,7 @@ __PACKAGE__->table("Builds");
 
   data_type: 'integer'
   default_value: 0
-  is_nullable: 1
+  is_nullable: 0
 
 =head2 iscurrent
 
@@ -150,22 +150,6 @@ __PACKAGE__->table("Builds");
   default_value: 0
   is_nullable: 0
 
-=head2 busy
-
-  data_type: 'integer'
-  default_value: 0
-  is_nullable: 0
-
-=head2 locker
-
-  data_type: 'text'
-  is_nullable: 1
-
-=head2 logfile
-
-  data_type: 'text'
-  is_nullable: 1
-
 =head2 starttime
 
   data_type: 'integer'
@@ -246,7 +230,7 @@ __PACKAGE__->add_columns(
   "timeout",
   { data_type => "integer", default_value => 36000, is_nullable => 1 },
   "ischannel",
-  { data_type => "integer", default_value => 0, is_nullable => 1 },
+  { data_type => "integer", default_value => 0, is_nullable => 0 },
   "iscurrent",
   { data_type => "integer", default_value => 0, is_nullable => 1 },
   "nixexprinput",
@@ -257,12 +241,6 @@ __PACKAGE__->add_columns(
   { data_type => "integer", default_value => 0, is_nullable => 0 },
   "globalpriority",
   { data_type => "integer", default_value => 0, is_nullable => 0 },
-  "busy",
-  { data_type => "integer", default_value => 0, is_nullable => 0 },
-  "locker",
-  { data_type => "text", is_nullable => 1 },
-  "logfile",
-  { data_type => "text", is_nullable => 1 },
   "starttime",
   { data_type => "integer", is_nullable => 1 },
   "stoptime",
@@ -566,8 +544,8 @@ __PACKAGE__->many_to_many(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.07043 @ 2015-09-10 17:34:23
+# Created by DBIx::Class::Schema::Loader v0.07043 @ 2015-10-27 13:54:16
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:JRelp13Cyfi+QVxC92xuqQ
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:iN9vwO5i3dURKOTgaPTHxw
 
 __PACKAGE__->has_many(
   "dependents",
@@ -647,7 +625,6 @@ my %hint = (
         'nixname',
         'system',
         'priority',
-        'busy',
         'buildstatus',
         'releasename'
     ],

src/root/build.tt

@@ -4,7 +4,12 @@
 [% USE HTML %]
 [% USE Date %]
 
-[% isAggregate = constituents.size > 0 %]
+[%
+isAggregate = constituents.size > 0;
+steps = build.buildsteps;
+busy = 0;
+FOR step IN steps; IF step.busy; busy = 1; END; END;
+%]
 
 [% BLOCK renderOutputs %]
   [% start=1; FOREACH output IN outputs %]
@@ -18,7 +23,7 @@
       <tr><th>Nr</th><th>What</th><th>Duration</th><th>Machine</th><th>Status</th></tr>
     </thead>
     <tbody>
-      [% FOREACH step IN build.buildsteps.reverse %]
+      [% FOREACH step IN steps.reverse %]
         [% IF ( type == "All" ) || ( type == "Failed" && step.status != 0 ) || ( type == "Running" && step.busy == 1 ) %]
           [% has_log = seen.${step.drvpath} ? 0 : buildStepLogExists(step);
              seen.${step.drvpath} = 1;
@@ -114,7 +119,7 @@
   [% IF isAggregate %]<li><a href="#tabs-constituents" data-toggle="tab">Constituents</a></li>[% END %]
   <li><a href="#tabs-details" data-toggle="tab">Details</a></li>
   <li><a href="#tabs-buildinputs" data-toggle="tab">Inputs</a></li>
-  [% IF build.buildsteps %]<li><a href="#tabs-buildsteps" data-toggle="tab">Build steps</a></li>[% END %]
+  [% IF steps %]<li><a href="#tabs-buildsteps" data-toggle="tab">Build steps</a></li>[% END %]
   [% IF build.dependents %]<li><a href="#tabs-usedby" data-toggle="tab">Used by</a></li>[% END%]
   [% IF drvAvailable %]<li><a href="#tabs-build-deps" data-toggle="tab">Build dependencies</a></li>[% END %]
   [% IF available %]<li><a href="#tabs-runtime-deps" data-toggle="tab">Runtime dependencies</a></li>[% END %]
@@ -138,7 +143,7 @@
             <tr>
               <th>Status:</th>
               <td>
-                [% INCLUDE renderStatus build=build icon=0 %]
+                [% INCLUDE renderStatus build=build icon=0 busy=busy %]
                 [% IF isAggregate;
                      nrConstituents = 0;
                      nrFinished = 0;
@@ -229,14 +234,14 @@
 
     [% END %]
 
-    [% IF build.busy %]
+    [% IF busy %]
       <h3>Running build steps</h3>
       [% INCLUDE renderBuildSteps type="Running" %]
     [% END %]
 
     [% IF build.finished %]
 
-      [% IF build.buildsteps && build.buildstatus != 0 && build.buildstatus != 6 %]
+      [% IF steps && build.buildstatus != 0 && build.buildstatus != 6 %]
         <h3>Failed build steps</h3>
         [% INCLUDE renderBuildSteps type="Failed" %]
       [% END %]
@@ -423,7 +428,7 @@
 
   </div>
 
-  [% IF build.buildsteps %]
+  [% IF steps %]
     <div id="tabs-buildsteps" class="tab-pane">
       [% INCLUDE renderBuildSteps type="All" %]
     </div>

src/root/common.tt

@@ -98,11 +98,11 @@ BLOCK renderBuildListBody;
     <tr>
       [% IF !hideResultInfo %]
         <td>
-          [% INCLUDE renderBuildStatusIcon size=16 busy=(showSchedulingInfo ? 1 : 0) buildstatus=build.buildstatus %]
+          [% INCLUDE renderBuildStatusIcon size=16 buildstatus=build.buildstatus %]
         </td>
       [% END %]
       [% IF showSchedulingInfo %]
-        <td>[% IF build.busy %]<span class="label label-success">Started</span>[% ELSE %]<span class="label">Queued</span>[% END %]</td>
+        <td>[% IF busy %]<span class="label label-success">Started</span>[% ELSE %]<span class="label">Queued</span>[% END %]</td>
       [% END %]
       <td><a class="row-link" href="[% link %]">[% build.id %]</a></td>
       [% IF !hideJobName %]
@@ -189,7 +189,6 @@ BLOCK renderBuildLink; INCLUDE renderBuildIdLink id=build.id; END;
 
 BLOCK renderBuildStatusIcon;
   finished = build != undef ? build.finished : 1;
-  busy = busy != undef ? busy : build.busy;
   buildstatus = buildstatus != undef ? buildstatus : build.buildstatus;
   IF finished;
     IF buildstatus == 0 %]
@@ -211,10 +210,8 @@ BLOCK renderBuildStatusIcon;
     [% ELSE %]
       <img src="[% c.uri_for("/static/images/error_${size}.png") %]" alt="Failed" class="build-status" />
     [% END;
-  ELSIF busy %]
+  ELSE %]
-    <img src="[% c.uri_for("/static/images/help_${size}.png") %]" alt="Busy" class="build-status" />
+    <img src="[% c.uri_for("/static/images/help_${size}.png") %]" alt="Queued" class="build-status" />
-  [% ELSE %]
-    <img src="[% c.uri_for("/static/images/help_${size}.png") %]" alt="Scheduled" class="build-status" />
   [% END;
 END;
 
@@ -243,9 +240,8 @@ BLOCK renderStatus;
       <span class="error">Aborted</span>
       (Hydra failure; see <a href="#nix-error">below</a>)
     [% END;
-  ELSIF build.busy %]
+  ELSIF busy %]
-    <strong>Build in progress</strong>
+    <strong>Build in progress</strong>[%-
-    since [% INCLUDE renderDateTime timestamp = build.starttime;
   ELSE %]
     <strong>Scheduled to be built</strong>
   [% END;

src/root/jobset-eval.tt

@@ -91,7 +91,7 @@ c.uri_for(c.controller('JobsetEval').action_for('view'),
   [% INCLUDE renderBuildListHeader unsortable=1 %]
   [% size = builds.size; max = full ? size : 250; %]
   [% INCLUDE renderBuildListBody builds=builds.slice(0, (size > max ? max : size) - 1)
-       hideProjectName=1 hideJobsetName=1 %]
+       hideProjectName=1 hideJobsetName=1 busy=0 %]
   [% IF size > max; params = c.req.params; params.full = 1 %]
   <tr><td class="centered" colspan="6"><a href="[% c.uri_for(c.controller('JobsetEval').action_for('view'), [eval.id], params) %]"><em>([% size - max %] more builds omitted)</em></a></td></tr>
   [% END %]

src/root/status.tt

@@ -7,7 +7,7 @@
 
 [% ELSE %]
 
-  [% INCLUDE renderBuildList builds=resource showSchedulingInfo=1 hideResultInfo=1 %]
+  [% INCLUDE renderBuildList builds=resource showSchedulingInfo=1 hideResultInfo=1 busy=1 %]
 
 [% END %]
 

src/sql/hydra.sql

@@ -173,12 +173,6 @@ create table Builds (
     -- the front of the queue via the web interface.
     globalPriority integer not null default 0,
 
-    -- FIXME: remove (obsolete with the new queue runner)
-    busy          integer not null default 0, -- true means someone is building this job now
-    locker        text, -- !!! hostname/pid of the process building this job?
-
-    logfile       text, -- if busy, the path of the logfile
-
     -- FIXME: remove startTime?
     startTime     integer, -- if busy/finished, time we started
     stopTime      integer, -- if finished, time we finished
@@ -641,7 +635,6 @@ create index IndexBuildStepsOnPropagatedFrom on BuildSteps(propagatedFrom) where
 create index IndexBuildStepsOnStopTime on BuildSteps(stopTime desc) where startTime is not null and stopTime is not null;
 create index IndexBuildStepOutputsOnPath on BuildStepOutputs(path);
 create index IndexBuildsOnFinished on Builds(finished) where finished = 0;
-create index IndexBuildsOnFinishedBusy on Builds(finished, busy) where finished = 0;
 create index IndexBuildsOnIsCurrent on Builds(isCurrent) where isCurrent = 1;
 create index IndexBuildsOnJobsetIsCurrent on Builds(project, jobset, isCurrent) where isCurrent = 1;
 create index IndexBuildsOnJobIsCurrent on Builds(project, jobset, job, isCurrent) where isCurrent = 1;

src/sql/upgrade-43.sql

@@ -0,0 +1,3 @@
+alter table Builds drop column busy, drop column locker, drop column logfile;
+
+drop index IndexBuildsOnFinishedBusy;