From 4d1816b1529e4877f9527f039facbe9953c187bd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 27 Oct 2015 15:37:17 +0100 Subject: [PATCH] Remove obsolete Builds columns and provide accurate "Running builds" This removes the "busy", "locker" and "logfile" columns, which are no longer used by the queue runner. The "Running builds" page now only shows builds that have an active build step. --- src/hydra-queue-runner/builder.cc | 5 +- src/hydra-queue-runner/hydra-queue-runner.cc | 3 +- src/hydra-queue-runner/queue-monitor.cc | 4 +- src/lib/Hydra/Controller/API.pm | 51 +------------------- src/lib/Hydra/Controller/Build.pm | 2 +- src/lib/Hydra/Controller/Root.pm | 19 ++++++-- src/lib/Hydra/Helper/AddBuilds.pm | 2 - src/lib/Hydra/Helper/CatalystUtils.pm | 2 +- src/lib/Hydra/Helper/Nix.pm | 2 - src/lib/Hydra/Schema/Builds.pm | 31 ++---------- src/root/build.tt | 19 +++++--- src/root/common.tt | 16 +++--- src/root/jobset-eval.tt | 2 +- src/root/status.tt | 2 +- src/sql/hydra.sql | 7 --- src/sql/upgrade-43.sql | 3 ++ 16 files changed, 50 insertions(+), 120 deletions(-) create mode 100644 src/sql/upgrade-43.sql diff --git a/src/hydra-queue-runner/builder.cc b/src/hydra-queue-runner/builder.cc index d13c545c..ae267a7d 100644 --- a/src/hydra-queue-runner/builder.cc +++ b/src/hydra-queue-runner/builder.cc @@ -110,11 +110,10 @@ bool State::doBuildStep(std::shared_ptr store, Step::ptr step, else { /* Create a build step record indicating that we started - building. Also, mark the selected build as busy. */ + building. */ { pqxx::work txn(*conn); stepNr = createBuildStep(txn, result.startTime, build, step, machine->sshName, bssBusy); - txn.parameterized("update Builds set busy = 1 where id = $1")(build->id).exec(); txn.commit(); } @@ -341,7 +340,7 @@ bool State::doBuildStep(std::shared_ptr store, Step::ptr step, if (build2->finishedInDB) continue; printMsg(lvlError, format("marking build %1% as failed") % build2->id); txn.parameterized - ("update Builds set finished = 1, busy = 0, buildStatus = $2, startTime = $3, stopTime = $4, isCachedBuild = $5 where id = $1 and finished = 0") + ("update Builds set finished = 1, buildStatus = $2, startTime = $3, stopTime = $4, isCachedBuild = $5 where id = $1 and finished = 0") (build2->id) ((int) (build2->drvPath != step->drvPath && buildStatus == bsFailed ? bsDepFailed : buildStatus)) (result.startTime) diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index 1e7ff7cc..4f8da044 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -159,7 +159,6 @@ void State::clearBusy(Connection & conn, time_t stopTime) ("update BuildSteps set busy = 0, status = $1, stopTime = $2 where busy = 1") ((int) bssAborted) (stopTime, stopTime != 0).exec(); - txn.exec("update Builds set busy = 0 where finished = 0 and busy = 1"); txn.commit(); } @@ -309,7 +308,7 @@ void State::markSucceededBuild(pqxx::work & txn, Build::ptr build, if (txn.parameterized("select 1 from Builds where id = $1 and finished = 0")(build->id).exec().empty()) return; txn.parameterized - ("update Builds set finished = 1, busy = 0, buildStatus = $2, startTime = $3, stopTime = $4, size = $5, closureSize = $6, releaseName = $7, isCachedBuild = $8 where id = $1") + ("update Builds set finished = 1, buildStatus = $2, startTime = $3, stopTime = $4, size = $5, closureSize = $6, releaseName = $7, isCachedBuild = $8 where id = $1") (build->id) ((int) (res.failed ? bsFailedWithOutput : bsSuccess)) (startTime) diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc index 3f3d55f2..8bce7aab 100644 --- a/src/hydra-queue-runner/queue-monitor.cc +++ b/src/hydra-queue-runner/queue-monitor.cc @@ -125,7 +125,7 @@ bool State::getQueuedBuilds(Connection & conn, std::shared_ptr store, if (!build->finishedInDB) { pqxx::work txn(conn); txn.parameterized - ("update Builds set finished = 1, busy = 0, buildStatus = $2, startTime = $3, stopTime = $3, errorMsg = $4 where id = $1 and finished = 0") + ("update Builds set finished = 1, buildStatus = $2, startTime = $3, stopTime = $3, errorMsg = $4 where id = $1 and finished = 0") (build->id) ((int) bsAborted) (time(0)) @@ -201,7 +201,7 @@ bool State::getQueuedBuilds(Connection & conn, std::shared_ptr store, createBuildStep(txn, 0, build, r, "", bssCachedFailure, "", propagatedFrom); txn.parameterized - ("update Builds set finished = 1, busy = 0, buildStatus = $2, startTime = $3, stopTime = $3, isCachedBuild = 1 where id = $1 and finished = 0") + ("update Builds set finished = 1, buildStatus = $2, startTime = $3, stopTime = $3, isCachedBuild = 1 where id = $1 and finished = 0") (build->id) ((int) (step == r ? bsFailed : bsDepFailed)) (time(0)).exec(); diff --git a/src/lib/Hydra/Controller/API.pm b/src/lib/Hydra/Controller/API.pm index fbbb70be..07311f7d 100644 --- a/src/lib/Hydra/Controller/API.pm +++ b/src/lib/Hydra/Controller/API.pm @@ -38,7 +38,6 @@ sub buildToHash { if($build->finished) { $result->{'buildstatus'} = $build->get_column("buildstatus"); } else { - $result->{'busy'} = $build->get_column("busy"); $result->{'priority'} = $build->get_column("priority"); } @@ -114,7 +113,7 @@ sub queue : Chained('api') PathPart('queue') Args(0) { my $nr = $c->request->params->{nr}; error($c, "Parameter not defined!") if !defined $nr; - my @builds = $c->model('DB::Builds')->search({finished => 0}, {rows => $nr, order_by => ["busy DESC", "priority DESC", "id"]}); + my @builds = $c->model('DB::Builds')->search({finished => 0}, {rows => $nr, order_by => ["priority DESC", "id"]}); my @list; push @list, buildToHash($_) foreach @builds; @@ -136,16 +135,6 @@ sub nrqueue : Chained('api') PathPart('nrqueue') Args(0) { } -sub nrrunning : Chained('api') PathPart('nrrunning') Args(0) { - my ($self, $c) = @_; - my $nrRunningBuilds = $c->model('DB::Builds')->search({finished => 0, busy => 1 })->count(); - $c->stash->{'plain'} = { - data => "$nrRunningBuilds" - }; - $c->forward('Hydra::View::Plain'); -} - - sub nrbuilds : Chained('api') PathPart('nrbuilds') Args(0) { my ($self, $c) = @_; my $nr = $c->request->params->{nr}; @@ -212,44 +201,6 @@ sub scmdiff : Chained('api') PathPart('scmdiff') Args(0) { } -sub readNormalizedLog { - my ($file) = @_; - my $pipe = (-f "$file.bz2" ? "cat $file.bz2 | bzip2 -d" : "cat $file"); - my $res = `$pipe`; - - $res =~ s/\/nix\/store\/[a-z0-9]*-/\/nix\/store\/...-/g; - $res =~ s/nix-build-[a-z0-9]*-/nix-build-...-/g; - $res =~ s/[0-9]{2}:[0-9]{2}:[0-9]{2}/00:00:00/g; - return $res; -} - - -sub logdiff : Chained('api') PathPart('logdiff') Args(2) { - my ($self, $c, $buildid1, $buildid2) = @_; - - my $diff = ""; - - my $build1 = getBuild($c, $buildid1); - notFound($c, "Build with ID $buildid1 doesn't exist.") - if !defined $build1; - my $build2 = getBuild($c, $buildid2); - notFound($c, "Build with ID $buildid2 doesn't exist.") - if !defined $build2; - - if (-f $build1->logfile && -f $build2->logfile) { - my $logtext1 = readNormalizedLog($build1->logfile); - my $logtext2 = readNormalizedLog($build2->logfile); - $diff = diff \$logtext1, \$logtext2; - } else { - $c->response->status(404); - } - - $c->response->content_type('text/x-diff'); - $c->stash->{'plain'} = { data => (scalar $diff) || " " }; - $c->forward('Hydra::View::Plain'); -} - - sub triggerJobset { my ($self, $c, $jobset) = @_; print STDERR "triggering jobset ", $jobset->project->name . ":" . $jobset->name, "\n"; diff --git a/src/lib/Hydra/Controller/Build.pm b/src/lib/Hydra/Controller/Build.pm index 7fc66b61..f35f5af3 100644 --- a/src/lib/Hydra/Controller/Build.pm +++ b/src/lib/Hydra/Controller/Build.pm @@ -167,7 +167,7 @@ sub showLog { my $url = $c->request->uri->as_string; $url =~ s/tail-reload/tail/g; $c->stash->{url} = $url; - $c->stash->{reload} = !$c->stash->{build}->finished && $c->stash->{build}->busy; + $c->stash->{reload} = !$c->stash->{build}->finished; $c->stash->{title} = ""; $c->stash->{contents} = (scalar logContents($logPath, 50)) || " "; $c->stash->{template} = 'plain-reload.tt'; diff --git a/src/lib/Hydra/Controller/Root.pm b/src/lib/Hydra/Controller/Root.pm index 14d22cb5..12fbd530 100644 --- a/src/lib/Hydra/Controller/Root.pm +++ b/src/lib/Hydra/Controller/Root.pm @@ -44,7 +44,9 @@ sub begin :Private { } if (scalar(@args) == 0 || $args[0] ne "static") { - $c->stash->{nrRunningBuilds} = $c->model('DB::Builds')->search({ finished => 0, busy => 1 }, {})->count(); + $c->stash->{nrRunningBuilds} = $c->model('DB::Builds')->search( + { finished => 0, 'buildsteps.busy' => 1 }, + { join => 'buildsteps', select => ["id"], distinct => 1 })->count(); $c->stash->{nrQueuedBuilds} = $c->model('DB::Builds')->search({ finished => 0 })->count(); } @@ -89,7 +91,11 @@ sub queue_GET { $c->stash->{flashMsg} //= $c->flash->{buildMsg}; $self->status_ok( $c, - entity => [$c->model('DB::Builds')->search({finished => 0}, { order_by => ["globalpriority desc", "id"]})] + entity => [$c->model('DB::Builds')->search( + { finished => 0 }, + { order_by => ["globalpriority desc", "id"], + , columns => [@buildListColumns] + })] ); } @@ -100,7 +106,12 @@ sub status_GET { my ($self, $c) = @_; $self->status_ok( $c, - entity => [$c->model('DB::Builds')->search({finished => 0, busy => 1}, { order_by => ["priority DESC", "id"]})] + entity => [$c->model('DB::Builds')->search( + { finished => 0, "buildsteps.busy" => 1 }, + { order_by => ["globalpriority DESC", "id"], + join => "buildsteps", + columns => [@buildListColumns] + })] ); } @@ -126,7 +137,7 @@ sub machines :Local Args(0) { $c->stash->{machines} = $machines; $c->stash->{steps} = [ $c->model('DB::BuildSteps')->search( - { finished => 0, 'me.busy' => 1, 'build.busy' => 1, }, + { finished => 0, 'me.busy' => 1, }, { join => [ 'build' ] , order_by => [ 'machine', 'stepnr' ] } ) ]; diff --git a/src/lib/Hydra/Helper/AddBuilds.pm b/src/lib/Hydra/Helper/AddBuilds.pm index 4d87e34c..3a515019 100644 --- a/src/lib/Hydra/Helper/AddBuilds.pm +++ b/src/lib/Hydra/Helper/AddBuilds.pm @@ -450,8 +450,6 @@ sub checkBuild { , nixexprpath => $jobset->nixexprpath , priority => $buildInfo->{schedulingPriority} , finished => 0 - , busy => 0 - , locker => "" , iscurrent => 1 , ischannel => $buildInfo->{isChannel} }); diff --git a/src/lib/Hydra/Helper/CatalystUtils.pm b/src/lib/Hydra/Helper/CatalystUtils.pm index db4640e9..43e1cc8b 100644 --- a/src/lib/Hydra/Helper/CatalystUtils.pm +++ b/src/lib/Hydra/Helper/CatalystUtils.pm @@ -29,7 +29,7 @@ our @EXPORT = qw( # Columns from the Builds table needed to render build lists. -Readonly our @buildListColumns => ('id', 'finished', 'timestamp', 'stoptime', 'project', 'jobset', 'job', 'nixname', 'system', 'priority', 'busy', 'buildstatus', 'releasename'); +Readonly our @buildListColumns => ('id', 'finished', 'timestamp', 'stoptime', 'project', 'jobset', 'job', 'nixname', 'system', 'buildstatus', 'releasename'); sub getBuild { diff --git a/src/lib/Hydra/Helper/Nix.pm b/src/lib/Hydra/Helper/Nix.pm index 7243166f..d25d58c0 100644 --- a/src/lib/Hydra/Helper/Nix.pm +++ b/src/lib/Hydra/Helper/Nix.pm @@ -455,8 +455,6 @@ sub restartBuilds($$) { txn_do($db, sub { $nrRestarted = $builds->update( { finished => 0 - , busy => 0 - , locker => "" , iscachedbuild => 0 }); diff --git a/src/lib/Hydra/Schema/Builds.pm b/src/lib/Hydra/Schema/Builds.pm index a22cd670..fbf64d45 100644 --- a/src/lib/Hydra/Schema/Builds.pm +++ b/src/lib/Hydra/Schema/Builds.pm @@ -120,7 +120,7 @@ __PACKAGE__->table("Builds"); data_type: 'integer' default_value: 0 - is_nullable: 1 + is_nullable: 0 =head2 iscurrent @@ -150,22 +150,6 @@ __PACKAGE__->table("Builds"); default_value: 0 is_nullable: 0 -=head2 busy - - data_type: 'integer' - default_value: 0 - is_nullable: 0 - -=head2 locker - - data_type: 'text' - is_nullable: 1 - -=head2 logfile - - data_type: 'text' - is_nullable: 1 - =head2 starttime data_type: 'integer' @@ -246,7 +230,7 @@ __PACKAGE__->add_columns( "timeout", { data_type => "integer", default_value => 36000, is_nullable => 1 }, "ischannel", - { data_type => "integer", default_value => 0, is_nullable => 1 }, + { data_type => "integer", default_value => 0, is_nullable => 0 }, "iscurrent", { data_type => "integer", default_value => 0, is_nullable => 1 }, "nixexprinput", @@ -257,12 +241,6 @@ __PACKAGE__->add_columns( { data_type => "integer", default_value => 0, is_nullable => 0 }, "globalpriority", { data_type => "integer", default_value => 0, is_nullable => 0 }, - "busy", - { data_type => "integer", default_value => 0, is_nullable => 0 }, - "locker", - { data_type => "text", is_nullable => 1 }, - "logfile", - { data_type => "text", is_nullable => 1 }, "starttime", { data_type => "integer", is_nullable => 1 }, "stoptime", @@ -566,8 +544,8 @@ __PACKAGE__->many_to_many( ); -# Created by DBIx::Class::Schema::Loader v0.07043 @ 2015-09-10 17:34:23 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:JRelp13Cyfi+QVxC92xuqQ +# Created by DBIx::Class::Schema::Loader v0.07043 @ 2015-10-27 13:54:16 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:iN9vwO5i3dURKOTgaPTHxw __PACKAGE__->has_many( "dependents", @@ -647,7 +625,6 @@ my %hint = ( 'nixname', 'system', 'priority', - 'busy', 'buildstatus', 'releasename' ], diff --git a/src/root/build.tt b/src/root/build.tt index 369e5648..8e6c4a48 100644 --- a/src/root/build.tt +++ b/src/root/build.tt @@ -4,7 +4,12 @@ [% USE HTML %] [% USE Date %] -[% isAggregate = constituents.size > 0 %] +[% +isAggregate = constituents.size > 0; +steps = build.buildsteps; +busy = 0; +FOR step IN steps; IF step.busy; busy = 1; END; END; +%] [% BLOCK renderOutputs %] [% start=1; FOREACH output IN outputs %] @@ -18,7 +23,7 @@ NrWhatDurationMachineStatus - [% FOREACH step IN build.buildsteps.reverse %] + [% FOREACH step IN steps.reverse %] [% IF ( type == "All" ) || ( type == "Failed" && step.status != 0 ) || ( type == "Running" && step.busy == 1 ) %] [% has_log = seen.${step.drvpath} ? 0 : buildStepLogExists(step); seen.${step.drvpath} = 1; @@ -114,7 +119,7 @@ [% IF isAggregate %]
  • Constituents
    • [% END %]
  • Details
  • Inputs
    • - [% IF build.buildsteps %]
  • Build steps
    • [% END %] + [% IF steps %]
  • Build steps
    • [% END %] [% IF build.dependents %]
  • Used by
    • [% END%] [% IF drvAvailable %]
  • Build dependencies
    • [% END %] [% IF available %]
  • Runtime dependencies
    • [% END %] @@ -138,7 +143,7 @@ Status: - [% INCLUDE renderStatus build=build icon=0 %] + [% INCLUDE renderStatus build=build icon=0 busy=busy %] [% IF isAggregate; nrConstituents = 0; nrFinished = 0; @@ -229,14 +234,14 @@ [% END %] - [% IF build.busy %] + [% IF busy %]

    Running build steps

    [% INCLUDE renderBuildSteps type="Running" %] [% END %] [% IF build.finished %] - [% IF build.buildsteps && build.buildstatus != 0 && build.buildstatus != 6 %] + [% IF steps && build.buildstatus != 0 && build.buildstatus != 6 %]

    Failed build steps

    [% INCLUDE renderBuildSteps type="Failed" %] [% END %] @@ -423,7 +428,7 @@ - [% IF build.buildsteps %] + [% IF steps %]
    [% INCLUDE renderBuildSteps type="All" %]
    diff --git a/src/root/common.tt b/src/root/common.tt index b4d21385..f844a919 100644 --- a/src/root/common.tt +++ b/src/root/common.tt @@ -98,11 +98,11 @@ BLOCK renderBuildListBody; [% IF !hideResultInfo %] - [% INCLUDE renderBuildStatusIcon size=16 busy=(showSchedulingInfo ? 1 : 0) buildstatus=build.buildstatus %] + [% INCLUDE renderBuildStatusIcon size=16 buildstatus=build.buildstatus %] [% END %] [% IF showSchedulingInfo %] - [% IF build.busy %]Started[% ELSE %]Queued[% END %] + [% IF busy %]Started[% ELSE %]Queued[% END %] [% END %] [% build.id %] [% IF !hideJobName %] @@ -189,7 +189,6 @@ BLOCK renderBuildLink; INCLUDE renderBuildIdLink id=build.id; END; BLOCK renderBuildStatusIcon; finished = build != undef ? build.finished : 1; - busy = busy != undef ? busy : build.busy; buildstatus = buildstatus != undef ? buildstatus : build.buildstatus; IF finished; IF buildstatus == 0 %] @@ -211,10 +210,8 @@ BLOCK renderBuildStatusIcon; [% ELSE %] Failed [% END; - ELSIF busy %] - Busy - [% ELSE %] - Scheduled + ELSE %] + Queued [% END; END; @@ -243,9 +240,8 @@ BLOCK renderStatus; Aborted (Hydra failure; see below) [% END; - ELSIF build.busy %] - Build in progress - since [% INCLUDE renderDateTime timestamp = build.starttime; + ELSIF busy %] + Build in progress[%- ELSE %] Scheduled to be built [% END; diff --git a/src/root/jobset-eval.tt b/src/root/jobset-eval.tt index 3a090a77..0380e2e5 100644 --- a/src/root/jobset-eval.tt +++ b/src/root/jobset-eval.tt @@ -91,7 +91,7 @@ c.uri_for(c.controller('JobsetEval').action_for('view'), [% INCLUDE renderBuildListHeader unsortable=1 %] [% size = builds.size; max = full ? size : 250; %] [% INCLUDE renderBuildListBody builds=builds.slice(0, (size > max ? max : size) - 1) - hideProjectName=1 hideJobsetName=1 %] + hideProjectName=1 hideJobsetName=1 busy=0 %] [% IF size > max; params = c.req.params; params.full = 1 %] ([% size - max %] more builds omitted) [% END %] diff --git a/src/root/status.tt b/src/root/status.tt index a1c2157c..f1ec70b9 100644 --- a/src/root/status.tt +++ b/src/root/status.tt @@ -7,7 +7,7 @@ [% ELSE %] - [% INCLUDE renderBuildList builds=resource showSchedulingInfo=1 hideResultInfo=1 %] + [% INCLUDE renderBuildList builds=resource showSchedulingInfo=1 hideResultInfo=1 busy=1 %] [% END %] diff --git a/src/sql/hydra.sql b/src/sql/hydra.sql index 65554627..2e7bf0ce 100644 --- a/src/sql/hydra.sql +++ b/src/sql/hydra.sql @@ -173,12 +173,6 @@ create table Builds ( -- the front of the queue via the web interface. globalPriority integer not null default 0, - -- FIXME: remove (obsolete with the new queue runner) - busy integer not null default 0, -- true means someone is building this job now - locker text, -- !!! hostname/pid of the process building this job? - - logfile text, -- if busy, the path of the logfile - -- FIXME: remove startTime? startTime integer, -- if busy/finished, time we started stopTime integer, -- if finished, time we finished @@ -641,7 +635,6 @@ create index IndexBuildStepsOnPropagatedFrom on BuildSteps(propagatedFrom) where create index IndexBuildStepsOnStopTime on BuildSteps(stopTime desc) where startTime is not null and stopTime is not null; create index IndexBuildStepOutputsOnPath on BuildStepOutputs(path); create index IndexBuildsOnFinished on Builds(finished) where finished = 0; -create index IndexBuildsOnFinishedBusy on Builds(finished, busy) where finished = 0; create index IndexBuildsOnIsCurrent on Builds(isCurrent) where isCurrent = 1; create index IndexBuildsOnJobsetIsCurrent on Builds(project, jobset, isCurrent) where isCurrent = 1; create index IndexBuildsOnJobIsCurrent on Builds(project, jobset, job, isCurrent) where isCurrent = 1; diff --git a/src/sql/upgrade-43.sql b/src/sql/upgrade-43.sql new file mode 100644 index 00000000..5fe7034b --- /dev/null +++ b/src/sql/upgrade-43.sql @@ -0,0 +1,3 @@ +alter table Builds drop column busy, drop column locker, drop column logfile; + +drop index IndexBuildsOnFinishedBusy;