From 47f877c5bbaed5b206a4fc134ca2105d61b1a51b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 7 Mar 2012 18:48:10 +0100 Subject: [PATCH] Evaluator cleanups * Don't use isCurrent anymore; instead look up builds in the previous jobset evaluation. (The isCurrent field is still maintained because it's still used in some other places.) * To determine whether to perform an evaluation, compare the hash of the current inputs with the inputs of the previous jobset evaluation, rather than checking if there was ever an evaluation with those inputs. This way, if the inputs of an evaluation change back to a previous state, we get a new jobset evaluation in the database (and thus the latest jobset evaluation correctly represents the latest state of the jobset). * Improve performance by removing some unnecessary operations and adding an index. --- src/lib/Hydra/Controller/Build.pm | 2 +- src/lib/Hydra/Helper/AddBuilds.pm | 70 ++++++++++++++++++----------- src/lib/Hydra/Schema/JobsetEvals.pm | 1 - src/script/hydra-evaluator | 53 ++++++++++------------ src/sql/hydra.sql | 10 ++++- src/sql/upgrade-4.sql | 1 + 6 files changed, 77 insertions(+), 60 deletions(-) create mode 100644 src/sql/upgrade-4.sql diff --git a/src/lib/Hydra/Controller/Build.pm b/src/lib/Hydra/Controller/Build.pm index a8462d4d..c422ba9c 100644 --- a/src/lib/Hydra/Controller/Build.pm +++ b/src/lib/Hydra/Controller/Build.pm @@ -538,7 +538,7 @@ sub clone_submit : Chained('build') PathPart('clone/submit') Args(0) { my %currentBuilds; my $newBuild = checkBuild( $c->model('DB'), $build->project, $build->jobset, - $inputInfo, $nixExprInput, $job, \%currentBuilds); + $inputInfo, $nixExprInput, $job, \%currentBuilds, undef); error($c, "This build has already been performed.") unless $newBuild; diff --git a/src/lib/Hydra/Helper/AddBuilds.pm b/src/lib/Hydra/Helper/AddBuilds.pm index 977c20a1..5749a73a 100644 --- a/src/lib/Hydra/Helper/AddBuilds.pm +++ b/src/lib/Hydra/Helper/AddBuilds.pm @@ -17,6 +17,7 @@ our @ISA = qw(Exporter); our @EXPORT = qw( fetchInput evalJobs checkBuild inputsToArgs captureStdoutStderr getReleaseName getBuildLog addBuildProducts restartBuild scmPath + getPrevJobsetEval ); @@ -787,9 +788,18 @@ sub addBuildProducts { } +# Return the most recent evaluation of the given jobset that had new +# builds, or undefined if no such evaluation exists. +sub getPrevJobsetEval { + my ($db, $jobset) = @_; + my ($prevEval) = $jobset->jobsetevals({ hasnewbuilds => 1 }, { order_by => "id DESC", rows => 1 }); + return $prevEval; +} + + # Check whether to add the build described by $buildInfo. sub checkBuild { - my ($db, $project, $jobset, $inputInfo, $nixExprInput, $buildInfo, $currentBuilds) = @_; + my ($db, $project, $jobset, $inputInfo, $nixExprInput, $buildInfo, $buildIds, $prevEval) = @_; my $jobName = $buildInfo->{jobName}; my $drvPath = $buildInfo->{drvPath}; @@ -822,19 +832,36 @@ sub checkBuild { # !!! Checking $outPath doesn't take meta-attributes into # account. For instance, do we want a new build to be # scheduled if the meta.maintainers field is changed? - my @previousBuilds = $job->builds->search({outPath => $outPath, isCurrent => 1}); - if (scalar(@previousBuilds) > 0) { - print STDERR "already scheduled/built\n"; - $currentBuilds->{$_->id} = 0 foreach @previousBuilds; - return; + if (defined $prevEval) { + my ($prevBuild) = $prevEval->builds->search({ job => $job->name, outPath => $outPath }, { rows => 1, columns => ['id'] }); + if (defined $prevBuild) { + print STDERR " already scheduled/built as build ", $prevBuild->id, "\n"; + $buildIds->{$prevBuild->id} = 0; + return; + } } my $time = time(); # Nope, so add it. + my %extraFlags; + if (isValidPath($outPath)) { + %extraFlags = + ( finished => 1 + , iscachedbuild => 1 + , buildstatus => 0 + , starttime => $time + , stoptime => $time + , logfile => getBuildLog($drvPath) + , errormsg => "" + , releasename => getReleaseName($outPath) + ); + } else { + %extraFlags = ( finished => 0 ); + } + $build = $job->builds->create( - { finished => 0 - , timestamp => $time + { timestamp => $time , description => $buildInfo->{description} , longdescription => $buildInfo->{longDescription} , license => $buildInfo->{license} @@ -849,30 +876,19 @@ sub checkBuild { , iscurrent => 1 , nixexprinput => $jobset->nixexprinput , nixexprpath => $jobset->nixexprpath + , priority => $priority + , busy => 0 + , locker => "" + , %extraFlags }); - $currentBuilds->{$build->id} = 1; + $buildIds->{$build->id} = 1; - if (isValidPath($outPath)) { - print STDERR "marked as cached build ", $build->id, "\n"; - $build->update( - { finished => 1 - , iscachedbuild => 1 - , buildstatus => 0 - , starttime => $time - , stoptime => $time - , logfile => getBuildLog($drvPath) - , errormsg => "" - , releasename => getReleaseName($outPath) - }); + if ($build->iscachedbuild) { + print STDERR " marked as cached build ", $build->id, "\n"; addBuildProducts($db, $build); } else { - print STDERR "added to queue as build ", $build->id, "\n"; - $build->update( - { priority => $priority - , busy => 0 - , locker => "" - }); + print STDERR " added to queue as build ", $build->id, "\n"; } my %inputs; diff --git a/src/lib/Hydra/Schema/JobsetEvals.pm b/src/lib/Hydra/Schema/JobsetEvals.pm index 36058ff1..f3feaa16 100644 --- a/src/lib/Hydra/Schema/JobsetEvals.pm +++ b/src/lib/Hydra/Schema/JobsetEvals.pm @@ -161,5 +161,4 @@ __PACKAGE__->has_many( __PACKAGE__->many_to_many(builds => 'buildIds', 'build'); -# You can replace this text with custom content, and it will be preserved on regeneration 1; diff --git a/src/script/hydra-evaluator b/src/script/hydra-evaluator index c2e71ebb..7f76f02a 100755 --- a/src/script/hydra-evaluator +++ b/src/script/hydra-evaluator @@ -80,11 +80,12 @@ sub sendJobsetErrorNotification() { ); $email->body_set($body); - print $email->as_string if $ENV{'HYDRA_MAIL_TEST'}; + print STDERR $email->as_string if $ENV{'HYDRA_MAIL_TEST'}; sendmail($email); } + sub permute { my @list = @_; for (my $n = scalar @list - 1; $n > 0; $n--) { @@ -105,13 +106,13 @@ sub checkJobset { my $checkoutStop = time; # Hash the arguments to hydra-eval-jobs and check the - # JobsetInputHashes to see if we've already evaluated this set of + # JobsetInputHashes to see if the previous evaluation had the same # inputs. If so, bail out. my @args = ($jobset->nixexprinput, $jobset->nixexprpath, inputsToArgs($inputInfo)); my $argsHash = sha256_hex("@args"); - - if (scalar($jobset->jobsetevals->search({hash => $argsHash})) > 0) { - print " already evaluated, skipping\n"; + my $prevEval = getPrevJobsetEval($db, $jobset); + if ($prevEval->hash eq $argsHash) { + print STDERR " jobset is unchanged, skipping\n"; txn_do($db, sub { $jobset->update({lastcheckedtime => time}); }); @@ -125,12 +126,17 @@ sub checkJobset { txn_do($db, sub { + # Clear the "current" flag on all builds. Since we're in a + # transaction this will only become visible after the new + # current builds have been added. + $jobset->builds->search({iscurrent => 1})->update({iscurrent => 0}); + # Schedule each successfully evaluated job. - my %currentBuilds; + my %buildIds; foreach my $job (permute @{$jobs->{job}}) { next if $job->{jobName} eq ""; - print "considering job " . $job->{jobName} . "\n"; - checkBuild($db, $project, $jobset, $inputInfo, $nixExprInput, $job, \%currentBuilds); + print STDERR " considering job " . $project->name, ":", $jobset->name, ":", $job->{jobName} . "\n"; + checkBuild($db, $project, $jobset, $inputInfo, $nixExprInput, $job, \%buildIds, $prevEval); } # Update the last checked times and error messages for each @@ -140,22 +146,11 @@ sub checkJobset { $jobset->update({lastcheckedtime => time}); - foreach my $job ($jobset->jobs->all) { - if ($failedJobNames{$job->name}) { - $job->update({errormsg => join '\n', @{$failedJobNames{$job->name}}}); - } else { - $job->update({errormsg => undef}); - } - } - - # Clear the "current" flag on all builds that are no longer - # current. - foreach my $build ($jobset->builds->search({iscurrent => 1})) { - $build->update({iscurrent => 0}) unless defined $currentBuilds{$build->id}; - } + $_->update({ errormsg => $failedJobNames{$_->name} ? join '\n', @{$failedJobNames{$_->name}} : undef }) + foreach $jobset->jobs->all; my $hasNewBuilds = 0; - while (my ($id, $new) = each %currentBuilds) { + while (my ($id, $new) = each %buildIds) { $hasNewBuilds = 1 if $new; } @@ -168,13 +163,13 @@ sub checkJobset { }); if ($hasNewBuilds) { - while (my ($id, $new) = each %currentBuilds) { + while (my ($id, $new) = each %buildIds) { $ev->jobsetevalmembers->create({ build => $id, isnew => $new }); } } }); - # Store the errors messages for jobs that failed to evaluate. + # Store the error messages for jobs that failed to evaluate. my $msg = ""; foreach my $error (@{$jobs->{error}}) { my $bindings = ""; @@ -197,7 +192,7 @@ sub checkJobset { sub checkJobsetWrapped { my ($project, $jobset) = @_; - print "considering jobset ", $jobset->name, " in ", $project->name, "\n"; + print STDERR "considering jobset ", $project->name, ":", $jobset->name, "\n"; eval { checkJobset($project, $jobset); @@ -205,7 +200,7 @@ sub checkJobsetWrapped { if ($@) { my $msg = $@; - print "error evaluating jobset ", $jobset->name, ": $msg"; + print STDERR "error evaluating jobset ", $jobset->name, ": $msg"; txn_do($db, sub { $jobset->update({lastcheckedtime => time}); setJobsetError($jobset, $msg); @@ -216,7 +211,7 @@ sub checkJobsetWrapped { sub checkProjects { foreach my $project ($db->resultset('Projects')->search({enabled => 1})) { - print "considering project ", $project->name, "\n"; + print STDERR "considering project ", $project->name, "\n"; checkJobsetWrapped($project, $_) foreach $project->jobsets->search({enabled => 1}); } @@ -237,7 +232,7 @@ while (1) { eval { checkProjects; }; - if ($@) { print "$@"; } - print "sleeping...\n"; + if ($@) { print STDERR "$@"; } + print STDERR "sleeping...\n"; sleep 30; } diff --git a/src/sql/hydra.sql b/src/sql/hydra.sql index 8ccc90be..7c384740 100644 --- a/src/sql/hydra.sql +++ b/src/sql/hydra.sql @@ -493,6 +493,7 @@ create table BuildMachineSystemTypes ( -- Some indices. + create index IndexBuildInputsOnBuild on BuildInputs(build); create index IndexBuildInputsOnDependency on BuildInputs(dependency); create index IndexBuildProducstOnBuildAndType on BuildProducts(build, type); @@ -517,8 +518,6 @@ create index IndexBuildsOnJobsetFinishedTimestamp on Builds(project, jobset, fin create index IndexBuildsOnJobFinishedId on builds(project, jobset, job, system, finished, id DESC); create index IndexBuildsOnJobSystemCurrent on Builds(project, jobset, job, system, isCurrent); create index IndexBuildsOnDrvPath on Builds(drvPath); -create index IndexBuildsOnKeep on Builds(keep); -- used by hydra-update-gc-roots -create index IndexMostRecentSuccessfulBuilds on Builds(project, jobset, job, system, finished, buildStatus, id desc); -- used by hydra-update-gc-roots create index IndexCachedHgInputsOnHash on CachedHgInputs(uri, branch, sha256hash); create index IndexCachedGitInputsOnHash on CachedGitInputs(uri, branch, sha256hash); create index IndexCachedSubversionInputsOnUriRevision on CachedSubversionInputs(uri, revision); @@ -528,3 +527,10 @@ create index IndexJobsetInputAltsOnInput on JobsetInputAlts(project, jobset, inp create index IndexJobsetInputAltsOnJobset on JobsetInputAlts(project, jobset); create index IndexProjectsOnEnabled on Projects(enabled); create index IndexReleaseMembersOnBuild on ReleaseMembers(build); + +-- For hydra-update-gc-roots. +create index IndexBuildsOnKeep on Builds(keep); +create index IndexMostRecentSuccessfulBuilds on Builds(project, jobset, job, system, finished, buildStatus, id desc); + +-- To get the most recent eval for a jobset. +create index IndexJobsetEvalsOnJobsetId on JobsetEvals(project, jobset, hasNewBuilds, id desc); diff --git a/src/sql/upgrade-4.sql b/src/sql/upgrade-4.sql new file mode 100644 index 00000000..c84b5298 --- /dev/null +++ b/src/sql/upgrade-4.sql @@ -0,0 +1 @@ +create index IndexJobsetEvalsOnJobsetId on JobsetEvals(project, jobset, hasNewBuilds, id desc);