From fca7fb20c417e7423d7af69e335c449f4b592119 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 9 Mar 2009 17:21:10 +0000 Subject: [PATCH] * Negative caching: don't perform a build if a dependency already failed in a previous build. This is essential for Nixpkgs: we don't want to keep doing the same failed dependency (say, Glibc) over and over again for a few hundred jobs. --- src/lib/Hydra/Schema.pm | 4 +- src/lib/Hydra/Schema/BuildInputs.pm | 4 +- src/lib/Hydra/Schema/BuildProducts.pm | 4 +- src/lib/Hydra/Schema/BuildResultInfo.pm | 14 +++++-- src/lib/Hydra/Schema/BuildSchedulingInfo.pm | 4 +- src/lib/Hydra/Schema/BuildSteps.pm | 6 +-- src/lib/Hydra/Schema/Builds.pm | 4 +- src/lib/Hydra/Schema/CachedPathInputs.pm | 4 +- .../Hydra/Schema/CachedSubversionInputs.pm | 4 +- src/lib/Hydra/Schema/JobsetInputAlts.pm | 4 +- src/lib/Hydra/Schema/JobsetInputs.pm | 4 +- src/lib/Hydra/Schema/Jobsets.pm | 4 +- src/lib/Hydra/Schema/Projects.pm | 4 +- src/lib/Hydra/Schema/ReleaseSetJobs.pm | 4 +- src/lib/Hydra/Schema/ReleaseSets.pm | 4 +- src/lib/Hydra/Schema/SystemTypes.pm | 4 +- src/lib/Hydra/Schema/UserRoles.pm | 4 +- src/lib/Hydra/Schema/Users.pm | 4 +- src/root/build.tt | 9 +++- src/script/hydra_build.pl | 41 ++++++++++++++++++- src/sql/hydra.sql | 9 +++- 21 files changed, 99 insertions(+), 44 deletions(-) diff --git a/src/lib/Hydra/Schema.pm b/src/lib/Hydra/Schema.pm index 7c96e47d..42340a27 100644 --- a/src/lib/Hydra/Schema.pm +++ b/src/lib/Hydra/Schema.pm @@ -8,8 +8,8 @@ use base 'DBIx::Class::Schema'; __PACKAGE__->load_classes; -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:hRwus0A1vxnFCzQxghFgTw +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:DQAmauNODAxmM2mF/AE2aQ # You can replace this text with custom content, and it will be preserved on regeneration diff --git a/src/lib/Hydra/Schema/BuildInputs.pm b/src/lib/Hydra/Schema/BuildInputs.pm index 579f53ff..0548fb8d 100644 --- a/src/lib/Hydra/Schema/BuildInputs.pm +++ b/src/lib/Hydra/Schema/BuildInputs.pm @@ -36,8 +36,8 @@ __PACKAGE__->belongs_to("build", "Hydra::Schema::Builds", { id => "build" }); __PACKAGE__->belongs_to("dependency", "Hydra::Schema::Builds", { id => "dependency" }); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:H25BhHe9wbU6nj6fSKjZnw +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:L6NP/+9zhMg4TRw3w911rg # You can replace this text with custom content, and it will be preserved on regeneration diff --git a/src/lib/Hydra/Schema/BuildProducts.pm b/src/lib/Hydra/Schema/BuildProducts.pm index a3fb4df1..e51fe9d5 100644 --- a/src/lib/Hydra/Schema/BuildProducts.pm +++ b/src/lib/Hydra/Schema/BuildProducts.pm @@ -35,8 +35,8 @@ __PACKAGE__->set_primary_key("build", "productnr"); __PACKAGE__->belongs_to("build", "Hydra::Schema::Builds", { id => "build" }); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:rfkj/A+Li7Q0hWydqtJHAw +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:yTJRsclWIpHApRSLqyR06Q # You can replace this text with custom content, and it will be preserved on regeneration diff --git a/src/lib/Hydra/Schema/BuildResultInfo.pm b/src/lib/Hydra/Schema/BuildResultInfo.pm index a8e24310..8c740d96 100644 --- a/src/lib/Hydra/Schema/BuildResultInfo.pm +++ b/src/lib/Hydra/Schema/BuildResultInfo.pm @@ -26,14 +26,22 @@ __PACKAGE__->add_columns( { data_type => "text", is_nullable => 0, size => undef }, "keep", { data_type => "integer", is_nullable => 0, size => undef }, + "faileddepbuild", + { data_type => "integer", is_nullable => 0, size => undef }, + "faileddepstepnr", + { data_type => "integer", is_nullable => 0, size => undef }, ); __PACKAGE__->set_primary_key("id"); __PACKAGE__->belongs_to("id", "Hydra::Schema::Builds", { id => "id" }); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:wmL9881G+dZrgHKM83dHXw +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:o1XAUgKd15pN76Rs8aX+IA +__PACKAGE__->belongs_to( + "failedDep", + "Hydra::Schema::BuildSteps", + { id => "faileddepbuild", stepnr => "faileddepstepnr" }, +); -# You can replace this text with custom content, and it will be preserved on regeneration 1; diff --git a/src/lib/Hydra/Schema/BuildSchedulingInfo.pm b/src/lib/Hydra/Schema/BuildSchedulingInfo.pm index a9d4cf36..fb8ca20e 100644 --- a/src/lib/Hydra/Schema/BuildSchedulingInfo.pm +++ b/src/lib/Hydra/Schema/BuildSchedulingInfo.pm @@ -27,8 +27,8 @@ __PACKAGE__->set_primary_key("id"); __PACKAGE__->belongs_to("id", "Hydra::Schema::Builds", { id => "id" }); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:6cTj5JovtTmtemvQjWtucQ +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:MeS9DLCs3kuhS4pTqfl4Lg # You can replace this text with custom content, and it will be preserved on regeneration diff --git a/src/lib/Hydra/Schema/BuildSteps.pm b/src/lib/Hydra/Schema/BuildSteps.pm index d7fef19e..4d927d59 100644 --- a/src/lib/Hydra/Schema/BuildSteps.pm +++ b/src/lib/Hydra/Schema/BuildSteps.pm @@ -35,9 +35,7 @@ __PACKAGE__->set_primary_key("id", "stepnr"); __PACKAGE__->belongs_to("id", "Hydra::Schema::Builds", { id => "id" }); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:LyNlkn5XjBnLp7M4ipB/ZQ +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:nFSksz60oxcM1+nYqAdjzw - -# You can replace this text with custom content, and it will be preserved on regeneration 1; diff --git a/src/lib/Hydra/Schema/Builds.pm b/src/lib/Hydra/Schema/Builds.pm index 8e6542d5..a3ed1233 100644 --- a/src/lib/Hydra/Schema/Builds.pm +++ b/src/lib/Hydra/Schema/Builds.pm @@ -76,8 +76,8 @@ __PACKAGE__->has_many( ); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:8U9CmcfeowLJVViKiR3n1g +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:p9SIzdyW9p4+eqwKwXzAdQ __PACKAGE__->has_many(dependents => 'Hydra::Schema::BuildInputs', 'dependency'); diff --git a/src/lib/Hydra/Schema/CachedPathInputs.pm b/src/lib/Hydra/Schema/CachedPathInputs.pm index cb70bc48..887d9cfe 100644 --- a/src/lib/Hydra/Schema/CachedPathInputs.pm +++ b/src/lib/Hydra/Schema/CachedPathInputs.pm @@ -22,8 +22,8 @@ __PACKAGE__->add_columns( __PACKAGE__->set_primary_key("srcpath", "sha256hash"); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:+F3EiYsvXuOjnGDrkhLxng +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:zvTGb3qhlacSWzc0H+7dfg # You can replace this text with custom content, and it will be preserved on regeneration diff --git a/src/lib/Hydra/Schema/CachedSubversionInputs.pm b/src/lib/Hydra/Schema/CachedSubversionInputs.pm index ea17cd02..2f4e55c4 100644 --- a/src/lib/Hydra/Schema/CachedSubversionInputs.pm +++ b/src/lib/Hydra/Schema/CachedSubversionInputs.pm @@ -20,8 +20,8 @@ __PACKAGE__->add_columns( __PACKAGE__->set_primary_key("uri", "revision"); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:O0QvXjOulMVTjhW4rRHkmQ +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:TY8SHfWiBibwiG3WPkP8Mg # You can replace this text with custom content, and it will be preserved on regeneration diff --git a/src/lib/Hydra/Schema/JobsetInputAlts.pm b/src/lib/Hydra/Schema/JobsetInputAlts.pm index 5dd7f153..2c7eb411 100644 --- a/src/lib/Hydra/Schema/JobsetInputAlts.pm +++ b/src/lib/Hydra/Schema/JobsetInputAlts.pm @@ -31,8 +31,8 @@ __PACKAGE__->belongs_to( ); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:qXh+0QbMUHIDmQCG9T5qdA +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:4iKXGIdrm56xZeHFxa/K4A # You can replace this text with custom content, and it will be preserved on regeneration diff --git a/src/lib/Hydra/Schema/JobsetInputs.pm b/src/lib/Hydra/Schema/JobsetInputs.pm index 8b0593f4..7f4da4e3 100644 --- a/src/lib/Hydra/Schema/JobsetInputs.pm +++ b/src/lib/Hydra/Schema/JobsetInputs.pm @@ -43,8 +43,8 @@ __PACKAGE__->has_many( ); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:TUTdwfhSsFMKBXa/wKenOQ +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:rh03/olFXpMiTA+zhRYltg # You can replace this text with custom content, and it will be preserved on regeneration diff --git a/src/lib/Hydra/Schema/Jobsets.pm b/src/lib/Hydra/Schema/Jobsets.pm index f6db720d..f5c8b8d3 100644 --- a/src/lib/Hydra/Schema/Jobsets.pm +++ b/src/lib/Hydra/Schema/Jobsets.pm @@ -50,8 +50,8 @@ __PACKAGE__->has_many( ); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:dVevQ8lPI2/IRpYoJgzLBA +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:IfDpZfiD9haRHLXGdkapGg # You can replace this text with custom content, and it will be preserved on regeneration diff --git a/src/lib/Hydra/Schema/Projects.pm b/src/lib/Hydra/Schema/Projects.pm index d0108e16..23418ede 100644 --- a/src/lib/Hydra/Schema/Projects.pm +++ b/src/lib/Hydra/Schema/Projects.pm @@ -45,8 +45,8 @@ __PACKAGE__->has_many( ); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Q5mSKzMxB9px2ja8NjK/9Q +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:7VRmssxrhgzfySJ6OGuhGg # You can replace this text with custom content, and it will be preserved on regeneration diff --git a/src/lib/Hydra/Schema/ReleaseSetJobs.pm b/src/lib/Hydra/Schema/ReleaseSetJobs.pm index cd0fb393..113e5c3a 100644 --- a/src/lib/Hydra/Schema/ReleaseSetJobs.pm +++ b/src/lib/Hydra/Schema/ReleaseSetJobs.pm @@ -32,8 +32,8 @@ __PACKAGE__->belongs_to( ); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:+Ky8V3sZIgT22hgF27Y0cw +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:GdOtS3nLs7dwpArSdGob0w # You can replace this text with custom content, and it will be preserved on regeneration diff --git a/src/lib/Hydra/Schema/ReleaseSets.pm b/src/lib/Hydra/Schema/ReleaseSets.pm index ea1f05f5..41daa1a1 100644 --- a/src/lib/Hydra/Schema/ReleaseSets.pm +++ b/src/lib/Hydra/Schema/ReleaseSets.pm @@ -29,8 +29,8 @@ __PACKAGE__->has_many( ); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:f+4AnWTJsi4RDfxoJxECgw +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:lQymTkBv/Av2y6iYjvP5PQ # You can replace this text with custom content, and it will be preserved on regeneration diff --git a/src/lib/Hydra/Schema/SystemTypes.pm b/src/lib/Hydra/Schema/SystemTypes.pm index 936f63c0..fb1d51e7 100644 --- a/src/lib/Hydra/Schema/SystemTypes.pm +++ b/src/lib/Hydra/Schema/SystemTypes.pm @@ -16,8 +16,8 @@ __PACKAGE__->add_columns( __PACKAGE__->set_primary_key("system"); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Af/LU15/hpXngfrBrDMI/A +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:KZzTedcMG8IxkLgEEkdn9A # You can replace this text with custom content, and it will be preserved on regeneration diff --git a/src/lib/Hydra/Schema/UserRoles.pm b/src/lib/Hydra/Schema/UserRoles.pm index 78e5bca6..5a827a90 100644 --- a/src/lib/Hydra/Schema/UserRoles.pm +++ b/src/lib/Hydra/Schema/UserRoles.pm @@ -17,8 +17,8 @@ __PACKAGE__->set_primary_key("username", "role"); __PACKAGE__->belongs_to("username", "Hydra::Schema::Users", { username => "username" }); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:/yDlbFhRYDzf+0VHzygrhA +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:9YGxnZD9hLoJJ8b1aejBuA # You can replace this text with custom content, and it will be preserved on regeneration diff --git a/src/lib/Hydra/Schema/Users.pm b/src/lib/Hydra/Schema/Users.pm index 65547eda..611db739 100644 --- a/src/lib/Hydra/Schema/Users.pm +++ b/src/lib/Hydra/Schema/Users.pm @@ -30,8 +30,8 @@ __PACKAGE__->has_many( ); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:d7M/Q6OucU9NUCSB5zZK7Q +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:HwKK4N8V5fzvR3XrKnrDEA # You can replace this text with custom content, and it will be preserved on regeneration diff --git a/src/root/build.tt b/src/root/build.tt index a7f74409..eac660de 100644 --- a/src/root/build.tt +++ b/src/root/build.tt @@ -36,13 +36,18 @@ Success [% ELSIF build.resultInfo.buildstatus == 1 %] Failed - Build returned a non-zero exit code + Build returned a non-zero exit code [% ELSIF build.resultInfo.buildstatus == 2 %] Failed A dependency of the build failed [% ELSIF build.resultInfo.buildstatus == 4 %] Failed Cancelled by user + [% ELSIF build.resultInfo.buildstatus == 5 %] + Failed + Build inhibited because a dependency previously failed to build + [% failedDep = build.resultInfo.failedDep %] + (namely, [% failedDep.outpath %]) [% ELSE %] Failed Build failed @@ -256,7 +261,7 @@ [% IF build.finished %] -[% IF build.resultInfo.errormsg %] +[% IF build.resultInfo.errormsg && build.resultInfo.buildstatus != 5 %]

Nix error output

diff --git a/src/script/hydra_build.pl b/src/script/hydra_build.pl index 8a1fef4d..e7c575d4 100755 --- a/src/script/hydra_build.pl +++ b/src/script/hydra_build.pl @@ -24,12 +24,47 @@ sub doBuild { my $buildStatus = 0; # = succeeded my $errormsg = undef; + + my $failedDepBuild; + my $failedDepStepNr; registerRoot $outPath; if (!isValidPath($outPath)) { $isCachedBuild = 0; + # Check whether a dependency of this build has previously + # failed. If so, don't even bother to build, since it will + # fail anyway. !!! Once Nix has negative caching, this code + # can go. + + my @drvDeps = split '\n', `nix-store --query --requisites --include-outputs $drvPath`; + die "cannot query dependencies of `$drvPath': $?" if $? != 0; + + @drvDeps = grep { $_ =~ /.drv$/ } @drvDeps; + + my @drvOutputs = split '\n', `nix-store --query --outputs @drvDeps`; + die "cannot query outputs of the dependencies of `$drvPath': $?" if $? != 0; + + foreach my $dep (@drvOutputs) { + # !!! This checks more than it has to, namely + # build-time-only dependencies of dependencies (which + # don't need to be built). However, it shouldn't matter: + # if the dependency was built, then presumably *its* + # dependencies were built as well. + # !!! should disregard fixed-output derivations (?) + if (!isValidPath($dep)) { + my ($step) = $db->resultset('BuildSteps')->search( + {outPath => $dep}, {rows => 1, order_by => "stopTime DESC"}); + if (defined $step && $step->status != 0) { + $buildStatus = 5; + $failedDepBuild = $step->id->id; + $failedDepStepNr = $step->stepnr; + goto done; + } + } + } + # Do the build. $startTime = time(); @@ -161,8 +196,10 @@ sub doBuild { $errormsg = undef unless $buildStatus == 3; } + done: + $db->txn_do(sub { - $build->({finished => 1, timestamp => time}); + $build->update({finished => 1, timestamp => time}); my $logPath = "/nix/var/log/nix/drvs/" . basename $drvPath; $logPath = undef unless -e $logPath; @@ -184,6 +221,8 @@ sub doBuild { , logfile => $logPath , errormsg => $errormsg , releasename => $releaseName + , faileddepbuild => $failedDepBuild + , faileddepstepnr => $failedDepStepNr }); if ($buildStatus == 0) { diff --git a/src/sql/hydra.sql b/src/sql/hydra.sql index 0ff8e36e..7b82d9f8 100644 --- a/src/sql/hydra.sql +++ b/src/sql/hydra.sql @@ -62,6 +62,7 @@ create table BuildResultInfo ( -- 2 = build of some dependency failed -- 3 = other failure (see errorMsg) -- 4 = build cancelled (removed from queue; never built) + -- 5 = build not done because a dependency failed previously buildStatus integer, errorMsg text, -- error message in case of a Nix failure @@ -74,18 +75,22 @@ create table BuildResultInfo ( releaseName text, -- e.g. "patchelf-0.5pre1234" keep integer not null default 0, -- true means never garbage-collect the build output + + -- If buildStatus == 5, the primary key of the failed build step. + failedDepBuild integer, + failedDepStepNr integer, foreign key (id) references Builds(id) on delete cascade -- ignored by sqlite ); create table BuildSteps ( - id integer not null, + id integer not null, -- !!! rename to "build" stepnr integer not null, type integer not null, -- 0 = build, 1 = substitution - drvPath text, + drvPath text, outPath text, logfile text,