* Negative caching: don't perform a build if a dependency already

failed in a previous build.  This is essential for Nixpkgs: we don't
  want to keep doing the same failed dependency (say, Glibc) over and
  over again for a few hundred jobs.
This commit is contained in:
Eelco Dolstra 2009-03-09 17:21:10 +00:00
parent 8725dc03ec
commit fca7fb20c4
21 changed files with 99 additions and 44 deletions

View file

@ -8,8 +8,8 @@ use base 'DBIx::Class::Schema';
__PACKAGE__->load_classes; __PACKAGE__->load_classes;
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12 # Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:hRwus0A1vxnFCzQxghFgTw # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:DQAmauNODAxmM2mF/AE2aQ
# You can replace this text with custom content, and it will be preserved on regeneration # You can replace this text with custom content, and it will be preserved on regeneration

View file

@ -36,8 +36,8 @@ __PACKAGE__->belongs_to("build", "Hydra::Schema::Builds", { id => "build" });
__PACKAGE__->belongs_to("dependency", "Hydra::Schema::Builds", { id => "dependency" }); __PACKAGE__->belongs_to("dependency", "Hydra::Schema::Builds", { id => "dependency" });
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12 # Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:H25BhHe9wbU6nj6fSKjZnw # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:L6NP/+9zhMg4TRw3w911rg
# You can replace this text with custom content, and it will be preserved on regeneration # You can replace this text with custom content, and it will be preserved on regeneration

View file

@ -35,8 +35,8 @@ __PACKAGE__->set_primary_key("build", "productnr");
__PACKAGE__->belongs_to("build", "Hydra::Schema::Builds", { id => "build" }); __PACKAGE__->belongs_to("build", "Hydra::Schema::Builds", { id => "build" });
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12 # Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:rfkj/A+Li7Q0hWydqtJHAw # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:yTJRsclWIpHApRSLqyR06Q
# You can replace this text with custom content, and it will be preserved on regeneration # You can replace this text with custom content, and it will be preserved on regeneration

View file

@ -26,14 +26,22 @@ __PACKAGE__->add_columns(
{ data_type => "text", is_nullable => 0, size => undef }, { data_type => "text", is_nullable => 0, size => undef },
"keep", "keep",
{ data_type => "integer", is_nullable => 0, size => undef }, { data_type => "integer", is_nullable => 0, size => undef },
"faileddepbuild",
{ data_type => "integer", is_nullable => 0, size => undef },
"faileddepstepnr",
{ data_type => "integer", is_nullable => 0, size => undef },
); );
__PACKAGE__->set_primary_key("id"); __PACKAGE__->set_primary_key("id");
__PACKAGE__->belongs_to("id", "Hydra::Schema::Builds", { id => "id" }); __PACKAGE__->belongs_to("id", "Hydra::Schema::Builds", { id => "id" });
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12 # Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:wmL9881G+dZrgHKM83dHXw # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:o1XAUgKd15pN76Rs8aX+IA
__PACKAGE__->belongs_to(
"failedDep",
"Hydra::Schema::BuildSteps",
{ id => "faileddepbuild", stepnr => "faileddepstepnr" },
);
# You can replace this text with custom content, and it will be preserved on regeneration
1; 1;

View file

@ -27,8 +27,8 @@ __PACKAGE__->set_primary_key("id");
__PACKAGE__->belongs_to("id", "Hydra::Schema::Builds", { id => "id" }); __PACKAGE__->belongs_to("id", "Hydra::Schema::Builds", { id => "id" });
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12 # Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:6cTj5JovtTmtemvQjWtucQ # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:MeS9DLCs3kuhS4pTqfl4Lg
# You can replace this text with custom content, and it will be preserved on regeneration # You can replace this text with custom content, and it will be preserved on regeneration

View file

@ -35,9 +35,7 @@ __PACKAGE__->set_primary_key("id", "stepnr");
__PACKAGE__->belongs_to("id", "Hydra::Schema::Builds", { id => "id" }); __PACKAGE__->belongs_to("id", "Hydra::Schema::Builds", { id => "id" });
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12 # Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:LyNlkn5XjBnLp7M4ipB/ZQ # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:nFSksz60oxcM1+nYqAdjzw
# You can replace this text with custom content, and it will be preserved on regeneration
1; 1;

View file

@ -76,8 +76,8 @@ __PACKAGE__->has_many(
); );
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12 # Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:8U9CmcfeowLJVViKiR3n1g # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:p9SIzdyW9p4+eqwKwXzAdQ
__PACKAGE__->has_many(dependents => 'Hydra::Schema::BuildInputs', 'dependency'); __PACKAGE__->has_many(dependents => 'Hydra::Schema::BuildInputs', 'dependency');

View file

@ -22,8 +22,8 @@ __PACKAGE__->add_columns(
__PACKAGE__->set_primary_key("srcpath", "sha256hash"); __PACKAGE__->set_primary_key("srcpath", "sha256hash");
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12 # Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:+F3EiYsvXuOjnGDrkhLxng # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:zvTGb3qhlacSWzc0H+7dfg
# You can replace this text with custom content, and it will be preserved on regeneration # You can replace this text with custom content, and it will be preserved on regeneration

View file

@ -20,8 +20,8 @@ __PACKAGE__->add_columns(
__PACKAGE__->set_primary_key("uri", "revision"); __PACKAGE__->set_primary_key("uri", "revision");
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12 # Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:O0QvXjOulMVTjhW4rRHkmQ # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:TY8SHfWiBibwiG3WPkP8Mg
# You can replace this text with custom content, and it will be preserved on regeneration # You can replace this text with custom content, and it will be preserved on regeneration

View file

@ -31,8 +31,8 @@ __PACKAGE__->belongs_to(
); );
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12 # Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:qXh+0QbMUHIDmQCG9T5qdA # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:4iKXGIdrm56xZeHFxa/K4A
# You can replace this text with custom content, and it will be preserved on regeneration # You can replace this text with custom content, and it will be preserved on regeneration

View file

@ -43,8 +43,8 @@ __PACKAGE__->has_many(
); );
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12 # Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:TUTdwfhSsFMKBXa/wKenOQ # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:rh03/olFXpMiTA+zhRYltg
# You can replace this text with custom content, and it will be preserved on regeneration # You can replace this text with custom content, and it will be preserved on regeneration

View file

@ -50,8 +50,8 @@ __PACKAGE__->has_many(
); );
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12 # Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:dVevQ8lPI2/IRpYoJgzLBA # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:IfDpZfiD9haRHLXGdkapGg
# You can replace this text with custom content, and it will be preserved on regeneration # You can replace this text with custom content, and it will be preserved on regeneration

View file

@ -45,8 +45,8 @@ __PACKAGE__->has_many(
); );
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12 # Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Q5mSKzMxB9px2ja8NjK/9Q # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:7VRmssxrhgzfySJ6OGuhGg
# You can replace this text with custom content, and it will be preserved on regeneration # You can replace this text with custom content, and it will be preserved on regeneration

View file

@ -32,8 +32,8 @@ __PACKAGE__->belongs_to(
); );
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12 # Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:+Ky8V3sZIgT22hgF27Y0cw # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:GdOtS3nLs7dwpArSdGob0w
# You can replace this text with custom content, and it will be preserved on regeneration # You can replace this text with custom content, and it will be preserved on regeneration

View file

@ -29,8 +29,8 @@ __PACKAGE__->has_many(
); );
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12 # Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:f+4AnWTJsi4RDfxoJxECgw # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:lQymTkBv/Av2y6iYjvP5PQ
# You can replace this text with custom content, and it will be preserved on regeneration # You can replace this text with custom content, and it will be preserved on regeneration

View file

@ -16,8 +16,8 @@ __PACKAGE__->add_columns(
__PACKAGE__->set_primary_key("system"); __PACKAGE__->set_primary_key("system");
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12 # Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Af/LU15/hpXngfrBrDMI/A # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:KZzTedcMG8IxkLgEEkdn9A
# You can replace this text with custom content, and it will be preserved on regeneration # You can replace this text with custom content, and it will be preserved on regeneration

View file

@ -17,8 +17,8 @@ __PACKAGE__->set_primary_key("username", "role");
__PACKAGE__->belongs_to("username", "Hydra::Schema::Users", { username => "username" }); __PACKAGE__->belongs_to("username", "Hydra::Schema::Users", { username => "username" });
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12 # Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:/yDlbFhRYDzf+0VHzygrhA # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:9YGxnZD9hLoJJ8b1aejBuA
# You can replace this text with custom content, and it will be preserved on regeneration # You can replace this text with custom content, and it will be preserved on regeneration

View file

@ -30,8 +30,8 @@ __PACKAGE__->has_many(
); );
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12 # Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:d7M/Q6OucU9NUCSB5zZK7Q # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:HwKK4N8V5fzvR3XrKnrDEA
# You can replace this text with custom content, and it will be preserved on regeneration # You can replace this text with custom content, and it will be preserved on regeneration

View file

@ -43,6 +43,11 @@
[% ELSIF build.resultInfo.buildstatus == 4 %] [% ELSIF build.resultInfo.buildstatus == 4 %]
<img src="/static/images/failure.gif" alt="Failed" /> <img src="/static/images/failure.gif" alt="Failed" />
<span class="error">Cancelled by user</span> <span class="error">Cancelled by user</span>
[% ELSIF build.resultInfo.buildstatus == 5 %]
<img src="/static/images/failure.gif" alt="Failed" />
<span class="error">Build inhibited because a dependency previously failed to build</span>
[% failedDep = build.resultInfo.failedDep %]
(namely, <a href="[% c.uri_for('/build' failedDep.id.id 'nixlog' failedDep.stepnr) %]"><tt>[% failedDep.outpath %]</tt></a>)
[% ELSE %] [% ELSE %]
<img src="/static/images/failure.gif" alt="Failed" /> <img src="/static/images/failure.gif" alt="Failed" />
<span class="error">Build failed</span> <span class="error">Build failed</span>
@ -256,7 +261,7 @@
[% IF build.finished %] [% IF build.finished %]
[% IF build.resultInfo.errormsg %] [% IF build.resultInfo.errormsg && build.resultInfo.buildstatus != 5 %]
<h2 id="nix-error">Nix error output</h2> <h2 id="nix-error">Nix error output</h2>

View file

@ -25,11 +25,46 @@ sub doBuild {
my $errormsg = undef; my $errormsg = undef;
my $failedDepBuild;
my $failedDepStepNr;
registerRoot $outPath; registerRoot $outPath;
if (!isValidPath($outPath)) { if (!isValidPath($outPath)) {
$isCachedBuild = 0; $isCachedBuild = 0;
# Check whether a dependency of this build has previously
# failed. If so, don't even bother to build, since it will
# fail anyway. !!! Once Nix has negative caching, this code
# can go.
my @drvDeps = split '\n', `nix-store --query --requisites --include-outputs $drvPath`;
die "cannot query dependencies of `$drvPath': $?" if $? != 0;
@drvDeps = grep { $_ =~ /.drv$/ } @drvDeps;
my @drvOutputs = split '\n', `nix-store --query --outputs @drvDeps`;
die "cannot query outputs of the dependencies of `$drvPath': $?" if $? != 0;
foreach my $dep (@drvOutputs) {
# !!! This checks more than it has to, namely
# build-time-only dependencies of dependencies (which
# don't need to be built). However, it shouldn't matter:
# if the dependency was built, then presumably *its*
# dependencies were built as well.
# !!! should disregard fixed-output derivations (?)
if (!isValidPath($dep)) {
my ($step) = $db->resultset('BuildSteps')->search(
{outPath => $dep}, {rows => 1, order_by => "stopTime DESC"});
if (defined $step && $step->status != 0) {
$buildStatus = 5;
$failedDepBuild = $step->id->id;
$failedDepStepNr = $step->stepnr;
goto done;
}
}
}
# Do the build. # Do the build.
$startTime = time(); $startTime = time();
@ -161,8 +196,10 @@ sub doBuild {
$errormsg = undef unless $buildStatus == 3; $errormsg = undef unless $buildStatus == 3;
} }
done:
$db->txn_do(sub { $db->txn_do(sub {
$build->({finished => 1, timestamp => time}); $build->update({finished => 1, timestamp => time});
my $logPath = "/nix/var/log/nix/drvs/" . basename $drvPath; my $logPath = "/nix/var/log/nix/drvs/" . basename $drvPath;
$logPath = undef unless -e $logPath; $logPath = undef unless -e $logPath;
@ -184,6 +221,8 @@ sub doBuild {
, logfile => $logPath , logfile => $logPath
, errormsg => $errormsg , errormsg => $errormsg
, releasename => $releaseName , releasename => $releaseName
, faileddepbuild => $failedDepBuild
, faileddepstepnr => $failedDepStepNr
}); });
if ($buildStatus == 0) { if ($buildStatus == 0) {

View file

@ -62,6 +62,7 @@ create table BuildResultInfo (
-- 2 = build of some dependency failed -- 2 = build of some dependency failed
-- 3 = other failure (see errorMsg) -- 3 = other failure (see errorMsg)
-- 4 = build cancelled (removed from queue; never built) -- 4 = build cancelled (removed from queue; never built)
-- 5 = build not done because a dependency failed previously
buildStatus integer, buildStatus integer,
errorMsg text, -- error message in case of a Nix failure errorMsg text, -- error message in case of a Nix failure
@ -75,12 +76,16 @@ create table BuildResultInfo (
keep integer not null default 0, -- true means never garbage-collect the build output keep integer not null default 0, -- true means never garbage-collect the build output
-- If buildStatus == 5, the primary key of the failed build step.
failedDepBuild integer,
failedDepStepNr integer,
foreign key (id) references Builds(id) on delete cascade -- ignored by sqlite foreign key (id) references Builds(id) on delete cascade -- ignored by sqlite
); );
create table BuildSteps ( create table BuildSteps (
id integer not null, id integer not null, -- !!! rename to "build"
stepnr integer not null, stepnr integer not null,
type integer not null, -- 0 = build, 1 = substitution type integer not null, -- 0 = build, 1 = substitution