the-distro/hydra
1
0
Fork 0
forked from lix-project/hydra
Code Pull requests Activity

* Negative caching: don't perform a build if a dependency already

Browse source 
failed in a previous build.  This is essential for Nixpkgs: we don't
  want to keep doing the same failed dependency (say, Glibc) over and
  over again for a few hundred jobs.
This commit is contained in:
Eelco Dolstra 2009-03-09 17:21:10 +00:00
parent 8725dc03ec
commit fca7fb20c4
21 changed files with 99 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/lib/Hydra/Schema.pm
View file

@ -8,8 +8,8 @@ use base 'DBIx::Class::Schema';
__PACKAGE__->load_classes;
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:hRwus0A1vxnFCzQxghFgTw
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:DQAmauNODAxmM2mF/AE2aQ
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/lib/Hydra/Schema/BuildInputs.pm
View file

@ -36,8 +36,8 @@ __PACKAGE__->belongs_to("build", "Hydra::Schema::Builds", { id => "build" });
__PACKAGE__->belongs_to("dependency", "Hydra::Schema::Builds", { id => "dependency" });
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:H25BhHe9wbU6nj6fSKjZnw
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:L6NP/+9zhMg4TRw3w911rg
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/lib/Hydra/Schema/BuildProducts.pm
View file

@ -35,8 +35,8 @@ __PACKAGE__->set_primary_key("build", "productnr");
__PACKAGE__->belongs_to("build", "Hydra::Schema::Builds", { id => "build" });
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:rfkj/A+Li7Q0hWydqtJHAw
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:yTJRsclWIpHApRSLqyR06Q
# You can replace this text with custom content, and it will be preserved on regeneration

14
src/lib/Hydra/Schema/BuildResultInfo.pm
View file

@ -26,14 +26,22 @@ __PACKAGE__->add_columns(
{ data_type => "text", is_nullable => 0, size => undef },
"keep",
{ data_type => "integer", is_nullable => 0, size => undef },
"faileddepbuild",
{ data_type => "integer", is_nullable => 0, size => undef },
"faileddepstepnr",
{ data_type => "integer", is_nullable => 0, size => undef },
);
__PACKAGE__->set_primary_key("id");
__PACKAGE__->belongs_to("id", "Hydra::Schema::Builds", { id => "id" });
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:wmL9881G+dZrgHKM83dHXw
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:o1XAUgKd15pN76Rs8aX+IA
__PACKAGE__->belongs_to(
"failedDep",
"Hydra::Schema::BuildSteps",
{ id => "faileddepbuild", stepnr => "faileddepstepnr" },
);
# You can replace this text with custom content, and it will be preserved on regeneration
1;

4
src/lib/Hydra/Schema/BuildSchedulingInfo.pm
View file

@ -27,8 +27,8 @@ __PACKAGE__->set_primary_key("id");
__PACKAGE__->belongs_to("id", "Hydra::Schema::Builds", { id => "id" });
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:6cTj5JovtTmtemvQjWtucQ
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:MeS9DLCs3kuhS4pTqfl4Lg
# You can replace this text with custom content, and it will be preserved on regeneration

6
src/lib/Hydra/Schema/BuildSteps.pm
View file

@ -35,9 +35,7 @@ __PACKAGE__->set_primary_key("id", "stepnr");
__PACKAGE__->belongs_to("id", "Hydra::Schema::Builds", { id => "id" });
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:LyNlkn5XjBnLp7M4ipB/ZQ
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:nFSksz60oxcM1+nYqAdjzw
# You can replace this text with custom content, and it will be preserved on regeneration
1;

4
src/lib/Hydra/Schema/Builds.pm
View file

@ -76,8 +76,8 @@ __PACKAGE__->has_many(
);
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:8U9CmcfeowLJVViKiR3n1g
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:p9SIzdyW9p4+eqwKwXzAdQ
__PACKAGE__->has_many(dependents => 'Hydra::Schema::BuildInputs', 'dependency');

4
src/lib/Hydra/Schema/CachedPathInputs.pm
View file

@ -22,8 +22,8 @@ __PACKAGE__->add_columns(
__PACKAGE__->set_primary_key("srcpath", "sha256hash");
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:+F3EiYsvXuOjnGDrkhLxng
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:zvTGb3qhlacSWzc0H+7dfg
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/lib/Hydra/Schema/CachedSubversionInputs.pm
View file

@ -20,8 +20,8 @@ __PACKAGE__->add_columns(
__PACKAGE__->set_primary_key("uri", "revision");
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:O0QvXjOulMVTjhW4rRHkmQ
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:TY8SHfWiBibwiG3WPkP8Mg
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/lib/Hydra/Schema/JobsetInputAlts.pm
View file

@ -31,8 +31,8 @@ __PACKAGE__->belongs_to(
);
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:qXh+0QbMUHIDmQCG9T5qdA
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:4iKXGIdrm56xZeHFxa/K4A
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/lib/Hydra/Schema/JobsetInputs.pm
View file

@ -43,8 +43,8 @@ __PACKAGE__->has_many(
);
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:TUTdwfhSsFMKBXa/wKenOQ
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:rh03/olFXpMiTA+zhRYltg
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/lib/Hydra/Schema/Jobsets.pm
View file

@ -50,8 +50,8 @@ __PACKAGE__->has_many(
);
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:dVevQ8lPI2/IRpYoJgzLBA
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:IfDpZfiD9haRHLXGdkapGg
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/lib/Hydra/Schema/Projects.pm
View file

@ -45,8 +45,8 @@ __PACKAGE__->has_many(
);
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Q5mSKzMxB9px2ja8NjK/9Q
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:7VRmssxrhgzfySJ6OGuhGg
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/lib/Hydra/Schema/ReleaseSetJobs.pm
View file

@ -32,8 +32,8 @@ __PACKAGE__->belongs_to(
);
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:+Ky8V3sZIgT22hgF27Y0cw
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:GdOtS3nLs7dwpArSdGob0w
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/lib/Hydra/Schema/ReleaseSets.pm
View file

@ -29,8 +29,8 @@ __PACKAGE__->has_many(
);
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:f+4AnWTJsi4RDfxoJxECgw
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:lQymTkBv/Av2y6iYjvP5PQ
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/lib/Hydra/Schema/SystemTypes.pm
View file

@ -16,8 +16,8 @@ __PACKAGE__->add_columns(
__PACKAGE__->set_primary_key("system");
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Af/LU15/hpXngfrBrDMI/A
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:KZzTedcMG8IxkLgEEkdn9A
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/lib/Hydra/Schema/UserRoles.pm
View file

@ -17,8 +17,8 @@ __PACKAGE__->set_primary_key("username", "role");
__PACKAGE__->belongs_to("username", "Hydra::Schema::Users", { username => "username" });
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:/yDlbFhRYDzf+0VHzygrhA
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:9YGxnZD9hLoJJ8b1aejBuA
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/lib/Hydra/Schema/Users.pm
View file

@ -30,8 +30,8 @@ __PACKAGE__->has_many(
);
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-06 14:20:12
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:d7M/Q6OucU9NUCSB5zZK7Q
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-09 18:05:06
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:HwKK4N8V5fzvR3XrKnrDEA
# You can replace this text with custom content, and it will be preserved on regeneration

9
src/root/build.tt
View file

@ -36,13 +36,18 @@
<strong>Success</strong>
[% ELSIF build.resultInfo.buildstatus == 1 %]
<img src="/static/images/failure.gif" alt="Failed" />
<span class="error">Build returned a non-zero exit code</span>
<span class="error">Build returned a non-zero exit code</span>
[% ELSIF build.resultInfo.buildstatus == 2 %]
<img src="/static/images/failure.gif" alt="Failed" />
<span class="error">A dependency of the build failed</span>
[% ELSIF build.resultInfo.buildstatus == 4 %]
<img src="/static/images/failure.gif" alt="Failed" />
<span class="error">Cancelled by user</span>
[% ELSIF build.resultInfo.buildstatus == 5 %]
<img src="/static/images/failure.gif" alt="Failed" />
<span class="error">Build inhibited because a dependency previously failed to build</span>
[% failedDep = build.resultInfo.failedDep %]
(namely, <a href="[% c.uri_for('/build' failedDep.id.id 'nixlog' failedDep.stepnr) %]"><tt>[% failedDep.outpath %]</tt></a>)
[% ELSE %]
<img src="/static/images/failure.gif" alt="Failed" />
<span class="error">Build failed</span>
@ -256,7 +261,7 @@
[% IF build.finished %]
[% IF build.resultInfo.errormsg %]
[% IF build.resultInfo.errormsg && build.resultInfo.buildstatus != 5 %]
<h2 id="nix-error">Nix error output</h2>

41
src/script/hydra_build.pl
View file

@ -24,12 +24,47 @@ sub doBuild {
my $buildStatus = 0; # = succeeded
my $errormsg = undef;
my $failedDepBuild;
my $failedDepStepNr;
registerRoot $outPath;
if (!isValidPath($outPath)) {
$isCachedBuild = 0;
# Check whether a dependency of this build has previously
# failed. If so, don't even bother to build, since it will
# fail anyway. !!! Once Nix has negative caching, this code
# can go.
my @drvDeps = split '\n', `nix-store --query --requisites --include-outputs $drvPath`;
die "cannot query dependencies of `$drvPath': $?" if $? != 0;
@drvDeps = grep { $_ =~ /.drv$/ } @drvDeps;
my @drvOutputs = split '\n', `nix-store --query --outputs @drvDeps`;
die "cannot query outputs of the dependencies of `$drvPath': $?" if $? != 0;
foreach my $dep (@drvOutputs) {
# !!! This checks more than it has to, namely
# build-time-only dependencies of dependencies (which
# don't need to be built). However, it shouldn't matter:
# if the dependency was built, then presumably *its*
# dependencies were built as well.
# !!! should disregard fixed-output derivations (?)
if (!isValidPath($dep)) {
my ($step) = $db->resultset('BuildSteps')->search(
{outPath => $dep}, {rows => 1, order_by => "stopTime DESC"});
if (defined $step && $step->status != 0) {
$buildStatus = 5;
$failedDepBuild = $step->id->id;
$failedDepStepNr = $step->stepnr;
goto done;
}
}
}
# Do the build.
$startTime = time();
@ -161,8 +196,10 @@ sub doBuild {
$errormsg = undef unless $buildStatus == 3;
}
done:
$db->txn_do(sub {
$build->({finished => 1, timestamp => time});
$build->update({finished => 1, timestamp => time});
my $logPath = "/nix/var/log/nix/drvs/" . basename $drvPath;
$logPath = undef unless -e $logPath;
@ -184,6 +221,8 @@ sub doBuild {
, logfile => $logPath
, errormsg => $errormsg
, releasename => $releaseName
, faileddepbuild => $failedDepBuild
, faileddepstepnr => $failedDepStepNr
});
if ($buildStatus == 0) {

9
src/sql/hydra.sql
View file

@ -62,6 +62,7 @@ create table BuildResultInfo (
-- 2 = build of some dependency failed
-- 3 = other failure (see errorMsg)
-- 4 = build cancelled (removed from queue; never built)
-- 5 = build not done because a dependency failed previously
buildStatus integer,
errorMsg text, -- error message in case of a Nix failure
@ -74,18 +75,22 @@ create table BuildResultInfo (
releaseName text, -- e.g. "patchelf-0.5pre1234"
keep integer not null default 0, -- true means never garbage-collect the build output
-- If buildStatus == 5, the primary key of the failed build step.
failedDepBuild integer,
failedDepStepNr integer,
foreign key (id) references Builds(id) on delete cascade -- ignored by sqlite
);
create table BuildSteps (
id integer not null,
id integer not null, -- !!! rename to "build"
stepnr integer not null,
type integer not null, -- 0 = build, 1 = substitution
drvPath text,
drvPath text,
outPath text,
logfile text,