ma27/hydra
0
0
Fork 0
forked from lix-project/hydra
Code Pull requests Activity

Add multiple output support

Browse source 
This requires turning the outPath columns in the Builds and BuildSteps
tables into separate tables, and so requires a schema upgrade.
This commit is contained in:
Eelco Dolstra 2013-02-13 16:49:28 +00:00
parent 799e5437bd
commit 10882a1ffd
23 changed files with 465 additions and 344 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
src/c/hydra-eval-jobs.cc
View file

@ -123,11 +123,12 @@ static void findJobsWrapped(EvalState & state, XMLWriter & doc,
XMLAttrs xmlAttrs;
Path drvPath;
DrvInfo::Outputs outputs = drv.queryOutputs(state);
xmlAttrs["jobName"] = attrPath;
xmlAttrs["nixName"] = drv.name;
xmlAttrs["system"] = drv.system;
xmlAttrs["drvPath"] = drvPath = drv.queryDrvPath(state);
xmlAttrs["outPath"] = drv.queryOutPath(state);
MetaInfo meta = drv.queryMetaInfo(state);
xmlAttrs["description"] = queryMetaFieldString(meta, "description");
xmlAttrs["longDescription"] = queryMetaFieldString(meta, "longDescription");
@ -164,6 +165,14 @@ static void findJobsWrapped(EvalState & state, XMLWriter & doc,
}
XMLOpenElement _(doc, "job", xmlAttrs);
foreach (DrvInfo::Outputs::iterator, j, outputs) {
XMLAttrs attrs2;
attrs2["name"] = j->first;
attrs2["path"] = j->second;
doc.writeEmptyElement("output", attrs2);
}
showArgsUsed(doc, argsUsed);
}

4
src/lib/Hydra/Base/Controller/ListBuilds.pm
View file

@ -78,7 +78,9 @@ sub nix : Chained('get_builds') PathPart('channel') CaptureArgs(1) {
$c->stash->{channelName} = $c->stash->{channelBaseName} . "-latest";
$c->stash->{channelBuilds} = $c->stash->{latestSucceeded}
->search_literal("exists (select 1 from buildproducts where build = me.id and type = 'nix-build')")
->search({}, { columns => [@buildListColumns, 'drvpath', 'outpath', 'description', 'homepage'] });
->search({}, { join => ["buildoutputs"]
, columns => [@buildListColumns, 'drvpath', 'description', 'homepage']
, '+select' => ['buildoutputs.path', 'buildoutputs.name'], '+as' => ['outpath', 'outname'] });
}
else {
notFound($c, "Unknown channel `$channelName'.");

31
src/lib/Hydra/Base/Controller/NixChannel.pm
View file

@ -3,6 +3,7 @@ package Hydra::Base::Controller::NixChannel;
use strict;
use warnings;
use base 'Catalyst::Controller';
use List::MoreUtils qw(all);
use Nix::Store;
use Hydra::Helper::Nix;
use Hydra::Helper::CatalystUtils;
@ -12,26 +13,23 @@ sub getChannelData {
my ($c, $checkValidity) = @_;
my @storePaths = ();
$c->stash->{nixPkgs} = [];
foreach my $build ($c->stash->{channelBuilds}->all) {
next if $checkValidity && !isValidPath($build->outpath);
#if (isValidPath($build->drvpath)) {
# # Adding `drvpath' implies adding `outpath' because of the
# # `--include-outputs' flag passed to `nix-store'.
# push @storePaths, $build->drvpath;
#} else {
# push @storePaths, $build->outpath;
#}
push @storePaths, $build->outpath;
my $pkgName = $build->nixname . "-" . $build->system . "-" . $build->id;
$c->stash->{nixPkgs}->{"${pkgName}.nixpkg"} = {build => $build, name => $pkgName};
my $outPath = $build->get_column("outpath");
my $outName = $build->get_column("outname");
next if $checkValidity && !isValidPath($outPath);
push @storePaths, $outPath;
my $pkgName = $build->nixname . "-" . $build->system . "-" . $build->id . ($outName ne "out" ? "-" . $outName : "");
push @{$c->stash->{nixPkgs}}, { build => $build, name => $pkgName, outPath => $outPath, outName => $outName };
# Put the system type in the manifest (for top-level paths) as
# a hint to the binary patch generator. (It shouldn't try to
# generate patches between builds for different systems.) It
# would be nice if Nix stored this info for every path but it
# doesn't.
$c->stash->{systemForPath}->{$build->outpath} = $build->system;
$c->stash->{systemForPath}->{$outPath} = $build->system;
};
print STDERR @storePaths, "\n";
$c->stash->{storePaths} = [@storePaths];
}
@ -42,6 +40,8 @@ sub closure : Chained('nix') PathPart {
getChannelData($c, 1);
# FIXME: get the closure of the selected path only.
# !!! quick hack; this is to make HEAD requests return the right
# MIME type. This is set in the view as well, but the view isn't
# called for HEAD requests. There should be a cleaner solution...
@ -62,11 +62,14 @@ sub pkg : Chained('nix') PathPart Args(1) {
if (!$c->stash->{build}) {
$pkgName =~ /-(\d+)\.nixpkg$/ or notFound($c, "Bad package name.");
# FIXME: need to handle multiple outputs: channelBuilds is
# joined with the build outputs, so find() can return multiple
# results.
$c->stash->{build} = $c->stash->{channelBuilds}->find({ id => $1 })
|| notFound($c, "No such package in this channel.");
}
if (!isValidPath($c->stash->{build}->outpath)) {
unless (all { isValidPath($_->path) } $c->stash->{build}->buildoutputs->all) {
$c->response->status(410); # "Gone"
error($c, "Build " . $c->stash->{build}->id . " is no longer available.");
}
@ -115,7 +118,7 @@ sub channel_contents : Chained('nix') PathPart('') Args(0) {
# channel.
getChannelData($c, 0);
$c->stash->{template} = 'channel-contents.tt';
$c->stash->{nixPkgs} = [sortPkgs (values %{$c->stash->{nixPkgs}})];
$c->stash->{nixPkgs} = [sortPkgs @{$c->stash->{nixPkgs}}];
}

89
src/lib/Hydra/Controller/Build.pm
View file

@ -10,6 +10,7 @@ use File::stat;
use File::Slurp;
use Data::Dump qw(dump);
use Nix::Store;
use List::MoreUtils qw(all);
sub build : Chained('/') PathPart CaptureArgs(1) {
@ -32,28 +33,44 @@ sub build : Chained('/') PathPart CaptureArgs(1) {
}
sub findBuildStepByOutPath {
my ($self, $c, $path, $status) = @_;
return $c->model('DB::BuildSteps')->search(
{ path => $path, busy => 0, status => $status },
{ join => ["buildstepoutputs"], order_by => ["stopTime"], limit => 1 })->single;
}
sub findBuildStepByDrvPath {
my ($self, $c, $drvPath, $status) = @_;
return $c->model('DB::BuildSteps')->search(
{ drvpath => $drvPath, busy => 0, status => $status },
{ order_by => ["stopTime"], limit => 1 })->single;
}
sub view_build : Chained('build') PathPart('') Args(0) {
my ($self, $c) = @_;
my $build = $c->stash->{build};
$c->stash->{template} = 'build.tt';
$c->stash->{available} = isValidPath $build->outpath;
$c->stash->{available} = all { isValidPath($_->path) } $build->buildoutputs->all;
$c->stash->{drvAvailable} = isValidPath $build->drvpath;
$c->stash->{flashMsg} = $c->flash->{buildMsg};
$c->stash->{pathHash} = $c->stash->{available} ? queryPathHash($build->outpath) : undef;
if (!$build->finished && $build->busy) {
$c->stash->{logtext} = read_file($build->logfile, err_mode => 'quiet') // "";
}
if ($build->finished && $build->iscachedbuild) {
(my $cachedBuildStep) = $c->model('DB::BuildSteps')->search({ outpath => $build->outpath }, {});
my $path = ($build->buildoutputs)[0]->path or die;
my $cachedBuildStep = findBuildStepByOutPath($self, $c, $path,
$build->buildstatus == 0 || $build->buildstatus == 6 ? 0 : 1);
$c->stash->{cachedBuild} = $cachedBuildStep->build if defined $cachedBuildStep;
}
if ($build->finished) {
if ($build->finished && 0) {
$c->stash->{prevBuilds} = [$c->model('DB::Builds')->search(
{ project => $c->stash->{project}->name
, jobset => $c->stash->{build}->jobset->name
@ -279,35 +296,31 @@ sub deps : Chained('build') PathPart('deps') {
my ($self, $c) = @_;
my $build = $c->stash->{build};
$c->stash->{available} = isValidPath $build->outpath;
$c->stash->{drvAvailable} = isValidPath $build->drvpath;
my $drvPath = $build->drvpath;
my @outPaths = map { $_->path } $build->buildoutputs->all;
my $drvpath = $build->drvpath;
my $outpath = $build->outpath;
$c->stash->{available} = all { isValidPath($_) } @outPaths;
$c->stash->{drvAvailable} = isValidPath $drvPath;
my @buildtimepaths = ();
my @buildtimepaths = $c->stash->{drvAvailable} ? computeFSClosure(0, 0, $drvPath) : ();
my @buildtimedeps = ();
@buildtimepaths = split '\n', `nix-store --query --requisites --include-outputs $drvpath` if isValidPath($build->drvpath);
my @runtimepaths = ();
my @runtimepaths = $c->stash->{available} ? computeFSClosure(0, 0, @outPaths) : ();
my @runtimedeps = ();
@runtimepaths = split '\n', `nix-store --query --requisites --include-outputs $outpath` if isValidPath($build->outpath);
foreach my $p (@buildtimepaths) {
my $buildStep;
($buildStep) = $c->model('DB::BuildSteps')->search({ outpath => $p }, {}) ;
my %dep = ( buildstep => $buildStep, path => $p ) ;
next unless $p =~ /\.drv$/;
my ($buildStep) = findBuildStepByDrvPath($self, $c, $p, 0);
my %dep = ( buildstep => $buildStep, path => $p );
push(@buildtimedeps, \%dep);
}
foreach my $p (@runtimepaths) {
my $buildStep;
($buildStep) = $c->model('DB::BuildSteps')->search({ outpath => $p }, {}) ;
my %dep = ( buildstep => $buildStep, path => $p ) ;
my ($buildStep) = findBuildStepByOutPath($self, $c, $p, 0);
my %dep = ( buildstep => $buildStep, path => $p );
push(@runtimedeps, \%dep);
}
$c->stash->{buildtimedeps} = \@buildtimedeps;
$c->stash->{runtimedeps} = \@runtimedeps;
@ -321,12 +334,17 @@ sub nix : Chained('build') PathPart('nix') CaptureArgs(0) {
my $build = $c->stash->{build};
notFound($c, "Build cannot be downloaded as a closure or Nix package.")
if !$build->buildproducts->find({type => "nix-build"});
if $build->buildproducts->search({type => "nix-build"})->count == 0;
notFound($c, "Path " . $build->outpath . " is no longer available.")
unless isValidPath($build->outpath);
foreach my $out ($build->buildoutputs) {
notFound($c, "Path " . $out->path . " is no longer available.")
unless isValidPath($out->path);
}
$c->stash->{channelBuilds} = $c->model('DB::Builds')->search({id => $build->id});
$c->stash->{channelBuilds} = $c->model('DB::Builds')->search(
{ id => $build->id },
{ join => ["buildoutputs"]
, '+select' => ['buildoutputs.path', 'buildoutputs.name'], '+as' => ['outpath', 'outname'] });
}
@ -377,22 +395,23 @@ sub cancel : Chained('build') PathPart Args(0) {
sub keep : Chained('build') PathPart Args(1) {
my ($self, $c, $newStatus) = @_;
my ($self, $c, $x) = @_;
my $keep = $x eq "1" ? 1 : 0;
my $build = $c->stash->{build};
requireProjectOwner($c, $build->project);
die unless $newStatus == 0 || $newStatus == 1;
registerRoot $build->outpath if $newStatus == 1;
if ($keep) {
registerRoot $_->path foreach $build->buildoutputs;
}
txn_do($c->model('DB')->schema, sub {
$build->update({keep => int $newStatus});
$build->update({keep => $keep});
});
$c->flash->{buildMsg} =
$newStatus == 0 ? "Build will not be kept." : "Build will be kept.";
$keep ? "Build will be kept." : "Build will not be kept.";
$c->res->redirect($c->uri_for($self->action_for("view_build"), $c->req->captures));
}
@ -414,9 +433,10 @@ sub add_to_release : Chained('build') PathPart('add-to-release') Args(0) {
error($c, "This build is already a part of release `$releaseName'.")
if $release->releasemembers->find({build => $build->id});
registerRoot $build->outpath;
error($c, "This build is no longer available.") unless isValidPath $build->outpath;
foreach my $output ($build->buildoutputs) {
error($c, "This build is no longer available.") unless isValidPath $output->path;
registerRoot $output->path;
}
$release->releasemembers->create({build => $build->id, description => $build->description});
@ -509,7 +529,8 @@ sub get_info : Chained('build') PathPart('api/get-info') Args(0) {
# !!! strip the json prefix
$c->stash->{jsonBuildId} = $build->id;
$c->stash->{jsonDrvPath} = $build->drvpath;
$c->stash->{jsonOutPath} = $build->outpath;
my $out = $build->buildoutputs->find({name => "out"});
$c->stash->{jsonOutPath} = $out->path if defined $out;
$c->forward('View::JSON');
}

5
src/lib/Hydra/Controller/JobsetEval.pm
View file

@ -141,7 +141,10 @@ sub nix : Chained('eval') PathPart('channel') CaptureArgs(0) {
$c->stash->{channelName} = $c->stash->{project}->name . "-" . $c->stash->{jobset}->name . "-latest";
$c->stash->{channelBuilds} = $c->stash->{eval}->builds
->search_literal("exists (select 1 from buildproducts where build = build.id and type = 'nix-build')")
->search({ finished => 1, buildstatus => 0 }, { columns => [@buildListColumns, 'drvpath', 'outpath', 'description', 'homepage'] });
->search({ finished => 1, buildstatus => 0 },
{ columns => [@buildListColumns, 'drvpath', 'description', 'homepage']
, join => ["buildoutputs"]
, '+select' => ['buildoutputs.path', 'buildoutputs.name'], '+as' => ['outpath', 'outname'] });
}

191
src/lib/Hydra/Helper/AddBuilds.pm
View file

@ -13,6 +13,7 @@ use File::Basename;
use File::stat;
use File::Path;
use File::Temp;
use File::Slurp;
our @ISA = qw(Exporter);
our @EXPORT = qw(
@ -43,14 +44,9 @@ sub getStorePathHash {
sub getReleaseName {
my ($outPath) = @_;
my $releaseName;
if (-e "$outPath/nix-support/hydra-release-name") {
open FILE, "$outPath/nix-support/hydra-release-name" or die;
$releaseName = <FILE>;
chomp $releaseName;
close FILE;
}
return undef unless -f "$outPath/nix-support/hydra-release-name";
my $releaseName = read_file("$outPath/nix-support/hydra-release-name");
chomp $releaseName;
return $releaseName;
}
@ -242,7 +238,7 @@ sub fetchInputBuild {
{ order_by => "me.id DESC", rows => 1
, where => \ attrsToSQL($attrs, "me.id") });
if (!defined $prevBuild || !isValidPath($prevBuild->outpath)) {
if (!defined $prevBuild || !isValidPath(getMainOutput($prevBuild)->path)) {
print STDERR "input `", $name, "': no previous build available\n";
return undef;
}
@ -256,7 +252,7 @@ sub fetchInputBuild {
my $version = $2 if $relName =~ /^($pkgNameRE)-($versionRE)$/;
return
{ storePath => $prevBuild->outpath
{ storePath => getMainOutput($prevBuild)->path
, id => $prevBuild->id
, version => $version
};
@ -275,7 +271,7 @@ sub fetchInputSystemBuild {
my @validBuilds = ();
foreach my $build (@latestBuilds) {
push(@validBuilds, $build) if isValidPath($build->outpath);
push(@validBuilds, $build) if !isValidPath(getMainOutput($build)->path);
}
if (scalar(@validBuilds) == 0) {
@ -293,7 +289,7 @@ sub fetchInputSystemBuild {
my $version = $2 if $relName =~ /^($pkgNameRE)-($versionRE)$/;
my $input =
{ storePath => $prevBuild->outpath
{ storePath => getMainOutput($prevBuild)->path
, id => $prevBuild->id
, version => $version
, system => $prevBuild->system
@ -666,6 +662,7 @@ sub buildInputToString {
return $result;
}
sub inputsToArgs {
my ($inputInfo, $exprType) = @_;
my @res = ();
@ -742,8 +739,8 @@ sub evalJobs {
my $jobs = XMLin(
$jobsXml,
ForceArray => ['error', 'job', 'arg'],
KeyAttr => [],
ForceArray => ['error', 'job', 'arg', 'output'],
KeyAttr => { output => "+name" },
SuppressEmpty => '')
or die "cannot parse XML output";
@ -751,7 +748,7 @@ sub evalJobs {
foreach my $job (@{$jobs->{job}}) {
my $validJob = 1;
foreach my $arg (@{$job->{arg}}) {
my $input = $inputInfo->{$arg->{name}}->[$arg->{altnr}] ;
my $input = $inputInfo->{$arg->{name}}->[$arg->{altnr}];
if ($input->{type} eq "sysbuild" && $input->{system} ne $job->{system}) {
$validJob = 0;
}
@ -769,60 +766,68 @@ sub evalJobs {
sub addBuildProducts {
my ($db, $build) = @_;
my $outPath = $build->outpath;
my $productnr = 1;
my $explicitProducts = 0;
if (-e "$outPath/nix-support/hydra-build-products") {
open LIST, "$outPath/nix-support/hydra-build-products" or die;
while (<LIST>) {
/^([\w\-]+)\s+([\w\-]+)\s+(\S+)(\s+(\S+))?$/ or next;
my $type = $1;
my $subtype = $2 eq "none" ? "" : $2;
my $path = $3;
my $defaultPath = $5;
next unless -e $path;
foreach my $output ($build->buildoutputs->all) {
my $outPath = $output->path;
if (-e "$outPath/nix-support/hydra-build-products") {
$explicitProducts = 1;
my $fileSize, my $sha1, my $sha256;
open LIST, "$outPath/nix-support/hydra-build-products" or die;
while (<LIST>) {
/^([\w\-]+)\s+([\w\-]+)\s+(\S+)(\s+(\S+))?$/ or next;
my $type = $1;
my $subtype = $2 eq "none" ? "" : $2;
my $path = $3;
my $defaultPath = $5;
next unless -e $path;
# !!! validate $path, $defaultPath
my $fileSize, my $sha1, my $sha256;
if (-f $path) {
my $st = stat($path) or die "cannot stat $path: $!";
$fileSize = $st->size;
# !!! validate $path, $defaultPath
$sha1 = `nix-hash --flat --type sha1 $path`
or die "cannot hash $path: $?";;
chomp $sha1;
if (-f $path) {
my $st = stat($path) or die "cannot stat $path: $!";
$fileSize = $st->size;
$sha256 = `nix-hash --flat --type sha256 $path`
or die "cannot hash $path: $?";;
chomp $sha256;
$sha1 = `nix-hash --flat --type sha1 $path`
or die "cannot hash $path: $?";;
chomp $sha1;
$sha256 = `nix-hash --flat --type sha256 $path`
or die "cannot hash $path: $?";;
chomp $sha256;
}
my $name = $path eq $outPath ? "" : basename $path;
$db->resultset('BuildProducts')->create(
{ build => $build->id
, productnr => $productnr++
, type => $type
, subtype => $subtype
, path => $path
, filesize => $fileSize
, sha1hash => $sha1
, sha256hash => $sha256
, name => $name
, defaultpath => $defaultPath
});
}
my $name = $path eq $outPath ? "" : basename $path;
$db->resultset('BuildProducts')->create(
{ build => $build->id
, productnr => $productnr++
, type => $type
, subtype => $subtype
, path => $path
, filesize => $fileSize
, sha1hash => $sha1
, sha256hash => $sha256
, name => $name
, defaultpath => $defaultPath
});
close LIST;
}
close LIST;
}
else {
return if $explicitProducts;
foreach my $output ($build->buildoutputs->all) {
my $outPath = $output->path;
$db->resultset('BuildProducts')->create(
{ build => $build->id
, productnr => $productnr++
, type => "nix-build"
, subtype => ""
, subtype => $output->name eq "out" ? "" : $output->name
, path => $outPath
, name => $build->nixname
});
@ -846,9 +851,17 @@ sub getPrevJobsetEval {
sub checkBuild {
my ($db, $project, $jobset, $inputInfo, $nixExprInput, $buildInfo, $buildIds, $prevEval, $jobOutPathMap) = @_;
my $jobName = $buildInfo->{jobName};
my $drvPath = $buildInfo->{drvPath};
my $outPath = $buildInfo->{outPath};
my @outputNames = sort keys %{$buildInfo->{output}};
die unless scalar @outputNames;
# In various checks we can use an arbitrary output (the first)
# rather than all outputs, since if one output is the same, the
# others will be as well.
my $firstOutputName = $outputNames[0];
my $firstOutputPath = $buildInfo->{output}->{$firstOutputName}->{path};
my $jobName = $buildInfo->{jobName} or die;
my $drvPath = $buildInfo->{drvPath} or die;
my $priority = 100;
$priority = int($buildInfo->{schedulingPriority})
@ -873,18 +886,21 @@ sub checkBuild {
# will be performed (though the last one will probably use the
# cached result from the first). This ensures that the builds
# with the highest ID will always be the ones that we want in
# the channels.
# !!! Checking $outPath doesn't take meta-attributes into
# account. For instance, do we want a new build to be
# scheduled if the meta.maintainers field is changed?
# the channels. FIXME: Checking the output paths doesn't take
# meta-attributes into account. For instance, do we want a
# new build to be scheduled if the meta.maintainers field is
# changed?
if (defined $prevEval) {
# Only check one output: if it's the same, the other will be as well.
my $firstOutput = $outputNames[0];
my ($prevBuild) = $prevEval->builds->search(
# The "project" and "jobset" constraints are
# semantically unnecessary (because they're implied by
# the eval), but they give a factor 1000 speedup on
# the Nixpkgs jobset with PostgreSQL.
{ project => $project->name, jobset => $jobset->name, job => $job->name, outPath => $outPath },
{ rows => 1, columns => ['id'] });
{ project => $project->name, jobset => $jobset->name, job => $job->name,
name => $firstOutputName, path => $firstOutputPath },
{ rows => 1, columns => ['id'], join => ['buildoutputs'] });
if (defined $prevBuild) {
print STDERR " already scheduled/built as build ", $prevBuild->id, "\n";
$buildIds->{$prevBuild->id} = 0;
@ -894,7 +910,7 @@ sub checkBuild {
# Prevent multiple builds with the same (job, outPath) from
# being added.
my $prev = $$jobOutPathMap{$job->name . "\t" . $outPath};
my $prev = $$jobOutPathMap{$job->name . "\t" . $firstOutputPath};
if (defined $prev) {
print STDERR " already scheduled as build ", $prev, "\n";
return;
@ -902,22 +918,41 @@ sub checkBuild {
my $time = time();
# Nope, so add it.
# Are the outputs already in the Nix store? Then add a cached
# build.
my %extraFlags;
if (isValidPath($outPath)) {
my $allValid = 1;
my $buildStatus;
my $releaseName;
foreach my $name (@outputNames) {
my $path = $buildInfo->{output}->{$name}->{path};
if (isValidPath($path)) {
if (-f "$path/nix-support/failed") {
$buildStatus = 6;
} else {
$buildStatus //= 0;
}
$releaseName //= getReleaseName($path);
} else {
$allValid = 0;
last;
}
}
if ($allValid) {
%extraFlags =
( finished => 1
, iscachedbuild => 1
, buildstatus => -f "$outPath/nix-support/failed" ? 6 : 0
, buildstatus => $buildStatus
, starttime => $time
, stoptime => $time
, errormsg => ""
, releasename => getReleaseName($outPath)
, releasename => $releaseName
);
} else {
%extraFlags = ( finished => 0 );
}
# Add the build to the database.
$build = $job->builds->create(
{ timestamp => $time
, description => $buildInfo->{description}
@ -929,7 +964,6 @@ sub checkBuild {
, timeout => $buildInfo->{timeout}
, nixname => $buildInfo->{nixName}
, drvpath => $drvPath
, outpath => $outPath
, system => $buildInfo->{system}
, nixexprinput => $jobset->nixexprinput
, nixexprpath => $jobset->nixexprpath
@ -939,8 +973,11 @@ sub checkBuild {
, %extraFlags
});
$build->buildoutputs->create({ name => $_, path => $buildInfo->{output}->{$_}->{path} })
foreach @outputNames;
$buildIds->{$build->id} = 1;
$$jobOutPathMap{$job->name . "\t" . $outPath} = $build->id;
$$jobOutPathMap{$job->name . "\t" . $firstOutputPath} = $build->id;
if ($build->iscachedbuild) {
print STDERR " marked as cached build ", $build->id, "\n";
@ -988,15 +1025,11 @@ sub restartBuild {
my ($db, $build) = @_;
txn_do($db, sub {
my $drvpath = $build->drvpath;
my $outpath = $build->outpath;
my @paths;
push @paths, $build->drvpath;
push @paths, $_->drvpath foreach $build->buildsteps;
my $paths = "";
foreach my $bs ($build->buildsteps) {
$paths = $paths . " " . $bs->outpath;
}
my $r = `nix-store --clear-failed-paths $paths $outpath`;
my $r = `nix-store --clear-failed-paths @paths`;
$build->update(
{ finished => 0

11
src/lib/Hydra/Helper/Nix.pm
View file

@ -14,7 +14,8 @@ our @EXPORT = qw(
getPrimaryBuildsForView
getPrimaryBuildTotal
getViewResult getLatestSuccessfulViewResult
jobsetOverview removeAsciiEscapes getDrvLogPath logContents);
jobsetOverview removeAsciiEscapes getDrvLogPath logContents
getMainOutput);
sub getHydraHome {
@ -278,4 +279,12 @@ sub removeAsciiEscapes {
}
sub getMainOutput {
my ($build) = @_;
return
$build->buildoutputs->find({name => "out"}) //
$build->buildoutputs->find({}, {limit => 1, order_by => ["name"]});
}
1;

26
src/lib/Hydra/Schema/BuildSteps.pm
View file

@ -44,11 +44,6 @@ __PACKAGE__->table("BuildSteps");
data_type: 'text'
is_nullable: 1
=head2 outpath
data_type: 'text'
is_nullable: 1
=head2 busy
data_type: 'integer'
@ -96,8 +91,6 @@ __PACKAGE__->add_columns(
{ data_type => "integer", is_nullable => 0 },
"drvpath",
{ data_type => "text", is_nullable => 1 },
"outpath",
{ data_type => "text", is_nullable => 1 },
"busy",
{ data_type => "integer", is_nullable => 0 },
"status",
@ -145,8 +138,23 @@ __PACKAGE__->belongs_to(
{ is_deferrable => 0, on_delete => "CASCADE", on_update => "NO ACTION" },
);
=head2 buildstepoutputs
# Created by DBIx::Class::Schema::Loader v0.07033 @ 2013-01-22 13:29:36
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:ItI1OvxHfLTzLVEqfPRjHg
Type: has_many
Related object: L<Hydra::Schema::BuildStepOutputs>
=cut
__PACKAGE__->has_many(
"buildstepoutputs",
"Hydra::Schema::BuildStepOutputs",
{ "foreign.build" => "self.build", "foreign.stepnr" => "self.stepnr" },
undef,
);
# Created by DBIx::Class::Schema::Loader v0.07033 @ 2013-01-30 16:36:03
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:ZiA1nv73Fpp0/DTi4sLfEQ
1;

55
src/lib/Hydra/Schema/Builds.pm
View file

@ -72,11 +72,6 @@ __PACKAGE__->table("Builds");
data_type: 'text'
is_nullable: 0
=head2 outpath
data_type: 'text'
is_nullable: 0
=head2 system
data_type: 'text'
@ -225,8 +220,6 @@ __PACKAGE__->add_columns(
{ data_type => "text", is_nullable => 1 },
"drvpath",
{ data_type => "text", is_nullable => 0 },
"outpath",
{ data_type => "text", is_nullable => 0 },
"system",
{ data_type => "text", is_nullable => 0 },
"longdescription",
@ -321,6 +314,21 @@ __PACKAGE__->has_many(
undef,
);
=head2 buildoutputs
Type: has_many
Related object: L<Hydra::Schema::BuildOutputs>
=cut
__PACKAGE__->has_many(
"buildoutputs",
"Hydra::Schema::BuildOutputs",
{ "foreign.build" => "self.id" },
undef,
);
=head2 buildproducts
Type: has_many
@ -336,6 +344,21 @@ __PACKAGE__->has_many(
undef,
);
=head2 buildstepoutputs
Type: has_many
Related object: L<Hydra::Schema::BuildStepOutputs>
=cut
__PACKAGE__->has_many(
"buildstepoutputs",
"Hydra::Schema::BuildStepOutputs",
{ "foreign.build" => "self.id" },
undef,
);
=head2 buildsteps
Type: has_many
@ -442,8 +465,8 @@ __PACKAGE__->has_many(
);
# Created by DBIx::Class::Schema::Loader v0.07033 @ 2013-01-22 13:34:39
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:wPBFqpUWncuD9xki8Pbnvg
# Created by DBIx::Class::Schema::Loader v0.07033 @ 2013-01-30 16:22:11
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:YBdqPWScG4dtGx+U3dJcwA
__PACKAGE__->has_many(
"dependents",
@ -459,13 +482,13 @@ __PACKAGE__->has_many(
{ "foreign.build" => "self.id" },
);
__PACKAGE__->has_one(
"actualBuildStep",
"Hydra::Schema::BuildSteps",
{ 'foreign.outpath' => 'self.outpath'
, 'foreign.build' => 'self.id'
},
);
#__PACKAGE__->has_one(
# "actualBuildStep",
# "Hydra::Schema::BuildSteps",
# { 'foreign.outpath' => 'self.outpath'
# , 'foreign.build' => 'self.id'
# },
#);
sub makeSource {
my ($name, $query) = @_;

8
src/lib/Hydra/View/NixExprs.pm
View file

@ -21,13 +21,13 @@ sub process {
my $res = "[\n";
foreach my $name (keys %{$c->stash->{nixPkgs}}) {
my $build = $c->stash->{nixPkgs}->{$name}->{build};
$res .= " # $name\n";
foreach my $pkg (@{$c->stash->{nixPkgs}}) {
my $build = $pkg->{build};
$res .= " # $pkg->{name}\n";
$res .= " { type = \"derivation\";\n";
$res .= " name = " . escape ($build->get_column("releasename") or $build->nixname) . ";\n";
$res .= " system = " . (escape $build->system) . ";\n";
$res .= " outPath = " . (escape $build->outpath) . ";\n";
$res .= " outPath = " . (escape $pkg->{outPath}) . ";\n";
$res .= " meta = {\n";
$res .= " description = " . (escape $build->description) . ";\n"
if $build->description;

3
src/lib/Hydra/View/NixPkg.pm
View file

@ -10,9 +10,10 @@ sub process {
my $build = $c->stash->{build};
# FIXME: add multiple output support
my $s = "NIXPKG1 " . $c->stash->{manifestUri}
. " " . $build->nixname . " " . $build->system
. " " . $build->drvpath . " " . $build->outpath
. " " . $build->drvpath . " " . $build->buildoutputs->find({name => "out"})->path
. " " . $c->uri_for('/');
$c->response->body($s);

2
src/root/admin.tt
View file

@ -26,7 +26,7 @@
<td><tt>[% INCLUDE renderFullJobName project = step.build.project.name jobset = step.build.jobset.name job = step.build.job.name %]</tt></td>
<td><tt>[% step.system %]</tt></td>
<td><a href="[% c.uri_for('/build' step.build.id) %]">[% step.build.id %]</a></td>
<td><tt>[% step.outpath.match('-(.*)').0 %]</tt></td>
<td><tt>[% step.drvpath.match('-(.*)').0 %]</tt></td>
<td class='right'>[% INCLUDE renderDuration duration = curTime - step.starttime %] </td>
</tr>
[% END %]

3
src/root/all.tt
View file

@ -6,8 +6,7 @@
[% ELSIF jobset %]for Jobset [% project.name %]:[% jobset.name %]
[% ELSIF project %] for Project <tt>[% project.name %]</tt>[% END %]</h1></div>
<p>Showing builds [% (page - 1) * resultsPerPage + 1 %] - [% (page - 1) * resultsPerPage + builds.size %]
out of [% total %] in order of descending timestamp.</p>
<p>Showing builds [% (page - 1) * resultsPerPage + 1 %] - [% (page - 1) * resultsPerPage + builds.size %] out of [% total %] in order of descending timestamp.</p>
[% INCLUDE renderPager %]
[% INCLUDE renderBuildList hideProjectName=project hideJobsetName=jobset hideJobName=job %]

158
src/root/build.tt
View file

@ -8,57 +8,60 @@
[% jobset = build.jobset %]
[% job = build.job %]
[% BLOCK renderBuildSteps %]
<table class="tablesorter table table-striped table-condensed">
<thead>
<tr><th>Nr</th><th>What</th><th>Duration</th><th>Machine</th><th>Status</th></tr>
</thead>
<tbody>
[% FOREACH step IN build.buildsteps -%]
[% IF ( type == "All" ) || ( type == "Failed" && step.status != 0 ) || ( type == "Running" && step.busy == 1 ) -%]
[% has_log = log_exists(step.drvpath);
log = c.uri_for('/build' build.id 'nixlog' step.stepnr); -%]
<tr [% IF has_log %] class="clickable" onclick="window.location = '[% log %]'" [% END %]>
<td>[% step.stepnr %]</td>
<td>
[% IF step.type == 0 %]
Build of <tt>[% step.outpath %]</tt>
[% ELSE %]
Substitution of <tt>[% step.outpath %]</tt>
[% END %]
</td>
<td>
[% IF step.busy == 0 %]
[% INCLUDE renderDuration duration = step.stoptime - step.starttime %]
[% ELSE %]
[% IF build.finished %]
[% INCLUDE renderDuration duration = build.stoptime - step.starttime %]
[% ELSE %]
[% INCLUDE renderDuration duration = curTime - step.starttime %]
[% END %]
[% END %]
</td>
<td>[% step.machine.split('@').1 %]</td>
<td>
[% IF step.busy == 1 %]
<strong>Building</strong>
[% ELSIF step.status == 0 %]
Succeeded
[% ELSIF step.status == 4 %]
<span class="error">Aborted</span>
[% ELSE %]
<span class="error">Failed: [% HTML.escape(step.errormsg) %]</span>
[% END %]
[% IF has_log %]
(<a href="[% log %]">log</a>, <a href="[% "$log/raw" %]">raw</a>, <a href="[% "$log/tail-reload" %]">tail</a>)
[% END %]
</td>
</tr>
[% END %]
[% END %]
</tbody>
</table>
[% BLOCK renderOutputs %]
[% start=1; FOREACH output IN outputs %]
[% IF !start %],<br/>[% END; start=0; output.path %]
[% END %]
[% END %]
[% BLOCK renderBuildSteps %]
<table class="tablesorter table table-striped table-condensed">
<thead>
<tr><th>Nr</th><th>What</th><th>Duration</th><th>Machine</th><th>Status</th></tr>
</thead>
<tbody>
[% FOREACH step IN build.buildsteps %]
[% IF ( type == "All" ) || ( type == "Failed" && step.status != 0 ) || ( type == "Running" && step.busy == 1 ) %]
[% has_log = log_exists(step.drvpath);
log = c.uri_for('/build' build.id 'nixlog' step.stepnr); %]
<tr [% IF has_log %] class="clickable" onclick="window.location = '[% log %]'" [% END %]>
<td>[% step.stepnr %]</td>
<td>
[% IF step.type == 0 %]
Build of <tt>[% INCLUDE renderOutputs outputs=step.buildstepoutputs %]</tt>
[% ELSE %]
Substitution of <tt>[% INCLUDE renderOutputs outputs=step.buildstepoutputs %]</tt>
[% END %]
</td>
<td>
[% IF step.busy == 0;
INCLUDE renderDuration duration = step.stoptime - step.starttime;
ELSIF build.finished;
INCLUDE renderDuration duration = build.stoptime - step.starttime;
ELSE;
INCLUDE renderDuration duration = curTime - step.starttime;
END %]
</td>
<td>[% step.machine.split('@').1 %]</td>
<td>
[% IF step.busy == 1 %]
<strong>Building</strong>
[% ELSIF step.status == 0 %]
Succeeded
[% ELSIF step.status == 4 %]
<span class="error">Aborted</span>
[% ELSE %]
<span class="error">Failed: [% HTML.escape(step.errormsg) %]</span>
[% END %]
[% IF has_log %]
(<a href="[% log %]">log</a>, <a href="[% "$log/raw" %]">raw</a>, <a href="[% "$log/tail-reload" %]">tail</a>)
[% END %]
</td>
</tr>
[% END %]
[% END %]
</tbody>
</table>
[% END %]
@ -110,15 +113,18 @@
<th>System:</th>
<td><tt>[% build.system %]</tt></td>
</tr>
[% IF build.finished %]
[% IF build.iscachedbuild %]
<tr>
<th>Cached from:</th>
<td>
[% INCLUDE renderFullBuildLink build=cachedBuild %]
</td>
</tr>
[% ELSE %]
<tr>
<th>Duration:</th>
<td>
[% IF build.iscachedbuild %]
(cached[% IF cachedBuild %] from [% INCLUDE renderFullBuildLink build=cachedBuild %][% END %])
[% ELSE %]
[% INCLUDE renderDuration duration = build.stoptime - build.starttime %] finished at [% INCLUDE renderDateTime timestamp = build.stoptime %]
[% END %]
[% INCLUDE renderDuration duration = build.stoptime - build.starttime %]; finished at [% INCLUDE renderDateTime timestamp = build.stoptime %]
</td>
</tr>
[% END %]
@ -217,28 +223,6 @@
<h2>Information</h2>
<table class="layoutTable">
<tr>
<th>Build ID:</th>
<td>[% build.id %]</td>
</tr>
<tr>
<th>Status:</th>
<td>
[% INCLUDE renderStatus build=build %]
</td>
</tr>
<tr>
<th>Project:</th>
<td>[% INCLUDE renderProjectName project=project.name %]</td>
</tr>
<tr>
<th>Jobset:</th>
<td>[% INCLUDE renderJobsetName project=project.name jobset=jobset.name %]</td>
</tr>
<tr>
<th>Job name:</th>
<td>[% INCLUDE renderJobName project=project.name jobset=jobset.name job=job.name %]</td>
</tr>
[% IF build.nixexprinput %]
<tr>
<th>Nix expression:</th>
@ -249,12 +233,6 @@
<th>Nix name:</th>
<td><tt>[% build.nixname %]</tt></td>
</tr>
[% IF build.releasename %]
<tr>
<th>Release name:</th>
<td><tt>[% HTML.escape(build.releasename) %]</tt></td>
</tr>
[% END %]
<tr>
<th>Short description:</th>
<td>[% IF build.description %][% HTML.escape(build.description) %][% ELSE %]<em>(not given)</em>[% END %]</td>
@ -289,22 +267,14 @@
</td>
</tr>
<tr>
<th>Output store path:</th>
<th>Output store paths:</th>
<td>
<tt>[% build.outpath %]</tt>
<tt>[% INCLUDE renderOutputs outputs=build.buildoutputs %]</tt>
[% IF available %]
(<a href="[% c.uri_for('/build' build.id 'deps') %]#runtime">runtime dependencies</a>)
[% END %]
</td>
</tr>
[% IF pathHash %]
<tr>
<th>Output store path hash:</th>
<td>
<tt>[% pathHash %]</tt>
</td>
</tr>
[% END %]
<tr>
<th>Time added:</th>
<td>[% INCLUDE renderDateTime timestamp = build.timestamp %]</td>

1
src/root/channel-contents.tt
View file

@ -64,6 +64,7 @@ install the package simply by clicking on the packages below.</p>
[% ELSE -%]
[% HTML.escape(build.description) -%]
[% END -%]
[% IF pkg.outName != 'out' %] [[% pkg.outName %]][% END %]
</td>
</tr>

6
src/root/common.tt
View file

@ -347,7 +347,7 @@
[% IF bi1.name == bi2.name %]
[% IF bi1.type == bi2.type %]
[% IF bi1.value != bi2.value || bi1.uri != bi2.uri %]
<tr><td><b>[% bi1.name %]</b></td><td><tt>[% INCLUDE renderShortInputValue input=bi1 %] to [% INCLUDE renderShortInputValue input=bi2 %]</tt></td></tr>
<tr><td><b>[% bi1.name %]</b></td><td><tt>[% INCLUDE renderShortInputValue input=bi1 %]</tt> to <tt>[% INCLUDE renderShortInputValue input=bi2 %]</tt></td></tr>
[% ELSIF bi1.uri == bi2.uri && bi1.revision != bi2.revision %]
[% IF bi1.type == "git" %]
<tr><td>
@ -360,7 +360,7 @@
[% END %]
[% ELSIF bi1.dependency.id != bi2.dependency.id || bi1.path != bi2.path %]
<tr><td>
<b>[% bi1.name %]</b></td><td><tt>[% INCLUDE renderShortInputValue input=bi1 %] to [% INCLUDE renderShortInputValue input=bi2 %]</tt>
<b>[% bi1.name %]</b></td><td><tt>[% INCLUDE renderShortInputValue input=bi1 %]</tt> to <tt>[% INCLUDE renderShortInputValue input=bi2 %]</tt>
<br/>
<br/>
[% INCLUDE renderInputDiff build1=bi1.dependency, build2=bi2.dependency, nestedDiff=1, nestLevel=nestLevel+1 %]
@ -395,7 +395,7 @@
<td><tt>[% step.system %]</tt></td>
<td><a href="[% c.uri_for('/build' step.build.id) %]">[% step.build.id %]</a></td>
<td><a href="[% c.uri_for('/build' step.build.id 'nixlog' step.stepnr 'tail-reload') %]">[% step.stepnr %]</a></td>
<td><tt>[% step.outpath.match('-(.*)').0 %]</tt></td>
<td><tt>[% step.drvpath.match('-(.*)').0 %]</tt></td>
<td class='right'>[% INCLUDE renderDuration duration = curTime - step.starttime %] </td>
</tr>
[% END %]

4
src/root/deps.tt
View file

@ -8,7 +8,7 @@
<a name="runtime"></a>
[% IF available %]
<h1>Runtime dependencies for [% build.outpath %]</h1>
<h1>Runtime dependencies</h1>
<ul>
[% FOREACH dep IN runtimedeps -%]
<li>
@ -26,7 +26,7 @@ Path not available anymore!<br />
<a name="buildtime"></a>
[% IF drvAvailable %]
<h1>Build time dependencies for [% build.drvpath %]</h1>
<h1>Build time dependencies</h1>
<ul>
[% FOREACH dep IN buildtimedeps -%]
<li>

4
src/root/jobset.tt
View file

@ -204,7 +204,7 @@
<blockquote>
[% IF activeJobs.size == 0 %]<em>(none)</em>[% END %]
[% FOREACH j IN activeJobs %] [% INCLUDE renderJobName project=project.name jobset=jobset.name job=j %] [% END %]
[% FOREACH j IN activeJobs %][% INCLUDE renderJobName project=project.name jobset=jobset.name job=j %]<br/>[% END %]
</blockquote>
</p>
@ -212,7 +212,7 @@
<blockquote>
[% IF inactiveJobs.size == 0 %]<em>(none)</em>[% END %]
[% FOREACH j IN inactiveJobs %] [% INCLUDE renderJobName project=project.name jobset=jobset.name job=j %] [% END %]
[% FOREACH j IN inactiveJobs %][% INCLUDE renderJobName project=project.name jobset=jobset.name job=j %]<br/>[% END %]
</blockquote>
</p>

2
src/root/log.tt
View file

@ -4,7 +4,7 @@
<div class="page-header"><h1>Build log of [% INCLUDE renderFullJobNameOfBuild %] build <a href="[% c.uri_for('/build' build.id) %]">[% build.id %]</a>[%IF step %] step [% step.stepnr %][% END %]</h1></div>
<p>
This is the build log of path <tt>[% IF step; step.outpath; ELSE; build.outpath; END %]</tt>.
This is the build log of derivation <tt>[% IF step; step.drvpath; ELSE; build.drvpath; END %]</tt>.
[% IF step && step.machine %]
It was built on <tt>[% step.machine %]</tt>.
[% END %]

4
src/root/product-list.tt
View file

@ -16,7 +16,7 @@
[% CASE "nix-build" %]
[% IF build.buildstatus == 6 %]
[% filename = "${build.nixname}.closure.gz" %]
[% filename = build.nixname _ (product.subtype ? "-" _ product.subtype : "") _ ".closure.gz" %]
[% uri = c.uri_for('/build' build.id 'nix' 'closure' filename ) %]
<tr class="product">
<td>
@ -70,7 +70,7 @@
<tr class="product">
<td>
[% filename = "${build.nixname}.closure.gz" %]
[% filename = build.nixname _ (product.subtype ? "-" _ product.subtype : "") _ ".closure.gz" %]
[% uri = c.uri_for('/build' build.id 'nix' 'closure' filename ) %]
<a href="[% uri %]">

94
src/script/hydra-build
View file

@ -1,6 +1,7 @@
#! /var/run/current-system/sw/bin/perl -w
use strict;
use List::MoreUtils qw(all);
use File::Basename;
use File::stat;
use Nix::Store;
@ -58,6 +59,7 @@ sub sendTwitterNotification {
warn "$@\n" if $@;
}
sub statusDescription {
my ($buildstatus) = @_;
@ -72,6 +74,7 @@ sub statusDescription {
return $status;
}
sub sendEmailNotification {
my ($build) = @_;
@ -127,7 +130,7 @@ sub sendEmailNotification {
[ "Maintainer(s):", $build->maintainers ],
[ "System:", $build->system ],
[ "Derivation store path:", $build->drvpath ],
[ "Output store path:", $build->outpath ],
[ "Output store path:", join(", ", map { $_->path } $build->buildoutputs) ],
[ "Time added:", showTime $build->timestamp ],
);
push @lines, (
@ -206,11 +209,21 @@ sub sendEmailNotification {
}
sub addBuildStepOutputs {
my ($step) = @_;
my $drv = derivationFromPath($step->drvpath);
$step->buildstepoutputs->create({ name => $_, path => $drv->{outputs}->{$_} })
foreach keys %{$drv->{outputs}};
}
sub doBuild {
my ($build) = @_;
my %outputs;
$outputs{$_->name} = $_->path foreach $build->buildoutputs->all;
my $drvPath = $build->drvpath;
my $outPath = $build->outpath;
my $maxsilent = $build->maxsilent;
my $timeout = $build->timeout;
@ -223,7 +236,7 @@ sub doBuild {
my $errormsg = undef;
if (!isValidPath($outPath)) {
unless (all { isValidPath($_) } values(%outputs)) {
$isCachedBuild = 0;
# Do the build.
@ -235,12 +248,12 @@ sub doBuild {
# Run Nix to perform the build, and monitor the stderr output
# to get notifications about specific build steps, the
# associated log files, etc.
# Note: `--timeout' was added in Nix 1.0pre27564, June 2011.
my $cmd = "nix-store --realise $drvPath " .
"--timeout $timeout " .
"--max-silent-time $maxsilent --keep-going --fallback " .
"--no-build-output --log-type flat --print-build-trace " .
"--add-root " . gcRootFor $outPath . " 2>&1";
"--add-root " . gcRootFor($outputs{out} // $outputs{(sort keys %outputs)[0]}) . " 2>&1";
print STDERR "$cmd\n";
my $max = $build->buildsteps->find(
{}, {select => {max => 'stepnr + 1'}, as => ['max']});
@ -261,15 +274,15 @@ sub doBuild {
if (/^@\s+build-started\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)$/) {
my $drvPathStep = $1;
txn_do($db, sub {
$build->buildsteps->create(
my $step = $build->buildsteps->create(
{ stepnr => ($buildSteps{$drvPathStep} = $buildStepNr++)
, type => 0 # = build
, drvpath => $drvPathStep
, outpath => $2
, system => $3
, busy => 1
, starttime => time
});
addBuildStepOutputs($step);
});
}
@ -305,46 +318,47 @@ sub doBuild {
# failed previously. This can happen if this is a
# restarted build.
elsif (scalar $build->buildsteps->search({drvpath => $drvPathStep, type => 0, busy => 0, status => 1}) == 0) {
$build->buildsteps->create(
my $step = $build->buildsteps->create(
{ stepnr => ($buildSteps{$drvPathStep} = $buildStepNr++)
, type => 0 # = build
, drvpath => $drvPathStep
, outpath => $2
, busy => 0
, status => 1
, starttime => time
, stoptime => time
, errormsg => $errorMsg
});
addBuildStepOutputs($step);
}
});
}
elsif (/^@\s+substituter-started\s+(\S+)\s+(\S+)$/) {
my $outPath = $1;
my $path = $1;
txn_do($db, sub {
$build->buildsteps->create(
{ stepnr => ($buildSteps{$outPath} = $buildStepNr++)
my $step = $build->buildsteps->create(
{ stepnr => ($buildSteps{$path} = $buildStepNr++)
, type => 1 # = substitution
, outpath => $1
, busy => 1
, starttime => time
});
# "out" is kinda fake (substitutions don't have named outputs).
$step->buildstepoutputs->create({ name => "out", path => $path });
});
}
elsif (/^@\s+substituter-succeeded\s+(\S+)$/) {
my $outPath = $1;
my $path = $1;
txn_do($db, sub {
my $step = $build->buildsteps->find({stepnr => $buildSteps{$outPath}}) or die;
my $step = $build->buildsteps->find({stepnr => $buildSteps{$path}}) or die;
$step->update({busy => 0, status => 0, stoptime => time});
});
}
elsif (/^@\s+substituter-failed\s+(\S+)\s+(\S+)\s+(\S+)$/) {
my $outPath = $1;
my $path = $1;
txn_do($db, sub {
my $step = $build->buildsteps->find({stepnr => $buildSteps{$outPath}}) or die;
my $step = $build->buildsteps->find({stepnr => $buildSteps{$path}}) or die;
$step->update({busy => 0, status => 1, errormsg => $3, stoptime => time});
});
}
@ -371,24 +385,34 @@ sub doBuild {
}
done:
my $size = 0;
my $closuresize = 0;
if (isValidPath($outPath)) {
my ($deriver, $hash, $time, $narSize, $refs) = queryPathInfo($outPath, 0);
$size = $narSize;
my @closure = computeFSClosure(0, 0, $outPath);
foreach my $path (@closure) {
my ($deriver, $hash, $time, $narSize, $refs) = queryPathInfo($path, 0);
$closuresize += $narSize;
}
}
txn_do($db, sub {
my $releaseName = getReleaseName($outPath);
if ($buildStatus == 0) {
$buildStatus = 6 if $buildStatus == 0 && -f "$outPath/nix-support/failed";
my $size = 0;
my $closureSize = 0;
my $releaseName;
my @closure = computeFSClosure(0, 0, values %outputs);
foreach my $path (@closure) {
my ($deriver, $hash, $time, $narSize, $refs) = queryPathInfo($path, 0);
$closureSize += $narSize;
$size += $narSize if grep { $path eq $_ } values(%outputs);
}
foreach my $path (values %outputs) {
$buildStatus = 6 if $buildStatus == 0 && -f "$path/nix-support/failed";
$releaseName //= getReleaseName($path);
}
$build->update(
{ releasename => $releaseName
, size => $size
, closuresize => $closureSize
});
addBuildProducts($db, $build);
}
$build->update(
{ finished => 1
@ -400,15 +424,9 @@ sub doBuild {
, buildstatus => $buildStatus
, starttime => $startTime
, stoptime => $stopTime
, size => $size
, closuresize => $closuresize
, errormsg => $errormsg
, releasename => $releaseName
});
if ($buildStatus == 0 || $buildStatus == 6) {
addBuildProducts($db, $build);
}
});
sendEmailNotification $build;

56
src/script/hydra-update-gc-roots
View file

@ -21,25 +21,27 @@ sub addRoot {
}
my @columns = ( "id", "project", "jobset", "job", "system", "finished", "outpath", "drvpath", "timestamp" );
my @columns = ( "id", "project", "jobset", "job", "system", "finished", "drvpath", "timestamp" );
sub keepBuild {
my ($build) = @_;
print STDERR " keeping ", ($build->finished ? "" : "scheduled "), "build ", $build->id, " (",
$build->get_column('project'), ":", $build->get_column('jobset'), ":", $build->get_column('job'), "; ",
$build->system, "; ",
$build->system, "; ",
strftime("%Y-%m-%d %H:%M:%S", localtime($build->timestamp)), ")\n";
if (isValidPath($build->outpath)) {
addRoot $build->outpath;
} else {
print STDERR " warning: output ", $build->outpath, " has disappeared\n" if $build->finished;
foreach my $out ($build->buildoutputs->all) {
if (isValidPath($out->path)) {
addRoot $out->path;
} else {
print STDERR " warning: output ", $out->path, " has disappeared\n" if $build->finished;
}
}
if (!$build->finished) {
if (isValidPath($build->drvpath)) {
addRoot $build->drvpath;
} else {
print STDERR " warning: derivation ", $build->drvpath, " has disappeared\n";
}
if (isValidPath($build->drvpath)) {
addRoot $build->drvpath;
} else {
print STDERR " warning: derivation ", $build->drvpath, " has disappeared\n";
}
}
}
@ -57,7 +59,8 @@ closedir DIR;
# Keep every build in every release of every project.
print STDERR "*** looking for release members\n";
keepBuild $_ foreach $db->resultset('Builds')->search_literal(
"exists (select 1 from releasemembers where build = me.id)", { order_by => ["project", "jobset", "job", "id"] });
"exists (select 1 from releasemembers where build = me.id)",
{ order_by => ["project", "jobset", "job", "id"], columns => [ @columns ] });
# Keep all builds that have been marked as "keep".
@ -74,8 +77,8 @@ foreach my $project ($db->resultset('Projects')->search({}, { order_by => ["name
foreach my $jobset ($project->jobsets->search({}, { order_by => ["name" ]})) {
my $keepnr = $jobset->keepnr;
# If the jobset has been hidden and disabled for more than one week, than
# don't keep its builds anymore.
# If the jobset has been hidden and disabled for more than one
# week, then don't keep its builds anymore.
if ($jobset->enabled == 0 && ($project->hidden == 1 || $jobset->hidden == 1) && (time() - ($jobset->lastcheckedtime || 0) > (7 * 24 * 3600))) {
print STDERR "*** skipping disabled jobset ", $project->name, ":", $jobset->name, "\n";
next;
@ -86,19 +89,20 @@ foreach my $project ($db->resultset('Projects')->search({}, { order_by => ["name
next;
}
print STDERR "*** looking for the $keepnr most recent successful builds of each job in jobset ",
$project->name, ":", $jobset->name, "\n";
# FIXME: base this on jobset evals?
print STDERR "*** looking for the $keepnr most recent successful builds of each job in jobset ",
$project->name, ":", $jobset->name, "\n";
keepBuild $_ foreach $jobset->builds->search(
{ 'me.id' => { 'in' => \
[ "select b2.id from Builds b2 join " .
" (select distinct job, system, coalesce( " .
keepBuild $_ foreach $jobset->builds->search(
{ 'me.id' => { 'in' => \
[ "select b2.id from Builds b2 join " .
" (select distinct job, system, coalesce( " .
" (select id from builds where project = b.project and jobset = b.jobset and job = b.job and system = b.system and finished = 1 and buildStatus = 0 order by id desc offset ? limit 1)" .
" , 0) as nth from builds b where project = ? and jobset = ? and isCurrent = 1) x " .
" on b2.project = ? and b2.jobset = ? and b2.job = x.job and b2.system = x.system and (id >= x.nth) where finished = 1 and buildStatus = 0"
, [ '', $keepnr - 1 ], [ '', $project->name ], [ '', $jobset->name ], [ '', $project->name ], [ '', $jobset->name ] ] }
},
{ order_by => ["job", "system", "id"], columns => [ @columns ] });
" on b2.project = ? and b2.jobset = ? and b2.job = x.job and b2.system = x.system and (id >= x.nth) where finished = 1 and buildStatus = 0"
, [ '', $keepnr - 1 ], [ '', $project->name ], [ '', $jobset->name ], [ '', $project->name ], [ '', $jobset->name ] ] }
},
{ order_by => ["job", "system", "id"], columns => [ @columns ] });
}
# Go over all views in this project.
@ -135,10 +139,10 @@ foreach my $link (@roots) {
my $path = "/nix/store/$link";
if (!defined $roots{$path}) {
print STDERR "removing root $path\n";
$rootsDeleted++;
$rootsDeleted++;
unlink "$gcRootsDir/$link" or warn "cannot remove $gcRootsDir/$link";
} else {
$rootsKept++;
$rootsKept++;
}
}

41
src/sql/hydra.sql
View file

@ -133,7 +133,6 @@ create table Builds (
nixName text, -- name attribute of the derivation
description text, -- meta.description
drvPath text not null,
outPath text not null,
system text not null,
longDescription text, -- meta.longDescription
@ -174,6 +173,7 @@ create table Builds (
-- 3 = other failure (see errorMsg)
-- 4 = build cancelled (removed from queue; never built)
-- 5 = build not done because a dependency failed previously (obsolete)
-- 6 = failure with output
buildStatus integer,
errorMsg text, -- error message in case of a Nix failure
@ -191,6 +191,15 @@ create table Builds (
);
create table BuildOutputs (
build integer not null,
name text not null,
path text not null,
primary key (build, name),
foreign key (build) references Builds(id) on delete cascade
);
create table BuildSteps (
build integer not null,
stepnr integer not null,
@ -198,7 +207,6 @@ create table BuildSteps (
type integer not null, -- 0 = build, 1 = substitution
drvPath text,
outPath text,
busy integer not null,
@ -217,6 +225,17 @@ create table BuildSteps (
);
create table BuildStepOutputs (
build integer not null,
stepnr integer not null,
name text not null,
path text not null,
primary key (build, stepnr, name),
foreign key (build) references Builds(id) on delete cascade,
foreign key (build, stepnr) references BuildSteps(build, stepnr) on delete cascade
);
-- Inputs of builds.
create table BuildInputs (
#ifdef POSTGRESQL
@ -494,13 +513,13 @@ create table NewsItems (
create table BuildMachines (
hostname text primary key NOT NULL,
username text DEFAULT '' NOT NULL,
ssh_key text DEFAULT '' NOT NULL,
options text DEFAULT '' NOT NULL,
maxconcurrent integer DEFAULT 2 NOT NULL,
speedfactor integer DEFAULT 1 NOT NULL,
enabled integer DEFAULT 0 NOT NULL
hostname text primary key not null,
username text default '' not null,
ssh_key text default '' not null,
options text default '' not null,
maxconcurrent integer default 2 not null,
speedfactor integer default 1 not null,
enabled integer default 0 not null
);
@ -518,11 +537,9 @@ create index IndexBuildInputsOnBuild on BuildInputs(build);
create index IndexBuildInputsOnDependency on BuildInputs(dependency);
create index IndexBuildProducstOnBuildAndType on BuildProducts(build, type);
create index IndexBuildProductsOnBuild on BuildProducts(build);
create index IndexBuildStepsOnBuild on BuildSteps(build);
create index IndexBuildStepsOnBusy on BuildSteps(busy);
create index IndexBuildStepsOnDrvpathTypeBusyStatus on BuildSteps(drvpath, type, busy, status);
create index IndexBuildStepsOnOutpath on BuildSteps(outpath);
create index IndexBuildStepsOnOutpathBuild on BuildSteps (outpath, build);
create index IndexBuildStepOutputsOnPath on BuildStepOutputs(path);
create index IndexBuildsOnFinished on Builds(finished);
create index IndexBuildsOnFinishedBusy on Builds(finished, busy);
create index IndexBuildsOnIsCurrent on Builds(isCurrent);