diff --git a/src/c/hydra-eval-jobs.cc b/src/c/hydra-eval-jobs.cc index 76d3d2da..8c003a84 100644 --- a/src/c/hydra-eval-jobs.cc +++ b/src/c/hydra-eval-jobs.cc @@ -123,11 +123,12 @@ static void findJobsWrapped(EvalState & state, XMLWriter & doc, XMLAttrs xmlAttrs; Path drvPath; + DrvInfo::Outputs outputs = drv.queryOutputs(state); + xmlAttrs["jobName"] = attrPath; xmlAttrs["nixName"] = drv.name; xmlAttrs["system"] = drv.system; xmlAttrs["drvPath"] = drvPath = drv.queryDrvPath(state); - xmlAttrs["outPath"] = drv.queryOutPath(state); MetaInfo meta = drv.queryMetaInfo(state); xmlAttrs["description"] = queryMetaFieldString(meta, "description"); xmlAttrs["longDescription"] = queryMetaFieldString(meta, "longDescription"); @@ -164,6 +165,14 @@ static void findJobsWrapped(EvalState & state, XMLWriter & doc, } XMLOpenElement _(doc, "job", xmlAttrs); + + foreach (DrvInfo::Outputs::iterator, j, outputs) { + XMLAttrs attrs2; + attrs2["name"] = j->first; + attrs2["path"] = j->second; + doc.writeEmptyElement("output", attrs2); + } + showArgsUsed(doc, argsUsed); } diff --git a/src/lib/Hydra/Base/Controller/ListBuilds.pm b/src/lib/Hydra/Base/Controller/ListBuilds.pm index 226e1d2e..3773ed11 100644 --- a/src/lib/Hydra/Base/Controller/ListBuilds.pm +++ b/src/lib/Hydra/Base/Controller/ListBuilds.pm @@ -78,7 +78,9 @@ sub nix : Chained('get_builds') PathPart('channel') CaptureArgs(1) { $c->stash->{channelName} = $c->stash->{channelBaseName} . "-latest"; $c->stash->{channelBuilds} = $c->stash->{latestSucceeded} ->search_literal("exists (select 1 from buildproducts where build = me.id and type = 'nix-build')") - ->search({}, { columns => [@buildListColumns, 'drvpath', 'outpath', 'description', 'homepage'] }); + ->search({}, { join => ["buildoutputs"] + , columns => [@buildListColumns, 'drvpath', 'description', 'homepage'] + , '+select' => ['buildoutputs.path', 'buildoutputs.name'], '+as' => ['outpath', 'outname'] }); } else { notFound($c, "Unknown channel `$channelName'."); diff --git a/src/lib/Hydra/Base/Controller/NixChannel.pm b/src/lib/Hydra/Base/Controller/NixChannel.pm index 0f100df6..3a7e19c4 100644 --- a/src/lib/Hydra/Base/Controller/NixChannel.pm +++ b/src/lib/Hydra/Base/Controller/NixChannel.pm @@ -3,6 +3,7 @@ package Hydra::Base::Controller::NixChannel; use strict; use warnings; use base 'Catalyst::Controller'; +use List::MoreUtils qw(all); use Nix::Store; use Hydra::Helper::Nix; use Hydra::Helper::CatalystUtils; @@ -12,26 +13,23 @@ sub getChannelData { my ($c, $checkValidity) = @_; my @storePaths = (); + $c->stash->{nixPkgs} = []; foreach my $build ($c->stash->{channelBuilds}->all) { - next if $checkValidity && !isValidPath($build->outpath); - #if (isValidPath($build->drvpath)) { - # # Adding `drvpath' implies adding `outpath' because of the - # # `--include-outputs' flag passed to `nix-store'. - # push @storePaths, $build->drvpath; - #} else { - # push @storePaths, $build->outpath; - #} - push @storePaths, $build->outpath; - my $pkgName = $build->nixname . "-" . $build->system . "-" . $build->id; - $c->stash->{nixPkgs}->{"${pkgName}.nixpkg"} = {build => $build, name => $pkgName}; + my $outPath = $build->get_column("outpath"); + my $outName = $build->get_column("outname"); + next if $checkValidity && !isValidPath($outPath); + push @storePaths, $outPath; + my $pkgName = $build->nixname . "-" . $build->system . "-" . $build->id . ($outName ne "out" ? "-" . $outName : ""); + push @{$c->stash->{nixPkgs}}, { build => $build, name => $pkgName, outPath => $outPath, outName => $outName }; # Put the system type in the manifest (for top-level paths) as # a hint to the binary patch generator. (It shouldn't try to # generate patches between builds for different systems.) It # would be nice if Nix stored this info for every path but it # doesn't. - $c->stash->{systemForPath}->{$build->outpath} = $build->system; + $c->stash->{systemForPath}->{$outPath} = $build->system; }; + print STDERR @storePaths, "\n"; $c->stash->{storePaths} = [@storePaths]; } @@ -42,6 +40,8 @@ sub closure : Chained('nix') PathPart { getChannelData($c, 1); + # FIXME: get the closure of the selected path only. + # !!! quick hack; this is to make HEAD requests return the right # MIME type. This is set in the view as well, but the view isn't # called for HEAD requests. There should be a cleaner solution... @@ -62,11 +62,14 @@ sub pkg : Chained('nix') PathPart Args(1) { if (!$c->stash->{build}) { $pkgName =~ /-(\d+)\.nixpkg$/ or notFound($c, "Bad package name."); + # FIXME: need to handle multiple outputs: channelBuilds is + # joined with the build outputs, so find() can return multiple + # results. $c->stash->{build} = $c->stash->{channelBuilds}->find({ id => $1 }) || notFound($c, "No such package in this channel."); } - if (!isValidPath($c->stash->{build}->outpath)) { + unless (all { isValidPath($_->path) } $c->stash->{build}->buildoutputs->all) { $c->response->status(410); # "Gone" error($c, "Build " . $c->stash->{build}->id . " is no longer available."); } @@ -115,7 +118,7 @@ sub channel_contents : Chained('nix') PathPart('') Args(0) { # channel. getChannelData($c, 0); $c->stash->{template} = 'channel-contents.tt'; - $c->stash->{nixPkgs} = [sortPkgs (values %{$c->stash->{nixPkgs}})]; + $c->stash->{nixPkgs} = [sortPkgs @{$c->stash->{nixPkgs}}]; } diff --git a/src/lib/Hydra/Controller/Build.pm b/src/lib/Hydra/Controller/Build.pm index 4786409c..2690e62c 100644 --- a/src/lib/Hydra/Controller/Build.pm +++ b/src/lib/Hydra/Controller/Build.pm @@ -10,6 +10,7 @@ use File::stat; use File::Slurp; use Data::Dump qw(dump); use Nix::Store; +use List::MoreUtils qw(all); sub build : Chained('/') PathPart CaptureArgs(1) { @@ -32,28 +33,44 @@ sub build : Chained('/') PathPart CaptureArgs(1) { } +sub findBuildStepByOutPath { + my ($self, $c, $path, $status) = @_; + return $c->model('DB::BuildSteps')->search( + { path => $path, busy => 0, status => $status }, + { join => ["buildstepoutputs"], order_by => ["stopTime"], limit => 1 })->single; +} + + +sub findBuildStepByDrvPath { + my ($self, $c, $drvPath, $status) = @_; + return $c->model('DB::BuildSteps')->search( + { drvpath => $drvPath, busy => 0, status => $status }, + { order_by => ["stopTime"], limit => 1 })->single; +} + + sub view_build : Chained('build') PathPart('') Args(0) { my ($self, $c) = @_; my $build = $c->stash->{build}; $c->stash->{template} = 'build.tt'; - $c->stash->{available} = isValidPath $build->outpath; + $c->stash->{available} = all { isValidPath($_->path) } $build->buildoutputs->all; $c->stash->{drvAvailable} = isValidPath $build->drvpath; $c->stash->{flashMsg} = $c->flash->{buildMsg}; - $c->stash->{pathHash} = $c->stash->{available} ? queryPathHash($build->outpath) : undef; - if (!$build->finished && $build->busy) { $c->stash->{logtext} = read_file($build->logfile, err_mode => 'quiet') // ""; } if ($build->finished && $build->iscachedbuild) { - (my $cachedBuildStep) = $c->model('DB::BuildSteps')->search({ outpath => $build->outpath }, {}); + my $path = ($build->buildoutputs)[0]->path or die; + my $cachedBuildStep = findBuildStepByOutPath($self, $c, $path, + $build->buildstatus == 0 || $build->buildstatus == 6 ? 0 : 1); $c->stash->{cachedBuild} = $cachedBuildStep->build if defined $cachedBuildStep; } - if ($build->finished) { + if ($build->finished && 0) { $c->stash->{prevBuilds} = [$c->model('DB::Builds')->search( { project => $c->stash->{project}->name , jobset => $c->stash->{build}->jobset->name @@ -279,35 +296,31 @@ sub deps : Chained('build') PathPart('deps') { my ($self, $c) = @_; my $build = $c->stash->{build}; - $c->stash->{available} = isValidPath $build->outpath; - $c->stash->{drvAvailable} = isValidPath $build->drvpath; + my $drvPath = $build->drvpath; + my @outPaths = map { $_->path } $build->buildoutputs->all; - my $drvpath = $build->drvpath; - my $outpath = $build->outpath; + $c->stash->{available} = all { isValidPath($_) } @outPaths; + $c->stash->{drvAvailable} = isValidPath $drvPath; - my @buildtimepaths = (); + my @buildtimepaths = $c->stash->{drvAvailable} ? computeFSClosure(0, 0, $drvPath) : (); my @buildtimedeps = (); - @buildtimepaths = split '\n', `nix-store --query --requisites --include-outputs $drvpath` if isValidPath($build->drvpath); - my @runtimepaths = (); + my @runtimepaths = $c->stash->{available} ? computeFSClosure(0, 0, @outPaths) : (); my @runtimedeps = (); - @runtimepaths = split '\n', `nix-store --query --requisites --include-outputs $outpath` if isValidPath($build->outpath); foreach my $p (@buildtimepaths) { - my $buildStep; - ($buildStep) = $c->model('DB::BuildSteps')->search({ outpath => $p }, {}) ; - my %dep = ( buildstep => $buildStep, path => $p ) ; + next unless $p =~ /\.drv$/; + my ($buildStep) = findBuildStepByDrvPath($self, $c, $p, 0); + my %dep = ( buildstep => $buildStep, path => $p ); push(@buildtimedeps, \%dep); } foreach my $p (@runtimepaths) { - my $buildStep; - ($buildStep) = $c->model('DB::BuildSteps')->search({ outpath => $p }, {}) ; - my %dep = ( buildstep => $buildStep, path => $p ) ; + my ($buildStep) = findBuildStepByOutPath($self, $c, $p, 0); + my %dep = ( buildstep => $buildStep, path => $p ); push(@runtimedeps, \%dep); } - $c->stash->{buildtimedeps} = \@buildtimedeps; $c->stash->{runtimedeps} = \@runtimedeps; @@ -321,12 +334,17 @@ sub nix : Chained('build') PathPart('nix') CaptureArgs(0) { my $build = $c->stash->{build}; notFound($c, "Build cannot be downloaded as a closure or Nix package.") - if !$build->buildproducts->find({type => "nix-build"}); + if $build->buildproducts->search({type => "nix-build"})->count == 0; - notFound($c, "Path " . $build->outpath . " is no longer available.") - unless isValidPath($build->outpath); + foreach my $out ($build->buildoutputs) { + notFound($c, "Path " . $out->path . " is no longer available.") + unless isValidPath($out->path); + } - $c->stash->{channelBuilds} = $c->model('DB::Builds')->search({id => $build->id}); + $c->stash->{channelBuilds} = $c->model('DB::Builds')->search( + { id => $build->id }, + { join => ["buildoutputs"] + , '+select' => ['buildoutputs.path', 'buildoutputs.name'], '+as' => ['outpath', 'outname'] }); } @@ -377,22 +395,23 @@ sub cancel : Chained('build') PathPart Args(0) { sub keep : Chained('build') PathPart Args(1) { - my ($self, $c, $newStatus) = @_; + my ($self, $c, $x) = @_; + my $keep = $x eq "1" ? 1 : 0; my $build = $c->stash->{build}; requireProjectOwner($c, $build->project); - die unless $newStatus == 0 || $newStatus == 1; - - registerRoot $build->outpath if $newStatus == 1; + if ($keep) { + registerRoot $_->path foreach $build->buildoutputs; + } txn_do($c->model('DB')->schema, sub { - $build->update({keep => int $newStatus}); + $build->update({keep => $keep}); }); $c->flash->{buildMsg} = - $newStatus == 0 ? "Build will not be kept." : "Build will be kept."; + $keep ? "Build will be kept." : "Build will not be kept."; $c->res->redirect($c->uri_for($self->action_for("view_build"), $c->req->captures)); } @@ -414,9 +433,10 @@ sub add_to_release : Chained('build') PathPart('add-to-release') Args(0) { error($c, "This build is already a part of release `$releaseName'.") if $release->releasemembers->find({build => $build->id}); - registerRoot $build->outpath; - - error($c, "This build is no longer available.") unless isValidPath $build->outpath; + foreach my $output ($build->buildoutputs) { + error($c, "This build is no longer available.") unless isValidPath $output->path; + registerRoot $output->path; + } $release->releasemembers->create({build => $build->id, description => $build->description}); @@ -509,7 +529,8 @@ sub get_info : Chained('build') PathPart('api/get-info') Args(0) { # !!! strip the json prefix $c->stash->{jsonBuildId} = $build->id; $c->stash->{jsonDrvPath} = $build->drvpath; - $c->stash->{jsonOutPath} = $build->outpath; + my $out = $build->buildoutputs->find({name => "out"}); + $c->stash->{jsonOutPath} = $out->path if defined $out; $c->forward('View::JSON'); } diff --git a/src/lib/Hydra/Controller/JobsetEval.pm b/src/lib/Hydra/Controller/JobsetEval.pm index b0807dc1..663c4b12 100644 --- a/src/lib/Hydra/Controller/JobsetEval.pm +++ b/src/lib/Hydra/Controller/JobsetEval.pm @@ -141,7 +141,10 @@ sub nix : Chained('eval') PathPart('channel') CaptureArgs(0) { $c->stash->{channelName} = $c->stash->{project}->name . "-" . $c->stash->{jobset}->name . "-latest"; $c->stash->{channelBuilds} = $c->stash->{eval}->builds ->search_literal("exists (select 1 from buildproducts where build = build.id and type = 'nix-build')") - ->search({ finished => 1, buildstatus => 0 }, { columns => [@buildListColumns, 'drvpath', 'outpath', 'description', 'homepage'] }); + ->search({ finished => 1, buildstatus => 0 }, + { columns => [@buildListColumns, 'drvpath', 'description', 'homepage'] + , join => ["buildoutputs"] + , '+select' => ['buildoutputs.path', 'buildoutputs.name'], '+as' => ['outpath', 'outname'] }); } diff --git a/src/lib/Hydra/Helper/AddBuilds.pm b/src/lib/Hydra/Helper/AddBuilds.pm index 421772dd..b274f21c 100644 --- a/src/lib/Hydra/Helper/AddBuilds.pm +++ b/src/lib/Hydra/Helper/AddBuilds.pm @@ -13,6 +13,7 @@ use File::Basename; use File::stat; use File::Path; use File::Temp; +use File::Slurp; our @ISA = qw(Exporter); our @EXPORT = qw( @@ -43,14 +44,9 @@ sub getStorePathHash { sub getReleaseName { my ($outPath) = @_; - - my $releaseName; - if (-e "$outPath/nix-support/hydra-release-name") { - open FILE, "$outPath/nix-support/hydra-release-name" or die; - $releaseName = ; - chomp $releaseName; - close FILE; - } + return undef unless -f "$outPath/nix-support/hydra-release-name"; + my $releaseName = read_file("$outPath/nix-support/hydra-release-name"); + chomp $releaseName; return $releaseName; } @@ -242,7 +238,7 @@ sub fetchInputBuild { { order_by => "me.id DESC", rows => 1 , where => \ attrsToSQL($attrs, "me.id") }); - if (!defined $prevBuild || !isValidPath($prevBuild->outpath)) { + if (!defined $prevBuild || !isValidPath(getMainOutput($prevBuild)->path)) { print STDERR "input `", $name, "': no previous build available\n"; return undef; } @@ -256,7 +252,7 @@ sub fetchInputBuild { my $version = $2 if $relName =~ /^($pkgNameRE)-($versionRE)$/; return - { storePath => $prevBuild->outpath + { storePath => getMainOutput($prevBuild)->path , id => $prevBuild->id , version => $version }; @@ -275,7 +271,7 @@ sub fetchInputSystemBuild { my @validBuilds = (); foreach my $build (@latestBuilds) { - push(@validBuilds, $build) if isValidPath($build->outpath); + push(@validBuilds, $build) if !isValidPath(getMainOutput($build)->path); } if (scalar(@validBuilds) == 0) { @@ -293,7 +289,7 @@ sub fetchInputSystemBuild { my $version = $2 if $relName =~ /^($pkgNameRE)-($versionRE)$/; my $input = - { storePath => $prevBuild->outpath + { storePath => getMainOutput($prevBuild)->path , id => $prevBuild->id , version => $version , system => $prevBuild->system @@ -666,6 +662,7 @@ sub buildInputToString { return $result; } + sub inputsToArgs { my ($inputInfo, $exprType) = @_; my @res = (); @@ -742,8 +739,8 @@ sub evalJobs { my $jobs = XMLin( $jobsXml, - ForceArray => ['error', 'job', 'arg'], - KeyAttr => [], + ForceArray => ['error', 'job', 'arg', 'output'], + KeyAttr => { output => "+name" }, SuppressEmpty => '') or die "cannot parse XML output"; @@ -751,7 +748,7 @@ sub evalJobs { foreach my $job (@{$jobs->{job}}) { my $validJob = 1; foreach my $arg (@{$job->{arg}}) { - my $input = $inputInfo->{$arg->{name}}->[$arg->{altnr}] ; + my $input = $inputInfo->{$arg->{name}}->[$arg->{altnr}]; if ($input->{type} eq "sysbuild" && $input->{system} ne $job->{system}) { $validJob = 0; } @@ -769,60 +766,68 @@ sub evalJobs { sub addBuildProducts { my ($db, $build) = @_; - my $outPath = $build->outpath; my $productnr = 1; + my $explicitProducts = 0; - if (-e "$outPath/nix-support/hydra-build-products") { - open LIST, "$outPath/nix-support/hydra-build-products" or die; - while () { - /^([\w\-]+)\s+([\w\-]+)\s+(\S+)(\s+(\S+))?$/ or next; - my $type = $1; - my $subtype = $2 eq "none" ? "" : $2; - my $path = $3; - my $defaultPath = $5; - next unless -e $path; + foreach my $output ($build->buildoutputs->all) { + my $outPath = $output->path; + if (-e "$outPath/nix-support/hydra-build-products") { + $explicitProducts = 1; - my $fileSize, my $sha1, my $sha256; + open LIST, "$outPath/nix-support/hydra-build-products" or die; + while () { + /^([\w\-]+)\s+([\w\-]+)\s+(\S+)(\s+(\S+))?$/ or next; + my $type = $1; + my $subtype = $2 eq "none" ? "" : $2; + my $path = $3; + my $defaultPath = $5; + next unless -e $path; - # !!! validate $path, $defaultPath + my $fileSize, my $sha1, my $sha256; - if (-f $path) { - my $st = stat($path) or die "cannot stat $path: $!"; - $fileSize = $st->size; + # !!! validate $path, $defaultPath - $sha1 = `nix-hash --flat --type sha1 $path` - or die "cannot hash $path: $?";; - chomp $sha1; + if (-f $path) { + my $st = stat($path) or die "cannot stat $path: $!"; + $fileSize = $st->size; - $sha256 = `nix-hash --flat --type sha256 $path` - or die "cannot hash $path: $?";; - chomp $sha256; + $sha1 = `nix-hash --flat --type sha1 $path` + or die "cannot hash $path: $?";; + chomp $sha1; + + $sha256 = `nix-hash --flat --type sha256 $path` + or die "cannot hash $path: $?";; + chomp $sha256; + } + + my $name = $path eq $outPath ? "" : basename $path; + + $db->resultset('BuildProducts')->create( + { build => $build->id + , productnr => $productnr++ + , type => $type + , subtype => $subtype + , path => $path + , filesize => $fileSize + , sha1hash => $sha1 + , sha256hash => $sha256 + , name => $name + , defaultpath => $defaultPath + }); } - - my $name = $path eq $outPath ? "" : basename $path; - - $db->resultset('BuildProducts')->create( - { build => $build->id - , productnr => $productnr++ - , type => $type - , subtype => $subtype - , path => $path - , filesize => $fileSize - , sha1hash => $sha1 - , sha256hash => $sha256 - , name => $name - , defaultpath => $defaultPath - }); + close LIST; } - close LIST; } - else { + return if $explicitProducts; + + foreach my $output ($build->buildoutputs->all) { + my $outPath = $output->path; $db->resultset('BuildProducts')->create( { build => $build->id , productnr => $productnr++ , type => "nix-build" - , subtype => "" + , subtype => $output->name eq "out" ? "" : $output->name , path => $outPath , name => $build->nixname }); @@ -846,9 +851,17 @@ sub getPrevJobsetEval { sub checkBuild { my ($db, $project, $jobset, $inputInfo, $nixExprInput, $buildInfo, $buildIds, $prevEval, $jobOutPathMap) = @_; - my $jobName = $buildInfo->{jobName}; - my $drvPath = $buildInfo->{drvPath}; - my $outPath = $buildInfo->{outPath}; + my @outputNames = sort keys %{$buildInfo->{output}}; + die unless scalar @outputNames; + + # In various checks we can use an arbitrary output (the first) + # rather than all outputs, since if one output is the same, the + # others will be as well. + my $firstOutputName = $outputNames[0]; + my $firstOutputPath = $buildInfo->{output}->{$firstOutputName}->{path}; + + my $jobName = $buildInfo->{jobName} or die; + my $drvPath = $buildInfo->{drvPath} or die; my $priority = 100; $priority = int($buildInfo->{schedulingPriority}) @@ -873,18 +886,21 @@ sub checkBuild { # will be performed (though the last one will probably use the # cached result from the first). This ensures that the builds # with the highest ID will always be the ones that we want in - # the channels. - # !!! Checking $outPath doesn't take meta-attributes into - # account. For instance, do we want a new build to be - # scheduled if the meta.maintainers field is changed? + # the channels. FIXME: Checking the output paths doesn't take + # meta-attributes into account. For instance, do we want a + # new build to be scheduled if the meta.maintainers field is + # changed? if (defined $prevEval) { + # Only check one output: if it's the same, the other will be as well. + my $firstOutput = $outputNames[0]; my ($prevBuild) = $prevEval->builds->search( # The "project" and "jobset" constraints are # semantically unnecessary (because they're implied by # the eval), but they give a factor 1000 speedup on # the Nixpkgs jobset with PostgreSQL. - { project => $project->name, jobset => $jobset->name, job => $job->name, outPath => $outPath }, - { rows => 1, columns => ['id'] }); + { project => $project->name, jobset => $jobset->name, job => $job->name, + name => $firstOutputName, path => $firstOutputPath }, + { rows => 1, columns => ['id'], join => ['buildoutputs'] }); if (defined $prevBuild) { print STDERR " already scheduled/built as build ", $prevBuild->id, "\n"; $buildIds->{$prevBuild->id} = 0; @@ -894,7 +910,7 @@ sub checkBuild { # Prevent multiple builds with the same (job, outPath) from # being added. - my $prev = $$jobOutPathMap{$job->name . "\t" . $outPath}; + my $prev = $$jobOutPathMap{$job->name . "\t" . $firstOutputPath}; if (defined $prev) { print STDERR " already scheduled as build ", $prev, "\n"; return; @@ -902,22 +918,41 @@ sub checkBuild { my $time = time(); - # Nope, so add it. + # Are the outputs already in the Nix store? Then add a cached + # build. my %extraFlags; - if (isValidPath($outPath)) { + my $allValid = 1; + my $buildStatus; + my $releaseName; + foreach my $name (@outputNames) { + my $path = $buildInfo->{output}->{$name}->{path}; + if (isValidPath($path)) { + if (-f "$path/nix-support/failed") { + $buildStatus = 6; + } else { + $buildStatus //= 0; + } + $releaseName //= getReleaseName($path); + } else { + $allValid = 0; + last; + } + } + + if ($allValid) { %extraFlags = ( finished => 1 , iscachedbuild => 1 - , buildstatus => -f "$outPath/nix-support/failed" ? 6 : 0 + , buildstatus => $buildStatus , starttime => $time , stoptime => $time - , errormsg => "" - , releasename => getReleaseName($outPath) + , releasename => $releaseName ); } else { %extraFlags = ( finished => 0 ); } + # Add the build to the database. $build = $job->builds->create( { timestamp => $time , description => $buildInfo->{description} @@ -929,7 +964,6 @@ sub checkBuild { , timeout => $buildInfo->{timeout} , nixname => $buildInfo->{nixName} , drvpath => $drvPath - , outpath => $outPath , system => $buildInfo->{system} , nixexprinput => $jobset->nixexprinput , nixexprpath => $jobset->nixexprpath @@ -939,8 +973,11 @@ sub checkBuild { , %extraFlags }); + $build->buildoutputs->create({ name => $_, path => $buildInfo->{output}->{$_}->{path} }) + foreach @outputNames; + $buildIds->{$build->id} = 1; - $$jobOutPathMap{$job->name . "\t" . $outPath} = $build->id; + $$jobOutPathMap{$job->name . "\t" . $firstOutputPath} = $build->id; if ($build->iscachedbuild) { print STDERR " marked as cached build ", $build->id, "\n"; @@ -988,15 +1025,11 @@ sub restartBuild { my ($db, $build) = @_; txn_do($db, sub { - my $drvpath = $build->drvpath; - my $outpath = $build->outpath; + my @paths; + push @paths, $build->drvpath; + push @paths, $_->drvpath foreach $build->buildsteps; - my $paths = ""; - foreach my $bs ($build->buildsteps) { - $paths = $paths . " " . $bs->outpath; - } - - my $r = `nix-store --clear-failed-paths $paths $outpath`; + my $r = `nix-store --clear-failed-paths @paths`; $build->update( { finished => 0 diff --git a/src/lib/Hydra/Helper/Nix.pm b/src/lib/Hydra/Helper/Nix.pm index 4b608966..87a5c797 100644 --- a/src/lib/Hydra/Helper/Nix.pm +++ b/src/lib/Hydra/Helper/Nix.pm @@ -14,7 +14,8 @@ our @EXPORT = qw( getPrimaryBuildsForView getPrimaryBuildTotal getViewResult getLatestSuccessfulViewResult - jobsetOverview removeAsciiEscapes getDrvLogPath logContents); + jobsetOverview removeAsciiEscapes getDrvLogPath logContents + getMainOutput); sub getHydraHome { @@ -278,4 +279,12 @@ sub removeAsciiEscapes { } +sub getMainOutput { + my ($build) = @_; + return + $build->buildoutputs->find({name => "out"}) // + $build->buildoutputs->find({}, {limit => 1, order_by => ["name"]}); +} + + 1; diff --git a/src/lib/Hydra/Schema/BuildSteps.pm b/src/lib/Hydra/Schema/BuildSteps.pm index c4ac04ef..fb2249bf 100644 --- a/src/lib/Hydra/Schema/BuildSteps.pm +++ b/src/lib/Hydra/Schema/BuildSteps.pm @@ -44,11 +44,6 @@ __PACKAGE__->table("BuildSteps"); data_type: 'text' is_nullable: 1 -=head2 outpath - - data_type: 'text' - is_nullable: 1 - =head2 busy data_type: 'integer' @@ -96,8 +91,6 @@ __PACKAGE__->add_columns( { data_type => "integer", is_nullable => 0 }, "drvpath", { data_type => "text", is_nullable => 1 }, - "outpath", - { data_type => "text", is_nullable => 1 }, "busy", { data_type => "integer", is_nullable => 0 }, "status", @@ -145,8 +138,23 @@ __PACKAGE__->belongs_to( { is_deferrable => 0, on_delete => "CASCADE", on_update => "NO ACTION" }, ); +=head2 buildstepoutputs -# Created by DBIx::Class::Schema::Loader v0.07033 @ 2013-01-22 13:29:36 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:ItI1OvxHfLTzLVEqfPRjHg +Type: has_many + +Related object: L + +=cut + +__PACKAGE__->has_many( + "buildstepoutputs", + "Hydra::Schema::BuildStepOutputs", + { "foreign.build" => "self.build", "foreign.stepnr" => "self.stepnr" }, + undef, +); + + +# Created by DBIx::Class::Schema::Loader v0.07033 @ 2013-01-30 16:36:03 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:ZiA1nv73Fpp0/DTi4sLfEQ 1; diff --git a/src/lib/Hydra/Schema/Builds.pm b/src/lib/Hydra/Schema/Builds.pm index 3cd02c46..9b0905a7 100644 --- a/src/lib/Hydra/Schema/Builds.pm +++ b/src/lib/Hydra/Schema/Builds.pm @@ -72,11 +72,6 @@ __PACKAGE__->table("Builds"); data_type: 'text' is_nullable: 0 -=head2 outpath - - data_type: 'text' - is_nullable: 0 - =head2 system data_type: 'text' @@ -225,8 +220,6 @@ __PACKAGE__->add_columns( { data_type => "text", is_nullable => 1 }, "drvpath", { data_type => "text", is_nullable => 0 }, - "outpath", - { data_type => "text", is_nullable => 0 }, "system", { data_type => "text", is_nullable => 0 }, "longdescription", @@ -321,6 +314,21 @@ __PACKAGE__->has_many( undef, ); +=head2 buildoutputs + +Type: has_many + +Related object: L + +=cut + +__PACKAGE__->has_many( + "buildoutputs", + "Hydra::Schema::BuildOutputs", + { "foreign.build" => "self.id" }, + undef, +); + =head2 buildproducts Type: has_many @@ -336,6 +344,21 @@ __PACKAGE__->has_many( undef, ); +=head2 buildstepoutputs + +Type: has_many + +Related object: L + +=cut + +__PACKAGE__->has_many( + "buildstepoutputs", + "Hydra::Schema::BuildStepOutputs", + { "foreign.build" => "self.id" }, + undef, +); + =head2 buildsteps Type: has_many @@ -442,8 +465,8 @@ __PACKAGE__->has_many( ); -# Created by DBIx::Class::Schema::Loader v0.07033 @ 2013-01-22 13:34:39 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:wPBFqpUWncuD9xki8Pbnvg +# Created by DBIx::Class::Schema::Loader v0.07033 @ 2013-01-30 16:22:11 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:YBdqPWScG4dtGx+U3dJcwA __PACKAGE__->has_many( "dependents", @@ -459,13 +482,13 @@ __PACKAGE__->has_many( { "foreign.build" => "self.id" }, ); -__PACKAGE__->has_one( - "actualBuildStep", - "Hydra::Schema::BuildSteps", - { 'foreign.outpath' => 'self.outpath' - , 'foreign.build' => 'self.id' - }, -); +#__PACKAGE__->has_one( +# "actualBuildStep", +# "Hydra::Schema::BuildSteps", +# { 'foreign.outpath' => 'self.outpath' +# , 'foreign.build' => 'self.id' +# }, +#); sub makeSource { my ($name, $query) = @_; diff --git a/src/lib/Hydra/View/NixExprs.pm b/src/lib/Hydra/View/NixExprs.pm index 9fb84c80..a88a0e0f 100644 --- a/src/lib/Hydra/View/NixExprs.pm +++ b/src/lib/Hydra/View/NixExprs.pm @@ -21,13 +21,13 @@ sub process { my $res = "[\n"; - foreach my $name (keys %{$c->stash->{nixPkgs}}) { - my $build = $c->stash->{nixPkgs}->{$name}->{build}; - $res .= " # $name\n"; + foreach my $pkg (@{$c->stash->{nixPkgs}}) { + my $build = $pkg->{build}; + $res .= " # $pkg->{name}\n"; $res .= " { type = \"derivation\";\n"; $res .= " name = " . escape ($build->get_column("releasename") or $build->nixname) . ";\n"; $res .= " system = " . (escape $build->system) . ";\n"; - $res .= " outPath = " . (escape $build->outpath) . ";\n"; + $res .= " outPath = " . (escape $pkg->{outPath}) . ";\n"; $res .= " meta = {\n"; $res .= " description = " . (escape $build->description) . ";\n" if $build->description; diff --git a/src/lib/Hydra/View/NixPkg.pm b/src/lib/Hydra/View/NixPkg.pm index 5631de2c..0e7b4222 100644 --- a/src/lib/Hydra/View/NixPkg.pm +++ b/src/lib/Hydra/View/NixPkg.pm @@ -10,9 +10,10 @@ sub process { my $build = $c->stash->{build}; + # FIXME: add multiple output support my $s = "NIXPKG1 " . $c->stash->{manifestUri} . " " . $build->nixname . " " . $build->system - . " " . $build->drvpath . " " . $build->outpath + . " " . $build->drvpath . " " . $build->buildoutputs->find({name => "out"})->path . " " . $c->uri_for('/'); $c->response->body($s); diff --git a/src/root/admin.tt b/src/root/admin.tt index 42c044b2..c1f4f641 100644 --- a/src/root/admin.tt +++ b/src/root/admin.tt @@ -26,7 +26,7 @@ [% INCLUDE renderFullJobName project = step.build.project.name jobset = step.build.jobset.name job = step.build.job.name %] [% step.system %] [% step.build.id %] - [% step.outpath.match('-(.*)').0 %] + [% step.drvpath.match('-(.*)').0 %] [% INCLUDE renderDuration duration = curTime - step.starttime %] [% END %] diff --git a/src/root/all.tt b/src/root/all.tt index f0f0b9a6..43377ca4 100644 --- a/src/root/all.tt +++ b/src/root/all.tt @@ -6,8 +6,7 @@ [% ELSIF jobset %]for Jobset [% project.name %]:[% jobset.name %] [% ELSIF project %] for Project [% project.name %][% END %] -

Showing builds [% (page - 1) * resultsPerPage + 1 %] - [% (page - 1) * resultsPerPage + builds.size %] -out of [% total %] in order of descending timestamp.

+

Showing builds [% (page - 1) * resultsPerPage + 1 %] - [% (page - 1) * resultsPerPage + builds.size %] out of [% total %] in order of descending timestamp.

[% INCLUDE renderPager %] [% INCLUDE renderBuildList hideProjectName=project hideJobsetName=jobset hideJobName=job %] diff --git a/src/root/build.tt b/src/root/build.tt index fb6cf94c..08f5cef0 100644 --- a/src/root/build.tt +++ b/src/root/build.tt @@ -8,57 +8,60 @@ [% jobset = build.jobset %] [% job = build.job %] -[% BLOCK renderBuildSteps %] - - - - - - [% FOREACH step IN build.buildsteps -%] - [% IF ( type == "All" ) || ( type == "Failed" && step.status != 0 ) || ( type == "Running" && step.busy == 1 ) -%] - [% has_log = log_exists(step.drvpath); - log = c.uri_for('/build' build.id 'nixlog' step.stepnr); -%] - - - - - - - - [% END %] - [% END %] - -
NrWhatDurationMachineStatus
[% step.stepnr %] - [% IF step.type == 0 %] - Build of [% step.outpath %] - [% ELSE %] - Substitution of [% step.outpath %] - [% END %] - - [% IF step.busy == 0 %] - [% INCLUDE renderDuration duration = step.stoptime - step.starttime %] - [% ELSE %] - [% IF build.finished %] - [% INCLUDE renderDuration duration = build.stoptime - step.starttime %] - [% ELSE %] - [% INCLUDE renderDuration duration = curTime - step.starttime %] - [% END %] - [% END %] - [% step.machine.split('@').1 %] - [% IF step.busy == 1 %] - Building - [% ELSIF step.status == 0 %] - Succeeded - [% ELSIF step.status == 4 %] - Aborted - [% ELSE %] - Failed: [% HTML.escape(step.errormsg) %] - [% END %] - [% IF has_log %] - (log, raw, tail) - [% END %] -
+[% BLOCK renderOutputs %] + [% start=1; FOREACH output IN outputs %] + [% IF !start %],
[% END; start=0; output.path %] + [% END %] +[% END %] +[% BLOCK renderBuildSteps %] + + + + + + [% FOREACH step IN build.buildsteps %] + [% IF ( type == "All" ) || ( type == "Failed" && step.status != 0 ) || ( type == "Running" && step.busy == 1 ) %] + [% has_log = log_exists(step.drvpath); + log = c.uri_for('/build' build.id 'nixlog' step.stepnr); %] + + + + + + + + [% END %] + [% END %] + +
NrWhatDurationMachineStatus
[% step.stepnr %] + [% IF step.type == 0 %] + Build of [% INCLUDE renderOutputs outputs=step.buildstepoutputs %] + [% ELSE %] + Substitution of [% INCLUDE renderOutputs outputs=step.buildstepoutputs %] + [% END %] + + [% IF step.busy == 0; + INCLUDE renderDuration duration = step.stoptime - step.starttime; + ELSIF build.finished; + INCLUDE renderDuration duration = build.stoptime - step.starttime; + ELSE; + INCLUDE renderDuration duration = curTime - step.starttime; + END %] + [% step.machine.split('@').1 %] + [% IF step.busy == 1 %] + Building + [% ELSIF step.status == 0 %] + Succeeded + [% ELSIF step.status == 4 %] + Aborted + [% ELSE %] + Failed: [% HTML.escape(step.errormsg) %] + [% END %] + [% IF has_log %] + (log, raw, tail) + [% END %] +
[% END %] @@ -110,15 +113,18 @@ System: [% build.system %] - [% IF build.finished %] + [% IF build.iscachedbuild %] + + Cached from: + + [% INCLUDE renderFullBuildLink build=cachedBuild %] + + + [% ELSE %] Duration: - [% IF build.iscachedbuild %] - (cached[% IF cachedBuild %] from [% INCLUDE renderFullBuildLink build=cachedBuild %][% END %]) - [% ELSE %] - [% INCLUDE renderDuration duration = build.stoptime - build.starttime %] finished at [% INCLUDE renderDateTime timestamp = build.stoptime %] - [% END %] + [% INCLUDE renderDuration duration = build.stoptime - build.starttime %]; finished at [% INCLUDE renderDateTime timestamp = build.stoptime %] [% END %] @@ -217,28 +223,6 @@

Information

- - - - - - - - - - - - - - - - - - - - [% IF build.nixexprinput %] @@ -249,12 +233,6 @@ - [% IF build.releasename %] - - - - - [% END %] @@ -289,22 +267,14 @@ - + - [% IF pathHash %] - - - - - [% END %] diff --git a/src/root/channel-contents.tt b/src/root/channel-contents.tt index 12e4b01e..fd3f7228 100644 --- a/src/root/channel-contents.tt +++ b/src/root/channel-contents.tt @@ -64,6 +64,7 @@ install the package simply by clicking on the packages below.

[% ELSE -%] [% HTML.escape(build.description) -%] [% END -%] + [% IF pkg.outName != 'out' %] [[% pkg.outName %]][% END %] diff --git a/src/root/common.tt b/src/root/common.tt index 1f254866..0f17aeb4 100644 --- a/src/root/common.tt +++ b/src/root/common.tt @@ -347,7 +347,7 @@ [% IF bi1.name == bi2.name %] [% IF bi1.type == bi2.type %] [% IF bi1.value != bi2.value || bi1.uri != bi2.uri %] - + [% ELSIF bi1.uri == bi2.uri && bi1.revision != bi2.revision %] [% IF bi1.type == "git" %] - + [% END %] diff --git a/src/root/deps.tt b/src/root/deps.tt index 8ab580d6..8a25492d 100644 --- a/src/root/deps.tt +++ b/src/root/deps.tt @@ -8,7 +8,7 @@ [% IF available %] -

Runtime dependencies for [% build.outpath %]

+

Runtime dependencies

    [% FOREACH dep IN runtimedeps -%]
  • @@ -26,7 +26,7 @@ Path not available anymore!
    [% IF drvAvailable %] -

    Build time dependencies for [% build.drvpath %]

    +

    Build time dependencies

      [% FOREACH dep IN buildtimedeps -%]
    • diff --git a/src/root/jobset.tt b/src/root/jobset.tt index bd81cc88..5128844c 100644 --- a/src/root/jobset.tt +++ b/src/root/jobset.tt @@ -204,7 +204,7 @@
      [% IF activeJobs.size == 0 %](none)[% END %] - [% FOREACH j IN activeJobs %] [% INCLUDE renderJobName project=project.name jobset=jobset.name job=j %] [% END %] + [% FOREACH j IN activeJobs %][% INCLUDE renderJobName project=project.name jobset=jobset.name job=j %]
      [% END %]

      @@ -212,7 +212,7 @@
      [% IF inactiveJobs.size == 0 %](none)[% END %] - [% FOREACH j IN inactiveJobs %] [% INCLUDE renderJobName project=project.name jobset=jobset.name job=j %] [% END %] + [% FOREACH j IN inactiveJobs %][% INCLUDE renderJobName project=project.name jobset=jobset.name job=j %]
      [% END %]

      diff --git a/src/root/log.tt b/src/root/log.tt index edc53b61..be1711ac 100644 --- a/src/root/log.tt +++ b/src/root/log.tt @@ -4,7 +4,7 @@

      Build log of [% INCLUDE renderFullJobNameOfBuild %] build [% build.id %][%IF step %] step [% step.stepnr %][% END %]

      - This is the build log of path [% IF step; step.outpath; ELSE; build.outpath; END %]. + This is the build log of derivation [% IF step; step.drvpath; ELSE; build.drvpath; END %]. [% IF step && step.machine %] It was built on [% step.machine %]. [% END %] diff --git a/src/root/product-list.tt b/src/root/product-list.tt index 65a3a11f..33282751 100644 --- a/src/root/product-list.tt +++ b/src/root/product-list.tt @@ -16,7 +16,7 @@ [% CASE "nix-build" %] [% IF build.buildstatus == 6 %] - [% filename = "${build.nixname}.closure.gz" %] + [% filename = build.nixname _ (product.subtype ? "-" _ product.subtype : "") _ ".closure.gz" %] [% uri = c.uri_for('/build' build.id 'nix' 'closure' filename ) %]

Build ID:[% build.id %]
Status: - [% INCLUDE renderStatus build=build %] -
Project:[% INCLUDE renderProjectName project=project.name %]
Jobset:[% INCLUDE renderJobsetName project=project.name jobset=jobset.name %]
Job name:[% INCLUDE renderJobName project=project.name jobset=jobset.name job=job.name %]
Nix expression:Nix name: [% build.nixname %]
Release name:[% HTML.escape(build.releasename) %]
Short description: [% IF build.description %][% HTML.escape(build.description) %][% ELSE %](not given)[% END %]
Output store path:Output store paths: - [% build.outpath %] + [% INCLUDE renderOutputs outputs=build.buildoutputs %] [% IF available %] (runtime dependencies) [% END %]
Output store path hash: - [% pathHash %] -
Time added: [% INCLUDE renderDateTime timestamp = build.timestamp %]
[% bi1.name %][% INCLUDE renderShortInputValue input=bi1 %] to [% INCLUDE renderShortInputValue input=bi2 %]
[% bi1.name %][% INCLUDE renderShortInputValue input=bi1 %] to [% INCLUDE renderShortInputValue input=bi2 %]
@@ -360,7 +360,7 @@ [% END %] [% ELSIF bi1.dependency.id != bi2.dependency.id || bi1.path != bi2.path %]
- [% bi1.name %][% INCLUDE renderShortInputValue input=bi1 %] to [% INCLUDE renderShortInputValue input=bi2 %] + [% bi1.name %][% INCLUDE renderShortInputValue input=bi1 %] to [% INCLUDE renderShortInputValue input=bi2 %]

[% INCLUDE renderInputDiff build1=bi1.dependency, build2=bi2.dependency, nestedDiff=1, nestLevel=nestLevel+1 %] @@ -395,7 +395,7 @@ 		[% step.system %] [% step.build.id %] [% step.stepnr %][% step.outpath.match('-(.*)').0 %][% step.drvpath.match('-(.*)').0 %] [% INCLUDE renderDuration duration = curTime - step.starttime %]
@@ -70,7 +70,7 @@
- [% filename = "${build.nixname}.closure.gz" %] + [% filename = build.nixname _ (product.subtype ? "-" _ product.subtype : "") _ ".closure.gz" %] [% uri = c.uri_for('/build' build.id 'nix' 'closure' filename ) %] diff --git a/src/script/hydra-build b/src/script/hydra-build index ebcbb9df..9b08c810 100755 --- a/src/script/hydra-build +++ b/src/script/hydra-build @@ -1,6 +1,7 @@ #! /var/run/current-system/sw/bin/perl -w use strict; +use List::MoreUtils qw(all); use File::Basename; use File::stat; use Nix::Store; @@ -58,6 +59,7 @@ sub sendTwitterNotification { warn "$@\n" if $@; } + sub statusDescription { my ($buildstatus) = @_; @@ -72,6 +74,7 @@ sub statusDescription { return $status; } + sub sendEmailNotification { my ($build) = @_; @@ -127,7 +130,7 @@ sub sendEmailNotification { [ "Maintainer(s):", $build->maintainers ], [ "System:", $build->system ], [ "Derivation store path:", $build->drvpath ], - [ "Output store path:", $build->outpath ], + [ "Output store path:", join(", ", map { $_->path } $build->buildoutputs) ], [ "Time added:", showTime $build->timestamp ], ); push @lines, ( @@ -206,11 +209,21 @@ sub sendEmailNotification { } +sub addBuildStepOutputs { + my ($step) = @_; + my $drv = derivationFromPath($step->drvpath); + $step->buildstepoutputs->create({ name => $_, path => $drv->{outputs}->{$_} }) + foreach keys %{$drv->{outputs}}; +} + + sub doBuild { my ($build) = @_; + my %outputs; + $outputs{$_->name} = $_->path foreach $build->buildoutputs->all; + my $drvPath = $build->drvpath; - my $outPath = $build->outpath; my $maxsilent = $build->maxsilent; my $timeout = $build->timeout; @@ -223,7 +236,7 @@ sub doBuild { my $errormsg = undef; - if (!isValidPath($outPath)) { + unless (all { isValidPath($_) } values(%outputs)) { $isCachedBuild = 0; # Do the build. @@ -235,12 +248,12 @@ sub doBuild { # Run Nix to perform the build, and monitor the stderr output # to get notifications about specific build steps, the # associated log files, etc. - # Note: `--timeout' was added in Nix 1.0pre27564, June 2011. my $cmd = "nix-store --realise $drvPath " . "--timeout $timeout " . "--max-silent-time $maxsilent --keep-going --fallback " . "--no-build-output --log-type flat --print-build-trace " . - "--add-root " . gcRootFor $outPath . " 2>&1"; + "--add-root " . gcRootFor($outputs{out} // $outputs{(sort keys %outputs)[0]}) . " 2>&1"; + print STDERR "$cmd\n"; my $max = $build->buildsteps->find( {}, {select => {max => 'stepnr + 1'}, as => ['max']}); @@ -261,15 +274,15 @@ sub doBuild { if (/^@\s+build-started\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)$/) { my $drvPathStep = $1; txn_do($db, sub { - $build->buildsteps->create( + my $step = $build->buildsteps->create( { stepnr => ($buildSteps{$drvPathStep} = $buildStepNr++) , type => 0 # = build , drvpath => $drvPathStep - , outpath => $2 , system => $3 , busy => 1 , starttime => time }); + addBuildStepOutputs($step); }); } @@ -305,46 +318,47 @@ sub doBuild { # failed previously. This can happen if this is a # restarted build. elsif (scalar $build->buildsteps->search({drvpath => $drvPathStep, type => 0, busy => 0, status => 1}) == 0) { - $build->buildsteps->create( + my $step = $build->buildsteps->create( { stepnr => ($buildSteps{$drvPathStep} = $buildStepNr++) , type => 0 # = build , drvpath => $drvPathStep - , outpath => $2 , busy => 0 , status => 1 , starttime => time , stoptime => time , errormsg => $errorMsg }); + addBuildStepOutputs($step); } }); } elsif (/^@\s+substituter-started\s+(\S+)\s+(\S+)$/) { - my $outPath = $1; + my $path = $1; txn_do($db, sub { - $build->buildsteps->create( - { stepnr => ($buildSteps{$outPath} = $buildStepNr++) + my $step = $build->buildsteps->create( + { stepnr => ($buildSteps{$path} = $buildStepNr++) , type => 1 # = substitution - , outpath => $1 , busy => 1 , starttime => time }); + # "out" is kinda fake (substitutions don't have named outputs). + $step->buildstepoutputs->create({ name => "out", path => $path }); }); } elsif (/^@\s+substituter-succeeded\s+(\S+)$/) { - my $outPath = $1; + my $path = $1; txn_do($db, sub { - my $step = $build->buildsteps->find({stepnr => $buildSteps{$outPath}}) or die; + my $step = $build->buildsteps->find({stepnr => $buildSteps{$path}}) or die; $step->update({busy => 0, status => 0, stoptime => time}); }); } elsif (/^@\s+substituter-failed\s+(\S+)\s+(\S+)\s+(\S+)$/) { - my $outPath = $1; + my $path = $1; txn_do($db, sub { - my $step = $build->buildsteps->find({stepnr => $buildSteps{$outPath}}) or die; + my $step = $build->buildsteps->find({stepnr => $buildSteps{$path}}) or die; $step->update({busy => 0, status => 1, errormsg => $3, stoptime => time}); }); } @@ -371,24 +385,34 @@ sub doBuild { } done: - my $size = 0; - my $closuresize = 0; - - if (isValidPath($outPath)) { - my ($deriver, $hash, $time, $narSize, $refs) = queryPathInfo($outPath, 0); - $size = $narSize; - - my @closure = computeFSClosure(0, 0, $outPath); - foreach my $path (@closure) { - my ($deriver, $hash, $time, $narSize, $refs) = queryPathInfo($path, 0); - $closuresize += $narSize; - } - } txn_do($db, sub { - my $releaseName = getReleaseName($outPath); + if ($buildStatus == 0) { - $buildStatus = 6 if $buildStatus == 0 && -f "$outPath/nix-support/failed"; + my $size = 0; + my $closureSize = 0; + my $releaseName; + + my @closure = computeFSClosure(0, 0, values %outputs); + foreach my $path (@closure) { + my ($deriver, $hash, $time, $narSize, $refs) = queryPathInfo($path, 0); + $closureSize += $narSize; + $size += $narSize if grep { $path eq $_ } values(%outputs); + } + + foreach my $path (values %outputs) { + $buildStatus = 6 if $buildStatus == 0 && -f "$path/nix-support/failed"; + $releaseName //= getReleaseName($path); + } + + $build->update( + { releasename => $releaseName + , size => $size + , closuresize => $closureSize + }); + + addBuildProducts($db, $build); + } $build->update( { finished => 1 @@ -400,15 +424,9 @@ sub doBuild { , buildstatus => $buildStatus , starttime => $startTime , stoptime => $stopTime - , size => $size - , closuresize => $closuresize , errormsg => $errormsg - , releasename => $releaseName }); - if ($buildStatus == 0 || $buildStatus == 6) { - addBuildProducts($db, $build); - } }); sendEmailNotification $build; diff --git a/src/script/hydra-update-gc-roots b/src/script/hydra-update-gc-roots index 8b403be9..43b17a4b 100755 --- a/src/script/hydra-update-gc-roots +++ b/src/script/hydra-update-gc-roots @@ -21,25 +21,27 @@ sub addRoot { } -my @columns = ( "id", "project", "jobset", "job", "system", "finished", "outpath", "drvpath", "timestamp" ); +my @columns = ( "id", "project", "jobset", "job", "system", "finished", "drvpath", "timestamp" ); sub keepBuild { my ($build) = @_; print STDERR " keeping ", ($build->finished ? "" : "scheduled "), "build ", $build->id, " (", $build->get_column('project'), ":", $build->get_column('jobset'), ":", $build->get_column('job'), "; ", - $build->system, "; ", + $build->system, "; ", strftime("%Y-%m-%d %H:%M:%S", localtime($build->timestamp)), ")\n"; - if (isValidPath($build->outpath)) { - addRoot $build->outpath; - } else { - print STDERR " warning: output ", $build->outpath, " has disappeared\n" if $build->finished; + foreach my $out ($build->buildoutputs->all) { + if (isValidPath($out->path)) { + addRoot $out->path; + } else { + print STDERR " warning: output ", $out->path, " has disappeared\n" if $build->finished; + } } if (!$build->finished) { - if (isValidPath($build->drvpath)) { - addRoot $build->drvpath; - } else { - print STDERR " warning: derivation ", $build->drvpath, " has disappeared\n"; - } + if (isValidPath($build->drvpath)) { + addRoot $build->drvpath; + } else { + print STDERR " warning: derivation ", $build->drvpath, " has disappeared\n"; + } } } @@ -57,7 +59,8 @@ closedir DIR; # Keep every build in every release of every project. print STDERR "*** looking for release members\n"; keepBuild $_ foreach $db->resultset('Builds')->search_literal( - "exists (select 1 from releasemembers where build = me.id)", { order_by => ["project", "jobset", "job", "id"] }); + "exists (select 1 from releasemembers where build = me.id)", + { order_by => ["project", "jobset", "job", "id"], columns => [ @columns ] }); # Keep all builds that have been marked as "keep". @@ -74,8 +77,8 @@ foreach my $project ($db->resultset('Projects')->search({}, { order_by => ["name foreach my $jobset ($project->jobsets->search({}, { order_by => ["name" ]})) { my $keepnr = $jobset->keepnr; - # If the jobset has been hidden and disabled for more than one week, than - # don't keep its builds anymore. + # If the jobset has been hidden and disabled for more than one + # week, then don't keep its builds anymore. if ($jobset->enabled == 0 && ($project->hidden == 1 || $jobset->hidden == 1) && (time() - ($jobset->lastcheckedtime || 0) > (7 * 24 * 3600))) { print STDERR "*** skipping disabled jobset ", $project->name, ":", $jobset->name, "\n"; next; @@ -86,19 +89,20 @@ foreach my $project ($db->resultset('Projects')->search({}, { order_by => ["name next; } - print STDERR "*** looking for the $keepnr most recent successful builds of each job in jobset ", - $project->name, ":", $jobset->name, "\n"; + # FIXME: base this on jobset evals? + print STDERR "*** looking for the $keepnr most recent successful builds of each job in jobset ", + $project->name, ":", $jobset->name, "\n"; - keepBuild $_ foreach $jobset->builds->search( - { 'me.id' => { 'in' => \ - [ "select b2.id from Builds b2 join " . - " (select distinct job, system, coalesce( " . + keepBuild $_ foreach $jobset->builds->search( + { 'me.id' => { 'in' => \ + [ "select b2.id from Builds b2 join " . + " (select distinct job, system, coalesce( " . " (select id from builds where project = b.project and jobset = b.jobset and job = b.job and system = b.system and finished = 1 and buildStatus = 0 order by id desc offset ? limit 1)" . " , 0) as nth from builds b where project = ? and jobset = ? and isCurrent = 1) x " . - " on b2.project = ? and b2.jobset = ? and b2.job = x.job and b2.system = x.system and (id >= x.nth) where finished = 1 and buildStatus = 0" - , [ '', $keepnr - 1 ], [ '', $project->name ], [ '', $jobset->name ], [ '', $project->name ], [ '', $jobset->name ] ] } - }, - { order_by => ["job", "system", "id"], columns => [ @columns ] }); + " on b2.project = ? and b2.jobset = ? and b2.job = x.job and b2.system = x.system and (id >= x.nth) where finished = 1 and buildStatus = 0" + , [ '', $keepnr - 1 ], [ '', $project->name ], [ '', $jobset->name ], [ '', $project->name ], [ '', $jobset->name ] ] } + }, + { order_by => ["job", "system", "id"], columns => [ @columns ] }); } # Go over all views in this project. @@ -135,10 +139,10 @@ foreach my $link (@roots) { my $path = "/nix/store/$link"; if (!defined $roots{$path}) { print STDERR "removing root $path\n"; - $rootsDeleted++; + $rootsDeleted++; unlink "$gcRootsDir/$link" or warn "cannot remove $gcRootsDir/$link"; } else { - $rootsKept++; + $rootsKept++; } } diff --git a/src/sql/hydra.sql b/src/sql/hydra.sql index aa5a3d13..3f236c78 100644 --- a/src/sql/hydra.sql +++ b/src/sql/hydra.sql @@ -133,7 +133,6 @@ create table Builds ( nixName text, -- name attribute of the derivation description text, -- meta.description drvPath text not null, - outPath text not null, system text not null, longDescription text, -- meta.longDescription @@ -174,6 +173,7 @@ create table Builds ( -- 3 = other failure (see errorMsg) -- 4 = build cancelled (removed from queue; never built) -- 5 = build not done because a dependency failed previously (obsolete) + -- 6 = failure with output buildStatus integer, errorMsg text, -- error message in case of a Nix failure @@ -191,6 +191,15 @@ create table Builds ( ); +create table BuildOutputs ( + build integer not null, + name text not null, + path text not null, + primary key (build, name), + foreign key (build) references Builds(id) on delete cascade +); + + create table BuildSteps ( build integer not null, stepnr integer not null, @@ -198,7 +207,6 @@ create table BuildSteps ( type integer not null, -- 0 = build, 1 = substitution drvPath text, - outPath text, busy integer not null, @@ -217,6 +225,17 @@ create table BuildSteps ( ); +create table BuildStepOutputs ( + build integer not null, + stepnr integer not null, + name text not null, + path text not null, + primary key (build, stepnr, name), + foreign key (build) references Builds(id) on delete cascade, + foreign key (build, stepnr) references BuildSteps(build, stepnr) on delete cascade +); + + -- Inputs of builds. create table BuildInputs ( #ifdef POSTGRESQL @@ -494,13 +513,13 @@ create table NewsItems ( create table BuildMachines ( - hostname text primary key NOT NULL, - username text DEFAULT '' NOT NULL, - ssh_key text DEFAULT '' NOT NULL, - options text DEFAULT '' NOT NULL, - maxconcurrent integer DEFAULT 2 NOT NULL, - speedfactor integer DEFAULT 1 NOT NULL, - enabled integer DEFAULT 0 NOT NULL + hostname text primary key not null, + username text default '' not null, + ssh_key text default '' not null, + options text default '' not null, + maxconcurrent integer default 2 not null, + speedfactor integer default 1 not null, + enabled integer default 0 not null ); @@ -518,11 +537,9 @@ create index IndexBuildInputsOnBuild on BuildInputs(build); create index IndexBuildInputsOnDependency on BuildInputs(dependency); create index IndexBuildProducstOnBuildAndType on BuildProducts(build, type); create index IndexBuildProductsOnBuild on BuildProducts(build); -create index IndexBuildStepsOnBuild on BuildSteps(build); create index IndexBuildStepsOnBusy on BuildSteps(busy); create index IndexBuildStepsOnDrvpathTypeBusyStatus on BuildSteps(drvpath, type, busy, status); -create index IndexBuildStepsOnOutpath on BuildSteps(outpath); -create index IndexBuildStepsOnOutpathBuild on BuildSteps (outpath, build); +create index IndexBuildStepOutputsOnPath on BuildStepOutputs(path); create index IndexBuildsOnFinished on Builds(finished); create index IndexBuildsOnFinishedBusy on Builds(finished, busy); create index IndexBuildsOnIsCurrent on Builds(isCurrent);