From fd7e37ef898a2cdc43d22c8db1c1e135a8dffaa0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 30 Aug 2013 13:53:25 +0000 Subject: [PATCH] Try harder to find build logs Due to the fixed-output derivation hashing scheme, there can be multiple derivations of the same output path. But build logs are indexed by derivation path. Thus, we may not be able to find the log of a build or build step using its derivation. So as a fallback, Hydra now looks for other derivations with the same output paths. --- src/lib/Hydra/Controller/Build.pm | 40 +++++++++++++++---------------- src/lib/Hydra/Helper/Nix.pm | 34 ++++++++++++++++++++++---- src/lib/Hydra/View/TT.pm | 15 ++++++++---- src/root/build.tt | 4 ++-- src/root/deps.tt | 2 +- 5 files changed, 64 insertions(+), 31 deletions(-) diff --git a/src/lib/Hydra/Controller/Build.pm b/src/lib/Hydra/Controller/Build.pm index 46fe1d5c..0bdfbda5 100644 --- a/src/lib/Hydra/Controller/Build.pm +++ b/src/lib/Hydra/Controller/Build.pm @@ -35,18 +35,18 @@ sub buildChain :Chained('/') :PathPart('build') :CaptureArgs(1) { sub findBuildStepByOutPath { - my ($self, $c, $path, $status) = @_; + my ($self, $c, $path) = @_; return $c->model('DB::BuildSteps')->search( - { path => $path, busy => 0, status => $status }, - { join => ["buildstepoutputs"], order_by => ["stopTime"], limit => 1 })->single; + { path => $path, busy => 0 }, + { join => ["buildstepoutputs"], order_by => ["status", "stopTime"], rows => 1 })->single; } sub findBuildStepByDrvPath { - my ($self, $c, $drvPath, $status) = @_; + my ($self, $c, $drvPath) = @_; return $c->model('DB::BuildSteps')->search( - { drvpath => $drvPath, busy => 0, status => $status }, - { order_by => ["stopTime"], limit => 1 })->single; + { drvpath => $drvPath, busy => 0 }, + { order_by => ["status", "stopTime"], rows => 1 })->single; } @@ -68,8 +68,7 @@ sub build_GET { if ($build->finished && $build->iscachedbuild) { my $path = ($build->buildoutputs)[0]->path or die; - my $cachedBuildStep = findBuildStepByOutPath($self, $c, $path, - $build->buildstatus == 0 || $build->buildstatus == 6 ? 0 : 1); + my $cachedBuildStep = findBuildStepByOutPath($self, $c, $path); $c->stash->{cachedBuild} = $cachedBuildStep->build if defined $cachedBuildStep; } @@ -95,9 +94,9 @@ sub build_GET { # Get the first eval of which this build was a part. ($c->stash->{nrEvals}) = $c->stash->{build}->jobsetevals->search({ hasnewbuilds => 1 })->count; - ($c->stash->{eval}) = $c->stash->{build}->jobsetevals->search( + $c->stash->{eval} = $c->stash->{build}->jobsetevals->search( { hasnewbuilds => 1}, - { limit => 1, order_by => ["id"] }); + { rows => 1, order_by => ["id"] })->single; $self->status_ok( $c, entity => $c->model('DB::Builds')->find($build->id,{ @@ -132,26 +131,27 @@ sub view_nixlog : Chained('buildChain') PathPart('nixlog') { $c->stash->{step} = $step; - showLog($c, $step->drvpath, $mode); + showLog($c, $mode, $step->drvpath, map { $_->path } $step->buildstepoutputs->all); } sub view_log : Chained('buildChain') PathPart('log') { my ($self, $c, $mode) = @_; - showLog($c, $c->stash->{build}->drvpath, $mode); + showLog($c, $mode, $c->stash->{build}->drvpath, map { $_->path } $c->stash->{build}->buildoutputs->all); } sub showLog { - my ($c, $drvPath, $mode) = @_; + my ($c, $mode, $drvPath, @outPaths) = @_; - my $logPath = getDrvLogPath($drvPath); + my $logPath = findLog($c, $drvPath, @outPaths); + print STDERR "log = $logPath\n"; notFound($c, "The build log of derivation ‘$drvPath’ is not available.") unless defined $logPath; if (!$mode) { # !!! quick hack - my $pipeline = "nix-store -l $drvPath" + my $pipeline = ($logPath =~ /.bz2$/ ? "bzip2 -d < $logPath" : "cat $logPath") . " | nix-log2xml | xsltproc " . $c->path_to("xsl/mark-errors.xsl") . " -" . " | xsltproc " . $c->path_to("xsl/log2html.xsl") . " - | tail -n +2"; $c->stash->{template} = 'log.tt'; @@ -159,7 +159,7 @@ sub showLog { } elsif ($mode eq "raw") { - $c->stash->{'plain'} = { data => (scalar logContents($drvPath)) || " " }; + $c->stash->{'plain'} = { data => (scalar logContents($logPath)) || " " }; $c->forward('Hydra::View::Plain'); } @@ -169,12 +169,12 @@ sub showLog { $c->stash->{url} = $url; $c->stash->{reload} = !$c->stash->{build}->finished && $c->stash->{build}->busy; $c->stash->{title} = ""; - $c->stash->{contents} = (scalar logContents($drvPath, 50)) || " "; + $c->stash->{contents} = (scalar logContents($logPath, 50)) || " "; $c->stash->{template} = 'plain-reload.tt'; } elsif ($mode eq "tail") { - $c->stash->{'plain'} = { data => (scalar logContents($drvPath, 50)) || " " }; + $c->stash->{'plain'} = { data => (scalar logContents($logPath, 50)) || " " }; $c->forward('Hydra::View::Plain'); } @@ -361,8 +361,8 @@ sub getDependencyGraph { { path => $path , name => $name , buildStep => $runtime - ? findBuildStepByOutPath($self, $c, $path, 0) - : findBuildStepByDrvPath($self, $c, $path, 0) + ? findBuildStepByOutPath($self, $c, $path) + : findBuildStepByDrvPath($self, $c, $path) }; $$done{$path} = $node; my @refs; diff --git a/src/lib/Hydra/Helper/Nix.pm b/src/lib/Hydra/Helper/Nix.pm index b4278027..64c9e26f 100644 --- a/src/lib/Hydra/Helper/Nix.pm +++ b/src/lib/Hydra/Helper/Nix.pm @@ -16,7 +16,7 @@ our @EXPORT = qw( getPrimaryBuildsForView getPrimaryBuildTotal getViewResult getLatestSuccessfulViewResult - jobsetOverview removeAsciiEscapes getDrvLogPath logContents + jobsetOverview removeAsciiEscapes getDrvLogPath findLog logContents getMainOutput getEvals getMachines pathIsInsidePrefix @@ -264,10 +264,36 @@ sub getDrvLogPath { } +# Find the log of the derivation denoted by $drvPath. It it doesn't +# exist, try other derivations that produced its outputs (@outPaths). +sub findLog { + my ($c, $drvPath, @outPaths) = @_; + + if (defined $drvPath) { + my $logPath = getDrvLogPath($drvPath); + return $logPath if defined $logPath; + } + + return undef if scalar @outPaths == 0; + + my @steps = $c->model('DB::BuildSteps')->search( + { path => { -in => [@outPaths] } }, + { select => ["drvpath"] + , distinct => 1 + , join => "buildstepoutputs" + }); + + foreach my $step (@steps) { + my $logPath = getDrvLogPath($step->drvpath); + return $logPath if defined $logPath; + } + + return undef; +} + + sub logContents { - my ($drvPath, $tail) = @_; - my $logPath = getDrvLogPath($drvPath); - die unless defined $logPath; + my ($logPath, $tail) = @_; my $cmd; if ($logPath =~ /.bz2$/) { $cmd = "bzip2 -d < $logPath"; diff --git a/src/lib/Hydra/View/TT.pm b/src/lib/Hydra/View/TT.pm index ece42f65..324d9a81 100644 --- a/src/lib/Hydra/View/TT.pm +++ b/src/lib/Hydra/View/TT.pm @@ -8,7 +8,7 @@ __PACKAGE__->config( TEMPLATE_EXTENSION => '.tt', PRE_CHOMP => 1, POST_CHOMP => 1, - expose_methods => [qw/log_exists ellipsize/]); + expose_methods => [qw/log_exists buildLogExists buildStepLogExists/]); sub log_exists { my ($self, $c, $drvPath) = @_; @@ -16,9 +16,16 @@ sub log_exists { return defined $x; } -sub ellipsize { - my ($self, $c, $s, $n) = @_; - return length $s <= $n ? $s : substr($s, 0, $n - 3) . "..."; +sub buildLogExists { + my ($self, $c, $build) = @_; + my @outPaths = map { $_->path } $build->buildoutputs->all; + return defined findLog($c, $build->drvpath, @outPaths); +} + +sub buildStepLogExists { + my ($self, $c, $step) = @_; + my @outPaths = map { $_->path } $step->buildstepoutputs->all; + return defined findLog($c, $step->drvpath, @outPaths); } 1; diff --git a/src/root/build.tt b/src/root/build.tt index 061e4afe..882b61c0 100644 --- a/src/root/build.tt +++ b/src/root/build.tt @@ -23,7 +23,7 @@ [% FOREACH step IN build.buildsteps %] [% IF ( type == "All" ) || ( type == "Failed" && step.status != 0 ) || ( type == "Running" && step.busy == 1 ) %] - [% has_log = log_exists(step.drvpath); + [% has_log = buildStepLogExists(step); log = c.uri_for('/build' build.id 'nixlog' step.stepnr); %] [% step.stepnr %] @@ -177,7 +177,7 @@ finished at [% INCLUDE renderDateTime timestamp = actualBuild.stoptime %] [% END %] - [% IF !isAggregate && log_exists(build.drvpath) %] + [% IF !isAggregate && buildLogExists(build) %] Logfile: diff --git a/src/root/deps.tt b/src/root/deps.tt index 3ad4c05c..a2c1fbba 100644 --- a/src/root/deps.tt +++ b/src/root/deps.tt @@ -12,7 +12,7 @@ [% IF node.buildStep %] [% node.name %] [% - IF log_exists(node.buildStep.drvpath); + IF buildStepLogExists(node.buildStep); INCLUDE renderLogLinks url=c.uri_for('/build' node.buildStep.get_column('build') 'nixlog' node.buildStep.stepnr); END %] [% ELSE %]