From a2f56ce574ce954adba4daa151f9d4687ac78917 Mon Sep 17 00:00:00 2001 From: Rob Vermaas Date: Tue, 26 Jun 2012 12:00:18 +0200 Subject: [PATCH] * Read logs using logContents function in stead of handling it everywhere separately. --- src/lib/Hydra/Controller/Build.pm | 16 +++++------- src/lib/Hydra/Helper/Nix.pm | 14 +++++++++- src/script/hydra-build | 43 +++++++++++++++---------------- 3 files changed, 40 insertions(+), 33 deletions(-) diff --git a/src/lib/Hydra/Controller/Build.pm b/src/lib/Hydra/Controller/Build.pm index 8678f9b6..b79eccbb 100644 --- a/src/lib/Hydra/Controller/Build.pm +++ b/src/lib/Hydra/Controller/Build.pm @@ -47,11 +47,9 @@ sub view_build : Chained('build') PathPart('') Args(0) { $c->stash->{pathHash} = $c->stash->{available} ? queryPathHash($build->outpath) : undef; - my $pipestart; if (!$build->finished && $build->busy) { my $logfile = $build->logfile; - $pipestart = cat_log_command($logfile); - $c->stash->{logtext} = `$pipestart` if defined $logfile && -e $logfile; + $c->stash->{logtext} = logContents($logfile); } if ($build->finished && $build->iscachedbuild) { @@ -62,8 +60,7 @@ sub view_build : Chained('build') PathPart('') Args(0) { (my $lastBuildStep) = $build->buildsteps->search({},{order_by => "stepnr DESC", rows => 1}); my $path = defined $lastBuildStep ? $lastBuildStep->logfile : "" ; if ($build->finished && ($build->buildstatus == 1 || $build->buildstatus == 6) && !($path eq "") && -f $lastBuildStep->logfile) { - $pipestart = cat_log_command($path); - my $logtext = `$pipestart | tail -n 50`; + my $logtext = logContents($path, 50); $c->stash->{logtext} = removeAsciiEscapes($logtext); } @@ -126,10 +123,9 @@ sub showLog { notFound($c, "Log file $path no longer exists.") unless -f $fallbackpath; $path = $fallbackpath; - my $pipestart = ($path =~ /.bz2$/ ? "cat $path | bzip2 -d" : "cat $path") ; - if (!$mode) { # !!! quick hack + my $pipestart = ($path =~ /.bz2$/ ? "cat $path | bzip2 -d" : "cat $path") ; my $pipeline = $pipestart . " | nix-log2xml | xsltproc " . $c->path_to("xsl/mark-errors.xsl") . " -" . " | xsltproc " . $c->path_to("xsl/log2html.xsl") . " - | tail -n +2"; @@ -139,7 +135,7 @@ sub showLog { } elsif ($mode eq "raw") { - $c->stash->{'plain'} = { data => (scalar `$pipestart`) || " " }; + $c->stash->{'plain'} = { data => (scalar logContents($path)) || " " }; $c->forward('Hydra::View::Plain'); } @@ -149,12 +145,12 @@ sub showLog { $c->stash->{url} = $url; $c->stash->{reload} = !$c->stash->{build}->finished && $c->stash->{build}->busy; $c->stash->{title} = ""; - $c->stash->{contents} = (scalar `$pipestart | tail -n 50`) || " "; + $c->stash->{contents} = (scalar logContents($path, 50)) || " "; $c->stash->{template} = 'plain-reload.tt'; } elsif ($mode eq "tail") { - $c->stash->{'plain'} = { data => (scalar `$pipestart | tail -n 50`) || " " }; + $c->stash->{'plain'} = { data => (scalar logContents($path, 50)) || " " }; $c->forward('Hydra::View::Plain'); } diff --git a/src/lib/Hydra/Helper/Nix.pm b/src/lib/Hydra/Helper/Nix.pm index 2c64902b..583a917c 100644 --- a/src/lib/Hydra/Helper/Nix.pm +++ b/src/lib/Hydra/Helper/Nix.pm @@ -164,7 +164,7 @@ sub jobsetOverview { my ($c, $project) = @_; return $project->jobsets->search( isProjectOwner($c, $project) ? {} : { hidden => 0 }, { order_by => "name" - , "+select" => + , "+select" => [ "(select count(*) from Builds as a where a.finished = 0 and me.project = a.project and me.name = a.jobset and a.isCurrent = 1)" , "(select count(*) from Builds as a where a.finished = 1 and me.project = a.project and me.name = a.jobset and buildstatus <> 0 and a.isCurrent = 1)" , "(select count(*) from Builds as a where a.finished = 1 and me.project = a.project and me.name = a.jobset and buildstatus = 0 and a.isCurrent = 1)" @@ -236,6 +236,18 @@ sub getLatestSuccessfulViewResult { return undef; } +sub logContents { + my ($path, $tail) = @_; + my $cmd; + if ($path =~ /.bz2$/) { + $cmd = "cat $path | bzip2 -d"; + $cmd = $cmd . " | tail -$tail" if defined $tail; + } + else { + $cmd = defined $tail ? "tail -$tail $path" : "cat $path"; + } + return `$cmd` if -e $path; +} sub removeAsciiEscapes { my ($logtext) = @_; diff --git a/src/script/hydra-build b/src/script/hydra-build index 67dab49c..6dcb60ae 100755 --- a/src/script/hydra-build +++ b/src/script/hydra-build @@ -37,7 +37,7 @@ sub sendTwitterNotification { my $jobName = $build->project->name . ":" . $build->jobset->name . ":" . $build->job->name; my $status = $build->buildstatus == 0 ? "SUCCEEDED" : "FAILED"; my $system = $build->system; - my $duration = ($build->stoptime - $build->starttime) . " seconds"; + my $duration = ($build->stoptime - $build->starttime) . " seconds"; my $url = $config{'base_uri'}."/build/".$build->id ; my $nt = Net::Twitter::Lite->new( @@ -47,11 +47,11 @@ sub sendTwitterNotification { ); my $tag = $build->project->name; - my $msg = "$jobName ($system): $status in $duration #$tag"; + my $msg = "$jobName ($system): $status in $duration #$tag"; if (length($msg) + 1 + length($url) <= 140) { $msg = "$msg $url" ; } - + eval { my $result = eval { $nt->update($msg) }; }; @@ -68,7 +68,7 @@ sub statusDescription { when (2) { $status = "Dependency failed"; } when (4) { $status = "Cancelled"; } } - + return $status; } @@ -76,7 +76,7 @@ sub sendEmailNotification { my ($build) = @_; die unless $build->finished; - + return if ! ( $build->jobset->enableemail && ($build->maintainers ne "" || $build->jobset->emailoverride ne "") ); # Do we want to send mail? @@ -86,7 +86,7 @@ sub sendEmailNotification { { project => $build->project->name , jobset => $build->jobset->name , job => $build->job->name - , system => $build->system + , system => $build->system , finished => 1 , id => { '<', $build->id } }, { order_by => ["id DESC"] } @@ -94,13 +94,13 @@ sub sendEmailNotification { # if there is a previous build with same buildstatus, do not send email if (defined $prevBuild && ($build->buildstatus == $prevBuild->buildstatus)) { - return; + return; } # if buildstatus of this build or the previous one is aborted, do # not send email if ($build->buildstatus == 3 || (defined $prevBuild && ($prevBuild->buildstatus == 3))) { - return; + return; } # Send mail. @@ -157,8 +157,7 @@ sub sendEmailNotification { $inputsTable->load(@lines); my $loglines = 50; - my $logfile = $build->logfile; - my $logtext = defined $logfile && -e $logfile ? `tail -$loglines $logfile` : "No logfile available.\n"; + my $logtext = logContents($build->logfile, $loglines); $logtext = removeAsciiEscapes($logtext); my $body = "Hi,\n" @@ -181,7 +180,7 @@ sub sendEmailNotification { . $inputsTable->body . "\n" . "Regards,\n\nThe Hydra build daemon.\n" - . ($build->buildstatus != 0 ? "\n---\n$logtext" : ""); + . ($build->buildstatus != 0 ? "\n---\n$logtext" : ""); # stripping trailing spaces from lines $body =~ s/[\ ]+$//gm; @@ -233,7 +232,7 @@ sub doBuild { my $thisBuildFailed = 0; my $someBuildFailed = 0; - + # Run Nix to perform the build, and monitor the stderr output # to get notifications about specific build steps, the # associated log files, etc. @@ -249,17 +248,17 @@ sub doBuild { my $buildStepNr = (defined $max && defined $max->get_column('max')) ? $max->get_column('max') : 1; my %buildSteps; - + open OUT, "$cmd |" or die; while () { $errormsg .= $_; - + unless (/^@\s+/) { print STDERR "$_"; next; } - + if (/^@\s+build-started\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)$/) { my $drvPathStep = $1; txn_do($db, sub { @@ -275,7 +274,7 @@ sub doBuild { }); }); } - + elsif (/^@\s+build-remote\s+(\S+)\s+(\S+)$/) { my $drvPathStep = $1; my $machine = $2; @@ -284,14 +283,14 @@ sub doBuild { $step->update({machine => $machine}); }); } - + elsif (/^@\s+build-succeeded\s+(\S+)\s+(\S+)$/) { my $drvPathStep = $1; txn_do($db, sub { my $step = $build->buildsteps->find({stepnr => $buildSteps{$drvPathStep}}) or die; my $stepOutpath = $step->outpath; my $stepStatus = 0; - + $step->update({busy => 0, status => $stepStatus, stoptime => time}); }); } @@ -360,7 +359,7 @@ sub doBuild { print STDERR "unknown Nix trace message: $_"; } } - + close OUT; my $res = $?; @@ -378,7 +377,7 @@ sub doBuild { } done: - my $logfile = getBuildLog($drvPath); + my $logfile = getBuildLog($drvPath); my $logsize = defined $logfile ? stat($logfile)->size : 0; @@ -392,13 +391,13 @@ sub doBuild { my @closure = computeFSClosure(0, 0, $outPath); foreach my $path (@closure) { my ($deriver, $hash, $time, $narSize, $refs) = queryPathInfo($path); - $closuresize += $narSize; + $closuresize += $narSize; } } txn_do($db, sub { my $releaseName = getReleaseName($outPath); - + $buildStatus = 6 if $buildStatus == 0 && -f "$outPath/nix-support/failed"; $build->update(