lix-project/hydra
3
0
Fork 6
Code Issues 2 Pull requests Packages Projects Releases Wiki Activity

hydra-evaluator: Send statistics to statsd

Browse source
This commit is contained in:
Eelco Dolstra 2015-07-10 16:40:50 +02:00
parent 5919e911db
commit 3e7bbec40b
1 changed files with 30 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

37
src/script/hydra-evaluator
View file

@ -12,6 +12,8 @@ use Digest::SHA qw(sha256_hex);
use Config::General; use Config::General;
use Data::Dump qw(dump); use Data::Dump qw(dump);
use Try::Tiny; use Try::Tiny;
use Net::Statsd;
use Time::HiRes qw(clock_gettime CLOCK_REALTIME);
STDOUT->autoflush(); STDOUT->autoflush();
STDERR->autoflush(1); STDERR->autoflush(1);
@ -102,11 +104,17 @@ sub checkJobsetWrapped {
my $exprType = $jobset->nixexprpath =~ /.scm$/ ? "guile" : "nix"; my $exprType = $jobset->nixexprpath =~ /.scm$/ ? "guile" : "nix";
# Fetch all values for all inputs. # Fetch all values for all inputs.
my $checkoutStart = time; my $checkoutStart = clock_gettime(CLOCK_REALTIME);
eval { eval {
fetchInputs($project, $jobset, $inputInfo); fetchInputs($project, $jobset, $inputInfo);
}; };
Net::Statsd::increment("hydra.evaluator.checkouts");
my $checkoutStop = clock_gettime(CLOCK_REALTIME);
Net::Statsd::timing("hydra.evaluator.checkout_time", int(($checkoutStop - $checkoutStart) * 1000));
if ($@) { if ($@) {
Net::Statsd::increment("hydra.evaluator.failed_checkouts");
my $msg = $@; my $msg = $@;
print STDERR $msg; print STDERR $msg;
txn_do($db, sub { txn_do($db, sub {
@ -114,7 +122,6 @@ sub checkJobsetWrapped {
}); });
return; return;
} }
my $checkoutStop = time;
# Hash the arguments to hydra-eval-jobs and check the # Hash the arguments to hydra-eval-jobs and check the
# JobsetInputHashes to see if the previous evaluation had the same # JobsetInputHashes to see if the previous evaluation had the same
@ -124,6 +131,7 @@ sub checkJobsetWrapped {
my $prevEval = getPrevJobsetEval($db, $jobset, 0); my $prevEval = getPrevJobsetEval($db, $jobset, 0);
if (defined $prevEval && $prevEval->hash eq $argsHash && !$dryRun) { if (defined $prevEval && $prevEval->hash eq $argsHash && !$dryRun) {
print STDERR " jobset is unchanged, skipping\n"; print STDERR " jobset is unchanged, skipping\n";
Net::Statsd::increment("hydra.evaluator.unchanged_checkouts");
txn_do($db, sub { txn_do($db, sub {
$jobset->update({ lastcheckedtime => time, fetcherrormsg => undef }); $jobset->update({ lastcheckedtime => time, fetcherrormsg => undef });
}); });
@ -131,9 +139,11 @@ sub checkJobsetWrapped {
} }
# Evaluate the job expression. # Evaluate the job expression.
my $evalStart = time; my $evalStart = clock_gettime(CLOCK_REALTIME);
my ($jobs, $nixExprInput) = evalJobs($inputInfo, $exprType, $jobset->nixexprinput, $jobset->nixexprpath); my ($jobs, $nixExprInput) = evalJobs($inputInfo, $exprType, $jobset->nixexprinput, $jobset->nixexprpath);
my $evalStop = time; my $evalStop = clock_gettime(CLOCK_REALTIME);
Net::Statsd::timing("hydra.evaluator.eval_time", int(($evalStop - $evalStart) * 1000));
if ($dryRun) { if ($dryRun) {
foreach my $name (keys %{$jobs}) { foreach my $name (keys %{$jobs}) {
@ -150,6 +160,8 @@ sub checkJobsetWrapped {
$jobs->{$_}->{jobName} = $_ for keys %{$jobs}; $jobs->{$_}->{jobName} = $_ for keys %{$jobs};
my $jobOutPathMap = {}; my $jobOutPathMap = {};
my $jobsetChanged = 0;
my $dbStart = clock_gettime(CLOCK_REALTIME);
txn_do($db, sub { txn_do($db, sub {
@ -169,15 +181,15 @@ sub checkJobsetWrapped {
} }
# Have any builds been added or removed since last time? # Have any builds been added or removed since last time?
my $jobsetChanged = $jobsetChanged =
(scalar(grep { $_->{new} } values(%buildMap)) > 0) (scalar(grep { $_->{new} } values(%buildMap)) > 0)
|| (defined $prevEval && $prevEval->jobsetevalmembers->count != scalar(keys %buildMap)); || (defined $prevEval && $prevEval->jobsetevalmembers->count != scalar(keys %buildMap));
my $ev = $jobset->jobsetevals->create( my $ev = $jobset->jobsetevals->create(
{ hash => $argsHash { hash => $argsHash
, timestamp => time , timestamp => time
, checkouttime => abs($checkoutStop - $checkoutStart) , checkouttime => abs(int($checkoutStop - $checkoutStart))
, evaltime => abs($evalStop - $evalStart) , evaltime => abs(int($evalStop - $evalStart))
, hasnewbuilds => $jobsetChanged ? 1 : 0 , hasnewbuilds => $jobsetChanged ? 1 : 0
, nrbuilds => $jobsetChanged ? scalar(keys %buildMap) : undef , nrbuilds => $jobsetChanged ? scalar(keys %buildMap) : undef
}); });
@ -245,6 +257,12 @@ sub checkJobsetWrapped {
$jobset->update({ lastcheckedtime => time }); $jobset->update({ lastcheckedtime => time });
}); });
my $dbStop = clock_gettime(CLOCK_REALTIME);
Net::Statsd::timing("hydra.evaluator.db_time", int(($dbStop - $dbStart) * 1000));
Net::Statsd::increment("hydra.evaluator.evals");
Net::Statsd::increment("hydra.evaluator.cached_evals") unless $jobsetChanged;
# Store the error messages for jobs that failed to evaluate. # Store the error messages for jobs that failed to evaluate.
my $msg = ""; my $msg = "";
foreach my $job (values %{$jobs}) { foreach my $job (values %{$jobs}) {
@ -267,10 +285,15 @@ sub checkJobset {
my $triggerTime = $jobset->triggertime; my $triggerTime = $jobset->triggertime;
my $startTime = clock_gettime(CLOCK_REALTIME);
eval { eval {
checkJobsetWrapped($jobset); checkJobsetWrapped($jobset);
}; };
my $stopTime = clock_gettime(CLOCK_REALTIME);
Net::Statsd::timing("hydra.evaluator.total_time", int(($stopTime - $startTime) * 1000));
my $failed = 0; my $failed = 0;
if ($@) { if ($@) {
my $msg = $@; my $msg = $@;