More stats

This commit is contained in:
Eelco Dolstra 2015-06-24 13:19:16 +02:00
parent 3f8891b6ff
commit 1a0e1eb5a0
4 changed files with 30 additions and 7 deletions

View file

@ -128,8 +128,8 @@ void buildRemote(std::shared_ptr<StoreAPI> store,
const string & sshName, const string & sshKey, const string & sshName, const string & sshKey,
const Path & drvPath, const Derivation & drv, const Path & drvPath, const Derivation & drv,
const nix::Path & logDir, unsigned int maxSilentTime, unsigned int buildTimeout, const nix::Path & logDir, unsigned int maxSilentTime, unsigned int buildTimeout,
TokenServer & copyClosureTokenServer, TokenServer & copyClosureTokenServer, RemoteResult & result,
RemoteResult & result, counter & nrStepsBuilding) counter & nrStepsBuilding, counter & nrStepsCopyingTo, counter & nrStepsCopyingFrom)
{ {
string base = baseNameOf(drvPath); string base = baseNameOf(drvPath);
result.logFile = logDir + "/" + string(base, 0, 2) + "/" + string(base, 2); result.logFile = logDir + "/" + string(base, 0, 2) + "/" + string(base, 2);
@ -178,7 +178,10 @@ void buildRemote(std::shared_ptr<StoreAPI> store,
/* Copy the input closure. */ /* Copy the input closure. */
printMsg(lvlDebug, format("sending closure of %1% to %2%") % drvPath % sshName); printMsg(lvlDebug, format("sending closure of %1% to %2%") % drvPath % sshName);
copyClosureTo(store, from, to, inputs, copyClosureTokenServer); {
MaintainCount mc(nrStepsCopyingTo);
copyClosureTo(store, from, to, inputs, copyClosureTokenServer);
}
autoDelete.cancel(); autoDelete.cancel();
@ -210,7 +213,10 @@ void buildRemote(std::shared_ptr<StoreAPI> store,
PathSet outputs; PathSet outputs;
for (auto & output : drv.outputs) for (auto & output : drv.outputs)
outputs.insert(output.second.path); outputs.insert(output.second.path);
copyClosureFrom(store, from, to, outputs); {
MaintainCount mc(nrStepsCopyingFrom);
copyClosureFrom(store, from, to, outputs);
}
/* Shut down the connection. */ /* Shut down the connection. */
child.to.close(); child.to.close();

View file

@ -23,5 +23,5 @@ void buildRemote(std::shared_ptr<nix::StoreAPI> store,
const std::string & sshName, const std::string & sshKey, const std::string & sshName, const std::string & sshKey,
const nix::Path & drvPath, const nix::Derivation & drv, const nix::Path & drvPath, const nix::Derivation & drv,
const nix::Path & logDir, unsigned int maxSilentTime, unsigned int buildTimeout, const nix::Path & logDir, unsigned int maxSilentTime, unsigned int buildTimeout,
TokenServer & copyClosureTokenServer, TokenServer & copyClosureTokenServer, RemoteResult & result,
RemoteResult & result, counter & nrStepsBuilding); counter & nrStepsBuilding, counter & nrStepsCopyingTo, counter & nrStepsCopyingFrom);

View file

@ -257,6 +257,8 @@ private:
counter nrStepsDone{0}; counter nrStepsDone{0};
counter nrActiveSteps{0}; counter nrActiveSteps{0};
counter nrStepsBuilding{0}; counter nrStepsBuilding{0};
counter nrStepsCopyingTo{0};
counter nrStepsCopyingFrom{0};
counter nrRetries{0}; counter nrRetries{0};
counter maxNrRetries{0}; counter maxNrRetries{0};
counter totalStepTime{0}; // total time for steps, including closure copying counter totalStepTime{0}; // total time for steps, including closure copying
@ -1114,7 +1116,7 @@ bool State::doBuildStep(std::shared_ptr<StoreAPI> store, Step::ptr step,
/* FIXME: referring builds may have conflicting timeouts. */ /* FIXME: referring builds may have conflicting timeouts. */
buildRemote(store, machine->sshName, machine->sshKey, step->drvPath, step->drv, buildRemote(store, machine->sshName, machine->sshKey, step->drvPath, step->drv,
logDir, build->maxSilentTime, build->buildTimeout, copyClosureTokenServer, logDir, build->maxSilentTime, build->buildTimeout, copyClosureTokenServer,
result, nrStepsBuilding); result, nrStepsBuilding, nrStepsCopyingTo, nrStepsCopyingFrom);
} catch (Error & e) { } catch (Error & e) {
result.status = RemoteResult::rrMiscFailure; result.status = RemoteResult::rrMiscFailure;
result.errorMsg = e.msg(); result.errorMsg = e.msg();
@ -1542,6 +1544,8 @@ void State::dumpStatus(Connection & conn, bool log)
} }
root.attr("nrActiveSteps", nrActiveSteps); root.attr("nrActiveSteps", nrActiveSteps);
root.attr("nrStepsBuilding", nrStepsBuilding); root.attr("nrStepsBuilding", nrStepsBuilding);
root.attr("nrStepsCopyingTo", nrStepsCopyingTo);
root.attr("nrStepsCopyingFrom", nrStepsCopyingFrom);
root.attr("nrBuildsRead", nrBuildsRead); root.attr("nrBuildsRead", nrBuildsRead);
root.attr("nrBuildsDone", nrBuildsDone); root.attr("nrBuildsDone", nrBuildsDone);
root.attr("nrStepsDone", nrStepsDone); root.attr("nrStepsDone", nrStepsDone);

View file

@ -3,6 +3,7 @@
use strict; use strict;
use utf8; use utf8;
use Net::Statsd; use Net::Statsd;
use File::Slurp;
use JSON; use JSON;
STDERR->autoflush(1); STDERR->autoflush(1);
@ -20,10 +21,14 @@ sub sendQueueRunnerStats {
my $json = decode_json($s) or die "cannot decode queue runner status"; my $json = decode_json($s) or die "cannot decode queue runner status";
gauge("hydra.queue.up", $json->{status} eq "up" ? 1 : 0);
return if $json->{status} ne "up"; return if $json->{status} ne "up";
gauge("hydra.queue.steps.active", $json->{nrActiveSteps}); gauge("hydra.queue.steps.active", $json->{nrActiveSteps});
gauge("hydra.queue.steps.building", $json->{nrStepsBuilding}); gauge("hydra.queue.steps.building", $json->{nrStepsBuilding});
gauge("hydra.queue.steps.copying_to", $json->{nrStepsCopyingTo});
gauge("hydra.queue.steps.copying_from", $json->{nrStepsCopyingFrom});
gauge("hydra.queue.steps.runnable", $json->{nrRunnableSteps}); gauge("hydra.queue.steps.runnable", $json->{nrRunnableSteps});
gauge("hydra.queue.steps.unfinished", $json->{nrUnfinishedSteps}); gauge("hydra.queue.steps.unfinished", $json->{nrUnfinishedSteps});
gauge("hydra.queue.steps.finished", $json->{nrStepsDone}); gauge("hydra.queue.steps.finished", $json->{nrStepsDone});
@ -44,5 +49,13 @@ sub sendQueueRunnerStats {
while (1) { while (1) {
eval { sendQueueRunnerStats(); }; eval { sendQueueRunnerStats(); };
if ($@) { warn "$@"; } if ($@) { warn "$@"; }
my $meminfo = read_file("/proc/meminfo", err_mode => 'quiet') // "";
$meminfo =~ m/Dirty:\s*(\d+) kB/;
if (defined $1) {
my $dirty = $1 / (1024.0 * 1024.0);
gauge("hydra.mem.dirty", $dirty);
}
sleep(30); sleep(30);
} }