From 1a0e1eb5a007588b1cbaacd8ccb1921ddfa2df77 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Jun 2015 13:19:16 +0200 Subject: [PATCH] More stats --- src/hydra-queue-runner/build-remote.cc | 14 ++++++++++---- src/hydra-queue-runner/build-remote.hh | 4 ++-- src/hydra-queue-runner/hydra-queue-runner.cc | 6 +++++- src/script/hydra-send-stats | 13 +++++++++++++ 4 files changed, 30 insertions(+), 7 deletions(-) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index befb13fa..73cf294c 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -128,8 +128,8 @@ void buildRemote(std::shared_ptr store, const string & sshName, const string & sshKey, const Path & drvPath, const Derivation & drv, const nix::Path & logDir, unsigned int maxSilentTime, unsigned int buildTimeout, - TokenServer & copyClosureTokenServer, - RemoteResult & result, counter & nrStepsBuilding) + TokenServer & copyClosureTokenServer, RemoteResult & result, + counter & nrStepsBuilding, counter & nrStepsCopyingTo, counter & nrStepsCopyingFrom) { string base = baseNameOf(drvPath); result.logFile = logDir + "/" + string(base, 0, 2) + "/" + string(base, 2); @@ -178,7 +178,10 @@ void buildRemote(std::shared_ptr store, /* Copy the input closure. */ printMsg(lvlDebug, format("sending closure of ‘%1%’ to ‘%2%’") % drvPath % sshName); - copyClosureTo(store, from, to, inputs, copyClosureTokenServer); + { + MaintainCount mc(nrStepsCopyingTo); + copyClosureTo(store, from, to, inputs, copyClosureTokenServer); + } autoDelete.cancel(); @@ -210,7 +213,10 @@ void buildRemote(std::shared_ptr store, PathSet outputs; for (auto & output : drv.outputs) outputs.insert(output.second.path); - copyClosureFrom(store, from, to, outputs); + { + MaintainCount mc(nrStepsCopyingFrom); + copyClosureFrom(store, from, to, outputs); + } /* Shut down the connection. */ child.to.close(); diff --git a/src/hydra-queue-runner/build-remote.hh b/src/hydra-queue-runner/build-remote.hh index 86ec767f..19625b19 100644 --- a/src/hydra-queue-runner/build-remote.hh +++ b/src/hydra-queue-runner/build-remote.hh @@ -23,5 +23,5 @@ void buildRemote(std::shared_ptr store, const std::string & sshName, const std::string & sshKey, const nix::Path & drvPath, const nix::Derivation & drv, const nix::Path & logDir, unsigned int maxSilentTime, unsigned int buildTimeout, - TokenServer & copyClosureTokenServer, - RemoteResult & result, counter & nrStepsBuilding); + TokenServer & copyClosureTokenServer, RemoteResult & result, + counter & nrStepsBuilding, counter & nrStepsCopyingTo, counter & nrStepsCopyingFrom); diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index c4ddee89..f6ba6179 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -257,6 +257,8 @@ private: counter nrStepsDone{0}; counter nrActiveSteps{0}; counter nrStepsBuilding{0}; + counter nrStepsCopyingTo{0}; + counter nrStepsCopyingFrom{0}; counter nrRetries{0}; counter maxNrRetries{0}; counter totalStepTime{0}; // total time for steps, including closure copying @@ -1114,7 +1116,7 @@ bool State::doBuildStep(std::shared_ptr store, Step::ptr step, /* FIXME: referring builds may have conflicting timeouts. */ buildRemote(store, machine->sshName, machine->sshKey, step->drvPath, step->drv, logDir, build->maxSilentTime, build->buildTimeout, copyClosureTokenServer, - result, nrStepsBuilding); + result, nrStepsBuilding, nrStepsCopyingTo, nrStepsCopyingFrom); } catch (Error & e) { result.status = RemoteResult::rrMiscFailure; result.errorMsg = e.msg(); @@ -1542,6 +1544,8 @@ void State::dumpStatus(Connection & conn, bool log) } root.attr("nrActiveSteps", nrActiveSteps); root.attr("nrStepsBuilding", nrStepsBuilding); + root.attr("nrStepsCopyingTo", nrStepsCopyingTo); + root.attr("nrStepsCopyingFrom", nrStepsCopyingFrom); root.attr("nrBuildsRead", nrBuildsRead); root.attr("nrBuildsDone", nrBuildsDone); root.attr("nrStepsDone", nrStepsDone); diff --git a/src/script/hydra-send-stats b/src/script/hydra-send-stats index dd893d77..b3e28968 100755 --- a/src/script/hydra-send-stats +++ b/src/script/hydra-send-stats @@ -3,6 +3,7 @@ use strict; use utf8; use Net::Statsd; +use File::Slurp; use JSON; STDERR->autoflush(1); @@ -20,10 +21,14 @@ sub sendQueueRunnerStats { my $json = decode_json($s) or die "cannot decode queue runner status"; + gauge("hydra.queue.up", $json->{status} eq "up" ? 1 : 0); + return if $json->{status} ne "up"; gauge("hydra.queue.steps.active", $json->{nrActiveSteps}); gauge("hydra.queue.steps.building", $json->{nrStepsBuilding}); + gauge("hydra.queue.steps.copying_to", $json->{nrStepsCopyingTo}); + gauge("hydra.queue.steps.copying_from", $json->{nrStepsCopyingFrom}); gauge("hydra.queue.steps.runnable", $json->{nrRunnableSteps}); gauge("hydra.queue.steps.unfinished", $json->{nrUnfinishedSteps}); gauge("hydra.queue.steps.finished", $json->{nrStepsDone}); @@ -44,5 +49,13 @@ sub sendQueueRunnerStats { while (1) { eval { sendQueueRunnerStats(); }; if ($@) { warn "$@"; } + + my $meminfo = read_file("/proc/meminfo", err_mode => 'quiet') // ""; + $meminfo =~ m/Dirty:\s*(\d+) kB/; + if (defined $1) { + my $dirty = $1 / (1024.0 * 1024.0); + gauge("hydra.mem.dirty", $dirty); + } + sleep(30); }