More stats

src/hydra-queue-runner/build-remote.cc

@@ -128,8 +128,8 @@ void buildRemote(std::shared_ptr<StoreAPI> store,
     const string & sshName, const string & sshKey,
     const Path & drvPath, const Derivation & drv,
     const nix::Path & logDir, unsigned int maxSilentTime, unsigned int buildTimeout,
-    TokenServer & copyClosureTokenServer,
-    RemoteResult & result, counter & nrStepsBuilding)
+    TokenServer & copyClosureTokenServer, RemoteResult & result,
+    counter & nrStepsBuilding, counter & nrStepsCopyingTo, counter & nrStepsCopyingFrom)
 {
     string base = baseNameOf(drvPath);
     result.logFile = logDir + "/" + string(base, 0, 2) + "/" + string(base, 2);
@@ -178,7 +178,10 @@ void buildRemote(std::shared_ptr<StoreAPI> store,
 
     /* Copy the input closure. */
     printMsg(lvlDebug, format("sending closure of ‘%1%’ to ‘%2%’") % drvPath % sshName);
-    copyClosureTo(store, from, to, inputs, copyClosureTokenServer);
+    {
+        MaintainCount mc(nrStepsCopyingTo);
+        copyClosureTo(store, from, to, inputs, copyClosureTokenServer);
+    }
 
     autoDelete.cancel();
 
@@ -210,7 +213,10 @@ void buildRemote(std::shared_ptr<StoreAPI> store,
     PathSet outputs;
     for (auto & output : drv.outputs)
         outputs.insert(output.second.path);
-    copyClosureFrom(store, from, to, outputs);
+    {
+        MaintainCount mc(nrStepsCopyingFrom);
+        copyClosureFrom(store, from, to, outputs);
+    }
 
     /* Shut down the connection. */
     child.to.close();

src/hydra-queue-runner/build-remote.hh

@@ -23,5 +23,5 @@ void buildRemote(std::shared_ptr<nix::StoreAPI> store,
     const std::string & sshName, const std::string & sshKey,
     const nix::Path & drvPath, const nix::Derivation & drv,
     const nix::Path & logDir, unsigned int maxSilentTime, unsigned int buildTimeout,
-    TokenServer & copyClosureTokenServer,
-    RemoteResult & result, counter & nrStepsBuilding);
+    TokenServer & copyClosureTokenServer, RemoteResult & result,
+    counter & nrStepsBuilding, counter & nrStepsCopyingTo, counter & nrStepsCopyingFrom);

src/hydra-queue-runner/hydra-queue-runner.cc

@@ -257,6 +257,8 @@ private:
     counter nrStepsDone{0};
     counter nrActiveSteps{0};
     counter nrStepsBuilding{0};
+    counter nrStepsCopyingTo{0};
+    counter nrStepsCopyingFrom{0};
     counter nrRetries{0};
     counter maxNrRetries{0};
     counter totalStepTime{0}; // total time for steps, including closure copying
@@ -1114,7 +1116,7 @@ bool State::doBuildStep(std::shared_ptr<StoreAPI> store, Step::ptr step,
             /* FIXME: referring builds may have conflicting timeouts. */
             buildRemote(store, machine->sshName, machine->sshKey, step->drvPath, step->drv,
                 logDir, build->maxSilentTime, build->buildTimeout, copyClosureTokenServer,
-                result, nrStepsBuilding);
+                result, nrStepsBuilding, nrStepsCopyingTo, nrStepsCopyingFrom);
         } catch (Error & e) {
             result.status = RemoteResult::rrMiscFailure;
             result.errorMsg = e.msg();
@@ -1542,6 +1544,8 @@ void State::dumpStatus(Connection & conn, bool log)
         }
         root.attr("nrActiveSteps", nrActiveSteps);
         root.attr("nrStepsBuilding", nrStepsBuilding);
+        root.attr("nrStepsCopyingTo", nrStepsCopyingTo);
+        root.attr("nrStepsCopyingFrom", nrStepsCopyingFrom);
         root.attr("nrBuildsRead", nrBuildsRead);
         root.attr("nrBuildsDone", nrBuildsDone);
         root.attr("nrStepsDone", nrStepsDone);

src/script/hydra-send-stats

@@ -3,6 +3,7 @@
 use strict;
 use utf8;
 use Net::Statsd;
+use File::Slurp;
 use JSON;
 
 STDERR->autoflush(1);
@@ -20,10 +21,14 @@ sub sendQueueRunnerStats {
 
     my $json = decode_json($s) or die "cannot decode queue runner status";
 
+    gauge("hydra.queue.up", $json->{status} eq "up" ? 1 : 0);
+
     return if $json->{status} ne "up";
 
     gauge("hydra.queue.steps.active", $json->{nrActiveSteps});
     gauge("hydra.queue.steps.building", $json->{nrStepsBuilding});
+    gauge("hydra.queue.steps.copying_to", $json->{nrStepsCopyingTo});
+    gauge("hydra.queue.steps.copying_from", $json->{nrStepsCopyingFrom});
     gauge("hydra.queue.steps.runnable", $json->{nrRunnableSteps});
     gauge("hydra.queue.steps.unfinished", $json->{nrUnfinishedSteps});
     gauge("hydra.queue.steps.finished", $json->{nrStepsDone});
@@ -44,5 +49,13 @@ sub sendQueueRunnerStats {
 while (1) {
     eval { sendQueueRunnerStats(); };
     if ($@) { warn "$@"; }
+
+    my $meminfo = read_file("/proc/meminfo", err_mode => 'quiet') // "";
+    $meminfo =~ m/Dirty:\s*(\d+) kB/;
+    if (defined $1) {
+        my $dirty = $1 / (1024.0 * 1024.0);
+        gauge("hydra.mem.dirty", $dirty);
+    }
+
     sleep(30);
 }