ma27/hydra
0
0
Fork 0
forked from lix-project/hydra
Code Pull requests Activity

hydra queue runner: instrument the process of loading new builds with prom

Browse source
This commit is contained in:
Graham Christensen 2022-04-06 20:18:29 -04:00
parent 5de08d412e
commit 1c12c5882f
3 changed files with 44 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
src/hydra-queue-runner/hydra-queue-runner.cc
View file

@ -47,6 +47,34 @@ State::PromMetrics::PromMetrics()
.Register(*registry)
.Add({})
)
, queue_build_loads(
prometheus::BuildCounter()
.Name("hydraqueuerunner_queue_build_loads_total")
.Help("Number of builds loaded")
.Register(*registry)
.Add({})
)
, queue_checks_early_exits(
prometheus::BuildCounter()
.Name("hydraqueuerunner_queue_checks_early_exits_total")
.Help("Number of times State::getQueuedBuilds() yielded to potential bumps")
.Register(*registry)
.Add({})
)
, queue_checks_finished(
prometheus::BuildCounter()
.Name("hydraqueuerunner_queue_checks_finished_total")
.Help("Number of times State::getQueuedBuilds() was completed")
.Register(*registry)
.Add({})
)
, queue_max_id(
prometheus::BuildGauge()
.Name("hydraqueuerunner_queue_max_build_id_info")
.Help("Maximum build record ID in the queue")
.Register(*registry)
.Add({})
)
{
}

13
src/hydra-queue-runner/queue-monitor.cc
View file

@ -109,7 +109,10 @@ bool State::getQueuedBuilds(Connection & conn,
auto builds_(builds.lock());
BuildID id = row["id"].as<BuildID>();
if (buildOne && id != buildOne) continue;
if (id > newLastBuildId) newLastBuildId = id;
if (id > newLastBuildId) {
newLastBuildId = id;
prom.queue_max_id.Set(id);
}
if (builds_->count(id)) continue;
auto build = std::make_shared<Build>(
@ -138,6 +141,7 @@ bool State::getQueuedBuilds(Connection & conn,
std::set<StorePath> finishedDrvs;
createBuild = [&](Build::ptr build) {
prom.queue_build_loads.Increment();
printMsg(lvlTalkative, format("loading build %1% (%2%)") % build->id % build->fullJobName());
nrAdded++;
newBuildsByID.erase(build->id);
@ -308,8 +312,13 @@ bool State::getQueuedBuilds(Connection & conn,
/* Stop after a certain time to allow priority bumps to be
processed. */
if (std::chrono::system_clock::now() > start + std::chrono::seconds(600)) break;
if (std::chrono::system_clock::now() > start + std::chrono::seconds(600)) {
prom.queue_checks_early_exits.Increment();
break;
}
}
prom.queue_checks_finished.Increment();
lastBuildId = newBuildsByID.empty() ? newLastBuildId : newBuildsByID.begin()->first - 1;
return newBuildsByID.empty();

5
src/hydra-queue-runner/state.hh
View file

@ -8,6 +8,7 @@
#include <queue>
#include <prometheus/counter.h>
#include <prometheus/gauge.h>
#include <prometheus/registry.h>
#include "db.hh"
@ -442,6 +443,10 @@ private:
std::shared_ptr<prometheus::Registry> registry;
prometheus::Counter& queue_checks_started;
prometheus::Counter& queue_build_loads;
prometheus::Counter& queue_checks_early_exits;
prometheus::Counter& queue_checks_finished;
prometheus::Gauge& queue_max_id;
PromMetrics();
};