hydra queue runner: instrument the process of loading new builds with prom

This commit is contained in:
Graham Christensen 2022-04-06 20:18:29 -04:00
parent 5de08d412e
commit 1c12c5882f
3 changed files with 44 additions and 2 deletions

View file

@ -47,6 +47,34 @@ State::PromMetrics::PromMetrics()
.Register(*registry) .Register(*registry)
.Add({}) .Add({})
) )
, queue_build_loads(
prometheus::BuildCounter()
.Name("hydraqueuerunner_queue_build_loads_total")
.Help("Number of builds loaded")
.Register(*registry)
.Add({})
)
, queue_checks_early_exits(
prometheus::BuildCounter()
.Name("hydraqueuerunner_queue_checks_early_exits_total")
.Help("Number of times State::getQueuedBuilds() yielded to potential bumps")
.Register(*registry)
.Add({})
)
, queue_checks_finished(
prometheus::BuildCounter()
.Name("hydraqueuerunner_queue_checks_finished_total")
.Help("Number of times State::getQueuedBuilds() was completed")
.Register(*registry)
.Add({})
)
, queue_max_id(
prometheus::BuildGauge()
.Name("hydraqueuerunner_queue_max_build_id_info")
.Help("Maximum build record ID in the queue")
.Register(*registry)
.Add({})
)
{ {
} }

View file

@ -109,7 +109,10 @@ bool State::getQueuedBuilds(Connection & conn,
auto builds_(builds.lock()); auto builds_(builds.lock());
BuildID id = row["id"].as<BuildID>(); BuildID id = row["id"].as<BuildID>();
if (buildOne && id != buildOne) continue; if (buildOne && id != buildOne) continue;
if (id > newLastBuildId) newLastBuildId = id; if (id > newLastBuildId) {
newLastBuildId = id;
prom.queue_max_id.Set(id);
}
if (builds_->count(id)) continue; if (builds_->count(id)) continue;
auto build = std::make_shared<Build>( auto build = std::make_shared<Build>(
@ -138,6 +141,7 @@ bool State::getQueuedBuilds(Connection & conn,
std::set<StorePath> finishedDrvs; std::set<StorePath> finishedDrvs;
createBuild = [&](Build::ptr build) { createBuild = [&](Build::ptr build) {
prom.queue_build_loads.Increment();
printMsg(lvlTalkative, format("loading build %1% (%2%)") % build->id % build->fullJobName()); printMsg(lvlTalkative, format("loading build %1% (%2%)") % build->id % build->fullJobName());
nrAdded++; nrAdded++;
newBuildsByID.erase(build->id); newBuildsByID.erase(build->id);
@ -308,9 +312,14 @@ bool State::getQueuedBuilds(Connection & conn,
/* Stop after a certain time to allow priority bumps to be /* Stop after a certain time to allow priority bumps to be
processed. */ processed. */
if (std::chrono::system_clock::now() > start + std::chrono::seconds(600)) break; if (std::chrono::system_clock::now() > start + std::chrono::seconds(600)) {
prom.queue_checks_early_exits.Increment();
break;
}
} }
prom.queue_checks_finished.Increment();
lastBuildId = newBuildsByID.empty() ? newLastBuildId : newBuildsByID.begin()->first - 1; lastBuildId = newBuildsByID.empty() ? newLastBuildId : newBuildsByID.begin()->first - 1;
return newBuildsByID.empty(); return newBuildsByID.empty();
} }

View file

@ -8,6 +8,7 @@
#include <queue> #include <queue>
#include <prometheus/counter.h> #include <prometheus/counter.h>
#include <prometheus/gauge.h>
#include <prometheus/registry.h> #include <prometheus/registry.h>
#include "db.hh" #include "db.hh"
@ -442,6 +443,10 @@ private:
std::shared_ptr<prometheus::Registry> registry; std::shared_ptr<prometheus::Registry> registry;
prometheus::Counter& queue_checks_started; prometheus::Counter& queue_checks_started;
prometheus::Counter& queue_build_loads;
prometheus::Counter& queue_checks_early_exits;
prometheus::Counter& queue_checks_finished;
prometheus::Gauge& queue_max_id;
PromMetrics(); PromMetrics();
}; };