ma27/hydra
0
0
Fork 0
forked from lix-project/hydra
Code Pull requests Activity

Merge pull request #1173 from DeterminateSystems/queue-runner-exporter

Browse source 
hydra-queue-runner metrics
This commit is contained in:
Graham Christensen 2022-04-07 12:27:33 -04:00 committed by GitHub
parent 8cbf065468 f8dc48f171
commit e1965250b5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 175 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
doc/manual/src/configuration.md
View file

@ -102,6 +102,22 @@ in the hydra configuration file, as below:
</hydra_notify>
```
hydra-queue-runner's Prometheus service
---------------------------------------
hydra-queue-runner supports running a Prometheus webserver for metrics. The
exporter's address defaults to exposing on `127.0.0.1:9198`, but is also
configurable through the hydra configuration file and a command line argument,
as below. A port of `:0` will make the exposer choose a random, available port.
```conf
queue_runner_exporter_address = 127.0.0.1:9198
```
```shell
$ hydra-queue-runner --prometheus-address 127.0.0.1:9198
```
Using LDAP as authentication backend (optional)
-----------------------------------------------

17
flake.lock
View file

@ -16,6 +16,22 @@
"type": "github"
}
},
"newNixpkgs": {
"locked": {
"lastModified": 1647380550,
"narHash": "sha256-909TI9poX7CIUiFx203WL29YON6m/I6k0ExbZvR7bLM=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "6e3ee8957637a60f5072e33d78e05c0f65c54366",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-unstable-small",
"repo": "nixpkgs",
"type": "github"
}
},
"nix": {
"inputs": {
"lowdown-src": "lowdown-src",
@ -67,6 +83,7 @@
},
"root": {
"inputs": {
"newNixpkgs": "newNixpkgs",
"nix": "nix",
"nixpkgs": [
"nix",

46
flake.nix
View file

@ -1,10 +1,13 @@
{
description = "A Nix-based continuous build system";
# FIXME: All the pinned versions of nix/nixpkgs have a broken foreman (yes,
# even 2.7.0's Nixpkgs pin).
inputs.newNixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable-small";
inputs.nixpkgs.follows = "nix/nixpkgs";
#inputs.nix.url = github:NixOS/nix/2.7.0;
outputs = { self, nixpkgs, nix }:
outputs = { self, newNixpkgs, nixpkgs, nix }:
let
version = "${builtins.readFile ./version.txt}.${builtins.substring 0 8 (self.lastModifiedDate or "19700101")}.${self.shortRev or "DIRTY"}";
@ -38,6 +41,12 @@
# A Nixpkgs overlay that provides a 'hydra' package.
overlay = final: prev: {
# Overlay these packages to use dependencies from the Nixpkgs everything
# else uses, to side-step the version difference: glibc is 2.32 in the
# nix-pinned Nixpkgs, but 2.33 in the newNixpkgs commit.
civetweb = final.callPackage "${newNixpkgs}/pkgs/development/libraries/civetweb" { };
prometheus-cpp = final.callPackage "${newNixpkgs}/pkgs/development/libraries/prometheus-cpp" { };
# Add LDAP dependencies that aren't currently found within nixpkgs.
perlPackages = prev.perlPackages // {
TestPostgreSQL = final.perlPackages.buildPerlModule {
@ -61,7 +70,7 @@
};
};
FunctionParameters = final.buildPerlPackage {
FunctionParameters = final.perlPackages.buildPerlPackage {
pname = "Function-Parameters";
version = "2.001003";
src = final.fetchurl {
@ -75,7 +84,7 @@
};
};
CatalystPluginPrometheusTiny = final.buildPerlPackage {
CatalystPluginPrometheusTiny = final.perlPackages.buildPerlPackage {
pname = "Catalyst-Plugin-PrometheusTiny";
version = "0.005";
src = final.fetchurl {
@ -104,7 +113,7 @@
};
};
CryptPassphrase = final.buildPerlPackage {
CryptPassphrase = final.perlPackages.buildPerlPackage {
pname = "Crypt-Passphrase";
version = "0.003";
src = final.fetchurl {
@ -117,7 +126,7 @@
};
};
CryptPassphraseArgon2 = final.buildPerlPackage {
CryptPassphraseArgon2 = final.perlPackages.buildPerlPackage {
pname = "Crypt-Passphrase-Argon2";
version = "0.002";
src = final.fetchurl {
@ -131,7 +140,7 @@
};
};
DataRandom = final.buildPerlPackage {
DataRandom = final.perlPackages.buildPerlPackage {
pname = "Data-Random";
version = "0.13";
src = final.fetchurl {
@ -145,7 +154,7 @@
};
};
DirSelf = final.buildPerlPackage {
DirSelf = final.perlPackages.buildPerlPackage {
pname = "Dir-Self";
version = "0.11";
src = final.fetchurl {
@ -173,7 +182,7 @@
};
};
PrometheusTiny = final.buildPerlPackage {
PrometheusTiny = final.perlPackages.buildPerlPackage {
pname = "Prometheus-Tiny";
version = "0.007";
src = final.fetchurl {
@ -188,7 +197,7 @@
};
};
PrometheusTinyShared = final.buildPerlPackage {
PrometheusTinyShared = final.perlPackages.buildPerlPackage {
pname = "Prometheus-Tiny-Shared";
version = "0.023";
src = final.fetchurl {
@ -219,7 +228,7 @@
};
};
TieHashMethod = final.buildPerlPackage {
TieHashMethod = final.perlPackages.buildPerlPackage {
pname = "Tie-Hash-Method";
version = "0.02";
src = final.fetchurl {
@ -232,7 +241,7 @@
};
};
Test2Harness = final.buildPerlPackage {
Test2Harness = final.perlPackages.buildPerlPackage {
pname = "Test2-Harness";
version = "1.000042";
src = final.fetchurl {
@ -281,7 +290,7 @@
};
};
LongJump = final.buildPerlPackage {
LongJump = final.perlPackages.buildPerlPackage {
pname = "Long-Jump";
version = "0.000001";
src = final.fetchurl {
@ -295,7 +304,7 @@
};
};
gotofile = final.buildPerlPackage {
gotofile = final.perlPackages.buildPerlPackage {
pname = "goto-file";
version = "0.005";
src = final.fetchurl {
@ -425,7 +434,7 @@
};
};
StringCompareConstantTime = final.buildPerlPackage {
StringCompareConstantTime = final.perlPackages.buildPerlPackage {
pname = "String-Compare-ConstantTime";
version = "0.321";
src = final.fetchurl {
@ -438,7 +447,7 @@
};
};
UUID4Tiny = final.buildPerlPackage {
UUID4Tiny = final.perlPackages.buildPerlPackage {
pname = "UUID4-Tiny";
version = "0.002";
src = final.fetchurl {
@ -562,13 +571,16 @@
(if lib.versionAtLeast lib.version "20.03pre"
then nlohmann_json
else nlohmann_json.override { multipleHeaders = true; })
prometheus-cpp
];
checkInputs = [
cacert
foreman
# FIXME: foreman is broken on all nix/nixpkgs pin, up to and
# including 2.7.0
newNixpkgs.legacyPackages.${final.system}.foreman
glibcLocales
netcat-openbsd
libressl.nc
openldap
python3
];

2
src/hydra-queue-runner/Makefile.am
View file

@ -4,5 +4,5 @@ hydra_queue_runner_SOURCES = hydra-queue-runner.cc queue-monitor.cc dispatcher.c
builder.cc build-result.cc build-remote.cc \
hydra-build-result.hh counter.hh state.hh db.hh \
nar-extractor.cc nar-extractor.hh
hydra_queue_runner_LDADD = $(NIX_LIBS) -lpqxx
hydra_queue_runner_LDADD = $(NIX_LIBS) -lpqxx -lprometheus-cpp-pull -lprometheus-cpp-core
hydra_queue_runner_CXXFLAGS = $(NIX_CFLAGS) -Wall -I ../libhydra -Wno-deprecated-declarations

73
src/hydra-queue-runner/hydra-queue-runner.cc
View file

@ -6,6 +6,8 @@
#include <sys/stat.h>
#include <fcntl.h>
#include <prometheus/exposer.h>
#include "state.hh"
#include "hydra-build-result.hh"
#include "store-api.hh"
@ -36,8 +38,55 @@ std::string getEnvOrDie(const std::string & key)
return *value;
}
State::PromMetrics::PromMetrics()
: registry(std::make_shared<prometheus::Registry>())
, queue_checks_started(
prometheus::BuildCounter()
.Name("hydraqueuerunner_queue_checks_started_total")
.Help("Number of times State::getQueuedBuilds() was started")
.Register(*registry)
.Add({})
)
, queue_build_loads(
prometheus::BuildCounter()
.Name("hydraqueuerunner_queue_build_loads_total")
.Help("Number of builds loaded")
.Register(*registry)
.Add({})
)
, queue_steps_created(
prometheus::BuildCounter()
.Name("hydraqueuerunner_queue_steps_created_total")
.Help("Number of steps created")
.Register(*registry)
.Add({})
)
, queue_checks_early_exits(
prometheus::BuildCounter()
.Name("hydraqueuerunner_queue_checks_early_exits_total")
.Help("Number of times State::getQueuedBuilds() yielded to potential bumps")
.Register(*registry)
.Add({})
)
, queue_checks_finished(
prometheus::BuildCounter()
.Name("hydraqueuerunner_queue_checks_finished_total")
.Help("Number of times State::getQueuedBuilds() was completed")
.Register(*registry)
.Add({})
)
, queue_max_id(
prometheus::BuildGauge()
.Name("hydraqueuerunner_queue_max_build_id_info")
.Help("Maximum build record ID in the queue")
.Register(*registry)
.Add({})
)
{
State::State()
}
State::State(std::optional<std::string> metricsAddrOpt)
: config(std::make_unique<HydraConfig>())
, maxUnsupportedTime(config->getIntOption("max_unsupported_time", 0))
, dbPool(config->getIntOption("max_db_connections", 128))
@ -45,11 +94,16 @@ State::State()
, maxLogSize(config->getIntOption("max_log_size", 64ULL << 20))
, uploadLogsToBinaryCache(config->getBoolOption("upload_logs_to_binary_cache", false))
, rootsDir(config->getStrOption("gc_roots_dir", fmt("%s/gcroots/per-user/%s/hydra-roots", settings.nixStateDir, getEnvOrDie("LOGNAME"))))
, metricsAddr(config->getStrOption("queue_runner_metrics_address", std::string{"127.0.0.1:9198"}))
{
hydraData = getEnvOrDie("HYDRA_DATA");
logDir = canonPath(hydraData + "/build-logs");
if (metricsAddrOpt.has_value()) {
metricsAddr = metricsAddrOpt.value();
}
/* handle deprecated store specification */
if (config->getStrOption("store_mode") != "")
throw Error("store_mode in hydra.conf is deprecated, please use store_uri");
@ -754,6 +808,18 @@ void State::run(BuildID buildOne)
if (!lock)
throw Error("hydra-queue-runner is already running");
std::cout << "Starting the Prometheus exporter on " << metricsAddr << std::endl;
/* Set up simple exporter, to show that we're still alive. */
prometheus::Exposer promExposer{metricsAddr};
auto exposerPort = promExposer.GetListeningPorts().front();
promExposer.RegisterCollectable(prom.registry);
std::cout << "Started the Prometheus exporter, listening on "
<< metricsAddr << "/metrics (port " << exposerPort << ")"
<< std::endl;
Store::Params localParams;
localParams["max-connections"] = "16";
localParams["max-connection-age"] = "600";
@ -864,6 +930,7 @@ int main(int argc, char * * argv)
bool unlock = false;
bool status = false;
BuildID buildOne = 0;
std::optional<std::string> metricsAddrOpt = std::nullopt;
parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) {
if (*arg == "--unlock")
@ -875,6 +942,8 @@ int main(int argc, char * * argv)
buildOne = *b;
else
throw Error("‘--build-one’ requires a build ID");
} else if (*arg == "--prometheus-address") {
metricsAddrOpt = getArg(*arg, arg, end);
} else
return false;
return true;
@ -883,7 +952,7 @@ int main(int argc, char * * argv)
settings.verboseBuild = true;
settings.lockCPU = false;
State state;
State state{metricsAddrOpt};
if (status)
state.showStatus();
else if (unlock)

17
src/hydra-queue-runner/queue-monitor.cc
View file

@ -82,6 +82,8 @@ struct PreviousFailure : public std::exception {
bool State::getQueuedBuilds(Connection & conn,
ref<Store> destStore, unsigned int & lastBuildId)
{
prom.queue_checks_started.Increment();
printInfo("checking the queue for builds > %d...", lastBuildId);
/* Grab the queued builds from the database, but don't process
@ -107,7 +109,10 @@ bool State::getQueuedBuilds(Connection & conn,
auto builds_(builds.lock());
BuildID id = row["id"].as<BuildID>();
if (buildOne && id != buildOne) continue;
if (id > newLastBuildId) newLastBuildId = id;
if (id > newLastBuildId) {
newLastBuildId = id;
prom.queue_max_id.Set(id);
}
if (builds_->count(id)) continue;
auto build = std::make_shared<Build>(
@ -136,6 +141,7 @@ bool State::getQueuedBuilds(Connection & conn,
std::set<StorePath> finishedDrvs;
createBuild = [&](Build::ptr build) {
prom.queue_build_loads.Increment();
printMsg(lvlTalkative, format("loading build %1% (%2%)") % build->id % build->fullJobName());
nrAdded++;
newBuildsByID.erase(build->id);
@ -306,9 +312,14 @@ bool State::getQueuedBuilds(Connection & conn,
/* Stop after a certain time to allow priority bumps to be
processed. */
if (std::chrono::system_clock::now() > start + std::chrono::seconds(600)) break;
if (std::chrono::system_clock::now() > start + std::chrono::seconds(600)) {
prom.queue_checks_early_exits.Increment();
break;
}
}
prom.queue_checks_finished.Increment();
lastBuildId = newBuildsByID.empty() ? newLastBuildId : newBuildsByID.begin()->first - 1;
return newBuildsByID.empty();
}
@ -437,6 +448,8 @@ Step::ptr State::createStep(ref<Store> destStore,
if (!isNew) return step;
prom.queue_steps_created.Increment();
printMsg(lvlDebug, "considering derivation ‘%1%’", localStore->printStorePath(drvPath));
/* Initialize the step. Note that the step may be visible in

25
src/hydra-queue-runner/state.hh
View file

@ -7,6 +7,10 @@
#include <memory>
#include <queue>
#include <prometheus/counter.h>
#include <prometheus/gauge.h>
#include <prometheus/registry.h>
#include "db.hh"
#include "parsed-derivations.hh"
@ -433,8 +437,25 @@ private:
via gc_roots_dir. */
nix::Path rootsDir;
std::string metricsAddr;
struct PromMetrics
{
std::shared_ptr<prometheus::Registry> registry;
prometheus::Counter& queue_checks_started;
prometheus::Counter& queue_build_loads;
prometheus::Counter& queue_steps_created;
prometheus::Counter& queue_checks_early_exits;
prometheus::Counter& queue_checks_finished;
prometheus::Gauge& queue_max_id;
PromMetrics();
};
PromMetrics prom;
public:
State();
State(std::optional<std::string> metricsAddrOpt);
private:
@ -544,6 +565,8 @@ private:
void addRoot(const nix::StorePath & storePath);
void runMetricsExporter();
public:
void showStatus();

1
t/Hydra/Config/include.t
View file

@ -20,6 +20,7 @@ write_file($ctx{'tmpdir'} . "/bar.conf", q|
|);
is(getHydraConfig(), {
queue_runner_metrics_address => "127.0.0.1:0",
foo => { bar => "baz" }
}, "Nested includes work.");

1
t/lib/HydraTestContext.pm
View file

@ -51,6 +51,7 @@ sub new {
$ENV{'HYDRA_CONFIG'} = "$dir/hydra.conf";
my $hydra_config = $opts{'hydra_config'} || "";
$hydra_config = "queue_runner_metrics_address = 127.0.0.1:0\n" . $hydra_config;
if ($opts{'use_external_destination_store'} // 1) {
$hydra_config = "store_uri = file:$dir/nix/dest-store\n" . $hydra_config;
}