ahuston-0/hydra
1
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

Merge pull request #1173 from DeterminateSystems/queue-runner-exporter

Browse Source 
hydra-queue-runner metrics
This commit is contained in:
Graham Christensen
2022-04-07 12:27:33 -04:00
committed by GitHub
parent 8cbf065468 f8dc48f171
commit e1965250b5
9 changed files with 175 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/hydra-queue-runner/Makefile.am
View File

@ -4,5 +4,5 @@ hydra_queue_runner_SOURCES = hydra-queue-runner.cc queue-monitor.cc dispatcher.c
builder.cc build-result.cc build-remote.cc \
hydra-build-result.hh counter.hh state.hh db.hh \
nar-extractor.cc nar-extractor.hh
hydra_queue_runner_LDADD = $(NIX_LIBS) -lpqxx
hydra_queue_runner_LDADD = $(NIX_LIBS) -lpqxx -lprometheus-cpp-pull -lprometheus-cpp-core
hydra_queue_runner_CXXFLAGS = $(NIX_CFLAGS) -Wall -I ../libhydra -Wno-deprecated-declarations

73
src/hydra-queue-runner/hydra-queue-runner.cc
View File

@ -6,6 +6,8 @@
#include <sys/stat.h>
#include <fcntl.h>
#include <prometheus/exposer.h>
#include "state.hh"
#include "hydra-build-result.hh"
#include "store-api.hh"
@ -36,8 +38,55 @@ std::string getEnvOrDie(const std::string & key)
return *value;
}
State::PromMetrics::PromMetrics()
: registry(std::make_shared<prometheus::Registry>())
, queue_checks_started(
prometheus::BuildCounter()
.Name("hydraqueuerunner_queue_checks_started_total")
.Help("Number of times State::getQueuedBuilds() was started")
.Register(*registry)
.Add({})
)
, queue_build_loads(
prometheus::BuildCounter()
.Name("hydraqueuerunner_queue_build_loads_total")
.Help("Number of builds loaded")
.Register(*registry)
.Add({})
)
, queue_steps_created(
prometheus::BuildCounter()
.Name("hydraqueuerunner_queue_steps_created_total")
.Help("Number of steps created")
.Register(*registry)
.Add({})
)
, queue_checks_early_exits(
prometheus::BuildCounter()
.Name("hydraqueuerunner_queue_checks_early_exits_total")
.Help("Number of times State::getQueuedBuilds() yielded to potential bumps")
.Register(*registry)
.Add({})
)
, queue_checks_finished(
prometheus::BuildCounter()
.Name("hydraqueuerunner_queue_checks_finished_total")
.Help("Number of times State::getQueuedBuilds() was completed")
.Register(*registry)
.Add({})
)
, queue_max_id(
prometheus::BuildGauge()
.Name("hydraqueuerunner_queue_max_build_id_info")
.Help("Maximum build record ID in the queue")
.Register(*registry)
.Add({})
)
{
State::State()
}
State::State(std::optional<std::string> metricsAddrOpt)
: config(std::make_unique<HydraConfig>())
, maxUnsupportedTime(config->getIntOption("max_unsupported_time", 0))
, dbPool(config->getIntOption("max_db_connections", 128))
@ -45,11 +94,16 @@ State::State()
, maxLogSize(config->getIntOption("max_log_size", 64ULL << 20))
, uploadLogsToBinaryCache(config->getBoolOption("upload_logs_to_binary_cache", false))
, rootsDir(config->getStrOption("gc_roots_dir", fmt("%s/gcroots/per-user/%s/hydra-roots", settings.nixStateDir, getEnvOrDie("LOGNAME"))))
, metricsAddr(config->getStrOption("queue_runner_metrics_address", std::string{"127.0.0.1:9198"}))
{
hydraData = getEnvOrDie("HYDRA_DATA");
logDir = canonPath(hydraData + "/build-logs");
if (metricsAddrOpt.has_value()) {
metricsAddr = metricsAddrOpt.value();
}
/* handle deprecated store specification */
if (config->getStrOption("store_mode") != "")
throw Error("store_mode in hydra.conf is deprecated, please use store_uri");
@ -754,6 +808,18 @@ void State::run(BuildID buildOne)
if (!lock)
throw Error("hydra-queue-runner is already running");
std::cout << "Starting the Prometheus exporter on " << metricsAddr << std::endl;
/* Set up simple exporter, to show that we're still alive. */
prometheus::Exposer promExposer{metricsAddr};
auto exposerPort = promExposer.GetListeningPorts().front();
promExposer.RegisterCollectable(prom.registry);
std::cout << "Started the Prometheus exporter, listening on "
<< metricsAddr << "/metrics (port " << exposerPort << ")"
<< std::endl;
Store::Params localParams;
localParams["max-connections"] = "16";
localParams["max-connection-age"] = "600";
@ -864,6 +930,7 @@ int main(int argc, char * * argv)
bool unlock = false;
bool status = false;
BuildID buildOne = 0;
std::optional<std::string> metricsAddrOpt = std::nullopt;
parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) {
if (*arg == "--unlock")
@ -875,6 +942,8 @@ int main(int argc, char * * argv)
buildOne = *b;
else
throw Error("‘--build-one’ requires a build ID");
} else if (*arg == "--prometheus-address") {
metricsAddrOpt = getArg(*arg, arg, end);
} else
return false;
return true;
@ -883,7 +952,7 @@ int main(int argc, char * * argv)
settings.verboseBuild = true;
settings.lockCPU = false;
State state;
State state{metricsAddrOpt};
if (status)
state.showStatus();
else if (unlock)

17
src/hydra-queue-runner/queue-monitor.cc
View File

@ -82,6 +82,8 @@ struct PreviousFailure : public std::exception {
bool State::getQueuedBuilds(Connection & conn,
ref<Store> destStore, unsigned int & lastBuildId)
{
prom.queue_checks_started.Increment();
printInfo("checking the queue for builds > %d...", lastBuildId);
/* Grab the queued builds from the database, but don't process
@ -107,7 +109,10 @@ bool State::getQueuedBuilds(Connection & conn,
auto builds_(builds.lock());
BuildID id = row["id"].as<BuildID>();
if (buildOne && id != buildOne) continue;
if (id > newLastBuildId) newLastBuildId = id;
if (id > newLastBuildId) {
newLastBuildId = id;
prom.queue_max_id.Set(id);
}
if (builds_->count(id)) continue;
auto build = std::make_shared<Build>(
@ -136,6 +141,7 @@ bool State::getQueuedBuilds(Connection & conn,
std::set<StorePath> finishedDrvs;
createBuild = [&](Build::ptr build) {
prom.queue_build_loads.Increment();
printMsg(lvlTalkative, format("loading build %1% (%2%)") % build->id % build->fullJobName());
nrAdded++;
newBuildsByID.erase(build->id);
@ -306,9 +312,14 @@ bool State::getQueuedBuilds(Connection & conn,
/* Stop after a certain time to allow priority bumps to be
processed. */
if (std::chrono::system_clock::now() > start + std::chrono::seconds(600)) break;
if (std::chrono::system_clock::now() > start + std::chrono::seconds(600)) {
prom.queue_checks_early_exits.Increment();
break;
}
}
prom.queue_checks_finished.Increment();
lastBuildId = newBuildsByID.empty() ? newLastBuildId : newBuildsByID.begin()->first - 1;
return newBuildsByID.empty();
}
@ -437,6 +448,8 @@ Step::ptr State::createStep(ref<Store> destStore,
if (!isNew) return step;
prom.queue_steps_created.Increment();
printMsg(lvlDebug, "considering derivation ‘%1%’", localStore->printStorePath(drvPath));
/* Initialize the step. Note that the step may be visible in

25
src/hydra-queue-runner/state.hh
View File

@ -7,6 +7,10 @@
#include <memory>
#include <queue>
#include <prometheus/counter.h>
#include <prometheus/gauge.h>
#include <prometheus/registry.h>
#include "db.hh"
#include "parsed-derivations.hh"
@ -433,8 +437,25 @@ private:
via gc_roots_dir. */
nix::Path rootsDir;
std::string metricsAddr;
struct PromMetrics
{
std::shared_ptr<prometheus::Registry> registry;
prometheus::Counter& queue_checks_started;
prometheus::Counter& queue_build_loads;
prometheus::Counter& queue_steps_created;
prometheus::Counter& queue_checks_early_exits;
prometheus::Counter& queue_checks_finished;
prometheus::Gauge& queue_max_id;
PromMetrics();
};
PromMetrics prom;
public:
State();
State(std::optional<std::string> metricsAddrOpt);
private:
@ -544,6 +565,8 @@ private:
void addRoot(const nix::StorePath & storePath);
void runMetricsExporter();
public:
void showStatus();