Compare commits
2 commits
3891ad77e3
...
fd7fd0ad65
Author | SHA1 | Date | |
---|---|---|---|
Pierre Bourdon | fd7fd0ad65 | ||
Pierre Bourdon | d3fcedbcf5 |
12
.clang-tidy
Normal file
12
.clang-tidy
Normal file
|
@ -0,0 +1,12 @@
|
||||||
|
UseColor: true
|
||||||
|
Checks:
|
||||||
|
- -*
|
||||||
|
|
||||||
|
- bugprone-*
|
||||||
|
# kind of nonsense
|
||||||
|
- -bugprone-easily-swappable-parameters
|
||||||
|
# many warnings due to not recognizing `assert` properly
|
||||||
|
- -bugprone-unchecked-optional-access
|
||||||
|
|
||||||
|
- modernize-*
|
||||||
|
- -modernize-use-trailing-return-type
|
|
@ -14,11 +14,12 @@
|
||||||
#include <sys/wait.h>
|
#include <sys/wait.h>
|
||||||
|
|
||||||
#include <boost/format.hpp>
|
#include <boost/format.hpp>
|
||||||
|
#include <utility>
|
||||||
|
|
||||||
using namespace nix;
|
using namespace nix;
|
||||||
using boost::format;
|
using boost::format;
|
||||||
|
|
||||||
typedef std::pair<std::string, std::string> JobsetName;
|
using JobsetName = std::pair<std::string, std::string>;
|
||||||
|
|
||||||
class JobsetId {
|
class JobsetId {
|
||||||
public:
|
public:
|
||||||
|
@ -28,8 +29,8 @@ class JobsetId {
|
||||||
int id;
|
int id;
|
||||||
|
|
||||||
|
|
||||||
JobsetId(const std::string & project, const std::string & jobset, int id)
|
JobsetId(std::string project, std::string jobset, int id)
|
||||||
: project{ project }, jobset{ jobset }, id{ id }
|
: project{std::move( project )}, jobset{std::move( jobset )}, id{ id }
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -41,7 +42,7 @@ class JobsetId {
|
||||||
friend bool operator== (const JobsetId & lhs, const JobsetName & rhs);
|
friend bool operator== (const JobsetId & lhs, const JobsetName & rhs);
|
||||||
friend bool operator!= (const JobsetId & lhs, const JobsetName & rhs);
|
friend bool operator!= (const JobsetId & lhs, const JobsetName & rhs);
|
||||||
|
|
||||||
std::string display() const {
|
[[nodiscard]] std::string display() const {
|
||||||
return str(format("%1%:%2% (jobset#%3%)") % project % jobset % id);
|
return str(format("%1%:%2% (jobset#%3%)") % project % jobset % id);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
@ -88,11 +89,11 @@ struct Evaluator
|
||||||
JobsetId name;
|
JobsetId name;
|
||||||
std::optional<EvaluationStyle> evaluation_style;
|
std::optional<EvaluationStyle> evaluation_style;
|
||||||
time_t lastCheckedTime, triggerTime;
|
time_t lastCheckedTime, triggerTime;
|
||||||
int checkInterval;
|
time_t checkInterval;
|
||||||
Pid pid;
|
Pid pid;
|
||||||
};
|
};
|
||||||
|
|
||||||
typedef std::map<JobsetId, Jobset> Jobsets;
|
using Jobsets = std::map<JobsetId, Jobset>;
|
||||||
|
|
||||||
std::optional<JobsetName> evalOne;
|
std::optional<JobsetName> evalOne;
|
||||||
|
|
||||||
|
@ -138,13 +139,15 @@ struct Evaluator
|
||||||
|
|
||||||
if (evalOne && name != *evalOne) continue;
|
if (evalOne && name != *evalOne) continue;
|
||||||
|
|
||||||
auto res = state->jobsets.try_emplace(name, Jobset{name});
|
auto res = state->jobsets.try_emplace(name, Jobset{.name=name});
|
||||||
|
|
||||||
auto & jobset = res.first->second;
|
auto & jobset = res.first->second;
|
||||||
jobset.lastCheckedTime = row["lastCheckedTime"].as<time_t>(0);
|
jobset.lastCheckedTime = row["lastCheckedTime"].as<time_t>(0);
|
||||||
jobset.triggerTime = row["triggerTime"].as<time_t>(notTriggered);
|
jobset.triggerTime = row["triggerTime"].as<time_t>(notTriggered);
|
||||||
jobset.checkInterval = row["checkInterval"].as<time_t>();
|
jobset.checkInterval = row["checkInterval"].as<time_t>();
|
||||||
switch (row["jobset_enabled"].as<int>(0)) {
|
|
||||||
|
int eval_style = row["jobset_enabled"].as<int>(0);
|
||||||
|
switch (eval_style) {
|
||||||
case 1:
|
case 1:
|
||||||
jobset.evaluation_style = EvaluationStyle::SCHEDULE;
|
jobset.evaluation_style = EvaluationStyle::SCHEDULE;
|
||||||
break;
|
break;
|
||||||
|
@ -154,6 +157,9 @@ struct Evaluator
|
||||||
case 3:
|
case 3:
|
||||||
jobset.evaluation_style = EvaluationStyle::ONE_AT_A_TIME;
|
jobset.evaluation_style = EvaluationStyle::ONE_AT_A_TIME;
|
||||||
break;
|
break;
|
||||||
|
default:
|
||||||
|
// Disabled or unknown. Leave as nullopt.
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
seen.insert(name);
|
seen.insert(name);
|
||||||
|
@ -175,7 +181,7 @@ struct Evaluator
|
||||||
|
|
||||||
void startEval(State & state, Jobset & jobset)
|
void startEval(State & state, Jobset & jobset)
|
||||||
{
|
{
|
||||||
time_t now = time(0);
|
time_t now = time(nullptr);
|
||||||
|
|
||||||
printInfo("starting evaluation of jobset ‘%s’ (last checked %d s ago)",
|
printInfo("starting evaluation of jobset ‘%s’ (last checked %d s ago)",
|
||||||
jobset.name.display(),
|
jobset.name.display(),
|
||||||
|
@ -228,7 +234,7 @@ struct Evaluator
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (jobset.lastCheckedTime + jobset.checkInterval <= time(0)) {
|
if (jobset.lastCheckedTime + jobset.checkInterval <= time(nullptr)) {
|
||||||
// Time to schedule a fresh evaluation. If the jobset
|
// Time to schedule a fresh evaluation. If the jobset
|
||||||
// is a ONE_AT_A_TIME jobset, ensure the previous jobset
|
// is a ONE_AT_A_TIME jobset, ensure the previous jobset
|
||||||
// has no remaining, unfinished work.
|
// has no remaining, unfinished work.
|
||||||
|
@ -301,7 +307,7 @@ struct Evaluator
|
||||||
|
|
||||||
/* Put jobsets in order of ascending trigger time, last checked
|
/* Put jobsets in order of ascending trigger time, last checked
|
||||||
time, and name. */
|
time, and name. */
|
||||||
std::sort(sorted.begin(), sorted.end(),
|
std::ranges::sort(sorted,
|
||||||
[](const Jobsets::iterator & a, const Jobsets::iterator & b) {
|
[](const Jobsets::iterator & a, const Jobsets::iterator & b) {
|
||||||
return
|
return
|
||||||
a->second.triggerTime != b->second.triggerTime
|
a->second.triggerTime != b->second.triggerTime
|
||||||
|
@ -324,7 +330,7 @@ struct Evaluator
|
||||||
|
|
||||||
while (true) {
|
while (true) {
|
||||||
|
|
||||||
time_t now = time(0);
|
time_t now = time(nullptr);
|
||||||
|
|
||||||
std::chrono::seconds sleepTime = std::chrono::seconds::max();
|
std::chrono::seconds sleepTime = std::chrono::seconds::max();
|
||||||
|
|
||||||
|
@ -411,7 +417,7 @@ struct Evaluator
|
||||||
printInfo("evaluation of jobset ‘%s’ %s",
|
printInfo("evaluation of jobset ‘%s’ %s",
|
||||||
jobset.name.display(), statusToString(status));
|
jobset.name.display(), statusToString(status));
|
||||||
|
|
||||||
auto now = time(0);
|
auto now = time(nullptr);
|
||||||
|
|
||||||
jobset.triggerTime = notTriggered;
|
jobset.triggerTime = notTriggered;
|
||||||
jobset.lastCheckedTime = now;
|
jobset.lastCheckedTime = now;
|
||||||
|
|
|
@ -1,5 +1,6 @@
|
||||||
#include <algorithm>
|
#include <algorithm>
|
||||||
#include <cmath>
|
#include <cmath>
|
||||||
|
#include <ranges>
|
||||||
|
|
||||||
#include <sys/types.h>
|
#include <sys/types.h>
|
||||||
#include <sys/stat.h>
|
#include <sys/stat.h>
|
||||||
|
@ -41,6 +42,7 @@ static Strings extraStoreArgs(std::string & machine)
|
||||||
}
|
}
|
||||||
} catch (BadURL &) {
|
} catch (BadURL &) {
|
||||||
// We just try to continue with `machine->sshName` here for backwards compat.
|
// We just try to continue with `machine->sshName` here for backwards compat.
|
||||||
|
printMsg(lvlWarn, "could not parse machine URL '%s', passing through to SSH", machine);
|
||||||
}
|
}
|
||||||
|
|
||||||
return result;
|
return result;
|
||||||
|
@ -133,8 +135,8 @@ static void copyClosureTo(
|
||||||
auto sorted = destStore.topoSortPaths(closure);
|
auto sorted = destStore.topoSortPaths(closure);
|
||||||
|
|
||||||
StorePathSet missing;
|
StorePathSet missing;
|
||||||
for (auto i = sorted.rbegin(); i != sorted.rend(); ++i)
|
for (auto & i : std::ranges::reverse_view(sorted))
|
||||||
if (!present.count(*i)) missing.insert(*i);
|
if (!present.count(i)) missing.insert(i);
|
||||||
|
|
||||||
printMsg(lvlDebug, "sending %d missing paths", missing.size());
|
printMsg(lvlDebug, "sending %d missing paths", missing.size());
|
||||||
|
|
||||||
|
@ -304,12 +306,12 @@ static BuildResult performBuild(
|
||||||
|
|
||||||
time_t startTime, stopTime;
|
time_t startTime, stopTime;
|
||||||
|
|
||||||
startTime = time(0);
|
startTime = time(nullptr);
|
||||||
{
|
{
|
||||||
MaintainCount<counter> mc(nrStepsBuilding);
|
MaintainCount<counter> mc(nrStepsBuilding);
|
||||||
result = ServeProto::Serialise<BuildResult>::read(localStore, conn);
|
result = ServeProto::Serialise<BuildResult>::read(localStore, conn);
|
||||||
}
|
}
|
||||||
stopTime = time(0);
|
stopTime = time(nullptr);
|
||||||
|
|
||||||
if (!result.startTime) {
|
if (!result.startTime) {
|
||||||
// If the builder gave `startTime = 0`, use our measurements
|
// If the builder gave `startTime = 0`, use our measurements
|
||||||
|
@ -338,10 +340,10 @@ static BuildResult performBuild(
|
||||||
// were known
|
// were known
|
||||||
assert(outputPath);
|
assert(outputPath);
|
||||||
auto outputHash = outputHashes.at(outputName);
|
auto outputHash = outputHashes.at(outputName);
|
||||||
auto drvOutput = DrvOutput { outputHash, outputName };
|
auto drvOutput = DrvOutput { .drvHash=outputHash, .outputName=outputName };
|
||||||
result.builtOutputs.insert_or_assign(
|
result.builtOutputs.insert_or_assign(
|
||||||
std::move(outputName),
|
std::move(outputName),
|
||||||
Realisation { drvOutput, *outputPath });
|
Realisation { .id=drvOutput, .outPath=*outputPath });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -634,7 +636,7 @@ void State::buildRemote(ref<Store> destStore,
|
||||||
* copying outputs and we end up building too many things that we
|
* copying outputs and we end up building too many things that we
|
||||||
* haven't been able to allow copy slots for. */
|
* haven't been able to allow copy slots for. */
|
||||||
assert(reservation.unique());
|
assert(reservation.unique());
|
||||||
reservation = 0;
|
reservation = nullptr;
|
||||||
wakeDispatcher();
|
wakeDispatcher();
|
||||||
|
|
||||||
StorePathSet outputs;
|
StorePathSet outputs;
|
||||||
|
@ -697,7 +699,7 @@ void State::buildRemote(ref<Store> destStore,
|
||||||
if (info->consecutiveFailures == 0 || info->lastFailure < now - std::chrono::seconds(30)) {
|
if (info->consecutiveFailures == 0 || info->lastFailure < now - std::chrono::seconds(30)) {
|
||||||
info->consecutiveFailures = std::min(info->consecutiveFailures + 1, (unsigned int) 4);
|
info->consecutiveFailures = std::min(info->consecutiveFailures + 1, (unsigned int) 4);
|
||||||
info->lastFailure = now;
|
info->lastFailure = now;
|
||||||
int delta = retryInterval * std::pow(retryBackoff, info->consecutiveFailures - 1) + (rand() % 30);
|
int delta = static_cast<int>(retryInterval * std::pow(retryBackoff, info->consecutiveFailures - 1) + (rand() % 30));
|
||||||
printMsg(lvlInfo, "will disable machine ‘%1%’ for %2%s", machine->sshName, delta);
|
printMsg(lvlInfo, "will disable machine ‘%1%’ for %2%s", machine->sshName, delta);
|
||||||
info->disabledUntil = now + std::chrono::seconds(delta);
|
info->disabledUntil = now + std::chrono::seconds(delta);
|
||||||
}
|
}
|
||||||
|
|
|
@ -58,7 +58,7 @@ void State::builder(MachineReservation::ptr reservation)
|
||||||
/* If the machine hasn't been released yet, release and wake up the dispatcher. */
|
/* If the machine hasn't been released yet, release and wake up the dispatcher. */
|
||||||
if (reservation) {
|
if (reservation) {
|
||||||
assert(reservation.unique());
|
assert(reservation.unique());
|
||||||
reservation = 0;
|
reservation = nullptr;
|
||||||
wakeDispatcher();
|
wakeDispatcher();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -72,7 +72,7 @@ void State::builder(MachineReservation::ptr reservation)
|
||||||
step_->tries++;
|
step_->tries++;
|
||||||
nrRetries++;
|
nrRetries++;
|
||||||
if (step_->tries > maxNrRetries) maxNrRetries = step_->tries; // yeah yeah, not atomic
|
if (step_->tries > maxNrRetries) maxNrRetries = step_->tries; // yeah yeah, not atomic
|
||||||
int delta = retryInterval * std::pow(retryBackoff, step_->tries - 1) + (rand() % 10);
|
int delta = static_cast<int>(retryInterval * std::pow(retryBackoff, step_->tries - 1) + (rand() % 10));
|
||||||
printMsg(lvlInfo, "will retry ‘%s’ after %ss", localStore->printStorePath(step->drvPath), delta);
|
printMsg(lvlInfo, "will retry ‘%s’ after %ss", localStore->printStorePath(step->drvPath), delta);
|
||||||
step_->after = std::chrono::system_clock::now() + std::chrono::seconds(delta);
|
step_->after = std::chrono::system_clock::now() + std::chrono::seconds(delta);
|
||||||
}
|
}
|
||||||
|
@ -190,7 +190,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
time_t stepStartTime = result.startTime = time(0);
|
time_t stepStartTime = result.startTime = time(nullptr);
|
||||||
|
|
||||||
/* If any of the outputs have previously failed, then don't bother
|
/* If any of the outputs have previously failed, then don't bother
|
||||||
building again. */
|
building again. */
|
||||||
|
@ -237,7 +237,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
time_t stepStopTime = time(0);
|
time_t stepStopTime = time(nullptr);
|
||||||
if (!result.stopTime) result.stopTime = stepStopTime;
|
if (!result.stopTime) result.stopTime = stepStopTime;
|
||||||
|
|
||||||
/* For standard failures, we don't care about the error
|
/* For standard failures, we don't care about the error
|
||||||
|
@ -251,7 +251,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
|
||||||
auto step_(step->state.lock());
|
auto step_(step->state.lock());
|
||||||
if (!step_->jobsets.empty()) {
|
if (!step_->jobsets.empty()) {
|
||||||
// FIXME: loss of precision.
|
// FIXME: loss of precision.
|
||||||
time_t charge = (result.stopTime - result.startTime) / step_->jobsets.size();
|
time_t charge = (result.stopTime - result.startTime) / static_cast<time_t>(step_->jobsets.size());
|
||||||
for (auto & jobset : step_->jobsets)
|
for (auto & jobset : step_->jobsets)
|
||||||
jobset->addStep(result.startTime, charge);
|
jobset->addStep(result.startTime, charge);
|
||||||
}
|
}
|
||||||
|
|
|
@ -46,7 +46,7 @@ void State::dispatcher()
|
||||||
auto t_after_work = std::chrono::steady_clock::now();
|
auto t_after_work = std::chrono::steady_clock::now();
|
||||||
|
|
||||||
prom.dispatcher_time_spent_running.Increment(
|
prom.dispatcher_time_spent_running.Increment(
|
||||||
std::chrono::duration_cast<std::chrono::microseconds>(t_after_work - t_before_work).count());
|
static_cast<double>(std::chrono::duration_cast<std::chrono::microseconds>(t_after_work - t_before_work).count()));
|
||||||
dispatchTimeMs += std::chrono::duration_cast<std::chrono::milliseconds>(t_after_work - t_before_work).count();
|
dispatchTimeMs += std::chrono::duration_cast<std::chrono::milliseconds>(t_after_work - t_before_work).count();
|
||||||
|
|
||||||
/* Sleep until we're woken up (either because a runnable build
|
/* Sleep until we're woken up (either because a runnable build
|
||||||
|
@ -63,7 +63,7 @@ void State::dispatcher()
|
||||||
|
|
||||||
auto t_after_sleep = std::chrono::steady_clock::now();
|
auto t_after_sleep = std::chrono::steady_clock::now();
|
||||||
prom.dispatcher_time_spent_waiting.Increment(
|
prom.dispatcher_time_spent_waiting.Increment(
|
||||||
std::chrono::duration_cast<std::chrono::microseconds>(t_after_sleep - t_after_work).count());
|
static_cast<double>(std::chrono::duration_cast<std::chrono::microseconds>(t_after_sleep - t_after_work).count()));
|
||||||
|
|
||||||
} catch (std::exception & e) {
|
} catch (std::exception & e) {
|
||||||
printError("dispatcher: %s", e.what());
|
printError("dispatcher: %s", e.what());
|
||||||
|
@ -190,7 +190,7 @@ system_time State::doDispatch()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
sort(runnableSorted.begin(), runnableSorted.end(),
|
std::ranges::sort(runnableSorted,
|
||||||
[](const StepInfo & a, const StepInfo & b)
|
[](const StepInfo & a, const StepInfo & b)
|
||||||
{
|
{
|
||||||
return
|
return
|
||||||
|
@ -240,11 +240,11 @@ system_time State::doDispatch()
|
||||||
- Then by speed factor.
|
- Then by speed factor.
|
||||||
|
|
||||||
- Finally by load. */
|
- Finally by load. */
|
||||||
sort(machinesSorted.begin(), machinesSorted.end(),
|
std::ranges::sort(machinesSorted,
|
||||||
[](const MachineInfo & a, const MachineInfo & b) -> bool
|
[](const MachineInfo & a, const MachineInfo & b) -> bool
|
||||||
{
|
{
|
||||||
float ta = std::round(a.currentJobs / a.machine->speedFactorFloat);
|
float ta = std::round(static_cast<float>(a.currentJobs) / a.machine->speedFactorFloat);
|
||||||
float tb = std::round(b.currentJobs / b.machine->speedFactorFloat);
|
float tb = std::round(static_cast<float>(b.currentJobs) / b.machine->speedFactorFloat);
|
||||||
return
|
return
|
||||||
ta != tb ? ta < tb :
|
ta != tb ? ta < tb :
|
||||||
a.machine->speedFactorFloat != b.machine->speedFactorFloat ? a.machine->speedFactorFloat > b.machine->speedFactorFloat :
|
a.machine->speedFactorFloat != b.machine->speedFactorFloat ? a.machine->speedFactorFloat > b.machine->speedFactorFloat :
|
||||||
|
@ -345,7 +345,7 @@ void State::abortUnsupported()
|
||||||
auto machines2 = *machines.lock();
|
auto machines2 = *machines.lock();
|
||||||
|
|
||||||
system_time now = std::chrono::system_clock::now();
|
system_time now = std::chrono::system_clock::now();
|
||||||
auto now2 = time(0);
|
auto now2 = time(nullptr);
|
||||||
|
|
||||||
std::unordered_set<Step::ptr> aborted;
|
std::unordered_set<Step::ptr> aborted;
|
||||||
|
|
||||||
|
@ -436,7 +436,7 @@ void Jobset::addStep(time_t startTime, time_t duration)
|
||||||
|
|
||||||
void Jobset::pruneSteps()
|
void Jobset::pruneSteps()
|
||||||
{
|
{
|
||||||
time_t now = time(0);
|
time_t now = time(nullptr);
|
||||||
auto steps_(steps.lock());
|
auto steps_(steps.lock());
|
||||||
while (!steps_->empty()) {
|
while (!steps_->empty()) {
|
||||||
auto i = steps_->begin();
|
auto i = steps_->begin();
|
||||||
|
@ -464,7 +464,7 @@ State::MachineReservation::~MachineReservation()
|
||||||
auto prev = machine->state->currentJobs--;
|
auto prev = machine->state->currentJobs--;
|
||||||
assert(prev);
|
assert(prev);
|
||||||
if (prev == 1)
|
if (prev == 1)
|
||||||
machine->state->idleSince = time(0);
|
machine->state->idleSince = time(nullptr);
|
||||||
|
|
||||||
{
|
{
|
||||||
auto machineTypes_(state.machineTypes.lock());
|
auto machineTypes_(state.machineTypes.lock());
|
||||||
|
|
|
@ -14,7 +14,7 @@ struct BuildProduct
|
||||||
bool isRegular = false;
|
bool isRegular = false;
|
||||||
std::optional<nix::Hash> sha256hash;
|
std::optional<nix::Hash> sha256hash;
|
||||||
std::optional<off_t> fileSize;
|
std::optional<off_t> fileSize;
|
||||||
BuildProduct() { }
|
BuildProduct() = default;
|
||||||
};
|
};
|
||||||
|
|
||||||
struct BuildMetric
|
struct BuildMetric
|
||||||
|
|
|
@ -105,7 +105,7 @@ State::State(std::optional<std::string> metricsAddrOpt)
|
||||||
: config(std::make_unique<HydraConfig>())
|
: config(std::make_unique<HydraConfig>())
|
||||||
, maxUnsupportedTime(config->getIntOption("max_unsupported_time", 0))
|
, maxUnsupportedTime(config->getIntOption("max_unsupported_time", 0))
|
||||||
, dbPool(config->getIntOption("max_db_connections", 128))
|
, dbPool(config->getIntOption("max_db_connections", 128))
|
||||||
, localWorkThrottler(config->getIntOption("max_local_worker_threads", std::min(maxSupportedLocalWorkers, std::max(4u, std::thread::hardware_concurrency()) - 2)))
|
, localWorkThrottler(static_cast<ptrdiff_t>(config->getIntOption("max_local_worker_threads", std::min(maxSupportedLocalWorkers, std::max(4u, std::thread::hardware_concurrency()) - 2))))
|
||||||
, maxOutputSize(config->getIntOption("max_output_size", 2ULL << 30))
|
, maxOutputSize(config->getIntOption("max_output_size", 2ULL << 30))
|
||||||
, maxLogSize(config->getIntOption("max_log_size", 64ULL << 20))
|
, maxLogSize(config->getIntOption("max_log_size", 64ULL << 20))
|
||||||
, uploadLogsToBinaryCache(config->getBoolOption("upload_logs_to_binary_cache", false))
|
, uploadLogsToBinaryCache(config->getBoolOption("upload_logs_to_binary_cache", false))
|
||||||
|
@ -138,7 +138,7 @@ nix::MaintainCount<counter> State::startDbUpdate()
|
||||||
{
|
{
|
||||||
if (nrActiveDbUpdates > 6)
|
if (nrActiveDbUpdates > 6)
|
||||||
printError("warning: %d concurrent database updates; PostgreSQL may be stalled", nrActiveDbUpdates.load());
|
printError("warning: %d concurrent database updates; PostgreSQL may be stalled", nrActiveDbUpdates.load());
|
||||||
return MaintainCount<counter>(nrActiveDbUpdates);
|
return {nrActiveDbUpdates};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -171,7 +171,7 @@ void State::parseMachines(const std::string & contents)
|
||||||
for (auto & f : mandatoryFeatures)
|
for (auto & f : mandatoryFeatures)
|
||||||
supportedFeatures.insert(f);
|
supportedFeatures.insert(f);
|
||||||
|
|
||||||
using MaxJobs = std::remove_const<decltype(nix::Machine::maxJobs)>::type;
|
using MaxJobs = std::remove_const_t<decltype(nix::Machine::maxJobs)>;
|
||||||
|
|
||||||
auto machine = std::make_shared<::Machine>(::Machine {{
|
auto machine = std::make_shared<::Machine>(::Machine {{
|
||||||
// `storeUri`, not yet used
|
// `storeUri`, not yet used
|
||||||
|
@ -198,7 +198,7 @@ void State::parseMachines(const std::string & contents)
|
||||||
|
|
||||||
machine->sshName = tokens[0];
|
machine->sshName = tokens[0];
|
||||||
machine->systemTypesSet = tokenizeString<StringSet>(tokens[1], ",");
|
machine->systemTypesSet = tokenizeString<StringSet>(tokens[1], ",");
|
||||||
machine->speedFactorFloat = atof(tokens[4].c_str());
|
machine->speedFactorFloat = static_cast<float>(atof(tokens[4].c_str()));
|
||||||
|
|
||||||
/* Re-use the State object of the previous machine with the
|
/* Re-use the State object of the previous machine with the
|
||||||
same name. */
|
same name. */
|
||||||
|
@ -412,7 +412,7 @@ void State::finishBuildStep(pqxx::work & txn, const RemoteResult & result,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
int State::createSubstitutionStep(pqxx::work & txn, time_t startTime, time_t stopTime,
|
unsigned int State::createSubstitutionStep(pqxx::work & txn, time_t startTime, time_t stopTime,
|
||||||
Build::ptr build, const StorePath & drvPath, const nix::Derivation drv, const std::string & outputName, const StorePath & storePath)
|
Build::ptr build, const StorePath & drvPath, const nix::Derivation drv, const std::string & outputName, const StorePath & storePath)
|
||||||
{
|
{
|
||||||
restart:
|
restart:
|
||||||
|
@ -594,7 +594,7 @@ std::shared_ptr<PathLocks> State::acquireGlobalLock()
|
||||||
createDirs(dirOf(lockPath));
|
createDirs(dirOf(lockPath));
|
||||||
|
|
||||||
auto lock = std::make_shared<PathLocks>();
|
auto lock = std::make_shared<PathLocks>();
|
||||||
if (!lock->lockPaths(PathSet({lockPath}), "", false)) return 0;
|
if (!lock->lockPaths(PathSet({lockPath}), "", false)) return nullptr;
|
||||||
|
|
||||||
return lock;
|
return lock;
|
||||||
}
|
}
|
||||||
|
@ -602,10 +602,10 @@ std::shared_ptr<PathLocks> State::acquireGlobalLock()
|
||||||
|
|
||||||
void State::dumpStatus(Connection & conn)
|
void State::dumpStatus(Connection & conn)
|
||||||
{
|
{
|
||||||
time_t now = time(0);
|
time_t now = time(nullptr);
|
||||||
json statusJson = {
|
json statusJson = {
|
||||||
{"status", "up"},
|
{"status", "up"},
|
||||||
{"time", time(0)},
|
{"time", time(nullptr)},
|
||||||
{"uptime", now - startedAt},
|
{"uptime", now - startedAt},
|
||||||
{"pid", getpid()},
|
{"pid", getpid()},
|
||||||
|
|
||||||
|
@ -620,7 +620,7 @@ void State::dumpStatus(Connection & conn)
|
||||||
{"bytesReceived", bytesReceived.load()},
|
{"bytesReceived", bytesReceived.load()},
|
||||||
{"nrBuildsRead", nrBuildsRead.load()},
|
{"nrBuildsRead", nrBuildsRead.load()},
|
||||||
{"buildReadTimeMs", buildReadTimeMs.load()},
|
{"buildReadTimeMs", buildReadTimeMs.load()},
|
||||||
{"buildReadTimeAvgMs", nrBuildsRead == 0 ? 0.0 : (float) buildReadTimeMs / nrBuildsRead},
|
{"buildReadTimeAvgMs", nrBuildsRead == 0 ? 0.0 : (float) buildReadTimeMs / (float) nrBuildsRead},
|
||||||
{"nrBuildsDone", nrBuildsDone.load()},
|
{"nrBuildsDone", nrBuildsDone.load()},
|
||||||
{"nrStepsStarted", nrStepsStarted.load()},
|
{"nrStepsStarted", nrStepsStarted.load()},
|
||||||
{"nrStepsDone", nrStepsDone.load()},
|
{"nrStepsDone", nrStepsDone.load()},
|
||||||
|
@ -629,7 +629,7 @@ void State::dumpStatus(Connection & conn)
|
||||||
{"nrQueueWakeups", nrQueueWakeups.load()},
|
{"nrQueueWakeups", nrQueueWakeups.load()},
|
||||||
{"nrDispatcherWakeups", nrDispatcherWakeups.load()},
|
{"nrDispatcherWakeups", nrDispatcherWakeups.load()},
|
||||||
{"dispatchTimeMs", dispatchTimeMs.load()},
|
{"dispatchTimeMs", dispatchTimeMs.load()},
|
||||||
{"dispatchTimeAvgMs", nrDispatcherWakeups == 0 ? 0.0 : (float) dispatchTimeMs / nrDispatcherWakeups},
|
{"dispatchTimeAvgMs", nrDispatcherWakeups == 0 ? 0.0 : (float) dispatchTimeMs / (float) nrDispatcherWakeups},
|
||||||
{"nrDbConnections", dbPool.count()},
|
{"nrDbConnections", dbPool.count()},
|
||||||
{"nrActiveDbUpdates", nrActiveDbUpdates.load()},
|
{"nrActiveDbUpdates", nrActiveDbUpdates.load()},
|
||||||
};
|
};
|
||||||
|
@ -649,8 +649,8 @@ void State::dumpStatus(Connection & conn)
|
||||||
if (nrStepsDone) {
|
if (nrStepsDone) {
|
||||||
statusJson["totalStepTime"] = totalStepTime.load();
|
statusJson["totalStepTime"] = totalStepTime.load();
|
||||||
statusJson["totalStepBuildTime"] = totalStepBuildTime.load();
|
statusJson["totalStepBuildTime"] = totalStepBuildTime.load();
|
||||||
statusJson["avgStepTime"] = (float) totalStepTime / nrStepsDone;
|
statusJson["avgStepTime"] = (float) totalStepTime / (float) nrStepsDone;
|
||||||
statusJson["avgStepBuildTime"] = (float) totalStepBuildTime / nrStepsDone;
|
statusJson["avgStepBuildTime"] = (float) totalStepBuildTime / (float) nrStepsDone;
|
||||||
}
|
}
|
||||||
|
|
||||||
{
|
{
|
||||||
|
@ -677,8 +677,8 @@ void State::dumpStatus(Connection & conn)
|
||||||
if (m->state->nrStepsDone) {
|
if (m->state->nrStepsDone) {
|
||||||
machine["totalStepTime"] = s->totalStepTime.load();
|
machine["totalStepTime"] = s->totalStepTime.load();
|
||||||
machine["totalStepBuildTime"] = s->totalStepBuildTime.load();
|
machine["totalStepBuildTime"] = s->totalStepBuildTime.load();
|
||||||
machine["avgStepTime"] = (float) s->totalStepTime / s->nrStepsDone;
|
machine["avgStepTime"] = (float) s->totalStepTime / (float) s->nrStepsDone;
|
||||||
machine["avgStepBuildTime"] = (float) s->totalStepBuildTime / s->nrStepsDone;
|
machine["avgStepBuildTime"] = (float) s->totalStepBuildTime / (float) s->nrStepsDone;
|
||||||
}
|
}
|
||||||
statusJson["machines"][m->sshName] = machine;
|
statusJson["machines"][m->sshName] = machine;
|
||||||
}
|
}
|
||||||
|
@ -706,7 +706,7 @@ void State::dumpStatus(Connection & conn)
|
||||||
};
|
};
|
||||||
if (i.second.runnable > 0)
|
if (i.second.runnable > 0)
|
||||||
machineTypeJson["waitTime"] = i.second.waitTime.count() +
|
machineTypeJson["waitTime"] = i.second.waitTime.count() +
|
||||||
i.second.runnable * (time(0) - lastDispatcherCheck);
|
i.second.runnable * (time(nullptr) - lastDispatcherCheck);
|
||||||
if (i.second.running == 0)
|
if (i.second.running == 0)
|
||||||
machineTypeJson["lastActive"] = std::chrono::system_clock::to_time_t(i.second.lastActive);
|
machineTypeJson["lastActive"] = std::chrono::system_clock::to_time_t(i.second.lastActive);
|
||||||
}
|
}
|
||||||
|
@ -732,11 +732,11 @@ void State::dumpStatus(Connection & conn)
|
||||||
{"narWriteCompressionTimeMs", stats.narWriteCompressionTimeMs.load()},
|
{"narWriteCompressionTimeMs", stats.narWriteCompressionTimeMs.load()},
|
||||||
{"narCompressionSavings",
|
{"narCompressionSavings",
|
||||||
stats.narWriteBytes
|
stats.narWriteBytes
|
||||||
? 1.0 - (double) stats.narWriteCompressedBytes / stats.narWriteBytes
|
? 1.0 - (double) stats.narWriteCompressedBytes / (double) stats.narWriteBytes
|
||||||
: 0.0},
|
: 0.0},
|
||||||
{"narCompressionSpeed", // MiB/s
|
{"narCompressionSpeed", // MiB/s
|
||||||
stats.narWriteCompressionTimeMs
|
stats.narWriteCompressionTimeMs
|
||||||
? (double) stats.narWriteBytes / stats.narWriteCompressionTimeMs * 1000.0 / (1024.0 * 1024.0)
|
? (double) stats.narWriteBytes / (double) stats.narWriteCompressionTimeMs * 1000.0 / (1024.0 * 1024.0)
|
||||||
: 0.0},
|
: 0.0},
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -749,20 +749,20 @@ void State::dumpStatus(Connection & conn)
|
||||||
{"putTimeMs", s3Stats.putTimeMs.load()},
|
{"putTimeMs", s3Stats.putTimeMs.load()},
|
||||||
{"putSpeed",
|
{"putSpeed",
|
||||||
s3Stats.putTimeMs
|
s3Stats.putTimeMs
|
||||||
? (double) s3Stats.putBytes / s3Stats.putTimeMs * 1000.0 / (1024.0 * 1024.0)
|
? (double) s3Stats.putBytes / (double) s3Stats.putTimeMs * 1000.0 / (1024.0 * 1024.0)
|
||||||
: 0.0},
|
: 0.0},
|
||||||
{"get", s3Stats.get.load()},
|
{"get", s3Stats.get.load()},
|
||||||
{"getBytes", s3Stats.getBytes.load()},
|
{"getBytes", s3Stats.getBytes.load()},
|
||||||
{"getTimeMs", s3Stats.getTimeMs.load()},
|
{"getTimeMs", s3Stats.getTimeMs.load()},
|
||||||
{"getSpeed",
|
{"getSpeed",
|
||||||
s3Stats.getTimeMs
|
s3Stats.getTimeMs
|
||||||
? (double) s3Stats.getBytes / s3Stats.getTimeMs * 1000.0 / (1024.0 * 1024.0)
|
? (double) s3Stats.getBytes / (double) s3Stats.getTimeMs * 1000.0 / (1024.0 * 1024.0)
|
||||||
: 0.0},
|
: 0.0},
|
||||||
{"head", s3Stats.head.load()},
|
{"head", s3Stats.head.load()},
|
||||||
{"costDollarApprox",
|
{"costDollarApprox",
|
||||||
(s3Stats.get + s3Stats.head) / 10000.0 * 0.004
|
(double) (s3Stats.get + s3Stats.head) / 10000.0 * 0.004
|
||||||
+ s3Stats.put / 1000.0 * 0.005 +
|
+ (double) s3Stats.put / 1000.0 * 0.005 +
|
||||||
+ s3Stats.getBytes / (1024.0 * 1024.0 * 1024.0) * 0.09},
|
+ (double) s3Stats.getBytes / (1024.0 * 1024.0 * 1024.0) * 0.09},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -848,7 +848,7 @@ void State::run(BuildID buildOne)
|
||||||
/* Can't be bothered to shut down cleanly. Goodbye! */
|
/* Can't be bothered to shut down cleanly. Goodbye! */
|
||||||
auto callback = createInterruptCallback([&]() { std::_Exit(0); });
|
auto callback = createInterruptCallback([&]() { std::_Exit(0); });
|
||||||
|
|
||||||
startedAt = time(0);
|
startedAt = time(nullptr);
|
||||||
this->buildOne = buildOne;
|
this->buildOne = buildOne;
|
||||||
|
|
||||||
auto lock = acquireGlobalLock();
|
auto lock = acquireGlobalLock();
|
||||||
|
|
|
@ -3,6 +3,7 @@
|
||||||
#include "archive.hh"
|
#include "archive.hh"
|
||||||
|
|
||||||
#include <unordered_set>
|
#include <unordered_set>
|
||||||
|
#include <utility>
|
||||||
|
|
||||||
using namespace nix;
|
using namespace nix;
|
||||||
|
|
||||||
|
@ -18,8 +19,8 @@ struct Extractor : ParseSink
|
||||||
NarMemberData * curMember = nullptr;
|
NarMemberData * curMember = nullptr;
|
||||||
Path prefix;
|
Path prefix;
|
||||||
|
|
||||||
Extractor(NarMemberDatas & members, const Path & prefix)
|
Extractor(NarMemberDatas & members, Path prefix)
|
||||||
: members(members), prefix(prefix)
|
: members(members), prefix(std::move(prefix))
|
||||||
{ }
|
{ }
|
||||||
|
|
||||||
void createDirectory(const Path & path) override
|
void createDirectory(const Path & path) override
|
||||||
|
|
|
@ -13,7 +13,7 @@ struct NarMemberData
|
||||||
std::optional<nix::Hash> sha256;
|
std::optional<nix::Hash> sha256;
|
||||||
};
|
};
|
||||||
|
|
||||||
typedef std::map<nix::Path, NarMemberData> NarMemberDatas;
|
using NarMemberDatas = std::map<nix::Path, NarMemberData>;
|
||||||
|
|
||||||
/* Read a NAR from a source and get to some info about every file
|
/* Read a NAR from a source and get to some info about every file
|
||||||
inside the NAR. */
|
inside the NAR. */
|
||||||
|
|
|
@ -4,7 +4,8 @@
|
||||||
#include "thread-pool.hh"
|
#include "thread-pool.hh"
|
||||||
|
|
||||||
#include <cstring>
|
#include <cstring>
|
||||||
#include <signal.h>
|
#include <utility>
|
||||||
|
#include <csignal>
|
||||||
|
|
||||||
using namespace nix;
|
using namespace nix;
|
||||||
|
|
||||||
|
@ -52,7 +53,7 @@ void State::queueMonitorLoop(Connection & conn)
|
||||||
auto t_after_work = std::chrono::steady_clock::now();
|
auto t_after_work = std::chrono::steady_clock::now();
|
||||||
|
|
||||||
prom.queue_monitor_time_spent_running.Increment(
|
prom.queue_monitor_time_spent_running.Increment(
|
||||||
std::chrono::duration_cast<std::chrono::microseconds>(t_after_work - t_before_work).count());
|
static_cast<double>(std::chrono::duration_cast<std::chrono::microseconds>(t_after_work - t_before_work).count()));
|
||||||
|
|
||||||
/* Sleep until we get notification from the database about an
|
/* Sleep until we get notification from the database about an
|
||||||
event. */
|
event. */
|
||||||
|
@ -79,7 +80,7 @@ void State::queueMonitorLoop(Connection & conn)
|
||||||
|
|
||||||
auto t_after_sleep = std::chrono::steady_clock::now();
|
auto t_after_sleep = std::chrono::steady_clock::now();
|
||||||
prom.queue_monitor_time_spent_waiting.Increment(
|
prom.queue_monitor_time_spent_waiting.Increment(
|
||||||
std::chrono::duration_cast<std::chrono::microseconds>(t_after_sleep - t_after_work).count());
|
static_cast<double>(std::chrono::duration_cast<std::chrono::microseconds>(t_after_sleep - t_after_work).count()));
|
||||||
}
|
}
|
||||||
|
|
||||||
exit(0);
|
exit(0);
|
||||||
|
@ -88,7 +89,7 @@ void State::queueMonitorLoop(Connection & conn)
|
||||||
|
|
||||||
struct PreviousFailure : public std::exception {
|
struct PreviousFailure : public std::exception {
|
||||||
Step::ptr step;
|
Step::ptr step;
|
||||||
PreviousFailure(Step::ptr step) : step(step) { }
|
PreviousFailure(Step::ptr step) : step(std::move(step)) { }
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
|
@ -117,7 +118,7 @@ bool State::getQueuedBuilds(Connection & conn,
|
||||||
|
|
||||||
for (auto const & row : res) {
|
for (auto const & row : res) {
|
||||||
auto builds_(builds.lock());
|
auto builds_(builds.lock());
|
||||||
BuildID id = row["id"].as<BuildID>();
|
auto id = row["id"].as<BuildID>();
|
||||||
if (buildOne && id != buildOne) continue;
|
if (buildOne && id != buildOne) continue;
|
||||||
if (builds_->count(id)) continue;
|
if (builds_->count(id)) continue;
|
||||||
|
|
||||||
|
@ -137,7 +138,7 @@ bool State::getQueuedBuilds(Connection & conn,
|
||||||
|
|
||||||
newIDs.push_back(id);
|
newIDs.push_back(id);
|
||||||
newBuildsByID[id] = build;
|
newBuildsByID[id] = build;
|
||||||
newBuildsByPath.emplace(std::make_pair(build->drvPath, id));
|
newBuildsByPath.emplace(build->drvPath, id);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -162,7 +163,7 @@ bool State::getQueuedBuilds(Connection & conn,
|
||||||
("update Builds set finished = 1, buildStatus = $2, startTime = $3, stopTime = $3 where id = $1 and finished = 0",
|
("update Builds set finished = 1, buildStatus = $2, startTime = $3, stopTime = $3 where id = $1 and finished = 0",
|
||||||
build->id,
|
build->id,
|
||||||
(int) bsAborted,
|
(int) bsAborted,
|
||||||
time(0));
|
time(nullptr));
|
||||||
txn.commit();
|
txn.commit();
|
||||||
build->finishedInDB = true;
|
build->finishedInDB = true;
|
||||||
nrBuildsDone++;
|
nrBuildsDone++;
|
||||||
|
@ -176,7 +177,7 @@ bool State::getQueuedBuilds(Connection & conn,
|
||||||
/* Create steps for this derivation and its dependencies. */
|
/* Create steps for this derivation and its dependencies. */
|
||||||
try {
|
try {
|
||||||
step = createStep(destStore, conn, build, build->drvPath,
|
step = createStep(destStore, conn, build, build->drvPath,
|
||||||
build, 0, finishedDrvs, newSteps, newRunnable);
|
build, nullptr, finishedDrvs, newSteps, newRunnable);
|
||||||
} catch (PreviousFailure & ex) {
|
} catch (PreviousFailure & ex) {
|
||||||
|
|
||||||
/* Some step previously failed, so mark the build as
|
/* Some step previously failed, so mark the build as
|
||||||
|
@ -221,7 +222,7 @@ bool State::getQueuedBuilds(Connection & conn,
|
||||||
"where id = $1 and finished = 0",
|
"where id = $1 and finished = 0",
|
||||||
build->id,
|
build->id,
|
||||||
(int) (ex.step->drvPath == build->drvPath ? bsFailed : bsDepFailed),
|
(int) (ex.step->drvPath == build->drvPath ? bsFailed : bsDepFailed),
|
||||||
time(0));
|
time(nullptr));
|
||||||
notifyBuildFinished(txn, build->id, {});
|
notifyBuildFinished(txn, build->id, {});
|
||||||
txn.commit();
|
txn.commit();
|
||||||
build->finishedInDB = true;
|
build->finishedInDB = true;
|
||||||
|
@ -254,7 +255,7 @@ bool State::getQueuedBuilds(Connection & conn,
|
||||||
{
|
{
|
||||||
auto mc = startDbUpdate();
|
auto mc = startDbUpdate();
|
||||||
pqxx::work txn(conn);
|
pqxx::work txn(conn);
|
||||||
time_t now = time(0);
|
time_t now = time(nullptr);
|
||||||
if (!buildOneDone && build->id == buildOne) buildOneDone = true;
|
if (!buildOneDone && build->id == buildOne) buildOneDone = true;
|
||||||
printMsg(lvlInfo, "marking build %1% as succeeded (cached)", build->id);
|
printMsg(lvlInfo, "marking build %1% as succeeded (cached)", build->id);
|
||||||
markSucceededBuild(txn, build, res, true, now, now);
|
markSucceededBuild(txn, build, res, true, now, now);
|
||||||
|
@ -355,7 +356,7 @@ void State::processQueueChange(Connection & conn)
|
||||||
pqxx::work txn(conn);
|
pqxx::work txn(conn);
|
||||||
auto res = txn.exec("select id, globalPriority from Builds where finished = 0");
|
auto res = txn.exec("select id, globalPriority from Builds where finished = 0");
|
||||||
for (auto const & row : res)
|
for (auto const & row : res)
|
||||||
currentIds[row["id"].as<BuildID>()] = row["globalPriority"].as<BuildID>();
|
currentIds[row["id"].as<BuildID>()] = row["globalPriority"].as<int>();
|
||||||
}
|
}
|
||||||
|
|
||||||
{
|
{
|
||||||
|
@ -438,7 +439,7 @@ Step::ptr State::createStep(ref<Store> destStore,
|
||||||
Build::ptr referringBuild, Step::ptr referringStep, std::set<StorePath> & finishedDrvs,
|
Build::ptr referringBuild, Step::ptr referringStep, std::set<StorePath> & finishedDrvs,
|
||||||
std::set<Step::ptr> & newSteps, std::set<Step::ptr> & newRunnable)
|
std::set<Step::ptr> & newSteps, std::set<Step::ptr> & newRunnable)
|
||||||
{
|
{
|
||||||
if (finishedDrvs.find(drvPath) != finishedDrvs.end()) return 0;
|
if (finishedDrvs.find(drvPath) != finishedDrvs.end()) return nullptr;
|
||||||
|
|
||||||
/* Check if the requested step already exists. If not, create a
|
/* Check if the requested step already exists. If not, create a
|
||||||
new step. In any case, make the step reachable from
|
new step. In any case, make the step reachable from
|
||||||
|
@ -516,7 +517,7 @@ Step::ptr State::createStep(ref<Store> destStore,
|
||||||
std::map<DrvOutput, std::optional<StorePath>> paths;
|
std::map<DrvOutput, std::optional<StorePath>> paths;
|
||||||
for (auto & [outputName, maybeOutputPath] : destStore->queryPartialDerivationOutputMap(drvPath, &*localStore)) {
|
for (auto & [outputName, maybeOutputPath] : destStore->queryPartialDerivationOutputMap(drvPath, &*localStore)) {
|
||||||
auto outputHash = outputHashes.at(outputName);
|
auto outputHash = outputHashes.at(outputName);
|
||||||
paths.insert({{outputHash, outputName}, maybeOutputPath});
|
paths.insert({{.drvHash=outputHash, .outputName=outputName}, maybeOutputPath});
|
||||||
}
|
}
|
||||||
|
|
||||||
auto missing = getMissingRemotePaths(destStore, paths);
|
auto missing = getMissingRemotePaths(destStore, paths);
|
||||||
|
@ -560,7 +561,7 @@ Step::ptr State::createStep(ref<Store> destStore,
|
||||||
auto & path = *pathOpt;
|
auto & path = *pathOpt;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
time_t startTime = time(0);
|
time_t startTime = time(nullptr);
|
||||||
|
|
||||||
if (localStore->isValidPath(path))
|
if (localStore->isValidPath(path))
|
||||||
printInfo("copying output ‘%1%’ of ‘%2%’ from local store",
|
printInfo("copying output ‘%1%’ of ‘%2%’ from local store",
|
||||||
|
@ -578,7 +579,7 @@ Step::ptr State::createStep(ref<Store> destStore,
|
||||||
StorePathSet { path },
|
StorePathSet { path },
|
||||||
NoRepair, CheckSigs, NoSubstitute);
|
NoRepair, CheckSigs, NoSubstitute);
|
||||||
|
|
||||||
time_t stopTime = time(0);
|
time_t stopTime = time(nullptr);
|
||||||
|
|
||||||
{
|
{
|
||||||
auto mc = startDbUpdate();
|
auto mc = startDbUpdate();
|
||||||
|
@ -602,7 +603,7 @@ Step::ptr State::createStep(ref<Store> destStore,
|
||||||
// FIXME: check whether all outputs are in the binary cache.
|
// FIXME: check whether all outputs are in the binary cache.
|
||||||
if (valid) {
|
if (valid) {
|
||||||
finishedDrvs.insert(drvPath);
|
finishedDrvs.insert(drvPath);
|
||||||
return 0;
|
return nullptr;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* No, we need to build. */
|
/* No, we need to build. */
|
||||||
|
@ -610,7 +611,7 @@ Step::ptr State::createStep(ref<Store> destStore,
|
||||||
|
|
||||||
/* Create steps for the dependencies. */
|
/* Create steps for the dependencies. */
|
||||||
for (auto & i : step->drv->inputDrvs.map) {
|
for (auto & i : step->drv->inputDrvs.map) {
|
||||||
auto dep = createStep(destStore, conn, build, i.first, 0, step, finishedDrvs, newSteps, newRunnable);
|
auto dep = createStep(destStore, conn, build, i.first, nullptr, step, finishedDrvs, newSteps, newRunnable);
|
||||||
if (dep) {
|
if (dep) {
|
||||||
auto step_(step->state.lock());
|
auto step_(step->state.lock());
|
||||||
step_->deps.insert(dep);
|
step_->deps.insert(dep);
|
||||||
|
@ -658,11 +659,11 @@ Jobset::ptr State::createJobset(pqxx::work & txn,
|
||||||
auto res2 = txn.exec_params
|
auto res2 = txn.exec_params
|
||||||
("select s.startTime, s.stopTime from BuildSteps s join Builds b on build = id "
|
("select s.startTime, s.stopTime from BuildSteps s join Builds b on build = id "
|
||||||
"where s.startTime is not null and s.stopTime > $1 and jobset_id = $2",
|
"where s.startTime is not null and s.stopTime > $1 and jobset_id = $2",
|
||||||
time(0) - Jobset::schedulingWindow * 10,
|
time(nullptr) - Jobset::schedulingWindow * 10,
|
||||||
jobsetID);
|
jobsetID);
|
||||||
for (auto const & row : res2) {
|
for (auto const & row : res2) {
|
||||||
time_t startTime = row["startTime"].as<time_t>();
|
auto startTime = row["startTime"].as<time_t>();
|
||||||
time_t stopTime = row["stopTime"].as<time_t>();
|
auto stopTime = row["stopTime"].as<time_t>();
|
||||||
jobset->addStep(startTime, stopTime - startTime);
|
jobset->addStep(startTime, stopTime - startTime);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -702,7 +703,7 @@ BuildOutput State::getBuildOutputCached(Connection & conn, nix::ref<nix::Store>
|
||||||
"where finished = 1 and (buildStatus = 0 or buildStatus = 6) and path = $1",
|
"where finished = 1 and (buildStatus = 0 or buildStatus = 6) and path = $1",
|
||||||
localStore->printStorePath(output));
|
localStore->printStorePath(output));
|
||||||
if (r.empty()) continue;
|
if (r.empty()) continue;
|
||||||
BuildID id = r[0][0].as<BuildID>();
|
auto id = r[0][0].as<BuildID>();
|
||||||
|
|
||||||
printInfo("reusing build %d", id);
|
printInfo("reusing build %d", id);
|
||||||
|
|
||||||
|
|
|
@ -8,6 +8,7 @@
|
||||||
#include <queue>
|
#include <queue>
|
||||||
#include <regex>
|
#include <regex>
|
||||||
#include <semaphore>
|
#include <semaphore>
|
||||||
|
#include <utility>
|
||||||
|
|
||||||
#include <prometheus/counter.h>
|
#include <prometheus/counter.h>
|
||||||
#include <prometheus/gauge.h>
|
#include <prometheus/gauge.h>
|
||||||
|
@ -26,16 +27,16 @@
|
||||||
#include "machines.hh"
|
#include "machines.hh"
|
||||||
|
|
||||||
|
|
||||||
typedef unsigned int BuildID;
|
using BuildID = unsigned int;
|
||||||
|
|
||||||
typedef unsigned int JobsetID;
|
using JobsetID = unsigned int;
|
||||||
|
|
||||||
typedef std::chrono::time_point<std::chrono::system_clock> system_time;
|
using system_time = std::chrono::time_point<std::chrono::system_clock>;
|
||||||
|
|
||||||
typedef std::atomic<unsigned long> counter;
|
using counter = std::atomic<unsigned long>;
|
||||||
|
|
||||||
|
|
||||||
typedef enum {
|
enum BuildStatus {
|
||||||
bsSuccess = 0,
|
bsSuccess = 0,
|
||||||
bsFailed = 1,
|
bsFailed = 1,
|
||||||
bsDepFailed = 2, // builds only
|
bsDepFailed = 2, // builds only
|
||||||
|
@ -49,10 +50,10 @@ typedef enum {
|
||||||
bsNarSizeLimitExceeded = 11,
|
bsNarSizeLimitExceeded = 11,
|
||||||
bsNotDeterministic = 12,
|
bsNotDeterministic = 12,
|
||||||
bsBusy = 100, // not stored
|
bsBusy = 100, // not stored
|
||||||
} BuildStatus;
|
};
|
||||||
|
|
||||||
|
|
||||||
typedef enum {
|
enum StepState {
|
||||||
ssPreparing = 1,
|
ssPreparing = 1,
|
||||||
ssConnecting = 10,
|
ssConnecting = 10,
|
||||||
ssSendingInputs = 20,
|
ssSendingInputs = 20,
|
||||||
|
@ -60,7 +61,7 @@ typedef enum {
|
||||||
ssWaitingForLocalSlot = 35,
|
ssWaitingForLocalSlot = 35,
|
||||||
ssReceivingOutputs = 40,
|
ssReceivingOutputs = 40,
|
||||||
ssPostProcessing = 50,
|
ssPostProcessing = 50,
|
||||||
} StepState;
|
};
|
||||||
|
|
||||||
|
|
||||||
struct RemoteResult
|
struct RemoteResult
|
||||||
|
@ -78,7 +79,7 @@ struct RemoteResult
|
||||||
unsigned int overhead = 0;
|
unsigned int overhead = 0;
|
||||||
nix::Path logFile;
|
nix::Path logFile;
|
||||||
|
|
||||||
BuildStatus buildStatus() const
|
[[nodiscard]] BuildStatus buildStatus() const
|
||||||
{
|
{
|
||||||
return stepStatus == bsCachedFailure ? bsFailed : stepStatus;
|
return stepStatus == bsCachedFailure ? bsFailed : stepStatus;
|
||||||
}
|
}
|
||||||
|
@ -95,10 +96,10 @@ class Jobset
|
||||||
{
|
{
|
||||||
public:
|
public:
|
||||||
|
|
||||||
typedef std::shared_ptr<Jobset> ptr;
|
using ptr = std::shared_ptr<Jobset>;
|
||||||
typedef std::weak_ptr<Jobset> wptr;
|
using wptr = std::weak_ptr<Jobset>;
|
||||||
|
|
||||||
static const time_t schedulingWindow = 24 * 60 * 60;
|
static const time_t schedulingWindow = static_cast<time_t>(24 * 60 * 60);
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
|
||||||
|
@ -115,7 +116,7 @@ public:
|
||||||
return (double) seconds / shares;
|
return (double) seconds / shares;
|
||||||
}
|
}
|
||||||
|
|
||||||
void setShares(int shares_)
|
void setShares(unsigned int shares_)
|
||||||
{
|
{
|
||||||
assert(shares_ > 0);
|
assert(shares_ > 0);
|
||||||
shares = shares_;
|
shares = shares_;
|
||||||
|
@ -131,8 +132,8 @@ public:
|
||||||
|
|
||||||
struct Build
|
struct Build
|
||||||
{
|
{
|
||||||
typedef std::shared_ptr<Build> ptr;
|
using ptr = std::shared_ptr<Build>;
|
||||||
typedef std::weak_ptr<Build> wptr;
|
using wptr = std::weak_ptr<Build>;
|
||||||
|
|
||||||
BuildID id;
|
BuildID id;
|
||||||
nix::StorePath drvPath;
|
nix::StorePath drvPath;
|
||||||
|
@ -163,8 +164,8 @@ struct Build
|
||||||
|
|
||||||
struct Step
|
struct Step
|
||||||
{
|
{
|
||||||
typedef std::shared_ptr<Step> ptr;
|
using ptr = std::shared_ptr<Step>;
|
||||||
typedef std::weak_ptr<Step> wptr;
|
using wptr = std::weak_ptr<Step>;
|
||||||
|
|
||||||
nix::StorePath drvPath;
|
nix::StorePath drvPath;
|
||||||
std::unique_ptr<nix::Derivation> drv;
|
std::unique_ptr<nix::Derivation> drv;
|
||||||
|
@ -221,13 +222,8 @@ struct Step
|
||||||
|
|
||||||
nix::Sync<State> state;
|
nix::Sync<State> state;
|
||||||
|
|
||||||
Step(const nix::StorePath & drvPath) : drvPath(drvPath)
|
Step(nix::StorePath drvPath) : drvPath(std::move(drvPath))
|
||||||
{ }
|
{ }
|
||||||
|
|
||||||
~Step()
|
|
||||||
{
|
|
||||||
//printMsg(lvlError, format("destroying step %1%") % drvPath);
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
|
@ -239,7 +235,7 @@ void visitDependencies(std::function<void(Step::ptr)> visitor, Step::ptr step);
|
||||||
|
|
||||||
struct Machine : nix::Machine
|
struct Machine : nix::Machine
|
||||||
{
|
{
|
||||||
typedef std::shared_ptr<Machine> ptr;
|
using ptr = std::shared_ptr<Machine>;
|
||||||
|
|
||||||
/* TODO Get rid of: `nix::Machine::storeUri` is normalized in a way
|
/* TODO Get rid of: `nix::Machine::storeUri` is normalized in a way
|
||||||
we are not yet used to, but once we are, we don't need this. */
|
we are not yet used to, but once we are, we don't need this. */
|
||||||
|
@ -254,7 +250,7 @@ struct Machine : nix::Machine
|
||||||
float speedFactorFloat = 1.0;
|
float speedFactorFloat = 1.0;
|
||||||
|
|
||||||
struct State {
|
struct State {
|
||||||
typedef std::shared_ptr<State> ptr;
|
using ptr = std::shared_ptr<State>;
|
||||||
counter currentJobs{0};
|
counter currentJobs{0};
|
||||||
counter nrStepsDone{0};
|
counter nrStepsDone{0};
|
||||||
counter totalStepTime{0}; // total time for steps, including closure copying
|
counter totalStepTime{0}; // total time for steps, including closure copying
|
||||||
|
@ -358,22 +354,22 @@ private:
|
||||||
bool useSubstitutes = false;
|
bool useSubstitutes = false;
|
||||||
|
|
||||||
/* The queued builds. */
|
/* The queued builds. */
|
||||||
typedef std::map<BuildID, Build::ptr> Builds;
|
using Builds = std::map<BuildID, Build::ptr>;
|
||||||
nix::Sync<Builds> builds;
|
nix::Sync<Builds> builds;
|
||||||
|
|
||||||
/* The jobsets. */
|
/* The jobsets. */
|
||||||
typedef std::map<std::pair<std::string, std::string>, Jobset::ptr> Jobsets;
|
using Jobsets = std::map<std::pair<std::string, std::string>, Jobset::ptr>;
|
||||||
nix::Sync<Jobsets> jobsets;
|
nix::Sync<Jobsets> jobsets;
|
||||||
|
|
||||||
/* All active or pending build steps (i.e. dependencies of the
|
/* All active or pending build steps (i.e. dependencies of the
|
||||||
queued builds). Note that these are weak pointers. Steps are
|
queued builds). Note that these are weak pointers. Steps are
|
||||||
kept alive by being reachable from Builds or by being in
|
kept alive by being reachable from Builds or by being in
|
||||||
progress. */
|
progress. */
|
||||||
typedef std::map<nix::StorePath, Step::wptr> Steps;
|
using Steps = std::map<nix::StorePath, Step::wptr>;
|
||||||
nix::Sync<Steps> steps;
|
nix::Sync<Steps> steps;
|
||||||
|
|
||||||
/* Build steps that have no unbuilt dependencies. */
|
/* Build steps that have no unbuilt dependencies. */
|
||||||
typedef std::list<Step::wptr> Runnable;
|
using Runnable = std::list<Step::wptr>;
|
||||||
nix::Sync<Runnable> runnable;
|
nix::Sync<Runnable> runnable;
|
||||||
|
|
||||||
/* CV for waking up the dispatcher. */
|
/* CV for waking up the dispatcher. */
|
||||||
|
@ -385,7 +381,7 @@ private:
|
||||||
|
|
||||||
/* The build machines. */
|
/* The build machines. */
|
||||||
std::mutex machinesReadyLock;
|
std::mutex machinesReadyLock;
|
||||||
typedef std::map<std::string, Machine::ptr> Machines;
|
using Machines = std::map<std::string, Machine::ptr>;
|
||||||
nix::Sync<Machines> machines; // FIXME: use atomic_shared_ptr
|
nix::Sync<Machines> machines; // FIXME: use atomic_shared_ptr
|
||||||
|
|
||||||
/* Throttler for CPU-bound local work. */
|
/* Throttler for CPU-bound local work. */
|
||||||
|
@ -431,7 +427,7 @@ private:
|
||||||
|
|
||||||
struct MachineReservation
|
struct MachineReservation
|
||||||
{
|
{
|
||||||
typedef std::shared_ptr<MachineReservation> ptr;
|
using ptr = std::shared_ptr<MachineReservation>;
|
||||||
State & state;
|
State & state;
|
||||||
Step::ptr step;
|
Step::ptr step;
|
||||||
Machine::ptr machine;
|
Machine::ptr machine;
|
||||||
|
@ -534,7 +530,7 @@ private:
|
||||||
void finishBuildStep(pqxx::work & txn, const RemoteResult & result, BuildID buildId, unsigned int stepNr,
|
void finishBuildStep(pqxx::work & txn, const RemoteResult & result, BuildID buildId, unsigned int stepNr,
|
||||||
const std::string & machine);
|
const std::string & machine);
|
||||||
|
|
||||||
int createSubstitutionStep(pqxx::work & txn, time_t startTime, time_t stopTime,
|
unsigned int createSubstitutionStep(pqxx::work & txn, time_t startTime, time_t stopTime,
|
||||||
Build::ptr build, const nix::StorePath & drvPath, const nix::Derivation drv, const std::string & outputName, const nix::StorePath & storePath);
|
Build::ptr build, const nix::StorePath & drvPath, const nix::Derivation drv, const std::string & outputName, const nix::StorePath & storePath);
|
||||||
|
|
||||||
void updateBuild(pqxx::work & txn, Build::ptr build, BuildStatus status);
|
void updateBuild(pqxx::work & txn, Build::ptr build, BuildStatus status);
|
||||||
|
|
Loading…
Reference in a new issue