delroth/hydra
0
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Merge branch 'master' into nix-next

Browse source
This commit is contained in:
John Ericson 2024-01-23 15:49:14 -05:00
parent 20b0ad3ba2 d02e20a4c1
commit 9a86da0e7b
5 changed files with 68 additions and 42 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
src/hydra-queue-runner/build-remote.cc
View file

@ -44,7 +44,7 @@ static Strings extraStoreArgs(std::string & machine)
}
static std::unique_ptr<SSHMaster::Connection> openConnection(
Machine::ptr machine, SSHMaster & master)
::Machine::ptr machine, SSHMaster & master)
{
Strings command = {"nix-store", "--serve", "--write"};
if (machine->isLocalhost()) {
@ -61,7 +61,7 @@ static std::unique_ptr<SSHMaster::Connection> openConnection(
static void copyClosureTo(
Machine::Connection & conn,
::Machine::Connection & conn,
Store & destStore,
const StorePathSet & paths,
SubstituteFlag useSubstitutes = NoSubstitute)
@ -148,7 +148,7 @@ static BasicDerivation sendInputs(
Step & step,
Store & localStore,
Store & destStore,
Machine::Connection & conn,
::Machine::Connection & conn,
unsigned int & overhead,
counter & nrStepsWaiting,
counter & nrStepsCopyingTo
@ -204,7 +204,7 @@ static BasicDerivation sendInputs(
}
static BuildResult performBuild(
Machine::Connection & conn,
::Machine::Connection & conn,
Store & localStore,
StorePath drvPath,
const BasicDerivation & drv,
@ -239,7 +239,7 @@ static BuildResult performBuild(
}
static std::map<StorePath, UnkeyedValidPathInfo> queryPathInfos(
Machine::Connection & conn,
::Machine::Connection & conn,
Store & localStore,
StorePathSet & outputs,
size_t & totalNarSize
@ -265,7 +265,7 @@ static std::map<StorePath, UnkeyedValidPathInfo> queryPathInfos(
}
static void copyPathFromRemote(
Machine::Connection & conn,
::Machine::Connection & conn,
NarMemberDatas & narMembers,
Store & localStore,
Store & destStore,
@ -295,7 +295,7 @@ static void copyPathFromRemote(
}
static void copyPathsFromRemote(
Machine::Connection & conn,
::Machine::Connection & conn,
NarMemberDatas & narMembers,
Store & localStore,
Store & destStore,
@ -374,7 +374,7 @@ void RemoteResult::updateWithBuildResult(const nix::BuildResult & buildResult)
void State::buildRemote(ref<Store> destStore,
Machine::ptr machine, Step::ptr step,
::Machine::ptr machine, Step::ptr step,
const ServeProto::BuildOptions & buildOptions,
RemoteResult & result, std::shared_ptr<ActiveStep> activeStep,
std::function<void(StepState)> updateStep,
@ -420,7 +420,7 @@ void State::buildRemote(ref<Store> destStore,
process. Meh. */
});
Machine::Connection conn {
::Machine::Connection conn {
{
.to = child->in.get(),
.from = child->out.get(),

2
src/hydra-queue-runner/builder.cc
View file

@ -403,7 +403,7 @@ void State::failStep(
Step::ptr step,
BuildID buildId,
const RemoteResult & result,
Machine::ptr machine,
::Machine::ptr machine,
bool & stepFinished)
{
/* Register failure in the database for all Build objects that

10
src/hydra-queue-runner/dispatcher.cc
View file

@ -199,7 +199,7 @@ system_time State::doDispatch()
filter out temporarily disabled machines. */
struct MachineInfo
{
Machine::ptr machine;
::Machine::ptr machine;
unsigned long currentJobs;
};
std::vector<MachineInfo> machinesSorted;
@ -231,11 +231,11 @@ system_time State::doDispatch()
sort(machinesSorted.begin(), machinesSorted.end(),
[](const MachineInfo & a, const MachineInfo & b) -> bool
{
float ta = std::round(a.currentJobs / a.machine->speedFactor);
float tb = std::round(b.currentJobs / b.machine->speedFactor);
float ta = std::round(a.currentJobs / a.machine->speedFactorFloat);
float tb = std::round(b.currentJobs / b.machine->speedFactorFloat);
return
ta != tb ? ta < tb :
a.machine->speedFactor != b.machine->speedFactor ? a.machine->speedFactor > b.machine->speedFactor :
a.machine->speedFactorFloat != b.machine->speedFactorFloat ? a.machine->speedFactorFloat > b.machine->speedFactorFloat :
a.currentJobs > b.currentJobs;
});
@ -435,7 +435,7 @@ void Jobset::pruneSteps()
}
State::MachineReservation::MachineReservation(State & state, Step::ptr step, Machine::ptr machine)
State::MachineReservation::MachineReservation(State & state, Step::ptr step, ::Machine::ptr machine)
: state(state), step(step), machine(machine)
{
machine->state->currentJobs++;

59
src/hydra-queue-runner/hydra-queue-runner.cc
View file

@ -1,6 +1,7 @@
#include <iostream>
#include <thread>
#include <optional>
#include <type_traits>
#include <sys/types.h>
#include <sys/stat.h>
@ -140,23 +141,43 @@ void State::parseMachines(const std::string & contents)
if (tokens.size() < 3) continue;
tokens.resize(8);
auto machine = std::make_shared<Machine>();
machine->sshName = tokens[0];
machine->systemTypes = tokenizeString<StringSet>(tokens[1], ",");
machine->sshKey = tokens[2] == "-" ? std::string("") : tokens[2];
if (tokens[3] != "")
machine->maxJobs = string2Int<decltype(machine->maxJobs)>(tokens[3]).value();
else
machine->maxJobs = 1;
machine->speedFactor = atof(tokens[4].c_str());
if (tokens[5] == "-") tokens[5] = "";
machine->supportedFeatures = tokenizeString<StringSet>(tokens[5], ",");
auto supportedFeatures = tokenizeString<StringSet>(tokens[5], ",");
if (tokens[6] == "-") tokens[6] = "";
machine->mandatoryFeatures = tokenizeString<StringSet>(tokens[6], ",");
for (auto & f : machine->mandatoryFeatures)
machine->supportedFeatures.insert(f);
if (tokens[7] != "" && tokens[7] != "-")
machine->sshPublicHostKey = base64Decode(tokens[7]);
auto mandatoryFeatures = tokenizeString<StringSet>(tokens[6], ",");
for (auto & f : mandatoryFeatures)
supportedFeatures.insert(f);
using MaxJobs = std::remove_const<decltype(nix::Machine::maxJobs)>::type;
auto machine = std::make_shared<::Machine>(nix::Machine {
// `storeUri`, not yet used
"",
// `systemTypes`, not yet used
{},
// `sshKey`
tokens[2] == "-" ? "" : tokens[2],
// `maxJobs`
tokens[3] != ""
? string2Int<MaxJobs>(tokens[3]).value()
: 1,
// `speedFactor`, not yet used
1,
// `supportedFeatures`
std::move(supportedFeatures),
// `mandatoryFeatures`
std::move(mandatoryFeatures),
// `sshPublicHostKey`
tokens[7] != "" && tokens[7] != "-"
? base64Decode(tokens[7])
: "",
});
machine->sshName = tokens[0];
machine->systemTypesSet = tokenizeString<StringSet>(tokens[1], ",");
machine->speedFactorFloat = atof(tokens[4].c_str());
/* Re-use the State object of the previous machine with the
same name. */
@ -166,7 +187,7 @@ void State::parseMachines(const std::string & contents)
else
printMsg(lvlChatty, "updating machine ‘%1%’", machine->sshName);
machine->state = i == oldMachines.end()
? std::make_shared<Machine::State>()
? std::make_shared<::Machine::State>()
: i->second->state;
newMachines[machine->sshName] = machine;
}
@ -175,9 +196,9 @@ void State::parseMachines(const std::string & contents)
if (newMachines.find(m.first) == newMachines.end()) {
if (m.second->enabled)
printInfo("removing machine ‘%1%’", m.first);
/* Add a disabled Machine object to make sure stats are
/* Add a disabled ::Machine object to make sure stats are
maintained. */
auto machine = std::make_shared<Machine>(*(m.second));
auto machine = std::make_shared<::Machine>(*(m.second));
machine->enabled = false;
newMachines[m.first] = machine;
}
@ -596,7 +617,7 @@ void State::dumpStatus(Connection & conn)
json machine = {
{"enabled", m->enabled},
{"systemTypes", m->systemTypes},
{"systemTypes", m->systemTypesSet},
{"supportedFeatures", m->supportedFeatures},
{"mandatoryFeatures", m->mandatoryFeatures},
{"nrStepsDone", s->nrStepsDone.load()},

21
src/hydra-queue-runner/state.hh
View file

@ -23,6 +23,7 @@
#include "nar-extractor.hh"
#include "serve-protocol.hh"
#include "serve-protocol-impl.hh"
#include "machines.hh"
typedef unsigned int BuildID;
@ -235,17 +236,21 @@ void getDependents(Step::ptr step, std::set<Build::ptr> & builds, std::set<Step:
void visitDependencies(std::function<void(Step::ptr)> visitor, Step::ptr step);
struct Machine
struct Machine : nix::Machine
{
typedef std::shared_ptr<Machine> ptr;
bool enabled{true};
/* TODO Get rid of: `nix::Machine::storeUri` is normalized in a way
we are not yet used to, but once we are, we don't need this. */
std::string sshName;
std::string sshName, sshKey;
std::set<std::string> systemTypes, supportedFeatures, mandatoryFeatures;
unsigned int maxJobs = 1;
float speedFactor = 1.0;
std::string sshPublicHostKey;
/* TODO Get rid once `nix::Machine::systemTypes` is a set not
vector. */
std::set<std::string> systemTypesSet;
/* TODO Get rid once `nix::Machine::systemTypes` is a `float` not
an `int`. */
float speedFactorFloat = 1.0;
struct State {
typedef std::shared_ptr<State> ptr;
@ -273,7 +278,7 @@ struct Machine
{
/* Check that this machine is of the type required by the
step. */
if (!systemTypes.count(step->drv->platform == "builtin" ? nix::settings.thisSystem : step->drv->platform))
if (!systemTypesSet.count(step->drv->platform == "builtin" ? nix::settings.thisSystem : step->drv->platform))
return false;
/* Check that the step requires all mandatory features of this