snaakey/hydra
0
0
Fork 0
forked from lix-project/hydra
Code Pull requests Activity

Fix build

Browse source
This commit is contained in:
Eelco Dolstra 2020-08-04 11:33:29 +02:00
parent 7b8a3acbf0
commit 7d3ba616a9
No known key found for this signature in database
GPG key ID: 8170B4726D7198DE
8 changed files with 51 additions and 51 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
src/hydra-eval-jobs/hydra-eval-jobs.cc
View file

@ -453,29 +453,29 @@ int main(int argc, char * * argv)
job["constituents"].push_back(drvPath2); job["constituents"].push_back(drvPath2);
} }
} else { } else {
std::string drvPath = job["drvPath"]; auto drvPath = store->parseStorePath((std::string) job["drvPath"]);
auto drv = readDerivation(*store, drvPath); auto drv = store->readDerivation(drvPath);
for (std::string jobName2 : *named) { for (std::string jobName2 : *named) {
auto job2 = state->jobs.find(jobName2); auto job2 = state->jobs.find(jobName2);
if (job2 == state->jobs.end()) if (job2 == state->jobs.end())
throw Error("aggregate job '%s' references non-existent job '%s'", jobName, jobName2); throw Error("aggregate job '%s' references non-existent job '%s'", jobName, jobName2);
std::string drvPath2 = (*job2)["drvPath"]; auto drvPath2 = store->parseStorePath((std::string) (*job2)["drvPath"]);
auto drv2 = readDerivation(*store, drvPath2); auto drv2 = store->readDerivation(drvPath2);
job["constituents"].push_back(drvPath2); job["constituents"].push_back(store->printStorePath(drvPath2));
drv.inputDrvs[store->parseStorePath(drvPath2)] = {drv2.outputs.begin()->first}; drv.inputDrvs[drvPath2] = {drv2.outputs.begin()->first};
} }
std::string drvName(store->parseStorePath(drvPath).name()); std::string drvName(drvPath.name());
assert(hasSuffix(drvName, drvExtension)); assert(hasSuffix(drvName, drvExtension));
drvName.resize(drvName.size() - drvExtension.size()); drvName.resize(drvName.size() - drvExtension.size());
auto h = hashDerivationModulo(*store, drv, true); auto h = hashDerivationModulo(*store, drv, true);
auto outPath = store->makeOutputPath("out", h, drvName); auto outPath = store->makeOutputPath("out", h, drvName);
drv.env["out"] = store->printStorePath(outPath); drv.env["out"] = store->printStorePath(outPath);
drv.outputs.insert_or_assign("out", DerivationOutput { .path = outPath }); drv.outputs.insert_or_assign("out", DerivationOutput { .output = DerivationOutputInputAddressed { .path = outPath } });
auto newDrvPath = store->printStorePath(writeDerivation(store, drv, drvName)); auto newDrvPath = store->printStorePath(writeDerivation(store, drv, drvName));
debug("rewrote aggregate derivation %s -> %s", drvPath, newDrvPath); debug("rewrote aggregate derivation %s -> %s", store->printStorePath(drvPath), newDrvPath);
job["drvPath"] = newDrvPath; job["drvPath"] = newDrvPath;
job["outputs"]["out"] = store->printStorePath(outPath); job["outputs"]["out"] = store->printStorePath(outPath);

13
src/hydra-queue-runner/build-remote.cc
View file

@ -260,12 +260,13 @@ void State::buildRemote(ref<Store> destStore,
inputs.insert(p); inputs.insert(p);
for (auto & input : step->drv->inputDrvs) { for (auto & input : step->drv->inputDrvs) {
Derivation drv2 = readDerivation(*localStore, localStore->printStorePath(input.first)); auto drv2 = localStore->readDerivation(input.first);
for (auto & name : input.second) { for (auto & name : input.second) {
auto i = drv2.outputs.find(name); if (auto i = get(drv2.outputs, name)) {
if (i == drv2.outputs.end()) continue; auto outPath = i->path(*localStore, drv2.name);
inputs.insert(i->second.path); inputs.insert(outPath);
basicDrv.inputSrcs.insert(i->second.path); basicDrv.inputSrcs.insert(outPath);
}
} }
} }
@ -433,7 +434,7 @@ void State::buildRemote(ref<Store> destStore,
auto now1 = std::chrono::steady_clock::now(); auto now1 = std::chrono::steady_clock::now();
auto outputs = step->drv->outputPaths(); auto outputs = step->drv->outputPaths(*localStore);
/* Get info about each output path. */ /* Get info about each output path. */
std::map<StorePath, ValidPathInfo> infos; std::map<StorePath, ValidPathInfo> infos;

7
src/hydra-queue-runner/build-result.cc
View file

@ -16,7 +16,7 @@ BuildOutput getBuildOutput(
BuildOutput res; BuildOutput res;
/* Compute the closure size. */ /* Compute the closure size. */
auto outputs = drv.outputPaths(); auto outputs = drv.outputPaths(*store);
StorePathSet closure; StorePathSet closure;
for (auto & output : outputs) for (auto & output : outputs)
store->computeFSClosure(output, closure); store->computeFSClosure(output, closure);
@ -106,10 +106,11 @@ BuildOutput getBuildOutput(
if (!explicitProducts) { if (!explicitProducts) {
for (auto & output : drv.outputs) { for (auto & output : drv.outputs) {
BuildProduct product; BuildProduct product;
product.path = store->printStorePath(output.second.path); auto outPath = output.second.path(*store, drv.name);
product.path = store->printStorePath(outPath);
product.type = "nix-build"; product.type = "nix-build";
product.subtype = output.first == "out" ? "" : output.first; product.subtype = output.first == "out" ? "" : output.first;
product.name = output.second.path.name(); product.name = outPath.name();
auto file = narMembers.find(product.path); auto file = narMembers.find(product.path);
assert(file != narMembers.end()); assert(file != narMembers.end());

4
src/hydra-queue-runner/build-result.hh
View file

@ -12,8 +12,8 @@ struct BuildProduct
nix::Path path, defaultPath; nix::Path path, defaultPath;
std::string type, subtype, name; std::string type, subtype, name;
bool isRegular = false; bool isRegular = false;
nix::Hash sha256hash; std::optional<nix::Hash> sha256hash;
off_t fileSize = 0; std::optional<off_t> fileSize;
BuildProduct() { } BuildProduct() { }
}; };

4
src/hydra-queue-runner/builder.cc
View file

@ -274,7 +274,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
assert(stepNr); assert(stepNr);
for (auto & path : step->drv->outputPaths()) for (auto & path : step->drv->outputPaths(*localStore))
addRoot(path); addRoot(path);
/* Register success in the database for all Build objects that /* Register success in the database for all Build objects that
@ -463,7 +463,7 @@ void State::failStep(
/* Remember failed paths in the database so that they /* Remember failed paths in the database so that they
won't be built again. */ won't be built again. */
if (result.stepStatus != bsCachedFailure && result.canCache) if (result.stepStatus != bsCachedFailure && result.canCache)
for (auto & path : step->drv->outputPaths()) for (auto & path : step->drv->outputPaths(*localStore))
txn.exec_params0("insert into FailedPaths values ($1)", localStore->printStorePath(path)); txn.exec_params0("insert into FailedPaths values ($1)", localStore->printStorePath(path));
txn.commit(); txn.commit();

13
src/hydra-queue-runner/hydra-queue-runner.cc
View file

@ -260,7 +260,7 @@ unsigned int State::createBuildStep(pqxx::work & txn, time_t startTime, BuildID
for (auto & output : step->drv->outputs) for (auto & output : step->drv->outputs)
txn.exec_params0 txn.exec_params0
("insert into BuildStepOutputs (build, stepnr, name, path) values ($1, $2, $3, $4)", ("insert into BuildStepOutputs (build, stepnr, name, path) values ($1, $2, $3, $4)",
buildId, stepNr, output.first, localStore->printStorePath(output.second.path)); buildId, stepNr, output.first, localStore->printStorePath(output.second.path(*localStore, step->drv->name)));
if (status == bsBusy) if (status == bsBusy)
txn.exec(fmt("notify step_started, '%d\t%d'", buildId, stepNr)); txn.exec(fmt("notify step_started, '%d\t%d'", buildId, stepNr));
@ -412,8 +412,8 @@ void State::markSucceededBuild(pqxx::work & txn, Build::ptr build,
productNr++, productNr++,
product.type, product.type,
product.subtype, product.subtype,
product.isRegular ? std::make_optional(product.fileSize) : std::nullopt, product.fileSize ? std::make_optional(*product.fileSize) : std::nullopt,
product.isRegular ? std::make_optional(product.sha256hash.to_string(Base16, false)) : std::nullopt, product.sha256hash ? std::make_optional(product.sha256hash->to_string(Base16, false)) : std::nullopt,
product.path, product.path,
product.name, product.name,
product.defaultPath); product.defaultPath);
@ -441,7 +441,7 @@ void State::markSucceededBuild(pqxx::work & txn, Build::ptr build,
bool State::checkCachedFailure(Step::ptr step, Connection & conn) bool State::checkCachedFailure(Step::ptr step, Connection & conn)
{ {
pqxx::work txn(conn); pqxx::work txn(conn);
for (auto & path : step->drv->outputPaths()) for (auto & path : step->drv->outputPaths(*localStore))
if (!txn.exec_params("select 1 from FailedPaths where path = $1", localStore->printStorePath(path)).empty()) if (!txn.exec_params("select 1 from FailedPaths where path = $1", localStore->printStorePath(path)).empty())
return true; return true;
return false; return false;
@ -637,12 +637,7 @@ void State::dumpStatus(Connection & conn)
auto nested2 = nested.object("s3"); auto nested2 = nested.object("s3");
auto & s3Stats = s3Store->getS3Stats(); auto & s3Stats = s3Store->getS3Stats();
nested2.attr("put", s3Stats.put); nested2.attr("put", s3Stats.put);
nested2.attr("putBytes", s3Stats.putBytes);
nested2.attr("putTimeMs", s3Stats.putTimeMs); nested2.attr("putTimeMs", s3Stats.putTimeMs);
nested2.attr("putSpeed",
s3Stats.putTimeMs
? (double) s3Stats.putBytes / s3Stats.putTimeMs * 1000.0 / (1024.0 * 1024.0)
: 0.0);
nested2.attr("get", s3Stats.get); nested2.attr("get", s3Stats.get);
nested2.attr("getBytes", s3Stats.getBytes); nested2.attr("getBytes", s3Stats.getBytes);
nested2.attr("getTimeMs", s3Stats.getTimeMs); nested2.attr("getTimeMs", s3Stats.getTimeMs);

4
src/hydra-queue-runner/nar-extractor.cc
View file

@ -39,13 +39,13 @@ struct Extractor : ParseSink
std::optional<unsigned long long> expectedSize; std::optional<unsigned long long> expectedSize;
std::unique_ptr<HashSink> hashSink; std::unique_ptr<HashSink> hashSink;
void preallocateContents(unsigned long long size) override void preallocateContents(uint64_t size) override
{ {
expectedSize = size; expectedSize = size;
hashSink = std::make_unique<HashSink>(htSHA256); hashSink = std::make_unique<HashSink>(htSHA256);
} }
void receiveContents(unsigned char * data, unsigned int len) override void receiveContents(unsigned char * data, size_t len) override
{ {
assert(expectedSize); assert(expectedSize);
assert(curMember); assert(curMember);

39
src/hydra-queue-runner/queue-monitor.cc
View file

@ -176,10 +176,10 @@ bool State::getQueuedBuilds(Connection & conn,
if (!res[0].is_null()) propagatedFrom = res[0].as<BuildID>(); if (!res[0].is_null()) propagatedFrom = res[0].as<BuildID>();
if (!propagatedFrom) { if (!propagatedFrom) {
for (auto & output : ex.step->drv->outputs) { for (auto & output : ex.step->drv->outputPaths(*localStore)) {
auto res = txn.exec_params auto res = txn.exec_params
("select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where path = $1 and startTime != 0 and stopTime != 0 and status = 1", ("select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where path = $1 and startTime != 0 and stopTime != 0 and status = 1",
localStore->printStorePath(output.second.path)); localStore->printStorePath(output));
if (!res[0][0].is_null()) { if (!res[0][0].is_null()) {
propagatedFrom = res[0][0].as<BuildID>(); propagatedFrom = res[0][0].as<BuildID>();
break; break;
@ -218,10 +218,10 @@ bool State::getQueuedBuilds(Connection & conn,
/* If we didn't get a step, it means the step's outputs are /* If we didn't get a step, it means the step's outputs are
all valid. So we mark this as a finished, cached build. */ all valid. So we mark this as a finished, cached build. */
if (!step) { if (!step) {
Derivation drv = readDerivation(*localStore, localStore->printStorePath(build->drvPath)); auto drv = localStore->readDerivation(build->drvPath);
BuildOutput res = getBuildOutputCached(conn, destStore, drv); BuildOutput res = getBuildOutputCached(conn, destStore, drv);
for (auto & path : drv.outputPaths()) for (auto & path : drv.outputPaths(*localStore))
addRoot(path); addRoot(path);
{ {
@ -427,7 +427,7 @@ Step::ptr State::createStep(ref<Store> destStore,
‘steps’ before this point, but that doesn't matter because ‘steps’ before this point, but that doesn't matter because
it's not runnable yet, and other threads won't make it it's not runnable yet, and other threads won't make it
runnable while step->created == false. */ runnable while step->created == false. */
step->drv = std::make_unique<Derivation>(readDerivation(*localStore, localStore->printStorePath(drvPath))); step->drv = std::make_unique<Derivation>(localStore->readDerivation(drvPath));
step->parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *step->drv); step->parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *step->drv);
step->preferLocalBuild = step->parsedDrv->willBuildLocally(); step->preferLocalBuild = step->parsedDrv->willBuildLocally();
@ -453,10 +453,10 @@ Step::ptr State::createStep(ref<Store> destStore,
/* Are all outputs valid? */ /* Are all outputs valid? */
bool valid = true; bool valid = true;
auto outputs = step->drv->outputPaths(); auto outputs = step->drv->outputPaths(*localStore);
DerivationOutputs missing; DerivationOutputs missing;
for (auto & i : step->drv->outputs) for (auto & i : step->drv->outputs)
if (!destStore->isValidPath(i.second.path)) { if (!destStore->isValidPath(i.second.path(*localStore, step->drv->name))) {
valid = false; valid = false;
missing.insert_or_assign(i.first, i.second); missing.insert_or_assign(i.first, i.second);
} }
@ -467,11 +467,12 @@ Step::ptr State::createStep(ref<Store> destStore,
size_t avail = 0; size_t avail = 0;
for (auto & i : missing) { for (auto & i : missing) {
if (/* localStore != destStore && */ localStore->isValidPath(i.second.path)) auto path = i.second.path(*localStore, step->drv->name);
if (/* localStore != destStore && */ localStore->isValidPath(path))
avail++; avail++;
else if (useSubstitutes) { else if (useSubstitutes) {
SubstitutablePathInfos infos; SubstitutablePathInfos infos;
localStore->querySubstitutablePathInfos({i.second.path}, infos); localStore->querySubstitutablePathInfos({{path, {}}}, infos);
if (infos.size() == 1) if (infos.size() == 1)
avail++; avail++;
} }
@ -480,35 +481,37 @@ Step::ptr State::createStep(ref<Store> destStore,
if (missing.size() == avail) { if (missing.size() == avail) {
valid = true; valid = true;
for (auto & i : missing) { for (auto & i : missing) {
auto path = i.second.path(*localStore, step->drv->name);
try { try {
time_t startTime = time(0); time_t startTime = time(0);
if (localStore->isValidPath(i.second.path)) if (localStore->isValidPath(path))
printInfo("copying output ‘%1%’ of ‘%2%’ from local store", printInfo("copying output ‘%1%’ of ‘%2%’ from local store",
localStore->printStorePath(i.second.path), localStore->printStorePath(path),
localStore->printStorePath(drvPath)); localStore->printStorePath(drvPath));
else { else {
printInfo("substituting output ‘%1%’ of ‘%2%’", printInfo("substituting output ‘%1%’ of ‘%2%’",
localStore->printStorePath(i.second.path), localStore->printStorePath(path),
localStore->printStorePath(drvPath)); localStore->printStorePath(drvPath));
localStore->ensurePath(i.second.path); localStore->ensurePath(path);
// FIXME: should copy directly from substituter to destStore. // FIXME: should copy directly from substituter to destStore.
} }
copyClosure(ref<Store>(localStore), destStore, {i.second.path}); copyClosure(ref<Store>(localStore), destStore, {path});
time_t stopTime = time(0); time_t stopTime = time(0);
{ {
auto mc = startDbUpdate(); auto mc = startDbUpdate();
pqxx::work txn(conn); pqxx::work txn(conn);
createSubstitutionStep(txn, startTime, stopTime, build, drvPath, "out", i.second.path); createSubstitutionStep(txn, startTime, stopTime, build, drvPath, "out", path);
txn.commit(); txn.commit();
} }
} catch (Error & e) { } catch (Error & e) {
printError("while copying/substituting output ‘%s’ of ‘%s’: %s", printError("while copying/substituting output ‘%s’ of ‘%s’: %s",
localStore->printStorePath(i.second.path), localStore->printStorePath(path),
localStore->printStorePath(drvPath), localStore->printStorePath(drvPath),
e.what()); e.what());
valid = false; valid = false;
@ -614,12 +617,12 @@ BuildOutput State::getBuildOutputCached(Connection & conn, nix::ref<nix::Store>
{ {
pqxx::work txn(conn); pqxx::work txn(conn);
for (auto & output : drv.outputs) { for (auto & output : drv.outputPaths(*localStore)) {
auto r = txn.exec_params auto r = txn.exec_params
("select id, buildStatus, releaseName, closureSize, size from Builds b " ("select id, buildStatus, releaseName, closureSize, size from Builds b "
"join BuildOutputs o on b.id = o.build " "join BuildOutputs o on b.id = o.build "
"where finished = 1 and (buildStatus = 0 or buildStatus = 6) and path = $1", "where finished = 1 and (buildStatus = 0 or buildStatus = 6) and path = $1",
localStore->printStorePath(output.second.path)); localStore->printStorePath(output));
if (r.empty()) continue; if (r.empty()) continue;
BuildID id = r[0][0].as<BuildID>(); BuildID id = r[0][0].as<BuildID>();