snaakey/hydra
0
0
Fork 0
forked from lix-project/hydra
Code Pull requests Activity

Fix build

Browse source
This commit is contained in:
Eelco Dolstra 2020-08-04 11:33:29 +02:00
parent 7b8a3acbf0
commit 7d3ba616a9
No known key found for this signature in database
GPG key ID: 8170B4726D7198DE
8 changed files with 51 additions and 51 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
src/hydra-eval-jobs/hydra-eval-jobs.cc
View file

@ -453,29 +453,29 @@ int main(int argc, char * * argv)
job["constituents"].push_back(drvPath2);
}
} else {
std::string drvPath = job["drvPath"];
auto drv = readDerivation(*store, drvPath);
auto drvPath = store->parseStorePath((std::string) job["drvPath"]);
auto drv = store->readDerivation(drvPath);
for (std::string jobName2 : *named) {
auto job2 = state->jobs.find(jobName2);
if (job2 == state->jobs.end())
throw Error("aggregate job '%s' references non-existent job '%s'", jobName, jobName2);
std::string drvPath2 = (*job2)["drvPath"];
auto drv2 = readDerivation(*store, drvPath2);
job["constituents"].push_back(drvPath2);
drv.inputDrvs[store->parseStorePath(drvPath2)] = {drv2.outputs.begin()->first};
auto drvPath2 = store->parseStorePath((std::string) (*job2)["drvPath"]);
auto drv2 = store->readDerivation(drvPath2);
job["constituents"].push_back(store->printStorePath(drvPath2));
drv.inputDrvs[drvPath2] = {drv2.outputs.begin()->first};
}
std::string drvName(store->parseStorePath(drvPath).name());
std::string drvName(drvPath.name());
assert(hasSuffix(drvName, drvExtension));
drvName.resize(drvName.size() - drvExtension.size());
auto h = hashDerivationModulo(*store, drv, true);
auto outPath = store->makeOutputPath("out", h, drvName);
drv.env["out"] = store->printStorePath(outPath);
drv.outputs.insert_or_assign("out", DerivationOutput { .path = outPath });
drv.outputs.insert_or_assign("out", DerivationOutput { .output = DerivationOutputInputAddressed { .path = outPath } });
auto newDrvPath = store->printStorePath(writeDerivation(store, drv, drvName));
debug("rewrote aggregate derivation %s -> %s", drvPath, newDrvPath);
debug("rewrote aggregate derivation %s -> %s", store->printStorePath(drvPath), newDrvPath);
job["drvPath"] = newDrvPath;
job["outputs"]["out"] = store->printStorePath(outPath);

13
src/hydra-queue-runner/build-remote.cc
View file

@ -260,12 +260,13 @@ void State::buildRemote(ref<Store> destStore,
inputs.insert(p);
for (auto & input : step->drv->inputDrvs) {
Derivation drv2 = readDerivation(*localStore, localStore->printStorePath(input.first));
auto drv2 = localStore->readDerivation(input.first);
for (auto & name : input.second) {
auto i = drv2.outputs.find(name);
if (i == drv2.outputs.end()) continue;
inputs.insert(i->second.path);
basicDrv.inputSrcs.insert(i->second.path);
if (auto i = get(drv2.outputs, name)) {
auto outPath = i->path(*localStore, drv2.name);
inputs.insert(outPath);
basicDrv.inputSrcs.insert(outPath);
}
}
}
@ -433,7 +434,7 @@ void State::buildRemote(ref<Store> destStore,
auto now1 = std::chrono::steady_clock::now();
auto outputs = step->drv->outputPaths();
auto outputs = step->drv->outputPaths(*localStore);
/* Get info about each output path. */
std::map<StorePath, ValidPathInfo> infos;

7
src/hydra-queue-runner/build-result.cc
View file

@ -16,7 +16,7 @@ BuildOutput getBuildOutput(
BuildOutput res;
/* Compute the closure size. */
auto outputs = drv.outputPaths();
auto outputs = drv.outputPaths(*store);
StorePathSet closure;
for (auto & output : outputs)
store->computeFSClosure(output, closure);
@ -106,10 +106,11 @@ BuildOutput getBuildOutput(
if (!explicitProducts) {
for (auto & output : drv.outputs) {
BuildProduct product;
product.path = store->printStorePath(output.second.path);
auto outPath = output.second.path(*store, drv.name);
product.path = store->printStorePath(outPath);
product.type = "nix-build";
product.subtype = output.first == "out" ? "" : output.first;
product.name = output.second.path.name();
product.name = outPath.name();
auto file = narMembers.find(product.path);
assert(file != narMembers.end());

4
src/hydra-queue-runner/build-result.hh
View file

@ -12,8 +12,8 @@ struct BuildProduct
nix::Path path, defaultPath;
std::string type, subtype, name;
bool isRegular = false;
nix::Hash sha256hash;
off_t fileSize = 0;
std::optional<nix::Hash> sha256hash;
std::optional<off_t> fileSize;
BuildProduct() { }
};

4
src/hydra-queue-runner/builder.cc
View file

@ -274,7 +274,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
assert(stepNr);
for (auto & path : step->drv->outputPaths())
for (auto & path : step->drv->outputPaths(*localStore))
addRoot(path);
/* Register success in the database for all Build objects that
@ -463,7 +463,7 @@ void State::failStep(
/* Remember failed paths in the database so that they
won't be built again. */
if (result.stepStatus != bsCachedFailure && result.canCache)
for (auto & path : step->drv->outputPaths())
for (auto & path : step->drv->outputPaths(*localStore))
txn.exec_params0("insert into FailedPaths values ($1)", localStore->printStorePath(path));
txn.commit();

13
src/hydra-queue-runner/hydra-queue-runner.cc
View file

@ -260,7 +260,7 @@ unsigned int State::createBuildStep(pqxx::work & txn, time_t startTime, BuildID
for (auto & output : step->drv->outputs)
txn.exec_params0
("insert into BuildStepOutputs (build, stepnr, name, path) values ($1, $2, $3, $4)",
buildId, stepNr, output.first, localStore->printStorePath(output.second.path));
buildId, stepNr, output.first, localStore->printStorePath(output.second.path(*localStore, step->drv->name)));
if (status == bsBusy)
txn.exec(fmt("notify step_started, '%d\t%d'", buildId, stepNr));
@ -412,8 +412,8 @@ void State::markSucceededBuild(pqxx::work & txn, Build::ptr build,
productNr++,
product.type,
product.subtype,
product.isRegular ? std::make_optional(product.fileSize) : std::nullopt,
product.isRegular ? std::make_optional(product.sha256hash.to_string(Base16, false)) : std::nullopt,
product.fileSize ? std::make_optional(*product.fileSize) : std::nullopt,
product.sha256hash ? std::make_optional(product.sha256hash->to_string(Base16, false)) : std::nullopt,
product.path,
product.name,
product.defaultPath);
@ -441,7 +441,7 @@ void State::markSucceededBuild(pqxx::work & txn, Build::ptr build,
bool State::checkCachedFailure(Step::ptr step, Connection & conn)
{
pqxx::work txn(conn);
for (auto & path : step->drv->outputPaths())
for (auto & path : step->drv->outputPaths(*localStore))
if (!txn.exec_params("select 1 from FailedPaths where path = $1", localStore->printStorePath(path)).empty())
return true;
return false;
@ -637,12 +637,7 @@ void State::dumpStatus(Connection & conn)
auto nested2 = nested.object("s3");
auto & s3Stats = s3Store->getS3Stats();
nested2.attr("put", s3Stats.put);
nested2.attr("putBytes", s3Stats.putBytes);
nested2.attr("putTimeMs", s3Stats.putTimeMs);
nested2.attr("putSpeed",
s3Stats.putTimeMs
? (double) s3Stats.putBytes / s3Stats.putTimeMs * 1000.0 / (1024.0 * 1024.0)
: 0.0);
nested2.attr("get", s3Stats.get);
nested2.attr("getBytes", s3Stats.getBytes);
nested2.attr("getTimeMs", s3Stats.getTimeMs);

4
src/hydra-queue-runner/nar-extractor.cc
View file

@ -39,13 +39,13 @@ struct Extractor : ParseSink
std::optional<unsigned long long> expectedSize;
std::unique_ptr<HashSink> hashSink;
void preallocateContents(unsigned long long size) override
void preallocateContents(uint64_t size) override
{
expectedSize = size;
hashSink = std::make_unique<HashSink>(htSHA256);
}
void receiveContents(unsigned char * data, unsigned int len) override
void receiveContents(unsigned char * data, size_t len) override
{
assert(expectedSize);
assert(curMember);

39
src/hydra-queue-runner/queue-monitor.cc
View file

@ -176,10 +176,10 @@ bool State::getQueuedBuilds(Connection & conn,
if (!res[0].is_null()) propagatedFrom = res[0].as<BuildID>();
if (!propagatedFrom) {
for (auto & output : ex.step->drv->outputs) {
for (auto & output : ex.step->drv->outputPaths(*localStore)) {
auto res = txn.exec_params
("select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where path = $1 and startTime != 0 and stopTime != 0 and status = 1",
localStore->printStorePath(output.second.path));
localStore->printStorePath(output));
if (!res[0][0].is_null()) {
propagatedFrom = res[0][0].as<BuildID>();
break;
@ -218,10 +218,10 @@ bool State::getQueuedBuilds(Connection & conn,
/* If we didn't get a step, it means the step's outputs are
all valid. So we mark this as a finished, cached build. */
if (!step) {
Derivation drv = readDerivation(*localStore, localStore->printStorePath(build->drvPath));
auto drv = localStore->readDerivation(build->drvPath);
BuildOutput res = getBuildOutputCached(conn, destStore, drv);
for (auto & path : drv.outputPaths())
for (auto & path : drv.outputPaths(*localStore))
addRoot(path);
{
@ -427,7 +427,7 @@ Step::ptr State::createStep(ref<Store> destStore,
‘steps’ before this point, but that doesn't matter because
it's not runnable yet, and other threads won't make it
runnable while step->created == false. */
step->drv = std::make_unique<Derivation>(readDerivation(*localStore, localStore->printStorePath(drvPath)));
step->drv = std::make_unique<Derivation>(localStore->readDerivation(drvPath));
step->parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *step->drv);
step->preferLocalBuild = step->parsedDrv->willBuildLocally();
@ -453,10 +453,10 @@ Step::ptr State::createStep(ref<Store> destStore,
/* Are all outputs valid? */
bool valid = true;
auto outputs = step->drv->outputPaths();
auto outputs = step->drv->outputPaths(*localStore);
DerivationOutputs missing;
for (auto & i : step->drv->outputs)
if (!destStore->isValidPath(i.second.path)) {
if (!destStore->isValidPath(i.second.path(*localStore, step->drv->name))) {
valid = false;
missing.insert_or_assign(i.first, i.second);
}
@ -467,11 +467,12 @@ Step::ptr State::createStep(ref<Store> destStore,
size_t avail = 0;
for (auto & i : missing) {
if (/* localStore != destStore && */ localStore->isValidPath(i.second.path))
auto path = i.second.path(*localStore, step->drv->name);
if (/* localStore != destStore && */ localStore->isValidPath(path))
avail++;
else if (useSubstitutes) {
SubstitutablePathInfos infos;
localStore->querySubstitutablePathInfos({i.second.path}, infos);
localStore->querySubstitutablePathInfos({{path, {}}}, infos);
if (infos.size() == 1)
avail++;
}
@ -480,35 +481,37 @@ Step::ptr State::createStep(ref<Store> destStore,
if (missing.size() == avail) {
valid = true;
for (auto & i : missing) {
auto path = i.second.path(*localStore, step->drv->name);
try {
time_t startTime = time(0);
if (localStore->isValidPath(i.second.path))
if (localStore->isValidPath(path))
printInfo("copying output ‘%1%’ of ‘%2%’ from local store",
localStore->printStorePath(i.second.path),
localStore->printStorePath(path),
localStore->printStorePath(drvPath));
else {
printInfo("substituting output ‘%1%’ of ‘%2%’",
localStore->printStorePath(i.second.path),
localStore->printStorePath(path),
localStore->printStorePath(drvPath));
localStore->ensurePath(i.second.path);
localStore->ensurePath(path);
// FIXME: should copy directly from substituter to destStore.
}
copyClosure(ref<Store>(localStore), destStore, {i.second.path});
copyClosure(ref<Store>(localStore), destStore, {path});
time_t stopTime = time(0);
{
auto mc = startDbUpdate();
pqxx::work txn(conn);
createSubstitutionStep(txn, startTime, stopTime, build, drvPath, "out", i.second.path);
createSubstitutionStep(txn, startTime, stopTime, build, drvPath, "out", path);
txn.commit();
}
} catch (Error & e) {
printError("while copying/substituting output ‘%s’ of ‘%s’: %s",
localStore->printStorePath(i.second.path),
localStore->printStorePath(path),
localStore->printStorePath(drvPath),
e.what());
valid = false;
@ -614,12 +617,12 @@ BuildOutput State::getBuildOutputCached(Connection & conn, nix::ref<nix::Store>
{
pqxx::work txn(conn);
for (auto & output : drv.outputs) {
for (auto & output : drv.outputPaths(*localStore)) {
auto r = txn.exec_params
("select id, buildStatus, releaseName, closureSize, size from Builds b "
"join BuildOutputs o on b.id = o.build "
"where finished = 1 and (buildStatus = 0 or buildStatus = 6) and path = $1",
localStore->printStorePath(output.second.path));
localStore->printStorePath(output));
if (r.empty()) continue;
BuildID id = r[0][0].as<BuildID>();