Fix build
This commit is contained in:
parent
7b8a3acbf0
commit
7d3ba616a9
|
@ -453,29 +453,29 @@ int main(int argc, char * * argv)
|
|||
job["constituents"].push_back(drvPath2);
|
||||
}
|
||||
} else {
|
||||
std::string drvPath = job["drvPath"];
|
||||
auto drv = readDerivation(*store, drvPath);
|
||||
auto drvPath = store->parseStorePath((std::string) job["drvPath"]);
|
||||
auto drv = store->readDerivation(drvPath);
|
||||
|
||||
for (std::string jobName2 : *named) {
|
||||
auto job2 = state->jobs.find(jobName2);
|
||||
if (job2 == state->jobs.end())
|
||||
throw Error("aggregate job '%s' references non-existent job '%s'", jobName, jobName2);
|
||||
std::string drvPath2 = (*job2)["drvPath"];
|
||||
auto drv2 = readDerivation(*store, drvPath2);
|
||||
job["constituents"].push_back(drvPath2);
|
||||
drv.inputDrvs[store->parseStorePath(drvPath2)] = {drv2.outputs.begin()->first};
|
||||
auto drvPath2 = store->parseStorePath((std::string) (*job2)["drvPath"]);
|
||||
auto drv2 = store->readDerivation(drvPath2);
|
||||
job["constituents"].push_back(store->printStorePath(drvPath2));
|
||||
drv.inputDrvs[drvPath2] = {drv2.outputs.begin()->first};
|
||||
}
|
||||
|
||||
std::string drvName(store->parseStorePath(drvPath).name());
|
||||
std::string drvName(drvPath.name());
|
||||
assert(hasSuffix(drvName, drvExtension));
|
||||
drvName.resize(drvName.size() - drvExtension.size());
|
||||
auto h = hashDerivationModulo(*store, drv, true);
|
||||
auto outPath = store->makeOutputPath("out", h, drvName);
|
||||
drv.env["out"] = store->printStorePath(outPath);
|
||||
drv.outputs.insert_or_assign("out", DerivationOutput { .path = outPath });
|
||||
drv.outputs.insert_or_assign("out", DerivationOutput { .output = DerivationOutputInputAddressed { .path = outPath } });
|
||||
auto newDrvPath = store->printStorePath(writeDerivation(store, drv, drvName));
|
||||
|
||||
debug("rewrote aggregate derivation %s -> %s", drvPath, newDrvPath);
|
||||
debug("rewrote aggregate derivation %s -> %s", store->printStorePath(drvPath), newDrvPath);
|
||||
|
||||
job["drvPath"] = newDrvPath;
|
||||
job["outputs"]["out"] = store->printStorePath(outPath);
|
||||
|
|
|
@ -260,12 +260,13 @@ void State::buildRemote(ref<Store> destStore,
|
|||
inputs.insert(p);
|
||||
|
||||
for (auto & input : step->drv->inputDrvs) {
|
||||
Derivation drv2 = readDerivation(*localStore, localStore->printStorePath(input.first));
|
||||
auto drv2 = localStore->readDerivation(input.first);
|
||||
for (auto & name : input.second) {
|
||||
auto i = drv2.outputs.find(name);
|
||||
if (i == drv2.outputs.end()) continue;
|
||||
inputs.insert(i->second.path);
|
||||
basicDrv.inputSrcs.insert(i->second.path);
|
||||
if (auto i = get(drv2.outputs, name)) {
|
||||
auto outPath = i->path(*localStore, drv2.name);
|
||||
inputs.insert(outPath);
|
||||
basicDrv.inputSrcs.insert(outPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -433,7 +434,7 @@ void State::buildRemote(ref<Store> destStore,
|
|||
|
||||
auto now1 = std::chrono::steady_clock::now();
|
||||
|
||||
auto outputs = step->drv->outputPaths();
|
||||
auto outputs = step->drv->outputPaths(*localStore);
|
||||
|
||||
/* Get info about each output path. */
|
||||
std::map<StorePath, ValidPathInfo> infos;
|
||||
|
|
|
@ -16,7 +16,7 @@ BuildOutput getBuildOutput(
|
|||
BuildOutput res;
|
||||
|
||||
/* Compute the closure size. */
|
||||
auto outputs = drv.outputPaths();
|
||||
auto outputs = drv.outputPaths(*store);
|
||||
StorePathSet closure;
|
||||
for (auto & output : outputs)
|
||||
store->computeFSClosure(output, closure);
|
||||
|
@ -106,10 +106,11 @@ BuildOutput getBuildOutput(
|
|||
if (!explicitProducts) {
|
||||
for (auto & output : drv.outputs) {
|
||||
BuildProduct product;
|
||||
product.path = store->printStorePath(output.second.path);
|
||||
auto outPath = output.second.path(*store, drv.name);
|
||||
product.path = store->printStorePath(outPath);
|
||||
product.type = "nix-build";
|
||||
product.subtype = output.first == "out" ? "" : output.first;
|
||||
product.name = output.second.path.name();
|
||||
product.name = outPath.name();
|
||||
|
||||
auto file = narMembers.find(product.path);
|
||||
assert(file != narMembers.end());
|
||||
|
|
|
@ -12,8 +12,8 @@ struct BuildProduct
|
|||
nix::Path path, defaultPath;
|
||||
std::string type, subtype, name;
|
||||
bool isRegular = false;
|
||||
nix::Hash sha256hash;
|
||||
off_t fileSize = 0;
|
||||
std::optional<nix::Hash> sha256hash;
|
||||
std::optional<off_t> fileSize;
|
||||
BuildProduct() { }
|
||||
};
|
||||
|
||||
|
|
|
@ -274,7 +274,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
|
|||
|
||||
assert(stepNr);
|
||||
|
||||
for (auto & path : step->drv->outputPaths())
|
||||
for (auto & path : step->drv->outputPaths(*localStore))
|
||||
addRoot(path);
|
||||
|
||||
/* Register success in the database for all Build objects that
|
||||
|
@ -463,7 +463,7 @@ void State::failStep(
|
|||
/* Remember failed paths in the database so that they
|
||||
won't be built again. */
|
||||
if (result.stepStatus != bsCachedFailure && result.canCache)
|
||||
for (auto & path : step->drv->outputPaths())
|
||||
for (auto & path : step->drv->outputPaths(*localStore))
|
||||
txn.exec_params0("insert into FailedPaths values ($1)", localStore->printStorePath(path));
|
||||
|
||||
txn.commit();
|
||||
|
|
|
@ -260,7 +260,7 @@ unsigned int State::createBuildStep(pqxx::work & txn, time_t startTime, BuildID
|
|||
for (auto & output : step->drv->outputs)
|
||||
txn.exec_params0
|
||||
("insert into BuildStepOutputs (build, stepnr, name, path) values ($1, $2, $3, $4)",
|
||||
buildId, stepNr, output.first, localStore->printStorePath(output.second.path));
|
||||
buildId, stepNr, output.first, localStore->printStorePath(output.second.path(*localStore, step->drv->name)));
|
||||
|
||||
if (status == bsBusy)
|
||||
txn.exec(fmt("notify step_started, '%d\t%d'", buildId, stepNr));
|
||||
|
@ -412,8 +412,8 @@ void State::markSucceededBuild(pqxx::work & txn, Build::ptr build,
|
|||
productNr++,
|
||||
product.type,
|
||||
product.subtype,
|
||||
product.isRegular ? std::make_optional(product.fileSize) : std::nullopt,
|
||||
product.isRegular ? std::make_optional(product.sha256hash.to_string(Base16, false)) : std::nullopt,
|
||||
product.fileSize ? std::make_optional(*product.fileSize) : std::nullopt,
|
||||
product.sha256hash ? std::make_optional(product.sha256hash->to_string(Base16, false)) : std::nullopt,
|
||||
product.path,
|
||||
product.name,
|
||||
product.defaultPath);
|
||||
|
@ -441,7 +441,7 @@ void State::markSucceededBuild(pqxx::work & txn, Build::ptr build,
|
|||
bool State::checkCachedFailure(Step::ptr step, Connection & conn)
|
||||
{
|
||||
pqxx::work txn(conn);
|
||||
for (auto & path : step->drv->outputPaths())
|
||||
for (auto & path : step->drv->outputPaths(*localStore))
|
||||
if (!txn.exec_params("select 1 from FailedPaths where path = $1", localStore->printStorePath(path)).empty())
|
||||
return true;
|
||||
return false;
|
||||
|
@ -637,12 +637,7 @@ void State::dumpStatus(Connection & conn)
|
|||
auto nested2 = nested.object("s3");
|
||||
auto & s3Stats = s3Store->getS3Stats();
|
||||
nested2.attr("put", s3Stats.put);
|
||||
nested2.attr("putBytes", s3Stats.putBytes);
|
||||
nested2.attr("putTimeMs", s3Stats.putTimeMs);
|
||||
nested2.attr("putSpeed",
|
||||
s3Stats.putTimeMs
|
||||
? (double) s3Stats.putBytes / s3Stats.putTimeMs * 1000.0 / (1024.0 * 1024.0)
|
||||
: 0.0);
|
||||
nested2.attr("get", s3Stats.get);
|
||||
nested2.attr("getBytes", s3Stats.getBytes);
|
||||
nested2.attr("getTimeMs", s3Stats.getTimeMs);
|
||||
|
|
|
@ -39,13 +39,13 @@ struct Extractor : ParseSink
|
|||
std::optional<unsigned long long> expectedSize;
|
||||
std::unique_ptr<HashSink> hashSink;
|
||||
|
||||
void preallocateContents(unsigned long long size) override
|
||||
void preallocateContents(uint64_t size) override
|
||||
{
|
||||
expectedSize = size;
|
||||
hashSink = std::make_unique<HashSink>(htSHA256);
|
||||
}
|
||||
|
||||
void receiveContents(unsigned char * data, unsigned int len) override
|
||||
void receiveContents(unsigned char * data, size_t len) override
|
||||
{
|
||||
assert(expectedSize);
|
||||
assert(curMember);
|
||||
|
|
|
@ -176,10 +176,10 @@ bool State::getQueuedBuilds(Connection & conn,
|
|||
if (!res[0].is_null()) propagatedFrom = res[0].as<BuildID>();
|
||||
|
||||
if (!propagatedFrom) {
|
||||
for (auto & output : ex.step->drv->outputs) {
|
||||
for (auto & output : ex.step->drv->outputPaths(*localStore)) {
|
||||
auto res = txn.exec_params
|
||||
("select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where path = $1 and startTime != 0 and stopTime != 0 and status = 1",
|
||||
localStore->printStorePath(output.second.path));
|
||||
localStore->printStorePath(output));
|
||||
if (!res[0][0].is_null()) {
|
||||
propagatedFrom = res[0][0].as<BuildID>();
|
||||
break;
|
||||
|
@ -218,10 +218,10 @@ bool State::getQueuedBuilds(Connection & conn,
|
|||
/* If we didn't get a step, it means the step's outputs are
|
||||
all valid. So we mark this as a finished, cached build. */
|
||||
if (!step) {
|
||||
Derivation drv = readDerivation(*localStore, localStore->printStorePath(build->drvPath));
|
||||
auto drv = localStore->readDerivation(build->drvPath);
|
||||
BuildOutput res = getBuildOutputCached(conn, destStore, drv);
|
||||
|
||||
for (auto & path : drv.outputPaths())
|
||||
for (auto & path : drv.outputPaths(*localStore))
|
||||
addRoot(path);
|
||||
|
||||
{
|
||||
|
@ -427,7 +427,7 @@ Step::ptr State::createStep(ref<Store> destStore,
|
|||
‘steps’ before this point, but that doesn't matter because
|
||||
it's not runnable yet, and other threads won't make it
|
||||
runnable while step->created == false. */
|
||||
step->drv = std::make_unique<Derivation>(readDerivation(*localStore, localStore->printStorePath(drvPath)));
|
||||
step->drv = std::make_unique<Derivation>(localStore->readDerivation(drvPath));
|
||||
step->parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *step->drv);
|
||||
|
||||
step->preferLocalBuild = step->parsedDrv->willBuildLocally();
|
||||
|
@ -453,10 +453,10 @@ Step::ptr State::createStep(ref<Store> destStore,
|
|||
|
||||
/* Are all outputs valid? */
|
||||
bool valid = true;
|
||||
auto outputs = step->drv->outputPaths();
|
||||
auto outputs = step->drv->outputPaths(*localStore);
|
||||
DerivationOutputs missing;
|
||||
for (auto & i : step->drv->outputs)
|
||||
if (!destStore->isValidPath(i.second.path)) {
|
||||
if (!destStore->isValidPath(i.second.path(*localStore, step->drv->name))) {
|
||||
valid = false;
|
||||
missing.insert_or_assign(i.first, i.second);
|
||||
}
|
||||
|
@ -467,11 +467,12 @@ Step::ptr State::createStep(ref<Store> destStore,
|
|||
|
||||
size_t avail = 0;
|
||||
for (auto & i : missing) {
|
||||
if (/* localStore != destStore && */ localStore->isValidPath(i.second.path))
|
||||
auto path = i.second.path(*localStore, step->drv->name);
|
||||
if (/* localStore != destStore && */ localStore->isValidPath(path))
|
||||
avail++;
|
||||
else if (useSubstitutes) {
|
||||
SubstitutablePathInfos infos;
|
||||
localStore->querySubstitutablePathInfos({i.second.path}, infos);
|
||||
localStore->querySubstitutablePathInfos({{path, {}}}, infos);
|
||||
if (infos.size() == 1)
|
||||
avail++;
|
||||
}
|
||||
|
@ -480,35 +481,37 @@ Step::ptr State::createStep(ref<Store> destStore,
|
|||
if (missing.size() == avail) {
|
||||
valid = true;
|
||||
for (auto & i : missing) {
|
||||
auto path = i.second.path(*localStore, step->drv->name);
|
||||
|
||||
try {
|
||||
time_t startTime = time(0);
|
||||
|
||||
if (localStore->isValidPath(i.second.path))
|
||||
if (localStore->isValidPath(path))
|
||||
printInfo("copying output ‘%1%’ of ‘%2%’ from local store",
|
||||
localStore->printStorePath(i.second.path),
|
||||
localStore->printStorePath(path),
|
||||
localStore->printStorePath(drvPath));
|
||||
else {
|
||||
printInfo("substituting output ‘%1%’ of ‘%2%’",
|
||||
localStore->printStorePath(i.second.path),
|
||||
localStore->printStorePath(path),
|
||||
localStore->printStorePath(drvPath));
|
||||
localStore->ensurePath(i.second.path);
|
||||
localStore->ensurePath(path);
|
||||
// FIXME: should copy directly from substituter to destStore.
|
||||
}
|
||||
|
||||
copyClosure(ref<Store>(localStore), destStore, {i.second.path});
|
||||
copyClosure(ref<Store>(localStore), destStore, {path});
|
||||
|
||||
time_t stopTime = time(0);
|
||||
|
||||
{
|
||||
auto mc = startDbUpdate();
|
||||
pqxx::work txn(conn);
|
||||
createSubstitutionStep(txn, startTime, stopTime, build, drvPath, "out", i.second.path);
|
||||
createSubstitutionStep(txn, startTime, stopTime, build, drvPath, "out", path);
|
||||
txn.commit();
|
||||
}
|
||||
|
||||
} catch (Error & e) {
|
||||
printError("while copying/substituting output ‘%s’ of ‘%s’: %s",
|
||||
localStore->printStorePath(i.second.path),
|
||||
localStore->printStorePath(path),
|
||||
localStore->printStorePath(drvPath),
|
||||
e.what());
|
||||
valid = false;
|
||||
|
@ -614,12 +617,12 @@ BuildOutput State::getBuildOutputCached(Connection & conn, nix::ref<nix::Store>
|
|||
{
|
||||
pqxx::work txn(conn);
|
||||
|
||||
for (auto & output : drv.outputs) {
|
||||
for (auto & output : drv.outputPaths(*localStore)) {
|
||||
auto r = txn.exec_params
|
||||
("select id, buildStatus, releaseName, closureSize, size from Builds b "
|
||||
"join BuildOutputs o on b.id = o.build "
|
||||
"where finished = 1 and (buildStatus = 0 or buildStatus = 6) and path = $1",
|
||||
localStore->printStorePath(output.second.path));
|
||||
localStore->printStorePath(output));
|
||||
if (r.empty()) continue;
|
||||
BuildID id = r[0][0].as<BuildID>();
|
||||
|
||||
|
|
Loading…
Reference in a new issue