Merge pull request #3689 from matthewbauer/substitute-other-storedir

Substitutions from different store dirs
This commit is contained in:
Eelco Dolstra 2020-07-30 21:56:59 +02:00 committed by GitHub
commit 0744f7f83b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 225 additions and 123 deletions

View file

@ -370,34 +370,6 @@ false</literal>.</para>
</varlistentry> </varlistentry>
<varlistentry xml:id="conf-hashed-mirrors"><term><literal>hashed-mirrors</literal></term>
<listitem><para>A list of web servers used by
<function>builtins.fetchurl</function> to obtain files by
hash. The default is
<literal>http://tarballs.nixos.org/</literal>. Given a hash type
<replaceable>ht</replaceable> and a base-16 hash
<replaceable>h</replaceable>, Nix will try to download the file
from
<literal>hashed-mirror/<replaceable>ht</replaceable>/<replaceable>h</replaceable></literal>.
This allows files to be downloaded even if they have disappeared
from their original URI. For example, given the default mirror
<literal>http://tarballs.nixos.org/</literal>, when building the derivation
<programlisting>
builtins.fetchurl {
url = "https://example.org/foo-1.2.3.tar.xz";
sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae";
}
</programlisting>
Nix will attempt to download this file from
<literal>http://tarballs.nixos.org/sha256/2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae</literal>
first. If it is not available there, if will try the original URI.</para></listitem>
</varlistentry>
<varlistentry xml:id="conf-http-connections"><term><literal>http-connections</literal></term> <varlistentry xml:id="conf-http-connections"><term><literal>http-connections</literal></term>
<listitem><para>The maximum number of parallel TCP connections <listitem><para>The maximum number of parallel TCP connections

View file

@ -297,7 +297,7 @@ public:
GoalPtr makeDerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); GoalPtr makeDerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(const StorePath & drvPath, std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(const StorePath & drvPath,
const BasicDerivation & drv, BuildMode buildMode = bmNormal); const BasicDerivation & drv, BuildMode buildMode = bmNormal);
GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair); GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
/* Remove a dead goal. */ /* Remove a dead goal. */
void removeGoal(GoalPtr goal); void removeGoal(GoalPtr goal);
@ -1206,7 +1206,7 @@ void DerivationGoal::haveDerivation()
them. */ them. */
if (settings.useSubstitutes && parsedDrv->substitutesAllowed()) if (settings.useSubstitutes && parsedDrv->substitutesAllowed())
for (auto & i : invalidOutputs) for (auto & i : invalidOutputs)
addWaitee(worker.makeSubstitutionGoal(i, buildMode == bmRepair ? Repair : NoRepair)); addWaitee(worker.makeSubstitutionGoal(i, buildMode == bmRepair ? Repair : NoRepair, getDerivationCA(*drv)));
if (waitees.empty()) /* to prevent hang (no wake-up event) */ if (waitees.empty()) /* to prevent hang (no wake-up event) */
outputsSubstituted(); outputsSubstituted();
@ -4276,6 +4276,10 @@ private:
/* The store path that should be realised through a substitute. */ /* The store path that should be realised through a substitute. */
StorePath storePath; StorePath storePath;
/* The path the substituter refers to the path as. This will be
* different when the stores have different names. */
std::optional<StorePath> subPath;
/* The remaining substituters. */ /* The remaining substituters. */
std::list<ref<Store>> subs; std::list<ref<Store>> subs;
@ -4309,8 +4313,11 @@ private:
typedef void (SubstitutionGoal::*GoalState)(); typedef void (SubstitutionGoal::*GoalState)();
GoalState state; GoalState state;
/* Content address for recomputing store path */
std::optional<ContentAddress> ca;
public: public:
SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair); SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
~SubstitutionGoal(); ~SubstitutionGoal();
void timedOut(Error && ex) override { abort(); }; void timedOut(Error && ex) override { abort(); };
@ -4340,10 +4347,11 @@ public:
}; };
SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair) SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
: Goal(worker) : Goal(worker)
, storePath(storePath) , storePath(storePath)
, repair(repair) , repair(repair)
, ca(ca)
{ {
state = &SubstitutionGoal::init; state = &SubstitutionGoal::init;
name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath)); name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath));
@ -4418,14 +4426,18 @@ void SubstitutionGoal::tryNext()
sub = subs.front(); sub = subs.front();
subs.pop_front(); subs.pop_front();
if (sub->storeDir != worker.store.storeDir) { if (ca) {
subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca);
if (sub->storeDir == worker.store.storeDir)
assert(subPath == storePath);
} else if (sub->storeDir != worker.store.storeDir) {
tryNext(); tryNext();
return; return;
} }
try { try {
// FIXME: make async // FIXME: make async
info = sub->queryPathInfo(storePath); info = sub->queryPathInfo(subPath ? *subPath : storePath);
} catch (InvalidPath &) { } catch (InvalidPath &) {
tryNext(); tryNext();
return; return;
@ -4444,6 +4456,19 @@ void SubstitutionGoal::tryNext()
throw; throw;
} }
if (info->path != storePath) {
if (info->isContentAddressed(*sub) && info->references.empty()) {
auto info2 = std::make_shared<ValidPathInfo>(*info);
info2->path = storePath;
info = info2;
} else {
printError("asked '%s' for '%s' but got '%s'",
sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path));
tryNext();
return;
}
}
/* Update the total expected download size. */ /* Update the total expected download size. */
auto narInfo = std::dynamic_pointer_cast<const NarInfo>(info); auto narInfo = std::dynamic_pointer_cast<const NarInfo>(info);
@ -4533,7 +4558,7 @@ void SubstitutionGoal::tryToRun()
PushActivity pact(act.id); PushActivity pact(act.id);
copyStorePath(ref<Store>(sub), ref<Store>(worker.store.shared_from_this()), copyStorePath(ref<Store>(sub), ref<Store>(worker.store.shared_from_this()),
storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs);
promise.set_value(); promise.set_value();
} catch (...) { } catch (...) {
@ -4666,11 +4691,11 @@ std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath
} }
GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair) GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca)
{ {
GoalPtr goal = substitutionGoals[path].lock(); // FIXME GoalPtr goal = substitutionGoals[path].lock(); // FIXME
if (!goal) { if (!goal) {
goal = std::make_shared<SubstitutionGoal>(path, *this, repair); goal = std::make_shared<SubstitutionGoal>(path, *this, repair, ca);
substitutionGoals.insert_or_assign(path, goal); substitutionGoals.insert_or_assign(path, goal);
wakeUp(goal); wakeUp(goal);
} }

View file

@ -58,26 +58,6 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
} }
}; };
/* We always have one output, and if it's a fixed-output derivation (as
checked below) it must be the only output */
auto & output = drv.outputs.begin()->second;
/* Try the hashed mirrors first. */
if (auto hash = std::get_if<DerivationOutputFixed>(&output.output)) {
if (hash->hash.method == FileIngestionMethod::Flat) {
for (auto hashedMirror : settings.hashedMirrors.get()) {
try {
if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
fetch(hashedMirror + printHashType(hash->hash.hash.type) + "/" + hash->hash.hash.to_string(Base16, false));
return;
} catch (Error & e) {
debug(e.what());
}
}
}
}
/* Otherwise try the specified URL. */
fetch(mainUrl); fetch(mainUrl);
} }

View file

@ -579,7 +579,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
auto path = store->parseStorePath(readString(from)); auto path = store->parseStorePath(readString(from));
logger->startWork(); logger->startWork();
SubstitutablePathInfos infos; SubstitutablePathInfos infos;
store->querySubstitutablePathInfos({path}, infos); store->querySubstitutablePathInfos({{path, std::nullopt}}, infos);
logger->stopWork(); logger->stopWork();
auto i = infos.find(path); auto i = infos.find(path);
if (i == infos.end()) if (i == infos.end())
@ -595,10 +595,16 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
} }
case wopQuerySubstitutablePathInfos: { case wopQuerySubstitutablePathInfos: {
auto paths = readStorePaths<StorePathSet>(*store, from);
logger->startWork();
SubstitutablePathInfos infos; SubstitutablePathInfos infos;
store->querySubstitutablePathInfos(paths, infos); StorePathCAMap pathsMap = {};
if (GET_PROTOCOL_MINOR(clientVersion) < 22) {
auto paths = readStorePaths<StorePathSet>(*store, from);
for (auto & path : paths)
pathsMap.emplace(path, std::nullopt);
} else
pathsMap = readStorePathCAMap(*store, from);
logger->startWork();
store->querySubstitutablePathInfos(pathsMap, infos);
logger->stopWork(); logger->stopWork();
to << infos.size(); to << infos.size();
for (auto & i : infos) { for (auto & i : infos) {

View file

@ -335,9 +335,6 @@ public:
"setuid/setgid bits or with file capabilities."}; "setuid/setgid bits or with file capabilities."};
#endif #endif
Setting<Strings> hashedMirrors{this, {"http://tarballs.nixos.org/"}, "hashed-mirrors",
"A list of servers used by builtins.fetchurl to fetch files by hash."};
Setting<uint64_t> minFree{this, 0, "min-free", Setting<uint64_t> minFree{this, 0, "min-free",
"Automatically run the garbage collector when free disk space drops below the specified amount."}; "Automatically run the garbage collector when free disk space drops below the specified amount."};

View file

@ -839,20 +839,32 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths)
} }
void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos)
SubstitutablePathInfos & infos)
{ {
if (!settings.useSubstitutes) return; if (!settings.useSubstitutes) return;
for (auto & sub : getDefaultSubstituters()) { for (auto & sub : getDefaultSubstituters()) {
if (sub->storeDir != storeDir) continue;
for (auto & path : paths) { for (auto & path : paths) {
if (infos.count(path)) continue; auto subPath(path.first);
debug("checking substituter '%s' for path '%s'", sub->getUri(), printStorePath(path));
// recompute store path so that we can use a different store root
if (path.second) {
subPath = makeFixedOutputPathFromCA(path.first.name(), *path.second);
if (sub->storeDir == storeDir)
assert(subPath == path.first);
if (subPath != path.first)
debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(path.first), sub->printStorePath(subPath), sub->getUri());
} else if (sub->storeDir != storeDir) continue;
debug("checking substituter '%s' for path '%s'", sub->getUri(), sub->printStorePath(subPath));
try { try {
auto info = sub->queryPathInfo(path); auto info = sub->queryPathInfo(subPath);
if (sub->storeDir != storeDir && !(info->isContentAddressed(*sub) && info->references.empty()))
continue;
auto narInfo = std::dynamic_pointer_cast<const NarInfo>( auto narInfo = std::dynamic_pointer_cast<const NarInfo>(
std::shared_ptr<const ValidPathInfo>(info)); std::shared_ptr<const ValidPathInfo>(info));
infos.insert_or_assign(path, SubstitutablePathInfo{ infos.insert_or_assign(path.first, SubstitutablePathInfo{
info->deriver, info->deriver,
info->references, info->references,
narInfo ? narInfo->fileSize : 0, narInfo ? narInfo->fileSize : 0,

View file

@ -139,7 +139,7 @@ public:
StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; StorePathSet querySubstitutablePaths(const StorePathSet & paths) override;
void querySubstitutablePathInfos(const StorePathSet & paths, void querySubstitutablePathInfos(const StorePathCAMap & paths,
SubstitutablePathInfos & infos) override; SubstitutablePathInfos & infos) override;
void addToStore(const ValidPathInfo & info, Source & source, void addToStore(const ValidPathInfo & info, Source & source,

View file

@ -108,6 +108,16 @@ void Store::computeFSClosure(const StorePath & startPath,
} }
std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv)
{
auto out = drv.outputs.find("out");
if (out != drv.outputs.end()) {
if (auto v = std::get_if<DerivationOutputFixed>(&out->second.output))
return v->hash;
}
return std::nullopt;
}
void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets, void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
StorePathSet & willBuild_, StorePathSet & willSubstitute_, StorePathSet & unknown_, StorePathSet & willBuild_, StorePathSet & willSubstitute_, StorePathSet & unknown_,
uint64_t & downloadSize_, uint64_t & narSize_) uint64_t & downloadSize_, uint64_t & narSize_)
@ -157,7 +167,7 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
auto outPath = parseStorePath(outPathS); auto outPath = parseStorePath(outPathS);
SubstitutablePathInfos infos; SubstitutablePathInfos infos;
querySubstitutablePathInfos({outPath}, infos); querySubstitutablePathInfos({{outPath, getDerivationCA(*drv)}}, infos);
if (infos.empty()) { if (infos.empty()) {
drvState_->lock()->done = true; drvState_->lock()->done = true;
@ -214,7 +224,7 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
if (isValidPath(path.path)) return; if (isValidPath(path.path)) return;
SubstitutablePathInfos infos; SubstitutablePathInfos infos;
querySubstitutablePathInfos({path.path}, infos); querySubstitutablePathInfos({{path.path, std::nullopt}}, infos);
if (infos.empty()) { if (infos.empty()) {
auto state(state_.lock()); auto state(state_.lock());

View file

@ -64,6 +64,8 @@ typedef std::set<StorePath> StorePathSet;
typedef std::vector<StorePath> StorePaths; typedef std::vector<StorePath> StorePaths;
typedef std::map<string, StorePath> OutputPathMap; typedef std::map<string, StorePath> OutputPathMap;
typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap;
/* Extension of derivations in the Nix store. */ /* Extension of derivations in the Nix store. */
const std::string drvExtension = ".drv"; const std::string drvExtension = ".drv";

View file

@ -39,6 +39,24 @@ void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths
out << store.printStorePath(i); out << store.printStorePath(i);
} }
StorePathCAMap readStorePathCAMap(const Store & store, Source & from)
{
StorePathCAMap paths;
auto count = readNum<size_t>(from);
while (count--)
paths.insert_or_assign(store.parseStorePath(readString(from)), parseContentAddressOpt(readString(from)));
return paths;
}
void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap & paths)
{
out << paths.size();
for (auto & i : paths) {
out << store.printStorePath(i.first);
out << renderContentAddress(i.second);
}
}
std::map<string, StorePath> readOutputPathMap(const Store & store, Source & from) std::map<string, StorePath> readOutputPathMap(const Store & store, Source & from)
{ {
std::map<string, StorePath> pathMap; std::map<string, StorePath> pathMap;
@ -332,18 +350,17 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths)
} }
void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, SubstitutablePathInfos & infos)
SubstitutablePathInfos & infos)
{ {
if (paths.empty()) return; if (pathsMap.empty()) return;
auto conn(getConnection()); auto conn(getConnection());
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) {
for (auto & i : paths) { for (auto & i : pathsMap) {
SubstitutablePathInfo info; SubstitutablePathInfo info;
conn->to << wopQuerySubstitutablePathInfo << printStorePath(i); conn->to << wopQuerySubstitutablePathInfo << printStorePath(i.first);
conn.processStderr(); conn.processStderr();
unsigned int reply = readInt(conn->from); unsigned int reply = readInt(conn->from);
if (reply == 0) continue; if (reply == 0) continue;
@ -353,13 +370,19 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths,
info.references = readStorePaths<StorePathSet>(*this, conn->from); info.references = readStorePaths<StorePathSet>(*this, conn->from);
info.downloadSize = readLongLong(conn->from); info.downloadSize = readLongLong(conn->from);
info.narSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from);
infos.insert_or_assign(i, std::move(info)); infos.insert_or_assign(i.first, std::move(info));
} }
} else { } else {
conn->to << wopQuerySubstitutablePathInfos; conn->to << wopQuerySubstitutablePathInfos;
writeStorePaths(*this, conn->to, paths); if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 22) {
StorePathSet paths;
for (auto & path : pathsMap)
paths.insert(path.first);
writeStorePaths(*this, conn->to, paths);
} else
writeStorePathCAMap(*this, conn->to, pathsMap);
conn.processStderr(); conn.processStderr();
size_t count = readNum<size_t>(conn->from); size_t count = readNum<size_t>(conn->from);
for (size_t n = 0; n < count; n++) { for (size_t n = 0; n < count; n++) {

View file

@ -56,7 +56,7 @@ public:
StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; StorePathSet querySubstitutablePaths(const StorePathSet & paths) override;
void querySubstitutablePathInfos(const StorePathSet & paths, void querySubstitutablePathInfos(const StorePathCAMap & paths,
SubstitutablePathInfos & infos) override; SubstitutablePathInfos & infos) override;
void addToStore(const ValidPathInfo & info, Source & nar, void addToStore(const ValidPathInfo & info, Source & nar,

View file

@ -193,6 +193,23 @@ StorePath Store::makeFixedOutputPath(
} }
} }
// FIXME Put this somewhere?
template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
StorePath Store::makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca,
const StorePathSet & references, bool hasSelfReference) const
{
// New template
return std::visit(overloaded {
[&](TextHash th) {
return makeTextPath(name, th.hash, references);
},
[&](FixedOutputHash fsh) {
return makeFixedOutputPath(fsh.method, fsh.hash, name, references, hasSelfReference);
}
}, ca);
}
StorePath Store::makeTextPath(std::string_view name, const Hash & hash, StorePath Store::makeTextPath(std::string_view name, const Hash & hash,
const StorePathSet & references) const const StorePathSet & references) const
@ -689,6 +706,15 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
uint64_t total = 0; uint64_t total = 0;
// recompute store path on the chance dstStore does it differently
if (info->ca && info->references.empty()) {
auto info2 = make_ref<ValidPathInfo>(*info);
info2->path = dstStore->makeFixedOutputPathFromCA(info->path.name(), *info->ca);
if (dstStore->storeDir == srcStore->storeDir)
assert(info->path == info2->path);
info = info2;
}
if (!info->narHash) { if (!info->narHash) {
StringSink sink; StringSink sink;
srcStore->narFromPath({storePath}, sink); srcStore->narFromPath({storePath}, sink);
@ -724,7 +750,7 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
} }
void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & storePaths, std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & storePaths,
RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute) RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute)
{ {
auto valid = dstStore->queryValidPaths(storePaths, substitute); auto valid = dstStore->queryValidPaths(storePaths, substitute);
@ -733,7 +759,11 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & st
for (auto & path : storePaths) for (auto & path : storePaths)
if (!valid.count(path)) missing.insert(path); if (!valid.count(path)) missing.insert(path);
if (missing.empty()) return; std::map<StorePath, StorePath> pathsMap;
for (auto & path : storePaths)
pathsMap.insert_or_assign(path, path);
if (missing.empty()) return pathsMap;
Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size())); Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size()));
@ -752,14 +782,23 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & st
StorePathSet(missing.begin(), missing.end()), StorePathSet(missing.begin(), missing.end()),
[&](const StorePath & storePath) { [&](const StorePath & storePath) {
auto info = srcStore->queryPathInfo(storePath);
auto storePathForDst = storePath;
if (info->ca && info->references.empty()) {
storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca);
if (dstStore->storeDir == srcStore->storeDir)
assert(storePathForDst == storePath);
if (storePathForDst != storePath)
debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri());
}
pathsMap.insert_or_assign(storePath, storePathForDst);
if (dstStore->isValidPath(storePath)) { if (dstStore->isValidPath(storePath)) {
nrDone++; nrDone++;
showProgress(); showProgress();
return StorePathSet(); return StorePathSet();
} }
auto info = srcStore->queryPathInfo(storePath);
bytesExpected += info->narSize; bytesExpected += info->narSize;
act.setExpected(actCopyPath, bytesExpected); act.setExpected(actCopyPath, bytesExpected);
@ -769,7 +808,19 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & st
[&](const StorePath & storePath) { [&](const StorePath & storePath) {
checkInterrupt(); checkInterrupt();
if (!dstStore->isValidPath(storePath)) { auto info = srcStore->queryPathInfo(storePath);
auto storePathForDst = storePath;
if (info->ca && info->references.empty()) {
storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca);
if (dstStore->storeDir == srcStore->storeDir)
assert(storePathForDst == storePath);
if (storePathForDst != storePath)
debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri());
}
pathsMap.insert_or_assign(storePath, storePathForDst);
if (!dstStore->isValidPath(storePathForDst)) {
MaintainCount<decltype(nrRunning)> mc(nrRunning); MaintainCount<decltype(nrRunning)> mc(nrRunning);
showProgress(); showProgress();
try { try {
@ -787,6 +838,8 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & st
nrDone++; nrDone++;
showProgress(); showProgress();
}); });
return pathsMap;
} }
@ -863,10 +916,6 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey)
sigs.insert(secretKey.signDetached(fingerprint(store))); sigs.insert(secretKey.signDetached(fingerprint(store)));
} }
// FIXME Put this somewhere?
template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
bool ValidPathInfo::isContentAddressed(const Store & store) const bool ValidPathInfo::isContentAddressed(const Store & store) const
{ {
if (! ca) return false; if (! ca) return false;

View file

@ -344,7 +344,11 @@ public:
bool hasSelfReference = false) const; bool hasSelfReference = false) const;
StorePath makeTextPath(std::string_view name, const Hash & hash, StorePath makeTextPath(std::string_view name, const Hash & hash,
const StorePathSet & references) const; const StorePathSet & references = {}) const;
StorePath makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca,
const StorePathSet & references = {},
bool hasSelfReference = false) const;
/* This is the preparatory part of addToStore(); it computes the /* This is the preparatory part of addToStore(); it computes the
store path to which srcPath is to be copied. Returns the store store path to which srcPath is to be copied. Returns the store
@ -436,9 +440,10 @@ public:
virtual StorePathSet querySubstitutablePaths(const StorePathSet & paths) { return {}; }; virtual StorePathSet querySubstitutablePaths(const StorePathSet & paths) { return {}; };
/* Query substitute info (i.e. references, derivers and download /* Query substitute info (i.e. references, derivers and download
sizes) of a set of paths. If a path does not have substitute sizes) of a map of paths to their optional ca values. If a path
info, it's omitted from the resulting infos map. */ does not have substitute info, it's omitted from the resulting
virtual void querySubstitutablePathInfos(const StorePathSet & paths, infos map. */
virtual void querySubstitutablePathInfos(const StorePathCAMap & paths,
SubstitutablePathInfos & infos) { return; }; SubstitutablePathInfos & infos) { return; };
/* Import a path into the store. */ /* Import a path into the store. */
@ -740,11 +745,13 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
/* Copy store paths from one store to another. The paths may be copied /* Copy store paths from one store to another. The paths may be copied
in parallel. They are copied in a topologically sorted order in parallel. They are copied in a topologically sorted order (i.e.
(i.e. if A is a reference of B, then A is copied before B), but if A is a reference of B, then A is copied before B), but the set
the set of store paths is not automatically closed; use of store paths is not automatically closed; use copyClosure() for
copyClosure() for that. */ that. Returns a map of what each path was copied to the dstStore
void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & storePaths, as. */
std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore,
const StorePathSet & storePaths,
RepairFlag repair = NoRepair, RepairFlag repair = NoRepair,
CheckSigsFlag checkSigs = CheckSigs, CheckSigsFlag checkSigs = CheckSigs,
SubstituteFlag substitute = NoSubstitute); SubstituteFlag substitute = NoSubstitute);
@ -843,4 +850,6 @@ std::optional<ValidPathInfo> decodeValidPathInfo(
/* Split URI into protocol+hierarchy part and its parameter set. */ /* Split URI into protocol+hierarchy part and its parameter set. */
std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri); std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri);
std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv);
} }

View file

@ -70,6 +70,10 @@ template<class T> T readStorePaths(const Store & store, Source & from);
void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths); void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths);
StorePathCAMap readStorePathCAMap(const Store & store, Source & from);
void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap & paths);
void writeOutputPathMap(const Store & store, Sink & out, const OutputPathMap & paths); void writeOutputPathMap(const Store & store, Sink & out, const OutputPathMap & paths);
} }

View file

@ -9,6 +9,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand
{ {
Path path; Path path;
std::optional<std::string> namePart; std::optional<std::string> namePart;
FileIngestionMethod ingestionMethod = FileIngestionMethod::Recursive;
CmdAddToStore() CmdAddToStore()
{ {
@ -21,6 +22,13 @@ struct CmdAddToStore : MixDryRun, StoreCommand
.labels = {"name"}, .labels = {"name"},
.handler = {&namePart}, .handler = {&namePart},
}); });
addFlag({
.longName = "flat",
.shortName = 0,
.description = "add flat file to the Nix store",
.handler = {&ingestionMethod, FileIngestionMethod::Flat},
});
} }
std::string description() override std::string description() override
@ -45,12 +53,19 @@ struct CmdAddToStore : MixDryRun, StoreCommand
auto narHash = hashString(htSHA256, *sink.s); auto narHash = hashString(htSHA256, *sink.s);
ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, *namePart)); Hash hash = narHash;
if (ingestionMethod == FileIngestionMethod::Flat) {
HashSink hsink(htSHA256);
readFile(path, hsink);
hash = hsink.finish().first;
}
ValidPathInfo info(store->makeFixedOutputPath(ingestionMethod, hash, *namePart));
info.narHash = narHash; info.narHash = narHash;
info.narSize = sink.s->size(); info.narSize = sink.s->size();
info.ca = std::optional { FixedOutputHash { info.ca = std::optional { FixedOutputHash {
.method = FileIngestionMethod::Recursive, .method = ingestionMethod,
.hash = *info.narHash, .hash = hash,
} }; } };
if (!dryRun) { if (!dryRun) {

View file

@ -61,30 +61,30 @@ nix-build check.nix -A nondeterministic --no-out-link --repeat 1 2> $TEST_ROOT/l
[ "$status" = "1" ] [ "$status" = "1" ]
grep 'differs from previous round' $TEST_ROOT/log grep 'differs from previous round' $TEST_ROOT/log
path=$(nix-build check.nix -A fetchurl --no-out-link --hashed-mirrors '') path=$(nix-build check.nix -A fetchurl --no-out-link)
chmod +w $path chmod +w $path
echo foo > $path echo foo > $path
chmod -w $path chmod -w $path
nix-build check.nix -A fetchurl --no-out-link --check --hashed-mirrors '' nix-build check.nix -A fetchurl --no-out-link --check
# Note: "check" doesn't repair anything, it just compares to the hash stored in the database. # Note: "check" doesn't repair anything, it just compares to the hash stored in the database.
[[ $(cat $path) = foo ]] [[ $(cat $path) = foo ]]
nix-build check.nix -A fetchurl --no-out-link --repair --hashed-mirrors '' nix-build check.nix -A fetchurl --no-out-link --repair
[[ $(cat $path) != foo ]] [[ $(cat $path) != foo ]]
nix-build check.nix -A hashmismatch --no-out-link --hashed-mirrors '' || status=$? nix-build check.nix -A hashmismatch --no-out-link || status=$?
[ "$status" = "102" ] [ "$status" = "102" ]
echo -n > ./dummy echo -n > ./dummy
nix-build check.nix -A hashmismatch --no-out-link --hashed-mirrors '' nix-build check.nix -A hashmismatch --no-out-link
echo 'Hello World' > ./dummy echo 'Hello World' > ./dummy
nix-build check.nix -A hashmismatch --no-out-link --check --hashed-mirrors '' || status=$? nix-build check.nix -A hashmismatch --no-out-link --check || status=$?
[ "$status" = "102" ] [ "$status" = "102" ]
# Multiple failures with --keep-going # Multiple failures with --keep-going
nix-build check.nix -A nondeterministic --no-out-link nix-build check.nix -A nondeterministic --no-out-link
nix-build check.nix -A nondeterministic -A hashmismatch --no-out-link --check --keep-going --hashed-mirrors '' || status=$? nix-build check.nix -A nondeterministic -A hashmismatch --no-out-link --check --keep-going || status=$?
[ "$status" = "110" ] [ "$status" = "110" ]

View file

@ -5,7 +5,7 @@ clearStore
# Test fetching a flat file. # Test fetching a flat file.
hash=$(nix-hash --flat --type sha256 ./fetchurl.sh) hash=$(nix-hash --flat --type sha256 ./fetchurl.sh)
outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr sha256 $hash --no-out-link --hashed-mirrors '') outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr sha256 $hash --no-out-link)
cmp $outPath fetchurl.sh cmp $outPath fetchurl.sh
@ -14,7 +14,7 @@ clearStore
hash=$(nix hash-file --type sha512 --base64 ./fetchurl.sh) hash=$(nix hash-file --type sha512 --base64 ./fetchurl.sh)
outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr sha512 $hash --no-out-link --hashed-mirrors '') outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr sha512 $hash --no-out-link)
cmp $outPath fetchurl.sh cmp $outPath fetchurl.sh
@ -25,26 +25,24 @@ hash=$(nix hash-file ./fetchurl.sh)
[[ $hash =~ ^sha256- ]] [[ $hash =~ ^sha256- ]]
outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr hash $hash --no-out-link --hashed-mirrors '') outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr hash $hash --no-out-link)
cmp $outPath fetchurl.sh cmp $outPath fetchurl.sh
# Test the hashed mirror feature. # Test that we can substitute from a different store dir.
clearStore clearStore
hash=$(nix hash-file --type sha512 --base64 ./fetchurl.sh) other_store=file://$TEST_ROOT/other_store?store=/fnord/store
hash32=$(nix hash-file --type sha512 --base16 ./fetchurl.sh)
mirror=$TEST_ROOT/hashed-mirror hash=$(nix hash-file --type sha256 --base16 ./fetchurl.sh)
rm -rf $mirror
mkdir -p $mirror/sha512
ln -s $(pwd)/fetchurl.sh $mirror/sha512/$hash32
outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha512 $hash --no-out-link --hashed-mirrors "file://$mirror") storePath=$(nix --store $other_store add-to-store --flat ./fetchurl.sh)
outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 $hash --no-out-link --substituters $other_store)
# Test hashed mirrors with an SRI hash. # Test hashed mirrors with an SRI hash.
nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix to-sri --type sha512 $hash) \ nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix to-sri --type sha256 $hash) \
--argstr name bla --no-out-link --hashed-mirrors "file://$mirror" --no-out-link --substituters $other_store
# Test unpacking a NAR. # Test unpacking a NAR.
rm -rf $TEST_ROOT/archive rm -rf $TEST_ROOT/archive