lix-project/lix-releng-staging
0
0
Fork 0
Code Packages Activity

Use PathReferences more widely

Browse source
This commit is contained in:
John Ericson 2020-10-07 13:52:20 +00:00
parent dae4409071
commit f8d562c0a7
29 changed files with 431 additions and 205 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
perl/lib/Nix/Store.xs
View file

@ -111,7 +111,7 @@ SV * queryPathInfo(char * path, int base32)
mXPUSHi(info->registrationTime);
mXPUSHi(info->narSize);
AV * arr = newAV();
for (auto & i : info->references)
for (auto & i : info->referencesPossiblyToSelf())
av_push(arr, newSVpv(store()->printStorePath(i).c_str(), 0));
XPUSHs(sv_2mortal(newRV((SV *) arr)));
} catch (Error & e) {
@ -287,7 +287,13 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name)
try {
auto h = Hash::parseAny(hash, parseHashType(algo));
auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
auto path = store()->makeFixedOutputPath(method, h, name);
auto path = store()->makeFixedOutputPath(name, FixedOutputInfo {
{
.method = method,
.hash = h,
},
{},
});
XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0)));
} catch (Error & e) {
croak("%s", e.what());

16
src/libexpr/primops.cc
View file

@ -1045,7 +1045,13 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
std::optional<HashType> ht = parseHashTypeOpt(outputHashAlgo);
Hash h = newHashAllowEmpty(*outputHash, ht);
auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName);
auto outPath = state.store->makeFixedOutputPath(drvName, FixedOutputInfo {
{
.method = ingestionMethod,
.hash = h,
},
{},
});
drv.env["out"] = state.store->printStorePath(outPath);
drv.outputs.insert_or_assign("out", DerivationOutput {
.output = DerivationOutputCAFixed {
@ -1764,7 +1770,13 @@ static void addPath(EvalState & state, const Pos & pos, const string & name, con
std::optional<StorePath> expectedStorePath;
if (expectedHash)
expectedStorePath = state.store->makeFixedOutputPath(method, *expectedHash, name);
expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputInfo {
{
.method = method,
.hash = *expectedHash,
},
{},
});
Path dstPath;
if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
dstPath = state.store->printStorePath(settings.readOnlyMode

2
src/libexpr/value-to-xml.hh
View file

@ -10,5 +10,5 @@ namespace nix {
void printValueAsXML(EvalState & state, bool strict, bool location,
Value & v, std::ostream & out, PathSet & context);
}

8
src/libfetchers/fetchers.cc
View file

@ -198,7 +198,13 @@ StorePath Input::computeStorePath(Store & store) const
auto narHash = getNarHash();
if (!narHash)
throw Error("cannot compute store path for mutable input '%s'", to_string());
return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, "source");
return store.makeFixedOutputPath("source", FixedOutputInfo {
{
.method = FileIngestionMethod::Recursive,
.hash = *narHash,
},
{},
});
}
std::string Input::getType() const

16
src/libfetchers/tarball.cc
View file

@ -71,14 +71,20 @@ DownloadFileResult downloadFile(
dumpString(*res.data, sink);
auto hash = hashString(htSHA256, *res.data);
ValidPathInfo info {
store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name),
*store,
{
.name = name,
.info = FixedOutputInfo {
{
.method = FileIngestionMethod::Flat,
.hash = hash,
},
{},
},
},
hashString(htSHA256, *sink.s),
};
info.narSize = sink.s->size();
info.ca = FixedOutputHash {
.method = FileIngestionMethod::Flat,
.hash = hash,
};
auto source = StringSource { *sink.s };
store->addToStore(info, source, NoRepair, NoCheckSigs);
storePath = std::move(info.path);

43
src/libstore/binary-cache-store.cc
View file

@ -322,7 +322,17 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, const string & nam
unsupported("addToStoreFromDump");
return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) {
ValidPathInfo info {
makeFixedOutputPath(method, nar.first, name),
*this,
{
.name = name,
.info = FixedOutputInfo {
{
.method = method,
.hash = nar.first,
},
{},
},
},
nar.first,
};
info.narSize = nar.second;
@ -412,14 +422,20 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
});
return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) {
ValidPathInfo info {
makeFixedOutputPath(method, h, name),
*this,
{
.name = name,
.info = FixedOutputInfo {
{
.method = method,
.hash = h,
},
{},
},
},
nar.first,
};
info.narSize = nar.second;
info.ca = FixedOutputHash {
.method = method,
.hash = h,
};
return info;
})->path;
}
@ -428,17 +444,26 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
const StorePathSet & references, RepairFlag repair)
{
auto textHash = hashString(htSHA256, s);
auto path = makeTextPath(name, textHash, references);
auto path = makeTextPath(name, TextInfo { textHash, references });
if (!repair && isValidPath(path))
return path;
auto source = StringSource { s };
return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) {
ValidPathInfo info { path, nar.first };
ValidPathInfo info {
*this,
{
.name = name,
.info = TextInfo {
{ .hash = textHash },
references,
},
},
nar.first,
};
info.narSize = nar.second;
info.ca = TextHash { textHash };
info.references = references;
return info;
})->path;
}

30
src/libstore/build.cc
View file

@ -4056,25 +4056,24 @@ void DerivationGoal::registerOutputs()
break;
}
auto got = caSink.finish().first;
auto refs = rewriteRefs();
HashModuloSink narSink { htSHA256, oldHashPart };
dumpPath(actualPath, narSink);
auto narHashAndSize = narSink.finish();
ValidPathInfo newInfo0 {
worker.store.makeFixedOutputPath(
outputHash.method,
got,
outputPathName(drv->name, outputName),
refs.references,
refs.hasSelfReference),
worker.store,
{
.name = outputPathName(drv->name, outputName),
.info = FixedOutputInfo {
{
.method = outputHash.method,
.hash = got,
},
rewriteRefs(),
},
},
narHashAndSize.first,
};
newInfo0.narSize = narHashAndSize.second;
newInfo0.ca = FixedOutputHash {
.method = outputHash.method,
.hash = got,
};
static_cast<PathReferences<StorePath> &>(newInfo0) = refs;
assert(newInfo0.ca);
return newInfo0;
@ -4861,7 +4860,10 @@ void SubstitutionGoal::tryNext()
subs.pop_front();
if (ca) {
subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca);
subPath = sub->makeFixedOutputPathFromCA({
.name = std::string { storePath.name() },
.info = caWithoutRefs(*ca),
});
if (sub->storeDir == worker.store.storeDir)
assert(subPath == storePath);
} else if (sub->storeDir != worker.store.storeDir) {
@ -4891,7 +4893,7 @@ void SubstitutionGoal::tryNext()
}
if (info->path != storePath) {
if (info->isContentAddressed(*sub) && info->references.empty()) {
if (info->isContentAddressed(*sub) && info->references.empty() && !info->hasSelfReference) {
auto info2 = std::make_shared<ValidPathInfo>(*info);
info2->path = storePath;
info = info2;

24
src/libstore/content-address.cc
View file

@ -9,6 +9,7 @@ std::string FixedOutputHash::printMethodAlgo() const
return makeFileIngestionPrefix(method) + printHashType(hash.type);
}
std::string makeFileIngestionPrefix(const FileIngestionMethod m)
{
switch (m) {
@ -16,9 +17,8 @@ std::string makeFileIngestionPrefix(const FileIngestionMethod m)
return "";
case FileIngestionMethod::Recursive:
return "r:";
default:
throw Error("impossible, caught both cases");
}
assert(false);
}
std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash)
@ -32,10 +32,13 @@ std::string renderContentAddress(ContentAddress ca)
{
return std::visit(overloaded {
[](TextHash th) {
return "text:" + th.hash.to_string(Base32, true);
return "text:"
+ th.hash.to_string(Base32, true);
},
[](FixedOutputHash fsh) {
return makeFixedOutputCA(fsh.method, fsh.hash);
return "fixed:"
+ makeFileIngestionPrefix(fsh.method)
+ fsh.hash.to_string(Base32, true);
}
}, ca);
}
@ -142,7 +145,18 @@ Hash getContentAddressHash(const ContentAddress & ca)
},
[](FixedOutputHash fsh) {
return fsh.hash;
}
},
}, ca);
}
ContentAddressWithReferences caWithoutRefs(const ContentAddress & ca) {
return std::visit(overloaded {
[&](TextHash h) -> ContentAddressWithReferences {
return TextInfo { h, {}};
},
[&](FixedOutputHash h) -> ContentAddressWithReferences {
return FixedOutputInfo { h, {}};
},
}, ca);
}

102
src/libstore/content-address.hh
View file

@ -2,14 +2,20 @@
#include <variant>
#include "hash.hh"
#include "path.hh"
namespace nix {
/*
* Mini content address
*/
enum struct FileIngestionMethod : uint8_t {
Flat = false,
Recursive = true
};
struct TextHash {
Hash hash;
};
@ -41,10 +47,6 @@ typedef std::variant<
ingested. */
std::string makeFileIngestionPrefix(const FileIngestionMethod m);
/* Compute the content-addressability assertion (ValidPathInfo::ca)
for paths created by makeFixedOutputPath() / addToStore(). */
std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash);
std::string renderContentAddress(ContentAddress ca);
std::string renderContentAddress(std::optional<ContentAddress> ca);
@ -74,4 +76,96 @@ ContentAddressMethod parseContentAddressMethod(std::string_view rawCaMethod);
std::string renderContentAddressMethod(ContentAddressMethod caMethod);
/*
* References set
*/
template<typename Ref>
struct PathReferences
{
std::set<Ref> references;
bool hasSelfReference = false;
bool operator == (const PathReferences<Ref> & other) const
{
return references == other.references
&& hasSelfReference == other.hasSelfReference;
}
/* Functions to view references + hasSelfReference as one set, mainly for
compatibility's sake. */
StorePathSet referencesPossiblyToSelf(const Ref & self) const;
void insertReferencePossiblyToSelf(const Ref & self, Ref && ref);
void setReferencesPossiblyToSelf(const Ref & self, std::set<Ref> && refs);
};
template<typename Ref>
StorePathSet PathReferences<Ref>::referencesPossiblyToSelf(const Ref & self) const
{
StorePathSet refs { references };
if (hasSelfReference)
refs.insert(self);
return refs;
}
template<typename Ref>
void PathReferences<Ref>::insertReferencePossiblyToSelf(const Ref & self, Ref && ref)
{
if (ref == self)
hasSelfReference = true;
else
references.insert(std::move(ref));
}
template<typename Ref>
void PathReferences<Ref>::setReferencesPossiblyToSelf(const Ref & self, std::set<Ref> && refs)
{
if (refs.count(self))
hasSelfReference = true;
refs.erase(self);
references = refs;
}
/*
* Full content address
*
* See the schema for store paths in store-api.cc
*/
// This matches the additional info that we need for makeTextPath
struct TextInfo : TextHash {
// References for the paths, self references disallowed
StorePathSet references;
};
struct FixedOutputInfo : FixedOutputHash {
// References for the paths
PathReferences<StorePath> references;
};
typedef std::variant<
TextInfo,
FixedOutputInfo
> ContentAddressWithReferences;
ContentAddressWithReferences caWithoutRefs(const ContentAddress &);
struct StorePathDescriptor {
std::string name;
ContentAddressWithReferences info;
bool operator == (const StorePathDescriptor & other) const
{
return name == other.name;
// FIXME second field
}
bool operator < (const StorePathDescriptor & other) const
{
return name < other.name;
// FIXME second field
}
};
}

4
src/libstore/derivations.cc
View file

@ -27,8 +27,8 @@ std::optional<StorePath> DerivationOutput::path(const Store & store, std::string
StorePath DerivationOutputCAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const {
return store.makeFixedOutputPath(
hash.method, hash.hash,
outputPathName(drvName, outputName));
outputPathName(drvName, outputName),
{ hash, {} });
}

4
src/libstore/derivations.hh
View file

@ -130,8 +130,8 @@ struct Derivation : BasicDerivation
/* Return the underlying basic derivation but with these changes:
1. Input drvs are emptied, but the outputs of them that were used are
added directly to input sources.
1. Input drvs are emptied, but the outputs of them that were used are
added directly to input sources.
2. Input placeholders are replaced with realized input store paths. */
std::optional<BasicDerivation> tryResolve(Store & store);

30
src/libstore/local-store.cc
View file

@ -567,7 +567,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
envHasRightPath(doia.path, i.first);
},
[&](DerivationOutputCAFixed dof) {
StorePath path = makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName);
StorePath path = makeFixedOutputPath(drvName, { dof.hash, {} });
envHasRightPath(path, i.first);
},
[&](DerivationOutputCAFloating _) {
@ -923,7 +923,10 @@ void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, Subst
// recompute store path so that we can use a different store root
if (path.second) {
subPath = makeFixedOutputPathFromCA(path.first.name(), *path.second);
subPath = makeFixedOutputPathFromCA({
.name = std::string { path.first.name() },
.info = caWithoutRefs(*path.second),
});
if (sub->storeDir == storeDir)
assert(subPath == path.first);
if (subPath != path.first)
@ -1164,7 +1167,18 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name,
auto [hash, size] = hashSink->finish();
auto dstPath = makeFixedOutputPath(method, hash, name);
auto desc = StorePathDescriptor {
name,
FixedOutputInfo {
{
.method = method,
.hash = hash,
},
{},
},
};
auto dstPath = makeFixedOutputPathFromCA(desc);
addTempRoot(dstPath);
@ -1184,7 +1198,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name,
autoGC();
if (inMemory) {
StringSource dumpSource { dump };
StringSource dumpSource { dump };
/* Restore from the NAR in memory. */
if (method == FileIngestionMethod::Recursive)
restorePath(realPath, dumpSource);
@ -1209,9 +1223,8 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name,
optimisePath(realPath);
ValidPathInfo info { dstPath, narHash.first };
ValidPathInfo info { *this, std::move(desc), narHash.first };
info.narSize = narHash.second;
info.ca = FixedOutputHash { .method = method, .hash = hash };
registerValidPath(info);
}
@ -1226,7 +1239,10 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair)
{
auto hash = hashString(htSHA256, s);
auto dstPath = makeTextPath(name, hash, references);
auto dstPath = makeTextPath(name, TextInfo {
{ .hash = hash },
references,
});
addTempRoot(dstPath);

3
src/libstore/nar-info.hh
View file

@ -17,6 +17,9 @@ struct NarInfo : ValidPathInfo
std::string system;
NarInfo() = delete;
NarInfo(const Store & store, StorePathDescriptor && ca, Hash narHash)
: ValidPathInfo(store, std::move(ca), narHash)
{ }
NarInfo(StorePath && path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { }
NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { }
NarInfo(const Store & store, const std::string & s, const std::string & whence);

47
src/libstore/path-info.hh
View file

@ -13,47 +13,6 @@ namespace nix {
class Store;
template<typename Ref>
struct PathReferences
{
std::set<Ref> references;
bool hasSelfReference = false;
/* Functions to view references + hasSelfReference as one set, mainly for
compatibility's sake. */
StorePathSet referencesPossiblyToSelf(const Ref & self) const;
void insertReferencePossiblyToSelf(const Ref & self, Ref && ref);
void setReferencesPossiblyToSelf(const Ref & self, std::set<Ref> && refs);
};
template<typename Ref>
StorePathSet PathReferences<Ref>::referencesPossiblyToSelf(const Ref & self) const
{
StorePathSet refs { references };
if (hasSelfReference)
refs.insert(self);
return refs;
}
template<typename Ref>
void PathReferences<Ref>::insertReferencePossiblyToSelf(const Ref & self, Ref && ref)
{
if (ref == self)
hasSelfReference = true;
else
references.insert(std::move(ref));
}
template<typename Ref>
void PathReferences<Ref>::setReferencesPossiblyToSelf(const Ref & self, std::set<Ref> && refs)
{
if (refs.count(self))
hasSelfReference = true;
refs.erase(self);
references = refs;
}
struct SubstitutablePathInfo : PathReferences<StorePath>
{
@ -68,7 +27,6 @@ struct ValidPathInfo : PathReferences<StorePath>
{
StorePath path;
std::optional<StorePath> deriver;
// TODO document this
Hash narHash;
time_t registrationTime = 0;
uint64_t narSize = 0; // 0 = unknown
@ -117,6 +75,8 @@ struct ValidPathInfo : PathReferences<StorePath>
void sign(const Store & store, const SecretKey & secretKey);
std::optional<StorePathDescriptor> fullStorePathDescriptorOpt() const;
/* Return true iff the path is verifiably content-addressed. */
bool isContentAddressed(const Store & store) const;
@ -143,6 +103,9 @@ struct ValidPathInfo : PathReferences<StorePath>
ValidPathInfo(StorePath && path, Hash narHash) : path(std::move(path)), narHash(narHash) { };
ValidPathInfo(const StorePath & path, Hash narHash) : path(path), narHash(narHash) { };
ValidPathInfo(const Store & store,
StorePathDescriptor && ca, Hash narHash);
virtual ~ValidPathInfo() { }
};

5
src/libstore/path.hh
View file

@ -1,6 +1,7 @@
#pragma once
#include "content-address.hh"
#include <string_view>
#include "types.hh"
namespace nix {
@ -64,8 +65,6 @@ typedef std::set<StorePath> StorePathSet;
typedef std::vector<StorePath> StorePaths;
typedef std::map<string, StorePath> OutputPathMap;
typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap;
/* Extension of derivations in the Nix store. */
const std::string drvExtension = ".drv";

171
src/libstore/store-api.cc
View file

@ -7,6 +7,7 @@
#include "thread-pool.hh"
#include "json.hh"
#include "url.hh"
#include "references.hh"
#include "archive.hh"
#include "callback.hh"
@ -163,63 +164,61 @@ StorePath Store::makeOutputPath(std::string_view id,
}
/* Stuff the references (if any) into the type. This is a bit
hacky, but we can't put them in `s' since that would be
ambiguous. */
static std::string makeType(
const Store & store,
string && type,
const StorePathSet & references,
bool hasSelfReference = false)
const PathReferences<StorePath> & references)
{
for (auto & i : references) {
for (auto & i : references.references) {
type += ":";
type += store.printStorePath(i);
}
if (hasSelfReference) type += ":self";
if (references.hasSelfReference) type += ":self";
return std::move(type);
}
StorePath Store::makeFixedOutputPath(
FileIngestionMethod method,
const Hash & hash,
std::string_view name,
const StorePathSet & references,
bool hasSelfReference) const
StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const
{
if (hash.type == htSHA256 && method == FileIngestionMethod::Recursive) {
return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name);
if (info.hash.type == htSHA256 && info.method == FileIngestionMethod::Recursive) {
return makeStorePath(makeType(*this, "source", info.references), info.hash, name);
} else {
assert(references.empty());
assert(info.references.references.size() == 0);
assert(!info.references.hasSelfReference);
return makeStorePath("output:out",
hashString(htSHA256,
"fixed:out:"
+ makeFileIngestionPrefix(method)
+ hash.to_string(Base16, true) + ":"),
+ makeFileIngestionPrefix(info.method)
+ info.hash.to_string(Base16, true) + ":"),
name);
}
}
StorePath Store::makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca,
const StorePathSet & references, bool hasSelfReference) const
StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) const
{
assert(info.hash.type == htSHA256);
return makeStorePath(
makeType(*this, "text", PathReferences<StorePath> { info.references }),
info.hash,
name);
}
StorePath Store::makeFixedOutputPathFromCA(const StorePathDescriptor & desc) const
{
// New template
return std::visit(overloaded {
[&](TextHash th) {
return makeTextPath(name, th.hash, references);
[&](TextInfo ti) {
return makeTextPath(desc.name, ti);
},
[&](FixedOutputHash fsh) {
return makeFixedOutputPath(fsh.method, fsh.hash, name, references, hasSelfReference);
[&](FixedOutputInfo foi) {
return makeFixedOutputPath(desc.name, foi);
}
}, ca);
}
StorePath Store::makeTextPath(std::string_view name, const Hash & hash,
const StorePathSet & references) const
{
assert(hash.type == htSHA256);
/* Stuff the references (if any) into the type. This is a bit
hacky, but we can't put them in `s' since that would be
ambiguous. */
return makeStorePath(makeType(*this, "text", references), hash, name);
}, desc.info);
}
@ -229,14 +228,24 @@ std::pair<StorePath, Hash> Store::computeStorePathForPath(std::string_view name,
Hash h = method == FileIngestionMethod::Recursive
? hashPath(hashAlgo, srcPath, filter).first
: hashFile(hashAlgo, srcPath);
return std::make_pair(makeFixedOutputPath(method, h, name), h);
FixedOutputInfo caInfo {
{
.method = method,
.hash = h,
},
{},
};
return std::make_pair(makeFixedOutputPath(name, caInfo), h);
}
StorePath Store::computeStorePathForText(const string & name, const string & s,
const StorePathSet & references) const
{
return makeTextPath(name, hashString(htSHA256, s), references);
return makeTextPath(name, TextInfo {
{ .hash = hashString(htSHA256, s) },
references,
});
}
@ -326,11 +335,20 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
throw Error("hash mismatch for '%s'", srcPath);
ValidPathInfo info {
makeFixedOutputPath(method, hash, name),
*this,
StorePathDescriptor {
std::string { name },
FixedOutputInfo {
{
.method = method,
.hash = hash,
},
{},
},
},
narHash,
};
info.narSize = narSize;
info.ca = FixedOutputHash { .method = method, .hash = hash };
if (!isValidPath(info.path)) {
auto source = sinkToSource([&](Sink & scratchpadSink) {
@ -496,7 +514,7 @@ void Store::queryPathInfo(const StorePath & storePath,
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
queryPathInfoUncached(storePath,
{[this, storePathS{printStorePath(storePath)}, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) {
{[this, storePath, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) {
try {
auto info = fut.get();
@ -509,11 +527,9 @@ void Store::queryPathInfo(const StorePath & storePath,
state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = info });
}
auto storePath = parseStorePath(storePathS);
if (!info || !goodStorePath(storePath, info->path)) {
stats.narInfoMissing++;
throw InvalidPath("path '%s' is not valid", storePathS);
throw InvalidPath("path '%s' is not valid", printStorePath(storePath));
}
(*callbackPtr)(ref<const ValidPathInfo>(info));
@ -536,13 +552,13 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m
std::condition_variable wakeup;
ThreadPool pool;
auto doQuery = [&](const Path & path) {
auto doQuery = [&](const StorePath & path) {
checkInterrupt();
queryPathInfo(parseStorePath(path), {[path, this, &state_, &wakeup](std::future<ref<const ValidPathInfo>> fut) {
queryPathInfo(path, {[path, this, &state_, &wakeup](std::future<ref<const ValidPathInfo>> fut) {
auto state(state_.lock());
try {
auto info = fut.get();
state->valid.insert(parseStorePath(path));
state->valid.insert(path);
} catch (InvalidPath &) {
} catch (...) {
state->exc = std::current_exception();
@ -554,7 +570,7 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m
};
for (auto & path : paths)
pool.enqueue(std::bind(doQuery, printStorePath(path))); // FIXME
pool.enqueue(std::bind(doQuery, path));
pool.process();
@ -737,7 +753,8 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
// recompute store path on the chance dstStore does it differently
if (info->ca && info->references.empty()) {
auto info2 = make_ref<ValidPathInfo>(*info);
info2->path = dstStore->makeFixedOutputPathFromCA(info->path.name(), *info->ca);
info2->path = dstStore->makeFixedOutputPathFromCA(
info->fullStorePathDescriptorOpt().value());
if (dstStore->storeDir == srcStore->storeDir)
assert(info->path == info2->path);
info = info2;
@ -799,7 +816,8 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
auto info = srcStore->queryPathInfo(storePath);
auto storePathForDst = storePath;
if (info->ca && info->references.empty()) {
storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca);
storePathForDst = dstStore->makeFixedOutputPathFromCA(
info->fullStorePathDescriptorOpt().value());
if (dstStore->storeDir == srcStore->storeDir)
assert(storePathForDst == storePath);
if (storePathForDst != storePath)
@ -826,7 +844,8 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
auto storePathForDst = storePath;
if (info->ca && info->references.empty()) {
storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca);
storePathForDst = dstStore->makeFixedOutputPathFromCA(
info->fullStorePathDescriptorOpt().value());
if (dstStore->storeDir == srcStore->storeDir)
assert(storePathForDst == storePath);
if (storePathForDst != storePath)
@ -947,19 +966,37 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey)
sigs.insert(secretKey.signDetached(fingerprint(store)));
}
std::optional<StorePathDescriptor> ValidPathInfo::fullStorePathDescriptorOpt() const
{
if (! ca)
return std::nullopt;
return StorePathDescriptor {
.name = std::string { path.name() },
.info = std::visit(overloaded {
[&](TextHash th) {
TextInfo info { th };
assert(!hasSelfReference);
info.references = references;
return ContentAddressWithReferences { info };
},
[&](FixedOutputHash foh) {
FixedOutputInfo info { foh };
info.references = static_cast<PathReferences<StorePath>>(*this);
return ContentAddressWithReferences { info };
},
}, *ca),
};
}
bool ValidPathInfo::isContentAddressed(const Store & store) const
{
if (! ca) return false;
auto fullCaOpt = fullStorePathDescriptorOpt();
auto caPath = std::visit(overloaded {
[&](TextHash th) {
assert(!hasSelfReference);
return store.makeTextPath(path.name(), th.hash, references);
},
[&](FixedOutputHash fsh) {
return store.makeFixedOutputPath(fsh.method, fsh.hash, path.name(), references, hasSelfReference);
}
}, *ca);
if (! fullCaOpt)
return false;
auto caPath = store.makeFixedOutputPathFromCA(*fullCaOpt);
bool res = caPath == path;
@ -997,6 +1034,26 @@ Strings ValidPathInfo::shortRefs() const
}
ValidPathInfo::ValidPathInfo(
const Store & store,
StorePathDescriptor && info,
Hash narHash)
: path(store.makeFixedOutputPathFromCA(info))
, narHash(narHash)
{
std::visit(overloaded {
[this](TextInfo ti) {
this->references = ti.references;
this->ca = TextHash { std::move(ti) };
},
[this](FixedOutputInfo foi) {
*(static_cast<PathReferences<StorePath> *>(this)) = foi.references;
this->ca = FixedOutputHash { (FixedOutputHash) std::move(foi) };
},
}, std::move(info.info));
}
Derivation Store::derivationFromPath(const StorePath & drvPath)
{
ensurePath(drvPath);

14
src/libstore/store-api.hh
View file

@ -170,6 +170,8 @@ struct BuildResult
}
};
typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap;
struct StoreConfig : public Config
{
using Config::Config;
@ -313,17 +315,11 @@ public:
StorePath makeOutputPath(std::string_view id,
const Hash & hash, std::string_view name) const;
StorePath makeFixedOutputPath(FileIngestionMethod method,
const Hash & hash, std::string_view name,
const StorePathSet & references = {},
bool hasSelfReference = false) const;
StorePath makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const;
StorePath makeTextPath(std::string_view name, const Hash & hash,
const StorePathSet & references = {}) const;
StorePath makeTextPath(std::string_view name, const TextInfo & info) const;
StorePath makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca,
const StorePathSet & references = {},
bool hasSelfReference = false) const;
StorePath makeFixedOutputPathFromCA(const StorePathDescriptor & info) const;
/* This is the preparatory part of addToStore(); it computes the
store path to which srcPath is to be copied. Returns the store

2
src/libutil/args.cc
View file

@ -244,7 +244,7 @@ nlohmann::json Args::toJSON()
return res;
}
static void hashTypeCompleter(size_t index, std::string_view prefix)
static void hashTypeCompleter(size_t index, std::string_view prefix)
{
for (auto & type : hashTypes)
if (hasPrefix(type, prefix))

3
src/libutil/error.hh
View file

@ -201,9 +201,8 @@ public:
template<typename... Args>
SysError(const Args & ... args)
: Error("")
: Error(""), errNo(errno)
{
errNo = errno;
auto hf = hintfmt(args...);
err.hint = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo));
}

2
src/libutil/fmt.hh
View file

@ -103,7 +103,7 @@ class hintformat
public:
hintformat(const string &format) :fmt(format)
{
fmt.exceptions(boost::io::all_error_bits ^
fmt.exceptions(boost::io::all_error_bits ^
boost::io::too_many_args_bit ^
boost::io::too_few_args_bit);
}

2
src/libutil/tests/logging.cc
View file

@ -370,7 +370,7 @@ namespace nix {
// constructing without access violation.
ErrPos ep(invalid);
// assignment without access violation.
ep = invalid;

1
src/libutil/types.hh
View file

@ -4,6 +4,7 @@
#include <list>
#include <set>
#include <string>
#include <map>
#include <vector>

10
src/nix-prefetch-url/nix-prefetch-url.cc
View file

@ -161,8 +161,14 @@ static int _main(int argc, char * * argv)
std::optional<StorePath> storePath;
if (args.size() == 2) {
expectedHash = Hash::parseAny(args[1], ht);
const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
storePath = store->makeFixedOutputPath(recursive, *expectedHash, name);
const auto method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
storePath = store->makeFixedOutputPath(name, FixedOutputInfo {
{
.method = method,
.hash = *expectedHash,
},
{},
});
if (store->isValidPath(*storePath))
hash = *expectedHash;
else

12
src/nix-store/nix-store.cc
View file

@ -195,10 +195,10 @@ static void opAddFixed(Strings opFlags, Strings opArgs)
/* Hack to support caching in `nix-prefetch-url'. */
static void opPrintFixedPath(Strings opFlags, Strings opArgs)
{
auto recursive = FileIngestionMethod::Flat;
auto method = FileIngestionMethod::Flat;
for (auto i : opFlags)
if (i == "--recursive") recursive = FileIngestionMethod::Recursive;
if (i == "--recursive") method = FileIngestionMethod::Recursive;
else throw UsageError("unknown flag '%1%'", i);
if (opArgs.size() != 3)
@ -209,7 +209,13 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs)
string hash = *i++;
string name = *i++;
cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(recursive, Hash::parseAny(hash, hashAlgo), name)));
cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(name, FixedOutputInfo {
{
.method = method,
.hash = Hash::parseAny(hash, hashAlgo),
},
{},
})));
}

16
src/nix/add-to-store.cc
View file

@ -69,14 +69,20 @@ struct CmdAddToStore : MixDryRun, StoreCommand
}
ValidPathInfo info {
store->makeFixedOutputPath(ingestionMethod, hash, *namePart),
*store,
StorePathDescriptor {
.name = *namePart,
.info = FixedOutputInfo {
{
.method = std::move(ingestionMethod),
.hash = std::move(hash),
},
{},
},
},
narHash,
};
info.narSize = sink.s->size();
info.ca = std::optional { FixedOutputHash {
.method = ingestionMethod,
.hash = hash,
} };
if (!dryRun) {
auto source = StringSource { *sink.s };

2
src/nix/bundle.cc
View file

@ -91,7 +91,7 @@ struct CmdBundle : InstallableCommand
mkString(*evalState->allocAttr(*arg, evalState->symbols.create("system")), settings.thisSystem.get());
arg->attrs->sort();
auto vRes = evalState->allocValue();
evalState->callFunction(*bundler.toValue(*evalState).first, *arg, *vRes, noPos);

38
src/nix/make-content-addressable.cc
View file

@ -55,19 +55,15 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON
StringMap rewrites;
StorePathSet references;
bool hasSelfReference = false;
PathReferences<StorePath> refs;
refs.hasSelfReference = oldInfo->hasSelfReference;
for (auto & ref : oldInfo->references) {
if (ref == path)
hasSelfReference = true;
else {
auto i = remappings.find(ref);
auto replacement = i != remappings.end() ? i->second : ref;
// FIXME: warn about unremapped paths?
if (replacement != ref)
rewrites.insert_or_assign(store->printStorePath(ref), store->printStorePath(replacement));
references.insert(std::move(replacement));
}
auto i = remappings.find(ref);
auto replacement = i != remappings.end() ? i->second : ref;
// FIXME: warn about unremapped paths?
if (replacement != ref)
rewrites.insert_or_assign(store->printStorePath(ref), store->printStorePath(replacement));
refs.references.insert(std::move(replacement));
}
*sink.s = rewriteStrings(*sink.s, rewrites);
@ -78,16 +74,20 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON
auto narHash = hashModuloSink.finish().first;
ValidPathInfo info {
store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference),
*store,
StorePathDescriptor {
.name = std::string { path.name() },
.info = FixedOutputInfo {
{
.method = FileIngestionMethod::Recursive,
.hash = narHash,
},
std::move(refs),
},
},
narHash,
};
info.references = std::move(references);
info.hasSelfReference = std::move(hasSelfReference);
info.narSize = sink.s->size();
info.ca = FixedOutputHash {
.method = FileIngestionMethod::Recursive,
.hash = info.narHash,
};
if (!json)
printInfo("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path));

13
src/nix/profile.cc
View file

@ -130,12 +130,21 @@ struct ProfileManifest
auto narHash = hashString(htSHA256, *sink.s);
ValidPathInfo info {
store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "profile", references),
*store,
StorePathDescriptor {
"profile",
FixedOutputInfo {
{
.method = FileIngestionMethod::Recursive,
.hash = narHash,
},
{ references },
},
},
narHash,
};
info.references = std::move(references);
info.narSize = sink.s->size();
info.ca = FixedOutputHash { .method = FileIngestionMethod::Recursive, .hash = info.narHash };
auto source = StringSource { *sink.s };
store->addToStore(info, source);

6
src/nix/verify.cc
View file

@ -73,14 +73,14 @@ struct CmdVerify : StorePathsCommand
ThreadPool pool;
auto doPath = [&](const Path & storePath) {
auto doPath = [&](const StorePath & storePath) {
try {
checkInterrupt();
MaintainCount<std::atomic<size_t>> mcActive(active);
update();
auto info = store->queryPathInfo(store->parseStorePath(storePath));
auto info = store->queryPathInfo(storePath);
// Note: info->path can be different from storePath
// for binary cache stores when using --all (since we
@ -178,7 +178,7 @@ struct CmdVerify : StorePathsCommand
};
for (auto & storePath : storePaths)
pool.enqueue(std::bind(doPath, store->printStorePath(storePath)));
pool.enqueue(std::bind(doPath, storePath));
pool.process();