lix-project/lix
20
57
Fork 31
Code Issues 350 Code Review (Gerrit) Projects 3 Releases Wiki Activity

Merge branch 'master' of github.com:NixOS/nix into hash-always-has-type

Browse source
This commit is contained in:
Carlo Nucera 2020-07-15 17:19:56 -04:00
parent 4415765385 36a1242603
commit d090562348
36 changed files with 394 additions and 354 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
perl/lib/Nix/Store.xs
View file

@ -182,7 +182,7 @@ void importPaths(int fd, int dontCheckSigs)
PPCODE:
try {
FdSource source(fd);
store()->importPaths(source, nullptr, dontCheckSigs ? NoCheckSigs : CheckSigs);
store()->importPaths(source, dontCheckSigs ? NoCheckSigs : CheckSigs);
} catch (Error & e) {
croak("%s", e.what());
}

2
src/libexpr/eval.cc
View file

@ -366,7 +366,7 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
if (store->isInStore(r.second)) {
StorePathSet closure;
store->computeFSClosure(store->parseStorePath(store->toStorePath(r.second)), closure);
store->computeFSClosure(store->toStorePath(r.second).first, closure);
for (auto & path : closure)
allowedPaths->insert(store->printStorePath(path));
} else

6
src/libexpr/primops.cc
View file

@ -883,10 +883,10 @@ static void prim_storePath(EvalState & state, const Pos & pos, Value * * args, V
.hint = hintfmt("path '%1%' is not in the Nix store", path),
.errPos = pos
});
Path path2 = state.store->toStorePath(path);
auto path2 = state.store->toStorePath(path).first;
if (!settings.readOnlyMode)
state.store->ensurePath(state.store->parseStorePath(path2));
context.insert(path2);
state.store->ensurePath(path2);
context.insert(state.store->printStorePath(path2));
mkString(v, path, context);
}

130
src/libstore/binary-cache-store.cc
View file

@ -15,6 +15,7 @@
#include <chrono>
#include <future>
#include <regex>
#include <fstream>
#include <nlohmann/json.hpp>
@ -57,6 +58,13 @@ void BinaryCacheStore::init()
}
}
void BinaryCacheStore::upsertFile(const std::string & path,
std::string && data,
const std::string & mimeType)
{
upsertFile(path, std::make_shared<std::stringstream>(std::move(data)), mimeType);
}
void BinaryCacheStore::getFile(const std::string & path,
Callback<std::shared_ptr<std::string>> callback) noexcept
{
@ -113,13 +121,74 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo));
}
void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
AutoCloseFD openFile(const Path & path)
{
// FIXME: See if we can use the original source to reduce memory usage.
auto nar = make_ref<std::string>(narSource.drain());
auto fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
if (!fd)
throw SysError("opening file '%1%'", path);
return fd;
}
if (!repair && isValidPath(info.path)) return;
struct FileSource : FdSource
{
AutoCloseFD fd2;
FileSource(const Path & path)
: fd2(openFile(path))
{
fd = fd2.get();
}
};
void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair, CheckSigsFlag checkSigs)
{
assert(info.narHash && info.narSize);
if (!repair && isValidPath(info.path)) {
// FIXME: copyNAR -> null sink
narSource.drain();
return;
}
auto [fdTemp, fnTemp] = createTempFile();
auto now1 = std::chrono::steady_clock::now();
/* Read the NAR simultaneously into a CompressionSink+FileSink (to
write the compressed NAR to disk), into a HashSink (to get the
NAR hash), and into a NarAccessor (to get the NAR listing). */
HashSink fileHashSink(htSHA256);
std::shared_ptr<FSAccessor> narAccessor;
{
FdSink fileSink(fdTemp.get());
TeeSink teeSink(fileSink, fileHashSink);
auto compressionSink = makeCompressionSink(compression, teeSink);
TeeSource teeSource(narSource, *compressionSink);
narAccessor = makeNarAccessor(teeSource);
compressionSink->finish();
}
auto now2 = std::chrono::steady_clock::now();
auto narInfo = make_ref<NarInfo>(info);
narInfo->narSize = info.narSize;
narInfo->narHash = info.narHash;
narInfo->compression = compression;
auto [fileHash, fileSize] = fileHashSink.finish();
narInfo->fileHash = fileHash;
narInfo->fileSize = fileSize;
narInfo->url = "nar/" + narInfo->fileHash->to_string(Base32, false) + ".nar"
+ (compression == "xz" ? ".xz" :
compression == "bzip2" ? ".bz2" :
compression == "br" ? ".br" :
"");
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache",
printStorePath(narInfo->path), info.narSize,
((1.0 - (double) fileSize / info.narSize) * 100.0),
duration);
/* Verify that all references are valid. This may do some .narinfo
reads, but typically they'll already be cached. */
@ -132,23 +201,6 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
printStorePath(info.path), printStorePath(ref));
}
assert(nar->compare(0, narMagic.size(), narMagic) == 0);
auto narInfo = make_ref<NarInfo>(info);
narInfo->narSize = nar->size();
narInfo->narHash = hashString(htSHA256, *nar);
if (info.narHash && info.narHash != narInfo->narHash)
throw Error("refusing to copy corrupted path '%1%' to binary cache", printStorePath(info.path));
auto accessor_ = std::dynamic_pointer_cast<RemoteFSAccessor>(accessor);
auto narAccessor = makeNarAccessor(nar);
if (accessor_)
accessor_->addToCache(printStorePath(info.path), *nar, narAccessor);
/* Optionally write a JSON file containing a listing of the
contents of the NAR. */
if (writeNARListing) {
@ -160,33 +212,13 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
{
auto res = jsonRoot.placeholder("root");
listNar(res, narAccessor, "", true);
listNar(res, ref<FSAccessor>(narAccessor), "", true);
}
}
upsertFile(std::string(info.path.to_string()) + ".ls", jsonOut.str(), "application/json");
}
/* Compress the NAR. */
narInfo->compression = compression;
auto now1 = std::chrono::steady_clock::now();
auto narCompressed = compress(compression, *nar, parallelCompression);
auto now2 = std::chrono::steady_clock::now();
narInfo->fileHash = hashString(htSHA256, *narCompressed);
narInfo->fileSize = narCompressed->size();
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache",
printStorePath(narInfo->path), narInfo->narSize,
((1.0 - (double) narCompressed->size() / nar->size()) * 100.0),
duration);
narInfo->url = "nar/" + narInfo->fileHash->to_string(Base32, false) + ".nar"
+ (compression == "xz" ? ".xz" :
compression == "bzip2" ? ".bz2" :
compression == "br" ? ".br" :
"");
/* Optionally maintain an index of DWARF debug info files
consisting of JSON files named 'debuginfo/<build-id>' that
specify the NAR file and member containing the debug info. */
@ -247,12 +279,14 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
/* Atomically write the NAR file. */
if (repair || !fileExists(narInfo->url)) {
stats.narWrite++;
upsertFile(narInfo->url, *narCompressed, "application/x-nix-nar");
upsertFile(narInfo->url,
std::make_shared<std::fstream>(fnTemp, std::ios_base::in),
"application/x-nix-nar");
} else
stats.narWriteAverted++;
stats.narWriteBytes += nar->size();
stats.narWriteCompressedBytes += narCompressed->size();
stats.narWriteBytes += info.narSize;
stats.narWriteCompressedBytes += fileSize;
stats.narWriteCompressionTimeMs += duration;
/* Atomically write the NAR info file.*/
@ -351,7 +385,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
ValidPathInfo info(makeFixedOutputPath(method, *h, name));
auto source = StringSource { *sink.s };
addToStore(info, source, repair, CheckSigs, nullptr);
addToStore(info, source, repair, CheckSigs);
return std::move(info.path);
}
@ -366,7 +400,7 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
StringSink sink;
dumpString(s, sink);
auto source = StringSource { *sink.s };
addToStore(info, source, repair, CheckSigs, nullptr);
addToStore(info, source, repair, CheckSigs);
}
return std::move(info.path);

9
src/libstore/binary-cache-store.hh
View file

@ -36,9 +36,13 @@ public:
virtual bool fileExists(const std::string & path) = 0;
virtual void upsertFile(const std::string & path,
const std::string & data,
std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType) = 0;
void upsertFile(const std::string & path,
std::string && data,
const std::string & mimeType);
/* Note: subclasses must implement at least one of the two
following getFile() methods. */
@ -75,8 +79,7 @@ public:
{ unsupported("queryPathFromHashPart"); }
void addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor) override;
RepairFlag repair, CheckSigsFlag checkSigs) override;
StorePath addToStore(const string & name, const Path & srcPath,
FileIngestionMethod method, HashType hashAlgo,

9
src/libstore/build.cc
View file

@ -2044,7 +2044,7 @@ void DerivationGoal::startBuilder()
auto storePathS = *i++;
if (!worker.store.isInStore(storePathS))
throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS);
auto storePath = worker.store.parseStorePath(worker.store.toStorePath(storePathS));
auto storePath = worker.store.toStorePath(storePathS).first;
/* Write closure info to <fileName>. */
writeFile(tmpDir + "/" + fileName,
@ -2083,7 +2083,7 @@ void DerivationGoal::startBuilder()
for (auto & i : dirsInChroot)
try {
if (worker.store.isInStore(i.second.source))
worker.store.computeFSClosure(worker.store.parseStorePath(worker.store.toStorePath(i.second.source)), closure);
worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure);
} catch (InvalidPath & e) {
} catch (Error & e) {
throw Error("while processing 'sandbox-paths': %s", e.what());
@ -2768,10 +2768,9 @@ struct RestrictedStore : public LocalFSStore
{ throw Error("addToStore"); }
void addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs,
std::shared_ptr<FSAccessor> accessor = 0) override
RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) override
{
next->addToStore(info, narSource, repair, checkSigs, accessor);
next->addToStore(info, narSource, repair, checkSigs);
goal.addDependency(info.path);
}

15
src/libstore/daemon.cc
View file

@ -391,7 +391,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
}
HashType hashAlgo = parseHashType(s);
TeeSource savedNAR(from);
StringSink savedNAR;
TeeSource savedNARSource(from, savedNAR);
RetrieveRegularNARSink savedRegular;
if (method == FileIngestionMethod::Recursive) {
@ -399,7 +400,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
a string so that we can pass it to
addToStoreFromDump(). */
ParseSink sink; /* null sink; just parse the NAR */
parseDump(sink, savedNAR);
parseDump(sink, savedNARSource);
} else
parseDump(savedRegular, from);
@ -407,7 +408,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
if (!savedRegular.regular) throw Error("regular file expected");
auto path = store->addToStoreFromDump(
method == FileIngestionMethod::Recursive ? *savedNAR.data : savedRegular.s,
method == FileIngestionMethod::Recursive ? *savedNAR.s : savedRegular.s,
baseName,
method,
hashAlgo);
@ -442,7 +443,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
case wopImportPaths: {
logger->startWork();
TunnelSource source(from, to);
auto paths = store->importPaths(source, nullptr,
auto paths = store->importPaths(source,
trusted ? NoCheckSigs : CheckSigs);
logger->stopWork();
Strings paths2;
@ -731,9 +732,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
if (GET_PROTOCOL_MINOR(clientVersion) >= 21)
source = std::make_unique<TunnelSource>(from, to);
else {
TeeSink tee(from);
TeeParseSink tee(from);
parseDump(tee, tee.source);
saved = std::move(*tee.source.data);
saved = std::move(*tee.saved.s);
source = std::make_unique<StringSource>(saved);
}
@ -741,7 +742,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
// FIXME: race if addToStore doesn't read source?
store->addToStore(info, *source, (RepairFlag) repair,
dontCheckSigs ? NoCheckSigs : CheckSigs, nullptr);
dontCheckSigs ? NoCheckSigs : CheckSigs);
logger->stopWork();
break;

7
src/libstore/derivations.cc
View file

@ -4,7 +4,6 @@
#include "util.hh"
#include "worker-protocol.hh"
#include "fs-accessor.hh"
#include "istringstream_nocopy.hh"
namespace nix {
@ -101,7 +100,7 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
}
static DerivationOutput parseDerivationOutput(const Store & store, istringstream_nocopy & str)
static DerivationOutput parseDerivationOutput(const Store & store, std::istringstream & str)
{
expect(str, ","); auto path = store.parseStorePath(parsePath(str));
expect(str, ","); auto hashAlgo = parseString(str);
@ -129,10 +128,10 @@ static DerivationOutput parseDerivationOutput(const Store & store, istringstream
}
static Derivation parseDerivation(const Store & store, const string & s)
static Derivation parseDerivation(const Store & store, std::string && s)
{
Derivation drv;
istringstream_nocopy str(s);
std::istringstream str(std::move(s));
expect(str, "Derive([");
/* Parse the list of outputs. */

43
src/libstore/export-import.cc
View file

@ -7,24 +7,6 @@
namespace nix {
struct HashAndWriteSink : Sink
{
Sink & writeSink;
HashSink hashSink;
HashAndWriteSink(Sink & writeSink) : writeSink(writeSink), hashSink(htSHA256)
{
}
virtual void operator () (const unsigned char * data, size_t len)
{
writeSink(data, len);
hashSink(data, len);
}
Hash currentHash()
{
return hashSink.currentHash().first;
}
};
void Store::exportPaths(const StorePathSet & paths, Sink & sink)
{
auto sorted = topoSortPaths(paths);
@ -47,28 +29,29 @@ void Store::exportPath(const StorePath & path, Sink & sink)
{
auto info = queryPathInfo(path);
HashAndWriteSink hashAndWriteSink(sink);
HashSink hashSink(htSHA256);
TeeSink teeSink(sink, hashSink);
narFromPath(path, hashAndWriteSink);
narFromPath(path, teeSink);
/* Refuse to export paths that have changed. This prevents
filesystem corruption from spreading to other machines.
Don't complain if the stored hash is zero (unknown). */
Hash hash = hashAndWriteSink.currentHash();
Hash hash = hashSink.currentHash().first;
if (hash != info->narHash && info->narHash != Hash(info->narHash->type))
throw Error("hash of path '%s' has changed from '%s' to '%s'!",
printStorePath(path), info->narHash->to_string(Base32, true), hash.to_string(Base32, true));
hashAndWriteSink
teeSink
<< exportMagic
<< printStorePath(path);
writeStorePaths(*this, hashAndWriteSink, info->references);
hashAndWriteSink
writeStorePaths(*this, teeSink, info->references);
teeSink
<< (info->deriver ? printStorePath(*info->deriver) : "")
<< 0;
}
StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, CheckSigsFlag checkSigs)
StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
{
StorePaths res;
while (true) {
@ -77,7 +60,7 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> acces
if (n != 1) throw Error("input doesn't look like something created by 'nix-store --export'");
/* Extract the NAR from the source. */
TeeSink tee(source);
TeeParseSink tee(source);
parseDump(tee, tee.source);
uint32_t magic = readInt(source);
@ -94,16 +77,16 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> acces
if (deriver != "")
info.deriver = parseStorePath(deriver);
info.narHash = hashString(htSHA256, *tee.source.data);
info.narSize = tee.source.data->size();
info.narHash = hashString(htSHA256, *tee.saved.s);
info.narSize = tee.saved.s->size();
// Ignore optional legacy signature.
if (readInt(source) == 1)
readString(source);
// Can't use underlying source, which would have been exhausted
auto source = StringSource { *tee.source.data };
addToStore(info, source, NoRepair, checkSigs, accessor);
auto source = StringSource { *tee.saved.s };
addToStore(info, source, NoRepair, checkSigs);
res.push_back(info.path);
}

30
src/libstore/gc.cc
View file

@ -262,11 +262,13 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
{
auto foundRoot = [&](const Path & path, const Path & target) {
auto storePath = maybeParseStorePath(toStorePath(target));
if (storePath && isValidPath(*storePath))
roots[std::move(*storePath)].emplace(path);
else
printInfo("skipping invalid root from '%1%' to '%2%'", path, target);
try {
auto storePath = toStorePath(target).first;
if (isValidPath(storePath))
roots[std::move(storePath)].emplace(path);
else
printInfo("skipping invalid root from '%1%' to '%2%'", path, target);
} catch (BadStorePath &) { }
};
try {
@ -472,15 +474,15 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
for (auto & [target, links] : unchecked) {
if (!isInStore(target)) continue;
Path pathS = toStorePath(target);
if (!isStorePath(pathS)) continue;
auto path = parseStorePath(pathS);
if (!isValidPath(path)) continue;
debug("got additional root '%1%'", pathS);
if (censor)
roots[path].insert(censored);
else
roots[path].insert(links.begin(), links.end());
try {
auto path = toStorePath(target).first;
if (!isValidPath(path)) continue;
debug("got additional root '%1%'", printStorePath(path));
if (censor)
roots[path].insert(censored);
else
roots[path].insert(links.begin(), links.end());
} catch (BadStorePath &) { }
}
}

4
src/libstore/http-binary-cache-store.cc
View file

@ -100,11 +100,11 @@ protected:
}
void upsertFile(const std::string & path,
const std::string & data,
std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType) override
{
auto req = FileTransferRequest(cacheUri + "/" + path);
req.data = std::make_shared<string>(data); // FIXME: inefficient
req.data = std::make_shared<string>(StreamToSourceAdapter(istream).drain());
req.mimeType = mimeType;
try {
getFileTransfer()->upload(req);

3
src/libstore/legacy-ssh-store.cc
View file

@ -126,8 +126,7 @@ struct LegacySSHStore : public Store
}
void addToStore(const ValidPathInfo & info, Source & source,
RepairFlag repair, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor) override
RepairFlag repair, CheckSigsFlag checkSigs) override
{
debug("adding path '%s' to remote host '%s'", printStorePath(info.path), host);

35
src/libstore/local-binary-cache-store.cc
View file

@ -31,8 +31,18 @@ protected:
bool fileExists(const std::string & path) override;
void upsertFile(const std::string & path,
const std::string & data,
const std::string & mimeType) override;
std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType) override
{
auto path2 = binaryCacheDir + "/" + path;
Path tmp = path2 + ".tmp." + std::to_string(getpid());
AutoDelete del(tmp, false);
StreamToSourceAdapter source(istream);
writeFile(tmp, source);
if (rename(tmp.c_str(), path2.c_str()))
throw SysError("renaming '%1%' to '%2%'", tmp, path2);
del.cancel();
}
void getFile(const std::string & path, Sink & sink) override
{
@ -52,7 +62,9 @@ protected:
if (entry.name.size() != 40 ||
!hasSuffix(entry.name, ".narinfo"))
continue;
paths.insert(parseStorePath(storeDir + "/" + entry.name.substr(0, entry.name.size() - 8)));
paths.insert(parseStorePath(
storeDir + "/" + entry.name.substr(0, entry.name.size() - 8)
+ "-" + MissingName));
}
return paths;
@ -68,28 +80,11 @@ void LocalBinaryCacheStore::init()
BinaryCacheStore::init();
}
static void atomicWrite(const Path & path, const std::string & s)
{
Path tmp = path + ".tmp." + std::to_string(getpid());
AutoDelete del(tmp, false);
writeFile(tmp, s);
if (rename(tmp.c_str(), path.c_str()))
throw SysError("renaming '%1%' to '%2%'", tmp, path);
del.cancel();
}
bool LocalBinaryCacheStore::fileExists(const std::string & path)
{
return pathExists(binaryCacheDir + "/" + path);
}
void LocalBinaryCacheStore::upsertFile(const std::string & path,
const std::string & data,
const std::string & mimeType)
{
atomicWrite(binaryCacheDir + "/" + path, data);
}
static RegisterStoreImplementation regStore([](
const std::string & uri, const Store::Params & params)
-> std::shared_ptr<Store>

6
src/libstore/local-fs-store.cc
View file

@ -20,9 +20,9 @@ struct LocalStoreAccessor : public FSAccessor
Path toRealPath(const Path & path)
{
Path storePath = store->toStorePath(path);
if (!store->isValidPath(store->parseStorePath(storePath)))
throw InvalidPath("path '%1%' is not a valid store path", storePath);
auto storePath = store->toStorePath(path).first;
if (!store->isValidPath(storePath))
throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath));
return store->getRealStoreDir() + std::string(path, store->storeDir.size());
}

22
src/libstore/local-store.cc
View file

@ -962,7 +962,7 @@ const PublicKeys & LocalStore::getPublicKeys()
void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
RepairFlag repair, CheckSigsFlag checkSigs)
{
if (!info.narHash)
throw Error("cannot add path '%s' because it lacks a hash", printStorePath(info.path));
@ -1097,13 +1097,16 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
{
Path srcPath(absPath(_srcPath));
if (method != FileIngestionMethod::Recursive)
return addToStoreFromDump(readFile(srcPath), name, method, hashAlgo, repair);
/* For computing the NAR hash. */
auto sha256Sink = std::make_unique<HashSink>(htSHA256);
/* For computing the store path. In recursive SHA-256 mode, this
is the same as the NAR hash, so no need to do it again. */
std::unique_ptr<HashSink> hashSink =
method == FileIngestionMethod::Recursive && hashAlgo == htSHA256
hashAlgo == htSHA256
? nullptr
: std::make_unique<HashSink>(hashAlgo);
@ -1136,10 +1139,7 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
if (!inMemory) sink(buf, len);
});
if (method == FileIngestionMethod::Recursive)
dumpPath(srcPath, sink2, filter);
else
readFile(srcPath, sink2);
dumpPath(srcPath, sink2, filter);
});
std::unique_ptr<AutoDelete> delTempDir;
@ -1155,10 +1155,7 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
delTempDir = std::make_unique<AutoDelete>(tempDir);
tempPath = tempDir + "/x";
if (method == FileIngestionMethod::Recursive)
restorePath(tempPath, *source);
else
writeFile(tempPath, *source);
restorePath(tempPath, *source);
} catch (EndOfFile &) {
if (!inMemory) throw;
@ -1191,10 +1188,7 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
if (inMemory) {
/* Restore from the NAR in memory. */
StringSource source(nar);
if (method == FileIngestionMethod::Recursive)
restorePath(realPath, source);
else
writeFile(realPath, source);
restorePath(realPath, source);
} else {
/* Move the temporary path we restored above. */
if (rename(tempPath.c_str(), realPath.c_str()))

3
src/libstore/local-store.hh
View file

@ -143,8 +143,7 @@ public:
SubstitutablePathInfos & infos) override;
void addToStore(const ValidPathInfo & info, Source & source,
RepairFlag repair, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor) override;
RepairFlag repair, CheckSigsFlag checkSigs) override;
StorePath addToStore(const string & name, const Path & srcPath,
FileIngestionMethod method, HashType hashAlgo,

46
src/libstore/nar-accessor.cc
View file

@ -18,7 +18,7 @@ struct NarMember
/* If this is a regular file, position of the contents of this
file in the NAR. */
size_t start = 0, size = 0;
uint64_t start = 0, size = 0;
std::string target;
@ -34,17 +34,19 @@ struct NarAccessor : public FSAccessor
NarMember root;
struct NarIndexer : ParseSink, StringSource
struct NarIndexer : ParseSink, Source
{
NarAccessor & acc;
Source & source;
std::stack<NarMember *> parents;
std::string currentStart;
bool isExec = false;
NarIndexer(NarAccessor & acc, const std::string & nar)
: StringSource(nar), acc(acc)
uint64_t pos = 0;
NarIndexer(NarAccessor & acc, Source & source)
: acc(acc), source(source)
{ }
void createMember(const Path & path, NarMember member) {
@ -79,31 +81,38 @@ struct NarAccessor : public FSAccessor
void preallocateContents(unsigned long long size) override
{
currentStart = string(s, pos, 16);
assert(size <= std::numeric_limits<size_t>::max());
parents.top()->size = (size_t)size;
assert(size <= std::numeric_limits<uint64_t>::max());
parents.top()->size = (uint64_t) size;
parents.top()->start = pos;
}
void receiveContents(unsigned char * data, unsigned int len) override
{
// Sanity check
if (!currentStart.empty()) {
assert(len < 16 || currentStart == string((char *) data, 16));
currentStart.clear();
}
}
{ }
void createSymlink(const Path & path, const string & target) override
{
createMember(path,
NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target});
}
size_t read(unsigned char * data, size_t len) override
{
auto n = source.read(data, len);
pos += n;
return n;
}
};
NarAccessor(ref<const std::string> nar) : nar(nar)
{
NarIndexer indexer(*this, *nar);
StringSource source(*nar);
NarIndexer indexer(*this, source);
parseDump(indexer, indexer);
}
NarAccessor(Source & source)
{
NarIndexer indexer(*this, source);
parseDump(indexer, indexer);
}
@ -219,6 +228,11 @@ ref<FSAccessor> makeNarAccessor(ref<const std::string> nar)
return make_ref<NarAccessor>(nar);
}
ref<FSAccessor> makeNarAccessor(Source & source)
{
return make_ref<NarAccessor>(source);
}
ref<FSAccessor> makeLazyNarAccessor(const std::string & listing,
GetNarBytes getNarBytes)
{

4
src/libstore/nar-accessor.hh
View file

@ -6,10 +6,14 @@
namespace nix {
struct Source;
/* Return an object that provides access to the contents of a NAR
file. */
ref<FSAccessor> makeNarAccessor(ref<const std::string> nar);
ref<FSAccessor> makeNarAccessor(Source & source);
/* Create a NAR accessor from a NAR listing (in the format produced by
listNar()). The callback getNarBytes(offset, length) is used by the
readFile() method of the accessor to get the contents of files

2
src/libstore/path.cc
View file

@ -2,8 +2,6 @@
namespace nix {
MakeError(BadStorePath, Error);
static void checkName(std::string_view path, std::string_view name)
{
if (name.empty())

33
src/libstore/remote-fs-accessor.cc
View file

@ -16,26 +16,26 @@ RemoteFSAccessor::RemoteFSAccessor(ref<Store> store, const Path & cacheDir)
createDirs(cacheDir);
}
Path RemoteFSAccessor::makeCacheFile(const Path & storePath, const std::string & ext)
Path RemoteFSAccessor::makeCacheFile(std::string_view hashPart, const std::string & ext)
{
assert(cacheDir != "");
return fmt("%s/%s.%s", cacheDir, store->parseStorePath(storePath).hashPart(), ext);
return fmt("%s/%s.%s", cacheDir, hashPart, ext);
}
void RemoteFSAccessor::addToCache(const Path & storePath, const std::string & nar,
void RemoteFSAccessor::addToCache(std::string_view hashPart, const std::string & nar,
ref<FSAccessor> narAccessor)
{
nars.emplace(storePath, narAccessor);
nars.emplace(hashPart, narAccessor);
if (cacheDir != "") {
try {
std::ostringstream str;
JSONPlaceholder jsonRoot(str);
listNar(jsonRoot, narAccessor, "", true);
writeFile(makeCacheFile(storePath, "ls"), str.str());
writeFile(makeCacheFile(hashPart, "ls"), str.str());
/* FIXME: do this asynchronously. */
writeFile(makeCacheFile(storePath, "nar"), nar);
writeFile(makeCacheFile(hashPart, "nar"), nar);
} catch (...) {
ignoreException();
@ -47,23 +47,22 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
{
auto path = canonPath(path_);
auto storePath = store->toStorePath(path);
std::string restPath = std::string(path, storePath.size());
auto [storePath, restPath] = store->toStorePath(path);
if (!store->isValidPath(store->parseStorePath(storePath)))
throw InvalidPath("path '%1%' is not a valid store path", storePath);
if (!store->isValidPath(storePath))
throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath));
auto i = nars.find(storePath);
auto i = nars.find(std::string(storePath.hashPart()));
if (i != nars.end()) return {i->second, restPath};
StringSink sink;
std::string listing;
Path cacheFile;
if (cacheDir != "" && pathExists(cacheFile = makeCacheFile(storePath, "nar"))) {
if (cacheDir != "" && pathExists(cacheFile = makeCacheFile(storePath.hashPart(), "nar"))) {
try {
listing = nix::readFile(makeCacheFile(storePath, "ls"));
listing = nix::readFile(makeCacheFile(storePath.hashPart(), "ls"));
auto narAccessor = makeLazyNarAccessor(listing,
[cacheFile](uint64_t offset, uint64_t length) {
@ -81,7 +80,7 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
return buf;
});
nars.emplace(storePath, narAccessor);
nars.emplace(storePath.hashPart(), narAccessor);
return {narAccessor, restPath};
} catch (SysError &) { }
@ -90,15 +89,15 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
*sink.s = nix::readFile(cacheFile);
auto narAccessor = makeNarAccessor(sink.s);
nars.emplace(storePath, narAccessor);
nars.emplace(storePath.hashPart(), narAccessor);
return {narAccessor, restPath};
} catch (SysError &) { }
}
store->narFromPath(store->parseStorePath(storePath), sink);
store->narFromPath(storePath, sink);
auto narAccessor = makeNarAccessor(sink.s);
addToCache(storePath, *sink.s, narAccessor);
addToCache(storePath.hashPart(), *sink.s, narAccessor);
return {narAccessor, restPath};
}

6
src/libstore/remote-fs-accessor.hh
View file

@ -10,7 +10,7 @@ class RemoteFSAccessor : public FSAccessor
{
ref<Store> store;
std::map<Path, ref<FSAccessor>> nars;
std::map<std::string, ref<FSAccessor>> nars;
Path cacheDir;
@ -18,9 +18,9 @@ class RemoteFSAccessor : public FSAccessor
friend class BinaryCacheStore;
Path makeCacheFile(const Path & storePath, const std::string & ext);
Path makeCacheFile(std::string_view hashPart, const std::string & ext);
void addToCache(const Path & storePath, const std::string & nar,
void addToCache(std::string_view hashPart, const std::string & nar,
ref<FSAccessor> narAccessor);
public:

2
src/libstore/remote-store.cc
View file

@ -466,7 +466,7 @@ std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string &
void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
RepairFlag repair, CheckSigsFlag checkSigs)
{
auto conn(getConnection());

3
src/libstore/remote-store.hh
View file

@ -60,8 +60,7 @@ public:
SubstitutablePathInfos & infos) override;
void addToStore(const ValidPathInfo & info, Source & nar,
RepairFlag repair, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor) override;
RepairFlag repair, CheckSigsFlag checkSigs) override;
StorePath addToStore(const string & name, const Path & srcPath,
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,

39
src/libstore/s3-binary-cache-store.cc
View file

@ -7,7 +7,6 @@
#include "globals.hh"
#include "compression.hh"
#include "filetransfer.hh"
#include "istringstream_nocopy.hh"
#include <aws/core/Aws.h>
#include <aws/core/VersionConfig.h>
@ -262,12 +261,11 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
std::shared_ptr<TransferManager> transferManager;
std::once_flag transferManagerCreated;
void uploadFile(const std::string & path, const std::string & data,
void uploadFile(const std::string & path,
std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType,
const std::string & contentEncoding)
{
auto stream = std::make_shared<istringstream_nocopy>(data);
auto maxThreads = std::thread::hardware_concurrency();
static std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor>
@ -307,7 +305,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
std::shared_ptr<TransferHandle> transferHandle =
transferManager->UploadFile(
stream, bucketName, path, mimeType,
istream, bucketName, path, mimeType,
Aws::Map<Aws::String, Aws::String>(),
nullptr /*, contentEncoding */);
@ -333,9 +331,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
if (contentEncoding != "")
request.SetContentEncoding(contentEncoding);
auto stream = std::make_shared<istringstream_nocopy>(data);
request.SetBody(stream);
request.SetBody(istream);
auto result = checkAws(fmt("AWS error uploading '%s'", path),
s3Helper.client->PutObject(request));
@ -347,25 +343,34 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1)
.count();
printInfo(format("uploaded 's3://%1%/%2%' (%3% bytes) in %4% ms") %
bucketName % path % data.size() % duration);
auto size = istream->tellg();
printInfo("uploaded 's3://%s/%s' (%d bytes) in %d ms",
bucketName, path, size, duration);
stats.putTimeMs += duration;
stats.putBytes += data.size();
stats.putBytes += size;
stats.put++;
}
void upsertFile(const std::string & path, const std::string & data,
void upsertFile(const std::string & path,
std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType) override
{
auto compress = [&](std::string compression)
{
auto compressed = nix::compress(compression, StreamToSourceAdapter(istream).drain());
return std::make_shared<std::stringstream>(std::move(*compressed));
};
if (narinfoCompression != "" && hasSuffix(path, ".narinfo"))
uploadFile(path, *compress(narinfoCompression, data), mimeType, narinfoCompression);
uploadFile(path, compress(narinfoCompression), mimeType, narinfoCompression);
else if (lsCompression != "" && hasSuffix(path, ".ls"))
uploadFile(path, *compress(lsCompression, data), mimeType, lsCompression);
uploadFile(path, compress(lsCompression), mimeType, lsCompression);
else if (logCompression != "" && hasPrefix(path, "log/"))
uploadFile(path, *compress(logCompression, data), mimeType, logCompression);
uploadFile(path, compress(logCompression), mimeType, logCompression);
else
uploadFile(path, data, mimeType, "");
uploadFile(path, istream, mimeType, "");
}
void getFile(const std::string & path, Sink & sink) override
@ -410,7 +415,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
for (auto object : contents) {
auto & key = object.GetKey();
if (key.size() != 40 || !hasSuffix(key, ".narinfo")) continue;
paths.insert(parseStorePath(storeDir + "/" + key.substr(0, key.size() - 8) + "-unknown"));
paths.insert(parseStorePath(storeDir + "/" + key.substr(0, key.size() - 8) + "-" + MissingName));
}
marker = res.GetNextMarker();

30
src/libstore/store-api.cc
View file

@ -21,15 +21,15 @@ bool Store::isInStore(const Path & path) const
}
Path Store::toStorePath(const Path & path) const
std::pair<StorePath, Path> Store::toStorePath(const Path & path) const
{
if (!isInStore(path))
throw Error("path '%1%' is not in the Nix store", path);
Path::size_type slash = path.find('/', storeDir.size() + 1);
if (slash == Path::npos)
return path;
return {parseStorePath(path), ""};
else
return Path(path, 0, slash);
return {parseStorePath(std::string_view(path).substr(0, slash)), path.substr(slash)};
}
@ -42,14 +42,14 @@ Path Store::followLinksToStore(std::string_view _path) const
path = absPath(target, dirOf(path));
}
if (!isInStore(path))
throw NotInStore("path '%1%' is not in the Nix store", path);
throw BadStorePath("path '%1%' is not in the Nix store", path);
return path;
}
StorePath Store::followLinksToStorePath(std::string_view path) const
{
return parseStorePath(toStorePath(followLinksToStore(path)));
return toStorePath(followLinksToStore(path)).first;
}
@ -351,6 +351,14 @@ ref<const ValidPathInfo> Store::queryPathInfo(const StorePath & storePath)
}
static bool goodStorePath(const StorePath & expected, const StorePath & actual)
{
return
expected.hashPart() == actual.hashPart()
&& (expected.name() == Store::MissingName || expected.name() == actual.name());
}
void Store::queryPathInfo(const StorePath & storePath,
Callback<ref<const ValidPathInfo>> callback) noexcept
{
@ -378,7 +386,7 @@ void Store::queryPathInfo(const StorePath & storePath,
state_->pathInfoCache.upsert(hashPart,
res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue{ .value = res.second });
if (res.first == NarInfoDiskCache::oInvalid ||
res.second->path != storePath)
!goodStorePath(storePath, res.second->path))
throw InvalidPath("path '%s' is not valid", printStorePath(storePath));
}
return callback(ref<const ValidPathInfo>(res.second));
@ -390,7 +398,7 @@ void Store::queryPathInfo(const StorePath & storePath,
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
queryPathInfoUncached(storePath,
{[this, storePath{printStorePath(storePath)}, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) {
{[this, storePathS{printStorePath(storePath)}, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) {
try {
auto info = fut.get();
@ -403,9 +411,11 @@ void Store::queryPathInfo(const StorePath & storePath,
state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = info });
}
if (!info || info->path != parseStorePath(storePath)) {
auto storePath = parseStorePath(storePathS);
if (!info || !goodStorePath(storePath, info->path)) {
stats.narInfoMissing++;
throw InvalidPath("path '%s' is not valid", storePath);
throw InvalidPath("path '%s' is not valid", storePathS);
}
(*callbackPtr)(ref<const ValidPathInfo>(info));
@ -550,7 +560,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store
if (!narInfo->url.empty())
jsonPath.attr("url", narInfo->url);
if (narInfo->fileHash)
jsonPath.attr("downloadHash", narInfo->fileHash->to_string(Base32, true));
jsonPath.attr("downloadHash", narInfo->fileHash->to_string(hashBase, true));
if (narInfo->fileSize)
jsonPath.attr("downloadSize", narInfo->fileSize);
if (showClosureSize)

23
src/libstore/store-api.hh
View file

@ -31,7 +31,7 @@ MakeError(InvalidPath, Error);
MakeError(Unsupported, Error);
MakeError(SubstituteGone, Error);
MakeError(SubstituterDisabled, Error);
MakeError(NotInStore, Error);
MakeError(BadStorePath, Error);
class FSAccessor;
@ -318,9 +318,9 @@ public:
the Nix store. */
bool isStorePath(std::string_view path) const;
/* Chop off the parts after the top-level store name, e.g.,
/nix/store/abcd-foo/bar => /nix/store/abcd-foo. */
Path toStorePath(const Path & path) const;
/* Split a path like /nix/store/<hash>-<name>/<bla> into
/nix/store/<hash>-<name> and /<bla>. */
std::pair<StorePath, Path> toStorePath(const Path & path) const;
/* Follow symlinks until we end up with a path in the Nix store. */
Path followLinksToStore(std::string_view path) const;
@ -385,13 +385,16 @@ public:
SubstituteFlag maybeSubstitute = NoSubstitute);
/* Query the set of all valid paths. Note that for some store
backends, the name part of store paths may be omitted
(i.e. you'll get /nix/store/<hash> rather than
backends, the name part of store paths may be replaced by 'x'
(i.e. you'll get /nix/store/<hash>-x rather than
/nix/store/<hash>-<name>). Use queryPathInfo() to obtain the
full store path. */
full store path. FIXME: should return a set of
std::variant<StorePath, HashPart> to get rid of this hack. */
virtual StorePathSet queryAllValidPaths()
{ unsupported("queryAllValidPaths"); }
constexpr static const char * MissingName = "x";
/* Query information about a valid path. It is permitted to omit
the name part of the store path. */
ref<const ValidPathInfo> queryPathInfo(const StorePath & path);
@ -440,8 +443,7 @@ public:
/* Import a path into the store. */
virtual void addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs,
std::shared_ptr<FSAccessor> accessor = 0) = 0;
RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) = 0;
/* Copy the contents of a path to the store and register the
validity the resulting path. The resulting path is returned.
@ -624,8 +626,7 @@ public:
the Nix store. Optionally, the contents of the NARs are
preloaded into the specified FS accessor to speed up subsequent
access. */
StorePaths importPaths(Source & source, std::shared_ptr<FSAccessor> accessor,
CheckSigsFlag checkSigs = CheckSigs);
StorePaths importPaths(Source & source, CheckSigsFlag checkSigs = CheckSigs);
struct Stats
{

4
src/libutil/ansicolor.hh
View file

@ -11,7 +11,7 @@ namespace nix {
#define ANSI_GREEN "\e[32;1m"
#define ANSI_YELLOW "\e[33;1m"
#define ANSI_BLUE "\e[34;1m"
#define ANSI_MAGENTA "\e[35m;1m"
#define ANSI_CYAN "\e[36m;1m"
#define ANSI_MAGENTA "\e[35;1m"
#define ANSI_CYAN "\e[36;1m"
}

5
src/libutil/archive.hh
View file

@ -63,11 +63,12 @@ struct ParseSink
virtual void createSymlink(const Path & path, const string & target) { };
};
struct TeeSink : ParseSink
struct TeeParseSink : ParseSink
{
StringSink saved;
TeeSource source;
TeeSink(Source & source) : source(source) { }
TeeParseSink(Source & source) : source(source, saved) { }
};
void parseDump(ParseSink & sink, Source & source);

1
src/libutil/hash.cc
View file

@ -8,7 +8,6 @@
#include "hash.hh"
#include "archive.hh"
#include "util.hh"
#include "istringstream_nocopy.hh"
#include <sys/types.h>
#include <sys/stat.h>

92
src/libutil/istringstream_nocopy.hh
View file

@ -1,92 +0,0 @@
/* This file provides a variant of std::istringstream that doesn't
copy its string argument. This is useful for large strings. The
caller must ensure that the string object is not destroyed while
it's referenced by this object. */
#pragma once
#include <string>
#include <iostream>
template <class CharT, class Traits = std::char_traits<CharT>, class Allocator = std::allocator<CharT>>
class basic_istringbuf_nocopy : public std::basic_streambuf<CharT, Traits>
{
public:
typedef std::basic_string<CharT, Traits, Allocator> string_type;
typedef typename std::basic_streambuf<CharT, Traits>::off_type off_type;
typedef typename std::basic_streambuf<CharT, Traits>::pos_type pos_type;
typedef typename std::basic_streambuf<CharT, Traits>::int_type int_type;
typedef typename std::basic_streambuf<CharT, Traits>::traits_type traits_type;
private:
const string_type & s;
off_type off;
public:
basic_istringbuf_nocopy(const string_type & s) : s{s}, off{0}
{
}
private:
pos_type seekoff(off_type off, std::ios_base::seekdir dir, std::ios_base::openmode which)
{
if (which & std::ios_base::in) {
this->off = dir == std::ios_base::beg
? off
: (dir == std::ios_base::end
? s.size() + off
: this->off + off);
}
return pos_type(this->off);
}
pos_type seekpos(pos_type pos, std::ios_base::openmode which)
{
return seekoff(pos, std::ios_base::beg, which);
}
std::streamsize showmanyc()
{
return s.size() - off;
}
int_type underflow()
{
if (typename string_type::size_type(off) == s.size())
return traits_type::eof();
return traits_type::to_int_type(s[off]);
}
int_type uflow()
{
if (typename string_type::size_type(off) == s.size())
return traits_type::eof();
return traits_type::to_int_type(s[off++]);
}
int_type pbackfail(int_type ch)
{
if (off == 0 || (ch != traits_type::eof() && ch != s[off - 1]))
return traits_type::eof();
return traits_type::to_int_type(s[--off]);
}
};
template <class CharT, class Traits = std::char_traits<CharT>, class Allocator = std::allocator<CharT>>
class basic_istringstream_nocopy : public std::basic_iostream<CharT, Traits>
{
typedef basic_istringbuf_nocopy<CharT, Traits, Allocator> buf_type;
buf_type buf;
public:
basic_istringstream_nocopy(const typename buf_type::string_type & s) :
std::basic_iostream<CharT, Traits>(&buf), buf(s) {};
};
typedef basic_istringstream_nocopy<char> istringstream_nocopy;

46
src/libutil/serialise.hh
View file

@ -166,17 +166,30 @@ struct StringSource : Source
};
/* Adapter class of a Source that saves all data read to `s'. */
/* A sink that writes all incoming data to two other sinks. */
struct TeeSink : Sink
{
Sink & sink1, & sink2;
TeeSink(Sink & sink1, Sink & sink2) : sink1(sink1), sink2(sink2) { }
virtual void operator () (const unsigned char * data, size_t len)
{
sink1(data, len);
sink2(data, len);
}
};
/* Adapter class of a Source that saves all data read to a sink. */
struct TeeSource : Source
{
Source & orig;
ref<std::string> data;
TeeSource(Source & orig)
: orig(orig), data(make_ref<std::string>()) { }
Sink & sink;
TeeSource(Source & orig, Sink & sink)
: orig(orig), sink(sink) { }
size_t read(unsigned char * data, size_t len)
{
size_t n = orig.read(data, len);
this->data->append((const char *) data, n);
sink(data, len);
return n;
}
};
@ -336,4 +349,27 @@ Source & operator >> (Source & in, bool & b)
}
/* An adapter that converts a std::basic_istream into a source. */
struct StreamToSourceAdapter : Source
{
std::shared_ptr<std::basic_istream<char>> istream;
StreamToSourceAdapter(std::shared_ptr<std::basic_istream<char>> istream)
: istream(istream)
{ }
size_t read(unsigned char * data, size_t len) override
{
if (!istream->read((char *) data, len)) {
if (istream->eof()) {
if (istream->gcount() == 0)
throw EndOfFile("end of file");
} else
throw Error("I/O error in StreamToSourceAdapter");
}
return istream->gcount();
}
};
}

4
src/nix-store/nix-store.cc
View file

@ -671,7 +671,7 @@ static void opImport(Strings opFlags, Strings opArgs)
if (!opArgs.empty()) throw UsageError("no arguments expected");
FdSource source(STDIN_FILENO);
auto paths = store->importPaths(source, nullptr, NoCheckSigs);
auto paths = store->importPaths(source, NoCheckSigs);
for (auto & i : paths)
cout << fmt("%s\n", store->printStorePath(i)) << std::flush;
@ -880,7 +880,7 @@ static void opServe(Strings opFlags, Strings opArgs)
case cmdImportPaths: {
if (!writeAllowed) throw Error("importing paths is not allowed");
store->importPaths(in, nullptr, NoCheckSigs); // FIXME: should we skip sig checking?
store->importPaths(in, NoCheckSigs); // FIXME: should we skip sig checking?
out << 1; // indicate success
break;
}

14
src/nix/installables.cc
View file

@ -94,8 +94,8 @@ struct InstallableStorePath : Installable
ref<Store> store;
StorePath storePath;
InstallableStorePath(ref<Store> store, const Path & storePath)
: store(store), storePath(store->parseStorePath(storePath)) { }
InstallableStorePath(ref<Store> store, StorePath && storePath)
: store(store), storePath(std::move(storePath)) { }
std::string what() override { return store->printStorePath(storePath); }
@ -228,11 +228,11 @@ static std::vector<std::shared_ptr<Installable>> parseInstallables(
result.push_back(std::make_shared<InstallableExpr>(cmd, s));
else if (s.find("/") != std::string::npos) {
auto path = store->toStorePath(store->followLinksToStore(s));
if (store->isStorePath(path))
result.push_back(std::make_shared<InstallableStorePath>(store, path));
try {
result.push_back(std::make_shared<InstallableStorePath>(
store,
store->toStorePath(store->followLinksToStore(s)).first));
} catch (BadStorePath &) { }
}
else if (s == "" || std::regex_match(s, attrPathRegex))

7
src/nix/verify.cc
View file

@ -77,13 +77,16 @@ struct CmdVerify : StorePathsCommand
try {
checkInterrupt();
Activity act2(*logger, lvlInfo, actUnknown, fmt("checking '%s'", storePath));
MaintainCount<std::atomic<size_t>> mcActive(active);
update();
auto info = store->queryPathInfo(store->parseStorePath(storePath));
// Note: info->path can be different from storePath
// for binary cache stores when using --all (since we
// can't enumerate names efficiently).
Activity act2(*logger, lvlInfo, actUnknown, fmt("checking '%s'", store->printStorePath(info->path)));
if (!noContents) {
std::unique_ptr<AbstractHashSink> hashSink;

8
src/nix/why-depends.cc
View file

@ -106,7 +106,11 @@ struct CmdWhyDepends : SourceExprCommand
std::map<StorePath, Node> graph;
for (auto & path : closure)
graph.emplace(path, Node { .path = path, .refs = store->queryPathInfo(path)->references });
graph.emplace(path, Node {
.path = path,
.refs = store->queryPathInfo(path)->references,
.dist = path == dependencyPath ? 0 : inf
});
// Transpose the graph.
for (auto & node : graph)
@ -115,8 +119,6 @@ struct CmdWhyDepends : SourceExprCommand
/* Run Dijkstra's shortest path algorithm to get the distance
of every path in the closure to 'dependency'. */
graph.emplace(dependencyPath, Node { .path = dependencyPath, .dist = 0 });
std::priority_queue<Node *> queue;
queue.push(&graph.at(dependencyPath));

53
tests/binary-cache.sh
View file

@ -182,3 +182,56 @@ clearCacheCache
nix-store -r $outPath --substituters "file://$cacheDir2 file://$cacheDir" --trusted-public-keys "$publicKey"
fi # HAVE_LIBSODIUM
unset _NIX_FORCE_HTTP
# Test 'nix verify --all' on a binary cache.
nix verify -vvvvv --all --store file://$cacheDir --no-trust
# Test local NAR caching.
narCache=$TEST_ROOT/nar-cache
rm -rf $narCache
mkdir $narCache
[[ $(nix cat-store --store "file://$cacheDir?local-nar-cache=$narCache" $outPath/foobar) = FOOBAR ]]
rm -rfv "$cacheDir/nar"
[[ $(nix cat-store --store "file://$cacheDir?local-nar-cache=$narCache" $outPath/foobar) = FOOBAR ]]
(! nix cat-store --store file://$cacheDir $outPath/foobar)
# Test NAR listing generation.
clearCache
outPath=$(nix-build --no-out-link -E '
with import ./config.nix;
mkDerivation {
name = "nar-listing";
buildCommand = "mkdir $out; echo foo > $out/bar; ln -s xyzzy $out/link";
}
')
nix copy --to file://$cacheDir?write-nar-listing=1 $outPath
[[ $(cat $cacheDir/$(basename $outPath).ls) = '{"version":1,"root":{"type":"directory","entries":{"bar":{"type":"regular","size":4,"narOffset":232},"link":{"type":"symlink","target":"xyzzy"}}}}' ]]
# Test debug info index generation.
clearCache
outPath=$(nix-build --no-out-link -E '
with import ./config.nix;
mkDerivation {
name = "debug-info";
buildCommand = "mkdir -p $out/lib/debug/.build-id/02; echo foo > $out/lib/debug/.build-id/02/623eda209c26a59b1a8638ff7752f6b945c26b.debug";
}
')
nix copy --to "file://$cacheDir?index-debug-info=1&compression=none" $outPath
[[ $(cat $cacheDir/debuginfo/02623eda209c26a59b1a8638ff7752f6b945c26b.debug) = '{"archive":"../nar/100vxs724qr46phz8m24iswmg9p3785hsyagz0kchf6q6gf06sw6.nar","member":"lib/debug/.build-id/02/623eda209c26a59b1a8638ff7752f6b945c26b.debug"}' ]]