lix-project/lix
20
55
Fork 31
Code Issues 344 Code Review (Gerrit) Projects 3 Releases Wiki Activity

Merge remote-tracking branch 'upstream/master' into optional-derivation-output-storepath

Browse source
This commit is contained in:
John Ericson 2020-07-11 02:27:45 +00:00
parent a7884970c5 8efa23bb99
commit 442d43178b
10 changed files with 203 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.dir-locals.el
View file

@ -1,6 +1,7 @@
((c++-mode . (
(c-file-style . "k&r")
(c-basic-offset . 4)
(c-block-comment-prefix . " ")
(indent-tabs-mode . nil)
(tab-width . 4)
(show-trailing-whitespace . t)

10
.github/workflows/test.yml vendored
View file

@ -12,3 +12,13 @@ jobs:
- uses: actions/checkout@v2
- uses: cachix/install-nix-action@v10
- run: nix-build release.nix --arg nix '{ outPath = ./.; revCount = 123; shortRev = "abcdefgh"; }' --arg systems '[ builtins.currentSystem ]' -A installerScript -A perlBindings
macos_perf_test:
runs-on: macos-latest
steps:
- name: Disable syspolicy assessments
run: |
spctl --status
sudo spctl --master-disable
- uses: actions/checkout@v2
- uses: cachix/install-nix-action@v10
- run: nix-build release.nix --arg nix '{ outPath = ./.; revCount = 123; shortRev = "abcdefgh"; }' --arg systems '[ builtins.currentSystem ]' -A installerScript -A perlBindings

12
src/libstore/content-address.cc
View file

@ -82,4 +82,16 @@ std::string renderContentAddress(std::optional<ContentAddress> ca) {
return ca ? renderContentAddress(*ca) : "";
}
Hash getContentAddressHash(const ContentAddress & ca)
{
return std::visit(overloaded {
[](TextHash th) {
return th.hash;
},
[](FixedOutputHash fsh) {
return fsh.hash;
}
}, ca);
}
}

2
src/libstore/content-address.hh
View file

@ -53,4 +53,6 @@ ContentAddress parseContentAddress(std::string_view rawCa);
std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt);
Hash getContentAddressHash(const ContentAddress & ca);
}

3
src/libstore/globals.hh
View file

@ -365,6 +365,9 @@ public:
Setting<bool> warnDirty{this, true, "warn-dirty",
"Whether to warn about dirty Git/Mercurial trees."};
Setting<size_t> narBufferSize{this, 32 * 1024 * 1024, "nar-buffer-size",
"Maximum size of NARs before spilling them to disk."};
};

135
src/libstore/local-store.cc
View file

@ -969,7 +969,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
PathLocks outputLock;
Path realPath = realStoreDir + "/" + std::string(info.path.to_string());
auto realPath = Store::toRealPath(info.path);
/* Lock the output path. But don't lock if we're being called
from a build hook (whose parent process already acquired a
@ -1040,8 +1040,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
/* The first check above is an optimisation to prevent
unnecessary lock acquisition. */
Path realPath = realStoreDir + "/";
realPath += dstPath.to_string();
auto realPath = Store::toRealPath(dstPath);
PathLocks outputLock({realPath});
@ -1091,16 +1090,125 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
{
Path srcPath(absPath(_srcPath));
/* Read the whole path into memory. This is not a very scalable
method for very large paths, but `copyPath' is mainly used for
small files. */
StringSink sink;
if (method == FileIngestionMethod::Recursive)
dumpPath(srcPath, sink, filter);
else
sink.s = make_ref<std::string>(readFile(srcPath));
/* For computing the NAR hash. */
auto sha256Sink = std::make_unique<HashSink>(htSHA256);
return addToStoreFromDump(*sink.s, name, method, hashAlgo, repair);
/* For computing the store path. In recursive SHA-256 mode, this
is the same as the NAR hash, so no need to do it again. */
std::unique_ptr<HashSink> hashSink =
method == FileIngestionMethod::Recursive && hashAlgo == htSHA256
? nullptr
: std::make_unique<HashSink>(hashAlgo);
/* Read the source path into memory, but only if it's up to
narBufferSize bytes. If it's larger, write it to a temporary
location in the Nix store. If the subsequently computed
destination store path is already valid, we just delete the
temporary path. Otherwise, we move it to the destination store
path. */
bool inMemory = true;
std::string nar;
auto source = sinkToSource([&](Sink & sink) {
LambdaSink sink2([&](const unsigned char * buf, size_t len) {
(*sha256Sink)(buf, len);
if (hashSink) (*hashSink)(buf, len);
if (inMemory) {
if (nar.size() + len > settings.narBufferSize) {
inMemory = false;
sink << 1;
sink((const unsigned char *) nar.data(), nar.size());
nar.clear();
} else {
nar.append((const char *) buf, len);
}
}
if (!inMemory) sink(buf, len);
});
if (method == FileIngestionMethod::Recursive)
dumpPath(srcPath, sink2, filter);
else
readFile(srcPath, sink2);
});
std::unique_ptr<AutoDelete> delTempDir;
Path tempPath;
try {
/* Wait for the source coroutine to give us some dummy
data. This is so that we don't create the temporary
directory if the NAR fits in memory. */
readInt(*source);
auto tempDir = createTempDir(realStoreDir, "add");
delTempDir = std::make_unique<AutoDelete>(tempDir);
tempPath = tempDir + "/x";
if (method == FileIngestionMethod::Recursive)
restorePath(tempPath, *source);
else
writeFile(tempPath, *source);
} catch (EndOfFile &) {
if (!inMemory) throw;
/* The NAR fits in memory, so we didn't do restorePath(). */
}
auto sha256 = sha256Sink->finish();
Hash hash = hashSink ? hashSink->finish().first : sha256.first;
auto dstPath = makeFixedOutputPath(method, hash, name);
addTempRoot(dstPath);
if (repair || !isValidPath(dstPath)) {
/* The first check above is an optimisation to prevent
unnecessary lock acquisition. */
auto realPath = Store::toRealPath(dstPath);
PathLocks outputLock({realPath});
if (repair || !isValidPath(dstPath)) {
deletePath(realPath);
autoGC();
if (inMemory) {
/* Restore from the NAR in memory. */
StringSource source(nar);
if (method == FileIngestionMethod::Recursive)
restorePath(realPath, source);
else
writeFile(realPath, source);
} else {
/* Move the temporary path we restored above. */
if (rename(tempPath.c_str(), realPath.c_str()))
throw Error("renaming '%s' to '%s'", tempPath, realPath);
}
canonicalisePathMetaData(realPath, -1); // FIXME: merge into restorePath
optimisePath(realPath);
ValidPathInfo info(dstPath);
info.narHash = sha256.first;
info.narSize = sha256.second;
info.ca = FixedOutputHash { .method = method, .hash = hash };
registerValidPath(info);
}
outputLock.setDeletion(true);
}
return dstPath;
}
@ -1114,8 +1222,7 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s,
if (repair || !isValidPath(dstPath)) {
Path realPath = realStoreDir + "/";
realPath += dstPath.to_string();
auto realPath = Store::toRealPath(dstPath);
PathLocks outputLock({realPath});

35
src/libstore/store-api.cc
View file

@ -7,6 +7,7 @@
#include "json.hh"
#include "derivations.hh"
#include "url.hh"
#include "archive.hh"
#include <future>
@ -221,6 +222,40 @@ StorePath Store::computeStorePathForText(const string & name, const string & s,
}
ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
FileIngestionMethod method, HashType hashAlgo,
std::optional<Hash> expectedCAHash)
{
/* FIXME: inefficient: we're reading/hashing 'tmpFile' three
times. */
auto [narHash, narSize] = hashPath(htSHA256, srcPath);
auto hash = method == FileIngestionMethod::Recursive
? hashAlgo == htSHA256
? narHash
: hashPath(hashAlgo, srcPath).first
: hashFile(hashAlgo, srcPath);
if (expectedCAHash && expectedCAHash != hash)
throw Error("hash mismatch for '%s'", srcPath);
ValidPathInfo info(makeFixedOutputPath(method, hash, name));
info.narHash = narHash;
info.narSize = narSize;
info.ca = FixedOutputHash { .method = method, .hash = hash };
if (!isValidPath(info.path)) {
auto source = sinkToSource([&](Sink & sink) {
dumpPath(srcPath, sink);
});
addToStore(info, *source);
}
return info;
}
Store::Store(const Params & params)
: Config(params)
, state({(size_t) pathInfoCacheSize})

7
src/libstore/store-api.hh
View file

@ -450,6 +450,13 @@ public:
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0;
/* Copy the contents of a path to the store and register the
validity the resulting path, using a constant amount of
memory. */
ValidPathInfo addToStoreSlow(std::string_view name, const Path & srcPath,
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
std::optional<Hash> expectedCAHash = {});
// FIXME: remove?
virtual StorePath addToStoreFromDump(const string & dump, const string & name,
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair)

27
src/nix-prefetch-url/nix-prefetch-url.cc
View file

@ -153,14 +153,15 @@ static int _main(int argc, char * * argv)
/* If an expected hash is given, the file may already exist in
the store. */
Hash hash, expectedHash(ht);
std::optional<Hash> expectedHash;
Hash hash;
std::optional<StorePath> storePath;
if (args.size() == 2) {
expectedHash = Hash(args[1], ht);
const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
storePath = store->makeFixedOutputPath(recursive, expectedHash, name);
storePath = store->makeFixedOutputPath(recursive, *expectedHash, name);
if (store->isValidPath(*storePath))
hash = expectedHash;
hash = *expectedHash;
else
storePath.reset();
}
@ -200,22 +201,12 @@ static int _main(int argc, char * * argv)
tmpFile = unpacked;
}
/* FIXME: inefficient; addToStore() will also hash
this. */
hash = unpack ? hashPath(ht, tmpFile).first : hashFile(ht, tmpFile);
const auto method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
if (expectedHash != Hash(ht) && expectedHash != hash)
throw Error("hash mismatch for '%1%'", uri);
const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
/* Copy the file to the Nix store. FIXME: if RemoteStore
implemented addToStoreFromDump() and downloadFile()
supported a sink, we could stream the download directly
into the Nix store. */
storePath = store->addToStore(name, tmpFile, recursive, ht);
assert(*storePath == store->makeFixedOutputPath(recursive, hash, name));
auto info = store->addToStoreSlow(name, tmpFile, method, ht, expectedHash);
storePath = info.path;
assert(info.ca);
hash = getContentAddressHash(*info.ca);
}
stopProgressBar();

6
src/nix-store/nix-store.cc
View file

@ -174,10 +174,10 @@ static void opAdd(Strings opFlags, Strings opArgs)
store. */
static void opAddFixed(Strings opFlags, Strings opArgs)
{
auto recursive = FileIngestionMethod::Flat;
auto method = FileIngestionMethod::Flat;
for (auto & i : opFlags)
if (i == "--recursive") recursive = FileIngestionMethod::Recursive;
if (i == "--recursive") method = FileIngestionMethod::Recursive;
else throw UsageError("unknown flag '%1%'", i);
if (opArgs.empty())
@ -187,7 +187,7 @@ static void opAddFixed(Strings opFlags, Strings opArgs)
opArgs.pop_front();
for (auto & i : opArgs)
cout << fmt("%s\n", store->printStorePath(store->addToStore(std::string(baseNameOf(i)), i, recursive, hashAlgo)));
std::cout << fmt("%s\n", store->printStorePath(store->addToStoreSlow(baseNameOf(i), i, method, hashAlgo).path));
}