forked from lix-project/lix
Merge remote-tracking branch 'upstream/master' into optional-derivation-output-storepath
This commit is contained in:
commit
442d43178b
|
@ -1,6 +1,7 @@
|
|||
((c++-mode . (
|
||||
(c-file-style . "k&r")
|
||||
(c-basic-offset . 4)
|
||||
(c-block-comment-prefix . " ")
|
||||
(indent-tabs-mode . nil)
|
||||
(tab-width . 4)
|
||||
(show-trailing-whitespace . t)
|
||||
|
|
10
.github/workflows/test.yml
vendored
10
.github/workflows/test.yml
vendored
|
@ -12,3 +12,13 @@ jobs:
|
|||
- uses: actions/checkout@v2
|
||||
- uses: cachix/install-nix-action@v10
|
||||
- run: nix-build release.nix --arg nix '{ outPath = ./.; revCount = 123; shortRev = "abcdefgh"; }' --arg systems '[ builtins.currentSystem ]' -A installerScript -A perlBindings
|
||||
macos_perf_test:
|
||||
runs-on: macos-latest
|
||||
steps:
|
||||
- name: Disable syspolicy assessments
|
||||
run: |
|
||||
spctl --status
|
||||
sudo spctl --master-disable
|
||||
- uses: actions/checkout@v2
|
||||
- uses: cachix/install-nix-action@v10
|
||||
- run: nix-build release.nix --arg nix '{ outPath = ./.; revCount = 123; shortRev = "abcdefgh"; }' --arg systems '[ builtins.currentSystem ]' -A installerScript -A perlBindings
|
||||
|
|
|
@ -82,4 +82,16 @@ std::string renderContentAddress(std::optional<ContentAddress> ca) {
|
|||
return ca ? renderContentAddress(*ca) : "";
|
||||
}
|
||||
|
||||
Hash getContentAddressHash(const ContentAddress & ca)
|
||||
{
|
||||
return std::visit(overloaded {
|
||||
[](TextHash th) {
|
||||
return th.hash;
|
||||
},
|
||||
[](FixedOutputHash fsh) {
|
||||
return fsh.hash;
|
||||
}
|
||||
}, ca);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -53,4 +53,6 @@ ContentAddress parseContentAddress(std::string_view rawCa);
|
|||
|
||||
std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt);
|
||||
|
||||
Hash getContentAddressHash(const ContentAddress & ca);
|
||||
|
||||
}
|
||||
|
|
|
@ -365,6 +365,9 @@ public:
|
|||
|
||||
Setting<bool> warnDirty{this, true, "warn-dirty",
|
||||
"Whether to warn about dirty Git/Mercurial trees."};
|
||||
|
||||
Setting<size_t> narBufferSize{this, 32 * 1024 * 1024, "nar-buffer-size",
|
||||
"Maximum size of NARs before spilling them to disk."};
|
||||
};
|
||||
|
||||
|
||||
|
|
|
@ -969,7 +969,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
|
|||
|
||||
PathLocks outputLock;
|
||||
|
||||
Path realPath = realStoreDir + "/" + std::string(info.path.to_string());
|
||||
auto realPath = Store::toRealPath(info.path);
|
||||
|
||||
/* Lock the output path. But don't lock if we're being called
|
||||
from a build hook (whose parent process already acquired a
|
||||
|
@ -1040,8 +1040,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
|
|||
/* The first check above is an optimisation to prevent
|
||||
unnecessary lock acquisition. */
|
||||
|
||||
Path realPath = realStoreDir + "/";
|
||||
realPath += dstPath.to_string();
|
||||
auto realPath = Store::toRealPath(dstPath);
|
||||
|
||||
PathLocks outputLock({realPath});
|
||||
|
||||
|
@ -1091,16 +1090,125 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
|
|||
{
|
||||
Path srcPath(absPath(_srcPath));
|
||||
|
||||
/* Read the whole path into memory. This is not a very scalable
|
||||
method for very large paths, but `copyPath' is mainly used for
|
||||
small files. */
|
||||
StringSink sink;
|
||||
if (method == FileIngestionMethod::Recursive)
|
||||
dumpPath(srcPath, sink, filter);
|
||||
else
|
||||
sink.s = make_ref<std::string>(readFile(srcPath));
|
||||
/* For computing the NAR hash. */
|
||||
auto sha256Sink = std::make_unique<HashSink>(htSHA256);
|
||||
|
||||
return addToStoreFromDump(*sink.s, name, method, hashAlgo, repair);
|
||||
/* For computing the store path. In recursive SHA-256 mode, this
|
||||
is the same as the NAR hash, so no need to do it again. */
|
||||
std::unique_ptr<HashSink> hashSink =
|
||||
method == FileIngestionMethod::Recursive && hashAlgo == htSHA256
|
||||
? nullptr
|
||||
: std::make_unique<HashSink>(hashAlgo);
|
||||
|
||||
/* Read the source path into memory, but only if it's up to
|
||||
narBufferSize bytes. If it's larger, write it to a temporary
|
||||
location in the Nix store. If the subsequently computed
|
||||
destination store path is already valid, we just delete the
|
||||
temporary path. Otherwise, we move it to the destination store
|
||||
path. */
|
||||
bool inMemory = true;
|
||||
std::string nar;
|
||||
|
||||
auto source = sinkToSource([&](Sink & sink) {
|
||||
|
||||
LambdaSink sink2([&](const unsigned char * buf, size_t len) {
|
||||
(*sha256Sink)(buf, len);
|
||||
if (hashSink) (*hashSink)(buf, len);
|
||||
|
||||
if (inMemory) {
|
||||
if (nar.size() + len > settings.narBufferSize) {
|
||||
inMemory = false;
|
||||
sink << 1;
|
||||
sink((const unsigned char *) nar.data(), nar.size());
|
||||
nar.clear();
|
||||
} else {
|
||||
nar.append((const char *) buf, len);
|
||||
}
|
||||
}
|
||||
|
||||
if (!inMemory) sink(buf, len);
|
||||
});
|
||||
|
||||
if (method == FileIngestionMethod::Recursive)
|
||||
dumpPath(srcPath, sink2, filter);
|
||||
else
|
||||
readFile(srcPath, sink2);
|
||||
});
|
||||
|
||||
std::unique_ptr<AutoDelete> delTempDir;
|
||||
Path tempPath;
|
||||
|
||||
try {
|
||||
/* Wait for the source coroutine to give us some dummy
|
||||
data. This is so that we don't create the temporary
|
||||
directory if the NAR fits in memory. */
|
||||
readInt(*source);
|
||||
|
||||
auto tempDir = createTempDir(realStoreDir, "add");
|
||||
delTempDir = std::make_unique<AutoDelete>(tempDir);
|
||||
tempPath = tempDir + "/x";
|
||||
|
||||
if (method == FileIngestionMethod::Recursive)
|
||||
restorePath(tempPath, *source);
|
||||
else
|
||||
writeFile(tempPath, *source);
|
||||
|
||||
} catch (EndOfFile &) {
|
||||
if (!inMemory) throw;
|
||||
/* The NAR fits in memory, so we didn't do restorePath(). */
|
||||
}
|
||||
|
||||
auto sha256 = sha256Sink->finish();
|
||||
|
||||
Hash hash = hashSink ? hashSink->finish().first : sha256.first;
|
||||
|
||||
auto dstPath = makeFixedOutputPath(method, hash, name);
|
||||
|
||||
addTempRoot(dstPath);
|
||||
|
||||
if (repair || !isValidPath(dstPath)) {
|
||||
|
||||
/* The first check above is an optimisation to prevent
|
||||
unnecessary lock acquisition. */
|
||||
|
||||
auto realPath = Store::toRealPath(dstPath);
|
||||
|
||||
PathLocks outputLock({realPath});
|
||||
|
||||
if (repair || !isValidPath(dstPath)) {
|
||||
|
||||
deletePath(realPath);
|
||||
|
||||
autoGC();
|
||||
|
||||
if (inMemory) {
|
||||
/* Restore from the NAR in memory. */
|
||||
StringSource source(nar);
|
||||
if (method == FileIngestionMethod::Recursive)
|
||||
restorePath(realPath, source);
|
||||
else
|
||||
writeFile(realPath, source);
|
||||
} else {
|
||||
/* Move the temporary path we restored above. */
|
||||
if (rename(tempPath.c_str(), realPath.c_str()))
|
||||
throw Error("renaming '%s' to '%s'", tempPath, realPath);
|
||||
}
|
||||
|
||||
canonicalisePathMetaData(realPath, -1); // FIXME: merge into restorePath
|
||||
|
||||
optimisePath(realPath);
|
||||
|
||||
ValidPathInfo info(dstPath);
|
||||
info.narHash = sha256.first;
|
||||
info.narSize = sha256.second;
|
||||
info.ca = FixedOutputHash { .method = method, .hash = hash };
|
||||
registerValidPath(info);
|
||||
}
|
||||
|
||||
outputLock.setDeletion(true);
|
||||
}
|
||||
|
||||
return dstPath;
|
||||
}
|
||||
|
||||
|
||||
|
@ -1114,8 +1222,7 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s,
|
|||
|
||||
if (repair || !isValidPath(dstPath)) {
|
||||
|
||||
Path realPath = realStoreDir + "/";
|
||||
realPath += dstPath.to_string();
|
||||
auto realPath = Store::toRealPath(dstPath);
|
||||
|
||||
PathLocks outputLock({realPath});
|
||||
|
||||
|
|
|
@ -7,6 +7,7 @@
|
|||
#include "json.hh"
|
||||
#include "derivations.hh"
|
||||
#include "url.hh"
|
||||
#include "archive.hh"
|
||||
|
||||
#include <future>
|
||||
|
||||
|
@ -221,6 +222,40 @@ StorePath Store::computeStorePathForText(const string & name, const string & s,
|
|||
}
|
||||
|
||||
|
||||
ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
|
||||
FileIngestionMethod method, HashType hashAlgo,
|
||||
std::optional<Hash> expectedCAHash)
|
||||
{
|
||||
/* FIXME: inefficient: we're reading/hashing 'tmpFile' three
|
||||
times. */
|
||||
|
||||
auto [narHash, narSize] = hashPath(htSHA256, srcPath);
|
||||
|
||||
auto hash = method == FileIngestionMethod::Recursive
|
||||
? hashAlgo == htSHA256
|
||||
? narHash
|
||||
: hashPath(hashAlgo, srcPath).first
|
||||
: hashFile(hashAlgo, srcPath);
|
||||
|
||||
if (expectedCAHash && expectedCAHash != hash)
|
||||
throw Error("hash mismatch for '%s'", srcPath);
|
||||
|
||||
ValidPathInfo info(makeFixedOutputPath(method, hash, name));
|
||||
info.narHash = narHash;
|
||||
info.narSize = narSize;
|
||||
info.ca = FixedOutputHash { .method = method, .hash = hash };
|
||||
|
||||
if (!isValidPath(info.path)) {
|
||||
auto source = sinkToSource([&](Sink & sink) {
|
||||
dumpPath(srcPath, sink);
|
||||
});
|
||||
addToStore(info, *source);
|
||||
}
|
||||
|
||||
return info;
|
||||
}
|
||||
|
||||
|
||||
Store::Store(const Params & params)
|
||||
: Config(params)
|
||||
, state({(size_t) pathInfoCacheSize})
|
||||
|
|
|
@ -450,6 +450,13 @@ public:
|
|||
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
|
||||
PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0;
|
||||
|
||||
/* Copy the contents of a path to the store and register the
|
||||
validity the resulting path, using a constant amount of
|
||||
memory. */
|
||||
ValidPathInfo addToStoreSlow(std::string_view name, const Path & srcPath,
|
||||
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
|
||||
std::optional<Hash> expectedCAHash = {});
|
||||
|
||||
// FIXME: remove?
|
||||
virtual StorePath addToStoreFromDump(const string & dump, const string & name,
|
||||
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair)
|
||||
|
|
|
@ -153,14 +153,15 @@ static int _main(int argc, char * * argv)
|
|||
|
||||
/* If an expected hash is given, the file may already exist in
|
||||
the store. */
|
||||
Hash hash, expectedHash(ht);
|
||||
std::optional<Hash> expectedHash;
|
||||
Hash hash;
|
||||
std::optional<StorePath> storePath;
|
||||
if (args.size() == 2) {
|
||||
expectedHash = Hash(args[1], ht);
|
||||
const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
|
||||
storePath = store->makeFixedOutputPath(recursive, expectedHash, name);
|
||||
storePath = store->makeFixedOutputPath(recursive, *expectedHash, name);
|
||||
if (store->isValidPath(*storePath))
|
||||
hash = expectedHash;
|
||||
hash = *expectedHash;
|
||||
else
|
||||
storePath.reset();
|
||||
}
|
||||
|
@ -200,22 +201,12 @@ static int _main(int argc, char * * argv)
|
|||
tmpFile = unpacked;
|
||||
}
|
||||
|
||||
/* FIXME: inefficient; addToStore() will also hash
|
||||
this. */
|
||||
hash = unpack ? hashPath(ht, tmpFile).first : hashFile(ht, tmpFile);
|
||||
const auto method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
|
||||
|
||||
if (expectedHash != Hash(ht) && expectedHash != hash)
|
||||
throw Error("hash mismatch for '%1%'", uri);
|
||||
|
||||
const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
|
||||
|
||||
/* Copy the file to the Nix store. FIXME: if RemoteStore
|
||||
implemented addToStoreFromDump() and downloadFile()
|
||||
supported a sink, we could stream the download directly
|
||||
into the Nix store. */
|
||||
storePath = store->addToStore(name, tmpFile, recursive, ht);
|
||||
|
||||
assert(*storePath == store->makeFixedOutputPath(recursive, hash, name));
|
||||
auto info = store->addToStoreSlow(name, tmpFile, method, ht, expectedHash);
|
||||
storePath = info.path;
|
||||
assert(info.ca);
|
||||
hash = getContentAddressHash(*info.ca);
|
||||
}
|
||||
|
||||
stopProgressBar();
|
||||
|
|
|
@ -174,10 +174,10 @@ static void opAdd(Strings opFlags, Strings opArgs)
|
|||
store. */
|
||||
static void opAddFixed(Strings opFlags, Strings opArgs)
|
||||
{
|
||||
auto recursive = FileIngestionMethod::Flat;
|
||||
auto method = FileIngestionMethod::Flat;
|
||||
|
||||
for (auto & i : opFlags)
|
||||
if (i == "--recursive") recursive = FileIngestionMethod::Recursive;
|
||||
if (i == "--recursive") method = FileIngestionMethod::Recursive;
|
||||
else throw UsageError("unknown flag '%1%'", i);
|
||||
|
||||
if (opArgs.empty())
|
||||
|
@ -187,7 +187,7 @@ static void opAddFixed(Strings opFlags, Strings opArgs)
|
|||
opArgs.pop_front();
|
||||
|
||||
for (auto & i : opArgs)
|
||||
cout << fmt("%s\n", store->printStorePath(store->addToStore(std::string(baseNameOf(i)), i, recursive, hashAlgo)));
|
||||
std::cout << fmt("%s\n", store->printStorePath(store->addToStoreSlow(baseNameOf(i), i, method, hashAlgo).path));
|
||||
}
|
||||
|
||||
|
||||
|
|
Loading…
Reference in a new issue