Merge remote-tracking branch 'upstream/master' into path-info

This commit is contained in:
John Ericson 2020-08-04 23:16:18 +00:00
commit 36dd7920a2
14 changed files with 69 additions and 55 deletions

View file

@ -153,6 +153,8 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
auto [fdTemp, fnTemp] = createTempFile(); auto [fdTemp, fnTemp] = createTempFile();
AutoDelete autoDelete(fnTemp);
auto now1 = std::chrono::steady_clock::now(); auto now1 = std::chrono::steady_clock::now();
/* Read the NAR simultaneously into a CompressionSink+FileSink (to /* Read the NAR simultaneously into a CompressionSink+FileSink (to
@ -167,6 +169,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
TeeSource teeSource(narSource, *compressionSink); TeeSource teeSource(narSource, *compressionSink);
narAccessor = makeNarAccessor(teeSource); narAccessor = makeNarAccessor(teeSource);
compressionSink->finish(); compressionSink->finish();
fileSink.flush();
} }
auto now2 = std::chrono::steady_clock::now(); auto now2 = std::chrono::steady_clock::now();
@ -279,7 +282,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
if (repair || !fileExists(narInfo->url)) { if (repair || !fileExists(narInfo->url)) {
stats.narWrite++; stats.narWrite++;
upsertFile(narInfo->url, upsertFile(narInfo->url,
std::make_shared<std::fstream>(fnTemp, std::ios_base::in), std::make_shared<std::fstream>(fnTemp, std::ios_base::in | std::ios_base::binary),
"application/x-nix-nar"); "application/x-nix-nar");
} else } else
stats.narWriteAverted++; stats.narWriteAverted++;

View file

@ -1041,20 +1041,6 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
} }
StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
{
Path srcPath(absPath(_srcPath));
auto source = sinkToSource([&](Sink & sink) {
if (method == FileIngestionMethod::Recursive)
dumpPath(srcPath, sink, filter);
else
readFile(srcPath, sink);
});
return addToStoreFromDump(*source, name, method, hashAlgo, repair);
}
StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name,
FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) FileIngestionMethod method, HashType hashAlgo, RepairFlag repair)
{ {

View file

@ -145,16 +145,8 @@ public:
void addToStore(const ValidPathInfo & info, Source & source, void addToStore(const ValidPathInfo & info, Source & source,
RepairFlag repair, CheckSigsFlag checkSigs) override; RepairFlag repair, CheckSigsFlag checkSigs) override;
StorePath addToStore(const string & name, const Path & srcPath,
FileIngestionMethod method, HashType hashAlgo,
PathFilter & filter, RepairFlag repair) override;
/* Like addToStore(), but the contents of the path are contained
in `dump', which is either a NAR serialisation (if recursive ==
true) or simply the contents of a regular file (if recursive ==
false). */
StorePath addToStoreFromDump(Source & dump, const string & name, StorePath addToStoreFromDump(Source & dump, const string & name,
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override; FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) override;
StorePath addTextToStore(const string & name, const string & s, StorePath addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair) override; const StorePathSet & references, RepairFlag repair) override;

View file

@ -1,6 +1,5 @@
#include "globals.hh" #include "globals.hh"
#include "nar-info.hh" #include "nar-info.hh"
#include "store-api.hh"
namespace nix { namespace nix {

View file

@ -2,7 +2,7 @@
#include "types.hh" #include "types.hh"
#include "hash.hh" #include "hash.hh"
#include "path-info.hh" #include "store-api.hh"
namespace nix { namespace nix {

View file

@ -1,17 +1,9 @@
#pragma once #pragma once
// TODO many of thes eare not needed.
#include "path.hh" #include "path.hh"
#include "hash.hh" #include "hash.hh"
#include "content-address.hh" #include "content-address.hh"
#include "serialise.hh"
#include "crypto.hh"
#include "lru-cache.hh"
#include "sync.hh"
#include "globals.hh"
#include "config.hh"
#include <limits>
#include <string> #include <string>
#include <optional> #include <optional>
@ -61,6 +53,16 @@ void PathReferences<Ref>::setReferencesPossiblyToSelf(const Ref & self, std::set
references = refs; references = refs;
} }
struct SubstitutablePathInfo : PathReferences<StorePath>
{
std::optional<StorePath> deriver;
uint64_t downloadSize; /* 0 = unknown or inapplicable */
uint64_t narSize; /* 0 = unknown */
};
typedef std::map<StorePath, SubstitutablePathInfo> SubstitutablePathInfos;
struct ValidPathInfo : PathReferences<StorePath> struct ValidPathInfo : PathReferences<StorePath>
{ {
StorePath path; StorePath path;
@ -145,13 +147,4 @@ struct ValidPathInfo : PathReferences<StorePath>
typedef list<ValidPathInfo> ValidPathInfos; typedef list<ValidPathInfo> ValidPathInfos;
struct SubstitutablePathInfo : PathReferences<StorePath>
{
std::optional<StorePath> deriver;
uint64_t downloadSize; /* 0 = unknown or inapplicable */
uint64_t narSize; /* 0 = unknown */
};
typedef std::map<StorePath, SubstitutablePathInfo> SubstitutablePathInfos;
} }

View file

@ -266,6 +266,10 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
const std::string & mimeType, const std::string & mimeType,
const std::string & contentEncoding) const std::string & contentEncoding)
{ {
istream->seekg(0, istream->end);
auto size = istream->tellg();
istream->seekg(0, istream->beg);
auto maxThreads = std::thread::hardware_concurrency(); auto maxThreads = std::thread::hardware_concurrency();
static std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor> static std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor>
@ -343,10 +347,11 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1) std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1)
.count(); .count();
printInfo("uploaded 's3://%s/%s' in %d ms", printInfo("uploaded 's3://%s/%s' (%d bytes) in %d ms",
bucketName, path, duration); bucketName, path, size, duration);
stats.putTimeMs += duration; stats.putTimeMs += duration;
stats.putBytes += std::max(size, (decltype(size)) 0);
stats.put++; stats.put++;
} }

View file

@ -19,6 +19,7 @@ public:
struct Stats struct Stats
{ {
std::atomic<uint64_t> put{0}; std::atomic<uint64_t> put{0};
std::atomic<uint64_t> putBytes{0};
std::atomic<uint64_t> putTimeMs{0}; std::atomic<uint64_t> putTimeMs{0};
std::atomic<uint64_t> get{0}; std::atomic<uint64_t> get{0};
std::atomic<uint64_t> getBytes{0}; std::atomic<uint64_t> getBytes{0};

View file

@ -239,6 +239,20 @@ StorePath Store::computeStorePathForText(const string & name, const string & s,
} }
StorePath Store::addToStore(const string & name, const Path & _srcPath,
FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
{
Path srcPath(absPath(_srcPath));
auto source = sinkToSource([&](Sink & sink) {
if (method == FileIngestionMethod::Recursive)
dumpPath(srcPath, sink, filter);
else
readFile(srcPath, sink);
});
return addToStoreFromDump(*source, name, method, hashAlgo, repair);
}
/* /*
The aim of this function is to compute in one pass the correct ValidPathInfo for The aim of this function is to compute in one pass the correct ValidPathInfo for
the files that we are trying to add to the store. To accomplish that in one the files that we are trying to add to the store. To accomplish that in one

View file

@ -368,7 +368,7 @@ public:
libutil/archive.hh). */ libutil/archive.hh). */
virtual StorePath addToStore(const string & name, const Path & srcPath, virtual StorePath addToStore(const string & name, const Path & srcPath,
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0; PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair);
/* Copy the contents of a path to the store and register the /* Copy the contents of a path to the store and register the
validity the resulting path, using a constant amount of validity the resulting path, using a constant amount of
@ -377,6 +377,10 @@ public:
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
std::optional<Hash> expectedCAHash = {}); std::optional<Hash> expectedCAHash = {});
/* Like addToStore(), but the contents of the path are contained
in `dump', which is either a NAR serialisation (if recursive ==
true) or simply the contents of a regular file (if recursive ==
false). */
// FIXME: remove? // FIXME: remove?
virtual StorePath addToStoreFromDump(Source & dump, const string & name, virtual StorePath addToStoreFromDump(Source & dump, const string & name,
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair)

View file

@ -494,6 +494,7 @@ std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix)
{ {
Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX"); Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX");
// Strictly speaking, this is UB, but who cares... // Strictly speaking, this is UB, but who cares...
// FIXME: use O_TMPFILE.
AutoCloseFD fd(mkstemp((char *) tmpl.c_str())); AutoCloseFD fd(mkstemp((char *) tmpl.c_str()));
if (!fd) if (!fd)
throw SysError("creating temporary file '%s'", tmpl); throw SysError("creating temporary file '%s'", tmpl);

View file

@ -437,7 +437,7 @@ ref<eval_cache::EvalCache> openEvalCache(
std::shared_ptr<flake::LockedFlake> lockedFlake, std::shared_ptr<flake::LockedFlake> lockedFlake,
bool useEvalCache) bool useEvalCache)
{ {
auto fingerprint = lockedFlake->getFingerprint(); auto fingerprint = lockedFlake->getFingerprint();
return make_ref<nix::eval_cache::EvalCache>( return make_ref<nix::eval_cache::EvalCache>(
useEvalCache && evalSettings.pureEval useEvalCache && evalSettings.pureEval
? std::optional { std::cref(fingerprint) } ? std::optional { std::cref(fingerprint) }

View file

@ -218,7 +218,9 @@ outPath=$(nix-build --no-out-link -E '
nix copy --to file://$cacheDir?write-nar-listing=1 $outPath nix copy --to file://$cacheDir?write-nar-listing=1 $outPath
[[ $(cat $cacheDir/$(basename $outPath).ls) = '{"version":1,"root":{"type":"directory","entries":{"bar":{"type":"regular","size":4,"narOffset":232},"link":{"type":"symlink","target":"xyzzy"}}}}' ]] diff -u \
<(jq -S < $cacheDir/$(basename $outPath).ls) \
<(echo '{"version":1,"root":{"type":"directory","entries":{"bar":{"type":"regular","size":4,"narOffset":232},"link":{"type":"symlink","target":"xyzzy"}}}}' | jq -S)
# Test debug info index generation. # Test debug info index generation.
@ -234,4 +236,6 @@ outPath=$(nix-build --no-out-link -E '
nix copy --to "file://$cacheDir?index-debug-info=1&compression=none" $outPath nix copy --to "file://$cacheDir?index-debug-info=1&compression=none" $outPath
[[ $(cat $cacheDir/debuginfo/02623eda209c26a59b1a8638ff7752f6b945c26b.debug) = '{"archive":"../nar/100vxs724qr46phz8m24iswmg9p3785hsyagz0kchf6q6gf06sw6.nar","member":"lib/debug/.build-id/02/623eda209c26a59b1a8638ff7752f6b945c26b.debug"}' ]] diff -u \
<(cat $cacheDir/debuginfo/02623eda209c26a59b1a8638ff7752f6b945c26b.debug | jq -S) \
<(echo '{"archive":"../nar/100vxs724qr46phz8m24iswmg9p3785hsyagz0kchf6q6gf06sw6.nar","member":"lib/debug/.build-id/02/623eda209c26a59b1a8638ff7752f6b945c26b.debug"}' | jq -S)

View file

@ -26,12 +26,24 @@ nix cat-store $storePath/foo/baz > baz.cat-nar
diff -u baz.cat-nar $storePath/foo/baz diff -u baz.cat-nar $storePath/foo/baz
# Test --json. # Test --json.
[[ $(nix ls-nar --json $narFile /) = '{"type":"directory","entries":{"foo":{},"foo-x":{},"qux":{},"zyx":{}}}' ]] diff -u \
[[ $(nix ls-nar --json -R $narFile /foo) = '{"type":"directory","entries":{"bar":{"type":"regular","size":0,"narOffset":368},"baz":{"type":"regular","size":0,"narOffset":552},"data":{"type":"regular","size":58,"narOffset":736}}}' ]] <(nix ls-nar --json $narFile / | jq -S) \
[[ $(nix ls-nar --json -R $narFile /foo/bar) = '{"type":"regular","size":0,"narOffset":368}' ]] <(echo '{"type":"directory","entries":{"foo":{},"foo-x":{},"qux":{},"zyx":{}}}' | jq -S)
[[ $(nix ls-store --json $storePath) = '{"type":"directory","entries":{"foo":{},"foo-x":{},"qux":{},"zyx":{}}}' ]] diff -u \
[[ $(nix ls-store --json -R $storePath/foo) = '{"type":"directory","entries":{"bar":{"type":"regular","size":0},"baz":{"type":"regular","size":0},"data":{"type":"regular","size":58}}}' ]] <(nix ls-nar --json -R $narFile /foo | jq -S) \
[[ $(nix ls-store --json -R $storePath/foo/bar) = '{"type":"regular","size":0}' ]] <(echo '{"type":"directory","entries":{"bar":{"type":"regular","size":0,"narOffset":368},"baz":{"type":"regular","size":0,"narOffset":552},"data":{"type":"regular","size":58,"narOffset":736}}}' | jq -S)
diff -u \
<(nix ls-nar --json -R $narFile /foo/bar | jq -S) \
<(echo '{"type":"regular","size":0,"narOffset":368}' | jq -S)
diff -u \
<(nix ls-store --json $storePath | jq -S) \
<(echo '{"type":"directory","entries":{"foo":{},"foo-x":{},"qux":{},"zyx":{}}}' | jq -S)
diff -u \
<(nix ls-store --json -R $storePath/foo | jq -S) \
<(echo '{"type":"directory","entries":{"bar":{"type":"regular","size":0},"baz":{"type":"regular","size":0},"data":{"type":"regular","size":58}}}' | jq -S)
diff -u \
<(nix ls-store --json -R $storePath/foo/bar| jq -S) \
<(echo '{"type":"regular","size":0}' | jq -S)
# Test missing files. # Test missing files.
nix ls-store --json -R $storePath/xyzzy 2>&1 | grep 'does not exist in NAR' nix ls-store --json -R $storePath/xyzzy 2>&1 | grep 'does not exist in NAR'