forked from lix-project/lix
Merge branch 'misc-ca' of github.com:obsidiansystems/nix into derivation-primop-floating-output
This commit is contained in:
commit
c36916eca0
25
README.md
25
README.md
|
@ -12,7 +12,7 @@ for more details.
|
||||||
On Linux and macOS the easiest way to Install Nix is to run the following shell command
|
On Linux and macOS the easiest way to Install Nix is to run the following shell command
|
||||||
(as a user other than root):
|
(as a user other than root):
|
||||||
|
|
||||||
```
|
```console
|
||||||
$ curl -L https://nixos.org/nix/install | sh
|
$ curl -L https://nixos.org/nix/install | sh
|
||||||
```
|
```
|
||||||
|
|
||||||
|
@ -20,27 +20,8 @@ Information on additional installation methods is available on the [Nix download
|
||||||
|
|
||||||
## Building And Developing
|
## Building And Developing
|
||||||
|
|
||||||
### Building Nix
|
See our [Hacking guide](https://hydra.nixos.org/job/nix/master/build.x86_64-linux/latest/download-by-type/doc/manual#chap-hacking) in our manual for instruction on how to
|
||||||
|
build nix from source with nix-build or how to get a development environment.
|
||||||
You can build Nix using one of the targets provided by [release.nix](./release.nix):
|
|
||||||
|
|
||||||
```
|
|
||||||
$ nix-build ./release.nix -A build.aarch64-linux
|
|
||||||
$ nix-build ./release.nix -A build.x86_64-darwin
|
|
||||||
$ nix-build ./release.nix -A build.i686-linux
|
|
||||||
$ nix-build ./release.nix -A build.x86_64-linux
|
|
||||||
```
|
|
||||||
|
|
||||||
### Development Environment
|
|
||||||
|
|
||||||
You can use the provided `shell.nix` to get a working development environment:
|
|
||||||
|
|
||||||
```
|
|
||||||
$ nix-shell
|
|
||||||
$ ./bootstrap.sh
|
|
||||||
$ ./configure
|
|
||||||
$ make
|
|
||||||
```
|
|
||||||
|
|
||||||
## Additional Resources
|
## Additional Resources
|
||||||
|
|
||||||
|
|
|
@ -7,15 +7,34 @@
|
||||||
<para>This section provides some notes on how to hack on Nix. To get
|
<para>This section provides some notes on how to hack on Nix. To get
|
||||||
the latest version of Nix from GitHub:
|
the latest version of Nix from GitHub:
|
||||||
<screen>
|
<screen>
|
||||||
$ git clone git://github.com/NixOS/nix.git
|
$ git clone https://github.com/NixOS/nix.git
|
||||||
$ cd nix
|
$ cd nix
|
||||||
</screen>
|
</screen>
|
||||||
</para>
|
</para>
|
||||||
|
|
||||||
<para>To build it and its dependencies:
|
<para>To build Nix for the current operating system/architecture use
|
||||||
|
|
||||||
<screen>
|
<screen>
|
||||||
$ nix-build release.nix -A build.x86_64-linux
|
$ nix-build
|
||||||
</screen>
|
</screen>
|
||||||
|
|
||||||
|
or if you have a flakes-enabled nix:
|
||||||
|
|
||||||
|
<screen>
|
||||||
|
$ nix build
|
||||||
|
</screen>
|
||||||
|
|
||||||
|
This will build <literal>defaultPackage</literal> attribute defined in the <literal>flake.nix</literal> file.
|
||||||
|
|
||||||
|
To build for other platforms add one of the following suffixes to it: aarch64-linux,
|
||||||
|
i686-linux, x86_64-darwin, x86_64-linux.
|
||||||
|
|
||||||
|
i.e.
|
||||||
|
|
||||||
|
<screen>
|
||||||
|
nix-build -A defaultPackage.x86_64-linux
|
||||||
|
</screen>
|
||||||
|
|
||||||
</para>
|
</para>
|
||||||
|
|
||||||
<para>To build all dependencies and start a shell in which all
|
<para>To build all dependencies and start a shell in which all
|
||||||
|
@ -27,13 +46,27 @@ $ nix-shell
|
||||||
To build Nix itself in this shell:
|
To build Nix itself in this shell:
|
||||||
<screen>
|
<screen>
|
||||||
[nix-shell]$ ./bootstrap.sh
|
[nix-shell]$ ./bootstrap.sh
|
||||||
[nix-shell]$ configurePhase
|
[nix-shell]$ ./configure $configureFlags
|
||||||
[nix-shell]$ make
|
[nix-shell]$ make -j $NIX_BUILD_CORES
|
||||||
</screen>
|
</screen>
|
||||||
To install it in <literal>$(pwd)/inst</literal> and test it:
|
To install it in <literal>$(pwd)/inst</literal> and test it:
|
||||||
<screen>
|
<screen>
|
||||||
[nix-shell]$ make install
|
[nix-shell]$ make install
|
||||||
[nix-shell]$ make installcheck
|
[nix-shell]$ make installcheck
|
||||||
|
[nix-shell]$ ./inst/bin/nix --version
|
||||||
|
nix (Nix) 2.4
|
||||||
|
</screen>
|
||||||
|
|
||||||
|
If you have a flakes-enabled nix you can replace:
|
||||||
|
|
||||||
|
<screen>
|
||||||
|
$ nix-shell
|
||||||
|
</screen>
|
||||||
|
|
||||||
|
by:
|
||||||
|
|
||||||
|
<screen>
|
||||||
|
$ nix develop
|
||||||
</screen>
|
</screen>
|
||||||
|
|
||||||
</para>
|
</para>
|
||||||
|
|
|
@ -2774,7 +2774,7 @@ struct RestrictedStore : public LocalFSStore
|
||||||
goal.addDependency(info.path);
|
goal.addDependency(info.path);
|
||||||
}
|
}
|
||||||
|
|
||||||
StorePath addToStoreFromDump(const string & dump, const string & name,
|
StorePath addToStoreFromDump(Source & dump, const string & name,
|
||||||
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override
|
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override
|
||||||
{
|
{
|
||||||
auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair);
|
auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair);
|
||||||
|
|
|
@ -350,21 +350,24 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
||||||
}
|
}
|
||||||
|
|
||||||
case wopAddToStore: {
|
case wopAddToStore: {
|
||||||
std::string s, baseName;
|
HashType hashAlgo;
|
||||||
|
std::string baseName;
|
||||||
FileIngestionMethod method;
|
FileIngestionMethod method;
|
||||||
{
|
{
|
||||||
bool fixed; uint8_t recursive;
|
bool fixed;
|
||||||
from >> baseName >> fixed /* obsolete */ >> recursive >> s;
|
uint8_t recursive;
|
||||||
|
std::string hashAlgoRaw;
|
||||||
|
from >> baseName >> fixed /* obsolete */ >> recursive >> hashAlgoRaw;
|
||||||
if (recursive > (uint8_t) FileIngestionMethod::Recursive)
|
if (recursive > (uint8_t) FileIngestionMethod::Recursive)
|
||||||
throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive);
|
throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive);
|
||||||
method = FileIngestionMethod { recursive };
|
method = FileIngestionMethod { recursive };
|
||||||
/* Compatibility hack. */
|
/* Compatibility hack. */
|
||||||
if (!fixed) {
|
if (!fixed) {
|
||||||
s = "sha256";
|
hashAlgoRaw = "sha256";
|
||||||
method = FileIngestionMethod::Recursive;
|
method = FileIngestionMethod::Recursive;
|
||||||
}
|
}
|
||||||
|
hashAlgo = parseHashType(hashAlgoRaw);
|
||||||
}
|
}
|
||||||
HashType hashAlgo = parseHashType(s);
|
|
||||||
|
|
||||||
StringSink saved;
|
StringSink saved;
|
||||||
TeeSource savedNARSource(from, saved);
|
TeeSource savedNARSource(from, saved);
|
||||||
|
@ -382,7 +385,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
||||||
logger->startWork();
|
logger->startWork();
|
||||||
if (!savedRegular.regular) throw Error("regular file expected");
|
if (!savedRegular.regular) throw Error("regular file expected");
|
||||||
|
|
||||||
auto path = store->addToStoreFromDump(*saved.s, baseName, method, hashAlgo);
|
// FIXME: try to stream directly from `from`.
|
||||||
|
StringSource dumpSource { *saved.s };
|
||||||
|
auto path = store->addToStoreFromDump(dumpSource, baseName, method, hashAlgo);
|
||||||
logger->stopWork();
|
logger->stopWork();
|
||||||
|
|
||||||
to << store->printStorePath(path);
|
to << store->printStorePath(path);
|
||||||
|
|
|
@ -124,7 +124,7 @@ struct curlFileTransfer : public FileTransfer
|
||||||
if (requestHeaders) curl_slist_free_all(requestHeaders);
|
if (requestHeaders) curl_slist_free_all(requestHeaders);
|
||||||
try {
|
try {
|
||||||
if (!done)
|
if (!done)
|
||||||
fail(FileTransferError(Interrupted, "download of '%s' was interrupted", request.uri));
|
fail(FileTransferError(Interrupted, nullptr, "download of '%s' was interrupted", request.uri));
|
||||||
} catch (...) {
|
} catch (...) {
|
||||||
ignoreException();
|
ignoreException();
|
||||||
}
|
}
|
||||||
|
@ -145,6 +145,7 @@ struct curlFileTransfer : public FileTransfer
|
||||||
|
|
||||||
LambdaSink finalSink;
|
LambdaSink finalSink;
|
||||||
std::shared_ptr<CompressionSink> decompressionSink;
|
std::shared_ptr<CompressionSink> decompressionSink;
|
||||||
|
std::optional<StringSink> errorSink;
|
||||||
|
|
||||||
std::exception_ptr writeException;
|
std::exception_ptr writeException;
|
||||||
|
|
||||||
|
@ -154,9 +155,19 @@ struct curlFileTransfer : public FileTransfer
|
||||||
size_t realSize = size * nmemb;
|
size_t realSize = size * nmemb;
|
||||||
result.bodySize += realSize;
|
result.bodySize += realSize;
|
||||||
|
|
||||||
if (!decompressionSink)
|
if (!decompressionSink) {
|
||||||
decompressionSink = makeDecompressionSink(encoding, finalSink);
|
decompressionSink = makeDecompressionSink(encoding, finalSink);
|
||||||
|
if (! successfulStatuses.count(getHTTPStatus())) {
|
||||||
|
// In this case we want to construct a TeeSink, to keep
|
||||||
|
// the response around (which we figure won't be big
|
||||||
|
// like an actual download should be) to improve error
|
||||||
|
// messages.
|
||||||
|
errorSink = StringSink { };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (errorSink)
|
||||||
|
(*errorSink)((unsigned char *) contents, realSize);
|
||||||
(*decompressionSink)((unsigned char *) contents, realSize);
|
(*decompressionSink)((unsigned char *) contents, realSize);
|
||||||
|
|
||||||
return realSize;
|
return realSize;
|
||||||
|
@ -412,16 +423,21 @@ struct curlFileTransfer : public FileTransfer
|
||||||
|
|
||||||
attempt++;
|
attempt++;
|
||||||
|
|
||||||
|
std::shared_ptr<std::string> response;
|
||||||
|
if (errorSink)
|
||||||
|
response = errorSink->s;
|
||||||
auto exc =
|
auto exc =
|
||||||
code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted
|
code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted
|
||||||
? FileTransferError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri))
|
? FileTransferError(Interrupted, response, "%s of '%s' was interrupted", request.verb(), request.uri)
|
||||||
: httpStatus != 0
|
: httpStatus != 0
|
||||||
? FileTransferError(err,
|
? FileTransferError(err,
|
||||||
|
response,
|
||||||
fmt("unable to %s '%s': HTTP error %d ('%s')",
|
fmt("unable to %s '%s': HTTP error %d ('%s')",
|
||||||
request.verb(), request.uri, httpStatus, statusMsg)
|
request.verb(), request.uri, httpStatus, statusMsg)
|
||||||
+ (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code)))
|
+ (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code)))
|
||||||
)
|
)
|
||||||
: FileTransferError(err,
|
: FileTransferError(err,
|
||||||
|
response,
|
||||||
fmt("unable to %s '%s': %s (%d)",
|
fmt("unable to %s '%s': %s (%d)",
|
||||||
request.verb(), request.uri, curl_easy_strerror(code), code));
|
request.verb(), request.uri, curl_easy_strerror(code), code));
|
||||||
|
|
||||||
|
@ -679,7 +695,7 @@ struct curlFileTransfer : public FileTransfer
|
||||||
auto s3Res = s3Helper.getObject(bucketName, key);
|
auto s3Res = s3Helper.getObject(bucketName, key);
|
||||||
FileTransferResult res;
|
FileTransferResult res;
|
||||||
if (!s3Res.data)
|
if (!s3Res.data)
|
||||||
throw FileTransferError(NotFound, fmt("S3 object '%s' does not exist", request.uri));
|
throw FileTransferError(NotFound, nullptr, "S3 object '%s' does not exist", request.uri);
|
||||||
res.data = s3Res.data;
|
res.data = s3Res.data;
|
||||||
callback(std::move(res));
|
callback(std::move(res));
|
||||||
#else
|
#else
|
||||||
|
@ -824,6 +840,21 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
template<typename... Args>
|
||||||
|
FileTransferError::FileTransferError(FileTransfer::Error error, std::shared_ptr<string> response, const Args & ... args)
|
||||||
|
: Error(args...), error(error), response(response)
|
||||||
|
{
|
||||||
|
const auto hf = hintfmt(args...);
|
||||||
|
// FIXME: Due to https://github.com/NixOS/nix/issues/3841 we don't know how
|
||||||
|
// to print different messages for different verbosity levels. For now
|
||||||
|
// we add some heuristics for detecting when we want to show the response.
|
||||||
|
if (response && (response->size() < 1024 || response->find("<html>") != string::npos)) {
|
||||||
|
err.hint = hintfmt("%1%\n\nresponse body:\n\n%2%", normaltxt(hf.str()), *response);
|
||||||
|
} else {
|
||||||
|
err.hint = hf;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
bool isUri(const string & s)
|
bool isUri(const string & s)
|
||||||
{
|
{
|
||||||
if (s.compare(0, 8, "channel:") == 0) return true;
|
if (s.compare(0, 8, "channel:") == 0) return true;
|
||||||
|
|
|
@ -103,10 +103,12 @@ class FileTransferError : public Error
|
||||||
{
|
{
|
||||||
public:
|
public:
|
||||||
FileTransfer::Error error;
|
FileTransfer::Error error;
|
||||||
|
std::shared_ptr<string> response; // intentionally optional
|
||||||
|
|
||||||
template<typename... Args>
|
template<typename... Args>
|
||||||
FileTransferError(FileTransfer::Error error, const Args & ... args)
|
FileTransferError(FileTransfer::Error error, std::shared_ptr<string> response, const Args & ... args);
|
||||||
: Error(args...), error(error)
|
|
||||||
{ }
|
virtual const char* sname() const override { return "FileTransferError"; }
|
||||||
};
|
};
|
||||||
|
|
||||||
bool isUri(const string & s);
|
bool isUri(const string & s);
|
||||||
|
|
|
@ -1041,82 +1041,26 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
StorePath LocalStore::addToStoreFromDump(const string & dump, const string & name,
|
|
||||||
FileIngestionMethod method, HashType hashAlgo, RepairFlag repair)
|
|
||||||
{
|
|
||||||
Hash h = hashString(hashAlgo, dump);
|
|
||||||
|
|
||||||
auto dstPath = makeFixedOutputPath(method, h, name);
|
|
||||||
|
|
||||||
addTempRoot(dstPath);
|
|
||||||
|
|
||||||
if (repair || !isValidPath(dstPath)) {
|
|
||||||
|
|
||||||
/* The first check above is an optimisation to prevent
|
|
||||||
unnecessary lock acquisition. */
|
|
||||||
|
|
||||||
auto realPath = Store::toRealPath(dstPath);
|
|
||||||
|
|
||||||
PathLocks outputLock({realPath});
|
|
||||||
|
|
||||||
if (repair || !isValidPath(dstPath)) {
|
|
||||||
|
|
||||||
deletePath(realPath);
|
|
||||||
|
|
||||||
autoGC();
|
|
||||||
|
|
||||||
if (method == FileIngestionMethod::Recursive) {
|
|
||||||
StringSource source(dump);
|
|
||||||
restorePath(realPath, source);
|
|
||||||
} else
|
|
||||||
writeFile(realPath, dump);
|
|
||||||
|
|
||||||
canonicalisePathMetaData(realPath, -1);
|
|
||||||
|
|
||||||
/* Register the SHA-256 hash of the NAR serialisation of
|
|
||||||
the path in the database. We may just have computed it
|
|
||||||
above (if called with recursive == true and hashAlgo ==
|
|
||||||
sha256); otherwise, compute it here. */
|
|
||||||
HashResult hash;
|
|
||||||
if (method == FileIngestionMethod::Recursive) {
|
|
||||||
hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump);
|
|
||||||
hash.second = dump.size();
|
|
||||||
} else
|
|
||||||
hash = hashPath(htSHA256, realPath);
|
|
||||||
|
|
||||||
optimisePath(realPath); // FIXME: combine with hashPath()
|
|
||||||
|
|
||||||
ValidPathInfo info(dstPath);
|
|
||||||
info.narHash = hash.first;
|
|
||||||
info.narSize = hash.second;
|
|
||||||
info.ca = FixedOutputHash { .method = method, .hash = h };
|
|
||||||
registerValidPath(info);
|
|
||||||
}
|
|
||||||
|
|
||||||
outputLock.setDeletion(true);
|
|
||||||
}
|
|
||||||
|
|
||||||
return dstPath;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
|
StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
|
||||||
FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
|
FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
|
||||||
{
|
{
|
||||||
Path srcPath(absPath(_srcPath));
|
Path srcPath(absPath(_srcPath));
|
||||||
|
auto source = sinkToSource([&](Sink & sink) {
|
||||||
|
if (method == FileIngestionMethod::Recursive)
|
||||||
|
dumpPath(srcPath, sink, filter);
|
||||||
|
else
|
||||||
|
readFile(srcPath, sink);
|
||||||
|
});
|
||||||
|
return addToStoreFromDump(*source, name, method, hashAlgo, repair);
|
||||||
|
}
|
||||||
|
|
||||||
if (method != FileIngestionMethod::Recursive)
|
|
||||||
return addToStoreFromDump(readFile(srcPath), name, method, hashAlgo, repair);
|
|
||||||
|
|
||||||
/* For computing the NAR hash. */
|
StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name,
|
||||||
auto sha256Sink = std::make_unique<HashSink>(htSHA256);
|
FileIngestionMethod method, HashType hashAlgo, RepairFlag repair)
|
||||||
|
{
|
||||||
/* For computing the store path. In recursive SHA-256 mode, this
|
/* For computing the store path. */
|
||||||
is the same as the NAR hash, so no need to do it again. */
|
auto hashSink = std::make_unique<HashSink>(hashAlgo);
|
||||||
std::unique_ptr<HashSink> hashSink =
|
TeeSource source { source0, *hashSink };
|
||||||
hashAlgo == htSHA256
|
|
||||||
? nullptr
|
|
||||||
: std::make_unique<HashSink>(hashAlgo);
|
|
||||||
|
|
||||||
/* Read the source path into memory, but only if it's up to
|
/* Read the source path into memory, but only if it's up to
|
||||||
narBufferSize bytes. If it's larger, write it to a temporary
|
narBufferSize bytes. If it's larger, write it to a temporary
|
||||||
|
@ -1124,55 +1068,49 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
|
||||||
destination store path is already valid, we just delete the
|
destination store path is already valid, we just delete the
|
||||||
temporary path. Otherwise, we move it to the destination store
|
temporary path. Otherwise, we move it to the destination store
|
||||||
path. */
|
path. */
|
||||||
bool inMemory = true;
|
bool inMemory = false;
|
||||||
std::string nar;
|
|
||||||
|
|
||||||
auto source = sinkToSource([&](Sink & sink) {
|
std::string dump;
|
||||||
|
|
||||||
LambdaSink sink2([&](const unsigned char * buf, size_t len) {
|
/* Fill out buffer, and decide whether we are working strictly in
|
||||||
(*sha256Sink)(buf, len);
|
memory based on whether we break out because the buffer is full
|
||||||
if (hashSink) (*hashSink)(buf, len);
|
or the original source is empty */
|
||||||
|
while (dump.size() < settings.narBufferSize) {
|
||||||
if (inMemory) {
|
auto oldSize = dump.size();
|
||||||
if (nar.size() + len > settings.narBufferSize) {
|
constexpr size_t chunkSize = 65536;
|
||||||
inMemory = false;
|
auto want = std::min(chunkSize, settings.narBufferSize - oldSize);
|
||||||
sink << 1;
|
dump.resize(oldSize + want);
|
||||||
sink((const unsigned char *) nar.data(), nar.size());
|
auto got = 0;
|
||||||
nar.clear();
|
try {
|
||||||
} else {
|
got = source.read((uint8_t *) dump.data() + oldSize, want);
|
||||||
nar.append((const char *) buf, len);
|
} catch (EndOfFile &) {
|
||||||
|
inMemory = true;
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
|
dump.resize(oldSize + got);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!inMemory) sink(buf, len);
|
|
||||||
});
|
|
||||||
|
|
||||||
dumpPath(srcPath, sink2, filter);
|
|
||||||
});
|
|
||||||
|
|
||||||
std::unique_ptr<AutoDelete> delTempDir;
|
std::unique_ptr<AutoDelete> delTempDir;
|
||||||
Path tempPath;
|
Path tempPath;
|
||||||
|
|
||||||
try {
|
if (!inMemory) {
|
||||||
/* Wait for the source coroutine to give us some dummy
|
/* Drain what we pulled so far, and then keep on pulling */
|
||||||
data. This is so that we don't create the temporary
|
StringSource dumpSource { dump };
|
||||||
directory if the NAR fits in memory. */
|
ChainSource bothSource { dumpSource, source };
|
||||||
readInt(*source);
|
|
||||||
|
|
||||||
auto tempDir = createTempDir(realStoreDir, "add");
|
auto tempDir = createTempDir(realStoreDir, "add");
|
||||||
delTempDir = std::make_unique<AutoDelete>(tempDir);
|
delTempDir = std::make_unique<AutoDelete>(tempDir);
|
||||||
tempPath = tempDir + "/x";
|
tempPath = tempDir + "/x";
|
||||||
|
|
||||||
restorePath(tempPath, *source);
|
if (method == FileIngestionMethod::Recursive)
|
||||||
|
restorePath(tempPath, bothSource);
|
||||||
|
else
|
||||||
|
writeFile(tempPath, bothSource);
|
||||||
|
|
||||||
} catch (EndOfFile &) {
|
dump.clear();
|
||||||
if (!inMemory) throw;
|
|
||||||
/* The NAR fits in memory, so we didn't do restorePath(). */
|
|
||||||
}
|
}
|
||||||
|
|
||||||
auto sha256 = sha256Sink->finish();
|
auto [hash, size] = hashSink->finish();
|
||||||
|
|
||||||
Hash hash = hashSink ? hashSink->finish().first : sha256.first;
|
|
||||||
|
|
||||||
auto dstPath = makeFixedOutputPath(method, hash, name);
|
auto dstPath = makeFixedOutputPath(method, hash, name);
|
||||||
|
|
||||||
|
@ -1194,22 +1132,34 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
|
||||||
autoGC();
|
autoGC();
|
||||||
|
|
||||||
if (inMemory) {
|
if (inMemory) {
|
||||||
|
StringSource dumpSource { dump };
|
||||||
/* Restore from the NAR in memory. */
|
/* Restore from the NAR in memory. */
|
||||||
StringSource source(nar);
|
if (method == FileIngestionMethod::Recursive)
|
||||||
restorePath(realPath, source);
|
restorePath(realPath, dumpSource);
|
||||||
|
else
|
||||||
|
writeFile(realPath, dumpSource);
|
||||||
} else {
|
} else {
|
||||||
/* Move the temporary path we restored above. */
|
/* Move the temporary path we restored above. */
|
||||||
if (rename(tempPath.c_str(), realPath.c_str()))
|
if (rename(tempPath.c_str(), realPath.c_str()))
|
||||||
throw Error("renaming '%s' to '%s'", tempPath, realPath);
|
throw Error("renaming '%s' to '%s'", tempPath, realPath);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* For computing the nar hash. In recursive SHA-256 mode, this
|
||||||
|
is the same as the store hash, so no need to do it again. */
|
||||||
|
auto narHash = std::pair { hash, size };
|
||||||
|
if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256) {
|
||||||
|
HashSink narSink { htSHA256 };
|
||||||
|
dumpPath(realPath, narSink);
|
||||||
|
narHash = narSink.finish();
|
||||||
|
}
|
||||||
|
|
||||||
canonicalisePathMetaData(realPath, -1); // FIXME: merge into restorePath
|
canonicalisePathMetaData(realPath, -1); // FIXME: merge into restorePath
|
||||||
|
|
||||||
optimisePath(realPath);
|
optimisePath(realPath);
|
||||||
|
|
||||||
ValidPathInfo info(dstPath);
|
ValidPathInfo info(dstPath);
|
||||||
info.narHash = sha256.first;
|
info.narHash = narHash.first;
|
||||||
info.narSize = sha256.second;
|
info.narSize = narHash.second;
|
||||||
info.ca = FixedOutputHash { .method = method, .hash = hash };
|
info.ca = FixedOutputHash { .method = method, .hash = hash };
|
||||||
registerValidPath(info);
|
registerValidPath(info);
|
||||||
}
|
}
|
||||||
|
|
|
@ -153,7 +153,7 @@ public:
|
||||||
in `dump', which is either a NAR serialisation (if recursive ==
|
in `dump', which is either a NAR serialisation (if recursive ==
|
||||||
true) or simply the contents of a regular file (if recursive ==
|
true) or simply the contents of a regular file (if recursive ==
|
||||||
false). */
|
false). */
|
||||||
StorePath addToStoreFromDump(const string & dump, const string & name,
|
StorePath addToStoreFromDump(Source & dump, const string & name,
|
||||||
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override;
|
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override;
|
||||||
|
|
||||||
StorePath addTextToStore(const string & name, const string & s,
|
StorePath addTextToStore(const string & name, const string & s,
|
||||||
|
|
|
@ -48,13 +48,12 @@ static void search(const unsigned char * s, size_t len,
|
||||||
|
|
||||||
struct RefScanSink : Sink
|
struct RefScanSink : Sink
|
||||||
{
|
{
|
||||||
HashSink hashSink;
|
|
||||||
StringSet hashes;
|
StringSet hashes;
|
||||||
StringSet seen;
|
StringSet seen;
|
||||||
|
|
||||||
string tail;
|
string tail;
|
||||||
|
|
||||||
RefScanSink() : hashSink(htSHA256) { }
|
RefScanSink() { }
|
||||||
|
|
||||||
void operator () (const unsigned char * data, size_t len);
|
void operator () (const unsigned char * data, size_t len);
|
||||||
};
|
};
|
||||||
|
@ -62,8 +61,6 @@ struct RefScanSink : Sink
|
||||||
|
|
||||||
void RefScanSink::operator () (const unsigned char * data, size_t len)
|
void RefScanSink::operator () (const unsigned char * data, size_t len)
|
||||||
{
|
{
|
||||||
hashSink(data, len);
|
|
||||||
|
|
||||||
/* It's possible that a reference spans the previous and current
|
/* It's possible that a reference spans the previous and current
|
||||||
fragment, so search in the concatenation of the tail of the
|
fragment, so search in the concatenation of the tail of the
|
||||||
previous fragment and the start of the current fragment. */
|
previous fragment and the start of the current fragment. */
|
||||||
|
@ -82,7 +79,9 @@ void RefScanSink::operator () (const unsigned char * data, size_t len)
|
||||||
PathSet scanForReferences(const string & path,
|
PathSet scanForReferences(const string & path,
|
||||||
const PathSet & refs, HashResult & hash)
|
const PathSet & refs, HashResult & hash)
|
||||||
{
|
{
|
||||||
RefScanSink sink;
|
RefScanSink refsSink;
|
||||||
|
HashSink hashSink { htSHA256 };
|
||||||
|
TeeSink sink { refsSink, hashSink };
|
||||||
std::map<string, Path> backMap;
|
std::map<string, Path> backMap;
|
||||||
|
|
||||||
/* For efficiency (and a higher hit rate), just search for the
|
/* For efficiency (and a higher hit rate), just search for the
|
||||||
|
@ -97,7 +96,7 @@ PathSet scanForReferences(const string & path,
|
||||||
assert(s.size() == refLength);
|
assert(s.size() == refLength);
|
||||||
assert(backMap.find(s) == backMap.end());
|
assert(backMap.find(s) == backMap.end());
|
||||||
// parseHash(htSHA256, s);
|
// parseHash(htSHA256, s);
|
||||||
sink.hashes.insert(s);
|
refsSink.hashes.insert(s);
|
||||||
backMap[s] = i;
|
backMap[s] = i;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -106,13 +105,13 @@ PathSet scanForReferences(const string & path,
|
||||||
|
|
||||||
/* Map the hashes found back to their store paths. */
|
/* Map the hashes found back to their store paths. */
|
||||||
PathSet found;
|
PathSet found;
|
||||||
for (auto & i : sink.seen) {
|
for (auto & i : refsSink.seen) {
|
||||||
std::map<string, Path>::iterator j;
|
std::map<string, Path>::iterator j;
|
||||||
if ((j = backMap.find(i)) == backMap.end()) abort();
|
if ((j = backMap.find(i)) == backMap.end()) abort();
|
||||||
found.insert(j->second);
|
found.insert(j->second);
|
||||||
}
|
}
|
||||||
|
|
||||||
hash = sink.hashSink.finish();
|
hash = hashSink.finish();
|
||||||
|
|
||||||
return found;
|
return found;
|
||||||
}
|
}
|
||||||
|
|
|
@ -963,12 +963,20 @@ ref<Store> openStore(const std::string & uri_,
|
||||||
throw Error("don't know how to open Nix store '%s'", uri);
|
throw Error("don't know how to open Nix store '%s'", uri);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static bool isNonUriPath(const std::string & spec) {
|
||||||
|
return
|
||||||
|
// is not a URL
|
||||||
|
spec.find("://") == std::string::npos
|
||||||
|
// Has at least one path separator, and so isn't a single word that
|
||||||
|
// might be special like "auto"
|
||||||
|
&& spec.find("/") != std::string::npos;
|
||||||
|
}
|
||||||
|
|
||||||
StoreType getStoreType(const std::string & uri, const std::string & stateDir)
|
StoreType getStoreType(const std::string & uri, const std::string & stateDir)
|
||||||
{
|
{
|
||||||
if (uri == "daemon") {
|
if (uri == "daemon") {
|
||||||
return tDaemon;
|
return tDaemon;
|
||||||
} else if (uri == "local" || hasPrefix(uri, "/")) {
|
} else if (uri == "local" || isNonUriPath(uri)) {
|
||||||
return tLocal;
|
return tLocal;
|
||||||
} else if (uri == "" || uri == "auto") {
|
} else if (uri == "" || uri == "auto") {
|
||||||
if (access(stateDir.c_str(), R_OK | W_OK) == 0)
|
if (access(stateDir.c_str(), R_OK | W_OK) == 0)
|
||||||
|
@ -992,8 +1000,9 @@ static RegisterStoreImplementation regStore([](
|
||||||
return std::shared_ptr<Store>(std::make_shared<UDSRemoteStore>(params));
|
return std::shared_ptr<Store>(std::make_shared<UDSRemoteStore>(params));
|
||||||
case tLocal: {
|
case tLocal: {
|
||||||
Store::Params params2 = params;
|
Store::Params params2 = params;
|
||||||
if (hasPrefix(uri, "/"))
|
if (isNonUriPath(uri)) {
|
||||||
params2["root"] = uri;
|
params2["root"] = absPath(uri);
|
||||||
|
}
|
||||||
return std::shared_ptr<Store>(std::make_shared<LocalStore>(params2));
|
return std::shared_ptr<Store>(std::make_shared<LocalStore>(params2));
|
||||||
}
|
}
|
||||||
default:
|
default:
|
||||||
|
|
|
@ -460,7 +460,7 @@ public:
|
||||||
std::optional<Hash> expectedCAHash = {});
|
std::optional<Hash> expectedCAHash = {});
|
||||||
|
|
||||||
// FIXME: remove?
|
// FIXME: remove?
|
||||||
virtual StorePath addToStoreFromDump(const string & dump, const string & name,
|
virtual StorePath addToStoreFromDump(Source & dump, const string & name,
|
||||||
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair)
|
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair)
|
||||||
{
|
{
|
||||||
throw Error("addToStoreFromDump() is not supported by this store");
|
throw Error("addToStoreFromDump() is not supported by this store");
|
||||||
|
|
|
@ -322,5 +322,18 @@ void StringSink::operator () (const unsigned char * data, size_t len)
|
||||||
s->append((const char *) data, len);
|
s->append((const char *) data, len);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
size_t ChainSource::read(unsigned char * data, size_t len)
|
||||||
|
{
|
||||||
|
if (useSecond) {
|
||||||
|
return source2.read(data, len);
|
||||||
|
} else {
|
||||||
|
try {
|
||||||
|
return source1.read(data, len);
|
||||||
|
} catch (EndOfFile &) {
|
||||||
|
useSecond = true;
|
||||||
|
return this->read(data, len);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -189,7 +189,7 @@ struct TeeSource : Source
|
||||||
size_t read(unsigned char * data, size_t len)
|
size_t read(unsigned char * data, size_t len)
|
||||||
{
|
{
|
||||||
size_t n = orig.read(data, len);
|
size_t n = orig.read(data, len);
|
||||||
sink(data, len);
|
sink(data, n);
|
||||||
return n;
|
return n;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
@ -256,6 +256,19 @@ struct LambdaSource : Source
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/* Chain two sources together so after the first is exhausted, the second is
|
||||||
|
used */
|
||||||
|
struct ChainSource : Source
|
||||||
|
{
|
||||||
|
Source & source1, & source2;
|
||||||
|
bool useSecond = false;
|
||||||
|
ChainSource(Source & s1, Source & s2)
|
||||||
|
: source1(s1), source2(s2)
|
||||||
|
{ }
|
||||||
|
|
||||||
|
size_t read(unsigned char * data, size_t len) override;
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
/* Convert a function that feeds data into a Sink into a Source. The
|
/* Convert a function that feeds data into a Sink into a Source. The
|
||||||
Source executes the function as a coroutine. */
|
Source executes the function as a coroutine. */
|
||||||
|
|
|
@ -111,6 +111,7 @@ struct CmdRegistryPin : virtual Args, EvalCommand
|
||||||
fetchers::Attrs extraAttrs;
|
fetchers::Attrs extraAttrs;
|
||||||
if (ref.subdir != "") extraAttrs["dir"] = ref.subdir;
|
if (ref.subdir != "") extraAttrs["dir"] = ref.subdir;
|
||||||
userRegistry->add(ref.input, resolved, extraAttrs);
|
userRegistry->add(ref.input, resolved, extraAttrs);
|
||||||
|
userRegistry->write(fetchers::getUserRegistryPath());
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
20
tests/local-store.sh
Normal file
20
tests/local-store.sh
Normal file
|
@ -0,0 +1,20 @@
|
||||||
|
source common.sh
|
||||||
|
|
||||||
|
cd $TEST_ROOT
|
||||||
|
|
||||||
|
echo example > example.txt
|
||||||
|
mkdir -p ./x
|
||||||
|
|
||||||
|
NIX_STORE_DIR=$TEST_ROOT/x
|
||||||
|
|
||||||
|
CORRECT_PATH=$(nix-store --store ./x --add example.txt)
|
||||||
|
|
||||||
|
PATH1=$(nix path-info --store ./x $CORRECT_PATH)
|
||||||
|
[ $CORRECT_PATH == $PATH1 ]
|
||||||
|
|
||||||
|
PATH2=$(nix path-info --store "$PWD/x" $CORRECT_PATH)
|
||||||
|
[ $CORRECT_PATH == $PATH2 ]
|
||||||
|
|
||||||
|
# FIXME we could also test the query parameter version:
|
||||||
|
# PATH3=$(nix path-info --store "local?store=$PWD/x" $CORRECT_PATH)
|
||||||
|
# [ $CORRECT_PATH == $PATH3 ]
|
|
@ -6,7 +6,7 @@ nix_tests = \
|
||||||
gc-auto.sh \
|
gc-auto.sh \
|
||||||
referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \
|
referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \
|
||||||
gc-runtime.sh check-refs.sh filter-source.sh \
|
gc-runtime.sh check-refs.sh filter-source.sh \
|
||||||
remote-store.sh export.sh export-graph.sh \
|
local-store.sh remote-store.sh export.sh export-graph.sh \
|
||||||
timeout.sh secure-drv-outputs.sh nix-channel.sh \
|
timeout.sh secure-drv-outputs.sh nix-channel.sh \
|
||||||
multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh \
|
multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh \
|
||||||
binary-cache.sh nix-profile.sh repair.sh dump-db.sh case-hack.sh \
|
binary-cache.sh nix-profile.sh repair.sh dump-db.sh case-hack.sh \
|
||||||
|
|
Loading…
Reference in a new issue