puck/lix
0
0
Fork 0
forked from lix-project/lix
Code Pull requests Activity

Merge remote-tracking branch 'upstream/master' into better-ca-parse-errors

Browse source
This commit is contained in:
John Ericson 2020-07-23 14:56:35 +00:00
parent cdb3f39b64 4bfba1305e
commit 66a2067288
9 changed files with 83 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
README.md
View file

@ -20,7 +20,7 @@ Information on additional installation methods is available on the [Nix download
## Building And Developing
See our [Hacking guide](hydra.nixos.org/job/nix/master/build.x86_64-linux/latest/download-by-type/doc/manual#chap-hacking) in our manual for instruction on how to
See our [Hacking guide](https://hydra.nixos.org/job/nix/master/build.x86_64-linux/latest/download-by-type/doc/manual#chap-hacking) in our manual for instruction on how to
build nix from source with nix-build or how to get a development environment.
## Additional Resources

39
src/libstore/filetransfer.cc
View file

@ -124,7 +124,7 @@ struct curlFileTransfer : public FileTransfer
if (requestHeaders) curl_slist_free_all(requestHeaders);
try {
if (!done)
fail(FileTransferError(Interrupted, "download of '%s' was interrupted", request.uri));
fail(FileTransferError(Interrupted, nullptr, "download of '%s' was interrupted", request.uri));
} catch (...) {
ignoreException();
}
@ -145,6 +145,7 @@ struct curlFileTransfer : public FileTransfer
LambdaSink finalSink;
std::shared_ptr<CompressionSink> decompressionSink;
std::optional<StringSink> errorSink;
std::exception_ptr writeException;
@ -154,9 +155,19 @@ struct curlFileTransfer : public FileTransfer
size_t realSize = size * nmemb;
result.bodySize += realSize;
if (!decompressionSink)
if (!decompressionSink) {
decompressionSink = makeDecompressionSink(encoding, finalSink);
if (! successfulStatuses.count(getHTTPStatus())) {
// In this case we want to construct a TeeSink, to keep
// the response around (which we figure won't be big
// like an actual download should be) to improve error
// messages.
errorSink = StringSink { };
}
}
if (errorSink)
(*errorSink)((unsigned char *) contents, realSize);
(*decompressionSink)((unsigned char *) contents, realSize);
return realSize;
@ -412,16 +423,21 @@ struct curlFileTransfer : public FileTransfer
attempt++;
std::shared_ptr<std::string> response;
if (errorSink)
response = errorSink->s;
auto exc =
code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted
? FileTransferError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri))
? FileTransferError(Interrupted, response, "%s of '%s' was interrupted", request.verb(), request.uri)
: httpStatus != 0
? FileTransferError(err,
response,
fmt("unable to %s '%s': HTTP error %d ('%s')",
request.verb(), request.uri, httpStatus, statusMsg)
+ (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code)))
)
: FileTransferError(err,
response,
fmt("unable to %s '%s': %s (%d)",
request.verb(), request.uri, curl_easy_strerror(code), code));
@ -679,7 +695,7 @@ struct curlFileTransfer : public FileTransfer
auto s3Res = s3Helper.getObject(bucketName, key);
FileTransferResult res;
if (!s3Res.data)
throw FileTransferError(NotFound, fmt("S3 object '%s' does not exist", request.uri));
throw FileTransferError(NotFound, nullptr, "S3 object '%s' does not exist", request.uri);
res.data = s3Res.data;
callback(std::move(res));
#else
@ -824,6 +840,21 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink)
}
}
template<typename... Args>
FileTransferError::FileTransferError(FileTransfer::Error error, std::shared_ptr<string> response, const Args & ... args)
: Error(args...), error(error), response(response)
{
const auto hf = hintfmt(args...);
// FIXME: Due to https://github.com/NixOS/nix/issues/3841 we don't know how
// to print different messages for different verbosity levels. For now
// we add some heuristics for detecting when we want to show the response.
if (response && (response->size() < 1024 || response->find("<html>") != string::npos)) {
err.hint = hintfmt("%1%\n\nresponse body:\n\n%2%", normaltxt(hf.str()), *response);
} else {
err.hint = hf;
}
}
bool isUri(const string & s)
{
if (s.compare(0, 8, "channel:") == 0) return true;

8
src/libstore/filetransfer.hh
View file

@ -103,10 +103,12 @@ class FileTransferError : public Error
{
public:
FileTransfer::Error error;
std::shared_ptr<string> response; // intentionally optional
template<typename... Args>
FileTransferError(FileTransfer::Error error, const Args & ... args)
: Error(args...), error(error)
{ }
FileTransferError(FileTransfer::Error error, std::shared_ptr<string> response, const Args & ... args);
virtual const char* sname() const override { return "FileTransferError"; }
};
bool isUri(const string & s);

2
src/libstore/local-store.cc
View file

@ -1069,7 +1069,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name,
or the original source is empty */
while (dump.size() < settings.narBufferSize) {
auto oldSize = dump.size();
constexpr size_t chunkSize = 1024;
constexpr size_t chunkSize = 65536;
auto want = std::min(chunkSize, settings.narBufferSize - oldSize);
dump.resize(oldSize + want);
auto got = 0;

15
src/libstore/references.cc
View file

@ -48,13 +48,12 @@ static void search(const unsigned char * s, size_t len,
struct RefScanSink : Sink
{
HashSink hashSink;
StringSet hashes;
StringSet seen;
string tail;
RefScanSink() : hashSink(htSHA256) { }
RefScanSink() { }
void operator () (const unsigned char * data, size_t len);
};
@ -62,8 +61,6 @@ struct RefScanSink : Sink
void RefScanSink::operator () (const unsigned char * data, size_t len)
{
hashSink(data, len);
/* It's possible that a reference spans the previous and current
fragment, so search in the concatenation of the tail of the
previous fragment and the start of the current fragment. */
@ -82,7 +79,9 @@ void RefScanSink::operator () (const unsigned char * data, size_t len)
std::pair<PathSet, HashResult> scanForReferences(const string & path,
const PathSet & refs)
{
RefScanSink sink;
RefScanSink refsSink;
HashSink hashSink { htSHA256 };
TeeSink sink { refsSink, hashSink };
std::map<string, Path> backMap;
/* For efficiency (and a higher hit rate), just search for the
@ -97,7 +96,7 @@ std::pair<PathSet, HashResult> scanForReferences(const string & path,
assert(s.size() == refLength);
assert(backMap.find(s) == backMap.end());
// parseHash(htSHA256, s);
sink.hashes.insert(s);
refsSink.hashes.insert(s);
backMap[s] = i;
}
@ -106,13 +105,13 @@ std::pair<PathSet, HashResult> scanForReferences(const string & path,
/* Map the hashes found back to their store paths. */
PathSet found;
for (auto & i : sink.seen) {
for (auto & i : refsSink.seen) {
std::map<string, Path>::iterator j;
if ((j = backMap.find(i)) == backMap.end()) abort();
found.insert(j->second);
}
auto hash = sink.hashSink.finish();
auto hash = hashSink.finish();
return std::pair<PathSet, HashResult>(found, hash);
}

15
src/libstore/store-api.cc
View file

@ -967,12 +967,20 @@ ref<Store> openStore(const std::string & uri_,
throw Error("don't know how to open Nix store '%s'", uri);
}
static bool isNonUriPath(const std::string & spec) {
return
// is not a URL
spec.find("://") == std::string::npos
// Has at least one path separator, and so isn't a single word that
// might be special like "auto"
&& spec.find("/") != std::string::npos;
}
StoreType getStoreType(const std::string & uri, const std::string & stateDir)
{
if (uri == "daemon") {
return tDaemon;
} else if (uri == "local" || hasPrefix(uri, "/")) {
} else if (uri == "local" || isNonUriPath(uri)) {
return tLocal;
} else if (uri == "" || uri == "auto") {
if (access(stateDir.c_str(), R_OK | W_OK) == 0)
@ -996,8 +1004,9 @@ static RegisterStoreImplementation regStore([](
return std::shared_ptr<Store>(std::make_shared<UDSRemoteStore>(params));
case tLocal: {
Store::Params params2 = params;
if (hasPrefix(uri, "/"))
params2["root"] = uri;
if (isNonUriPath(uri)) {
params2["root"] = absPath(uri);
}
return std::shared_ptr<Store>(std::make_shared<LocalStore>(params2));
}
default:

1
src/nix/registry.cc
View file

@ -111,6 +111,7 @@ struct CmdRegistryPin : virtual Args, EvalCommand
fetchers::Attrs extraAttrs;
if (ref.subdir != "") extraAttrs["dir"] = ref.subdir;
userRegistry->add(ref.input, resolved, extraAttrs);
userRegistry->write(fetchers::getUserRegistryPath());
}
};

20
tests/local-store.sh Normal file
View file

@ -0,0 +1,20 @@
source common.sh
cd $TEST_ROOT
echo example > example.txt
mkdir -p ./x
NIX_STORE_DIR=$TEST_ROOT/x
CORRECT_PATH=$(nix-store --store ./x --add example.txt)
PATH1=$(nix path-info --store ./x $CORRECT_PATH)
[ $CORRECT_PATH == $PATH1 ]
PATH2=$(nix path-info --store "$PWD/x" $CORRECT_PATH)
[ $CORRECT_PATH == $PATH2 ]
# FIXME we could also test the query parameter version:
# PATH3=$(nix path-info --store "local?store=$PWD/x" $CORRECT_PATH)
# [ $CORRECT_PATH == $PATH3 ]

2
tests/local.mk
View file

@ -6,7 +6,7 @@ nix_tests = \
gc-auto.sh \
referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \
gc-runtime.sh check-refs.sh filter-source.sh \
remote-store.sh export.sh export-graph.sh \
local-store.sh remote-store.sh export.sh export-graph.sh \
timeout.sh secure-drv-outputs.sh nix-channel.sh \
multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh \
binary-cache.sh nix-profile.sh repair.sh dump-db.sh case-hack.sh \