Merge remote-tracking branch 'upstream/master' into no-stringly-typed-derivation-output

This commit is contained in:
John Ericson 2020-06-04 21:04:35 +00:00
commit e5cc1ebc5d
20 changed files with 296 additions and 44 deletions

View file

@ -282,7 +282,10 @@ struct GitInput : Input
// FIXME: git stderr messes up our progress indicator, so // FIXME: git stderr messes up our progress indicator, so
// we're using --quiet for now. Should process its stderr. // we're using --quiet for now. Should process its stderr.
try { try {
runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", *input->ref, *input->ref) }); auto fetchRef = input->ref->compare(0, 5, "refs/") == 0
? *input->ref
: "refs/heads/" + *input->ref;
runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) });
} catch (Error & e) { } catch (Error & e) {
if (!pathExists(localRefFile)) throw; if (!pathExists(localRefFile)) throw;
warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl); warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl);
@ -418,7 +421,7 @@ struct GitInputScheme : InputScheme
auto input = std::make_unique<GitInput>(parseURL(getStrAttr(attrs, "url"))); auto input = std::make_unique<GitInput>(parseURL(getStrAttr(attrs, "url")));
if (auto ref = maybeGetStrAttr(attrs, "ref")) { if (auto ref = maybeGetStrAttr(attrs, "ref")) {
if (!std::regex_match(*ref, refRegex)) if (std::regex_search(*ref, badGitRefRegex))
throw BadURL("invalid Git branch/tag name '%s'", *ref); throw BadURL("invalid Git branch/tag name '%s'", *ref);
input->ref = *ref; input->ref = *ref;
} }

View file

@ -71,7 +71,8 @@ DownloadFileResult downloadFile(
info.narHash = hashString(htSHA256, *sink.s); info.narHash = hashString(htSHA256, *sink.s);
info.narSize = sink.s->size(); info.narSize = sink.s->size();
info.ca = makeFixedOutputCA(FileIngestionMethod::Flat, hash); info.ca = makeFixedOutputCA(FileIngestionMethod::Flat, hash);
store->addToStore(info, sink.s, NoRepair, NoCheckSigs); auto source = StringSource { *sink.s };
store->addToStore(info, source, NoRepair, NoCheckSigs);
storePath = std::move(info.path); storePath = std::move(info.path);
} }

View file

@ -113,9 +113,12 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo)); diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo));
} }
void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::string> & nar, void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
{ {
// FIXME: See if we can use the original source to reduce memory usage.
auto nar = make_ref<std::string>(narSource.drain());
if (!repair && isValidPath(info.path)) return; if (!repair && isValidPath(info.path)) return;
/* Verify that all references are valid. This may do some .narinfo /* Verify that all references are valid. This may do some .narinfo
@ -347,7 +350,8 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
ValidPathInfo info(makeFixedOutputPath(method, h, name)); ValidPathInfo info(makeFixedOutputPath(method, h, name));
addToStore(info, sink.s, repair, CheckSigs, nullptr); auto source = StringSource { *sink.s };
addToStore(info, source, repair, CheckSigs, nullptr);
return std::move(info.path); return std::move(info.path);
} }
@ -361,7 +365,8 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
if (repair || !isValidPath(info.path)) { if (repair || !isValidPath(info.path)) {
StringSink sink; StringSink sink;
dumpString(s, sink); dumpString(s, sink);
addToStore(info, sink.s, repair, CheckSigs, nullptr); auto source = StringSource { *sink.s };
addToStore(info, source, repair, CheckSigs, nullptr);
} }
return std::move(info.path); return std::move(info.path);

View file

@ -74,7 +74,7 @@ public:
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
{ unsupported("queryPathFromHashPart"); } { unsupported("queryPathFromHashPart"); }
void addToStore(const ValidPathInfo & info, const ref<std::string> & nar, void addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair, CheckSigsFlag checkSigs, RepairFlag repair, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor) override; std::shared_ptr<FSAccessor> accessor) override;

View file

@ -1,3 +1,4 @@
#include "serialise.hh"
#include "store-api.hh" #include "store-api.hh"
#include "archive.hh" #include "archive.hh"
#include "worker-protocol.hh" #include "worker-protocol.hh"
@ -100,7 +101,9 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> acces
if (readInt(source) == 1) if (readInt(source) == 1)
readString(source); readString(source);
addToStore(info, tee.source.data, NoRepair, checkSigs, accessor); // Can't use underlying source, which would have been exhausted
auto source = StringSource { *tee.source.data };
addToStore(info, source, NoRepair, checkSigs, accessor);
res.push_back(info.path.clone()); res.push_back(info.path.clone());
} }

View file

@ -833,21 +833,6 @@ std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash)
} }
void Store::addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor)
{
addToStore(info, make_ref<std::string>(narSource.drain()), repair, checkSigs, accessor);
}
void Store::addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
RepairFlag repair, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor)
{
StringSource source(*nar);
addToStore(info, source, repair, checkSigs, accessor);
}
} }

View file

@ -451,12 +451,7 @@ public:
/* Import a path into the store. */ /* Import a path into the store. */
virtual void addToStore(const ValidPathInfo & info, Source & narSource, virtual void addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs,
std::shared_ptr<FSAccessor> accessor = 0); std::shared_ptr<FSAccessor> accessor = 0) = 0;
// FIXME: remove
virtual void addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs,
std::shared_ptr<FSAccessor> accessor = 0);
/* Copy the contents of a path to the store and register the /* Copy the contents of a path to the store and register the
validity the resulting path. The resulting path is returned. validity the resulting path. The resulting path is returned.

View file

@ -217,10 +217,15 @@ MultiCommand::MultiCommand(const Commands & commands)
{ {
expectedArgs.push_back(ExpectedArg{"command", 1, true, [=](std::vector<std::string> ss) { expectedArgs.push_back(ExpectedArg{"command", 1, true, [=](std::vector<std::string> ss) {
assert(!command); assert(!command);
auto i = commands.find(ss[0]); auto cmd = ss[0];
if (auto alias = get(deprecatedAliases, cmd)) {
warn("'%s' is a deprecated alias for '%s'", cmd, *alias);
cmd = *alias;
}
auto i = commands.find(cmd);
if (i == commands.end()) if (i == commands.end())
throw UsageError("'%s' is not a recognised command", ss[0]); throw UsageError("'%s' is not a recognised command", cmd);
command = {ss[0], i->second()}; command = {cmd, i->second()};
}}); }});
categories[Command::catDefault] = "Available commands"; categories[Command::catDefault] = "Available commands";

View file

@ -234,6 +234,8 @@ public:
std::map<Command::Category, std::string> categories; std::map<Command::Category, std::string> categories;
std::map<std::string, std::string> deprecatedAliases;
// Selected command, if any. // Selected command, if any.
std::optional<std::pair<std::string, ref<Command>>> command; std::optional<std::pair<std::string, ref<Command>>> command;

View file

@ -1,5 +1,6 @@
#pragma once #pragma once
#include <cassert>
#include <map> #include <map>
#include <list> #include <list>
#include <optional> #include <optional>

View file

@ -0,0 +1,130 @@
#include "lru-cache.hh"
#include <gtest/gtest.h>
namespace nix {
/* ----------------------------------------------------------------------------
* size
* --------------------------------------------------------------------------*/
TEST(LRUCache, sizeOfEmptyCacheIsZero) {
LRUCache<std::string, std::string> c(10);
ASSERT_EQ(c.size(), 0);
}
TEST(LRUCache, sizeOfSingleElementCacheIsOne) {
LRUCache<std::string, std::string> c(10);
c.upsert("foo", "bar");
ASSERT_EQ(c.size(), 1);
}
/* ----------------------------------------------------------------------------
* upsert / get
* --------------------------------------------------------------------------*/
TEST(LRUCache, getFromEmptyCache) {
LRUCache<std::string, std::string> c(10);
auto val = c.get("x");
ASSERT_EQ(val.has_value(), false);
}
TEST(LRUCache, getExistingValue) {
LRUCache<std::string, std::string> c(10);
c.upsert("foo", "bar");
auto val = c.get("foo");
ASSERT_EQ(val, "bar");
}
TEST(LRUCache, getNonExistingValueFromNonEmptyCache) {
LRUCache<std::string, std::string> c(10);
c.upsert("foo", "bar");
auto val = c.get("another");
ASSERT_EQ(val.has_value(), false);
}
TEST(LRUCache, upsertOnZeroCapacityCache) {
LRUCache<std::string, std::string> c(0);
c.upsert("foo", "bar");
auto val = c.get("foo");
ASSERT_EQ(val.has_value(), false);
}
TEST(LRUCache, updateExistingValue) {
LRUCache<std::string, std::string> c(1);
c.upsert("foo", "bar");
auto val = c.get("foo");
ASSERT_EQ(val.value_or("error"), "bar");
ASSERT_EQ(c.size(), 1);
c.upsert("foo", "changed");
val = c.get("foo");
ASSERT_EQ(val.value_or("error"), "changed");
ASSERT_EQ(c.size(), 1);
}
TEST(LRUCache, overwriteOldestWhenCapacityIsReached) {
LRUCache<std::string, std::string> c(3);
c.upsert("one", "eins");
c.upsert("two", "zwei");
c.upsert("three", "drei");
ASSERT_EQ(c.size(), 3);
ASSERT_EQ(c.get("one").value_or("error"), "eins");
// exceed capacity
c.upsert("another", "whatever");
ASSERT_EQ(c.size(), 3);
// Retrieving "one" makes it the most recent element thus
// two will be the oldest one and thus replaced.
ASSERT_EQ(c.get("two").has_value(), false);
ASSERT_EQ(c.get("another").value(), "whatever");
}
/* ----------------------------------------------------------------------------
* clear
* --------------------------------------------------------------------------*/
TEST(LRUCache, clearEmptyCache) {
LRUCache<std::string, std::string> c(10);
c.clear();
ASSERT_EQ(c.size(), 0);
}
TEST(LRUCache, clearNonEmptyCache) {
LRUCache<std::string, std::string> c(10);
c.upsert("one", "eins");
c.upsert("two", "zwei");
c.upsert("three", "drei");
ASSERT_EQ(c.size(), 3);
c.clear();
ASSERT_EQ(c.size(), 0);
}
/* ----------------------------------------------------------------------------
* erase
* --------------------------------------------------------------------------*/
TEST(LRUCache, eraseFromEmptyCache) {
LRUCache<std::string, std::string> c(10);
ASSERT_EQ(c.erase("foo"), false);
ASSERT_EQ(c.size(), 0);
}
TEST(LRUCache, eraseMissingFromNonEmptyCache) {
LRUCache<std::string, std::string> c(10);
c.upsert("one", "eins");
ASSERT_EQ(c.erase("foo"), false);
ASSERT_EQ(c.size(), 1);
ASSERT_EQ(c.get("one").value_or("error"), "eins");
}
TEST(LRUCache, eraseFromNonEmptyCache) {
LRUCache<std::string, std::string> c(10);
c.upsert("one", "eins");
ASSERT_EQ(c.erase("one"), true);
ASSERT_EQ(c.size(), 0);
ASSERT_EQ(c.get("one").value_or("empty"), "empty");
}
}

View file

@ -4,6 +4,7 @@
namespace nix { namespace nix {
std::regex refRegex(refRegexS, std::regex::ECMAScript); std::regex refRegex(refRegexS, std::regex::ECMAScript);
std::regex badGitRefRegex(badGitRefRegexS, std::regex::ECMAScript);
std::regex revRegex(revRegexS, std::regex::ECMAScript); std::regex revRegex(revRegexS, std::regex::ECMAScript);
std::regex flakeIdRegex(flakeIdRegexS, std::regex::ECMAScript); std::regex flakeIdRegex(flakeIdRegexS, std::regex::ECMAScript);

View file

@ -49,6 +49,12 @@ const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRege
const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.-]*"; // FIXME: check const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.-]*"; // FIXME: check
extern std::regex refRegex; extern std::regex refRegex;
// Instead of defining what a good Git Ref is, we define what a bad Git Ref is
// This is because of the definition of a ref in refs.c in https://github.com/git/git
// See tests/fetchGitRefs.sh for the full definition
const static std::string badGitRefRegexS = "//|^[./]|/\\.|\\.\\.|[[:cntrl:][:space:]:?^~\[]|\\\\|\\*|\\.lock$|\\.lock/|@\\{|[/.]$|^@$|^$";
extern std::regex badGitRefRegex;
// A Git revision (a SHA-1 commit hash). // A Git revision (a SHA-1 commit hash).
const static std::string revRegexS = "[0-9a-fA-F]{40}"; const static std::string revRegexS = "[0-9a-fA-F]{40}";
extern std::regex revRegex; extern std::regex revRegex;

View file

@ -459,8 +459,7 @@ string base64Encode(const string & s);
string base64Decode(const string & s); string base64Decode(const string & s);
/* Get a value for the specified key from an associate container, or a /* Get a value for the specified key from an associate container. */
default value if the key doesn't exist. */
template <class T> template <class T>
std::optional<typename T::mapped_type> get(const T & map, const typename T::key_type & key) std::optional<typename T::mapped_type> get(const T & map, const typename T::key_type & key)
{ {

View file

@ -50,8 +50,10 @@ struct CmdAddToStore : MixDryRun, StoreCommand
info.narSize = sink.s->size(); info.narSize = sink.s->size();
info.ca = makeFixedOutputCA(FileIngestionMethod::Recursive, info.narHash); info.ca = makeFixedOutputCA(FileIngestionMethod::Recursive, info.narHash);
if (!dryRun) if (!dryRun) {
store->addToStore(info, sink.s); auto source = StringSource { *sink.s };
store->addToStore(info, source);
}
logger->stdout("%s", store->printStorePath(info.path)); logger->stdout("%s", store->printStorePath(info.path));
} }

View file

@ -110,7 +110,7 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
auto builder = baseNameOf(drv.builder); auto builder = baseNameOf(drv.builder);
if (builder != "bash") if (builder != "bash")
throw Error("'nix dev-shell' only works on derivations that use 'bash' as their builder"); throw Error("'nix develop' only works on derivations that use 'bash' as their builder");
auto getEnvShPath = store->addTextToStore("get-env.sh", getEnvSh, {}); auto getEnvShPath = store->addTextToStore("get-env.sh", getEnvSh, {});
@ -233,11 +233,11 @@ struct Common : InstallableCommand, MixProfile
} }
}; };
struct CmdDevShell : Common, MixEnvironment struct CmdDevelop : Common, MixEnvironment
{ {
std::vector<std::string> command; std::vector<std::string> command;
CmdDevShell() CmdDevelop()
{ {
addFlag({ addFlag({
.longName = "command", .longName = "command",
@ -261,15 +261,15 @@ struct CmdDevShell : Common, MixEnvironment
return { return {
Example{ Example{
"To get the build environment of GNU hello:", "To get the build environment of GNU hello:",
"nix dev-shell nixpkgs.hello" "nix develop nixpkgs.hello"
}, },
Example{ Example{
"To store the build environment in a profile:", "To store the build environment in a profile:",
"nix dev-shell --profile /tmp/my-shell nixpkgs.hello" "nix develop --profile /tmp/my-shell nixpkgs.hello"
}, },
Example{ Example{
"To use a build environment previously recorded in a profile:", "To use a build environment previously recorded in a profile:",
"nix dev-shell /tmp/my-shell" "nix develop /tmp/my-shell"
}, },
}; };
} }
@ -343,4 +343,4 @@ struct CmdPrintDevEnv : Common
}; };
static auto r1 = registerCommand<CmdPrintDevEnv>("print-dev-env"); static auto r1 = registerCommand<CmdPrintDevEnv>("print-dev-env");
static auto r2 = registerCommand<CmdDevShell>("dev-shell"); static auto r2 = registerCommand<CmdDevelop>("develop");

View file

@ -28,4 +28,4 @@ $(eval $(call install-symlink, $(bindir)/nix, $(libexecdir)/nix/build-remote))
src/nix-env/user-env.cc: src/nix-env/buildenv.nix.gen.hh src/nix-env/user-env.cc: src/nix-env/buildenv.nix.gen.hh
src/nix/dev-shell.cc: src/nix/get-env.sh.gen.hh src/nix/develop.cc: src/nix/get-env.sh.gen.hh

View file

@ -110,6 +110,8 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
.description = "consider all previously downloaded files out-of-date", .description = "consider all previously downloaded files out-of-date",
.handler = {[&]() { refresh = true; }}, .handler = {[&]() { refresh = true; }},
}); });
deprecatedAliases.insert({"dev-shell", "develop"});
} }
void printFlags(std::ostream & out) override void printFlags(std::ostream & out) override

111
tests/fetchGitRefs.sh Normal file
View file

@ -0,0 +1,111 @@
source common.sh
if [[ -z $(type -p git) ]]; then
echo "Git not installed; skipping Git tests"
exit 99
fi
clearStore
repo="$TEST_ROOT/git"
rm -rf "$repo" "${repo}-tmp" "$TEST_HOME/.cache/nix"
git init "$repo"
git -C "$repo" config user.email "foobar@example.com"
git -C "$repo" config user.name "Foobar"
echo utrecht > "$repo"/hello
git -C "$repo" add hello
git -C "$repo" commit -m 'Bla1'
path=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = \"master\"; }).outPath")
# Test various combinations of ref names
# (taken from the git project)
# git help check-ref-format
# Git imposes the following rules on how references are named:
#
# 1. They can include slash / for hierarchical (directory) grouping, but no slash-separated component can begin with a dot . or end with the sequence .lock.
# 2. They must contain at least one /. This enforces the presence of a category like heads/, tags/ etc. but the actual names are not restricted. If the --allow-onelevel option is used, this rule is waived.
# 3. They cannot have two consecutive dots .. anywhere.
# 4. They cannot have ASCII control characters (i.e. bytes whose values are lower than \040, or \177 DEL), space, tilde ~, caret ^, or colon : anywhere.
# 5. They cannot have question-mark ?, asterisk *, or open bracket [ anywhere. See the --refspec-pattern option below for an exception to this rule.
# 6. They cannot begin or end with a slash / or contain multiple consecutive slashes (see the --normalize option below for an exception to this rule)
# 7. They cannot end with a dot ..
# 8. They cannot contain a sequence @{.
# 9. They cannot be the single character @.
# 10. They cannot contain a \.
valid_ref() {
{ set +x; printf >&2 '\n>>>>>>>>>> valid_ref %s\b <<<<<<<<<<\n' $(printf %s "$1" | sed -n -e l); set -x; }
git check-ref-format --branch "$1" >/dev/null
git -C "$repo" branch "$1" master >/dev/null
path1=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = ''$1''; }).outPath")
[[ $path1 = $path ]]
git -C "$repo" branch -D "$1" >/dev/null
}
invalid_ref() {
{ set +x; printf >&2 '\n>>>>>>>>>> invalid_ref %s\b <<<<<<<<<<\n' $(printf %s "$1" | sed -n -e l); set -x; }
# special case for a sole @:
# --branch @ will try to interpret @ as a branch reference and not fail. Thus we need --allow-onelevel
if [ "$1" = "@" ]; then
(! git check-ref-format --allow-onelevel "$1" >/dev/null 2>&1)
else
(! git check-ref-format --branch "$1" >/dev/null 2>&1)
fi
nix --debug eval --raw "(builtins.fetchGit { url = $repo; ref = ''$1''; }).outPath" 2>&1 | grep 'error: invalid Git branch/tag name' >/dev/null
}
valid_ref 'foox'
valid_ref '1337'
valid_ref 'foo.baz'
valid_ref 'foo/bar/baz'
valid_ref 'foo./bar'
valid_ref 'heads/foo@bar'
valid_ref "$(printf 'heads/fu\303\237')"
valid_ref 'foo-bar-baz'
valid_ref '$1'
valid_ref 'foo.locke'
invalid_ref 'refs///heads/foo'
invalid_ref 'heads/foo/'
invalid_ref '///heads/foo'
invalid_ref '.foo'
invalid_ref './foo'
invalid_ref './foo/bar'
invalid_ref 'foo/./bar'
invalid_ref 'foo/bar/.'
invalid_ref 'foo bar'
invalid_ref 'foo?bar'
invalid_ref 'foo^bar'
invalid_ref 'foo~bar'
invalid_ref 'foo:bar'
invalid_ref 'foo[bar'
invalid_ref 'foo/bar/.'
invalid_ref '.refs/foo'
invalid_ref 'refs/heads/foo.'
invalid_ref 'heads/foo..bar'
invalid_ref 'heads/foo?bar'
invalid_ref 'heads/foo.lock'
invalid_ref 'heads///foo.lock'
invalid_ref 'foo.lock/bar'
invalid_ref 'foo.lock///bar'
invalid_ref 'heads/v@{ation'
invalid_ref 'heads/foo\.ar' # should fail due to \
invalid_ref 'heads/foo\bar' # should fail due to \
invalid_ref "$(printf 'heads/foo\t')" # should fail because it has a TAB
invalid_ref "$(printf 'heads/foo\177')"
invalid_ref '@'
invalid_ref 'foo/*'
invalid_ref '*/foo'
invalid_ref 'foo/*/bar'
invalid_ref '*'
invalid_ref 'foo/*/*'
invalid_ref '*/foo/*'
invalid_ref '/foo'
invalid_ref ''

View file

@ -18,6 +18,7 @@ nix_tests = \
nar-access.sh \ nar-access.sh \
structured-attrs.sh \ structured-attrs.sh \
fetchGit.sh \ fetchGit.sh \
fetchGitRefs.sh \
fetchGitSubmodules.sh \ fetchGitSubmodules.sh \
fetchMercurial.sh \ fetchMercurial.sh \
signing.sh \ signing.sh \