Merge pull request #2907 from NixOS/subdir

Subdirectory improvements
This commit is contained in:
Eelco Dolstra 2019-06-03 09:21:31 +02:00 committed by GitHub
commit 95bdfaa8bd
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 103 additions and 51 deletions

View file

@ -121,7 +121,7 @@ nlohmann::json flakeEntryToJson(const LockFile::FlakeEntry & entry)
return json; return json;
} }
void writeLockFile(const LockFile & lockFile, const Path & path) std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile)
{ {
nlohmann::json json; nlohmann::json json;
json["version"] = 1; json["version"] = 1;
@ -133,8 +133,14 @@ void writeLockFile(const LockFile & lockFile, const Path & path)
json["inputs"] = nlohmann::json::object(); json["inputs"] = nlohmann::json::object();
for (auto & x : lockFile.flakeEntries) for (auto & x : lockFile.flakeEntries)
json["inputs"][x.first.to_string()] = flakeEntryToJson(x.second); json["inputs"][x.first.to_string()] = flakeEntryToJson(x.second);
stream << json.dump(4); // '4' = indentation in json file
return stream;
}
void writeLockFile(const LockFile & lockFile, const Path & path)
{
createDirs(dirOf(path)); createDirs(dirOf(path));
writeFile(path, json.dump(4) + "\n"); // '4' = indentation in json file writeFile(path, fmt("%s\n", lockFile));
} }
Path getUserRegistryPath() Path getUserRegistryPath()
@ -478,7 +484,10 @@ ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, HandleLoc
if (!recreateLockFile(handleLockFile)) { if (!recreateLockFile(handleLockFile)) {
// If recreateLockFile, start with an empty lockfile // If recreateLockFile, start with an empty lockfile
oldLockFile = readLockFile(flake.sourceInfo.storePath + "/flake.lock"); // FIXME: symlink attack // FIXME: symlink attack
oldLockFile = readLockFile(
state.store->toRealPath(flake.sourceInfo.storePath)
+ "/" + flake.sourceInfo.resolvedRef.subdir + "/flake.lock");
} }
LockFile lockFile(oldLockFile); LockFile lockFile(oldLockFile);

View file

@ -1,4 +1,5 @@
#include "flakeref.hh" #include "flakeref.hh"
#include "store-api.hh"
#include <regex> #include <regex>
@ -30,10 +31,6 @@ const static std::string schemeRegex = "(?:http|https|ssh|git|file)";
const static std::string authorityRegex = "[a-zA-Z0-9._~-]*"; const static std::string authorityRegex = "[a-zA-Z0-9._~-]*";
const static std::string segmentRegex = "[a-zA-Z0-9._~-]+"; const static std::string segmentRegex = "[a-zA-Z0-9._~-]+";
const static std::string pathRegex = "/?" + segmentRegex + "(?:/" + segmentRegex + ")*"; const static std::string pathRegex = "/?" + segmentRegex + "(?:/" + segmentRegex + ")*";
// FIXME: support escaping in query string.
// Note: '/' is not a valid query parameter, but so what...
const static std::string paramRegex = "[a-z]+=[/a-zA-Z0-9._-]*";
const static std::string paramsRegex = "(?:[?](" + paramRegex + "(?:&" + paramRegex + ")*))";
// 'dir' path elements cannot start with a '.'. We also reject // 'dir' path elements cannot start with a '.'. We also reject
// potentially dangerous characters like ';'. // potentially dangerous characters like ';'.
@ -41,7 +38,7 @@ const static std::string subDirElemRegex = "(?:[a-zA-Z0-9_-]+[a-zA-Z0-9._-]*)";
const static std::string subDirRegex = subDirElemRegex + "(?:/" + subDirElemRegex + ")*"; const static std::string subDirRegex = subDirElemRegex + "(?:/" + subDirElemRegex + ")*";
FlakeRef::FlakeRef(const std::string & uri, bool allowRelative) FlakeRef::FlakeRef(const std::string & uri_, bool allowRelative)
{ {
// FIXME: could combine this into one regex. // FIXME: could combine this into one regex.
@ -50,21 +47,46 @@ FlakeRef::FlakeRef(const std::string & uri, bool allowRelative)
std::regex::ECMAScript); std::regex::ECMAScript);
static std::regex githubRegex( static std::regex githubRegex(
"github:(" + ownerRegex + ")/(" + repoRegex + ")(?:/" + revOrRefRegex + ")?" "github:(" + ownerRegex + ")/(" + repoRegex + ")(?:/" + revOrRefRegex + ")?",
+ paramsRegex + "?",
std::regex::ECMAScript); std::regex::ECMAScript);
static std::regex uriRegex( static std::regex uriRegex(
"((" + schemeRegex + "):" + "((" + schemeRegex + "):" +
"(?://(" + authorityRegex + "))?" + "(?://(" + authorityRegex + "))?" +
"(" + pathRegex + "))" + "(" + pathRegex + "))",
paramsRegex + "?",
std::regex::ECMAScript); std::regex::ECMAScript);
static std::regex refRegex2(refRegex, std::regex::ECMAScript); static std::regex refRegex2(refRegex, std::regex::ECMAScript);
static std::regex subDirRegex2(subDirRegex, std::regex::ECMAScript); static std::regex subDirRegex2(subDirRegex, std::regex::ECMAScript);
auto [uri, params] = splitUriAndParams(uri_);
auto handleSubdir = [&](const std::string & name, const std::string & value) {
if (name == "dir") {
if (value != "" && !std::regex_match(value, subDirRegex2))
throw BadFlakeRef("flake '%s' has invalid subdirectory '%s'", uri, value);
subdir = value;
return true;
} else
return false;
};
auto handleGitParams = [&](const std::string & name, const std::string & value) {
if (name == "rev") {
if (!std::regex_match(value, revRegex))
throw BadFlakeRef("invalid Git revision '%s'", value);
rev = Hash(value, htSHA1);
} else if (name == "ref") {
if (!std::regex_match(value, refRegex2))
throw BadFlakeRef("invalid Git ref '%s'", value);
ref = value;
} else if (handleSubdir(name, value))
;
else return false;
return true;
};
std::cmatch match; std::cmatch match;
if (std::regex_match(uri.c_str(), match, flakeRegex)) { if (std::regex_match(uri.c_str(), match, flakeRegex)) {
IsAlias d; IsAlias d;
@ -88,17 +110,11 @@ FlakeRef::FlakeRef(const std::string & uri, bool allowRelative)
else if (match[4].matched) { else if (match[4].matched) {
ref = match[4]; ref = match[4];
} }
for (auto & param : tokenizeString<Strings>(match[5], "&")) { for (auto & param : params) {
auto n = param.find('='); if (handleSubdir(param.first, param.second))
assert(n != param.npos); ;
std::string name(param, 0, n); else
std::string value(param, n + 1); throw BadFlakeRef("invalid Git flakeref parameter '%s', in '%s'", param.first, uri);
if (name == "dir") {
if (value != "" && !std::regex_match(value, subDirRegex2))
throw Error("flake '%s' has invalid subdirectory '%s'", uri, value);
subdir = value;
} else
throw Error("invalid Git flake reference parameter '%s', in '%s'", name, uri);
} }
data = d; data = d;
} }
@ -108,40 +124,44 @@ FlakeRef::FlakeRef(const std::string & uri, bool allowRelative)
{ {
IsGit d; IsGit d;
d.uri = match[1]; d.uri = match[1];
for (auto & param : tokenizeString<Strings>(match[5], "&")) { for (auto & param : params) {
auto n = param.find('='); if (handleGitParams(param.first, param.second))
assert(n != param.npos); ;
std::string name(param, 0, n); else
std::string value(param, n + 1);
if (name == "rev") {
if (!std::regex_match(value, revRegex))
throw Error("invalid Git revision '%s'", value);
rev = Hash(value, htSHA1);
} else if (name == "ref") {
if (!std::regex_match(value, refRegex2))
throw Error("invalid Git ref '%s'", value);
ref = value;
} else if (name == "dir") {
if (value != "" && !std::regex_match(value, subDirRegex2))
throw Error("flake '%s' has invalid subdirectory '%s'", uri, value);
subdir = value;
} else
// FIXME: should probably pass through unknown parameters // FIXME: should probably pass through unknown parameters
throw Error("invalid Git flake reference parameter '%s', in '%s'", name, uri); throw BadFlakeRef("invalid Git flakeref parameter '%s', in '%s'", param.first, uri);
} }
if (rev && !ref) if (rev && !ref)
throw Error("flake URI '%s' lacks a Git ref", uri); throw BadFlakeRef("flake URI '%s' lacks a Git ref", uri);
data = d; data = d;
} }
else if (hasPrefix(uri, "/") || (allowRelative && (hasPrefix(uri, "./") || hasPrefix(uri, "../") || uri == "."))) { else if ((hasPrefix(uri, "/") || (allowRelative && (hasPrefix(uri, "./") || hasPrefix(uri, "../") || uri == ".")))
&& uri.find(':') == std::string::npos)
{
IsPath d; IsPath d;
d.path = allowRelative ? absPath(uri) : canonPath(uri); if (allowRelative) {
d.path = absPath(uri);
while (true) {
if (pathExists(d.path + "/.git")) break;
subdir = baseNameOf(d.path) + (subdir.empty() ? "" : "/" + subdir);
d.path = dirOf(d.path);
if (d.path == "/")
throw BadFlakeRef("path '%s' does not reference a Git repository", uri);
}
} else
d.path = canonPath(uri);
data = d; data = d;
for (auto & param : params) {
if (handleGitParams(param.first, param.second))
;
else
throw BadFlakeRef("invalid Git flakeref parameter '%s', in '%s'", param.first, uri);
}
} }
else else
throw Error("'%s' is not a valid flake reference", uri); throw BadFlakeRef("'%s' is not a valid flake reference", uri);
} }
std::string FlakeRef::to_string() const std::string FlakeRef::to_string() const
@ -165,10 +185,10 @@ std::string FlakeRef::to_string() const
} }
else if (auto refData = std::get_if<FlakeRef::IsPath>(&data)) { else if (auto refData = std::get_if<FlakeRef::IsPath>(&data)) {
assert(subdir == ""); string = refData->path;
if (ref) addParam("ref", *ref); if (ref) addParam("ref", *ref);
if (rev) addParam("rev", rev->gitRev()); if (rev) addParam("rev", rev->gitRev());
return refData->path; if (subdir != "") addParam("dir", subdir);
} }
else if (auto refData = std::get_if<FlakeRef::IsGitHub>(&data)) { else if (auto refData = std::get_if<FlakeRef::IsGitHub>(&data)) {
@ -217,4 +237,15 @@ FlakeRef FlakeRef::baseRef() const // Removes the ref and rev from a FlakeRef.
result.rev = std::nullopt; result.rev = std::nullopt;
return result; return result;
} }
std::optional<FlakeRef> parseFlakeRef(
const std::string & uri, bool allowRelative)
{
try {
return FlakeRef(uri, allowRelative);
} catch (BadFlakeRef & e) {
return {};
}
}
} }

View file

@ -180,4 +180,9 @@ struct FlakeRef
std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef); std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef);
MakeError(BadFlakeRef, Error);
std::optional<FlakeRef> parseFlakeRef(
const std::string & uri, bool allowRelative = false);
} }

View file

@ -3170,7 +3170,7 @@ void DerivationGoal::registerOutputs()
valid. */ valid. */
delayedException = std::make_exception_ptr( delayedException = std::make_exception_ptr(
BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s",
dest, h.to_string(), h2.to_string())); dest, h.to_string(SRI), h2.to_string(SRI)));
Path actualDest = worker.store.toRealPath(dest); Path actualDest = worker.store.toRealPath(dest);

View file

@ -314,6 +314,10 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
Strings{"packages." + std::string(s, 8)})); Strings{"packages." + std::string(s, 8)}));
} }
else if (auto flakeRef = parseFlakeRef(s, true))
result.push_back(std::make_shared<InstallableFlake>(*this, std::move(*flakeRef),
getDefaultFlakeAttrPaths()));
else if ((colon = s.rfind(':')) != std::string::npos) { else if ((colon = s.rfind(':')) != std::string::npos) {
auto flakeRef = std::string(s, 0, colon); auto flakeRef = std::string(s, 0, colon);
auto attrPath = std::string(s, colon + 1); auto attrPath = std::string(s, colon + 1);

View file

@ -131,9 +131,12 @@ nix build -o $TEST_ROOT/result --flake-registry $registry flake1:foo
[[ -e $TEST_ROOT/result/hello ]] [[ -e $TEST_ROOT/result/hello ]]
# Test defaultPackage. # Test defaultPackage.
nix build -o $TEST_ROOT/result --flake-registry $registry flake1: nix build -o $TEST_ROOT/result --flake-registry $registry flake1
[[ -e $TEST_ROOT/result/hello ]] [[ -e $TEST_ROOT/result/hello ]]
nix build -o $TEST_ROOT/result --flake-registry $registry $flake1Dir
nix build -o $TEST_ROOT/result --flake-registry $registry file://$flake1Dir
# Building a flake with an unlocked dependency should fail in pure mode. # Building a flake with an unlocked dependency should fail in pure mode.
(! nix eval "(builtins.getFlake "$flake2Dir")") (! nix eval "(builtins.getFlake "$flake2Dir")")