Merge remote-tracking branch 'tweag/subdir' into flakes

This commit is contained in:
Eelco Dolstra 2019-05-03 12:54:59 +02:00
commit 2aafa6901e
4 changed files with 69 additions and 27 deletions

View file

@ -204,11 +204,8 @@ static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, const
return flakeRef; return flakeRef;
} }
static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef flakeRef, bool impureIsAllowed = false) static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef fRef, bool impureIsAllowed = false)
{ {
FlakeRef fRef = lookupFlake(state, flakeRef,
impureIsAllowed ? state.getFlakeRegistries() : std::vector<std::shared_ptr<FlakeRegistry>>());
if (evalSettings.pureEval && !impureIsAllowed && !fRef.isImmutable()) if (evalSettings.pureEval && !impureIsAllowed && !fRef.isImmutable())
throw Error("requested to fetch mutable flake '%s' in pure mode", fRef); throw Error("requested to fetch mutable flake '%s' in pure mode", fRef);
@ -276,26 +273,34 @@ static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef flakeRef, bo
// This will return the flake which corresponds to a given FlakeRef. The lookupFlake is done within this function. // This will return the flake which corresponds to a given FlakeRef. The lookupFlake is done within this function.
Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowed = false) Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowed = false)
{ {
FlakeSourceInfo sourceInfo = fetchFlake(state, flakeRef, impureIsAllowed); FlakeRef resolvedRef = lookupFlake(state, flakeRef,
impureIsAllowed ? state.getFlakeRegistries() : std::vector<std::shared_ptr<FlakeRegistry>>());
FlakeSourceInfo sourceInfo = fetchFlake(state, resolvedRef, impureIsAllowed);
debug("got flake source '%s' with revision %s", debug("got flake source '%s' with revision %s",
sourceInfo.storePath, sourceInfo.rev.value_or(Hash(htSHA1)).to_string(Base16, false)); sourceInfo.storePath, sourceInfo.rev.value_or(Hash(htSHA1)).to_string(Base16, false));
resolvedRef = sourceInfo.flakeRef; // `resolvedRef` is now immutable
state.store->assertStorePath(sourceInfo.storePath); state.store->assertStorePath(sourceInfo.storePath);
if (state.allowedPaths) if (state.allowedPaths)
state.allowedPaths->insert(sourceInfo.storePath); state.allowedPaths->insert(sourceInfo.storePath);
Flake flake(flakeRef, std::move(sourceInfo)); Flake flake(resolvedRef, std::move(sourceInfo));
if (std::get_if<FlakeRef::IsGitHub>(&flakeRef.data)) { if (std::get_if<FlakeRef::IsGitHub>(&resolvedRef.data)) {
// FIXME: ehm? // FIXME: ehm?
if (flake.sourceInfo.rev) if (flake.sourceInfo.rev)
flake.ref = FlakeRef(flakeRef.baseRef().to_string() flake.ref = FlakeRef(resolvedRef.baseRef().to_string()
+ "/" + flake.sourceInfo.rev->to_string(Base16, false)); + "/" + flake.sourceInfo.rev->to_string(Base16, false));
} }
Path flakeFile = sourceInfo.storePath + "/flake.nix"; // Guard against symlink attacks.
auto flakeFile = canonPath(sourceInfo.storePath + "/" + resolvedRef.subdir + "/flake.nix");
if (!isInDir(flakeFile, sourceInfo.storePath))
throw Error("flake file '%s' escapes from '%s'", resolvedRef, sourceInfo.storePath);
if (!pathExists(flakeFile)) if (!pathExists(flakeFile))
throw Error("source tree referenced by '%s' does not contain a 'flake.nix' file", flakeRef); throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", resolvedRef, resolvedRef.subdir);
Value vInfo; Value vInfo;
state.evalFile(flakeFile, vInfo); // FIXME: symlink attack state.evalFile(flakeFile, vInfo); // FIXME: symlink attack
@ -375,7 +380,7 @@ ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef,
LockFile lockFile; LockFile lockFile;
if (isTopFlake) if (isTopFlake)
lockFile = readLockFile(flake.sourceInfo.storePath + "/flake.lock"); // FIXME: symlink attack lockFile = readLockFile(flake.sourceInfo.storePath + "/" + flake.ref.subdir + "/flake.lock"); // FIXME: symlink attack
ResolvedFlake deps(flake); ResolvedFlake deps(flake);
@ -415,16 +420,19 @@ static LockFile makeLockFile(EvalState & evalState, FlakeRef & flakeRef)
return lockFile; return lockFile;
} }
void updateLockFile(EvalState & state, const Path & path) void updateLockFile(EvalState & state, const FlakeUri & flakeUri)
{ {
// FIXME: We are writing the lockfile to the store here! Very bad practice! // FIXME: We are writing the lockfile to the store here! Very bad practice!
FlakeRef flakeRef = FlakeRef(path); FlakeRef flakeRef = FlakeRef(flakeUri);
if (auto refData = std::get_if<FlakeRef::IsPath>(&flakeRef.data)) {
auto lockFile = makeLockFile(state, flakeRef); auto lockFile = makeLockFile(state, flakeRef);
writeLockFile(lockFile, path + "/flake.lock"); writeLockFile(lockFile, refData->path + "/" + flakeRef.subdir + "/flake.lock");
// Hack: Make sure that flake.lock is visible to Git. Otherwise, // Hack: Make sure that flake.lock is visible to Git. Otherwise,
// exportGit will fail to copy it to the Nix store. // exportGit will fail to copy it to the Nix store.
runProgram("git", true, { "-C", path, "add", "flake.lock" }); runProgram("git", true, { "-C", refData->path, "add", flakeRef.subdir + "/flake.lock" });
} else
throw Error("flakeUri %s can't be updated because it is not a path", flakeUri);
} }
void callFlake(EvalState & state, const ResolvedFlake & resFlake, Value & v) void callFlake(EvalState & state, const ResolvedFlake & resFlake, Value & v)

View file

@ -90,7 +90,7 @@ struct ResolvedFlake
ResolvedFlake resolveFlake(EvalState &, const FlakeRef &, RegistryAccess registryAccess, bool isTopFlake = true); ResolvedFlake resolveFlake(EvalState &, const FlakeRef &, RegistryAccess registryAccess, bool isTopFlake = true);
void updateLockFile(EvalState &, const Path & path); void updateLockFile(EvalState &, const FlakeUri &);
void gitCloneFlake (std::string flakeUri, EvalState &, Registries, Path); void gitCloneFlake (std::string flakeUri, EvalState &, Registries, Path);
} }

View file

@ -30,7 +30,16 @@ const static std::string schemeRegex = "(?:http|https|ssh|git|file)";
const static std::string authorityRegex = "[a-zA-Z0-9._~-]*"; const static std::string authorityRegex = "[a-zA-Z0-9._~-]*";
const static std::string segmentRegex = "[a-zA-Z0-9._~-]+"; const static std::string segmentRegex = "[a-zA-Z0-9._~-]+";
const static std::string pathRegex = "/?" + segmentRegex + "(?:/" + segmentRegex + ")*"; const static std::string pathRegex = "/?" + segmentRegex + "(?:/" + segmentRegex + ")*";
const static std::string paramRegex = "[a-z]+=[a-zA-Z0-9._-]*"; // FIXME: support escaping in query string.
// Note: '/' is not a valid query parameter, but so what...
const static std::string paramRegex = "[a-z]+=[/a-zA-Z0-9._-]*";
const static std::string paramsRegex = "(?:[?](" + paramRegex + "(?:&" + paramRegex + ")*))";
// 'dir' path elements cannot start with a '.'. We also reject
// potentially dangerous characters like ';'.
const static std::string subDirElemRegex = "(?:[a-zA-Z0-9_-]+[a-zA-Z0-9._-]*)";
const static std::string subDirRegex = subDirElemRegex + "(?:/" + subDirElemRegex + ")*";
FlakeRef::FlakeRef(const std::string & uri, bool allowRelative) FlakeRef::FlakeRef(const std::string & uri, bool allowRelative)
{ {
@ -41,18 +50,21 @@ FlakeRef::FlakeRef(const std::string & uri, bool allowRelative)
std::regex::ECMAScript); std::regex::ECMAScript);
static std::regex githubRegex( static std::regex githubRegex(
"github:(" + ownerRegex + ")/(" + repoRegex + ")(?:/" + revOrRefRegex + ")?", "github:(" + ownerRegex + ")/(" + repoRegex + ")(?:/" + revOrRefRegex + ")?"
+ paramsRegex + "?",
std::regex::ECMAScript); std::regex::ECMAScript);
static std::regex uriRegex( static std::regex uriRegex(
"((" + schemeRegex + "):" + "((" + schemeRegex + "):" +
"(?://(" + authorityRegex + "))?" + "(?://(" + authorityRegex + "))?" +
"(" + pathRegex + "))" + "(" + pathRegex + "))" +
"(?:[?](" + paramRegex + "(?:&" + paramRegex + ")*))?", paramsRegex + "?",
std::regex::ECMAScript); std::regex::ECMAScript);
static std::regex refRegex2(refRegex, std::regex::ECMAScript); static std::regex refRegex2(refRegex, std::regex::ECMAScript);
static std::regex subDirRegex2(subDirRegex, std::regex::ECMAScript);
std::cmatch match; std::cmatch match;
if (std::regex_match(uri.c_str(), match, flakeRegex)) { if (std::regex_match(uri.c_str(), match, flakeRegex)) {
IsAlias d; IsAlias d;
@ -76,6 +88,18 @@ FlakeRef::FlakeRef(const std::string & uri, bool allowRelative)
else if (match[4].matched) { else if (match[4].matched) {
ref = match[4]; ref = match[4];
} }
for (auto & param : tokenizeString<Strings>(match[5], "&")) {
auto n = param.find('=');
assert(n != param.npos);
std::string name(param, 0, n);
std::string value(param, n + 1);
if (name == "dir") {
if (value != "" && !std::regex_match(value, subDirRegex2))
throw Error("flake '%s' has invalid subdirectory '%s'", uri, value);
subdir = value;
} else
throw Error("invalid Git flake reference parameter '%s', in '%s'", name, uri);
}
data = d; data = d;
} }
@ -97,6 +121,10 @@ FlakeRef::FlakeRef(const std::string & uri, bool allowRelative)
if (!std::regex_match(value, refRegex2)) if (!std::regex_match(value, refRegex2))
throw Error("invalid Git ref '%s'", value); throw Error("invalid Git ref '%s'", value);
ref = value; ref = value;
} else if (name == "dir") {
if (value != "" && !std::regex_match(value, subDirRegex2))
throw Error("flake '%s' has invalid subdirectory '%s'", uri, value);
subdir = value;
} else } else
// FIXME: should probably pass through unknown parameters // FIXME: should probably pass through unknown parameters
throw Error("invalid Git flake reference parameter '%s', in '%s'", name, uri); throw Error("invalid Git flake reference parameter '%s', in '%s'", name, uri);
@ -119,6 +147,7 @@ FlakeRef::FlakeRef(const std::string & uri, bool allowRelative)
std::string FlakeRef::to_string() const std::string FlakeRef::to_string() const
{ {
std::string string; std::string string;
if (auto refData = std::get_if<FlakeRef::IsAlias>(&data)) if (auto refData = std::get_if<FlakeRef::IsAlias>(&data))
string = refData->alias; string = refData->alias;
@ -137,8 +166,12 @@ std::string FlakeRef::to_string() const
else abort(); else abort();
// FIXME: need to use ?rev etc. for IsGit URIs.
string += (ref ? "/" + *ref : "") + string += (ref ? "/" + *ref : "") +
(rev ? "/" + rev->to_string(Base16, false) : ""); (rev ? "/" + rev->to_string(Base16, false) : "");
if (subdir != "") string += "?dir=" + subdir;
return string; return string;
} }

View file

@ -69,7 +69,7 @@ namespace nix {
https://example.org/my/repo.git https://example.org/my/repo.git
https://example.org/my/repo.git?ref=release-1.2.3 https://example.org/my/repo.git?ref=release-1.2.3
https://example.org/my/repo.git?rev=e72daba8250068216d79d2aeef40d4d95aff6666 https://example.org/my/repo.git?rev=e72daba8250068216d79d2aeef40d4d95aff6666
git://github.com/edolstra/dwarffs.git\?ref=flake\&rev=2efca4bc9da70fb001b26c3dc858c6397d3c4817 git://github.com/edolstra/dwarffs.git?ref=flake&rev=2efca4bc9da70fb001b26c3dc858c6397d3c4817
* /path.git(\?attr(&attr)*)? * /path.git(\?attr(&attr)*)?
@ -144,17 +144,18 @@ struct FlakeRef
std::optional<std::string> ref; std::optional<std::string> ref;
std::optional<Hash> rev; std::optional<Hash> rev;
Path subdir = ""; // This is a relative path pointing at the flake.nix file's directory, relative to the git root.
bool operator<(const FlakeRef & flakeRef) const bool operator<(const FlakeRef & flakeRef) const
{ {
return std::make_tuple(this->data, ref, rev) < return std::make_tuple(data, ref, rev, subdir) <
std::make_tuple(flakeRef.data, flakeRef.ref, flakeRef.rev); std::make_tuple(flakeRef.data, flakeRef.ref, flakeRef.rev, subdir);
} }
bool operator==(const FlakeRef & flakeRef) const bool operator==(const FlakeRef & flakeRef) const
{ {
return std::make_tuple(this->data, ref, rev) == return std::make_tuple(data, ref, rev, subdir) ==
std::make_tuple(flakeRef.data, flakeRef.ref, flakeRef.rev); std::make_tuple(flakeRef.data, flakeRef.ref, flakeRef.rev, flakeRef.subdir);
} }
// Parse a flake URI. // Parse a flake URI.