Merge remote-tracking branch 'tweag/subdir' into flakes

src/libexpr/primops/flake.cc

@@ -204,11 +204,8 @@ static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, const
     return flakeRef;
 }
 
-static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef flakeRef, bool impureIsAllowed = false)
+static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef fRef, bool impureIsAllowed = false)
 {
-    FlakeRef fRef = lookupFlake(state, flakeRef,
-        impureIsAllowed ? state.getFlakeRegistries() : std::vector<std::shared_ptr<FlakeRegistry>>());
-
     if (evalSettings.pureEval && !impureIsAllowed && !fRef.isImmutable())
         throw Error("requested to fetch mutable flake '%s' in pure mode", fRef);
 
@@ -276,26 +273,34 @@ static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef flakeRef, bo
 // This will return the flake which corresponds to a given FlakeRef. The lookupFlake is done within this function.
 Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowed = false)
 {
-    FlakeSourceInfo sourceInfo = fetchFlake(state, flakeRef, impureIsAllowed);
+    FlakeRef resolvedRef = lookupFlake(state, flakeRef,
+        impureIsAllowed ? state.getFlakeRegistries() : std::vector<std::shared_ptr<FlakeRegistry>>());
+
+    FlakeSourceInfo sourceInfo = fetchFlake(state, resolvedRef, impureIsAllowed);
     debug("got flake source '%s' with revision %s",
         sourceInfo.storePath, sourceInfo.rev.value_or(Hash(htSHA1)).to_string(Base16, false));
 
+    resolvedRef = sourceInfo.flakeRef; // `resolvedRef` is now immutable
+
     state.store->assertStorePath(sourceInfo.storePath);
 
     if (state.allowedPaths)
         state.allowedPaths->insert(sourceInfo.storePath);
 
-    Flake flake(flakeRef, std::move(sourceInfo));
+    Flake flake(resolvedRef, std::move(sourceInfo));
-    if (std::get_if<FlakeRef::IsGitHub>(&flakeRef.data)) {
+    if (std::get_if<FlakeRef::IsGitHub>(&resolvedRef.data)) {
         // FIXME: ehm?
         if (flake.sourceInfo.rev)
-            flake.ref = FlakeRef(flakeRef.baseRef().to_string()
+            flake.ref = FlakeRef(resolvedRef.baseRef().to_string()
                 + "/" + flake.sourceInfo.rev->to_string(Base16, false));
     }
 
-    Path flakeFile = sourceInfo.storePath + "/flake.nix";
+    // Guard against symlink attacks.
+    auto flakeFile = canonPath(sourceInfo.storePath + "/" + resolvedRef.subdir + "/flake.nix");
+    if (!isInDir(flakeFile, sourceInfo.storePath))
+        throw Error("flake file '%s' escapes from '%s'", resolvedRef, sourceInfo.storePath);
     if (!pathExists(flakeFile))
-        throw Error("source tree referenced by '%s' does not contain a 'flake.nix' file", flakeRef);
+        throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", resolvedRef, resolvedRef.subdir);
 
     Value vInfo;
     state.evalFile(flakeFile, vInfo); // FIXME: symlink attack
@@ -375,7 +380,7 @@ ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef,
     LockFile lockFile;
 
     if (isTopFlake)
-        lockFile = readLockFile(flake.sourceInfo.storePath + "/flake.lock"); // FIXME: symlink attack
+        lockFile = readLockFile(flake.sourceInfo.storePath + "/" + flake.ref.subdir + "/flake.lock"); // FIXME: symlink attack
 
     ResolvedFlake deps(flake);
 
@@ -415,16 +420,19 @@ static LockFile makeLockFile(EvalState & evalState, FlakeRef & flakeRef)
     return lockFile;
 }
 
-void updateLockFile(EvalState & state, const Path & path)
+void updateLockFile(EvalState & state, const FlakeUri & flakeUri)
 {
     // FIXME: We are writing the lockfile to the store here! Very bad practice!
-    FlakeRef flakeRef = FlakeRef(path);
+    FlakeRef flakeRef = FlakeRef(flakeUri);
-    auto lockFile = makeLockFile(state, flakeRef);
+    if (auto refData = std::get_if<FlakeRef::IsPath>(&flakeRef.data)) {
-    writeLockFile(lockFile, path + "/flake.lock");
+        auto lockFile = makeLockFile(state, flakeRef);
+        writeLockFile(lockFile, refData->path + "/" + flakeRef.subdir + "/flake.lock");
 
-    // Hack: Make sure that flake.lock is visible to Git. Otherwise,
+        // Hack: Make sure that flake.lock is visible to Git. Otherwise,
-    // exportGit will fail to copy it to the Nix store.
+        // exportGit will fail to copy it to the Nix store.
-    runProgram("git", true, { "-C", path, "add", "flake.lock" });
+        runProgram("git", true, { "-C", refData->path, "add", flakeRef.subdir + "/flake.lock" });
+    } else
+        throw Error("flakeUri %s can't be updated because it is not a path", flakeUri);
 }
 
 void callFlake(EvalState & state, const ResolvedFlake & resFlake, Value & v)

src/libexpr/primops/flake.hh

@@ -90,7 +90,7 @@ struct ResolvedFlake
 
 ResolvedFlake resolveFlake(EvalState &, const FlakeRef &, RegistryAccess registryAccess, bool isTopFlake = true);
 
-void updateLockFile(EvalState &, const Path & path);
+void updateLockFile(EvalState &, const FlakeUri &);
 
 void gitCloneFlake (std::string flakeUri, EvalState &, Registries, Path);
 }

src/libexpr/primops/flakeref.cc

@@ -30,7 +30,16 @@ const static std::string schemeRegex = "(?:http|https|ssh|git|file)";
 const static std::string authorityRegex = "[a-zA-Z0-9._~-]*";
 const static std::string segmentRegex = "[a-zA-Z0-9._~-]+";
 const static std::string pathRegex = "/?" + segmentRegex + "(?:/" + segmentRegex + ")*";
-const static std::string paramRegex = "[a-z]+=[a-zA-Z0-9._-]*";
+// FIXME: support escaping in query string.
+// Note: '/' is not a valid query parameter, but so what...
+const static std::string paramRegex = "[a-z]+=[/a-zA-Z0-9._-]*";
+const static std::string paramsRegex = "(?:[?](" + paramRegex + "(?:&" + paramRegex + ")*))";
+
+// 'dir' path elements cannot start with a '.'. We also reject
+// potentially dangerous characters like ';'.
+const static std::string subDirElemRegex = "(?:[a-zA-Z0-9_-]+[a-zA-Z0-9._-]*)";
+const static std::string subDirRegex = subDirElemRegex + "(?:/" + subDirElemRegex + ")*";
+
 
 FlakeRef::FlakeRef(const std::string & uri, bool allowRelative)
 {
@@ -41,18 +50,21 @@ FlakeRef::FlakeRef(const std::string & uri, bool allowRelative)
         std::regex::ECMAScript);
 
     static std::regex githubRegex(
-        "github:(" + ownerRegex + ")/(" + repoRegex + ")(?:/" + revOrRefRegex + ")?",
+        "github:(" + ownerRegex + ")/(" + repoRegex + ")(?:/" + revOrRefRegex + ")?"
+        + paramsRegex + "?",
         std::regex::ECMAScript);
 
     static std::regex uriRegex(
         "((" + schemeRegex + "):" +
         "(?://(" + authorityRegex + "))?" +
         "(" + pathRegex + "))" +
-        "(?:[?](" + paramRegex + "(?:&" + paramRegex + ")*))?",
+        paramsRegex + "?",
         std::regex::ECMAScript);
 
     static std::regex refRegex2(refRegex, std::regex::ECMAScript);
 
+    static std::regex subDirRegex2(subDirRegex, std::regex::ECMAScript);
+
     std::cmatch match;
     if (std::regex_match(uri.c_str(), match, flakeRegex)) {
         IsAlias d;
@@ -76,6 +88,18 @@ FlakeRef::FlakeRef(const std::string & uri, bool allowRelative)
         else if (match[4].matched) {
             ref = match[4];
         }
+        for (auto & param : tokenizeString<Strings>(match[5], "&")) {
+            auto n = param.find('=');
+            assert(n != param.npos);
+            std::string name(param, 0, n);
+            std::string value(param, n + 1);
+            if (name == "dir") {
+                if (value != "" && !std::regex_match(value, subDirRegex2))
+                    throw Error("flake '%s' has invalid subdirectory '%s'", uri, value);
+                subdir = value;
+            } else
+                throw Error("invalid Git flake reference parameter '%s', in '%s'", name, uri);
+        }
         data = d;
     }
 
@@ -97,6 +121,10 @@ FlakeRef::FlakeRef(const std::string & uri, bool allowRelative)
                 if (!std::regex_match(value, refRegex2))
                     throw Error("invalid Git ref '%s'", value);
                 ref = value;
+            } else if (name == "dir") {
+                if (value != "" && !std::regex_match(value, subDirRegex2))
+                    throw Error("flake '%s' has invalid subdirectory '%s'", uri, value);
+                subdir = value;
             } else
                 // FIXME: should probably pass through unknown parameters
                 throw Error("invalid Git flake reference parameter '%s', in '%s'", name, uri);
@@ -119,6 +147,7 @@ FlakeRef::FlakeRef(const std::string & uri, bool allowRelative)
 std::string FlakeRef::to_string() const
 {
     std::string string;
+
     if (auto refData = std::get_if<FlakeRef::IsAlias>(&data))
         string = refData->alias;
 
@@ -137,8 +166,12 @@ std::string FlakeRef::to_string() const
 
     else abort();
 
+    // FIXME: need to use ?rev etc. for IsGit URIs.
     string += (ref ? "/" + *ref : "") +
               (rev ? "/" + rev->to_string(Base16, false) : "");
+
+    if (subdir != "") string += "?dir=" + subdir;
+
     return string;
 }
 

src/libexpr/primops/flakeref.hh

@@ -69,7 +69,7 @@ namespace nix {
        https://example.org/my/repo.git
        https://example.org/my/repo.git?ref=release-1.2.3
        https://example.org/my/repo.git?rev=e72daba8250068216d79d2aeef40d4d95aff6666
-       git://github.com/edolstra/dwarffs.git\?ref=flake\&rev=2efca4bc9da70fb001b26c3dc858c6397d3c4817
+       git://github.com/edolstra/dwarffs.git?ref=flake&rev=2efca4bc9da70fb001b26c3dc858c6397d3c4817
 
    * /path.git(\?attr(&attr)*)?
 
@@ -144,17 +144,18 @@ struct FlakeRef
 
     std::optional<std::string> ref;
     std::optional<Hash> rev;
+    Path subdir = ""; // This is a relative path pointing at the flake.nix file's directory, relative to the git root.
 
     bool operator<(const FlakeRef & flakeRef) const
     {
-        return std::make_tuple(this->data, ref, rev) <
+        return std::make_tuple(data, ref, rev, subdir) <
-            std::make_tuple(flakeRef.data, flakeRef.ref, flakeRef.rev);
+            std::make_tuple(flakeRef.data, flakeRef.ref, flakeRef.rev, subdir);
     }
 
     bool operator==(const FlakeRef & flakeRef) const
     {
-        return std::make_tuple(this->data, ref, rev) ==
+        return std::make_tuple(data, ref, rev, subdir) ==
-            std::make_tuple(flakeRef.data, flakeRef.ref, flakeRef.rev);
+            std::make_tuple(flakeRef.data, flakeRef.ref, flakeRef.rev, flakeRef.subdir);
     }
 
     // Parse a flake URI.