Merge pull request #5347 from edolstra/allow-context

Make addPath() work on paths with a context

src/libexpr/primops.cc

@@ -56,13 +56,9 @@ void EvalState::realiseContext(const PathSet & context)
             "cannot build '%1%' during evaluation because the option 'allow-import-from-derivation' is disabled",
             store->printStorePath(drvs.begin()->drvPath));
 
-    /* For performance, prefetch all substitute info. */
+    /* Build/substitute the context. */
-    StorePathSet willBuild, willSubstitute, unknown;
-    uint64_t downloadSize, narSize;
     std::vector<DerivedPath> buildReqs;
     for (auto & d : drvs) buildReqs.emplace_back(DerivedPath { d });
-    store->queryMissing(buildReqs, willBuild, willSubstitute, unknown, downloadSize, narSize);
-
     store->buildPaths(buildReqs);
 
     /* Add the output of this derivations to the allowed
@@ -1847,52 +1843,80 @@ static RegisterPrimOp primop_toFile({
     .fun = prim_toFile,
 });
 
-static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_,
+static void addPath(
-    Value * filterFun, FileIngestionMethod method, const std::optional<Hash> expectedHash, Value & v)
+    EvalState & state,
+    const Pos & pos,
+    const string & name,
+    Path path,
+    Value * filterFun,
+    FileIngestionMethod method,
+    const std::optional<Hash> expectedHash,
+    Value & v,
+    const PathSet & context)
 {
-    const auto path = evalSettings.pureEval && expectedHash ?
+    try {
-        path_ :
+        // FIXME: handle CA derivation outputs (where path needs to
-        state.checkSourcePath(path_);
+        // be rewritten to the actual output).
-    PathFilter filter = filterFun ? ([&](const Path & path) {
+        state.realiseContext(context);
-        auto st = lstat(path);
 
-        /* Call the filter function.  The first argument is the path,
+        path = evalSettings.pureEval && expectedHash
-           the second is a string indicating the type of the file. */
+            ? path
-        Value arg1;
+            : state.checkSourcePath(path);
-        mkString(arg1, path);
 
-        Value fun2;
+        if (state.store->isInStore(path)) {
-        state.callFunction(*filterFun, arg1, fun2, noPos);
+            auto storePath = state.store->toStorePath(path).first;
+            auto info = state.store->queryPathInfo(storePath);
+            if (!info->references.empty())
+                throw EvalError("store path '%s' is not allowed to have references",
+                    state.store->printStorePath(storePath));
+        }
 
-        Value arg2;
+        PathFilter filter = filterFun ? ([&](const Path & path) {
-        mkString(arg2,
+            auto st = lstat(path);
-            S_ISREG(st.st_mode) ? "regular" :
-            S_ISDIR(st.st_mode) ? "directory" :
-            S_ISLNK(st.st_mode) ? "symlink" :
-            "unknown" /* not supported, will fail! */);
 
-        Value res;
+            /* Call the filter function.  The first argument is the path,
-        state.callFunction(fun2, arg2, res, noPos);
+               the second is a string indicating the type of the file. */
+            Value arg1;
+            mkString(arg1, path);
 
-        return state.forceBool(res, pos);
+            Value fun2;
-    }) : defaultPathFilter;
+            state.callFunction(*filterFun, arg1, fun2, noPos);
 
-    std::optional<StorePath> expectedStorePath;
+            Value arg2;
-    if (expectedHash)
+            mkString(arg2,
-        expectedStorePath = state.store->makeFixedOutputPath(method, *expectedHash, name);
+                S_ISREG(st.st_mode) ? "regular" :
-    Path dstPath;
+                S_ISDIR(st.st_mode) ? "directory" :
-    if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
+                S_ISLNK(st.st_mode) ? "symlink" :
-        dstPath = state.store->printStorePath(settings.readOnlyMode
+                "unknown" /* not supported, will fail! */);
-            ? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first
-            : state.store->addToStore(name, path, method, htSHA256, filter, state.repair));
-        if (expectedHash && expectedStorePath != state.store->parseStorePath(dstPath))
-            throw Error("store path mismatch in (possibly filtered) path added from '%s'", path);
-    } else
-        dstPath = state.store->printStorePath(*expectedStorePath);
 
-    mkString(v, dstPath, {dstPath});
+            Value res;
+            state.callFunction(fun2, arg2, res, noPos);
 
-    state.allowPath(v.string.s);
+            return state.forceBool(res, pos);
+        }) : defaultPathFilter;
+
+        std::optional<StorePath> expectedStorePath;
+        if (expectedHash)
+            expectedStorePath = state.store->makeFixedOutputPath(method, *expectedHash, name);
+
+        Path dstPath;
+        if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
+            dstPath = state.store->printStorePath(settings.readOnlyMode
+                ? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first
+                : state.store->addToStore(name, path, method, htSHA256, filter, state.repair));
+            if (expectedHash && expectedStorePath != state.store->parseStorePath(dstPath))
+                throw Error("store path mismatch in (possibly filtered) path added from '%s'", path);
+        } else
+            dstPath = state.store->printStorePath(*expectedStorePath);
+
+        mkString(v, dstPath, {dstPath});
+
+        state.allowPath(v.string.s);
+
+    } catch (Error & e) {
+        e.addTrace(pos, "while adding path '%s'", path);
+        throw;
+    }
 }
 
 
@@ -1900,11 +1924,6 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
 {
     PathSet context;
     Path path = state.coerceToPath(pos, *args[1], context);
-    if (!context.empty())
-        throw EvalError({
-            .msg = hintfmt("string '%1%' cannot refer to other paths", path),
-            .errPos = pos
-        });
 
     state.forceValue(*args[0], pos);
     if (args[0]->type() != nFunction)
@@ -1915,7 +1934,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
             .errPos = pos
         });
 
-    addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v);
+    addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v, context);
 }
 
 static RegisterPrimOp primop_filterSource({
@@ -1981,18 +2000,13 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
     Value * filterFun = nullptr;
     auto method = FileIngestionMethod::Recursive;
     std::optional<Hash> expectedHash;
+    PathSet context;
 
     for (auto & attr : *args[0]->attrs) {
         const string & n(attr.name);
-        if (n == "path") {
+        if (n == "path")
-            PathSet context;
             path = state.coerceToPath(*attr.pos, *attr.value, context);
-            if (!context.empty())
+        else if (attr.name == state.sName)
-                throw EvalError({
-                    .msg = hintfmt("string '%1%' cannot refer to other paths", path),
-                    .errPos = *attr.pos
-                });
-        } else if (attr.name == state.sName)
             name = state.forceStringNoCtx(*attr.value, *attr.pos);
         else if (n == "filter") {
             state.forceValue(*attr.value, pos);
@@ -2015,7 +2029,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
     if (name.empty())
         name = baseNameOf(path);
 
-    addPath(state, pos, name, path, filterFun, method, expectedHash, v);
+    addPath(state, pos, name, path, filterFun, method, expectedHash, v, context);
 }
 
 static RegisterPrimOp primop_path({

src/libstore/build/worker.cc

@@ -239,7 +239,7 @@ void Worker::run(const Goals & _topGoals)
         }
     }
 
-    /* Call queryMissing() efficiently query substitutes. */
+    /* Call queryMissing() to efficiently query substitutes. */
     StorePathSet willBuild, willSubstitute, unknown;
     uint64_t downloadSize, narSize;
     store.queryMissing(topPaths, willBuild, willSubstitute, unknown, downloadSize, narSize);

tests/flakes.sh

@@ -262,6 +262,7 @@ cat > $flake3Dir/flake.nix <<EOF
         inherit system;
         name = "fnord";
         dummy = builtins.readFile (builtins.path { name = "source"; path = ./.; filter = path: type: baseNameOf path == "config.nix"; } + "/config.nix");
+        dummy2 = builtins.readFile (builtins.path { name = "source"; path = inputs.flake1; filter = path: type: baseNameOf path == "simple.nix"; } + "/simple.nix");
         buildCommand = ''
           cat \${inputs.nonFlake}/README.md > \$out
         '';