Introduce MemoryInputAccessor and use it for corepkgs

MemoryInputAccessor is an in-memory virtual filesystem that returns
files like <nix/fetchurl.nix>. This removes the need for special hacks
to handle those files.

doc/manual/generate-manpage.nix

@@ -2,7 +2,7 @@ let
   inherit (builtins)
     attrNames attrValues fromJSON listToAttrs mapAttrs groupBy
     concatStringsSep concatMap length lessThan replaceStrings sort;
-  inherit (import ./utils.nix) attrsToList concatStrings optionalString filterAttrs trim squash unique;
+  inherit (import <nix/utils.nix>) attrsToList concatStrings optionalString filterAttrs trim squash unique;
   showStoreDocs = import ./generate-store-info.nix;
 in
 

doc/manual/local.mk

@@ -32,7 +32,7 @@ dummy-env = env -i \
 	NIX_STATE_DIR=/dummy \
 	NIX_CONFIG='cores = 0'
 
-nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix/corepkgs=corepkgs --store dummy:// --impure --raw
+nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix=doc/manual --store dummy:// --impure --raw
 
 # re-implement mdBook's include directive to make it usable for terminal output and for proper @docroot@ substitution
 define process-includes

src/libexpr/eval.cc

@@ -13,6 +13,7 @@
 #include "profiles.hh"
 #include "print.hh"
 #include "fs-input-accessor.hh"
+#include "memory-input-accessor.hh"
 
 #include <algorithm>
 #include <chrono>
@@ -516,7 +517,16 @@ EvalState::EvalState(
                     : "in restricted mode";
                 throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", path, modeInformation);
             }))
-    , derivationInternal(rootPath(CanonPath("/builtin/derivation.nix")))
+    , corepkgsFS(makeMemoryInputAccessor())
+    , internalFS(makeMemoryInputAccessor())
+    , derivationInternal{corepkgsFS->addFile(
+        CanonPath("derivation-internal.nix"),
+        #include "primops/derivation.nix.gen.hh"
+    )}
+    , callFlakeInternal{internalFS->addFile(
+        CanonPath("call-flake.nix"),
+        #include "flake/call-flake.nix.gen.hh"
+    )}
     , store(store)
     , buildStore(buildStore ? buildStore : store)
     , debugRepl(nullptr)
@@ -531,7 +541,8 @@ EvalState::EvalState(
     , baseEnv(allocEnv(128))
     , staticBaseEnv{std::make_shared<StaticEnv>(false, nullptr)}
 {
-    rootFS->allowPath(CanonPath::root); // FIXME
+    // For now, don't rely on FSInputAccessor for access control.
+    rootFS->allowPath(CanonPath::root);
 
     countCalls = getEnv("NIX_COUNT_CALLS").value_or("0") != "0";
 
@@ -570,6 +581,11 @@ EvalState::EvalState(
         }
     }
 
+    corepkgsFS->addFile(
+        CanonPath("fetchurl.nix"),
+        #include "fetchurl.nix.gen.hh"
+    );
+
     createBaseEnv();
 }
 
@@ -600,6 +616,7 @@ void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value &
 
 SourcePath EvalState::checkSourcePath(const SourcePath & path_)
 {
+    if (path_.accessor != rootFS) return path_;
     if (!allowedPaths) return path_;
 
     auto i = resolvedPaths.find(path_.path.abs());
@@ -614,8 +631,6 @@ SourcePath EvalState::checkSourcePath(const SourcePath & path_)
      */
     Path abspath = canonPath(path_.path.abs());
 
-    if (hasPrefix(abspath, corepkgsPrefix)) return rootPath(CanonPath(abspath));
-
     for (auto & i : *allowedPaths) {
         if (isDirOrInDir(abspath, i)) {
             found = true;
@@ -1180,24 +1195,6 @@ void EvalState::evalFile(const SourcePath & path_, Value & v, bool mustBeTrivial
     if (!e)
         e = parseExprFromFile(checkSourcePath(resolvedPath));
 
-    cacheFile(path, resolvedPath, e, v, mustBeTrivial);
-}
-
-
-void EvalState::resetFileCache()
-{
-    fileEvalCache.clear();
-    fileParseCache.clear();
-}
-
-
-void EvalState::cacheFile(
-    const SourcePath & path,
-    const SourcePath & resolvedPath,
-    Expr * e,
-    Value & v,
-    bool mustBeTrivial)
-{
     fileParseCache[resolvedPath] = e;
 
     try {
@@ -1226,6 +1223,13 @@ void EvalState::cacheFile(
 }
 
 
+void EvalState::resetFileCache()
+{
+    fileEvalCache.clear();
+    fileParseCache.clear();
+}
+
+
 void EvalState::eval(Expr * e, Value & v)
 {
     e->eval(*this, baseEnv, v);
@@ -2341,10 +2345,31 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat
 
 SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx)
 {
-    auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned();
+    try {
-    if (path == "" || path[0] != '/')
+        forceValue(v, pos);
-        error("string '%1%' doesn't represent an absolute path", path).withTrace(pos, errorCtx).debugThrow<EvalError>();
+
-    return rootPath(CanonPath(path));
+        if (v.type() == nString) {
+            copyContext(v, context);
+            auto s = v.string_view();
+            if (!hasPrefix(s, "/"))
+                error("string '%s' doesn't represent an absolute path", s).atPos(pos).debugThrow<EvalError>();
+            return rootPath(CanonPath(s));
+        }
+    } catch (Error & e) {
+        e.addTrace(positions[pos], errorCtx);
+        throw;
+    }
+
+    if (v.type() == nPath)
+        return v.path();
+
+    if (v.type() == nAttrs) {
+        auto i = v.attrs->find(sOutPath);
+        if (i != v.attrs->end())
+            return coerceToPath(pos, *i->value, context, errorCtx);
+    }
+
+    error("cannot coerce %1% to a path", showType(v)).withTrace(pos, errorCtx).debugThrow<TypeError>();
 }
 
 

src/libexpr/eval.hh

@@ -25,6 +25,7 @@ class StorePath;
 struct SingleDerivedPath;
 enum RepairFlag : bool;
 struct FSInputAccessor;
+struct MemoryInputAccessor;
 
 
 /**
@@ -212,10 +213,26 @@ public:
 
     Bindings emptyBindings;
 
+    /**
+     * The accessor for the root filesystem.
+     */
     const ref<FSInputAccessor> rootFS;
 
+    /**
+     * The in-memory filesystem for <nix/...> paths.
+     */
+    const ref<MemoryInputAccessor> corepkgsFS;
+
+    /**
+     * In-memory filesystem for internal, non-user-callable Nix
+     * expressions like call-flake.nix.
+     */
+    const ref<MemoryInputAccessor> internalFS;
+
     const SourcePath derivationInternal;
 
+    const SourcePath callFlakeInternal;
+
     /**
      * Store used to materialise .drv files.
      */
@@ -226,7 +243,6 @@ public:
      */
     const ref<Store> buildStore;
 
-    RootValue vCallFlake = nullptr;
     RootValue vImportedDrvToDerivation = nullptr;
 
     /**
@@ -408,16 +424,6 @@ public:
      */
     void evalFile(const SourcePath & path, Value & v, bool mustBeTrivial = false);
 
-    /**
-     * Like `evalFile`, but with an already parsed expression.
-     */
-    void cacheFile(
-        const SourcePath & path,
-        const SourcePath & resolvedPath,
-        Expr * e,
-        Value & v,
-        bool mustBeTrivial = false);
-
     void resetFileCache();
 
     /**
@@ -427,7 +433,7 @@ public:
     SourcePath findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos);
 
     /**
-     * Try to resolve a search path value (not the optinal key part)
+     * Try to resolve a search path value (not the optional key part)
      *
      * If the specified search path element is a URI, download it.
      *
@@ -832,8 +838,6 @@ struct InvalidPathError : EvalError
 #endif
 };
 
-static const std::string corepkgsPrefix{"/__corepkgs__/"};
-
 template<class ErrorType>
 void ErrorBuilder::debugThrow()
 {

src/libexpr/flake/flake.cc

@@ -737,14 +737,10 @@ void callFlake(EvalState & state,
 
     vRootSubdir->mkString(lockedFlake.flake.lockedRef.subdir);
 
-    if (!state.vCallFlake) {
+    auto vCallFlake = state.allocValue();
-        state.vCallFlake = allocRootValue(state.allocValue());
+    state.evalFile(state.callFlakeInternal, *vCallFlake);
-        state.eval(state.parseExprFromString(
-            #include "call-flake.nix.gen.hh"
-            , state.rootPath(CanonPath::root)), **state.vCallFlake);
-    }
 
-    state.callFunction(**state.vCallFlake, *vLocks, *vTmp1, noPos);
+    state.callFunction(*vCallFlake, *vLocks, *vTmp1, noPos);
     state.callFunction(*vTmp1, *vRootSrc, *vTmp2, noPos);
     state.callFunction(*vTmp2, *vRootSubdir, vRes, noPos);
 }

src/libexpr/parser.y

@@ -64,7 +64,6 @@ struct StringToken {
 
 #include "parser-tab.hh"
 #include "lexer-tab.hh"
-#include "fs-input-accessor.hh"
 
 YY_DECL;
 
@@ -650,6 +649,8 @@ formal
 #include "fetchers.hh"
 #include "store-api.hh"
 #include "flake/flake.hh"
+#include "fs-input-accessor.hh"
+#include "memory-input-accessor.hh"
 
 
 namespace nix {
@@ -761,7 +762,7 @@ SourcePath EvalState::findFile(const SearchPath & searchPath, const std::string_
     }
 
     if (hasPrefix(path, "nix/"))
-        return rootPath(CanonPath(concatStrings(corepkgsPrefix, path.substr(4))));
+        return {corepkgsFS, CanonPath(path.substr(3))};
 
     debugThrow(ThrownError({
         .msg = hintfmt(evalSettings.pureEval

src/libexpr/primops.cc

@@ -121,13 +121,15 @@ static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v, co
     auto path = state.coerceToPath(noPos, v, context, "while realising the context of a path");
 
     try {
-        StringMap rewrites = state.realiseContext(context);
+        if (!context.empty()) {
-
+            auto rewrites = state.realiseContext(context);
-        auto realPath = state.rootPath(CanonPath(state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context)));
+            auto realPath = state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context);
+            return {path.accessor, CanonPath(realPath)};
+        }
 
         return flags.checkForPureEval
-            ? state.checkSourcePath(realPath)
+            ? state.checkSourcePath(path)
-            : realPath;
+            : path;
     } catch (Error & e) {
         e.addTrace(state.positions[pos], "while realising the context of path '%s'", path);
         throw;
@@ -210,12 +212,6 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v
         state.forceAttrs(v, pos, "while calling imported-drv-to-derivation.nix.gen.hh");
     }
 
-    else if (path2 == corepkgsPrefix + "fetchurl.nix") {
-        state.eval(state.parseExprFromString(
-            #include "fetchurl.nix.gen.hh"
-            , state.rootPath(CanonPath::root)), v);
-    }
-
     else {
         if (!vScope)
             state.evalFile(path, v);
@@ -1486,7 +1482,7 @@ static void prim_storePath(EvalState & state, const PosIdx pos, Value * * args,
         }));
 
     NixStringContext context;
-    auto path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to builtins.storePath")).path;
+    auto path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to 'builtins.storePath'")).path;
     /* Resolve symlinks in ‘path’, unless ‘path’ itself is a symlink
        directly in the store.  The latter condition is necessary so
        e.g. nix-push does the right thing. */
@@ -2257,7 +2253,7 @@ static void prim_filterSource(EvalState & state, const PosIdx pos, Value * * arg
 {
     NixStringContext context;
     auto path = state.coerceToPath(pos, *args[1], context,
-        "while evaluating the second argument (the path to filter) passed to builtins.filterSource");
+        "while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'");
     state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filterSource");
     addPath(state, pos, path.baseName(), path.path.abs(), args[0], FileIngestionMethod::Recursive, std::nullopt, v, context);
 }
@@ -4444,12 +4440,7 @@ void EvalState::createBaseEnv()
 
     /* Note: we have to initialize the 'derivation' constant *after*
        building baseEnv/staticBaseEnv because it uses 'builtins'. */
-    char code[] =
+    evalFile(derivationInternal, *vDerivation);
-        #include "primops/derivation.nix.gen.hh"
-        // the parser needs two NUL bytes as terminators; one of them
-        // is implied by being a C string.
-        "\0";
-    eval(parse(code, sizeof(code), derivationInternal, rootPath(CanonPath::root), staticBaseEnv), *vDerivation);
 }
 
 

src/libexpr/tests/error_traces.cc

@@ -295,7 +295,7 @@ namespace nix {
     TEST_F(ErrorTraceTest, toPath) {
         ASSERT_TRACE2("toPath []",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string", "a list"),
+                      hintfmt("cannot coerce %s to a path", "a list"),
                       hintfmt("while evaluating the first argument passed to builtins.toPath"));
 
         ASSERT_TRACE2("toPath \"foo\"",
@@ -309,8 +309,8 @@ namespace nix {
     TEST_F(ErrorTraceTest, storePath) {
         ASSERT_TRACE2("storePath true",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string", "a Boolean"),
+                      hintfmt("cannot coerce %s to a path", "a Boolean"),
-                      hintfmt("while evaluating the first argument passed to builtins.storePath"));
+                      hintfmt("while evaluating the first argument passed to 'builtins.storePath'"));
 
     }
 
@@ -318,7 +318,7 @@ namespace nix {
     TEST_F(ErrorTraceTest, pathExists) {
         ASSERT_TRACE2("pathExists []",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string", "a list"),
+                      hintfmt("cannot coerce %s to a path", "a list"),
                       hintfmt("while realising the context of a path"));
 
         ASSERT_TRACE2("pathExists \"zorglub\"",
@@ -377,13 +377,13 @@ namespace nix {
     TEST_F(ErrorTraceTest, filterSource) {
         ASSERT_TRACE2("filterSource [] []",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string", "a list"),
+                      hintfmt("cannot coerce %s to a path", "a list"),
-                      hintfmt("while evaluating the second argument (the path to filter) passed to builtins.filterSource"));
+                      hintfmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'"));
 
         ASSERT_TRACE2("filterSource [] \"foo\"",
                       EvalError,
                       hintfmt("string '%s' doesn't represent an absolute path", "foo"),
-                      hintfmt("while evaluating the second argument (the path to filter) passed to builtins.filterSource"));
+                      hintfmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'"));
 
         ASSERT_TRACE2("filterSource [] ./.",
                       TypeError,

src/libfetchers/memory-input-accessor.cc

@@ -0,0 +1,54 @@
+#include "memory-input-accessor.hh"
+
+namespace nix {
+
+struct MemoryInputAccessorImpl : MemoryInputAccessor
+{
+    std::map<CanonPath, std::string> files;
+
+    std::string readFile(const CanonPath & path) override
+    {
+        auto i = files.find(path);
+        if (i == files.end())
+            throw Error("file '%s' does not exist", path);
+        return i->second;
+    }
+
+    bool pathExists(const CanonPath & path) override
+    {
+        auto i = files.find(path);
+        return i != files.end();
+    }
+
+    Stat lstat(const CanonPath & path) override
+    {
+        auto i = files.find(path);
+        if (i != files.end())
+            return Stat { .type = tRegular, .isExecutable = false };
+        throw Error("file '%s' does not exist", path);
+    }
+
+    DirEntries readDirectory(const CanonPath & path) override
+    {
+        return {};
+    }
+
+    std::string readLink(const CanonPath & path) override
+    {
+        throw UnimplementedError("MemoryInputAccessor::readLink");
+    }
+
+    SourcePath addFile(CanonPath path, std::string && contents) override
+    {
+        files.emplace(path, std::move(contents));
+
+        return {ref(shared_from_this()), std::move(path)};
+    }
+};
+
+ref<MemoryInputAccessor> makeMemoryInputAccessor()
+{
+    return make_ref<MemoryInputAccessorImpl>();
+}
+
+}

src/libfetchers/memory-input-accessor.hh

@@ -0,0 +1,15 @@
+#include "input-accessor.hh"
+
+namespace nix {
+
+/**
+ * An input accessor for an in-memory file system.
+ */
+struct MemoryInputAccessor : InputAccessor
+{
+    virtual SourcePath addFile(CanonPath path, std::string && contents) = 0;
+};
+
+ref<MemoryInputAccessor> makeMemoryInputAccessor();
+
+}

src/nix/main.cc

@@ -12,6 +12,7 @@
 #include "finally.hh"
 #include "loggers.hh"
 #include "markdown.hh"
+#include "memory-input-accessor.hh"
 
 #include <sys/types.h>
 #include <sys/socket.h>
@@ -206,29 +207,20 @@ static void showHelp(std::vector<std::string> subcommand, NixArgs & toplevel)
         #include "generate-manpage.nix.gen.hh"
         , state.rootPath(CanonPath::root)), *vGenerateManpage);
 
-    auto vUtils = state.allocValue();
+    state.corepkgsFS->addFile(
-    state.cacheFile(
+        CanonPath("utils.nix"),
-        state.rootPath(CanonPath("/utils.nix")), state.rootPath(CanonPath("/utils.nix")),
-        state.parseExprFromString(
         #include "utils.nix.gen.hh"
-            , state.rootPath(CanonPath::root)),
+        );
-        *vUtils);
 
-    auto vSettingsInfo = state.allocValue();
+    state.corepkgsFS->addFile(
-    state.cacheFile(
+        CanonPath("/generate-settings.nix"),
-        state.rootPath(CanonPath("/generate-settings.nix")), state.rootPath(CanonPath("/generate-settings.nix")),
-        state.parseExprFromString(
         #include "generate-settings.nix.gen.hh"
-            , state.rootPath(CanonPath::root)),
+        );
-        *vSettingsInfo);
 
-    auto vStoreInfo = state.allocValue();
+    state.corepkgsFS->addFile(
-    state.cacheFile(
+        CanonPath("/generate-store-info.nix"),
-        state.rootPath(CanonPath("/generate-store-info.nix")), state.rootPath(CanonPath("/generate-store-info.nix")),
-        state.parseExprFromString(
         #include "generate-store-info.nix.gen.hh"
-            , state.rootPath(CanonPath::root)),
+        );
-        *vStoreInfo);
 
     auto vDump = state.allocValue();
     vDump->mkString(toplevel.dumpCli());