Merge pull request #9177 from edolstra/input-accessors

Backport FSInputAccessor and MemoryInputAccessor from lazy-trees

doc/manual/generate-manpage.nix

@@ -2,7 +2,7 @@ let
   inherit (builtins)
     attrNames attrValues fromJSON listToAttrs mapAttrs groupBy
     concatStringsSep concatMap length lessThan replaceStrings sort;
-  inherit (import ./utils.nix) attrsToList concatStrings optionalString filterAttrs trim squash unique;
+  inherit (import <nix/utils.nix>) attrsToList concatStrings optionalString filterAttrs trim squash unique;
   showStoreDocs = import ./generate-store-info.nix;
 in
 

doc/manual/local.mk

@@ -32,7 +32,7 @@ dummy-env = env -i \
 	NIX_STATE_DIR=/dummy \
 	NIX_CONFIG='cores = 0'
 
-nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix/corepkgs=corepkgs --store dummy:// --impure --raw
+nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix=doc/manual --store dummy:// --impure --raw
 
 # re-implement mdBook's include directive to make it usable for terminal output and for proper @docroot@ substitution
 define process-includes
@@ -125,7 +125,7 @@ $(d)/src/command-ref/experimental-features-shortlist.md: $(d)/xp-features.json $
 	@mv $@.tmp $@
 
 $(d)/xp-features.json: $(bindir)/nix
-	$(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-xp-features > $@.tmp
+	$(trace-gen) $(dummy-env) $(bindir)/nix __dump-xp-features > $@.tmp
 	@mv $@.tmp $@
 
 $(d)/src/language/builtins.md: $(d)/language.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix
@@ -141,7 +141,7 @@ $(d)/src/language/builtin-constants.md: $(d)/language.json $(d)/generate-builtin
 	@mv $@.tmp $@
 
 $(d)/language.json: $(bindir)/nix
-	$(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-language > $@.tmp
+	$(trace-gen) $(dummy-env) $(bindir)/nix __dump-language > $@.tmp
 	@mv $@.tmp $@
 
 # Generate the HTML manual.

src/libexpr/attr-path.cc

@@ -132,7 +132,7 @@ std::pair<SourcePath, uint32_t> findPackageFilename(EvalState & state, Value & v
         if (colon == std::string::npos) fail();
         std::string filename(fn, 0, colon);
         auto lineno = std::stoi(std::string(fn, colon + 1, std::string::npos));
-        return {CanonPath(fn.substr(0, colon)), lineno};
+        return {SourcePath{path.accessor, CanonPath(fn.substr(0, colon))}, lineno};
     } catch (std::invalid_argument & e) {
         fail();
         abort();

src/libexpr/eval.cc

@@ -12,6 +12,8 @@
 #include "function-trace.hh"
 #include "profiles.hh"
 #include "print.hh"
+#include "fs-input-accessor.hh"
+#include "memory-input-accessor.hh"
 
 #include <algorithm>
 #include <chrono>
@@ -503,7 +505,17 @@ EvalState::EvalState(
     , sOutputSpecified(symbols.create("outputSpecified"))
     , repair(NoRepair)
     , emptyBindings(0)
-    , derivationInternal(rootPath(CanonPath("/builtin/derivation.nix")))
+    , rootFS(makeFSInputAccessor(CanonPath::root))
+    , corepkgsFS(makeMemoryInputAccessor())
+    , internalFS(makeMemoryInputAccessor())
+    , derivationInternal{corepkgsFS->addFile(
+        CanonPath("derivation-internal.nix"),
+        #include "primops/derivation.nix.gen.hh"
+    )}
+    , callFlakeInternal{internalFS->addFile(
+        CanonPath("call-flake.nix"),
+        #include "flake/call-flake.nix.gen.hh"
+    )}
     , store(store)
     , buildStore(buildStore ? buildStore : store)
     , debugRepl(nullptr)
@@ -555,6 +567,11 @@ EvalState::EvalState(
         }
     }
 
+    corepkgsFS->addFile(
+        CanonPath("fetchurl.nix"),
+        #include "fetchurl.nix.gen.hh"
+    );
+
     createBaseEnv();
 }
 
@@ -585,6 +602,9 @@ void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value &
 
 SourcePath EvalState::checkSourcePath(const SourcePath & path_)
 {
+    // Don't check non-rootFS accessors, they're in a different namespace.
+    if (path_.accessor != ref<InputAccessor>(rootFS)) return path_;
+
     if (!allowedPaths) return path_;
 
     auto i = resolvedPaths.find(path_.path.abs());
@@ -599,8 +619,6 @@ SourcePath EvalState::checkSourcePath(const SourcePath & path_)
      */
     Path abspath = canonPath(path_.path.abs());
 
-    if (hasPrefix(abspath, corepkgsPrefix)) return CanonPath(abspath);
-
     for (auto & i : *allowedPaths) {
         if (isDirOrInDir(abspath, i)) {
             found = true;
@@ -617,7 +635,7 @@ SourcePath EvalState::checkSourcePath(const SourcePath & path_)
 
     /* Resolve symlinks. */
     debug("checking access to '%s'", abspath);
-    SourcePath path = CanonPath(canonPath(abspath, true));
+    SourcePath path = rootPath(CanonPath(canonPath(abspath, true)));
 
     for (auto & i : *allowedPaths) {
         if (isDirOrInDir(path.path.abs(), i)) {
@@ -649,12 +667,12 @@ void EvalState::checkURI(const std::string & uri)
     /* If the URI is a path, then check it against allowedPaths as
        well. */
     if (hasPrefix(uri, "/")) {
-        checkSourcePath(CanonPath(uri));
+        checkSourcePath(rootPath(CanonPath(uri)));
         return;
     }
 
     if (hasPrefix(uri, "file://")) {
-        checkSourcePath(CanonPath(std::string(uri, 7)));
+        checkSourcePath(rootPath(CanonPath(std::string(uri, 7))));
         return;
     }
 
@@ -950,7 +968,7 @@ void Value::mkStringMove(const char * s, const NixStringContext & context)
 
 void Value::mkPath(const SourcePath & path)
 {
-    mkPath(makeImmutableString(path.path.abs()));
+    mkPath(&*path.accessor, makeImmutableString(path.path.abs()));
 }
 
 
@@ -1165,24 +1183,6 @@ void EvalState::evalFile(const SourcePath & path_, Value & v, bool mustBeTrivial
     if (!e)
         e = parseExprFromFile(checkSourcePath(resolvedPath));
 
-    cacheFile(path, resolvedPath, e, v, mustBeTrivial);
-}
-
-
-void EvalState::resetFileCache()
-{
-    fileEvalCache.clear();
-    fileParseCache.clear();
-}
-
-
-void EvalState::cacheFile(
-    const SourcePath & path,
-    const SourcePath & resolvedPath,
-    Expr * e,
-    Value & v,
-    bool mustBeTrivial)
-{
     fileParseCache[resolvedPath] = e;
 
     try {
@@ -1211,6 +1211,13 @@ void EvalState::cacheFile(
 }
 
 
+void EvalState::resetFileCache()
+{
+    fileEvalCache.clear();
+    fileParseCache.clear();
+}
+
+
 void EvalState::eval(Expr * e, Value & v)
 {
     e->eval(*this, baseEnv, v);
@@ -2037,7 +2044,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
     else if (firstType == nPath) {
         if (!context.empty())
             state.error("a string that refers to a store path cannot be appended to a path").atPos(pos).withFrame(env, *this).debugThrow<EvalError>();
-        v.mkPath(CanonPath(canonPath(str())));
+        v.mkPath(state.rootPath(CanonPath(canonPath(str()))));
     } else
         v.mkStringMove(c_str(), context);
 }
@@ -2236,7 +2243,7 @@ BackedStringView EvalState::coerceToString(
             !canonicalizePath && !copyToStore
             ? // FIXME: hack to preserve path literals that end in a
               // slash, as in /foo/${x}.
-              v._path
+              v._path.path
             : copyToStore
             ? store->printStorePath(copyPathToStore(context, v.path()))
             : std::string(v.path().path.abs());
@@ -2310,7 +2317,7 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat
     auto dstPath = i != srcToStore.end()
         ? i->second
         : [&]() {
-            auto dstPath = path.fetchToStore(store, path.baseName(), nullptr, repair);
+            auto dstPath = path.fetchToStore(store, path.baseName(), FileIngestionMethod::Recursive, nullptr, repair);
             allowPath(dstPath);
             srcToStore.insert_or_assign(path, dstPath);
             printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath));
@@ -2326,10 +2333,34 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat
 
 SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx)
 {
+    try {
+        forceValue(v, pos);
+    } catch (Error & e) {
+        e.addTrace(positions[pos], errorCtx);
+        throw;
+    }
+
+    /* Handle path values directly, without coercing to a string. */
+    if (v.type() == nPath)
+        return v.path();
+
+    /* Similarly, handle __toString where the result may be a path
+       value. */
+    if (v.type() == nAttrs) {
+        auto i = v.attrs->find(sToString);
+        if (i != v.attrs->end()) {
+            Value v1;
+            callFunction(*i->value, v, v1, pos);
+            return coerceToPath(pos, v1, context, errorCtx);
+        }
+    }
+
+    /* Any other value should be coercable to a string, interpreted
+       relative to the root filesystem. */
     auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned();
     if (path == "" || path[0] != '/')
         error("string '%1%' doesn't represent an absolute path", path).withTrace(pos, errorCtx).debugThrow<EvalError>();
-    return CanonPath(path);
+    return rootPath(CanonPath(path));
 }
 
 
@@ -2429,7 +2460,10 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v
             return v1.string_view().compare(v2.string_view()) == 0;
 
         case nPath:
-            return strcmp(v1._path, v2._path) == 0;
+            return
+                // FIXME: compare accessors by their fingerprint.
+                v1._path.accessor == v2._path.accessor
+                && strcmp(v1._path.path, v2._path.path) == 0;
 
         case nNull:
             return true;

src/libexpr/eval.hh

@@ -24,6 +24,8 @@ class EvalState;
 class StorePath;
 struct SingleDerivedPath;
 enum RepairFlag : bool;
+struct FSInputAccessor;
+struct MemoryInputAccessor;
 
 
 /**
@@ -211,8 +213,26 @@ public:
 
     Bindings emptyBindings;
 
+    /**
+     * The accessor for the root filesystem.
+     */
+    const ref<FSInputAccessor> rootFS;
+
+    /**
+     * The in-memory filesystem for <nix/...> paths.
+     */
+    const ref<MemoryInputAccessor> corepkgsFS;
+
+    /**
+     * In-memory filesystem for internal, non-user-callable Nix
+     * expressions like call-flake.nix.
+     */
+    const ref<MemoryInputAccessor> internalFS;
+
     const SourcePath derivationInternal;
 
+    const SourcePath callFlakeInternal;
+
     /**
      * Store used to materialise .drv files.
      */
@@ -223,7 +243,6 @@ public:
      */
     const ref<Store> buildStore;
 
-    RootValue vCallFlake = nullptr;
     RootValue vImportedDrvToDerivation = nullptr;
 
     /**
@@ -405,16 +424,6 @@ public:
      */
     void evalFile(const SourcePath & path, Value & v, bool mustBeTrivial = false);
 
-    /**
-     * Like `evalFile`, but with an already parsed expression.
-     */
-    void cacheFile(
-        const SourcePath & path,
-        const SourcePath & resolvedPath,
-        Expr * e,
-        Value & v,
-        bool mustBeTrivial = false);
-
     void resetFileCache();
 
     /**
@@ -424,7 +433,7 @@ public:
     SourcePath findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos);
 
     /**
-     * Try to resolve a search path value (not the optinal key part)
+     * Try to resolve a search path value (not the optional key part)
      *
      * If the specified search path element is a URI, download it.
      *
@@ -829,8 +838,6 @@ struct InvalidPathError : EvalError
 #endif
 };
 
-static const std::string corepkgsPrefix{"/__corepkgs__/"};
-
 template<class ErrorType>
 void ErrorBuilder::debugThrow()
 {

src/libexpr/flake/flake.cc

@@ -223,9 +223,9 @@ static Flake getFlake(
         throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", lockedRef, lockedRef.subdir);
 
     Value vInfo;
-    state.evalFile(CanonPath(flakeFile), vInfo, true); // FIXME: symlink attack
+    state.evalFile(state.rootPath(CanonPath(flakeFile)), vInfo, true); // FIXME: symlink attack
 
-    expectType(state, nAttrs, vInfo, state.positions.add({CanonPath(flakeFile)}, 1, 1));
+    expectType(state, nAttrs, vInfo, state.positions.add({state.rootPath(CanonPath(flakeFile))}, 1, 1));
 
     if (auto description = vInfo.attrs->get(state.sDescription)) {
         expectType(state, nString, *description->value, description->pos);
@@ -737,14 +737,10 @@ void callFlake(EvalState & state,
 
     vRootSubdir->mkString(lockedFlake.flake.lockedRef.subdir);
 
-    if (!state.vCallFlake) {
+    auto vCallFlake = state.allocValue();
-        state.vCallFlake = allocRootValue(state.allocValue());
+    state.evalFile(state.callFlakeInternal, *vCallFlake);
-        state.eval(state.parseExprFromString(
-            #include "call-flake.nix.gen.hh"
-            , CanonPath::root), **state.vCallFlake);
-    }
 
-    state.callFunction(**state.vCallFlake, *vLocks, *vTmp1, noPos);
+    state.callFunction(*vCallFlake, *vLocks, *vTmp1, noPos);
     state.callFunction(*vTmp1, *vRootSrc, *vTmp2, noPos);
     state.callFunction(*vTmp2, *vRootSubdir, vRes, noPos);
 }

src/libexpr/nixexpr.hh

@@ -20,7 +20,6 @@ MakeError(Abort, EvalError);
 MakeError(TypeError, EvalError);
 MakeError(UndefinedVarError, Error);
 MakeError(MissingArgumentError, EvalError);
-MakeError(RestrictedPathError, Error);
 
 /**
  * Position objects.
@@ -200,9 +199,13 @@ struct ExprString : Expr
 
 struct ExprPath : Expr
 {
+    ref<InputAccessor> accessor;
     std::string s;
     Value v;
-    ExprPath(std::string s) : s(std::move(s)) { v.mkPath(this->s.c_str()); };
+    ExprPath(ref<InputAccessor> accessor, std::string s) : accessor(accessor), s(std::move(s))
+    {
+        v.mkPath(&*accessor, this->s.c_str());
+    }
     Value * maybeThunk(EvalState & state, Env & env) override;
     COMMON_METHODS
 };

src/libexpr/parser.y

@@ -520,7 +520,7 @@ path_start
     /* add back in the trailing '/' to the first segment */
     if ($1.p[$1.l-1] == '/' && $1.l > 1)
       path += "/";
-    $$ = new ExprPath(std::move(path));
+    $$ = new ExprPath(ref<InputAccessor>(data->state.rootFS), std::move(path));
   }
   | HPATH {
     if (evalSettings.pureEval) {
@@ -530,7 +530,7 @@ path_start
         );
     }
     Path path(getHome() + std::string($1.p + 1, $1.l - 1));
-    $$ = new ExprPath(std::move(path));
+    $$ = new ExprPath(ref<InputAccessor>(data->state.rootFS), std::move(path));
   }
   ;
 
@@ -649,6 +649,8 @@ formal
 #include "tarball.hh"
 #include "store-api.hh"
 #include "flake/flake.hh"
+#include "fs-input-accessor.hh"
+#include "memory-input-accessor.hh"
 
 
 namespace nix {
@@ -756,11 +758,11 @@ SourcePath EvalState::findFile(const SearchPath & searchPath, const std::string_
         auto r = *rOpt;
 
         Path res = suffix == "" ? r : concatStrings(r, "/", suffix);
-        if (pathExists(res)) return CanonPath(canonPath(res));
+        if (pathExists(res)) return rootPath(CanonPath(canonPath(res)));
     }
 
     if (hasPrefix(path, "nix/"))
-        return CanonPath(concatStrings(corepkgsPrefix, path.substr(4)));
+        return {corepkgsFS, CanonPath(path.substr(3))};
 
     debugThrow(ThrownError({
         .msg = hintfmt(evalSettings.pureEval

src/libexpr/paths.cc

@@ -1,10 +1,11 @@
 #include "eval.hh"
+#include "fs-input-accessor.hh"
 
 namespace nix {
 
 SourcePath EvalState::rootPath(CanonPath path)
 {
-    return path;
+    return {rootFS, std::move(path)};
 }
 
 }

src/libexpr/primops.cc

@@ -121,13 +121,15 @@ static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v, co
     auto path = state.coerceToPath(noPos, v, context, "while realising the context of a path");
 
     try {
-        StringMap rewrites = state.realiseContext(context);
+        if (!context.empty()) {
-
+            auto rewrites = state.realiseContext(context);
-        auto realPath = state.rootPath(CanonPath(state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context)));
+            auto realPath = state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context);
+            return {path.accessor, CanonPath(realPath)};
+        }
 
         return flags.checkForPureEval
-            ? state.checkSourcePath(realPath)
+            ? state.checkSourcePath(path)
-            : realPath;
+            : path;
     } catch (Error & e) {
         e.addTrace(state.positions[pos], "while realising the context of path '%s'", path);
         throw;
@@ -202,7 +204,7 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v
             state.vImportedDrvToDerivation = allocRootValue(state.allocValue());
             state.eval(state.parseExprFromString(
                 #include "imported-drv-to-derivation.nix.gen.hh"
-                , CanonPath::root), **state.vImportedDrvToDerivation);
+                , state.rootPath(CanonPath::root)), **state.vImportedDrvToDerivation);
         }
 
         state.forceFunction(**state.vImportedDrvToDerivation, pos, "while evaluating imported-drv-to-derivation.nix.gen.hh");
@@ -210,12 +212,6 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v
         state.forceAttrs(v, pos, "while calling imported-drv-to-derivation.nix.gen.hh");
     }
 
-    else if (path2 == corepkgsPrefix + "fetchurl.nix") {
-        state.eval(state.parseExprFromString(
-            #include "fetchurl.nix.gen.hh"
-            , CanonPath::root), v);
-    }
-
     else {
         if (!vScope)
             state.evalFile(path, v);
@@ -599,7 +595,10 @@ struct CompareValues
                 case nString:
                     return v1->string_view().compare(v2->string_view()) < 0;
                 case nPath:
-                    return strcmp(v1->_path, v2->_path) < 0;
+                    // Note: we don't take the accessor into account
+                    // since it's not obvious how to compare them in a
+                    // reproducible way.
+                    return strcmp(v1->_path.path, v2->_path.path) < 0;
                 case nList:
                     // Lexicographic comparison
                     for (size_t i = 0;; i++) {
@@ -1493,7 +1492,7 @@ static void prim_storePath(EvalState & state, const PosIdx pos, Value * * args,
         }));
 
     NixStringContext context;
-    auto path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to builtins.storePath")).path;
+    auto path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to 'builtins.storePath'")).path;
     /* Resolve symlinks in ‘path’, unless ‘path’ itself is a symlink
        directly in the store.  The latter condition is necessary so
        e.g. nix-push does the right thing. */
@@ -2211,7 +2210,7 @@ static void addPath(
 
         path = evalSettings.pureEval && expectedHash
             ? path
-            : state.checkSourcePath(CanonPath(path)).path.abs();
+            : state.checkSourcePath(state.rootPath(CanonPath(path))).path.abs();
 
         PathFilter filter = filterFun ? ([&](const Path & path) {
             auto st = lstat(path);
@@ -2244,9 +2243,7 @@ static void addPath(
             });
 
         if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
-            StorePath dstPath = settings.readOnlyMode
+            auto dstPath = state.rootPath(CanonPath(path)).fetchToStore(state.store, name, method, &filter, state.repair);
-                ? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first
-                : state.store->addToStore(name, path, method, htSHA256, filter, state.repair, refs);
             if (expectedHash && expectedStorePath != dstPath)
                 state.debugThrowLastTrace(Error("store path mismatch in (possibly filtered) path added from '%s'", path));
             state.allowAndSetStorePathString(dstPath, v);
@@ -2263,7 +2260,7 @@ static void prim_filterSource(EvalState & state, const PosIdx pos, Value * * arg
 {
     NixStringContext context;
     auto path = state.coerceToPath(pos, *args[1], context,
-        "while evaluating the second argument (the path to filter) passed to builtins.filterSource");
+        "while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'");
     state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filterSource");
     addPath(state, pos, path.baseName(), path.path.abs(), args[0], FileIngestionMethod::Recursive, std::nullopt, v, context);
 }
@@ -4545,12 +4542,7 @@ void EvalState::createBaseEnv()
 
     /* Note: we have to initialize the 'derivation' constant *after*
        building baseEnv/staticBaseEnv because it uses 'builtins'. */
-    char code[] =
+    evalFile(derivationInternal, *vDerivation);
-        #include "primops/derivation.nix.gen.hh"
-        // the parser needs two NUL bytes as terminators; one of them
-        // is implied by being a C string.
-        "\0";
-    eval(parse(code, sizeof(code), derivationInternal, {CanonPath::root}, staticBaseEnv), *vDerivation);
 }
 
 

src/libexpr/tests/error_traces.cc

@@ -310,7 +310,7 @@ namespace nix {
         ASSERT_TRACE2("storePath true",
                       TypeError,
                       hintfmt("cannot coerce %s to a string", "a Boolean"),
-                      hintfmt("while evaluating the first argument passed to builtins.storePath"));
+                      hintfmt("while evaluating the first argument passed to 'builtins.storePath'"));
 
     }
 
@@ -378,12 +378,12 @@ namespace nix {
         ASSERT_TRACE2("filterSource [] []",
                       TypeError,
                       hintfmt("cannot coerce %s to a string", "a list"),
-                      hintfmt("while evaluating the second argument (the path to filter) passed to builtins.filterSource"));
+                      hintfmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'"));
 
         ASSERT_TRACE2("filterSource [] \"foo\"",
                       EvalError,
                       hintfmt("string '%s' doesn't represent an absolute path", "foo"),
-                      hintfmt("while evaluating the second argument (the path to filter) passed to builtins.filterSource"));
+                      hintfmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'"));
 
         ASSERT_TRACE2("filterSource [] ./.",
                       TypeError,

src/libexpr/tests/json.cc

@@ -62,7 +62,7 @@ namespace nix {
     // not supported by store 'dummy'" thrown in the test body.
     TEST_F(JSONValueTest, DISABLED_Path) {
         Value v;
-        v.mkPath("test");
+        v.mkPath(state.rootPath(CanonPath("/test")));
         ASSERT_EQ(getJSONValue(v), "\"/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x\"");
     }
 } /* namespace nix */

src/libexpr/tests/libexpr.hh

@@ -103,14 +103,17 @@ namespace nix {
     }
 
     MATCHER_P(IsPathEq, p, fmt("Is a path equal to \"%1%\"", p)) {
-            if (arg.type() != nPath) {
+        if (arg.type() != nPath) {
-                *result_listener << "Expected a path got " << arg.type();
+            *result_listener << "Expected a path got " << arg.type();
-                return false;
+            return false;
-            } else if (std::string_view(arg._path) != p) {
+        } else {
-                *result_listener << "Expected a path that equals \"" << p << "\" but got: " << arg.c_str();
+            auto path = arg.path();
+            if (path.path != CanonPath(p)) {
+                *result_listener << "Expected a path that equals \"" << p << "\" but got: " << path.path;
                 return false;
             }
-            return true;
+        }
+        return true;
     }
 
 

src/libexpr/value.hh

@@ -189,7 +189,12 @@ public:
             const char * c_str;
             const char * * context; // must be in sorted order
         } string;
-        const char * _path;
+
+        struct {
+            InputAccessor * accessor;
+            const char * path;
+        } _path;
+
         Bindings * attrs;
         struct {
             size_t size;
@@ -286,11 +291,12 @@ public:
 
     void mkPath(const SourcePath & path);
 
-    inline void mkPath(const char * path)
+    inline void mkPath(InputAccessor * accessor, const char * path)
     {
         clearValue();
         internalType = tPath;
-        _path = path;
+        _path.accessor = accessor;
+        _path.path = path;
     }
 
     inline void mkNull()
@@ -437,7 +443,10 @@ public:
     SourcePath path() const
     {
         assert(internalType == tPath);
-        return SourcePath{CanonPath(_path)};
+        return SourcePath {
+            .accessor = ref(_path.accessor->shared_from_this()),
+            .path = CanonPath(CanonPath::unchecked_t(), _path.path)
+        };
     }
 
     std::string_view string_view() const

src/libfetchers/fs-input-accessor.cc

@@ -0,0 +1,130 @@
+#include "fs-input-accessor.hh"
+#include "posix-source-accessor.hh"
+#include "store-api.hh"
+
+namespace nix {
+
+struct FSInputAccessorImpl : FSInputAccessor, PosixSourceAccessor
+{
+    CanonPath root;
+    std::optional<std::set<CanonPath>> allowedPaths;
+    MakeNotAllowedError makeNotAllowedError;
+
+    FSInputAccessorImpl(
+        const CanonPath & root,
+        std::optional<std::set<CanonPath>> && allowedPaths,
+        MakeNotAllowedError && makeNotAllowedError)
+        : root(root)
+        , allowedPaths(std::move(allowedPaths))
+        , makeNotAllowedError(std::move(makeNotAllowedError))
+    {
+    }
+
+    void readFile(
+        const CanonPath & path,
+        Sink & sink,
+        std::function<void(uint64_t)> sizeCallback) override
+    {
+        auto absPath = makeAbsPath(path);
+        checkAllowed(absPath);
+        PosixSourceAccessor::readFile(absPath, sink, sizeCallback);
+    }
+
+    bool pathExists(const CanonPath & path) override
+    {
+        auto absPath = makeAbsPath(path);
+        return isAllowed(absPath) && PosixSourceAccessor::pathExists(absPath);
+    }
+
+    Stat lstat(const CanonPath & path) override
+    {
+        auto absPath = makeAbsPath(path);
+        checkAllowed(absPath);
+        return PosixSourceAccessor::lstat(absPath);
+    }
+
+    DirEntries readDirectory(const CanonPath & path) override
+    {
+        auto absPath = makeAbsPath(path);
+        checkAllowed(absPath);
+        DirEntries res;
+        for (auto & entry : PosixSourceAccessor::readDirectory(absPath))
+            if (isAllowed(absPath + entry.first))
+                res.emplace(entry);
+        return res;
+    }
+
+    std::string readLink(const CanonPath & path) override
+    {
+        auto absPath = makeAbsPath(path);
+        checkAllowed(absPath);
+        return PosixSourceAccessor::readLink(absPath);
+    }
+
+    CanonPath makeAbsPath(const CanonPath & path)
+    {
+        return root + path;
+    }
+
+    void checkAllowed(const CanonPath & absPath) override
+    {
+        if (!isAllowed(absPath))
+            throw makeNotAllowedError
+                ? makeNotAllowedError(absPath)
+                : RestrictedPathError("access to path '%s' is forbidden", absPath);
+    }
+
+    bool isAllowed(const CanonPath & absPath)
+    {
+        if (!absPath.isWithin(root))
+            return false;
+
+        if (allowedPaths) {
+            auto p = absPath.removePrefix(root);
+            if (!p.isAllowed(*allowedPaths))
+                return false;
+        }
+
+        return true;
+    }
+
+    void allowPath(CanonPath path) override
+    {
+        if (allowedPaths)
+            allowedPaths->insert(std::move(path));
+    }
+
+    bool hasAccessControl() override
+    {
+        return (bool) allowedPaths;
+    }
+
+    std::optional<CanonPath> getPhysicalPath(const CanonPath & path) override
+    {
+        return makeAbsPath(path);
+    }
+};
+
+ref<FSInputAccessor> makeFSInputAccessor(
+    const CanonPath & root,
+    std::optional<std::set<CanonPath>> && allowedPaths,
+    MakeNotAllowedError && makeNotAllowedError)
+{
+    return make_ref<FSInputAccessorImpl>(root, std::move(allowedPaths), std::move(makeNotAllowedError));
+}
+
+ref<FSInputAccessor> makeStorePathAccessor(
+    ref<Store> store,
+    const StorePath & storePath,
+    MakeNotAllowedError && makeNotAllowedError)
+{
+    return makeFSInputAccessor(CanonPath(store->toRealPath(storePath)), {}, std::move(makeNotAllowedError));
+}
+
+SourcePath getUnfilteredRootPath(CanonPath path)
+{
+    static auto rootFS = makeFSInputAccessor(CanonPath::root);
+    return {rootFS, path};
+}
+
+}

src/libfetchers/fs-input-accessor.hh

@@ -0,0 +1,33 @@
+#pragma once
+
+#include "input-accessor.hh"
+
+namespace nix {
+
+class StorePath;
+class Store;
+
+struct FSInputAccessor : InputAccessor
+{
+    virtual void checkAllowed(const CanonPath & absPath) = 0;
+
+    virtual void allowPath(CanonPath path) = 0;
+
+    virtual bool hasAccessControl() = 0;
+};
+
+typedef std::function<RestrictedPathError(const CanonPath & path)> MakeNotAllowedError;
+
+ref<FSInputAccessor> makeFSInputAccessor(
+    const CanonPath & root,
+    std::optional<std::set<CanonPath>> && allowedPaths = {},
+    MakeNotAllowedError && makeNotAllowedError = {});
+
+ref<FSInputAccessor> makeStorePathAccessor(
+    ref<Store> store,
+    const StorePath & storePath,
+    MakeNotAllowedError && makeNotAllowedError = {});
+
+SourcePath getUnfilteredRootPath(CanonPath path);
+
+}

src/libfetchers/input-accessor.cc

@@ -3,12 +3,52 @@
 
 namespace nix {
 
+StorePath InputAccessor::fetchToStore(
+    ref<Store> store,
+    const CanonPath & path,
+    std::string_view name,
+    FileIngestionMethod method,
+    PathFilter * filter,
+    RepairFlag repair)
+{
+    Activity act(*logger, lvlChatty, actUnknown, fmt("copying '%s' to the store", showPath(path)));
+
+    auto source = sinkToSource([&](Sink & sink) {
+        if (method == FileIngestionMethod::Recursive)
+            dumpPath(path, sink, filter ? *filter : defaultPathFilter);
+        else
+            readFile(path, sink);
+    });
+
+    auto storePath =
+        settings.readOnlyMode
+        ? store->computeStorePathFromDump(*source, name, method, htSHA256).first
+        : store->addToStoreFromDump(*source, name, method, htSHA256, repair);
+
+    return storePath;
+}
+
+SourcePath InputAccessor::root()
+{
+    return {ref(shared_from_this()), CanonPath::root};
+}
+
 std::ostream & operator << (std::ostream & str, const SourcePath & path)
 {
     str << path.to_string();
     return str;
 }
 
+StorePath SourcePath::fetchToStore(
+    ref<Store> store,
+    std::string_view name,
+    FileIngestionMethod method,
+    PathFilter * filter,
+    RepairFlag repair) const
+{
+    return accessor->fetchToStore(store, path, name, method, filter, repair);
+}
+
 std::string_view SourcePath::baseName() const
 {
     return path.baseName().value_or("source");
@@ -18,60 +58,12 @@ SourcePath SourcePath::parent() const
 {
     auto p = path.parent();
     assert(p);
-    return std::move(*p);
+    return {accessor, std::move(*p)};
-}
-
-InputAccessor::Stat SourcePath::lstat() const
-{
-    auto st = nix::lstat(path.abs());
-    return InputAccessor::Stat {
-        .type =
-            S_ISREG(st.st_mode) ? InputAccessor::tRegular :
-            S_ISDIR(st.st_mode) ? InputAccessor::tDirectory :
-            S_ISLNK(st.st_mode) ? InputAccessor::tSymlink :
-            InputAccessor::tMisc,
-        .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR
-    };
-}
-
-std::optional<InputAccessor::Stat> SourcePath::maybeLstat() const
-{
-    // FIXME: merge these into one operation.
-    if (!pathExists())
-        return {};
-    return lstat();
-}
-
-InputAccessor::DirEntries SourcePath::readDirectory() const
-{
-    InputAccessor::DirEntries res;
-    for (auto & entry : nix::readDirectory(path.abs())) {
-        std::optional<InputAccessor::Type> type;
-        switch (entry.type) {
-        case DT_REG: type = InputAccessor::Type::tRegular; break;
-        case DT_LNK: type = InputAccessor::Type::tSymlink; break;
-        case DT_DIR: type = InputAccessor::Type::tDirectory; break;
-        }
-        res.emplace(entry.name, type);
-    }
-    return res;
-}
-
-StorePath SourcePath::fetchToStore(
-    ref<Store> store,
-    std::string_view name,
-    PathFilter * filter,
-    RepairFlag repair) const
-{
-    return
-        settings.readOnlyMode
-        ? store->computeStorePathForPath(name, path.abs(), FileIngestionMethod::Recursive, htSHA256, filter ? *filter : defaultPathFilter).first
-        : store->addToStore(name, path.abs(), FileIngestionMethod::Recursive, htSHA256, filter ? *filter : defaultPathFilter, repair);
 }
 
 SourcePath SourcePath::resolveSymlinks() const
 {
-    SourcePath res(CanonPath::root);
+    auto res = accessor->root();
 
     int linksAllowed = 1024;
 

src/libfetchers/input-accessor.hh

@@ -1,40 +1,39 @@
 #pragma once
 
+#include "source-accessor.hh"
 #include "ref.hh"
 #include "types.hh"
-#include "archive.hh"
-#include "canon-path.hh"
 #include "repair-flag.hh"
+#include "content-address.hh"
 
 namespace nix {
 
+MakeError(RestrictedPathError, Error);
+
+struct SourcePath;
 class StorePath;
 class Store;
 
-struct InputAccessor
+struct InputAccessor : SourceAccessor, std::enable_shared_from_this<InputAccessor>
 {
-    enum Type {
+    /**
-      tRegular, tSymlink, tDirectory,
+     * Return the maximum last-modified time of the files in this
-      /**
+     * tree, if available.
-        Any other node types that may be encountered on the file system, such as device nodes, sockets, named pipe, and possibly even more exotic things.
+     */
-
+    virtual std::optional<time_t> getLastModified()
-        Responsible for `"unknown"` from `builtins.readFileType "/dev/null"`.
-
-        Unlike `DT_UNKNOWN`, this must not be used for deferring the lookup of types.
-      */
-      tMisc
-    };
-
-    struct Stat
     {
-        Type type = tMisc;
+        return std::nullopt;
-        //uint64_t fileSize = 0; // regular files only
+    }
-        bool isExecutable = false; // regular files only
-    };
 
-    typedef std::optional<Type> DirEntry;
+    StorePath fetchToStore(
+        ref<Store> store,
+        const CanonPath & path,
+        std::string_view name = "source",
+        FileIngestionMethod method = FileIngestionMethod::Recursive,
+        PathFilter * filter = nullptr,
+        RepairFlag repair = NoRepair);
 
-    typedef std::map<std::string, DirEntry> DirEntries;
+    SourcePath root();
 };
 
 /**
@@ -45,12 +44,9 @@ struct InputAccessor
  */
 struct SourcePath
 {
+    ref<InputAccessor> accessor;
     CanonPath path;
 
-    SourcePath(CanonPath path)
-        : path(std::move(path))
-    { }
-
     std::string_view baseName() const;
 
     /**
@@ -64,39 +60,42 @@ struct SourcePath
      * return its contents; otherwise throw an error.
      */
     std::string readFile() const
-    { return nix::readFile(path.abs()); }
+    { return accessor->readFile(path); }
 
     /**
      * Return whether this `SourcePath` denotes a file (of any type)
      * that exists
     */
     bool pathExists() const
-    { return nix::pathExists(path.abs()); }
+    { return accessor->pathExists(path); }
 
     /**
      * Return stats about this `SourcePath`, or throw an exception if
      * it doesn't exist.
      */
-    InputAccessor::Stat lstat() const;
+    InputAccessor::Stat lstat() const
+    { return accessor->lstat(path); }
 
     /**
      * Return stats about this `SourcePath`, or std::nullopt if it
      * doesn't exist.
      */
-    std::optional<InputAccessor::Stat> maybeLstat() const;
+    std::optional<InputAccessor::Stat> maybeLstat() const
+    { return accessor->maybeLstat(path); }
 
     /**
      * If this `SourcePath` denotes a directory (not a symlink),
      * return its directory entries; otherwise throw an error.
      */
-    InputAccessor::DirEntries readDirectory() const;
+    InputAccessor::DirEntries readDirectory() const
+    { return accessor->readDirectory(path); }
 
     /**
      * If this `SourcePath` denotes a symlink, return its target;
      * otherwise throw an error.
      */
     std::string readLink() const
-    { return nix::readLink(path.abs()); }
+    { return accessor->readLink(path); }
 
     /**
      * Dump this `SourcePath` to `sink` as a NAR archive.
@@ -104,7 +103,7 @@ struct SourcePath
     void dumpPath(
         Sink & sink,
         PathFilter & filter = defaultPathFilter) const
-    { return nix::dumpPath(path.abs(), sink, filter); }
+    { return accessor->dumpPath(path, sink, filter); }
 
     /**
      * Copy this `SourcePath` to the Nix store.
@@ -112,6 +111,7 @@ struct SourcePath
     StorePath fetchToStore(
         ref<Store> store,
         std::string_view name = "source",
+        FileIngestionMethod method = FileIngestionMethod::Recursive,
         PathFilter * filter = nullptr,
         RepairFlag repair = NoRepair) const;
 
@@ -120,7 +120,7 @@ struct SourcePath
      * it has a physical location.
      */
     std::optional<CanonPath> getPhysicalPath() const
-    { return path; }
+    { return accessor->getPhysicalPath(path); }
 
     std::string to_string() const
     { return path.abs(); }
@@ -129,7 +129,7 @@ struct SourcePath
      * Append a `CanonPath` to this path.
      */
     SourcePath operator + (const CanonPath & x) const
-    { return {path + x}; }
+    { return {accessor, path + x}; }
 
     /**
      * Append a single component `c` to this path. `c` must not
@@ -137,21 +137,21 @@ struct SourcePath
      * and `c`.
      */
     SourcePath operator + (std::string_view c) const
-    {  return {path + c}; }
+    {  return {accessor, path + c}; }
 
     bool operator == (const SourcePath & x) const
     {
-        return path == x.path;
+        return std::tie(accessor, path) == std::tie(x.accessor, x.path);
     }
 
     bool operator != (const SourcePath & x) const
     {
-        return path != x.path;
+        return std::tie(accessor, path) != std::tie(x.accessor, x.path);
     }
 
     bool operator < (const SourcePath & x) const
     {
-        return path < x.path;
+        return std::tie(accessor, path) < std::tie(x.accessor, x.path);
     }
 
     /**

src/libfetchers/memory-input-accessor.cc

@@ -0,0 +1,54 @@
+#include "memory-input-accessor.hh"
+
+namespace nix {
+
+struct MemoryInputAccessorImpl : MemoryInputAccessor
+{
+    std::map<CanonPath, std::string> files;
+
+    std::string readFile(const CanonPath & path) override
+    {
+        auto i = files.find(path);
+        if (i == files.end())
+            throw Error("file '%s' does not exist", path);
+        return i->second;
+    }
+
+    bool pathExists(const CanonPath & path) override
+    {
+        auto i = files.find(path);
+        return i != files.end();
+    }
+
+    Stat lstat(const CanonPath & path) override
+    {
+        auto i = files.find(path);
+        if (i != files.end())
+            return Stat { .type = tRegular, .isExecutable = false };
+        throw Error("file '%s' does not exist", path);
+    }
+
+    DirEntries readDirectory(const CanonPath & path) override
+    {
+        return {};
+    }
+
+    std::string readLink(const CanonPath & path) override
+    {
+        throw UnimplementedError("MemoryInputAccessor::readLink");
+    }
+
+    SourcePath addFile(CanonPath path, std::string && contents) override
+    {
+        files.emplace(path, std::move(contents));
+
+        return {ref(shared_from_this()), std::move(path)};
+    }
+};
+
+ref<MemoryInputAccessor> makeMemoryInputAccessor()
+{
+    return make_ref<MemoryInputAccessorImpl>();
+}
+
+}

src/libfetchers/memory-input-accessor.hh

@@ -0,0 +1,15 @@
+#include "input-accessor.hh"
+
+namespace nix {
+
+/**
+ * An input accessor for an in-memory file system.
+ */
+struct MemoryInputAccessor : InputAccessor
+{
+    virtual SourcePath addFile(CanonPath path, std::string && contents) = 0;
+};
+
+ref<MemoryInputAccessor> makeMemoryInputAccessor();
+
+}

src/libstore/store-api.cc

@@ -225,12 +225,16 @@ StorePath Store::makeFixedOutputPathFromCA(std::string_view name, const ContentA
 }
 
 
-std::pair<StorePath, Hash> Store::computeStorePathForPath(std::string_view name,
+std::pair<StorePath, Hash> Store::computeStorePathFromDump(
-    const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, PathFilter & filter) const
+    Source & dump,
+    std::string_view name,
+    FileIngestionMethod method,
+    HashType hashAlgo,
+    const StorePathSet & references) const
 {
-    Hash h = method == FileIngestionMethod::Recursive
+    HashSink sink(hashAlgo);
-        ? hashPath(hashAlgo, srcPath, filter).first
+    dump.drainInto(sink);
-        : hashFile(hashAlgo, srcPath);
+    auto h = sink.finish().first;
     FixedOutputInfo caInfo {
         .method = method,
         .hash = h,

src/libstore/store-api.hh

@@ -292,14 +292,15 @@ public:
     StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const;
 
     /**
-     * Preparatory part of addToStore().
+     * Read-only variant of addToStoreFromDump(). It returns the store
-     *
+     * path to which a NAR or flat file would be written.
-     * @return the store path to which srcPath is to be copied
-     * and the cryptographic hash of the contents of srcPath.
      */
-    std::pair<StorePath, Hash> computeStorePathForPath(std::string_view name,
+    std::pair<StorePath, Hash> computeStorePathFromDump(
-        const Path & srcPath, FileIngestionMethod method = FileIngestionMethod::Recursive,
+        Source & dump,
-        HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const;
+        std::string_view name,
+        FileIngestionMethod method = FileIngestionMethod::Recursive,
+        HashType hashAlgo = htSHA256,
+        const StorePathSet & references = {}) const;
 
     /**
      * Preparatory part of addTextToStore().

src/libutil/archive.cc

@@ -14,6 +14,7 @@
 #include "archive.hh"
 #include "util.hh"
 #include "config.hh"
+#include "posix-source-accessor.hh"
 
 namespace nix {
 
@@ -36,91 +37,87 @@ static GlobalConfig::Register rArchiveSettings(&archiveSettings);
 PathFilter defaultPathFilter = [](const Path &) { return true; };
 
 
-static void dumpContents(const Path & path, off_t size,
+void SourceAccessor::dumpPath(
-    Sink & sink)
+    const CanonPath & path,
+    Sink & sink,
+    PathFilter & filter)
 {
-    sink << "contents" << size;
+    auto dumpContents = [&](const CanonPath & path)
+    {
+        sink << "contents";
+        std::optional<uint64_t> size;
+        readFile(path, sink, [&](uint64_t _size)
+        {
+            size = _size;
+            sink << _size;
+        });
+        assert(size);
+        writePadding(*size, sink);
+    };
 
-    AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
+    std::function<void(const CanonPath & path)> dump;
-    if (!fd) throw SysError("opening file '%1%'", path);
 
-    std::vector<char> buf(65536);
+    dump = [&](const CanonPath & path) {
-    size_t left = size;
+        checkInterrupt();
 
-    while (left > 0) {
+        auto st = lstat(path);
-        auto n = std::min(left, buf.size());
-        readFull(fd.get(), buf.data(), n);
-        left -= n;
-        sink({buf.data(), n});
-    }
 
-    writePadding(size, sink);
+        sink << "(";
-}
 
+        if (st.type == tRegular) {
+            sink << "type" << "regular";
+            if (st.isExecutable)
+                sink << "executable" << "";
+            dumpContents(path);
+        }
 
-static time_t dump(const Path & path, Sink & sink, PathFilter & filter)
+        else if (st.type == tDirectory) {
-{
+            sink << "type" << "directory";
-    checkInterrupt();
 
-    auto st = lstat(path);
+            /* If we're on a case-insensitive system like macOS, undo
-    time_t result = st.st_mtime;
+               the case hack applied by restorePath(). */
+            std::map<std::string, std::string> unhacked;
+            for (auto & i : readDirectory(path))
+                if (archiveSettings.useCaseHack) {
+                    std::string name(i.first);
+                    size_t pos = i.first.find(caseHackSuffix);
+                    if (pos != std::string::npos) {
+                        debug("removing case hack suffix from '%s'", path + i.first);
+                        name.erase(pos);
+                    }
+                    if (!unhacked.emplace(name, i.first).second)
+                        throw Error("file name collision in between '%s' and '%s'",
+                            (path + unhacked[name]),
+                            (path + i.first));
+                } else
+                    unhacked.emplace(i.first, i.first);
 
-    sink << "(";
+            for (auto & i : unhacked)
-
+                if (filter((path + i.first).abs())) {
-    if (S_ISREG(st.st_mode)) {
+                    sink << "entry" << "(" << "name" << i.first << "node";
-        sink << "type" << "regular";
+                    dump(path + i.second);
-        if (st.st_mode & S_IXUSR)
+                    sink << ")";
-            sink << "executable" << "";
-        dumpContents(path, st.st_size, sink);
-    }
-
-    else if (S_ISDIR(st.st_mode)) {
-        sink << "type" << "directory";
-
-        /* If we're on a case-insensitive system like macOS, undo
-           the case hack applied by restorePath(). */
-        std::map<std::string, std::string> unhacked;
-        for (auto & i : readDirectory(path))
-            if (archiveSettings.useCaseHack) {
-                std::string name(i.name);
-                size_t pos = i.name.find(caseHackSuffix);
-                if (pos != std::string::npos) {
-                    debug("removing case hack suffix from '%1%'", path + "/" + i.name);
-                    name.erase(pos);
                 }
-                if (!unhacked.emplace(name, i.name).second)
+        }
-                    throw Error("file name collision in between '%1%' and '%2%'",
-                       (path + "/" + unhacked[name]),
-                       (path + "/" + i.name));
-            } else
-                unhacked.emplace(i.name, i.name);
 
-        for (auto & i : unhacked)
+        else if (st.type == tSymlink)
-            if (filter(path + "/" + i.first)) {
+            sink << "type" << "symlink" << "target" << readLink(path);
-                sink << "entry" << "(" << "name" << i.first << "node";
-                auto tmp_mtime = dump(path + "/" + i.second, sink, filter);
-                if (tmp_mtime > result) {
-                    result = tmp_mtime;
-                }
-                sink << ")";
-            }
-    }
 
-    else if (S_ISLNK(st.st_mode))
+        else throw Error("file '%s' has an unsupported type", path);
-        sink << "type" << "symlink" << "target" << readLink(path);
 
-    else throw Error("file '%1%' has an unsupported type", path);
+        sink << ")";
+    };
 
-    sink << ")";
+    sink << narVersionMagic1;
-
+    dump(path);
-    return result;
 }
 
 
 time_t dumpPathAndGetMtime(const Path & path, Sink & sink, PathFilter & filter)
 {
-    sink << narVersionMagic1;
+    PosixSourceAccessor accessor;
-    return dump(path, sink, filter);
+    accessor.dumpPath(CanonPath::fromCwd(path), sink, filter);
+    return accessor.mtime;
 }
 
 void dumpPath(const Path & path, Sink & sink, PathFilter & filter)
@@ -141,17 +138,6 @@ static SerialisationError badArchive(const std::string & s)
 }
 
 
-#if 0
-static void skipGeneric(Source & source)
-{
-    if (readString(source) == "(") {
-        while (readString(source) != ")")
-            skipGeneric(source);
-    }
-}
-#endif
-
-
 static void parseContents(ParseSink & sink, Source & source, const Path & path)
 {
     uint64_t size = readLongLong(source);

src/libutil/posix-source-accessor.cc

@@ -0,0 +1,86 @@
+#include "posix-source-accessor.hh"
+
+namespace nix {
+
+void PosixSourceAccessor::readFile(
+    const CanonPath & path,
+    Sink & sink,
+    std::function<void(uint64_t)> sizeCallback)
+{
+    // FIXME: add O_NOFOLLOW since symlinks should be resolved by the
+    // caller?
+    AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
+    if (!fd)
+        throw SysError("opening file '%1%'", path);
+
+    struct stat st;
+    if (fstat(fd.get(), &st) == -1)
+        throw SysError("statting file");
+
+    sizeCallback(st.st_size);
+
+    off_t left = st.st_size;
+
+    std::vector<unsigned char> buf(64 * 1024);
+    while (left) {
+        checkInterrupt();
+        ssize_t rd = read(fd.get(), buf.data(), (size_t) std::min(left, (off_t) buf.size()));
+        if (rd == -1) {
+            if (errno != EINTR)
+                throw SysError("reading from file '%s'", showPath(path));
+        }
+        else if (rd == 0)
+            throw SysError("unexpected end-of-file reading '%s'", showPath(path));
+        else {
+            assert(rd <= left);
+            sink({(char *) buf.data(), (size_t) rd});
+            left -= rd;
+        }
+    }
+}
+
+bool PosixSourceAccessor::pathExists(const CanonPath & path)
+{
+    return nix::pathExists(path.abs());
+}
+
+SourceAccessor::Stat PosixSourceAccessor::lstat(const CanonPath & path)
+{
+    auto st = nix::lstat(path.abs());
+    mtime = std::max(mtime, st.st_mtime);
+    return Stat {
+        .type =
+            S_ISREG(st.st_mode) ? tRegular :
+            S_ISDIR(st.st_mode) ? tDirectory :
+            S_ISLNK(st.st_mode) ? tSymlink :
+            tMisc,
+        .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR
+    };
+}
+
+SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath & path)
+{
+    DirEntries res;
+    for (auto & entry : nix::readDirectory(path.abs())) {
+        std::optional<Type> type;
+        switch (entry.type) {
+        case DT_REG: type = Type::tRegular; break;
+        case DT_LNK: type = Type::tSymlink; break;
+        case DT_DIR: type = Type::tDirectory; break;
+        }
+        res.emplace(entry.name, type);
+    }
+    return res;
+}
+
+std::string PosixSourceAccessor::readLink(const CanonPath & path)
+{
+    return nix::readLink(path.abs());
+}
+
+std::optional<CanonPath> PosixSourceAccessor::getPhysicalPath(const CanonPath & path)
+{
+    return path;
+}
+
+}

src/libutil/posix-source-accessor.hh

@@ -0,0 +1,34 @@
+#pragma once
+
+#include "source-accessor.hh"
+
+namespace nix {
+
+/**
+ * A source accessor that uses the Unix filesystem.
+ */
+struct PosixSourceAccessor : SourceAccessor
+{
+    /**
+     * The most recent mtime seen by lstat(). This is a hack to
+     * support dumpPathAndGetMtime(). Should remove this eventually.
+     */
+    time_t mtime = 0;
+
+    void readFile(
+        const CanonPath & path,
+        Sink & sink,
+        std::function<void(uint64_t)> sizeCallback) override;
+
+    bool pathExists(const CanonPath & path) override;
+
+    Stat lstat(const CanonPath & path) override;
+
+    DirEntries readDirectory(const CanonPath & path) override;
+
+    std::string readLink(const CanonPath & path) override;
+
+    std::optional<CanonPath> getPhysicalPath(const CanonPath & path) override;
+};
+
+}

src/libutil/source-accessor.cc

@@ -0,0 +1,58 @@
+#include "source-accessor.hh"
+#include "archive.hh"
+
+namespace nix {
+
+static std::atomic<size_t> nextNumber{0};
+
+SourceAccessor::SourceAccessor()
+    : number(++nextNumber)
+{
+}
+
+std::string SourceAccessor::readFile(const CanonPath & path)
+{
+    StringSink sink;
+    std::optional<uint64_t> size;
+    readFile(path, sink, [&](uint64_t _size)
+    {
+        size = _size;
+    });
+    assert(size && *size == sink.s.size());
+    return std::move(sink.s);
+}
+
+void SourceAccessor::readFile(
+    const CanonPath & path,
+    Sink & sink,
+    std::function<void(uint64_t)> sizeCallback)
+{
+    auto s = readFile(path);
+    sizeCallback(s.size());
+    sink(s);
+}
+
+Hash SourceAccessor::hashPath(
+    const CanonPath & path,
+    PathFilter & filter,
+    HashType ht)
+{
+    HashSink sink(ht);
+    dumpPath(path, sink, filter);
+    return sink.finish().first;
+}
+
+std::optional<SourceAccessor::Stat> SourceAccessor::maybeLstat(const CanonPath & path)
+{
+    // FIXME: merge these into one operation.
+    if (!pathExists(path))
+        return {};
+    return lstat(path);
+}
+
+std::string SourceAccessor::showPath(const CanonPath & path)
+{
+    return path.abs();
+}
+
+}

src/libutil/source-accessor.hh

@@ -0,0 +1,107 @@
+#pragma once
+
+#include "canon-path.hh"
+#include "hash.hh"
+
+namespace nix {
+
+struct Sink;
+
+/**
+ * A read-only filesystem abstraction. This is used by the Nix
+ * evaluator and elsewhere for accessing sources in various
+ * filesystem-like entities (such as the real filesystem, tarballs or
+ * Git repositories).
+ */
+struct SourceAccessor
+{
+    const size_t number;
+
+    SourceAccessor();
+
+    virtual ~SourceAccessor()
+    { }
+
+    /**
+     * Return the contents of a file as a string.
+     */
+    virtual std::string readFile(const CanonPath & path);
+
+    /**
+     * Write the contents of a file as a sink. `sizeCallback` must be
+     * called with the size of the file before any data is written to
+     * the sink.
+     *
+     * Note: subclasses of `SourceAccessor` need to implement at least
+     * one of the `readFile()` variants.
+     */
+    virtual void readFile(
+        const CanonPath & path,
+        Sink & sink,
+        std::function<void(uint64_t)> sizeCallback = [](uint64_t size){});
+
+    virtual bool pathExists(const CanonPath & path) = 0;
+
+    enum Type {
+      tRegular, tSymlink, tDirectory,
+      /**
+        Any other node types that may be encountered on the file system, such as device nodes, sockets, named pipe, and possibly even more exotic things.
+
+        Responsible for `"unknown"` from `builtins.readFileType "/dev/null"`.
+
+        Unlike `DT_UNKNOWN`, this must not be used for deferring the lookup of types.
+      */
+      tMisc
+    };
+
+    struct Stat
+    {
+        Type type = tMisc;
+        //uint64_t fileSize = 0; // regular files only
+        bool isExecutable = false; // regular files only
+    };
+
+    virtual Stat lstat(const CanonPath & path) = 0;
+
+    std::optional<Stat> maybeLstat(const CanonPath & path);
+
+    typedef std::optional<Type> DirEntry;
+
+    typedef std::map<std::string, DirEntry> DirEntries;
+
+    virtual DirEntries readDirectory(const CanonPath & path) = 0;
+
+    virtual std::string readLink(const CanonPath & path) = 0;
+
+    virtual void dumpPath(
+        const CanonPath & path,
+        Sink & sink,
+        PathFilter & filter = defaultPathFilter);
+
+    Hash hashPath(
+        const CanonPath & path,
+        PathFilter & filter = defaultPathFilter,
+        HashType ht = htSHA256);
+
+    /**
+     * Return a corresponding path in the root filesystem, if
+     * possible. This is only possible for filesystems that are
+     * materialized in the root filesystem.
+     */
+    virtual std::optional<CanonPath> getPhysicalPath(const CanonPath & path)
+    { return std::nullopt; }
+
+    bool operator == (const SourceAccessor & x) const
+    {
+        return number == x.number;
+    }
+
+    bool operator < (const SourceAccessor & x) const
+    {
+        return number < x.number;
+    }
+
+    virtual std::string showPath(const CanonPath & path);
+};
+
+}

src/nix/main.cc

@@ -12,6 +12,7 @@
 #include "finally.hh"
 #include "loggers.hh"
 #include "markdown.hh"
+#include "memory-input-accessor.hh"
 
 #include <sys/types.h>
 #include <sys/socket.h>
@@ -204,31 +205,22 @@ static void showHelp(std::vector<std::string> subcommand, NixArgs & toplevel)
     auto vGenerateManpage = state.allocValue();
     state.eval(state.parseExprFromString(
         #include "generate-manpage.nix.gen.hh"
-        , CanonPath::root), *vGenerateManpage);
+        , state.rootPath(CanonPath::root)), *vGenerateManpage);
 
-    auto vUtils = state.allocValue();
+    state.corepkgsFS->addFile(
-    state.cacheFile(
+        CanonPath("utils.nix"),
-        CanonPath("/utils.nix"), CanonPath("/utils.nix"),
+        #include "utils.nix.gen.hh"
-        state.parseExprFromString(
+        );
-            #include "utils.nix.gen.hh"
-            , CanonPath::root),
-        *vUtils);
 
-    auto vSettingsInfo = state.allocValue();
+    state.corepkgsFS->addFile(
-    state.cacheFile(
+        CanonPath("/generate-settings.nix"),
-        CanonPath("/generate-settings.nix"), CanonPath("/generate-settings.nix"),
+        #include "generate-settings.nix.gen.hh"
-        state.parseExprFromString(
+        );
-            #include "generate-settings.nix.gen.hh"
-            , CanonPath::root),
-        *vSettingsInfo);
 
-    auto vStoreInfo = state.allocValue();
+    state.corepkgsFS->addFile(
-    state.cacheFile(
+        CanonPath("/generate-store-info.nix"),
-        CanonPath("/generate-store-info.nix"), CanonPath("/generate-store-info.nix"),
+        #include "generate-store-info.nix.gen.hh"
-        state.parseExprFromString(
+        );
-            #include "generate-store-info.nix.gen.hh"
-            , CanonPath::root),
-        *vStoreInfo);
 
     auto vDump = state.allocValue();
     vDump->mkString(toplevel.dumpCli());

tests/functional/lang/eval-okay-path.exp

@@ -1 +1 @@
-"/nix/store/ya937r4ydw0l6kayq8jkyqaips9c75jm-output"
+[ "/nix/store/ya937r4ydw0l6kayq8jkyqaips9c75jm-output" "/nix/store/m7y372g6jb0g4hh1dzmj847rd356fhnz-output" ]

tests/functional/lang/eval-okay-path.nix

@@ -1,7 +1,15 @@
-builtins.path
+[
-  { path = ./.;
+  (builtins.path
-    filter = path: _: baseNameOf path == "data";
+    { path = ./.;
-    recursive = true;
+      filter = path: _: baseNameOf path == "data";
-    sha256 = "1yhm3gwvg5a41yylymgblsclk95fs6jy72w0wv925mmidlhcq4sw";
+      recursive = true;
-    name = "output";
+      sha256 = "1yhm3gwvg5a41yylymgblsclk95fs6jy72w0wv925mmidlhcq4sw";
-  }
+      name = "output";
+    })
+  (builtins.path
+    { path = ./data;
+      recursive = false;
+      sha256 = "0k4lwj58f2w5yh92ilrwy9917pycipbrdrr13vbb3yd02j09vfxm";
+      name = "output";
+    })
+]

tests/functional/lang/eval-okay-pathexists.nix

@@ -25,5 +25,7 @@ builtins.pathExists (./lib.nix)
 && builtins.pathExists (builtins.toString ./. + "/../lang/..//")
 && builtins.pathExists (builtins.toPath (builtins.toString ./lib.nix))
 && !builtins.pathExists (builtins.toPath (builtins.toString ./bla.nix))
+&& builtins.pathExists (builtins.toPath { __toString = x: builtins.toString ./lib.nix; })
+&& builtins.pathExists (builtins.toPath { outPath = builtins.toString ./lib.nix; })
 && builtins.pathExists ./lib.nix
 && !builtins.pathExists ./bla.nix