Merge pull request #9177 from edolstra/input-accessors

Backport FSInputAccessor and MemoryInputAccessor from lazy-trees
This commit is contained in:
Eelco Dolstra 2023-10-23 11:42:04 +02:00 committed by GitHub
commit 955bbe53c5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
31 changed files with 861 additions and 312 deletions

View file

@ -2,7 +2,7 @@ let
inherit (builtins) inherit (builtins)
attrNames attrValues fromJSON listToAttrs mapAttrs groupBy attrNames attrValues fromJSON listToAttrs mapAttrs groupBy
concatStringsSep concatMap length lessThan replaceStrings sort; concatStringsSep concatMap length lessThan replaceStrings sort;
inherit (import ./utils.nix) attrsToList concatStrings optionalString filterAttrs trim squash unique; inherit (import <nix/utils.nix>) attrsToList concatStrings optionalString filterAttrs trim squash unique;
showStoreDocs = import ./generate-store-info.nix; showStoreDocs = import ./generate-store-info.nix;
in in

View file

@ -32,7 +32,7 @@ dummy-env = env -i \
NIX_STATE_DIR=/dummy \ NIX_STATE_DIR=/dummy \
NIX_CONFIG='cores = 0' NIX_CONFIG='cores = 0'
nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix/corepkgs=corepkgs --store dummy:// --impure --raw nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix=doc/manual --store dummy:// --impure --raw
# re-implement mdBook's include directive to make it usable for terminal output and for proper @docroot@ substitution # re-implement mdBook's include directive to make it usable for terminal output and for proper @docroot@ substitution
define process-includes define process-includes
@ -125,7 +125,7 @@ $(d)/src/command-ref/experimental-features-shortlist.md: $(d)/xp-features.json $
@mv $@.tmp $@ @mv $@.tmp $@
$(d)/xp-features.json: $(bindir)/nix $(d)/xp-features.json: $(bindir)/nix
$(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-xp-features > $@.tmp $(trace-gen) $(dummy-env) $(bindir)/nix __dump-xp-features > $@.tmp
@mv $@.tmp $@ @mv $@.tmp $@
$(d)/src/language/builtins.md: $(d)/language.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix $(d)/src/language/builtins.md: $(d)/language.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix
@ -141,7 +141,7 @@ $(d)/src/language/builtin-constants.md: $(d)/language.json $(d)/generate-builtin
@mv $@.tmp $@ @mv $@.tmp $@
$(d)/language.json: $(bindir)/nix $(d)/language.json: $(bindir)/nix
$(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-language > $@.tmp $(trace-gen) $(dummy-env) $(bindir)/nix __dump-language > $@.tmp
@mv $@.tmp $@ @mv $@.tmp $@
# Generate the HTML manual. # Generate the HTML manual.

View file

@ -132,7 +132,7 @@ std::pair<SourcePath, uint32_t> findPackageFilename(EvalState & state, Value & v
if (colon == std::string::npos) fail(); if (colon == std::string::npos) fail();
std::string filename(fn, 0, colon); std::string filename(fn, 0, colon);
auto lineno = std::stoi(std::string(fn, colon + 1, std::string::npos)); auto lineno = std::stoi(std::string(fn, colon + 1, std::string::npos));
return {CanonPath(fn.substr(0, colon)), lineno}; return {SourcePath{path.accessor, CanonPath(fn.substr(0, colon))}, lineno};
} catch (std::invalid_argument & e) { } catch (std::invalid_argument & e) {
fail(); fail();
abort(); abort();

View file

@ -12,6 +12,8 @@
#include "function-trace.hh" #include "function-trace.hh"
#include "profiles.hh" #include "profiles.hh"
#include "print.hh" #include "print.hh"
#include "fs-input-accessor.hh"
#include "memory-input-accessor.hh"
#include <algorithm> #include <algorithm>
#include <chrono> #include <chrono>
@ -503,7 +505,17 @@ EvalState::EvalState(
, sOutputSpecified(symbols.create("outputSpecified")) , sOutputSpecified(symbols.create("outputSpecified"))
, repair(NoRepair) , repair(NoRepair)
, emptyBindings(0) , emptyBindings(0)
, derivationInternal(rootPath(CanonPath("/builtin/derivation.nix"))) , rootFS(makeFSInputAccessor(CanonPath::root))
, corepkgsFS(makeMemoryInputAccessor())
, internalFS(makeMemoryInputAccessor())
, derivationInternal{corepkgsFS->addFile(
CanonPath("derivation-internal.nix"),
#include "primops/derivation.nix.gen.hh"
)}
, callFlakeInternal{internalFS->addFile(
CanonPath("call-flake.nix"),
#include "flake/call-flake.nix.gen.hh"
)}
, store(store) , store(store)
, buildStore(buildStore ? buildStore : store) , buildStore(buildStore ? buildStore : store)
, debugRepl(nullptr) , debugRepl(nullptr)
@ -555,6 +567,11 @@ EvalState::EvalState(
} }
} }
corepkgsFS->addFile(
CanonPath("fetchurl.nix"),
#include "fetchurl.nix.gen.hh"
);
createBaseEnv(); createBaseEnv();
} }
@ -585,6 +602,9 @@ void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value &
SourcePath EvalState::checkSourcePath(const SourcePath & path_) SourcePath EvalState::checkSourcePath(const SourcePath & path_)
{ {
// Don't check non-rootFS accessors, they're in a different namespace.
if (path_.accessor != ref<InputAccessor>(rootFS)) return path_;
if (!allowedPaths) return path_; if (!allowedPaths) return path_;
auto i = resolvedPaths.find(path_.path.abs()); auto i = resolvedPaths.find(path_.path.abs());
@ -599,8 +619,6 @@ SourcePath EvalState::checkSourcePath(const SourcePath & path_)
*/ */
Path abspath = canonPath(path_.path.abs()); Path abspath = canonPath(path_.path.abs());
if (hasPrefix(abspath, corepkgsPrefix)) return CanonPath(abspath);
for (auto & i : *allowedPaths) { for (auto & i : *allowedPaths) {
if (isDirOrInDir(abspath, i)) { if (isDirOrInDir(abspath, i)) {
found = true; found = true;
@ -617,7 +635,7 @@ SourcePath EvalState::checkSourcePath(const SourcePath & path_)
/* Resolve symlinks. */ /* Resolve symlinks. */
debug("checking access to '%s'", abspath); debug("checking access to '%s'", abspath);
SourcePath path = CanonPath(canonPath(abspath, true)); SourcePath path = rootPath(CanonPath(canonPath(abspath, true)));
for (auto & i : *allowedPaths) { for (auto & i : *allowedPaths) {
if (isDirOrInDir(path.path.abs(), i)) { if (isDirOrInDir(path.path.abs(), i)) {
@ -649,12 +667,12 @@ void EvalState::checkURI(const std::string & uri)
/* If the URI is a path, then check it against allowedPaths as /* If the URI is a path, then check it against allowedPaths as
well. */ well. */
if (hasPrefix(uri, "/")) { if (hasPrefix(uri, "/")) {
checkSourcePath(CanonPath(uri)); checkSourcePath(rootPath(CanonPath(uri)));
return; return;
} }
if (hasPrefix(uri, "file://")) { if (hasPrefix(uri, "file://")) {
checkSourcePath(CanonPath(std::string(uri, 7))); checkSourcePath(rootPath(CanonPath(std::string(uri, 7))));
return; return;
} }
@ -950,7 +968,7 @@ void Value::mkStringMove(const char * s, const NixStringContext & context)
void Value::mkPath(const SourcePath & path) void Value::mkPath(const SourcePath & path)
{ {
mkPath(makeImmutableString(path.path.abs())); mkPath(&*path.accessor, makeImmutableString(path.path.abs()));
} }
@ -1165,24 +1183,6 @@ void EvalState::evalFile(const SourcePath & path_, Value & v, bool mustBeTrivial
if (!e) if (!e)
e = parseExprFromFile(checkSourcePath(resolvedPath)); e = parseExprFromFile(checkSourcePath(resolvedPath));
cacheFile(path, resolvedPath, e, v, mustBeTrivial);
}
void EvalState::resetFileCache()
{
fileEvalCache.clear();
fileParseCache.clear();
}
void EvalState::cacheFile(
const SourcePath & path,
const SourcePath & resolvedPath,
Expr * e,
Value & v,
bool mustBeTrivial)
{
fileParseCache[resolvedPath] = e; fileParseCache[resolvedPath] = e;
try { try {
@ -1211,6 +1211,13 @@ void EvalState::cacheFile(
} }
void EvalState::resetFileCache()
{
fileEvalCache.clear();
fileParseCache.clear();
}
void EvalState::eval(Expr * e, Value & v) void EvalState::eval(Expr * e, Value & v)
{ {
e->eval(*this, baseEnv, v); e->eval(*this, baseEnv, v);
@ -2037,7 +2044,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
else if (firstType == nPath) { else if (firstType == nPath) {
if (!context.empty()) if (!context.empty())
state.error("a string that refers to a store path cannot be appended to a path").atPos(pos).withFrame(env, *this).debugThrow<EvalError>(); state.error("a string that refers to a store path cannot be appended to a path").atPos(pos).withFrame(env, *this).debugThrow<EvalError>();
v.mkPath(CanonPath(canonPath(str()))); v.mkPath(state.rootPath(CanonPath(canonPath(str()))));
} else } else
v.mkStringMove(c_str(), context); v.mkStringMove(c_str(), context);
} }
@ -2236,7 +2243,7 @@ BackedStringView EvalState::coerceToString(
!canonicalizePath && !copyToStore !canonicalizePath && !copyToStore
? // FIXME: hack to preserve path literals that end in a ? // FIXME: hack to preserve path literals that end in a
// slash, as in /foo/${x}. // slash, as in /foo/${x}.
v._path v._path.path
: copyToStore : copyToStore
? store->printStorePath(copyPathToStore(context, v.path())) ? store->printStorePath(copyPathToStore(context, v.path()))
: std::string(v.path().path.abs()); : std::string(v.path().path.abs());
@ -2310,7 +2317,7 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat
auto dstPath = i != srcToStore.end() auto dstPath = i != srcToStore.end()
? i->second ? i->second
: [&]() { : [&]() {
auto dstPath = path.fetchToStore(store, path.baseName(), nullptr, repair); auto dstPath = path.fetchToStore(store, path.baseName(), FileIngestionMethod::Recursive, nullptr, repair);
allowPath(dstPath); allowPath(dstPath);
srcToStore.insert_or_assign(path, dstPath); srcToStore.insert_or_assign(path, dstPath);
printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath)); printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath));
@ -2326,10 +2333,34 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat
SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx) SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx)
{ {
try {
forceValue(v, pos);
} catch (Error & e) {
e.addTrace(positions[pos], errorCtx);
throw;
}
/* Handle path values directly, without coercing to a string. */
if (v.type() == nPath)
return v.path();
/* Similarly, handle __toString where the result may be a path
value. */
if (v.type() == nAttrs) {
auto i = v.attrs->find(sToString);
if (i != v.attrs->end()) {
Value v1;
callFunction(*i->value, v, v1, pos);
return coerceToPath(pos, v1, context, errorCtx);
}
}
/* Any other value should be coercable to a string, interpreted
relative to the root filesystem. */
auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned(); auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned();
if (path == "" || path[0] != '/') if (path == "" || path[0] != '/')
error("string '%1%' doesn't represent an absolute path", path).withTrace(pos, errorCtx).debugThrow<EvalError>(); error("string '%1%' doesn't represent an absolute path", path).withTrace(pos, errorCtx).debugThrow<EvalError>();
return CanonPath(path); return rootPath(CanonPath(path));
} }
@ -2429,7 +2460,10 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v
return v1.string_view().compare(v2.string_view()) == 0; return v1.string_view().compare(v2.string_view()) == 0;
case nPath: case nPath:
return strcmp(v1._path, v2._path) == 0; return
// FIXME: compare accessors by their fingerprint.
v1._path.accessor == v2._path.accessor
&& strcmp(v1._path.path, v2._path.path) == 0;
case nNull: case nNull:
return true; return true;

View file

@ -24,6 +24,8 @@ class EvalState;
class StorePath; class StorePath;
struct SingleDerivedPath; struct SingleDerivedPath;
enum RepairFlag : bool; enum RepairFlag : bool;
struct FSInputAccessor;
struct MemoryInputAccessor;
/** /**
@ -211,8 +213,26 @@ public:
Bindings emptyBindings; Bindings emptyBindings;
/**
* The accessor for the root filesystem.
*/
const ref<FSInputAccessor> rootFS;
/**
* The in-memory filesystem for <nix/...> paths.
*/
const ref<MemoryInputAccessor> corepkgsFS;
/**
* In-memory filesystem for internal, non-user-callable Nix
* expressions like call-flake.nix.
*/
const ref<MemoryInputAccessor> internalFS;
const SourcePath derivationInternal; const SourcePath derivationInternal;
const SourcePath callFlakeInternal;
/** /**
* Store used to materialise .drv files. * Store used to materialise .drv files.
*/ */
@ -223,7 +243,6 @@ public:
*/ */
const ref<Store> buildStore; const ref<Store> buildStore;
RootValue vCallFlake = nullptr;
RootValue vImportedDrvToDerivation = nullptr; RootValue vImportedDrvToDerivation = nullptr;
/** /**
@ -405,16 +424,6 @@ public:
*/ */
void evalFile(const SourcePath & path, Value & v, bool mustBeTrivial = false); void evalFile(const SourcePath & path, Value & v, bool mustBeTrivial = false);
/**
* Like `evalFile`, but with an already parsed expression.
*/
void cacheFile(
const SourcePath & path,
const SourcePath & resolvedPath,
Expr * e,
Value & v,
bool mustBeTrivial = false);
void resetFileCache(); void resetFileCache();
/** /**
@ -424,7 +433,7 @@ public:
SourcePath findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos); SourcePath findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos);
/** /**
* Try to resolve a search path value (not the optinal key part) * Try to resolve a search path value (not the optional key part)
* *
* If the specified search path element is a URI, download it. * If the specified search path element is a URI, download it.
* *
@ -829,8 +838,6 @@ struct InvalidPathError : EvalError
#endif #endif
}; };
static const std::string corepkgsPrefix{"/__corepkgs__/"};
template<class ErrorType> template<class ErrorType>
void ErrorBuilder::debugThrow() void ErrorBuilder::debugThrow()
{ {

View file

@ -223,9 +223,9 @@ static Flake getFlake(
throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", lockedRef, lockedRef.subdir); throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", lockedRef, lockedRef.subdir);
Value vInfo; Value vInfo;
state.evalFile(CanonPath(flakeFile), vInfo, true); // FIXME: symlink attack state.evalFile(state.rootPath(CanonPath(flakeFile)), vInfo, true); // FIXME: symlink attack
expectType(state, nAttrs, vInfo, state.positions.add({CanonPath(flakeFile)}, 1, 1)); expectType(state, nAttrs, vInfo, state.positions.add({state.rootPath(CanonPath(flakeFile))}, 1, 1));
if (auto description = vInfo.attrs->get(state.sDescription)) { if (auto description = vInfo.attrs->get(state.sDescription)) {
expectType(state, nString, *description->value, description->pos); expectType(state, nString, *description->value, description->pos);
@ -737,14 +737,10 @@ void callFlake(EvalState & state,
vRootSubdir->mkString(lockedFlake.flake.lockedRef.subdir); vRootSubdir->mkString(lockedFlake.flake.lockedRef.subdir);
if (!state.vCallFlake) { auto vCallFlake = state.allocValue();
state.vCallFlake = allocRootValue(state.allocValue()); state.evalFile(state.callFlakeInternal, *vCallFlake);
state.eval(state.parseExprFromString(
#include "call-flake.nix.gen.hh"
, CanonPath::root), **state.vCallFlake);
}
state.callFunction(**state.vCallFlake, *vLocks, *vTmp1, noPos); state.callFunction(*vCallFlake, *vLocks, *vTmp1, noPos);
state.callFunction(*vTmp1, *vRootSrc, *vTmp2, noPos); state.callFunction(*vTmp1, *vRootSrc, *vTmp2, noPos);
state.callFunction(*vTmp2, *vRootSubdir, vRes, noPos); state.callFunction(*vTmp2, *vRootSubdir, vRes, noPos);
} }

View file

@ -20,7 +20,6 @@ MakeError(Abort, EvalError);
MakeError(TypeError, EvalError); MakeError(TypeError, EvalError);
MakeError(UndefinedVarError, Error); MakeError(UndefinedVarError, Error);
MakeError(MissingArgumentError, EvalError); MakeError(MissingArgumentError, EvalError);
MakeError(RestrictedPathError, Error);
/** /**
* Position objects. * Position objects.
@ -200,9 +199,13 @@ struct ExprString : Expr
struct ExprPath : Expr struct ExprPath : Expr
{ {
ref<InputAccessor> accessor;
std::string s; std::string s;
Value v; Value v;
ExprPath(std::string s) : s(std::move(s)) { v.mkPath(this->s.c_str()); }; ExprPath(ref<InputAccessor> accessor, std::string s) : accessor(accessor), s(std::move(s))
{
v.mkPath(&*accessor, this->s.c_str());
}
Value * maybeThunk(EvalState & state, Env & env) override; Value * maybeThunk(EvalState & state, Env & env) override;
COMMON_METHODS COMMON_METHODS
}; };

View file

@ -520,7 +520,7 @@ path_start
/* add back in the trailing '/' to the first segment */ /* add back in the trailing '/' to the first segment */
if ($1.p[$1.l-1] == '/' && $1.l > 1) if ($1.p[$1.l-1] == '/' && $1.l > 1)
path += "/"; path += "/";
$$ = new ExprPath(std::move(path)); $$ = new ExprPath(ref<InputAccessor>(data->state.rootFS), std::move(path));
} }
| HPATH { | HPATH {
if (evalSettings.pureEval) { if (evalSettings.pureEval) {
@ -530,7 +530,7 @@ path_start
); );
} }
Path path(getHome() + std::string($1.p + 1, $1.l - 1)); Path path(getHome() + std::string($1.p + 1, $1.l - 1));
$$ = new ExprPath(std::move(path)); $$ = new ExprPath(ref<InputAccessor>(data->state.rootFS), std::move(path));
} }
; ;
@ -649,6 +649,8 @@ formal
#include "tarball.hh" #include "tarball.hh"
#include "store-api.hh" #include "store-api.hh"
#include "flake/flake.hh" #include "flake/flake.hh"
#include "fs-input-accessor.hh"
#include "memory-input-accessor.hh"
namespace nix { namespace nix {
@ -756,11 +758,11 @@ SourcePath EvalState::findFile(const SearchPath & searchPath, const std::string_
auto r = *rOpt; auto r = *rOpt;
Path res = suffix == "" ? r : concatStrings(r, "/", suffix); Path res = suffix == "" ? r : concatStrings(r, "/", suffix);
if (pathExists(res)) return CanonPath(canonPath(res)); if (pathExists(res)) return rootPath(CanonPath(canonPath(res)));
} }
if (hasPrefix(path, "nix/")) if (hasPrefix(path, "nix/"))
return CanonPath(concatStrings(corepkgsPrefix, path.substr(4))); return {corepkgsFS, CanonPath(path.substr(3))};
debugThrow(ThrownError({ debugThrow(ThrownError({
.msg = hintfmt(evalSettings.pureEval .msg = hintfmt(evalSettings.pureEval

View file

@ -1,10 +1,11 @@
#include "eval.hh" #include "eval.hh"
#include "fs-input-accessor.hh"
namespace nix { namespace nix {
SourcePath EvalState::rootPath(CanonPath path) SourcePath EvalState::rootPath(CanonPath path)
{ {
return path; return {rootFS, std::move(path)};
} }
} }

View file

@ -121,13 +121,15 @@ static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v, co
auto path = state.coerceToPath(noPos, v, context, "while realising the context of a path"); auto path = state.coerceToPath(noPos, v, context, "while realising the context of a path");
try { try {
StringMap rewrites = state.realiseContext(context); if (!context.empty()) {
auto rewrites = state.realiseContext(context);
auto realPath = state.rootPath(CanonPath(state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context))); auto realPath = state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context);
return {path.accessor, CanonPath(realPath)};
}
return flags.checkForPureEval return flags.checkForPureEval
? state.checkSourcePath(realPath) ? state.checkSourcePath(path)
: realPath; : path;
} catch (Error & e) { } catch (Error & e) {
e.addTrace(state.positions[pos], "while realising the context of path '%s'", path); e.addTrace(state.positions[pos], "while realising the context of path '%s'", path);
throw; throw;
@ -202,7 +204,7 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v
state.vImportedDrvToDerivation = allocRootValue(state.allocValue()); state.vImportedDrvToDerivation = allocRootValue(state.allocValue());
state.eval(state.parseExprFromString( state.eval(state.parseExprFromString(
#include "imported-drv-to-derivation.nix.gen.hh" #include "imported-drv-to-derivation.nix.gen.hh"
, CanonPath::root), **state.vImportedDrvToDerivation); , state.rootPath(CanonPath::root)), **state.vImportedDrvToDerivation);
} }
state.forceFunction(**state.vImportedDrvToDerivation, pos, "while evaluating imported-drv-to-derivation.nix.gen.hh"); state.forceFunction(**state.vImportedDrvToDerivation, pos, "while evaluating imported-drv-to-derivation.nix.gen.hh");
@ -210,12 +212,6 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v
state.forceAttrs(v, pos, "while calling imported-drv-to-derivation.nix.gen.hh"); state.forceAttrs(v, pos, "while calling imported-drv-to-derivation.nix.gen.hh");
} }
else if (path2 == corepkgsPrefix + "fetchurl.nix") {
state.eval(state.parseExprFromString(
#include "fetchurl.nix.gen.hh"
, CanonPath::root), v);
}
else { else {
if (!vScope) if (!vScope)
state.evalFile(path, v); state.evalFile(path, v);
@ -599,7 +595,10 @@ struct CompareValues
case nString: case nString:
return v1->string_view().compare(v2->string_view()) < 0; return v1->string_view().compare(v2->string_view()) < 0;
case nPath: case nPath:
return strcmp(v1->_path, v2->_path) < 0; // Note: we don't take the accessor into account
// since it's not obvious how to compare them in a
// reproducible way.
return strcmp(v1->_path.path, v2->_path.path) < 0;
case nList: case nList:
// Lexicographic comparison // Lexicographic comparison
for (size_t i = 0;; i++) { for (size_t i = 0;; i++) {
@ -1493,7 +1492,7 @@ static void prim_storePath(EvalState & state, const PosIdx pos, Value * * args,
})); }));
NixStringContext context; NixStringContext context;
auto path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to builtins.storePath")).path; auto path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to 'builtins.storePath'")).path;
/* Resolve symlinks in path, unless path itself is a symlink /* Resolve symlinks in path, unless path itself is a symlink
directly in the store. The latter condition is necessary so directly in the store. The latter condition is necessary so
e.g. nix-push does the right thing. */ e.g. nix-push does the right thing. */
@ -2211,7 +2210,7 @@ static void addPath(
path = evalSettings.pureEval && expectedHash path = evalSettings.pureEval && expectedHash
? path ? path
: state.checkSourcePath(CanonPath(path)).path.abs(); : state.checkSourcePath(state.rootPath(CanonPath(path))).path.abs();
PathFilter filter = filterFun ? ([&](const Path & path) { PathFilter filter = filterFun ? ([&](const Path & path) {
auto st = lstat(path); auto st = lstat(path);
@ -2244,9 +2243,7 @@ static void addPath(
}); });
if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
StorePath dstPath = settings.readOnlyMode auto dstPath = state.rootPath(CanonPath(path)).fetchToStore(state.store, name, method, &filter, state.repair);
? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first
: state.store->addToStore(name, path, method, htSHA256, filter, state.repair, refs);
if (expectedHash && expectedStorePath != dstPath) if (expectedHash && expectedStorePath != dstPath)
state.debugThrowLastTrace(Error("store path mismatch in (possibly filtered) path added from '%s'", path)); state.debugThrowLastTrace(Error("store path mismatch in (possibly filtered) path added from '%s'", path));
state.allowAndSetStorePathString(dstPath, v); state.allowAndSetStorePathString(dstPath, v);
@ -2263,7 +2260,7 @@ static void prim_filterSource(EvalState & state, const PosIdx pos, Value * * arg
{ {
NixStringContext context; NixStringContext context;
auto path = state.coerceToPath(pos, *args[1], context, auto path = state.coerceToPath(pos, *args[1], context,
"while evaluating the second argument (the path to filter) passed to builtins.filterSource"); "while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'");
state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filterSource"); state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filterSource");
addPath(state, pos, path.baseName(), path.path.abs(), args[0], FileIngestionMethod::Recursive, std::nullopt, v, context); addPath(state, pos, path.baseName(), path.path.abs(), args[0], FileIngestionMethod::Recursive, std::nullopt, v, context);
} }
@ -4545,12 +4542,7 @@ void EvalState::createBaseEnv()
/* Note: we have to initialize the 'derivation' constant *after* /* Note: we have to initialize the 'derivation' constant *after*
building baseEnv/staticBaseEnv because it uses 'builtins'. */ building baseEnv/staticBaseEnv because it uses 'builtins'. */
char code[] = evalFile(derivationInternal, *vDerivation);
#include "primops/derivation.nix.gen.hh"
// the parser needs two NUL bytes as terminators; one of them
// is implied by being a C string.
"\0";
eval(parse(code, sizeof(code), derivationInternal, {CanonPath::root}, staticBaseEnv), *vDerivation);
} }

View file

@ -310,7 +310,7 @@ namespace nix {
ASSERT_TRACE2("storePath true", ASSERT_TRACE2("storePath true",
TypeError, TypeError,
hintfmt("cannot coerce %s to a string", "a Boolean"), hintfmt("cannot coerce %s to a string", "a Boolean"),
hintfmt("while evaluating the first argument passed to builtins.storePath")); hintfmt("while evaluating the first argument passed to 'builtins.storePath'"));
} }
@ -378,12 +378,12 @@ namespace nix {
ASSERT_TRACE2("filterSource [] []", ASSERT_TRACE2("filterSource [] []",
TypeError, TypeError,
hintfmt("cannot coerce %s to a string", "a list"), hintfmt("cannot coerce %s to a string", "a list"),
hintfmt("while evaluating the second argument (the path to filter) passed to builtins.filterSource")); hintfmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'"));
ASSERT_TRACE2("filterSource [] \"foo\"", ASSERT_TRACE2("filterSource [] \"foo\"",
EvalError, EvalError,
hintfmt("string '%s' doesn't represent an absolute path", "foo"), hintfmt("string '%s' doesn't represent an absolute path", "foo"),
hintfmt("while evaluating the second argument (the path to filter) passed to builtins.filterSource")); hintfmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'"));
ASSERT_TRACE2("filterSource [] ./.", ASSERT_TRACE2("filterSource [] ./.",
TypeError, TypeError,

View file

@ -62,7 +62,7 @@ namespace nix {
// not supported by store 'dummy'" thrown in the test body. // not supported by store 'dummy'" thrown in the test body.
TEST_F(JSONValueTest, DISABLED_Path) { TEST_F(JSONValueTest, DISABLED_Path) {
Value v; Value v;
v.mkPath("test"); v.mkPath(state.rootPath(CanonPath("/test")));
ASSERT_EQ(getJSONValue(v), "\"/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x\""); ASSERT_EQ(getJSONValue(v), "\"/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x\"");
} }
} /* namespace nix */ } /* namespace nix */

View file

@ -106,10 +106,13 @@ namespace nix {
if (arg.type() != nPath) { if (arg.type() != nPath) {
*result_listener << "Expected a path got " << arg.type(); *result_listener << "Expected a path got " << arg.type();
return false; return false;
} else if (std::string_view(arg._path) != p) { } else {
*result_listener << "Expected a path that equals \"" << p << "\" but got: " << arg.c_str(); auto path = arg.path();
if (path.path != CanonPath(p)) {
*result_listener << "Expected a path that equals \"" << p << "\" but got: " << path.path;
return false; return false;
} }
}
return true; return true;
} }

View file

@ -189,7 +189,12 @@ public:
const char * c_str; const char * c_str;
const char * * context; // must be in sorted order const char * * context; // must be in sorted order
} string; } string;
const char * _path;
struct {
InputAccessor * accessor;
const char * path;
} _path;
Bindings * attrs; Bindings * attrs;
struct { struct {
size_t size; size_t size;
@ -286,11 +291,12 @@ public:
void mkPath(const SourcePath & path); void mkPath(const SourcePath & path);
inline void mkPath(const char * path) inline void mkPath(InputAccessor * accessor, const char * path)
{ {
clearValue(); clearValue();
internalType = tPath; internalType = tPath;
_path = path; _path.accessor = accessor;
_path.path = path;
} }
inline void mkNull() inline void mkNull()
@ -437,7 +443,10 @@ public:
SourcePath path() const SourcePath path() const
{ {
assert(internalType == tPath); assert(internalType == tPath);
return SourcePath{CanonPath(_path)}; return SourcePath {
.accessor = ref(_path.accessor->shared_from_this()),
.path = CanonPath(CanonPath::unchecked_t(), _path.path)
};
} }
std::string_view string_view() const std::string_view string_view() const

View file

@ -0,0 +1,130 @@
#include "fs-input-accessor.hh"
#include "posix-source-accessor.hh"
#include "store-api.hh"
namespace nix {
struct FSInputAccessorImpl : FSInputAccessor, PosixSourceAccessor
{
CanonPath root;
std::optional<std::set<CanonPath>> allowedPaths;
MakeNotAllowedError makeNotAllowedError;
FSInputAccessorImpl(
const CanonPath & root,
std::optional<std::set<CanonPath>> && allowedPaths,
MakeNotAllowedError && makeNotAllowedError)
: root(root)
, allowedPaths(std::move(allowedPaths))
, makeNotAllowedError(std::move(makeNotAllowedError))
{
}
void readFile(
const CanonPath & path,
Sink & sink,
std::function<void(uint64_t)> sizeCallback) override
{
auto absPath = makeAbsPath(path);
checkAllowed(absPath);
PosixSourceAccessor::readFile(absPath, sink, sizeCallback);
}
bool pathExists(const CanonPath & path) override
{
auto absPath = makeAbsPath(path);
return isAllowed(absPath) && PosixSourceAccessor::pathExists(absPath);
}
Stat lstat(const CanonPath & path) override
{
auto absPath = makeAbsPath(path);
checkAllowed(absPath);
return PosixSourceAccessor::lstat(absPath);
}
DirEntries readDirectory(const CanonPath & path) override
{
auto absPath = makeAbsPath(path);
checkAllowed(absPath);
DirEntries res;
for (auto & entry : PosixSourceAccessor::readDirectory(absPath))
if (isAllowed(absPath + entry.first))
res.emplace(entry);
return res;
}
std::string readLink(const CanonPath & path) override
{
auto absPath = makeAbsPath(path);
checkAllowed(absPath);
return PosixSourceAccessor::readLink(absPath);
}
CanonPath makeAbsPath(const CanonPath & path)
{
return root + path;
}
void checkAllowed(const CanonPath & absPath) override
{
if (!isAllowed(absPath))
throw makeNotAllowedError
? makeNotAllowedError(absPath)
: RestrictedPathError("access to path '%s' is forbidden", absPath);
}
bool isAllowed(const CanonPath & absPath)
{
if (!absPath.isWithin(root))
return false;
if (allowedPaths) {
auto p = absPath.removePrefix(root);
if (!p.isAllowed(*allowedPaths))
return false;
}
return true;
}
void allowPath(CanonPath path) override
{
if (allowedPaths)
allowedPaths->insert(std::move(path));
}
bool hasAccessControl() override
{
return (bool) allowedPaths;
}
std::optional<CanonPath> getPhysicalPath(const CanonPath & path) override
{
return makeAbsPath(path);
}
};
ref<FSInputAccessor> makeFSInputAccessor(
const CanonPath & root,
std::optional<std::set<CanonPath>> && allowedPaths,
MakeNotAllowedError && makeNotAllowedError)
{
return make_ref<FSInputAccessorImpl>(root, std::move(allowedPaths), std::move(makeNotAllowedError));
}
ref<FSInputAccessor> makeStorePathAccessor(
ref<Store> store,
const StorePath & storePath,
MakeNotAllowedError && makeNotAllowedError)
{
return makeFSInputAccessor(CanonPath(store->toRealPath(storePath)), {}, std::move(makeNotAllowedError));
}
SourcePath getUnfilteredRootPath(CanonPath path)
{
static auto rootFS = makeFSInputAccessor(CanonPath::root);
return {rootFS, path};
}
}

View file

@ -0,0 +1,33 @@
#pragma once
#include "input-accessor.hh"
namespace nix {
class StorePath;
class Store;
struct FSInputAccessor : InputAccessor
{
virtual void checkAllowed(const CanonPath & absPath) = 0;
virtual void allowPath(CanonPath path) = 0;
virtual bool hasAccessControl() = 0;
};
typedef std::function<RestrictedPathError(const CanonPath & path)> MakeNotAllowedError;
ref<FSInputAccessor> makeFSInputAccessor(
const CanonPath & root,
std::optional<std::set<CanonPath>> && allowedPaths = {},
MakeNotAllowedError && makeNotAllowedError = {});
ref<FSInputAccessor> makeStorePathAccessor(
ref<Store> store,
const StorePath & storePath,
MakeNotAllowedError && makeNotAllowedError = {});
SourcePath getUnfilteredRootPath(CanonPath path);
}

View file

@ -3,12 +3,52 @@
namespace nix { namespace nix {
StorePath InputAccessor::fetchToStore(
ref<Store> store,
const CanonPath & path,
std::string_view name,
FileIngestionMethod method,
PathFilter * filter,
RepairFlag repair)
{
Activity act(*logger, lvlChatty, actUnknown, fmt("copying '%s' to the store", showPath(path)));
auto source = sinkToSource([&](Sink & sink) {
if (method == FileIngestionMethod::Recursive)
dumpPath(path, sink, filter ? *filter : defaultPathFilter);
else
readFile(path, sink);
});
auto storePath =
settings.readOnlyMode
? store->computeStorePathFromDump(*source, name, method, htSHA256).first
: store->addToStoreFromDump(*source, name, method, htSHA256, repair);
return storePath;
}
SourcePath InputAccessor::root()
{
return {ref(shared_from_this()), CanonPath::root};
}
std::ostream & operator << (std::ostream & str, const SourcePath & path) std::ostream & operator << (std::ostream & str, const SourcePath & path)
{ {
str << path.to_string(); str << path.to_string();
return str; return str;
} }
StorePath SourcePath::fetchToStore(
ref<Store> store,
std::string_view name,
FileIngestionMethod method,
PathFilter * filter,
RepairFlag repair) const
{
return accessor->fetchToStore(store, path, name, method, filter, repair);
}
std::string_view SourcePath::baseName() const std::string_view SourcePath::baseName() const
{ {
return path.baseName().value_or("source"); return path.baseName().value_or("source");
@ -18,60 +58,12 @@ SourcePath SourcePath::parent() const
{ {
auto p = path.parent(); auto p = path.parent();
assert(p); assert(p);
return std::move(*p); return {accessor, std::move(*p)};
}
InputAccessor::Stat SourcePath::lstat() const
{
auto st = nix::lstat(path.abs());
return InputAccessor::Stat {
.type =
S_ISREG(st.st_mode) ? InputAccessor::tRegular :
S_ISDIR(st.st_mode) ? InputAccessor::tDirectory :
S_ISLNK(st.st_mode) ? InputAccessor::tSymlink :
InputAccessor::tMisc,
.isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR
};
}
std::optional<InputAccessor::Stat> SourcePath::maybeLstat() const
{
// FIXME: merge these into one operation.
if (!pathExists())
return {};
return lstat();
}
InputAccessor::DirEntries SourcePath::readDirectory() const
{
InputAccessor::DirEntries res;
for (auto & entry : nix::readDirectory(path.abs())) {
std::optional<InputAccessor::Type> type;
switch (entry.type) {
case DT_REG: type = InputAccessor::Type::tRegular; break;
case DT_LNK: type = InputAccessor::Type::tSymlink; break;
case DT_DIR: type = InputAccessor::Type::tDirectory; break;
}
res.emplace(entry.name, type);
}
return res;
}
StorePath SourcePath::fetchToStore(
ref<Store> store,
std::string_view name,
PathFilter * filter,
RepairFlag repair) const
{
return
settings.readOnlyMode
? store->computeStorePathForPath(name, path.abs(), FileIngestionMethod::Recursive, htSHA256, filter ? *filter : defaultPathFilter).first
: store->addToStore(name, path.abs(), FileIngestionMethod::Recursive, htSHA256, filter ? *filter : defaultPathFilter, repair);
} }
SourcePath SourcePath::resolveSymlinks() const SourcePath SourcePath::resolveSymlinks() const
{ {
SourcePath res(CanonPath::root); auto res = accessor->root();
int linksAllowed = 1024; int linksAllowed = 1024;

View file

@ -1,40 +1,39 @@
#pragma once #pragma once
#include "source-accessor.hh"
#include "ref.hh" #include "ref.hh"
#include "types.hh" #include "types.hh"
#include "archive.hh"
#include "canon-path.hh"
#include "repair-flag.hh" #include "repair-flag.hh"
#include "content-address.hh"
namespace nix { namespace nix {
MakeError(RestrictedPathError, Error);
struct SourcePath;
class StorePath; class StorePath;
class Store; class Store;
struct InputAccessor struct InputAccessor : SourceAccessor, std::enable_shared_from_this<InputAccessor>
{ {
enum Type {
tRegular, tSymlink, tDirectory,
/** /**
Any other node types that may be encountered on the file system, such as device nodes, sockets, named pipe, and possibly even more exotic things. * Return the maximum last-modified time of the files in this
* tree, if available.
Responsible for `"unknown"` from `builtins.readFileType "/dev/null"`.
Unlike `DT_UNKNOWN`, this must not be used for deferring the lookup of types.
*/ */
tMisc virtual std::optional<time_t> getLastModified()
};
struct Stat
{ {
Type type = tMisc; return std::nullopt;
//uint64_t fileSize = 0; // regular files only }
bool isExecutable = false; // regular files only
};
typedef std::optional<Type> DirEntry; StorePath fetchToStore(
ref<Store> store,
const CanonPath & path,
std::string_view name = "source",
FileIngestionMethod method = FileIngestionMethod::Recursive,
PathFilter * filter = nullptr,
RepairFlag repair = NoRepair);
typedef std::map<std::string, DirEntry> DirEntries; SourcePath root();
}; };
/** /**
@ -45,12 +44,9 @@ struct InputAccessor
*/ */
struct SourcePath struct SourcePath
{ {
ref<InputAccessor> accessor;
CanonPath path; CanonPath path;
SourcePath(CanonPath path)
: path(std::move(path))
{ }
std::string_view baseName() const; std::string_view baseName() const;
/** /**
@ -64,39 +60,42 @@ struct SourcePath
* return its contents; otherwise throw an error. * return its contents; otherwise throw an error.
*/ */
std::string readFile() const std::string readFile() const
{ return nix::readFile(path.abs()); } { return accessor->readFile(path); }
/** /**
* Return whether this `SourcePath` denotes a file (of any type) * Return whether this `SourcePath` denotes a file (of any type)
* that exists * that exists
*/ */
bool pathExists() const bool pathExists() const
{ return nix::pathExists(path.abs()); } { return accessor->pathExists(path); }
/** /**
* Return stats about this `SourcePath`, or throw an exception if * Return stats about this `SourcePath`, or throw an exception if
* it doesn't exist. * it doesn't exist.
*/ */
InputAccessor::Stat lstat() const; InputAccessor::Stat lstat() const
{ return accessor->lstat(path); }
/** /**
* Return stats about this `SourcePath`, or std::nullopt if it * Return stats about this `SourcePath`, or std::nullopt if it
* doesn't exist. * doesn't exist.
*/ */
std::optional<InputAccessor::Stat> maybeLstat() const; std::optional<InputAccessor::Stat> maybeLstat() const
{ return accessor->maybeLstat(path); }
/** /**
* If this `SourcePath` denotes a directory (not a symlink), * If this `SourcePath` denotes a directory (not a symlink),
* return its directory entries; otherwise throw an error. * return its directory entries; otherwise throw an error.
*/ */
InputAccessor::DirEntries readDirectory() const; InputAccessor::DirEntries readDirectory() const
{ return accessor->readDirectory(path); }
/** /**
* If this `SourcePath` denotes a symlink, return its target; * If this `SourcePath` denotes a symlink, return its target;
* otherwise throw an error. * otherwise throw an error.
*/ */
std::string readLink() const std::string readLink() const
{ return nix::readLink(path.abs()); } { return accessor->readLink(path); }
/** /**
* Dump this `SourcePath` to `sink` as a NAR archive. * Dump this `SourcePath` to `sink` as a NAR archive.
@ -104,7 +103,7 @@ struct SourcePath
void dumpPath( void dumpPath(
Sink & sink, Sink & sink,
PathFilter & filter = defaultPathFilter) const PathFilter & filter = defaultPathFilter) const
{ return nix::dumpPath(path.abs(), sink, filter); } { return accessor->dumpPath(path, sink, filter); }
/** /**
* Copy this `SourcePath` to the Nix store. * Copy this `SourcePath` to the Nix store.
@ -112,6 +111,7 @@ struct SourcePath
StorePath fetchToStore( StorePath fetchToStore(
ref<Store> store, ref<Store> store,
std::string_view name = "source", std::string_view name = "source",
FileIngestionMethod method = FileIngestionMethod::Recursive,
PathFilter * filter = nullptr, PathFilter * filter = nullptr,
RepairFlag repair = NoRepair) const; RepairFlag repair = NoRepair) const;
@ -120,7 +120,7 @@ struct SourcePath
* it has a physical location. * it has a physical location.
*/ */
std::optional<CanonPath> getPhysicalPath() const std::optional<CanonPath> getPhysicalPath() const
{ return path; } { return accessor->getPhysicalPath(path); }
std::string to_string() const std::string to_string() const
{ return path.abs(); } { return path.abs(); }
@ -129,7 +129,7 @@ struct SourcePath
* Append a `CanonPath` to this path. * Append a `CanonPath` to this path.
*/ */
SourcePath operator + (const CanonPath & x) const SourcePath operator + (const CanonPath & x) const
{ return {path + x}; } { return {accessor, path + x}; }
/** /**
* Append a single component `c` to this path. `c` must not * Append a single component `c` to this path. `c` must not
@ -137,21 +137,21 @@ struct SourcePath
* and `c`. * and `c`.
*/ */
SourcePath operator + (std::string_view c) const SourcePath operator + (std::string_view c) const
{ return {path + c}; } { return {accessor, path + c}; }
bool operator == (const SourcePath & x) const bool operator == (const SourcePath & x) const
{ {
return path == x.path; return std::tie(accessor, path) == std::tie(x.accessor, x.path);
} }
bool operator != (const SourcePath & x) const bool operator != (const SourcePath & x) const
{ {
return path != x.path; return std::tie(accessor, path) != std::tie(x.accessor, x.path);
} }
bool operator < (const SourcePath & x) const bool operator < (const SourcePath & x) const
{ {
return path < x.path; return std::tie(accessor, path) < std::tie(x.accessor, x.path);
} }
/** /**

View file

@ -0,0 +1,54 @@
#include "memory-input-accessor.hh"
namespace nix {
struct MemoryInputAccessorImpl : MemoryInputAccessor
{
std::map<CanonPath, std::string> files;
std::string readFile(const CanonPath & path) override
{
auto i = files.find(path);
if (i == files.end())
throw Error("file '%s' does not exist", path);
return i->second;
}
bool pathExists(const CanonPath & path) override
{
auto i = files.find(path);
return i != files.end();
}
Stat lstat(const CanonPath & path) override
{
auto i = files.find(path);
if (i != files.end())
return Stat { .type = tRegular, .isExecutable = false };
throw Error("file '%s' does not exist", path);
}
DirEntries readDirectory(const CanonPath & path) override
{
return {};
}
std::string readLink(const CanonPath & path) override
{
throw UnimplementedError("MemoryInputAccessor::readLink");
}
SourcePath addFile(CanonPath path, std::string && contents) override
{
files.emplace(path, std::move(contents));
return {ref(shared_from_this()), std::move(path)};
}
};
ref<MemoryInputAccessor> makeMemoryInputAccessor()
{
return make_ref<MemoryInputAccessorImpl>();
}
}

View file

@ -0,0 +1,15 @@
#include "input-accessor.hh"
namespace nix {
/**
* An input accessor for an in-memory file system.
*/
struct MemoryInputAccessor : InputAccessor
{
virtual SourcePath addFile(CanonPath path, std::string && contents) = 0;
};
ref<MemoryInputAccessor> makeMemoryInputAccessor();
}

View file

@ -225,12 +225,16 @@ StorePath Store::makeFixedOutputPathFromCA(std::string_view name, const ContentA
} }
std::pair<StorePath, Hash> Store::computeStorePathForPath(std::string_view name, std::pair<StorePath, Hash> Store::computeStorePathFromDump(
const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, PathFilter & filter) const Source & dump,
std::string_view name,
FileIngestionMethod method,
HashType hashAlgo,
const StorePathSet & references) const
{ {
Hash h = method == FileIngestionMethod::Recursive HashSink sink(hashAlgo);
? hashPath(hashAlgo, srcPath, filter).first dump.drainInto(sink);
: hashFile(hashAlgo, srcPath); auto h = sink.finish().first;
FixedOutputInfo caInfo { FixedOutputInfo caInfo {
.method = method, .method = method,
.hash = h, .hash = h,

View file

@ -292,14 +292,15 @@ public:
StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const; StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const;
/** /**
* Preparatory part of addToStore(). * Read-only variant of addToStoreFromDump(). It returns the store
* * path to which a NAR or flat file would be written.
* @return the store path to which srcPath is to be copied
* and the cryptographic hash of the contents of srcPath.
*/ */
std::pair<StorePath, Hash> computeStorePathForPath(std::string_view name, std::pair<StorePath, Hash> computeStorePathFromDump(
const Path & srcPath, FileIngestionMethod method = FileIngestionMethod::Recursive, Source & dump,
HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const; std::string_view name,
FileIngestionMethod method = FileIngestionMethod::Recursive,
HashType hashAlgo = htSHA256,
const StorePathSet & references = {}) const;
/** /**
* Preparatory part of addTextToStore(). * Preparatory part of addTextToStore().

View file

@ -14,6 +14,7 @@
#include "archive.hh" #include "archive.hh"
#include "util.hh" #include "util.hh"
#include "config.hh" #include "config.hh"
#include "posix-source-accessor.hh"
namespace nix { namespace nix {
@ -36,45 +37,41 @@ static GlobalConfig::Register rArchiveSettings(&archiveSettings);
PathFilter defaultPathFilter = [](const Path &) { return true; }; PathFilter defaultPathFilter = [](const Path &) { return true; };
static void dumpContents(const Path & path, off_t size, void SourceAccessor::dumpPath(
Sink & sink) const CanonPath & path,
Sink & sink,
PathFilter & filter)
{ {
sink << "contents" << size; auto dumpContents = [&](const CanonPath & path)
{
sink << "contents";
std::optional<uint64_t> size;
readFile(path, sink, [&](uint64_t _size)
{
size = _size;
sink << _size;
});
assert(size);
writePadding(*size, sink);
};
AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); std::function<void(const CanonPath & path)> dump;
if (!fd) throw SysError("opening file '%1%'", path);
std::vector<char> buf(65536); dump = [&](const CanonPath & path) {
size_t left = size;
while (left > 0) {
auto n = std::min(left, buf.size());
readFull(fd.get(), buf.data(), n);
left -= n;
sink({buf.data(), n});
}
writePadding(size, sink);
}
static time_t dump(const Path & path, Sink & sink, PathFilter & filter)
{
checkInterrupt(); checkInterrupt();
auto st = lstat(path); auto st = lstat(path);
time_t result = st.st_mtime;
sink << "("; sink << "(";
if (S_ISREG(st.st_mode)) { if (st.type == tRegular) {
sink << "type" << "regular"; sink << "type" << "regular";
if (st.st_mode & S_IXUSR) if (st.isExecutable)
sink << "executable" << ""; sink << "executable" << "";
dumpContents(path, st.st_size, sink); dumpContents(path);
} }
else if (S_ISDIR(st.st_mode)) { else if (st.type == tDirectory) {
sink << "type" << "directory"; sink << "type" << "directory";
/* If we're on a case-insensitive system like macOS, undo /* If we're on a case-insensitive system like macOS, undo
@ -82,45 +79,45 @@ static time_t dump(const Path & path, Sink & sink, PathFilter & filter)
std::map<std::string, std::string> unhacked; std::map<std::string, std::string> unhacked;
for (auto & i : readDirectory(path)) for (auto & i : readDirectory(path))
if (archiveSettings.useCaseHack) { if (archiveSettings.useCaseHack) {
std::string name(i.name); std::string name(i.first);
size_t pos = i.name.find(caseHackSuffix); size_t pos = i.first.find(caseHackSuffix);
if (pos != std::string::npos) { if (pos != std::string::npos) {
debug("removing case hack suffix from '%1%'", path + "/" + i.name); debug("removing case hack suffix from '%s'", path + i.first);
name.erase(pos); name.erase(pos);
} }
if (!unhacked.emplace(name, i.name).second) if (!unhacked.emplace(name, i.first).second)
throw Error("file name collision in between '%1%' and '%2%'", throw Error("file name collision in between '%s' and '%s'",
(path + "/" + unhacked[name]), (path + unhacked[name]),
(path + "/" + i.name)); (path + i.first));
} else } else
unhacked.emplace(i.name, i.name); unhacked.emplace(i.first, i.first);
for (auto & i : unhacked) for (auto & i : unhacked)
if (filter(path + "/" + i.first)) { if (filter((path + i.first).abs())) {
sink << "entry" << "(" << "name" << i.first << "node"; sink << "entry" << "(" << "name" << i.first << "node";
auto tmp_mtime = dump(path + "/" + i.second, sink, filter); dump(path + i.second);
if (tmp_mtime > result) {
result = tmp_mtime;
}
sink << ")"; sink << ")";
} }
} }
else if (S_ISLNK(st.st_mode)) else if (st.type == tSymlink)
sink << "type" << "symlink" << "target" << readLink(path); sink << "type" << "symlink" << "target" << readLink(path);
else throw Error("file '%1%' has an unsupported type", path); else throw Error("file '%s' has an unsupported type", path);
sink << ")"; sink << ")";
};
return result; sink << narVersionMagic1;
dump(path);
} }
time_t dumpPathAndGetMtime(const Path & path, Sink & sink, PathFilter & filter) time_t dumpPathAndGetMtime(const Path & path, Sink & sink, PathFilter & filter)
{ {
sink << narVersionMagic1; PosixSourceAccessor accessor;
return dump(path, sink, filter); accessor.dumpPath(CanonPath::fromCwd(path), sink, filter);
return accessor.mtime;
} }
void dumpPath(const Path & path, Sink & sink, PathFilter & filter) void dumpPath(const Path & path, Sink & sink, PathFilter & filter)
@ -141,17 +138,6 @@ static SerialisationError badArchive(const std::string & s)
} }
#if 0
static void skipGeneric(Source & source)
{
if (readString(source) == "(") {
while (readString(source) != ")")
skipGeneric(source);
}
}
#endif
static void parseContents(ParseSink & sink, Source & source, const Path & path) static void parseContents(ParseSink & sink, Source & source, const Path & path)
{ {
uint64_t size = readLongLong(source); uint64_t size = readLongLong(source);

View file

@ -0,0 +1,86 @@
#include "posix-source-accessor.hh"
namespace nix {
void PosixSourceAccessor::readFile(
const CanonPath & path,
Sink & sink,
std::function<void(uint64_t)> sizeCallback)
{
// FIXME: add O_NOFOLLOW since symlinks should be resolved by the
// caller?
AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
if (!fd)
throw SysError("opening file '%1%'", path);
struct stat st;
if (fstat(fd.get(), &st) == -1)
throw SysError("statting file");
sizeCallback(st.st_size);
off_t left = st.st_size;
std::vector<unsigned char> buf(64 * 1024);
while (left) {
checkInterrupt();
ssize_t rd = read(fd.get(), buf.data(), (size_t) std::min(left, (off_t) buf.size()));
if (rd == -1) {
if (errno != EINTR)
throw SysError("reading from file '%s'", showPath(path));
}
else if (rd == 0)
throw SysError("unexpected end-of-file reading '%s'", showPath(path));
else {
assert(rd <= left);
sink({(char *) buf.data(), (size_t) rd});
left -= rd;
}
}
}
bool PosixSourceAccessor::pathExists(const CanonPath & path)
{
return nix::pathExists(path.abs());
}
SourceAccessor::Stat PosixSourceAccessor::lstat(const CanonPath & path)
{
auto st = nix::lstat(path.abs());
mtime = std::max(mtime, st.st_mtime);
return Stat {
.type =
S_ISREG(st.st_mode) ? tRegular :
S_ISDIR(st.st_mode) ? tDirectory :
S_ISLNK(st.st_mode) ? tSymlink :
tMisc,
.isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR
};
}
SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath & path)
{
DirEntries res;
for (auto & entry : nix::readDirectory(path.abs())) {
std::optional<Type> type;
switch (entry.type) {
case DT_REG: type = Type::tRegular; break;
case DT_LNK: type = Type::tSymlink; break;
case DT_DIR: type = Type::tDirectory; break;
}
res.emplace(entry.name, type);
}
return res;
}
std::string PosixSourceAccessor::readLink(const CanonPath & path)
{
return nix::readLink(path.abs());
}
std::optional<CanonPath> PosixSourceAccessor::getPhysicalPath(const CanonPath & path)
{
return path;
}
}

View file

@ -0,0 +1,34 @@
#pragma once
#include "source-accessor.hh"
namespace nix {
/**
* A source accessor that uses the Unix filesystem.
*/
struct PosixSourceAccessor : SourceAccessor
{
/**
* The most recent mtime seen by lstat(). This is a hack to
* support dumpPathAndGetMtime(). Should remove this eventually.
*/
time_t mtime = 0;
void readFile(
const CanonPath & path,
Sink & sink,
std::function<void(uint64_t)> sizeCallback) override;
bool pathExists(const CanonPath & path) override;
Stat lstat(const CanonPath & path) override;
DirEntries readDirectory(const CanonPath & path) override;
std::string readLink(const CanonPath & path) override;
std::optional<CanonPath> getPhysicalPath(const CanonPath & path) override;
};
}

View file

@ -0,0 +1,58 @@
#include "source-accessor.hh"
#include "archive.hh"
namespace nix {
static std::atomic<size_t> nextNumber{0};
SourceAccessor::SourceAccessor()
: number(++nextNumber)
{
}
std::string SourceAccessor::readFile(const CanonPath & path)
{
StringSink sink;
std::optional<uint64_t> size;
readFile(path, sink, [&](uint64_t _size)
{
size = _size;
});
assert(size && *size == sink.s.size());
return std::move(sink.s);
}
void SourceAccessor::readFile(
const CanonPath & path,
Sink & sink,
std::function<void(uint64_t)> sizeCallback)
{
auto s = readFile(path);
sizeCallback(s.size());
sink(s);
}
Hash SourceAccessor::hashPath(
const CanonPath & path,
PathFilter & filter,
HashType ht)
{
HashSink sink(ht);
dumpPath(path, sink, filter);
return sink.finish().first;
}
std::optional<SourceAccessor::Stat> SourceAccessor::maybeLstat(const CanonPath & path)
{
// FIXME: merge these into one operation.
if (!pathExists(path))
return {};
return lstat(path);
}
std::string SourceAccessor::showPath(const CanonPath & path)
{
return path.abs();
}
}

View file

@ -0,0 +1,107 @@
#pragma once
#include "canon-path.hh"
#include "hash.hh"
namespace nix {
struct Sink;
/**
* A read-only filesystem abstraction. This is used by the Nix
* evaluator and elsewhere for accessing sources in various
* filesystem-like entities (such as the real filesystem, tarballs or
* Git repositories).
*/
struct SourceAccessor
{
const size_t number;
SourceAccessor();
virtual ~SourceAccessor()
{ }
/**
* Return the contents of a file as a string.
*/
virtual std::string readFile(const CanonPath & path);
/**
* Write the contents of a file as a sink. `sizeCallback` must be
* called with the size of the file before any data is written to
* the sink.
*
* Note: subclasses of `SourceAccessor` need to implement at least
* one of the `readFile()` variants.
*/
virtual void readFile(
const CanonPath & path,
Sink & sink,
std::function<void(uint64_t)> sizeCallback = [](uint64_t size){});
virtual bool pathExists(const CanonPath & path) = 0;
enum Type {
tRegular, tSymlink, tDirectory,
/**
Any other node types that may be encountered on the file system, such as device nodes, sockets, named pipe, and possibly even more exotic things.
Responsible for `"unknown"` from `builtins.readFileType "/dev/null"`.
Unlike `DT_UNKNOWN`, this must not be used for deferring the lookup of types.
*/
tMisc
};
struct Stat
{
Type type = tMisc;
//uint64_t fileSize = 0; // regular files only
bool isExecutable = false; // regular files only
};
virtual Stat lstat(const CanonPath & path) = 0;
std::optional<Stat> maybeLstat(const CanonPath & path);
typedef std::optional<Type> DirEntry;
typedef std::map<std::string, DirEntry> DirEntries;
virtual DirEntries readDirectory(const CanonPath & path) = 0;
virtual std::string readLink(const CanonPath & path) = 0;
virtual void dumpPath(
const CanonPath & path,
Sink & sink,
PathFilter & filter = defaultPathFilter);
Hash hashPath(
const CanonPath & path,
PathFilter & filter = defaultPathFilter,
HashType ht = htSHA256);
/**
* Return a corresponding path in the root filesystem, if
* possible. This is only possible for filesystems that are
* materialized in the root filesystem.
*/
virtual std::optional<CanonPath> getPhysicalPath(const CanonPath & path)
{ return std::nullopt; }
bool operator == (const SourceAccessor & x) const
{
return number == x.number;
}
bool operator < (const SourceAccessor & x) const
{
return number < x.number;
}
virtual std::string showPath(const CanonPath & path);
};
}

View file

@ -12,6 +12,7 @@
#include "finally.hh" #include "finally.hh"
#include "loggers.hh" #include "loggers.hh"
#include "markdown.hh" #include "markdown.hh"
#include "memory-input-accessor.hh"
#include <sys/types.h> #include <sys/types.h>
#include <sys/socket.h> #include <sys/socket.h>
@ -204,31 +205,22 @@ static void showHelp(std::vector<std::string> subcommand, NixArgs & toplevel)
auto vGenerateManpage = state.allocValue(); auto vGenerateManpage = state.allocValue();
state.eval(state.parseExprFromString( state.eval(state.parseExprFromString(
#include "generate-manpage.nix.gen.hh" #include "generate-manpage.nix.gen.hh"
, CanonPath::root), *vGenerateManpage); , state.rootPath(CanonPath::root)), *vGenerateManpage);
auto vUtils = state.allocValue(); state.corepkgsFS->addFile(
state.cacheFile( CanonPath("utils.nix"),
CanonPath("/utils.nix"), CanonPath("/utils.nix"),
state.parseExprFromString(
#include "utils.nix.gen.hh" #include "utils.nix.gen.hh"
, CanonPath::root), );
*vUtils);
auto vSettingsInfo = state.allocValue(); state.corepkgsFS->addFile(
state.cacheFile( CanonPath("/generate-settings.nix"),
CanonPath("/generate-settings.nix"), CanonPath("/generate-settings.nix"),
state.parseExprFromString(
#include "generate-settings.nix.gen.hh" #include "generate-settings.nix.gen.hh"
, CanonPath::root), );
*vSettingsInfo);
auto vStoreInfo = state.allocValue(); state.corepkgsFS->addFile(
state.cacheFile( CanonPath("/generate-store-info.nix"),
CanonPath("/generate-store-info.nix"), CanonPath("/generate-store-info.nix"),
state.parseExprFromString(
#include "generate-store-info.nix.gen.hh" #include "generate-store-info.nix.gen.hh"
, CanonPath::root), );
*vStoreInfo);
auto vDump = state.allocValue(); auto vDump = state.allocValue();
vDump->mkString(toplevel.dumpCli()); vDump->mkString(toplevel.dumpCli());

View file

@ -1 +1 @@
"/nix/store/ya937r4ydw0l6kayq8jkyqaips9c75jm-output" [ "/nix/store/ya937r4ydw0l6kayq8jkyqaips9c75jm-output" "/nix/store/m7y372g6jb0g4hh1dzmj847rd356fhnz-output" ]

View file

@ -1,7 +1,15 @@
builtins.path [
(builtins.path
{ path = ./.; { path = ./.;
filter = path: _: baseNameOf path == "data"; filter = path: _: baseNameOf path == "data";
recursive = true; recursive = true;
sha256 = "1yhm3gwvg5a41yylymgblsclk95fs6jy72w0wv925mmidlhcq4sw"; sha256 = "1yhm3gwvg5a41yylymgblsclk95fs6jy72w0wv925mmidlhcq4sw";
name = "output"; name = "output";
} })
(builtins.path
{ path = ./data;
recursive = false;
sha256 = "0k4lwj58f2w5yh92ilrwy9917pycipbrdrr13vbb3yd02j09vfxm";
name = "output";
})
]

View file

@ -25,5 +25,7 @@ builtins.pathExists (./lib.nix)
&& builtins.pathExists (builtins.toString ./. + "/../lang/..//") && builtins.pathExists (builtins.toString ./. + "/../lang/..//")
&& builtins.pathExists (builtins.toPath (builtins.toString ./lib.nix)) && builtins.pathExists (builtins.toPath (builtins.toString ./lib.nix))
&& !builtins.pathExists (builtins.toPath (builtins.toString ./bla.nix)) && !builtins.pathExists (builtins.toPath (builtins.toString ./bla.nix))
&& builtins.pathExists (builtins.toPath { __toString = x: builtins.toString ./lib.nix; })
&& builtins.pathExists (builtins.toPath { outPath = builtins.toString ./lib.nix; })
&& builtins.pathExists ./lib.nix && builtins.pathExists ./lib.nix
&& !builtins.pathExists ./bla.nix && !builtins.pathExists ./bla.nix