Merge remote-tracking branch 'upstream/master' into trustless-remote-builder-simple

This commit is contained in:
John Ericson 2023-04-07 19:54:34 -04:00
commit 3ebd66c00e
57 changed files with 1991 additions and 946 deletions

View file

@ -1,8 +1,8 @@
diff --git a/darwin_stop_world.c b/darwin_stop_world.c diff --git a/darwin_stop_world.c b/darwin_stop_world.c
index 3dbaa3fb..36a1d1f7 100644 index 0468aaec..b348d869 100644
--- a/darwin_stop_world.c --- a/darwin_stop_world.c
+++ b/darwin_stop_world.c +++ b/darwin_stop_world.c
@@ -352,6 +352,7 @@ GC_INNER void GC_push_all_stacks(void) @@ -356,6 +356,7 @@ GC_INNER void GC_push_all_stacks(void)
int nthreads = 0; int nthreads = 0;
word total_size = 0; word total_size = 0;
mach_msg_type_number_t listcount = (mach_msg_type_number_t)THREAD_TABLE_SZ; mach_msg_type_number_t listcount = (mach_msg_type_number_t)THREAD_TABLE_SZ;
@ -10,7 +10,7 @@ index 3dbaa3fb..36a1d1f7 100644
if (!EXPECT(GC_thr_initialized, TRUE)) if (!EXPECT(GC_thr_initialized, TRUE))
GC_thr_init(); GC_thr_init();
@@ -407,6 +408,19 @@ GC_INNER void GC_push_all_stacks(void) @@ -411,6 +412,19 @@ GC_INNER void GC_push_all_stacks(void)
GC_push_all_stack_sections(lo, hi, p->traced_stack_sect); GC_push_all_stack_sections(lo, hi, p->traced_stack_sect);
} }
if (altstack_lo) { if (altstack_lo) {
@ -30,6 +30,22 @@ index 3dbaa3fb..36a1d1f7 100644
total_size += altstack_hi - altstack_lo; total_size += altstack_hi - altstack_lo;
GC_push_all_stack(altstack_lo, altstack_hi); GC_push_all_stack(altstack_lo, altstack_hi);
} }
diff --git a/include/gc.h b/include/gc.h
index edab6c22..f2c61282 100644
--- a/include/gc.h
+++ b/include/gc.h
@@ -2172,6 +2172,11 @@ GC_API void GC_CALL GC_win32_free_heap(void);
(*GC_amiga_allocwrapper_do)(a,GC_malloc_atomic_ignore_off_page)
#endif /* _AMIGA && !GC_AMIGA_MAKINGLIB */
+#if !__APPLE__
+/* Patch doesn't work on apple */
+#define NIX_BOEHM_PATCH_VERSION 1
+#endif
+
#ifdef __cplusplus
} /* extern "C" */
#endif
diff --git a/pthread_stop_world.c b/pthread_stop_world.c diff --git a/pthread_stop_world.c b/pthread_stop_world.c
index b5d71e62..aed7b0bf 100644 index b5d71e62..aed7b0bf 100644
--- a/pthread_stop_world.c --- a/pthread_stop_world.c

View file

@ -4,8 +4,8 @@ namespace nix {
void InstallableValueCommand::run(ref<Store> store, ref<Installable> installable) void InstallableValueCommand::run(ref<Store> store, ref<Installable> installable)
{ {
auto installableValue = InstallableValue::require(installable); auto installableValue = InstallableValue::require(installable);
run(store, installableValue); run(store, installableValue);
} }
} }

View file

@ -5,8 +5,10 @@
namespace nix { namespace nix {
/* Helper function to generate args that invoke $EDITOR on /**
filename:lineno. */ * Helper function to generate args that invoke $EDITOR on
* filename:lineno.
*/
Strings editorFor(const Path & file, uint32_t line); Strings editorFor(const Path & file, uint32_t line);
} }

View file

@ -22,7 +22,7 @@ InstallableValue::getCursor(EvalState & state)
static UsageError nonValueInstallable(Installable & installable) static UsageError nonValueInstallable(Installable & installable)
{ {
return UsageError("installable '%s' does not correspond to a Nix language value", installable.what()); return UsageError("installable '%s' does not correspond to a Nix language value", installable.what());
} }
InstallableValue & InstallableValue::require(Installable & installable) InstallableValue & InstallableValue::require(Installable & installable)

View file

@ -17,7 +17,9 @@ std::pair<Value *, PosIdx> findAlongAttrPath(
Bindings & autoArgs, Bindings & autoArgs,
Value & vIn); Value & vIn);
/* Heuristic to find the filename and lineno or a nix value. */ /**
* Heuristic to find the filename and lineno or a nix value.
*/
std::pair<std::string, uint32_t> findPackageFilename(EvalState & state, Value & v, std::string what); std::pair<std::string, uint32_t> findPackageFilename(EvalState & state, Value & v, std::string what);
std::vector<Symbol> parseAttrPath(EvalState & state, std::string_view s); std::vector<Symbol> parseAttrPath(EvalState & state, std::string_view s);

View file

@ -13,7 +13,9 @@ namespace nix {
class EvalState; class EvalState;
struct Value; struct Value;
/* Map one attribute name to its value. */ /**
* Map one attribute name to its value.
*/
struct Attr struct Attr
{ {
/* the placement of `name` and `pos` in this struct is important. /* the placement of `name` and `pos` in this struct is important.
@ -37,10 +39,12 @@ static_assert(sizeof(Attr) == 2 * sizeof(uint32_t) + sizeof(Value *),
"avoid introducing any padding into Attr if at all possible, and do not " "avoid introducing any padding into Attr if at all possible, and do not "
"introduce new fields that need not be present for almost every instance."); "introduce new fields that need not be present for almost every instance.");
/* Bindings contains all the attributes of an attribute set. It is defined /**
by its size and its capacity, the capacity being the number of Attr * Bindings contains all the attributes of an attribute set. It is defined
elements allocated after this structure, while the size corresponds to * by its size and its capacity, the capacity being the number of Attr
the number of elements already inserted in this structure. */ * elements allocated after this structure, while the size corresponds to
* the number of elements already inserted in this structure.
*/
class Bindings class Bindings
{ {
public: public:
@ -95,7 +99,9 @@ public:
size_t capacity() { return capacity_; } size_t capacity() { return capacity_; }
/* Returns the attributes in lexicographically sorted order. */ /**
* Returns the attributes in lexicographically sorted order.
*/
std::vector<const Attr *> lexicographicOrder(const SymbolTable & symbols) const std::vector<const Attr *> lexicographicOrder(const SymbolTable & symbols) const
{ {
std::vector<const Attr *> res; std::vector<const Attr *> res;
@ -112,9 +118,11 @@ public:
friend class EvalState; friend class EvalState;
}; };
/* A wrapper around Bindings that ensures that its always in sorted /**
order at the end. The only way to consume a BindingsBuilder is to * A wrapper around Bindings that ensures that its always in sorted
call finish(), which sorts the bindings. */ * order at the end. The only way to consume a BindingsBuilder is to
* call finish(), which sorts the bindings.
*/
class BindingsBuilder class BindingsBuilder
{ {
Bindings * bindings; Bindings * bindings;

View file

@ -110,8 +110,10 @@ public:
ref<AttrCursor> getAttr(std::string_view name); ref<AttrCursor> getAttr(std::string_view name);
/* Get an attribute along a chain of attrsets. Note that this does /**
not auto-call functors or functions. */ * Get an attribute along a chain of attrsets. Note that this does
* not auto-call functors or functions.
*/
OrSuggestions<ref<AttrCursor>> findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force = false); OrSuggestions<ref<AttrCursor>> findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force = false);
std::string getString(); std::string getString();
@ -130,7 +132,9 @@ public:
Value & forceValue(); Value & forceValue();
/* Force creation of the .drv file in the Nix store. */ /**
* Force creation of the .drv file in the Nix store.
*/
StorePath forceDerivation(); StorePath forceDerivation();
}; };

View file

@ -5,7 +5,9 @@
namespace nix { namespace nix {
/* Note: Various places expect the allocated memory to be zeroed. */ /**
* Note: Various places expect the allocated memory to be zeroed.
*/
[[gnu::always_inline]] [[gnu::always_inline]]
inline void * allocBytes(size_t n) inline void * allocBytes(size_t n)
{ {

View file

@ -344,6 +344,22 @@ static Symbol getName(const AttrName & name, EvalState & state, Env & env)
} }
} }
#if HAVE_BOEHMGC
/* Disable GC while this object lives. Used by CoroutineContext.
*
* Boehm keeps a count of GC_disable() and GC_enable() calls,
* and only enables GC when the count matches.
*/
class BoehmDisableGC {
public:
BoehmDisableGC() {
GC_disable();
};
~BoehmDisableGC() {
GC_enable();
};
};
#endif
static bool gcInitialised = false; static bool gcInitialised = false;
@ -368,6 +384,15 @@ void initGC()
StackAllocator::defaultAllocator = &boehmGCStackAllocator; StackAllocator::defaultAllocator = &boehmGCStackAllocator;
#if NIX_BOEHM_PATCH_VERSION != 1
printTalkative("Unpatched BoehmGC, disabling GC inside coroutines");
/* Used to disable GC when entering coroutines on macOS */
create_coro_gc_hook = []() -> std::shared_ptr<void> {
return std::make_shared<BoehmDisableGC>();
};
#endif
/* Set the initial heap size to something fairly big (25% of /* Set the initial heap size to something fairly big (25% of
physical RAM, up to a maximum of 384 MiB) so that in most cases physical RAM, up to a maximum of 384 MiB) so that in most cases
we don't need to garbage collect at all. (Collection has a we don't need to garbage collect at all. (Collection has a

View file

@ -43,7 +43,10 @@ struct PrimOp
struct Env struct Env
{ {
Env * up; Env * up;
unsigned short prevWith:14; // nr of levels up to next `with' environment /**
* Number of of levels up to next `with` environment
*/
unsigned short prevWith:14;
enum { Plain = 0, HasWithExpr, HasWithAttrs } type:2; enum { Plain = 0, HasWithExpr, HasWithAttrs } type:2;
Value * values[0]; Value * values[0];
}; };
@ -56,8 +59,10 @@ std::unique_ptr<ValMap> mapStaticEnvBindings(const SymbolTable & st, const Stati
void copyContext(const Value & v, PathSet & context); void copyContext(const Value & v, PathSet & context);
/* Cache for calls to addToStore(); maps source paths to the store /**
paths. */ * Cache for calls to addToStore(); maps source paths to the store
* paths.
*/
typedef std::map<Path, StorePath> SrcToStore; typedef std::map<Path, StorePath> SrcToStore;
@ -69,7 +74,9 @@ typedef std::pair<std::string, std::string> SearchPathElem;
typedef std::list<SearchPathElem> SearchPath; typedef std::list<SearchPathElem> SearchPath;
/* Initialise the Boehm GC, if applicable. */ /**
* Initialise the Boehm GC, if applicable.
*/
void initGC(); void initGC();
@ -144,26 +151,36 @@ public:
sOutputSpecified; sOutputSpecified;
Symbol sDerivationNix; Symbol sDerivationNix;
/* If set, force copying files to the Nix store even if they /**
already exist there. */ * If set, force copying files to the Nix store even if they
* already exist there.
*/
RepairFlag repair; RepairFlag repair;
/* The allowed filesystem paths in restricted or pure evaluation /**
mode. */ * The allowed filesystem paths in restricted or pure evaluation
* mode.
*/
std::optional<PathSet> allowedPaths; std::optional<PathSet> allowedPaths;
Bindings emptyBindings; Bindings emptyBindings;
/* Store used to materialise .drv files. */ /**
* Store used to materialise .drv files.
*/
const ref<Store> store; const ref<Store> store;
/* Store used to build stuff. */ /**
* Store used to build stuff.
*/
const ref<Store> buildStore; const ref<Store> buildStore;
RootValue vCallFlake = nullptr; RootValue vCallFlake = nullptr;
RootValue vImportedDrvToDerivation = nullptr; RootValue vImportedDrvToDerivation = nullptr;
/* Debugger */ /**
* Debugger
*/
void (* debugRepl)(ref<EvalState> es, const ValMap & extraEnv); void (* debugRepl)(ref<EvalState> es, const ValMap & extraEnv);
bool debugStop; bool debugStop;
bool debugQuit; bool debugQuit;
@ -219,7 +236,9 @@ public:
private: private:
SrcToStore srcToStore; SrcToStore srcToStore;
/* A cache from path names to parse trees. */ /**
* A cache from path names to parse trees.
*/
#if HAVE_BOEHMGC #if HAVE_BOEHMGC
typedef std::map<Path, Expr *, std::less<Path>, traceable_allocator<std::pair<const Path, Expr *>>> FileParseCache; typedef std::map<Path, Expr *, std::less<Path>, traceable_allocator<std::pair<const Path, Expr *>>> FileParseCache;
#else #else
@ -227,7 +246,9 @@ private:
#endif #endif
FileParseCache fileParseCache; FileParseCache fileParseCache;
/* A cache from path names to values. */ /**
* A cache from path names to values.
*/
#if HAVE_BOEHMGC #if HAVE_BOEHMGC
typedef std::map<Path, Value, std::less<Path>, traceable_allocator<std::pair<const Path, Value>>> FileEvalCache; typedef std::map<Path, Value, std::less<Path>, traceable_allocator<std::pair<const Path, Value>>> FileEvalCache;
#else #else
@ -239,17 +260,25 @@ private:
std::map<std::string, std::pair<bool, std::string>> searchPathResolved; std::map<std::string, std::pair<bool, std::string>> searchPathResolved;
/* Cache used by checkSourcePath(). */ /**
* Cache used by checkSourcePath().
*/
std::unordered_map<Path, Path> resolvedPaths; std::unordered_map<Path, Path> resolvedPaths;
/* Cache used by prim_match(). */ /**
* Cache used by prim_match().
*/
std::shared_ptr<RegexCache> regexCache; std::shared_ptr<RegexCache> regexCache;
#if HAVE_BOEHMGC #if HAVE_BOEHMGC
/* Allocation cache for GC'd Value objects. */ /**
* Allocation cache for GC'd Value objects.
*/
std::shared_ptr<void *> valueAllocCache; std::shared_ptr<void *> valueAllocCache;
/* Allocation cache for size-1 Env objects. */ /**
* Allocation cache for size-1 Env objects.
*/
std::shared_ptr<void *> env1AllocCache; std::shared_ptr<void *> env1AllocCache;
#endif #endif
@ -265,47 +294,65 @@ public:
SearchPath getSearchPath() { return searchPath; } SearchPath getSearchPath() { return searchPath; }
/* Allow access to a path. */ /**
* Allow access to a path.
*/
void allowPath(const Path & path); void allowPath(const Path & path);
/* Allow access to a store path. Note that this gets remapped to /**
the real store path if `store` is a chroot store. */ * Allow access to a store path. Note that this gets remapped to
* the real store path if `store` is a chroot store.
*/
void allowPath(const StorePath & storePath); void allowPath(const StorePath & storePath);
/* Allow access to a store path and return it as a string. */ /**
* Allow access to a store path and return it as a string.
*/
void allowAndSetStorePathString(const StorePath & storePath, Value & v); void allowAndSetStorePathString(const StorePath & storePath, Value & v);
/* Check whether access to a path is allowed and throw an error if /**
not. Otherwise return the canonicalised path. */ * Check whether access to a path is allowed and throw an error if
* not. Otherwise return the canonicalised path.
*/
Path checkSourcePath(const Path & path); Path checkSourcePath(const Path & path);
void checkURI(const std::string & uri); void checkURI(const std::string & uri);
/* When using a diverted store and 'path' is in the Nix store, map /**
'path' to the diverted location (e.g. /nix/store/foo is mapped * When using a diverted store and 'path' is in the Nix store, map
to /home/alice/my-nix/nix/store/foo). However, this is only * 'path' to the diverted location (e.g. /nix/store/foo is mapped
done if the context is not empty, since otherwise we're * to /home/alice/my-nix/nix/store/foo). However, this is only
probably trying to read from the actual /nix/store. This is * done if the context is not empty, since otherwise we're
intended to distinguish between import-from-derivation and * probably trying to read from the actual /nix/store. This is
sources stored in the actual /nix/store. */ * intended to distinguish between import-from-derivation and
* sources stored in the actual /nix/store.
*/
Path toRealPath(const Path & path, const PathSet & context); Path toRealPath(const Path & path, const PathSet & context);
/* Parse a Nix expression from the specified file. */ /**
* Parse a Nix expression from the specified file.
*/
Expr * parseExprFromFile(const Path & path); Expr * parseExprFromFile(const Path & path);
Expr * parseExprFromFile(const Path & path, std::shared_ptr<StaticEnv> & staticEnv); Expr * parseExprFromFile(const Path & path, std::shared_ptr<StaticEnv> & staticEnv);
/* Parse a Nix expression from the specified string. */ /**
* Parse a Nix expression from the specified string.
*/
Expr * parseExprFromString(std::string s, const Path & basePath, std::shared_ptr<StaticEnv> & staticEnv); Expr * parseExprFromString(std::string s, const Path & basePath, std::shared_ptr<StaticEnv> & staticEnv);
Expr * parseExprFromString(std::string s, const Path & basePath); Expr * parseExprFromString(std::string s, const Path & basePath);
Expr * parseStdin(); Expr * parseStdin();
/* Evaluate an expression read from the given file to normal /**
form. Optionally enforce that the top-level expression is * Evaluate an expression read from the given file to normal
trivial (i.e. doesn't require arbitrary computation). */ * form. Optionally enforce that the top-level expression is
* trivial (i.e. doesn't require arbitrary computation).
*/
void evalFile(const Path & path, Value & v, bool mustBeTrivial = false); void evalFile(const Path & path, Value & v, bool mustBeTrivial = false);
/* Like `evalFile`, but with an already parsed expression. */ /**
* Like `evalFile`, but with an already parsed expression.
*/
void cacheFile( void cacheFile(
const Path & path, const Path & path,
const Path & resolvedPath, const Path & resolvedPath,
@ -315,37 +362,52 @@ public:
void resetFileCache(); void resetFileCache();
/* Look up a file in the search path. */ /**
* Look up a file in the search path.
*/
Path findFile(const std::string_view path); Path findFile(const std::string_view path);
Path findFile(SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos); Path findFile(SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos);
/* If the specified search path element is a URI, download it. */ /**
* If the specified search path element is a URI, download it.
*/
std::pair<bool, std::string> resolveSearchPathElem(const SearchPathElem & elem); std::pair<bool, std::string> resolveSearchPathElem(const SearchPathElem & elem);
/* Evaluate an expression to normal form, storing the result in /**
value `v'. */ * Evaluate an expression to normal form
*
* @param [out] v The resulting is stored here.
*/
void eval(Expr * e, Value & v); void eval(Expr * e, Value & v);
/* Evaluation the expression, then verify that it has the expected /**
type. */ * Evaluation the expression, then verify that it has the expected
* type.
*/
inline bool evalBool(Env & env, Expr * e); inline bool evalBool(Env & env, Expr * e);
inline bool evalBool(Env & env, Expr * e, const PosIdx pos, std::string_view errorCtx); inline bool evalBool(Env & env, Expr * e, const PosIdx pos, std::string_view errorCtx);
inline void evalAttrs(Env & env, Expr * e, Value & v, const PosIdx pos, std::string_view errorCtx); inline void evalAttrs(Env & env, Expr * e, Value & v, const PosIdx pos, std::string_view errorCtx);
/* If `v' is a thunk, enter it and overwrite `v' with the result /**
of the evaluation of the thunk. If `v' is a delayed function * If `v` is a thunk, enter it and overwrite `v` with the result
application, call the function and overwrite `v' with the * of the evaluation of the thunk. If `v` is a delayed function
result. Otherwise, this is a no-op. */ * application, call the function and overwrite `v` with the
* result. Otherwise, this is a no-op.
*/
inline void forceValue(Value & v, const PosIdx pos); inline void forceValue(Value & v, const PosIdx pos);
template <typename Callable> template <typename Callable>
inline void forceValue(Value & v, Callable getPos); inline void forceValue(Value & v, Callable getPos);
/* Force a value, then recursively force list elements and /**
attributes. */ * Force a value, then recursively force list elements and
* attributes.
*/
void forceValueDeep(Value & v); void forceValueDeep(Value & v);
/* Force `v', and then verify that it has the expected type. */ /**
* Force `v`, and then verify that it has the expected type.
*/
NixInt forceInt(Value & v, const PosIdx pos, std::string_view errorCtx); NixInt forceInt(Value & v, const PosIdx pos, std::string_view errorCtx);
NixFloat forceFloat(Value & v, const PosIdx pos, std::string_view errorCtx); NixFloat forceFloat(Value & v, const PosIdx pos, std::string_view errorCtx);
bool forceBool(Value & v, const PosIdx pos, std::string_view errorCtx); bool forceBool(Value & v, const PosIdx pos, std::string_view errorCtx);
@ -356,7 +418,10 @@ public:
inline void forceAttrs(Value & v, Callable getPos, std::string_view errorCtx); inline void forceAttrs(Value & v, Callable getPos, std::string_view errorCtx);
inline void forceList(Value & v, const PosIdx pos, std::string_view errorCtx); inline void forceList(Value & v, const PosIdx pos, std::string_view errorCtx);
void forceFunction(Value & v, const PosIdx pos, std::string_view errorCtx); // either lambda or primop /**
* @param v either lambda or primop
*/
void forceFunction(Value & v, const PosIdx pos, std::string_view errorCtx);
std::string_view forceString(Value & v, const PosIdx pos, std::string_view errorCtx); std::string_view forceString(Value & v, const PosIdx pos, std::string_view errorCtx);
std::string_view forceString(Value & v, PathSet & context, const PosIdx pos, std::string_view errorCtx); std::string_view forceString(Value & v, PathSet & context, const PosIdx pos, std::string_view errorCtx);
std::string_view forceStringNoCtx(Value & v, const PosIdx pos, std::string_view errorCtx); std::string_view forceStringNoCtx(Value & v, const PosIdx pos, std::string_view errorCtx);
@ -367,17 +432,23 @@ public:
void addErrorTrace(Error & e, const PosIdx pos, const char * s, const std::string & s2, bool frame = false) const; void addErrorTrace(Error & e, const PosIdx pos, const char * s, const std::string & s2, bool frame = false) const;
public: public:
/* Return true iff the value `v' denotes a derivation (i.e. a /**
set with attribute `type = "derivation"'). */ * @return true iff the value `v` denotes a derivation (i.e. a
* set with attribute `type = "derivation"`).
*/
bool isDerivation(Value & v); bool isDerivation(Value & v);
std::optional<std::string> tryAttrsToString(const PosIdx pos, Value & v, std::optional<std::string> tryAttrsToString(const PosIdx pos, Value & v,
PathSet & context, bool coerceMore = false, bool copyToStore = true); PathSet & context, bool coerceMore = false, bool copyToStore = true);
/* String coercion. Converts strings, paths and derivations to a /**
string. If `coerceMore' is set, also converts nulls, integers, * String coercion.
booleans and lists to a string. If `copyToStore' is set, *
referenced paths are copied to the Nix store as a side effect. */ * Converts strings, paths and derivations to a
* string. If `coerceMore` is set, also converts nulls, integers,
* booleans and lists to a string. If `copyToStore` is set,
* referenced paths are copied to the Nix store as a side effect.
*/
BackedStringView coerceToString(const PosIdx pos, Value & v, PathSet & context, BackedStringView coerceToString(const PosIdx pos, Value & v, PathSet & context,
std::string_view errorCtx, std::string_view errorCtx,
bool coerceMore = false, bool copyToStore = true, bool coerceMore = false, bool copyToStore = true,
@ -385,21 +456,31 @@ public:
StorePath copyPathToStore(PathSet & context, const Path & path); StorePath copyPathToStore(PathSet & context, const Path & path);
/* Path coercion. Converts strings, paths and derivations to a /**
path. The result is guaranteed to be a canonicalised, absolute * Path coercion.
path. Nothing is copied to the store. */ *
* Converts strings, paths and derivations to a
* path. The result is guaranteed to be a canonicalised, absolute
* path. Nothing is copied to the store.
*/
Path coerceToPath(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx); Path coerceToPath(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx);
/* Like coerceToPath, but the result must be a store path. */ /**
* Like coerceToPath, but the result must be a store path.
*/
StorePath coerceToStorePath(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx); StorePath coerceToStorePath(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx);
public: public:
/* The base environment, containing the builtin functions and /**
values. */ * The base environment, containing the builtin functions and
* values.
*/
Env & baseEnv; Env & baseEnv;
/* The same, but used during parsing to resolve variables. */ /**
* The same, but used during parsing to resolve variables.
*/
std::shared_ptr<StaticEnv> staticBaseEnv; // !!! should be private std::shared_ptr<StaticEnv> staticBaseEnv; // !!! should be private
private: private:
@ -449,8 +530,10 @@ private:
public: public:
/* Do a deep equality test between two values. That is, list /**
elements and attributes are compared recursively. */ * Do a deep equality test between two values. That is, list
* elements and attributes are compared recursively.
*/
bool eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx); bool eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx);
bool isFunctor(Value & fun); bool isFunctor(Value & fun);
@ -464,11 +547,15 @@ public:
callFunction(fun, 1, args, vRes, pos); callFunction(fun, 1, args, vRes, pos);
} }
/* Automatically call a function for which each argument has a /**
default value or has a binding in the `args' map. */ * Automatically call a function for which each argument has a
* default value or has a binding in the `args` map.
*/
void autoCallFunction(Bindings & args, Value & fun, Value & res); void autoCallFunction(Bindings & args, Value & fun, Value & res);
/* Allocation primitives. */ /**
* Allocation primitives.
*/
inline Value * allocValue(); inline Value * allocValue();
inline Env & allocEnv(size_t size); inline Env & allocEnv(size_t size);
@ -488,10 +575,13 @@ public:
void concatLists(Value & v, size_t nrLists, Value * * lists, const PosIdx pos, std::string_view errorCtx); void concatLists(Value & v, size_t nrLists, Value * * lists, const PosIdx pos, std::string_view errorCtx);
/* Print statistics. */ /**
* Print statistics.
*/
void printStats(); void printStats();
/* Realise the given context, and return a mapping from the placeholders /**
* Realise the given context, and return a mapping from the placeholders
* used to construct the associated value to their final store path * used to construct the associated value to their final store path
*/ */
[[nodiscard]] StringMap realiseContext(const PathSet & context); [[nodiscard]] StringMap realiseContext(const PathSet & context);
@ -551,11 +641,15 @@ struct DebugTraceStacker {
DebugTrace trace; DebugTrace trace;
}; };
/* Return a string representing the type of the value `v'. */ /**
* @return A string representing the type of the value `v`.
*/
std::string_view showType(ValueType type); std::string_view showType(ValueType type);
std::string showType(const Value & v); std::string showType(const Value & v);
/* If `path' refers to a directory, then append "/default.nix". */ /**
* If `path` refers to a directory, then append "/default.nix".
*/
Path resolveExprPath(Path path); Path resolveExprPath(Path path);
struct InvalidPathError : EvalError struct InvalidPathError : EvalError

View file

@ -18,7 +18,8 @@ struct FlakeInput;
typedef std::map<FlakeId, FlakeInput> FlakeInputs; typedef std::map<FlakeId, FlakeInput> FlakeInputs;
/* FlakeInput is the 'Flake'-level parsed form of the "input" entries /**
* FlakeInput is the 'Flake'-level parsed form of the "input" entries
* in the flake file. * in the flake file.
* *
* A FlakeInput is normally constructed by the 'parseFlakeInput' * A FlakeInput is normally constructed by the 'parseFlakeInput'
@ -42,7 +43,12 @@ typedef std::map<FlakeId, FlakeInput> FlakeInputs;
struct FlakeInput struct FlakeInput
{ {
std::optional<FlakeRef> ref; std::optional<FlakeRef> ref;
bool isFlake = true; // true = process flake to get outputs, false = (fetched) static source path /**
* true = process flake to get outputs
*
* false = (fetched) static source path
*/
bool isFlake = true;
std::optional<InputPath> follows; std::optional<InputPath> follows;
FlakeInputs overrides; FlakeInputs overrides;
}; };
@ -56,23 +62,42 @@ struct ConfigFile
void apply(); void apply();
}; };
/* The contents of a flake.nix file. */ /**
* The contents of a flake.nix file.
*/
struct Flake struct Flake
{ {
FlakeRef originalRef; // the original flake specification (by the user) /**
FlakeRef resolvedRef; // registry references and caching resolved to the specific underlying flake * The original flake specification (by the user)
FlakeRef lockedRef; // the specific local store result of invoking the fetcher */
bool forceDirty = false; // pretend that 'lockedRef' is dirty FlakeRef originalRef;
/**
* registry references and caching resolved to the specific underlying flake
*/
FlakeRef resolvedRef;
/**
* the specific local store result of invoking the fetcher
*/
FlakeRef lockedRef;
/**
* pretend that 'lockedRef' is dirty
*/
bool forceDirty = false;
std::optional<std::string> description; std::optional<std::string> description;
std::shared_ptr<const fetchers::Tree> sourceInfo; std::shared_ptr<const fetchers::Tree> sourceInfo;
FlakeInputs inputs; FlakeInputs inputs;
ConfigFile config; // 'nixConfig' attribute /**
* 'nixConfig' attribute
*/
ConfigFile config;
~Flake(); ~Flake();
}; };
Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool allowLookup); Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool allowLookup);
/* Fingerprint of a locked flake; used as a cache key. */ /**
* Fingerprint of a locked flake; used as a cache key.
*/
typedef Hash Fingerprint; typedef Hash Fingerprint;
struct LockedFlake struct LockedFlake
@ -85,50 +110,72 @@ struct LockedFlake
struct LockFlags struct LockFlags
{ {
/* Whether to ignore the existing lock file, creating a new one /**
from scratch. */ * Whether to ignore the existing lock file, creating a new one
* from scratch.
*/
bool recreateLockFile = false; bool recreateLockFile = false;
/* Whether to update the lock file at all. If set to false, if any /**
change to the lock file is needed (e.g. when an input has been * Whether to update the lock file at all. If set to false, if any
added to flake.nix), you get a fatal error. */ * change to the lock file is needed (e.g. when an input has been
* added to flake.nix), you get a fatal error.
*/
bool updateLockFile = true; bool updateLockFile = true;
/* Whether to write the lock file to disk. If set to true, if the /**
any changes to the lock file are needed and the flake is not * Whether to write the lock file to disk. If set to true, if the
writable (i.e. is not a local Git working tree or similar), you * any changes to the lock file are needed and the flake is not
get a fatal error. If set to false, Nix will use the modified * writable (i.e. is not a local Git working tree or similar), you
lock file in memory only, without writing it to disk. */ * get a fatal error. If set to false, Nix will use the modified
* lock file in memory only, without writing it to disk.
*/
bool writeLockFile = true; bool writeLockFile = true;
/* Whether to use the registries to lookup indirect flake /**
references like 'nixpkgs'. */ * Whether to use the registries to lookup indirect flake
* references like 'nixpkgs'.
*/
std::optional<bool> useRegistries = std::nullopt; std::optional<bool> useRegistries = std::nullopt;
/* Whether to apply flake's nixConfig attribute to the configuration */ /**
* Whether to apply flake's nixConfig attribute to the configuration
*/
bool applyNixConfig = false; bool applyNixConfig = false;
/* Whether unlocked flake references (i.e. those without a Git /**
revision or similar) without a corresponding lock are * Whether unlocked flake references (i.e. those without a Git
allowed. Unlocked flake references with a lock are always * revision or similar) without a corresponding lock are
allowed. */ * allowed. Unlocked flake references with a lock are always
* allowed.
*/
bool allowUnlocked = true; bool allowUnlocked = true;
/* Whether to commit changes to flake.lock. */ /**
* Whether to commit changes to flake.lock.
*/
bool commitLockFile = false; bool commitLockFile = false;
/* The path to a lock file to read instead of the `flake.lock` file in the top-level flake */ /**
* The path to a lock file to read instead of the `flake.lock` file in the top-level flake
*/
std::optional<std::string> referenceLockFilePath; std::optional<std::string> referenceLockFilePath;
/* The path to a lock file to write to instead of the `flake.lock` file in the top-level flake */ /**
* The path to a lock file to write to instead of the `flake.lock` file in the top-level flake
*/
std::optional<Path> outputLockFilePath; std::optional<Path> outputLockFilePath;
/* Flake inputs to be overridden. */ /**
* Flake inputs to be overridden.
*/
std::map<InputPath, FlakeRef> inputOverrides; std::map<InputPath, FlakeRef> inputOverrides;
/* Flake inputs to be updated. This means that any existing lock /**
for those inputs will be ignored. */ * Flake inputs to be updated. This means that any existing lock
* for those inputs will be ignored.
*/
std::set<InputPath> inputUpdates; std::set<InputPath> inputUpdates;
}; };

View file

@ -14,7 +14,8 @@ class Store;
typedef std::string FlakeId; typedef std::string FlakeId;
/* A flake reference specifies how to fetch a flake or raw source /**
* A flake reference specifies how to fetch a flake or raw source
* (e.g. from a Git repository). It is created from a URL-like syntax * (e.g. from a Git repository). It is created from a URL-like syntax
* (e.g. 'github:NixOS/patchelf'), an attrset representation (e.g. '{ * (e.g. 'github:NixOS/patchelf'), an attrset representation (e.g. '{
* type="github"; owner = "NixOS"; repo = "patchelf"; }'), or a local * type="github"; owner = "NixOS"; repo = "patchelf"; }'), or a local
@ -33,14 +34,17 @@ typedef std::string FlakeId;
* be lazy), but the fetcher can be invoked at any time via the * be lazy), but the fetcher can be invoked at any time via the
* FlakeRef to ensure the store is populated with this input. * FlakeRef to ensure the store is populated with this input.
*/ */
struct FlakeRef struct FlakeRef
{ {
/* Fetcher-specific representation of the input, sufficient to /**
perform the fetch operation. */ * Fetcher-specific representation of the input, sufficient to
* perform the fetch operation.
*/
fetchers::Input input; fetchers::Input input;
/* sub-path within the fetched input that represents this input */ /**
* sub-path within the fetched input that represents this input
*/
Path subdir; Path subdir;
bool operator==(const FlakeRef & other) const; bool operator==(const FlakeRef & other) const;

View file

@ -16,9 +16,11 @@ typedef std::vector<FlakeId> InputPath;
struct LockedNode; struct LockedNode;
/* A node in the lock file. It has outgoing edges to other nodes (its /**
inputs). Only the root node has this type; all other nodes have * A node in the lock file. It has outgoing edges to other nodes (its
type LockedNode. */ * inputs). Only the root node has this type; all other nodes have
* type LockedNode.
*/
struct Node : std::enable_shared_from_this<Node> struct Node : std::enable_shared_from_this<Node>
{ {
typedef std::variant<ref<LockedNode>, InputPath> Edge; typedef std::variant<ref<LockedNode>, InputPath> Edge;
@ -28,7 +30,9 @@ struct Node : std::enable_shared_from_this<Node>
virtual ~Node() { } virtual ~Node() { }
}; };
/* A non-root node in the lock file. */ /**
* A non-root node in the lock file.
*/
struct LockedNode : Node struct LockedNode : Node
{ {
FlakeRef lockedRef, originalRef; FlakeRef lockedRef, originalRef;
@ -63,7 +67,9 @@ struct LockFile
void write(const Path & path) const; void write(const Path & path) const;
/* Check whether this lock file has any unlocked inputs. */ /**
* Check whether this lock file has any unlocked inputs.
*/
std::optional<FlakeRef> isUnlocked() const; std::optional<FlakeRef> isUnlocked() const;
bool operator ==(const LockFile & other) const; bool operator ==(const LockFile & other) const;
@ -74,7 +80,9 @@ struct LockFile
static std::string diff(const LockFile & oldLocks, const LockFile & newLocks); static std::string diff(const LockFile & oldLocks, const LockFile & newLocks);
/* Check that every 'follows' input target exists. */ /**
* Check that every 'follows' input target exists.
*/
void check(); void check();
}; };

View file

@ -26,7 +26,10 @@ private:
mutable std::string outputName; mutable std::string outputName;
Outputs outputs; Outputs outputs;
bool failed = false; // set if we get an AssertionError /**
* Set if we get an AssertionError
*/
bool failed = false;
Bindings * attrs = nullptr, * meta = nullptr; Bindings * attrs = nullptr, * meta = nullptr;
@ -35,7 +38,10 @@ private:
bool checkMeta(Value & v); bool checkMeta(Value & v);
public: public:
std::string attrPath; /* path towards the derivation */ /**
* path towards the derivation
*/
std::string attrPath;
DrvInfo(EvalState & state) : state(&state) { }; DrvInfo(EvalState & state) : state(&state) { };
DrvInfo(EvalState & state, std::string attrPath, Bindings * attrs); DrvInfo(EvalState & state, std::string attrPath, Bindings * attrs);
@ -47,8 +53,10 @@ public:
StorePath requireDrvPath() const; StorePath requireDrvPath() const;
StorePath queryOutPath() const; StorePath queryOutPath() const;
std::string queryOutputName() const; std::string queryOutputName() const;
/** Return the unordered map of output names to (optional) output paths. /**
* The "outputs to install" are determined by `meta.outputsToInstall`. */ * Return the unordered map of output names to (optional) output paths.
* The "outputs to install" are determined by `meta.outputsToInstall`.
*/
Outputs queryOutputs(bool withPaths = true, bool onlyOutputsToInstall = false); Outputs queryOutputs(bool withPaths = true, bool onlyOutputsToInstall = false);
StringSet queryMetaNames(); StringSet queryMetaNames();
@ -80,8 +88,10 @@ typedef std::list<DrvInfo> DrvInfos;
#endif #endif
/* If value `v' denotes a derivation, return a DrvInfo object /**
describing it. Otherwise return nothing. */ * If value `v` denotes a derivation, return a DrvInfo object
* describing it. Otherwise return nothing.
*/
std::optional<DrvInfo> getDerivation(EvalState & state, std::optional<DrvInfo> getDerivation(EvalState & state,
Value & v, bool ignoreAssertionFailures); Value & v, bool ignoreAssertionFailures);

View file

@ -22,7 +22,9 @@ MakeError(UndefinedVarError, Error);
MakeError(MissingArgumentError, EvalError); MakeError(MissingArgumentError, EvalError);
MakeError(RestrictedPathError, Error); MakeError(RestrictedPathError, Error);
/* Position objects. */ /**
* Position objects.
*/
struct Pos struct Pos
{ {
uint32_t line; uint32_t line;
@ -133,7 +135,9 @@ class EvalState;
struct StaticEnv; struct StaticEnv;
/* An attribute path is a sequence of attribute names. */ /**
* An attribute path is a sequence of attribute names.
*/
struct AttrName struct AttrName
{ {
Symbol symbol; Symbol symbol;
@ -213,11 +217,11 @@ struct ExprVar : Expr
or function argument) or from a "with". */ or function argument) or from a "with". */
bool fromWith; bool fromWith;
/* In the former case, the value is obtained by going `level' /* In the former case, the value is obtained by going `level`
levels up from the current environment and getting the levels up from the current environment and getting the
`displ'th value in that environment. In the latter case, the `displ`th value in that environment. In the latter case, the
value is obtained by getting the attribute named `name' from value is obtained by getting the attribute named `name` from
the set stored in the environment that is `level' levels up the set stored in the environment that is `level` levels up
from the current one.*/ from the current one.*/
Level level; Level level;
Displacement displ; Displacement displ;

View file

@ -23,9 +23,11 @@ struct RegisterPrimOp
typedef std::vector<Info> PrimOps; typedef std::vector<Info> PrimOps;
static PrimOps * primOps; static PrimOps * primOps;
/* You can register a constant by passing an arity of 0. fun /**
will get called during EvalState initialization, so there * You can register a constant by passing an arity of 0. fun
may be primops not yet added and builtins is not yet sorted. */ * will get called during EvalState initialization, so there
* may be primops not yet added and builtins is not yet sorted.
*/
RegisterPrimOp( RegisterPrimOp(
std::string name, std::string name,
size_t arity, size_t arity,
@ -38,10 +40,14 @@ struct RegisterPrimOp
may wish to use them in limited contexts without globally enabling may wish to use them in limited contexts without globally enabling
them. */ them. */
/* Load a ValueInitializer from a DSO and return whatever it initializes */ /**
* Load a ValueInitializer from a DSO and return whatever it initializes
*/
void prim_importNative(EvalState & state, const PosIdx pos, Value * * args, Value & v); void prim_importNative(EvalState & state, const PosIdx pos, Value * * args, Value & v);
/* Execute a program and parse its output */ /**
* Execute a program and parse its output
*/
void prim_exec(EvalState & state, const PosIdx pos, Value * * args, Value & v); void prim_exec(EvalState & state, const PosIdx pos, Value * * args, Value & v);
} }

View file

@ -10,15 +10,11 @@
namespace nix { namespace nix {
/* Symbol table used by the parser and evaluator to represent and look /**
up identifiers and attributes efficiently. SymbolTable::create() * This class mainly exists to give us an operator<< for ostreams. We could also
converts a string into a symbol. Symbols have the property that * return plain strings from SymbolTable, but then we'd have to wrap every
they can be compared efficiently (using an equality test), * instance of a symbol that is fmt()ed, which is inconvenient and error-prone.
because the symbol table stores only one copy of each string. */ */
/* This class mainly exists to give us an operator<< for ostreams. We could also
return plain strings from SymbolTable, but then we'd have to wrap every
instance of a symbol that is fmt()ed, which is inconvenient and error-prone. */
class SymbolStr class SymbolStr
{ {
friend class SymbolTable; friend class SymbolTable;
@ -47,6 +43,11 @@ public:
friend std::ostream & operator <<(std::ostream & os, const SymbolStr & symbol); friend std::ostream & operator <<(std::ostream & os, const SymbolStr & symbol);
}; };
/**
* Symbols have the property that they can be compared efficiently
* (using an equality test), because the symbol table stores only one
* copy of each string.
*/
class Symbol class Symbol
{ {
friend class SymbolTable; friend class SymbolTable;
@ -66,6 +67,10 @@ public:
bool operator!=(const Symbol other) const { return id != other.id; } bool operator!=(const Symbol other) const { return id != other.id; }
}; };
/**
* Symbol table used by the parser and evaluator to represent and look
* up identifiers and attributes efficiently.
*/
class SymbolTable class SymbolTable
{ {
private: private:
@ -74,6 +79,9 @@ private:
public: public:
/**
* converts a string into a symbol.
*/
Symbol create(std::string_view s) Symbol create(std::string_view s)
{ {
// Most symbols are looked up more than once, so we trade off insertion performance // Most symbols are looked up more than once, so we trade off insertion performance

View file

@ -36,9 +36,11 @@ typedef enum {
tFloat tFloat
} InternalType; } InternalType;
// This type abstracts over all actual value types in the language, /**
// grouping together implementation details like tList*, different function * This type abstracts over all actual value types in the language,
// types, and types in non-normal form (so thunks and co.) * grouping together implementation details like tList*, different function
* types, and types in non-normal form (so thunks and co.)
*/
typedef enum { typedef enum {
nThunk, nThunk,
nInt, nInt,
@ -70,38 +72,51 @@ class XMLWriter;
typedef int64_t NixInt; typedef int64_t NixInt;
typedef double NixFloat; typedef double NixFloat;
/* External values must descend from ExternalValueBase, so that /**
* External values must descend from ExternalValueBase, so that
* type-agnostic nix functions (e.g. showType) can be implemented * type-agnostic nix functions (e.g. showType) can be implemented
*/ */
class ExternalValueBase class ExternalValueBase
{ {
friend std::ostream & operator << (std::ostream & str, const ExternalValueBase & v); friend std::ostream & operator << (std::ostream & str, const ExternalValueBase & v);
protected: protected:
/* Print out the value */ /**
* Print out the value
*/
virtual std::ostream & print(std::ostream & str) const = 0; virtual std::ostream & print(std::ostream & str) const = 0;
public: public:
/* Return a simple string describing the type */ /**
* Return a simple string describing the type
*/
virtual std::string showType() const = 0; virtual std::string showType() const = 0;
/* Return a string to be used in builtins.typeOf */ /**
* Return a string to be used in builtins.typeOf
*/
virtual std::string typeOf() const = 0; virtual std::string typeOf() const = 0;
/* Coerce the value to a string. Defaults to uncoercable, i.e. throws an /**
* Coerce the value to a string. Defaults to uncoercable, i.e. throws an
* error. * error.
*/ */
virtual std::string coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const; virtual std::string coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const;
/* Compare to another value of the same type. Defaults to uncomparable, /**
* Compare to another value of the same type. Defaults to uncomparable,
* i.e. always false. * i.e. always false.
*/ */
virtual bool operator ==(const ExternalValueBase & b) const; virtual bool operator ==(const ExternalValueBase & b) const;
/* Print the value as JSON. Defaults to unconvertable, i.e. throws an error */ /**
* Print the value as JSON. Defaults to unconvertable, i.e. throws an error
*/
virtual nlohmann::json printValueAsJSON(EvalState & state, bool strict, virtual nlohmann::json printValueAsJSON(EvalState & state, bool strict,
PathSet & context, bool copyToStore = true) const; PathSet & context, bool copyToStore = true) const;
/* Print the value as XML. Defaults to unevaluated */ /**
* Print the value as XML. Defaults to unevaluated
*/
virtual void printValueAsXML(EvalState & state, bool strict, bool location, virtual void printValueAsXML(EvalState & state, bool strict, bool location,
XMLWriter & doc, PathSet & context, PathSet & drvsSeen, XMLWriter & doc, PathSet & context, PathSet & drvsSeen,
const PosIdx pos) const; const PosIdx pos) const;
@ -146,26 +161,28 @@ public:
NixInt integer; NixInt integer;
bool boolean; bool boolean;
/* Strings in the evaluator carry a so-called `context' which /**
is a list of strings representing store paths. This is to * Strings in the evaluator carry a so-called `context` which
allow users to write things like * is a list of strings representing store paths. This is to
* allow users to write things like
"--with-freetype2-library=" + freetype + "/lib" * "--with-freetype2-library=" + freetype + "/lib"
where `freetype' is a derivation (or a source to be copied * where `freetype` is a derivation (or a source to be copied
to the store). If we just concatenated the strings without * to the store). If we just concatenated the strings without
keeping track of the referenced store paths, then if the * keeping track of the referenced store paths, then if the
string is used as a derivation attribute, the derivation * string is used as a derivation attribute, the derivation
will not have the correct dependencies in its inputDrvs and * will not have the correct dependencies in its inputDrvs and
inputSrcs. * inputSrcs.
The semantics of the context is as follows: when a string * The semantics of the context is as follows: when a string
with context C is used as a derivation attribute, then the * with context C is used as a derivation attribute, then the
derivations in C will be added to the inputDrvs of the * derivations in C will be added to the inputDrvs of the
derivation, and the other store paths in C will be added to * derivation, and the other store paths in C will be added to
the inputSrcs of the derivations. * the inputSrcs of the derivations.
For canonicity, the store paths should be in sorted order. */ * For canonicity, the store paths should be in sorted order.
*/
struct { struct {
const char * s; const char * s;
const char * * context; // must be in sorted order const char * * context; // must be in sorted order
@ -197,8 +214,10 @@ public:
NixFloat fpoint; NixFloat fpoint;
}; };
// Returns the normal type of a Value. This only returns nThunk if the /**
// Value hasn't been forceValue'd * Returns the normal type of a Value. This only returns nThunk if
* the Value hasn't been forceValue'd
*/
inline ValueType type() const inline ValueType type() const
{ {
switch (internalType) { switch (internalType) {
@ -217,8 +236,10 @@ public:
abort(); abort();
} }
/* After overwriting an app node, be sure to clear pointers in the /**
Value to ensure that the target isn't kept alive unnecessarily. */ * After overwriting an app node, be sure to clear pointers in the
* Value to ensure that the target isn't kept alive unnecessarily.
*/
inline void clearValue() inline void clearValue()
{ {
app.left = app.right = 0; app.left = app.right = 0;
@ -366,9 +387,11 @@ public:
PosIdx determinePos(const PosIdx pos) const; PosIdx determinePos(const PosIdx pos) const;
/* Check whether forcing this value requires a trivial amount of /**
computation. In particular, function applications are * Check whether forcing this value requires a trivial amount of
non-trivial. */ * computation. In particular, function applications are
* non-trivial.
*/
bool isTrivial() const; bool isTrivial() const;
NixStringContext getContext(const Store &); NixStringContext getContext(const Store &);
@ -414,7 +437,9 @@ typedef std::map<Symbol, ValueVector> ValueVectorMap;
#endif #endif
/* A value allocated in traceable memory. */ /**
* A value allocated in traceable memory.
*/
typedef std::shared_ptr<Value *> RootValue; typedef std::shared_ptr<Value *> RootValue;
RootValue allocRootValue(Value * v); RootValue allocRootValue(Value * v);

View file

@ -28,34 +28,37 @@ public:
class Store; class Store;
/* Plain opaque path to some store object. /**
* Plain opaque path to some store object.
Encoded as just the path: <path>. *
*/ * Encoded as just the path: <path>.
*/
struct NixStringContextElem_Opaque { struct NixStringContextElem_Opaque {
StorePath path; StorePath path;
GENERATE_CMP(NixStringContextElem_Opaque, me->path); GENERATE_CMP(NixStringContextElem_Opaque, me->path);
}; };
/* Path to a derivation and its entire build closure. /**
* Path to a derivation and its entire build closure.
The path doesn't just refer to derivation itself and its closure, but *
also all outputs of all derivations in that closure (including the * The path doesn't just refer to derivation itself and its closure, but
root derivation). * also all outputs of all derivations in that closure (including the
* root derivation).
Encoded in the form =<drvPath>. *
*/ * Encoded in the form =<drvPath>.
*/
struct NixStringContextElem_DrvDeep { struct NixStringContextElem_DrvDeep {
StorePath drvPath; StorePath drvPath;
GENERATE_CMP(NixStringContextElem_DrvDeep, me->drvPath); GENERATE_CMP(NixStringContextElem_DrvDeep, me->drvPath);
}; };
/* Derivation output. /**
* Derivation output.
Encoded in the form !<output>!<drvPath>. *
*/ * Encoded in the form !<output>!<drvPath>.
*/
struct NixStringContextElem_Built { struct NixStringContextElem_Built {
StorePath drvPath; StorePath drvPath;
std::string output; std::string output;
@ -84,11 +87,12 @@ struct NixStringContextElem : _NixStringContextElem_Raw {
return static_cast<Raw &>(*this); return static_cast<Raw &>(*this);
} }
/* Decode a context string, one of: /**
- <path> * Decode a context string, one of:
- =<path> * - <path>
- !<name>!<path> * - =<path>
*/ * - !<name>!<path>
*/
static NixStringContextElem parse(const Store & store, std::string_view s); static NixStringContextElem parse(const Store & store, std::string_view s);
std::string to_string(const Store & store) const; std::string to_string(const Store & store) const;
}; };

View file

@ -21,14 +21,14 @@ struct Tree
struct InputScheme; struct InputScheme;
/* The Input object is generated by a specific fetcher, based on the /**
* The Input object is generated by a specific fetcher, based on the
* user-supplied input attribute in the flake.nix file, and contains * user-supplied input attribute in the flake.nix file, and contains
* the information that the specific fetcher needs to perform the * the information that the specific fetcher needs to perform the
* actual fetch. The Input object is most commonly created via the * actual fetch. The Input object is most commonly created via the
* "fromURL()" or "fromAttrs()" static functions which are provided * "fromURL()" or "fromAttrs()" static functions which are provided
* the url or attrset specified in the flake file. * the url or attrset specified in the flake file.
*/ */
struct Input struct Input
{ {
friend struct InputScheme; friend struct InputScheme;
@ -38,7 +38,9 @@ struct Input
bool locked = false; bool locked = false;
bool direct = true; bool direct = true;
/* path of the parent of this input, used for relative path resolution */ /**
* path of the parent of this input, used for relative path resolution
*/
std::optional<Path> parent; std::optional<Path> parent;
public: public:
@ -56,27 +58,35 @@ public:
Attrs toAttrs() const; Attrs toAttrs() const;
/* Check whether this is a "direct" input, that is, not /**
one that goes through a registry. */ * Check whether this is a "direct" input, that is, not
* one that goes through a registry.
*/
bool isDirect() const { return direct; } bool isDirect() const { return direct; }
/* Check whether this is a "locked" input, that is, /**
one that contains a commit hash or content hash. */ * Check whether this is a "locked" input, that is,
* one that contains a commit hash or content hash.
*/
bool isLocked() const { return locked; } bool isLocked() const { return locked; }
/* Check whether the input carries all necessary info required /**
for cache insertion and substitution. * Check whether the input carries all necessary info required
These fields are used to uniquely identify cached trees * for cache insertion and substitution.
within the "tarball TTL" window without necessarily * These fields are used to uniquely identify cached trees
indicating that the input's origin is unchanged. */ * within the "tarball TTL" window without necessarily
* indicating that the input's origin is unchanged.
*/
bool hasAllInfo() const; bool hasAllInfo() const;
bool operator ==(const Input & other) const; bool operator ==(const Input & other) const;
bool contains(const Input & other) const; bool contains(const Input & other) const;
/* Fetch the input into the Nix store, returning the location in /**
the Nix store and the locked input. */ * Fetch the input into the Nix store, returning the location in
* the Nix store and the locked input.
*/
std::pair<Tree, Input> fetch(ref<Store> store) const; std::pair<Tree, Input> fetch(ref<Store> store) const;
Input applyOverrides( Input applyOverrides(
@ -105,7 +115,8 @@ public:
}; };
/* The InputScheme represents a type of fetcher. Each fetcher /**
* The InputScheme represents a type of fetcher. Each fetcher
* registers with nix at startup time. When processing an input for a * registers with nix at startup time. When processing an input for a
* flake, each scheme is given an opportunity to "recognize" that * flake, each scheme is given an opportunity to "recognize" that
* input from the url or attributes in the flake file's specification * input from the url or attributes in the flake file's specification

View file

@ -25,7 +25,9 @@ public:
int handleExceptions(const std::string & programName, std::function<void()> fun); int handleExceptions(const std::string & programName, std::function<void()> fun);
/* Don't forget to call initPlugins() after settings are initialized! */ /**
* Don't forget to call initPlugins() after settings are initialized!
*/
void initNix(); void initNix();
void parseCmdLine(int argc, char * * argv, void parseCmdLine(int argc, char * * argv,
@ -36,7 +38,9 @@ void parseCmdLine(const std::string & programName, const Strings & args,
void printVersion(const std::string & programName); void printVersion(const std::string & programName);
/* Ugh. No better place to put this. */ /**
* Ugh. No better place to put this.
*/
void printGCWarning(); void printGCWarning();
class Store; class Store;
@ -75,11 +79,15 @@ struct LegacyArgs : public MixCommonArgs
}; };
/* Show the manual page for the specified program. */ /**
* Show the manual page for the specified program.
*/
void showManPage(const std::string & name); void showManPage(const std::string & name);
/* The constructor of this class starts a pager if stdout is a /**
terminal and $PAGER is set. Stdout is redirected to the pager. */ * The constructor of this class starts a pager if stdout is a
* terminal and $PAGER is set. Stdout is redirected to the pager.
*/
class RunPager class RunPager
{ {
public: public:
@ -110,28 +118,34 @@ struct PrintFreed
}; };
/* Install a SIGSEGV handler to detect stack overflows. */ /**
* Install a SIGSEGV handler to detect stack overflows.
*/
void detectStackOverflow(); void detectStackOverflow();
/* Pluggable behavior to run in case of a stack overflow. /**
* Pluggable behavior to run in case of a stack overflow.
Default value: defaultStackOverflowHandler. *
* Default value: defaultStackOverflowHandler.
This is called by the handler installed by detectStackOverflow(). *
* This is called by the handler installed by detectStackOverflow().
This gives Nix library consumers a limit opportunity to report the error *
condition. The handler should exit the process. * This gives Nix library consumers a limit opportunity to report the error
See defaultStackOverflowHandler() for a reference implementation. * condition. The handler should exit the process.
* See defaultStackOverflowHandler() for a reference implementation.
NOTE: Use with diligence, because this runs in the signal handler, with very *
limited stack space and a potentially a corrupted heap, all while the failed * NOTE: Use with diligence, because this runs in the signal handler, with very
thread is blocked indefinitely. All functions called must be reentrant. */ * limited stack space and a potentially a corrupted heap, all while the failed
* thread is blocked indefinitely. All functions called must be reentrant.
*/
extern std::function<void(siginfo_t * info, void * ctx)> stackOverflowHandler; extern std::function<void(siginfo_t * info, void * ctx)> stackOverflowHandler;
/* The default, robust implementation of stackOverflowHandler. /**
* The default, robust implementation of stackOverflowHandler.
Prints an error message directly to stderr using a syscall instead of the *
logger. Exits the process immediately after. */ * Prints an error message directly to stderr using a syscall instead of the
* logger. Exits the process immediately after.
*/
void defaultStackOverflowHandler(siginfo_t * info, void * ctx); void defaultStackOverflowHandler(siginfo_t * info, void * ctx);
} }

View file

@ -46,6 +46,11 @@ struct BinaryCacheStoreConfig : virtual StoreConfig
)"}; )"};
}; };
/**
* @note subclasses must implement at least one of the two
* virtual getFile() methods.
*/
class BinaryCacheStore : public virtual BinaryCacheStoreConfig, class BinaryCacheStore : public virtual BinaryCacheStoreConfig,
public virtual Store, public virtual Store,
public virtual LogStore public virtual LogStore
@ -75,14 +80,15 @@ public:
std::string && data, std::string && data,
const std::string & mimeType); const std::string & mimeType);
/* Note: subclasses must implement at least one of the two /**
following getFile() methods. */ * Dump the contents of the specified file to a sink.
*/
/* Dump the contents of the specified file to a sink. */
virtual void getFile(const std::string & path, Sink & sink); virtual void getFile(const std::string & path, Sink & sink);
/* Fetch the specified file and call the specified callback with /**
the result. A subclass may implement this asynchronously. */ * Fetch the specified file and call the specified callback with
* the result. A subclass may implement this asynchronously.
*/
virtual void getFile( virtual void getFile(
const std::string & path, const std::string & path,
Callback<std::optional<std::string>> callback) noexcept; Callback<std::optional<std::string>> callback) noexcept;

View file

@ -12,9 +12,12 @@ namespace nix {
struct BuildResult struct BuildResult
{ {
/* Note: don't remove status codes, and only add new status codes /**
at the end of the list, to prevent client/server * @note This is directly used in the nix-store --serve protocol.
incompatibilities in the nix-store --serve protocol. */ * That means we need to worry about compatability across versions.
* Therefore, don't remove status codes, and only add new status
* codes at the end of the list.
*/
enum Status { enum Status {
Built = 0, Built = 0,
Substituted, Substituted,
@ -22,8 +25,10 @@ struct BuildResult
PermanentFailure, PermanentFailure,
InputRejected, InputRejected,
OutputRejected, OutputRejected,
TransientFailure, // possibly transient /// possibly transient
CachedFailure, // no longer used TransientFailure,
/// no longer used
CachedFailure,
TimedOut, TimedOut,
MiscFailure, MiscFailure,
DependencyFailed, DependencyFailed,
@ -33,7 +38,12 @@ struct BuildResult
NoSubstituters, NoSubstituters,
} status = MiscFailure; } status = MiscFailure;
// FIXME: include entire ErrorInfo object. /**
* Information about the error if the build failed.
*
* @todo This should be an entire ErrorInfo object, not just a
* string, for richer information.
*/
std::string errorMsg; std::string errorMsg;
std::string toString() const { std::string toString() const {
@ -60,27 +70,39 @@ struct BuildResult
return strStatus + ((errorMsg == "") ? "" : " : " + errorMsg); return strStatus + ((errorMsg == "") ? "" : " : " + errorMsg);
} }
/* How many times this build was performed. */ /**
* How many times this build was performed.
*/
unsigned int timesBuilt = 0; unsigned int timesBuilt = 0;
/* If timesBuilt > 1, whether some builds did not produce the same /**
result. (Note that 'isNonDeterministic = false' does not mean * If timesBuilt > 1, whether some builds did not produce the same
the build is deterministic, just that we don't have evidence of * result. (Note that 'isNonDeterministic = false' does not mean
non-determinism.) */ * the build is deterministic, just that we don't have evidence of
* non-determinism.)
*/
bool isNonDeterministic = false; bool isNonDeterministic = false;
/* The derivation we built or the store path we substituted. */ /**
* The derivation we built or the store path we substituted.
*/
DerivedPath path; DerivedPath path;
/* For derivations, a mapping from the names of the wanted outputs /**
to actual paths. */ * For derivations, a mapping from the names of the wanted outputs
* to actual paths.
*/
DrvOutputs builtOutputs; DrvOutputs builtOutputs;
/* The start/stop times of the build (or one of the rounds, if it /**
was repeated). */ * The start/stop times of the build (or one of the rounds, if it
* was repeated).
*/
time_t startTime = 0, stopTime = 0; time_t startTime = 0, stopTime = 0;
/* User and system CPU time the build took. */ /**
* User and system CPU time the build took.
*/
std::optional<std::chrono::microseconds> cpuUser, cpuSystem; std::optional<std::chrono::microseconds> cpuUser, cpuSystem;
bool success() bool success()

View file

@ -16,8 +16,10 @@ struct HookInstance;
typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
/* Unless we are repairing, we don't both to test validity and just assume it, /**
so the choices are `Absent` or `Valid`. */ * Unless we are repairing, we don't both to test validity and just assume it,
* so the choices are `Absent` or `Valid`.
*/
enum struct PathStatus { enum struct PathStatus {
Corrupt, Corrupt,
Absent, Absent,
@ -27,11 +29,15 @@ enum struct PathStatus {
struct InitialOutputStatus { struct InitialOutputStatus {
StorePath path; StorePath path;
PathStatus status; PathStatus status;
/* Valid in the store, and additionally non-corrupt if we are repairing */ /**
* Valid in the store, and additionally non-corrupt if we are repairing
*/
bool isValid() const { bool isValid() const {
return status == PathStatus::Valid; return status == PathStatus::Valid;
} }
/* Merely present, allowed to be corrupt */ /**
* Merely present, allowed to be corrupt
*/
bool isPresent() const { bool isPresent() const {
return status == PathStatus::Corrupt return status == PathStatus::Corrupt
|| status == PathStatus::Valid; || status == PathStatus::Valid;
@ -46,59 +52,87 @@ struct InitialOutput {
struct DerivationGoal : public Goal struct DerivationGoal : public Goal
{ {
/* Whether to use an on-disk .drv file. */ /**
* Whether to use an on-disk .drv file.
*/
bool useDerivation; bool useDerivation;
/* The path of the derivation. */ /** The path of the derivation. */
StorePath drvPath; StorePath drvPath;
/* The goal for the corresponding resolved derivation */ /**
* The goal for the corresponding resolved derivation
*/
std::shared_ptr<DerivationGoal> resolvedDrvGoal; std::shared_ptr<DerivationGoal> resolvedDrvGoal;
/* The specific outputs that we need to build. Empty means all of /**
them. */ * The specific outputs that we need to build. Empty means all of
* them.
*/
OutputsSpec wantedOutputs; OutputsSpec wantedOutputs;
/* Mapping from input derivations + output names to actual store /**
paths. This is filled in by waiteeDone() as each dependency * Mapping from input derivations + output names to actual store
finishes, before inputsRealised() is reached, */ * paths. This is filled in by waiteeDone() as each dependency
* finishes, before inputsRealised() is reached.
*/
std::map<std::pair<StorePath, std::string>, StorePath> inputDrvOutputs; std::map<std::pair<StorePath, std::string>, StorePath> inputDrvOutputs;
/* Whether additional wanted outputs have been added. */ /**
* Whether additional wanted outputs have been added.
*/
bool needRestart = false; bool needRestart = false;
/* Whether to retry substituting the outputs after building the /**
inputs. This is done in case of an incomplete closure. */ * Whether to retry substituting the outputs after building the
* inputs. This is done in case of an incomplete closure.
*/
bool retrySubstitution = false; bool retrySubstitution = false;
/* Whether we've retried substitution, in which case we won't try /**
again. */ * Whether we've retried substitution, in which case we won't try
* again.
*/
bool retriedSubstitution = false; bool retriedSubstitution = false;
/* The derivation stored at drvPath. */ /**
* The derivation stored at drvPath.
*/
std::unique_ptr<Derivation> drv; std::unique_ptr<Derivation> drv;
std::unique_ptr<ParsedDerivation> parsedDrv; std::unique_ptr<ParsedDerivation> parsedDrv;
/* The remainder is state held during the build. */ /**
* The remainder is state held during the build.
*/
/* Locks on (fixed) output paths. */ /**
* Locks on (fixed) output paths.
*/
PathLocks outputLocks; PathLocks outputLocks;
/* All input paths (that is, the union of FS closures of the /**
immediate input paths). */ * All input paths (that is, the union of FS closures of the
* immediate input paths).
*/
StorePathSet inputPaths; StorePathSet inputPaths;
std::map<std::string, InitialOutput> initialOutputs; std::map<std::string, InitialOutput> initialOutputs;
/* File descriptor for the log file. */ /**
* File descriptor for the log file.
*/
AutoCloseFD fdLogFile; AutoCloseFD fdLogFile;
std::shared_ptr<BufferedSink> logFileSink, logSink; std::shared_ptr<BufferedSink> logFileSink, logSink;
/* Number of bytes received from the builder's stdout/stderr. */ /**
* Number of bytes received from the builder's stdout/stderr.
*/
unsigned long logSize; unsigned long logSize;
/* The most recent log lines. */ /**
* The most recent log lines.
*/
std::list<std::string> logTail; std::list<std::string> logTail;
std::string currentLogLine; std::string currentLogLine;
@ -106,10 +140,14 @@ struct DerivationGoal : public Goal
std::string currentHookLine; std::string currentHookLine;
/* The build hook. */ /**
* The build hook.
*/
std::unique_ptr<HookInstance> hook; std::unique_ptr<HookInstance> hook;
/* The sort of derivation we are building. */ /**
* The sort of derivation we are building.
*/
DerivationType derivationType; DerivationType derivationType;
typedef void (DerivationGoal::*GoalState)(); typedef void (DerivationGoal::*GoalState)();
@ -121,12 +159,16 @@ struct DerivationGoal : public Goal
std::unique_ptr<Activity> act; std::unique_ptr<Activity> act;
/* Activity that denotes waiting for a lock. */ /**
* Activity that denotes waiting for a lock.
*/
std::unique_ptr<Activity> actLock; std::unique_ptr<Activity> actLock;
std::map<ActivityId, Activity> builderActivities; std::map<ActivityId, Activity> builderActivities;
/* The remote machine on which we're building. */ /**
* The remote machine on which we're building.
*/
std::string machineName; std::string machineName;
DerivationGoal(const StorePath & drvPath, DerivationGoal(const StorePath & drvPath,
@ -143,10 +185,14 @@ struct DerivationGoal : public Goal
void work() override; void work() override;
/* Add wanted outputs to an already existing derivation goal. */ /**
* Add wanted outputs to an already existing derivation goal.
*/
void addWantedOutputs(const OutputsSpec & outputs); void addWantedOutputs(const OutputsSpec & outputs);
/* The states. */ /**
* The states.
*/
void getDerivation(); void getDerivation();
void loadDerivation(); void loadDerivation();
void haveDerivation(); void haveDerivation();
@ -160,28 +206,42 @@ struct DerivationGoal : public Goal
void resolvedFinished(); void resolvedFinished();
/* Is the build hook willing to perform the build? */ /**
* Is the build hook willing to perform the build?
*/
HookReply tryBuildHook(); HookReply tryBuildHook();
virtual int getChildStatus(); virtual int getChildStatus();
/* Check that the derivation outputs all exist and register them /**
as valid. */ * Check that the derivation outputs all exist and register them
* as valid.
*/
virtual DrvOutputs registerOutputs(); virtual DrvOutputs registerOutputs();
/* Open a log file and a pipe to it. */ /**
* Open a log file and a pipe to it.
*/
Path openLogFile(); Path openLogFile();
/* Sign the newly built realisation if the store allows it */ /**
* Sign the newly built realisation if the store allows it
*/
virtual void signRealisation(Realisation&) {} virtual void signRealisation(Realisation&) {}
/* Close the log file. */ /**
* Close the log file.
*/
void closeLogFile(); void closeLogFile();
/* Close the read side of the logger pipe. */ /**
* Close the read side of the logger pipe.
*/
virtual void closeReadPipes(); virtual void closeReadPipes();
/* Cleanup hooks for buildDone() */ /**
* Cleanup hooks for buildDone()
*/
virtual void cleanupHookFinally(); virtual void cleanupHookFinally();
virtual void cleanupPreChildKill(); virtual void cleanupPreChildKill();
virtual void cleanupPostChildKill(); virtual void cleanupPostChildKill();
@ -191,30 +251,40 @@ struct DerivationGoal : public Goal
virtual bool isReadDesc(int fd); virtual bool isReadDesc(int fd);
/* Callback used by the worker to write to the log. */ /**
* Callback used by the worker to write to the log.
*/
void handleChildOutput(int fd, std::string_view data) override; void handleChildOutput(int fd, std::string_view data) override;
void handleEOF(int fd) override; void handleEOF(int fd) override;
void flushLine(); void flushLine();
/* Wrappers around the corresponding Store methods that first consult the /**
derivation. This is currently needed because when there is no drv file * Wrappers around the corresponding Store methods that first consult the
there also is no DB entry. */ * derivation. This is currently needed because when there is no drv file
* there also is no DB entry.
*/
std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(); std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap();
OutputPathMap queryDerivationOutputMap(); OutputPathMap queryDerivationOutputMap();
/* Update 'initialOutputs' to determine the current status of the /**
outputs of the derivation. Also returns a Boolean denoting * Update 'initialOutputs' to determine the current status of the
whether all outputs are valid and non-corrupt, and a * outputs of the derivation. Also returns a Boolean denoting
'DrvOutputs' structure containing the valid and wanted * whether all outputs are valid and non-corrupt, and a
outputs. */ * 'DrvOutputs' structure containing the valid and wanted
* outputs.
*/
std::pair<bool, DrvOutputs> checkPathValidity(); std::pair<bool, DrvOutputs> checkPathValidity();
/* Aborts if any output is not valid or corrupt, and otherwise /**
returns a 'DrvOutputs' structure containing the wanted * Aborts if any output is not valid or corrupt, and otherwise
outputs. */ * returns a 'DrvOutputs' structure containing the wanted
* outputs.
*/
DrvOutputs assertPathValidity(); DrvOutputs assertPathValidity();
/* Forcibly kill the child process, if any. */ /**
* Forcibly kill the child process, if any.
*/
virtual void killChild(); virtual void killChild();
void repairClosure(); void repairClosure();

View file

@ -11,24 +11,34 @@ namespace nix {
class Worker; class Worker;
// Substitution of a derivation output. /**
// This is done in three steps: * Substitution of a derivation output.
// 1. Fetch the output info from a substituter * This is done in three steps:
// 2. Substitute the corresponding output path * 1. Fetch the output info from a substituter
// 3. Register the output info * 2. Substitute the corresponding output path
* 3. Register the output info
*/
class DrvOutputSubstitutionGoal : public Goal { class DrvOutputSubstitutionGoal : public Goal {
// The drv output we're trying to substitue /**
* The drv output we're trying to substitue
*/
DrvOutput id; DrvOutput id;
// The realisation corresponding to the given output id. /**
// Will be filled once we can get it. * The realisation corresponding to the given output id.
* Will be filled once we can get it.
*/
std::shared_ptr<const Realisation> outputInfo; std::shared_ptr<const Realisation> outputInfo;
/* The remaining substituters. */ /**
* The remaining substituters.
*/
std::list<ref<Store>> subs; std::list<ref<Store>> subs;
/* The current substituter. */ /**
* The current substituter.
*/
std::shared_ptr<Store> sub; std::shared_ptr<Store> sub;
struct DownloadState struct DownloadState
@ -39,7 +49,9 @@ class DrvOutputSubstitutionGoal : public Goal {
std::shared_ptr<DownloadState> downloadState; std::shared_ptr<DownloadState> downloadState;
/* Whether a substituter failed. */ /**
* Whether a substituter failed.
*/
bool substituterFailed = false; bool substituterFailed = false;
public: public:

View file

@ -7,11 +7,15 @@
namespace nix { namespace nix {
/* Forward definition. */ /**
* Forward definition.
*/
struct Goal; struct Goal;
class Worker; class Worker;
/* A pointer to a goal. */ /**
* A pointer to a goal.
*/
typedef std::shared_ptr<Goal> GoalPtr; typedef std::shared_ptr<Goal> GoalPtr;
typedef std::weak_ptr<Goal> WeakGoalPtr; typedef std::weak_ptr<Goal> WeakGoalPtr;
@ -19,48 +23,72 @@ struct CompareGoalPtrs {
bool operator() (const GoalPtr & a, const GoalPtr & b) const; bool operator() (const GoalPtr & a, const GoalPtr & b) const;
}; };
/* Set of goals. */ /**
* Set of goals.
*/
typedef std::set<GoalPtr, CompareGoalPtrs> Goals; typedef std::set<GoalPtr, CompareGoalPtrs> Goals;
typedef std::set<WeakGoalPtr, std::owner_less<WeakGoalPtr>> WeakGoals; typedef std::set<WeakGoalPtr, std::owner_less<WeakGoalPtr>> WeakGoals;
/* A map of paths to goals (and the other way around). */ /**
* A map of paths to goals (and the other way around).
*/
typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap; typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap;
struct Goal : public std::enable_shared_from_this<Goal> struct Goal : public std::enable_shared_from_this<Goal>
{ {
typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode; typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode;
/* Backlink to the worker. */ /**
* Backlink to the worker.
*/
Worker & worker; Worker & worker;
/* Goals that this goal is waiting for. */ /**
* Goals that this goal is waiting for.
*/
Goals waitees; Goals waitees;
/* Goals waiting for this one to finish. Must use weak pointers /**
here to prevent cycles. */ * Goals waiting for this one to finish. Must use weak pointers
* here to prevent cycles.
*/
WeakGoals waiters; WeakGoals waiters;
/* Number of goals we are/were waiting for that have failed. */ /**
* Number of goals we are/were waiting for that have failed.
*/
size_t nrFailed = 0; size_t nrFailed = 0;
/* Number of substitution goals we are/were waiting for that /**
failed because there are no substituters. */ * Number of substitution goals we are/were waiting for that
* failed because there are no substituters.
*/
size_t nrNoSubstituters = 0; size_t nrNoSubstituters = 0;
/* Number of substitution goals we are/were waiting for that /**
failed because they had unsubstitutable references. */ * Number of substitution goals we are/were waiting for that
* failed because they had unsubstitutable references.
*/
size_t nrIncompleteClosure = 0; size_t nrIncompleteClosure = 0;
/* Name of this goal for debugging purposes. */ /**
* Name of this goal for debugging purposes.
*/
std::string name; std::string name;
/* Whether the goal is finished. */ /**
* Whether the goal is finished.
*/
ExitCode exitCode = ecBusy; ExitCode exitCode = ecBusy;
/* Build result. */ /**
* Build result.
*/
BuildResult buildResult; BuildResult buildResult;
/* Exception containing an error message, if any. */ /**
* Exception containing an error message, if any.
*/
std::optional<Error> ex; std::optional<Error> ex;
Goal(Worker & worker, DerivedPath path) Goal(Worker & worker, DerivedPath path)
@ -96,9 +124,11 @@ struct Goal : public std::enable_shared_from_this<Goal>
return name; return name;
} }
/* Callback in case of a timeout. It should wake up its waiters, /**
get rid of any running child processes that are being monitored * Callback in case of a timeout. It should wake up its waiters,
by the worker (important!), etc. */ * get rid of any running child processes that are being monitored
* by the worker (important!), etc.
*/
virtual void timedOut(Error && ex) = 0; virtual void timedOut(Error && ex) = 0;
virtual std::string key() = 0; virtual std::string key() = 0;

View file

@ -8,16 +8,24 @@ namespace nix {
struct HookInstance struct HookInstance
{ {
/* Pipes for talking to the build hook. */ /**
* Pipes for talking to the build hook.
*/
Pipe toHook; Pipe toHook;
/* Pipe for the hook's standard output/error. */ /**
* Pipe for the hook's standard output/error.
*/
Pipe fromHook; Pipe fromHook;
/* Pipe for the builder's standard output/error. */ /**
* Pipe for the builder's standard output/error.
*/
Pipe builderOut; Pipe builderOut;
/* The process ID of the hook. */ /**
* The process ID of the hook.
*/
Pid pid; Pid pid;
FdSink sink; FdSink sink;

View file

@ -10,49 +10,75 @@ struct LocalDerivationGoal : public DerivationGoal
{ {
LocalStore & getLocalStore(); LocalStore & getLocalStore();
/* User selected for running the builder. */ /**
* User selected for running the builder.
*/
std::unique_ptr<UserLock> buildUser; std::unique_ptr<UserLock> buildUser;
/* The process ID of the builder. */ /**
* The process ID of the builder.
*/
Pid pid; Pid pid;
/* The cgroup of the builder, if any. */ /**
* The cgroup of the builder, if any.
*/
std::optional<Path> cgroup; std::optional<Path> cgroup;
/* The temporary directory. */ /**
* The temporary directory.
*/
Path tmpDir; Path tmpDir;
/* The path of the temporary directory in the sandbox. */ /**
* The path of the temporary directory in the sandbox.
*/
Path tmpDirInSandbox; Path tmpDirInSandbox;
/* Master side of the pseudoterminal used for the builder's /**
standard output/error. */ * Master side of the pseudoterminal used for the builder's
* standard output/error.
*/
AutoCloseFD builderOut; AutoCloseFD builderOut;
/* Pipe for synchronising updates to the builder namespaces. */ /**
* Pipe for synchronising updates to the builder namespaces.
*/
Pipe userNamespaceSync; Pipe userNamespaceSync;
/* The mount namespace and user namespace of the builder, used to add additional /**
paths to the sandbox as a result of recursive Nix calls. */ * The mount namespace and user namespace of the builder, used to add additional
* paths to the sandbox as a result of recursive Nix calls.
*/
AutoCloseFD sandboxMountNamespace; AutoCloseFD sandboxMountNamespace;
AutoCloseFD sandboxUserNamespace; AutoCloseFD sandboxUserNamespace;
/* On Linux, whether we're doing the build in its own user /**
namespace. */ * On Linux, whether we're doing the build in its own user
* namespace.
*/
bool usingUserNamespace = true; bool usingUserNamespace = true;
/* Whether we're currently doing a chroot build. */ /**
* Whether we're currently doing a chroot build.
*/
bool useChroot = false; bool useChroot = false;
Path chrootRootDir; Path chrootRootDir;
/* RAII object to delete the chroot directory. */ /**
* RAII object to delete the chroot directory.
*/
std::shared_ptr<AutoDelete> autoDelChroot; std::shared_ptr<AutoDelete> autoDelChroot;
/* Whether to run the build in a private network namespace. */ /**
* Whether to run the build in a private network namespace.
*/
bool privateNetwork = false; bool privateNetwork = false;
/* Stuff we need to pass to initChild(). */ /**
* Stuff we need to pass to initChild().
*/
struct ChrootPath { struct ChrootPath {
Path source; Path source;
bool optional; bool optional;
@ -71,30 +97,35 @@ struct LocalDerivationGoal : public DerivationGoal
SandboxProfile additionalSandboxProfile; SandboxProfile additionalSandboxProfile;
#endif #endif
/* Hash rewriting. */ /**
* Hash rewriting.
*/
StringMap inputRewrites, outputRewrites; StringMap inputRewrites, outputRewrites;
typedef map<StorePath, StorePath> RedirectedOutputs; typedef map<StorePath, StorePath> RedirectedOutputs;
RedirectedOutputs redirectedOutputs; RedirectedOutputs redirectedOutputs;
/* The outputs paths used during the build. /**
* The outputs paths used during the build.
- Input-addressed derivations or fixed content-addressed outputs are *
sometimes built when some of their outputs already exist, and can not * - Input-addressed derivations or fixed content-addressed outputs are
be hidden via sandboxing. We use temporary locations instead and * sometimes built when some of their outputs already exist, and can not
rewrite after the build. Otherwise the regular predetermined paths are * be hidden via sandboxing. We use temporary locations instead and
put here. * rewrite after the build. Otherwise the regular predetermined paths are
* put here.
- Floating content-addressed derivations do not know their final build *
output paths until the outputs are hashed, so random locations are * - Floating content-addressed derivations do not know their final build
used, and then renamed. The randomness helps guard against hidden * output paths until the outputs are hashed, so random locations are
self-references. * used, and then renamed. The randomness helps guard against hidden
* self-references.
*/ */
OutputPathMap scratchOutputs; OutputPathMap scratchOutputs;
/* Path registration info from the previous round, if we're /**
building multiple times. Since this contains the hash, it * Path registration info from the previous round, if we're
allows us to compare whether two rounds produced the same * building multiple times. Since this contains the hash, it
result. */ * allows us to compare whether two rounds produced the same
* result.
*/
std::map<Path, ValidPathInfo> prevInfos; std::map<Path, ValidPathInfo> prevInfos;
uid_t sandboxUid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 1000 : 0) : buildUser->getUID(); } uid_t sandboxUid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 1000 : 0) : buildUser->getUID(); }
@ -102,25 +133,37 @@ struct LocalDerivationGoal : public DerivationGoal
const static Path homeDir; const static Path homeDir;
/* The recursive Nix daemon socket. */ /**
* The recursive Nix daemon socket.
*/
AutoCloseFD daemonSocket; AutoCloseFD daemonSocket;
/* The daemon main thread. */ /**
* The daemon main thread.
*/
std::thread daemonThread; std::thread daemonThread;
/* The daemon worker threads. */ /**
* The daemon worker threads.
*/
std::vector<std::thread> daemonWorkerThreads; std::vector<std::thread> daemonWorkerThreads;
/* Paths that were added via recursive Nix calls. */ /**
* Paths that were added via recursive Nix calls.
*/
StorePathSet addedPaths; StorePathSet addedPaths;
/* Realisations that were added via recursive Nix calls. */ /**
* Realisations that were added via recursive Nix calls.
*/
std::set<DrvOutput> addedDrvOutputs; std::set<DrvOutput> addedDrvOutputs;
/* Recursive Nix calls are only allowed to build or realize paths /**
in the original input closure or added via a recursive Nix call * Recursive Nix calls are only allowed to build or realize paths
(so e.g. you can't do 'nix-store -r /nix/store/<bla>' where * in the original input closure or added via a recursive Nix call
/nix/store/<bla> is some arbitrary path in a binary cache). */ * (so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
* /nix/store/<bla> is some arbitrary path in a binary cache).
*/
bool isAllowed(const StorePath & path) bool isAllowed(const StorePath & path)
{ {
return inputPaths.count(path) || addedPaths.count(path); return inputPaths.count(path) || addedPaths.count(path);
@ -138,55 +181,81 @@ struct LocalDerivationGoal : public DerivationGoal
virtual ~LocalDerivationGoal() override; virtual ~LocalDerivationGoal() override;
/* Whether we need to perform hash rewriting if there are valid output paths. */ /**
* Whether we need to perform hash rewriting if there are valid output paths.
*/
bool needsHashRewrite(); bool needsHashRewrite();
/* The additional states. */ /**
* The additional states.
*/
void tryLocalBuild() override; void tryLocalBuild() override;
/* Start building a derivation. */ /**
* Start building a derivation.
*/
void startBuilder(); void startBuilder();
/* Fill in the environment for the builder. */ /**
* Fill in the environment for the builder.
*/
void initEnv(); void initEnv();
/* Setup tmp dir location. */ /**
* Setup tmp dir location.
*/
void initTmpDir(); void initTmpDir();
/* Write a JSON file containing the derivation attributes. */ /**
* Write a JSON file containing the derivation attributes.
*/
void writeStructuredAttrs(); void writeStructuredAttrs();
void startDaemon(); void startDaemon();
void stopDaemon(); void stopDaemon();
/* Add 'path' to the set of paths that may be referenced by the /**
outputs, and make it appear in the sandbox. */ * Add 'path' to the set of paths that may be referenced by the
* outputs, and make it appear in the sandbox.
*/
void addDependency(const StorePath & path); void addDependency(const StorePath & path);
/* Make a file owned by the builder. */ /**
* Make a file owned by the builder.
*/
void chownToBuilder(const Path & path); void chownToBuilder(const Path & path);
int getChildStatus() override; int getChildStatus() override;
/* Run the builder's process. */ /**
* Run the builder's process.
*/
void runChild(); void runChild();
/* Check that the derivation outputs all exist and register them /**
as valid. */ * Check that the derivation outputs all exist and register them
* as valid.
*/
DrvOutputs registerOutputs() override; DrvOutputs registerOutputs() override;
void signRealisation(Realisation &) override; void signRealisation(Realisation &) override;
/* Check that an output meets the requirements specified by the /**
'outputChecks' attribute (or the legacy * Check that an output meets the requirements specified by the
'{allowed,disallowed}{References,Requisites}' attributes). */ * 'outputChecks' attribute (or the legacy
* '{allowed,disallowed}{References,Requisites}' attributes).
*/
void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs); void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs);
/* Close the read side of the logger pipe. */ /**
* Close the read side of the logger pipe.
*/
void closeReadPipes() override; void closeReadPipes() override;
/* Cleanup hooks for buildDone() */ /**
* Cleanup hooks for buildDone()
*/
void cleanupHookFinally() override; void cleanupHookFinally() override;
void cleanupPreChildKill() override; void cleanupPreChildKill() override;
void cleanupPostChildKill() override; void cleanupPostChildKill() override;
@ -196,24 +265,36 @@ struct LocalDerivationGoal : public DerivationGoal
bool isReadDesc(int fd) override; bool isReadDesc(int fd) override;
/* Delete the temporary directory, if we have one. */ /**
* Delete the temporary directory, if we have one.
*/
void deleteTmpDir(bool force); void deleteTmpDir(bool force);
/* Forcibly kill the child process, if any. */ /**
* Forcibly kill the child process, if any.
*/
void killChild() override; void killChild() override;
/* Kill any processes running under the build user UID or in the /**
cgroup of the build. */ * Kill any processes running under the build user UID or in the
* cgroup of the build.
*/
void killSandbox(bool getStats); void killSandbox(bool getStats);
/* Create alternative path calculated from but distinct from the /**
input, so we can avoid overwriting outputs (or other store paths) * Create alternative path calculated from but distinct from the
that already exist. */ * input, so we can avoid overwriting outputs (or other store paths)
* that already exist.
*/
StorePath makeFallbackPath(const StorePath & path); StorePath makeFallbackPath(const StorePath & path);
/* Make a path to another based on the output name along with the
derivation hash. */ /**
/* FIXME add option to randomize, so we can audit whether our * Make a path to another based on the output name along with the
rewrites caught everything */ * derivation hash.
*
* @todo Add option to randomize, so we can audit whether our
* rewrites caught everything
*/
StorePath makeFallbackPath(std::string_view outputName); StorePath makeFallbackPath(std::string_view outputName);
}; };

View file

@ -11,38 +11,58 @@ class Worker;
struct PathSubstitutionGoal : public Goal struct PathSubstitutionGoal : public Goal
{ {
/* The store path that should be realised through a substitute. */ /**
* The store path that should be realised through a substitute.
*/
StorePath storePath; StorePath storePath;
/* The path the substituter refers to the path as. This will be /**
different when the stores have different names. */ * The path the substituter refers to the path as. This will be
* different when the stores have different names.
*/
std::optional<StorePath> subPath; std::optional<StorePath> subPath;
/* The remaining substituters. */ /**
* The remaining substituters.
*/
std::list<ref<Store>> subs; std::list<ref<Store>> subs;
/* The current substituter. */ /**
* The current substituter.
*/
std::shared_ptr<Store> sub; std::shared_ptr<Store> sub;
/* Whether a substituter failed. */ /**
* Whether a substituter failed.
*/
bool substituterFailed = false; bool substituterFailed = false;
/* Path info returned by the substituter's query info operation. */ /**
* Path info returned by the substituter's query info operation.
*/
std::shared_ptr<const ValidPathInfo> info; std::shared_ptr<const ValidPathInfo> info;
/* Pipe for the substituter's standard output. */ /**
* Pipe for the substituter's standard output.
*/
Pipe outPipe; Pipe outPipe;
/* The substituter thread. */ /**
* The substituter thread.
*/
std::thread thr; std::thread thr;
std::promise<void> promise; std::promise<void> promise;
/* Whether to try to repair a valid path. */ /**
* Whether to try to repair a valid path.
*/
RepairFlag repair; RepairFlag repair;
/* Location where we're downloading the substitute. Differs from /**
storePath when doing a repair. */ * Location where we're downloading the substitute. Differs from
* storePath when doing a repair.
*/
Path destPath; Path destPath;
std::unique_ptr<MaintainCount<uint64_t>> maintainExpectedSubstitutions, std::unique_ptr<MaintainCount<uint64_t>> maintainExpectedSubstitutions,
@ -51,7 +71,9 @@ struct PathSubstitutionGoal : public Goal
typedef void (PathSubstitutionGoal::*GoalState)(); typedef void (PathSubstitutionGoal::*GoalState)();
GoalState state; GoalState state;
/* Content address for recomputing store path */ /**
* Content address for recomputing store path
*/
std::optional<ContentAddress> ca; std::optional<ContentAddress> ca;
void done( void done(
@ -65,16 +87,20 @@ public:
void timedOut(Error && ex) override { abort(); }; void timedOut(Error && ex) override { abort(); };
/**
* We prepend "a$" to the key name to ensure substitution goals
* happen before derivation goals.
*/
std::string key() override std::string key() override
{ {
/* "a$" ensures substitution goals happen before derivation
goals. */
return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath); return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath);
} }
void work() override; void work() override;
/* The states. */ /**
* The states.
*/
void init(); void init();
void tryNext(); void tryNext();
void gotInfo(); void gotInfo();
@ -82,7 +108,9 @@ public:
void tryToRun(); void tryToRun();
void finished(); void finished();
/* Callback used by the worker to write to the log. */ /**
* Callback used by the worker to write to the log.
*/
void handleChildOutput(int fd, std::string_view data) override; void handleChildOutput(int fd, std::string_view data) override;
void handleEOF(int fd) override; void handleEOF(int fd) override;

View file

@ -17,24 +17,29 @@ struct DerivationGoal;
struct PathSubstitutionGoal; struct PathSubstitutionGoal;
class DrvOutputSubstitutionGoal; class DrvOutputSubstitutionGoal;
/* Workaround for not being able to declare a something like /**
* Workaround for not being able to declare a something like
class PathSubstitutionGoal : public Goal; *
* ```c++
even when Goal is a complete type. * class PathSubstitutionGoal : public Goal;
* ```
This is still a static cast. The purpose of exporting it is to define it in * even when Goal is a complete type.
a place where `PathSubstitutionGoal` is concrete, and use it in a place where it *
is opaque. */ * This is still a static cast. The purpose of exporting it is to define it in
* a place where `PathSubstitutionGoal` is concrete, and use it in a place where it
* is opaque.
*/
GoalPtr upcast_goal(std::shared_ptr<PathSubstitutionGoal> subGoal); GoalPtr upcast_goal(std::shared_ptr<PathSubstitutionGoal> subGoal);
GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal); GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal);
typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point; typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
/* A mapping used to remember for each child process to what goal it /**
belongs, and file descriptors for receiving log data and output * A mapping used to remember for each child process to what goal it
path creation commands. */ * belongs, and file descriptors for receiving log data and output
* path creation commands.
*/
struct Child struct Child
{ {
WeakGoalPtr goal; WeakGoalPtr goal;
@ -42,14 +47,19 @@ struct Child
std::set<int> fds; std::set<int> fds;
bool respectTimeouts; bool respectTimeouts;
bool inBuildSlot; bool inBuildSlot;
steady_time_point lastOutput; /* time we last got output on stdout/stderr */ /**
* Time we last got output on stdout/stderr
*/
steady_time_point lastOutput;
steady_time_point timeStarted; steady_time_point timeStarted;
}; };
/* Forward definition. */ /* Forward definition. */
struct HookInstance; struct HookInstance;
/* The worker class. */ /**
* The worker class.
*/
class Worker class Worker
{ {
private: private:
@ -57,38 +67,58 @@ private:
/* Note: the worker should only have strong pointers to the /* Note: the worker should only have strong pointers to the
top-level goals. */ top-level goals. */
/* The top-level goals of the worker. */ /**
* The top-level goals of the worker.
*/
Goals topGoals; Goals topGoals;
/* Goals that are ready to do some work. */ /**
* Goals that are ready to do some work.
*/
WeakGoals awake; WeakGoals awake;
/* Goals waiting for a build slot. */ /**
* Goals waiting for a build slot.
*/
WeakGoals wantingToBuild; WeakGoals wantingToBuild;
/* Child processes currently running. */ /**
* Child processes currently running.
*/
std::list<Child> children; std::list<Child> children;
/* Number of build slots occupied. This includes local builds and /**
substitutions but not remote builds via the build hook. */ * Number of build slots occupied. This includes local builds and
* substitutions but not remote builds via the build hook.
*/
unsigned int nrLocalBuilds; unsigned int nrLocalBuilds;
/* Maps used to prevent multiple instantiations of a goal for the /**
same derivation / path. */ * Maps used to prevent multiple instantiations of a goal for the
* same derivation / path.
*/
std::map<StorePath, std::weak_ptr<DerivationGoal>> derivationGoals; std::map<StorePath, std::weak_ptr<DerivationGoal>> derivationGoals;
std::map<StorePath, std::weak_ptr<PathSubstitutionGoal>> substitutionGoals; std::map<StorePath, std::weak_ptr<PathSubstitutionGoal>> substitutionGoals;
std::map<DrvOutput, std::weak_ptr<DrvOutputSubstitutionGoal>> drvOutputSubstitutionGoals; std::map<DrvOutput, std::weak_ptr<DrvOutputSubstitutionGoal>> drvOutputSubstitutionGoals;
/* Goals waiting for busy paths to be unlocked. */ /**
* Goals waiting for busy paths to be unlocked.
*/
WeakGoals waitingForAnyGoal; WeakGoals waitingForAnyGoal;
/* Goals sleeping for a few seconds (polling a lock). */ /**
* Goals sleeping for a few seconds (polling a lock).
*/
WeakGoals waitingForAWhile; WeakGoals waitingForAWhile;
/* Last time the goals in `waitingForAWhile' where woken up. */ /**
* Last time the goals in `waitingForAWhile` where woken up.
*/
steady_time_point lastWokenUp; steady_time_point lastWokenUp;
/* Cache for pathContentsGood(). */ /**
* Cache for pathContentsGood().
*/
std::map<StorePath, bool> pathContentsGoodCache; std::map<StorePath, bool> pathContentsGoodCache;
public: public:
@ -97,17 +127,25 @@ public:
const Activity actDerivations; const Activity actDerivations;
const Activity actSubstitutions; const Activity actSubstitutions;
/* Set if at least one derivation had a BuildError (i.e. permanent /**
failure). */ * Set if at least one derivation had a BuildError (i.e. permanent
* failure).
*/
bool permanentFailure; bool permanentFailure;
/* Set if at least one derivation had a timeout. */ /**
* Set if at least one derivation had a timeout.
*/
bool timedOut; bool timedOut;
/* Set if at least one derivation fails with a hash mismatch. */ /**
* Set if at least one derivation fails with a hash mismatch.
*/
bool hashMismatch; bool hashMismatch;
/* Set if at least one derivation is not deterministic in check mode. */ /**
* Set if at least one derivation is not deterministic in check mode.
*/
bool checkMismatch; bool checkMismatch;
Store & store; Store & store;
@ -129,16 +167,22 @@ public:
uint64_t expectedNarSize = 0; uint64_t expectedNarSize = 0;
uint64_t doneNarSize = 0; uint64_t doneNarSize = 0;
/* Whether to ask the build hook if it can build a derivation. If /**
it answers with "decline-permanently", we don't try again. */ * Whether to ask the build hook if it can build a derivation. If
* it answers with "decline-permanently", we don't try again.
*/
bool tryBuildHook = true; bool tryBuildHook = true;
Worker(Store & store, Store & evalStore); Worker(Store & store, Store & evalStore);
~Worker(); ~Worker();
/* Make a goal (with caching). */ /**
* Make a goal (with caching).
*/
/* derivation goal */ /**
* derivation goal
*/
private: private:
std::shared_ptr<DerivationGoal> makeDerivationGoalCommon( std::shared_ptr<DerivationGoal> makeDerivationGoalCommon(
const StorePath & drvPath, const OutputsSpec & wantedOutputs, const StorePath & drvPath, const OutputsSpec & wantedOutputs,
@ -151,56 +195,80 @@ public:
const StorePath & drvPath, const BasicDerivation & drv, const StorePath & drvPath, const BasicDerivation & drv,
const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal); const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal);
/* substitution goal */ /**
* substitution goal
*/
std::shared_ptr<PathSubstitutionGoal> makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt); std::shared_ptr<PathSubstitutionGoal> makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
std::shared_ptr<DrvOutputSubstitutionGoal> makeDrvOutputSubstitutionGoal(const DrvOutput & id, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt); std::shared_ptr<DrvOutputSubstitutionGoal> makeDrvOutputSubstitutionGoal(const DrvOutput & id, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
/* Remove a dead goal. */ /**
* Remove a dead goal.
*/
void removeGoal(GoalPtr goal); void removeGoal(GoalPtr goal);
/* Wake up a goal (i.e., there is something for it to do). */ /**
* Wake up a goal (i.e., there is something for it to do).
*/
void wakeUp(GoalPtr goal); void wakeUp(GoalPtr goal);
/* Return the number of local build and substitution processes /**
currently running (but not remote builds via the build * Return the number of local build and substitution processes
hook). */ * currently running (but not remote builds via the build
* hook).
*/
unsigned int getNrLocalBuilds(); unsigned int getNrLocalBuilds();
/* Registers a running child process. `inBuildSlot' means that /**
the process counts towards the jobs limit. */ * Registers a running child process. `inBuildSlot` means that
* the process counts towards the jobs limit.
*/
void childStarted(GoalPtr goal, const std::set<int> & fds, void childStarted(GoalPtr goal, const std::set<int> & fds,
bool inBuildSlot, bool respectTimeouts); bool inBuildSlot, bool respectTimeouts);
/* Unregisters a running child process. `wakeSleepers' should be /**
false if there is no sense in waking up goals that are sleeping * Unregisters a running child process. `wakeSleepers` should be
because they can't run yet (e.g., there is no free build slot, * false if there is no sense in waking up goals that are sleeping
or the hook would still say `postpone'). */ * because they can't run yet (e.g., there is no free build slot,
* or the hook would still say `postpone`).
*/
void childTerminated(Goal * goal, bool wakeSleepers = true); void childTerminated(Goal * goal, bool wakeSleepers = true);
/* Put `goal' to sleep until a build slot becomes available (which /**
might be right away). */ * Put `goal` to sleep until a build slot becomes available (which
* might be right away).
*/
void waitForBuildSlot(GoalPtr goal); void waitForBuildSlot(GoalPtr goal);
/* Wait for any goal to finish. Pretty indiscriminate way to /**
wait for some resource that some other goal is holding. */ * Wait for any goal to finish. Pretty indiscriminate way to
* wait for some resource that some other goal is holding.
*/
void waitForAnyGoal(GoalPtr goal); void waitForAnyGoal(GoalPtr goal);
/* Wait for a few seconds and then retry this goal. Used when /**
waiting for a lock held by another process. This kind of * Wait for a few seconds and then retry this goal. Used when
polling is inefficient, but POSIX doesn't really provide a way * waiting for a lock held by another process. This kind of
to wait for multiple locks in the main select() loop. */ * polling is inefficient, but POSIX doesn't really provide a way
* to wait for multiple locks in the main select() loop.
*/
void waitForAWhile(GoalPtr goal); void waitForAWhile(GoalPtr goal);
/* Loop until the specified top-level goals have finished. */ /**
* Loop until the specified top-level goals have finished.
*/
void run(const Goals & topGoals); void run(const Goals & topGoals);
/* Wait for input to become available. */ /**
* Wait for input to become available.
*/
void waitForInput(); void waitForInput();
unsigned int exitStatus(); unsigned int exitStatus();
/* Check whether the given valid path exists and has the right /**
contents. */ * Check whether the given valid path exists and has the right
* contents.
*/
bool pathContentsGood(const StorePath & path); bool pathContentsGood(const StorePath & path);
void markContentsGood(const StorePath & path); void markContentsGood(const StorePath & path);

View file

@ -12,8 +12,10 @@ struct Key
std::string name; std::string name;
std::string key; std::string key;
/* Construct Key from a string in the format /**
<name>:<key-in-base64>. */ * Construct Key from a string in the format
* <name>:<key-in-base64>.
*/
Key(std::string_view s); Key(std::string_view s);
std::string to_string() const; std::string to_string() const;
@ -29,7 +31,9 @@ struct SecretKey : Key
{ {
SecretKey(std::string_view s); SecretKey(std::string_view s);
/* Return a detached signature of the given string. */ /**
* Return a detached signature of the given string.
*/
std::string signDetached(std::string_view s) const; std::string signDetached(std::string_view s) const;
PublicKey toPublicKey() const; PublicKey toPublicKey() const;
@ -53,8 +57,10 @@ private:
typedef std::map<std::string, PublicKey> PublicKeys; typedef std::map<std::string, PublicKey> PublicKeys;
/* Return true iff sig is a correct signature over data using one /**
of the given public keys. */ * @return true iff sig is a correct signature over data using one
* of the given public keys.
*/
bool verifyDetached(const std::string & data, const std::string & sig, bool verifyDetached(const std::string & data, const std::string & sig,
const PublicKeys & publicKeys); const PublicKeys & publicKeys);

View file

@ -258,8 +258,14 @@ struct DerivationType : _DerivationTypeRaw {
struct BasicDerivation struct BasicDerivation
{ {
DerivationOutputs outputs; /* keyed on symbolic IDs */ /**
StorePathSet inputSrcs; /* inputs that are sources */ * keyed on symbolic IDs
*/
DerivationOutputs outputs;
/**
* inputs that are sources
*/
StorePathSet inputSrcs;
std::string platform; std::string platform;
Path builder; Path builder;
Strings args; Strings args;
@ -429,12 +435,12 @@ void operator |= (DrvHash::Kind & self, const DrvHash::Kind & other) noexcept;
* *
* A fixed-output derivation is a derivation whose outputs have a * A fixed-output derivation is a derivation whose outputs have a
* specified content hash and hash algorithm. (Currently they must have * specified content hash and hash algorithm. (Currently they must have
* exactly one output (`out'), which is specified using the `outputHash' * exactly one output (`out`), which is specified using the `outputHash`
* and `outputHashAlgo' attributes, but the algorithm doesn't assume * and `outputHashAlgo` attributes, but the algorithm doesn't assume
* this.) We don't want changes to such derivations to propagate upwards * this.) We don't want changes to such derivations to propagate upwards
* through the dependency graph, changing output paths everywhere. * through the dependency graph, changing output paths everywhere.
* *
* For instance, if we change the url in a call to the `fetchurl' * For instance, if we change the url in a call to the `fetchurl`
* function, we do not want to rebuild everything depending on it---after * function, we do not want to rebuild everything depending on it---after
* all, (the hash of) the file being downloaded is unchanged. So the * all, (the hash of) the file being downloaded is unchanged. So the
* *output paths* should not change. On the other hand, the *derivation * *output paths* should not change. On the other hand, the *derivation

View file

@ -5,8 +5,10 @@
namespace nix { namespace nix {
/* An abstract class for accessing a filesystem-like structure, such /**
as a (possibly remote) Nix store or the contents of a NAR file. */ * An abstract class for accessing a filesystem-like structure, such
* as a (possibly remote) Nix store or the contents of a NAR file.
*/
class FSAccessor class FSAccessor
{ {
public: public:
@ -15,8 +17,17 @@ public:
struct Stat struct Stat
{ {
Type type = tMissing; Type type = tMissing;
uint64_t fileSize = 0; // regular files only /**
* regular files only
*/
uint64_t fileSize = 0;
/**
* regular files only
*/
bool isExecutable = false; // regular files only bool isExecutable = false; // regular files only
/**
* regular files only
*/
uint64_t narOffset = 0; // regular files only uint64_t narOffset = 0; // regular files only
}; };

View file

@ -12,19 +12,20 @@ typedef std::unordered_map<StorePath, std::unordered_set<std::string>> Roots;
struct GCOptions struct GCOptions
{ {
/* Garbage collector operation: /**
* Garbage collector operation:
- `gcReturnLive': return the set of paths reachable from *
(i.e. in the closure of) the roots. * - `gcReturnLive`: return the set of paths reachable from
* (i.e. in the closure of) the roots.
- `gcReturnDead': return the set of paths not reachable from *
the roots. * - `gcReturnDead`: return the set of paths not reachable from
* the roots.
- `gcDeleteDead': actually delete the latter set. *
* - `gcDeleteDead`: actually delete the latter set.
- `gcDeleteSpecific': delete the paths listed in *
`pathsToDelete', insofar as they are not reachable. * - `gcDeleteSpecific`: delete the paths listed in
*/ * `pathsToDelete`, insofar as they are not reachable.
*/
typedef enum { typedef enum {
gcReturnLive, gcReturnLive,
gcReturnDead, gcReturnDead,
@ -34,28 +35,38 @@ struct GCOptions
GCAction action{gcDeleteDead}; GCAction action{gcDeleteDead};
/* If `ignoreLiveness' is set, then reachability from the roots is /**
ignored (dangerous!). However, the paths must still be * If `ignoreLiveness` is set, then reachability from the roots is
unreferenced *within* the store (i.e., there can be no other * ignored (dangerous!). However, the paths must still be
store paths that depend on them). */ * unreferenced *within* the store (i.e., there can be no other
* store paths that depend on them).
*/
bool ignoreLiveness{false}; bool ignoreLiveness{false};
/* For `gcDeleteSpecific', the paths to delete. */ /**
* For `gcDeleteSpecific`, the paths to delete.
*/
StorePathSet pathsToDelete; StorePathSet pathsToDelete;
/* Stop after at least `maxFreed' bytes have been freed. */ /**
* Stop after at least `maxFreed` bytes have been freed.
*/
uint64_t maxFreed{std::numeric_limits<uint64_t>::max()}; uint64_t maxFreed{std::numeric_limits<uint64_t>::max()};
}; };
struct GCResults struct GCResults
{ {
/* Depending on the action, the GC roots, or the paths that would /**
be or have been deleted. */ * Depending on the action, the GC roots, or the paths that would
* be or have been deleted.
*/
PathSet paths; PathSet paths;
/* For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the /**
number of bytes that would be or was freed. */ * For `gcReturnDead`, `gcDeleteDead` and `gcDeleteSpecific`, the
* number of bytes that would be or was freed.
*/
uint64_t bytesFreed = 0; uint64_t bytesFreed = 0;
}; };
@ -64,21 +75,27 @@ struct GcStore : public virtual Store
{ {
inline static std::string operationName = "Garbage collection"; inline static std::string operationName = "Garbage collection";
/* Add an indirect root, which is merely a symlink to `path' from /**
/nix/var/nix/gcroots/auto/<hash of `path'>. `path' is supposed * Add an indirect root, which is merely a symlink to `path` from
to be a symlink to a store path. The garbage collector will * `/nix/var/nix/gcroots/auto/<hash of path>`. `path` is supposed
automatically remove the indirect root when it finds that * to be a symlink to a store path. The garbage collector will
`path' has disappeared. */ * automatically remove the indirect root when it finds that
* `path` has disappeared.
*/
virtual void addIndirectRoot(const Path & path) = 0; virtual void addIndirectRoot(const Path & path) = 0;
/* Find the roots of the garbage collector. Each root is a pair /**
(link, storepath) where `link' is the path of the symlink * Find the roots of the garbage collector. Each root is a pair
outside of the Nix store that point to `storePath'. If * `(link, storepath)` where `link` is the path of the symlink
'censor' is true, privacy-sensitive information about roots * outside of the Nix store that point to `storePath`. If
found in /proc is censored. */ * `censor` is true, privacy-sensitive information about roots
* found in `/proc` is censored.
*/
virtual Roots findRoots(bool censor) = 0; virtual Roots findRoots(bool censor) = 0;
/* Perform a garbage collection. */ /**
* Perform a garbage collection.
*/
virtual void collectGarbage(const GCOptions & options, GCResults & results) = 0; virtual void collectGarbage(const GCOptions & options, GCResults & results) = 0;
}; };

View file

@ -72,30 +72,46 @@ public:
Path nixPrefix; Path nixPrefix;
/* The directory where we store sources and derived files. */ /**
* The directory where we store sources and derived files.
*/
Path nixStore; Path nixStore;
Path nixDataDir; /* !!! fix */ Path nixDataDir; /* !!! fix */
/* The directory where we log various operations. */ /**
* The directory where we log various operations.
*/
Path nixLogDir; Path nixLogDir;
/* The directory where state is stored. */ /**
* The directory where state is stored.
*/
Path nixStateDir; Path nixStateDir;
/* The directory where system configuration files are stored. */ /**
* The directory where system configuration files are stored.
*/
Path nixConfDir; Path nixConfDir;
/* A list of user configuration files to load. */ /**
* A list of user configuration files to load.
*/
std::vector<Path> nixUserConfFiles; std::vector<Path> nixUserConfFiles;
/* The directory where the main programs are stored. */ /**
* The directory where the main programs are stored.
*/
Path nixBinDir; Path nixBinDir;
/* The directory where the man pages are stored. */ /**
* The directory where the man pages are stored.
*/
Path nixManDir; Path nixManDir;
/* File name of the socket the daemon listens to. */ /**
* File name of the socket the daemon listens to.
*/
Path nixDaemonSocketFile; Path nixDaemonSocketFile;
Setting<std::string> storeUri{this, getEnv("NIX_REMOTE").value_or("auto"), "store", Setting<std::string> storeUri{this, getEnv("NIX_REMOTE").value_or("auto"), "store",
@ -121,7 +137,9 @@ public:
)", )",
{"build-fallback"}}; {"build-fallback"}};
/* Whether to show build log output in real time. */ /**
* Whether to show build log output in real time.
*/
bool verboseBuild = true; bool verboseBuild = true;
Setting<size_t> logLines{this, 10, "log-lines", Setting<size_t> logLines{this, 10, "log-lines",
@ -157,8 +175,10 @@ public:
)", )",
{"build-cores"}, false}; {"build-cores"}, false};
/* Read-only mode. Don't copy stuff to the store, don't change /**
the database. */ * Read-only mode. Don't copy stuff to the store, don't change
* the database.
*/
bool readOnlyMode = false; bool readOnlyMode = false;
Setting<std::string> thisSystem{ Setting<std::string> thisSystem{
@ -458,7 +478,9 @@ public:
)", )",
{"env-keep-derivations"}}; {"env-keep-derivations"}};
/* Whether to lock the Nix client and worker to the same CPU. */ /**
* Whether to lock the Nix client and worker to the same CPU.
*/
bool lockCPU; bool lockCPU;
Setting<SandboxMode> sandboxMode{ Setting<SandboxMode> sandboxMode{
@ -997,8 +1019,10 @@ public:
// FIXME: don't use a global variable. // FIXME: don't use a global variable.
extern Settings settings; extern Settings settings;
/* This should be called after settings are initialized, but before /**
anything else */ * This should be called after settings are initialized, but before
* anything else
*/
void initPlugins(); void initPlugins();
void loadConfFile(); void loadConfFile();
@ -1008,12 +1032,16 @@ std::vector<Path> getUserConfigFiles();
extern const std::string nixVersion; extern const std::string nixVersion;
/* NB: This is not sufficient. You need to call initNix() */ /**
* NB: This is not sufficient. You need to call initNix()
*/
void initLibStore(); void initLibStore();
/* It's important to initialize before doing _anything_, which is why we /**
call upon the programmer to handle this correctly. However, we only add * It's important to initialize before doing _anything_, which is why we
this in a key locations, so as not to litter the code. */ * call upon the programmer to handle this correctly. However, we only add
* this in a key locations, so as not to litter the code.
*/
void assertLibStoreInitialized(); void assertLibStoreInitialized();
} }

View file

@ -48,7 +48,9 @@ public:
void narFromPath(const StorePath & path, Sink & sink) override; void narFromPath(const StorePath & path, Sink & sink) override;
ref<FSAccessor> getFSAccessor() override; ref<FSAccessor> getFSAccessor() override;
/* Register a permanent GC root. */ /**
* Register a permanent GC root.
*/
Path addPermRoot(const StorePath & storePath, const Path & gcRoot); Path addPermRoot(const StorePath & storePath, const Path & gcRoot);
virtual Path getRealStoreDir() { return realStoreDir; } virtual Path getRealStoreDir() { return realStoreDir; }

View file

@ -19,10 +19,14 @@
namespace nix { namespace nix {
/* Nix store and database schema version. Version 1 (or 0) was Nix <= /**
0.7. Version 2 was Nix 0.8 and 0.9. Version 3 is Nix 0.10. * Nix store and database schema version.
Version 4 is Nix 0.11. Version 5 is Nix 0.12-0.16. Version 6 is *
Nix 1.0. Version 7 is Nix 1.3. Version 10 is 2.0. */ * Version 1 (or 0) was Nix <=
* 0.7. Version 2 was Nix 0.8 and 0.9. Version 3 is Nix 0.10.
* Version 4 is Nix 0.11. Version 5 is Nix 0.12-0.16. Version 6 is
* Nix 1.0. Version 7 is Nix 1.3. Version 10 is 2.0.
*/
const int nixSchemaVersion = 10; const int nixSchemaVersion = 10;
@ -51,30 +55,40 @@ class LocalStore : public virtual LocalStoreConfig, public virtual LocalFSStore,
{ {
private: private:
/* Lock file used for upgrading. */ /**
* Lock file used for upgrading.
*/
AutoCloseFD globalLock; AutoCloseFD globalLock;
struct State struct State
{ {
/* The SQLite database object. */ /**
* The SQLite database object.
*/
SQLite db; SQLite db;
struct Stmts; struct Stmts;
std::unique_ptr<Stmts> stmts; std::unique_ptr<Stmts> stmts;
/* The last time we checked whether to do an auto-GC, or an /**
auto-GC finished. */ * The last time we checked whether to do an auto-GC, or an
* auto-GC finished.
*/
std::chrono::time_point<std::chrono::steady_clock> lastGCCheck; std::chrono::time_point<std::chrono::steady_clock> lastGCCheck;
/* Whether auto-GC is running. If so, get gcFuture to wait for /**
the GC to finish. */ * Whether auto-GC is running. If so, get gcFuture to wait for
* the GC to finish.
*/
bool gcRunning = false; bool gcRunning = false;
std::shared_future<void> gcFuture; std::shared_future<void> gcFuture;
/* How much disk space was available after the previous /**
auto-GC. If the current available disk space is below * How much disk space was available after the previous
minFree but not much below availAfterGC, then there is no * auto-GC. If the current available disk space is below
point in starting a new GC. */ * minFree but not much below availAfterGC, then there is no
* point in starting a new GC.
*/
uint64_t availAfterGC = std::numeric_limits<uint64_t>::max(); uint64_t availAfterGC = std::numeric_limits<uint64_t>::max();
std::unique_ptr<PublicKeys> publicKeys; std::unique_ptr<PublicKeys> publicKeys;
@ -97,11 +111,15 @@ private:
public: public:
// Hack for build-remote.cc. /**
* Hack for build-remote.cc.
*/
PathSet locksHeld; PathSet locksHeld;
/* Initialise the local store, upgrading the schema if /**
necessary. */ * Initialise the local store, upgrading the schema if
* necessary.
*/
LocalStore(const Params & params); LocalStore(const Params & params);
LocalStore(std::string scheme, std::string path, const Params & params); LocalStore(std::string scheme, std::string path, const Params & params);
@ -110,7 +128,9 @@ public:
static std::set<std::string> uriSchemes() static std::set<std::string> uriSchemes()
{ return {}; } { return {}; }
/* Implementations of abstract store API methods. */ /**
* Implementations of abstract store API methods.
*/
std::string getUri() override; std::string getUri() override;
@ -155,13 +175,19 @@ private:
void createTempRootsFile(); void createTempRootsFile();
/* The file to which we write our temporary roots. */ /**
* The file to which we write our temporary roots.
*/
Sync<AutoCloseFD> _fdTempRoots; Sync<AutoCloseFD> _fdTempRoots;
/* The global GC lock. */ /**
* The global GC lock.
*/
Sync<AutoCloseFD> _fdGCLock; Sync<AutoCloseFD> _fdGCLock;
/* Connection to the garbage collector. */ /**
* Connection to the garbage collector.
*/
Sync<AutoCloseFD> _fdRootsSocket; Sync<AutoCloseFD> _fdRootsSocket;
public: public:
@ -180,24 +206,30 @@ public:
void collectGarbage(const GCOptions & options, GCResults & results) override; void collectGarbage(const GCOptions & options, GCResults & results) override;
/* Optimise the disk space usage of the Nix store by hard-linking /**
files with the same contents. */ * Optimise the disk space usage of the Nix store by hard-linking
* files with the same contents.
*/
void optimiseStore(OptimiseStats & stats); void optimiseStore(OptimiseStats & stats);
void optimiseStore() override; void optimiseStore() override;
/* Optimise a single store path. Optionally, test the encountered /**
symlinks for corruption. */ * Optimise a single store path. Optionally, test the encountered
* symlinks for corruption.
*/
void optimisePath(const Path & path, RepairFlag repair); void optimisePath(const Path & path, RepairFlag repair);
bool verifyStore(bool checkContents, RepairFlag repair) override; bool verifyStore(bool checkContents, RepairFlag repair) override;
/* Register the validity of a path, i.e., that `path' exists, that /**
the paths referenced by it exists, and in the case of an output * Register the validity of a path, i.e., that `path` exists, that
path of a derivation, that it has been produced by a successful * the paths referenced by it exists, and in the case of an output
execution of the derivation (or something equivalent). Also * path of a derivation, that it has been produced by a successful
register the hash of the file system contents of the path. The * execution of the derivation (or something equivalent). Also
hash must be a SHA-256 hash. */ * register the hash of the file system contents of the path. The
* hash must be a SHA-256 hash.
*/
void registerValidPath(const ValidPathInfo & info); void registerValidPath(const ValidPathInfo & info);
void registerValidPaths(const ValidPathInfos & infos); void registerValidPaths(const ValidPathInfos & infos);
@ -212,12 +244,16 @@ public:
void addSignatures(const StorePath & storePath, const StringSet & sigs) override; void addSignatures(const StorePath & storePath, const StringSet & sigs) override;
/* If free disk space in /nix/store if below minFree, delete /**
garbage until it exceeds maxFree. */ * If free disk space in /nix/store if below minFree, delete
* garbage until it exceeds maxFree.
*/
void autoGC(bool sync = true); void autoGC(bool sync = true);
/* Register the store path 'output' as the output named 'outputName' of /**
derivation 'deriver'. */ * Register the store path 'output' as the output named 'outputName' of
* derivation 'deriver'.
*/
void registerDrvOutput(const Realisation & info) override; void registerDrvOutput(const Realisation & info) override;
void registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) override; void registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) override;
void cacheDrvOutputMapping( void cacheDrvOutputMapping(
@ -247,7 +283,9 @@ private:
void invalidatePath(State & state, const StorePath & path); void invalidatePath(State & state, const StorePath & path);
/* Delete a path from the Nix store. */ /**
* Delete a path from the Nix store.
*/
void invalidatePathChecked(const StorePath & path); void invalidatePathChecked(const StorePath & path);
void verifyPath(const Path & path, const StringSet & store, void verifyPath(const Path & path, const StringSet & store,
@ -280,8 +318,10 @@ private:
bool isValidPath_(State & state, const StorePath & path); bool isValidPath_(State & state, const StorePath & path);
void queryReferrers(State & state, const StorePath & path, StorePathSet & referrers); void queryReferrers(State & state, const StorePath & path, StorePathSet & referrers);
/* Add signatures to a ValidPathInfo or Realisation using the secret keys /**
specified by the secret-key-files option. */ * Add signatures to a ValidPathInfo or Realisation using the secret keys
* specified by the secret-key-files option.
*/
void signPathInfo(ValidPathInfo & info); void signPathInfo(ValidPathInfo & info);
void signRealisation(Realisation &); void signRealisation(Realisation &);
@ -311,18 +351,23 @@ typedef std::pair<dev_t, ino_t> Inode;
typedef std::set<Inode> InodesSeen; typedef std::set<Inode> InodesSeen;
/* "Fix", or canonicalise, the meta-data of the files in a store path /**
after it has been built. In particular: * "Fix", or canonicalise, the meta-data of the files in a store path
- the last modification date on each file is set to 1 (i.e., * after it has been built. In particular:
00:00:01 1/1/1970 UTC) *
- the permissions are set of 444 or 555 (i.e., read-only with or * - the last modification date on each file is set to 1 (i.e.,
without execute permission; setuid bits etc. are cleared) * 00:00:01 1/1/1970 UTC)
- the owner and group are set to the Nix user and group, if we're *
running as root. * - the permissions are set of 444 or 555 (i.e., read-only with or
If uidRange is not empty, this function will throw an error if it * without execute permission; setuid bits etc. are cleared)
encounters files owned by a user outside of the closed interval *
[uidRange->first, uidRange->second]. * - the owner and group are set to the Nix user and group, if we're
*/ * running as root.
*
* If uidRange is not empty, this function will throw an error if it
* encounters files owned by a user outside of the closed interval
* [uidRange->first, uidRange->second].
*/
void canonicalisePathMetaData( void canonicalisePathMetaData(
const Path & path, const Path & path,
std::optional<std::pair<uid_t, uid_t>> uidRange, std::optional<std::pair<uid_t, uid_t>> uidRange,

View file

@ -13,14 +13,18 @@ struct UserLock
{ {
virtual ~UserLock() { } virtual ~UserLock() { }
/* Get the first and last UID. */ /**
* Get the first and last UID.
*/
std::pair<uid_t, uid_t> getUIDRange() std::pair<uid_t, uid_t> getUIDRange()
{ {
auto first = getUID(); auto first = getUID();
return {first, first + getUIDCount() - 1}; return {first, first + getUIDCount() - 1};
} }
/* Get the first UID. */ /**
* Get the first UID.
*/
virtual uid_t getUID() = 0; virtual uid_t getUID() = 0;
virtual uid_t getUIDCount() = 0; virtual uid_t getUIDCount() = 0;
@ -30,8 +34,10 @@ struct UserLock
virtual std::vector<gid_t> getSupplementaryGIDs() = 0; virtual std::vector<gid_t> getSupplementaryGIDs() = 0;
}; };
/* Acquire a user lock for a UID range of size `nrIds`. Note that this /**
may return nullptr if no user is available. */ * Acquire a user lock for a UID range of size `nrIds`. Note that this
* may return nullptr if no user is available.
*/
std::unique_ptr<UserLock> acquireUserLock(uid_t nrIds, bool useUserNamespace); std::unique_ptr<UserLock> acquireUserLock(uid_t nrIds, bool useUserNamespace);
bool useBuildUsers(); bool useBuildUsers();

View file

@ -10,8 +10,10 @@ struct LogStore : public virtual Store
{ {
inline static std::string operationName = "Build log storage and retrieval"; inline static std::string operationName = "Build log storage and retrieval";
/* Return the build log of the specified store path, if available, /**
or null otherwise. */ * Return the build log of the specified store path, if available,
* or null otherwise.
*/
std::optional<std::string> getBuildLog(const StorePath & path); std::optional<std::string> getBuildLog(const StorePath & path);
virtual std::optional<std::string> getBuildLogExact(const StorePath & path) = 0; virtual std::optional<std::string> getBuildLogExact(const StorePath & path) = 0;

View file

@ -10,24 +10,30 @@ namespace nix {
struct Source; struct Source;
/* Return an object that provides access to the contents of a NAR /**
file. */ * Return an object that provides access to the contents of a NAR
* file.
*/
ref<FSAccessor> makeNarAccessor(std::string && nar); ref<FSAccessor> makeNarAccessor(std::string && nar);
ref<FSAccessor> makeNarAccessor(Source & source); ref<FSAccessor> makeNarAccessor(Source & source);
/* Create a NAR accessor from a NAR listing (in the format produced by /**
listNar()). The callback getNarBytes(offset, length) is used by the * Create a NAR accessor from a NAR listing (in the format produced by
readFile() method of the accessor to get the contents of files * listNar()). The callback getNarBytes(offset, length) is used by the
inside the NAR. */ * readFile() method of the accessor to get the contents of files
* inside the NAR.
*/
typedef std::function<std::string(uint64_t, uint64_t)> GetNarBytes; typedef std::function<std::string(uint64_t, uint64_t)> GetNarBytes;
ref<FSAccessor> makeLazyNarAccessor( ref<FSAccessor> makeLazyNarAccessor(
const std::string & listing, const std::string & listing,
GetNarBytes getNarBytes); GetNarBytes getNarBytes);
/* Write a JSON representation of the contents of a NAR (except file /**
contents). */ * Write a JSON representation of the contents of a NAR (except file
* contents).
*/
nlohmann::json listNar(ref<FSAccessor> accessor, const Path & path, bool recurse); nlohmann::json listNar(ref<FSAccessor> accessor, const Path & path, bool recurse);
} }

View file

@ -43,8 +43,10 @@ public:
const std::string & uri, const DrvOutput & id) = 0; const std::string & uri, const DrvOutput & id) = 0;
}; };
/* Return a singleton cache object that can be used concurrently by /**
multiple threads. */ * Return a singleton cache object that can be used concurrently by
* multiple threads.
*/
ref<NarInfoDiskCache> getNarInfoDiskCache(); ref<NarInfoDiskCache> getNarInfoDiskCache();
ref<NarInfoDiskCache> getTestNarInfoDiskCache(Path dbPath); ref<NarInfoDiskCache> getTestNarInfoDiskCache(Path dbPath);

View file

@ -19,8 +19,14 @@ struct SubstitutablePathInfo
{ {
std::optional<StorePath> deriver; std::optional<StorePath> deriver;
StorePathSet references; StorePathSet references;
uint64_t downloadSize; /* 0 = unknown or inapplicable */ /**
uint64_t narSize; /* 0 = unknown */ * 0 = unknown or inapplicable
*/
uint64_t downloadSize;
/**
* 0 = unknown
*/
uint64_t narSize;
}; };
typedef std::map<StorePath, SubstitutablePathInfo> SubstitutablePathInfos; typedef std::map<StorePath, SubstitutablePathInfo> SubstitutablePathInfos;
@ -30,35 +36,40 @@ struct ValidPathInfo
{ {
StorePath path; StorePath path;
std::optional<StorePath> deriver; std::optional<StorePath> deriver;
// TODO document this /**
* \todo document this
*/
Hash narHash; Hash narHash;
StorePathSet references; StorePathSet references;
time_t registrationTime = 0; time_t registrationTime = 0;
uint64_t narSize = 0; // 0 = unknown uint64_t narSize = 0; // 0 = unknown
uint64_t id; // internal use only uint64_t id; // internal use only
/* Whether the path is ultimately trusted, that is, it's a /**
derivation output that was built locally. */ * Whether the path is ultimately trusted, that is, it's a
* derivation output that was built locally.
*/
bool ultimate = false; bool ultimate = false;
StringSet sigs; // note: not necessarily verified StringSet sigs; // note: not necessarily verified
/* If non-empty, an assertion that the path is content-addressed, /**
i.e., that the store path is computed from a cryptographic hash * If non-empty, an assertion that the path is content-addressed,
of the contents of the path, plus some other bits of data like * i.e., that the store path is computed from a cryptographic hash
the "name" part of the path. Such a path doesn't need * of the contents of the path, plus some other bits of data like
signatures, since we don't have to trust anybody's claim that * the "name" part of the path. Such a path doesn't need
the path is the output of a particular derivation. (In the * signatures, since we don't have to trust anybody's claim that
extensional store model, we have to trust that the *contents* * the path is the output of a particular derivation. (In the
of an output path of a derivation were actually produced by * extensional store model, we have to trust that the *contents*
that derivation. In the intensional model, we have to trust * of an output path of a derivation were actually produced by
that a particular output path was produced by a derivation; the * that derivation. In the intensional model, we have to trust
path then implies the contents.) * that a particular output path was produced by a derivation; the
* path then implies the contents.)
Ideally, the content-addressability assertion would just be a Boolean, *
and the store path would be computed from the name component, narHash * Ideally, the content-addressability assertion would just be a Boolean,
and references. However, we support many types of content addresses. * and the store path would be computed from the name component, narHash
*/ * and references. However, we support many types of content addresses.
*/
std::optional<ContentAddress> ca; std::optional<ContentAddress> ca;
bool operator == (const ValidPathInfo & i) const bool operator == (const ValidPathInfo & i) const
@ -69,27 +80,35 @@ struct ValidPathInfo
&& references == i.references; && references == i.references;
} }
/* Return a fingerprint of the store path to be used in binary /**
cache signatures. It contains the store path, the base-32 * Return a fingerprint of the store path to be used in binary
SHA-256 hash of the NAR serialisation of the path, the size of * cache signatures. It contains the store path, the base-32
the NAR, and the sorted references. The size field is strictly * SHA-256 hash of the NAR serialisation of the path, the size of
speaking superfluous, but might prevent endless/excessive data * the NAR, and the sorted references. The size field is strictly
attacks. */ * speaking superfluous, but might prevent endless/excessive data
* attacks.
*/
std::string fingerprint(const Store & store) const; std::string fingerprint(const Store & store) const;
void sign(const Store & store, const SecretKey & secretKey); void sign(const Store & store, const SecretKey & secretKey);
/* Return true iff the path is verifiably content-addressed. */ /**
* @return true iff the path is verifiably content-addressed.
*/
bool isContentAddressed(const Store & store) const; bool isContentAddressed(const Store & store) const;
static const size_t maxSigs = std::numeric_limits<size_t>::max(); static const size_t maxSigs = std::numeric_limits<size_t>::max();
/* Return the number of signatures on this .narinfo that were /**
produced by one of the specified keys, or maxSigs if the path * Return the number of signatures on this .narinfo that were
is content-addressed. */ * produced by one of the specified keys, or maxSigs if the path
* is content-addressed.
*/
size_t checkSignatures(const Store & store, const PublicKeys & publicKeys) const; size_t checkSignatures(const Store & store, const PublicKeys & publicKeys) const;
/* Verify a single signature. */ /**
* Verify a single signature.
*/
bool checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const; bool checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const;
Strings shortRefs() const; Strings shortRefs() const;

View file

@ -6,13 +6,14 @@
namespace nix { namespace nix {
/* This is a deprecated old type just for use by the old CLI, and older /**
versions of the RPC protocols. In new code don't use it; you want * This is a deprecated old type just for use by the old CLI, and older
`DerivedPath` instead. * versions of the RPC protocols. In new code don't use it; you want
* `DerivedPath` instead.
`DerivedPath` is better because it handles more cases, and does so more *
explicitly without devious punning tricks. * `DerivedPath` is better because it handles more cases, and does so more
*/ * explicitly without devious punning tricks.
*/
struct StorePathWithOutputs struct StorePathWithOutputs
{ {
StorePath path; StorePath path;
@ -31,9 +32,11 @@ std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s);
class Store; class Store;
/* Split a string specifying a derivation and a set of outputs /**
(/nix/store/hash-foo!out1,out2,...) into the derivation path * Split a string specifying a derivation and a set of outputs
and the outputs. */ * (/nix/store/hash-foo!out1,out2,...) into the derivation path
* and the outputs.
*/
StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view pathWithOutputs); StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view pathWithOutputs);
StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs); StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs);

View file

@ -5,12 +5,16 @@
namespace nix { namespace nix {
/* Open (possibly create) a lock file and return the file descriptor. /**
-1 is returned if create is false and the lock could not be opened * Open (possibly create) a lock file and return the file descriptor.
because it doesn't exist. Any other error throws an exception. */ * -1 is returned if create is false and the lock could not be opened
* because it doesn't exist. Any other error throws an exception.
*/
AutoCloseFD openLockFile(const Path & path, bool create); AutoCloseFD openLockFile(const Path & path, bool create);
/* Delete an open lock file. */ /**
* Delete an open lock file.
*/
void deleteLockFile(const Path & path, int fd); void deleteLockFile(const Path & path, int fd);
enum LockType { ltRead, ltWrite, ltNone }; enum LockType { ltRead, ltWrite, ltNone };

View file

@ -1,7 +1,7 @@
#pragma once #pragma once
///@file ///@file
#include "types.hh" #include "types.hh"
#include "pathlocks.hh" #include "pathlocks.hh"
#include <time.h> #include <time.h>
@ -24,9 +24,11 @@ struct Generation
typedef std::list<Generation> Generations; typedef std::list<Generation> Generations;
/* Returns the list of currently present generations for the specified /**
profile, sorted by generation number. Also returns the number of * Returns the list of currently present generations for the specified
the current generation. */ * profile, sorted by generation number. Also returns the number of
* the current generation.
*/
std::pair<Generations, std::optional<GenerationNumber>> findGenerations(Path profile); std::pair<Generations, std::optional<GenerationNumber>> findGenerations(Path profile);
class LocalFSStore; class LocalFSStore;
@ -47,26 +49,32 @@ void deleteGenerationsOlderThan(const Path & profile, std::string_view timeSpec,
void switchLink(Path link, Path target); void switchLink(Path link, Path target);
/* Roll back a profile to the specified generation, or to the most /**
recent one older than the current. */ * Roll back a profile to the specified generation, or to the most
* recent one older than the current.
*/
void switchGeneration( void switchGeneration(
const Path & profile, const Path & profile,
std::optional<GenerationNumber> dstGen, std::optional<GenerationNumber> dstGen,
bool dryRun); bool dryRun);
/* Ensure exclusive access to a profile. Any command that modifies /**
the profile first acquires this lock. */ * Ensure exclusive access to a profile. Any command that modifies
* the profile first acquires this lock.
*/
void lockProfile(PathLocks & lock, const Path & profile); void lockProfile(PathLocks & lock, const Path & profile);
/* Optimistic locking is used by long-running operations like `nix-env /**
-i'. Instead of acquiring the exclusive lock for the entire * Optimistic locking is used by long-running operations like `nix-env
duration of the operation, we just perform the operation * -i'. Instead of acquiring the exclusive lock for the entire
optimistically (without an exclusive lock), and check at the end * duration of the operation, we just perform the operation
whether the profile changed while we were busy (i.e., the symlink * optimistically (without an exclusive lock), and check at the end
target changed). If so, the operation is restarted. Restarting is * whether the profile changed while we were busy (i.e., the symlink
generally cheap, since the build results are still in the Nix * target changed). If so, the operation is restarted. Restarting is
store. Most of the time, only the user environment has to be * generally cheap, since the build results are still in the Nix
rebuilt. */ * store. Most of the time, only the user environment has to be
* rebuilt.
*/
std::string optimisticLockProfile(const Path & profile); std::string optimisticLockProfile(const Path & profile);
/** /**

View file

@ -32,8 +32,10 @@ struct RemoteStoreConfig : virtual StoreConfig
"Maximum age of a connection before it is closed."}; "Maximum age of a connection before it is closed."};
}; };
/* FIXME: RemoteStore is a misnomer - should be something like /**
DaemonStore. */ * \todo RemoteStore is a misnomer - should be something like
* DaemonStore.
*/
class RemoteStore : public virtual RemoteStoreConfig, class RemoteStore : public virtual RemoteStoreConfig,
public virtual Store, public virtual Store,
public virtual GcStore, public virtual GcStore,
@ -69,7 +71,9 @@ public:
void querySubstitutablePathInfos(const StorePathCAMap & paths, void querySubstitutablePathInfos(const StorePathCAMap & paths,
SubstitutablePathInfos & infos) override; SubstitutablePathInfos & infos) override;
/* Add a content-addressable store path. `dump` will be drained. */ /**
* Add a content-addressable store path. `dump` will be drained.
*/
ref<const ValidPathInfo> addCAToStore( ref<const ValidPathInfo> addCAToStore(
Source & dump, Source & dump,
std::string_view name, std::string_view name,
@ -77,7 +81,9 @@ public:
const StorePathSet & references, const StorePathSet & references,
RepairFlag repair); RepairFlag repair);
/* Add a content-addressable store path. Does not support references. `dump` will be drained. */ /**
* Add a content-addressable store path. Does not support references. `dump` will be drained.
*/
StorePath addToStoreFromDump(Source & dump, std::string_view name, StorePath addToStoreFromDump(Source & dump, std::string_view name,
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()) override; FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()) override;

View file

@ -11,7 +11,9 @@ struct sqlite3_stmt;
namespace nix { namespace nix {
/* RAII wrapper to close a SQLite database automatically. */ /**
* RAII wrapper to close a SQLite database automatically.
*/
struct SQLite struct SQLite
{ {
sqlite3 * db = 0; sqlite3 * db = 0;
@ -23,7 +25,9 @@ struct SQLite
~SQLite(); ~SQLite();
operator sqlite3 * () { return db; } operator sqlite3 * () { return db; }
/* Disable synchronous mode, set truncate journal mode. */ /**
* Disable synchronous mode, set truncate journal mode.
*/
void isCache(); void isCache();
void exec(const std::string & stmt); void exec(const std::string & stmt);
@ -31,7 +35,9 @@ struct SQLite
uint64_t getLastInsertedRowId(); uint64_t getLastInsertedRowId();
}; };
/* RAII wrapper to create and destroy SQLite prepared statements. */ /**
* RAII wrapper to create and destroy SQLite prepared statements.
*/
struct SQLiteStmt struct SQLiteStmt
{ {
sqlite3 * db = 0; sqlite3 * db = 0;
@ -43,7 +49,9 @@ struct SQLiteStmt
~SQLiteStmt(); ~SQLiteStmt();
operator sqlite3_stmt * () { return stmt; } operator sqlite3_stmt * () { return stmt; }
/* Helper for binding / executing statements. */ /**
* Helper for binding / executing statements.
*/
class Use class Use
{ {
friend struct SQLiteStmt; friend struct SQLiteStmt;
@ -56,7 +64,9 @@ struct SQLiteStmt
~Use(); ~Use();
/* Bind the next parameter. */ /**
* Bind the next parameter.
*/
Use & operator () (std::string_view value, bool notNull = true); Use & operator () (std::string_view value, bool notNull = true);
Use & operator () (const unsigned char * data, size_t len, bool notNull = true); Use & operator () (const unsigned char * data, size_t len, bool notNull = true);
Use & operator () (int64_t value, bool notNull = true); Use & operator () (int64_t value, bool notNull = true);
@ -64,11 +74,15 @@ struct SQLiteStmt
int step(); int step();
/* Execute a statement that does not return rows. */ /**
* Execute a statement that does not return rows.
*/
void exec(); void exec();
/* For statements that return 0 or more rows. Returns true iff /**
a row is available. */ * For statements that return 0 or more rows. Returns true iff
* a row is available.
*/
bool next(); bool next();
std::string getStr(int col); std::string getStr(int col);
@ -82,8 +96,10 @@ struct SQLiteStmt
} }
}; };
/* RAII helper that ensures transactions are aborted unless explicitly /**
committed. */ * RAII helper that ensures transactions are aborted unless explicitly
* committed.
*/
struct SQLiteTxn struct SQLiteTxn
{ {
bool active = false; bool active = false;
@ -125,8 +141,10 @@ MakeError(SQLiteBusy, SQLiteError);
void handleSQLiteBusy(const SQLiteBusy & e); void handleSQLiteBusy(const SQLiteBusy & e);
/* Convenience function for retrying a SQLite transaction when the /**
database is busy. */ * Convenience function for retrying a SQLite transaction when the
* database is busy.
*/
template<typename T, typename F> template<typename T, typename F>
T retrySQLite(F && fun) T retrySQLite(F && fun)
{ {

View file

@ -411,17 +411,17 @@ public:
{ unsupported("queryReferrers"); } { unsupported("queryReferrers"); }
/** /**
* @return all currently valid derivations that have `path' as an * @return all currently valid derivations that have `path` as an
* output. * output.
* *
* (Note that the result of `queryDeriver()' is the derivation that * (Note that the result of `queryDeriver()` is the derivation that
* was actually used to produce `path', which may not exist * was actually used to produce `path`, which may not exist
* anymore.) * anymore.)
*/ */
virtual StorePathSet queryValidDerivers(const StorePath & path) { return {}; }; virtual StorePathSet queryValidDerivers(const StorePath & path) { return {}; };
/** /**
* Query the outputs of the derivation denoted by `path'. * Query the outputs of the derivation denoted by `path`.
*/ */
virtual StorePathSet queryDerivationOutputs(const StorePath & path); virtual StorePathSet queryDerivationOutputs(const StorePath & path);
@ -513,7 +513,7 @@ public:
/** /**
* Like addToStore(), but the contents of the path are contained * Like addToStore(), but the contents of the path are contained
* in `dump', which is either a NAR serialisation (if recursive == * in `dump`, which is either a NAR serialisation (if recursive ==
* true) or simply the contents of a regular file (if recursive == * true) or simply the contents of a regular file (if recursive ==
* false). * false).
* `dump` may be drained * `dump` may be drained
@ -634,8 +634,8 @@ public:
/** /**
* @return a string representing information about the path that * @return a string representing information about the path that
* can be loaded into the database using `nix-store --load-db' or * can be loaded into the database using `nix-store --load-db` or
* `nix-store --register-validity'. * `nix-store --register-validity`.
*/ */
std::string makeValidityRegistration(const StorePathSet & paths, std::string makeValidityRegistration(const StorePathSet & paths,
bool showDerivers, bool showHash); bool showDerivers, bool showHash);
@ -715,12 +715,12 @@ public:
/** /**
* @param [out] out Place in here the set of all store paths in the * @param [out] out Place in here the set of all store paths in the
* file system closure of `storePath'; that is, all paths than can * file system closure of `storePath`; that is, all paths than can
* be directly or indirectly reached from it. `out' is not cleared. * be directly or indirectly reached from it. `out` is not cleared.
* *
* @param flipDirection If true, the set of paths that can reach * @param flipDirection If true, the set of paths that can reach
* `storePath' is returned; that is, the closures under the * `storePath` is returned; that is, the closures under the
* `referrers' relation instead of the `references' relation is * `referrers` relation instead of the `references` relation is
* returned. * returned.
*/ */
virtual void computeFSClosure(const StorePathSet & paths, virtual void computeFSClosure(const StorePathSet & paths,

View file

@ -1,9 +1,12 @@
#pragma once #pragma once
///@file /**
* @file
*
* @brief Some ANSI escape sequences.
*/
namespace nix { namespace nix {
/* Some ANSI escape sequences. */
#define ANSI_NORMAL "\e[0m" #define ANSI_NORMAL "\e[0m"
#define ANSI_BOLD "\e[1m" #define ANSI_BOLD "\e[1m"
#define ANSI_FAINT "\e[2m" #define ANSI_FAINT "\e[2m"

View file

@ -20,39 +20,41 @@ namespace nix {
* *
* The format is as follows: * The format is as follows:
* *
* IF path points to a REGULAR FILE: * ```
* dump(path) = attrs( * IF path points to a REGULAR FILE:
* [ ("type", "regular") * dump(path) = attrs(
* , ("contents", contents(path)) * [ ("type", "regular")
* ]) * , ("contents", contents(path))
* ])
* *
* IF path points to a DIRECTORY: * IF path points to a DIRECTORY:
* dump(path) = attrs( * dump(path) = attrs(
* [ ("type", "directory") * [ ("type", "directory")
* , ("entries", concat(map(f, sort(entries(path))))) * , ("entries", concat(map(f, sort(entries(path)))))
* ]) * ])
* where f(fn) = attrs( * where f(fn) = attrs(
* [ ("name", fn) * [ ("name", fn)
* , ("file", dump(path + "/" + fn)) * , ("file", dump(path + "/" + fn))
* ]) * ])
* *
* where: * where:
* *
* attrs(as) = concat(map(attr, as)) + encN(0) * attrs(as) = concat(map(attr, as)) + encN(0)
* attrs((a, b)) = encS(a) + encS(b) * attrs((a, b)) = encS(a) + encS(b)
* *
* encS(s) = encN(len(s)) + s + (padding until next 64-bit boundary) * encS(s) = encN(len(s)) + s + (padding until next 64-bit boundary)
* *
* encN(n) = 64-bit little-endian encoding of n. * encN(n) = 64-bit little-endian encoding of n.
* *
* contents(path) = the contents of a regular file. * contents(path) = the contents of a regular file.
* *
* sort(strings) = lexicographic sort by 8-bit value (strcmp). * sort(strings) = lexicographic sort by 8-bit value (strcmp).
* *
* entries(path) = the entries of a directory, without `.' and * entries(path) = the entries of a directory, without `.` and
* `..'. * `..`.
* *
* `+' denotes string concatenation. * `+` denotes string concatenation.
* ```
*/ */
void dumpPath(const Path & path, Sink & sink, void dumpPath(const Path & path, Sink & sink,
PathFilter & filter = defaultPathFilter); PathFilter & filter = defaultPathFilter);
@ -88,7 +90,7 @@ struct ParseSink
/** /**
* If the NAR archive contains a single file at top-level, then save * If the NAR archive contains a single file at top-level, then save
* the contents of the file to `s'. Otherwise barf. * the contents of the file to `s`. Otherwise barf.
*/ */
struct RetrieveRegularNARSink : ParseSink struct RetrieveRegularNARSink : ParseSink
{ {

View file

@ -192,8 +192,10 @@ public:
*/ */
bool isAllowed(const std::set<CanonPath> & allowed) const; bool isAllowed(const std::set<CanonPath> & allowed) const;
/* Return a representation `x` of `path` relative to `this`, i.e. /**
`CanonPath(this.makeRelative(x), this) == path`. */ * Return a representation `x` of `path` relative to `this`, i.e.
* `CanonPath(this.makeRelative(x), this) == path`.
*/
std::string makeRelative(const CanonPath & path) const; std::string makeRelative(const CanonPath & path) const;
}; };

View file

@ -1,5 +1,19 @@
#pragma once #pragma once
///@file /**
* @file
*
* @brief This file defines two main structs/classes used in nix error handling.
*
* ErrorInfo provides a standard payload of error information, with conversion to string
* happening in the logger rather than at the call site.
*
* BaseError is the ancestor of nix specific exceptions (and Interrupted), and contains
* an ErrorInfo.
*
* ErrorInfo structs are sent to the logger as part of an exception, or directly with the
* logError or logWarning macros.
* See libutil/tests/logging.cc for usage examples.
*/
#include "suggestions.hh" #include "suggestions.hh"
#include "ref.hh" #include "ref.hh"
@ -27,22 +41,6 @@
namespace nix { namespace nix {
/*
This file defines two main structs/classes used in nix error handling.
ErrorInfo provides a standard payload of error information, with conversion to string
happening in the logger rather than at the call site.
BaseError is the ancestor of nix specific exceptions (and Interrupted), and contains
an ErrorInfo.
ErrorInfo structs are sent to the logger as part of an exception, or directly with the
logError or logWarning macros.
See libutil/tests/logging.cc for usage examples.
*/
typedef enum { typedef enum {
lvlError = 0, lvlError = 0,

View file

@ -183,12 +183,17 @@ bool handleJSONLogMessage(const std::string & msg,
const Activity & act, std::map<ActivityId, Activity> & activities, const Activity & act, std::map<ActivityId, Activity> & activities,
bool trusted); bool trusted);
extern Verbosity verbosity; /* suppress msgs > this */ /**
* suppress msgs > this
*/
extern Verbosity verbosity;
/* Print a message with the standard ErrorInfo format. /**
In general, use these 'log' macros for reporting problems that may require user * Print a message with the standard ErrorInfo format.
intervention or that need more explanation. Use the 'print' macros for more * In general, use these 'log' macros for reporting problems that may require user
lightweight status messages. */ * intervention or that need more explanation. Use the 'print' macros for more
* lightweight status messages.
*/
#define logErrorInfo(level, errorInfo...) \ #define logErrorInfo(level, errorInfo...) \
do { \ do { \
if ((level) <= nix::verbosity) { \ if ((level) <= nix::verbosity) { \
@ -199,9 +204,11 @@ extern Verbosity verbosity; /* suppress msgs > this */
#define logError(errorInfo...) logErrorInfo(lvlError, errorInfo) #define logError(errorInfo...) logErrorInfo(lvlError, errorInfo)
#define logWarning(errorInfo...) logErrorInfo(lvlWarn, errorInfo) #define logWarning(errorInfo...) logErrorInfo(lvlWarn, errorInfo)
/* Print a string message if the current log level is at least the specified /**
level. Note that this has to be implemented as a macro to ensure that the * Print a string message if the current log level is at least the specified
arguments are evaluated lazily. */ * level. Note that this has to be implemented as a macro to ensure that the
* arguments are evaluated lazily.
*/
#define printMsgUsing(loggerParam, level, args...) \ #define printMsgUsing(loggerParam, level, args...) \
do { \ do { \
auto __lvl = level; \ auto __lvl = level; \

View file

@ -186,6 +186,22 @@ static DefaultStackAllocator defaultAllocatorSingleton;
StackAllocator *StackAllocator::defaultAllocator = &defaultAllocatorSingleton; StackAllocator *StackAllocator::defaultAllocator = &defaultAllocatorSingleton;
std::shared_ptr<void> (*create_coro_gc_hook)() = []() -> std::shared_ptr<void> {
return {};
};
/* This class is used for entry and exit hooks on coroutines */
class CoroutineContext {
/* Disable GC when entering the coroutine without the boehm patch,
* since it doesn't find the main thread stack in this case.
* std::shared_ptr<void> performs type-erasure, so it will call the right
* deleter. */
const std::shared_ptr<void> coro_gc_hook = create_coro_gc_hook();
public:
CoroutineContext() {};
~CoroutineContext() {};
};
std::unique_ptr<FinishSink> sourceToSink(std::function<void(Source &)> fun) std::unique_ptr<FinishSink> sourceToSink(std::function<void(Source &)> fun)
{ {
struct SourceToSink : FinishSink struct SourceToSink : FinishSink
@ -206,7 +222,8 @@ std::unique_ptr<FinishSink> sourceToSink(std::function<void(Source &)> fun)
if (in.empty()) return; if (in.empty()) return;
cur = in; cur = in;
if (!coro) if (!coro) {
CoroutineContext ctx;
coro = coro_t::push_type(VirtualStackAllocator{}, [&](coro_t::pull_type & yield) { coro = coro_t::push_type(VirtualStackAllocator{}, [&](coro_t::pull_type & yield) {
LambdaSource source([&](char *out, size_t out_len) { LambdaSource source([&](char *out, size_t out_len) {
if (cur.empty()) { if (cur.empty()) {
@ -223,17 +240,24 @@ std::unique_ptr<FinishSink> sourceToSink(std::function<void(Source &)> fun)
}); });
fun(source); fun(source);
}); });
}
if (!*coro) { abort(); } if (!*coro) { abort(); }
if (!cur.empty()) (*coro)(false); if (!cur.empty()) {
CoroutineContext ctx;
(*coro)(false);
}
} }
void finish() override void finish() override
{ {
if (!coro) return; if (!coro) return;
if (!*coro) abort(); if (!*coro) abort();
(*coro)(true); {
CoroutineContext ctx;
(*coro)(true);
}
if (*coro) abort(); if (*coro) abort();
} }
}; };
@ -264,18 +288,23 @@ std::unique_ptr<Source> sinkToSource(
size_t read(char * data, size_t len) override size_t read(char * data, size_t len) override
{ {
if (!coro) if (!coro) {
CoroutineContext ctx;
coro = coro_t::pull_type(VirtualStackAllocator{}, [&](coro_t::push_type & yield) { coro = coro_t::pull_type(VirtualStackAllocator{}, [&](coro_t::push_type & yield) {
LambdaSink sink([&](std::string_view data) { LambdaSink sink([&](std::string_view data) {
if (!data.empty()) yield(std::string(data)); if (!data.empty()) yield(std::string(data));
}); });
fun(sink); fun(sink);
}); });
}
if (!*coro) { eof(); abort(); } if (!*coro) { eof(); abort(); }
if (pos == cur.size()) { if (pos == cur.size()) {
if (!cur.empty()) (*coro)(); if (!cur.empty()) {
CoroutineContext ctx;
(*coro)();
}
cur = coro->get(); cur = coro->get();
pos = 0; pos = 0;
} }

View file

@ -551,4 +551,10 @@ struct StackAllocator {
static StackAllocator *defaultAllocator; static StackAllocator *defaultAllocator;
}; };
/* Disabling GC when entering a coroutine (without the boehm patch).
mutable to avoid boehm gc dependency in libutil.
*/
extern std::shared_ptr<void> (*create_coro_gc_hook)();
} }

View file

@ -41,14 +41,14 @@ public:
/** /**
* Execute work items until the queue is empty. * Execute work items until the queue is empty.
* *
* \note Note that work items are allowed to add new items to the * \note Note that work items are allowed to add new items to the
* queue; this is handled correctly. * queue; this is handled correctly.
* *
* Queue processing stops prematurely if any work item throws an * Queue processing stops prematurely if any work item throws an
* exception. This exception is propagated to the calling thread. If * exception. This exception is propagated to the calling thread. If
* multiple work items throw an exception concurrently, only one * multiple work items throw an exception concurrently, only one
* item is propagated; the others are printed on stderr and * item is propagated; the others are printed on stderr and
* otherwise ignored. * otherwise ignored.
*/ */
void process(); void process();

View file

@ -33,77 +33,112 @@ struct Sink;
struct Source; struct Source;
/* The system for which Nix is compiled. */ /**
* The system for which Nix is compiled.
*/
extern const std::string nativeSystem; extern const std::string nativeSystem;
/* Return an environment variable. */ /**
* @return an environment variable.
*/
std::optional<std::string> getEnv(const std::string & key); std::optional<std::string> getEnv(const std::string & key);
/* Return a non empty environment variable. Returns nullopt if the env /**
variable is set to "" */ * @return a non empty environment variable. Returns nullopt if the env
* variable is set to ""
*/
std::optional<std::string> getEnvNonEmpty(const std::string & key); std::optional<std::string> getEnvNonEmpty(const std::string & key);
/* Get the entire environment. */ /**
* Get the entire environment.
*/
std::map<std::string, std::string> getEnv(); std::map<std::string, std::string> getEnv();
/* Clear the environment. */ /**
* Clear the environment.
*/
void clearEnv(); void clearEnv();
/* Return an absolutized path, resolving paths relative to the /**
specified directory, or the current directory otherwise. The path * @return An absolutized path, resolving paths relative to the
is also canonicalised. */ * specified directory, or the current directory otherwise. The path
* is also canonicalised.
*/
Path absPath(Path path, Path absPath(Path path,
std::optional<PathView> dir = {}, std::optional<PathView> dir = {},
bool resolveSymlinks = false); bool resolveSymlinks = false);
/* Canonicalise a path by removing all `.' or `..' components and /**
double or trailing slashes. Optionally resolves all symlink * Canonicalise a path by removing all `.` or `..` components and
components such that each component of the resulting path is *not* * double or trailing slashes. Optionally resolves all symlink
a symbolic link. */ * components such that each component of the resulting path is *not*
* a symbolic link.
*/
Path canonPath(PathView path, bool resolveSymlinks = false); Path canonPath(PathView path, bool resolveSymlinks = false);
/* Return the directory part of the given canonical path, i.e., /**
everything before the final `/'. If the path is the root or an * @return The directory part of the given canonical path, i.e.,
immediate child thereof (e.g., `/foo'), this means `/' * everything before the final `/`. If the path is the root or an
is returned.*/ * immediate child thereof (e.g., `/foo`), this means `/`
* is returned.
*/
Path dirOf(const PathView path); Path dirOf(const PathView path);
/* Return the base name of the given canonical path, i.e., everything /**
following the final `/' (trailing slashes are removed). */ * @return the base name of the given canonical path, i.e., everything
* following the final `/` (trailing slashes are removed).
*/
std::string_view baseNameOf(std::string_view path); std::string_view baseNameOf(std::string_view path);
/* Perform tilde expansion on a path. */ /**
* Perform tilde expansion on a path.
*/
std::string expandTilde(std::string_view path); std::string expandTilde(std::string_view path);
/* Check whether 'path' is a descendant of 'dir'. Both paths must be /**
canonicalized. */ * Check whether 'path' is a descendant of 'dir'. Both paths must be
* canonicalized.
*/
bool isInDir(std::string_view path, std::string_view dir); bool isInDir(std::string_view path, std::string_view dir);
/* Check whether 'path' is equal to 'dir' or a descendant of /**
'dir'. Both paths must be canonicalized. */ * Check whether 'path' is equal to 'dir' or a descendant of
* 'dir'. Both paths must be canonicalized.
*/
bool isDirOrInDir(std::string_view path, std::string_view dir); bool isDirOrInDir(std::string_view path, std::string_view dir);
/* Get status of `path'. */ /**
* Get status of `path`.
*/
struct stat stat(const Path & path); struct stat stat(const Path & path);
struct stat lstat(const Path & path); struct stat lstat(const Path & path);
/* Return true iff the given path exists. */ /**
* @return true iff the given path exists.
*/
bool pathExists(const Path & path); bool pathExists(const Path & path);
/* Read the contents (target) of a symbolic link. The result is not /**
in any way canonicalised. */ * Read the contents (target) of a symbolic link. The result is not
* in any way canonicalised.
*/
Path readLink(const Path & path); Path readLink(const Path & path);
bool isLink(const Path & path); bool isLink(const Path & path);
/* Read the contents of a directory. The entries `.' and `..' are /**
removed. */ * Read the contents of a directory. The entries `.` and `..` are
* removed.
*/
struct DirEntry struct DirEntry
{ {
std::string name; std::string name;
ino_t ino; ino_t ino;
unsigned char type; // one of DT_* /**
* one of DT_*
*/
unsigned char type;
DirEntry(std::string name, ino_t ino, unsigned char type) DirEntry(std::string name, ino_t ino, unsigned char type)
: name(std::move(name)), ino(ino), type(type) { } : name(std::move(name)), ino(ino), type(type) { }
}; };
@ -114,74 +149,110 @@ DirEntries readDirectory(const Path & path);
unsigned char getFileType(const Path & path); unsigned char getFileType(const Path & path);
/* Read the contents of a file into a string. */ /**
* Read the contents of a file into a string.
*/
std::string readFile(int fd); std::string readFile(int fd);
std::string readFile(const Path & path); std::string readFile(const Path & path);
void readFile(const Path & path, Sink & sink); void readFile(const Path & path, Sink & sink);
/* Write a string to a file. */ /**
* Write a string to a file.
*/
void writeFile(const Path & path, std::string_view s, mode_t mode = 0666, bool sync = false); void writeFile(const Path & path, std::string_view s, mode_t mode = 0666, bool sync = false);
void writeFile(const Path & path, Source & source, mode_t mode = 0666, bool sync = false); void writeFile(const Path & path, Source & source, mode_t mode = 0666, bool sync = false);
/* Flush a file's parent directory to disk */ /**
* Flush a file's parent directory to disk
*/
void syncParent(const Path & path); void syncParent(const Path & path);
/* Read a line from a file descriptor. */ /**
* Read a line from a file descriptor.
*/
std::string readLine(int fd); std::string readLine(int fd);
/* Write a line to a file descriptor. */ /**
* Write a line to a file descriptor.
*/
void writeLine(int fd, std::string s); void writeLine(int fd, std::string s);
/* Delete a path; i.e., in the case of a directory, it is deleted /**
recursively. It's not an error if the path does not exist. The * Delete a path; i.e., in the case of a directory, it is deleted
second variant returns the number of bytes and blocks freed. */ * recursively. It's not an error if the path does not exist. The
* second variant returns the number of bytes and blocks freed.
*/
void deletePath(const Path & path); void deletePath(const Path & path);
void deletePath(const Path & path, uint64_t & bytesFreed); void deletePath(const Path & path, uint64_t & bytesFreed);
std::string getUserName(); std::string getUserName();
/* Return the given user's home directory from /etc/passwd. */ /**
* @return the given user's home directory from /etc/passwd.
*/
Path getHomeOf(uid_t userId); Path getHomeOf(uid_t userId);
/* Return $HOME or the user's home directory from /etc/passwd. */ /**
* @return $HOME or the user's home directory from /etc/passwd.
*/
Path getHome(); Path getHome();
/* Return $XDG_CACHE_HOME or $HOME/.cache. */ /**
* @return $XDG_CACHE_HOME or $HOME/.cache.
*/
Path getCacheDir(); Path getCacheDir();
/* Return $XDG_CONFIG_HOME or $HOME/.config. */ /**
* @return $XDG_CONFIG_HOME or $HOME/.config.
*/
Path getConfigDir(); Path getConfigDir();
/* Return the directories to search for user configuration files */ /**
* @return the directories to search for user configuration files
*/
std::vector<Path> getConfigDirs(); std::vector<Path> getConfigDirs();
/* Return $XDG_DATA_HOME or $HOME/.local/share. */ /**
* @return $XDG_DATA_HOME or $HOME/.local/share.
*/
Path getDataDir(); Path getDataDir();
/* Return the path of the current executable. */ /**
* @return the path of the current executable.
*/
std::optional<Path> getSelfExe(); std::optional<Path> getSelfExe();
/* Return $XDG_STATE_HOME or $HOME/.local/state. */ /**
* @return $XDG_STATE_HOME or $HOME/.local/state.
*/
Path getStateDir(); Path getStateDir();
/* Create the Nix state directory and return the path to it. */ /**
* Create the Nix state directory and return the path to it.
*/
Path createNixStateDir(); Path createNixStateDir();
/* Create a directory and all its parents, if necessary. Returns the /**
list of created directories, in order of creation. */ * Create a directory and all its parents, if necessary. Returns the
* list of created directories, in order of creation.
*/
Paths createDirs(const Path & path); Paths createDirs(const Path & path);
inline Paths createDirs(PathView path) inline Paths createDirs(PathView path)
{ {
return createDirs(Path(path)); return createDirs(Path(path));
} }
/* Create a symlink. */ /**
* Create a symlink.
*/
void createSymlink(const Path & target, const Path & link, void createSymlink(const Path & target, const Path & link,
std::optional<time_t> mtime = {}); std::optional<time_t> mtime = {});
/* Atomically create or replace a symlink. */ /**
* Atomically create or replace a symlink.
*/
void replaceSymlink(const Path & target, const Path & link, void replaceSymlink(const Path & target, const Path & link,
std::optional<time_t> mtime = {}); std::optional<time_t> mtime = {});
@ -197,24 +268,32 @@ void renameFile(const Path & src, const Path & dst);
void moveFile(const Path & src, const Path & dst); void moveFile(const Path & src, const Path & dst);
/* Wrappers arount read()/write() that read/write exactly the /**
requested number of bytes. */ * Wrappers arount read()/write() that read/write exactly the
* requested number of bytes.
*/
void readFull(int fd, char * buf, size_t count); void readFull(int fd, char * buf, size_t count);
void writeFull(int fd, std::string_view s, bool allowInterrupts = true); void writeFull(int fd, std::string_view s, bool allowInterrupts = true);
MakeError(EndOfFile, Error); MakeError(EndOfFile, Error);
/* Read a file descriptor until EOF occurs. */ /**
* Read a file descriptor until EOF occurs.
*/
std::string drainFD(int fd, bool block = true, const size_t reserveSize=0); std::string drainFD(int fd, bool block = true, const size_t reserveSize=0);
void drainFD(int fd, Sink & sink, bool block = true); void drainFD(int fd, Sink & sink, bool block = true);
/* If cgroups are active, attempt to calculate the number of CPUs available. /**
If cgroups are unavailable or if cpu.max is set to "max", return 0. */ * If cgroups are active, attempt to calculate the number of CPUs available.
* If cgroups are unavailable or if cpu.max is set to "max", return 0.
*/
unsigned int getMaxCPU(); unsigned int getMaxCPU();
/* Automatic cleanup of resources. */ /**
* Automatic cleanup of resources.
*/
class AutoDelete class AutoDelete
@ -252,11 +331,15 @@ public:
}; };
/* Create a temporary directory. */ /**
* Create a temporary directory.
*/
Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix",
bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755);
/* Create a temporary file, returning a file handle and its path. */ /**
* Create a temporary file, returning a file handle and its path.
*/
std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix = "nix"); std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix = "nix");
@ -299,27 +382,36 @@ public:
}; };
/* Kill all processes running under the specified uid by sending them /**
a SIGKILL. */ * Kill all processes running under the specified uid by sending them
* a SIGKILL.
*/
void killUser(uid_t uid); void killUser(uid_t uid);
/* Fork a process that runs the given function, and return the child /**
pid to the caller. */ * Fork a process that runs the given function, and return the child
* pid to the caller.
*/
struct ProcessOptions struct ProcessOptions
{ {
std::string errorPrefix = ""; std::string errorPrefix = "";
bool dieWithParent = true; bool dieWithParent = true;
bool runExitHandlers = false; bool runExitHandlers = false;
bool allowVfork = false; bool allowVfork = false;
int cloneFlags = 0; // use clone() with the specified flags (Linux only) /**
* use clone() with the specified flags (Linux only)
*/
int cloneFlags = 0;
}; };
pid_t startProcess(std::function<void()> fun, const ProcessOptions & options = ProcessOptions()); pid_t startProcess(std::function<void()> fun, const ProcessOptions & options = ProcessOptions());
/* Run a program and return its stdout in a string (i.e., like the /**
shell backtick operator). */ * Run a program and return its stdout in a string (i.e., like the
* shell backtick operator).
*/
std::string runProgram(Path program, bool searchPath = false, std::string runProgram(Path program, bool searchPath = false,
const Strings & args = Strings(), const Strings & args = Strings(),
const std::optional<std::string> & input = {}); const std::optional<std::string> & input = {});
@ -344,25 +436,35 @@ std::pair<int, std::string> runProgram(RunOptions && options);
void runProgram2(const RunOptions & options); void runProgram2(const RunOptions & options);
/* Change the stack size. */ /**
* Change the stack size.
*/
void setStackSize(size_t stackSize); void setStackSize(size_t stackSize);
/* Restore the original inherited Unix process context (such as signal /**
masks, stack size). */ * Restore the original inherited Unix process context (such as signal
* masks, stack size).
*/
void restoreProcessContext(bool restoreMounts = true); void restoreProcessContext(bool restoreMounts = true);
/* Save the current mount namespace. Ignored if called more than /**
once. */ * Save the current mount namespace. Ignored if called more than
* once.
*/
void saveMountNamespace(); void saveMountNamespace();
/* Restore the mount namespace saved by saveMountNamespace(). Ignored /**
if saveMountNamespace() was never called. */ * Restore the mount namespace saved by saveMountNamespace(). Ignored
* if saveMountNamespace() was never called.
*/
void restoreMountNamespace(); void restoreMountNamespace();
/* Cause this thread to not share any FS attributes with the main /**
thread, because this causes setns() in restoreMountNamespace() to * Cause this thread to not share any FS attributes with the main
fail. */ * thread, because this causes setns() in restoreMountNamespace() to
* fail.
*/
void unshareFilesystem(); void unshareFilesystem();
@ -377,16 +479,22 @@ public:
{ } { }
}; };
/* Convert a list of strings to a null-terminated vector of char /**
*'s. The result must not be accessed beyond the lifetime of the * Convert a list of strings to a null-terminated vector of `char
list of strings. */ * *`s. The result must not be accessed beyond the lifetime of the
* list of strings.
*/
std::vector<char *> stringsToCharPtrs(const Strings & ss); std::vector<char *> stringsToCharPtrs(const Strings & ss);
/* Close all file descriptors except those listed in the given set. /**
Good practice in child processes. */ * Close all file descriptors except those listed in the given set.
* Good practice in child processes.
*/
void closeMostFDs(const std::set<int> & exceptions); void closeMostFDs(const std::set<int> & exceptions);
/* Set the close-on-exec flag for the given file descriptor. */ /**
* Set the close-on-exec flag for the given file descriptor.
*/
void closeOnExec(int fd); void closeOnExec(int fd);
@ -412,12 +520,16 @@ MakeError(Interrupted, BaseError);
MakeError(FormatError, Error); MakeError(FormatError, Error);
/* String tokenizer. */ /**
* String tokenizer.
*/
template<class C> C tokenizeString(std::string_view s, std::string_view separators = " \t\n\r"); template<class C> C tokenizeString(std::string_view s, std::string_view separators = " \t\n\r");
/* Concatenate the given strings with a separator between the /**
elements. */ * Concatenate the given strings with a separator between the
* elements.
*/
template<class C> template<class C>
std::string concatStringsSep(const std::string_view sep, const C & ss) std::string concatStringsSep(const std::string_view sep, const C & ss)
{ {
@ -442,7 +554,9 @@ auto concatStrings(Parts && ... parts)
} }
/* Add quotes around a collection of strings. */ /**
* Add quotes around a collection of strings.
*/
template<class C> Strings quoteStrings(const C & c) template<class C> Strings quoteStrings(const C & c)
{ {
Strings res; Strings res;
@ -451,16 +565,23 @@ template<class C> Strings quoteStrings(const C & c)
return res; return res;
} }
/* Remove trailing whitespace from a string. FIXME: return /**
std::string_view. */ * Remove trailing whitespace from a string.
*
* \todo return std::string_view.
*/
std::string chomp(std::string_view s); std::string chomp(std::string_view s);
/* Remove whitespace from the start and end of a string. */ /**
* Remove whitespace from the start and end of a string.
*/
std::string trim(std::string_view s, std::string_view whitespace = " \n\r\t"); std::string trim(std::string_view s, std::string_view whitespace = " \n\r\t");
/* Replace all occurrences of a string inside another string. */ /**
* Replace all occurrences of a string inside another string.
*/
std::string replaceStrings( std::string replaceStrings(
std::string s, std::string s,
std::string_view from, std::string_view from,
@ -470,14 +591,18 @@ std::string replaceStrings(
std::string rewriteStrings(std::string s, const StringMap & rewrites); std::string rewriteStrings(std::string s, const StringMap & rewrites);
/* Convert the exit status of a child as returned by wait() into an /**
error string. */ * Convert the exit status of a child as returned by wait() into an
* error string.
*/
std::string statusToString(int status); std::string statusToString(int status);
bool statusOk(int status); bool statusOk(int status);
/* Parse a string into an integer. */ /**
* Parse a string into an integer.
*/
template<class N> template<class N>
std::optional<N> string2Int(const std::string_view s) std::optional<N> string2Int(const std::string_view s)
{ {
@ -490,8 +615,10 @@ std::optional<N> string2Int(const std::string_view s)
} }
} }
/* Like string2Int(), but support an optional suffix 'K', 'M', 'G' or /**
'T' denoting a binary unit prefix. */ * Like string2Int(), but support an optional suffix 'K', 'M', 'G' or
* 'T' denoting a binary unit prefix.
*/
template<class N> template<class N>
N string2IntWithUnitPrefix(std::string_view s) N string2IntWithUnitPrefix(std::string_view s)
{ {
@ -512,7 +639,9 @@ N string2IntWithUnitPrefix(std::string_view s)
throw UsageError("'%s' is not an integer", s); throw UsageError("'%s' is not an integer", s);
} }
/* Parse a string into a float. */ /**
* Parse a string into a float.
*/
template<class N> template<class N>
std::optional<N> string2Float(const std::string_view s) std::optional<N> string2Float(const std::string_view s)
{ {
@ -524,7 +653,9 @@ std::optional<N> string2Float(const std::string_view s)
} }
/* Convert a little-endian integer to host order. */ /**
* Convert a little-endian integer to host order.
*/
template<typename T> template<typename T>
T readLittleEndian(unsigned char * p) T readLittleEndian(unsigned char * p)
{ {
@ -536,66 +667,90 @@ T readLittleEndian(unsigned char * p)
} }
/* Return true iff `s' starts with `prefix'. */ /**
* @return true iff `s` starts with `prefix`.
*/
bool hasPrefix(std::string_view s, std::string_view prefix); bool hasPrefix(std::string_view s, std::string_view prefix);
/* Return true iff `s' ends in `suffix'. */ /**
* @return true iff `s` ends in `suffix`.
*/
bool hasSuffix(std::string_view s, std::string_view suffix); bool hasSuffix(std::string_view s, std::string_view suffix);
/* Convert a string to lower case. */ /**
* Convert a string to lower case.
*/
std::string toLower(const std::string & s); std::string toLower(const std::string & s);
/* Escape a string as a shell word. */ /**
* Escape a string as a shell word.
*/
std::string shellEscape(const std::string_view s); std::string shellEscape(const std::string_view s);
/* Exception handling in destructors: print an error message, then /**
ignore the exception. */ * Exception handling in destructors: print an error message, then
* ignore the exception.
*/
void ignoreException(Verbosity lvl = lvlError); void ignoreException(Verbosity lvl = lvlError);
/* Tree formatting. */ /**
* Tree formatting.
*/
constexpr char treeConn[] = "├───"; constexpr char treeConn[] = "├───";
constexpr char treeLast[] = "└───"; constexpr char treeLast[] = "└───";
constexpr char treeLine[] = ""; constexpr char treeLine[] = "";
constexpr char treeNull[] = " "; constexpr char treeNull[] = " ";
/* Determine whether ANSI escape sequences are appropriate for the /**
present output. */ * Determine whether ANSI escape sequences are appropriate for the
* present output.
*/
bool shouldANSI(); bool shouldANSI();
/* Truncate a string to 'width' printable characters. If 'filterAll' /**
is true, all ANSI escape sequences are filtered out. Otherwise, * Truncate a string to 'width' printable characters. If 'filterAll'
some escape sequences (such as colour setting) are copied but not * is true, all ANSI escape sequences are filtered out. Otherwise,
included in the character count. Also, tabs are expanded to * some escape sequences (such as colour setting) are copied but not
spaces. */ * included in the character count. Also, tabs are expanded to
* spaces.
*/
std::string filterANSIEscapes(std::string_view s, std::string filterANSIEscapes(std::string_view s,
bool filterAll = false, bool filterAll = false,
unsigned int width = std::numeric_limits<unsigned int>::max()); unsigned int width = std::numeric_limits<unsigned int>::max());
/* Base64 encoding/decoding. */ /**
* Base64 encoding/decoding.
*/
std::string base64Encode(std::string_view s); std::string base64Encode(std::string_view s);
std::string base64Decode(std::string_view s); std::string base64Decode(std::string_view s);
/* Remove common leading whitespace from the lines in the string /**
's'. For example, if every line is indented by at least 3 spaces, * Remove common leading whitespace from the lines in the string
then we remove 3 spaces from the start of every line. */ * 's'. For example, if every line is indented by at least 3 spaces,
* then we remove 3 spaces from the start of every line.
*/
std::string stripIndentation(std::string_view s); std::string stripIndentation(std::string_view s);
/* Get the prefix of 's' up to and excluding the next line break (LF /**
optionally preceded by CR), and the remainder following the line * Get the prefix of 's' up to and excluding the next line break (LF
break. */ * optionally preceded by CR), and the remainder following the line
* break.
*/
std::pair<std::string_view, std::string_view> getLine(std::string_view s); std::pair<std::string_view, std::string_view> getLine(std::string_view s);
/* Get a value for the specified key from an associate container. */ /**
* Get a value for the specified key from an associate container.
*/
template <class T> template <class T>
const typename T::mapped_type * get(const T & map, const typename T::key_type & key) const typename T::mapped_type * get(const T & map, const typename T::key_type & key)
{ {
@ -612,7 +767,9 @@ typename T::mapped_type * get(T & map, const typename T::key_type & key)
return &i->second; return &i->second;
} }
/* Get a value for the specified key from an associate container, or a default value if the key isn't present. */ /**
* Get a value for the specified key from an associate container, or a default value if the key isn't present.
*/
template <class T> template <class T>
const typename T::mapped_type & getOr(T & map, const typename T::mapped_type & getOr(T & map,
const typename T::key_type & key, const typename T::key_type & key,
@ -623,7 +780,9 @@ const typename T::mapped_type & getOr(T & map,
return i->second; return i->second;
} }
/* Remove and return the first item from a container. */ /**
* Remove and return the first item from a container.
*/
template <class T> template <class T>
std::optional<typename T::value_type> remove_begin(T & c) std::optional<typename T::value_type> remove_begin(T & c)
{ {
@ -635,7 +794,9 @@ std::optional<typename T::value_type> remove_begin(T & c)
} }
/* Remove and return the first item from a container. */ /**
* Remove and return the first item from a container.
*/
template <class T> template <class T>
std::optional<typename T::value_type> pop(T & c) std::optional<typename T::value_type> pop(T & c)
{ {
@ -650,8 +811,10 @@ template<typename T>
class Callback; class Callback;
/* Start a thread that handles various signals. Also block those signals /**
on the current thread (and thus any threads created by it). */ * Start a thread that handles various signals. Also block those signals
* on the current thread (and thus any threads created by it).
*/
void startSignalHandlerThread(); void startSignalHandlerThread();
struct InterruptCallback struct InterruptCallback
@ -659,16 +822,20 @@ struct InterruptCallback
virtual ~InterruptCallback() { }; virtual ~InterruptCallback() { };
}; };
/* Register a function that gets called on SIGINT (in a non-signal /**
context). */ * Register a function that gets called on SIGINT (in a non-signal
* context).
*/
std::unique_ptr<InterruptCallback> createInterruptCallback( std::unique_ptr<InterruptCallback> createInterruptCallback(
std::function<void()> callback); std::function<void()> callback);
void triggerInterrupt(); void triggerInterrupt();
/* A RAII class that causes the current thread to receive SIGUSR1 when /**
the signal handler thread receives SIGINT. That is, this allows * A RAII class that causes the current thread to receive SIGUSR1 when
SIGINT to be multiplexed to multiple threads. */ * the signal handler thread receives SIGINT. That is, this allows
* SIGINT to be multiplexed to multiple threads.
*/
struct ReceiveInterrupts struct ReceiveInterrupts
{ {
pthread_t target; pthread_t target;
@ -682,8 +849,10 @@ struct ReceiveInterrupts
/* A RAII helper that increments a counter on construction and /**
decrements it on destruction. */ * A RAII helper that increments a counter on construction and
* decrements it on destruction.
*/
template<typename T> template<typename T>
struct MaintainCount struct MaintainCount
{ {
@ -694,33 +863,50 @@ struct MaintainCount
}; };
/* Return the number of rows and columns of the terminal. */ /**
* @return the number of rows and columns of the terminal.
*/
std::pair<unsigned short, unsigned short> getWindowSize(); std::pair<unsigned short, unsigned short> getWindowSize();
/* Used in various places. */ /**
* Used in various places.
*/
typedef std::function<bool(const Path & path)> PathFilter; typedef std::function<bool(const Path & path)> PathFilter;
extern PathFilter defaultPathFilter; extern PathFilter defaultPathFilter;
/* Common initialisation performed in child processes. */ /**
* Common initialisation performed in child processes.
*/
void commonChildInit(); void commonChildInit();
/* Create a Unix domain socket. */ /**
* Create a Unix domain socket.
*/
AutoCloseFD createUnixDomainSocket(); AutoCloseFD createUnixDomainSocket();
/* Create a Unix domain socket in listen mode. */ /**
* Create a Unix domain socket in listen mode.
*/
AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode); AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode);
/* Bind a Unix domain socket to a path. */ /**
* Bind a Unix domain socket to a path.
*/
void bind(int fd, const std::string & path); void bind(int fd, const std::string & path);
/* Connect to a Unix domain socket. */ /**
* Connect to a Unix domain socket.
*/
void connect(int fd, const std::string & path); void connect(int fd, const std::string & path);
// A Rust/Python-like enumerate() iterator adapter. /**
// Borrowed from http://reedbeta.com/blog/python-like-enumerate-in-cpp17. * A Rust/Python-like enumerate() iterator adapter.
*
* Borrowed from http://reedbeta.com/blog/python-like-enumerate-in-cpp17.
*/
template <typename T, template <typename T,
typename TIter = decltype(std::begin(std::declval<T>())), typename TIter = decltype(std::begin(std::declval<T>())),
typename = decltype(std::end(std::declval<T>()))> typename = decltype(std::end(std::declval<T>()))>
@ -746,7 +932,9 @@ constexpr auto enumerate(T && iterable)
} }
// C++17 std::visit boilerplate /**
* C++17 std::visit boilerplate
*/
template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; }; template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>; template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
@ -754,8 +942,10 @@ template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
std::string showBytes(uint64_t bytes); std::string showBytes(uint64_t bytes);
/* Provide an addition operator between strings and string_views /**
inexplicably omitted from the standard library. */ * Provide an addition operator between strings and string_views
* inexplicably omitted from the standard library.
*/
inline std::string operator + (const std::string & s1, std::string_view s2) inline std::string operator + (const std::string & s1, std::string_view s2)
{ {
auto s = s1; auto s = s1;