From 468d30e0533b7fd947acb73c0ddf197b934f6a07 Mon Sep 17 00:00:00 2001 From: Tom Hubrecht Date: Thu, 30 May 2024 03:19:57 +0200 Subject: [PATCH] primops: Move functions to primops/import.cc Moved builtins: import, importNative, scopedImport Change-Id: I7c525a03f877ad4a6586e055b37f8e4db51ad721 --- src/libexpr/meson.build | 1 + src/libexpr/primops.cc | 219 --------------------------- src/libexpr/primops.hh | 10 ++ src/libexpr/primops/import.cc | 270 ++++++++++++++++++++++++++++++++++ 4 files changed, 281 insertions(+), 219 deletions(-) create mode 100644 src/libexpr/primops/import.cc diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build index f91cd3d38..520ec2d83 100644 --- a/src/libexpr/meson.build +++ b/src/libexpr/meson.build @@ -95,6 +95,7 @@ libexpr_sources = files( 'primops/fetchMercurial.cc', 'primops/fetchTree.cc', 'primops/fromTOML.cc', + 'primops/import.cc', 'primops/list.cc', 'primops/string.cc', 'primops/types.cc', diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index d28be73af..093c3862a 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -135,225 +135,6 @@ SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v, const Rea } } -/** - * Add and attribute to the given attribute map from the output name to - * the output path, or a placeholder. - * - * Where possible the path is used, but for floating CA derivations we - * may not know it. For sake of determinism we always assume we don't - * and instead put in a place holder. In either case, however, the - * string context will contain the drv path and output name, so - * downstream derivations will have the proper dependency, and in - * addition, before building, the placeholder will be rewritten to be - * the actual path. - * - * The 'drv' and 'drvPath' outputs must correspond. - */ -static void mkOutputString( - EvalState & state, - BindingsBuilder & attrs, - const StorePath & drvPath, - const std::pair & o) -{ - state.mkOutputString( - attrs.alloc(o.first), - SingleDerivedPath::Built { - .drvPath = makeConstantStorePathRef(drvPath), - .output = o.first, - }, - o.second.path(*state.store, Derivation::nameFromPath(drvPath), o.first)); -} - -/* Load and evaluate an expression from path specified by the - argument. */ -static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * vScope, Value & v) -{ - auto path = realisePath(state, pos, vPath); - auto path2 = path.path.abs(); - - // FIXME - auto isValidDerivationInStore = [&]() -> std::optional { - if (!state.store->isStorePath(path2)) - return std::nullopt; - auto storePath = state.store->parseStorePath(path2); - if (!(state.store->isValidPath(storePath) && isDerivation(path2))) - return std::nullopt; - return storePath; - }; - - if (auto storePath = isValidDerivationInStore()) { - Derivation drv = state.store->readDerivation(*storePath); - auto attrs = state.buildBindings(3 + drv.outputs.size()); - attrs.alloc(state.sDrvPath).mkString(path2, { - NixStringContextElem::DrvDeep { .drvPath = *storePath }, - }); - attrs.alloc(state.sName).mkString(drv.env["name"]); - auto & outputsVal = attrs.alloc(state.sOutputs); - state.mkList(outputsVal, drv.outputs.size()); - - for (const auto & [i, o] : enumerate(drv.outputs)) { - mkOutputString(state, attrs, *storePath, o); - (outputsVal.listElems()[i] = state.allocValue())->mkString(o.first); - } - - auto w = state.allocValue(); - w->mkAttrs(attrs); - - if (!state.vImportedDrvToDerivation) { - state.vImportedDrvToDerivation = allocRootValue(state.allocValue()); - state.eval(state.parseExprFromString( - #include "imported-drv-to-derivation.nix.gen.hh" - , CanonPath::root), **state.vImportedDrvToDerivation); - } - - state.forceFunction(**state.vImportedDrvToDerivation, pos, "while evaluating imported-drv-to-derivation.nix.gen.hh"); - v.mkApp(*state.vImportedDrvToDerivation, w); - state.forceAttrs(v, pos, "while calling imported-drv-to-derivation.nix.gen.hh"); - } - - else if (path2 == corepkgsPrefix + "fetchurl.nix") { - state.eval(state.parseExprFromString( - #include "fetchurl.nix.gen.hh" - , CanonPath::root), v); - } - - else { - if (!vScope) - state.evalFile(path, v); - else { - state.forceAttrs(*vScope, pos, "while evaluating the first argument passed to builtins.scopedImport"); - - Env * env = &state.allocEnv(vScope->attrs->size()); - env->up = &state.baseEnv; - - auto staticEnv = std::make_shared(nullptr, state.staticBaseEnv.get(), vScope->attrs->size()); - - unsigned int displ = 0; - for (auto & attr : *vScope->attrs) { - staticEnv->vars.emplace_back(attr.name, displ); - env->values[displ++] = attr.value; - } - - // No need to call staticEnv.sort(), because - // args[0]->attrs is already sorted. - - debug("evaluating file '%1%'", path); - Expr * e = state.parseExprFromFile(resolveExprPath(path), staticEnv); - - e->eval(state, *env, v); - } - } -} - -static RegisterPrimOp primop_scopedImport(PrimOp { - .name = "scopedImport", - .arity = 2, - .fun = [](EvalState & state, const PosIdx pos, Value * * args, Value & v) - { - import(state, pos, *args[1], args[0], v); - } -}); - -static RegisterPrimOp primop_import({ - .name = "import", - .args = {"path"}, - // TODO turn "normal path values" into link below - .doc = R"( - Load, parse and return the Nix expression in the file *path*. - - The value *path* can be a path, a string, or an attribute set with an - `__toString` attribute or a `outPath` attribute (as derivations or flake - inputs typically have). - - If *path* is a directory, the file `default.nix` in that directory - is loaded. - - Evaluation aborts if the file doesn’t exist or contains - an incorrect Nix expression. `import` implements Nix’s module - system: you can put any Nix expression (such as a set or a - function) in a separate file, and use it from Nix expressions in - other files. - - > **Note** - > - > Unlike some languages, `import` is a regular function in Nix. - > Paths using the angle bracket syntax (e.g., `import` *\*) - > are normal [path values](@docroot@/language/values.md#type-path). - - A Nix expression loaded by `import` must not contain any *free - variables* (identifiers that are not defined in the Nix expression - itself and are not built-in). Therefore, it cannot refer to - variables that are in scope at the call site. For instance, if you - have a calling expression - - ```nix - rec { - x = 123; - y = import ./foo.nix; - } - ``` - - then the following `foo.nix` will give an error: - - ```nix - x + 456 - ``` - - since `x` is not in scope in `foo.nix`. If you want `x` to be - available in `foo.nix`, you should pass it as a function argument: - - ```nix - rec { - x = 123; - y = import ./foo.nix x; - } - ``` - - and - - ```nix - x: x + 456 - ``` - - (The function argument doesn’t have to be called `x` in `foo.nix`; - any name would work.) - )", - .fun = [](EvalState & state, const PosIdx pos, Value * * args, Value & v) - { - import(state, pos, *args[0], nullptr, v); - } -}); - -/* Want reasonable symbol names, so extern C */ -/* !!! Should we pass the Pos or the file name too? */ -extern "C" typedef void (*ValueInitializer)(EvalState & state, Value & v); - -/* Load a ValueInitializer from a DSO and return whatever it initializes */ -void prim_importNative(EvalState & state, const PosIdx pos, Value * * args, Value & v) -{ - auto path = realisePath(state, pos, *args[0]); - - std::string sym(state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument passed to builtins.importNative")); - - void *handle = dlopen(path.path.c_str(), RTLD_LAZY | RTLD_LOCAL); - if (!handle) - state.error("could not open '%1%': %2%", path, dlerror()).debugThrow(); - - dlerror(); - ValueInitializer func = (ValueInitializer) dlsym(handle, sym.c_str()); - if(!func) { - char *message = dlerror(); - if (message) - state.error("could not load symbol '%1%' from '%2%': %3%", sym, path, message).debugThrow(); - else - state.error("symbol '%1%' from '%2%' resolved to NULL when a function pointer was expected", sym, path).debugThrow(); - } - - (func)(state, v); - - /* We don't dlclose because v may be a primop referencing a function in the shared object file */ -} - /* Execute a program and parse its output */ void prim_exec(EvalState & state, const PosIdx pos, Value * * args, Value & v) diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh index b326cfd59..26b5fa682 100644 --- a/src/libexpr/primops.hh +++ b/src/libexpr/primops.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include "derivations.hh" #include "eval.hh" #include @@ -53,8 +54,17 @@ void prim_exec(EvalState & state, const PosIdx pos, Value ** args, Value & v); void prim_lessThan(EvalState & state, const PosIdx pos, Value ** args, Value & v); +void prim_importNative(EvalState & state, const PosIdx pos, Value ** args, Value & v); + void makePositionThunks(EvalState & state, const PosIdx pos, Value & line, Value & column); +void mkOutputString( + EvalState & state, + BindingsBuilder & attrs, + const StorePath & drvPath, + const std::pair & o +); + #if HAVE_BOEHMGC typedef std::list> ValueList; #else diff --git a/src/libexpr/primops/import.cc b/src/libexpr/primops/import.cc new file mode 100644 index 000000000..fbabc0bc4 --- /dev/null +++ b/src/libexpr/primops/import.cc @@ -0,0 +1,270 @@ +#include "derivations.hh" +#include "primops.hh" +#include "store-api.hh" + +namespace nix { + +/* Want reasonable symbol names, so extern C */ +/* !!! Should we pass the Pos or the file name too? */ +extern "C" typedef void (*ValueInitializer)(EvalState & state, Value & v); + +/** + * Add and attribute to the given attribute map from the output name to + * the output path, or a placeholder. + * + * Where possible the path is used, but for floating CA derivations we + * may not know it. For sake of determinism we always assume we don't + * and instead put in a place holder. In either case, however, the + * string context will contain the drv path and output name, so + * downstream derivations will have the proper dependency, and in + * addition, before building, the placeholder will be rewritten to be + * the actual path. + * + * The 'drv' and 'drvPath' outputs must correspond. + */ + +void mkOutputString( + EvalState & state, + BindingsBuilder & attrs, + const StorePath & drvPath, + const std::pair & o +) +{ + state.mkOutputString( + attrs.alloc(o.first), + SingleDerivedPath::Built{ + .drvPath = makeConstantStorePathRef(drvPath), + .output = o.first, + }, + o.second.path(*state.store, Derivation::nameFromPath(drvPath), o.first) + ); +} + +/* Load and evaluate an expression from path specified by the + argument. */ +static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * vScope, Value & v) +{ + auto path = realisePath(state, pos, vPath); + auto path2 = path.path.abs(); + + // FIXME + auto isValidDerivationInStore = [&]() -> std::optional { + if (!state.store->isStorePath(path2)) { + return std::nullopt; + } + auto storePath = state.store->parseStorePath(path2); + if (!(state.store->isValidPath(storePath) && isDerivation(path2))) { + return std::nullopt; + } + return storePath; + }; + + if (auto storePath = isValidDerivationInStore()) { + Derivation drv = state.store->readDerivation(*storePath); + auto attrs = state.buildBindings(3 + drv.outputs.size()); + attrs.alloc(state.sDrvPath) + .mkString( + path2, + { + NixStringContextElem::DrvDeep{.drvPath = *storePath}, + } + ); + attrs.alloc(state.sName).mkString(drv.env["name"]); + auto & outputsVal = attrs.alloc(state.sOutputs); + state.mkList(outputsVal, drv.outputs.size()); + + for (const auto & [i, o] : enumerate(drv.outputs)) { + mkOutputString(state, attrs, *storePath, o); + (outputsVal.listElems()[i] = state.allocValue())->mkString(o.first); + } + + auto w = state.allocValue(); + w->mkAttrs(attrs); + + if (!state.vImportedDrvToDerivation) { + state.vImportedDrvToDerivation = allocRootValue(state.allocValue()); + state.eval( + state.parseExprFromString( +#include "imported-drv-to-derivation.nix.gen.hh" + , CanonPath::root + ), + **state.vImportedDrvToDerivation + ); + } + + state.forceFunction( + **state.vImportedDrvToDerivation, + pos, + "while evaluating imported-drv-to-derivation.nix.gen.hh" + ); + v.mkApp(*state.vImportedDrvToDerivation, w); + state.forceAttrs(v, pos, "while calling imported-drv-to-derivation.nix.gen.hh"); + } + + else if (path2 == corepkgsPrefix + "fetchurl.nix") + { + state.eval( + state.parseExprFromString( +#include "fetchurl.nix.gen.hh" + , CanonPath::root + ), + v + ); + } + + else + { + if (!vScope) { + state.evalFile(path, v); + } else { + state.forceAttrs( + *vScope, pos, "while evaluating the first argument passed to builtins.scopedImport" + ); + + Env * env = &state.allocEnv(vScope->attrs->size()); + env->up = &state.baseEnv; + + auto staticEnv = std::make_shared( + nullptr, state.staticBaseEnv.get(), vScope->attrs->size() + ); + + unsigned int displ = 0; + for (auto & attr : *vScope->attrs) { + staticEnv->vars.emplace_back(attr.name, displ); + env->values[displ++] = attr.value; + } + + // No need to call staticEnv.sort(), because + // args[0]->attrs is already sorted. + + debug("evaluating file '%1%'", path); + Expr * e = state.parseExprFromFile(resolveExprPath(path), staticEnv); + + e->eval(state, *env, v); + } + } +} + +static void prim_import(EvalState & state, const PosIdx pos, Value ** args, Value & v) +{ + import(state, pos, *args[0], nullptr, v); +} + +static RegisterPrimOp primop_import({ + .name = "import", + .args = {"path"}, + // TODO turn "normal path values" into link below + .doc = R"( + Load, parse and return the Nix expression in the file *path*. + + The value *path* can be a path, a string, or an attribute set with an + `__toString` attribute or a `outPath` attribute (as derivations or flake + inputs typically have). + + If *path* is a directory, the file `default.nix` in that directory + is loaded. + + Evaluation aborts if the file doesn’t exist or contains + an incorrect Nix expression. `import` implements Nix’s module + system: you can put any Nix expression (such as a set or a + function) in a separate file, and use it from Nix expressions in + other files. + + > **Note** + > + > Unlike some languages, `import` is a regular function in Nix. + > Paths using the angle bracket syntax (e.g., `import` *\*) + > are normal [path values](@docroot@/language/values.md#type-path). + + A Nix expression loaded by `import` must not contain any *free + variables* (identifiers that are not defined in the Nix expression + itself and are not built-in). Therefore, it cannot refer to + variables that are in scope at the call site. For instance, if you + have a calling expression + + ```nix + rec { + x = 123; + y = import ./foo.nix; + } + ``` + + then the following `foo.nix` will give an error: + + ```nix + x + 456 + ``` + + since `x` is not in scope in `foo.nix`. If you want `x` to be + available in `foo.nix`, you should pass it as a function argument: + + ```nix + rec { + x = 123; + y = import ./foo.nix x; + } + ``` + + and + + ```nix + x: x + 456 + ``` + + (The function argument doesn’t have to be called `x` in `foo.nix`; + any name would work.) + )", + .fun = prim_import, +}); + +/* Load a ValueInitializer from a DSO and return whatever it initializes */ +void prim_importNative(EvalState & state, const PosIdx pos, Value ** args, Value & v) +{ + auto path = realisePath(state, pos, *args[0]); + + std::string sym(state.forceStringNoCtx( + *args[1], pos, "while evaluating the second argument passed to builtins.importNative" + )); + + void * handle = dlopen(path.path.c_str(), RTLD_LAZY | RTLD_LOCAL); + if (!handle) { + state.error("could not open '%1%': %2%", path, dlerror()).debugThrow(); + } + + dlerror(); + ValueInitializer func = (ValueInitializer) dlsym(handle, sym.c_str()); + if (!func) { + char * message = dlerror(); + if (message) { + state + .error("could not load symbol '%1%' from '%2%': %3%", sym, path, message) + .debugThrow(); + } else { + state + .error( + "symbol '%1%' from '%2%' resolved to NULL when a function pointer was expected", + sym, + path + ) + .debugThrow(); + } + } + + (func)(state, v); + + /* We don't dlclose because v may be a primop referencing a function in the shared object file + */ +} + +static void prim_scopedImport(EvalState & state, const PosIdx pos, Value ** args, Value & v) +{ + import(state, pos, *args[1], args[0], v); +} + +static RegisterPrimOp primop_scopedImport(PrimOp{ + .name = "scopedImport", + .arity = 2, + .fun = prim_scopedImport, +}); + +}