lix-project/lix
20
57
Fork 31
Code Issues 347 Code Review (Gerrit) Projects 3 Releases Wiki Activity

Parse string context elements properly

Browse source 
Prior to this change, we had a bunch of ad-hoc string manipulation code
scattered around. This made it hard to figure out what data model for
string contexts is.

Now, we still store string contexts most of the time as encoded strings
--- I was wary of the performance implications of changing that --- but
whenever we parse them we do so only through the
`NixStringContextElem::parse` method, which handles all cases. This
creates a data type that is very similar to `DerivedPath` but:

 - Represents the funky `=<drvpath>` case as properly distinct from the
   others.

 - Only encodes a single output, no wildcards and no set, for the
   "built" case.

(I would like to deprecate `=<path>`, after which we are in spitting
distance of `DerivedPath` and could maybe get away with fewer types, but
that is another topic for another day.)
This commit is contained in:
John Ericson 2023-01-03 11:44:59 -05:00
parent da64f026dd
commit 5576d5e987
14 changed files with 347 additions and 111 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
src/libexpr/eval-cache.cc
View file

@ -300,7 +300,7 @@ struct AttrDb
NixStringContext context;
if (!queryAttribute.isNull(3))
for (auto & s : tokenizeString<std::vector<std::string>>(queryAttribute.getStr(3), ";"))
context.push_back(decodeContext(cfg, s));
context.push_back(NixStringContextElem::parse(cfg, s));
return {{rowId, string_t{queryAttribute.getStr(2), context}}};
}
case AttrType::Bool:
@ -592,7 +592,18 @@ string_t AttrCursor::getStringWithContext()
if (auto s = std::get_if<string_t>(&cachedValue->second)) {
bool valid = true;
for (auto & c : s->second) {
if (!root->state.store->isValidPath(c.first)) {
const StorePath & path = std::visit(overloaded {
[&](const NixStringContextElem::DrvDeep & d) -> const StorePath & {
return d.drvPath;
},
[&](const NixStringContextElem::Built & b) -> const StorePath & {
return b.drvPath;
},
[&](const NixStringContextElem::Opaque & o) -> const StorePath & {
return o.path;
},
}, c.raw());
if (!root->state.store->isValidPath(path)) {
valid = false;
break;
}

23
src/libexpr/eval.cc
View file

@ -2068,27 +2068,6 @@ std::string_view EvalState::forceString(Value & v, const PosIdx pos, std::string
}
/* Decode a context string ‘!<name>!<path>’ into a pair <path,
name>. */
NixStringContextElem decodeContext(const Store & store, std::string_view s)
{
if (s.at(0) == '!') {
size_t index = s.find("!", 1);
return {
store.parseStorePath(s.substr(index + 1)),
std::string(s.substr(1, index - 1)),
};
} else
return {
store.parseStorePath(
s.at(0) == '/'
? s
: s.substr(1)),
"",
};
}
void copyContext(const Value & v, PathSet & context)
{
if (v.string.context)
@ -2103,7 +2082,7 @@ NixStringContext Value::getContext(const Store & store)
assert(internalType == tString);
if (string.context)
for (const char * * p = string.context; *p; ++p)
res.push_back(decodeContext(store, *p));
res.push_back(NixStringContextElem::parse(store, *p));
return res;
}

4
src/libexpr/eval.hh
View file

@ -551,10 +551,6 @@ struct DebugTraceStacker {
std::string_view showType(ValueType type);
std::string showType(const Value & v);
/* Decode a context string ‘!<name>!<path>’ into a pair <path,
name>. */
NixStringContextElem decodeContext(const Store & store, std::string_view s);
/* If `path' refers to a directory, then append "/default.nix". */
Path resolveExprPath(Path path);

3
src/libexpr/local.mk
View file

@ -6,6 +6,7 @@ libexpr_DIR := $(d)
libexpr_SOURCES := \
$(wildcard $(d)/*.cc) \
$(wildcard $(d)/value/*.cc) \
$(wildcard $(d)/primops/*.cc) \
$(wildcard $(d)/flake/*.cc) \
$(d)/lexer-tab.cc \
@ -37,6 +38,8 @@ clean-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexe
$(eval $(call install-file-in, $(d)/nix-expr.pc, $(libdir)/pkgconfig, 0644))
$(foreach i, $(wildcard src/libexpr/value/*.hh), \
$(eval $(call install-file-in, $(i), $(includedir)/nix/value, 0644)))
$(foreach i, $(wildcard src/libexpr/flake/*.hh), \
$(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644)))

90
src/libexpr/primops.cc
View file

@ -43,16 +43,32 @@ StringMap EvalState::realiseContext(const PathSet & context)
std::vector<DerivedPath::Built> drvs;
StringMap res;
for (auto & i : context) {
auto [ctx, outputName] = decodeContext(*store, i);
auto ctxS = store->printStorePath(ctx);
if (!store->isValidPath(ctx))
debugThrowLastTrace(InvalidPathError(store->printStorePath(ctx)));
if (!outputName.empty() && ctx.isDerivation()) {
drvs.push_back({ctx, {outputName}});
} else {
res.insert_or_assign(ctxS, ctxS);
}
for (auto & c_ : context) {
auto ensureValid = [&](const StorePath & p) {
if (!store->isValidPath(p))
debugThrowLastTrace(InvalidPathError(store->printStorePath(p)));
};
auto c = NixStringContextElem::parse(*store, c_);
std::visit(overloaded {
[&](const NixStringContextElem::Built & b) {
drvs.push_back(DerivedPath::Built {
.drvPath = b.drvPath,
.outputs = std::set { b.output },
});
ensureValid(b.drvPath);
},
[&](const NixStringContextElem::Opaque & o) {
auto ctxS = store->printStorePath(o.path);
res.insert_or_assign(ctxS, ctxS);
ensureValid(o.path);
},
[&](const NixStringContextElem::DrvDeep & d) {
/* Treat same as Opaque */
auto ctxS = store->printStorePath(d.drvPath);
res.insert_or_assign(ctxS, ctxS);
ensureValid(d.drvPath);
},
}, c.raw());
}
if (drvs.empty()) return {};
@ -1179,35 +1195,31 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * *
/* Everything in the context of the strings in the derivation
attributes should be added as dependencies of the resulting
derivation. */
for (auto & path : context) {
/* Paths marked with `=' denote that the path of a derivation
is explicitly passed to the builder. Since that allows the
builder to gain access to every path in the dependency
graph of the derivation (including all outputs), all paths
in the graph must be added to this derivation's list of
inputs to ensure that they are available when the builder
runs. */
if (path.at(0) == '=') {
/* !!! This doesn't work if readOnlyMode is set. */
StorePathSet refs;
state.store->computeFSClosure(state.store->parseStorePath(std::string_view(path).substr(1)), refs);
for (auto & j : refs) {
drv.inputSrcs.insert(j);
if (j.isDerivation())
drv.inputDrvs[j] = state.store->readDerivation(j).outputNames();
}
}
/* Handle derivation outputs of the form ‘!<name>!<path>’. */
else if (path.at(0) == '!') {
auto ctx = decodeContext(*state.store, path);
drv.inputDrvs[ctx.first].insert(ctx.second);
}
/* Otherwise it's a source file. */
else
drv.inputSrcs.insert(state.store->parseStorePath(path));
for (auto & c_ : context) {
auto c = NixStringContextElem::parse(*state.store, c_);
std::visit(overloaded {
/* Since this allows the builder to gain access to every
path in the dependency graph of the derivation (including
all outputs), all paths in the graph must be added to
this derivation's list of inputs to ensure that they are
available when the builder runs. */
[&](const NixStringContextElem::DrvDeep & d) {
/* !!! This doesn't work if readOnlyMode is set. */
StorePathSet refs;
state.store->computeFSClosure(d.drvPath, refs);
for (auto & j : refs) {
drv.inputSrcs.insert(j);
if (j.isDerivation())
drv.inputDrvs[j] = state.store->readDerivation(j).outputNames();
}
},
[&](const NixStringContextElem::Built & b) {
drv.inputDrvs[b.drvPath].insert(b.output);
},
[&](const NixStringContextElem::Opaque & o) {
drv.inputSrcs.insert(o.path);
},
}, c.raw());
}
/* Do we have all required attributes? */

51
src/libexpr/primops/context.cc
View file

@ -37,8 +37,15 @@ static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx p
auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.unsafeDiscardOutputDependency");
PathSet context2;
for (auto & p : context)
context2.insert(p.at(0) == '=' ? std::string(p, 1) : p);
for (auto && p : context) {
auto c = NixStringContextElem::parse(*state.store, p);
if (auto * ptr = std::get_if<NixStringContextElem::DrvDeep>(&c)) {
context2.emplace(state.store->printStorePath(ptr->drvPath));
} else {
/* Can reuse original item */
context2.emplace(std::move(p));
}
}
v.mkString(*s, context2);
}
@ -74,34 +81,22 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args,
};
PathSet context;
state.forceString(*args[0], context, pos, "while evaluating the argument passed to builtins.getContext");
auto contextInfos = std::map<Path, ContextInfo>();
auto contextInfos = std::map<StorePath, ContextInfo>();
for (const auto & p : context) {
Path drv;
std::string output;
const Path * path = &p;
if (p.at(0) == '=') {
drv = std::string(p, 1);
path = &drv;
} else if (p.at(0) == '!') {
NixStringContextElem ctx = decodeContext(*state.store, p);
drv = state.store->printStorePath(ctx.first);
output = ctx.second;
path = &drv;
}
auto isPath = drv.empty();
auto isAllOutputs = (!drv.empty()) && output.empty();
auto iter = contextInfos.find(*path);
if (iter == contextInfos.end()) {
contextInfos.emplace(*path, ContextInfo{isPath, isAllOutputs, output.empty() ? Strings{} : Strings{std::move(output)}});
} else {
if (isPath)
iter->second.path = true;
else if (isAllOutputs)
iter->second.allOutputs = true;
else
iter->second.outputs.emplace_back(std::move(output));
}
NixStringContextElem ctx = NixStringContextElem::parse(*state.store, p);
std::visit(overloaded {
[&](NixStringContextElem::DrvDeep & d) {
contextInfos[d.drvPath].allOutputs = true;
},
[&](NixStringContextElem::Built & b) {
contextInfos[b.drvPath].outputs.emplace_back(std::move(output));
},
[&](NixStringContextElem::Opaque & o) {
contextInfos[o.path].path = true;
},
}, ctx.raw());
}
auto attrs = state.buildBindings(contextInfos.size());
@ -120,7 +115,7 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args,
for (const auto & [i, output] : enumerate(info.second.outputs))
(outputsVal.listElems()[i] = state.allocValue())->mkString(output);
}
attrs.alloc(info.first).mkAttrs(infoAttrs);
attrs.alloc(state.store->printStorePath(info.first)).mkAttrs(infoAttrs);
}
v.mkAttrs(attrs);

4
src/libexpr/tests/local.mk
View file

@ -6,7 +6,9 @@ libexpr-tests_DIR := $(d)
libexpr-tests_INSTALL_DIR :=
libexpr-tests_SOURCES := $(wildcard $(d)/*.cc)
libexpr-tests_SOURCES := \
$(wildcard $(d)/*.cc) \
$(wildcard $(d)/value/*.cc)
libexpr-tests_CXXFLAGS += -I src/libexpr -I src/libutil -I src/libstore -I src/libexpr/tests

72
src/libexpr/tests/value/context.cc Normal file
View file

@ -0,0 +1,72 @@
#include "value/context.hh"
#include "libexprtests.hh"
namespace nix {
// Testing of trivial expressions
struct NixStringContextElemTest : public LibExprTest {
const Store & store() const {
return *LibExprTest::store;
}
};
TEST_F(NixStringContextElemTest, empty_invalid) {
EXPECT_THROW(
NixStringContextElem::parse(store(), ""),
BadNixStringContextElem);
}
TEST_F(NixStringContextElemTest, single_bang_invalid) {
EXPECT_THROW(
NixStringContextElem::parse(store(), "!"),
BadNixStringContextElem);
}
TEST_F(NixStringContextElemTest, double_bang_invalid) {
EXPECT_THROW(
NixStringContextElem::parse(store(), "!!/"),
BadStorePath);
}
TEST_F(NixStringContextElemTest, eq_slash_invalid) {
EXPECT_THROW(
NixStringContextElem::parse(store(), "=/"),
BadStorePath);
}
TEST_F(NixStringContextElemTest, slash_invalid) {
EXPECT_THROW(
NixStringContextElem::parse(store(), "/"),
BadStorePath);
}
TEST_F(NixStringContextElemTest, opaque) {
std::string_view opaque = "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x";
auto elem = NixStringContextElem::parse(store(), opaque);
auto * p = std::get_if<NixStringContextElem::Opaque>(&elem);
ASSERT_TRUE(p);
ASSERT_EQ(p->path, store().parseStorePath(opaque));
ASSERT_EQ(elem.to_string(store()), opaque);
}
TEST_F(NixStringContextElemTest, drvDeep) {
std::string_view drvDeep = "=/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv";
auto elem = NixStringContextElem::parse(store(), drvDeep);
auto * p = std::get_if<NixStringContextElem::DrvDeep>(&elem);
ASSERT_TRUE(p);
ASSERT_EQ(p->drvPath, store().parseStorePath(drvDeep.substr(1)));
ASSERT_EQ(elem.to_string(store()), drvDeep);
}
TEST_F(NixStringContextElemTest, built) {
std::string_view built = "!foo!/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv";
auto elem = NixStringContextElem::parse(store(), built);
auto * p = std::get_if<NixStringContextElem::Built>(&elem);
ASSERT_TRUE(p);
ASSERT_EQ(p->output, "foo");
ASSERT_EQ(p->drvPath, store().parseStorePath(built.substr(5)));
ASSERT_EQ(elem.to_string(store()), built);
}
}

3
src/libexpr/value.hh
View file

@ -3,6 +3,7 @@
#include <cassert>
#include "symbol-table.hh"
#include "value/context.hh"
#if HAVE_BOEHMGC
#include <gc/gc_allocator.h>
@ -67,8 +68,6 @@ class XMLWriter;
typedef int64_t NixInt;
typedef double NixFloat;
typedef std::pair<StorePath, std::string> NixStringContextElem;
typedef std::vector<NixStringContextElem> NixStringContext;
/* External values must descend from ExternalValueBase, so that
* type-agnostic nix functions (e.g. showType) can be implemented

67
src/libexpr/value/context.cc Normal file
View file

@ -0,0 +1,67 @@
#include "value/context.hh"
#include "store-api.hh"
#include <optional>
namespace nix {
NixStringContextElem NixStringContextElem::parse(const Store & store, std::string_view s0)
{
std::string_view s = s0;
if (s.size() == 0) {
throw BadNixStringContextElem(s0,
"String context element should never be an empty string");
}
switch (s.at(0)) {
case '!': {
s = s.substr(1); // advance string to parse after first !
size_t index = s.find("!");
// This makes index + 1 safe. Index can be the length (one after index
// of last character), so given any valid character index --- a
// successful find --- we can add one.
if (index == std::string_view::npos) {
throw BadNixStringContextElem(s0,
"String content element beginning with '!' should have a second '!'");
}
return NixStringContextElem::Built {
.drvPath = store.parseStorePath(s.substr(index + 1)),
.output = std::string(s.substr(0, index)),
};
}
case '=': {
return NixStringContextElem::DrvDeep {
.drvPath = store.parseStorePath(s.substr(1)),
};
}
default: {
return NixStringContextElem::Opaque {
.path = store.parseStorePath(s),
};
}
}
}
std::string NixStringContextElem::to_string(const Store & store) const {
return std::visit(overloaded {
[&](const NixStringContextElem::Built & b) {
std::string res;
res += '!';
res += b.output;
res += '!';
res += store.printStorePath(b.drvPath);
return res;
},
[&](const NixStringContextElem::DrvDeep & d) {
std::string res;
res += '=';
res += store.printStorePath(d.drvPath);
return res;
},
[&](const NixStringContextElem::Opaque & o) {
return store.printStorePath(o.path);
},
}, raw());
}
}

90
src/libexpr/value/context.hh Normal file
View file

@ -0,0 +1,90 @@
#pragma once
#include "util.hh"
#include "path.hh"
#include <optional>
#include <nlohmann/json_fwd.hpp>
namespace nix {
class BadNixStringContextElem : public Error
{
public:
std::string_view raw;
template<typename... Args>
BadNixStringContextElem(std::string_view raw_, const Args & ... args)
: Error("")
{
raw = raw_;
auto hf = hintfmt(args...);
err.msg = hintfmt("Bad String Context element: %1%: %2%", normaltxt(hf.str()), raw);
}
};
class Store;
/* Plain opaque path to some store object.
Encoded as just the path: ‘<path>’.
*/
struct NixStringContextElem_Opaque {
StorePath path;
};
/* Path to a derivation and its entire build closure.
The path doesn't just refer to derivation itself and its closure, but
also all outputs of all derivations in that closure (including the
root derivation).
Encoded in the form ‘=<drvPath>’.
*/
struct NixStringContextElem_DrvDeep {
StorePath drvPath;
};
/* Derivation output.
Encoded in the form ‘!<output>!<drvPath>’.
*/
struct NixStringContextElem_Built {
StorePath drvPath;
std::string output;
};
using _NixStringContextElem_Raw = std::variant<
NixStringContextElem_Opaque,
NixStringContextElem_DrvDeep,
NixStringContextElem_Built
>;
struct NixStringContextElem : _NixStringContextElem_Raw {
using Raw = _NixStringContextElem_Raw;
using Raw::Raw;
using Opaque = NixStringContextElem_Opaque;
using DrvDeep = NixStringContextElem_DrvDeep;
using Built = NixStringContextElem_Built;
inline const Raw & raw() const {
return static_cast<const Raw &>(*this);
}
inline Raw & raw() {
return static_cast<Raw &>(*this);
}
/* Decode a context string, one of:
- ‘<path>’
- ‘=<path>’
- ‘!<name>!<path>’
*/
static NixStringContextElem parse(const Store & store, std::string_view s);
std::string to_string(const Store & store) const;
};
typedef std::vector<NixStringContextElem> NixStringContext;
}

32
src/nix/app.cc
View file

@ -80,17 +80,27 @@ UnresolvedApp Installable::toApp(EvalState & state)
auto [program, context] = cursor->getAttr("program")->getStringWithContext();
std::vector<DerivedPath> context2;
for (auto & [path, name] : context) {
context2.push_back(name != "" || path.isDerivation()
? (DerivedPath) DerivedPath::Built {
.drvPath = path,
.outputs = name != ""
? StringSet { name }
: StringSet { },
}
: (DerivedPath) DerivedPath::Opaque {
.path = path,
});
for (auto & c : context) {
context2.emplace_back(std::visit(overloaded {
[&](const NixStringContextElem::DrvDeep & d) -> DerivedPath {
/* We want all outputs of the drv */
return DerivedPath::Built {
.drvPath = d.drvPath,
.outputs = {},
};
},
[&](const NixStringContextElem::Built & b) -> DerivedPath {
return DerivedPath::Built {
.drvPath = b.drvPath,
.outputs = { b.output },
};
},
[&](const NixStringContextElem::Opaque & o) -> DerivedPath {
return DerivedPath::Opaque {
.path = o.path,
};
},
}, c.raw()));
}
return UnresolvedApp{App {

2
src/nix/flake.cc
View file

@ -348,7 +348,7 @@ struct CmdFlakeCheck : FlakeCommand
// FIXME
auto app = App(*state, v);
for (auto & i : app.context) {
auto [drvPathS, outputName] = decodeContext(i);
auto [drvPathS, outputName] = NixStringContextElem::parse(i);
store->parseStorePath(drvPathS);
}
#endif

2
tests/plugins/local.mk
View file

@ -8,4 +8,4 @@ libplugintest_ALLOW_UNDEFINED := 1
libplugintest_EXCLUDE_FROM_LIBRARY_LIST := 1
libplugintest_CXXFLAGS := -I src/libutil -I src/libexpr
libplugintest_CXXFLAGS := -I src/libutil -I src/libstore -I src/libexpr