Parse string context elements properly

Prior to this change, we had a bunch of ad-hoc string manipulation code
scattered around. This made it hard to figure out what data model for
string contexts is.

Now, we still store string contexts most of the time as encoded strings
--- I was wary of the performance implications of changing that --- but
whenever we parse them we do so only through the
`NixStringContextElem::parse` method, which handles all cases. This
creates a data type that is very similar to `DerivedPath` but:

 - Represents the funky `=<drvpath>` case as properly distinct from the
   others.

 - Only encodes a single output, no wildcards and no set, for the
   "built" case.

(I would like to deprecate `=<path>`, after which we are in spitting
distance of `DerivedPath` and could maybe get away with fewer types, but
that is another topic for another day.)
This commit is contained in:
John Ericson 2023-01-03 11:44:59 -05:00
parent da64f026dd
commit 5576d5e987
14 changed files with 347 additions and 111 deletions

View file

@ -300,7 +300,7 @@ struct AttrDb
NixStringContext context; NixStringContext context;
if (!queryAttribute.isNull(3)) if (!queryAttribute.isNull(3))
for (auto & s : tokenizeString<std::vector<std::string>>(queryAttribute.getStr(3), ";")) for (auto & s : tokenizeString<std::vector<std::string>>(queryAttribute.getStr(3), ";"))
context.push_back(decodeContext(cfg, s)); context.push_back(NixStringContextElem::parse(cfg, s));
return {{rowId, string_t{queryAttribute.getStr(2), context}}}; return {{rowId, string_t{queryAttribute.getStr(2), context}}};
} }
case AttrType::Bool: case AttrType::Bool:
@ -592,7 +592,18 @@ string_t AttrCursor::getStringWithContext()
if (auto s = std::get_if<string_t>(&cachedValue->second)) { if (auto s = std::get_if<string_t>(&cachedValue->second)) {
bool valid = true; bool valid = true;
for (auto & c : s->second) { for (auto & c : s->second) {
if (!root->state.store->isValidPath(c.first)) { const StorePath & path = std::visit(overloaded {
[&](const NixStringContextElem::DrvDeep & d) -> const StorePath & {
return d.drvPath;
},
[&](const NixStringContextElem::Built & b) -> const StorePath & {
return b.drvPath;
},
[&](const NixStringContextElem::Opaque & o) -> const StorePath & {
return o.path;
},
}, c.raw());
if (!root->state.store->isValidPath(path)) {
valid = false; valid = false;
break; break;
} }

View file

@ -2068,27 +2068,6 @@ std::string_view EvalState::forceString(Value & v, const PosIdx pos, std::string
} }
/* Decode a context string !<name>!<path> into a pair <path,
name>. */
NixStringContextElem decodeContext(const Store & store, std::string_view s)
{
if (s.at(0) == '!') {
size_t index = s.find("!", 1);
return {
store.parseStorePath(s.substr(index + 1)),
std::string(s.substr(1, index - 1)),
};
} else
return {
store.parseStorePath(
s.at(0) == '/'
? s
: s.substr(1)),
"",
};
}
void copyContext(const Value & v, PathSet & context) void copyContext(const Value & v, PathSet & context)
{ {
if (v.string.context) if (v.string.context)
@ -2103,7 +2082,7 @@ NixStringContext Value::getContext(const Store & store)
assert(internalType == tString); assert(internalType == tString);
if (string.context) if (string.context)
for (const char * * p = string.context; *p; ++p) for (const char * * p = string.context; *p; ++p)
res.push_back(decodeContext(store, *p)); res.push_back(NixStringContextElem::parse(store, *p));
return res; return res;
} }

View file

@ -551,10 +551,6 @@ struct DebugTraceStacker {
std::string_view showType(ValueType type); std::string_view showType(ValueType type);
std::string showType(const Value & v); std::string showType(const Value & v);
/* Decode a context string !<name>!<path> into a pair <path,
name>. */
NixStringContextElem decodeContext(const Store & store, std::string_view s);
/* If `path' refers to a directory, then append "/default.nix". */ /* If `path' refers to a directory, then append "/default.nix". */
Path resolveExprPath(Path path); Path resolveExprPath(Path path);

View file

@ -6,6 +6,7 @@ libexpr_DIR := $(d)
libexpr_SOURCES := \ libexpr_SOURCES := \
$(wildcard $(d)/*.cc) \ $(wildcard $(d)/*.cc) \
$(wildcard $(d)/value/*.cc) \
$(wildcard $(d)/primops/*.cc) \ $(wildcard $(d)/primops/*.cc) \
$(wildcard $(d)/flake/*.cc) \ $(wildcard $(d)/flake/*.cc) \
$(d)/lexer-tab.cc \ $(d)/lexer-tab.cc \
@ -37,6 +38,8 @@ clean-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexe
$(eval $(call install-file-in, $(d)/nix-expr.pc, $(libdir)/pkgconfig, 0644)) $(eval $(call install-file-in, $(d)/nix-expr.pc, $(libdir)/pkgconfig, 0644))
$(foreach i, $(wildcard src/libexpr/value/*.hh), \
$(eval $(call install-file-in, $(i), $(includedir)/nix/value, 0644)))
$(foreach i, $(wildcard src/libexpr/flake/*.hh), \ $(foreach i, $(wildcard src/libexpr/flake/*.hh), \
$(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644))) $(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644)))

View file

@ -43,16 +43,32 @@ StringMap EvalState::realiseContext(const PathSet & context)
std::vector<DerivedPath::Built> drvs; std::vector<DerivedPath::Built> drvs;
StringMap res; StringMap res;
for (auto & i : context) { for (auto & c_ : context) {
auto [ctx, outputName] = decodeContext(*store, i); auto ensureValid = [&](const StorePath & p) {
auto ctxS = store->printStorePath(ctx); if (!store->isValidPath(p))
if (!store->isValidPath(ctx)) debugThrowLastTrace(InvalidPathError(store->printStorePath(p)));
debugThrowLastTrace(InvalidPathError(store->printStorePath(ctx))); };
if (!outputName.empty() && ctx.isDerivation()) { auto c = NixStringContextElem::parse(*store, c_);
drvs.push_back({ctx, {outputName}}); std::visit(overloaded {
} else { [&](const NixStringContextElem::Built & b) {
drvs.push_back(DerivedPath::Built {
.drvPath = b.drvPath,
.outputs = std::set { b.output },
});
ensureValid(b.drvPath);
},
[&](const NixStringContextElem::Opaque & o) {
auto ctxS = store->printStorePath(o.path);
res.insert_or_assign(ctxS, ctxS); res.insert_or_assign(ctxS, ctxS);
} ensureValid(o.path);
},
[&](const NixStringContextElem::DrvDeep & d) {
/* Treat same as Opaque */
auto ctxS = store->printStorePath(d.drvPath);
res.insert_or_assign(ctxS, ctxS);
ensureValid(d.drvPath);
},
}, c.raw());
} }
if (drvs.empty()) return {}; if (drvs.empty()) return {};
@ -1179,35 +1195,31 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * *
/* Everything in the context of the strings in the derivation /* Everything in the context of the strings in the derivation
attributes should be added as dependencies of the resulting attributes should be added as dependencies of the resulting
derivation. */ derivation. */
for (auto & path : context) { for (auto & c_ : context) {
auto c = NixStringContextElem::parse(*state.store, c_);
/* Paths marked with `=' denote that the path of a derivation std::visit(overloaded {
is explicitly passed to the builder. Since that allows the /* Since this allows the builder to gain access to every
builder to gain access to every path in the dependency path in the dependency graph of the derivation (including
graph of the derivation (including all outputs), all paths all outputs), all paths in the graph must be added to
in the graph must be added to this derivation's list of this derivation's list of inputs to ensure that they are
inputs to ensure that they are available when the builder available when the builder runs. */
runs. */ [&](const NixStringContextElem::DrvDeep & d) {
if (path.at(0) == '=') {
/* !!! This doesn't work if readOnlyMode is set. */ /* !!! This doesn't work if readOnlyMode is set. */
StorePathSet refs; StorePathSet refs;
state.store->computeFSClosure(state.store->parseStorePath(std::string_view(path).substr(1)), refs); state.store->computeFSClosure(d.drvPath, refs);
for (auto & j : refs) { for (auto & j : refs) {
drv.inputSrcs.insert(j); drv.inputSrcs.insert(j);
if (j.isDerivation()) if (j.isDerivation())
drv.inputDrvs[j] = state.store->readDerivation(j).outputNames(); drv.inputDrvs[j] = state.store->readDerivation(j).outputNames();
} }
} },
[&](const NixStringContextElem::Built & b) {
/* Handle derivation outputs of the form !<name>!<path>. */ drv.inputDrvs[b.drvPath].insert(b.output);
else if (path.at(0) == '!') { },
auto ctx = decodeContext(*state.store, path); [&](const NixStringContextElem::Opaque & o) {
drv.inputDrvs[ctx.first].insert(ctx.second); drv.inputSrcs.insert(o.path);
} },
}, c.raw());
/* Otherwise it's a source file. */
else
drv.inputSrcs.insert(state.store->parseStorePath(path));
} }
/* Do we have all required attributes? */ /* Do we have all required attributes? */

View file

@ -37,8 +37,15 @@ static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx p
auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.unsafeDiscardOutputDependency"); auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.unsafeDiscardOutputDependency");
PathSet context2; PathSet context2;
for (auto & p : context) for (auto && p : context) {
context2.insert(p.at(0) == '=' ? std::string(p, 1) : p); auto c = NixStringContextElem::parse(*state.store, p);
if (auto * ptr = std::get_if<NixStringContextElem::DrvDeep>(&c)) {
context2.emplace(state.store->printStorePath(ptr->drvPath));
} else {
/* Can reuse original item */
context2.emplace(std::move(p));
}
}
v.mkString(*s, context2); v.mkString(*s, context2);
} }
@ -74,34 +81,22 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args,
}; };
PathSet context; PathSet context;
state.forceString(*args[0], context, pos, "while evaluating the argument passed to builtins.getContext"); state.forceString(*args[0], context, pos, "while evaluating the argument passed to builtins.getContext");
auto contextInfos = std::map<Path, ContextInfo>(); auto contextInfos = std::map<StorePath, ContextInfo>();
for (const auto & p : context) { for (const auto & p : context) {
Path drv; Path drv;
std::string output; std::string output;
const Path * path = &p; NixStringContextElem ctx = NixStringContextElem::parse(*state.store, p);
if (p.at(0) == '=') { std::visit(overloaded {
drv = std::string(p, 1); [&](NixStringContextElem::DrvDeep & d) {
path = &drv; contextInfos[d.drvPath].allOutputs = true;
} else if (p.at(0) == '!') { },
NixStringContextElem ctx = decodeContext(*state.store, p); [&](NixStringContextElem::Built & b) {
drv = state.store->printStorePath(ctx.first); contextInfos[b.drvPath].outputs.emplace_back(std::move(output));
output = ctx.second; },
path = &drv; [&](NixStringContextElem::Opaque & o) {
} contextInfos[o.path].path = true;
auto isPath = drv.empty(); },
auto isAllOutputs = (!drv.empty()) && output.empty(); }, ctx.raw());
auto iter = contextInfos.find(*path);
if (iter == contextInfos.end()) {
contextInfos.emplace(*path, ContextInfo{isPath, isAllOutputs, output.empty() ? Strings{} : Strings{std::move(output)}});
} else {
if (isPath)
iter->second.path = true;
else if (isAllOutputs)
iter->second.allOutputs = true;
else
iter->second.outputs.emplace_back(std::move(output));
}
} }
auto attrs = state.buildBindings(contextInfos.size()); auto attrs = state.buildBindings(contextInfos.size());
@ -120,7 +115,7 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args,
for (const auto & [i, output] : enumerate(info.second.outputs)) for (const auto & [i, output] : enumerate(info.second.outputs))
(outputsVal.listElems()[i] = state.allocValue())->mkString(output); (outputsVal.listElems()[i] = state.allocValue())->mkString(output);
} }
attrs.alloc(info.first).mkAttrs(infoAttrs); attrs.alloc(state.store->printStorePath(info.first)).mkAttrs(infoAttrs);
} }
v.mkAttrs(attrs); v.mkAttrs(attrs);

View file

@ -6,7 +6,9 @@ libexpr-tests_DIR := $(d)
libexpr-tests_INSTALL_DIR := libexpr-tests_INSTALL_DIR :=
libexpr-tests_SOURCES := $(wildcard $(d)/*.cc) libexpr-tests_SOURCES := \
$(wildcard $(d)/*.cc) \
$(wildcard $(d)/value/*.cc)
libexpr-tests_CXXFLAGS += -I src/libexpr -I src/libutil -I src/libstore -I src/libexpr/tests libexpr-tests_CXXFLAGS += -I src/libexpr -I src/libutil -I src/libstore -I src/libexpr/tests

View file

@ -0,0 +1,72 @@
#include "value/context.hh"
#include "libexprtests.hh"
namespace nix {
// Testing of trivial expressions
struct NixStringContextElemTest : public LibExprTest {
const Store & store() const {
return *LibExprTest::store;
}
};
TEST_F(NixStringContextElemTest, empty_invalid) {
EXPECT_THROW(
NixStringContextElem::parse(store(), ""),
BadNixStringContextElem);
}
TEST_F(NixStringContextElemTest, single_bang_invalid) {
EXPECT_THROW(
NixStringContextElem::parse(store(), "!"),
BadNixStringContextElem);
}
TEST_F(NixStringContextElemTest, double_bang_invalid) {
EXPECT_THROW(
NixStringContextElem::parse(store(), "!!/"),
BadStorePath);
}
TEST_F(NixStringContextElemTest, eq_slash_invalid) {
EXPECT_THROW(
NixStringContextElem::parse(store(), "=/"),
BadStorePath);
}
TEST_F(NixStringContextElemTest, slash_invalid) {
EXPECT_THROW(
NixStringContextElem::parse(store(), "/"),
BadStorePath);
}
TEST_F(NixStringContextElemTest, opaque) {
std::string_view opaque = "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x";
auto elem = NixStringContextElem::parse(store(), opaque);
auto * p = std::get_if<NixStringContextElem::Opaque>(&elem);
ASSERT_TRUE(p);
ASSERT_EQ(p->path, store().parseStorePath(opaque));
ASSERT_EQ(elem.to_string(store()), opaque);
}
TEST_F(NixStringContextElemTest, drvDeep) {
std::string_view drvDeep = "=/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv";
auto elem = NixStringContextElem::parse(store(), drvDeep);
auto * p = std::get_if<NixStringContextElem::DrvDeep>(&elem);
ASSERT_TRUE(p);
ASSERT_EQ(p->drvPath, store().parseStorePath(drvDeep.substr(1)));
ASSERT_EQ(elem.to_string(store()), drvDeep);
}
TEST_F(NixStringContextElemTest, built) {
std::string_view built = "!foo!/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv";
auto elem = NixStringContextElem::parse(store(), built);
auto * p = std::get_if<NixStringContextElem::Built>(&elem);
ASSERT_TRUE(p);
ASSERT_EQ(p->output, "foo");
ASSERT_EQ(p->drvPath, store().parseStorePath(built.substr(5)));
ASSERT_EQ(elem.to_string(store()), built);
}
}

View file

@ -3,6 +3,7 @@
#include <cassert> #include <cassert>
#include "symbol-table.hh" #include "symbol-table.hh"
#include "value/context.hh"
#if HAVE_BOEHMGC #if HAVE_BOEHMGC
#include <gc/gc_allocator.h> #include <gc/gc_allocator.h>
@ -67,8 +68,6 @@ class XMLWriter;
typedef int64_t NixInt; typedef int64_t NixInt;
typedef double NixFloat; typedef double NixFloat;
typedef std::pair<StorePath, std::string> NixStringContextElem;
typedef std::vector<NixStringContextElem> NixStringContext;
/* External values must descend from ExternalValueBase, so that /* External values must descend from ExternalValueBase, so that
* type-agnostic nix functions (e.g. showType) can be implemented * type-agnostic nix functions (e.g. showType) can be implemented

View file

@ -0,0 +1,67 @@
#include "value/context.hh"
#include "store-api.hh"
#include <optional>
namespace nix {
NixStringContextElem NixStringContextElem::parse(const Store & store, std::string_view s0)
{
std::string_view s = s0;
if (s.size() == 0) {
throw BadNixStringContextElem(s0,
"String context element should never be an empty string");
}
switch (s.at(0)) {
case '!': {
s = s.substr(1); // advance string to parse after first !
size_t index = s.find("!");
// This makes index + 1 safe. Index can be the length (one after index
// of last character), so given any valid character index --- a
// successful find --- we can add one.
if (index == std::string_view::npos) {
throw BadNixStringContextElem(s0,
"String content element beginning with '!' should have a second '!'");
}
return NixStringContextElem::Built {
.drvPath = store.parseStorePath(s.substr(index + 1)),
.output = std::string(s.substr(0, index)),
};
}
case '=': {
return NixStringContextElem::DrvDeep {
.drvPath = store.parseStorePath(s.substr(1)),
};
}
default: {
return NixStringContextElem::Opaque {
.path = store.parseStorePath(s),
};
}
}
}
std::string NixStringContextElem::to_string(const Store & store) const {
return std::visit(overloaded {
[&](const NixStringContextElem::Built & b) {
std::string res;
res += '!';
res += b.output;
res += '!';
res += store.printStorePath(b.drvPath);
return res;
},
[&](const NixStringContextElem::DrvDeep & d) {
std::string res;
res += '=';
res += store.printStorePath(d.drvPath);
return res;
},
[&](const NixStringContextElem::Opaque & o) {
return store.printStorePath(o.path);
},
}, raw());
}
}

View file

@ -0,0 +1,90 @@
#pragma once
#include "util.hh"
#include "path.hh"
#include <optional>
#include <nlohmann/json_fwd.hpp>
namespace nix {
class BadNixStringContextElem : public Error
{
public:
std::string_view raw;
template<typename... Args>
BadNixStringContextElem(std::string_view raw_, const Args & ... args)
: Error("")
{
raw = raw_;
auto hf = hintfmt(args...);
err.msg = hintfmt("Bad String Context element: %1%: %2%", normaltxt(hf.str()), raw);
}
};
class Store;
/* Plain opaque path to some store object.
Encoded as just the path: <path>.
*/
struct NixStringContextElem_Opaque {
StorePath path;
};
/* Path to a derivation and its entire build closure.
The path doesn't just refer to derivation itself and its closure, but
also all outputs of all derivations in that closure (including the
root derivation).
Encoded in the form =<drvPath>.
*/
struct NixStringContextElem_DrvDeep {
StorePath drvPath;
};
/* Derivation output.
Encoded in the form !<output>!<drvPath>.
*/
struct NixStringContextElem_Built {
StorePath drvPath;
std::string output;
};
using _NixStringContextElem_Raw = std::variant<
NixStringContextElem_Opaque,
NixStringContextElem_DrvDeep,
NixStringContextElem_Built
>;
struct NixStringContextElem : _NixStringContextElem_Raw {
using Raw = _NixStringContextElem_Raw;
using Raw::Raw;
using Opaque = NixStringContextElem_Opaque;
using DrvDeep = NixStringContextElem_DrvDeep;
using Built = NixStringContextElem_Built;
inline const Raw & raw() const {
return static_cast<const Raw &>(*this);
}
inline Raw & raw() {
return static_cast<Raw &>(*this);
}
/* Decode a context string, one of:
- <path>
- =<path>
- !<name>!<path>
*/
static NixStringContextElem parse(const Store & store, std::string_view s);
std::string to_string(const Store & store) const;
};
typedef std::vector<NixStringContextElem> NixStringContext;
}

View file

@ -80,17 +80,27 @@ UnresolvedApp Installable::toApp(EvalState & state)
auto [program, context] = cursor->getAttr("program")->getStringWithContext(); auto [program, context] = cursor->getAttr("program")->getStringWithContext();
std::vector<DerivedPath> context2; std::vector<DerivedPath> context2;
for (auto & [path, name] : context) { for (auto & c : context) {
context2.push_back(name != "" || path.isDerivation() context2.emplace_back(std::visit(overloaded {
? (DerivedPath) DerivedPath::Built { [&](const NixStringContextElem::DrvDeep & d) -> DerivedPath {
.drvPath = path, /* We want all outputs of the drv */
.outputs = name != "" return DerivedPath::Built {
? StringSet { name } .drvPath = d.drvPath,
: StringSet { }, .outputs = {},
} };
: (DerivedPath) DerivedPath::Opaque { },
.path = path, [&](const NixStringContextElem::Built & b) -> DerivedPath {
}); return DerivedPath::Built {
.drvPath = b.drvPath,
.outputs = { b.output },
};
},
[&](const NixStringContextElem::Opaque & o) -> DerivedPath {
return DerivedPath::Opaque {
.path = o.path,
};
},
}, c.raw()));
} }
return UnresolvedApp{App { return UnresolvedApp{App {

View file

@ -348,7 +348,7 @@ struct CmdFlakeCheck : FlakeCommand
// FIXME // FIXME
auto app = App(*state, v); auto app = App(*state, v);
for (auto & i : app.context) { for (auto & i : app.context) {
auto [drvPathS, outputName] = decodeContext(i); auto [drvPathS, outputName] = NixStringContextElem::parse(i);
store->parseStorePath(drvPathS); store->parseStorePath(drvPathS);
} }
#endif #endif

View file

@ -8,4 +8,4 @@ libplugintest_ALLOW_UNDEFINED := 1
libplugintest_EXCLUDE_FROM_LIBRARY_LIST := 1 libplugintest_EXCLUDE_FROM_LIBRARY_LIST := 1
libplugintest_CXXFLAGS := -I src/libutil -I src/libexpr libplugintest_CXXFLAGS := -I src/libutil -I src/libstore -I src/libexpr