From 66fccd5832d125e9162abc5ed351aa37708e9623 Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Sun, 5 Jan 2020 00:41:18 +0100 Subject: [PATCH 01/17] build: fix sandboxing on darwin Starting ba87b08f8529e4d9f8c58d8c625152058ceadb75 getEnv now returns an std::optional which means these getEnv() != "" conditions no longer happen if the variables are not defined. --- src/libstore/build.cc | 33 ++++++++++++++---------- src/libstore/gc.cc | 2 +- src/nix-prefetch-url/nix-prefetch-url.cc | 2 +- 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 78f39fed1..c3d69f387 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3338,7 +3338,7 @@ void DerivationGoal::runChild() ; } #if __APPLE__ - else if (getEnv("_NIX_TEST_NO_SANDBOX") == "") { + else { /* This has to appear before import statements. */ std::string sandboxProfile = "(version 1)\n"; @@ -3447,25 +3447,32 @@ void DerivationGoal::runChild() /* They don't like trailing slashes on subpath directives */ if (globalTmpDir.back() == '/') globalTmpDir.pop_back(); - builder = "/usr/bin/sandbox-exec"; - args.push_back("sandbox-exec"); - args.push_back("-f"); - args.push_back(sandboxFile); - args.push_back("-D"); - args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir); - args.push_back("-D"); - args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/"); - if (allowLocalNetworking) { + if (getEnv("_NIX_TEST_NO_SANDBOX") != "1") { + builder = "/usr/bin/sandbox-exec"; + args.push_back("sandbox-exec"); + args.push_back("-f"); + args.push_back(sandboxFile); args.push_back("-D"); - args.push_back(string("_ALLOW_LOCAL_NETWORKING=1")); + args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir); + args.push_back("-D"); + args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/"); + if (allowLocalNetworking) { + args.push_back("-D"); + args.push_back(string("_ALLOW_LOCAL_NETWORKING=1")); + } + args.push_back(drv->builder); + } else { + printError("warning: running in sandboxing test mode, sandbox disabled"); + builder = drv->builder.c_str(); + args.push_back(std::string(baseNameOf(drv->builder))); } - args.push_back(drv->builder); } -#endif +#else else { builder = drv->builder.c_str(); args.push_back(std::string(baseNameOf(drv->builder))); } +#endif for (auto & i : drv->args) args.push_back(rewriteStrings(i, inputRewrites)); diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index ed81186af..690febc5b 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -443,7 +443,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) // lsof is really slow on OS X. This actually causes the gc-concurrent.sh test to fail. // See: https://github.com/NixOS/nix/issues/3011 // Because of this we disable lsof when running the tests. - if (getEnv("_NIX_TEST_NO_LSOF") == "") { + if (getEnv("_NIX_TEST_NO_LSOF") != "1") { try { std::regex lsofRegex(R"(^n(/.*)$)"); auto lsofLines = diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index 2ec0b07ac..cc0891811 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -53,7 +53,7 @@ static int _main(int argc, char * * argv) { HashType ht = htSHA256; std::vector args; - bool printPath = getEnv("PRINT_PATH") != ""; + bool printPath = getEnv("PRINT_PATH") == "1"; bool fromExpr = false; string attrPath; bool unpack = false; From bc22a7ee6a6d9367f3197cf31a5fdd477b155974 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 6 Jan 2020 22:18:00 +0100 Subject: [PATCH 02/17] Fix use of uninitialized store path Fixes 'building of '/nix/store/00000000000000000000000000000000-': ...'. --- src/libstore/build.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 4c07865e9..e1171082b 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1016,7 +1016,7 @@ DerivationGoal::DerivationGoal(StorePath && drvPath, const StringSet & wantedOut , buildMode(buildMode) { state = &DerivationGoal::getDerivation; - name = fmt("building of '%s'", worker.store.printStorePath(drvPath)); + name = fmt("building of '%s'", worker.store.printStorePath(this->drvPath)); trace("created"); mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); @@ -1041,7 +1041,7 @@ DerivationGoal::DerivationGoal(StorePath && drvPath, const BasicDerivation & drv /* Prevent the .chroot directory from being garbage-collected. (See isActiveTempFile() in gc.cc.) */ - worker.store.addTempRoot(drvPath); + worker.store.addTempRoot(this->drvPath); } @@ -4259,7 +4259,7 @@ SubstitutionGoal::SubstitutionGoal(StorePath && storePath, Worker & worker, Repa , repair(repair) { state = &SubstitutionGoal::init; - name = fmt("substitution of '%s'", worker.store.printStorePath(storePath)); + name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath)); trace("created"); maintainExpectedSubstitutions = std::make_unique>(worker.expectedSubstitutions); } From cb2d348d48cfd66fa34edd5df8c255cf74170a84 Mon Sep 17 00:00:00 2001 From: Nikola Knezevic Date: Mon, 6 Jan 2020 16:30:56 +0100 Subject: [PATCH 03/17] Remove redundant check in parseJSONString --- src/libexpr/json-to-value.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index 8bae986f9..96cd0fc72 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -22,7 +22,6 @@ static string parseJSONString(const char * & s) if (*s == '"') res += '"'; else if (*s == '\\') res += '\\'; else if (*s == '/') res += '/'; - else if (*s == '/') res += '/'; else if (*s == 'b') res += '\b'; else if (*s == 'f') res += '\f'; else if (*s == 'n') res += '\n'; From 52a8f9295b828872586c5b9e5587064a25dae9b2 Mon Sep 17 00:00:00 2001 From: Nikola Knezevic Date: Tue, 7 Jan 2020 00:06:49 +0100 Subject: [PATCH 04/17] Add support for \u escape in fromJSON As fromTOML supports \u and \U escapes, bring fromJSON on par. As JSON defaults to UTF-8 encoding (every JSON parser must support UTF-8), this change parses the `\u hex hex hex hex` sequence (\u followed by 4 hexadecimal digits) into an UTF-8 representation. Add a test to verify correct parsing, using all escape sequences from json.org. --- src/libexpr/json-to-value.cc | 88 ++++++++++++++++++++++- tests/lang/eval-okay-fromjson-escapes.exp | 1 + tests/lang/eval-okay-fromjson-escapes.nix | 3 + 3 files changed, 90 insertions(+), 2 deletions(-) create mode 100644 tests/lang/eval-okay-fromjson-escapes.exp create mode 100644 tests/lang/eval-okay-fromjson-escapes.nix diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index 96cd0fc72..47cab2bb5 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -11,6 +11,87 @@ static void skipWhitespace(const char * & s) } +/* + Parse an unicode escape sequence (4 hex characters following \u) in JSON string +*/ +static string parseUnicodeEscapeSequence(const char * & s) +{ + int codepoint = 0; + + const auto factors = { 12u, 8u, 4u, 0u }; + for (const auto factor : factors) + { + if (!*s) throw JSONParseError("got end-of-string in JSON string while parsing \\u sequence"); + + if (*s >= '0' and *s <= '9') { + codepoint += static_cast((static_cast(*s) - 0x30u) << factor); + } else if (*s >= 'A' and *s <= 'F') { + codepoint += static_cast((static_cast(*s) - 0x37u) << factor); + } else if (*s >= 'a' and *s <= 'f') { + codepoint += static_cast((static_cast(*s) - 0x57u) << factor); + } else { + throw JSONParseError(format("illegal character '%1%' in \\u escape sequence.") % *s); + } + s++; + } + + if ((codepoint > 0xd7ff && codepoint < 0xe000) || codepoint > 0x10ffff) { + throw JSONParseError("Unicode escape sequence is not a Unicode scalar value"); + } + + // taken from cpptoml.h + std::string result; + // See Table 3-6 of the Unicode standard + if (codepoint <= 0x7f) + { + // 1-byte codepoints: 00000000 0xxxxxxx + // repr: 0xxxxxxx + result += static_cast(codepoint & 0x7f); + } + else if (codepoint <= 0x7ff) + { + // 2-byte codepoints: 00000yyy yyxxxxxx + // repr: 110yyyyy 10xxxxxx + // + // 0x1f = 00011111 + // 0xc0 = 11000000 + // + result += static_cast(0xc0 | ((codepoint >> 6) & 0x1f)); + // + // 0x80 = 10000000 + // 0x3f = 00111111 + // + result += static_cast(0x80 | (codepoint & 0x3f)); + } + else if (codepoint <= 0xffff) + { + // 3-byte codepoints: zzzzyyyy yyxxxxxx + // repr: 1110zzzz 10yyyyyy 10xxxxxx + // + // 0xe0 = 11100000 + // 0x0f = 00001111 + // + result += static_cast(0xe0 | ((codepoint >> 12) & 0x0f)); + result += static_cast(0x80 | ((codepoint >> 6) & 0x1f)); + result += static_cast(0x80 | (codepoint & 0x3f)); + } + else + { + // 4-byte codepoints: 000uuuuu zzzzyyyy yyxxxxxx + // repr: 11110uuu 10uuzzzz 10yyyyyy 10xxxxxx + // + // 0xf0 = 11110000 + // 0x07 = 00000111 + // + result += static_cast(0xf0 | ((codepoint >> 18) & 0x07)); + result += static_cast(0x80 | ((codepoint >> 12) & 0x3f)); + result += static_cast(0x80 | ((codepoint >> 6) & 0x3f)); + result += static_cast(0x80 | (codepoint & 0x3f)); + } + return result; +} + + static string parseJSONString(const char * & s) { string res; @@ -27,8 +108,11 @@ static string parseJSONString(const char * & s) else if (*s == 'n') res += '\n'; else if (*s == 'r') res += '\r'; else if (*s == 't') res += '\t'; - else if (*s == 'u') throw JSONParseError("\\u characters in JSON strings are currently not supported"); - else throw JSONParseError("invalid escaped character in JSON string"); + else if (*s == 'u') { + res += parseUnicodeEscapeSequence(++s); + // to neuter the outside s++ + s--; + } else throw JSONParseError("invalid escaped character in JSON string"); s++; } else res += *s++; diff --git a/tests/lang/eval-okay-fromjson-escapes.exp b/tests/lang/eval-okay-fromjson-escapes.exp new file mode 100644 index 000000000..add5505a8 --- /dev/null +++ b/tests/lang/eval-okay-fromjson-escapes.exp @@ -0,0 +1 @@ +"quote \" reverse solidus \\ solidus / backspace  formfeed newline \n carriage return \r horizontal tab \t 1 char unicode encoded backspace  1 char unicode encoded e with accent é 2 char unicode encoded s with caron š 3 char unicode encoded rightwards arrow →" diff --git a/tests/lang/eval-okay-fromjson-escapes.nix b/tests/lang/eval-okay-fromjson-escapes.nix new file mode 100644 index 000000000..f00713507 --- /dev/null +++ b/tests/lang/eval-okay-fromjson-escapes.nix @@ -0,0 +1,3 @@ +# This string contains all supported escapes in a JSON string, per json.org +# \b and \f are not supported by Nix +builtins.fromJSON ''"quote \" reverse solidus \\ solidus \/ backspace \b formfeed \f newline \n carriage return \r horizontal tab \t 1 char unicode encoded backspace \u0008 1 char unicode encoded e with accent \u00e9 2 char unicode encoded s with caron \u0161 3 char unicode encoded rightwards arrow \u2192"'' From f1fac0b5c3b75efab781949fdff2b67ffdda2cb3 Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Thu, 9 Jan 2020 17:38:27 +0100 Subject: [PATCH 05/17] builtins.fromJSON: use nlohmann/json parser instead of custom parser --- src/libexpr/json-to-value.cc | 343 ++++++++++++++--------------------- 1 file changed, 138 insertions(+), 205 deletions(-) diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index 47cab2bb5..19d9a9b90 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -1,232 +1,165 @@ #include "json-to-value.hh" -#include +#include +#include +#include + +using json = nlohmann::json; namespace nix { +// for more information, refer to +// https://github.com/nlohmann/json/blob/master/include/nlohmann/detail/input/json_sax.hpp +class JSONSax : nlohmann::json_sax { + class JSONState { + protected: + JSONState* parent; + Value * v; + public: + virtual JSONState* resolve(EvalState &) + { + throw std::logic_error("tried to close toplevel json parser state"); + }; + explicit JSONState(JSONState* p) : parent(p), v(nullptr) {}; + explicit JSONState(Value* v) : v(v) {}; + JSONState(JSONState& p) = delete; + Value& value(EvalState & state) + { + if (v == nullptr) + v = state.allocValue(); + return *v; + }; + virtual ~JSONState() {}; + virtual void add() {}; + }; -static void skipWhitespace(const char * & s) -{ - while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r') s++; -} - - -/* - Parse an unicode escape sequence (4 hex characters following \u) in JSON string -*/ -static string parseUnicodeEscapeSequence(const char * & s) -{ - int codepoint = 0; - - const auto factors = { 12u, 8u, 4u, 0u }; - for (const auto factor : factors) - { - if (!*s) throw JSONParseError("got end-of-string in JSON string while parsing \\u sequence"); - - if (*s >= '0' and *s <= '9') { - codepoint += static_cast((static_cast(*s) - 0x30u) << factor); - } else if (*s >= 'A' and *s <= 'F') { - codepoint += static_cast((static_cast(*s) - 0x37u) << factor); - } else if (*s >= 'a' and *s <= 'f') { - codepoint += static_cast((static_cast(*s) - 0x57u) << factor); - } else { - throw JSONParseError(format("illegal character '%1%' in \\u escape sequence.") % *s); + class JSONObjectState : public JSONState { + using JSONState::JSONState; + ValueMap attrs = ValueMap(); + virtual JSONState* resolve(EvalState & state) override + { + Value& v = parent->value(state); + state.mkAttrs(v, attrs.size()); + for (auto & i : attrs) + v.attrs->push_back(Attr(i.first, i.second)); + return parent; } - s++; - } - - if ((codepoint > 0xd7ff && codepoint < 0xe000) || codepoint > 0x10ffff) { - throw JSONParseError("Unicode escape sequence is not a Unicode scalar value"); - } - - // taken from cpptoml.h - std::string result; - // See Table 3-6 of the Unicode standard - if (codepoint <= 0x7f) - { - // 1-byte codepoints: 00000000 0xxxxxxx - // repr: 0xxxxxxx - result += static_cast(codepoint & 0x7f); - } - else if (codepoint <= 0x7ff) - { - // 2-byte codepoints: 00000yyy yyxxxxxx - // repr: 110yyyyy 10xxxxxx - // - // 0x1f = 00011111 - // 0xc0 = 11000000 - // - result += static_cast(0xc0 | ((codepoint >> 6) & 0x1f)); - // - // 0x80 = 10000000 - // 0x3f = 00111111 - // - result += static_cast(0x80 | (codepoint & 0x3f)); - } - else if (codepoint <= 0xffff) - { - // 3-byte codepoints: zzzzyyyy yyxxxxxx - // repr: 1110zzzz 10yyyyyy 10xxxxxx - // - // 0xe0 = 11100000 - // 0x0f = 00001111 - // - result += static_cast(0xe0 | ((codepoint >> 12) & 0x0f)); - result += static_cast(0x80 | ((codepoint >> 6) & 0x1f)); - result += static_cast(0x80 | (codepoint & 0x3f)); - } - else - { - // 4-byte codepoints: 000uuuuu zzzzyyyy yyxxxxxx - // repr: 11110uuu 10uuzzzz 10yyyyyy 10xxxxxx - // - // 0xf0 = 11110000 - // 0x07 = 00000111 - // - result += static_cast(0xf0 | ((codepoint >> 18) & 0x07)); - result += static_cast(0x80 | ((codepoint >> 12) & 0x3f)); - result += static_cast(0x80 | ((codepoint >> 6) & 0x3f)); - result += static_cast(0x80 | (codepoint & 0x3f)); - } - return result; -} - - -static string parseJSONString(const char * & s) -{ - string res; - if (*s++ != '"') throw JSONParseError("expected JSON string"); - while (*s != '"') { - if (!*s) throw JSONParseError("got end-of-string in JSON string"); - if (*s == '\\') { - s++; - if (*s == '"') res += '"'; - else if (*s == '\\') res += '\\'; - else if (*s == '/') res += '/'; - else if (*s == 'b') res += '\b'; - else if (*s == 'f') res += '\f'; - else if (*s == 'n') res += '\n'; - else if (*s == 'r') res += '\r'; - else if (*s == 't') res += '\t'; - else if (*s == 'u') { - res += parseUnicodeEscapeSequence(++s); - // to neuter the outside s++ - s--; - } else throw JSONParseError("invalid escaped character in JSON string"); - s++; - } else - res += *s++; - } - s++; - return res; -} - - -static void parseJSON(EvalState & state, const char * & s, Value & v) -{ - skipWhitespace(s); - - if (!*s) throw JSONParseError("expected JSON value"); - - if (*s == '[') { - s++; - ValueVector values; - values.reserve(128); - skipWhitespace(s); - while (1) { - if (values.empty() && *s == ']') break; - Value * v2 = state.allocValue(); - parseJSON(state, s, *v2); - values.push_back(v2); - skipWhitespace(s); - if (*s == ']') break; - if (*s != ',') throw JSONParseError("expected ',' or ']' after JSON array element"); - s++; + virtual void add() override { v = nullptr; }; + public: + void key(string_t& name, EvalState & state) + { + attrs[state.symbols.create(name)] = &value(state); } - s++; - state.mkList(v, values.size()); - for (size_t n = 0; n < values.size(); ++n) - v.listElems()[n] = values[n]; - } + }; - else if (*s == '{') { - s++; - ValueMap attrs; - while (1) { - skipWhitespace(s); - if (attrs.empty() && *s == '}') break; - string name = parseJSONString(s); - skipWhitespace(s); - if (*s != ':') throw JSONParseError("expected ':' in JSON object"); - s++; - Value * v2 = state.allocValue(); - parseJSON(state, s, *v2); - attrs[state.symbols.create(name)] = v2; - skipWhitespace(s); - if (*s == '}') break; - if (*s != ',') throw JSONParseError("expected ',' or '}' after JSON member"); - s++; + class JSONListState : public JSONState { + ValueVector values = ValueVector(); + virtual JSONState* resolve(EvalState & state) override + { + Value& v = parent->value(state); + state.mkList(v, values.size()); + for (size_t n = 0; n < values.size(); ++n) { + v.listElems()[n] = values[n]; + } + return parent; } - state.mkAttrs(v, attrs.size()); - for (auto & i : attrs) - v.attrs->push_back(Attr(i.first, i.second)); - v.attrs->sort(); - s++; - } - - else if (*s == '"') { - mkString(v, parseJSONString(s)); - } - - else if (isdigit(*s) || *s == '-' || *s == '.' ) { - // Buffer into a string first, then use built-in C++ conversions - std::string tmp_number; - ValueType number_type = tInt; - - while (isdigit(*s) || *s == '-' || *s == '.' || *s == 'e' || *s == 'E') { - if (*s == '.' || *s == 'e' || *s == 'E') - number_type = tFloat; - tmp_number += *s++; + virtual void add() override { + values.push_back(v); + v = nullptr; + }; + public: + JSONListState(JSONState* p, std::size_t reserve) : JSONState(p) + { + values.reserve(reserve); } + }; - try { - if (number_type == tFloat) - mkFloat(v, stod(tmp_number)); - else - mkInt(v, stol(tmp_number)); - } catch (std::invalid_argument & e) { - throw JSONParseError("invalid JSON number"); - } catch (std::out_of_range & e) { - throw JSONParseError("out-of-range JSON number"); - } + EvalState & state; + JSONState* rs; + + template inline bool handle_value(T f, Args... args) + { + f(rs->value(state), args...); + rs->add(); + return true; } - else if (strncmp(s, "true", 4) == 0) { - s += 4; - mkBool(v, true); +public: + JSONSax(EvalState & state, Value & v) : state(state), rs(new JSONState(&v)) {}; + ~JSONSax() { delete rs; }; + + bool null() + { + return handle_value(mkNull); } - else if (strncmp(s, "false", 5) == 0) { - s += 5; - mkBool(v, false); + bool boolean(bool val) + { + return handle_value(mkBool, val); } - else if (strncmp(s, "null", 4) == 0) { - s += 4; - mkNull(v); + bool number_integer(number_integer_t val) + { + return handle_value(mkInt, val); } - else throw JSONParseError("unrecognised JSON value"); -} + bool number_unsigned(number_unsigned_t val) + { + return handle_value(mkInt, val); + } + bool number_float(number_float_t val, const string_t& s) + { + return handle_value(mkFloat, val); + } + + bool string(string_t& val) + { + return handle_value(mkString, val.c_str()); + } + + bool start_object(std::size_t len) + { + JSONState* old = rs; + rs = new JSONObjectState(old); + return true; + } + + bool key(string_t& name) + { + dynamic_cast(rs)->key(name, state); + return true; + } + + bool end_object() { + JSONState* old = rs; + rs = old->resolve(state); + delete old; + rs->add(); + return true; + } + + bool end_array() { + return end_object(); + } + + bool start_array(size_t len) { + JSONState* old = rs; + rs = new JSONListState(old, len != std::numeric_limits::max() ? len : 128); + return true; + } + + bool parse_error(std::size_t, const std::string&, const nlohmann::detail::exception& ex) { + throw JSONParseError(ex.what()); + } +}; void parseJSON(EvalState & state, const string & s_, Value & v) { - const char * s = s_.c_str(); - parseJSON(state, s, v); - skipWhitespace(s); - if (*s) throw JSONParseError(format("expected end-of-string while parsing JSON value: %1%") % s); + JSONSax parser(state, v); + bool res = json::sax_parse(s_, &parser); + if (!res) + throw JSONParseError("Invalid JSON Value"); } - } From a350d0beb0e13d0f58698510bd6a96d894cd06fd Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Thu, 9 Jan 2020 22:46:41 +0100 Subject: [PATCH 06/17] json-to-value: use unique_ptr instead of raw pointers --- src/libexpr/json-to-value.cc | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index 19d9a9b90..67feb9e1f 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -5,6 +5,7 @@ #include using json = nlohmann::json; +using std::unique_ptr; namespace nix { @@ -13,14 +14,14 @@ namespace nix { class JSONSax : nlohmann::json_sax { class JSONState { protected: - JSONState* parent; + unique_ptr parent; Value * v; public: - virtual JSONState* resolve(EvalState &) + virtual unique_ptr resolve(EvalState &) { throw std::logic_error("tried to close toplevel json parser state"); }; - explicit JSONState(JSONState* p) : parent(p), v(nullptr) {}; + explicit JSONState(unique_ptr&& p) : parent(std::move(p)), v(nullptr) {}; explicit JSONState(Value* v) : v(v) {}; JSONState(JSONState& p) = delete; Value& value(EvalState & state) @@ -36,13 +37,13 @@ class JSONSax : nlohmann::json_sax { class JSONObjectState : public JSONState { using JSONState::JSONState; ValueMap attrs = ValueMap(); - virtual JSONState* resolve(EvalState & state) override + virtual unique_ptr resolve(EvalState & state) override { Value& v = parent->value(state); state.mkAttrs(v, attrs.size()); for (auto & i : attrs) v.attrs->push_back(Attr(i.first, i.second)); - return parent; + return std::move(parent); } virtual void add() override { v = nullptr; }; public: @@ -54,28 +55,28 @@ class JSONSax : nlohmann::json_sax { class JSONListState : public JSONState { ValueVector values = ValueVector(); - virtual JSONState* resolve(EvalState & state) override + virtual unique_ptr resolve(EvalState & state) override { Value& v = parent->value(state); state.mkList(v, values.size()); for (size_t n = 0; n < values.size(); ++n) { v.listElems()[n] = values[n]; } - return parent; + return std::move(parent); } virtual void add() override { values.push_back(v); v = nullptr; }; public: - JSONListState(JSONState* p, std::size_t reserve) : JSONState(p) + JSONListState(unique_ptr&& p, std::size_t reserve) : JSONState(std::move(p)) { values.reserve(reserve); } }; EvalState & state; - JSONState* rs; + unique_ptr rs; template inline bool handle_value(T f, Args... args) { @@ -86,7 +87,6 @@ class JSONSax : nlohmann::json_sax { public: JSONSax(EvalState & state, Value & v) : state(state), rs(new JSONState(&v)) {}; - ~JSONSax() { delete rs; }; bool null() { @@ -120,21 +120,18 @@ public: bool start_object(std::size_t len) { - JSONState* old = rs; - rs = new JSONObjectState(old); + rs = std::make_unique(std::move(rs)); return true; } bool key(string_t& name) { - dynamic_cast(rs)->key(name, state); + dynamic_cast(rs.get())->key(name, state); return true; } bool end_object() { - JSONState* old = rs; - rs = old->resolve(state); - delete old; + rs = rs->resolve(state); rs->add(); return true; } @@ -144,8 +141,8 @@ public: } bool start_array(size_t len) { - JSONState* old = rs; - rs = new JSONListState(old, len != std::numeric_limits::max() ? len : 128); + rs = std::make_unique(std::move(rs), + len != std::numeric_limits::max() ? len : 128); return true; } From 7ccfa7ca4f85d71fdd9e0fa1cb537989ec6095f2 Mon Sep 17 00:00:00 2001 From: Joe Pea Date: Thu, 9 Jan 2020 14:15:19 -0800 Subject: [PATCH 07/17] Add a link to official channels in the Channels chapter. --- doc/manual/packages/channels.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/manual/packages/channels.xml b/doc/manual/packages/channels.xml index 15c119fcb..1ed2bba52 100644 --- a/doc/manual/packages/channels.xml +++ b/doc/manual/packages/channels.xml @@ -17,6 +17,9 @@ a set of Nix expressions and a manifest. Using the command nix-channel you can automatically stay up to date with whatever is available at that URL. + +To see the list of official NixOS channels, visit . You can “subscribe” to a channel using nix-channel --add, e.g., From 3895e7879456218acb82774670273305a6b3f08c Mon Sep 17 00:00:00 2001 From: Joe Pea Date: Thu, 9 Jan 2020 14:20:08 -0800 Subject: [PATCH 08/17] Add link to official channels in nix-channel command ref --- doc/manual/command-ref/nix-channel.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/manual/command-ref/nix-channel.xml b/doc/manual/command-ref/nix-channel.xml index 9fea59167..ebcf56aff 100644 --- a/doc/manual/command-ref/nix-channel.xml +++ b/doc/manual/command-ref/nix-channel.xml @@ -36,6 +36,9 @@ stay up-to-date with a set of pre-built Nix expressions. A Nix channel is just a URL that points to a place containing a set of Nix expressions. See also . + +To see the list of official NixOS channels, visit . This command has the following operations: From 307bcb9a8e7a16bfc451e055a620b766df9d3f7d Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Sat, 11 Jan 2020 15:06:57 +0100 Subject: [PATCH 09/17] libexpr: show expression in assertion errors Includes the expression of the condition in the assertion message if the assertion failed, making assertions much easier to debug. eg. error: assertion (withPython -> (python2Packages != null)) failed at pkgs/tools/security/nmap/default.nix:11:1 --- src/libexpr/eval.cc | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 1c787645d..3a3810f24 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -531,9 +531,9 @@ LocalNoInlineNoReturn(void throwTypeError(const char * s, const ExprLambda & fun throw TypeError(format(s) % fun.showNamePos() % s2 % pos); } -LocalNoInlineNoReturn(void throwAssertionError(const char * s, const Pos & pos)) +LocalNoInlineNoReturn(void throwAssertionError(const char * s, const string & s1, const Pos & pos)) { - throw AssertionError(format(s) % pos); + throw AssertionError(format(s) % s1 % pos); } LocalNoInlineNoReturn(void throwUndefinedVarError(const char * s, const string & s1, const Pos & pos)) @@ -1262,8 +1262,11 @@ void ExprIf::eval(EvalState & state, Env & env, Value & v) void ExprAssert::eval(EvalState & state, Env & env, Value & v) { - if (!state.evalBool(env, cond, pos)) - throwAssertionError("assertion failed at %1%", pos); + if (!state.evalBool(env, cond, pos)) { + std::ostringstream out; + cond->show(out); + throwAssertionError("assertion %1% failed at %2%", out.str(), pos); + } body->eval(state, env, v); } From 30c9ca3b055a793339a850fc787fd5974e1405a8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 13 Jan 2020 21:11:56 +0100 Subject: [PATCH 10/17] Fix Nixpkgs dependency --- release.nix | 2 +- shell.nix | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/release.nix b/release.nix index 9202c2ff0..287b5294a 100644 --- a/release.nix +++ b/release.nix @@ -1,5 +1,5 @@ { nix ? builtins.fetchGit ./. -, nixpkgs ? builtins.fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.09.tar.gz +, nixpkgs ? builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-19.09.tar.gz , officialRelease ? false , systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ] }: diff --git a/shell.nix b/shell.nix index c709f6a5f..e5a2b2c91 100644 --- a/shell.nix +++ b/shell.nix @@ -1,6 +1,6 @@ { useClang ? false }: -with import (builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/bb1013511e1e5edcf314df8321acf2f3c536df0d.tar.gz) {}; +with import (builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-19.09.tar.gz) {}; with import ./release-common.nix { inherit pkgs; }; From 835e541144272853f1c6e5e6f342d19926ba4e6f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 13 Jan 2020 21:34:54 +0100 Subject: [PATCH 11/17] Fix build https://hydra.nixos.org/eval/1564374 --- src/libexpr/json-to-value.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index 67feb9e1f..1fdce1983 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -2,7 +2,6 @@ #include #include -#include using json = nlohmann::json; using std::unique_ptr; From c86c71c2b1feac19f3363662cf20abc88ab47e03 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 13 Jan 2020 21:44:35 +0100 Subject: [PATCH 12/17] Test PRECOMPILE_HEADERS=0 --- release.nix | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/release.nix b/release.nix index 287b5294a..db2b5ef05 100644 --- a/release.nix +++ b/release.nix @@ -240,6 +240,8 @@ let name = "nix-build"; src = tarball; + enableParallelBuilding = true; + buildInputs = buildDeps; dontInstall = false; @@ -252,6 +254,9 @@ let # syntax-check generated dot files, it still requires some # fonts. So provide those. FONTCONFIG_FILE = texFunctions.fontsConf; + + # To test building without precompiled headers. + makeFlagsArray = [ "PRECOMPILE_HEADERS=0" ]; }; From d64ab5131c376a0e96075ec133dfd4185505594f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 12 Jan 2020 18:12:34 -0500 Subject: [PATCH 13/17] unbreak build without pch --- src/libstore/globals.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 5f1ae5ab5..a8945996e 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -7,6 +7,7 @@ #include #include #include +#include namespace nix { From c3181e21e75d9e31254cd97c8e56858be7072d5f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 13 Jan 2020 21:52:03 +0100 Subject: [PATCH 14/17] Tweak error message --- src/libexpr/eval.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 3a3810f24..dac32b6f5 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1265,7 +1265,7 @@ void ExprAssert::eval(EvalState & state, Env & env, Value & v) if (!state.evalBool(env, cond, pos)) { std::ostringstream out; cond->show(out); - throwAssertionError("assertion %1% failed at %2%", out.str(), pos); + throwAssertionError("assertion '%1%' failed at %2%", out.str(), pos); } body->eval(state, env, v); } From 8b3217f8321ebd367c96d254df79e827d1076b3d Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Mon, 13 Jan 2020 22:09:18 +0100 Subject: [PATCH 15/17] build: remove warning when in sandboxing test mode Introduced in 66fccd5832d125e9162abc5ed351aa37708e9623, but somehow breaks the secure-drv-outputs test. --- src/libstore/build.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index e1171082b..74a68e1f5 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3464,7 +3464,6 @@ void DerivationGoal::runChild() } args.push_back(drv->builder); } else { - printError("warning: running in sandboxing test mode, sandbox disabled"); builder = drv->builder.c_str(); args.push_back(std::string(baseNameOf(drv->builder))); } From 5d24e18e29ea1fff8fa316701fd95be6941da770 Mon Sep 17 00:00:00 2001 From: Benjamin Hipple Date: Tue, 14 Jan 2020 00:14:03 -0500 Subject: [PATCH 16/17] doc: touchup release notes for 2.3 - At the top of the release notes, we announce sandboxing is now enabled by default, then at the bottom it says it's now disabled when missing kernel support. These can be merged into one point for clarity. - The point about `max-jobs` defaulting to 1 appears unrelated to sandboxing. --- doc/manual/release-notes/rl-2.3.xml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/doc/manual/release-notes/rl-2.3.xml b/doc/manual/release-notes/rl-2.3.xml index 80fbd6396..0ad7d641f 100644 --- a/doc/manual/release-notes/rl-2.3.xml +++ b/doc/manual/release-notes/rl-2.3.xml @@ -33,9 +33,13 @@ incompatible changes: - The installer now enables sandboxing by default on - Linux. The max-jobs setting now defaults to - 1. + The installer now enables sandboxing by default on Linux when the + system has the necessary kernel support. + + + + + The max-jobs setting now defaults to 1. @@ -82,11 +86,6 @@ incompatible changes: the duration of Nix function calls to stderr. - - On Linux, sandboxing is now disabled by default on systems - that don’t have the necessary kernel support. - - From aef635da78d33bf679f49fd10e7130d918a82549 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 21 Jan 2020 21:14:13 +0100 Subject: [PATCH 17/17] Fix derivation computation with __structuredAttrs and multiple outputs Fixes error: derivation '/nix/store/klivma7r7h5lndb99f7xxmlh5whyayvg-zlib-1.2.11.drv' has incorrect output '/nix/store/fv98nnx5ykgbq8sqabilkgkbc4169q05-zlib-1.2.11-dev', should be '/nix/store/adm7pilzlj3z5k249s8b4wv3scprhzi1-zlib-1.2.11-dev' --- src/libexpr/primops.cc | 2 ++ src/libstore/derivations.cc | 32 ++++++++++---------------------- src/libstore/nar-info.cc | 2 +- src/libstore/path.cc | 5 ++++- src/libstore/path.hh | 2 ++ tests/structured-attrs.nix | 6 +++++- tests/structured-attrs.sh | 5 +++-- 7 files changed, 27 insertions(+), 27 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 3ef827ebc..29302c9b6 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -731,6 +731,8 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * the hash. */ for (auto & i : outputs) { if (!jsonObject) drv.env[i] = ""; + drv.outputs.insert_or_assign(i, + DerivationOutput(StorePath::dummy.clone(), "", "")); } Hash h = hashDerivationModulo(*state.store, Derivation(drv), true); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 1a061149a..d9da8769c 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -246,30 +246,18 @@ string Derivation::unparse(const Store & store, bool maskOutputs, s.reserve(65536); s += "Derive(["; - StringSet maskedOutputs; - - if (maskOutputs) { - bool first = true; - maskedOutputs = tokenizeString(get(env, "outputs").value_or("out"), " "); - for (auto & i : maskedOutputs) { - if (first) first = false; else s += ','; - s += '('; printString(s, i); - s += ",\"\",\"\",\"\")"; - } - } else { - bool first = true; - for (auto & i : outputs) { - if (first) first = false; else s += ','; - s += '('; printString(s, i.first); - s += ','; printString(s, store.printStorePath(i.second.path)); - s += ','; printString(s, i.second.hashAlgo); - s += ','; printString(s, i.second.hash); - s += ')'; - } + bool first = true; + for (auto & i : outputs) { + if (first) first = false; else s += ','; + s += '('; printString(s, i.first); + s += ','; printString(s, maskOutputs ? "" : store.printStorePath(i.second.path)); + s += ','; printString(s, i.second.hashAlgo); + s += ','; printString(s, i.second.hash); + s += ')'; } s += "],["; - bool first = true; + first = true; if (actualInputs) { for (auto & i : *actualInputs) { if (first) first = false; else s += ','; @@ -299,7 +287,7 @@ string Derivation::unparse(const Store & store, bool maskOutputs, for (auto & i : env) { if (first) first = false; else s += ','; s += '('; printString(s, i.first); - s += ','; printString(s, maskOutputs && maskedOutputs.count(i.first) ? "" : i.second); + s += ','; printString(s, maskOutputs && outputs.count(i.first) ? "" : i.second); s += ')'; } diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index fb02cf3fd..1375094b5 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -4,7 +4,7 @@ namespace nix { NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & whence) - : ValidPathInfo(StorePath::make((unsigned char *) "xxxxxxxxxxxxxxxxxxxx", "x")) // FIXME: hack + : ValidPathInfo(StorePath::dummy.clone()) // FIXME: hack { auto corrupt = [&]() { throw Error(format("NAR info file '%1%' is corrupt") % whence); diff --git a/src/libstore/path.cc b/src/libstore/path.cc index cda5f9968..a33bec3ed 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -46,12 +46,15 @@ std::string_view StorePath::name() const return ffi_StorePath_name(*this); } +StorePath StorePath::dummy( + StorePath::make( + (unsigned char *) "xxxxxxxxxxxxxxxxxxxx", "x")); + StorePath Store::parseStorePath(std::string_view path) const { return StorePath::make(path, storeDir); } - StorePathSet Store::parseStorePathSet(const PathSet & paths) const { StorePathSet res; diff --git a/src/libstore/path.hh b/src/libstore/path.hh index 5ebb57480..6b29c3566 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -53,6 +53,8 @@ struct StorePath : rust::Value<3 * sizeof(void *) + 24, ffi_StorePath_drop> { return ffi_StorePath_hash_data(*this); } + + static StorePath dummy; }; typedef std::set StorePathSet; diff --git a/tests/structured-attrs.nix b/tests/structured-attrs.nix index 6c77a4391..c39c3a346 100644 --- a/tests/structured-attrs.nix +++ b/tests/structured-attrs.nix @@ -16,6 +16,8 @@ mkDerivation { __structuredAttrs = true; + outputs = [ "out" "dev" ]; + buildCommand = '' set -x @@ -30,8 +32,9 @@ mkDerivation { [[ -v nothing ]] [[ -z $nothing ]] - mkdir ''${outputs[out]} + mkdir ''${outputs[out]} ''${outputs[dev]} echo bar > $dest + echo foo > $dest2 json=$(cat .attrs.json) [[ $json =~ '"narHash":"sha256:1r7yc43zqnzl5b0als5vnyp649gk17i37s7mj00xr8kc47rjcybk"' ]] @@ -57,6 +60,7 @@ mkDerivation { nothing = null; dest = "${placeholder "out"}/foo"; + dest2 = "${placeholder "dev"}/foo"; "foo bar" = "BAD"; "1foobar" = "BAD"; diff --git a/tests/structured-attrs.sh b/tests/structured-attrs.sh index 9ba2672b6..646bdb876 100644 --- a/tests/structured-attrs.sh +++ b/tests/structured-attrs.sh @@ -2,6 +2,7 @@ source common.sh clearStore -outPath=$(nix-build structured-attrs.nix --no-out-link) +nix-build structured-attrs.nix -A all -o $TEST_ROOT/result -[[ $(cat $outPath/foo) = bar ]] +[[ $(cat $TEST_ROOT/result/foo) = bar ]] +[[ $(cat $TEST_ROOT/result-dev/foo) = foo ]]