Merge remote-tracking branch 'origin/master' into flakes

This commit is contained in:
Eelco Dolstra 2020-01-21 21:18:52 +01:00
commit 6fadb3fc03
20 changed files with 217 additions and 181 deletions

View file

@ -37,6 +37,9 @@ channel is just a URL that points to a place containing a set of Nix
expressions. <phrase condition="manual">See also <xref expressions. <phrase condition="manual">See also <xref
linkend="sec-channels" />.</phrase></para> linkend="sec-channels" />.</phrase></para>
<para>To see the list of official NixOS channels, visit <link
xlink:href="https://nixos.org/channels" />.</para>
<para>This command has the following operations: <para>This command has the following operations:
<variablelist> <variablelist>

View file

@ -18,6 +18,9 @@ linkend="sec-nix-channel"><command>nix-channel</command></link> you
can automatically stay up to date with whatever is available at that can automatically stay up to date with whatever is available at that
URL.</para> URL.</para>
<para>To see the list of official NixOS channels, visit <link
xlink:href="https://nixos.org/channels" />.</para>
<para>You can “subscribe” to a channel using <para>You can “subscribe” to a channel using
<command>nix-channel --add</command>, e.g., <command>nix-channel --add</command>, e.g.,

View file

@ -33,9 +33,13 @@ incompatible changes:</para>
</listitem> </listitem>
<listitem> <listitem>
<para>The installer now enables sandboxing by default on <para>The installer now enables sandboxing by default on Linux when the
Linux. The <literal>max-jobs</literal> setting now defaults to system has the necessary kernel support.
1.</para> </para>
</listitem>
<listitem>
<para>The <literal>max-jobs</literal> setting now defaults to 1.</para>
</listitem> </listitem>
<listitem> <listitem>
@ -82,11 +86,6 @@ incompatible changes:</para>
the duration of Nix function calls to stderr.</para> the duration of Nix function calls to stderr.</para>
</listitem> </listitem>
<listitem>
<para>On Linux, sandboxing is now disabled by default on systems
that dont have the necessary kernel support.</para>
</listitem>
</itemizedlist> </itemizedlist>
</section> </section>

View file

@ -354,6 +354,8 @@
name = "nix-build"; name = "nix-build";
src = self.hydraJobs.tarball; src = self.hydraJobs.tarball;
enableParallelBuilding = true;
buildInputs = buildDeps; buildInputs = buildDeps;
dontInstall = false; dontInstall = false;
@ -366,6 +368,9 @@
# syntax-check generated dot files, it still requires some # syntax-check generated dot files, it still requires some
# fonts. So provide those. # fonts. So provide those.
FONTCONFIG_FILE = texFunctions.fontsConf; FONTCONFIG_FILE = texFunctions.fontsConf;
# To test building without precompiled headers.
makeFlagsArray = [ "PRECOMPILE_HEADERS=0" ];
}; };
# System tests. # System tests.

View file

@ -1,6 +1,6 @@
{ useClang ? false }: { useClang ? false }:
with import (builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/bb1013511e1e5edcf314df8321acf2f3c536df0d.tar.gz) {}; with import (builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-19.09.tar.gz) {};
with import ./release-common.nix { inherit pkgs; }; with import ./release-common.nix { inherit pkgs; };

View file

@ -566,9 +566,9 @@ LocalNoInlineNoReturn(void throwTypeError(const char * s, const ExprLambda & fun
throw TypeError(format(s) % fun.showNamePos() % s2 % pos); throw TypeError(format(s) % fun.showNamePos() % s2 % pos);
} }
LocalNoInlineNoReturn(void throwAssertionError(const char * s, const Pos & pos)) LocalNoInlineNoReturn(void throwAssertionError(const char * s, const string & s1, const Pos & pos))
{ {
throw AssertionError(format(s) % pos); throw AssertionError(format(s) % s1 % pos);
} }
LocalNoInlineNoReturn(void throwUndefinedVarError(const char * s, const string & s1, const Pos & pos)) LocalNoInlineNoReturn(void throwUndefinedVarError(const char * s, const string & s1, const Pos & pos))
@ -1302,8 +1302,11 @@ void ExprIf::eval(EvalState & state, Env & env, Value & v)
void ExprAssert::eval(EvalState & state, Env & env, Value & v) void ExprAssert::eval(EvalState & state, Env & env, Value & v)
{ {
if (!state.evalBool(env, cond, pos)) if (!state.evalBool(env, cond, pos)) {
throwAssertionError("assertion failed at %1%", pos); std::ostringstream out;
cond->show(out);
throwAssertionError("assertion '%1%' failed at %2%", out.str(), pos);
}
body->eval(state, env, v); body->eval(state, env, v);
} }

View file

@ -1,149 +1,161 @@
#include "json-to-value.hh" #include "json-to-value.hh"
#include <cstring> #include <variant>
#include <nlohmann/json.hpp>
using json = nlohmann::json;
using std::unique_ptr;
namespace nix { namespace nix {
// for more information, refer to
// https://github.com/nlohmann/json/blob/master/include/nlohmann/detail/input/json_sax.hpp
class JSONSax : nlohmann::json_sax<json> {
class JSONState {
protected:
unique_ptr<JSONState> parent;
Value * v;
public:
virtual unique_ptr<JSONState> resolve(EvalState &)
{
throw std::logic_error("tried to close toplevel json parser state");
};
explicit JSONState(unique_ptr<JSONState>&& p) : parent(std::move(p)), v(nullptr) {};
explicit JSONState(Value* v) : v(v) {};
JSONState(JSONState& p) = delete;
Value& value(EvalState & state)
{
if (v == nullptr)
v = state.allocValue();
return *v;
};
virtual ~JSONState() {};
virtual void add() {};
};
static void skipWhitespace(const char * & s) class JSONObjectState : public JSONState {
{ using JSONState::JSONState;
while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r') s++; ValueMap attrs = ValueMap();
} virtual unique_ptr<JSONState> resolve(EvalState & state) override
{
Value& v = parent->value(state);
static string parseJSONString(const char * & s) state.mkAttrs(v, attrs.size());
{ for (auto & i : attrs)
string res; v.attrs->push_back(Attr(i.first, i.second));
if (*s++ != '"') throw JSONParseError("expected JSON string"); return std::move(parent);
while (*s != '"') {
if (!*s) throw JSONParseError("got end-of-string in JSON string");
if (*s == '\\') {
s++;
if (*s == '"') res += '"';
else if (*s == '\\') res += '\\';
else if (*s == '/') res += '/';
else if (*s == '/') res += '/';
else if (*s == 'b') res += '\b';
else if (*s == 'f') res += '\f';
else if (*s == 'n') res += '\n';
else if (*s == 'r') res += '\r';
else if (*s == 't') res += '\t';
else if (*s == 'u') throw JSONParseError("\\u characters in JSON strings are currently not supported");
else throw JSONParseError("invalid escaped character in JSON string");
s++;
} else
res += *s++;
}
s++;
return res;
}
static void parseJSON(EvalState & state, const char * & s, Value & v)
{
skipWhitespace(s);
if (!*s) throw JSONParseError("expected JSON value");
if (*s == '[') {
s++;
ValueVector values;
values.reserve(128);
skipWhitespace(s);
while (1) {
if (values.empty() && *s == ']') break;
Value * v2 = state.allocValue();
parseJSON(state, s, *v2);
values.push_back(v2);
skipWhitespace(s);
if (*s == ']') break;
if (*s != ',') throw JSONParseError("expected ',' or ']' after JSON array element");
s++;
} }
s++; virtual void add() override { v = nullptr; };
state.mkList(v, values.size()); public:
for (size_t n = 0; n < values.size(); ++n) void key(string_t& name, EvalState & state)
v.listElems()[n] = values[n]; {
} attrs[state.symbols.create(name)] = &value(state);
else if (*s == '{') {
s++;
ValueMap attrs;
while (1) {
skipWhitespace(s);
if (attrs.empty() && *s == '}') break;
string name = parseJSONString(s);
skipWhitespace(s);
if (*s != ':') throw JSONParseError("expected ':' in JSON object");
s++;
Value * v2 = state.allocValue();
parseJSON(state, s, *v2);
attrs[state.symbols.create(name)] = v2;
skipWhitespace(s);
if (*s == '}') break;
if (*s != ',') throw JSONParseError("expected ',' or '}' after JSON member");
s++;
} }
state.mkAttrs(v, attrs.size()); };
for (auto & i : attrs)
v.attrs->push_back(Attr(i.first, i.second));
v.attrs->sort();
s++;
}
else if (*s == '"') { class JSONListState : public JSONState {
mkString(v, parseJSONString(s)); ValueVector values = ValueVector();
} virtual unique_ptr<JSONState> resolve(EvalState & state) override
{
else if (isdigit(*s) || *s == '-' || *s == '.' ) { Value& v = parent->value(state);
// Buffer into a string first, then use built-in C++ conversions state.mkList(v, values.size());
std::string tmp_number; for (size_t n = 0; n < values.size(); ++n) {
ValueType number_type = tInt; v.listElems()[n] = values[n];
}
while (isdigit(*s) || *s == '-' || *s == '.' || *s == 'e' || *s == 'E') { return std::move(parent);
if (*s == '.' || *s == 'e' || *s == 'E')
number_type = tFloat;
tmp_number += *s++;
} }
virtual void add() override {
try { values.push_back(v);
if (number_type == tFloat) v = nullptr;
mkFloat(v, stod(tmp_number)); };
else public:
mkInt(v, stol(tmp_number)); JSONListState(unique_ptr<JSONState>&& p, std::size_t reserve) : JSONState(std::move(p))
} catch (std::invalid_argument & e) { {
throw JSONParseError("invalid JSON number"); values.reserve(reserve);
} catch (std::out_of_range & e) {
throw JSONParseError("out-of-range JSON number");
} }
};
EvalState & state;
unique_ptr<JSONState> rs;
template<typename T, typename... Args> inline bool handle_value(T f, Args... args)
{
f(rs->value(state), args...);
rs->add();
return true;
} }
else if (strncmp(s, "true", 4) == 0) { public:
s += 4; JSONSax(EvalState & state, Value & v) : state(state), rs(new JSONState(&v)) {};
mkBool(v, true);
bool null()
{
return handle_value(mkNull);
} }
else if (strncmp(s, "false", 5) == 0) { bool boolean(bool val)
s += 5; {
mkBool(v, false); return handle_value(mkBool, val);
} }
else if (strncmp(s, "null", 4) == 0) { bool number_integer(number_integer_t val)
s += 4; {
mkNull(v); return handle_value(mkInt, val);
} }
else throw JSONParseError("unrecognised JSON value"); bool number_unsigned(number_unsigned_t val)
} {
return handle_value(mkInt, val);
}
bool number_float(number_float_t val, const string_t& s)
{
return handle_value(mkFloat, val);
}
bool string(string_t& val)
{
return handle_value<void(Value&, const char*)>(mkString, val.c_str());
}
bool start_object(std::size_t len)
{
rs = std::make_unique<JSONObjectState>(std::move(rs));
return true;
}
bool key(string_t& name)
{
dynamic_cast<JSONObjectState*>(rs.get())->key(name, state);
return true;
}
bool end_object() {
rs = rs->resolve(state);
rs->add();
return true;
}
bool end_array() {
return end_object();
}
bool start_array(size_t len) {
rs = std::make_unique<JSONListState>(std::move(rs),
len != std::numeric_limits<size_t>::max() ? len : 128);
return true;
}
bool parse_error(std::size_t, const std::string&, const nlohmann::detail::exception& ex) {
throw JSONParseError(ex.what());
}
};
void parseJSON(EvalState & state, const string & s_, Value & v) void parseJSON(EvalState & state, const string & s_, Value & v)
{ {
const char * s = s_.c_str(); JSONSax parser(state, v);
parseJSON(state, s, v); bool res = json::sax_parse(s_, &parser);
skipWhitespace(s); if (!res)
if (*s) throw JSONParseError(format("expected end-of-string while parsing JSON value: %1%") % s); throw JSONParseError("Invalid JSON Value");
} }
} }

View file

@ -732,6 +732,8 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
the hash. */ the hash. */
for (auto & i : outputs) { for (auto & i : outputs) {
if (!jsonObject) drv.env[i] = ""; if (!jsonObject) drv.env[i] = "";
drv.outputs.insert_or_assign(i,
DerivationOutput(StorePath::dummy.clone(), "", ""));
} }
Hash h = hashDerivationModulo(*state.store, Derivation(drv), true); Hash h = hashDerivationModulo(*state.store, Derivation(drv), true);

View file

@ -1017,7 +1017,7 @@ DerivationGoal::DerivationGoal(StorePath && drvPath, const StringSet & wantedOut
, buildMode(buildMode) , buildMode(buildMode)
{ {
state = &DerivationGoal::getDerivation; state = &DerivationGoal::getDerivation;
name = fmt("building of '%s'", worker.store.printStorePath(drvPath)); name = fmt("building of '%s'", worker.store.printStorePath(this->drvPath));
trace("created"); trace("created");
mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds); mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
@ -1042,7 +1042,7 @@ DerivationGoal::DerivationGoal(StorePath && drvPath, const BasicDerivation & drv
/* Prevent the .chroot directory from being /* Prevent the .chroot directory from being
garbage-collected. (See isActiveTempFile() in gc.cc.) */ garbage-collected. (See isActiveTempFile() in gc.cc.) */
worker.store.addTempRoot(drvPath); worker.store.addTempRoot(this->drvPath);
} }
@ -3341,7 +3341,7 @@ void DerivationGoal::runChild()
; ;
} }
#if __APPLE__ #if __APPLE__
else if (getEnv("_NIX_TEST_NO_SANDBOX") == "") { else {
/* This has to appear before import statements. */ /* This has to appear before import statements. */
std::string sandboxProfile = "(version 1)\n"; std::string sandboxProfile = "(version 1)\n";
@ -3450,25 +3450,31 @@ void DerivationGoal::runChild()
/* They don't like trailing slashes on subpath directives */ /* They don't like trailing slashes on subpath directives */
if (globalTmpDir.back() == '/') globalTmpDir.pop_back(); if (globalTmpDir.back() == '/') globalTmpDir.pop_back();
builder = "/usr/bin/sandbox-exec"; if (getEnv("_NIX_TEST_NO_SANDBOX") != "1") {
args.push_back("sandbox-exec"); builder = "/usr/bin/sandbox-exec";
args.push_back("-f"); args.push_back("sandbox-exec");
args.push_back(sandboxFile); args.push_back("-f");
args.push_back("-D"); args.push_back(sandboxFile);
args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir);
args.push_back("-D");
args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/");
if (allowLocalNetworking) {
args.push_back("-D"); args.push_back("-D");
args.push_back(string("_ALLOW_LOCAL_NETWORKING=1")); args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir);
args.push_back("-D");
args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/");
if (allowLocalNetworking) {
args.push_back("-D");
args.push_back(string("_ALLOW_LOCAL_NETWORKING=1"));
}
args.push_back(drv->builder);
} else {
builder = drv->builder.c_str();
args.push_back(std::string(baseNameOf(drv->builder)));
} }
args.push_back(drv->builder);
} }
#endif #else
else { else {
builder = drv->builder.c_str(); builder = drv->builder.c_str();
args.push_back(std::string(baseNameOf(drv->builder))); args.push_back(std::string(baseNameOf(drv->builder)));
} }
#endif
for (auto & i : drv->args) for (auto & i : drv->args)
args.push_back(rewriteStrings(i, inputRewrites)); args.push_back(rewriteStrings(i, inputRewrites));
@ -4253,7 +4259,7 @@ SubstitutionGoal::SubstitutionGoal(StorePath && storePath, Worker & worker, Repa
, repair(repair) , repair(repair)
{ {
state = &SubstitutionGoal::init; state = &SubstitutionGoal::init;
name = fmt("substitution of '%s'", worker.store.printStorePath(storePath)); name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath));
trace("created"); trace("created");
maintainExpectedSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.expectedSubstitutions); maintainExpectedSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.expectedSubstitutions);
} }

View file

@ -246,30 +246,18 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
s.reserve(65536); s.reserve(65536);
s += "Derive(["; s += "Derive([";
StringSet maskedOutputs; bool first = true;
for (auto & i : outputs) {
if (maskOutputs) { if (first) first = false; else s += ',';
bool first = true; s += '('; printString(s, i.first);
maskedOutputs = tokenizeString<StringSet>(get(env, "outputs").value_or("out"), " "); s += ','; printString(s, maskOutputs ? "" : store.printStorePath(i.second.path));
for (auto & i : maskedOutputs) { s += ','; printString(s, i.second.hashAlgo);
if (first) first = false; else s += ','; s += ','; printString(s, i.second.hash);
s += '('; printString(s, i); s += ')';
s += ",\"\",\"\",\"\")";
}
} else {
bool first = true;
for (auto & i : outputs) {
if (first) first = false; else s += ',';
s += '('; printString(s, i.first);
s += ','; printString(s, store.printStorePath(i.second.path));
s += ','; printString(s, i.second.hashAlgo);
s += ','; printString(s, i.second.hash);
s += ')';
}
} }
s += "],["; s += "],[";
bool first = true; first = true;
if (actualInputs) { if (actualInputs) {
for (auto & i : *actualInputs) { for (auto & i : *actualInputs) {
if (first) first = false; else s += ','; if (first) first = false; else s += ',';
@ -299,7 +287,7 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
for (auto & i : env) { for (auto & i : env) {
if (first) first = false; else s += ','; if (first) first = false; else s += ',';
s += '('; printString(s, i.first); s += '('; printString(s, i.first);
s += ','; printString(s, maskOutputs && maskedOutputs.count(i.first) ? "" : i.second); s += ','; printString(s, maskOutputs && outputs.count(i.first) ? "" : i.second);
s += ')'; s += ')';
} }

View file

@ -443,7 +443,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
// lsof is really slow on OS X. This actually causes the gc-concurrent.sh test to fail. // lsof is really slow on OS X. This actually causes the gc-concurrent.sh test to fail.
// See: https://github.com/NixOS/nix/issues/3011 // See: https://github.com/NixOS/nix/issues/3011
// Because of this we disable lsof when running the tests. // Because of this we disable lsof when running the tests.
if (getEnv("_NIX_TEST_NO_LSOF") == "") { if (getEnv("_NIX_TEST_NO_LSOF") != "1") {
try { try {
std::regex lsofRegex(R"(^n(/.*)$)"); std::regex lsofRegex(R"(^n(/.*)$)");
auto lsofLines = auto lsofLines =

View file

@ -7,6 +7,7 @@
#include <map> #include <map>
#include <thread> #include <thread>
#include <dlfcn.h> #include <dlfcn.h>
#include <sys/utsname.h>
namespace nix { namespace nix {

View file

@ -4,7 +4,7 @@
namespace nix { namespace nix {
NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & whence) NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & whence)
: ValidPathInfo(StorePath::make((unsigned char *) "xxxxxxxxxxxxxxxxxxxx", "x")) // FIXME: hack : ValidPathInfo(StorePath::dummy.clone()) // FIXME: hack
{ {
auto corrupt = [&]() { auto corrupt = [&]() {
throw Error(format("NAR info file '%1%' is corrupt") % whence); throw Error(format("NAR info file '%1%' is corrupt") % whence);

View file

@ -46,12 +46,15 @@ std::string_view StorePath::name() const
return ffi_StorePath_name(*this); return ffi_StorePath_name(*this);
} }
StorePath StorePath::dummy(
StorePath::make(
(unsigned char *) "xxxxxxxxxxxxxxxxxxxx", "x"));
StorePath Store::parseStorePath(std::string_view path) const StorePath Store::parseStorePath(std::string_view path) const
{ {
return StorePath::make(path, storeDir); return StorePath::make(path, storeDir);
} }
StorePathSet Store::parseStorePathSet(const PathSet & paths) const StorePathSet Store::parseStorePathSet(const PathSet & paths) const
{ {
StorePathSet res; StorePathSet res;

View file

@ -53,6 +53,8 @@ struct StorePath : rust::Value<3 * sizeof(void *) + 24, ffi_StorePath_drop>
{ {
return ffi_StorePath_hash_data(*this); return ffi_StorePath_hash_data(*this);
} }
static StorePath dummy;
}; };
typedef std::set<StorePath> StorePathSet; typedef std::set<StorePath> StorePathSet;

View file

@ -53,7 +53,7 @@ static int _main(int argc, char * * argv)
{ {
HashType ht = htSHA256; HashType ht = htSHA256;
std::vector<string> args; std::vector<string> args;
bool printPath = getEnv("PRINT_PATH") != ""; bool printPath = getEnv("PRINT_PATH") == "1";
bool fromExpr = false; bool fromExpr = false;
string attrPath; string attrPath;
bool unpack = false; bool unpack = false;

View file

@ -0,0 +1 @@
"quote \" reverse solidus \\ solidus / backspace  formfeed newline \n carriage return \r horizontal tab \t 1 char unicode encoded backspace  1 char unicode encoded e with accent é 2 char unicode encoded s with caron š 3 char unicode encoded rightwards arrow →"

View file

@ -0,0 +1,3 @@
# This string contains all supported escapes in a JSON string, per json.org
# \b and \f are not supported by Nix
builtins.fromJSON ''"quote \" reverse solidus \\ solidus \/ backspace \b formfeed \f newline \n carriage return \r horizontal tab \t 1 char unicode encoded backspace \u0008 1 char unicode encoded e with accent \u00e9 2 char unicode encoded s with caron \u0161 3 char unicode encoded rightwards arrow \u2192"''

View file

@ -16,6 +16,8 @@ mkDerivation {
__structuredAttrs = true; __structuredAttrs = true;
outputs = [ "out" "dev" ];
buildCommand = '' buildCommand = ''
set -x set -x
@ -30,8 +32,9 @@ mkDerivation {
[[ -v nothing ]] [[ -v nothing ]]
[[ -z $nothing ]] [[ -z $nothing ]]
mkdir ''${outputs[out]} mkdir ''${outputs[out]} ''${outputs[dev]}
echo bar > $dest echo bar > $dest
echo foo > $dest2
json=$(cat .attrs.json) json=$(cat .attrs.json)
[[ $json =~ '"narHash":"sha256:1r7yc43zqnzl5b0als5vnyp649gk17i37s7mj00xr8kc47rjcybk"' ]] [[ $json =~ '"narHash":"sha256:1r7yc43zqnzl5b0als5vnyp649gk17i37s7mj00xr8kc47rjcybk"' ]]
@ -57,6 +60,7 @@ mkDerivation {
nothing = null; nothing = null;
dest = "${placeholder "out"}/foo"; dest = "${placeholder "out"}/foo";
dest2 = "${placeholder "dev"}/foo";
"foo bar" = "BAD"; "foo bar" = "BAD";
"1foobar" = "BAD"; "1foobar" = "BAD";

View file

@ -2,6 +2,7 @@ source common.sh
clearStore clearStore
outPath=$(nix-build structured-attrs.nix --no-out-link) nix-build structured-attrs.nix -A all -o $TEST_ROOT/result
[[ $(cat $outPath/foo) = bar ]] [[ $(cat $TEST_ROOT/result/foo) = bar ]]
[[ $(cat $TEST_ROOT/result-dev/foo) = foo ]]