Merge branch 'master' of https://github.com/NixOS/nix into parallel-xz

This commit is contained in:
AmineChikhaoui 2018-02-07 17:53:50 +01:00
commit 163e39547a
No known key found for this signature in database
GPG key ID: C0C8C84C15BCCD1C
14 changed files with 221 additions and 103 deletions

View file

@ -308,8 +308,9 @@ stdenv.mkDerivation { … }
</varlistentry> </varlistentry>
<varlistentry><term><function>builtins.filterSource</function> <varlistentry xml:id='builtin-filterSource'>
<replaceable>e1</replaceable> <replaceable>e2</replaceable></term> <term><function>builtins.filterSource</function>
<replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
<listitem> <listitem>
@ -768,6 +769,75 @@ Evaluates to <literal>[ "foo" ]</literal>.
</varlistentry> </varlistentry>
<varlistentry>
<term>
<function>builtins.path</function>
<replaceable>args</replaceable>
</term>
<listitem>
<para>
An enrichment of the built-in path type, based on the attributes
present in <replaceable>args</replaceable>. All are optional
except <varname>path</varname>:
</para>
<variablelist>
<varlistentry>
<term>path</term>
<listitem>
<para>The underlying path.</para>
</listitem>
</varlistentry>
<varlistentry>
<term>name</term>
<listitem>
<para>
The name of the path when added to the store. This can
used to reference paths that have nix-illegal characters
in their names, like <literal>@</literal>.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>filter</term>
<listitem>
<para>
A function of the type expected by
<link linkend="builtin-filterSource">builtins.filterSource</link>,
with the same semantics.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>recursive</term>
<listitem>
<para>
When <literal>false</literal>, when
<varname>path</varname> is added to the store it is with a
flat hash, rather than a hash of the NAR serialization of
the file. Thus, <varname>path</varname> must refer to a
regular file, not a directory. This allows similar
behavior to <literal>fetchurl</literal>. Defaults to
<literal>true</literal>.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>sha256</term>
<listitem>
<para>
When provided, this is the expected hash of the file at
the path. Evaluation will fail if the hash is incorrect,
and providing a hash allows
<literal>builtins.path</literal> to be used even when the
<literal>pure-eval</literal> nix config option is on.
</para>
</listitem>
</varlistentry>
</variablelist>
</listitem>
</varlistentry>
<varlistentry><term><function>builtins.pathExists</function> <varlistentry><term><function>builtins.pathExists</function>
<replaceable>path</replaceable></term> <replaceable>path</replaceable></term>

View file

@ -224,11 +224,13 @@ let
nix = build.x86_64-linux; system = "x86_64-linux"; nix = build.x86_64-linux; system = "x86_64-linux";
}); });
tests.setuid = pkgs.lib.genAttrs (pkgs.lib.filter (pkgs.lib.hasSuffix "-linux") systems) (system: tests.setuid = pkgs.lib.genAttrs
import ./tests/setuid.nix rec { ["i686-linux" "x86_64-linux"]
inherit nixpkgs; (system:
nix = build.${system}; inherit system; import ./tests/setuid.nix rec {
}); inherit nixpkgs;
nix = build.${system}; inherit system;
});
tests.binaryTarball = tests.binaryTarball =
with import nixpkgs { system = "x86_64-linux"; }; with import nixpkgs { system = "x86_64-linux"; };

View file

@ -1578,7 +1578,7 @@ string EvalState::copyPathToStore(PathSet & context, const Path & path)
dstPath = srcToStore[path]; dstPath = srcToStore[path];
else { else {
dstPath = settings.readOnlyMode dstPath = settings.readOnlyMode
? store->computeStorePathForPath(checkSourcePath(path)).first ? store->computeStorePathForPath(baseNameOf(path), checkSourcePath(path)).first
: store->addToStore(baseNameOf(path), checkSourcePath(path), true, htSHA256, defaultPathFilter, repair); : store->addToStore(baseNameOf(path), checkSourcePath(path), true, htSHA256, defaultPathFilter, repair);
srcToStore[path] = dstPath; srcToStore[path] = dstPath;
printMsg(lvlChatty, format("copied source '%1%' -> '%2%'") printMsg(lvlChatty, format("copied source '%1%' -> '%2%'")

View file

@ -1023,20 +1023,13 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu
} }
static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args, Value & v) static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_,
Value * filterFun, bool recursive, const Hash & expectedHash, Value & v)
{ {
PathSet context; const auto path = settings.pureEval && expectedHash ?
Path path = state.coerceToPath(pos, *args[1], context); path_ :
if (!context.empty()) state.checkSourcePath(path_);
throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % pos); PathFilter filter = filterFun ? ([&](const Path & path) {
state.forceValue(*args[0]);
if (args[0]->type != tLambda)
throw TypeError(format("first argument in call to 'filterSource' is not a function but %1%, at %2%") % showType(*args[0]) % pos);
path = state.checkSourcePath(path);
PathFilter filter = [&](const Path & path) {
auto st = lstat(path); auto st = lstat(path);
/* Call the filter function. The first argument is the path, /* Call the filter function. The first argument is the path,
@ -1045,7 +1038,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
mkString(arg1, path); mkString(arg1, path);
Value fun2; Value fun2;
state.callFunction(*args[0], arg1, fun2, noPos); state.callFunction(*filterFun, arg1, fun2, noPos);
Value arg2; Value arg2;
mkString(arg2, mkString(arg2,
@ -1058,16 +1051,79 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
state.callFunction(fun2, arg2, res, noPos); state.callFunction(fun2, arg2, res, noPos);
return state.forceBool(res, pos); return state.forceBool(res, pos);
}; }) : defaultPathFilter;
Path dstPath = settings.readOnlyMode Path expectedStorePath;
? state.store->computeStorePathForPath(path, true, htSHA256, filter).first if (expectedHash) {
: state.store->addToStore(baseNameOf(path), path, true, htSHA256, filter, state.repair); expectedStorePath =
state.store->makeFixedOutputPath(recursive, expectedHash, name);
}
Path dstPath;
if (!expectedHash || !state.store->isValidPath(expectedStorePath)) {
dstPath = settings.readOnlyMode
? state.store->computeStorePathForPath(name, path, recursive, htSHA256, filter).first
: state.store->addToStore(name, path, recursive, htSHA256, filter, state.repair);
if (expectedHash && expectedStorePath != dstPath) {
throw Error(format("store path mismatch in (possibly filtered) path added from '%1%'") % path);
}
} else
dstPath = expectedStorePath;
mkString(v, dstPath, {dstPath}); mkString(v, dstPath, {dstPath});
} }
static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
PathSet context;
Path path = state.coerceToPath(pos, *args[1], context);
if (!context.empty())
throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % pos);
state.forceValue(*args[0]);
if (args[0]->type != tLambda)
throw TypeError(format("first argument in call to 'filterSource' is not a function but %1%, at %2%") % showType(*args[0]) % pos);
addPath(state, pos, baseNameOf(path), path, args[0], true, Hash(), v);
}
static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
state.forceAttrs(*args[0], pos);
Path path;
string name;
Value * filterFun = nullptr;
auto recursive = true;
Hash expectedHash;
for (auto & attr : *args[0]->attrs) {
const string & n(attr.name);
if (n == "path") {
PathSet context;
path = state.coerceToPath(*attr.pos, *attr.value, context);
if (!context.empty())
throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % *attr.pos);
} else if (attr.name == state.sName)
name = state.forceStringNoCtx(*attr.value, *attr.pos);
else if (n == "filter") {
state.forceValue(*attr.value);
filterFun = attr.value;
} else if (n == "recursive")
recursive = state.forceBool(*attr.value, *attr.pos);
else if (n == "sha256")
expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256);
else
throw EvalError(format("unsupported argument '%1%' to 'addPath', at %2%") % attr.name % *attr.pos);
}
if (path.empty())
throw EvalError(format("'path' required, at %1%") % pos);
if (name.empty())
name = baseNameOf(path);
addPath(state, pos, name, path, filterFun, recursive, expectedHash, v);
}
/************************************************************* /*************************************************************
* Sets * Sets
*************************************************************/ *************************************************************/
@ -2085,6 +2141,7 @@ void EvalState::createBaseEnv()
addPrimOp("__fromJSON", 1, prim_fromJSON); addPrimOp("__fromJSON", 1, prim_fromJSON);
addPrimOp("__toFile", 2, prim_toFile); addPrimOp("__toFile", 2, prim_toFile);
addPrimOp("__filterSource", 2, prim_filterSource); addPrimOp("__filterSource", 2, prim_filterSource);
addPrimOp("__path", 1, prim_path);
// Sets // Sets
addPrimOp("__attrNames", 1, prim_attrNames); addPrimOp("__attrNames", 1, prim_attrNames);

View file

@ -3428,7 +3428,7 @@ void DerivationGoal::flushLine()
else { else {
if (settings.verboseBuild && if (settings.verboseBuild &&
(settings.printRepeatedBuilds || curRound == 1)) (settings.printRepeatedBuilds || curRound == 1))
printError(filterANSIEscapes(currentLogLine, true)); printError(currentLogLine);
else { else {
logTail.push_back(currentLogLine); logTail.push_back(currentLogLine);
if (logTail.size() > settings.logLines) logTail.pop_front(); if (logTail.size() > settings.logLines) logTail.pop_front();

View file

@ -222,11 +222,10 @@ Path Store::makeTextPath(const string & name, const Hash & hash,
} }
std::pair<Path, Hash> Store::computeStorePathForPath(const Path & srcPath, std::pair<Path, Hash> Store::computeStorePathForPath(const string & name,
bool recursive, HashType hashAlgo, PathFilter & filter) const const Path & srcPath, bool recursive, HashType hashAlgo, PathFilter & filter) const
{ {
Hash h = recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath); Hash h = recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath);
string name = baseNameOf(srcPath);
Path dstPath = makeFixedOutputPath(recursive, h, name); Path dstPath = makeFixedOutputPath(recursive, h, name);
return std::pair<Path, Hash>(dstPath, h); return std::pair<Path, Hash>(dstPath, h);
} }

View file

@ -307,9 +307,9 @@ public:
/* This is the preparatory part of addToStore(); it computes the /* This is the preparatory part of addToStore(); it computes the
store path to which srcPath is to be copied. Returns the store store path to which srcPath is to be copied. Returns the store
path and the cryptographic hash of the contents of srcPath. */ path and the cryptographic hash of the contents of srcPath. */
std::pair<Path, Hash> computeStorePathForPath(const Path & srcPath, std::pair<Path, Hash> computeStorePathForPath(const string & name,
bool recursive = true, HashType hashAlgo = htSHA256, const Path & srcPath, bool recursive = true,
PathFilter & filter = defaultPathFilter) const; HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const;
/* Preparatory part of addTextToStore(). /* Preparatory part of addTextToStore().

View file

@ -44,7 +44,7 @@ public:
prefix = std::string("<") + c + ">"; prefix = std::string("<") + c + ">";
} }
writeToStderr(prefix + (tty ? fs.s : filterANSIEscapes(fs.s)) + "\n"); writeToStderr(prefix + filterANSIEscapes(fs.s) + "\n");
} }
void startActivity(ActivityId act, Verbosity lvl, ActivityType type, void startActivity(ActivityId act, Verbosity lvl, ActivityType type,

View file

@ -1178,36 +1178,51 @@ void ignoreException()
} }
string filterANSIEscapes(const string & s, bool nixOnly) std::string filterANSIEscapes(const std::string & s, unsigned int width)
{ {
string t, r; std::string t, e;
enum { stTop, stEscape, stCSI } state = stTop; size_t w = 0;
for (auto c : s) { auto i = s.begin();
if (state == stTop) {
if (c == '\e') { while (w < (size_t) width && i != s.end()) {
state = stEscape;
r = c; if (*i == '\e') {
} else std::string e;
t += c; e += *i++;
} else if (state == stEscape) { char last = 0;
r += c;
if (c == '[') if (i != s.end() && *i == '[') {
state = stCSI; e += *i++;
else { // eat parameter bytes
t += r; while (i != s.end() && *i >= 0x30 && *i <= 0x3f) e += *i++;
state = stTop; // eat intermediate bytes
while (i != s.end() && *i >= 0x20 && *i <= 0x2f) e += *i++;
// eat final byte
if (i != s.end() && *i >= 0x40 && *i <= 0x7e) e += last = *i++;
} else {
if (i != s.end() && *i >= 0x40 && *i <= 0x5f) e += *i++;
} }
} else {
r += c; if (last == 'm')
if (c >= 0x40 && c <= 0x7e) { t += e;
if (nixOnly && (c != 'p' && c != 'q' && c != 's' && c != 'a' && c != 'b')) }
t += r;
state = stTop; else if (*i == '\t') {
r.clear(); i++; t += ' '; w++;
while (w < (size_t) width && w % 8) {
t += ' '; w++;
} }
} }
else if (*i == '\r')
// do nothing for now
;
else {
t += *i++; w++;
}
} }
t += r;
return t; return t;
} }

View file

@ -388,10 +388,12 @@ void ignoreException();
#define ANSI_BLUE "\e[34;1m" #define ANSI_BLUE "\e[34;1m"
/* Filter out ANSI escape codes from the given string. If nixOnly is /* Truncate a string to 'width' printable characters. Certain ANSI
set, only filter escape codes generated by Nixpkgs' stdenv (used to escape sequences (such as colour setting) are copied but not
denote nesting etc.). */ included in the character count. Other ANSI escape sequences are
string filterANSIEscapes(const string & s, bool nixOnly = false); filtered. Also, tabs are expanded to spaces. */
std::string filterANSIEscapes(const std::string & s,
unsigned int width = std::numeric_limits<unsigned int>::max());
/* Base64 encoding/decoding. */ /* Base64 encoding/decoding. */

View file

@ -23,44 +23,6 @@ static uint64_t getI(const std::vector<Logger::Field> & fields, size_t n)
return fields[n].i; return fields[n].i;
} }
/* Truncate a string to 'width' printable characters. ANSI escape
sequences are copied but not included in the character count. Also,
tabs are expanded to spaces. */
static std::string ansiTruncate(const std::string & s, int width)
{
if (width <= 0) return s;
std::string t;
size_t w = 0;
auto i = s.begin();
while (w < (size_t) width && i != s.end()) {
if (*i == '\e') {
t += *i++;
if (i != s.end() && *i == '[') {
t += *i++;
while (i != s.end() && (*i < 0x40 || *i > 0x7e)) {
t += *i++;
}
if (i != s.end()) t += *i++;
}
}
else if (*i == '\t') {
t += ' '; w++;
while (w < (size_t) width && w & 8) {
t += ' '; w++;
}
}
else {
t += *i++; w++;
}
}
return t;
}
class ProgressBar : public Logger class ProgressBar : public Logger
{ {
private: private:
@ -343,7 +305,10 @@ public:
} }
} }
writeToStderr("\r" + ansiTruncate(line, getWindowSize().second) + "\e[K"); auto width = getWindowSize().second;
if (width <= 0) std::numeric_limits<decltype(width)>::max();
writeToStderr("\r" + filterANSIEscapes(line, width) + "\e[K");
} }
std::string getStatus(State & state) std::string getStatus(State & state)

1
tests/lang/data Normal file
View file

@ -0,0 +1 @@
foo

View file

@ -0,0 +1,7 @@
builtins.path
{ path = ./.;
filter = path: _: baseNameOf path == "data";
recursive = true;
sha256 = "1yhm3gwvg5a41yylymgblsclk95fs6jy72w0wv925mmidlhcq4sw";
name = "output";
}

View file

@ -16,4 +16,4 @@ nix-env --foo 2>&1 | grep "no operation"
nix-env -q --foo 2>&1 | grep "unknown flag" nix-env -q --foo 2>&1 | grep "unknown flag"
# Eval Errors. # Eval Errors.
nix-instantiate --eval -E 'let a = {} // a; in a.foo' 2>&1 | grep "infinite recursion encountered, at (string):1:15$" nix-instantiate --eval -E 'let a = {} // a; in a.foo' 2>&1 | grep "infinite recursion encountered, at .*(string).*:1:15$"