From e4ece83b1aad1bbbdc7cd39d9d0829f85330e505 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 12 Dec 2017 08:31:31 -0500 Subject: [PATCH 1/5] tests.setuid: only on i686 and x86_64 linuxs --- release.nix | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/release.nix b/release.nix index c5c2170f7..538211ca9 100644 --- a/release.nix +++ b/release.nix @@ -225,10 +225,12 @@ let nix = build.x86_64-linux; system = "x86_64-linux"; }); - tests.setuid = pkgs.lib.genAttrs (pkgs.lib.filter (pkgs.lib.hasSuffix "-linux") systems) (system: - import ./tests/setuid.nix rec { - nix = build.${system}; inherit system; - }); + tests.setuid = pkgs.lib.genAttrs + (pkgs.lib.filter (system: system == "x86_64-linux" || system == "i686-linux") systems) + (system: + import ./tests/setuid.nix rec { + nix = build.${system}; inherit system; + }); tests.binaryTarball = with import { system = "x86_64-linux"; }; From 69d82e5c58bf6d7e16fc296f598c352da2a618d0 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Thu, 25 Jan 2018 07:05:57 -0800 Subject: [PATCH 2/5] Add path primop. builtins.path allows specifying the name of a path (which makes paths with store-illegal names now addable), allows adding paths with flat instead of recursive hashes, allows specifying a filter (so is a generalization of filterSource), and allows specifying an expected hash (enabling safe path adding in pure mode). --- doc/manual/expressions/builtins.xml | 74 ++++++++++++++++++++++- src/libexpr/eval.cc | 2 +- src/libexpr/primops.cc | 93 +++++++++++++++++++++++------ src/libstore/store-api.cc | 5 +- src/libstore/store-api.hh | 6 +- tests/lang/data | 1 + tests/lang/eval-okay-path.exp | 1 + tests/lang/eval-okay-path.nix | 7 +++ 8 files changed, 162 insertions(+), 27 deletions(-) create mode 100644 tests/lang/data create mode 100644 tests/lang/eval-okay-path.exp create mode 100644 tests/lang/eval-okay-path.nix diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index 5a3a8645c..81770bcf6 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -308,8 +308,9 @@ stdenv.mkDerivation { … } - builtins.filterSource - e1 e2 + + builtins.filterSource + e1 e2 @@ -768,6 +769,75 @@ Evaluates to [ "foo" ]. + + + builtins.path + args + + + + + An enrichment of the built-in path type, based on the attributes + present in args. All are optional + except path: + + + + + path + + The underlying path. + + + + name + + + The name of the path when added to the store. This can + used to reference paths that have nix-illegal characters + in their names, like @. + + + + + filter + + + A function of the type expected by + builtins.filterSource, + with the same semantics. + + + + + recursive + + + When false, when + path is added to the store it is with a + flat hash, rather than a hash of the NAR serialization of + the file. Thus, path must refer to a + regular file, not a directory. This allows similar + behavior to fetchurl. Defaults to + true. + + + + + sha256 + + + When provided, this is the expected hash of the file at + the path. Evaluation will fail if the hash is incorrect, + and providing a hash allows + builtins.path to be used even when the + pure-eval nix config option is on. + + + + + + builtins.pathExists path diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 33a9bc614..9499ebe70 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1566,7 +1566,7 @@ string EvalState::copyPathToStore(PathSet & context, const Path & path) dstPath = srcToStore[path]; else { dstPath = settings.readOnlyMode - ? store->computeStorePathForPath(checkSourcePath(path)).first + ? store->computeStorePathForPath(baseNameOf(path), checkSourcePath(path)).first : store->addToStore(baseNameOf(path), checkSourcePath(path), true, htSHA256, defaultPathFilter, repair); srcToStore[path] = dstPath; printMsg(lvlChatty, format("copied source '%1%' -> '%2%'") diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 975f0e830..5c8dfd9df 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1009,20 +1009,13 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu } -static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_, + Value * filterFun, bool recursive, const Hash & expectedHash, Value & v) { - PathSet context; - Path path = state.coerceToPath(pos, *args[1], context); - if (!context.empty()) - throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % pos); - - state.forceValue(*args[0]); - if (args[0]->type != tLambda) - throw TypeError(format("first argument in call to 'filterSource' is not a function but %1%, at %2%") % showType(*args[0]) % pos); - - path = state.checkSourcePath(path); - - PathFilter filter = [&](const Path & path) { + const auto path = settings.pureEval && expectedHash ? + path_ : + state.checkSourcePath(path_); + PathFilter filter = filterFun ? ([&](const Path & path) { auto st = lstat(path); /* Call the filter function. The first argument is the path, @@ -1031,7 +1024,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args mkString(arg1, path); Value fun2; - state.callFunction(*args[0], arg1, fun2, noPos); + state.callFunction(*filterFun, arg1, fun2, noPos); Value arg2; mkString(arg2, @@ -1044,16 +1037,79 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args state.callFunction(fun2, arg2, res, noPos); return state.forceBool(res, pos); - }; + }) : defaultPathFilter; - Path dstPath = settings.readOnlyMode - ? state.store->computeStorePathForPath(path, true, htSHA256, filter).first - : state.store->addToStore(baseNameOf(path), path, true, htSHA256, filter, state.repair); + Path expectedStorePath; + if (expectedHash) { + expectedStorePath = + state.store->makeFixedOutputPath(recursive, expectedHash, name); + } + Path dstPath; + if (!expectedHash || !state.store->isValidPath(expectedStorePath)) { + dstPath = settings.readOnlyMode + ? state.store->computeStorePathForPath(name, path, recursive, htSHA256, filter).first + : state.store->addToStore(name, path, recursive, htSHA256, filter, state.repair); + if (expectedHash && expectedStorePath != dstPath) { + throw Error(format("store path mismatch in (possibly filtered) path added from '%1%'") % path); + } + } else + dstPath = expectedStorePath; mkString(v, dstPath, {dstPath}); } +static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet context; + Path path = state.coerceToPath(pos, *args[1], context); + if (!context.empty()) + throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % pos); + + state.forceValue(*args[0]); + if (args[0]->type != tLambda) + throw TypeError(format("first argument in call to 'filterSource' is not a function but %1%, at %2%") % showType(*args[0]) % pos); + + addPath(state, pos, baseNameOf(path), path, args[0], true, Hash(), v); +} + +static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceAttrs(*args[0], pos); + Path path; + string name; + Value * filterFun = nullptr; + auto recursive = true; + Hash expectedHash; + + for (auto & attr : *args[0]->attrs) { + const string & n(attr.name); + if (n == "path") { + PathSet context; + path = state.coerceToPath(*attr.pos, *attr.value, context); + if (!context.empty()) + throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % *attr.pos); + } else if (attr.name == state.sName) + name = state.forceStringNoCtx(*attr.value, *attr.pos); + else if (n == "filter") { + state.forceValue(*attr.value); + filterFun = attr.value; + } else if (n == "recursive") + recursive = state.forceBool(*attr.value, *attr.pos); + else if (n == "sha256") + expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); + else + throw EvalError(format("unsupported argument '%1%' to 'addPath', at %2%") % attr.name % *attr.pos); + } + if (path.empty()) + throw EvalError(format("'path' required, at %1%") % pos); + if (name.empty()) + name = baseNameOf(path); + + addPath(state, pos, name, path, filterFun, recursive, expectedHash, v); +} + + /************************************************************* * Sets *************************************************************/ @@ -2071,6 +2127,7 @@ void EvalState::createBaseEnv() addPrimOp("__fromJSON", 1, prim_fromJSON); addPrimOp("__toFile", 2, prim_toFile); addPrimOp("__filterSource", 2, prim_filterSource); + addPrimOp("__path", 1, prim_path); // Sets addPrimOp("__attrNames", 1, prim_attrNames); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 77ab87ef7..7abb300a9 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -222,11 +222,10 @@ Path Store::makeTextPath(const string & name, const Hash & hash, } -std::pair Store::computeStorePathForPath(const Path & srcPath, - bool recursive, HashType hashAlgo, PathFilter & filter) const +std::pair Store::computeStorePathForPath(const string & name, + const Path & srcPath, bool recursive, HashType hashAlgo, PathFilter & filter) const { Hash h = recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath); - string name = baseNameOf(srcPath); Path dstPath = makeFixedOutputPath(recursive, h, name); return std::pair(dstPath, h); } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index c0e735cd3..bf0862ef1 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -305,9 +305,9 @@ public: /* This is the preparatory part of addToStore(); it computes the store path to which srcPath is to be copied. Returns the store path and the cryptographic hash of the contents of srcPath. */ - std::pair computeStorePathForPath(const Path & srcPath, - bool recursive = true, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter) const; + std::pair computeStorePathForPath(const string & name, + const Path & srcPath, bool recursive = true, + HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const; /* Preparatory part of addTextToStore(). diff --git a/tests/lang/data b/tests/lang/data new file mode 100644 index 000000000..257cc5642 --- /dev/null +++ b/tests/lang/data @@ -0,0 +1 @@ +foo diff --git a/tests/lang/eval-okay-path.exp b/tests/lang/eval-okay-path.exp new file mode 100644 index 000000000..6827d49ff --- /dev/null +++ b/tests/lang/eval-okay-path.exp @@ -0,0 +1 @@ +"/run/user/1000/nix-test/store/wjagrv37lfvfx92g2gf3yqflwypj0q1y-output" diff --git a/tests/lang/eval-okay-path.nix b/tests/lang/eval-okay-path.nix new file mode 100644 index 000000000..e67168cf3 --- /dev/null +++ b/tests/lang/eval-okay-path.nix @@ -0,0 +1,7 @@ +builtins.path + { path = ./.; + filter = path: _: baseNameOf path == "data"; + recursive = true; + sha256 = "1yhm3gwvg5a41yylymgblsclk95fs6jy72w0wv925mmidlhcq4sw"; + name = "output"; + } From cfdfad5c3451731879a5a693059c094f107c2bc8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 7 Feb 2018 14:15:20 +0100 Subject: [PATCH 3/5] Simplify --- release.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release.nix b/release.nix index d9c4f1efb..9e04f0b67 100644 --- a/release.nix +++ b/release.nix @@ -225,7 +225,7 @@ let }); tests.setuid = pkgs.lib.genAttrs - (pkgs.lib.filter (system: system == "x86_64-linux" || system == "i686-linux") systems) + ["i686-linux" "x86_64-linux"] (system: import ./tests/setuid.nix rec { inherit nixpkgs; From 84989d3af23c717744b8ddeacd6828bc87e7eda1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 7 Feb 2018 15:19:10 +0100 Subject: [PATCH 4/5] Improve filtering of ANSI escape sequences in build logs All ANSI sequences except color setting are now filtered out. In particular, terminal resets (such as from NixOS VM tests) are filtered out. Also, fix the completely broken tab character handling. --- src/libstore/build.cc | 2 +- src/libutil/logging.cc | 2 +- src/libutil/util.cc | 65 +++++++++++++++++++++++++---------------- src/libutil/util.hh | 10 ++++--- src/nix/progress-bar.cc | 43 +++------------------------ tests/misc.sh | 2 +- 6 files changed, 53 insertions(+), 71 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 5be7ce60d..9f669f7e4 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3428,7 +3428,7 @@ void DerivationGoal::flushLine() else { if (settings.verboseBuild && (settings.printRepeatedBuilds || curRound == 1)) - printError(filterANSIEscapes(currentLogLine, true)); + printError(currentLogLine); else { logTail.push_back(currentLogLine); if (logTail.size() > settings.logLines) logTail.pop_front(); diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 6924e0080..27a631a37 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -44,7 +44,7 @@ public: prefix = std::string("<") + c + ">"; } - writeToStderr(prefix + (tty ? fs.s : filterANSIEscapes(fs.s)) + "\n"); + writeToStderr(prefix + filterANSIEscapes(fs.s) + "\n"); } void startActivity(ActivityId act, Verbosity lvl, ActivityType type, diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 272997397..f7a12d21b 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1178,36 +1178,51 @@ void ignoreException() } -string filterANSIEscapes(const string & s, bool nixOnly) +std::string filterANSIEscapes(const std::string & s, unsigned int width) { - string t, r; - enum { stTop, stEscape, stCSI } state = stTop; - for (auto c : s) { - if (state == stTop) { - if (c == '\e') { - state = stEscape; - r = c; - } else - t += c; - } else if (state == stEscape) { - r += c; - if (c == '[') - state = stCSI; - else { - t += r; - state = stTop; + std::string t, e; + size_t w = 0; + auto i = s.begin(); + + while (w < (size_t) width && i != s.end()) { + + if (*i == '\e') { + std::string e; + e += *i++; + char last = 0; + + if (i != s.end() && *i == '[') { + e += *i++; + // eat parameter bytes + while (i != s.end() && *i >= 0x30 && *i <= 0x3f) e += *i++; + // eat intermediate bytes + while (i != s.end() && *i >= 0x20 && *i <= 0x2f) e += *i++; + // eat final byte + if (i != s.end() && *i >= 0x40 && *i <= 0x7e) e += last = *i++; + } else { + if (i != s.end() && *i >= 0x40 && *i <= 0x5f) e += *i++; } - } else { - r += c; - if (c >= 0x40 && c <= 0x7e) { - if (nixOnly && (c != 'p' && c != 'q' && c != 's' && c != 'a' && c != 'b')) - t += r; - state = stTop; - r.clear(); + + if (last == 'm') + t += e; + } + + else if (*i == '\t') { + i++; t += ' '; w++; + while (w < (size_t) width && w % 8) { + t += ' '; w++; } } + + else if (*i == '\r') + // do nothing for now + ; + + else { + t += *i++; w++; + } } - t += r; + return t; } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 75eb97515..47e02bc89 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -388,10 +388,12 @@ void ignoreException(); #define ANSI_BLUE "\e[34;1m" -/* Filter out ANSI escape codes from the given string. If ‘nixOnly’ is - set, only filter escape codes generated by Nixpkgs' stdenv (used to - denote nesting etc.). */ -string filterANSIEscapes(const string & s, bool nixOnly = false); +/* Truncate a string to 'width' printable characters. Certain ANSI + escape sequences (such as colour setting) are copied but not + included in the character count. Other ANSI escape sequences are + filtered. Also, tabs are expanded to spaces. */ +std::string filterANSIEscapes(const std::string & s, + unsigned int width = std::numeric_limits::max()); /* Base64 encoding/decoding. */ diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc index 252d12c5d..e6553c06f 100644 --- a/src/nix/progress-bar.cc +++ b/src/nix/progress-bar.cc @@ -23,44 +23,6 @@ static uint64_t getI(const std::vector & fields, size_t n) return fields[n].i; } -/* Truncate a string to 'width' printable characters. ANSI escape - sequences are copied but not included in the character count. Also, - tabs are expanded to spaces. */ -static std::string ansiTruncate(const std::string & s, int width) -{ - if (width <= 0) return s; - - std::string t; - size_t w = 0; - auto i = s.begin(); - - while (w < (size_t) width && i != s.end()) { - if (*i == '\e') { - t += *i++; - if (i != s.end() && *i == '[') { - t += *i++; - while (i != s.end() && (*i < 0x40 || *i > 0x7e)) { - t += *i++; - } - if (i != s.end()) t += *i++; - } - } - - else if (*i == '\t') { - t += ' '; w++; - while (w < (size_t) width && w & 8) { - t += ' '; w++; - } - } - - else { - t += *i++; w++; - } - } - - return t; -} - class ProgressBar : public Logger { private: @@ -343,7 +305,10 @@ public: } } - writeToStderr("\r" + ansiTruncate(line, getWindowSize().second) + "\e[K"); + auto width = getWindowSize().second; + if (width <= 0) std::numeric_limits::max(); + + writeToStderr("\r" + filterANSIEscapes(line, width) + "\e[K"); } std::string getStatus(State & state) diff --git a/tests/misc.sh b/tests/misc.sh index 6d0ab3adc..eda016416 100644 --- a/tests/misc.sh +++ b/tests/misc.sh @@ -16,4 +16,4 @@ nix-env --foo 2>&1 | grep "no operation" nix-env -q --foo 2>&1 | grep "unknown flag" # Eval Errors. -nix-instantiate --eval -E 'let a = {} // a; in a.foo' 2>&1 | grep "infinite recursion encountered, at (string):1:15$" +nix-instantiate --eval -E 'let a = {} // a; in a.foo' 2>&1 | grep "infinite recursion encountered, at .*(string).*:1:15$" From 48c192ca2d5bc65b69d2336c8577258f8eb80cf8 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Wed, 7 Feb 2018 10:26:53 -0500 Subject: [PATCH 5/5] builtins.path test: Don't rely on shlevy's XDG_RUNTIME_DIR --- tests/lang/eval-okay-path.exp | 1 - 1 file changed, 1 deletion(-) delete mode 100644 tests/lang/eval-okay-path.exp diff --git a/tests/lang/eval-okay-path.exp b/tests/lang/eval-okay-path.exp deleted file mode 100644 index 6827d49ff..000000000 --- a/tests/lang/eval-okay-path.exp +++ /dev/null @@ -1 +0,0 @@ -"/run/user/1000/nix-test/store/wjagrv37lfvfx92g2gf3yqflwypj0q1y-output"