Pass lists/attrsets to bash as (associative) arrays

This commit is contained in:
Eelco Dolstra 2017-10-25 13:01:50 +02:00
parent ac12517f3e
commit 2d5b1b24bf
No known key found for this signature in database
GPG key ID: 8170B4726D7198DE
10 changed files with 166 additions and 26 deletions

View file

@ -713,7 +713,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
if (outputHashRecursive) outputHashAlgo = "r:" + outputHashAlgo; if (outputHashRecursive) outputHashAlgo = "r:" + outputHashAlgo;
Path outPath = state.store->makeFixedOutputPath(outputHashRecursive, h, drvName); Path outPath = state.store->makeFixedOutputPath(outputHashRecursive, h, drvName);
drv.env["out"] = outPath; if (!jsonObject) drv.env["out"] = outPath;
drv.outputs["out"] = DerivationOutput(outPath, outputHashAlgo, *outputHash); drv.outputs["out"] = DerivationOutput(outPath, outputHashAlgo, *outputHash);
} }
@ -724,7 +724,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
an empty value. This ensures that changes in the set of an empty value. This ensures that changes in the set of
output names do get reflected in the hash. */ output names do get reflected in the hash. */
for (auto & i : outputs) { for (auto & i : outputs) {
drv.env[i] = ""; if (!jsonObject) drv.env[i] = "";
drv.outputs[i] = DerivationOutput("", "", ""); drv.outputs[i] = DerivationOutput("", "", "");
} }
@ -735,7 +735,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
for (auto & i : drv.outputs) for (auto & i : drv.outputs)
if (i.second.path == "") { if (i.second.path == "") {
Path outPath = state.store->makeOutputPath(i.first, h, drvName); Path outPath = state.store->makeOutputPath(i.first, h, drvName);
drv.env[i.first] = outPath; if (!jsonObject) drv.env[i.first] = outPath;
i.second.path = outPath; i.second.path = outPath;
} }
} }

View file

@ -18,6 +18,7 @@
#include <thread> #include <thread>
#include <future> #include <future>
#include <chrono> #include <chrono>
#include <regex>
#include <limits.h> #include <limits.h>
#include <sys/time.h> #include <sys/time.h>
@ -55,6 +56,8 @@
#include <sys/statvfs.h> #include <sys/statvfs.h>
#endif #endif
#include <nlohmann/json.hpp>
namespace nix { namespace nix {
@ -2286,12 +2289,99 @@ void DerivationGoal::initEnv()
} }
static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*");
void DerivationGoal::writeStructuredAttrs() void DerivationGoal::writeStructuredAttrs()
{ {
auto json = drv->env.find("__json"); auto jsonAttr = drv->env.find("__json");
if (json == drv->env.end()) return; if (jsonAttr == drv->env.end()) return;
writeFile(tmpDir + "/.attrs.json", rewriteStrings(json->second, inputRewrites)); try {
auto jsonStr = rewriteStrings(jsonAttr->second, inputRewrites);
auto json = nlohmann::json::parse(jsonStr);
/* Add an "outputs" object containing the output paths. */
nlohmann::json outputs;
for (auto & i : drv->outputs)
outputs[i.first] = rewriteStrings(i.second.path, inputRewrites);
json["outputs"] = outputs;
writeFile(tmpDir + "/.attrs.json", json.dump());
/* As a convenience to bash scripts, write a shell file that
maps all attributes that are representable in bash -
namely, strings, integers, nulls, Booleans, and arrays and
objects consisting entirely of those values. (So nested
arrays or objects are not supported.) */
auto handleSimpleType = [](const nlohmann::json & value) -> std::experimental::optional<std::string> {
if (value.is_string())
return shellEscape(value);
if (value.is_number()) {
auto f = value.get<float>();
if (std::ceil(f) == f)
return std::to_string(value.get<int>());
}
if (value.is_null())
return "''";
if (value.is_boolean())
return value.get<bool>() ? "1" : "";
return {};
};
std::string jsonSh;
for (auto i = json.begin(); i != json.end(); ++i) {
if (!std::regex_match(i.key(), shVarName)) continue;
auto & value = i.value();
auto s = handleSimpleType(value);
if (s)
jsonSh += fmt("declare %s=%s\n", i.key(), *s);
else if (value.is_array()) {
std::string s2;
bool good = true;
for (auto i = value.begin(); i != value.end(); ++i) {
auto s3 = handleSimpleType(i.value());
if (!s3) { good = false; break; }
s2 += *s3; s2 += ' ';
}
if (good)
jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2);
}
else if (value.is_object()) {
std::string s2;
bool good = true;
for (auto i = value.begin(); i != value.end(); ++i) {
auto s3 = handleSimpleType(i.value());
if (!s3) { good = false; break; }
s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3);
}
if (good)
jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2);
}
}
writeFile(tmpDir + "/.attrs.sh", jsonSh);
} catch (std::exception & e) {
throw Error("cannot process __json attribute of '%s': %s", drvPath, e.what());
}
} }

View file

@ -1142,6 +1142,16 @@ std::string toLower(const std::string & s)
} }
std::string shellEscape(const std::string & s)
{
std::string r = "'";
for (auto & i : s)
if (i == '\'') r += "'\\''"; else r += i;
r += '\'';
return r;
}
void ignoreException() void ignoreException()
{ {
try { try {

View file

@ -352,10 +352,8 @@ bool hasSuffix(const string & s, const string & suffix);
std::string toLower(const std::string & s); std::string toLower(const std::string & s);
/* Escape a string that contains octal-encoded escape codes such as /* Escape a string as a shell word. */
used in /etc/fstab and /proc/mounts (e.g. "foo\040bar" decodes to std::string shellEscape(const std::string & s);
"foo bar"). */
string decodeOctalEscaped(const string & s);
/* Exception handling in destructors: print an error message, then /* Exception handling in destructors: print an error message, then

View file

@ -196,10 +196,6 @@ void mainWrapped(int argc, char * * argv)
interactive = false; interactive = false;
auto execArgs = ""; auto execArgs = "";
auto shellEscape = [](const string & s) {
return "'" + std::regex_replace(s, std::regex("'"), "'\\''") + "'";
};
// Überhack to support Perl. Perl examines the shebang and // Überhack to support Perl. Perl examines the shebang and
// executes it unless it contains the string "perl" or "indir", // executes it unless it contains the string "perl" or "indir",
// or (undocumented) argv[0] does not contain "perl". Exploit // or (undocumented) argv[0] does not contain "perl". Exploit

View file

@ -440,15 +440,6 @@ static void opQuery(Strings opFlags, Strings opArgs)
} }
static string shellEscape(const string & s)
{
string r;
for (auto & i : s)
if (i == '\'') r += "'\\''"; else r += i;
return r;
}
static void opPrintEnv(Strings opFlags, Strings opArgs) static void opPrintEnv(Strings opFlags, Strings opArgs)
{ {
if (!opFlags.empty()) throw UsageError("unknown flag"); if (!opFlags.empty()) throw UsageError("unknown flag");
@ -460,7 +451,7 @@ static void opPrintEnv(Strings opFlags, Strings opArgs)
/* Print each environment variable in the derivation in a format /* Print each environment variable in the derivation in a format
that can be sourced by the shell. */ that can be sourced by the shell. */
for (auto & i : drv.env) for (auto & i : drv.env)
cout << format("export %1%; %1%='%2%'\n") % i.first % shellEscape(i.second); cout << format("export %1%; %1%=%2%\n") % i.first % shellEscape(i.second);
/* Also output the arguments. This doesn't preserve whitespace in /* Also output the arguments. This doesn't preserve whitespace in
arguments. */ arguments. */

View file

@ -13,7 +13,7 @@ rec {
derivation ({ derivation ({
inherit system; inherit system;
builder = shell; builder = shell;
args = ["-e" args.builder or (builtins.toFile "builder.sh" "eval \"$buildCommand\"")]; args = ["-e" args.builder or (builtins.toFile "builder.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")];
PATH = path; PATH = path;
} // removeAttrs args ["builder" "meta"]) } // removeAttrs args ["builder" "meta"])
// { meta = args.meta or {}; }; // { meta = args.meta or {}; };

View file

@ -14,7 +14,8 @@ nix_tests = \
placeholders.sh nix-shell.sh \ placeholders.sh nix-shell.sh \
linux-sandbox.sh \ linux-sandbox.sh \
build-remote.sh \ build-remote.sh \
nar-index.sh nar-index.sh \
structured-attrs.sh
# parallel.sh # parallel.sh
install-tests += $(foreach x, $(nix_tests), tests/$(x)) install-tests += $(foreach x, $(nix_tests), tests/$(x))

View file

@ -0,0 +1,47 @@
with import ./config.nix;
mkDerivation {
name = "structured";
__structuredAttrs = true;
buildCommand = ''
set -x
[[ $int = 123456789 ]]
[[ -z $float ]]
[[ -n $boolTrue ]]
[[ -z $boolFalse ]]
[[ -n ''${hardening[format]} ]]
[[ -z ''${hardening[fortify]} ]]
[[ ''${#buildInputs[@]} = 7 ]]
[[ ''${buildInputs[2]} = c ]]
[[ -v nothing ]]
[[ -z $nothing ]]
mkdir ''${outputs[out]}
echo bar > $dest
'';
buildInputs = [ "a" "b" "c" 123 "'" "\"" null ];
hardening.format = true;
hardening.fortify = false;
outer.inner = [ 1 2 3 ];
int = 123456789;
float = 123.456;
boolTrue = true;
boolFalse = false;
nothing = null;
dest = "${placeholder "out"}/foo";
"foo bar" = "BAD";
"1foobar" = "BAD";
"foo$" = "BAD";
}

View file

@ -0,0 +1,7 @@
source common.sh
clearStore
outPath=$(nix-build structured-attrs.nix --no-out-link)
[[ $(cat $outPath/foo) = bar ]]