lix-project/lix
20
55
Fork 31
Code Issues 345 Code Review (Gerrit) Projects 3 Releases Wiki Activity

Merge remote-tracking branch 'upstream/master' into multi-output-hashDerivationModulo

Browse source
This commit is contained in:
John Ericson 2020-06-21 16:43:17 +00:00
parent 02928f76fd 984e521392
commit fdeabf7160
48 changed files with 490 additions and 227 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
doc/manual/advanced-topics/diff-hook.xml
View file

@ -70,7 +70,7 @@ path just built.</para>
<screen>
$ nix-build ./deterministic.nix -A stable
these derivations will be built:
this derivation will be built:
/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv
building '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'...
/nix/store/yyxlzw3vqaas7wfp04g0b1xg51f2czgq-stable
@ -85,7 +85,7 @@ checking outputs of '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'...
<screen>
$ nix-build ./deterministic.nix -A unstable
these derivations will be built:
this derivation will be built:
/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv
building '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'...
/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable
@ -193,7 +193,7 @@ repeat = 1
An example output of this configuration:
<screen>
$ nix-build ./test.nix -A unstable
these derivations will be built:
this derivation will be built:
/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv
building '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' (round 1/2)...
building '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' (round 2/2)...

2
doc/manual/advanced-topics/post-build-hook.xml
View file

@ -122,7 +122,7 @@ post-build-hook = /etc/nix/upload-to-cache.sh
<screen>
$ nix-build -E '(import &lt;nixpkgs&gt; {}).writeText "example" (builtins.toString builtins.currentTime)'
these derivations will be built:
this derivation will be built:
/nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv
building '/nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv'...
running post-build-hook '/home/grahamc/projects/github.com/NixOS/nix/post-hook.sh'...

2
doc/manual/command-ref/nix-env.xml
View file

@ -516,7 +516,7 @@ source:
$ nix-env -f '&lt;nixpkgs>' -iA hello --dry-run
(dry run; not doing anything)
installing ‘hello-2.10’
these paths will be fetched (0.04 MiB download, 0.19 MiB unpacked):
this path will be fetched (0.04 MiB download, 0.19 MiB unpacked):
/nix/store/wkhdf9jinag5750mqlax6z2zbwhqb76n-hello-2.10
<replaceable>...</replaceable></screen>

2
doc/manual/installation/env-variables.xml
View file

@ -39,7 +39,7 @@ bundle.</para>
<step><para>Set the environment variable and install Nix</para>
<screen>
$ export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt
$ sh &lt;(curl https://nixos.org/nix/install)
$ sh &lt;(curl -L https://nixos.org/nix/install)
</screen></step>
<step><para>In the shell profile and rc files (for example,

4
doc/manual/installation/installing-binary.xml
View file

@ -12,7 +12,7 @@
</para>
<screen>
$ sh &lt;(curl https://nixos.org/nix/install)
$ sh &lt;(curl -L https://nixos.org/nix/install)
</screen>
<para>
@ -39,7 +39,7 @@
To explicitly select a single-user installation on your system:
<screen>
sh &lt;(curl https://nixos.org/nix/install) --no-daemon
sh &lt;(curl -L https://nixos.org/nix/install) --no-daemon
</screen>
</para>

2
doc/manual/introduction/quick-start.xml
View file

@ -15,7 +15,7 @@ to subsequent chapters.</para>
<step><para>Install single-user Nix by running the following:
<screen>
$ bash &lt;(curl https://nixos.org/nix/install)
$ bash &lt;(curl -L https://nixos.org/nix/install)
</screen>
This will install Nix in <filename>/nix</filename>. The install script

2
shell.nix
View file

@ -7,7 +7,7 @@ with import ./release-common.nix { inherit pkgs; };
(if useClang then clangStdenv else stdenv).mkDerivation {
name = "nix";
buildInputs = buildDeps ++ propagatedDeps ++ perlDeps ++ [ pkgs.rustfmt ];
buildInputs = buildDeps ++ propagatedDeps ++ perlDeps;
inherit configureFlags;

13
src/libexpr/attr-path.cc
View file

@ -6,11 +6,11 @@
namespace nix {
static Strings parseAttrPath(const string & s)
static Strings parseAttrPath(std::string_view s)
{
Strings res;
string cur;
string::const_iterator i = s.begin();
auto i = s.begin();
while (i != s.end()) {
if (*i == '.') {
res.push_back(cur);
@ -32,6 +32,15 @@ static Strings parseAttrPath(const string & s)
}
std::vector<Symbol> parseAttrPath(EvalState & state, std::string_view s)
{
std::vector<Symbol> res;
for (auto & a : parseAttrPath(s))
res.push_back(state.symbols.create(a));
return res;
}
std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attrPath,
Bindings & autoArgs, Value & vIn)
{

2
src/libexpr/attr-path.hh
View file

@ -16,4 +16,6 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
/* Heuristic to find the filename and lineno or a nix value. */
Pos findDerivationFilename(EvalState & state, Value & v, std::string what);
std::vector<Symbol> parseAttrPath(EvalState & state, std::string_view s);
}

39
src/libexpr/eval.cc
View file

@ -161,12 +161,12 @@ const Value *getPrimOp(const Value &v) {
}
string showType(const Value & v)
string showType(ValueType type)
{
switch (v.type) {
switch (type) {
case tInt: return "an integer";
case tBool: return "a boolean";
case tString: return v.string.context ? "a string with context" : "a string";
case tBool: return "a Boolean";
case tString: return "a string";
case tPath: return "a path";
case tNull: return "null";
case tAttrs: return "a set";
@ -175,14 +175,27 @@ string showType(const Value & v)
case tApp: return "a function application";
case tLambda: return "a function";
case tBlackhole: return "a black hole";
case tPrimOp: return "a built-in function";
case tPrimOpApp: return "a partially applied built-in function";
case tExternal: return "an external value";
case tFloat: return "a float";
}
abort();
}
string showType(const Value & v)
{
switch (v.type) {
case tString: return v.string.context ? "a string with context" : "a string";
case tPrimOp:
return fmt("the built-in function '%s'", string(v.primOp->name));
case tPrimOpApp:
return fmt("the partially applied built-in function '%s'", string(getPrimOp(v)->primOp->name));
case tExternal: return v.external->showType();
case tFloat: return "a float";
default:
return showType(v.type);
}
abort();
}
@ -323,6 +336,7 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
, sOutputHash(symbols.create("outputHash"))
, sOutputHashAlgo(symbols.create("outputHashAlgo"))
, sOutputHashMode(symbols.create("outputHashMode"))
, sRecurseForDerivations(symbols.create("recurseForDerivations"))
, repair(NoRepair)
, store(store)
, baseEnv(allocEnv(128))
@ -471,14 +485,21 @@ Value * EvalState::addConstant(const string & name, Value & v)
Value * EvalState::addPrimOp(const string & name,
size_t arity, PrimOpFun primOp)
{
auto name2 = string(name, 0, 2) == "__" ? string(name, 2) : name;
Symbol sym = symbols.create(name2);
/* Hack to make constants lazy: turn them into a application of
the primop to a dummy value. */
if (arity == 0) {
auto vPrimOp = allocValue();
vPrimOp->type = tPrimOp;
vPrimOp->primOp = new PrimOp(primOp, 1, sym);
Value v;
primOp(*this, noPos, nullptr, v);
mkApp(v, *vPrimOp, *vPrimOp);
return addConstant(name, v);
}
Value * v = allocValue();
string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name;
Symbol sym = symbols.create(name2);
v->type = tPrimOp;
v->primOp = new PrimOp(primOp, arity, sym);
staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl;

6
src/libexpr/eval.hh
View file

@ -18,7 +18,7 @@ namespace nix {
class Store;
class EvalState;
struct StorePath;
class StorePath;
enum RepairFlag : bool;
@ -74,7 +74,8 @@ public:
sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls,
sFile, sLine, sColumn, sFunctor, sToString,
sRight, sWrong, sStructuredAttrs, sBuilder, sArgs,
sOutputHash, sOutputHashAlgo, sOutputHashMode;
sOutputHash, sOutputHashAlgo, sOutputHashMode,
sRecurseForDerivations;
Symbol sDerivationNix;
/* If set, force copying files to the Nix store even if they
@ -324,6 +325,7 @@ private:
/* Return a string representing the type of the value `v'. */
string showType(ValueType type);
string showType(const Value & v);
/* Decode a context string ‘!<name>!<path>’ into a pair <path,

2
src/libexpr/get-drvs.cc
View file

@ -348,7 +348,7 @@ static void getDerivations(EvalState & state, Value & vIn,
should we recurse into it? => Only if it has a
`recurseForDerivations = true' attribute. */
if (i->value->type == tAttrs) {
Bindings::iterator j = i->value->attrs->find(state.symbols.create("recurseForDerivations"));
Bindings::iterator j = i->value->attrs->find(state.sRecurseForDerivations);
if (j != i->value->attrs->end() && state.forceBool(*j->value, *j->pos))
getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
}

57
src/libexpr/primops.cc
View file

@ -50,20 +50,20 @@ void EvalState::realiseContext(const PathSet & context)
std::vector<StorePathWithOutputs> drvs;
for (auto & i : context) {
std::pair<string, string> decoded = decodeContext(i);
auto ctx = store->parseStorePath(decoded.first);
auto [ctxS, outputName] = decodeContext(i);
auto ctx = store->parseStorePath(ctxS);
if (!store->isValidPath(ctx))
throw InvalidPathError(store->printStorePath(ctx));
if (!decoded.second.empty() && ctx.isDerivation()) {
drvs.push_back(StorePathWithOutputs{ctx, {decoded.second}});
if (!outputName.empty() && ctx.isDerivation()) {
drvs.push_back(StorePathWithOutputs{ctx, {outputName}});
/* Add the output of this derivation to the allowed
paths. */
if (allowedPaths) {
auto drv = store->derivationFromPath(store->parseStorePath(decoded.first));
DerivationOutputs::iterator i = drv.outputs.find(decoded.second);
auto drv = store->derivationFromPath(ctx);
DerivationOutputs::iterator i = drv.outputs.find(outputName);
if (i == drv.outputs.end())
throw Error("derivation '%s' does not have an output named '%s'", decoded.first, decoded.second);
throw Error("derivation '%s' does not have an output named '%s'", ctxS, outputName);
allowedPaths->insert(store->printStorePath(i->second.path));
}
}
@ -79,6 +79,7 @@ void EvalState::realiseContext(const PathSet & context)
StorePathSet willBuild, willSubstitute, unknown;
unsigned long long downloadSize, narSize;
store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize);
store->buildPaths(drvs);
}
@ -768,17 +769,17 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
.nixCode = NixCode { .errPos = posDrvName }
});
HashType ht = outputHashAlgo.empty() ? htUnknown : parseHashType(outputHashAlgo);
std::optional<HashType> ht = parseHashTypeOpt(outputHashAlgo);
Hash h = newHashAllowEmpty(*outputHash, ht);
auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName);
if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath);
drv.outputs.insert_or_assign("out", DerivationOutput {
std::move(outPath),
(ingestionMethod == FileIngestionMethod::Recursive ? "r:" : "")
+ printHashType(h.type),
h.to_string(Base16, false),
.path = std::move(outPath),
.hash = DerivationOutputHash {
.method = ingestionMethod,
.hash = std::move(h),
},
});
}
@ -792,7 +793,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
for (auto & i : outputs) {
if (!jsonObject) drv.env[i] = "";
drv.outputs.insert_or_assign(i,
DerivationOutput { StorePath::dummy, "", "" });
DerivationOutput {
.path = StorePath::dummy,
.hash = std::optional<DerivationOutputHash> {},
});
}
// Regular, non-CA derivation should always return a single hash and not
@ -803,7 +807,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
auto outPath = state.store->makeOutputPath(i, h, drvName);
if (!jsonObject) drv.env[i] = state.store->printStorePath(outPath);
drv.outputs.insert_or_assign(i,
DerivationOutput { std::move(outPath), "", "" });
DerivationOutput {
.path = std::move(outPath),
.hash = std::optional<DerivationOutputHash>(),
});
}
}
@ -1000,8 +1007,8 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
string type = state.forceStringNoCtx(*args[0], pos);
HashType ht = parseHashType(type);
if (ht == htUnknown)
std::optional<HashType> ht = parseHashType(type);
if (!ht)
throw Error({
.hint = hintfmt("unknown hash type '%1%'", type),
.nixCode = NixCode { .errPos = pos }
@ -1010,7 +1017,7 @@ static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Va
PathSet context; // discarded
Path p = state.coerceToPath(pos, *args[1], context);
mkString(v, hashFile(ht, state.checkSourcePath(p)).to_string(Base16, false), context);
mkString(v, hashFile(*ht, state.checkSourcePath(p)).to_string(Base16, false), context);
}
/* Read a directory (without . or ..) */
@ -1937,8 +1944,8 @@ static void prim_stringLength(EvalState & state, const Pos & pos, Value * * args
static void prim_hashString(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
string type = state.forceStringNoCtx(*args[0], pos);
HashType ht = parseHashType(type);
if (ht == htUnknown)
std::optional<HashType> ht = parseHashType(type);
if (!ht)
throw Error({
.hint = hintfmt("unknown hash type '%1%'", type),
.nixCode = NixCode { .errPos = pos }
@ -1947,7 +1954,7 @@ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args,
PathSet context; // discarded
string s = state.forceString(*args[1], context, pos);
mkString(v, hashString(ht, s).to_string(Base16, false), context);
mkString(v, hashString(*ht, s).to_string(Base16, false), context);
}
@ -2203,10 +2210,11 @@ static void prim_splitVersion(EvalState & state, const Pos & pos, Value * * args
RegisterPrimOp::PrimOps * RegisterPrimOp::primOps;
RegisterPrimOp::RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun)
RegisterPrimOp::RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun,
std::optional<std::string> requiredFeature)
{
if (!primOps) primOps = new PrimOps;
primOps->emplace_back(name, arity, fun);
primOps->push_back({name, arity, fun, requiredFeature});
}
@ -2398,7 +2406,8 @@ void EvalState::createBaseEnv()
if (RegisterPrimOp::primOps)
for (auto & primOp : *RegisterPrimOp::primOps)
addPrimOp(std::get<0>(primOp), std::get<1>(primOp), std::get<2>(primOp));
if (!primOp.requiredFeature || settings.isExperimentalFeatureEnabled(*primOp.requiredFeature))
addPrimOp(primOp.name, primOp.arity, primOp.primOp);
/* Now that we've added all primops, sort the `builtins' set,
because attribute lookups expect it to be sorted. */

17
src/libexpr/primops.hh
View file

@ -7,12 +7,25 @@ namespace nix {
struct RegisterPrimOp
{
typedef std::vector<std::tuple<std::string, size_t, PrimOpFun>> PrimOps;
struct Info
{
std::string name;
size_t arity;
PrimOpFun primOp;
std::optional<std::string> requiredFeature;
};
typedef std::vector<Info> PrimOps;
static PrimOps * primOps;
/* You can register a constant by passing an arity of 0. fun
will get called during EvalState initialization, so there
may be primops not yet added and builtins is not yet sorted. */
RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun);
RegisterPrimOp(
std::string name,
size_t arity,
PrimOpFun fun,
std::optional<std::string> requiredFeature = {});
};
/* These primops are disabled without enableNativeCode, but plugins

2
src/libfetchers/fetchers.cc
View file

@ -36,7 +36,7 @@ std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs)
if (res) {
if (auto narHash = maybeGetStrAttr(attrs, "narHash"))
// FIXME: require SRI hash.
res->narHash = newHashAllowEmpty(*narHash, htUnknown);
res->narHash = newHashAllowEmpty(*narHash, {});
return res;
}
}

2
src/libfetchers/tarball.cc
View file

@ -264,7 +264,7 @@ struct TarballInputScheme : InputScheme
auto input = std::make_unique<TarballInput>(parseURL(getStrAttr(attrs, "url")));
if (auto hash = maybeGetStrAttr(attrs, "hash"))
input->hash = newHashAllowEmpty(*hash, htUnknown);
input->hash = newHashAllowEmpty(*hash, {});
return input;
}

20
src/libmain/shared.cc
View file

@ -48,7 +48,10 @@ void printMissing(ref<Store> store, const StorePathSet & willBuild,
unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl)
{
if (!willBuild.empty()) {
printMsg(lvl, "these derivations will be built:");
if (willBuild.size() == 1)
printMsg(lvl, fmt("this derivation will be built:"));
else
printMsg(lvl, fmt("these %d derivations will be built:", willBuild.size()));
auto sorted = store->topoSortPaths(willBuild);
reverse(sorted.begin(), sorted.end());
for (auto & i : sorted)
@ -56,9 +59,18 @@ void printMissing(ref<Store> store, const StorePathSet & willBuild,
}
if (!willSubstitute.empty()) {
printMsg(lvl, fmt("these paths will be fetched (%.2f MiB download, %.2f MiB unpacked):",
downloadSize / (1024.0 * 1024.0),
narSize / (1024.0 * 1024.0)));
const float downloadSizeMiB = downloadSize / (1024.f * 1024.f);
const float narSizeMiB = narSize / (1024.f * 1024.f);
if (willSubstitute.size() == 1) {
printMsg(lvl, fmt("this path will be fetched (%.2f MiB download, %.2f MiB unpacked):",
downloadSizeMiB,
narSizeMiB));
} else {
printMsg(lvl, fmt("these %d paths will be fetched (%.2f MiB download, %.2f MiB unpacked):",
willSubstitute.size(),
downloadSizeMiB,
narSizeMiB));
}
for (auto & i : willSubstitute)
printMsg(lvl, fmt(" %s", store->printStorePath(i)));
}

2
src/libstore/binary-cache-store.cc
View file

@ -388,8 +388,6 @@ void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSe
narInfo->sigs.insert(sigs.begin(), sigs.end());
auto narInfoFile = narInfoFileFor(narInfo->path);
writeNarInfo(narInfo);
}

31
src/libstore/build.cc
View file

@ -86,7 +86,7 @@ struct HookInstance;
/* A pointer to a goal. */
class Goal;
struct Goal;
class DerivationGoal;
typedef std::shared_ptr<Goal> GoalPtr;
typedef std::weak_ptr<Goal> WeakGoalPtr;
@ -1195,6 +1195,12 @@ void DerivationGoal::haveDerivation()
parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv);
if (parsedDrv->contentAddressed()) {
settings.requireExperimentalFeature("ca-derivations");
throw Error("ca-derivations isn't implemented yet");
}
/* We are first going to try to create the invalid output paths
through substitutes. If that doesn't work, we'll build
them. */
@ -3712,10 +3718,7 @@ void DerivationGoal::registerOutputs()
if (fixedOutput) {
FileIngestionMethod outputHashMode; Hash h;
i.second.parseHashInfo(outputHashMode, h);
if (outputHashMode == FileIngestionMethod::Flat) {
if (i.second.hash->method == FileIngestionMethod::Flat) {
/* The output path should be a regular file without execute permission. */
if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0)
throw BuildError(
@ -3726,20 +3729,22 @@ void DerivationGoal::registerOutputs()
/* Check the hash. In hash mode, move the path produced by
the derivation to its content-addressed location. */
Hash h2 = outputHashMode == FileIngestionMethod::Recursive
? hashPath(h.type, actualPath).first
: hashFile(h.type, actualPath);
Hash h2 = i.second.hash->method == FileIngestionMethod::Recursive
? hashPath(*i.second.hash->hash.type, actualPath).first
: hashFile(*i.second.hash->hash.type, actualPath);
auto dest = worker.store.makeFixedOutputPath(outputHashMode, h2, i.second.path.name());
auto dest = worker.store.makeFixedOutputPath(i.second.hash->method, h2, i.second.path.name());
if (h != h2) {
if (i.second.hash->hash != h2) {
/* Throw an error after registering the path as
valid. */
worker.hashMismatch = true;
delayedException = std::make_exception_ptr(
BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s",
worker.store.printStorePath(dest), h.to_string(SRI, true), h2.to_string(SRI, true)));
worker.store.printStorePath(dest),
i.second.hash->hash.to_string(SRI, true),
h2.to_string(SRI, true)));
Path actualDest = worker.store.Store::toRealPath(dest);
@ -3759,7 +3764,7 @@ void DerivationGoal::registerOutputs()
else
assert(worker.store.parseStorePath(path) == dest);
ca = makeFixedOutputCA(outputHashMode, h2);
ca = makeFixedOutputCA(i.second.hash->method, h2);
}
/* Get rid of all weird permissions. This also checks that
@ -4992,7 +4997,7 @@ bool Worker::pathContentsGood(const StorePath & path)
if (!pathExists(store.printStorePath(path)))
res = false;
else {
HashResult current = hashPath(info->narHash.type, store.printStorePath(path));
HashResult current = hashPath(*info->narHash.type, store.printStorePath(path));
Hash nullHash(htSHA256);
res = info->narHash == nullHash || info->narHash == current.first;
}

4
src/libstore/builtins/fetchurl.cc
View file

@ -63,9 +63,9 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
for (auto hashedMirror : settings.hashedMirrors.get())
try {
if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
auto ht = parseHashType(getAttr("outputHashAlgo"));
auto ht = parseHashTypeOpt(getAttr("outputHashAlgo"));
auto h = Hash(getAttr("outputHash"), ht);
fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false));
fetch(hashedMirror + printHashType(*h.type) + "/" + h.to_string(Base16, false));
return;
} catch (Error & e) {
debug(e.what());

145
src/libstore/derivations.cc
View file

@ -8,22 +8,8 @@
namespace nix {
void DerivationOutput::parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const
{
recursive = FileIngestionMethod::Flat;
string algo = hashAlgo;
if (string(algo, 0, 2) == "r:") {
recursive = FileIngestionMethod::Recursive;
algo = string(algo, 2);
}
HashType hashType = parseHashType(algo);
if (hashType == htUnknown)
throw Error("unknown hash algorithm '%s'", algo);
hash = Hash(this->hash, hashType);
std::string DerivationOutputHash::printMethodAlgo() const {
return makeFileIngestionPrefix(method) + printHashType(*hash.type);
}
@ -120,6 +106,34 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
}
static DerivationOutput parseDerivationOutput(const Store & store, istringstream_nocopy & str)
{
expect(str, ","); auto path = store.parseStorePath(parsePath(str));
expect(str, ","); auto hashAlgo = parseString(str);
expect(str, ","); const auto hash = parseString(str);
expect(str, ")");
std::optional<DerivationOutputHash> fsh;
if (hashAlgo != "") {
auto method = FileIngestionMethod::Flat;
if (string(hashAlgo, 0, 2) == "r:") {
method = FileIngestionMethod::Recursive;
hashAlgo = string(hashAlgo, 2);
}
const HashType hashType = parseHashType(hashAlgo);
fsh = DerivationOutputHash {
.method = std::move(method),
.hash = Hash(hash, hashType),
};
}
return DerivationOutput {
.path = std::move(path),
.hash = std::move(fsh),
};
}
static Derivation parseDerivation(const Store & store, const string & s)
{
Derivation drv;
@ -129,15 +143,8 @@ static Derivation parseDerivation(const Store & store, const string & s)
/* Parse the list of outputs. */
while (!endOfList(str)) {
expect(str, "("); std::string id = parseString(str);
expect(str, ","); auto path = store.parseStorePath(parsePath(str));
expect(str, ","); auto hashAlgo = parseString(str);
expect(str, ","); auto hash = parseString(str);
expect(str, ")");
drv.outputs.emplace(id, DerivationOutput {
.path = std::move(path),
.hashAlgo = std::move(hashAlgo),
.hash = std::move(hash)
});
auto output = parseDerivationOutput(store, str);
drv.outputs.emplace(std::move(id), std::move(output));
}
/* Parse the list of input derivations. */
@ -263,8 +270,9 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
if (first) first = false; else s += ',';
s += '('; printUnquotedString(s, i.first);
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path));
s += ','; printUnquotedString(s, i.second.hashAlgo);
s += ','; printUnquotedString(s, i.second.hash);
s += ','; printUnquotedString(s, i.second.hash ? i.second.hash->printMethodAlgo() : "");
s += ','; printUnquotedString(s,
i.second.hash ? i.second.hash->hash.to_string(Base16, false) : "");
s += ')';
}
@ -320,7 +328,7 @@ bool BasicDerivation::isFixedOutput() const
{
return outputs.size() == 1 &&
outputs.begin()->first == "out" &&
outputs.begin()->second.hash != "";
outputs.begin()->second.hash;
}
@ -332,7 +340,7 @@ DrvHashes drvHashes;
/* Look up the derivation by value and memoize the
`hashDerivationModulo` call.
*/
static DrvHashModulo & pathDerivationModulo(Store & store, const StorePath & drvPath)
static const DrvHashModulo & pathDerivationModulo(Store & store, const StorePath & drvPath)
{
auto h = drvHashes.find(drvPath);
if (h == drvHashes.end()) {
@ -348,6 +356,10 @@ static DrvHashModulo & pathDerivationModulo(Store & store, const StorePath & drv
return h->second;
}
// FIXME: Boilerplatflate for `std::visit`, put this somewhere?
template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
/* See the header for interface details. These are the implementation details.
For fixed-output derivations, each hash in the map is not the
@ -372,8 +384,8 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
std::map<std::string, Hash> outputHashes;
for (const auto & i : drv.outputs) {
const Hash h = hashString(htSHA256, "fixed:out:"
+ i.second.hashAlgo + ":"
+ i.second.hash + ":"
+ i.second.hash->printMethodAlgo() + ":"
+ i.second.hash->hash.to_string(Base16, false) + ":"
+ store.printStorePath(i.second.path));
outputHashes.insert_or_assign(std::string(i.first), std::move(h));
}
@ -384,21 +396,24 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
calls to this function. */
std::map<std::string, StringSet> inputs2;
for (auto & i : drv.inputDrvs) {
const auto res = pathDerivationModulo(store, i.first);
if (const Hash *pval = std::get_if<0>(&res)) {
// regular non-CA derivation, replace derivation
inputs2.insert_or_assign(pval->to_string(Base16, false), i.second);
} else if (const std::map<std::string, Hash> *pval = std::get_if<1>(&res)) {
const auto & res = pathDerivationModulo(store, i.first);
std::visit(overloaded {
// Regular non-CA derivation, replace derivation
[&](Hash drvHash) {
inputs2.insert_or_assign(drvHash.to_string(Base16, false), i.second);
},
// CA derivation's output hashes
std::set justOut = { std::string("out") };
for (auto & output : i.second) {
/* Put each one in with a single "out" output.. */
const auto h = pval->at(output);
inputs2.insert_or_assign(
h.to_string(Base16, false),
justOut);
}
}
[&](CaOutputHashes outputHashes) {
std::set justOut = { std::string("out") };
for (auto & output : i.second) {
/* Put each one in with a single "out" output.. */
const auto h = outputHashes.at(output);
inputs2.insert_or_assign(
h.to_string(Base16, false),
justOut);
}
},
}, res);
}
return hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2));
@ -427,6 +442,31 @@ StorePathSet BasicDerivation::outputPaths() const
return paths;
}
static DerivationOutput readDerivationOutput(Source & in, const Store & store)
{
auto path = store.parseStorePath(readString(in));
auto hashAlgo = readString(in);
const auto hash = readString(in);
std::optional<DerivationOutputHash> fsh;
if (hashAlgo != "") {
auto method = FileIngestionMethod::Flat;
if (string(hashAlgo, 0, 2) == "r:") {
method = FileIngestionMethod::Recursive;
hashAlgo = string(hashAlgo, 2);
}
const HashType hashType = parseHashType(hashAlgo);
fsh = DerivationOutputHash {
.method = std::move(method),
.hash = Hash(hash, hashType),
};
}
return DerivationOutput {
.path = std::move(path),
.hash = std::move(fsh),
};
}
StringSet BasicDerivation::outputNames() const
{
@ -443,14 +483,8 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv)
auto nr = readNum<size_t>(in);
for (size_t n = 0; n < nr; n++) {
auto name = readString(in);
auto path = store.parseStorePath(readString(in));
auto hashAlgo = readString(in);
auto hash = readString(in);
drv.outputs.emplace(name, DerivationOutput {
.path = std::move(path),
.hashAlgo = std::move(hashAlgo),
.hash = std::move(hash)
});
auto output = readDerivationOutput(in, store);
drv.outputs.emplace(std::move(name), std::move(output));
}
drv.inputSrcs = readStorePaths<StorePathSet>(store, in);
@ -472,7 +506,10 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
{
out << drv.outputs.size();
for (auto & i : drv.outputs)
out << i.first << store.printStorePath(i.second.path) << i.second.hashAlgo << i.second.hash;
out << i.first
<< store.printStorePath(i.second.path)
<< i.second.hash->printMethodAlgo()
<< i.second.hash->hash.to_string(Base16, false);
writeStorePaths(store, out, drv.inputSrcs);
out << drv.platform << drv.builder << drv.args;
out << drv.env.size();

16
src/libstore/derivations.hh
View file

@ -13,11 +13,17 @@ namespace nix {
/* Abstract syntax of derivations. */
/// Pair of a hash, and how the file system was ingested
struct DerivationOutputHash {
FileIngestionMethod method;
Hash hash;
std::string printMethodAlgo() const;
};
struct DerivationOutput
{
StorePath path;
std::string hashAlgo; /* hash used for expected hash computation */
std::string hash; /* expected hash, may be null */
std::optional<DerivationOutputHash> hash; /* hash used for expected hash computation */
void parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const;
};
@ -82,9 +88,13 @@ Derivation readDerivation(const Store & store, const Path & drvPath);
// FIXME: remove
bool isDerivation(const string & fileName);
// known CA drv's output hashes, current just for fixed-output derivations
// whose output hashes are always known since they are fixed up-front.
typedef std::map<std::string, Hash> CaOutputHashes;
typedef std::variant<
Hash, // regular DRV normalized hash
std::map<std::string, Hash> // known CA drv's output hashes
CaOutputHashes
> DrvHashModulo;
/* Returns hashes with the details of fixed-output subderivations

2
src/libstore/export-import.cc
View file

@ -55,7 +55,7 @@ void Store::exportPath(const StorePath & path, Sink & sink)
filesystem corruption from spreading to other machines.
Don't complain if the stored hash is zero (unknown). */
Hash hash = hashAndWriteSink.currentHash();
if (hash != info->narHash && info->narHash != Hash(info->narHash.type))
if (hash != info->narHash && info->narHash != Hash(*info->narHash.type))
throw Error("hash of path '%s' has changed from '%s' to '%s'!",
printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true));

6
src/libstore/filetransfer.cc
View file

@ -72,6 +72,7 @@ struct curlFileTransfer : public FileTransfer
curl_off_t writtenToSink = 0;
inline static const std::set<long> successfulStatuses {200, 201, 204, 206, 304, 0 /* other protocol */};
/* Get the HTTP status code, or 0 for other protocols. */
long getHTTPStatus()
{
@ -98,7 +99,7 @@ struct curlFileTransfer : public FileTransfer
/* Only write data to the sink if this is a
successful response. */
if (httpStatus == 0 || httpStatus == 200 || httpStatus == 201 || httpStatus == 206) {
if (successfulStatuses.count(httpStatus)) {
writtenToSink += len;
this->request.dataCallback((char *) data, len);
}
@ -352,8 +353,7 @@ struct curlFileTransfer : public FileTransfer
if (writeException)
failEx(writeException);
else if (code == CURLE_OK &&
(httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 206 || httpStatus == 304 || httpStatus == 0 /* other protocol */))
else if (code == CURLE_OK && successfulStatuses.count(httpStatus))
{
result.cached = httpStatus == 304;
act.progress(result.bodySize, result.bodySize);

14
src/libstore/local-store.cc
View file

@ -561,10 +561,12 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
if (out == drv.outputs.end())
throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath));
FileIngestionMethod method; Hash h;
out->second.parseHashInfo(method, h);
check(makeFixedOutputPath(method, h, drvName), out->second.path, "out");
check(
makeFixedOutputPath(
out->second.hash->method,
out->second.hash->hash,
drvName),
out->second.path, "out");
}
else {
@ -1255,9 +1257,9 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
std::unique_ptr<AbstractHashSink> hashSink;
if (info->ca == "" || !info->references.count(info->path))
hashSink = std::make_unique<HashSink>(info->narHash.type);
hashSink = std::make_unique<HashSink>(*info->narHash.type);
else
hashSink = std::make_unique<HashModuloSink>(info->narHash.type, std::string(info->path.hashPart()));
hashSink = std::make_unique<HashModuloSink>(*info->narHash.type, std::string(info->path.hashPart()));
dumpPath(Store::toRealPath(i), *hashSink);
auto current = hashSink->finish();

5
src/libstore/parsed-derivations.cc
View file

@ -117,4 +117,9 @@ bool ParsedDerivation::substitutesAllowed() const
return getBoolAttr("allowSubstitutes", true);
}
bool ParsedDerivation::contentAddressed() const
{
return getBoolAttr("__contentAddressed", false);
}
}

2
src/libstore/parsed-derivations.hh
View file

@ -34,6 +34,8 @@ public:
bool willBuildLocally() const;
bool substitutesAllowed() const;
bool contentAddressed() const;
};
}

2
src/libstore/remote-store.cc
View file

@ -228,7 +228,7 @@ struct ConnectionHandle
~ConnectionHandle()
{
if (!daemonException && std::uncaught_exception()) {
if (!daemonException && std::uncaught_exceptions()) {
handle.markBad();
debug("closing daemon connection because of an exception");
}

40
src/libstore/store-api.cc
View file

@ -61,7 +61,7 @@ StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view p
/* Store paths have the following form:
<store>/<h>-<name>
<realized-path> = <store>/<h>-<name>
where
@ -85,11 +85,14 @@ StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view p
<type> = one of:
"text:<r1>:<r2>:...<rN>"
for plain text files written to the store using
addTextToStore(); <r1> ... <rN> are the references of the
path.
"source"
addTextToStore(); <r1> ... <rN> are the store paths referenced
by this path, in the form described by <realized-path>
"source:<r1>:<r2>:...:<rN>:self"
for paths copied to the store using addToStore() when recursive
= true and hashAlgo = "sha256"
= true and hashAlgo = "sha256". Just like in the text case, we
can have the store paths referenced by the path.
Additionally, we can have an optional :self label to denote self
reference.
"output:<id>"
for either the outputs created by derivations, OR paths copied
to the store using addToStore() with recursive != true or
@ -117,6 +120,12 @@ StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view p
the contents of the path (or expected contents of the
path for fixed-output derivations)
Note that since an output derivation has always type output, while
something added by addToStore can have type output or source depending
on the hash, this means that the same input can be hashed differently
if added to the store via addToStore or via a derivation, in the sha256
recursive case.
It would have been nicer to handle fixed-output derivations under
"source", e.g. have something like "source:<rec><algo>", but we're
stuck with this for now...
@ -164,20 +173,20 @@ static std::string makeType(
StorePath Store::makeFixedOutputPath(
FileIngestionMethod recursive,
FileIngestionMethod method,
const Hash & hash,
std::string_view name,
const StorePathSet & references,
bool hasSelfReference) const
{
if (hash.type == htSHA256 && recursive == FileIngestionMethod::Recursive) {
if (hash.type == htSHA256 && method == FileIngestionMethod::Recursive) {
return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name);
} else {
assert(references.empty());
return makeStorePath("output:out",
hashString(htSHA256,
"fixed:out:"
+ (recursive == FileIngestionMethod::Recursive ? (string) "r:" : "")
+ makeFileIngestionPrefix(method)
+ hash.to_string(Base16, true) + ":"),
name);
}
@ -813,10 +822,21 @@ Strings ValidPathInfo::shortRefs() const
}
std::string makeFixedOutputCA(FileIngestionMethod recursive, const Hash & hash)
std::string makeFileIngestionPrefix(const FileIngestionMethod m) {
switch (m) {
case FileIngestionMethod::Flat:
return "";
case FileIngestionMethod::Recursive:
return "r:";
default:
throw Error("impossible, caught both cases");
}
}
std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash)
{
return "fixed:"
+ (recursive == FileIngestionMethod::Recursive ? (std::string) "r:" : "")
+ makeFileIngestionPrefix(method)
+ hash.to_string(Base32, true);
}

8
src/libstore/store-api.hh
View file

@ -189,9 +189,10 @@ struct ValidPathInfo
Strings shortRefs() const;
ValidPathInfo(const StorePath & path) : path(path) { }
ValidPathInfo(const ValidPathInfo & other) = default;
ValidPathInfo(StorePath && path) : path(std::move(path)) { }
ValidPathInfo(StorePath && path) : path(std::move(path)) { };
ValidPathInfo(const StorePath & path) : path(path) { };
virtual ~ValidPathInfo() { }
};
@ -838,6 +839,9 @@ std::optional<ValidPathInfo> decodeValidPathInfo(
std::istream & str,
bool hashGiven = false);
/* Compute the prefix to the hash algorithm which indicates how the files were
ingested. */
std::string makeFileIngestionPrefix(const FileIngestionMethod m);
/* Compute the content-addressability assertion (ValidPathInfo::ca)
for paths created by makeFixedOutputPath() / addToStore(). */

2
src/libutil/ansicolor.hh
View file

@ -11,5 +11,7 @@ namespace nix {
#define ANSI_GREEN "\e[32;1m"
#define ANSI_YELLOW "\e[33;1m"
#define ANSI_BLUE "\e[34;1m"
#define ANSI_MAGENTA "\e[35m;1m"
#define ANSI_CYAN "\e[36m;1m"
}

14
src/libutil/args.cc
View file

@ -162,8 +162,18 @@ Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashType * ht)
.labels = {"hash-algo"},
.handler = {[ht](std::string s) {
*ht = parseHashType(s);
if (*ht == htUnknown)
throw UsageError("unknown hash type '%1%'", s);
}}
};
}
Args::Flag Args::Flag::mkHashTypeOptFlag(std::string && longName, std::optional<HashType> * oht)
{
return Flag {
.longName = std::move(longName),
.description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512'). Optional as can also be gotten from SRI hash itself.",
.labels = {"hash-algo"},
.handler = {[oht](std::string s) {
*oht = std::optional<HashType> { parseHashType(s) };
}}
};
}

1
src/libutil/args.hh
View file

@ -85,6 +85,7 @@ protected:
Handler handler;
static Flag mkHashTypeFlag(std::string && longName, HashType * ht);
static Flag mkHashTypeOptFlag(std::string && longName, std::optional<HashType> * oht);
};
std::map<std::string, Flag::ptr> longFlags;

71
src/libutil/hash.cc
View file

@ -4,6 +4,7 @@
#include <openssl/md5.h>
#include <openssl/sha.h>
#include "args.hh"
#include "hash.hh"
#include "archive.hh"
#include "util.hh"
@ -18,11 +19,13 @@ namespace nix {
void Hash::init()
{
if (type == htMD5) hashSize = md5HashSize;
else if (type == htSHA1) hashSize = sha1HashSize;
else if (type == htSHA256) hashSize = sha256HashSize;
else if (type == htSHA512) hashSize = sha512HashSize;
else abort();
if (!type) abort();
switch (*type) {
case htMD5: hashSize = md5HashSize; break;
case htSHA1: hashSize = sha1HashSize; break;
case htSHA256: hashSize = sha256HashSize; break;
case htSHA512: hashSize = sha512HashSize; break;
}
assert(hashSize <= maxHashSize);
memset(hash, 0, maxHashSize);
}
@ -102,11 +105,18 @@ string printHash16or32(const Hash & hash)
}
HashType assertInitHashType(const Hash & h) {
if (h.type)
return *h.type;
else
abort();
}
std::string Hash::to_string(Base base, bool includeType) const
{
std::string s;
if (base == SRI || includeType) {
s += printHashType(type);
s += printHashType(assertInitHashType(*this));
s += base == SRI ? '-' : ':';
}
switch (base) {
@ -124,8 +134,10 @@ std::string Hash::to_string(Base base, bool includeType) const
return s;
}
Hash::Hash(std::string_view s, HashType type) : Hash(s, std::optional { type }) { }
Hash::Hash(std::string_view s) : Hash(s, std::optional<HashType>{}) { }
Hash::Hash(std::string_view s, HashType type)
Hash::Hash(std::string_view s, std::optional<HashType> type)
: type(type)
{
size_t pos = 0;
@ -136,17 +148,17 @@ Hash::Hash(std::string_view s, HashType type)
sep = s.find('-');
if (sep != string::npos) {
isSRI = true;
} else if (type == htUnknown)
} else if (! type)
throw BadHash("hash '%s' does not include a type", s);
}
if (sep != string::npos) {
string hts = string(s, 0, sep);
this->type = parseHashType(hts);
if (this->type == htUnknown)
if (!this->type)
throw BadHash("unknown hash type '%s'", hts);
if (type != htUnknown && type != this->type)
throw BadHash("hash '%s' should have type '%s'", s, printHashType(type));
if (type && type != this->type)
throw BadHash("hash '%s' should have type '%s'", s, printHashType(*type));
pos = sep + 1;
}
@ -202,14 +214,16 @@ Hash::Hash(std::string_view s, HashType type)
}
else
throw BadHash("hash '%s' has wrong length for hash type '%s'", s, printHashType(type));
throw BadHash("hash '%s' has wrong length for hash type '%s'", s, printHashType(*type));
}
Hash newHashAllowEmpty(std::string hashStr, HashType ht)
Hash newHashAllowEmpty(std::string hashStr, std::optional<HashType> ht)
{
if (hashStr.empty()) {
Hash h(ht);
warn("found empty hash, assuming '%s'", h.to_string(SRI, true));
if (!ht)
throw BadHash("empty hash requires explicit hash type");
Hash h(*ht);
warn("found empty hash, assuming '%s'", h.to_string(Base::SRI, true));
return h;
} else
return Hash(hashStr, ht);
@ -328,24 +342,35 @@ Hash compressHash(const Hash & hash, unsigned int newSize)
}
HashType parseHashType(const string & s)
std::optional<HashType> parseHashTypeOpt(const string & s)
{
if (s == "md5") return htMD5;
else if (s == "sha1") return htSHA1;
else if (s == "sha256") return htSHA256;
else if (s == "sha512") return htSHA512;
else return htUnknown;
else return std::optional<HashType> {};
}
HashType parseHashType(const string & s)
{
auto opt_h = parseHashTypeOpt(s);
if (opt_h)
return *opt_h;
else
throw UsageError("unknown hash algorithm '%1%'", s);
}
string printHashType(HashType ht)
{
if (ht == htMD5) return "md5";
else if (ht == htSHA1) return "sha1";
else if (ht == htSHA256) return "sha256";
else if (ht == htSHA512) return "sha512";
else abort();
switch (ht) {
case htMD5: return "md5"; break;
case htSHA1: return "sha1"; break;
case htSHA256: return "sha256"; break;
case htSHA512: return "sha512"; break;
}
// illegal hash type enum value internally, as opposed to external input
// which should be validated with nice error message.
abort();
}
}

18
src/libutil/hash.hh
View file

@ -10,7 +10,7 @@ namespace nix {
MakeError(BadHash, Error);
enum HashType : char { htUnknown, htMD5, htSHA1, htSHA256, htSHA512 };
enum HashType : char { htMD5, htSHA1, htSHA256, htSHA512 };
const int md5HashSize = 16;
@ -29,7 +29,7 @@ struct Hash
unsigned int hashSize = 0;
unsigned char hash[maxHashSize] = {};
HashType type = htUnknown;
std::optional<HashType> type = {};
/* Create an unset hash object. */
Hash() { };
@ -40,14 +40,18 @@ struct Hash
/* Initialize the hash from a string representation, in the format
"[<type>:]<base16|base32|base64>" or "<type>-<base64>" (a
Subresource Integrity hash expression). If the 'type' argument
is htUnknown, then the hash type must be specified in the
is not present, then the hash type must be specified in the
string. */
Hash(std::string_view s, HashType type = htUnknown);
Hash(std::string_view s, std::optional<HashType> type);
// type must be provided
Hash(std::string_view s, HashType type);
// hash type must be part of string
Hash(std::string_view s);
void init();
/* Check whether a hash is set. */
operator bool () const { return type != htUnknown; }
operator bool () const { return (bool) type; }
/* Check whether two hash are equal. */
bool operator == (const Hash & h2) const;
@ -95,7 +99,7 @@ struct Hash
};
/* Helper that defaults empty hashes to the 0 hash. */
Hash newHashAllowEmpty(std::string hashStr, HashType ht);
Hash newHashAllowEmpty(std::string hashStr, std::optional<HashType> ht);
/* Print a hash in base-16 if it's MD5, or base-32 otherwise. */
string printHash16or32(const Hash & hash);
@ -118,6 +122,8 @@ Hash compressHash(const Hash & hash, unsigned int newSize);
/* Parse a string representing a hash type. */
HashType parseHashType(const string & s);
/* Will return nothing on parse error */
std::optional<HashType> parseHashTypeOpt(const string & s);
/* And the reverse. */
string printHashType(HashType ht);

2
src/libutil/json.cc
View file

@ -173,7 +173,7 @@ JSONObject JSONPlaceholder::object()
JSONPlaceholder::~JSONPlaceholder()
{
assert(!first || std::uncaught_exception());
assert(!first || std::uncaught_exceptions());
}
}

78
src/libutil/tests/compression.cc Normal file
View file

@ -0,0 +1,78 @@
#include "compression.hh"
#include <gtest/gtest.h>
namespace nix {
/* ----------------------------------------------------------------------------
* compress / decompress
* --------------------------------------------------------------------------*/
TEST(compress, compressWithUnknownMethod) {
ASSERT_THROW(compress("invalid-method", "something-to-compress"), UnknownCompressionMethod);
}
TEST(compress, noneMethodDoesNothingToTheInput) {
ref<std::string> o = compress("none", "this-is-a-test");
ASSERT_EQ(*o, "this-is-a-test");
}
TEST(decompress, decompressXzCompressed) {
auto method = "xz";
auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf";
ref<std::string> o = decompress(method, *compress(method, str));
ASSERT_EQ(*o, str);
}
TEST(decompress, decompressBzip2Compressed) {
auto method = "bzip2";
auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf";
ref<std::string> o = decompress(method, *compress(method, str));
ASSERT_EQ(*o, str);
}
TEST(decompress, decompressBrCompressed) {
auto method = "br";
auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf";
ref<std::string> o = decompress(method, *compress(method, str));
ASSERT_EQ(*o, str);
}
TEST(decompress, decompressInvalidInputThrowsCompressionError) {
auto method = "bzip2";
auto str = "this is a string that does not qualify as valid bzip2 data";
ASSERT_THROW(decompress(method, str), CompressionError);
}
/* ----------------------------------------------------------------------------
* compression sinks
* --------------------------------------------------------------------------*/
TEST(makeCompressionSink, noneSinkDoesNothingToInput) {
StringSink strSink;
auto inputString = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf";
auto sink = makeCompressionSink("none", strSink);
(*sink)(inputString);
sink->finish();
ASSERT_STREQ((*strSink.s).c_str(), inputString);
}
TEST(makeCompressionSink, compressAndDecompress) {
StringSink strSink;
auto inputString = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf";
auto decompressionSink = makeDecompressionSink("bzip2", strSink);
auto sink = makeCompressionSink("bzip2", *decompressionSink);
(*sink)(inputString);
sink->finish();
decompressionSink->finish();
ASSERT_STREQ((*strSink.s).c_str(), inputString);
}
}

5
src/libutil/tests/hash.cc
View file

@ -72,9 +72,4 @@ namespace nix {
"7299aeadb6889018501d289e4900f7e4331b99dec4b5433a"
"c7d329eeb6dd26545e96e55b874be909");
}
TEST(hashString, hashingWithUnknownAlgoExits) {
auto s = "unknown";
ASSERT_DEATH(hashString(HashType::htUnknown, s), "");
}
}

4
src/libutil/util.cc
View file

@ -35,7 +35,7 @@
#endif
extern char * * environ;
extern char * * environ __attribute__((weak));
namespace nix {
@ -1199,7 +1199,7 @@ void _interrupted()
/* Block user interrupts while an exception is being handled.
Throwing an exception while another exception is being handled
kills the program! */
if (!interruptThrown && !std::uncaught_exception()) {
if (!interruptThrown && !std::uncaught_exceptions()) {
interruptThrown = true;
throw Interrupted("interrupted by the user");
}

2
src/nix-build/nix-build.cc
View file

@ -21,7 +21,7 @@
using namespace nix;
using namespace std::string_literals;
extern char * * environ;
extern char * * environ __attribute__((weak));
/* Recreate the effect of the perl shellwords function, breaking up a
* string into arguments like a shell word, including escapes

2
src/nix-prefetch-url/nix-prefetch-url.cc
View file

@ -72,8 +72,6 @@ static int _main(int argc, char * * argv)
else if (*arg == "--type") {
string s = getArg(*arg, arg, end);
ht = parseHashType(s);
if (ht == htUnknown)
throw UsageError("unknown hash type '%1%'", s);
}
else if (*arg == "--print-path")
printPath = true;

2
src/nix-store/nix-store.cc
View file

@ -725,7 +725,7 @@ static void opVerifyPath(Strings opFlags, Strings opArgs)
auto path = store->followLinksToStorePath(i);
printMsg(lvlTalkative, "checking path '%s'...", store->printStorePath(path));
auto info = store->queryPathInfo(path);
HashSink sink(info->narHash.type);
HashSink sink(*info->narHash.type);
store->narFromPath(path, sink);
auto current = sink.finish();
if (current.first != info->narHash) {

2
src/nix/command.cc
View file

@ -4,7 +4,7 @@
#include "nixexpr.hh"
#include "profiles.hh"
extern char * * environ;
extern char * * environ __attribute__((weak));
namespace nix {

8
src/nix/develop.cc
View file

@ -135,7 +135,13 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
drv.inputSrcs.insert(std::move(getEnvShPath));
Hash h = std::get<0>(hashDerivationModulo(*store, drv, true));
auto shellOutPath = store->makeOutputPath("out", h, drvName);
drv.outputs.insert_or_assign("out", DerivationOutput { shellOutPath, "", "" });
drv.outputs.insert_or_assign("out", DerivationOutput {
.path = shellOutPath,
.hash = DerivationOutputHash {
.method = FileIngestionMethod::Flat,
.hash = Hash { },
},
});
drv.env["out"] = store->printStorePath(shellOutPath);
auto shellDrvPath2 = writeDerivation(store, drv, drvName);

6
src/nix/hash.cc
View file

@ -79,12 +79,12 @@ static RegisterCommand r2("hash-path", [](){ return make_ref<CmdHash>(FileIngest
struct CmdToBase : Command
{
Base base;
HashType ht = htUnknown;
std::optional<HashType> ht;
std::vector<std::string> args;
CmdToBase(Base base) : base(base)
{
addFlag(Flag::mkHashTypeFlag("type", &ht));
addFlag(Flag::mkHashTypeOptFlag("type", &ht));
expectArgs("strings", &args);
}
@ -132,8 +132,6 @@ static int compatNixHash(int argc, char * * argv)
else if (*arg == "--type") {
string s = getArg(*arg, arg, end);
ht = parseHashType(s);
if (ht == htUnknown)
throw UsageError("unknown hash type '%1%'", s);
}
else if (*arg == "--to-base16") op = opTo16;
else if (*arg == "--to-base32") op = opTo32;

33
src/nix/repl.cc
View file

@ -19,6 +19,7 @@ extern "C" {
}
#endif
#include "ansicolor.hh"
#include "shared.hh"
#include "eval.hh"
#include "eval-inline.hh"
@ -37,14 +38,6 @@ extern "C" {
namespace nix {
#define ESC_RED "\033[31m"
#define ESC_GRE "\033[32m"
#define ESC_YEL "\033[33m"
#define ESC_BLU "\033[34;1m"
#define ESC_MAG "\033[35m"
#define ESC_CYA "\033[36m"
#define ESC_END "\033[0m"
struct NixRepl : gc
{
string curDir;
@ -645,25 +638,25 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
switch (v.type) {
case tInt:
str << ESC_CYA << v.integer << ESC_END;
str << ANSI_CYAN << v.integer << ANSI_NORMAL;
break;
case tBool:
str << ESC_CYA << (v.boolean ? "true" : "false") << ESC_END;
str << ANSI_CYAN << (v.boolean ? "true" : "false") << ANSI_NORMAL;
break;
case tString:
str << ESC_YEL;
str << ANSI_YELLOW;
printStringValue(str, v.string.s);
str << ESC_END;
str << ANSI_NORMAL;
break;
case tPath:
str << ESC_GRE << v.path << ESC_END; // !!! escaping?
str << ANSI_GREEN << v.path << ANSI_NORMAL; // !!! escaping?
break;
case tNull:
str << ESC_CYA "null" ESC_END;
str << ANSI_CYAN "null" ANSI_NORMAL;
break;
case tAttrs: {
@ -699,7 +692,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
try {
printValue(str, *i.second, maxDepth - 1, seen);
} catch (AssertionError & e) {
str << ESC_RED "«error: " << e.msg() << "»" ESC_END;
str << ANSI_RED "«error: " << e.msg() << "»" ANSI_NORMAL;
}
str << "; ";
}
@ -725,7 +718,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
try {
printValue(str, *v.listElems()[n], maxDepth - 1, seen);
} catch (AssertionError & e) {
str << ESC_RED "«error: " << e.msg() << "»" ESC_END;
str << ANSI_RED "«error: " << e.msg() << "»" ANSI_NORMAL;
}
str << " ";
}
@ -737,16 +730,16 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
case tLambda: {
std::ostringstream s;
s << v.lambda.fun->pos;
str << ESC_BLU "«lambda @ " << filterANSIEscapes(s.str()) << "»" ESC_END;
str << ANSI_BLUE "«lambda @ " << filterANSIEscapes(s.str()) << "»" ANSI_NORMAL;
break;
}
case tPrimOp:
str << ESC_MAG "«primop»" ESC_END;
str << ANSI_MAGENTA "«primop»" ANSI_NORMAL;
break;
case tPrimOpApp:
str << ESC_BLU "«primop-app»" ESC_END;
str << ANSI_BLUE "«primop-app»" ANSI_NORMAL;
break;
case tFloat:
@ -754,7 +747,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
break;
default:
str << ESC_RED "«unknown»" ESC_END;
str << ANSI_RED "«unknown»" ANSI_NORMAL;
break;
}

6
src/nix/show-derivation.cc
View file

@ -70,9 +70,9 @@ struct CmdShowDerivation : InstallablesCommand
for (auto & output : drv.outputs) {
auto outputObj(outputsObj.object(output.first));
outputObj.attr("path", store->printStorePath(output.second.path));
if (output.second.hash != "") {
outputObj.attr("hashAlgo", output.second.hashAlgo);
outputObj.attr("hash", output.second.hash);
if (output.second.hash) {
outputObj.attr("hashAlgo", output.second.hash->printMethodAlgo());
outputObj.attr("hash", output.second.hash->hash.to_string(Base16, false));
}
}
}

4
src/nix/verify.cc
View file

@ -88,9 +88,9 @@ struct CmdVerify : StorePathsCommand
std::unique_ptr<AbstractHashSink> hashSink;
if (info->ca == "")
hashSink = std::make_unique<HashSink>(info->narHash.type);
hashSink = std::make_unique<HashSink>(*info->narHash.type);
else
hashSink = std::make_unique<HashModuloSink>(info->narHash.type, std::string(info->path.hashPart()));
hashSink = std::make_unique<HashModuloSink>(*info->narHash.type, std::string(info->path.hashPart()));
store->narFromPath(info->path, *hashSink);