diff --git a/doc/manual/advanced-topics/diff-hook.xml b/doc/manual/advanced-topics/diff-hook.xml index fb4bf819f..f01ab71b3 100644 --- a/doc/manual/advanced-topics/diff-hook.xml +++ b/doc/manual/advanced-topics/diff-hook.xml @@ -70,7 +70,7 @@ path just built. $ nix-build ./deterministic.nix -A stable -these derivations will be built: +this derivation will be built: /nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv building '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'... /nix/store/yyxlzw3vqaas7wfp04g0b1xg51f2czgq-stable @@ -85,7 +85,7 @@ checking outputs of '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'... $ nix-build ./deterministic.nix -A unstable -these derivations will be built: +this derivation will be built: /nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv building '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'... /nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable @@ -193,7 +193,7 @@ repeat = 1 An example output of this configuration: $ nix-build ./test.nix -A unstable -these derivations will be built: +this derivation will be built: /nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv building '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' (round 1/2)... building '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' (round 2/2)... diff --git a/doc/manual/advanced-topics/post-build-hook.xml b/doc/manual/advanced-topics/post-build-hook.xml index acfe9e3cc..6cc286ee1 100644 --- a/doc/manual/advanced-topics/post-build-hook.xml +++ b/doc/manual/advanced-topics/post-build-hook.xml @@ -122,7 +122,7 @@ post-build-hook = /etc/nix/upload-to-cache.sh $ nix-build -E '(import <nixpkgs> {}).writeText "example" (builtins.toString builtins.currentTime)' -these derivations will be built: +this derivation will be built: /nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv building '/nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv'... running post-build-hook '/home/grahamc/projects/github.com/NixOS/nix/post-hook.sh'... diff --git a/doc/manual/command-ref/nix-env.xml b/doc/manual/command-ref/nix-env.xml index 2b95b6819..55f25d959 100644 --- a/doc/manual/command-ref/nix-env.xml +++ b/doc/manual/command-ref/nix-env.xml @@ -516,7 +516,7 @@ source: $ nix-env -f '<nixpkgs>' -iA hello --dry-run (dry run; not doing anything) installing ‘hello-2.10’ -these paths will be fetched (0.04 MiB download, 0.19 MiB unpacked): +this path will be fetched (0.04 MiB download, 0.19 MiB unpacked): /nix/store/wkhdf9jinag5750mqlax6z2zbwhqb76n-hello-2.10 ... diff --git a/doc/manual/installation/env-variables.xml b/doc/manual/installation/env-variables.xml index e2b8fc867..cc52f5b4a 100644 --- a/doc/manual/installation/env-variables.xml +++ b/doc/manual/installation/env-variables.xml @@ -39,7 +39,7 @@ bundle. Set the environment variable and install Nix $ export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt -$ sh <(curl https://nixos.org/nix/install) +$ sh <(curl -L https://nixos.org/nix/install) In the shell profile and rc files (for example, diff --git a/doc/manual/installation/installing-binary.xml b/doc/manual/installation/installing-binary.xml index 8d548f0ea..d25c46b85 100644 --- a/doc/manual/installation/installing-binary.xml +++ b/doc/manual/installation/installing-binary.xml @@ -12,7 +12,7 @@ - $ sh <(curl https://nixos.org/nix/install) + $ sh <(curl -L https://nixos.org/nix/install) @@ -39,7 +39,7 @@ To explicitly select a single-user installation on your system: - sh <(curl https://nixos.org/nix/install) --no-daemon + sh <(curl -L https://nixos.org/nix/install) --no-daemon diff --git a/doc/manual/introduction/quick-start.xml b/doc/manual/introduction/quick-start.xml index 1ce6c8d50..1992c14ed 100644 --- a/doc/manual/introduction/quick-start.xml +++ b/doc/manual/introduction/quick-start.xml @@ -15,7 +15,7 @@ to subsequent chapters. Install single-user Nix by running the following: -$ bash <(curl https://nixos.org/nix/install) +$ bash <(curl -L https://nixos.org/nix/install) This will install Nix in /nix. The install script diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index 8980bc09d..2e2a17b14 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -6,11 +6,11 @@ namespace nix { -static Strings parseAttrPath(const string & s) +static Strings parseAttrPath(std::string_view s) { Strings res; string cur; - string::const_iterator i = s.begin(); + auto i = s.begin(); while (i != s.end()) { if (*i == '.') { res.push_back(cur); @@ -32,6 +32,15 @@ static Strings parseAttrPath(const string & s) } +std::vector parseAttrPath(EvalState & state, std::string_view s) +{ + std::vector res; + for (auto & a : parseAttrPath(s)) + res.push_back(state.symbols.create(a)); + return res; +} + + std::pair findAlongAttrPath(EvalState & state, const string & attrPath, Bindings & autoArgs, Value & vIn) { diff --git a/src/libexpr/attr-path.hh b/src/libexpr/attr-path.hh index fce160da7..d9d74ab2d 100644 --- a/src/libexpr/attr-path.hh +++ b/src/libexpr/attr-path.hh @@ -16,4 +16,6 @@ std::pair findAlongAttrPath(EvalState & state, const string & attr /* Heuristic to find the filename and lineno or a nix value. */ Pos findDerivationFilename(EvalState & state, Value & v, std::string what); +std::vector parseAttrPath(EvalState & state, std::string_view s); + } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 8e71db2b8..b90a64357 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -161,12 +161,12 @@ const Value *getPrimOp(const Value &v) { } -string showType(const Value & v) +string showType(ValueType type) { - switch (v.type) { + switch (type) { case tInt: return "an integer"; - case tBool: return "a boolean"; - case tString: return v.string.context ? "a string with context" : "a string"; + case tBool: return "a Boolean"; + case tString: return "a string"; case tPath: return "a path"; case tNull: return "null"; case tAttrs: return "a set"; @@ -175,14 +175,27 @@ string showType(const Value & v) case tApp: return "a function application"; case tLambda: return "a function"; case tBlackhole: return "a black hole"; + case tPrimOp: return "a built-in function"; + case tPrimOpApp: return "a partially applied built-in function"; + case tExternal: return "an external value"; + case tFloat: return "a float"; + } + abort(); +} + + +string showType(const Value & v) +{ + switch (v.type) { + case tString: return v.string.context ? "a string with context" : "a string"; case tPrimOp: return fmt("the built-in function '%s'", string(v.primOp->name)); case tPrimOpApp: return fmt("the partially applied built-in function '%s'", string(getPrimOp(v)->primOp->name)); case tExternal: return v.external->showType(); - case tFloat: return "a float"; + default: + return showType(v.type); } - abort(); } @@ -323,6 +336,7 @@ EvalState::EvalState(const Strings & _searchPath, ref store) , sOutputHash(symbols.create("outputHash")) , sOutputHashAlgo(symbols.create("outputHashAlgo")) , sOutputHashMode(symbols.create("outputHashMode")) + , sRecurseForDerivations(symbols.create("recurseForDerivations")) , repair(NoRepair) , store(store) , baseEnv(allocEnv(128)) @@ -471,14 +485,21 @@ Value * EvalState::addConstant(const string & name, Value & v) Value * EvalState::addPrimOp(const string & name, size_t arity, PrimOpFun primOp) { + auto name2 = string(name, 0, 2) == "__" ? string(name, 2) : name; + Symbol sym = symbols.create(name2); + + /* Hack to make constants lazy: turn them into a application of + the primop to a dummy value. */ if (arity == 0) { + auto vPrimOp = allocValue(); + vPrimOp->type = tPrimOp; + vPrimOp->primOp = new PrimOp(primOp, 1, sym); Value v; - primOp(*this, noPos, nullptr, v); + mkApp(v, *vPrimOp, *vPrimOp); return addConstant(name, v); } + Value * v = allocValue(); - string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name; - Symbol sym = symbols.create(name2); v->type = tPrimOp; v->primOp = new PrimOp(primOp, arity, sym); staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl; diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 1485dc7fe..863365259 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -18,7 +18,7 @@ namespace nix { class Store; class EvalState; -struct StorePath; +class StorePath; enum RepairFlag : bool; @@ -74,7 +74,8 @@ public: sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls, sFile, sLine, sColumn, sFunctor, sToString, sRight, sWrong, sStructuredAttrs, sBuilder, sArgs, - sOutputHash, sOutputHashAlgo, sOutputHashMode; + sOutputHash, sOutputHashAlgo, sOutputHashMode, + sRecurseForDerivations; Symbol sDerivationNix; /* If set, force copying files to the Nix store even if they @@ -324,6 +325,7 @@ private: /* Return a string representing the type of the value `v'. */ +string showType(ValueType type); string showType(const Value & v); /* Decode a context string ‘!!’ into a pair Only if it has a `recurseForDerivations = true' attribute. */ if (i->value->type == tAttrs) { - Bindings::iterator j = i->value->attrs->find(state.symbols.create("recurseForDerivations")); + Bindings::iterator j = i->value->attrs->find(state.sRecurseForDerivations); if (j != i->value->attrs->end() && state.forceBool(*j->value, *j->pos)) getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 907f15246..fe5358962 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -50,20 +50,20 @@ void EvalState::realiseContext(const PathSet & context) std::vector drvs; for (auto & i : context) { - std::pair decoded = decodeContext(i); - auto ctx = store->parseStorePath(decoded.first); + auto [ctxS, outputName] = decodeContext(i); + auto ctx = store->parseStorePath(ctxS); if (!store->isValidPath(ctx)) throw InvalidPathError(store->printStorePath(ctx)); - if (!decoded.second.empty() && ctx.isDerivation()) { - drvs.push_back(StorePathWithOutputs{ctx, {decoded.second}}); + if (!outputName.empty() && ctx.isDerivation()) { + drvs.push_back(StorePathWithOutputs{ctx, {outputName}}); /* Add the output of this derivation to the allowed paths. */ if (allowedPaths) { - auto drv = store->derivationFromPath(store->parseStorePath(decoded.first)); - DerivationOutputs::iterator i = drv.outputs.find(decoded.second); + auto drv = store->derivationFromPath(ctx); + DerivationOutputs::iterator i = drv.outputs.find(outputName); if (i == drv.outputs.end()) - throw Error("derivation '%s' does not have an output named '%s'", decoded.first, decoded.second); + throw Error("derivation '%s' does not have an output named '%s'", ctxS, outputName); allowedPaths->insert(store->printStorePath(i->second.path)); } } @@ -79,6 +79,7 @@ void EvalState::realiseContext(const PathSet & context) StorePathSet willBuild, willSubstitute, unknown; unsigned long long downloadSize, narSize; store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize); + store->buildPaths(drvs); } @@ -775,10 +776,11 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName); if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign("out", DerivationOutput { - std::move(outPath), - (ingestionMethod == FileIngestionMethod::Recursive ? "r:" : "") - + printHashType(h.type), - h.to_string(Base16, false), + .path = std::move(outPath), + .hash = DerivationOutputHash { + .method = ingestionMethod, + .hash = std::move(h), + }, }); } @@ -792,7 +794,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * for (auto & i : outputs) { if (!jsonObject) drv.env[i] = ""; drv.outputs.insert_or_assign(i, - DerivationOutput { StorePath::dummy, "", "" }); + DerivationOutput { + .path = StorePath::dummy, + .hash = std::optional {}, + }); } Hash h = hashDerivationModulo(*state.store, Derivation(drv), true); @@ -801,7 +806,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * auto outPath = state.store->makeOutputPath(i, h, drvName); if (!jsonObject) drv.env[i] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign(i, - DerivationOutput { std::move(outPath), "", "" }); + DerivationOutput { + .path = std::move(outPath), + .hash = std::optional(), + }); } } @@ -2201,10 +2209,11 @@ static void prim_splitVersion(EvalState & state, const Pos & pos, Value * * args RegisterPrimOp::PrimOps * RegisterPrimOp::primOps; -RegisterPrimOp::RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun) +RegisterPrimOp::RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun, + std::optional requiredFeature) { if (!primOps) primOps = new PrimOps; - primOps->emplace_back(name, arity, fun); + primOps->push_back({name, arity, fun, requiredFeature}); } @@ -2396,7 +2405,8 @@ void EvalState::createBaseEnv() if (RegisterPrimOp::primOps) for (auto & primOp : *RegisterPrimOp::primOps) - addPrimOp(std::get<0>(primOp), std::get<1>(primOp), std::get<2>(primOp)); + if (!primOp.requiredFeature || settings.isExperimentalFeatureEnabled(*primOp.requiredFeature)) + addPrimOp(primOp.name, primOp.arity, primOp.primOp); /* Now that we've added all primops, sort the `builtins' set, because attribute lookups expect it to be sorted. */ diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh index 05d0792ef..75c460ecf 100644 --- a/src/libexpr/primops.hh +++ b/src/libexpr/primops.hh @@ -7,12 +7,25 @@ namespace nix { struct RegisterPrimOp { - typedef std::vector> PrimOps; + struct Info + { + std::string name; + size_t arity; + PrimOpFun primOp; + std::optional requiredFeature; + }; + + typedef std::vector PrimOps; static PrimOps * primOps; + /* You can register a constant by passing an arity of 0. fun will get called during EvalState initialization, so there may be primops not yet added and builtins is not yet sorted. */ - RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun); + RegisterPrimOp( + std::string name, + size_t arity, + PrimOpFun fun, + std::optional requiredFeature = {}); }; /* These primops are disabled without enableNativeCode, but plugins diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index dc6d5e413..1cb422967 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -48,7 +48,10 @@ void printMissing(ref store, const StorePathSet & willBuild, unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl) { if (!willBuild.empty()) { - printMsg(lvl, "these derivations will be built:"); + if (willBuild.size() == 1) + printMsg(lvl, fmt("this derivation will be built:")); + else + printMsg(lvl, fmt("these %d derivations will be built:", willBuild.size())); auto sorted = store->topoSortPaths(willBuild); reverse(sorted.begin(), sorted.end()); for (auto & i : sorted) @@ -56,9 +59,18 @@ void printMissing(ref store, const StorePathSet & willBuild, } if (!willSubstitute.empty()) { - printMsg(lvl, fmt("these paths will be fetched (%.2f MiB download, %.2f MiB unpacked):", - downloadSize / (1024.0 * 1024.0), - narSize / (1024.0 * 1024.0))); + const float downloadSizeMiB = downloadSize / (1024.f * 1024.f); + const float narSizeMiB = narSize / (1024.f * 1024.f); + if (willSubstitute.size() == 1) { + printMsg(lvl, fmt("this path will be fetched (%.2f MiB download, %.2f MiB unpacked):", + downloadSizeMiB, + narSizeMiB)); + } else { + printMsg(lvl, fmt("these %d paths will be fetched (%.2f MiB download, %.2f MiB unpacked):", + willSubstitute.size(), + downloadSizeMiB, + narSizeMiB)); + } for (auto & i : willSubstitute) printMsg(lvl, fmt(" %s", store->printStorePath(i))); } diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index f8eff508c..9f52ddafa 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -388,8 +388,6 @@ void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSe narInfo->sigs.insert(sigs.begin(), sigs.end()); - auto narInfoFile = narInfoFileFor(narInfo->path); - writeNarInfo(narInfo); } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 63cb70ef2..bd2b38833 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -86,7 +86,7 @@ struct HookInstance; /* A pointer to a goal. */ -class Goal; +struct Goal; class DerivationGoal; typedef std::shared_ptr GoalPtr; typedef std::weak_ptr WeakGoalPtr; @@ -1196,8 +1196,8 @@ void DerivationGoal::haveDerivation() parsedDrv = std::make_unique(drvPath, *drv); if (parsedDrv->contentAddressed()) { - settings.requireExperimentalFeature("content-addressed-paths"); - throw Error("content-addressed-paths isn't implemented yet"); + settings.requireExperimentalFeature("ca-derivations"); + throw Error("ca-derivations isn't implemented yet"); } @@ -3718,10 +3718,7 @@ void DerivationGoal::registerOutputs() if (fixedOutput) { - FileIngestionMethod outputHashMode; Hash h; - i.second.parseHashInfo(outputHashMode, h); - - if (outputHashMode == FileIngestionMethod::Flat) { + if (i.second.hash->method == FileIngestionMethod::Flat) { /* The output path should be a regular file without execute permission. */ if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) throw BuildError( @@ -3732,20 +3729,22 @@ void DerivationGoal::registerOutputs() /* Check the hash. In hash mode, move the path produced by the derivation to its content-addressed location. */ - Hash h2 = outputHashMode == FileIngestionMethod::Recursive - ? hashPath(h.type, actualPath).first - : hashFile(h.type, actualPath); + Hash h2 = i.second.hash->method == FileIngestionMethod::Recursive + ? hashPath(i.second.hash->hash.type, actualPath).first + : hashFile(i.second.hash->hash.type, actualPath); - auto dest = worker.store.makeFixedOutputPath(outputHashMode, h2, i.second.path.name()); + auto dest = worker.store.makeFixedOutputPath(i.second.hash->method, h2, i.second.path.name()); - if (h != h2) { + if (i.second.hash->hash != h2) { /* Throw an error after registering the path as valid. */ worker.hashMismatch = true; delayedException = std::make_exception_ptr( BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", - worker.store.printStorePath(dest), h.to_string(SRI, true), h2.to_string(SRI, true))); + worker.store.printStorePath(dest), + i.second.hash->hash.to_string(SRI, true), + h2.to_string(SRI, true))); Path actualDest = worker.store.Store::toRealPath(dest); @@ -3765,7 +3764,7 @@ void DerivationGoal::registerOutputs() else assert(worker.store.parseStorePath(path) == dest); - ca = makeFixedOutputCA(outputHashMode, h2); + ca = makeFixedOutputCA(i.second.hash->method, h2); } /* Get rid of all weird permissions. This also checks that diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index b95f7bfdc..528b7ccea 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -8,22 +8,8 @@ namespace nix { - -void DerivationOutput::parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const -{ - recursive = FileIngestionMethod::Flat; - string algo = hashAlgo; - - if (string(algo, 0, 2) == "r:") { - recursive = FileIngestionMethod::Recursive; - algo = string(algo, 2); - } - - HashType hashType = parseHashType(algo); - if (hashType == htUnknown) - throw Error("unknown hash algorithm '%s'", algo); - - hash = Hash(this->hash, hashType); +std::string DerivationOutputHash::printMethodAlgo() const { + return makeFileIngestionPrefix(method) + printHashType(hash.type); } @@ -120,6 +106,36 @@ static StringSet parseStrings(std::istream & str, bool arePaths) } +static DerivationOutput parseDerivationOutput(const Store & store, istringstream_nocopy & str) +{ + expect(str, ","); auto path = store.parseStorePath(parsePath(str)); + expect(str, ","); auto hashAlgo = parseString(str); + expect(str, ","); const auto hash = parseString(str); + expect(str, ")"); + + std::optional fsh; + if (hashAlgo != "") { + auto method = FileIngestionMethod::Flat; + if (string(hashAlgo, 0, 2) == "r:") { + method = FileIngestionMethod::Recursive; + hashAlgo = string(hashAlgo, 2); + } + const HashType hashType = parseHashType(hashAlgo); + if (hashType == htUnknown) + throw Error("unknown hash hashAlgorithm '%s'", hashAlgo); + fsh = DerivationOutputHash { + .method = std::move(method), + .hash = Hash(hash, hashType), + }; + } + + return DerivationOutput { + .path = std::move(path), + .hash = std::move(fsh), + }; +} + + static Derivation parseDerivation(const Store & store, const string & s) { Derivation drv; @@ -129,15 +145,8 @@ static Derivation parseDerivation(const Store & store, const string & s) /* Parse the list of outputs. */ while (!endOfList(str)) { expect(str, "("); std::string id = parseString(str); - expect(str, ","); auto path = store.parseStorePath(parsePath(str)); - expect(str, ","); auto hashAlgo = parseString(str); - expect(str, ","); auto hash = parseString(str); - expect(str, ")"); - drv.outputs.emplace(id, DerivationOutput { - .path = std::move(path), - .hashAlgo = std::move(hashAlgo), - .hash = std::move(hash) - }); + auto output = parseDerivationOutput(store, str); + drv.outputs.emplace(std::move(id), std::move(output)); } /* Parse the list of input derivations. */ @@ -263,8 +272,9 @@ string Derivation::unparse(const Store & store, bool maskOutputs, if (first) first = false; else s += ','; s += '('; printUnquotedString(s, i.first); s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path)); - s += ','; printUnquotedString(s, i.second.hashAlgo); - s += ','; printUnquotedString(s, i.second.hash); + s += ','; printUnquotedString(s, i.second.hash ? i.second.hash->printMethodAlgo() : ""); + s += ','; printUnquotedString(s, + i.second.hash ? i.second.hash->hash.to_string(Base16, false) : ""); s += ')'; } @@ -320,7 +330,7 @@ bool BasicDerivation::isFixedOutput() const { return outputs.size() == 1 && outputs.begin()->first == "out" && - outputs.begin()->second.hash != ""; + outputs.begin()->second.hash; } @@ -353,8 +363,8 @@ Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutput if (drv.isFixedOutput()) { DerivationOutputs::const_iterator i = drv.outputs.begin(); return hashString(htSHA256, "fixed:out:" - + i->second.hashAlgo + ":" - + i->second.hash + ":" + + i->second.hash->printMethodAlgo() + ":" + + i->second.hash->hash.to_string(Base16, false) + ":" + store.printStorePath(i->second.path)); } @@ -397,6 +407,33 @@ StorePathSet BasicDerivation::outputPaths() const return paths; } +static DerivationOutput readDerivationOutput(Source & in, const Store & store) +{ + auto path = store.parseStorePath(readString(in)); + auto hashAlgo = readString(in); + const auto hash = readString(in); + + std::optional fsh; + if (hashAlgo != "") { + auto method = FileIngestionMethod::Flat; + if (string(hashAlgo, 0, 2) == "r:") { + method = FileIngestionMethod::Recursive; + hashAlgo = string(hashAlgo, 2); + } + const HashType hashType = parseHashType(hashAlgo); + if (hashType == htUnknown) + throw Error("unknown hash hashAlgorithm '%s'", hashAlgo); + fsh = DerivationOutputHash { + .method = std::move(method), + .hash = Hash(hash, hashType), + }; + } + + return DerivationOutput { + .path = std::move(path), + .hash = std::move(fsh), + }; +} StringSet BasicDerivation::outputNames() const { @@ -413,14 +450,8 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv) auto nr = readNum(in); for (size_t n = 0; n < nr; n++) { auto name = readString(in); - auto path = store.parseStorePath(readString(in)); - auto hashAlgo = readString(in); - auto hash = readString(in); - drv.outputs.emplace(name, DerivationOutput { - .path = std::move(path), - .hashAlgo = std::move(hashAlgo), - .hash = std::move(hash) - }); + auto output = readDerivationOutput(in, store); + drv.outputs.emplace(std::move(name), std::move(output)); } drv.inputSrcs = readStorePaths(store, in); @@ -442,7 +473,10 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr { out << drv.outputs.size(); for (auto & i : drv.outputs) - out << i.first << store.printStorePath(i.second.path) << i.second.hashAlgo << i.second.hash; + out << i.first + << store.printStorePath(i.second.path) + << i.second.hash->printMethodAlgo() + << i.second.hash->hash.to_string(Base16, false); writeStorePaths(store, out, drv.inputSrcs); out << drv.platform << drv.builder << drv.args; out << drv.env.size(); diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index d349c6d4d..7b677ca49 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -12,11 +12,17 @@ namespace nix { /* Abstract syntax of derivations. */ +/// Pair of a hash, and how the file system was ingested +struct DerivationOutputHash { + FileIngestionMethod method; + Hash hash; + std::string printMethodAlgo() const; +}; + struct DerivationOutput { StorePath path; - std::string hashAlgo; /* hash used for expected hash computation */ - std::string hash; /* expected hash, may be null */ + std::optional hash; /* hash used for expected hash computation */ void parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const; }; diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index c954ace7f..531b85af8 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -72,6 +72,7 @@ struct curlFileTransfer : public FileTransfer curl_off_t writtenToSink = 0; + inline static const std::set successfulStatuses {200, 201, 204, 206, 304, 0 /* other protocol */}; /* Get the HTTP status code, or 0 for other protocols. */ long getHTTPStatus() { @@ -98,7 +99,7 @@ struct curlFileTransfer : public FileTransfer /* Only write data to the sink if this is a successful response. */ - if (httpStatus == 0 || httpStatus == 200 || httpStatus == 201 || httpStatus == 206) { + if (successfulStatuses.count(httpStatus)) { writtenToSink += len; this->request.dataCallback((char *) data, len); } @@ -352,8 +353,7 @@ struct curlFileTransfer : public FileTransfer if (writeException) failEx(writeException); - else if (code == CURLE_OK && - (httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 206 || httpStatus == 304 || httpStatus == 0 /* other protocol */)) + else if (code == CURLE_OK && successfulStatuses.count(httpStatus)) { result.cached = httpStatus == 304; act.progress(result.bodySize, result.bodySize); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 3901d532a..ca6f3562a 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -561,10 +561,12 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat if (out == drv.outputs.end()) throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath)); - FileIngestionMethod method; Hash h; - out->second.parseHashInfo(method, h); - - check(makeFixedOutputPath(method, h, drvName), out->second.path, "out"); + check( + makeFixedOutputPath( + out->second.hash->method, + out->second.hash->hash, + drvName), + out->second.path, "out"); } else { diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c8097da52..661dea141 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -173,20 +173,20 @@ static std::string makeType( StorePath Store::makeFixedOutputPath( - FileIngestionMethod recursive, + FileIngestionMethod method, const Hash & hash, std::string_view name, const StorePathSet & references, bool hasSelfReference) const { - if (hash.type == htSHA256 && recursive == FileIngestionMethod::Recursive) { + if (hash.type == htSHA256 && method == FileIngestionMethod::Recursive) { return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name); } else { assert(references.empty()); return makeStorePath("output:out", hashString(htSHA256, "fixed:out:" - + (recursive == FileIngestionMethod::Recursive ? (string) "r:" : "") + + makeFileIngestionPrefix(method) + hash.to_string(Base16, true) + ":"), name); } @@ -864,10 +864,21 @@ Strings ValidPathInfo::shortRefs() const } -std::string makeFixedOutputCA(FileIngestionMethod recursive, const Hash & hash) +std::string makeFileIngestionPrefix(const FileIngestionMethod m) { + switch (m) { + case FileIngestionMethod::Flat: + return ""; + case FileIngestionMethod::Recursive: + return "r:"; + default: + throw Error("impossible, caught both cases"); + } +} + +std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash) { return "fixed:" - + (recursive == FileIngestionMethod::Recursive ? (std::string) "r:" : "") + + makeFileIngestionPrefix(method) + hash.to_string(Base32, true); } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 12a6b6939..fd3652588 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -189,9 +189,10 @@ struct ValidPathInfo Strings shortRefs() const; - ValidPathInfo(const StorePath & path) : path(path) { } + ValidPathInfo(const ValidPathInfo & other) = default; - ValidPathInfo(StorePath && path) : path(std::move(path)) { } + ValidPathInfo(StorePath && path) : path(std::move(path)) { }; + ValidPathInfo(const StorePath & path) : path(path) { }; virtual ~ValidPathInfo() { } }; @@ -842,6 +843,9 @@ std::optional decodeValidPathInfo( std::istream & str, bool hashGiven = false); +/* Compute the prefix to the hash algorithm which indicates how the files were + ingested. */ +std::string makeFileIngestionPrefix(const FileIngestionMethod m); /* Compute the content-addressability assertion (ValidPathInfo::ca) for paths created by makeFixedOutputPath() / addToStore(). */ diff --git a/src/libutil/tests/compression.cc b/src/libutil/tests/compression.cc new file mode 100644 index 000000000..5b7a2c5b9 --- /dev/null +++ b/src/libutil/tests/compression.cc @@ -0,0 +1,78 @@ +#include "compression.hh" +#include + +namespace nix { + + /* ---------------------------------------------------------------------------- + * compress / decompress + * --------------------------------------------------------------------------*/ + + TEST(compress, compressWithUnknownMethod) { + ASSERT_THROW(compress("invalid-method", "something-to-compress"), UnknownCompressionMethod); + } + + TEST(compress, noneMethodDoesNothingToTheInput) { + ref o = compress("none", "this-is-a-test"); + + ASSERT_EQ(*o, "this-is-a-test"); + } + + TEST(decompress, decompressXzCompressed) { + auto method = "xz"; + auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf"; + ref o = decompress(method, *compress(method, str)); + + ASSERT_EQ(*o, str); + } + + TEST(decompress, decompressBzip2Compressed) { + auto method = "bzip2"; + auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf"; + ref o = decompress(method, *compress(method, str)); + + ASSERT_EQ(*o, str); + } + + TEST(decompress, decompressBrCompressed) { + auto method = "br"; + auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf"; + ref o = decompress(method, *compress(method, str)); + + ASSERT_EQ(*o, str); + } + + TEST(decompress, decompressInvalidInputThrowsCompressionError) { + auto method = "bzip2"; + auto str = "this is a string that does not qualify as valid bzip2 data"; + + ASSERT_THROW(decompress(method, str), CompressionError); + } + + /* ---------------------------------------------------------------------------- + * compression sinks + * --------------------------------------------------------------------------*/ + + TEST(makeCompressionSink, noneSinkDoesNothingToInput) { + StringSink strSink; + auto inputString = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf"; + auto sink = makeCompressionSink("none", strSink); + (*sink)(inputString); + sink->finish(); + + ASSERT_STREQ((*strSink.s).c_str(), inputString); + } + + TEST(makeCompressionSink, compressAndDecompress) { + StringSink strSink; + auto inputString = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf"; + auto decompressionSink = makeDecompressionSink("bzip2", strSink); + auto sink = makeCompressionSink("bzip2", *decompressionSink); + + (*sink)(inputString); + sink->finish(); + decompressionSink->finish(); + + ASSERT_STREQ((*strSink.s).c_str(), inputString); + } + +} diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 05a9b9cd9..8b85caf82 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -135,7 +135,13 @@ StorePath getDerivationEnvironment(ref store, const StorePath & drvPath) drv.inputSrcs.insert(std::move(getEnvShPath)); Hash h = hashDerivationModulo(*store, drv, true); auto shellOutPath = store->makeOutputPath("out", h, drvName); - drv.outputs.insert_or_assign("out", DerivationOutput { shellOutPath, "", "" }); + drv.outputs.insert_or_assign("out", DerivationOutput { + .path = shellOutPath, + .hash = DerivationOutputHash { + .method = FileIngestionMethod::Flat, + .hash = Hash { }, + }, + }); drv.env["out"] = store->printStorePath(shellOutPath); auto shellDrvPath2 = writeDerivation(store, drv, drvName); diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index 2d31894c2..5d77cfdca 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -70,9 +70,9 @@ struct CmdShowDerivation : InstallablesCommand for (auto & output : drv.outputs) { auto outputObj(outputsObj.object(output.first)); outputObj.attr("path", store->printStorePath(output.second.path)); - if (output.second.hash != "") { - outputObj.attr("hashAlgo", output.second.hashAlgo); - outputObj.attr("hash", output.second.hash); + if (output.second.hash) { + outputObj.attr("hashAlgo", output.second.hash->printMethodAlgo()); + outputObj.attr("hash", output.second.hash->hash.to_string(Base16, false)); } } }