Merge remote-tracking branch 'upstream/master' into no-hash-type-unknown

This commit is contained in:
John Ericson 2020-06-19 17:52:01 +00:00
commit 68294746ae
9 changed files with 138 additions and 71 deletions

View file

@ -775,10 +775,11 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName); auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName);
if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath); if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath);
drv.outputs.insert_or_assign("out", DerivationOutput { drv.outputs.insert_or_assign("out", DerivationOutput {
std::move(outPath), .path = std::move(outPath),
(ingestionMethod == FileIngestionMethod::Recursive ? "r:" : "") .hash = DerivationOutputHash {
+ printHashType(*h.type), .method = ingestionMethod,
h.to_string(Base16, false), .hash = std::move(h),
},
}); });
} }
@ -792,7 +793,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
for (auto & i : outputs) { for (auto & i : outputs) {
if (!jsonObject) drv.env[i] = ""; if (!jsonObject) drv.env[i] = "";
drv.outputs.insert_or_assign(i, drv.outputs.insert_or_assign(i,
DerivationOutput { StorePath::dummy, "", "" }); DerivationOutput {
.path = StorePath::dummy,
.hash = std::optional<DerivationOutputHash> {},
});
} }
Hash h = hashDerivationModulo(*state.store, Derivation(drv), true); Hash h = hashDerivationModulo(*state.store, Derivation(drv), true);
@ -801,7 +805,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
auto outPath = state.store->makeOutputPath(i, h, drvName); auto outPath = state.store->makeOutputPath(i, h, drvName);
if (!jsonObject) drv.env[i] = state.store->printStorePath(outPath); if (!jsonObject) drv.env[i] = state.store->printStorePath(outPath);
drv.outputs.insert_or_assign(i, drv.outputs.insert_or_assign(i,
DerivationOutput { std::move(outPath), "", "" }); DerivationOutput {
.path = std::move(outPath),
.hash = std::optional<DerivationOutputHash>(),
});
} }
} }

View file

@ -3718,10 +3718,7 @@ void DerivationGoal::registerOutputs()
if (fixedOutput) { if (fixedOutput) {
FileIngestionMethod outputHashMode; Hash h; if (i.second.hash->method == FileIngestionMethod::Flat) {
i.second.parseHashInfo(outputHashMode, h);
if (outputHashMode == FileIngestionMethod::Flat) {
/* The output path should be a regular file without execute permission. */ /* The output path should be a regular file without execute permission. */
if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0)
throw BuildError( throw BuildError(
@ -3732,20 +3729,22 @@ void DerivationGoal::registerOutputs()
/* Check the hash. In hash mode, move the path produced by /* Check the hash. In hash mode, move the path produced by
the derivation to its content-addressed location. */ the derivation to its content-addressed location. */
Hash h2 = outputHashMode == FileIngestionMethod::Recursive Hash h2 = i.second.hash->method == FileIngestionMethod::Recursive
? hashPath(*h.type, actualPath).first ? hashPath(*i.second.hash->hash.type, actualPath).first
: hashFile(*h.type, actualPath); : hashFile(*i.second.hash->hash.type, actualPath);
auto dest = worker.store.makeFixedOutputPath(outputHashMode, h2, i.second.path.name()); auto dest = worker.store.makeFixedOutputPath(i.second.hash->method, h2, i.second.path.name());
if (h != h2) { if (i.second.hash->hash != h2) {
/* Throw an error after registering the path as /* Throw an error after registering the path as
valid. */ valid. */
worker.hashMismatch = true; worker.hashMismatch = true;
delayedException = std::make_exception_ptr( delayedException = std::make_exception_ptr(
BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s",
worker.store.printStorePath(dest), h.to_string(SRI, true), h2.to_string(SRI, true))); worker.store.printStorePath(dest),
i.second.hash->hash.to_string(SRI, true),
h2.to_string(SRI, true)));
Path actualDest = worker.store.Store::toRealPath(dest); Path actualDest = worker.store.Store::toRealPath(dest);
@ -3765,7 +3764,7 @@ void DerivationGoal::registerOutputs()
else else
assert(worker.store.parseStorePath(path) == dest); assert(worker.store.parseStorePath(path) == dest);
ca = makeFixedOutputCA(outputHashMode, h2); ca = makeFixedOutputCA(i.second.hash->method, h2);
} }
/* Get rid of all weird permissions. This also checks that /* Get rid of all weird permissions. This also checks that

View file

@ -8,20 +8,8 @@
namespace nix { namespace nix {
std::string DerivationOutputHash::printMethodAlgo() const {
void DerivationOutput::parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const return makeFileIngestionPrefix(method) + printHashType(*hash.type);
{
recursive = FileIngestionMethod::Flat;
string algo = hashAlgo;
if (string(algo, 0, 2) == "r:") {
recursive = FileIngestionMethod::Recursive;
algo = string(algo, 2);
}
HashType hashType = parseHashType(algo);
hash = Hash(this->hash, hashType);
} }
@ -118,6 +106,34 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
} }
static DerivationOutput parseDerivationOutput(const Store & store, istringstream_nocopy & str)
{
expect(str, ","); auto path = store.parseStorePath(parsePath(str));
expect(str, ","); auto hashAlgo = parseString(str);
expect(str, ","); const auto hash = parseString(str);
expect(str, ")");
std::optional<DerivationOutputHash> fsh;
if (hashAlgo != "") {
auto method = FileIngestionMethod::Flat;
if (string(hashAlgo, 0, 2) == "r:") {
method = FileIngestionMethod::Recursive;
hashAlgo = string(hashAlgo, 2);
}
const HashType hashType = parseHashType(hashAlgo);
fsh = DerivationOutputHash {
.method = std::move(method),
.hash = Hash(hash, hashType),
};
}
return DerivationOutput {
.path = std::move(path),
.hash = std::move(fsh),
};
}
static Derivation parseDerivation(const Store & store, const string & s) static Derivation parseDerivation(const Store & store, const string & s)
{ {
Derivation drv; Derivation drv;
@ -127,15 +143,8 @@ static Derivation parseDerivation(const Store & store, const string & s)
/* Parse the list of outputs. */ /* Parse the list of outputs. */
while (!endOfList(str)) { while (!endOfList(str)) {
expect(str, "("); std::string id = parseString(str); expect(str, "("); std::string id = parseString(str);
expect(str, ","); auto path = store.parseStorePath(parsePath(str)); auto output = parseDerivationOutput(store, str);
expect(str, ","); auto hashAlgo = parseString(str); drv.outputs.emplace(std::move(id), std::move(output));
expect(str, ","); auto hash = parseString(str);
expect(str, ")");
drv.outputs.emplace(id, DerivationOutput {
.path = std::move(path),
.hashAlgo = std::move(hashAlgo),
.hash = std::move(hash)
});
} }
/* Parse the list of input derivations. */ /* Parse the list of input derivations. */
@ -261,8 +270,9 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
if (first) first = false; else s += ','; if (first) first = false; else s += ',';
s += '('; printUnquotedString(s, i.first); s += '('; printUnquotedString(s, i.first);
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path)); s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path));
s += ','; printUnquotedString(s, i.second.hashAlgo); s += ','; printUnquotedString(s, i.second.hash ? i.second.hash->printMethodAlgo() : "");
s += ','; printUnquotedString(s, i.second.hash); s += ','; printUnquotedString(s,
i.second.hash ? i.second.hash->hash.to_string(Base16, false) : "");
s += ')'; s += ')';
} }
@ -318,7 +328,7 @@ bool BasicDerivation::isFixedOutput() const
{ {
return outputs.size() == 1 && return outputs.size() == 1 &&
outputs.begin()->first == "out" && outputs.begin()->first == "out" &&
outputs.begin()->second.hash != ""; outputs.begin()->second.hash;
} }
@ -351,8 +361,8 @@ Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutput
if (drv.isFixedOutput()) { if (drv.isFixedOutput()) {
DerivationOutputs::const_iterator i = drv.outputs.begin(); DerivationOutputs::const_iterator i = drv.outputs.begin();
return hashString(htSHA256, "fixed:out:" return hashString(htSHA256, "fixed:out:"
+ i->second.hashAlgo + ":" + i->second.hash->printMethodAlgo() + ":"
+ i->second.hash + ":" + i->second.hash->hash.to_string(Base16, false) + ":"
+ store.printStorePath(i->second.path)); + store.printStorePath(i->second.path));
} }
@ -395,6 +405,31 @@ StorePathSet BasicDerivation::outputPaths() const
return paths; return paths;
} }
static DerivationOutput readDerivationOutput(Source & in, const Store & store)
{
auto path = store.parseStorePath(readString(in));
auto hashAlgo = readString(in);
const auto hash = readString(in);
std::optional<DerivationOutputHash> fsh;
if (hashAlgo != "") {
auto method = FileIngestionMethod::Flat;
if (string(hashAlgo, 0, 2) == "r:") {
method = FileIngestionMethod::Recursive;
hashAlgo = string(hashAlgo, 2);
}
const HashType hashType = parseHashType(hashAlgo);
fsh = DerivationOutputHash {
.method = std::move(method),
.hash = Hash(hash, hashType),
};
}
return DerivationOutput {
.path = std::move(path),
.hash = std::move(fsh),
};
}
StringSet BasicDerivation::outputNames() const StringSet BasicDerivation::outputNames() const
{ {
@ -411,14 +446,8 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv)
auto nr = readNum<size_t>(in); auto nr = readNum<size_t>(in);
for (size_t n = 0; n < nr; n++) { for (size_t n = 0; n < nr; n++) {
auto name = readString(in); auto name = readString(in);
auto path = store.parseStorePath(readString(in)); auto output = readDerivationOutput(in, store);
auto hashAlgo = readString(in); drv.outputs.emplace(std::move(name), std::move(output));
auto hash = readString(in);
drv.outputs.emplace(name, DerivationOutput {
.path = std::move(path),
.hashAlgo = std::move(hashAlgo),
.hash = std::move(hash)
});
} }
drv.inputSrcs = readStorePaths<StorePathSet>(store, in); drv.inputSrcs = readStorePaths<StorePathSet>(store, in);
@ -440,7 +469,10 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
{ {
out << drv.outputs.size(); out << drv.outputs.size();
for (auto & i : drv.outputs) for (auto & i : drv.outputs)
out << i.first << store.printStorePath(i.second.path) << i.second.hashAlgo << i.second.hash; out << i.first
<< store.printStorePath(i.second.path)
<< i.second.hash->printMethodAlgo()
<< i.second.hash->hash.to_string(Base16, false);
writeStorePaths(store, out, drv.inputSrcs); writeStorePaths(store, out, drv.inputSrcs);
out << drv.platform << drv.builder << drv.args; out << drv.platform << drv.builder << drv.args;
out << drv.env.size(); out << drv.env.size();

View file

@ -12,11 +12,17 @@ namespace nix {
/* Abstract syntax of derivations. */ /* Abstract syntax of derivations. */
/// Pair of a hash, and how the file system was ingested
struct DerivationOutputHash {
FileIngestionMethod method;
Hash hash;
std::string printMethodAlgo() const;
};
struct DerivationOutput struct DerivationOutput
{ {
StorePath path; StorePath path;
std::string hashAlgo; /* hash used for expected hash computation */ std::optional<DerivationOutputHash> hash; /* hash used for expected hash computation */
std::string hash; /* expected hash, may be null */
void parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const; void parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const;
}; };

View file

@ -561,10 +561,12 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
if (out == drv.outputs.end()) if (out == drv.outputs.end())
throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath)); throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath));
FileIngestionMethod method; Hash h; check(
out->second.parseHashInfo(method, h); makeFixedOutputPath(
out->second.hash->method,
check(makeFixedOutputPath(method, h, drvName), out->second.path, "out"); out->second.hash->hash,
drvName),
out->second.path, "out");
} }
else { else {

View file

@ -173,20 +173,20 @@ static std::string makeType(
StorePath Store::makeFixedOutputPath( StorePath Store::makeFixedOutputPath(
FileIngestionMethod recursive, FileIngestionMethod method,
const Hash & hash, const Hash & hash,
std::string_view name, std::string_view name,
const StorePathSet & references, const StorePathSet & references,
bool hasSelfReference) const bool hasSelfReference) const
{ {
if (hash.type == htSHA256 && recursive == FileIngestionMethod::Recursive) { if (hash.type == htSHA256 && method == FileIngestionMethod::Recursive) {
return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name); return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name);
} else { } else {
assert(references.empty()); assert(references.empty());
return makeStorePath("output:out", return makeStorePath("output:out",
hashString(htSHA256, hashString(htSHA256,
"fixed:out:" "fixed:out:"
+ (recursive == FileIngestionMethod::Recursive ? (string) "r:" : "") + makeFileIngestionPrefix(method)
+ hash.to_string(Base16, true) + ":"), + hash.to_string(Base16, true) + ":"),
name); name);
} }
@ -822,10 +822,21 @@ Strings ValidPathInfo::shortRefs() const
} }
std::string makeFixedOutputCA(FileIngestionMethod recursive, const Hash & hash) std::string makeFileIngestionPrefix(const FileIngestionMethod m) {
switch (m) {
case FileIngestionMethod::Flat:
return "";
case FileIngestionMethod::Recursive:
return "r:";
default:
throw Error("impossible, caught both cases");
}
}
std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash)
{ {
return "fixed:" return "fixed:"
+ (recursive == FileIngestionMethod::Recursive ? (std::string) "r:" : "") + makeFileIngestionPrefix(method)
+ hash.to_string(Base32, true); + hash.to_string(Base32, true);
} }

View file

@ -189,9 +189,10 @@ struct ValidPathInfo
Strings shortRefs() const; Strings shortRefs() const;
ValidPathInfo(const StorePath & path) : path(path) { } ValidPathInfo(const ValidPathInfo & other) = default;
ValidPathInfo(StorePath && path) : path(std::move(path)) { } ValidPathInfo(StorePath && path) : path(std::move(path)) { };
ValidPathInfo(const StorePath & path) : path(path) { };
virtual ~ValidPathInfo() { } virtual ~ValidPathInfo() { }
}; };
@ -838,6 +839,9 @@ std::optional<ValidPathInfo> decodeValidPathInfo(
std::istream & str, std::istream & str,
bool hashGiven = false); bool hashGiven = false);
/* Compute the prefix to the hash algorithm which indicates how the files were
ingested. */
std::string makeFileIngestionPrefix(const FileIngestionMethod m);
/* Compute the content-addressability assertion (ValidPathInfo::ca) /* Compute the content-addressability assertion (ValidPathInfo::ca)
for paths created by makeFixedOutputPath() / addToStore(). */ for paths created by makeFixedOutputPath() / addToStore(). */

View file

@ -135,7 +135,13 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
drv.inputSrcs.insert(std::move(getEnvShPath)); drv.inputSrcs.insert(std::move(getEnvShPath));
Hash h = hashDerivationModulo(*store, drv, true); Hash h = hashDerivationModulo(*store, drv, true);
auto shellOutPath = store->makeOutputPath("out", h, drvName); auto shellOutPath = store->makeOutputPath("out", h, drvName);
drv.outputs.insert_or_assign("out", DerivationOutput { shellOutPath, "", "" }); drv.outputs.insert_or_assign("out", DerivationOutput {
.path = shellOutPath,
.hash = DerivationOutputHash {
.method = FileIngestionMethod::Flat,
.hash = Hash { },
},
});
drv.env["out"] = store->printStorePath(shellOutPath); drv.env["out"] = store->printStorePath(shellOutPath);
auto shellDrvPath2 = writeDerivation(store, drv, drvName); auto shellDrvPath2 = writeDerivation(store, drv, drvName);

View file

@ -70,9 +70,9 @@ struct CmdShowDerivation : InstallablesCommand
for (auto & output : drv.outputs) { for (auto & output : drv.outputs) {
auto outputObj(outputsObj.object(output.first)); auto outputObj(outputsObj.object(output.first));
outputObj.attr("path", store->printStorePath(output.second.path)); outputObj.attr("path", store->printStorePath(output.second.path));
if (output.second.hash != "") { if (output.second.hash) {
outputObj.attr("hashAlgo", output.second.hashAlgo); outputObj.attr("hashAlgo", output.second.hash->printMethodAlgo());
outputObj.attr("hash", output.second.hash); outputObj.attr("hash", output.second.hash->hash.to_string(Base16, false));
} }
} }
} }