lix-project/lix-releng-staging
0
0
Fork 0
Code Packages Activity

Store parsed hashes in DerivationOutput

Browse source 
It's best to detect invalid data as soon as possible, with data types
that make storing it impossible.
This commit is contained in:
John Ericson 2020-03-22 23:43:07 -04:00
parent f5494d9442
commit 832bd534dc
8 changed files with 128 additions and 63 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
src/libexpr/primops.cc
View file

@ -724,9 +724,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
auto outPath = state.store->makeFixedOutputPath(outputHashRecursive, h, drvName);
if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath);
drv.outputs.insert_or_assign("out", DerivationOutput(std::move(outPath),
(static_cast<bool>(outputHashRecursive) ? "r:" : "") + printHashType(h.type),
h.to_string(Base16, false)));
drv.outputs.insert_or_assign("out", DerivationOutput(
std::move(outPath),
FileSystemHash(outputHashRecursive, std::move(h))));
}
else {
@ -739,7 +739,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
for (auto & i : outputs) {
if (!jsonObject) drv.env[i] = "";
drv.outputs.insert_or_assign(i,
DerivationOutput(StorePath::dummy.clone(), "", ""));
DerivationOutput(StorePath::dummy.clone(), std::optional<FileSystemHash>()));
}
Hash h = hashDerivationModulo(*state.store, Derivation(drv), true);
@ -748,7 +748,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
auto outPath = state.store->makeOutputPath(i, h, drvName);
if (!jsonObject) drv.env[i] = state.store->printStorePath(outPath);
drv.outputs.insert_or_assign(i,
DerivationOutput(std::move(outPath), "", ""));
DerivationOutput(std::move(outPath), std::optional<FileSystemHash>()));
}
}

19
src/libstore/build.cc
View file

@ -3647,10 +3647,7 @@ void DerivationGoal::registerOutputs()
if (fixedOutput) {
FileIngestionMethod recursive; Hash h;
i.second.parseHashInfo(recursive, h);
if (!static_cast<bool>(recursive)) {
if (i.second.hash->method == FileIngestionMethod::Flat) {
/* The output path should be a regular file without execute permission. */
if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0)
throw BuildError(
@ -3659,18 +3656,22 @@ void DerivationGoal::registerOutputs()
/* Check the hash. In hash mode, move the path produced by
the derivation to its content-addressed location. */
Hash h2 = static_cast<bool>(recursive) ? hashPath(h.type, actualPath).first : hashFile(h.type, actualPath);
Hash h2 = i.second.hash->method == FileIngestionMethod::Recursive
? hashPath(i.second.hash->hash.type, actualPath).first
: hashFile(i.second.hash->hash.type, actualPath);
auto dest = worker.store.makeFixedOutputPath(recursive, h2, i.second.path.name());
auto dest = worker.store.makeFixedOutputPath(i.second.hash->method, h2, i.second.path.name());
if (h != h2) {
if (i.second.hash->hash != h2) {
/* Throw an error after registering the path as
valid. */
worker.hashMismatch = true;
delayedException = std::make_exception_ptr(
BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s",
worker.store.printStorePath(dest), h.to_string(SRI), h2.to_string(SRI)));
worker.store.printStorePath(dest),
i.second.hash->hash.to_string(SRI),
h2.to_string(SRI)));
Path actualDest = worker.store.Store::toRealPath(dest);
@ -3690,7 +3691,7 @@ void DerivationGoal::registerOutputs()
else
assert(worker.store.parseStorePath(path) == dest);
ca = makeFixedOutputCA(recursive, h2);
ca = makeFixedOutputCA(i.second.hash->method, h2);
}
/* Get rid of all weird permissions. This also checks that

99
src/libstore/derivations.cc
View file

@ -8,22 +8,8 @@
namespace nix {
void DerivationOutput::parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const
{
recursive = FileIngestionMethod::Flat;
string algo = hashAlgo;
if (string(algo, 0, 2) == "r:") {
recursive = FileIngestionMethod::Recursive;
algo = string(algo, 2);
}
HashType hashType = parseHashType(algo);
if (hashType == htUnknown)
throw Error("unknown hash algorithm '%s'", algo);
hash = Hash(this->hash, hashType);
std::string FileSystemHash::printMethodAlgo() const {
return makeFileIngestionPrefix(method) + printHashType(hash.type);
}
@ -35,7 +21,7 @@ BasicDerivation::BasicDerivation(const BasicDerivation & other)
{
for (auto & i : other.outputs)
outputs.insert_or_assign(i.first,
DerivationOutput(i.second.path.clone(), std::string(i.second.hashAlgo), std::string(i.second.hash)));
DerivationOutput(i.second.path.clone(), std::optional<FileSystemHash>(i.second.hash)));
for (auto & i : other.inputSrcs)
inputSrcs.insert(i.clone());
}
@ -142,6 +128,33 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
}
static DerivationOutput parseDerivationOutput(const Store & store, istringstream_nocopy & str)
{
expect(str, ","); auto path = store.parseStorePath(parsePath(str));
expect(str, ","); auto hashAlgo = parseString(str);
expect(str, ","); const auto hash = parseString(str);
expect(str, ")");
auto method = FileIngestionMethod::Flat;
std::optional<FileSystemHash> fsh;
if (hashAlgo != "") {
if (string(hashAlgo, 0, 2) == "r:") {
method = FileIngestionMethod::Recursive;
hashAlgo = string(hashAlgo, 2);
}
const HashType hashType = parseHashType(hashAlgo);
if (hashType == htUnknown)
throw Error("unknown hash hashAlgorithm '%s'", hashAlgo);
fsh = FileSystemHash {
std::move(method),
Hash(hash, hashType),
};
}
return DerivationOutput(std::move(path), std::move(fsh));
}
static Derivation parseDerivation(const Store & store, const string & s)
{
Derivation drv;
@ -151,11 +164,7 @@ static Derivation parseDerivation(const Store & store, const string & s)
/* Parse the list of outputs. */
while (!endOfList(str)) {
expect(str, "("); std::string id = parseString(str);
expect(str, ","); auto path = store.parseStorePath(parsePath(str));
expect(str, ","); auto hashAlgo = parseString(str);
expect(str, ","); auto hash = parseString(str);
expect(str, ")");
drv.outputs.emplace(id, DerivationOutput(std::move(path), std::move(hashAlgo), std::move(hash)));
drv.outputs.emplace(id, parseDerivationOutput(store, str));
}
/* Parse the list of input derivations. */
@ -275,8 +284,9 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
if (first) first = false; else s += ',';
s += '('; printUnquotedString(s, i.first);
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path));
s += ','; printUnquotedString(s, i.second.hashAlgo);
s += ','; printUnquotedString(s, i.second.hash);
s += ','; printUnquotedString(s, i.second.hash ? i.second.hash->printMethodAlgo() : "");
s += ','; printUnquotedString(s,
i.second.hash ? i.second.hash->hash.to_string(Base16, false) : "");
s += ')';
}
@ -332,7 +342,7 @@ bool BasicDerivation::isFixedOutput() const
{
return outputs.size() == 1 &&
outputs.begin()->first == "out" &&
outputs.begin()->second.hash != "";
outputs.begin()->second.hash;
}
@ -365,8 +375,8 @@ Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutput
if (drv.isFixedOutput()) {
DerivationOutputs::const_iterator i = drv.outputs.begin();
return hashString(htSHA256, "fixed:out:"
+ i->second.hashAlgo + ":"
+ i->second.hash + ":"
+ i->second.hash->printMethodAlgo() + ":"
+ i->second.hash->hash.to_string(Base16, false) + ":"
+ store.printStorePath(i->second.path));
}
@ -409,6 +419,30 @@ StorePathSet BasicDerivation::outputPaths() const
return paths;
}
static DerivationOutput readDerivationOutput(Source & in, const Store & store)
{
auto path = store.parseStorePath(readString(in));
auto hashAlgo = readString(in);
const auto hash = readString(in);
auto method = FileIngestionMethod::Flat;
std::optional<FileSystemHash> fsh;
if (hashAlgo != "") {
if (string(hashAlgo, 0, 2) == "r:") {
method = FileIngestionMethod::Recursive;
hashAlgo = string(hashAlgo, 2);
}
HashType hashType = parseHashType(hashAlgo);
if (hashType == htUnknown)
throw Error("unknown hash hashAlgorithm '%s'", hashAlgo);
fsh = FileSystemHash {
std::move(method),
Hash(hash, hashType),
};
}
return DerivationOutput(std::move(path), std::move(fsh));
}
Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv)
{
@ -416,10 +450,8 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv)
auto nr = readNum<size_t>(in);
for (size_t n = 0; n < nr; n++) {
auto name = readString(in);
auto path = store.parseStorePath(readString(in));
auto hashAlgo = readString(in);
auto hash = readString(in);
drv.outputs.emplace(name, DerivationOutput(std::move(path), std::move(hashAlgo), std::move(hash)));
auto output = readDerivationOutput(in, store);
drv.outputs.emplace(name, output);
}
drv.inputSrcs = readStorePaths<StorePathSet>(store, in);
@ -441,7 +473,10 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
{
out << drv.outputs.size();
for (auto & i : drv.outputs)
out << i.first << store.printStorePath(i.second.path) << i.second.hashAlgo << i.second.hash;
out << i.first
<< store.printStorePath(i.second.path)
<< i.second.hash->printMethodAlgo()
<< i.second.hash->hash.to_string(Base16, false);
writeStorePaths(store, out, drv.inputSrcs);
out << drv.platform << drv.builder << drv.args;
out << drv.env.size();

21
src/libstore/derivations.hh
View file

@ -12,20 +12,31 @@ namespace nix {
/* Abstract syntax of derivations. */
/// Pair of a hash, and how the file system was ingested
struct FileSystemHash {
FileIngestionMethod method;
Hash hash;
FileSystemHash(FileIngestionMethod method, Hash hash)
: method(std::move(method))
, hash(std::move(hash))
{ }
FileSystemHash(const FileSystemHash &) = default;
FileSystemHash(FileSystemHash &&) = default;
FileSystemHash & operator = (const FileSystemHash &) = default;
std::string printMethodAlgo() const;
};
struct DerivationOutput
{
StorePath path;
std::string hashAlgo; /* hash used for expected hash computation */
std::string hash; /* expected hash, may be null */
DerivationOutput(StorePath && path, std::string && hashAlgo, std::string && hash)
std::optional<FileSystemHash> hash; /* hash used for expected hash computation */
DerivationOutput(StorePath && path, std::optional<FileSystemHash> && hash)
: path(std::move(path))
, hashAlgo(std::move(hashAlgo))
, hash(std::move(hash))
{ }
DerivationOutput(const DerivationOutput &) = default;
DerivationOutput(DerivationOutput &&) = default;
DerivationOutput & operator = (const DerivationOutput &) = default;
void parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const;
};
typedef std::map<string, DerivationOutput> DerivationOutputs;

10
src/libstore/local-store.cc
View file

@ -557,10 +557,12 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
if (out == drv.outputs.end())
throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath));
FileIngestionMethod recursive; Hash h;
out->second.parseHashInfo(recursive, h);
check(makeFixedOutputPath(recursive, h, drvName), out->second.path, "out");
check(
makeFixedOutputPath(
out->second.hash->method,
out->second.hash->hash,
drvName),
out->second.path, "out");
}
else {

23
src/libstore/store-api.cc
View file

@ -171,18 +171,18 @@ static std::string makeType(
StorePath Store::makeFixedOutputPath(
FileIngestionMethod recursive,
FileIngestionMethod method,
const Hash & hash,
std::string_view name,
const StorePathSet & references,
bool hasSelfReference) const
{
if (hash.type == htSHA256 && recursive == FileIngestionMethod::Recursive) {
if (hash.type == htSHA256 && method == FileIngestionMethod::Recursive) {
return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name);
} else {
assert(references.empty());
return makeStorePath("output:out", hashString(htSHA256,
"fixed:out:" + (static_cast<bool>(recursive) ? (string) "r:" : "") +
"fixed:out:" + makeFileIngestionPrefix(method) +
hash.to_string(Base16) + ":"), name);
}
}
@ -811,9 +811,22 @@ Strings ValidPathInfo::shortRefs() const
}
std::string makeFixedOutputCA(FileIngestionMethod recursive, const Hash & hash)
std::string makeFileIngestionPrefix(const FileIngestionMethod m) {
switch (m) {
case FileIngestionMethod::Flat:
return "";
case FileIngestionMethod::Recursive:
return "r:";
default:
throw Error("impossible, caught both cases");
}
}
std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash)
{
return "fixed:" + (static_cast<bool>(recursive) ? (std::string) "r:" : "") + hash.to_string();
return "fixed:"
+ makeFileIngestionPrefix(method)
+ hash.to_string();
}

3
src/libstore/store-api.hh
View file

@ -847,6 +847,9 @@ std::optional<ValidPathInfo> decodeValidPathInfo(
std::istream & str,
bool hashGiven = false);
/* Compute the prefix to the hash algorithm which indicates how the files were
ingested. */
std::string makeFileIngestionPrefix(const FileIngestionMethod m);
/* Compute the content-addressability assertion (ValidPathInfo::ca)
for paths created by makeFixedOutputPath() / addToStore(). */

6
src/nix/show-derivation.cc
View file

@ -69,9 +69,9 @@ struct CmdShowDerivation : InstallablesCommand
for (auto & output : drv.outputs) {
auto outputObj(outputsObj.object(output.first));
outputObj.attr("path", store->printStorePath(output.second.path));
if (output.second.hash != "") {
outputObj.attr("hashAlgo", output.second.hashAlgo);
outputObj.attr("hash", output.second.hash);
if (output.second.hash) {
outputObj.attr("hashAlgo", output.second.hash->printMethodAlgo());
outputObj.attr("hash", output.second.hash->hash.to_string(Base16, false));
}
}
}