forked from lix-project/lix
Merge pull request #3746 from obsidiansystems/path-info
Introduce `StoreReferences` and `ContentAddressWithReferences`
This commit is contained in:
commit
e641de085b
|
@ -293,7 +293,13 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name)
|
||||||
try {
|
try {
|
||||||
auto h = Hash::parseAny(hash, parseHashType(algo));
|
auto h = Hash::parseAny(hash, parseHashType(algo));
|
||||||
auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
|
auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
|
||||||
auto path = store()->makeFixedOutputPath(method, h, name);
|
auto path = store()->makeFixedOutputPath(name, FixedOutputInfo {
|
||||||
|
.hash = {
|
||||||
|
.method = method,
|
||||||
|
.hash = h,
|
||||||
|
},
|
||||||
|
.references = {},
|
||||||
|
});
|
||||||
XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0)));
|
XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0)));
|
||||||
} catch (Error & e) {
|
} catch (Error & e) {
|
||||||
croak("%s", e.what());
|
croak("%s", e.what());
|
||||||
|
|
|
@ -1293,7 +1293,13 @@ drvName, Bindings * attrs, Value & v)
|
||||||
auto h = newHashAllowEmpty(*outputHash, parseHashTypeOpt(outputHashAlgo));
|
auto h = newHashAllowEmpty(*outputHash, parseHashTypeOpt(outputHashAlgo));
|
||||||
|
|
||||||
auto method = ingestionMethod.value_or(FileIngestionMethod::Flat);
|
auto method = ingestionMethod.value_or(FileIngestionMethod::Flat);
|
||||||
auto outPath = state.store->makeFixedOutputPath(method, h, drvName);
|
auto outPath = state.store->makeFixedOutputPath(drvName, FixedOutputInfo {
|
||||||
|
.hash = {
|
||||||
|
.method = method,
|
||||||
|
.hash = h,
|
||||||
|
},
|
||||||
|
.references = {},
|
||||||
|
});
|
||||||
drv.env["out"] = state.store->printStorePath(outPath);
|
drv.env["out"] = state.store->printStorePath(outPath);
|
||||||
drv.outputs.insert_or_assign("out",
|
drv.outputs.insert_or_assign("out",
|
||||||
DerivationOutput::CAFixed {
|
DerivationOutput::CAFixed {
|
||||||
|
@ -2103,7 +2109,13 @@ static void addPath(
|
||||||
|
|
||||||
std::optional<StorePath> expectedStorePath;
|
std::optional<StorePath> expectedStorePath;
|
||||||
if (expectedHash)
|
if (expectedHash)
|
||||||
expectedStorePath = state.store->makeFixedOutputPath(method, *expectedHash, name);
|
expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputInfo {
|
||||||
|
.hash = {
|
||||||
|
.method = method,
|
||||||
|
.hash = *expectedHash,
|
||||||
|
},
|
||||||
|
.references = {},
|
||||||
|
});
|
||||||
|
|
||||||
if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
|
if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
|
||||||
StorePath dstPath = settings.readOnlyMode
|
StorePath dstPath = settings.readOnlyMode
|
||||||
|
|
|
@ -243,10 +243,15 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
|
||||||
|
|
||||||
// early exit if pinned and already in the store
|
// early exit if pinned and already in the store
|
||||||
if (expectedHash && expectedHash->type == htSHA256) {
|
if (expectedHash && expectedHash->type == htSHA256) {
|
||||||
auto expectedPath =
|
auto expectedPath = state.store->makeFixedOutputPath(
|
||||||
unpack
|
name,
|
||||||
? state.store->makeFixedOutputPath(FileIngestionMethod::Recursive, *expectedHash, name, {})
|
FixedOutputInfo {
|
||||||
: state.store->makeFixedOutputPath(FileIngestionMethod::Flat, *expectedHash, name, {});
|
.hash = {
|
||||||
|
.method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat,
|
||||||
|
.hash = *expectedHash,
|
||||||
|
},
|
||||||
|
.references = {}
|
||||||
|
});
|
||||||
|
|
||||||
if (state.store->isValidPath(expectedPath)) {
|
if (state.store->isValidPath(expectedPath)) {
|
||||||
state.allowAndSetStorePathString(expectedPath, v);
|
state.allowAndSetStorePathString(expectedPath, v);
|
||||||
|
|
|
@ -210,7 +210,13 @@ StorePath Input::computeStorePath(Store & store) const
|
||||||
auto narHash = getNarHash();
|
auto narHash = getNarHash();
|
||||||
if (!narHash)
|
if (!narHash)
|
||||||
throw Error("cannot compute store path for unlocked input '%s'", to_string());
|
throw Error("cannot compute store path for unlocked input '%s'", to_string());
|
||||||
return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, getName());
|
return store.makeFixedOutputPath(getName(), FixedOutputInfo {
|
||||||
|
.hash = {
|
||||||
|
.method = FileIngestionMethod::Recursive,
|
||||||
|
.hash = *narHash,
|
||||||
|
},
|
||||||
|
.references = {},
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
std::string Input::getType() const
|
std::string Input::getType() const
|
||||||
|
|
|
@ -71,15 +71,19 @@ DownloadFileResult downloadFile(
|
||||||
dumpString(res.data, sink);
|
dumpString(res.data, sink);
|
||||||
auto hash = hashString(htSHA256, res.data);
|
auto hash = hashString(htSHA256, res.data);
|
||||||
ValidPathInfo info {
|
ValidPathInfo info {
|
||||||
store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name),
|
*store,
|
||||||
|
name,
|
||||||
|
FixedOutputInfo {
|
||||||
|
.hash = {
|
||||||
|
.method = FileIngestionMethod::Flat,
|
||||||
|
.hash = hash,
|
||||||
|
},
|
||||||
|
.references = {},
|
||||||
|
},
|
||||||
hashString(htSHA256, sink.s),
|
hashString(htSHA256, sink.s),
|
||||||
};
|
};
|
||||||
info.narSize = sink.s.size();
|
info.narSize = sink.s.size();
|
||||||
info.ca = FixedOutputHash {
|
auto source = StringSource { sink.s };
|
||||||
.method = FileIngestionMethod::Flat,
|
|
||||||
.hash = hash,
|
|
||||||
};
|
|
||||||
auto source = StringSource(sink.s);
|
|
||||||
store->addToStore(info, source, NoRepair, NoCheckSigs);
|
store->addToStore(info, source, NoRepair, NoCheckSigs);
|
||||||
storePath = std::move(info.path);
|
storePath = std::move(info.path);
|
||||||
}
|
}
|
||||||
|
|
|
@ -306,11 +306,22 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n
|
||||||
unsupported("addToStoreFromDump");
|
unsupported("addToStoreFromDump");
|
||||||
return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) {
|
return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) {
|
||||||
ValidPathInfo info {
|
ValidPathInfo info {
|
||||||
makeFixedOutputPath(method, nar.first, name, references),
|
*this,
|
||||||
|
name,
|
||||||
|
FixedOutputInfo {
|
||||||
|
.hash = {
|
||||||
|
.method = method,
|
||||||
|
.hash = nar.first,
|
||||||
|
},
|
||||||
|
.references = {
|
||||||
|
.others = references,
|
||||||
|
// caller is not capable of creating a self-reference, because this is content-addressed without modulus
|
||||||
|
.self = false,
|
||||||
|
},
|
||||||
|
},
|
||||||
nar.first,
|
nar.first,
|
||||||
};
|
};
|
||||||
info.narSize = nar.second;
|
info.narSize = nar.second;
|
||||||
info.references = references;
|
|
||||||
return info;
|
return info;
|
||||||
})->path;
|
})->path;
|
||||||
}
|
}
|
||||||
|
@ -414,15 +425,22 @@ StorePath BinaryCacheStore::addToStore(
|
||||||
});
|
});
|
||||||
return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) {
|
return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) {
|
||||||
ValidPathInfo info {
|
ValidPathInfo info {
|
||||||
makeFixedOutputPath(method, h, name, references),
|
*this,
|
||||||
|
name,
|
||||||
|
FixedOutputInfo {
|
||||||
|
.hash = {
|
||||||
|
.method = method,
|
||||||
|
.hash = h,
|
||||||
|
},
|
||||||
|
.references = {
|
||||||
|
.others = references,
|
||||||
|
// caller is not capable of creating a self-reference, because this is content-addressed without modulus
|
||||||
|
.self = false,
|
||||||
|
},
|
||||||
|
},
|
||||||
nar.first,
|
nar.first,
|
||||||
};
|
};
|
||||||
info.narSize = nar.second;
|
info.narSize = nar.second;
|
||||||
info.references = references;
|
|
||||||
info.ca = FixedOutputHash {
|
|
||||||
.method = method,
|
|
||||||
.hash = h,
|
|
||||||
};
|
|
||||||
return info;
|
return info;
|
||||||
})->path;
|
})->path;
|
||||||
}
|
}
|
||||||
|
@ -434,7 +452,7 @@ StorePath BinaryCacheStore::addTextToStore(
|
||||||
RepairFlag repair)
|
RepairFlag repair)
|
||||||
{
|
{
|
||||||
auto textHash = hashString(htSHA256, s);
|
auto textHash = hashString(htSHA256, s);
|
||||||
auto path = makeTextPath(name, textHash, references);
|
auto path = makeTextPath(name, TextInfo { { textHash }, references });
|
||||||
|
|
||||||
if (!repair && isValidPath(path))
|
if (!repair && isValidPath(path))
|
||||||
return path;
|
return path;
|
||||||
|
@ -443,10 +461,16 @@ StorePath BinaryCacheStore::addTextToStore(
|
||||||
dumpString(s, sink);
|
dumpString(s, sink);
|
||||||
StringSource source(sink.s);
|
StringSource source(sink.s);
|
||||||
return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) {
|
return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) {
|
||||||
ValidPathInfo info { path, nar.first };
|
ValidPathInfo info {
|
||||||
|
*this,
|
||||||
|
std::string { name },
|
||||||
|
TextInfo {
|
||||||
|
{ .hash = textHash },
|
||||||
|
references,
|
||||||
|
},
|
||||||
|
nar.first,
|
||||||
|
};
|
||||||
info.narSize = nar.second;
|
info.narSize = nar.second;
|
||||||
info.ca = TextHash { textHash };
|
|
||||||
info.references = references;
|
|
||||||
return info;
|
return info;
|
||||||
})->path;
|
})->path;
|
||||||
}
|
}
|
||||||
|
|
|
@ -2395,27 +2395,26 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
auto rewriteRefs = [&]() -> std::pair<bool, StorePathSet> {
|
auto rewriteRefs = [&]() -> StoreReferences {
|
||||||
/* In the CA case, we need the rewritten refs to calculate the
|
/* In the CA case, we need the rewritten refs to calculate the
|
||||||
final path, therefore we look for a *non-rewritten
|
final path, therefore we look for a *non-rewritten
|
||||||
self-reference, and use a bool rather try to solve the
|
self-reference, and use a bool rather try to solve the
|
||||||
computationally intractable fixed point. */
|
computationally intractable fixed point. */
|
||||||
std::pair<bool, StorePathSet> res {
|
StoreReferences res {
|
||||||
false,
|
.self = false,
|
||||||
{},
|
|
||||||
};
|
};
|
||||||
for (auto & r : references) {
|
for (auto & r : references) {
|
||||||
auto name = r.name();
|
auto name = r.name();
|
||||||
auto origHash = std::string { r.hashPart() };
|
auto origHash = std::string { r.hashPart() };
|
||||||
if (r == *scratchPath) {
|
if (r == *scratchPath) {
|
||||||
res.first = true;
|
res.self = true;
|
||||||
} else if (auto outputRewrite = get(outputRewrites, origHash)) {
|
} else if (auto outputRewrite = get(outputRewrites, origHash)) {
|
||||||
std::string newRef = *outputRewrite;
|
std::string newRef = *outputRewrite;
|
||||||
newRef += '-';
|
newRef += '-';
|
||||||
newRef += name;
|
newRef += name;
|
||||||
res.second.insert(StorePath { newRef });
|
res.others.insert(StorePath { newRef });
|
||||||
} else {
|
} else {
|
||||||
res.second.insert(r);
|
res.others.insert(r);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return res;
|
return res;
|
||||||
|
@ -2448,18 +2447,22 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
auto got = caSink.finish().first;
|
auto got = caSink.finish().first;
|
||||||
auto refs = rewriteRefs();
|
ValidPathInfo newInfo0 {
|
||||||
|
worker.store,
|
||||||
auto finalPath = worker.store.makeFixedOutputPath(
|
|
||||||
outputHash.method,
|
|
||||||
got,
|
|
||||||
outputPathName(drv->name, outputName),
|
outputPathName(drv->name, outputName),
|
||||||
refs.second,
|
FixedOutputInfo {
|
||||||
refs.first);
|
.hash = {
|
||||||
if (*scratchPath != finalPath) {
|
.method = outputHash.method,
|
||||||
|
.hash = got,
|
||||||
|
},
|
||||||
|
.references = rewriteRefs(),
|
||||||
|
},
|
||||||
|
Hash::dummy,
|
||||||
|
};
|
||||||
|
if (*scratchPath != newInfo0.path) {
|
||||||
// Also rewrite the output path
|
// Also rewrite the output path
|
||||||
auto source = sinkToSource([&](Sink & nextSink) {
|
auto source = sinkToSource([&](Sink & nextSink) {
|
||||||
RewritingSink rsink2(oldHashPart, std::string(finalPath.hashPart()), nextSink);
|
RewritingSink rsink2(oldHashPart, std::string(newInfo0.path.hashPart()), nextSink);
|
||||||
dumpPath(actualPath, rsink2);
|
dumpPath(actualPath, rsink2);
|
||||||
rsink2.flush();
|
rsink2.flush();
|
||||||
});
|
});
|
||||||
|
@ -2470,19 +2473,8 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
|
||||||
}
|
}
|
||||||
|
|
||||||
HashResult narHashAndSize = hashPath(htSHA256, actualPath);
|
HashResult narHashAndSize = hashPath(htSHA256, actualPath);
|
||||||
ValidPathInfo newInfo0 {
|
newInfo0.narHash = narHashAndSize.first;
|
||||||
finalPath,
|
|
||||||
narHashAndSize.first,
|
|
||||||
};
|
|
||||||
|
|
||||||
newInfo0.narSize = narHashAndSize.second;
|
newInfo0.narSize = narHashAndSize.second;
|
||||||
newInfo0.ca = FixedOutputHash {
|
|
||||||
.method = outputHash.method,
|
|
||||||
.hash = got,
|
|
||||||
};
|
|
||||||
newInfo0.references = refs.second;
|
|
||||||
if (refs.first)
|
|
||||||
newInfo0.references.insert(newInfo0.path);
|
|
||||||
|
|
||||||
assert(newInfo0.ca);
|
assert(newInfo0.ca);
|
||||||
return newInfo0;
|
return newInfo0;
|
||||||
|
@ -2504,8 +2496,8 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
|
||||||
ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first };
|
ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first };
|
||||||
newInfo0.narSize = narHashAndSize.second;
|
newInfo0.narSize = narHashAndSize.second;
|
||||||
auto refs = rewriteRefs();
|
auto refs = rewriteRefs();
|
||||||
newInfo0.references = refs.second;
|
newInfo0.references = std::move(refs.others);
|
||||||
if (refs.first)
|
if (refs.self)
|
||||||
newInfo0.references.insert(newInfo0.path);
|
newInfo0.references.insert(newInfo0.path);
|
||||||
return newInfo0;
|
return newInfo0;
|
||||||
},
|
},
|
||||||
|
@ -2519,7 +2511,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
|
||||||
/* Check wanted hash */
|
/* Check wanted hash */
|
||||||
const Hash & wanted = dof.hash.hash;
|
const Hash & wanted = dof.hash.hash;
|
||||||
assert(newInfo0.ca);
|
assert(newInfo0.ca);
|
||||||
auto got = getContentAddressHash(*newInfo0.ca);
|
auto got = newInfo0.ca->getHash();
|
||||||
if (wanted != got) {
|
if (wanted != got) {
|
||||||
/* Throw an error after registering the path as
|
/* Throw an error after registering the path as
|
||||||
valid. */
|
valid. */
|
||||||
|
|
|
@ -95,7 +95,9 @@ void PathSubstitutionGoal::tryNext()
|
||||||
subs.pop_front();
|
subs.pop_front();
|
||||||
|
|
||||||
if (ca) {
|
if (ca) {
|
||||||
subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca);
|
subPath = sub->makeFixedOutputPathFromCA(
|
||||||
|
std::string { storePath.name() },
|
||||||
|
ContentAddressWithReferences::withoutRefs(*ca));
|
||||||
if (sub->storeDir == worker.store.storeDir)
|
if (sub->storeDir == worker.store.storeDir)
|
||||||
assert(subPath == storePath);
|
assert(subPath == storePath);
|
||||||
} else if (sub->storeDir != worker.store.storeDir) {
|
} else if (sub->storeDir != worker.store.storeDir) {
|
||||||
|
|
|
@ -9,7 +9,7 @@ std::string FixedOutputHash::printMethodAlgo() const
|
||||||
return makeFileIngestionPrefix(method) + printHashType(hash.type);
|
return makeFileIngestionPrefix(method) + printHashType(hash.type);
|
||||||
}
|
}
|
||||||
|
|
||||||
std::string makeFileIngestionPrefix(const FileIngestionMethod m)
|
std::string makeFileIngestionPrefix(FileIngestionMethod m)
|
||||||
{
|
{
|
||||||
switch (m) {
|
switch (m) {
|
||||||
case FileIngestionMethod::Flat:
|
case FileIngestionMethod::Flat:
|
||||||
|
@ -21,39 +21,35 @@ std::string makeFileIngestionPrefix(const FileIngestionMethod m)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash)
|
std::string ContentAddress::render() const
|
||||||
{
|
|
||||||
return "fixed:"
|
|
||||||
+ makeFileIngestionPrefix(method)
|
|
||||||
+ hash.to_string(Base32, true);
|
|
||||||
}
|
|
||||||
|
|
||||||
std::string renderContentAddress(ContentAddress ca)
|
|
||||||
{
|
{
|
||||||
return std::visit(overloaded {
|
return std::visit(overloaded {
|
||||||
[](TextHash & th) {
|
[](const TextHash & th) {
|
||||||
return "text:" + th.hash.to_string(Base32, true);
|
return "text:"
|
||||||
|
+ th.hash.to_string(Base32, true);
|
||||||
},
|
},
|
||||||
[](FixedOutputHash & fsh) {
|
[](const FixedOutputHash & fsh) {
|
||||||
return makeFixedOutputCA(fsh.method, fsh.hash);
|
return "fixed:"
|
||||||
|
+ makeFileIngestionPrefix(fsh.method)
|
||||||
|
+ fsh.hash.to_string(Base32, true);
|
||||||
}
|
}
|
||||||
}, ca);
|
}, raw);
|
||||||
}
|
}
|
||||||
|
|
||||||
std::string renderContentAddressMethod(ContentAddressMethod cam)
|
std::string ContentAddressMethod::render() const
|
||||||
{
|
{
|
||||||
return std::visit(overloaded {
|
return std::visit(overloaded {
|
||||||
[](TextHashMethod & th) {
|
[](const TextHashMethod & th) {
|
||||||
return std::string{"text:"} + printHashType(htSHA256);
|
return std::string{"text:"} + printHashType(htSHA256);
|
||||||
},
|
},
|
||||||
[](FixedOutputHashMethod & fshm) {
|
[](const FixedOutputHashMethod & fshm) {
|
||||||
return "fixed:" + makeFileIngestionPrefix(fshm.fileIngestionMethod) + printHashType(fshm.hashType);
|
return "fixed:" + makeFileIngestionPrefix(fshm.fileIngestionMethod) + printHashType(fshm.hashType);
|
||||||
}
|
}
|
||||||
}, cam);
|
}, raw);
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/**
|
||||||
Parses content address strings up to the hash.
|
* Parses content address strings up to the hash.
|
||||||
*/
|
*/
|
||||||
static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & rest)
|
static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & rest)
|
||||||
{
|
{
|
||||||
|
@ -97,7 +93,7 @@ static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & r
|
||||||
throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix);
|
throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix);
|
||||||
}
|
}
|
||||||
|
|
||||||
ContentAddress parseContentAddress(std::string_view rawCa) {
|
ContentAddress ContentAddress::parse(std::string_view rawCa) {
|
||||||
auto rest = rawCa;
|
auto rest = rawCa;
|
||||||
|
|
||||||
ContentAddressMethod caMethod = parseContentAddressMethodPrefix(rest);
|
ContentAddressMethod caMethod = parseContentAddressMethodPrefix(rest);
|
||||||
|
@ -115,10 +111,10 @@ ContentAddress parseContentAddress(std::string_view rawCa) {
|
||||||
.hash = Hash::parseNonSRIUnprefixed(rest, std::move(fohMethod.hashType)),
|
.hash = Hash::parseNonSRIUnprefixed(rest, std::move(fohMethod.hashType)),
|
||||||
});
|
});
|
||||||
},
|
},
|
||||||
}, caMethod);
|
}, caMethod.raw);
|
||||||
}
|
}
|
||||||
|
|
||||||
ContentAddressMethod parseContentAddressMethod(std::string_view caMethod)
|
ContentAddressMethod ContentAddressMethod::parse(std::string_view caMethod)
|
||||||
{
|
{
|
||||||
std::string asPrefix = std::string{caMethod} + ":";
|
std::string asPrefix = std::string{caMethod} + ":";
|
||||||
// parseContentAddressMethodPrefix takes its argument by reference
|
// parseContentAddressMethodPrefix takes its argument by reference
|
||||||
|
@ -126,26 +122,55 @@ ContentAddressMethod parseContentAddressMethod(std::string_view caMethod)
|
||||||
return parseContentAddressMethodPrefix(asPrefixView);
|
return parseContentAddressMethodPrefix(asPrefixView);
|
||||||
}
|
}
|
||||||
|
|
||||||
std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt)
|
std::optional<ContentAddress> ContentAddress::parseOpt(std::string_view rawCaOpt)
|
||||||
{
|
{
|
||||||
return rawCaOpt == "" ? std::optional<ContentAddress>() : parseContentAddress(rawCaOpt);
|
return rawCaOpt == ""
|
||||||
|
? std::nullopt
|
||||||
|
: std::optional { ContentAddress::parse(rawCaOpt) };
|
||||||
};
|
};
|
||||||
|
|
||||||
std::string renderContentAddress(std::optional<ContentAddress> ca)
|
std::string renderContentAddress(std::optional<ContentAddress> ca)
|
||||||
{
|
{
|
||||||
return ca ? renderContentAddress(*ca) : "";
|
return ca ? ca->render() : "";
|
||||||
}
|
}
|
||||||
|
|
||||||
Hash getContentAddressHash(const ContentAddress & ca)
|
const Hash & ContentAddress::getHash() const
|
||||||
{
|
{
|
||||||
return std::visit(overloaded {
|
return std::visit(overloaded {
|
||||||
[](const TextHash & th) {
|
[](const TextHash & th) -> auto & {
|
||||||
return th.hash;
|
return th.hash;
|
||||||
},
|
},
|
||||||
[](const FixedOutputHash & fsh) {
|
[](const FixedOutputHash & fsh) -> auto & {
|
||||||
return fsh.hash;
|
return fsh.hash;
|
||||||
|
},
|
||||||
|
}, raw);
|
||||||
}
|
}
|
||||||
}, ca);
|
|
||||||
|
bool StoreReferences::empty() const
|
||||||
|
{
|
||||||
|
return !self && others.empty();
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t StoreReferences::size() const
|
||||||
|
{
|
||||||
|
return (self ? 1 : 0) + others.size();
|
||||||
|
}
|
||||||
|
|
||||||
|
ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const ContentAddress & ca) {
|
||||||
|
return std::visit(overloaded {
|
||||||
|
[&](const TextHash & h) -> ContentAddressWithReferences {
|
||||||
|
return TextInfo {
|
||||||
|
.hash = h,
|
||||||
|
.references = {},
|
||||||
|
};
|
||||||
|
},
|
||||||
|
[&](const FixedOutputHash & h) -> ContentAddressWithReferences {
|
||||||
|
return FixedOutputInfo {
|
||||||
|
.hash = h,
|
||||||
|
.references = {},
|
||||||
|
};
|
||||||
|
},
|
||||||
|
}, ca.raw);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,12 +3,30 @@
|
||||||
|
|
||||||
#include <variant>
|
#include <variant>
|
||||||
#include "hash.hh"
|
#include "hash.hh"
|
||||||
|
#include "path.hh"
|
||||||
#include "comparator.hh"
|
#include "comparator.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Content addressing method
|
||||||
|
*/
|
||||||
|
|
||||||
|
/* We only have one way to hash text with references, so this is a single-value
|
||||||
|
type, mainly useful with std::variant.
|
||||||
|
*/
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* An enumeration of the ways we can serialize file system objects.
|
* The single way we can serialize "text" file system objects.
|
||||||
|
*
|
||||||
|
* Somewhat obscure, used by \ref Derivation derivations and
|
||||||
|
* `builtins.toFile` currently.
|
||||||
|
*/
|
||||||
|
struct TextHashMethod : std::monostate { };
|
||||||
|
|
||||||
|
/**
|
||||||
|
* An enumeration of the main ways we can serialize file system
|
||||||
|
* objects.
|
||||||
*/
|
*/
|
||||||
enum struct FileIngestionMethod : uint8_t {
|
enum struct FileIngestionMethod : uint8_t {
|
||||||
/**
|
/**
|
||||||
|
@ -22,6 +40,53 @@ enum struct FileIngestionMethod : uint8_t {
|
||||||
Recursive = true
|
Recursive = true
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Compute the prefix to the hash algorithm which indicates how the
|
||||||
|
* files were ingested.
|
||||||
|
*/
|
||||||
|
std::string makeFileIngestionPrefix(FileIngestionMethod m);
|
||||||
|
|
||||||
|
struct FixedOutputHashMethod {
|
||||||
|
FileIngestionMethod fileIngestionMethod;
|
||||||
|
HashType hashType;
|
||||||
|
|
||||||
|
GENERATE_CMP(FixedOutputHashMethod, me->fileIngestionMethod, me->hashType);
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* An enumeration of all the ways we can serialize file system objects.
|
||||||
|
*
|
||||||
|
* Just the type of a content address. Combine with the hash itself, and
|
||||||
|
* we have a `ContentAddress` as defined below. Combine that, in turn,
|
||||||
|
* with info on references, and we have `ContentAddressWithReferences`,
|
||||||
|
* as defined further below.
|
||||||
|
*/
|
||||||
|
struct ContentAddressMethod
|
||||||
|
{
|
||||||
|
typedef std::variant<
|
||||||
|
TextHashMethod,
|
||||||
|
FixedOutputHashMethod
|
||||||
|
> Raw;
|
||||||
|
|
||||||
|
Raw raw;
|
||||||
|
|
||||||
|
GENERATE_CMP(ContentAddressMethod, me->raw);
|
||||||
|
|
||||||
|
/* The moral equivalent of `using Raw::Raw;` */
|
||||||
|
ContentAddressMethod(auto &&... arg)
|
||||||
|
: raw(std::forward<decltype(arg)>(arg)...)
|
||||||
|
{ }
|
||||||
|
|
||||||
|
static ContentAddressMethod parse(std::string_view rawCaMethod);
|
||||||
|
|
||||||
|
std::string render() const;
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Mini content address
|
||||||
|
*/
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Somewhat obscure, used by \ref Derivation derivations and
|
* Somewhat obscure, used by \ref Derivation derivations and
|
||||||
* `builtins.toFile` currently.
|
* `builtins.toFile` currently.
|
||||||
|
@ -36,7 +101,7 @@ struct TextHash {
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* For path computed by makeFixedOutputPath.
|
* Used by most store objects that are content-addressed.
|
||||||
*/
|
*/
|
||||||
struct FixedOutputHash {
|
struct FixedOutputHash {
|
||||||
/**
|
/**
|
||||||
|
@ -65,41 +130,96 @@ struct FixedOutputHash {
|
||||||
* - ‘fixed:<r?>:<ht>:<h>’: For paths computed by
|
* - ‘fixed:<r?>:<ht>:<h>’: For paths computed by
|
||||||
* Store::makeFixedOutputPath() / Store::addToStore().
|
* Store::makeFixedOutputPath() / Store::addToStore().
|
||||||
*/
|
*/
|
||||||
|
struct ContentAddress
|
||||||
|
{
|
||||||
typedef std::variant<
|
typedef std::variant<
|
||||||
TextHash,
|
TextHash,
|
||||||
FixedOutputHash
|
FixedOutputHash
|
||||||
> ContentAddress;
|
> Raw;
|
||||||
|
|
||||||
/**
|
Raw raw;
|
||||||
* Compute the prefix to the hash algorithm which indicates how the
|
|
||||||
* files were ingested.
|
GENERATE_CMP(ContentAddress, me->raw);
|
||||||
*/
|
|
||||||
std::string makeFileIngestionPrefix(const FileIngestionMethod m);
|
/* The moral equivalent of `using Raw::Raw;` */
|
||||||
|
ContentAddress(auto &&... arg)
|
||||||
|
: raw(std::forward<decltype(arg)>(arg)...)
|
||||||
|
{ }
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Compute the content-addressability assertion (ValidPathInfo::ca) for
|
* Compute the content-addressability assertion (ValidPathInfo::ca) for
|
||||||
* paths created by Store::makeFixedOutputPath() / Store::addToStore().
|
* paths created by Store::makeFixedOutputPath() / Store::addToStore().
|
||||||
*/
|
*/
|
||||||
std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash);
|
std::string render() const;
|
||||||
|
|
||||||
std::string renderContentAddress(ContentAddress ca);
|
static ContentAddress parse(std::string_view rawCa);
|
||||||
|
|
||||||
|
static std::optional<ContentAddress> parseOpt(std::string_view rawCaOpt);
|
||||||
|
|
||||||
|
const Hash & getHash() const;
|
||||||
|
};
|
||||||
|
|
||||||
std::string renderContentAddress(std::optional<ContentAddress> ca);
|
std::string renderContentAddress(std::optional<ContentAddress> ca);
|
||||||
|
|
||||||
ContentAddress parseContentAddress(std::string_view rawCa);
|
|
||||||
|
|
||||||
std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt);
|
|
||||||
|
|
||||||
Hash getContentAddressHash(const ContentAddress & ca);
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
We only have one way to hash text with references, so this is single-value
|
* Full content address
|
||||||
type is only useful in std::variant.
|
*
|
||||||
|
* See the schema for store paths in store-api.cc
|
||||||
*/
|
*/
|
||||||
struct TextHashMethod { };
|
|
||||||
struct FixedOutputHashMethod {
|
/**
|
||||||
FileIngestionMethod fileIngestionMethod;
|
* A set of references to other store objects.
|
||||||
HashType hashType;
|
*
|
||||||
|
* References to other store objects are tracked with store paths, self
|
||||||
|
* references however are tracked with a boolean.
|
||||||
|
*/
|
||||||
|
struct StoreReferences {
|
||||||
|
/**
|
||||||
|
* References to other store objects
|
||||||
|
*/
|
||||||
|
StorePathSet others;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reference to this store object
|
||||||
|
*/
|
||||||
|
bool self = false;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return true iff no references, i.e. others is empty and self is
|
||||||
|
* false.
|
||||||
|
*/
|
||||||
|
bool empty() const;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns the numbers of references, i.e. the size of others + 1
|
||||||
|
* iff self is true.
|
||||||
|
*/
|
||||||
|
size_t size() const;
|
||||||
|
|
||||||
|
GENERATE_CMP(StoreReferences, me->self, me->others);
|
||||||
|
};
|
||||||
|
|
||||||
|
// This matches the additional info that we need for makeTextPath
|
||||||
|
struct TextInfo {
|
||||||
|
TextHash hash;
|
||||||
|
/**
|
||||||
|
* References to other store objects only; self references
|
||||||
|
* disallowed
|
||||||
|
*/
|
||||||
|
StorePathSet references;
|
||||||
|
|
||||||
|
GENERATE_CMP(TextInfo, me->hash, me->references);
|
||||||
|
};
|
||||||
|
|
||||||
|
struct FixedOutputInfo {
|
||||||
|
FixedOutputHash hash;
|
||||||
|
/**
|
||||||
|
* References to other store objects or this one.
|
||||||
|
*/
|
||||||
|
StoreReferences references;
|
||||||
|
|
||||||
|
GENERATE_CMP(FixedOutputInfo, me->hash, me->references);
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -107,13 +227,27 @@ struct FixedOutputHashMethod {
|
||||||
*
|
*
|
||||||
* A ContentAddress without a Hash.
|
* A ContentAddress without a Hash.
|
||||||
*/
|
*/
|
||||||
|
struct ContentAddressWithReferences
|
||||||
|
{
|
||||||
typedef std::variant<
|
typedef std::variant<
|
||||||
TextHashMethod,
|
TextInfo,
|
||||||
FixedOutputHashMethod
|
FixedOutputInfo
|
||||||
> ContentAddressMethod;
|
> Raw;
|
||||||
|
|
||||||
ContentAddressMethod parseContentAddressMethod(std::string_view rawCaMethod);
|
Raw raw;
|
||||||
|
|
||||||
std::string renderContentAddressMethod(ContentAddressMethod caMethod);
|
GENERATE_CMP(ContentAddressWithReferences, me->raw);
|
||||||
|
|
||||||
|
/* The moral equivalent of `using Raw::Raw;` */
|
||||||
|
ContentAddressWithReferences(auto &&... arg)
|
||||||
|
: raw(std::forward<decltype(arg)>(arg)...)
|
||||||
|
{ }
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a ContentAddressWithReferences from a mere ContentAddress, by
|
||||||
|
* assuming no references in all cases.
|
||||||
|
*/
|
||||||
|
static ContentAddressWithReferences withoutRefs(const ContentAddress &);
|
||||||
|
};
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -401,21 +401,21 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
||||||
logger->startWork();
|
logger->startWork();
|
||||||
auto pathInfo = [&]() {
|
auto pathInfo = [&]() {
|
||||||
// NB: FramedSource must be out of scope before logger->stopWork();
|
// NB: FramedSource must be out of scope before logger->stopWork();
|
||||||
ContentAddressMethod contentAddressMethod = parseContentAddressMethod(camStr);
|
ContentAddressMethod contentAddressMethod = ContentAddressMethod::parse(camStr);
|
||||||
FramedSource source(from);
|
FramedSource source(from);
|
||||||
// TODO this is essentially RemoteStore::addCAToStore. Move it up to Store.
|
// TODO this is essentially RemoteStore::addCAToStore. Move it up to Store.
|
||||||
return std::visit(overloaded {
|
return std::visit(overloaded {
|
||||||
[&](TextHashMethod &) {
|
[&](const TextHashMethod &) {
|
||||||
// We could stream this by changing Store
|
// We could stream this by changing Store
|
||||||
std::string contents = source.drain();
|
std::string contents = source.drain();
|
||||||
auto path = store->addTextToStore(name, contents, refs, repair);
|
auto path = store->addTextToStore(name, contents, refs, repair);
|
||||||
return store->queryPathInfo(path);
|
return store->queryPathInfo(path);
|
||||||
},
|
},
|
||||||
[&](FixedOutputHashMethod & fohm) {
|
[&](const FixedOutputHashMethod & fohm) {
|
||||||
auto path = store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair, refs);
|
auto path = store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair, refs);
|
||||||
return store->queryPathInfo(path);
|
return store->queryPathInfo(path);
|
||||||
},
|
},
|
||||||
}, contentAddressMethod);
|
}, contentAddressMethod.raw);
|
||||||
}();
|
}();
|
||||||
logger->stopWork();
|
logger->stopWork();
|
||||||
|
|
||||||
|
@ -880,7 +880,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
||||||
info.references = worker_proto::read(*store, from, Phantom<StorePathSet> {});
|
info.references = worker_proto::read(*store, from, Phantom<StorePathSet> {});
|
||||||
from >> info.registrationTime >> info.narSize >> info.ultimate;
|
from >> info.registrationTime >> info.narSize >> info.ultimate;
|
||||||
info.sigs = readStrings<StringSet>(from);
|
info.sigs = readStrings<StringSet>(from);
|
||||||
info.ca = parseContentAddressOpt(readString(from));
|
info.ca = ContentAddress::parseOpt(readString(from));
|
||||||
from >> repair >> dontCheckSigs;
|
from >> repair >> dontCheckSigs;
|
||||||
if (!trusted && dontCheckSigs)
|
if (!trusted && dontCheckSigs)
|
||||||
dontCheckSigs = false;
|
dontCheckSigs = false;
|
||||||
|
|
|
@ -36,8 +36,8 @@ std::optional<StorePath> DerivationOutput::path(const Store & store, std::string
|
||||||
StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const
|
StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const
|
||||||
{
|
{
|
||||||
return store.makeFixedOutputPath(
|
return store.makeFixedOutputPath(
|
||||||
hash.method, hash.hash,
|
outputPathName(drvName, outputName),
|
||||||
outputPathName(drvName, outputName));
|
{ hash, {} });
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -942,7 +942,7 @@ void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const
|
||||||
envHasRightPath(doia.path, i.first);
|
envHasRightPath(doia.path, i.first);
|
||||||
},
|
},
|
||||||
[&](const DerivationOutput::CAFixed & dof) {
|
[&](const DerivationOutput::CAFixed & dof) {
|
||||||
StorePath path = store.makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName);
|
StorePath path = store.makeFixedOutputPath(drvName, { dof.hash, {} });
|
||||||
envHasRightPath(path, i.first);
|
envHasRightPath(path, i.first);
|
||||||
},
|
},
|
||||||
[&](const DerivationOutput::CAFloating &) {
|
[&](const DerivationOutput::CAFloating &) {
|
||||||
|
|
|
@ -156,7 +156,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
|
||||||
throw Error("NAR hash is now mandatory");
|
throw Error("NAR hash is now mandatory");
|
||||||
info->narHash = Hash::parseAnyPrefixed(s);
|
info->narHash = Hash::parseAnyPrefixed(s);
|
||||||
}
|
}
|
||||||
info->ca = parseContentAddressOpt(readString(conn->from));
|
info->ca = ContentAddress::parseOpt(readString(conn->from));
|
||||||
info->sigs = readStrings<StringSet>(conn->from);
|
info->sigs = readStrings<StringSet>(conn->from);
|
||||||
|
|
||||||
auto s = readString(conn->from);
|
auto s = readString(conn->from);
|
||||||
|
|
|
@ -710,6 +710,7 @@ void canonicalisePathMetaData(const Path & path,
|
||||||
canonicalisePathMetaData(path, uidRange, inodesSeen);
|
canonicalisePathMetaData(path, uidRange, inodesSeen);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs)
|
void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs)
|
||||||
{
|
{
|
||||||
experimentalFeatureSettings.require(Xp::CaDerivations);
|
experimentalFeatureSettings.require(Xp::CaDerivations);
|
||||||
|
@ -888,7 +889,7 @@ std::shared_ptr<const ValidPathInfo> LocalStore::queryPathInfoInternal(State & s
|
||||||
if (s) info->sigs = tokenizeString<StringSet>(s, " ");
|
if (s) info->sigs = tokenizeString<StringSet>(s, " ");
|
||||||
|
|
||||||
s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 7);
|
s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 7);
|
||||||
if (s) info->ca = parseContentAddressOpt(s);
|
if (s) info->ca = ContentAddress::parseOpt(s);
|
||||||
|
|
||||||
/* Get the references. */
|
/* Get the references. */
|
||||||
auto useQueryReferences(state.stmts->QueryReferences.use()(info->id));
|
auto useQueryReferences(state.stmts->QueryReferences.use()(info->id));
|
||||||
|
@ -1221,7 +1222,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
|
||||||
printStorePath(info.path), info.narSize, hashResult.second);
|
printStorePath(info.path), info.narSize, hashResult.second);
|
||||||
|
|
||||||
if (info.ca) {
|
if (info.ca) {
|
||||||
if (auto foHash = std::get_if<FixedOutputHash>(&*info.ca)) {
|
if (auto foHash = std::get_if<FixedOutputHash>(&info.ca->raw)) {
|
||||||
auto actualFoHash = hashCAPath(
|
auto actualFoHash = hashCAPath(
|
||||||
foHash->method,
|
foHash->method,
|
||||||
foHash->hash.type,
|
foHash->hash.type,
|
||||||
|
@ -1234,7 +1235,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
|
||||||
actualFoHash.hash.to_string(Base32, true));
|
actualFoHash.hash.to_string(Base32, true));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (auto textHash = std::get_if<TextHash>(&*info.ca)) {
|
if (auto textHash = std::get_if<TextHash>(&info.ca->raw)) {
|
||||||
auto actualTextHash = hashString(htSHA256, readFile(realPath));
|
auto actualTextHash = hashString(htSHA256, readFile(realPath));
|
||||||
if (textHash->hash != actualTextHash) {
|
if (textHash->hash != actualTextHash) {
|
||||||
throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s",
|
throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s",
|
||||||
|
@ -1320,7 +1321,19 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name
|
||||||
|
|
||||||
auto [hash, size] = hashSink->finish();
|
auto [hash, size] = hashSink->finish();
|
||||||
|
|
||||||
auto dstPath = makeFixedOutputPath(method, hash, name, references);
|
ContentAddressWithReferences desc = FixedOutputInfo {
|
||||||
|
.hash = {
|
||||||
|
.method = method,
|
||||||
|
.hash = hash,
|
||||||
|
},
|
||||||
|
.references = {
|
||||||
|
.others = references,
|
||||||
|
// caller is not capable of creating a self-reference, because this is content-addressed without modulus
|
||||||
|
.self = false,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
auto dstPath = makeFixedOutputPathFromCA(name, desc);
|
||||||
|
|
||||||
addTempRoot(dstPath);
|
addTempRoot(dstPath);
|
||||||
|
|
||||||
|
@ -1364,10 +1377,13 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name
|
||||||
|
|
||||||
optimisePath(realPath, repair);
|
optimisePath(realPath, repair);
|
||||||
|
|
||||||
ValidPathInfo info { dstPath, narHash.first };
|
ValidPathInfo info {
|
||||||
|
*this,
|
||||||
|
name,
|
||||||
|
std::move(desc),
|
||||||
|
narHash.first
|
||||||
|
};
|
||||||
info.narSize = narHash.second;
|
info.narSize = narHash.second;
|
||||||
info.references = references;
|
|
||||||
info.ca = FixedOutputHash { .method = method, .hash = hash };
|
|
||||||
registerValidPath(info);
|
registerValidPath(info);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1384,7 +1400,10 @@ StorePath LocalStore::addTextToStore(
|
||||||
const StorePathSet & references, RepairFlag repair)
|
const StorePathSet & references, RepairFlag repair)
|
||||||
{
|
{
|
||||||
auto hash = hashString(htSHA256, s);
|
auto hash = hashString(htSHA256, s);
|
||||||
auto dstPath = makeTextPath(name, hash, references);
|
auto dstPath = makeTextPath(name, TextInfo {
|
||||||
|
{ .hash = hash },
|
||||||
|
references,
|
||||||
|
});
|
||||||
|
|
||||||
addTempRoot(dstPath);
|
addTempRoot(dstPath);
|
||||||
|
|
||||||
|
|
|
@ -27,18 +27,17 @@ std::map<StorePath, StorePath> makeContentAddressed(
|
||||||
|
|
||||||
StringMap rewrites;
|
StringMap rewrites;
|
||||||
|
|
||||||
StorePathSet references;
|
StoreReferences refs;
|
||||||
bool hasSelfReference = false;
|
|
||||||
for (auto & ref : oldInfo->references) {
|
for (auto & ref : oldInfo->references) {
|
||||||
if (ref == path)
|
if (ref == path)
|
||||||
hasSelfReference = true;
|
refs.self = true;
|
||||||
else {
|
else {
|
||||||
auto i = remappings.find(ref);
|
auto i = remappings.find(ref);
|
||||||
auto replacement = i != remappings.end() ? i->second : ref;
|
auto replacement = i != remappings.end() ? i->second : ref;
|
||||||
// FIXME: warn about unremapped paths?
|
// FIXME: warn about unremapped paths?
|
||||||
if (replacement != ref)
|
if (replacement != ref)
|
||||||
rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement));
|
rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement));
|
||||||
references.insert(std::move(replacement));
|
refs.others.insert(std::move(replacement));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -49,24 +48,28 @@ std::map<StorePath, StorePath> makeContentAddressed(
|
||||||
|
|
||||||
auto narModuloHash = hashModuloSink.finish().first;
|
auto narModuloHash = hashModuloSink.finish().first;
|
||||||
|
|
||||||
auto dstPath = dstStore.makeFixedOutputPath(
|
ValidPathInfo info {
|
||||||
FileIngestionMethod::Recursive, narModuloHash, path.name(), references, hasSelfReference);
|
dstStore,
|
||||||
|
path.name(),
|
||||||
|
FixedOutputInfo {
|
||||||
|
.hash = {
|
||||||
|
.method = FileIngestionMethod::Recursive,
|
||||||
|
.hash = narModuloHash,
|
||||||
|
},
|
||||||
|
.references = std::move(refs),
|
||||||
|
},
|
||||||
|
Hash::dummy,
|
||||||
|
};
|
||||||
|
|
||||||
printInfo("rewriting '%s' to '%s'", pathS, srcStore.printStorePath(dstPath));
|
printInfo("rewriting '%s' to '%s'", pathS, dstStore.printStorePath(info.path));
|
||||||
|
|
||||||
StringSink sink2;
|
StringSink sink2;
|
||||||
RewritingSink rsink2(oldHashPart, std::string(dstPath.hashPart()), sink2);
|
RewritingSink rsink2(oldHashPart, std::string(info.path.hashPart()), sink2);
|
||||||
rsink2(sink.s);
|
rsink2(sink.s);
|
||||||
rsink2.flush();
|
rsink2.flush();
|
||||||
|
|
||||||
ValidPathInfo info { dstPath, hashString(htSHA256, sink2.s) };
|
info.narHash = hashString(htSHA256, sink2.s);
|
||||||
info.references = std::move(references);
|
|
||||||
if (hasSelfReference) info.references.insert(info.path);
|
|
||||||
info.narSize = sink.s.size();
|
info.narSize = sink.s.size();
|
||||||
info.ca = FixedOutputHash {
|
|
||||||
.method = FileIngestionMethod::Recursive,
|
|
||||||
.hash = narModuloHash,
|
|
||||||
};
|
|
||||||
|
|
||||||
StringSource source(sink2.s);
|
StringSource source(sink2.s);
|
||||||
dstStore.addToStore(info, source);
|
dstStore.addToStore(info, source);
|
||||||
|
|
|
@ -273,7 +273,7 @@ public:
|
||||||
narInfo->deriver = StorePath(queryNAR.getStr(9));
|
narInfo->deriver = StorePath(queryNAR.getStr(9));
|
||||||
for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(10), " "))
|
for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(10), " "))
|
||||||
narInfo->sigs.insert(sig);
|
narInfo->sigs.insert(sig);
|
||||||
narInfo->ca = parseContentAddressOpt(queryNAR.getStr(11));
|
narInfo->ca = ContentAddress::parseOpt(queryNAR.getStr(11));
|
||||||
|
|
||||||
return {oValid, narInfo};
|
return {oValid, narInfo};
|
||||||
});
|
});
|
||||||
|
|
|
@ -74,7 +74,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
|
||||||
else if (name == "CA") {
|
else if (name == "CA") {
|
||||||
if (ca) throw corrupt();
|
if (ca) throw corrupt();
|
||||||
// FIXME: allow blank ca or require skipping field?
|
// FIXME: allow blank ca or require skipping field?
|
||||||
ca = parseContentAddressOpt(value);
|
ca = ContentAddress::parseOpt(value);
|
||||||
}
|
}
|
||||||
|
|
||||||
pos = eol + 1;
|
pos = eol + 1;
|
||||||
|
|
|
@ -17,6 +17,9 @@ struct NarInfo : ValidPathInfo
|
||||||
uint64_t fileSize = 0;
|
uint64_t fileSize = 0;
|
||||||
|
|
||||||
NarInfo() = delete;
|
NarInfo() = delete;
|
||||||
|
NarInfo(const Store & store, std::string && name, ContentAddressWithReferences && ca, Hash narHash)
|
||||||
|
: ValidPathInfo(store, std::move(name), std::move(ca), narHash)
|
||||||
|
{ }
|
||||||
NarInfo(StorePath && path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { }
|
NarInfo(StorePath && path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { }
|
||||||
NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { }
|
NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { }
|
||||||
NarInfo(const Store & store, const std::string & s, const std::string & whence);
|
NarInfo(const Store & store, const std::string & s, const std::string & whence);
|
||||||
|
|
|
@ -21,25 +21,45 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey)
|
||||||
sigs.insert(secretKey.signDetached(fingerprint(store)));
|
sigs.insert(secretKey.signDetached(fingerprint(store)));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
std::optional<ContentAddressWithReferences> ValidPathInfo::contentAddressWithReferences() const
|
||||||
bool ValidPathInfo::isContentAddressed(const Store & store) const
|
|
||||||
{
|
{
|
||||||
if (! ca) return false;
|
if (! ca)
|
||||||
|
return std::nullopt;
|
||||||
|
|
||||||
auto caPath = std::visit(overloaded {
|
return std::visit(overloaded {
|
||||||
[&](const TextHash & th) {
|
[&](const TextHash & th) -> ContentAddressWithReferences {
|
||||||
return store.makeTextPath(path.name(), th.hash, references);
|
assert(references.count(path) == 0);
|
||||||
|
return TextInfo {
|
||||||
|
.hash = th,
|
||||||
|
.references = references,
|
||||||
|
};
|
||||||
},
|
},
|
||||||
[&](const FixedOutputHash & fsh) {
|
[&](const FixedOutputHash & foh) -> ContentAddressWithReferences {
|
||||||
auto refs = references;
|
auto refs = references;
|
||||||
bool hasSelfReference = false;
|
bool hasSelfReference = false;
|
||||||
if (refs.count(path)) {
|
if (refs.count(path)) {
|
||||||
hasSelfReference = true;
|
hasSelfReference = true;
|
||||||
refs.erase(path);
|
refs.erase(path);
|
||||||
}
|
}
|
||||||
return store.makeFixedOutputPath(fsh.method, fsh.hash, path.name(), refs, hasSelfReference);
|
return FixedOutputInfo {
|
||||||
|
.hash = foh,
|
||||||
|
.references = {
|
||||||
|
.others = std::move(refs),
|
||||||
|
.self = hasSelfReference,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
},
|
||||||
|
}, ca->raw);
|
||||||
}
|
}
|
||||||
}, *ca);
|
|
||||||
|
bool ValidPathInfo::isContentAddressed(const Store & store) const
|
||||||
|
{
|
||||||
|
auto fullCaOpt = contentAddressWithReferences();
|
||||||
|
|
||||||
|
if (! fullCaOpt)
|
||||||
|
return false;
|
||||||
|
|
||||||
|
auto caPath = store.makeFixedOutputPathFromCA(path.name(), *fullCaOpt);
|
||||||
|
|
||||||
bool res = caPath == path;
|
bool res = caPath == path;
|
||||||
|
|
||||||
|
@ -77,6 +97,29 @@ Strings ValidPathInfo::shortRefs() const
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
ValidPathInfo::ValidPathInfo(
|
||||||
|
const Store & store,
|
||||||
|
std::string_view name,
|
||||||
|
ContentAddressWithReferences && ca,
|
||||||
|
Hash narHash)
|
||||||
|
: path(store.makeFixedOutputPathFromCA(name, ca))
|
||||||
|
, narHash(narHash)
|
||||||
|
{
|
||||||
|
std::visit(overloaded {
|
||||||
|
[this](TextInfo && ti) {
|
||||||
|
this->references = std::move(ti.references);
|
||||||
|
this->ca = std::move((TextHash &&) ti);
|
||||||
|
},
|
||||||
|
[this](FixedOutputInfo && foi) {
|
||||||
|
this->references = std::move(foi.references.others);
|
||||||
|
if (foi.references.self)
|
||||||
|
this->references.insert(path);
|
||||||
|
this->ca = std::move((FixedOutputHash &&) foi);
|
||||||
|
},
|
||||||
|
}, std::move(ca).raw);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned int format)
|
ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned int format)
|
||||||
{
|
{
|
||||||
return read(source, store, format, store.parseStorePath(readString(source)));
|
return read(source, store, format, store.parseStorePath(readString(source)));
|
||||||
|
@ -93,7 +136,7 @@ ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned
|
||||||
if (format >= 16) {
|
if (format >= 16) {
|
||||||
source >> info.ultimate;
|
source >> info.ultimate;
|
||||||
info.sigs = readStrings<StringSet>(source);
|
info.sigs = readStrings<StringSet>(source);
|
||||||
info.ca = parseContentAddressOpt(readString(source));
|
info.ca = ContentAddress::parseOpt(readString(source));
|
||||||
}
|
}
|
||||||
return info;
|
return info;
|
||||||
}
|
}
|
||||||
|
|
|
@ -92,6 +92,13 @@ struct ValidPathInfo
|
||||||
|
|
||||||
void sign(const Store & store, const SecretKey & secretKey);
|
void sign(const Store & store, const SecretKey & secretKey);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return The `ContentAddressWithReferences` that determines the
|
||||||
|
* store path for a content-addressed store object, `std::nullopt`
|
||||||
|
* for an input-addressed store object.
|
||||||
|
*/
|
||||||
|
std::optional<ContentAddressWithReferences> contentAddressWithReferences() const;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @return true iff the path is verifiably content-addressed.
|
* @return true iff the path is verifiably content-addressed.
|
||||||
*/
|
*/
|
||||||
|
@ -118,6 +125,9 @@ struct ValidPathInfo
|
||||||
ValidPathInfo(StorePath && path, Hash narHash) : path(std::move(path)), narHash(narHash) { };
|
ValidPathInfo(StorePath && path, Hash narHash) : path(std::move(path)), narHash(narHash) { };
|
||||||
ValidPathInfo(const StorePath & path, Hash narHash) : path(path), narHash(narHash) { };
|
ValidPathInfo(const StorePath & path, Hash narHash) : path(path), narHash(narHash) { };
|
||||||
|
|
||||||
|
ValidPathInfo(const Store & store,
|
||||||
|
std::string_view name, ContentAddressWithReferences && ca, Hash narHash);
|
||||||
|
|
||||||
virtual ~ValidPathInfo() { }
|
virtual ~ValidPathInfo() { }
|
||||||
|
|
||||||
static ValidPathInfo read(Source & source, const Store & store, unsigned int format);
|
static ValidPathInfo read(Source & source, const Store & store, unsigned int format);
|
||||||
|
|
|
@ -78,7 +78,7 @@ void write(const Store & store, Sink & out, const std::optional<TrustedFlag> & o
|
||||||
|
|
||||||
ContentAddress read(const Store & store, Source & from, Phantom<ContentAddress> _)
|
ContentAddress read(const Store & store, Source & from, Phantom<ContentAddress> _)
|
||||||
{
|
{
|
||||||
return parseContentAddress(readString(from));
|
return ContentAddress::parse(readString(from));
|
||||||
}
|
}
|
||||||
|
|
||||||
void write(const Store & store, Sink & out, const ContentAddress & ca)
|
void write(const Store & store, Sink & out, const ContentAddress & ca)
|
||||||
|
@ -168,7 +168,7 @@ void write(const Store & store, Sink & out, const std::optional<StorePath> & sto
|
||||||
|
|
||||||
std::optional<ContentAddress> read(const Store & store, Source & from, Phantom<std::optional<ContentAddress>> _)
|
std::optional<ContentAddress> read(const Store & store, Source & from, Phantom<std::optional<ContentAddress>> _)
|
||||||
{
|
{
|
||||||
return parseContentAddressOpt(readString(from));
|
return ContentAddress::parseOpt(readString(from));
|
||||||
}
|
}
|
||||||
|
|
||||||
void write(const Store & store, Sink & out, const std::optional<ContentAddress> & caOpt)
|
void write(const Store & store, Sink & out, const std::optional<ContentAddress> & caOpt)
|
||||||
|
@ -586,7 +586,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
|
||||||
conn->to
|
conn->to
|
||||||
<< wopAddToStore
|
<< wopAddToStore
|
||||||
<< name
|
<< name
|
||||||
<< renderContentAddressMethod(caMethod);
|
<< caMethod.render();
|
||||||
worker_proto::write(*this, conn->to, references);
|
worker_proto::write(*this, conn->to, references);
|
||||||
conn->to << repair;
|
conn->to << repair;
|
||||||
|
|
||||||
|
@ -644,7 +644,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
}, caMethod);
|
}, caMethod.raw);
|
||||||
auto path = parseStorePath(readString(conn->from));
|
auto path = parseStorePath(readString(conn->from));
|
||||||
// Release our connection to prevent a deadlock in queryPathInfo().
|
// Release our connection to prevent a deadlock in queryPathInfo().
|
||||||
conn_.reset();
|
conn_.reset();
|
||||||
|
|
|
@ -7,6 +7,7 @@
|
||||||
#include "nar-info-disk-cache.hh"
|
#include "nar-info-disk-cache.hh"
|
||||||
#include "thread-pool.hh"
|
#include "thread-pool.hh"
|
||||||
#include "url.hh"
|
#include "url.hh"
|
||||||
|
#include "references.hh"
|
||||||
#include "archive.hh"
|
#include "archive.hh"
|
||||||
#include "callback.hh"
|
#include "callback.hh"
|
||||||
#include "remote-store.hh"
|
#include "remote-store.hh"
|
||||||
|
@ -98,10 +99,12 @@ StorePath Store::followLinksToStorePath(std::string_view path) const
|
||||||
silly, but it's done that way for compatibility). <id> is the
|
silly, but it's done that way for compatibility). <id> is the
|
||||||
name of the output (usually, "out").
|
name of the output (usually, "out").
|
||||||
|
|
||||||
<h2> = base-16 representation of a SHA-256 hash of:
|
<h2> = base-16 representation of a SHA-256 hash of <s2>
|
||||||
|
|
||||||
|
<s2> =
|
||||||
if <type> = "text:...":
|
if <type> = "text:...":
|
||||||
the string written to the resulting store path
|
the string written to the resulting store path
|
||||||
if <type> = "source":
|
if <type> = "source:...":
|
||||||
the serialisation of the path from which this store path is
|
the serialisation of the path from which this store path is
|
||||||
copied, as returned by hashPath()
|
copied, as returned by hashPath()
|
||||||
if <type> = "output:<id>":
|
if <type> = "output:<id>":
|
||||||
|
@ -162,63 +165,63 @@ StorePath Store::makeOutputPath(std::string_view id,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/* Stuff the references (if any) into the type. This is a bit
|
||||||
|
hacky, but we can't put them in, say, <s2> (per the grammar above)
|
||||||
|
since that would be ambiguous. */
|
||||||
static std::string makeType(
|
static std::string makeType(
|
||||||
const Store & store,
|
const Store & store,
|
||||||
std::string && type,
|
std::string && type,
|
||||||
const StorePathSet & references,
|
const StoreReferences & references)
|
||||||
bool hasSelfReference = false)
|
|
||||||
{
|
{
|
||||||
for (auto & i : references) {
|
for (auto & i : references.others) {
|
||||||
type += ":";
|
type += ":";
|
||||||
type += store.printStorePath(i);
|
type += store.printStorePath(i);
|
||||||
}
|
}
|
||||||
if (hasSelfReference) type += ":self";
|
if (references.self) type += ":self";
|
||||||
return std::move(type);
|
return std::move(type);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
StorePath Store::makeFixedOutputPath(
|
StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const
|
||||||
FileIngestionMethod method,
|
|
||||||
const Hash & hash,
|
|
||||||
std::string_view name,
|
|
||||||
const StorePathSet & references,
|
|
||||||
bool hasSelfReference) const
|
|
||||||
{
|
{
|
||||||
if (hash.type == htSHA256 && method == FileIngestionMethod::Recursive) {
|
if (info.hash.hash.type == htSHA256 && info.hash.method == FileIngestionMethod::Recursive) {
|
||||||
return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name);
|
return makeStorePath(makeType(*this, "source", info.references), info.hash.hash, name);
|
||||||
} else {
|
} else {
|
||||||
assert(references.empty());
|
assert(info.references.size() == 0);
|
||||||
return makeStorePath("output:out",
|
return makeStorePath("output:out",
|
||||||
hashString(htSHA256,
|
hashString(htSHA256,
|
||||||
"fixed:out:"
|
"fixed:out:"
|
||||||
+ makeFileIngestionPrefix(method)
|
+ makeFileIngestionPrefix(info.hash.method)
|
||||||
+ hash.to_string(Base16, true) + ":"),
|
+ info.hash.hash.to_string(Base16, true) + ":"),
|
||||||
name);
|
name);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
StorePath Store::makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca,
|
|
||||||
const StorePathSet & references, bool hasSelfReference) const
|
StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) const
|
||||||
|
{
|
||||||
|
assert(info.hash.hash.type == htSHA256);
|
||||||
|
return makeStorePath(
|
||||||
|
makeType(*this, "text", StoreReferences {
|
||||||
|
.others = info.references,
|
||||||
|
.self = false,
|
||||||
|
}),
|
||||||
|
info.hash.hash,
|
||||||
|
name);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
StorePath Store::makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const
|
||||||
{
|
{
|
||||||
// New template
|
// New template
|
||||||
return std::visit(overloaded {
|
return std::visit(overloaded {
|
||||||
[&](const TextHash & th) {
|
[&](const TextInfo & ti) {
|
||||||
return makeTextPath(name, th.hash, references);
|
return makeTextPath(name, ti);
|
||||||
},
|
},
|
||||||
[&](const FixedOutputHash & fsh) {
|
[&](const FixedOutputInfo & foi) {
|
||||||
return makeFixedOutputPath(fsh.method, fsh.hash, name, references, hasSelfReference);
|
return makeFixedOutputPath(name, foi);
|
||||||
}
|
}
|
||||||
}, ca);
|
}, ca.raw);
|
||||||
}
|
|
||||||
|
|
||||||
StorePath Store::makeTextPath(std::string_view name, const Hash & hash,
|
|
||||||
const StorePathSet & references) const
|
|
||||||
{
|
|
||||||
assert(hash.type == htSHA256);
|
|
||||||
/* Stuff the references (if any) into the type. This is a bit
|
|
||||||
hacky, but we can't put them in `s' since that would be
|
|
||||||
ambiguous. */
|
|
||||||
return makeStorePath(makeType(*this, "text", references), hash, name);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -228,7 +231,14 @@ std::pair<StorePath, Hash> Store::computeStorePathForPath(std::string_view name,
|
||||||
Hash h = method == FileIngestionMethod::Recursive
|
Hash h = method == FileIngestionMethod::Recursive
|
||||||
? hashPath(hashAlgo, srcPath, filter).first
|
? hashPath(hashAlgo, srcPath, filter).first
|
||||||
: hashFile(hashAlgo, srcPath);
|
: hashFile(hashAlgo, srcPath);
|
||||||
return std::make_pair(makeFixedOutputPath(method, h, name), h);
|
FixedOutputInfo caInfo {
|
||||||
|
.hash = {
|
||||||
|
.method = method,
|
||||||
|
.hash = h,
|
||||||
|
},
|
||||||
|
.references = {},
|
||||||
|
};
|
||||||
|
return std::make_pair(makeFixedOutputPath(name, caInfo), h);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -237,7 +247,10 @@ StorePath Store::computeStorePathForText(
|
||||||
std::string_view s,
|
std::string_view s,
|
||||||
const StorePathSet & references) const
|
const StorePathSet & references) const
|
||||||
{
|
{
|
||||||
return makeTextPath(name, hashString(htSHA256, s), references);
|
return makeTextPath(name, TextInfo {
|
||||||
|
{ .hash = hashString(htSHA256, s) },
|
||||||
|
references,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -425,11 +438,18 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
|
||||||
throw Error("hash mismatch for '%s'", srcPath);
|
throw Error("hash mismatch for '%s'", srcPath);
|
||||||
|
|
||||||
ValidPathInfo info {
|
ValidPathInfo info {
|
||||||
makeFixedOutputPath(method, hash, name),
|
*this,
|
||||||
|
name,
|
||||||
|
FixedOutputInfo {
|
||||||
|
.hash = {
|
||||||
|
.method = method,
|
||||||
|
.hash = hash,
|
||||||
|
},
|
||||||
|
.references = {},
|
||||||
|
},
|
||||||
narHash,
|
narHash,
|
||||||
};
|
};
|
||||||
info.narSize = narSize;
|
info.narSize = narSize;
|
||||||
info.ca = FixedOutputHash { .method = method, .hash = hash };
|
|
||||||
|
|
||||||
if (!isValidPath(info.path)) {
|
if (!isValidPath(info.path)) {
|
||||||
auto source = sinkToSource([&](Sink & scratchpadSink) {
|
auto source = sinkToSource([&](Sink & scratchpadSink) {
|
||||||
|
@ -521,7 +541,9 @@ void Store::querySubstitutablePathInfos(const StorePathCAMap & paths, Substituta
|
||||||
|
|
||||||
// Recompute store path so that we can use a different store root.
|
// Recompute store path so that we can use a different store root.
|
||||||
if (path.second) {
|
if (path.second) {
|
||||||
subPath = makeFixedOutputPathFromCA(path.first.name(), *path.second);
|
subPath = makeFixedOutputPathFromCA(
|
||||||
|
path.first.name(),
|
||||||
|
ContentAddressWithReferences::withoutRefs(*path.second));
|
||||||
if (sub->storeDir == storeDir)
|
if (sub->storeDir == storeDir)
|
||||||
assert(subPath == path.first);
|
assert(subPath == path.first);
|
||||||
if (subPath != path.first)
|
if (subPath != path.first)
|
||||||
|
@ -538,10 +560,11 @@ void Store::querySubstitutablePathInfos(const StorePathCAMap & paths, Substituta
|
||||||
auto narInfo = std::dynamic_pointer_cast<const NarInfo>(
|
auto narInfo = std::dynamic_pointer_cast<const NarInfo>(
|
||||||
std::shared_ptr<const ValidPathInfo>(info));
|
std::shared_ptr<const ValidPathInfo>(info));
|
||||||
infos.insert_or_assign(path.first, SubstitutablePathInfo{
|
infos.insert_or_assign(path.first, SubstitutablePathInfo{
|
||||||
info->deriver,
|
.deriver = info->deriver,
|
||||||
info->references,
|
.references = info->references,
|
||||||
narInfo ? narInfo->fileSize : 0,
|
.downloadSize = narInfo ? narInfo->fileSize : 0,
|
||||||
info->narSize});
|
.narSize = info->narSize,
|
||||||
|
});
|
||||||
} catch (InvalidPath &) {
|
} catch (InvalidPath &) {
|
||||||
} catch (SubstituterDisabled &) {
|
} catch (SubstituterDisabled &) {
|
||||||
} catch (Error & e) {
|
} catch (Error & e) {
|
||||||
|
@ -1025,7 +1048,9 @@ void copyStorePath(
|
||||||
// recompute store path on the chance dstStore does it differently
|
// recompute store path on the chance dstStore does it differently
|
||||||
if (info->ca && info->references.empty()) {
|
if (info->ca && info->references.empty()) {
|
||||||
auto info2 = make_ref<ValidPathInfo>(*info);
|
auto info2 = make_ref<ValidPathInfo>(*info);
|
||||||
info2->path = dstStore.makeFixedOutputPathFromCA(info->path.name(), *info->ca);
|
info2->path = dstStore.makeFixedOutputPathFromCA(
|
||||||
|
info->path.name(),
|
||||||
|
info->contentAddressWithReferences().value());
|
||||||
if (dstStore.storeDir == srcStore.storeDir)
|
if (dstStore.storeDir == srcStore.storeDir)
|
||||||
assert(info->path == info2->path);
|
assert(info->path == info2->path);
|
||||||
info = info2;
|
info = info2;
|
||||||
|
@ -1137,7 +1162,9 @@ std::map<StorePath, StorePath> copyPaths(
|
||||||
auto storePathForSrc = currentPathInfo.path;
|
auto storePathForSrc = currentPathInfo.path;
|
||||||
auto storePathForDst = storePathForSrc;
|
auto storePathForDst = storePathForSrc;
|
||||||
if (currentPathInfo.ca && currentPathInfo.references.empty()) {
|
if (currentPathInfo.ca && currentPathInfo.references.empty()) {
|
||||||
storePathForDst = dstStore.makeFixedOutputPathFromCA(storePathForSrc.name(), *currentPathInfo.ca);
|
storePathForDst = dstStore.makeFixedOutputPathFromCA(
|
||||||
|
currentPathInfo.path.name(),
|
||||||
|
currentPathInfo.contentAddressWithReferences().value());
|
||||||
if (dstStore.storeDir == srcStore.storeDir)
|
if (dstStore.storeDir == srcStore.storeDir)
|
||||||
assert(storePathForDst == storePathForSrc);
|
assert(storePathForDst == storePathForSrc);
|
||||||
if (storePathForDst != storePathForSrc)
|
if (storePathForDst != storePathForSrc)
|
||||||
|
|
|
@ -268,17 +268,11 @@ public:
|
||||||
StorePath makeOutputPath(std::string_view id,
|
StorePath makeOutputPath(std::string_view id,
|
||||||
const Hash & hash, std::string_view name) const;
|
const Hash & hash, std::string_view name) const;
|
||||||
|
|
||||||
StorePath makeFixedOutputPath(FileIngestionMethod method,
|
StorePath makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const;
|
||||||
const Hash & hash, std::string_view name,
|
|
||||||
const StorePathSet & references = {},
|
|
||||||
bool hasSelfReference = false) const;
|
|
||||||
|
|
||||||
StorePath makeTextPath(std::string_view name, const Hash & hash,
|
StorePath makeTextPath(std::string_view name, const TextInfo & info) const;
|
||||||
const StorePathSet & references = {}) const;
|
|
||||||
|
|
||||||
StorePath makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca,
|
StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const;
|
||||||
const StorePathSet & references = {},
|
|
||||||
bool hasSelfReference = false) const;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Preparatory part of addToStore().
|
* Preparatory part of addToStore().
|
||||||
|
|
|
@ -204,10 +204,10 @@ static void opAddFixed(Strings opFlags, Strings opArgs)
|
||||||
/* Hack to support caching in `nix-prefetch-url'. */
|
/* Hack to support caching in `nix-prefetch-url'. */
|
||||||
static void opPrintFixedPath(Strings opFlags, Strings opArgs)
|
static void opPrintFixedPath(Strings opFlags, Strings opArgs)
|
||||||
{
|
{
|
||||||
auto recursive = FileIngestionMethod::Flat;
|
auto method = FileIngestionMethod::Flat;
|
||||||
|
|
||||||
for (auto i : opFlags)
|
for (auto i : opFlags)
|
||||||
if (i == "--recursive") recursive = FileIngestionMethod::Recursive;
|
if (i == "--recursive") method = FileIngestionMethod::Recursive;
|
||||||
else throw UsageError("unknown flag '%1%'", i);
|
else throw UsageError("unknown flag '%1%'", i);
|
||||||
|
|
||||||
if (opArgs.size() != 3)
|
if (opArgs.size() != 3)
|
||||||
|
@ -218,7 +218,13 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs)
|
||||||
std::string hash = *i++;
|
std::string hash = *i++;
|
||||||
std::string name = *i++;
|
std::string name = *i++;
|
||||||
|
|
||||||
cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(recursive, Hash::parseAny(hash, hashAlgo), name)));
|
cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(name, FixedOutputInfo {
|
||||||
|
.hash = {
|
||||||
|
.method = method,
|
||||||
|
.hash = Hash::parseAny(hash, hashAlgo),
|
||||||
|
},
|
||||||
|
.references = {},
|
||||||
|
})));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -964,7 +970,7 @@ static void opServe(Strings opFlags, Strings opArgs)
|
||||||
info.references = worker_proto::read(*store, in, Phantom<StorePathSet> {});
|
info.references = worker_proto::read(*store, in, Phantom<StorePathSet> {});
|
||||||
in >> info.registrationTime >> info.narSize >> info.ultimate;
|
in >> info.registrationTime >> info.narSize >> info.ultimate;
|
||||||
info.sigs = readStrings<StringSet>(in);
|
info.sigs = readStrings<StringSet>(in);
|
||||||
info.ca = parseContentAddressOpt(readString(in));
|
info.ca = ContentAddress::parseOpt(readString(in));
|
||||||
|
|
||||||
if (info.narSize == 0)
|
if (info.narSize == 0)
|
||||||
throw Error("narInfo is too old and missing the narSize field");
|
throw Error("narInfo is too old and missing the narSize field");
|
||||||
|
|
|
@ -42,14 +42,18 @@ struct CmdAddToStore : MixDryRun, StoreCommand
|
||||||
}
|
}
|
||||||
|
|
||||||
ValidPathInfo info {
|
ValidPathInfo info {
|
||||||
store->makeFixedOutputPath(ingestionMethod, hash, *namePart),
|
*store,
|
||||||
|
std::move(*namePart),
|
||||||
|
FixedOutputInfo {
|
||||||
|
.hash = {
|
||||||
|
.method = std::move(ingestionMethod),
|
||||||
|
.hash = std::move(hash),
|
||||||
|
},
|
||||||
|
.references = {},
|
||||||
|
},
|
||||||
narHash,
|
narHash,
|
||||||
};
|
};
|
||||||
info.narSize = sink.s.size();
|
info.narSize = sink.s.size();
|
||||||
info.ca = std::optional { FixedOutputHash {
|
|
||||||
.method = ingestionMethod,
|
|
||||||
.hash = hash,
|
|
||||||
} };
|
|
||||||
|
|
||||||
if (!dryRun) {
|
if (!dryRun) {
|
||||||
auto source = StringSource(sink.s);
|
auto source = StringSource(sink.s);
|
||||||
|
|
|
@ -67,7 +67,13 @@ std::tuple<StorePath, Hash> prefetchFile(
|
||||||
the store. */
|
the store. */
|
||||||
if (expectedHash) {
|
if (expectedHash) {
|
||||||
hashType = expectedHash->type;
|
hashType = expectedHash->type;
|
||||||
storePath = store->makeFixedOutputPath(ingestionMethod, *expectedHash, *name);
|
storePath = store->makeFixedOutputPath(*name, FixedOutputInfo {
|
||||||
|
.hash = {
|
||||||
|
.method = ingestionMethod,
|
||||||
|
.hash = *expectedHash,
|
||||||
|
},
|
||||||
|
.references = {},
|
||||||
|
});
|
||||||
if (store->isValidPath(*storePath))
|
if (store->isValidPath(*storePath))
|
||||||
hash = expectedHash;
|
hash = expectedHash;
|
||||||
else
|
else
|
||||||
|
@ -118,7 +124,7 @@ std::tuple<StorePath, Hash> prefetchFile(
|
||||||
auto info = store->addToStoreSlow(*name, tmpFile, ingestionMethod, hashType, expectedHash);
|
auto info = store->addToStoreSlow(*name, tmpFile, ingestionMethod, hashType, expectedHash);
|
||||||
storePath = info.path;
|
storePath = info.path;
|
||||||
assert(info.ca);
|
assert(info.ca);
|
||||||
hash = getContentAddressHash(*info.ca);
|
hash = info.ca->getHash();
|
||||||
}
|
}
|
||||||
|
|
||||||
return {storePath.value(), hash.value()};
|
return {storePath.value(), hash.value()};
|
||||||
|
|
|
@ -200,12 +200,22 @@ struct ProfileManifest
|
||||||
auto narHash = hashString(htSHA256, sink.s);
|
auto narHash = hashString(htSHA256, sink.s);
|
||||||
|
|
||||||
ValidPathInfo info {
|
ValidPathInfo info {
|
||||||
store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "profile", references),
|
*store,
|
||||||
|
"profile",
|
||||||
|
FixedOutputInfo {
|
||||||
|
.hash = {
|
||||||
|
.method = FileIngestionMethod::Recursive,
|
||||||
|
.hash = narHash,
|
||||||
|
},
|
||||||
|
.references = {
|
||||||
|
.others = std::move(references),
|
||||||
|
// profiles never refer to themselves
|
||||||
|
.self = false,
|
||||||
|
},
|
||||||
|
},
|
||||||
narHash,
|
narHash,
|
||||||
};
|
};
|
||||||
info.references = std::move(references);
|
|
||||||
info.narSize = sink.s.size();
|
info.narSize = sink.s.size();
|
||||||
info.ca = FixedOutputHash { .method = FileIngestionMethod::Recursive, .hash = info.narHash };
|
|
||||||
|
|
||||||
StringSource source(sink.s);
|
StringSource source(sink.s);
|
||||||
store->addToStore(info, source);
|
store->addToStore(info, source);
|
||||||
|
|
Loading…
Reference in a new issue