CA derivations that depend on other CA derivations

Co-authored-by: Théophane Hufschmitt <regnat@users.noreply.github.com>
This commit is contained in:
John Ericson 2020-08-22 20:44:47 +00:00
parent e0b0e18905
commit 8eb73a8724
6 changed files with 189 additions and 28 deletions

View file

@ -984,6 +984,8 @@ private:
void tryLocalBuild(); void tryLocalBuild();
void buildDone(); void buildDone();
void resolvedFinished();
/* Is the build hook willing to perform the build? */ /* Is the build hook willing to perform the build? */
HookReply tryBuildHook(); HookReply tryBuildHook();
@ -1451,8 +1453,39 @@ void DerivationGoal::inputsRealised()
/* Determine the full set of input paths. */ /* Determine the full set of input paths. */
/* First, the input derivations. */ /* First, the input derivations. */
if (useDerivation) if (useDerivation) {
for (auto & [depDrvPath, wantedDepOutputs] : dynamic_cast<Derivation *>(drv.get())->inputDrvs) { auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
if (!fullDrv.inputDrvs.empty() && fullDrv.type() == DerivationType::CAFloating) {
/* We are be able to resolve this derivation based on the
now-known results of dependencies. If so, we become a stub goal
aliasing that resolved derivation goal */
Derivation drvResolved { fullDrv.resolve(worker.store) };
auto pathResolved = writeDerivation(worker.store, drvResolved);
/* Add to memotable to speed up downstream goal's queries with the
original derivation. */
drvPathResolutions.lock()->insert_or_assign(drvPath, pathResolved);
auto msg = fmt("Resolved derivation: '%s' -> '%s'",
worker.store.printStorePath(drvPath),
worker.store.printStorePath(pathResolved));
act = std::make_unique<Activity>(*logger, lvlInfo, actBuildWaiting, msg,
Logger::Fields {
worker.store.printStorePath(drvPath),
worker.store.printStorePath(pathResolved),
});
auto resolvedGoal = worker.makeDerivationGoal(
pathResolved, wantedOutputs,
buildMode == bmRepair ? bmRepair : bmNormal);
addWaitee(resolvedGoal);
state = &DerivationGoal::resolvedFinished;
return;
}
for (auto & [depDrvPath, wantedDepOutputs] : fullDrv.inputDrvs) {
/* Add the relevant output closures of the input derivation /* Add the relevant output closures of the input derivation
`i' as input paths. Only add the closures of output paths `i' as input paths. Only add the closures of output paths
that are specified as inputs. */ that are specified as inputs. */
@ -1472,6 +1505,7 @@ void DerivationGoal::inputsRealised()
worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath)); worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath));
} }
} }
}
/* Second, the input sources. */ /* Second, the input sources. */
worker.store.computeFSClosure(drv->inputSrcs, inputPaths); worker.store.computeFSClosure(drv->inputSrcs, inputPaths);
@ -1893,6 +1927,9 @@ void DerivationGoal::buildDone()
done(BuildResult::Built); done(BuildResult::Built);
} }
void DerivationGoal::resolvedFinished() {
done(BuildResult::Built);
}
HookReply DerivationGoal::tryBuildHook() HookReply DerivationGoal::tryBuildHook()
{ {
@ -4248,10 +4285,14 @@ void DerivationGoal::registerOutputs()
{ {
ValidPathInfos infos2; ValidPathInfos infos2;
for (auto & [outputName, newInfo] : infos) { for (auto & [outputName, newInfo] : infos) {
/* FIXME: we will want to track this mapping in the DB whether or
not we have a drv file. */
if (useDerivation) if (useDerivation)
worker.store.linkDeriverToPath(drvPath, outputName, newInfo.path); worker.store.linkDeriverToPath(drvPath, outputName, newInfo.path);
else {
/* Once a floating CA derivations reaches this point, it must
already be resolved, drvPath the basic derivation path, and
a file existsing at that path for sake of the DB's foreign key. */
assert(drv->type() != DerivationType::CAFloating);
}
infos2.push_back(newInfo); infos2.push_back(newInfo);
} }
worker.store.registerValidPaths(infos2); worker.store.registerValidPaths(infos2);

View file

@ -672,4 +672,57 @@ std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath
return "/" + hashString(htSHA256, clearText).to_string(Base32, false); return "/" + hashString(htSHA256, clearText).to_string(Base32, false);
} }
// N.B. Outputs are left unchanged
static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) {
debug("Rewriting the derivation");
for (auto &rewrite: rewrites) {
debug("rewriting %s as %s", rewrite.first, rewrite.second);
}
drv.builder = rewriteStrings(drv.builder, rewrites);
for (auto & arg: drv.args) {
arg = rewriteStrings(arg, rewrites);
}
StringPairs newEnv;
for (auto & envVar: drv.env) {
auto envName = rewriteStrings(envVar.first, rewrites);
auto envValue = rewriteStrings(envVar.second, rewrites);
newEnv.emplace(envName, envValue);
}
drv.env = newEnv;
}
Sync<DrvPathResolutions> drvPathResolutions;
BasicDerivation Derivation::resolve(Store & store) {
BasicDerivation resolved { *this };
// Input paths that we'll want to rewrite in the derivation
StringMap inputRewrites;
for (auto & input : inputDrvs) {
auto inputDrvOutputs = store.queryPartialDerivationOutputMap(input.first);
StringSet newOutputNames;
for (auto & outputName : input.second) {
auto actualPathOpt = inputDrvOutputs.at(outputName);
if (!actualPathOpt)
throw Error("input drv '%s' wasn't yet built", store.printStorePath(input.first));
auto actualPath = *actualPathOpt;
inputRewrites.emplace(
downstreamPlaceholder(store, input.first, outputName),
store.printStorePath(actualPath));
resolved.inputSrcs.insert(std::move(actualPath));
}
}
rewriteDerivation(store, resolved, inputRewrites);
return resolved;
}
} }

View file

@ -4,6 +4,7 @@
#include "types.hh" #include "types.hh"
#include "hash.hh" #include "hash.hh"
#include "content-address.hh" #include "content-address.hh"
#include "sync.hh"
#include <map> #include <map>
#include <variant> #include <variant>
@ -127,6 +128,13 @@ struct Derivation : BasicDerivation
std::string unparse(const Store & store, bool maskOutputs, std::string unparse(const Store & store, bool maskOutputs,
std::map<std::string, StringSet> * actualInputs = nullptr) const; std::map<std::string, StringSet> * actualInputs = nullptr) const;
/* Return the underlying basic derivation but with
1. input drv outputs moved to input sources.
2. placeholders replaced with realized input store paths. */
BasicDerivation resolve(Store & store);
Derivation() = default; Derivation() = default;
Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { } Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { }
}; };
@ -187,6 +195,16 @@ typedef std::map<StorePath, DrvHashModulo> DrvHashes;
extern DrvHashes drvHashes; // FIXME: global, not thread-safe extern DrvHashes drvHashes; // FIXME: global, not thread-safe
/* Memoisation of `readDerivation(..).resove()`. */
typedef std::map<
StorePath,
std::optional<StorePath>
> DrvPathResolutions;
// FIXME: global, though at least thread-safe.
// FIXME: arguably overlaps with hashDerivationModulo memo table.
extern Sync<DrvPathResolutions> drvPathResolutions;
bool wantOutput(const string & output, const std::set<string> & wanted); bool wantOutput(const string & output, const std::set<string> & wanted);
struct Source; struct Source;

View file

@ -803,13 +803,36 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path)
} }
std::map<std::string, std::optional<StorePath>> LocalStore::queryPartialDerivationOutputMap(const StorePath & path) std::map<std::string, std::optional<StorePath>> LocalStore::queryPartialDerivationOutputMap(const StorePath & path_)
{ {
auto path = path_;
std::map<std::string, std::optional<StorePath>> outputs; std::map<std::string, std::optional<StorePath>> outputs;
BasicDerivation drv = readDerivation(path); Derivation drv = readDerivation(path);
for (auto & [outName, _] : drv.outputs) { for (auto & [outName, _] : drv.outputs) {
outputs.insert_or_assign(outName, std::nullopt); outputs.insert_or_assign(outName, std::nullopt);
} }
bool haveCached = false;
{
auto resolutions = drvPathResolutions.lock();
auto resolvedPathOptIter = resolutions->find(path);
if (resolvedPathOptIter != resolutions->end()) {
auto & [_, resolvedPathOpt] = *resolvedPathOptIter;
if (resolvedPathOpt)
path = *resolvedPathOpt;
haveCached = true;
}
}
/* can't just use else-if instead of `!haveCached` because we need to unlock
`drvPathResolutions` before it is locked in `Derivation::resolve`. */
if (!haveCached && drv.type() == DerivationType::CAFloating) {
/* Resolve drv and use that path instead. */
auto pathResolved = writeDerivation(*this, drv.resolve(*this));
/* Store in memo table. */
/* FIXME: memo logic should not be local-store specific, should have
wrapper-method instead. */
drvPathResolutions.lock()->insert_or_assign(path, pathResolved);
path = std::move(pathResolved);
}
return retrySQLite<std::map<std::string, std::optional<StorePath>>>([&]() { return retrySQLite<std::map<std::string, std::optional<StorePath>>>([&]() {
auto state(_state.lock()); auto state(_state.lock());

View file

@ -4,16 +4,40 @@ with import ./config.nix;
# A simple content-addressed derivation. # A simple content-addressed derivation.
# The derivation can be arbitrarily modified by passing a different `seed`, # The derivation can be arbitrarily modified by passing a different `seed`,
# but the output will always be the same # but the output will always be the same
mkDerivation { rec {
name = "simple-content-addressed"; root = mkDerivation {
buildCommand = '' name = "simple-content-addressed";
set -x buildCommand = ''
echo "Building a CA derivation" set -x
echo "The seed is ${toString seed}" echo "Building a CA derivation"
mkdir -p $out echo "The seed is ${toString seed}"
echo "Hello World" > $out/hello mkdir -p $out
''; echo "Hello World" > $out/hello
__contentAddressed = true; '';
outputHashMode = "recursive"; __contentAddressed = true;
outputHashAlgo = "sha256"; outputHashMode = "recursive";
outputHashAlgo = "sha256";
};
dependent = mkDerivation {
name = "dependent";
buildCommand = ''
echo "building a dependent derivation"
mkdir -p $out
echo ${root}/hello > $out/dep
'';
__contentAddressed = true;
outputHashMode = "recursive";
outputHashAlgo = "sha256";
};
transitivelyDependent = mkDerivation {
name = "transitively-dependent";
buildCommand = ''
echo "building transitively-dependent"
cat ${dependent}/dep
echo ${dependent} > $out
'';
__contentAddressed = true;
outputHashMode = "recursive";
outputHashAlgo = "sha256";
};
} }

View file

@ -2,15 +2,17 @@
source common.sh source common.sh
clearStore drv=$(nix-instantiate --experimental-features ca-derivations ./content-addressed.nix -A root --arg seed 1)
clearCache
export REMOTE_STORE=file://$cacheDir
drv=$(nix-instantiate --experimental-features ca-derivations ./content-addressed.nix --arg seed 1)
nix --experimental-features 'nix-command ca-derivations' show-derivation --derivation "$drv" --arg seed 1 nix --experimental-features 'nix-command ca-derivations' show-derivation --derivation "$drv" --arg seed 1
out1=$(nix-build --experimental-features ca-derivations ./content-addressed.nix --arg seed 1 --no-out-link) testDerivation () {
out2=$(nix-build --experimental-features ca-derivations ./content-addressed.nix --arg seed 2 --no-out-link) local derivationPath=$1
local commonArgs=("--experimental-features" "ca-derivations" "./content-addressed.nix" "-A" "$derivationPath" "--no-out-link")
local out1=$(nix-build "${commonArgs[@]}" --arg seed 1)
local out2=$(nix-build "${commonArgs[@]}" --arg seed 2)
test $out1 == $out2
}
test $out1 == $out2 testDerivation root
testDerivation dependent
testDerivation transitivelyDependent