Try to realise CA derivations during queryMissing

This enables nix to correctly report what will be fetched in the case
that everything is a cache hit.

Note however that if an intermediate build of something which is not
cached could still cause products to end up being substituted if the
intermediate build results in a CA path which is in the cache.

Fixes #8615.

Signed-off-by: Peter Waller <p@pwaller.net>
This commit is contained in:
Peter Waller 2023-06-30 16:11:12 +01:00
parent d00fe5f225
commit 4b1bd822ac
5 changed files with 101 additions and 11 deletions

View file

@ -200,6 +200,36 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets,
auto drv = make_ref<Derivation>(derivationFromPath(bfd.drvPath)); auto drv = make_ref<Derivation>(derivationFromPath(bfd.drvPath));
ParsedDerivation parsedDrv(StorePath(bfd.drvPath), *drv); ParsedDerivation parsedDrv(StorePath(bfd.drvPath), *drv);
if (!knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
experimentalFeatureSettings.require(Xp::CaDerivations);
// If there are unknown output paths, attempt to find if the
// paths are known to substituters through a realisation.
auto outputHashes = staticOutputHashes(*this, *drv);
knownOutputPaths = true;
for (auto [outputName, hash] : outputHashes) {
if (!bfd.outputs.contains(outputName))
continue;
bool found = false;
for (auto &sub : getDefaultSubstituters()) {
auto realisation = sub->queryRealisation({hash, outputName});
if (!realisation)
continue;
found = true;
if (!isValidPath(realisation->outPath))
invalid.insert(realisation->outPath);
break;
}
if (!found) {
// Some paths did not have a realisation, this must be built.
knownOutputPaths = false;
break;
}
}
}
if (knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) { if (knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size())); auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size()));
for (auto & output : invalid) for (auto & output : invalid)

51
tests/ca/build-cache.sh Normal file
View file

@ -0,0 +1,51 @@
#!/usr/bin/env bash
source common.sh
# The substituters didn't work prior to this time.
requireDaemonNewerThan "2.18.0pre20230808"
drv=$(nix-instantiate ./content-addressed.nix -A rootCA --arg seed 1)^out
nix derivation show "$drv" --arg seed 1
buildAttr () {
local derivationPath=$1
local seedValue=$2
shift; shift
local args=("./content-addressed.nix" "-A" "$derivationPath" --arg seed "$seedValue" "--no-out-link")
args+=("$@")
nix-build "${args[@]}"
}
copyAttr () {
local derivationPath=$1
local seedValue=$2
shift; shift
local args=("-f" "./content-addressed.nix" "$derivationPath" --arg seed "$seedValue")
args+=("$@")
# Note: to copy CA derivations, we need to copy the realisations, which
# currently requires naming the installables, not just the derivation output
# path.
nix copy --to file://$cacheDir "${args[@]}"
}
testRemoteCacheFor () {
local derivationPath=$1
clearCache
copyAttr "$derivationPath" 1
clearStore
# Check nothing gets built.
buildAttr "$derivationPath" 1 --option substituters file://$cacheDir --no-require-sigs |& grepQuietInverse " will be built:"
}
testRemoteCache () {
testRemoteCacheFor rootCA
testRemoteCacheFor dependentCA
testRemoteCacheFor dependentNonCA
testRemoteCacheFor dependentFixedOutput
testRemoteCacheFor dependentForBuildCA
testRemoteCacheFor dependentForBuildNonCA
}
clearStore
testRemoteCache

View file

@ -2,7 +2,7 @@
source common.sh source common.sh
drv=$(nix-instantiate ./content-addressed.nix -A rootCA --arg seed 1) drv=$(nix-instantiate ./content-addressed.nix -A rootCA --arg seed 1)^out
nix derivation show "$drv" --arg seed 1 nix derivation show "$drv" --arg seed 1
buildAttr () { buildAttr () {
@ -14,14 +14,6 @@ buildAttr () {
nix-build "${args[@]}" nix-build "${args[@]}"
} }
testRemoteCache () {
clearCache
local outPath=$(buildAttr dependentNonCA 1)
nix copy --to file://$cacheDir $outPath
clearStore
buildAttr dependentNonCA 1 --option substituters file://$cacheDir --no-require-sigs |& grepQuietInverse "building dependent-non-ca"
}
testDeterministicCA () { testDeterministicCA () {
[[ $(buildAttr rootCA 1) = $(buildAttr rootCA 2) ]] [[ $(buildAttr rootCA 1) = $(buildAttr rootCA 2) ]]
} }
@ -66,8 +58,6 @@ testNormalization () {
test "$(stat -c %Y $outPath)" -eq 1 test "$(stat -c %Y $outPath)" -eq 1
} }
# Disabled until we have it properly working
# testRemoteCache
clearStore clearStore
testNormalization testNormalization
testDeterministicCA testDeterministicCA

View file

@ -61,6 +61,24 @@ rec {
echo ${rootCA}/non-ca-hello > $out/dep echo ${rootCA}/non-ca-hello > $out/dep
''; '';
}; };
dependentForBuildCA = mkCADerivation {
name = "dependent-for-build-ca";
buildCommand = ''
echo "Depends on rootCA for building only"
mkdir -p $out
echo ${rootCA}
touch $out
'';
};
dependentForBuildNonCA = mkDerivation {
name = "dependent-for-build-non-ca";
buildCommand = ''
echo "Depends on rootCA for building only"
mkdir -p $out
echo ${rootCA}
touch $out
'';
};
dependentFixedOutput = mkDerivation { dependentFixedOutput = mkDerivation {
name = "dependent-fixed-output"; name = "dependent-fixed-output";
outputHashMode = "recursive"; outputHashMode = "recursive";

View file

@ -1,6 +1,7 @@
ca-tests := \ ca-tests := \
$(d)/build-with-garbage-path.sh \ $(d)/build-with-garbage-path.sh \
$(d)/build.sh \ $(d)/build.sh \
$(d)/build-cache.sh \
$(d)/concurrent-builds.sh \ $(d)/concurrent-builds.sh \
$(d)/derivation-json.sh \ $(d)/derivation-json.sh \
$(d)/duplicate-realisation-in-closure.sh \ $(d)/duplicate-realisation-in-closure.sh \