Merge pull request #4487 from NixOS/ca/copy-drv-outputs

Copy ca derivation outputs
This commit is contained in:
Eelco Dolstra 2021-02-26 12:20:42 +01:00 committed by GitHub
commit d2803406b5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 98 additions and 33 deletions

View file

@ -165,10 +165,15 @@ bool Settings::isExperimentalFeatureEnabled(const std::string & name)
return std::find(f.begin(), f.end(), name) != f.end();
}
MissingExperimentalFeature::MissingExperimentalFeature(std::string feature)
: Error("experimental Nix feature '%1%' is disabled; use '--experimental-features %1%' to override", feature)
, missingFeature(feature)
{}
void Settings::requireExperimentalFeature(const std::string & name)
{
if (!isExperimentalFeatureEnabled(name))
throw Error("experimental Nix feature '%1%' is disabled; use '--experimental-features %1%' to override", name);
throw MissingExperimentalFeature(name);
}
bool Settings::isWSL1()

View file

@ -45,6 +45,15 @@ struct PluginFilesSetting : public BaseSetting<Paths>
void set(const std::string & str, bool append = false) override;
};
class MissingExperimentalFeature: public Error
{
public:
std::string missingFeature;
MissingExperimentalFeature(std::string feature);
virtual const char* sname() const override { return "MissingExperimentalFeature"; }
};
class Settings : public Config {
unsigned int getDefaultCores();

View file

@ -655,6 +655,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
void LocalStore::registerDrvOutput(const Realisation & info)
{
settings.requireExperimentalFeature("ca-derivations");
auto state(_state.lock());
retrySQLite<void>([&]() {
state->stmts->RegisterRealisedOutput.use()

View file

@ -783,6 +783,36 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
}
std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore, const RealisedPath::Set & paths,
RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute)
{
StorePathSet storePaths;
std::set<Realisation> realisations;
for (auto & path : paths) {
storePaths.insert(path.path());
if (auto realisation = std::get_if<Realisation>(&path.raw)) {
settings.requireExperimentalFeature("ca-derivations");
realisations.insert(*realisation);
}
}
auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute);
try {
for (auto & realisation : realisations) {
dstStore->registerDrvOutput(realisation);
}
} catch (MissingExperimentalFeature & e) {
// Don't fail if the remote doesn't support CA derivations is it might
// not be within our control to change that, and we might still want
// to at least copy the output paths.
if (e.missingFeature == "ca-derivations")
ignoreException();
else
throw;
}
return pathsMap;
}
std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & storePaths,
RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute)
{
@ -796,7 +826,6 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
for (auto & path : storePaths)
pathsMap.insert_or_assign(path, path);
if (missing.empty()) return pathsMap;
Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size()));
@ -871,21 +900,9 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
nrDone++;
showProgress();
});
return pathsMap;
}
void copyClosure(ref<Store> srcStore, ref<Store> dstStore,
const StorePathSet & storePaths, RepairFlag repair, CheckSigsFlag checkSigs,
SubstituteFlag substitute)
{
StorePathSet closure;
srcStore->computeFSClosure(storePaths, closure);
copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute);
}
std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istream & str, std::optional<HashResult> hashGiven)
{
std::string path;

View file

@ -752,15 +752,12 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
that. Returns a map of what each path was copied to the dstStore
as. */
std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore,
const StorePathSet & storePaths,
const RealisedPath::Set &,
RepairFlag repair = NoRepair,
CheckSigsFlag checkSigs = CheckSigs,
SubstituteFlag substitute = NoSubstitute);
/* Copy the closure of the specified paths from one store to another. */
void copyClosure(ref<Store> srcStore, ref<Store> dstStore,
const StorePathSet & storePaths,
std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore,
const StorePathSet& paths,
RepairFlag repair = NoRepair,
CheckSigsFlag checkSigs = CheckSigs,
SubstituteFlag substitute = NoSubstitute);

View file

@ -50,12 +50,12 @@ static int main_nix_copy_closure(int argc, char ** argv)
auto to = toMode ? openStore(remoteUri) : openStore();
auto from = toMode ? openStore() : openStore(remoteUri);
StorePathSet storePaths2;
RealisedPath::Set storePaths2;
for (auto & path : storePaths)
storePaths2.insert(from->followLinksToStorePath(path));
StorePathSet closure;
from->computeFSClosure(storePaths2, closure, false, includeOutputs);
RealisedPath::Set closure;
RealisedPath::closure(*from, storePaths2, closure);
copyPaths(from, to, closure, NoRepair, NoCheckSigs, useSubstitutes);

View file

@ -8,7 +8,7 @@
using namespace nix;
struct CmdCopy : StorePathsCommand
struct CmdCopy : RealisedPathsCommand
{
std::string srcUri, dstUri;
@ -16,10 +16,10 @@ struct CmdCopy : StorePathsCommand
SubstituteFlag substitute = NoSubstitute;
using StorePathsCommand::run;
using RealisedPathsCommand::run;
CmdCopy()
: StorePathsCommand(true)
: RealisedPathsCommand(true)
{
addFlag({
.longName = "from",
@ -75,14 +75,15 @@ struct CmdCopy : StorePathsCommand
if (srcUri.empty() && dstUri.empty())
throw UsageError("you must pass '--from' and/or '--to'");
StorePathsCommand::run(store);
RealisedPathsCommand::run(store);
}
void run(ref<Store> srcStore, StorePaths storePaths) override
void run(ref<Store> srcStore, std::vector<RealisedPath> paths) override
{
ref<Store> dstStore = dstUri.empty() ? openStore() : openStore(dstUri);
copyPaths(srcStore, dstStore, StorePathSet(storePaths.begin(), storePaths.end()),
copyPaths(
srcStore, dstStore, RealisedPath::Set(paths.begin(), paths.end()),
NoRepair, checkSigs, substitute);
}
};

View file

@ -38,6 +38,7 @@ nix_tests = \
describe-stores.sh \
flakes.sh \
content-addressed.sh \
nix-copy-content-addressed.sh \
build.sh \
compute-levels.sh
# parallel.sh

View file

@ -0,0 +1,34 @@
#!/usr/bin/env bash
source common.sh
# Globally enable the ca derivations experimental flag
sed -i 's/experimental-features = .*/& ca-derivations ca-references/' "$NIX_CONF_DIR/nix.conf"
export REMOTE_STORE_DIR="$TEST_ROOT/remote_store"
export REMOTE_STORE="file://$REMOTE_STORE_DIR"
ensureCorrectlyCopied () {
attrPath="$1"
nix build --store "$REMOTE_STORE" --file ./content-addressed.nix "$attrPath"
}
testOneCopy () {
clearStore
rm -rf "$REMOTE_STORE_DIR"
attrPath="$1"
nix copy --to $REMOTE_STORE "$attrPath" --file ./content-addressed.nix
ensureCorrectlyCopied "$attrPath"
# Ensure that we can copy back what we put in the store
clearStore
nix copy --from $REMOTE_STORE \
--file ./content-addressed.nix "$attrPath" \
--no-check-sigs
}
for attrPath in rootCA dependentCA transitivelyDependentCA dependentNonCA dependentFixedOutput; do
testOneCopy "$attrPath"
done