Merge pull request #4487 from NixOS/ca/copy-drv-outputs

Copy ca derivation outputs

src/libstore/globals.cc

@@ -165,10 +165,15 @@ bool Settings::isExperimentalFeatureEnabled(const std::string & name)
     return std::find(f.begin(), f.end(), name) != f.end();
 }
 
+MissingExperimentalFeature::MissingExperimentalFeature(std::string feature)
+    : Error("experimental Nix feature '%1%' is disabled; use '--experimental-features %1%' to override", feature)
+    , missingFeature(feature)
+    {}
+
 void Settings::requireExperimentalFeature(const std::string & name)
 {
     if (!isExperimentalFeatureEnabled(name))
-        throw Error("experimental Nix feature '%1%' is disabled; use '--experimental-features %1%' to override", name);
+        throw MissingExperimentalFeature(name);
 }
 
 bool Settings::isWSL1()

src/libstore/globals.hh

@@ -45,6 +45,15 @@ struct PluginFilesSetting : public BaseSetting<Paths>
     void set(const std::string & str, bool append = false) override;
 };
 
+class MissingExperimentalFeature: public Error
+{
+public:
+    std::string missingFeature;
+
+    MissingExperimentalFeature(std::string feature);
+    virtual const char* sname() const override { return "MissingExperimentalFeature"; }
+};
+
 class Settings : public Config {
 
     unsigned int getDefaultCores();
@@ -632,7 +641,7 @@ public:
           is `root`.
 
           > **Warning**
-          > 
+          >
           > Adding a user to `trusted-users` is essentially equivalent to
           > giving that user root access to the system. For example, the user
           > can set `sandbox-paths` and thereby obtain read access to
@@ -722,13 +731,13 @@ public:
           The program executes with no arguments. The program's environment
           contains the following environment variables:
 
-            - `DRV_PATH`  
+            - `DRV_PATH`
               The derivation for the built paths.
 
               Example:
               `/nix/store/5nihn1a7pa8b25l9zafqaqibznlvvp3f-bash-4.4-p23.drv`
 
-            - `OUT_PATHS`  
+            - `OUT_PATHS`
               Output paths of the built derivation, separated by a space
               character.
 
@@ -759,7 +768,7 @@ public:
           documentation](https://ec.haxx.se/usingcurl-netrc.html).
 
           > **Note**
-          > 
+          >
           > This must be an absolute path, and `~` is not resolved. For
           > example, `~/.netrc` won't resolve to your home directory's
           > `.netrc`.

src/libstore/local-store.cc

@@ -655,6 +655,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
 
 void LocalStore::registerDrvOutput(const Realisation & info)
 {
+    settings.requireExperimentalFeature("ca-derivations");
     auto state(_state.lock());
     retrySQLite<void>([&]() {
         state->stmts->RegisterRealisedOutput.use()

src/libstore/store-api.cc

@@ -783,6 +783,36 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
 }
 
 
+std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore, const RealisedPath::Set & paths,
+    RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute)
+{
+    StorePathSet storePaths;
+    std::set<Realisation> realisations;
+    for (auto & path : paths) {
+        storePaths.insert(path.path());
+        if (auto realisation = std::get_if<Realisation>(&path.raw)) {
+            settings.requireExperimentalFeature("ca-derivations");
+            realisations.insert(*realisation);
+        }
+    }
+    auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute);
+    try {
+        for (auto & realisation : realisations) {
+            dstStore->registerDrvOutput(realisation);
+        }
+    } catch (MissingExperimentalFeature & e) {
+        // Don't fail if the remote doesn't support CA derivations is it might
+        // not be within our control to change that, and we might still want
+        // to at least copy the output paths.
+        if (e.missingFeature == "ca-derivations")
+            ignoreException();
+        else
+            throw;
+    }
+
+    return pathsMap;
+}
+
 std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & storePaths,
     RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute)
 {
@@ -796,7 +826,6 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
     for (auto & path : storePaths)
         pathsMap.insert_or_assign(path, path);
 
-    if (missing.empty()) return pathsMap;
 
     Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size()));
 
@@ -871,21 +900,9 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
             nrDone++;
             showProgress();
         });
-
     return pathsMap;
 }
 
-
-void copyClosure(ref<Store> srcStore, ref<Store> dstStore,
-    const StorePathSet & storePaths, RepairFlag repair, CheckSigsFlag checkSigs,
-    SubstituteFlag substitute)
-{
-    StorePathSet closure;
-    srcStore->computeFSClosure(storePaths, closure);
-    copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute);
-}
-
-
 std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istream & str, std::optional<HashResult> hashGiven)
 {
     std::string path;

src/libstore/store-api.hh

@@ -752,15 +752,12 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
    that. Returns a map of what each path was copied to the dstStore
    as. */
 std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore,
-    const StorePathSet & storePaths,
+    const RealisedPath::Set &,
     RepairFlag repair = NoRepair,
     CheckSigsFlag checkSigs = CheckSigs,
     SubstituteFlag substitute = NoSubstitute);
-
-
-/* Copy the closure of the specified paths from one store to another. */
-void copyClosure(ref<Store> srcStore, ref<Store> dstStore,
-    const StorePathSet & storePaths,
+std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore,
+    const StorePathSet& paths,
     RepairFlag repair = NoRepair,
     CheckSigsFlag checkSigs = CheckSigs,
     SubstituteFlag substitute = NoSubstitute);

src/nix-copy-closure/nix-copy-closure.cc

@@ -50,12 +50,12 @@ static int main_nix_copy_closure(int argc, char ** argv)
         auto to = toMode ? openStore(remoteUri) : openStore();
         auto from = toMode ? openStore() : openStore(remoteUri);
 
-        StorePathSet storePaths2;
+        RealisedPath::Set storePaths2;
         for (auto & path : storePaths)
             storePaths2.insert(from->followLinksToStorePath(path));
 
-        StorePathSet closure;
-        from->computeFSClosure(storePaths2, closure, false, includeOutputs);
+        RealisedPath::Set closure;
+        RealisedPath::closure(*from, storePaths2, closure);
 
         copyPaths(from, to, closure, NoRepair, NoCheckSigs, useSubstitutes);
 

src/nix/copy.cc

@@ -8,7 +8,7 @@
 
 using namespace nix;
 
-struct CmdCopy : StorePathsCommand
+struct CmdCopy : RealisedPathsCommand
 {
     std::string srcUri, dstUri;
 
@@ -16,10 +16,10 @@ struct CmdCopy : StorePathsCommand
 
     SubstituteFlag substitute = NoSubstitute;
 
-    using StorePathsCommand::run;
+    using RealisedPathsCommand::run;
 
     CmdCopy()
-        : StorePathsCommand(true)
+        : RealisedPathsCommand(true)
     {
         addFlag({
             .longName = "from",
@@ -75,14 +75,15 @@ struct CmdCopy : StorePathsCommand
         if (srcUri.empty() && dstUri.empty())
             throw UsageError("you must pass '--from' and/or '--to'");
 
-        StorePathsCommand::run(store);
+        RealisedPathsCommand::run(store);
     }
 
-    void run(ref<Store> srcStore, StorePaths storePaths) override
+    void run(ref<Store> srcStore, std::vector<RealisedPath> paths) override
     {
         ref<Store> dstStore = dstUri.empty() ? openStore() : openStore(dstUri);
 
-        copyPaths(srcStore, dstStore, StorePathSet(storePaths.begin(), storePaths.end()),
+        copyPaths(
+            srcStore, dstStore, RealisedPath::Set(paths.begin(), paths.end()),
             NoRepair, checkSigs, substitute);
     }
 };

tests/local.mk

@@ -38,6 +38,7 @@ nix_tests = \
   describe-stores.sh \
   flakes.sh \
   content-addressed.sh \
+  nix-copy-content-addressed.sh \
   build.sh \
   compute-levels.sh
   # parallel.sh

tests/nix-copy-content-addressed.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+source common.sh
+
+# Globally enable the ca derivations experimental flag
+sed -i 's/experimental-features = .*/& ca-derivations ca-references/' "$NIX_CONF_DIR/nix.conf"
+
+export REMOTE_STORE_DIR="$TEST_ROOT/remote_store"
+export REMOTE_STORE="file://$REMOTE_STORE_DIR"
+
+ensureCorrectlyCopied () {
+    attrPath="$1"
+    nix build --store "$REMOTE_STORE" --file ./content-addressed.nix "$attrPath"
+}
+
+testOneCopy () {
+    clearStore
+    rm -rf "$REMOTE_STORE_DIR"
+
+    attrPath="$1"
+    nix copy --to $REMOTE_STORE "$attrPath" --file ./content-addressed.nix
+
+    ensureCorrectlyCopied "$attrPath"
+
+    # Ensure that we can copy back what we put in the store
+    clearStore
+    nix copy --from $REMOTE_STORE \
+        --file ./content-addressed.nix "$attrPath" \
+        --no-check-sigs
+}
+
+for attrPath in rootCA dependentCA transitivelyDependentCA dependentNonCA dependentFixedOutput; do
+    testOneCopy "$attrPath"
+done