Add command 'nix store copy-log'

Fixes #5222.

doc/manual/src/release-notes/rl-next.md

@@ -7,3 +7,5 @@
   set or toggle display of error traces.
 * New builtin function `builtins.zipAttrsWith` with same functionality
   as `lib.zipAttrsWith` from nixpkgs, but much more efficient.
+* New command `nix store copy-log` to copy build logs from one store
+  to another.

src/libcmd/command.cc

@@ -54,6 +54,36 @@ void StoreCommand::run()
     run(getStore());
 }
 
+CopyCommand::CopyCommand()
+{
+    addFlag({
+        .longName = "from",
+        .description = "URL of the source Nix store.",
+        .labels = {"store-uri"},
+        .handler = {&srcUri},
+    });
+
+    addFlag({
+        .longName = "to",
+        .description = "URL of the destination Nix store.",
+        .labels = {"store-uri"},
+        .handler = {&dstUri},
+    });
+}
+
+ref<Store> CopyCommand::createStore()
+{
+    return srcUri.empty() ? StoreCommand::createStore() : openStore(srcUri);
+}
+
+ref<Store> CopyCommand::getDstStore()
+{
+    if (srcUri.empty() && dstUri.empty())
+        throw UsageError("you must pass '--from' and/or '--to'");
+
+    return dstUri.empty() ? openStore() : openStore(dstUri);
+}
+
 EvalCommand::EvalCommand()
 {
 }
@@ -159,43 +189,6 @@ void StorePathsCommand::run(ref<Store> store, BuiltPaths && paths)
     run(store, std::move(sorted));
 }
 
-CopyCommand::CopyCommand()
-    : BuiltPathsCommand(true)
-{
-    addFlag({
-        .longName = "from",
-        .description = "URL of the source Nix store.",
-        .labels = {"store-uri"},
-        .handler = {&srcUri},
-    });
-
-    addFlag({
-        .longName = "to",
-        .description = "URL of the destination Nix store.",
-        .labels = {"store-uri"},
-        .handler = {&dstUri},
-    });
-}
-
-ref<Store> CopyCommand::createStore()
-{
-    return srcUri.empty() ? StoreCommand::createStore() : openStore(srcUri);
-}
-
-void CopyCommand::run(ref<Store> store)
-{
-    if (srcUri.empty() && dstUri.empty())
-        throw UsageError("you must pass '--from' and/or '--to'");
-
-    BuiltPathsCommand::run(store);
-}
-
-void CopyCommand::run(ref<Store> srcStore, BuiltPaths && paths)
-{
-    ref<Store> dstStore = dstUri.empty() ? openStore() : openStore(dstUri);
-    run(srcStore, dstStore, std::move(paths));
-}
-
 void StorePathCommand::run(ref<Store> store, std::vector<StorePath> && storePaths)
 {
     if (storePaths.size() != 1)

src/libcmd/command.hh

@@ -43,6 +43,19 @@ private:
     std::shared_ptr<Store> _store;
 };
 
+/* A command that copies something between `--from` and `--to`
+   stores. */
+struct CopyCommand : virtual StoreCommand
+{
+    std::string srcUri, dstUri;
+
+    CopyCommand();
+
+    ref<Store> createStore() override;
+
+    ref<Store> getDstStore();
+};
+
 struct EvalCommand : virtual StoreCommand, MixEvalArgs
 {
     EvalCommand();
@@ -176,23 +189,6 @@ public:
     bool useDefaultInstallables() override { return !all; }
 };
 
-/* A command that copies something between `--from` and `--to`
-   stores. */
-struct CopyCommand : virtual BuiltPathsCommand
-{
-    std::string srcUri, dstUri;
-
-    CopyCommand();
-
-    ref<Store> createStore() override;
-
-    void run(ref<Store> store) override;
-
-    void run(ref<Store> srcStore, BuiltPaths && paths) override;
-
-    virtual void run(ref<Store> srcStore, ref<Store> dstStore, BuiltPaths && paths) = 0;
-};
-
 struct StorePathsCommand : public BuiltPathsCommand
 {
     StorePathsCommand(bool recursive = false);

src/libstore/daemon.cc

@@ -468,10 +468,12 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
             dontCheckSigs = false;
 
         logger->startWork();
-        FramedSource source(from);
+        {
-        store->addMultipleToStore(source,
+            FramedSource source(from);
-            RepairFlag{repair},
+            store->addMultipleToStore(source,
-            dontCheckSigs ? NoCheckSigs : CheckSigs);
+                RepairFlag{repair},
+                dontCheckSigs ? NoCheckSigs : CheckSigs);
+        }
         logger->stopWork();
         break;
     }
@@ -920,6 +922,22 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         break;
     }
 
+    case wopAddBuildLog: {
+        StorePath path{readString(from)};
+        logger->startWork();
+        if (!trusted)
+            throw Error("you are not privileged to add logs");
+        {
+            FramedSource source(from);
+            StringSink sink;
+            source.drainInto(sink);
+            store->addBuildLog(path, sink.s);
+        }
+        logger->stopWork();
+        to << 1;
+        break;
+    }
+
     default:
         throw Error("invalid operation %1%", op);
     }

src/libstore/local-store.cc

@@ -9,6 +9,7 @@
 #include "callback.hh"
 #include "topo-sort.hh"
 #include "finally.hh"
+#include "compression.hh"
 
 #include <iostream>
 #include <algorithm>
@@ -1898,4 +1899,24 @@ FixedOutputHash LocalStore::hashCAPath(
     };
 }
 
+void LocalStore::addBuildLog(const StorePath & drvPath, std::string_view log)
+{
+    assert(drvPath.isDerivation());
+
+    auto baseName = drvPath.to_string();
+
+    auto logPath = fmt("%s/%s/%s/%s.bz2", logDir, drvsLogDir, baseName.substr(0, 2), baseName.substr(2));
+
+    if (pathExists(logPath)) return;
+
+    createDirs(dirOf(logPath));
+
+    auto tmpFile = fmt("%s.tmp.%d", logPath, getpid());
+
+    writeFile(tmpFile, compress("bzip2", log));
+
+    if (rename(tmpFile.c_str(), logPath.c_str()) != 0)
+        throw SysError("renaming '%1%' to '%2%'", tmpFile, logPath);
+}
+
 }  // namespace nix

src/libstore/local-store.hh

@@ -280,6 +280,8 @@ private:
         const std::string_view pathHash
     );
 
+    void addBuildLog(const StorePath & drvPath, std::string_view log) override;
+
     friend struct LocalDerivationGoal;
     friend struct PathSubstitutionGoal;
     friend struct SubstitutionGoal;

src/libstore/remote-store.cc

@@ -908,6 +908,18 @@ void RemoteStore::queryMissing(const std::vector<DerivedPath> & targets,
 }
 
 
+void RemoteStore::addBuildLog(const StorePath & drvPath, std::string_view log)
+{
+    auto conn(getConnection());
+    conn->to << wopAddBuildLog << drvPath.to_string();
+    StringSource source(log);
+    conn.withFramedSink([&](Sink & sink) {
+        source.drainInto(sink);
+    });
+    readInt(conn->from);
+}
+
+
 void RemoteStore::connect()
 {
     auto conn(getConnection());

src/libstore/remote-store.hh

@@ -116,6 +116,8 @@ public:
         StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
         uint64_t & downloadSize, uint64_t & narSize) override;
 
+    void addBuildLog(const StorePath & drvPath, std::string_view log) override;
+
     void connect() override;
 
     unsigned int getProtocol() override;

src/libstore/store-api.hh

@@ -727,6 +727,9 @@ public:
     virtual std::optional<std::string> getBuildLog(const StorePath & path)
     { return std::nullopt; }
 
+    virtual void addBuildLog(const StorePath & path, std::string_view log)
+    { unsupported("addBuildLog"); }
+
     /* Hack to allow long-running processes like hydra-queue-runner to
        occasionally flush their path info cache. */
     void clearPathInfoCache()

src/libstore/worker-protocol.hh

@@ -56,6 +56,7 @@ typedef enum {
     wopRegisterDrvOutput = 42,
     wopQueryRealisation = 43,
     wopAddMultipleToStore = 44,
+    wopAddBuildLog = 45,
 } WorkerOp;
 
 

src/nix/copy.cc

@@ -4,7 +4,7 @@
 
 using namespace nix;
 
-struct CmdCopy : CopyCommand
+struct CmdCopy : virtual CopyCommand, virtual BuiltPathsCommand
 {
     CheckSigsFlag checkSigs = CheckSigs;
 
@@ -45,8 +45,10 @@ struct CmdCopy : CopyCommand
 
     Category category() override { return catSecondary; }
 
-    void run(ref<Store> srcStore, ref<Store> dstStore, BuiltPaths && paths) override
+    void run(ref<Store> srcStore, BuiltPaths && paths) override
     {
+        auto dstStore = getDstStore();
+
         RealisedPath::Set stuffToCopy;
 
         for (auto & builtPath : paths) {

src/nix/store-copy-log.cc

@@ -0,0 +1,40 @@
+#include "command.hh"
+#include "shared.hh"
+#include "store-api.hh"
+#include "sync.hh"
+#include "thread-pool.hh"
+
+#include <atomic>
+
+using namespace nix;
+
+struct CmdCopyLog : virtual CopyCommand, virtual InstallablesCommand
+{
+    std::string description() override
+    {
+        return "copy build logs between Nix stores";
+    }
+
+    std::string doc() override
+    {
+        return
+          #include "store-copy-log.md"
+          ;
+    }
+
+    Category category() override { return catUtility; }
+
+    void run(ref<Store> srcStore) override
+    {
+        auto dstStore = getDstStore();
+
+        for (auto & path : toDerivations(srcStore, installables, true)) {
+            if (auto log = srcStore->getBuildLog(path))
+                dstStore->addBuildLog(path, *log);
+            else
+                throw Error("build log for '%s' is not available", srcStore->printStorePath(path));
+        }
+    }
+};
+
+static auto rCmdCopyLog = registerCommand2<CmdCopyLog>({"store", "copy-log"});

src/nix/store-copy-log.md

@@ -0,0 +1,13 @@
+R""(
+
+# Examples
+
+TODO
+
+# Description
+
+`nix store copy-log` copies build logs between two Nix stores. The
+source store is specified using `--from` and the destination using
+`--to`. If one of these is omitted, it defaults to the local store.
+
+)""