From 4dda1f92aae05dd9d633152458d65a3815bcd03c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 17 Jan 2022 19:45:21 +0100
Subject: [PATCH] Add command 'nix store copy-log'

Fixes #5222.
---
 doc/manual/src/release-notes/rl-next.md |  2 +
 src/libcmd/command.cc                   | 67 +++++++++++--------------
 src/libcmd/command.hh                   | 30 +++++------
 src/libstore/daemon.cc                  | 26 ++++++++--
 src/libstore/local-store.cc             | 21 ++++++++
 src/libstore/local-store.hh             |  2 +
 src/libstore/remote-store.cc            | 12 +++++
 src/libstore/remote-store.hh            |  2 +
 src/libstore/store-api.hh               |  3 ++
 src/libstore/worker-protocol.hh         |  1 +
 src/nix/copy.cc                         |  6 ++-
 src/nix/store-copy-log.cc               | 40 +++++++++++++++
 src/nix/store-copy-log.md               | 13 +++++
 13 files changed, 165 insertions(+), 60 deletions(-)
 create mode 100644 src/nix/store-copy-log.cc
 create mode 100644 src/nix/store-copy-log.md

diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md
index f51969ced..adf3010c0 100644
--- a/doc/manual/src/release-notes/rl-next.md
+++ b/doc/manual/src/release-notes/rl-next.md
@@ -7,3 +7,5 @@
   set or toggle display of error traces.
 * New builtin function `builtins.zipAttrsWith` with same functionality
   as `lib.zipAttrsWith` from nixpkgs, but much more efficient.
+* New command `nix store copy-log` to copy build logs from one store
+  to another.
diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc
index 5e6d4a857..6d183dfad 100644
--- a/src/libcmd/command.cc
+++ b/src/libcmd/command.cc
@@ -54,6 +54,36 @@ void StoreCommand::run()
     run(getStore());
 }
 
+CopyCommand::CopyCommand()
+{
+    addFlag({
+        .longName = "from",
+        .description = "URL of the source Nix store.",
+        .labels = {"store-uri"},
+        .handler = {&srcUri},
+    });
+
+    addFlag({
+        .longName = "to",
+        .description = "URL of the destination Nix store.",
+        .labels = {"store-uri"},
+        .handler = {&dstUri},
+    });
+}
+
+ref<Store> CopyCommand::createStore()
+{
+    return srcUri.empty() ? StoreCommand::createStore() : openStore(srcUri);
+}
+
+ref<Store> CopyCommand::getDstStore()
+{
+    if (srcUri.empty() && dstUri.empty())
+        throw UsageError("you must pass '--from' and/or '--to'");
+
+    return dstUri.empty() ? openStore() : openStore(dstUri);
+}
+
 EvalCommand::EvalCommand()
 {
 }
@@ -159,43 +189,6 @@ void StorePathsCommand::run(ref<Store> store, BuiltPaths && paths)
     run(store, std::move(sorted));
 }
 
-CopyCommand::CopyCommand()
-    : BuiltPathsCommand(true)
-{
-    addFlag({
-        .longName = "from",
-        .description = "URL of the source Nix store.",
-        .labels = {"store-uri"},
-        .handler = {&srcUri},
-    });
-
-    addFlag({
-        .longName = "to",
-        .description = "URL of the destination Nix store.",
-        .labels = {"store-uri"},
-        .handler = {&dstUri},
-    });
-}
-
-ref<Store> CopyCommand::createStore()
-{
-    return srcUri.empty() ? StoreCommand::createStore() : openStore(srcUri);
-}
-
-void CopyCommand::run(ref<Store> store)
-{
-    if (srcUri.empty() && dstUri.empty())
-        throw UsageError("you must pass '--from' and/or '--to'");
-
-    BuiltPathsCommand::run(store);
-}
-
-void CopyCommand::run(ref<Store> srcStore, BuiltPaths && paths)
-{
-    ref<Store> dstStore = dstUri.empty() ? openStore() : openStore(dstUri);
-    run(srcStore, dstStore, std::move(paths));
-}
-
 void StorePathCommand::run(ref<Store> store, std::vector<StorePath> && storePaths)
 {
     if (storePaths.size() != 1)
diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh
index 0c3e29e25..bd2a0a7ee 100644
--- a/src/libcmd/command.hh
+++ b/src/libcmd/command.hh
@@ -43,6 +43,19 @@ private:
     std::shared_ptr<Store> _store;
 };
 
+/* A command that copies something between `--from` and `--to`
+   stores. */
+struct CopyCommand : virtual StoreCommand
+{
+    std::string srcUri, dstUri;
+
+    CopyCommand();
+
+    ref<Store> createStore() override;
+
+    ref<Store> getDstStore();
+};
+
 struct EvalCommand : virtual StoreCommand, MixEvalArgs
 {
     EvalCommand();
@@ -176,23 +189,6 @@ public:
     bool useDefaultInstallables() override { return !all; }
 };
 
-/* A command that copies something between `--from` and `--to`
-   stores. */
-struct CopyCommand : virtual BuiltPathsCommand
-{
-    std::string srcUri, dstUri;
-
-    CopyCommand();
-
-    ref<Store> createStore() override;
-
-    void run(ref<Store> store) override;
-
-    void run(ref<Store> srcStore, BuiltPaths && paths) override;
-
-    virtual void run(ref<Store> srcStore, ref<Store> dstStore, BuiltPaths && paths) = 0;
-};
-
 struct StorePathsCommand : public BuiltPathsCommand
 {
     StorePathsCommand(bool recursive = false);
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 5b817c587..101aa13a5 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -468,10 +468,12 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
             dontCheckSigs = false;
 
         logger->startWork();
-        FramedSource source(from);
-        store->addMultipleToStore(source,
-            RepairFlag{repair},
-            dontCheckSigs ? NoCheckSigs : CheckSigs);
+        {
+            FramedSource source(from);
+            store->addMultipleToStore(source,
+                RepairFlag{repair},
+                dontCheckSigs ? NoCheckSigs : CheckSigs);
+        }
         logger->stopWork();
         break;
     }
@@ -920,6 +922,22 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         break;
     }
 
+    case wopAddBuildLog: {
+        StorePath path{readString(from)};
+        logger->startWork();
+        if (!trusted)
+            throw Error("you are not privileged to add logs");
+        {
+            FramedSource source(from);
+            StringSink sink;
+            source.drainInto(sink);
+            store->addBuildLog(path, sink.s);
+        }
+        logger->stopWork();
+        to << 1;
+        break;
+    }
+
     default:
         throw Error("invalid operation %1%", op);
     }
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index d3cebe720..1807940d8 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -9,6 +9,7 @@
 #include "callback.hh"
 #include "topo-sort.hh"
 #include "finally.hh"
+#include "compression.hh"
 
 #include <iostream>
 #include <algorithm>
@@ -1898,4 +1899,24 @@ FixedOutputHash LocalStore::hashCAPath(
     };
 }
 
+void LocalStore::addBuildLog(const StorePath & drvPath, std::string_view log)
+{
+    assert(drvPath.isDerivation());
+
+    auto baseName = drvPath.to_string();
+
+    auto logPath = fmt("%s/%s/%s/%s.bz2", logDir, drvsLogDir, baseName.substr(0, 2), baseName.substr(2));
+
+    if (pathExists(logPath)) return;
+
+    createDirs(dirOf(logPath));
+
+    auto tmpFile = fmt("%s.tmp.%d", logPath, getpid());
+
+    writeFile(tmpFile, compress("bzip2", log));
+
+    if (rename(tmpFile.c_str(), logPath.c_str()) != 0)
+        throw SysError("renaming '%1%' to '%2%'", tmpFile, logPath);
+}
+
 }  // namespace nix
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index c4d7b80bd..6d867d778 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -280,6 +280,8 @@ private:
         const std::string_view pathHash
     );
 
+    void addBuildLog(const StorePath & drvPath, std::string_view log) override;
+
     friend struct LocalDerivationGoal;
     friend struct PathSubstitutionGoal;
     friend struct SubstitutionGoal;
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 6886103e1..aac2965e0 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -908,6 +908,18 @@ void RemoteStore::queryMissing(const std::vector<DerivedPath> & targets,
 }
 
 
+void RemoteStore::addBuildLog(const StorePath & drvPath, std::string_view log)
+{
+    auto conn(getConnection());
+    conn->to << wopAddBuildLog << drvPath.to_string();
+    StringSource source(log);
+    conn.withFramedSink([&](Sink & sink) {
+        source.drainInto(sink);
+    });
+    readInt(conn->from);
+}
+
+
 void RemoteStore::connect()
 {
     auto conn(getConnection());
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 0fd67f371..4754ff45a 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -116,6 +116,8 @@ public:
         StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
         uint64_t & downloadSize, uint64_t & narSize) override;
 
+    void addBuildLog(const StorePath & drvPath, std::string_view log) override;
+
     void connect() override;
 
     unsigned int getProtocol() override;
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 3567dcd1c..07f45d1e9 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -727,6 +727,9 @@ public:
     virtual std::optional<std::string> getBuildLog(const StorePath & path)
     { return std::nullopt; }
 
+    virtual void addBuildLog(const StorePath & path, std::string_view log)
+    { unsupported("addBuildLog"); }
+
     /* Hack to allow long-running processes like hydra-queue-runner to
        occasionally flush their path info cache. */
     void clearPathInfoCache()
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index 93cf546d2..ecf42a5d0 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -56,6 +56,7 @@ typedef enum {
     wopRegisterDrvOutput = 42,
     wopQueryRealisation = 43,
     wopAddMultipleToStore = 44,
+    wopAddBuildLog = 45,
 } WorkerOp;
 
 
diff --git a/src/nix/copy.cc b/src/nix/copy.cc
index 9f7cef304..8730a9a5c 100644
--- a/src/nix/copy.cc
+++ b/src/nix/copy.cc
@@ -4,7 +4,7 @@
 
 using namespace nix;
 
-struct CmdCopy : CopyCommand
+struct CmdCopy : virtual CopyCommand, virtual BuiltPathsCommand
 {
     CheckSigsFlag checkSigs = CheckSigs;
 
@@ -45,8 +45,10 @@ struct CmdCopy : CopyCommand
 
     Category category() override { return catSecondary; }
 
-    void run(ref<Store> srcStore, ref<Store> dstStore, BuiltPaths && paths) override
+    void run(ref<Store> srcStore, BuiltPaths && paths) override
     {
+        auto dstStore = getDstStore();
+
         RealisedPath::Set stuffToCopy;
 
         for (auto & builtPath : paths) {
diff --git a/src/nix/store-copy-log.cc b/src/nix/store-copy-log.cc
new file mode 100644
index 000000000..fa6436cd0
--- /dev/null
+++ b/src/nix/store-copy-log.cc
@@ -0,0 +1,40 @@
+#include "command.hh"
+#include "shared.hh"
+#include "store-api.hh"
+#include "sync.hh"
+#include "thread-pool.hh"
+
+#include <atomic>
+
+using namespace nix;
+
+struct CmdCopyLog : virtual CopyCommand, virtual InstallablesCommand
+{
+    std::string description() override
+    {
+        return "copy build logs between Nix stores";
+    }
+
+    std::string doc() override
+    {
+        return
+          #include "store-copy-log.md"
+          ;
+    }
+
+    Category category() override { return catUtility; }
+
+    void run(ref<Store> srcStore) override
+    {
+        auto dstStore = getDstStore();
+
+        for (auto & path : toDerivations(srcStore, installables, true)) {
+            if (auto log = srcStore->getBuildLog(path))
+                dstStore->addBuildLog(path, *log);
+            else
+                throw Error("build log for '%s' is not available", srcStore->printStorePath(path));
+        }
+    }
+};
+
+static auto rCmdCopyLog = registerCommand2<CmdCopyLog>({"store", "copy-log"});
diff --git a/src/nix/store-copy-log.md b/src/nix/store-copy-log.md
new file mode 100644
index 000000000..f0cb66e57
--- /dev/null
+++ b/src/nix/store-copy-log.md
@@ -0,0 +1,13 @@
+R""(
+
+# Examples
+
+TODO
+
+# Description
+
+`nix store copy-log` copies build logs between two Nix stores. The
+source store is specified using `--from` and the destination using
+`--to`. If one of these is omitted, it defaults to the local store.
+
+)""