Merge pull request #5929 from edolstra/copy-logs

Add command to copy build logs between stores

doc/manual/src/release-notes/rl-next.md

@@ -7,3 +7,5 @@
   set or toggle display of error traces.
 * New builtin function `builtins.zipAttrsWith` with same functionality
   as `lib.zipAttrsWith` from nixpkgs, but much more efficient.
+* New command `nix store copy-log` to copy build logs from one store
+  to another.

src/libcmd/command.cc

@@ -54,6 +54,36 @@ void StoreCommand::run()
     run(getStore());
 }
 
+CopyCommand::CopyCommand()
+{
+    addFlag({
+        .longName = "from",
+        .description = "URL of the source Nix store.",
+        .labels = {"store-uri"},
+        .handler = {&srcUri},
+    });
+
+    addFlag({
+        .longName = "to",
+        .description = "URL of the destination Nix store.",
+        .labels = {"store-uri"},
+        .handler = {&dstUri},
+    });
+}
+
+ref<Store> CopyCommand::createStore()
+{
+    return srcUri.empty() ? StoreCommand::createStore() : openStore(srcUri);
+}
+
+ref<Store> CopyCommand::getDstStore()
+{
+    if (srcUri.empty() && dstUri.empty())
+        throw UsageError("you must pass '--from' and/or '--to'");
+
+    return dstUri.empty() ? openStore() : openStore(dstUri);
+}
+
 EvalCommand::EvalCommand()
 {
 }
@@ -73,13 +103,16 @@ ref<Store> EvalCommand::getEvalStore()
 
 ref<EvalState> EvalCommand::getEvalState()
 {
-    if (!evalState) evalState =
+    if (!evalState)
+        evalState =
             #if HAVE_BOEHMGC
             std::allocate_shared<EvalState>(traceable_allocator<EvalState>(),
+                searchPath, getEvalStore(), getStore())
             #else
             std::make_shared<EvalState>(
+                searchPath, getEvalStore(), getStore())
             #endif
-            searchPath, getEvalStore(), getStore());
+            ;
     return ref<EvalState>(evalState);
 }
 

src/libcmd/command.hh

@@ -43,6 +43,19 @@ private:
     std::shared_ptr<Store> _store;
 };
 
+/* A command that copies something between `--from` and `--to`
+   stores. */
+struct CopyCommand : virtual StoreCommand
+{
+    std::string srcUri, dstUri;
+
+    CopyCommand();
+
+    ref<Store> createStore() override;
+
+    ref<Store> getDstStore();
+};
+
 struct EvalCommand : virtual StoreCommand, MixEvalArgs
 {
     EvalCommand();

src/libcmd/installables.cc

@@ -345,6 +345,18 @@ Installable::getCursor(EvalState & state)
     return cursors[0];
 }
 
+static StorePath getDeriver(
+    ref<Store> store,
+    const Installable & i,
+    const StorePath & drvPath)
+{
+    auto derivers = store->queryValidDerivers(drvPath);
+    if (derivers.empty())
+        throw Error("'%s' does not have a known deriver", i.what());
+    // FIXME: use all derivers?
+    return *derivers.begin();
+}
+
 struct InstallableStorePath : Installable
 {
     ref<Store> store;
@@ -353,7 +365,7 @@ struct InstallableStorePath : Installable
     InstallableStorePath(ref<Store> store, StorePath && storePath)
         : store(store), storePath(std::move(storePath)) { }
 
-    std::string what() override { return store->printStorePath(storePath); }
+    std::string what() const override { return store->printStorePath(storePath); }
 
     DerivedPaths toDerivedPaths() override
     {
@@ -374,6 +386,15 @@ struct InstallableStorePath : Installable
         }
     }
 
+    StorePathSet toDrvPaths(ref<Store> store) override
+    {
+        if (storePath.isDerivation()) {
+            return {storePath};
+        } else {
+            return {getDeriver(store, *this, storePath)};
+        }
+    }
+
     std::optional<StorePath> getStorePath() override
     {
         return storePath;
@@ -402,6 +423,14 @@ DerivedPaths InstallableValue::toDerivedPaths()
     return res;
 }
 
+StorePathSet InstallableValue::toDrvPaths(ref<Store> store)
+{
+    StorePathSet res;
+    for (auto & drv : toDerivations())
+        res.insert(drv.drvPath);
+    return res;
+}
+
 struct InstallableAttrPath : InstallableValue
 {
     SourceExprCommand & cmd;
@@ -412,7 +441,7 @@ struct InstallableAttrPath : InstallableValue
         : InstallableValue(state), cmd(cmd), v(allocRootValue(v)), attrPath(attrPath)
     { }
 
-    std::string what() override { return attrPath; }
+    std::string what() const override { return attrPath; }
 
     std::pair<Value *, Pos> toValue(EvalState & state) override
     {
@@ -836,11 +865,7 @@ StorePathSet toDerivations(
                 [&](const DerivedPath::Opaque & bo) {
                     if (!useDeriver)
                         throw Error("argument '%s' did not evaluate to a derivation", i->what());
-                    auto derivers = store->queryValidDerivers(bo.path);
+                    drvPaths.insert(getDeriver(store, *i, bo.path));
-                    if (derivers.empty())
-                        throw Error("'%s' does not have a known deriver", i->what());
-                    // FIXME: use all derivers?
-                    drvPaths.insert(*derivers.begin());
                 },
                 [&](const DerivedPath::Built & bfd) {
                     drvPaths.insert(bfd.drvPath);

src/libcmd/installables.hh

@@ -33,10 +33,15 @@ struct Installable
 {
     virtual ~Installable() { }
 
-    virtual std::string what() = 0;
+    virtual std::string what() const = 0;
 
     virtual DerivedPaths toDerivedPaths() = 0;
 
+    virtual StorePathSet toDrvPaths(ref<Store> store)
+    {
+        throw Error("'%s' cannot be converted to a derivation path", what());
+    }
+
     DerivedPath toDerivedPath();
 
     UnresolvedApp toApp(EvalState & state);
@@ -81,6 +86,8 @@ struct InstallableValue : Installable
     virtual std::vector<DerivationInfo> toDerivations() = 0;
 
     DerivedPaths toDerivedPaths() override;
+
+    StorePathSet toDrvPaths(ref<Store> store) override;
 };
 
 struct InstallableFlake : InstallableValue
@@ -99,7 +106,7 @@ struct InstallableFlake : InstallableValue
         Strings && prefixes,
         const flake::LockFlags & lockFlags);
 
-    std::string what() override { return flakeRef.to_string() + "#" + *attrPaths.begin(); }
+    std::string what() const override { return flakeRef.to_string() + "#" + *attrPaths.begin(); }
 
     std::vector<std::string> getActualAttrPaths();
 

src/libstore/binary-cache-store.cc

@@ -512,4 +512,14 @@ std::optional<std::string> BinaryCacheStore::getBuildLog(const StorePath & path)
     return getFile(logPath);
 }
 
+void BinaryCacheStore::addBuildLog(const StorePath & drvPath, std::string_view log)
+{
+    assert(drvPath.isDerivation());
+
+    upsertFile(
+        "log/" + std::string(drvPath.to_string()),
+        (std::string) log, // FIXME: don't copy
+        "text/plain; charset=utf-8");
+}
+
 }

src/libstore/binary-cache-store.hh

@@ -51,6 +51,7 @@ public:
         const std::string & mimeType) = 0;
 
     void upsertFile(const std::string & path,
+        // FIXME: use std::string_view
         std::string && data,
         const std::string & mimeType);
 
@@ -120,6 +121,8 @@ public:
 
     std::optional<std::string> getBuildLog(const StorePath & path) override;
 
+    void addBuildLog(const StorePath & drvPath, std::string_view log) override;
+
 };
 
 MakeError(NoSuchBinaryCacheFile, Error);

src/libstore/daemon.cc

@@ -468,10 +468,12 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
             dontCheckSigs = false;
 
         logger->startWork();
+        {
             FramedSource source(from);
             store->addMultipleToStore(source,
                 RepairFlag{repair},
                 dontCheckSigs ? NoCheckSigs : CheckSigs);
+        }
         logger->stopWork();
         break;
     }
@@ -920,6 +922,22 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         break;
     }
 
+    case wopAddBuildLog: {
+        StorePath path{readString(from)};
+        logger->startWork();
+        if (!trusted)
+            throw Error("you are not privileged to add logs");
+        {
+            FramedSource source(from);
+            StringSink sink;
+            source.drainInto(sink);
+            store->addBuildLog(path, sink.s);
+        }
+        logger->stopWork();
+        to << 1;
+        break;
+    }
+
     default:
         throw Error("invalid operation %1%", op);
     }

src/libstore/local-binary-cache-store.cc

@@ -96,6 +96,7 @@ void LocalBinaryCacheStore::init()
     createDirs(binaryCacheDir + "/" + realisationsPrefix);
     if (writeDebugInfo)
         createDirs(binaryCacheDir + "/debuginfo");
+    createDirs(binaryCacheDir + "/log");
     BinaryCacheStore::init();
 }
 

src/libstore/local-store.cc

@@ -9,6 +9,7 @@
 #include "callback.hh"
 #include "topo-sort.hh"
 #include "finally.hh"
+#include "compression.hh"
 
 #include <iostream>
 #include <algorithm>
@@ -1898,4 +1899,24 @@ FixedOutputHash LocalStore::hashCAPath(
     };
 }
 
+void LocalStore::addBuildLog(const StorePath & drvPath, std::string_view log)
+{
+    assert(drvPath.isDerivation());
+
+    auto baseName = drvPath.to_string();
+
+    auto logPath = fmt("%s/%s/%s/%s.bz2", logDir, drvsLogDir, baseName.substr(0, 2), baseName.substr(2));
+
+    if (pathExists(logPath)) return;
+
+    createDirs(dirOf(logPath));
+
+    auto tmpFile = fmt("%s.tmp.%d", logPath, getpid());
+
+    writeFile(tmpFile, compress("bzip2", log));
+
+    if (rename(tmpFile.c_str(), logPath.c_str()) != 0)
+        throw SysError("renaming '%1%' to '%2%'", tmpFile, logPath);
+}
+
 }  // namespace nix

src/libstore/local-store.hh

@@ -280,6 +280,8 @@ private:
         const std::string_view pathHash
     );
 
+    void addBuildLog(const StorePath & drvPath, std::string_view log) override;
+
     friend struct LocalDerivationGoal;
     friend struct PathSubstitutionGoal;
     friend struct SubstitutionGoal;

src/libstore/remote-store.cc

@@ -908,6 +908,18 @@ void RemoteStore::queryMissing(const std::vector<DerivedPath> & targets,
 }
 
 
+void RemoteStore::addBuildLog(const StorePath & drvPath, std::string_view log)
+{
+    auto conn(getConnection());
+    conn->to << wopAddBuildLog << drvPath.to_string();
+    StringSource source(log);
+    conn.withFramedSink([&](Sink & sink) {
+        source.drainInto(sink);
+    });
+    readInt(conn->from);
+}
+
+
 void RemoteStore::connect()
 {
     auto conn(getConnection());

src/libstore/remote-store.hh

@@ -116,6 +116,8 @@ public:
         StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
         uint64_t & downloadSize, uint64_t & narSize) override;
 
+    void addBuildLog(const StorePath & drvPath, std::string_view log) override;
+
     void connect() override;
 
     unsigned int getProtocol() override;

src/libstore/store-api.hh

@@ -727,6 +727,9 @@ public:
     virtual std::optional<std::string> getBuildLog(const StorePath & path)
     { return std::nullopt; }
 
+    virtual void addBuildLog(const StorePath & path, std::string_view log)
+    { unsupported("addBuildLog"); }
+
     /* Hack to allow long-running processes like hydra-queue-runner to
        occasionally flush their path info cache. */
     void clearPathInfoCache()

src/libstore/worker-protocol.hh

@@ -56,6 +56,7 @@ typedef enum {
     wopRegisterDrvOutput = 42,
     wopQueryRealisation = 43,
     wopAddMultipleToStore = 44,
+    wopAddBuildLog = 45,
 } WorkerOp;
 
 

src/nix/app.cc

@@ -19,7 +19,7 @@ struct InstallableDerivedPath : Installable
     }
 
 
-    std::string what() override { return derivedPath.to_string(*store); }
+    std::string what() const override { return derivedPath.to_string(*store); }
 
     DerivedPaths toDerivedPaths() override
     {

src/nix/copy.cc

@@ -1,17 +1,11 @@
 #include "command.hh"
 #include "shared.hh"
 #include "store-api.hh"
-#include "sync.hh"
-#include "thread-pool.hh"
-
-#include <atomic>
 
 using namespace nix;
 
-struct CmdCopy : BuiltPathsCommand
+struct CmdCopy : virtual CopyCommand, virtual BuiltPathsCommand
 {
-    std::string srcUri, dstUri;
-
     CheckSigsFlag checkSigs = CheckSigs;
 
     SubstituteFlag substitute = NoSubstitute;
@@ -21,20 +15,6 @@ struct CmdCopy : BuiltPathsCommand
     CmdCopy()
         : BuiltPathsCommand(true)
     {
-        addFlag({
-            .longName = "from",
-            .description = "URL of the source Nix store.",
-            .labels = {"store-uri"},
-            .handler = {&srcUri},
-        });
-
-        addFlag({
-            .longName = "to",
-            .description = "URL of the destination Nix store.",
-            .labels = {"store-uri"},
-            .handler = {&dstUri},
-        });
-
         addFlag({
             .longName = "no-check-sigs",
             .description = "Do not require that paths are signed by trusted keys.",
@@ -65,22 +45,9 @@ struct CmdCopy : BuiltPathsCommand
 
     Category category() override { return catSecondary; }
 
-    ref<Store> createStore() override
-    {
-        return srcUri.empty() ? StoreCommand::createStore() : openStore(srcUri);
-    }
-
-    void run(ref<Store> store) override
-    {
-        if (srcUri.empty() && dstUri.empty())
-            throw UsageError("you must pass '--from' and/or '--to'");
-
-        BuiltPathsCommand::run(store);
-    }
-
     void run(ref<Store> srcStore, BuiltPaths && paths) override
     {
-        ref<Store> dstStore = dstUri.empty() ? openStore() : openStore(dstUri);
+        auto dstStore = getDstStore();
 
         RealisedPath::Set stuffToCopy;
 

src/nix/store-copy-log.cc

@@ -0,0 +1,46 @@
+#include "command.hh"
+#include "shared.hh"
+#include "store-api.hh"
+#include "sync.hh"
+#include "thread-pool.hh"
+
+#include <atomic>
+
+using namespace nix;
+
+struct CmdCopyLog : virtual CopyCommand, virtual InstallablesCommand
+{
+    std::string description() override
+    {
+        return "copy build logs between Nix stores";
+    }
+
+    std::string doc() override
+    {
+        return
+          #include "store-copy-log.md"
+          ;
+    }
+
+    Category category() override { return catUtility; }
+
+    void run(ref<Store> srcStore) override
+    {
+        auto dstStore = getDstStore();
+
+        StorePathSet drvPaths;
+
+        for (auto & i : installables)
+            for (auto & drvPath : i->toDrvPaths(getEvalStore()))
+                drvPaths.insert(drvPath);
+
+        for (auto & drvPath : drvPaths) {
+            if (auto log = srcStore->getBuildLog(drvPath))
+                dstStore->addBuildLog(drvPath, *log);
+            else
+                throw Error("build log for '%s' is not available", srcStore->printStorePath(drvPath));
+        }
+    }
+};
+
+static auto rCmdCopyLog = registerCommand2<CmdCopyLog>({"store", "copy-log"});

src/nix/store-copy-log.md

@@ -0,0 +1,33 @@
+R""(
+
+# Examples
+
+* To copy the build log of the `hello` package from
+  https://cache.nixos.org to the local store:
+
+  ```console
+  # nix store copy-log --from https://cache.nixos.org --eval-store auto nixpkgs#hello
+  ```
+
+  You can verify that the log is available locally:
+
+  ```console
+  # nix log --substituters '' nixpkgs#hello
+  ```
+
+  (The flag `--substituters ''` avoids querying
+  `https://cache.nixos.org` for the log.)
+
+* To copy the log for a specific store derivation via SSH:
+
+  ```console
+  # nix store copy-log --to ssh-ng://machine /nix/store/ilgm50plpmcgjhcp33z6n4qbnpqfhxym-glibc-2.33-59.drv
+  ```
+
+# Description
+
+`nix store copy-log` copies build logs between two Nix stores. The
+source store is specified using `--from` and the destination using
+`--to`. If one of these is omitted, it defaults to the local store.
+
+)""

tests/binary-cache.sh

@@ -14,6 +14,17 @@ outPath=$(nix-build dependencies.nix --no-out-link)
 
 nix copy --to file://$cacheDir $outPath
 
+# Test copying build logs to the binary cache.
+nix log --store file://$cacheDir $outPath 2>&1 | grep 'is not available'
+nix store copy-log --to file://$cacheDir $outPath
+nix log --store file://$cacheDir $outPath | grep FOO
+rm -rf $TEST_ROOT/var/log/nix
+nix log $outPath 2>&1 | grep 'is not available'
+nix log --substituters file://$cacheDir $outPath | grep FOO
+
+# Test copying build logs from the binary cache.
+nix store copy-log --from file://$cacheDir $(nix-store -qd $outPath)
+nix log $outPath | grep FOO
 
 basicDownloadTests() {
     # No uploading tests bcause upload with force HTTP doesn't work.