From 4dda1f92aae05dd9d633152458d65a3815bcd03c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 17 Jan 2022 19:45:21 +0100 Subject: [PATCH] Add command 'nix store copy-log' Fixes #5222. --- doc/manual/src/release-notes/rl-next.md | 2 + src/libcmd/command.cc | 67 +++++++++++-------------- src/libcmd/command.hh | 30 +++++------ src/libstore/daemon.cc | 26 ++++++++-- src/libstore/local-store.cc | 21 ++++++++ src/libstore/local-store.hh | 2 + src/libstore/remote-store.cc | 12 +++++ src/libstore/remote-store.hh | 2 + src/libstore/store-api.hh | 3 ++ src/libstore/worker-protocol.hh | 1 + src/nix/copy.cc | 6 ++- src/nix/store-copy-log.cc | 40 +++++++++++++++ src/nix/store-copy-log.md | 13 +++++ 13 files changed, 165 insertions(+), 60 deletions(-) create mode 100644 src/nix/store-copy-log.cc create mode 100644 src/nix/store-copy-log.md diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index f51969ced..adf3010c0 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -7,3 +7,5 @@ set or toggle display of error traces. * New builtin function `builtins.zipAttrsWith` with same functionality as `lib.zipAttrsWith` from nixpkgs, but much more efficient. +* New command `nix store copy-log` to copy build logs from one store + to another. diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 5e6d4a857..6d183dfad 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -54,6 +54,36 @@ void StoreCommand::run() run(getStore()); } +CopyCommand::CopyCommand() +{ + addFlag({ + .longName = "from", + .description = "URL of the source Nix store.", + .labels = {"store-uri"}, + .handler = {&srcUri}, + }); + + addFlag({ + .longName = "to", + .description = "URL of the destination Nix store.", + .labels = {"store-uri"}, + .handler = {&dstUri}, + }); +} + +ref CopyCommand::createStore() +{ + return srcUri.empty() ? StoreCommand::createStore() : openStore(srcUri); +} + +ref CopyCommand::getDstStore() +{ + if (srcUri.empty() && dstUri.empty()) + throw UsageError("you must pass '--from' and/or '--to'"); + + return dstUri.empty() ? openStore() : openStore(dstUri); +} + EvalCommand::EvalCommand() { } @@ -159,43 +189,6 @@ void StorePathsCommand::run(ref store, BuiltPaths && paths) run(store, std::move(sorted)); } -CopyCommand::CopyCommand() - : BuiltPathsCommand(true) -{ - addFlag({ - .longName = "from", - .description = "URL of the source Nix store.", - .labels = {"store-uri"}, - .handler = {&srcUri}, - }); - - addFlag({ - .longName = "to", - .description = "URL of the destination Nix store.", - .labels = {"store-uri"}, - .handler = {&dstUri}, - }); -} - -ref CopyCommand::createStore() -{ - return srcUri.empty() ? StoreCommand::createStore() : openStore(srcUri); -} - -void CopyCommand::run(ref store) -{ - if (srcUri.empty() && dstUri.empty()) - throw UsageError("you must pass '--from' and/or '--to'"); - - BuiltPathsCommand::run(store); -} - -void CopyCommand::run(ref srcStore, BuiltPaths && paths) -{ - ref dstStore = dstUri.empty() ? openStore() : openStore(dstUri); - run(srcStore, dstStore, std::move(paths)); -} - void StorePathCommand::run(ref store, std::vector && storePaths) { if (storePaths.size() != 1) diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index 0c3e29e25..bd2a0a7ee 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -43,6 +43,19 @@ private: std::shared_ptr _store; }; +/* A command that copies something between `--from` and `--to` + stores. */ +struct CopyCommand : virtual StoreCommand +{ + std::string srcUri, dstUri; + + CopyCommand(); + + ref createStore() override; + + ref getDstStore(); +}; + struct EvalCommand : virtual StoreCommand, MixEvalArgs { EvalCommand(); @@ -176,23 +189,6 @@ public: bool useDefaultInstallables() override { return !all; } }; -/* A command that copies something between `--from` and `--to` - stores. */ -struct CopyCommand : virtual BuiltPathsCommand -{ - std::string srcUri, dstUri; - - CopyCommand(); - - ref createStore() override; - - void run(ref store) override; - - void run(ref srcStore, BuiltPaths && paths) override; - - virtual void run(ref srcStore, ref dstStore, BuiltPaths && paths) = 0; -}; - struct StorePathsCommand : public BuiltPathsCommand { StorePathsCommand(bool recursive = false); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 5b817c587..101aa13a5 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -468,10 +468,12 @@ static void performOp(TunnelLogger * logger, ref store, dontCheckSigs = false; logger->startWork(); - FramedSource source(from); - store->addMultipleToStore(source, - RepairFlag{repair}, - dontCheckSigs ? NoCheckSigs : CheckSigs); + { + FramedSource source(from); + store->addMultipleToStore(source, + RepairFlag{repair}, + dontCheckSigs ? NoCheckSigs : CheckSigs); + } logger->stopWork(); break; } @@ -920,6 +922,22 @@ static void performOp(TunnelLogger * logger, ref store, break; } + case wopAddBuildLog: { + StorePath path{readString(from)}; + logger->startWork(); + if (!trusted) + throw Error("you are not privileged to add logs"); + { + FramedSource source(from); + StringSink sink; + source.drainInto(sink); + store->addBuildLog(path, sink.s); + } + logger->stopWork(); + to << 1; + break; + } + default: throw Error("invalid operation %1%", op); } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index d3cebe720..1807940d8 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -9,6 +9,7 @@ #include "callback.hh" #include "topo-sort.hh" #include "finally.hh" +#include "compression.hh" #include #include @@ -1898,4 +1899,24 @@ FixedOutputHash LocalStore::hashCAPath( }; } +void LocalStore::addBuildLog(const StorePath & drvPath, std::string_view log) +{ + assert(drvPath.isDerivation()); + + auto baseName = drvPath.to_string(); + + auto logPath = fmt("%s/%s/%s/%s.bz2", logDir, drvsLogDir, baseName.substr(0, 2), baseName.substr(2)); + + if (pathExists(logPath)) return; + + createDirs(dirOf(logPath)); + + auto tmpFile = fmt("%s.tmp.%d", logPath, getpid()); + + writeFile(tmpFile, compress("bzip2", log)); + + if (rename(tmpFile.c_str(), logPath.c_str()) != 0) + throw SysError("renaming '%1%' to '%2%'", tmpFile, logPath); +} + } // namespace nix diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index c4d7b80bd..6d867d778 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -280,6 +280,8 @@ private: const std::string_view pathHash ); + void addBuildLog(const StorePath & drvPath, std::string_view log) override; + friend struct LocalDerivationGoal; friend struct PathSubstitutionGoal; friend struct SubstitutionGoal; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 6886103e1..aac2965e0 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -908,6 +908,18 @@ void RemoteStore::queryMissing(const std::vector & targets, } +void RemoteStore::addBuildLog(const StorePath & drvPath, std::string_view log) +{ + auto conn(getConnection()); + conn->to << wopAddBuildLog << drvPath.to_string(); + StringSource source(log); + conn.withFramedSink([&](Sink & sink) { + source.drainInto(sink); + }); + readInt(conn->from); +} + + void RemoteStore::connect() { auto conn(getConnection()); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 0fd67f371..4754ff45a 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -116,6 +116,8 @@ public: StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, uint64_t & downloadSize, uint64_t & narSize) override; + void addBuildLog(const StorePath & drvPath, std::string_view log) override; + void connect() override; unsigned int getProtocol() override; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 3567dcd1c..07f45d1e9 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -727,6 +727,9 @@ public: virtual std::optional getBuildLog(const StorePath & path) { return std::nullopt; } + virtual void addBuildLog(const StorePath & path, std::string_view log) + { unsupported("addBuildLog"); } + /* Hack to allow long-running processes like hydra-queue-runner to occasionally flush their path info cache. */ void clearPathInfoCache() diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 93cf546d2..ecf42a5d0 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -56,6 +56,7 @@ typedef enum { wopRegisterDrvOutput = 42, wopQueryRealisation = 43, wopAddMultipleToStore = 44, + wopAddBuildLog = 45, } WorkerOp; diff --git a/src/nix/copy.cc b/src/nix/copy.cc index 9f7cef304..8730a9a5c 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -4,7 +4,7 @@ using namespace nix; -struct CmdCopy : CopyCommand +struct CmdCopy : virtual CopyCommand, virtual BuiltPathsCommand { CheckSigsFlag checkSigs = CheckSigs; @@ -45,8 +45,10 @@ struct CmdCopy : CopyCommand Category category() override { return catSecondary; } - void run(ref srcStore, ref dstStore, BuiltPaths && paths) override + void run(ref srcStore, BuiltPaths && paths) override { + auto dstStore = getDstStore(); + RealisedPath::Set stuffToCopy; for (auto & builtPath : paths) { diff --git a/src/nix/store-copy-log.cc b/src/nix/store-copy-log.cc new file mode 100644 index 000000000..fa6436cd0 --- /dev/null +++ b/src/nix/store-copy-log.cc @@ -0,0 +1,40 @@ +#include "command.hh" +#include "shared.hh" +#include "store-api.hh" +#include "sync.hh" +#include "thread-pool.hh" + +#include + +using namespace nix; + +struct CmdCopyLog : virtual CopyCommand, virtual InstallablesCommand +{ + std::string description() override + { + return "copy build logs between Nix stores"; + } + + std::string doc() override + { + return + #include "store-copy-log.md" + ; + } + + Category category() override { return catUtility; } + + void run(ref srcStore) override + { + auto dstStore = getDstStore(); + + for (auto & path : toDerivations(srcStore, installables, true)) { + if (auto log = srcStore->getBuildLog(path)) + dstStore->addBuildLog(path, *log); + else + throw Error("build log for '%s' is not available", srcStore->printStorePath(path)); + } + } +}; + +static auto rCmdCopyLog = registerCommand2({"store", "copy-log"}); diff --git a/src/nix/store-copy-log.md b/src/nix/store-copy-log.md new file mode 100644 index 000000000..f0cb66e57 --- /dev/null +++ b/src/nix/store-copy-log.md @@ -0,0 +1,13 @@ +R""( + +# Examples + +TODO + +# Description + +`nix store copy-log` copies build logs between two Nix stores. The +source store is specified using `--from` and the destination using +`--to`. If one of these is omitted, it defaults to the local store. + +)""