diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index f51969ced..adf3010c0 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -7,3 +7,5 @@ set or toggle display of error traces. * New builtin function `builtins.zipAttrsWith` with same functionality as `lib.zipAttrsWith` from nixpkgs, but much more efficient. +* New command `nix store copy-log` to copy build logs from one store + to another. diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 429cd32cc..6d183dfad 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -54,6 +54,36 @@ void StoreCommand::run() run(getStore()); } +CopyCommand::CopyCommand() +{ + addFlag({ + .longName = "from", + .description = "URL of the source Nix store.", + .labels = {"store-uri"}, + .handler = {&srcUri}, + }); + + addFlag({ + .longName = "to", + .description = "URL of the destination Nix store.", + .labels = {"store-uri"}, + .handler = {&dstUri}, + }); +} + +ref CopyCommand::createStore() +{ + return srcUri.empty() ? StoreCommand::createStore() : openStore(srcUri); +} + +ref CopyCommand::getDstStore() +{ + if (srcUri.empty() && dstUri.empty()) + throw UsageError("you must pass '--from' and/or '--to'"); + + return dstUri.empty() ? openStore() : openStore(dstUri); +} + EvalCommand::EvalCommand() { } @@ -73,13 +103,16 @@ ref EvalCommand::getEvalStore() ref EvalCommand::getEvalState() { - if (!evalState) evalState = -#if HAVE_BOEHMGC - std::allocate_shared(traceable_allocator(), -#else - std::make_shared( -#endif - searchPath, getEvalStore(), getStore()); + if (!evalState) + evalState = + #if HAVE_BOEHMGC + std::allocate_shared(traceable_allocator(), + searchPath, getEvalStore(), getStore()) + #else + std::make_shared( + searchPath, getEvalStore(), getStore()) + #endif + ; return ref(evalState); } diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index 07f398468..bd2a0a7ee 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -43,6 +43,19 @@ private: std::shared_ptr _store; }; +/* A command that copies something between `--from` and `--to` + stores. */ +struct CopyCommand : virtual StoreCommand +{ + std::string srcUri, dstUri; + + CopyCommand(); + + ref createStore() override; + + ref getDstStore(); +}; + struct EvalCommand : virtual StoreCommand, MixEvalArgs { EvalCommand(); diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 6c20bb7b1..0c2db0399 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -345,6 +345,18 @@ Installable::getCursor(EvalState & state) return cursors[0]; } +static StorePath getDeriver( + ref store, + const Installable & i, + const StorePath & drvPath) +{ + auto derivers = store->queryValidDerivers(drvPath); + if (derivers.empty()) + throw Error("'%s' does not have a known deriver", i.what()); + // FIXME: use all derivers? + return *derivers.begin(); +} + struct InstallableStorePath : Installable { ref store; @@ -353,7 +365,7 @@ struct InstallableStorePath : Installable InstallableStorePath(ref store, StorePath && storePath) : store(store), storePath(std::move(storePath)) { } - std::string what() override { return store->printStorePath(storePath); } + std::string what() const override { return store->printStorePath(storePath); } DerivedPaths toDerivedPaths() override { @@ -374,6 +386,15 @@ struct InstallableStorePath : Installable } } + StorePathSet toDrvPaths(ref store) override + { + if (storePath.isDerivation()) { + return {storePath}; + } else { + return {getDeriver(store, *this, storePath)}; + } + } + std::optional getStorePath() override { return storePath; @@ -402,6 +423,14 @@ DerivedPaths InstallableValue::toDerivedPaths() return res; } +StorePathSet InstallableValue::toDrvPaths(ref store) +{ + StorePathSet res; + for (auto & drv : toDerivations()) + res.insert(drv.drvPath); + return res; +} + struct InstallableAttrPath : InstallableValue { SourceExprCommand & cmd; @@ -412,7 +441,7 @@ struct InstallableAttrPath : InstallableValue : InstallableValue(state), cmd(cmd), v(allocRootValue(v)), attrPath(attrPath) { } - std::string what() override { return attrPath; } + std::string what() const override { return attrPath; } std::pair toValue(EvalState & state) override { @@ -836,11 +865,7 @@ StorePathSet toDerivations( [&](const DerivedPath::Opaque & bo) { if (!useDeriver) throw Error("argument '%s' did not evaluate to a derivation", i->what()); - auto derivers = store->queryValidDerivers(bo.path); - if (derivers.empty()) - throw Error("'%s' does not have a known deriver", i->what()); - // FIXME: use all derivers? - drvPaths.insert(*derivers.begin()); + drvPaths.insert(getDeriver(store, *i, bo.path)); }, [&](const DerivedPath::Built & bfd) { drvPaths.insert(bfd.drvPath); diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index 79931ad3e..ced6b3f10 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -33,10 +33,15 @@ struct Installable { virtual ~Installable() { } - virtual std::string what() = 0; + virtual std::string what() const = 0; virtual DerivedPaths toDerivedPaths() = 0; + virtual StorePathSet toDrvPaths(ref store) + { + throw Error("'%s' cannot be converted to a derivation path", what()); + } + DerivedPath toDerivedPath(); UnresolvedApp toApp(EvalState & state); @@ -81,6 +86,8 @@ struct InstallableValue : Installable virtual std::vector toDerivations() = 0; DerivedPaths toDerivedPaths() override; + + StorePathSet toDrvPaths(ref store) override; }; struct InstallableFlake : InstallableValue @@ -99,7 +106,7 @@ struct InstallableFlake : InstallableValue Strings && prefixes, const flake::LockFlags & lockFlags); - std::string what() override { return flakeRef.to_string() + "#" + *attrPaths.begin(); } + std::string what() const override { return flakeRef.to_string() + "#" + *attrPaths.begin(); } std::vector getActualAttrPaths(); diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 7d25e2160..6e4458f7a 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -512,4 +512,14 @@ std::optional BinaryCacheStore::getBuildLog(const StorePath & path) return getFile(logPath); } +void BinaryCacheStore::addBuildLog(const StorePath & drvPath, std::string_view log) +{ + assert(drvPath.isDerivation()); + + upsertFile( + "log/" + std::string(drvPath.to_string()), + (std::string) log, // FIXME: don't copy + "text/plain; charset=utf-8"); +} + } diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 46ff67c77..7599230d9 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -51,6 +51,7 @@ public: const std::string & mimeType) = 0; void upsertFile(const std::string & path, + // FIXME: use std::string_view std::string && data, const std::string & mimeType); @@ -120,6 +121,8 @@ public: std::optional getBuildLog(const StorePath & path) override; + void addBuildLog(const StorePath & drvPath, std::string_view log) override; + }; MakeError(NoSuchBinaryCacheFile, Error); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 5b817c587..101aa13a5 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -468,10 +468,12 @@ static void performOp(TunnelLogger * logger, ref store, dontCheckSigs = false; logger->startWork(); - FramedSource source(from); - store->addMultipleToStore(source, - RepairFlag{repair}, - dontCheckSigs ? NoCheckSigs : CheckSigs); + { + FramedSource source(from); + store->addMultipleToStore(source, + RepairFlag{repair}, + dontCheckSigs ? NoCheckSigs : CheckSigs); + } logger->stopWork(); break; } @@ -920,6 +922,22 @@ static void performOp(TunnelLogger * logger, ref store, break; } + case wopAddBuildLog: { + StorePath path{readString(from)}; + logger->startWork(); + if (!trusted) + throw Error("you are not privileged to add logs"); + { + FramedSource source(from); + StringSink sink; + source.drainInto(sink); + store->addBuildLog(path, sink.s); + } + logger->stopWork(); + to << 1; + break; + } + default: throw Error("invalid operation %1%", op); } diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index f93111fce..f754770f9 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -96,6 +96,7 @@ void LocalBinaryCacheStore::init() createDirs(binaryCacheDir + "/" + realisationsPrefix); if (writeDebugInfo) createDirs(binaryCacheDir + "/debuginfo"); + createDirs(binaryCacheDir + "/log"); BinaryCacheStore::init(); } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index d3cebe720..1807940d8 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -9,6 +9,7 @@ #include "callback.hh" #include "topo-sort.hh" #include "finally.hh" +#include "compression.hh" #include #include @@ -1898,4 +1899,24 @@ FixedOutputHash LocalStore::hashCAPath( }; } +void LocalStore::addBuildLog(const StorePath & drvPath, std::string_view log) +{ + assert(drvPath.isDerivation()); + + auto baseName = drvPath.to_string(); + + auto logPath = fmt("%s/%s/%s/%s.bz2", logDir, drvsLogDir, baseName.substr(0, 2), baseName.substr(2)); + + if (pathExists(logPath)) return; + + createDirs(dirOf(logPath)); + + auto tmpFile = fmt("%s.tmp.%d", logPath, getpid()); + + writeFile(tmpFile, compress("bzip2", log)); + + if (rename(tmpFile.c_str(), logPath.c_str()) != 0) + throw SysError("renaming '%1%' to '%2%'", tmpFile, logPath); +} + } // namespace nix diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index c4d7b80bd..6d867d778 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -280,6 +280,8 @@ private: const std::string_view pathHash ); + void addBuildLog(const StorePath & drvPath, std::string_view log) override; + friend struct LocalDerivationGoal; friend struct PathSubstitutionGoal; friend struct SubstitutionGoal; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 6886103e1..aac2965e0 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -908,6 +908,18 @@ void RemoteStore::queryMissing(const std::vector & targets, } +void RemoteStore::addBuildLog(const StorePath & drvPath, std::string_view log) +{ + auto conn(getConnection()); + conn->to << wopAddBuildLog << drvPath.to_string(); + StringSource source(log); + conn.withFramedSink([&](Sink & sink) { + source.drainInto(sink); + }); + readInt(conn->from); +} + + void RemoteStore::connect() { auto conn(getConnection()); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 0fd67f371..4754ff45a 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -116,6 +116,8 @@ public: StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, uint64_t & downloadSize, uint64_t & narSize) override; + void addBuildLog(const StorePath & drvPath, std::string_view log) override; + void connect() override; unsigned int getProtocol() override; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 3567dcd1c..07f45d1e9 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -727,6 +727,9 @@ public: virtual std::optional getBuildLog(const StorePath & path) { return std::nullopt; } + virtual void addBuildLog(const StorePath & path, std::string_view log) + { unsupported("addBuildLog"); } + /* Hack to allow long-running processes like hydra-queue-runner to occasionally flush their path info cache. */ void clearPathInfoCache() diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 93cf546d2..ecf42a5d0 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -56,6 +56,7 @@ typedef enum { wopRegisterDrvOutput = 42, wopQueryRealisation = 43, wopAddMultipleToStore = 44, + wopAddBuildLog = 45, } WorkerOp; diff --git a/src/nix/app.cc b/src/nix/app.cc index 2fcf4752c..e104cc9c1 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -19,7 +19,7 @@ struct InstallableDerivedPath : Installable } - std::string what() override { return derivedPath.to_string(*store); } + std::string what() const override { return derivedPath.to_string(*store); } DerivedPaths toDerivedPaths() override { diff --git a/src/nix/copy.cc b/src/nix/copy.cc index 197c85316..8730a9a5c 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -1,17 +1,11 @@ #include "command.hh" #include "shared.hh" #include "store-api.hh" -#include "sync.hh" -#include "thread-pool.hh" - -#include using namespace nix; -struct CmdCopy : BuiltPathsCommand +struct CmdCopy : virtual CopyCommand, virtual BuiltPathsCommand { - std::string srcUri, dstUri; - CheckSigsFlag checkSigs = CheckSigs; SubstituteFlag substitute = NoSubstitute; @@ -21,20 +15,6 @@ struct CmdCopy : BuiltPathsCommand CmdCopy() : BuiltPathsCommand(true) { - addFlag({ - .longName = "from", - .description = "URL of the source Nix store.", - .labels = {"store-uri"}, - .handler = {&srcUri}, - }); - - addFlag({ - .longName = "to", - .description = "URL of the destination Nix store.", - .labels = {"store-uri"}, - .handler = {&dstUri}, - }); - addFlag({ .longName = "no-check-sigs", .description = "Do not require that paths are signed by trusted keys.", @@ -65,22 +45,9 @@ struct CmdCopy : BuiltPathsCommand Category category() override { return catSecondary; } - ref createStore() override - { - return srcUri.empty() ? StoreCommand::createStore() : openStore(srcUri); - } - - void run(ref store) override - { - if (srcUri.empty() && dstUri.empty()) - throw UsageError("you must pass '--from' and/or '--to'"); - - BuiltPathsCommand::run(store); - } - void run(ref srcStore, BuiltPaths && paths) override { - ref dstStore = dstUri.empty() ? openStore() : openStore(dstUri); + auto dstStore = getDstStore(); RealisedPath::Set stuffToCopy; diff --git a/src/nix/store-copy-log.cc b/src/nix/store-copy-log.cc new file mode 100644 index 000000000..079cd6b3e --- /dev/null +++ b/src/nix/store-copy-log.cc @@ -0,0 +1,46 @@ +#include "command.hh" +#include "shared.hh" +#include "store-api.hh" +#include "sync.hh" +#include "thread-pool.hh" + +#include + +using namespace nix; + +struct CmdCopyLog : virtual CopyCommand, virtual InstallablesCommand +{ + std::string description() override + { + return "copy build logs between Nix stores"; + } + + std::string doc() override + { + return + #include "store-copy-log.md" + ; + } + + Category category() override { return catUtility; } + + void run(ref srcStore) override + { + auto dstStore = getDstStore(); + + StorePathSet drvPaths; + + for (auto & i : installables) + for (auto & drvPath : i->toDrvPaths(getEvalStore())) + drvPaths.insert(drvPath); + + for (auto & drvPath : drvPaths) { + if (auto log = srcStore->getBuildLog(drvPath)) + dstStore->addBuildLog(drvPath, *log); + else + throw Error("build log for '%s' is not available", srcStore->printStorePath(drvPath)); + } + } +}; + +static auto rCmdCopyLog = registerCommand2({"store", "copy-log"}); diff --git a/src/nix/store-copy-log.md b/src/nix/store-copy-log.md new file mode 100644 index 000000000..19ae57079 --- /dev/null +++ b/src/nix/store-copy-log.md @@ -0,0 +1,33 @@ +R""( + +# Examples + +* To copy the build log of the `hello` package from + https://cache.nixos.org to the local store: + + ```console + # nix store copy-log --from https://cache.nixos.org --eval-store auto nixpkgs#hello + ``` + + You can verify that the log is available locally: + + ```console + # nix log --substituters '' nixpkgs#hello + ``` + + (The flag `--substituters ''` avoids querying + `https://cache.nixos.org` for the log.) + +* To copy the log for a specific store derivation via SSH: + + ```console + # nix store copy-log --to ssh-ng://machine /nix/store/ilgm50plpmcgjhcp33z6n4qbnpqfhxym-glibc-2.33-59.drv + ``` + +# Description + +`nix store copy-log` copies build logs between two Nix stores. The +source store is specified using `--from` and the destination using +`--to`. If one of these is omitted, it defaults to the local store. + +)"" diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh index d7bc1507b..2368884f7 100644 --- a/tests/binary-cache.sh +++ b/tests/binary-cache.sh @@ -14,6 +14,17 @@ outPath=$(nix-build dependencies.nix --no-out-link) nix copy --to file://$cacheDir $outPath +# Test copying build logs to the binary cache. +nix log --store file://$cacheDir $outPath 2>&1 | grep 'is not available' +nix store copy-log --to file://$cacheDir $outPath +nix log --store file://$cacheDir $outPath | grep FOO +rm -rf $TEST_ROOT/var/log/nix +nix log $outPath 2>&1 | grep 'is not available' +nix log --substituters file://$cacheDir $outPath | grep FOO + +# Test copying build logs from the binary cache. +nix store copy-log --from file://$cacheDir $(nix-store -qd $outPath) +nix log $outPath | grep FOO basicDownloadTests() { # No uploading tests bcause upload with force HTTP doesn't work.