2006-11-30 19:54:43 +00:00
|
|
|
|
#include "serialise.hh"
|
|
|
|
|
#include "util.hh"
|
2021-03-02 03:50:41 +00:00
|
|
|
|
#include "path-with-outputs.hh"
|
2020-09-07 09:26:09 +00:00
|
|
|
|
#include "remote-fs-accessor.hh"
|
2006-11-30 18:35:50 +00:00
|
|
|
|
#include "remote-store.hh"
|
2006-11-30 20:13:59 +00:00
|
|
|
|
#include "worker-protocol.hh"
|
2006-11-30 20:45:20 +00:00
|
|
|
|
#include "archive.hh"
|
2013-08-07 11:51:55 +00:00
|
|
|
|
#include "affinity.hh"
|
2006-12-04 13:09:16 +00:00
|
|
|
|
#include "globals.hh"
|
2015-09-03 10:56:59 +00:00
|
|
|
|
#include "derivations.hh"
|
2016-02-23 14:00:59 +00:00
|
|
|
|
#include "pool.hh"
|
2018-04-16 09:14:39 +00:00
|
|
|
|
#include "finally.hh"
|
2020-07-02 15:04:31 +00:00
|
|
|
|
#include "logging.hh"
|
2020-09-21 16:40:11 +00:00
|
|
|
|
#include "callback.hh"
|
2020-11-02 12:57:58 +00:00
|
|
|
|
#include "filetransfer.hh"
|
2021-01-26 09:48:41 +00:00
|
|
|
|
#include <nlohmann/json.hpp>
|
2006-11-30 18:35:50 +00:00
|
|
|
|
|
|
|
|
|
namespace nix {
|
|
|
|
|
|
2020-09-30 00:39:06 +00:00
|
|
|
|
namespace worker_proto {
|
|
|
|
|
|
|
|
|
|
std::string read(const Store & store, Source & from, Phantom<std::string> _)
|
2020-06-10 09:20:52 +00:00
|
|
|
|
{
|
2020-08-04 19:02:05 +00:00
|
|
|
|
return readString(from);
|
2020-06-10 09:20:52 +00:00
|
|
|
|
}
|
|
|
|
|
|
2020-09-30 00:39:06 +00:00
|
|
|
|
void write(const Store & store, Sink & out, const std::string & str)
|
2020-06-19 22:06:19 +00:00
|
|
|
|
{
|
2020-08-04 19:02:05 +00:00
|
|
|
|
out << str;
|
2020-06-19 22:06:19 +00:00
|
|
|
|
}
|
|
|
|
|
|
2020-06-10 09:20:52 +00:00
|
|
|
|
|
2020-09-30 00:39:06 +00:00
|
|
|
|
StorePath read(const Store & store, Source & from, Phantom<StorePath> _)
|
2020-06-19 22:06:19 +00:00
|
|
|
|
{
|
2020-08-04 19:02:05 +00:00
|
|
|
|
return store.parseStorePath(readString(from));
|
2020-06-19 22:06:19 +00:00
|
|
|
|
}
|
|
|
|
|
|
2020-09-30 00:39:06 +00:00
|
|
|
|
void write(const Store & store, Sink & out, const StorePath & storePath)
|
2020-06-10 09:20:52 +00:00
|
|
|
|
{
|
2020-08-04 19:02:05 +00:00
|
|
|
|
out << store.printStorePath(storePath);
|
2020-06-10 09:20:52 +00:00
|
|
|
|
}
|
|
|
|
|
|
2020-08-04 22:28:10 +00:00
|
|
|
|
|
2020-09-30 00:39:06 +00:00
|
|
|
|
ContentAddress read(const Store & store, Source & from, Phantom<ContentAddress> _)
|
2020-08-04 19:02:05 +00:00
|
|
|
|
{
|
|
|
|
|
return parseContentAddress(readString(from));
|
|
|
|
|
}
|
|
|
|
|
|
2020-09-30 00:39:06 +00:00
|
|
|
|
void write(const Store & store, Sink & out, const ContentAddress & ca)
|
2020-06-10 09:20:52 +00:00
|
|
|
|
{
|
2020-08-04 19:02:05 +00:00
|
|
|
|
out << renderContentAddress(ca);
|
2020-06-10 09:20:52 +00:00
|
|
|
|
}
|
2011-12-16 22:31:25 +00:00
|
|
|
|
|
2021-03-02 03:50:41 +00:00
|
|
|
|
|
2021-04-05 13:48:18 +00:00
|
|
|
|
DerivedPath read(const Store & store, Source & from, Phantom<DerivedPath> _)
|
2021-03-02 03:50:41 +00:00
|
|
|
|
{
|
|
|
|
|
auto s = readString(from);
|
2021-04-05 13:48:18 +00:00
|
|
|
|
return DerivedPath::parse(store, s);
|
2021-03-02 03:50:41 +00:00
|
|
|
|
}
|
|
|
|
|
|
2021-04-05 13:48:18 +00:00
|
|
|
|
void write(const Store & store, Sink & out, const DerivedPath & req)
|
2021-03-02 03:50:41 +00:00
|
|
|
|
{
|
2021-04-05 13:24:42 +00:00
|
|
|
|
out << req.to_string(store);
|
2021-03-02 03:50:41 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
2021-01-26 09:48:41 +00:00
|
|
|
|
Realisation read(const Store & store, Source & from, Phantom<Realisation> _)
|
|
|
|
|
{
|
|
|
|
|
std::string rawInput = readString(from);
|
|
|
|
|
return Realisation::fromJSON(
|
|
|
|
|
nlohmann::json::parse(rawInput),
|
|
|
|
|
"remote-protocol"
|
|
|
|
|
);
|
|
|
|
|
}
|
2021-03-02 03:50:41 +00:00
|
|
|
|
|
2021-01-26 09:48:41 +00:00
|
|
|
|
void write(const Store & store, Sink & out, const Realisation & realisation)
|
2021-03-02 03:50:41 +00:00
|
|
|
|
{
|
|
|
|
|
out << realisation.toJSON().dump();
|
|
|
|
|
}
|
|
|
|
|
|
2021-01-26 09:48:41 +00:00
|
|
|
|
|
|
|
|
|
DrvOutput read(const Store & store, Source & from, Phantom<DrvOutput> _)
|
2021-02-28 18:42:46 +00:00
|
|
|
|
{
|
|
|
|
|
return DrvOutput::parse(readString(from));
|
|
|
|
|
}
|
|
|
|
|
|
2021-01-26 09:48:41 +00:00
|
|
|
|
void write(const Store & store, Sink & out, const DrvOutput & drvOutput)
|
2021-02-28 18:42:46 +00:00
|
|
|
|
{
|
|
|
|
|
out << drvOutput.to_string();
|
|
|
|
|
}
|
|
|
|
|
|
2020-07-24 21:02:51 +00:00
|
|
|
|
|
2020-09-30 00:39:06 +00:00
|
|
|
|
std::optional<StorePath> read(const Store & store, Source & from, Phantom<std::optional<StorePath>> _)
|
2020-08-07 17:05:14 +00:00
|
|
|
|
{
|
2020-08-21 19:19:14 +00:00
|
|
|
|
auto s = readString(from);
|
2020-08-07 17:05:14 +00:00
|
|
|
|
return s == "" ? std::optional<StorePath> {} : store.parseStorePath(s);
|
|
|
|
|
}
|
|
|
|
|
|
2020-09-30 00:39:06 +00:00
|
|
|
|
void write(const Store & store, Sink & out, const std::optional<StorePath> & storePathOpt)
|
2020-08-07 17:05:14 +00:00
|
|
|
|
{
|
|
|
|
|
out << (storePathOpt ? store.printStorePath(*storePathOpt) : "");
|
|
|
|
|
}
|
|
|
|
|
|
2020-08-07 18:51:01 +00:00
|
|
|
|
|
2020-09-30 00:39:06 +00:00
|
|
|
|
std::optional<ContentAddress> read(const Store & store, Source & from, Phantom<std::optional<ContentAddress>> _)
|
2020-08-07 18:51:01 +00:00
|
|
|
|
{
|
|
|
|
|
return parseContentAddressOpt(readString(from));
|
|
|
|
|
}
|
|
|
|
|
|
2020-09-30 00:39:06 +00:00
|
|
|
|
void write(const Store & store, Sink & out, const std::optional<ContentAddress> & caOpt)
|
2020-08-07 18:51:01 +00:00
|
|
|
|
{
|
|
|
|
|
out << (caOpt ? renderContentAddress(*caOpt) : "");
|
2020-08-05 20:37:48 +00:00
|
|
|
|
}
|
|
|
|
|
|
2020-09-30 00:39:06 +00:00
|
|
|
|
}
|
|
|
|
|
|
2020-07-24 21:02:51 +00:00
|
|
|
|
|
2016-09-02 18:15:04 +00:00
|
|
|
|
/* TODO: Separate these store impls into different files, give them better names */
|
2017-03-03 18:05:50 +00:00
|
|
|
|
RemoteStore::RemoteStore(const Params & params)
|
2020-12-20 15:33:12 +00:00
|
|
|
|
: RemoteStoreConfig(params)
|
|
|
|
|
, Store(params)
|
2016-06-01 12:49:12 +00:00
|
|
|
|
, connections(make_ref<Pool<Connection>>(
|
2017-04-13 13:55:38 +00:00
|
|
|
|
std::max(1, (int) maxConnections),
|
2020-08-19 19:34:47 +00:00
|
|
|
|
[this]() {
|
|
|
|
|
auto conn = openConnectionWrapper();
|
|
|
|
|
try {
|
|
|
|
|
initConnection(*conn);
|
|
|
|
|
} catch (...) {
|
|
|
|
|
failed = true;
|
|
|
|
|
throw;
|
|
|
|
|
}
|
|
|
|
|
return conn;
|
|
|
|
|
},
|
2017-09-14 16:10:38 +00:00
|
|
|
|
[this](const ref<Connection> & r) {
|
|
|
|
|
return
|
|
|
|
|
r->to.good()
|
|
|
|
|
&& r->from.good()
|
|
|
|
|
&& std::chrono::duration_cast<std::chrono::seconds>(
|
|
|
|
|
std::chrono::steady_clock::now() - r->startTime).count() < maxConnectionAge;
|
|
|
|
|
}
|
2016-02-24 10:39:56 +00:00
|
|
|
|
))
|
2006-12-04 13:28:14 +00:00
|
|
|
|
{
|
2008-12-11 14:30:25 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
2017-03-03 18:35:34 +00:00
|
|
|
|
ref<RemoteStore::Connection> RemoteStore::openConnectionWrapper()
|
|
|
|
|
{
|
|
|
|
|
if (failed)
|
2017-07-30 11:27:57 +00:00
|
|
|
|
throw Error("opening a connection to remote store '%s' previously failed", getUri());
|
2017-03-03 18:35:34 +00:00
|
|
|
|
try {
|
|
|
|
|
return openConnection();
|
|
|
|
|
} catch (...) {
|
|
|
|
|
failed = true;
|
|
|
|
|
throw;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
2016-09-02 18:15:04 +00:00
|
|
|
|
void RemoteStore::initConnection(Connection & conn)
|
|
|
|
|
{
|
2016-02-23 14:00:59 +00:00
|
|
|
|
/* Send the magic greeting, check for the reply. */
|
|
|
|
|
try {
|
2016-09-02 18:15:04 +00:00
|
|
|
|
conn.to << WORKER_MAGIC_1;
|
|
|
|
|
conn.to.flush();
|
2021-09-23 15:48:52 +00:00
|
|
|
|
StringSink saved;
|
|
|
|
|
try {
|
|
|
|
|
TeeSource tee(conn.from, saved);
|
|
|
|
|
unsigned int magic = readInt(tee);
|
|
|
|
|
if (magic != WORKER_MAGIC_2)
|
|
|
|
|
throw Error("protocol mismatch");
|
|
|
|
|
} catch (SerialisationError & e) {
|
2021-09-23 16:01:04 +00:00
|
|
|
|
/* In case the other side is waiting for our input, close
|
|
|
|
|
it. */
|
|
|
|
|
conn.closeWrite();
|
2021-09-23 15:48:52 +00:00
|
|
|
|
auto msg = conn.from.drain();
|
|
|
|
|
throw Error("protocol mismatch, got '%s'", chomp(*saved.s + msg));
|
|
|
|
|
}
|
2016-02-23 14:00:59 +00:00
|
|
|
|
|
2017-03-01 12:52:54 +00:00
|
|
|
|
conn.from >> conn.daemonVersion;
|
2016-09-02 18:15:04 +00:00
|
|
|
|
if (GET_PROTOCOL_MAJOR(conn.daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION))
|
2016-02-23 14:00:59 +00:00
|
|
|
|
throw Error("Nix daemon protocol version not supported");
|
2016-09-02 18:15:04 +00:00
|
|
|
|
if (GET_PROTOCOL_MINOR(conn.daemonVersion) < 10)
|
2016-07-27 13:03:20 +00:00
|
|
|
|
throw Error("the Nix daemon version is too old");
|
2016-09-02 18:15:04 +00:00
|
|
|
|
conn.to << PROTOCOL_VERSION;
|
2016-02-23 14:00:59 +00:00
|
|
|
|
|
2016-09-02 18:15:04 +00:00
|
|
|
|
if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 14) {
|
2019-10-11 16:48:59 +00:00
|
|
|
|
int cpu = sameMachine() && settings.lockCPU ? lockToCurrentCPU() : -1;
|
2016-02-23 14:00:59 +00:00
|
|
|
|
if (cpu != -1)
|
2016-09-02 18:15:04 +00:00
|
|
|
|
conn.to << 1 << cpu;
|
2016-02-23 14:00:59 +00:00
|
|
|
|
else
|
2016-09-02 18:15:04 +00:00
|
|
|
|
conn.to << 0;
|
2016-02-23 14:00:59 +00:00
|
|
|
|
}
|
|
|
|
|
|
2016-09-02 18:15:04 +00:00
|
|
|
|
if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 11)
|
|
|
|
|
conn.to << false;
|
2016-02-23 14:00:59 +00:00
|
|
|
|
|
2018-10-16 21:36:15 +00:00
|
|
|
|
auto ex = conn.processStderr();
|
|
|
|
|
if (ex) std::rethrow_exception(ex);
|
2016-02-23 14:00:59 +00:00
|
|
|
|
}
|
|
|
|
|
catch (Error & e) {
|
2017-07-30 11:27:57 +00:00
|
|
|
|
throw Error("cannot open connection to remote store '%s': %s", getUri(), e.what());
|
2016-02-23 14:00:59 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
setOptions(conn);
|
2006-11-30 18:35:50 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
2016-09-02 18:15:04 +00:00
|
|
|
|
void RemoteStore::setOptions(Connection & conn)
|
2007-09-18 09:11:20 +00:00
|
|
|
|
{
|
2016-09-02 18:15:04 +00:00
|
|
|
|
conn.to << wopSetOptions
|
2015-07-19 23:16:16 +00:00
|
|
|
|
<< settings.keepFailed
|
|
|
|
|
<< settings.keepGoing
|
|
|
|
|
<< settings.tryFallback
|
2020-06-18 22:09:22 +00:00
|
|
|
|
<< verbosity
|
2015-07-19 23:16:16 +00:00
|
|
|
|
<< settings.maxBuildJobs
|
2016-07-27 13:03:20 +00:00
|
|
|
|
<< settings.maxSilentTime
|
2017-10-24 09:00:16 +00:00
|
|
|
|
<< true
|
2020-06-18 22:09:22 +00:00
|
|
|
|
<< (settings.verboseBuild ? lvlError : lvlVomit)
|
2016-07-27 13:03:20 +00:00
|
|
|
|
<< 0 // obsolete log type
|
|
|
|
|
<< 0 /* obsolete print build trace */
|
|
|
|
|
<< settings.buildCores
|
|
|
|
|
<< settings.useSubstitutes;
|
2012-07-30 21:13:25 +00:00
|
|
|
|
|
2016-09-02 18:15:04 +00:00
|
|
|
|
if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 12) {
|
2018-03-27 16:41:31 +00:00
|
|
|
|
std::map<std::string, Config::SettingInfo> overrides;
|
2020-11-02 12:57:58 +00:00
|
|
|
|
settings.getSettings(overrides, true); // libstore settings
|
|
|
|
|
fileTransferSettings.getSettings(overrides, true);
|
2019-08-28 14:29:44 +00:00
|
|
|
|
overrides.erase(settings.keepFailed.name);
|
|
|
|
|
overrides.erase(settings.keepGoing.name);
|
|
|
|
|
overrides.erase(settings.tryFallback.name);
|
|
|
|
|
overrides.erase(settings.maxBuildJobs.name);
|
|
|
|
|
overrides.erase(settings.maxSilentTime.name);
|
|
|
|
|
overrides.erase(settings.buildCores.name);
|
|
|
|
|
overrides.erase(settings.useSubstitutes.name);
|
2020-07-02 15:04:31 +00:00
|
|
|
|
overrides.erase(loggerSettings.showTrace.name);
|
2021-07-30 08:34:50 +00:00
|
|
|
|
overrides.erase(settings.experimentalFeatures.name);
|
2016-09-02 18:15:04 +00:00
|
|
|
|
conn.to << overrides.size();
|
2015-07-19 23:16:16 +00:00
|
|
|
|
for (auto & i : overrides)
|
2018-03-27 16:41:31 +00:00
|
|
|
|
conn.to << i.first << i.second.value;
|
2012-07-31 22:19:44 +00:00
|
|
|
|
}
|
|
|
|
|
|
2018-10-16 21:36:15 +00:00
|
|
|
|
auto ex = conn.processStderr();
|
|
|
|
|
if (ex) std::rethrow_exception(ex);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/* A wrapper around Pool<RemoteStore::Connection>::Handle that marks
|
|
|
|
|
the connection as bad (causing it to be closed) if a non-daemon
|
|
|
|
|
exception is thrown before the handle is closed. Such an exception
|
|
|
|
|
causes a deviation from the expected protocol and therefore a
|
|
|
|
|
desynchronization between the client and daemon. */
|
|
|
|
|
struct ConnectionHandle
|
|
|
|
|
{
|
|
|
|
|
Pool<RemoteStore::Connection>::Handle handle;
|
|
|
|
|
bool daemonException = false;
|
|
|
|
|
|
|
|
|
|
ConnectionHandle(Pool<RemoteStore::Connection>::Handle && handle)
|
|
|
|
|
: handle(std::move(handle))
|
|
|
|
|
{ }
|
|
|
|
|
|
|
|
|
|
ConnectionHandle(ConnectionHandle && h)
|
|
|
|
|
: handle(std::move(h.handle))
|
|
|
|
|
{ }
|
|
|
|
|
|
|
|
|
|
~ConnectionHandle()
|
|
|
|
|
{
|
2020-06-17 02:15:47 +00:00
|
|
|
|
if (!daemonException && std::uncaught_exceptions()) {
|
2018-10-16 21:36:15 +00:00
|
|
|
|
handle.markBad();
|
|
|
|
|
debug("closing daemon connection because of an exception");
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
RemoteStore::Connection * operator -> () { return &*handle; }
|
|
|
|
|
|
2020-08-27 12:48:08 +00:00
|
|
|
|
void processStderr(Sink * sink = 0, Source * source = 0, bool flush = true)
|
2018-10-16 21:36:15 +00:00
|
|
|
|
{
|
2020-08-27 12:48:08 +00:00
|
|
|
|
auto ex = handle->processStderr(sink, source, flush);
|
2018-10-16 21:36:15 +00:00
|
|
|
|
if (ex) {
|
|
|
|
|
daemonException = true;
|
|
|
|
|
std::rethrow_exception(ex);
|
|
|
|
|
}
|
|
|
|
|
}
|
2020-09-17 15:36:16 +00:00
|
|
|
|
|
|
|
|
|
void withFramedSink(std::function<void(Sink & sink)> fun);
|
2018-10-16 21:36:15 +00:00
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
ConnectionHandle RemoteStore::getConnection()
|
|
|
|
|
{
|
|
|
|
|
return ConnectionHandle(connections->get());
|
2007-09-18 09:11:20 +00:00
|
|
|
|
}
|
|
|
|
|
|
2021-11-05 15:17:49 +00:00
|
|
|
|
void RemoteStore::setOptions()
|
|
|
|
|
{
|
|
|
|
|
setOptions(*(getConnection().handle));
|
|
|
|
|
}
|
2007-09-18 09:11:20 +00:00
|
|
|
|
|
2019-12-05 18:11:09 +00:00
|
|
|
|
bool RemoteStore::isValidPathUncached(const StorePath & path)
|
2006-11-30 18:35:50 +00:00
|
|
|
|
{
|
2018-10-16 21:36:15 +00:00
|
|
|
|
auto conn(getConnection());
|
2019-12-05 18:11:09 +00:00
|
|
|
|
conn->to << wopIsValidPath << printStorePath(path);
|
2018-10-16 21:36:15 +00:00
|
|
|
|
conn.processStderr();
|
2017-03-01 12:52:54 +00:00
|
|
|
|
return readInt(conn->from);
|
2006-11-30 18:35:50 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
2019-12-05 18:11:09 +00:00
|
|
|
|
StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute)
|
2012-07-11 15:08:47 +00:00
|
|
|
|
{
|
2018-10-16 21:36:15 +00:00
|
|
|
|
auto conn(getConnection());
|
2016-02-23 14:00:59 +00:00
|
|
|
|
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) {
|
2019-12-05 18:11:09 +00:00
|
|
|
|
StorePathSet res;
|
2015-07-17 17:24:28 +00:00
|
|
|
|
for (auto & i : paths)
|
2020-06-16 20:20:18 +00:00
|
|
|
|
if (isValidPath(i)) res.insert(i);
|
2012-07-11 15:08:47 +00:00
|
|
|
|
return res;
|
|
|
|
|
} else {
|
2019-12-05 18:11:09 +00:00
|
|
|
|
conn->to << wopQueryValidPaths;
|
2020-09-30 00:41:18 +00:00
|
|
|
|
worker_proto::write(*this, conn->to, paths);
|
2020-10-21 19:31:19 +00:00
|
|
|
|
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 27) {
|
|
|
|
|
conn->to << (settings.buildersUseSubstitutes ? 1 : 0);
|
|
|
|
|
}
|
2018-10-16 21:36:15 +00:00
|
|
|
|
conn.processStderr();
|
2020-09-30 00:39:06 +00:00
|
|
|
|
return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
|
2012-07-11 15:08:47 +00:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
2019-12-05 18:11:09 +00:00
|
|
|
|
StorePathSet RemoteStore::queryAllValidPaths()
|
2008-01-29 18:17:36 +00:00
|
|
|
|
{
|
2018-10-16 21:36:15 +00:00
|
|
|
|
auto conn(getConnection());
|
2016-02-23 14:00:59 +00:00
|
|
|
|
conn->to << wopQueryAllValidPaths;
|
2018-10-16 21:36:15 +00:00
|
|
|
|
conn.processStderr();
|
2020-09-30 00:41:18 +00:00
|
|
|
|
return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
|
2008-01-29 18:17:36 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
2019-12-05 18:11:09 +00:00
|
|
|
|
StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths)
|
2006-11-30 22:43:55 +00:00
|
|
|
|
{
|
2018-10-16 21:36:15 +00:00
|
|
|
|
auto conn(getConnection());
|
2016-02-23 14:00:59 +00:00
|
|
|
|
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) {
|
2019-12-05 18:11:09 +00:00
|
|
|
|
StorePathSet res;
|
2015-07-17 17:24:28 +00:00
|
|
|
|
for (auto & i : paths) {
|
2019-12-05 18:11:09 +00:00
|
|
|
|
conn->to << wopHasSubstitutes << printStorePath(i);
|
2018-10-16 21:36:15 +00:00
|
|
|
|
conn.processStderr();
|
2020-06-16 20:20:18 +00:00
|
|
|
|
if (readInt(conn->from)) res.insert(i);
|
2012-07-11 21:52:18 +00:00
|
|
|
|
}
|
|
|
|
|
return res;
|
|
|
|
|
} else {
|
2019-12-05 18:11:09 +00:00
|
|
|
|
conn->to << wopQuerySubstitutablePaths;
|
2020-09-30 00:41:18 +00:00
|
|
|
|
worker_proto::write(*this, conn->to, paths);
|
2018-10-16 21:36:15 +00:00
|
|
|
|
conn.processStderr();
|
2020-09-30 00:39:06 +00:00
|
|
|
|
return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
|
2012-07-11 21:52:18 +00:00
|
|
|
|
}
|
2006-11-30 18:35:50 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
2020-06-17 19:03:05 +00:00
|
|
|
|
void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, SubstitutablePathInfos & infos)
|
download-from-binary-cache: parallelise fetching of NAR info files
Getting substitute information using the binary cache substituter has
non-trivial latency overhead. A package or NixOS system configuration
can have hundreds of dependencies, and in the worst case (when the
local info cache is empty) we have to do a separate HTTP request for
each of these. If the ping time to the server is t, getting N info
files will take tN seconds; e.g., with a ping time of 0.1s to
nixos.org, sequentially downloading 1000 info files (a typical NixOS
config) will take at least 100 seconds.
To fix this problem, the binary cache substituter can now perform
requests in parallel. This required changing the substituter
interface to support a function querySubstitutablePathInfos() that
queries multiple paths at the same time, and rewriting queryMissing()
to take advantage of parallelism. (Due to local caching,
parallelising queryMissing() is sufficient for most use cases, since
it's almost always called before building a derivation and thus fills
the local info cache.)
For example, parallelism speeds up querying all 1056 paths in a
particular NixOS system configuration from 116s to 2.6s. It works so
well because the eccentricity of the top-level derivation in the
dependency graph is only 9. So we only need 10 round-trips (when
using an unlimited number of parallel connections) to get everything.
Currently we do a maximum of 150 parallel connections to the server.
Thus it's important that the binary cache server (e.g. nixos.org) has
a high connection limit. Alternatively we could use HTTP pipelining,
but WWW::Curl doesn't support it and libcurl has a hard-coded limit of
5 requests per pipeline.
2012-07-06 23:08:20 +00:00
|
|
|
|
{
|
2020-06-17 19:03:05 +00:00
|
|
|
|
if (pathsMap.empty()) return;
|
2012-07-11 14:43:24 +00:00
|
|
|
|
|
2018-10-16 21:36:15 +00:00
|
|
|
|
auto conn(getConnection());
|
2012-07-30 21:13:25 +00:00
|
|
|
|
|
2016-02-23 14:00:59 +00:00
|
|
|
|
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) {
|
2012-07-30 21:13:25 +00:00
|
|
|
|
|
2020-06-17 19:03:05 +00:00
|
|
|
|
for (auto & i : pathsMap) {
|
2012-07-11 14:43:24 +00:00
|
|
|
|
SubstitutablePathInfo info;
|
2020-06-17 19:03:05 +00:00
|
|
|
|
conn->to << wopQuerySubstitutablePathInfo << printStorePath(i.first);
|
2018-10-16 21:36:15 +00:00
|
|
|
|
conn.processStderr();
|
2016-02-23 14:00:59 +00:00
|
|
|
|
unsigned int reply = readInt(conn->from);
|
2012-07-11 14:43:24 +00:00
|
|
|
|
if (reply == 0) continue;
|
2019-12-05 18:11:09 +00:00
|
|
|
|
auto deriver = readString(conn->from);
|
|
|
|
|
if (deriver != "")
|
|
|
|
|
info.deriver = parseStorePath(deriver);
|
2020-09-30 00:39:06 +00:00
|
|
|
|
info.references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
|
2016-02-23 14:00:59 +00:00
|
|
|
|
info.downloadSize = readLongLong(conn->from);
|
2016-07-27 13:03:20 +00:00
|
|
|
|
info.narSize = readLongLong(conn->from);
|
2020-06-17 19:03:05 +00:00
|
|
|
|
infos.insert_or_assign(i.first, std::move(info));
|
2012-07-11 14:43:24 +00:00
|
|
|
|
}
|
2012-07-30 21:13:25 +00:00
|
|
|
|
|
2012-07-11 14:43:24 +00:00
|
|
|
|
} else {
|
2012-07-30 21:13:25 +00:00
|
|
|
|
|
2019-12-05 18:11:09 +00:00
|
|
|
|
conn->to << wopQuerySubstitutablePathInfos;
|
2020-06-19 22:06:19 +00:00
|
|
|
|
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 22) {
|
|
|
|
|
StorePathSet paths;
|
|
|
|
|
for (auto & path : pathsMap)
|
|
|
|
|
paths.insert(path.first);
|
2020-09-30 00:39:06 +00:00
|
|
|
|
worker_proto::write(*this, conn->to, paths);
|
2020-06-19 22:06:19 +00:00
|
|
|
|
} else
|
2020-09-30 00:39:06 +00:00
|
|
|
|
worker_proto::write(*this, conn->to, pathsMap);
|
2018-10-16 21:36:15 +00:00
|
|
|
|
conn.processStderr();
|
2017-03-01 12:52:54 +00:00
|
|
|
|
size_t count = readNum<size_t>(conn->from);
|
|
|
|
|
for (size_t n = 0; n < count; n++) {
|
2019-12-05 18:11:09 +00:00
|
|
|
|
SubstitutablePathInfo & info(infos[parseStorePath(readString(conn->from))]);
|
|
|
|
|
auto deriver = readString(conn->from);
|
|
|
|
|
if (deriver != "")
|
|
|
|
|
info.deriver = parseStorePath(deriver);
|
2020-09-30 00:39:06 +00:00
|
|
|
|
info.references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
|
2016-02-23 14:00:59 +00:00
|
|
|
|
info.downloadSize = readLongLong(conn->from);
|
|
|
|
|
info.narSize = readLongLong(conn->from);
|
2012-07-11 14:43:24 +00:00
|
|
|
|
}
|
2012-07-30 21:13:25 +00:00
|
|
|
|
|
download-from-binary-cache: parallelise fetching of NAR info files
Getting substitute information using the binary cache substituter has
non-trivial latency overhead. A package or NixOS system configuration
can have hundreds of dependencies, and in the worst case (when the
local info cache is empty) we have to do a separate HTTP request for
each of these. If the ping time to the server is t, getting N info
files will take tN seconds; e.g., with a ping time of 0.1s to
nixos.org, sequentially downloading 1000 info files (a typical NixOS
config) will take at least 100 seconds.
To fix this problem, the binary cache substituter can now perform
requests in parallel. This required changing the substituter
interface to support a function querySubstitutablePathInfos() that
queries multiple paths at the same time, and rewriting queryMissing()
to take advantage of parallelism. (Due to local caching,
parallelising queryMissing() is sufficient for most use cases, since
it's almost always called before building a derivation and thus fills
the local info cache.)
For example, parallelism speeds up querying all 1056 paths in a
particular NixOS system configuration from 116s to 2.6s. It works so
well because the eccentricity of the top-level derivation in the
dependency graph is only 9. So we only need 10 round-trips (when
using an unlimited number of parallel connections) to get everything.
Currently we do a maximum of 150 parallel connections to the server.
Thus it's important that the binary cache server (e.g. nixos.org) has
a high connection limit. Alternatively we could use HTTP pipelining,
but WWW::Curl doesn't support it and libcurl has a hard-coded limit of
5 requests per pipeline.
2012-07-06 23:08:20 +00:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
2019-12-05 18:11:09 +00:00
|
|
|
|
void RemoteStore::queryPathInfoUncached(const StorePath & path,
|
2018-09-25 16:54:16 +00:00
|
|
|
|
Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept
|
2010-11-16 17:11:46 +00:00
|
|
|
|
{
|
2018-03-27 20:16:01 +00:00
|
|
|
|
try {
|
2020-09-18 09:22:13 +00:00
|
|
|
|
std::shared_ptr<const ValidPathInfo> info;
|
2018-04-09 19:26:16 +00:00
|
|
|
|
{
|
2018-10-16 21:36:15 +00:00
|
|
|
|
auto conn(getConnection());
|
2019-12-05 18:11:09 +00:00
|
|
|
|
conn->to << wopQueryPathInfo << printStorePath(path);
|
2018-04-09 19:26:16 +00:00
|
|
|
|
try {
|
2018-10-16 21:36:15 +00:00
|
|
|
|
conn.processStderr();
|
2018-04-09 19:26:16 +00:00
|
|
|
|
} catch (Error & e) {
|
|
|
|
|
// Ugly backwards compatibility hack.
|
|
|
|
|
if (e.msg().find("is not valid") != std::string::npos)
|
2020-04-30 00:57:05 +00:00
|
|
|
|
throw InvalidPath(e.info());
|
2018-04-09 19:26:16 +00:00
|
|
|
|
throw;
|
|
|
|
|
}
|
|
|
|
|
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) {
|
|
|
|
|
bool valid; conn->from >> valid;
|
2019-12-05 18:11:09 +00:00
|
|
|
|
if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path));
|
2018-04-09 19:26:16 +00:00
|
|
|
|
}
|
2021-07-26 11:31:09 +00:00
|
|
|
|
info = std::make_shared<ValidPathInfo>(
|
|
|
|
|
ValidPathInfo::read(conn->from, *this, GET_PROTOCOL_MINOR(conn->daemonVersion), StorePath{path}));
|
2016-09-16 16:54:14 +00:00
|
|
|
|
}
|
2018-03-27 20:16:01 +00:00
|
|
|
|
callback(std::move(info));
|
|
|
|
|
} catch (...) { callback.rethrow(); }
|
2010-11-16 17:11:46 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
2019-12-05 18:11:09 +00:00
|
|
|
|
void RemoteStore::queryReferrers(const StorePath & path,
|
|
|
|
|
StorePathSet & referrers)
|
2006-11-30 18:35:50 +00:00
|
|
|
|
{
|
2018-10-16 21:36:15 +00:00
|
|
|
|
auto conn(getConnection());
|
2019-12-05 18:11:09 +00:00
|
|
|
|
conn->to << wopQueryReferrers << printStorePath(path);
|
2018-10-16 21:36:15 +00:00
|
|
|
|
conn.processStderr();
|
2020-09-30 00:39:06 +00:00
|
|
|
|
for (auto & i : worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}))
|
2020-06-16 20:20:18 +00:00
|
|
|
|
referrers.insert(i);
|
2006-11-30 18:35:50 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
2019-12-05 18:11:09 +00:00
|
|
|
|
StorePathSet RemoteStore::queryValidDerivers(const StorePath & path)
|
2012-12-20 17:41:44 +00:00
|
|
|
|
{
|
2018-10-16 21:36:15 +00:00
|
|
|
|
auto conn(getConnection());
|
2019-12-05 18:11:09 +00:00
|
|
|
|
conn->to << wopQueryValidDerivers << printStorePath(path);
|
2018-10-16 21:36:15 +00:00
|
|
|
|
conn.processStderr();
|
2020-09-30 00:39:06 +00:00
|
|
|
|
return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
|
2012-12-20 17:41:44 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
2019-12-05 18:11:09 +00:00
|
|
|
|
StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path)
|
2010-02-22 12:44:36 +00:00
|
|
|
|
{
|
2020-11-16 07:35:50 +00:00
|
|
|
|
if (GET_PROTOCOL_MINOR(getProtocol()) >= 0x16) {
|
2020-06-10 09:20:52 +00:00
|
|
|
|
return Store::queryDerivationOutputs(path);
|
|
|
|
|
}
|
2020-11-16 07:35:50 +00:00
|
|
|
|
auto conn(getConnection());
|
2019-12-05 18:11:09 +00:00
|
|
|
|
conn->to << wopQueryDerivationOutputs << printStorePath(path);
|
2018-10-16 21:36:15 +00:00
|
|
|
|
conn.processStderr();
|
2020-09-30 00:39:06 +00:00
|
|
|
|
return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
|
2010-02-22 12:44:36 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
2020-08-20 14:12:51 +00:00
|
|
|
|
std::map<std::string, std::optional<StorePath>> RemoteStore::queryPartialDerivationOutputMap(const StorePath & path)
|
2020-06-10 09:20:52 +00:00
|
|
|
|
{
|
2020-09-07 09:26:09 +00:00
|
|
|
|
if (GET_PROTOCOL_MINOR(getProtocol()) >= 0x16) {
|
|
|
|
|
auto conn(getConnection());
|
|
|
|
|
conn->to << wopQueryDerivationOutputMap << printStorePath(path);
|
|
|
|
|
conn.processStderr();
|
2020-09-30 00:39:06 +00:00
|
|
|
|
return worker_proto::read(*this, conn->from, Phantom<std::map<std::string, std::optional<StorePath>>> {});
|
2020-09-07 09:26:09 +00:00
|
|
|
|
} else {
|
|
|
|
|
// Fallback for old daemon versions.
|
|
|
|
|
// For floating-CA derivations (and their co-dependencies) this is an
|
|
|
|
|
// under-approximation as it only returns the paths that can be inferred
|
|
|
|
|
// from the derivation itself (and not the ones that are known because
|
|
|
|
|
// the have been built), but as old stores don't handle floating-CA
|
|
|
|
|
// derivations this shouldn't matter
|
|
|
|
|
auto derivation = readDerivation(path);
|
|
|
|
|
auto outputsWithOptPaths = derivation.outputsAndOptPaths(*this);
|
|
|
|
|
std::map<std::string, std::optional<StorePath>> ret;
|
|
|
|
|
for (auto & [outputName, outputAndPath] : outputsWithOptPaths) {
|
|
|
|
|
ret.emplace(outputName, outputAndPath.second);
|
|
|
|
|
}
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
2020-06-10 09:20:52 +00:00
|
|
|
|
}
|
|
|
|
|
|
2019-12-05 18:11:09 +00:00
|
|
|
|
std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string & hashPart)
|
2012-07-17 22:55:39 +00:00
|
|
|
|
{
|
2018-10-16 21:36:15 +00:00
|
|
|
|
auto conn(getConnection());
|
2016-02-23 14:00:59 +00:00
|
|
|
|
conn->to << wopQueryPathFromHashPart << hashPart;
|
2018-10-16 21:36:15 +00:00
|
|
|
|
conn.processStderr();
|
2016-02-23 14:00:59 +00:00
|
|
|
|
Path path = readString(conn->from);
|
2019-12-05 18:11:09 +00:00
|
|
|
|
if (path.empty()) return {};
|
|
|
|
|
return parseStorePath(path);
|
2012-07-17 22:55:39 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
2020-09-22 09:40:19 +00:00
|
|
|
|
ref<const ValidPathInfo> RemoteStore::addCAToStore(
|
|
|
|
|
Source & dump,
|
|
|
|
|
const string & name,
|
|
|
|
|
ContentAddressMethod caMethod,
|
|
|
|
|
const StorePathSet & references,
|
|
|
|
|
RepairFlag repair)
|
2020-09-17 17:27:11 +00:00
|
|
|
|
{
|
2020-09-22 13:28:20 +00:00
|
|
|
|
std::optional<ConnectionHandle> conn_(getConnection());
|
|
|
|
|
auto & conn = *conn_;
|
2020-09-17 17:27:11 +00:00
|
|
|
|
|
|
|
|
|
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 25) {
|
|
|
|
|
|
|
|
|
|
conn->to
|
|
|
|
|
<< wopAddToStore
|
|
|
|
|
<< name
|
2020-09-17 18:19:15 +00:00
|
|
|
|
<< renderContentAddressMethod(caMethod);
|
2020-09-30 00:39:06 +00:00
|
|
|
|
worker_proto::write(*this, conn->to, references);
|
2020-09-18 08:06:34 +00:00
|
|
|
|
conn->to << repair;
|
2020-09-17 17:27:11 +00:00
|
|
|
|
|
2020-10-30 20:47:34 +00:00
|
|
|
|
// The dump source may invoke the store, so we need to make some room.
|
|
|
|
|
connections->incCapacity();
|
|
|
|
|
{
|
|
|
|
|
Finally cleanup([&]() { connections->decCapacity(); });
|
|
|
|
|
conn.withFramedSink([&](Sink & sink) {
|
|
|
|
|
dump.drainInto(sink);
|
|
|
|
|
});
|
|
|
|
|
}
|
2020-09-17 17:27:11 +00:00
|
|
|
|
|
2021-07-26 11:31:09 +00:00
|
|
|
|
return make_ref<ValidPathInfo>(
|
|
|
|
|
ValidPathInfo::read(conn->from, *this, GET_PROTOCOL_MINOR(conn->daemonVersion)));
|
2020-09-17 17:27:11 +00:00
|
|
|
|
}
|
|
|
|
|
else {
|
|
|
|
|
if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25");
|
|
|
|
|
|
2020-09-17 18:19:15 +00:00
|
|
|
|
std::visit(overloaded {
|
2021-09-30 21:31:21 +00:00
|
|
|
|
[&](const TextHashMethod & thm) -> void {
|
2020-09-17 18:19:15 +00:00
|
|
|
|
std::string s = dump.drain();
|
|
|
|
|
conn->to << wopAddTextToStore << name << s;
|
2020-09-30 00:39:06 +00:00
|
|
|
|
worker_proto::write(*this, conn->to, references);
|
2020-09-17 18:19:15 +00:00
|
|
|
|
conn.processStderr();
|
|
|
|
|
},
|
2021-09-30 21:31:21 +00:00
|
|
|
|
[&](const FixedOutputHashMethod & fohm) -> void {
|
2020-09-17 18:19:15 +00:00
|
|
|
|
conn->to
|
|
|
|
|
<< wopAddToStore
|
|
|
|
|
<< name
|
|
|
|
|
<< ((fohm.hashType == htSHA256 && fohm.fileIngestionMethod == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */
|
|
|
|
|
<< (fohm.fileIngestionMethod == FileIngestionMethod::Recursive ? 1 : 0)
|
|
|
|
|
<< printHashType(fohm.hashType);
|
|
|
|
|
|
2020-09-17 17:27:11 +00:00
|
|
|
|
try {
|
2020-09-17 18:19:15 +00:00
|
|
|
|
conn->to.written = 0;
|
|
|
|
|
conn->to.warn = true;
|
|
|
|
|
connections->incCapacity();
|
|
|
|
|
{
|
|
|
|
|
Finally cleanup([&]() { connections->decCapacity(); });
|
|
|
|
|
if (fohm.fileIngestionMethod == FileIngestionMethod::Recursive) {
|
|
|
|
|
dump.drainInto(conn->to);
|
|
|
|
|
} else {
|
|
|
|
|
std::string contents = dump.drain();
|
|
|
|
|
dumpString(contents, conn->to);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
conn->to.warn = false;
|
2020-09-17 17:27:11 +00:00
|
|
|
|
conn.processStderr();
|
2020-09-17 18:19:15 +00:00
|
|
|
|
} catch (SysError & e) {
|
|
|
|
|
/* Daemon closed while we were sending the path. Probably OOM
|
|
|
|
|
or I/O error. */
|
|
|
|
|
if (e.errNo == EPIPE)
|
|
|
|
|
try {
|
|
|
|
|
conn.processStderr();
|
|
|
|
|
} catch (EndOfFile & e) { }
|
|
|
|
|
throw;
|
|
|
|
|
}
|
2020-09-17 17:27:11 +00:00
|
|
|
|
|
2020-09-17 18:19:15 +00:00
|
|
|
|
}
|
|
|
|
|
}, caMethod);
|
2020-09-18 09:22:13 +00:00
|
|
|
|
auto path = parseStorePath(readString(conn->from));
|
2020-09-22 13:28:20 +00:00
|
|
|
|
// Release our connection to prevent a deadlock in queryPathInfo().
|
|
|
|
|
conn_.reset();
|
2020-09-18 09:22:13 +00:00
|
|
|
|
return queryPathInfo(path);
|
2020-09-17 17:27:11 +00:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2020-09-22 09:40:19 +00:00
|
|
|
|
|
2020-09-17 18:19:15 +00:00
|
|
|
|
StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name,
|
2021-11-09 09:24:49 +00:00
|
|
|
|
FileIngestionMethod method, HashType hashType, RepairFlag repair, const StorePathSet & references)
|
2020-09-17 18:19:15 +00:00
|
|
|
|
{
|
2020-09-18 09:22:13 +00:00
|
|
|
|
return addCAToStore(dump, name, FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }, references, repair)->path;
|
2020-09-17 18:19:15 +00:00
|
|
|
|
}
|
|
|
|
|
|
2020-09-17 17:27:11 +00:00
|
|
|
|
|
2018-03-21 22:42:21 +00:00
|
|
|
|
void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
|
2020-07-13 15:37:44 +00:00
|
|
|
|
RepairFlag repair, CheckSigsFlag checkSigs)
|
2016-05-04 11:36:54 +00:00
|
|
|
|
{
|
2018-10-16 21:36:15 +00:00
|
|
|
|
auto conn(getConnection());
|
2016-11-09 17:45:06 +00:00
|
|
|
|
|
2016-11-09 17:57:22 +00:00
|
|
|
|
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 18) {
|
|
|
|
|
conn->to << wopImportPaths;
|
|
|
|
|
|
2018-03-21 22:46:03 +00:00
|
|
|
|
auto source2 = sinkToSource([&](Sink & sink) {
|
|
|
|
|
sink << 1 // == path follows
|
|
|
|
|
;
|
|
|
|
|
copyNAR(source, sink);
|
|
|
|
|
sink
|
|
|
|
|
<< exportMagic
|
2019-12-05 18:11:09 +00:00
|
|
|
|
<< printStorePath(info.path);
|
2020-09-30 00:39:06 +00:00
|
|
|
|
worker_proto::write(*this, sink, info.references);
|
2019-12-05 18:11:09 +00:00
|
|
|
|
sink
|
|
|
|
|
<< (info.deriver ? printStorePath(*info.deriver) : "")
|
2018-03-21 22:46:03 +00:00
|
|
|
|
<< 0 // == no legacy signature
|
|
|
|
|
<< 0 // == no path follows
|
|
|
|
|
;
|
|
|
|
|
});
|
|
|
|
|
|
2018-10-16 21:36:15 +00:00
|
|
|
|
conn.processStderr(0, source2.get());
|
2016-11-09 17:57:22 +00:00
|
|
|
|
|
2020-09-30 00:39:06 +00:00
|
|
|
|
auto importedPaths = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
|
2016-11-09 17:57:22 +00:00
|
|
|
|
assert(importedPaths.size() <= 1);
|
|
|
|
|
}
|
2016-11-09 17:45:06 +00:00
|
|
|
|
|
2016-11-09 17:57:22 +00:00
|
|
|
|
else {
|
|
|
|
|
conn->to << wopAddToStoreNar
|
2019-12-05 18:11:09 +00:00
|
|
|
|
<< printStorePath(info.path)
|
|
|
|
|
<< (info.deriver ? printStorePath(*info.deriver) : "")
|
2020-08-05 18:42:48 +00:00
|
|
|
|
<< info.narHash.to_string(Base16, false);
|
2020-09-30 00:39:06 +00:00
|
|
|
|
worker_proto::write(*this, conn->to, info.references);
|
2019-12-05 18:11:09 +00:00
|
|
|
|
conn->to << info.registrationTime << info.narSize
|
2020-06-02 00:37:43 +00:00
|
|
|
|
<< info.ultimate << info.sigs << renderContentAddress(info.ca)
|
2017-06-28 16:11:01 +00:00
|
|
|
|
<< repair << !checkSigs;
|
2020-07-28 22:48:39 +00:00
|
|
|
|
|
|
|
|
|
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 23) {
|
2020-09-17 15:36:16 +00:00
|
|
|
|
conn.withFramedSink([&](Sink & sink) {
|
2020-07-28 22:48:39 +00:00
|
|
|
|
copyNAR(source, sink);
|
2020-09-17 15:36:16 +00:00
|
|
|
|
});
|
2020-07-28 22:48:39 +00:00
|
|
|
|
} else if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21) {
|
|
|
|
|
conn.processStderr(0, &source);
|
|
|
|
|
} else {
|
|
|
|
|
copyNAR(source, conn->to);
|
|
|
|
|
conn.processStderr(0, nullptr);
|
|
|
|
|
}
|
2016-11-09 17:57:22 +00:00
|
|
|
|
}
|
2016-05-04 11:36:54 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
2021-07-26 11:31:09 +00:00
|
|
|
|
void RemoteStore::addMultipleToStore(
|
|
|
|
|
Source & source,
|
|
|
|
|
RepairFlag repair,
|
|
|
|
|
CheckSigsFlag checkSigs)
|
|
|
|
|
{
|
|
|
|
|
if (GET_PROTOCOL_MINOR(getConnection()->daemonVersion) >= 32) {
|
|
|
|
|
auto conn(getConnection());
|
|
|
|
|
conn->to
|
|
|
|
|
<< wopAddMultipleToStore
|
|
|
|
|
<< repair
|
|
|
|
|
<< !checkSigs;
|
|
|
|
|
conn.withFramedSink([&](Sink & sink) {
|
|
|
|
|
source.drainInto(sink);
|
|
|
|
|
});
|
|
|
|
|
} else
|
|
|
|
|
Store::addMultipleToStore(source, repair, checkSigs);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
2019-12-05 18:11:09 +00:00
|
|
|
|
StorePath RemoteStore::addTextToStore(const string & name, const string & s,
|
|
|
|
|
const StorePathSet & references, RepairFlag repair)
|
2006-11-30 18:35:50 +00:00
|
|
|
|
{
|
2020-09-17 18:19:15 +00:00
|
|
|
|
StringSource source(s);
|
2020-09-18 09:22:13 +00:00
|
|
|
|
return addCAToStore(source, name, TextHashMethod{}, references, repair)->path;
|
2006-11-30 18:35:50 +00:00
|
|
|
|
}
|
|
|
|
|
|
2020-10-08 15:36:51 +00:00
|
|
|
|
void RemoteStore::registerDrvOutput(const Realisation & info)
|
|
|
|
|
{
|
|
|
|
|
auto conn(getConnection());
|
|
|
|
|
conn->to << wopRegisterDrvOutput;
|
2021-05-07 09:06:17 +00:00
|
|
|
|
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) {
|
|
|
|
|
conn->to << info.id.to_string();
|
|
|
|
|
conn->to << std::string(info.outPath.to_string());
|
|
|
|
|
} else {
|
|
|
|
|
worker_proto::write(*this, conn->to, info);
|
|
|
|
|
}
|
2020-10-08 15:36:51 +00:00
|
|
|
|
conn.processStderr();
|
|
|
|
|
}
|
|
|
|
|
|
2021-10-27 09:36:51 +00:00
|
|
|
|
void RemoteStore::queryRealisationUncached(const DrvOutput & id,
|
|
|
|
|
Callback<std::shared_ptr<const Realisation>> callback) noexcept
|
2020-10-08 15:36:51 +00:00
|
|
|
|
{
|
|
|
|
|
auto conn(getConnection());
|
2021-12-06 10:45:18 +00:00
|
|
|
|
|
|
|
|
|
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 27) {
|
|
|
|
|
warn("The daemon is too old for content-addressed derivations. I’ll do what I can");
|
|
|
|
|
try {
|
|
|
|
|
callback(nullptr);
|
|
|
|
|
} catch (...) { return callback.rethrow(); }
|
|
|
|
|
}
|
|
|
|
|
|
2020-10-08 15:36:51 +00:00
|
|
|
|
conn->to << wopQueryRealisation;
|
|
|
|
|
conn->to << id.to_string();
|
|
|
|
|
conn.processStderr();
|
2021-10-27 09:36:51 +00:00
|
|
|
|
|
|
|
|
|
auto real = [&]() -> std::shared_ptr<const Realisation> {
|
|
|
|
|
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) {
|
|
|
|
|
auto outPaths = worker_proto::read(
|
|
|
|
|
*this, conn->from, Phantom<std::set<StorePath>> {});
|
|
|
|
|
if (outPaths.empty())
|
|
|
|
|
return nullptr;
|
|
|
|
|
return std::make_shared<const Realisation>(Realisation { .id = id, .outPath = *outPaths.begin() });
|
|
|
|
|
} else {
|
|
|
|
|
auto realisations = worker_proto::read(
|
|
|
|
|
*this, conn->from, Phantom<std::set<Realisation>> {});
|
|
|
|
|
if (realisations.empty())
|
|
|
|
|
return nullptr;
|
|
|
|
|
return std::make_shared<const Realisation>(*realisations.begin());
|
|
|
|
|
}
|
|
|
|
|
}();
|
|
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
callback(std::shared_ptr<const Realisation>(real));
|
|
|
|
|
} catch (...) { return callback.rethrow(); }
|
2020-10-08 15:36:51 +00:00
|
|
|
|
}
|
|
|
|
|
|
2021-04-05 13:48:18 +00:00
|
|
|
|
static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, const std::vector<DerivedPath> & reqs)
|
2021-03-02 03:50:41 +00:00
|
|
|
|
{
|
2021-05-02 23:12:23 +00:00
|
|
|
|
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 30) {
|
2021-03-02 03:50:41 +00:00
|
|
|
|
worker_proto::write(store, conn->to, reqs);
|
|
|
|
|
} else {
|
|
|
|
|
Strings ss;
|
|
|
|
|
for (auto & p : reqs) {
|
2021-04-05 13:48:18 +00:00
|
|
|
|
auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(p);
|
2021-03-02 03:50:41 +00:00
|
|
|
|
std::visit(overloaded {
|
2021-09-30 21:31:21 +00:00
|
|
|
|
[&](const StorePathWithOutputs & s) {
|
2021-03-02 03:50:41 +00:00
|
|
|
|
ss.push_back(s.to_string(store));
|
|
|
|
|
},
|
2021-09-30 21:31:21 +00:00
|
|
|
|
[&](const StorePath & drvPath) {
|
2021-03-02 03:50:41 +00:00
|
|
|
|
throw Error("trying to request '%s', but daemon protocol %d.%d is too old (< 1.29) to request a derivation file",
|
|
|
|
|
store.printStorePath(drvPath),
|
|
|
|
|
GET_PROTOCOL_MAJOR(conn->daemonVersion),
|
|
|
|
|
GET_PROTOCOL_MINOR(conn->daemonVersion));
|
|
|
|
|
},
|
|
|
|
|
}, sOrDrvPath);
|
|
|
|
|
}
|
|
|
|
|
conn->to << ss;
|
|
|
|
|
}
|
|
|
|
|
}
|
2006-11-30 18:35:50 +00:00
|
|
|
|
|
2021-07-19 13:43:08 +00:00
|
|
|
|
void RemoteStore::buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMode buildMode, std::shared_ptr<Store> evalStore)
|
2006-11-30 18:35:50 +00:00
|
|
|
|
{
|
2021-07-22 20:50:48 +00:00
|
|
|
|
if (evalStore && evalStore.get() != this) {
|
|
|
|
|
/* The remote doesn't have a way to access evalStore, so copy
|
|
|
|
|
the .drvs. */
|
|
|
|
|
RealisedPath::Set drvPaths2;
|
|
|
|
|
for (auto & i : drvPaths)
|
|
|
|
|
if (auto p = std::get_if<DerivedPath::Built>(&i))
|
|
|
|
|
drvPaths2.insert(p->drvPath);
|
|
|
|
|
copyClosure(*evalStore, *this, drvPaths2);
|
|
|
|
|
}
|
2021-07-19 13:43:08 +00:00
|
|
|
|
|
2018-10-16 21:36:15 +00:00
|
|
|
|
auto conn(getConnection());
|
2016-02-23 14:00:59 +00:00
|
|
|
|
conn->to << wopBuildPaths;
|
2019-12-05 18:11:09 +00:00
|
|
|
|
assert(GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13);
|
2021-04-05 13:48:18 +00:00
|
|
|
|
writeDerivedPaths(*this, conn, drvPaths);
|
2019-12-05 18:11:09 +00:00
|
|
|
|
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15)
|
|
|
|
|
conn->to << buildMode;
|
|
|
|
|
else
|
|
|
|
|
/* Old daemons did not take a 'buildMode' parameter, so we
|
|
|
|
|
need to validate it here on the client side. */
|
|
|
|
|
if (buildMode != bmNormal)
|
|
|
|
|
throw Error("repairing or checking is not supported when building through the Nix daemon");
|
2018-10-16 21:36:15 +00:00
|
|
|
|
conn.processStderr();
|
2016-02-23 14:00:59 +00:00
|
|
|
|
readInt(conn->from);
|
2006-11-30 18:35:50 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
2019-12-05 18:11:09 +00:00
|
|
|
|
BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
|
Allow remote builds without sending the derivation closure
Previously, to build a derivation remotely, we had to copy the entire
closure of the .drv file to the remote machine, even though we only
need the top-level derivation. This is very wasteful: the closure can
contain thousands of store paths, and in some Hydra use cases, include
source paths that are very large (e.g. Git/Mercurial checkouts).
So now there is a new operation, StoreAPI::buildDerivation(), that
performs a build from an in-memory representation of a derivation
(BasicDerivation) rather than from a on-disk .drv file. The only files
that need to be in the Nix store are the sources of the derivation
(drv.inputSrcs), and the needed output paths of the dependencies (as
described by drv.inputDrvs). "nix-store --serve" exposes this
interface.
Note that this is a privileged operation, because you can construct a
derivation that builds any store path whatsoever. Fixing this will
require changing the hashing scheme (i.e., the output paths should be
computed from the other fields in BasicDerivation, allowing them to be
verified without access to other derivations). However, this would be
quite nice because it would allow .drv-free building (e.g. "nix-env
-i" wouldn't have to write any .drv files to disk).
Fixes #173.
2015-07-17 15:57:40 +00:00
|
|
|
|
BuildMode buildMode)
|
|
|
|
|
{
|
2018-10-16 21:36:15 +00:00
|
|
|
|
auto conn(getConnection());
|
2019-12-05 18:11:09 +00:00
|
|
|
|
conn->to << wopBuildDerivation << printStorePath(drvPath);
|
|
|
|
|
writeDerivation(conn->to, *this, drv);
|
|
|
|
|
conn->to << buildMode;
|
2018-10-16 21:36:15 +00:00
|
|
|
|
conn.processStderr();
|
2015-09-03 10:56:59 +00:00
|
|
|
|
BuildResult res;
|
2021-02-28 18:42:46 +00:00
|
|
|
|
res.status = (BuildResult::Status) readInt(conn->from);
|
|
|
|
|
conn->from >> res.errorMsg;
|
|
|
|
|
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) {
|
2021-03-22 15:18:48 +00:00
|
|
|
|
conn->from >> res.timesBuilt >> res.isNonDeterministic >> res.startTime >> res.stopTime;
|
2021-02-28 18:42:46 +00:00
|
|
|
|
}
|
|
|
|
|
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 28) {
|
2021-01-25 10:08:38 +00:00
|
|
|
|
auto builtOutputs = worker_proto::read(*this, conn->from, Phantom<DrvOutputs> {});
|
|
|
|
|
res.builtOutputs = builtOutputs;
|
|
|
|
|
}
|
2015-09-03 10:56:59 +00:00
|
|
|
|
return res;
|
Allow remote builds without sending the derivation closure
Previously, to build a derivation remotely, we had to copy the entire
closure of the .drv file to the remote machine, even though we only
need the top-level derivation. This is very wasteful: the closure can
contain thousands of store paths, and in some Hydra use cases, include
source paths that are very large (e.g. Git/Mercurial checkouts).
So now there is a new operation, StoreAPI::buildDerivation(), that
performs a build from an in-memory representation of a derivation
(BasicDerivation) rather than from a on-disk .drv file. The only files
that need to be in the Nix store are the sources of the derivation
(drv.inputSrcs), and the needed output paths of the dependencies (as
described by drv.inputDrvs). "nix-store --serve" exposes this
interface.
Note that this is a privileged operation, because you can construct a
derivation that builds any store path whatsoever. Fixing this will
require changing the hashing scheme (i.e., the output paths should be
computed from the other fields in BasicDerivation, allowing them to be
verified without access to other derivations). However, this would be
quite nice because it would allow .drv-free building (e.g. "nix-env
-i" wouldn't have to write any .drv files to disk).
Fixes #173.
2015-07-17 15:57:40 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
2019-12-05 18:11:09 +00:00
|
|
|
|
void RemoteStore::ensurePath(const StorePath & path)
|
2006-11-30 18:35:50 +00:00
|
|
|
|
{
|
2018-10-16 21:36:15 +00:00
|
|
|
|
auto conn(getConnection());
|
2019-12-05 18:11:09 +00:00
|
|
|
|
conn->to << wopEnsurePath << printStorePath(path);
|
2018-10-16 21:36:15 +00:00
|
|
|
|
conn.processStderr();
|
2016-02-23 14:00:59 +00:00
|
|
|
|
readInt(conn->from);
|
2006-11-30 18:35:50 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
2019-12-05 18:11:09 +00:00
|
|
|
|
void RemoteStore::addTempRoot(const StorePath & path)
|
2006-12-02 16:41:36 +00:00
|
|
|
|
{
|
2018-10-16 21:36:15 +00:00
|
|
|
|
auto conn(getConnection());
|
2019-12-05 18:11:09 +00:00
|
|
|
|
conn->to << wopAddTempRoot << printStorePath(path);
|
2018-10-16 21:36:15 +00:00
|
|
|
|
conn.processStderr();
|
2016-02-23 14:00:59 +00:00
|
|
|
|
readInt(conn->from);
|
2006-12-02 16:41:36 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
2006-12-04 23:29:16 +00:00
|
|
|
|
void RemoteStore::addIndirectRoot(const Path & path)
|
|
|
|
|
{
|
2018-10-16 21:36:15 +00:00
|
|
|
|
auto conn(getConnection());
|
2016-02-23 14:00:59 +00:00
|
|
|
|
conn->to << wopAddIndirectRoot << path;
|
2018-10-16 21:36:15 +00:00
|
|
|
|
conn.processStderr();
|
2016-02-23 14:00:59 +00:00
|
|
|
|
readInt(conn->from);
|
2006-12-04 23:29:16 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
2019-03-14 12:50:07 +00:00
|
|
|
|
Roots RemoteStore::findRoots(bool censor)
|
2006-12-05 01:31:45 +00:00
|
|
|
|
{
|
2018-10-16 21:36:15 +00:00
|
|
|
|
auto conn(getConnection());
|
2016-02-23 14:00:59 +00:00
|
|
|
|
conn->to << wopFindRoots;
|
2018-10-16 21:36:15 +00:00
|
|
|
|
conn.processStderr();
|
2017-03-01 12:52:54 +00:00
|
|
|
|
size_t count = readNum<size_t>(conn->from);
|
2006-12-05 01:31:45 +00:00
|
|
|
|
Roots result;
|
|
|
|
|
while (count--) {
|
2016-02-23 14:00:59 +00:00
|
|
|
|
Path link = readString(conn->from);
|
2019-12-05 18:11:09 +00:00
|
|
|
|
auto target = parseStorePath(readString(conn->from));
|
|
|
|
|
result[std::move(target)].emplace(link);
|
2006-12-05 01:31:45 +00:00
|
|
|
|
}
|
|
|
|
|
return result;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
2008-06-18 09:34:17 +00:00
|
|
|
|
void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
|
2006-12-05 02:18:46 +00:00
|
|
|
|
{
|
2018-10-16 21:36:15 +00:00
|
|
|
|
auto conn(getConnection());
|
2012-07-30 21:13:25 +00:00
|
|
|
|
|
2016-07-27 13:03:20 +00:00
|
|
|
|
conn->to
|
2019-12-05 18:11:09 +00:00
|
|
|
|
<< wopCollectGarbage << options.action;
|
2020-09-30 00:39:06 +00:00
|
|
|
|
worker_proto::write(*this, conn->to, options.pathsToDelete);
|
2019-12-05 18:11:09 +00:00
|
|
|
|
conn->to << options.ignoreLiveness
|
2016-07-27 13:03:20 +00:00
|
|
|
|
<< options.maxFreed
|
2009-11-20 17:12:38 +00:00
|
|
|
|
/* removed options */
|
2016-07-27 13:03:20 +00:00
|
|
|
|
<< 0 << 0 << 0;
|
2012-07-30 21:13:25 +00:00
|
|
|
|
|
2018-10-16 21:36:15 +00:00
|
|
|
|
conn.processStderr();
|
2012-07-30 21:13:25 +00:00
|
|
|
|
|
2016-02-23 14:00:59 +00:00
|
|
|
|
results.paths = readStrings<PathSet>(conn->from);
|
|
|
|
|
results.bytesFreed = readLongLong(conn->from);
|
|
|
|
|
readLongLong(conn->from); // obsolete
|
2016-04-19 16:50:15 +00:00
|
|
|
|
|
|
|
|
|
{
|
|
|
|
|
auto state_(Store::state.lock());
|
|
|
|
|
state_->pathInfoCache.clear();
|
|
|
|
|
}
|
2006-12-05 02:18:46 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
2014-09-01 20:21:42 +00:00
|
|
|
|
void RemoteStore::optimiseStore()
|
|
|
|
|
{
|
2018-10-16 21:36:15 +00:00
|
|
|
|
auto conn(getConnection());
|
2016-02-23 14:00:59 +00:00
|
|
|
|
conn->to << wopOptimiseStore;
|
2018-10-16 21:36:15 +00:00
|
|
|
|
conn.processStderr();
|
2016-02-23 14:00:59 +00:00
|
|
|
|
readInt(conn->from);
|
2014-09-01 20:21:42 +00:00
|
|
|
|
}
|
2010-05-04 10:45:10 +00:00
|
|
|
|
|
2016-04-08 16:16:53 +00:00
|
|
|
|
|
2017-06-28 16:11:01 +00:00
|
|
|
|
bool RemoteStore::verifyStore(bool checkContents, RepairFlag repair)
|
2015-06-01 21:20:11 +00:00
|
|
|
|
{
|
2018-10-16 21:36:15 +00:00
|
|
|
|
auto conn(getConnection());
|
2016-02-23 14:00:59 +00:00
|
|
|
|
conn->to << wopVerifyStore << checkContents << repair;
|
2018-10-16 21:36:15 +00:00
|
|
|
|
conn.processStderr();
|
2017-03-01 12:52:54 +00:00
|
|
|
|
return readInt(conn->from);
|
2015-06-01 21:20:11 +00:00
|
|
|
|
}
|
|
|
|
|
|
2016-02-23 15:40:16 +00:00
|
|
|
|
|
2019-12-05 18:11:09 +00:00
|
|
|
|
void RemoteStore::addSignatures(const StorePath & storePath, const StringSet & sigs)
|
2016-04-05 13:30:22 +00:00
|
|
|
|
{
|
2018-10-16 21:36:15 +00:00
|
|
|
|
auto conn(getConnection());
|
2019-12-05 18:11:09 +00:00
|
|
|
|
conn->to << wopAddSignatures << printStorePath(storePath) << sigs;
|
2018-10-16 21:36:15 +00:00
|
|
|
|
conn.processStderr();
|
2016-04-05 13:30:22 +00:00
|
|
|
|
readInt(conn->from);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
2021-04-05 13:48:18 +00:00
|
|
|
|
void RemoteStore::queryMissing(const std::vector<DerivedPath> & targets,
|
2019-12-05 18:11:09 +00:00
|
|
|
|
StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
|
2020-07-30 11:10:49 +00:00
|
|
|
|
uint64_t & downloadSize, uint64_t & narSize)
|
2017-04-06 16:40:19 +00:00
|
|
|
|
{
|
|
|
|
|
{
|
2018-10-16 21:36:15 +00:00
|
|
|
|
auto conn(getConnection());
|
2017-04-06 16:40:19 +00:00
|
|
|
|
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 19)
|
|
|
|
|
// Don't hold the connection handle in the fallback case
|
|
|
|
|
// to prevent a deadlock.
|
|
|
|
|
goto fallback;
|
2019-12-05 18:11:09 +00:00
|
|
|
|
conn->to << wopQueryMissing;
|
2021-04-05 13:48:18 +00:00
|
|
|
|
writeDerivedPaths(*this, conn, targets);
|
2018-10-16 21:36:15 +00:00
|
|
|
|
conn.processStderr();
|
2020-09-30 00:39:06 +00:00
|
|
|
|
willBuild = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
|
|
|
|
|
willSubstitute = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
|
|
|
|
|
unknown = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
|
2017-04-06 16:40:19 +00:00
|
|
|
|
conn->from >> downloadSize >> narSize;
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fallback:
|
|
|
|
|
return Store::queryMissing(targets, willBuild, willSubstitute,
|
|
|
|
|
unknown, downloadSize, narSize);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
2017-05-02 12:18:46 +00:00
|
|
|
|
void RemoteStore::connect()
|
|
|
|
|
{
|
2018-10-16 21:36:15 +00:00
|
|
|
|
auto conn(getConnection());
|
2017-05-02 12:18:46 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
2018-08-30 21:28:47 +00:00
|
|
|
|
unsigned int RemoteStore::getProtocol()
|
|
|
|
|
{
|
|
|
|
|
auto conn(connections->get());
|
|
|
|
|
return conn->daemonVersion;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
2017-09-14 16:10:38 +00:00
|
|
|
|
void RemoteStore::flushBadConnections()
|
|
|
|
|
{
|
|
|
|
|
connections->flushBad();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
2016-02-23 15:40:16 +00:00
|
|
|
|
RemoteStore::Connection::~Connection()
|
|
|
|
|
{
|
|
|
|
|
try {
|
|
|
|
|
to.flush();
|
|
|
|
|
} catch (...) {
|
|
|
|
|
ignoreException();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2020-09-07 09:26:09 +00:00
|
|
|
|
void RemoteStore::narFromPath(const StorePath & path, Sink & sink)
|
|
|
|
|
{
|
|
|
|
|
auto conn(connections->get());
|
|
|
|
|
conn->to << wopNarFromPath << printStorePath(path);
|
|
|
|
|
conn->processStderr();
|
|
|
|
|
copyNAR(conn->from, sink);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ref<FSAccessor> RemoteStore::getFSAccessor()
|
|
|
|
|
{
|
|
|
|
|
return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()));
|
|
|
|
|
}
|
2016-02-23 15:40:16 +00:00
|
|
|
|
|
2017-08-28 16:49:42 +00:00
|
|
|
|
static Logger::Fields readFields(Source & from)
|
|
|
|
|
{
|
|
|
|
|
Logger::Fields fields;
|
|
|
|
|
size_t size = readInt(from);
|
|
|
|
|
for (size_t n = 0; n < size; n++) {
|
|
|
|
|
auto type = (decltype(Logger::Field::type)) readInt(from);
|
|
|
|
|
if (type == Logger::Field::tInt)
|
|
|
|
|
fields.push_back(readNum<uint64_t>(from));
|
|
|
|
|
else if (type == Logger::Field::tString)
|
|
|
|
|
fields.push_back(readString(from));
|
|
|
|
|
else
|
|
|
|
|
throw Error("got unsupported field type %x from Nix daemon", (int) type);
|
|
|
|
|
}
|
|
|
|
|
return fields;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
2020-08-27 12:48:08 +00:00
|
|
|
|
std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * source, bool flush)
|
2006-12-03 02:08:13 +00:00
|
|
|
|
{
|
2020-08-27 12:48:08 +00:00
|
|
|
|
if (flush)
|
|
|
|
|
to.flush();
|
2017-08-28 16:49:42 +00:00
|
|
|
|
|
|
|
|
|
while (true) {
|
|
|
|
|
|
|
|
|
|
auto msg = readNum<uint64_t>(from);
|
|
|
|
|
|
2007-02-21 17:34:02 +00:00
|
|
|
|
if (msg == STDERR_WRITE) {
|
|
|
|
|
string s = readString(from);
|
2007-02-21 16:34:00 +00:00
|
|
|
|
if (!sink) throw Error("no sink");
|
2016-05-04 13:46:25 +00:00
|
|
|
|
(*sink)(s);
|
2007-02-21 16:34:00 +00:00
|
|
|
|
}
|
2017-08-28 16:49:42 +00:00
|
|
|
|
|
2007-02-21 17:34:02 +00:00
|
|
|
|
else if (msg == STDERR_READ) {
|
|
|
|
|
if (!source) throw Error("no source");
|
2017-03-01 12:52:54 +00:00
|
|
|
|
size_t len = readNum<size_t>(from);
|
2020-12-02 13:10:56 +00:00
|
|
|
|
auto buf = std::make_unique<char[]>(len);
|
2020-12-02 13:00:43 +00:00
|
|
|
|
writeString({(const char *) buf.get(), source->read(buf.get(), len)}, to);
|
2011-12-14 23:30:06 +00:00
|
|
|
|
to.flush();
|
2007-02-21 17:34:02 +00:00
|
|
|
|
}
|
2017-08-28 16:49:42 +00:00
|
|
|
|
|
|
|
|
|
else if (msg == STDERR_ERROR) {
|
2020-10-07 15:13:54 +00:00
|
|
|
|
if (GET_PROTOCOL_MINOR(daemonVersion) >= 26) {
|
|
|
|
|
return std::make_exception_ptr(readError(from));
|
|
|
|
|
} else {
|
|
|
|
|
string error = readString(from);
|
|
|
|
|
unsigned int status = readInt(from);
|
|
|
|
|
return std::make_exception_ptr(Error(status, error));
|
|
|
|
|
}
|
2017-08-28 16:49:42 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
else if (msg == STDERR_NEXT)
|
2016-09-21 14:11:01 +00:00
|
|
|
|
printError(chomp(readString(from)));
|
2017-08-28 16:49:42 +00:00
|
|
|
|
|
|
|
|
|
else if (msg == STDERR_START_ACTIVITY) {
|
|
|
|
|
auto act = readNum<ActivityId>(from);
|
2017-08-28 17:13:24 +00:00
|
|
|
|
auto lvl = (Verbosity) readInt(from);
|
2017-08-28 16:49:42 +00:00
|
|
|
|
auto type = (ActivityType) readInt(from);
|
|
|
|
|
auto s = readString(from);
|
|
|
|
|
auto fields = readFields(from);
|
|
|
|
|
auto parent = readNum<ActivityId>(from);
|
2017-08-28 17:13:24 +00:00
|
|
|
|
logger->startActivity(act, lvl, type, s, fields, parent);
|
2017-08-28 16:49:42 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
else if (msg == STDERR_STOP_ACTIVITY) {
|
|
|
|
|
auto act = readNum<ActivityId>(from);
|
|
|
|
|
logger->stopActivity(act);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
else if (msg == STDERR_RESULT) {
|
|
|
|
|
auto act = readNum<ActivityId>(from);
|
|
|
|
|
auto type = (ResultType) readInt(from);
|
|
|
|
|
auto fields = readFields(from);
|
|
|
|
|
logger->result(act, type, fields);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
else if (msg == STDERR_LAST)
|
|
|
|
|
break;
|
|
|
|
|
|
|
|
|
|
else
|
|
|
|
|
throw Error("got unknown message type %x from Nix daemon", msg);
|
2006-12-03 02:08:13 +00:00
|
|
|
|
}
|
2018-10-16 21:36:15 +00:00
|
|
|
|
|
|
|
|
|
return nullptr;
|
2006-12-03 02:08:13 +00:00
|
|
|
|
}
|
|
|
|
|
|
2021-07-26 11:31:09 +00:00
|
|
|
|
void ConnectionHandle::withFramedSink(std::function<void(Sink & sink)> fun)
|
2020-09-22 09:40:19 +00:00
|
|
|
|
{
|
2020-09-17 15:36:16 +00:00
|
|
|
|
(*this)->to.flush();
|
|
|
|
|
|
|
|
|
|
std::exception_ptr ex;
|
|
|
|
|
|
2021-07-26 11:31:09 +00:00
|
|
|
|
/* Handle log messages / exceptions from the remote on a separate
|
|
|
|
|
thread. */
|
2020-09-17 15:36:16 +00:00
|
|
|
|
std::thread stderrThread([&]()
|
|
|
|
|
{
|
|
|
|
|
try {
|
|
|
|
|
processStderr(nullptr, nullptr, false);
|
|
|
|
|
} catch (...) {
|
|
|
|
|
ex = std::current_exception();
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
Finally joinStderrThread([&]()
|
|
|
|
|
{
|
|
|
|
|
if (stderrThread.joinable()) {
|
|
|
|
|
stderrThread.join();
|
|
|
|
|
if (ex) {
|
|
|
|
|
try {
|
|
|
|
|
std::rethrow_exception(ex);
|
|
|
|
|
} catch (...) {
|
|
|
|
|
ignoreException();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
{
|
2020-09-17 20:01:35 +00:00
|
|
|
|
FramedSink sink((*this)->to, ex);
|
2020-09-17 15:36:16 +00:00
|
|
|
|
fun(sink);
|
|
|
|
|
sink.flush();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
stderrThread.join();
|
|
|
|
|
if (ex)
|
|
|
|
|
std::rethrow_exception(ex);
|
|
|
|
|
}
|
|
|
|
|
|
2006-11-30 18:35:50 +00:00
|
|
|
|
}
|