Run the daemon worker on the same CPU as the client

On a system with multiple CPUs, running Nix operations through the
daemon is significantly slower than "direct" mode:

$ NIX_REMOTE= nix-instantiate '<nixos>' -A system
real    0m0.974s
user    0m0.875s
sys     0m0.088s

$ NIX_REMOTE=daemon nix-instantiate '<nixos>' -A system
real    0m2.118s
user    0m1.463s
sys     0m0.218s

The main reason seems to be that the client and the worker get moved
to a different CPU after every call to the worker.  This patch adds a
hack to lock them to the same CPU.  With this, the overhead of going
through the daemon is very small:

$ NIX_REMOTE=daemon nix-instantiate '<nixos>' -A system
real    0m1.074s
user    0m0.809s
sys     0m0.098s
This commit is contained in:
Eelco Dolstra 2013-08-07 11:51:55 +00:00
parent 263d668222
commit a583a2bc59
9 changed files with 92 additions and 4 deletions

View file

@ -127,6 +127,10 @@ AC_CHECK_HEADERS([sys/mount.h], [], [],
AC_CHECK_FUNCS([lutimes]) AC_CHECK_FUNCS([lutimes])
# Check for sched_setaffinity.
AC_CHECK_FUNCS([sched_setaffinity])
# Check whether the store optimiser can optimise symlinks. # Check whether the store optimiser can optimise symlinks.
AC_MSG_CHECKING([whether it is possible to create a link to a symlink]) AC_MSG_CHECKING([whether it is possible to create a link to a symlink])
ln -s bla tmp_link ln -s bla tmp_link

View file

@ -7,6 +7,7 @@
#include "local-store.hh" #include "local-store.hh"
#include "util.hh" #include "util.hh"
#include "archive.hh" #include "archive.hh"
#include "affinity.hh"
#include <map> #include <map>
#include <sstream> #include <sstream>
@ -366,6 +367,8 @@ void Goal::trace(const format & f)
/* Common initialisation performed in child processes. */ /* Common initialisation performed in child processes. */
static void commonChildInit(Pipe & logPipe) static void commonChildInit(Pipe & logPipe)
{ {
restoreAffinity();
/* Put the child in a separate session (and thus a separate /* Put the child in a separate session (and thus a separate
process group) so that it has no controlling terminal (meaning process group) so that it has no controlling terminal (meaning
that e.g. ssh cannot open /dev/tty) and it doesn't receive that e.g. ssh cannot open /dev/tty) and it doesn't receive
@ -568,6 +571,7 @@ static void runSetuidHelper(const string & command,
args.push_back(0); args.push_back(0);
restoreSIGPIPE(); restoreSIGPIPE();
restoreAffinity();
execve(program.c_str(), (char * *) &args[0], 0); execve(program.c_str(), (char * *) &args[0], 0);
throw SysError(format("executing `%1%'") % program); throw SysError(format("executing `%1%'") % program);

View file

@ -5,6 +5,7 @@
#include "pathlocks.hh" #include "pathlocks.hh"
#include "worker-protocol.hh" #include "worker-protocol.hh"
#include "derivations.hh" #include "derivations.hh"
#include "affinity.hh"
#include <iostream> #include <iostream>
#include <algorithm> #include <algorithm>
@ -1021,6 +1022,7 @@ void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter &
case 0: /* child */ case 0: /* child */
try { try {
restoreAffinity();
if (dup2(toPipe.readSide, STDIN_FILENO) == -1) if (dup2(toPipe.readSide, STDIN_FILENO) == -1)
throw SysError("dupping stdin"); throw SysError("dupping stdin");
if (dup2(fromPipe.writeSide, STDOUT_FILENO) == -1) if (dup2(fromPipe.writeSide, STDOUT_FILENO) == -1)

View file

@ -3,6 +3,7 @@
#include "remote-store.hh" #include "remote-store.hh"
#include "worker-protocol.hh" #include "worker-protocol.hh"
#include "archive.hh" #include "archive.hh"
#include "affinity.hh"
#include "globals.hh" #include "globals.hh"
#include <sys/types.h> #include <sys/types.h>
@ -15,7 +16,6 @@
#include <unistd.h> #include <unistd.h>
#include <cstring> #include <cstring>
namespace nix { namespace nix {
@ -71,8 +71,19 @@ void RemoteStore::openConnection(bool reserveSpace)
if (GET_PROTOCOL_MAJOR(daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION)) if (GET_PROTOCOL_MAJOR(daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION))
throw Error("Nix daemon protocol version not supported"); throw Error("Nix daemon protocol version not supported");
writeInt(PROTOCOL_VERSION, to); writeInt(PROTOCOL_VERSION, to);
if (GET_PROTOCOL_MINOR(daemonVersion) >= 14) {
int cpu = lockToCurrentCPU();
if (cpu != -1) {
writeInt(1, to);
writeInt(cpu, to);
} else
writeInt(0, to);
}
if (GET_PROTOCOL_MINOR(daemonVersion) >= 11) if (GET_PROTOCOL_MINOR(daemonVersion) >= 11)
writeInt(reserveSpace, to); writeInt(reserveSpace, to);
processStderr(); processStderr();
} }
catch (Error & e) { catch (Error & e) {

View file

@ -6,7 +6,7 @@ namespace nix {
#define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_1 0x6e697863
#define WORKER_MAGIC_2 0x6478696f #define WORKER_MAGIC_2 0x6478696f
#define PROTOCOL_VERSION 0x10d #define PROTOCOL_VERSION 0x10e
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)

View file

@ -1,12 +1,12 @@
pkglib_LTLIBRARIES = libutil.la pkglib_LTLIBRARIES = libutil.la
libutil_la_SOURCES = util.cc hash.cc serialise.cc \ libutil_la_SOURCES = util.cc hash.cc serialise.cc \
archive.cc xml-writer.cc archive.cc xml-writer.cc affinity.cc
libutil_la_LIBADD = ../boost/format/libformat.la libutil_la_LIBADD = ../boost/format/libformat.la
pkginclude_HEADERS = util.hh hash.hh serialise.hh \ pkginclude_HEADERS = util.hh hash.hh serialise.hh \
archive.hh xml-writer.hh types.hh archive.hh xml-writer.hh types.hh affinity.hh
if !HAVE_OPENSSL if !HAVE_OPENSSL
libutil_la_SOURCES += \ libutil_la_SOURCES += \

54
src/libutil/affinity.cc Normal file
View file

@ -0,0 +1,54 @@
#include "types.hh"
#include "util.hh"
#include "affinity.hh"
#if HAVE_SCHED_H
#include <sched.h>
#endif
namespace nix {
static bool didSaveAffinity = false;
static cpu_set_t savedAffinity;
void setAffinityTo(int cpu)
{
#if HAVE_SCHED_SETAFFINITY
if (sched_getaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) return;
didSaveAffinity = true;
printMsg(lvlDebug, format("locking this thread to CPU %1%") % cpu);
cpu_set_t newAffinity;
CPU_ZERO(&newAffinity);
CPU_SET(cpu, &newAffinity);
if (sched_setaffinity(0, sizeof(cpu_set_t), &newAffinity) == -1)
printMsg(lvlError, format("failed to lock thread to CPU %1%") % cpu);
#endif
}
int lockToCurrentCPU()
{
#if HAVE_SCHED_SETAFFINITY
if (getEnv("NIX_AFFINITY_HACK", "1") == "1") {
int cpu = sched_getcpu();
if (cpu != -1) setAffinityTo(cpu);
return cpu;
}
#endif
return -1;
}
void restoreAffinity()
{
#if HAVE_SCHED_SETAFFINITY
if (!didSaveAffinity) return;
if (sched_setaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1)
printMsg(lvlError, "failed to restore affinity %1%");
#endif
}
}

9
src/libutil/affinity.hh Normal file
View file

@ -0,0 +1,9 @@
#pragma once
namespace nix {
void setAffinityTo(int cpu);
int lockToCurrentCPU();
void restoreAffinity();
}

View file

@ -4,6 +4,7 @@
#include "serialise.hh" #include "serialise.hh"
#include "worker-protocol.hh" #include "worker-protocol.hh"
#include "archive.hh" #include "archive.hh"
#include "affinity.hh"
#include "globals.hh" #include "globals.hh"
#include <cstring> #include <cstring>
@ -671,6 +672,9 @@ static void processConnection(bool trusted)
to.flush(); to.flush();
unsigned int clientVersion = readInt(from); unsigned int clientVersion = readInt(from);
if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from))
setAffinityTo(readInt(from));
bool reserveSpace = true; bool reserveSpace = true;
if (GET_PROTOCOL_MINOR(clientVersion) >= 11) if (GET_PROTOCOL_MINOR(clientVersion) >= 11)
reserveSpace = readInt(from) != 0; reserveSpace = readInt(from) != 0;