From a583a2bc59a4ee2b067e5520f6c5bc0c61852c32 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 7 Aug 2013 11:51:55 +0000 Subject: [PATCH] Run the daemon worker on the same CPU as the client On a system with multiple CPUs, running Nix operations through the daemon is significantly slower than "direct" mode: $ NIX_REMOTE= nix-instantiate '' -A system real 0m0.974s user 0m0.875s sys 0m0.088s $ NIX_REMOTE=daemon nix-instantiate '' -A system real 0m2.118s user 0m1.463s sys 0m0.218s The main reason seems to be that the client and the worker get moved to a different CPU after every call to the worker. This patch adds a hack to lock them to the same CPU. With this, the overhead of going through the daemon is very small: $ NIX_REMOTE=daemon nix-instantiate '' -A system real 0m1.074s user 0m0.809s sys 0m0.098s --- configure.ac | 4 +++ src/libstore/build.cc | 4 +++ src/libstore/local-store.cc | 2 ++ src/libstore/remote-store.cc | 13 +++++++- src/libstore/worker-protocol.hh | 2 +- src/libutil/Makefile.am | 4 +-- src/libutil/affinity.cc | 54 +++++++++++++++++++++++++++++++++ src/libutil/affinity.hh | 9 ++++++ src/nix-daemon/nix-daemon.cc | 4 +++ 9 files changed, 92 insertions(+), 4 deletions(-) create mode 100644 src/libutil/affinity.cc create mode 100644 src/libutil/affinity.hh diff --git a/configure.ac b/configure.ac index 9ffefa914..89f009923 100644 --- a/configure.ac +++ b/configure.ac @@ -127,6 +127,10 @@ AC_CHECK_HEADERS([sys/mount.h], [], [], AC_CHECK_FUNCS([lutimes]) +# Check for sched_setaffinity. +AC_CHECK_FUNCS([sched_setaffinity]) + + # Check whether the store optimiser can optimise symlinks. AC_MSG_CHECKING([whether it is possible to create a link to a symlink]) ln -s bla tmp_link diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 1ca99dcc5..67e38da0b 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -7,6 +7,7 @@ #include "local-store.hh" #include "util.hh" #include "archive.hh" +#include "affinity.hh" #include #include @@ -366,6 +367,8 @@ void Goal::trace(const format & f) /* Common initialisation performed in child processes. */ static void commonChildInit(Pipe & logPipe) { + restoreAffinity(); + /* Put the child in a separate session (and thus a separate process group) so that it has no controlling terminal (meaning that e.g. ssh cannot open /dev/tty) and it doesn't receive @@ -568,6 +571,7 @@ static void runSetuidHelper(const string & command, args.push_back(0); restoreSIGPIPE(); + restoreAffinity(); execve(program.c_str(), (char * *) &args[0], 0); throw SysError(format("executing `%1%'") % program); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 29095e1ea..34b4a5158 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -5,6 +5,7 @@ #include "pathlocks.hh" #include "worker-protocol.hh" #include "derivations.hh" +#include "affinity.hh" #include #include @@ -1021,6 +1022,7 @@ void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter & case 0: /* child */ try { + restoreAffinity(); if (dup2(toPipe.readSide, STDIN_FILENO) == -1) throw SysError("dupping stdin"); if (dup2(fromPipe.writeSide, STDOUT_FILENO) == -1) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 2b5a93213..3764b4813 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -3,6 +3,7 @@ #include "remote-store.hh" #include "worker-protocol.hh" #include "archive.hh" +#include "affinity.hh" #include "globals.hh" #include @@ -15,7 +16,6 @@ #include #include - namespace nix { @@ -71,8 +71,19 @@ void RemoteStore::openConnection(bool reserveSpace) if (GET_PROTOCOL_MAJOR(daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION)) throw Error("Nix daemon protocol version not supported"); writeInt(PROTOCOL_VERSION, to); + + if (GET_PROTOCOL_MINOR(daemonVersion) >= 14) { + int cpu = lockToCurrentCPU(); + if (cpu != -1) { + writeInt(1, to); + writeInt(cpu, to); + } else + writeInt(0, to); + } + if (GET_PROTOCOL_MINOR(daemonVersion) >= 11) writeInt(reserveSpace, to); + processStderr(); } catch (Error & e) { diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 07f825b92..9317f89c3 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -6,7 +6,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x10d +#define PROTOCOL_VERSION 0x10e #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) diff --git a/src/libutil/Makefile.am b/src/libutil/Makefile.am index fe896eec5..0c4073e66 100644 --- a/src/libutil/Makefile.am +++ b/src/libutil/Makefile.am @@ -1,12 +1,12 @@ pkglib_LTLIBRARIES = libutil.la libutil_la_SOURCES = util.cc hash.cc serialise.cc \ - archive.cc xml-writer.cc + archive.cc xml-writer.cc affinity.cc libutil_la_LIBADD = ../boost/format/libformat.la pkginclude_HEADERS = util.hh hash.hh serialise.hh \ - archive.hh xml-writer.hh types.hh + archive.hh xml-writer.hh types.hh affinity.hh if !HAVE_OPENSSL libutil_la_SOURCES += \ diff --git a/src/libutil/affinity.cc b/src/libutil/affinity.cc new file mode 100644 index 000000000..3a20fd277 --- /dev/null +++ b/src/libutil/affinity.cc @@ -0,0 +1,54 @@ +#include "types.hh" +#include "util.hh" +#include "affinity.hh" + +#if HAVE_SCHED_H +#include +#endif + +namespace nix { + + +static bool didSaveAffinity = false; +static cpu_set_t savedAffinity; + + +void setAffinityTo(int cpu) +{ +#if HAVE_SCHED_SETAFFINITY + if (sched_getaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) return; + didSaveAffinity = true; + printMsg(lvlDebug, format("locking this thread to CPU %1%") % cpu); + cpu_set_t newAffinity; + CPU_ZERO(&newAffinity); + CPU_SET(cpu, &newAffinity); + if (sched_setaffinity(0, sizeof(cpu_set_t), &newAffinity) == -1) + printMsg(lvlError, format("failed to lock thread to CPU %1%") % cpu); +#endif +} + + +int lockToCurrentCPU() +{ +#if HAVE_SCHED_SETAFFINITY + if (getEnv("NIX_AFFINITY_HACK", "1") == "1") { + int cpu = sched_getcpu(); + if (cpu != -1) setAffinityTo(cpu); + return cpu; + } +#endif + return -1; +} + + +void restoreAffinity() +{ +#if HAVE_SCHED_SETAFFINITY + if (!didSaveAffinity) return; + if (sched_setaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) + printMsg(lvlError, "failed to restore affinity %1%"); +#endif +} + + +} diff --git a/src/libutil/affinity.hh b/src/libutil/affinity.hh new file mode 100644 index 000000000..c1bd28e13 --- /dev/null +++ b/src/libutil/affinity.hh @@ -0,0 +1,9 @@ +#pragma once + +namespace nix { + +void setAffinityTo(int cpu); +int lockToCurrentCPU(); +void restoreAffinity(); + +} diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index a6f29b7c9..86f5c0a24 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -4,6 +4,7 @@ #include "serialise.hh" #include "worker-protocol.hh" #include "archive.hh" +#include "affinity.hh" #include "globals.hh" #include @@ -671,6 +672,9 @@ static void processConnection(bool trusted) to.flush(); unsigned int clientVersion = readInt(from); + if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from)) + setAffinityTo(readInt(from)); + bool reserveSpace = true; if (GET_PROTOCOL_MINOR(clientVersion) >= 11) reserveSpace = readInt(from) != 0;