Merge branch 'master' into no-manifests

This commit is contained in:
Eelco Dolstra 2012-08-27 11:09:07 -04:00
commit e94806d030
20 changed files with 325 additions and 139 deletions

View file

@ -46,7 +46,7 @@ Flags:
--no-out-link: do not create the `result' symlink --no-out-link: do not create the `result' symlink
--out-link / -o NAME: create symlink NAME instead of `result' --out-link / -o NAME: create symlink NAME instead of `result'
--attr / -A ATTR: select a specific attribute from the Nix expression --attr / -A ATTR: select a specific attribute from the Nix expression
--run-env: build dependencies of the specified derivation, then start a --run-env: build dependencies of the specified derivation, then start a
shell with the environment of the derivation shell with the environment of the derivation
--command: command to run with `--run-env' --command: command to run with `--run-env'
@ -119,7 +119,7 @@ EOF
push @buildArgs, "--dry-run"; push @buildArgs, "--dry-run";
$dryRun = 1; $dryRun = 1;
} }
elsif ($arg eq "--show-trace") { elsif ($arg eq "--show-trace") {
push @instArgs, $arg; push @instArgs, $arg;
} }
@ -127,22 +127,22 @@ EOF
elsif ($arg eq "-") { elsif ($arg eq "-") {
@exprs = ("-"); @exprs = ("-");
} }
elsif ($arg eq "--verbose" or substr($arg, 0, 2) eq "-v") { elsif ($arg eq "--verbose" or substr($arg, 0, 2) eq "-v") {
push @buildArgs, $arg; push @buildArgs, $arg;
push @instArgs, $arg; push @instArgs, $arg;
$verbose = 1; $verbose = 1;
} }
elsif ($arg eq "--quiet") { elsif ($arg eq "--quiet") {
push @buildArgs, $arg; push @buildArgs, $arg;
push @instArgs, $arg; push @instArgs, $arg;
} }
elsif ($arg eq "--run-env") { elsif ($arg eq "--run-env") {
$runEnv = 1; $runEnv = 1;
} }
elsif ($arg eq "--command") { elsif ($arg eq "--command") {
$n++; $n++;
die "$0: `$arg' requires an argument\n" unless $n < scalar @ARGV; die "$0: `$arg' requires an argument\n" unless $n < scalar @ARGV;

View file

@ -144,6 +144,8 @@ EvalState::EvalState()
{ {
nrEnvs = nrValuesInEnvs = nrValues = nrListElems = 0; nrEnvs = nrValuesInEnvs = nrValues = nrListElems = 0;
nrAttrsets = nrOpUpdates = nrOpUpdateValuesCopied = 0; nrAttrsets = nrOpUpdates = nrOpUpdateValuesCopied = 0;
nrListConcats = nrPrimOpCalls = nrFunctionCalls = 0;
countCalls = getEnv("NIX_COUNT_CALLS", "0") != "0";
#if HAVE_BOEHMGC #if HAVE_BOEHMGC
static bool gcInitialised = true; static bool gcInitialised = true;
@ -300,8 +302,10 @@ inline Value * EvalState::lookupVar(Env * env, const VarRef & var)
if (var.fromWith) { if (var.fromWith) {
while (1) { while (1) {
Bindings::iterator j = env->values[0]->attrs->find(var.name); Bindings::iterator j = env->values[0]->attrs->find(var.name);
if (j != env->values[0]->attrs->end()) if (j != env->values[0]->attrs->end()) {
if (countCalls && j->pos) attrSelects[*j->pos]++;
return j->value; return j->value;
}
if (env->prevWith == 0) if (env->prevWith == 0)
throwEvalError("undefined variable `%1%'", var.name); throwEvalError("undefined variable `%1%'", var.name);
for (unsigned int l = env->prevWith; l; --l, env = env->up) ; for (unsigned int l = env->prevWith; l; --l, env = env->up) ;
@ -344,7 +348,7 @@ void EvalState::mkList(Value & v, unsigned int length)
{ {
v.type = tList; v.type = tList;
v.list.length = length; v.list.length = length;
v.list.elems = (Value * *) GC_MALLOC(length * sizeof(Value *)); v.list.elems = length ? (Value * *) GC_MALLOC(length * sizeof(Value *)) : 0;
nrListElems += length; nrListElems += length;
} }
@ -619,8 +623,10 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
} }
vAttrs = j->value; vAttrs = j->value;
pos = j->pos; pos = j->pos;
if (state.countCalls && pos) state.attrSelects[*pos]++;
} }
state.forceValue(*vAttrs); state.forceValue(*vAttrs);
} catch (Error & e) { } catch (Error & e) {
@ -700,6 +706,8 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v)
vArgs[n--] = arg->primOpApp.right; vArgs[n--] = arg->primOpApp.right;
/* And call the primop. */ /* And call the primop. */
nrPrimOpCalls++;
if (countCalls) primOpCalls[primOp->primOp->name]++;
try { try {
primOp->primOp->fun(*this, vArgs, v); primOp->primOp->fun(*this, vArgs, v);
} catch (Error & e) { } catch (Error & e) {
@ -716,7 +724,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v)
} }
if (fun.type != tLambda) if (fun.type != tLambda)
throwTypeError("attempt to call something which is neither a function nor a primop (built-in operation) but %1%", throwTypeError("attempt to call something which is not a function but %1%",
showType(fun)); showType(fun));
unsigned int size = unsigned int size =
@ -760,6 +768,9 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v)
throwTypeError("function at %1% called with unexpected argument", fun.lambda.fun->pos); throwTypeError("function at %1% called with unexpected argument", fun.lambda.fun->pos);
} }
nrFunctionCalls++;
if (countCalls) functionCalls[fun.lambda.fun->pos]++;
try { try {
fun.lambda.fun->body->eval(*this, env2, v); fun.lambda.fun->body->eval(*this, env2, v);
} catch (Error & e) { } catch (Error & e) {
@ -902,14 +913,36 @@ void ExprOpUpdate::eval(EvalState & state, Env & env, Value & v)
void ExprOpConcatLists::eval(EvalState & state, Env & env, Value & v) void ExprOpConcatLists::eval(EvalState & state, Env & env, Value & v)
{ {
Value v1; e1->eval(state, env, v1); Value v1; e1->eval(state, env, v1);
state.forceList(v1);
Value v2; e2->eval(state, env, v2); Value v2; e2->eval(state, env, v2);
state.forceList(v2); Value * lists[2] = { &v1, &v2 };
state.mkList(v, v1.list.length + v2.list.length); state.concatLists(v, 2, lists);
for (unsigned int n = 0; n < v1.list.length; ++n) }
v.list.elems[n] = v1.list.elems[n];
for (unsigned int n = 0; n < v2.list.length; ++n)
v.list.elems[n + v1.list.length] = v2.list.elems[n]; void EvalState::concatLists(Value & v, unsigned int nrLists, Value * * lists)
{
nrListConcats++;
Value * nonEmpty = 0;
unsigned int len = 0;
for (unsigned int n = 0; n < nrLists; ++n) {
forceList(*lists[n]);
unsigned int l = lists[n]->list.length;
len += l;
if (l) nonEmpty = lists[n];
}
if (nonEmpty && len == nonEmpty->list.length) {
v = *nonEmpty;
return;
}
mkList(v, len);
for (unsigned int n = 0, pos = 0; n < nrLists; ++n) {
unsigned int l = lists[n]->list.length;
memcpy(v.list.elems + pos, lists[n]->list.elems, l * sizeof(Value *));
pos += l;
}
} }
@ -932,7 +965,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
isPath = vStr.type == tPath; isPath = vStr.type == tPath;
first = false; first = false;
} }
s << state.coerceToString(vStr, context, false, !isPath); s << state.coerceToString(vStr, context, false, !isPath);
} }
@ -1207,6 +1240,7 @@ void EvalState::printStats()
% nrEnvs % (nrEnvs * sizeof(Env) + nrValuesInEnvs * sizeof(Value *))); % nrEnvs % (nrEnvs * sizeof(Env) + nrValuesInEnvs * sizeof(Value *)));
printMsg(v, format(" list elements: %1% (%2% bytes)") printMsg(v, format(" list elements: %1% (%2% bytes)")
% nrListElems % (nrListElems * sizeof(Value *))); % nrListElems % (nrListElems * sizeof(Value *)));
printMsg(v, format(" list concatenations: %1%") % nrListConcats);
printMsg(v, format(" values allocated: %1% (%2% bytes)") printMsg(v, format(" values allocated: %1% (%2% bytes)")
% nrValues % (nrValues * sizeof(Value))); % nrValues % (nrValues * sizeof(Value)));
printMsg(v, format(" attribute sets allocated: %1%") % nrAttrsets); printMsg(v, format(" attribute sets allocated: %1%") % nrAttrsets);
@ -1216,6 +1250,36 @@ void EvalState::printStats()
printMsg(v, format(" number of thunks: %1%") % nrThunks); printMsg(v, format(" number of thunks: %1%") % nrThunks);
printMsg(v, format(" number of thunks avoided: %1%") % nrAvoided); printMsg(v, format(" number of thunks avoided: %1%") % nrAvoided);
printMsg(v, format(" number of attr lookups: %1%") % nrLookups); printMsg(v, format(" number of attr lookups: %1%") % nrLookups);
printMsg(v, format(" number of primop calls: %1%") % nrPrimOpCalls);
printMsg(v, format(" number of function calls: %1%") % nrFunctionCalls);
if (countCalls) {
printMsg(v, format("calls to %1% primops:") % primOpCalls.size());
typedef std::multimap<unsigned int, Symbol> PrimOpCalls_;
std::multimap<unsigned int, Symbol> primOpCalls_;
foreach (PrimOpCalls::iterator, i, primOpCalls)
primOpCalls_.insert(std::pair<unsigned int, Symbol>(i->second, i->first));
foreach_reverse (PrimOpCalls_::reverse_iterator, i, primOpCalls_)
printMsg(v, format("%1$10d %2%") % i->first % i->second);
printMsg(v, format("calls to %1% functions:") % functionCalls.size());
typedef std::multimap<unsigned int, Pos> FunctionCalls_;
std::multimap<unsigned int, Pos> functionCalls_;
foreach (FunctionCalls::iterator, i, functionCalls)
functionCalls_.insert(std::pair<unsigned int, Pos>(i->second, i->first));
foreach_reverse (FunctionCalls_::reverse_iterator, i, functionCalls_)
printMsg(v, format("%1$10d %2%") % i->first % i->second);
printMsg(v, format("evaluations of %1% attributes:") % attrSelects.size());
typedef std::multimap<unsigned int, Pos> AttrSelects_;
std::multimap<unsigned int, Pos> attrSelects_;
foreach (AttrSelects::iterator, i, attrSelects)
attrSelects_.insert(std::pair<unsigned int, Pos>(i->second, i->first));
foreach_reverse (AttrSelects_::reverse_iterator, i, attrSelects_)
printMsg(v, format("%1$10d %2%") % i->first % i->second);
}
} }

View file

@ -232,11 +232,13 @@ public:
void mkAttrs(Value & v, unsigned int expected); void mkAttrs(Value & v, unsigned int expected);
void mkThunk_(Value & v, Expr * expr); void mkThunk_(Value & v, Expr * expr);
void concatLists(Value & v, unsigned int nrLists, Value * * lists);
/* Print statistics. */ /* Print statistics. */
void printStats(); void printStats();
private: private:
unsigned long nrEnvs; unsigned long nrEnvs;
unsigned long nrValuesInEnvs; unsigned long nrValuesInEnvs;
unsigned long nrValues; unsigned long nrValues;
@ -244,9 +246,25 @@ private:
unsigned long nrAttrsets; unsigned long nrAttrsets;
unsigned long nrOpUpdates; unsigned long nrOpUpdates;
unsigned long nrOpUpdateValuesCopied; unsigned long nrOpUpdateValuesCopied;
unsigned long nrListConcats;
friend class RecursionCounter; unsigned long nrPrimOpCalls;
unsigned long nrFunctionCalls;
bool countCalls;
typedef std::map<Symbol, unsigned int> PrimOpCalls;
PrimOpCalls primOpCalls;
typedef std::map<Pos, unsigned int> FunctionCalls;
FunctionCalls functionCalls;
typedef std::map<Pos, unsigned int> AttrSelects;
AttrSelects attrSelects;
friend class ExprOpUpdate; friend class ExprOpUpdate;
friend class ExprOpConcatLists;
friend class ExprSelect;
friend void prim_getAttr(EvalState & state, Value * * args, Value & v);
}; };

View file

@ -27,6 +27,15 @@ struct Pos
Pos() : line(0), column(0) { }; Pos() : line(0), column(0) { };
Pos(const string & file, unsigned int line, unsigned int column) Pos(const string & file, unsigned int line, unsigned int column)
: file(file), line(line), column(column) { }; : file(file), line(line), column(column) { };
bool operator < (const Pos & p2) const
{
int d = file.compare(p2.file);
if (d < 0) return true;
if (d > 0) return false;
if (line < p2.line) return true;
if (line > p2.line) return false;
return column < p2.column;
}
}; };
extern Pos noPos; extern Pos noPos;

View file

@ -203,7 +203,7 @@ static Expr * stripIndentation(SymbolTable & symbols, vector<Expr *> & es)
es2->push_back(new ExprString(symbols.create(s2))); es2->push_back(new ExprString(symbols.create(s2)));
} }
return new ExprConcatStrings(es2); return es2->size() == 1 ? (*es2)[0] : new ExprConcatStrings(es2);
} }

View file

@ -719,7 +719,7 @@ static void prim_attrNames(EvalState & state, Value * * args, Value & v)
/* Dynamic version of the `.' operator. */ /* Dynamic version of the `.' operator. */
static void prim_getAttr(EvalState & state, Value * * args, Value & v) void prim_getAttr(EvalState & state, Value * * args, Value & v)
{ {
string attr = state.forceStringNoCtx(*args[0]); string attr = state.forceStringNoCtx(*args[0]);
state.forceAttrs(*args[1]); state.forceAttrs(*args[1]);
@ -728,6 +728,7 @@ static void prim_getAttr(EvalState & state, Value * * args, Value & v)
if (i == args[1]->attrs->end()) if (i == args[1]->attrs->end())
throw EvalError(format("attribute `%1%' missing") % attr); throw EvalError(format("attribute `%1%' missing") % attr);
// !!! add to stack trace? // !!! add to stack trace?
if (state.countCalls && i->pos) state.attrSelects[*i->pos]++;
state.forceValue(*i->value); state.forceValue(*i->value);
v = *i->value; v = *i->value;
} }
@ -873,19 +874,33 @@ static void prim_isList(EvalState & state, Value * * args, Value & v)
} }
static void elemAt(EvalState & state, Value & list, int n, Value & v)
{
state.forceList(list);
if (n < 0 || n >= list.list.length)
throw Error(format("list index %1% is out of bounds") % n);
state.forceValue(*list.list.elems[n]);
v = *list.list.elems[n];
}
/* Return the n-1'th element of a list. */
static void prim_elemAt(EvalState & state, Value * * args, Value & v)
{
elemAt(state, *args[0], state.forceInt(*args[1]), v);
}
/* Return the first element of a list. */ /* Return the first element of a list. */
static void prim_head(EvalState & state, Value * * args, Value & v) static void prim_head(EvalState & state, Value * * args, Value & v)
{ {
state.forceList(*args[0]); elemAt(state, *args[0], 0, v);
if (args[0]->list.length == 0)
throw Error("`head' called on an empty list");
state.forceValue(*args[0]->list.elems[0]);
v = *args[0]->list.elems[0];
} }
/* Return a list consisting of everything but the the first element of /* Return a list consisting of everything but the the first element of
a list. */ a list. Warning: this function takes O(n) time, so you probably
don't want to use it! */
static void prim_tail(EvalState & state, Value * * args, Value & v) static void prim_tail(EvalState & state, Value * * args, Value & v)
{ {
state.forceList(*args[0]); state.forceList(*args[0]);
@ -911,6 +926,52 @@ static void prim_map(EvalState & state, Value * * args, Value & v)
} }
/* Filter a list using a predicate; that is, return a list containing
every element from the list for which the predicate function
returns true. */
static void prim_filter(EvalState & state, Value * * args, Value & v)
{
state.forceFunction(*args[0]);
state.forceList(*args[1]);
// FIXME: putting this on the stack is risky.
Value * vs[args[1]->list.length];
unsigned int k = 0;
for (unsigned int n = 0; n < args[1]->list.length; ++n) {
Value res;
state.callFunction(*args[0], *args[1]->list.elems[n], res);
if (state.forceBool(res))
vs[k++] = args[1]->list.elems[n];
}
state.mkList(v, k);
for (unsigned int n = 0; n < k; ++n) v.list.elems[n] = vs[n];
}
/* Return true if a list contains a given element. */
static void prim_elem(EvalState & state, Value * * args, Value & v)
{
bool res = false;
state.forceList(*args[1]);
for (unsigned int n = 0; n < args[1]->list.length; ++n)
if (state.eqValues(*args[0], *args[1]->list.elems[n])) {
res = true;
break;
}
mkBool(v, res);
}
/* Concatenate a list of lists. */
static void prim_concatLists(EvalState & state, Value * * args, Value & v)
{
state.forceList(*args[0]);
state.concatLists(v, args[0]->list.length, args[0]->list.elems);
}
/* Return the length of a list. This is an O(1) time operation. */ /* Return the length of a list. This is an O(1) time operation. */
static void prim_length(EvalState & state, Value * * args, Value & v) static void prim_length(EvalState & state, Value * * args, Value & v)
{ {
@ -1122,11 +1183,15 @@ void EvalState::createBaseEnv()
// Lists // Lists
addPrimOp("__isList", 1, prim_isList); addPrimOp("__isList", 1, prim_isList);
addPrimOp("__elemAt", 2, prim_elemAt);
addPrimOp("__head", 1, prim_head); addPrimOp("__head", 1, prim_head);
addPrimOp("__tail", 1, prim_tail); addPrimOp("__tail", 1, prim_tail);
addPrimOp("map", 2, prim_map); addPrimOp("map", 2, prim_map);
addPrimOp("__filter", 2, prim_filter);
addPrimOp("__elem", 2, prim_elem);
addPrimOp("__concatLists", 1, prim_concatLists);
addPrimOp("__length", 1, prim_length); addPrimOp("__length", 1, prim_length);
// Integer arithmetic // Integer arithmetic
addPrimOp("__add", 2, prim_add); addPrimOp("__add", 2, prim_add);
addPrimOp("__sub", 2, prim_sub); addPrimOp("__sub", 2, prim_sub);

View file

@ -33,16 +33,6 @@ static void sigintHandler(int signo)
} }
Path makeRootName(const Path & gcRoot, int & counter)
{
counter++;
if (counter == 1)
return gcRoot;
else
return (format("%1%-%2%") % gcRoot % counter).str();
}
void printGCWarning() void printGCWarning()
{ {
static bool haveWarned = false; static bool haveWarned = false;

View file

@ -26,7 +26,6 @@ MakeError(UsageError, nix::Error);
class StoreAPI; class StoreAPI;
/* Ugh. No better place to put this. */ /* Ugh. No better place to put this. */
Path makeRootName(const Path & gcRoot, int & counter);
void printGCWarning(); void printGCWarning();
void printMissing(StoreAPI & store, const PathSet & paths); void printMissing(StoreAPI & store, const PathSet & paths);

View file

@ -45,7 +45,7 @@
#include <sched.h> #include <sched.h>
#endif #endif
#define CHROOT_ENABLED HAVE_CHROOT && HAVE_UNSHARE && HAVE_SYS_MOUNT_H && defined(MS_BIND) && defined(CLONE_NEWNS) #define CHROOT_ENABLED HAVE_CHROOT && HAVE_UNSHARE && HAVE_SYS_MOUNT_H && defined(MS_BIND) && defined(MS_PRIVATE) && defined(CLONE_NEWNS)
#if CHROOT_ENABLED #if CHROOT_ENABLED
#include <sys/socket.h> #include <sys/socket.h>
@ -604,18 +604,17 @@ void getOwnership(const Path & path)
} }
void deletePathWrapped(const Path & path, void deletePathWrapped(const Path & path, unsigned long long & bytesFreed)
unsigned long long & bytesFreed, unsigned long long & blocksFreed)
{ {
try { try {
/* First try to delete it ourselves. */ /* First try to delete it ourselves. */
deletePath(path, bytesFreed, blocksFreed); deletePath(path, bytesFreed);
} catch (SysError & e) { } catch (SysError & e) {
/* If this failed due to a permission error, then try it with /* If this failed due to a permission error, then try it with
the setuid helper. */ the setuid helper. */
if (settings.buildUsersGroup != "" && !amPrivileged()) { if (settings.buildUsersGroup != "" && !amPrivileged()) {
getOwnership(path); getOwnership(path);
deletePath(path, bytesFreed, blocksFreed); deletePath(path, bytesFreed);
} else } else
throw; throw;
} }
@ -624,8 +623,8 @@ void deletePathWrapped(const Path & path,
void deletePathWrapped(const Path & path) void deletePathWrapped(const Path & path)
{ {
unsigned long long dummy1, dummy2; unsigned long long dummy1;
deletePathWrapped(path, dummy1, dummy2); deletePathWrapped(path, dummy1);
} }
@ -1470,9 +1469,9 @@ HookReply DerivationGoal::tryBuildHook()
} }
void chmod(const Path & path, mode_t mode) void chmod_(const Path & path, mode_t mode)
{ {
if (::chmod(path.c_str(), 01777) == -1) if (chmod(path.c_str(), mode) == -1)
throw SysError(format("setting permissions on `%1%'") % path); throw SysError(format("setting permissions on `%1%'") % path);
} }
@ -1674,7 +1673,7 @@ void DerivationGoal::startBuilder()
instead.) */ instead.) */
Path chrootTmpDir = chrootRootDir + "/tmp"; Path chrootTmpDir = chrootRootDir + "/tmp";
createDirs(chrootTmpDir); createDirs(chrootTmpDir);
chmod(chrootTmpDir, 01777); chmod_(chrootTmpDir, 01777);
/* Create a /etc/passwd with entries for the build user and the /* Create a /etc/passwd with entries for the build user and the
nobody account. The latter is kind of a hack to support nobody account. The latter is kind of a hack to support
@ -1710,7 +1709,7 @@ void DerivationGoal::startBuilder()
precaution, make the fake Nix store only writable by the precaution, make the fake Nix store only writable by the
build user. */ build user. */
createDirs(chrootRootDir + settings.nixStore); createDirs(chrootRootDir + settings.nixStore);
chmod(chrootRootDir + settings.nixStore, 01777); chmod_(chrootRootDir + settings.nixStore, 01777);
foreach (PathSet::iterator, i, inputPaths) { foreach (PathSet::iterator, i, inputPaths) {
struct stat st; struct stat st;
@ -1844,22 +1843,40 @@ void DerivationGoal::initChild()
char domainname[] = "(none)"; // kernel default char domainname[] = "(none)"; // kernel default
setdomainname(domainname, sizeof(domainname)); setdomainname(domainname, sizeof(domainname));
/* Make all filesystems private. This is necessary
because subtrees may have been mounted as "shared"
(MS_SHARED). (Systemd does this, for instance.) Even
though we have a private mount namespace, mounting
filesystems on top of a shared subtree still propagates
outside of the namespace. Making a subtree private is
local to the namespace, though, so setting MS_PRIVATE
does not affect the outside world. */
Strings mounts = tokenizeString(readFile("/proc/self/mountinfo", true), "\n");
foreach (Strings::iterator, i, mounts) {
Strings fields = tokenizeString(*i, " ");
assert(fields.size() >= 5);
Strings::iterator j = fields.begin();
std::advance(j, 4);
if (mount(0, j->c_str(), 0, MS_PRIVATE, 0) == -1)
throw SysError(format("unable to make filesystem `%1%' private") % *j);
}
/* Bind-mount all the directories from the "host" /* Bind-mount all the directories from the "host"
filesystem that we want in the chroot filesystem that we want in the chroot
environment. */ environment. */
foreach (PathSet::iterator, i, dirsInChroot) { foreach (PathSet::iterator, i, dirsInChroot) {
Path source = *i; Path source = *i;
Path target = chrootRootDir + source; Path target = chrootRootDir + source;
if (source == "/proc") continue; // backwards compatibility
debug(format("bind mounting `%1%' to `%2%'") % source % target); debug(format("bind mounting `%1%' to `%2%'") % source % target);
createDirs(target); createDirs(target);
if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1) if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1)
throw SysError(format("bind mount from `%1%' to `%2%' failed") % source % target); throw SysError(format("bind mount from `%1%' to `%2%' failed") % source % target);
} }
/* Bind a new instance of procfs on /proc to reflect our /* Bind a new instance of procfs on /proc to reflect our
private PID namespace. */ private PID namespace. */
createDirs(chrootRootDir + "/proc");
if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1) if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1)
throw SysError("mounting /proc"); throw SysError("mounting /proc");

View file

@ -425,10 +425,9 @@ bool LocalStore::isActiveTempFile(const GCState & state,
void LocalStore::deleteGarbage(GCState & state, const Path & path) void LocalStore::deleteGarbage(GCState & state, const Path & path)
{ {
printMsg(lvlInfo, format("deleting `%1%'") % path); printMsg(lvlInfo, format("deleting `%1%'") % path);
unsigned long long bytesFreed, blocksFreed; unsigned long long bytesFreed;
deletePathWrapped(path, bytesFreed, blocksFreed); deletePathWrapped(path, bytesFreed);
state.results.bytesFreed += bytesFreed; state.results.bytesFreed += bytesFreed;
state.results.blocksFreed += blocksFreed;
} }
@ -550,7 +549,7 @@ bool LocalStore::tryToDelete(GCState & state, const Path & path)
} else } else
deleteGarbage(state, path); deleteGarbage(state, path);
if (state.options.maxFreed && state.results.bytesFreed + state.bytesInvalidated > state.options.maxFreed) { if (state.results.bytesFreed + state.bytesInvalidated > state.options.maxFreed) {
printMsg(lvlInfo, format("deleted or invalidated more than %1% bytes; stopping") % state.options.maxFreed); printMsg(lvlInfo, format("deleted or invalidated more than %1% bytes; stopping") % state.options.maxFreed);
throw GCLimitReached(); throw GCLimitReached();
} }
@ -576,11 +575,13 @@ bool LocalStore::tryToDelete(GCState & state, const Path & path)
safely deleted. FIXME: race condition with optimisePath(): we safely deleted. FIXME: race condition with optimisePath(): we
might see a link count of 1 just before optimisePath() increases might see a link count of 1 just before optimisePath() increases
the link count. */ the link count. */
void LocalStore::removeUnusedLinks() void LocalStore::removeUnusedLinks(const GCState & state)
{ {
AutoCloseDir dir = opendir(linksDir.c_str()); AutoCloseDir dir = opendir(linksDir.c_str());
if (!dir) throw SysError(format("opening directory `%1%'") % linksDir); if (!dir) throw SysError(format("opening directory `%1%'") % linksDir);
long long actualSize = 0, unsharedSize = 0;
struct dirent * dirent; struct dirent * dirent;
while (errno = 0, dirent = readdir(dir)) { while (errno = 0, dirent = readdir(dir)) {
checkInterrupt(); checkInterrupt();
@ -592,13 +593,28 @@ void LocalStore::removeUnusedLinks()
if (lstat(path.c_str(), &st) == -1) if (lstat(path.c_str(), &st) == -1)
throw SysError(format("statting `%1%'") % path); throw SysError(format("statting `%1%'") % path);
if (st.st_nlink != 1) continue; if (st.st_nlink != 1) {
unsigned long long size = st.st_blocks * 512ULL;
actualSize += size;
unsharedSize += (st.st_nlink - 1) * size;
continue;
}
printMsg(lvlTalkative, format("deleting unused link `%1%'") % path); printMsg(lvlTalkative, format("deleting unused link `%1%'") % path);
if (unlink(path.c_str()) == -1) if (unlink(path.c_str()) == -1)
throw SysError(format("deleting `%1%'") % path); throw SysError(format("deleting `%1%'") % path);
state.results.bytesFreed += st.st_blocks * 512;
} }
struct stat st;
if (stat(linksDir.c_str(), &st) == -1)
throw SysError(format("statting `%1%'") % linksDir);
long long overhead = st.st_blocks * 512ULL;
printMsg(lvlInfo, format("note: currently hard linking saves %.2f MiB")
% ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0)));
} }
@ -660,7 +676,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
throw Error(format("cannot delete path `%1%' since it is still alive") % *i); throw Error(format("cannot delete path `%1%' since it is still alive") % *i);
} }
} else { } else if (options.maxFreed > 0) {
if (shouldDelete(state.options.action)) if (shouldDelete(state.options.action))
printMsg(lvlError, format("deleting garbage...")); printMsg(lvlError, format("deleting garbage..."));
@ -718,7 +734,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
/* Clean up the links directory. */ /* Clean up the links directory. */
printMsg(lvlError, format("deleting unused links...")); printMsg(lvlError, format("deleting unused links..."));
removeUnusedLinks(); removeUnusedLinks(state);
} }

View file

@ -261,7 +261,7 @@ private:
int openGCLock(LockType lockType); int openGCLock(LockType lockType);
void removeUnusedLinks(); void removeUnusedLinks(const GCState & state);
void startSubstituter(const Path & substituter, void startSubstituter(const Path & substituter,
RunningSubstituter & runningSubstituter); RunningSubstituter & runningSubstituter);
@ -298,8 +298,7 @@ void getOwnership(const Path & path);
/* Like deletePath(), but changes the ownership of `path' using the /* Like deletePath(), but changes the ownership of `path' using the
setuid wrapper if necessary (and possible). */ setuid wrapper if necessary (and possible). */
void deletePathWrapped(const Path & path, void deletePathWrapped(const Path & path, unsigned long long & bytesFreed);
unsigned long long & bytesFreed, unsigned long long & blocksFreed);
void deletePathWrapped(const Path & path); void deletePathWrapped(const Path & path);

View file

@ -102,11 +102,11 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path)
/* Nope, create a hard link in the links directory. */ /* Nope, create a hard link in the links directory. */
makeMutable(path); makeMutable(path);
MakeImmutable mk1(path); MakeImmutable mk1(path);
if (link(path.c_str(), linkPath.c_str()) == 0) return;
if (link(path.c_str(), linkPath.c_str()) == -1) if (errno != EEXIST)
throw SysError(format("cannot link `%1%' to `%2%'") % linkPath % path); throw SysError(format("cannot link `%1%' to `%2%'") % linkPath % path);
/* Fall through if another process created linkPath before
return; we did. */
} }
/* Yes! We've seen a file with the same contents. Replace the /* Yes! We've seen a file with the same contents. Replace the
@ -123,9 +123,6 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path)
printMsg(lvlTalkative, format("linking `%1%' to `%2%'") % path % linkPath); printMsg(lvlTalkative, format("linking `%1%' to `%2%'") % path % linkPath);
Path tempLink = (format("%1%/.tmp-link-%2%-%3%")
% settings.nixStore % getpid() % rand()).str();
/* Make the containing directory writable, but only if it's not /* Make the containing directory writable, but only if it's not
the store itself (we don't want or need to mess with its the store itself (we don't want or need to mess with its
permissions). */ permissions). */
@ -140,40 +137,55 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path)
so make it mutable first (and make it immutable again when so make it mutable first (and make it immutable again when
we're done). We also have to make path mutable, otherwise we're done). We also have to make path mutable, otherwise
rename() will fail to delete it. */ rename() will fail to delete it. */
makeMutable(linkPath);
MakeImmutable mk1(linkPath);
makeMutable(path); makeMutable(path);
MakeImmutable mk2(path); MakeImmutable mk2(path);
if (link(linkPath.c_str(), tempLink.c_str()) == -1) { /* Another process might be doing the same thing (creating a new
if (errno == EMLINK) { link to linkPath) and make linkPath immutable before we're
/* Too many links to the same file (>= 32000 on most file done. In that case, just retry. */
systems). This is likely to happen with empty files. unsigned int retries = 1024;
Just shrug and ignore. */ while (--retries > 0) {
printMsg(lvlInfo, format("`%1%' has maximum number of links") % linkPath); makeMutable(linkPath);
return; MakeImmutable mk1(linkPath);
Path tempLink = (format("%1%/.tmp-link-%2%-%3%")
% settings.nixStore % getpid() % rand()).str();
if (link(linkPath.c_str(), tempLink.c_str()) == -1) {
if (errno == EMLINK) {
/* Too many links to the same file (>= 32000 on most
file systems). This is likely to happen with empty
files. Just shrug and ignore. */
if (st.st_size)
printMsg(lvlInfo, format("`%1%' has maximum number of links") % linkPath);
return;
}
if (errno == EPERM) continue;
throw SysError(format("cannot link `%1%' to `%2%'") % tempLink % linkPath);
} }
throw SysError(format("cannot link `%1%' to `%2%'") % tempLink % linkPath);
}
/* Atomically replace the old file with the new hard link. */ /* Atomically replace the old file with the new hard link. */
if (rename(tempLink.c_str(), path.c_str()) == -1) { if (rename(tempLink.c_str(), path.c_str()) == -1) {
if (errno == EMLINK) {
/* Some filesystems generate too many links on the rename,
rather than on the original link. (Probably it
temporarily increases the st_nlink field before
decreasing it again.) */
printMsg(lvlInfo, format("`%1%' has maximum number of links") % linkPath);
/* Unlink the temp link. */
if (unlink(tempLink.c_str()) == -1) if (unlink(tempLink.c_str()) == -1)
printMsg(lvlError, format("unable to unlink `%1%'") % tempLink); printMsg(lvlError, format("unable to unlink `%1%'") % tempLink);
return; if (errno == EMLINK) {
/* Some filesystems generate too many links on the
rename, rather than on the original link.
(Probably it temporarily increases the st_nlink
field before decreasing it again.) */
if (st.st_size)
printMsg(lvlInfo, format("`%1%' has maximum number of links") % linkPath);
return;
}
if (errno == EPERM) continue;
throw SysError(format("cannot rename `%1%' to `%2%'") % tempLink % path);
} }
throw SysError(format("cannot rename `%1%' to `%2%'") % tempLink % path);
break;
} }
if (retries == 0) throw Error(format("cannot link `%1%' to `%2%'") % path % linkPath);
stats.filesLinked++; stats.filesLinked++;
stats.bytesFreed += st.st_size; stats.bytesFreed += st.st_size;
stats.blocksFreed += st.st_blocks; stats.blocksFreed += st.st_blocks;

View file

@ -558,7 +558,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
results.paths = readStrings<PathSet>(from); results.paths = readStrings<PathSet>(from);
results.bytesFreed = readLongLong(from); results.bytesFreed = readLongLong(from);
results.blocksFreed = readLongLong(from); readLongLong(from); // obsolete
} }

View file

@ -2,7 +2,7 @@
#include "globals.hh" #include "globals.hh"
#include "util.hh" #include "util.hh"
#include <limits.h> #include <climits>
namespace nix { namespace nix {
@ -12,7 +12,7 @@ GCOptions::GCOptions()
{ {
action = gcDeleteDead; action = gcDeleteDead;
ignoreLiveness = false; ignoreLiveness = false;
maxFreed = 0; maxFreed = ULLONG_MAX;
} }

View file

@ -48,8 +48,7 @@ struct GCOptions
/* For `gcDeleteSpecific', the paths to delete. */ /* For `gcDeleteSpecific', the paths to delete. */
PathSet pathsToDelete; PathSet pathsToDelete;
/* Stop after at least `maxFreed' bytes have been freed. 0 means /* Stop after at least `maxFreed' bytes have been freed. */
no limit. */
unsigned long long maxFreed; unsigned long long maxFreed;
GCOptions(); GCOptions();
@ -66,13 +65,9 @@ struct GCResults
number of bytes that would be or was freed. */ number of bytes that would be or was freed. */
unsigned long long bytesFreed; unsigned long long bytesFreed;
/* The number of file system blocks that would be or was freed. */
unsigned long long blocksFreed;
GCResults() GCResults()
{ {
bytesFreed = 0; bytesFreed = 0;
blocksFreed = 0;
} }
}; };

View file

@ -224,12 +224,12 @@ string readFile(int fd)
} }
string readFile(const Path & path) string readFile(const Path & path, bool drain)
{ {
AutoCloseFD fd = open(path.c_str(), O_RDONLY); AutoCloseFD fd = open(path.c_str(), O_RDONLY);
if (fd == -1) if (fd == -1)
throw SysError(format("opening file `%1%'") % path); throw SysError(format("opening file `%1%'") % path);
return readFile(fd); return drain ? drainFD(fd) : readFile(fd);
} }
@ -297,8 +297,7 @@ void computePathSize(const Path & path,
} }
static void _deletePath(const Path & path, unsigned long long & bytesFreed, static void _deletePath(const Path & path, unsigned long long & bytesFreed)
unsigned long long & blocksFreed)
{ {
checkInterrupt(); checkInterrupt();
@ -308,10 +307,8 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed,
if (S_ISDIR(st.st_mode) || S_ISREG(st.st_mode)) makeMutable(path); if (S_ISDIR(st.st_mode) || S_ISREG(st.st_mode)) makeMutable(path);
if (!S_ISDIR(st.st_mode) && st.st_nlink == 1) { if (!S_ISDIR(st.st_mode) && st.st_nlink == 1)
bytesFreed += st.st_size; bytesFreed += st.st_blocks * 512;
blocksFreed += st.st_blocks;
}
if (S_ISDIR(st.st_mode)) { if (S_ISDIR(st.st_mode)) {
Strings names = readDirectory(path); Strings names = readDirectory(path);
@ -323,7 +320,7 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed,
} }
for (Strings::iterator i = names.begin(); i != names.end(); ++i) for (Strings::iterator i = names.begin(); i != names.end(); ++i)
_deletePath(path + "/" + *i, bytesFreed, blocksFreed); _deletePath(path + "/" + *i, bytesFreed);
} }
if (remove(path.c_str()) == -1) if (remove(path.c_str()) == -1)
@ -333,19 +330,17 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed,
void deletePath(const Path & path) void deletePath(const Path & path)
{ {
unsigned long long dummy1, dummy2; unsigned long long dummy;
deletePath(path, dummy1, dummy2); deletePath(path, dummy);
} }
void deletePath(const Path & path, unsigned long long & bytesFreed, void deletePath(const Path & path, unsigned long long & bytesFreed)
unsigned long long & blocksFreed)
{ {
startNest(nest, lvlDebug, startNest(nest, lvlDebug,
format("recursively deleting path `%1%'") % path); format("recursively deleting path `%1%'") % path);
bytesFreed = 0; bytesFreed = 0;
blocksFreed = 0; _deletePath(path, bytesFreed);
_deletePath(path, bytesFreed, blocksFreed);
} }

View file

@ -17,6 +17,9 @@ namespace nix {
#define foreach(it_type, it, collection) \ #define foreach(it_type, it, collection) \
for (it_type it = (collection).begin(); it != (collection).end(); ++it) for (it_type it = (collection).begin(); it != (collection).end(); ++it)
#define foreach_reverse(it_type, it, collection) \
for (it_type it = (collection).rbegin(); it != (collection).rend(); ++it)
/* Return an environment variable. */ /* Return an environment variable. */
string getEnv(const string & key, const string & def = ""); string getEnv(const string & key, const string & def = "");
@ -60,7 +63,7 @@ Strings readDirectory(const Path & path);
/* Read the contents of a file into a string. */ /* Read the contents of a file into a string. */
string readFile(int fd); string readFile(int fd);
string readFile(const Path & path); string readFile(const Path & path, bool drain = false);
/* Write a string to a file. */ /* Write a string to a file. */
void writeFile(const Path & path, const string & s); void writeFile(const Path & path, const string & s);
@ -80,8 +83,7 @@ void computePathSize(const Path & path,
returns the number of bytes and blocks freed. */ returns the number of bytes and blocks freed. */
void deletePath(const Path & path); void deletePath(const Path & path);
void deletePath(const Path & path, unsigned long long & bytesFreed, void deletePath(const Path & path, unsigned long long & bytesFreed);
unsigned long long & blocksFreed);
/* Make a path read-only recursively. */ /* Make a path read-only recursively. */
void makePathReadOnly(const Path & path); void makePathReadOnly(const Path & path);

View file

@ -64,9 +64,11 @@ void processExpr(EvalState & state, const Strings & attrPaths,
Path drvPath = i->queryDrvPath(state); Path drvPath = i->queryDrvPath(state);
if (gcRoot == "") if (gcRoot == "")
printGCWarning(); printGCWarning();
else else {
drvPath = addPermRoot(*store, drvPath, Path rootName = gcRoot;
makeRootName(gcRoot, rootNr), indirectRoot); if (++rootNr > 1) rootName += "-" + int2String(rootNr);
drvPath = addPermRoot(*store, drvPath, rootName, indirectRoot);
}
std::cout << format("%1%\n") % drvPath; std::cout << format("%1%\n") % drvPath;
} }
} }

View file

@ -64,15 +64,19 @@ static PathSet realisePath(const Path & path)
if (isDerivation(path)) { if (isDerivation(path)) {
store->buildPaths(singleton<PathSet>(path)); store->buildPaths(singleton<PathSet>(path));
Derivation drv = derivationFromPath(*store, path); Derivation drv = derivationFromPath(*store, path);
rootNr++;
PathSet outputs; PathSet outputs;
foreach (DerivationOutputs::iterator, i, drv.outputs) { foreach (DerivationOutputs::iterator, i, drv.outputs) {
Path outPath = i->second.path; Path outPath = i->second.path;
if (gcRoot == "") if (gcRoot == "")
printGCWarning(); printGCWarning();
else else {
outPath = addPermRoot(*store, outPath, Path rootName = gcRoot;
makeRootName(gcRoot, rootNr), indirectRoot); if (rootNr > 1) rootName += "-" + int2String(rootNr);
if (i->first != "out") rootName += "-" + i->first;
outPath = addPermRoot(*store, outPath, rootName, indirectRoot);
}
outputs.insert(outPath); outputs.insert(outPath);
} }
return outputs; return outputs;
@ -544,10 +548,9 @@ static void opCheckValidity(Strings opFlags, Strings opArgs)
} }
static string showBytes(unsigned long long bytes, unsigned long long blocks) static string showBytes(unsigned long long bytes)
{ {
return (format("%d bytes (%.2f MiB, %d blocks)") return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str();
% bytes % (bytes / (1024.0 * 1024.0)) % blocks).str();
} }
@ -562,7 +565,7 @@ struct PrintFreed
if (show) if (show)
cout << format("%1% store paths deleted, %2% freed\n") cout << format("%1% store paths deleted, %2% freed\n")
% results.paths.size() % results.paths.size()
% showBytes(results.bytesFreed, results.blocksFreed); % showBytes(results.bytesFreed);
} }
}; };
@ -583,7 +586,7 @@ static void opGC(Strings opFlags, Strings opArgs)
else if (*i == "--delete") options.action = GCOptions::gcDeleteDead; else if (*i == "--delete") options.action = GCOptions::gcDeleteDead;
else if (*i == "--max-freed") { else if (*i == "--max-freed") {
long long maxFreed = getIntArg<long long>(*i, i, opFlags.end()); long long maxFreed = getIntArg<long long>(*i, i, opFlags.end());
options.maxFreed = maxFreed >= 1 ? maxFreed : 1; options.maxFreed = maxFreed >= 0 ? maxFreed : 0;
} }
else throw UsageError(format("bad sub-operation `%1%' in GC") % *i); else throw UsageError(format("bad sub-operation `%1%' in GC") % *i);
@ -735,7 +738,7 @@ static void showOptimiseStats(OptimiseStats & stats)
{ {
printMsg(lvlError, printMsg(lvlError,
format("%1% freed by hard-linking %2% files; there are %3% files with equal contents out of %4% files in total") format("%1% freed by hard-linking %2% files; there are %3% files with equal contents out of %4% files in total")
% showBytes(stats.bytesFreed, stats.blocksFreed) % showBytes(stats.bytesFreed)
% stats.filesLinked % stats.filesLinked
% stats.sameContents % stats.sameContents
% stats.totalFiles); % stats.totalFiles);

View file

@ -521,7 +521,7 @@ static void performOp(unsigned int clientVersion,
writeStrings(results.paths, to); writeStrings(results.paths, to);
writeLongLong(results.bytesFreed, to); writeLongLong(results.bytesFreed, to);
writeLongLong(results.blocksFreed, to); writeLongLong(0, to); // obsolete
break; break;
} }
@ -661,6 +661,10 @@ static void processConnection()
to.flush(); to.flush();
unsigned int clientVersion = readInt(from); unsigned int clientVersion = readInt(from);
bool reserveSpace = true;
if (GET_PROTOCOL_MINOR(clientVersion) >= 11)
reserveSpace = readInt(from) != 0;
/* Send startup error messages to the client. */ /* Send startup error messages to the client. */
startWork(); startWork();
@ -676,10 +680,6 @@ static void processConnection()
throw Error("if you run `nix-worker' as root, then you MUST set `build-users-group'!"); throw Error("if you run `nix-worker' as root, then you MUST set `build-users-group'!");
#endif #endif
bool reserveSpace = true;
if (GET_PROTOCOL_MINOR(clientVersion) >= 11)
reserveSpace = readInt(from) != 0;
/* Open the store. */ /* Open the store. */
store = boost::shared_ptr<StoreAPI>(new LocalStore(reserveSpace)); store = boost::shared_ptr<StoreAPI>(new LocalStore(reserveSpace));