Merge branch 'master' into no-manifests

doc/manual/conf-file.xml

@@ -297,6 +297,16 @@ build-use-chroot = /dev /proc /bin</programlisting>
   </varlistentry>
 
 
+  <varlistentry><term><literal>build-compress-log</literal></term>
+
+    <listitem><para>If set to <literal>true</literal> (the default),
+    build logs written to <filename>/nix/var/log/nix/drvs</filename>
+    will be compressed on the fly using bzip2.  Otherwise, they will
+    not be compressed.</para></listitem>
+
+  </varlistentry>
+
+
   <varlistentry><term><literal>binary-caches</literal></term>
 
     <listitem><para>A list of URLs of binary caches, separated by
@@ -305,7 +315,7 @@ build-use-chroot = /dev /proc /bin</programlisting>
     <literal>http://nixos.org/binary-cache</literal>.</para></listitem>
 
   </varlistentry>
-  
+
 
   <varlistentry><term><literal>binary-caches-parallel-connections</literal></term>
 
@@ -315,7 +325,7 @@ build-use-chroot = /dev /proc /bin</programlisting>
     150.</para></listitem>
 
   </varlistentry>
-  
+
 
   <varlistentry><term><literal>system</literal></term>
 

doc/manual/introduction.xml

@@ -225,7 +225,7 @@ href="docs/papers.html#servicecm">SCM-12 paper</a>.</para>
 <simplesect><title>Portability</title>
 
 <para>Nix should run on most Unix systems, including Linux, FreeBSD and
-Mac OS X.  It is also supported on Windows using Cygwin.</para>
+Mac OS X.<!-- It is also supported on Windows using Cygwin.--></para>
 
 </simplesect>
 

doc/manual/release-notes.xml

@@ -15,25 +15,84 @@
 <itemizedlist>
 
   <listitem>
-    <para>Builds in chroots are now executed in a private network
-    namespace, meaning that they do not see any network interfaces
-    except a private loopback interface.  This ensures that builds
-    cannot communicate with processes outside of the chroot, or clash
-    with other builds by listening on an identical port number.</para>
+    <para>On Linux, when doing a chroot build, Nix now uses various
+    namespace features provided by the Linux kernel to improve
+    build isolation.  Namely:
+    <itemizedlist>
+      <listitem><para>The private network namespace ensures that
+      builders cannot talk to the outside world (or vice versa): each
+      build only sees a private loopback interface.  This also means
+      that two concurrent builds can listen on the same port (e.g. as
+      part of a test) without conflicting with each
+      other.</para></listitem>
+      <listitem><para>The PID namespace causes each build to start as
+      PID 1.  Processes outside of the chroot are not visible to those
+      on the inside.  On the other hand, processes inside the chroot
+      <emphasis>are</emphasis> visible from the outside (though with
+      different PIDs).</para></listitem>
+      <listitem><para>The IPC namespace prevents the builder from
+      communicating with outside processes using SysV IPC mechanisms
+      (shared memory, message queues, semaphores).  It also ensures
+      that all IPC objects are destroyed when the builder
+      exits.</para></listitem>
+      <listitem><para>The UTS namespace ensures that builders see a
+      hostname of <literal>localhost</literal> rather than the actual
+      hostname.</para></listitem>
+      <listitem><para>The private mount namespace was already used by
+      Nix to ensure that the bind-mounts used to set up the chroot are
+      cleaned up automatically.</para></listitem>
+    </itemizedlist>
+    </para>
   </listitem>
 
   <listitem>
     <para>Build logs are now compressed using
     <command>bzip2</command>.  The command <command>nix-store
-    -l</command> decompresses them on the fly.</para>
+    -l</command> decompresses them on the fly.  This can be disabled
+    by setting the option <literal>build-compress-log</literal> to
+    <literal>false</literal>.</para>
   </listitem>
 
   <listitem>
     <para>The creation of build logs in
     <filename>/nix/var/log/nix/drvs</filename> can be disabled by
     setting the new option <literal>build-keep-log</literal> to
-    <literal>false</literal>.</para>
-  </listitem>    
+    <literal>false</literal>.  This is useful, for instance, for Hydra
+    build machines.</para>
+  </listitem>
+
+  <listitem>
+    <para>Nix now reserves some space in
+    <filename>/nix/var/nix/db/reserved</filename> to ensure that the
+    garbage collector can run successfully if the disk is full.  This
+    is necessary because SQLite transactions fail if the disk is
+    full.</para>
+  </listitem>
+
+  <listitem>
+    <para>Added a basic <function>fetchurl</function> function.  This
+    is not intended to replace the <function>fetchurl</function> in
+    Nixpkgs, but is useful for bootstrapping; e.g., it will allow us
+    to get rid of the bootstrap binaries in the Nixpkgs source tree
+    and download them instead.  You can use it by doing
+    <literal>import &lt;nix/fetchurl.nix> { url =
+    <replaceable>url</replaceable>; sha256 =
+    "<replaceable>hash</replaceable>"; }</literal>. (Shea Levy)</para>
+  </listitem>
+
+  <listitem>
+    <para>Improved RPM spec file. (Michel Alexandre Salim)</para>
+  </listitem>
+
+  <listitem>
+    <para>Support for on-demand socket-based activation in the Nix
+    daemon with <command>systemd</command>.</para>
+  </listitem>
+
+  <listitem>
+    <para>Added a manpage for
+    <citerefentry><refentrytitle>nix.conf</refentrytitle><manvolnum>5</manvolnum></citerefentry>.</para>
+  </listitem>
 
   <listitem>
     <para>When using the Nix daemon, the <option>-s</option> flag in

perl/lib/Nix/Store.pm

@@ -14,6 +14,7 @@ our @EXPORT_OK = ( @{ $EXPORT_TAGS{'all'} } );
 
 our @EXPORT = qw(
     isValidPath queryReferences queryPathInfo queryDeriver queryPathHash
+    queryPathFromHashPart
     topoSortPaths computeFSClosure followLinksToStorePath exportPaths
     hashPath hashFile hashString
     addToStore makeFixedOutputPath

perl/lib/Nix/Store.xs

@@ -108,6 +108,17 @@ SV * queryPathInfo(char * path, int base32)
         }
 
 
+SV * queryPathFromHashPart(char * hashPart)
+    PPCODE:
+        try {
+            doInit();
+            Path path = store->queryPathFromHashPart(hashPart);
+            XPUSHs(sv_2mortal(newSVpv(path.c_str(), 0)));
+        } catch (Error & e) {
+            croak(e.what());
+        }
+
+
 SV * computeFSClosure(int flipDirection, int includeOutputs, ...)
     PPCODE:
         try {

src/libexpr/eval.cc

@@ -143,9 +143,7 @@ EvalState::EvalState()
     , staticBaseEnv(false, 0)
 {
     nrEnvs = nrValuesInEnvs = nrValues = nrListElems = 0;
-    nrEvaluated = recursionDepth = maxRecursionDepth = 0;
     nrAttrsets = nrOpUpdates = nrOpUpdateValuesCopied = 0;
-    deepestStack = (char *) -1;
 
 #if HAVE_BOEHMGC
     static bool gcInitialised = true;
@@ -190,7 +188,6 @@ EvalState::EvalState()
 
 EvalState::~EvalState()
 {
-    assert(recursionDepth == 0);
 }
 
 
@@ -1206,12 +1203,6 @@ void EvalState::printStats()
 
     printMsg(v, format("  time elapsed: %1%") % cpuTime);
     printMsg(v, format("  size of a value: %1%") % sizeof(Value));
-    printMsg(v, format("  expressions evaluated: %1%") % nrEvaluated);
-    char x;
-    printMsg(v, format("  stack space used: %1% bytes") % (&x - deepestStack));
-    printMsg(v, format("  max eval() nesting depth: %1%") % maxRecursionDepth);
-    printMsg(v, format("  stack space per eval() level: %1% bytes")
-        % ((&x - deepestStack) / (float) maxRecursionDepth));
     printMsg(v, format("  environments allocated: %1% (%2% bytes)")
         % nrEnvs % (nrEnvs * sizeof(Env) + nrValuesInEnvs * sizeof(Value *)));
     printMsg(v, format("  list elements: %1% (%2% bytes)")

src/libexpr/eval.hh

@@ -242,13 +242,9 @@ private:
     unsigned long nrValuesInEnvs;
     unsigned long nrValues;
     unsigned long nrListElems;
-    unsigned long nrEvaluated;
     unsigned long nrAttrsets;
     unsigned long nrOpUpdates;
     unsigned long nrOpUpdateValuesCopied;
-    unsigned int recursionDepth;
-    unsigned int maxRecursionDepth;
-    char * deepestStack; /* for measuring stack usage */
     
     friend class RecursionCounter;
     friend class ExprOpUpdate;

src/libexpr/primops.cc

@@ -508,7 +508,11 @@ static void prim_toPath(EvalState & state, Value * * args, Value & v)
 static void prim_storePath(EvalState & state, Value * * args, Value & v)
 {
     PathSet context;
-    Path path = canonPath(state.coerceToPath(*args[0], context));
+    Path path = state.coerceToPath(*args[0], context);
+    /* Resolve symlinks in ‘path’, unless ‘path’ itself is a symlink
+       directly in the store.  The latter condition is necessary so
+       e.g. nix-push does the right thing. */
+    if (!isStorePath(path)) path = canonPath(path, true);
     if (!isInStore(path))
         throw EvalError(format("path `%1%' is not in the Nix store") % path);
     Path path2 = toStorePath(path);

src/libstore/build.cc

@@ -777,6 +777,7 @@ private:
     /* File descriptor for the log file. */
     FILE * fLogFile;
     BZFILE * bzLogFile;
+    AutoCloseFD fdLogFile;
 
     /* Pipe for the builder's standard output/error. */
     Pipe builderOut;
@@ -1801,6 +1802,9 @@ void DerivationGoal::startBuilder()
          with outside processes using SysV IPC mechanisms (shared
          memory, message queues, semaphores).  It also ensures that
          all IPC objects are destroyed when the builder exits.
+
+       - The UTS namespace ensures that builders see a hostname of
+         localhost rather than the actual hostname.
     */
 #if CHROOT_ENABLED
     if (useChroot) {
@@ -2125,20 +2129,29 @@ Path DerivationGoal::openLogFile()
     Path dir = (format("%1%/%2%") % nixLogDir % drvsLogDir).str();
     createDirs(dir);
 
-    Path logFileName = (format("%1%/%2%.bz2") % dir % baseNameOf(drvPath)).str();
-    AutoCloseFD fd = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0666);
-    if (fd == -1)
-        throw SysError(format("creating log file `%1%'") % logFileName);
-    closeOnExec(fd);
+    if (queryBoolSetting("build-compress-log", true)) {
 
-    if (!(fLogFile = fdopen(fd.borrow(), "w")))
-        throw SysError(format("opening file `%1%'") % logFileName);
+        Path logFileName = (format("%1%/%2%.bz2") % dir % baseNameOf(drvPath)).str();
+        AutoCloseFD fd = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0666);
+        if (fd == -1) throw SysError(format("creating log file `%1%'") % logFileName);
+        closeOnExec(fd);
 
-    int err;
-    if (!(bzLogFile = BZ2_bzWriteOpen(&err, fLogFile, 9, 0, 0)))
-        throw Error(format("cannot open compressed log file `%1%'") % logFileName);
+        if (!(fLogFile = fdopen(fd.borrow(), "w")))
+            throw SysError(format("opening file `%1%'") % logFileName);
 
-    return logFileName;
+        int err;
+        if (!(bzLogFile = BZ2_bzWriteOpen(&err, fLogFile, 9, 0, 0)))
+            throw Error(format("cannot open compressed log file `%1%'") % logFileName);
+
+        return logFileName;
+
+    } else {
+        Path logFileName = (format("%1%/%2%") % dir % baseNameOf(drvPath)).str();
+        fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0666);
+        if (fdLogFile == -1) throw SysError(format("creating log file `%1%'") % logFileName);
+        closeOnExec(fdLogFile);
+        return logFileName;
+    }
 }
 
 
@@ -2155,6 +2168,8 @@ void DerivationGoal::closeLogFile()
         fclose(fLogFile);
         fLogFile = 0;
     }
+
+    fdLogFile.close();
 }
 
 
@@ -2186,7 +2201,8 @@ void DerivationGoal::handleChildOutput(int fd, const string & data)
             int err;
             BZ2_bzWrite(&err, bzLogFile, (unsigned char *) data.data(), data.size());
             if (err != BZ_OK) throw Error(format("cannot write to compressed log file (BZip2 error = %1%)") % err);
-        }
+        } else if (fdLogFile != -1)
+            writeFull(fdLogFile, (unsigned char *) data.data(), data.size());
     }
 
     if (hook && fd == hook->fromHook.readSide)
@@ -2224,6 +2240,7 @@ bool DerivationGoal::pathFailed(const Path & path)
     if (printBuildTrace)
         printMsg(lvlError, format("@ build-failed %1% %2% cached") % drvPath % path);
     
+    worker.permanentFailure = true;
     amDone(ecFailed);
 
     return true;

src/libstore/local-store.cc

@@ -405,6 +405,10 @@ void LocalStore::openDB(bool create)
         "select v.id, v.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where d.path = ?;");
     stmtQueryDerivationOutputs.create(db,
         "select id, path from DerivationOutputs where drv = ?;");
+    // Use "path >= ?" with limit 1 rather than "path like '?%'" to
+    // ensure efficient lookup.
+    stmtQueryPathFromHashPart.create(db,
+        "select path from ValidPaths where path >= ? limit 1;");
 }
 
 
@@ -874,6 +878,26 @@ StringSet LocalStore::queryDerivationOutputNames(const Path & path)
 }
 
 
+Path LocalStore::queryPathFromHashPart(const string & hashPart)
+{
+    if (hashPart.size() != 32) throw Error("invalid hash part");
+    
+    SQLiteTxn txn(db);
+
+    Path prefix = nixStore + "/" + hashPart;
+    
+    SQLiteStmtUse use(stmtQueryPathFromHashPart);
+    stmtQueryPathFromHashPart.bind(prefix);
+
+    int res = sqlite3_step(stmtQueryPathFromHashPart);
+    if (res == SQLITE_DONE) return "";
+    if (res != SQLITE_ROW) throwSQLiteError(db, "finding path in database");
+
+    const char * s = (const char *) sqlite3_column_text(stmtQueryPathFromHashPart, 0);
+    return s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0 ? s : "";
+}
+
+
 void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter & run)
 {
     if (run.pid != -1) return;

src/libstore/local-store.hh

@@ -123,6 +123,8 @@ public:
 
     StringSet queryDerivationOutputNames(const Path & path);
     
+    Path queryPathFromHashPart(const string & hashPart);
+    
     PathSet querySubstitutablePaths(const PathSet & paths);
 
     void querySubstitutablePathInfos(const Path & substituter,
@@ -217,6 +219,7 @@ private:
     SQLiteStmt stmtAddDerivationOutput;
     SQLiteStmt stmtQueryValidDerivers;
     SQLiteStmt stmtQueryDerivationOutputs;
+    SQLiteStmt stmtQueryPathFromHashPart;
 
     int getSchema();
 

src/libstore/remote-store.cc

@@ -395,6 +395,18 @@ PathSet RemoteStore::queryDerivationOutputNames(const Path & path)
 }
 
 
+Path RemoteStore::queryPathFromHashPart(const string & hashPart)
+{
+    openConnection();
+    writeInt(wopQueryPathFromHashPart, to);
+    writeString(hashPart, to);
+    processStderr();
+    Path path = readString(from);
+    if (!path.empty()) assertStorePath(path);
+    return path;
+}
+
+
 Path RemoteStore::addToStore(const Path & _srcPath,
     bool recursive, HashType hashAlgo, PathFilter & filter)
 {

src/libstore/remote-store.hh

@@ -45,6 +45,8 @@ public:
     
     StringSet queryDerivationOutputNames(const Path & path);
 
+    Path queryPathFromHashPart(const string & hashPart);
+    
     PathSet querySubstitutablePaths(const PathSet & paths);
     
     void querySubstitutablePathInfos(const PathSet & paths,

src/libstore/store-api.hh

@@ -144,6 +144,10 @@ public:
 
     /* Query the output names of the derivation denoted by `path'. */
     virtual StringSet queryDerivationOutputNames(const Path & path) = 0;
+
+    /* Query the full store path given the hash part of a valid store
+       path, or "" if the path doesn't exist. */
+    virtual Path queryPathFromHashPart(const string & hashPart) = 0;
     
     /* Query which of the given paths have substitutes. */
     virtual PathSet querySubstitutablePaths(const PathSet & paths) = 0;

src/libstore/worker-protocol.hh

@@ -40,9 +40,10 @@ typedef enum {
     wopQueryPathInfo = 26,
     wopImportPaths = 27,
     wopQueryDerivationOutputNames = 28,
-    wopQuerySubstitutablePathInfos = 29,
-    wopQueryValidPaths = 30,
-    wopQuerySubstitutablePaths = 31,
+    wopQueryPathFromHashPart = 29,
+    wopQuerySubstitutablePathInfos = 30,
+    wopQueryValidPaths = 31,
+    wopQuerySubstitutablePaths = 32,
 } WorkerOp;
 
 

src/nix-worker/nix-worker.cc

@@ -368,6 +368,15 @@ static void performOp(unsigned int clientVersion,
         break;
     }
 
+    case wopQueryPathFromHashPart: {
+        string hashPart = readString(from);
+        startWork();
+        Path path = store->queryPathFromHashPart(hashPart);
+        stopWork();
+        writeString(path, to);
+        break;
+    }
+
     case wopAddToStore: {
         string baseName = readString(from);
         bool fixed = readInt(from) == 1; /* obsolete */