* New primop builtins.filterSource, which can be used to filter files

from a source directory.  All files for which a predicate function
  returns true are copied to the store.  Typical example is to leave
  out the .svn directory:

    stdenv.mkDerivation {
      ...
      src = builtins.filterSource
        (path: baseNameOf (toString path) != ".svn")
        ./source-dir;
      # as opposed to
      #   src = ./source-dir;
    }

  This is important because the .svn directory influences the hash in
  a rather unpredictable and variable way.

doc/manual/release-notes.xml

@@ -12,7 +12,7 @@
 <itemizedlist>
 
 
-  <listitem><para>TODO: multi-user  support.</para></listitem>
+  <listitem><para>TODO: multi-user support.</para></listitem>
 
   
   <listitem><para><command>nix-store</command> has a new operation
@@ -50,6 +50,11 @@
   <option>--set</option>.</para></listitem>
   
 
+  <listitem><para>TODO: new built-ins
+  <function>builtins.attrNames</function>,
+  <function>builtins.filterSource</function>.</para></listitem>
+  
+
 </itemizedlist>
 
 </section>

src/libexpr/primops.cc

@@ -3,6 +3,7 @@
 #include "globals.hh"
 #include "store-api.hh"
 #include "util.hh"
+#include "archive.hh"
 #include "expr-to-xml.hh"
 #include "nixexpr-ast.hh"
 
@@ -726,6 +727,42 @@ static Expr primLessThan(EvalState & state, const ATermVector & args)
 }
 
 
+struct FilterFromExpr : PathFilter
+{
+    EvalState & state;
+    Expr filter;
+    
+    FilterFromExpr(EvalState & state, Expr filter)
+        : state(state), filter(filter)
+    {
+    }
+
+    bool operator () (const Path & path)
+    {
+        printMsg(lvlError, format("filter %1%") % path);
+        Expr call = makeCall(filter, makePath(toATerm(path)));
+        return evalBool(state, call);
+    }
+};
+
+
+static Expr primFilterSource(EvalState & state, const ATermVector & args)
+{
+    PathSet context;
+    Path path = coerceToPath(state, args[1], context);
+    if (!context.empty())
+        throw EvalError(format("string `%1%' cannot refer to other paths") % path);
+
+    FilterFromExpr filter(state, args[0]);
+
+    Path dstPath = readOnlyMode
+        ? computeStorePathForPath(path, false, false, "", filter).first
+        : store->addToStore(path, false, false, "", filter);
+
+    return makeStr(dstPath, singleton<PathSet>(dstPath));
+}
+
+
 void EvalState::addPrimOps()
 {
     addPrimOp("builtins", 0, primBuiltins);
@@ -762,6 +799,7 @@ void EvalState::addPrimOps()
     addPrimOp("__add", 2, primAdd);
     addPrimOp("__lessThan", 2, primLessThan);
     addPrimOp("__toFile", 2, primToFile);
+    addPrimOp("__filterSource", 2, primFilterSource);
 }
 
  

src/libstore/build.cc

@@ -949,7 +949,7 @@ void DerivationGoal::buildDone()
                as that means that someone else can have interfered
                with the build.  Also, the output should be owned by
                the build user. */
-            if ((st.st_mode & (S_IWGRP | S_IWOTH)) ||
+            if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) ||
                 (buildUser.enabled() && st.st_uid != buildUser.getUID()))
                 throw BuildError(format("suspicious ownership or permission on `%1%'; rejecting this build output") % path);
 #endif

src/libstore/local-store.cc

@@ -171,33 +171,7 @@ void createStoreTransaction(Transaction & txn)
 }
 
 
-/* Path copying. */
-
-struct CopySink : Sink
-{
-    string s;
-    virtual void operator () (const unsigned char * data, unsigned int len)
-    {
-        s.append((const char *) data, len);
-    }
-};
-
-
-struct CopySource : Source
-{
-    string & s;
-    unsigned int pos;
-    CopySource(string & _s) : s(_s), pos(0) { }
-    virtual void operator () (unsigned char * data, unsigned int len)
-    {
-        s.copy((char *) data, len, pos);
-        pos += len;
-        assert(pos <= s.size());
-    }
-};
-
-
-void copyPath(const Path & src, const Path & dst)
+void copyPath(const Path & src, const Path & dst, PathFilter & filter)
 {
     debug(format("copying `%1%' to `%2%'") % src % dst);
 
@@ -206,10 +180,10 @@ void copyPath(const Path & src, const Path & dst)
        for very large paths, but `copyPath' is mainly used for small
        files. */ 
 
-    CopySink sink;
-    dumpPath(src, sink);
+    StringSink sink;
+    dumpPath(src, sink, filter);
 
-    CopySource source(sink.s);
+    StringSource source(sink.s);
     restorePath(dst, source);
 }
 
@@ -646,13 +620,13 @@ static void invalidatePath(Transaction & txn, const Path & path)
 
 
 Path LocalStore::addToStore(const Path & _srcPath, bool fixed,
-    bool recursive, string hashAlgo)
+    bool recursive, string hashAlgo, PathFilter & filter)
 {
     Path srcPath(absPath(_srcPath));
     debug(format("adding `%1%' to the store") % srcPath);
 
     std::pair<Path, Hash> pr =
-        computeStorePathForPath(srcPath, fixed, recursive, hashAlgo);
+        computeStorePathForPath(srcPath, fixed, recursive, hashAlgo, filter);
     Path & dstPath(pr.first);
     Hash & h(pr.second);
 
@@ -669,9 +643,9 @@ Path LocalStore::addToStore(const Path & _srcPath, bool fixed,
 
             if (pathExists(dstPath)) deletePathWrapped(dstPath);
 
-            copyPath(srcPath, dstPath);
+            copyPath(srcPath, dstPath, filter);
 
-            Hash h2 = hashPath(htSHA256, dstPath);
+            Hash h2 = hashPath(htSHA256, dstPath, filter);
             if (h != h2)
                 throw Error(format("contents of `%1%' changed while copying it to `%2%' (%3% -> %4%)")
                     % srcPath % dstPath % printHash(h) % printHash(h2));

src/libstore/local-store.hh

@@ -50,7 +50,8 @@ public:
     void queryReferrers(const Path & path, PathSet & referrers);
 
     Path addToStore(const Path & srcPath, bool fixed = false,
-        bool recursive = false, string hashAlgo = "");
+        bool recursive = false, string hashAlgo = "",
+        PathFilter & filter = defaultPathFilter);
 
     Path addTextToStore(const string & suffix, const string & s,
         const PathSet & references);

src/libstore/remote-store.cc

@@ -213,7 +213,7 @@ void RemoteStore::queryReferrers(const Path & path,
 
 
 Path RemoteStore::addToStore(const Path & _srcPath, bool fixed,
-    bool recursive, string hashAlgo)
+    bool recursive, string hashAlgo, PathFilter & filter)
 {
     Path srcPath(absPath(_srcPath));
     
@@ -222,7 +222,7 @@ Path RemoteStore::addToStore(const Path & _srcPath, bool fixed,
     writeInt(fixed ? 1 : 0, to);
     writeInt(recursive ? 1 : 0, to);
     writeString(hashAlgo, to);
-    dumpPath(srcPath, to);
+    dumpPath(srcPath, to, filter);
     processStderr();
     Path path = readStorePath(from);
     return path;

src/libstore/remote-store.hh

@@ -38,7 +38,8 @@ public:
     void queryReferrers(const Path & path, PathSet & referrers);
 
     Path addToStore(const Path & srcPath, bool fixed = false,
-        bool recursive = false, string hashAlgo = "");
+        bool recursive = false, string hashAlgo = "",
+        PathFilter & filter = defaultPathFilter);
 
     Path addTextToStore(const string & suffix, const string & s,
         const PathSet & references);

src/libstore/store-api.cc

@@ -94,9 +94,9 @@ Path makeFixedOutputPath(bool recursive,
 
 
 std::pair<Path, Hash> computeStorePathForPath(const Path & srcPath,
-    bool fixed, bool recursive, string hashAlgo)
+    bool fixed, bool recursive, string hashAlgo, PathFilter & filter)
 {
-    Hash h = hashPath(htSHA256, srcPath);
+    Hash h = hashPath(htSHA256, srcPath, filter);
 
     string baseName = baseNameOf(srcPath);
 
@@ -104,7 +104,7 @@ std::pair<Path, Hash> computeStorePathForPath(const Path & srcPath,
     
     if (fixed) {
         HashType ht(parseHashType(hashAlgo));
-        Hash h2 = recursive ? hashPath(ht, srcPath) : hashFile(ht, srcPath);
+        Hash h2 = recursive ? hashPath(ht, srcPath, filter) : hashFile(ht, srcPath);
         dstPath = makeFixedOutputPath(recursive, hashAlgo, h2, baseName);
     }
         

src/libstore/store-api.hh

@@ -79,9 +79,12 @@ public:
     /* Copy the contents of a path to the store and register the
        validity the resulting path.  The resulting path is returned.
        If `fixed' is true, then the output of a fixed-output
-       derivation is pre-loaded into the Nix store. */
+       derivation is pre-loaded into the Nix store.  The function
+       object `filter' can be used to exclude files (see
+       libutil/archive.hh). */
     virtual Path addToStore(const Path & srcPath, bool fixed = false,
-        bool recursive = false, string hashAlgo = "") = 0;
+        bool recursive = false, string hashAlgo = "",
+        PathFilter & filter = defaultPathFilter) = 0;
 
     /* Like addToStore, but the contents written to the output path is
        a regular file containing the given string. */
@@ -195,7 +198,8 @@ Path makeFixedOutputPath(bool recursive,
    Returns the store path and the cryptographic hash of the
    contents of srcPath. */
 std::pair<Path, Hash> computeStorePathForPath(const Path & srcPath,
-    bool fixed = false, bool recursive = false, string hashAlgo = "");
+    bool fixed = false, bool recursive = false, string hashAlgo = "",
+    PathFilter & filter = defaultPathFilter);
 
 /* Preparatory part of addTextToStore().
 

src/libutil/archive.cc

@@ -18,28 +18,29 @@ namespace nix {
 static string archiveVersion1 = "nix-archive-1";
 
 
-DumpFilter defaultDumpFilter;
+PathFilter defaultPathFilter;
 
 
-static void dump(const string & path, Sink & sink, DumpFilter & filter);
+static void dump(const string & path, Sink & sink, PathFilter & filter);
 
 
-static void dumpEntries(const Path & path, Sink & sink, DumpFilter & filter)
+static void dumpEntries(const Path & path, Sink & sink, PathFilter & filter)
 {
     Strings names = readDirectory(path);
     vector<string> names2(names.begin(), names.end());
     sort(names2.begin(), names2.end());
 
-    for (vector<string>::iterator it = names2.begin();
-         it != names2.end(); it++)
+    for (vector<string>::iterator i = names2.begin();
+         i != names2.end(); ++i)
     {
-        if (filter(path)) {
+        Path entry = path + "/" + *i;
+        if (filter(entry)) {
             writeString("entry", sink);
             writeString("(", sink);
             writeString("name", sink);
-            writeString(*it, sink);
+            writeString(*i, sink);
             writeString("node", sink);
-            dump(path + "/" + *it, sink, filter);
+            dump(entry, sink, filter);
             writeString(")", sink);
         }
     }
@@ -69,7 +70,7 @@ static void dumpContents(const Path & path, unsigned int size,
 }
 
 
-static void dump(const Path & path, Sink & sink, DumpFilter & filter)
+static void dump(const Path & path, Sink & sink, PathFilter & filter)
 {
     struct stat st;
     if (lstat(path.c_str(), &st))
@@ -106,7 +107,7 @@ static void dump(const Path & path, Sink & sink, DumpFilter & filter)
 }
 
 
-void dumpPath(const Path & path, Sink & sink, DumpFilter & filter)
+void dumpPath(const Path & path, Sink & sink, PathFilter & filter)
 {
     writeString(archiveVersion1, sink);
     dump(path, sink, filter);

src/libutil/archive.hh

@@ -45,16 +45,16 @@ namespace nix {
 
      `+' denotes string concatenation. */
 
-struct DumpFilter
+struct PathFilter
 {
-    virtual ~DumpFilter() { }
+    virtual ~PathFilter() { }
     virtual bool operator () (const Path & path) { return true; }
 };
 
-extern DumpFilter defaultDumpFilter;
+extern PathFilter defaultPathFilter;
 
 void dumpPath(const Path & path, Sink & sink,
-    DumpFilter & filter = defaultDumpFilter);
+    PathFilter & filter = defaultPathFilter);
 
 void restorePath(const Path & path, Source & source);
 

src/libutil/hash.cc

@@ -294,13 +294,13 @@ struct HashSink : Sink
 };
 
 
-Hash hashPath(HashType ht, const Path & path)
+Hash hashPath(HashType ht, const Path & path, PathFilter & filter)
 {
     HashSink sink;
     sink.ht = ht;
     Hash hash(ht);
     start(ht, sink.ctx);
-    dumpPath(path, sink);
+    dumpPath(path, sink, filter);
     finish(ht, sink.ctx, hash.hash);
     return hash;
 }

src/libutil/hash.hh

@@ -69,7 +69,10 @@ Hash hashFile(HashType ht, const Path & path);
 
 /* Compute the hash of the given path.  The hash is defined as
    (essentially) hashString(ht, dumpPath(path)). */
-Hash hashPath(HashType ht, const Path & path);
+struct PathFilter;
+extern PathFilter defaultPathFilter;
+Hash hashPath(HashType ht, const Path & path,
+    PathFilter & filter = defaultPathFilter);
 
 /* Compress a hash to the specified number of bytes by cyclically
    XORing bytes together. */

src/libutil/serialise.hh

@@ -66,6 +66,33 @@ struct FdSource : Source
 };
 
 
+/* A sink that writes data to a string. */
+struct StringSink : Sink
+{
+    string s;
+    virtual void operator () (const unsigned char * data, unsigned int len)
+    {
+        s.append((const char *) data, len);
+    }
+};
+
+
+/* A source that reads data from a string. */
+struct StringSource : Source
+{
+    string & s;
+    unsigned int pos;
+    StringSource(string & _s) : s(_s), pos(0) { }
+    virtual void operator () (unsigned char * data, unsigned int len)
+    {
+        s.copy((char *) data, len, pos);
+        pos += len;
+        if (pos > s.size())
+            throw Error("end of string reached");
+    }
+};
+
+
 void writePadding(unsigned int len, Sink & sink);
 void writeInt(unsigned int n, Sink & sink);
 void writeString(const string & s, Sink & sink);

src/libutil/util.hh

@@ -164,10 +164,10 @@ struct AutoDeleteArray
 
 class AutoDelete
 {
-    string path;
+    Path path;
     bool del;
 public:
-    AutoDelete(const string & p);
+    AutoDelete(const Path & p);
     ~AutoDelete();
     void cancel();
 };

src/nix-worker/nix-worker.cc

@@ -240,6 +240,7 @@ static void performOp(Source & from, Sink & to, unsigned int op)
         string hashAlgo = readString(from);
         
         Path tmp = createTempDir();
+        AutoDelete delTmp(tmp);
         Path tmp2 = tmp + "/" + baseName;
         restorePath(tmp2, from);
 
@@ -248,8 +249,6 @@ static void performOp(Source & from, Sink & to, unsigned int op)
         stopWork();
         
         writeString(path, to);
-            
-        deletePath(tmp);
         break;
     }
 

tests/Makefile.am

@@ -15,12 +15,13 @@ user-envs.sh: user-envs.nix
 fixed.sh: fixed.nix
 gc-runtime.sh: gc-runtime.nix
 check-refs.sh: check-refs.nix
+filter-source.sh: filter-source.nix
 
 TESTS = init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \
   locking.sh parallel.sh build-hook.sh substitutes.sh substitutes2.sh \
   fallback.sh nix-push.sh gc.sh gc-concurrent.sh verify.sh nix-pull.sh \
   referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \
-  gc-runtime.sh install-package.sh check-refs.sh
+  gc-runtime.sh install-package.sh check-refs.sh filter-source.sh
 
 XFAIL_TESTS =
 
@@ -44,5 +45,6 @@ EXTRA_DIST = $(TESTS) \
   fixed.nix.in fixed.builder1.sh fixed.builder2.sh \
   gc-runtime.nix.in gc-runtime.builder.sh \
   check-refs.nix.in \
+  filter-source.nix.in \
   $(wildcard lang/*.nix) $(wildcard lang/*.exp) $(wildcard lang/*.exp.xml) \
   common.sh.in

tests/filter-source.nix.in

@@ -0,0 +1,7 @@
+derivation {
+  name = "filter";
+  system = "@system@";
+  builder = "@shell@";
+  args = ["-e" "-x" (builtins.toFile "builder" "PATH=@testPath@; ln -s $input $out")];
+  input = builtins.filterSource (path: baseNameOf (toString path) != "foo") ./test-tmp/filterin;
+}

tests/filter-source.sh

@@ -0,0 +1,13 @@
+source common.sh
+
+rm -rf $TEST_ROOT/filterin
+mkdir $TEST_ROOT/filterin
+mkdir $TEST_ROOT/filterin/foo
+touch $TEST_ROOT/filterin/foo/bar
+touch $TEST_ROOT/filterin/xyzzy
+
+$NIX_BIN_DIR/nix-build ./filter-source.nix -o $TEST_ROOT/filterout
+
+set -x
+test ! -e $TEST_ROOT/filterout/foo/bar
+test -e $TEST_ROOT/filterout/xyzzy