* New primop builtins.filterSource, which can be used to filter files

from a source directory.  All files for which a predicate function
  returns true are copied to the store.  Typical example is to leave
  out the .svn directory:

    stdenv.mkDerivation {
      ...
      src = builtins.filterSource
        (path: baseNameOf (toString path) != ".svn")
        ./source-dir;
      # as opposed to
      #   src = ./source-dir;
    }

  This is important because the .svn directory influences the hash in
  a rather unpredictable and variable way.
This commit is contained in:
Eelco Dolstra 2006-12-12 23:05:01 +00:00
parent b438d37558
commit a3e6415ba8
19 changed files with 143 additions and 68 deletions

View file

@ -12,7 +12,7 @@
<itemizedlist> <itemizedlist>
<listitem><para>TODO: multi-user support.</para></listitem> <listitem><para>TODO: multi-user support.</para></listitem>
<listitem><para><command>nix-store</command> has a new operation <listitem><para><command>nix-store</command> has a new operation
@ -50,6 +50,11 @@
<option>--set</option>.</para></listitem> <option>--set</option>.</para></listitem>
<listitem><para>TODO: new built-ins
<function>builtins.attrNames</function>,
<function>builtins.filterSource</function>.</para></listitem>
</itemizedlist> </itemizedlist>
</section> </section>

View file

@ -3,6 +3,7 @@
#include "globals.hh" #include "globals.hh"
#include "store-api.hh" #include "store-api.hh"
#include "util.hh" #include "util.hh"
#include "archive.hh"
#include "expr-to-xml.hh" #include "expr-to-xml.hh"
#include "nixexpr-ast.hh" #include "nixexpr-ast.hh"
@ -726,6 +727,42 @@ static Expr primLessThan(EvalState & state, const ATermVector & args)
} }
struct FilterFromExpr : PathFilter
{
EvalState & state;
Expr filter;
FilterFromExpr(EvalState & state, Expr filter)
: state(state), filter(filter)
{
}
bool operator () (const Path & path)
{
printMsg(lvlError, format("filter %1%") % path);
Expr call = makeCall(filter, makePath(toATerm(path)));
return evalBool(state, call);
}
};
static Expr primFilterSource(EvalState & state, const ATermVector & args)
{
PathSet context;
Path path = coerceToPath(state, args[1], context);
if (!context.empty())
throw EvalError(format("string `%1%' cannot refer to other paths") % path);
FilterFromExpr filter(state, args[0]);
Path dstPath = readOnlyMode
? computeStorePathForPath(path, false, false, "", filter).first
: store->addToStore(path, false, false, "", filter);
return makeStr(dstPath, singleton<PathSet>(dstPath));
}
void EvalState::addPrimOps() void EvalState::addPrimOps()
{ {
addPrimOp("builtins", 0, primBuiltins); addPrimOp("builtins", 0, primBuiltins);
@ -762,6 +799,7 @@ void EvalState::addPrimOps()
addPrimOp("__add", 2, primAdd); addPrimOp("__add", 2, primAdd);
addPrimOp("__lessThan", 2, primLessThan); addPrimOp("__lessThan", 2, primLessThan);
addPrimOp("__toFile", 2, primToFile); addPrimOp("__toFile", 2, primToFile);
addPrimOp("__filterSource", 2, primFilterSource);
} }

View file

@ -949,7 +949,7 @@ void DerivationGoal::buildDone()
as that means that someone else can have interfered as that means that someone else can have interfered
with the build. Also, the output should be owned by with the build. Also, the output should be owned by
the build user. */ the build user. */
if ((st.st_mode & (S_IWGRP | S_IWOTH)) || if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) ||
(buildUser.enabled() && st.st_uid != buildUser.getUID())) (buildUser.enabled() && st.st_uid != buildUser.getUID()))
throw BuildError(format("suspicious ownership or permission on `%1%'; rejecting this build output") % path); throw BuildError(format("suspicious ownership or permission on `%1%'; rejecting this build output") % path);
#endif #endif

View file

@ -171,33 +171,7 @@ void createStoreTransaction(Transaction & txn)
} }
/* Path copying. */ void copyPath(const Path & src, const Path & dst, PathFilter & filter)
struct CopySink : Sink
{
string s;
virtual void operator () (const unsigned char * data, unsigned int len)
{
s.append((const char *) data, len);
}
};
struct CopySource : Source
{
string & s;
unsigned int pos;
CopySource(string & _s) : s(_s), pos(0) { }
virtual void operator () (unsigned char * data, unsigned int len)
{
s.copy((char *) data, len, pos);
pos += len;
assert(pos <= s.size());
}
};
void copyPath(const Path & src, const Path & dst)
{ {
debug(format("copying `%1%' to `%2%'") % src % dst); debug(format("copying `%1%' to `%2%'") % src % dst);
@ -206,10 +180,10 @@ void copyPath(const Path & src, const Path & dst)
for very large paths, but `copyPath' is mainly used for small for very large paths, but `copyPath' is mainly used for small
files. */ files. */
CopySink sink; StringSink sink;
dumpPath(src, sink); dumpPath(src, sink, filter);
CopySource source(sink.s); StringSource source(sink.s);
restorePath(dst, source); restorePath(dst, source);
} }
@ -646,13 +620,13 @@ static void invalidatePath(Transaction & txn, const Path & path)
Path LocalStore::addToStore(const Path & _srcPath, bool fixed, Path LocalStore::addToStore(const Path & _srcPath, bool fixed,
bool recursive, string hashAlgo) bool recursive, string hashAlgo, PathFilter & filter)
{ {
Path srcPath(absPath(_srcPath)); Path srcPath(absPath(_srcPath));
debug(format("adding `%1%' to the store") % srcPath); debug(format("adding `%1%' to the store") % srcPath);
std::pair<Path, Hash> pr = std::pair<Path, Hash> pr =
computeStorePathForPath(srcPath, fixed, recursive, hashAlgo); computeStorePathForPath(srcPath, fixed, recursive, hashAlgo, filter);
Path & dstPath(pr.first); Path & dstPath(pr.first);
Hash & h(pr.second); Hash & h(pr.second);
@ -669,9 +643,9 @@ Path LocalStore::addToStore(const Path & _srcPath, bool fixed,
if (pathExists(dstPath)) deletePathWrapped(dstPath); if (pathExists(dstPath)) deletePathWrapped(dstPath);
copyPath(srcPath, dstPath); copyPath(srcPath, dstPath, filter);
Hash h2 = hashPath(htSHA256, dstPath); Hash h2 = hashPath(htSHA256, dstPath, filter);
if (h != h2) if (h != h2)
throw Error(format("contents of `%1%' changed while copying it to `%2%' (%3% -> %4%)") throw Error(format("contents of `%1%' changed while copying it to `%2%' (%3% -> %4%)")
% srcPath % dstPath % printHash(h) % printHash(h2)); % srcPath % dstPath % printHash(h) % printHash(h2));

View file

@ -50,7 +50,8 @@ public:
void queryReferrers(const Path & path, PathSet & referrers); void queryReferrers(const Path & path, PathSet & referrers);
Path addToStore(const Path & srcPath, bool fixed = false, Path addToStore(const Path & srcPath, bool fixed = false,
bool recursive = false, string hashAlgo = ""); bool recursive = false, string hashAlgo = "",
PathFilter & filter = defaultPathFilter);
Path addTextToStore(const string & suffix, const string & s, Path addTextToStore(const string & suffix, const string & s,
const PathSet & references); const PathSet & references);

View file

@ -213,7 +213,7 @@ void RemoteStore::queryReferrers(const Path & path,
Path RemoteStore::addToStore(const Path & _srcPath, bool fixed, Path RemoteStore::addToStore(const Path & _srcPath, bool fixed,
bool recursive, string hashAlgo) bool recursive, string hashAlgo, PathFilter & filter)
{ {
Path srcPath(absPath(_srcPath)); Path srcPath(absPath(_srcPath));
@ -222,7 +222,7 @@ Path RemoteStore::addToStore(const Path & _srcPath, bool fixed,
writeInt(fixed ? 1 : 0, to); writeInt(fixed ? 1 : 0, to);
writeInt(recursive ? 1 : 0, to); writeInt(recursive ? 1 : 0, to);
writeString(hashAlgo, to); writeString(hashAlgo, to);
dumpPath(srcPath, to); dumpPath(srcPath, to, filter);
processStderr(); processStderr();
Path path = readStorePath(from); Path path = readStorePath(from);
return path; return path;

View file

@ -38,7 +38,8 @@ public:
void queryReferrers(const Path & path, PathSet & referrers); void queryReferrers(const Path & path, PathSet & referrers);
Path addToStore(const Path & srcPath, bool fixed = false, Path addToStore(const Path & srcPath, bool fixed = false,
bool recursive = false, string hashAlgo = ""); bool recursive = false, string hashAlgo = "",
PathFilter & filter = defaultPathFilter);
Path addTextToStore(const string & suffix, const string & s, Path addTextToStore(const string & suffix, const string & s,
const PathSet & references); const PathSet & references);

View file

@ -94,9 +94,9 @@ Path makeFixedOutputPath(bool recursive,
std::pair<Path, Hash> computeStorePathForPath(const Path & srcPath, std::pair<Path, Hash> computeStorePathForPath(const Path & srcPath,
bool fixed, bool recursive, string hashAlgo) bool fixed, bool recursive, string hashAlgo, PathFilter & filter)
{ {
Hash h = hashPath(htSHA256, srcPath); Hash h = hashPath(htSHA256, srcPath, filter);
string baseName = baseNameOf(srcPath); string baseName = baseNameOf(srcPath);
@ -104,7 +104,7 @@ std::pair<Path, Hash> computeStorePathForPath(const Path & srcPath,
if (fixed) { if (fixed) {
HashType ht(parseHashType(hashAlgo)); HashType ht(parseHashType(hashAlgo));
Hash h2 = recursive ? hashPath(ht, srcPath) : hashFile(ht, srcPath); Hash h2 = recursive ? hashPath(ht, srcPath, filter) : hashFile(ht, srcPath);
dstPath = makeFixedOutputPath(recursive, hashAlgo, h2, baseName); dstPath = makeFixedOutputPath(recursive, hashAlgo, h2, baseName);
} }

View file

@ -79,9 +79,12 @@ public:
/* Copy the contents of a path to the store and register the /* Copy the contents of a path to the store and register the
validity the resulting path. The resulting path is returned. validity the resulting path. The resulting path is returned.
If `fixed' is true, then the output of a fixed-output If `fixed' is true, then the output of a fixed-output
derivation is pre-loaded into the Nix store. */ derivation is pre-loaded into the Nix store. The function
object `filter' can be used to exclude files (see
libutil/archive.hh). */
virtual Path addToStore(const Path & srcPath, bool fixed = false, virtual Path addToStore(const Path & srcPath, bool fixed = false,
bool recursive = false, string hashAlgo = "") = 0; bool recursive = false, string hashAlgo = "",
PathFilter & filter = defaultPathFilter) = 0;
/* Like addToStore, but the contents written to the output path is /* Like addToStore, but the contents written to the output path is
a regular file containing the given string. */ a regular file containing the given string. */
@ -195,7 +198,8 @@ Path makeFixedOutputPath(bool recursive,
Returns the store path and the cryptographic hash of the Returns the store path and the cryptographic hash of the
contents of srcPath. */ contents of srcPath. */
std::pair<Path, Hash> computeStorePathForPath(const Path & srcPath, std::pair<Path, Hash> computeStorePathForPath(const Path & srcPath,
bool fixed = false, bool recursive = false, string hashAlgo = ""); bool fixed = false, bool recursive = false, string hashAlgo = "",
PathFilter & filter = defaultPathFilter);
/* Preparatory part of addTextToStore(). /* Preparatory part of addTextToStore().

View file

@ -18,28 +18,29 @@ namespace nix {
static string archiveVersion1 = "nix-archive-1"; static string archiveVersion1 = "nix-archive-1";
DumpFilter defaultDumpFilter; PathFilter defaultPathFilter;
static void dump(const string & path, Sink & sink, DumpFilter & filter); static void dump(const string & path, Sink & sink, PathFilter & filter);
static void dumpEntries(const Path & path, Sink & sink, DumpFilter & filter) static void dumpEntries(const Path & path, Sink & sink, PathFilter & filter)
{ {
Strings names = readDirectory(path); Strings names = readDirectory(path);
vector<string> names2(names.begin(), names.end()); vector<string> names2(names.begin(), names.end());
sort(names2.begin(), names2.end()); sort(names2.begin(), names2.end());
for (vector<string>::iterator it = names2.begin(); for (vector<string>::iterator i = names2.begin();
it != names2.end(); it++) i != names2.end(); ++i)
{ {
if (filter(path)) { Path entry = path + "/" + *i;
if (filter(entry)) {
writeString("entry", sink); writeString("entry", sink);
writeString("(", sink); writeString("(", sink);
writeString("name", sink); writeString("name", sink);
writeString(*it, sink); writeString(*i, sink);
writeString("node", sink); writeString("node", sink);
dump(path + "/" + *it, sink, filter); dump(entry, sink, filter);
writeString(")", sink); writeString(")", sink);
} }
} }
@ -69,7 +70,7 @@ static void dumpContents(const Path & path, unsigned int size,
} }
static void dump(const Path & path, Sink & sink, DumpFilter & filter) static void dump(const Path & path, Sink & sink, PathFilter & filter)
{ {
struct stat st; struct stat st;
if (lstat(path.c_str(), &st)) if (lstat(path.c_str(), &st))
@ -106,7 +107,7 @@ static void dump(const Path & path, Sink & sink, DumpFilter & filter)
} }
void dumpPath(const Path & path, Sink & sink, DumpFilter & filter) void dumpPath(const Path & path, Sink & sink, PathFilter & filter)
{ {
writeString(archiveVersion1, sink); writeString(archiveVersion1, sink);
dump(path, sink, filter); dump(path, sink, filter);

View file

@ -45,16 +45,16 @@ namespace nix {
`+' denotes string concatenation. */ `+' denotes string concatenation. */
struct DumpFilter struct PathFilter
{ {
virtual ~DumpFilter() { } virtual ~PathFilter() { }
virtual bool operator () (const Path & path) { return true; } virtual bool operator () (const Path & path) { return true; }
}; };
extern DumpFilter defaultDumpFilter; extern PathFilter defaultPathFilter;
void dumpPath(const Path & path, Sink & sink, void dumpPath(const Path & path, Sink & sink,
DumpFilter & filter = defaultDumpFilter); PathFilter & filter = defaultPathFilter);
void restorePath(const Path & path, Source & source); void restorePath(const Path & path, Source & source);

View file

@ -294,13 +294,13 @@ struct HashSink : Sink
}; };
Hash hashPath(HashType ht, const Path & path) Hash hashPath(HashType ht, const Path & path, PathFilter & filter)
{ {
HashSink sink; HashSink sink;
sink.ht = ht; sink.ht = ht;
Hash hash(ht); Hash hash(ht);
start(ht, sink.ctx); start(ht, sink.ctx);
dumpPath(path, sink); dumpPath(path, sink, filter);
finish(ht, sink.ctx, hash.hash); finish(ht, sink.ctx, hash.hash);
return hash; return hash;
} }

View file

@ -69,7 +69,10 @@ Hash hashFile(HashType ht, const Path & path);
/* Compute the hash of the given path. The hash is defined as /* Compute the hash of the given path. The hash is defined as
(essentially) hashString(ht, dumpPath(path)). */ (essentially) hashString(ht, dumpPath(path)). */
Hash hashPath(HashType ht, const Path & path); struct PathFilter;
extern PathFilter defaultPathFilter;
Hash hashPath(HashType ht, const Path & path,
PathFilter & filter = defaultPathFilter);
/* Compress a hash to the specified number of bytes by cyclically /* Compress a hash to the specified number of bytes by cyclically
XORing bytes together. */ XORing bytes together. */

View file

@ -66,6 +66,33 @@ struct FdSource : Source
}; };
/* A sink that writes data to a string. */
struct StringSink : Sink
{
string s;
virtual void operator () (const unsigned char * data, unsigned int len)
{
s.append((const char *) data, len);
}
};
/* A source that reads data from a string. */
struct StringSource : Source
{
string & s;
unsigned int pos;
StringSource(string & _s) : s(_s), pos(0) { }
virtual void operator () (unsigned char * data, unsigned int len)
{
s.copy((char *) data, len, pos);
pos += len;
if (pos > s.size())
throw Error("end of string reached");
}
};
void writePadding(unsigned int len, Sink & sink); void writePadding(unsigned int len, Sink & sink);
void writeInt(unsigned int n, Sink & sink); void writeInt(unsigned int n, Sink & sink);
void writeString(const string & s, Sink & sink); void writeString(const string & s, Sink & sink);

View file

@ -164,10 +164,10 @@ struct AutoDeleteArray
class AutoDelete class AutoDelete
{ {
string path; Path path;
bool del; bool del;
public: public:
AutoDelete(const string & p); AutoDelete(const Path & p);
~AutoDelete(); ~AutoDelete();
void cancel(); void cancel();
}; };

View file

@ -240,6 +240,7 @@ static void performOp(Source & from, Sink & to, unsigned int op)
string hashAlgo = readString(from); string hashAlgo = readString(from);
Path tmp = createTempDir(); Path tmp = createTempDir();
AutoDelete delTmp(tmp);
Path tmp2 = tmp + "/" + baseName; Path tmp2 = tmp + "/" + baseName;
restorePath(tmp2, from); restorePath(tmp2, from);
@ -248,8 +249,6 @@ static void performOp(Source & from, Sink & to, unsigned int op)
stopWork(); stopWork();
writeString(path, to); writeString(path, to);
deletePath(tmp);
break; break;
} }

View file

@ -15,12 +15,13 @@ user-envs.sh: user-envs.nix
fixed.sh: fixed.nix fixed.sh: fixed.nix
gc-runtime.sh: gc-runtime.nix gc-runtime.sh: gc-runtime.nix
check-refs.sh: check-refs.nix check-refs.sh: check-refs.nix
filter-source.sh: filter-source.nix
TESTS = init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \ TESTS = init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \
locking.sh parallel.sh build-hook.sh substitutes.sh substitutes2.sh \ locking.sh parallel.sh build-hook.sh substitutes.sh substitutes2.sh \
fallback.sh nix-push.sh gc.sh gc-concurrent.sh verify.sh nix-pull.sh \ fallback.sh nix-push.sh gc.sh gc-concurrent.sh verify.sh nix-pull.sh \
referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \ referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \
gc-runtime.sh install-package.sh check-refs.sh gc-runtime.sh install-package.sh check-refs.sh filter-source.sh
XFAIL_TESTS = XFAIL_TESTS =
@ -44,5 +45,6 @@ EXTRA_DIST = $(TESTS) \
fixed.nix.in fixed.builder1.sh fixed.builder2.sh \ fixed.nix.in fixed.builder1.sh fixed.builder2.sh \
gc-runtime.nix.in gc-runtime.builder.sh \ gc-runtime.nix.in gc-runtime.builder.sh \
check-refs.nix.in \ check-refs.nix.in \
filter-source.nix.in \
$(wildcard lang/*.nix) $(wildcard lang/*.exp) $(wildcard lang/*.exp.xml) \ $(wildcard lang/*.nix) $(wildcard lang/*.exp) $(wildcard lang/*.exp.xml) \
common.sh.in common.sh.in

View file

@ -0,0 +1,7 @@
derivation {
name = "filter";
system = "@system@";
builder = "@shell@";
args = ["-e" "-x" (builtins.toFile "builder" "PATH=@testPath@; ln -s $input $out")];
input = builtins.filterSource (path: baseNameOf (toString path) != "foo") ./test-tmp/filterin;
}

13
tests/filter-source.sh Normal file
View file

@ -0,0 +1,13 @@
source common.sh
rm -rf $TEST_ROOT/filterin
mkdir $TEST_ROOT/filterin
mkdir $TEST_ROOT/filterin/foo
touch $TEST_ROOT/filterin/foo/bar
touch $TEST_ROOT/filterin/xyzzy
$NIX_BIN_DIR/nix-build ./filter-source.nix -o $TEST_ROOT/filterout
set -x
test ! -e $TEST_ROOT/filterout/foo/bar
test -e $TEST_ROOT/filterout/xyzzy