* Basic work on allowing derive expressions to build multiple paths.

This is not entirely trivial since this introduces the possibility
  of mutual recursion.
* Made normal forms self-contained.
* Use unique ids, not content hashes, for content referencing.
This commit is contained in:
Eelco Dolstra 2003-07-15 16:28:54 +00:00
parent 8898e86b4f
commit f5b6fa5256
7 changed files with 367 additions and 194 deletions

View file

@ -154,55 +154,47 @@ FState hash2fstate(Hash hash)
}
ATerm termFromHash(const Hash & hash, string * p)
ATerm termFromId(const FSId & id, string * p)
{
string path = expandHash(hash);
string path = expandId(id);
if (p) *p = path;
ATerm t = ATreadFromNamedFile(path.c_str());
if (!t) throw Error(format("cannot read aterm %1%") % path);
if (!t) throw Error(format("cannot read aterm from `%1%'") % path);
return t;
}
Hash writeTerm(ATerm t, const string & suffix, string * p)
FSId writeTerm(ATerm t, const string & suffix, string * p)
{
string path = nixStore + "/tmp.nix"; /* !!! */
FSId id = hashTerm(t);
string path = canonPath(nixStore + "/" +
(string) id + suffix + ".nix");
if (!ATwriteToNamedTextFile(t, path.c_str()))
throw Error(format("cannot write aterm %1%") % path);
Hash hash = hashPath(path);
string path2 = canonPath(nixStore + "/" +
(string) hash + suffix + ".nix");
if (rename(path.c_str(), path2.c_str()) == -1)
throw SysError(format("renaming %1% to %2%") % path % path2);
registerPath(path2, hash);
if (p) *p = path2;
return hash;
registerPath(path, id);
if (p) *p = path;
return id;
}
void registerSuccessor(const Hash & fsHash, const Hash & scHash)
void registerSuccessor(const FSId & id1, const FSId & id2)
{
setDB(nixDB, dbSuccessors, fsHash, scHash);
setDB(nixDB, dbSuccessors, id1, id2);
}
FState storeSuccessor(FState fs, FState sc, StringSet & paths)
static FSId storeSuccessor(const FSId & id1, FState sc)
{
if (fs == sc) return sc;
FSId id2 = writeTerm(sc, "-s-" + (string) id1, 0);
registerSuccessor(id1, id2);
return id2;
}
string path;
Hash fsHash = hashTerm(fs);
Hash scHash = writeTerm(sc, "-s-" + (string) fsHash, &path);
registerSuccessor(fsHash, scHash);
paths.insert(path);
#if 0
return ATmake("Include(<str>)", ((string) scHash).c_str());
#endif
return sc;
}
static FState realise(FState fs, StringSet & paths)
{
char * s1, * s2, * s3;
@ -421,3 +413,193 @@ void fstateRefs(FState fs, StringSet & paths)
{
fstateRefs2(fs, paths);
}
#endif
static void parseIds(ATermList ids, FSIds & out)
{
while (!ATisEmpty(ids)) {
char * s;
ATerm id = ATgetFirst(ids);
if (!ATmatch(id, "<str>", &s))
throw badTerm("not an id", id);
out.push_back(parseHash(s));
debug(s);
ids = ATgetNext(ids);
}
}
/* Parse a slice. */
static Slice parseSlice(FState fs)
{
Slice slice;
ATermList roots, elems;
if (!ATmatch(fs, "Slice([<list>], [<list>])", &roots, &elems))
throw badTerm("not a slice", fs);
parseIds(roots, slice.roots);
while (!ATisEmpty(elems)) {
char * s1, * s2;
ATermList refs;
ATerm t = ATgetFirst(elems);
if (!ATmatch(t, "(<str>, <str>, [<list>])", &s1, &s2, &refs))
throw badTerm("not a slice element", t);
SliceElem elem;
elem.path = s1;
elem.id = parseHash(s2);
parseIds(refs, elem.refs);
slice.elems.push_back(elem);
elems = ATgetNext(elems);
}
return slice;
}
Slice normaliseFState(FSId id)
{
debug(format("normalising fstate"));
Nest nest(true);
/* Try to substitute $id$ by any known successors in order to
speed up the rewrite process. */
string idSucc;
while (queryDB(nixDB, dbSuccessors, id, idSucc)) {
debug(format("successor %1% -> %2%") % (string) id % idSucc);
id = parseHash(idSucc);
}
/* Get the fstate expression. */
FState fs = termFromId(id);
/* Already in normal form (i.e., a slice)? */
if (ATgetType(fs) == AT_APPL &&
(string) ATgetName(ATgetAFun(fs)) == "Slice")
return parseSlice(fs);
/* Then we it's a Derive node. */
ATermList outs, ins, bnds;
char * builder;
char * platform;
if (!ATmatch(fs, "Derive([<list>], [<list>], <str>, <str>, [<list>])",
&outs, &ins, &builder, &platform, &bnds))
throw badTerm("not a derive", fs);
/* Right platform? */
checkPlatform(platform);
/* Realise inputs (and remember all input paths). */
FSIds inIds;
parseIds(ins, inIds);
SliceElems inElems; /* !!! duplicates */
StringSet inPathsSet;
for (FSIds::iterator i = inIds.begin(); i != inIds.end(); i++) {
Slice slice = normaliseFState(*i);
realiseSlice(slice);
for (SliceElems::iterator j = slice.elems.begin();
j != slice.elems.end(); j++)
{
inElems.push_back(*j);
inPathsSet.insert(j->path);
}
}
Strings inPaths;
copy(inPathsSet.begin(), inPathsSet.end(),
inserter(inPaths, inPaths.begin()));
/* Build the environment. */
Environment env;
while (!ATisEmpty(bnds)) {
char * s1, * s2;
ATerm bnd = ATgetFirst(bnds);
if (!ATmatch(bnd, "(<str>, <str>)", &s1, &s2))
throw badTerm("tuple of strings expected", bnd);
env[s1] = s2;
bnds = ATgetNext(bnds);
}
/* Check that none of the output paths exist. */
typedef pair<string, FSId> OutPath;
list<OutPath> outPaths;
while (!ATisEmpty(outs)) {
ATerm t = ATgetFirst(outs);
char * s1, * s2;
if (!ATmatch(t, "(<str>, <str>)", &s1, &s2))
throw badTerm("string expected", t);
outPaths.push_back(OutPath(s1, parseHash(s2)));
outs = ATgetNext(outs);
}
for (list<OutPath>::iterator i = outPaths.begin();
i != outPaths.end(); i++)
if (pathExists(i->first))
throw Error(format("path `%1%' exists") % i->first);
/* Run the builder. */
runProgram(builder, env);
Slice slice;
/* Check whether the output paths were created, and register each
one. */
for (list<OutPath>::iterator i = outPaths.begin();
i != outPaths.end(); i++)
{
string path = i->first;
if (!pathExists(path))
throw Error(format("path `%1%' does not exist") % path);
registerPath(path, i->second);
slice.roots.push_back(i->second);
Strings outPaths = filterReferences(path, inPaths);
}
return slice;
}
void realiseSlice(Slice slice)
{
debug(format("realising slice"));
Nest nest(true);
if (slice.elems.size() == 0)
throw Error("empty slice");
/* Perhaps all paths already contain the right id? */
bool missing = false;
for (SliceElems::iterator i = slice.elems.begin();
i != slice.elems.end(); i++)
{
SliceElem elem = *i;
string id;
if (!queryDB(nixDB, dbPath2Id, elem.path, id)) {
if (pathExists(elem.path))
throw Error(format("path `%1%' obstructed") % elem.path);
missing = true;
break;
}
if (parseHash(id) != elem.id)
throw Error(format("path `%1%' obstructed") % elem.path);
}
if (!missing) {
debug(format("already installed"));
return;
}
/* For each element, expand its id at its path. */
for (SliceElems::iterator i = slice.elems.begin();
i != slice.elems.end(); i++)
{
SliceElem elem = *i;
expandId(elem.id, elem.path);
}
}

View file

@ -8,6 +8,7 @@ extern "C" {
}
#include "hash.hh"
#include "store.hh"
using namespace std;
@ -17,33 +18,24 @@ using namespace std;
A Nix file system state expression, or FState, describes a
(partial) state of the file system.
Path : Path * Content * [FState] -> FState
Slice : [Id] * [(Path, Id, [Id])] -> FState
(update)
Path(path, content, refs) specifies a file object (its full path
and contents), along with all file objects referenced by it (that
is, that it has pointers to). We assume that all files are
self-referential. This prevents us from having to deal with
cycles.
Derive : String * Path * [FState] * Path * [(String, String)] -> FState
Derive : [(Path, Id)] * [FStateId] * Path * [(String, String)] -> FState
(update)
Derive(platform, builder, ins, outs, env) specifies the creation of
new file objects (in paths declared by `outs') by the execution of
a program `builder' on a platform `platform'. This execution takes
place in a file system state given by `ins'. `env' specifies a
mapping of strings to strings.
[ !!! NOT IMPLEMENTED
Regular : String -> Content
Directory : [(String, Content)] -> Content
(this complicates unambiguous normalisation)
]
CHash : Hash -> Content
File content, given either in situ, or through an external reference
to the file system or url-space decorated with a hash to preserve
purity.
A FState expression is in {\em $f$-normal form} if all Derive nodes
have been reduced to File nodes.
@ -63,7 +55,26 @@ typedef ATerm Content;
typedef set<string> StringSet;
typedef list<FSId> FSIds;
struct SliceElem
{
string path;
FSId id;
FSIds refs;
};
typedef list<SliceElem> SliceElems;
struct Slice
{
FSIds roots;
SliceElems elems;
};
#if 0
/* Realise an fstate expression in the file system. This requires
execution of all Derive() nodes. */
FState realiseFState(FState fs, StringSet & paths);
@ -74,6 +85,8 @@ string fstatePath(FState fs);
/* Return the paths referenced by fstate expression. */
void fstateRefs(FState fs, StringSet & paths);
#endif
/* Return a canonical textual representation of an expression. */
string printTerm(ATerm t);
@ -85,16 +98,22 @@ Error badTerm(const format & f, ATerm t);
/* Hash an aterm. */
Hash hashTerm(ATerm t);
FState hash2fstate(Hash hash);
/* Read an aterm from disk, given its hash. */
ATerm termFromHash(const Hash & hash, string * p = 0);
/* Read an aterm from disk, given its id. */
ATerm termFromId(const FSId & id, string * p = 0);
/* Write an aterm to the Nix store directory, and return its hash. */
Hash writeTerm(ATerm t, const string & suffix, string * p = 0);
FSId writeTerm(ATerm t, const string & suffix, string * p = 0);
/* Register a successor. */
void registerSuccessor(const Hash & fsHash, const Hash & scHash);
void registerSuccessor(const FSId & id1, const FSId & id2);
/* Normalise an fstate-expression, that is, return an equivalent
Slice. */
Slice normaliseFState(FSId id);
/* Realise a Slice in the file system. */
void realiseSlice(Slice slice);
#endif /* !__FSTATE_H */

View file

@ -2,7 +2,8 @@
#include "db.hh"
string dbHash2Paths = "hash2paths";
string dbPath2Id = "path2id";
string dbId2Paths = "id2paths";
string dbSuccessors = "successors";
string dbSubstitutes = "substitutes";
@ -15,7 +16,8 @@ string nixDB = "/UNINIT";
void initDB()
{
createDB(nixDB, dbHash2Paths);
createDB(nixDB, dbPath2Id);
createDB(nixDB, dbId2Paths);
createDB(nixDB, dbSuccessors);
createDB(nixDB, dbSubstitutes);
}

View file

@ -8,33 +8,36 @@ using namespace std;
/* Database names. */
/* dbHash2Paths :: Hash -> [Path]
/* dbPath2Id :: Path -> FSId
Maintains a mapping from hashes to lists of paths. This is what we
use to resolve Hash(hash) content descriptors. */
extern string dbHash2Paths;
Each pair (p, id) records that path $p$ contains an expansion of
$id$. */
extern string dbPath2Id;
/* dbSuccessors :: Hash -> Hash
Each pair (h1, h2) in this mapping records the fact that a
successor of an fstate expression with hash h1 is stored in a file
with hash h2.
/* dbId2Paths :: FSId -> [Path]
A mapping from ids to lists of paths. */
extern string dbId2Paths;
/* dbSuccessors :: FSId -> FSId
Each pair $(id_1, id_2)$ in this mapping records the fact that a
successor of an fstate expression stored in a file with identifier
$id_1$ is stored in a file with identifier $id_2$.
Note that a term $y$ is successor of $x$ iff there exists a
sequence of rewrite steps that rewrites $x$ into $y$.
Also note that instead of a successor, $y$ can be any term
equivalent to $x$, that is, reducing to the same result, as long as
$x$ is equal to or a successor of $y$. (This is useful, e.g., for
shared derivate caching over the network).
*/
extern string dbSuccessors;
/* dbSubstitutes :: Hash -> [Hash]
Each pair $(h, [hs])$ tells Nix that it can realise any of the
fstate expressions referenced by the hashes in $hs$ to obtain a Nix
archive that, when unpacked, will produce a path with hash $h$.
/* dbSubstitutes :: FSId -> [FSId]
Each pair $(id, [ids])$ tells Nix that it can realise any of the
fstate expressions referenced by the identifiers in $ids$ to
generate a path with identifier $id$.
The main purpose of this is for distributed caching of derivates.
One system can compute a derivate with hash $h$ and put it on a

View file

@ -84,39 +84,36 @@ void copyPath(string src, string dst)
}
void registerSubstitute(const Hash & srcHash, const Hash & subHash)
void registerSubstitute(const FSId & srcId, const FSId & subId)
{
Strings subs;
queryListDB(nixDB, dbSubstitutes, srcHash, subs); /* non-existence = ok */
queryListDB(nixDB, dbSubstitutes, srcId, subs); /* non-existence = ok */
for (Strings::iterator it = subs.begin(); it != subs.end(); it++)
if (parseHash(*it) == subHash) return;
if (parseHash(*it) == subId) return;
subs.push_back(subHash);
subs.push_back(subId);
setListDB(nixDB, dbSubstitutes, srcHash, subs);
setListDB(nixDB, dbSubstitutes, srcId, subs);
}
Hash registerPath(const string & _path, Hash hash)
void registerPath(const string & _path, const FSId & id)
{
string path(canonPath(_path));
if (hash == Hash()) hash = hashPath(path);
setDB(nixDB, dbPath2Id, path, id);
Strings paths;
queryListDB(nixDB, dbHash2Paths, hash, paths); /* non-existence = ok */
queryListDB(nixDB, dbId2Paths, id, paths); /* non-existence = ok */
for (Strings::iterator it = paths.begin();
it != paths.end(); it++)
if (*it == path) goto exists;
if (*it == path) return;
paths.push_back(path);
setListDB(nixDB, dbHash2Paths, hash, paths);
exists:
return hash;
setListDB(nixDB, dbId2Paths, id, paths);
}
@ -124,10 +121,15 @@ void unregisterPath(const string & _path)
{
string path(canonPath(_path));
Hash hash = hashPath(path);
string _id;
if (!queryDB(nixDB, dbPath2Id, path, _id))
return;
FSId id(parseHash(_id));
/* begin transaction */
Strings paths, paths2;
queryListDB(nixDB, dbHash2Paths, hash, paths); /* non-existence = ok */
queryListDB(nixDB, dbId2Paths, id, paths); /* non-existence = ok */
bool changed = false;
for (Strings::iterator it = paths.begin();
@ -135,7 +137,9 @@ void unregisterPath(const string & _path)
if (*it != path) paths2.push_back(*it); else changed = true;
if (changed)
setListDB(nixDB, dbHash2Paths, hash, paths2);
setListDB(nixDB, dbId2Paths, id, paths2);
/* end transaction */
}
@ -146,7 +150,7 @@ bool isInPrefix(const string & path, const string & _prefix)
}
string expandHash(const Hash & hash, const string & target,
string expandId(const FSId & id, const string & target,
const string & prefix)
{
Strings paths;
@ -154,9 +158,7 @@ string expandHash(const Hash & hash, const string & target,
if (!target.empty() && !isInPrefix(target, prefix))
abort();
queryListDB(nixDB, dbHash2Paths, hash, paths);
/* !!! we shouldn't check for staleness by default --- too slow */
queryListDB(nixDB, dbId2Paths, id, paths);
/* Pick one equal to `target'. */
if (!target.empty()) {
@ -165,16 +167,8 @@ string expandHash(const Hash & hash, const string & target,
i != paths.end(); i++)
{
string path = *i;
try {
#if 0
if (path == target && hashPath(path) == hash)
#endif
if (path == target && pathExists(path))
return path;
} catch (Error & e) {
debug(format("stale path: %1%") % e.msg());
/* try next one */
}
}
}
@ -184,28 +178,26 @@ string expandHash(const Hash & hash, const string & target,
it != paths.end(); it++)
{
string path = *it;
try {
if (isInPrefix(path, prefix) && hashPath(path) == hash) {
if (isInPrefix(path, prefix) && pathExists(path)) {
if (target.empty())
return path;
else {
copyPath(path, target);
registerPath(target, id);
return target;
}
}
} catch (Error & e) {
debug(format("stale path: %1%") % e.msg());
/* try next one */
}
}
#if 0
/* Try to realise the substitutes. */
Strings subs;
queryListDB(nixDB, dbSubstitutes, hash, subs); /* non-existence = ok */
queryListDB(nixDB, dbSubstitutes, id, subs); /* non-existence = ok */
for (Strings::iterator it = subs.begin(); it != subs.end(); it++) {
StringSet dummy;
realiseSlice(normaliseFState(*it));
FState nf = realiseFState(hash2fstate(parseHash(*it)), dummy);
string path = fstatePath(nf);
@ -222,29 +214,30 @@ string expandHash(const Hash & hash, const string & target,
return target;
}
}
#endif
throw Error(format("cannot expand hash `%1%'") % (string) hash);
throw Error(format("cannot expand id `%1%'") % (string) id);
}
void addToStore(string srcPath, string & dstPath, Hash & hash,
void addToStore(string srcPath, string & dstPath, FSId & id,
bool deterministicName)
{
srcPath = absPath(srcPath);
hash = hashPath(srcPath);
id = hashPath(srcPath);
string baseName = baseNameOf(srcPath);
dstPath = canonPath(nixStore + "/" + (string) hash + "-" + baseName);
dstPath = canonPath(nixStore + "/" + (string) id + "-" + baseName);
try {
/* !!! should not use the substitutes! */
dstPath = expandHash(hash, deterministicName ? dstPath : "", nixStore);
dstPath = expandId(id, deterministicName ? dstPath : "", nixStore);
return;
} catch (...) {
}
copyPath(srcPath, dstPath);
registerPath(dstPath, hash);
registerPath(dstPath, id);
}

View file

@ -8,29 +8,28 @@
using namespace std;
typedef Hash FSId;
/* Copy a path recursively. */
void copyPath(string src, string dst);
/* Register a substitute. */
void registerSubstitute(const Hash & srcHash, const Hash & subHash);
void registerSubstitute(const FSId & srcId, const FSId & subId);
/* Register a path keyed on its hash. */
Hash registerPath(const string & path, Hash hash = Hash());
/* Register a path keyed on its id. */
void registerPath(const string & path, const FSId & id);
/* Return a path whose contents have the given hash. If target is
not empty, ensure that such a path is realised in target (if
necessary by copying from another location). If prefix is not
empty, only return a path that is an descendent of prefix.
If no path with the given hash is known to exist in the file
system,
*/
string expandHash(const Hash & hash, const string & target = "",
empty, only return a path that is an descendent of prefix. */
string expandId(const FSId & id, const string & target = "",
const string & prefix = "/");
/* Copy a file to the nixStore directory and register it in dbRefs.
Return the hash code of the value. */
void addToStore(string srcPath, string & dstPath, Hash & hash,
void addToStore(string srcPath, string & dstPath, FSId & id,
bool deterministicName = false);
/* Delete a value from the nixStore directory. */

View file

@ -11,14 +11,15 @@
#include "globals.hh"
void realise(FState fs)
void realise(FSId id)
{
cout << format("%1% => %2%\n")
% printTerm(fs)
% printTerm(realiseFState(fs));
cout << format("realising %1%\n") % (string) id;
Slice slice = normaliseFState(id);
realiseSlice(slice);
}
#if 0
void realiseFail(FState fs)
{
try {
@ -28,6 +29,7 @@ void realiseFail(FState fs)
cout << "error (expected): " << e.what() << endl;
}
}
#endif
struct MySink : DumpSink
@ -111,54 +113,47 @@ void runTests()
/* Expression evaluation. */
#if 0
eval(whNormalise,
ATmake("Str(\"Hello World\")"));
eval(whNormalise,
ATmake("Bool(True)"));
eval(whNormalise,
ATmake("Bool(False)"));
eval(whNormalise,
ATmake("App(Lam(\"x\", Var(\"x\")), Str(\"Hello World\"))"));
eval(whNormalise,
ATmake("App(App(Lam(\"x\", Lam(\"y\", Var(\"x\"))), Str(\"Hello World\")), Str(\"Hallo Wereld\"))"));
eval(whNormalise,
ATmake("App(Lam(\"sys\", Lam(\"x\", [Var(\"x\"), Var(\"sys\")])), Str(\"i686-suse-linux\"))"));
evalFail(whNormalise,
ATmake("Foo(123)"));
string builder1fn = absPath("./test-builder-1.sh");
Hash builder1h = hashPath(builder1fn);
string fn1 = nixValues + "/builder-1.sh";
Expr e1 = ATmake("Path(<str>, ExtFile(<str>, <str>), [])",
fn1.c_str(),
builder1h.c_str(),
builder1fn.c_str());
eval(fNormalise, e1);
string fn2 = nixValues + "/refer.txt";
Expr e2 = ATmake("Path(<str>, Regular(<str>), [<term>])",
fn2.c_str(),
("I refer to " + fn1).c_str(),
e1);
eval(fNormalise, e2);
realise(e2);
#endif
Hash builder1h;
FSId builder1id;
string builder1fn;
addToStore("./test-builder-1.sh", builder1fn, builder1h);
addToStore("./test-builder-1.sh", builder1fn, builder1id);
FState fs1 = ATmake(
"Path(<str>, Hash(<str>), [])",
"Slice([<str>], [(<str>, <str>, [])])",
((string) builder1id).c_str(),
builder1fn.c_str(),
((string) builder1h).c_str());
realise(fs1);
realise(fs1);
((string) builder1id).c_str());
FSId fs1id = writeTerm(fs1, "", 0);
realise(fs1id);
realise(fs1id);
FState fs2 = ATmake(
"Slice([<str>], [(<str>, <str>, [])])",
((string) builder1id).c_str(),
(builder1fn + "_bla").c_str(),
((string) builder1id).c_str());
FSId fs2id = writeTerm(fs2, "", 0);
realise(fs2id);
realise(fs2id);
string out1fn = nixStore + "/hello.txt";
string out1id = hashString("foo"); /* !!! bad */
FState fs3 = ATmake(
"Derive([(<str>, <str>)], [<str>], <str>, <str>, [(\"out\", <str>)])",
out1fn.c_str(),
((string) out1id).c_str(),
((string) fs1id).c_str(),
((string) builder1fn).c_str(),
thisSystem.c_str(),
out1fn.c_str());
debug(printTerm(fs3));
FSId fs3id = writeTerm(fs3, "", 0);
realise(fs3id);
realise(fs3id);
#if 0
FState fs2 = ATmake(
"Path(<str>, Hash(<str>), [])",
(builder1fn + "_bla").c_str(),
@ -175,28 +170,8 @@ void runTests()
out1fn.c_str(),
out1fn.c_str());
realise(fs3);
#if 0
Expr e1 = ATmake("Exec(Str(<str>), Hash(<str>), [])",
thisSystem.c_str(), ((string) builder1).c_str());
eval(e1);
Hash builder2 = addValue("./test-builder-2.sh");
Expr e2 = ATmake(
"Exec(Str(<str>), Hash(<str>), [Tup(Str(\"src\"), <term>)])",
thisSystem.c_str(), ((string) builder2).c_str(), e1);
eval(e2);
Hash h3 = addValue("./test-expr-1.nix");
Expr e3 = ATmake("Deref(Hash(<str>))", ((string) h3).c_str());
eval(e3);
deleteValue(h3);
#endif
}