From 3da9687854e029e9df3b612fd592d2d5a622bb20 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 27 Jun 2003 13:55:12 +0000 Subject: [PATCH] * Realisation of File(...) expressions. --- src/eval.cc | 280 +++++++++---------------------------------------- src/eval.hh | 79 +++----------- src/globals.cc | 2 +- src/globals.hh | 19 ++-- src/hash.cc | 4 +- src/hash.hh | 4 +- src/test.cc | 36 +++++-- src/util.cc | 4 +- src/util.hh | 7 +- src/values.cc | 50 ++++----- src/values.hh | 20 ++-- 11 files changed, 137 insertions(+), 368 deletions(-) diff --git a/src/eval.cc b/src/eval.cc index 831464c18..4f59bcc21 100644 --- a/src/eval.cc +++ b/src/eval.cc @@ -30,6 +30,7 @@ bool pathExists(string path) } +#if 0 /* Compute a derived value by running a program. */ static Hash computeDerived(Hash sourceHash, string targetName, string platform, Hash prog, Environment env) @@ -175,6 +176,7 @@ static Hash computeDerived(Hash sourceHash, string targetName, return targetHash; } +#endif /* Throw an exception if the given platform string is not supported by @@ -182,54 +184,32 @@ static Hash computeDerived(Hash sourceHash, string targetName, static void checkPlatform(string platform) { if (platform != thisSystem) - throw Error("a `" + platform + - "' is required, but I am a `" + thisSystem + "'"); + throw Error(format("a `%1%' is required, but I am a `%2%'") + % platform % thisSystem); } -string printExpr(Expr e) +string printTerm(ATerm t) { - char * s = ATwriteToString(e); + char * s = ATwriteToString(t); return s; } /* Throw an exception with an error message containing the given aterm. */ -static Error badTerm(const string & msg, Expr e) +static Error badTerm(const format & f, ATerm t) { - return Error(msg + ", in `" + printExpr(e) + "'"); + return Error(format("%1%, in `%2%'") % f.str() % printTerm(t)); } -Hash hashExpr(Expr e) +Hash hashTerm(ATerm t) { - return hashString(printExpr(e)); + return hashString(printTerm(t)); } -/* Evaluate an expression; the result must be a string. */ -static string evalString(Expr e) -{ - e = whNormalise(e); - char * s; - if (ATmatch(e, "Str()", &s)) return s; - else throw badTerm("string value expected", e); -} - - -#if 0 -/* Evaluate an expression; the result must be a value reference. */ -static Hash evalHash(Expr e) -{ - e = evalValue(e); - char * s; - if (ATmatch(e, "Hash()", &s)) return parseHash(s); - else throw badTerm("value reference expected", e); -} -#endif - - #if 0 /* Evaluate a list of arguments into normal form. */ void evalArgs(ATermList args, ATermList & argsNF, Environment & env) @@ -262,225 +242,63 @@ void evalArgs(ATermList args, ATermList & argsNF, Environment & env) #endif -Expr substExpr(string x, Expr rep, Expr e) -{ - char * s; - Expr e2; - - if (ATmatch(e, "Var()", &s)) - if (x == s) - return rep; - else - return e; - - if (ATmatch(e, "Lam(, )", &s, &e2)) - if (x == s) - return e; - /* !!! unfair substitutions */ - - /* Generically substitute in subterms. */ - - if (ATgetType(e) == AT_APPL) { - AFun fun = ATgetAFun(e); - int arity = ATgetArity(fun); - ATermList args = ATempty; - - for (int i = arity - 1; i >= 0; i--) - args = ATinsert(args, substExpr(x, rep, ATgetArgument(e, i))); - - return (ATerm) ATmakeApplList(fun, args); - } - - if (ATgetType(e) == AT_LIST) { - ATermList in = (ATermList) e; - ATermList out = ATempty; - - while (!ATisEmpty(in)) { - out = ATinsert(out, substExpr(x, rep, ATgetFirst(in))); - in = ATgetNext(in); - } - - return (ATerm) ATreverse(out); - } - - throw badTerm("do not know how to substitute", e); -} - - -#if 0 -Expr evalValue(Expr e) -{ - char * s; - Expr eBuildPlatform, eProg, e2, e3, e4; - ATermList args; - - /* Value references. */ - if (ATmatch(e, "Hash()", &s)) { - parseHash(s); /* i.e., throw exception if not valid */ - return e; - } - - /* External expression. */ - if (ATmatch(e, "Deref()", &e2)) { - string fn = queryValuePath(evalHash(e2)); - ATerm e3 = ATreadFromNamedFile(fn.c_str()); - if (!e3) throw Error("reading aterm from " + fn); - return evalValue(e3); - } - - /* Execution primitive. */ - - if (ATmatch(e, "Exec(, , [])", - &eBuildPlatform, &eProg, &args)) - { - string buildPlatform = evalString(eBuildPlatform); - - checkPlatform(buildPlatform); - - Hash prog = evalHash(eProg); - - Environment env; - ATermList argsNF; - evalArgs(args, argsNF, env); - - Hash sourceHash = hashExpr( - ATmake("Exec(Str(), Hash(), )", - buildPlatform.c_str(), ((string) prog).c_str(), argsNF)); - - /* Do we know a normal form for sourceHash? */ - Hash targetHash; - string targetHashS; - if (queryDB(nixDB, dbNFs, sourceHash, targetHashS)) { - /* Yes. */ - targetHash = parseHash(targetHashS); - debug("already built: " + (string) sourceHash - + " -> " + (string) targetHash); - } else { - /* No, so we compute one. */ - targetHash = computeDerived(sourceHash, - (string) sourceHash + "-nf", buildPlatform, prog, env); - } - - return ATmake("Hash()", ((string) targetHash).c_str()); - } - - /* Barf. */ - throw badTerm("invalid expression", e); -} -#endif - - -Expr whNormalise(Expr e) -{ - char * s; - Expr e2, e3, e4, e5; - - /* Normal forms. */ - if (ATmatch(e, "Str()", &s) || - ATmatch(e, "Bool(True)") || - ATmatch(e, "Bool(False)") || - ATmatch(e, "Lam(, )", &s, &e2) || - ATmatch(e, "File(, , )", &s, &e2, &e3) || - ATmatch(e, "Derive(, , , )", &e2, &e3, &e4, &e5)) - return e; - - /* Application. */ - if (ATmatch(e, "App(, )", &e2, &e3)) { - e2 = whNormalise(e2); - if (!ATmatch(e2, "Lam(, )", &s, &e4)) - throw badTerm("expecting lambda", e2); - return whNormalise(substExpr(s, e3, e4)); - } - - throw badTerm("invalid expression", e); -} - - -Expr dNormalise(Expr e) -{ - e = whNormalise(e); - /* !!! todo */ - return e; -} - - -Expr fNormalise(Expr e) -{ - e = dNormalise(e); - - char * s; - Expr e2, e3; - - if (ATmatch(e, "File(, , [])", &s, &e2, &e3)) { - - ATermList refs = (ATermList) e3, refs2 = ATempty; - while (!ATisEmpty(refs)) { - ATerm ref = ATgetFirst(refs); - refs2 = ATinsert(refs2, fNormalise(ref)); - refs = ATgetNext(refs); - } - refs2 = ATreverse(refs2); - - return ATmake("File(, , )", s, e2, refs2); - - } - - else return e; -} - - -void writeContent(string path, Content content) -{ - char * s; - - if (ATmatch(content, "Regular()", &s)) { - - int fd; /* !!! close on exception */ - fd = open(path.c_str(), O_CREAT | O_EXCL | O_WRONLY, 0666); - if (fd == -1) - throw SysError("creating file " + path); - - int len = strlen(s); - if (write(fd, s, len) != len) - throw SysError("writing file " + path); - - close(fd); - } - - else throw badTerm("ill-formed content", content); -} - - struct RStatus { /* !!! the comparator of this hash should match the semantics of the file system */ - map paths; +// map paths; }; -static void realise2(RStatus & status, Expr e) +static void realise(RStatus & status, FState fs) { char * s; Content content; ATermList refs; - if (!ATmatch(e, "File(, , [])", &s, &content, &refs)) - throw badTerm("not f-normalised", e); + if (ATmatch(fs, "File(, , [])", &s, &content, &refs)) { + string path(s); - string path(s); + if (path[0] != '/') throw Error("absolute path expected: " + path); - while (!ATisEmpty(refs)) { - realise2(status, ATgetFirst(refs)); - refs = ATgetNext(refs); + /* Realise referenced paths. */ + while (!ATisEmpty(refs)) { + realise(status, ATgetFirst(refs)); + refs = ATgetNext(refs); + } + + if (!ATmatch(content, "Hash()", &s)) + throw badTerm("hash expected", content); + Hash hash = parseHash(s); + + /* Perhaps the path already exists and has the right hash? */ + if (pathExists(path)) { + if (hash == hashPath(path)) { + debug(format("path %1% already has hash %2%") + % path % (string) hash); + return; + } + + throw Error(format("path %1% exists, but does not have hash %2%") + % path % (string) hash); + } + + /* Do we know a path with that hash? If so, copy it. */ + string path2 = queryFromStore(hash); + copyFile(path2, path); } - - writeContent(path, content); + + else if (ATmatch(fs, "Derive()")) { + + + } + + else throw badTerm("bad file system state expression", fs); } -void realise(Expr e) +void realiseFState(FState fs) { RStatus status; - realise2(status, e); + realise(status, fs); } diff --git a/src/eval.hh b/src/eval.hh index 807f98f85..f90d5ba02 100644 --- a/src/eval.hh +++ b/src/eval.hh @@ -10,14 +10,12 @@ extern "C" { using namespace std; -/* \section{Abstract syntax of Nix expressions} +/* \section{Abstract syntax of Nix file system state expressions} - An expression describes a (partial) state of the file system in a - referentially transparent way. The operational effect of - evaluating an expression is that the state described by the - expression is realised. + A Nix file system state expression, or FState, describes a + (partial) state of the file system. - File : Path * Content * [Expr] -> Expr + File : Path * Content * [FState] -> FState File(path, content, refs) specifies a file object (its full path and contents), along with all file objects referenced by it (that @@ -25,7 +23,7 @@ using namespace std; self-referential. This prevents us from having to deal with cycles. - Derive : String * Path * [Expr] * [Expr] * [Expr] -> Expr + Derive : String * Path * [FState] * [Path] * [(String, String)] -> [FState] Derive(platform, builder, ins, outs, env) specifies the creation of new file objects (in paths declared by `outs') by the execution of @@ -33,14 +31,6 @@ using namespace std; place in a file system state given by `ins'. `env' specifies a mapping of strings to strings. - Str : String -> Expr - - A string constant. - - Tup : Expr * Expr -> Expr - - Tuples of expressions. - [ !!! NOT IMPLEMENTED Regular : String -> Content Directory : [(String, Content)] -> Content @@ -49,7 +39,11 @@ using namespace std; CHash : Hash -> Content File content, given either in situ, or through an external reference - to the file system or url-space decorated with a hash to preserve purity. + to the file system or url-space decorated with a hash to preserve + purity. + + A FState expression is in {\em $f$-normal form} if all Derive nodes + have been reduced to File nodes. DISCUSSION: the idea is that a Regular/Directory is interchangeable with its CHash. This would appear to break referential @@ -60,63 +54,20 @@ using namespace std; CHash, we should also export the file object referenced by that CHash. - - \section{Reduction rules} - - ... - - - \section{Normals forms} - - An expression is in {\em weak head normal form} if it is a lambda, - a string or boolean value, or a File or Derive value. - - An expression is in {\em $d$-normal form} if it matches the - signature FExpr: - - File : String * Content * [DExpr] -> DExpr - Derive : String * Path * [Tup] * [Tup2] -> DExpr - - Tup : Str * DExpr -> Tup - Tup : Str * Str -> Tup - - Tup : Str * Str -> Tup2 - - Str : String -> Str - - These are Nix expressions in which the file system result of Derive - expressions has not yet been computed. This is useful for, e.g., - querying dependencies. - - An expression is in {\em $f$-normal form} if it matches the - signature FExpr: - - File : String * Content * [FExpr] -> FExpr - - These are completely evaluated Nix expressions. - */ -typedef ATerm Expr; +typedef ATerm FState; typedef ATerm Content; -/* Expression normalisation. */ -Expr whNormalise(Expr e); -Expr dNormalise(Expr e); -Expr fNormalise(Expr e); - /* Realise a $f$-normalised expression in the file system. */ -void realise(Expr e); +void realiseFState(FState fs); /* Return a canonical textual representation of an expression. */ -string printExpr(Expr e); +string printTerm(ATerm t); -/* Perform variable substitution. */ -Expr substExpr(string x, Expr rep, Expr e); - -/* Hash an expression. */ -Hash hashExpr(Expr e); +/* Hash an aterm. */ +Hash hashTerm(ATerm t); #endif /* !__EVAL_H */ diff --git a/src/globals.cc b/src/globals.cc index 14fb431d8..640e960b1 100644 --- a/src/globals.cc +++ b/src/globals.cc @@ -6,7 +6,7 @@ string dbRefs = "refs"; string dbNFs = "nfs"; string dbNetSources = "netsources"; -string nixValues = "/UNINIT"; +string nixStore = "/UNINIT"; string nixLogDir = "/UNINIT"; string nixDB = "/UNINIT"; diff --git a/src/globals.hh b/src/globals.hh index b81a78714..3cb231ee2 100644 --- a/src/globals.hh +++ b/src/globals.hh @@ -8,15 +8,10 @@ using namespace std; /* Database names. */ -/* dbRefs :: Hash -> FileName +/* dbRefs :: Hash -> Path - Maintains a mapping from hashes to filenames within the NixValues - directory. This mapping is for performance only; it can be - reconstructed unambiguously. The reason is that names in this - directory are not printed hashes but also might carry some - descriptive element (e.g., "aterm-2.0-ae749a..."). Without this - mapping, looking up a value would take O(n) time because we would - need to read the entire directory. */ + Maintains a mapping from hashes to paths. This is what we use to + resolve CHash(hash) content descriptors. */ extern string dbRefs; /* dbNFs :: Hash -> Hash @@ -45,11 +40,11 @@ extern string dbNetSources; /* Path names. */ -/* nixValues is the directory where all Nix values (both files and - directories, and both normal and non-normal forms) live. */ -extern string nixValues; +/* nixStore is the directory where we generally store atomic and + derived files. */ +extern string nixStore; -/* nixLogDir is the directory where we log evaluations. */ +/* nixLogDir is the directory where we log various operations. */ extern string nixLogDir; /* nixDB is the file name of the Berkeley DB database where we diff --git a/src/hash.cc b/src/hash.cc index 765b7ba04..72dcd790c 100644 --- a/src/hash.cc +++ b/src/hash.cc @@ -14,7 +14,7 @@ Hash::Hash() } -bool Hash::operator == (Hash & h2) +bool Hash::operator == (Hash h2) { for (unsigned int i = 0; i < hashSize; i++) if (hash[i] != h2.hash[i]) return false; @@ -22,7 +22,7 @@ bool Hash::operator == (Hash & h2) } -bool Hash::operator != (Hash & h2) +bool Hash::operator != (Hash h2) { return !(*this == h2); } diff --git a/src/hash.hh b/src/hash.hh index cbc195c1f..509a27912 100644 --- a/src/hash.hh +++ b/src/hash.hh @@ -17,10 +17,10 @@ struct Hash Hash(); /* Check whether two hash are equal. */ - bool operator == (Hash & h2); + bool operator == (Hash h2); /* Check whether two hash are not equal. */ - bool operator != (Hash & h2); + bool operator != (Hash h2); /* Convert a hash code into a hexadecimal representation. */ operator string() const; diff --git a/src/test.cc b/src/test.cc index d912eaa6a..aafae8ee3 100644 --- a/src/test.cc +++ b/src/test.cc @@ -11,20 +11,16 @@ #include "globals.hh" -typedef Expr (* Normaliser) (Expr); - - -void eval(Normaliser n, Expr e) +void realise(FState fs) { - e = n(e); - cout << (string) hashExpr(e) << ": " << printExpr(e) << endl; + realiseFState(fs); } -void evalFail(Normaliser n, Expr e) +void realiseFail(FState fs) { try { - e = n(e); + realiseFState(fs); abort(); } catch (Error e) { cout << "error (expected): " << e.what() << endl; @@ -96,7 +92,7 @@ void runTests() string testDir = absPath("scratch"); cout << testDir << endl; - nixValues = testDir; + nixStore = testDir; nixLogDir = testDir; nixDB = testDir + "/db"; @@ -104,6 +100,7 @@ void runTests() /* Expression evaluation. */ +#if 0 eval(whNormalise, ATmake("Str(\"Hello World\")")); eval(whNormalise, @@ -138,10 +135,27 @@ void runTests() eval(fNormalise, e2); realise(e2); +#endif + + Hash builder1h; + string builder1fn; + addToStore("./test-builder-1.sh", builder1fn, builder1h); + + FState fs1 = ATmake( + "File(, Hash(), [])", + builder1fn.c_str(), + ((string) builder1h).c_str()); + realiseFState(fs1); + realiseFState(fs1); + + FState fs2 = ATmake( + "File(, Hash(), [])", + (builder1fn + "_bla").c_str(), + ((string) builder1h).c_str()); + realiseFState(fs2); + realiseFState(fs2); #if 0 - Hash builder1 = addValue("./test-builder-1.sh"); - Expr e1 = ATmake("Exec(Str(), Hash(), [])", thisSystem.c_str(), ((string) builder1).c_str()); diff --git a/src/util.cc b/src/util.cc index 4dada48ba..c6a0c1199 100644 --- a/src/util.cc +++ b/src/util.cc @@ -78,7 +78,7 @@ void deletePath(string path) } -void debug(string s) +void debug(const format & f) { - cerr << "debug: " << s << endl; + cerr << format("debug: %1%\n") % f.str(); } diff --git a/src/util.hh b/src/util.hh index 7d5f00a2e..3efac928b 100644 --- a/src/util.hh +++ b/src/util.hh @@ -7,7 +7,10 @@ #include +#include + using namespace std; +using namespace boost; class Error : public exception @@ -16,7 +19,7 @@ protected: string err; public: Error() { } - Error(string _err) { err = _err; } + Error(format f) { err = f.str(); } ~Error() throw () { } const char * what() const throw () { return err.c_str(); } }; @@ -59,7 +62,7 @@ string baseNameOf(string path); void deletePath(string path); -void debug(string s); +void debug(const format & f); #endif /* !__UTIL_H */ diff --git a/src/values.cc b/src/values.cc index c8a3b58cb..e23624ce5 100644 --- a/src/values.cc +++ b/src/values.cc @@ -34,7 +34,7 @@ struct CopySource : RestoreSource }; -static void copyFile(string src, string dst) +void copyFile(string src, string dst) { /* Unfortunately C++ doesn't support coprocedures, so we have no nice way to chain CopySink and CopySource together. Instead we @@ -83,33 +83,25 @@ static void copyFile(string src, string dst) } -static string absValuePath(string s) +void addToStore(string srcPath, string & dstPath, Hash & hash) { - return nixValues + "/" + s; -} + srcPath = absPath(srcPath); + hash = hashPath(srcPath); -Hash addValue(string path) -{ - path = absPath(path); - - Hash hash = hashPath(path); - - string name; - if (queryDB(nixDB, dbRefs, hash, name)) { + string path; + if (queryDB(nixDB, dbRefs, hash, path)) { debug((string) hash + " already known"); - return hash; + dstPath = path; + return; } - string baseName = baseNameOf(path); - - string targetName = (string) hash + "-" + baseName; + string baseName = baseNameOf(srcPath); + dstPath = nixStore + "/" + (string) hash + "-" + baseName; - copyFile(path, absValuePath(targetName)); + copyFile(srcPath, dstPath); - setDB(nixDB, dbRefs, hash, targetName); - - return hash; + setDB(nixDB, dbRefs, hash, dstPath); } @@ -135,28 +127,28 @@ string fetchURL(string url) #endif -void deleteValue(Hash hash) +void deleteFromStore(Hash hash) { - string name; - if (queryDB(nixDB, dbRefs, hash, name)) { - string fn = absValuePath(name); + string fn; + if (queryDB(nixDB, dbRefs, hash, fn)) { + string prefix = nixStore + "/"; + if (string(fn, prefix.size()) != prefix) + throw Error("path " + fn + " is not in the store"); deletePath(fn); delDB(nixDB, dbRefs, hash); } } -/* !!! bad name, "query" implies no side effect => getValuePath() */ -string queryValuePath(Hash hash) +string queryFromStore(Hash hash) { bool checkedNet = false; while (1) { - string name, url; + string fn, url; - if (queryDB(nixDB, dbRefs, hash, name)) { - string fn = absValuePath(name); + if (queryDB(nixDB, dbRefs, hash, fn)) { /* Verify that the file hasn't changed. !!! race !!! slow */ if (hashPath(fn) != hash) diff --git a/src/values.hh b/src/values.hh index d66ae770f..1bb00a9dd 100644 --- a/src/values.hh +++ b/src/values.hh @@ -8,21 +8,17 @@ using namespace std; -/* Copy a value to the nixValues directory and register it in dbRefs. +void copyFile(string src, string dst); + +/* Copy a file to the nixStore directory and register it in dbRefs. Return the hash code of the value. */ -Hash addValue(string pathName); +void addToStore(string srcPath, string & dstPath, Hash & hash); +/* Delete a value from the nixStore directory. */ +void deleteFromStore(Hash hash); -/* Delete a value from the nixValues directory. */ -void deleteValue(Hash hash); - - -/* Obtain the path of a value with the given hash. If a file with - that hash is known to exist in the local file system (as indicated - by the dbRefs database), we use that. Otherwise, we attempt to - fetch it from the network (using dbNetSources). We verify that the - file has the right hash. */ -string queryValuePath(Hash hash); +/* !!! */ +string queryFromStore(Hash hash); #endif /* !__VALUES_H */