* Caching of expression successors.

This commit is contained in:
Eelco Dolstra 2003-07-04 12:18:06 +00:00
parent 40b5936691
commit 207ff2caf0
8 changed files with 145 additions and 43 deletions

View file

@ -12,7 +12,7 @@ AC_PROG_CXX
AC_PROG_RANLIB AC_PROG_RANLIB
# Unix shell scripting should die a slow and painful death. # Unix shell scripting should die a slow and painful death.
AC_DEFINE_UNQUOTED(NIX_VALUES_DIR, "$(eval echo $prefix/values)", Nix values directory.) AC_DEFINE_UNQUOTED(NIX_STORE_DIR, "$(eval echo $prefix/store)", Nix store directory.)
AC_DEFINE_UNQUOTED(NIX_STATE_DIR, "$(eval echo $localstatedir/nix)", Nix state directory.) AC_DEFINE_UNQUOTED(NIX_STATE_DIR, "$(eval echo $localstatedir/nix)", Nix state directory.)
AC_DEFINE_UNQUOTED(NIX_LOG_DIR, "$(eval echo $localstatedir/log/nix)", Nix log file directory.) AC_DEFINE_UNQUOTED(NIX_LOG_DIR, "$(eval echo $localstatedir/log/nix)", Nix log file directory.)

View file

@ -23,5 +23,5 @@ install-data-local:
# $(INSTALL) -d $(localstatedir)/nix/prebuilts/imports # $(INSTALL) -d $(localstatedir)/nix/prebuilts/imports
# $(INSTALL) -d $(localstatedir)/nix/prebuilts/exports # $(INSTALL) -d $(localstatedir)/nix/prebuilts/exports
$(INSTALL) -d $(localstatedir)/log/nix $(INSTALL) -d $(localstatedir)/log/nix
$(INSTALL) -d $(prefix)/values $(INSTALL) -d $(prefix)/store
$(bindir)/nix --init $(bindir)/nix --init

View file

@ -25,7 +25,7 @@ bool pathExists(const string & path)
res = stat(path.c_str(), &st); res = stat(path.c_str(), &st);
if (!res) return true; if (!res) return true;
if (errno != ENOENT) if (errno != ENOENT)
throw SysError("getting status of " + path); throw SysError(format("getting status of %1%") % path);
return false; return false;
} }
@ -105,7 +105,7 @@ static void runProgram(const string & program, Environment env)
throw SysError(format("unable to execute %1%") % program); throw SysError(format("unable to execute %1%") % program);
} catch (exception & e) { } catch (exception & e) {
cerr << "build error: " << e.what() << endl; cerr << format("build error: %1%\n") % e.what();
} }
_exit(1); _exit(1);
@ -199,15 +199,62 @@ struct RStatus
}; };
static ATerm termFromHash(const Hash & hash)
{
string path = queryFromStore(hash);
ATerm t = ATreadFromNamedFile(path.c_str());
if (!t) throw Error(format("cannot read aterm %1%") % path);
return t;
}
static Hash writeTerm(ATerm t)
{
string path = nixStore + "/tmp.nix"; /* !!! */
if (!ATwriteToNamedTextFile(t, path.c_str()))
throw Error(format("cannot write aterm %1%") % path);
Hash hash = hashPath(path);
string path2 = nixStore + "/" + (string) hash + ".nix";
if (rename(path.c_str(), path2.c_str()) == -1)
throw SysError(format("renaming %1% to %2%") % path % path2);
setDB(nixDB, dbRefs, hash, path2);
return hash;
}
static FState realise(RStatus & status, FState fs) static FState realise(RStatus & status, FState fs)
{ {
char * s1, * s2, * s3; char * s1, * s2, * s3;
Content content; Content content;
ATermList refs, ins, outs, bnds; ATermList refs, ins, bnds;
if (ATmatch(fs, "File(<str>, <term>, [<list>])", &s1, &content, &refs)) { /* First repeatedly try to substitute $fs$ by any known successors
in order to speed up the rewrite process. */
{
string fsHash, scHash;
while (queryDB(nixDB, dbSuccessors, fsHash = hashTerm(fs), scHash)) {
debug(format("successor %1% -> %2%") % (string) fsHash % scHash);
FState fs2 = termFromHash(parseHash(scHash));
if (fs == fs2) {
debug(format("successor cycle detected in %1%") % printTerm(fs));
break;
}
fs = fs2;
}
}
/* Fall through. */
if (ATmatch(fs, "Include(<str>)", &s1)) {
return realise(status, termFromHash(parseHash(s1)));
}
else if (ATmatch(fs, "File(<str>, <term>, [<list>])", &s1, &content, &refs)) {
string path(s1); string path(s1);
msg(format("realising atomic path %1%") % path);
Nest nest(true);
if (path[0] != '/') throw Error("absolute path expected: " + path); if (path[0] != '/') throw Error("absolute path expected: " + path);
/* Realise referenced paths. */ /* Realise referenced paths. */
@ -223,9 +270,15 @@ static FState realise(RStatus & status, FState fs)
Hash hash = parseHash(s1); Hash hash = parseHash(s1);
/* Normal form. */ /* Normal form. */
ATerm nf = ATmake("File(<str>, <term>, <list>)", ATerm nf = ATmake("File(<str>, <term>, <term>)",
path.c_str(), content, refs2); path.c_str(), content, refs2);
/* Register the normal form. */
if (fs != nf) {
Hash nfHash = writeTerm(nf);
setDB(nixDB, dbSuccessors, hashTerm(fs), nfHash);
}
/* Perhaps the path already exists and has the right hash? */ /* Perhaps the path already exists and has the right hash? */
if (pathExists(path)) { if (pathExists(path)) {
if (hash == hashPath(path)) { if (hash == hashPath(path)) {
@ -250,6 +303,9 @@ static FState realise(RStatus & status, FState fs)
{ {
string platform(s1), builder(s2), outPath(s3); string platform(s1), builder(s2), outPath(s3);
msg(format("realising derivate path %1%") % outPath);
Nest nest(true);
checkPlatform(platform); checkPlatform(platform);
/* Realise inputs. */ /* Realise inputs. */
@ -297,15 +353,13 @@ static FState realise(RStatus & status, FState fs)
values.cc. */ values.cc. */
setDB(nixDB, dbRefs, outHash, outPath); setDB(nixDB, dbRefs, outHash, outPath);
#if 0 /* Register the normal form of fs. */
/* Register that targetHash was produced by evaluating FState nf = ATmake("File(<str>, Hash(<str>), <term>)",
sourceHash; i.e., that targetHash is a normal form of
sourceHash. !!! this shouldn't be here */
setDB(nixDB, dbNFs, sourceHash, targetHash);
#endif
return ATmake("File(<str>, Hash(<str>), <list>)",
outPath.c_str(), ((string) outHash).c_str(), ins2); outPath.c_str(), ((string) outHash).c_str(), ins2);
Hash nfHash = writeTerm(nf);
setDB(nixDB, dbSuccessors, hashTerm(fs), nfHash);
return nf;
} }
throw badTerm("bad file system state expression", fs); throw badTerm("bad file system state expression", fs);

View file

@ -3,7 +3,7 @@
string dbRefs = "refs"; string dbRefs = "refs";
string dbNFs = "nfs"; string dbSuccessors = "successors";
string dbNetSources = "netsources"; string dbNetSources = "netsources";
string nixStore = "/UNINIT"; string nixStore = "/UNINIT";
@ -14,6 +14,6 @@ string nixDB = "/UNINIT";
void initDB() void initDB()
{ {
createDB(nixDB, dbRefs); createDB(nixDB, dbRefs);
createDB(nixDB, dbNFs); createDB(nixDB, dbSuccessors);
createDB(nixDB, dbNetSources); createDB(nixDB, dbNetSources);
} }

View file

@ -14,17 +14,15 @@ using namespace std;
resolve CHash(hash) content descriptors. */ resolve CHash(hash) content descriptors. */
extern string dbRefs; extern string dbRefs;
/* dbNFs :: Hash -> Hash /* dbSuccessors :: Hash -> Hash
Each pair (h1, h2) in this mapping records the fact that the normal Each pair (h1, h2) in this mapping records the fact that a
form of an expression with hash h1 is Hash(h2). successor of an fstate expression with hash h1 is stored in a file
with hash h2.
TODO: maybe this should be that the normal form of an expression Note that a term $y$ is successor of $x$ iff there exists a
with hash h1 is an expression with hash h2; this would be more sequence of rewrite steps that rewrites $x$ into $y$. */
general, but would require us to store lots of small expressions in extern string dbSuccessors;
the file system just to support the caching mechanism.
*/
extern string dbNFs;
/* dbNetSources :: Hash -> URL /* dbNetSources :: Hash -> URL

View file

@ -22,7 +22,7 @@ static ArgType argType = atpUnknown;
Operations: Operations:
--evaluate / -e: evaluate values --realise / -r: realise values
--delete / -d: delete values --delete / -d: delete values
--query / -q: query stored values --query / -q: query stored values
--add: add values --add: add values
@ -87,8 +87,8 @@ static void getArgType(Strings & flags)
} }
/* Evaluate values. */ /* Realise values. */
static void opEvaluate(Strings opFlags, Strings opArgs) static void opRealise(Strings opFlags, Strings opArgs)
{ {
getArgType(opFlags); getArgType(opFlags);
if (!opFlags.empty()) throw UsageError("unknown flag"); if (!opFlags.empty()) throw UsageError("unknown flag");
@ -101,16 +101,19 @@ static void opEvaluate(Strings opFlags, Strings opArgs)
hash = parseHash(*it); hash = parseHash(*it);
else if (argType == atpName) else if (argType == atpName)
throw Error("not implemented"); throw Error("not implemented");
else if (argType == atpPath) else if (argType == atpPath) {
hash = addValue(*it); string path;
Expr e = ATmake("Deref(Hash(<str>))", ((string) hash).c_str()); addToStore(*it, path, hash);
cerr << printExpr(evalValue(e)) << endl; }
FState fs = ATmake("Include(<str>)", ((string) hash).c_str());
realiseFState(fs);
} }
} }
static void opDelete(Strings opFlags, Strings opArgs) static void opDelete(Strings opFlags, Strings opArgs)
{ {
#if 0
getArgType(opFlags); getArgType(opFlags);
if (!opFlags.empty()) throw UsageError("unknown flag"); if (!opFlags.empty()) throw UsageError("unknown flag");
@ -126,6 +129,7 @@ static void opDelete(Strings opFlags, Strings opArgs)
throw Error("invalid argument type"); throw Error("invalid argument type");
deleteValue(hash); deleteValue(hash);
} }
#endif
} }
@ -138,7 +142,12 @@ static void opAdd(Strings opFlags, Strings opArgs)
for (Strings::iterator it = opArgs.begin(); for (Strings::iterator it = opArgs.begin();
it != opArgs.end(); it++) it != opArgs.end(); it++)
cout << (string) addValue(*it) << endl; {
string path;
Hash hash;
addToStore(*it, path, hash);
cout << format("%1% %2%\n") % (string) hash % path;
}
} }
@ -158,6 +167,7 @@ struct StdoutSink : DumpSink
output. */ output. */
static void opDump(Strings opFlags, Strings opArgs) static void opDump(Strings opFlags, Strings opArgs)
{ {
#if 0
getArgType(opFlags); getArgType(opFlags);
if (!opFlags.empty()) throw UsageError("unknown flag"); if (!opFlags.empty()) throw UsageError("unknown flag");
if (opArgs.size() != 1) throw UsageError("only one argument allowed"); if (opArgs.size() != 1) throw UsageError("only one argument allowed");
@ -174,6 +184,7 @@ static void opDump(Strings opFlags, Strings opArgs)
path = arg; path = arg;
dumpPath(path, sink); dumpPath(path, sink);
#endif
} }
@ -218,7 +229,7 @@ static void opInit(Strings opFlags, Strings opArgs)
static void run(int argc, char * * argv) static void run(int argc, char * * argv)
{ {
/* Setup Nix paths. */ /* Setup Nix paths. */
nixValues = NIX_VALUES_DIR; nixStore = NIX_STORE_DIR;
nixLogDir = NIX_LOG_DIR; nixLogDir = NIX_LOG_DIR;
nixDB = (string) NIX_STATE_DIR + "/nixstate.db"; nixDB = (string) NIX_STATE_DIR + "/nixstate.db";
@ -253,8 +264,8 @@ static void run(int argc, char * * argv)
Operation oldOp = op; Operation oldOp = op;
if (arg == "--evaluate" || arg == "-e") if (arg == "--realise" || arg == "-r")
op = opEvaluate; op = opRealise;
else if (arg == "--delete" || arg == "-d") else if (arg == "--delete" || arg == "-d")
op = opDelete; op = opDelete;
else if (arg == "--add") else if (arg == "--add")

View file

@ -36,7 +36,7 @@ string absPath(string path, string dir)
/* !!! canonicalise */ /* !!! canonicalise */
char resolved[PATH_MAX]; char resolved[PATH_MAX];
if (!realpath(path.c_str(), resolved)) if (!realpath(path.c_str(), resolved))
throw SysError("cannot canonicalise path " + path); throw SysError(format("cannot canonicalise path %1%") % path);
path = resolved; path = resolved;
} }
return path; return path;
@ -46,7 +46,8 @@ string absPath(string path, string dir)
string dirOf(string path) string dirOf(string path)
{ {
unsigned int pos = path.rfind('/'); unsigned int pos = path.rfind('/');
if (pos == string::npos) throw Error("invalid file name: " + path); if (pos == string::npos)
throw Error(format("invalid file name: %1%") % path);
return string(path, 0, pos); return string(path, 0, pos);
} }
@ -54,7 +55,8 @@ string dirOf(string path)
string baseNameOf(string path) string baseNameOf(string path)
{ {
unsigned int pos = path.rfind('/'); unsigned int pos = path.rfind('/');
if (pos == string::npos) throw Error("invalid file name: " + path); if (pos == string::npos)
throw Error(format("invalid file name %1% ") % path);
return string(path, pos + 1); return string(path, pos + 1);
} }
@ -63,7 +65,7 @@ void deletePath(string path)
{ {
struct stat st; struct stat st;
if (lstat(path.c_str(), &st)) if (lstat(path.c_str(), &st))
throw SysError("getting attributes of path " + path); throw SysError(format("getting attributes of path %1%") % path);
if (S_ISDIR(st.st_mode)) { if (S_ISDIR(st.st_mode)) {
DIR * dir = opendir(path.c_str()); DIR * dir = opendir(path.c_str());
@ -79,11 +81,36 @@ void deletePath(string path)
} }
if (remove(path.c_str()) == -1) if (remove(path.c_str()) == -1)
throw SysError("cannot unlink " + path); throw SysError(format("cannot unlink %1%") % path);
}
static int nestingLevel = 0;
Nest::Nest(bool nest)
{
this->nest = nest;
if (nest) nestingLevel++;
}
Nest::~Nest()
{
if (nest) nestingLevel--;
}
void msg(const format & f)
{
string spaces;
for (int i = 0; i < nestingLevel; i++)
spaces += " ";
cerr << format("%1%%2%\n") % spaces % f.str();
} }
void debug(const format & f) void debug(const format & f)
{ {
cerr << format("debug: %1%\n") % f.str(); msg(format("debug: %1%") % f.str());
} }

View file

@ -61,6 +61,18 @@ string baseNameOf(string path);
void deletePath(string path); void deletePath(string path);
/* Messages. */
class Nest
{
private:
bool nest;
public:
Nest(bool nest);
~Nest();
};
void msg(const format & f);
void debug(const format & f); void debug(const format & f);