lix-project/lix
20
57
Fork 31
Code Issues 347 Code Review (Gerrit) Projects 3 Releases Wiki Activity

* Get rid of identifiers since they are redundant now. This greatly

Browse source 
simplifies stuff.

* The format of Nix expressions and the database schema changed
  because of this, so it's best to delete old Nix installations.
This commit is contained in:
Eelco Dolstra 2003-10-08 15:06:59 +00:00
parent b9f4942bd2
commit 6baa2c4420
23 changed files with 485 additions and 631 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
src/archive.cc
View file

@ -49,7 +49,7 @@ static void writeString(const string & s, DumpSink & sink)
static void dump(const string & path, DumpSink & sink);
static void dumpEntries(const string & path, DumpSink & sink)
static void dumpEntries(const Path & path, DumpSink & sink)
{
DIR * dir = opendir(path.c_str());
if (!dir) throw SysError("opening directory " + path);
@ -82,7 +82,7 @@ static void dumpEntries(const string & path, DumpSink & sink)
}
static void dumpContents(const string & path, unsigned int size,
static void dumpContents(const Path & path, unsigned int size,
DumpSink & sink)
{
writeString("contents", sink);
@ -110,7 +110,7 @@ static void dumpContents(const string & path, unsigned int size,
}
static void dump(const string & path, DumpSink & sink)
static void dump(const Path & path, DumpSink & sink)
{
struct stat st;
if (lstat(path.c_str(), &st))
@ -150,7 +150,7 @@ static void dump(const string & path, DumpSink & sink)
}
void dumpPath(const string & path, DumpSink & sink)
void dumpPath(const Path & path, DumpSink & sink)
{
writeString(archiveVersion1, sink);
dump(path, sink);
@ -207,10 +207,10 @@ static void skipGeneric(RestoreSource & source)
}
static void restore(const string & path, RestoreSource & source);
static void restore(const Path & path, RestoreSource & source);
static void restoreEntry(const string & path, RestoreSource & source)
static void restoreEntry(const Path & path, RestoreSource & source)
{
string s, name;
@ -235,7 +235,7 @@ static void restoreEntry(const string & path, RestoreSource & source)
}
static void restoreContents(int fd, const string & path, RestoreSource & source)
static void restoreContents(int fd, const Path & path, RestoreSource & source)
{
unsigned int size = readInt(source);
unsigned int left = size;
@ -254,7 +254,7 @@ static void restoreContents(int fd, const string & path, RestoreSource & source)
}
static void restore(const string & path, RestoreSource & source)
static void restore(const Path & path, RestoreSource & source)
{
string s;
@ -331,7 +331,7 @@ static void restore(const string & path, RestoreSource & source)
}
void restorePath(const string & path, RestoreSource & source)
void restorePath(const Path & path, RestoreSource & source)
{
if (readString(source) != archiveVersion1)
throw badArchive("expected Nix archive");

6
src/archive.hh
View file

@ -1,6 +1,6 @@
#include <string>
using namespace std;
#include "util.hh"
/* dumpPath creates a Nix archive of the specified path. The format
@ -45,7 +45,7 @@ struct DumpSink
virtual void operator () (const unsigned char * data, unsigned int len) = 0;
};
void dumpPath(const string & path, DumpSink & sink);
void dumpPath(const Path & path, DumpSink & sink);
struct RestoreSource
@ -57,4 +57,4 @@ struct RestoreSource
virtual void operator () (unsigned char * data, unsigned int len) = 0;
};
void restorePath(const string & path, RestoreSource & source);
void restorePath(const Path & path, RestoreSource & source);

2
src/db.cc
View file

@ -258,6 +258,8 @@ void Database::delPair(const Transaction & txn, TableId table,
Db * db = getDb(table);
Dbt kt((void *) key.c_str(), key.length());
db->del(txn.txn, &kt, 0);
/* Non-existence of a pair with the given key is not an
error. */
} catch (DbException e) { rethrow(e); }
}

50
src/dotgraph.cc
View file

@ -45,24 +45,24 @@ static string symbolicName(const string & path)
}
string pathLabel(const FSId & id, const string & path)
string pathLabel(const Path & nePath, const string & elemPath)
{
return (string) id + "-" + path;
return (string) nePath + "-" + elemPath;
}
void printClosure(const FSId & id, const NixExpr & fs)
void printClosure(const Path & nePath, const NixExpr & fs)
{
Strings workList(fs.closure.roots.begin(), fs.closure.roots.end());
StringSet doneSet;
PathSet workList(fs.closure.roots);
PathSet doneSet;
for (Strings::iterator i = workList.begin(); i != workList.end(); i++) {
cout << makeEdge(pathLabel(id, *i), id);
for (PathSet::iterator i = workList.begin(); i != workList.end(); i++) {
cout << makeEdge(pathLabel(nePath, *i), nePath);
}
while (!workList.empty()) {
string path = workList.front();
workList.pop_front();
Path path = *(workList.begin());
workList.erase(path);
if (doneSet.find(path) == doneSet.end()) {
doneSet.insert(path);
@ -74,41 +74,41 @@ void printClosure(const FSId & id, const NixExpr & fs)
for (StringSet::const_iterator i = elem->second.refs.begin();
i != elem->second.refs.end(); i++)
{
workList.push_back(*i);
cout << makeEdge(pathLabel(id, *i), pathLabel(id, path));
workList.insert(*i);
cout << makeEdge(pathLabel(nePath, *i), pathLabel(nePath, path));
}
cout << makeNode(pathLabel(id, path),
cout << makeNode(pathLabel(nePath, path),
symbolicName(path), "#ff0000");
}
}
}
void printDotGraph(const FSIds & roots)
void printDotGraph(const PathSet & roots)
{
FSIds workList(roots.begin(), roots.end());
FSIdSet doneSet;
PathSet workList(roots);
PathSet doneSet;
cout << "digraph G {\n";
while (!workList.empty()) {
FSId id = workList.front();
workList.pop_front();
Path nePath = *(workList.begin());
workList.erase(nePath);
if (doneSet.find(id) == doneSet.end()) {
doneSet.insert(id);
if (doneSet.find(nePath) == doneSet.end()) {
doneSet.insert(nePath);
NixExpr ne = parseNixExpr(termFromId(id));
NixExpr ne = parseNixExpr(termFromPath(nePath));
string label, colour;
if (ne.type == NixExpr::neDerivation) {
for (FSIdSet::iterator i = ne.derivation.inputs.begin();
for (PathSet::iterator i = ne.derivation.inputs.begin();
i != ne.derivation.inputs.end(); i++)
{
workList.push_back(*i);
cout << makeEdge(*i, id);
workList.insert(*i);
cout << makeEdge(*i, nePath);
}
label = "derivation";
@ -121,12 +121,12 @@ void printDotGraph(const FSIds & roots)
else if (ne.type == NixExpr::neClosure) {
label = "<closure>";
colour = "#00ffff";
printClosure(id, ne);
printClosure(nePath, ne);
}
else abort();
cout << makeNode(id, label, colour);
cout << makeNode(nePath, label, colour);
}
}

2
src/dotgraph.hh
View file

@ -3,6 +3,6 @@
#include "expr.hh"
void printDotGraph(const FSIds & roots);
void printDotGraph(const PathSet & roots);
#endif /* !__DOTGRAPH_H */

75
src/expr.cc
View file

@ -22,37 +22,33 @@ Hash hashTerm(ATerm t)
}
ATerm termFromId(const FSId & id)
ATerm termFromPath(const Path & path)
{
string path = expandId(id);
ATerm t = ATreadFromNamedFile(path.c_str());
if (!t) throw Error(format("cannot read aterm from `%1%'") % path);
return t;
}
FSId writeTerm(ATerm t, const string & suffix, FSId id)
Path writeTerm(ATerm t, const string & suffix)
{
/* By default, the id of a term is its hash. */
if (id == FSId()) id = hashTerm(t);
/* The id of a term is its hash. */
Hash h = hashTerm(t);
string path = canonPath(nixStore + "/" +
(string) id + suffix + ".nix");
Path path = canonPath(nixStore + "/" +
(string) h + suffix + ".nix");
if (!ATwriteToNamedTextFile(t, path.c_str()))
throw Error(format("cannot write aterm %1%") % path);
// debug(format("written term %1% = %2%") % (string) id %
// printTerm(t));
Transaction txn(nixDB);
registerPath(txn, path, id);
registerValidPath(txn, path);
txn.commit();
return id;
return path;
}
static void parsePaths(ATermList paths, StringSet & out)
static void parsePaths(ATermList paths, PathSet & out)
{
while (!ATisEmpty(paths)) {
char * s;
@ -70,19 +66,19 @@ static void checkClosure(const Closure & closure)
if (closure.elems.size() == 0)
throw Error("empty closure");
StringSet decl;
PathSet decl;
for (ClosureElems::const_iterator i = closure.elems.begin();
i != closure.elems.end(); i++)
decl.insert(i->first);
for (StringSet::const_iterator i = closure.roots.begin();
for (PathSet::const_iterator i = closure.roots.begin();
i != closure.roots.end(); i++)
if (decl.find(*i) == decl.end())
throw Error(format("undefined root path `%1%'") % *i);
for (ClosureElems::const_iterator i = closure.elems.begin();
i != closure.elems.end(); i++)
for (StringSet::const_iterator j = i->second.refs.begin();
for (PathSet::const_iterator j = i->second.refs.begin();
j != i->second.refs.end(); j++)
if (decl.find(*j) == decl.end())
throw Error(
@ -102,13 +98,12 @@ static bool parseClosure(ATerm t, Closure & closure)
parsePaths(roots, closure.roots);
while (!ATisEmpty(elems)) {
char * s1, * s2;
char * s1;
ATermList refs;
ATerm t = ATgetFirst(elems);
if (!ATmatch(t, "(<str>, <str>, [<list>])", &s1, &s2, &refs))
if (!ATmatch(t, "(<str>, [<list>])", &s1, &refs))
throw badTerm("not a closure element", t);
ClosureElem elem;
elem.id = parseHash(s2);
parsePaths(refs, elem.refs);
closure.elems[s1] = elem;
elems = ATgetNext(elems);
@ -135,23 +130,8 @@ static bool parseDerivation(ATerm t, Derivation & derivation)
args = ATempty;
}
while (!ATisEmpty(outs)) {
char * s1, * s2;
ATerm t = ATgetFirst(outs);
if (!ATmatch(t, "(<str>, <str>)", &s1, &s2))
throw badTerm("not a derivation output", t);
derivation.outputs[s1] = parseHash(s2);
outs = ATgetNext(outs);
}
while (!ATisEmpty(ins)) {
char * s;
ATerm t = ATgetFirst(ins);
if (!ATmatch(t, "<str>", &s))
throw badTerm("not an id", t);
derivation.inputs.insert(parseHash(s));
ins = ATgetNext(ins);
}
parsePaths(outs, derivation.outputs);
parsePaths(ins, derivation.inputs);
derivation.builder = builder;
derivation.platform = platform;
@ -190,10 +170,10 @@ NixExpr parseNixExpr(ATerm t)
}
static ATermList unparsePaths(const StringSet & paths)
static ATermList unparsePaths(const PathSet & paths)
{
ATermList l = ATempty;
for (StringSet::const_iterator i = paths.begin();
for (PathSet::const_iterator i = paths.begin();
i != paths.end(); i++)
l = ATinsert(l, ATmake("<str>", i->c_str()));
return ATreverse(l);
@ -208,9 +188,8 @@ static ATerm unparseClosure(const Closure & closure)
for (ClosureElems::const_iterator i = closure.elems.begin();
i != closure.elems.end(); i++)
elems = ATinsert(elems,
ATmake("(<str>, <str>, <term>)",
ATmake("(<str>, <term>)",
i->first.c_str(),
((string) i->second.id).c_str(),
unparsePaths(i->second.refs)));
return ATmake("Closure(<term>, <term>)", roots, elems);
@ -219,18 +198,6 @@ static ATerm unparseClosure(const Closure & closure)
static ATerm unparseDerivation(const Derivation & derivation)
{
ATermList outs = ATempty;
for (DerivationOutputs::const_iterator i = derivation.outputs.begin();
i != derivation.outputs.end(); i++)
outs = ATinsert(outs,
ATmake("(<str>, <str>)",
i->first.c_str(), ((string) i->second).c_str()));
ATermList ins = ATempty;
for (FSIdSet::const_iterator i = derivation.inputs.begin();
i != derivation.inputs.end(); i++)
ins = ATinsert(ins, ATmake("<str>", ((string) *i).c_str()));
ATermList args = ATempty;
for (Strings::const_iterator i = derivation.args.begin();
i != derivation.args.end(); i++)
@ -244,8 +211,8 @@ static ATerm unparseDerivation(const Derivation & derivation)
i->first.c_str(), i->second.c_str()));
return ATmake("Derive(<term>, <term>, <str>, <str>, <term>, <term>)",
ATreverse(outs),
ATreverse(ins),
unparsePaths(derivation.outputs),
unparsePaths(derivation.inputs),
derivation.platform.c_str(),
derivation.builder.c_str(),
ATreverse(args),

24
src/expr.hh
View file

@ -10,31 +10,27 @@ extern "C" {
/* Abstract syntax of Nix expressions. */
typedef list<FSId> FSIds;
struct ClosureElem
{
FSId id;
StringSet refs;
PathSet refs;
};
typedef map<string, ClosureElem> ClosureElems;
typedef map<Path, ClosureElem> ClosureElems;
struct Closure
{
StringSet roots;
PathSet roots;
ClosureElems elems;
};
typedef map<string, FSId> DerivationOutputs;
typedef map<string, string> StringPairs;
struct Derivation
{
DerivationOutputs outputs;
FSIdSet inputs;
PathSet outputs;
PathSet inputs; /* Nix expressions, not actual inputs */
string platform;
string builder;
Path builder;
Strings args;
StringPairs env;
};
@ -57,11 +53,11 @@ Error badTerm(const format & f, ATerm t);
/* Hash an aterm. */
Hash hashTerm(ATerm t);
/* Read an aterm from disk, given its id. */
ATerm termFromId(const FSId & id);
/* Read an aterm from disk. */
ATerm termFromPath(const Path & path);
/* Write an aterm to the Nix store directory, and return its hash. */
FSId writeTerm(ATerm t, const string & suffix, FSId id = FSId());
/* Write an aterm to the Nix store directory, and return its path. */
Path writeTerm(ATerm t, const string & suffix);
/* Parse a Nix expression. */
NixExpr parseNixExpr(ATerm t);

109
src/fix.cc
View file

@ -9,12 +9,12 @@
typedef ATerm Expr;
typedef map<ATerm, ATerm> NormalForms;
typedef map<FSId, Strings> PkgPaths;
typedef map<FSId, Hash> PkgHashes;
typedef map<Path, PathSet> PkgPaths;
typedef map<Path, Hash> PkgHashes;
struct EvalState
{
Strings searchDirs;
Paths searchDirs;
NormalForms normalForms;
PkgPaths pkgPaths;
PkgHashes pkgHashes; /* normalised package hashes */
@ -28,18 +28,18 @@ struct EvalState
};
static Expr evalFile(EvalState & state, string fileName);
static Expr evalFile(EvalState & state, const Path & path);
static Expr evalExpr(EvalState & state, Expr e);
static string searchPath(const Strings & searchDirs, string relPath)
static Path searchPath(const Paths & searchDirs, const Path & relPath)
{
if (string(relPath, 0, 1) == "/") return relPath;
for (Strings::const_iterator i = searchDirs.begin();
for (Paths::const_iterator i = searchDirs.begin();
i != searchDirs.end(); i++)
{
string path = *i + "/" + relPath;
Path path = *i + "/" + relPath;
if (pathExists(path)) return path;
}
@ -121,14 +121,14 @@ static Expr substExprMany(ATermList formals, ATermList args, Expr body)
}
static Strings nixExprPathsCached(EvalState & state, const FSId & id)
static PathSet nixExprRootsCached(EvalState & state, const Path & nePath)
{
PkgPaths::iterator i = state.pkgPaths.find(id);
PkgPaths::iterator i = state.pkgPaths.find(nePath);
if (i != state.pkgPaths.end())
return i->second;
else {
Strings paths = nixExprPaths(id);
state.pkgPaths[id] = paths;
PathSet paths = nixExprRoots(nePath);
state.pkgPaths[nePath] = paths;
return paths;
}
}
@ -137,13 +137,13 @@ static Strings nixExprPathsCached(EvalState & state, const FSId & id)
static Hash hashPackage(EvalState & state, NixExpr ne)
{
if (ne.type == NixExpr::neDerivation) {
FSIdSet inputs2;
for (FSIdSet::iterator i = ne.derivation.inputs.begin();
PathSet inputs2;
for (PathSet::iterator i = ne.derivation.inputs.begin();
i != ne.derivation.inputs.end(); i++)
{
PkgHashes::iterator j = state.pkgHashes.find(*i);
if (j == state.pkgHashes.end())
throw Error(format("unknown package id %1%") % (string) *i);
throw Error(format("don't know expression `%1%'") % (string) *i);
inputs2.insert(j->second);
}
ne.derivation.inputs = inputs2;
@ -156,12 +156,12 @@ static string processBinding(EvalState & state, Expr e, NixExpr & ne)
{
char * s1;
if (ATmatch(e, "FSId(<str>)", &s1)) {
FSId id = parseHash(s1);
Strings paths = nixExprPathsCached(state, id);
if (ATmatch(e, "NixExpr(<str>)", &s1)) {
Path nePath(s1);
PathSet paths = nixExprRootsCached(state, nePath);
if (paths.size() != 1) abort();
string path = *(paths.begin());
ne.derivation.inputs.insert(id);
Path path = *(paths.begin());
ne.derivation.inputs.insert(nePath);
return path;
}
@ -200,14 +200,14 @@ static Expr evalExpr2(EvalState & state, Expr e)
ATmatch(e, "True") ||
ATmatch(e, "False") ||
ATmatch(e, "Function([<list>], <term>)", &e1, &e2) ||
ATmatch(e, "FSId(<str>)", &s1))
ATmatch(e, "NixExpr(<str>)", &s1))
return e;
try {
Hash pkgHash = hashPackage(state, parseNixExpr(e));
FSId pkgId = writeTerm(e, "");
state.pkgHashes[pkgId] = pkgHash;
return ATmake("FSId(<str>)", ((string) pkgId).c_str());
Path pkgPath = writeTerm(e, "");
state.pkgHashes[pkgPath] = pkgHash;
return ATmake("NixExpr(<str>)", pkgPath.c_str());
} catch (...) { /* !!! catch parse errors only */
}
@ -254,32 +254,29 @@ static Expr evalExpr2(EvalState & state, Expr e)
/* Fix inclusion. */
if (ATmatch(e, "IncludeFix(<str>)", &s1)) {
string fileName(s1);
Path fileName(s1);
return evalFile(state, s1);
}
/* Relative files. */
if (ATmatch(e, "Relative(<str>)", &s1)) {
string srcPath = searchPath(state.searchDirs, s1);
string dstPath;
FSId id;
addToStore(srcPath, dstPath, id, true);
Path srcPath = searchPath(state.searchDirs, s1);
Path dstPath = addToStore(srcPath);
ClosureElem elem;
elem.id = id;
NixExpr ne;
ne.type = NixExpr::neClosure;
ne.closure.roots.insert(dstPath);
ne.closure.elems[dstPath] = elem;
Hash pkgHash = hashPackage(state, ne);
FSId pkgId = writeTerm(unparseNixExpr(ne), "");
state.pkgHashes[pkgId] = pkgHash;
Path pkgPath = writeTerm(unparseNixExpr(ne), "");
state.pkgHashes[pkgPath] = pkgHash;
msg(lvlChatty, format("copied `%1%' -> %2%")
% srcPath % (string) pkgId);
msg(lvlChatty, format("copied `%1%' -> closure `%2%'")
% srcPath % pkgPath);
return ATmake("FSId(<str>)", ((string) pkgId).c_str());
return ATmake("NixExpr(<str>)", pkgPath.c_str());
}
/* Packages are transformed into Nix derivation expressions. */
@ -302,8 +299,8 @@ static Expr evalExpr2(EvalState & state, Expr e)
ne.type = NixExpr::neDerivation;
ne.derivation.platform = SYSTEM;
string name;
FSId outId;
bool outIdGiven = false;
Hash outHash;
bool outHashGiven = false;
bnds = ATempty;
for (map<string, ATerm>::iterator it = bndMap.begin();
@ -331,8 +328,8 @@ static Expr evalExpr2(EvalState & state, Expr e)
if (key == "build") ne.derivation.builder = s;
if (key == "name") name = s;
if (key == "id") {
outId = parseHash(s);
outIdGiven = true;
outHash = parseHash(s);
outHashGiven = true;
}
}
@ -348,23 +345,23 @@ static Expr evalExpr2(EvalState & state, Expr e)
/* Hash the Nix expression with no outputs to produce a
unique but deterministic path name for this package. */
if (!outIdGiven) outId = hashPackage(state, ne);
string outPath =
canonPath(nixStore + "/" + ((string) outId).c_str() + "-" + name);
if (!outHashGiven) outHash = hashPackage(state, ne);
Path outPath =
canonPath(nixStore + "/" + ((string) outHash).c_str() + "-" + name);
ne.derivation.env["out"] = outPath;
ne.derivation.outputs[outPath] = outId;
ne.derivation.outputs.insert(outPath);
/* Write the resulting term into the Nix store directory. */
Hash pkgHash = outIdGiven
? hashString((string) outId + outPath)
Hash pkgHash = outHashGiven
? hashString((string) outHash + outPath)
: hashPackage(state, ne);
FSId pkgId = writeTerm(unparseNixExpr(ne), "-d-" + name);
state.pkgHashes[pkgId] = pkgHash;
Path pkgPath = writeTerm(unparseNixExpr(ne), "-d-" + name);
state.pkgHashes[pkgPath] = pkgHash;
msg(lvlChatty, format("instantiated `%1%' -> %2%")
% name % (string) pkgId);
msg(lvlChatty, format("instantiated `%1%' -> `%2%'")
% name % pkgPath);
return ATmake("FSId(<str>)", ((string) pkgId).c_str());
return ATmake("NixExpr(<str>)", pkgPath.c_str());
}
/* BaseName primitive function. */
@ -401,9 +398,9 @@ static Expr evalExpr(EvalState & state, Expr e)
}
static Expr evalFile(EvalState & state, string relPath)
static Expr evalFile(EvalState & state, const Path & relPath)
{
string path = searchPath(state.searchDirs, relPath);
Path path = searchPath(state.searchDirs, relPath);
Nest nest(lvlTalkative, format("evaluating file `%1%'") % path);
Expr e = ATreadFromNamedFile(path.c_str());
if (!e)
@ -422,16 +419,16 @@ static Expr evalStdin(EvalState & state)
}
static void printFSId(EvalState & state, Expr e)
static void printNixExpr(EvalState & state, Expr e)
{
ATermList es;
char * s;
if (ATmatch(e, "FSId(<str>)", &s)) {
if (ATmatch(e, "NixExpr(<str>)", &s)) {
cout << format("%1%\n") % s;
}
else if (ATmatch(e, "[<list>]", &es)) {
while (!ATisEmpty(es)) {
printFSId(state, evalExpr(state, ATgetFirst(es)));
printNixExpr(state, evalExpr(state, ATgetFirst(es)));
es = ATgetNext(es);
}
}
@ -472,14 +469,14 @@ void run(Strings args)
if (readStdin) {
Expr e = evalStdin(state);
printFSId(state, e);
printNixExpr(state, e);
}
for (Strings::iterator it = files.begin();
it != files.end(); it++)
{
Expr e = evalFile(state, *it);
printFSId(state, e);
printNixExpr(state, e);
}
}

6
src/globals.cc
View file

@ -5,8 +5,7 @@
Database nixDB;
TableId dbPath2Id;
TableId dbId2Paths;
TableId dbValidPaths;
TableId dbSuccessors;
TableId dbSubstitutes;
@ -23,8 +22,7 @@ bool keepFailed = false;
void openDB()
{
nixDB.open(nixDBPath);
dbPath2Id = nixDB.openTable("path2id");
dbId2Paths = nixDB.openTable("id2paths");
dbValidPaths = nixDB.openTable("validpaths");
dbSuccessors = nixDB.openTable("successors");
dbSubstitutes = nixDB.openTable("substitutes");
}

40
src/globals.hh
View file

@ -13,42 +13,36 @@ extern Database nixDB;
/* Database tables. */
/* dbPath2Id :: Path -> FSId
Each pair (p, id) records that path $p$ contains an expansion of
$id$. */
extern TableId dbPath2Id;
/* dbValidPaths :: Path -> ()
The existence of a key $p$ indicates that path $p$ is valid (that
is, produced by a succesful build). */
extern TableId dbValidPaths;
/* dbId2Paths :: FSId -> [Path]
/* dbSuccessors :: Path -> Path
A mapping from ids to lists of paths. */
extern TableId dbId2Paths;
Each pair $(p_1, p_2)$ in this mapping records the fact that the
Nix expression stored at path $p_1$ has a successor expression
stored at path $p_2$.
/* dbSuccessors :: FSId -> FSId
Each pair $(id_1, id_2)$ in this mapping records the fact that a
successor of a Nix expression stored in a file with identifier
$id_1$ is stored in a file with identifier $id_2$.
Note that a term $y$ is successor of $x$ iff there exists a
Note that a term $y$ is a successor of $x$ iff there exists a
sequence of rewrite steps that rewrites $x$ into $y$.
*/
extern TableId dbSuccessors;
/* dbSubstitutes :: FSId -> [FSId]
/* dbSubstitutes :: Path -> [Path]
Each pair $(id, [ids])$ tells Nix that it can realise any of the
Nix expressions referenced by the identifiers in $ids$ to
generate a path with identifier $id$.
Each pair $(p, [ps])$ tells Nix that it can realise any of the
Nix expressions stored at paths $ps$ to produce a path $p$.
The main purpose of this is for distributed caching of derivates.
One system can compute a derivate with hash $h$ and put it on a
website (as a Nix archive), for instance, and then another system
can register a substitute for that derivate. The substitute in
this case might be a Nix expression that fetches the Nix archive.
One system can compute a derivate and put it on a website (as a Nix
archive), for instance, and then another system can register a
substitute for that derivate. The substitute in this case might be
a Nix expression that fetches the Nix archive.
*/
extern TableId dbSubstitutes;

14
src/hash.cc
View file

@ -54,11 +54,11 @@ Hash parseHash(const string & s)
{
Hash hash;
if (s.length() != Hash::hashSize * 2)
throw BadRefError("invalid hash: " + s);
throw Error(format("invalid hash `%1%'") % s);
for (unsigned int i = 0; i < Hash::hashSize; i++) {
string s2(s, i * 2, 2);
if (!isxdigit(s2[0]) || !isxdigit(s2[1]))
throw BadRefError("invalid hash: " + s);
throw Error(format("invalid hash `%1%'") % s);
istringstream str(s2);
int n;
str >> hex >> n;
@ -89,15 +89,15 @@ Hash hashString(const string & s)
}
Hash hashFile(const string & fileName)
Hash hashFile(const Path & path)
{
Hash hash;
FILE * file = fopen(fileName.c_str(), "rb");
FILE * file = fopen(path.c_str(), "rb");
if (!file)
throw SysError("file `" + fileName + "' does not exist");
throw SysError(format("file `%1%' does not exist") % path);
int err = md5_stream(file, hash.hash);
fclose(file);
if (err) throw SysError("cannot hash file " + fileName);
if (err) throw SysError(format("cannot hash file `%1%'") % path);
return hash;
}
@ -113,7 +113,7 @@ struct HashSink : DumpSink
};
Hash hashPath(const string & path)
Hash hashPath(const Path & path)
{
Hash hash;
HashSink sink;

11
src/hash.hh
View file

@ -30,13 +30,6 @@ struct Hash
};
class BadRefError : public Error
{
public:
BadRefError(string _err) : Error(_err) { };
};
/* Parse a hexadecimal representation of a hash code. */
Hash parseHash(const string & s);
@ -47,12 +40,12 @@ bool isHash(const string & s);
Hash hashString(const string & s);
/* Compute the hash of the given file. */
Hash hashFile(const string & fileName);
Hash hashFile(const Path & path);
/* Compute the hash of the given path. The hash is defined as
md5(dump(path)).
*/
Hash hashPath(const string & path);
Hash hashPath(const Path & path);
#endif /* !__HASH_H */

5
src/nix-help.txt
View file

@ -19,16 +19,11 @@ Operations:
--version: output version information
--help: display help
Source selection for --install, --dump:
--path / -p: by file name !!! -> path
Query flags:
--list / -l: query the output paths (roots) of a Nix expression (default)
--requisites / -r: print all paths necessary to realise expression
--generators / -g: find expressions producing a subset of given ids
--expansion / -e: print a path containing id
--graph: print a dot graph rooted at given ids
Options:

86
src/nix.cc
View file

@ -11,9 +11,6 @@
typedef void (* Operation) (Strings opFlags, Strings opArgs);
static bool pathArgs = false;
static void printHelp()
{
cout <<
@ -24,16 +21,9 @@ static void printHelp()
static FSId argToId(const string & arg)
static Path checkPath(const Path & arg)
{
if (!pathArgs)
return parseHash(arg);
else {
FSId id;
if (!queryPathId(arg, id))
throw Error(format("don't know id of `%1%'") % arg);
return id;
}
return arg; /* !!! check that arg is in the store */
}
@ -42,12 +32,12 @@ static void opInstall(Strings opFlags, Strings opArgs)
{
if (!opFlags.empty()) throw UsageError("unknown flag");
for (Strings::iterator it = opArgs.begin();
it != opArgs.end(); it++)
for (Strings::iterator i = opArgs.begin();
i != opArgs.end(); i++)
{
FSId id = normaliseNixExpr(argToId(*it));
realiseClosure(id);
cout << format("%1%\n") % (string) id;
Path nfPath = normaliseNixExpr(checkPath(*i));
realiseClosure(nfPath);
cout << format("%1%\n") % (string) nfPath;
}
}
@ -59,7 +49,7 @@ static void opDelete(Strings opFlags, Strings opArgs)
for (Strings::iterator it = opArgs.begin();
it != opArgs.end(); it++)
deleteFromStore(absPath(*it));
deleteFromStore(checkPath(*it));
}
@ -69,27 +59,21 @@ static void opAdd(Strings opFlags, Strings opArgs)
{
if (!opFlags.empty()) throw UsageError("unknown flag");
for (Strings::iterator it = opArgs.begin();
it != opArgs.end(); it++)
{
string path;
FSId id;
addToStore(*it, path, id);
cout << format("%1% %2%\n") % (string) id % path;
}
for (Strings::iterator i = opArgs.begin(); i != opArgs.end(); i++)
cout << format("%1%\n") % addToStore(*i);
}
FSId maybeNormalise(const FSId & id, bool normalise)
Path maybeNormalise(const Path & ne, bool normalise)
{
return normalise ? normaliseNixExpr(id) : id;
return normalise ? normaliseNixExpr(ne) : ne;
}
/* Perform various sorts of queries. */
static void opQuery(Strings opFlags, Strings opArgs)
{
enum { qList, qRequisites, qGenerators, qExpansion, qGraph
enum { qList, qRequisites, qGenerators, qGraph
} query = qList;
bool normalise = false;
bool includeExprs = true;
@ -100,7 +84,6 @@ static void opQuery(Strings opFlags, Strings opArgs)
if (*i == "--list" || *i == "-l") query = qList;
else if (*i == "--requisites" || *i == "-r") query = qRequisites;
else if (*i == "--generators" || *i == "-g") query = qGenerators;
else if (*i == "--expansion" || *i == "-e") query = qExpansion;
else if (*i == "--graph") query = qGraph;
else if (*i == "--normalise" || *i == "-n") normalise = true;
else if (*i == "--exclude-exprs") includeExprs = false;
@ -110,12 +93,12 @@ static void opQuery(Strings opFlags, Strings opArgs)
switch (query) {
case qList: {
StringSet paths;
PathSet paths;
for (Strings::iterator i = opArgs.begin();
i != opArgs.end(); i++)
{
Strings paths2 = nixExprPaths(
maybeNormalise(argToId(*i), normalise));
StringSet paths2 = nixExprRoots(
maybeNormalise(checkPath(*i), normalise));
paths.insert(paths2.begin(), paths2.end());
}
for (StringSet::iterator i = paths.begin();
@ -129,8 +112,8 @@ static void opQuery(Strings opFlags, Strings opArgs)
for (Strings::iterator i = opArgs.begin();
i != opArgs.end(); i++)
{
Strings paths2 = nixExprRequisites(
maybeNormalise(argToId(*i), normalise),
StringSet paths2 = nixExprRequisites(
maybeNormalise(checkPath(*i), normalise),
includeExprs, includeSuccessors);
paths.insert(paths2.begin(), paths2.end());
}
@ -140,11 +123,12 @@ static void opQuery(Strings opFlags, Strings opArgs)
break;
}
#if 0
case qGenerators: {
FSIds outIds;
for (Strings::iterator i = opArgs.begin();
i != opArgs.end(); i++)
outIds.push_back(argToId(*i));
outIds.push_back(checkPath(*i));
FSIds genIds = findGenerators(outIds);
@ -153,21 +137,13 @@ static void opQuery(Strings opFlags, Strings opArgs)
cout << format("%s\n") % expandId(*i);
break;
}
case qExpansion: {
for (Strings::iterator i = opArgs.begin();
i != opArgs.end(); i++)
/* !!! should not use substitutes; this is a query,
it should not have side-effects */
cout << format("%s\n") % expandId(parseHash(*i));
break;
}
#endif
case qGraph: {
FSIds roots;
PathSet roots;
for (Strings::iterator i = opArgs.begin();
i != opArgs.end(); i++)
roots.push_back(maybeNormalise(argToId(*i), normalise));
roots.insert(maybeNormalise(checkPath(*i), normalise));
printDotGraph(roots);
break;
}
@ -187,9 +163,9 @@ static void opSuccessor(Strings opFlags, Strings opArgs)
for (Strings::iterator i = opArgs.begin();
i != opArgs.end(); )
{
FSId id1 = parseHash(*i++);
FSId id2 = parseHash(*i++);
registerSuccessor(txn, id1, id2);
Path path1 = checkPath(*i++);
Path path2 = checkPath(*i++);
registerSuccessor(txn, path1, path2);
}
txn.commit();
}
@ -203,8 +179,8 @@ static void opSubstitute(Strings opFlags, Strings opArgs)
for (Strings::iterator i = opArgs.begin();
i != opArgs.end(); )
{
FSId src = parseHash(*i++);
FSId sub = parseHash(*i++);
Path src = checkPath(*i++);
Path sub = checkPath(*i++);
registerSubstitute(src, sub);
}
}
@ -229,9 +205,7 @@ static void opDump(Strings opFlags, Strings opArgs)
if (opArgs.size() != 1) throw UsageError("only one argument allowed");
StdoutSink sink;
string arg = *opArgs.begin();
string path = pathArgs ? arg : expandId(parseHash(arg));
string path = *opArgs.begin();
dumpPath(path, sink);
}
@ -311,8 +285,6 @@ void run(Strings args)
op = opInit;
else if (arg == "--verify")
op = opVerify;
else if (arg == "--path" || arg == "-p")
pathArgs = true;
else if (arg == "--verbose" || arg == "-v")
verbosity = (Verbosity) ((int) verbosity + 1);
else if (arg == "--keep-failed" || arg == "-K")

285
src/normalise.cc
View file

@ -9,47 +9,131 @@
void registerSuccessor(const Transaction & txn,
const FSId & id1, const FSId & id2)
const Path & path1, const Path & path2)
{
nixDB.setString(txn, dbSuccessors, id1, id2);
nixDB.setString(txn, dbSuccessors, path1, path2);
}
static FSId useSuccessor(const FSId & id)
static Path useSuccessor(const Path & path)
{
string idSucc;
if (nixDB.queryString(noTxn, dbSuccessors, id, idSucc)) {
debug(format("successor %1% -> %2%") % (string) id % idSucc);
return parseHash(idSucc);
string pathSucc;
if (nixDB.queryString(noTxn, dbSuccessors, path, pathSucc)) {
debug(format("successor %1% -> %2%") % (string) path % pathSucc);
return pathSucc;
} else
return id;
return path;
}
Strings pathsFromOutputs(const DerivationOutputs & ps)
#if 0
/* Return a path whose contents have the given hash. If target is
not empty, ensure that such a path is realised in target (if
necessary by copying from another location). If prefix is not
empty, only return a path that is an descendent of prefix. */
string expandId(const FSId & id, const string & target = "",
const string & prefix = "/", FSIdSet pending = FSIdSet(),
bool ignoreSubstitutes = false)
{
Strings ss;
for (DerivationOutputs::const_iterator i = ps.begin();
i != ps.end(); i++)
ss.push_back(i->first);
return ss;
xxx
}
FSId normaliseNixExpr(FSId id, FSIdSet pending)
string expandId(const FSId & id, const string & target,
const string & prefix, FSIdSet pending, bool ignoreSubstitutes)
{
Nest nest(lvlDebug, format("expanding %1%") % (string) id);
Strings paths;
if (!target.empty() && !isInPrefix(target, prefix))
abort();
nixDB.queryStrings(noTxn, dbId2Paths, id, paths);
/* Pick one equal to `target'. */
if (!target.empty()) {
for (Strings::iterator i = paths.begin();
i != paths.end(); i++)
{
string path = *i;
if (path == target && pathExists(path))
return path;
}
}
/* Arbitrarily pick the first one that exists and isn't stale. */
for (Strings::iterator it = paths.begin();
it != paths.end(); it++)
{
string path = *it;
if (isInPrefix(path, prefix) && pathExists(path)) {
if (target.empty())
return path;
else {
/* Acquire a lock on the target path. */
Strings lockPaths;
lockPaths.push_back(target);
PathLocks outputLock(lockPaths);
/* Copy. */
copyPath(path, target);
/* Register the target path. */
Transaction txn(nixDB);
registerPath(txn, target, id);
txn.commit();
return target;
}
}
}
if (!ignoreSubstitutes) {
if (pending.find(id) != pending.end())
throw Error(format("id %1% already being expanded") % (string) id);
pending.insert(id);
/* Try to realise the substitutes, but only if this id is not
already being realised by a substitute. */
Strings subs;
nixDB.queryStrings(noTxn, dbSubstitutes, id, subs); /* non-existence = ok */
for (Strings::iterator it = subs.begin(); it != subs.end(); it++) {
FSId subId = parseHash(*it);
debug(format("trying substitute %1%") % (string) subId);
realiseClosure(normaliseNixExpr(subId, pending), pending);
return expandId(id, target, prefix, pending);
}
}
throw Error(format("cannot expand id `%1%'") % (string) id);
}
#endif
Path normaliseNixExpr(const Path & _nePath, PathSet pending)
{
Nest nest(lvlTalkative,
format("normalising nix expression %1%") % (string) id);
format("normalising expression in `%1%'") % (string) _nePath);
/* Try to substitute $id$ by any known successors in order to
speed up the rewrite process. */
id = useSuccessor(id);
/* Try to substitute the expression by any known successors in
order to speed up the rewrite process. */
Path nePath = useSuccessor(_nePath);
/* Get the Nix expression. */
NixExpr ne = parseNixExpr(termFromId(id));
NixExpr ne = parseNixExpr(termFromPath(nePath));
/* If this is a normal form (i.e., a closure) we are done. */
if (ne.type == NixExpr::neClosure) return id;
if (ne.type == NixExpr::neClosure) return nePath;
if (ne.type != NixExpr::neDerivation) abort();
@ -62,8 +146,8 @@ FSId normaliseNixExpr(FSId id, FSIdSet pending)
/* Input paths, with their closure elements. */
ClosureElems inClosures;
/* Referencable paths (i.e., input and output paths). */
StringSet allPaths;
/* Referenceable paths (i.e., input and output paths). */
PathSet allPaths;
/* The environment to be passed to the builder. */
Environment env;
@ -73,17 +157,17 @@ FSId normaliseNixExpr(FSId id, FSIdSet pending)
nf.type = NixExpr::neClosure;
/* Parse the outputs. */
for (DerivationOutputs::iterator i = ne.derivation.outputs.begin();
/* The outputs are referenceable paths. */
for (PathSet::iterator i = ne.derivation.outputs.begin();
i != ne.derivation.outputs.end(); i++)
{
debug(format("building %1% in `%2%'") % (string) i->second % i->first);
allPaths.insert(i->first);
debug(format("building path `%1%'") % *i);
allPaths.insert(*i);
}
/* Obtain locks on all output paths. The locks are automatically
released when we exit this function or Nix crashes. */
PathLocks outputLocks(pathsFromOutputs(ne.derivation.outputs));
PathLocks outputLocks(ne.derivation.outputs);
/* Now check again whether there is a successor. This is because
another process may have started building in parallel. After
@ -94,13 +178,13 @@ FSId normaliseNixExpr(FSId id, FSIdSet pending)
other process can build this expression, so no further checks
are necessary. */
{
FSId id2 = useSuccessor(id);
if (id2 != id) {
NixExpr ne = parseNixExpr(termFromId(id2));
debug(format("skipping build of %1%, someone beat us to it")
% (string) id);
Path nePath2 = useSuccessor(nePath);
if (nePath != nePath2) {
NixExpr ne = parseNixExpr(termFromPath(nePath2));
debug(format("skipping build of expression `%1%', someone beat us to it")
% (string) nePath);
if (ne.type != NixExpr::neClosure) abort();
return id2;
return nePath2;
}
}
@ -110,14 +194,14 @@ FSId normaliseNixExpr(FSId id, FSIdSet pending)
% ne.derivation.platform % thisSystem);
/* Realise inputs (and remember all input paths). */
for (FSIdSet::iterator i = ne.derivation.inputs.begin();
for (PathSet::iterator i = ne.derivation.inputs.begin();
i != ne.derivation.inputs.end(); i++)
{
FSId nf = normaliseNixExpr(*i, pending);
realiseClosure(nf, pending);
/* !!! nf should be a root of the garbage collector while we
are building */
NixExpr ne = parseNixExpr(termFromId(nf));
Path nfPath = normaliseNixExpr(*i, pending);
realiseClosure(nfPath, pending);
/* !!! nfPath should be a root of the garbage collector while
we are building */
NixExpr ne = parseNixExpr(termFromPath(nfPath));
if (ne.type != NixExpr::neClosure) abort();
for (ClosureElems::iterator j = ne.closure.elems.begin();
j != ne.closure.elems.end(); j++)
@ -147,8 +231,10 @@ FSId normaliseNixExpr(FSId id, FSIdSet pending)
/* We can skip running the builder if we can expand all output
paths from their ids. */
bool fastBuild = false;
#if 0
bool fastBuild = true;
for (DerivationOutputs::iterator i = ne.derivation.outputs.begin();
for (PathSet::iterator i = ne.derivation.outputs.begin();
i != ne.derivation.outputs.end(); i++)
{
try {
@ -160,17 +246,17 @@ FSId normaliseNixExpr(FSId id, FSIdSet pending)
break;
}
}
#endif
if (!fastBuild) {
/* If any of the outputs already exist but are not registered,
delete them. */
for (DerivationOutputs::iterator i = ne.derivation.outputs.begin();
for (PathSet::iterator i = ne.derivation.outputs.begin();
i != ne.derivation.outputs.end(); i++)
{
string path = i->first;
FSId id;
if (queryPathId(path, id))
Path path = *i;
if (isValidPath(path))
throw Error(format("obstructed build: path `%1%' exists") % path);
if (pathExists(path)) {
debug(format("removing unregistered path `%1%'") % path);
@ -189,11 +275,11 @@ FSId normaliseNixExpr(FSId id, FSIdSet pending)
/* Check whether the output paths were created, and grep each
output path to determine what other paths it references. Also make all
output paths read-only. */
StringSet usedPaths;
for (DerivationOutputs::iterator i = ne.derivation.outputs.begin();
PathSet usedPaths;
for (PathSet::iterator i = ne.derivation.outputs.begin();
i != ne.derivation.outputs.end(); i++)
{
string path = i->first;
Path path = *i;
if (!pathExists(path))
throw Error(format("path `%1%' does not exist") % path);
nf.closure.roots.insert(path);
@ -207,15 +293,14 @@ FSId normaliseNixExpr(FSId id, FSIdSet pending)
/* Construct a closure element for this output path. */
ClosureElem elem;
elem.id = i->second;
/* For each path referenced by this output path, add its id to the
closure element and add the id to the `usedPaths' set (so that the
elements referenced by *its* closure are added below). */
for (Strings::iterator j = refPaths.begin();
for (Paths::iterator j = refPaths.begin();
j != refPaths.end(); j++)
{
string path = *j;
Path path = *j;
elem.refs.insert(path);
if (inClosures.find(path) != inClosures.end())
usedPaths.insert(path);
@ -228,11 +313,11 @@ FSId normaliseNixExpr(FSId id, FSIdSet pending)
/* Close the closure. That is, for any referenced path, add the paths
referenced by it. */
StringSet donePaths;
PathSet donePaths;
while (!usedPaths.empty()) {
StringSet::iterator i = usedPaths.begin();
string path = *i;
PathSet::iterator i = usedPaths.begin();
Path path = *i;
usedPaths.erase(i);
if (donePaths.find(path) != donePaths.end()) continue;
@ -243,7 +328,7 @@ FSId normaliseNixExpr(FSId id, FSIdSet pending)
nf.closure.elems[path] = j->second;
for (StringSet::iterator k = j->second.refs.begin();
for (PathSet::iterator k = j->second.refs.begin();
k != j->second.refs.end(); k++)
usedPaths.insert(*k);
}
@ -252,7 +337,7 @@ FSId normaliseNixExpr(FSId id, FSIdSet pending)
for (ClosureElems::iterator i = inClosures.begin();
i != inClosures.end(); i++)
{
StringSet::iterator j = donePaths.find(i->first);
PathSet::iterator j = donePaths.find(i->first);
if (j == donePaths.end())
debug(format("NOT referenced: `%1%'") % i->first);
else
@ -263,7 +348,7 @@ FSId normaliseNixExpr(FSId id, FSIdSet pending)
transaction below because writing terms is idem-potent. */
ATerm nfTerm = unparseNixExpr(nf);
msg(lvlVomit, format("normal form: %1%") % printTerm(nfTerm));
FSId idNF = writeTerm(nfTerm, "-s-" + (string) id);
Path nfPath = writeTerm(nfTerm, "-s");
/* Register each outpat path, and register the normal form. This
is wrapped in one database transaction to ensure that if we
@ -272,63 +357,58 @@ FSId normaliseNixExpr(FSId id, FSIdSet pending)
deleted arbitrarily, while registered paths can only be deleted
by running the garbage collector. */
Transaction txn(nixDB);
for (DerivationOutputs::iterator i = ne.derivation.outputs.begin();
for (PathSet::iterator i = ne.derivation.outputs.begin();
i != ne.derivation.outputs.end(); i++)
registerPath(txn, i->first, i->second);
registerSuccessor(txn, id, idNF);
registerValidPath(txn, *i);
registerSuccessor(txn, nePath, nfPath);
txn.commit();
return idNF;
return nfPath;
}
void realiseClosure(const FSId & id, FSIdSet pending)
void realiseClosure(const Path & nePath, PathSet pending)
{
Nest nest(lvlDebug,
format("realising closure %1%") % (string) id);
Nest nest(lvlDebug, format("realising closure `%1%'") % nePath);
NixExpr ne = parseNixExpr(termFromId(id));
NixExpr ne = parseNixExpr(termFromPath(nePath));
if (ne.type != NixExpr::neClosure)
throw Error(format("expected closure in %1%") % (string) id);
throw Error(format("expected closure in `%1%'") % nePath);
for (ClosureElems::const_iterator i = ne.closure.elems.begin();
i != ne.closure.elems.end(); i++)
assert(isValidPath(i->first));
#if 0
expandId(i->second.id, i->first, "/", pending);
#endif
}
Strings nixExprPaths(const FSId & id)
PathSet nixExprRoots(const Path & nePath)
{
Strings paths;
PathSet paths;
NixExpr ne = parseNixExpr(termFromId(id));
if (ne.type == NixExpr::neClosure) {
for (StringSet::const_iterator i = ne.closure.roots.begin();
i != ne.closure.roots.end(); i++)
paths.push_back(*i);
}
else if (ne.type == NixExpr::neDerivation) {
for (DerivationOutputs::iterator i = ne.derivation.outputs.begin();
i != ne.derivation.outputs.end(); i++)
paths.push_back(i->first);
}
NixExpr ne = parseNixExpr(termFromPath(nePath));
if (ne.type == NixExpr::neClosure)
paths.insert(ne.closure.roots.begin(), ne.closure.roots.end());
else if (ne.type == NixExpr::neDerivation)
paths.insert(ne.derivation.outputs.begin(),
ne.derivation.outputs.end());
else abort();
return paths;
}
static void nixExprRequisitesSet(const FSId & id,
bool includeExprs, bool includeSuccessors, StringSet & paths,
FSIdSet & doneSet)
static void requisitesWorker(const Path & nePath,
bool includeExprs, bool includeSuccessors,
PathSet & paths, PathSet & doneSet)
{
if (doneSet.find(id) != doneSet.end()) return;
doneSet.insert(id);
if (doneSet.find(nePath) != doneSet.end()) return;
doneSet.insert(nePath);
NixExpr ne = parseNixExpr(termFromId(id));
NixExpr ne = parseNixExpr(termFromPath(nePath));
if (ne.type == NixExpr::neClosure)
for (ClosureElems::iterator i = ne.closure.elems.begin();
@ -336,35 +416,35 @@ static void nixExprRequisitesSet(const FSId & id,
paths.insert(i->first);
else if (ne.type == NixExpr::neDerivation)
for (FSIdSet::iterator i = ne.derivation.inputs.begin();
for (PathSet::iterator i = ne.derivation.inputs.begin();
i != ne.derivation.inputs.end(); i++)
nixExprRequisitesSet(*i,
requisitesWorker(*i,
includeExprs, includeSuccessors, paths, doneSet);
else abort();
if (includeExprs)
paths.insert(expandId(id));
if (includeExprs) paths.insert(nePath);
string idSucc;
if (includeSuccessors &&
nixDB.queryString(noTxn, dbSuccessors, id, idSucc))
nixExprRequisitesSet(parseHash(idSucc),
includeExprs, includeSuccessors, paths, doneSet);
string nfPath;
if (includeSuccessors && (nfPath = useSuccessor(nePath)) != nePath)
requisitesWorker(nfPath, includeExprs, includeSuccessors,
paths, doneSet);
}
Strings nixExprRequisites(const FSId & id,
PathSet nixExprRequisites(const Path & nePath,
bool includeExprs, bool includeSuccessors)
{
StringSet paths;
FSIdSet doneSet;
nixExprRequisitesSet(id, includeExprs, includeSuccessors, paths, doneSet);
return Strings(paths.begin(), paths.end());
PathSet paths;
PathSet doneSet;
requisitesWorker(nePath, includeExprs, includeSuccessors,
paths, doneSet);
return paths;
}
FSIds findGenerators(const FSIds & _ids)
#if 0
PathSet findGenerators(const PathSet & outputs)
{
FSIdSet ids(_ids.begin(), _ids.end());
FSIds generators;
@ -407,3 +487,4 @@ FSIds findGenerators(const FSIds & _ids)
return generators;
}
#endif

29
src/normalise.hh
View file

@ -4,15 +4,22 @@
#include "expr.hh"
/* Normalise a Nix expression, that is, return an equivalent
closure. (For the meaning of `pending', see expandId()). */
FSId normaliseNixExpr(FSId id, FSIdSet pending = FSIdSet());
/* Normalise a Nix expression. That is, if the expression is a
derivation, a path containing an equivalent closure expression is
returned. This requires that the derivation is performed, unless a
successor is known. */
Path normaliseNixExpr(const Path & nePath, PathSet pending = PathSet());
/* Realise a Closure in the file system. */
void realiseClosure(const FSId & id, FSIdSet pending = FSIdSet());
/* Realise a closure expression in the file system.
The pending paths are those that are already being realised. This
prevents infinite recursion for paths realised through a substitute
(since when we build the substitute, we would first try to realise
its output paths through substitutes... kaboom!). */
void realiseClosure(const Path & nePath, PathSet pending = PathSet());
/* Get the list of root (output) paths of the given Nix expression. */
Strings nixExprPaths(const FSId & id);
PathSet nixExprRoots(const Path & nePath);
/* Get the list of paths that are required to realise the given
expression. For a derive expression, this is the union of
@ -20,16 +27,16 @@ Strings nixExprPaths(const FSId & id);
each element in the closure. If `includeExprs' is true, include the
paths of the Nix expressions themselves. If `includeSuccessors' is
true, include the requisites of successors. */
Strings nixExprRequisites(const FSId & id,
PathSet nixExprRequisites(const Path & nePath,
bool includeExprs, bool includeSuccessors);
/* Return the list of the ids of all known Nix expressions whose
output ids are completely contained in `ids'. */
FSIds findGenerators(const FSIds & ids);
/* Return the list of the paths of all known Nix expressions whose
output paths are completely contained in the set `outputs'. */
PathSet findGenerators(const PathSet & outputs);
/* Register a successor. */
void registerSuccessor(const Transaction & txn,
const FSId & id1, const FSId & id2);
const Path & path1, const Path & path2);
#endif /* !__NORMALISE_H */

12
src/pathlocks.cc
View file

@ -13,20 +13,20 @@
static StringSet lockedPaths; /* !!! not thread-safe */
PathLocks::PathLocks(const Strings & _paths)
PathLocks::PathLocks(const PathSet & _paths)
{
/* Note that `fds' is built incrementally so that the destructor
will only release those locks that we have already acquired. */
/* Sort the paths. This assures that locks are always acquired in
the same order, thus preventing deadlocks. */
Strings paths(_paths);
Paths paths(_paths.begin(), _paths.end());
paths.sort();
/* Acquire the lock for each path. */
for (Strings::iterator i = paths.begin(); i != paths.end(); i++) {
string path = *i;
string lockPath = path + ".lock";
for (Paths::iterator i = paths.begin(); i != paths.end(); i++) {
Path path = *i;
Path lockPath = path + ".lock";
debug(format("locking path `%1%'") % path);
@ -64,6 +64,6 @@ PathLocks::~PathLocks()
for (list<int>::iterator i = fds.begin(); i != fds.end(); i++)
close(*i);
for (Strings::iterator i = paths.begin(); i != paths.end(); i++)
for (Paths::iterator i = paths.begin(); i != paths.end(); i++)
lockedPaths.erase(*i);
}

4
src/pathlocks.hh
View file

@ -8,10 +8,10 @@ class PathLocks
{
private:
list<int> fds;
Strings paths;
Paths paths;
public:
PathLocks(const Strings & _paths);
PathLocks(const PathSet & _paths);
~PathLocks();
};

2
src/references.hh
View file

@ -4,7 +4,7 @@
#include "util.hh"
Strings filterReferences(const string & path, const Strings & refs);
Strings filterReferences(const Path & path, const Strings & refs);
#endif /* !__VALUES_H */

2
src/shared.cc
View file

@ -72,5 +72,3 @@ int main(int argc, char * * argv)
return 0;
}

208
src/store.cc
View file

@ -8,7 +8,6 @@
#include "db.hh"
#include "archive.hh"
#include "pathlocks.hh"
#include "normalise.hh"
struct CopySink : DumpSink
@ -31,7 +30,7 @@ struct CopySource : RestoreSource
};
void copyPath(string src, string dst)
void copyPath(const Path & src, const Path & dst)
{
debug(format("copying `%1%' to `%2%'") % src % dst);
@ -82,7 +81,7 @@ void copyPath(string src, string dst)
}
void registerSubstitute(const FSId & srcId, const FSId & subId)
void registerSubstitute(const Path & srcPath, const Path & subPath)
{
#if 0
Strings subs;
@ -98,202 +97,89 @@ void registerSubstitute(const FSId & srcId, const FSId & subId)
/* For now, accept only one substitute per id. */
Strings subs;
subs.push_back(subId);
subs.push_back(subPath);
Transaction txn(nixDB);
nixDB.setStrings(txn, dbSubstitutes, srcId, subs);
nixDB.setStrings(txn, dbSubstitutes, srcPath, subs);
txn.commit();
}
void registerPath(const Transaction & txn,
const string & _path, const FSId & id)
void registerValidPath(const Transaction & txn, const Path & _path)
{
string path(canonPath(_path));
debug(format("registering path `%1%' with id %2%")
% path % (string) id);
string oldId;
if (nixDB.queryString(txn, dbPath2Id, path, oldId)) {
if (id != parseHash(oldId))
throw Error(format("path `%1%' already contains id %2%")
% path % oldId);
return;
}
nixDB.setString(txn, dbPath2Id, path, id);
Strings paths;
nixDB.queryStrings(txn, dbId2Paths, id, paths); /* non-existence = ok */
paths.push_back(path);
nixDB.setStrings(txn, dbId2Paths, id, paths);
Path path(canonPath(_path));
debug(format("registering path `%1%'") % path);
nixDB.setString(txn, dbValidPaths, path, "");
}
void unregisterPath(const string & _path)
bool isValidPath(const Path & path)
{
string path(canonPath(_path));
string s;
return nixDB.queryString(noTxn, dbValidPaths, path, s);
}
void unregisterValidPath(const Path & _path)
{
Path path(canonPath(_path));
Transaction txn(nixDB);
debug(format("unregistering path `%1%'") % path);
string _id;
if (!nixDB.queryString(txn, dbPath2Id, path, _id)) {
txn.abort();
return;
}
FSId id(parseHash(_id));
nixDB.delPair(txn, dbPath2Id, path);
Strings paths, paths2;
nixDB.queryStrings(txn, dbId2Paths, id, paths); /* non-existence = ok */
for (Strings::iterator it = paths.begin();
it != paths.end(); it++)
if (*it != path) paths2.push_back(*it);
nixDB.setStrings(txn, dbId2Paths, id, paths2);
nixDB.delPair(txn, dbValidPaths, path);
txn.commit();
}
bool queryPathId(const string & path, FSId & id)
{
string s;
if (!nixDB.queryString(noTxn, dbPath2Id, absPath(path), s)) return false;
id = parseHash(s);
return true;
}
bool isInPrefix(const string & path, const string & _prefix)
static bool isInPrefix(const string & path, const string & _prefix)
{
string prefix = canonPath(_prefix + "/");
return string(path, 0, prefix.size()) == prefix;
}
string expandId(const FSId & id, const string & target,
const string & prefix, FSIdSet pending, bool ignoreSubstitutes)
{
Nest nest(lvlDebug, format("expanding %1%") % (string) id);
Strings paths;
if (!target.empty() && !isInPrefix(target, prefix))
abort();
nixDB.queryStrings(noTxn, dbId2Paths, id, paths);
/* Pick one equal to `target'. */
if (!target.empty()) {
for (Strings::iterator i = paths.begin();
i != paths.end(); i++)
{
string path = *i;
if (path == target && pathExists(path))
return path;
}
}
/* Arbitrarily pick the first one that exists and isn't stale. */
for (Strings::iterator it = paths.begin();
it != paths.end(); it++)
{
string path = *it;
if (isInPrefix(path, prefix) && pathExists(path)) {
if (target.empty())
return path;
else {
/* Acquire a lock on the target path. */
Strings lockPaths;
lockPaths.push_back(target);
PathLocks outputLock(lockPaths);
/* Copy. */
copyPath(path, target);
/* Register the target path. */
Transaction txn(nixDB);
registerPath(txn, target, id);
txn.commit();
return target;
}
}
}
if (!ignoreSubstitutes) {
if (pending.find(id) != pending.end())
throw Error(format("id %1% already being expanded") % (string) id);
pending.insert(id);
/* Try to realise the substitutes, but only if this id is not
already being realised by a substitute. */
Strings subs;
nixDB.queryStrings(noTxn, dbSubstitutes, id, subs); /* non-existence = ok */
for (Strings::iterator it = subs.begin(); it != subs.end(); it++) {
FSId subId = parseHash(*it);
debug(format("trying substitute %1%") % (string) subId);
realiseClosure(normaliseNixExpr(subId, pending), pending);
return expandId(id, target, prefix, pending);
}
}
throw Error(format("cannot expand id `%1%'") % (string) id);
}
void addToStore(string srcPath, string & dstPath, FSId & id,
bool deterministicName)
Path addToStore(const Path & _srcPath)
{
Path srcPath(absPath(_srcPath));
debug(format("adding `%1%' to the store") % srcPath);
srcPath = absPath(srcPath);
id = hashPath(srcPath);
Hash h = hashPath(srcPath);
string baseName = baseNameOf(srcPath);
dstPath = canonPath(nixStore + "/" + (string) id + "-" + baseName);
Path dstPath = canonPath(nixStore + "/" + (string) h + "-" + baseName);
try {
dstPath = expandId(id, deterministicName ? dstPath : "",
nixStore, FSIdSet(), true);
return;
} catch (...) {
if (!isValidPath(dstPath)) {
/* The first check above is an optimisation to prevent
unnecessary lock acquisition. */
PathSet lockPaths;
lockPaths.insert(dstPath);
PathLocks outputLock(lockPaths);
if (!isValidPath(dstPath)) {
copyPath(srcPath, dstPath);
Transaction txn(nixDB);
registerValidPath(txn, dstPath);
txn.commit();
}
}
Strings lockPaths;
lockPaths.push_back(dstPath);
PathLocks outputLock(lockPaths);
copyPath(srcPath, dstPath);
Transaction txn(nixDB);
registerPath(txn, dstPath, id);
txn.commit();
return dstPath;
}
void deleteFromStore(const string & path)
void deleteFromStore(const Path & _path)
{
string prefix = + "/";
if (!isInPrefix(path, nixStore))
throw Error(format("path %1% is not in the store") % path);
Path path(canonPath(_path));
unregisterPath(path);
if (!isInPrefix(path, nixStore))
throw Error(format("path `%1%' is not in the store") % path);
unregisterValidPath(path);
deletePath(path);
}
@ -305,6 +191,7 @@ void verifyStore()
/* !!! verify that the result is consistent */
#if 0
Strings paths;
nixDB.enumTable(txn, dbPath2Id, paths);
@ -421,6 +308,7 @@ void verifyStore()
}
}
}
#endif
txn.commit();
}

41
src/store.hh
View file

@ -9,44 +9,27 @@
using namespace std;
typedef Hash FSId;
typedef set<FSId> FSIdSet;
/* Copy a path recursively. */
void copyPath(string src, string dst);
void copyPath(const Path & src, const Path & dst);
/* Register a substitute. */
void registerSubstitute(const FSId & srcId, const FSId & subId);
void registerSubstitute(const Path & srcPath, const Path & subPath);
/* Register a path keyed on its id. */
void registerPath(const Transaction & txn,
const string & path, const FSId & id);
/* Register the validity of a path. */
void registerValidPath(const Transaction & txn, const Path & path);
/* Query the id of a path. */
bool queryPathId(const string & path, FSId & id);
/* Unregister the validity of a path. */
void unregisterValidPath(const Path & path);
/* Return a path whose contents have the given hash. If target is
not empty, ensure that such a path is realised in target (if
necessary by copying from another location). If prefix is not
empty, only return a path that is an descendent of prefix.
/* Checks whether a path is valid. */
bool isValidPath(const Path & path);
The list of pending ids are those that already being expanded.
This prevents infinite recursion for ids realised through a
substitute (since when we build the substitute, we would first try
to expand the id... kaboom!). */
string expandId(const FSId & id, const string & target = "",
const string & prefix = "/", FSIdSet pending = FSIdSet(),
bool ignoreSubstitutes = false);
/* Copy a file to the nixStore directory and register it in dbRefs.
Return the hash code of the value. */
void addToStore(string srcPath, string & dstPath, FSId & id,
bool deterministicName = false);
/* Copy the contents of a path to the store and register the validity
the resulting path. The resulting path is returned. */
Path addToStore(const Path & srcPath);
/* Delete a value from the nixStore directory. */
void deleteFromStore(const string & path);
void deleteFromStore(const Path & path);
void verifyStore();

85
src/test.cc
View file

@ -10,10 +10,10 @@
#include "globals.hh"
void realise(FSId id)
void realise(Path nePath)
{
Nest nest(lvlDebug, format("TEST: realising %1%") % (string) id);
realiseClosure(normaliseNixExpr(id));
Nest nest(lvlDebug, format("TEST: realising `%1%'") % nePath);
realiseClosure(normaliseNixExpr(nePath));
}
@ -48,12 +48,12 @@ void runTests()
try {
h = parseHash("blah blah");
abort();
} catch (BadRefError err) { };
} catch (Error err) { };
try {
h = parseHash("0b0ffd0538622bfe20b92c4aa57254d99");
abort();
} catch (BadRefError err) { };
} catch (Error err) { };
/* Path canonicalisation. */
cout << canonPath("/./../././//") << endl;
@ -97,76 +97,59 @@ void runTests()
/* Expression evaluation. */
FSId builder1id;
string builder1fn;
addToStore("./test-builder-1.sh", builder1fn, builder1id);
Path builder1fn;
builder1fn = addToStore("./test-builder-1.sh");
ATerm fs1 = ATmake(
"Closure([<str>], [(<str>, <str>, [])])",
"Closure([<str>], [(<str>, [])])",
builder1fn.c_str(),
builder1fn.c_str(),
((string) builder1id).c_str());
FSId fs1id = writeTerm(fs1, "");
builder1fn.c_str());
Path fs1ne = writeTerm(fs1, "-c");
realise(fs1id);
realise(fs1id);
realise(fs1ne);
realise(fs1ne);
ATerm fs2 = ATmake(
"Closure([<str>], [(<str>, <str>, [])])",
(builder1fn + "_bla").c_str(),
(builder1fn + "_bla").c_str(),
((string) builder1id).c_str());
FSId fs2id = writeTerm(fs2, "");
realise(fs2id);
realise(fs2id);
string out1id = hashString("foo"); /* !!! bad */
string out1fn = nixStore + "/" + (string) out1id + "-hello.txt";
string out1h = hashString("foo"); /* !!! bad */
Path out1fn = nixStore + "/" + (string) out1h + "-hello.txt";
ATerm fs3 = ATmake(
"Derive([(<str>, <str>)], [<str>], <str>, <str>, [], [(\"out\", <str>)])",
"Derive([<str>], [<str>], <str>, <str>, [], [(\"out\", <str>)])",
out1fn.c_str(),
((string) out1id).c_str(),
((string) fs1id).c_str(),
fs1ne.c_str(),
thisSystem.c_str(),
((string) builder1fn).c_str(),
builder1fn.c_str(),
out1fn.c_str());
debug(printTerm(fs3));
FSId fs3id = writeTerm(fs3, "");
Path fs3ne = writeTerm(fs3, "-d");
realise(fs3id);
realise(fs3id);
realise(fs3ne);
realise(fs3ne);
FSId builder4id;
string builder4fn;
addToStore("./test-builder-2.sh", builder4fn, builder4id);
Path builder4fn = addToStore("./test-builder-2.sh");
ATerm fs4 = ATmake(
"Closure([<str>], [(<str>, <str>, [])])",
"Closure([<str>], [(<str>, [])])",
builder4fn.c_str(),
builder4fn.c_str(),
((string) builder4id).c_str());
FSId fs4id = writeTerm(fs4, "");
builder4fn.c_str());
Path fs4ne = writeTerm(fs4, "-c");
realise(fs4id);
realise(fs4ne);
string out5id = hashString("bar"); /* !!! bad */
string out5fn = nixStore + "/" + (string) out5id + "-hello2";
string out5h = hashString("bar"); /* !!! bad */
Path out5fn = nixStore + "/" + (string) out5h + "-hello2";
ATerm fs5 = ATmake(
"Derive([(<str>, <str>)], [<str>], <str>, <str>, [], [(\"out\", <str>), (\"builder\", <str>)])",
"Derive([<str>], [<str>], <str>, <str>, [], [(\"out\", <str>), (\"builder\", <str>)])",
out5fn.c_str(),
((string) out5id).c_str(),
((string) fs4id).c_str(),
fs4ne.c_str(),
thisSystem.c_str(),
((string) builder4fn).c_str(),
builder4fn.c_str(),
out5fn.c_str(),
((string) builder4fn).c_str());
builder4fn.c_str());
debug(printTerm(fs5));
FSId fs5id = writeTerm(fs5, "");
Path fs5ne = writeTerm(fs5, "-d");
realise(fs5id);
realise(fs5id);
realise(fs5ne);
realise(fs5ne);
}