* Use maps and sets in the FState data type. This ensures normalisation of

slices and derivations w.r.t. order of paths, slice elements, etc.
This commit is contained in:
Eelco Dolstra 2003-08-20 14:11:40 +00:00
parent 624c48260f
commit 956801fcc2
5 changed files with 86 additions and 105 deletions

View file

@ -137,14 +137,16 @@ static Strings fstatePathsCached(EvalState & state, const FSId & id)
static Hash hashPackage(EvalState & state, FState fs) static Hash hashPackage(EvalState & state, FState fs)
{ {
if (fs.type == FState::fsDerive) { if (fs.type == FState::fsDerive) {
for (FSIds::iterator i = fs.derive.inputs.begin(); FSIdSet inputs2;
for (FSIdSet::iterator i = fs.derive.inputs.begin();
i != fs.derive.inputs.end(); i++) i != fs.derive.inputs.end(); i++)
{ {
PkgHashes::iterator j = state.pkgHashes.find(*i); PkgHashes::iterator j = state.pkgHashes.find(*i);
if (j == state.pkgHashes.end()) if (j == state.pkgHashes.end())
throw Error(format("unknown package id %1%") % (string) *i); throw Error(format("unknown package id %1%") % (string) *i);
*i = j->second; inputs2.insert(j->second);
} }
fs.derive.inputs = inputs2;
} }
return hashTerm(unparseFState(fs)); return hashTerm(unparseFState(fs));
} }
@ -159,7 +161,7 @@ static string processBinding(EvalState & state, Expr e, FState & fs)
Strings paths = fstatePathsCached(state, id); Strings paths = fstatePathsCached(state, id);
if (paths.size() != 1) abort(); if (paths.size() != 1) abort();
string path = *(paths.begin()); string path = *(paths.begin());
fs.derive.inputs.push_back(id); fs.derive.inputs.insert(id);
return path; return path;
} }
@ -264,12 +266,11 @@ static Expr evalExpr2(EvalState & state, Expr e)
addToStore(srcPath, dstPath, id, true); addToStore(srcPath, dstPath, id, true);
SliceElem elem; SliceElem elem;
elem.path = dstPath;
elem.id = id; elem.id = id;
FState fs; FState fs;
fs.type = FState::fsSlice; fs.type = FState::fsSlice;
fs.slice.roots.push_back(dstPath); fs.slice.roots.insert(dstPath);
fs.slice.elems.push_back(elem); fs.slice.elems[dstPath] = elem;
Hash pkgHash = hashPackage(state, fs); Hash pkgHash = hashPackage(state, fs);
FSId pkgId = writeTerm(unparseFState(fs), ""); FSId pkgId = writeTerm(unparseFState(fs), "");
@ -324,7 +325,7 @@ static Expr evalExpr2(EvalState & state, Expr e)
else { else {
string s = processBinding(state, value, fs); string s = processBinding(state, value, fs);
fs.derive.env.push_back(StringPair(key, s)); fs.derive.env[key] = s;
if (key == "build") fs.derive.builder = s; if (key == "build") fs.derive.builder = s;
if (key == "name") name = s; if (key == "name") name = s;
@ -349,8 +350,8 @@ static Expr evalExpr2(EvalState & state, Expr e)
if (!outIdGiven) outId = hashPackage(state, fs); if (!outIdGiven) outId = hashPackage(state, fs);
string outPath = string outPath =
canonPath(nixStore + "/" + ((string) outId).c_str() + "-" + name); canonPath(nixStore + "/" + ((string) outId).c_str() + "-" + name);
fs.derive.env.push_back(StringPair("out", outPath)); fs.derive.env["out"] = outPath;
fs.derive.outputs.push_back(DeriveOutput(outPath, outId)); fs.derive.outputs[outPath] = outId;
/* Write the resulting term into the Nix store directory. */ /* Write the resulting term into the Nix store directory. */
Hash pkgHash = outIdGiven Hash pkgHash = outIdGiven

View file

@ -52,14 +52,14 @@ FSId writeTerm(ATerm t, const string & suffix, FSId id)
} }
static void parsePaths(ATermList paths, Strings & out) static void parsePaths(ATermList paths, StringSet & out)
{ {
while (!ATisEmpty(paths)) { while (!ATisEmpty(paths)) {
char * s; char * s;
ATerm t = ATgetFirst(paths); ATerm t = ATgetFirst(paths);
if (!ATmatch(t, "<str>", &s)) if (!ATmatch(t, "<str>", &s))
throw badTerm("not a path", t); throw badTerm("not a path", t);
out.push_back(s); out.insert(s);
paths = ATgetNext(paths); paths = ATgetNext(paths);
} }
} }
@ -73,21 +73,21 @@ static void checkSlice(const Slice & slice)
StringSet decl; StringSet decl;
for (SliceElems::const_iterator i = slice.elems.begin(); for (SliceElems::const_iterator i = slice.elems.begin();
i != slice.elems.end(); i++) i != slice.elems.end(); i++)
decl.insert(i->path); decl.insert(i->first);
for (Strings::const_iterator i = slice.roots.begin(); for (StringSet::const_iterator i = slice.roots.begin();
i != slice.roots.end(); i++) i != slice.roots.end(); i++)
if (decl.find(*i) == decl.end()) if (decl.find(*i) == decl.end())
throw Error(format("undefined root path `%1%'") % *i); throw Error(format("undefined root path `%1%'") % *i);
for (SliceElems::const_iterator i = slice.elems.begin(); for (SliceElems::const_iterator i = slice.elems.begin();
i != slice.elems.end(); i++) i != slice.elems.end(); i++)
for (Strings::const_iterator j = i->refs.begin(); for (StringSet::const_iterator j = i->second.refs.begin();
j != i->refs.end(); j++) j != i->second.refs.end(); j++)
if (decl.find(*j) == decl.end()) if (decl.find(*j) == decl.end())
throw Error( throw Error(
format("undefined path `%1%' referenced by `%2%'") format("undefined path `%1%' referenced by `%2%'")
% *j % i->path); % *j % i->first);
} }
@ -108,10 +108,9 @@ static bool parseSlice(ATerm t, Slice & slice)
if (!ATmatch(t, "(<str>, <str>, [<list>])", &s1, &s2, &refs)) if (!ATmatch(t, "(<str>, <str>, [<list>])", &s1, &s2, &refs))
throw badTerm("not a slice element", t); throw badTerm("not a slice element", t);
SliceElem elem; SliceElem elem;
elem.path = s1;
elem.id = parseHash(s2); elem.id = parseHash(s2);
parsePaths(refs, elem.refs); parsePaths(refs, elem.refs);
slice.elems.push_back(elem); slice.elems[s1] = elem;
elems = ATgetNext(elems); elems = ATgetNext(elems);
} }
@ -141,7 +140,7 @@ static bool parseDerive(ATerm t, Derive & derive)
ATerm t = ATgetFirst(outs); ATerm t = ATgetFirst(outs);
if (!ATmatch(t, "(<str>, <str>)", &s1, &s2)) if (!ATmatch(t, "(<str>, <str>)", &s1, &s2))
throw badTerm("not a derive output", t); throw badTerm("not a derive output", t);
derive.outputs.push_back(DeriveOutput(s1, parseHash(s2))); derive.outputs[s1] = parseHash(s2);
outs = ATgetNext(outs); outs = ATgetNext(outs);
} }
@ -150,7 +149,7 @@ static bool parseDerive(ATerm t, Derive & derive)
ATerm t = ATgetFirst(ins); ATerm t = ATgetFirst(ins);
if (!ATmatch(t, "<str>", &s)) if (!ATmatch(t, "<str>", &s))
throw badTerm("not an id", t); throw badTerm("not an id", t);
derive.inputs.push_back(parseHash(s)); derive.inputs.insert(parseHash(s));
ins = ATgetNext(ins); ins = ATgetNext(ins);
} }
@ -171,7 +170,7 @@ static bool parseDerive(ATerm t, Derive & derive)
ATerm bnd = ATgetFirst(bnds); ATerm bnd = ATgetFirst(bnds);
if (!ATmatch(bnd, "(<str>, <str>)", &s1, &s2)) if (!ATmatch(bnd, "(<str>, <str>)", &s1, &s2))
throw badTerm("tuple of strings expected", bnd); throw badTerm("tuple of strings expected", bnd);
derive.env.push_back(StringPair(s1, s2)); derive.env[s1] = s2;
bnds = ATgetNext(bnds); bnds = ATgetNext(bnds);
} }
@ -191,10 +190,10 @@ FState parseFState(ATerm t)
} }
static ATermList unparsePaths(const Strings & paths) static ATermList unparsePaths(const StringSet & paths)
{ {
ATermList l = ATempty; ATermList l = ATempty;
for (Strings::const_iterator i = paths.begin(); for (StringSet::const_iterator i = paths.begin();
i != paths.end(); i++) i != paths.end(); i++)
l = ATinsert(l, ATmake("<str>", i->c_str())); l = ATinsert(l, ATmake("<str>", i->c_str()));
return ATreverse(l); return ATreverse(l);
@ -210,9 +209,9 @@ static ATerm unparseSlice(const Slice & slice)
i != slice.elems.end(); i++) i != slice.elems.end(); i++)
elems = ATinsert(elems, elems = ATinsert(elems,
ATmake("(<str>, <str>, <term>)", ATmake("(<str>, <str>, <term>)",
i->path.c_str(), i->first.c_str(),
((string) i->id).c_str(), ((string) i->second.id).c_str(),
unparsePaths(i->refs))); unparsePaths(i->second.refs)));
return ATmake("Slice(<term>, <term>)", roots, elems); return ATmake("Slice(<term>, <term>)", roots, elems);
} }
@ -228,7 +227,7 @@ static ATerm unparseDerive(const Derive & derive)
i->first.c_str(), ((string) i->second).c_str())); i->first.c_str(), ((string) i->second).c_str()));
ATermList ins = ATempty; ATermList ins = ATempty;
for (FSIds::const_iterator i = derive.inputs.begin(); for (FSIdSet::const_iterator i = derive.inputs.begin();
i != derive.inputs.end(); i++) i != derive.inputs.end(); i++)
ins = ATinsert(ins, ATmake("<str>", ((string) *i).c_str())); ins = ATinsert(ins, ATmake("<str>", ((string) *i).c_str()));

View file

@ -14,28 +14,25 @@ typedef list<FSId> FSIds;
struct SliceElem struct SliceElem
{ {
string path;
FSId id; FSId id;
Strings refs; StringSet refs;
}; };
typedef list<SliceElem> SliceElems; typedef map<string, SliceElem> SliceElems;
struct Slice struct Slice
{ {
Strings roots; StringSet roots;
SliceElems elems; SliceElems elems;
}; };
typedef pair<string, FSId> DeriveOutput; typedef map<string, FSId> DeriveOutputs;
typedef pair<string, string> StringPair; typedef map<string, string> StringPairs;
typedef list<DeriveOutput> DeriveOutputs;
typedef list<StringPair> StringPairs;
struct Derive struct Derive
{ {
DeriveOutputs outputs; DeriveOutputs outputs;
FSIds inputs; FSIdSet inputs;
string platform; string platform;
string builder; string builder;
Strings args; Strings args;

View file

@ -193,7 +193,7 @@ static void opQuery(Strings opFlags, Strings opArgs)
string label, shape; string label, shape;
if (fs.type == FState::fsDerive) { if (fs.type == FState::fsDerive) {
for (FSIds::iterator i = fs.derive.inputs.begin(); for (FSIdSet::iterator i = fs.derive.inputs.begin();
i != fs.derive.inputs.end(); i++) i != fs.derive.inputs.end(); i++)
{ {
workList.push_back(*i); workList.push_back(*i);
@ -209,7 +209,7 @@ static void opQuery(Strings opFlags, Strings opArgs)
} }
else if (fs.type == FState::fsSlice) { else if (fs.type == FState::fsSlice) {
label = baseNameOf((*fs.slice.elems.begin()).path); label = baseNameOf((*fs.slice.elems.begin()).first);
shape = "ellipse"; shape = "ellipse";
if (isHash(string(label, 0, Hash::hashSize * 2)) && if (isHash(string(label, 0, Hash::hashSize * 2)) &&
label[Hash::hashSize * 2] == '-') label[Hash::hashSize * 2] == '-')

View file

@ -26,14 +26,10 @@ static FSId useSuccessor(const FSId & id)
} }
typedef map<string, FSId> OutPaths; Strings pathsFromOutputs(const DeriveOutputs & ps)
typedef map<string, SliceElem> ElemMap;
Strings pathsFromOutPaths(const OutPaths & ps)
{ {
Strings ss; Strings ss;
for (OutPaths::const_iterator i = ps.begin(); for (DeriveOutputs::const_iterator i = ps.begin();
i != ps.end(); i++) i != ps.end(); i++)
ss.push_back(i->first); ss.push_back(i->first);
return ss; return ss;
@ -62,31 +58,31 @@ FSId normaliseFState(FSId id, FSIdSet pending)
/* Some variables. */ /* Some variables. */
/* Output paths, with their ids. */
OutPaths outPaths;
/* Input paths, with their slice elements. */ /* Input paths, with their slice elements. */
ElemMap inMap; SliceElems inSlices;
/* Referencable paths (i.e., input and output paths). */ /* Referencable paths (i.e., input and output paths). */
Strings allPaths; StringSet allPaths;
/* The environment to be passed to the builder. */ /* The environment to be passed to the builder. */
Environment env; Environment env;
/* The result. */
FState nfFS;
nfFS.type = FState::fsSlice;
/* Parse the outputs. */ /* Parse the outputs. */
for (DeriveOutputs::iterator i = fs.derive.outputs.begin(); for (DeriveOutputs::iterator i = fs.derive.outputs.begin();
i != fs.derive.outputs.end(); i++) i != fs.derive.outputs.end(); i++)
{ {
debug(format("building %1% in `%2%'") % (string) i->second % i->first); debug(format("building %1% in `%2%'") % (string) i->second % i->first);
outPaths[i->first] = i->second; allPaths.insert(i->first);
allPaths.push_back(i->first);
} }
/* Obtain locks on all output paths. The locks are automatically /* Obtain locks on all output paths. The locks are automatically
released when we exit this function or Nix crashes. */ released when we exit this function or Nix crashes. */
PathLocks outputLocks(pathsFromOutPaths(outPaths)); PathLocks outputLocks(pathsFromOutputs(fs.derive.outputs));
/* Now check again whether there is a successor. This is because /* Now check again whether there is a successor. This is because
another process may have started building in parallel. After another process may have started building in parallel. After
@ -113,8 +109,9 @@ FSId normaliseFState(FSId id, FSIdSet pending)
% fs.derive.platform % thisSystem); % fs.derive.platform % thisSystem);
/* Realise inputs (and remember all input paths). */ /* Realise inputs (and remember all input paths). */
for (FSIds::iterator i = fs.derive.inputs.begin(); for (FSIdSet::iterator i = fs.derive.inputs.begin();
i != fs.derive.inputs.end(); i++) { i != fs.derive.inputs.end(); i++)
{
FSId nf = normaliseFState(*i, pending); FSId nf = normaliseFState(*i, pending);
realiseSlice(nf, pending); realiseSlice(nf, pending);
/* !!! nf should be a root of the garbage collector while we /* !!! nf should be a root of the garbage collector while we
@ -123,11 +120,11 @@ FSId normaliseFState(FSId id, FSIdSet pending)
if (fs.type != FState::fsSlice) abort(); if (fs.type != FState::fsSlice) abort();
for (SliceElems::iterator j = fs.slice.elems.begin(); for (SliceElems::iterator j = fs.slice.elems.begin();
j != fs.slice.elems.end(); j++) j != fs.slice.elems.end(); j++)
inMap[j->path] = *j; {
inSlices[j->first] = j->second;
allPaths.insert(j->first);
}
} }
for (ElemMap::iterator i = inMap.begin(); i != inMap.end(); i++)
allPaths.push_back(i->second.path);
/* Most shells initialise PATH to some default (/bin:/usr/bin:...) when /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when
PATH is not set. We don't want this, so we fill it in with some dummy PATH is not set. We don't want this, so we fill it in with some dummy
@ -142,8 +139,8 @@ FSId normaliseFState(FSId id, FSIdSet pending)
/* We can skip running the builder if we can expand all output /* We can skip running the builder if we can expand all output
paths from their ids. */ paths from their ids. */
bool fastBuild = true; bool fastBuild = true;
for (OutPaths::iterator i = outPaths.begin(); for (DeriveOutputs::iterator i = fs.derive.outputs.begin();
i != outPaths.end(); i++) i != fs.derive.outputs.end(); i++)
{ {
try { try {
expandId(i->second, i->first, "/", pending); expandId(i->second, i->first, "/", pending);
@ -159,8 +156,8 @@ FSId normaliseFState(FSId id, FSIdSet pending)
/* If any of the outputs already exist but are not registered, /* If any of the outputs already exist but are not registered,
delete them. */ delete them. */
for (OutPaths::iterator i = outPaths.begin(); for (DeriveOutputs::iterator i = fs.derive.outputs.begin();
i != outPaths.end(); i++) i != fs.derive.outputs.end(); i++)
{ {
string path = i->first; string path = i->first;
FSId id; FSId id;
@ -183,21 +180,21 @@ FSId normaliseFState(FSId id, FSIdSet pending)
/* Check whether the output paths were created, and grep each /* Check whether the output paths were created, and grep each
output path to determine what other paths it references. */ output path to determine what other paths it references. */
StringSet usedPaths; StringSet usedPaths;
for (OutPaths::iterator i = outPaths.begin(); for (DeriveOutputs::iterator i = fs.derive.outputs.begin();
i != outPaths.end(); i++) i != fs.derive.outputs.end(); i++)
{ {
string path = i->first; string path = i->first;
if (!pathExists(path)) if (!pathExists(path))
throw Error(format("path `%1%' does not exist") % path); throw Error(format("path `%1%' does not exist") % path);
fs.slice.roots.push_back(path); nfFS.slice.roots.insert(path);
/* For this output path, find the references to other paths contained /* For this output path, find the references to other paths contained
in it. */ in it. */
Strings refPaths = filterReferences(path, allPaths); Strings refPaths = filterReferences(path,
Strings(allPaths.begin(), allPaths.end()));
/* Construct a slice element for this output path. */ /* Construct a slice element for this output path. */
SliceElem elem; SliceElem elem;
elem.path = path;
elem.id = i->second; elem.id = i->second;
/* For each path referenced by this output path, add its id to the /* For each path referenced by this output path, add its id to the
@ -207,18 +204,14 @@ FSId normaliseFState(FSId id, FSIdSet pending)
j != refPaths.end(); j++) j != refPaths.end(); j++)
{ {
string path = *j; string path = *j;
ElemMap::iterator k; elem.refs.insert(path);
OutPaths::iterator l; if (inSlices.find(path) != inSlices.end())
usedPaths.insert(path);
elem.refs.push_back(path); else if (fs.derive.outputs.find(path) == fs.derive.outputs.end())
if ((k = inMap.find(path)) != inMap.end())
usedPaths.insert(k->second.path);
else if ((l = outPaths.find(path)) == outPaths.end())
abort(); abort();
} }
fs.slice.elems.push_back(elem); nfFS.slice.elems[path] = elem;
} }
/* Close the slice. That is, for any referenced path, add the paths /* Close the slice. That is, for any referenced path, add the paths
@ -233,31 +226,30 @@ FSId normaliseFState(FSId id, FSIdSet pending)
if (donePaths.find(path) != donePaths.end()) continue; if (donePaths.find(path) != donePaths.end()) continue;
donePaths.insert(path); donePaths.insert(path);
ElemMap::iterator j = inMap.find(path); SliceElems::iterator j = inSlices.find(path);
if (j == inMap.end()) abort(); if (j == inSlices.end()) abort();
fs.slice.elems.push_back(j->second); nfFS.slice.elems[path] = j->second;
for (Strings::iterator k = j->second.refs.begin(); for (StringSet::iterator k = j->second.refs.begin();
k != j->second.refs.end(); k++) k != j->second.refs.end(); k++)
usedPaths.insert(*k); usedPaths.insert(*k);
} }
/* For debugging, print out the referenced and unreferenced paths. */ /* For debugging, print out the referenced and unreferenced paths. */
for (ElemMap::iterator i = inMap.begin(); for (SliceElems::iterator i = inSlices.begin();
i != inMap.end(); i++) i != inSlices.end(); i++)
{ {
StringSet::iterator j = donePaths.find(i->second.path); StringSet::iterator j = donePaths.find(i->first);
if (j == donePaths.end()) if (j == donePaths.end())
debug(format("NOT referenced: `%1%'") % i->second.path); debug(format("NOT referenced: `%1%'") % i->first);
else else
debug(format("referenced: `%1%'") % i->second.path); debug(format("referenced: `%1%'") % i->first);
} }
/* Write the normal form. This does not have to occur in the /* Write the normal form. This does not have to occur in the
transaction below because writing terms is idem-potent. */ transaction below because writing terms is idem-potent. */
fs.type = FState::fsSlice; ATerm nf = unparseFState(nfFS);
ATerm nf = unparseFState(fs);
msg(lvlVomit, format("normal form: %1%") % printTerm(nf)); msg(lvlVomit, format("normal form: %1%") % printTerm(nf));
FSId idNF = writeTerm(nf, "-s-" + (string) id); FSId idNF = writeTerm(nf, "-s-" + (string) id);
@ -268,8 +260,8 @@ FSId normaliseFState(FSId id, FSIdSet pending)
deleted arbitrarily, while registered paths can only be deleted deleted arbitrarily, while registered paths can only be deleted
by running the garbage collector. */ by running the garbage collector. */
Transaction txn(nixDB); Transaction txn(nixDB);
for (OutPaths::iterator i = outPaths.begin(); for (DeriveOutputs::iterator i = fs.derive.outputs.begin();
i != outPaths.end(); i++) i != fs.derive.outputs.end(); i++)
registerPath(txn, i->first, i->second); registerPath(txn, i->first, i->second);
registerSuccessor(txn, id, idNF); registerSuccessor(txn, id, idNF);
txn.commit(); txn.commit();
@ -289,10 +281,7 @@ void realiseSlice(const FSId & id, FSIdSet pending)
for (SliceElems::const_iterator i = fs.slice.elems.begin(); for (SliceElems::const_iterator i = fs.slice.elems.begin();
i != fs.slice.elems.end(); i++) i != fs.slice.elems.end(); i++)
{ expandId(i->second.id, i->first, "/", pending);
SliceElem elem = *i;
expandId(elem.id, elem.path, "/", pending);
}
} }
@ -303,12 +292,9 @@ Strings fstatePaths(const FSId & id)
FState fs = parseFState(termFromId(id)); FState fs = parseFState(termFromId(id));
if (fs.type == FState::fsSlice) { if (fs.type == FState::fsSlice) {
/* !!! fix complexity */ for (StringSet::const_iterator i = fs.slice.roots.begin();
for (Strings::const_iterator i = fs.slice.roots.begin();
i != fs.slice.roots.end(); i++) i != fs.slice.roots.end(); i++)
for (SliceElems::const_iterator j = fs.slice.elems.begin(); paths.push_back(*i);
j != fs.slice.elems.end(); j++)
if (*i == j->path) paths.push_back(j->path);
} }
else if (fs.type == FState::fsDerive) { else if (fs.type == FState::fsDerive) {
@ -328,18 +314,16 @@ static void fstateRequisitesSet(const FSId & id,
{ {
FState fs = parseFState(termFromId(id)); FState fs = parseFState(termFromId(id));
if (fs.type == FState::fsSlice) { if (fs.type == FState::fsSlice)
for (SliceElems::iterator i = fs.slice.elems.begin(); for (SliceElems::iterator i = fs.slice.elems.begin();
i != fs.slice.elems.end(); i++) i != fs.slice.elems.end(); i++)
paths.insert(i->path); paths.insert(i->first);
}
else if (fs.type == FState::fsDerive) { else if (fs.type == FState::fsDerive)
for (FSIds::iterator i = fs.derive.inputs.begin(); for (FSIdSet::iterator i = fs.derive.inputs.begin();
i != fs.derive.inputs.end(); i++) i != fs.derive.inputs.end(); i++)
fstateRequisitesSet(*i, fstateRequisitesSet(*i,
includeExprs, includeSuccessors, paths); includeExprs, includeSuccessors, paths);
}
else abort(); else abort();
@ -394,7 +378,7 @@ FSIds findGenerators(const FSIds & _ids)
bool okay = true; bool okay = true;
for (SliceElems::const_iterator i = fs.slice.elems.begin(); for (SliceElems::const_iterator i = fs.slice.elems.begin();
i != fs.slice.elems.end(); i++) i != fs.slice.elems.end(); i++)
if (ids.find(i->id) == ids.end()) { if (ids.find(i->second.id) == ids.end()) {
okay = false; okay = false;
break; break;
} }