* Substitutes now should produce a path with the same id as they are

substituting for (obvious, really).

* For greater efficiency, nix-pull/unnar will place the output in a
  path that is probably the same as what is actually needed, thus
  preventing a path copy.

* Even if a output id is given in a Fix package expression, ensure
  that the resulting Nix derive expression has a different id.  This
  is because Nix expressions that are semantically equivalent (i.e.,
  build the same result) might be different w.r.t. efficiency or
  divergence.  It is absolutely vital for the substitute mechanism
  that such expressions are not used interchangeably.
This commit is contained in:
Eelco Dolstra 2003-07-22 15:15:15 +00:00
parent df648c4967
commit e877c69d78
8 changed files with 86 additions and 50 deletions

View file

@ -3,6 +3,7 @@ Function(["nar", "name"],
[ ("name", Var("name")) [ ("name", Var("name"))
, ("build", Relative("nar/unnar.sh")) , ("build", Relative("nar/unnar.sh"))
, ("nar", Var("nar")) , ("nar", Var("nar"))
, ("id", Var("id"))
] ]
) )
) )

View file

@ -28,10 +28,18 @@ while (<CONFFILE>) {
my $fn = $1; my $fn = $1;
next if $fn =~ /\.\./; next if $fn =~ /\.\./;
next if $fn =~ /\//; next if $fn =~ /\//;
next unless $fn =~ /^([0-9a-z]{32})-([0-9a-z]{32})(-s-([0-9a-z]{32}))?.*\.nar\.bz2$/; next unless $fn =~ /^([0-9a-z]{32})-([0-9a-z]{32})(.*)\.nar\.bz2$/;
my $hash = $1; my $hash = $1;
my $id = $2; my $id = $2;
my $fsid = $4; my $outname = $3;
my $fsid;
if ($outname =~ /^-/) {
next unless $outname =~ /^-((s-([0-9a-z]{32}))?.*)$/;
$outname = $1;
$fsid = $3;
} else {
$outname = "";
}
print "registering $id -> $url/$fn\n"; print "registering $id -> $url/$fn\n";
@ -43,7 +51,8 @@ while (<CONFFILE>) {
my $fixexpr = my $fixexpr =
"App(IncludeFix(\"nar/unnar.fix\"), " . "App(IncludeFix(\"nar/unnar.fix\"), " .
"[ (\"nar\", $fetch)" . "[ (\"nar\", $fetch)" .
", (\"name\", \"fetched-$id\")" . ", (\"name\", \"$outname\")" .
", (\"id\", \"$id\")" .
"])"; "])";
my $fixfile = "/tmp/nix-pull-tmp.fix"; my $fixfile = "/tmp/nix-pull-tmp.fix";

View file

@ -9,11 +9,13 @@
typedef ATerm Expr; typedef ATerm Expr;
typedef map<ATerm, ATerm> NormalForms; typedef map<ATerm, ATerm> NormalForms;
typedef map<FSId, Hash> PkgHashes;
struct EvalState struct EvalState
{ {
Strings searchDirs; Strings searchDirs;
NormalForms normalForms; NormalForms normalForms;
PkgHashes pkgHashes; /* normalised package hashes */
}; };
@ -104,6 +106,23 @@ static Expr substExprMany(ATermList formals, ATermList args, Expr body)
} }
Hash hashPackage(EvalState & state, FState fs)
{
if (fs.type == FState::fsDerive) {
for (FSIds::iterator i = fs.derive.inputs.begin();
i != fs.derive.inputs.end(); i++)
{
PkgHashes::iterator j = state.pkgHashes.find(*i);
if (j == state.pkgHashes.end())
throw Error(format("unknown package id %1%") % (string) *i);
*i = j->second;
}
}
debug(printTerm(unparseFState(fs)));
return hashTerm(unparseFState(fs));
}
static Expr evalExpr2(EvalState & state, Expr e) static Expr evalExpr2(EvalState & state, Expr e)
{ {
char * s1; char * s1;
@ -117,9 +136,10 @@ static Expr evalExpr2(EvalState & state, Expr e)
return e; return e;
try { try {
parseFState(e); Hash pkgHash = hashPackage(state, parseFState(e));
return ATmake("FSId(<str>)", FSId pkgId = writeTerm(e, "");
((string) writeTerm(e, "")).c_str()); state.pkgHashes[pkgId] = pkgHash;
return ATmake("FSId(<str>)", ((string) pkgId).c_str());
} catch (...) { /* !!! catch parse errors only */ } catch (...) { /* !!! catch parse errors only */
} }
@ -153,10 +173,10 @@ static Expr evalExpr2(EvalState & state, Expr e)
fs.slice.roots.push_back(id); fs.slice.roots.push_back(id);
fs.slice.elems.push_back(elem); fs.slice.elems.push_back(elem);
FSId termId = hashString("producer-" + (string) id Hash pkgHash = hashPackage(state, fs);
+ "-" + dstPath); FSId pkgId = writeTerm(unparseFState(fs), "");
writeTerm(unparseFState(fs), "", termId); state.pkgHashes[pkgId] = pkgHash;
return ATmake("FSId(<str>)", ((string) termId).c_str()); return ATmake("FSId(<str>)", ((string) pkgId).c_str());
} }
/* Packages are transformed into Derive fstate expressions. */ /* Packages are transformed into Derive fstate expressions. */
@ -179,6 +199,7 @@ static Expr evalExpr2(EvalState & state, Expr e)
fs.derive.platform = SYSTEM; fs.derive.platform = SYSTEM;
string name; string name;
FSId outId; FSId outId;
bool outIdGiven = false;
bnds = ATempty; bnds = ATempty;
for (map<string, ATerm>::iterator it = bndMap.begin(); for (map<string, ATerm>::iterator it = bndMap.begin();
@ -198,7 +219,10 @@ static Expr evalExpr2(EvalState & state, Expr e)
} }
else if (ATmatch(value, "<str>", &s1)) { else if (ATmatch(value, "<str>", &s1)) {
if (key == "name") name = s1; if (key == "name") name = s1;
if (key == "id") outId = parseHash(s1); if (key == "id") {
outId = parseHash(s1);
outIdGiven = true;
}
fs.derive.env.push_back(StringPair(key, s1)); fs.derive.env.push_back(StringPair(key, s1));
} }
else throw badTerm("invalid package argument", value); else throw badTerm("invalid package argument", value);
@ -215,8 +239,7 @@ static Expr evalExpr2(EvalState & state, Expr e)
/* Hash the fstate-expression with no outputs to produce a /* Hash the fstate-expression with no outputs to produce a
unique but deterministic path name for this package. */ unique but deterministic path name for this package. */
if (outId == FSId()) if (!outIdGiven) outId = hashPackage(state, fs);
outId = hashTerm(unparseFState(fs));
string outPath = string outPath =
canonPath(nixStore + "/" + ((string) outId).c_str() + "-" + name); canonPath(nixStore + "/" + ((string) outId).c_str() + "-" + name);
fs.derive.env.push_back(StringPair("out", outPath)); fs.derive.env.push_back(StringPair("out", outPath));
@ -224,10 +247,12 @@ static Expr evalExpr2(EvalState & state, Expr e)
debug(format("%1%: %2%") % (string) outId % name); debug(format("%1%: %2%") % (string) outId % name);
/* Write the resulting term into the Nix store directory. */ /* Write the resulting term into the Nix store directory. */
FSId termId = hashString("producer-" + (string) outId Hash pkgHash = outIdGiven
+ "-" + outPath); ? hashString((string) outId + outPath)
writeTerm(unparseFState(fs), "-d-" + name, termId); : hashPackage(state, fs);
return ATmake("FSId(<str>)", ((string) termId).c_str()); FSId pkgId = writeTerm(unparseFState(fs), "-d-" + name);
state.pkgHashes[pkgId] = pkgHash;
return ATmake("FSId(<str>)", ((string) pkgId).c_str());
} }
/* BaseName primitive function. */ /* BaseName primitive function. */

View file

@ -62,9 +62,6 @@ static void parseIds(ATermList ids, FSIds & out)
} }
typedef set<FSId> FSIdSet;
static void checkSlice(const Slice & slice) static void checkSlice(const Slice & slice)
{ {
if (slice.elems.size() == 0) if (slice.elems.size() == 0)

View file

@ -24,7 +24,7 @@ static FSId storeSuccessor(const FSId & id1, ATerm sc)
typedef set<FSId> FSIdSet; typedef set<FSId> FSIdSet;
Slice normaliseFState(FSId id) Slice normaliseFState(FSId id, FSIdSet pending)
{ {
debug(format("normalising fstate %1%") % (string) id); debug(format("normalising fstate %1%") % (string) id);
Nest nest(true); Nest nest(true);
@ -57,8 +57,8 @@ Slice normaliseFState(FSId id)
for (FSIds::iterator i = fs.derive.inputs.begin(); for (FSIds::iterator i = fs.derive.inputs.begin();
i != fs.derive.inputs.end(); i++) { i != fs.derive.inputs.end(); i++) {
Slice slice = normaliseFState(*i); Slice slice = normaliseFState(*i, pending);
realiseSlice(slice); realiseSlice(slice, pending);
for (SliceElems::iterator j = slice.elems.begin(); for (SliceElems::iterator j = slice.elems.begin();
j != slice.elems.end(); j++) j != slice.elems.end(); j++)
@ -93,7 +93,7 @@ Slice normaliseFState(FSId id)
i != outPaths.end(); i++) i != outPaths.end(); i++)
{ {
try { try {
expandId(i->second, i->first); expandId(i->second, i->first, "/", pending);
} catch (Error & e) { } catch (Error & e) {
debug(format("fast build failed: %1%") % e.what()); debug(format("fast build failed: %1%") % e.what());
fastBuild = false; fastBuild = false;
@ -175,7 +175,7 @@ Slice normaliseFState(FSId id)
} }
void realiseSlice(const Slice & slice) void realiseSlice(const Slice & slice, FSIdSet pending)
{ {
debug(format("realising slice")); debug(format("realising slice"));
Nest nest(true); Nest nest(true);
@ -209,7 +209,7 @@ void realiseSlice(const Slice & slice)
{ {
SliceElem elem = *i; SliceElem elem = *i;
debug(format("expanding %1% in %2%") % (string) elem.id % elem.path); debug(format("expanding %1% in %2%") % (string) elem.id % elem.path);
expandId(elem.id, elem.path); expandId(elem.id, elem.path, "/", pending);
} }
} }

View file

@ -5,11 +5,11 @@
/* Normalise an fstate-expression, that is, return an equivalent /* Normalise an fstate-expression, that is, return an equivalent
Slice. */ Slice. (For the meaning of `pending', see expandId()). */
Slice normaliseFState(FSId id); Slice normaliseFState(FSId id, FSIdSet pending = FSIdSet());
/* Realise a Slice in the file system. */ /* Realise a Slice in the file system. */
void realiseSlice(const Slice & slice); void realiseSlice(const Slice & slice, FSIdSet pending = FSIdSet());
/* Get the list of root (output) paths of the given /* Get the list of root (output) paths of the given
fstate-expression. */ fstate-expression. */

View file

@ -165,8 +165,11 @@ bool isInPrefix(const string & path, const string & _prefix)
string expandId(const FSId & id, const string & target, string expandId(const FSId & id, const string & target,
const string & prefix) const string & prefix, FSIdSet pending)
{ {
debug(format("expanding %1%") % (string) id);
Nest nest(true);
Strings paths; Strings paths;
if (!target.empty() && !isInPrefix(target, prefix)) if (!target.empty() && !isInPrefix(target, prefix))
@ -203,30 +206,24 @@ string expandId(const FSId & id, const string & target,
} }
} }
/* Try to realise the substitutes. */ if (pending.find(id) != pending.end())
throw Error(format("id %1% already being expanded") % (string) id);
pending.insert(id);
/* Try to realise the substitutes, but only if this id is not
already being realised by a substitute. */
Strings subs; Strings subs;
queryListDB(nixDB, dbSubstitutes, id, subs); /* non-existence = ok */ queryListDB(nixDB, dbSubstitutes, id, subs); /* non-existence = ok */
for (Strings::iterator it = subs.begin(); it != subs.end(); it++) { for (Strings::iterator it = subs.begin(); it != subs.end(); it++) {
FSId subId = parseHash(*it); FSId subId = parseHash(*it);
Slice slice = normaliseFState(subId);
realiseSlice(slice);
Strings paths = fstatePaths(subId, true); debug(format("trying substitute %1%") % (string) subId);
if (paths.size() != 1)
throw Error("substitute created more than 1 path");
string path = *(paths.begin());
if (target.empty()) Slice slice = normaliseFState(subId, pending);
return path; /* !!! prefix */ realiseSlice(slice, pending);
else {
if (path != target) { return expandId(id, target, prefix, pending);
copyPath(path, target);
registerPath(target, id);
}
return target;
}
} }
throw Error(format("cannot expand id `%1%'") % (string) id); throw Error(format("cannot expand id `%1%'") % (string) id);

View file

@ -10,6 +10,8 @@ using namespace std;
typedef Hash FSId; typedef Hash FSId;
typedef set<FSId> FSIdSet;
/* Copy a path recursively. */ /* Copy a path recursively. */
void copyPath(string src, string dst); void copyPath(string src, string dst);
@ -26,9 +28,14 @@ bool queryPathId(const string & path, FSId & id);
/* Return a path whose contents have the given hash. If target is /* Return a path whose contents have the given hash. If target is
not empty, ensure that such a path is realised in target (if not empty, ensure that such a path is realised in target (if
necessary by copying from another location). If prefix is not necessary by copying from another location). If prefix is not
empty, only return a path that is an descendent of prefix. */ empty, only return a path that is an descendent of prefix.
The list of pending ids are those that already being expanded.
This prevents infinite recursion for ids realised through a
substitute (since when we build the substitute, we would first try
to expand the id... kaboom!). */
string expandId(const FSId & id, const string & target = "", string expandId(const FSId & id, const string & target = "",
const string & prefix = "/"); const string & prefix = "/", FSIdSet pending = FSIdSet());
/* Copy a file to the nixStore directory and register it in dbRefs. /* Copy a file to the nixStore directory and register it in dbRefs.
Return the hash code of the value. */ Return the hash code of the value. */