* Substitutes now should produce a path with the same id as they are

substituting for (obvious, really).

* For greater efficiency, nix-pull/unnar will place the output in a
  path that is probably the same as what is actually needed, thus
  preventing a path copy.

* Even if a output id is given in a Fix package expression, ensure
  that the resulting Nix derive expression has a different id.  This
  is because Nix expressions that are semantically equivalent (i.e.,
  build the same result) might be different w.r.t. efficiency or
  divergence.  It is absolutely vital for the substitute mechanism
  that such expressions are not used interchangeably.

corepkgs/nar/unnar.fix

@@ -3,6 +3,7 @@ Function(["nar", "name"],
     [ ("name", Var("name"))
     , ("build", Relative("nar/unnar.sh"))
     , ("nar", Var("nar"))
+    , ("id", Var("id"))
     ]
   )
 )

scripts/nix-pull.in

@@ -28,10 +28,18 @@ while (<CONFFILE>) {
             my $fn = $1;
             next if $fn =~ /\.\./;
             next if $fn =~ /\//;
-            next unless $fn =~ /^([0-9a-z]{32})-([0-9a-z]{32})(-s-([0-9a-z]{32}))?.*\.nar\.bz2$/;
+            next unless $fn =~ /^([0-9a-z]{32})-([0-9a-z]{32})(.*)\.nar\.bz2$/;
             my $hash = $1;
-            my $id = $2;
+	    my $id = $2;
-            my $fsid = $4;
+	    my $outname = $3;
+	    my $fsid;
+	    if ($outname =~ /^-/) {
+		next unless $outname =~ /^-((s-([0-9a-z]{32}))?.*)$/;
+		$outname = $1;
+		$fsid = $3;
+	    } else {
+		$outname = "";
+	    }
 
             print "registering $id -> $url/$fn\n";
 
@@ -43,7 +51,8 @@ while (<CONFFILE>) {
             my $fixexpr = 
                 "App(IncludeFix(\"nar/unnar.fix\"), " .
                 "[ (\"nar\", $fetch)" .
-                ", (\"name\", \"fetched-$id\")" .
+                ", (\"name\", \"$outname\")" .
+                ", (\"id\", \"$id\")" .
                 "])";
             
             my $fixfile = "/tmp/nix-pull-tmp.fix";

src/fix.cc

@@ -9,11 +9,13 @@
 typedef ATerm Expr;
 
 typedef map<ATerm, ATerm> NormalForms;
+typedef map<FSId, Hash> PkgHashes;
 
 struct EvalState 
 {
     Strings searchDirs;
     NormalForms normalForms;
+    PkgHashes pkgHashes; /* normalised package hashes */
 };
 
 
@@ -104,6 +106,23 @@ static Expr substExprMany(ATermList formals, ATermList args, Expr body)
 }
 
 
+Hash hashPackage(EvalState & state, FState fs)
+{
+    if (fs.type == FState::fsDerive) {
+        for (FSIds::iterator i = fs.derive.inputs.begin();
+             i != fs.derive.inputs.end(); i++)
+        {
+            PkgHashes::iterator j = state.pkgHashes.find(*i);
+            if (j == state.pkgHashes.end())
+                throw Error(format("unknown package id %1%") % (string) *i);
+            *i = j->second;
+        }
+    }
+    debug(printTerm(unparseFState(fs)));
+    return hashTerm(unparseFState(fs));
+}
+
+
 static Expr evalExpr2(EvalState & state, Expr e)
 {
     char * s1;
@@ -117,9 +136,10 @@ static Expr evalExpr2(EvalState & state, Expr e)
         return e;
 
     try {
-        parseFState(e);
+        Hash pkgHash = hashPackage(state, parseFState(e));
-        return ATmake("FSId(<str>)", 
+        FSId pkgId = writeTerm(e, "");
-            ((string) writeTerm(e, "")).c_str());
+        state.pkgHashes[pkgId] = pkgHash;
+        return ATmake("FSId(<str>)", ((string) pkgId).c_str());
     } catch (...) { /* !!! catch parse errors only */
     }
 
@@ -153,10 +173,10 @@ static Expr evalExpr2(EvalState & state, Expr e)
         fs.slice.roots.push_back(id);
         fs.slice.elems.push_back(elem);
 
-        FSId termId = hashString("producer-" + (string) id
+        Hash pkgHash = hashPackage(state, fs);
-            + "-" + dstPath);
+        FSId pkgId = writeTerm(unparseFState(fs), "");
-        writeTerm(unparseFState(fs), "", termId);
+        state.pkgHashes[pkgId] = pkgHash;
-        return ATmake("FSId(<str>)", ((string) termId).c_str());
+        return ATmake("FSId(<str>)", ((string) pkgId).c_str());
     }
 
     /* Packages are transformed into Derive fstate expressions. */
@@ -179,6 +199,7 @@ static Expr evalExpr2(EvalState & state, Expr e)
         fs.derive.platform = SYSTEM;
         string name;
         FSId outId;
+        bool outIdGiven = false;
         bnds = ATempty;
 
         for (map<string, ATerm>::iterator it = bndMap.begin();
@@ -198,7 +219,10 @@ static Expr evalExpr2(EvalState & state, Expr e)
             }
             else if (ATmatch(value, "<str>", &s1)) {
                 if (key == "name") name = s1;
-                if (key == "id") outId = parseHash(s1);
+                if (key == "id") { 
+                    outId = parseHash(s1);
+                    outIdGiven = true;
+                }
                 fs.derive.env.push_back(StringPair(key, s1));
             }
             else throw badTerm("invalid package argument", value);
@@ -215,8 +239,7 @@ static Expr evalExpr2(EvalState & state, Expr e)
         
         /* Hash the fstate-expression with no outputs to produce a
            unique but deterministic path name for this package. */
-        if (outId == FSId())
+        if (!outIdGiven) outId = hashPackage(state, fs);
-            outId = hashTerm(unparseFState(fs));
         string outPath = 
             canonPath(nixStore + "/" + ((string) outId).c_str() + "-" + name);
         fs.derive.env.push_back(StringPair("out", outPath));
@@ -224,10 +247,12 @@ static Expr evalExpr2(EvalState & state, Expr e)
         debug(format("%1%: %2%") % (string) outId % name);
 
         /* Write the resulting term into the Nix store directory. */
-        FSId termId = hashString("producer-" + (string) outId
+        Hash pkgHash = outIdGiven
-            + "-" + outPath);
+            ? hashString((string) outId + outPath)
-        writeTerm(unparseFState(fs), "-d-" + name, termId);
+            : hashPackage(state, fs);
-        return ATmake("FSId(<str>)", ((string) termId).c_str());
+        FSId pkgId = writeTerm(unparseFState(fs), "-d-" + name);
+        state.pkgHashes[pkgId] = pkgHash;
+        return ATmake("FSId(<str>)", ((string) pkgId).c_str());
     }
 
     /* BaseName primitive function. */

src/fstate.cc

@@ -62,9 +62,6 @@ static void parseIds(ATermList ids, FSIds & out)
 }
 
 
-typedef set<FSId> FSIdSet;
-
-
 static void checkSlice(const Slice & slice)
 {
     if (slice.elems.size() == 0)

src/normalise.cc

@@ -24,7 +24,7 @@ static FSId storeSuccessor(const FSId & id1, ATerm sc)
 typedef set<FSId> FSIdSet;
 
 
-Slice normaliseFState(FSId id)
+Slice normaliseFState(FSId id, FSIdSet pending)
 {
     debug(format("normalising fstate %1%") % (string) id);
     Nest nest(true);
@@ -57,8 +57,8 @@ Slice normaliseFState(FSId id)
 
     for (FSIds::iterator i = fs.derive.inputs.begin();
          i != fs.derive.inputs.end(); i++) {
-        Slice slice = normaliseFState(*i);
+        Slice slice = normaliseFState(*i, pending);
-        realiseSlice(slice);
+        realiseSlice(slice, pending);
 
         for (SliceElems::iterator j = slice.elems.begin();
              j != slice.elems.end(); j++)
@@ -93,7 +93,7 @@ Slice normaliseFState(FSId id)
          i != outPaths.end(); i++)
     {
         try {
-            expandId(i->second, i->first);
+            expandId(i->second, i->first, "/", pending);
         } catch (Error & e) {
             debug(format("fast build failed: %1%") % e.what());
             fastBuild = false;
@@ -175,7 +175,7 @@ Slice normaliseFState(FSId id)
 }
 
 
-void realiseSlice(const Slice & slice)
+void realiseSlice(const Slice & slice, FSIdSet pending)
 {
     debug(format("realising slice"));
     Nest nest(true);
@@ -209,7 +209,7 @@ void realiseSlice(const Slice & slice)
     {
         SliceElem elem = *i;
         debug(format("expanding %1% in %2%") % (string) elem.id % elem.path);
-        expandId(elem.id, elem.path);
+        expandId(elem.id, elem.path, "/", pending);
     }
 }
 

src/normalise.hh

@@ -5,11 +5,11 @@
 
 
 /* Normalise an fstate-expression, that is, return an equivalent
-   Slice. */
+   Slice.  (For the meaning of `pending', see expandId()). */
-Slice normaliseFState(FSId id);
+Slice normaliseFState(FSId id, FSIdSet pending = FSIdSet());
 
 /* Realise a Slice in the file system. */
-void realiseSlice(const Slice & slice);
+void realiseSlice(const Slice & slice, FSIdSet pending = FSIdSet());
 
 /* Get the list of root (output) paths of the given
    fstate-expression. */

src/store.cc

@@ -165,8 +165,11 @@ bool isInPrefix(const string & path, const string & _prefix)
 
 
 string expandId(const FSId & id, const string & target,
-    const string & prefix)
+    const string & prefix, FSIdSet pending)
 {
+    debug(format("expanding %1%") % (string) id);
+    Nest nest(true);
+
     Strings paths;
 
     if (!target.empty() && !isInPrefix(target, prefix))
@@ -203,30 +206,24 @@ string expandId(const FSId & id, const string & target,
         }
     }
 
-    /* Try to realise the substitutes. */
+    if (pending.find(id) != pending.end())
+        throw Error(format("id %1% already being expanded") % (string) id);
+    pending.insert(id);
 
+    /* Try to realise the substitutes, but only if this id is not
+       already being realised by a substitute. */
     Strings subs;
     queryListDB(nixDB, dbSubstitutes, id, subs); /* non-existence = ok */
 
     for (Strings::iterator it = subs.begin(); it != subs.end(); it++) {
         FSId subId = parseHash(*it);
-        Slice slice = normaliseFState(subId);
-        realiseSlice(slice);
-        
-        Strings paths = fstatePaths(subId, true);
-        if (paths.size() != 1) 
-            throw Error("substitute created more than 1 path");
-        string path = *(paths.begin());
 
-        if (target.empty())
+        debug(format("trying substitute %1%") % (string) subId);
-            return path; /* !!! prefix */
+
-        else {
+        Slice slice = normaliseFState(subId, pending);
-            if (path != target) {
+        realiseSlice(slice, pending);
-                copyPath(path, target);
+
-                registerPath(target, id);
+        return expandId(id, target, prefix, pending);
-            }
-            return target;
-        }
     }
     
     throw Error(format("cannot expand id `%1%'") % (string) id);

src/store.hh

@@ -10,6 +10,8 @@ using namespace std;
 
 typedef Hash FSId;
 
+typedef set<FSId> FSIdSet;
+
 
 /* Copy a path recursively. */
 void copyPath(string src, string dst);
@@ -26,9 +28,14 @@ bool queryPathId(const string & path, FSId & id);
 /* Return a path whose contents have the given hash.  If target is
    not empty, ensure that such a path is realised in target (if
    necessary by copying from another location).  If prefix is not
-   empty, only return a path that is an descendent of prefix. */
+   empty, only return a path that is an descendent of prefix.
+
+   The list of pending ids are those that already being expanded.
+   This prevents infinite recursion for ids realised through a
+   substitute (since when we build the substitute, we would first try
+   to expand the id... kaboom!). */
 string expandId(const FSId & id, const string & target = "",
-    const string & prefix = "/");
+    const string & prefix = "/", FSIdSet pending = FSIdSet());
 
 /* Copy a file to the nixStore directory and register it in dbRefs.
    Return the hash code of the value. */