* Another change to low-level derivations. The last one this year, I

promise :-) This allows derivations to specify on *what* output
  paths of input derivations they are dependent.  This helps to
  prevent unnecessary downloads.  For instance, a build might be
  dependent on the `devel' and `lib' outputs of some library
  component, but not the `docs' output.

src/libexpr/primops.cc

@@ -68,17 +68,17 @@ static Hash hashDerivationModulo(EvalState & state, Derivation drv)
 
     /* For other derivations, replace the inputs paths with recursive
        calls to this function.*/
-    PathSet inputs2;
+    DerivationInputs inputs2;
-    for (PathSet::iterator i = drv.inputDrvs.begin();
+    for (DerivationInputs::iterator i = drv.inputDrvs.begin();
          i != drv.inputDrvs.end(); ++i)
     {
-        Hash h = state.drvHashes[*i];
+        Hash h = state.drvHashes[i->first];
         if (h.type == htUnknown) {
-            Derivation drv2 = derivationFromPath(*i);
+            Derivation drv2 = derivationFromPath(i->first);
             h = hashDerivationModulo(state, drv2);
-            state.drvHashes[*i] = h;
+            state.drvHashes[i->first] = h;
         }
-        inputs2.insert(printHash(h));
+        inputs2[printHash(h)] = i->second;
     }
     drv.inputDrvs = inputs2;
     
@@ -119,7 +119,9 @@ static void processBinding(EvalState & state, Expr e, Derivation & drv,
             /* !!! supports only single output path */
             Path outPath = evalPath(state, a);
 
-            drv.inputDrvs.insert(drvPath);
+            StringSet ids;
+            ids.insert("out");
+            drv.inputDrvs[drvPath] = ids;
             ss.push_back(outPath);
         } else
             throw Error("invalid derivation attribute");

src/libstore/build.cc

@@ -448,9 +448,10 @@ void DerivationGoal::haveStoreExpr()
     }
 
     /* Inputs must be built before we can build this goal. */
-    for (PathSet::iterator i = drv.inputDrvs.begin();
+    /* !!! but if possible, only install the paths that we need */
+    for (DerivationInputs::iterator i = drv.inputDrvs.begin();
          i != drv.inputDrvs.end(); ++i)
-        addWaitee(worker.makeDerivationGoal(*i));
+        addWaitee(worker.makeDerivationGoal(i->first));
 
     for (PathSet::iterator i = drv.inputSrcs.begin();
          i != drv.inputSrcs.end(); ++i)
@@ -812,18 +813,23 @@ bool DerivationGoal::prepareBuild()
     /* Determine the full set of input paths. */
 
     /* First, the input derivations. */
-    for (PathSet::iterator i = drv.inputDrvs.begin();
+    for (DerivationInputs::iterator i = drv.inputDrvs.begin();
          i != drv.inputDrvs.end(); ++i)
     {
-        /* Add all the output closures of the input derivation `*i' as
+        /* Add the relevant output closures of the input derivation
-           input paths.  !!!  there should be a way to indicate
+           `*i' as input paths.  Only add the closures of output paths
-           specific outputs. */
+           that are specified as inputs. */
         /* !!! is `*i' present? */
-        assert(isValidPath(*i));
+        assert(isValidPath(i->first));
-        Derivation inDrv = derivationFromPath(*i);
+        Derivation inDrv = derivationFromPath(i->first);
-        for (DerivationOutputs::iterator j = inDrv.outputs.begin();
+        for (StringSet::iterator j = i->second.begin();
-             j != inDrv.outputs.end(); ++j)
+             j != i->second.begin(); ++j)
-            computeFSClosure(j->second.path, inputPaths);
+            if (inDrv.outputs.find(*j) != inDrv.outputs.end())
+                computeFSClosure(inDrv.outputs[*j].path, inputPaths);
+            else
+                throw Error(
+                    format("derivation `%1%' requires non-existent output `%2%' from input derivation `%3%'")
+                    % drvPath % *j % i->first);
     }
 
     for (PathSet::iterator i = inputPaths.begin(); i != inputPaths.end(); ++i)

src/libstore/derivations-ast.def

@@ -3,4 +3,5 @@ init initDerivationsHelpers
 Derive | ATermList ATermList ATermList string string ATermList ATermList | ATerm |
 
 | string string | ATerm | EnvBinding |
+| string ATermList | ATerm | DerivationInput |
 | string string string string | ATerm | DerivationOutput |

src/libstore/derivations.cc

@@ -26,13 +26,13 @@ static void checkPath(const string & s)
 }
     
 
-static void parsePaths(ATermList paths, PathSet & out)
+static void parseStrings(ATermList paths, StringSet & out, bool arePaths)
 {
     for (ATermIterator i(paths); i; ++i) {
         if (ATgetType(*i) != AT_APPL)
             throw badTerm("not a path", *i);
         string s = aterm2String(*i);
-        checkPath(s);
+        if (arePaths) checkPath(s);
         out.insert(s);
     }
 }
@@ -65,8 +65,19 @@ Derivation parseDerivation(ATerm t)
         drv.outputs[aterm2String(id)] = out;
     }
 
-    parsePaths(inDrvs, drv.inputDrvs);
+    for (ATermIterator i(inDrvs); i; ++i) {
-    parsePaths(inSrcs, drv.inputSrcs);
+        ATerm drvPath;
+        ATermList ids;
+        if (!matchDerivationInput(*i, drvPath, ids))
+            throwBadDrv(t);
+        Path drvPath2 = aterm2String(drvPath);
+        checkPath(drvPath2);
+        StringSet ids2;
+        parseStrings(ids, ids2, false);
+        drv.inputDrvs[drvPath2] = ids2;
+    }
+    
+    parseStrings(inSrcs, drv.inputSrcs, true);
 
     drv.builder = aterm2String(builder);
     drv.platform = aterm2String(platform);
@@ -88,11 +99,11 @@ Derivation parseDerivation(ATerm t)
 }
 
 
-static ATermList unparsePaths(const PathSet & paths)
+static ATermList unparseStrings(const StringSet & paths)
 {
     ATermList l = ATempty;
     for (PathSet::const_iterator i = paths.begin();
-         i != paths.end(); i++)
+         i != paths.end(); ++i)
         l = ATinsert(l, toATerm(*i));
     return ATreverse(l);
 }
@@ -102,7 +113,7 @@ ATerm unparseDerivation(const Derivation & drv)
 {
     ATermList outputs = ATempty;
     for (DerivationOutputs::const_iterator i = drv.outputs.begin();
-         i != drv.outputs.end(); i++)
+         i != drv.outputs.end(); ++i)
         outputs = ATinsert(outputs,
             makeDerivationOutput(
                 toATerm(i->first),
@@ -110,14 +121,22 @@ ATerm unparseDerivation(const Derivation & drv)
                 toATerm(i->second.hashAlgo),
                 toATerm(i->second.hash)));
 
+    ATermList inDrvs = ATempty;
+    for (DerivationInputs::const_iterator i = drv.inputDrvs.begin();
+         i != drv.inputDrvs.end(); ++i)
+        inDrvs = ATinsert(inDrvs,
+            makeDerivationInput(
+                toATerm(i->first),
+                unparseStrings(i->second)));
+    
     ATermList args = ATempty;
     for (Strings::const_iterator i = drv.args.begin();
-         i != drv.args.end(); i++)
+         i != drv.args.end(); ++i)
         args = ATinsert(args, toATerm(*i));
 
     ATermList env = ATempty;
     for (StringPairs::const_iterator i = drv.env.begin();
-         i != drv.env.end(); i++)
+         i != drv.env.end(); ++i)
         env = ATinsert(env,
             makeEnvBinding(
                 toATerm(i->first),
@@ -125,8 +144,8 @@ ATerm unparseDerivation(const Derivation & drv)
 
     return makeDerive(
         ATreverse(outputs),
-        unparsePaths(drv.inputDrvs),
+        ATreverse(inDrvs),
-        unparsePaths(drv.inputSrcs),
+        unparseStrings(drv.inputSrcs),
         toATerm(drv.platform),
         toATerm(drv.builder),
         ATreverse(args),

src/libstore/derivations.hh

@@ -28,12 +28,17 @@ struct DerivationOutput
 };
 
 typedef map<string, DerivationOutput> DerivationOutputs;
+
+/* For inputs that are sub-derivations, we specify exactly which
+   output IDs we are interested in. */
+typedef map<Path, StringSet> DerivationInputs;
+
 typedef map<string, string> StringPairs;
 
 struct Derivation
 {
     DerivationOutputs outputs; /* keyed on symbolic IDs */
-    PathSet inputDrvs; /* inputs that are sub-derivations */
+    DerivationInputs inputDrvs; /* inputs that are sub-derivations */
     PathSet inputSrcs; /* inputs that are sources */
     string platform;
     Path builder;

src/libstore/misc.cc

@@ -39,9 +39,12 @@ void storePathRequisites(const Path & storePath,
         
         Derivation drv = derivationFromPath(storePath);
 
-        for (PathSet::iterator i = drv.inputDrvs.begin();
+        for (DerivationInputs::iterator i = drv.inputDrvs.begin();
              i != drv.inputDrvs.end(); ++i)
-            storePathRequisites(*i, includeOutputs, paths);
+            /* !!! Maybe this is too strict, since it will include
+               *all* output paths of the input derivation, not just
+               the ones needed by this derivation. */
+            storePathRequisites(i->first, includeOutputs, paths);
 
         for (PathSet::iterator i = drv.inputSrcs.begin();
              i != drv.inputSrcs.end(); ++i)

src/libutil/util.cc

@@ -587,6 +587,8 @@ Strings unpackStrings(const string & s)
         len |= ((unsigned char) *i++) << 8;
         len |= ((unsigned char) *i++) << 16;
         len |= ((unsigned char) *i++) << 24;
+
+        if (len == 0xffffffff) return strings; /* explicit end-of-list */
         
         if (i + len > s.end())
             throw Error(format("short db entry: `%1%'") % s);