* Another change to low-level derivations. The last one this year, I

promise :-) This allows derivations to specify on *what* output
  paths of input derivations they are dependent.  This helps to
  prevent unnecessary downloads.  For instance, a build might be
  dependent on the `devel' and `lib' outputs of some library
  component, but not the `docs' output.
This commit is contained in:
Eelco Dolstra 2005-01-20 14:10:19 +00:00
parent 6ff48e77f6
commit 05f0430de1
7 changed files with 70 additions and 32 deletions

View file

@ -68,17 +68,17 @@ static Hash hashDerivationModulo(EvalState & state, Derivation drv)
/* For other derivations, replace the inputs paths with recursive /* For other derivations, replace the inputs paths with recursive
calls to this function.*/ calls to this function.*/
PathSet inputs2; DerivationInputs inputs2;
for (PathSet::iterator i = drv.inputDrvs.begin(); for (DerivationInputs::iterator i = drv.inputDrvs.begin();
i != drv.inputDrvs.end(); ++i) i != drv.inputDrvs.end(); ++i)
{ {
Hash h = state.drvHashes[*i]; Hash h = state.drvHashes[i->first];
if (h.type == htUnknown) { if (h.type == htUnknown) {
Derivation drv2 = derivationFromPath(*i); Derivation drv2 = derivationFromPath(i->first);
h = hashDerivationModulo(state, drv2); h = hashDerivationModulo(state, drv2);
state.drvHashes[*i] = h; state.drvHashes[i->first] = h;
} }
inputs2.insert(printHash(h)); inputs2[printHash(h)] = i->second;
} }
drv.inputDrvs = inputs2; drv.inputDrvs = inputs2;
@ -119,7 +119,9 @@ static void processBinding(EvalState & state, Expr e, Derivation & drv,
/* !!! supports only single output path */ /* !!! supports only single output path */
Path outPath = evalPath(state, a); Path outPath = evalPath(state, a);
drv.inputDrvs.insert(drvPath); StringSet ids;
ids.insert("out");
drv.inputDrvs[drvPath] = ids;
ss.push_back(outPath); ss.push_back(outPath);
} else } else
throw Error("invalid derivation attribute"); throw Error("invalid derivation attribute");

View file

@ -448,9 +448,10 @@ void DerivationGoal::haveStoreExpr()
} }
/* Inputs must be built before we can build this goal. */ /* Inputs must be built before we can build this goal. */
for (PathSet::iterator i = drv.inputDrvs.begin(); /* !!! but if possible, only install the paths that we need */
for (DerivationInputs::iterator i = drv.inputDrvs.begin();
i != drv.inputDrvs.end(); ++i) i != drv.inputDrvs.end(); ++i)
addWaitee(worker.makeDerivationGoal(*i)); addWaitee(worker.makeDerivationGoal(i->first));
for (PathSet::iterator i = drv.inputSrcs.begin(); for (PathSet::iterator i = drv.inputSrcs.begin();
i != drv.inputSrcs.end(); ++i) i != drv.inputSrcs.end(); ++i)
@ -812,18 +813,23 @@ bool DerivationGoal::prepareBuild()
/* Determine the full set of input paths. */ /* Determine the full set of input paths. */
/* First, the input derivations. */ /* First, the input derivations. */
for (PathSet::iterator i = drv.inputDrvs.begin(); for (DerivationInputs::iterator i = drv.inputDrvs.begin();
i != drv.inputDrvs.end(); ++i) i != drv.inputDrvs.end(); ++i)
{ {
/* Add all the output closures of the input derivation `*i' as /* Add the relevant output closures of the input derivation
input paths. !!! there should be a way to indicate `*i' as input paths. Only add the closures of output paths
specific outputs. */ that are specified as inputs. */
/* !!! is `*i' present? */ /* !!! is `*i' present? */
assert(isValidPath(*i)); assert(isValidPath(i->first));
Derivation inDrv = derivationFromPath(*i); Derivation inDrv = derivationFromPath(i->first);
for (DerivationOutputs::iterator j = inDrv.outputs.begin(); for (StringSet::iterator j = i->second.begin();
j != inDrv.outputs.end(); ++j) j != i->second.begin(); ++j)
computeFSClosure(j->second.path, inputPaths); if (inDrv.outputs.find(*j) != inDrv.outputs.end())
computeFSClosure(inDrv.outputs[*j].path, inputPaths);
else
throw Error(
format("derivation `%1%' requires non-existent output `%2%' from input derivation `%3%'")
% drvPath % *j % i->first);
} }
for (PathSet::iterator i = inputPaths.begin(); i != inputPaths.end(); ++i) for (PathSet::iterator i = inputPaths.begin(); i != inputPaths.end(); ++i)

View file

@ -3,4 +3,5 @@ init initDerivationsHelpers
Derive | ATermList ATermList ATermList string string ATermList ATermList | ATerm | Derive | ATermList ATermList ATermList string string ATermList ATermList | ATerm |
| string string | ATerm | EnvBinding | | string string | ATerm | EnvBinding |
| string ATermList | ATerm | DerivationInput |
| string string string string | ATerm | DerivationOutput | | string string string string | ATerm | DerivationOutput |

View file

@ -26,13 +26,13 @@ static void checkPath(const string & s)
} }
static void parsePaths(ATermList paths, PathSet & out) static void parseStrings(ATermList paths, StringSet & out, bool arePaths)
{ {
for (ATermIterator i(paths); i; ++i) { for (ATermIterator i(paths); i; ++i) {
if (ATgetType(*i) != AT_APPL) if (ATgetType(*i) != AT_APPL)
throw badTerm("not a path", *i); throw badTerm("not a path", *i);
string s = aterm2String(*i); string s = aterm2String(*i);
checkPath(s); if (arePaths) checkPath(s);
out.insert(s); out.insert(s);
} }
} }
@ -65,8 +65,19 @@ Derivation parseDerivation(ATerm t)
drv.outputs[aterm2String(id)] = out; drv.outputs[aterm2String(id)] = out;
} }
parsePaths(inDrvs, drv.inputDrvs); for (ATermIterator i(inDrvs); i; ++i) {
parsePaths(inSrcs, drv.inputSrcs); ATerm drvPath;
ATermList ids;
if (!matchDerivationInput(*i, drvPath, ids))
throwBadDrv(t);
Path drvPath2 = aterm2String(drvPath);
checkPath(drvPath2);
StringSet ids2;
parseStrings(ids, ids2, false);
drv.inputDrvs[drvPath2] = ids2;
}
parseStrings(inSrcs, drv.inputSrcs, true);
drv.builder = aterm2String(builder); drv.builder = aterm2String(builder);
drv.platform = aterm2String(platform); drv.platform = aterm2String(platform);
@ -88,11 +99,11 @@ Derivation parseDerivation(ATerm t)
} }
static ATermList unparsePaths(const PathSet & paths) static ATermList unparseStrings(const StringSet & paths)
{ {
ATermList l = ATempty; ATermList l = ATempty;
for (PathSet::const_iterator i = paths.begin(); for (PathSet::const_iterator i = paths.begin();
i != paths.end(); i++) i != paths.end(); ++i)
l = ATinsert(l, toATerm(*i)); l = ATinsert(l, toATerm(*i));
return ATreverse(l); return ATreverse(l);
} }
@ -102,7 +113,7 @@ ATerm unparseDerivation(const Derivation & drv)
{ {
ATermList outputs = ATempty; ATermList outputs = ATempty;
for (DerivationOutputs::const_iterator i = drv.outputs.begin(); for (DerivationOutputs::const_iterator i = drv.outputs.begin();
i != drv.outputs.end(); i++) i != drv.outputs.end(); ++i)
outputs = ATinsert(outputs, outputs = ATinsert(outputs,
makeDerivationOutput( makeDerivationOutput(
toATerm(i->first), toATerm(i->first),
@ -110,14 +121,22 @@ ATerm unparseDerivation(const Derivation & drv)
toATerm(i->second.hashAlgo), toATerm(i->second.hashAlgo),
toATerm(i->second.hash))); toATerm(i->second.hash)));
ATermList inDrvs = ATempty;
for (DerivationInputs::const_iterator i = drv.inputDrvs.begin();
i != drv.inputDrvs.end(); ++i)
inDrvs = ATinsert(inDrvs,
makeDerivationInput(
toATerm(i->first),
unparseStrings(i->second)));
ATermList args = ATempty; ATermList args = ATempty;
for (Strings::const_iterator i = drv.args.begin(); for (Strings::const_iterator i = drv.args.begin();
i != drv.args.end(); i++) i != drv.args.end(); ++i)
args = ATinsert(args, toATerm(*i)); args = ATinsert(args, toATerm(*i));
ATermList env = ATempty; ATermList env = ATempty;
for (StringPairs::const_iterator i = drv.env.begin(); for (StringPairs::const_iterator i = drv.env.begin();
i != drv.env.end(); i++) i != drv.env.end(); ++i)
env = ATinsert(env, env = ATinsert(env,
makeEnvBinding( makeEnvBinding(
toATerm(i->first), toATerm(i->first),
@ -125,8 +144,8 @@ ATerm unparseDerivation(const Derivation & drv)
return makeDerive( return makeDerive(
ATreverse(outputs), ATreverse(outputs),
unparsePaths(drv.inputDrvs), ATreverse(inDrvs),
unparsePaths(drv.inputSrcs), unparseStrings(drv.inputSrcs),
toATerm(drv.platform), toATerm(drv.platform),
toATerm(drv.builder), toATerm(drv.builder),
ATreverse(args), ATreverse(args),

View file

@ -28,12 +28,17 @@ struct DerivationOutput
}; };
typedef map<string, DerivationOutput> DerivationOutputs; typedef map<string, DerivationOutput> DerivationOutputs;
/* For inputs that are sub-derivations, we specify exactly which
output IDs we are interested in. */
typedef map<Path, StringSet> DerivationInputs;
typedef map<string, string> StringPairs; typedef map<string, string> StringPairs;
struct Derivation struct Derivation
{ {
DerivationOutputs outputs; /* keyed on symbolic IDs */ DerivationOutputs outputs; /* keyed on symbolic IDs */
PathSet inputDrvs; /* inputs that are sub-derivations */ DerivationInputs inputDrvs; /* inputs that are sub-derivations */
PathSet inputSrcs; /* inputs that are sources */ PathSet inputSrcs; /* inputs that are sources */
string platform; string platform;
Path builder; Path builder;

View file

@ -39,9 +39,12 @@ void storePathRequisites(const Path & storePath,
Derivation drv = derivationFromPath(storePath); Derivation drv = derivationFromPath(storePath);
for (PathSet::iterator i = drv.inputDrvs.begin(); for (DerivationInputs::iterator i = drv.inputDrvs.begin();
i != drv.inputDrvs.end(); ++i) i != drv.inputDrvs.end(); ++i)
storePathRequisites(*i, includeOutputs, paths); /* !!! Maybe this is too strict, since it will include
*all* output paths of the input derivation, not just
the ones needed by this derivation. */
storePathRequisites(i->first, includeOutputs, paths);
for (PathSet::iterator i = drv.inputSrcs.begin(); for (PathSet::iterator i = drv.inputSrcs.begin();
i != drv.inputSrcs.end(); ++i) i != drv.inputSrcs.end(); ++i)

View file

@ -588,6 +588,8 @@ Strings unpackStrings(const string & s)
len |= ((unsigned char) *i++) << 16; len |= ((unsigned char) *i++) << 16;
len |= ((unsigned char) *i++) << 24; len |= ((unsigned char) *i++) << 24;
if (len == 0xffffffff) return strings; /* explicit end-of-list */
if (i + len > s.end()) if (i + len > s.end())
throw Error(format("short db entry: `%1%'") % s); throw Error(format("short db entry: `%1%'") % s);