Squashed get CA derivations building

This commit is contained in:
John Ericson 2020-08-07 19:09:26 +00:00
parent f7ba16f9cb
commit e913a2989f
29 changed files with 1021 additions and 485 deletions

View file

@ -39,7 +39,9 @@ DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPat
if (i == drv.outputs.end()) if (i == drv.outputs.end())
throw Error("derivation '%s' does not have output '%s'", store->printStorePath(drvPath), outputName); throw Error("derivation '%s' does not have output '%s'", store->printStorePath(drvPath), outputName);
outPath = store->printStorePath(i->second.path(*store, drv.name)); auto optStorePath = i->second.pathOpt(*store, drv.name);
if (optStorePath)
outPath = store->printStorePath(*optStorePath);
} }
@ -77,12 +79,15 @@ string DrvInfo::queryDrvPath() const
string DrvInfo::queryOutPath() const string DrvInfo::queryOutPath() const
{ {
if (outPath == "" && attrs) { if (!outPath && attrs) {
Bindings::iterator i = attrs->find(state->sOutPath); Bindings::iterator i = attrs->find(state->sOutPath);
PathSet context; PathSet context;
outPath = i != attrs->end() ? state->coerceToPath(*i->pos, *i->value, context) : ""; if (i != attrs->end())
outPath = state->coerceToPath(*i->pos, *i->value, context);
} }
return outPath; if (!outPath)
throw UnimplementedError("CA derivations are not yet supported");
return *outPath;
} }

View file

@ -20,7 +20,7 @@ private:
mutable string name; mutable string name;
mutable string system; mutable string system;
mutable string drvPath; mutable string drvPath;
mutable string outPath; mutable std::optional<string> outPath;
mutable string outputName; mutable string outputName;
Outputs outputs; Outputs outputs;

View file

@ -44,16 +44,6 @@ void EvalState::realiseContext(const PathSet & context)
throw InvalidPathError(store->printStorePath(ctx)); throw InvalidPathError(store->printStorePath(ctx));
if (!outputName.empty() && ctx.isDerivation()) { if (!outputName.empty() && ctx.isDerivation()) {
drvs.push_back(StorePathWithOutputs{ctx, {outputName}}); drvs.push_back(StorePathWithOutputs{ctx, {outputName}});
/* Add the output of this derivation to the allowed
paths. */
if (allowedPaths) {
auto drv = store->derivationFromPath(ctx);
DerivationOutputs::iterator i = drv.outputs.find(outputName);
if (i == drv.outputs.end())
throw Error("derivation '%s' does not have an output named '%s'", ctxS, outputName);
allowedPaths->insert(store->printStorePath(i->second.path(*store, drv.name)));
}
} }
} }
@ -69,8 +59,38 @@ void EvalState::realiseContext(const PathSet & context)
store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize); store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize);
store->buildPaths(drvs); store->buildPaths(drvs);
/* Add the output of this derivations to the allowed
paths. */
if (allowedPaths) {
for (auto & [drvPath, outputs] : drvs) {
auto outputPaths = store->queryDerivationOutputMapAssumeTotal(drvPath);
for (auto & outputName : outputs) {
if (outputPaths.count(outputName) == 0)
throw Error("derivation '%s' does not have an output named '%s'",
store->printStorePath(drvPath), outputName);
allowedPaths->insert(store->printStorePath(outputPaths.at(outputName)));
}
}
}
} }
static void mkOutputString(EvalState & state, Value & v,
const StorePath & drvPath, const BasicDerivation & drv,
std::pair<string, DerivationOutput> o)
{
auto optOutputPath = o.second.pathOpt(*state.store, drv.name);
mkString(
*state.allocAttr(v, state.symbols.create(o.first)),
state.store->printStorePath(optOutputPath
? *optOutputPath
/* Downstream we would substitute this for an actual path once
we build the floating CA derivation */
/* FIXME: we need to depend on the basic derivation, not
derivation */
: downstreamPlaceholder(*state.store, drvPath, o.first)),
{"!" + o.first + "!" + state.store->printStorePath(drvPath)});
}
/* Load and evaluate an expression from path specified by the /* Load and evaluate an expression from path specified by the
argument. */ argument. */
@ -114,8 +134,7 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
unsigned int outputs_index = 0; unsigned int outputs_index = 0;
for (const auto & o : drv.outputs) { for (const auto & o : drv.outputs) {
v2 = state.allocAttr(w, state.symbols.create(o.first)); mkOutputString(state, w, storePath, drv, o);
mkString(*v2, state.store->printStorePath(o.second.path(*state.store, drv.name)), {"!" + o.first + "!" + path});
outputsVal->listElems()[outputs_index] = state.allocValue(); outputsVal->listElems()[outputs_index] = state.allocValue();
mkString(*(outputsVal->listElems()[outputs_index++]), o.first); mkString(*(outputsVal->listElems()[outputs_index++]), o.first);
} }
@ -846,16 +865,18 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
/* Optimisation, but required in read-only mode! because in that /* Optimisation, but required in read-only mode! because in that
case we don't actually write store derivations, so we can't case we don't actually write store derivations, so we can't
read them later. */ read them later.
drvHashes.insert_or_assign(drvPath,
hashDerivationModulo(*state.store, Derivation(drv), false)); However, we don't bother doing this for floating CA derivations because
their "hash modulo" is indeterminate until built. */
if (drv.type() != DerivationType::CAFloating)
drvHashes.insert_or_assign(drvPath,
hashDerivationModulo(*state.store, Derivation(drv), false));
state.mkAttrs(v, 1 + drv.outputs.size()); state.mkAttrs(v, 1 + drv.outputs.size());
mkString(*state.allocAttr(v, state.sDrvPath), drvPathS, {"=" + drvPathS}); mkString(*state.allocAttr(v, state.sDrvPath), drvPathS, {"=" + drvPathS});
for (auto & i : drv.outputs) { for (auto & i : drv.outputs)
mkString(*state.allocAttr(v, state.symbols.create(i.first)), mkOutputString(state, v, drvPath, drv, i);
state.store->printStorePath(i.second.path(*state.store, drv.name)), {"!" + i.first + "!" + drvPathS});
}
v.attrs->sort(); v.attrs->sort();
} }

File diff suppressed because it is too large Load diff

View file

@ -15,7 +15,8 @@ std::optional<StorePath> DerivationOutput::pathOpt(const Store & store, std::str
}, },
[&](DerivationOutputCAFixed dof) -> std::optional<StorePath> { [&](DerivationOutputCAFixed dof) -> std::optional<StorePath> {
return { return {
store.makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName) // FIXME if we intend to support multiple CA outputs.
dof.path(store, drvName, "out")
}; };
}, },
[](DerivationOutputCAFloating dof) -> std::optional<StorePath> { [](DerivationOutputCAFloating dof) -> std::optional<StorePath> {
@ -25,6 +26,13 @@ std::optional<StorePath> DerivationOutput::pathOpt(const Store & store, std::str
} }
StorePath DerivationOutputCAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const {
return store.makeFixedOutputPath(
hash.method, hash.hash,
outputPathName(drvName, outputName));
}
bool derivationIsCA(DerivationType dt) { bool derivationIsCA(DerivationType dt) {
switch (dt) { switch (dt) {
case DerivationType::InputAddressed: return false; case DerivationType::InputAddressed: return false;
@ -106,12 +114,15 @@ static string parseString(std::istream & str)
return res; return res;
} }
static void validatePath(std::string_view s) {
if (s.size() == 0 || s[0] != '/')
throw FormatError("bad path '%1%' in derivation", s);
}
static Path parsePath(std::istream & str) static Path parsePath(std::istream & str)
{ {
string s = parseString(str); auto s = parseString(str);
if (s.size() == 0 || s[0] != '/') validatePath(s);
throw FormatError("bad path '%1%' in derivation", s);
return s; return s;
} }
@ -141,7 +152,7 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
static DerivationOutput parseDerivationOutput(const Store & store, std::istringstream & str) static DerivationOutput parseDerivationOutput(const Store & store, std::istringstream & str)
{ {
expect(str, ","); auto path = store.parseStorePath(parsePath(str)); expect(str, ","); auto pathS = parseString(str);
expect(str, ","); auto hashAlgo = parseString(str); expect(str, ","); auto hashAlgo = parseString(str);
expect(str, ","); const auto hash = parseString(str); expect(str, ","); const auto hash = parseString(str);
expect(str, ")"); expect(str, ")");
@ -152,30 +163,35 @@ static DerivationOutput parseDerivationOutput(const Store & store, std::istrings
method = FileIngestionMethod::Recursive; method = FileIngestionMethod::Recursive;
hashAlgo = string(hashAlgo, 2); hashAlgo = string(hashAlgo, 2);
} }
const HashType hashType = parseHashType(hashAlgo); const auto hashType = parseHashType(hashAlgo);
if (hash != "") {
return hash != "" validatePath(pathS);
? DerivationOutput { return DerivationOutput {
.output = DerivationOutputCAFixed { .output = DerivationOutputCAFixed {
.hash = FixedOutputHash { .hash = FixedOutputHash {
.method = std::move(method), .method = std::move(method),
.hash = Hash::parseNonSRIUnprefixed(hash, hashType), .hash = Hash::parseNonSRIUnprefixed(hash, hashType),
}, },
} },
} };
: (settings.requireExperimentalFeature("ca-derivations"), } else {
DerivationOutput { settings.requireExperimentalFeature("ca-derivations");
.output = DerivationOutputCAFloating { assert(pathS == "");
.method = std::move(method), return DerivationOutput {
.hashType = std::move(hashType), .output = DerivationOutputCAFloating {
}, .method = std::move(method),
}); .hashType = std::move(hashType),
} else },
};
}
} else {
validatePath(pathS);
return DerivationOutput { return DerivationOutput {
.output = DerivationOutputInputAddressed { .output = DerivationOutputInputAddressed {
.path = std::move(path), .path = store.parseStorePath(pathS),
} }
}; };
}
} }
@ -316,17 +332,19 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
for (auto & i : outputs) { for (auto & i : outputs) {
if (first) first = false; else s += ','; if (first) first = false; else s += ',';
s += '('; printUnquotedString(s, i.first); s += '('; printUnquotedString(s, i.first);
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path(store, name)));
std::visit(overloaded { std::visit(overloaded {
[&](DerivationOutputInputAddressed doi) { [&](DerivationOutputInputAddressed doi) {
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(doi.path));
s += ','; printUnquotedString(s, ""); s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, ""); s += ','; printUnquotedString(s, "");
}, },
[&](DerivationOutputCAFixed dof) { [&](DerivationOutputCAFixed dof) {
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first)));
s += ','; printUnquotedString(s, dof.hash.printMethodAlgo()); s += ','; printUnquotedString(s, dof.hash.printMethodAlgo());
s += ','; printUnquotedString(s, dof.hash.hash.to_string(Base16, false)); s += ','; printUnquotedString(s, dof.hash.hash.to_string(Base16, false));
}, },
[&](DerivationOutputCAFloating dof) { [&](DerivationOutputCAFloating dof) {
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)); s += ','; printUnquotedString(s, makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType));
s += ','; printUnquotedString(s, ""); s += ','; printUnquotedString(s, "");
}, },
@ -382,6 +400,16 @@ bool isDerivation(const string & fileName)
} }
std::string outputPathName(std::string_view drvName, std::string_view outputName) {
std::string res { drvName };
if (outputName != "out") {
res += "-";
res += outputName;
}
return res;
}
DerivationType BasicDerivation::type() const DerivationType BasicDerivation::type() const
{ {
std::set<std::string_view> inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs; std::set<std::string_view> inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs;
@ -479,7 +507,7 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
auto hash = hashString(htSHA256, "fixed:out:" auto hash = hashString(htSHA256, "fixed:out:"
+ dof.hash.printMethodAlgo() + ":" + dof.hash.printMethodAlgo() + ":"
+ dof.hash.hash.to_string(Base16, false) + ":" + dof.hash.hash.to_string(Base16, false) + ":"
+ store.printStorePath(i.second.path(store, drv.name))); + store.printStorePath(dof.path(store, drv.name, i.first)));
outputHashes.insert_or_assign(i.first, std::move(hash)); outputHashes.insert_or_assign(i.first, std::move(hash));
} }
return outputHashes; return outputHashes;
@ -530,17 +558,9 @@ bool wantOutput(const string & output, const std::set<string> & wanted)
} }
StorePathSet BasicDerivation::outputPaths(const Store & store) const
{
StorePathSet paths;
for (auto & i : outputs)
paths.insert(i.second.path(store, name));
return paths;
}
static DerivationOutput readDerivationOutput(Source & in, const Store & store) static DerivationOutput readDerivationOutput(Source & in, const Store & store)
{ {
auto path = store.parseStorePath(readString(in)); auto pathS = readString(in);
auto hashAlgo = readString(in); auto hashAlgo = readString(in);
auto hash = readString(in); auto hash = readString(in);
@ -550,29 +570,35 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store)
method = FileIngestionMethod::Recursive; method = FileIngestionMethod::Recursive;
hashAlgo = string(hashAlgo, 2); hashAlgo = string(hashAlgo, 2);
} }
auto hashType = parseHashType(hashAlgo); const auto hashType = parseHashType(hashAlgo);
return hash != "" if (hash != "") {
? DerivationOutput { validatePath(pathS);
.output = DerivationOutputCAFixed { return DerivationOutput {
.hash = FixedOutputHash { .output = DerivationOutputCAFixed {
.method = std::move(method), .hash = FixedOutputHash {
.hash = Hash::parseNonSRIUnprefixed(hash, hashType), .method = std::move(method),
}, .hash = Hash::parseNonSRIUnprefixed(hash, hashType),
} },
} },
: (settings.requireExperimentalFeature("ca-derivations"), };
DerivationOutput { } else {
.output = DerivationOutputCAFloating { settings.requireExperimentalFeature("ca-derivations");
.method = std::move(method), assert(pathS == "");
.hashType = std::move(hashType), return DerivationOutput {
}, .output = DerivationOutputCAFloating {
}); .method = std::move(method),
} else .hashType = std::move(hashType),
},
};
}
} else {
validatePath(pathS);
return DerivationOutput { return DerivationOutput {
.output = DerivationOutputInputAddressed { .output = DerivationOutputInputAddressed {
.path = std::move(path), .path = store.parseStorePath(pathS),
} }
}; };
}
} }
StringSet BasicDerivation::outputNames() const StringSet BasicDerivation::outputNames() const
@ -624,18 +650,21 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
{ {
out << drv.outputs.size(); out << drv.outputs.size();
for (auto & i : drv.outputs) { for (auto & i : drv.outputs) {
out << i.first out << i.first;
<< store.printStorePath(i.second.path(store, drv.name));
std::visit(overloaded { std::visit(overloaded {
[&](DerivationOutputInputAddressed doi) { [&](DerivationOutputInputAddressed doi) {
out << "" << ""; out << store.printStorePath(doi.path)
<< ""
<< "";
}, },
[&](DerivationOutputCAFixed dof) { [&](DerivationOutputCAFixed dof) {
out << dof.hash.printMethodAlgo() out << store.printStorePath(dof.path(store, drv.name, i.first))
<< dof.hash.printMethodAlgo()
<< dof.hash.hash.to_string(Base16, false); << dof.hash.hash.to_string(Base16, false);
}, },
[&](DerivationOutputCAFloating dof) { [&](DerivationOutputCAFloating dof) {
out << (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)) out << ""
<< (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType))
<< ""; << "";
}, },
}, i.second.output); }, i.second.output);
@ -654,5 +683,14 @@ std::string hashPlaceholder(const std::string & outputName)
return "/" + hashString(htSHA256, "nix-output:" + outputName).to_string(Base32, false); return "/" + hashString(htSHA256, "nix-output:" + outputName).to_string(Base32, false);
} }
StorePath downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName)
{
auto drvNameWithExtension = drvPath.name();
auto drvName = drvNameWithExtension.substr(0, drvNameWithExtension.size() - 4);
return store.makeStorePath(
"downstream-placeholder:" + std::string { drvPath.name() } + ":" + std::string { outputName },
"compressed:" + std::string { drvPath.hashPart() },
outputPathName(drvName, outputName));
}
} }

View file

@ -27,6 +27,7 @@ struct DerivationOutputInputAddressed
struct DerivationOutputCAFixed struct DerivationOutputCAFixed
{ {
FixedOutputHash hash; /* hash used for expected hash computation */ FixedOutputHash hash; /* hash used for expected hash computation */
StorePath path(const Store & store, std::string_view drvName, std::string_view outputName) const;
}; };
/* Floating-output derivations, whose output paths are content addressed, but /* Floating-output derivations, whose output paths are content addressed, but
@ -48,12 +49,6 @@ struct DerivationOutput
> output; > output;
std::optional<HashType> hashAlgoOpt(const Store & store) const; std::optional<HashType> hashAlgoOpt(const Store & store) const;
std::optional<StorePath> pathOpt(const Store & store, std::string_view drvName) const; std::optional<StorePath> pathOpt(const Store & store, std::string_view drvName) const;
/* DEPRECATED: Remove after CA drvs are fully implemented */
StorePath path(const Store & store, std::string_view drvName) const {
auto p = pathOpt(store, drvName);
if (!p) throw UnimplementedError("floating content-addressed derivations are not yet implemented");
return *p;
}
}; };
typedef std::map<string, DerivationOutput> DerivationOutputs; typedef std::map<string, DerivationOutput> DerivationOutputs;
@ -101,9 +96,6 @@ struct BasicDerivation
/* Return true iff this is a fixed-output derivation. */ /* Return true iff this is a fixed-output derivation. */
DerivationType type() const; DerivationType type() const;
/* Return the output paths of a derivation. */
StorePathSet outputPaths(const Store & store) const;
/* Return the output names of a derivation. */ /* Return the output names of a derivation. */
StringSet outputNames() const; StringSet outputNames() const;
@ -136,6 +128,8 @@ Derivation readDerivation(const Store & store, const Path & drvPath, std::string
// FIXME: remove // FIXME: remove
bool isDerivation(const string & fileName); bool isDerivation(const string & fileName);
std::string outputPathName(std::string_view drvName, std::string_view outputName);
// known CA drv's output hashes, current just for fixed-output derivations // known CA drv's output hashes, current just for fixed-output derivations
// whose output hashes are always known since they are fixed up-front. // whose output hashes are always known since they are fixed up-front.
typedef std::map<std::string, Hash> CaOutputHashes; typedef std::map<std::string, Hash> CaOutputHashes;
@ -185,4 +179,6 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
std::string hashPlaceholder(const std::string & outputName); std::string hashPlaceholder(const std::string & outputName);
StorePath downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName);
} }

View file

@ -578,13 +578,32 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
envHasRightPath(path, i.first); envHasRightPath(path, i.first);
}, },
[&](DerivationOutputCAFloating _) { [&](DerivationOutputCAFloating _) {
throw UnimplementedError("floating CA output derivations are not yet implemented"); /* Nothing to check */
}, },
}, i.second.output); }, i.second.output);
} }
} }
void LocalStore::linkDeriverToPath(const StorePath & deriver, const string & outputName, const StorePath & output)
{
auto state(_state.lock());
return linkDeriverToPath(*state, queryValidPathId(*state, deriver), outputName, output);
}
void LocalStore::linkDeriverToPath(State & state, uint64_t deriver, const string & outputName, const StorePath & output)
{
retrySQLite<void>([&]() {
state.stmtAddDerivationOutput.use()
(deriver)
(outputName)
(printStorePath(output))
.exec();
});
}
uint64_t LocalStore::addValidPath(State & state, uint64_t LocalStore::addValidPath(State & state,
const ValidPathInfo & info, bool checkOutputs) const ValidPathInfo & info, bool checkOutputs)
{ {
@ -619,11 +638,11 @@ uint64_t LocalStore::addValidPath(State & state,
if (checkOutputs) checkDerivationOutputs(info.path, drv); if (checkOutputs) checkDerivationOutputs(info.path, drv);
for (auto & i : drv.outputs) { for (auto & i : drv.outputs) {
state.stmtAddDerivationOutput.use() /* Floating CA derivations have indeterminate output paths until
(id) they are built, so don't register anything in that case */
(i.first) auto optPath = i.second.pathOpt(*this, drv.name);
(printStorePath(i.second.path(*this, drv.name))) if (optPath)
.exec(); linkDeriverToPath(state, id, i.first, *optPath);
} }
} }

View file

@ -282,6 +282,12 @@ private:
specified by the secret-key-files option. */ specified by the secret-key-files option. */
void signPathInfo(ValidPathInfo & info); void signPathInfo(ValidPathInfo & info);
/* Add a mapping from the deriver of the path info (if specified) to its
* out path
*/
void linkDeriverToPath(const StorePath & deriver, const string & outputName, const StorePath & output);
void linkDeriverToPath(State & state, uint64_t deriver, const string & outputName, const StorePath & output);
Path getRealStoreDir() override { return realStoreDir; } Path getRealStoreDir() override { return realStoreDir; }
void createUser(const std::string & userName, uid_t userId) override; void createUser(const std::string & userName, uid_t userId) override;

View file

@ -203,17 +203,24 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
return; return;
} }
PathSet invalid;
/* true for regular derivations, and CA derivations for which we
have a trust mapping for all wanted outputs. */
auto knownOutputPaths = true;
for (auto & [outputName, pathOpt] : queryDerivationOutputMap(path.path)) {
if (!pathOpt) {
knownOutputPaths = false;
break;
}
if (wantOutput(outputName, path.outputs) && !isValidPath(*pathOpt))
invalid.insert(printStorePath(*pathOpt));
}
if (knownOutputPaths && invalid.empty()) return;
auto drv = make_ref<Derivation>(derivationFromPath(path.path)); auto drv = make_ref<Derivation>(derivationFromPath(path.path));
ParsedDerivation parsedDrv(StorePath(path.path), *drv); ParsedDerivation parsedDrv(StorePath(path.path), *drv);
PathSet invalid; if (knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
for (auto & j : drv->outputs)
if (wantOutput(j.first, path.outputs)
&& !isValidPath(j.second.path(*this, drv->name)))
invalid.insert(printStorePath(j.second.path(*this, drv->name)));
if (invalid.empty()) return;
if (settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size())); auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size()));
for (auto & output : invalid) for (auto & output : invalid)
pool.enqueue(std::bind(checkOutput, printStorePath(path.path), drv, output, drvState)); pool.enqueue(std::bind(checkOutput, printStorePath(path.path), drv, output, drvState));

View file

@ -79,9 +79,17 @@ void RefScanSink::operator () (const unsigned char * data, size_t len)
std::pair<PathSet, HashResult> scanForReferences(const string & path, std::pair<PathSet, HashResult> scanForReferences(const string & path,
const PathSet & refs) const PathSet & refs)
{ {
RefScanSink refsSink;
HashSink hashSink { htSHA256 }; HashSink hashSink { htSHA256 };
TeeSink sink { refsSink, hashSink }; auto found = scanForReferences(hashSink, path, refs);
auto hash = hashSink.finish();
return std::pair<PathSet, HashResult>(found, hash);
}
PathSet scanForReferences(Sink & toTee,
const string & path, const PathSet & refs)
{
RefScanSink refsSink;
TeeSink sink { refsSink, toTee };
std::map<string, Path> backMap; std::map<string, Path> backMap;
/* For efficiency (and a higher hit rate), just search for the /* For efficiency (and a higher hit rate), just search for the
@ -111,9 +119,7 @@ std::pair<PathSet, HashResult> scanForReferences(const string & path,
found.insert(j->second); found.insert(j->second);
} }
auto hash = hashSink.finish(); return found;
return std::pair<PathSet, HashResult>(found, hash);
} }

View file

@ -7,6 +7,8 @@ namespace nix {
std::pair<PathSet, HashResult> scanForReferences(const Path & path, const PathSet & refs); std::pair<PathSet, HashResult> scanForReferences(const Path & path, const PathSet & refs);
PathSet scanForReferences(Sink & toTee, const Path & path, const PathSet & refs);
struct RewritingSink : Sink struct RewritingSink : Sink
{ {
std::string from, to, prev; std::string from, to, prev;

View file

@ -74,7 +74,7 @@ void write(const Store & store, Sink & out, const StorePath & storePath)
template<> template<>
std::optional<StorePath> read(const Store & store, Source & from, Phantom<std::optional<StorePath>> _) std::optional<StorePath> read(const Store & store, Source & from, Phantom<std::optional<StorePath>> _)
{ {
auto s = readString(from); auto s = readString(from);
return s == "" ? std::optional<StorePath> {} : store.parseStorePath(s); return s == "" ? std::optional<StorePath> {} : store.parseStorePath(s);
} }

View file

@ -140,21 +140,28 @@ StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view p
*/ */
StorePath Store::makeStorePath(const string & type, StorePath Store::makeStorePath(std::string_view type,
const Hash & hash, std::string_view name) const std::string_view hash, std::string_view name) const
{ {
/* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */ /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */
string s = type + ":" + hash.to_string(Base16, true) + ":" + storeDir + ":" + std::string(name); string s = std::string { type } + ":" + std::string { hash }
+ ":" + storeDir + ":" + std::string { name };
auto h = compressHash(hashString(htSHA256, s), 20); auto h = compressHash(hashString(htSHA256, s), 20);
return StorePath(h, name); return StorePath(h, name);
} }
StorePath Store::makeOutputPath(const string & id, StorePath Store::makeStorePath(std::string_view type,
const Hash & hash, std::string_view name) const const Hash & hash, std::string_view name) const
{ {
return makeStorePath("output:" + id, hash, return makeStorePath(type, hash.to_string(Base16, true), name);
std::string(name) + (id == "out" ? "" : "-" + id)); }
StorePath Store::makeOutputPath(std::string_view id,
const Hash & hash, std::string_view name) const
{
return makeStorePath("output:" + std::string { id }, hash, outputPathName(name, id));
} }

View file

@ -244,10 +244,12 @@ public:
StorePathWithOutputs followLinksToStorePathWithOutputs(std::string_view path) const; StorePathWithOutputs followLinksToStorePathWithOutputs(std::string_view path) const;
/* Constructs a unique store path name. */ /* Constructs a unique store path name. */
StorePath makeStorePath(const string & type, StorePath makeStorePath(std::string_view type,
std::string_view hash, std::string_view name) const;
StorePath makeStorePath(std::string_view type,
const Hash & hash, std::string_view name) const; const Hash & hash, std::string_view name) const;
StorePath makeOutputPath(const string & id, StorePath makeOutputPath(std::string_view id,
const Hash & hash, std::string_view name) const; const Hash & hash, std::string_view name) const;
StorePath makeFixedOutputPath(FileIngestionMethod method, StorePath makeFixedOutputPath(FileIngestionMethod method,

View file

@ -22,6 +22,12 @@ struct Sink
} }
}; };
/* Just throws away data. */
struct NullSink : Sink
{
void operator () (const unsigned char * data, size_t len) override
{ }
};
/* A buffered abstract sink. */ /* A buffered abstract sink. */
struct BufferedSink : virtual Sink struct BufferedSink : virtual Sink

View file

@ -487,50 +487,56 @@ static void _main(int argc, char * * argv)
std::vector<StorePathWithOutputs> pathsToBuild; std::vector<StorePathWithOutputs> pathsToBuild;
std::map<Path, Path> drvPrefixes; std::map<StorePath, std::pair<size_t, StringSet>> drvMap;
std::map<Path, Path> resultSymlinks;
std::vector<Path> outPaths;
for (auto & drvInfo : drvs) { for (auto & drvInfo : drvs) {
auto drvPath = drvInfo.queryDrvPath(); auto drvPath = store->parseStorePath(drvInfo.queryDrvPath());
auto outPath = drvInfo.queryOutPath();
auto outputName = drvInfo.queryOutputName(); auto outputName = drvInfo.queryOutputName();
if (outputName == "") if (outputName == "")
throw Error("derivation '%s' lacks an 'outputName' attribute", drvPath); throw Error("derivation '%s' lacks an 'outputName' attribute", store->printStorePath(drvPath));
pathsToBuild.push_back({store->parseStorePath(drvPath), {outputName}}); pathsToBuild.push_back({drvPath, {outputName}});
std::string drvPrefix; auto i = drvMap.find(drvPath);
auto i = drvPrefixes.find(drvPath); if (i != drvMap.end())
if (i != drvPrefixes.end()) i->second.second.insert(outputName);
drvPrefix = i->second;
else { else {
drvPrefix = outLink; drvMap[drvPath] = {drvMap.size(), {outputName}};
if (drvPrefixes.size())
drvPrefix += fmt("-%d", drvPrefixes.size() + 1);
drvPrefixes[drvPath] = drvPrefix;
} }
std::string symlink = drvPrefix;
if (outputName != "out") symlink += "-" + outputName;
resultSymlinks[symlink] = outPath;
outPaths.push_back(outPath);
} }
buildPaths(pathsToBuild); buildPaths(pathsToBuild);
if (dryRun) return; if (dryRun) return;
for (auto & symlink : resultSymlinks) std::vector<StorePath> outPaths;
if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
store2->addPermRoot(store->parseStorePath(symlink.second), absPath(symlink.first), true); for (auto & [drvPath, info] : drvMap) {
auto & [counter, wantedOutputs] = info;
std::string drvPrefix = outLink;
if (counter)
drvPrefix += fmt("-%d", counter + 1);
auto builtOutputs = store->queryDerivationOutputMapAssumeTotal(drvPath);
for (auto & outputName : wantedOutputs) {
auto outputPath = builtOutputs.at(outputName);
if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) {
std::string symlink = drvPrefix;
if (outputName != "out") symlink += "-" + outputName;
store2->addPermRoot(outputPath, absPath(symlink), true);
}
outPaths.push_back(outputPath);
}
}
logger->stop(); logger->stop();
for (auto & path : outPaths) for (auto & path : outPaths)
std::cout << path << '\n'; std::cout << store->printStorePath(path) << '\n';
} }
} }

View file

@ -65,6 +65,7 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true)
if (path.path.isDerivation()) { if (path.path.isDerivation()) {
if (build) store->buildPaths({path}); if (build) store->buildPaths({path});
auto outputPaths = store->queryDerivationOutputMapAssumeTotal(path.path);
Derivation drv = store->derivationFromPath(path.path); Derivation drv = store->derivationFromPath(path.path);
rootNr++; rootNr++;
@ -77,7 +78,8 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true)
if (i == drv.outputs.end()) if (i == drv.outputs.end())
throw Error("derivation '%s' does not have an output named '%s'", throw Error("derivation '%s' does not have an output named '%s'",
store2->printStorePath(path.path), j); store2->printStorePath(path.path), j);
auto outPath = store2->printStorePath(i->second.path(*store, drv.name)); auto outPath = outputPaths.at(i->first);
auto retPath = store->printStorePath(outPath);
if (store2) { if (store2) {
if (gcRoot == "") if (gcRoot == "")
printGCWarning(); printGCWarning();
@ -85,10 +87,10 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true)
Path rootName = gcRoot; Path rootName = gcRoot;
if (rootNr > 1) rootName += "-" + std::to_string(rootNr); if (rootNr > 1) rootName += "-" + std::to_string(rootNr);
if (i->first != "out") rootName += "-" + i->first; if (i->first != "out") rootName += "-" + i->first;
outPath = store2->addPermRoot(store->parseStorePath(outPath), rootName, indirectRoot); retPath = store2->addPermRoot(outPath, rootName, indirectRoot);
} }
} }
outputs.insert(outPath); outputs.insert(retPath);
} }
return outputs; return outputs;
} }
@ -218,8 +220,14 @@ static StorePathSet maybeUseOutputs(const StorePath & storePath, bool useOutput,
if (useOutput && storePath.isDerivation()) { if (useOutput && storePath.isDerivation()) {
auto drv = store->derivationFromPath(storePath); auto drv = store->derivationFromPath(storePath);
StorePathSet outputs; StorePathSet outputs;
for (auto & i : drv.outputs) if (forceRealise)
outputs.insert(i.second.path(*store, drv.name)); return store->queryDerivationOutputs(storePath);
for (auto & i : drv.outputs) {
auto optPath = i.second.pathOpt(*store, drv.name);
if (!optPath)
throw UsageError("Cannot use output path of floating content-addressed derivation until we know what it is (e.g. by building it)");
outputs.insert(*optPath);
}
return outputs; return outputs;
} }
else return {storePath}; else return {storePath};
@ -309,11 +317,9 @@ static void opQuery(Strings opFlags, Strings opArgs)
case qOutputs: { case qOutputs: {
for (auto & i : opArgs) { for (auto & i : opArgs) {
auto i2 = store->followLinksToStorePath(i); auto outputs = maybeUseOutputs(store->followLinksToStorePath(i), true, forceRealise);
if (forceRealise) realisePath({i2}); for (auto & outputPath : outputs)
Derivation drv = store->derivationFromPath(i2); cout << fmt("%1%\n", store->printStorePath(outputPath));
for (auto & j : drv.outputs)
cout << fmt("%1%\n", store->printStorePath(j.second.path(*store, drv.name)));
} }
break; break;
} }

View file

@ -74,7 +74,8 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile
store2->addPermRoot(bo.path, absPath(symlink), true); store2->addPermRoot(bo.path, absPath(symlink), true);
}, },
[&](BuildableFromDrv bfd) { [&](BuildableFromDrv bfd) {
for (auto & output : bfd.outputs) { auto builtOutputs = store->queryDerivationOutputMapAssumeTotal(bfd.drvPath);
for (auto & output : builtOutputs) {
std::string symlink = outLink; std::string symlink = outLink;
if (i) symlink += fmt("-%d", i); if (i) symlink += fmt("-%d", i);
if (output.first != "out") symlink += fmt("-%s", output.first); if (output.first != "out") symlink += fmt("-%s", output.first);

View file

@ -137,7 +137,9 @@ void MixProfile::updateProfile(const Buildables & buildables)
}, },
[&](BuildableFromDrv bfd) { [&](BuildableFromDrv bfd) {
for (auto & output : bfd.outputs) { for (auto & output : bfd.outputs) {
result.push_back(output.second); if (!output.second)
throw Error("output path should be known because we just tried to build it");
result.push_back(*output.second);
} }
}, },
}, buildable); }, buildable);

View file

@ -302,10 +302,10 @@ struct InstallableStorePath : Installable
Buildables toBuildables() override Buildables toBuildables() override
{ {
if (storePath.isDerivation()) { if (storePath.isDerivation()) {
std::map<std::string, StorePath> outputs; std::map<std::string, std::optional<StorePath>> outputs;
auto drv = store->readDerivation(storePath); auto drv = store->readDerivation(storePath);
for (auto & [name, output] : drv.outputs) for (auto & [name, output] : drv.outputs)
outputs.emplace(name, output.path(*store, drv.name)); outputs.emplace(name, output.pathOpt(*store, drv.name));
return { return {
BuildableFromDrv { BuildableFromDrv {
.drvPath = storePath, .drvPath = storePath,
@ -331,7 +331,7 @@ Buildables InstallableValue::toBuildables()
{ {
Buildables res; Buildables res;
std::map<StorePath, OutputPathMap> drvsToOutputs; std::map<StorePath, std::map<std::string, std::optional<StorePath>>> drvsToOutputs;
// Group by derivation, helps with .all in particular // Group by derivation, helps with .all in particular
for (auto & drv : toDerivations()) { for (auto & drv : toDerivations()) {
@ -674,8 +674,11 @@ StorePathSet toStorePaths(ref<Store> store,
outPaths.insert(bo.path); outPaths.insert(bo.path);
}, },
[&](BuildableFromDrv bfd) { [&](BuildableFromDrv bfd) {
for (auto & output : bfd.outputs) for (auto & output : bfd.outputs) {
outPaths.insert(output.second); if (!output.second)
throw Error("Cannot operate on output of unbuilt CA drv");
outPaths.insert(*output.second);
}
}, },
}, b); }, b);
} else { } else {

View file

@ -20,7 +20,7 @@ struct BuildableOpaque {
struct BuildableFromDrv { struct BuildableFromDrv {
StorePath drvPath; StorePath drvPath;
std::map<std::string, StorePath> outputs; std::map<std::string, std::optional<StorePath>> outputs;
}; };
typedef std::variant< typedef std::variant<
@ -82,7 +82,7 @@ struct InstallableValue : Installable
struct DerivationInfo struct DerivationInfo
{ {
StorePath drvPath; StorePath drvPath;
StorePath outPath; std::optional<StorePath> outPath;
std::string outputName; std::string outputName;
}; };

View file

@ -178,7 +178,9 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
auto [attrPath, resolvedRef, drv] = installable2->toDerivation(); auto [attrPath, resolvedRef, drv] = installable2->toDerivation();
ProfileElement element; ProfileElement element;
element.storePaths = {drv.outPath}; // FIXME if (!drv.outPath)
throw UnimplementedError("CA derivations are not yet supported by 'nix profile'");
element.storePaths = {*drv.outPath}; // FIXME
element.source = ProfileElementSource{ element.source = ProfileElementSource{
installable2->flakeRef, installable2->flakeRef,
resolvedRef, resolvedRef,
@ -189,7 +191,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
manifest.elements.emplace_back(std::move(element)); manifest.elements.emplace_back(std::move(element));
} else } else
throw Error("'nix profile install' does not support argument '%s'", installable->what()); throw UnimplementedError("'nix profile install' does not support argument '%s'", installable->what());
} }
store->buildPaths(pathsToBuild); store->buildPaths(pathsToBuild);
@ -347,7 +349,9 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
printInfo("upgrading '%s' from flake '%s' to '%s'", printInfo("upgrading '%s' from flake '%s' to '%s'",
element.source->attrPath, element.source->resolvedRef, resolvedRef); element.source->attrPath, element.source->resolvedRef, resolvedRef);
element.storePaths = {drv.outPath}; // FIXME if (!drv.outPath)
throw UnimplementedError("CA derivations are not yet supported by 'nix profile'");
element.storePaths = {*drv.outPath}; // FIXME
element.source = ProfileElementSource{ element.source = ProfileElementSource{
installable.flakeRef, installable.flakeRef,
resolvedRef, resolvedRef,

View file

@ -488,10 +488,10 @@ bool NixRepl::processLine(string line)
but doing it in a child makes it easier to recover from but doing it in a child makes it easier to recover from
problems / SIGINT. */ problems / SIGINT. */
if (runProgram(settings.nixBinDir + "/nix", Strings{"build", "--no-link", drvPath}) == 0) { if (runProgram(settings.nixBinDir + "/nix", Strings{"build", "--no-link", drvPath}) == 0) {
auto drv = readDerivation(*state->store, drvPath, Derivation::nameFromPath(state->store->parseStorePath(drvPath))); auto outputs = state->store->queryDerivationOutputMapAssumeTotal(state->store->parseStorePath(drvPath));
std::cout << std::endl << "this derivation produced the following outputs:" << std::endl; std::cout << std::endl << "this derivation produced the following outputs:" << std::endl;
for (auto & i : drv.outputs) for (auto & i : outputs)
std::cout << fmt(" %s -> %s\n", i.first, state->store->printStorePath(i.second.path(*state->store, drv.name))); std::cout << fmt(" %s -> %s\n", i.first, state->store->printStorePath(i.second));
} }
} else if (command == ":i") { } else if (command == ":i") {
runProgram(settings.nixBinDir + "/nix-env", Strings{"-i", drvPath}); runProgram(settings.nixBinDir + "/nix-env", Strings{"-i", drvPath});

View file

@ -69,12 +69,12 @@ struct CmdShowDerivation : InstallablesCommand
auto outputsObj(drvObj.object("outputs")); auto outputsObj(drvObj.object("outputs"));
for (auto & output : drv.outputs) { for (auto & output : drv.outputs) {
auto outputObj(outputsObj.object(output.first)); auto outputObj(outputsObj.object(output.first));
outputObj.attr("path", store->printStorePath(output.second.path(*store, drv.name)));
std::visit(overloaded { std::visit(overloaded {
[&](DerivationOutputInputAddressed doi) { [&](DerivationOutputInputAddressed doi) {
outputObj.attr("path", store->printStorePath(doi.path));
}, },
[&](DerivationOutputCAFixed dof) { [&](DerivationOutputCAFixed dof) {
outputObj.attr("path", store->printStorePath(dof.path(*store, drv.name, output.first)));
outputObj.attr("hashAlgo", dof.hash.printMethodAlgo()); outputObj.attr("hashAlgo", dof.hash.printMethodAlgo());
outputObj.attr("hash", dof.hash.hash.to_string(Base16, false)); outputObj.attr("hash", dof.hash.hash.to_string(Base16, false));
}, },

View file

@ -0,0 +1,19 @@
with import ./config.nix;
{ seed ? 0 }:
# A simple content-addressed derivation.
# The derivation can be arbitrarily modified by passing a different `seed`,
# but the output will always be the same
mkDerivation {
name = "simple-content-addressed";
buildCommand = ''
set -x
echo "Building a CA derivation"
echo "The seed is ${toString seed}"
mkdir -p $out
echo "Hello World" > $out/hello
'';
__contentAddressed = true;
outputHashMode = "recursive";
outputHashAlgo = "sha256";
}

View file

@ -0,0 +1,16 @@
#!/usr/bin/env bash
source common.sh
clearStore
clearCache
export REMOTE_STORE=file://$cacheDir
drv=$(nix-instantiate --experimental-features ca-derivations ./content-addressed.nix --arg seed 1)
nix --experimental-features 'nix-command ca-derivations' show-derivation --derivation "$drv" --arg seed 1
out1=$(nix-build --experimental-features ca-derivations ./content-addressed.nix --arg seed 1 --no-out-link)
out2=$(nix-build --experimental-features ca-derivations ./content-addressed.nix --arg seed 2 --no-out-link)
test $out1 == $out2

View file

@ -32,7 +32,8 @@ nix_tests = \
post-hook.sh \ post-hook.sh \
function-trace.sh \ function-trace.sh \
recursive.sh \ recursive.sh \
flakes.sh flakes.sh \
content-addressed.sh
# parallel.sh # parallel.sh
install-tests += $(foreach x, $(nix_tests), tests/$(x)) install-tests += $(foreach x, $(nix_tests), tests/$(x))

View file

@ -2,6 +2,21 @@ with import ./config.nix;
rec { rec {
# Want to ensure that "out" doesn't get a suffix on it's path.
nameCheck = mkDerivation {
name = "multiple-outputs-a";
outputs = [ "out" "dev" ];
builder = builtins.toFile "builder.sh"
''
mkdir $first $second
test -z $all
echo "first" > $first/file
echo "second" > $second/file
ln -s $first $second/link
'';
helloString = "Hello, world!";
};
a = mkDerivation { a = mkDerivation {
name = "multiple-outputs-a"; name = "multiple-outputs-a";
outputs = [ "first" "second" ]; outputs = [ "first" "second" ];

View file

@ -4,6 +4,12 @@ clearStore
rm -f $TEST_ROOT/result* rm -f $TEST_ROOT/result*
# Test whether the output names match our expectations
outPath=$(nix-instantiate multiple-outputs.nix --eval -A nameCheck.out.outPath)
[ "$(echo "$outPath" | sed -E 's_^".*/[^-/]*-([^/]*)"$_\1_')" = "multiple-outputs-a" ]
outPath=$(nix-instantiate multiple-outputs.nix --eval -A nameCheck.dev.outPath)
[ "$(echo "$outPath" | sed -E 's_^".*/[^-/]*-([^/]*)"$_\1_')" = "multiple-outputs-a-dev" ]
# Test whether read-only evaluation works when referring to the # Test whether read-only evaluation works when referring to the
# drvPath attribute. # drvPath attribute.
echo "evaluating c..." echo "evaluating c..."