lix-project/lix
19
42
Fork 18
Code Issues 295 Code Review (Gerrit) Projects 1 Releases Wiki Activity

Merge branch 'single-ca-drv-build' of https://github.com/obsidiansystems/nix

Browse source
This commit is contained in:
Eelco Dolstra 2020-09-16 12:14:01 +02:00
parent 733d2e9402 3a5cdd737c
commit 609a6d6d9f
30 changed files with 1100 additions and 515 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
perl/lib/Nix/Store.xs
View file

@ -303,11 +303,14 @@ SV * derivationFromPath(char * drvPath)
hash = newHV();
HV * outputs = newHV();
for (auto & i : drv.outputsAndPaths(*store()))
for (auto & i : drv.outputsAndOptPaths(*store())) {
hv_store(
outputs, i.first.c_str(), i.first.size(),
newSVpv(store()->printStorePath(i.second.second).c_str(), 0),
!i.second.second
? newSV(0) /* null value */
: newSVpv(store()->printStorePath(*i.second.second).c_str(), 0),
0);
}
hv_stores(hash, "outputs", newRV((SV *) outputs));
AV * inputDrvs = newAV();

14
src/libexpr/get-drvs.cc
View file

@ -38,8 +38,11 @@ DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPat
auto i = drv.outputs.find(outputName);
if (i == drv.outputs.end())
throw Error("derivation '%s' does not have output '%s'", store->printStorePath(drvPath), outputName);
auto & [outputName, output] = *i;
outPath = store->printStorePath(i->second.path(*store, drv.name));
auto optStorePath = output.path(*store, drv.name, outputName);
if (optStorePath)
outPath = store->printStorePath(*optStorePath);
}
@ -77,12 +80,15 @@ string DrvInfo::queryDrvPath() const
string DrvInfo::queryOutPath() const
{
if (outPath == "" && attrs) {
if (!outPath && attrs) {
Bindings::iterator i = attrs->find(state->sOutPath);
PathSet context;
outPath = i != attrs->end() ? state->coerceToPath(*i->pos, *i->value, context) : "";
if (i != attrs->end())
outPath = state->coerceToPath(*i->pos, *i->value, context);
}
return outPath;
if (!outPath)
throw UnimplementedError("CA derivations are not yet supported");
return *outPath;
}

2
src/libexpr/get-drvs.hh
View file

@ -20,7 +20,7 @@ private:
mutable string name;
mutable string system;
mutable string drvPath;
mutable string outPath;
mutable std::optional<string> outPath;
mutable string outputName;
Outputs outputs;

73
src/libexpr/primops.cc
View file

@ -44,16 +44,6 @@ void EvalState::realiseContext(const PathSet & context)
throw InvalidPathError(store->printStorePath(ctx));
if (!outputName.empty() && ctx.isDerivation()) {
drvs.push_back(StorePathWithOutputs{ctx, {outputName}});
/* Add the output of this derivation to the allowed
paths. */
if (allowedPaths) {
auto drv = store->derivationFromPath(ctx);
DerivationOutputs::iterator i = drv.outputs.find(outputName);
if (i == drv.outputs.end())
throw Error("derivation '%s' does not have an output named '%s'", ctxS, outputName);
allowedPaths->insert(store->printStorePath(i->second.path(*store, drv.name)));
}
}
}
@ -69,8 +59,50 @@ void EvalState::realiseContext(const PathSet & context)
store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize);
store->buildPaths(drvs);
/* Add the output of this derivations to the allowed
paths. */
if (allowedPaths) {
for (auto & [drvPath, outputs] : drvs) {
auto outputPaths = store->queryDerivationOutputMap(drvPath);
for (auto & outputName : outputs) {
if (outputPaths.count(outputName) == 0)
throw Error("derivation '%s' does not have an output named '%s'",
store->printStorePath(drvPath), outputName);
allowedPaths->insert(store->printStorePath(outputPaths.at(outputName)));
}
}
}
}
/* Add and attribute to the given attribute map from the output name to
the output path, or a placeholder.
Where possible the path is used, but for floating CA derivations we
may not know it. For sake of determinism we always assume we don't
and instead put in a place holder. In either case, however, the
string context will contain the drv path and output name, so
downstream derivations will have the proper dependency, and in
addition, before building, the placeholder will be rewritten to be
the actual path.
The 'drv' and 'drvPath' outputs must correspond. */
static void mkOutputString(EvalState & state, Value & v,
const StorePath & drvPath, const BasicDerivation & drv,
std::pair<string, DerivationOutput> o)
{
auto optOutputPath = o.second.path(*state.store, drv.name, o.first);
mkString(
*state.allocAttr(v, state.symbols.create(o.first)),
optOutputPath
? state.store->printStorePath(*optOutputPath)
/* Downstream we would substitute this for an actual path once
we build the floating CA derivation */
/* FIXME: we need to depend on the basic derivation, not
derivation */
: downstreamPlaceholder(*state.store, drvPath, o.first),
{"!" + o.first + "!" + state.store->printStorePath(drvPath)});
}
/* Load and evaluate an expression from path specified by the
argument. */
@ -114,9 +146,8 @@ static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vS
state.mkList(*outputsVal, drv.outputs.size());
unsigned int outputs_index = 0;
for (const auto & o : drv.outputsAndPaths(*state.store)) {
v2 = state.allocAttr(w, state.symbols.create(o.first));
mkString(*v2, state.store->printStorePath(o.second.second), {"!" + o.first + "!" + path});
for (const auto & o : drv.outputs) {
mkOutputString(state, w, storePath, drv, o);
outputsVal->listElems()[outputs_index] = state.allocValue();
mkString(*(outputsVal->listElems()[outputs_index++]), o.first);
}
@ -1080,16 +1111,18 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
/* Optimisation, but required in read-only mode! because in that
case we don't actually write store derivations, so we can't
read them later. */
drvHashes.insert_or_assign(drvPath,
hashDerivationModulo(*state.store, Derivation(drv), false));
read them later.
However, we don't bother doing this for floating CA derivations because
their "hash modulo" is indeterminate until built. */
if (drv.type() != DerivationType::CAFloating)
drvHashes.insert_or_assign(drvPath,
hashDerivationModulo(*state.store, Derivation(drv), false));
state.mkAttrs(v, 1 + drv.outputs.size());
mkString(*state.allocAttr(v, state.sDrvPath), drvPathS, {"=" + drvPathS});
for (auto & i : drv.outputsAndPaths(*state.store)) {
mkString(*state.allocAttr(v, state.symbols.create(i.first)),
state.store->printStorePath(i.second.second), {"!" + i.first + "!" + drvPathS});
}
for (auto & i : drv.outputs)
mkOutputString(state, v, drvPath, drv, i);
v.attrs->sort();
}

1017
src/libstore/build.cc
View file

File diff suppressed because it is too large Load diff

140
src/libstore/derivations.cc
View file

@ -7,7 +7,7 @@
namespace nix {
std::optional<StorePath> DerivationOutput::pathOpt(const Store & store, std::string_view drvName) const
std::optional<StorePath> DerivationOutput::path(const Store & store, std::string_view drvName, std::string_view outputName) const
{
return std::visit(overloaded {
[](DerivationOutputInputAddressed doi) -> std::optional<StorePath> {
@ -15,7 +15,7 @@ std::optional<StorePath> DerivationOutput::pathOpt(const Store & store, std::str
},
[&](DerivationOutputCAFixed dof) -> std::optional<StorePath> {
return {
store.makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName)
dof.path(store, drvName, outputName)
};
},
[](DerivationOutputCAFloating dof) -> std::optional<StorePath> {
@ -25,6 +25,13 @@ std::optional<StorePath> DerivationOutput::pathOpt(const Store & store, std::str
}
StorePath DerivationOutputCAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const {
return store.makeFixedOutputPath(
hash.method, hash.hash,
outputPathName(drvName, outputName));
}
bool derivationIsCA(DerivationType dt) {
switch (dt) {
case DerivationType::InputAddressed: return false;
@ -106,12 +113,15 @@ static string parseString(std::istream & str)
return res;
}
static void validatePath(std::string_view s) {
if (s.size() == 0 || s[0] != '/')
throw FormatError("bad path '%1%' in derivation", s);
}
static Path parsePath(std::istream & str)
{
string s = parseString(str);
if (s.size() == 0 || s[0] != '/')
throw FormatError("bad path '%1%' in derivation", s);
auto s = parseString(str);
validatePath(s);
return s;
}
@ -140,7 +150,7 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
static DerivationOutput parseDerivationOutput(const Store & store,
StorePath path, std::string_view hashAlgo, std::string_view hash)
std::string_view pathS, std::string_view hashAlgo, std::string_view hash)
{
if (hashAlgo != "") {
auto method = FileIngestionMethod::Flat;
@ -148,40 +158,45 @@ static DerivationOutput parseDerivationOutput(const Store & store,
method = FileIngestionMethod::Recursive;
hashAlgo = hashAlgo.substr(2);
}
const HashType hashType = parseHashType(hashAlgo);
return hash != ""
? DerivationOutput {
.output = DerivationOutputCAFixed {
.hash = FixedOutputHash {
.method = std::move(method),
.hash = Hash::parseNonSRIUnprefixed(hash, hashType),
},
}
}
: (settings.requireExperimentalFeature("ca-derivations"),
DerivationOutput {
.output = DerivationOutputCAFloating {
.method = std::move(method),
.hashType = std::move(hashType),
},
});
} else
const auto hashType = parseHashType(hashAlgo);
if (hash != "") {
validatePath(pathS);
return DerivationOutput {
.output = DerivationOutputCAFixed {
.hash = FixedOutputHash {
.method = std::move(method),
.hash = Hash::parseNonSRIUnprefixed(hash, hashType),
},
},
};
} else {
settings.requireExperimentalFeature("ca-derivations");
assert(pathS == "");
return DerivationOutput {
.output = DerivationOutputCAFloating {
.method = std::move(method),
.hashType = std::move(hashType),
},
};
}
} else {
validatePath(pathS);
return DerivationOutput {
.output = DerivationOutputInputAddressed {
.path = std::move(path),
.path = store.parseStorePath(pathS),
}
};
}
}
static DerivationOutput parseDerivationOutput(const Store & store, std::istringstream & str)
{
expect(str, ","); auto path = store.parseStorePath(parsePath(str));
expect(str, ","); const auto pathS = parseString(str);
expect(str, ","); const auto hashAlgo = parseString(str);
expect(str, ","); const auto hash = parseString(str);
expect(str, ")");
return parseDerivationOutput(store, std::move(path), hashAlgo, hash);
return parseDerivationOutput(store, pathS, hashAlgo, hash);
}
@ -294,17 +309,19 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
for (auto & i : outputs) {
if (first) first = false; else s += ',';
s += '('; printUnquotedString(s, i.first);
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path(store, name)));
std::visit(overloaded {
[&](DerivationOutputInputAddressed doi) {
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(doi.path));
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, "");
},
[&](DerivationOutputCAFixed dof) {
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first)));
s += ','; printUnquotedString(s, dof.hash.printMethodAlgo());
s += ','; printUnquotedString(s, dof.hash.hash.to_string(Base16, false));
},
[&](DerivationOutputCAFloating dof) {
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType));
s += ','; printUnquotedString(s, "");
},
@ -360,6 +377,16 @@ bool isDerivation(const string & fileName)
}
std::string outputPathName(std::string_view drvName, std::string_view outputName) {
std::string res { drvName };
if (outputName != "out") {
res += "-";
res += outputName;
}
return res;
}
DerivationType BasicDerivation::type() const
{
std::set<std::string_view> inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs;
@ -452,12 +479,12 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
throw Error("Regular input-addressed derivations are not yet allowed to depend on CA derivations");
case DerivationType::CAFixed: {
std::map<std::string, Hash> outputHashes;
for (const auto & i : drv.outputsAndPaths(store)) {
auto & dof = std::get<DerivationOutputCAFixed>(i.second.first.output);
for (const auto & i : drv.outputs) {
auto & dof = std::get<DerivationOutputCAFixed>(i.second.output);
auto hash = hashString(htSHA256, "fixed:out:"
+ dof.hash.printMethodAlgo() + ":"
+ dof.hash.hash.to_string(Base16, false) + ":"
+ store.printStorePath(i.second.second));
+ store.printStorePath(dof.path(store, drv.name, i.first)));
outputHashes.insert_or_assign(i.first, std::move(hash));
}
return outputHashes;
@ -508,21 +535,13 @@ bool wantOutput(const string & output, const std::set<string> & wanted)
}
StorePathSet BasicDerivation::outputPaths(const Store & store) const
{
StorePathSet paths;
for (auto & i : outputsAndPaths(store))
paths.insert(i.second.second);
return paths;
}
static DerivationOutput readDerivationOutput(Source & in, const Store & store)
{
auto path = store.parseStorePath(readString(in));
const auto pathS = readString(in);
const auto hashAlgo = readString(in);
const auto hash = readString(in);
return parseDerivationOutput(store, std::move(path), hashAlgo, hash);
return parseDerivationOutput(store, pathS, hashAlgo, hash);
}
StringSet BasicDerivation::outputNames() const
@ -533,23 +552,12 @@ StringSet BasicDerivation::outputNames() const
return names;
}
DerivationOutputsAndPaths BasicDerivation::outputsAndPaths(const Store & store) const {
DerivationOutputsAndPaths outsAndPaths;
for (auto output : outputs)
outsAndPaths.insert(std::make_pair(
output.first,
std::make_pair(output.second, output.second.path(store, name))
)
);
return outsAndPaths;
}
DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const {
DerivationOutputsAndOptPaths outsAndOptPaths;
for (auto output : outputs)
outsAndOptPaths.insert(std::make_pair(
output.first,
std::make_pair(output.second, output.second.pathOpt(store, output.first))
std::make_pair(output.second, output.second.path(store, name, output.first))
)
);
return outsAndOptPaths;
@ -594,22 +602,25 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv,
void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv)
{
out << drv.outputs.size();
for (auto & i : drv.outputsAndPaths(store)) {
out << i.first
<< store.printStorePath(i.second.second);
for (auto & i : drv.outputs) {
out << i.first;
std::visit(overloaded {
[&](DerivationOutputInputAddressed doi) {
out << "" << "";
out << store.printStorePath(doi.path)
<< ""
<< "";
},
[&](DerivationOutputCAFixed dof) {
out << dof.hash.printMethodAlgo()
out << store.printStorePath(dof.path(store, drv.name, i.first))
<< dof.hash.printMethodAlgo()
<< dof.hash.hash.to_string(Base16, false);
},
[&](DerivationOutputCAFloating dof) {
out << (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType))
out << ""
<< (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType))
<< "";
},
}, i.second.first.output);
}, i.second.output);
}
writeStorePaths(store, out, drv.inputSrcs);
out << drv.platform << drv.builder << drv.args;
@ -625,5 +636,12 @@ std::string hashPlaceholder(const std::string & outputName)
return "/" + hashString(htSHA256, "nix-output:" + outputName).to_string(Base32, false);
}
std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName)
{
auto drvNameWithExtension = drvPath.name();
auto drvName = drvNameWithExtension.substr(0, drvNameWithExtension.size() - 4);
auto clearText = "nix-upstream-output:" + std::string { drvPath.hashPart() } + ":" + outputPathName(drvName, outputName);
return "/" + hashString(htSHA256, clearText).to_string(Base32, false);
}
}

42
src/libstore/derivations.hh
View file

@ -27,6 +27,7 @@ struct DerivationOutputInputAddressed
struct DerivationOutputCAFixed
{
FixedOutputHash hash; /* hash used for expected hash computation */
StorePath path(const Store & store, std::string_view drvName, std::string_view outputName) const;
};
/* Floating-output derivations, whose output paths are content addressed, but
@ -49,14 +50,8 @@ struct DerivationOutput
std::optional<HashType> hashAlgoOpt(const Store & store) const;
/* Note, when you use this function you should make sure that you're passing
the right derivation name. When in doubt, you should use the safer
interface provided by BasicDerivation::outputsAndPaths */
std::optional<StorePath> pathOpt(const Store & store, std::string_view drvName) const;
/* DEPRECATED: Remove after CA drvs are fully implemented */
StorePath path(const Store & store, std::string_view drvName) const {
auto p = pathOpt(store, drvName);
if (!p) throw UnimplementedError("floating content-addressed derivations are not yet implemented");
return *p;
}
interface provided by BasicDerivation::outputsAndOptPaths */
std::optional<StorePath> path(const Store & store, std::string_view drvName, std::string_view outputName) const;
};
typedef std::map<string, DerivationOutput> DerivationOutputs;
@ -113,17 +108,12 @@ struct BasicDerivation
/* Return true iff this is a fixed-output derivation. */
DerivationType type() const;
/* Return the output paths of a derivation. */
StorePathSet outputPaths(const Store & store) const;
/* Return the output names of a derivation. */
StringSet outputNames() const;
/* Calculates the maps that contains all the DerivationOutputs, but
augmented with knowledge of the Store paths they would be written into.
The first one of these functions will be removed when the CA work is
completed */
DerivationOutputsAndPaths outputsAndPaths(const Store & store) const;
augmented with knowledge of the Store paths they would be written
into. */
DerivationOutputsAndOptPaths outputsAndOptPaths(const Store & store) const;
static std::string_view nameFromPath(const StorePath & storePath);
@ -155,6 +145,13 @@ Derivation parseDerivation(const Store & store, std::string && s, std::string_vi
// FIXME: remove
bool isDerivation(const string & fileName);
/* Calculate the name that will be used for the store path for this
output.
This is usually <drv-name>-<output-name>, but is just <drv-name> when
the output name is "out". */
std::string outputPathName(std::string_view drvName, std::string_view outputName);
// known CA drv's output hashes, current just for fixed-output derivations
// whose output hashes are always known since they are fixed up-front.
typedef std::map<std::string, Hash> CaOutputHashes;
@ -202,6 +199,21 @@ struct Sink;
Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name);
void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv);
/* This creates an opaque and almost certainly unique string
deterministically from the output name.
It is used as a placeholder to allow derivations to refer to their
own outputs without needing to use the hash of a derivation in
itself, making the hash near-impossible to calculate. */
std::string hashPlaceholder(const std::string & outputName);
/* This creates an opaque and almost certainly unique string
deterministically from a derivation path and output name.
It is used as a placeholder to allow derivations to refer to
content-addressed paths whose content --- and thus the path
themselves --- isn't yet known. This occurs when a derivation has a
dependency which is a CA derivation. */
std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName);
}

32
src/libstore/local-store.cc
View file

@ -578,13 +578,32 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
envHasRightPath(path, i.first);
},
[&](DerivationOutputCAFloating _) {
throw UnimplementedError("floating CA output derivations are not yet implemented");
/* Nothing to check */
},
}, i.second.output);
}
}
void LocalStore::linkDeriverToPath(const StorePath & deriver, const string & outputName, const StorePath & output)
{
auto state(_state.lock());
return linkDeriverToPath(*state, queryValidPathId(*state, deriver), outputName, output);
}
void LocalStore::linkDeriverToPath(State & state, uint64_t deriver, const string & outputName, const StorePath & output)
{
retrySQLite<void>([&]() {
state.stmtAddDerivationOutput.use()
(deriver)
(outputName)
(printStorePath(output))
.exec();
});
}
uint64_t LocalStore::addValidPath(State & state,
const ValidPathInfo & info, bool checkOutputs)
{
@ -618,12 +637,11 @@ uint64_t LocalStore::addValidPath(State & state,
registration above is undone. */
if (checkOutputs) checkDerivationOutputs(info.path, drv);
for (auto & i : drv.outputsAndPaths(*this)) {
state.stmtAddDerivationOutput.use()
(id)
(i.first)
(printStorePath(i.second.second))
.exec();
for (auto & i : drv.outputsAndOptPaths(*this)) {
/* Floating CA derivations have indeterminate output paths until
they are built, so don't register anything in that case */
if (i.second.second)
linkDeriverToPath(state, id, i.first, *i.second.second);
}
}

5
src/libstore/local-store.hh
View file

@ -279,6 +279,11 @@ private:
specified by the ‘secret-key-files’ option. */
void signPathInfo(ValidPathInfo & info);
/* Register the store path 'output' as the output named 'outputName' of
derivation 'deriver'. */
void linkDeriverToPath(const StorePath & deriver, const string & outputName, const StorePath & output);
void linkDeriverToPath(State & state, uint64_t deriver, const string & outputName, const StorePath & output);
Path getRealStoreDir() override { return realStoreDir; }
void createUser(const std::string & userName, uid_t userId) override;

23
src/libstore/misc.cc
View file

@ -203,17 +203,24 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
return;
}
PathSet invalid;
/* true for regular derivations, and CA derivations for which we
have a trust mapping for all wanted outputs. */
auto knownOutputPaths = true;
for (auto & [outputName, pathOpt] : queryPartialDerivationOutputMap(path.path)) {
if (!pathOpt) {
knownOutputPaths = false;
break;
}
if (wantOutput(outputName, path.outputs) && !isValidPath(*pathOpt))
invalid.insert(printStorePath(*pathOpt));
}
if (knownOutputPaths && invalid.empty()) return;
auto drv = make_ref<Derivation>(derivationFromPath(path.path));
ParsedDerivation parsedDrv(StorePath(path.path), *drv);
PathSet invalid;
for (auto & j : drv->outputsAndPaths(*this))
if (wantOutput(j.first, path.outputs)
&& !isValidPath(j.second.second))
invalid.insert(printStorePath(j.second.second));
if (invalid.empty()) return;
if (settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
if (knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size()));
for (auto & output : invalid)
pool.enqueue(std::bind(checkOutput, printStorePath(path.path), drv, output, drvState));

16
src/libstore/references.cc
View file

@ -79,9 +79,17 @@ void RefScanSink::operator () (const unsigned char * data, size_t len)
std::pair<PathSet, HashResult> scanForReferences(const string & path,
const PathSet & refs)
{
RefScanSink refsSink;
HashSink hashSink { htSHA256 };
TeeSink sink { refsSink, hashSink };
auto found = scanForReferences(hashSink, path, refs);
auto hash = hashSink.finish();
return std::pair<PathSet, HashResult>(found, hash);
}
PathSet scanForReferences(Sink & toTee,
const string & path, const PathSet & refs)
{
RefScanSink refsSink;
TeeSink sink { refsSink, toTee };
std::map<string, Path> backMap;
/* For efficiency (and a higher hit rate), just search for the
@ -111,9 +119,7 @@ std::pair<PathSet, HashResult> scanForReferences(const string & path,
found.insert(j->second);
}
auto hash = hashSink.finish();
return std::pair<PathSet, HashResult>(found, hash);
return found;
}

2
src/libstore/references.hh
View file

@ -7,6 +7,8 @@ namespace nix {
std::pair<PathSet, HashResult> scanForReferences(const Path & path, const PathSet & refs);
PathSet scanForReferences(Sink & toTee, const Path & path, const PathSet & refs);
struct RewritingSink : Sink
{
std::string from, to, prev;

19
src/libstore/store-api.cc
View file

@ -140,21 +140,28 @@ StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view p
*/
StorePath Store::makeStorePath(const string & type,
const Hash & hash, std::string_view name) const
StorePath Store::makeStorePath(std::string_view type,
std::string_view hash, std::string_view name) const
{
/* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */
string s = type + ":" + hash.to_string(Base16, true) + ":" + storeDir + ":" + std::string(name);
string s = std::string { type } + ":" + std::string { hash }
+ ":" + storeDir + ":" + std::string { name };
auto h = compressHash(hashString(htSHA256, s), 20);
return StorePath(h, name);
}
StorePath Store::makeOutputPath(const string & id,
StorePath Store::makeStorePath(std::string_view type,
const Hash & hash, std::string_view name) const
{
return makeStorePath("output:" + id, hash,
std::string(name) + (id == "out" ? "" : "-" + id));
return makeStorePath(type, hash.to_string(Base16, true), name);
}
StorePath Store::makeOutputPath(std::string_view id,
const Hash & hash, std::string_view name) const
{
return makeStorePath("output:" + std::string { id }, hash, outputPathName(name, id));
}

6
src/libstore/store-api.hh
View file

@ -247,10 +247,12 @@ public:
StorePathWithOutputs followLinksToStorePathWithOutputs(std::string_view path) const;
/* Constructs a unique store path name. */
StorePath makeStorePath(const string & type,
StorePath makeStorePath(std::string_view type,
std::string_view hash, std::string_view name) const;
StorePath makeStorePath(std::string_view type,
const Hash & hash, std::string_view name) const;
StorePath makeOutputPath(const string & id,
StorePath makeOutputPath(std::string_view id,
const Hash & hash, std::string_view name) const;
StorePath makeFixedOutputPath(FileIngestionMethod method,

6
src/libutil/serialise.hh
View file

@ -22,6 +22,12 @@ struct Sink
}
};
/* Just throws away data. */
struct NullSink : Sink
{
void operator () (const unsigned char * data, size_t len) override
{ }
};
/* A buffered abstract sink. Warning: a BufferedSink should not be
used from multiple threads concurrently. */

56
src/nix-build/nix-build.cc
View file

@ -487,50 +487,56 @@ static void _main(int argc, char * * argv)
std::vector<StorePathWithOutputs> pathsToBuild;
std::map<Path, Path> drvPrefixes;
std::map<Path, Path> resultSymlinks;
std::vector<Path> outPaths;
std::map<StorePath, std::pair<size_t, StringSet>> drvMap;
for (auto & drvInfo : drvs) {
auto drvPath = drvInfo.queryDrvPath();
auto outPath = drvInfo.queryOutPath();
auto drvPath = store->parseStorePath(drvInfo.queryDrvPath());
auto outputName = drvInfo.queryOutputName();
if (outputName == "")
throw Error("derivation '%s' lacks an 'outputName' attribute", drvPath);
throw Error("derivation '%s' lacks an 'outputName' attribute", store->printStorePath(drvPath));
pathsToBuild.push_back({store->parseStorePath(drvPath), {outputName}});
pathsToBuild.push_back({drvPath, {outputName}});
std::string drvPrefix;
auto i = drvPrefixes.find(drvPath);
if (i != drvPrefixes.end())
drvPrefix = i->second;
auto i = drvMap.find(drvPath);
if (i != drvMap.end())
i->second.second.insert(outputName);
else {
drvPrefix = outLink;
if (drvPrefixes.size())
drvPrefix += fmt("-%d", drvPrefixes.size() + 1);
drvPrefixes[drvPath] = drvPrefix;
drvMap[drvPath] = {drvMap.size(), {outputName}};
}
std::string symlink = drvPrefix;
if (outputName != "out") symlink += "-" + outputName;
resultSymlinks[symlink] = outPath;
outPaths.push_back(outPath);
}
buildPaths(pathsToBuild);
if (dryRun) return;
for (auto & symlink : resultSymlinks)
if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
store2->addPermRoot(store->parseStorePath(symlink.second), absPath(symlink.first));
std::vector<StorePath> outPaths;
for (auto & [drvPath, info] : drvMap) {
auto & [counter, wantedOutputs] = info;
std::string drvPrefix = outLink;
if (counter)
drvPrefix += fmt("-%d", counter + 1);
auto builtOutputs = store->queryDerivationOutputMap(drvPath);
for (auto & outputName : wantedOutputs) {
auto outputPath = builtOutputs.at(outputName);
if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) {
std::string symlink = drvPrefix;
if (outputName != "out") symlink += "-" + outputName;
store2->addPermRoot(outputPath, absPath(symlink));
}
outPaths.push_back(outputPath);
}
}
logger->stop();
for (auto & path : outPaths)
std::cout << path << '\n';
std::cout << store->printStorePath(path) << '\n';
}
}

25
src/nix-store/nix-store.cc
View file

@ -64,6 +64,7 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true)
if (path.path.isDerivation()) {
if (build) store->buildPaths({path});
auto outputPaths = store->queryDerivationOutputMap(path.path);
Derivation drv = store->derivationFromPath(path.path);
rootNr++;
@ -76,7 +77,8 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true)
if (i == drv.outputs.end())
throw Error("derivation '%s' does not have an output named '%s'",
store2->printStorePath(path.path), j);
auto outPath = store2->printStorePath(i->second.path(*store, drv.name));
auto outPath = outputPaths.at(i->first);
auto retPath = store->printStorePath(outPath);
if (store2) {
if (gcRoot == "")
printGCWarning();
@ -84,10 +86,10 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true)
Path rootName = gcRoot;
if (rootNr > 1) rootName += "-" + std::to_string(rootNr);
if (i->first != "out") rootName += "-" + i->first;
outPath = store2->addPermRoot(store->parseStorePath(outPath), rootName);
retPath = store2->addPermRoot(outPath, rootName);
}
}
outputs.insert(outPath);
outputs.insert(retPath);
}
return outputs;
}
@ -217,8 +219,13 @@ static StorePathSet maybeUseOutputs(const StorePath & storePath, bool useOutput,
if (useOutput && storePath.isDerivation()) {
auto drv = store->derivationFromPath(storePath);
StorePathSet outputs;
for (auto & i : drv.outputsAndPaths(*store))
outputs.insert(i.second.second);
if (forceRealise)
return store->queryDerivationOutputs(storePath);
for (auto & i : drv.outputsAndOptPaths(*store)) {
if (!i.second.second)
throw UsageError("Cannot use output path of floating content-addressed derivation until we know what it is (e.g. by building it)");
outputs.insert(*i.second.second);
}
return outputs;
}
else return {storePath};
@ -308,11 +315,9 @@ static void opQuery(Strings opFlags, Strings opArgs)
case qOutputs: {
for (auto & i : opArgs) {
auto i2 = store->followLinksToStorePath(i);
if (forceRealise) realisePath({i2});
Derivation drv = store->derivationFromPath(i2);
for (auto & j : drv.outputsAndPaths(*store))
cout << fmt("%1%\n", store->printStorePath(j.second.second));
auto outputs = maybeUseOutputs(store->followLinksToStorePath(i), true, forceRealise);
for (auto & outputPath : outputs)
cout << fmt("%1%\n", store->printStorePath(outputPath));
}
break;
}

3
src/nix/build.cc
View file

@ -74,7 +74,8 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile
store2->addPermRoot(bo.path, absPath(symlink));
},
[&](BuildableFromDrv bfd) {
for (auto & output : bfd.outputs) {
auto builtOutputs = store->queryDerivationOutputMap(bfd.drvPath);
for (auto & output : builtOutputs) {
std::string symlink = outLink;
if (i) symlink += fmt("-%d", i);
if (output.first != "out") symlink += fmt("-%s", output.first);

5
src/nix/command.cc
View file

@ -150,7 +150,10 @@ void MixProfile::updateProfile(const Buildables & buildables)
},
[&](BuildableFromDrv bfd) {
for (auto & output : bfd.outputs) {
result.push_back(output.second);
/* Output path should be known because we just tried to
build it. */
assert(!output.second);
result.push_back(*output.second);
}
},
}, buildable);

5
src/nix/develop.cc
View file

@ -145,7 +145,10 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
/* Build the derivation. */
store->buildPaths({{shellDrvPath}});
for (auto & outPath : drv.outputPaths(*store)) {
for (auto & [_0, outputAndOptPath] : drv.outputsAndOptPaths(*store)) {
auto & [_1, optPath] = outputAndOptPath;
assert(optPath);
auto & outPath = *optPath;
assert(store->isValidPath(outPath));
auto outPathS = store->toRealPath(outPath);
if (lstat(outPathS).st_size)

15
src/nix/installables.cc
View file

@ -302,10 +302,10 @@ struct InstallableStorePath : Installable
Buildables toBuildables() override
{
if (storePath.isDerivation()) {
std::map<std::string, StorePath> outputs;
std::map<std::string, std::optional<StorePath>> outputs;
auto drv = store->readDerivation(storePath);
for (auto & i : drv.outputsAndPaths(*store))
outputs.emplace(i.first, i.second.second);
for (auto & [name, output] : drv.outputsAndOptPaths(*store))
outputs.emplace(name, output.second);
return {
BuildableFromDrv {
.drvPath = storePath,
@ -331,7 +331,7 @@ Buildables InstallableValue::toBuildables()
{
Buildables res;
std::map<StorePath, OutputPathMap> drvsToOutputs;
std::map<StorePath, std::map<std::string, std::optional<StorePath>>> drvsToOutputs;
// Group by derivation, helps with .all in particular
for (auto & drv : toDerivations()) {
@ -674,8 +674,11 @@ StorePathSet toStorePaths(ref<Store> store,
outPaths.insert(bo.path);
},
[&](BuildableFromDrv bfd) {
for (auto & output : bfd.outputs)
outPaths.insert(output.second);
for (auto & output : bfd.outputs) {
if (!output.second)
throw Error("Cannot operate on output of unbuilt CA drv");
outPaths.insert(*output.second);
}
},
}, b);
} else {

4
src/nix/installables.hh
View file

@ -20,7 +20,7 @@ struct BuildableOpaque {
struct BuildableFromDrv {
StorePath drvPath;
std::map<std::string, StorePath> outputs;
std::map<std::string, std::optional<StorePath>> outputs;
};
typedef std::variant<
@ -82,7 +82,7 @@ struct InstallableValue : Installable
struct DerivationInfo
{
StorePath drvPath;
StorePath outPath;
std::optional<StorePath> outPath;
std::string outputName;
};

10
src/nix/profile.cc
View file

@ -180,7 +180,9 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
auto [attrPath, resolvedRef, drv] = installable2->toDerivation();
ProfileElement element;
element.storePaths = {drv.outPath}; // FIXME
if (!drv.outPath)
throw UnimplementedError("CA derivations are not yet supported by 'nix profile'");
element.storePaths = {*drv.outPath}; // FIXME
element.source = ProfileElementSource{
installable2->flakeRef,
resolvedRef,
@ -191,7 +193,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
manifest.elements.emplace_back(std::move(element));
} else
throw Error("'nix profile install' does not support argument '%s'", installable->what());
throw UnimplementedError("'nix profile install' does not support argument '%s'", installable->what());
}
store->buildPaths(pathsToBuild);
@ -349,7 +351,9 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
printInfo("upgrading '%s' from flake '%s' to '%s'",
element.source->attrPath, element.source->resolvedRef, resolvedRef);
element.storePaths = {drv.outPath}; // FIXME
if (!drv.outPath)
throw UnimplementedError("CA derivations are not yet supported by 'nix profile'");
element.storePaths = {*drv.outPath}; // FIXME
element.source = ProfileElementSource{
installable.flakeRef,
resolvedRef,

4
src/nix/repl.cc
View file

@ -496,8 +496,8 @@ bool NixRepl::processLine(string line)
if (runProgram(settings.nixBinDir + "/nix", Strings{"build", "--no-link", drvPathRaw}) == 0) {
auto drv = state->store->readDerivation(drvPath);
std::cout << std::endl << "this derivation produced the following outputs:" << std::endl;
for (auto & i : drv.outputsAndPaths(*state->store))
std::cout << fmt(" %s -> %s\n", i.first, state->store->printStorePath(i.second.second));
for (auto & i : drv.outputsAndOptPaths(*state->store))
std::cout << fmt(" %s -> %s\n", i.first, state->store->printStorePath(*i.second.second));
}
} else if (command == ":i") {
runProgram(settings.nixBinDir + "/nix-env", Strings{"-i", drvPathRaw});

11
src/nix/show-derivation.cc
View file

@ -67,21 +67,22 @@ struct CmdShowDerivation : InstallablesCommand
{
auto outputsObj(drvObj.object("outputs"));
for (auto & output : drv.outputsAndPaths(*store)) {
auto outputObj(outputsObj.object(output.first));
outputObj.attr("path", store->printStorePath(output.second.second));
for (auto & [_outputName, output] : drv.outputs) {
auto & outputName = _outputName; // work around clang bug
auto outputObj { outputsObj.object(outputName) };
std::visit(overloaded {
[&](DerivationOutputInputAddressed doi) {
outputObj.attr("path", store->printStorePath(doi.path));
},
[&](DerivationOutputCAFixed dof) {
outputObj.attr("path", store->printStorePath(dof.path(*store, drv.name, outputName)));
outputObj.attr("hashAlgo", dof.hash.printMethodAlgo());
outputObj.attr("hash", dof.hash.hash.to_string(Base16, false));
},
[&](DerivationOutputCAFloating dof) {
outputObj.attr("hashAlgo", makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType));
},
}, output.second.first.output);
}, output.output);
}
}

32
tests/content-addressed.nix Normal file
View file

@ -0,0 +1,32 @@
with import ./config.nix;
{ seed ? 0 }:
# A simple content-addressed derivation.
# The derivation can be arbitrarily modified by passing a different `seed`,
# but the output will always be the same
rec {
rootLegacy = mkDerivation {
name = "simple-input-addressed";
buildCommand = ''
set -x
echo "Building a legacy derivation"
mkdir -p $out
echo "Hello World" > $out/hello
'';
};
rootCA = mkDerivation {
name = "dependent";
outputs = [ "out" "dev" ];
buildCommand = ''
echo "building a CA derivation"
echo "The seed is ${toString seed}"
mkdir -p $out
echo ${rootLegacy}/hello > $out/dep
# test symlink at root
ln -s $out $dev
'';
__contentAddressed = true;
outputHashMode = "recursive";
outputHashAlgo = "sha256";
};
}

17
tests/content-addressed.sh Normal file
View file

@ -0,0 +1,17 @@
#!/usr/bin/env bash
source common.sh
clearStore
clearCache
export REMOTE_STORE=file://$cacheDir
drv=$(nix-instantiate --experimental-features ca-derivations ./content-addressed.nix -A rootCA --arg seed 1)
nix --experimental-features 'nix-command ca-derivations' show-derivation --derivation "$drv" --arg seed 1
commonArgs=("--experimental-features" "ca-derivations" "./content-addressed.nix" "-A" "rootCA" "--no-out-link")
out1=$(nix-build "${commonArgs[@]}" ./content-addressed.nix --arg seed 1)
out2=$(nix-build "${commonArgs[@]}" ./content-addressed.nix --arg seed 2)
test $out1 == $out2

3
tests/local.mk
View file

@ -32,7 +32,8 @@ nix_tests = \
post-hook.sh \
function-trace.sh \
recursive.sh \
flakes.sh
flakes.sh \
content-addressed.sh
# parallel.sh
# build-remote-content-addressed-fixed.sh \

15
tests/multiple-outputs.nix
View file

@ -2,6 +2,21 @@ with import ./config.nix;
rec {
# Want to ensure that "out" doesn't get a suffix on it's path.
nameCheck = mkDerivation {
name = "multiple-outputs-a";
outputs = [ "out" "dev" ];
builder = builtins.toFile "builder.sh"
''
mkdir $first $second
test -z $all
echo "first" > $first/file
echo "second" > $second/file
ln -s $first $second/link
'';
helloString = "Hello, world!";
};
a = mkDerivation {
name = "multiple-outputs-a";
outputs = [ "first" "second" ];

6
tests/multiple-outputs.sh
View file

@ -4,6 +4,12 @@ clearStore
rm -f $TEST_ROOT/result*
# Test whether the output names match our expectations
outPath=$(nix-instantiate multiple-outputs.nix --eval -A nameCheck.out.outPath)
[ "$(echo "$outPath" | sed -E 's_^".*/[^-/]*-([^/]*)"$_\1_')" = "multiple-outputs-a" ]
outPath=$(nix-instantiate multiple-outputs.nix --eval -A nameCheck.dev.outPath)
[ "$(echo "$outPath" | sed -E 's_^".*/[^-/]*-([^/]*)"$_\1_')" = "multiple-outputs-a-dev" ]
# Test whether read-only evaluation works when referring to the
# ‘drvPath’ attribute.
echo "evaluating c..."