kloenk/lix
0
0
Fork 0
forked from lix-project/lix
Code Pull requests Activity

Merge remote-tracking branch 'origin/master' into debug-exploratory-PR

Browse source
This commit is contained in:
Eelco Dolstra 2022-05-04 14:10:21 +02:00
parent c81ffa692e 470e27ce80
commit c98648bef0
58 changed files with 1041 additions and 330 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
doc/manual/src/release-notes/rl-next.md
View file

@ -14,3 +14,13 @@
* `nix build` has a new `--print-out-paths` flag to print the resulting output paths. * `nix build` has a new `--print-out-paths` flag to print the resulting output paths.
This matches the default behaviour of `nix-build`. This matches the default behaviour of `nix-build`.
* You can now specify which outputs of a derivation `nix` should
operate on using the syntax `installable^outputs`,
e.g. `nixpkgs#glibc^dev,static` or `nixpkgs#glibc^*`. By default,
`nix` will use the outputs specified by the derivation's
`meta.outputsToInstall` attribute if it exists, or all outputs
otherwise.
Selecting derivation outputs using the attribute selection syntax
(e.g. `nixpkgs#glibc.dev`) no longer works.

83
src/libcmd/installables.cc
View file

@ -440,9 +440,7 @@ DerivedPaths InstallableValue::toDerivedPaths()
// Group by derivation, helps with .all in particular // Group by derivation, helps with .all in particular
for (auto & drv : toDerivations()) { for (auto & drv : toDerivations()) {
auto outputName = drv.outputName; for (auto & outputName : drv.outputsToInstall)
if (outputName == "")
throw Error("derivation '%s' lacks an 'outputName' attribute", state->store->printStorePath(drv.drvPath));
drvsToOutputs[drv.drvPath].insert(outputName); drvsToOutputs[drv.drvPath].insert(outputName);
drvsToCopy.insert(drv.drvPath); drvsToCopy.insert(drv.drvPath);
} }
@ -466,9 +464,19 @@ struct InstallableAttrPath : InstallableValue
SourceExprCommand & cmd; SourceExprCommand & cmd;
RootValue v; RootValue v;
std::string attrPath; std::string attrPath;
OutputsSpec outputsSpec;
InstallableAttrPath(ref<EvalState> state, SourceExprCommand & cmd, Value * v, const std::string & attrPath) InstallableAttrPath(
: InstallableValue(state), cmd(cmd), v(allocRootValue(v)), attrPath(attrPath) ref<EvalState> state,
SourceExprCommand & cmd,
Value * v,
const std::string & attrPath,
OutputsSpec outputsSpec)
: InstallableValue(state)
, cmd(cmd)
, v(allocRootValue(v))
, attrPath(attrPath)
, outputsSpec(std::move(outputsSpec))
{ } { }
std::string what() const override { return attrPath; } std::string what() const override { return attrPath; }
@ -497,7 +505,19 @@ std::vector<InstallableValue::DerivationInfo> InstallableAttrPath::toDerivations
auto drvPath = drvInfo.queryDrvPath(); auto drvPath = drvInfo.queryDrvPath();
if (!drvPath) if (!drvPath)
throw Error("'%s' is not a derivation", what()); throw Error("'%s' is not a derivation", what());
res.push_back({ *drvPath, drvInfo.queryOutputName() });
std::set<std::string> outputsToInstall;
if (auto outputNames = std::get_if<OutputNames>(&outputsSpec))
outputsToInstall = *outputNames;
else
for (auto & output : drvInfo.queryOutputs(false, std::get_if<DefaultOutputs>(&outputsSpec)))
outputsToInstall.insert(output.first);
res.push_back(DerivationInfo {
.drvPath = *drvPath,
.outputsToInstall = std::move(outputsToInstall)
});
} }
return res; return res;
@ -574,6 +594,7 @@ InstallableFlake::InstallableFlake(
ref<EvalState> state, ref<EvalState> state,
FlakeRef && flakeRef, FlakeRef && flakeRef,
std::string_view fragment, std::string_view fragment,
OutputsSpec outputsSpec,
Strings attrPaths, Strings attrPaths,
Strings prefixes, Strings prefixes,
const flake::LockFlags & lockFlags) const flake::LockFlags & lockFlags)
@ -581,6 +602,7 @@ InstallableFlake::InstallableFlake(
flakeRef(flakeRef), flakeRef(flakeRef),
attrPaths(fragment == "" ? attrPaths : Strings{(std::string) fragment}), attrPaths(fragment == "" ? attrPaths : Strings{(std::string) fragment}),
prefixes(fragment == "" ? Strings{} : prefixes), prefixes(fragment == "" ? Strings{} : prefixes),
outputsSpec(std::move(outputsSpec)),
lockFlags(lockFlags) lockFlags(lockFlags)
{ {
if (cmd && cmd->getAutoArgs(*state)->size()) if (cmd && cmd->getAutoArgs(*state)->size())
@ -598,9 +620,29 @@ std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableF
auto drvPath = attr->forceDerivation(); auto drvPath = attr->forceDerivation();
std::set<std::string> outputsToInstall;
if (auto aMeta = attr->maybeGetAttr(state->sMeta))
if (auto aOutputsToInstall = aMeta->maybeGetAttr("outputsToInstall"))
for (auto & s : aOutputsToInstall->getListOfStrings())
outputsToInstall.insert(s);
if (outputsToInstall.empty() || std::get_if<AllOutputs>(&outputsSpec)) {
outputsToInstall.clear();
if (auto aOutputs = attr->maybeGetAttr(state->sOutputs))
for (auto & s : aOutputs->getListOfStrings())
outputsToInstall.insert(s);
}
if (outputsToInstall.empty())
outputsToInstall.insert("out");
if (auto outputNames = std::get_if<OutputNames>(&outputsSpec))
outputsToInstall = *outputNames;
auto drvInfo = DerivationInfo { auto drvInfo = DerivationInfo {
std::move(drvPath), .drvPath = std::move(drvPath),
attr->getAttr(state->sOutputName)->getString() .outputsToInstall = std::move(outputsToInstall),
}; };
return {attrPath, getLockedFlake()->flake.lockedRef, std::move(drvInfo)}; return {attrPath, getLockedFlake()->flake.lockedRef, std::move(drvInfo)};
@ -723,8 +765,14 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
state->eval(e, *vFile); state->eval(e, *vFile);
} }
for (auto & s : ss) for (auto & s : ss) {
result.push_back(std::make_shared<InstallableAttrPath>(state, *this, vFile, s == "." ? "" : s)); auto [prefix, outputsSpec] = parseOutputsSpec(s);
result.push_back(
std::make_shared<InstallableAttrPath>(
state, *this, vFile,
prefix == "." ? "" : prefix,
outputsSpec));
}
} else { } else {
@ -743,12 +791,13 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
} }
try { try {
auto [flakeRef, fragment] = parseFlakeRefWithFragment(s, absPath(".")); auto [flakeRef, fragment, outputsSpec] = parseFlakeRefWithFragmentAndOutputsSpec(s, absPath("."));
result.push_back(std::make_shared<InstallableFlake>( result.push_back(std::make_shared<InstallableFlake>(
this, this,
getEvalState(), getEvalState(),
std::move(flakeRef), std::move(flakeRef),
fragment, fragment,
outputsSpec,
getDefaultFlakeAttrPaths(), getDefaultFlakeAttrPaths(),
getDefaultFlakeAttrPathPrefixes(), getDefaultFlakeAttrPathPrefixes(),
lockFlags)); lockFlags));
@ -822,12 +871,13 @@ std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> Installable::bui
auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive
auto drvOutputs = drv.outputsAndOptPaths(*store); auto drvOutputs = drv.outputsAndOptPaths(*store);
for (auto & output : bfd.outputs) { for (auto & output : bfd.outputs) {
if (!outputHashes.count(output)) auto outputHash = get(outputHashes, output);
if (!outputHash)
throw Error( throw Error(
"the derivation '%s' doesn't have an output named '%s'", "the derivation '%s' doesn't have an output named '%s'",
store->printStorePath(bfd.drvPath), output); store->printStorePath(bfd.drvPath), output);
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
DrvOutput outputId { outputHashes.at(output), output }; DrvOutput outputId { *outputHash, output };
auto realisation = store->queryRealisation(outputId); auto realisation = store->queryRealisation(outputId);
if (!realisation) if (!realisation)
throw Error( throw Error(
@ -838,10 +888,11 @@ std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> Installable::bui
} else { } else {
// If ca-derivations isn't enabled, assume that // If ca-derivations isn't enabled, assume that
// the output path is statically known. // the output path is statically known.
assert(drvOutputs.count(output)); auto drvOutput = get(drvOutputs, output);
assert(drvOutputs.at(output).second); assert(drvOutput);
assert(drvOutput->second);
outputs.insert_or_assign( outputs.insert_or_assign(
output, *drvOutputs.at(output).second); output, *drvOutput->second);
} }
} }
res.push_back({installable, BuiltPath::Built { bfd.drvPath, outputs }}); res.push_back({installable, BuiltPath::Built { bfd.drvPath, outputs }});

4
src/libcmd/installables.hh
View file

@ -141,7 +141,7 @@ struct InstallableValue : Installable
struct DerivationInfo struct DerivationInfo
{ {
StorePath drvPath; StorePath drvPath;
std::string outputName; std::set<std::string> outputsToInstall;
}; };
virtual std::vector<DerivationInfo> toDerivations() = 0; virtual std::vector<DerivationInfo> toDerivations() = 0;
@ -156,6 +156,7 @@ struct InstallableFlake : InstallableValue
FlakeRef flakeRef; FlakeRef flakeRef;
Strings attrPaths; Strings attrPaths;
Strings prefixes; Strings prefixes;
OutputsSpec outputsSpec;
const flake::LockFlags & lockFlags; const flake::LockFlags & lockFlags;
mutable std::shared_ptr<flake::LockedFlake> _lockedFlake; mutable std::shared_ptr<flake::LockedFlake> _lockedFlake;
@ -164,6 +165,7 @@ struct InstallableFlake : InstallableValue
ref<EvalState> state, ref<EvalState> state,
FlakeRef && flakeRef, FlakeRef && flakeRef,
std::string_view fragment, std::string_view fragment,
OutputsSpec outputsSpec,
Strings attrPaths, Strings attrPaths,
Strings prefixes, Strings prefixes,
const flake::LockFlags & lockFlags); const flake::LockFlags & lockFlags);

56
src/libexpr/eval-cache.cc
View file

@ -47,7 +47,7 @@ struct AttrDb
{ {
auto state(_state->lock()); auto state(_state->lock());
Path cacheDir = getCacheDir() + "/nix/eval-cache-v2"; Path cacheDir = getCacheDir() + "/nix/eval-cache-v3";
createDirs(cacheDir); createDirs(cacheDir);
Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite"; Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite";
@ -175,6 +175,24 @@ struct AttrDb
}); });
} }
AttrId setListOfStrings(
AttrKey key,
const std::vector<std::string> & l)
{
return doSQLite([&]()
{
auto state(_state->lock());
state->insertAttribute.use()
(key.first)
(symbols[key.second])
(AttrType::ListOfStrings)
(concatStringsSep("\t", l)).exec();
return state->db.getLastInsertedRowId();
});
}
AttrId setPlaceholder(AttrKey key) AttrId setPlaceholder(AttrKey key)
{ {
return doSQLite([&]() return doSQLite([&]()
@ -269,6 +287,8 @@ struct AttrDb
} }
case AttrType::Bool: case AttrType::Bool:
return {{rowId, queryAttribute.getInt(2) != 0}}; return {{rowId, queryAttribute.getInt(2) != 0}};
case AttrType::ListOfStrings:
return {{rowId, tokenizeString<std::vector<std::string>>(queryAttribute.getStr(2), "\t")}};
case AttrType::Missing: case AttrType::Missing:
return {{rowId, missing_t()}}; return {{rowId, missing_t()}};
case AttrType::Misc: case AttrType::Misc:
@ -385,7 +405,7 @@ std::string AttrCursor::getAttrPathStr(Symbol name) const
Value & AttrCursor::forceValue() Value & AttrCursor::forceValue()
{ {
debug("evaluating uncached attribute %s", getAttrPathStr()); debug("evaluating uncached attribute '%s'", getAttrPathStr());
auto & v = getValue(); auto & v = getValue();
@ -626,6 +646,38 @@ bool AttrCursor::getBool()
return v.boolean; return v.boolean;
} }
std::vector<std::string> AttrCursor::getListOfStrings()
{
if (root->db) {
if (!cachedValue)
cachedValue = root->db->getAttr(getKey());
if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
if (auto l = std::get_if<std::vector<std::string>>(&cachedValue->second)) {
debug("using cached list of strings attribute '%s'", getAttrPathStr());
return *l;
} else
throw TypeError("'%s' is not a list of strings", getAttrPathStr());
}
}
debug("evaluating uncached attribute '%s'", getAttrPathStr());
auto & v = getValue();
root->state.forceValue(v, noPos);
if (v.type() != nList)
throw TypeError("'%s' is not a list", getAttrPathStr());
std::vector<std::string> res;
for (auto & elem : v.listItems())
res.push_back(std::string(root->state.forceStringNoCtx(*elem)));
cachedValue = {root->db->setListOfStrings(getKey(), res), res};
return res;
}
std::vector<Symbol> AttrCursor::getAttrs() std::vector<Symbol> AttrCursor::getAttrs()
{ {
if (root->db) { if (root->db) {

6
src/libexpr/eval-cache.hh
View file

@ -44,6 +44,7 @@ enum AttrType {
Misc = 4, Misc = 4,
Failed = 5, Failed = 5,
Bool = 6, Bool = 6,
ListOfStrings = 7,
}; };
struct placeholder_t {}; struct placeholder_t {};
@ -61,7 +62,8 @@ typedef std::variant<
missing_t, missing_t,
misc_t, misc_t,
failed_t, failed_t,
bool bool,
std::vector<std::string>
> AttrValue; > AttrValue;
class AttrCursor : public std::enable_shared_from_this<AttrCursor> class AttrCursor : public std::enable_shared_from_this<AttrCursor>
@ -114,6 +116,8 @@ public:
bool getBool(); bool getBool();
std::vector<std::string> getListOfStrings();
std::vector<Symbol> getAttrs(); std::vector<Symbol> getAttrs();
bool isDerivation(); bool isDerivation();

11
src/libexpr/flake/config.cc
View file

@ -50,13 +50,11 @@ void ConfigFile::apply()
else else
assert(false); assert(false);
if (!whitelist.count(baseName)) { if (!whitelist.count(baseName) && !nix::fetchSettings.acceptFlakeConfig) {
auto trustedList = readTrustedList();
bool trusted = false; bool trusted = false;
if (nix::fetchSettings.acceptFlakeConfig){ auto trustedList = readTrustedList();
trusted = true; auto tlname = get(trustedList, name);
} else if (auto saved = get(get(trustedList, name).value_or(std::map<std::string, bool>()), valueS)) { if (auto saved = tlname ? get(*tlname, valueS) : nullptr) {
trusted = *saved; trusted = *saved;
warn("Using saved setting for '%s = %s' from ~/.local/share/nix/trusted-settings.json.", name,valueS); warn("Using saved setting for '%s = %s' from ~/.local/share/nix/trusted-settings.json.", name,valueS);
} else { } else {
@ -69,7 +67,6 @@ void ConfigFile::apply()
writeTrustedList(trustedList); writeTrustedList(trustedList);
} }
} }
if (!trusted) { if (!trusted) {
warn("ignoring untrusted flake configuration setting '%s'", name); warn("ignoring untrusted flake configuration setting '%s'", name);
continue; continue;

17
src/libexpr/flake/flakeref.cc
View file

@ -176,7 +176,7 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
parsedURL.query.insert_or_assign("shallow", "1"); parsedURL.query.insert_or_assign("shallow", "1");
return std::make_pair( return std::make_pair(
FlakeRef(Input::fromURL(parsedURL), get(parsedURL.query, "dir").value_or("")), FlakeRef(Input::fromURL(parsedURL), getOr(parsedURL.query, "dir", "")),
fragment); fragment);
} }
@ -189,7 +189,7 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
if (!hasPrefix(path, "/")) if (!hasPrefix(path, "/"))
throw BadURL("flake reference '%s' is not an absolute path", url); throw BadURL("flake reference '%s' is not an absolute path", url);
auto query = decodeQuery(match[2]); auto query = decodeQuery(match[2]);
path = canonPath(path + "/" + get(query, "dir").value_or("")); path = canonPath(path + "/" + getOr(query, "dir", ""));
} }
fetchers::Attrs attrs; fetchers::Attrs attrs;
@ -208,7 +208,7 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
input.parent = baseDir; input.parent = baseDir;
return std::make_pair( return std::make_pair(
FlakeRef(std::move(input), get(parsedURL.query, "dir").value_or("")), FlakeRef(std::move(input), getOr(parsedURL.query, "dir", "")),
fragment); fragment);
} }
} }
@ -238,4 +238,15 @@ std::pair<fetchers::Tree, FlakeRef> FlakeRef::fetchTree(ref<Store> store) const
return {std::move(tree), FlakeRef(std::move(lockedInput), subdir)}; return {std::move(tree), FlakeRef(std::move(lockedInput), subdir)};
} }
std::tuple<FlakeRef, std::string, OutputsSpec> parseFlakeRefWithFragmentAndOutputsSpec(
const std::string & url,
const std::optional<Path> & baseDir,
bool allowMissing,
bool isFlake)
{
auto [prefix, outputsSpec] = parseOutputsSpec(url);
auto [flakeRef, fragment] = parseFlakeRefWithFragment(prefix, baseDir, allowMissing, isFlake);
return {std::move(flakeRef), fragment, outputsSpec};
}
} }

8
src/libexpr/flake/flakeref.hh
View file

@ -3,6 +3,7 @@
#include "types.hh" #include "types.hh"
#include "hash.hh" #include "hash.hh"
#include "fetchers.hh" #include "fetchers.hh"
#include "path-with-outputs.hh"
#include <variant> #include <variant>
@ -79,4 +80,11 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
std::optional<std::pair<FlakeRef, std::string>> maybeParseFlakeRefWithFragment( std::optional<std::pair<FlakeRef, std::string>> maybeParseFlakeRefWithFragment(
const std::string & url, const std::optional<Path> & baseDir = {}); const std::string & url, const std::optional<Path> & baseDir = {});
std::tuple<FlakeRef, std::string, OutputsSpec> parseFlakeRefWithFragmentAndOutputsSpec(
const std::string & url,
const std::optional<Path> & baseDir = {},
bool allowMissing = false,
bool isFlake = true);
} }

2
src/libexpr/get-drvs.cc
View file

@ -34,7 +34,7 @@ DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPat
outputName = outputName =
selectedOutputs.empty() selectedOutputs.empty()
? get(drv.env, "outputName").value_or("out") ? getOr(drv.env, "outputName", "out")
: *selectedOutputs.begin(); : *selectedOutputs.begin();
auto i = drv.outputs.find(outputName); auto i = drv.outputs.find(outputName);

17
src/libexpr/primops.cc
View file

@ -76,10 +76,10 @@ StringMap EvalState::realiseContext(const PathSet & context)
/* Get all the output paths corresponding to the placeholders we had */ /* Get all the output paths corresponding to the placeholders we had */
for (auto & [drvPath, outputs] : drvs) { for (auto & [drvPath, outputs] : drvs) {
auto outputPaths = store->queryDerivationOutputMap(drvPath); const auto outputPaths = store->queryDerivationOutputMap(drvPath);
for (auto & outputName : outputs) { for (auto & outputName : outputs) {
if (outputPaths.count(outputName) == 0) auto outputPath = get(outputPaths, outputName);
{ if (!outputPath) {
auto e = Error("derivation '%s' does not have an output named '%s'", auto e = Error("derivation '%s' does not have an output named '%s'",
store->printStorePath(drvPath), outputName); store->printStorePath(drvPath), outputName);
debugLastTrace(e); debugLastTrace(e);
@ -87,7 +87,7 @@ StringMap EvalState::realiseContext(const PathSet & context)
} }
res.insert_or_assign( res.insert_or_assign(
downstreamPlaceholder(*store, drvPath, outputName), downstreamPlaceholder(*store, drvPath, outputName),
store->printStorePath(outputPaths.at(outputName)) store->printStorePath(*outputPath)
); );
} }
} }
@ -1369,8 +1369,13 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * *
switch (hashModulo.kind) { switch (hashModulo.kind) {
case DrvHash::Kind::Regular: case DrvHash::Kind::Regular:
for (auto & i : outputs) { for (auto & i : outputs) {
auto h = hashModulo.hashes.at(i); auto h = get(hashModulo.hashes, i);
auto outPath = state.store->makeOutputPath(i, h, drvName); if (!h)
throw AssertionError({
.msg = hintfmt("derivation produced no hash for output '%s'", i),
.errPos = state.positions[posDrvName],
});
auto outPath = state.store->makeOutputPath(i, *h, drvName);
drv.env[i] = state.store->printStorePath(outPath); drv.env[i] = state.store->printStorePath(outPath);
drv.outputs.insert_or_assign( drv.outputs.insert_or_assign(
i, i,

27
src/libfetchers/git-utils.cc Normal file
View file

@ -0,0 +1,27 @@
#include "git-utils.hh"
#include <regex>
std::optional<std::string> parseListReferenceHeadRef(std::string_view line)
{
const static std::regex head_ref_regex("^ref: ([^\\s]+)\\t+HEAD$");
std::match_results<std::string_view::const_iterator> match;
if (std::regex_match(line.cbegin(), line.cend(), match, head_ref_regex)) {
return match[1];
} else {
return std::nullopt;
}
}
std::optional<std::string> parseListReferenceForRev(std::string_view rev, std::string_view line)
{
const static std::regex rev_regex("^([^\\t]+)\\t+(.*)$");
std::match_results<std::string_view::const_iterator> match;
if (!std::regex_match(line.cbegin(), line.cend(), match, rev_regex)) {
return std::nullopt;
}
if (rev != match[2].str()) {
return std::nullopt;
}
return match[1];
}

23
src/libfetchers/git-utils.hh Normal file
View file

@ -0,0 +1,23 @@
#pragma once
#include <string>
#include <string_view>
#include <optional>
// Parses the HEAD ref as reported by `git ls-remote --symref`
//
// Returns the head branch name as reported by `git ls-remote --symref`, e.g., if
// ls-remote returns the output below, "main" is returned based on the ref line.
//
// ref: refs/heads/main HEAD
//
// If the repository is in 'detached head' state (HEAD is pointing to a rev
// instead of a branch), parseListReferenceForRev("HEAD") may be used instead.
std::optional<std::string> parseListReferenceHeadRef(std::string_view line);
// Parses a reference line from `git ls-remote --symref`, e.g.,
// parseListReferenceForRev("refs/heads/master", line) will return 6926...
// given the line below.
//
// 6926beab444c33fb57b21819b6642d032016bb1e refs/heads/master
std::optional<std::string> parseListReferenceForRev(std::string_view rev, std::string_view line);

373
src/libfetchers/git.cc
View file

@ -5,15 +5,20 @@
#include "store-api.hh" #include "store-api.hh"
#include "url-parts.hh" #include "url-parts.hh"
#include "pathlocks.hh" #include "pathlocks.hh"
#include "util.hh"
#include "git-utils.hh"
#include "fetch-settings.hh" #include "fetch-settings.hh"
#include <regex>
#include <string.h>
#include <sys/time.h> #include <sys/time.h>
#include <sys/wait.h> #include <sys/wait.h>
using namespace std::string_literals; using namespace std::string_literals;
namespace nix::fetchers { namespace nix::fetchers {
namespace {
// Explicit initial branch of our bare repo to suppress warnings from new version of git. // Explicit initial branch of our bare repo to suppress warnings from new version of git.
// The value itself does not matter, since we always fetch a specific revision or branch. // The value itself does not matter, since we always fetch a specific revision or branch.
@ -21,25 +26,228 @@ namespace nix::fetchers {
// old version of git, which will ignore unrecognized `-c` options. // old version of git, which will ignore unrecognized `-c` options.
const std::string gitInitialBranch = "__nix_dummy_branch"; const std::string gitInitialBranch = "__nix_dummy_branch";
static std::string getGitDir() std::string getGitDir()
{ {
auto gitDir = getEnv("GIT_DIR"); return getEnv("GIT_DIR").value_or(".git");
if (!gitDir) {
return ".git";
}
return *gitDir;
} }
static std::string readHead(const Path & path) bool isCacheFileWithinTtl(const time_t now, const struct stat & st)
{ {
return chomp(runProgram("git", true, { "-C", path, "--git-dir", ".git", "rev-parse", "--abbrev-ref", "HEAD" })); return st.st_mtime + settings.tarballTtl > now;
} }
static bool isNotDotGitDirectory(const Path & path) bool touchCacheFile(const Path& path, const time_t& touch_time)
{
struct timeval times[2];
times[0].tv_sec = touch_time;
times[0].tv_usec = 0;
times[1].tv_sec = touch_time;
times[1].tv_usec = 0;
return lutimes(path.c_str(), times) == 0;
}
Path getCachePath(std::string key)
{
return getCacheDir() + "/nix/gitv3/" +
hashString(htSHA256, key).to_string(Base32, false);
}
// Returns the name of the HEAD branch.
//
// Returns the head branch name as reported by git ls-remote --symref, e.g., if
// ls-remote returns the output below, "main" is returned based on the ref line.
//
// ref: refs/heads/main HEAD
// ...
std::optional<std::string> readHead(const Path & path)
{
auto [exit_code, output] = runProgram(RunOptions {
.program = "git",
.args = {"ls-remote", "--symref", path},
});
if (exit_code != 0) {
return std::nullopt;
}
std::string_view line = output;
line = line.substr(0, line.find("\n"));
if (const auto ref = parseListReferenceHeadRef(line); ref) {
debug("resolved HEAD ref '%s' for repo '%s'", *ref, path);
return *ref;
}
if (const auto rev = parseListReferenceForRev("HEAD", line); rev) {
debug("resolved HEAD rev '%s' for repo '%s'", *rev, path);
return *rev;
}
return std::nullopt;
}
// Persist the HEAD ref from the remote repo in the local cached repo.
bool storeCachedHead(const std::string& actualUrl, const std::string& headRef)
{
Path cacheDir = getCachePath(actualUrl);
try {
runProgram("git", true, { "-C", cacheDir, "symbolic-ref", "--", "HEAD", headRef });
} catch (ExecError &e) {
if (!WIFEXITED(e.status)) throw;
return false;
}
/* No need to touch refs/HEAD, because `git symbolic-ref` updates the mtime. */
return true;
}
std::optional<std::string> readHeadCached(const std::string& actualUrl)
{
// Create a cache path to store the branch of the HEAD ref. Append something
// in front of the URL to prevent collision with the repository itself.
Path cacheDir = getCachePath(actualUrl);
Path headRefFile = cacheDir + "/HEAD";
time_t now = time(0);
struct stat st;
std::optional<std::string> cachedRef;
if (stat(headRefFile.c_str(), &st) == 0) {
cachedRef = readHead(cacheDir);
if (cachedRef != std::nullopt &&
*cachedRef != gitInitialBranch &&
isCacheFileWithinTtl(now, st)) {
debug("using cached HEAD ref '%s' for repo '%s'", *cachedRef, actualUrl);
return cachedRef;
}
}
auto ref = readHead(actualUrl);
if (ref) {
return ref;
}
if (cachedRef) {
// If the cached git ref is expired in fetch() below, and the 'git fetch'
// fails, it falls back to continuing with the most recent version.
// This function must behave the same way, so we return the expired
// cached ref here.
warn("could not get HEAD ref for repository '%s'; using expired cached ref '%s'", actualUrl, *cachedRef);
return *cachedRef;
}
return std::nullopt;
}
bool isNotDotGitDirectory(const Path & path)
{ {
return baseNameOf(path) != ".git"; return baseNameOf(path) != ".git";
} }
struct WorkdirInfo
{
bool clean = false;
bool hasHead = false;
};
// Returns whether a git workdir is clean and has commits.
WorkdirInfo getWorkdirInfo(const Input & input, const Path & workdir)
{
const bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
auto gitDir = getGitDir();
auto env = getEnv();
// Set LC_ALL to C: because we rely on the error messages from git rev-parse to determine what went wrong
// that way unknown errors can lead to a failure instead of continuing through the wrong code path
env["LC_ALL"] = "C";
/* Check whether HEAD points to something that looks like a commit,
since that is the refrence we want to use later on. */
auto result = runProgram(RunOptions {
.program = "git",
.args = { "-C", workdir, "--git-dir", gitDir, "rev-parse", "--verify", "--no-revs", "HEAD^{commit}" },
.environment = env,
.mergeStderrToStdout = true
});
auto exitCode = WEXITSTATUS(result.first);
auto errorMessage = result.second;
if (errorMessage.find("fatal: not a git repository") != std::string::npos) {
throw Error("'%s' is not a Git repository", workdir);
} else if (errorMessage.find("fatal: Needed a single revision") != std::string::npos) {
// indicates that the repo does not have any commits
// we want to proceed and will consider it dirty later
} else if (exitCode != 0) {
// any other errors should lead to a failure
throw Error("getting the HEAD of the Git tree '%s' failed with exit code %d:\n%s", workdir, exitCode, errorMessage);
}
bool clean = false;
bool hasHead = exitCode == 0;
try {
if (hasHead) {
// Using git diff is preferrable over lower-level operations here,
// because its conceptually simpler and we only need the exit code anyways.
auto gitDiffOpts = Strings({ "-C", workdir, "diff", "HEAD", "--quiet"});
if (!submodules) {
// Changes in submodules should only make the tree dirty
// when those submodules will be copied as well.
gitDiffOpts.emplace_back("--ignore-submodules");
}
gitDiffOpts.emplace_back("--");
runProgram("git", true, gitDiffOpts);
clean = true;
}
} catch (ExecError & e) {
if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw;
}
return WorkdirInfo { .clean = clean, .hasHead = hasHead };
}
std::pair<StorePath, Input> fetchFromWorkdir(ref<Store> store, Input & input, const Path & workdir, const WorkdirInfo & workdirInfo)
{
const bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
if (!fetchSettings.allowDirty)
throw Error("Git tree '%s' is dirty", workdir);
if (fetchSettings.warnDirty)
warn("Git tree '%s' is dirty", workdir);
auto gitOpts = Strings({ "-C", workdir, "ls-files", "-z" });
if (submodules)
gitOpts.emplace_back("--recurse-submodules");
auto files = tokenizeString<std::set<std::string>>(
runProgram("git", true, gitOpts), "\0"s);
Path actualPath(absPath(workdir));
PathFilter filter = [&](const Path & p) -> bool {
assert(hasPrefix(p, actualPath));
std::string file(p, actualPath.size() + 1);
auto st = lstat(p);
if (S_ISDIR(st.st_mode)) {
auto prefix = file + "/";
auto i = files.lower_bound(prefix);
return i != files.end() && hasPrefix(*i, prefix);
}
return files.count(file);
};
auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, htSHA256, filter);
// FIXME: maybe we should use the timestamp of the last
// modified dirty file?
input.attrs.insert_or_assign(
"lastModified",
workdirInfo.hasHead ? std::stoull(runProgram("git", true, { "-C", actualPath, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0);
return {std::move(storePath), input};
}
} // end namespace
struct GitInputScheme : InputScheme struct GitInputScheme : InputScheme
{ {
std::optional<Input> inputFromURL(const ParsedURL & url) override std::optional<Input> inputFromURL(const ParsedURL & url) override
@ -234,106 +442,16 @@ struct GitInputScheme : InputScheme
auto [isLocal, actualUrl_] = getActualUrl(input); auto [isLocal, actualUrl_] = getActualUrl(input);
auto actualUrl = actualUrl_; // work around clang bug auto actualUrl = actualUrl_; // work around clang bug
// If this is a local directory and no ref or revision is /* If this is a local directory and no ref or revision is given,
// given, then allow the use of an unclean working tree. allow fetching directly from a dirty workdir. */
if (!input.getRef() && !input.getRev() && isLocal) { if (!input.getRef() && !input.getRev() && isLocal) {
bool clean = false; auto workdirInfo = getWorkdirInfo(input, actualUrl);
if (!workdirInfo.clean) {
auto env = getEnv(); return fetchFromWorkdir(store, input, actualUrl, workdirInfo);
// Set LC_ALL to C: because we rely on the error messages from git rev-parse to determine what went wrong
// that way unknown errors can lead to a failure instead of continuing through the wrong code path
env["LC_ALL"] = "C";
/* Check whether HEAD points to something that looks like a commit,
since that is the refrence we want to use later on. */
auto result = runProgram(RunOptions {
.program = "git",
.args = { "-C", actualUrl, "--git-dir", gitDir, "rev-parse", "--verify", "--no-revs", "HEAD^{commit}" },
.environment = env,
.mergeStderrToStdout = true
});
auto exitCode = WEXITSTATUS(result.first);
auto errorMessage = result.second;
if (errorMessage.find("fatal: not a git repository") != std::string::npos) {
throw Error("'%s' is not a Git repository", actualUrl);
} else if (errorMessage.find("fatal: Needed a single revision") != std::string::npos) {
// indicates that the repo does not have any commits
// we want to proceed and will consider it dirty later
} else if (exitCode != 0) {
// any other errors should lead to a failure
throw Error("getting the HEAD of the Git tree '%s' failed with exit code %d:\n%s", actualUrl, exitCode, errorMessage);
}
bool hasHead = exitCode == 0;
try {
if (hasHead) {
// Using git diff is preferrable over lower-level operations here,
// because its conceptually simpler and we only need the exit code anyways.
auto gitDiffOpts = Strings({ "-C", actualUrl, "--git-dir", gitDir, "diff", "HEAD", "--quiet"});
if (!submodules) {
// Changes in submodules should only make the tree dirty
// when those submodules will be copied as well.
gitDiffOpts.emplace_back("--ignore-submodules");
}
gitDiffOpts.emplace_back("--");
runProgram("git", true, gitDiffOpts);
clean = true;
}
} catch (ExecError & e) {
if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw;
}
if (!clean) {
/* This is an unclean working tree. So copy all tracked files. */
if (!fetchSettings.allowDirty)
throw Error("Git tree '%s' is dirty", actualUrl);
if (fetchSettings.warnDirty)
warn("Git tree '%s' is dirty", actualUrl);
auto gitOpts = Strings({ "-C", actualUrl, "--git-dir", gitDir, "ls-files", "-z" });
if (submodules)
gitOpts.emplace_back("--recurse-submodules");
auto files = tokenizeString<std::set<std::string>>(
runProgram("git", true, gitOpts), "\0"s);
Path actualPath(absPath(actualUrl));
PathFilter filter = [&](const Path & p) -> bool {
assert(hasPrefix(p, actualPath));
std::string file(p, actualPath.size() + 1);
auto st = lstat(p);
if (S_ISDIR(st.st_mode)) {
auto prefix = file + "/";
auto i = files.lower_bound(prefix);
return i != files.end() && hasPrefix(*i, prefix);
}
return files.count(file);
};
auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, htSHA256, filter);
// FIXME: maybe we should use the timestamp of the last
// modified dirty file?
input.attrs.insert_or_assign(
"lastModified",
hasHead ? std::stoull(runProgram("git", true, { "-C", actualPath, "--git-dir", gitDir, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0);
return {std::move(storePath), input};
} }
} }
if (!input.getRef()) input.attrs.insert_or_assign("ref", isLocal ? readHead(actualUrl) : "master"); const Attrs unlockedAttrs({
Attrs unlockedAttrs({
{"type", cacheType}, {"type", cacheType},
{"name", name}, {"name", name},
{"url", actualUrl}, {"url", actualUrl},
@ -343,14 +461,30 @@ struct GitInputScheme : InputScheme
Path repoDir; Path repoDir;
if (isLocal) { if (isLocal) {
if (!input.getRef()) {
auto head = readHead(actualUrl);
if (!head) {
warn("could not read HEAD ref from repo at '%s', using 'master'", actualUrl);
head = "master";
}
input.attrs.insert_or_assign("ref", *head);
}
if (!input.getRev()) if (!input.getRev())
input.attrs.insert_or_assign("rev", input.attrs.insert_or_assign("rev",
Hash::parseAny(chomp(runProgram("git", true, { "-C", actualUrl, "--git-dir", gitDir, "rev-parse", *input.getRef() })), htSHA1).gitRev()); Hash::parseAny(chomp(runProgram("git", true, { "-C", actualUrl, "--git-dir", gitDir, "rev-parse", *input.getRef() })), htSHA1).gitRev());
repoDir = actualUrl; repoDir = actualUrl;
} else { } else {
const bool useHeadRef = !input.getRef();
if (useHeadRef) {
auto head = readHeadCached(actualUrl);
if (!head) {
warn("could not read HEAD ref from repo at '%s', using 'master'", actualUrl);
head = "master";
}
input.attrs.insert_or_assign("ref", *head);
}
if (auto res = getCache()->lookup(store, unlockedAttrs)) { if (auto res = getCache()->lookup(store, unlockedAttrs)) {
auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), htSHA1); auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), htSHA1);
@ -360,7 +494,7 @@ struct GitInputScheme : InputScheme
} }
} }
Path cacheDir = getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, actualUrl).to_string(Base32, false); Path cacheDir = getCachePath(actualUrl);
repoDir = cacheDir; repoDir = cacheDir;
gitDir = "."; gitDir = ".";
@ -383,7 +517,7 @@ struct GitInputScheme : InputScheme
repo. */ repo. */
if (input.getRev()) { if (input.getRev()) {
try { try {
runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input.getRev()->gitRev() }); runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "cat-file", "-e", input.getRev()->gitRev() });
doFetch = false; doFetch = false;
} catch (ExecError & e) { } catch (ExecError & e) {
if (WIFEXITED(e.status)) { if (WIFEXITED(e.status)) {
@ -400,7 +534,7 @@ struct GitInputScheme : InputScheme
git fetch to update the local ref to the remote ref. */ git fetch to update the local ref to the remote ref. */
struct stat st; struct stat st;
doFetch = stat(localRefFile.c_str(), &st) != 0 || doFetch = stat(localRefFile.c_str(), &st) != 0 ||
(uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now; !isCacheFileWithinTtl(now, st);
} }
} }
@ -418,19 +552,16 @@ struct GitInputScheme : InputScheme
: ref == "HEAD" : ref == "HEAD"
? *ref ? *ref
: "refs/heads/" + *ref; : "refs/heads/" + *ref;
runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) }); runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) });
} catch (Error & e) { } catch (Error & e) {
if (!pathExists(localRefFile)) throw; if (!pathExists(localRefFile)) throw;
warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl); warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl);
} }
struct timeval times[2]; if (!touchCacheFile(localRefFile, now))
times[0].tv_sec = now; warn("could not update mtime for file '%s': %s", localRefFile, strerror(errno));
times[0].tv_usec = 0; if (useHeadRef && !storeCachedHead(actualUrl, *input.getRef()))
times[1].tv_sec = now; warn("could not update cached head '%s' for '%s'", *input.getRef(), actualUrl);
times[1].tv_usec = 0;
utimes(localRefFile.c_str(), times);
} }
if (!input.getRev()) if (!input.getRev())
@ -459,7 +590,7 @@ struct GitInputScheme : InputScheme
auto result = runProgram(RunOptions { auto result = runProgram(RunOptions {
.program = "git", .program = "git",
.args = { "-C", repoDir, "cat-file", "commit", input.getRev()->gitRev() }, .args = { "-C", repoDir, "--git-dir", gitDir, "cat-file", "commit", input.getRev()->gitRev() },
.mergeStderrToStdout = true .mergeStderrToStdout = true
}); });
if (WEXITSTATUS(result.first) == 128 if (WEXITSTATUS(result.first) == 128
@ -498,7 +629,7 @@ struct GitInputScheme : InputScheme
auto source = sinkToSource([&](Sink & sink) { auto source = sinkToSource([&](Sink & sink) {
runProgram2({ runProgram2({
.program = "git", .program = "git",
.args = { "-C", repoDir, "archive", input.getRev()->gitRev() }, .args = { "-C", repoDir, "--git-dir", gitDir, "archive", input.getRev()->gitRev() },
.standardOut = &sink .standardOut = &sink
}); });
}); });
@ -508,7 +639,7 @@ struct GitInputScheme : InputScheme
auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter); auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter);
auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", "--no-show-signature", input.getRev()->gitRev() })); auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "log", "-1", "--format=%ct", "--no-show-signature", input.getRev()->gitRev() }));
Attrs infoAttrs({ Attrs infoAttrs({
{"rev", input.getRev()->gitRev()}, {"rev", input.getRev()->gitRev()},
@ -517,7 +648,7 @@ struct GitInputScheme : InputScheme
if (!shallow) if (!shallow)
infoAttrs.insert_or_assign("revCount", infoAttrs.insert_or_assign("revCount",
std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input.getRev()->gitRev() }))); std::stoull(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "rev-list", "--count", input.getRev()->gitRev() })));
if (!_input.getRev()) if (!_input.getRev())
getCache()->add( getCache()->add(

28
src/libfetchers/github.cc
View file

@ -4,7 +4,7 @@
#include "store-api.hh" #include "store-api.hh"
#include "types.hh" #include "types.hh"
#include "url-parts.hh" #include "url-parts.hh"
#include "git-utils.hh"
#include "fetchers.hh" #include "fetchers.hh"
#include "fetch-settings.hh" #include "fetch-settings.hh"
@ -383,35 +383,29 @@ struct SourceHutInputScheme : GitArchiveInputScheme
std::string line; std::string line;
getline(is, line); getline(is, line);
auto ref_index = line.find("ref: "); auto r = parseListReferenceHeadRef(line);
if (ref_index == std::string::npos) { if (!r) {
throw BadURL("in '%d', couldn't resolve HEAD ref '%d'", input.to_string(), ref); throw BadURL("in '%d', couldn't resolve HEAD ref '%d'", input.to_string(), ref);
} }
ref_uri = *r;
ref_uri = line.substr(ref_index+5, line.length()-1); } else {
} else
ref_uri = fmt("refs/(heads|tags)/%s", ref); ref_uri = fmt("refs/(heads|tags)/%s", ref);
}
auto file = store->toRealPath( auto file = store->toRealPath(
downloadFile(store, fmt("%s/info/refs", base_url), "source", false, headers).storePath); downloadFile(store, fmt("%s/info/refs", base_url), "source", false, headers).storePath);
std::ifstream is(file); std::ifstream is(file);
std::string line; std::string line;
std::string id; std::optional<std::string> id;
while(getline(is, line)) { while(!id && getline(is, line)) {
// Append $ to avoid partial name matches id = parseListReferenceForRev(ref_uri, line);
std::regex pattern(fmt("%s$", ref_uri));
if (std::regex_search(line, pattern)) {
id = line.substr(0, line.find('\t'));
break;
}
} }
if(id.empty()) if(!id)
throw BadURL("in '%d', couldn't find ref '%d'", input.to_string(), ref); throw BadURL("in '%d', couldn't find ref '%d'", input.to_string(), ref);
auto rev = Hash::parseAny(id, htSHA1); auto rev = Hash::parseAny(*id, htSHA1);
debug("HEAD revision for '%s' is %s", fmt("%s/%s", base_url, ref), rev.gitRev()); debug("HEAD revision for '%s' is %s", fmt("%s/%s", base_url, ref), rev.gitRev());
return rev; return rev;
} }

33
src/libstore/build/derivation-goal.cc
View file

@ -984,21 +984,28 @@ void DerivationGoal::resolvedFinished()
realWantedOutputs = resolvedDrv.outputNames(); realWantedOutputs = resolvedDrv.outputNames();
for (auto & wantedOutput : realWantedOutputs) { for (auto & wantedOutput : realWantedOutputs) {
assert(initialOutputs.count(wantedOutput) != 0); auto initialOutput = get(initialOutputs, wantedOutput);
assert(resolvedHashes.count(wantedOutput) != 0); auto resolvedHash = get(resolvedHashes, wantedOutput);
auto realisation = resolvedResult.builtOutputs.at( if ((!initialOutput) || (!resolvedHash))
DrvOutput { resolvedHashes.at(wantedOutput), wantedOutput }); throw Error(
"derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,resolve)",
worker.store.printStorePath(drvPath), wantedOutput);
auto realisation = get(resolvedResult.builtOutputs, DrvOutput { *resolvedHash, wantedOutput });
if (!realisation)
throw Error(
"derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,realisation)",
worker.store.printStorePath(resolvedDrvGoal->drvPath), wantedOutput);
if (drv->type().isPure()) { if (drv->type().isPure()) {
auto newRealisation = realisation; auto newRealisation = *realisation;
newRealisation.id = DrvOutput { initialOutputs.at(wantedOutput).outputHash, wantedOutput }; newRealisation.id = DrvOutput { initialOutput->outputHash, wantedOutput };
newRealisation.signatures.clear(); newRealisation.signatures.clear();
if (!drv->type().isFixed()) if (!drv->type().isFixed())
newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation.outPath); newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation->outPath);
signRealisation(newRealisation); signRealisation(newRealisation);
worker.store.registerDrvOutput(newRealisation); worker.store.registerDrvOutput(newRealisation);
} }
outputPaths.insert(realisation.outPath); outputPaths.insert(realisation->outPath);
builtOutputs.emplace(realisation.id, realisation); builtOutputs.emplace(realisation->id, *realisation);
} }
runPostBuildHook( runPostBuildHook(
@ -1294,7 +1301,11 @@ std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity()
DrvOutputs validOutputs; DrvOutputs validOutputs;
for (auto & i : queryPartialDerivationOutputMap()) { for (auto & i : queryPartialDerivationOutputMap()) {
InitialOutput & info = initialOutputs.at(i.first); auto initialOutput = get(initialOutputs, i.first);
if (!initialOutput)
// this is an invalid output, gets catched with (!wantedOutputsLeft.empty())
continue;
auto & info = *initialOutput;
info.wanted = wantOutput(i.first, wantedOutputs); info.wanted = wantOutput(i.first, wantedOutputs);
if (info.wanted) if (info.wanted)
wantedOutputsLeft.erase(i.first); wantedOutputsLeft.erase(i.first);
@ -1309,7 +1320,7 @@ std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity()
: PathStatus::Corrupt, : PathStatus::Corrupt,
}; };
} }
auto drvOutput = DrvOutput{initialOutputs.at(i.first).outputHash, i.first}; auto drvOutput = DrvOutput{info.outputHash, i.first};
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
if (auto real = worker.store.queryRealisation(drvOutput)) { if (auto real = worker.store.queryRealisation(drvOutput)) {
info.known = { info.known = {

118
src/libstore/build/local-derivation-goal.cc
View file

@ -14,6 +14,7 @@
#include "worker-protocol.hh" #include "worker-protocol.hh"
#include "topo-sort.hh" #include "topo-sort.hh"
#include "callback.hh" #include "callback.hh"
#include "json-utils.hh"
#include <regex> #include <regex>
#include <queue> #include <queue>
@ -56,8 +57,6 @@
#include <pwd.h> #include <pwd.h>
#include <grp.h> #include <grp.h>
#include <nlohmann/json.hpp>
namespace nix { namespace nix {
void handleDiffHook( void handleDiffHook(
@ -482,7 +481,7 @@ void LocalDerivationGoal::startBuilder()
temporary build directory. The text files have the format used temporary build directory. The text files have the format used
by `nix-store --register-validity'. However, the deriver by `nix-store --register-validity'. However, the deriver
fields are left empty. */ fields are left empty. */
auto s = get(drv->env, "exportReferencesGraph").value_or(""); auto s = getOr(drv->env, "exportReferencesGraph", "");
Strings ss = tokenizeString<Strings>(s); Strings ss = tokenizeString<Strings>(s);
if (ss.size() % 2 != 0) if (ss.size() % 2 != 0)
throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s); throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s);
@ -989,7 +988,7 @@ void LocalDerivationGoal::initTmpDir() {
there is no size constraint). */ there is no size constraint). */
if (!parsedDrv->getStructuredAttrs()) { if (!parsedDrv->getStructuredAttrs()) {
StringSet passAsFile = tokenizeString<StringSet>(get(drv->env, "passAsFile").value_or("")); StringSet passAsFile = tokenizeString<StringSet>(getOr(drv->env, "passAsFile", ""));
for (auto & i : drv->env) { for (auto & i : drv->env) {
if (passAsFile.find(i.first) == passAsFile.end()) { if (passAsFile.find(i.first) == passAsFile.end()) {
env[i.first] = i.second; env[i.first] = i.second;
@ -2128,12 +2127,22 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
std::map<std::string, std::variant<AlreadyRegistered, PerhapsNeedToRegister>> outputReferencesIfUnregistered; std::map<std::string, std::variant<AlreadyRegistered, PerhapsNeedToRegister>> outputReferencesIfUnregistered;
std::map<std::string, struct stat> outputStats; std::map<std::string, struct stat> outputStats;
for (auto & [outputName, _] : drv->outputs) { for (auto & [outputName, _] : drv->outputs) {
auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchOutputs.at(outputName))); auto scratchOutput = get(scratchOutputs, outputName);
if (!scratchOutput)
throw BuildError(
"builder for '%s' has no scratch output for '%s'",
worker.store.printStorePath(drvPath), outputName);
auto actualPath = toRealPathChroot(worker.store.printStorePath(*scratchOutput));
outputsToSort.insert(outputName); outputsToSort.insert(outputName);
/* Updated wanted info to remove the outputs we definitely don't need to register */ /* Updated wanted info to remove the outputs we definitely don't need to register */
auto & initialInfo = initialOutputs.at(outputName); auto initialOutput = get(initialOutputs, outputName);
if (!initialOutput)
throw BuildError(
"builder for '%s' has no initial output for '%s'",
worker.store.printStorePath(drvPath), outputName);
auto & initialInfo = *initialOutput;
/* Don't register if already valid, and not checking */ /* Don't register if already valid, and not checking */
initialInfo.wanted = buildMode == bmCheck initialInfo.wanted = buildMode == bmCheck
@ -2185,6 +2194,11 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
auto sortedOutputNames = topoSort(outputsToSort, auto sortedOutputNames = topoSort(outputsToSort,
{[&](const std::string & name) { {[&](const std::string & name) {
auto orifu = get(outputReferencesIfUnregistered, name);
if (!orifu)
throw BuildError(
"no output reference for '%s' in build of '%s'",
name, worker.store.printStorePath(drvPath));
return std::visit(overloaded { return std::visit(overloaded {
/* Since we'll use the already installed versions of these, we /* Since we'll use the already installed versions of these, we
can treat them as leaves and ignore any references they can treat them as leaves and ignore any references they
@ -2199,7 +2213,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
referencedOutputs.insert(o); referencedOutputs.insert(o);
return referencedOutputs; return referencedOutputs;
}, },
}, outputReferencesIfUnregistered.at(name)); }, *orifu);
}}, }},
{[&](const std::string & path, const std::string & parent) { {[&](const std::string & path, const std::string & parent) {
// TODO with more -vvvv also show the temporary paths for manual inspection. // TODO with more -vvvv also show the temporary paths for manual inspection.
@ -2213,9 +2227,10 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
OutputPathMap finalOutputs; OutputPathMap finalOutputs;
for (auto & outputName : sortedOutputNames) { for (auto & outputName : sortedOutputNames) {
auto output = drv->outputs.at(outputName); auto output = get(drv->outputs, outputName);
auto & scratchPath = scratchOutputs.at(outputName); auto scratchPath = get(scratchOutputs, outputName);
auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchPath)); assert(output && scratchPath);
auto actualPath = toRealPathChroot(worker.store.printStorePath(*scratchPath));
auto finish = [&](StorePath finalStorePath) { auto finish = [&](StorePath finalStorePath) {
/* Store the final path */ /* Store the final path */
@ -2223,10 +2238,13 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
/* The rewrite rule will be used in downstream outputs that refer to /* The rewrite rule will be used in downstream outputs that refer to
use. This is why the topological sort is essential to do first use. This is why the topological sort is essential to do first
before this for loop. */ before this for loop. */
if (scratchPath != finalStorePath) if (*scratchPath != finalStorePath)
outputRewrites[std::string { scratchPath.hashPart() }] = std::string { finalStorePath.hashPart() }; outputRewrites[std::string { scratchPath->hashPart() }] = std::string { finalStorePath.hashPart() };
}; };
auto orifu = get(outputReferencesIfUnregistered, outputName);
assert(orifu);
std::optional<StorePathSet> referencesOpt = std::visit(overloaded { std::optional<StorePathSet> referencesOpt = std::visit(overloaded {
[&](const AlreadyRegistered & skippedFinalPath) -> std::optional<StorePathSet> { [&](const AlreadyRegistered & skippedFinalPath) -> std::optional<StorePathSet> {
finish(skippedFinalPath.path); finish(skippedFinalPath.path);
@ -2235,7 +2253,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
[&](const PerhapsNeedToRegister & r) -> std::optional<StorePathSet> { [&](const PerhapsNeedToRegister & r) -> std::optional<StorePathSet> {
return r.refs; return r.refs;
}, },
}, outputReferencesIfUnregistered.at(outputName)); }, *orifu);
if (!referencesOpt) if (!referencesOpt)
continue; continue;
@ -2268,25 +2286,29 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
for (auto & r : references) { for (auto & r : references) {
auto name = r.name(); auto name = r.name();
auto origHash = std::string { r.hashPart() }; auto origHash = std::string { r.hashPart() };
if (r == scratchPath) if (r == *scratchPath) {
res.first = true; res.first = true;
else if (outputRewrites.count(origHash) == 0) } else if (auto outputRewrite = get(outputRewrites, origHash)) {
res.second.insert(r); std::string newRef = *outputRewrite;
else {
std::string newRef = outputRewrites.at(origHash);
newRef += '-'; newRef += '-';
newRef += name; newRef += name;
res.second.insert(StorePath { newRef }); res.second.insert(StorePath { newRef });
} else {
res.second.insert(r);
} }
} }
return res; return res;
}; };
auto newInfoFromCA = [&](const DerivationOutput::CAFloating outputHash) -> ValidPathInfo { auto newInfoFromCA = [&](const DerivationOutput::CAFloating outputHash) -> ValidPathInfo {
auto & st = outputStats.at(outputName); auto st = get(outputStats, outputName);
if (!st)
throw BuildError(
"output path %1% without valid stats info",
actualPath);
if (outputHash.method == FileIngestionMethod::Flat) { if (outputHash.method == FileIngestionMethod::Flat) {
/* The output path should be a regular file without execute permission. */ /* The output path should be a regular file without execute permission. */
if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) if (!S_ISREG(st->st_mode) || (st->st_mode & S_IXUSR) != 0)
throw BuildError( throw BuildError(
"output path '%1%' should be a non-executable regular file " "output path '%1%' should be a non-executable regular file "
"since recursive hashing is not enabled (outputHashMode=flat)", "since recursive hashing is not enabled (outputHashMode=flat)",
@ -2294,7 +2316,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
} }
rewriteOutput(); rewriteOutput();
/* FIXME optimize and deduplicate with addToStore */ /* FIXME optimize and deduplicate with addToStore */
std::string oldHashPart { scratchPath.hashPart() }; std::string oldHashPart { scratchPath->hashPart() };
HashModuloSink caSink { outputHash.hashType, oldHashPart }; HashModuloSink caSink { outputHash.hashType, oldHashPart };
switch (outputHash.method) { switch (outputHash.method) {
case FileIngestionMethod::Recursive: case FileIngestionMethod::Recursive:
@ -2313,7 +2335,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
outputPathName(drv->name, outputName), outputPathName(drv->name, outputName),
refs.second, refs.second,
refs.first); refs.first);
if (scratchPath != finalPath) { if (*scratchPath != finalPath) {
// Also rewrite the output path // Also rewrite the output path
auto source = sinkToSource([&](Sink & nextSink) { auto source = sinkToSource([&](Sink & nextSink) {
StringSink sink; StringSink sink;
@ -2354,9 +2376,9 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
auto requiredFinalPath = output.path; auto requiredFinalPath = output.path;
/* Preemptively add rewrite rule for final hash, as that is /* Preemptively add rewrite rule for final hash, as that is
what the NAR hash will use rather than normalized-self references */ what the NAR hash will use rather than normalized-self references */
if (scratchPath != requiredFinalPath) if (*scratchPath != requiredFinalPath)
outputRewrites.insert_or_assign( outputRewrites.insert_or_assign(
std::string { scratchPath.hashPart() }, std::string { scratchPath->hashPart() },
std::string { requiredFinalPath.hashPart() }); std::string { requiredFinalPath.hashPart() });
rewriteOutput(); rewriteOutput();
auto narHashAndSize = hashPath(htSHA256, actualPath); auto narHashAndSize = hashPath(htSHA256, actualPath);
@ -2409,7 +2431,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
}); });
}, },
}, output.raw()); }, output->raw());
/* FIXME: set proper permissions in restorePath() so /* FIXME: set proper permissions in restorePath() so
we don't have to do another traversal. */ we don't have to do another traversal. */
@ -2425,7 +2447,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
derivations. */ derivations. */
PathLocks dynamicOutputLock; PathLocks dynamicOutputLock;
dynamicOutputLock.setDeletion(true); dynamicOutputLock.setDeletion(true);
auto optFixedPath = output.path(worker.store, drv->name, outputName); auto optFixedPath = output->path(worker.store, drv->name, outputName);
if (!optFixedPath || if (!optFixedPath ||
worker.store.printStorePath(*optFixedPath) != finalDestPath) worker.store.printStorePath(*optFixedPath) != finalDestPath)
{ {
@ -2491,11 +2513,10 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
/* For debugging, print out the referenced and unreferenced paths. */ /* For debugging, print out the referenced and unreferenced paths. */
for (auto & i : inputPaths) { for (auto & i : inputPaths) {
auto j = references.find(i); if (references.count(i))
if (j == references.end())
debug("unreferenced input: '%1%'", worker.store.printStorePath(i));
else
debug("referenced input: '%1%'", worker.store.printStorePath(i)); debug("referenced input: '%1%'", worker.store.printStorePath(i));
else
debug("unreferenced input: '%1%'", worker.store.printStorePath(i));
} }
if (curRound == nrRounds) { if (curRound == nrRounds) {
@ -2612,9 +2633,11 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
DrvOutputs builtOutputs; DrvOutputs builtOutputs;
for (auto & [outputName, newInfo] : infos) { for (auto & [outputName, newInfo] : infos) {
auto oldinfo = get(initialOutputs, outputName);
assert(oldinfo);
auto thisRealisation = Realisation { auto thisRealisation = Realisation {
.id = DrvOutput { .id = DrvOutput {
initialOutputs.at(outputName).outputHash, oldinfo->outputHash,
outputName outputName
}, },
.outPath = newInfo.path .outPath = newInfo.path
@ -2710,9 +2733,10 @@ void LocalDerivationGoal::checkOutputs(const std::map<std::string, ValidPathInfo
for (auto & i : *value) { for (auto & i : *value) {
if (worker.store.isStorePath(i)) if (worker.store.isStorePath(i))
spec.insert(worker.store.parseStorePath(i)); spec.insert(worker.store.parseStorePath(i));
else if (outputs.count(i)) else if (auto output = get(outputs, i))
spec.insert(outputs.at(i).path); spec.insert(output->path);
else throw BuildError("derivation contains an illegal reference specifier '%s'", i); else
throw BuildError("derivation contains an illegal reference specifier '%s'", i);
} }
auto used = recursive auto used = recursive
@ -2751,24 +2775,18 @@ void LocalDerivationGoal::checkOutputs(const std::map<std::string, ValidPathInfo
}; };
if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) { if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) {
auto outputChecks = structuredAttrs->find("outputChecks"); if (auto outputChecks = get(*structuredAttrs, "outputChecks")) {
if (outputChecks != structuredAttrs->end()) { if (auto output = get(*outputChecks, outputName)) {
auto output = outputChecks->find(outputName);
if (output != outputChecks->end()) {
Checks checks; Checks checks;
auto maxSize = output->find("maxSize"); if (auto maxSize = get(*output, "maxSize"))
if (maxSize != output->end())
checks.maxSize = maxSize->get<uint64_t>(); checks.maxSize = maxSize->get<uint64_t>();
auto maxClosureSize = output->find("maxClosureSize"); if (auto maxClosureSize = get(*output, "maxClosureSize"))
if (maxClosureSize != output->end())
checks.maxClosureSize = maxClosureSize->get<uint64_t>(); checks.maxClosureSize = maxClosureSize->get<uint64_t>();
auto get = [&](const std::string & name) -> std::optional<Strings> { auto get_ = [&](const std::string & name) -> std::optional<Strings> {
auto i = output->find(name); if (auto i = get(*output, name)) {
if (i != output->end()) {
Strings res; Strings res;
for (auto j = i->begin(); j != i->end(); ++j) { for (auto j = i->begin(); j != i->end(); ++j) {
if (!j->is_string()) if (!j->is_string())
@ -2781,10 +2799,10 @@ void LocalDerivationGoal::checkOutputs(const std::map<std::string, ValidPathInfo
return {}; return {};
}; };
checks.allowedReferences = get("allowedReferences"); checks.allowedReferences = get_("allowedReferences");
checks.allowedRequisites = get("allowedRequisites"); checks.allowedRequisites = get_("allowedRequisites");
checks.disallowedReferences = get("disallowedReferences"); checks.disallowedReferences = get_("disallowedReferences");
checks.disallowedRequisites = get("disallowedRequisites"); checks.disallowedRequisites = get_("disallowedRequisites");
applyChecks(checks); applyChecks(checks);
} }

7
src/libstore/build/worker.cc
View file

@ -350,7 +350,7 @@ void Worker::waitForInput()
become `available'. Note that `available' (i.e., non-blocking) become `available'. Note that `available' (i.e., non-blocking)
includes EOF. */ includes EOF. */
std::vector<struct pollfd> pollStatus; std::vector<struct pollfd> pollStatus;
std::map <int, int> fdToPollStatus; std::map<int, size_t> fdToPollStatus;
for (auto & i : children) { for (auto & i : children) {
for (auto & j : i.fds) { for (auto & j : i.fds) {
pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN }); pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN });
@ -380,7 +380,10 @@ void Worker::waitForInput()
std::set<int> fds2(j->fds); std::set<int> fds2(j->fds);
std::vector<unsigned char> buffer(4096); std::vector<unsigned char> buffer(4096);
for (auto & k : fds2) { for (auto & k : fds2) {
if (pollStatus.at(fdToPollStatus.at(k)).revents) { const auto fdPollStatusId = get(fdToPollStatus, k);
assert(fdPollStatusId);
assert(*fdPollStatusId < pollStatus.size());
if (pollStatus.at(*fdPollStatusId).revents) {
ssize_t rd = ::read(k, buffer.data(), buffer.size()); ssize_t rd = ::read(k, buffer.data(), buffer.size());
// FIXME: is there a cleaner way to handle pt close // FIXME: is there a cleaner way to handle pt close
// than EIO? Is this even standard? // than EIO? Is this even standard?

2
src/libstore/builtins/fetchurl.cc
View file

@ -24,7 +24,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
Path storePath = getAttr("out"); Path storePath = getAttr("out");
auto mainUrl = getAttr("url"); auto mainUrl = getAttr("url");
bool unpack = get(drv.env, "unpack").value_or("") == "1"; bool unpack = getOr(drv.env, "unpack", "") == "1";
/* Note: have to use a fresh fileTransfer here because we're in /* Note: have to use a fresh fileTransfer here because we're in
a forked process. */ a forked process. */

13
src/libstore/derivations.cc
View file

@ -661,8 +661,10 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut
if (res.kind == DrvHash::Kind::Deferred) if (res.kind == DrvHash::Kind::Deferred)
kind = DrvHash::Kind::Deferred; kind = DrvHash::Kind::Deferred;
for (auto & outputName : inputOutputs) { for (auto & outputName : inputOutputs) {
const auto h = res.hashes.at(outputName); const auto h = get(res.hashes, outputName);
inputs2[h.to_string(Base16, false)].insert(outputName); if (!h)
throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name);
inputs2[h->to_string(Base16, false)].insert(outputName);
} }
} }
@ -836,8 +838,11 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String
auto hashModulo = hashDerivationModulo(store, Derivation(drv), true); auto hashModulo = hashDerivationModulo(store, Derivation(drv), true);
for (auto & [outputName, output] : drv.outputs) { for (auto & [outputName, output] : drv.outputs) {
if (std::holds_alternative<DerivationOutput::Deferred>(output.raw())) { if (std::holds_alternative<DerivationOutput::Deferred>(output.raw())) {
auto & h = hashModulo.hashes.at(outputName); auto h = get(hashModulo.hashes, outputName);
auto outPath = store.makeOutputPath(outputName, h, drv.name); if (!h)
throw Error("derivation '%s' output '%s' has no hash (derivations.cc/rewriteDerivation)",
drv.name, outputName);
auto outPath = store.makeOutputPath(outputName, *h, drv.name);
drv.env[outputName] = store.printStorePath(outPath); drv.env[outputName] = store.printStorePath(outPath);
output = DerivationOutput::InputAddressed { output = DerivationOutput::InputAddressed {
.path = std::move(outPath), .path = std::move(outPath),

23
src/libstore/derived-path.cc
View file

@ -4,6 +4,8 @@
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
#include <optional>
namespace nix { namespace nix {
nlohmann::json DerivedPath::Opaque::toJSON(ref<Store> store) const { nlohmann::json DerivedPath::Opaque::toJSON(ref<Store> store) const {
@ -17,12 +19,12 @@ nlohmann::json DerivedPath::Built::toJSON(ref<Store> store) const {
res["drvPath"] = store->printStorePath(drvPath); res["drvPath"] = store->printStorePath(drvPath);
// Fallback for the input-addressed derivation case: We expect to always be // Fallback for the input-addressed derivation case: We expect to always be
// able to print the output paths, so let’s do it // able to print the output paths, so let’s do it
auto knownOutputs = store->queryPartialDerivationOutputMap(drvPath); const auto knownOutputs = store->queryPartialDerivationOutputMap(drvPath);
for (const auto& output : outputs) { for (const auto& output : outputs) {
if (knownOutputs.at(output)) auto knownOutput = get(knownOutputs, output);
res["outputs"][output] = store->printStorePath(knownOutputs.at(output).value()); res["outputs"][output] = (knownOutput && *knownOutput)
else ? store->printStorePath(**knownOutput)
res["outputs"][output] = nullptr; : nullptr;
} }
return res; return res;
} }
@ -123,10 +125,15 @@ RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const
for (auto& [outputName, outputPath] : p.outputs) { for (auto& [outputName, outputPath] : p.outputs) {
if (settings.isExperimentalFeatureEnabled( if (settings.isExperimentalFeatureEnabled(
Xp::CaDerivations)) { Xp::CaDerivations)) {
auto drvOutput = get(drvHashes, outputName);
if (!drvOutput)
throw Error(
"the derivation '%s' has unrealised output '%s' (derived-path.cc/toRealisedPaths)",
store.printStorePath(p.drvPath), outputName);
auto thisRealisation = store.queryRealisation( auto thisRealisation = store.queryRealisation(
DrvOutput{drvHashes.at(outputName), outputName}); DrvOutput{*drvOutput, outputName});
assert(thisRealisation); // We’ve built it, so we must h assert(thisRealisation); // We’ve built it, so we must
// ve the realisation // have the realisation
res.insert(*thisRealisation); res.insert(*thisRealisation);
} else { } else {
res.insert(outputPath); res.insert(outputPath);

8
src/libstore/filetransfer.cc
View file

@ -692,10 +692,10 @@ struct curlFileTransfer : public FileTransfer
#if ENABLE_S3 #if ENABLE_S3
auto [bucketName, key, params] = parseS3Uri(request.uri); auto [bucketName, key, params] = parseS3Uri(request.uri);
std::string profile = get(params, "profile").value_or(""); std::string profile = getOr(params, "profile", "");
std::string region = get(params, "region").value_or(Aws::Region::US_EAST_1); std::string region = getOr(params, "region", Aws::Region::US_EAST_1);
std::string scheme = get(params, "scheme").value_or(""); std::string scheme = getOr(params, "scheme", "");
std::string endpoint = get(params, "endpoint").value_or(""); std::string endpoint = getOr(params, "endpoint", "");
S3Helper s3Helper(profile, region, scheme, endpoint); S3Helper s3Helper(profile, region, scheme, endpoint);

6
src/libstore/local-store.cc
View file

@ -718,7 +718,11 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
// somewhat expensive so we do lazily // somewhat expensive so we do lazily
hashesModulo = hashDerivationModulo(*this, drv, true); hashesModulo = hashDerivationModulo(*this, drv, true);
} }
StorePath recomputed = makeOutputPath(i.first, hashesModulo->hashes.at(i.first), drvName); auto currentOutputHash = get(hashesModulo->hashes, i.first);
if (!currentOutputHash)
throw Error("derivation '%s' has unexpected output '%s' (local-store / hashesModulo) named '%s'",
printStorePath(drvPath), printStorePath(doia.path), i.first);
StorePath recomputed = makeOutputPath(i.first, *currentOutputHash, drvName);
if (doia.path != recomputed) if (doia.path != recomputed)
throw Error("derivation '%s' has incorrect output '%s', should be '%s'", throw Error("derivation '%s' has incorrect output '%s', should be '%s'",
printStorePath(drvPath), printStorePath(doia.path), printStorePath(recomputed)); printStorePath(drvPath), printStorePath(doia.path), printStorePath(recomputed));

9
src/libstore/misc.cc
View file

@ -278,11 +278,16 @@ std::map<DrvOutput, StorePath> drvOutputReferences(
std::set<Realisation> inputRealisations; std::set<Realisation> inputRealisations;
for (const auto & [inputDrv, outputNames] : drv.inputDrvs) { for (const auto & [inputDrv, outputNames] : drv.inputDrvs) {
auto outputHashes = const auto outputHashes =
staticOutputHashes(store, store.readDerivation(inputDrv)); staticOutputHashes(store, store.readDerivation(inputDrv));
for (const auto & outputName : outputNames) { for (const auto & outputName : outputNames) {
auto outputHash = get(outputHashes, outputName);
if (!outputHash)
throw Error(
"output '%s' of derivation '%s' isn't realised", outputName,
store.printStorePath(inputDrv));
auto thisRealisation = store.queryRealisation( auto thisRealisation = store.queryRealisation(
DrvOutput{outputHashes.at(outputName), outputName}); DrvOutput{*outputHash, outputName});
if (!thisRealisation) if (!thisRealisation)
throw Error( throw Error(
"output '%s' of derivation '%s' isn't built", outputName, "output '%s' of derivation '%s' isn't built", outputName,

56
src/libstore/path-with-outputs.cc
View file

@ -1,5 +1,8 @@
#include "path-with-outputs.hh" #include "path-with-outputs.hh"
#include "store-api.hh" #include "store-api.hh"
#include "nlohmann/json.hpp"
#include <regex>
namespace nix { namespace nix {
@ -68,4 +71,57 @@ StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std:
return StorePathWithOutputs { store.followLinksToStorePath(path), std::move(outputs) }; return StorePathWithOutputs { store.followLinksToStorePath(path), std::move(outputs) };
} }
std::pair<std::string, OutputsSpec> parseOutputsSpec(const std::string & s)
{
static std::regex regex(R"((.*)\^((\*)|([a-z]+(,[a-z]+)*)))");
std::smatch match;
if (!std::regex_match(s, match, regex))
return {s, DefaultOutputs()};
if (match[3].matched)
return {match[1], AllOutputs()};
return {match[1], tokenizeString<OutputNames>(match[4].str(), ",")};
}
std::string printOutputsSpec(const OutputsSpec & outputsSpec)
{
if (std::get_if<DefaultOutputs>(&outputsSpec))
return "";
if (std::get_if<AllOutputs>(&outputsSpec))
return "^*";
if (auto outputNames = std::get_if<OutputNames>(&outputsSpec))
return "^" + concatStringsSep(",", *outputNames);
assert(false);
}
void to_json(nlohmann::json & json, const OutputsSpec & outputsSpec)
{
if (std::get_if<DefaultOutputs>(&outputsSpec))
json = nullptr;
else if (std::get_if<AllOutputs>(&outputsSpec))
json = std::vector<std::string>({"*"});
else if (auto outputNames = std::get_if<OutputNames>(&outputsSpec))
json = *outputNames;
}
void from_json(const nlohmann::json & json, OutputsSpec & outputsSpec)
{
if (json.is_null())
outputsSpec = DefaultOutputs();
else {
auto names = json.get<OutputNames>();
if (names == OutputNames({"*"}))
outputsSpec = AllOutputs();
else
outputsSpec = names;
}
}
} }

22
src/libstore/path-with-outputs.hh
View file

@ -4,6 +4,7 @@
#include "path.hh" #include "path.hh"
#include "derived-path.hh" #include "derived-path.hh"
#include "nlohmann/json_fwd.hpp"
namespace nix { namespace nix {
@ -32,4 +33,25 @@ StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view
StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs); StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs);
typedef std::set<std::string> OutputNames;
struct AllOutputs {
bool operator < (const AllOutputs & _) const { return false; }
};
struct DefaultOutputs {
bool operator < (const DefaultOutputs & _) const { return false; }
};
typedef std::variant<DefaultOutputs, AllOutputs, OutputNames> OutputsSpec;
/* Parse a string of the form 'prefix^output1,...outputN' or
'prefix^*', returning the prefix and the outputs spec. */
std::pair<std::string, OutputsSpec> parseOutputsSpec(const std::string & s);
std::string printOutputsSpec(const OutputsSpec & outputsSpec);
void to_json(nlohmann::json &, const OutputsSpec &);
void from_json(const nlohmann::json &, OutputsSpec &);
} }

17
src/libstore/remote-store.cc
View file

@ -853,15 +853,15 @@ std::vector<BuildResult> RemoteStore::buildPathsWithResults(
OutputPathMap outputs; OutputPathMap outputs;
auto drv = evalStore->readDerivation(bfd.drvPath); auto drv = evalStore->readDerivation(bfd.drvPath);
auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive const auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive
auto drvOutputs = drv.outputsAndOptPaths(*this); const auto drvOutputs = drv.outputsAndOptPaths(*this);
for (auto & output : bfd.outputs) { for (auto & output : bfd.outputs) {
if (!outputHashes.count(output)) auto outputHash = get(outputHashes, output);
if (!outputHash)
throw Error( throw Error(
"the derivation '%s' doesn't have an output named '%s'", "the derivation '%s' doesn't have an output named '%s'",
printStorePath(bfd.drvPath), output); printStorePath(bfd.drvPath), output);
auto outputId = auto outputId = DrvOutput{ *outputHash, output };
DrvOutput{outputHashes.at(output), output};
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
auto realisation = auto realisation =
queryRealisation(outputId); queryRealisation(outputId);
@ -874,13 +874,14 @@ std::vector<BuildResult> RemoteStore::buildPathsWithResults(
} else { } else {
// If ca-derivations isn't enabled, assume that // If ca-derivations isn't enabled, assume that
// the output path is statically known. // the output path is statically known.
assert(drvOutputs.count(output)); const auto drvOutput = get(drvOutputs, output);
assert(drvOutputs.at(output).second); assert(drvOutput);
assert(drvOutput->second);
res.builtOutputs.emplace( res.builtOutputs.emplace(
outputId, outputId,
Realisation { Realisation {
.id = outputId, .id = outputId,
.outPath = *drvOutputs.at(output).second .outPath = *drvOutput->second,
}); });
} }
} }

2
src/libstore/store-api.cc
View file

@ -1314,7 +1314,7 @@ static bool isNonUriPath(const std::string & spec) {
std::shared_ptr<Store> openFromNonUri(const std::string & uri, const Store::Params & params) std::shared_ptr<Store> openFromNonUri(const std::string & uri, const Store::Params & params)
{ {
if (uri == "" || uri == "auto") { if (uri == "" || uri == "auto") {
auto stateDir = get(params, "state").value_or(settings.nixStateDir); auto stateDir = getOr(params, "state", settings.nixStateDir);
if (access(stateDir.c_str(), R_OK | W_OK) == 0) if (access(stateDir.c_str(), R_OK | W_OK) == 0)
return std::make_shared<LocalStore>(params); return std::make_shared<LocalStore>(params);
else if (pathExists(settings.nixDaemonSocketFile)) else if (pathExists(settings.nixDaemonSocketFile))

46
src/libstore/tests/path-with-outputs.cc Normal file
View file

@ -0,0 +1,46 @@
#include "path-with-outputs.hh"
#include <gtest/gtest.h>
namespace nix {
TEST(parseOutputsSpec, basic)
{
{
auto [prefix, outputsSpec] = parseOutputsSpec("foo");
ASSERT_EQ(prefix, "foo");
ASSERT_TRUE(std::get_if<DefaultOutputs>(&outputsSpec));
}
{
auto [prefix, outputsSpec] = parseOutputsSpec("foo^*");
ASSERT_EQ(prefix, "foo");
ASSERT_TRUE(std::get_if<AllOutputs>(&outputsSpec));
}
{
auto [prefix, outputsSpec] = parseOutputsSpec("foo^out");
ASSERT_EQ(prefix, "foo");
ASSERT_TRUE(std::get<OutputNames>(outputsSpec) == OutputNames({"out"}));
}
{
auto [prefix, outputsSpec] = parseOutputsSpec("foo^out,bin");
ASSERT_EQ(prefix, "foo");
ASSERT_TRUE(std::get<OutputNames>(outputsSpec) == OutputNames({"out", "bin"}));
}
{
auto [prefix, outputsSpec] = parseOutputsSpec("foo^bar^out,bin");
ASSERT_EQ(prefix, "foo^bar");
ASSERT_TRUE(std::get<OutputNames>(outputsSpec) == OutputNames({"out", "bin"}));
}
{
auto [prefix, outputsSpec] = parseOutputsSpec("foo^&*()");
ASSERT_EQ(prefix, "foo^&*()");
ASSERT_TRUE(std::get_if<DefaultOutputs>(&outputsSpec));
}
}
}

10
src/libutil/experimental-features.cc
View file

@ -35,7 +35,9 @@ const std::optional<ExperimentalFeature> parseExperimentalFeature(const std::str
std::string_view showExperimentalFeature(const ExperimentalFeature feature) std::string_view showExperimentalFeature(const ExperimentalFeature feature)
{ {
return stringifiedXpFeatures.at(feature); const auto ret = get(stringifiedXpFeatures, feature);
assert(ret);
return *ret;
} }
std::set<ExperimentalFeature> parseFeatures(const std::set<std::string> & rawFeatures) std::set<ExperimentalFeature> parseFeatures(const std::set<std::string> & rawFeatures)
@ -58,11 +60,13 @@ std::ostream & operator <<(std::ostream & str, const ExperimentalFeature & featu
return str << showExperimentalFeature(feature); return str << showExperimentalFeature(feature);
} }
void to_json(nlohmann::json& j, const ExperimentalFeature& feature) { void to_json(nlohmann::json & j, const ExperimentalFeature & feature)
{
j = showExperimentalFeature(feature); j = showExperimentalFeature(feature);
} }
void from_json(const nlohmann::json& j, ExperimentalFeature& feature) { void from_json(const nlohmann::json & j, ExperimentalFeature & feature)
{
const std::string input = j; const std::string input = j;
const auto parsed = parseExperimentalFeature(input); const auto parsed = parseExperimentalFeature(input);

21
src/libutil/json-utils.hh Normal file
View file

@ -0,0 +1,21 @@
#pragma once
#include <nlohmann/json.hpp>
namespace nix {
const nlohmann::json * get(const nlohmann::json & map, const std::string & key)
{
auto i = map.find(key);
if (i == map.end()) return nullptr;
return &*i;
}
nlohmann::json * get(nlohmann::json & map, const std::string & key)
{
auto i = map.find(key);
if (i == map.end()) return nullptr;
return &*i;
}
}

20
src/libutil/tests/tests.cc
View file

@ -548,7 +548,7 @@ namespace nix {
TEST(get, emptyContainer) { TEST(get, emptyContainer) {
StringMap s = { }; StringMap s = { };
auto expected = std::nullopt; auto expected = nullptr;
ASSERT_EQ(get(s, "one"), expected); ASSERT_EQ(get(s, "one"), expected);
} }
@ -559,7 +559,23 @@ namespace nix {
s["two"] = "er"; s["two"] = "er";
auto expected = "yi"; auto expected = "yi";
ASSERT_EQ(get(s, "one"), expected); ASSERT_EQ(*get(s, "one"), expected);
}
TEST(getOr, emptyContainer) {
StringMap s = { };
auto expected = "yi";
ASSERT_EQ(getOr(s, "one", "yi"), expected);
}
TEST(getOr, getFromContainer) {
StringMap s;
s["one"] = "yi";
s["two"] = "er";
auto expected = "yi";
ASSERT_EQ(getOr(s, "one", "nope"), expected);
} }
/* ---------------------------------------------------------------------------- /* ----------------------------------------------------------------------------

1
src/libutil/util.cc
View file

@ -1588,7 +1588,6 @@ std::string stripIndentation(std::string_view s)
////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////
static Sync<std::pair<unsigned short, unsigned short>> windowSize{{0, 0}}; static Sync<std::pair<unsigned short, unsigned short>> windowSize{{0, 0}};

24
src/libutil/util.hh
View file

@ -543,13 +543,31 @@ std::string stripIndentation(std::string_view s);
/* Get a value for the specified key from an associate container. */ /* Get a value for the specified key from an associate container. */
template <class T> template <class T>
std::optional<typename T::mapped_type> get(const T & map, const typename T::key_type & key) const typename T::mapped_type * get(const T & map, const typename T::key_type & key)
{ {
auto i = map.find(key); auto i = map.find(key);
if (i == map.end()) return {}; if (i == map.end()) return nullptr;
return std::optional<typename T::mapped_type>(i->second); return &i->second;
} }
template <class T>
typename T::mapped_type * get(T & map, const typename T::key_type & key)
{
auto i = map.find(key);
if (i == map.end()) return nullptr;
return &i->second;
}
/* Get a value for the specified key from an associate container, or a default value if the key isn't present. */
template <class T>
const typename T::mapped_type & getOr(T & map,
const typename T::key_type & key,
const typename T::mapped_type & defaultValue)
{
auto i = map.find(key);
if (i == map.end()) return defaultValue;
return i->second;
}
/* Remove and return the first item from a container. */ /* Remove and return the first item from a container. */
template <class T> template <class T>

2
src/nix-build/nix-build.cc Executable file → Normal file
View file

@ -440,7 +440,7 @@ static void main_nix_build(int argc, char * * argv)
env["NIX_STORE"] = store->storeDir; env["NIX_STORE"] = store->storeDir;
env["NIX_BUILD_CORES"] = std::to_string(settings.buildCores); env["NIX_BUILD_CORES"] = std::to_string(settings.buildCores);
auto passAsFile = tokenizeString<StringSet>(get(drv.env, "passAsFile").value_or("")); auto passAsFile = tokenizeString<StringSet>(getOr(drv.env, "passAsFile", ""));
bool keepTmp = false; bool keepTmp = false;
int fileNr = 0; int fileNr = 0;

2
src/nix/app.cc
View file

@ -89,7 +89,7 @@ UnresolvedApp Installable::toApp(EvalState & state)
auto outputName = cursor->getAttr(state.sOutputName)->getString(); auto outputName = cursor->getAttr(state.sOutputName)->getString();
auto name = cursor->getAttr(state.sName)->getString(); auto name = cursor->getAttr(state.sName)->getString();
auto aPname = cursor->maybeGetAttr("pname"); auto aPname = cursor->maybeGetAttr("pname");
auto aMeta = cursor->maybeGetAttr("meta"); auto aMeta = cursor->maybeGetAttr(state.sMeta);
auto aMainProgram = aMeta ? aMeta->maybeGetAttr("mainProgram") : nullptr; auto aMainProgram = aMeta ? aMeta->maybeGetAttr("mainProgram") : nullptr;
auto mainProgram = auto mainProgram =
aMainProgram aMainProgram

4
src/nix/bundle.cc
View file

@ -75,10 +75,10 @@ struct CmdBundle : InstallableCommand
auto val = installable->toValue(*evalState).first; auto val = installable->toValue(*evalState).first;
auto [bundlerFlakeRef, bundlerName] = parseFlakeRefWithFragment(bundler, absPath(".")); auto [bundlerFlakeRef, bundlerName, outputsSpec] = parseFlakeRefWithFragmentAndOutputsSpec(bundler, absPath("."));
const flake::LockFlags lockFlags{ .writeLockFile = false }; const flake::LockFlags lockFlags{ .writeLockFile = false };
InstallableFlake bundler{this, InstallableFlake bundler{this,
evalState, std::move(bundlerFlakeRef), bundlerName, evalState, std::move(bundlerFlakeRef), bundlerName, outputsSpec,
{"bundlers." + settings.thisSystem.get() + ".default", {"bundlers." + settings.thisSystem.get() + ".default",
"defaultBundler." + settings.thisSystem.get() "defaultBundler." + settings.thisSystem.get()
}, },

1
src/nix/develop.cc
View file

@ -507,6 +507,7 @@ struct CmdDevelop : Common, MixEnvironment
state, state,
installable->nixpkgsFlakeRef(), installable->nixpkgsFlakeRef(),
"bashInteractive", "bashInteractive",
DefaultOutputs(),
Strings{}, Strings{},
Strings{"legacyPackages." + settings.thisSystem.get() + "."}, Strings{"legacyPackages." + settings.thisSystem.get() + "."},
nixpkgsLockFlags); nixpkgsLockFlags);

6
src/nix/flake.cc
View file

@ -724,7 +724,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand
auto [templateFlakeRef, templateName] = parseFlakeRefWithFragment(templateUrl, absPath(".")); auto [templateFlakeRef, templateName] = parseFlakeRefWithFragment(templateUrl, absPath("."));
auto installable = InstallableFlake(nullptr, auto installable = InstallableFlake(nullptr,
evalState, std::move(templateFlakeRef), templateName, evalState, std::move(templateFlakeRef), templateName, DefaultOutputs(),
defaultTemplateAttrPaths, defaultTemplateAttrPaths,
defaultTemplateAttrPathsPrefixes, defaultTemplateAttrPathsPrefixes,
lockFlags); lockFlags);
@ -1015,8 +1015,8 @@ struct CmdFlakeShow : FlakeCommand, MixJSON
auto name = visitor.getAttr(state->sName)->getString(); auto name = visitor.getAttr(state->sName)->getString();
if (json) { if (json) {
std::optional<std::string> description; std::optional<std::string> description;
if (auto aMeta = visitor.maybeGetAttr("meta")) { if (auto aMeta = visitor.maybeGetAttr(state->sMeta)) {
if (auto aDescription = aMeta->maybeGetAttr("description")) if (auto aDescription = aMeta->maybeGetAttr(state->sDescription))
description = aDescription->getString(); description = aDescription->getString();
} }
j.emplace("type", "derivation"); j.emplace("type", "derivation");

7
src/nix/flake.md
View file

@ -153,7 +153,7 @@ Currently the `type` attribute can be one of the following:
git(+http|+https|+ssh|+git|+file|):(//<server>)?<path>(\?<params>)? git(+http|+https|+ssh|+git|+file|):(//<server>)?<path>(\?<params>)?
``` ```
The `ref` attribute defaults to `master`. The `ref` attribute defaults to resolving the `HEAD` reference.
The `rev` attribute must denote a commit that exists in the branch The `rev` attribute must denote a commit that exists in the branch
or tag specified by the `ref` attribute, since Nix doesn't do a full or tag specified by the `ref` attribute, since Nix doesn't do a full
@ -161,6 +161,11 @@ Currently the `type` attribute can be one of the following:
doesn't allow fetching a `rev` without a known `ref`). The default doesn't allow fetching a `rev` without a known `ref`). The default
is the commit currently pointed to by `ref`. is the commit currently pointed to by `ref`.
When `git+file` is used without specifying `ref` or `rev`, files are
fetched directly from the local `path` as long as they have been added
to the Git repository. If there are uncommitted changes, the reference
is treated as dirty and a warning is printed.
For example, the following are valid Git flake references: For example, the following are valid Git flake references:
* `git+https://example.org/my/repo` * `git+https://example.org/my/repo`

45
src/nix/nix.md
View file

@ -146,6 +146,51 @@ For most commands, if no installable is specified, the default is `.`,
i.e. Nix will operate on the default flake output attribute of the i.e. Nix will operate on the default flake output attribute of the
flake in the current directory. flake in the current directory.
## Derivation output selection
Derivations can have multiple outputs, each corresponding to a
different store path. For instance, a package can have a `bin` output
that contains programs, and a `dev` output that provides development
artifacts like C/C++ header files. The outputs on which `nix` commands
operate are determined as follows:
* You can explicitly specify the desired outputs using the syntax
*installable*`^`*output1*`,`*...*`,`*outputN*. For example, you can
obtain the `dev` and `static` outputs of the `glibc` package:
```console
# nix build 'nixpkgs#glibc^dev,static'
# ls ./result-dev/include/ ./result-static/lib/
…
```
* You can also specify that *all* outputs should be used using the
syntax *installable*`^*`. For example, the following shows the size
of all outputs of the `glibc` package in the binary cache:
```console
# nix path-info -S --eval-store auto --store https://cache.nixos.org 'nixpkgs#glibc^*'
/nix/store/g02b1lpbddhymmcjb923kf0l7s9nww58-glibc-2.33-123 33208200
/nix/store/851dp95qqiisjifi639r0zzg5l465ny4-glibc-2.33-123-bin 36142896
/nix/store/kdgs3q6r7xdff1p7a9hnjr43xw2404z7-glibc-2.33-123-debug 155787312
/nix/store/n4xa8h6pbmqmwnq0mmsz08l38abb06zc-glibc-2.33-123-static 42488328
/nix/store/q6580lr01jpcsqs4r5arlh4ki2c1m9rv-glibc-2.33-123-dev 44200560
```
* If you didn't specify the desired outputs, but the derivation has an
attribute `meta.outputsToInstall`, Nix will use those outputs. For
example, since the package `nixpkgs#libxml2` has this attribute:
```console
# nix eval 'nixpkgs#libxml2.meta.outputsToInstall'
[ "bin" "man" ]
```
a command like `nix shell nixpkgs#libxml2` will provide only those
two outputs by default.
* Otherwise, Nix will use all outputs of the derivation.
# Nix stores # Nix stores
Most `nix` subcommands operate on a *Nix store*. Most `nix` subcommands operate on a *Nix store*.

7
src/nix/profile-install.md
View file

@ -20,6 +20,13 @@ R""(
# nix profile install nixpkgs/d73407e8e6002646acfdef0e39ace088bacc83da#hello # nix profile install nixpkgs/d73407e8e6002646acfdef0e39ace088bacc83da#hello
``` ```
* Install a specific output of a package:
```console
# nix profile install nixpkgs#bash^man
```
# Description # Description
This command adds *installables* to a Nix profile. This command adds *installables* to a Nix profile.

20
src/nix/profile.cc
View file

@ -22,13 +22,13 @@ struct ProfileElementSource
// FIXME: record original attrpath. // FIXME: record original attrpath.
FlakeRef resolvedRef; FlakeRef resolvedRef;
std::string attrPath; std::string attrPath;
// FIXME: output names OutputsSpec outputs;
bool operator < (const ProfileElementSource & other) const bool operator < (const ProfileElementSource & other) const
{ {
return return
std::pair(originalRef.to_string(), attrPath) < std::tuple(originalRef.to_string(), attrPath, outputs) <
std::pair(other.originalRef.to_string(), other.attrPath); std::tuple(other.originalRef.to_string(), other.attrPath, other.outputs);
} }
}; };
@ -42,7 +42,7 @@ struct ProfileElement
std::string describe() const std::string describe() const
{ {
if (source) if (source)
return fmt("%s#%s", source->originalRef, source->attrPath); return fmt("%s#%s%s", source->originalRef, source->attrPath, printOutputsSpec(source->outputs));
StringSet names; StringSet names;
for (auto & path : storePaths) for (auto & path : storePaths)
names.insert(DrvName(path.name()).name); names.insert(DrvName(path.name()).name);
@ -67,7 +67,6 @@ struct ProfileElement
ref<Store> store, ref<Store> store,
const BuiltPaths & builtPaths) const BuiltPaths & builtPaths)
{ {
// FIXME: respect meta.outputsToInstall
storePaths.clear(); storePaths.clear();
for (auto & buildable : builtPaths) { for (auto & buildable : builtPaths) {
std::visit(overloaded { std::visit(overloaded {
@ -121,7 +120,8 @@ struct ProfileManifest
element.source = ProfileElementSource { element.source = ProfileElementSource {
parseFlakeRef(e[sOriginalUrl]), parseFlakeRef(e[sOriginalUrl]),
parseFlakeRef(e[sUrl]), parseFlakeRef(e[sUrl]),
e["attrPath"] e["attrPath"],
e["outputs"].get<OutputsSpec>()
}; };
} }
elements.emplace_back(std::move(element)); elements.emplace_back(std::move(element));
@ -157,6 +157,7 @@ struct ProfileManifest
obj["originalUrl"] = element.source->originalRef.to_string(); obj["originalUrl"] = element.source->originalRef.to_string();
obj["url"] = element.source->resolvedRef.to_string(); obj["url"] = element.source->resolvedRef.to_string();
obj["attrPath"] = element.source->attrPath; obj["attrPath"] = element.source->attrPath;
obj["outputs"] = element.source->outputs;
} }
array.push_back(obj); array.push_back(obj);
} }
@ -288,6 +289,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
installable2->flakeRef, installable2->flakeRef,
resolvedRef, resolvedRef,
attrPath, attrPath,
installable2->outputsSpec
}; };
} }
@ -444,6 +446,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
getEvalState(), getEvalState(),
FlakeRef(element.source->originalRef), FlakeRef(element.source->originalRef),
"", "",
element.source->outputs,
Strings{element.source->attrPath}, Strings{element.source->attrPath},
Strings{}, Strings{},
lockFlags); lockFlags);
@ -459,6 +462,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
installable->flakeRef, installable->flakeRef,
resolvedRef, resolvedRef,
attrPath, attrPath,
installable->outputsSpec
}; };
installables.push_back(installable); installables.push_back(installable);
@ -514,8 +518,8 @@ struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultPro
for (size_t i = 0; i < manifest.elements.size(); ++i) { for (size_t i = 0; i < manifest.elements.size(); ++i) {
auto & element(manifest.elements[i]); auto & element(manifest.elements[i]);
logger->cout("%d %s %s %s", i, logger->cout("%d %s %s %s", i,
element.source ? element.source->originalRef.to_string() + "#" + element.source->attrPath : "-", element.source ? element.source->originalRef.to_string() + "#" + element.source->attrPath + printOutputsSpec(element.source->outputs) : "-",
element.source ? element.source->resolvedRef.to_string() + "#" + element.source->attrPath : "-", element.source ? element.source->resolvedRef.to_string() + "#" + element.source->attrPath + printOutputsSpec(element.source->outputs) : "-",
concatStringsSep(" ", store->printStorePathSet(element.storePaths))); concatStringsSep(" ", store->printStorePathSet(element.storePaths)));
} }
} }

6
src/nix/search.cc
View file

@ -93,10 +93,10 @@ struct CmdSearch : InstallableCommand, MixJSON
}; };
if (cursor.isDerivation()) { if (cursor.isDerivation()) {
DrvName name(cursor.getAttr("name")->getString()); DrvName name(cursor.getAttr(state->sName)->getString());
auto aMeta = cursor.maybeGetAttr("meta"); auto aMeta = cursor.maybeGetAttr(state->sMeta);
auto aDescription = aMeta ? aMeta->maybeGetAttr("description") : nullptr; auto aDescription = aMeta ? aMeta->maybeGetAttr(state->sDescription) : nullptr;
auto description = aDescription ? aDescription->getString() : ""; auto description = aDescription ? aDescription->getString() : "";
std::replace(description.begin(), description.end(), '\n', ' '); std::replace(description.begin(), description.end(), '\n', ' ');
auto attrPath2 = concatStringsSep(".", attrPathS); auto attrPath2 = concatStringsSep(".", attrPathS);

2
src/resolve-system-dependencies/resolve-system-dependencies.cc
View file

@ -176,7 +176,7 @@ int main(int argc, char ** argv)
impurePaths.insert(argv[2]); impurePaths.insert(argv[2]);
else { else {
auto drv = store->derivationFromPath(store->parseStorePath(argv[1])); auto drv = store->derivationFromPath(store->parseStorePath(argv[1]));
impurePaths = tokenizeString<StringSet>(get(drv.env, "__impureHostDeps").value_or("")); impurePaths = tokenizeString<StringSet>(getOr(drv.env, "__impureHostDeps", ""));
impurePaths.insert("/usr/lib/libSystem.dylib"); impurePaths.insert("/usr/lib/libSystem.dylib");
} }

3
tests/build-remote.sh
View file

@ -34,7 +34,7 @@ outPath=$(readlink -f $TEST_ROOT/result)
grep 'FOO BAR BAZ' $TEST_ROOT/machine0/$outPath grep 'FOO BAR BAZ' $TEST_ROOT/machine0/$outPath
testPrintOutPath=$(nix build -L -v -f $file --print-out-paths --max-jobs 0 \ testPrintOutPath=$(nix build -L -v -f $file --no-link --print-out-paths --max-jobs 0 \
--arg busybox $busybox \ --arg busybox $busybox \
--store $TEST_ROOT/machine0 \ --store $TEST_ROOT/machine0 \
--builders "$(join_by '; ' "${builders[@]}")" --builders "$(join_by '; ' "${builders[@]}")"
@ -72,6 +72,7 @@ fi
# Behavior of keep-failed # Behavior of keep-failed
out="$(nix-build 2>&1 failing.nix \ out="$(nix-build 2>&1 failing.nix \
--no-out-link \
--builders "$(join_by '; ' "${builders[@]}")" \ --builders "$(join_by '; ' "${builders[@]}")" \
--keep-failed \ --keep-failed \
--store $TEST_ROOT/machine0 \ --store $TEST_ROOT/machine0 \

50
tests/build.sh
View file

@ -2,15 +2,10 @@ source common.sh
clearStore clearStore
# Make sure that 'nix build' only returns the outputs we asked for. set -o pipefail
nix build -f multiple-outputs.nix --json a --no-link | jq --exit-status '
(.[0] |
(.drvPath | match(".*multiple-outputs-a.drv")) and
(.outputs | keys | length == 1) and
(.outputs.first | match(".*multiple-outputs-a-first")))
'
nix build -f multiple-outputs.nix --json a.all b.all --no-link | jq --exit-status ' # Make sure that 'nix build' returns all outputs by default.
nix build -f multiple-outputs.nix --json a b --no-link | jq --exit-status '
(.[0] | (.[0] |
(.drvPath | match(".*multiple-outputs-a.drv")) and (.drvPath | match(".*multiple-outputs-a.drv")) and
(.outputs | keys | length == 2) and (.outputs | keys | length == 2) and
@ -22,6 +17,45 @@ nix build -f multiple-outputs.nix --json a.all b.all --no-link | jq --exit-statu
(.outputs.out | match(".*multiple-outputs-b"))) (.outputs.out | match(".*multiple-outputs-b")))
' '
# Test output selection using the '^' syntax.
nix build -f multiple-outputs.nix --json a^first --no-link | jq --exit-status '
(.[0] |
(.drvPath | match(".*multiple-outputs-a.drv")) and
(.outputs | keys == ["first"]))
'
nix build -f multiple-outputs.nix --json a^second,first --no-link | jq --exit-status '
(.[0] |
(.drvPath | match(".*multiple-outputs-a.drv")) and
(.outputs | keys == ["first", "second"]))
'
nix build -f multiple-outputs.nix --json 'a^*' --no-link | jq --exit-status '
(.[0] |
(.drvPath | match(".*multiple-outputs-a.drv")) and
(.outputs | keys == ["first", "second"]))
'
# Test that 'outputsToInstall' is respected by default.
nix build -f multiple-outputs.nix --json e --no-link | jq --exit-status '
(.[0] |
(.drvPath | match(".*multiple-outputs-e.drv")) and
(.outputs | keys == ["a", "b"]))
'
# But not when it's overriden.
nix build -f multiple-outputs.nix --json e^a --no-link | jq --exit-status '
(.[0] |
(.drvPath | match(".*multiple-outputs-e.drv")) and
(.outputs | keys == ["a"]))
'
nix build -f multiple-outputs.nix --json 'e^*' --no-link | jq --exit-status '
(.[0] |
(.drvPath | match(".*multiple-outputs-e.drv")) and
(.outputs | keys == ["a", "b", "c"]))
'
testNormalization () { testNormalization () {
clearStore clearStore
outPath=$(nix-build ./simple.nix --no-out-link) outPath=$(nix-build ./simple.nix --no-out-link)

3
tests/ca/substitute.sh
View file

@ -25,7 +25,8 @@ buildDrvs --substitute --substituters $REMOTE_STORE --no-require-sigs -j0 transi
# Check that the thing we’ve just substituted has its realisation stored # Check that the thing we’ve just substituted has its realisation stored
nix realisation info --file ./content-addressed.nix transitivelyDependentCA nix realisation info --file ./content-addressed.nix transitivelyDependentCA
# Check that its dependencies have it too # Check that its dependencies have it too
nix realisation info --file ./content-addressed.nix dependentCA rootCA nix realisation info --file ./content-addressed.nix dependentCA
# nix realisation info --file ./content-addressed.nix rootCA --outputs out
# Same thing, but # Same thing, but
# 1. With non-ca derivations # 1. With non-ca derivations

11
tests/common.sh.in
View file

@ -157,11 +157,12 @@ expect() {
local expected res local expected res
expected="$1" expected="$1"
shift shift
set +e "$@" || res="$?"
"$@" if [[ $res -ne $expected ]]; then
res="$?" echo "Expected '$expected' but got '$res' while running '$*'"
set -e return 1
[[ $res -eq $expected ]] fi
return 0
} }
needLocalStore() { needLocalStore() {

2
tests/fetchClosure.sh
View file

@ -7,7 +7,7 @@ clearStore
clearCacheCache clearCacheCache
# Initialize binary cache. # Initialize binary cache.
nonCaPath=$(nix build --json --file ./dependencies.nix | jq -r .[].outputs.out) nonCaPath=$(nix build --json --file ./dependencies.nix --no-link | jq -r .[].outputs.out)
caPath=$(nix store make-content-addressed --json $nonCaPath | jq -r '.rewrites | map(.) | .[]') caPath=$(nix store make-content-addressed --json $nonCaPath | jq -r '.rewrites | map(.) | .[]')
nix copy --to file://$cacheDir $nonCaPath nix copy --to file://$cacheDir $nonCaPath

8
tests/fetchGit.sh
View file

@ -161,6 +161,14 @@ path4=$(nix eval --impure --raw --expr "(builtins.fetchGit $repo).outPath")
[[ $(cat $path4/hello) = dev ]] [[ $(cat $path4/hello) = dev ]]
[[ $path3 = $path4 ]] [[ $path3 = $path4 ]]
# Using remote path with branch other than 'master' should fetch the HEAD revision.
# (--tarball-ttl 0 to prevent using the cached repo above)
export _NIX_FORCE_HTTP=1
path4=$(nix eval --tarball-ttl 0 --impure --raw --expr "(builtins.fetchGit $repo).outPath")
[[ $(cat $path4/hello) = dev ]]
[[ $path3 = $path4 ]]
unset _NIX_FORCE_HTTP
# Confirm same as 'dev' branch # Confirm same as 'dev' branch
path5=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath") path5=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath")
[[ $path3 = $path5 ]] [[ $path3 = $path5 ]]

9
tests/flakes.sh
View file

@ -31,7 +31,14 @@ flakeFollowsE=$TEST_ROOT/follows/flakeA/flakeE
for repo in $flake1Dir $flake2Dir $flake3Dir $flake7Dir $templatesDir $nonFlakeDir $flakeA $flakeB $flakeFollowsA; do for repo in $flake1Dir $flake2Dir $flake3Dir $flake7Dir $templatesDir $nonFlakeDir $flakeA $flakeB $flakeFollowsA; do
rm -rf $repo $repo.tmp rm -rf $repo $repo.tmp
mkdir -p $repo mkdir -p $repo
git -C $repo init
# Give one repo a non-master initial branch.
extraArgs=
if [[ $repo == $flake2Dir ]]; then
extraArgs="--initial-branch=main"
fi
git -C $repo init $extraArgs
git -C $repo config user.email "foobar@example.com" git -C $repo config user.email "foobar@example.com"
git -C $repo config user.name "Foobar" git -C $repo config user.name "Foobar"
done done

11
tests/lang.sh
View file

@ -4,6 +4,7 @@ export TEST_VAR=foo # for eval-okay-getenv.nix
export NIX_REMOTE=dummy:// export NIX_REMOTE=dummy://
nix-instantiate --eval -E 'builtins.trace "Hello" 123' 2>&1 | grep -q Hello nix-instantiate --eval -E 'builtins.trace "Hello" 123' 2>&1 | grep -q Hello
nix-instantiate --eval -E 'builtins.addErrorContext "Hello" 123' 2>&1
(! nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" 123' 2>&1 | grep -q Hello) (! nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" 123' 2>&1 | grep -q Hello)
nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" (throw "Foo")' 2>&1 | grep -q Hello nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" (throw "Foo")' 2>&1 | grep -q Hello
@ -14,7 +15,7 @@ fail=0
for i in lang/parse-fail-*.nix; do for i in lang/parse-fail-*.nix; do
echo "parsing $i (should fail)"; echo "parsing $i (should fail)";
i=$(basename $i .nix) i=$(basename $i .nix)
if nix-instantiate --parse - < lang/$i.nix; then if ! expect 1 nix-instantiate --parse - < lang/$i.nix; then
echo "FAIL: $i shouldn't parse" echo "FAIL: $i shouldn't parse"
fail=1 fail=1
fi fi
@ -23,7 +24,7 @@ done
for i in lang/parse-okay-*.nix; do for i in lang/parse-okay-*.nix; do
echo "parsing $i (should succeed)"; echo "parsing $i (should succeed)";
i=$(basename $i .nix) i=$(basename $i .nix)
if ! nix-instantiate --parse - < lang/$i.nix > lang/$i.out; then if ! expect 0 nix-instantiate --parse - < lang/$i.nix > lang/$i.out; then
echo "FAIL: $i should parse" echo "FAIL: $i should parse"
fail=1 fail=1
fi fi
@ -32,7 +33,7 @@ done
for i in lang/eval-fail-*.nix; do for i in lang/eval-fail-*.nix; do
echo "evaluating $i (should fail)"; echo "evaluating $i (should fail)";
i=$(basename $i .nix) i=$(basename $i .nix)
if nix-instantiate --eval lang/$i.nix; then if ! expect 1 nix-instantiate --eval lang/$i.nix; then
echo "FAIL: $i shouldn't evaluate" echo "FAIL: $i shouldn't evaluate"
fail=1 fail=1
fi fi
@ -47,7 +48,7 @@ for i in lang/eval-okay-*.nix; do
if test -e lang/$i.flags; then if test -e lang/$i.flags; then
flags=$(cat lang/$i.flags) flags=$(cat lang/$i.flags)
fi fi
if ! NIX_PATH=lang/dir3:lang/dir4 nix-instantiate $flags --eval --strict lang/$i.nix > lang/$i.out; then if ! expect 0 env NIX_PATH=lang/dir3:lang/dir4 nix-instantiate $flags --eval --strict lang/$i.nix > lang/$i.out; then
echo "FAIL: $i should evaluate" echo "FAIL: $i should evaluate"
fail=1 fail=1
elif ! diff lang/$i.out lang/$i.exp; then elif ! diff lang/$i.out lang/$i.exp; then
@ -57,7 +58,7 @@ for i in lang/eval-okay-*.nix; do
fi fi
if test -e lang/$i.exp.xml; then if test -e lang/$i.exp.xml; then
if ! nix-instantiate --eval --xml --no-location --strict \ if ! expect 0 nix-instantiate --eval --xml --no-location --strict \
lang/$i.nix > lang/$i.out.xml; then lang/$i.nix > lang/$i.out.xml; then
echo "FAIL: $i should evaluate" echo "FAIL: $i should evaluate"
fail=1 fail=1

7
tests/multiple-outputs.nix
View file

@ -80,4 +80,11 @@ rec {
''; '';
}).a; }).a;
e = mkDerivation {
name = "multiple-outputs-e";
outputs = [ "a" "b" "c" ];
meta.outputsToInstall = [ "a" "b" ];
buildCommand = "mkdir $a $b $c";
};
} }

29
tests/nix-profile.sh
View file

@ -17,6 +17,7 @@ cat > $flake1Dir/flake.nix <<EOF
outputs = { self }: with import ./config.nix; rec { outputs = { self }: with import ./config.nix; rec {
packages.$system.default = mkDerivation { packages.$system.default = mkDerivation {
name = "profile-test-\${builtins.readFile ./version}"; name = "profile-test-\${builtins.readFile ./version}";
outputs = [ "out" "man" "dev" ];
builder = builtins.toFile "builder.sh" builder = builtins.toFile "builder.sh"
'' ''
mkdir -p \$out/bin mkdir -p \$out/bin
@ -26,10 +27,13 @@ cat > $flake1Dir/flake.nix <<EOF
EOF EOF
chmod +x \$out/bin/hello chmod +x \$out/bin/hello
echo DONE echo DONE
mkdir -p \$man/share/man
mkdir -p \$dev/include
''; '';
__contentAddressed = import ./ca.nix; __contentAddressed = import ./ca.nix;
outputHashMode = "recursive"; outputHashMode = "recursive";
outputHashAlgo = "sha256"; outputHashAlgo = "sha256";
meta.outputsToInstall = [ "out" "man" ];
}; };
}; };
} }
@ -46,6 +50,8 @@ nix-env -f ./user-envs.nix -i foo-1.0
nix profile list | grep '0 - - .*-foo-1.0' nix profile list | grep '0 - - .*-foo-1.0'
nix profile install $flake1Dir -L nix profile install $flake1Dir -L
[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]] [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]]
[ -e $TEST_HOME/.nix-profile/share/man ]
(! [ -e $TEST_HOME/.nix-profile/include ])
nix profile history nix profile history
nix profile history | grep "packages.$system.default: ∅ -> 1.0" nix profile history | grep "packages.$system.default: ∅ -> 1.0"
nix profile diff-closures | grep 'env-manifest.nix: ε → ∅' nix profile diff-closures | grep 'env-manifest.nix: ε → ∅'
@ -55,7 +61,7 @@ printf NixOS > $flake1Dir/who
printf 2.0 > $flake1Dir/version printf 2.0 > $flake1Dir/version
nix profile upgrade 1 nix profile upgrade 1
[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello NixOS" ]] [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello NixOS" ]]
nix profile history | grep "packages.$system.default: 1.0 -> 2.0" nix profile history | grep "packages.$system.default: 1.0, 1.0-man -> 2.0, 2.0-man"
# Test 'history', 'diff-closures'. # Test 'history', 'diff-closures'.
nix profile diff-closures nix profile diff-closures
@ -86,7 +92,7 @@ nix profile wipe-history
printf true > $flake1Dir/ca.nix printf true > $flake1Dir/ca.nix
printf 3.0 > $flake1Dir/version printf 3.0 > $flake1Dir/version
nix profile upgrade 0 nix profile upgrade 0
nix profile history | grep "packages.$system.default: 1.0 -> 3.0" nix profile history | grep "packages.$system.default: 1.0, 1.0-man -> 3.0, 3.0-man"
# Test new install of CA package. # Test new install of CA package.
nix profile remove 0 nix profile remove 0
@ -95,3 +101,22 @@ printf Utrecht > $flake1Dir/who
nix profile install $flake1Dir nix profile install $flake1Dir
[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello Utrecht" ]] [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello Utrecht" ]]
[[ $(nix path-info --json $(realpath $TEST_HOME/.nix-profile/bin/hello) | jq -r .[].ca) =~ fixed:r:sha256: ]] [[ $(nix path-info --json $(realpath $TEST_HOME/.nix-profile/bin/hello) | jq -r .[].ca) =~ fixed:r:sha256: ]]
# Override the outputs.
nix profile remove 0 1
nix profile install "$flake1Dir^*"
[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello Utrecht" ]]
[ -e $TEST_HOME/.nix-profile/share/man ]
[ -e $TEST_HOME/.nix-profile/include ]
printf Nix > $flake1Dir/who
nix profile upgrade 0
[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello Nix" ]]
[ -e $TEST_HOME/.nix-profile/share/man ]
[ -e $TEST_HOME/.nix-profile/include ]
nix profile remove 0
nix profile install "$flake1Dir^man"
(! [ -e $TEST_HOME/.nix-profile/bin/hello ])
[ -e $TEST_HOME/.nix-profile/share/man ]
(! [ -e $TEST_HOME/.nix-profile/include ])

11
tests/shell-hello.nix
View file

@ -3,15 +3,24 @@ with import ./config.nix;
{ {
hello = mkDerivation { hello = mkDerivation {
name = "hello"; name = "hello";
outputs = [ "out" "dev" ];
meta.outputsToInstall = [ "out" ];
buildCommand = buildCommand =
'' ''
mkdir -p $out/bin mkdir -p $out/bin $dev/bin
cat > $out/bin/hello <<EOF cat > $out/bin/hello <<EOF
#! ${shell} #! ${shell}
who=\$1 who=\$1
echo "Hello \''${who:-World} from $out/bin/hello" echo "Hello \''${who:-World} from $out/bin/hello"
EOF EOF
chmod +x $out/bin/hello chmod +x $out/bin/hello
cat > $dev/bin/hello2 <<EOF
#! ${shell}
echo "Hello2"
EOF
chmod +x $dev/bin/hello2
''; '';
}; };
} }

4
tests/shell.sh
View file

@ -6,6 +6,10 @@ clearCache
nix shell -f shell-hello.nix hello -c hello | grep 'Hello World' nix shell -f shell-hello.nix hello -c hello | grep 'Hello World'
nix shell -f shell-hello.nix hello -c hello NixOS | grep 'Hello NixOS' nix shell -f shell-hello.nix hello -c hello NixOS | grep 'Hello NixOS'
# Test output selection.
nix shell -f shell-hello.nix hello^dev -c hello2 | grep 'Hello2'
nix shell -f shell-hello.nix 'hello^*' -c hello2 | grep 'Hello2'
if ! canUseSandbox; then exit 99; fi if ! canUseSandbox; then exit 99; fi
chmod -R u+w $TEST_ROOT/store0 || true chmod -R u+w $TEST_ROOT/store0 || true