Merge remote-tracking branch 'origin/master' into debug-exploratory-PR

This commit is contained in:
Eelco Dolstra 2022-05-04 14:10:21 +02:00
commit c98648bef0
58 changed files with 1041 additions and 330 deletions

View file

@ -14,3 +14,13 @@
* `nix build` has a new `--print-out-paths` flag to print the resulting output paths.
This matches the default behaviour of `nix-build`.
* You can now specify which outputs of a derivation `nix` should
operate on using the syntax `installable^outputs`,
e.g. `nixpkgs#glibc^dev,static` or `nixpkgs#glibc^*`. By default,
`nix` will use the outputs specified by the derivation's
`meta.outputsToInstall` attribute if it exists, or all outputs
otherwise.
Selecting derivation outputs using the attribute selection syntax
(e.g. `nixpkgs#glibc.dev`) no longer works.

View file

@ -440,9 +440,7 @@ DerivedPaths InstallableValue::toDerivedPaths()
// Group by derivation, helps with .all in particular
for (auto & drv : toDerivations()) {
auto outputName = drv.outputName;
if (outputName == "")
throw Error("derivation '%s' lacks an 'outputName' attribute", state->store->printStorePath(drv.drvPath));
for (auto & outputName : drv.outputsToInstall)
drvsToOutputs[drv.drvPath].insert(outputName);
drvsToCopy.insert(drv.drvPath);
}
@ -466,9 +464,19 @@ struct InstallableAttrPath : InstallableValue
SourceExprCommand & cmd;
RootValue v;
std::string attrPath;
OutputsSpec outputsSpec;
InstallableAttrPath(ref<EvalState> state, SourceExprCommand & cmd, Value * v, const std::string & attrPath)
: InstallableValue(state), cmd(cmd), v(allocRootValue(v)), attrPath(attrPath)
InstallableAttrPath(
ref<EvalState> state,
SourceExprCommand & cmd,
Value * v,
const std::string & attrPath,
OutputsSpec outputsSpec)
: InstallableValue(state)
, cmd(cmd)
, v(allocRootValue(v))
, attrPath(attrPath)
, outputsSpec(std::move(outputsSpec))
{ }
std::string what() const override { return attrPath; }
@ -497,7 +505,19 @@ std::vector<InstallableValue::DerivationInfo> InstallableAttrPath::toDerivations
auto drvPath = drvInfo.queryDrvPath();
if (!drvPath)
throw Error("'%s' is not a derivation", what());
res.push_back({ *drvPath, drvInfo.queryOutputName() });
std::set<std::string> outputsToInstall;
if (auto outputNames = std::get_if<OutputNames>(&outputsSpec))
outputsToInstall = *outputNames;
else
for (auto & output : drvInfo.queryOutputs(false, std::get_if<DefaultOutputs>(&outputsSpec)))
outputsToInstall.insert(output.first);
res.push_back(DerivationInfo {
.drvPath = *drvPath,
.outputsToInstall = std::move(outputsToInstall)
});
}
return res;
@ -574,6 +594,7 @@ InstallableFlake::InstallableFlake(
ref<EvalState> state,
FlakeRef && flakeRef,
std::string_view fragment,
OutputsSpec outputsSpec,
Strings attrPaths,
Strings prefixes,
const flake::LockFlags & lockFlags)
@ -581,6 +602,7 @@ InstallableFlake::InstallableFlake(
flakeRef(flakeRef),
attrPaths(fragment == "" ? attrPaths : Strings{(std::string) fragment}),
prefixes(fragment == "" ? Strings{} : prefixes),
outputsSpec(std::move(outputsSpec)),
lockFlags(lockFlags)
{
if (cmd && cmd->getAutoArgs(*state)->size())
@ -598,9 +620,29 @@ std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableF
auto drvPath = attr->forceDerivation();
std::set<std::string> outputsToInstall;
if (auto aMeta = attr->maybeGetAttr(state->sMeta))
if (auto aOutputsToInstall = aMeta->maybeGetAttr("outputsToInstall"))
for (auto & s : aOutputsToInstall->getListOfStrings())
outputsToInstall.insert(s);
if (outputsToInstall.empty() || std::get_if<AllOutputs>(&outputsSpec)) {
outputsToInstall.clear();
if (auto aOutputs = attr->maybeGetAttr(state->sOutputs))
for (auto & s : aOutputs->getListOfStrings())
outputsToInstall.insert(s);
}
if (outputsToInstall.empty())
outputsToInstall.insert("out");
if (auto outputNames = std::get_if<OutputNames>(&outputsSpec))
outputsToInstall = *outputNames;
auto drvInfo = DerivationInfo {
std::move(drvPath),
attr->getAttr(state->sOutputName)->getString()
.drvPath = std::move(drvPath),
.outputsToInstall = std::move(outputsToInstall),
};
return {attrPath, getLockedFlake()->flake.lockedRef, std::move(drvInfo)};
@ -723,8 +765,14 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
state->eval(e, *vFile);
}
for (auto & s : ss)
result.push_back(std::make_shared<InstallableAttrPath>(state, *this, vFile, s == "." ? "" : s));
for (auto & s : ss) {
auto [prefix, outputsSpec] = parseOutputsSpec(s);
result.push_back(
std::make_shared<InstallableAttrPath>(
state, *this, vFile,
prefix == "." ? "" : prefix,
outputsSpec));
}
} else {
@ -743,12 +791,13 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
}
try {
auto [flakeRef, fragment] = parseFlakeRefWithFragment(s, absPath("."));
auto [flakeRef, fragment, outputsSpec] = parseFlakeRefWithFragmentAndOutputsSpec(s, absPath("."));
result.push_back(std::make_shared<InstallableFlake>(
this,
getEvalState(),
std::move(flakeRef),
fragment,
outputsSpec,
getDefaultFlakeAttrPaths(),
getDefaultFlakeAttrPathPrefixes(),
lockFlags));
@ -822,12 +871,13 @@ std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> Installable::bui
auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive
auto drvOutputs = drv.outputsAndOptPaths(*store);
for (auto & output : bfd.outputs) {
if (!outputHashes.count(output))
auto outputHash = get(outputHashes, output);
if (!outputHash)
throw Error(
"the derivation '%s' doesn't have an output named '%s'",
store->printStorePath(bfd.drvPath), output);
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
DrvOutput outputId { outputHashes.at(output), output };
DrvOutput outputId { *outputHash, output };
auto realisation = store->queryRealisation(outputId);
if (!realisation)
throw Error(
@ -838,10 +888,11 @@ std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> Installable::bui
} else {
// If ca-derivations isn't enabled, assume that
// the output path is statically known.
assert(drvOutputs.count(output));
assert(drvOutputs.at(output).second);
auto drvOutput = get(drvOutputs, output);
assert(drvOutput);
assert(drvOutput->second);
outputs.insert_or_assign(
output, *drvOutputs.at(output).second);
output, *drvOutput->second);
}
}
res.push_back({installable, BuiltPath::Built { bfd.drvPath, outputs }});

View file

@ -141,7 +141,7 @@ struct InstallableValue : Installable
struct DerivationInfo
{
StorePath drvPath;
std::string outputName;
std::set<std::string> outputsToInstall;
};
virtual std::vector<DerivationInfo> toDerivations() = 0;
@ -156,6 +156,7 @@ struct InstallableFlake : InstallableValue
FlakeRef flakeRef;
Strings attrPaths;
Strings prefixes;
OutputsSpec outputsSpec;
const flake::LockFlags & lockFlags;
mutable std::shared_ptr<flake::LockedFlake> _lockedFlake;
@ -164,6 +165,7 @@ struct InstallableFlake : InstallableValue
ref<EvalState> state,
FlakeRef && flakeRef,
std::string_view fragment,
OutputsSpec outputsSpec,
Strings attrPaths,
Strings prefixes,
const flake::LockFlags & lockFlags);

View file

@ -47,7 +47,7 @@ struct AttrDb
{
auto state(_state->lock());
Path cacheDir = getCacheDir() + "/nix/eval-cache-v2";
Path cacheDir = getCacheDir() + "/nix/eval-cache-v3";
createDirs(cacheDir);
Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite";
@ -175,6 +175,24 @@ struct AttrDb
});
}
AttrId setListOfStrings(
AttrKey key,
const std::vector<std::string> & l)
{
return doSQLite([&]()
{
auto state(_state->lock());
state->insertAttribute.use()
(key.first)
(symbols[key.second])
(AttrType::ListOfStrings)
(concatStringsSep("\t", l)).exec();
return state->db.getLastInsertedRowId();
});
}
AttrId setPlaceholder(AttrKey key)
{
return doSQLite([&]()
@ -269,6 +287,8 @@ struct AttrDb
}
case AttrType::Bool:
return {{rowId, queryAttribute.getInt(2) != 0}};
case AttrType::ListOfStrings:
return {{rowId, tokenizeString<std::vector<std::string>>(queryAttribute.getStr(2), "\t")}};
case AttrType::Missing:
return {{rowId, missing_t()}};
case AttrType::Misc:
@ -385,7 +405,7 @@ std::string AttrCursor::getAttrPathStr(Symbol name) const
Value & AttrCursor::forceValue()
{
debug("evaluating uncached attribute %s", getAttrPathStr());
debug("evaluating uncached attribute '%s'", getAttrPathStr());
auto & v = getValue();
@ -626,6 +646,38 @@ bool AttrCursor::getBool()
return v.boolean;
}
std::vector<std::string> AttrCursor::getListOfStrings()
{
if (root->db) {
if (!cachedValue)
cachedValue = root->db->getAttr(getKey());
if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
if (auto l = std::get_if<std::vector<std::string>>(&cachedValue->second)) {
debug("using cached list of strings attribute '%s'", getAttrPathStr());
return *l;
} else
throw TypeError("'%s' is not a list of strings", getAttrPathStr());
}
}
debug("evaluating uncached attribute '%s'", getAttrPathStr());
auto & v = getValue();
root->state.forceValue(v, noPos);
if (v.type() != nList)
throw TypeError("'%s' is not a list", getAttrPathStr());
std::vector<std::string> res;
for (auto & elem : v.listItems())
res.push_back(std::string(root->state.forceStringNoCtx(*elem)));
cachedValue = {root->db->setListOfStrings(getKey(), res), res};
return res;
}
std::vector<Symbol> AttrCursor::getAttrs()
{
if (root->db) {

View file

@ -44,6 +44,7 @@ enum AttrType {
Misc = 4,
Failed = 5,
Bool = 6,
ListOfStrings = 7,
};
struct placeholder_t {};
@ -61,7 +62,8 @@ typedef std::variant<
missing_t,
misc_t,
failed_t,
bool
bool,
std::vector<std::string>
> AttrValue;
class AttrCursor : public std::enable_shared_from_this<AttrCursor>
@ -114,6 +116,8 @@ public:
bool getBool();
std::vector<std::string> getListOfStrings();
std::vector<Symbol> getAttrs();
bool isDerivation();

View file

@ -50,13 +50,11 @@ void ConfigFile::apply()
else
assert(false);
if (!whitelist.count(baseName)) {
auto trustedList = readTrustedList();
if (!whitelist.count(baseName) && !nix::fetchSettings.acceptFlakeConfig) {
bool trusted = false;
if (nix::fetchSettings.acceptFlakeConfig){
trusted = true;
} else if (auto saved = get(get(trustedList, name).value_or(std::map<std::string, bool>()), valueS)) {
auto trustedList = readTrustedList();
auto tlname = get(trustedList, name);
if (auto saved = tlname ? get(*tlname, valueS) : nullptr) {
trusted = *saved;
warn("Using saved setting for '%s = %s' from ~/.local/share/nix/trusted-settings.json.", name,valueS);
} else {
@ -69,7 +67,6 @@ void ConfigFile::apply()
writeTrustedList(trustedList);
}
}
if (!trusted) {
warn("ignoring untrusted flake configuration setting '%s'", name);
continue;

View file

@ -176,7 +176,7 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
parsedURL.query.insert_or_assign("shallow", "1");
return std::make_pair(
FlakeRef(Input::fromURL(parsedURL), get(parsedURL.query, "dir").value_or("")),
FlakeRef(Input::fromURL(parsedURL), getOr(parsedURL.query, "dir", "")),
fragment);
}
@ -189,7 +189,7 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
if (!hasPrefix(path, "/"))
throw BadURL("flake reference '%s' is not an absolute path", url);
auto query = decodeQuery(match[2]);
path = canonPath(path + "/" + get(query, "dir").value_or(""));
path = canonPath(path + "/" + getOr(query, "dir", ""));
}
fetchers::Attrs attrs;
@ -208,7 +208,7 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
input.parent = baseDir;
return std::make_pair(
FlakeRef(std::move(input), get(parsedURL.query, "dir").value_or("")),
FlakeRef(std::move(input), getOr(parsedURL.query, "dir", "")),
fragment);
}
}
@ -238,4 +238,15 @@ std::pair<fetchers::Tree, FlakeRef> FlakeRef::fetchTree(ref<Store> store) const
return {std::move(tree), FlakeRef(std::move(lockedInput), subdir)};
}
std::tuple<FlakeRef, std::string, OutputsSpec> parseFlakeRefWithFragmentAndOutputsSpec(
const std::string & url,
const std::optional<Path> & baseDir,
bool allowMissing,
bool isFlake)
{
auto [prefix, outputsSpec] = parseOutputsSpec(url);
auto [flakeRef, fragment] = parseFlakeRefWithFragment(prefix, baseDir, allowMissing, isFlake);
return {std::move(flakeRef), fragment, outputsSpec};
}
}

View file

@ -3,6 +3,7 @@
#include "types.hh"
#include "hash.hh"
#include "fetchers.hh"
#include "path-with-outputs.hh"
#include <variant>
@ -79,4 +80,11 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
std::optional<std::pair<FlakeRef, std::string>> maybeParseFlakeRefWithFragment(
const std::string & url, const std::optional<Path> & baseDir = {});
std::tuple<FlakeRef, std::string, OutputsSpec> parseFlakeRefWithFragmentAndOutputsSpec(
const std::string & url,
const std::optional<Path> & baseDir = {},
bool allowMissing = false,
bool isFlake = true);
}

View file

@ -34,7 +34,7 @@ DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPat
outputName =
selectedOutputs.empty()
? get(drv.env, "outputName").value_or("out")
? getOr(drv.env, "outputName", "out")
: *selectedOutputs.begin();
auto i = drv.outputs.find(outputName);

View file

@ -76,10 +76,10 @@ StringMap EvalState::realiseContext(const PathSet & context)
/* Get all the output paths corresponding to the placeholders we had */
for (auto & [drvPath, outputs] : drvs) {
auto outputPaths = store->queryDerivationOutputMap(drvPath);
const auto outputPaths = store->queryDerivationOutputMap(drvPath);
for (auto & outputName : outputs) {
if (outputPaths.count(outputName) == 0)
{
auto outputPath = get(outputPaths, outputName);
if (!outputPath) {
auto e = Error("derivation '%s' does not have an output named '%s'",
store->printStorePath(drvPath), outputName);
debugLastTrace(e);
@ -87,7 +87,7 @@ StringMap EvalState::realiseContext(const PathSet & context)
}
res.insert_or_assign(
downstreamPlaceholder(*store, drvPath, outputName),
store->printStorePath(outputPaths.at(outputName))
store->printStorePath(*outputPath)
);
}
}
@ -1369,8 +1369,13 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * *
switch (hashModulo.kind) {
case DrvHash::Kind::Regular:
for (auto & i : outputs) {
auto h = hashModulo.hashes.at(i);
auto outPath = state.store->makeOutputPath(i, h, drvName);
auto h = get(hashModulo.hashes, i);
if (!h)
throw AssertionError({
.msg = hintfmt("derivation produced no hash for output '%s'", i),
.errPos = state.positions[posDrvName],
});
auto outPath = state.store->makeOutputPath(i, *h, drvName);
drv.env[i] = state.store->printStorePath(outPath);
drv.outputs.insert_or_assign(
i,

View file

@ -0,0 +1,27 @@
#include "git-utils.hh"
#include <regex>
std::optional<std::string> parseListReferenceHeadRef(std::string_view line)
{
const static std::regex head_ref_regex("^ref: ([^\\s]+)\\t+HEAD$");
std::match_results<std::string_view::const_iterator> match;
if (std::regex_match(line.cbegin(), line.cend(), match, head_ref_regex)) {
return match[1];
} else {
return std::nullopt;
}
}
std::optional<std::string> parseListReferenceForRev(std::string_view rev, std::string_view line)
{
const static std::regex rev_regex("^([^\\t]+)\\t+(.*)$");
std::match_results<std::string_view::const_iterator> match;
if (!std::regex_match(line.cbegin(), line.cend(), match, rev_regex)) {
return std::nullopt;
}
if (rev != match[2].str()) {
return std::nullopt;
}
return match[1];
}

View file

@ -0,0 +1,23 @@
#pragma once
#include <string>
#include <string_view>
#include <optional>
// Parses the HEAD ref as reported by `git ls-remote --symref`
//
// Returns the head branch name as reported by `git ls-remote --symref`, e.g., if
// ls-remote returns the output below, "main" is returned based on the ref line.
//
// ref: refs/heads/main HEAD
//
// If the repository is in 'detached head' state (HEAD is pointing to a rev
// instead of a branch), parseListReferenceForRev("HEAD") may be used instead.
std::optional<std::string> parseListReferenceHeadRef(std::string_view line);
// Parses a reference line from `git ls-remote --symref`, e.g.,
// parseListReferenceForRev("refs/heads/master", line) will return 6926...
// given the line below.
//
// 6926beab444c33fb57b21819b6642d032016bb1e refs/heads/master
std::optional<std::string> parseListReferenceForRev(std::string_view rev, std::string_view line);

View file

@ -5,15 +5,20 @@
#include "store-api.hh"
#include "url-parts.hh"
#include "pathlocks.hh"
#include "util.hh"
#include "git-utils.hh"
#include "fetch-settings.hh"
#include <regex>
#include <string.h>
#include <sys/time.h>
#include <sys/wait.h>
using namespace std::string_literals;
namespace nix::fetchers {
namespace {
// Explicit initial branch of our bare repo to suppress warnings from new version of git.
// The value itself does not matter, since we always fetch a specific revision or branch.
@ -21,25 +26,228 @@ namespace nix::fetchers {
// old version of git, which will ignore unrecognized `-c` options.
const std::string gitInitialBranch = "__nix_dummy_branch";
static std::string getGitDir()
std::string getGitDir()
{
auto gitDir = getEnv("GIT_DIR");
if (!gitDir) {
return ".git";
return getEnv("GIT_DIR").value_or(".git");
}
bool isCacheFileWithinTtl(const time_t now, const struct stat & st)
{
return st.st_mtime + settings.tarballTtl > now;
}
bool touchCacheFile(const Path& path, const time_t& touch_time)
{
struct timeval times[2];
times[0].tv_sec = touch_time;
times[0].tv_usec = 0;
times[1].tv_sec = touch_time;
times[1].tv_usec = 0;
return lutimes(path.c_str(), times) == 0;
}
Path getCachePath(std::string key)
{
return getCacheDir() + "/nix/gitv3/" +
hashString(htSHA256, key).to_string(Base32, false);
}
// Returns the name of the HEAD branch.
//
// Returns the head branch name as reported by git ls-remote --symref, e.g., if
// ls-remote returns the output below, "main" is returned based on the ref line.
//
// ref: refs/heads/main HEAD
// ...
std::optional<std::string> readHead(const Path & path)
{
auto [exit_code, output] = runProgram(RunOptions {
.program = "git",
.args = {"ls-remote", "--symref", path},
});
if (exit_code != 0) {
return std::nullopt;
}
return *gitDir;
std::string_view line = output;
line = line.substr(0, line.find("\n"));
if (const auto ref = parseListReferenceHeadRef(line); ref) {
debug("resolved HEAD ref '%s' for repo '%s'", *ref, path);
return *ref;
}
if (const auto rev = parseListReferenceForRev("HEAD", line); rev) {
debug("resolved HEAD rev '%s' for repo '%s'", *rev, path);
return *rev;
}
return std::nullopt;
}
static std::string readHead(const Path & path)
// Persist the HEAD ref from the remote repo in the local cached repo.
bool storeCachedHead(const std::string& actualUrl, const std::string& headRef)
{
return chomp(runProgram("git", true, { "-C", path, "--git-dir", ".git", "rev-parse", "--abbrev-ref", "HEAD" }));
Path cacheDir = getCachePath(actualUrl);
try {
runProgram("git", true, { "-C", cacheDir, "symbolic-ref", "--", "HEAD", headRef });
} catch (ExecError &e) {
if (!WIFEXITED(e.status)) throw;
return false;
}
/* No need to touch refs/HEAD, because `git symbolic-ref` updates the mtime. */
return true;
}
static bool isNotDotGitDirectory(const Path & path)
std::optional<std::string> readHeadCached(const std::string& actualUrl)
{
// Create a cache path to store the branch of the HEAD ref. Append something
// in front of the URL to prevent collision with the repository itself.
Path cacheDir = getCachePath(actualUrl);
Path headRefFile = cacheDir + "/HEAD";
time_t now = time(0);
struct stat st;
std::optional<std::string> cachedRef;
if (stat(headRefFile.c_str(), &st) == 0) {
cachedRef = readHead(cacheDir);
if (cachedRef != std::nullopt &&
*cachedRef != gitInitialBranch &&
isCacheFileWithinTtl(now, st)) {
debug("using cached HEAD ref '%s' for repo '%s'", *cachedRef, actualUrl);
return cachedRef;
}
}
auto ref = readHead(actualUrl);
if (ref) {
return ref;
}
if (cachedRef) {
// If the cached git ref is expired in fetch() below, and the 'git fetch'
// fails, it falls back to continuing with the most recent version.
// This function must behave the same way, so we return the expired
// cached ref here.
warn("could not get HEAD ref for repository '%s'; using expired cached ref '%s'", actualUrl, *cachedRef);
return *cachedRef;
}
return std::nullopt;
}
bool isNotDotGitDirectory(const Path & path)
{
return baseNameOf(path) != ".git";
}
struct WorkdirInfo
{
bool clean = false;
bool hasHead = false;
};
// Returns whether a git workdir is clean and has commits.
WorkdirInfo getWorkdirInfo(const Input & input, const Path & workdir)
{
const bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
auto gitDir = getGitDir();
auto env = getEnv();
// Set LC_ALL to C: because we rely on the error messages from git rev-parse to determine what went wrong
// that way unknown errors can lead to a failure instead of continuing through the wrong code path
env["LC_ALL"] = "C";
/* Check whether HEAD points to something that looks like a commit,
since that is the refrence we want to use later on. */
auto result = runProgram(RunOptions {
.program = "git",
.args = { "-C", workdir, "--git-dir", gitDir, "rev-parse", "--verify", "--no-revs", "HEAD^{commit}" },
.environment = env,
.mergeStderrToStdout = true
});
auto exitCode = WEXITSTATUS(result.first);
auto errorMessage = result.second;
if (errorMessage.find("fatal: not a git repository") != std::string::npos) {
throw Error("'%s' is not a Git repository", workdir);
} else if (errorMessage.find("fatal: Needed a single revision") != std::string::npos) {
// indicates that the repo does not have any commits
// we want to proceed and will consider it dirty later
} else if (exitCode != 0) {
// any other errors should lead to a failure
throw Error("getting the HEAD of the Git tree '%s' failed with exit code %d:\n%s", workdir, exitCode, errorMessage);
}
bool clean = false;
bool hasHead = exitCode == 0;
try {
if (hasHead) {
// Using git diff is preferrable over lower-level operations here,
// because its conceptually simpler and we only need the exit code anyways.
auto gitDiffOpts = Strings({ "-C", workdir, "diff", "HEAD", "--quiet"});
if (!submodules) {
// Changes in submodules should only make the tree dirty
// when those submodules will be copied as well.
gitDiffOpts.emplace_back("--ignore-submodules");
}
gitDiffOpts.emplace_back("--");
runProgram("git", true, gitDiffOpts);
clean = true;
}
} catch (ExecError & e) {
if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw;
}
return WorkdirInfo { .clean = clean, .hasHead = hasHead };
}
std::pair<StorePath, Input> fetchFromWorkdir(ref<Store> store, Input & input, const Path & workdir, const WorkdirInfo & workdirInfo)
{
const bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
if (!fetchSettings.allowDirty)
throw Error("Git tree '%s' is dirty", workdir);
if (fetchSettings.warnDirty)
warn("Git tree '%s' is dirty", workdir);
auto gitOpts = Strings({ "-C", workdir, "ls-files", "-z" });
if (submodules)
gitOpts.emplace_back("--recurse-submodules");
auto files = tokenizeString<std::set<std::string>>(
runProgram("git", true, gitOpts), "\0"s);
Path actualPath(absPath(workdir));
PathFilter filter = [&](const Path & p) -> bool {
assert(hasPrefix(p, actualPath));
std::string file(p, actualPath.size() + 1);
auto st = lstat(p);
if (S_ISDIR(st.st_mode)) {
auto prefix = file + "/";
auto i = files.lower_bound(prefix);
return i != files.end() && hasPrefix(*i, prefix);
}
return files.count(file);
};
auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, htSHA256, filter);
// FIXME: maybe we should use the timestamp of the last
// modified dirty file?
input.attrs.insert_or_assign(
"lastModified",
workdirInfo.hasHead ? std::stoull(runProgram("git", true, { "-C", actualPath, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0);
return {std::move(storePath), input};
}
} // end namespace
struct GitInputScheme : InputScheme
{
std::optional<Input> inputFromURL(const ParsedURL & url) override
@ -234,106 +442,16 @@ struct GitInputScheme : InputScheme
auto [isLocal, actualUrl_] = getActualUrl(input);
auto actualUrl = actualUrl_; // work around clang bug
// If this is a local directory and no ref or revision is
// given, then allow the use of an unclean working tree.
/* If this is a local directory and no ref or revision is given,
allow fetching directly from a dirty workdir. */
if (!input.getRef() && !input.getRev() && isLocal) {
bool clean = false;
auto env = getEnv();
// Set LC_ALL to C: because we rely on the error messages from git rev-parse to determine what went wrong
// that way unknown errors can lead to a failure instead of continuing through the wrong code path
env["LC_ALL"] = "C";
/* Check whether HEAD points to something that looks like a commit,
since that is the refrence we want to use later on. */
auto result = runProgram(RunOptions {
.program = "git",
.args = { "-C", actualUrl, "--git-dir", gitDir, "rev-parse", "--verify", "--no-revs", "HEAD^{commit}" },
.environment = env,
.mergeStderrToStdout = true
});
auto exitCode = WEXITSTATUS(result.first);
auto errorMessage = result.second;
if (errorMessage.find("fatal: not a git repository") != std::string::npos) {
throw Error("'%s' is not a Git repository", actualUrl);
} else if (errorMessage.find("fatal: Needed a single revision") != std::string::npos) {
// indicates that the repo does not have any commits
// we want to proceed and will consider it dirty later
} else if (exitCode != 0) {
// any other errors should lead to a failure
throw Error("getting the HEAD of the Git tree '%s' failed with exit code %d:\n%s", actualUrl, exitCode, errorMessage);
}
bool hasHead = exitCode == 0;
try {
if (hasHead) {
// Using git diff is preferrable over lower-level operations here,
// because its conceptually simpler and we only need the exit code anyways.
auto gitDiffOpts = Strings({ "-C", actualUrl, "--git-dir", gitDir, "diff", "HEAD", "--quiet"});
if (!submodules) {
// Changes in submodules should only make the tree dirty
// when those submodules will be copied as well.
gitDiffOpts.emplace_back("--ignore-submodules");
}
gitDiffOpts.emplace_back("--");
runProgram("git", true, gitDiffOpts);
clean = true;
}
} catch (ExecError & e) {
if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw;
}
if (!clean) {
/* This is an unclean working tree. So copy all tracked files. */
if (!fetchSettings.allowDirty)
throw Error("Git tree '%s' is dirty", actualUrl);
if (fetchSettings.warnDirty)
warn("Git tree '%s' is dirty", actualUrl);
auto gitOpts = Strings({ "-C", actualUrl, "--git-dir", gitDir, "ls-files", "-z" });
if (submodules)
gitOpts.emplace_back("--recurse-submodules");
auto files = tokenizeString<std::set<std::string>>(
runProgram("git", true, gitOpts), "\0"s);
Path actualPath(absPath(actualUrl));
PathFilter filter = [&](const Path & p) -> bool {
assert(hasPrefix(p, actualPath));
std::string file(p, actualPath.size() + 1);
auto st = lstat(p);
if (S_ISDIR(st.st_mode)) {
auto prefix = file + "/";
auto i = files.lower_bound(prefix);
return i != files.end() && hasPrefix(*i, prefix);
}
return files.count(file);
};
auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, htSHA256, filter);
// FIXME: maybe we should use the timestamp of the last
// modified dirty file?
input.attrs.insert_or_assign(
"lastModified",
hasHead ? std::stoull(runProgram("git", true, { "-C", actualPath, "--git-dir", gitDir, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0);
return {std::move(storePath), input};
auto workdirInfo = getWorkdirInfo(input, actualUrl);
if (!workdirInfo.clean) {
return fetchFromWorkdir(store, input, actualUrl, workdirInfo);
}
}
if (!input.getRef()) input.attrs.insert_or_assign("ref", isLocal ? readHead(actualUrl) : "master");
Attrs unlockedAttrs({
const Attrs unlockedAttrs({
{"type", cacheType},
{"name", name},
{"url", actualUrl},
@ -343,14 +461,30 @@ struct GitInputScheme : InputScheme
Path repoDir;
if (isLocal) {
if (!input.getRef()) {
auto head = readHead(actualUrl);
if (!head) {
warn("could not read HEAD ref from repo at '%s', using 'master'", actualUrl);
head = "master";
}
input.attrs.insert_or_assign("ref", *head);
}
if (!input.getRev())
input.attrs.insert_or_assign("rev",
Hash::parseAny(chomp(runProgram("git", true, { "-C", actualUrl, "--git-dir", gitDir, "rev-parse", *input.getRef() })), htSHA1).gitRev());
repoDir = actualUrl;
} else {
const bool useHeadRef = !input.getRef();
if (useHeadRef) {
auto head = readHeadCached(actualUrl);
if (!head) {
warn("could not read HEAD ref from repo at '%s', using 'master'", actualUrl);
head = "master";
}
input.attrs.insert_or_assign("ref", *head);
}
if (auto res = getCache()->lookup(store, unlockedAttrs)) {
auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), htSHA1);
@ -360,7 +494,7 @@ struct GitInputScheme : InputScheme
}
}
Path cacheDir = getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, actualUrl).to_string(Base32, false);
Path cacheDir = getCachePath(actualUrl);
repoDir = cacheDir;
gitDir = ".";
@ -383,7 +517,7 @@ struct GitInputScheme : InputScheme
repo. */
if (input.getRev()) {
try {
runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input.getRev()->gitRev() });
runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "cat-file", "-e", input.getRev()->gitRev() });
doFetch = false;
} catch (ExecError & e) {
if (WIFEXITED(e.status)) {
@ -400,7 +534,7 @@ struct GitInputScheme : InputScheme
git fetch to update the local ref to the remote ref. */
struct stat st;
doFetch = stat(localRefFile.c_str(), &st) != 0 ||
(uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now;
!isCacheFileWithinTtl(now, st);
}
}
@ -418,19 +552,16 @@ struct GitInputScheme : InputScheme
: ref == "HEAD"
? *ref
: "refs/heads/" + *ref;
runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) });
runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) });
} catch (Error & e) {
if (!pathExists(localRefFile)) throw;
warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl);
}
struct timeval times[2];
times[0].tv_sec = now;
times[0].tv_usec = 0;
times[1].tv_sec = now;
times[1].tv_usec = 0;
utimes(localRefFile.c_str(), times);
if (!touchCacheFile(localRefFile, now))
warn("could not update mtime for file '%s': %s", localRefFile, strerror(errno));
if (useHeadRef && !storeCachedHead(actualUrl, *input.getRef()))
warn("could not update cached head '%s' for '%s'", *input.getRef(), actualUrl);
}
if (!input.getRev())
@ -459,7 +590,7 @@ struct GitInputScheme : InputScheme
auto result = runProgram(RunOptions {
.program = "git",
.args = { "-C", repoDir, "cat-file", "commit", input.getRev()->gitRev() },
.args = { "-C", repoDir, "--git-dir", gitDir, "cat-file", "commit", input.getRev()->gitRev() },
.mergeStderrToStdout = true
});
if (WEXITSTATUS(result.first) == 128
@ -498,7 +629,7 @@ struct GitInputScheme : InputScheme
auto source = sinkToSource([&](Sink & sink) {
runProgram2({
.program = "git",
.args = { "-C", repoDir, "archive", input.getRev()->gitRev() },
.args = { "-C", repoDir, "--git-dir", gitDir, "archive", input.getRev()->gitRev() },
.standardOut = &sink
});
});
@ -508,7 +639,7 @@ struct GitInputScheme : InputScheme
auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter);
auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", "--no-show-signature", input.getRev()->gitRev() }));
auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "log", "-1", "--format=%ct", "--no-show-signature", input.getRev()->gitRev() }));
Attrs infoAttrs({
{"rev", input.getRev()->gitRev()},
@ -517,7 +648,7 @@ struct GitInputScheme : InputScheme
if (!shallow)
infoAttrs.insert_or_assign("revCount",
std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input.getRev()->gitRev() })));
std::stoull(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "rev-list", "--count", input.getRev()->gitRev() })));
if (!_input.getRev())
getCache()->add(

View file

@ -4,7 +4,7 @@
#include "store-api.hh"
#include "types.hh"
#include "url-parts.hh"
#include "git-utils.hh"
#include "fetchers.hh"
#include "fetch-settings.hh"
@ -383,35 +383,29 @@ struct SourceHutInputScheme : GitArchiveInputScheme
std::string line;
getline(is, line);
auto ref_index = line.find("ref: ");
if (ref_index == std::string::npos) {
auto r = parseListReferenceHeadRef(line);
if (!r) {
throw BadURL("in '%d', couldn't resolve HEAD ref '%d'", input.to_string(), ref);
}
ref_uri = line.substr(ref_index+5, line.length()-1);
} else
ref_uri = *r;
} else {
ref_uri = fmt("refs/(heads|tags)/%s", ref);
}
auto file = store->toRealPath(
downloadFile(store, fmt("%s/info/refs", base_url), "source", false, headers).storePath);
std::ifstream is(file);
std::string line;
std::string id;
while(getline(is, line)) {
// Append $ to avoid partial name matches
std::regex pattern(fmt("%s$", ref_uri));
if (std::regex_search(line, pattern)) {
id = line.substr(0, line.find('\t'));
break;
}
std::optional<std::string> id;
while(!id && getline(is, line)) {
id = parseListReferenceForRev(ref_uri, line);
}
if(id.empty())
if(!id)
throw BadURL("in '%d', couldn't find ref '%d'", input.to_string(), ref);
auto rev = Hash::parseAny(id, htSHA1);
auto rev = Hash::parseAny(*id, htSHA1);
debug("HEAD revision for '%s' is %s", fmt("%s/%s", base_url, ref), rev.gitRev());
return rev;
}

View file

@ -984,21 +984,28 @@ void DerivationGoal::resolvedFinished()
realWantedOutputs = resolvedDrv.outputNames();
for (auto & wantedOutput : realWantedOutputs) {
assert(initialOutputs.count(wantedOutput) != 0);
assert(resolvedHashes.count(wantedOutput) != 0);
auto realisation = resolvedResult.builtOutputs.at(
DrvOutput { resolvedHashes.at(wantedOutput), wantedOutput });
auto initialOutput = get(initialOutputs, wantedOutput);
auto resolvedHash = get(resolvedHashes, wantedOutput);
if ((!initialOutput) || (!resolvedHash))
throw Error(
"derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,resolve)",
worker.store.printStorePath(drvPath), wantedOutput);
auto realisation = get(resolvedResult.builtOutputs, DrvOutput { *resolvedHash, wantedOutput });
if (!realisation)
throw Error(
"derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,realisation)",
worker.store.printStorePath(resolvedDrvGoal->drvPath), wantedOutput);
if (drv->type().isPure()) {
auto newRealisation = realisation;
newRealisation.id = DrvOutput { initialOutputs.at(wantedOutput).outputHash, wantedOutput };
auto newRealisation = *realisation;
newRealisation.id = DrvOutput { initialOutput->outputHash, wantedOutput };
newRealisation.signatures.clear();
if (!drv->type().isFixed())
newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation.outPath);
newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation->outPath);
signRealisation(newRealisation);
worker.store.registerDrvOutput(newRealisation);
}
outputPaths.insert(realisation.outPath);
builtOutputs.emplace(realisation.id, realisation);
outputPaths.insert(realisation->outPath);
builtOutputs.emplace(realisation->id, *realisation);
}
runPostBuildHook(
@ -1294,7 +1301,11 @@ std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity()
DrvOutputs validOutputs;
for (auto & i : queryPartialDerivationOutputMap()) {
InitialOutput & info = initialOutputs.at(i.first);
auto initialOutput = get(initialOutputs, i.first);
if (!initialOutput)
// this is an invalid output, gets catched with (!wantedOutputsLeft.empty())
continue;
auto & info = *initialOutput;
info.wanted = wantOutput(i.first, wantedOutputs);
if (info.wanted)
wantedOutputsLeft.erase(i.first);
@ -1309,7 +1320,7 @@ std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity()
: PathStatus::Corrupt,
};
}
auto drvOutput = DrvOutput{initialOutputs.at(i.first).outputHash, i.first};
auto drvOutput = DrvOutput{info.outputHash, i.first};
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
if (auto real = worker.store.queryRealisation(drvOutput)) {
info.known = {

View file

@ -14,6 +14,7 @@
#include "worker-protocol.hh"
#include "topo-sort.hh"
#include "callback.hh"
#include "json-utils.hh"
#include <regex>
#include <queue>
@ -56,8 +57,6 @@
#include <pwd.h>
#include <grp.h>
#include <nlohmann/json.hpp>
namespace nix {
void handleDiffHook(
@ -482,7 +481,7 @@ void LocalDerivationGoal::startBuilder()
temporary build directory. The text files have the format used
by `nix-store --register-validity'. However, the deriver
fields are left empty. */
auto s = get(drv->env, "exportReferencesGraph").value_or("");
auto s = getOr(drv->env, "exportReferencesGraph", "");
Strings ss = tokenizeString<Strings>(s);
if (ss.size() % 2 != 0)
throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s);
@ -989,7 +988,7 @@ void LocalDerivationGoal::initTmpDir() {
there is no size constraint). */
if (!parsedDrv->getStructuredAttrs()) {
StringSet passAsFile = tokenizeString<StringSet>(get(drv->env, "passAsFile").value_or(""));
StringSet passAsFile = tokenizeString<StringSet>(getOr(drv->env, "passAsFile", ""));
for (auto & i : drv->env) {
if (passAsFile.find(i.first) == passAsFile.end()) {
env[i.first] = i.second;
@ -2128,12 +2127,22 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
std::map<std::string, std::variant<AlreadyRegistered, PerhapsNeedToRegister>> outputReferencesIfUnregistered;
std::map<std::string, struct stat> outputStats;
for (auto & [outputName, _] : drv->outputs) {
auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchOutputs.at(outputName)));
auto scratchOutput = get(scratchOutputs, outputName);
if (!scratchOutput)
throw BuildError(
"builder for '%s' has no scratch output for '%s'",
worker.store.printStorePath(drvPath), outputName);
auto actualPath = toRealPathChroot(worker.store.printStorePath(*scratchOutput));
outputsToSort.insert(outputName);
/* Updated wanted info to remove the outputs we definitely don't need to register */
auto & initialInfo = initialOutputs.at(outputName);
auto initialOutput = get(initialOutputs, outputName);
if (!initialOutput)
throw BuildError(
"builder for '%s' has no initial output for '%s'",
worker.store.printStorePath(drvPath), outputName);
auto & initialInfo = *initialOutput;
/* Don't register if already valid, and not checking */
initialInfo.wanted = buildMode == bmCheck
@ -2185,6 +2194,11 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
auto sortedOutputNames = topoSort(outputsToSort,
{[&](const std::string & name) {
auto orifu = get(outputReferencesIfUnregistered, name);
if (!orifu)
throw BuildError(
"no output reference for '%s' in build of '%s'",
name, worker.store.printStorePath(drvPath));
return std::visit(overloaded {
/* Since we'll use the already installed versions of these, we
can treat them as leaves and ignore any references they
@ -2199,7 +2213,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
referencedOutputs.insert(o);
return referencedOutputs;
},
}, outputReferencesIfUnregistered.at(name));
}, *orifu);
}},
{[&](const std::string & path, const std::string & parent) {
// TODO with more -vvvv also show the temporary paths for manual inspection.
@ -2213,9 +2227,10 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
OutputPathMap finalOutputs;
for (auto & outputName : sortedOutputNames) {
auto output = drv->outputs.at(outputName);
auto & scratchPath = scratchOutputs.at(outputName);
auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchPath));
auto output = get(drv->outputs, outputName);
auto scratchPath = get(scratchOutputs, outputName);
assert(output && scratchPath);
auto actualPath = toRealPathChroot(worker.store.printStorePath(*scratchPath));
auto finish = [&](StorePath finalStorePath) {
/* Store the final path */
@ -2223,10 +2238,13 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
/* The rewrite rule will be used in downstream outputs that refer to
use. This is why the topological sort is essential to do first
before this for loop. */
if (scratchPath != finalStorePath)
outputRewrites[std::string { scratchPath.hashPart() }] = std::string { finalStorePath.hashPart() };
if (*scratchPath != finalStorePath)
outputRewrites[std::string { scratchPath->hashPart() }] = std::string { finalStorePath.hashPart() };
};
auto orifu = get(outputReferencesIfUnregistered, outputName);
assert(orifu);
std::optional<StorePathSet> referencesOpt = std::visit(overloaded {
[&](const AlreadyRegistered & skippedFinalPath) -> std::optional<StorePathSet> {
finish(skippedFinalPath.path);
@ -2235,7 +2253,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
[&](const PerhapsNeedToRegister & r) -> std::optional<StorePathSet> {
return r.refs;
},
}, outputReferencesIfUnregistered.at(outputName));
}, *orifu);
if (!referencesOpt)
continue;
@ -2268,25 +2286,29 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
for (auto & r : references) {
auto name = r.name();
auto origHash = std::string { r.hashPart() };
if (r == scratchPath)
if (r == *scratchPath) {
res.first = true;
else if (outputRewrites.count(origHash) == 0)
res.second.insert(r);
else {
std::string newRef = outputRewrites.at(origHash);
} else if (auto outputRewrite = get(outputRewrites, origHash)) {
std::string newRef = *outputRewrite;
newRef += '-';
newRef += name;
res.second.insert(StorePath { newRef });
} else {
res.second.insert(r);
}
}
return res;
};
auto newInfoFromCA = [&](const DerivationOutput::CAFloating outputHash) -> ValidPathInfo {
auto & st = outputStats.at(outputName);
auto st = get(outputStats, outputName);
if (!st)
throw BuildError(
"output path %1% without valid stats info",
actualPath);
if (outputHash.method == FileIngestionMethod::Flat) {
/* The output path should be a regular file without execute permission. */
if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0)
if (!S_ISREG(st->st_mode) || (st->st_mode & S_IXUSR) != 0)
throw BuildError(
"output path '%1%' should be a non-executable regular file "
"since recursive hashing is not enabled (outputHashMode=flat)",
@ -2294,7 +2316,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
}
rewriteOutput();
/* FIXME optimize and deduplicate with addToStore */
std::string oldHashPart { scratchPath.hashPart() };
std::string oldHashPart { scratchPath->hashPart() };
HashModuloSink caSink { outputHash.hashType, oldHashPart };
switch (outputHash.method) {
case FileIngestionMethod::Recursive:
@ -2313,7 +2335,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
outputPathName(drv->name, outputName),
refs.second,
refs.first);
if (scratchPath != finalPath) {
if (*scratchPath != finalPath) {
// Also rewrite the output path
auto source = sinkToSource([&](Sink & nextSink) {
StringSink sink;
@ -2354,9 +2376,9 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
auto requiredFinalPath = output.path;
/* Preemptively add rewrite rule for final hash, as that is
what the NAR hash will use rather than normalized-self references */
if (scratchPath != requiredFinalPath)
if (*scratchPath != requiredFinalPath)
outputRewrites.insert_or_assign(
std::string { scratchPath.hashPart() },
std::string { scratchPath->hashPart() },
std::string { requiredFinalPath.hashPart() });
rewriteOutput();
auto narHashAndSize = hashPath(htSHA256, actualPath);
@ -2409,7 +2431,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
});
},
}, output.raw());
}, output->raw());
/* FIXME: set proper permissions in restorePath() so
we don't have to do another traversal. */
@ -2425,7 +2447,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
derivations. */
PathLocks dynamicOutputLock;
dynamicOutputLock.setDeletion(true);
auto optFixedPath = output.path(worker.store, drv->name, outputName);
auto optFixedPath = output->path(worker.store, drv->name, outputName);
if (!optFixedPath ||
worker.store.printStorePath(*optFixedPath) != finalDestPath)
{
@ -2491,11 +2513,10 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
/* For debugging, print out the referenced and unreferenced paths. */
for (auto & i : inputPaths) {
auto j = references.find(i);
if (j == references.end())
debug("unreferenced input: '%1%'", worker.store.printStorePath(i));
else
if (references.count(i))
debug("referenced input: '%1%'", worker.store.printStorePath(i));
else
debug("unreferenced input: '%1%'", worker.store.printStorePath(i));
}
if (curRound == nrRounds) {
@ -2612,9 +2633,11 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
DrvOutputs builtOutputs;
for (auto & [outputName, newInfo] : infos) {
auto oldinfo = get(initialOutputs, outputName);
assert(oldinfo);
auto thisRealisation = Realisation {
.id = DrvOutput {
initialOutputs.at(outputName).outputHash,
oldinfo->outputHash,
outputName
},
.outPath = newInfo.path
@ -2710,9 +2733,10 @@ void LocalDerivationGoal::checkOutputs(const std::map<std::string, ValidPathInfo
for (auto & i : *value) {
if (worker.store.isStorePath(i))
spec.insert(worker.store.parseStorePath(i));
else if (outputs.count(i))
spec.insert(outputs.at(i).path);
else throw BuildError("derivation contains an illegal reference specifier '%s'", i);
else if (auto output = get(outputs, i))
spec.insert(output->path);
else
throw BuildError("derivation contains an illegal reference specifier '%s'", i);
}
auto used = recursive
@ -2751,24 +2775,18 @@ void LocalDerivationGoal::checkOutputs(const std::map<std::string, ValidPathInfo
};
if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) {
auto outputChecks = structuredAttrs->find("outputChecks");
if (outputChecks != structuredAttrs->end()) {
auto output = outputChecks->find(outputName);
if (output != outputChecks->end()) {
if (auto outputChecks = get(*structuredAttrs, "outputChecks")) {
if (auto output = get(*outputChecks, outputName)) {
Checks checks;
auto maxSize = output->find("maxSize");
if (maxSize != output->end())
if (auto maxSize = get(*output, "maxSize"))
checks.maxSize = maxSize->get<uint64_t>();
auto maxClosureSize = output->find("maxClosureSize");
if (maxClosureSize != output->end())
if (auto maxClosureSize = get(*output, "maxClosureSize"))
checks.maxClosureSize = maxClosureSize->get<uint64_t>();
auto get = [&](const std::string & name) -> std::optional<Strings> {
auto i = output->find(name);
if (i != output->end()) {
auto get_ = [&](const std::string & name) -> std::optional<Strings> {
if (auto i = get(*output, name)) {
Strings res;
for (auto j = i->begin(); j != i->end(); ++j) {
if (!j->is_string())
@ -2781,10 +2799,10 @@ void LocalDerivationGoal::checkOutputs(const std::map<std::string, ValidPathInfo
return {};
};
checks.allowedReferences = get("allowedReferences");
checks.allowedRequisites = get("allowedRequisites");
checks.disallowedReferences = get("disallowedReferences");
checks.disallowedRequisites = get("disallowedRequisites");
checks.allowedReferences = get_("allowedReferences");
checks.allowedRequisites = get_("allowedRequisites");
checks.disallowedReferences = get_("disallowedReferences");
checks.disallowedRequisites = get_("disallowedRequisites");
applyChecks(checks);
}

View file

@ -350,7 +350,7 @@ void Worker::waitForInput()
become `available'. Note that `available' (i.e., non-blocking)
includes EOF. */
std::vector<struct pollfd> pollStatus;
std::map <int, int> fdToPollStatus;
std::map<int, size_t> fdToPollStatus;
for (auto & i : children) {
for (auto & j : i.fds) {
pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN });
@ -380,7 +380,10 @@ void Worker::waitForInput()
std::set<int> fds2(j->fds);
std::vector<unsigned char> buffer(4096);
for (auto & k : fds2) {
if (pollStatus.at(fdToPollStatus.at(k)).revents) {
const auto fdPollStatusId = get(fdToPollStatus, k);
assert(fdPollStatusId);
assert(*fdPollStatusId < pollStatus.size());
if (pollStatus.at(*fdPollStatusId).revents) {
ssize_t rd = ::read(k, buffer.data(), buffer.size());
// FIXME: is there a cleaner way to handle pt close
// than EIO? Is this even standard?

View file

@ -24,7 +24,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
Path storePath = getAttr("out");
auto mainUrl = getAttr("url");
bool unpack = get(drv.env, "unpack").value_or("") == "1";
bool unpack = getOr(drv.env, "unpack", "") == "1";
/* Note: have to use a fresh fileTransfer here because we're in
a forked process. */

View file

@ -661,8 +661,10 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut
if (res.kind == DrvHash::Kind::Deferred)
kind = DrvHash::Kind::Deferred;
for (auto & outputName : inputOutputs) {
const auto h = res.hashes.at(outputName);
inputs2[h.to_string(Base16, false)].insert(outputName);
const auto h = get(res.hashes, outputName);
if (!h)
throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name);
inputs2[h->to_string(Base16, false)].insert(outputName);
}
}
@ -836,8 +838,11 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String
auto hashModulo = hashDerivationModulo(store, Derivation(drv), true);
for (auto & [outputName, output] : drv.outputs) {
if (std::holds_alternative<DerivationOutput::Deferred>(output.raw())) {
auto & h = hashModulo.hashes.at(outputName);
auto outPath = store.makeOutputPath(outputName, h, drv.name);
auto h = get(hashModulo.hashes, outputName);
if (!h)
throw Error("derivation '%s' output '%s' has no hash (derivations.cc/rewriteDerivation)",
drv.name, outputName);
auto outPath = store.makeOutputPath(outputName, *h, drv.name);
drv.env[outputName] = store.printStorePath(outPath);
output = DerivationOutput::InputAddressed {
.path = std::move(outPath),

View file

@ -4,6 +4,8 @@
#include <nlohmann/json.hpp>
#include <optional>
namespace nix {
nlohmann::json DerivedPath::Opaque::toJSON(ref<Store> store) const {
@ -17,12 +19,12 @@ nlohmann::json DerivedPath::Built::toJSON(ref<Store> store) const {
res["drvPath"] = store->printStorePath(drvPath);
// Fallback for the input-addressed derivation case: We expect to always be
// able to print the output paths, so lets do it
auto knownOutputs = store->queryPartialDerivationOutputMap(drvPath);
const auto knownOutputs = store->queryPartialDerivationOutputMap(drvPath);
for (const auto& output : outputs) {
if (knownOutputs.at(output))
res["outputs"][output] = store->printStorePath(knownOutputs.at(output).value());
else
res["outputs"][output] = nullptr;
auto knownOutput = get(knownOutputs, output);
res["outputs"][output] = (knownOutput && *knownOutput)
? store->printStorePath(**knownOutput)
: nullptr;
}
return res;
}
@ -123,10 +125,15 @@ RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const
for (auto& [outputName, outputPath] : p.outputs) {
if (settings.isExperimentalFeatureEnabled(
Xp::CaDerivations)) {
auto drvOutput = get(drvHashes, outputName);
if (!drvOutput)
throw Error(
"the derivation '%s' has unrealised output '%s' (derived-path.cc/toRealisedPaths)",
store.printStorePath(p.drvPath), outputName);
auto thisRealisation = store.queryRealisation(
DrvOutput{drvHashes.at(outputName), outputName});
assert(thisRealisation); // Weve built it, so we must h
// ve the realisation
DrvOutput{*drvOutput, outputName});
assert(thisRealisation); // Weve built it, so we must
// have the realisation
res.insert(*thisRealisation);
} else {
res.insert(outputPath);

View file

@ -692,10 +692,10 @@ struct curlFileTransfer : public FileTransfer
#if ENABLE_S3
auto [bucketName, key, params] = parseS3Uri(request.uri);
std::string profile = get(params, "profile").value_or("");
std::string region = get(params, "region").value_or(Aws::Region::US_EAST_1);
std::string scheme = get(params, "scheme").value_or("");
std::string endpoint = get(params, "endpoint").value_or("");
std::string profile = getOr(params, "profile", "");
std::string region = getOr(params, "region", Aws::Region::US_EAST_1);
std::string scheme = getOr(params, "scheme", "");
std::string endpoint = getOr(params, "endpoint", "");
S3Helper s3Helper(profile, region, scheme, endpoint);

View file

@ -718,7 +718,11 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
// somewhat expensive so we do lazily
hashesModulo = hashDerivationModulo(*this, drv, true);
}
StorePath recomputed = makeOutputPath(i.first, hashesModulo->hashes.at(i.first), drvName);
auto currentOutputHash = get(hashesModulo->hashes, i.first);
if (!currentOutputHash)
throw Error("derivation '%s' has unexpected output '%s' (local-store / hashesModulo) named '%s'",
printStorePath(drvPath), printStorePath(doia.path), i.first);
StorePath recomputed = makeOutputPath(i.first, *currentOutputHash, drvName);
if (doia.path != recomputed)
throw Error("derivation '%s' has incorrect output '%s', should be '%s'",
printStorePath(drvPath), printStorePath(doia.path), printStorePath(recomputed));

View file

@ -278,11 +278,16 @@ std::map<DrvOutput, StorePath> drvOutputReferences(
std::set<Realisation> inputRealisations;
for (const auto & [inputDrv, outputNames] : drv.inputDrvs) {
auto outputHashes =
const auto outputHashes =
staticOutputHashes(store, store.readDerivation(inputDrv));
for (const auto & outputName : outputNames) {
auto outputHash = get(outputHashes, outputName);
if (!outputHash)
throw Error(
"output '%s' of derivation '%s' isn't realised", outputName,
store.printStorePath(inputDrv));
auto thisRealisation = store.queryRealisation(
DrvOutput{outputHashes.at(outputName), outputName});
DrvOutput{*outputHash, outputName});
if (!thisRealisation)
throw Error(
"output '%s' of derivation '%s' isn't built", outputName,

View file

@ -1,5 +1,8 @@
#include "path-with-outputs.hh"
#include "store-api.hh"
#include "nlohmann/json.hpp"
#include <regex>
namespace nix {
@ -68,4 +71,57 @@ StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std:
return StorePathWithOutputs { store.followLinksToStorePath(path), std::move(outputs) };
}
std::pair<std::string, OutputsSpec> parseOutputsSpec(const std::string & s)
{
static std::regex regex(R"((.*)\^((\*)|([a-z]+(,[a-z]+)*)))");
std::smatch match;
if (!std::regex_match(s, match, regex))
return {s, DefaultOutputs()};
if (match[3].matched)
return {match[1], AllOutputs()};
return {match[1], tokenizeString<OutputNames>(match[4].str(), ",")};
}
std::string printOutputsSpec(const OutputsSpec & outputsSpec)
{
if (std::get_if<DefaultOutputs>(&outputsSpec))
return "";
if (std::get_if<AllOutputs>(&outputsSpec))
return "^*";
if (auto outputNames = std::get_if<OutputNames>(&outputsSpec))
return "^" + concatStringsSep(",", *outputNames);
assert(false);
}
void to_json(nlohmann::json & json, const OutputsSpec & outputsSpec)
{
if (std::get_if<DefaultOutputs>(&outputsSpec))
json = nullptr;
else if (std::get_if<AllOutputs>(&outputsSpec))
json = std::vector<std::string>({"*"});
else if (auto outputNames = std::get_if<OutputNames>(&outputsSpec))
json = *outputNames;
}
void from_json(const nlohmann::json & json, OutputsSpec & outputsSpec)
{
if (json.is_null())
outputsSpec = DefaultOutputs();
else {
auto names = json.get<OutputNames>();
if (names == OutputNames({"*"}))
outputsSpec = AllOutputs();
else
outputsSpec = names;
}
}
}

View file

@ -4,6 +4,7 @@
#include "path.hh"
#include "derived-path.hh"
#include "nlohmann/json_fwd.hpp"
namespace nix {
@ -32,4 +33,25 @@ StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view
StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs);
typedef std::set<std::string> OutputNames;
struct AllOutputs {
bool operator < (const AllOutputs & _) const { return false; }
};
struct DefaultOutputs {
bool operator < (const DefaultOutputs & _) const { return false; }
};
typedef std::variant<DefaultOutputs, AllOutputs, OutputNames> OutputsSpec;
/* Parse a string of the form 'prefix^output1,...outputN' or
'prefix^*', returning the prefix and the outputs spec. */
std::pair<std::string, OutputsSpec> parseOutputsSpec(const std::string & s);
std::string printOutputsSpec(const OutputsSpec & outputsSpec);
void to_json(nlohmann::json &, const OutputsSpec &);
void from_json(const nlohmann::json &, OutputsSpec &);
}

View file

@ -853,15 +853,15 @@ std::vector<BuildResult> RemoteStore::buildPathsWithResults(
OutputPathMap outputs;
auto drv = evalStore->readDerivation(bfd.drvPath);
auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive
auto drvOutputs = drv.outputsAndOptPaths(*this);
const auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive
const auto drvOutputs = drv.outputsAndOptPaths(*this);
for (auto & output : bfd.outputs) {
if (!outputHashes.count(output))
auto outputHash = get(outputHashes, output);
if (!outputHash)
throw Error(
"the derivation '%s' doesn't have an output named '%s'",
printStorePath(bfd.drvPath), output);
auto outputId =
DrvOutput{outputHashes.at(output), output};
auto outputId = DrvOutput{ *outputHash, output };
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
auto realisation =
queryRealisation(outputId);
@ -874,13 +874,14 @@ std::vector<BuildResult> RemoteStore::buildPathsWithResults(
} else {
// If ca-derivations isn't enabled, assume that
// the output path is statically known.
assert(drvOutputs.count(output));
assert(drvOutputs.at(output).second);
const auto drvOutput = get(drvOutputs, output);
assert(drvOutput);
assert(drvOutput->second);
res.builtOutputs.emplace(
outputId,
Realisation {
.id = outputId,
.outPath = *drvOutputs.at(output).second
.outPath = *drvOutput->second,
});
}
}

View file

@ -1314,7 +1314,7 @@ static bool isNonUriPath(const std::string & spec) {
std::shared_ptr<Store> openFromNonUri(const std::string & uri, const Store::Params & params)
{
if (uri == "" || uri == "auto") {
auto stateDir = get(params, "state").value_or(settings.nixStateDir);
auto stateDir = getOr(params, "state", settings.nixStateDir);
if (access(stateDir.c_str(), R_OK | W_OK) == 0)
return std::make_shared<LocalStore>(params);
else if (pathExists(settings.nixDaemonSocketFile))

View file

@ -0,0 +1,46 @@
#include "path-with-outputs.hh"
#include <gtest/gtest.h>
namespace nix {
TEST(parseOutputsSpec, basic)
{
{
auto [prefix, outputsSpec] = parseOutputsSpec("foo");
ASSERT_EQ(prefix, "foo");
ASSERT_TRUE(std::get_if<DefaultOutputs>(&outputsSpec));
}
{
auto [prefix, outputsSpec] = parseOutputsSpec("foo^*");
ASSERT_EQ(prefix, "foo");
ASSERT_TRUE(std::get_if<AllOutputs>(&outputsSpec));
}
{
auto [prefix, outputsSpec] = parseOutputsSpec("foo^out");
ASSERT_EQ(prefix, "foo");
ASSERT_TRUE(std::get<OutputNames>(outputsSpec) == OutputNames({"out"}));
}
{
auto [prefix, outputsSpec] = parseOutputsSpec("foo^out,bin");
ASSERT_EQ(prefix, "foo");
ASSERT_TRUE(std::get<OutputNames>(outputsSpec) == OutputNames({"out", "bin"}));
}
{
auto [prefix, outputsSpec] = parseOutputsSpec("foo^bar^out,bin");
ASSERT_EQ(prefix, "foo^bar");
ASSERT_TRUE(std::get<OutputNames>(outputsSpec) == OutputNames({"out", "bin"}));
}
{
auto [prefix, outputsSpec] = parseOutputsSpec("foo^&*()");
ASSERT_EQ(prefix, "foo^&*()");
ASSERT_TRUE(std::get_if<DefaultOutputs>(&outputsSpec));
}
}
}

View file

@ -35,7 +35,9 @@ const std::optional<ExperimentalFeature> parseExperimentalFeature(const std::str
std::string_view showExperimentalFeature(const ExperimentalFeature feature)
{
return stringifiedXpFeatures.at(feature);
const auto ret = get(stringifiedXpFeatures, feature);
assert(ret);
return *ret;
}
std::set<ExperimentalFeature> parseFeatures(const std::set<std::string> & rawFeatures)
@ -58,11 +60,13 @@ std::ostream & operator <<(std::ostream & str, const ExperimentalFeature & featu
return str << showExperimentalFeature(feature);
}
void to_json(nlohmann::json& j, const ExperimentalFeature& feature) {
void to_json(nlohmann::json & j, const ExperimentalFeature & feature)
{
j = showExperimentalFeature(feature);
}
void from_json(const nlohmann::json& j, ExperimentalFeature& feature) {
void from_json(const nlohmann::json & j, ExperimentalFeature & feature)
{
const std::string input = j;
const auto parsed = parseExperimentalFeature(input);

View file

@ -55,7 +55,7 @@ public:
* Semi-magic conversion to and from json.
* See the nlohmann/json readme for more details.
*/
void to_json(nlohmann::json&, const ExperimentalFeature&);
void from_json(const nlohmann::json&, ExperimentalFeature&);
void to_json(nlohmann::json &, const ExperimentalFeature &);
void from_json(const nlohmann::json &, ExperimentalFeature &);
}

21
src/libutil/json-utils.hh Normal file
View file

@ -0,0 +1,21 @@
#pragma once
#include <nlohmann/json.hpp>
namespace nix {
const nlohmann::json * get(const nlohmann::json & map, const std::string & key)
{
auto i = map.find(key);
if (i == map.end()) return nullptr;
return &*i;
}
nlohmann::json * get(nlohmann::json & map, const std::string & key)
{
auto i = map.find(key);
if (i == map.end()) return nullptr;
return &*i;
}
}

View file

@ -548,7 +548,7 @@ namespace nix {
TEST(get, emptyContainer) {
StringMap s = { };
auto expected = std::nullopt;
auto expected = nullptr;
ASSERT_EQ(get(s, "one"), expected);
}
@ -559,7 +559,23 @@ namespace nix {
s["two"] = "er";
auto expected = "yi";
ASSERT_EQ(get(s, "one"), expected);
ASSERT_EQ(*get(s, "one"), expected);
}
TEST(getOr, emptyContainer) {
StringMap s = { };
auto expected = "yi";
ASSERT_EQ(getOr(s, "one", "yi"), expected);
}
TEST(getOr, getFromContainer) {
StringMap s;
s["one"] = "yi";
s["two"] = "er";
auto expected = "yi";
ASSERT_EQ(getOr(s, "one", "nope"), expected);
}
/* ----------------------------------------------------------------------------

View file

@ -1588,7 +1588,6 @@ std::string stripIndentation(std::string_view s)
//////////////////////////////////////////////////////////////////////
static Sync<std::pair<unsigned short, unsigned short>> windowSize{{0, 0}};

View file

@ -543,13 +543,31 @@ std::string stripIndentation(std::string_view s);
/* Get a value for the specified key from an associate container. */
template <class T>
std::optional<typename T::mapped_type> get(const T & map, const typename T::key_type & key)
const typename T::mapped_type * get(const T & map, const typename T::key_type & key)
{
auto i = map.find(key);
if (i == map.end()) return {};
return std::optional<typename T::mapped_type>(i->second);
if (i == map.end()) return nullptr;
return &i->second;
}
template <class T>
typename T::mapped_type * get(T & map, const typename T::key_type & key)
{
auto i = map.find(key);
if (i == map.end()) return nullptr;
return &i->second;
}
/* Get a value for the specified key from an associate container, or a default value if the key isn't present. */
template <class T>
const typename T::mapped_type & getOr(T & map,
const typename T::key_type & key,
const typename T::mapped_type & defaultValue)
{
auto i = map.find(key);
if (i == map.end()) return defaultValue;
return i->second;
}
/* Remove and return the first item from a container. */
template <class T>

2
src/nix-build/nix-build.cc Executable file → Normal file
View file

@ -440,7 +440,7 @@ static void main_nix_build(int argc, char * * argv)
env["NIX_STORE"] = store->storeDir;
env["NIX_BUILD_CORES"] = std::to_string(settings.buildCores);
auto passAsFile = tokenizeString<StringSet>(get(drv.env, "passAsFile").value_or(""));
auto passAsFile = tokenizeString<StringSet>(getOr(drv.env, "passAsFile", ""));
bool keepTmp = false;
int fileNr = 0;

View file

@ -89,7 +89,7 @@ UnresolvedApp Installable::toApp(EvalState & state)
auto outputName = cursor->getAttr(state.sOutputName)->getString();
auto name = cursor->getAttr(state.sName)->getString();
auto aPname = cursor->maybeGetAttr("pname");
auto aMeta = cursor->maybeGetAttr("meta");
auto aMeta = cursor->maybeGetAttr(state.sMeta);
auto aMainProgram = aMeta ? aMeta->maybeGetAttr("mainProgram") : nullptr;
auto mainProgram =
aMainProgram

View file

@ -75,10 +75,10 @@ struct CmdBundle : InstallableCommand
auto val = installable->toValue(*evalState).first;
auto [bundlerFlakeRef, bundlerName] = parseFlakeRefWithFragment(bundler, absPath("."));
auto [bundlerFlakeRef, bundlerName, outputsSpec] = parseFlakeRefWithFragmentAndOutputsSpec(bundler, absPath("."));
const flake::LockFlags lockFlags{ .writeLockFile = false };
InstallableFlake bundler{this,
evalState, std::move(bundlerFlakeRef), bundlerName,
evalState, std::move(bundlerFlakeRef), bundlerName, outputsSpec,
{"bundlers." + settings.thisSystem.get() + ".default",
"defaultBundler." + settings.thisSystem.get()
},

View file

@ -507,6 +507,7 @@ struct CmdDevelop : Common, MixEnvironment
state,
installable->nixpkgsFlakeRef(),
"bashInteractive",
DefaultOutputs(),
Strings{},
Strings{"legacyPackages." + settings.thisSystem.get() + "."},
nixpkgsLockFlags);

View file

@ -724,7 +724,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand
auto [templateFlakeRef, templateName] = parseFlakeRefWithFragment(templateUrl, absPath("."));
auto installable = InstallableFlake(nullptr,
evalState, std::move(templateFlakeRef), templateName,
evalState, std::move(templateFlakeRef), templateName, DefaultOutputs(),
defaultTemplateAttrPaths,
defaultTemplateAttrPathsPrefixes,
lockFlags);
@ -1015,8 +1015,8 @@ struct CmdFlakeShow : FlakeCommand, MixJSON
auto name = visitor.getAttr(state->sName)->getString();
if (json) {
std::optional<std::string> description;
if (auto aMeta = visitor.maybeGetAttr("meta")) {
if (auto aDescription = aMeta->maybeGetAttr("description"))
if (auto aMeta = visitor.maybeGetAttr(state->sMeta)) {
if (auto aDescription = aMeta->maybeGetAttr(state->sDescription))
description = aDescription->getString();
}
j.emplace("type", "derivation");

View file

@ -153,7 +153,7 @@ Currently the `type` attribute can be one of the following:
git(+http|+https|+ssh|+git|+file|):(//<server>)?<path>(\?<params>)?
```
The `ref` attribute defaults to `master`.
The `ref` attribute defaults to resolving the `HEAD` reference.
The `rev` attribute must denote a commit that exists in the branch
or tag specified by the `ref` attribute, since Nix doesn't do a full
@ -161,6 +161,11 @@ Currently the `type` attribute can be one of the following:
doesn't allow fetching a `rev` without a known `ref`). The default
is the commit currently pointed to by `ref`.
When `git+file` is used without specifying `ref` or `rev`, files are
fetched directly from the local `path` as long as they have been added
to the Git repository. If there are uncommitted changes, the reference
is treated as dirty and a warning is printed.
For example, the following are valid Git flake references:
* `git+https://example.org/my/repo`

View file

@ -146,6 +146,51 @@ For most commands, if no installable is specified, the default is `.`,
i.e. Nix will operate on the default flake output attribute of the
flake in the current directory.
## Derivation output selection
Derivations can have multiple outputs, each corresponding to a
different store path. For instance, a package can have a `bin` output
that contains programs, and a `dev` output that provides development
artifacts like C/C++ header files. The outputs on which `nix` commands
operate are determined as follows:
* You can explicitly specify the desired outputs using the syntax
*installable*`^`*output1*`,`*...*`,`*outputN*. For example, you can
obtain the `dev` and `static` outputs of the `glibc` package:
```console
# nix build 'nixpkgs#glibc^dev,static'
# ls ./result-dev/include/ ./result-static/lib/
```
* You can also specify that *all* outputs should be used using the
syntax *installable*`^*`. For example, the following shows the size
of all outputs of the `glibc` package in the binary cache:
```console
# nix path-info -S --eval-store auto --store https://cache.nixos.org 'nixpkgs#glibc^*'
/nix/store/g02b1lpbddhymmcjb923kf0l7s9nww58-glibc-2.33-123 33208200
/nix/store/851dp95qqiisjifi639r0zzg5l465ny4-glibc-2.33-123-bin 36142896
/nix/store/kdgs3q6r7xdff1p7a9hnjr43xw2404z7-glibc-2.33-123-debug 155787312
/nix/store/n4xa8h6pbmqmwnq0mmsz08l38abb06zc-glibc-2.33-123-static 42488328
/nix/store/q6580lr01jpcsqs4r5arlh4ki2c1m9rv-glibc-2.33-123-dev 44200560
```
* If you didn't specify the desired outputs, but the derivation has an
attribute `meta.outputsToInstall`, Nix will use those outputs. For
example, since the package `nixpkgs#libxml2` has this attribute:
```console
# nix eval 'nixpkgs#libxml2.meta.outputsToInstall'
[ "bin" "man" ]
```
a command like `nix shell nixpkgs#libxml2` will provide only those
two outputs by default.
* Otherwise, Nix will use all outputs of the derivation.
# Nix stores
Most `nix` subcommands operate on a *Nix store*.

View file

@ -20,6 +20,13 @@ R""(
# nix profile install nixpkgs/d73407e8e6002646acfdef0e39ace088bacc83da#hello
```
* Install a specific output of a package:
```console
# nix profile install nixpkgs#bash^man
```
# Description
This command adds *installables* to a Nix profile.

View file

@ -22,13 +22,13 @@ struct ProfileElementSource
// FIXME: record original attrpath.
FlakeRef resolvedRef;
std::string attrPath;
// FIXME: output names
OutputsSpec outputs;
bool operator < (const ProfileElementSource & other) const
{
return
std::pair(originalRef.to_string(), attrPath) <
std::pair(other.originalRef.to_string(), other.attrPath);
std::tuple(originalRef.to_string(), attrPath, outputs) <
std::tuple(other.originalRef.to_string(), other.attrPath, other.outputs);
}
};
@ -42,7 +42,7 @@ struct ProfileElement
std::string describe() const
{
if (source)
return fmt("%s#%s", source->originalRef, source->attrPath);
return fmt("%s#%s%s", source->originalRef, source->attrPath, printOutputsSpec(source->outputs));
StringSet names;
for (auto & path : storePaths)
names.insert(DrvName(path.name()).name);
@ -67,7 +67,6 @@ struct ProfileElement
ref<Store> store,
const BuiltPaths & builtPaths)
{
// FIXME: respect meta.outputsToInstall
storePaths.clear();
for (auto & buildable : builtPaths) {
std::visit(overloaded {
@ -99,7 +98,7 @@ struct ProfileManifest
auto version = json.value("version", 0);
std::string sUrl;
std::string sOriginalUrl;
switch(version){
switch (version) {
case 1:
sUrl = "uri";
sOriginalUrl = "originalUri";
@ -117,11 +116,12 @@ struct ProfileManifest
for (auto & p : e["storePaths"])
element.storePaths.insert(state.store->parseStorePath((std::string) p));
element.active = e["active"];
if (e.value(sUrl,"") != "") {
element.source = ProfileElementSource{
if (e.value(sUrl, "") != "") {
element.source = ProfileElementSource {
parseFlakeRef(e[sOriginalUrl]),
parseFlakeRef(e[sUrl]),
e["attrPath"]
e["attrPath"],
e["outputs"].get<OutputsSpec>()
};
}
elements.emplace_back(std::move(element));
@ -157,6 +157,7 @@ struct ProfileManifest
obj["originalUrl"] = element.source->originalRef.to_string();
obj["url"] = element.source->resolvedRef.to_string();
obj["attrPath"] = element.source->attrPath;
obj["outputs"] = element.source->outputs;
}
array.push_back(obj);
}
@ -284,10 +285,11 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
if (auto installable2 = std::dynamic_pointer_cast<InstallableFlake>(installable)) {
// FIXME: make build() return this?
auto [attrPath, resolvedRef, drv] = installable2->toDerivation();
element.source = ProfileElementSource{
element.source = ProfileElementSource {
installable2->flakeRef,
resolvedRef,
attrPath,
installable2->outputsSpec
};
}
@ -444,6 +446,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
getEvalState(),
FlakeRef(element.source->originalRef),
"",
element.source->outputs,
Strings{element.source->attrPath},
Strings{},
lockFlags);
@ -455,10 +458,11 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
printInfo("upgrading '%s' from flake '%s' to '%s'",
element.source->attrPath, element.source->resolvedRef, resolvedRef);
element.source = ProfileElementSource{
element.source = ProfileElementSource {
installable->flakeRef,
resolvedRef,
attrPath,
installable->outputsSpec
};
installables.push_back(installable);
@ -514,8 +518,8 @@ struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultPro
for (size_t i = 0; i < manifest.elements.size(); ++i) {
auto & element(manifest.elements[i]);
logger->cout("%d %s %s %s", i,
element.source ? element.source->originalRef.to_string() + "#" + element.source->attrPath : "-",
element.source ? element.source->resolvedRef.to_string() + "#" + element.source->attrPath : "-",
element.source ? element.source->originalRef.to_string() + "#" + element.source->attrPath + printOutputsSpec(element.source->outputs) : "-",
element.source ? element.source->resolvedRef.to_string() + "#" + element.source->attrPath + printOutputsSpec(element.source->outputs) : "-",
concatStringsSep(" ", store->printStorePathSet(element.storePaths)));
}
}

View file

@ -93,10 +93,10 @@ struct CmdSearch : InstallableCommand, MixJSON
};
if (cursor.isDerivation()) {
DrvName name(cursor.getAttr("name")->getString());
DrvName name(cursor.getAttr(state->sName)->getString());
auto aMeta = cursor.maybeGetAttr("meta");
auto aDescription = aMeta ? aMeta->maybeGetAttr("description") : nullptr;
auto aMeta = cursor.maybeGetAttr(state->sMeta);
auto aDescription = aMeta ? aMeta->maybeGetAttr(state->sDescription) : nullptr;
auto description = aDescription ? aDescription->getString() : "";
std::replace(description.begin(), description.end(), '\n', ' ');
auto attrPath2 = concatStringsSep(".", attrPathS);

View file

@ -176,7 +176,7 @@ int main(int argc, char ** argv)
impurePaths.insert(argv[2]);
else {
auto drv = store->derivationFromPath(store->parseStorePath(argv[1]));
impurePaths = tokenizeString<StringSet>(get(drv.env, "__impureHostDeps").value_or(""));
impurePaths = tokenizeString<StringSet>(getOr(drv.env, "__impureHostDeps", ""));
impurePaths.insert("/usr/lib/libSystem.dylib");
}

View file

@ -34,7 +34,7 @@ outPath=$(readlink -f $TEST_ROOT/result)
grep 'FOO BAR BAZ' $TEST_ROOT/machine0/$outPath
testPrintOutPath=$(nix build -L -v -f $file --print-out-paths --max-jobs 0 \
testPrintOutPath=$(nix build -L -v -f $file --no-link --print-out-paths --max-jobs 0 \
--arg busybox $busybox \
--store $TEST_ROOT/machine0 \
--builders "$(join_by '; ' "${builders[@]}")"
@ -72,6 +72,7 @@ fi
# Behavior of keep-failed
out="$(nix-build 2>&1 failing.nix \
--no-out-link \
--builders "$(join_by '; ' "${builders[@]}")" \
--keep-failed \
--store $TEST_ROOT/machine0 \

View file

@ -2,15 +2,10 @@ source common.sh
clearStore
# Make sure that 'nix build' only returns the outputs we asked for.
nix build -f multiple-outputs.nix --json a --no-link | jq --exit-status '
(.[0] |
(.drvPath | match(".*multiple-outputs-a.drv")) and
(.outputs | keys | length == 1) and
(.outputs.first | match(".*multiple-outputs-a-first")))
'
set -o pipefail
nix build -f multiple-outputs.nix --json a.all b.all --no-link | jq --exit-status '
# Make sure that 'nix build' returns all outputs by default.
nix build -f multiple-outputs.nix --json a b --no-link | jq --exit-status '
(.[0] |
(.drvPath | match(".*multiple-outputs-a.drv")) and
(.outputs | keys | length == 2) and
@ -22,6 +17,45 @@ nix build -f multiple-outputs.nix --json a.all b.all --no-link | jq --exit-statu
(.outputs.out | match(".*multiple-outputs-b")))
'
# Test output selection using the '^' syntax.
nix build -f multiple-outputs.nix --json a^first --no-link | jq --exit-status '
(.[0] |
(.drvPath | match(".*multiple-outputs-a.drv")) and
(.outputs | keys == ["first"]))
'
nix build -f multiple-outputs.nix --json a^second,first --no-link | jq --exit-status '
(.[0] |
(.drvPath | match(".*multiple-outputs-a.drv")) and
(.outputs | keys == ["first", "second"]))
'
nix build -f multiple-outputs.nix --json 'a^*' --no-link | jq --exit-status '
(.[0] |
(.drvPath | match(".*multiple-outputs-a.drv")) and
(.outputs | keys == ["first", "second"]))
'
# Test that 'outputsToInstall' is respected by default.
nix build -f multiple-outputs.nix --json e --no-link | jq --exit-status '
(.[0] |
(.drvPath | match(".*multiple-outputs-e.drv")) and
(.outputs | keys == ["a", "b"]))
'
# But not when it's overriden.
nix build -f multiple-outputs.nix --json e^a --no-link | jq --exit-status '
(.[0] |
(.drvPath | match(".*multiple-outputs-e.drv")) and
(.outputs | keys == ["a"]))
'
nix build -f multiple-outputs.nix --json 'e^*' --no-link | jq --exit-status '
(.[0] |
(.drvPath | match(".*multiple-outputs-e.drv")) and
(.outputs | keys == ["a", "b", "c"]))
'
testNormalization () {
clearStore
outPath=$(nix-build ./simple.nix --no-out-link)

View file

@ -23,7 +23,7 @@ rec {
};
rootCA = mkCADerivation {
name = "rootCA";
outputs = [ "out" "dev" "foo"];
outputs = [ "out" "dev" "foo" ];
buildCommand = ''
echo "building a CA derivation"
echo "The seed is ${toString seed}"

View file

@ -25,7 +25,8 @@ buildDrvs --substitute --substituters $REMOTE_STORE --no-require-sigs -j0 transi
# Check that the thing weve just substituted has its realisation stored
nix realisation info --file ./content-addressed.nix transitivelyDependentCA
# Check that its dependencies have it too
nix realisation info --file ./content-addressed.nix dependentCA rootCA
nix realisation info --file ./content-addressed.nix dependentCA
# nix realisation info --file ./content-addressed.nix rootCA --outputs out
# Same thing, but
# 1. With non-ca derivations

View file

@ -157,11 +157,12 @@ expect() {
local expected res
expected="$1"
shift
set +e
"$@"
res="$?"
set -e
[[ $res -eq $expected ]]
"$@" || res="$?"
if [[ $res -ne $expected ]]; then
echo "Expected '$expected' but got '$res' while running '$*'"
return 1
fi
return 0
}
needLocalStore() {

View file

@ -7,7 +7,7 @@ clearStore
clearCacheCache
# Initialize binary cache.
nonCaPath=$(nix build --json --file ./dependencies.nix | jq -r .[].outputs.out)
nonCaPath=$(nix build --json --file ./dependencies.nix --no-link | jq -r .[].outputs.out)
caPath=$(nix store make-content-addressed --json $nonCaPath | jq -r '.rewrites | map(.) | .[]')
nix copy --to file://$cacheDir $nonCaPath

View file

@ -161,6 +161,14 @@ path4=$(nix eval --impure --raw --expr "(builtins.fetchGit $repo).outPath")
[[ $(cat $path4/hello) = dev ]]
[[ $path3 = $path4 ]]
# Using remote path with branch other than 'master' should fetch the HEAD revision.
# (--tarball-ttl 0 to prevent using the cached repo above)
export _NIX_FORCE_HTTP=1
path4=$(nix eval --tarball-ttl 0 --impure --raw --expr "(builtins.fetchGit $repo).outPath")
[[ $(cat $path4/hello) = dev ]]
[[ $path3 = $path4 ]]
unset _NIX_FORCE_HTTP
# Confirm same as 'dev' branch
path5=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath")
[[ $path3 = $path5 ]]

View file

@ -31,7 +31,14 @@ flakeFollowsE=$TEST_ROOT/follows/flakeA/flakeE
for repo in $flake1Dir $flake2Dir $flake3Dir $flake7Dir $templatesDir $nonFlakeDir $flakeA $flakeB $flakeFollowsA; do
rm -rf $repo $repo.tmp
mkdir -p $repo
git -C $repo init
# Give one repo a non-master initial branch.
extraArgs=
if [[ $repo == $flake2Dir ]]; then
extraArgs="--initial-branch=main"
fi
git -C $repo init $extraArgs
git -C $repo config user.email "foobar@example.com"
git -C $repo config user.name "Foobar"
done

View file

@ -4,6 +4,7 @@ export TEST_VAR=foo # for eval-okay-getenv.nix
export NIX_REMOTE=dummy://
nix-instantiate --eval -E 'builtins.trace "Hello" 123' 2>&1 | grep -q Hello
nix-instantiate --eval -E 'builtins.addErrorContext "Hello" 123' 2>&1
(! nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" 123' 2>&1 | grep -q Hello)
nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" (throw "Foo")' 2>&1 | grep -q Hello
@ -14,7 +15,7 @@ fail=0
for i in lang/parse-fail-*.nix; do
echo "parsing $i (should fail)";
i=$(basename $i .nix)
if nix-instantiate --parse - < lang/$i.nix; then
if ! expect 1 nix-instantiate --parse - < lang/$i.nix; then
echo "FAIL: $i shouldn't parse"
fail=1
fi
@ -23,7 +24,7 @@ done
for i in lang/parse-okay-*.nix; do
echo "parsing $i (should succeed)";
i=$(basename $i .nix)
if ! nix-instantiate --parse - < lang/$i.nix > lang/$i.out; then
if ! expect 0 nix-instantiate --parse - < lang/$i.nix > lang/$i.out; then
echo "FAIL: $i should parse"
fail=1
fi
@ -32,7 +33,7 @@ done
for i in lang/eval-fail-*.nix; do
echo "evaluating $i (should fail)";
i=$(basename $i .nix)
if nix-instantiate --eval lang/$i.nix; then
if ! expect 1 nix-instantiate --eval lang/$i.nix; then
echo "FAIL: $i shouldn't evaluate"
fail=1
fi
@ -47,7 +48,7 @@ for i in lang/eval-okay-*.nix; do
if test -e lang/$i.flags; then
flags=$(cat lang/$i.flags)
fi
if ! NIX_PATH=lang/dir3:lang/dir4 nix-instantiate $flags --eval --strict lang/$i.nix > lang/$i.out; then
if ! expect 0 env NIX_PATH=lang/dir3:lang/dir4 nix-instantiate $flags --eval --strict lang/$i.nix > lang/$i.out; then
echo "FAIL: $i should evaluate"
fail=1
elif ! diff lang/$i.out lang/$i.exp; then
@ -57,7 +58,7 @@ for i in lang/eval-okay-*.nix; do
fi
if test -e lang/$i.exp.xml; then
if ! nix-instantiate --eval --xml --no-location --strict \
if ! expect 0 nix-instantiate --eval --xml --no-location --strict \
lang/$i.nix > lang/$i.out.xml; then
echo "FAIL: $i should evaluate"
fail=1

View file

@ -80,4 +80,11 @@ rec {
'';
}).a;
e = mkDerivation {
name = "multiple-outputs-e";
outputs = [ "a" "b" "c" ];
meta.outputsToInstall = [ "a" "b" ];
buildCommand = "mkdir $a $b $c";
};
}

View file

@ -17,6 +17,7 @@ cat > $flake1Dir/flake.nix <<EOF
outputs = { self }: with import ./config.nix; rec {
packages.$system.default = mkDerivation {
name = "profile-test-\${builtins.readFile ./version}";
outputs = [ "out" "man" "dev" ];
builder = builtins.toFile "builder.sh"
''
mkdir -p \$out/bin
@ -26,10 +27,13 @@ cat > $flake1Dir/flake.nix <<EOF
EOF
chmod +x \$out/bin/hello
echo DONE
mkdir -p \$man/share/man
mkdir -p \$dev/include
'';
__contentAddressed = import ./ca.nix;
outputHashMode = "recursive";
outputHashAlgo = "sha256";
meta.outputsToInstall = [ "out" "man" ];
};
};
}
@ -46,6 +50,8 @@ nix-env -f ./user-envs.nix -i foo-1.0
nix profile list | grep '0 - - .*-foo-1.0'
nix profile install $flake1Dir -L
[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]]
[ -e $TEST_HOME/.nix-profile/share/man ]
(! [ -e $TEST_HOME/.nix-profile/include ])
nix profile history
nix profile history | grep "packages.$system.default: ∅ -> 1.0"
nix profile diff-closures | grep 'env-manifest.nix: ε → ∅'
@ -55,7 +61,7 @@ printf NixOS > $flake1Dir/who
printf 2.0 > $flake1Dir/version
nix profile upgrade 1
[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello NixOS" ]]
nix profile history | grep "packages.$system.default: 1.0 -> 2.0"
nix profile history | grep "packages.$system.default: 1.0, 1.0-man -> 2.0, 2.0-man"
# Test 'history', 'diff-closures'.
nix profile diff-closures
@ -86,7 +92,7 @@ nix profile wipe-history
printf true > $flake1Dir/ca.nix
printf 3.0 > $flake1Dir/version
nix profile upgrade 0
nix profile history | grep "packages.$system.default: 1.0 -> 3.0"
nix profile history | grep "packages.$system.default: 1.0, 1.0-man -> 3.0, 3.0-man"
# Test new install of CA package.
nix profile remove 0
@ -95,3 +101,22 @@ printf Utrecht > $flake1Dir/who
nix profile install $flake1Dir
[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello Utrecht" ]]
[[ $(nix path-info --json $(realpath $TEST_HOME/.nix-profile/bin/hello) | jq -r .[].ca) =~ fixed:r:sha256: ]]
# Override the outputs.
nix profile remove 0 1
nix profile install "$flake1Dir^*"
[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello Utrecht" ]]
[ -e $TEST_HOME/.nix-profile/share/man ]
[ -e $TEST_HOME/.nix-profile/include ]
printf Nix > $flake1Dir/who
nix profile upgrade 0
[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello Nix" ]]
[ -e $TEST_HOME/.nix-profile/share/man ]
[ -e $TEST_HOME/.nix-profile/include ]
nix profile remove 0
nix profile install "$flake1Dir^man"
(! [ -e $TEST_HOME/.nix-profile/bin/hello ])
[ -e $TEST_HOME/.nix-profile/share/man ]
(! [ -e $TEST_HOME/.nix-profile/include ])

View file

@ -3,15 +3,24 @@ with import ./config.nix;
{
hello = mkDerivation {
name = "hello";
outputs = [ "out" "dev" ];
meta.outputsToInstall = [ "out" ];
buildCommand =
''
mkdir -p $out/bin
mkdir -p $out/bin $dev/bin
cat > $out/bin/hello <<EOF
#! ${shell}
who=\$1
echo "Hello \''${who:-World} from $out/bin/hello"
EOF
chmod +x $out/bin/hello
cat > $dev/bin/hello2 <<EOF
#! ${shell}
echo "Hello2"
EOF
chmod +x $dev/bin/hello2
'';
};
}

View file

@ -6,6 +6,10 @@ clearCache
nix shell -f shell-hello.nix hello -c hello | grep 'Hello World'
nix shell -f shell-hello.nix hello -c hello NixOS | grep 'Hello NixOS'
# Test output selection.
nix shell -f shell-hello.nix hello^dev -c hello2 | grep 'Hello2'
nix shell -f shell-hello.nix 'hello^*' -c hello2 | grep 'Hello2'
if ! canUseSandbox; then exit 99; fi
chmod -R u+w $TEST_ROOT/store0 || true