Merge remote-tracking branch 'upstream/master' into path-info

This commit is contained in:
John Ericson 2023-01-14 14:27:28 -05:00
commit 056cc1c1b9
81 changed files with 1773 additions and 846 deletions

View file

@ -10,3 +10,12 @@ git-repository-url = "https://github.com/NixOS/nix"
[preprocessor.anchors] [preprocessor.anchors]
renderers = ["html"] renderers = ["html"]
command = "jq --from-file doc/manual/anchors.jq" command = "jq --from-file doc/manual/anchors.jq"
[output.linkcheck]
# no Internet during the build (in the sandbox)
follow-web-links = false
# mdbook-linkcheck does not understand [foo]{#bar} style links, resulting in
# excessive "Potential incomplete link" warnings. No other kind of warning was
# produced at the time of writing.
warning-policy = "ignore"

View file

@ -50,11 +50,16 @@ $(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli
$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix $(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix
@rm -rf $@ @rm -rf $@
$(trace-gen) $(nix-eval) --write-to $@ --expr 'import doc/manual/generate-manpage.nix { toplevel = builtins.readFile $<; }' $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix { toplevel = builtins.readFile $<; }'
# @docroot@: https://nixos.org/manual/nix/unstable/contributing/hacking.html#docroot-variable
$(trace-gen) sed -i $@.tmp/*.md -e 's^@docroot@^../..^g'
@mv $@.tmp $@
$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/generate-options.nix $(d)/src/command-ref/conf-file-prefix.md $(bindir)/nix $(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/generate-options.nix $(d)/src/command-ref/conf-file-prefix.md $(bindir)/nix
@cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp @cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp
$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-options.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp # @docroot@: https://nixos.org/manual/nix/unstable/contributing/hacking.html#docroot-variable
$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-options.nix (builtins.fromJSON (builtins.readFile $<))' \
| sed -e 's^@docroot@^..^g'>> $@.tmp
@mv $@.tmp $@ @mv $@.tmp $@
$(d)/nix.json: $(bindir)/nix $(d)/nix.json: $(bindir)/nix
@ -67,7 +72,9 @@ $(d)/conf-file.json: $(bindir)/nix
$(d)/src/language/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix $(d)/src/language/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix
@cat doc/manual/src/language/builtins-prefix.md > $@.tmp @cat doc/manual/src/language/builtins-prefix.md > $@.tmp
$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp # @docroot@: https://nixos.org/manual/nix/unstable/contributing/hacking.html#docroot-variable
$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' \
| sed -e 's^@docroot@^..^g' >> $@.tmp
@cat doc/manual/src/language/builtins-suffix.md >> $@.tmp @cat doc/manual/src/language/builtins-suffix.md >> $@.tmp
@mv $@.tmp $@ @mv $@.tmp $@
@ -102,6 +109,12 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli
@touch $@ @touch $@
$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md
$(trace-gen) RUST_LOG=warn mdbook build doc/manual -d $(DESTDIR)$(docdir)/manual $(trace-gen) \
set -euo pipefail; \
RUST_LOG=warn mdbook build doc/manual -d $(DESTDIR)$(docdir)/manual.tmp 2>&1 \
| { grep -Fv "because fragment resolution isn't implemented" || :; }
@rm -rf $(DESTDIR)$(docdir)/manual
@mv $(DESTDIR)$(docdir)/manual.tmp/html $(DESTDIR)$(docdir)/manual
@rm -rf $(DESTDIR)$(docdir)/manual.tmp
endif endif

View file

@ -68,7 +68,7 @@ It can also execute build plans to produce new data, which are made available to
A build plan itself is a series of *build tasks*, together with their build inputs. A build plan itself is a series of *build tasks*, together with their build inputs.
> **Important** > **Important**
> A build task in Nix is called [derivation](../glossary#gloss-derivation). > A build task in Nix is called [derivation](../glossary.md#gloss-derivation).
Each build task has a special build input executed as *build instructions* in order to perform the build. Each build task has a special build input executed as *build instructions* in order to perform the build.
The result of a build task can be input to another build task. The result of a build task can be input to another build task.

View file

@ -11,7 +11,7 @@ Most Nix commands interpret the following environment variables:
expressions using [paths](../language/values.md#type-path) expressions using [paths](../language/values.md#type-path)
enclosed in angle brackets (i.e., `<path>`), enclosed in angle brackets (i.e., `<path>`),
e.g. `/home/eelco/Dev:/etc/nixos`. It can be extended using the e.g. `/home/eelco/Dev:/etc/nixos`. It can be extended using the
[`-I` option](./opt-common#opt-I). [`-I` option](./opt-common.md#opt-I).
- [`NIX_IGNORE_SYMLINK_STORE`]{#env-NIX_IGNORE_SYMLINK_STORE}\ - [`NIX_IGNORE_SYMLINK_STORE`]{#env-NIX_IGNORE_SYMLINK_STORE}\
Normally, the Nix store directory (typically `/nix/store`) is not Normally, the Nix store directory (typically `/nix/store`) is not

View file

@ -49,7 +49,7 @@ authentication, you can avoid typing the passphrase with `ssh-agent`.
- `--include-outputs`\ - `--include-outputs`\
Also copy the outputs of [store derivation]s included in the closure. Also copy the outputs of [store derivation]s included in the closure.
[store derivation]: ../../glossary.md#gloss-store-derivation [store derivation]: ../glossary.md#gloss-store-derivation
- `--use-substitutes` / `-s`\ - `--use-substitutes` / `-s`\
Attempt to download missing paths on the target machine using Nixs Attempt to download missing paths on the target machine using Nixs

View file

@ -155,6 +155,12 @@ To test whether a previously-built derivation is deterministic:
$ nix-build '<nixpkgs>' -A hello --check -K $ nix-build '<nixpkgs>' -A hello --check -K
``` ```
Use [`--read-log`](#operation---read-log) to show the stderr and stdout of a build:
```console
$ nix-store --read-log $(nix-instantiate ./test.nix)
```
# Operation `--serve` # Operation `--serve`
## Synopsis ## Synopsis

View file

@ -249,3 +249,36 @@ search/replaced in it for each new build.
The installer now supports a `--tarball-url-prefix` flag which _may_ have The installer now supports a `--tarball-url-prefix` flag which _may_ have
solved this need? solved this need?
--> -->
### Checking links in the manual
The build checks for broken internal links.
This happens late in the process, so `nix build` is not suitable for iterating.
To build the manual incrementally, run:
```console
make html -j $NIX_BUILD_CORES
```
In order to reflect changes to the [Makefile], clear all generated files before re-building:
[Makefile]: https://github.com/NixOS/nix/blob/master/doc/manual/local.mk
```console
rm $(git ls-files doc/manual/ -o | grep -F '.md') && rmdir doc/manual/src/command-ref/new-cli && make html -j $NIX_BUILD_CORES
```
[`mdbook-linkcheck`] does not implement checking [URI fragments] yet.
[`mdbook-linkcheck`]: https://github.com/Michael-F-Bryan/mdbook-linkcheck
[URI fragments]: https://en.m.wikipedia.org/wiki/URI_fragment
#### `@docroot@` variable
`@docroot@` provides a base path for links that occur in reusable snippets or other documentation that doesn't have a base path of its own.
If a broken link occurs in a snippet that was inserted into multiple generated files in different directories, use `@docroot@` to reference the `doc/manual/src` directory.
If the `@docroot@` literal appears in an error message from the `mdbook-linkcheck` tool, the `@docroot@` replacement needs to be applied to the generated source file that mentions it.
See existing `@docroot@` logic in the [Makefile].
Regular markdown files used for the manual have a base path of their own and they can use relative paths instead of `@docroot@`.

View file

@ -96,6 +96,7 @@
buildPackages.flex buildPackages.flex
(lib.getBin buildPackages.lowdown-nix) (lib.getBin buildPackages.lowdown-nix)
buildPackages.mdbook buildPackages.mdbook
buildPackages.mdbook-linkcheck
buildPackages.autoconf-archive buildPackages.autoconf-archive
buildPackages.autoreconfHook buildPackages.autoreconfHook
buildPackages.pkg-config buildPackages.pkg-config

View file

@ -36,17 +36,45 @@ Issues on the board progress through the following states:
- No Status - No Status
Team members can add pull requests or issues to discuss or review together.
During the discussion meeting, the team triages new items. During the discussion meeting, the team triages new items.
To be considered, issues and pull requests must have a high-level description to provide the whole team with the necessary context at a glance.
On every meeting, at least one item from each of the following categories is inspected:
1. [critical](https://github.com/NixOS/nix/labels/critical)
2. [security](https://github.com/NixOS/nix/labels/security)
3. [regression](https://github.com/NixOS/nix/labels/regression)
4. [bug](https://github.com/NixOS/nix/issues?q=is%3Aopen+label%3Abug+sort%3Areactions-%2B1-desc)
- [oldest pull requests](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+sort%3Acreated-asc)
- [most popular pull requests](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+sort%3Areactions-%2B1-desc)
- [oldest issues](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Acreated-asc)
- [most popular issues](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc)
Team members can also add pull requests or issues they would like the whole team to consider.
If there is disagreement on the general idea behind an issue or pull request, it is moved to _To discuss_, otherwise to _In review_. If there is disagreement on the general idea behind an issue or pull request, it is moved to _To discuss_, otherwise to _In review_.
- To discuss - To discuss
Pull requests and issues that are important and controversial are discussed by the team during discussion meetings. Pull requests and issues that are deemed important and controversial are discussed by the team during discussion meetings.
This may be where the merit of the change itself or the implementation strategy is contested by a team member. This may be where the merit of the change itself or the implementation strategy is contested by a team member.
As a general guideline, the order of items is determined as follows:
- Prioritise pull requests over issues
Contributors who took the time to implement concrete change proposals should not wait indefinitely.
- Prioritise fixing bugs over documentation, improvements or new features
The team values stability and accessibility higher than raw functionality.
- Interleave issues and PRs
This way issues without attempts at a solution get a chance to get addressed.
- In review - In review
Pull requests in this column are reviewed together during work meetings. Pull requests in this column are reviewed together during work meetings.

View file

@ -1,3 +1,15 @@
function add_path --argument-names new_path
if type -q fish_add_path
# fish 3.2.0 or newer
fish_add_path --prepend --global $new_path
else
# older versions of fish
if not contains $new_path $fish_user_paths
set --global fish_user_paths $new_path $fish_user_paths
end
end
end
# Only execute this file once per shell. # Only execute this file once per shell.
if test -n "$__ETC_PROFILE_NIX_SOURCED" if test -n "$__ETC_PROFILE_NIX_SOURCED"
exit exit
@ -31,5 +43,7 @@ else
end end
end end
fish_add_path --prepend --global "@localstatedir@/nix/profiles/default/bin" add_path "@localstatedir@/nix/profiles/default/bin"
fish_add_path --prepend --global "$HOME/.nix-profile/bin" add_path "$HOME/.nix-profile/bin"
functions -e add_path

View file

@ -1,3 +1,15 @@
function add_path --argument-names new_path
if type -q fish_add_path
# fish 3.2.0 or newer
fish_add_path --prepend --global $new_path
else
# older versions of fish
if not contains $new_path $fish_user_paths
set --global fish_user_paths $new_path $fish_user_paths
end
end
end
if test -n "$HOME" && test -n "$USER" if test -n "$HOME" && test -n "$USER"
# Set up the per-user profile. # Set up the per-user profile.
@ -32,6 +44,8 @@ if test -n "$HOME" && test -n "$USER"
set --export --prepend --path MANPATH "$NIX_LINK/share/man" set --export --prepend --path MANPATH "$NIX_LINK/share/man"
end end
fish_add_path --prepend --global "$NIX_LINK/bin" add_path "$NIX_LINK/bin"
set --erase NIX_LINK set --erase NIX_LINK
end end
functions -e add_path

View file

@ -34,8 +34,8 @@ MixEvalArgs::MixEvalArgs()
.shortName = 'I', .shortName = 'I',
.description = R"( .description = R"(
Add *path* to the Nix search path. The Nix search path is Add *path* to the Nix search path. The Nix search path is
initialized from the colon-separated [`NIX_PATH`](./env-common.md#env-NIX_PATH) environment initialized from the colon-separated [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH) environment
variable, and is used to look up the location of Nix expressions using [paths](../language/values.md#type-path) enclosed in angle variable, and is used to look up the location of Nix expressions using [paths](@docroot@/language/values.md#type-path) enclosed in angle
brackets (i.e., `<nixpkgs>`). brackets (i.e., `<nixpkgs>`).
For instance, passing For instance, passing

View file

@ -1,5 +1,6 @@
#include "globals.hh" #include "globals.hh"
#include "installables.hh" #include "installables.hh"
#include "outputs-spec.hh"
#include "util.hh" #include "util.hh"
#include "command.hh" #include "command.hh"
#include "attr-path.hh" #include "attr-path.hh"
@ -358,7 +359,7 @@ void completeFlakeRef(ref<Store> store, std::string_view prefix)
} }
} }
DerivedPath Installable::toDerivedPath() DerivedPathWithInfo Installable::toDerivedPath()
{ {
auto buildables = toDerivedPaths(); auto buildables = toDerivedPaths();
if (buildables.size() != 1) if (buildables.size() != 1)
@ -401,18 +402,6 @@ struct InstallableStorePath : Installable
ref<Store> store; ref<Store> store;
DerivedPath req; DerivedPath req;
InstallableStorePath(ref<Store> store, StorePath && storePath)
: store(store),
req(storePath.isDerivation()
? (DerivedPath) DerivedPath::Built {
.drvPath = std::move(storePath),
.outputs = {},
}
: (DerivedPath) DerivedPath::Opaque {
.path = std::move(storePath),
})
{ }
InstallableStorePath(ref<Store> store, DerivedPath && req) InstallableStorePath(ref<Store> store, DerivedPath && req)
: store(store), req(std::move(req)) : store(store), req(std::move(req))
{ } { }
@ -422,21 +411,9 @@ struct InstallableStorePath : Installable
return req.to_string(*store); return req.to_string(*store);
} }
DerivedPaths toDerivedPaths() override DerivedPathsWithInfo toDerivedPaths() override
{ {
return { req }; return {{.path = req, .info = {} }};
}
StorePathSet toDrvPaths(ref<Store> store) override
{
return std::visit(overloaded {
[&](const DerivedPath::Built & bfd) -> StorePathSet {
return { bfd.drvPath };
},
[&](const DerivedPath::Opaque & bo) -> StorePathSet {
return { getDeriver(store, *this, bo.path) };
},
}, req.raw());
} }
std::optional<StorePath> getStorePath() override std::optional<StorePath> getStorePath() override
@ -452,52 +429,24 @@ struct InstallableStorePath : Installable
} }
}; };
DerivedPaths InstallableValue::toDerivedPaths()
{
DerivedPaths res;
std::map<StorePath, std::set<std::string>> drvsToOutputs;
RealisedPath::Set drvsToCopy;
// Group by derivation, helps with .all in particular
for (auto & drv : toDerivations()) {
for (auto & outputName : drv.outputsToInstall)
drvsToOutputs[drv.drvPath].insert(outputName);
drvsToCopy.insert(drv.drvPath);
}
for (auto & i : drvsToOutputs)
res.push_back(DerivedPath::Built { i.first, i.second });
return res;
}
StorePathSet InstallableValue::toDrvPaths(ref<Store> store)
{
StorePathSet res;
for (auto & drv : toDerivations())
res.insert(drv.drvPath);
return res;
}
struct InstallableAttrPath : InstallableValue struct InstallableAttrPath : InstallableValue
{ {
SourceExprCommand & cmd; SourceExprCommand & cmd;
RootValue v; RootValue v;
std::string attrPath; std::string attrPath;
OutputsSpec outputsSpec; ExtendedOutputsSpec extendedOutputsSpec;
InstallableAttrPath( InstallableAttrPath(
ref<EvalState> state, ref<EvalState> state,
SourceExprCommand & cmd, SourceExprCommand & cmd,
Value * v, Value * v,
const std::string & attrPath, const std::string & attrPath,
OutputsSpec outputsSpec) ExtendedOutputsSpec extendedOutputsSpec)
: InstallableValue(state) : InstallableValue(state)
, cmd(cmd) , cmd(cmd)
, v(allocRootValue(v)) , v(allocRootValue(v))
, attrPath(attrPath) , attrPath(attrPath)
, outputsSpec(std::move(outputsSpec)) , extendedOutputsSpec(std::move(extendedOutputsSpec))
{ } { }
std::string what() const override { return attrPath; } std::string what() const override { return attrPath; }
@ -509,40 +458,54 @@ struct InstallableAttrPath : InstallableValue
return {vRes, pos}; return {vRes, pos};
} }
virtual std::vector<InstallableValue::DerivationInfo> toDerivations() override; DerivedPathsWithInfo toDerivedPaths() override
}; {
auto v = toValue(*state).first;
std::vector<InstallableValue::DerivationInfo> InstallableAttrPath::toDerivations() Bindings & autoArgs = *cmd.getAutoArgs(*state);
{
auto v = toValue(*state).first;
Bindings & autoArgs = *cmd.getAutoArgs(*state); DrvInfos drvInfos;
getDerivations(*state, *v, "", autoArgs, drvInfos, false);
DrvInfos drvInfos; // Backward compatibility hack: group results by drvPath. This
getDerivations(*state, *v, "", autoArgs, drvInfos, false); // helps keep .all output together.
std::map<StorePath, OutputsSpec> byDrvPath;
std::vector<DerivationInfo> res; for (auto & drvInfo : drvInfos) {
for (auto & drvInfo : drvInfos) { auto drvPath = drvInfo.queryDrvPath();
auto drvPath = drvInfo.queryDrvPath(); if (!drvPath)
if (!drvPath) throw Error("'%s' is not a derivation", what());
throw Error("'%s' is not a derivation", what());
std::set<std::string> outputsToInstall; auto newOutputs = std::visit(overloaded {
[&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec {
std::set<std::string> outputsToInstall;
for (auto & output : drvInfo.queryOutputs(false, true))
outputsToInstall.insert(output.first);
return OutputsSpec::Names { std::move(outputsToInstall) };
},
[&](const ExtendedOutputsSpec::Explicit & e) -> OutputsSpec {
return e;
},
}, extendedOutputsSpec.raw());
if (auto outputNames = std::get_if<OutputNames>(&outputsSpec)) auto [iter, didInsert] = byDrvPath.emplace(*drvPath, newOutputs);
outputsToInstall = *outputNames;
else
for (auto & output : drvInfo.queryOutputs(false, std::get_if<DefaultOutputs>(&outputsSpec)))
outputsToInstall.insert(output.first);
res.push_back(DerivationInfo { if (!didInsert)
.drvPath = *drvPath, iter->second = iter->second.union_(newOutputs);
.outputsToInstall = std::move(outputsToInstall) }
});
DerivedPathsWithInfo res;
for (auto & [drvPath, outputs] : byDrvPath)
res.push_back({
.path = DerivedPath::Built {
.drvPath = drvPath,
.outputs = outputs,
},
});
return res;
} }
};
return res;
}
std::vector<std::string> InstallableFlake::getActualAttrPaths() std::vector<std::string> InstallableFlake::getActualAttrPaths()
{ {
@ -615,7 +578,7 @@ InstallableFlake::InstallableFlake(
ref<EvalState> state, ref<EvalState> state,
FlakeRef && flakeRef, FlakeRef && flakeRef,
std::string_view fragment, std::string_view fragment,
OutputsSpec outputsSpec, ExtendedOutputsSpec extendedOutputsSpec,
Strings attrPaths, Strings attrPaths,
Strings prefixes, Strings prefixes,
const flake::LockFlags & lockFlags) const flake::LockFlags & lockFlags)
@ -623,14 +586,14 @@ InstallableFlake::InstallableFlake(
flakeRef(flakeRef), flakeRef(flakeRef),
attrPaths(fragment == "" ? attrPaths : Strings{(std::string) fragment}), attrPaths(fragment == "" ? attrPaths : Strings{(std::string) fragment}),
prefixes(fragment == "" ? Strings{} : prefixes), prefixes(fragment == "" ? Strings{} : prefixes),
outputsSpec(std::move(outputsSpec)), extendedOutputsSpec(std::move(extendedOutputsSpec)),
lockFlags(lockFlags) lockFlags(lockFlags)
{ {
if (cmd && cmd->getAutoArgs(*state)->size()) if (cmd && cmd->getAutoArgs(*state)->size())
throw UsageError("'--arg' and '--argstr' are incompatible with flakes"); throw UsageError("'--arg' and '--argstr' are incompatible with flakes");
} }
std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableFlake::toDerivation() DerivedPathsWithInfo InstallableFlake::toDerivedPaths()
{ {
Activity act(*logger, lvlTalkative, actUnknown, fmt("evaluating derivation '%s'", what())); Activity act(*logger, lvlTalkative, actUnknown, fmt("evaluating derivation '%s'", what()));
@ -638,56 +601,84 @@ std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableF
auto attrPath = attr->getAttrPathStr(); auto attrPath = attr->getAttrPathStr();
if (!attr->isDerivation()) if (!attr->isDerivation()) {
throw Error("flake output attribute '%s' is not a derivation", attrPath);
// FIXME: use eval cache?
auto v = attr->forceValue();
if (v.type() == nPath) {
PathSet context;
auto storePath = state->copyPathToStore(context, Path(v.path));
return {{
.path = DerivedPath::Opaque {
.path = std::move(storePath),
}
}};
}
else if (v.type() == nString) {
PathSet context;
auto s = state->forceString(v, context, noPos, fmt("while evaluating the flake output attribute '%s'", attrPath));
auto storePath = state->store->maybeParseStorePath(s);
if (storePath && context.count(std::string(s))) {
return {{
.path = DerivedPath::Opaque {
.path = std::move(*storePath),
}
}};
} else
throw Error("flake output attribute '%s' evaluates to the string '%s' which is not a store path", attrPath, s);
}
else
throw Error("flake output attribute '%s' is not a derivation or path", attrPath);
}
auto drvPath = attr->forceDerivation(); auto drvPath = attr->forceDerivation();
std::set<std::string> outputsToInstall;
std::optional<NixInt> priority; std::optional<NixInt> priority;
if (auto aOutputSpecified = attr->maybeGetAttr(state->sOutputSpecified)) { if (attr->maybeGetAttr(state->sOutputSpecified)) {
if (aOutputSpecified->getBool()) { } else if (auto aMeta = attr->maybeGetAttr(state->sMeta)) {
if (auto aOutputName = attr->maybeGetAttr("outputName"))
outputsToInstall = { aOutputName->getString() };
}
}
else if (auto aMeta = attr->maybeGetAttr(state->sMeta)) {
if (auto aOutputsToInstall = aMeta->maybeGetAttr("outputsToInstall"))
for (auto & s : aOutputsToInstall->getListOfStrings())
outputsToInstall.insert(s);
if (auto aPriority = aMeta->maybeGetAttr("priority")) if (auto aPriority = aMeta->maybeGetAttr("priority"))
priority = aPriority->getInt(); priority = aPriority->getInt();
} }
if (outputsToInstall.empty() || std::get_if<AllOutputs>(&outputsSpec)) { return {{
outputsToInstall.clear(); .path = DerivedPath::Built {
if (auto aOutputs = attr->maybeGetAttr(state->sOutputs)) .drvPath = std::move(drvPath),
for (auto & s : aOutputs->getListOfStrings()) .outputs = std::visit(overloaded {
outputsToInstall.insert(s); [&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec {
} std::set<std::string> outputsToInstall;
if (auto aOutputSpecified = attr->maybeGetAttr(state->sOutputSpecified)) {
if (aOutputSpecified->getBool()) {
if (auto aOutputName = attr->maybeGetAttr("outputName"))
outputsToInstall = { aOutputName->getString() };
}
} else if (auto aMeta = attr->maybeGetAttr(state->sMeta)) {
if (auto aOutputsToInstall = aMeta->maybeGetAttr("outputsToInstall"))
for (auto & s : aOutputsToInstall->getListOfStrings())
outputsToInstall.insert(s);
}
if (outputsToInstall.empty()) if (outputsToInstall.empty())
outputsToInstall.insert("out"); outputsToInstall.insert("out");
if (auto outputNames = std::get_if<OutputNames>(&outputsSpec)) return OutputsSpec::Names { std::move(outputsToInstall) };
outputsToInstall = *outputNames; },
[&](const ExtendedOutputsSpec::Explicit & e) -> OutputsSpec {
auto drvInfo = DerivationInfo { return e;
.drvPath = std::move(drvPath), },
.outputsToInstall = std::move(outputsToInstall), }, extendedOutputsSpec.raw()),
.priority = priority, },
}; .info = {
.priority = priority,
return {attrPath, getLockedFlake()->flake.lockedRef, std::move(drvInfo)}; .originalRef = flakeRef,
} .resolvedRef = getLockedFlake()->flake.lockedRef,
.attrPath = attrPath,
std::vector<InstallableValue::DerivationInfo> InstallableFlake::toDerivations() .extendedOutputsSpec = extendedOutputsSpec,
{ }
std::vector<DerivationInfo> res; }};
res.push_back(std::get<2>(toDerivation()));
return res;
} }
std::pair<Value *, PosIdx> InstallableFlake::toValue(EvalState & state) std::pair<Value *, PosIdx> InstallableFlake::toValue(EvalState & state)
@ -802,12 +793,12 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
} }
for (auto & s : ss) { for (auto & s : ss) {
auto [prefix, outputsSpec] = parseOutputsSpec(s); auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(s);
result.push_back( result.push_back(
std::make_shared<InstallableAttrPath>( std::make_shared<InstallableAttrPath>(
state, *this, vFile, state, *this, vFile,
prefix == "." ? "" : prefix, prefix == "." ? "" : std::string { prefix },
outputsSpec)); extendedOutputsSpec));
} }
} else { } else {
@ -815,24 +806,46 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
for (auto & s : ss) { for (auto & s : ss) {
std::exception_ptr ex; std::exception_ptr ex;
auto found = s.rfind('^'); auto [prefix_, extendedOutputsSpec_] = ExtendedOutputsSpec::parse(s);
if (found != std::string::npos) { // To avoid clang's pedantry
try { auto prefix = std::move(prefix_);
result.push_back(std::make_shared<InstallableStorePath>( auto extendedOutputsSpec = std::move(extendedOutputsSpec_);
store,
DerivedPath::Built::parse(*store, s.substr(0, found), s.substr(found + 1))));
continue;
} catch (BadStorePath &) {
} catch (...) {
if (!ex)
ex = std::current_exception();
}
}
found = s.find('/'); auto found = prefix.find('/');
if (found != std::string::npos) { if (found != std::string::npos) {
try { try {
result.push_back(std::make_shared<InstallableStorePath>(store, store->followLinksToStorePath(s))); auto derivedPath = std::visit(overloaded {
// If the user did not use ^, we treat the output more liberally.
[&](const ExtendedOutputsSpec::Default &) -> DerivedPath {
// First, we accept a symlink chain or an actual store path.
auto storePath = store->followLinksToStorePath(prefix);
// Second, we see if the store path ends in `.drv` to decide what sort
// of derived path they want.
//
// This handling predates the `^` syntax. The `^*` in
// `/nix/store/hash-foo.drv^*` unambiguously means "do the
// `DerivedPath::Built` case", so plain `/nix/store/hash-foo.drv` could
// also unambiguously mean "do the DerivedPath::Opaque` case".
//
// Issue #7261 tracks reconsidering this `.drv` dispatching.
return storePath.isDerivation()
? (DerivedPath) DerivedPath::Built {
.drvPath = std::move(storePath),
.outputs = OutputsSpec::All {},
}
: (DerivedPath) DerivedPath::Opaque {
.path = std::move(storePath),
};
},
// If the user did use ^, we just do exactly what is written.
[&](const ExtendedOutputsSpec::Explicit & outputSpec) -> DerivedPath {
return DerivedPath::Built {
.drvPath = store->parseStorePath(prefix),
.outputs = outputSpec,
};
},
}, extendedOutputsSpec.raw());
result.push_back(std::make_shared<InstallableStorePath>(store, std::move(derivedPath)));
continue; continue;
} catch (BadStorePath &) { } catch (BadStorePath &) {
} catch (...) { } catch (...) {
@ -842,13 +855,13 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
} }
try { try {
auto [flakeRef, fragment, outputsSpec] = parseFlakeRefWithFragmentAndOutputsSpec(s, absPath(".")); auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, absPath("."));
result.push_back(std::make_shared<InstallableFlake>( result.push_back(std::make_shared<InstallableFlake>(
this, this,
getEvalState(), getEvalState(),
std::move(flakeRef), std::move(flakeRef),
fragment, fragment,
outputsSpec, extendedOutputsSpec,
getDefaultFlakeAttrPaths(), getDefaultFlakeAttrPaths(),
getDefaultFlakeAttrPathPrefixes(), getDefaultFlakeAttrPathPrefixes(),
lockFlags)); lockFlags));
@ -895,13 +908,19 @@ std::vector<std::pair<std::shared_ptr<Installable>, BuiltPathWithResult>> Instal
if (mode == Realise::Nothing) if (mode == Realise::Nothing)
settings.readOnlyMode = true; settings.readOnlyMode = true;
struct Aux
{
ExtraPathInfo info;
std::shared_ptr<Installable> installable;
};
std::vector<DerivedPath> pathsToBuild; std::vector<DerivedPath> pathsToBuild;
std::map<DerivedPath, std::vector<std::shared_ptr<Installable>>> backmap; std::map<DerivedPath, std::vector<Aux>> backmap;
for (auto & i : installables) { for (auto & i : installables) {
for (auto b : i->toDerivedPaths()) { for (auto b : i->toDerivedPaths()) {
pathsToBuild.push_back(b); pathsToBuild.push_back(b.path);
backmap[b].push_back(i); backmap[b.path].push_back({.info = b.info, .installable = i});
} }
} }
@ -914,39 +933,18 @@ std::vector<std::pair<std::shared_ptr<Installable>, BuiltPathWithResult>> Instal
printMissing(store, pathsToBuild, lvlError); printMissing(store, pathsToBuild, lvlError);
for (auto & path : pathsToBuild) { for (auto & path : pathsToBuild) {
for (auto & installable : backmap[path]) { for (auto & aux : backmap[path]) {
std::visit(overloaded { std::visit(overloaded {
[&](const DerivedPath::Built & bfd) { [&](const DerivedPath::Built & bfd) {
OutputPathMap outputs; auto outputs = resolveDerivedPath(*store, bfd, &*evalStore);
auto drv = evalStore->readDerivation(bfd.drvPath); res.push_back({aux.installable, {
auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive .path = BuiltPath::Built { bfd.drvPath, outputs },
auto drvOutputs = drv.outputsAndOptPaths(*store); .info = aux.info}});
for (auto & output : bfd.outputs) {
auto outputHash = get(outputHashes, output);
if (!outputHash)
throw Error(
"the derivation '%s' doesn't have an output named '%s'",
store->printStorePath(bfd.drvPath), output);
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
DrvOutput outputId { *outputHash, output };
auto realisation = store->queryRealisation(outputId);
if (!realisation)
throw MissingRealisation(outputId);
outputs.insert_or_assign(output, realisation->outPath);
} else {
// If ca-derivations isn't enabled, assume that
// the output path is statically known.
auto drvOutput = get(drvOutputs, output);
assert(drvOutput);
assert(drvOutput->second);
outputs.insert_or_assign(
output, *drvOutput->second);
}
}
res.push_back({installable, {.path = BuiltPath::Built { bfd.drvPath, outputs }}});
}, },
[&](const DerivedPath::Opaque & bo) { [&](const DerivedPath::Opaque & bo) {
res.push_back({installable, {.path = BuiltPath::Opaque { bo.path }}}); res.push_back({aux.installable, {
.path = BuiltPath::Opaque { bo.path },
.info = aux.info}});
}, },
}, path.raw()); }, path.raw());
} }
@ -962,16 +960,22 @@ std::vector<std::pair<std::shared_ptr<Installable>, BuiltPathWithResult>> Instal
if (!buildResult.success()) if (!buildResult.success())
buildResult.rethrow(); buildResult.rethrow();
for (auto & installable : backmap[buildResult.path]) { for (auto & aux : backmap[buildResult.path]) {
std::visit(overloaded { std::visit(overloaded {
[&](const DerivedPath::Built & bfd) { [&](const DerivedPath::Built & bfd) {
std::map<std::string, StorePath> outputs; std::map<std::string, StorePath> outputs;
for (auto & path : buildResult.builtOutputs) for (auto & path : buildResult.builtOutputs)
outputs.emplace(path.first.outputName, path.second.outPath); outputs.emplace(path.first.outputName, path.second.outPath);
res.push_back({installable, {.path = BuiltPath::Built { bfd.drvPath, outputs }, .result = buildResult}}); res.push_back({aux.installable, {
.path = BuiltPath::Built { bfd.drvPath, outputs },
.info = aux.info,
.result = buildResult}});
}, },
[&](const DerivedPath::Opaque & bo) { [&](const DerivedPath::Opaque & bo) {
res.push_back({installable, {.path = BuiltPath::Opaque { bo.path }, .result = buildResult}}); res.push_back({aux.installable, {
.path = BuiltPath::Opaque { bo.path },
.info = aux.info,
.result = buildResult}});
}, },
}, buildResult.path.raw()); }, buildResult.path.raw());
} }
@ -1056,7 +1060,7 @@ StorePathSet Installable::toDerivations(
[&](const DerivedPath::Built & bfd) { [&](const DerivedPath::Built & bfd) {
drvPaths.insert(bfd.drvPath); drvPaths.insert(bfd.drvPath);
}, },
}, b.raw()); }, b.path.raw());
return drvPaths; return drvPaths;
} }

View file

@ -2,7 +2,7 @@
#include "util.hh" #include "util.hh"
#include "path.hh" #include "path.hh"
#include "path-with-outputs.hh" #include "outputs-spec.hh"
#include "derived-path.hh" #include "derived-path.hh"
#include "eval.hh" #include "eval.hh"
#include "store-api.hh" #include "store-api.hh"
@ -20,7 +20,7 @@ namespace eval_cache { class EvalCache; class AttrCursor; }
struct App struct App
{ {
std::vector<StorePathWithOutputs> context; std::vector<DerivedPath> context;
Path program; Path program;
// FIXME: add args, sandbox settings, metadata, ... // FIXME: add args, sandbox settings, metadata, ...
}; };
@ -52,26 +52,42 @@ enum class OperateOn {
Derivation Derivation
}; };
struct ExtraPathInfo
{
std::optional<NixInt> priority;
std::optional<FlakeRef> originalRef;
std::optional<FlakeRef> resolvedRef;
std::optional<std::string> attrPath;
// FIXME: merge with DerivedPath's 'outputs' field?
std::optional<ExtendedOutputsSpec> extendedOutputsSpec;
};
/* A derived path with any additional info that commands might
need from the derivation. */
struct DerivedPathWithInfo
{
DerivedPath path;
ExtraPathInfo info;
};
struct BuiltPathWithResult struct BuiltPathWithResult
{ {
BuiltPath path; BuiltPath path;
ExtraPathInfo info;
std::optional<BuildResult> result; std::optional<BuildResult> result;
}; };
typedef std::vector<DerivedPathWithInfo> DerivedPathsWithInfo;
struct Installable struct Installable
{ {
virtual ~Installable() { } virtual ~Installable() { }
virtual std::string what() const = 0; virtual std::string what() const = 0;
virtual DerivedPaths toDerivedPaths() = 0; virtual DerivedPathsWithInfo toDerivedPaths() = 0;
virtual StorePathSet toDrvPaths(ref<Store> store) DerivedPathWithInfo toDerivedPath();
{
throw Error("'%s' cannot be converted to a derivation path", what());
}
DerivedPath toDerivedPath();
UnresolvedApp toApp(EvalState & state); UnresolvedApp toApp(EvalState & state);
@ -146,19 +162,6 @@ struct InstallableValue : Installable
ref<EvalState> state; ref<EvalState> state;
InstallableValue(ref<EvalState> state) : state(state) {} InstallableValue(ref<EvalState> state) : state(state) {}
struct DerivationInfo
{
StorePath drvPath;
std::set<std::string> outputsToInstall;
std::optional<NixInt> priority;
};
virtual std::vector<DerivationInfo> toDerivations() = 0;
DerivedPaths toDerivedPaths() override;
StorePathSet toDrvPaths(ref<Store> store) override;
}; };
struct InstallableFlake : InstallableValue struct InstallableFlake : InstallableValue
@ -166,7 +169,7 @@ struct InstallableFlake : InstallableValue
FlakeRef flakeRef; FlakeRef flakeRef;
Strings attrPaths; Strings attrPaths;
Strings prefixes; Strings prefixes;
OutputsSpec outputsSpec; ExtendedOutputsSpec extendedOutputsSpec;
const flake::LockFlags & lockFlags; const flake::LockFlags & lockFlags;
mutable std::shared_ptr<flake::LockedFlake> _lockedFlake; mutable std::shared_ptr<flake::LockedFlake> _lockedFlake;
@ -175,7 +178,7 @@ struct InstallableFlake : InstallableValue
ref<EvalState> state, ref<EvalState> state,
FlakeRef && flakeRef, FlakeRef && flakeRef,
std::string_view fragment, std::string_view fragment,
OutputsSpec outputsSpec, ExtendedOutputsSpec extendedOutputsSpec,
Strings attrPaths, Strings attrPaths,
Strings prefixes, Strings prefixes,
const flake::LockFlags & lockFlags); const flake::LockFlags & lockFlags);
@ -186,9 +189,7 @@ struct InstallableFlake : InstallableValue
Value * getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake); Value * getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake);
std::tuple<std::string, FlakeRef, DerivationInfo> toDerivation(); DerivedPathsWithInfo toDerivedPaths() override;
std::vector<DerivationInfo> toDerivations() override;
std::pair<Value *, PosIdx> toValue(EvalState & state) override; std::pair<Value *, PosIdx> toValue(EvalState & state) override;

View file

@ -641,7 +641,12 @@ bool NixRepl::processLine(std::string line)
Path drvPathRaw = state->store->printStorePath(drvPath); Path drvPathRaw = state->store->printStorePath(drvPath);
if (command == ":b" || command == ":bl") { if (command == ":b" || command == ":bl") {
state->store->buildPaths({DerivedPath::Built{drvPath}}); state->store->buildPaths({
DerivedPath::Built {
.drvPath = drvPath,
.outputs = OutputsSpec::All { },
},
});
auto drv = state->store->readDerivation(drvPath); auto drv = state->store->readDerivation(drvPath);
logger->cout("\nThis derivation produced the following outputs:"); logger->cout("\nThis derivation produced the following outputs:");
for (auto & [outputName, outputPath] : state->store->queryDerivationOutputMap(drvPath)) { for (auto & [outputName, outputPath] : state->store->queryDerivationOutputMap(drvPath)) {

View file

@ -300,7 +300,7 @@ struct AttrDb
NixStringContext context; NixStringContext context;
if (!queryAttribute.isNull(3)) if (!queryAttribute.isNull(3))
for (auto & s : tokenizeString<std::vector<std::string>>(queryAttribute.getStr(3), ";")) for (auto & s : tokenizeString<std::vector<std::string>>(queryAttribute.getStr(3), ";"))
context.push_back(decodeContext(cfg, s)); context.push_back(NixStringContextElem::parse(cfg, s));
return {{rowId, string_t{queryAttribute.getStr(2), context}}}; return {{rowId, string_t{queryAttribute.getStr(2), context}}};
} }
case AttrType::Bool: case AttrType::Bool:
@ -592,7 +592,18 @@ string_t AttrCursor::getStringWithContext()
if (auto s = std::get_if<string_t>(&cachedValue->second)) { if (auto s = std::get_if<string_t>(&cachedValue->second)) {
bool valid = true; bool valid = true;
for (auto & c : s->second) { for (auto & c : s->second) {
if (!root->state.store->isValidPath(c.first)) { const StorePath & path = std::visit(overloaded {
[&](const NixStringContextElem::DrvDeep & d) -> const StorePath & {
return d.drvPath;
},
[&](const NixStringContextElem::Built & b) -> const StorePath & {
return b.drvPath;
},
[&](const NixStringContextElem::Opaque & o) -> const StorePath & {
return o.path;
},
}, c.raw());
if (!root->state.store->isValidPath(path)) {
valid = false; valid = false;
break; break;
} }

View file

@ -2068,27 +2068,6 @@ std::string_view EvalState::forceString(Value & v, const PosIdx pos, std::string
} }
/* Decode a context string !<name>!<path> into a pair <path,
name>. */
NixStringContextElem decodeContext(const Store & store, std::string_view s)
{
if (s.at(0) == '!') {
size_t index = s.find("!", 1);
return {
store.parseStorePath(s.substr(index + 1)),
std::string(s.substr(1, index - 1)),
};
} else
return {
store.parseStorePath(
s.at(0) == '/'
? s
: s.substr(1)),
"",
};
}
void copyContext(const Value & v, PathSet & context) void copyContext(const Value & v, PathSet & context)
{ {
if (v.string.context) if (v.string.context)
@ -2103,7 +2082,7 @@ NixStringContext Value::getContext(const Store & store)
assert(internalType == tString); assert(internalType == tString);
if (string.context) if (string.context)
for (const char * * p = string.context; *p; ++p) for (const char * * p = string.context; *p; ++p)
res.push_back(decodeContext(store, *p)); res.push_back(NixStringContextElem::parse(store, *p));
return res; return res;
} }
@ -2166,7 +2145,7 @@ BackedStringView EvalState::coerceToString(const PosIdx pos, Value & v, PathSet
if (canonicalizePath) if (canonicalizePath)
path = canonPath(*path); path = canonPath(*path);
if (copyToStore) if (copyToStore)
path = copyPathToStore(context, std::move(path).toOwned()); path = store->printStorePath(copyPathToStore(context, std::move(path).toOwned()));
return path; return path;
} }
@ -2215,26 +2194,26 @@ BackedStringView EvalState::coerceToString(const PosIdx pos, Value & v, PathSet
} }
std::string EvalState::copyPathToStore(PathSet & context, const Path & path) StorePath EvalState::copyPathToStore(PathSet & context, const Path & path)
{ {
if (nix::isDerivation(path)) if (nix::isDerivation(path))
error("file names are not allowed to end in '%1%'", drvExtension).debugThrow<EvalError>(); error("file names are not allowed to end in '%1%'", drvExtension).debugThrow<EvalError>();
Path dstPath; auto dstPath = [&]() -> StorePath
auto i = srcToStore.find(path); {
if (i != srcToStore.end()) auto i = srcToStore.find(path);
dstPath = store->printStorePath(i->second); if (i != srcToStore.end()) return i->second;
else {
auto p = settings.readOnlyMode auto dstPath = settings.readOnlyMode
? store->computeStorePathForPath(std::string(baseNameOf(path)), checkSourcePath(path)).first ? store->computeStorePathForPath(std::string(baseNameOf(path)), checkSourcePath(path)).first
: store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, repair); : store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, repair);
dstPath = store->printStorePath(p); allowPath(dstPath);
allowPath(p); srcToStore.insert_or_assign(path, dstPath);
srcToStore.insert_or_assign(path, std::move(p)); printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath));
printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, dstPath); return dstPath;
} }();
context.insert(dstPath); context.insert(store->printStorePath(dstPath));
return dstPath; return dstPath;
} }

View file

@ -379,7 +379,7 @@ public:
bool canonicalizePath = true, bool canonicalizePath = true,
std::string_view errorCtx = ""); std::string_view errorCtx = "");
std::string copyPathToStore(PathSet & context, const Path & path); StorePath copyPathToStore(PathSet & context, const Path & path);
/* Path coercion. Converts strings, paths and derivations to a /* Path coercion. Converts strings, paths and derivations to a
path. The result is guaranteed to be a canonicalised, absolute path. The result is guaranteed to be a canonicalised, absolute
@ -551,10 +551,6 @@ struct DebugTraceStacker {
std::string_view showType(ValueType type); std::string_view showType(ValueType type);
std::string showType(const Value & v); std::string showType(const Value & v);
/* Decode a context string !<name>!<path> into a pair <path,
name>. */
NixStringContextElem decodeContext(const Store & store, std::string_view s);
/* If `path' refers to a directory, then append "/default.nix". */ /* If `path' refers to a directory, then append "/default.nix". */
Path resolveExprPath(Path path); Path resolveExprPath(Path path);

View file

@ -238,15 +238,15 @@ std::pair<fetchers::Tree, FlakeRef> FlakeRef::fetchTree(ref<Store> store) const
return {std::move(tree), FlakeRef(std::move(lockedInput), subdir)}; return {std::move(tree), FlakeRef(std::move(lockedInput), subdir)};
} }
std::tuple<FlakeRef, std::string, OutputsSpec> parseFlakeRefWithFragmentAndOutputsSpec( std::tuple<FlakeRef, std::string, ExtendedOutputsSpec> parseFlakeRefWithFragmentAndExtendedOutputsSpec(
const std::string & url, const std::string & url,
const std::optional<Path> & baseDir, const std::optional<Path> & baseDir,
bool allowMissing, bool allowMissing,
bool isFlake) bool isFlake)
{ {
auto [prefix, outputsSpec] = parseOutputsSpec(url); auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(url);
auto [flakeRef, fragment] = parseFlakeRefWithFragment(prefix, baseDir, allowMissing, isFlake); auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, baseDir, allowMissing, isFlake);
return {std::move(flakeRef), fragment, outputsSpec}; return {std::move(flakeRef), fragment, extendedOutputsSpec};
} }
} }

View file

@ -3,7 +3,7 @@
#include "types.hh" #include "types.hh"
#include "hash.hh" #include "hash.hh"
#include "fetchers.hh" #include "fetchers.hh"
#include "path-with-outputs.hh" #include "outputs-spec.hh"
#include <variant> #include <variant>
@ -80,7 +80,7 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
std::optional<std::pair<FlakeRef, std::string>> maybeParseFlakeRefWithFragment( std::optional<std::pair<FlakeRef, std::string>> maybeParseFlakeRefWithFragment(
const std::string & url, const std::optional<Path> & baseDir = {}); const std::string & url, const std::optional<Path> & baseDir = {});
std::tuple<FlakeRef, std::string, OutputsSpec> parseFlakeRefWithFragmentAndOutputsSpec( std::tuple<FlakeRef, std::string, ExtendedOutputsSpec> parseFlakeRefWithFragmentAndExtendedOutputsSpec(
const std::string & url, const std::string & url,
const std::optional<Path> & baseDir = {}, const std::optional<Path> & baseDir = {},
bool allowMissing = false, bool allowMissing = false,

View file

@ -6,6 +6,7 @@ libexpr_DIR := $(d)
libexpr_SOURCES := \ libexpr_SOURCES := \
$(wildcard $(d)/*.cc) \ $(wildcard $(d)/*.cc) \
$(wildcard $(d)/value/*.cc) \
$(wildcard $(d)/primops/*.cc) \ $(wildcard $(d)/primops/*.cc) \
$(wildcard $(d)/flake/*.cc) \ $(wildcard $(d)/flake/*.cc) \
$(d)/lexer-tab.cc \ $(d)/lexer-tab.cc \
@ -37,6 +38,8 @@ clean-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexe
$(eval $(call install-file-in, $(d)/nix-expr.pc, $(libdir)/pkgconfig, 0644)) $(eval $(call install-file-in, $(d)/nix-expr.pc, $(libdir)/pkgconfig, 0644))
$(foreach i, $(wildcard src/libexpr/value/*.hh), \
$(eval $(call install-file-in, $(i), $(includedir)/nix/value, 0644)))
$(foreach i, $(wildcard src/libexpr/flake/*.hh), \ $(foreach i, $(wildcard src/libexpr/flake/*.hh), \
$(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644))) $(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644)))

View file

@ -43,16 +43,32 @@ StringMap EvalState::realiseContext(const PathSet & context)
std::vector<DerivedPath::Built> drvs; std::vector<DerivedPath::Built> drvs;
StringMap res; StringMap res;
for (auto & i : context) { for (auto & c_ : context) {
auto [ctx, outputName] = decodeContext(*store, i); auto ensureValid = [&](const StorePath & p) {
auto ctxS = store->printStorePath(ctx); if (!store->isValidPath(p))
if (!store->isValidPath(ctx)) debugThrowLastTrace(InvalidPathError(store->printStorePath(p)));
debugThrowLastTrace(InvalidPathError(store->printStorePath(ctx))); };
if (!outputName.empty() && ctx.isDerivation()) { auto c = NixStringContextElem::parse(*store, c_);
drvs.push_back({ctx, {outputName}}); std::visit(overloaded {
} else { [&](const NixStringContextElem::Built & b) {
res.insert_or_assign(ctxS, ctxS); drvs.push_back(DerivedPath::Built {
} .drvPath = b.drvPath,
.outputs = OutputsSpec::Names { b.output },
});
ensureValid(b.drvPath);
},
[&](const NixStringContextElem::Opaque & o) {
auto ctxS = store->printStorePath(o.path);
res.insert_or_assign(ctxS, ctxS);
ensureValid(o.path);
},
[&](const NixStringContextElem::DrvDeep & d) {
/* Treat same as Opaque */
auto ctxS = store->printStorePath(d.drvPath);
res.insert_or_assign(ctxS, ctxS);
ensureValid(d.drvPath);
},
}, c.raw());
} }
if (drvs.empty()) return {}; if (drvs.empty()) return {};
@ -68,16 +84,12 @@ StringMap EvalState::realiseContext(const PathSet & context)
store->buildPaths(buildReqs); store->buildPaths(buildReqs);
/* Get all the output paths corresponding to the placeholders we had */ /* Get all the output paths corresponding to the placeholders we had */
for (auto & [drvPath, outputs] : drvs) { for (auto & drv : drvs) {
const auto outputPaths = store->queryDerivationOutputMap(drvPath); auto outputs = resolveDerivedPath(*store, drv);
for (auto & outputName : outputs) { for (auto & [outputName, outputPath] : outputs) {
auto outputPath = get(outputPaths, outputName);
if (!outputPath)
debugThrowLastTrace(Error("derivation '%s' does not have an output named '%s'",
store->printStorePath(drvPath), outputName));
res.insert_or_assign( res.insert_or_assign(
downstreamPlaceholder(*store, drvPath, outputName), downstreamPlaceholder(*store, drv.drvPath, outputName),
store->printStorePath(*outputPath) store->printStorePath(outputPath)
); );
} }
} }
@ -240,6 +252,7 @@ static RegisterPrimOp primop_scopedImport(RegisterPrimOp::Info {
static RegisterPrimOp primop_import({ static RegisterPrimOp primop_import({
.name = "import", .name = "import",
.args = {"path"}, .args = {"path"},
// TODO turn "normal path values" into link below
.doc = R"( .doc = R"(
Load, parse and return the Nix expression in the file *path*. If Load, parse and return the Nix expression in the file *path*. If
*path* is a directory, the file ` default.nix ` in that directory *path* is a directory, the file ` default.nix ` in that directory
@ -253,7 +266,7 @@ static RegisterPrimOp primop_import({
> >
> Unlike some languages, `import` is a regular function in Nix. > Unlike some languages, `import` is a regular function in Nix.
> Paths using the angle bracket syntax (e.g., `import` *\<foo\>*) > Paths using the angle bracket syntax (e.g., `import` *\<foo\>*)
> are [normal path values](language-values.md). > are normal [path values](@docroot@/language/values.md#type-path).
A Nix expression loaded by `import` must not contain any *free A Nix expression loaded by `import` must not contain any *free
variables* (identifiers that are not defined in the Nix expression variables* (identifiers that are not defined in the Nix expression
@ -1179,35 +1192,31 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * *
/* Everything in the context of the strings in the derivation /* Everything in the context of the strings in the derivation
attributes should be added as dependencies of the resulting attributes should be added as dependencies of the resulting
derivation. */ derivation. */
for (auto & path : context) { for (auto & c_ : context) {
auto c = NixStringContextElem::parse(*state.store, c_);
/* Paths marked with `=' denote that the path of a derivation std::visit(overloaded {
is explicitly passed to the builder. Since that allows the /* Since this allows the builder to gain access to every
builder to gain access to every path in the dependency path in the dependency graph of the derivation (including
graph of the derivation (including all outputs), all paths all outputs), all paths in the graph must be added to
in the graph must be added to this derivation's list of this derivation's list of inputs to ensure that they are
inputs to ensure that they are available when the builder available when the builder runs. */
runs. */ [&](const NixStringContextElem::DrvDeep & d) {
if (path.at(0) == '=') { /* !!! This doesn't work if readOnlyMode is set. */
/* !!! This doesn't work if readOnlyMode is set. */ StorePathSet refs;
StorePathSet refs; state.store->computeFSClosure(d.drvPath, refs);
state.store->computeFSClosure(state.store->parseStorePath(std::string_view(path).substr(1)), refs); for (auto & j : refs) {
for (auto & j : refs) { drv.inputSrcs.insert(j);
drv.inputSrcs.insert(j); if (j.isDerivation())
if (j.isDerivation()) drv.inputDrvs[j] = state.store->readDerivation(j).outputNames();
drv.inputDrvs[j] = state.store->readDerivation(j).outputNames(); }
} },
} [&](const NixStringContextElem::Built & b) {
drv.inputDrvs[b.drvPath].insert(b.output);
/* Handle derivation outputs of the form !<name>!<path>. */ },
else if (path.at(0) == '!') { [&](const NixStringContextElem::Opaque & o) {
auto ctx = decodeContext(*state.store, path); drv.inputSrcs.insert(o.path);
drv.inputDrvs[ctx.first].insert(ctx.second); },
} }, c.raw());
/* Otherwise it's a source file. */
else
drv.inputSrcs.insert(state.store->parseStorePath(path));
} }
/* Do we have all required attributes? */ /* Do we have all required attributes? */
@ -1879,8 +1888,7 @@ static RegisterPrimOp primop_toFile({
path. The file has suffix *name*. This file can be used as an path. The file has suffix *name*. This file can be used as an
input to derivations. One application is to write builders input to derivations. One application is to write builders
inline. For instance, the following Nix expression combines the inline. For instance, the following Nix expression combines the
[Nix expression for GNU Hello](expression-syntax.md) and its Nix expression for GNU Hello and its build script into one file:
[build script](build-script.md) into one file:
```nix ```nix
{ stdenv, fetchurl, perl }: { stdenv, fetchurl, perl }:
@ -1924,7 +1932,7 @@ static RegisterPrimOp primop_toFile({
``` ```
Note that `${configFile}` is a Note that `${configFile}` is a
[string interpolation](language/values.md#type-string), so the result of the [string interpolation](@docroot@/language/values.md#type-string), so the result of the
expression `configFile` expression `configFile`
(i.e., a path like `/nix/store/m7p7jfny445k...-foo.conf`) will be (i.e., a path like `/nix/store/m7p7jfny445k...-foo.conf`) will be
spliced into the resulting string. spliced into the resulting string.
@ -2820,7 +2828,7 @@ static RegisterPrimOp primop_map({
example, example,
```nix ```nix
map (x"foo" + x) [ "bar" "bla" "abc" ] map (x: "foo" + x) [ "bar" "bla" "abc" ]
``` ```
evaluates to `[ "foobar" "foobla" "fooabc" ]`. evaluates to `[ "foobar" "foobla" "fooabc" ]`.

View file

@ -37,8 +37,15 @@ static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx p
auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.unsafeDiscardOutputDependency"); auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.unsafeDiscardOutputDependency");
PathSet context2; PathSet context2;
for (auto & p : context) for (auto && p : context) {
context2.insert(p.at(0) == '=' ? std::string(p, 1) : p); auto c = NixStringContextElem::parse(*state.store, p);
if (auto * ptr = std::get_if<NixStringContextElem::DrvDeep>(&c)) {
context2.emplace(state.store->printStorePath(ptr->drvPath));
} else {
/* Can reuse original item */
context2.emplace(std::move(p));
}
}
v.mkString(*s, context2); v.mkString(*s, context2);
} }
@ -74,34 +81,22 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args,
}; };
PathSet context; PathSet context;
state.forceString(*args[0], context, pos, "while evaluating the argument passed to builtins.getContext"); state.forceString(*args[0], context, pos, "while evaluating the argument passed to builtins.getContext");
auto contextInfos = std::map<Path, ContextInfo>(); auto contextInfos = std::map<StorePath, ContextInfo>();
for (const auto & p : context) { for (const auto & p : context) {
Path drv; Path drv;
std::string output; std::string output;
const Path * path = &p; NixStringContextElem ctx = NixStringContextElem::parse(*state.store, p);
if (p.at(0) == '=') { std::visit(overloaded {
drv = std::string(p, 1); [&](NixStringContextElem::DrvDeep & d) {
path = &drv; contextInfos[d.drvPath].allOutputs = true;
} else if (p.at(0) == '!') { },
NixStringContextElem ctx = decodeContext(*state.store, p); [&](NixStringContextElem::Built & b) {
drv = state.store->printStorePath(ctx.first); contextInfos[b.drvPath].outputs.emplace_back(std::move(output));
output = ctx.second; },
path = &drv; [&](NixStringContextElem::Opaque & o) {
} contextInfos[o.path].path = true;
auto isPath = drv.empty(); },
auto isAllOutputs = (!drv.empty()) && output.empty(); }, ctx.raw());
auto iter = contextInfos.find(*path);
if (iter == contextInfos.end()) {
contextInfos.emplace(*path, ContextInfo{isPath, isAllOutputs, output.empty() ? Strings{} : Strings{std::move(output)}});
} else {
if (isPath)
iter->second.path = true;
else if (isAllOutputs)
iter->second.allOutputs = true;
else
iter->second.outputs.emplace_back(std::move(output));
}
} }
auto attrs = state.buildBindings(contextInfos.size()); auto attrs = state.buildBindings(contextInfos.size());
@ -120,7 +115,7 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args,
for (const auto & [i, output] : enumerate(info.second.outputs)) for (const auto & [i, output] : enumerate(info.second.outputs))
(outputsVal.listElems()[i] = state.allocValue())->mkString(output); (outputsVal.listElems()[i] = state.allocValue())->mkString(output);
} }
attrs.alloc(info.first).mkAttrs(infoAttrs); attrs.alloc(state.store->printStorePath(info.first)).mkAttrs(infoAttrs);
} }
v.mkAttrs(attrs); v.mkAttrs(attrs);

View file

@ -6,7 +6,9 @@ libexpr-tests_DIR := $(d)
libexpr-tests_INSTALL_DIR := libexpr-tests_INSTALL_DIR :=
libexpr-tests_SOURCES := $(wildcard $(d)/*.cc) libexpr-tests_SOURCES := \
$(wildcard $(d)/*.cc) \
$(wildcard $(d)/value/*.cc)
libexpr-tests_CXXFLAGS += -I src/libexpr -I src/libutil -I src/libstore -I src/libexpr/tests libexpr-tests_CXXFLAGS += -I src/libexpr -I src/libutil -I src/libstore -I src/libexpr/tests

View file

@ -0,0 +1,72 @@
#include "value/context.hh"
#include "libexprtests.hh"
namespace nix {
// Testing of trivial expressions
struct NixStringContextElemTest : public LibExprTest {
const Store & store() const {
return *LibExprTest::store;
}
};
TEST_F(NixStringContextElemTest, empty_invalid) {
EXPECT_THROW(
NixStringContextElem::parse(store(), ""),
BadNixStringContextElem);
}
TEST_F(NixStringContextElemTest, single_bang_invalid) {
EXPECT_THROW(
NixStringContextElem::parse(store(), "!"),
BadNixStringContextElem);
}
TEST_F(NixStringContextElemTest, double_bang_invalid) {
EXPECT_THROW(
NixStringContextElem::parse(store(), "!!/"),
BadStorePath);
}
TEST_F(NixStringContextElemTest, eq_slash_invalid) {
EXPECT_THROW(
NixStringContextElem::parse(store(), "=/"),
BadStorePath);
}
TEST_F(NixStringContextElemTest, slash_invalid) {
EXPECT_THROW(
NixStringContextElem::parse(store(), "/"),
BadStorePath);
}
TEST_F(NixStringContextElemTest, opaque) {
std::string_view opaque = "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x";
auto elem = NixStringContextElem::parse(store(), opaque);
auto * p = std::get_if<NixStringContextElem::Opaque>(&elem);
ASSERT_TRUE(p);
ASSERT_EQ(p->path, store().parseStorePath(opaque));
ASSERT_EQ(elem.to_string(store()), opaque);
}
TEST_F(NixStringContextElemTest, drvDeep) {
std::string_view drvDeep = "=/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv";
auto elem = NixStringContextElem::parse(store(), drvDeep);
auto * p = std::get_if<NixStringContextElem::DrvDeep>(&elem);
ASSERT_TRUE(p);
ASSERT_EQ(p->drvPath, store().parseStorePath(drvDeep.substr(1)));
ASSERT_EQ(elem.to_string(store()), drvDeep);
}
TEST_F(NixStringContextElemTest, built) {
std::string_view built = "!foo!/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv";
auto elem = NixStringContextElem::parse(store(), built);
auto * p = std::get_if<NixStringContextElem::Built>(&elem);
ASSERT_TRUE(p);
ASSERT_EQ(p->output, "foo");
ASSERT_EQ(p->drvPath, store().parseStorePath(built.substr(5)));
ASSERT_EQ(elem.to_string(store()), built);
}
}

View file

@ -1,6 +1,7 @@
#include "value-to-json.hh" #include "value-to-json.hh"
#include "eval-inline.hh" #include "eval-inline.hh"
#include "util.hh" #include "util.hh"
#include "store-api.hh"
#include <cstdlib> #include <cstdlib>
#include <iomanip> #include <iomanip>
@ -35,7 +36,7 @@ json printValueAsJSON(EvalState & state, bool strict,
case nPath: case nPath:
if (copyToStore) if (copyToStore)
out = state.copyPathToStore(context, v.path); out = state.store->printStorePath(state.copyPathToStore(context, v.path));
else else
out = v.path; out = v.path;
break; break;

View file

@ -3,6 +3,7 @@
#include <cassert> #include <cassert>
#include "symbol-table.hh" #include "symbol-table.hh"
#include "value/context.hh"
#if HAVE_BOEHMGC #if HAVE_BOEHMGC
#include <gc/gc_allocator.h> #include <gc/gc_allocator.h>
@ -67,8 +68,6 @@ class XMLWriter;
typedef int64_t NixInt; typedef int64_t NixInt;
typedef double NixFloat; typedef double NixFloat;
typedef std::pair<StorePath, std::string> NixStringContextElem;
typedef std::vector<NixStringContextElem> NixStringContext;
/* External values must descend from ExternalValueBase, so that /* External values must descend from ExternalValueBase, so that
* type-agnostic nix functions (e.g. showType) can be implemented * type-agnostic nix functions (e.g. showType) can be implemented

View file

@ -0,0 +1,67 @@
#include "value/context.hh"
#include "store-api.hh"
#include <optional>
namespace nix {
NixStringContextElem NixStringContextElem::parse(const Store & store, std::string_view s0)
{
std::string_view s = s0;
if (s.size() == 0) {
throw BadNixStringContextElem(s0,
"String context element should never be an empty string");
}
switch (s.at(0)) {
case '!': {
s = s.substr(1); // advance string to parse after first !
size_t index = s.find("!");
// This makes index + 1 safe. Index can be the length (one after index
// of last character), so given any valid character index --- a
// successful find --- we can add one.
if (index == std::string_view::npos) {
throw BadNixStringContextElem(s0,
"String content element beginning with '!' should have a second '!'");
}
return NixStringContextElem::Built {
.drvPath = store.parseStorePath(s.substr(index + 1)),
.output = std::string(s.substr(0, index)),
};
}
case '=': {
return NixStringContextElem::DrvDeep {
.drvPath = store.parseStorePath(s.substr(1)),
};
}
default: {
return NixStringContextElem::Opaque {
.path = store.parseStorePath(s),
};
}
}
}
std::string NixStringContextElem::to_string(const Store & store) const {
return std::visit(overloaded {
[&](const NixStringContextElem::Built & b) {
std::string res;
res += '!';
res += b.output;
res += '!';
res += store.printStorePath(b.drvPath);
return res;
},
[&](const NixStringContextElem::DrvDeep & d) {
std::string res;
res += '=';
res += store.printStorePath(d.drvPath);
return res;
},
[&](const NixStringContextElem::Opaque & o) {
return store.printStorePath(o.path);
},
}, raw());
}
}

View file

@ -0,0 +1,90 @@
#pragma once
#include "util.hh"
#include "path.hh"
#include <optional>
#include <nlohmann/json_fwd.hpp>
namespace nix {
class BadNixStringContextElem : public Error
{
public:
std::string_view raw;
template<typename... Args>
BadNixStringContextElem(std::string_view raw_, const Args & ... args)
: Error("")
{
raw = raw_;
auto hf = hintfmt(args...);
err.msg = hintfmt("Bad String Context element: %1%: %2%", normaltxt(hf.str()), raw);
}
};
class Store;
/* Plain opaque path to some store object.
Encoded as just the path: <path>.
*/
struct NixStringContextElem_Opaque {
StorePath path;
};
/* Path to a derivation and its entire build closure.
The path doesn't just refer to derivation itself and its closure, but
also all outputs of all derivations in that closure (including the
root derivation).
Encoded in the form =<drvPath>.
*/
struct NixStringContextElem_DrvDeep {
StorePath drvPath;
};
/* Derivation output.
Encoded in the form !<output>!<drvPath>.
*/
struct NixStringContextElem_Built {
StorePath drvPath;
std::string output;
};
using _NixStringContextElem_Raw = std::variant<
NixStringContextElem_Opaque,
NixStringContextElem_DrvDeep,
NixStringContextElem_Built
>;
struct NixStringContextElem : _NixStringContextElem_Raw {
using Raw = _NixStringContextElem_Raw;
using Raw::Raw;
using Opaque = NixStringContextElem_Opaque;
using DrvDeep = NixStringContextElem_DrvDeep;
using Built = NixStringContextElem_Built;
inline const Raw & raw() const {
return static_cast<const Raw &>(*this);
}
inline Raw & raw() {
return static_cast<Raw &>(*this);
}
/* Decode a context string, one of:
- <path>
- =<path>
- !<name>!<path>
*/
static NixStringContextElem parse(const Store & store, std::string_view s);
std::string to_string(const Store & store) const;
};
typedef std::vector<NixStringContextElem> NixStringContext;
}

View file

@ -39,7 +39,6 @@ void printVersion(const std::string & programName);
void printGCWarning(); void printGCWarning();
class Store; class Store;
struct StorePathWithOutputs;
void printMissing( void printMissing(
ref<Store> store, ref<Store> store,

View file

@ -530,22 +530,9 @@ void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSe
writeNarInfo(narInfo); writeNarInfo(narInfo);
} }
std::optional<std::string> BinaryCacheStore::getBuildLog(const StorePath & path) std::optional<std::string> BinaryCacheStore::getBuildLogExact(const StorePath & path)
{ {
auto drvPath = path; auto logPath = "log/" + std::string(baseNameOf(printStorePath(path)));
if (!path.isDerivation()) {
try {
auto info = queryPathInfo(path);
// FIXME: add a "Log" field to .narinfo
if (!info->deriver) return std::nullopt;
drvPath = *info->deriver;
} catch (InvalidPath &) {
return std::nullopt;
}
}
auto logPath = "log/" + std::string(baseNameOf(printStorePath(drvPath)));
debug("fetching build log from binary cache '%s/%s'", getUri(), logPath); debug("fetching build log from binary cache '%s/%s'", getUri(), logPath);

View file

@ -129,7 +129,7 @@ public:
void addSignatures(const StorePath & storePath, const StringSet & sigs) override; void addSignatures(const StorePath & storePath, const StringSet & sigs) override;
std::optional<std::string> getBuildLog(const StorePath & path) override; std::optional<std::string> getBuildLogExact(const StorePath & path) override;
void addBuildLog(const StorePath & drvPath, std::string_view log) override; void addBuildLog(const StorePath & drvPath, std::string_view log) override;

View file

@ -63,7 +63,7 @@
namespace nix { namespace nix {
DerivationGoal::DerivationGoal(const StorePath & drvPath, DerivationGoal::DerivationGoal(const StorePath & drvPath,
const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode)
: Goal(worker, DerivedPath::Built { .drvPath = drvPath, .outputs = wantedOutputs }) : Goal(worker, DerivedPath::Built { .drvPath = drvPath, .outputs = wantedOutputs })
, useDerivation(true) , useDerivation(true)
, drvPath(drvPath) , drvPath(drvPath)
@ -82,7 +82,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath,
DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode)
: Goal(worker, DerivedPath::Built { .drvPath = drvPath, .outputs = wantedOutputs }) : Goal(worker, DerivedPath::Built { .drvPath = drvPath, .outputs = wantedOutputs })
, useDerivation(false) , useDerivation(false)
, drvPath(drvPath) , drvPath(drvPath)
@ -142,18 +142,12 @@ void DerivationGoal::work()
(this->*state)(); (this->*state)();
} }
void DerivationGoal::addWantedOutputs(const StringSet & outputs) void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs)
{ {
/* If we already want all outputs, there is nothing to do. */ auto newWanted = wantedOutputs.union_(outputs);
if (wantedOutputs.empty()) return; if (!newWanted.isSubsetOf(wantedOutputs))
if (outputs.empty()) {
wantedOutputs.clear();
needRestart = true; needRestart = true;
} else wantedOutputs = newWanted;
for (auto & i : outputs)
if (wantedOutputs.insert(i).second)
needRestart = true;
} }
@ -390,7 +384,7 @@ void DerivationGoal::repairClosure()
auto outputs = queryDerivationOutputMap(); auto outputs = queryDerivationOutputMap();
StorePathSet outputClosure; StorePathSet outputClosure;
for (auto & i : outputs) { for (auto & i : outputs) {
if (!wantOutput(i.first, wantedOutputs)) continue; if (!wantedOutputs.contains(i.first)) continue;
worker.store.computeFSClosure(i.second, outputClosure); worker.store.computeFSClosure(i.second, outputClosure);
} }
@ -422,7 +416,7 @@ void DerivationGoal::repairClosure()
if (drvPath2 == outputsToDrv.end()) if (drvPath2 == outputsToDrv.end())
addWaitee(upcast_goal(worker.makePathSubstitutionGoal(i, Repair))); addWaitee(upcast_goal(worker.makePathSubstitutionGoal(i, Repair)));
else else
addWaitee(worker.makeDerivationGoal(drvPath2->second, StringSet(), bmRepair)); addWaitee(worker.makeDerivationGoal(drvPath2->second, OutputsSpec::All(), bmRepair));
} }
if (waitees.empty()) { if (waitees.empty()) {
@ -991,10 +985,15 @@ void DerivationGoal::resolvedFinished()
StorePathSet outputPaths; StorePathSet outputPaths;
// `wantedOutputs` might be empty, which means “all the outputs” // `wantedOutputs` might merely indicate “all the outputs”
auto realWantedOutputs = wantedOutputs; auto realWantedOutputs = std::visit(overloaded {
if (realWantedOutputs.empty()) [&](const OutputsSpec::All &) {
realWantedOutputs = resolvedDrv.outputNames(); return resolvedDrv.outputNames();
},
[&](const OutputsSpec::Names & names) {
return static_cast<std::set<std::string>>(names);
},
}, wantedOutputs.raw());
for (auto & wantedOutput : realWantedOutputs) { for (auto & wantedOutput : realWantedOutputs) {
auto initialOutput = get(initialOutputs, wantedOutput); auto initialOutput = get(initialOutputs, wantedOutput);
@ -1322,7 +1321,14 @@ std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity()
if (!drv->type().isPure()) return { false, {} }; if (!drv->type().isPure()) return { false, {} };
bool checkHash = buildMode == bmRepair; bool checkHash = buildMode == bmRepair;
auto wantedOutputsLeft = wantedOutputs; auto wantedOutputsLeft = std::visit(overloaded {
[&](const OutputsSpec::All &) {
return StringSet {};
},
[&](const OutputsSpec::Names & names) {
return static_cast<StringSet>(names);
},
}, wantedOutputs.raw());
DrvOutputs validOutputs; DrvOutputs validOutputs;
for (auto & i : queryPartialDerivationOutputMap()) { for (auto & i : queryPartialDerivationOutputMap()) {
@ -1331,7 +1337,7 @@ std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity()
// this is an invalid output, gets catched with (!wantedOutputsLeft.empty()) // this is an invalid output, gets catched with (!wantedOutputsLeft.empty())
continue; continue;
auto & info = *initialOutput; auto & info = *initialOutput;
info.wanted = wantOutput(i.first, wantedOutputs); info.wanted = wantedOutputs.contains(i.first);
if (info.wanted) if (info.wanted)
wantedOutputsLeft.erase(i.first); wantedOutputsLeft.erase(i.first);
if (i.second) { if (i.second) {
@ -1369,7 +1375,7 @@ std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity()
validOutputs.emplace(drvOutput, Realisation { drvOutput, info.known->path }); validOutputs.emplace(drvOutput, Realisation { drvOutput, info.known->path });
} }
// If we requested all the outputs via the empty set, we are always fine. // If we requested all the outputs, we are always fine.
// If we requested specific elements, the loop above removes all the valid // If we requested specific elements, the loop above removes all the valid
// ones, so any that are left must be invalid. // ones, so any that are left must be invalid.
if (!wantedOutputsLeft.empty()) if (!wantedOutputsLeft.empty())

View file

@ -2,6 +2,7 @@
#include "parsed-derivations.hh" #include "parsed-derivations.hh"
#include "lock.hh" #include "lock.hh"
#include "outputs-spec.hh"
#include "store-api.hh" #include "store-api.hh"
#include "pathlocks.hh" #include "pathlocks.hh"
#include "goal.hh" #include "goal.hh"
@ -55,7 +56,7 @@ struct DerivationGoal : public Goal
/* The specific outputs that we need to build. Empty means all of /* The specific outputs that we need to build. Empty means all of
them. */ them. */
StringSet wantedOutputs; OutputsSpec wantedOutputs;
/* Mapping from input derivations + output names to actual store /* Mapping from input derivations + output names to actual store
paths. This is filled in by waiteeDone() as each dependency paths. This is filled in by waiteeDone() as each dependency
@ -128,10 +129,10 @@ struct DerivationGoal : public Goal
std::string machineName; std::string machineName;
DerivationGoal(const StorePath & drvPath, DerivationGoal(const StorePath & drvPath,
const StringSet & wantedOutputs, Worker & worker, const OutputsSpec & wantedOutputs, Worker & worker,
BuildMode buildMode = bmNormal); BuildMode buildMode = bmNormal);
DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
const StringSet & wantedOutputs, Worker & worker, const OutputsSpec & wantedOutputs, Worker & worker,
BuildMode buildMode = bmNormal); BuildMode buildMode = bmNormal);
virtual ~DerivationGoal(); virtual ~DerivationGoal();
@ -142,7 +143,7 @@ struct DerivationGoal : public Goal
void work() override; void work() override;
/* Add wanted outputs to an already existing derivation goal. */ /* Add wanted outputs to an already existing derivation goal. */
void addWantedOutputs(const StringSet & outputs); void addWantedOutputs(const OutputsSpec & outputs);
/* The states. */ /* The states. */
void getDerivation(); void getDerivation();

View file

@ -80,7 +80,7 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat
BuildMode buildMode) BuildMode buildMode)
{ {
Worker worker(*this, *this); Worker worker(*this, *this);
auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode); auto goal = worker.makeBasicDerivationGoal(drvPath, drv, OutputsSpec::All {}, buildMode);
try { try {
worker.run(Goals{goal}); worker.run(Goals{goal});
@ -89,7 +89,10 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat
return BuildResult { return BuildResult {
.status = BuildResult::MiscFailure, .status = BuildResult::MiscFailure,
.errorMsg = e.msg(), .errorMsg = e.msg(),
.path = DerivedPath::Built { .drvPath = drvPath }, .path = DerivedPath::Built {
.drvPath = drvPath,
.outputs = OutputsSpec::All { },
},
}; };
}; };
} }
@ -130,7 +133,8 @@ void LocalStore::repairPath(const StorePath & path)
auto info = queryPathInfo(path); auto info = queryPathInfo(path);
if (info->deriver && isValidPath(*info->deriver)) { if (info->deriver && isValidPath(*info->deriver)) {
goals.clear(); goals.clear();
goals.insert(worker.makeDerivationGoal(*info->deriver, StringSet(), bmRepair)); // FIXME: Should just build the specific output we need.
goals.insert(worker.makeDerivationGoal(*info->deriver, OutputsSpec::All { }, bmRepair));
worker.run(goals); worker.run(goals);
} else } else
throw Error(worker.exitStatus(), "cannot repair path '%s'", printStorePath(path)); throw Error(worker.exitStatus(), "cannot repair path '%s'", printStorePath(path));

View file

@ -1459,7 +1459,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
unknown, downloadSize, narSize); unknown, downloadSize, narSize);
} }
virtual std::optional<std::string> getBuildLog(const StorePath & path) override virtual std::optional<std::string> getBuildLogExact(const StorePath & path) override
{ return std::nullopt; } { return std::nullopt; }
virtual void addBuildLog(const StorePath & path, std::string_view log) override virtual void addBuildLog(const StorePath & path, std::string_view log) override
@ -2726,7 +2726,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
signRealisation(thisRealisation); signRealisation(thisRealisation);
worker.store.registerDrvOutput(thisRealisation); worker.store.registerDrvOutput(thisRealisation);
} }
if (wantOutput(outputName, wantedOutputs)) if (wantedOutputs.contains(outputName))
builtOutputs.emplace(thisRealisation.id, thisRealisation); builtOutputs.emplace(thisRealisation.id, thisRealisation);
} }

View file

@ -42,7 +42,7 @@ Worker::~Worker()
std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon( std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon(
const StorePath & drvPath, const StorePath & drvPath,
const StringSet & wantedOutputs, const OutputsSpec & wantedOutputs,
std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal) std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal)
{ {
std::weak_ptr<DerivationGoal> & goal_weak = derivationGoals[drvPath]; std::weak_ptr<DerivationGoal> & goal_weak = derivationGoals[drvPath];
@ -59,7 +59,7 @@ std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon(
std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drvPath, std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drvPath,
const StringSet & wantedOutputs, BuildMode buildMode) const OutputsSpec & wantedOutputs, BuildMode buildMode)
{ {
return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr<DerivationGoal> { return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr<DerivationGoal> {
return !dynamic_cast<LocalStore *>(&store) return !dynamic_cast<LocalStore *>(&store)
@ -70,7 +70,7 @@ std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drv
std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath & drvPath, std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath & drvPath,
const BasicDerivation & drv, const StringSet & wantedOutputs, BuildMode buildMode) const BasicDerivation & drv, const OutputsSpec & wantedOutputs, BuildMode buildMode)
{ {
return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr<DerivationGoal> { return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr<DerivationGoal> {
return !dynamic_cast<LocalStore *>(&store) return !dynamic_cast<LocalStore *>(&store)

View file

@ -140,15 +140,15 @@ public:
/* derivation goal */ /* derivation goal */
private: private:
std::shared_ptr<DerivationGoal> makeDerivationGoalCommon( std::shared_ptr<DerivationGoal> makeDerivationGoalCommon(
const StorePath & drvPath, const StringSet & wantedOutputs, const StorePath & drvPath, const OutputsSpec & wantedOutputs,
std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal); std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal);
public: public:
std::shared_ptr<DerivationGoal> makeDerivationGoal( std::shared_ptr<DerivationGoal> makeDerivationGoal(
const StorePath & drvPath, const StorePath & drvPath,
const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal);
std::shared_ptr<DerivationGoal> makeBasicDerivationGoal( std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(
const StorePath & drvPath, const BasicDerivation & drv, const StorePath & drvPath, const BasicDerivation & drv,
const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal);
/* substitution goal */ /* substitution goal */
std::shared_ptr<PathSubstitutionGoal> makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt); std::shared_ptr<PathSubstitutionGoal> makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);

View file

@ -688,12 +688,6 @@ std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation &
} }
bool wantOutput(const std::string & output, const std::set<std::string> & wanted)
{
return wanted.empty() || wanted.find(output) != wanted.end();
}
static DerivationOutput readDerivationOutput(Source & in, const Store & store) static DerivationOutput readDerivationOutput(Source & in, const Store & store)
{ {
const auto pathS = readString(in); const auto pathS = readString(in);

View file

@ -294,8 +294,6 @@ typedef std::map<StorePath, DrvHash> DrvHashes;
// FIXME: global, though at least thread-safe. // FIXME: global, though at least thread-safe.
extern Sync<DrvHashes> drvHashes; extern Sync<DrvHashes> drvHashes;
bool wantOutput(const std::string & output, const std::set<std::string> & wanted);
struct Source; struct Source;
struct Sink; struct Sink;

View file

@ -19,11 +19,11 @@ nlohmann::json DerivedPath::Built::toJSON(ref<Store> store) const {
res["drvPath"] = store->printStorePath(drvPath); res["drvPath"] = store->printStorePath(drvPath);
// Fallback for the input-addressed derivation case: We expect to always be // Fallback for the input-addressed derivation case: We expect to always be
// able to print the output paths, so lets do it // able to print the output paths, so lets do it
const auto knownOutputs = store->queryPartialDerivationOutputMap(drvPath); const auto outputMap = store->queryPartialDerivationOutputMap(drvPath);
for (const auto & output : outputs) { for (const auto & [output, outputPathOpt] : outputMap) {
auto knownOutput = get(knownOutputs, output); if (!outputs.contains(output)) continue;
if (knownOutput && *knownOutput) if (outputPathOpt)
res["outputs"][output] = store->printStorePath(**knownOutput); res["outputs"][output] = store->printStorePath(*outputPathOpt);
else else
res["outputs"][output] = nullptr; res["outputs"][output] = nullptr;
} }
@ -63,7 +63,7 @@ std::string DerivedPath::Built::to_string(const Store & store) const
{ {
return store.printStorePath(drvPath) return store.printStorePath(drvPath)
+ "!" + "!"
+ (outputs.empty() ? std::string { "*" } : concatStringsSep(",", outputs)); + outputs.to_string();
} }
std::string DerivedPath::to_string(const Store & store) const std::string DerivedPath::to_string(const Store & store) const
@ -81,15 +81,10 @@ DerivedPath::Opaque DerivedPath::Opaque::parse(const Store & store, std::string_
DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_view drvS, std::string_view outputsS) DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_view drvS, std::string_view outputsS)
{ {
auto drvPath = store.parseStorePath(drvS); return {
std::set<std::string> outputs; .drvPath = store.parseStorePath(drvS),
if (outputsS != "*") { .outputs = OutputsSpec::parse(outputsS),
outputs = tokenizeString<std::set<std::string>>(outputsS, ","); };
if (outputs.empty())
throw Error(
"Explicit list of wanted outputs '%s' must not be empty. Consider using '*' as a wildcard meaning all outputs if no output in particular is wanted.", outputsS);
}
return {drvPath, outputs};
} }
DerivedPath DerivedPath::parse(const Store & store, std::string_view s) DerivedPath DerivedPath::parse(const Store & store, std::string_view s)

View file

@ -3,6 +3,7 @@
#include "util.hh" #include "util.hh"
#include "path.hh" #include "path.hh"
#include "realisation.hh" #include "realisation.hh"
#include "outputs-spec.hh"
#include <optional> #include <optional>
#include <variant> #include <variant>
@ -45,7 +46,7 @@ struct DerivedPathOpaque {
*/ */
struct DerivedPathBuilt { struct DerivedPathBuilt {
StorePath drvPath; StorePath drvPath;
std::set<std::string> outputs; OutputsSpec outputs;
std::string to_string(const Store & store) const; std::string to_string(const Store & store) const;
static DerivedPathBuilt parse(const Store & store, std::string_view, std::string_view); static DerivedPathBuilt parse(const Store & store, std::string_view, std::string_view);

View file

@ -676,7 +676,7 @@ public:
- the store object is signed by one of the [`trusted-public-keys`](#conf-trusted-public-keys) - the store object is signed by one of the [`trusted-public-keys`](#conf-trusted-public-keys)
- the substituter is in the [`trusted-substituters`](#conf-trusted-substituters) list - the substituter is in the [`trusted-substituters`](#conf-trusted-substituters) list
- the [`require-sigs`](#conf-require-sigs) option has been set to `false` - the [`require-sigs`](#conf-require-sigs) option has been set to `false`
- the store object is [output-addressed](glossary.md#gloss-output-addressed-store-object) - the store object is [output-addressed](@docroot@/glossary.md#gloss-output-addressed-store-object)
)", )",
{"binary-caches"}}; {"binary-caches"}};

View file

@ -278,7 +278,12 @@ public:
conn->to.flush(); conn->to.flush();
BuildResult status { .path = DerivedPath::Built { .drvPath = drvPath } }; BuildResult status {
.path = DerivedPath::Built {
.drvPath = drvPath,
.outputs = OutputsSpec::All { },
},
};
status.status = (BuildResult::Status) readInt(conn->from); status.status = (BuildResult::Status) readInt(conn->from);
conn->from >> status.errorMsg; conn->from >> status.errorMsg;

View file

@ -87,20 +87,8 @@ void LocalFSStore::narFromPath(const StorePath & path, Sink & sink)
const std::string LocalFSStore::drvsLogDir = "drvs"; const std::string LocalFSStore::drvsLogDir = "drvs";
std::optional<std::string> LocalFSStore::getBuildLog(const StorePath & path_) std::optional<std::string> LocalFSStore::getBuildLogExact(const StorePath & path)
{ {
auto path = path_;
if (!path.isDerivation()) {
try {
auto info = queryPathInfo(path);
if (!info->deriver) return std::nullopt;
path = *info->deriver;
} catch (InvalidPath &) {
return std::nullopt;
}
}
auto baseName = path.to_string(); auto baseName = path.to_string();
for (int j = 0; j < 2; j++) { for (int j = 0; j < 2; j++) {

View file

@ -50,7 +50,7 @@ public:
return getRealStoreDir() + "/" + std::string(storePath, storeDir.size() + 1); return getRealStoreDir() + "/" + std::string(storePath, storeDir.size() + 1);
} }
std::optional<std::string> getBuildLog(const StorePath & path) override; std::optional<std::string> getBuildLogExact(const StorePath & path) override;
}; };

12
src/libstore/log-store.cc Normal file
View file

@ -0,0 +1,12 @@
#include "log-store.hh"
namespace nix {
std::optional<std::string> LogStore::getBuildLog(const StorePath & path) {
auto maybePath = getBuildDerivationPath(path);
if (!maybePath)
return std::nullopt;
return getBuildLogExact(maybePath.value());
}
}

View file

@ -11,7 +11,9 @@ struct LogStore : public virtual Store
/* Return the build log of the specified store path, if available, /* Return the build log of the specified store path, if available,
or null otherwise. */ or null otherwise. */
virtual std::optional<std::string> getBuildLog(const StorePath & path) = 0; std::optional<std::string> getBuildLog(const StorePath & path);
virtual std::optional<std::string> getBuildLogExact(const StorePath & path) = 0;
virtual void addBuildLog(const StorePath & path, std::string_view log) = 0; virtual void addBuildLog(const StorePath & path, std::string_view log) = 0;

View file

@ -184,7 +184,7 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets,
knownOutputPaths = false; knownOutputPaths = false;
break; break;
} }
if (wantOutput(outputName, bfd.outputs) && !isValidPath(*pathOpt)) if (bfd.outputs.contains(outputName) && !isValidPath(*pathOpt))
invalid.insert(*pathOpt); invalid.insert(*pathOpt);
} }
if (knownOutputPaths && invalid.empty()) return; if (knownOutputPaths && invalid.empty()) return;
@ -300,4 +300,47 @@ std::map<DrvOutput, StorePath> drvOutputReferences(
return drvOutputReferences(Realisation::closure(store, inputRealisations), info->referencesPossiblyToSelf()); return drvOutputReferences(Realisation::closure(store, inputRealisations), info->referencesPossiblyToSelf());
} }
OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, Store * evalStore_)
{
auto & evalStore = evalStore_ ? *evalStore_ : store;
OutputPathMap outputs;
auto drv = evalStore.readDerivation(bfd.drvPath);
auto outputHashes = staticOutputHashes(store, drv);
auto drvOutputs = drv.outputsAndOptPaths(store);
auto outputNames = std::visit(overloaded {
[&](const OutputsSpec::All &) {
StringSet names;
for (auto & [outputName, _] : drv.outputs)
names.insert(outputName);
return names;
},
[&](const OutputsSpec::Names & names) {
return static_cast<std::set<std::string>>(names);
},
}, bfd.outputs);
for (auto & output : outputNames) {
auto outputHash = get(outputHashes, output);
if (!outputHash)
throw Error(
"the derivation '%s' doesn't have an output named '%s'",
store.printStorePath(bfd.drvPath), output);
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
DrvOutput outputId { *outputHash, output };
auto realisation = store.queryRealisation(outputId);
if (!realisation)
throw MissingRealisation(outputId);
outputs.insert_or_assign(output, realisation->outPath);
} else {
// If ca-derivations isn't enabled, assume that
// the output path is statically known.
auto drvOutput = get(drvOutputs, output);
assert(drvOutput);
assert(drvOutput->second);
outputs.insert_or_assign(output, *drvOutput->second);
}
}
return outputs;
}
} }

View file

@ -0,0 +1,189 @@
#include "util.hh"
#include "outputs-spec.hh"
#include "nlohmann/json.hpp"
#include <regex>
namespace nix {
bool OutputsSpec::contains(const std::string & outputName) const
{
return std::visit(overloaded {
[&](const OutputsSpec::All &) {
return true;
},
[&](const OutputsSpec::Names & outputNames) {
return outputNames.count(outputName) > 0;
},
}, raw());
}
std::optional<OutputsSpec> OutputsSpec::parseOpt(std::string_view s)
{
static std::regex regex(R"((\*)|([a-z]+(,[a-z]+)*))");
std::smatch match;
std::string s2 { s }; // until some improves std::regex
if (!std::regex_match(s2, match, regex))
return std::nullopt;
if (match[1].matched)
return { OutputsSpec::All {} };
if (match[2].matched)
return OutputsSpec::Names { tokenizeString<StringSet>(match[2].str(), ",") };
assert(false);
}
OutputsSpec OutputsSpec::parse(std::string_view s)
{
std::optional spec = parseOpt(s);
if (!spec)
throw Error("Invalid outputs specifier: '%s'", s);
return *spec;
}
std::optional<std::pair<std::string_view, ExtendedOutputsSpec>> ExtendedOutputsSpec::parseOpt(std::string_view s)
{
auto found = s.rfind('^');
if (found == std::string::npos)
return std::pair { s, ExtendedOutputsSpec::Default {} };
auto specOpt = OutputsSpec::parseOpt(s.substr(found + 1));
if (!specOpt)
return std::nullopt;
return std::pair { s.substr(0, found), ExtendedOutputsSpec::Explicit { *std::move(specOpt) } };
}
std::pair<std::string_view, ExtendedOutputsSpec> ExtendedOutputsSpec::parse(std::string_view s)
{
std::optional spec = parseOpt(s);
if (!spec)
throw Error("Invalid extended outputs specifier: '%s'", s);
return *spec;
}
std::string OutputsSpec::to_string() const
{
return std::visit(overloaded {
[&](const OutputsSpec::All &) -> std::string {
return "*";
},
[&](const OutputsSpec::Names & outputNames) -> std::string {
return concatStringsSep(",", outputNames);
},
}, raw());
}
std::string ExtendedOutputsSpec::to_string() const
{
return std::visit(overloaded {
[&](const ExtendedOutputsSpec::Default &) -> std::string {
return "";
},
[&](const ExtendedOutputsSpec::Explicit & outputSpec) -> std::string {
return "^" + outputSpec.to_string();
},
}, raw());
}
OutputsSpec OutputsSpec::union_(const OutputsSpec & that) const
{
return std::visit(overloaded {
[&](const OutputsSpec::All &) -> OutputsSpec {
return OutputsSpec::All { };
},
[&](const OutputsSpec::Names & theseNames) -> OutputsSpec {
return std::visit(overloaded {
[&](const OutputsSpec::All &) -> OutputsSpec {
return OutputsSpec::All {};
},
[&](const OutputsSpec::Names & thoseNames) -> OutputsSpec {
OutputsSpec::Names ret = theseNames;
ret.insert(thoseNames.begin(), thoseNames.end());
return ret;
},
}, that.raw());
},
}, raw());
}
bool OutputsSpec::isSubsetOf(const OutputsSpec & that) const
{
return std::visit(overloaded {
[&](const OutputsSpec::All &) {
return true;
},
[&](const OutputsSpec::Names & thoseNames) {
return std::visit(overloaded {
[&](const OutputsSpec::All &) {
return false;
},
[&](const OutputsSpec::Names & theseNames) {
bool ret = true;
for (auto & o : theseNames)
if (thoseNames.count(o) == 0)
ret = false;
return ret;
},
}, raw());
},
}, that.raw());
}
}
namespace nlohmann {
using namespace nix;
OutputsSpec adl_serializer<OutputsSpec>::from_json(const json & json) {
auto names = json.get<StringSet>();
if (names == StringSet({"*"}))
return OutputsSpec::All {};
else
return OutputsSpec::Names { std::move(names) };
}
void adl_serializer<OutputsSpec>::to_json(json & json, OutputsSpec t) {
std::visit(overloaded {
[&](const OutputsSpec::All &) {
json = std::vector<std::string>({"*"});
},
[&](const OutputsSpec::Names & names) {
json = names;
},
}, t);
}
ExtendedOutputsSpec adl_serializer<ExtendedOutputsSpec>::from_json(const json & json) {
if (json.is_null())
return ExtendedOutputsSpec::Default {};
else {
return ExtendedOutputsSpec::Explicit { json.get<OutputsSpec>() };
}
}
void adl_serializer<ExtendedOutputsSpec>::to_json(json & json, ExtendedOutputsSpec t) {
std::visit(overloaded {
[&](const ExtendedOutputsSpec::Default &) {
json = nullptr;
},
[&](const ExtendedOutputsSpec::Explicit & e) {
adl_serializer<OutputsSpec>::to_json(json, e);
},
}, t);
}
}

View file

@ -0,0 +1,95 @@
#pragma once
#include <cassert>
#include <optional>
#include <set>
#include <variant>
#include "json-impls.hh"
namespace nix {
struct OutputNames : std::set<std::string> {
using std::set<std::string>::set;
/* These need to be "inherited manually" */
OutputNames(const std::set<std::string> & s)
: std::set<std::string>(s)
{ assert(!empty()); }
OutputNames(std::set<std::string> && s)
: std::set<std::string>(s)
{ assert(!empty()); }
/* This set should always be non-empty, so we delete this
constructor in order make creating empty ones by mistake harder.
*/
OutputNames() = delete;
};
struct AllOutputs : std::monostate { };
typedef std::variant<AllOutputs, OutputNames> _OutputsSpecRaw;
struct OutputsSpec : _OutputsSpecRaw {
using Raw = _OutputsSpecRaw;
using Raw::Raw;
/* Force choosing a variant */
OutputsSpec() = delete;
using Names = OutputNames;
using All = AllOutputs;
inline const Raw & raw() const {
return static_cast<const Raw &>(*this);
}
inline Raw & raw() {
return static_cast<Raw &>(*this);
}
bool contains(const std::string & output) const;
/* Create a new OutputsSpec which is the union of this and that. */
OutputsSpec union_(const OutputsSpec & that) const;
/* Whether this OutputsSpec is a subset of that. */
bool isSubsetOf(const OutputsSpec & outputs) const;
/* Parse a string of the form 'output1,...outputN' or
'*', returning the outputs spec. */
static OutputsSpec parse(std::string_view s);
static std::optional<OutputsSpec> parseOpt(std::string_view s);
std::string to_string() const;
};
struct DefaultOutputs : std::monostate { };
typedef std::variant<DefaultOutputs, OutputsSpec> _ExtendedOutputsSpecRaw;
struct ExtendedOutputsSpec : _ExtendedOutputsSpecRaw {
using Raw = _ExtendedOutputsSpecRaw;
using Raw::Raw;
using Default = DefaultOutputs;
using Explicit = OutputsSpec;
inline const Raw & raw() const {
return static_cast<const Raw &>(*this);
}
/* Parse a string of the form 'prefix^output1,...outputN' or
'prefix^*', returning the prefix and the extended outputs spec. */
static std::pair<std::string_view, ExtendedOutputsSpec> parse(std::string_view s);
static std::optional<std::pair<std::string_view, ExtendedOutputsSpec>> parseOpt(std::string_view s);
std::string to_string() const;
};
}
JSON_IMPL(OutputsSpec)
JSON_IMPL(ExtendedOutputsSpec)

View file

@ -3,6 +3,117 @@
namespace nix { namespace nix {
std::string ValidPathInfo::fingerprint(const Store & store) const
{
if (narSize == 0)
throw Error("cannot calculate fingerprint of path '%s' because its size is not known",
store.printStorePath(path));
return
"1;" + store.printStorePath(path) + ";"
+ narHash.to_string(Base32, true) + ";"
+ std::to_string(narSize) + ";"
+ concatStringsSep(",", store.printStorePathSet(referencesPossiblyToSelf()));
}
void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey)
{
sigs.insert(secretKey.signDetached(fingerprint(store)));
}
std::optional<StorePathDescriptor> ValidPathInfo::fullStorePathDescriptorOpt() const
{
if (! ca)
return std::nullopt;
return StorePathDescriptor {
.name = std::string { path.name() },
.info = std::visit(overloaded {
[&](const TextHash & th) -> ContentAddressWithReferences {
assert(!references.self);
return TextInfo {
th,
.references = references.others,
};
},
[&](const FixedOutputHash & foh) -> ContentAddressWithReferences {
return FixedOutputInfo {
foh,
.references = references,
};
},
}, *ca),
};
}
bool ValidPathInfo::isContentAddressed(const Store & store) const
{
auto fullCaOpt = fullStorePathDescriptorOpt();
if (! fullCaOpt)
return false;
auto caPath = store.makeFixedOutputPathFromCA(*fullCaOpt);
bool res = caPath == path;
if (!res)
printError("warning: path '%s' claims to be content-addressed but isn't", store.printStorePath(path));
return res;
}
size_t ValidPathInfo::checkSignatures(const Store & store, const PublicKeys & publicKeys) const
{
if (isContentAddressed(store)) return maxSigs;
size_t good = 0;
for (auto & sig : sigs)
if (checkSignature(store, publicKeys, sig))
good++;
return good;
}
bool ValidPathInfo::checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const
{
return verifyDetached(fingerprint(store), sig, publicKeys);
}
Strings ValidPathInfo::shortRefs() const
{
Strings refs;
for (auto & r : referencesPossiblyToSelf())
refs.push_back(std::string(r.to_string()));
return refs;
}
ValidPathInfo::ValidPathInfo(
const Store & store,
StorePathDescriptor && info,
Hash narHash)
: path(store.makeFixedOutputPathFromCA(info))
, narHash(narHash)
{
std::visit(overloaded {
[this](TextInfo && ti) {
this->references = {
.others = std::move(ti.references),
.self = false,
};
this->ca = std::move((TextHash &&) ti);
},
[this](FixedOutputInfo && foi) {
this->references = std::move(foi.references);
this->ca = std::move((FixedOutputHash &&) foi);
},
}, std::move(info.info));
}
StorePathSet ValidPathInfo::referencesPossiblyToSelf() const StorePathSet ValidPathInfo::referencesPossiblyToSelf() const
{ {
return references.possiblyToSelf(path); return references.possiblyToSelf(path);
@ -18,6 +129,7 @@ void ValidPathInfo::setReferencesPossiblyToSelf(StorePathSet && refs)
return references.setPossiblyToSelf(path, std::move(refs)); return references.setPossiblyToSelf(path, std::move(refs));
} }
ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned int format) ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned int format)
{ {
return read(source, store, format, store.parseStorePath(readString(source))); return read(source, store, format, store.parseStorePath(readString(source)));
@ -39,6 +151,7 @@ ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned
return info; return info;
} }
void ValidPathInfo::write( void ValidPathInfo::write(
Sink & sink, Sink & sink,
const Store & store, const Store & store,

View file

@ -1,6 +1,5 @@
#include "path-with-outputs.hh" #include "path-with-outputs.hh"
#include "store-api.hh" #include "store-api.hh"
#include "nlohmann/json.hpp"
#include <regex> #include <regex>
@ -16,10 +15,14 @@ std::string StorePathWithOutputs::to_string(const Store & store) const
DerivedPath StorePathWithOutputs::toDerivedPath() const DerivedPath StorePathWithOutputs::toDerivedPath() const
{ {
if (!outputs.empty() || path.isDerivation()) if (!outputs.empty()) {
return DerivedPath::Built { path, outputs }; return DerivedPath::Built { path, OutputsSpec::Names { outputs } };
else } else if (path.isDerivation()) {
assert(outputs.empty());
return DerivedPath::Built { path, OutputsSpec::All { } };
} else {
return DerivedPath::Opaque { path }; return DerivedPath::Opaque { path };
}
} }
@ -42,7 +45,18 @@ std::variant<StorePathWithOutputs, StorePath> StorePathWithOutputs::tryFromDeriv
return StorePathWithOutputs { bo.path }; return StorePathWithOutputs { bo.path };
}, },
[&](const DerivedPath::Built & bfd) -> std::variant<StorePathWithOutputs, StorePath> { [&](const DerivedPath::Built & bfd) -> std::variant<StorePathWithOutputs, StorePath> {
return StorePathWithOutputs { bfd.drvPath, bfd.outputs }; return StorePathWithOutputs {
.path = bfd.drvPath,
// Use legacy encoding of wildcard as empty set
.outputs = std::visit(overloaded {
[&](const OutputsSpec::All &) -> StringSet {
return {};
},
[&](const OutputsSpec::Names & outputs) {
return static_cast<StringSet>(outputs);
},
}, bfd.outputs.raw()),
};
}, },
}, p.raw()); }, p.raw());
} }
@ -53,8 +67,8 @@ std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s)
size_t n = s.find("!"); size_t n = s.find("!");
return n == s.npos return n == s.npos
? std::make_pair(s, std::set<std::string>()) ? std::make_pair(s, std::set<std::string>())
: std::make_pair(((std::string_view) s).substr(0, n), : std::make_pair(s.substr(0, n),
tokenizeString<std::set<std::string>>(((std::string_view) s).substr(n + 1), ",")); tokenizeString<std::set<std::string>>(s.substr(n + 1), ","));
} }
@ -71,57 +85,4 @@ StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std:
return StorePathWithOutputs { store.followLinksToStorePath(path), std::move(outputs) }; return StorePathWithOutputs { store.followLinksToStorePath(path), std::move(outputs) };
} }
std::pair<std::string, OutputsSpec> parseOutputsSpec(const std::string & s)
{
static std::regex regex(R"((.*)\^((\*)|([a-z]+(,[a-z]+)*)))");
std::smatch match;
if (!std::regex_match(s, match, regex))
return {s, DefaultOutputs()};
if (match[3].matched)
return {match[1], AllOutputs()};
return {match[1], tokenizeString<OutputNames>(match[4].str(), ",")};
}
std::string printOutputsSpec(const OutputsSpec & outputsSpec)
{
if (std::get_if<DefaultOutputs>(&outputsSpec))
return "";
if (std::get_if<AllOutputs>(&outputsSpec))
return "^*";
if (auto outputNames = std::get_if<OutputNames>(&outputsSpec))
return "^" + concatStringsSep(",", *outputNames);
assert(false);
}
void to_json(nlohmann::json & json, const OutputsSpec & outputsSpec)
{
if (std::get_if<DefaultOutputs>(&outputsSpec))
json = nullptr;
else if (std::get_if<AllOutputs>(&outputsSpec))
json = std::vector<std::string>({"*"});
else if (auto outputNames = std::get_if<OutputNames>(&outputsSpec))
json = *outputNames;
}
void from_json(const nlohmann::json & json, OutputsSpec & outputsSpec)
{
if (json.is_null())
outputsSpec = DefaultOutputs();
else {
auto names = json.get<OutputNames>();
if (names == OutputNames({"*"}))
outputsSpec = AllOutputs();
else
outputsSpec = names;
}
}
} }

View file

@ -1,13 +1,17 @@
#pragma once #pragma once
#include <variant>
#include "path.hh" #include "path.hh"
#include "derived-path.hh" #include "derived-path.hh"
#include "nlohmann/json_fwd.hpp"
namespace nix { namespace nix {
/* This is a deprecated old type just for use by the old CLI, and older
versions of the RPC protocols. In new code don't use it; you want
`DerivedPath` instead.
`DerivedPath` is better because it handles more cases, and does so more
explicitly without devious punning tricks.
*/
struct StorePathWithOutputs struct StorePathWithOutputs
{ {
StorePath path; StorePath path;
@ -33,25 +37,4 @@ StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view
StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs); StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs);
typedef std::set<std::string> OutputNames;
struct AllOutputs {
bool operator < (const AllOutputs & _) const { return false; }
};
struct DefaultOutputs {
bool operator < (const DefaultOutputs & _) const { return false; }
};
typedef std::variant<DefaultOutputs, AllOutputs, OutputNames> OutputsSpec;
/* Parse a string of the form 'prefix^output1,...outputN' or
'prefix^*', returning the prefix and the outputs spec. */
std::pair<std::string, OutputsSpec> parseOutputsSpec(const std::string & s);
std::string printOutputsSpec(const OutputsSpec & outputsSpec);
void to_json(nlohmann::json &, const OutputsSpec &);
void from_json(const nlohmann::json &, OutputsSpec &);
} }

View file

@ -65,7 +65,6 @@ public:
typedef std::set<StorePath> StorePathSet; typedef std::set<StorePath> StorePathSet;
typedef std::vector<StorePath> StorePaths; typedef std::vector<StorePath> StorePaths;
typedef std::map<std::string, StorePath> OutputPathMap;
/* Extension of derivations in the Nix store. */ /* Extension of derivations in the Nix store. */
const std::string drvExtension = ".drv"; const std::string drvExtension = ".drv";

View file

@ -868,8 +868,8 @@ std::vector<BuildResult> RemoteStore::buildPathsWithResults(
OutputPathMap outputs; OutputPathMap outputs;
auto drv = evalStore->readDerivation(bfd.drvPath); auto drv = evalStore->readDerivation(bfd.drvPath);
const auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive const auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive
const auto drvOutputs = drv.outputsAndOptPaths(*this); auto built = resolveDerivedPath(*this, bfd, &*evalStore);
for (auto & output : bfd.outputs) { for (auto & [output, outputPath] : built) {
auto outputHash = get(outputHashes, output); auto outputHash = get(outputHashes, output);
if (!outputHash) if (!outputHash)
throw Error( throw Error(
@ -883,16 +883,11 @@ std::vector<BuildResult> RemoteStore::buildPathsWithResults(
throw MissingRealisation(outputId); throw MissingRealisation(outputId);
res.builtOutputs.emplace(realisation->id, *realisation); res.builtOutputs.emplace(realisation->id, *realisation);
} else { } else {
// If ca-derivations isn't enabled, assume that
// the output path is statically known.
const auto drvOutput = get(drvOutputs, output);
assert(drvOutput);
assert(drvOutput->second);
res.builtOutputs.emplace( res.builtOutputs.emplace(
outputId, outputId,
Realisation { Realisation {
.id = outputId, .id = outputId,
.outPath = *drvOutput->second, .outPath = outputPath,
}); });
} }
} }
@ -916,7 +911,12 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD
writeDerivation(conn->to, *this, drv); writeDerivation(conn->to, *this, drv);
conn->to << buildMode; conn->to << buildMode;
conn.processStderr(); conn.processStderr();
BuildResult res { .path = DerivedPath::Built { .drvPath = drvPath } }; BuildResult res {
.path = DerivedPath::Built {
.drvPath = drvPath,
.outputs = OutputsSpec::All { },
},
};
res.status = (BuildResult::Status) readInt(conn->from); res.status = (BuildResult::Status) readInt(conn->from);
conn->from >> res.errorMsg; conn->from >> res.errorMsg;
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) { if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) {

View file

@ -53,8 +53,8 @@ public:
{ return false; } { return false; }
// FIXME extend daemon protocol, move implementation to RemoteStore // FIXME extend daemon protocol, move implementation to RemoteStore
std::optional<std::string> getBuildLog(const StorePath & path) override std::optional<std::string> getBuildLogExact(const StorePath & path) override
{ unsupported("getBuildLog"); } { unsupported("getBuildLogExact"); }
private: private:

View file

@ -1228,117 +1228,6 @@ std::string showPaths(const PathSet & paths)
return concatStringsSep(", ", quoteStrings(paths)); return concatStringsSep(", ", quoteStrings(paths));
} }
std::string ValidPathInfo::fingerprint(const Store & store) const
{
if (narSize == 0)
throw Error("cannot calculate fingerprint of path '%s' because its size is not known",
store.printStorePath(path));
return
"1;" + store.printStorePath(path) + ";"
+ narHash.to_string(Base32, true) + ";"
+ std::to_string(narSize) + ";"
+ concatStringsSep(",", store.printStorePathSet(referencesPossiblyToSelf()));
}
void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey)
{
sigs.insert(secretKey.signDetached(fingerprint(store)));
}
std::optional<StorePathDescriptor> ValidPathInfo::fullStorePathDescriptorOpt() const
{
if (! ca)
return std::nullopt;
return StorePathDescriptor {
.name = std::string { path.name() },
.info = std::visit(overloaded {
[&](const TextHash & th) -> ContentAddressWithReferences {
assert(!references.self);
return TextInfo {
th,
.references = references.others,
};
},
[&](const FixedOutputHash & foh) -> ContentAddressWithReferences {
return FixedOutputInfo {
foh,
.references = references,
};
},
}, *ca),
};
}
bool ValidPathInfo::isContentAddressed(const Store & store) const
{
auto fullCaOpt = fullStorePathDescriptorOpt();
if (! fullCaOpt)
return false;
auto caPath = store.makeFixedOutputPathFromCA(*fullCaOpt);
bool res = caPath == path;
if (!res)
printError("warning: path '%s' claims to be content-addressed but isn't", store.printStorePath(path));
return res;
}
size_t ValidPathInfo::checkSignatures(const Store & store, const PublicKeys & publicKeys) const
{
if (isContentAddressed(store)) return maxSigs;
size_t good = 0;
for (auto & sig : sigs)
if (checkSignature(store, publicKeys, sig))
good++;
return good;
}
bool ValidPathInfo::checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const
{
return verifyDetached(fingerprint(store), sig, publicKeys);
}
Strings ValidPathInfo::shortRefs() const
{
Strings refs;
for (auto & r : referencesPossiblyToSelf())
refs.push_back(std::string(r.to_string()));
return refs;
}
ValidPathInfo::ValidPathInfo(
const Store & store,
StorePathDescriptor && info,
Hash narHash)
: path(store.makeFixedOutputPathFromCA(info))
, narHash(narHash)
{
std::visit(overloaded {
[this](TextInfo && ti) {
this->references = {
.others = std::move(ti.references),
.self = false,
};
this->ca = std::move((TextHash &&) ti);
},
[this](FixedOutputInfo && foi) {
this->references = std::move(foi.references);
this->ca = std::move((FixedOutputHash &&) foi);
},
}, std::move(info.info));
}
Derivation Store::derivationFromPath(const StorePath & drvPath) Derivation Store::derivationFromPath(const StorePath & drvPath)
{ {
ensurePath(drvPath); ensurePath(drvPath);
@ -1357,6 +1246,34 @@ Derivation readDerivationCommon(Store& store, const StorePath& drvPath, bool req
} }
} }
std::optional<StorePath> Store::getBuildDerivationPath(const StorePath & path)
{
if (!path.isDerivation()) {
try {
auto info = queryPathInfo(path);
if (!info->deriver) return std::nullopt;
return *info->deriver;
} catch (InvalidPath &) {
return std::nullopt;
}
}
if (!settings.isExperimentalFeatureEnabled(Xp::CaDerivations) || !isValidPath(path))
return path;
auto drv = readDerivation(path);
if (!drv.type().hasKnownOutputPaths()) {
// The build log is actually attached to the corresponding
// resolved derivation, so we need to get it first
auto resolvedDrv = drv.tryResolve(*this);
if (resolvedDrv)
return writeDerivation(*this, *resolvedDrv, NoRepair, true);
}
return path;
}
Derivation Store::readDerivation(const StorePath & drvPath) Derivation Store::readDerivation(const StorePath & drvPath)
{ return readDerivationCommon(*this, drvPath, true); } { return readDerivationCommon(*this, drvPath, true); }

View file

@ -71,6 +71,9 @@ class NarInfoDiskCache;
class Store; class Store;
typedef std::map<std::string, StorePath> OutputPathMap;
enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true }; enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true };
enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true }; enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true };
enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true }; enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true };
@ -122,6 +125,8 @@ public:
typedef std::map<std::string, std::string> Params; typedef std::map<std::string, std::string> Params;
protected: protected:
struct PathInfoCacheValue { struct PathInfoCacheValue {
@ -614,6 +619,13 @@ public:
*/ */
StorePathSet exportReferences(const StorePathSet & storePaths, const StorePathSet & inputPaths); StorePathSet exportReferences(const StorePathSet & storePaths, const StorePathSet & inputPaths);
/**
* Given a store path, return the realisation actually used in the realisation of this path:
* - If the path is a content-addressed derivation, try to resolve it
* - Otherwise, find one of its derivers
*/
std::optional<StorePath> getBuildDerivationPath(const StorePath &);
/* Hack to allow long-running processes like hydra-queue-runner to /* Hack to allow long-running processes like hydra-queue-runner to
occasionally flush their path info cache. */ occasionally flush their path info cache. */
void clearPathInfoCache() void clearPathInfoCache()
@ -715,6 +727,11 @@ void copyClosure(
void removeTempRoots(); void removeTempRoots();
/* Resolve the derived path completely, failing if any derivation output
is unknown. */
OutputPathMap resolveDerivedPath(Store &, const DerivedPath::Built &, Store * evalStore = nullptr);
/* Return a Store object to access the Nix store denoted by /* Return a Store object to access the Nix store denoted by
uri (slight misnomer...). Supported values are: uri (slight misnomer...). Supported values are:

View file

@ -0,0 +1,187 @@
#include "outputs-spec.hh"
#include <nlohmann/json.hpp>
#include <gtest/gtest.h>
namespace nix {
#ifndef NDEBUG
TEST(OutputsSpec, no_empty_names) {
ASSERT_DEATH(OutputsSpec::Names { std::set<std::string> { } }, "");
}
#endif
#define TEST_DONT_PARSE(NAME, STR) \
TEST(OutputsSpec, bad_ ## NAME) { \
std::optional OutputsSpecOpt = \
OutputsSpec::parseOpt(STR); \
ASSERT_FALSE(OutputsSpecOpt); \
}
TEST_DONT_PARSE(empty, "")
TEST_DONT_PARSE(garbage, "&*()")
TEST_DONT_PARSE(double_star, "**")
TEST_DONT_PARSE(star_first, "*,foo")
TEST_DONT_PARSE(star_second, "foo,*")
#undef TEST_DONT_PARSE
TEST(OutputsSpec, all) {
std::string_view str = "*";
OutputsSpec expected = OutputsSpec::All { };
ASSERT_EQ(OutputsSpec::parse(str), expected);
ASSERT_EQ(expected.to_string(), str);
}
TEST(OutputsSpec, names_out) {
std::string_view str = "out";
OutputsSpec expected = OutputsSpec::Names { "out" };
ASSERT_EQ(OutputsSpec::parse(str), expected);
ASSERT_EQ(expected.to_string(), str);
}
TEST(OutputsSpec, names_out_bin) {
OutputsSpec expected = OutputsSpec::Names { "out", "bin" };
ASSERT_EQ(OutputsSpec::parse("out,bin"), expected);
// N.B. This normalization is OK.
ASSERT_EQ(expected.to_string(), "bin,out");
}
#define TEST_SUBSET(X, THIS, THAT) \
X((OutputsSpec { THIS }).isSubsetOf(THAT));
TEST(OutputsSpec, subsets_all_all) {
TEST_SUBSET(ASSERT_TRUE, OutputsSpec::All { }, OutputsSpec::All { });
}
TEST(OutputsSpec, subsets_names_all) {
TEST_SUBSET(ASSERT_TRUE, OutputsSpec::Names { "a" }, OutputsSpec::All { });
}
TEST(OutputsSpec, subsets_names_names_eq) {
TEST_SUBSET(ASSERT_TRUE, OutputsSpec::Names { "a" }, OutputsSpec::Names { "a" });
}
TEST(OutputsSpec, subsets_names_names_noneq) {
TEST_SUBSET(ASSERT_TRUE, OutputsSpec::Names { "a" }, (OutputsSpec::Names { "a", "b" }));
}
TEST(OutputsSpec, not_subsets_all_names) {
TEST_SUBSET(ASSERT_FALSE, OutputsSpec::All { }, OutputsSpec::Names { "a" });
}
TEST(OutputsSpec, not_subsets_names_names) {
TEST_SUBSET(ASSERT_FALSE, (OutputsSpec::Names { "a", "b" }), (OutputsSpec::Names { "a" }));
}
#undef TEST_SUBSET
#define TEST_UNION(RES, THIS, THAT) \
ASSERT_EQ(OutputsSpec { RES }, (OutputsSpec { THIS }).union_(THAT));
TEST(OutputsSpec, union_all_all) {
TEST_UNION(OutputsSpec::All { }, OutputsSpec::All { }, OutputsSpec::All { });
}
TEST(OutputsSpec, union_all_names) {
TEST_UNION(OutputsSpec::All { }, OutputsSpec::All { }, OutputsSpec::Names { "a" });
}
TEST(OutputsSpec, union_names_all) {
TEST_UNION(OutputsSpec::All { }, OutputsSpec::Names { "a" }, OutputsSpec::All { });
}
TEST(OutputsSpec, union_names_names) {
TEST_UNION((OutputsSpec::Names { "a", "b" }), OutputsSpec::Names { "a" }, OutputsSpec::Names { "b" });
}
#undef TEST_UNION
#define TEST_DONT_PARSE(NAME, STR) \
TEST(ExtendedOutputsSpec, bad_ ## NAME) { \
std::optional extendedOutputsSpecOpt = \
ExtendedOutputsSpec::parseOpt(STR); \
ASSERT_FALSE(extendedOutputsSpecOpt); \
}
TEST_DONT_PARSE(carot_empty, "^")
TEST_DONT_PARSE(prefix_carot_empty, "foo^")
TEST_DONT_PARSE(garbage, "^&*()")
TEST_DONT_PARSE(double_star, "^**")
TEST_DONT_PARSE(star_first, "^*,foo")
TEST_DONT_PARSE(star_second, "^foo,*")
#undef TEST_DONT_PARSE
TEST(ExtendedOutputsSpec, defeault) {
std::string_view str = "foo";
auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(str);
ASSERT_EQ(prefix, "foo");
ExtendedOutputsSpec expected = ExtendedOutputsSpec::Default { };
ASSERT_EQ(extendedOutputsSpec, expected);
ASSERT_EQ(std::string { prefix } + expected.to_string(), str);
}
TEST(ExtendedOutputsSpec, all) {
std::string_view str = "foo^*";
auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(str);
ASSERT_EQ(prefix, "foo");
ExtendedOutputsSpec expected = OutputsSpec::All { };
ASSERT_EQ(extendedOutputsSpec, expected);
ASSERT_EQ(std::string { prefix } + expected.to_string(), str);
}
TEST(ExtendedOutputsSpec, out) {
std::string_view str = "foo^out";
auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(str);
ASSERT_EQ(prefix, "foo");
ExtendedOutputsSpec expected = OutputsSpec::Names { "out" };
ASSERT_EQ(extendedOutputsSpec, expected);
ASSERT_EQ(std::string { prefix } + expected.to_string(), str);
}
TEST(ExtendedOutputsSpec, out_bin) {
auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse("foo^out,bin");
ASSERT_EQ(prefix, "foo");
ExtendedOutputsSpec expected = OutputsSpec::Names { "out", "bin" };
ASSERT_EQ(extendedOutputsSpec, expected);
ASSERT_EQ(std::string { prefix } + expected.to_string(), "foo^bin,out");
}
TEST(ExtendedOutputsSpec, many_carrot) {
auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse("foo^bar^out,bin");
ASSERT_EQ(prefix, "foo^bar");
ExtendedOutputsSpec expected = OutputsSpec::Names { "out", "bin" };
ASSERT_EQ(extendedOutputsSpec, expected);
ASSERT_EQ(std::string { prefix } + expected.to_string(), "foo^bar^bin,out");
}
#define TEST_JSON(TYPE, NAME, STR, VAL) \
\
TEST(TYPE, NAME ## _to_json) { \
using nlohmann::literals::operator "" _json; \
ASSERT_EQ( \
STR ## _json, \
((nlohmann::json) TYPE { VAL })); \
} \
\
TEST(TYPE, NAME ## _from_json) { \
using nlohmann::literals::operator "" _json; \
ASSERT_EQ( \
TYPE { VAL }, \
(STR ## _json).get<TYPE>()); \
}
TEST_JSON(OutputsSpec, all, R"(["*"])", OutputsSpec::All { })
TEST_JSON(OutputsSpec, name, R"(["a"])", OutputsSpec::Names { "a" })
TEST_JSON(OutputsSpec, names, R"(["a","b"])", (OutputsSpec::Names { "a", "b" }))
TEST_JSON(ExtendedOutputsSpec, def, R"(null)", ExtendedOutputsSpec::Default { })
TEST_JSON(ExtendedOutputsSpec, all, R"(["*"])", ExtendedOutputsSpec::Explicit { OutputsSpec::All { } })
TEST_JSON(ExtendedOutputsSpec, name, R"(["a"])", ExtendedOutputsSpec::Explicit { OutputsSpec::Names { "a" } })
TEST_JSON(ExtendedOutputsSpec, names, R"(["a","b"])", (ExtendedOutputsSpec::Explicit { OutputsSpec::Names { "a", "b" } }))
#undef TEST_JSON
}

View file

@ -1,46 +0,0 @@
#include "path-with-outputs.hh"
#include <gtest/gtest.h>
namespace nix {
TEST(parseOutputsSpec, basic)
{
{
auto [prefix, outputsSpec] = parseOutputsSpec("foo");
ASSERT_EQ(prefix, "foo");
ASSERT_TRUE(std::get_if<DefaultOutputs>(&outputsSpec));
}
{
auto [prefix, outputsSpec] = parseOutputsSpec("foo^*");
ASSERT_EQ(prefix, "foo");
ASSERT_TRUE(std::get_if<AllOutputs>(&outputsSpec));
}
{
auto [prefix, outputsSpec] = parseOutputsSpec("foo^out");
ASSERT_EQ(prefix, "foo");
ASSERT_TRUE(std::get<OutputNames>(outputsSpec) == OutputNames({"out"}));
}
{
auto [prefix, outputsSpec] = parseOutputsSpec("foo^out,bin");
ASSERT_EQ(prefix, "foo");
ASSERT_TRUE(std::get<OutputNames>(outputsSpec) == OutputNames({"out", "bin"}));
}
{
auto [prefix, outputsSpec] = parseOutputsSpec("foo^bar^out,bin");
ASSERT_EQ(prefix, "foo^bar");
ASSERT_TRUE(std::get<OutputNames>(outputsSpec) == OutputNames({"out", "bin"}));
}
{
auto [prefix, outputsSpec] = parseOutputsSpec("foo^&*()");
ASSERT_EQ(prefix, "foo^&*()");
ASSERT_TRUE(std::get_if<DefaultOutputs>(&outputsSpec));
}
}
}

14
src/libutil/json-impls.hh Normal file
View file

@ -0,0 +1,14 @@
#pragma once
#include "nlohmann/json_fwd.hpp"
// Following https://github.com/nlohmann/json#how-can-i-use-get-for-non-default-constructiblenon-copyable-types
#define JSON_IMPL(TYPE) \
namespace nlohmann { \
using namespace nix; \
template <> \
struct adl_serializer<TYPE> { \
static TYPE from_json(const json & json); \
static void to_json(json & json, TYPE t); \
}; \
}

View file

@ -311,6 +311,42 @@ namespace nix {
ASSERT_THROW(base64Decode("cXVvZCBlcm_0IGRlbW9uc3RyYW5kdW0="), Error); ASSERT_THROW(base64Decode("cXVvZCBlcm_0IGRlbW9uc3RyYW5kdW0="), Error);
} }
/* ----------------------------------------------------------------------------
* getLine
* --------------------------------------------------------------------------*/
TEST(getLine, all) {
{
auto [line, rest] = getLine("foo\nbar\nxyzzy");
ASSERT_EQ(line, "foo");
ASSERT_EQ(rest, "bar\nxyzzy");
}
{
auto [line, rest] = getLine("foo\r\nbar\r\nxyzzy");
ASSERT_EQ(line, "foo");
ASSERT_EQ(rest, "bar\r\nxyzzy");
}
{
auto [line, rest] = getLine("foo\n");
ASSERT_EQ(line, "foo");
ASSERT_EQ(rest, "");
}
{
auto [line, rest] = getLine("foo");
ASSERT_EQ(line, "foo");
ASSERT_EQ(rest, "");
}
{
auto [line, rest] = getLine("");
ASSERT_EQ(line, "");
ASSERT_EQ(rest, "");
}
}
/* ---------------------------------------------------------------------------- /* ----------------------------------------------------------------------------
* toLower * toLower
* --------------------------------------------------------------------------*/ * --------------------------------------------------------------------------*/

View file

@ -397,7 +397,7 @@ static void main_nix_build(int argc, char * * argv)
auto bashDrv = drv->requireDrvPath(); auto bashDrv = drv->requireDrvPath();
pathsToBuild.push_back(DerivedPath::Built { pathsToBuild.push_back(DerivedPath::Built {
.drvPath = bashDrv, .drvPath = bashDrv,
.outputs = {"out"}, .outputs = OutputsSpec::Names {"out"},
}); });
pathsToCopy.insert(bashDrv); pathsToCopy.insert(bashDrv);
shellDrv = bashDrv; shellDrv = bashDrv;
@ -421,7 +421,7 @@ static void main_nix_build(int argc, char * * argv)
{ {
pathsToBuild.push_back(DerivedPath::Built { pathsToBuild.push_back(DerivedPath::Built {
.drvPath = inputDrv, .drvPath = inputDrv,
.outputs = inputOutputs .outputs = OutputsSpec::Names { inputOutputs },
}); });
pathsToCopy.insert(inputDrv); pathsToCopy.insert(inputDrv);
} }
@ -591,7 +591,7 @@ static void main_nix_build(int argc, char * * argv)
if (outputName == "") if (outputName == "")
throw Error("derivation '%s' lacks an 'outputName' attribute", store->printStorePath(drvPath)); throw Error("derivation '%s' lacks an 'outputName' attribute", store->printStorePath(drvPath));
pathsToBuild.push_back(DerivedPath::Built{drvPath, {outputName}}); pathsToBuild.push_back(DerivedPath::Built{drvPath, OutputsSpec::Names{outputName}});
pathsToBuildOrdered.push_back({drvPath, {outputName}}); pathsToBuildOrdered.push_back({drvPath, {outputName}});
drvsToCopy.insert(drvPath); drvsToCopy.insert(drvPath);

View file

@ -478,9 +478,14 @@ static void printMissing(EvalState & state, DrvInfos & elems)
std::vector<DerivedPath> targets; std::vector<DerivedPath> targets;
for (auto & i : elems) for (auto & i : elems)
if (auto drvPath = i.queryDrvPath()) if (auto drvPath = i.queryDrvPath())
targets.push_back(DerivedPath::Built{*drvPath}); targets.push_back(DerivedPath::Built{
.drvPath = *drvPath,
.outputs = OutputsSpec::All { },
});
else else
targets.push_back(DerivedPath::Opaque{i.queryOutPath()}); targets.push_back(DerivedPath::Opaque{
.path = i.queryOutPath(),
});
printMissing(state.store, targets); printMissing(state.store, targets);
} }
@ -751,8 +756,13 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs)
auto drvPath = drv.queryDrvPath(); auto drvPath = drv.queryDrvPath();
std::vector<DerivedPath> paths { std::vector<DerivedPath> paths {
drvPath drvPath
? (DerivedPath) (DerivedPath::Built { *drvPath }) ? (DerivedPath) (DerivedPath::Built {
: (DerivedPath) (DerivedPath::Opaque { drv.queryOutPath() }), .drvPath = *drvPath,
.outputs = OutputsSpec::All { },
})
: (DerivedPath) (DerivedPath::Opaque {
.path = drv.queryOutPath(),
}),
}; };
printMissing(globals.state->store, paths); printMissing(globals.state->store, paths);
if (globals.dryRun) return; if (globals.dryRun) return;

View file

@ -19,12 +19,11 @@ struct InstallableDerivedPath : Installable
{ {
} }
std::string what() const override { return derivedPath.to_string(*store); } std::string what() const override { return derivedPath.to_string(*store); }
DerivedPaths toDerivedPaths() override DerivedPathsWithInfo toDerivedPaths() override
{ {
return {derivedPath}; return {{derivedPath}};
} }
std::optional<StorePath> getStorePath() override std::optional<StorePath> getStorePath() override
@ -80,9 +79,29 @@ UnresolvedApp Installable::toApp(EvalState & state)
if (type == "app") { if (type == "app") {
auto [program, context] = cursor->getAttr("program")->getStringWithContext(); auto [program, context] = cursor->getAttr("program")->getStringWithContext();
std::vector<StorePathWithOutputs> context2; std::vector<DerivedPath> context2;
for (auto & [path, name] : context) for (auto & c : context) {
context2.push_back({path, {name}}); context2.emplace_back(std::visit(overloaded {
[&](const NixStringContextElem::DrvDeep & d) -> DerivedPath {
/* We want all outputs of the drv */
return DerivedPath::Built {
.drvPath = d.drvPath,
.outputs = OutputsSpec::All {},
};
},
[&](const NixStringContextElem::Built & b) -> DerivedPath {
return DerivedPath::Built {
.drvPath = b.drvPath,
.outputs = OutputsSpec::Names { b.output },
};
},
[&](const NixStringContextElem::Opaque & o) -> DerivedPath {
return DerivedPath::Opaque {
.path = o.path,
};
},
}, c.raw()));
}
return UnresolvedApp{App { return UnresolvedApp{App {
.context = std::move(context2), .context = std::move(context2),
@ -106,7 +125,10 @@ UnresolvedApp Installable::toApp(EvalState & state)
: DrvName(name).name; : DrvName(name).name;
auto program = outPath + "/bin/" + mainProgram; auto program = outPath + "/bin/" + mainProgram;
return UnresolvedApp { App { return UnresolvedApp { App {
.context = { { drvPath, {outputName} } }, .context = { DerivedPath::Built {
.drvPath = drvPath,
.outputs = OutputsSpec::Names { outputName },
} },
.program = program, .program = program,
}}; }};
} }
@ -124,7 +146,7 @@ App UnresolvedApp::resolve(ref<Store> evalStore, ref<Store> store)
for (auto & ctxElt : unresolved.context) for (auto & ctxElt : unresolved.context)
installableContext.push_back( installableContext.push_back(
std::make_shared<InstallableDerivedPath>(store, ctxElt.toDerivedPath())); std::make_shared<InstallableDerivedPath>(store, ctxElt));
auto builtContext = Installable::build(evalStore, store, Realise::Outputs, installableContext); auto builtContext = Installable::build(evalStore, store, Realise::Outputs, installableContext);
res.program = resolveString(*store, unresolved.program, builtContext); res.program = resolveString(*store, unresolved.program, builtContext);

View file

@ -94,13 +94,15 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile
if (dryRun) { if (dryRun) {
std::vector<DerivedPath> pathsToBuild; std::vector<DerivedPath> pathsToBuild;
for (auto & i : installables) { for (auto & i : installables)
auto b = i->toDerivedPaths(); for (auto & b : i->toDerivedPaths())
pathsToBuild.insert(pathsToBuild.end(), b.begin(), b.end()); pathsToBuild.push_back(b.path);
}
printMissing(store, pathsToBuild, lvlError); printMissing(store, pathsToBuild, lvlError);
if (json) if (json)
logger->cout("%s", derivedPathsToJSON(pathsToBuild, store).dump()); logger->cout("%s", derivedPathsToJSON(pathsToBuild, store).dump());
return; return;
} }

View file

@ -75,10 +75,10 @@ struct CmdBundle : InstallableCommand
auto val = installable->toValue(*evalState).first; auto val = installable->toValue(*evalState).first;
auto [bundlerFlakeRef, bundlerName, outputsSpec] = parseFlakeRefWithFragmentAndOutputsSpec(bundler, absPath(".")); auto [bundlerFlakeRef, bundlerName, extendedOutputsSpec] = parseFlakeRefWithFragmentAndExtendedOutputsSpec(bundler, absPath("."));
const flake::LockFlags lockFlags{ .writeLockFile = false }; const flake::LockFlags lockFlags{ .writeLockFile = false };
InstallableFlake bundler{this, InstallableFlake bundler{this,
evalState, std::move(bundlerFlakeRef), bundlerName, outputsSpec, evalState, std::move(bundlerFlakeRef), bundlerName, extendedOutputsSpec,
{"bundlers." + settings.thisSystem.get() + ".default", {"bundlers." + settings.thisSystem.get() + ".default",
"defaultBundler." + settings.thisSystem.get() "defaultBundler." + settings.thisSystem.get()
}, },
@ -105,7 +105,12 @@ struct CmdBundle : InstallableCommand
auto outPath = evalState->coerceToStorePath(attr2->pos, *attr2->value, context2, ""); auto outPath = evalState->coerceToStorePath(attr2->pos, *attr2->value, context2, "");
store->buildPaths({ DerivedPath::Built { drvPath } }); store->buildPaths({
DerivedPath::Built {
.drvPath = drvPath,
.outputs = OutputsSpec::All { },
},
});
auto outPathS = store->printStorePath(outPath); auto outPathS = store->printStorePath(outPath);

View file

@ -3,7 +3,7 @@
#include "common-args.hh" #include "common-args.hh"
#include "shared.hh" #include "shared.hh"
#include "store-api.hh" #include "store-api.hh"
#include "path-with-outputs.hh" #include "outputs-spec.hh"
#include "derivations.hh" #include "derivations.hh"
#include "progress-bar.hh" #include "progress-bar.hh"
#include "run.hh" #include "run.hh"
@ -232,7 +232,12 @@ static StorePath getDerivationEnvironment(ref<Store> store, ref<Store> evalStore
auto shellDrvPath = writeDerivation(*evalStore, drv); auto shellDrvPath = writeDerivation(*evalStore, drv);
/* Build the derivation. */ /* Build the derivation. */
store->buildPaths({DerivedPath::Built{shellDrvPath}}, bmNormal, evalStore); store->buildPaths(
{ DerivedPath::Built {
.drvPath = shellDrvPath,
.outputs = OutputsSpec::All { },
}},
bmNormal, evalStore);
for (auto & [_0, optPath] : evalStore->queryPartialDerivationOutputMap(shellDrvPath)) { for (auto & [_0, optPath] : evalStore->queryPartialDerivationOutputMap(shellDrvPath)) {
assert(optPath); assert(optPath);

View file

@ -7,7 +7,7 @@
#include "get-drvs.hh" #include "get-drvs.hh"
#include "store-api.hh" #include "store-api.hh"
#include "derivations.hh" #include "derivations.hh"
#include "path-with-outputs.hh" #include "outputs-spec.hh"
#include "attr-path.hh" #include "attr-path.hh"
#include "fetchers.hh" #include "fetchers.hh"
#include "registry.hh" #include "registry.hh"
@ -348,7 +348,7 @@ struct CmdFlakeCheck : FlakeCommand
// FIXME // FIXME
auto app = App(*state, v); auto app = App(*state, v);
for (auto & i : app.context) { for (auto & i : app.context) {
auto [drvPathS, outputName] = decodeContext(i); auto [drvPathS, outputName] = NixStringContextElem::parse(i);
store->parseStorePath(drvPathS); store->parseStorePath(drvPathS);
} }
#endif #endif
@ -513,8 +513,12 @@ struct CmdFlakeCheck : FlakeCommand
auto drvPath = checkDerivation( auto drvPath = checkDerivation(
fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]),
*attr2.value, attr2.pos); *attr2.value, attr2.pos);
if (drvPath && attr_name == settings.thisSystem.get()) if (drvPath && attr_name == settings.thisSystem.get()) {
drvPaths.push_back(DerivedPath::Built{*drvPath}); drvPaths.push_back(DerivedPath::Built {
.drvPath = *drvPath,
.outputs = OutputsSpec::All { },
});
}
} }
} }
} }

View file

@ -49,7 +49,7 @@ struct CmdLog : InstallableCommand
[&](const DerivedPath::Built & bfd) { [&](const DerivedPath::Built & bfd) {
return logSub.getBuildLog(bfd.drvPath); return logSub.getBuildLog(bfd.drvPath);
}, },
}, b.raw()); }, b.path.raw());
if (!log) continue; if (!log) continue;
stopProgressBar(); stopProgressBar();
printInfo("got build log for '%s' from '%s'", installable->what(), logSub.getUri()); printInfo("got build log for '%s' from '%s'", installable->what(), logSub.getUri());

View file

@ -22,7 +22,7 @@ struct ProfileElementSource
// FIXME: record original attrpath. // FIXME: record original attrpath.
FlakeRef resolvedRef; FlakeRef resolvedRef;
std::string attrPath; std::string attrPath;
OutputsSpec outputs; ExtendedOutputsSpec outputs;
bool operator < (const ProfileElementSource & other) const bool operator < (const ProfileElementSource & other) const
{ {
@ -32,17 +32,19 @@ struct ProfileElementSource
} }
}; };
const int defaultPriority = 5;
struct ProfileElement struct ProfileElement
{ {
StorePathSet storePaths; StorePathSet storePaths;
std::optional<ProfileElementSource> source; std::optional<ProfileElementSource> source;
bool active = true; bool active = true;
int priority = 5; int priority = defaultPriority;
std::string describe() const std::string describe() const
{ {
if (source) if (source)
return fmt("%s#%s%s", source->originalRef, source->attrPath, printOutputsSpec(source->outputs)); return fmt("%s#%s%s", source->originalRef, source->attrPath, source->outputs.to_string());
StringSet names; StringSet names;
for (auto & path : storePaths) for (auto & path : storePaths)
names.insert(DrvName(path.name()).name); names.insert(DrvName(path.name()).name);
@ -124,7 +126,7 @@ struct ProfileManifest
parseFlakeRef(e[sOriginalUrl]), parseFlakeRef(e[sOriginalUrl]),
parseFlakeRef(e[sUrl]), parseFlakeRef(e[sUrl]),
e["attrPath"], e["attrPath"],
e["outputs"].get<OutputsSpec>() e["outputs"].get<ExtendedOutputsSpec>()
}; };
} }
elements.emplace_back(std::move(element)); elements.emplace_back(std::move(element));
@ -262,13 +264,20 @@ struct ProfileManifest
} }
}; };
static std::map<Installable *, BuiltPaths> static std::map<Installable *, std::pair<BuiltPaths, ExtraPathInfo>>
builtPathsPerInstallable( builtPathsPerInstallable(
const std::vector<std::pair<std::shared_ptr<Installable>, BuiltPathWithResult>> & builtPaths) const std::vector<std::pair<std::shared_ptr<Installable>, BuiltPathWithResult>> & builtPaths)
{ {
std::map<Installable *, BuiltPaths> res; std::map<Installable *, std::pair<BuiltPaths, ExtraPathInfo>> res;
for (auto & [installable, builtPath] : builtPaths) for (auto & [installable, builtPath] : builtPaths) {
res[installable.get()].push_back(builtPath.path); auto & r = res[installable.get()];
/* Note that there could be conflicting info
(e.g. meta.priority fields) if the installable returned
multiple derivations. So pick one arbitrarily. FIXME:
print a warning? */
r.first.push_back(builtPath.path);
r.second = builtPath.info;
}
return res; return res;
} }
@ -308,28 +317,25 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
for (auto & installable : installables) { for (auto & installable : installables) {
ProfileElement element; ProfileElement element;
auto & [res, info] = builtPaths[installable.get()];
if (info.originalRef && info.resolvedRef && info.attrPath && info.extendedOutputsSpec) {
if (auto installable2 = std::dynamic_pointer_cast<InstallableFlake>(installable)) {
// FIXME: make build() return this?
auto [attrPath, resolvedRef, drv] = installable2->toDerivation();
element.source = ProfileElementSource { element.source = ProfileElementSource {
installable2->flakeRef, .originalRef = *info.originalRef,
resolvedRef, .resolvedRef = *info.resolvedRef,
attrPath, .attrPath = *info.attrPath,
installable2->outputsSpec .outputs = *info.extendedOutputsSpec,
}; };
if(drv.priority) {
element.priority = *drv.priority;
}
} }
if(priority) { // if --priority was specified we want to override the priority of the installable // If --priority was specified we want to override the
element.priority = *priority; // priority of the installable.
}; element.priority =
priority
? *priority
: info.priority.value_or(defaultPriority);
element.updateStorePaths(getEvalStore(), store, builtPaths[installable.get()]); element.updateStorePaths(getEvalStore(), store, res);
manifest.elements.push_back(std::move(element)); manifest.elements.push_back(std::move(element));
} }
@ -487,18 +493,22 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
Strings{}, Strings{},
lockFlags); lockFlags);
auto [attrPath, resolvedRef, drv] = installable->toDerivation(); auto derivedPaths = installable->toDerivedPaths();
if (derivedPaths.empty()) continue;
auto & info = derivedPaths[0].info;
if (element.source->resolvedRef == resolvedRef) continue; assert(info.resolvedRef && info.attrPath);
if (element.source->resolvedRef == info.resolvedRef) continue;
printInfo("upgrading '%s' from flake '%s' to '%s'", printInfo("upgrading '%s' from flake '%s' to '%s'",
element.source->attrPath, element.source->resolvedRef, resolvedRef); element.source->attrPath, element.source->resolvedRef, *info.resolvedRef);
element.source = ProfileElementSource { element.source = ProfileElementSource {
installable->flakeRef, .originalRef = installable->flakeRef,
resolvedRef, .resolvedRef = *info.resolvedRef,
attrPath, .attrPath = *info.attrPath,
installable->outputsSpec .outputs = installable->extendedOutputsSpec,
}; };
installables.push_back(installable); installables.push_back(installable);
@ -526,7 +536,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
for (size_t i = 0; i < installables.size(); ++i) { for (size_t i = 0; i < installables.size(); ++i) {
auto & installable = installables.at(i); auto & installable = installables.at(i);
auto & element = manifest.elements[indices.at(i)]; auto & element = manifest.elements[indices.at(i)];
element.updateStorePaths(getEvalStore(), store, builtPaths[installable.get()]); element.updateStorePaths(getEvalStore(), store, builtPaths[installable.get()].first);
} }
updateProfile(manifest.build(store)); updateProfile(manifest.build(store));
@ -554,8 +564,8 @@ struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultPro
for (size_t i = 0; i < manifest.elements.size(); ++i) { for (size_t i = 0; i < manifest.elements.size(); ++i) {
auto & element(manifest.elements[i]); auto & element(manifest.elements[i]);
logger->cout("%d %s %s %s", i, logger->cout("%d %s %s %s", i,
element.source ? element.source->originalRef.to_string() + "#" + element.source->attrPath + printOutputsSpec(element.source->outputs) : "-", element.source ? element.source->originalRef.to_string() + "#" + element.source->attrPath + element.source->outputs.to_string() : "-",
element.source ? element.source->resolvedRef.to_string() + "#" + element.source->attrPath + printOutputsSpec(element.source->outputs) : "-", element.source ? element.source->resolvedRef.to_string() + "#" + element.source->attrPath + element.source->outputs.to_string() : "-",
concatStringsSep(" ", store->printStorePathSet(element.storePaths))); concatStringsSep(" ", store->printStorePathSet(element.storePaths)));
} }
} }

View file

@ -33,13 +33,7 @@ struct CmdCopyLog : virtual CopyCommand, virtual InstallablesCommand
auto dstStore = getDstStore(); auto dstStore = getDstStore();
auto & dstLogStore = require<LogStore>(*dstStore); auto & dstLogStore = require<LogStore>(*dstStore);
StorePathSet drvPaths; for (auto & drvPath : Installable::toDerivations(getEvalStore(), installables, true)) {
for (auto & i : installables)
for (auto & drvPath : i->toDrvPaths(getEvalStore()))
drvPaths.insert(drvPath);
for (auto & drvPath : drvPaths) {
if (auto log = srcLogStore.getBuildLog(drvPath)) if (auto log = srcLogStore.getBuildLog(drvPath))
dstLogStore.addBuildLog(drvPath, *log); dstLogStore.addBuildLog(drvPath, *log);
else else

View file

@ -1,53 +1,6 @@
{ busybox }: { busybox }:
with import ./config.nix; import ./build-hook.nix {
inherit busybox;
let contentAddressed = true;
}
mkDerivation = args:
derivation ({
inherit system;
builder = busybox;
args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")];
outputHashMode = "recursive";
outputHashAlgo = "sha256";
__contentAddressed = true;
} // removeAttrs args ["builder" "meta"])
// { meta = args.meta or {}; };
input1 = mkDerivation {
shell = busybox;
name = "build-remote-input-1";
buildCommand = "echo FOO > $out";
requiredSystemFeatures = ["foo"];
};
input2 = mkDerivation {
shell = busybox;
name = "build-remote-input-2";
buildCommand = "echo BAR > $out";
requiredSystemFeatures = ["bar"];
};
input3 = mkDerivation {
shell = busybox;
name = "build-remote-input-3";
buildCommand = ''
read x < ${input2}
echo $x BAZ > $out
'';
requiredSystemFeatures = ["baz"];
};
in
mkDerivation {
shell = busybox;
name = "build-remote";
buildCommand =
''
read x < ${input1}
read y < ${input3}
echo "$x $y" > $out
'';
}

View file

@ -1,15 +1,22 @@
{ busybox }: { busybox, contentAddressed ? false }:
with import ./config.nix; with import ./config.nix;
let let
caArgs = if contentAddressed then {
outputHashMode = "recursive";
outputHashAlgo = "sha256";
__contentAddressed = true;
} else {};
mkDerivation = args: mkDerivation = args:
derivation ({ derivation ({
inherit system; inherit system;
builder = busybox; builder = busybox;
args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")];
} // removeAttrs args ["builder" "meta" "passthru"]) } // removeAttrs args ["builder" "meta" "passthru"]
// caArgs)
// { meta = args.meta or {}; passthru = args.passthru or {}; }; // { meta = args.meta or {}; passthru = args.passthru or {}; };
input1 = mkDerivation { input1 = mkDerivation {

View file

@ -63,12 +63,9 @@ nix path-info --store $TEST_ROOT/machine3 --all \
| grep builder-build-remote-input-3.sh | grep builder-build-remote-input-3.sh
# Temporarily disabled because of https://github.com/NixOS/nix/issues/6209 for i in input1 input3; do
if [[ -z "$CONTENT_ADDRESSED" ]]; then nix log --store $TEST_ROOT/machine0 --file "$file" --arg busybox $busybox passthru."$i" | grep hi-$i
for i in input1 input3; do done
nix log --store $TEST_ROOT/machine0 --file "$file" --arg busybox $busybox passthru."$i" | grep hi-$i
done
fi
# Behavior of keep-failed # Behavior of keep-failed
out="$(nix-build 2>&1 failing.nix \ out="$(nix-build 2>&1 failing.nix \

View file

@ -16,7 +16,7 @@ makeTest ({
{ virtualisation.writableStore = true; { virtualisation.writableStore = true;
virtualisation.diskSize = 2048; virtualisation.diskSize = 2048;
virtualisation.additionalPaths = virtualisation.additionalPaths =
[ pkgs.stdenv [ pkgs.stdenvNoCC
(import ./systemd-nspawn.nix { inherit nixpkgs; }).toplevel (import ./systemd-nspawn.nix { inherit nixpkgs; }).toplevel
]; ];
virtualisation.memorySize = 4096; virtualisation.memorySize = 4096;
@ -38,30 +38,30 @@ makeTest ({
# Test that 'id' gives the expected result in various configurations. # Test that 'id' gives the expected result in various configurations.
# Existing UIDs, sandbox. # Existing UIDs, sandbox.
host.succeed("nix build --no-auto-allocate-uids --sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-1") host.succeed("nix build -v --no-auto-allocate-uids --sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-1")
host.succeed("[[ $(cat ./result) = 'uid=1000(nixbld) gid=100(nixbld) groups=100(nixbld)' ]]") host.succeed("[[ $(cat ./result) = 'uid=1000(nixbld) gid=100(nixbld) groups=100(nixbld)' ]]")
# Existing UIDs, no sandbox. # Existing UIDs, no sandbox.
host.succeed("nix build --no-auto-allocate-uids --no-sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-2") host.succeed("nix build -v --no-auto-allocate-uids --no-sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-2")
host.succeed("[[ $(cat ./result) = 'uid=30001(nixbld1) gid=30000(nixbld) groups=30000(nixbld)' ]]") host.succeed("[[ $(cat ./result) = 'uid=30001(nixbld1) gid=30000(nixbld) groups=30000(nixbld)' ]]")
# Auto-allocated UIDs, sandbox. # Auto-allocated UIDs, sandbox.
host.succeed("nix build --auto-allocate-uids --sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-3") host.succeed("nix build -v --auto-allocate-uids --sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-3")
host.succeed("[[ $(cat ./result) = 'uid=1000(nixbld) gid=100(nixbld) groups=100(nixbld)' ]]") host.succeed("[[ $(cat ./result) = 'uid=1000(nixbld) gid=100(nixbld) groups=100(nixbld)' ]]")
# Auto-allocated UIDs, no sandbox. # Auto-allocated UIDs, no sandbox.
host.succeed("nix build --auto-allocate-uids --no-sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-4") host.succeed("nix build -v --auto-allocate-uids --no-sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-4")
host.succeed("[[ $(cat ./result) = 'uid=872415232 gid=30000(nixbld) groups=30000(nixbld)' ]]") host.succeed("[[ $(cat ./result) = 'uid=872415232 gid=30000(nixbld) groups=30000(nixbld)' ]]")
# Auto-allocated UIDs, UID range, sandbox. # Auto-allocated UIDs, UID range, sandbox.
host.succeed("nix build --auto-allocate-uids --sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-5 --arg uidRange true") host.succeed("nix build -v --auto-allocate-uids --sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-5 --arg uidRange true")
host.succeed("[[ $(cat ./result) = 'uid=0(root) gid=0(root) groups=0(root)' ]]") host.succeed("[[ $(cat ./result) = 'uid=0(root) gid=0(root) groups=0(root)' ]]")
# Auto-allocated UIDs, UID range, no sandbox. # Auto-allocated UIDs, UID range, no sandbox.
host.fail("nix build --auto-allocate-uids --no-sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-6 --arg uidRange true") host.fail("nix build -v --auto-allocate-uids --no-sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-6 --arg uidRange true")
# Run systemd-nspawn in a Nix build. # Run systemd-nspawn in a Nix build.
host.succeed("nix build --auto-allocate-uids --sandbox -L --offline --impure --file ${./systemd-nspawn.nix} --argstr nixpkgs ${nixpkgs}") host.succeed("nix build -v --auto-allocate-uids --sandbox -L --offline --impure --file ${./systemd-nspawn.nix} --argstr nixpkgs ${nixpkgs}")
host.succeed("[[ $(cat ./result/msg) = 'Hello World' ]]") host.succeed("[[ $(cat ./result/msg) = 'Hello World' ]]")
''; '';

View file

@ -0,0 +1,66 @@
source ./common.sh
flake1Dir=$TEST_ROOT/flake1
flake2Dir=$TEST_ROOT/flake2
mkdir -p $flake1Dir $flake2Dir
writeSimpleFlake $flake2Dir
tar cfz $TEST_ROOT/flake.tar.gz -C $TEST_ROOT flake2
hash=$(nix hash path $flake2Dir)
dep=$(nix store add-path ./common.sh)
cat > $flake1Dir/flake.nix <<EOF
{
inputs.flake2.url = "file://$TEST_ROOT/flake.tar.gz";
outputs = { self, flake2 }: {
a1 = builtins.fetchTarball {
#type = "tarball";
url = "file://$TEST_ROOT/flake.tar.gz";
sha256 = "$hash";
};
a2 = ./foo;
a3 = ./.;
a4 = self.outPath;
# FIXME
a5 = self;
a6 = flake2.outPath;
# FIXME
a7 = "${flake2}/config.nix";
# This is only allowed in impure mode.
a8 = builtins.storePath $dep;
a9 = "$dep";
};
}
EOF
echo bar > $flake1Dir/foo
nix build --json --out-link $TEST_ROOT/result $flake1Dir#a1
[[ -e $TEST_ROOT/result/simple.nix ]]
nix build --json --out-link $TEST_ROOT/result $flake1Dir#a2
[[ $(cat $TEST_ROOT/result) = bar ]]
nix build --json --out-link $TEST_ROOT/result $flake1Dir#a3
nix build --json --out-link $TEST_ROOT/result $flake1Dir#a4
nix build --json --out-link $TEST_ROOT/result $flake1Dir#a6
[[ -e $TEST_ROOT/result/simple.nix ]]
nix build --impure --json --out-link $TEST_ROOT/result $flake1Dir#a8
diff common.sh $TEST_ROOT/result
(! nix build --impure --json --out-link $TEST_ROOT/result $flake1Dir#a9)

View file

@ -9,6 +9,7 @@ nix_tests = \
flakes/check.sh \ flakes/check.sh \
flakes/unlocked-override.sh \ flakes/unlocked-override.sh \
flakes/absolute-paths.sh \ flakes/absolute-paths.sh \
flakes/build-paths.sh \
ca/gc.sh \ ca/gc.sh \
gc.sh \ gc.sh \
remote-store.sh \ remote-store.sh \

View file

@ -8,4 +8,4 @@ libplugintest_ALLOW_UNDEFINED := 1
libplugintest_EXCLUDE_FROM_LIBRARY_LIST := 1 libplugintest_EXCLUDE_FROM_LIBRARY_LIST := 1
libplugintest_CXXFLAGS := -I src/libutil -I src/libexpr libplugintest_CXXFLAGS := -I src/libutil -I src/libstore -I src/libexpr

View file

@ -15,7 +15,7 @@ makeTest {
{ virtualisation.writableStore = true; { virtualisation.writableStore = true;
nix.settings.substituters = lib.mkForce [ ]; nix.settings.substituters = lib.mkForce [ ];
nix.nixPath = [ "nixpkgs=${lib.cleanSource pkgs.path}" ]; nix.nixPath = [ "nixpkgs=${lib.cleanSource pkgs.path}" ];
virtualisation.additionalPaths = [ pkgs.stdenv pkgs.pkgsi686Linux.stdenv ]; virtualisation.additionalPaths = [ pkgs.stdenvNoCC pkgs.pkgsi686Linux.stdenvNoCC ];
}; };
testScript = { nodes }: '' testScript = { nodes }: ''