lunaphied/lix
0
0
Fork 0
forked from lix-project/lix
Code Pull requests Activity

Merge remote-tracking branch 'upstream/master' into remove-storetype-delegate-regStore

Browse source
This commit is contained in:
John Ericson 2020-09-16 22:35:24 +00:00
parent c08c9f08c7 5080d4e7b2
commit f60b380a7f
62 changed files with 1694 additions and 857 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
doc/manual/generate-builtins.jq
View file

@ -1,6 +0,0 @@
. | to_entries | sort_by(.key) | map(
" - `builtins." + .key + "` "
+ (.value.args | map("*" + . + "*") | join(" "))
+ " \n\n"
+ (.value.doc | split("\n") | map(" " + . + "\n") | join("")) + "\n\n"
) | join("")

14
doc/manual/generate-builtins.nix Normal file
View file

@ -0,0 +1,14 @@
with builtins;
with import ./utils.nix;
builtins:
concatStrings (map
(name:
let builtin = builtins.${name}; in
" - `builtins.${name}` " + concatStringsSep " " (map (s: "*${s}*") builtin.args)
+ " \n\n"
+ concatStrings (map (s: " ${s}\n") (splitLines builtin.doc)) + "\n\n"
)
(attrNames builtins))

44
doc/manual/generate-manpage.jq
View file

@ -1,44 +0,0 @@
def show_flags:
.flags
| map_values(select(.category != "config"))
| to_entries
| map(
" - `--" + .key + "`"
+ (if .value.shortName then " / `" + .value.shortName + "`" else "" end)
+ (if .value.labels then " " + (.value.labels | map("*" + . + "*") | join(" ")) else "" end)
+ " \n"
+ " " + .value.description + "\n\n")
| join("")
;
def show_synopsis:
"`" + .command + "` [*flags*...] " + (.args | map("*" + .label + "*" + (if has("arity") then "" else "..." end)) | join(" ")) + "\n\n"
;
def show_command:
. as $top |
.section + " Name\n\n"
+ "`" + .command + "` - " + .def.description + "\n\n"
+ .section + " Synopsis\n\n"
+ ({"command": .command, "args": .def.args} | show_synopsis)
+ (if .def | has("doc")
then .section + " Description\n\n" + .def.doc + "\n\n"
else ""
end)
+ (if (.def.flags | length) > 0 then
.section + " Flags\n\n"
+ (.def | show_flags)
else "" end)
+ (if (.def.examples | length) > 0 then
.section + " Examples\n\n"
+ (.def.examples | map(.description + "\n\n```console\n" + .command + "\n```\n" ) | join("\n"))
+ "\n"
else "" end)
+ (if .def.commands then .def.commands | to_entries | map(
"# Subcommand `" + ($top.command + " " + .key) + "`\n\n"
+ ({"command": ($top.command + " " + .key), "section": "##", "def": .value} | show_command)
) | join("") else "" end)
;
"Title: nix\n\n"
+ ({"command": "nix", "section": "#", "def": .} | show_command)

56
doc/manual/generate-manpage.nix Normal file
View file

@ -0,0 +1,56 @@
with builtins;
with import ./utils.nix;
let
showCommand =
{ command, section, def }:
"${section} Name\n\n"
+ "`${command}` - ${def.description}\n\n"
+ "${section} Synopsis\n\n"
+ showSynopsis { inherit command; args = def.args; }
+ (if def ? doc
then "${section} Description\n\n" + def.doc + "\n\n"
else "")
+ (let s = showFlags def.flags; in
if s != ""
then "${section} Flags\n\n${s}"
else "")
+ (if def.examples or [] != []
then
"${section} Examples\n\n"
+ concatStrings (map ({ description, command }: "${description}\n\n```console\n${command}\n```\n\n") def.examples)
else "")
+ (if def.commands or [] != []
then concatStrings (
map (name:
"# Subcommand `${command} ${name}`\n\n"
+ showCommand { command = command + " " + name; section = "##"; def = def.commands.${name}; })
(attrNames def.commands))
else "");
showFlags = flags:
concatStrings
(map (longName:
let flag = flags.${longName}; in
if flag.category or "" != "config"
then
" - `--${longName}`"
+ (if flag ? shortName then " / `${flag.shortName}`" else "")
+ (if flag ? labels then " " + (concatStringsSep " " (map (s: "*${s}*") flag.labels)) else "")
+ " \n"
+ " " + flag.description + "\n\n"
else "")
(attrNames flags));
showSynopsis =
{ command, args }:
"`${command}` [*flags*...] ${concatStringsSep " "
(map (arg: "*${arg.label}*" + (if arg ? arity then "" else "...")) args)}\n\n";
in
command:
"Title: nix\n\n"
+ showCommand { command = "nix"; section = "#"; def = command; }

16
doc/manual/generate-options.jq
View file

@ -1,16 +0,0 @@
. | to_entries | sort_by(.key) | map(
" - `" + .key + "` \n\n"
+ (.value.description | split("\n") | map(" " + . + "\n") | join("")) + "\n\n"
+ " **Default:** " + (
if .value.value == "" or .value.value == []
then "*empty*"
elif (.value.value | type) == "array"
then "`" + (.value.value | join(" ")) + "`"
else "`" + (.value.value | tostring) + "`"
end)
+ "\n\n"
+ (if (.value.aliases | length) > 0
then " **Deprecated alias:** " + (.value.aliases | map("`" + . + "`") | join(", ")) + "\n\n"
else ""
end)
) | join("")

21
doc/manual/generate-options.nix Normal file
View file

@ -0,0 +1,21 @@
with builtins;
with import ./utils.nix;
options:
concatStrings (map
(name:
let option = options.${name}; in
" - `${name}` \n\n"
+ concatStrings (map (s: " ${s}\n") (splitLines option.description)) + "\n\n"
+ " **Default:** " + (
if option.value == "" || option.value == []
then "*empty*"
else if isBool option.value
then (if option.value then "`true`" else "`false`")
else "`" + toString option.value + "`") + "\n\n"
+ (if option.aliases != []
then " **Deprecated alias:** " + (concatStringsSep ", " (map (s: "`${s}`") option.aliases)) + "\n\n"
else "")
)
(attrNames options))

30
doc/manual/local.mk
View file

@ -15,6 +15,8 @@ clean-files += $(d)/*.1 $(d)/*.5 $(d)/*.8
dist-files += $(man-pages) dist-files += $(man-pages)
nix-eval = $(bindir)/nix eval --experimental-features nix-command -I nix/corepkgs=corepkgs --store dummy:// --impure --raw --expr
$(d)/%.1: $(d)/src/command-ref/%.md $(d)/%.1: $(d)/src/command-ref/%.md
$(trace-gen) lowdown -sT man $^ -o $@ $(trace-gen) lowdown -sT man $^ -o $@
@ -24,25 +26,31 @@ $(d)/%.8: $(d)/src/command-ref/%.md
$(d)/nix.conf.5: $(d)/src/command-ref/conf-file.md $(d)/nix.conf.5: $(d)/src/command-ref/conf-file.md
$(trace-gen) lowdown -sT man $^ -o $@ $(trace-gen) lowdown -sT man $^ -o $@
$(d)/src/command-ref/nix.md: $(d)/nix.json $(d)/generate-manpage.jq $(d)/src/command-ref/nix.md: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix
jq -r -f doc/manual/generate-manpage.jq $< > $@ $(trace-gen) $(nix-eval) 'import doc/manual/generate-manpage.nix (builtins.fromJSON (builtins.readFile $<))' > $@.tmp
@mv $@.tmp $@
$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/generate-options.jq $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/generate-options.nix $(d)/src/command-ref/conf-file-prefix.md $(bindir)/nix
cat doc/manual/src/command-ref/conf-file-prefix.md > $@ @cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp
jq -r -f doc/manual/generate-options.jq $< >> $@ $(trace-gen) $(nix-eval) 'import doc/manual/generate-options.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp
@mv $@.tmp $@
$(d)/nix.json: $(bindir)/nix $(d)/nix.json: $(bindir)/nix
$(trace-gen) $(bindir)/nix __dump-args > $@ $(trace-gen) $(bindir)/nix __dump-args > $@.tmp
@mv $@.tmp $@
$(d)/conf-file.json: $(bindir)/nix $(d)/conf-file.json: $(bindir)/nix
$(trace-gen) env -i NIX_CONF_DIR=/dummy HOME=/dummy $(bindir)/nix show-config --json --experimental-features nix-command > $@ $(trace-gen) env -i NIX_CONF_DIR=/dummy HOME=/dummy $(bindir)/nix show-config --json --experimental-features nix-command > $@.tmp
@mv $@.tmp $@
$(d)/src/expressions/builtins.md: $(d)/builtins.json $(d)/generate-builtins.jq $(d)/src/expressions/builtins-prefix.md $(d)/src/expressions/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/expressions/builtins-prefix.md $(bindir)/nix
cat doc/manual/src/expressions/builtins-prefix.md > $@ @cat doc/manual/src/expressions/builtins-prefix.md > $@.tmp
jq -r -f doc/manual/generate-builtins.jq $< >> $@ $(trace-gen) $(nix-eval) 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp
@mv $@.tmp $@
$(d)/builtins.json: $(bindir)/nix $(d)/builtins.json: $(bindir)/nix
$(trace-gen) $(bindir)/nix __dump-builtins > $@ $(trace-gen) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-builtins > $@.tmp
mv $@.tmp $@
# Generate the HTML manual. # Generate the HTML manual.
install: $(docdir)/manual/index.html install: $(docdir)/manual/index.html

7
doc/manual/utils.nix Normal file
View file

@ -0,0 +1,7 @@
with builtins;
{
splitLines = s: filter (x: !isList x) (split "\n" s);
concatStrings = concatStringsSep "";
}

7
perl/lib/Nix/Store.xs
View file

@ -303,11 +303,14 @@ SV * derivationFromPath(char * drvPath)
hash = newHV(); hash = newHV();
HV * outputs = newHV(); HV * outputs = newHV();
for (auto & i : drv.outputsAndPaths(*store())) for (auto & i : drv.outputsAndOptPaths(*store())) {
hv_store( hv_store(
outputs, i.first.c_str(), i.first.size(), outputs, i.first.c_str(), i.first.size(),
newSVpv(store()->printStorePath(i.second.second).c_str(), 0), !i.second.second
? newSV(0) /* null value */
: newSVpv(store()->printStorePath(*i.second.second).c_str(), 0),
0); 0);
}
hv_stores(hash, "outputs", newRV((SV *) outputs)); hv_stores(hash, "outputs", newRV((SV *) outputs));
AV * inputDrvs = newAV(); AV * inputDrvs = newAV();

6
src/libexpr/eval.cc
View file

@ -370,7 +370,11 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
for (auto & i : _searchPath) addToSearchPath(i); for (auto & i : _searchPath) addToSearchPath(i);
for (auto & i : evalSettings.nixPath.get()) addToSearchPath(i); for (auto & i : evalSettings.nixPath.get()) addToSearchPath(i);
} }
addToSearchPath("nix=" + canonPath(settings.nixDataDir + "/nix/corepkgs", true));
try {
addToSearchPath("nix=" + canonPath(settings.nixDataDir + "/nix/corepkgs", true));
} catch (Error &) {
}
if (evalSettings.restrictEval || evalSettings.pureEval) { if (evalSettings.restrictEval || evalSettings.pureEval) {
allowedPaths = PathSet(); allowedPaths = PathSet();

2
src/libexpr/flake/lockfile.hh
View file

@ -6,7 +6,7 @@
namespace nix { namespace nix {
class Store; class Store;
struct StorePath; class StorePath;
} }
namespace nix::flake { namespace nix::flake {

14
src/libexpr/get-drvs.cc
View file

@ -38,8 +38,11 @@ DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPat
auto i = drv.outputs.find(outputName); auto i = drv.outputs.find(outputName);
if (i == drv.outputs.end()) if (i == drv.outputs.end())
throw Error("derivation '%s' does not have output '%s'", store->printStorePath(drvPath), outputName); throw Error("derivation '%s' does not have output '%s'", store->printStorePath(drvPath), outputName);
auto & [outputName, output] = *i;
outPath = store->printStorePath(i->second.path(*store, drv.name)); auto optStorePath = output.path(*store, drv.name, outputName);
if (optStorePath)
outPath = store->printStorePath(*optStorePath);
} }
@ -77,12 +80,15 @@ string DrvInfo::queryDrvPath() const
string DrvInfo::queryOutPath() const string DrvInfo::queryOutPath() const
{ {
if (outPath == "" && attrs) { if (!outPath && attrs) {
Bindings::iterator i = attrs->find(state->sOutPath); Bindings::iterator i = attrs->find(state->sOutPath);
PathSet context; PathSet context;
outPath = i != attrs->end() ? state->coerceToPath(*i->pos, *i->value, context) : ""; if (i != attrs->end())
outPath = state->coerceToPath(*i->pos, *i->value, context);
} }
return outPath; if (!outPath)
throw UnimplementedError("CA derivations are not yet supported");
return *outPath;
} }

2
src/libexpr/get-drvs.hh
View file

@ -20,7 +20,7 @@ private:
mutable string name; mutable string name;
mutable string system; mutable string system;
mutable string drvPath; mutable string drvPath;
mutable string outPath; mutable std::optional<string> outPath;
mutable string outputName; mutable string outputName;
Outputs outputs; Outputs outputs;

86
src/libexpr/primops.cc
View file

@ -44,16 +44,6 @@ void EvalState::realiseContext(const PathSet & context)
throw InvalidPathError(store->printStorePath(ctx)); throw InvalidPathError(store->printStorePath(ctx));
if (!outputName.empty() && ctx.isDerivation()) { if (!outputName.empty() && ctx.isDerivation()) {
drvs.push_back(StorePathWithOutputs{ctx, {outputName}}); drvs.push_back(StorePathWithOutputs{ctx, {outputName}});
/* Add the output of this derivation to the allowed
paths. */
if (allowedPaths) {
auto drv = store->derivationFromPath(ctx);
DerivationOutputs::iterator i = drv.outputs.find(outputName);
if (i == drv.outputs.end())
throw Error("derivation '%s' does not have an output named '%s'", ctxS, outputName);
allowedPaths->insert(store->printStorePath(i->second.path(*store, drv.name)));
}
} }
} }
@ -69,8 +59,50 @@ void EvalState::realiseContext(const PathSet & context)
store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize); store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize);
store->buildPaths(drvs); store->buildPaths(drvs);
/* Add the output of this derivations to the allowed
paths. */
if (allowedPaths) {
for (auto & [drvPath, outputs] : drvs) {
auto outputPaths = store->queryDerivationOutputMap(drvPath);
for (auto & outputName : outputs) {
if (outputPaths.count(outputName) == 0)
throw Error("derivation '%s' does not have an output named '%s'",
store->printStorePath(drvPath), outputName);
allowedPaths->insert(store->printStorePath(outputPaths.at(outputName)));
}
}
}
} }
/* Add and attribute to the given attribute map from the output name to
the output path, or a placeholder.
Where possible the path is used, but for floating CA derivations we
may not know it. For sake of determinism we always assume we don't
and instead put in a place holder. In either case, however, the
string context will contain the drv path and output name, so
downstream derivations will have the proper dependency, and in
addition, before building, the placeholder will be rewritten to be
the actual path.
The 'drv' and 'drvPath' outputs must correspond. */
static void mkOutputString(EvalState & state, Value & v,
const StorePath & drvPath, const BasicDerivation & drv,
std::pair<string, DerivationOutput> o)
{
auto optOutputPath = o.second.path(*state.store, drv.name, o.first);
mkString(
*state.allocAttr(v, state.symbols.create(o.first)),
optOutputPath
? state.store->printStorePath(*optOutputPath)
/* Downstream we would substitute this for an actual path once
we build the floating CA derivation */
/* FIXME: we need to depend on the basic derivation, not
derivation */
: downstreamPlaceholder(*state.store, drvPath, o.first),
{"!" + o.first + "!" + state.store->printStorePath(drvPath)});
}
/* Load and evaluate an expression from path specified by the /* Load and evaluate an expression from path specified by the
argument. */ argument. */
@ -114,9 +146,8 @@ static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vS
state.mkList(*outputsVal, drv.outputs.size()); state.mkList(*outputsVal, drv.outputs.size());
unsigned int outputs_index = 0; unsigned int outputs_index = 0;
for (const auto & o : drv.outputsAndPaths(*state.store)) { for (const auto & o : drv.outputs) {
v2 = state.allocAttr(w, state.symbols.create(o.first)); mkOutputString(state, w, storePath, drv, o);
mkString(*v2, state.store->printStorePath(o.second.second), {"!" + o.first + "!" + path});
outputsVal->listElems()[outputs_index] = state.allocValue(); outputsVal->listElems()[outputs_index] = state.allocValue();
mkString(*(outputsVal->listElems()[outputs_index++]), o.first); mkString(*(outputsVal->listElems()[outputs_index++]), o.first);
} }
@ -1080,16 +1111,18 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
/* Optimisation, but required in read-only mode! because in that /* Optimisation, but required in read-only mode! because in that
case we don't actually write store derivations, so we can't case we don't actually write store derivations, so we can't
read them later. */ read them later.
drvHashes.insert_or_assign(drvPath,
hashDerivationModulo(*state.store, Derivation(drv), false)); However, we don't bother doing this for floating CA derivations because
their "hash modulo" is indeterminate until built. */
if (drv.type() != DerivationType::CAFloating)
drvHashes.insert_or_assign(drvPath,
hashDerivationModulo(*state.store, Derivation(drv), false));
state.mkAttrs(v, 1 + drv.outputs.size()); state.mkAttrs(v, 1 + drv.outputs.size());
mkString(*state.allocAttr(v, state.sDrvPath), drvPathS, {"=" + drvPathS}); mkString(*state.allocAttr(v, state.sDrvPath), drvPathS, {"=" + drvPathS});
for (auto & i : drv.outputsAndPaths(*state.store)) { for (auto & i : drv.outputs)
mkString(*state.allocAttr(v, state.symbols.create(i.first)), mkOutputString(state, v, drvPath, drv, i);
state.store->printStorePath(i.second.second), {"!" + i.first + "!" + drvPathS});
}
v.attrs->sort(); v.attrs->sort();
} }
@ -1671,7 +1704,7 @@ static RegisterPrimOp primop_toFile({
... ...
cp ${configFile} $out/etc/foo.conf cp ${configFile} $out/etc/foo.conf
"; ";
``` ```
Note that `${configFile}` is an Note that `${configFile}` is an
[antiquotation](language-values.md), so the result of the [antiquotation](language-values.md), so the result of the
@ -3532,10 +3565,13 @@ void EvalState::createBaseEnv()
/* Add a wrapper around the derivation primop that computes the /* Add a wrapper around the derivation primop that computes the
`drvPath' and `outPath' attributes lazily. */ `drvPath' and `outPath' attributes lazily. */
string path = canonPath(settings.nixDataDir + "/nix/corepkgs/derivation.nix", true); try {
sDerivationNix = symbols.create(path); string path = canonPath(settings.nixDataDir + "/nix/corepkgs/derivation.nix", true);
evalFile(path, v); sDerivationNix = symbols.create(path);
addConstant("derivation", v); evalFile(path, v);
addConstant("derivation", v);
} catch (SysError &) {
}
/* Now that we've added all primops, sort the `builtins' set, /* Now that we've added all primops, sort the `builtins' set,
because attribute lookups expect it to be sorted. */ because attribute lookups expect it to be sorted. */

2
src/libfetchers/fetchers.hh
View file

@ -23,7 +23,7 @@ struct InputScheme;
struct Input struct Input
{ {
friend class InputScheme; friend struct InputScheme;
std::shared_ptr<InputScheme> scheme; // note: can be null std::shared_ptr<InputScheme> scheme; // note: can be null
Attrs attrs; Attrs attrs;

3
src/libstore/binary-cache-store.cc
View file

@ -22,7 +22,8 @@
namespace nix { namespace nix {
BinaryCacheStore::BinaryCacheStore(const Params & params) BinaryCacheStore::BinaryCacheStore(const Params & params)
: Store(params) : BinaryCacheStoreConfig(params)
, Store(params)
{ {
if (secretKeyFile != "") if (secretKeyFile != "")
secretKey = std::unique_ptr<SecretKey>(new SecretKey(readFile(secretKeyFile))); secretKey = std::unique_ptr<SecretKey>(new SecretKey(readFile(secretKeyFile)));

22
src/libstore/binary-cache-store.hh
View file

@ -11,17 +11,21 @@ namespace nix {
struct NarInfo; struct NarInfo;
class BinaryCacheStore : public Store struct BinaryCacheStoreConfig : virtual StoreConfig
{ {
public: using StoreConfig::StoreConfig;
const Setting<std::string> compression{this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"}; const Setting<std::string> compression{(StoreConfig*) this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"};
const Setting<bool> writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"}; const Setting<bool> writeNARListing{(StoreConfig*) this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"};
const Setting<bool> writeDebugInfo{this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"}; const Setting<bool> writeDebugInfo{(StoreConfig*) this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"};
const Setting<Path> secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"}; const Setting<Path> secretKeyFile{(StoreConfig*) this, "", "secret-key", "path to secret key used to sign the binary cache"};
const Setting<Path> localNarCache{this, "", "local-nar-cache", "path to a local cache of NARs"}; const Setting<Path> localNarCache{(StoreConfig*) this, "", "local-nar-cache", "path to a local cache of NARs"};
const Setting<bool> parallelCompression{this, false, "parallel-compression", const Setting<bool> parallelCompression{(StoreConfig*) this, false, "parallel-compression",
"enable multi-threading compression, available for xz only currently"}; "enable multi-threading compression, available for xz only currently"};
};
class BinaryCacheStore : public Store, public virtual BinaryCacheStoreConfig
{
private: private:
@ -58,7 +62,7 @@ public:
public: public:
virtual void init(); virtual void init() override;
private: private:

1026
src/libstore/build.cc
View file

File diff suppressed because it is too large Load diff

140
src/libstore/derivations.cc
View file

@ -7,7 +7,7 @@
namespace nix { namespace nix {
std::optional<StorePath> DerivationOutput::pathOpt(const Store & store, std::string_view drvName) const std::optional<StorePath> DerivationOutput::path(const Store & store, std::string_view drvName, std::string_view outputName) const
{ {
return std::visit(overloaded { return std::visit(overloaded {
[](DerivationOutputInputAddressed doi) -> std::optional<StorePath> { [](DerivationOutputInputAddressed doi) -> std::optional<StorePath> {
@ -15,7 +15,7 @@ std::optional<StorePath> DerivationOutput::pathOpt(const Store & store, std::str
}, },
[&](DerivationOutputCAFixed dof) -> std::optional<StorePath> { [&](DerivationOutputCAFixed dof) -> std::optional<StorePath> {
return { return {
store.makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName) dof.path(store, drvName, outputName)
}; };
}, },
[](DerivationOutputCAFloating dof) -> std::optional<StorePath> { [](DerivationOutputCAFloating dof) -> std::optional<StorePath> {
@ -25,6 +25,13 @@ std::optional<StorePath> DerivationOutput::pathOpt(const Store & store, std::str
} }
StorePath DerivationOutputCAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const {
return store.makeFixedOutputPath(
hash.method, hash.hash,
outputPathName(drvName, outputName));
}
bool derivationIsCA(DerivationType dt) { bool derivationIsCA(DerivationType dt) {
switch (dt) { switch (dt) {
case DerivationType::InputAddressed: return false; case DerivationType::InputAddressed: return false;
@ -106,12 +113,15 @@ static string parseString(std::istream & str)
return res; return res;
} }
static void validatePath(std::string_view s) {
if (s.size() == 0 || s[0] != '/')
throw FormatError("bad path '%1%' in derivation", s);
}
static Path parsePath(std::istream & str) static Path parsePath(std::istream & str)
{ {
string s = parseString(str); auto s = parseString(str);
if (s.size() == 0 || s[0] != '/') validatePath(s);
throw FormatError("bad path '%1%' in derivation", s);
return s; return s;
} }
@ -140,7 +150,7 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
static DerivationOutput parseDerivationOutput(const Store & store, static DerivationOutput parseDerivationOutput(const Store & store,
StorePath path, std::string_view hashAlgo, std::string_view hash) std::string_view pathS, std::string_view hashAlgo, std::string_view hash)
{ {
if (hashAlgo != "") { if (hashAlgo != "") {
auto method = FileIngestionMethod::Flat; auto method = FileIngestionMethod::Flat;
@ -148,40 +158,45 @@ static DerivationOutput parseDerivationOutput(const Store & store,
method = FileIngestionMethod::Recursive; method = FileIngestionMethod::Recursive;
hashAlgo = hashAlgo.substr(2); hashAlgo = hashAlgo.substr(2);
} }
const HashType hashType = parseHashType(hashAlgo); const auto hashType = parseHashType(hashAlgo);
if (hash != "") {
return hash != "" validatePath(pathS);
? DerivationOutput { return DerivationOutput {
.output = DerivationOutputCAFixed { .output = DerivationOutputCAFixed {
.hash = FixedOutputHash { .hash = FixedOutputHash {
.method = std::move(method), .method = std::move(method),
.hash = Hash::parseNonSRIUnprefixed(hash, hashType), .hash = Hash::parseNonSRIUnprefixed(hash, hashType),
}, },
} },
} };
: (settings.requireExperimentalFeature("ca-derivations"), } else {
DerivationOutput { settings.requireExperimentalFeature("ca-derivations");
.output = DerivationOutputCAFloating { assert(pathS == "");
.method = std::move(method), return DerivationOutput {
.hashType = std::move(hashType), .output = DerivationOutputCAFloating {
}, .method = std::move(method),
}); .hashType = std::move(hashType),
} else },
};
}
} else {
validatePath(pathS);
return DerivationOutput { return DerivationOutput {
.output = DerivationOutputInputAddressed { .output = DerivationOutputInputAddressed {
.path = std::move(path), .path = store.parseStorePath(pathS),
} }
}; };
}
} }
static DerivationOutput parseDerivationOutput(const Store & store, std::istringstream & str) static DerivationOutput parseDerivationOutput(const Store & store, std::istringstream & str)
{ {
expect(str, ","); auto path = store.parseStorePath(parsePath(str)); expect(str, ","); const auto pathS = parseString(str);
expect(str, ","); const auto hashAlgo = parseString(str); expect(str, ","); const auto hashAlgo = parseString(str);
expect(str, ","); const auto hash = parseString(str); expect(str, ","); const auto hash = parseString(str);
expect(str, ")"); expect(str, ")");
return parseDerivationOutput(store, std::move(path), hashAlgo, hash); return parseDerivationOutput(store, pathS, hashAlgo, hash);
} }
@ -294,17 +309,19 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
for (auto & i : outputs) { for (auto & i : outputs) {
if (first) first = false; else s += ','; if (first) first = false; else s += ',';
s += '('; printUnquotedString(s, i.first); s += '('; printUnquotedString(s, i.first);
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path(store, name)));
std::visit(overloaded { std::visit(overloaded {
[&](DerivationOutputInputAddressed doi) { [&](DerivationOutputInputAddressed doi) {
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(doi.path));
s += ','; printUnquotedString(s, ""); s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, ""); s += ','; printUnquotedString(s, "");
}, },
[&](DerivationOutputCAFixed dof) { [&](DerivationOutputCAFixed dof) {
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first)));
s += ','; printUnquotedString(s, dof.hash.printMethodAlgo()); s += ','; printUnquotedString(s, dof.hash.printMethodAlgo());
s += ','; printUnquotedString(s, dof.hash.hash.to_string(Base16, false)); s += ','; printUnquotedString(s, dof.hash.hash.to_string(Base16, false));
}, },
[&](DerivationOutputCAFloating dof) { [&](DerivationOutputCAFloating dof) {
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)); s += ','; printUnquotedString(s, makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType));
s += ','; printUnquotedString(s, ""); s += ','; printUnquotedString(s, "");
}, },
@ -360,6 +377,16 @@ bool isDerivation(const string & fileName)
} }
std::string outputPathName(std::string_view drvName, std::string_view outputName) {
std::string res { drvName };
if (outputName != "out") {
res += "-";
res += outputName;
}
return res;
}
DerivationType BasicDerivation::type() const DerivationType BasicDerivation::type() const
{ {
std::set<std::string_view> inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs; std::set<std::string_view> inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs;
@ -452,12 +479,12 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
throw Error("Regular input-addressed derivations are not yet allowed to depend on CA derivations"); throw Error("Regular input-addressed derivations are not yet allowed to depend on CA derivations");
case DerivationType::CAFixed: { case DerivationType::CAFixed: {
std::map<std::string, Hash> outputHashes; std::map<std::string, Hash> outputHashes;
for (const auto & i : drv.outputsAndPaths(store)) { for (const auto & i : drv.outputs) {
auto & dof = std::get<DerivationOutputCAFixed>(i.second.first.output); auto & dof = std::get<DerivationOutputCAFixed>(i.second.output);
auto hash = hashString(htSHA256, "fixed:out:" auto hash = hashString(htSHA256, "fixed:out:"
+ dof.hash.printMethodAlgo() + ":" + dof.hash.printMethodAlgo() + ":"
+ dof.hash.hash.to_string(Base16, false) + ":" + dof.hash.hash.to_string(Base16, false) + ":"
+ store.printStorePath(i.second.second)); + store.printStorePath(dof.path(store, drv.name, i.first)));
outputHashes.insert_or_assign(i.first, std::move(hash)); outputHashes.insert_or_assign(i.first, std::move(hash));
} }
return outputHashes; return outputHashes;
@ -508,21 +535,13 @@ bool wantOutput(const string & output, const std::set<string> & wanted)
} }
StorePathSet BasicDerivation::outputPaths(const Store & store) const
{
StorePathSet paths;
for (auto & i : outputsAndPaths(store))
paths.insert(i.second.second);
return paths;
}
static DerivationOutput readDerivationOutput(Source & in, const Store & store) static DerivationOutput readDerivationOutput(Source & in, const Store & store)
{ {
auto path = store.parseStorePath(readString(in)); const auto pathS = readString(in);
const auto hashAlgo = readString(in); const auto hashAlgo = readString(in);
const auto hash = readString(in); const auto hash = readString(in);
return parseDerivationOutput(store, std::move(path), hashAlgo, hash); return parseDerivationOutput(store, pathS, hashAlgo, hash);
} }
StringSet BasicDerivation::outputNames() const StringSet BasicDerivation::outputNames() const
@ -533,23 +552,12 @@ StringSet BasicDerivation::outputNames() const
return names; return names;
} }
DerivationOutputsAndPaths BasicDerivation::outputsAndPaths(const Store & store) const {
DerivationOutputsAndPaths outsAndPaths;
for (auto output : outputs)
outsAndPaths.insert(std::make_pair(
output.first,
std::make_pair(output.second, output.second.path(store, name))
)
);
return outsAndPaths;
}
DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const { DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const {
DerivationOutputsAndOptPaths outsAndOptPaths; DerivationOutputsAndOptPaths outsAndOptPaths;
for (auto output : outputs) for (auto output : outputs)
outsAndOptPaths.insert(std::make_pair( outsAndOptPaths.insert(std::make_pair(
output.first, output.first,
std::make_pair(output.second, output.second.pathOpt(store, output.first)) std::make_pair(output.second, output.second.path(store, name, output.first))
) )
); );
return outsAndOptPaths; return outsAndOptPaths;
@ -594,22 +602,25 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv,
void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv) void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv)
{ {
out << drv.outputs.size(); out << drv.outputs.size();
for (auto & i : drv.outputsAndPaths(store)) { for (auto & i : drv.outputs) {
out << i.first out << i.first;
<< store.printStorePath(i.second.second);
std::visit(overloaded { std::visit(overloaded {
[&](DerivationOutputInputAddressed doi) { [&](DerivationOutputInputAddressed doi) {
out << "" << ""; out << store.printStorePath(doi.path)
<< ""
<< "";
}, },
[&](DerivationOutputCAFixed dof) { [&](DerivationOutputCAFixed dof) {
out << dof.hash.printMethodAlgo() out << store.printStorePath(dof.path(store, drv.name, i.first))
<< dof.hash.printMethodAlgo()
<< dof.hash.hash.to_string(Base16, false); << dof.hash.hash.to_string(Base16, false);
}, },
[&](DerivationOutputCAFloating dof) { [&](DerivationOutputCAFloating dof) {
out << (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)) out << ""
<< (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType))
<< ""; << "";
}, },
}, i.second.first.output); }, i.second.output);
} }
writeStorePaths(store, out, drv.inputSrcs); writeStorePaths(store, out, drv.inputSrcs);
out << drv.platform << drv.builder << drv.args; out << drv.platform << drv.builder << drv.args;
@ -625,5 +636,12 @@ std::string hashPlaceholder(const std::string & outputName)
return "/" + hashString(htSHA256, "nix-output:" + outputName).to_string(Base32, false); return "/" + hashString(htSHA256, "nix-output:" + outputName).to_string(Base32, false);
} }
std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName)
{
auto drvNameWithExtension = drvPath.name();
auto drvName = drvNameWithExtension.substr(0, drvNameWithExtension.size() - 4);
auto clearText = "nix-upstream-output:" + std::string { drvPath.hashPart() } + ":" + outputPathName(drvName, outputName);
return "/" + hashString(htSHA256, clearText).to_string(Base32, false);
}
} }

42
src/libstore/derivations.hh
View file

@ -27,6 +27,7 @@ struct DerivationOutputInputAddressed
struct DerivationOutputCAFixed struct DerivationOutputCAFixed
{ {
FixedOutputHash hash; /* hash used for expected hash computation */ FixedOutputHash hash; /* hash used for expected hash computation */
StorePath path(const Store & store, std::string_view drvName, std::string_view outputName) const;
}; };
/* Floating-output derivations, whose output paths are content addressed, but /* Floating-output derivations, whose output paths are content addressed, but
@ -49,14 +50,8 @@ struct DerivationOutput
std::optional<HashType> hashAlgoOpt(const Store & store) const; std::optional<HashType> hashAlgoOpt(const Store & store) const;
/* Note, when you use this function you should make sure that you're passing /* Note, when you use this function you should make sure that you're passing
the right derivation name. When in doubt, you should use the safer the right derivation name. When in doubt, you should use the safer
interface provided by BasicDerivation::outputsAndPaths */ interface provided by BasicDerivation::outputsAndOptPaths */
std::optional<StorePath> pathOpt(const Store & store, std::string_view drvName) const; std::optional<StorePath> path(const Store & store, std::string_view drvName, std::string_view outputName) const;
/* DEPRECATED: Remove after CA drvs are fully implemented */
StorePath path(const Store & store, std::string_view drvName) const {
auto p = pathOpt(store, drvName);
if (!p) throw UnimplementedError("floating content-addressed derivations are not yet implemented");
return *p;
}
}; };
typedef std::map<string, DerivationOutput> DerivationOutputs; typedef std::map<string, DerivationOutput> DerivationOutputs;
@ -113,17 +108,12 @@ struct BasicDerivation
/* Return true iff this is a fixed-output derivation. */ /* Return true iff this is a fixed-output derivation. */
DerivationType type() const; DerivationType type() const;
/* Return the output paths of a derivation. */
StorePathSet outputPaths(const Store & store) const;
/* Return the output names of a derivation. */ /* Return the output names of a derivation. */
StringSet outputNames() const; StringSet outputNames() const;
/* Calculates the maps that contains all the DerivationOutputs, but /* Calculates the maps that contains all the DerivationOutputs, but
augmented with knowledge of the Store paths they would be written into. augmented with knowledge of the Store paths they would be written
The first one of these functions will be removed when the CA work is into. */
completed */
DerivationOutputsAndPaths outputsAndPaths(const Store & store) const;
DerivationOutputsAndOptPaths outputsAndOptPaths(const Store & store) const; DerivationOutputsAndOptPaths outputsAndOptPaths(const Store & store) const;
static std::string_view nameFromPath(const StorePath & storePath); static std::string_view nameFromPath(const StorePath & storePath);
@ -155,6 +145,13 @@ Derivation parseDerivation(const Store & store, std::string && s, std::string_vi
// FIXME: remove // FIXME: remove
bool isDerivation(const string & fileName); bool isDerivation(const string & fileName);
/* Calculate the name that will be used for the store path for this
output.
This is usually <drv-name>-<output-name>, but is just <drv-name> when
the output name is "out". */
std::string outputPathName(std::string_view drvName, std::string_view outputName);
// known CA drv's output hashes, current just for fixed-output derivations // known CA drv's output hashes, current just for fixed-output derivations
// whose output hashes are always known since they are fixed up-front. // whose output hashes are always known since they are fixed up-front.
typedef std::map<std::string, Hash> CaOutputHashes; typedef std::map<std::string, Hash> CaOutputHashes;
@ -202,6 +199,21 @@ struct Sink;
Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name); Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name);
void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv); void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv);
/* This creates an opaque and almost certainly unique string
deterministically from the output name.
It is used as a placeholder to allow derivations to refer to their
own outputs without needing to use the hash of a derivation in
itself, making the hash near-impossible to calculate. */
std::string hashPlaceholder(const std::string & outputName); std::string hashPlaceholder(const std::string & outputName);
/* This creates an opaque and almost certainly unique string
deterministically from a derivation path and output name.
It is used as a placeholder to allow derivations to refer to
content-addressed paths whose content --- and thus the path
themselves --- isn't yet known. This occurs when a derivation has a
dependency which is a CA derivation. */
std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName);
} }

32
src/libstore/dummy-store.cc
View file

@ -2,17 +2,27 @@
namespace nix { namespace nix {
static std::string uriScheme = "dummy://"; struct DummyStoreConfig : virtual StoreConfig {
using StoreConfig::StoreConfig;
struct DummyStore : public Store const std::string name() override { return "Dummy Store"; }
};
struct DummyStore : public Store, public virtual DummyStoreConfig
{ {
DummyStore(const Params & params) DummyStore(const std::string scheme, const std::string uri, const Params & params)
: Store(params) : DummyStore(params)
{ } { }
DummyStore(const Params & params)
: StoreConfig(params)
, Store(params)
{
}
string getUri() override string getUri() override
{ {
return uriScheme; return *uriSchemes().begin();
} }
void queryPathInfoUncached(const StorePath & path, void queryPathInfoUncached(const StorePath & path,
@ -21,6 +31,10 @@ struct DummyStore : public Store
callback(nullptr); callback(nullptr);
} }
static std::set<std::string> uriSchemes() {
return {"dummy"};
}
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
{ unsupported("queryPathFromHashPart"); } { unsupported("queryPathFromHashPart"); }
@ -48,12 +62,6 @@ struct DummyStore : public Store
{ unsupported("buildDerivation"); } { unsupported("buildDerivation"); }
}; };
static RegisterStoreImplementation regStore([]( static RegisterStoreImplementation<DummyStore, DummyStoreConfig> regStore;
const std::string & uri, const Store::Params & params)
-> std::shared_ptr<Store>
{
if (uri != uriScheme) return nullptr;
return std::make_shared<DummyStore>(params);
});
} }

5
src/libstore/globals.cc
View file

@ -162,11 +162,6 @@ template<> std::string BaseSetting<SandboxMode>::to_string() const
else abort(); else abort();
} }
template<> nlohmann::json BaseSetting<SandboxMode>::toJSON()
{
return AbstractSetting::toJSON();
}
template<> void BaseSetting<SandboxMode>::convertToArg(Args & args, const std::string & category) template<> void BaseSetting<SandboxMode>::convertToArg(Args & args, const std::string & category)
{ {
args.addFlag({ args.addFlag({

1
src/libstore/globals.hh
View file

@ -2,6 +2,7 @@
#include "types.hh" #include "types.hh"
#include "config.hh" #include "config.hh"
#include "abstractsettingtojson.hh"
#include "util.hh" #include "util.hh"
#include <map> #include <map>

39
src/libstore/http-binary-cache-store.cc
View file

@ -7,7 +7,14 @@ namespace nix {
MakeError(UploadToHTTP, Error); MakeError(UploadToHTTP, Error);
class HttpBinaryCacheStore : public BinaryCacheStore struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
{
using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
const std::string name() override { return "Http Binary Cache Store"; }
};
class HttpBinaryCacheStore : public BinaryCacheStore, public HttpBinaryCacheStoreConfig
{ {
private: private:
@ -24,9 +31,12 @@ private:
public: public:
HttpBinaryCacheStore( HttpBinaryCacheStore(
const Params & params, const Path & _cacheUri) const std::string & scheme,
: BinaryCacheStore(params) const Path & _cacheUri,
, cacheUri(_cacheUri) const Params & params)
: StoreConfig(params)
, BinaryCacheStore(params)
, cacheUri(scheme + "://" + _cacheUri)
{ {
if (cacheUri.back() == '/') if (cacheUri.back() == '/')
cacheUri.pop_back(); cacheUri.pop_back();
@ -55,6 +65,13 @@ public:
} }
} }
static std::set<std::string> uriSchemes()
{
static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1";
auto ret = std::set<std::string>({"http", "https"});
if (forceHttp) ret.insert("file");
return ret;
}
protected: protected:
void maybeDisable() void maybeDisable()
@ -162,18 +179,6 @@ protected:
}; };
static RegisterStoreImplementation regStore([]( static RegisterStoreImplementation<HttpBinaryCacheStore, HttpBinaryCacheStoreConfig> regStore;
const std::string & uri, const Store::Params & params)
-> std::shared_ptr<Store>
{
static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1";
if (std::string(uri, 0, 7) != "http://" &&
std::string(uri, 0, 8) != "https://" &&
(!forceHttp || std::string(uri, 0, 7) != "file://"))
return 0;
auto store = std::make_shared<HttpBinaryCacheStore>(params, uri);
store->init();
return store;
});
} }

41
src/libstore/legacy-ssh-store.cc
View file

@ -9,18 +9,24 @@
namespace nix { namespace nix {
static std::string uriScheme = "ssh://"; struct LegacySSHStoreConfig : virtual StoreConfig
struct LegacySSHStore : public Store
{ {
const Setting<int> maxConnections{this, 1, "max-connections", "maximum number of concurrent SSH connections"}; using StoreConfig::StoreConfig;
const Setting<Path> sshKey{this, "", "ssh-key", "path to an SSH private key"}; const Setting<int> maxConnections{(StoreConfig*) this, 1, "max-connections", "maximum number of concurrent SSH connections"};
const Setting<bool> compress{this, false, "compress", "whether to compress the connection"}; const Setting<Path> sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"};
const Setting<Path> remoteProgram{this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"}; const Setting<bool> compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"};
const Setting<std::string> remoteStore{this, "", "remote-store", "URI of the store on the remote system"}; const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"};
const Setting<std::string> remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"};
const std::string name() override { return "Legacy SSH Store"; }
};
struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig
{
// Hack for getting remote build log output. // Hack for getting remote build log output.
const Setting<int> logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"}; // Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in
// the documentation
const Setting<int> logFD{(StoreConfig*) this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"};
struct Connection struct Connection
{ {
@ -37,8 +43,11 @@ struct LegacySSHStore : public Store
SSHMaster master; SSHMaster master;
LegacySSHStore(const string & host, const Params & params) static std::set<std::string> uriSchemes() { return {"ssh"}; }
: Store(params)
LegacySSHStore(const string & scheme, const string & host, const Params & params)
: StoreConfig(params)
, Store(params)
, host(host) , host(host)
, connections(make_ref<Pool<Connection>>( , connections(make_ref<Pool<Connection>>(
std::max(1, (int) maxConnections), std::max(1, (int) maxConnections),
@ -84,7 +93,7 @@ struct LegacySSHStore : public Store
string getUri() override string getUri() override
{ {
return uriScheme + host; return *uriSchemes().begin() + "://" + host;
} }
void queryPathInfoUncached(const StorePath & path, void queryPathInfoUncached(const StorePath & path,
@ -325,12 +334,6 @@ public:
} }
}; };
static RegisterStoreImplementation regStore([]( static RegisterStoreImplementation<LegacySSHStore, LegacySSHStoreConfig> regStore;
const std::string & uri, const Store::Params & params)
-> std::shared_ptr<Store>
{
if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0;
return std::make_shared<LegacySSHStore>(std::string(uri, uriScheme.size()), params);
});
} }

36
src/libstore/local-binary-cache-store.cc
View file

@ -4,7 +4,14 @@
namespace nix { namespace nix {
class LocalBinaryCacheStore : public BinaryCacheStore struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
{
using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
const std::string name() override { return "Local Binary Cache Store"; }
};
class LocalBinaryCacheStore : public BinaryCacheStore, public virtual LocalBinaryCacheStoreConfig
{ {
private: private:
@ -13,8 +20,11 @@ private:
public: public:
LocalBinaryCacheStore( LocalBinaryCacheStore(
const Params & params, const Path & binaryCacheDir) const std::string scheme,
: BinaryCacheStore(params) const Path & binaryCacheDir,
const Params & params)
: StoreConfig(params)
, BinaryCacheStore(params)
, binaryCacheDir(binaryCacheDir) , binaryCacheDir(binaryCacheDir)
{ {
} }
@ -26,6 +36,8 @@ public:
return "file://" + binaryCacheDir; return "file://" + binaryCacheDir;
} }
static std::set<std::string> uriSchemes();
protected: protected:
bool fileExists(const std::string & path) override; bool fileExists(const std::string & path) override;
@ -85,16 +97,14 @@ bool LocalBinaryCacheStore::fileExists(const std::string & path)
return pathExists(binaryCacheDir + "/" + path); return pathExists(binaryCacheDir + "/" + path);
} }
static RegisterStoreImplementation regStore([]( std::set<std::string> LocalBinaryCacheStore::uriSchemes()
const std::string & uri, const Store::Params & params)
-> std::shared_ptr<Store>
{ {
if (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") == "1" || if (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") == "1")
std::string(uri, 0, 7) != "file://") return {};
return 0; else
auto store = std::make_shared<LocalBinaryCacheStore>(params, std::string(uri, 7)); return {"file"};
store->init(); }
return store;
}); static RegisterStoreImplementation<LocalBinaryCacheStore, LocalBinaryCacheStoreConfig> regStore;
} }

57
src/libstore/local-store.cc
View file

@ -42,7 +42,8 @@ namespace nix {
LocalStore::LocalStore(const Params & params) LocalStore::LocalStore(const Params & params)
: Store(params) : StoreConfig(params)
, Store(params)
, LocalFSStore(params) , LocalFSStore(params)
, realStoreDir_{this, false, rootDir != "" ? rootDir + "/nix/store" : storeDir, "real", , realStoreDir_{this, false, rootDir != "" ? rootDir + "/nix/store" : storeDir, "real",
"physical path to the Nix store"} "physical path to the Nix store"}
@ -578,13 +579,32 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
envHasRightPath(path, i.first); envHasRightPath(path, i.first);
}, },
[&](DerivationOutputCAFloating _) { [&](DerivationOutputCAFloating _) {
throw UnimplementedError("floating CA output derivations are not yet implemented"); /* Nothing to check */
}, },
}, i.second.output); }, i.second.output);
} }
} }
void LocalStore::linkDeriverToPath(const StorePath & deriver, const string & outputName, const StorePath & output)
{
auto state(_state.lock());
return linkDeriverToPath(*state, queryValidPathId(*state, deriver), outputName, output);
}
void LocalStore::linkDeriverToPath(State & state, uint64_t deriver, const string & outputName, const StorePath & output)
{
retrySQLite<void>([&]() {
state.stmtAddDerivationOutput.use()
(deriver)
(outputName)
(printStorePath(output))
.exec();
});
}
uint64_t LocalStore::addValidPath(State & state, uint64_t LocalStore::addValidPath(State & state,
const ValidPathInfo & info, bool checkOutputs) const ValidPathInfo & info, bool checkOutputs)
{ {
@ -618,12 +638,11 @@ uint64_t LocalStore::addValidPath(State & state,
registration above is undone. */ registration above is undone. */
if (checkOutputs) checkDerivationOutputs(info.path, drv); if (checkOutputs) checkDerivationOutputs(info.path, drv);
for (auto & i : drv.outputsAndPaths(*this)) { for (auto & i : drv.outputsAndOptPaths(*this)) {
state.stmtAddDerivationOutput.use() /* Floating CA derivations have indeterminate output paths until
(id) they are built, so don't register anything in that case */
(i.first) if (i.second.second)
(printStorePath(i.second.second)) linkDeriverToPath(state, id, i.first, *i.second.second);
.exec();
} }
} }
@ -1533,27 +1552,5 @@ void LocalStore::createUser(const std::string & userName, uid_t userId)
} }
} }
static bool isNonUriPath(const std::string & spec) {
return
// is not a URL
spec.find("://") == std::string::npos
// Has at least one path separator, and so isn't a single word that
// might be special like "auto"
&& spec.find("/") != std::string::npos;
}
static RegisterStoreImplementation regStore([](
const std::string & uri, const Store::Params & params)
-> std::shared_ptr<Store>
{
Store::Params params2 = params;
if (uri == "local") {
} else if (isNonUriPath(uri)) {
params2["root"] = absPath(uri);
} else {
return nullptr;
}
return std::shared_ptr<Store>(std::make_shared<LocalStore>(params2));
});
} }

22
src/libstore/local-store.hh
View file

@ -30,8 +30,19 @@ struct OptimiseStats
uint64_t blocksFreed = 0; uint64_t blocksFreed = 0;
}; };
struct LocalStoreConfig : virtual LocalFSStoreConfig
{
using LocalFSStoreConfig::LocalFSStoreConfig;
class LocalStore : public LocalFSStore Setting<bool> requireSigs{(StoreConfig*) this,
settings.requireSigs,
"require-sigs", "whether store paths should have a trusted signature on import"};
const std::string name() override { return "Local Store"; }
};
class LocalStore : public LocalFSStore, public virtual LocalStoreConfig
{ {
private: private:
@ -95,10 +106,6 @@ public:
private: private:
Setting<bool> requireSigs{(Store*) this,
settings.requireSigs,
"require-sigs", "whether store paths should have a trusted signature on import"};
const PublicKeys & getPublicKeys(); const PublicKeys & getPublicKeys();
public: public:
@ -279,6 +286,11 @@ private:
specified by the ‘secret-key-files’ option. */ specified by the ‘secret-key-files’ option. */
void signPathInfo(ValidPathInfo & info); void signPathInfo(ValidPathInfo & info);
/* Register the store path 'output' as the output named 'outputName' of
derivation 'deriver'. */
void linkDeriverToPath(const StorePath & deriver, const string & outputName, const StorePath & output);
void linkDeriverToPath(State & state, uint64_t deriver, const string & outputName, const StorePath & output);
Path getRealStoreDir() override { return realStoreDir; } Path getRealStoreDir() override { return realStoreDir; }
void createUser(const std::string & userName, uid_t userId) override; void createUser(const std::string & userName, uid_t userId) override;

23
src/libstore/misc.cc
View file

@ -203,17 +203,24 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
return; return;
} }
PathSet invalid;
/* true for regular derivations, and CA derivations for which we
have a trust mapping for all wanted outputs. */
auto knownOutputPaths = true;
for (auto & [outputName, pathOpt] : queryPartialDerivationOutputMap(path.path)) {
if (!pathOpt) {
knownOutputPaths = false;
break;
}
if (wantOutput(outputName, path.outputs) && !isValidPath(*pathOpt))
invalid.insert(printStorePath(*pathOpt));
}
if (knownOutputPaths && invalid.empty()) return;
auto drv = make_ref<Derivation>(derivationFromPath(path.path)); auto drv = make_ref<Derivation>(derivationFromPath(path.path));
ParsedDerivation parsedDrv(StorePath(path.path), *drv); ParsedDerivation parsedDrv(StorePath(path.path), *drv);
PathSet invalid; if (knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
for (auto & j : drv->outputsAndPaths(*this))
if (wantOutput(j.first, path.outputs)
&& !isValidPath(j.second.second))
invalid.insert(printStorePath(j.second.second));
if (invalid.empty()) return;
if (settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size())); auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size()));
for (auto & output : invalid) for (auto & output : invalid)
pool.enqueue(std::bind(checkOutput, printStorePath(path.path), drv, output, drvState)); pool.enqueue(std::bind(checkOutput, printStorePath(path.path), drv, output, drvState));

16
src/libstore/references.cc
View file

@ -79,9 +79,17 @@ void RefScanSink::operator () (const unsigned char * data, size_t len)
std::pair<PathSet, HashResult> scanForReferences(const string & path, std::pair<PathSet, HashResult> scanForReferences(const string & path,
const PathSet & refs) const PathSet & refs)
{ {
RefScanSink refsSink;
HashSink hashSink { htSHA256 }; HashSink hashSink { htSHA256 };
TeeSink sink { refsSink, hashSink }; auto found = scanForReferences(hashSink, path, refs);
auto hash = hashSink.finish();
return std::pair<PathSet, HashResult>(found, hash);
}
PathSet scanForReferences(Sink & toTee,
const string & path, const PathSet & refs)
{
RefScanSink refsSink;
TeeSink sink { refsSink, toTee };
std::map<string, Path> backMap; std::map<string, Path> backMap;
/* For efficiency (and a higher hit rate), just search for the /* For efficiency (and a higher hit rate), just search for the
@ -111,9 +119,7 @@ std::pair<PathSet, HashResult> scanForReferences(const string & path,
found.insert(j->second); found.insert(j->second);
} }
auto hash = hashSink.finish(); return found;
return std::pair<PathSet, HashResult>(found, hash);
} }

2
src/libstore/references.hh
View file

@ -7,6 +7,8 @@ namespace nix {
std::pair<PathSet, HashResult> scanForReferences(const Path & path, const PathSet & refs); std::pair<PathSet, HashResult> scanForReferences(const Path & path, const PathSet & refs);
PathSet scanForReferences(Sink & toTee, const Path & path, const PathSet & refs);
struct RewritingSink : Sink struct RewritingSink : Sink
{ {
std::string from, to, prev; std::string from, to, prev;

29
src/libstore/remote-store.cc
View file

@ -94,6 +94,7 @@ void write(const Store & store, Sink & out, const std::optional<StorePath> & sto
/* TODO: Separate these store impls into different files, give them better names */ /* TODO: Separate these store impls into different files, give them better names */
RemoteStore::RemoteStore(const Params & params) RemoteStore::RemoteStore(const Params & params)
: Store(params) : Store(params)
, RemoteStoreConfig(params)
, connections(make_ref<Pool<Connection>>( , connections(make_ref<Pool<Connection>>(
std::max(1, (int) maxConnections), std::max(1, (int) maxConnections),
[this]() { [this]() {
@ -132,19 +133,21 @@ ref<RemoteStore::Connection> RemoteStore::openConnectionWrapper()
UDSRemoteStore::UDSRemoteStore(const Params & params) UDSRemoteStore::UDSRemoteStore(const Params & params)
: Store(params) : StoreConfig(params)
, Store(params)
, LocalFSStore(params) , LocalFSStore(params)
, RemoteStore(params) , RemoteStore(params)
{ {
} }
UDSRemoteStore::UDSRemoteStore(std::string socket_path, const Params & params) UDSRemoteStore::UDSRemoteStore(
: Store(params) const std::string scheme,
, LocalFSStore(params) std::string socket_path,
, RemoteStore(params) const Params & params)
, path(socket_path) : UDSRemoteStore(params)
{ {
path.emplace(socket_path);
} }
@ -989,18 +992,6 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source *
return nullptr; return nullptr;
} }
static std::string_view uriScheme = "unix://"; static RegisterStoreImplementation<UDSRemoteStore, UDSRemoteStoreConfig> regStore;
static RegisterStoreImplementation regStore([](
const std::string & uri, const Store::Params & params)
-> std::shared_ptr<Store>
{
if (hasPrefix(uri, uriScheme))
return std::make_shared<UDSRemoteStore>(std::string(uri, uriScheme.size()), params);
else if (uri == "daemon")
return std::make_shared<UDSRemoteStore>(params);
else
return nullptr;
});
} }

42
src/libstore/remote-store.hh
View file

@ -16,19 +16,23 @@ struct FdSource;
template<typename T> class Pool; template<typename T> class Pool;
struct ConnectionHandle; struct ConnectionHandle;
struct RemoteStoreConfig : virtual StoreConfig
{
using StoreConfig::StoreConfig;
const Setting<int> maxConnections{(StoreConfig*) this, 1,
"max-connections", "maximum number of concurrent connections to the Nix daemon"};
const Setting<unsigned int> maxConnectionAge{(StoreConfig*) this, std::numeric_limits<unsigned int>::max(),
"max-connection-age", "number of seconds to reuse a connection"};
};
/* FIXME: RemoteStore is a misnomer - should be something like /* FIXME: RemoteStore is a misnomer - should be something like
DaemonStore. */ DaemonStore. */
class RemoteStore : public virtual Store class RemoteStore : public virtual Store, public virtual RemoteStoreConfig
{ {
public: public:
const Setting<int> maxConnections{(Store*) this, 1,
"max-connections", "maximum number of concurrent connections to the Nix daemon"};
const Setting<unsigned int> maxConnectionAge{(Store*) this, std::numeric_limits<unsigned int>::max(),
"max-connection-age", "number of seconds to reuse a connection"};
virtual bool sameMachine() = 0; virtual bool sameMachine() = 0;
RemoteStore(const Params & params); RemoteStore(const Params & params);
@ -141,15 +145,35 @@ private:
}; };
class UDSRemoteStore : public LocalFSStore, public RemoteStore struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreConfig
{
UDSRemoteStoreConfig(const Store::Params & params)
: StoreConfig(params)
, LocalFSStoreConfig(params)
, RemoteStoreConfig(params)
{
}
UDSRemoteStoreConfig()
: UDSRemoteStoreConfig(Store::Params({}))
{
}
const std::string name() override { return "Local Daemon Store"; }
};
class UDSRemoteStore : public LocalFSStore, public RemoteStore, public virtual UDSRemoteStoreConfig
{ {
public: public:
UDSRemoteStore(const Params & params); UDSRemoteStore(const Params & params);
UDSRemoteStore(std::string path, const Params & params); UDSRemoteStore(const std::string scheme, std::string path, const Params & params);
std::string getUri() override; std::string getUri() override;
static std::set<std::string> uriSchemes()
{ return {"unix"}; }
bool sameMachine() override bool sameMachine() override
{ return true; } { return true; }

45
src/libstore/s3-binary-cache-store.cc
View file

@ -172,20 +172,26 @@ S3Helper::FileTransferResult S3Helper::getObject(
return res; return res;
} }
struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
{ {
const Setting<std::string> profile{this, "", "profile", "The name of the AWS configuration profile to use."}; using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
const Setting<std::string> region{this, Aws::Region::US_EAST_1, "region", {"aws-region"}}; const Setting<std::string> profile{(StoreConfig*) this, "", "profile", "The name of the AWS configuration profile to use."};
const Setting<std::string> scheme{this, "", "scheme", "The scheme to use for S3 requests, https by default."}; const Setting<std::string> region{(StoreConfig*) this, Aws::Region::US_EAST_1, "region", {"aws-region"}};
const Setting<std::string> endpoint{this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."}; const Setting<std::string> scheme{(StoreConfig*) this, "", "scheme", "The scheme to use for S3 requests, https by default."};
const Setting<std::string> narinfoCompression{this, "", "narinfo-compression", "compression method for .narinfo files"}; const Setting<std::string> endpoint{(StoreConfig*) this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."};
const Setting<std::string> lsCompression{this, "", "ls-compression", "compression method for .ls files"}; const Setting<std::string> narinfoCompression{(StoreConfig*) this, "", "narinfo-compression", "compression method for .narinfo files"};
const Setting<std::string> logCompression{this, "", "log-compression", "compression method for log/* files"}; const Setting<std::string> lsCompression{(StoreConfig*) this, "", "ls-compression", "compression method for .ls files"};
const Setting<std::string> logCompression{(StoreConfig*) this, "", "log-compression", "compression method for log/* files"};
const Setting<bool> multipartUpload{ const Setting<bool> multipartUpload{
this, false, "multipart-upload", "whether to use multi-part uploads"}; (StoreConfig*) this, false, "multipart-upload", "whether to use multi-part uploads"};
const Setting<uint64_t> bufferSize{ const Setting<uint64_t> bufferSize{
this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"}; (StoreConfig*) this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"};
const std::string name() override { return "S3 Binary Cache Store"; }
};
struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore, virtual S3BinaryCacheStoreConfig
{
std::string bucketName; std::string bucketName;
Stats stats; Stats stats;
@ -193,8 +199,11 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
S3Helper s3Helper; S3Helper s3Helper;
S3BinaryCacheStoreImpl( S3BinaryCacheStoreImpl(
const Params & params, const std::string & bucketName) const std::string & scheme,
: S3BinaryCacheStore(params) const std::string & bucketName,
const Params & params)
: StoreConfig(params)
, S3BinaryCacheStore(params)
, bucketName(bucketName) , bucketName(bucketName)
, s3Helper(profile, region, scheme, endpoint) , s3Helper(profile, region, scheme, endpoint)
{ {
@ -426,17 +435,11 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
return paths; return paths;
} }
static std::set<std::string> uriSchemes() { return {"s3"}; }
}; };
static RegisterStoreImplementation regStore([]( static RegisterStoreImplementation<S3BinaryCacheStoreImpl, S3BinaryCacheStoreConfig> regStore;
const std::string & uri, const Store::Params & params)
-> std::shared_ptr<Store>
{
if (std::string(uri, 0, 5) != "s3://") return 0;
auto store = std::make_shared<S3BinaryCacheStoreImpl>(params, std::string(uri, 5));
store->init();
return store;
});
} }

36
src/libstore/ssh-store.cc
View file

@ -8,19 +8,25 @@
namespace nix { namespace nix {
static std::string uriScheme = "ssh-ng://"; struct SSHStoreConfig : virtual RemoteStoreConfig
{
using RemoteStoreConfig::RemoteStoreConfig;
class SSHStore : public RemoteStore const Setting<Path> sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"};
const Setting<bool> compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"};
const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-daemon", "remote-program", "path to the nix-daemon executable on the remote system"};
const Setting<std::string> remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"};
const std::string name() override { return "SSH Store"; }
};
class SSHStore : public virtual RemoteStore, public virtual SSHStoreConfig
{ {
public: public:
const Setting<Path> sshKey{(Store*) this, "", "ssh-key", "path to an SSH private key"}; SSHStore(const std::string & scheme, const std::string & host, const Params & params)
const Setting<bool> compress{(Store*) this, false, "compress", "whether to compress the connection"}; : StoreConfig(params)
const Setting<Path> remoteProgram{(Store*) this, "nix-daemon", "remote-program", "path to the nix-daemon executable on the remote system"}; , Store(params)
const Setting<std::string> remoteStore{(Store*) this, "", "remote-store", "URI of the store on the remote system"};
SSHStore(const std::string & host, const Params & params)
: Store(params)
, RemoteStore(params) , RemoteStore(params)
, host(host) , host(host)
, master( , master(
@ -32,9 +38,11 @@ public:
{ {
} }
static std::set<std::string> uriSchemes() { return {"ssh-ng"}; }
std::string getUri() override std::string getUri() override
{ {
return uriScheme + host; return *uriSchemes().begin() + "://" + host;
} }
bool sameMachine() override bool sameMachine() override
@ -75,12 +83,6 @@ ref<RemoteStore::Connection> SSHStore::openConnection()
return conn; return conn;
} }
static RegisterStoreImplementation regStore([]( static RegisterStoreImplementation<SSHStore, SSHStoreConfig> regStore;
const std::string & uri, const Store::Params & params)
-> std::shared_ptr<Store>
{
if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0;
return std::make_shared<SSHStore>(std::string(uri, uriScheme.size()), params);
});
} }

109
src/libstore/store-api.cc
View file

@ -140,21 +140,28 @@ StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view p
*/ */
StorePath Store::makeStorePath(const string & type, StorePath Store::makeStorePath(std::string_view type,
const Hash & hash, std::string_view name) const std::string_view hash, std::string_view name) const
{ {
/* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */ /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */
string s = type + ":" + hash.to_string(Base16, true) + ":" + storeDir + ":" + std::string(name); string s = std::string { type } + ":" + std::string { hash }
+ ":" + storeDir + ":" + std::string { name };
auto h = compressHash(hashString(htSHA256, s), 20); auto h = compressHash(hashString(htSHA256, s), 20);
return StorePath(h, name); return StorePath(h, name);
} }
StorePath Store::makeOutputPath(const string & id, StorePath Store::makeStorePath(std::string_view type,
const Hash & hash, std::string_view name) const const Hash & hash, std::string_view name) const
{ {
return makeStorePath("output:" + id, hash, return makeStorePath(type, hash.to_string(Base16, true), name);
std::string(name) + (id == "out" ? "" : "-" + id)); }
StorePath Store::makeOutputPath(std::string_view id,
const Hash & hash, std::string_view name) const
{
return makeStorePath("output:" + std::string { id }, hash, outputPathName(name, id));
} }
@ -339,7 +346,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
Store::Store(const Params & params) Store::Store(const Params & params)
: Config(params) : StoreConfig(params)
, state({(size_t) pathInfoCacheSize}) , state({(size_t) pathInfoCacheSize})
{ {
} }
@ -1002,7 +1009,6 @@ Derivation Store::readDerivation(const StorePath & drvPath)
} }
} }
} }
@ -1012,9 +1018,6 @@ Derivation Store::readDerivation(const StorePath & drvPath)
namespace nix { namespace nix {
RegisterStoreImplementation::Implementations * RegisterStoreImplementation::implementations = 0;
/* Split URI into protocol+hierarchy part and its parameter set. */ /* Split URI into protocol+hierarchy part and its parameter set. */
std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_) std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_)
{ {
@ -1028,44 +1031,71 @@ std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_
return {uri, params}; return {uri, params};
} }
ref<Store> openStore(const std::string & uri_, static bool isNonUriPath(const std::string & spec) {
const Store::Params & extraParams) return
{ // is not a URL
auto [uri, uriParams] = splitUriAndParams(uri_); spec.find("://") == std::string::npos
auto params = extraParams; // Has at least one path separator, and so isn't a single word that
params.insert(uriParams.begin(), uriParams.end()); // might be special like "auto"
&& spec.find("/") != std::string::npos;
for (auto fun : *RegisterStoreImplementation::implementations) {
auto store = fun(uri, params);
if (store) {
store->warnUnknownSettings();
return ref<Store>(store);
}
}
throw Error("don't know how to open Nix store '%s'", uri);
} }
std::shared_ptr<Store> openFromNonUri(const std::string & uri, const Store::Params & params)
// Specific prefixes are handled by the specific types of store, while here we
// handle the general cases not covered by the other ones.
static RegisterStoreImplementation regStore([](
const std::string & uri, const Store::Params & params)
-> std::shared_ptr<Store>
{ {
auto stateDir = get(params, "state").value_or(settings.nixStateDir); if (uri == "" || uri == "auto") {
if (uri == "" || uri == "auto") { auto stateDir = get(params, "state").value_or(settings.nixStateDir);
if (access(stateDir.c_str(), R_OK | W_OK) == 0) if (access(stateDir.c_str(), R_OK | W_OK) == 0)
return std::make_shared<LocalStore>(params); return std::make_shared<LocalStore>(params);
else if (pathExists(settings.nixDaemonSocketFile)) else if (pathExists(settings.nixDaemonSocketFile))
return std::make_shared<UDSRemoteStore>(params); return std::make_shared<UDSRemoteStore>(params);
else else
return std::make_shared<LocalStore>(params); return std::make_shared<LocalStore>(params);
} else { } else if (uri == "daemon") {
return nullptr; return std::make_shared<UDSRemoteStore>(params);
} } else if (uri == "local") {
}); return std::make_shared<LocalStore>(params);
} else if (isNonUriPath(uri)) {
Store::Params params2 = params;
params2["root"] = absPath(uri);
return std::make_shared<LocalStore>(params2);
} else {
return nullptr;
}
}
ref<Store> openStore(const std::string & uri_,
const Store::Params & extraParams)
{
auto params = extraParams;
try {
auto parsedUri = parseURL(uri_);
params.insert(parsedUri.query.begin(), parsedUri.query.end());
auto baseURI = parsedUri.authority.value_or("") + parsedUri.path;
for (auto implem : *Implementations::registered) {
if (implem.uriSchemes.count(parsedUri.scheme)) {
auto store = implem.create(parsedUri.scheme, baseURI, params);
if (store) {
store->init();
store->warnUnknownSettings();
return ref<Store>(store);
}
}
}
}
catch (BadURL &) {
auto [uri, uriParams] = splitUriAndParams(uri_);
params.insert(uriParams.begin(), uriParams.end());
if (auto store = openFromNonUri(uri, params)) {
store->warnUnknownSettings();
return ref<Store>(store);
}
}
throw Error("don't know how to open Nix store '%s'", uri_);
}
std::list<ref<Store>> getDefaultSubstituters() std::list<ref<Store>> getDefaultSubstituters()
{ {
@ -1099,5 +1129,6 @@ std::list<ref<Store>> getDefaultSubstituters()
return stores; return stores;
} }
std::vector<StoreFactory> * Implementations::registered = 0;
} }

133
src/libstore/store-api.hh
View file

@ -24,6 +24,31 @@
namespace nix { namespace nix {
/**
* About the class hierarchy of the store implementations:
*
* Each store type `Foo` consists of two classes:
*
* 1. A class `FooConfig : virtual StoreConfig` that contains the configuration
* for the store
*
* It should only contain members of type `const Setting<T>` (or subclasses
* of it) and inherit the constructors of `StoreConfig`
* (`using StoreConfig::StoreConfig`).
*
* 2. A class `Foo : virtual Store, virtual FooConfig` that contains the
* implementation of the store.
*
* This class is expected to have a constructor `Foo(const Params & params)`
* that calls `StoreConfig(params)` (otherwise you're gonna encounter an
* `assertion failure` when trying to instantiate it).
*
* You can then register the new store using:
*
* ```
* cpp static RegisterStoreImplementation<Foo, FooConfig> regStore;
* ```
*/
MakeError(SubstError, Error); MakeError(SubstError, Error);
MakeError(BuildError, Error); // denotes a permanent build failure MakeError(BuildError, Error); // denotes a permanent build failure
@ -33,6 +58,7 @@ MakeError(SubstituteGone, Error);
MakeError(SubstituterDisabled, Error); MakeError(SubstituterDisabled, Error);
MakeError(BadStorePath, Error); MakeError(BadStorePath, Error);
MakeError(InvalidStoreURI, Error);
class FSAccessor; class FSAccessor;
class NarInfoDiskCache; class NarInfoDiskCache;
@ -144,12 +170,31 @@ struct BuildResult
} }
}; };
struct StoreConfig : public Config
class Store : public std::enable_shared_from_this<Store>, public Config
{ {
public: using Config::Config;
typedef std::map<std::string, std::string> Params; /**
* When constructing a store implementation, we pass in a map `params` of
* parameters that's supposed to initialize the associated config.
* To do that, we must use the `StoreConfig(StringMap & params)`
* constructor, so we'd like to `delete` its default constructor to enforce
* it.
*
* However, actually deleting it means that all the subclasses of
* `StoreConfig` will have their default constructor deleted (because it's
* supposed to call the deleted default constructor of `StoreConfig`). But
* because we're always using virtual inheritance, the constructors of
* child classes will never implicitely call this one, so deleting it will
* be more painful than anything else.
*
* So we `assert(false)` here to ensure at runtime that the right
* constructor is always called without having to redefine a custom
* constructor for each `*Config` class.
*/
StoreConfig() { assert(false); }
virtual const std::string name() = 0;
const PathSetting storeDir_{this, false, settings.nixStore, const PathSetting storeDir_{this, false, settings.nixStore,
"store", "path to the Nix store"}; "store", "path to the Nix store"};
@ -167,6 +212,14 @@ public:
"system-features", "system-features",
"Optional features that the system this store builds on implements (like \"kvm\")."}; "Optional features that the system this store builds on implements (like \"kvm\")."};
};
class Store : public std::enable_shared_from_this<Store>, public virtual StoreConfig
{
public:
typedef std::map<std::string, std::string> Params;
protected: protected:
struct PathInfoCacheValue { struct PathInfoCacheValue {
@ -200,6 +253,11 @@ protected:
Store(const Params & params); Store(const Params & params);
public: public:
/**
* Perform any necessary effectful operation to make the store up and
* running
*/
virtual void init() {};
virtual ~Store() { } virtual ~Store() { }
@ -247,10 +305,12 @@ public:
StorePathWithOutputs followLinksToStorePathWithOutputs(std::string_view path) const; StorePathWithOutputs followLinksToStorePathWithOutputs(std::string_view path) const;
/* Constructs a unique store path name. */ /* Constructs a unique store path name. */
StorePath makeStorePath(const string & type, StorePath makeStorePath(std::string_view type,
std::string_view hash, std::string_view name) const;
StorePath makeStorePath(std::string_view type,
const Hash & hash, std::string_view name) const; const Hash & hash, std::string_view name) const;
StorePath makeOutputPath(const string & id, StorePath makeOutputPath(std::string_view id,
const Hash & hash, std::string_view name) const; const Hash & hash, std::string_view name) const;
StorePath makeFixedOutputPath(FileIngestionMethod method, StorePath makeFixedOutputPath(FileIngestionMethod method,
@ -624,22 +684,25 @@ protected:
}; };
struct LocalFSStoreConfig : virtual StoreConfig
class LocalFSStore : public virtual Store
{ {
public: using StoreConfig::StoreConfig;
// FIXME: the (StoreConfig*) cast works around a bug in gcc that causes
// FIXME: the (Store*) cast works around a bug in gcc that causes
// it to omit the call to the Setting constructor. Clang works fine // it to omit the call to the Setting constructor. Clang works fine
// either way. // either way.
const PathSetting rootDir{(Store*) this, true, "", const PathSetting rootDir{(StoreConfig*) this, true, "",
"root", "directory prefixed to all other paths"}; "root", "directory prefixed to all other paths"};
const PathSetting stateDir{(Store*) this, false, const PathSetting stateDir{(StoreConfig*) this, false,
rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir, rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir,
"state", "directory where Nix will store state"}; "state", "directory where Nix will store state"};
const PathSetting logDir{(Store*) this, false, const PathSetting logDir{(StoreConfig*) this, false,
rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir, rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir,
"log", "directory where Nix will store state"}; "log", "directory where Nix will store state"};
};
class LocalFSStore : public virtual Store, public virtual LocalFSStoreConfig
{
public:
const static string drvsLogDir; const static string drvsLogDir;
@ -732,23 +795,43 @@ ref<Store> openStore(const std::string & uri = settings.storeUri.get(),
‘substituters’ option and various legacy options. */ ‘substituters’ option and various legacy options. */
std::list<ref<Store>> getDefaultSubstituters(); std::list<ref<Store>> getDefaultSubstituters();
struct StoreFactory
/* Store implementation registration. */
typedef std::function<std::shared_ptr<Store>(
const std::string & uri, const Store::Params & params)> OpenStore;
struct RegisterStoreImplementation
{ {
typedef std::vector<OpenStore> Implementations; std::set<std::string> uriSchemes;
static Implementations * implementations; std::function<std::shared_ptr<Store> (const std::string & scheme, const std::string & uri, const Store::Params & params)> create;
std::function<std::shared_ptr<StoreConfig> ()> getConfig;
};
struct Implementations
{
static std::vector<StoreFactory> * registered;
RegisterStoreImplementation(OpenStore fun) template<typename T, typename TConfig>
static void add()
{ {
if (!implementations) implementations = new Implementations; if (!registered) registered = new std::vector<StoreFactory>();
implementations->push_back(fun); StoreFactory factory{
.uriSchemes = T::uriSchemes(),
.create =
([](const std::string & scheme, const std::string & uri, const Store::Params & params)
-> std::shared_ptr<Store>
{ return std::make_shared<T>(scheme, uri, params); }),
.getConfig =
([]()
-> std::shared_ptr<StoreConfig>
{ return std::make_shared<TConfig>(StringMap({})); })
};
registered->push_back(factory);
} }
}; };
template<typename T, typename TConfig>
struct RegisterStoreImplementation
{
RegisterStoreImplementation()
{
Implementations::add<T, TConfig>();
}
};
/* Display a set of paths in human-readable form (i.e., between quotes /* Display a set of paths in human-readable form (i.e., between quotes

15
src/libutil/abstractsettingtojson.hh Normal file
View file

@ -0,0 +1,15 @@
#pragma once
#include <nlohmann/json.hpp>
#include "config.hh"
namespace nix {
template<typename T>
std::map<std::string, nlohmann::json> BaseSetting<T>::toJSONObject()
{
auto obj = AbstractSetting::toJSONObject();
obj.emplace("value", value);
obj.emplace("defaultValue", defaultValue);
return obj;
}
}

33
src/libutil/config.cc
View file

@ -1,5 +1,6 @@
#include "config.hh" #include "config.hh"
#include "args.hh" #include "args.hh"
#include "abstractsettingtojson.hh"
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
@ -137,11 +138,7 @@ nlohmann::json Config::toJSON()
auto res = nlohmann::json::object(); auto res = nlohmann::json::object();
for (auto & s : _settings) for (auto & s : _settings)
if (!s.second.isAlias) { if (!s.second.isAlias) {
auto obj = nlohmann::json::object(); res.emplace(s.first, s.second.setting->toJSON());
obj.emplace("description", s.second.setting->description);
obj.emplace("aliases", s.second.setting->aliases);
obj.emplace("value", s.second.setting->toJSON());
res.emplace(s.first, obj);
} }
return res; return res;
} }
@ -168,19 +165,21 @@ void AbstractSetting::setDefault(const std::string & str)
nlohmann::json AbstractSetting::toJSON() nlohmann::json AbstractSetting::toJSON()
{ {
return to_string(); return nlohmann::json(toJSONObject());
}
std::map<std::string, nlohmann::json> AbstractSetting::toJSONObject()
{
std::map<std::string, nlohmann::json> obj;
obj.emplace("description", description);
obj.emplace("aliases", aliases);
return obj;
} }
void AbstractSetting::convertToArg(Args & args, const std::string & category) void AbstractSetting::convertToArg(Args & args, const std::string & category)
{ {
} }
template<typename T>
nlohmann::json BaseSetting<T>::toJSON()
{
return value;
}
template<typename T> template<typename T>
void BaseSetting<T>::convertToArg(Args & args, const std::string & category) void BaseSetting<T>::convertToArg(Args & args, const std::string & category)
{ {
@ -259,11 +258,6 @@ template<> std::string BaseSetting<Strings>::to_string() const
return concatStringsSep(" ", value); return concatStringsSep(" ", value);
} }
template<> nlohmann::json BaseSetting<Strings>::toJSON()
{
return value;
}
template<> void BaseSetting<StringSet>::set(const std::string & str) template<> void BaseSetting<StringSet>::set(const std::string & str)
{ {
value = tokenizeString<StringSet>(str); value = tokenizeString<StringSet>(str);
@ -274,11 +268,6 @@ template<> std::string BaseSetting<StringSet>::to_string() const
return concatStringsSep(" ", value); return concatStringsSep(" ", value);
} }
template<> nlohmann::json BaseSetting<StringSet>::toJSON()
{
return value;
}
template class BaseSetting<int>; template class BaseSetting<int>;
template class BaseSetting<unsigned int>; template class BaseSetting<unsigned int>;
template class BaseSetting<long>; template class BaseSetting<long>;

8
src/libutil/config.hh
View file

@ -206,7 +206,9 @@ protected:
virtual std::string to_string() const = 0; virtual std::string to_string() const = 0;
virtual nlohmann::json toJSON(); nlohmann::json toJSON();
virtual std::map<std::string, nlohmann::json> toJSONObject();
virtual void convertToArg(Args & args, const std::string & category); virtual void convertToArg(Args & args, const std::string & category);
@ -220,6 +222,7 @@ class BaseSetting : public AbstractSetting
protected: protected:
T value; T value;
const T defaultValue;
public: public:
@ -229,6 +232,7 @@ public:
const std::set<std::string> & aliases = {}) const std::set<std::string> & aliases = {})
: AbstractSetting(name, description, aliases) : AbstractSetting(name, description, aliases)
, value(def) , value(def)
, defaultValue(def)
{ } { }
operator const T &() const { return value; } operator const T &() const { return value; }
@ -251,7 +255,7 @@ public:
void convertToArg(Args & args, const std::string & category) override; void convertToArg(Args & args, const std::string & category) override;
nlohmann::json toJSON() override; std::map<std::string, nlohmann::json> toJSONObject() override;
}; };
template<typename T> template<typename T>

6
src/libutil/serialise.hh
View file

@ -22,6 +22,12 @@ struct Sink
} }
}; };
/* Just throws away data. */
struct NullSink : Sink
{
void operator () (const unsigned char * data, size_t len) override
{ }
};
/* A buffered abstract sink. Warning: a BufferedSink should not be /* A buffered abstract sink. Warning: a BufferedSink should not be
used from multiple threads concurrently. */ used from multiple threads concurrently. */

2
src/libutil/tests/config.cc
View file

@ -161,7 +161,7 @@ namespace nix {
Setting<std::string> setting{&config, "", "name-of-the-setting", "description"}; Setting<std::string> setting{&config, "", "name-of-the-setting", "description"};
setting.assign("value"); setting.assign("value");
ASSERT_EQ(config.toJSON().dump(), R"#({"name-of-the-setting":{"aliases":[],"description":"description\n","value":"value"}})#"); ASSERT_EQ(config.toJSON().dump(), R"#({"name-of-the-setting":{"aliases":[],"defaultValue":"","description":"description\n","value":"value"}})#");
} }
TEST(Config, setSettingAlias) { TEST(Config, setSettingAlias) {

2
src/libutil/url.hh
View file

@ -31,7 +31,7 @@ ParsedURL parseURL(const std::string & url);
// URI stuff. // URI stuff.
const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])"; const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])";
const static std::string schemeRegex = "(?:[a-z+]+)"; const static std::string schemeRegex = "(?:[a-z+.-]+)";
const static std::string ipv6AddressRegex = "(?:\\[[0-9a-fA-F:]+\\])"; const static std::string ipv6AddressRegex = "(?:\\[[0-9a-fA-F:]+\\])";
const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])"; const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])";
const static std::string subdelimsRegex = "(?:[!$&'\"()*+,;=])"; const static std::string subdelimsRegex = "(?:[!$&'\"()*+,;=])";

56
src/nix-build/nix-build.cc
View file

@ -487,50 +487,56 @@ static void _main(int argc, char * * argv)
std::vector<StorePathWithOutputs> pathsToBuild; std::vector<StorePathWithOutputs> pathsToBuild;
std::map<Path, Path> drvPrefixes; std::map<StorePath, std::pair<size_t, StringSet>> drvMap;
std::map<Path, Path> resultSymlinks;
std::vector<Path> outPaths;
for (auto & drvInfo : drvs) { for (auto & drvInfo : drvs) {
auto drvPath = drvInfo.queryDrvPath(); auto drvPath = store->parseStorePath(drvInfo.queryDrvPath());
auto outPath = drvInfo.queryOutPath();
auto outputName = drvInfo.queryOutputName(); auto outputName = drvInfo.queryOutputName();
if (outputName == "") if (outputName == "")
throw Error("derivation '%s' lacks an 'outputName' attribute", drvPath); throw Error("derivation '%s' lacks an 'outputName' attribute", store->printStorePath(drvPath));
pathsToBuild.push_back({store->parseStorePath(drvPath), {outputName}}); pathsToBuild.push_back({drvPath, {outputName}});
std::string drvPrefix; auto i = drvMap.find(drvPath);
auto i = drvPrefixes.find(drvPath); if (i != drvMap.end())
if (i != drvPrefixes.end()) i->second.second.insert(outputName);
drvPrefix = i->second;
else { else {
drvPrefix = outLink; drvMap[drvPath] = {drvMap.size(), {outputName}};
if (drvPrefixes.size())
drvPrefix += fmt("-%d", drvPrefixes.size() + 1);
drvPrefixes[drvPath] = drvPrefix;
} }
std::string symlink = drvPrefix;
if (outputName != "out") symlink += "-" + outputName;
resultSymlinks[symlink] = outPath;
outPaths.push_back(outPath);
} }
buildPaths(pathsToBuild); buildPaths(pathsToBuild);
if (dryRun) return; if (dryRun) return;
for (auto & symlink : resultSymlinks) std::vector<StorePath> outPaths;
if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
store2->addPermRoot(store->parseStorePath(symlink.second), absPath(symlink.first)); for (auto & [drvPath, info] : drvMap) {
auto & [counter, wantedOutputs] = info;
std::string drvPrefix = outLink;
if (counter)
drvPrefix += fmt("-%d", counter + 1);
auto builtOutputs = store->queryDerivationOutputMap(drvPath);
for (auto & outputName : wantedOutputs) {
auto outputPath = builtOutputs.at(outputName);
if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) {
std::string symlink = drvPrefix;
if (outputName != "out") symlink += "-" + outputName;
store2->addPermRoot(outputPath, absPath(symlink));
}
outPaths.push_back(outputPath);
}
}
logger->stop(); logger->stop();
for (auto & path : outPaths) for (auto & path : outPaths)
std::cout << path << '\n'; std::cout << store->printStorePath(path) << '\n';
} }
} }

25
src/nix-store/nix-store.cc
View file

@ -64,6 +64,7 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true)
if (path.path.isDerivation()) { if (path.path.isDerivation()) {
if (build) store->buildPaths({path}); if (build) store->buildPaths({path});
auto outputPaths = store->queryDerivationOutputMap(path.path);
Derivation drv = store->derivationFromPath(path.path); Derivation drv = store->derivationFromPath(path.path);
rootNr++; rootNr++;
@ -76,7 +77,8 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true)
if (i == drv.outputs.end()) if (i == drv.outputs.end())
throw Error("derivation '%s' does not have an output named '%s'", throw Error("derivation '%s' does not have an output named '%s'",
store2->printStorePath(path.path), j); store2->printStorePath(path.path), j);
auto outPath = store2->printStorePath(i->second.path(*store, drv.name)); auto outPath = outputPaths.at(i->first);
auto retPath = store->printStorePath(outPath);
if (store2) { if (store2) {
if (gcRoot == "") if (gcRoot == "")
printGCWarning(); printGCWarning();
@ -84,10 +86,10 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true)
Path rootName = gcRoot; Path rootName = gcRoot;
if (rootNr > 1) rootName += "-" + std::to_string(rootNr); if (rootNr > 1) rootName += "-" + std::to_string(rootNr);
if (i->first != "out") rootName += "-" + i->first; if (i->first != "out") rootName += "-" + i->first;
outPath = store2->addPermRoot(store->parseStorePath(outPath), rootName); retPath = store2->addPermRoot(outPath, rootName);
} }
} }
outputs.insert(outPath); outputs.insert(retPath);
} }
return outputs; return outputs;
} }
@ -217,8 +219,13 @@ static StorePathSet maybeUseOutputs(const StorePath & storePath, bool useOutput,
if (useOutput && storePath.isDerivation()) { if (useOutput && storePath.isDerivation()) {
auto drv = store->derivationFromPath(storePath); auto drv = store->derivationFromPath(storePath);
StorePathSet outputs; StorePathSet outputs;
for (auto & i : drv.outputsAndPaths(*store)) if (forceRealise)
outputs.insert(i.second.second); return store->queryDerivationOutputs(storePath);
for (auto & i : drv.outputsAndOptPaths(*store)) {
if (!i.second.second)
throw UsageError("Cannot use output path of floating content-addressed derivation until we know what it is (e.g. by building it)");
outputs.insert(*i.second.second);
}
return outputs; return outputs;
} }
else return {storePath}; else return {storePath};
@ -308,11 +315,9 @@ static void opQuery(Strings opFlags, Strings opArgs)
case qOutputs: { case qOutputs: {
for (auto & i : opArgs) { for (auto & i : opArgs) {
auto i2 = store->followLinksToStorePath(i); auto outputs = maybeUseOutputs(store->followLinksToStorePath(i), true, forceRealise);
if (forceRealise) realisePath({i2}); for (auto & outputPath : outputs)
Derivation drv = store->derivationFromPath(i2); cout << fmt("%1%\n", store->printStorePath(outputPath));
for (auto & j : drv.outputsAndPaths(*store))
cout << fmt("%1%\n", store->printStorePath(j.second.second));
} }
break; break;
} }

3
src/nix/build.cc
View file

@ -74,7 +74,8 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile
store2->addPermRoot(bo.path, absPath(symlink)); store2->addPermRoot(bo.path, absPath(symlink));
}, },
[&](BuildableFromDrv bfd) { [&](BuildableFromDrv bfd) {
for (auto & output : bfd.outputs) { auto builtOutputs = store->queryDerivationOutputMap(bfd.drvPath);
for (auto & output : builtOutputs) {
std::string symlink = outLink; std::string symlink = outLink;
if (i) symlink += fmt("-%d", i); if (i) symlink += fmt("-%d", i);
if (output.first != "out") symlink += fmt("-%s", output.first); if (output.first != "out") symlink += fmt("-%s", output.first);

5
src/nix/command.cc
View file

@ -150,7 +150,10 @@ void MixProfile::updateProfile(const Buildables & buildables)
}, },
[&](BuildableFromDrv bfd) { [&](BuildableFromDrv bfd) {
for (auto & output : bfd.outputs) { for (auto & output : bfd.outputs) {
result.push_back(output.second); /* Output path should be known because we just tried to
build it. */
assert(!output.second);
result.push_back(*output.second);
} }
}, },
}, buildable); }, buildable);

44
src/nix/describe-stores.cc Normal file
View file

@ -0,0 +1,44 @@
#include "command.hh"
#include "common-args.hh"
#include "shared.hh"
#include "store-api.hh"
#include <nlohmann/json.hpp>
using namespace nix;
struct CmdDescribeStores : Command, MixJSON
{
std::string description() override
{
return "show registered store types and their available options";
}
Category category() override { return catUtility; }
void run() override
{
auto res = nlohmann::json::object();
for (auto & implem : *Implementations::registered) {
auto storeConfig = implem.getConfig();
auto storeName = storeConfig->name();
res[storeName] = storeConfig->toJSON();
}
if (json) {
std::cout << res;
} else {
for (auto & [storeName, storeConfig] : res.items()) {
std::cout << "## " << storeName << std::endl << std::endl;
for (auto & [optionName, optionDesc] : storeConfig.items()) {
std::cout << "### " << optionName << std::endl << std::endl;
std::cout << optionDesc["description"].get<std::string>() << std::endl;
std::cout << "default: " << optionDesc["defaultValue"] << std::endl <<std::endl;
if (!optionDesc["aliases"].empty())
std::cout << "aliases: " << optionDesc["aliases"] << std::endl << std::endl;
}
}
}
}
};
static auto r1 = registerCommand<CmdDescribeStores>("describe-stores");

7
src/nix/develop.cc
View file

@ -145,7 +145,10 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
/* Build the derivation. */ /* Build the derivation. */
store->buildPaths({{shellDrvPath}}); store->buildPaths({{shellDrvPath}});
for (auto & outPath : drv.outputPaths(*store)) { for (auto & [_0, outputAndOptPath] : drv.outputsAndOptPaths(*store)) {
auto & [_1, optPath] = outputAndOptPath;
assert(optPath);
auto & outPath = *optPath;
assert(store->isValidPath(outPath)); assert(store->isValidPath(outPath));
auto outPathS = store->toRealPath(outPath); auto outPathS = store->toRealPath(outPath);
if (lstat(outPathS).st_size) if (lstat(outPathS).st_size)
@ -389,7 +392,7 @@ struct CmdDevelop : Common, MixEnvironment
auto bashInstallable = std::make_shared<InstallableFlake>( auto bashInstallable = std::make_shared<InstallableFlake>(
state, state,
std::move(installable->nixpkgsFlakeRef()), installable->nixpkgsFlakeRef(),
Strings{"bashInteractive"}, Strings{"bashInteractive"},
Strings{"legacyPackages." + settings.thisSystem.get() + "."}, Strings{"legacyPackages." + settings.thisSystem.get() + "."},
lockFlags); lockFlags);

2
src/nix/diff-closures.cc
View file

@ -81,7 +81,7 @@ void printClosureDiff(
auto beforeSize = totalSize(beforeVersions); auto beforeSize = totalSize(beforeVersions);
auto afterSize = totalSize(afterVersions); auto afterSize = totalSize(afterVersions);
auto sizeDelta = (int64_t) afterSize - (int64_t) beforeSize; auto sizeDelta = (int64_t) afterSize - (int64_t) beforeSize;
auto showDelta = abs(sizeDelta) >= 8 * 1024; auto showDelta = std::abs(sizeDelta) >= 8 * 1024;
std::set<std::string> removed, unchanged; std::set<std::string> removed, unchanged;
for (auto & [version, _] : beforeVersions) for (auto & [version, _] : beforeVersions)

15
src/nix/installables.cc
View file

@ -302,10 +302,10 @@ struct InstallableStorePath : Installable
Buildables toBuildables() override Buildables toBuildables() override
{ {
if (storePath.isDerivation()) { if (storePath.isDerivation()) {
std::map<std::string, StorePath> outputs; std::map<std::string, std::optional<StorePath>> outputs;
auto drv = store->readDerivation(storePath); auto drv = store->readDerivation(storePath);
for (auto & i : drv.outputsAndPaths(*store)) for (auto & [name, output] : drv.outputsAndOptPaths(*store))
outputs.emplace(i.first, i.second.second); outputs.emplace(name, output.second);
return { return {
BuildableFromDrv { BuildableFromDrv {
.drvPath = storePath, .drvPath = storePath,
@ -331,7 +331,7 @@ Buildables InstallableValue::toBuildables()
{ {
Buildables res; Buildables res;
std::map<StorePath, OutputPathMap> drvsToOutputs; std::map<StorePath, std::map<std::string, std::optional<StorePath>>> drvsToOutputs;
// Group by derivation, helps with .all in particular // Group by derivation, helps with .all in particular
for (auto & drv : toDerivations()) { for (auto & drv : toDerivations()) {
@ -674,8 +674,11 @@ StorePathSet toStorePaths(ref<Store> store,
outPaths.insert(bo.path); outPaths.insert(bo.path);
}, },
[&](BuildableFromDrv bfd) { [&](BuildableFromDrv bfd) {
for (auto & output : bfd.outputs) for (auto & output : bfd.outputs) {
outPaths.insert(output.second); if (!output.second)
throw Error("Cannot operate on output of unbuilt CA drv");
outPaths.insert(*output.second);
}
}, },
}, b); }, b);
} else { } else {

6
src/nix/installables.hh
View file

@ -20,7 +20,7 @@ struct BuildableOpaque {
struct BuildableFromDrv { struct BuildableFromDrv {
StorePath drvPath; StorePath drvPath;
std::map<std::string, StorePath> outputs; std::map<std::string, std::optional<StorePath>> outputs;
}; };
typedef std::variant< typedef std::variant<
@ -69,7 +69,7 @@ struct Installable
virtual FlakeRef nixpkgsFlakeRef() const virtual FlakeRef nixpkgsFlakeRef() const
{ {
return std::move(FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}})); return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}});
} }
}; };
@ -82,7 +82,7 @@ struct InstallableValue : Installable
struct DerivationInfo struct DerivationInfo
{ {
StorePath drvPath; StorePath drvPath;
StorePath outPath; std::optional<StorePath> outPath;
std::string outputName; std::string outputName;
}; };

1
src/nix/main.cc
View file

@ -185,6 +185,7 @@ void mainWrapped(int argc, char * * argv)
} }
if (argc == 2 && std::string(argv[1]) == "__dump-builtins") { if (argc == 2 && std::string(argv[1]) == "__dump-builtins") {
evalSettings.pureEval = false;
EvalState state({}, openStore("dummy://")); EvalState state({}, openStore("dummy://"));
auto res = nlohmann::json::object(); auto res = nlohmann::json::object();
auto builtins = state.baseEnv.values[0]->attrs; auto builtins = state.baseEnv.values[0]->attrs;

10
src/nix/profile.cc
View file

@ -180,7 +180,9 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
auto [attrPath, resolvedRef, drv] = installable2->toDerivation(); auto [attrPath, resolvedRef, drv] = installable2->toDerivation();
ProfileElement element; ProfileElement element;
element.storePaths = {drv.outPath}; // FIXME if (!drv.outPath)
throw UnimplementedError("CA derivations are not yet supported by 'nix profile'");
element.storePaths = {*drv.outPath}; // FIXME
element.source = ProfileElementSource{ element.source = ProfileElementSource{
installable2->flakeRef, installable2->flakeRef,
resolvedRef, resolvedRef,
@ -191,7 +193,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
manifest.elements.emplace_back(std::move(element)); manifest.elements.emplace_back(std::move(element));
} else } else
throw Error("'nix profile install' does not support argument '%s'", installable->what()); throw UnimplementedError("'nix profile install' does not support argument '%s'", installable->what());
} }
store->buildPaths(pathsToBuild); store->buildPaths(pathsToBuild);
@ -349,7 +351,9 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
printInfo("upgrading '%s' from flake '%s' to '%s'", printInfo("upgrading '%s' from flake '%s' to '%s'",
element.source->attrPath, element.source->resolvedRef, resolvedRef); element.source->attrPath, element.source->resolvedRef, resolvedRef);
element.storePaths = {drv.outPath}; // FIXME if (!drv.outPath)
throw UnimplementedError("CA derivations are not yet supported by 'nix profile'");
element.storePaths = {*drv.outPath}; // FIXME
element.source = ProfileElementSource{ element.source = ProfileElementSource{
installable.flakeRef, installable.flakeRef,
resolvedRef, resolvedRef,

4
src/nix/repl.cc
View file

@ -496,8 +496,8 @@ bool NixRepl::processLine(string line)
if (runProgram(settings.nixBinDir + "/nix", Strings{"build", "--no-link", drvPathRaw}) == 0) { if (runProgram(settings.nixBinDir + "/nix", Strings{"build", "--no-link", drvPathRaw}) == 0) {
auto drv = state->store->readDerivation(drvPath); auto drv = state->store->readDerivation(drvPath);
std::cout << std::endl << "this derivation produced the following outputs:" << std::endl; std::cout << std::endl << "this derivation produced the following outputs:" << std::endl;
for (auto & i : drv.outputsAndPaths(*state->store)) for (auto & i : drv.outputsAndOptPaths(*state->store))
std::cout << fmt(" %s -> %s\n", i.first, state->store->printStorePath(i.second.second)); std::cout << fmt(" %s -> %s\n", i.first, state->store->printStorePath(*i.second.second));
} }
} else if (command == ":i") { } else if (command == ":i") {
runProgram(settings.nixBinDir + "/nix-env", Strings{"-i", drvPathRaw}); runProgram(settings.nixBinDir + "/nix-env", Strings{"-i", drvPathRaw});

11
src/nix/show-derivation.cc
View file

@ -67,21 +67,22 @@ struct CmdShowDerivation : InstallablesCommand
{ {
auto outputsObj(drvObj.object("outputs")); auto outputsObj(drvObj.object("outputs"));
for (auto & output : drv.outputsAndPaths(*store)) { for (auto & [_outputName, output] : drv.outputs) {
auto outputObj(outputsObj.object(output.first)); auto & outputName = _outputName; // work around clang bug
outputObj.attr("path", store->printStorePath(output.second.second)); auto outputObj { outputsObj.object(outputName) };
std::visit(overloaded { std::visit(overloaded {
[&](DerivationOutputInputAddressed doi) { [&](DerivationOutputInputAddressed doi) {
outputObj.attr("path", store->printStorePath(doi.path));
}, },
[&](DerivationOutputCAFixed dof) { [&](DerivationOutputCAFixed dof) {
outputObj.attr("path", store->printStorePath(dof.path(*store, drv.name, outputName)));
outputObj.attr("hashAlgo", dof.hash.printMethodAlgo()); outputObj.attr("hashAlgo", dof.hash.printMethodAlgo());
outputObj.attr("hash", dof.hash.hash.to_string(Base16, false)); outputObj.attr("hash", dof.hash.hash.to_string(Base16, false));
}, },
[&](DerivationOutputCAFloating dof) { [&](DerivationOutputCAFloating dof) {
outputObj.attr("hashAlgo", makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)); outputObj.attr("hashAlgo", makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType));
}, },
}, output.second.first.output); }, output.output);
} }
} }

32
tests/content-addressed.nix Normal file
View file

@ -0,0 +1,32 @@
with import ./config.nix;
{ seed ? 0 }:
# A simple content-addressed derivation.
# The derivation can be arbitrarily modified by passing a different `seed`,
# but the output will always be the same
rec {
rootLegacy = mkDerivation {
name = "simple-input-addressed";
buildCommand = ''
set -x
echo "Building a legacy derivation"
mkdir -p $out
echo "Hello World" > $out/hello
'';
};
rootCA = mkDerivation {
name = "dependent";
outputs = [ "out" "dev" ];
buildCommand = ''
echo "building a CA derivation"
echo "The seed is ${toString seed}"
mkdir -p $out
echo ${rootLegacy}/hello > $out/dep
# test symlink at root
ln -s $out $dev
'';
__contentAddressed = true;
outputHashMode = "recursive";
outputHashAlgo = "sha256";
};
}

17
tests/content-addressed.sh Normal file
View file

@ -0,0 +1,17 @@
#!/usr/bin/env bash
source common.sh
clearStore
clearCache
export REMOTE_STORE=file://$cacheDir
drv=$(nix-instantiate --experimental-features ca-derivations ./content-addressed.nix -A rootCA --arg seed 1)
nix --experimental-features 'nix-command ca-derivations' show-derivation --derivation "$drv" --arg seed 1
commonArgs=("--experimental-features" "ca-derivations" "./content-addressed.nix" "-A" "rootCA" "--no-out-link")
out1=$(nix-build "${commonArgs[@]}" ./content-addressed.nix --arg seed 1)
out2=$(nix-build "${commonArgs[@]}" ./content-addressed.nix --arg seed 2)
test $out1 == $out2

8
tests/describe-stores.sh Normal file
View file

@ -0,0 +1,8 @@
source common.sh
# Query an arbitrary value in `nix describe-stores --json`'s output just to
# check that it has the right structure
[[ $(nix --experimental-features 'nix-command flakes' describe-stores --json | jq '.["SSH Store"]["compress"]["defaultValue"]') == false ]]
# Ensure that the output of `nix describe-stores` isn't empty
[[ -n $(nix --experimental-features 'nix-command flakes' describe-stores) ]]

4
tests/local.mk
View file

@ -33,7 +33,9 @@ nix_tests = \
post-hook.sh \ post-hook.sh \
function-trace.sh \ function-trace.sh \
recursive.sh \ recursive.sh \
flakes.sh describe-stores.sh \
flakes.sh \
content-addressed.sh
# parallel.sh # parallel.sh
# build-remote-content-addressed-fixed.sh \ # build-remote-content-addressed-fixed.sh \

15
tests/multiple-outputs.nix
View file

@ -2,6 +2,21 @@ with import ./config.nix;
rec { rec {
# Want to ensure that "out" doesn't get a suffix on it's path.
nameCheck = mkDerivation {
name = "multiple-outputs-a";
outputs = [ "out" "dev" ];
builder = builtins.toFile "builder.sh"
''
mkdir $first $second
test -z $all
echo "first" > $first/file
echo "second" > $second/file
ln -s $first $second/link
'';
helloString = "Hello, world!";
};
a = mkDerivation { a = mkDerivation {
name = "multiple-outputs-a"; name = "multiple-outputs-a";
outputs = [ "first" "second" ]; outputs = [ "first" "second" ];

6
tests/multiple-outputs.sh
View file

@ -4,6 +4,12 @@ clearStore
rm -f $TEST_ROOT/result* rm -f $TEST_ROOT/result*
# Test whether the output names match our expectations
outPath=$(nix-instantiate multiple-outputs.nix --eval -A nameCheck.out.outPath)
[ "$(echo "$outPath" | sed -E 's_^".*/[^-/]*-([^/]*)"$_\1_')" = "multiple-outputs-a" ]
outPath=$(nix-instantiate multiple-outputs.nix --eval -A nameCheck.dev.outPath)
[ "$(echo "$outPath" | sed -E 's_^".*/[^-/]*-([^/]*)"$_\1_')" = "multiple-outputs-a-dev" ]
# Test whether read-only evaluation works when referring to the # Test whether read-only evaluation works when referring to the
# ‘drvPath’ attribute. # ‘drvPath’ attribute.
echo "evaluating c..." echo "evaluating c..."