2005-01-19 16:39:47 +00:00
|
|
|
#include "derivations.hh"
|
2006-11-30 17:43:04 +00:00
|
|
|
#include "store-api.hh"
|
2006-12-01 21:00:39 +00:00
|
|
|
#include "globals.hh"
|
2008-08-25 13:31:57 +00:00
|
|
|
#include "util.hh"
|
Allow remote builds without sending the derivation closure
Previously, to build a derivation remotely, we had to copy the entire
closure of the .drv file to the remote machine, even though we only
need the top-level derivation. This is very wasteful: the closure can
contain thousands of store paths, and in some Hydra use cases, include
source paths that are very large (e.g. Git/Mercurial checkouts).
So now there is a new operation, StoreAPI::buildDerivation(), that
performs a build from an in-memory representation of a derivation
(BasicDerivation) rather than from a on-disk .drv file. The only files
that need to be in the Nix store are the sources of the derivation
(drv.inputSrcs), and the needed output paths of the dependencies (as
described by drv.inputDrvs). "nix-store --serve" exposes this
interface.
Note that this is a privileged operation, because you can construct a
derivation that builds any store path whatsoever. Fixing this will
require changing the hashing scheme (i.e., the output paths should be
computed from the other fields in BasicDerivation, allowing them to be
verified without access to other derivations). However, this would be
quite nice because it would allow .drv-free building (e.g. "nix-env
-i" wouldn't have to write any .drv files to disk).
Fixes #173.
2015-07-17 15:57:40 +00:00
|
|
|
#include "worker-protocol.hh"
|
2016-06-02 16:43:36 +00:00
|
|
|
#include "fs-accessor.hh"
|
2022-01-19 14:20:46 +00:00
|
|
|
#include <boost/container/small_vector.hpp>
|
2023-02-17 23:37:35 +00:00
|
|
|
#include <nlohmann/json.hpp>
|
2003-06-16 13:33:38 +00:00
|
|
|
|
2006-09-04 21:06:23 +00:00
|
|
|
namespace nix {
|
|
|
|
|
2020-09-15 15:21:39 +00:00
|
|
|
std::optional<StorePath> DerivationOutput::path(const Store & store, std::string_view drvName, std::string_view outputName) const
|
2020-07-08 23:11:39 +00:00
|
|
|
{
|
|
|
|
return std::visit(overloaded {
|
2022-03-17 22:29:15 +00:00
|
|
|
[](const DerivationOutput::InputAddressed & doi) -> std::optional<StorePath> {
|
2020-07-12 16:12:21 +00:00
|
|
|
return { doi.path };
|
|
|
|
},
|
2022-03-17 22:29:15 +00:00
|
|
|
[&](const DerivationOutput::CAFixed & dof) -> std::optional<StorePath> {
|
2020-07-12 16:12:21 +00:00
|
|
|
return {
|
2020-08-14 17:00:13 +00:00
|
|
|
dof.path(store, drvName, outputName)
|
2020-07-12 16:12:21 +00:00
|
|
|
};
|
|
|
|
},
|
2022-03-17 22:29:15 +00:00
|
|
|
[](const DerivationOutput::CAFloating & dof) -> std::optional<StorePath> {
|
2020-07-12 16:12:21 +00:00
|
|
|
return std::nullopt;
|
2020-07-08 23:11:39 +00:00
|
|
|
},
|
2022-03-17 22:29:15 +00:00
|
|
|
[](const DerivationOutput::Deferred &) -> std::optional<StorePath> {
|
2020-09-23 14:30:42 +00:00
|
|
|
return std::nullopt;
|
|
|
|
},
|
2022-03-30 14:31:01 +00:00
|
|
|
[](const DerivationOutput::Impure &) -> std::optional<StorePath> {
|
|
|
|
return std::nullopt;
|
|
|
|
},
|
2022-03-17 22:29:15 +00:00
|
|
|
}, raw());
|
2020-07-08 23:11:39 +00:00
|
|
|
}
|
|
|
|
|
Eliminate the "store" global variable
Also, move a few free-standing functions into StoreAPI and Derivation.
Also, introduce a non-nullable smart pointer, ref<T>, which is just a
wrapper around std::shared_ptr ensuring that the pointer is never
null. (For reference-counted values, this is better than passing a
"T&", because the latter doesn't maintain the refcount. Usually, the
caller will have a shared_ptr keeping the value alive, but that's not
always the case, e.g., when passing a reference to a std::thread via
std::bind.)
2016-02-04 13:28:26 +00:00
|
|
|
|
2022-03-30 14:31:01 +00:00
|
|
|
StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const
|
|
|
|
{
|
2020-08-07 19:09:26 +00:00
|
|
|
return store.makeFixedOutputPath(
|
2020-10-07 13:52:20 +00:00
|
|
|
outputPathName(drvName, outputName),
|
|
|
|
{ hash, {} });
|
2020-08-07 19:09:26 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2022-03-30 14:31:01 +00:00
|
|
|
bool DerivationType::isCA() const
|
|
|
|
{
|
2022-03-18 00:36:52 +00:00
|
|
|
/* Normally we do the full `std::visit` to make sure we have
|
|
|
|
exhaustively handled all variants, but so long as there is a
|
|
|
|
variant called `ContentAddressed`, it must be the only one for
|
|
|
|
which `isCA` is true for this to make sense!. */
|
2022-03-30 14:31:01 +00:00
|
|
|
return std::visit(overloaded {
|
|
|
|
[](const InputAddressed & ia) {
|
|
|
|
return false;
|
|
|
|
},
|
|
|
|
[](const ContentAddressed & ca) {
|
|
|
|
return true;
|
|
|
|
},
|
|
|
|
[](const Impure &) {
|
|
|
|
return true;
|
|
|
|
},
|
|
|
|
}, raw());
|
2020-06-03 17:38:54 +00:00
|
|
|
}
|
|
|
|
|
2022-03-30 14:31:01 +00:00
|
|
|
bool DerivationType::isFixed() const
|
|
|
|
{
|
2022-03-18 00:36:52 +00:00
|
|
|
return std::visit(overloaded {
|
|
|
|
[](const InputAddressed & ia) {
|
|
|
|
return false;
|
|
|
|
},
|
|
|
|
[](const ContentAddressed & ca) {
|
|
|
|
return ca.fixed;
|
|
|
|
},
|
2022-03-30 14:31:01 +00:00
|
|
|
[](const Impure &) {
|
|
|
|
return false;
|
|
|
|
},
|
2022-03-18 00:36:52 +00:00
|
|
|
}, raw());
|
2020-06-03 17:38:54 +00:00
|
|
|
}
|
|
|
|
|
2022-03-30 14:31:01 +00:00
|
|
|
bool DerivationType::hasKnownOutputPaths() const
|
|
|
|
{
|
2022-03-18 00:36:52 +00:00
|
|
|
return std::visit(overloaded {
|
|
|
|
[](const InputAddressed & ia) {
|
|
|
|
return !ia.deferred;
|
|
|
|
},
|
|
|
|
[](const ContentAddressed & ca) {
|
|
|
|
return ca.fixed;
|
|
|
|
},
|
2022-03-30 14:31:01 +00:00
|
|
|
[](const Impure &) {
|
|
|
|
return false;
|
|
|
|
},
|
2022-03-18 00:36:52 +00:00
|
|
|
}, raw());
|
2021-02-26 15:34:33 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2022-03-31 14:06:40 +00:00
|
|
|
bool DerivationType::isSandboxed() const
|
2022-03-30 14:31:01 +00:00
|
|
|
{
|
2022-03-18 00:36:52 +00:00
|
|
|
return std::visit(overloaded {
|
|
|
|
[](const InputAddressed & ia) {
|
2022-03-31 14:06:40 +00:00
|
|
|
return true;
|
2022-03-18 00:36:52 +00:00
|
|
|
},
|
|
|
|
[](const ContentAddressed & ca) {
|
2022-03-31 14:12:25 +00:00
|
|
|
return ca.sandboxed;
|
2022-03-18 00:36:52 +00:00
|
|
|
},
|
2022-03-30 14:31:01 +00:00
|
|
|
[](const Impure &) {
|
2022-03-31 14:06:40 +00:00
|
|
|
return false;
|
2022-03-30 14:31:01 +00:00
|
|
|
},
|
|
|
|
}, raw());
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
bool DerivationType::isPure() const
|
|
|
|
{
|
|
|
|
return std::visit(overloaded {
|
|
|
|
[](const InputAddressed & ia) {
|
|
|
|
return true;
|
|
|
|
},
|
|
|
|
[](const ContentAddressed & ca) {
|
|
|
|
return true;
|
|
|
|
},
|
|
|
|
[](const Impure &) {
|
|
|
|
return false;
|
|
|
|
},
|
2022-03-18 00:36:52 +00:00
|
|
|
}, raw());
|
2020-06-03 17:38:54 +00:00
|
|
|
}
|
|
|
|
|
Eliminate the "store" global variable
Also, move a few free-standing functions into StoreAPI and Derivation.
Also, introduce a non-nullable smart pointer, ref<T>, which is just a
wrapper around std::shared_ptr ensuring that the pointer is never
null. (For reference-counted values, this is better than passing a
"T&", because the latter doesn't maintain the refcount. Usually, the
caller will have a shared_ptr keeping the value alive, but that's not
always the case, e.g., when passing a reference to a std::thread via
std::bind.)
2016-02-04 13:28:26 +00:00
|
|
|
|
|
|
|
bool BasicDerivation::isBuiltin() const
|
|
|
|
{
|
2022-02-25 15:00:00 +00:00
|
|
|
return builder.substr(0, 8) == "builtin:";
|
Eliminate the "store" global variable
Also, move a few free-standing functions into StoreAPI and Derivation.
Also, introduce a non-nullable smart pointer, ref<T>, which is just a
wrapper around std::shared_ptr ensuring that the pointer is never
null. (For reference-counted values, this is better than passing a
"T&", because the latter doesn't maintain the refcount. Usually, the
caller will have a shared_ptr keeping the value alive, but that's not
always the case, e.g., when passing a reference to a std::thread via
std::bind.)
2016-02-04 13:28:26 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2020-08-23 15:00:25 +00:00
|
|
|
StorePath writeDerivation(Store & store,
|
2020-09-04 18:33:58 +00:00
|
|
|
const Derivation & drv, RepairFlag repair, bool readOnly)
|
2003-07-04 12:18:06 +00:00
|
|
|
{
|
2020-06-16 20:20:18 +00:00
|
|
|
auto references = drv.inputSrcs;
|
2015-07-17 17:24:28 +00:00
|
|
|
for (auto & i : drv.inputDrvs)
|
2020-06-16 20:20:18 +00:00
|
|
|
references.insert(i.first);
|
2005-01-25 21:28:25 +00:00
|
|
|
/* Note that the outputs of a derivation are *not* references
|
|
|
|
(that can be missing (of course) and should not necessarily be
|
|
|
|
held during a garbage collection). */
|
2020-08-09 20:32:35 +00:00
|
|
|
auto suffix = std::string(drv.name) + drvExtension;
|
2020-08-23 15:00:25 +00:00
|
|
|
auto contents = drv.unparse(store, false);
|
2020-09-04 18:33:58 +00:00
|
|
|
return readOnly || settings.readOnlyMode
|
2020-08-23 15:00:25 +00:00
|
|
|
? store.computeStorePathForText(suffix, contents, references)
|
|
|
|
: store.addTextToStore(suffix, contents, references, repair);
|
2003-07-04 12:18:06 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2016-09-14 08:54:57 +00:00
|
|
|
/* Read string `s' from stream `str'. */
|
2022-02-25 15:00:00 +00:00
|
|
|
static void expect(std::istream & str, std::string_view s)
|
2016-09-14 08:54:57 +00:00
|
|
|
{
|
|
|
|
char s2[s.size()];
|
|
|
|
str.read(s2, s.size());
|
2022-02-25 15:00:00 +00:00
|
|
|
if (std::string(s2, s.size()) != s)
|
2020-04-21 23:07:07 +00:00
|
|
|
throw FormatError("expected string '%1%'", s);
|
2016-09-14 08:54:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/* Read a C-style string from stream `str'. */
|
2022-02-25 15:00:00 +00:00
|
|
|
static std::string parseString(std::istream & str)
|
2016-09-14 08:54:57 +00:00
|
|
|
{
|
2022-02-25 15:00:00 +00:00
|
|
|
std::string res;
|
2016-09-14 08:54:57 +00:00
|
|
|
expect(str, "\"");
|
|
|
|
int c;
|
|
|
|
while ((c = str.get()) != '"')
|
|
|
|
if (c == '\\') {
|
|
|
|
c = str.get();
|
|
|
|
if (c == 'n') res += '\n';
|
|
|
|
else if (c == 'r') res += '\r';
|
|
|
|
else if (c == 't') res += '\t';
|
|
|
|
else res += c;
|
|
|
|
}
|
|
|
|
else res += c;
|
|
|
|
return res;
|
|
|
|
}
|
|
|
|
|
2020-08-07 19:09:26 +00:00
|
|
|
static void validatePath(std::string_view s) {
|
|
|
|
if (s.size() == 0 || s[0] != '/')
|
|
|
|
throw FormatError("bad path '%1%' in derivation", s);
|
|
|
|
}
|
2016-09-14 08:54:57 +00:00
|
|
|
|
2010-04-19 13:46:58 +00:00
|
|
|
static Path parsePath(std::istream & str)
|
* Removed the `id' attribute hack.
* Formalise the notion of fixed-output derivations, i.e., derivations
for which a cryptographic hash of the output is known in advance.
Changes to such derivations should not propagate upwards through the
dependency graph. Previously this was done by specifying the hash
component of the output path through the `id' attribute, but this is
insecure since you can lie about it (i.e., you can specify any hash
and then produce a completely different output). Now the
responsibility for checking the output is moved from the builder to
Nix itself.
A fixed-output derivation can be created by specifying the
`outputHash' and `outputHashAlgo' attributes, the latter taking
values `md5', `sha1', and `sha256', and the former specifying the
actual hash in hexadecimal or in base-32 (auto-detected by looking
at the length of the attribute value). MD5 is included for
compatibility but should be considered deprecated.
* Removed the `drvPath' pseudo-attribute in derivation results. It's
no longer necessary.
* Cleaned up the support for multiple output paths in derivation store
expressions. Each output now has a unique identifier (e.g., `out',
`devel', `docs'). Previously there was no way to tell output paths
apart at the store expression level.
* `nix-hash' now has a flag `--base32' to specify that the hash should
be printed in base-32 notation.
* `fetchurl' accepts parameters `sha256' and `sha1' in addition to
`md5'.
* `nix-prefetch-url' now prints out a SHA-1 hash in base-32. (TODO: a
flag to specify the hash.)
2005-01-17 16:55:19 +00:00
|
|
|
{
|
2020-08-07 19:09:26 +00:00
|
|
|
auto s = parseString(str);
|
|
|
|
validatePath(s);
|
2010-04-19 13:46:58 +00:00
|
|
|
return s;
|
* Removed the `id' attribute hack.
* Formalise the notion of fixed-output derivations, i.e., derivations
for which a cryptographic hash of the output is known in advance.
Changes to such derivations should not propagate upwards through the
dependency graph. Previously this was done by specifying the hash
component of the output path through the `id' attribute, but this is
insecure since you can lie about it (i.e., you can specify any hash
and then produce a completely different output). Now the
responsibility for checking the output is moved from the builder to
Nix itself.
A fixed-output derivation can be created by specifying the
`outputHash' and `outputHashAlgo' attributes, the latter taking
values `md5', `sha1', and `sha256', and the former specifying the
actual hash in hexadecimal or in base-32 (auto-detected by looking
at the length of the attribute value). MD5 is included for
compatibility but should be considered deprecated.
* Removed the `drvPath' pseudo-attribute in derivation results. It's
no longer necessary.
* Cleaned up the support for multiple output paths in derivation store
expressions. Each output now has a unique identifier (e.g., `out',
`devel', `docs'). Previously there was no way to tell output paths
apart at the store expression level.
* `nix-hash' now has a flag `--base32' to specify that the hash should
be printed in base-32 notation.
* `fetchurl' accepts parameters `sha256' and `sha1' in addition to
`md5'.
* `nix-prefetch-url' now prints out a SHA-1 hash in base-32. (TODO: a
flag to specify the hash.)
2005-01-17 16:55:19 +00:00
|
|
|
}
|
2012-07-30 23:55:41 +00:00
|
|
|
|
* Removed the `id' attribute hack.
* Formalise the notion of fixed-output derivations, i.e., derivations
for which a cryptographic hash of the output is known in advance.
Changes to such derivations should not propagate upwards through the
dependency graph. Previously this was done by specifying the hash
component of the output path through the `id' attribute, but this is
insecure since you can lie about it (i.e., you can specify any hash
and then produce a completely different output). Now the
responsibility for checking the output is moved from the builder to
Nix itself.
A fixed-output derivation can be created by specifying the
`outputHash' and `outputHashAlgo' attributes, the latter taking
values `md5', `sha1', and `sha256', and the former specifying the
actual hash in hexadecimal or in base-32 (auto-detected by looking
at the length of the attribute value). MD5 is included for
compatibility but should be considered deprecated.
* Removed the `drvPath' pseudo-attribute in derivation results. It's
no longer necessary.
* Cleaned up the support for multiple output paths in derivation store
expressions. Each output now has a unique identifier (e.g., `out',
`devel', `docs'). Previously there was no way to tell output paths
apart at the store expression level.
* `nix-hash' now has a flag `--base32' to specify that the hash should
be printed in base-32 notation.
* `fetchurl' accepts parameters `sha256' and `sha1' in addition to
`md5'.
* `nix-prefetch-url' now prints out a SHA-1 hash in base-32. (TODO: a
flag to specify the hash.)
2005-01-17 16:55:19 +00:00
|
|
|
|
2016-09-14 08:54:57 +00:00
|
|
|
static bool endOfList(std::istream & str)
|
|
|
|
{
|
|
|
|
if (str.peek() == ',') {
|
|
|
|
str.get();
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
if (str.peek() == ']') {
|
|
|
|
str.get();
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2010-04-19 13:46:58 +00:00
|
|
|
static StringSet parseStrings(std::istream & str, bool arePaths)
|
2003-07-15 16:28:54 +00:00
|
|
|
{
|
2010-04-19 13:46:58 +00:00
|
|
|
StringSet res;
|
|
|
|
while (!endOfList(str))
|
|
|
|
res.insert(arePaths ? parsePath(str) : parseString(str));
|
|
|
|
return res;
|
2003-07-15 16:28:54 +00:00
|
|
|
}
|
2012-07-30 23:55:41 +00:00
|
|
|
|
2003-07-15 16:28:54 +00:00
|
|
|
|
2020-08-09 20:51:34 +00:00
|
|
|
static DerivationOutput parseDerivationOutput(const Store & store,
|
2020-08-10 01:57:54 +00:00
|
|
|
std::string_view pathS, std::string_view hashAlgo, std::string_view hash)
|
2020-03-23 03:43:07 +00:00
|
|
|
{
|
|
|
|
if (hashAlgo != "") {
|
2020-06-17 04:55:47 +00:00
|
|
|
auto method = FileIngestionMethod::Flat;
|
2022-02-25 15:00:00 +00:00
|
|
|
if (hashAlgo.substr(0, 2) == "r:") {
|
2020-03-23 03:43:07 +00:00
|
|
|
method = FileIngestionMethod::Recursive;
|
2020-08-09 20:51:34 +00:00
|
|
|
hashAlgo = hashAlgo.substr(2);
|
2020-03-23 03:43:07 +00:00
|
|
|
}
|
2020-08-07 19:09:26 +00:00
|
|
|
const auto hashType = parseHashType(hashAlgo);
|
2022-03-30 14:31:01 +00:00
|
|
|
if (hash == "impure") {
|
2023-03-17 14:33:48 +00:00
|
|
|
experimentalFeatureSettings.require(Xp::ImpureDerivations);
|
2022-03-30 14:31:01 +00:00
|
|
|
assert(pathS == "");
|
2022-03-31 14:39:18 +00:00
|
|
|
return DerivationOutput::Impure {
|
|
|
|
.method = std::move(method),
|
|
|
|
.hashType = std::move(hashType),
|
2022-03-30 14:31:01 +00:00
|
|
|
};
|
|
|
|
} else if (hash != "") {
|
2020-08-07 19:09:26 +00:00
|
|
|
validatePath(pathS);
|
2022-03-17 22:29:15 +00:00
|
|
|
return DerivationOutput::CAFixed {
|
|
|
|
.hash = FixedOutputHash {
|
|
|
|
.method = std::move(method),
|
|
|
|
.hash = Hash::parseNonSRIUnprefixed(hash, hashType),
|
2020-08-07 19:09:26 +00:00
|
|
|
},
|
|
|
|
};
|
|
|
|
} else {
|
2023-03-17 14:33:48 +00:00
|
|
|
experimentalFeatureSettings.require(Xp::CaDerivations);
|
2020-08-07 19:09:26 +00:00
|
|
|
assert(pathS == "");
|
2022-03-17 22:29:15 +00:00
|
|
|
return DerivationOutput::CAFloating {
|
|
|
|
.method = std::move(method),
|
|
|
|
.hashType = std::move(hashType),
|
2020-08-07 19:09:26 +00:00
|
|
|
};
|
|
|
|
}
|
|
|
|
} else {
|
2020-09-23 14:30:42 +00:00
|
|
|
if (pathS == "") {
|
2022-03-17 22:29:15 +00:00
|
|
|
return DerivationOutput::Deferred { };
|
2020-09-23 14:30:42 +00:00
|
|
|
}
|
2020-08-07 19:09:26 +00:00
|
|
|
validatePath(pathS);
|
2022-03-17 22:29:15 +00:00
|
|
|
return DerivationOutput::InputAddressed {
|
|
|
|
.path = store.parseStorePath(pathS),
|
2020-07-08 23:11:39 +00:00
|
|
|
};
|
2020-08-07 19:09:26 +00:00
|
|
|
}
|
2020-03-23 03:43:07 +00:00
|
|
|
}
|
|
|
|
|
2020-08-09 20:51:34 +00:00
|
|
|
static DerivationOutput parseDerivationOutput(const Store & store, std::istringstream & str)
|
|
|
|
{
|
2020-08-11 00:12:54 +00:00
|
|
|
expect(str, ","); const auto pathS = parseString(str);
|
2020-08-09 20:51:34 +00:00
|
|
|
expect(str, ","); const auto hashAlgo = parseString(str);
|
|
|
|
expect(str, ","); const auto hash = parseString(str);
|
|
|
|
expect(str, ")");
|
|
|
|
|
2020-08-10 01:57:54 +00:00
|
|
|
return parseDerivationOutput(store, pathS, hashAlgo, hash);
|
2020-08-09 20:51:34 +00:00
|
|
|
}
|
|
|
|
|
2020-03-23 03:43:07 +00:00
|
|
|
|
2020-08-01 19:38:35 +00:00
|
|
|
Derivation parseDerivation(const Store & store, std::string && s, std::string_view name)
|
2003-07-15 21:24:05 +00:00
|
|
|
{
|
2005-01-19 11:16:11 +00:00
|
|
|
Derivation drv;
|
2020-07-08 19:38:01 +00:00
|
|
|
drv.name = name;
|
|
|
|
|
2020-07-13 16:22:56 +00:00
|
|
|
std::istringstream str(std::move(s));
|
2010-04-19 13:46:58 +00:00
|
|
|
expect(str, "Derive([");
|
2003-11-16 17:46:31 +00:00
|
|
|
|
2010-04-19 13:46:58 +00:00
|
|
|
/* Parse the list of outputs. */
|
|
|
|
while (!endOfList(str)) {
|
2019-12-05 18:11:09 +00:00
|
|
|
expect(str, "("); std::string id = parseString(str);
|
2020-06-17 04:55:47 +00:00
|
|
|
auto output = parseDerivationOutput(store, str);
|
|
|
|
drv.outputs.emplace(std::move(id), std::move(output));
|
* Removed the `id' attribute hack.
* Formalise the notion of fixed-output derivations, i.e., derivations
for which a cryptographic hash of the output is known in advance.
Changes to such derivations should not propagate upwards through the
dependency graph. Previously this was done by specifying the hash
component of the output path through the `id' attribute, but this is
insecure since you can lie about it (i.e., you can specify any hash
and then produce a completely different output). Now the
responsibility for checking the output is moved from the builder to
Nix itself.
A fixed-output derivation can be created by specifying the
`outputHash' and `outputHashAlgo' attributes, the latter taking
values `md5', `sha1', and `sha256', and the former specifying the
actual hash in hexadecimal or in base-32 (auto-detected by looking
at the length of the attribute value). MD5 is included for
compatibility but should be considered deprecated.
* Removed the `drvPath' pseudo-attribute in derivation results. It's
no longer necessary.
* Cleaned up the support for multiple output paths in derivation store
expressions. Each output now has a unique identifier (e.g., `out',
`devel', `docs'). Previously there was no way to tell output paths
apart at the store expression level.
* `nix-hash' now has a flag `--base32' to specify that the hash should
be printed in base-32 notation.
* `fetchurl' accepts parameters `sha256' and `sha1' in addition to
`md5'.
* `nix-prefetch-url' now prints out a SHA-1 hash in base-32. (TODO: a
flag to specify the hash.)
2005-01-17 16:55:19 +00:00
|
|
|
}
|
|
|
|
|
2010-04-19 13:46:58 +00:00
|
|
|
/* Parse the list of input derivations. */
|
|
|
|
expect(str, ",[");
|
|
|
|
while (!endOfList(str)) {
|
|
|
|
expect(str, "(");
|
|
|
|
Path drvPath = parsePath(str);
|
|
|
|
expect(str, ",[");
|
2019-12-05 18:11:09 +00:00
|
|
|
drv.inputDrvs.insert_or_assign(store.parseStorePath(drvPath), parseStrings(str, false));
|
2010-04-19 13:46:58 +00:00
|
|
|
expect(str, ")");
|
2005-01-20 14:10:19 +00:00
|
|
|
}
|
2003-07-15 16:28:54 +00:00
|
|
|
|
2019-12-05 18:11:09 +00:00
|
|
|
expect(str, ",["); drv.inputSrcs = store.parseStorePathSet(parseStrings(str, true));
|
2010-04-19 13:46:58 +00:00
|
|
|
expect(str, ","); drv.platform = parseString(str);
|
|
|
|
expect(str, ","); drv.builder = parseString(str);
|
|
|
|
|
|
|
|
/* Parse the builder arguments. */
|
|
|
|
expect(str, ",[");
|
|
|
|
while (!endOfList(str))
|
|
|
|
drv.args.push_back(parseString(str));
|
|
|
|
|
|
|
|
/* Parse the environment variables. */
|
|
|
|
expect(str, ",[");
|
|
|
|
while (!endOfList(str)) {
|
2022-02-25 15:00:00 +00:00
|
|
|
expect(str, "("); auto name = parseString(str);
|
|
|
|
expect(str, ","); auto value = parseString(str);
|
2010-04-19 13:46:58 +00:00
|
|
|
expect(str, ")");
|
|
|
|
drv.env[name] = value;
|
2003-08-15 12:32:37 +00:00
|
|
|
}
|
2012-07-30 23:55:41 +00:00
|
|
|
|
2010-04-19 13:46:58 +00:00
|
|
|
expect(str, ")");
|
|
|
|
return drv;
|
|
|
|
}
|
2003-08-15 12:32:37 +00:00
|
|
|
|
2003-07-15 16:28:54 +00:00
|
|
|
|
2023-04-09 20:42:20 +00:00
|
|
|
/**
|
2023-04-15 18:56:51 +00:00
|
|
|
* Print a derivation string literal to an `std::string`.
|
2023-04-09 20:42:20 +00:00
|
|
|
*
|
|
|
|
* This syntax does not generalize to the expression language, which needs to
|
|
|
|
* escape `$`.
|
|
|
|
*
|
|
|
|
* @param res Where to print to
|
|
|
|
* @param s Which logical string to print
|
|
|
|
*/
|
2022-02-25 15:00:00 +00:00
|
|
|
static void printString(std::string & res, std::string_view s)
|
2020-02-23 15:36:19 +00:00
|
|
|
{
|
2022-01-21 16:25:37 +00:00
|
|
|
boost::container::small_vector<char, 64 * 1024> buffer;
|
2022-01-19 14:20:46 +00:00
|
|
|
buffer.reserve(s.size() * 2 + 2);
|
|
|
|
char * buf = buffer.data();
|
|
|
|
char * p = buf;
|
|
|
|
*p++ = '"';
|
|
|
|
for (auto c : s)
|
|
|
|
if (c == '\"' || c == '\\') { *p++ = '\\'; *p++ = c; }
|
|
|
|
else if (c == '\n') { *p++ = '\\'; *p++ = 'n'; }
|
|
|
|
else if (c == '\r') { *p++ = '\\'; *p++ = 'r'; }
|
|
|
|
else if (c == '\t') { *p++ = '\\'; *p++ = 't'; }
|
|
|
|
else *p++ = c;
|
|
|
|
*p++ = '"';
|
|
|
|
res.append(buf, p - buf);
|
2020-02-23 15:36:19 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2022-02-25 15:00:00 +00:00
|
|
|
static void printUnquotedString(std::string & res, std::string_view s)
|
2010-04-19 13:46:58 +00:00
|
|
|
{
|
2010-04-21 19:25:50 +00:00
|
|
|
res += '"';
|
2020-02-23 15:36:19 +00:00
|
|
|
res.append(s);
|
2010-04-21 19:25:50 +00:00
|
|
|
res += '"';
|
2003-07-16 11:05:59 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2010-04-19 13:46:58 +00:00
|
|
|
template<class ForwardIterator>
|
2022-02-25 15:00:00 +00:00
|
|
|
static void printStrings(std::string & res, ForwardIterator i, ForwardIterator j)
|
2003-07-15 22:28:27 +00:00
|
|
|
{
|
2010-04-21 19:25:50 +00:00
|
|
|
res += '[';
|
2010-04-19 13:46:58 +00:00
|
|
|
bool first = true;
|
|
|
|
for ( ; i != j; ++i) {
|
2010-04-21 19:25:50 +00:00
|
|
|
if (first) first = false; else res += ',';
|
|
|
|
printString(res, *i);
|
2010-04-19 13:46:58 +00:00
|
|
|
}
|
2010-04-21 19:25:50 +00:00
|
|
|
res += ']';
|
2010-04-19 13:46:58 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2020-02-23 15:36:19 +00:00
|
|
|
template<class ForwardIterator>
|
2022-02-25 15:00:00 +00:00
|
|
|
static void printUnquotedStrings(std::string & res, ForwardIterator i, ForwardIterator j)
|
2020-02-23 15:36:19 +00:00
|
|
|
{
|
|
|
|
res += '[';
|
|
|
|
bool first = true;
|
|
|
|
for ( ; i != j; ++i) {
|
|
|
|
if (first) first = false; else res += ',';
|
|
|
|
printUnquotedString(res, *i);
|
|
|
|
}
|
|
|
|
res += ']';
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2022-02-25 15:00:00 +00:00
|
|
|
std::string Derivation::unparse(const Store & store, bool maskOutputs,
|
2019-12-05 18:11:09 +00:00
|
|
|
std::map<std::string, StringSet> * actualInputs) const
|
2010-04-19 13:46:58 +00:00
|
|
|
{
|
2022-02-25 15:00:00 +00:00
|
|
|
std::string s;
|
2010-04-21 19:25:50 +00:00
|
|
|
s.reserve(65536);
|
|
|
|
s += "Derive([";
|
2010-04-19 13:46:58 +00:00
|
|
|
|
2020-01-21 20:14:13 +00:00
|
|
|
bool first = true;
|
|
|
|
for (auto & i : outputs) {
|
|
|
|
if (first) first = false; else s += ',';
|
2020-02-23 15:36:19 +00:00
|
|
|
s += '('; printUnquotedString(s, i.first);
|
2020-07-12 16:12:21 +00:00
|
|
|
std::visit(overloaded {
|
2022-03-17 22:29:15 +00:00
|
|
|
[&](const DerivationOutput::InputAddressed & doi) {
|
2020-08-07 19:09:26 +00:00
|
|
|
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(doi.path));
|
2020-07-12 16:12:21 +00:00
|
|
|
s += ','; printUnquotedString(s, "");
|
|
|
|
s += ','; printUnquotedString(s, "");
|
|
|
|
},
|
2022-03-17 22:29:15 +00:00
|
|
|
[&](const DerivationOutput::CAFixed & dof) {
|
2020-08-07 19:09:26 +00:00
|
|
|
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first)));
|
2020-07-12 16:12:21 +00:00
|
|
|
s += ','; printUnquotedString(s, dof.hash.printMethodAlgo());
|
|
|
|
s += ','; printUnquotedString(s, dof.hash.hash.to_string(Base16, false));
|
|
|
|
},
|
2022-03-17 22:29:15 +00:00
|
|
|
[&](const DerivationOutput::CAFloating & dof) {
|
2020-08-07 19:09:26 +00:00
|
|
|
s += ','; printUnquotedString(s, "");
|
2020-07-12 16:12:21 +00:00
|
|
|
s += ','; printUnquotedString(s, makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType));
|
|
|
|
s += ','; printUnquotedString(s, "");
|
|
|
|
},
|
2022-03-17 22:29:15 +00:00
|
|
|
[&](const DerivationOutput::Deferred &) {
|
2020-09-23 14:30:42 +00:00
|
|
|
s += ','; printUnquotedString(s, "");
|
|
|
|
s += ','; printUnquotedString(s, "");
|
|
|
|
s += ','; printUnquotedString(s, "");
|
2022-03-30 14:31:01 +00:00
|
|
|
},
|
|
|
|
[&](const DerivationOutputImpure & doi) {
|
|
|
|
// FIXME
|
|
|
|
s += ','; printUnquotedString(s, "");
|
|
|
|
s += ','; printUnquotedString(s, makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType));
|
|
|
|
s += ','; printUnquotedString(s, "impure");
|
2020-09-23 14:30:42 +00:00
|
|
|
}
|
2022-03-17 22:29:15 +00:00
|
|
|
}, i.second.raw());
|
2020-01-21 20:14:13 +00:00
|
|
|
s += ')';
|
2010-04-19 13:46:58 +00:00
|
|
|
}
|
|
|
|
|
2010-04-21 19:25:50 +00:00
|
|
|
s += "],[";
|
2020-01-21 20:14:13 +00:00
|
|
|
first = true;
|
2019-12-05 18:11:09 +00:00
|
|
|
if (actualInputs) {
|
|
|
|
for (auto & i : *actualInputs) {
|
|
|
|
if (first) first = false; else s += ',';
|
2020-02-23 15:36:19 +00:00
|
|
|
s += '('; printUnquotedString(s, i.first);
|
|
|
|
s += ','; printUnquotedStrings(s, i.second.begin(), i.second.end());
|
2019-12-05 18:11:09 +00:00
|
|
|
s += ')';
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
for (auto & i : inputDrvs) {
|
|
|
|
if (first) first = false; else s += ',';
|
2020-02-23 15:36:19 +00:00
|
|
|
s += '('; printUnquotedString(s, store.printStorePath(i.first));
|
|
|
|
s += ','; printUnquotedStrings(s, i.second.begin(), i.second.end());
|
2019-12-05 18:11:09 +00:00
|
|
|
s += ')';
|
|
|
|
}
|
2010-04-19 13:46:58 +00:00
|
|
|
}
|
|
|
|
|
2010-04-21 19:25:50 +00:00
|
|
|
s += "],";
|
2019-12-05 18:11:09 +00:00
|
|
|
auto paths = store.printStorePathSet(inputSrcs); // FIXME: slow
|
2020-02-23 15:36:19 +00:00
|
|
|
printUnquotedStrings(s, paths.begin(), paths.end());
|
2012-07-30 23:55:41 +00:00
|
|
|
|
2020-02-23 15:36:19 +00:00
|
|
|
s += ','; printUnquotedString(s, platform);
|
Eliminate the "store" global variable
Also, move a few free-standing functions into StoreAPI and Derivation.
Also, introduce a non-nullable smart pointer, ref<T>, which is just a
wrapper around std::shared_ptr ensuring that the pointer is never
null. (For reference-counted values, this is better than passing a
"T&", because the latter doesn't maintain the refcount. Usually, the
caller will have a shared_ptr keeping the value alive, but that's not
always the case, e.g., when passing a reference to a std::thread via
std::bind.)
2016-02-04 13:28:26 +00:00
|
|
|
s += ','; printString(s, builder);
|
|
|
|
s += ','; printStrings(s, args.begin(), args.end());
|
2010-04-19 13:46:58 +00:00
|
|
|
|
2010-04-21 19:25:50 +00:00
|
|
|
s += ",[";
|
2010-04-19 13:46:58 +00:00
|
|
|
first = true;
|
Eliminate the "store" global variable
Also, move a few free-standing functions into StoreAPI and Derivation.
Also, introduce a non-nullable smart pointer, ref<T>, which is just a
wrapper around std::shared_ptr ensuring that the pointer is never
null. (For reference-counted values, this is better than passing a
"T&", because the latter doesn't maintain the refcount. Usually, the
caller will have a shared_ptr keeping the value alive, but that's not
always the case, e.g., when passing a reference to a std::thread via
std::bind.)
2016-02-04 13:28:26 +00:00
|
|
|
for (auto & i : env) {
|
2010-04-21 19:25:50 +00:00
|
|
|
if (first) first = false; else s += ',';
|
2015-07-17 17:24:28 +00:00
|
|
|
s += '('; printString(s, i.first);
|
2020-01-21 20:14:13 +00:00
|
|
|
s += ','; printString(s, maskOutputs && outputs.count(i.first) ? "" : i.second);
|
2010-04-21 19:25:50 +00:00
|
|
|
s += ')';
|
2010-04-19 13:46:58 +00:00
|
|
|
}
|
2012-07-30 23:55:41 +00:00
|
|
|
|
2010-04-21 19:25:50 +00:00
|
|
|
s += "])";
|
2012-07-30 23:55:41 +00:00
|
|
|
|
2010-04-21 19:25:50 +00:00
|
|
|
return s;
|
2003-07-15 22:28:27 +00:00
|
|
|
}
|
2005-01-19 14:36:00 +00:00
|
|
|
|
|
|
|
|
2019-12-05 18:11:09 +00:00
|
|
|
// FIXME: remove
|
2022-12-07 11:58:58 +00:00
|
|
|
bool isDerivation(std::string_view fileName)
|
2005-01-19 14:36:00 +00:00
|
|
|
{
|
2008-08-25 13:31:57 +00:00
|
|
|
return hasSuffix(fileName, drvExtension);
|
2005-01-19 14:36:00 +00:00
|
|
|
}
|
2006-09-04 21:06:23 +00:00
|
|
|
|
2012-07-30 23:55:41 +00:00
|
|
|
|
2020-08-07 19:09:26 +00:00
|
|
|
std::string outputPathName(std::string_view drvName, std::string_view outputName) {
|
|
|
|
std::string res { drvName };
|
|
|
|
if (outputName != "out") {
|
|
|
|
res += "-";
|
|
|
|
res += outputName;
|
|
|
|
}
|
|
|
|
return res;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2020-03-15 06:23:17 +00:00
|
|
|
DerivationType BasicDerivation::type() const
|
2011-07-20 18:10:47 +00:00
|
|
|
{
|
2022-03-30 14:31:01 +00:00
|
|
|
std::set<std::string_view>
|
|
|
|
inputAddressedOutputs,
|
|
|
|
fixedCAOutputs,
|
|
|
|
floatingCAOutputs,
|
|
|
|
deferredIAOutputs,
|
|
|
|
impureOutputs;
|
2020-07-17 19:55:41 +00:00
|
|
|
std::optional<HashType> floatingHashType;
|
2022-03-30 14:31:01 +00:00
|
|
|
|
2020-07-17 15:33:27 +00:00
|
|
|
for (auto & i : outputs) {
|
|
|
|
std::visit(overloaded {
|
2022-03-17 22:29:15 +00:00
|
|
|
[&](const DerivationOutput::InputAddressed &) {
|
2020-07-17 15:33:27 +00:00
|
|
|
inputAddressedOutputs.insert(i.first);
|
|
|
|
},
|
2022-03-17 22:29:15 +00:00
|
|
|
[&](const DerivationOutput::CAFixed &) {
|
2020-07-17 15:33:27 +00:00
|
|
|
fixedCAOutputs.insert(i.first);
|
|
|
|
},
|
2022-03-17 22:29:15 +00:00
|
|
|
[&](const DerivationOutput::CAFloating & dof) {
|
2020-07-17 19:55:41 +00:00
|
|
|
floatingCAOutputs.insert(i.first);
|
|
|
|
if (!floatingHashType) {
|
|
|
|
floatingHashType = dof.hashType;
|
|
|
|
} else {
|
|
|
|
if (*floatingHashType != dof.hashType)
|
2022-03-30 14:31:01 +00:00
|
|
|
throw Error("all floating outputs must use the same hash type");
|
2020-07-17 19:55:41 +00:00
|
|
|
}
|
2020-07-17 15:33:27 +00:00
|
|
|
},
|
2022-03-17 22:29:15 +00:00
|
|
|
[&](const DerivationOutput::Deferred &) {
|
2022-03-30 14:31:01 +00:00
|
|
|
deferredIAOutputs.insert(i.first);
|
|
|
|
},
|
|
|
|
[&](const DerivationOutput::Impure &) {
|
|
|
|
impureOutputs.insert(i.first);
|
2020-09-23 14:30:42 +00:00
|
|
|
},
|
2022-03-17 22:29:15 +00:00
|
|
|
}, i.second.raw());
|
2020-07-17 15:33:27 +00:00
|
|
|
}
|
|
|
|
|
2022-03-30 14:31:01 +00:00
|
|
|
if (inputAddressedOutputs.empty()
|
|
|
|
&& fixedCAOutputs.empty()
|
|
|
|
&& floatingCAOutputs.empty()
|
|
|
|
&& deferredIAOutputs.empty()
|
|
|
|
&& impureOutputs.empty())
|
|
|
|
throw Error("must have at least one output");
|
|
|
|
|
|
|
|
if (!inputAddressedOutputs.empty()
|
|
|
|
&& fixedCAOutputs.empty()
|
|
|
|
&& floatingCAOutputs.empty()
|
|
|
|
&& deferredIAOutputs.empty()
|
|
|
|
&& impureOutputs.empty())
|
2022-03-18 00:36:52 +00:00
|
|
|
return DerivationType::InputAddressed {
|
|
|
|
.deferred = false,
|
|
|
|
};
|
2022-03-30 14:31:01 +00:00
|
|
|
|
|
|
|
if (inputAddressedOutputs.empty()
|
|
|
|
&& !fixedCAOutputs.empty()
|
|
|
|
&& floatingCAOutputs.empty()
|
|
|
|
&& deferredIAOutputs.empty()
|
|
|
|
&& impureOutputs.empty())
|
|
|
|
{
|
2020-07-17 15:33:27 +00:00
|
|
|
if (fixedCAOutputs.size() > 1)
|
|
|
|
// FIXME: Experimental feature?
|
2022-03-30 14:31:01 +00:00
|
|
|
throw Error("only one fixed output is allowed for now");
|
2020-07-17 15:33:27 +00:00
|
|
|
if (*fixedCAOutputs.begin() != "out")
|
2022-03-30 14:31:01 +00:00
|
|
|
throw Error("single fixed output must be named \"out\"");
|
2022-03-18 00:36:52 +00:00
|
|
|
return DerivationType::ContentAddressed {
|
2022-03-31 14:12:25 +00:00
|
|
|
.sandboxed = false,
|
2022-03-18 00:36:52 +00:00
|
|
|
.fixed = true,
|
|
|
|
};
|
2022-03-30 14:31:01 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if (inputAddressedOutputs.empty()
|
|
|
|
&& fixedCAOutputs.empty()
|
|
|
|
&& !floatingCAOutputs.empty()
|
|
|
|
&& deferredIAOutputs.empty()
|
|
|
|
&& impureOutputs.empty())
|
2022-03-18 00:36:52 +00:00
|
|
|
return DerivationType::ContentAddressed {
|
2022-03-31 14:12:25 +00:00
|
|
|
.sandboxed = true,
|
2022-03-18 00:36:52 +00:00
|
|
|
.fixed = false,
|
|
|
|
};
|
2022-03-30 14:31:01 +00:00
|
|
|
|
|
|
|
if (inputAddressedOutputs.empty()
|
|
|
|
&& fixedCAOutputs.empty()
|
|
|
|
&& floatingCAOutputs.empty()
|
|
|
|
&& !deferredIAOutputs.empty()
|
|
|
|
&& impureOutputs.empty())
|
2022-03-18 00:36:52 +00:00
|
|
|
return DerivationType::InputAddressed {
|
|
|
|
.deferred = true,
|
|
|
|
};
|
2022-03-30 14:31:01 +00:00
|
|
|
|
|
|
|
if (inputAddressedOutputs.empty()
|
|
|
|
&& fixedCAOutputs.empty()
|
|
|
|
&& floatingCAOutputs.empty()
|
|
|
|
&& deferredIAOutputs.empty()
|
|
|
|
&& !impureOutputs.empty())
|
|
|
|
return DerivationType::Impure { };
|
|
|
|
|
|
|
|
throw Error("can't mix derivation output types");
|
2011-07-20 18:10:47 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2020-11-19 16:50:06 +00:00
|
|
|
Sync<DrvHashes> drvHashes;
|
2011-07-20 18:10:47 +00:00
|
|
|
|
2020-03-19 04:37:57 +00:00
|
|
|
/* pathDerivationModulo and hashDerivationModulo are mutually recursive
|
|
|
|
*/
|
2011-07-20 18:10:47 +00:00
|
|
|
|
2020-03-19 04:37:57 +00:00
|
|
|
/* Look up the derivation by value and memoize the
|
|
|
|
`hashDerivationModulo` call.
|
|
|
|
*/
|
2022-03-16 13:21:09 +00:00
|
|
|
static const DrvHash pathDerivationModulo(Store & store, const StorePath & drvPath)
|
2020-03-19 04:37:57 +00:00
|
|
|
{
|
2020-11-19 16:50:06 +00:00
|
|
|
{
|
|
|
|
auto hashes = drvHashes.lock();
|
|
|
|
auto h = hashes->find(drvPath);
|
|
|
|
if (h != hashes->end()) {
|
|
|
|
return h->second;
|
|
|
|
}
|
2020-03-19 04:37:57 +00:00
|
|
|
}
|
2020-11-19 16:50:06 +00:00
|
|
|
auto h = hashDerivationModulo(
|
|
|
|
store,
|
|
|
|
store.readInvalidDerivation(drvPath),
|
|
|
|
false);
|
|
|
|
// Cache it
|
|
|
|
drvHashes.lock()->insert_or_assign(drvPath, h);
|
|
|
|
return h;
|
2020-03-19 04:37:57 +00:00
|
|
|
}
|
2011-07-20 18:10:47 +00:00
|
|
|
|
2020-03-19 04:37:57 +00:00
|
|
|
/* See the header for interface details. These are the implementation details.
|
|
|
|
|
2020-03-20 03:37:52 +00:00
|
|
|
For fixed-output derivations, each hash in the map is not the
|
2020-03-19 04:37:57 +00:00
|
|
|
corresponding output's content hash, but a hash of that hash along
|
|
|
|
with other constant data. The key point is that the value is a pure
|
|
|
|
function of the output's contents, and there are no preimage attacks
|
2020-03-20 03:37:52 +00:00
|
|
|
either spoofing an output's contents for a derivation, or
|
|
|
|
spoofing a derivation for an output's contents.
|
2020-03-19 04:37:57 +00:00
|
|
|
|
|
|
|
For regular derivations, it looks up each subderivation from its hash
|
|
|
|
and recurs. If the subderivation is also regular, it simply
|
|
|
|
substitutes the derivation path with its hash. If the subderivation
|
|
|
|
is fixed-output, however, it takes each output hash and pretends it
|
|
|
|
is a derivation hash producing a single "out" output. This is so we
|
|
|
|
don't leak the provenance of fixed outputs, reducing pointless cache
|
|
|
|
misses as the build itself won't know this.
|
|
|
|
*/
|
2022-03-16 13:21:09 +00:00
|
|
|
DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs)
|
2011-07-20 18:10:47 +00:00
|
|
|
{
|
2022-03-18 00:36:52 +00:00
|
|
|
auto type = drv.type();
|
|
|
|
|
2011-07-20 18:10:47 +00:00
|
|
|
/* Return a fixed hash for fixed-output derivations. */
|
2022-03-18 00:36:52 +00:00
|
|
|
if (type.isFixed()) {
|
2020-03-19 04:37:57 +00:00
|
|
|
std::map<std::string, Hash> outputHashes;
|
|
|
|
for (const auto & i : drv.outputs) {
|
2022-03-17 22:29:15 +00:00
|
|
|
auto & dof = std::get<DerivationOutput::CAFixed>(i.second.raw());
|
2020-07-17 14:28:33 +00:00
|
|
|
auto hash = hashString(htSHA256, "fixed:out:"
|
|
|
|
+ dof.hash.printMethodAlgo() + ":"
|
|
|
|
+ dof.hash.hash.to_string(Base16, false) + ":"
|
2020-08-07 19:09:26 +00:00
|
|
|
+ store.printStorePath(dof.path(store, drv.name, i.first)));
|
2020-07-17 14:28:33 +00:00
|
|
|
outputHashes.insert_or_assign(i.first, std::move(hash));
|
2020-03-19 04:37:57 +00:00
|
|
|
}
|
2022-03-30 14:31:01 +00:00
|
|
|
return DrvHash {
|
2022-03-16 13:21:09 +00:00
|
|
|
.hashes = outputHashes,
|
|
|
|
.kind = DrvHash::Kind::Regular,
|
|
|
|
};
|
2011-07-20 18:10:47 +00:00
|
|
|
}
|
2022-03-18 00:36:52 +00:00
|
|
|
|
2022-03-30 14:31:01 +00:00
|
|
|
if (!type.isPure()) {
|
|
|
|
std::map<std::string, Hash> outputHashes;
|
|
|
|
for (const auto & [outputName, _] : drv.outputs)
|
|
|
|
outputHashes.insert_or_assign(outputName, impureOutputHash);
|
|
|
|
return DrvHash {
|
|
|
|
.hashes = outputHashes,
|
|
|
|
.kind = DrvHash::Kind::Deferred,
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
2022-03-18 00:36:52 +00:00
|
|
|
auto kind = std::visit(overloaded {
|
|
|
|
[](const DerivationType::InputAddressed & ia) {
|
|
|
|
/* This might be a "pesimistically" deferred output, so we don't
|
|
|
|
"taint" the kind yet. */
|
|
|
|
return DrvHash::Kind::Regular;
|
|
|
|
},
|
|
|
|
[](const DerivationType::ContentAddressed & ca) {
|
|
|
|
return ca.fixed
|
|
|
|
? DrvHash::Kind::Regular
|
|
|
|
: DrvHash::Kind::Deferred;
|
|
|
|
},
|
2022-03-30 14:31:01 +00:00
|
|
|
[](const DerivationType::Impure &) -> DrvHash::Kind {
|
|
|
|
assert(false);
|
|
|
|
}
|
2022-03-18 00:36:52 +00:00
|
|
|
}, drv.type().raw());
|
2011-07-20 18:10:47 +00:00
|
|
|
|
2019-12-05 18:11:09 +00:00
|
|
|
std::map<std::string, StringSet> inputs2;
|
2021-10-01 18:05:53 +00:00
|
|
|
for (auto & [drvPath, inputOutputs0] : drv.inputDrvs) {
|
|
|
|
// Avoid lambda capture restriction with standard / Clang
|
|
|
|
auto & inputOutputs = inputOutputs0;
|
|
|
|
const auto & res = pathDerivationModulo(store, drvPath);
|
2022-03-16 13:21:09 +00:00
|
|
|
if (res.kind == DrvHash::Kind::Deferred)
|
|
|
|
kind = DrvHash::Kind::Deferred;
|
|
|
|
for (auto & outputName : inputOutputs) {
|
2022-05-04 05:44:32 +00:00
|
|
|
const auto h = get(res.hashes, outputName);
|
|
|
|
if (!h)
|
|
|
|
throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name);
|
|
|
|
inputs2[h->to_string(Base16, false)].insert(outputName);
|
2022-03-16 13:21:09 +00:00
|
|
|
}
|
2011-07-20 18:10:47 +00:00
|
|
|
}
|
2012-07-30 23:55:41 +00:00
|
|
|
|
2020-12-09 15:56:56 +00:00
|
|
|
auto hash = hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2));
|
|
|
|
|
2022-03-16 13:21:09 +00:00
|
|
|
std::map<std::string, Hash> outputHashes;
|
|
|
|
for (const auto & [outputName, _] : drv.outputs) {
|
|
|
|
outputHashes.insert_or_assign(outputName, hash);
|
2021-10-01 18:05:53 +00:00
|
|
|
}
|
2022-03-16 13:21:09 +00:00
|
|
|
|
|
|
|
return DrvHash {
|
|
|
|
.hashes = outputHashes,
|
|
|
|
.kind = kind,
|
|
|
|
};
|
2020-12-09 15:56:56 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2021-07-16 14:04:47 +00:00
|
|
|
std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation & drv)
|
2020-12-09 15:56:56 +00:00
|
|
|
{
|
2022-03-16 13:21:09 +00:00
|
|
|
return hashDerivationModulo(store, drv, true).hashes;
|
2011-07-20 18:10:47 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2020-03-23 03:43:07 +00:00
|
|
|
static DerivationOutput readDerivationOutput(Source & in, const Store & store)
|
|
|
|
{
|
2020-08-10 01:57:54 +00:00
|
|
|
const auto pathS = readString(in);
|
2020-08-09 20:51:34 +00:00
|
|
|
const auto hashAlgo = readString(in);
|
|
|
|
const auto hash = readString(in);
|
2020-03-23 03:43:07 +00:00
|
|
|
|
2020-08-10 01:57:54 +00:00
|
|
|
return parseDerivationOutput(store, pathS, hashAlgo, hash);
|
2020-03-23 03:43:07 +00:00
|
|
|
}
|
2015-06-10 14:17:06 +00:00
|
|
|
|
2020-06-12 10:46:33 +00:00
|
|
|
StringSet BasicDerivation::outputNames() const
|
|
|
|
{
|
|
|
|
StringSet names;
|
|
|
|
for (auto & i : outputs)
|
|
|
|
names.insert(i.first);
|
|
|
|
return names;
|
|
|
|
}
|
|
|
|
|
2022-03-30 14:31:01 +00:00
|
|
|
DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const
|
|
|
|
{
|
2020-07-27 20:42:02 +00:00
|
|
|
DerivationOutputsAndOptPaths outsAndOptPaths;
|
|
|
|
for (auto output : outputs)
|
|
|
|
outsAndOptPaths.insert(std::make_pair(
|
|
|
|
output.first,
|
2020-09-15 15:21:39 +00:00
|
|
|
std::make_pair(output.second, output.second.path(store, name, output.first))
|
2020-07-27 20:42:02 +00:00
|
|
|
)
|
|
|
|
);
|
|
|
|
return outsAndOptPaths;
|
|
|
|
}
|
2020-06-12 10:46:33 +00:00
|
|
|
|
2022-03-30 14:31:01 +00:00
|
|
|
std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath)
|
|
|
|
{
|
2020-07-12 15:26:30 +00:00
|
|
|
auto nameWithSuffix = drvPath.name();
|
|
|
|
constexpr std::string_view extension = ".drv";
|
|
|
|
assert(hasSuffix(nameWithSuffix, extension));
|
|
|
|
nameWithSuffix.remove_suffix(extension.size());
|
|
|
|
return nameWithSuffix;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2020-07-12 15:02:36 +00:00
|
|
|
Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name)
|
Allow remote builds without sending the derivation closure
Previously, to build a derivation remotely, we had to copy the entire
closure of the .drv file to the remote machine, even though we only
need the top-level derivation. This is very wasteful: the closure can
contain thousands of store paths, and in some Hydra use cases, include
source paths that are very large (e.g. Git/Mercurial checkouts).
So now there is a new operation, StoreAPI::buildDerivation(), that
performs a build from an in-memory representation of a derivation
(BasicDerivation) rather than from a on-disk .drv file. The only files
that need to be in the Nix store are the sources of the derivation
(drv.inputSrcs), and the needed output paths of the dependencies (as
described by drv.inputDrvs). "nix-store --serve" exposes this
interface.
Note that this is a privileged operation, because you can construct a
derivation that builds any store path whatsoever. Fixing this will
require changing the hashing scheme (i.e., the output paths should be
computed from the other fields in BasicDerivation, allowing them to be
verified without access to other derivations). However, this would be
quite nice because it would allow .drv-free building (e.g. "nix-env
-i" wouldn't have to write any .drv files to disk).
Fixes #173.
2015-07-17 15:57:40 +00:00
|
|
|
{
|
2020-07-08 19:38:01 +00:00
|
|
|
drv.name = name;
|
|
|
|
|
Allow remote builds without sending the derivation closure
Previously, to build a derivation remotely, we had to copy the entire
closure of the .drv file to the remote machine, even though we only
need the top-level derivation. This is very wasteful: the closure can
contain thousands of store paths, and in some Hydra use cases, include
source paths that are very large (e.g. Git/Mercurial checkouts).
So now there is a new operation, StoreAPI::buildDerivation(), that
performs a build from an in-memory representation of a derivation
(BasicDerivation) rather than from a on-disk .drv file. The only files
that need to be in the Nix store are the sources of the derivation
(drv.inputSrcs), and the needed output paths of the dependencies (as
described by drv.inputDrvs). "nix-store --serve" exposes this
interface.
Note that this is a privileged operation, because you can construct a
derivation that builds any store path whatsoever. Fixing this will
require changing the hashing scheme (i.e., the output paths should be
computed from the other fields in BasicDerivation, allowing them to be
verified without access to other derivations). However, this would be
quite nice because it would allow .drv-free building (e.g. "nix-env
-i" wouldn't have to write any .drv files to disk).
Fixes #173.
2015-07-17 15:57:40 +00:00
|
|
|
drv.outputs.clear();
|
2017-03-01 12:52:54 +00:00
|
|
|
auto nr = readNum<size_t>(in);
|
|
|
|
for (size_t n = 0; n < nr; n++) {
|
Allow remote builds without sending the derivation closure
Previously, to build a derivation remotely, we had to copy the entire
closure of the .drv file to the remote machine, even though we only
need the top-level derivation. This is very wasteful: the closure can
contain thousands of store paths, and in some Hydra use cases, include
source paths that are very large (e.g. Git/Mercurial checkouts).
So now there is a new operation, StoreAPI::buildDerivation(), that
performs a build from an in-memory representation of a derivation
(BasicDerivation) rather than from a on-disk .drv file. The only files
that need to be in the Nix store are the sources of the derivation
(drv.inputSrcs), and the needed output paths of the dependencies (as
described by drv.inputDrvs). "nix-store --serve" exposes this
interface.
Note that this is a privileged operation, because you can construct a
derivation that builds any store path whatsoever. Fixing this will
require changing the hashing scheme (i.e., the output paths should be
computed from the other fields in BasicDerivation, allowing them to be
verified without access to other derivations). However, this would be
quite nice because it would allow .drv-free building (e.g. "nix-env
-i" wouldn't have to write any .drv files to disk).
Fixes #173.
2015-07-17 15:57:40 +00:00
|
|
|
auto name = readString(in);
|
2020-03-23 03:43:07 +00:00
|
|
|
auto output = readDerivationOutput(in, store);
|
2020-06-17 04:55:47 +00:00
|
|
|
drv.outputs.emplace(std::move(name), std::move(output));
|
Allow remote builds without sending the derivation closure
Previously, to build a derivation remotely, we had to copy the entire
closure of the .drv file to the remote machine, even though we only
need the top-level derivation. This is very wasteful: the closure can
contain thousands of store paths, and in some Hydra use cases, include
source paths that are very large (e.g. Git/Mercurial checkouts).
So now there is a new operation, StoreAPI::buildDerivation(), that
performs a build from an in-memory representation of a derivation
(BasicDerivation) rather than from a on-disk .drv file. The only files
that need to be in the Nix store are the sources of the derivation
(drv.inputSrcs), and the needed output paths of the dependencies (as
described by drv.inputDrvs). "nix-store --serve" exposes this
interface.
Note that this is a privileged operation, because you can construct a
derivation that builds any store path whatsoever. Fixing this will
require changing the hashing scheme (i.e., the output paths should be
computed from the other fields in BasicDerivation, allowing them to be
verified without access to other derivations). However, this would be
quite nice because it would allow .drv-free building (e.g. "nix-env
-i" wouldn't have to write any .drv files to disk).
Fixes #173.
2015-07-17 15:57:40 +00:00
|
|
|
}
|
|
|
|
|
2020-09-30 00:41:18 +00:00
|
|
|
drv.inputSrcs = worker_proto::read(store, in, Phantom<StorePathSet> {});
|
Allow remote builds without sending the derivation closure
Previously, to build a derivation remotely, we had to copy the entire
closure of the .drv file to the remote machine, even though we only
need the top-level derivation. This is very wasteful: the closure can
contain thousands of store paths, and in some Hydra use cases, include
source paths that are very large (e.g. Git/Mercurial checkouts).
So now there is a new operation, StoreAPI::buildDerivation(), that
performs a build from an in-memory representation of a derivation
(BasicDerivation) rather than from a on-disk .drv file. The only files
that need to be in the Nix store are the sources of the derivation
(drv.inputSrcs), and the needed output paths of the dependencies (as
described by drv.inputDrvs). "nix-store --serve" exposes this
interface.
Note that this is a privileged operation, because you can construct a
derivation that builds any store path whatsoever. Fixing this will
require changing the hashing scheme (i.e., the output paths should be
computed from the other fields in BasicDerivation, allowing them to be
verified without access to other derivations). However, this would be
quite nice because it would allow .drv-free building (e.g. "nix-env
-i" wouldn't have to write any .drv files to disk).
Fixes #173.
2015-07-17 15:57:40 +00:00
|
|
|
in >> drv.platform >> drv.builder;
|
|
|
|
drv.args = readStrings<Strings>(in);
|
|
|
|
|
2017-03-01 12:52:54 +00:00
|
|
|
nr = readNum<size_t>(in);
|
|
|
|
for (size_t n = 0; n < nr; n++) {
|
Allow remote builds without sending the derivation closure
Previously, to build a derivation remotely, we had to copy the entire
closure of the .drv file to the remote machine, even though we only
need the top-level derivation. This is very wasteful: the closure can
contain thousands of store paths, and in some Hydra use cases, include
source paths that are very large (e.g. Git/Mercurial checkouts).
So now there is a new operation, StoreAPI::buildDerivation(), that
performs a build from an in-memory representation of a derivation
(BasicDerivation) rather than from a on-disk .drv file. The only files
that need to be in the Nix store are the sources of the derivation
(drv.inputSrcs), and the needed output paths of the dependencies (as
described by drv.inputDrvs). "nix-store --serve" exposes this
interface.
Note that this is a privileged operation, because you can construct a
derivation that builds any store path whatsoever. Fixing this will
require changing the hashing scheme (i.e., the output paths should be
computed from the other fields in BasicDerivation, allowing them to be
verified without access to other derivations). However, this would be
quite nice because it would allow .drv-free building (e.g. "nix-env
-i" wouldn't have to write any .drv files to disk).
Fixes #173.
2015-07-17 15:57:40 +00:00
|
|
|
auto key = readString(in);
|
|
|
|
auto value = readString(in);
|
|
|
|
drv.env[key] = value;
|
|
|
|
}
|
|
|
|
|
|
|
|
return in;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2019-12-05 18:11:09 +00:00
|
|
|
void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv)
|
Allow remote builds without sending the derivation closure
Previously, to build a derivation remotely, we had to copy the entire
closure of the .drv file to the remote machine, even though we only
need the top-level derivation. This is very wasteful: the closure can
contain thousands of store paths, and in some Hydra use cases, include
source paths that are very large (e.g. Git/Mercurial checkouts).
So now there is a new operation, StoreAPI::buildDerivation(), that
performs a build from an in-memory representation of a derivation
(BasicDerivation) rather than from a on-disk .drv file. The only files
that need to be in the Nix store are the sources of the derivation
(drv.inputSrcs), and the needed output paths of the dependencies (as
described by drv.inputDrvs). "nix-store --serve" exposes this
interface.
Note that this is a privileged operation, because you can construct a
derivation that builds any store path whatsoever. Fixing this will
require changing the hashing scheme (i.e., the output paths should be
computed from the other fields in BasicDerivation, allowing them to be
verified without access to other derivations). However, this would be
quite nice because it would allow .drv-free building (e.g. "nix-env
-i" wouldn't have to write any .drv files to disk).
Fixes #173.
2015-07-17 15:57:40 +00:00
|
|
|
{
|
|
|
|
out << drv.outputs.size();
|
2020-06-25 13:50:30 +00:00
|
|
|
for (auto & i : drv.outputs) {
|
2020-08-07 19:09:26 +00:00
|
|
|
out << i.first;
|
2020-07-12 16:12:21 +00:00
|
|
|
std::visit(overloaded {
|
2022-03-17 22:29:15 +00:00
|
|
|
[&](const DerivationOutput::InputAddressed & doi) {
|
2020-08-07 19:09:26 +00:00
|
|
|
out << store.printStorePath(doi.path)
|
|
|
|
<< ""
|
|
|
|
<< "";
|
2020-07-12 16:12:21 +00:00
|
|
|
},
|
2022-03-17 22:29:15 +00:00
|
|
|
[&](const DerivationOutput::CAFixed & dof) {
|
2020-08-07 19:09:26 +00:00
|
|
|
out << store.printStorePath(dof.path(store, drv.name, i.first))
|
|
|
|
<< dof.hash.printMethodAlgo()
|
2020-07-12 16:12:21 +00:00
|
|
|
<< dof.hash.hash.to_string(Base16, false);
|
|
|
|
},
|
2022-03-17 22:29:15 +00:00
|
|
|
[&](const DerivationOutput::CAFloating & dof) {
|
2020-08-07 19:09:26 +00:00
|
|
|
out << ""
|
|
|
|
<< (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType))
|
2020-07-12 16:12:21 +00:00
|
|
|
<< "";
|
|
|
|
},
|
2022-03-17 22:29:15 +00:00
|
|
|
[&](const DerivationOutput::Deferred &) {
|
2020-09-23 14:30:42 +00:00
|
|
|
out << ""
|
|
|
|
<< ""
|
|
|
|
<< "";
|
|
|
|
},
|
2022-03-30 14:31:01 +00:00
|
|
|
[&](const DerivationOutput::Impure & doi) {
|
|
|
|
out << ""
|
|
|
|
<< (makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType))
|
|
|
|
<< "impure";
|
|
|
|
},
|
2022-03-17 22:29:15 +00:00
|
|
|
}, i.second.raw());
|
2020-06-25 13:50:30 +00:00
|
|
|
}
|
2020-09-30 00:41:18 +00:00
|
|
|
worker_proto::write(store, out, drv.inputSrcs);
|
2019-12-05 18:11:09 +00:00
|
|
|
out << drv.platform << drv.builder << drv.args;
|
Allow remote builds without sending the derivation closure
Previously, to build a derivation remotely, we had to copy the entire
closure of the .drv file to the remote machine, even though we only
need the top-level derivation. This is very wasteful: the closure can
contain thousands of store paths, and in some Hydra use cases, include
source paths that are very large (e.g. Git/Mercurial checkouts).
So now there is a new operation, StoreAPI::buildDerivation(), that
performs a build from an in-memory representation of a derivation
(BasicDerivation) rather than from a on-disk .drv file. The only files
that need to be in the Nix store are the sources of the derivation
(drv.inputSrcs), and the needed output paths of the dependencies (as
described by drv.inputDrvs). "nix-store --serve" exposes this
interface.
Note that this is a privileged operation, because you can construct a
derivation that builds any store path whatsoever. Fixing this will
require changing the hashing scheme (i.e., the output paths should be
computed from the other fields in BasicDerivation, allowing them to be
verified without access to other derivations). However, this would be
quite nice because it would allow .drv-free building (e.g. "nix-env
-i" wouldn't have to write any .drv files to disk).
Fixes #173.
2015-07-17 15:57:40 +00:00
|
|
|
out << drv.env.size();
|
|
|
|
for (auto & i : drv.env)
|
|
|
|
out << i.first << i.second;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2022-01-21 13:44:00 +00:00
|
|
|
std::string hashPlaceholder(const std::string_view outputName)
|
2016-08-17 13:12:54 +00:00
|
|
|
{
|
|
|
|
// FIXME: memoize?
|
2022-01-21 13:44:00 +00:00
|
|
|
return "/" + hashString(htSHA256, concatStrings("nix-output:", outputName)).to_string(Base32, false);
|
2016-08-17 13:12:54 +00:00
|
|
|
}
|
|
|
|
|
2020-08-21 19:35:35 +00:00
|
|
|
std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName)
|
2020-08-07 19:09:26 +00:00
|
|
|
{
|
|
|
|
auto drvNameWithExtension = drvPath.name();
|
|
|
|
auto drvName = drvNameWithExtension.substr(0, drvNameWithExtension.size() - 4);
|
2020-08-21 19:35:35 +00:00
|
|
|
auto clearText = "nix-upstream-output:" + std::string { drvPath.hashPart() } + ":" + outputPathName(drvName, outputName);
|
|
|
|
return "/" + hashString(htSHA256, clearText).to_string(Base32, false);
|
2020-08-07 19:09:26 +00:00
|
|
|
}
|
2016-08-17 13:12:54 +00:00
|
|
|
|
2020-08-22 20:44:47 +00:00
|
|
|
|
2022-03-30 14:31:01 +00:00
|
|
|
static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites)
|
|
|
|
{
|
|
|
|
for (auto & rewrite : rewrites) {
|
2020-08-22 20:44:47 +00:00
|
|
|
debug("rewriting %s as %s", rewrite.first, rewrite.second);
|
|
|
|
}
|
|
|
|
|
|
|
|
drv.builder = rewriteStrings(drv.builder, rewrites);
|
2022-03-30 14:31:01 +00:00
|
|
|
for (auto & arg : drv.args) {
|
2020-08-22 20:44:47 +00:00
|
|
|
arg = rewriteStrings(arg, rewrites);
|
|
|
|
}
|
|
|
|
|
|
|
|
StringPairs newEnv;
|
2022-03-30 14:31:01 +00:00
|
|
|
for (auto & envVar : drv.env) {
|
2020-08-22 20:44:47 +00:00
|
|
|
auto envName = rewriteStrings(envVar.first, rewrites);
|
|
|
|
auto envValue = rewriteStrings(envVar.second, rewrites);
|
|
|
|
newEnv.emplace(envName, envValue);
|
|
|
|
}
|
|
|
|
drv.env = newEnv;
|
2020-09-23 14:30:42 +00:00
|
|
|
|
|
|
|
auto hashModulo = hashDerivationModulo(store, Derivation(drv), true);
|
|
|
|
for (auto & [outputName, output] : drv.outputs) {
|
2022-03-17 22:29:15 +00:00
|
|
|
if (std::holds_alternative<DerivationOutput::Deferred>(output.raw())) {
|
2022-05-04 05:44:32 +00:00
|
|
|
auto h = get(hashModulo.hashes, outputName);
|
|
|
|
if (!h)
|
|
|
|
throw Error("derivation '%s' output '%s' has no hash (derivations.cc/rewriteDerivation)",
|
|
|
|
drv.name, outputName);
|
|
|
|
auto outPath = store.makeOutputPath(outputName, *h, drv.name);
|
2020-09-23 14:30:42 +00:00
|
|
|
drv.env[outputName] = store.printStorePath(outPath);
|
2022-03-17 22:29:15 +00:00
|
|
|
output = DerivationOutput::InputAddressed {
|
|
|
|
.path = std::move(outPath),
|
2020-09-23 14:30:42 +00:00
|
|
|
};
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-08-22 20:44:47 +00:00
|
|
|
}
|
|
|
|
|
2022-03-30 14:31:01 +00:00
|
|
|
std::optional<BasicDerivation> Derivation::tryResolve(Store & store) const
|
2021-10-01 18:05:53 +00:00
|
|
|
{
|
2022-03-30 14:31:01 +00:00
|
|
|
std::map<std::pair<StorePath, std::string>, StorePath> inputDrvOutputs;
|
2021-10-01 18:05:53 +00:00
|
|
|
|
2022-03-30 14:31:01 +00:00
|
|
|
for (auto & input : inputDrvs)
|
|
|
|
for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(input.first))
|
|
|
|
if (outputPath)
|
|
|
|
inputDrvOutputs.insert_or_assign({input.first, outputName}, *outputPath);
|
2021-10-01 18:05:53 +00:00
|
|
|
|
2022-03-30 14:31:01 +00:00
|
|
|
return tryResolve(store, inputDrvOutputs);
|
2021-10-01 18:05:53 +00:00
|
|
|
}
|
|
|
|
|
2022-03-30 14:31:01 +00:00
|
|
|
std::optional<BasicDerivation> Derivation::tryResolve(
|
|
|
|
Store & store,
|
|
|
|
const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs) const
|
|
|
|
{
|
2020-08-22 20:44:47 +00:00
|
|
|
BasicDerivation resolved { *this };
|
|
|
|
|
|
|
|
// Input paths that we'll want to rewrite in the derivation
|
|
|
|
StringMap inputRewrites;
|
|
|
|
|
2022-03-30 14:31:01 +00:00
|
|
|
for (auto & [inputDrv, inputOutputs] : inputDrvs) {
|
|
|
|
for (auto & outputName : inputOutputs) {
|
|
|
|
if (auto actualPath = get(inputDrvOutputs, { inputDrv, outputName })) {
|
|
|
|
inputRewrites.emplace(
|
|
|
|
downstreamPlaceholder(store, inputDrv, outputName),
|
|
|
|
store.printStorePath(*actualPath));
|
|
|
|
resolved.inputSrcs.insert(*actualPath);
|
|
|
|
} else {
|
|
|
|
warn("output '%s' of input '%s' missing, aborting the resolving",
|
|
|
|
outputName,
|
|
|
|
store.printStorePath(inputDrv));
|
|
|
|
return {};
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2020-08-22 20:44:47 +00:00
|
|
|
|
|
|
|
rewriteDerivation(store, resolved, inputRewrites);
|
|
|
|
|
|
|
|
return resolved;
|
|
|
|
}
|
|
|
|
|
2023-02-17 23:37:35 +00:00
|
|
|
|
2023-03-01 21:57:36 +00:00
|
|
|
void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const
|
|
|
|
{
|
|
|
|
assert(drvPath.isDerivation());
|
|
|
|
std::string drvName(drvPath.name());
|
|
|
|
drvName = drvName.substr(0, drvName.size() - drvExtension.size());
|
|
|
|
|
|
|
|
if (drvName != name) {
|
|
|
|
throw Error("Derivation '%s' has name '%s' which does not match its path", store.printStorePath(drvPath), name);
|
|
|
|
}
|
|
|
|
|
|
|
|
auto envHasRightPath = [&](const StorePath & actual, const std::string & varName)
|
|
|
|
{
|
|
|
|
auto j = env.find(varName);
|
|
|
|
if (j == env.end() || store.parseStorePath(j->second) != actual)
|
|
|
|
throw Error("derivation '%s' has incorrect environment variable '%s', should be '%s'",
|
|
|
|
store.printStorePath(drvPath), varName, store.printStorePath(actual));
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
// Don't need the answer, but do this anyways to assert is proper
|
|
|
|
// combination. The code below is more general and naturally allows
|
|
|
|
// combinations that are currently prohibited.
|
|
|
|
type();
|
|
|
|
|
|
|
|
std::optional<DrvHash> hashesModulo;
|
|
|
|
for (auto & i : outputs) {
|
|
|
|
std::visit(overloaded {
|
|
|
|
[&](const DerivationOutput::InputAddressed & doia) {
|
|
|
|
if (!hashesModulo) {
|
|
|
|
// somewhat expensive so we do lazily
|
|
|
|
hashesModulo = hashDerivationModulo(store, *this, true);
|
|
|
|
}
|
|
|
|
auto currentOutputHash = get(hashesModulo->hashes, i.first);
|
|
|
|
if (!currentOutputHash)
|
|
|
|
throw Error("derivation '%s' has unexpected output '%s' (local-store / hashesModulo) named '%s'",
|
|
|
|
store.printStorePath(drvPath), store.printStorePath(doia.path), i.first);
|
|
|
|
StorePath recomputed = store.makeOutputPath(i.first, *currentOutputHash, drvName);
|
|
|
|
if (doia.path != recomputed)
|
|
|
|
throw Error("derivation '%s' has incorrect output '%s', should be '%s'",
|
|
|
|
store.printStorePath(drvPath), store.printStorePath(doia.path), store.printStorePath(recomputed));
|
|
|
|
envHasRightPath(doia.path, i.first);
|
|
|
|
},
|
|
|
|
[&](const DerivationOutput::CAFixed & dof) {
|
2023-04-08 00:39:04 +00:00
|
|
|
StorePath path = store.makeFixedOutputPath(drvName, { dof.hash, {} });
|
2023-03-01 21:57:36 +00:00
|
|
|
envHasRightPath(path, i.first);
|
|
|
|
},
|
|
|
|
[&](const DerivationOutput::CAFloating &) {
|
|
|
|
/* Nothing to check */
|
|
|
|
},
|
|
|
|
[&](const DerivationOutput::Deferred &) {
|
|
|
|
/* Nothing to check */
|
|
|
|
},
|
|
|
|
[&](const DerivationOutput::Impure &) {
|
|
|
|
/* Nothing to check */
|
|
|
|
},
|
|
|
|
}, i.second.raw());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2022-03-30 14:31:01 +00:00
|
|
|
const Hash impureOutputHash = hashString(htSHA256, "impure");
|
|
|
|
|
2023-02-17 23:37:35 +00:00
|
|
|
nlohmann::json DerivationOutput::toJSON(
|
|
|
|
const Store & store, std::string_view drvName, std::string_view outputName) const
|
|
|
|
{
|
|
|
|
nlohmann::json res = nlohmann::json::object();
|
|
|
|
std::visit(overloaded {
|
|
|
|
[&](const DerivationOutput::InputAddressed & doi) {
|
|
|
|
res["path"] = store.printStorePath(doi.path);
|
|
|
|
},
|
|
|
|
[&](const DerivationOutput::CAFixed & dof) {
|
|
|
|
res["path"] = store.printStorePath(dof.path(store, drvName, outputName));
|
|
|
|
res["hashAlgo"] = dof.hash.printMethodAlgo();
|
|
|
|
res["hash"] = dof.hash.hash.to_string(Base16, false);
|
|
|
|
},
|
|
|
|
[&](const DerivationOutput::CAFloating & dof) {
|
|
|
|
res["hashAlgo"] = makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType);
|
|
|
|
},
|
|
|
|
[&](const DerivationOutput::Deferred &) {},
|
|
|
|
[&](const DerivationOutput::Impure & doi) {
|
|
|
|
res["hashAlgo"] = makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType);
|
|
|
|
res["impure"] = true;
|
|
|
|
},
|
|
|
|
}, raw());
|
|
|
|
return res;
|
|
|
|
}
|
|
|
|
|
2023-02-17 23:37:35 +00:00
|
|
|
|
|
|
|
DerivationOutput DerivationOutput::fromJSON(
|
|
|
|
const Store & store, std::string_view drvName, std::string_view outputName,
|
2023-04-17 15:22:31 +00:00
|
|
|
const nlohmann::json & _json,
|
|
|
|
const ExperimentalFeatureSettings & xpSettings)
|
2023-02-17 23:37:35 +00:00
|
|
|
{
|
|
|
|
std::set<std::string_view> keys;
|
|
|
|
auto json = (std::map<std::string, nlohmann::json>) _json;
|
|
|
|
|
|
|
|
for (const auto & [key, _] : json)
|
|
|
|
keys.insert(key);
|
|
|
|
|
|
|
|
auto methodAlgo = [&]() -> std::pair<FileIngestionMethod, HashType> {
|
|
|
|
std::string hashAlgo = json["hashAlgo"];
|
|
|
|
auto method = FileIngestionMethod::Flat;
|
|
|
|
if (hashAlgo.substr(0, 2) == "r:") {
|
|
|
|
method = FileIngestionMethod::Recursive;
|
|
|
|
hashAlgo = hashAlgo.substr(2);
|
|
|
|
}
|
|
|
|
auto hashType = parseHashType(hashAlgo);
|
|
|
|
return { method, hashType };
|
|
|
|
};
|
|
|
|
|
|
|
|
if (keys == (std::set<std::string_view> { "path" })) {
|
|
|
|
return DerivationOutput::InputAddressed {
|
|
|
|
.path = store.parseStorePath((std::string) json["path"]),
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
|
|
|
else if (keys == (std::set<std::string_view> { "path", "hashAlgo", "hash" })) {
|
|
|
|
auto [method, hashType] = methodAlgo();
|
|
|
|
auto dof = DerivationOutput::CAFixed {
|
|
|
|
.hash = {
|
|
|
|
.method = method,
|
|
|
|
.hash = Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashType),
|
|
|
|
},
|
|
|
|
};
|
|
|
|
if (dof.path(store, drvName, outputName) != store.parseStorePath((std::string) json["path"]))
|
|
|
|
throw Error("Path doesn't match derivation output");
|
|
|
|
return dof;
|
|
|
|
}
|
|
|
|
|
|
|
|
else if (keys == (std::set<std::string_view> { "hashAlgo" })) {
|
2023-04-17 15:22:31 +00:00
|
|
|
xpSettings.require(Xp::CaDerivations);
|
2023-02-17 23:37:35 +00:00
|
|
|
auto [method, hashType] = methodAlgo();
|
|
|
|
return DerivationOutput::CAFloating {
|
|
|
|
.method = method,
|
|
|
|
.hashType = hashType,
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
|
|
|
else if (keys == (std::set<std::string_view> { })) {
|
|
|
|
return DerivationOutput::Deferred {};
|
|
|
|
}
|
|
|
|
|
|
|
|
else if (keys == (std::set<std::string_view> { "hashAlgo", "impure" })) {
|
2023-04-17 15:22:31 +00:00
|
|
|
xpSettings.require(Xp::ImpureDerivations);
|
2023-02-17 23:37:35 +00:00
|
|
|
auto [method, hashType] = methodAlgo();
|
|
|
|
return DerivationOutput::Impure {
|
|
|
|
.method = method,
|
|
|
|
.hashType = hashType,
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
|
|
|
else {
|
|
|
|
throw Error("invalid JSON for derivation output");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2023-02-17 23:37:35 +00:00
|
|
|
nlohmann::json Derivation::toJSON(const Store & store) const
|
|
|
|
{
|
|
|
|
nlohmann::json res = nlohmann::json::object();
|
|
|
|
|
2023-03-30 15:06:52 +00:00
|
|
|
res["name"] = name;
|
|
|
|
|
2023-02-17 23:37:35 +00:00
|
|
|
{
|
|
|
|
nlohmann::json & outputsObj = res["outputs"];
|
|
|
|
outputsObj = nlohmann::json::object();
|
|
|
|
for (auto & [outputName, output] : outputs) {
|
|
|
|
outputsObj[outputName] = output.toJSON(store, name, outputName);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
{
|
|
|
|
auto& inputsList = res["inputSrcs"];
|
|
|
|
inputsList = nlohmann::json ::array();
|
|
|
|
for (auto & input : inputSrcs)
|
|
|
|
inputsList.emplace_back(store.printStorePath(input));
|
|
|
|
}
|
|
|
|
|
|
|
|
{
|
|
|
|
auto& inputDrvsObj = res["inputDrvs"];
|
|
|
|
inputDrvsObj = nlohmann::json ::object();
|
|
|
|
for (auto & input : inputDrvs)
|
|
|
|
inputDrvsObj[store.printStorePath(input.first)] = input.second;
|
|
|
|
}
|
|
|
|
|
|
|
|
res["system"] = platform;
|
|
|
|
res["builder"] = builder;
|
|
|
|
res["args"] = args;
|
2023-02-20 22:32:19 +00:00
|
|
|
res["env"] = env;
|
2023-02-17 23:37:35 +00:00
|
|
|
|
|
|
|
return res;
|
|
|
|
}
|
|
|
|
|
2023-02-17 23:37:35 +00:00
|
|
|
|
|
|
|
Derivation Derivation::fromJSON(
|
2023-03-30 15:06:52 +00:00
|
|
|
const Store & store,
|
2023-02-17 23:37:35 +00:00
|
|
|
const nlohmann::json & json)
|
|
|
|
{
|
|
|
|
Derivation res;
|
|
|
|
|
2023-03-30 15:06:52 +00:00
|
|
|
res.name = json["name"];
|
|
|
|
|
2023-02-17 23:37:35 +00:00
|
|
|
{
|
|
|
|
auto & outputsObj = json["outputs"];
|
|
|
|
for (auto & [outputName, output] : outputsObj.items()) {
|
|
|
|
res.outputs.insert_or_assign(
|
|
|
|
outputName,
|
2023-03-30 15:06:52 +00:00
|
|
|
DerivationOutput::fromJSON(store, res.name, outputName, output));
|
2023-02-17 23:37:35 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
{
|
|
|
|
auto & inputsList = json["inputSrcs"];
|
|
|
|
for (auto & input : inputsList)
|
|
|
|
res.inputSrcs.insert(store.parseStorePath(static_cast<const std::string &>(input)));
|
|
|
|
}
|
|
|
|
|
|
|
|
{
|
|
|
|
auto & inputDrvsObj = json["inputDrvs"];
|
|
|
|
for (auto & [inputDrvPath, inputOutputs] : inputDrvsObj.items())
|
|
|
|
res.inputDrvs[store.parseStorePath(inputDrvPath)] =
|
|
|
|
static_cast<const StringSet &>(inputOutputs);
|
|
|
|
}
|
|
|
|
|
|
|
|
res.platform = json["system"];
|
|
|
|
res.builder = json["builder"];
|
|
|
|
res.args = json["args"];
|
|
|
|
res.env = json["env"];
|
|
|
|
|
|
|
|
return res;
|
|
|
|
}
|
|
|
|
|
2006-09-04 21:06:23 +00:00
|
|
|
}
|