diff --git a/.version b/.version
index 7208c2182..f398a2061 100644
--- a/.version
+++ b/.version
@@ -1 +1 @@
-2.4
\ No newline at end of file
+3.0
\ No newline at end of file
diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index 1fa74a143..9c55526a3 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -370,34 +370,6 @@ false.
- hashed-mirrors
-
- A list of web servers used by
- builtins.fetchurl to obtain files by
- hash. The default is
- http://tarballs.nixos.org/. Given a hash type
- ht and a base-16 hash
- h, Nix will try to download the file
- from
- hashed-mirror/ht/h.
- This allows files to be downloaded even if they have disappeared
- from their original URI. For example, given the default mirror
- http://tarballs.nixos.org/, when building the derivation
-
-
-builtins.fetchurl {
- url = "https://example.org/foo-1.2.3.tar.xz";
- sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae";
-}
-
-
- Nix will attempt to download this file from
- http://tarballs.nixos.org/sha256/2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae
- first. If it is not available there, if will try the original URI.
-
-
-
-
http-connections
The maximum number of parallel TCP connections
diff --git a/mk/precompiled-headers.mk b/mk/precompiled-headers.mk
index 1c0452dc2..500c99e4a 100644
--- a/mk/precompiled-headers.mk
+++ b/mk/precompiled-headers.mk
@@ -21,13 +21,13 @@ clean-files += $(GCH) $(PCH)
ifeq ($(PRECOMPILE_HEADERS), 1)
- ifeq ($(CXX), g++)
+ ifeq ($(findstring g++,$(CXX)), g++)
GLOBAL_CXXFLAGS_PCH += -include $(buildprefix)precompiled-headers.h -Winvalid-pch
GLOBAL_ORDER_AFTER += $(GCH)
- else ifeq ($(CXX), clang++)
+ else ifeq ($(findstring clang++,$(CXX)), clang++)
GLOBAL_CXXFLAGS_PCH += -include-pch $(PCH) -Winvalid-pch
diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs
index f14c3f73f..c412e4640 100644
--- a/perl/lib/Nix/Store.xs
+++ b/perl/lib/Nix/Store.xs
@@ -80,7 +80,7 @@ SV * queryReferences(char * path)
SV * queryPathHash(char * path)
PPCODE:
try {
- auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(Base32, true);
+ auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash->to_string(Base32, true);
XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
} catch (Error & e) {
croak("%s", e.what());
@@ -106,7 +106,7 @@ SV * queryPathInfo(char * path, int base32)
XPUSHs(&PL_sv_undef);
else
XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0)));
- auto s = info->narHash.to_string(base32 ? Base32 : Base16, true);
+ auto s = info->narHash->to_string(base32 ? Base32 : Base16, true);
XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
mXPUSHi(info->registrationTime);
mXPUSHi(info->narSize);
@@ -304,7 +304,10 @@ SV * derivationFromPath(char * drvPath)
HV * outputs = newHV();
for (auto & i : drv.outputs)
- hv_store(outputs, i.first.c_str(), i.first.size(), newSVpv(store()->printStorePath(i.second.path).c_str(), 0), 0);
+ hv_store(
+ outputs, i.first.c_str(), i.first.size(),
+ newSVpv(store()->printStorePath(i.second.path(*store(), drv.name)).c_str(), 0),
+ 0);
hv_stores(hash, "outputs", newRV((SV *) outputs));
AV * inputDrvs = newAV();
diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh
index 00c9d540b..e5cc4d7ed 100644
--- a/scripts/install-multi-user.sh
+++ b/scripts/install-multi-user.sh
@@ -37,6 +37,8 @@ readonly PROFILE_NIX_FILE="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix-
readonly NIX_INSTALLED_NIX="@nix@"
readonly NIX_INSTALLED_CACERT="@cacert@"
+#readonly NIX_INSTALLED_NIX="/nix/store/j8dbv5w6jl34caywh2ygdy88knx1mdf7-nix-2.3.6"
+#readonly NIX_INSTALLED_CACERT="/nix/store/7dxhzymvy330i28ii676fl1pqwcahv2f-nss-cacert-3.49.2"
readonly EXTRACTED_NIX_PATH="$(dirname "$0")"
readonly ROOT_HOME=$(echo ~root)
@@ -69,9 +71,11 @@ uninstall_directions() {
subheader "Uninstalling nix:"
local step=0
- if poly_service_installed_check; then
+ if [ -e /run/systemd/system ] && poly_service_installed_check; then
step=$((step + 1))
poly_service_uninstall_directions "$step"
+ else
+ step=$((step + 1))
fi
for profile_target in "${PROFILE_TARGETS[@]}"; do
@@ -250,7 +254,9 @@ function finish_success {
echo "But fetching the nixpkgs channel failed. (Are you offline?)"
echo "To try again later, run \"sudo -i nix-channel --update nixpkgs\"."
fi
- cat <&2
-elif [ "$(uname -s)" = "Linux" ] && [ -e /run/systemd/system ]; then
+elif [ "$(uname -s)" = "Linux" ]; then
echo "Note: a multi-user installation is possible. See https://nixos.org/nix/manual/#sect-multi-user-installation" >&2
fi
@@ -122,7 +122,7 @@ if [ "$(uname -s)" = "Darwin" ]; then
fi
if [ "$INSTALL_MODE" = "daemon" ]; then
- printf '\e[1;31mSwitching to the Daemon-based Installer\e[0m\n'
+ printf '\e[1;31mSwitching to the Multi-user Installer\e[0m\n'
exec "$self/install-multi-user"
exit 0
fi
diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index e07117496..3579d8fff 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -33,7 +33,7 @@ std::string escapeUri(std::string uri)
static string currentLoad;
-static AutoCloseFD openSlotLock(const Machine & m, unsigned long long slot)
+static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot)
{
return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true);
}
@@ -119,7 +119,7 @@ static int _main(int argc, char * * argv)
bool rightType = false;
Machine * bestMachine = nullptr;
- unsigned long long bestLoad = 0;
+ uint64_t bestLoad = 0;
for (auto & m : machines) {
debug("considering building on remote machine '%s'", m.storeUri);
@@ -130,8 +130,8 @@ static int _main(int argc, char * * argv)
m.mandatoryMet(requiredFeatures)) {
rightType = true;
AutoCloseFD free;
- unsigned long long load = 0;
- for (unsigned long long slot = 0; slot < m.maxJobs; ++slot) {
+ uint64_t load = 0;
+ for (uint64_t slot = 0; slot < m.maxJobs; ++slot) {
auto slotLock = openSlotLock(m, slot);
if (lockFile(slotLock.get(), ltWrite, false)) {
if (!free) {
diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc
index 919de8a4e..deb32484f 100644
--- a/src/libexpr/eval-cache.cc
+++ b/src/libexpr/eval-cache.cc
@@ -285,11 +285,10 @@ static std::shared_ptr makeAttrDb(const Hash & fingerprint)
}
EvalCache::EvalCache(
- bool useCache,
- const Hash & fingerprint,
+ std::optional> useCache,
EvalState & state,
RootLoader rootLoader)
- : db(useCache ? makeAttrDb(fingerprint) : nullptr)
+ : db(useCache ? makeAttrDb(*useCache) : nullptr)
, state(state)
, rootLoader(rootLoader)
{
diff --git a/src/libexpr/eval-cache.hh b/src/libexpr/eval-cache.hh
index 674bb03c1..afee85fa9 100644
--- a/src/libexpr/eval-cache.hh
+++ b/src/libexpr/eval-cache.hh
@@ -4,6 +4,7 @@
#include "hash.hh"
#include "eval.hh"
+#include
#include
namespace nix::eval_cache {
@@ -26,8 +27,7 @@ class EvalCache : public std::enable_shared_from_this
public:
EvalCache(
- bool useCache,
- const Hash & fingerprint,
+ std::optional> useCache,
EvalState & state,
RootLoader rootLoader);
diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh
index 77f3abdeb..c2bb2888b 100644
--- a/src/libexpr/flake/flake.hh
+++ b/src/libexpr/flake/flake.hh
@@ -106,6 +106,6 @@ void emitTreeAttrs(
EvalState & state,
const fetchers::Tree & tree,
const fetchers::Input & input,
- Value & v);
+ Value & v, bool emptyRevFallback = false);
}
diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc
index 9055f59a1..5d6e39aa0 100644
--- a/src/libexpr/get-drvs.cc
+++ b/src/libexpr/get-drvs.cc
@@ -39,7 +39,7 @@ DrvInfo::DrvInfo(EvalState & state, ref store, const std::string & drvPat
if (i == drv.outputs.end())
throw Error("derivation '%s' does not have output '%s'", store->printStorePath(drvPath), outputName);
- outPath = store->printStorePath(i->second.path);
+ outPath = store->printStorePath(i->second.path(*store, drv.name));
}
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 9f877f765..05d499d1f 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -52,7 +52,7 @@ void EvalState::realiseContext(const PathSet & context)
DerivationOutputs::iterator i = drv.outputs.find(outputName);
if (i == drv.outputs.end())
throw Error("derivation '%s' does not have an output named '%s'", ctxS, outputName);
- allowedPaths->insert(store->printStorePath(i->second.path));
+ allowedPaths->insert(store->printStorePath(i->second.path(*store, drv.name)));
}
}
}
@@ -65,7 +65,7 @@ void EvalState::realiseContext(const PathSet & context)
/* For performance, prefetch all substitute info. */
StorePathSet willBuild, willSubstitute, unknown;
- unsigned long long downloadSize, narSize;
+ uint64_t downloadSize, narSize;
store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize);
store->buildPaths(drvs);
@@ -91,8 +91,17 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
Path realPath = state.checkSourcePath(state.toRealPath(path, context));
// FIXME
- if (state.store->isStorePath(path) && state.store->isValidPath(state.store->parseStorePath(path)) && isDerivation(path)) {
- Derivation drv = readDerivation(*state.store, realPath);
+ auto isValidDerivationInStore = [&]() -> std::optional {
+ if (!state.store->isStorePath(path))
+ return std::nullopt;
+ auto storePath = state.store->parseStorePath(path);
+ if (!(state.store->isValidPath(storePath) && isDerivation(path)))
+ return std::nullopt;
+ return storePath;
+ };
+ if (auto optStorePath = isValidDerivationInStore()) {
+ auto storePath = *optStorePath;
+ Derivation drv = readDerivation(*state.store, realPath, Derivation::nameFromPath(storePath));
Value & w = *state.allocValue();
state.mkAttrs(w, 3 + drv.outputs.size());
Value * v2 = state.allocAttr(w, state.sDrvPath);
@@ -106,7 +115,7 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
for (const auto & o : drv.outputs) {
v2 = state.allocAttr(w, state.symbols.create(o.first));
- mkString(*v2, state.store->printStorePath(o.second.path), {"!" + o.first + "!" + path});
+ mkString(*v2, state.store->printStorePath(o.second.path(*state.store, drv.name)), {"!" + o.first + "!" + path});
outputsVal->listElems()[outputs_index] = state.allocValue();
mkString(*(outputsVal->listElems()[outputs_index++]), o.first);
}
@@ -570,6 +579,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
/* Build the derivation expression by processing the attributes. */
Derivation drv;
+ drv.name = drvName;
PathSet context;
@@ -764,11 +774,12 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName);
if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath);
drv.outputs.insert_or_assign("out", DerivationOutput {
- .path = std::move(outPath),
- .hash = FixedOutputHash {
- .method = ingestionMethod,
- .hash = std::move(h),
- },
+ .output = DerivationOutputFixed {
+ .hash = FixedOutputHash {
+ .method = ingestionMethod,
+ .hash = std::move(h),
+ },
+ },
});
}
@@ -783,8 +794,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
if (!jsonObject) drv.env[i] = "";
drv.outputs.insert_or_assign(i,
DerivationOutput {
- .path = StorePath::dummy,
- .hash = std::optional {},
+ .output = DerivationOutputInputAddressed {
+ .path = StorePath::dummy,
+ },
});
}
@@ -795,8 +807,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
if (!jsonObject) drv.env[i] = state.store->printStorePath(outPath);
drv.outputs.insert_or_assign(i,
DerivationOutput {
- .path = std::move(outPath),
- .hash = std::optional(),
+ .output = DerivationOutputInputAddressed {
+ .path = std::move(outPath),
+ },
});
}
}
@@ -817,7 +830,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
mkString(*state.allocAttr(v, state.sDrvPath), drvPathS, {"=" + drvPathS});
for (auto & i : drv.outputs) {
mkString(*state.allocAttr(v, state.symbols.create(i.first)),
- state.store->printStorePath(i.second.path), {"!" + i.first + "!" + drvPathS});
+ state.store->printStorePath(i.second.path(*state.store, drv.name)), {"!" + i.first + "!" + drvPathS});
}
v.attrs->sort();
}
@@ -1111,7 +1124,7 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu
static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_,
- Value * filterFun, FileIngestionMethod method, const Hash & expectedHash, Value & v)
+ Value * filterFun, FileIngestionMethod method, const std::optional expectedHash, Value & v)
{
const auto path = evalSettings.pureEval && expectedHash ?
path_ :
@@ -1142,7 +1155,7 @@ static void addPath(EvalState & state, const Pos & pos, const string & name, con
std::optional expectedStorePath;
if (expectedHash)
- expectedStorePath = state.store->makeFixedOutputPath(method, expectedHash, name);
+ expectedStorePath = state.store->makeFixedOutputPath(method, *expectedHash, name);
Path dstPath;
if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
dstPath = state.store->printStorePath(settings.readOnlyMode
@@ -1176,7 +1189,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
.errPos = pos
});
- addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, Hash(), v);
+ addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v);
}
static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value & v)
@@ -1186,7 +1199,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
string name;
Value * filterFun = nullptr;
auto method = FileIngestionMethod::Recursive;
- Hash expectedHash;
+ std::optional expectedHash;
for (auto & attr : *args[0]->attrs) {
const string & n(attr.name);
diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc
deleted file mode 100644
index 5013e74f0..000000000
--- a/src/libexpr/primops/fetchGit.cc
+++ /dev/null
@@ -1,91 +0,0 @@
-#include "primops.hh"
-#include "eval-inline.hh"
-#include "store-api.hh"
-#include "hash.hh"
-#include "fetchers.hh"
-#include "url.hh"
-
-namespace nix {
-
-static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v)
-{
- std::string url;
- std::optional ref;
- std::optional rev;
- std::string name = "source";
- bool fetchSubmodules = false;
- PathSet context;
-
- state.forceValue(*args[0]);
-
- if (args[0]->type == tAttrs) {
-
- state.forceAttrs(*args[0], pos);
-
- for (auto & attr : *args[0]->attrs) {
- string n(attr.name);
- if (n == "url")
- url = state.coerceToString(*attr.pos, *attr.value, context, false, false);
- else if (n == "ref")
- ref = state.forceStringNoCtx(*attr.value, *attr.pos);
- else if (n == "rev")
- rev = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA1);
- else if (n == "name")
- name = state.forceStringNoCtx(*attr.value, *attr.pos);
- else if (n == "submodules")
- fetchSubmodules = state.forceBool(*attr.value, *attr.pos);
- else
- throw EvalError({
- .hint = hintfmt("unsupported argument '%s' to 'fetchGit'", attr.name),
- .errPos = *attr.pos
- });
- }
-
- if (url.empty())
- throw EvalError({
- .hint = hintfmt("'url' argument required"),
- .errPos = pos
- });
-
- } else
- url = state.coerceToString(pos, *args[0], context, false, false);
-
- // FIXME: git externals probably can be used to bypass the URI
- // whitelist. Ah well.
- state.checkURI(url);
-
- if (evalSettings.pureEval && !rev)
- throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision");
-
- fetchers::Attrs attrs;
- attrs.insert_or_assign("type", "git");
- attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url);
- if (ref) attrs.insert_or_assign("ref", *ref);
- if (rev) attrs.insert_or_assign("rev", rev->gitRev());
- if (fetchSubmodules) attrs.insert_or_assign("submodules", fetchers::Explicit{true});
- auto input = fetchers::Input::fromAttrs(std::move(attrs));
-
- // FIXME: use name?
- auto [tree, input2] = input.fetch(state.store);
-
- state.mkAttrs(v, 8);
- auto storePath = state.store->printStorePath(tree.storePath);
- mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
- // Backward compatibility: set 'rev' to
- // 0000000000000000000000000000000000000000 for a dirty tree.
- auto rev2 = input2.getRev().value_or(Hash(htSHA1));
- mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev());
- mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev2.gitShortRev());
- // Backward compatibility: set 'revCount' to 0 for a dirty tree.
- mkInt(*state.allocAttr(v, state.symbols.create("revCount")),
- input2.getRevCount().value_or(0));
- mkBool(*state.allocAttr(v, state.symbols.create("submodules")), fetchSubmodules);
- v.attrs->sort();
-
- if (state.allowedPaths)
- state.allowedPaths->insert(tree.actualPath);
-}
-
-static RegisterPrimOp r("fetchGit", 1, prim_fetchGit);
-
-}
diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index 6a796f3d3..cddcf0e59 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -14,7 +14,8 @@ void emitTreeAttrs(
EvalState & state,
const fetchers::Tree & tree,
const fetchers::Input & input,
- Value & v)
+ Value & v,
+ bool emptyRevFallback)
{
assert(input.isImmutable());
@@ -34,10 +35,20 @@ void emitTreeAttrs(
if (auto rev = input.getRev()) {
mkString(*state.allocAttr(v, state.symbols.create("rev")), rev->gitRev());
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev->gitShortRev());
+ } else if (emptyRevFallback) {
+ // Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev
+ auto emptyHash = Hash(htSHA1);
+ mkString(*state.allocAttr(v, state.symbols.create("rev")), emptyHash.gitRev());
+ mkString(*state.allocAttr(v, state.symbols.create("shortRev")), emptyHash.gitRev());
}
+ if (input.getType() == "git")
+ mkBool(*state.allocAttr(v, state.symbols.create("submodules")), maybeGetBoolAttr(input.attrs, "submodules").value_or(false));
+
if (auto revCount = input.getRevCount())
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount);
+ else if (emptyRevFallback)
+ mkInt(*state.allocAttr(v, state.symbols.create("revCount")), 0);
if (auto lastModified = input.getLastModified()) {
mkInt(*state.allocAttr(v, state.symbols.create("lastModified")), *lastModified);
@@ -48,10 +59,26 @@ void emitTreeAttrs(
v.attrs->sort();
}
-static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v)
+std::string fixURI(std::string uri, EvalState &state)
{
- settings.requireExperimentalFeature("flakes");
+ state.checkURI(uri);
+ return uri.find("://") != std::string::npos ? uri : "file://" + uri;
+}
+void addURI(EvalState &state, fetchers::Attrs &attrs, Symbol name, std::string v)
+{
+ string n(name);
+ attrs.emplace(name, n == "url" ? fixURI(v, state) : v);
+}
+
+static void fetchTree(
+ EvalState &state,
+ const Pos &pos,
+ Value **args,
+ Value &v,
+ const std::optional type,
+ bool emptyRevFallback = false
+) {
fetchers::Input input;
PathSet context;
@@ -64,8 +91,15 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V
for (auto & attr : *args[0]->attrs) {
state.forceValue(*attr.value);
- if (attr.value->type == tString)
- attrs.emplace(attr.name, attr.value->string.s);
+ if (attr.value->type == tPath || attr.value->type == tString)
+ addURI(
+ state,
+ attrs,
+ attr.name,
+ state.coerceToString(*attr.pos, *attr.value, context, false, false)
+ );
+ else if (attr.value->type == tString)
+ addURI(state, attrs, attr.name, attr.value->string.s);
else if (attr.value->type == tBool)
attrs.emplace(attr.name, fetchers::Explicit{attr.value->boolean});
else if (attr.value->type == tInt)
@@ -75,6 +109,9 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V
attr.name, showType(*attr.value));
}
+ if (type)
+ attrs.emplace("type", type.value());
+
if (!attrs.count("type"))
throw Error({
.hint = hintfmt("attribute 'type' is missing in call to 'fetchTree'"),
@@ -82,8 +119,18 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V
});
input = fetchers::Input::fromAttrs(std::move(attrs));
- } else
- input = fetchers::Input::fromURL(state.coerceToString(pos, *args[0], context, false, false));
+ } else {
+ auto url = fixURI(state.coerceToString(pos, *args[0], context, false, false), state);
+
+ if (type == "git") {
+ fetchers::Attrs attrs;
+ attrs.emplace("type", "git");
+ attrs.emplace("url", url);
+ input = fetchers::Input::fromAttrs(std::move(attrs));
+ } else {
+ input = fetchers::Input::fromURL(url);
+ }
+ }
if (!evalSettings.pureEval && !input.isDirect())
input = lookupInRegistries(state.store, input).first;
@@ -96,7 +143,13 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V
if (state.allowedPaths)
state.allowedPaths->insert(tree.actualPath);
- emitTreeAttrs(state, tree, input2, v);
+ emitTreeAttrs(state, tree, input2, v, emptyRevFallback);
+}
+
+static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+ settings.requireExperimentalFeature("flakes");
+ fetchTree(state, pos, args, v, std::nullopt);
}
static RegisterPrimOp r("fetchTree", 1, prim_fetchTree);
@@ -159,7 +212,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
: hashFile(htSHA256, path);
if (hash != *expectedHash)
throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s",
- *url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true));
+ *url, expectedHash->to_string(Base32, true), hash->to_string(Base32, true));
}
if (state.allowedPaths)
@@ -178,7 +231,13 @@ static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args
fetch(state, pos, args, v, "fetchTarball", true, "source");
}
+static void prim_fetchGit(EvalState &state, const Pos &pos, Value **args, Value &v)
+{
+ fetchTree(state, pos, args, v, "git", true);
+}
+
static RegisterPrimOp r2("__fetchurl", 1, prim_fetchurl);
static RegisterPrimOp r3("fetchTarball", 1, prim_fetchTarball);
+static RegisterPrimOp r4("fetchGit", 1, prim_fetchGit);
}
diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index 2b6173df9..28db8aa9c 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -130,12 +130,12 @@ std::pair Input::fetch(ref store) const
tree.actualPath = store->toRealPath(tree.storePath);
auto narHash = store->queryPathInfo(tree.storePath)->narHash;
- input.attrs.insert_or_assign("narHash", narHash.to_string(SRI, true));
+ input.attrs.insert_or_assign("narHash", narHash->to_string(SRI, true));
if (auto prevNarHash = getNarHash()) {
if (narHash != *prevNarHash)
- throw Error("NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'",
- to_string(), tree.actualPath, prevNarHash->to_string(SRI, true), narHash.to_string(SRI, true));
+ throw Error((unsigned int) 102, "NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'",
+ to_string(), tree.actualPath, prevNarHash->to_string(SRI, true), narHash->to_string(SRI, true));
}
if (auto prevLastModified = getLastModified()) {
diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index 52718c231..2b1f25ca3 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -36,7 +36,7 @@ void printGCWarning()
void printMissing(ref store, const std::vector & paths, Verbosity lvl)
{
- unsigned long long downloadSize, narSize;
+ uint64_t downloadSize, narSize;
StorePathSet willBuild, willSubstitute, unknown;
store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize);
printMissing(store, willBuild, willSubstitute, unknown, downloadSize, narSize, lvl);
@@ -45,7 +45,7 @@ void printMissing(ref store, const std::vector & pa
void printMissing(ref store, const StorePathSet & willBuild,
const StorePathSet & willSubstitute, const StorePathSet & unknown,
- unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl)
+ uint64_t downloadSize, uint64_t narSize, Verbosity lvl)
{
if (!willBuild.empty()) {
if (willBuild.size() == 1)
@@ -384,7 +384,7 @@ RunPager::~RunPager()
}
-string showBytes(unsigned long long bytes)
+string showBytes(uint64_t bytes)
{
return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str();
}
diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh
index f558247c0..ffae5d796 100644
--- a/src/libmain/shared.hh
+++ b/src/libmain/shared.hh
@@ -47,7 +47,7 @@ void printMissing(
void printMissing(ref store, const StorePathSet & willBuild,
const StorePathSet & willSubstitute, const StorePathSet & unknown,
- unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl = lvlInfo);
+ uint64_t downloadSize, uint64_t narSize, Verbosity lvl = lvlInfo);
string getArg(const string & opt,
Strings::iterator & i, const Strings::iterator & end);
@@ -110,7 +110,7 @@ extern volatile ::sig_atomic_t blockInt;
/* GC helpers. */
-string showBytes(unsigned long long bytes);
+string showBytes(uint64_t bytes);
struct GCResults;
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index b791c125b..30150eeba 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -178,7 +178,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
auto [fileHash, fileSize] = fileHashSink.finish();
narInfo->fileHash = fileHash;
narInfo->fileSize = fileSize;
- narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar"
+ narInfo->url = "nar/" + narInfo->fileHash->to_string(Base32, false) + ".nar"
+ (compression == "xz" ? ".xz" :
compression == "bzip2" ? ".bz2" :
compression == "br" ? ".br" :
@@ -372,7 +372,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
method for very large paths, but `copyPath' is mainly used for
small files. */
StringSink sink;
- Hash h;
+ std::optional h;
if (method == FileIngestionMethod::Recursive) {
dumpPath(srcPath, sink, filter);
h = hashString(hashAlgo, *sink.s);
@@ -382,7 +382,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
h = hashString(hashAlgo, s);
}
- ValidPathInfo info(makeFixedOutputPath(method, h, name));
+ ValidPathInfo info(makeFixedOutputPath(method, *h, name));
auto source = StringSource { *sink.s };
addToStore(info, source, repair, CheckSigs);
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 3a55f24cb..1f40dc42a 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -297,7 +297,7 @@ public:
GoalPtr makeDerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
std::shared_ptr makeBasicDerivationGoal(const StorePath & drvPath,
const BasicDerivation & drv, BuildMode buildMode = bmNormal);
- GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair);
+ GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt);
/* Remove a dead goal. */
void removeGoal(GoalPtr goal);
@@ -1047,7 +1047,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation
{
this->drv = std::make_unique(BasicDerivation(drv));
state = &DerivationGoal::haveDerivation;
- name = fmt("building of %s", worker.store.showPaths(drv.outputPaths()));
+ name = fmt("building of %s", worker.store.showPaths(drv.outputPaths(worker.store)));
trace("created");
mcExpectedBuilds = std::make_unique>(worker.expectedBuilds);
@@ -1182,7 +1182,7 @@ void DerivationGoal::haveDerivation()
retrySubstitution = false;
for (auto & i : drv->outputs)
- worker.store.addTempRoot(i.second.path);
+ worker.store.addTempRoot(i.second.path(worker.store, drv->name));
/* Check what outputs paths are not already valid. */
auto invalidOutputs = checkPathValidity(false, buildMode == bmRepair);
@@ -1206,7 +1206,7 @@ void DerivationGoal::haveDerivation()
them. */
if (settings.useSubstitutes && parsedDrv->substitutesAllowed())
for (auto & i : invalidOutputs)
- addWaitee(worker.makeSubstitutionGoal(i, buildMode == bmRepair ? Repair : NoRepair));
+ addWaitee(worker.makeSubstitutionGoal(i, buildMode == bmRepair ? Repair : NoRepair, getDerivationCA(*drv)));
if (waitees.empty()) /* to prevent hang (no wake-up event) */
outputsSubstituted();
@@ -1290,12 +1290,12 @@ void DerivationGoal::repairClosure()
StorePathSet outputClosure;
for (auto & i : drv->outputs) {
if (!wantOutput(i.first, wantedOutputs)) continue;
- worker.store.computeFSClosure(i.second.path, outputClosure);
+ worker.store.computeFSClosure(i.second.path(worker.store, drv->name), outputClosure);
}
/* Filter out our own outputs (which we have already checked). */
for (auto & i : drv->outputs)
- outputClosure.erase(i.second.path);
+ outputClosure.erase(i.second.path(worker.store, drv->name));
/* Get all dependencies of this derivation so that we know which
derivation is responsible for which path in the output
@@ -1307,7 +1307,7 @@ void DerivationGoal::repairClosure()
if (i.isDerivation()) {
Derivation drv = worker.store.derivationFromPath(i);
for (auto & j : drv.outputs)
- outputsToDrv.insert_or_assign(j.second.path, i);
+ outputsToDrv.insert_or_assign(j.second.path(worker.store, drv.name), i);
}
/* Check each path (slow!). */
@@ -1379,7 +1379,7 @@ void DerivationGoal::inputsRealised()
for (auto & j : i.second) {
auto k = inDrv.outputs.find(j);
if (k != inDrv.outputs.end())
- worker.store.computeFSClosure(k->second.path, inputPaths);
+ worker.store.computeFSClosure(k->second.path(worker.store, inDrv.name), inputPaths);
else
throw Error(
"derivation '%s' requires non-existent output '%s' from input derivation '%s'",
@@ -1432,7 +1432,7 @@ void DerivationGoal::tryToBuild()
goal can start a build, and if not, the main loop will sleep a
few seconds and then retry this goal. */
PathSet lockFiles;
- for (auto & outPath : drv->outputPaths())
+ for (auto & outPath : drv->outputPaths(worker.store))
lockFiles.insert(worker.store.Store::toRealPath(outPath));
if (!outputLocks.lockPaths(lockFiles, "", false)) {
@@ -1460,16 +1460,16 @@ void DerivationGoal::tryToBuild()
return;
}
- missingPaths = drv->outputPaths();
+ missingPaths = drv->outputPaths(worker.store);
if (buildMode != bmCheck)
for (auto & i : validPaths) missingPaths.erase(i);
/* If any of the outputs already exist but are not valid, delete
them. */
for (auto & i : drv->outputs) {
- if (worker.store.isValidPath(i.second.path)) continue;
- debug("removing invalid path '%s'", worker.store.printStorePath(i.second.path));
- deletePath(worker.store.Store::toRealPath(i.second.path));
+ if (worker.store.isValidPath(i.second.path(worker.store, drv->name))) continue;
+ debug("removing invalid path '%s'", worker.store.printStorePath(i.second.path(worker.store, drv->name)));
+ deletePath(worker.store.Store::toRealPath(i.second.path(worker.store, drv->name)));
}
/* Don't do a remote build if the derivation has the attribute
@@ -1646,13 +1646,13 @@ void DerivationGoal::buildDone()
So instead, check if the disk is (nearly) full now. If
so, we don't mark this build as a permanent failure. */
#if HAVE_STATVFS
- unsigned long long required = 8ULL * 1024 * 1024; // FIXME: make configurable
+ uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable
struct statvfs st;
if (statvfs(worker.store.realStoreDir.c_str(), &st) == 0 &&
- (unsigned long long) st.f_bavail * st.f_bsize < required)
+ (uint64_t) st.f_bavail * st.f_bsize < required)
diskFull = true;
if (statvfs(tmpDir.c_str(), &st) == 0 &&
- (unsigned long long) st.f_bavail * st.f_bsize < required)
+ (uint64_t) st.f_bavail * st.f_bsize < required)
diskFull = true;
#endif
@@ -1692,7 +1692,7 @@ void DerivationGoal::buildDone()
fmt("running post-build-hook '%s'", settings.postBuildHook),
Logger::Fields{worker.store.printStorePath(drvPath)});
PushActivity pact(act.id);
- auto outputPaths = drv->outputPaths();
+ auto outputPaths = drv->outputPaths(worker.store);
std::map hookEnvironment = getEnv();
hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath));
@@ -1920,7 +1920,7 @@ StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths)
if (j.isDerivation()) {
Derivation drv = worker.store.derivationFromPath(j);
for (auto & k : drv.outputs)
- worker.store.computeFSClosure(k.second.path, paths);
+ worker.store.computeFSClosure(k.second.path(worker.store, drv.name), paths);
}
}
@@ -2015,7 +2015,7 @@ void DerivationGoal::startBuilder()
/* Substitute output placeholders with the actual output paths. */
for (auto & output : drv->outputs)
- inputRewrites[hashPlaceholder(output.first)] = worker.store.printStorePath(output.second.path);
+ inputRewrites[hashPlaceholder(output.first)] = worker.store.printStorePath(output.second.path(worker.store, drv->name));
/* Construct the environment passed to the builder. */
initEnv();
@@ -2200,7 +2200,7 @@ void DerivationGoal::startBuilder()
(typically the dependencies of /bin/sh). Throw them
out. */
for (auto & i : drv->outputs)
- dirsInChroot.erase(worker.store.printStorePath(i.second.path));
+ dirsInChroot.erase(worker.store.printStorePath(i.second.path(worker.store, drv->name)));
#elif __APPLE__
/* We don't really have any parent prep work to do (yet?)
@@ -2613,7 +2613,7 @@ void DerivationGoal::writeStructuredAttrs()
/* Add an "outputs" object containing the output paths. */
nlohmann::json outputs;
for (auto & i : drv->outputs)
- outputs[i.first] = rewriteStrings(worker.store.printStorePath(i.second.path), inputRewrites);
+ outputs[i.first] = rewriteStrings(worker.store.printStorePath(i.second.path(worker.store, drv->name)), inputRewrites);
json["outputs"] = outputs;
/* Handle exportReferencesGraph. */
@@ -2821,7 +2821,7 @@ struct RestrictedStore : public LocalFSStore
auto drv = derivationFromPath(path.path);
for (auto & output : drv.outputs)
if (wantOutput(output.first, path.outputs))
- newPaths.insert(output.second.path);
+ newPaths.insert(output.second.path(*this, drv.name));
} else if (!goal.isAllowed(path.path))
throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path));
}
@@ -2855,7 +2855,7 @@ struct RestrictedStore : public LocalFSStore
void queryMissing(const std::vector & targets,
StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
- unsigned long long & downloadSize, unsigned long long & narSize) override
+ uint64_t & downloadSize, uint64_t & narSize) override
{
/* This is slightly impure since it leaks information to the
client about what paths will be built/substituted or are
@@ -3583,7 +3583,7 @@ StorePathSet parseReferenceSpecifiers(Store & store, const BasicDerivation & drv
if (store.isStorePath(i))
result.insert(store.parseStorePath(i));
else if (drv.outputs.count(i))
- result.insert(drv.outputs.find(i)->second.path);
+ result.insert(drv.outputs.find(i)->second.path(store, drv.name));
else throw BuildError("derivation contains an illegal reference specifier '%s'", i);
}
return result;
@@ -3621,7 +3621,7 @@ void DerivationGoal::registerOutputs()
if (hook) {
bool allValid = true;
for (auto & i : drv->outputs)
- if (!worker.store.isValidPath(i.second.path)) allValid = false;
+ if (!worker.store.isValidPath(i.second.path(worker.store, drv->name))) allValid = false;
if (allValid) return;
}
@@ -3642,23 +3642,23 @@ void DerivationGoal::registerOutputs()
Nix calls. */
StorePathSet referenceablePaths;
for (auto & p : inputPaths) referenceablePaths.insert(p);
- for (auto & i : drv->outputs) referenceablePaths.insert(i.second.path);
+ for (auto & i : drv->outputs) referenceablePaths.insert(i.second.path(worker.store, drv->name));
for (auto & p : addedPaths) referenceablePaths.insert(p);
/* Check whether the output paths were created, and grep each
output path to determine what other paths it references. Also make all
output paths read-only. */
for (auto & i : drv->outputs) {
- auto path = worker.store.printStorePath(i.second.path);
- if (!missingPaths.count(i.second.path)) continue;
+ auto path = worker.store.printStorePath(i.second.path(worker.store, drv->name));
+ if (!missingPaths.count(i.second.path(worker.store, drv->name))) continue;
Path actualPath = path;
if (needsHashRewrite()) {
- auto r = redirectedOutputs.find(i.second.path);
+ auto r = redirectedOutputs.find(i.second.path(worker.store, drv->name));
if (r != redirectedOutputs.end()) {
auto redirected = worker.store.Store::toRealPath(r->second);
if (buildMode == bmRepair
- && redirectedBadOutputs.count(i.second.path)
+ && redirectedBadOutputs.count(i.second.path(worker.store, drv->name))
&& pathExists(redirected))
replaceValidPath(path, redirected);
if (buildMode == bmCheck)
@@ -3727,7 +3727,9 @@ void DerivationGoal::registerOutputs()
if (fixedOutput) {
- if (i.second.hash->method == FileIngestionMethod::Flat) {
+ FixedOutputHash outputHash = std::get(i.second.output).hash;
+
+ if (outputHash.method == FileIngestionMethod::Flat) {
/* The output path should be a regular file without execute permission. */
if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0)
throw BuildError(
@@ -3738,13 +3740,13 @@ void DerivationGoal::registerOutputs()
/* Check the hash. In hash mode, move the path produced by
the derivation to its content-addressed location. */
- Hash h2 = i.second.hash->method == FileIngestionMethod::Recursive
- ? hashPath(*i.second.hash->hash.type, actualPath).first
- : hashFile(*i.second.hash->hash.type, actualPath);
+ Hash h2 = outputHash.method == FileIngestionMethod::Recursive
+ ? hashPath(outputHash.hash.type, actualPath).first
+ : hashFile(outputHash.hash.type, actualPath);
- auto dest = worker.store.makeFixedOutputPath(i.second.hash->method, h2, i.second.path.name());
+ auto dest = worker.store.makeFixedOutputPath(outputHash.method, h2, i.second.path(worker.store, drv->name).name());
- if (i.second.hash->hash != h2) {
+ if (outputHash.hash != h2) {
/* Throw an error after registering the path as
valid. */
@@ -3752,7 +3754,7 @@ void DerivationGoal::registerOutputs()
delayedException = std::make_exception_ptr(
BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s",
worker.store.printStorePath(dest),
- i.second.hash->hash.to_string(SRI, true),
+ outputHash.hash.to_string(SRI, true),
h2.to_string(SRI, true)));
Path actualDest = worker.store.Store::toRealPath(dest);
@@ -3774,7 +3776,7 @@ void DerivationGoal::registerOutputs()
assert(worker.store.parseStorePath(path) == dest);
ca = FixedOutputHash {
- .method = i.second.hash->method,
+ .method = outputHash.method,
.hash = h2,
};
}
@@ -3789,8 +3791,10 @@ void DerivationGoal::registerOutputs()
time. The hash is stored in the database so that we can
verify later on whether nobody has messed with the store. */
debug("scanning for references inside '%1%'", path);
- HashResult hash;
- auto references = worker.store.parseStorePathSet(scanForReferences(actualPath, worker.store.printStorePathSet(referenceablePaths), hash));
+ // HashResult hash;
+ auto pathSetAndHash = scanForReferences(actualPath, worker.store.printStorePathSet(referenceablePaths));
+ auto references = worker.store.parseStorePathSet(pathSetAndHash.first);
+ HashResult hash = pathSetAndHash.second;
if (buildMode == bmCheck) {
if (!worker.store.isValidPath(worker.store.parseStorePath(path))) continue;
@@ -3898,7 +3902,7 @@ void DerivationGoal::registerOutputs()
/* If this is the first round of several, then move the output out of the way. */
if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) {
for (auto & i : drv->outputs) {
- auto path = worker.store.printStorePath(i.second.path);
+ auto path = worker.store.printStorePath(i.second.path(worker.store, drv->name));
Path prev = path + checkSuffix;
deletePath(prev);
Path dst = path + checkSuffix;
@@ -3916,7 +3920,7 @@ void DerivationGoal::registerOutputs()
if the result was not determistic? */
if (curRound == nrRounds) {
for (auto & i : drv->outputs) {
- Path prev = worker.store.printStorePath(i.second.path) + checkSuffix;
+ Path prev = worker.store.printStorePath(i.second.path(worker.store, drv->name)) + checkSuffix;
deletePath(prev);
}
}
@@ -4217,9 +4221,9 @@ StorePathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash)
for (auto & i : drv->outputs) {
if (!wantOutput(i.first, wantedOutputs)) continue;
bool good =
- worker.store.isValidPath(i.second.path) &&
- (!checkHash || worker.pathContentsGood(i.second.path));
- if (good == returnValid) result.insert(i.second.path);
+ worker.store.isValidPath(i.second.path(worker.store, drv->name)) &&
+ (!checkHash || worker.pathContentsGood(i.second.path(worker.store, drv->name)));
+ if (good == returnValid) result.insert(i.second.path(worker.store, drv->name));
}
return result;
}
@@ -4276,6 +4280,10 @@ private:
/* The store path that should be realised through a substitute. */
StorePath storePath;
+ /* The path the substituter refers to the path as. This will be
+ * different when the stores have different names. */
+ std::optional subPath;
+
/* The remaining substituters. */
std::list> subs;
@@ -4309,8 +4317,11 @@ private:
typedef void (SubstitutionGoal::*GoalState)();
GoalState state;
+ /* Content address for recomputing store path */
+ std::optional ca;
+
public:
- SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair);
+ SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt);
~SubstitutionGoal();
void timedOut(Error && ex) override { abort(); };
@@ -4340,10 +4351,11 @@ public:
};
-SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair)
+SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional ca)
: Goal(worker)
, storePath(storePath)
, repair(repair)
+ , ca(ca)
{
state = &SubstitutionGoal::init;
name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath));
@@ -4418,14 +4430,18 @@ void SubstitutionGoal::tryNext()
sub = subs.front();
subs.pop_front();
- if (sub->storeDir != worker.store.storeDir) {
+ if (ca) {
+ subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca);
+ if (sub->storeDir == worker.store.storeDir)
+ assert(subPath == storePath);
+ } else if (sub->storeDir != worker.store.storeDir) {
tryNext();
return;
}
try {
// FIXME: make async
- info = sub->queryPathInfo(storePath);
+ info = sub->queryPathInfo(subPath ? *subPath : storePath);
} catch (InvalidPath &) {
tryNext();
return;
@@ -4444,6 +4460,19 @@ void SubstitutionGoal::tryNext()
throw;
}
+ if (info->path != storePath) {
+ if (info->isContentAddressed(*sub) && info->references.empty()) {
+ auto info2 = std::make_shared(*info);
+ info2->path = storePath;
+ info = info2;
+ } else {
+ printError("asked '%s' for '%s' but got '%s'",
+ sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path));
+ tryNext();
+ return;
+ }
+ }
+
/* Update the total expected download size. */
auto narInfo = std::dynamic_pointer_cast(info);
@@ -4533,7 +4562,7 @@ void SubstitutionGoal::tryToRun()
PushActivity pact(act.id);
copyStorePath(ref(sub), ref(worker.store.shared_from_this()),
- storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs);
+ subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs);
promise.set_value();
} catch (...) {
@@ -4666,11 +4695,11 @@ std::shared_ptr Worker::makeBasicDerivationGoal(const StorePath
}
-GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair)
+GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca)
{
GoalPtr goal = substitutionGoals[path].lock(); // FIXME
if (!goal) {
- goal = std::make_shared(path, *this, repair);
+ goal = std::make_shared(path, *this, repair, ca);
substitutionGoals.insert_or_assign(path, goal);
wakeUp(goal);
}
@@ -5012,7 +5041,7 @@ bool Worker::pathContentsGood(const StorePath & path)
if (!pathExists(store.printStorePath(path)))
res = false;
else {
- HashResult current = hashPath(*info->narHash.type, store.printStorePath(path));
+ HashResult current = hashPath(info->narHash->type, store.printStorePath(path));
Hash nullHash(htSHA256);
res = info->narHash == nullHash || info->narHash == current.first;
}
@@ -5038,7 +5067,7 @@ void Worker::markContentsGood(const StorePath & path)
static void primeCache(Store & store, const std::vector & paths)
{
StorePathSet willBuild, willSubstitute, unknown;
- unsigned long long downloadSize, narSize;
+ uint64_t downloadSize, narSize;
store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize);
if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty())
diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc
index e630cf6f1..6585a480d 100644
--- a/src/libstore/builtins/fetchurl.cc
+++ b/src/libstore/builtins/fetchurl.cc
@@ -58,23 +58,6 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
}
};
- /* We always have one output, and if it's a fixed-output derivation (as
- checked below) it must be the only output */
- auto & output = drv.outputs.begin()->second;
-
- /* Try the hashed mirrors first. */
- if (output.hash && output.hash->method == FileIngestionMethod::Flat)
- for (auto hashedMirror : settings.hashedMirrors.get())
- try {
- if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
- auto & h = output.hash->hash;
- fetch(hashedMirror + printHashType(*h.type) + "/" + h.to_string(Base16, false));
- return;
- } catch (Error & e) {
- debug(e.what());
- }
-
- /* Otherwise try the specified URL. */
fetch(mainUrl);
}
diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc
index 6cb69d0a9..f83b98a98 100644
--- a/src/libstore/content-address.cc
+++ b/src/libstore/content-address.cc
@@ -3,7 +3,7 @@
namespace nix {
std::string FixedOutputHash::printMethodAlgo() const {
- return makeFileIngestionPrefix(method) + printHashType(*hash.type);
+ return makeFileIngestionPrefix(method) + printHashType(hash.type);
}
std::string makeFileIngestionPrefix(const FileIngestionMethod m) {
@@ -46,7 +46,7 @@ ContentAddress parseContentAddress(std::string_view rawCa) {
if (prefix == "text") {
auto hashTypeAndHash = rawCa.substr(prefixSeparator+1, string::npos);
Hash hash = Hash(string(hashTypeAndHash));
- if (*hash.type != htSHA256) {
+ if (hash.type != htSHA256) {
throw Error("parseContentAddress: the text hash should have type SHA256");
}
return TextHash { hash };
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 82833f5c3..9f7be6e1a 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -86,7 +86,7 @@ struct TunnelLogger : public Logger
}
/* startWork() means that we're starting an operation for which we
- want to send out stderr to the client. */
+ want to send out stderr to the client. */
void startWork()
{
auto state(state_.lock());
@@ -289,7 +289,7 @@ static void performOp(TunnelLogger * logger, ref store,
logger->startWork();
auto hash = store->queryPathInfo(path)->narHash;
logger->stopWork();
- to << hash.to_string(Base16, false);
+ to << hash->to_string(Base16, false);
break;
}
@@ -451,7 +451,7 @@ static void performOp(TunnelLogger * logger, ref store,
case wopBuildDerivation: {
auto drvPath = store->parseStorePath(readString(from));
BasicDerivation drv;
- readDerivation(from, *store, drv);
+ readDerivation(from, *store, drv, Derivation::nameFromPath(drvPath));
BuildMode buildMode = (BuildMode) readInt(from);
logger->startWork();
if (!trusted)
@@ -579,7 +579,7 @@ static void performOp(TunnelLogger * logger, ref store,
auto path = store->parseStorePath(readString(from));
logger->startWork();
SubstitutablePathInfos infos;
- store->querySubstitutablePathInfos({path}, infos);
+ store->querySubstitutablePathInfos({{path, std::nullopt}}, infos);
logger->stopWork();
auto i = infos.find(path);
if (i == infos.end())
@@ -595,10 +595,16 @@ static void performOp(TunnelLogger * logger, ref store,
}
case wopQuerySubstitutablePathInfos: {
- auto paths = readStorePaths(*store, from);
- logger->startWork();
SubstitutablePathInfos infos;
- store->querySubstitutablePathInfos(paths, infos);
+ StorePathCAMap pathsMap = {};
+ if (GET_PROTOCOL_MINOR(clientVersion) < 22) {
+ auto paths = readStorePaths(*store, from);
+ for (auto & path : paths)
+ pathsMap.emplace(path, std::nullopt);
+ } else
+ pathsMap = readStorePathCAMap(*store, from);
+ logger->startWork();
+ store->querySubstitutablePathInfos(pathsMap, infos);
logger->stopWork();
to << infos.size();
for (auto & i : infos) {
@@ -632,7 +638,7 @@ static void performOp(TunnelLogger * logger, ref store,
if (GET_PROTOCOL_MINOR(clientVersion) >= 17)
to << 1;
to << (info->deriver ? store->printStorePath(*info->deriver) : "")
- << info->narHash.to_string(Base16, false);
+ << info->narHash->to_string(Base16, false);
writeStorePaths(*store, to, info->references);
to << info->registrationTime << info->narSize;
if (GET_PROTOCOL_MINOR(clientVersion) >= 16) {
@@ -703,24 +709,84 @@ static void performOp(TunnelLogger * logger, ref store,
if (!trusted)
info.ultimate = false;
- std::unique_ptr source;
- if (GET_PROTOCOL_MINOR(clientVersion) >= 21)
- source = std::make_unique(from, to);
- else {
- StringSink saved;
- TeeSource tee { from, saved };
- ParseSink ether;
- parseDump(ether, tee);
- source = std::make_unique(std::move(*saved.s));
+ if (GET_PROTOCOL_MINOR(clientVersion) >= 23) {
+
+ struct FramedSource : Source
+ {
+ Source & from;
+ bool eof = false;
+ std::vector pending;
+ size_t pos = 0;
+
+ FramedSource(Source & from) : from(from)
+ { }
+
+ ~FramedSource()
+ {
+ if (!eof) {
+ while (true) {
+ auto n = readInt(from);
+ if (!n) break;
+ std::vector data(n);
+ from(data.data(), n);
+ }
+ }
+ }
+
+ size_t read(unsigned char * data, size_t len) override
+ {
+ if (eof) throw EndOfFile("reached end of FramedSource");
+
+ if (pos >= pending.size()) {
+ size_t len = readInt(from);
+ if (!len) {
+ eof = true;
+ return 0;
+ }
+ pending = std::vector(len);
+ pos = 0;
+ from(pending.data(), len);
+ }
+
+ auto n = std::min(len, pending.size() - pos);
+ memcpy(data, pending.data() + pos, n);
+ pos += n;
+ return n;
+ }
+ };
+
+ logger->startWork();
+
+ {
+ FramedSource source(from);
+ store->addToStore(info, source, (RepairFlag) repair,
+ dontCheckSigs ? NoCheckSigs : CheckSigs);
+ }
+
+ logger->stopWork();
}
- logger->startWork();
+ else {
+ std::unique_ptr source;
+ if (GET_PROTOCOL_MINOR(clientVersion) >= 21)
+ source = std::make_unique(from, to);
+ else {
+ StringSink saved;
+ TeeSource tee { from, saved };
+ ParseSink ether;
+ parseDump(ether, tee);
+ source = std::make_unique(std::move(*saved.s));
+ }
- // FIXME: race if addToStore doesn't read source?
- store->addToStore(info, *source, (RepairFlag) repair,
- dontCheckSigs ? NoCheckSigs : CheckSigs);
+ logger->startWork();
+
+ // FIXME: race if addToStore doesn't read source?
+ store->addToStore(info, *source, (RepairFlag) repair,
+ dontCheckSigs ? NoCheckSigs : CheckSigs);
+
+ logger->stopWork();
+ }
- logger->stopWork();
break;
}
@@ -730,7 +796,7 @@ static void performOp(TunnelLogger * logger, ref store,
targets.push_back(store->parsePathWithOutputs(s));
logger->startWork();
StorePathSet willBuild, willSubstitute, unknown;
- unsigned long long downloadSize, narSize;
+ uint64_t downloadSize, narSize;
store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize);
logger->stopWork();
writeStorePaths(*store, to, willBuild);
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 870901421..7d0a5abeb 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -7,6 +7,22 @@
namespace nix {
+// FIXME Put this somewhere?
+template struct overloaded : Ts... { using Ts::operator()...; };
+template overloaded(Ts...) -> overloaded;
+
+StorePath DerivationOutput::path(const Store & store, std::string_view drvName) const
+{
+ return std::visit(overloaded {
+ [](DerivationOutputInputAddressed doi) {
+ return doi.path;
+ },
+ [&](DerivationOutputFixed dof) {
+ return store.makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName);
+ }
+ }, output);
+}
+
bool BasicDerivation::isBuiltin() const
{
@@ -99,7 +115,6 @@ static DerivationOutput parseDerivationOutput(const Store & store, std::istrings
expect(str, ","); const auto hash = parseString(str);
expect(str, ")");
- std::optional fsh;
if (hashAlgo != "") {
auto method = FileIngestionMethod::Flat;
if (string(hashAlgo, 0, 2) == "r:") {
@@ -107,22 +122,29 @@ static DerivationOutput parseDerivationOutput(const Store & store, std::istrings
hashAlgo = string(hashAlgo, 2);
}
const HashType hashType = parseHashType(hashAlgo);
- fsh = FixedOutputHash {
- .method = std::move(method),
- .hash = Hash(hash, hashType),
- };
- }
- return DerivationOutput {
- .path = std::move(path),
- .hash = std::move(fsh),
- };
+ return DerivationOutput {
+ .output = DerivationOutputFixed {
+ .hash = FixedOutputHash {
+ .method = std::move(method),
+ .hash = Hash(hash, hashType),
+ },
+ }
+ };
+ } else
+ return DerivationOutput {
+ .output = DerivationOutputInputAddressed {
+ .path = std::move(path),
+ }
+ };
}
-static Derivation parseDerivation(const Store & store, std::string && s)
+static Derivation parseDerivation(const Store & store, std::string && s, std::string_view name)
{
Derivation drv;
+ drv.name = name;
+
std::istringstream str(std::move(s));
expect(str, "Derive([");
@@ -166,10 +188,10 @@ static Derivation parseDerivation(const Store & store, std::string && s)
}
-Derivation readDerivation(const Store & store, const Path & drvPath)
+Derivation readDerivation(const Store & store, const Path & drvPath, std::string_view name)
{
try {
- return parseDerivation(store, readFile(drvPath));
+ return parseDerivation(store, readFile(drvPath), name);
} catch (FormatError & e) {
throw Error("error parsing derivation '%1%': %2%", drvPath, e.msg());
}
@@ -187,7 +209,7 @@ Derivation Store::readDerivation(const StorePath & drvPath)
{
auto accessor = getFSAccessor();
try {
- return parseDerivation(*this, accessor->readFile(printStorePath(drvPath)));
+ return parseDerivation(*this, accessor->readFile(printStorePath(drvPath)), Derivation::nameFromPath(drvPath));
} catch (FormatError & e) {
throw Error("error parsing derivation '%s': %s", printStorePath(drvPath), e.msg());
}
@@ -255,10 +277,14 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
for (auto & i : outputs) {
if (first) first = false; else s += ',';
s += '('; printUnquotedString(s, i.first);
- s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path));
- s += ','; printUnquotedString(s, i.second.hash ? i.second.hash->printMethodAlgo() : "");
- s += ','; printUnquotedString(s,
- i.second.hash ? i.second.hash->hash.to_string(Base16, false) : "");
+ s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path(store, name)));
+ if (auto hash = std::get_if(&i.second.output)) {
+ s += ','; printUnquotedString(s, hash->hash.printMethodAlgo());
+ s += ','; printUnquotedString(s, hash->hash.hash.to_string(Base16, false));
+ } else {
+ s += ','; printUnquotedString(s, "");
+ s += ','; printUnquotedString(s, "");
+ }
s += ')';
}
@@ -314,7 +340,7 @@ bool BasicDerivation::isFixedOutput() const
{
return outputs.size() == 1 &&
outputs.begin()->first == "out" &&
- outputs.begin()->second.hash;
+ std::holds_alternative(outputs.begin()->second.output);
}
@@ -346,10 +372,11 @@ Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutput
/* Return a fixed hash for fixed-output derivations. */
if (drv.isFixedOutput()) {
DerivationOutputs::const_iterator i = drv.outputs.begin();
+ auto hash = std::get(i->second.output);
return hashString(htSHA256, "fixed:out:"
- + i->second.hash->printMethodAlgo() + ":"
- + i->second.hash->hash.to_string(Base16, false) + ":"
- + store.printStorePath(i->second.path));
+ + hash.hash.printMethodAlgo() + ":"
+ + hash.hash.hash.to_string(Base16, false) + ":"
+ + store.printStorePath(i->second.path(store, drv.name)));
}
/* For other derivations, replace the inputs paths with recursive
@@ -383,11 +410,11 @@ bool wantOutput(const string & output, const std::set & wanted)
}
-StorePathSet BasicDerivation::outputPaths() const
+StorePathSet BasicDerivation::outputPaths(const Store & store) const
{
StorePathSet paths;
for (auto & i : outputs)
- paths.insert(i.second.path);
+ paths.insert(i.second.path(store, name));
return paths;
}
@@ -397,7 +424,6 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store)
auto hashAlgo = readString(in);
auto hash = readString(in);
- std::optional fsh;
if (hashAlgo != "") {
auto method = FileIngestionMethod::Flat;
if (string(hashAlgo, 0, 2) == "r:") {
@@ -405,16 +431,20 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store)
hashAlgo = string(hashAlgo, 2);
}
auto hashType = parseHashType(hashAlgo);
- fsh = FixedOutputHash {
- .method = std::move(method),
- .hash = Hash(hash, hashType),
+ return DerivationOutput {
+ .output = DerivationOutputFixed {
+ .hash = FixedOutputHash {
+ .method = std::move(method),
+ .hash = Hash(hash, hashType),
+ },
+ }
+ };
+ } else
+ return DerivationOutput {
+ .output = DerivationOutputInputAddressed {
+ .path = std::move(path),
+ }
};
- }
-
- return DerivationOutput {
- .path = std::move(path),
- .hash = std::move(fsh),
- };
}
StringSet BasicDerivation::outputNames() const
@@ -426,8 +456,19 @@ StringSet BasicDerivation::outputNames() const
}
-Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv)
+std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath) {
+ auto nameWithSuffix = drvPath.name();
+ constexpr std::string_view extension = ".drv";
+ assert(hasSuffix(nameWithSuffix, extension));
+ nameWithSuffix.remove_suffix(extension.size());
+ return nameWithSuffix;
+}
+
+
+Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name)
{
+ drv.name = name;
+
drv.outputs.clear();
auto nr = readNum(in);
for (size_t n = 0; n < nr; n++) {
@@ -456,10 +497,10 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
out << drv.outputs.size();
for (auto & i : drv.outputs) {
out << i.first
- << store.printStorePath(i.second.path);
- if (i.second.hash) {
- out << i.second.hash->printMethodAlgo()
- << i.second.hash->hash.to_string(Base16, false);
+ << store.printStorePath(i.second.path(store, drv.name));
+ if (auto hash = std::get_if(&i.second.output)) {
+ out << hash->hash.printMethodAlgo()
+ << hash->hash.hash.to_string(Base16, false);
} else {
out << "" << "";
}
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index 00cf1bb08..133ffe50e 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -13,10 +13,20 @@ namespace nix {
/* Abstract syntax of derivations. */
-struct DerivationOutput
+struct DerivationOutputInputAddressed
{
StorePath path;
- std::optional hash; /* hash used for expected hash computation */
+};
+
+struct DerivationOutputFixed
+{
+ FixedOutputHash hash; /* hash used for expected hash computation */
+};
+
+struct DerivationOutput
+{
+ std::variant output;
+ StorePath path(const Store & store, std::string_view drvName) const;
};
typedef std::map DerivationOutputs;
@@ -35,6 +45,7 @@ struct BasicDerivation
Path builder;
Strings args;
StringPairs env;
+ std::string name;
BasicDerivation() { }
virtual ~BasicDerivation() { };
@@ -45,10 +56,12 @@ struct BasicDerivation
bool isFixedOutput() const;
/* Return the output paths of a derivation. */
- StorePathSet outputPaths() const;
+ StorePathSet outputPaths(const Store & store) const;
/* Return the output names of a derivation. */
StringSet outputNames() const;
+
+ static std::string_view nameFromPath(const StorePath & storePath);
};
struct Derivation : BasicDerivation
@@ -72,7 +85,7 @@ StorePath writeDerivation(ref store,
const Derivation & drv, std::string_view name, RepairFlag repair = NoRepair);
/* Read a derivation from a file. */
-Derivation readDerivation(const Store & store, const Path & drvPath);
+Derivation readDerivation(const Store & store, const Path & drvPath, std::string_view name);
// FIXME: remove
bool isDerivation(const string & fileName);
@@ -89,7 +102,7 @@ bool wantOutput(const string & output, const std::set & wanted);
struct Source;
struct Sink;
-Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv);
+Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name);
void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv);
std::string hashPlaceholder(const std::string & outputName);
diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc
index b963d64d7..a0fc22264 100644
--- a/src/libstore/export-import.cc
+++ b/src/libstore/export-import.cc
@@ -38,9 +38,9 @@ void Store::exportPath(const StorePath & path, Sink & sink)
filesystem corruption from spreading to other machines.
Don't complain if the stored hash is zero (unknown). */
Hash hash = hashSink.currentHash().first;
- if (hash != info->narHash && info->narHash != Hash(*info->narHash.type))
+ if (hash != info->narHash && info->narHash != Hash(info->narHash->type))
throw Error("hash of path '%s' has changed from '%s' to '%s'!",
- printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true));
+ printStorePath(path), info->narHash->to_string(Base32, true), hash.to_string(Base32, true));
teeSink
<< exportMagic
diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index aaed5c218..e74382ed2 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -500,7 +500,7 @@ struct LocalStore::GCState
StorePathSet alive;
bool gcKeepOutputs;
bool gcKeepDerivations;
- unsigned long long bytesInvalidated;
+ uint64_t bytesInvalidated;
bool moveToTrash = true;
bool shouldDelete;
GCState(const GCOptions & options, GCResults & results)
@@ -518,7 +518,7 @@ bool LocalStore::isActiveTempFile(const GCState & state,
void LocalStore::deleteGarbage(GCState & state, const Path & path)
{
- unsigned long long bytesFreed;
+ uint64_t bytesFreed;
deletePath(path, bytesFreed);
state.results.bytesFreed += bytesFreed;
}
@@ -528,7 +528,7 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path)
{
checkInterrupt();
- unsigned long long size = 0;
+ uint64_t size = 0;
auto storePath = maybeParseStorePath(path);
if (storePath && isValidPath(*storePath)) {
@@ -687,7 +687,7 @@ void LocalStore::removeUnusedLinks(const GCState & state)
AutoCloseDir dir(opendir(linksDir.c_str()));
if (!dir) throw SysError("opening directory '%1%'", linksDir);
- long long actualSize = 0, unsharedSize = 0;
+ int64_t actualSize = 0, unsharedSize = 0;
struct dirent * dirent;
while (errno = 0, dirent = readdir(dir.get())) {
@@ -717,10 +717,10 @@ void LocalStore::removeUnusedLinks(const GCState & state)
struct stat st;
if (stat(linksDir.c_str(), &st) == -1)
throw SysError("statting '%1%'", linksDir);
- long long overhead = st.st_blocks * 512ULL;
+ auto overhead = st.st_blocks * 512ULL;
- printInfo(format("note: currently hard linking saves %.2f MiB")
- % ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0)));
+ printInfo("note: currently hard linking saves %.2f MiB",
+ ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0)));
}
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index d47e0b6b5..3406a9331 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -335,9 +335,6 @@ public:
"setuid/setgid bits or with file capabilities."};
#endif
- Setting hashedMirrors{this, {"http://tarballs.nixos.org/"}, "hashed-mirrors",
- "A list of servers used by builtins.fetchurl to fetch files by hash."};
-
Setting minFree{this, 0, "min-free",
"Automatically run the garbage collector when free disk space drops below the specified amount."};
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index a8bd8a972..5d7566121 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -113,7 +113,7 @@ struct LegacySSHStore : public Store
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) {
auto s = readString(conn->from);
- info->narHash = s.empty() ? Hash() : Hash(s);
+ info->narHash = s.empty() ? std::optional{} : Hash{s};
info->ca = parseContentAddressOpt(readString(conn->from));
info->sigs = readStrings(conn->from);
}
@@ -138,7 +138,7 @@ struct LegacySSHStore : public Store
<< cmdAddToStoreNar
<< printStorePath(info.path)
<< (info.deriver ? printStorePath(*info.deriver) : "")
- << info.narHash.to_string(Base16, false);
+ << info.narHash->to_string(Base16, false);
writeStorePaths(*this, conn->to, info.references);
conn->to
<< info.registrationTime
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 4db4f7d6a..e2cd3c0a5 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -560,19 +560,12 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
DerivationOutputs::const_iterator out = drv.outputs.find("out");
if (out == drv.outputs.end())
throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath));
-
- check(
- makeFixedOutputPath(
- out->second.hash->method,
- out->second.hash->hash,
- drvName),
- out->second.path, "out");
}
else {
Hash h = hashDerivationModulo(*this, drv, true);
for (auto & i : drv.outputs)
- check(makeOutputPath(i.first, h, drvName), i.second.path, i.first);
+ check(makeOutputPath(i.first, h, drvName), i.second.path(*this, drv.name), i.first);
}
}
@@ -586,7 +579,7 @@ uint64_t LocalStore::addValidPath(State & state,
state.stmtRegisterValidPath.use()
(printStorePath(info.path))
- (info.narHash.to_string(Base16, true))
+ (info.narHash->to_string(Base16, true))
(info.registrationTime == 0 ? time(0) : info.registrationTime)
(info.deriver ? printStorePath(*info.deriver) : "", (bool) info.deriver)
(info.narSize, info.narSize != 0)
@@ -614,7 +607,7 @@ uint64_t LocalStore::addValidPath(State & state,
state.stmtAddDerivationOutput.use()
(id)
(i.first)
- (printStorePath(i.second.path))
+ (printStorePath(i.second.path(*this, drv.name)))
.exec();
}
}
@@ -686,7 +679,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info)
{
state.stmtUpdatePathInfo.use()
(info.narSize, info.narSize != 0)
- (info.narHash.to_string(Base16, true))
+ (info.narHash->to_string(Base16, true))
(info.ultimate ? 1 : 0, info.ultimate)
(concatStringsSep(" ", info.sigs), !info.sigs.empty())
(renderContentAddress(info.ca), (bool) info.ca)
@@ -850,20 +843,32 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths)
}
-void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths,
- SubstitutablePathInfos & infos)
+void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos)
{
if (!settings.useSubstitutes) return;
for (auto & sub : getDefaultSubstituters()) {
- if (sub->storeDir != storeDir) continue;
for (auto & path : paths) {
- if (infos.count(path)) continue;
- debug("checking substituter '%s' for path '%s'", sub->getUri(), printStorePath(path));
+ auto subPath(path.first);
+
+ // recompute store path so that we can use a different store root
+ if (path.second) {
+ subPath = makeFixedOutputPathFromCA(path.first.name(), *path.second);
+ if (sub->storeDir == storeDir)
+ assert(subPath == path.first);
+ if (subPath != path.first)
+ debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(path.first), sub->printStorePath(subPath), sub->getUri());
+ } else if (sub->storeDir != storeDir) continue;
+
+ debug("checking substituter '%s' for path '%s'", sub->getUri(), sub->printStorePath(subPath));
try {
- auto info = sub->queryPathInfo(path);
+ auto info = sub->queryPathInfo(subPath);
+
+ if (sub->storeDir != storeDir && !(info->isContentAddressed(*sub) && info->references.empty()))
+ continue;
+
auto narInfo = std::dynamic_pointer_cast(
std::shared_ptr(info));
- infos.insert_or_assign(path, SubstitutablePathInfo{
+ infos.insert_or_assign(path.first, SubstitutablePathInfo{
info->deriver,
info->references,
narInfo ? narInfo->fileSize : 0,
@@ -904,7 +909,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
StorePathSet paths;
for (auto & i : infos) {
- assert(i.narHash.type == htSHA256);
+ assert(i.narHash && i.narHash->type == htSHA256);
if (isValidPath_(*state, i.path))
updatePathInfo(*state, i);
else
@@ -1017,7 +1022,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
if (hashResult.first != info.narHash)
throw Error("hash mismatch importing path '%s';\n wanted: %s\n got: %s",
- printStorePath(info.path), info.narHash.to_string(Base32, true), hashResult.first.to_string(Base32, true));
+ printStorePath(info.path), info.narHash->to_string(Base32, true), hashResult.first.to_string(Base32, true));
if (hashResult.second != info.narSize)
throw Error("size mismatch importing path '%s';\n wanted: %s\n got: %s",
@@ -1313,9 +1318,9 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
std::unique_ptr hashSink;
if (!info->ca || !info->references.count(info->path))
- hashSink = std::make_unique(*info->narHash.type);
+ hashSink = std::make_unique(info->narHash->type);
else
- hashSink = std::make_unique(*info->narHash.type, std::string(info->path.hashPart()));
+ hashSink = std::make_unique(info->narHash->type, std::string(info->path.hashPart()));
dumpPath(Store::toRealPath(i), *hashSink);
auto current = hashSink->finish();
@@ -1324,7 +1329,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
logError({
.name = "Invalid hash - path modified",
.hint = hintfmt("path '%s' was modified! expected hash '%s', got '%s'",
- printStorePath(i), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true))
+ printStorePath(i), info->narHash->to_string(Base32, true), current.first.to_string(Base32, true))
});
if (repair) repairPath(i); else errors = true;
} else {
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 56ef1d251..285f46658 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -29,8 +29,8 @@ struct Derivation;
struct OptimiseStats
{
unsigned long filesLinked = 0;
- unsigned long long bytesFreed = 0;
- unsigned long long blocksFreed = 0;
+ uint64_t bytesFreed = 0;
+ uint64_t blocksFreed = 0;
};
@@ -139,7 +139,7 @@ public:
StorePathSet querySubstitutablePaths(const StorePathSet & paths) override;
- void querySubstitutablePathInfos(const StorePathSet & paths,
+ void querySubstitutablePathInfos(const StorePathCAMap & paths,
SubstitutablePathInfos & infos) override;
void addToStore(const ValidPathInfo & info, Source & source,
diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index e68edb38c..7f1b62f26 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -108,9 +108,19 @@ void Store::computeFSClosure(const StorePath & startPath,
}
+std::optional getDerivationCA(const BasicDerivation & drv)
+{
+ auto out = drv.outputs.find("out");
+ if (out != drv.outputs.end()) {
+ if (auto v = std::get_if(&out->second.output))
+ return v->hash;
+ }
+ return std::nullopt;
+}
+
void Store::queryMissing(const std::vector & targets,
StorePathSet & willBuild_, StorePathSet & willSubstitute_, StorePathSet & unknown_,
- unsigned long long & downloadSize_, unsigned long long & narSize_)
+ uint64_t & downloadSize_, uint64_t & narSize_)
{
Activity act(*logger, lvlDebug, actUnknown, "querying info about missing paths");
@@ -122,8 +132,8 @@ void Store::queryMissing(const std::vector & targets,
{
std::unordered_set done;
StorePathSet & unknown, & willSubstitute, & willBuild;
- unsigned long long & downloadSize;
- unsigned long long & narSize;
+ uint64_t & downloadSize;
+ uint64_t & narSize;
};
struct DrvState
@@ -157,7 +167,7 @@ void Store::queryMissing(const std::vector & targets,
auto outPath = parseStorePath(outPathS);
SubstitutablePathInfos infos;
- querySubstitutablePathInfos({outPath}, infos);
+ querySubstitutablePathInfos({{outPath, getDerivationCA(*drv)}}, infos);
if (infos.empty()) {
drvState_->lock()->done = true;
@@ -198,8 +208,8 @@ void Store::queryMissing(const std::vector & targets,
PathSet invalid;
for (auto & j : drv->outputs)
if (wantOutput(j.first, path.outputs)
- && !isValidPath(j.second.path))
- invalid.insert(printStorePath(j.second.path));
+ && !isValidPath(j.second.path(*this, drv->name)))
+ invalid.insert(printStorePath(j.second.path(*this, drv->name)));
if (invalid.empty()) return;
if (settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
@@ -214,7 +224,7 @@ void Store::queryMissing(const std::vector & targets,
if (isValidPath(path.path)) return;
SubstitutablePathInfos infos;
- querySubstitutablePathInfos({path.path}, infos);
+ querySubstitutablePathInfos({{path.path, std::nullopt}}, infos);
if (infos.empty()) {
auto state(state_.lock());
diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc
index d884a131e..59ec164b6 100644
--- a/src/libstore/nar-accessor.cc
+++ b/src/libstore/nar-accessor.cc
@@ -79,14 +79,14 @@ struct NarAccessor : public FSAccessor
parents.top()->isExecutable = true;
}
- void preallocateContents(unsigned long long size) override
+ void preallocateContents(uint64_t size) override
{
assert(size <= std::numeric_limits::max());
parents.top()->size = (uint64_t) size;
parents.top()->start = pos;
}
- void receiveContents(unsigned char * data, unsigned int len) override
+ void receiveContents(unsigned char * data, size_t len) override
{ }
void createSymlink(const Path & path, const string & target) override
diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc
index 012dea6ea..9ddb9957f 100644
--- a/src/libstore/nar-info-disk-cache.cc
+++ b/src/libstore/nar-info-disk-cache.cc
@@ -230,9 +230,9 @@ public:
(std::string(info->path.name()))
(narInfo ? narInfo->url : "", narInfo != 0)
(narInfo ? narInfo->compression : "", narInfo != 0)
- (narInfo && narInfo->fileHash ? narInfo->fileHash.to_string(Base32, true) : "", narInfo && narInfo->fileHash)
+ (narInfo && narInfo->fileHash ? narInfo->fileHash->to_string(Base32, true) : "", narInfo && narInfo->fileHash)
(narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize)
- (info->narHash.to_string(Base32, true))
+ (info->narHash->to_string(Base32, true))
(info->narSize)
(concatStringsSep(" ", info->shortRefs()))
(info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver)
diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc
index 04550ed97..ca471463c 100644
--- a/src/libstore/nar-info.cc
+++ b/src/libstore/nar-info.cc
@@ -7,15 +7,14 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
: ValidPathInfo(StorePath(StorePath::dummy)) // FIXME: hack
{
auto corrupt = [&]() {
- throw Error("NAR info file '%1%' is corrupt", whence);
+ return Error("NAR info file '%1%' is corrupt", whence);
};
auto parseHashField = [&](const string & s) {
try {
return Hash(s);
} catch (BadHash &) {
- corrupt();
- return Hash(); // never reached
+ throw corrupt();
}
};
@@ -25,12 +24,12 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
while (pos < s.size()) {
size_t colon = s.find(':', pos);
- if (colon == std::string::npos) corrupt();
+ if (colon == std::string::npos) throw corrupt();
std::string name(s, pos, colon - pos);
size_t eol = s.find('\n', colon + 2);
- if (eol == std::string::npos) corrupt();
+ if (eol == std::string::npos) throw corrupt();
std::string value(s, colon + 2, eol - colon - 2);
@@ -45,16 +44,16 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
else if (name == "FileHash")
fileHash = parseHashField(value);
else if (name == "FileSize") {
- if (!string2Int(value, fileSize)) corrupt();
+ if (!string2Int(value, fileSize)) throw corrupt();
}
else if (name == "NarHash")
narHash = parseHashField(value);
else if (name == "NarSize") {
- if (!string2Int(value, narSize)) corrupt();
+ if (!string2Int(value, narSize)) throw corrupt();
}
else if (name == "References") {
auto refs = tokenizeString(value, " ");
- if (!references.empty()) corrupt();
+ if (!references.empty()) throw corrupt();
for (auto & r : refs)
references.insert(StorePath(r));
}
@@ -67,7 +66,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
else if (name == "Sig")
sigs.insert(value);
else if (name == "CA") {
- if (ca) corrupt();
+ if (ca) throw corrupt();
// FIXME: allow blank ca or require skipping field?
ca = parseContentAddressOpt(value);
}
@@ -77,7 +76,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
if (compression == "") compression = "bzip2";
- if (!havePath || url.empty() || narSize == 0 || !narHash) corrupt();
+ if (!havePath || url.empty() || narSize == 0 || !narHash) throw corrupt();
}
std::string NarInfo::to_string(const Store & store) const
@@ -87,11 +86,11 @@ std::string NarInfo::to_string(const Store & store) const
res += "URL: " + url + "\n";
assert(compression != "");
res += "Compression: " + compression + "\n";
- assert(fileHash.type == htSHA256);
- res += "FileHash: " + fileHash.to_string(Base32, true) + "\n";
+ assert(fileHash && fileHash->type == htSHA256);
+ res += "FileHash: " + fileHash->to_string(Base32, true) + "\n";
res += "FileSize: " + std::to_string(fileSize) + "\n";
- assert(narHash.type == htSHA256);
- res += "NarHash: " + narHash.to_string(Base32, true) + "\n";
+ assert(narHash && narHash->type == htSHA256);
+ res += "NarHash: " + narHash->to_string(Base32, true) + "\n";
res += "NarSize: " + std::to_string(narSize) + "\n";
res += "References: " + concatStringsSep(" ", shortRefs()) + "\n";
diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh
index 373c33427..eff19f0ef 100644
--- a/src/libstore/nar-info.hh
+++ b/src/libstore/nar-info.hh
@@ -10,7 +10,7 @@ struct NarInfo : ValidPathInfo
{
std::string url;
std::string compression;
- Hash fileHash;
+ std::optional fileHash;
uint64_t fileSize = 0;
std::string system;
diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc
index b2b2412a3..e4b4b6213 100644
--- a/src/libstore/optimise-store.cc
+++ b/src/libstore/optimise-store.cc
@@ -282,7 +282,7 @@ void LocalStore::optimiseStore(OptimiseStats & stats)
}
}
-static string showBytes(unsigned long long bytes)
+static string showBytes(uint64_t bytes)
{
return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str();
}
diff --git a/src/libstore/path.hh b/src/libstore/path.hh
index e43a8b50c..b03a0f69d 100644
--- a/src/libstore/path.hh
+++ b/src/libstore/path.hh
@@ -64,6 +64,8 @@ typedef std::set StorePathSet;
typedef std::vector StorePaths;
typedef std::map OutputPathMap;
+typedef std::map> StorePathCAMap;
+
/* Extension of derivations in the Nix store. */
const std::string drvExtension = ".drv";
diff --git a/src/libstore/references.cc b/src/libstore/references.cc
index 968e2e425..62a3cda61 100644
--- a/src/libstore/references.cc
+++ b/src/libstore/references.cc
@@ -76,8 +76,8 @@ void RefScanSink::operator () (const unsigned char * data, size_t len)
}
-PathSet scanForReferences(const string & path,
- const PathSet & refs, HashResult & hash)
+std::pair scanForReferences(const string & path,
+ const PathSet & refs)
{
RefScanSink refsSink;
HashSink hashSink { htSHA256 };
@@ -111,9 +111,9 @@ PathSet scanForReferences(const string & path,
found.insert(j->second);
}
- hash = hashSink.finish();
+ auto hash = hashSink.finish();
- return found;
+ return std::pair(found, hash);
}
diff --git a/src/libstore/references.hh b/src/libstore/references.hh
index c38bdd720..598a3203a 100644
--- a/src/libstore/references.hh
+++ b/src/libstore/references.hh
@@ -5,8 +5,7 @@
namespace nix {
-PathSet scanForReferences(const Path & path, const PathSet & refs,
- HashResult & hash);
+std::pair scanForReferences(const Path & path, const PathSet & refs);
struct RewritingSink : Sink
{
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 3b3cdbc02..1200ab200 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -39,13 +39,45 @@ void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths
out << store.printStorePath(i);
}
-
StorePath read(const Store & store, Source & from, Proxy _)
{
auto path = readString(from);
return store.parseStorePath(path);
}
+StorePathCAMap readStorePathCAMap(const Store & store, Source & from)
+{
+ StorePathCAMap paths;
+ auto count = readNum(from);
+ while (count--)
+ paths.insert_or_assign(store.parseStorePath(readString(from)), parseContentAddressOpt(readString(from)));
+ return paths;
+}
+
+void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap & paths)
+{
+ out << paths.size();
+ for (auto & i : paths) {
+ out << store.printStorePath(i.first);
+ out << renderContentAddress(i.second);
+ }
+}
+
+std::map readOutputPathMap(const Store & store, Source & from)
+{
+ std::map pathMap;
+ auto rawInput = readStrings(from);
+ if (rawInput.size() % 2)
+ throw Error("got an odd number of elements from the daemon when trying to read a output path map");
+ auto curInput = rawInput.begin();
+ while (curInput != rawInput.end()) {
+ auto thisKey = *curInput++;
+ auto thisValue = *curInput++;
+ pathMap.emplace(thisKey, store.parseStorePath(thisValue));
+ }
+ return pathMap;
+}
+
void write(const Store & store, Sink & out, const StorePath & storePath)
{
@@ -323,18 +355,17 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths)
}
-void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths,
- SubstitutablePathInfos & infos)
+void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, SubstitutablePathInfos & infos)
{
- if (paths.empty()) return;
+ if (pathsMap.empty()) return;
auto conn(getConnection());
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) {
- for (auto & i : paths) {
+ for (auto & i : pathsMap) {
SubstitutablePathInfo info;
- conn->to << wopQuerySubstitutablePathInfo << printStorePath(i);
+ conn->to << wopQuerySubstitutablePathInfo << printStorePath(i.first);
conn.processStderr();
unsigned int reply = readInt(conn->from);
if (reply == 0) continue;
@@ -344,13 +375,19 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths,
info.references = readStorePaths(*this, conn->from);
info.downloadSize = readLongLong(conn->from);
info.narSize = readLongLong(conn->from);
- infos.insert_or_assign(i, std::move(info));
+ infos.insert_or_assign(i.first, std::move(info));
}
} else {
conn->to << wopQuerySubstitutablePathInfos;
- writeStorePaths(*this, conn->to, paths);
+ if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 22) {
+ StorePathSet paths;
+ for (auto & path : pathsMap)
+ paths.insert(path.first);
+ writeStorePaths(*this, conn->to, paths);
+ } else
+ writeStorePathCAMap(*this, conn->to, pathsMap);
conn.processStderr();
size_t count = readNum(conn->from);
for (size_t n = 0; n < count; n++) {
@@ -489,14 +526,89 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
conn->to << wopAddToStoreNar
<< printStorePath(info.path)
<< (info.deriver ? printStorePath(*info.deriver) : "")
- << info.narHash.to_string(Base16, false);
+ << info.narHash->to_string(Base16, false);
writeStorePaths(*this, conn->to, info.references);
conn->to << info.registrationTime << info.narSize
<< info.ultimate << info.sigs << renderContentAddress(info.ca)
<< repair << !checkSigs;
- bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21;
- if (!tunnel) copyNAR(source, conn->to);
- conn.processStderr(0, tunnel ? &source : nullptr);
+
+ if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 23) {
+
+ std::exception_ptr ex;
+
+ struct FramedSink : BufferedSink
+ {
+ ConnectionHandle & conn;
+ std::exception_ptr & ex;
+
+ FramedSink(ConnectionHandle & conn, std::exception_ptr & ex) : conn(conn), ex(ex)
+ { }
+
+ ~FramedSink()
+ {
+ try {
+ conn->to << 0;
+ conn->to.flush();
+ } catch (...) {
+ ignoreException();
+ }
+ }
+
+ void write(const unsigned char * data, size_t len) override
+ {
+ /* Don't send more data if the remote has
+ encountered an error. */
+ if (ex) {
+ auto ex2 = ex;
+ ex = nullptr;
+ std::rethrow_exception(ex2);
+ }
+ conn->to << len;
+ conn->to(data, len);
+ };
+ };
+
+ /* Handle log messages / exceptions from the remote on a
+ separate thread. */
+ std::thread stderrThread([&]()
+ {
+ try {
+ conn.processStderr();
+ } catch (...) {
+ ex = std::current_exception();
+ }
+ });
+
+ Finally joinStderrThread([&]()
+ {
+ if (stderrThread.joinable()) {
+ stderrThread.join();
+ if (ex) {
+ try {
+ std::rethrow_exception(ex);
+ } catch (...) {
+ ignoreException();
+ }
+ }
+ }
+ });
+
+ {
+ FramedSink sink(conn, ex);
+ copyNAR(source, sink);
+ sink.flush();
+ }
+
+ stderrThread.join();
+ if (ex)
+ std::rethrow_exception(ex);
+
+ } else if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21) {
+ conn.processStderr(0, &source);
+ } else {
+ copyNAR(source, conn->to);
+ conn.processStderr(0, nullptr);
+ }
}
}
@@ -698,7 +810,7 @@ void RemoteStore::addSignatures(const StorePath & storePath, const StringSet & s
void RemoteStore::queryMissing(const std::vector & targets,
StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
- unsigned long long & downloadSize, unsigned long long & narSize)
+ uint64_t & downloadSize, uint64_t & narSize)
{
{
auto conn(getConnection());
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index b39a8de48..4b093ad3b 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -56,7 +56,7 @@ public:
StorePathSet querySubstitutablePaths(const StorePathSet & paths) override;
- void querySubstitutablePathInfos(const StorePathSet & paths,
+ void querySubstitutablePathInfos(const StorePathCAMap & paths,
SubstitutablePathInfos & infos) override;
void addToStore(const ValidPathInfo & info, Source & nar,
@@ -94,7 +94,7 @@ public:
void queryMissing(const std::vector & targets,
StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
- unsigned long long & downloadSize, unsigned long long & narSize) override;
+ uint64_t & downloadSize, uint64_t & narSize) override;
void connect() override;
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 1b7dff085..67935f3ba 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -343,13 +343,10 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
std::chrono::duration_cast(now2 - now1)
.count();
- auto size = istream->tellg();
-
- printInfo("uploaded 's3://%s/%s' (%d bytes) in %d ms",
- bucketName, path, size, duration);
+ printInfo("uploaded 's3://%s/%s' in %d ms",
+ bucketName, path, duration);
stats.putTimeMs += duration;
- stats.putBytes += size;
stats.put++;
}
diff --git a/src/libstore/s3-binary-cache-store.hh b/src/libstore/s3-binary-cache-store.hh
index 4d43fe4d2..b2b75d498 100644
--- a/src/libstore/s3-binary-cache-store.hh
+++ b/src/libstore/s3-binary-cache-store.hh
@@ -19,7 +19,6 @@ public:
struct Stats
{
std::atomic put{0};
- std::atomic putBytes{0};
std::atomic putTimeMs{0};
std::atomic get{0};
std::atomic getBytes{0};
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index a624fafdd..f46e22874 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -193,6 +193,23 @@ StorePath Store::makeFixedOutputPath(
}
}
+// FIXME Put this somewhere?
+template struct overloaded : Ts... { using Ts::operator()...; };
+template overloaded(Ts...) -> overloaded;
+
+StorePath Store::makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca,
+ const StorePathSet & references, bool hasSelfReference) const
+{
+ // New template
+ return std::visit(overloaded {
+ [&](TextHash th) {
+ return makeTextPath(name, th.hash, references);
+ },
+ [&](FixedOutputHash fsh) {
+ return makeFixedOutputPath(fsh.method, fsh.hash, name, references, hasSelfReference);
+ }
+ }, ca);
+}
StorePath Store::makeTextPath(std::string_view name, const Hash & hash,
const StorePathSet & references) const
@@ -549,7 +566,7 @@ string Store::makeValidityRegistration(const StorePathSet & paths,
auto info = queryPathInfo(i);
if (showHash) {
- s += info->narHash.to_string(Base16, false) + "\n";
+ s += info->narHash->to_string(Base16, false) + "\n";
s += (format("%1%\n") % info->narSize).str();
}
@@ -581,7 +598,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store
auto info = queryPathInfo(storePath);
jsonPath
- .attr("narHash", info->narHash.to_string(hashBase, true))
+ .attr("narHash", info->narHash->to_string(hashBase, true))
.attr("narSize", info->narSize);
{
@@ -624,7 +641,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store
if (!narInfo->url.empty())
jsonPath.attr("url", narInfo->url);
if (narInfo->fileHash)
- jsonPath.attr("downloadHash", narInfo->fileHash.to_string(hashBase, true));
+ jsonPath.attr("downloadHash", narInfo->fileHash->to_string(hashBase, true));
if (narInfo->fileSize)
jsonPath.attr("downloadSize", narInfo->fileSize);
if (showClosureSize)
@@ -700,6 +717,15 @@ void copyStorePath(ref srcStore, ref dstStore,
uint64_t total = 0;
+ // recompute store path on the chance dstStore does it differently
+ if (info->ca && info->references.empty()) {
+ auto info2 = make_ref(*info);
+ info2->path = dstStore->makeFixedOutputPathFromCA(info->path.name(), *info->ca);
+ if (dstStore->storeDir == srcStore->storeDir)
+ assert(info->path == info2->path);
+ info = info2;
+ }
+
if (!info->narHash) {
StringSink sink;
srcStore->narFromPath({storePath}, sink);
@@ -735,16 +761,20 @@ void copyStorePath(ref srcStore, ref dstStore,
}
-void copyPaths(ref srcStore, ref dstStore, const StorePathSet & storePaths,
+std::map copyPaths(ref srcStore, ref dstStore, const StorePathSet & storePaths,
RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute)
{
auto valid = dstStore->queryValidPaths(storePaths, substitute);
- PathSet missing;
+ StorePathSet missing;
for (auto & path : storePaths)
- if (!valid.count(path)) missing.insert(srcStore->printStorePath(path));
+ if (!valid.count(path)) missing.insert(path);
- if (missing.empty()) return;
+ std::map pathsMap;
+ for (auto & path : storePaths)
+ pathsMap.insert_or_assign(path, path);
+
+ if (missing.empty()) return pathsMap;
Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size()));
@@ -759,30 +789,49 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st
ThreadPool pool;
- processGraph(pool,
- PathSet(missing.begin(), missing.end()),
+ processGraph(pool,
+ StorePathSet(missing.begin(), missing.end()),
- [&](const Path & storePath) {
- if (dstStore->isValidPath(dstStore->parseStorePath(storePath))) {
+ [&](const StorePath & storePath) {
+ auto info = srcStore->queryPathInfo(storePath);
+ auto storePathForDst = storePath;
+ if (info->ca && info->references.empty()) {
+ storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca);
+ if (dstStore->storeDir == srcStore->storeDir)
+ assert(storePathForDst == storePath);
+ if (storePathForDst != storePath)
+ debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri());
+ }
+ pathsMap.insert_or_assign(storePath, storePathForDst);
+
+ if (dstStore->isValidPath(storePath)) {
nrDone++;
showProgress();
- return PathSet();
+ return StorePathSet();
}
- auto info = srcStore->queryPathInfo(srcStore->parseStorePath(storePath));
-
bytesExpected += info->narSize;
act.setExpected(actCopyPath, bytesExpected);
- return srcStore->printStorePathSet(info->references);
+ return info->references;
},
- [&](const Path & storePathS) {
+ [&](const StorePath & storePath) {
checkInterrupt();
- auto storePath = dstStore->parseStorePath(storePathS);
+ auto info = srcStore->queryPathInfo(storePath);
- if (!dstStore->isValidPath(storePath)) {
+ auto storePathForDst = storePath;
+ if (info->ca && info->references.empty()) {
+ storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca);
+ if (dstStore->storeDir == srcStore->storeDir)
+ assert(storePathForDst == storePath);
+ if (storePathForDst != storePath)
+ debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri());
+ }
+ pathsMap.insert_or_assign(storePath, storePathForDst);
+
+ if (!dstStore->isValidPath(storePathForDst)) {
MaintainCount mc(nrRunning);
showProgress();
try {
@@ -791,7 +840,7 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st
nrFailed++;
if (!settings.keepGoing)
throw e;
- logger->log(lvlError, fmt("could not copy %s: %s", storePathS, e.what()));
+ logger->log(lvlError, fmt("could not copy %s: %s", dstStore->printStorePath(storePath), e.what()));
showProgress();
return;
}
@@ -800,6 +849,8 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st
nrDone++;
showProgress();
});
+
+ return pathsMap;
}
@@ -865,7 +916,7 @@ std::string ValidPathInfo::fingerprint(const Store & store) const
store.printStorePath(path));
return
"1;" + store.printStorePath(path) + ";"
- + narHash.to_string(Base32, true) + ";"
+ + narHash->to_string(Base32, true) + ";"
+ std::to_string(narSize) + ";"
+ concatStringsSep(",", store.printStorePathSet(references));
}
@@ -876,10 +927,6 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey)
sigs.insert(secretKey.signDetached(fingerprint(store)));
}
-// FIXME Put this somewhere?
-template struct overloaded : Ts... { using Ts::operator()...; };
-template overloaded(Ts...) -> overloaded;
-
bool ValidPathInfo::isContentAddressed(const Store & store) const
{
if (! ca) return false;
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index a83a2ff10..3f8866a2c 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -85,7 +85,7 @@ struct GCOptions
StorePathSet pathsToDelete;
/* Stop after at least `maxFreed' bytes have been freed. */
- unsigned long long maxFreed{std::numeric_limits::max()};
+ uint64_t maxFreed{std::numeric_limits::max()};
};
@@ -97,7 +97,7 @@ struct GCResults
/* For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the
number of bytes that would be or was freed. */
- unsigned long long bytesFreed = 0;
+ uint64_t bytesFreed = 0;
};
@@ -105,8 +105,8 @@ struct SubstitutablePathInfo
{
std::optional deriver;
StorePathSet references;
- unsigned long long downloadSize; /* 0 = unknown or inapplicable */
- unsigned long long narSize; /* 0 = unknown */
+ uint64_t downloadSize; /* 0 = unknown or inapplicable */
+ uint64_t narSize; /* 0 = unknown */
};
typedef std::map SubstitutablePathInfos;
@@ -115,7 +115,8 @@ struct ValidPathInfo
{
StorePath path;
std::optional deriver;
- Hash narHash;
+ // TODO document this
+ std::optional narHash;
StorePathSet references;
time_t registrationTime = 0;
uint64_t narSize = 0; // 0 = unknown
@@ -343,7 +344,11 @@ public:
bool hasSelfReference = false) const;
StorePath makeTextPath(std::string_view name, const Hash & hash,
- const StorePathSet & references) const;
+ const StorePathSet & references = {}) const;
+
+ StorePath makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca,
+ const StorePathSet & references = {},
+ bool hasSelfReference = false) const;
/* This is the preparatory part of addToStore(); it computes the
store path to which srcPath is to be copied. Returns the store
@@ -441,9 +446,10 @@ public:
virtual StorePathSet querySubstitutablePaths(const StorePathSet & paths) { return {}; };
/* Query substitute info (i.e. references, derivers and download
- sizes) of a set of paths. If a path does not have substitute
- info, it's omitted from the resulting ‘infos’ map. */
- virtual void querySubstitutablePathInfos(const StorePathSet & paths,
+ sizes) of a map of paths to their optional ca values. If a path
+ does not have substitute info, it's omitted from the resulting
+ ‘infos’ map. */
+ virtual void querySubstitutablePathInfos(const StorePathCAMap & paths,
SubstitutablePathInfos & infos) { return; };
/* Import a path into the store. */
@@ -615,7 +621,7 @@ public:
that will be substituted. */
virtual void queryMissing(const std::vector & targets,
StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
- unsigned long long & downloadSize, unsigned long long & narSize);
+ uint64_t & downloadSize, uint64_t & narSize);
/* Sort a set of paths topologically under the references
relation. If p refers to q, then p precedes q in this list. */
@@ -745,11 +751,13 @@ void copyStorePath(ref srcStore, ref dstStore,
/* Copy store paths from one store to another. The paths may be copied
- in parallel. They are copied in a topologically sorted order
- (i.e. if A is a reference of B, then A is copied before B), but
- the set of store paths is not automatically closed; use
- copyClosure() for that. */
-void copyPaths(ref srcStore, ref dstStore, const StorePathSet & storePaths,
+ in parallel. They are copied in a topologically sorted order (i.e.
+ if A is a reference of B, then A is copied before B), but the set
+ of store paths is not automatically closed; use copyClosure() for
+ that. Returns a map of what each path was copied to the dstStore
+ as. */
+std::map copyPaths(ref srcStore, ref dstStore,
+ const StorePathSet & storePaths,
RepairFlag repair = NoRepair,
CheckSigsFlag checkSigs = CheckSigs,
SubstituteFlag substitute = NoSubstitute);
@@ -848,4 +856,6 @@ std::optional decodeValidPathInfo(
/* Split URI into protocol+hierarchy part and its parameter set. */
std::pair splitUriAndParams(const std::string & uri);
+std::optional getDerivationCA(const BasicDerivation & drv);
+
}
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index 6a6e29640..ad5854c85 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -6,7 +6,7 @@ namespace nix {
#define WORKER_MAGIC_1 0x6e697863
#define WORKER_MAGIC_2 0x6478696f
-#define PROTOCOL_VERSION 0x116
+#define PROTOCOL_VERSION 0x117
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
@@ -122,4 +122,10 @@ StorePath read(const Store & store, Source & from, Proxy _);
void write(const Store & store, Sink & out, const StorePath & storePath);
+StorePathCAMap readStorePathCAMap(const Store & store, Source & from);
+
+void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap & paths);
+
+void writeOutputPathMap(const Store & store, Sink & out, const OutputPathMap & paths);
+
}
diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc
index 51c88537e..ce7cf9754 100644
--- a/src/libutil/archive.cc
+++ b/src/libutil/archive.cc
@@ -150,17 +150,17 @@ static void skipGeneric(Source & source)
static void parseContents(ParseSink & sink, Source & source, const Path & path)
{
- unsigned long long size = readLongLong(source);
+ uint64_t size = readLongLong(source);
sink.preallocateContents(size);
- unsigned long long left = size;
+ uint64_t left = size;
std::vector buf(65536);
while (left) {
checkInterrupt();
auto n = buf.size();
- if ((unsigned long long)n > left) n = left;
+ if ((uint64_t)n > left) n = left;
source(buf.data(), n);
sink.receiveContents(buf.data(), n);
left -= n;
@@ -323,7 +323,7 @@ struct RestoreSink : ParseSink
throw SysError("fchmod");
}
- void preallocateContents(unsigned long long len)
+ void preallocateContents(uint64_t len)
{
#if HAVE_POSIX_FALLOCATE
if (len) {
@@ -338,7 +338,7 @@ struct RestoreSink : ParseSink
#endif
}
- void receiveContents(unsigned char * data, unsigned int len)
+ void receiveContents(unsigned char * data, size_t len)
{
writeFull(fd.get(), data, len);
}
diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh
index 57780d16a..5665732d2 100644
--- a/src/libutil/archive.hh
+++ b/src/libutil/archive.hh
@@ -57,8 +57,8 @@ struct ParseSink
virtual void createRegularFile(const Path & path) { };
virtual void isExecutable() { };
- virtual void preallocateContents(unsigned long long size) { };
- virtual void receiveContents(unsigned char * data, unsigned int len) { };
+ virtual void preallocateContents(uint64_t size) { };
+ virtual void receiveContents(unsigned char * data, size_t len) { };
virtual void createSymlink(const Path & path, const string & target) { };
};
@@ -77,7 +77,7 @@ struct RetrieveRegularNARSink : ParseSink
regular = false;
}
- void receiveContents(unsigned char * data, unsigned int len)
+ void receiveContents(unsigned char * data, size_t len)
{
sink(data, len);
}
diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc
index 5578a618e..2b0390da4 100644
--- a/src/libutil/hash.cc
+++ b/src/libutil/hash.cc
@@ -15,19 +15,22 @@
namespace nix {
+static size_t regularHashSize(HashType type) {
+ switch (type) {
+ case htMD5: return md5HashSize;
+ case htSHA1: return sha1HashSize;
+ case htSHA256: return sha256HashSize;
+ case htSHA512: return sha512HashSize;
+ }
+ abort();
+}
std::set hashTypes = { "md5", "sha1", "sha256", "sha512" };
void Hash::init()
{
- assert(type);
- switch (*type) {
- case htMD5: hashSize = md5HashSize; break;
- case htSHA1: hashSize = sha1HashSize; break;
- case htSHA256: hashSize = sha256HashSize; break;
- case htSHA512: hashSize = sha512HashSize; break;
- }
+ hashSize = regularHashSize(type);
assert(hashSize <= maxHashSize);
memset(hash, 0, maxHashSize);
}
@@ -108,17 +111,11 @@ string printHash16or32(const Hash & hash)
}
-HashType assertInitHashType(const Hash & h)
-{
- assert(h.type);
- return *h.type;
-}
-
std::string Hash::to_string(Base base, bool includeType) const
{
std::string s;
if (base == SRI || includeType) {
- s += printHashType(assertInitHashType(*this));
+ s += printHashType(type);
s += base == SRI ? '-' : ':';
}
switch (base) {
@@ -139,60 +136,66 @@ std::string Hash::to_string(Base base, bool includeType) const
Hash::Hash(std::string_view s, HashType type) : Hash(s, std::optional { type }) { }
Hash::Hash(std::string_view s) : Hash(s, std::optional{}) { }
-Hash::Hash(std::string_view s, std::optional type)
- : type(type)
+Hash::Hash(std::string_view original, std::optional optType)
{
+ auto rest = original;
+
size_t pos = 0;
bool isSRI = false;
- auto sep = s.find(':');
- if (sep == string::npos) {
- sep = s.find('-');
- if (sep != string::npos) {
- isSRI = true;
- } else if (! type)
- throw BadHash("hash '%s' does not include a type", s);
+ // Parse the has type before the separater, if there was one.
+ std::optional optParsedType;
+ {
+ auto sep = rest.find(':');
+ if (sep == std::string_view::npos) {
+ sep = rest.find('-');
+ if (sep != std::string_view::npos)
+ isSRI = true;
+ }
+ if (sep != std::string_view::npos) {
+ auto hashRaw = rest.substr(0, sep);
+ optParsedType = parseHashType(hashRaw);
+ rest = rest.substr(sep + 1);
+ }
}
- if (sep != string::npos) {
- string hts = string(s, 0, sep);
- this->type = parseHashType(hts);
- if (!this->type)
- throw BadHash("unknown hash type '%s'", hts);
- if (type && type != this->type)
- throw BadHash("hash '%s' should have type '%s'", s, printHashType(*type));
- pos = sep + 1;
+ // Either the string or user must provide the type, if they both do they
+ // must agree.
+ if (!optParsedType && !optType) {
+ throw BadHash("hash '%s' does not include a type, nor is the type otherwise known from context.", rest);
+ } else {
+ this->type = optParsedType ? *optParsedType : *optType;
+ if (optParsedType && optType && *optParsedType != *optType)
+ throw BadHash("hash '%s' should have type '%s'", original, printHashType(*optType));
}
init();
- size_t size = s.size() - pos;
-
- if (!isSRI && size == base16Len()) {
+ if (!isSRI && rest.size() == base16Len()) {
auto parseHexDigit = [&](char c) {
if (c >= '0' && c <= '9') return c - '0';
if (c >= 'A' && c <= 'F') return c - 'A' + 10;
if (c >= 'a' && c <= 'f') return c - 'a' + 10;
- throw BadHash("invalid base-16 hash '%s'", s);
+ throw BadHash("invalid base-16 hash '%s'", original);
};
for (unsigned int i = 0; i < hashSize; i++) {
hash[i] =
- parseHexDigit(s[pos + i * 2]) << 4
- | parseHexDigit(s[pos + i * 2 + 1]);
+ parseHexDigit(rest[pos + i * 2]) << 4
+ | parseHexDigit(rest[pos + i * 2 + 1]);
}
}
- else if (!isSRI && size == base32Len()) {
+ else if (!isSRI && rest.size() == base32Len()) {
- for (unsigned int n = 0; n < size; ++n) {
- char c = s[pos + size - n - 1];
+ for (unsigned int n = 0; n < rest.size(); ++n) {
+ char c = rest[rest.size() - n - 1];
unsigned char digit;
for (digit = 0; digit < base32Chars.size(); ++digit) /* !!! slow */
if (base32Chars[digit] == c) break;
if (digit >= 32)
- throw BadHash("invalid base-32 hash '%s'", s);
+ throw BadHash("invalid base-32 hash '%s'", original);
unsigned int b = n * 5;
unsigned int i = b / 8;
unsigned int j = b % 8;
@@ -202,21 +205,21 @@ Hash::Hash(std::string_view s, std::optional type)
hash[i + 1] |= digit >> (8 - j);
} else {
if (digit >> (8 - j))
- throw BadHash("invalid base-32 hash '%s'", s);
+ throw BadHash("invalid base-32 hash '%s'", original);
}
}
}
- else if (isSRI || size == base64Len()) {
- auto d = base64Decode(s.substr(pos));
+ else if (isSRI || rest.size() == base64Len()) {
+ auto d = base64Decode(rest);
if (d.size() != hashSize)
- throw BadHash("invalid %s hash '%s'", isSRI ? "SRI" : "base-64", s);
+ throw BadHash("invalid %s hash '%s'", isSRI ? "SRI" : "base-64", original);
assert(hashSize);
memcpy(hash, d.data(), hashSize);
}
else
- throw BadHash("hash '%s' has wrong length for hash type '%s'", s, printHashType(*type));
+ throw BadHash("hash '%s' has wrong length for hash type '%s'", rest, printHashType(this->type));
}
Hash newHashAllowEmpty(std::string hashStr, std::optional ht)
@@ -269,7 +272,7 @@ static void finish(HashType ht, Ctx & ctx, unsigned char * hash)
}
-Hash hashString(HashType ht, const string & s)
+Hash hashString(HashType ht, std::string_view s)
{
Ctx ctx;
Hash hash(ht);
@@ -336,7 +339,7 @@ HashResult hashPath(
Hash compressHash(const Hash & hash, unsigned int newSize)
{
- Hash h;
+ Hash h(hash.type);
h.hashSize = newSize;
for (unsigned int i = 0; i < hash.hashSize; ++i)
h.hash[i % newSize] ^= hash.hash[i];
@@ -344,7 +347,7 @@ Hash compressHash(const Hash & hash, unsigned int newSize)
}
-std::optional parseHashTypeOpt(const string & s)
+std::optional parseHashTypeOpt(std::string_view s)
{
if (s == "md5") return htMD5;
else if (s == "sha1") return htSHA1;
@@ -353,7 +356,7 @@ std::optional parseHashTypeOpt(const string & s)
else return std::optional {};
}
-HashType parseHashType(const string & s)
+HashType parseHashType(std::string_view s)
{
auto opt_h = parseHashTypeOpt(s);
if (opt_h)
diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh
index ad6093fca..abcd58f24 100644
--- a/src/libutil/hash.hh
+++ b/src/libutil/hash.hh
@@ -27,14 +27,11 @@ enum Base : int { Base64, Base32, Base16, SRI };
struct Hash
{
- static const unsigned int maxHashSize = 64;
- unsigned int hashSize = 0;
- unsigned char hash[maxHashSize] = {};
+ constexpr static size_t maxHashSize = 64;
+ size_t hashSize = 0;
+ uint8_t hash[maxHashSize] = {};
- std::optional type = {};
-
- /* Create an unset hash object. */
- Hash() { };
+ HashType type;
/* Create a zero-filled hash object. */
Hash(HashType type) : type(type) { init(); };
@@ -107,14 +104,14 @@ Hash newHashAllowEmpty(std::string hashStr, std::optional ht);
string printHash16or32(const Hash & hash);
/* Compute the hash of the given string. */
-Hash hashString(HashType ht, const string & s);
+Hash hashString(HashType ht, std::string_view s);
/* Compute the hash of the given file. */
Hash hashFile(HashType ht, const Path & path);
/* Compute the hash of the given path. The hash is defined as
(essentially) hashString(ht, dumpPath(path)). */
-typedef std::pair HashResult;
+typedef std::pair HashResult;
HashResult hashPath(HashType ht, const Path & path,
PathFilter & filter = defaultPathFilter);
@@ -123,10 +120,10 @@ HashResult hashPath(HashType ht, const Path & path,
Hash compressHash(const Hash & hash, unsigned int newSize);
/* Parse a string representing a hash type. */
-HashType parseHashType(const string & s);
+HashType parseHashType(std::string_view s);
/* Will return nothing on parse error */
-std::optional parseHashTypeOpt(const string & s);
+std::optional parseHashTypeOpt(std::string_view s);
/* And the reverse. */
string printHashType(HashType ht);
@@ -144,7 +141,7 @@ class HashSink : public BufferedSink, public AbstractHashSink
private:
HashType ht;
Ctx * ctx;
- unsigned long long bytes;
+ uint64_t bytes;
public:
HashSink(HashType ht);
diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index 5d9acf887..c29c6b29b 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -312,14 +312,14 @@ T readNum(Source & source)
source(buf, sizeof(buf));
uint64_t n =
- ((unsigned long long) buf[0]) |
- ((unsigned long long) buf[1] << 8) |
- ((unsigned long long) buf[2] << 16) |
- ((unsigned long long) buf[3] << 24) |
- ((unsigned long long) buf[4] << 32) |
- ((unsigned long long) buf[5] << 40) |
- ((unsigned long long) buf[6] << 48) |
- ((unsigned long long) buf[7] << 56);
+ ((uint64_t) buf[0]) |
+ ((uint64_t) buf[1] << 8) |
+ ((uint64_t) buf[2] << 16) |
+ ((uint64_t) buf[3] << 24) |
+ ((uint64_t) buf[4] << 32) |
+ ((uint64_t) buf[5] << 40) |
+ ((uint64_t) buf[6] << 48) |
+ ((uint64_t) buf[7] << 56);
if (n > std::numeric_limits::max())
throw SerialisationError("serialised integer %d is too large for type '%s'", n, typeid(T).name());
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index a0a8ff4d3..97d278581 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -374,7 +374,7 @@ void writeLine(int fd, string s)
}
-static void _deletePath(int parentfd, const Path & path, unsigned long long & bytesFreed)
+static void _deletePath(int parentfd, const Path & path, uint64_t & bytesFreed)
{
checkInterrupt();
@@ -414,7 +414,7 @@ static void _deletePath(int parentfd, const Path & path, unsigned long long & by
}
}
-static void _deletePath(const Path & path, unsigned long long & bytesFreed)
+static void _deletePath(const Path & path, uint64_t & bytesFreed)
{
Path dir = dirOf(path);
if (dir == "")
@@ -435,12 +435,12 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed)
void deletePath(const Path & path)
{
- unsigned long long dummy;
+ uint64_t dummy;
deletePath(path, dummy);
}
-void deletePath(const Path & path, unsigned long long & bytesFreed)
+void deletePath(const Path & path, uint64_t & bytesFreed)
{
//Activity act(*logger, lvlDebug, format("recursively deleting path '%1%'") % path);
bytesFreed = 0;
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 42130f6dc..6850b5a7a 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -125,7 +125,7 @@ void writeLine(int fd, string s);
second variant returns the number of bytes and blocks freed. */
void deletePath(const Path & path);
-void deletePath(const Path & path, unsigned long long & bytesFreed);
+void deletePath(const Path & path, uint64_t & bytesFreed);
std::string getUserName();
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index f77de56ea..94412042f 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -174,7 +174,7 @@ static void _main(int argc, char * * argv)
else if (*arg == "--run-env") // obsolete
runEnv = true;
- else if (*arg == "--command" || *arg == "--run") {
+ else if (runEnv && (*arg == "--command" || *arg == "--run")) {
if (*arg == "--run")
interactive = false;
envCommand = getArg(*arg, arg, end) + "\nexit";
@@ -192,7 +192,7 @@ static void _main(int argc, char * * argv)
else if (*arg == "--pure") pure = true;
else if (*arg == "--impure") pure = false;
- else if (*arg == "--packages" || *arg == "-p")
+ else if (runEnv && (*arg == "--packages" || *arg == "-p"))
packages = true;
else if (inShebang && *arg == "-i") {
@@ -325,7 +325,7 @@ static void _main(int argc, char * * argv)
auto buildPaths = [&](const std::vector & paths) {
/* Note: we do this even when !printMissing to efficiently
fetch binary cache data. */
- unsigned long long downloadSize, narSize;
+ uint64_t downloadSize, narSize;
StorePathSet willBuild, willSubstitute, unknown;
store->queryMissing(paths,
willBuild, willSubstitute, unknown, downloadSize, narSize);
diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc
index aa5ada3a6..bcf1d8518 100644
--- a/src/nix-collect-garbage/nix-collect-garbage.cc
+++ b/src/nix-collect-garbage/nix-collect-garbage.cc
@@ -67,10 +67,8 @@ static int _main(int argc, char * * argv)
deleteOlderThan = getArg(*arg, arg, end);
}
else if (*arg == "--dry-run") dryRun = true;
- else if (*arg == "--max-freed") {
- long long maxFreed = getIntArg(*arg, arg, end, true);
- options.maxFreed = maxFreed >= 0 ? maxFreed : 0;
- }
+ else if (*arg == "--max-freed")
+ options.maxFreed = std::max(getIntArg(*arg, arg, end, true), (int64_t) 0);
else
return false;
return true;
diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc
index 961e7fb6d..65d8ec6b6 100644
--- a/src/nix-prefetch-url/nix-prefetch-url.cc
+++ b/src/nix-prefetch-url/nix-prefetch-url.cc
@@ -154,7 +154,7 @@ static int _main(int argc, char * * argv)
/* If an expected hash is given, the file may already exist in
the store. */
std::optional expectedHash;
- Hash hash;
+ Hash hash(ht);
std::optional storePath;
if (args.size() == 2) {
expectedHash = Hash(args[1], ht);
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 23bb48d88..7b26970ef 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -77,7 +77,7 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true)
if (i == drv.outputs.end())
throw Error("derivation '%s' does not have an output named '%s'",
store2->printStorePath(path.path), j);
- auto outPath = store2->printStorePath(i->second.path);
+ auto outPath = store2->printStorePath(i->second.path(*store, drv.name));
if (store2) {
if (gcRoot == "")
printGCWarning();
@@ -130,7 +130,7 @@ static void opRealise(Strings opFlags, Strings opArgs)
for (auto & i : opArgs)
paths.push_back(store->followLinksToStorePathWithOutputs(i));
- unsigned long long downloadSize, narSize;
+ uint64_t downloadSize, narSize;
StorePathSet willBuild, willSubstitute, unknown;
store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize);
@@ -219,7 +219,7 @@ static StorePathSet maybeUseOutputs(const StorePath & storePath, bool useOutput,
auto drv = store->derivationFromPath(storePath);
StorePathSet outputs;
for (auto & i : drv.outputs)
- outputs.insert(i.second.path);
+ outputs.insert(i.second.path(*store, drv.name));
return outputs;
}
else return {storePath};
@@ -313,7 +313,7 @@ static void opQuery(Strings opFlags, Strings opArgs)
if (forceRealise) realisePath({i2});
Derivation drv = store->derivationFromPath(i2);
for (auto & j : drv.outputs)
- cout << fmt("%1%\n", store->printStorePath(j.second.path));
+ cout << fmt("%1%\n", store->printStorePath(j.second.path(*store, drv.name)));
}
break;
}
@@ -372,8 +372,8 @@ static void opQuery(Strings opFlags, Strings opArgs)
for (auto & j : maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise)) {
auto info = store->queryPathInfo(j);
if (query == qHash) {
- assert(info->narHash.type == htSHA256);
- cout << fmt("%s\n", info->narHash.to_string(Base32, true));
+ assert(info->narHash && info->narHash->type == htSHA256);
+ cout << fmt("%s\n", info->narHash->to_string(Base32, true));
} else if (query == qSize)
cout << fmt("%d\n", info->narSize);
}
@@ -572,10 +572,8 @@ static void opGC(Strings opFlags, Strings opArgs)
if (*i == "--print-roots") printRoots = true;
else if (*i == "--print-live") options.action = GCOptions::gcReturnLive;
else if (*i == "--print-dead") options.action = GCOptions::gcReturnDead;
- else if (*i == "--max-freed") {
- long long maxFreed = getIntArg(*i, i, opFlags.end(), true);
- options.maxFreed = maxFreed >= 0 ? maxFreed : 0;
- }
+ else if (*i == "--max-freed")
+ options.maxFreed = std::max(getIntArg(*i, i, opFlags.end(), true), (int64_t) 0);
else throw UsageError("bad sub-operation '%1%' in GC", *i);
if (!opArgs.empty()) throw UsageError("no arguments expected");
@@ -725,7 +723,7 @@ static void opVerifyPath(Strings opFlags, Strings opArgs)
auto path = store->followLinksToStorePath(i);
printMsg(lvlTalkative, "checking path '%s'...", store->printStorePath(path));
auto info = store->queryPathInfo(path);
- HashSink sink(*info->narHash.type);
+ HashSink sink(info->narHash->type);
store->narFromPath(path, sink);
auto current = sink.finish();
if (current.first != info->narHash) {
@@ -734,7 +732,7 @@ static void opVerifyPath(Strings opFlags, Strings opArgs)
.hint = hintfmt(
"path '%s' was modified! expected hash '%s', got '%s'",
store->printStorePath(path),
- info->narHash.to_string(Base32, true),
+ info->narHash->to_string(Base32, true),
current.first.to_string(Base32, true))
});
status = 1;
@@ -831,7 +829,7 @@ static void opServe(Strings opFlags, Strings opArgs)
for (auto & path : paths)
if (!path.isDerivation())
paths2.push_back({path});
- unsigned long long downloadSize, narSize;
+ uint64_t downloadSize, narSize;
StorePathSet willBuild, willSubstitute, unknown;
store->queryMissing(paths2,
willBuild, willSubstitute, unknown, downloadSize, narSize);
@@ -864,7 +862,9 @@ static void opServe(Strings opFlags, Strings opArgs)
out << info->narSize // downloadSize
<< info->narSize;
if (GET_PROTOCOL_MINOR(clientVersion) >= 4)
- out << (info->narHash ? info->narHash.to_string(Base32, true) : "") << renderContentAddress(info->ca) << info->sigs;
+ out << (info->narHash ? info->narHash->to_string(Base32, true) : "")
+ << renderContentAddress(info->ca)
+ << info->sigs;
} catch (InvalidPath &) {
}
}
@@ -914,9 +914,9 @@ static void opServe(Strings opFlags, Strings opArgs)
if (!writeAllowed) throw Error("building paths is not allowed");
- auto drvPath = store->parseStorePath(readString(in)); // informational only
+ auto drvPath = store->parseStorePath(readString(in));
BasicDerivation drv;
- readDerivation(in, *store, drv);
+ readDerivation(in, *store, drv, Derivation::nameFromPath(drvPath));
getBuildSettings();
diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc
index f9d6de16e..e183cb8b5 100644
--- a/src/nix/add-to-store.cc
+++ b/src/nix/add-to-store.cc
@@ -9,6 +9,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand
{
Path path;
std::optional namePart;
+ FileIngestionMethod ingestionMethod = FileIngestionMethod::Recursive;
CmdAddToStore()
{
@@ -21,6 +22,13 @@ struct CmdAddToStore : MixDryRun, StoreCommand
.labels = {"name"},
.handler = {&namePart},
});
+
+ addFlag({
+ .longName = "flat",
+ .shortName = 0,
+ .description = "add flat file to the Nix store",
+ .handler = {&ingestionMethod, FileIngestionMethod::Flat},
+ });
}
std::string description() override
@@ -45,12 +53,19 @@ struct CmdAddToStore : MixDryRun, StoreCommand
auto narHash = hashString(htSHA256, *sink.s);
- ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, *namePart));
+ Hash hash = narHash;
+ if (ingestionMethod == FileIngestionMethod::Flat) {
+ HashSink hsink(htSHA256);
+ readFile(path, hsink);
+ hash = hsink.finish().first;
+ }
+
+ ValidPathInfo info(store->makeFixedOutputPath(ingestionMethod, hash, *namePart));
info.narHash = narHash;
info.narSize = sink.s->size();
info.ca = std::optional { FixedOutputHash {
- .method = FileIngestionMethod::Recursive,
- .hash = info.narHash,
+ .method = ingestionMethod,
+ .hash = hash,
} };
if (!dryRun) {
diff --git a/src/nix/build.cc b/src/nix/build.cc
index 0f7e0e123..613cc15eb 100644
--- a/src/nix/build.cc
+++ b/src/nix/build.cc
@@ -9,6 +9,7 @@ using namespace nix;
struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile
{
Path outLink = "result";
+ BuildMode buildMode = bmNormal;
CmdBuild()
{
@@ -26,6 +27,12 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile
.description = "do not create a symlink to the build result",
.handler = {&outLink, Path("")},
});
+
+ addFlag({
+ .longName = "rebuild",
+ .description = "rebuild an already built package and compare the result to the existing store paths",
+ .handler = {&buildMode, bmCheck},
+ });
}
std::string description() override
@@ -53,7 +60,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile
void run(ref store) override
{
- auto buildables = build(store, dryRun ? Realise::Nothing : Realise::Outputs, installables);
+ auto buildables = build(store, dryRun ? Realise::Nothing : Realise::Outputs, installables, buildMode);
if (dryRun) return;
diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc
new file mode 100644
index 000000000..eb3339f5d
--- /dev/null
+++ b/src/nix/bundle.cc
@@ -0,0 +1,129 @@
+#include "command.hh"
+#include "common-args.hh"
+#include "shared.hh"
+#include "store-api.hh"
+#include "fs-accessor.hh"
+
+using namespace nix;
+
+struct CmdBundle : InstallableCommand
+{
+ std::string bundler = "github:matthewbauer/nix-bundle";
+ std::optional outLink;
+
+ CmdBundle()
+ {
+ addFlag({
+ .longName = "bundler",
+ .description = "use custom bundler",
+ .labels = {"flake-url"},
+ .handler = {&bundler},
+ .completer = {[&](size_t, std::string_view prefix) {
+ completeFlakeRef(getStore(), prefix);
+ }}
+ });
+
+ addFlag({
+ .longName = "out-link",
+ .shortName = 'o',
+ .description = "path of the symlink to the build result",
+ .labels = {"path"},
+ .handler = {&outLink},
+ .completer = completePath
+ });
+ }
+
+ std::string description() override
+ {
+ return "bundle an application so that it works outside of the Nix store";
+ }
+
+ Examples examples() override
+ {
+ return {
+ Example{
+ "To bundle Hello:",
+ "nix bundle hello"
+ },
+ };
+ }
+
+ Category category() override { return catSecondary; }
+
+ Strings getDefaultFlakeAttrPaths() override
+ {
+ Strings res{"defaultApp." + settings.thisSystem.get()};
+ for (auto & s : SourceExprCommand::getDefaultFlakeAttrPaths())
+ res.push_back(s);
+ return res;
+ }
+
+ Strings getDefaultFlakeAttrPathPrefixes() override
+ {
+ Strings res{"apps." + settings.thisSystem.get() + ".", "packages"};
+ for (auto & s : SourceExprCommand::getDefaultFlakeAttrPathPrefixes())
+ res.push_back(s);
+ return res;
+ }
+
+ void run(ref store) override
+ {
+ auto evalState = getEvalState();
+
+ auto app = installable->toApp(*evalState);
+ store->buildPaths(app.context);
+
+ auto [bundlerFlakeRef, bundlerName] = parseFlakeRefWithFragment(bundler, absPath("."));
+ const flake::LockFlags lockFlags{ .writeLockFile = false };
+ auto bundler = InstallableFlake(
+ evalState, std::move(bundlerFlakeRef),
+ Strings{bundlerName == "" ? "defaultBundler" : bundlerName},
+ Strings({"bundlers."}), lockFlags);
+
+ Value * arg = evalState->allocValue();
+ evalState->mkAttrs(*arg, 2);
+
+ PathSet context;
+ for (auto & i : app.context)
+ context.insert("=" + store->printStorePath(i.path));
+ mkString(*evalState->allocAttr(*arg, evalState->symbols.create("program")), app.program, context);
+
+ mkString(*evalState->allocAttr(*arg, evalState->symbols.create("system")), settings.thisSystem.get());
+
+ arg->attrs->sort();
+
+ auto vRes = evalState->allocValue();
+ evalState->callFunction(*bundler.toValue(*evalState).first, *arg, *vRes, noPos);
+
+ if (!evalState->isDerivation(*vRes))
+ throw Error("the bundler '%s' does not produce a derivation", bundler.what());
+
+ auto attr1 = vRes->attrs->find(evalState->sDrvPath);
+ if (!attr1)
+ throw Error("the bundler '%s' does not produce a derivation", bundler.what());
+
+ PathSet context2;
+ StorePath drvPath = store->parseStorePath(evalState->coerceToPath(*attr1->pos, *attr1->value, context2));
+
+ auto attr2 = vRes->attrs->find(evalState->sOutPath);
+ if (!attr2)
+ throw Error("the bundler '%s' does not produce a derivation", bundler.what());
+
+ StorePath outPath = store->parseStorePath(evalState->coerceToPath(*attr2->pos, *attr2->value, context2));
+
+ store->buildPaths({{drvPath}});
+
+ auto outPathS = store->printStorePath(outPath);
+
+ auto info = store->queryPathInfo(outPath);
+ if (!info->references.empty())
+ throw Error("'%s' has references; a bundler must not leave any references", outPathS);
+
+ if (!outLink)
+ outLink = baseNameOf(app.program);
+
+ store.dynamic_pointer_cast()->addPermRoot(outPath, absPath(*outLink), true);
+ }
+};
+
+static auto r2 = registerCommand("bundle");
diff --git a/src/nix/command.hh b/src/nix/command.hh
index 856721ebf..bc46a2028 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -5,6 +5,7 @@
#include "common-eval-args.hh"
#include "path.hh"
#include "flake/lockfile.hh"
+#include "store-api.hh"
#include
@@ -185,7 +186,7 @@ static RegisterCommand registerCommand(const std::string & name)
}
Buildables build(ref store, Realise mode,
- std::vector> installables);
+ std::vector> installables, BuildMode bMode = bmNormal);
std::set toStorePaths(ref store,
Realise mode, OperateOn operateOn,
diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index a6d7d6add..a0c119e43 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -68,22 +68,22 @@ BuildEnvironment readEnvironment(const Path & path)
std::smatch match;
- if (std::regex_search(pos, file.cend(), match, declareRegex)) {
+ if (std::regex_search(pos, file.cend(), match, declareRegex, std::regex_constants::match_continuous)) {
pos = match[0].second;
exported.insert(match[1]);
}
- else if (std::regex_search(pos, file.cend(), match, varRegex)) {
+ else if (std::regex_search(pos, file.cend(), match, varRegex, std::regex_constants::match_continuous)) {
pos = match[0].second;
res.env.insert({match[1], Var { .exported = exported.count(match[1]) > 0, .value = match[2] }});
}
- else if (std::regex_search(pos, file.cend(), match, assocArrayRegex)) {
+ else if (std::regex_search(pos, file.cend(), match, assocArrayRegex, std::regex_constants::match_continuous)) {
pos = match[0].second;
res.env.insert({match[1], Var { .associative = true, .value = match[2] }});
}
- else if (std::regex_search(pos, file.cend(), match, functionRegex)) {
+ else if (std::regex_search(pos, file.cend(), match, functionRegex, std::regex_constants::match_continuous)) {
res.bashFunctions = std::string(pos, file.cend());
break;
}
@@ -130,14 +130,16 @@ StorePath getDerivationEnvironment(ref store, const StorePath & drvPath)
drvName += "-env";
for (auto & output : drv.outputs)
drv.env.erase(output.first);
- drv.outputs = {{"out", DerivationOutput { .path = StorePath::dummy }}};
+ drv.outputs = {{"out", DerivationOutput { .output = DerivationOutputInputAddressed { .path = StorePath::dummy }}}};
drv.env["out"] = "";
drv.env["_outputs_saved"] = drv.env["outputs"];
drv.env["outputs"] = "out";
drv.inputSrcs.insert(std::move(getEnvShPath));
Hash h = hashDerivationModulo(*store, drv, true);
auto shellOutPath = store->makeOutputPath("out", h, drvName);
- drv.outputs.insert_or_assign("out", DerivationOutput { .path = shellOutPath });
+ drv.outputs.insert_or_assign("out", DerivationOutput { .output = DerivationOutputInputAddressed {
+ .path = shellOutPath
+ } });
drv.env["out"] = store->printStorePath(shellOutPath);
auto shellDrvPath2 = writeDerivation(store, drv, drvName);
diff --git a/src/nix/flake.cc b/src/nix/flake.cc
index 027a9871e..80d8654bc 100644
--- a/src/nix/flake.cc
+++ b/src/nix/flake.cc
@@ -368,6 +368,21 @@ struct CmdFlakeCheck : FlakeCommand
}
};
+ auto checkBundler = [&](const std::string & attrPath, Value & v, const Pos & pos) {
+ try {
+ state->forceValue(v, pos);
+ if (v.type != tLambda)
+ throw Error("bundler must be a function");
+ if (!v.lambda.fun->formals ||
+ v.lambda.fun->formals->argNames.find(state->symbols.create("program")) == v.lambda.fun->formals->argNames.end() ||
+ v.lambda.fun->formals->argNames.find(state->symbols.create("system")) == v.lambda.fun->formals->argNames.end())
+ throw Error("bundler must take formal arguments 'program' and 'system'");
+ } catch (Error & e) {
+ e.addTrace(pos, hintfmt("while checking the template '%s'", attrPath));
+ throw;
+ }
+ };
+
{
Activity act(*logger, lvlInfo, actUnknown, "evaluating flake");
@@ -490,6 +505,16 @@ struct CmdFlakeCheck : FlakeCommand
*attr.value, *attr.pos);
}
+ else if (name == "defaultBundler")
+ checkBundler(name, vOutput, pos);
+
+ else if (name == "bundlers") {
+ state->forceAttrs(vOutput, pos);
+ for (auto & attr : *vOutput.attrs)
+ checkBundler(fmt("%s.%s", name, attr.name),
+ *attr.value, *attr.pos);
+ }
+
else
warn("unknown flake output '%s'", name);
diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index a13e5a3df..babb0e5fe 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -276,7 +276,7 @@ std::vector, std::string>>
Installable::getCursors(EvalState & state, bool useEvalCache)
{
auto evalCache =
- std::make_shared(false, Hash(), state,
+ std::make_shared(std::nullopt, state,
[&]() { return toValue(state).first; });
return {{evalCache->getRoot(), ""}};
}
@@ -304,8 +304,9 @@ struct InstallableStorePath : Installable
{
if (storePath.isDerivation()) {
std::map outputs;
- for (auto & [name, output] : store->readDerivation(storePath).outputs)
- outputs.emplace(name, output.path);
+ auto drv = store->readDerivation(storePath);
+ for (auto & [name, output] : drv.outputs)
+ outputs.emplace(name, output.path(*store, drv.name));
return {
Buildable {
.drvPath = storePath,
@@ -436,9 +437,11 @@ ref openEvalCache(
std::shared_ptr lockedFlake,
bool useEvalCache)
{
- return ref(std::make_shared(
- useEvalCache && evalSettings.pureEval,
- lockedFlake->getFingerprint(),
+ auto fingerprint = lockedFlake->getFingerprint();
+ return make_ref(
+ useEvalCache && evalSettings.pureEval
+ ? std::optional { std::cref(fingerprint) }
+ : std::nullopt,
state,
[&state, lockedFlake]()
{
@@ -456,7 +459,7 @@ ref openEvalCache(
assert(aOutputs);
return aOutputs->value;
- }));
+ });
}
static std::string showAttrPaths(const std::vector & paths)
@@ -642,7 +645,7 @@ std::shared_ptr SourceExprCommand::parseInstallable(
}
Buildables build(ref store, Realise mode,
- std::vector> installables)
+ std::vector> installables, BuildMode bMode)
{
if (mode == Realise::Nothing)
settings.readOnlyMode = true;
@@ -668,7 +671,7 @@ Buildables build(ref store, Realise mode,
if (mode == Realise::Nothing)
printMissing(store, pathsToBuild, lvlError);
else if (mode == Realise::Outputs)
- store->buildPaths(pathsToBuild);
+ store->buildPaths(pathsToBuild, bMode);
return buildables;
}
diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc
index 712043978..2fe2e2fb2 100644
--- a/src/nix/make-content-addressable.cc
+++ b/src/nix/make-content-addressable.cc
@@ -84,7 +84,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON
info.narSize = sink.s->size();
info.ca = FixedOutputHash {
.method = FileIngestionMethod::Recursive,
- .hash = info.narHash,
+ .hash = *info.narHash,
};
if (!json)
diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc
index 65f73cd94..0c12efaf0 100644
--- a/src/nix/path-info.cc
+++ b/src/nix/path-info.cc
@@ -61,7 +61,7 @@ struct CmdPathInfo : StorePathsCommand, MixJSON
};
}
- void printSize(unsigned long long value)
+ void printSize(uint64_t value)
{
if (!humanReadable) {
std::cout << fmt("\t%11d", value);
diff --git a/src/nix/profile.cc b/src/nix/profile.cc
index c6cd88c49..7dcc0b6d4 100644
--- a/src/nix/profile.cc
+++ b/src/nix/profile.cc
@@ -133,7 +133,7 @@ struct ProfileManifest
info.references = std::move(references);
info.narHash = narHash;
info.narSize = sink.s->size();
- info.ca = FixedOutputHash { .method = FileIngestionMethod::Recursive, .hash = info.narHash };
+ info.ca = FixedOutputHash { .method = FileIngestionMethod::Recursive, .hash = *info.narHash };
auto source = StringSource { *sink.s };
store->addToStore(info, source);
diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index 8eb58f62a..fb9050d0d 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -483,10 +483,10 @@ bool NixRepl::processLine(string line)
but doing it in a child makes it easier to recover from
problems / SIGINT. */
if (runProgram(settings.nixBinDir + "/nix", Strings{"build", "--no-link", drvPath}) == 0) {
- auto drv = readDerivation(*state->store, drvPath);
+ auto drv = readDerivation(*state->store, drvPath, Derivation::nameFromPath(state->store->parseStorePath(drvPath)));
std::cout << std::endl << "this derivation produced the following outputs:" << std::endl;
for (auto & i : drv.outputs)
- std::cout << fmt(" %s -> %s\n", i.first, state->store->printStorePath(i.second.path));
+ std::cout << fmt(" %s -> %s\n", i.first, state->store->printStorePath(i.second.path(*state->store, drv.name)));
}
} else if (command == ":i") {
runProgram(settings.nixBinDir + "/nix-env", Strings{"-i", drvPath});
diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc
index b5434f982..9fd26e2d7 100644
--- a/src/nix/show-derivation.cc
+++ b/src/nix/show-derivation.cc
@@ -69,10 +69,10 @@ struct CmdShowDerivation : InstallablesCommand
auto outputsObj(drvObj.object("outputs"));
for (auto & output : drv.outputs) {
auto outputObj(outputsObj.object(output.first));
- outputObj.attr("path", store->printStorePath(output.second.path));
- if (output.second.hash) {
- outputObj.attr("hashAlgo", output.second.hash->printMethodAlgo());
- outputObj.attr("hash", output.second.hash->hash.to_string(Base16, false));
+ outputObj.attr("path", store->printStorePath(output.second.path(*store, drv.name)));
+ if (auto hash = std::get_if(&output.second.output)) {
+ outputObj.attr("hashAlgo", hash->hash.printMethodAlgo());
+ outputObj.attr("hash", hash->hash.hash.to_string(Base16, false));
}
}
}
diff --git a/src/nix/verify.cc b/src/nix/verify.cc
index ce90b0f6d..fc7a9765c 100644
--- a/src/nix/verify.cc
+++ b/src/nix/verify.cc
@@ -91,15 +91,15 @@ struct CmdVerify : StorePathsCommand
std::unique_ptr hashSink;
if (!info->ca)
- hashSink = std::make_unique(*info->narHash.type);
+ hashSink = std::make_unique(info->narHash->type);
else
- hashSink = std::make_unique(*info->narHash.type, std::string(info->path.hashPart()));
+ hashSink = std::make_unique(info->narHash->type, std::string(info->path.hashPart()));
store->narFromPath(info->path, *hashSink);
auto hash = hashSink->finish();
- if (hash.first != info->narHash) {
+ if (hash.first != *info->narHash) {
corrupted++;
act2.result(resCorruptedPath, store->printStorePath(info->path));
logError({
@@ -107,7 +107,7 @@ struct CmdVerify : StorePathsCommand
.hint = hintfmt(
"path '%s' was modified! expected hash '%s', got '%s'",
store->printStorePath(info->path),
- info->narHash.to_string(Base32, true),
+ info->narHash->to_string(Base32, true),
hash.first.to_string(Base32, true))
});
}
diff --git a/tests/check.sh b/tests/check.sh
index 5f25d04cb..5f4997e28 100644
--- a/tests/check.sh
+++ b/tests/check.sh
@@ -61,30 +61,30 @@ nix-build check.nix -A nondeterministic --no-out-link --repeat 1 2> $TEST_ROOT/l
[ "$status" = "1" ]
grep 'differs from previous round' $TEST_ROOT/log
-path=$(nix-build check.nix -A fetchurl --no-out-link --hashed-mirrors '')
+path=$(nix-build check.nix -A fetchurl --no-out-link)
chmod +w $path
echo foo > $path
chmod -w $path
-nix-build check.nix -A fetchurl --no-out-link --check --hashed-mirrors ''
+nix-build check.nix -A fetchurl --no-out-link --check
# Note: "check" doesn't repair anything, it just compares to the hash stored in the database.
[[ $(cat $path) = foo ]]
-nix-build check.nix -A fetchurl --no-out-link --repair --hashed-mirrors ''
+nix-build check.nix -A fetchurl --no-out-link --repair
[[ $(cat $path) != foo ]]
-nix-build check.nix -A hashmismatch --no-out-link --hashed-mirrors '' || status=$?
+nix-build check.nix -A hashmismatch --no-out-link || status=$?
[ "$status" = "102" ]
echo -n > ./dummy
-nix-build check.nix -A hashmismatch --no-out-link --hashed-mirrors ''
+nix-build check.nix -A hashmismatch --no-out-link
echo 'Hello World' > ./dummy
-nix-build check.nix -A hashmismatch --no-out-link --check --hashed-mirrors '' || status=$?
+nix-build check.nix -A hashmismatch --no-out-link --check || status=$?
[ "$status" = "102" ]
# Multiple failures with --keep-going
nix-build check.nix -A nondeterministic --no-out-link
-nix-build check.nix -A nondeterministic -A hashmismatch --no-out-link --check --keep-going --hashed-mirrors '' || status=$?
+nix-build check.nix -A nondeterministic -A hashmismatch --no-out-link --check --keep-going || status=$?
[ "$status" = "110" ]
diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh
index 9faa5d9f6..cedd796f7 100644
--- a/tests/fetchGit.sh
+++ b/tests/fetchGit.sh
@@ -32,6 +32,8 @@ rev2=$(git -C $repo rev-parse HEAD)
# Fetch a worktree
unset _NIX_FORCE_HTTP
path0=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$TEST_ROOT/worktree).outPath")
+path0_=$(nix eval --impure --raw --expr "(builtins.fetchTree { type = \"git\"; url = file://$TEST_ROOT/worktree; }).outPath")
+[[ $path0 = $path0_ ]]
export _NIX_FORCE_HTTP=1
[[ $(tail -n 1 $path0/hello) = "hello" ]]
@@ -102,6 +104,12 @@ git -C $repo commit -m 'Bla3' -a
path4=$(nix eval --impure --refresh --raw --expr "(builtins.fetchGit file://$repo).outPath")
[[ $path2 = $path4 ]]
+nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; rev = \"$rev2\"; narHash = \"sha256-B5yIPHhEm0eysJKEsO7nqxprh9vcblFxpJG11gXJus1=\"; }).outPath" || status=$?
+[[ "$status" = "102" ]]
+
+path5=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; rev = \"$rev2\"; narHash = \"sha256-Hr8g6AqANb3xqX28eu1XnjK/3ab8Gv6TJSnkb1LezG9=\"; }).outPath")
+[[ $path = $path5 ]]
+
# tarball-ttl should be ignored if we specify a rev
echo delft > $repo/hello
git -C $repo add hello
diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh
index 2535651b0..0f2044342 100644
--- a/tests/fetchurl.sh
+++ b/tests/fetchurl.sh
@@ -5,7 +5,7 @@ clearStore
# Test fetching a flat file.
hash=$(nix-hash --flat --type sha256 ./fetchurl.sh)
-outPath=$(nix-build '' --argstr url file://$(pwd)/fetchurl.sh --argstr sha256 $hash --no-out-link --hashed-mirrors '')
+outPath=$(nix-build '' --argstr url file://$(pwd)/fetchurl.sh --argstr sha256 $hash --no-out-link)
cmp $outPath fetchurl.sh
@@ -14,7 +14,7 @@ clearStore
hash=$(nix hash-file --type sha512 --base64 ./fetchurl.sh)
-outPath=$(nix-build '' --argstr url file://$(pwd)/fetchurl.sh --argstr sha512 $hash --no-out-link --hashed-mirrors '')
+outPath=$(nix-build '' --argstr url file://$(pwd)/fetchurl.sh --argstr sha512 $hash --no-out-link)
cmp $outPath fetchurl.sh
@@ -25,26 +25,24 @@ hash=$(nix hash-file ./fetchurl.sh)
[[ $hash =~ ^sha256- ]]
-outPath=$(nix-build '' --argstr url file://$(pwd)/fetchurl.sh --argstr hash $hash --no-out-link --hashed-mirrors '')
+outPath=$(nix-build '' --argstr url file://$(pwd)/fetchurl.sh --argstr hash $hash --no-out-link)
cmp $outPath fetchurl.sh
-# Test the hashed mirror feature.
+# Test that we can substitute from a different store dir.
clearStore
-hash=$(nix hash-file --type sha512 --base64 ./fetchurl.sh)
-hash32=$(nix hash-file --type sha512 --base16 ./fetchurl.sh)
+other_store=file://$TEST_ROOT/other_store?store=/fnord/store
-mirror=$TEST_ROOT/hashed-mirror
-rm -rf $mirror
-mkdir -p $mirror/sha512
-ln -s $(pwd)/fetchurl.sh $mirror/sha512/$hash32
+hash=$(nix hash-file --type sha256 --base16 ./fetchurl.sh)
-outPath=$(nix-build '' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha512 $hash --no-out-link --hashed-mirrors "file://$mirror")
+storePath=$(nix --store $other_store add-to-store --flat ./fetchurl.sh)
+
+outPath=$(nix-build '' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 $hash --no-out-link --substituters $other_store)
# Test hashed mirrors with an SRI hash.
-nix-build '' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix to-sri --type sha512 $hash) \
- --argstr name bla --no-out-link --hashed-mirrors "file://$mirror"
+nix-build '' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix to-sri --type sha256 $hash) \
+ --no-out-link --substituters $other_store
# Test unpacking a NAR.
rm -rf $TEST_ROOT/archive
diff --git a/tests/filter-source.sh b/tests/filter-source.sh
index 1f8dceee5..ba34d2eac 100644
--- a/tests/filter-source.sh
+++ b/tests/filter-source.sh
@@ -10,10 +10,16 @@ touch $TEST_ROOT/filterin/bak
touch $TEST_ROOT/filterin/bla.c.bak
ln -s xyzzy $TEST_ROOT/filterin/link
-nix-build ./filter-source.nix -o $TEST_ROOT/filterout
+checkFilter() {
+ test ! -e $1/foo/bar
+ test -e $1/xyzzy
+ test -e $1/bak
+ test ! -e $1/bla.c.bak
+ test ! -L $1/link
+}
-test ! -e $TEST_ROOT/filterout/foo/bar
-test -e $TEST_ROOT/filterout/xyzzy
-test -e $TEST_ROOT/filterout/bak
-test ! -e $TEST_ROOT/filterout/bla.c.bak
-test ! -L $TEST_ROOT/filterout/link
+nix-build ./filter-source.nix -o $TEST_ROOT/filterout1
+checkFilter $TEST_ROOT/filterout1
+
+nix-build ./path.nix -o $TEST_ROOT/filterout2
+checkFilter $TEST_ROOT/filterout2
diff --git a/tests/path.nix b/tests/path.nix
new file mode 100644
index 000000000..883c3c41b
--- /dev/null
+++ b/tests/path.nix
@@ -0,0 +1,14 @@
+with import ./config.nix;
+
+mkDerivation {
+ name = "filter";
+ builder = builtins.toFile "builder" "ln -s $input $out";
+ input =
+ builtins.path {
+ path = ((builtins.getEnv "TEST_ROOT") + "/filterin");
+ filter = path: type:
+ type != "symlink"
+ && baseNameOf path != "foo"
+ && !((import ./lang/lib.nix).hasSuffix ".bak" (baseNameOf path));
+ };
+}
diff --git a/tests/tarball.sh b/tests/tarball.sh
index b3ec16d40..88a1a07a0 100644
--- a/tests/tarball.sh
+++ b/tests/tarball.sh
@@ -27,10 +27,13 @@ test_tarball() {
nix-build -o $TEST_ROOT/result -E "import (fetchTarball file://$tarball)"
- nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree file://$tarball)"
- nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; })"
- nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })"
- nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"sha256-xdKv2pq/IiwLSnBBJXW8hNowI4MrdZfW+SYqDQs7Tzc=\"; })" 2>&1 | grep 'NAR hash mismatch in input'
+ nix-build -o $TEST_ROOT/result -E "import (fetchTree file://$tarball)"
+ nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; })"
+ nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })"
+ nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"sha256-xdKv2pq/IiwLSnBBJXW8hNowI4MrdZfW+SYqDQs7Tzc=\"; })" 2>&1 | grep 'NAR hash mismatch in input'
+
+ nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" >&2
+ nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" 2>&1 | grep 'true'
nix-instantiate --eval -E '1 + 2' -I fnord=file://no-such-tarball.tar$ext
nix-instantiate --eval -E 'with ; 1 + 2' -I fnord=file://no-such-tarball$ext