diff --git a/.version b/.version
index 7208c2182..f398a2061 100644
--- a/.version
+++ b/.version
@@ -1 +1 @@
-2.4
\ No newline at end of file
+3.0
\ No newline at end of file
diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index 1fa74a143..9c55526a3 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -370,34 +370,6 @@ false.
- hashed-mirrors
-
- A list of web servers used by
- builtins.fetchurl to obtain files by
- hash. The default is
- http://tarballs.nixos.org/. Given a hash type
- ht and a base-16 hash
- h, Nix will try to download the file
- from
- hashed-mirror/ht/h.
- This allows files to be downloaded even if they have disappeared
- from their original URI. For example, given the default mirror
- http://tarballs.nixos.org/, when building the derivation
-
-
-builtins.fetchurl {
- url = "https://example.org/foo-1.2.3.tar.xz";
- sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae";
-}
-
-
- Nix will attempt to download this file from
- http://tarballs.nixos.org/sha256/2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae
- first. If it is not available there, if will try the original URI.
-
-
-
-
http-connections
The maximum number of parallel TCP connections
diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs
index c412e4640..de60fb939 100644
--- a/perl/lib/Nix/Store.xs
+++ b/perl/lib/Nix/Store.xs
@@ -224,7 +224,7 @@ SV * hashString(char * algo, int base32, char * s)
SV * convertHash(char * algo, char * s, int toBase32)
PPCODE:
try {
- Hash h(s, parseHashType(algo));
+ auto h = Hash::parseAny(s, parseHashType(algo));
string s = h.to_string(toBase32 ? Base32 : Base16, false);
XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
} catch (Error & e) {
@@ -285,7 +285,7 @@ SV * addToStore(char * srcPath, int recursive, char * algo)
SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name)
PPCODE:
try {
- Hash h(hash, parseHashType(algo));
+ auto h = Hash::parseAny(hash, parseHashType(algo));
auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
auto path = store()->makeFixedOutputPath(method, h, name);
XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0)));
diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh
index 00c9d540b..e5cc4d7ed 100644
--- a/scripts/install-multi-user.sh
+++ b/scripts/install-multi-user.sh
@@ -37,6 +37,8 @@ readonly PROFILE_NIX_FILE="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix-
readonly NIX_INSTALLED_NIX="@nix@"
readonly NIX_INSTALLED_CACERT="@cacert@"
+#readonly NIX_INSTALLED_NIX="/nix/store/j8dbv5w6jl34caywh2ygdy88knx1mdf7-nix-2.3.6"
+#readonly NIX_INSTALLED_CACERT="/nix/store/7dxhzymvy330i28ii676fl1pqwcahv2f-nss-cacert-3.49.2"
readonly EXTRACTED_NIX_PATH="$(dirname "$0")"
readonly ROOT_HOME=$(echo ~root)
@@ -69,9 +71,11 @@ uninstall_directions() {
subheader "Uninstalling nix:"
local step=0
- if poly_service_installed_check; then
+ if [ -e /run/systemd/system ] && poly_service_installed_check; then
step=$((step + 1))
poly_service_uninstall_directions "$step"
+ else
+ step=$((step + 1))
fi
for profile_target in "${PROFILE_TARGETS[@]}"; do
@@ -250,7 +254,9 @@ function finish_success {
echo "But fetching the nixpkgs channel failed. (Are you offline?)"
echo "To try again later, run \"sudo -i nix-channel --update nixpkgs\"."
fi
- cat <&2
-elif [ "$(uname -s)" = "Linux" ] && [ -e /run/systemd/system ]; then
+elif [ "$(uname -s)" = "Linux" ]; then
echo "Note: a multi-user installation is possible. See https://nixos.org/nix/manual/#sect-multi-user-installation" >&2
fi
@@ -122,7 +122,7 @@ if [ "$(uname -s)" = "Darwin" ]; then
fi
if [ "$INSTALL_MODE" = "daemon" ]; then
- printf '\e[1;31mSwitching to the Daemon-based Installer\e[0m\n'
+ printf '\e[1;31mSwitching to the Multi-user Installer\e[0m\n'
exec "$self/install-multi-user"
exit 0
fi
diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index e07117496..3579d8fff 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -33,7 +33,7 @@ std::string escapeUri(std::string uri)
static string currentLoad;
-static AutoCloseFD openSlotLock(const Machine & m, unsigned long long slot)
+static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot)
{
return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true);
}
@@ -119,7 +119,7 @@ static int _main(int argc, char * * argv)
bool rightType = false;
Machine * bestMachine = nullptr;
- unsigned long long bestLoad = 0;
+ uint64_t bestLoad = 0;
for (auto & m : machines) {
debug("considering building on remote machine '%s'", m.storeUri);
@@ -130,8 +130,8 @@ static int _main(int argc, char * * argv)
m.mandatoryMet(requiredFeatures)) {
rightType = true;
AutoCloseFD free;
- unsigned long long load = 0;
- for (unsigned long long slot = 0; slot < m.maxJobs; ++slot) {
+ uint64_t load = 0;
+ for (uint64_t slot = 0; slot < m.maxJobs; ++slot) {
auto slotLock = openSlotLock(m, slot);
if (lockFile(slotLock.get(), ltWrite, false)) {
if (!free) {
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 32b09fb0d..0123070d1 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -1257,10 +1257,10 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
try {
lambda.body->eval(*this, env2, v);
} catch (Error & e) {
- addErrorTrace(e, lambda.pos, "while evaluating %s",
- (lambda.name.set()
- ? "'" + (string) lambda.name + "'"
- : "anonymous lambdaction"));
+ addErrorTrace(e, lambda.pos, "while evaluating %s",
+ (lambda.name.set()
+ ? "'" + (string) lambda.name + "'"
+ : "anonymous lambda"));
addErrorTrace(e, pos, "from call site%s", "");
throw;
}
diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh
index 77f3abdeb..c2bb2888b 100644
--- a/src/libexpr/flake/flake.hh
+++ b/src/libexpr/flake/flake.hh
@@ -106,6 +106,6 @@ void emitTreeAttrs(
EvalState & state,
const fetchers::Tree & tree,
const fetchers::Input & input,
- Value & v);
+ Value & v, bool emptyRevFallback = false);
}
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 7bc424d52..9ecf99f17 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -65,7 +65,7 @@ void EvalState::realiseContext(const PathSet & context)
/* For performance, prefetch all substitute info. */
StorePathSet willBuild, willSubstitute, unknown;
- unsigned long long downloadSize, narSize;
+ uint64_t downloadSize, narSize;
store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize);
store->buildPaths(drvs);
@@ -1223,7 +1223,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
string name;
Value * filterFun = nullptr;
auto method = FileIngestionMethod::Recursive;
- Hash expectedHash(htSHA256);
+ std::optional expectedHash;
for (auto & attr : *args[0]->attrs) {
const string & n(attr.name);
diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc
deleted file mode 100644
index 5013e74f0..000000000
--- a/src/libexpr/primops/fetchGit.cc
+++ /dev/null
@@ -1,91 +0,0 @@
-#include "primops.hh"
-#include "eval-inline.hh"
-#include "store-api.hh"
-#include "hash.hh"
-#include "fetchers.hh"
-#include "url.hh"
-
-namespace nix {
-
-static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v)
-{
- std::string url;
- std::optional ref;
- std::optional rev;
- std::string name = "source";
- bool fetchSubmodules = false;
- PathSet context;
-
- state.forceValue(*args[0]);
-
- if (args[0]->type == tAttrs) {
-
- state.forceAttrs(*args[0], pos);
-
- for (auto & attr : *args[0]->attrs) {
- string n(attr.name);
- if (n == "url")
- url = state.coerceToString(*attr.pos, *attr.value, context, false, false);
- else if (n == "ref")
- ref = state.forceStringNoCtx(*attr.value, *attr.pos);
- else if (n == "rev")
- rev = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA1);
- else if (n == "name")
- name = state.forceStringNoCtx(*attr.value, *attr.pos);
- else if (n == "submodules")
- fetchSubmodules = state.forceBool(*attr.value, *attr.pos);
- else
- throw EvalError({
- .hint = hintfmt("unsupported argument '%s' to 'fetchGit'", attr.name),
- .errPos = *attr.pos
- });
- }
-
- if (url.empty())
- throw EvalError({
- .hint = hintfmt("'url' argument required"),
- .errPos = pos
- });
-
- } else
- url = state.coerceToString(pos, *args[0], context, false, false);
-
- // FIXME: git externals probably can be used to bypass the URI
- // whitelist. Ah well.
- state.checkURI(url);
-
- if (evalSettings.pureEval && !rev)
- throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision");
-
- fetchers::Attrs attrs;
- attrs.insert_or_assign("type", "git");
- attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url);
- if (ref) attrs.insert_or_assign("ref", *ref);
- if (rev) attrs.insert_or_assign("rev", rev->gitRev());
- if (fetchSubmodules) attrs.insert_or_assign("submodules", fetchers::Explicit{true});
- auto input = fetchers::Input::fromAttrs(std::move(attrs));
-
- // FIXME: use name?
- auto [tree, input2] = input.fetch(state.store);
-
- state.mkAttrs(v, 8);
- auto storePath = state.store->printStorePath(tree.storePath);
- mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
- // Backward compatibility: set 'rev' to
- // 0000000000000000000000000000000000000000 for a dirty tree.
- auto rev2 = input2.getRev().value_or(Hash(htSHA1));
- mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev());
- mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev2.gitShortRev());
- // Backward compatibility: set 'revCount' to 0 for a dirty tree.
- mkInt(*state.allocAttr(v, state.symbols.create("revCount")),
- input2.getRevCount().value_or(0));
- mkBool(*state.allocAttr(v, state.symbols.create("submodules")), fetchSubmodules);
- v.attrs->sort();
-
- if (state.allowedPaths)
- state.allowedPaths->insert(tree.actualPath);
-}
-
-static RegisterPrimOp r("fetchGit", 1, prim_fetchGit);
-
-}
diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc
index fc2a6a1c2..cef85cfef 100644
--- a/src/libexpr/primops/fetchMercurial.cc
+++ b/src/libexpr/primops/fetchMercurial.cc
@@ -31,7 +31,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
// be both a revision or a branch/tag name.
auto value = state.forceStringNoCtx(*attr.value, *attr.pos);
if (std::regex_match(value, revRegex))
- rev = Hash(value, htSHA1);
+ rev = Hash::parseAny(value, htSHA1);
else
ref = value;
}
diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index 5f480d919..cddcf0e59 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -14,7 +14,8 @@ void emitTreeAttrs(
EvalState & state,
const fetchers::Tree & tree,
const fetchers::Input & input,
- Value & v)
+ Value & v,
+ bool emptyRevFallback)
{
assert(input.isImmutable());
@@ -34,10 +35,20 @@ void emitTreeAttrs(
if (auto rev = input.getRev()) {
mkString(*state.allocAttr(v, state.symbols.create("rev")), rev->gitRev());
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev->gitShortRev());
+ } else if (emptyRevFallback) {
+ // Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev
+ auto emptyHash = Hash(htSHA1);
+ mkString(*state.allocAttr(v, state.symbols.create("rev")), emptyHash.gitRev());
+ mkString(*state.allocAttr(v, state.symbols.create("shortRev")), emptyHash.gitRev());
}
+ if (input.getType() == "git")
+ mkBool(*state.allocAttr(v, state.symbols.create("submodules")), maybeGetBoolAttr(input.attrs, "submodules").value_or(false));
+
if (auto revCount = input.getRevCount())
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount);
+ else if (emptyRevFallback)
+ mkInt(*state.allocAttr(v, state.symbols.create("revCount")), 0);
if (auto lastModified = input.getLastModified()) {
mkInt(*state.allocAttr(v, state.symbols.create("lastModified")), *lastModified);
@@ -48,10 +59,26 @@ void emitTreeAttrs(
v.attrs->sort();
}
-static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v)
+std::string fixURI(std::string uri, EvalState &state)
{
- settings.requireExperimentalFeature("flakes");
+ state.checkURI(uri);
+ return uri.find("://") != std::string::npos ? uri : "file://" + uri;
+}
+void addURI(EvalState &state, fetchers::Attrs &attrs, Symbol name, std::string v)
+{
+ string n(name);
+ attrs.emplace(name, n == "url" ? fixURI(v, state) : v);
+}
+
+static void fetchTree(
+ EvalState &state,
+ const Pos &pos,
+ Value **args,
+ Value &v,
+ const std::optional type,
+ bool emptyRevFallback = false
+) {
fetchers::Input input;
PathSet context;
@@ -64,8 +91,15 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V
for (auto & attr : *args[0]->attrs) {
state.forceValue(*attr.value);
- if (attr.value->type == tString)
- attrs.emplace(attr.name, attr.value->string.s);
+ if (attr.value->type == tPath || attr.value->type == tString)
+ addURI(
+ state,
+ attrs,
+ attr.name,
+ state.coerceToString(*attr.pos, *attr.value, context, false, false)
+ );
+ else if (attr.value->type == tString)
+ addURI(state, attrs, attr.name, attr.value->string.s);
else if (attr.value->type == tBool)
attrs.emplace(attr.name, fetchers::Explicit{attr.value->boolean});
else if (attr.value->type == tInt)
@@ -75,6 +109,9 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V
attr.name, showType(*attr.value));
}
+ if (type)
+ attrs.emplace("type", type.value());
+
if (!attrs.count("type"))
throw Error({
.hint = hintfmt("attribute 'type' is missing in call to 'fetchTree'"),
@@ -82,8 +119,18 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V
});
input = fetchers::Input::fromAttrs(std::move(attrs));
- } else
- input = fetchers::Input::fromURL(state.coerceToString(pos, *args[0], context, false, false));
+ } else {
+ auto url = fixURI(state.coerceToString(pos, *args[0], context, false, false), state);
+
+ if (type == "git") {
+ fetchers::Attrs attrs;
+ attrs.emplace("type", "git");
+ attrs.emplace("url", url);
+ input = fetchers::Input::fromAttrs(std::move(attrs));
+ } else {
+ input = fetchers::Input::fromURL(url);
+ }
+ }
if (!evalSettings.pureEval && !input.isDirect())
input = lookupInRegistries(state.store, input).first;
@@ -96,7 +143,13 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V
if (state.allowedPaths)
state.allowedPaths->insert(tree.actualPath);
- emitTreeAttrs(state, tree, input2, v);
+ emitTreeAttrs(state, tree, input2, v, emptyRevFallback);
+}
+
+static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+ settings.requireExperimentalFeature("flakes");
+ fetchTree(state, pos, args, v, std::nullopt);
}
static RegisterPrimOp r("fetchTree", 1, prim_fetchTree);
@@ -178,7 +231,13 @@ static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args
fetch(state, pos, args, v, "fetchTarball", true, "source");
}
+static void prim_fetchGit(EvalState &state, const Pos &pos, Value **args, Value &v)
+{
+ fetchTree(state, pos, args, v, "git", true);
+}
+
static RegisterPrimOp r2("__fetchurl", 1, prim_fetchurl);
static RegisterPrimOp r3("fetchTarball", 1, prim_fetchTarball);
+static RegisterPrimOp r4("fetchGit", 1, prim_fetchGit);
}
diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index c3ee9bf43..9c69fc564 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -134,7 +134,7 @@ std::pair Input::fetch(ref store) const
if (auto prevNarHash = getNarHash()) {
if (narHash != *prevNarHash)
- throw Error("NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'",
+ throw Error((unsigned int) 102, "NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'",
to_string(), tree.actualPath, prevNarHash->to_string(SRI, true), narHash->to_string(SRI, true));
}
@@ -200,9 +200,12 @@ std::string Input::getType() const
std::optional Input::getNarHash() const
{
- if (auto s = maybeGetStrAttr(attrs, "narHash"))
- // FIXME: require SRI hash.
- return newHashAllowEmpty(*s, htSHA256);
+ if (auto s = maybeGetStrAttr(attrs, "narHash")) {
+ auto hash = s->empty() ? Hash(htSHA256) : Hash::parseSRI(*s);
+ if (hash.type != htSHA256)
+ throw UsageError("narHash must use SHA-256");
+ return hash;
+ }
return {};
}
@@ -216,7 +219,7 @@ std::optional Input::getRef() const
std::optional Input::getRev() const
{
if (auto s = maybeGetStrAttr(attrs, "rev"))
- return Hash(*s, htSHA1);
+ return Hash::parseAny(*s, htSHA1);
return {};
}
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index b1b47c45f..8b6e047f1 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -293,14 +293,14 @@ struct GitInputScheme : InputScheme
if (!input.getRev())
input.attrs.insert_or_assign("rev",
- Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input.getRef() })), htSHA1).gitRev());
+ Hash::parseAny(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input.getRef() })), htSHA1).gitRev());
repoDir = actualUrl;
} else {
if (auto res = getCache()->lookup(store, mutableAttrs)) {
- auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1);
+ auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), htSHA1);
if (!input.getRev() || input.getRev() == rev2) {
input.attrs.insert_or_assign("rev", rev2.gitRev());
return makeResult(res->first, std::move(res->second));
@@ -370,7 +370,7 @@ struct GitInputScheme : InputScheme
}
if (!input.getRev())
- input.attrs.insert_or_assign("rev", Hash(chomp(readFile(localRefFile)), htSHA1).gitRev());
+ input.attrs.insert_or_assign("rev", Hash::parseAny(chomp(readFile(localRefFile)), htSHA1).gitRev());
}
bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true";
diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
index 8bb7c2c1d..9f84ffb68 100644
--- a/src/libfetchers/github.cc
+++ b/src/libfetchers/github.cc
@@ -29,7 +29,7 @@ struct GitArchiveInputScheme : InputScheme
if (path.size() == 2) {
} else if (path.size() == 3) {
if (std::regex_match(path[2], revRegex))
- rev = Hash(path[2], htSHA1);
+ rev = Hash::parseAny(path[2], htSHA1);
else if (std::regex_match(path[2], refRegex))
ref = path[2];
else
@@ -41,7 +41,7 @@ struct GitArchiveInputScheme : InputScheme
if (name == "rev") {
if (rev)
throw BadURL("URL '%s' contains multiple commit hashes", url.url);
- rev = Hash(value, htSHA1);
+ rev = Hash::parseAny(value, htSHA1);
}
else if (name == "ref") {
if (!std::regex_match(value, refRegex))
@@ -191,7 +191,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
readFile(
store->toRealPath(
downloadFile(store, url, "source", false).storePath)));
- auto rev = Hash(std::string { json["sha"] }, htSHA1);
+ auto rev = Hash::parseAny(std::string { json["sha"] }, htSHA1);
debug("HEAD revision for '%s' is %s", url, rev.gitRev());
return rev;
}
@@ -235,7 +235,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
readFile(
store->toRealPath(
downloadFile(store, url, "source", false).storePath)));
- auto rev = Hash(std::string(json[0]["id"]), htSHA1);
+ auto rev = Hash::parseAny(std::string(json[0]["id"]), htSHA1);
debug("HEAD revision for '%s' is %s", url, rev.gitRev());
return rev;
}
diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc
index 91dc83740..b981d4d8e 100644
--- a/src/libfetchers/indirect.cc
+++ b/src/libfetchers/indirect.cc
@@ -18,7 +18,7 @@ struct IndirectInputScheme : InputScheme
if (path.size() == 1) {
} else if (path.size() == 2) {
if (std::regex_match(path[1], revRegex))
- rev = Hash(path[1], htSHA1);
+ rev = Hash::parseAny(path[1], htSHA1);
else if (std::regex_match(path[1], refRegex))
ref = path[1];
else
@@ -29,7 +29,7 @@ struct IndirectInputScheme : InputScheme
ref = path[1];
if (!std::regex_match(path[2], revRegex))
throw BadURL("in flake URL '%s', '%s' is not a commit hash", url.url, path[2]);
- rev = Hash(path[2], htSHA1);
+ rev = Hash::parseAny(path[2], htSHA1);
} else
throw BadURL("GitHub URL '%s' is invalid", url.url);
diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc
index c48cb6fd1..3e76ffc4d 100644
--- a/src/libfetchers/mercurial.cc
+++ b/src/libfetchers/mercurial.cc
@@ -209,7 +209,7 @@ struct MercurialInputScheme : InputScheme
});
if (auto res = getCache()->lookup(store, mutableAttrs)) {
- auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1);
+ auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), htSHA1);
if (!input.getRev() || input.getRev() == rev2) {
input.attrs.insert_or_assign("rev", rev2.gitRev());
return makeResult(res->first, std::move(res->second));
@@ -252,7 +252,7 @@ struct MercurialInputScheme : InputScheme
runProgram("hg", true, { "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" }));
assert(tokens.size() == 3);
- input.attrs.insert_or_assign("rev", Hash(tokens[0], htSHA1).gitRev());
+ input.attrs.insert_or_assign("rev", Hash::parseAny(tokens[0], htSHA1).gitRev());
auto revCount = std::stoull(tokens[1]);
input.attrs.insert_or_assign("ref", tokens[2]);
diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index 52718c231..2b1f25ca3 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -36,7 +36,7 @@ void printGCWarning()
void printMissing(ref store, const std::vector & paths, Verbosity lvl)
{
- unsigned long long downloadSize, narSize;
+ uint64_t downloadSize, narSize;
StorePathSet willBuild, willSubstitute, unknown;
store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize);
printMissing(store, willBuild, willSubstitute, unknown, downloadSize, narSize, lvl);
@@ -45,7 +45,7 @@ void printMissing(ref store, const std::vector & pa
void printMissing(ref store, const StorePathSet & willBuild,
const StorePathSet & willSubstitute, const StorePathSet & unknown,
- unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl)
+ uint64_t downloadSize, uint64_t narSize, Verbosity lvl)
{
if (!willBuild.empty()) {
if (willBuild.size() == 1)
@@ -384,7 +384,7 @@ RunPager::~RunPager()
}
-string showBytes(unsigned long long bytes)
+string showBytes(uint64_t bytes)
{
return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str();
}
diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh
index f558247c0..ffae5d796 100644
--- a/src/libmain/shared.hh
+++ b/src/libmain/shared.hh
@@ -47,7 +47,7 @@ void printMissing(
void printMissing(ref store, const StorePathSet & willBuild,
const StorePathSet & willSubstitute, const StorePathSet & unknown,
- unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl = lvlInfo);
+ uint64_t downloadSize, uint64_t narSize, Verbosity lvl = lvlInfo);
string getArg(const string & opt,
Strings::iterator & i, const Strings::iterator & end);
@@ -110,7 +110,7 @@ extern volatile ::sig_atomic_t blockInt;
/* GC helpers. */
-string showBytes(unsigned long long bytes);
+string showBytes(uint64_t bytes);
struct GCResults;
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 30150eeba..9682db730 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -153,6 +153,8 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
auto [fdTemp, fnTemp] = createTempFile();
+ AutoDelete autoDelete(fnTemp);
+
auto now1 = std::chrono::steady_clock::now();
/* Read the NAR simultaneously into a CompressionSink+FileSink (to
@@ -167,6 +169,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
TeeSource teeSource(narSource, *compressionSink);
narAccessor = makeNarAccessor(teeSource);
compressionSink->finish();
+ fileSink.flush();
}
auto now2 = std::chrono::steady_clock::now();
@@ -280,7 +283,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
if (repair || !fileExists(narInfo->url)) {
stats.narWrite++;
upsertFile(narInfo->url,
- std::make_shared(fnTemp, std::ios_base::in),
+ std::make_shared(fnTemp, std::ios_base::in | std::ios_base::binary),
"application/x-nix-nar");
} else
stats.narWriteAverted++;
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 38897a9e2..94c398b2f 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -297,7 +297,7 @@ public:
GoalPtr makeDerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
std::shared_ptr makeBasicDerivationGoal(const StorePath & drvPath,
const BasicDerivation & drv, BuildMode buildMode = bmNormal);
- GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair);
+ GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt);
/* Remove a dead goal. */
void removeGoal(GoalPtr goal);
@@ -1206,7 +1206,7 @@ void DerivationGoal::haveDerivation()
them. */
if (settings.useSubstitutes && parsedDrv->substitutesAllowed())
for (auto & i : invalidOutputs)
- addWaitee(worker.makeSubstitutionGoal(i, buildMode == bmRepair ? Repair : NoRepair));
+ addWaitee(worker.makeSubstitutionGoal(i, buildMode == bmRepair ? Repair : NoRepair, getDerivationCA(*drv)));
if (waitees.empty()) /* to prevent hang (no wake-up event) */
outputsSubstituted();
@@ -1646,13 +1646,13 @@ void DerivationGoal::buildDone()
So instead, check if the disk is (nearly) full now. If
so, we don't mark this build as a permanent failure. */
#if HAVE_STATVFS
- unsigned long long required = 8ULL * 1024 * 1024; // FIXME: make configurable
+ uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable
struct statvfs st;
if (statvfs(worker.store.realStoreDir.c_str(), &st) == 0 &&
- (unsigned long long) st.f_bavail * st.f_bsize < required)
+ (uint64_t) st.f_bavail * st.f_bsize < required)
diskFull = true;
if (statvfs(tmpDir.c_str(), &st) == 0 &&
- (unsigned long long) st.f_bavail * st.f_bsize < required)
+ (uint64_t) st.f_bavail * st.f_bsize < required)
diskFull = true;
#endif
@@ -2851,7 +2851,7 @@ struct RestrictedStore : public LocalFSStore
void queryMissing(const std::vector & targets,
StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
- unsigned long long & downloadSize, unsigned long long & narSize) override
+ uint64_t & downloadSize, uint64_t & narSize) override
{
/* This is slightly impure since it leaks information to the
client about what paths will be built/substituted or are
@@ -4300,6 +4300,10 @@ private:
/* The store path that should be realised through a substitute. */
StorePath storePath;
+ /* The path the substituter refers to the path as. This will be
+ * different when the stores have different names. */
+ std::optional subPath;
+
/* The remaining substituters. */
std::list> subs;
@@ -4333,8 +4337,11 @@ private:
typedef void (SubstitutionGoal::*GoalState)();
GoalState state;
+ /* Content address for recomputing store path */
+ std::optional ca;
+
public:
- SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair);
+ SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt);
~SubstitutionGoal();
void timedOut(Error && ex) override { abort(); };
@@ -4364,10 +4371,11 @@ public:
};
-SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair)
+SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional ca)
: Goal(worker)
, storePath(storePath)
, repair(repair)
+ , ca(ca)
{
state = &SubstitutionGoal::init;
name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath));
@@ -4442,14 +4450,18 @@ void SubstitutionGoal::tryNext()
sub = subs.front();
subs.pop_front();
- if (sub->storeDir != worker.store.storeDir) {
+ if (ca) {
+ subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca);
+ if (sub->storeDir == worker.store.storeDir)
+ assert(subPath == storePath);
+ } else if (sub->storeDir != worker.store.storeDir) {
tryNext();
return;
}
try {
// FIXME: make async
- info = sub->queryPathInfo(storePath);
+ info = sub->queryPathInfo(subPath ? *subPath : storePath);
} catch (InvalidPath &) {
tryNext();
return;
@@ -4468,6 +4480,19 @@ void SubstitutionGoal::tryNext()
throw;
}
+ if (info->path != storePath) {
+ if (info->isContentAddressed(*sub) && info->references.empty()) {
+ auto info2 = std::make_shared(*info);
+ info2->path = storePath;
+ info = info2;
+ } else {
+ printError("asked '%s' for '%s' but got '%s'",
+ sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path));
+ tryNext();
+ return;
+ }
+ }
+
/* Update the total expected download size. */
auto narInfo = std::dynamic_pointer_cast(info);
@@ -4557,7 +4582,7 @@ void SubstitutionGoal::tryToRun()
PushActivity pact(act.id);
copyStorePath(ref(sub), ref(worker.store.shared_from_this()),
- storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs);
+ subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs);
promise.set_value();
} catch (...) {
@@ -4690,11 +4715,11 @@ std::shared_ptr Worker::makeBasicDerivationGoal(const StorePath
}
-GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair)
+GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca)
{
GoalPtr goal = substitutionGoals[path].lock(); // FIXME
if (!goal) {
- goal = std::make_shared(path, *this, repair);
+ goal = std::make_shared(path, *this, repair, ca);
substitutionGoals.insert_or_assign(path, goal);
wakeUp(goal);
}
@@ -5062,7 +5087,7 @@ void Worker::markContentsGood(const StorePath & path)
static void primeCache(Store & store, const std::vector & paths)
{
StorePathSet willBuild, willSubstitute, unknown;
- unsigned long long downloadSize, narSize;
+ uint64_t downloadSize, narSize;
store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize);
if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty())
diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc
index 8291c745c..6585a480d 100644
--- a/src/libstore/builtins/fetchurl.cc
+++ b/src/libstore/builtins/fetchurl.cc
@@ -58,26 +58,6 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
}
};
- /* We always have one output, and if it's a fixed-output derivation (as
- checked below) it must be the only output */
- auto & output = drv.outputs.begin()->second;
-
- /* Try the hashed mirrors first. */
- if (auto hash = std::get_if(&output.output)) {
- if (hash->hash.method == FileIngestionMethod::Flat) {
- for (auto hashedMirror : settings.hashedMirrors.get()) {
- try {
- if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
- fetch(hashedMirror + printHashType(hash->hash.hash.type) + "/" + hash->hash.hash.to_string(Base16, false));
- return;
- } catch (Error & e) {
- debug(e.what());
- }
- }
- }
- }
-
- /* Otherwise try the specified URL. */
fetch(mainUrl);
}
diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc
index 0e139ff39..38acaa4bf 100644
--- a/src/libstore/content-address.cc
+++ b/src/libstore/content-address.cc
@@ -1,4 +1,6 @@
+#include "args.hh"
#include "content-address.hh"
+#include "split.hh"
namespace nix {
@@ -36,38 +38,46 @@ std::string renderContentAddress(ContentAddress ca) {
}
ContentAddress parseContentAddress(std::string_view rawCa) {
- auto prefixSeparator = rawCa.find(':');
- if (prefixSeparator != string::npos) {
- auto prefix = string(rawCa, 0, prefixSeparator);
- if (prefix == "text") {
- auto hashTypeAndHash = rawCa.substr(prefixSeparator+1, string::npos);
- Hash hash = Hash(string(hashTypeAndHash));
- if (hash.type != htSHA256) {
- throw Error("parseContentAddress: the text hash should have type SHA256");
- }
- return TextHash { hash };
- } else if (prefix == "fixed") {
- // This has to be an inverse of makeFixedOutputCA
- auto methodAndHash = rawCa.substr(prefixSeparator+1, string::npos);
- if (methodAndHash.substr(0,2) == "r:") {
- std::string_view hashRaw = methodAndHash.substr(2,string::npos);
- return FixedOutputHash {
- .method = FileIngestionMethod::Recursive,
- .hash = Hash(string(hashRaw)),
- };
- } else {
- std::string_view hashRaw = methodAndHash;
- return FixedOutputHash {
- .method = FileIngestionMethod::Flat,
- .hash = Hash(string(hashRaw)),
- };
- }
- } else {
- throw Error("parseContentAddress: format not recognized; has to be text or fixed");
- }
- } else {
- throw Error("Not a content address because it lacks an appropriate prefix");
+ auto rest = rawCa;
+
+ std::string_view prefix;
+ {
+ auto optPrefix = splitPrefixTo(rest, ':');
+ if (!optPrefix)
+ throw UsageError("not a content address because it is not in the form \":\": %s", rawCa);
+ prefix = *optPrefix;
}
+
+ auto parseHashType_ = [&](){
+ auto hashTypeRaw = splitPrefixTo(rest, ':');
+ if (!hashTypeRaw)
+ throw UsageError("content address hash must be in form \":\", but found: %s", rawCa);
+ HashType hashType = parseHashType(*hashTypeRaw);
+ return std::move(hashType);
+ };
+
+ // Switch on prefix
+ if (prefix == "text") {
+ // No parsing of the method, "text" only support flat.
+ HashType hashType = parseHashType_();
+ if (hashType != htSHA256)
+ throw Error("text content address hash should use %s, but instead uses %s",
+ printHashType(htSHA256), printHashType(hashType));
+ return TextHash {
+ .hash = Hash::parseNonSRIUnprefixed(rest, std::move(hashType)),
+ };
+ } else if (prefix == "fixed") {
+ // Parse method
+ auto method = FileIngestionMethod::Flat;
+ if (splitPrefix(rest, "r:"))
+ method = FileIngestionMethod::Recursive;
+ HashType hashType = parseHashType_();
+ return FixedOutputHash {
+ .method = method,
+ .hash = Hash::parseNonSRIUnprefixed(rest, std::move(hashType)),
+ };
+ } else
+ throw UsageError("content address prefix \"%s\" is unrecognized. Recogonized prefixes are \"text\" or \"fixed\"", prefix);
};
std::optional parseContentAddressOpt(std::string_view rawCaOpt) {
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 503e04f92..5e568fc94 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -86,7 +86,7 @@ struct TunnelLogger : public Logger
}
/* startWork() means that we're starting an operation for which we
- want to send out stderr to the client. */
+ want to send out stderr to the client. */
void startWork()
{
auto state(state_.lock());
@@ -579,7 +579,7 @@ static void performOp(TunnelLogger * logger, ref store,
auto path = store->parseStorePath(readString(from));
logger->startWork();
SubstitutablePathInfos infos;
- store->querySubstitutablePathInfos({path}, infos);
+ store->querySubstitutablePathInfos({{path, std::nullopt}}, infos);
logger->stopWork();
auto i = infos.find(path);
if (i == infos.end())
@@ -595,10 +595,16 @@ static void performOp(TunnelLogger * logger, ref store,
}
case wopQuerySubstitutablePathInfos: {
- auto paths = readStorePaths(*store, from);
- logger->startWork();
SubstitutablePathInfos infos;
- store->querySubstitutablePathInfos(paths, infos);
+ StorePathCAMap pathsMap = {};
+ if (GET_PROTOCOL_MINOR(clientVersion) < 22) {
+ auto paths = readStorePaths(*store, from);
+ for (auto & path : paths)
+ pathsMap.emplace(path, std::nullopt);
+ } else
+ pathsMap = readStorePathCAMap(*store, from);
+ logger->startWork();
+ store->querySubstitutablePathInfos(pathsMap, infos);
logger->stopWork();
to << infos.size();
for (auto & i : infos) {
@@ -692,7 +698,7 @@ static void performOp(TunnelLogger * logger, ref store,
auto deriver = readString(from);
if (deriver != "")
info.deriver = store->parseStorePath(deriver);
- info.narHash = Hash(readString(from), htSHA256);
+ info.narHash = Hash::parseAny(readString(from), htSHA256);
info.references = readStorePaths(*store, from);
from >> info.registrationTime >> info.narSize >> info.ultimate;
info.sigs = readStrings(from);
@@ -703,24 +709,84 @@ static void performOp(TunnelLogger * logger, ref store,
if (!trusted)
info.ultimate = false;
- std::unique_ptr source;
- if (GET_PROTOCOL_MINOR(clientVersion) >= 21)
- source = std::make_unique(from, to);
- else {
- StringSink saved;
- TeeSource tee { from, saved };
- ParseSink ether;
- parseDump(ether, tee);
- source = std::make_unique(std::move(*saved.s));
+ if (GET_PROTOCOL_MINOR(clientVersion) >= 23) {
+
+ struct FramedSource : Source
+ {
+ Source & from;
+ bool eof = false;
+ std::vector pending;
+ size_t pos = 0;
+
+ FramedSource(Source & from) : from(from)
+ { }
+
+ ~FramedSource()
+ {
+ if (!eof) {
+ while (true) {
+ auto n = readInt(from);
+ if (!n) break;
+ std::vector data(n);
+ from(data.data(), n);
+ }
+ }
+ }
+
+ size_t read(unsigned char * data, size_t len) override
+ {
+ if (eof) throw EndOfFile("reached end of FramedSource");
+
+ if (pos >= pending.size()) {
+ size_t len = readInt(from);
+ if (!len) {
+ eof = true;
+ return 0;
+ }
+ pending = std::vector(len);
+ pos = 0;
+ from(pending.data(), len);
+ }
+
+ auto n = std::min(len, pending.size() - pos);
+ memcpy(data, pending.data() + pos, n);
+ pos += n;
+ return n;
+ }
+ };
+
+ logger->startWork();
+
+ {
+ FramedSource source(from);
+ store->addToStore(info, source, (RepairFlag) repair,
+ dontCheckSigs ? NoCheckSigs : CheckSigs);
+ }
+
+ logger->stopWork();
}
- logger->startWork();
+ else {
+ std::unique_ptr source;
+ if (GET_PROTOCOL_MINOR(clientVersion) >= 21)
+ source = std::make_unique(from, to);
+ else {
+ StringSink saved;
+ TeeSource tee { from, saved };
+ ParseSink ether;
+ parseDump(ether, tee);
+ source = std::make_unique(std::move(*saved.s));
+ }
- // FIXME: race if addToStore doesn't read source?
- store->addToStore(info, *source, (RepairFlag) repair,
- dontCheckSigs ? NoCheckSigs : CheckSigs);
+ logger->startWork();
+
+ // FIXME: race if addToStore doesn't read source?
+ store->addToStore(info, *source, (RepairFlag) repair,
+ dontCheckSigs ? NoCheckSigs : CheckSigs);
+
+ logger->stopWork();
+ }
- logger->stopWork();
break;
}
@@ -730,7 +796,7 @@ static void performOp(TunnelLogger * logger, ref store,
targets.push_back(store->parsePathWithOutputs(s));
logger->startWork();
StorePathSet willBuild, willSubstitute, unknown;
- unsigned long long downloadSize, narSize;
+ uint64_t downloadSize, narSize;
store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize);
logger->stopWork();
writeStorePaths(*store, to, willBuild);
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 03cdc1760..68b081058 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -33,7 +33,7 @@ bool derivationIsCA(DerivationType dt) {
};
// Since enums can have non-variant values, but making a `default:` would
// disable exhaustiveness warnings.
- abort();
+ assert(false);
}
bool derivationIsFixed(DerivationType dt) {
@@ -42,7 +42,7 @@ bool derivationIsFixed(DerivationType dt) {
case DerivationType::CAFixed: return true;
case DerivationType::CAFloating: return false;
};
- abort();
+ assert(false);
}
bool derivationIsImpure(DerivationType dt) {
@@ -51,7 +51,7 @@ bool derivationIsImpure(DerivationType dt) {
case DerivationType::CAFixed: return true;
case DerivationType::CAFloating: return false;
};
- abort();
+ assert(false);
}
@@ -159,7 +159,7 @@ static DerivationOutput parseDerivationOutput(const Store & store, std::istrings
.output = DerivationOutputCAFixed {
.hash = FixedOutputHash {
.method = std::move(method),
- .hash = Hash(hash, hashType),
+ .hash = Hash::parseNonSRIUnprefixed(hash, hashType),
},
}
}
@@ -556,7 +556,7 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store)
.output = DerivationOutputCAFixed {
.hash = FixedOutputHash {
.method = std::move(method),
- .hash = Hash(hash, hashType),
+ .hash = Hash::parseNonSRIUnprefixed(hash, hashType),
},
}
}
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index 09d51649e..14e0e947a 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -14,6 +14,7 @@ namespace nix {
/* Abstract syntax of derivations. */
+/* The traditional non-fixed-output derivation type. */
struct DerivationOutputInputAddressed
{
/* Will need to become `std::optional` once input-addressed
@@ -21,11 +22,16 @@ struct DerivationOutputInputAddressed
StorePath path;
};
+/* Fixed-output derivations, whose output paths are content addressed
+ according to that fixed output. */
struct DerivationOutputCAFixed
{
FixedOutputHash hash; /* hash used for expected hash computation */
};
+/* Floating-output derivations, whose output paths are content addressed, but
+ not fixed, and so are dynamically calculated from whatever the output ends
+ up being. */
struct DerivationOutputCAFloating
{
/* information used for expected hash computation */
diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index aaed5c218..e74382ed2 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -500,7 +500,7 @@ struct LocalStore::GCState
StorePathSet alive;
bool gcKeepOutputs;
bool gcKeepDerivations;
- unsigned long long bytesInvalidated;
+ uint64_t bytesInvalidated;
bool moveToTrash = true;
bool shouldDelete;
GCState(const GCOptions & options, GCResults & results)
@@ -518,7 +518,7 @@ bool LocalStore::isActiveTempFile(const GCState & state,
void LocalStore::deleteGarbage(GCState & state, const Path & path)
{
- unsigned long long bytesFreed;
+ uint64_t bytesFreed;
deletePath(path, bytesFreed);
state.results.bytesFreed += bytesFreed;
}
@@ -528,7 +528,7 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path)
{
checkInterrupt();
- unsigned long long size = 0;
+ uint64_t size = 0;
auto storePath = maybeParseStorePath(path);
if (storePath && isValidPath(*storePath)) {
@@ -687,7 +687,7 @@ void LocalStore::removeUnusedLinks(const GCState & state)
AutoCloseDir dir(opendir(linksDir.c_str()));
if (!dir) throw SysError("opening directory '%1%'", linksDir);
- long long actualSize = 0, unsharedSize = 0;
+ int64_t actualSize = 0, unsharedSize = 0;
struct dirent * dirent;
while (errno = 0, dirent = readdir(dir.get())) {
@@ -717,10 +717,10 @@ void LocalStore::removeUnusedLinks(const GCState & state)
struct stat st;
if (stat(linksDir.c_str(), &st) == -1)
throw SysError("statting '%1%'", linksDir);
- long long overhead = st.st_blocks * 512ULL;
+ auto overhead = st.st_blocks * 512ULL;
- printInfo(format("note: currently hard linking saves %.2f MiB")
- % ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0)));
+ printInfo("note: currently hard linking saves %.2f MiB",
+ ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0)));
}
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index d47e0b6b5..3406a9331 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -335,9 +335,6 @@ public:
"setuid/setgid bits or with file capabilities."};
#endif
- Setting hashedMirrors{this, {"http://tarballs.nixos.org/"}, "hashed-mirrors",
- "A list of servers used by builtins.fetchurl to fetch files by hash."};
-
Setting minFree{this, 0, "min-free",
"Automatically run the garbage collector when free disk space drops below the specified amount."};
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 5d7566121..c6eeab548 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -113,7 +113,7 @@ struct LegacySSHStore : public Store
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) {
auto s = readString(conn->from);
- info->narHash = s.empty() ? std::optional{} : Hash{s};
+ info->narHash = s.empty() ? std::optional{} : Hash::parseAnyPrefixed(s);
info->ca = parseContentAddressOpt(readString(conn->from));
info->sigs = readStrings(conn->from);
}
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index edcb17607..de7ddb84b 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -655,7 +655,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path,
info->id = useQueryPathInfo.getInt(0);
try {
- info->narHash = Hash(useQueryPathInfo.getStr(1));
+ info->narHash = Hash::parseAnyPrefixed(useQueryPathInfo.getStr(1));
} catch (BadHash & e) {
throw Error("in valid-path entry for '%s': %s", printStorePath(path), e.what());
}
@@ -854,20 +854,32 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths)
}
-void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths,
- SubstitutablePathInfos & infos)
+void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos)
{
if (!settings.useSubstitutes) return;
for (auto & sub : getDefaultSubstituters()) {
- if (sub->storeDir != storeDir) continue;
for (auto & path : paths) {
- if (infos.count(path)) continue;
- debug("checking substituter '%s' for path '%s'", sub->getUri(), printStorePath(path));
+ auto subPath(path.first);
+
+ // recompute store path so that we can use a different store root
+ if (path.second) {
+ subPath = makeFixedOutputPathFromCA(path.first.name(), *path.second);
+ if (sub->storeDir == storeDir)
+ assert(subPath == path.first);
+ if (subPath != path.first)
+ debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(path.first), sub->printStorePath(subPath), sub->getUri());
+ } else if (sub->storeDir != storeDir) continue;
+
+ debug("checking substituter '%s' for path '%s'", sub->getUri(), sub->printStorePath(subPath));
try {
- auto info = sub->queryPathInfo(path);
+ auto info = sub->queryPathInfo(subPath);
+
+ if (sub->storeDir != storeDir && !(info->isContentAddressed(*sub) && info->references.empty()))
+ continue;
+
auto narInfo = std::dynamic_pointer_cast(
std::shared_ptr(info));
- infos.insert_or_assign(path, SubstitutablePathInfo{
+ infos.insert_or_assign(path.first, SubstitutablePathInfo{
info->deriver,
info->references,
narInfo ? narInfo->fileSize : 0,
@@ -1041,20 +1053,6 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
}
-StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
- FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
-{
- Path srcPath(absPath(_srcPath));
- auto source = sinkToSource([&](Sink & sink) {
- if (method == FileIngestionMethod::Recursive)
- dumpPath(srcPath, sink, filter);
- else
- readFile(srcPath, sink);
- });
- return addToStoreFromDump(*source, name, method, hashAlgo, repair);
-}
-
-
StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name,
FileIngestionMethod method, HashType hashAlgo, RepairFlag repair)
{
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 355c2814f..31e6587ac 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -29,8 +29,8 @@ struct Derivation;
struct OptimiseStats
{
unsigned long filesLinked = 0;
- unsigned long long bytesFreed = 0;
- unsigned long long blocksFreed = 0;
+ uint64_t bytesFreed = 0;
+ uint64_t blocksFreed = 0;
};
@@ -139,22 +139,14 @@ public:
StorePathSet querySubstitutablePaths(const StorePathSet & paths) override;
- void querySubstitutablePathInfos(const StorePathSet & paths,
+ void querySubstitutablePathInfos(const StorePathCAMap & paths,
SubstitutablePathInfos & infos) override;
void addToStore(const ValidPathInfo & info, Source & source,
RepairFlag repair, CheckSigsFlag checkSigs) override;
- StorePath addToStore(const string & name, const Path & srcPath,
- FileIngestionMethod method, HashType hashAlgo,
- PathFilter & filter, RepairFlag repair) override;
-
- /* Like addToStore(), but the contents of the path are contained
- in `dump', which is either a NAR serialisation (if recursive ==
- true) or simply the contents of a regular file (if recursive ==
- false). */
StorePath addToStoreFromDump(Source & dump, const string & name,
- FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override;
+ FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) override;
StorePath addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair) override;
diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index c4d22a634..0ae1ceaad 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -4,6 +4,7 @@
#include "local-store.hh"
#include "store-api.hh"
#include "thread-pool.hh"
+#include "topo-sort.hh"
namespace nix {
@@ -108,9 +109,19 @@ void Store::computeFSClosure(const StorePath & startPath,
}
+std::optional getDerivationCA(const BasicDerivation & drv)
+{
+ auto out = drv.outputs.find("out");
+ if (out != drv.outputs.end()) {
+ if (auto v = std::get_if(&out->second.output))
+ return v->hash;
+ }
+ return std::nullopt;
+}
+
void Store::queryMissing(const std::vector & targets,
StorePathSet & willBuild_, StorePathSet & willSubstitute_, StorePathSet & unknown_,
- unsigned long long & downloadSize_, unsigned long long & narSize_)
+ uint64_t & downloadSize_, uint64_t & narSize_)
{
Activity act(*logger, lvlDebug, actUnknown, "querying info about missing paths");
@@ -122,8 +133,8 @@ void Store::queryMissing(const std::vector & targets,
{
std::unordered_set done;
StorePathSet & unknown, & willSubstitute, & willBuild;
- unsigned long long & downloadSize;
- unsigned long long & narSize;
+ uint64_t & downloadSize;
+ uint64_t & narSize;
};
struct DrvState
@@ -157,7 +168,7 @@ void Store::queryMissing(const std::vector & targets,
auto outPath = parseStorePath(outPathS);
SubstitutablePathInfos infos;
- querySubstitutablePathInfos({outPath}, infos);
+ querySubstitutablePathInfos({{outPath, getDerivationCA(*drv)}}, infos);
if (infos.empty()) {
drvState_->lock()->done = true;
@@ -214,7 +225,7 @@ void Store::queryMissing(const std::vector & targets,
if (isValidPath(path.path)) return;
SubstitutablePathInfos infos;
- querySubstitutablePathInfos({path.path}, infos);
+ querySubstitutablePathInfos({{path.path, std::nullopt}}, infos);
if (infos.empty()) {
auto state(state_.lock());
@@ -246,41 +257,21 @@ void Store::queryMissing(const std::vector & targets,
StorePaths Store::topoSortPaths(const StorePathSet & paths)
{
- StorePaths sorted;
- StorePathSet visited, parents;
-
- std::function dfsVisit;
-
- dfsVisit = [&](const StorePath & path, const StorePath * parent) {
- if (parents.count(path))
- throw BuildError("cycle detected in the references of '%s' from '%s'",
- printStorePath(path), printStorePath(*parent));
-
- if (!visited.insert(path).second) return;
- parents.insert(path);
-
- StorePathSet references;
- try {
- references = queryPathInfo(path)->references;
- } catch (InvalidPath &) {
- }
-
- for (auto & i : references)
- /* Don't traverse into paths that don't exist. That can
- happen due to substitutes for non-existent paths. */
- if (i != path && paths.count(i))
- dfsVisit(i, &path);
-
- sorted.push_back(path);
- parents.erase(path);
- };
-
- for (auto & i : paths)
- dfsVisit(i, nullptr);
-
- std::reverse(sorted.begin(), sorted.end());
-
- return sorted;
+ return topoSort(paths,
+ {[&](const StorePath & path) {
+ StorePathSet references;
+ try {
+ references = queryPathInfo(path)->references;
+ } catch (InvalidPath &) {
+ }
+ return references;
+ }},
+ {[&](const StorePath & path, const StorePath & parent) {
+ return BuildError(
+ "cycle detected in the references of '%s' from '%s'",
+ printStorePath(path),
+ printStorePath(parent));
+ }});
}
diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc
index d884a131e..59ec164b6 100644
--- a/src/libstore/nar-accessor.cc
+++ b/src/libstore/nar-accessor.cc
@@ -79,14 +79,14 @@ struct NarAccessor : public FSAccessor
parents.top()->isExecutable = true;
}
- void preallocateContents(unsigned long long size) override
+ void preallocateContents(uint64_t size) override
{
assert(size <= std::numeric_limits::max());
parents.top()->size = (uint64_t) size;
parents.top()->start = pos;
}
- void receiveContents(unsigned char * data, unsigned int len) override
+ void receiveContents(unsigned char * data, size_t len) override
{ }
void createSymlink(const Path & path, const string & target) override
diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc
index 9ddb9957f..92da14e23 100644
--- a/src/libstore/nar-info-disk-cache.cc
+++ b/src/libstore/nar-info-disk-cache.cc
@@ -193,9 +193,9 @@ public:
narInfo->url = queryNAR.getStr(2);
narInfo->compression = queryNAR.getStr(3);
if (!queryNAR.isNull(4))
- narInfo->fileHash = Hash(queryNAR.getStr(4));
+ narInfo->fileHash = Hash::parseAnyPrefixed(queryNAR.getStr(4));
narInfo->fileSize = queryNAR.getInt(5);
- narInfo->narHash = Hash(queryNAR.getStr(6));
+ narInfo->narHash = Hash::parseAnyPrefixed(queryNAR.getStr(6));
narInfo->narSize = queryNAR.getInt(7);
for (auto & r : tokenizeString(queryNAR.getStr(8), " "))
narInfo->references.insert(StorePath(r));
diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc
index ca471463c..5812aa4ac 100644
--- a/src/libstore/nar-info.cc
+++ b/src/libstore/nar-info.cc
@@ -12,7 +12,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
auto parseHashField = [&](const string & s) {
try {
- return Hash(s);
+ return Hash::parseAnyPrefixed(s);
} catch (BadHash &) {
throw corrupt();
}
diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc
index b2b2412a3..e4b4b6213 100644
--- a/src/libstore/optimise-store.cc
+++ b/src/libstore/optimise-store.cc
@@ -282,7 +282,7 @@ void LocalStore::optimiseStore(OptimiseStats & stats)
}
}
-static string showBytes(unsigned long long bytes)
+static string showBytes(uint64_t bytes)
{
return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str();
}
diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh
new file mode 100644
index 000000000..2a015ea3c
--- /dev/null
+++ b/src/libstore/path-info.hh
@@ -0,0 +1,111 @@
+#pragma once
+
+#include "path.hh"
+#include "hash.hh"
+#include "content-address.hh"
+
+#include
+#include
+
+namespace nix {
+
+
+class Store;
+
+
+struct SubstitutablePathInfo
+{
+ std::optional deriver;
+ StorePathSet references;
+ uint64_t downloadSize; /* 0 = unknown or inapplicable */
+ uint64_t narSize; /* 0 = unknown */
+};
+
+typedef std::map SubstitutablePathInfos;
+
+
+struct ValidPathInfo
+{
+ StorePath path;
+ std::optional deriver;
+ // TODO document this
+ std::optional narHash;
+ StorePathSet references;
+ time_t registrationTime = 0;
+ uint64_t narSize = 0; // 0 = unknown
+ uint64_t id; // internal use only
+
+ /* Whether the path is ultimately trusted, that is, it's a
+ derivation output that was built locally. */
+ bool ultimate = false;
+
+ StringSet sigs; // note: not necessarily verified
+
+ /* If non-empty, an assertion that the path is content-addressed,
+ i.e., that the store path is computed from a cryptographic hash
+ of the contents of the path, plus some other bits of data like
+ the "name" part of the path. Such a path doesn't need
+ signatures, since we don't have to trust anybody's claim that
+ the path is the output of a particular derivation. (In the
+ extensional store model, we have to trust that the *contents*
+ of an output path of a derivation were actually produced by
+ that derivation. In the intensional model, we have to trust
+ that a particular output path was produced by a derivation; the
+ path then implies the contents.)
+
+ Ideally, the content-addressability assertion would just be a Boolean,
+ and the store path would be computed from the name component, ‘narHash’
+ and ‘references’. However, we support many types of content addresses.
+ */
+ std::optional ca;
+
+ bool operator == (const ValidPathInfo & i) const
+ {
+ return
+ path == i.path
+ && narHash == i.narHash
+ && references == i.references;
+ }
+
+ /* Return a fingerprint of the store path to be used in binary
+ cache signatures. It contains the store path, the base-32
+ SHA-256 hash of the NAR serialisation of the path, the size of
+ the NAR, and the sorted references. The size field is strictly
+ speaking superfluous, but might prevent endless/excessive data
+ attacks. */
+ std::string fingerprint(const Store & store) const;
+
+ void sign(const Store & store, const SecretKey & secretKey);
+
+ /* Return true iff the path is verifiably content-addressed. */
+ bool isContentAddressed(const Store & store) const;
+
+ /* Functions to view references + hasSelfReference as one set, mainly for
+ compatibility's sake. */
+ StorePathSet referencesPossiblyToSelf() const;
+ void insertReferencePossiblyToSelf(StorePath && ref);
+ void setReferencesPossiblyToSelf(StorePathSet && refs);
+
+ static const size_t maxSigs = std::numeric_limits::max();
+
+ /* Return the number of signatures on this .narinfo that were
+ produced by one of the specified keys, or maxSigs if the path
+ is content-addressed. */
+ size_t checkSignatures(const Store & store, const PublicKeys & publicKeys) const;
+
+ /* Verify a single signature. */
+ bool checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const;
+
+ Strings shortRefs() const;
+
+ ValidPathInfo(const ValidPathInfo & other) = default;
+
+ ValidPathInfo(StorePath && path) : path(std::move(path)) { };
+ ValidPathInfo(const StorePath & path) : path(path) { };
+
+ virtual ~ValidPathInfo() { }
+};
+
+typedef list ValidPathInfos;
+
+}
diff --git a/src/libstore/path.hh b/src/libstore/path.hh
index e43a8b50c..b03a0f69d 100644
--- a/src/libstore/path.hh
+++ b/src/libstore/path.hh
@@ -64,6 +64,8 @@ typedef std::set StorePathSet;
typedef std::vector StorePaths;
typedef std::map OutputPathMap;
+typedef std::map> StorePathCAMap;
+
/* Extension of derivations in the Nix store. */
const std::string drvExtension = ".drv";
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 8d01c6667..33d1e431b 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -39,6 +39,24 @@ void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths
out << store.printStorePath(i);
}
+StorePathCAMap readStorePathCAMap(const Store & store, Source & from)
+{
+ StorePathCAMap paths;
+ auto count = readNum(from);
+ while (count--)
+ paths.insert_or_assign(store.parseStorePath(readString(from)), parseContentAddressOpt(readString(from)));
+ return paths;
+}
+
+void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap & paths)
+{
+ out << paths.size();
+ for (auto & i : paths) {
+ out << store.printStorePath(i.first);
+ out << renderContentAddress(i.second);
+ }
+}
+
std::map readOutputPathMap(const Store & store, Source & from)
{
std::map pathMap;
@@ -332,18 +350,17 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths)
}
-void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths,
- SubstitutablePathInfos & infos)
+void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, SubstitutablePathInfos & infos)
{
- if (paths.empty()) return;
+ if (pathsMap.empty()) return;
auto conn(getConnection());
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) {
- for (auto & i : paths) {
+ for (auto & i : pathsMap) {
SubstitutablePathInfo info;
- conn->to << wopQuerySubstitutablePathInfo << printStorePath(i);
+ conn->to << wopQuerySubstitutablePathInfo << printStorePath(i.first);
conn.processStderr();
unsigned int reply = readInt(conn->from);
if (reply == 0) continue;
@@ -353,13 +370,19 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths,
info.references = readStorePaths(*this, conn->from);
info.downloadSize = readLongLong(conn->from);
info.narSize = readLongLong(conn->from);
- infos.insert_or_assign(i, std::move(info));
+ infos.insert_or_assign(i.first, std::move(info));
}
} else {
conn->to << wopQuerySubstitutablePathInfos;
- writeStorePaths(*this, conn->to, paths);
+ if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 22) {
+ StorePathSet paths;
+ for (auto & path : pathsMap)
+ paths.insert(path.first);
+ writeStorePaths(*this, conn->to, paths);
+ } else
+ writeStorePathCAMap(*this, conn->to, pathsMap);
conn.processStderr();
size_t count = readNum(conn->from);
for (size_t n = 0; n < count; n++) {
@@ -399,7 +422,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path,
info = std::make_shared(StorePath(path));
auto deriver = readString(conn->from);
if (deriver != "") info->deriver = parseStorePath(deriver);
- info->narHash = Hash(readString(conn->from), htSHA256);
+ info->narHash = Hash::parseAny(readString(conn->from), htSHA256);
info->references = readStorePaths(*this, conn->from);
conn->from >> info->registrationTime >> info->narSize;
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) {
@@ -503,9 +526,84 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
conn->to << info.registrationTime << info.narSize
<< info.ultimate << info.sigs << renderContentAddress(info.ca)
<< repair << !checkSigs;
- bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21;
- if (!tunnel) copyNAR(source, conn->to);
- conn.processStderr(0, tunnel ? &source : nullptr);
+
+ if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 23) {
+
+ std::exception_ptr ex;
+
+ struct FramedSink : BufferedSink
+ {
+ ConnectionHandle & conn;
+ std::exception_ptr & ex;
+
+ FramedSink(ConnectionHandle & conn, std::exception_ptr & ex) : conn(conn), ex(ex)
+ { }
+
+ ~FramedSink()
+ {
+ try {
+ conn->to << 0;
+ conn->to.flush();
+ } catch (...) {
+ ignoreException();
+ }
+ }
+
+ void write(const unsigned char * data, size_t len) override
+ {
+ /* Don't send more data if the remote has
+ encountered an error. */
+ if (ex) {
+ auto ex2 = ex;
+ ex = nullptr;
+ std::rethrow_exception(ex2);
+ }
+ conn->to << len;
+ conn->to(data, len);
+ };
+ };
+
+ /* Handle log messages / exceptions from the remote on a
+ separate thread. */
+ std::thread stderrThread([&]()
+ {
+ try {
+ conn.processStderr();
+ } catch (...) {
+ ex = std::current_exception();
+ }
+ });
+
+ Finally joinStderrThread([&]()
+ {
+ if (stderrThread.joinable()) {
+ stderrThread.join();
+ if (ex) {
+ try {
+ std::rethrow_exception(ex);
+ } catch (...) {
+ ignoreException();
+ }
+ }
+ }
+ });
+
+ {
+ FramedSink sink(conn, ex);
+ copyNAR(source, sink);
+ sink.flush();
+ }
+
+ stderrThread.join();
+ if (ex)
+ std::rethrow_exception(ex);
+
+ } else if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21) {
+ conn.processStderr(0, &source);
+ } else {
+ copyNAR(source, conn->to);
+ conn.processStderr(0, nullptr);
+ }
}
}
@@ -707,7 +805,7 @@ void RemoteStore::addSignatures(const StorePath & storePath, const StringSet & s
void RemoteStore::queryMissing(const std::vector & targets,
StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
- unsigned long long & downloadSize, unsigned long long & narSize)
+ uint64_t & downloadSize, uint64_t & narSize)
{
{
auto conn(getConnection());
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 3c1b78b6a..72d2a6689 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -56,7 +56,7 @@ public:
StorePathSet querySubstitutablePaths(const StorePathSet & paths) override;
- void querySubstitutablePathInfos(const StorePathSet & paths,
+ void querySubstitutablePathInfos(const StorePathCAMap & paths,
SubstitutablePathInfos & infos) override;
void addToStore(const ValidPathInfo & info, Source & nar,
@@ -94,7 +94,7 @@ public:
void queryMissing(const std::vector & targets,
StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
- unsigned long long & downloadSize, unsigned long long & narSize) override;
+ uint64_t & downloadSize, uint64_t & narSize) override;
void connect() override;
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 1b7dff085..a0a446bd3 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -266,6 +266,10 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
const std::string & mimeType,
const std::string & contentEncoding)
{
+ istream->seekg(0, istream->end);
+ auto size = istream->tellg();
+ istream->seekg(0, istream->beg);
+
auto maxThreads = std::thread::hardware_concurrency();
static std::shared_ptr
@@ -343,13 +347,11 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
std::chrono::duration_cast(now2 - now1)
.count();
- auto size = istream->tellg();
-
printInfo("uploaded 's3://%s/%s' (%d bytes) in %d ms",
bucketName, path, size, duration);
stats.putTimeMs += duration;
- stats.putBytes += size;
+ stats.putBytes += std::max(size, (decltype(size)) 0);
stats.put++;
}
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index ab4dfa908..2837910b9 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -193,6 +193,19 @@ StorePath Store::makeFixedOutputPath(
}
}
+StorePath Store::makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca,
+ const StorePathSet & references, bool hasSelfReference) const
+{
+ // New template
+ return std::visit(overloaded {
+ [&](TextHash th) {
+ return makeTextPath(name, th.hash, references);
+ },
+ [&](FixedOutputHash fsh) {
+ return makeFixedOutputPath(fsh.method, fsh.hash, name, references, hasSelfReference);
+ }
+ }, ca);
+}
StorePath Store::makeTextPath(std::string_view name, const Hash & hash,
const StorePathSet & references) const
@@ -222,6 +235,20 @@ StorePath Store::computeStorePathForText(const string & name, const string & s,
}
+StorePath Store::addToStore(const string & name, const Path & _srcPath,
+ FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
+{
+ Path srcPath(absPath(_srcPath));
+ auto source = sinkToSource([&](Sink & sink) {
+ if (method == FileIngestionMethod::Recursive)
+ dumpPath(srcPath, sink, filter);
+ else
+ readFile(srcPath, sink);
+ });
+ return addToStoreFromDump(*source, name, method, hashAlgo, repair);
+}
+
+
/*
The aim of this function is to compute in one pass the correct ValidPathInfo for
the files that we are trying to add to the store. To accomplish that in one
@@ -689,6 +716,15 @@ void copyStorePath(ref srcStore, ref dstStore,
uint64_t total = 0;
+ // recompute store path on the chance dstStore does it differently
+ if (info->ca && info->references.empty()) {
+ auto info2 = make_ref(*info);
+ info2->path = dstStore->makeFixedOutputPathFromCA(info->path.name(), *info->ca);
+ if (dstStore->storeDir == srcStore->storeDir)
+ assert(info->path == info2->path);
+ info = info2;
+ }
+
if (!info->narHash) {
StringSink sink;
srcStore->narFromPath({storePath}, sink);
@@ -724,16 +760,20 @@ void copyStorePath(ref srcStore, ref dstStore,
}
-void copyPaths(ref srcStore, ref dstStore, const StorePathSet & storePaths,
+std::map copyPaths(ref srcStore, ref dstStore, const StorePathSet & storePaths,
RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute)
{
auto valid = dstStore->queryValidPaths(storePaths, substitute);
- PathSet missing;
+ StorePathSet missing;
for (auto & path : storePaths)
- if (!valid.count(path)) missing.insert(srcStore->printStorePath(path));
+ if (!valid.count(path)) missing.insert(path);
- if (missing.empty()) return;
+ std::map pathsMap;
+ for (auto & path : storePaths)
+ pathsMap.insert_or_assign(path, path);
+
+ if (missing.empty()) return pathsMap;
Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size()));
@@ -748,30 +788,49 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st
ThreadPool pool;
- processGraph(pool,
- PathSet(missing.begin(), missing.end()),
+ processGraph(pool,
+ StorePathSet(missing.begin(), missing.end()),
- [&](const Path & storePath) {
- if (dstStore->isValidPath(dstStore->parseStorePath(storePath))) {
+ [&](const StorePath & storePath) {
+ auto info = srcStore->queryPathInfo(storePath);
+ auto storePathForDst = storePath;
+ if (info->ca && info->references.empty()) {
+ storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca);
+ if (dstStore->storeDir == srcStore->storeDir)
+ assert(storePathForDst == storePath);
+ if (storePathForDst != storePath)
+ debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri());
+ }
+ pathsMap.insert_or_assign(storePath, storePathForDst);
+
+ if (dstStore->isValidPath(storePath)) {
nrDone++;
showProgress();
- return PathSet();
+ return StorePathSet();
}
- auto info = srcStore->queryPathInfo(srcStore->parseStorePath(storePath));
-
bytesExpected += info->narSize;
act.setExpected(actCopyPath, bytesExpected);
- return srcStore->printStorePathSet(info->references);
+ return info->references;
},
- [&](const Path & storePathS) {
+ [&](const StorePath & storePath) {
checkInterrupt();
- auto storePath = dstStore->parseStorePath(storePathS);
+ auto info = srcStore->queryPathInfo(storePath);
- if (!dstStore->isValidPath(storePath)) {
+ auto storePathForDst = storePath;
+ if (info->ca && info->references.empty()) {
+ storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca);
+ if (dstStore->storeDir == srcStore->storeDir)
+ assert(storePathForDst == storePath);
+ if (storePathForDst != storePath)
+ debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri());
+ }
+ pathsMap.insert_or_assign(storePath, storePathForDst);
+
+ if (!dstStore->isValidPath(storePathForDst)) {
MaintainCount mc(nrRunning);
showProgress();
try {
@@ -780,7 +839,7 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st
nrFailed++;
if (!settings.keepGoing)
throw e;
- logger->log(lvlError, fmt("could not copy %s: %s", storePathS, e.what()));
+ logger->log(lvlError, fmt("could not copy %s: %s", dstStore->printStorePath(storePath), e.what()));
showProgress();
return;
}
@@ -789,6 +848,8 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st
nrDone++;
showProgress();
});
+
+ return pathsMap;
}
@@ -811,7 +872,7 @@ std::optional decodeValidPathInfo(const Store & store, std::istre
if (hashGiven) {
string s;
getline(str, s);
- info.narHash = Hash(s, htSHA256);
+ info.narHash = Hash::parseAny(s, htSHA256);
getline(str, s);
if (!string2Int(s, info.narSize)) throw Error("number expected");
}
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index a7288d0cc..e94d975c5 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -10,6 +10,7 @@
#include "globals.hh"
#include "config.hh"
#include "derivations.hh"
+#include "path-info.hh"
#include
#include
@@ -85,7 +86,7 @@ struct GCOptions
StorePathSet pathsToDelete;
/* Stop after at least `maxFreed' bytes have been freed. */
- unsigned long long maxFreed{std::numeric_limits::max()};
+ uint64_t maxFreed{std::numeric_limits::max()};
};
@@ -97,99 +98,10 @@ struct GCResults
/* For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the
number of bytes that would be or was freed. */
- unsigned long long bytesFreed = 0;
+ uint64_t bytesFreed = 0;
};
-struct SubstitutablePathInfo
-{
- std::optional deriver;
- StorePathSet references;
- unsigned long long downloadSize; /* 0 = unknown or inapplicable */
- unsigned long long narSize; /* 0 = unknown */
-};
-
-typedef std::map SubstitutablePathInfos;
-
-struct ValidPathInfo
-{
- StorePath path;
- std::optional deriver;
- // TODO document this
- std::optional narHash;
- StorePathSet references;
- time_t registrationTime = 0;
- uint64_t narSize = 0; // 0 = unknown
- uint64_t id; // internal use only
-
- /* Whether the path is ultimately trusted, that is, it's a
- derivation output that was built locally. */
- bool ultimate = false;
-
- StringSet sigs; // note: not necessarily verified
-
- /* If non-empty, an assertion that the path is content-addressed,
- i.e., that the store path is computed from a cryptographic hash
- of the contents of the path, plus some other bits of data like
- the "name" part of the path. Such a path doesn't need
- signatures, since we don't have to trust anybody's claim that
- the path is the output of a particular derivation. (In the
- extensional store model, we have to trust that the *contents*
- of an output path of a derivation were actually produced by
- that derivation. In the intensional model, we have to trust
- that a particular output path was produced by a derivation; the
- path then implies the contents.)
-
- Ideally, the content-addressability assertion would just be a Boolean,
- and the store path would be computed from the name component, ‘narHash’
- and ‘references’. However, we support many types of content addresses.
- */
- std::optional ca;
-
- bool operator == (const ValidPathInfo & i) const
- {
- return
- path == i.path
- && narHash == i.narHash
- && references == i.references;
- }
-
- /* Return a fingerprint of the store path to be used in binary
- cache signatures. It contains the store path, the base-32
- SHA-256 hash of the NAR serialisation of the path, the size of
- the NAR, and the sorted references. The size field is strictly
- speaking superfluous, but might prevent endless/excessive data
- attacks. */
- std::string fingerprint(const Store & store) const;
-
- void sign(const Store & store, const SecretKey & secretKey);
-
- /* Return true iff the path is verifiably content-addressed. */
- bool isContentAddressed(const Store & store) const;
-
- static const size_t maxSigs = std::numeric_limits::max();
-
- /* Return the number of signatures on this .narinfo that were
- produced by one of the specified keys, or maxSigs if the path
- is content-addressed. */
- size_t checkSignatures(const Store & store, const PublicKeys & publicKeys) const;
-
- /* Verify a single signature. */
- bool checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const;
-
- Strings shortRefs() const;
-
- ValidPathInfo(const ValidPathInfo & other) = default;
-
- ValidPathInfo(StorePath && path) : path(std::move(path)) { };
- ValidPathInfo(const StorePath & path) : path(path) { };
-
- virtual ~ValidPathInfo() { }
-};
-
-typedef list ValidPathInfos;
-
-
enum BuildMode { bmNormal, bmRepair, bmCheck };
@@ -344,7 +256,11 @@ public:
bool hasSelfReference = false) const;
StorePath makeTextPath(std::string_view name, const Hash & hash,
- const StorePathSet & references) const;
+ const StorePathSet & references = {}) const;
+
+ StorePath makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca,
+ const StorePathSet & references = {},
+ bool hasSelfReference = false) const;
/* This is the preparatory part of addToStore(); it computes the
store path to which srcPath is to be copied. Returns the store
@@ -436,9 +352,10 @@ public:
virtual StorePathSet querySubstitutablePaths(const StorePathSet & paths) { return {}; };
/* Query substitute info (i.e. references, derivers and download
- sizes) of a set of paths. If a path does not have substitute
- info, it's omitted from the resulting ‘infos’ map. */
- virtual void querySubstitutablePathInfos(const StorePathSet & paths,
+ sizes) of a map of paths to their optional ca values. If a path
+ does not have substitute info, it's omitted from the resulting
+ ‘infos’ map. */
+ virtual void querySubstitutablePathInfos(const StorePathCAMap & paths,
SubstitutablePathInfos & infos) { return; };
/* Import a path into the store. */
@@ -451,7 +368,7 @@ public:
libutil/archive.hh). */
virtual StorePath addToStore(const string & name, const Path & srcPath,
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
- PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0;
+ PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair);
/* Copy the contents of a path to the store and register the
validity the resulting path, using a constant amount of
@@ -460,6 +377,10 @@ public:
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
std::optional expectedCAHash = {});
+ /* Like addToStore(), but the contents of the path are contained
+ in `dump', which is either a NAR serialisation (if recursive ==
+ true) or simply the contents of a regular file (if recursive ==
+ false). */
// FIXME: remove?
virtual StorePath addToStoreFromDump(Source & dump, const string & name,
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair)
@@ -610,7 +531,7 @@ public:
that will be substituted. */
virtual void queryMissing(const std::vector & targets,
StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
- unsigned long long & downloadSize, unsigned long long & narSize);
+ uint64_t & downloadSize, uint64_t & narSize);
/* Sort a set of paths topologically under the references
relation. If p refers to q, then p precedes q in this list. */
@@ -740,11 +661,13 @@ void copyStorePath(ref srcStore, ref dstStore,
/* Copy store paths from one store to another. The paths may be copied
- in parallel. They are copied in a topologically sorted order
- (i.e. if A is a reference of B, then A is copied before B), but
- the set of store paths is not automatically closed; use
- copyClosure() for that. */
-void copyPaths(ref srcStore, ref dstStore, const StorePathSet & storePaths,
+ in parallel. They are copied in a topologically sorted order (i.e.
+ if A is a reference of B, then A is copied before B), but the set
+ of store paths is not automatically closed; use copyClosure() for
+ that. Returns a map of what each path was copied to the dstStore
+ as. */
+std::map copyPaths(ref srcStore, ref dstStore,
+ const StorePathSet & storePaths,
RepairFlag repair = NoRepair,
CheckSigsFlag checkSigs = CheckSigs,
SubstituteFlag substitute = NoSubstitute);
@@ -843,4 +766,6 @@ std::optional decodeValidPathInfo(
/* Split URI into protocol+hierarchy part and its parameter set. */
std::pair splitUriAndParams(const std::string & uri);
+std::optional getDerivationCA(const BasicDerivation & drv);
+
}
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index 8b538f6da..f76b13fb4 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -6,7 +6,7 @@ namespace nix {
#define WORKER_MAGIC_1 0x6e697863
#define WORKER_MAGIC_2 0x6478696f
-#define PROTOCOL_VERSION 0x116
+#define PROTOCOL_VERSION 0x117
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
@@ -70,6 +70,10 @@ template T readStorePaths(const Store & store, Source & from);
void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths);
+StorePathCAMap readStorePathCAMap(const Store & store, Source & from);
+
+void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap & paths);
+
void writeOutputPathMap(const Store & store, Sink & out, const OutputPathMap & paths);
}
diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc
index 51c88537e..ce7cf9754 100644
--- a/src/libutil/archive.cc
+++ b/src/libutil/archive.cc
@@ -150,17 +150,17 @@ static void skipGeneric(Source & source)
static void parseContents(ParseSink & sink, Source & source, const Path & path)
{
- unsigned long long size = readLongLong(source);
+ uint64_t size = readLongLong(source);
sink.preallocateContents(size);
- unsigned long long left = size;
+ uint64_t left = size;
std::vector buf(65536);
while (left) {
checkInterrupt();
auto n = buf.size();
- if ((unsigned long long)n > left) n = left;
+ if ((uint64_t)n > left) n = left;
source(buf.data(), n);
sink.receiveContents(buf.data(), n);
left -= n;
@@ -323,7 +323,7 @@ struct RestoreSink : ParseSink
throw SysError("fchmod");
}
- void preallocateContents(unsigned long long len)
+ void preallocateContents(uint64_t len)
{
#if HAVE_POSIX_FALLOCATE
if (len) {
@@ -338,7 +338,7 @@ struct RestoreSink : ParseSink
#endif
}
- void receiveContents(unsigned char * data, unsigned int len)
+ void receiveContents(unsigned char * data, size_t len)
{
writeFull(fd.get(), data, len);
}
diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh
index 57780d16a..5665732d2 100644
--- a/src/libutil/archive.hh
+++ b/src/libutil/archive.hh
@@ -57,8 +57,8 @@ struct ParseSink
virtual void createRegularFile(const Path & path) { };
virtual void isExecutable() { };
- virtual void preallocateContents(unsigned long long size) { };
- virtual void receiveContents(unsigned char * data, unsigned int len) { };
+ virtual void preallocateContents(uint64_t size) { };
+ virtual void receiveContents(unsigned char * data, size_t len) { };
virtual void createSymlink(const Path & path, const string & target) { };
};
@@ -77,7 +77,7 @@ struct RetrieveRegularNARSink : ParseSink
regular = false;
}
- void receiveContents(unsigned char * data, unsigned int len)
+ void receiveContents(unsigned char * data, size_t len)
{
sink(data, len);
}
diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc
index 2b0390da4..dfb3668f1 100644
--- a/src/libutil/hash.cc
+++ b/src/libutil/hash.cc
@@ -7,6 +7,7 @@
#include "args.hh"
#include "hash.hh"
#include "archive.hh"
+#include "split.hh"
#include "util.hh"
#include
@@ -15,6 +16,7 @@
namespace nix {
+
static size_t regularHashSize(HashType type) {
switch (type) {
case htMD5: return md5HashSize;
@@ -25,10 +27,11 @@ static size_t regularHashSize(HashType type) {
abort();
}
+
std::set hashTypes = { "md5", "sha1", "sha256", "sha512" };
-void Hash::init()
+Hash::Hash(HashType type) : type(type)
{
hashSize = regularHashSize(type);
assert(hashSize <= maxHashSize);
@@ -133,57 +136,89 @@ std::string Hash::to_string(Base base, bool includeType) const
return s;
}
-Hash::Hash(std::string_view s, HashType type) : Hash(s, std::optional { type }) { }
-Hash::Hash(std::string_view s) : Hash(s, std::optional{}) { }
-
-Hash::Hash(std::string_view original, std::optional optType)
-{
+Hash Hash::parseSRI(std::string_view original) {
auto rest = original;
- size_t pos = 0;
+ // Parse the has type before the separater, if there was one.
+ auto hashRaw = splitPrefixTo(rest, '-');
+ if (!hashRaw)
+ throw BadHash("hash '%s' is not SRI", original);
+ HashType parsedType = parseHashType(*hashRaw);
+
+ return Hash(rest, parsedType, true);
+}
+
+// Mutates the string to eliminate the prefixes when found
+static std::pair, bool> getParsedTypeAndSRI(std::string_view & rest) {
bool isSRI = false;
// Parse the has type before the separater, if there was one.
std::optional optParsedType;
{
- auto sep = rest.find(':');
- if (sep == std::string_view::npos) {
- sep = rest.find('-');
- if (sep != std::string_view::npos)
+ auto hashRaw = splitPrefixTo(rest, ':');
+
+ if (!hashRaw) {
+ hashRaw = splitPrefixTo(rest, '-');
+ if (hashRaw)
isSRI = true;
}
- if (sep != std::string_view::npos) {
- auto hashRaw = rest.substr(0, sep);
- optParsedType = parseHashType(hashRaw);
- rest = rest.substr(sep + 1);
- }
+ if (hashRaw)
+ optParsedType = parseHashType(*hashRaw);
}
+ return {optParsedType, isSRI};
+}
+
+Hash Hash::parseAnyPrefixed(std::string_view original)
+{
+ auto rest = original;
+ auto [optParsedType, isSRI] = getParsedTypeAndSRI(rest);
+
// Either the string or user must provide the type, if they both do they
// must agree.
- if (!optParsedType && !optType) {
+ if (!optParsedType)
+ throw BadHash("hash '%s' does not include a type", rest);
+
+ return Hash(rest, *optParsedType, isSRI);
+}
+
+Hash Hash::parseAny(std::string_view original, std::optional optType)
+{
+ auto rest = original;
+ auto [optParsedType, isSRI] = getParsedTypeAndSRI(rest);
+
+ // Either the string or user must provide the type, if they both do they
+ // must agree.
+ if (!optParsedType && !optType)
throw BadHash("hash '%s' does not include a type, nor is the type otherwise known from context.", rest);
- } else {
- this->type = optParsedType ? *optParsedType : *optType;
- if (optParsedType && optType && *optParsedType != *optType)
- throw BadHash("hash '%s' should have type '%s'", original, printHashType(*optType));
- }
+ else if (optParsedType && optType && *optParsedType != *optType)
+ throw BadHash("hash '%s' should have type '%s'", original, printHashType(*optType));
- init();
+ HashType hashType = optParsedType ? *optParsedType : *optType;
+ return Hash(rest, hashType, isSRI);
+}
+Hash Hash::parseNonSRIUnprefixed(std::string_view s, HashType type)
+{
+ return Hash(s, type, false);
+}
+
+Hash::Hash(std::string_view rest, HashType type, bool isSRI)
+ : Hash(type)
+{
if (!isSRI && rest.size() == base16Len()) {
auto parseHexDigit = [&](char c) {
if (c >= '0' && c <= '9') return c - '0';
if (c >= 'A' && c <= 'F') return c - 'A' + 10;
if (c >= 'a' && c <= 'f') return c - 'a' + 10;
- throw BadHash("invalid base-16 hash '%s'", original);
+ throw BadHash("invalid base-16 hash '%s'", rest);
};
for (unsigned int i = 0; i < hashSize; i++) {
hash[i] =
- parseHexDigit(rest[pos + i * 2]) << 4
- | parseHexDigit(rest[pos + i * 2 + 1]);
+ parseHexDigit(rest[i * 2]) << 4
+ | parseHexDigit(rest[i * 2 + 1]);
}
}
@@ -195,7 +230,7 @@ Hash::Hash(std::string_view original, std::optional optType)
for (digit = 0; digit < base32Chars.size(); ++digit) /* !!! slow */
if (base32Chars[digit] == c) break;
if (digit >= 32)
- throw BadHash("invalid base-32 hash '%s'", original);
+ throw BadHash("invalid base-32 hash '%s'", rest);
unsigned int b = n * 5;
unsigned int i = b / 8;
unsigned int j = b % 8;
@@ -205,7 +240,7 @@ Hash::Hash(std::string_view original, std::optional optType)
hash[i + 1] |= digit >> (8 - j);
} else {
if (digit >> (8 - j))
- throw BadHash("invalid base-32 hash '%s'", original);
+ throw BadHash("invalid base-32 hash '%s'", rest);
}
}
}
@@ -213,7 +248,7 @@ Hash::Hash(std::string_view original, std::optional optType)
else if (isSRI || rest.size() == base64Len()) {
auto d = base64Decode(rest);
if (d.size() != hashSize)
- throw BadHash("invalid %s hash '%s'", isSRI ? "SRI" : "base-64", original);
+ throw BadHash("invalid %s hash '%s'", isSRI ? "SRI" : "base-64", rest);
assert(hashSize);
memcpy(hash, d.data(), hashSize);
}
@@ -231,7 +266,7 @@ Hash newHashAllowEmpty(std::string hashStr, std::optional ht)
warn("found empty hash, assuming '%s'", h.to_string(SRI, true));
return h;
} else
- return Hash(hashStr, ht);
+ return Hash::parseAny(hashStr, ht);
}
diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh
index 98ee1bed0..00ce7bb6f 100644
--- a/src/libutil/hash.hh
+++ b/src/libutil/hash.hh
@@ -34,21 +34,31 @@ struct Hash
HashType type;
/* Create a zero-filled hash object. */
- Hash(HashType type) : type(type) { init(); };
+ Hash(HashType type);
- /* Initialize the hash from a string representation, in the format
+ /* Parse the hash from a string representation in the format
"[:]" or "-" (a
Subresource Integrity hash expression). If the 'type' argument
is not present, then the hash type must be specified in the
string. */
- Hash(std::string_view s, std::optional type);
- // type must be provided
- Hash(std::string_view s, HashType type);
- // hash type must be part of string
- Hash(std::string_view s);
+ static Hash parseAny(std::string_view s, std::optional type);
- void init();
+ /* Parse a hash from a string representation like the above, except the
+ type prefix is mandatory is there is no separate arguement. */
+ static Hash parseAnyPrefixed(std::string_view s);
+ /* Parse a plain hash that musst not have any prefix indicating the type.
+ The type is passed in to disambiguate. */
+ static Hash parseNonSRIUnprefixed(std::string_view s, HashType type);
+
+ static Hash parseSRI(std::string_view original);
+
+private:
+ /* The type must be provided, the string view must not include
+ prefix. `isSRI` helps disambigate the various base-* encodings. */
+ Hash(std::string_view s, HashType type, bool isSRI);
+
+public:
/* Check whether a hash is set. */
operator bool () const { return (bool) type; }
@@ -111,7 +121,7 @@ Hash hashFile(HashType ht, const Path & path);
/* Compute the hash of the given path. The hash is defined as
(essentially) hashString(ht, dumpPath(path)). */
-typedef std::pair HashResult;
+typedef std::pair HashResult;
HashResult hashPath(HashType ht, const Path & path,
PathFilter & filter = defaultPathFilter);
@@ -141,7 +151,7 @@ class HashSink : public BufferedSink, public AbstractHashSink
private:
HashType ht;
Ctx * ctx;
- unsigned long long bytes;
+ uint64_t bytes;
public:
HashSink(HashType ht);
diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index 5d9acf887..c29c6b29b 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -312,14 +312,14 @@ T readNum(Source & source)
source(buf, sizeof(buf));
uint64_t n =
- ((unsigned long long) buf[0]) |
- ((unsigned long long) buf[1] << 8) |
- ((unsigned long long) buf[2] << 16) |
- ((unsigned long long) buf[3] << 24) |
- ((unsigned long long) buf[4] << 32) |
- ((unsigned long long) buf[5] << 40) |
- ((unsigned long long) buf[6] << 48) |
- ((unsigned long long) buf[7] << 56);
+ ((uint64_t) buf[0]) |
+ ((uint64_t) buf[1] << 8) |
+ ((uint64_t) buf[2] << 16) |
+ ((uint64_t) buf[3] << 24) |
+ ((uint64_t) buf[4] << 32) |
+ ((uint64_t) buf[5] << 40) |
+ ((uint64_t) buf[6] << 48) |
+ ((uint64_t) buf[7] << 56);
if (n > std::numeric_limits::max())
throw SerialisationError("serialised integer %d is too large for type '%s'", n, typeid(T).name());
diff --git a/src/libutil/split.hh b/src/libutil/split.hh
new file mode 100644
index 000000000..d19d7d8ed
--- /dev/null
+++ b/src/libutil/split.hh
@@ -0,0 +1,33 @@
+#pragma once
+
+#include
+#include
+
+#include "util.hh"
+
+namespace nix {
+
+// If `separator` is found, we return the portion of the string before the
+// separator, and modify the string argument to contain only the part after the
+// separator. Otherwise, wer return `std::nullopt`, and we leave the argument
+// string alone.
+static inline std::optional splitPrefixTo(std::string_view & string, char separator) {
+ auto sepInstance = string.find(separator);
+
+ if (sepInstance != std::string_view::npos) {
+ auto prefix = string.substr(0, sepInstance);
+ string.remove_prefix(sepInstance+1);
+ return prefix;
+ }
+
+ return std::nullopt;
+}
+
+static inline bool splitPrefix(std::string_view & string, std::string_view prefix) {
+ bool res = hasPrefix(string, prefix);
+ if (res)
+ string.remove_prefix(prefix.length());
+ return res;
+}
+
+}
diff --git a/src/libutil/topo-sort.hh b/src/libutil/topo-sort.hh
new file mode 100644
index 000000000..7a68ff169
--- /dev/null
+++ b/src/libutil/topo-sort.hh
@@ -0,0 +1,40 @@
+#include "error.hh"
+
+namespace nix {
+
+template
+std::vector topoSort(std::set items,
+ std::function(const T &)> getChildren,
+ std::function makeCycleError)
+{
+ std::vector sorted;
+ std::set visited, parents;
+
+ std::function dfsVisit;
+
+ dfsVisit = [&](const T & path, const T * parent) {
+ if (parents.count(path)) throw makeCycleError(path, *parent);
+
+ if (!visited.insert(path).second) return;
+ parents.insert(path);
+
+ std::set references = getChildren(path);
+
+ for (auto & i : references)
+ /* Don't traverse into items that don't exist in our starting set. */
+ if (i != path && items.count(i))
+ dfsVisit(i, &path);
+
+ sorted.push_back(path);
+ parents.erase(path);
+ };
+
+ for (auto & i : items)
+ dfsVisit(i, nullptr);
+
+ std::reverse(sorted.begin(), sorted.end());
+
+ return sorted;
+}
+
+}
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index a0a8ff4d3..c0b9698ee 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -374,7 +374,7 @@ void writeLine(int fd, string s)
}
-static void _deletePath(int parentfd, const Path & path, unsigned long long & bytesFreed)
+static void _deletePath(int parentfd, const Path & path, uint64_t & bytesFreed)
{
checkInterrupt();
@@ -414,7 +414,7 @@ static void _deletePath(int parentfd, const Path & path, unsigned long long & by
}
}
-static void _deletePath(const Path & path, unsigned long long & bytesFreed)
+static void _deletePath(const Path & path, uint64_t & bytesFreed)
{
Path dir = dirOf(path);
if (dir == "")
@@ -435,12 +435,12 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed)
void deletePath(const Path & path)
{
- unsigned long long dummy;
+ uint64_t dummy;
deletePath(path, dummy);
}
-void deletePath(const Path & path, unsigned long long & bytesFreed)
+void deletePath(const Path & path, uint64_t & bytesFreed)
{
//Activity act(*logger, lvlDebug, format("recursively deleting path '%1%'") % path);
bytesFreed = 0;
@@ -494,6 +494,7 @@ std::pair createTempFile(const Path & prefix)
{
Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX");
// Strictly speaking, this is UB, but who cares...
+ // FIXME: use O_TMPFILE.
AutoCloseFD fd(mkstemp((char *) tmpl.c_str()));
if (!fd)
throw SysError("creating temporary file '%s'", tmpl);
@@ -1449,7 +1450,7 @@ string base64Decode(std::string_view s)
char digit = decode[(unsigned char) c];
if (digit == -1)
- throw Error("invalid character in Base64 string");
+ throw Error("invalid character in Base64 string: '%c'", c);
bits += 6;
d = d << 6 | digit;
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 630303a5d..3a20679a8 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -125,7 +125,7 @@ void writeLine(int fd, string s);
second variant returns the number of bytes and blocks freed. */
void deletePath(const Path & path);
-void deletePath(const Path & path, unsigned long long & bytesFreed);
+void deletePath(const Path & path, uint64_t & bytesFreed);
std::string getUserName();
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index f77de56ea..94412042f 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -174,7 +174,7 @@ static void _main(int argc, char * * argv)
else if (*arg == "--run-env") // obsolete
runEnv = true;
- else if (*arg == "--command" || *arg == "--run") {
+ else if (runEnv && (*arg == "--command" || *arg == "--run")) {
if (*arg == "--run")
interactive = false;
envCommand = getArg(*arg, arg, end) + "\nexit";
@@ -192,7 +192,7 @@ static void _main(int argc, char * * argv)
else if (*arg == "--pure") pure = true;
else if (*arg == "--impure") pure = false;
- else if (*arg == "--packages" || *arg == "-p")
+ else if (runEnv && (*arg == "--packages" || *arg == "-p"))
packages = true;
else if (inShebang && *arg == "-i") {
@@ -325,7 +325,7 @@ static void _main(int argc, char * * argv)
auto buildPaths = [&](const std::vector & paths) {
/* Note: we do this even when !printMissing to efficiently
fetch binary cache data. */
- unsigned long long downloadSize, narSize;
+ uint64_t downloadSize, narSize;
StorePathSet willBuild, willSubstitute, unknown;
store->queryMissing(paths,
willBuild, willSubstitute, unknown, downloadSize, narSize);
diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc
index aa5ada3a6..bcf1d8518 100644
--- a/src/nix-collect-garbage/nix-collect-garbage.cc
+++ b/src/nix-collect-garbage/nix-collect-garbage.cc
@@ -67,10 +67,8 @@ static int _main(int argc, char * * argv)
deleteOlderThan = getArg(*arg, arg, end);
}
else if (*arg == "--dry-run") dryRun = true;
- else if (*arg == "--max-freed") {
- long long maxFreed = getIntArg(*arg, arg, end, true);
- options.maxFreed = maxFreed >= 0 ? maxFreed : 0;
- }
+ else if (*arg == "--max-freed")
+ options.maxFreed = std::max(getIntArg(*arg, arg, end, true), (int64_t) 0);
else
return false;
return true;
diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc
index 65d8ec6b6..1001f27af 100644
--- a/src/nix-prefetch-url/nix-prefetch-url.cc
+++ b/src/nix-prefetch-url/nix-prefetch-url.cc
@@ -157,7 +157,7 @@ static int _main(int argc, char * * argv)
Hash hash(ht);
std::optional storePath;
if (args.size() == 2) {
- expectedHash = Hash(args[1], ht);
+ expectedHash = Hash::parseAny(args[1], ht);
const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
storePath = store->makeFixedOutputPath(recursive, *expectedHash, name);
if (store->isValidPath(*storePath))
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 76bf5154a..2996e36c4 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -130,7 +130,7 @@ static void opRealise(Strings opFlags, Strings opArgs)
for (auto & i : opArgs)
paths.push_back(store->followLinksToStorePathWithOutputs(i));
- unsigned long long downloadSize, narSize;
+ uint64_t downloadSize, narSize;
StorePathSet willBuild, willSubstitute, unknown;
store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize);
@@ -208,7 +208,7 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs)
string hash = *i++;
string name = *i++;
- cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(recursive, Hash(hash, hashAlgo), name)));
+ cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(recursive, Hash::parseAny(hash, hashAlgo), name)));
}
@@ -572,10 +572,8 @@ static void opGC(Strings opFlags, Strings opArgs)
if (*i == "--print-roots") printRoots = true;
else if (*i == "--print-live") options.action = GCOptions::gcReturnLive;
else if (*i == "--print-dead") options.action = GCOptions::gcReturnDead;
- else if (*i == "--max-freed") {
- long long maxFreed = getIntArg(*i, i, opFlags.end(), true);
- options.maxFreed = maxFreed >= 0 ? maxFreed : 0;
- }
+ else if (*i == "--max-freed")
+ options.maxFreed = std::max(getIntArg(*i, i, opFlags.end(), true), (int64_t) 0);
else throw UsageError("bad sub-operation '%1%' in GC", *i);
if (!opArgs.empty()) throw UsageError("no arguments expected");
@@ -831,7 +829,7 @@ static void opServe(Strings opFlags, Strings opArgs)
for (auto & path : paths)
if (!path.isDerivation())
paths2.push_back({path});
- unsigned long long downloadSize, narSize;
+ uint64_t downloadSize, narSize;
StorePathSet willBuild, willSubstitute, unknown;
store->queryMissing(paths2,
willBuild, willSubstitute, unknown, downloadSize, narSize);
@@ -950,7 +948,7 @@ static void opServe(Strings opFlags, Strings opArgs)
auto deriver = readString(in);
if (deriver != "")
info.deriver = store->parseStorePath(deriver);
- info.narHash = Hash(readString(in), htSHA256);
+ info.narHash = Hash::parseAny(readString(in), htSHA256);
info.references = readStorePaths(*store, in);
in >> info.registrationTime >> info.narSize >> info.ultimate;
info.sigs = readStrings(in);
diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc
index ad1f9e91f..e183cb8b5 100644
--- a/src/nix/add-to-store.cc
+++ b/src/nix/add-to-store.cc
@@ -9,6 +9,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand
{
Path path;
std::optional namePart;
+ FileIngestionMethod ingestionMethod = FileIngestionMethod::Recursive;
CmdAddToStore()
{
@@ -21,6 +22,13 @@ struct CmdAddToStore : MixDryRun, StoreCommand
.labels = {"name"},
.handler = {&namePart},
});
+
+ addFlag({
+ .longName = "flat",
+ .shortName = 0,
+ .description = "add flat file to the Nix store",
+ .handler = {&ingestionMethod, FileIngestionMethod::Flat},
+ });
}
std::string description() override
@@ -45,12 +53,19 @@ struct CmdAddToStore : MixDryRun, StoreCommand
auto narHash = hashString(htSHA256, *sink.s);
- ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, *namePart));
+ Hash hash = narHash;
+ if (ingestionMethod == FileIngestionMethod::Flat) {
+ HashSink hsink(htSHA256);
+ readFile(path, hsink);
+ hash = hsink.finish().first;
+ }
+
+ ValidPathInfo info(store->makeFixedOutputPath(ingestionMethod, hash, *namePart));
info.narHash = narHash;
info.narSize = sink.s->size();
info.ca = std::optional { FixedOutputHash {
- .method = FileIngestionMethod::Recursive,
- .hash = *info.narHash,
+ .method = ingestionMethod,
+ .hash = hash,
} };
if (!dryRun) {
diff --git a/src/nix/build.cc b/src/nix/build.cc
index 0f7e0e123..613cc15eb 100644
--- a/src/nix/build.cc
+++ b/src/nix/build.cc
@@ -9,6 +9,7 @@ using namespace nix;
struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile
{
Path outLink = "result";
+ BuildMode buildMode = bmNormal;
CmdBuild()
{
@@ -26,6 +27,12 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile
.description = "do not create a symlink to the build result",
.handler = {&outLink, Path("")},
});
+
+ addFlag({
+ .longName = "rebuild",
+ .description = "rebuild an already built package and compare the result to the existing store paths",
+ .handler = {&buildMode, bmCheck},
+ });
}
std::string description() override
@@ -53,7 +60,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile
void run(ref store) override
{
- auto buildables = build(store, dryRun ? Realise::Nothing : Realise::Outputs, installables);
+ auto buildables = build(store, dryRun ? Realise::Nothing : Realise::Outputs, installables, buildMode);
if (dryRun) return;
diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc
new file mode 100644
index 000000000..eb3339f5d
--- /dev/null
+++ b/src/nix/bundle.cc
@@ -0,0 +1,129 @@
+#include "command.hh"
+#include "common-args.hh"
+#include "shared.hh"
+#include "store-api.hh"
+#include "fs-accessor.hh"
+
+using namespace nix;
+
+struct CmdBundle : InstallableCommand
+{
+ std::string bundler = "github:matthewbauer/nix-bundle";
+ std::optional outLink;
+
+ CmdBundle()
+ {
+ addFlag({
+ .longName = "bundler",
+ .description = "use custom bundler",
+ .labels = {"flake-url"},
+ .handler = {&bundler},
+ .completer = {[&](size_t, std::string_view prefix) {
+ completeFlakeRef(getStore(), prefix);
+ }}
+ });
+
+ addFlag({
+ .longName = "out-link",
+ .shortName = 'o',
+ .description = "path of the symlink to the build result",
+ .labels = {"path"},
+ .handler = {&outLink},
+ .completer = completePath
+ });
+ }
+
+ std::string description() override
+ {
+ return "bundle an application so that it works outside of the Nix store";
+ }
+
+ Examples examples() override
+ {
+ return {
+ Example{
+ "To bundle Hello:",
+ "nix bundle hello"
+ },
+ };
+ }
+
+ Category category() override { return catSecondary; }
+
+ Strings getDefaultFlakeAttrPaths() override
+ {
+ Strings res{"defaultApp." + settings.thisSystem.get()};
+ for (auto & s : SourceExprCommand::getDefaultFlakeAttrPaths())
+ res.push_back(s);
+ return res;
+ }
+
+ Strings getDefaultFlakeAttrPathPrefixes() override
+ {
+ Strings res{"apps." + settings.thisSystem.get() + ".", "packages"};
+ for (auto & s : SourceExprCommand::getDefaultFlakeAttrPathPrefixes())
+ res.push_back(s);
+ return res;
+ }
+
+ void run(ref store) override
+ {
+ auto evalState = getEvalState();
+
+ auto app = installable->toApp(*evalState);
+ store->buildPaths(app.context);
+
+ auto [bundlerFlakeRef, bundlerName] = parseFlakeRefWithFragment(bundler, absPath("."));
+ const flake::LockFlags lockFlags{ .writeLockFile = false };
+ auto bundler = InstallableFlake(
+ evalState, std::move(bundlerFlakeRef),
+ Strings{bundlerName == "" ? "defaultBundler" : bundlerName},
+ Strings({"bundlers."}), lockFlags);
+
+ Value * arg = evalState->allocValue();
+ evalState->mkAttrs(*arg, 2);
+
+ PathSet context;
+ for (auto & i : app.context)
+ context.insert("=" + store->printStorePath(i.path));
+ mkString(*evalState->allocAttr(*arg, evalState->symbols.create("program")), app.program, context);
+
+ mkString(*evalState->allocAttr(*arg, evalState->symbols.create("system")), settings.thisSystem.get());
+
+ arg->attrs->sort();
+
+ auto vRes = evalState->allocValue();
+ evalState->callFunction(*bundler.toValue(*evalState).first, *arg, *vRes, noPos);
+
+ if (!evalState->isDerivation(*vRes))
+ throw Error("the bundler '%s' does not produce a derivation", bundler.what());
+
+ auto attr1 = vRes->attrs->find(evalState->sDrvPath);
+ if (!attr1)
+ throw Error("the bundler '%s' does not produce a derivation", bundler.what());
+
+ PathSet context2;
+ StorePath drvPath = store->parseStorePath(evalState->coerceToPath(*attr1->pos, *attr1->value, context2));
+
+ auto attr2 = vRes->attrs->find(evalState->sOutPath);
+ if (!attr2)
+ throw Error("the bundler '%s' does not produce a derivation", bundler.what());
+
+ StorePath outPath = store->parseStorePath(evalState->coerceToPath(*attr2->pos, *attr2->value, context2));
+
+ store->buildPaths({{drvPath}});
+
+ auto outPathS = store->printStorePath(outPath);
+
+ auto info = store->queryPathInfo(outPath);
+ if (!info->references.empty())
+ throw Error("'%s' has references; a bundler must not leave any references", outPathS);
+
+ if (!outLink)
+ outLink = baseNameOf(app.program);
+
+ store.dynamic_pointer_cast()->addPermRoot(outPath, absPath(*outLink), true);
+ }
+};
+
+static auto r2 = registerCommand("bundle");
diff --git a/src/nix/command.hh b/src/nix/command.hh
index 856721ebf..bc46a2028 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -5,6 +5,7 @@
#include "common-eval-args.hh"
#include "path.hh"
#include "flake/lockfile.hh"
+#include "store-api.hh"
#include
@@ -185,7 +186,7 @@ static RegisterCommand registerCommand(const std::string & name)
}
Buildables build(ref store, Realise mode,
- std::vector> installables);
+ std::vector> installables, BuildMode bMode = bmNormal);
std::set toStorePaths(ref store,
Realise mode, OperateOn operateOn,
diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index 43b86c64c..12658078a 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -68,22 +68,22 @@ BuildEnvironment readEnvironment(const Path & path)
std::smatch match;
- if (std::regex_search(pos, file.cend(), match, declareRegex)) {
+ if (std::regex_search(pos, file.cend(), match, declareRegex, std::regex_constants::match_continuous)) {
pos = match[0].second;
exported.insert(match[1]);
}
- else if (std::regex_search(pos, file.cend(), match, varRegex)) {
+ else if (std::regex_search(pos, file.cend(), match, varRegex, std::regex_constants::match_continuous)) {
pos = match[0].second;
res.env.insert({match[1], Var { .exported = exported.count(match[1]) > 0, .value = match[2] }});
}
- else if (std::regex_search(pos, file.cend(), match, assocArrayRegex)) {
+ else if (std::regex_search(pos, file.cend(), match, assocArrayRegex, std::regex_constants::match_continuous)) {
pos = match[0].second;
res.env.insert({match[1], Var { .associative = true, .value = match[2] }});
}
- else if (std::regex_search(pos, file.cend(), match, functionRegex)) {
+ else if (std::regex_search(pos, file.cend(), match, functionRegex, std::regex_constants::match_continuous)) {
res.bashFunctions = std::string(pos, file.cend());
break;
}
diff --git a/src/nix/flake.cc b/src/nix/flake.cc
index 027a9871e..80d8654bc 100644
--- a/src/nix/flake.cc
+++ b/src/nix/flake.cc
@@ -368,6 +368,21 @@ struct CmdFlakeCheck : FlakeCommand
}
};
+ auto checkBundler = [&](const std::string & attrPath, Value & v, const Pos & pos) {
+ try {
+ state->forceValue(v, pos);
+ if (v.type != tLambda)
+ throw Error("bundler must be a function");
+ if (!v.lambda.fun->formals ||
+ v.lambda.fun->formals->argNames.find(state->symbols.create("program")) == v.lambda.fun->formals->argNames.end() ||
+ v.lambda.fun->formals->argNames.find(state->symbols.create("system")) == v.lambda.fun->formals->argNames.end())
+ throw Error("bundler must take formal arguments 'program' and 'system'");
+ } catch (Error & e) {
+ e.addTrace(pos, hintfmt("while checking the template '%s'", attrPath));
+ throw;
+ }
+ };
+
{
Activity act(*logger, lvlInfo, actUnknown, "evaluating flake");
@@ -490,6 +505,16 @@ struct CmdFlakeCheck : FlakeCommand
*attr.value, *attr.pos);
}
+ else if (name == "defaultBundler")
+ checkBundler(name, vOutput, pos);
+
+ else if (name == "bundlers") {
+ state->forceAttrs(vOutput, pos);
+ for (auto & attr : *vOutput.attrs)
+ checkBundler(fmt("%s.%s", name, attr.name),
+ *attr.value, *attr.pos);
+ }
+
else
warn("unknown flake output '%s'", name);
diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index b94751e45..0eca4f8ea 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -107,7 +107,7 @@ struct CmdToBase : Command
void run() override
{
for (auto s : args)
- logger->stdout(Hash(s, ht).to_string(base, base == SRI));
+ logger->stdout(Hash::parseAny(s, ht).to_string(base, base == SRI));
}
};
diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index 72aae7ef7..59b52ce95 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -437,7 +437,7 @@ ref openEvalCache(
std::shared_ptr lockedFlake,
bool useEvalCache)
{
- auto fingerprint = lockedFlake->getFingerprint();
+ auto fingerprint = lockedFlake->getFingerprint();
return make_ref(
useEvalCache && evalSettings.pureEval
? std::optional { std::cref(fingerprint) }
@@ -645,7 +645,7 @@ std::shared_ptr SourceExprCommand::parseInstallable(
}
Buildables build(ref store, Realise mode,
- std::vector> installables)
+ std::vector> installables, BuildMode bMode)
{
if (mode == Realise::Nothing)
settings.readOnlyMode = true;
@@ -671,7 +671,7 @@ Buildables build(ref store, Realise mode,
if (mode == Realise::Nothing)
printMissing(store, pathsToBuild, lvlError);
else if (mode == Realise::Outputs)
- store->buildPaths(pathsToBuild);
+ store->buildPaths(pathsToBuild, bMode);
return buildables;
}
diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc
index 65f73cd94..0c12efaf0 100644
--- a/src/nix/path-info.cc
+++ b/src/nix/path-info.cc
@@ -61,7 +61,7 @@ struct CmdPathInfo : StorePathsCommand, MixJSON
};
}
- void printSize(unsigned long long value)
+ void printSize(uint64_t value)
{
if (!humanReadable) {
std::cout << fmt("\t%11d", value);
diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh
index 40f1a4f76..b05a7d167 100644
--- a/tests/binary-cache.sh
+++ b/tests/binary-cache.sh
@@ -218,7 +218,9 @@ outPath=$(nix-build --no-out-link -E '
nix copy --to file://$cacheDir?write-nar-listing=1 $outPath
-[[ $(cat $cacheDir/$(basename $outPath).ls) = '{"version":1,"root":{"type":"directory","entries":{"bar":{"type":"regular","size":4,"narOffset":232},"link":{"type":"symlink","target":"xyzzy"}}}}' ]]
+diff -u \
+ <(jq -S < $cacheDir/$(basename $outPath).ls) \
+ <(echo '{"version":1,"root":{"type":"directory","entries":{"bar":{"type":"regular","size":4,"narOffset":232},"link":{"type":"symlink","target":"xyzzy"}}}}' | jq -S)
# Test debug info index generation.
@@ -234,4 +236,6 @@ outPath=$(nix-build --no-out-link -E '
nix copy --to "file://$cacheDir?index-debug-info=1&compression=none" $outPath
-[[ $(cat $cacheDir/debuginfo/02623eda209c26a59b1a8638ff7752f6b945c26b.debug) = '{"archive":"../nar/100vxs724qr46phz8m24iswmg9p3785hsyagz0kchf6q6gf06sw6.nar","member":"lib/debug/.build-id/02/623eda209c26a59b1a8638ff7752f6b945c26b.debug"}' ]]
+diff -u \
+ <(cat $cacheDir/debuginfo/02623eda209c26a59b1a8638ff7752f6b945c26b.debug | jq -S) \
+ <(echo '{"archive":"../nar/100vxs724qr46phz8m24iswmg9p3785hsyagz0kchf6q6gf06sw6.nar","member":"lib/debug/.build-id/02/623eda209c26a59b1a8638ff7752f6b945c26b.debug"}' | jq -S)
diff --git a/tests/check.sh b/tests/check.sh
index 5f25d04cb..5f4997e28 100644
--- a/tests/check.sh
+++ b/tests/check.sh
@@ -61,30 +61,30 @@ nix-build check.nix -A nondeterministic --no-out-link --repeat 1 2> $TEST_ROOT/l
[ "$status" = "1" ]
grep 'differs from previous round' $TEST_ROOT/log
-path=$(nix-build check.nix -A fetchurl --no-out-link --hashed-mirrors '')
+path=$(nix-build check.nix -A fetchurl --no-out-link)
chmod +w $path
echo foo > $path
chmod -w $path
-nix-build check.nix -A fetchurl --no-out-link --check --hashed-mirrors ''
+nix-build check.nix -A fetchurl --no-out-link --check
# Note: "check" doesn't repair anything, it just compares to the hash stored in the database.
[[ $(cat $path) = foo ]]
-nix-build check.nix -A fetchurl --no-out-link --repair --hashed-mirrors ''
+nix-build check.nix -A fetchurl --no-out-link --repair
[[ $(cat $path) != foo ]]
-nix-build check.nix -A hashmismatch --no-out-link --hashed-mirrors '' || status=$?
+nix-build check.nix -A hashmismatch --no-out-link || status=$?
[ "$status" = "102" ]
echo -n > ./dummy
-nix-build check.nix -A hashmismatch --no-out-link --hashed-mirrors ''
+nix-build check.nix -A hashmismatch --no-out-link
echo 'Hello World' > ./dummy
-nix-build check.nix -A hashmismatch --no-out-link --check --hashed-mirrors '' || status=$?
+nix-build check.nix -A hashmismatch --no-out-link --check || status=$?
[ "$status" = "102" ]
# Multiple failures with --keep-going
nix-build check.nix -A nondeterministic --no-out-link
-nix-build check.nix -A nondeterministic -A hashmismatch --no-out-link --check --keep-going --hashed-mirrors '' || status=$?
+nix-build check.nix -A nondeterministic -A hashmismatch --no-out-link --check --keep-going || status=$?
[ "$status" = "110" ]
diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh
index 9faa5d9f6..cedd796f7 100644
--- a/tests/fetchGit.sh
+++ b/tests/fetchGit.sh
@@ -32,6 +32,8 @@ rev2=$(git -C $repo rev-parse HEAD)
# Fetch a worktree
unset _NIX_FORCE_HTTP
path0=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$TEST_ROOT/worktree).outPath")
+path0_=$(nix eval --impure --raw --expr "(builtins.fetchTree { type = \"git\"; url = file://$TEST_ROOT/worktree; }).outPath")
+[[ $path0 = $path0_ ]]
export _NIX_FORCE_HTTP=1
[[ $(tail -n 1 $path0/hello) = "hello" ]]
@@ -102,6 +104,12 @@ git -C $repo commit -m 'Bla3' -a
path4=$(nix eval --impure --refresh --raw --expr "(builtins.fetchGit file://$repo).outPath")
[[ $path2 = $path4 ]]
+nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; rev = \"$rev2\"; narHash = \"sha256-B5yIPHhEm0eysJKEsO7nqxprh9vcblFxpJG11gXJus1=\"; }).outPath" || status=$?
+[[ "$status" = "102" ]]
+
+path5=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; rev = \"$rev2\"; narHash = \"sha256-Hr8g6AqANb3xqX28eu1XnjK/3ab8Gv6TJSnkb1LezG9=\"; }).outPath")
+[[ $path = $path5 ]]
+
# tarball-ttl should be ignored if we specify a rev
echo delft > $repo/hello
git -C $repo add hello
diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh
index 2535651b0..0f2044342 100644
--- a/tests/fetchurl.sh
+++ b/tests/fetchurl.sh
@@ -5,7 +5,7 @@ clearStore
# Test fetching a flat file.
hash=$(nix-hash --flat --type sha256 ./fetchurl.sh)
-outPath=$(nix-build '' --argstr url file://$(pwd)/fetchurl.sh --argstr sha256 $hash --no-out-link --hashed-mirrors '')
+outPath=$(nix-build '' --argstr url file://$(pwd)/fetchurl.sh --argstr sha256 $hash --no-out-link)
cmp $outPath fetchurl.sh
@@ -14,7 +14,7 @@ clearStore
hash=$(nix hash-file --type sha512 --base64 ./fetchurl.sh)
-outPath=$(nix-build '' --argstr url file://$(pwd)/fetchurl.sh --argstr sha512 $hash --no-out-link --hashed-mirrors '')
+outPath=$(nix-build '' --argstr url file://$(pwd)/fetchurl.sh --argstr sha512 $hash --no-out-link)
cmp $outPath fetchurl.sh
@@ -25,26 +25,24 @@ hash=$(nix hash-file ./fetchurl.sh)
[[ $hash =~ ^sha256- ]]
-outPath=$(nix-build '' --argstr url file://$(pwd)/fetchurl.sh --argstr hash $hash --no-out-link --hashed-mirrors '')
+outPath=$(nix-build '' --argstr url file://$(pwd)/fetchurl.sh --argstr hash $hash --no-out-link)
cmp $outPath fetchurl.sh
-# Test the hashed mirror feature.
+# Test that we can substitute from a different store dir.
clearStore
-hash=$(nix hash-file --type sha512 --base64 ./fetchurl.sh)
-hash32=$(nix hash-file --type sha512 --base16 ./fetchurl.sh)
+other_store=file://$TEST_ROOT/other_store?store=/fnord/store
-mirror=$TEST_ROOT/hashed-mirror
-rm -rf $mirror
-mkdir -p $mirror/sha512
-ln -s $(pwd)/fetchurl.sh $mirror/sha512/$hash32
+hash=$(nix hash-file --type sha256 --base16 ./fetchurl.sh)
-outPath=$(nix-build '' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha512 $hash --no-out-link --hashed-mirrors "file://$mirror")
+storePath=$(nix --store $other_store add-to-store --flat ./fetchurl.sh)
+
+outPath=$(nix-build '' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 $hash --no-out-link --substituters $other_store)
# Test hashed mirrors with an SRI hash.
-nix-build '' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix to-sri --type sha512 $hash) \
- --argstr name bla --no-out-link --hashed-mirrors "file://$mirror"
+nix-build '' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix to-sri --type sha256 $hash) \
+ --no-out-link --substituters $other_store
# Test unpacking a NAR.
rm -rf $TEST_ROOT/archive
diff --git a/tests/filter-source.sh b/tests/filter-source.sh
index 1f8dceee5..ba34d2eac 100644
--- a/tests/filter-source.sh
+++ b/tests/filter-source.sh
@@ -10,10 +10,16 @@ touch $TEST_ROOT/filterin/bak
touch $TEST_ROOT/filterin/bla.c.bak
ln -s xyzzy $TEST_ROOT/filterin/link
-nix-build ./filter-source.nix -o $TEST_ROOT/filterout
+checkFilter() {
+ test ! -e $1/foo/bar
+ test -e $1/xyzzy
+ test -e $1/bak
+ test ! -e $1/bla.c.bak
+ test ! -L $1/link
+}
-test ! -e $TEST_ROOT/filterout/foo/bar
-test -e $TEST_ROOT/filterout/xyzzy
-test -e $TEST_ROOT/filterout/bak
-test ! -e $TEST_ROOT/filterout/bla.c.bak
-test ! -L $TEST_ROOT/filterout/link
+nix-build ./filter-source.nix -o $TEST_ROOT/filterout1
+checkFilter $TEST_ROOT/filterout1
+
+nix-build ./path.nix -o $TEST_ROOT/filterout2
+checkFilter $TEST_ROOT/filterout2
diff --git a/tests/nar-access.sh b/tests/nar-access.sh
index 553d6ca89..88b997ca6 100644
--- a/tests/nar-access.sh
+++ b/tests/nar-access.sh
@@ -26,12 +26,24 @@ nix cat-store $storePath/foo/baz > baz.cat-nar
diff -u baz.cat-nar $storePath/foo/baz
# Test --json.
-[[ $(nix ls-nar --json $narFile /) = '{"type":"directory","entries":{"foo":{},"foo-x":{},"qux":{},"zyx":{}}}' ]]
-[[ $(nix ls-nar --json -R $narFile /foo) = '{"type":"directory","entries":{"bar":{"type":"regular","size":0,"narOffset":368},"baz":{"type":"regular","size":0,"narOffset":552},"data":{"type":"regular","size":58,"narOffset":736}}}' ]]
-[[ $(nix ls-nar --json -R $narFile /foo/bar) = '{"type":"regular","size":0,"narOffset":368}' ]]
-[[ $(nix ls-store --json $storePath) = '{"type":"directory","entries":{"foo":{},"foo-x":{},"qux":{},"zyx":{}}}' ]]
-[[ $(nix ls-store --json -R $storePath/foo) = '{"type":"directory","entries":{"bar":{"type":"regular","size":0},"baz":{"type":"regular","size":0},"data":{"type":"regular","size":58}}}' ]]
-[[ $(nix ls-store --json -R $storePath/foo/bar) = '{"type":"regular","size":0}' ]]
+diff -u \
+ <(nix ls-nar --json $narFile / | jq -S) \
+ <(echo '{"type":"directory","entries":{"foo":{},"foo-x":{},"qux":{},"zyx":{}}}' | jq -S)
+diff -u \
+ <(nix ls-nar --json -R $narFile /foo | jq -S) \
+ <(echo '{"type":"directory","entries":{"bar":{"type":"regular","size":0,"narOffset":368},"baz":{"type":"regular","size":0,"narOffset":552},"data":{"type":"regular","size":58,"narOffset":736}}}' | jq -S)
+diff -u \
+ <(nix ls-nar --json -R $narFile /foo/bar | jq -S) \
+ <(echo '{"type":"regular","size":0,"narOffset":368}' | jq -S)
+diff -u \
+ <(nix ls-store --json $storePath | jq -S) \
+ <(echo '{"type":"directory","entries":{"foo":{},"foo-x":{},"qux":{},"zyx":{}}}' | jq -S)
+diff -u \
+ <(nix ls-store --json -R $storePath/foo | jq -S) \
+ <(echo '{"type":"directory","entries":{"bar":{"type":"regular","size":0},"baz":{"type":"regular","size":0},"data":{"type":"regular","size":58}}}' | jq -S)
+diff -u \
+ <(nix ls-store --json -R $storePath/foo/bar| jq -S) \
+ <(echo '{"type":"regular","size":0}' | jq -S)
# Test missing files.
nix ls-store --json -R $storePath/xyzzy 2>&1 | grep 'does not exist in NAR'
diff --git a/tests/path.nix b/tests/path.nix
new file mode 100644
index 000000000..883c3c41b
--- /dev/null
+++ b/tests/path.nix
@@ -0,0 +1,14 @@
+with import ./config.nix;
+
+mkDerivation {
+ name = "filter";
+ builder = builtins.toFile "builder" "ln -s $input $out";
+ input =
+ builtins.path {
+ path = ((builtins.getEnv "TEST_ROOT") + "/filterin");
+ filter = path: type:
+ type != "symlink"
+ && baseNameOf path != "foo"
+ && !((import ./lang/lib.nix).hasSuffix ".bak" (baseNameOf path));
+ };
+}
diff --git a/tests/tarball.sh b/tests/tarball.sh
index b3ec16d40..88a1a07a0 100644
--- a/tests/tarball.sh
+++ b/tests/tarball.sh
@@ -27,10 +27,13 @@ test_tarball() {
nix-build -o $TEST_ROOT/result -E "import (fetchTarball file://$tarball)"
- nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree file://$tarball)"
- nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; })"
- nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })"
- nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"sha256-xdKv2pq/IiwLSnBBJXW8hNowI4MrdZfW+SYqDQs7Tzc=\"; })" 2>&1 | grep 'NAR hash mismatch in input'
+ nix-build -o $TEST_ROOT/result -E "import (fetchTree file://$tarball)"
+ nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; })"
+ nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })"
+ nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"sha256-xdKv2pq/IiwLSnBBJXW8hNowI4MrdZfW+SYqDQs7Tzc=\"; })" 2>&1 | grep 'NAR hash mismatch in input'
+
+ nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" >&2
+ nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" 2>&1 | grep 'true'
nix-instantiate --eval -E '1 + 2' -I fnord=file://no-such-tarball.tar$ext
nix-instantiate --eval -E 'with ; 1 + 2' -I fnord=file://no-such-tarball$ext