lix-project/lix-releng-staging
0
0
Fork 0
Code Packages Activity

Merge branch 'master' of github.com:NixOS/nix into better-ca-parse-errors

Browse source
This commit is contained in:
Carlo Nucera 2020-07-30 18:27:25 -04:00
parent 1d7d94ceea 0744f7f83b
commit eee6ef86cd
44 changed files with 550 additions and 344 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.version
View file

@ -1 +1 @@
2.4
3.0

28
doc/manual/command-ref/conf-file.xml
View file

@ -370,34 +370,6 @@ false</literal>.</para>
</varlistentry>
<varlistentry xml:id="conf-hashed-mirrors"><term><literal>hashed-mirrors</literal></term>
<listitem><para>A list of web servers used by
<function>builtins.fetchurl</function> to obtain files by
hash. The default is
<literal>http://tarballs.nixos.org/</literal>. Given a hash type
<replaceable>ht</replaceable> and a base-16 hash
<replaceable>h</replaceable>, Nix will try to download the file
from
<literal>hashed-mirror/<replaceable>ht</replaceable>/<replaceable>h</replaceable></literal>.
This allows files to be downloaded even if they have disappeared
from their original URI. For example, given the default mirror
<literal>http://tarballs.nixos.org/</literal>, when building the derivation
<programlisting>
builtins.fetchurl {
url = "https://example.org/foo-1.2.3.tar.xz";
sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae";
}
</programlisting>
Nix will attempt to download this file from
<literal>http://tarballs.nixos.org/sha256/2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae</literal>
first. If it is not available there, if will try the original URI.</para></listitem>
</varlistentry>
<varlistentry xml:id="conf-http-connections"><term><literal>http-connections</literal></term>
<listitem><para>The maximum number of parallel TCP connections

8
src/build-remote/build-remote.cc
View file

@ -33,7 +33,7 @@ std::string escapeUri(std::string uri)
static string currentLoad;
static AutoCloseFD openSlotLock(const Machine & m, unsigned long long slot)
static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot)
{
return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true);
}
@ -119,7 +119,7 @@ static int _main(int argc, char * * argv)
bool rightType = false;
Machine * bestMachine = nullptr;
unsigned long long bestLoad = 0;
uint64_t bestLoad = 0;
for (auto & m : machines) {
debug("considering building on remote machine '%s'", m.storeUri);
@ -130,8 +130,8 @@ static int _main(int argc, char * * argv)
m.mandatoryMet(requiredFeatures)) {
rightType = true;
AutoCloseFD free;
unsigned long long load = 0;
for (unsigned long long slot = 0; slot < m.maxJobs; ++slot) {
uint64_t load = 0;
for (uint64_t slot = 0; slot < m.maxJobs; ++slot) {
auto slotLock = openSlotLock(m, slot);
if (lockFile(slotLock.get(), ltWrite, false)) {
if (!free) {

2
src/libexpr/flake/flake.hh
View file

@ -106,6 +106,6 @@ void emitTreeAttrs(
EvalState & state,
const fetchers::Tree & tree,
const fetchers::Input & input,
Value & v);
Value & v, bool emptyRevFallback = false);
}

2
src/libexpr/primops.cc
View file

@ -65,7 +65,7 @@ void EvalState::realiseContext(const PathSet & context)
/* For performance, prefetch all substitute info. */
StorePathSet willBuild, willSubstitute, unknown;
unsigned long long downloadSize, narSize;
uint64_t downloadSize, narSize;
store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize);
store->buildPaths(drvs);

91
src/libexpr/primops/fetchGit.cc
View file

@ -1,91 +0,0 @@
#include "primops.hh"
#include "eval-inline.hh"
#include "store-api.hh"
#include "hash.hh"
#include "fetchers.hh"
#include "url.hh"
namespace nix {
static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
std::string url;
std::optional<std::string> ref;
std::optional<Hash> rev;
std::string name = "source";
bool fetchSubmodules = false;
PathSet context;
state.forceValue(*args[0]);
if (args[0]->type == tAttrs) {
state.forceAttrs(*args[0], pos);
for (auto & attr : *args[0]->attrs) {
string n(attr.name);
if (n == "url")
url = state.coerceToString(*attr.pos, *attr.value, context, false, false);
else if (n == "ref")
ref = state.forceStringNoCtx(*attr.value, *attr.pos);
else if (n == "rev")
rev = Hash::parseAny(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA1);
else if (n == "name")
name = state.forceStringNoCtx(*attr.value, *attr.pos);
else if (n == "submodules")
fetchSubmodules = state.forceBool(*attr.value, *attr.pos);
else
throw EvalError({
.hint = hintfmt("unsupported argument '%s' to 'fetchGit'", attr.name),
.errPos = *attr.pos
});
}
if (url.empty())
throw EvalError({
.hint = hintfmt("'url' argument required"),
.errPos = pos
});
} else
url = state.coerceToString(pos, *args[0], context, false, false);
// FIXME: git externals probably can be used to bypass the URI
// whitelist. Ah well.
state.checkURI(url);
if (evalSettings.pureEval && !rev)
throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision");
fetchers::Attrs attrs;
attrs.insert_or_assign("type", "git");
attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url);
if (ref) attrs.insert_or_assign("ref", *ref);
if (rev) attrs.insert_or_assign("rev", rev->gitRev());
if (fetchSubmodules) attrs.insert_or_assign("submodules", fetchers::Explicit<bool>{true});
auto input = fetchers::Input::fromAttrs(std::move(attrs));
// FIXME: use name?
auto [tree, input2] = input.fetch(state.store);
state.mkAttrs(v, 8);
auto storePath = state.store->printStorePath(tree.storePath);
mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
// Backward compatibility: set 'rev' to
// 0000000000000000000000000000000000000000 for a dirty tree.
auto rev2 = input2.getRev().value_or(Hash(htSHA1));
mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev());
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev2.gitShortRev());
// Backward compatibility: set 'revCount' to 0 for a dirty tree.
mkInt(*state.allocAttr(v, state.symbols.create("revCount")),
input2.getRevCount().value_or(0));
mkBool(*state.allocAttr(v, state.symbols.create("submodules")), fetchSubmodules);
v.attrs->sort();
if (state.allowedPaths)
state.allowedPaths->insert(tree.actualPath);
}
static RegisterPrimOp r("fetchGit", 1, prim_fetchGit);
}

75
src/libexpr/primops/fetchTree.cc
View file

@ -14,7 +14,8 @@ void emitTreeAttrs(
EvalState & state,
const fetchers::Tree & tree,
const fetchers::Input & input,
Value & v)
Value & v,
bool emptyRevFallback)
{
assert(input.isImmutable());
@ -34,10 +35,20 @@ void emitTreeAttrs(
if (auto rev = input.getRev()) {
mkString(*state.allocAttr(v, state.symbols.create("rev")), rev->gitRev());
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev->gitShortRev());
} else if (emptyRevFallback) {
// Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev
auto emptyHash = Hash(htSHA1);
mkString(*state.allocAttr(v, state.symbols.create("rev")), emptyHash.gitRev());
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), emptyHash.gitRev());
}
if (input.getType() == "git")
mkBool(*state.allocAttr(v, state.symbols.create("submodules")), maybeGetBoolAttr(input.attrs, "submodules").value_or(false));
if (auto revCount = input.getRevCount())
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount);
else if (emptyRevFallback)
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), 0);
if (auto lastModified = input.getLastModified()) {
mkInt(*state.allocAttr(v, state.symbols.create("lastModified")), *lastModified);
@ -48,10 +59,26 @@ void emitTreeAttrs(
v.attrs->sort();
}
static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v)
std::string fixURI(std::string uri, EvalState &state)
{
settings.requireExperimentalFeature("flakes");
state.checkURI(uri);
return uri.find("://") != std::string::npos ? uri : "file://" + uri;
}
void addURI(EvalState &state, fetchers::Attrs &attrs, Symbol name, std::string v)
{
string n(name);
attrs.emplace(name, n == "url" ? fixURI(v, state) : v);
}
static void fetchTree(
EvalState &state,
const Pos &pos,
Value **args,
Value &v,
const std::optional<std::string> type,
bool emptyRevFallback = false
) {
fetchers::Input input;
PathSet context;
@ -64,8 +91,15 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V
for (auto & attr : *args[0]->attrs) {
state.forceValue(*attr.value);
if (attr.value->type == tString)
attrs.emplace(attr.name, attr.value->string.s);
if (attr.value->type == tPath || attr.value->type == tString)
addURI(
state,
attrs,
attr.name,
state.coerceToString(*attr.pos, *attr.value, context, false, false)
);
else if (attr.value->type == tString)
addURI(state, attrs, attr.name, attr.value->string.s);
else if (attr.value->type == tBool)
attrs.emplace(attr.name, fetchers::Explicit<bool>{attr.value->boolean});
else if (attr.value->type == tInt)
@ -75,6 +109,9 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V
attr.name, showType(*attr.value));
}
if (type)
attrs.emplace("type", type.value());
if (!attrs.count("type"))
throw Error({
.hint = hintfmt("attribute 'type' is missing in call to 'fetchTree'"),
@ -82,8 +119,18 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V
});
input = fetchers::Input::fromAttrs(std::move(attrs));
} else
input = fetchers::Input::fromURL(state.coerceToString(pos, *args[0], context, false, false));
} else {
auto url = fixURI(state.coerceToString(pos, *args[0], context, false, false), state);
if (type == "git") {
fetchers::Attrs attrs;
attrs.emplace("type", "git");
attrs.emplace("url", url);
input = fetchers::Input::fromAttrs(std::move(attrs));
} else {
input = fetchers::Input::fromURL(url);
}
}
if (!evalSettings.pureEval && !input.isDirect())
input = lookupInRegistries(state.store, input).first;
@ -96,7 +143,13 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V
if (state.allowedPaths)
state.allowedPaths->insert(tree.actualPath);
emitTreeAttrs(state, tree, input2, v);
emitTreeAttrs(state, tree, input2, v, emptyRevFallback);
}
static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
settings.requireExperimentalFeature("flakes");
fetchTree(state, pos, args, v, std::nullopt);
}
static RegisterPrimOp r("fetchTree", 1, prim_fetchTree);
@ -178,7 +231,13 @@ static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args
fetch(state, pos, args, v, "fetchTarball", true, "source");
}
static void prim_fetchGit(EvalState &state, const Pos &pos, Value **args, Value &v)
{
fetchTree(state, pos, args, v, "git", true);
}
static RegisterPrimOp r2("__fetchurl", 1, prim_fetchurl);
static RegisterPrimOp r3("fetchTarball", 1, prim_fetchTarball);
static RegisterPrimOp r4("fetchGit", 1, prim_fetchGit);
}

2
src/libfetchers/fetchers.cc
View file

@ -134,7 +134,7 @@ std::pair<Tree, Input> Input::fetch(ref<Store> store) const
if (auto prevNarHash = getNarHash()) {
if (narHash != *prevNarHash)
throw Error("NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'",
throw Error((unsigned int) 102, "NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'",
to_string(), tree.actualPath, prevNarHash->to_string(SRI, true), narHash->to_string(SRI, true));
}

6
src/libmain/shared.cc
View file

@ -36,7 +36,7 @@ void printGCWarning()
void printMissing(ref<Store> store, const std::vector<StorePathWithOutputs> & paths, Verbosity lvl)
{
unsigned long long downloadSize, narSize;
uint64_t downloadSize, narSize;
StorePathSet willBuild, willSubstitute, unknown;
store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize);
printMissing(store, willBuild, willSubstitute, unknown, downloadSize, narSize, lvl);
@ -45,7 +45,7 @@ void printMissing(ref<Store> store, const std::vector<StorePathWithOutputs> & pa
void printMissing(ref<Store> store, const StorePathSet & willBuild,
const StorePathSet & willSubstitute, const StorePathSet & unknown,
unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl)
uint64_t downloadSize, uint64_t narSize, Verbosity lvl)
{
if (!willBuild.empty()) {
if (willBuild.size() == 1)
@ -384,7 +384,7 @@ RunPager::~RunPager()
}
string showBytes(unsigned long long bytes)
string showBytes(uint64_t bytes)
{
return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str();
}

4
src/libmain/shared.hh
View file

@ -47,7 +47,7 @@ void printMissing(
void printMissing(ref<Store> store, const StorePathSet & willBuild,
const StorePathSet & willSubstitute, const StorePathSet & unknown,
unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl = lvlInfo);
uint64_t downloadSize, uint64_t narSize, Verbosity lvl = lvlInfo);
string getArg(const string & opt,
Strings::iterator & i, const Strings::iterator & end);
@ -110,7 +110,7 @@ extern volatile ::sig_atomic_t blockInt;
/* GC helpers. */
string showBytes(unsigned long long bytes);
string showBytes(uint64_t bytes);
struct GCResults;

53
src/libstore/build.cc
View file

@ -297,7 +297,7 @@ public:
GoalPtr makeDerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(const StorePath & drvPath,
const BasicDerivation & drv, BuildMode buildMode = bmNormal);
GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair);
GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
/* Remove a dead goal. */
void removeGoal(GoalPtr goal);
@ -1206,7 +1206,7 @@ void DerivationGoal::haveDerivation()
them. */
if (settings.useSubstitutes && parsedDrv->substitutesAllowed())
for (auto & i : invalidOutputs)
addWaitee(worker.makeSubstitutionGoal(i, buildMode == bmRepair ? Repair : NoRepair));
addWaitee(worker.makeSubstitutionGoal(i, buildMode == bmRepair ? Repair : NoRepair, getDerivationCA(*drv)));
if (waitees.empty()) /* to prevent hang (no wake-up event) */
outputsSubstituted();
@ -1646,13 +1646,13 @@ void DerivationGoal::buildDone()
So instead, check if the disk is (nearly) full now. If
so, we don't mark this build as a permanent failure. */
#if HAVE_STATVFS
unsigned long long required = 8ULL * 1024 * 1024; // FIXME: make configurable
uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable
struct statvfs st;
if (statvfs(worker.store.realStoreDir.c_str(), &st) == 0 &&
(unsigned long long) st.f_bavail * st.f_bsize < required)
(uint64_t) st.f_bavail * st.f_bsize < required)
diskFull = true;
if (statvfs(tmpDir.c_str(), &st) == 0 &&
(unsigned long long) st.f_bavail * st.f_bsize < required)
(uint64_t) st.f_bavail * st.f_bsize < required)
diskFull = true;
#endif
@ -2851,7 +2851,7 @@ struct RestrictedStore : public LocalFSStore
void queryMissing(const std::vector<StorePathWithOutputs> & targets,
StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
unsigned long long & downloadSize, unsigned long long & narSize) override
uint64_t & downloadSize, uint64_t & narSize) override
{
/* This is slightly impure since it leaks information to the
client about what paths will be built/substituted or are
@ -4276,6 +4276,10 @@ private:
/* The store path that should be realised through a substitute. */
StorePath storePath;
/* The path the substituter refers to the path as. This will be
* different when the stores have different names. */
std::optional<StorePath> subPath;
/* The remaining substituters. */
std::list<ref<Store>> subs;
@ -4309,8 +4313,11 @@ private:
typedef void (SubstitutionGoal::*GoalState)();
GoalState state;
/* Content address for recomputing store path */
std::optional<ContentAddress> ca;
public:
SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair);
SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
~SubstitutionGoal();
void timedOut(Error && ex) override { abort(); };
@ -4340,10 +4347,11 @@ public:
};
SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair)
SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
: Goal(worker)
, storePath(storePath)
, repair(repair)
, ca(ca)
{
state = &SubstitutionGoal::init;
name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath));
@ -4418,14 +4426,18 @@ void SubstitutionGoal::tryNext()
sub = subs.front();
subs.pop_front();
if (sub->storeDir != worker.store.storeDir) {
if (ca) {
subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca);
if (sub->storeDir == worker.store.storeDir)
assert(subPath == storePath);
} else if (sub->storeDir != worker.store.storeDir) {
tryNext();
return;
}
try {
// FIXME: make async
info = sub->queryPathInfo(storePath);
info = sub->queryPathInfo(subPath ? *subPath : storePath);
} catch (InvalidPath &) {
tryNext();
return;
@ -4444,6 +4456,19 @@ void SubstitutionGoal::tryNext()
throw;
}
if (info->path != storePath) {
if (info->isContentAddressed(*sub) && info->references.empty()) {
auto info2 = std::make_shared<ValidPathInfo>(*info);
info2->path = storePath;
info = info2;
} else {
printError("asked '%s' for '%s' but got '%s'",
sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path));
tryNext();
return;
}
}
/* Update the total expected download size. */
auto narInfo = std::dynamic_pointer_cast<const NarInfo>(info);
@ -4533,7 +4558,7 @@ void SubstitutionGoal::tryToRun()
PushActivity pact(act.id);
copyStorePath(ref<Store>(sub), ref<Store>(worker.store.shared_from_this()),
storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs);
subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs);
promise.set_value();
} catch (...) {
@ -4666,11 +4691,11 @@ std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath
}
GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair)
GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca)
{
GoalPtr goal = substitutionGoals[path].lock(); // FIXME
if (!goal) {
goal = std::make_shared<SubstitutionGoal>(path, *this, repair);
goal = std::make_shared<SubstitutionGoal>(path, *this, repair, ca);
substitutionGoals.insert_or_assign(path, goal);
wakeUp(goal);
}
@ -5038,7 +5063,7 @@ void Worker::markContentsGood(const StorePath & path)
static void primeCache(Store & store, const std::vector<StorePathWithOutputs> & paths)
{
StorePathSet willBuild, willSubstitute, unknown;
unsigned long long downloadSize, narSize;
uint64_t downloadSize, narSize;
store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize);
if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty())

20
src/libstore/builtins/fetchurl.cc
View file

@ -58,26 +58,6 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
}
};
/* We always have one output, and if it's a fixed-output derivation (as
checked below) it must be the only output */
auto & output = drv.outputs.begin()->second;
/* Try the hashed mirrors first. */
if (auto hash = std::get_if<DerivationOutputFixed>(&output.output)) {
if (hash->hash.method == FileIngestionMethod::Flat) {
for (auto hashedMirror : settings.hashedMirrors.get()) {
try {
if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
fetch(hashedMirror + printHashType(hash->hash.hash.type) + "/" + hash->hash.hash.to_string(Base16, false));
return;
} catch (Error & e) {
debug(e.what());
}
}
}
}
/* Otherwise try the specified URL. */
fetch(mainUrl);
}

106
src/libstore/daemon.cc
View file

@ -86,7 +86,7 @@ struct TunnelLogger : public Logger
}
/* startWork() means that we're starting an operation for which we
want to send out stderr to the client. */
want to send out stderr to the client. */
void startWork()
{
auto state(state_.lock());
@ -579,7 +579,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
auto path = store->parseStorePath(readString(from));
logger->startWork();
SubstitutablePathInfos infos;
store->querySubstitutablePathInfos({path}, infos);
store->querySubstitutablePathInfos({{path, std::nullopt}}, infos);
logger->stopWork();
auto i = infos.find(path);
if (i == infos.end())
@ -595,10 +595,16 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
}
case wopQuerySubstitutablePathInfos: {
auto paths = readStorePaths<StorePathSet>(*store, from);
logger->startWork();
SubstitutablePathInfos infos;
store->querySubstitutablePathInfos(paths, infos);
StorePathCAMap pathsMap = {};
if (GET_PROTOCOL_MINOR(clientVersion) < 22) {
auto paths = readStorePaths<StorePathSet>(*store, from);
for (auto & path : paths)
pathsMap.emplace(path, std::nullopt);
} else
pathsMap = readStorePathCAMap(*store, from);
logger->startWork();
store->querySubstitutablePathInfos(pathsMap, infos);
logger->stopWork();
to << infos.size();
for (auto & i : infos) {
@ -703,24 +709,84 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
if (!trusted)
info.ultimate = false;
std::unique_ptr<Source> source;
if (GET_PROTOCOL_MINOR(clientVersion) >= 21)
source = std::make_unique<TunnelSource>(from, to);
else {
StringSink saved;
TeeSource tee { from, saved };
ParseSink ether;
parseDump(ether, tee);
source = std::make_unique<StringSource>(std::move(*saved.s));
if (GET_PROTOCOL_MINOR(clientVersion) >= 23) {
struct FramedSource : Source
{
Source & from;
bool eof = false;
std::vector<unsigned char> pending;
size_t pos = 0;
FramedSource(Source & from) : from(from)
{ }
~FramedSource()
{
if (!eof) {
while (true) {
auto n = readInt(from);
if (!n) break;
std::vector<unsigned char> data(n);
from(data.data(), n);
}
}
}
size_t read(unsigned char * data, size_t len) override
{
if (eof) throw EndOfFile("reached end of FramedSource");
if (pos >= pending.size()) {
size_t len = readInt(from);
if (!len) {
eof = true;
return 0;
}
pending = std::vector<unsigned char>(len);
pos = 0;
from(pending.data(), len);
}
auto n = std::min(len, pending.size() - pos);
memcpy(data, pending.data() + pos, n);
pos += n;
return n;
}
};
logger->startWork();
{
FramedSource source(from);
store->addToStore(info, source, (RepairFlag) repair,
dontCheckSigs ? NoCheckSigs : CheckSigs);
}
logger->stopWork();
}
logger->startWork();
else {
std::unique_ptr<Source> source;
if (GET_PROTOCOL_MINOR(clientVersion) >= 21)
source = std::make_unique<TunnelSource>(from, to);
else {
StringSink saved;
TeeSource tee { from, saved };
ParseSink ether;
parseDump(ether, tee);
source = std::make_unique<StringSource>(std::move(*saved.s));
}
// FIXME: race if addToStore doesn't read source?
store->addToStore(info, *source, (RepairFlag) repair,
dontCheckSigs ? NoCheckSigs : CheckSigs);
logger->startWork();
// FIXME: race if addToStore doesn't read source?
store->addToStore(info, *source, (RepairFlag) repair,
dontCheckSigs ? NoCheckSigs : CheckSigs);
logger->stopWork();
}
logger->stopWork();
break;
}
@ -730,7 +796,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
targets.push_back(store->parsePathWithOutputs(s));
logger->startWork();
StorePathSet willBuild, willSubstitute, unknown;
unsigned long long downloadSize, narSize;
uint64_t downloadSize, narSize;
store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize);
logger->stopWork();
writeStorePaths(*store, to, willBuild);

14
src/libstore/gc.cc
View file

@ -500,7 +500,7 @@ struct LocalStore::GCState
StorePathSet alive;
bool gcKeepOutputs;
bool gcKeepDerivations;
unsigned long long bytesInvalidated;
uint64_t bytesInvalidated;
bool moveToTrash = true;
bool shouldDelete;
GCState(const GCOptions & options, GCResults & results)
@ -518,7 +518,7 @@ bool LocalStore::isActiveTempFile(const GCState & state,
void LocalStore::deleteGarbage(GCState & state, const Path & path)
{
unsigned long long bytesFreed;
uint64_t bytesFreed;
deletePath(path, bytesFreed);
state.results.bytesFreed += bytesFreed;
}
@ -528,7 +528,7 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path)
{
checkInterrupt();
unsigned long long size = 0;
uint64_t size = 0;
auto storePath = maybeParseStorePath(path);
if (storePath && isValidPath(*storePath)) {
@ -687,7 +687,7 @@ void LocalStore::removeUnusedLinks(const GCState & state)
AutoCloseDir dir(opendir(linksDir.c_str()));
if (!dir) throw SysError("opening directory '%1%'", linksDir);
long long actualSize = 0, unsharedSize = 0;
int64_t actualSize = 0, unsharedSize = 0;
struct dirent * dirent;
while (errno = 0, dirent = readdir(dir.get())) {
@ -717,10 +717,10 @@ void LocalStore::removeUnusedLinks(const GCState & state)
struct stat st;
if (stat(linksDir.c_str(), &st) == -1)
throw SysError("statting '%1%'", linksDir);
long long overhead = st.st_blocks * 512ULL;
auto overhead = st.st_blocks * 512ULL;
printInfo(format("note: currently hard linking saves %.2f MiB")
% ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0)));
printInfo("note: currently hard linking saves %.2f MiB",
((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0)));
}

3
src/libstore/globals.hh
View file

@ -335,9 +335,6 @@ public:
"setuid/setgid bits or with file capabilities."};
#endif
Setting<Strings> hashedMirrors{this, {"http://tarballs.nixos.org/"}, "hashed-mirrors",
"A list of servers used by builtins.fetchurl to fetch files by hash."};
Setting<uint64_t> minFree{this, 0, "min-free",
"Automatically run the garbage collector when free disk space drops below the specified amount."};

26
src/libstore/local-store.cc
View file

@ -839,20 +839,32 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths)
}
void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths,
SubstitutablePathInfos & infos)
void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos)
{
if (!settings.useSubstitutes) return;
for (auto & sub : getDefaultSubstituters()) {
if (sub->storeDir != storeDir) continue;
for (auto & path : paths) {
if (infos.count(path)) continue;
debug("checking substituter '%s' for path '%s'", sub->getUri(), printStorePath(path));
auto subPath(path.first);
// recompute store path so that we can use a different store root
if (path.second) {
subPath = makeFixedOutputPathFromCA(path.first.name(), *path.second);
if (sub->storeDir == storeDir)
assert(subPath == path.first);
if (subPath != path.first)
debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(path.first), sub->printStorePath(subPath), sub->getUri());
} else if (sub->storeDir != storeDir) continue;
debug("checking substituter '%s' for path '%s'", sub->getUri(), sub->printStorePath(subPath));
try {
auto info = sub->queryPathInfo(path);
auto info = sub->queryPathInfo(subPath);
if (sub->storeDir != storeDir && !(info->isContentAddressed(*sub) && info->references.empty()))
continue;
auto narInfo = std::dynamic_pointer_cast<const NarInfo>(
std::shared_ptr<const ValidPathInfo>(info));
infos.insert_or_assign(path, SubstitutablePathInfo{
infos.insert_or_assign(path.first, SubstitutablePathInfo{
info->deriver,
info->references,
narInfo ? narInfo->fileSize : 0,

6
src/libstore/local-store.hh
View file

@ -29,8 +29,8 @@ struct Derivation;
struct OptimiseStats
{
unsigned long filesLinked = 0;
unsigned long long bytesFreed = 0;
unsigned long long blocksFreed = 0;
uint64_t bytesFreed = 0;
uint64_t blocksFreed = 0;
};
@ -139,7 +139,7 @@ public:
StorePathSet querySubstitutablePaths(const StorePathSet & paths) override;
void querySubstitutablePathInfos(const StorePathSet & paths,
void querySubstitutablePathInfos(const StorePathCAMap & paths,
SubstitutablePathInfos & infos) override;
void addToStore(const ValidPathInfo & info, Source & source,

20
src/libstore/misc.cc
View file

@ -108,9 +108,19 @@ void Store::computeFSClosure(const StorePath & startPath,
}
std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv)
{
auto out = drv.outputs.find("out");
if (out != drv.outputs.end()) {
if (auto v = std::get_if<DerivationOutputFixed>(&out->second.output))
return v->hash;
}
return std::nullopt;
}
void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
StorePathSet & willBuild_, StorePathSet & willSubstitute_, StorePathSet & unknown_,
unsigned long long & downloadSize_, unsigned long long & narSize_)
uint64_t & downloadSize_, uint64_t & narSize_)
{
Activity act(*logger, lvlDebug, actUnknown, "querying info about missing paths");
@ -122,8 +132,8 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
{
std::unordered_set<std::string> done;
StorePathSet & unknown, & willSubstitute, & willBuild;
unsigned long long & downloadSize;
unsigned long long & narSize;
uint64_t & downloadSize;
uint64_t & narSize;
};
struct DrvState
@ -157,7 +167,7 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
auto outPath = parseStorePath(outPathS);
SubstitutablePathInfos infos;
querySubstitutablePathInfos({outPath}, infos);
querySubstitutablePathInfos({{outPath, getDerivationCA(*drv)}}, infos);
if (infos.empty()) {
drvState_->lock()->done = true;
@ -214,7 +224,7 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
if (isValidPath(path.path)) return;
SubstitutablePathInfos infos;
querySubstitutablePathInfos({path.path}, infos);
querySubstitutablePathInfos({{path.path, std::nullopt}}, infos);
if (infos.empty()) {
auto state(state_.lock());

4
src/libstore/nar-accessor.cc
View file

@ -79,14 +79,14 @@ struct NarAccessor : public FSAccessor
parents.top()->isExecutable = true;
}
void preallocateContents(unsigned long long size) override
void preallocateContents(uint64_t size) override
{
assert(size <= std::numeric_limits<uint64_t>::max());
parents.top()->size = (uint64_t) size;
parents.top()->start = pos;
}
void receiveContents(unsigned char * data, unsigned int len) override
void receiveContents(unsigned char * data, size_t len) override
{ }
void createSymlink(const Path & path, const string & target) override

2
src/libstore/optimise-store.cc
View file

@ -282,7 +282,7 @@ void LocalStore::optimiseStore(OptimiseStats & stats)
}
}
static string showBytes(unsigned long long bytes)
static string showBytes(uint64_t bytes)
{
return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str();
}

2
src/libstore/path.hh
View file

@ -64,6 +64,8 @@ typedef std::set<StorePath> StorePathSet;
typedef std::vector<StorePath> StorePaths;
typedef std::map<string, StorePath> OutputPathMap;
typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap;
/* Extension of derivations in the Nix store. */
const std::string drvExtension = ".drv";

120
src/libstore/remote-store.cc
View file

@ -39,6 +39,24 @@ void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths
out << store.printStorePath(i);
}
StorePathCAMap readStorePathCAMap(const Store & store, Source & from)
{
StorePathCAMap paths;
auto count = readNum<size_t>(from);
while (count--)
paths.insert_or_assign(store.parseStorePath(readString(from)), parseContentAddressOpt(readString(from)));
return paths;
}
void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap & paths)
{
out << paths.size();
for (auto & i : paths) {
out << store.printStorePath(i.first);
out << renderContentAddress(i.second);
}
}
std::map<string, StorePath> readOutputPathMap(const Store & store, Source & from)
{
std::map<string, StorePath> pathMap;
@ -332,18 +350,17 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths)
}
void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths,
SubstitutablePathInfos & infos)
void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, SubstitutablePathInfos & infos)
{
if (paths.empty()) return;
if (pathsMap.empty()) return;
auto conn(getConnection());
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) {
for (auto & i : paths) {
for (auto & i : pathsMap) {
SubstitutablePathInfo info;
conn->to << wopQuerySubstitutablePathInfo << printStorePath(i);
conn->to << wopQuerySubstitutablePathInfo << printStorePath(i.first);
conn.processStderr();
unsigned int reply = readInt(conn->from);
if (reply == 0) continue;
@ -353,13 +370,19 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths,
info.references = readStorePaths<StorePathSet>(*this, conn->from);
info.downloadSize = readLongLong(conn->from);
info.narSize = readLongLong(conn->from);
infos.insert_or_assign(i, std::move(info));
infos.insert_or_assign(i.first, std::move(info));
}
} else {
conn->to << wopQuerySubstitutablePathInfos;
writeStorePaths(*this, conn->to, paths);
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 22) {
StorePathSet paths;
for (auto & path : pathsMap)
paths.insert(path.first);
writeStorePaths(*this, conn->to, paths);
} else
writeStorePathCAMap(*this, conn->to, pathsMap);
conn.processStderr();
size_t count = readNum<size_t>(conn->from);
for (size_t n = 0; n < count; n++) {
@ -503,9 +526,84 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
conn->to << info.registrationTime << info.narSize
<< info.ultimate << info.sigs << renderContentAddress(info.ca)
<< repair << !checkSigs;
bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21;
if (!tunnel) copyNAR(source, conn->to);
conn.processStderr(0, tunnel ? &source : nullptr);
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 23) {
std::exception_ptr ex;
struct FramedSink : BufferedSink
{
ConnectionHandle & conn;
std::exception_ptr & ex;
FramedSink(ConnectionHandle & conn, std::exception_ptr & ex) : conn(conn), ex(ex)
{ }
~FramedSink()
{
try {
conn->to << 0;
conn->to.flush();
} catch (...) {
ignoreException();
}
}
void write(const unsigned char * data, size_t len) override
{
/* Don't send more data if the remote has
encountered an error. */
if (ex) {
auto ex2 = ex;
ex = nullptr;
std::rethrow_exception(ex2);
}
conn->to << len;
conn->to(data, len);
};
};
/* Handle log messages / exceptions from the remote on a
separate thread. */
std::thread stderrThread([&]()
{
try {
conn.processStderr();
} catch (...) {
ex = std::current_exception();
}
});
Finally joinStderrThread([&]()
{
if (stderrThread.joinable()) {
stderrThread.join();
if (ex) {
try {
std::rethrow_exception(ex);
} catch (...) {
ignoreException();
}
}
}
});
{
FramedSink sink(conn, ex);
copyNAR(source, sink);
sink.flush();
}
stderrThread.join();
if (ex)
std::rethrow_exception(ex);
} else if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21) {
conn.processStderr(0, &source);
} else {
copyNAR(source, conn->to);
conn.processStderr(0, nullptr);
}
}
}
@ -707,7 +805,7 @@ void RemoteStore::addSignatures(const StorePath & storePath, const StringSet & s
void RemoteStore::queryMissing(const std::vector<StorePathWithOutputs> & targets,
StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
unsigned long long & downloadSize, unsigned long long & narSize)
uint64_t & downloadSize, uint64_t & narSize)
{
{
auto conn(getConnection());

4
src/libstore/remote-store.hh
View file

@ -56,7 +56,7 @@ public:
StorePathSet querySubstitutablePaths(const StorePathSet & paths) override;
void querySubstitutablePathInfos(const StorePathSet & paths,
void querySubstitutablePathInfos(const StorePathCAMap & paths,
SubstitutablePathInfos & infos) override;
void addToStore(const ValidPathInfo & info, Source & nar,
@ -94,7 +94,7 @@ public:
void queryMissing(const std::vector<StorePathWithOutputs> & targets,
StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
unsigned long long & downloadSize, unsigned long long & narSize) override;
uint64_t & downloadSize, uint64_t & narSize) override;
void connect() override;

7
src/libstore/s3-binary-cache-store.cc
View file

@ -343,13 +343,10 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1)
.count();
auto size = istream->tellg();
printInfo("uploaded 's3://%s/%s' (%d bytes) in %d ms",
bucketName, path, size, duration);
printInfo("uploaded 's3://%s/%s' in %d ms",
bucketName, path, duration);
stats.putTimeMs += duration;
stats.putBytes += size;
stats.put++;
}

1
src/libstore/s3-binary-cache-store.hh
View file

@ -19,7 +19,6 @@ public:
struct Stats
{
std::atomic<uint64_t> put{0};
std::atomic<uint64_t> putBytes{0};
std::atomic<uint64_t> putTimeMs{0};
std::atomic<uint64_t> get{0};
std::atomic<uint64_t> getBytes{0};

87
src/libstore/store-api.cc
View file

@ -193,6 +193,23 @@ StorePath Store::makeFixedOutputPath(
}
}
// FIXME Put this somewhere?
template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
StorePath Store::makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca,
const StorePathSet & references, bool hasSelfReference) const
{
// New template
return std::visit(overloaded {
[&](TextHash th) {
return makeTextPath(name, th.hash, references);
},
[&](FixedOutputHash fsh) {
return makeFixedOutputPath(fsh.method, fsh.hash, name, references, hasSelfReference);
}
}, ca);
}
StorePath Store::makeTextPath(std::string_view name, const Hash & hash,
const StorePathSet & references) const
@ -689,6 +706,15 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
uint64_t total = 0;
// recompute store path on the chance dstStore does it differently
if (info->ca && info->references.empty()) {
auto info2 = make_ref<ValidPathInfo>(*info);
info2->path = dstStore->makeFixedOutputPathFromCA(info->path.name(), *info->ca);
if (dstStore->storeDir == srcStore->storeDir)
assert(info->path == info2->path);
info = info2;
}
if (!info->narHash) {
StringSink sink;
srcStore->narFromPath({storePath}, sink);
@ -724,16 +750,20 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
}
void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & storePaths,
std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & storePaths,
RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute)
{
auto valid = dstStore->queryValidPaths(storePaths, substitute);
PathSet missing;
StorePathSet missing;
for (auto & path : storePaths)
if (!valid.count(path)) missing.insert(srcStore->printStorePath(path));
if (!valid.count(path)) missing.insert(path);
if (missing.empty()) return;
std::map<StorePath, StorePath> pathsMap;
for (auto & path : storePaths)
pathsMap.insert_or_assign(path, path);
if (missing.empty()) return pathsMap;
Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size()));
@ -748,30 +778,49 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & st
ThreadPool pool;
processGraph<Path>(pool,
PathSet(missing.begin(), missing.end()),
processGraph<StorePath>(pool,
StorePathSet(missing.begin(), missing.end()),
[&](const Path & storePath) {
if (dstStore->isValidPath(dstStore->parseStorePath(storePath))) {
[&](const StorePath & storePath) {
auto info = srcStore->queryPathInfo(storePath);
auto storePathForDst = storePath;
if (info->ca && info->references.empty()) {
storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca);
if (dstStore->storeDir == srcStore->storeDir)
assert(storePathForDst == storePath);
if (storePathForDst != storePath)
debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri());
}
pathsMap.insert_or_assign(storePath, storePathForDst);
if (dstStore->isValidPath(storePath)) {
nrDone++;
showProgress();
return PathSet();
return StorePathSet();
}
auto info = srcStore->queryPathInfo(srcStore->parseStorePath(storePath));
bytesExpected += info->narSize;
act.setExpected(actCopyPath, bytesExpected);
return srcStore->printStorePathSet(info->references);
return info->references;
},
[&](const Path & storePathS) {
[&](const StorePath & storePath) {
checkInterrupt();
auto storePath = dstStore->parseStorePath(storePathS);
auto info = srcStore->queryPathInfo(storePath);
if (!dstStore->isValidPath(storePath)) {
auto storePathForDst = storePath;
if (info->ca && info->references.empty()) {
storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca);
if (dstStore->storeDir == srcStore->storeDir)
assert(storePathForDst == storePath);
if (storePathForDst != storePath)
debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri());
}
pathsMap.insert_or_assign(storePath, storePathForDst);
if (!dstStore->isValidPath(storePathForDst)) {
MaintainCount<decltype(nrRunning)> mc(nrRunning);
showProgress();
try {
@ -780,7 +829,7 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & st
nrFailed++;
if (!settings.keepGoing)
throw e;
logger->log(lvlError, fmt("could not copy %s: %s", storePathS, e.what()));
logger->log(lvlError, fmt("could not copy %s: %s", dstStore->printStorePath(storePath), e.what()));
showProgress();
return;
}
@ -789,6 +838,8 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & st
nrDone++;
showProgress();
});
return pathsMap;
}
@ -865,10 +916,6 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey)
sigs.insert(secretKey.signDetached(fingerprint(store)));
}
// FIXME Put this somewhere?
template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
bool ValidPathInfo::isContentAddressed(const Store & store) const
{
if (! ca) return false;

37
src/libstore/store-api.hh
View file

@ -85,7 +85,7 @@ struct GCOptions
StorePathSet pathsToDelete;
/* Stop after at least `maxFreed' bytes have been freed. */
unsigned long long maxFreed{std::numeric_limits<unsigned long long>::max()};
uint64_t maxFreed{std::numeric_limits<uint64_t>::max()};
};
@ -97,7 +97,7 @@ struct GCResults
/* For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the
number of bytes that would be or was freed. */
unsigned long long bytesFreed = 0;
uint64_t bytesFreed = 0;
};
@ -105,8 +105,8 @@ struct SubstitutablePathInfo
{
std::optional<StorePath> deriver;
StorePathSet references;
unsigned long long downloadSize; /* 0 = unknown or inapplicable */
unsigned long long narSize; /* 0 = unknown */
uint64_t downloadSize; /* 0 = unknown or inapplicable */
uint64_t narSize; /* 0 = unknown */
};
typedef std::map<StorePath, SubstitutablePathInfo> SubstitutablePathInfos;
@ -344,7 +344,11 @@ public:
bool hasSelfReference = false) const;
StorePath makeTextPath(std::string_view name, const Hash & hash,
const StorePathSet & references) const;
const StorePathSet & references = {}) const;
StorePath makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca,
const StorePathSet & references = {},
bool hasSelfReference = false) const;
/* This is the preparatory part of addToStore(); it computes the
store path to which srcPath is to be copied. Returns the store
@ -436,9 +440,10 @@ public:
virtual StorePathSet querySubstitutablePaths(const StorePathSet & paths) { return {}; };
/* Query substitute info (i.e. references, derivers and download
sizes) of a set of paths. If a path does not have substitute
info, it's omitted from the resulting ‘infos’ map. */
virtual void querySubstitutablePathInfos(const StorePathSet & paths,
sizes) of a map of paths to their optional ca values. If a path
does not have substitute info, it's omitted from the resulting
‘infos’ map. */
virtual void querySubstitutablePathInfos(const StorePathCAMap & paths,
SubstitutablePathInfos & infos) { return; };
/* Import a path into the store. */
@ -610,7 +615,7 @@ public:
that will be substituted. */
virtual void queryMissing(const std::vector<StorePathWithOutputs> & targets,
StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
unsigned long long & downloadSize, unsigned long long & narSize);
uint64_t & downloadSize, uint64_t & narSize);
/* Sort a set of paths topologically under the references
relation. If p refers to q, then p precedes q in this list. */
@ -740,11 +745,13 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
/* Copy store paths from one store to another. The paths may be copied
in parallel. They are copied in a topologically sorted order
(i.e. if A is a reference of B, then A is copied before B), but
the set of store paths is not automatically closed; use
copyClosure() for that. */
void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & storePaths,
in parallel. They are copied in a topologically sorted order (i.e.
if A is a reference of B, then A is copied before B), but the set
of store paths is not automatically closed; use copyClosure() for
that. Returns a map of what each path was copied to the dstStore
as. */
std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore,
const StorePathSet & storePaths,
RepairFlag repair = NoRepair,
CheckSigsFlag checkSigs = CheckSigs,
SubstituteFlag substitute = NoSubstitute);
@ -843,4 +850,6 @@ std::optional<ValidPathInfo> decodeValidPathInfo(
/* Split URI into protocol+hierarchy part and its parameter set. */
std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri);
std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv);
}

6
src/libstore/worker-protocol.hh
View file

@ -6,7 +6,7 @@ namespace nix {
#define WORKER_MAGIC_1 0x6e697863
#define WORKER_MAGIC_2 0x6478696f
#define PROTOCOL_VERSION 0x116
#define PROTOCOL_VERSION 0x117
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
@ -70,6 +70,10 @@ template<class T> T readStorePaths(const Store & store, Source & from);
void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths);
StorePathCAMap readStorePathCAMap(const Store & store, Source & from);
void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap & paths);
void writeOutputPathMap(const Store & store, Sink & out, const OutputPathMap & paths);
}

10
src/libutil/archive.cc
View file

@ -150,17 +150,17 @@ static void skipGeneric(Source & source)
static void parseContents(ParseSink & sink, Source & source, const Path & path)
{
unsigned long long size = readLongLong(source);
uint64_t size = readLongLong(source);
sink.preallocateContents(size);
unsigned long long left = size;
uint64_t left = size;
std::vector<unsigned char> buf(65536);
while (left) {
checkInterrupt();
auto n = buf.size();
if ((unsigned long long)n > left) n = left;
if ((uint64_t)n > left) n = left;
source(buf.data(), n);
sink.receiveContents(buf.data(), n);
left -= n;
@ -323,7 +323,7 @@ struct RestoreSink : ParseSink
throw SysError("fchmod");
}
void preallocateContents(unsigned long long len)
void preallocateContents(uint64_t len)
{
#if HAVE_POSIX_FALLOCATE
if (len) {
@ -338,7 +338,7 @@ struct RestoreSink : ParseSink
#endif
}
void receiveContents(unsigned char * data, unsigned int len)
void receiveContents(unsigned char * data, size_t len)
{
writeFull(fd.get(), data, len);
}

6
src/libutil/archive.hh
View file

@ -57,8 +57,8 @@ struct ParseSink
virtual void createRegularFile(const Path & path) { };
virtual void isExecutable() { };
virtual void preallocateContents(unsigned long long size) { };
virtual void receiveContents(unsigned char * data, unsigned int len) { };
virtual void preallocateContents(uint64_t size) { };
virtual void receiveContents(unsigned char * data, size_t len) { };
virtual void createSymlink(const Path & path, const string & target) { };
};
@ -77,7 +77,7 @@ struct RetrieveRegularNARSink : ParseSink
regular = false;
}
void receiveContents(unsigned char * data, unsigned int len)
void receiveContents(unsigned char * data, size_t len)
{
sink(data, len);
}

4
src/libutil/hash.hh
View file

@ -116,7 +116,7 @@ Hash hashFile(HashType ht, const Path & path);
/* Compute the hash of the given path. The hash is defined as
(essentially) hashString(ht, dumpPath(path)). */
typedef std::pair<Hash, unsigned long long> HashResult;
typedef std::pair<Hash, uint64_t> HashResult;
HashResult hashPath(HashType ht, const Path & path,
PathFilter & filter = defaultPathFilter);
@ -146,7 +146,7 @@ class HashSink : public BufferedSink, public AbstractHashSink
private:
HashType ht;
Ctx * ctx;
unsigned long long bytes;
uint64_t bytes;
public:
HashSink(HashType ht);

16
src/libutil/serialise.hh
View file

@ -312,14 +312,14 @@ T readNum(Source & source)
source(buf, sizeof(buf));
uint64_t n =
((unsigned long long) buf[0]) |
((unsigned long long) buf[1] << 8) |
((unsigned long long) buf[2] << 16) |
((unsigned long long) buf[3] << 24) |
((unsigned long long) buf[4] << 32) |
((unsigned long long) buf[5] << 40) |
((unsigned long long) buf[6] << 48) |
((unsigned long long) buf[7] << 56);
((uint64_t) buf[0]) |
((uint64_t) buf[1] << 8) |
((uint64_t) buf[2] << 16) |
((uint64_t) buf[3] << 24) |
((uint64_t) buf[4] << 32) |
((uint64_t) buf[5] << 40) |
((uint64_t) buf[6] << 48) |
((uint64_t) buf[7] << 56);
if (n > std::numeric_limits<T>::max())
throw SerialisationError("serialised integer %d is too large for type '%s'", n, typeid(T).name());

8
src/libutil/util.cc
View file

@ -374,7 +374,7 @@ void writeLine(int fd, string s)
}
static void _deletePath(int parentfd, const Path & path, unsigned long long & bytesFreed)
static void _deletePath(int parentfd, const Path & path, uint64_t & bytesFreed)
{
checkInterrupt();
@ -414,7 +414,7 @@ static void _deletePath(int parentfd, const Path & path, unsigned long long & by
}
}
static void _deletePath(const Path & path, unsigned long long & bytesFreed)
static void _deletePath(const Path & path, uint64_t & bytesFreed)
{
Path dir = dirOf(path);
if (dir == "")
@ -435,12 +435,12 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed)
void deletePath(const Path & path)
{
unsigned long long dummy;
uint64_t dummy;
deletePath(path, dummy);
}
void deletePath(const Path & path, unsigned long long & bytesFreed)
void deletePath(const Path & path, uint64_t & bytesFreed)
{
//Activity act(*logger, lvlDebug, format("recursively deleting path '%1%'") % path);
bytesFreed = 0;

2
src/libutil/util.hh
View file

@ -125,7 +125,7 @@ void writeLine(int fd, string s);
second variant returns the number of bytes and blocks freed. */
void deletePath(const Path & path);
void deletePath(const Path & path, unsigned long long & bytesFreed);
void deletePath(const Path & path, uint64_t & bytesFreed);
std::string getUserName();

6
src/nix-build/nix-build.cc
View file

@ -174,7 +174,7 @@ static void _main(int argc, char * * argv)
else if (*arg == "--run-env") // obsolete
runEnv = true;
else if (*arg == "--command" || *arg == "--run") {
else if (runEnv && (*arg == "--command" || *arg == "--run")) {
if (*arg == "--run")
interactive = false;
envCommand = getArg(*arg, arg, end) + "\nexit";
@ -192,7 +192,7 @@ static void _main(int argc, char * * argv)
else if (*arg == "--pure") pure = true;
else if (*arg == "--impure") pure = false;
else if (*arg == "--packages" || *arg == "-p")
else if (runEnv && (*arg == "--packages" || *arg == "-p"))
packages = true;
else if (inShebang && *arg == "-i") {
@ -325,7 +325,7 @@ static void _main(int argc, char * * argv)
auto buildPaths = [&](const std::vector<StorePathWithOutputs> & paths) {
/* Note: we do this even when !printMissing to efficiently
fetch binary cache data. */
unsigned long long downloadSize, narSize;
uint64_t downloadSize, narSize;
StorePathSet willBuild, willSubstitute, unknown;
store->queryMissing(paths,
willBuild, willSubstitute, unknown, downloadSize, narSize);

6
src/nix-collect-garbage/nix-collect-garbage.cc
View file

@ -67,10 +67,8 @@ static int _main(int argc, char * * argv)
deleteOlderThan = getArg(*arg, arg, end);
}
else if (*arg == "--dry-run") dryRun = true;
else if (*arg == "--max-freed") {
long long maxFreed = getIntArg<long long>(*arg, arg, end, true);
options.maxFreed = maxFreed >= 0 ? maxFreed : 0;
}
else if (*arg == "--max-freed")
options.maxFreed = std::max(getIntArg<int64_t>(*arg, arg, end, true), (int64_t) 0);
else
return false;
return true;

10
src/nix-store/nix-store.cc
View file

@ -130,7 +130,7 @@ static void opRealise(Strings opFlags, Strings opArgs)
for (auto & i : opArgs)
paths.push_back(store->followLinksToStorePathWithOutputs(i));
unsigned long long downloadSize, narSize;
uint64_t downloadSize, narSize;
StorePathSet willBuild, willSubstitute, unknown;
store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize);
@ -572,10 +572,8 @@ static void opGC(Strings opFlags, Strings opArgs)
if (*i == "--print-roots") printRoots = true;
else if (*i == "--print-live") options.action = GCOptions::gcReturnLive;
else if (*i == "--print-dead") options.action = GCOptions::gcReturnDead;
else if (*i == "--max-freed") {
long long maxFreed = getIntArg<long long>(*i, i, opFlags.end(), true);
options.maxFreed = maxFreed >= 0 ? maxFreed : 0;
}
else if (*i == "--max-freed")
options.maxFreed = std::max(getIntArg<int64_t>(*i, i, opFlags.end(), true), (int64_t) 0);
else throw UsageError("bad sub-operation '%1%' in GC", *i);
if (!opArgs.empty()) throw UsageError("no arguments expected");
@ -831,7 +829,7 @@ static void opServe(Strings opFlags, Strings opArgs)
for (auto & path : paths)
if (!path.isDerivation())
paths2.push_back({path});
unsigned long long downloadSize, narSize;
uint64_t downloadSize, narSize;
StorePathSet willBuild, willSubstitute, unknown;
store->queryMissing(paths2,
willBuild, willSubstitute, unknown, downloadSize, narSize);

21
src/nix/add-to-store.cc
View file

@ -9,6 +9,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand
{
Path path;
std::optional<std::string> namePart;
FileIngestionMethod ingestionMethod = FileIngestionMethod::Recursive;
CmdAddToStore()
{
@ -21,6 +22,13 @@ struct CmdAddToStore : MixDryRun, StoreCommand
.labels = {"name"},
.handler = {&namePart},
});
addFlag({
.longName = "flat",
.shortName = 0,
.description = "add flat file to the Nix store",
.handler = {&ingestionMethod, FileIngestionMethod::Flat},
});
}
std::string description() override
@ -45,12 +53,19 @@ struct CmdAddToStore : MixDryRun, StoreCommand
auto narHash = hashString(htSHA256, *sink.s);
ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, *namePart));
Hash hash = narHash;
if (ingestionMethod == FileIngestionMethod::Flat) {
HashSink hsink(htSHA256);
readFile(path, hsink);
hash = hsink.finish().first;
}
ValidPathInfo info(store->makeFixedOutputPath(ingestionMethod, hash, *namePart));
info.narHash = narHash;
info.narSize = sink.s->size();
info.ca = std::optional { FixedOutputHash {
.method = FileIngestionMethod::Recursive,
.hash = *info.narHash,
.method = ingestionMethod,
.hash = hash,
} };
if (!dryRun) {

8
src/nix/develop.cc
View file

@ -68,22 +68,22 @@ BuildEnvironment readEnvironment(const Path & path)
std::smatch match;
if (std::regex_search(pos, file.cend(), match, declareRegex)) {
if (std::regex_search(pos, file.cend(), match, declareRegex, std::regex_constants::match_continuous)) {
pos = match[0].second;
exported.insert(match[1]);
}
else if (std::regex_search(pos, file.cend(), match, varRegex)) {
else if (std::regex_search(pos, file.cend(), match, varRegex, std::regex_constants::match_continuous)) {
pos = match[0].second;
res.env.insert({match[1], Var { .exported = exported.count(match[1]) > 0, .value = match[2] }});
}
else if (std::regex_search(pos, file.cend(), match, assocArrayRegex)) {
else if (std::regex_search(pos, file.cend(), match, assocArrayRegex, std::regex_constants::match_continuous)) {
pos = match[0].second;
res.env.insert({match[1], Var { .associative = true, .value = match[2] }});
}
else if (std::regex_search(pos, file.cend(), match, functionRegex)) {
else if (std::regex_search(pos, file.cend(), match, functionRegex, std::regex_constants::match_continuous)) {
res.bashFunctions = std::string(pos, file.cend());
break;
}

2
src/nix/path-info.cc
View file

@ -61,7 +61,7 @@ struct CmdPathInfo : StorePathsCommand, MixJSON
};
}
void printSize(unsigned long long value)
void printSize(uint64_t value)
{
if (!humanReadable) {
std::cout << fmt("\t%11d", value);

14
tests/check.sh
View file

@ -61,30 +61,30 @@ nix-build check.nix -A nondeterministic --no-out-link --repeat 1 2> $TEST_ROOT/l
[ "$status" = "1" ]
grep 'differs from previous round' $TEST_ROOT/log
path=$(nix-build check.nix -A fetchurl --no-out-link --hashed-mirrors '')
path=$(nix-build check.nix -A fetchurl --no-out-link)
chmod +w $path
echo foo > $path
chmod -w $path
nix-build check.nix -A fetchurl --no-out-link --check --hashed-mirrors ''
nix-build check.nix -A fetchurl --no-out-link --check
# Note: "check" doesn't repair anything, it just compares to the hash stored in the database.
[[ $(cat $path) = foo ]]
nix-build check.nix -A fetchurl --no-out-link --repair --hashed-mirrors ''
nix-build check.nix -A fetchurl --no-out-link --repair
[[ $(cat $path) != foo ]]
nix-build check.nix -A hashmismatch --no-out-link --hashed-mirrors '' || status=$?
nix-build check.nix -A hashmismatch --no-out-link || status=$?
[ "$status" = "102" ]
echo -n > ./dummy
nix-build check.nix -A hashmismatch --no-out-link --hashed-mirrors ''
nix-build check.nix -A hashmismatch --no-out-link
echo 'Hello World' > ./dummy
nix-build check.nix -A hashmismatch --no-out-link --check --hashed-mirrors '' || status=$?
nix-build check.nix -A hashmismatch --no-out-link --check || status=$?
[ "$status" = "102" ]
# Multiple failures with --keep-going
nix-build check.nix -A nondeterministic --no-out-link
nix-build check.nix -A nondeterministic -A hashmismatch --no-out-link --check --keep-going --hashed-mirrors '' || status=$?
nix-build check.nix -A nondeterministic -A hashmismatch --no-out-link --check --keep-going || status=$?
[ "$status" = "110" ]

8
tests/fetchGit.sh
View file

@ -32,6 +32,8 @@ rev2=$(git -C $repo rev-parse HEAD)
# Fetch a worktree
unset _NIX_FORCE_HTTP
path0=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$TEST_ROOT/worktree).outPath")
path0_=$(nix eval --impure --raw --expr "(builtins.fetchTree { type = \"git\"; url = file://$TEST_ROOT/worktree; }).outPath")
[[ $path0 = $path0_ ]]
export _NIX_FORCE_HTTP=1
[[ $(tail -n 1 $path0/hello) = "hello" ]]
@ -102,6 +104,12 @@ git -C $repo commit -m 'Bla3' -a
path4=$(nix eval --impure --refresh --raw --expr "(builtins.fetchGit file://$repo).outPath")
[[ $path2 = $path4 ]]
nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; rev = \"$rev2\"; narHash = \"sha256-B5yIPHhEm0eysJKEsO7nqxprh9vcblFxpJG11gXJus1=\"; }).outPath" || status=$?
[[ "$status" = "102" ]]
path5=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; rev = \"$rev2\"; narHash = \"sha256-Hr8g6AqANb3xqX28eu1XnjK/3ab8Gv6TJSnkb1LezG9=\"; }).outPath")
[[ $path = $path5 ]]
# tarball-ttl should be ignored if we specify a rev
echo delft > $repo/hello
git -C $repo add hello

24
tests/fetchurl.sh
View file

@ -5,7 +5,7 @@ clearStore
# Test fetching a flat file.
hash=$(nix-hash --flat --type sha256 ./fetchurl.sh)
outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr sha256 $hash --no-out-link --hashed-mirrors '')
outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr sha256 $hash --no-out-link)
cmp $outPath fetchurl.sh
@ -14,7 +14,7 @@ clearStore
hash=$(nix hash-file --type sha512 --base64 ./fetchurl.sh)
outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr sha512 $hash --no-out-link --hashed-mirrors '')
outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr sha512 $hash --no-out-link)
cmp $outPath fetchurl.sh
@ -25,26 +25,24 @@ hash=$(nix hash-file ./fetchurl.sh)
[[ $hash =~ ^sha256- ]]
outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr hash $hash --no-out-link --hashed-mirrors '')
outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr hash $hash --no-out-link)
cmp $outPath fetchurl.sh
# Test the hashed mirror feature.
# Test that we can substitute from a different store dir.
clearStore
hash=$(nix hash-file --type sha512 --base64 ./fetchurl.sh)
hash32=$(nix hash-file --type sha512 --base16 ./fetchurl.sh)
other_store=file://$TEST_ROOT/other_store?store=/fnord/store
mirror=$TEST_ROOT/hashed-mirror
rm -rf $mirror
mkdir -p $mirror/sha512
ln -s $(pwd)/fetchurl.sh $mirror/sha512/$hash32
hash=$(nix hash-file --type sha256 --base16 ./fetchurl.sh)
outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha512 $hash --no-out-link --hashed-mirrors "file://$mirror")
storePath=$(nix --store $other_store add-to-store --flat ./fetchurl.sh)
outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 $hash --no-out-link --substituters $other_store)
# Test hashed mirrors with an SRI hash.
nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix to-sri --type sha512 $hash) \
--argstr name bla --no-out-link --hashed-mirrors "file://$mirror"
nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix to-sri --type sha256 $hash) \
--no-out-link --substituters $other_store
# Test unpacking a NAR.
rm -rf $TEST_ROOT/archive

11
tests/tarball.sh
View file

@ -27,10 +27,13 @@ test_tarball() {
nix-build -o $TEST_ROOT/result -E "import (fetchTarball file://$tarball)"
nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree file://$tarball)"
nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; })"
nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })"
nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"sha256-xdKv2pq/IiwLSnBBJXW8hNowI4MrdZfW+SYqDQs7Tzc=\"; })" 2>&1 | grep 'NAR hash mismatch in input'
nix-build -o $TEST_ROOT/result -E "import (fetchTree file://$tarball)"
nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; })"
nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })"
nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"sha256-xdKv2pq/IiwLSnBBJXW8hNowI4MrdZfW+SYqDQs7Tzc=\"; })" 2>&1 | grep 'NAR hash mismatch in input'
nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" >&2
nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" 2>&1 | grep 'true'
nix-instantiate --eval -E '1 + 2' -I fnord=file://no-such-tarball.tar$ext
nix-instantiate --eval -E 'with <fnord/xyzzy>; 1 + 2' -I fnord=file://no-such-tarball$ext