Split {,local-}derivation-goal.{cc,hh}

This separates the scheduling logic (including simple hook pathway) from
the local-store needing code.

This should be the final split for now. I'm reasonably happy with how
it's turning out, even before I'm done moving code into
`local-derivation-goal`. Benefits:

1. This will help "witness" that the hook case is indeed a lot simpler,
   and also compensate for the increased complexity that comes from
   content-addressed derivation outputs.

2. It also moves us ever so slightly towards a world where we could use
   off-the-shelf storage or sandboxing, since `local-derivation-goal`
   would be gutted in those cases, but `derivation-goal` should remain
   nearly the same.

The new `#if 0` in the new files will be deleted in the following
commit. I keep it here so if it turns out more stuff can be moved over,
it's easy to do so in a way that preserves ordering --- and thus
prevents conflicts.

N.B.
```sh
git diff HEAD^^ --color-moved --find-copies-harder --patience --stat
```
makes nicer output.
This commit is contained in:
John Ericson 2021-02-26 15:20:33 +00:00
parent 05cc5a8587
commit 68f4c728ec
7 changed files with 329 additions and 3045 deletions

File diff suppressed because it is too large Load diff

View file

@ -2,7 +2,8 @@
#include "parsed-derivations.hh"
#include "lock.hh"
#include "local-store.hh"
#include "store-api.hh"
#include "pathlocks.hh"
#include "goal.hh"
namespace nix {
@ -79,18 +80,6 @@ struct DerivationGoal : public Goal
std::map<std::string, InitialOutput> initialOutputs;
/* User selected for running the builder. */
std::unique_ptr<UserLock> buildUser;
/* The process ID of the builder. */
Pid pid;
/* The temporary directory. */
Path tmpDir;
/* The path of the temporary directory in the sandbox. */
Path tmpDirInSandbox;
/* File descriptor for the log file. */
AutoCloseFD fdLogFile;
std::shared_ptr<BufferedSink> logFileSink, logSink;
@ -106,79 +95,15 @@ struct DerivationGoal : public Goal
std::string currentHookLine;
/* Pipe for the builder's standard output/error. */
Pipe builderOut;
/* Pipe for synchronising updates to the builder namespaces. */
Pipe userNamespaceSync;
/* The mount namespace of the builder, used to add additional
paths to the sandbox as a result of recursive Nix calls. */
AutoCloseFD sandboxMountNamespace;
/* On Linux, whether we're doing the build in its own user
namespace. */
bool usingUserNamespace = true;
/* The build hook. */
std::unique_ptr<HookInstance> hook;
/* Whether we're currently doing a chroot build. */
bool useChroot = false;
Path chrootRootDir;
/* RAII object to delete the chroot directory. */
std::shared_ptr<AutoDelete> autoDelChroot;
/* The sort of derivation we are building. */
DerivationType derivationType;
/* Whether to run the build in a private network namespace. */
bool privateNetwork = false;
typedef void (DerivationGoal::*GoalState)();
GoalState state;
/* Stuff we need to pass to initChild(). */
struct ChrootPath {
Path source;
bool optional;
ChrootPath(Path source = "", bool optional = false)
: source(source), optional(optional)
{ }
};
typedef map<Path, ChrootPath> DirsInChroot; // maps target path to source path
DirsInChroot dirsInChroot;
typedef map<string, string> Environment;
Environment env;
#if __APPLE__
typedef string SandboxProfile;
SandboxProfile additionalSandboxProfile;
#endif
/* Hash rewriting. */
StringMap inputRewrites, outputRewrites;
typedef map<StorePath, StorePath> RedirectedOutputs;
RedirectedOutputs redirectedOutputs;
/* The outputs paths used during the build.
- Input-addressed derivations or fixed content-addressed outputs are
sometimes built when some of their outputs already exist, and can not
be hidden via sandboxing. We use temporary locations instead and
rewrite after the build. Otherwise the regular predetermined paths are
put here.
- Floating content-addressed derivations do not know their final build
output paths until the outputs are hashed, so random locations are
used, and then renamed. The randomness helps guard against hidden
self-references.
*/
OutputPathMap scratchOutputs;
/* The final output paths of the build.
- For input-addressed derivations, always the precomputed paths
@ -191,11 +116,6 @@ struct DerivationGoal : public Goal
BuildMode buildMode;
/* If we're repairing without a chroot, there may be outputs that
are valid but corrupt. So we redirect these outputs to
temporary paths. */
StorePathSet redirectedBadOutputs;
BuildResult result;
/* The current round, if we're building multiple times. */
@ -203,17 +123,6 @@ struct DerivationGoal : public Goal
size_t nrRounds;
/* Path registration info from the previous round, if we're
building multiple times. Since this contains the hash, it
allows us to compare whether two rounds produced the same
result. */
std::map<Path, ValidPathInfo> prevInfos;
uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); }
gid_t sandboxGid() { return usingUserNamespace ? 100 : buildUser->getGID(); }
const static Path homeDir;
std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds, mcRunningBuilds;
std::unique_ptr<Activity> act;
@ -226,39 +135,13 @@ struct DerivationGoal : public Goal
/* The remote machine on which we're building. */
std::string machineName;
/* The recursive Nix daemon socket. */
AutoCloseFD daemonSocket;
/* The daemon main thread. */
std::thread daemonThread;
/* The daemon worker threads. */
std::vector<std::thread> daemonWorkerThreads;
/* Paths that were added via recursive Nix calls. */
StorePathSet addedPaths;
/* Recursive Nix calls are only allowed to build or realize paths
in the original input closure or added via a recursive Nix call
(so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
/nix/store/<bla> is some arbitrary path in a binary cache). */
bool isAllowed(const StorePath & path)
{
return inputPaths.count(path) || addedPaths.count(path);
}
friend struct RestrictedStore;
DerivationGoal(const StorePath & drvPath,
const StringSet & wantedOutputs, Worker & worker,
BuildMode buildMode = bmNormal);
DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
const StringSet & wantedOutputs, Worker & worker,
BuildMode buildMode = bmNormal);
~DerivationGoal();
/* Whether we need to perform hash rewriting if there are valid output paths. */
bool needsHashRewrite();
virtual ~DerivationGoal();
void timedOut(Error && ex) override;
@ -280,7 +163,7 @@ struct DerivationGoal : public Goal
void closureRepaired();
void inputsRealised();
void tryToBuild();
void tryLocalBuild();
virtual void tryLocalBuild();
void buildDone();
void resolvedFinished();
@ -288,40 +171,11 @@ struct DerivationGoal : public Goal
/* Is the build hook willing to perform the build? */
HookReply tryBuildHook();
/* Start building a derivation. */
void startBuilder();
/* Fill in the environment for the builder. */
void initEnv();
/* Setup tmp dir location. */
void initTmpDir();
/* Write a JSON file containing the derivation attributes. */
void writeStructuredAttrs();
void startDaemon();
void stopDaemon();
/* Add 'path' to the set of paths that may be referenced by the
outputs, and make it appear in the sandbox. */
void addDependency(const StorePath & path);
/* Make a file owned by the builder. */
void chownToBuilder(const Path & path);
/* Run the builder's process. */
void runChild();
virtual int getChildStatus();
/* Check that the derivation outputs all exist and register them
as valid. */
void registerOutputs();
/* Check that an output meets the requirements specified by the
'outputChecks' attribute (or the legacy
'{allowed,disallowed}{References,Requisites}' attributes). */
void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs);
virtual void registerOutputs();
/* Open a log file and a pipe to it. */
Path openLogFile();
@ -329,8 +183,18 @@ struct DerivationGoal : public Goal
/* Close the log file. */
void closeLogFile();
/* Delete the temporary directory, if we have one. */
void deleteTmpDir(bool force);
/* Close the read side of the logger pipe. */
virtual void closeReadPipes();
/* Cleanup hooks for buildDone() */
virtual void cleanupHookFinally();
virtual void cleanupPreChildKill();
virtual void cleanupPostChildKill();
virtual bool cleanupDecideWhetherDiskFull();
virtual void cleanupPostOutputsRegisteredModeCheck();
virtual void cleanupPostOutputsRegisteredModeNonCheck();
virtual bool isReadDesc(int fd);
/* Callback used by the worker to write to the log. */
void handleChildOutput(int fd, const string & data) override;
@ -347,17 +211,7 @@ struct DerivationGoal : public Goal
void checkPathValidity();
/* Forcibly kill the child process, if any. */
void killChild();
/* Create alternative path calculated from but distinct from the
input, so we can avoid overwriting outputs (or other store paths)
that already exist. */
StorePath makeFallbackPath(const StorePath & path);
/* Make a path to another based on the output name along with the
derivation hash. */
/* FIXME add option to randomize, so we can audit whether our
rewrites caught everything */
StorePath makeFallbackPath(std::string_view outputName);
virtual void killChild();
void repairClosure();
@ -370,4 +224,6 @@ struct DerivationGoal : public Goal
StorePathSet exportReferences(const StorePathSet & storePaths);
};
MakeError(NotDeterministic, BuildError);
}

View file

@ -2,6 +2,7 @@
#include "worker.hh"
#include "substitution-goal.hh"
#include "derivation-goal.hh"
#include "local-store.hh"
namespace nix {

View file

@ -1,4 +1,4 @@
#include "derivation-goal.hh"
#include "local-derivation-goal.hh"
#include "hook-instance.hh"
#include "worker.hh"
#include "builtins.hh"
@ -75,7 +75,6 @@ void handleDiffHook(
diffHookOptions.uid = uid;
diffHookOptions.gid = gid;
diffHookOptions.chdir = "/";
auto diffRes = runProgram(diffHookOptions);
if (!statusOk(diffRes.first))
throw ExecError(diffRes.first,
@ -94,8 +93,9 @@ void handleDiffHook(
}
}
const Path DerivationGoal::homeDir = "/homeless-shelter";
const Path LocalDerivationGoal::homeDir = "/homeless-shelter";
#if 0
DerivationGoal::DerivationGoal(const StorePath & drvPath,
const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode)
: Goal(worker)
@ -138,19 +138,20 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation
garbage-collected. (See isActiveTempFile() in gc.cc.) */
worker.store.addTempRoot(this->drvPath);
}
#endif
DerivationGoal::~DerivationGoal()
LocalDerivationGoal::~LocalDerivationGoal()
{
/* Careful: we should never ever throw an exception from a
destructor. */
try { deleteTmpDir(false); } catch (...) { ignoreException(); }
try { killChild(); } catch (...) { ignoreException(); }
try { stopDaemon(); } catch (...) { ignoreException(); }
try { deleteTmpDir(false); } catch (...) { ignoreException(); }
try { closeLogFile(); } catch (...) { ignoreException(); }
}
#if 0
string DerivationGoal::key()
{
/* Ensure that derivations get built in order of their name,
@ -159,9 +160,10 @@ string DerivationGoal::key()
derivation goals (due to "b$"). */
return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath);
}
#endif
inline bool DerivationGoal::needsHashRewrite()
inline bool LocalDerivationGoal::needsHashRewrite()
{
#if __linux__
return !useChroot;
@ -172,7 +174,15 @@ inline bool DerivationGoal::needsHashRewrite()
}
void DerivationGoal::killChild()
LocalStore & LocalDerivationGoal::getLocalStore()
{
auto p = dynamic_cast<LocalStore *>(&worker.store);
assert(p);
return *p;
}
void LocalDerivationGoal::killChild()
{
if (pid != -1) {
worker.childTerminated(this);
@ -193,17 +203,11 @@ void DerivationGoal::killChild()
assert(pid == -1);
}
hook.reset();
}
void DerivationGoal::timedOut(Error && ex)
{
killChild();
done(BuildResult::TimedOut, ex);
DerivationGoal::killChild();
}
#if 0
void DerivationGoal::work()
{
(this->*state)();
@ -695,15 +699,9 @@ void DerivationGoal::tryToBuild()
state = &DerivationGoal::tryLocalBuild;
worker.wakeUp(shared_from_this());
}
#endif
void DerivationGoal::tryLocalBuild() {
/* Make sure that we are allowed to start a build. */
if (!dynamic_cast<LocalStore *>(&worker.store)) {
throw Error(
"unable to build with a primary store that isn't a local store; "
"either pass a different '--store' or enable remote builds."
"\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
}
void LocalDerivationGoal::tryLocalBuild() {
unsigned int curBuilds = worker.getNrLocalBuilds();
if (curBuilds >= settings.maxBuildJobs) {
worker.waitForBuildSlot(shared_from_this());
@ -757,7 +755,6 @@ void DerivationGoal::tryLocalBuild() {
started();
}
static void chmod_(const Path & path, mode_t mode)
{
if (chmod(path.c_str(), mode) == -1)
@ -785,70 +782,40 @@ static void movePath(const Path & src, const Path & dst)
}
void replaceValidPath(const Path & storePath, const Path & tmpPath)
extern void replaceValidPath(const Path & storePath, const Path & tmpPath);
int LocalDerivationGoal::getChildStatus()
{
/* We can't atomically replace storePath (the original) with
tmpPath (the replacement), so we have to move it out of the
way first. We'd better not be interrupted here, because if
we're repairing (say) Glibc, we end up with a broken system. */
Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % random()).str();
if (pathExists(storePath))
movePath(storePath, oldPath);
try {
movePath(tmpPath, storePath);
} catch (...) {
try {
// attempt to recover
movePath(oldPath, storePath);
} catch (...) {
ignoreException();
}
throw;
return hook ? DerivationGoal::getChildStatus() : pid.kill();
}
deletePath(oldPath);
}
MakeError(NotDeterministic, BuildError);
void DerivationGoal::buildDone()
void LocalDerivationGoal::closeReadPipes()
{
trace("build done");
if (hook) {
DerivationGoal::closeReadPipes();
} else
builderOut.readSide = -1;
}
void LocalDerivationGoal::cleanupHookFinally()
{
/* Release the build user at the end of this function. We don't do
it right away because we don't want another build grabbing this
uid and then messing around with our output. */
Finally releaseBuildUser([&]() { buildUser.reset(); });
buildUser.reset();
}
void LocalDerivationGoal::cleanupPreChildKill()
{
sandboxMountNamespace = -1;
}
/* Since we got an EOF on the logger pipe, the builder is presumed
to have terminated. In fact, the builder could also have
simply have closed its end of the pipe, so just to be sure,
kill it. */
int status = hook ? hook->pid.kill() : pid.kill();
debug("builder process for '%s' finished", worker.store.printStorePath(drvPath));
result.timesBuilt++;
result.stopTime = time(0);
/* So the child is gone now. */
worker.childTerminated(this);
/* Close the read side of the logger pipe. */
if (hook) {
hook->builderOut.readSide = -1;
hook->fromHook.readSide = -1;
} else
builderOut.readSide = -1;
/* Close the log file. */
closeLogFile();
void LocalDerivationGoal::cleanupPostChildKill()
{
/* When running under a build user, make sure that all processes
running under that uid are gone. This is to prevent a
malicious user from leaving behind a process that keeps files
@ -858,24 +825,24 @@ void DerivationGoal::buildDone()
/* Terminate the recursive Nix daemon. */
stopDaemon();
}
bool LocalDerivationGoal::cleanupDecideWhetherDiskFull()
{
bool diskFull = false;
try {
/* Check the exit status. */
if (!statusOk(status)) {
/* Heuristically check whether the build failure may have
been caused by a disk full condition. We have no way
of knowing whether the build actually got an ENOSPC.
So instead, check if the disk is (nearly) full now. If
so, we don't mark this build as a permanent failure. */
#if HAVE_STATVFS
if (auto localStore = dynamic_cast<LocalStore *>(&worker.store)) {
{
auto & localStore = getLocalStore();
uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable
struct statvfs st;
if (statvfs(localStore->realStoreDir.c_str(), &st) == 0 &&
if (statvfs(localStore.realStoreDir.c_str(), &st) == 0 &&
(uint64_t) st.f_bavail * st.f_bsize < required)
diskFull = true;
if (statvfs(tmpDir.c_str(), &st) == 0 &&
@ -897,6 +864,69 @@ void DerivationGoal::buildDone()
rename((chrootRootDir + p).c_str(), p.c_str());
}
return diskFull;
}
void LocalDerivationGoal::cleanupPostOutputsRegisteredModeCheck()
{
deleteTmpDir(true);
}
void LocalDerivationGoal::cleanupPostOutputsRegisteredModeNonCheck()
{
/* Delete unused redirected outputs (when doing hash rewriting). */
for (auto & i : redirectedOutputs)
deletePath(worker.store.Store::toRealPath(i.second));
/* Delete the chroot (if we were using one). */
autoDelChroot.reset(); /* this runs the destructor */
cleanupPostOutputsRegisteredModeCheck();
}
#if 0
void DerivationGoal::buildDone()
{
trace("build done");
Finally releaseBuildUser([&](){ this->cleanupHookFinally(); });
cleanupPreChildKill();
/* Since we got an EOF on the logger pipe, the builder is presumed
to have terminated. In fact, the builder could also have
simply have closed its end of the pipe, so just to be sure,
kill it. */
int status = getChildStatus();
debug("builder process for '%s' finished", worker.store.printStorePath(drvPath));
result.timesBuilt++;
result.stopTime = time(0);
/* So the child is gone now. */
worker.childTerminated(this);
/* Close the read side of the logger pipe. */
closeReadPipes();
/* Close the log file. */
closeLogFile();
cleanupPostChildKill();
bool diskFull = false;
try {
/* Check the exit status. */
if (!statusOk(status)) {
diskFull |= cleanupDecideWhetherDiskFull();
auto msg = fmt("builder for '%s' %s",
yellowtxt(worker.store.printStorePath(drvPath)),
statusToString(status));
@ -975,19 +1005,12 @@ void DerivationGoal::buildDone()
}
if (buildMode == bmCheck) {
deleteTmpDir(true);
cleanupPostOutputsRegisteredModeCheck();
done(BuildResult::Built);
return;
}
/* Delete unused redirected outputs (when doing hash rewriting). */
for (auto & i : redirectedOutputs)
deletePath(worker.store.Store::toRealPath(i.second));
/* Delete the chroot (if we were using one). */
autoDelChroot.reset(); /* this runs the destructor */
deleteTmpDir(true);
cleanupPostOutputsRegisteredModeNonCheck();
/* Repeat the build if necessary. */
if (curRound++ < nrRounds) {
@ -1169,15 +1192,17 @@ HookReply DerivationGoal::tryBuildHook()
return rpAccept;
}
#endif
int childEntry(void * arg)
{
((DerivationGoal *) arg)->runChild();
((LocalDerivationGoal *) arg)->runChild();
return 1;
}
#if 0
StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths)
{
StorePathSet paths;
@ -1212,6 +1237,7 @@ StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths)
return paths;
}
#endif
static std::once_flag dns_resolve_flag;
@ -1230,7 +1256,7 @@ static void preloadNSS() {
}
void linkOrCopy(const Path & from, const Path & to)
static void linkOrCopy(const Path & from, const Path & to)
{
if (link(from.c_str(), to.c_str()) == -1) {
/* Hard-linking fails if we exceed the maximum link count on a
@ -1247,7 +1273,7 @@ void linkOrCopy(const Path & from, const Path & to)
}
void DerivationGoal::startBuilder()
void LocalDerivationGoal::startBuilder()
{
/* Right platform? */
if (!parsedDrv->canBuildLocally(worker.store))
@ -1285,8 +1311,7 @@ void DerivationGoal::startBuilder()
useChroot = !(derivationIsImpure(derivationType)) && !noChroot;
}
if (auto localStoreP = dynamic_cast<LocalStore *>(&worker.store)) {
auto & localStore = *localStoreP;
auto & localStore = getLocalStore();
if (localStore.storeDir != localStore.realStoreDir) {
#if __linux__
useChroot = true;
@ -1294,7 +1319,6 @@ void DerivationGoal::startBuilder()
throw Error("building using a diverted store is not supported on this platform");
#endif
}
}
/* Create a temporary directory where the build will take
place. */
@ -1850,7 +1874,7 @@ void DerivationGoal::startBuilder()
}
void DerivationGoal::initTmpDir() {
void LocalDerivationGoal::initTmpDir() {
/* In a sandbox, for determinism, always use the same temporary
directory. */
#if __linux__
@ -1899,7 +1923,7 @@ void DerivationGoal::initTmpDir() {
}
void DerivationGoal::initEnv()
void LocalDerivationGoal::initEnv()
{
env.clear();
@ -1960,7 +1984,7 @@ void DerivationGoal::initEnv()
static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*");
void DerivationGoal::writeStructuredAttrs()
void LocalDerivationGoal::writeStructuredAttrs()
{
auto structuredAttrs = parsedDrv->getStructuredAttrs();
if (!structuredAttrs) return;
@ -2079,9 +2103,9 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
{
ref<LocalStore> next;
DerivationGoal & goal;
LocalDerivationGoal & goal;
RestrictedStore(const Params & params, ref<LocalStore> next, DerivationGoal & goal)
RestrictedStore(const Params & params, ref<LocalStore> next, LocalDerivationGoal & goal)
: StoreConfig(params)
, LocalFSStoreConfig(params)
, RestrictedStoreConfig(params)
@ -2256,15 +2280,14 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
};
void DerivationGoal::startDaemon()
void LocalDerivationGoal::startDaemon()
{
settings.requireExperimentalFeature("recursive-nix");
Store::Params params;
params["path-info-cache-size"] = "0";
params["store"] = worker.store.storeDir;
if (auto localStore = dynamic_cast<LocalStore *>(&worker.store))
params["root"] = localStore->rootDir;
params["root"] = getLocalStore().rootDir;
params["state"] = "/no-such-path";
params["log"] = "/no-such-path";
auto store = make_ref<RestrictedStore>(params,
@ -2322,7 +2345,7 @@ void DerivationGoal::startDaemon()
}
void DerivationGoal::stopDaemon()
void LocalDerivationGoal::stopDaemon()
{
if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1)
throw SysError("shutting down daemon socket");
@ -2340,7 +2363,7 @@ void DerivationGoal::stopDaemon()
}
void DerivationGoal::addDependency(const StorePath & path)
void LocalDerivationGoal::addDependency(const StorePath & path)
{
if (isAllowed(path)) return;
@ -2397,8 +2420,7 @@ void DerivationGoal::addDependency(const StorePath & path)
}
}
void DerivationGoal::chownToBuilder(const Path & path)
void LocalDerivationGoal::chownToBuilder(const Path & path)
{
if (!buildUser) return;
if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1)
@ -2469,7 +2491,7 @@ void setupSeccomp()
}
void DerivationGoal::runChild()
void LocalDerivationGoal::runChild()
{
/* Warning: in the child we should absolutely not make any SQLite
calls! */
@ -2977,7 +2999,7 @@ void DerivationGoal::runChild()
}
void DerivationGoal::registerOutputs()
void LocalDerivationGoal::registerOutputs()
{
/* When using a build hook, the build hook can register the output
as valid (by doing `nix-store --import'). If so we don't have
@ -2987,14 +3009,8 @@ void DerivationGoal::registerOutputs()
floating content-addressed derivations this isn't the case.
*/
if (hook) {
bool allValid = true;
for (auto & [outputName, outputPath] : worker.store.queryPartialDerivationOutputMap(drvPath)) {
if (!outputPath || !worker.store.isValidPath(*outputPath))
allValid = false;
else
finalOutputs.insert_or_assign(outputName, *outputPath);
}
if (allValid) return;
DerivationGoal::registerOutputs();
return;
}
std::map<std::string, ValidPathInfo> infos;
@ -3349,10 +3365,7 @@ void DerivationGoal::registerOutputs()
}
}
auto localStoreP = dynamic_cast<LocalStore *>(&worker.store);
if (!localStoreP)
throw Unsupported("can only register outputs with local store, but this is %s", worker.store.getUri());
auto & localStore = *localStoreP;
auto & localStore = getLocalStore();
if (buildMode == bmCheck) {
@ -3481,10 +3494,7 @@ void DerivationGoal::registerOutputs()
paths referenced by each of them. If there are cycles in the
outputs, this will fail. */
{
auto localStoreP = dynamic_cast<LocalStore *>(&worker.store);
if (!localStoreP)
throw Unsupported("can only register outputs with local store, but this is %s", worker.store.getUri());
auto & localStore = *localStoreP;
auto & localStore = getLocalStore();
ValidPathInfos infos2;
for (auto & [outputName, newInfo] : infos) {
@ -3513,7 +3523,7 @@ void DerivationGoal::registerOutputs()
}
void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
void LocalDerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
{
std::map<Path, const ValidPathInfo &> outputsByPath;
for (auto & output : outputs)
@ -3678,6 +3688,7 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
}
#if 0
Path DerivationGoal::openLogFile()
{
logSize = 0;
@ -3720,9 +3731,10 @@ void DerivationGoal::closeLogFile()
logSink = logFileSink = 0;
fdLogFile = -1;
}
#endif
void DerivationGoal::deleteTmpDir(bool force)
void LocalDerivationGoal::deleteTmpDir(bool force)
{
if (tmpDir != "") {
/* Don't keep temporary directories for builtins because they
@ -3738,10 +3750,17 @@ void DerivationGoal::deleteTmpDir(bool force)
}
bool LocalDerivationGoal::isReadDesc(int fd)
{
return (hook && DerivationGoal::isReadDesc(fd)) ||
(!hook && fd == builderOut.readSide.get());
}
#if 0
void DerivationGoal::handleChildOutput(int fd, const string & data)
{
if ((hook && fd == hook->builderOut.readSide.get()) ||
(!hook && fd == builderOut.readSide.get()))
if (isReadDesc(fd))
{
logSize += data.size();
if (settings.maxLogSize && logSize > settings.maxLogSize) {
@ -3855,9 +3874,10 @@ void DerivationGoal::checkPathValidity()
}
}
}
#endif
StorePath DerivationGoal::makeFallbackPath(std::string_view outputName)
StorePath LocalDerivationGoal::makeFallbackPath(std::string_view outputName)
{
return worker.store.makeStorePath(
"rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName),
@ -3865,7 +3885,7 @@ StorePath DerivationGoal::makeFallbackPath(std::string_view outputName)
}
StorePath DerivationGoal::makeFallbackPath(const StorePath & path)
StorePath LocalDerivationGoal::makeFallbackPath(const StorePath & path)
{
return worker.store.makeStorePath(
"rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()),
@ -3873,6 +3893,7 @@ StorePath DerivationGoal::makeFallbackPath(const StorePath & path)
}
#if 0
void DerivationGoal::done(BuildResult::Status status, std::optional<Error> ex)
{
result.status = status;
@ -3897,6 +3918,7 @@ void DerivationGoal::done(BuildResult::Status status, std::optional<Error> ex)
worker.updateProgress();
}
#endif
}

View file

@ -1,48 +1,15 @@
#pragma once
#include "parsed-derivations.hh"
#include "lock.hh"
#include "derivation-goal.hh"
#include "local-store.hh"
#include "goal.hh"
namespace nix {
using std::map;
struct HookInstance;
typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
/* Unless we are repairing, we don't both to test validity and just assume it,
so the choices are `Absent` or `Valid`. */
enum struct PathStatus {
Corrupt,
Absent,
Valid,
};
struct InitialOutputStatus {
StorePath path;
PathStatus status;
/* Valid in the store, and additionally non-corrupt if we are repairing */
bool isValid() const {
return status == PathStatus::Valid;
}
/* Merely present, allowed to be corrupt */
bool isPresent() const {
return status == PathStatus::Corrupt
|| status == PathStatus::Valid;
}
};
struct InitialOutput {
bool wanted;
Hash outputHash;
std::optional<InitialOutputStatus> known;
};
struct DerivationGoal : public Goal
struct LocalDerivationGoal : public DerivationGoal
{
LocalStore & getLocalStore();
#if 0
/* Whether to use an on-disk .drv file. */
bool useDerivation;
@ -78,6 +45,7 @@ struct DerivationGoal : public Goal
StorePathSet inputPaths;
std::map<std::string, InitialOutput> initialOutputs;
#endif
/* User selected for running the builder. */
std::unique_ptr<UserLock> buildUser;
@ -91,6 +59,7 @@ struct DerivationGoal : public Goal
/* The path of the temporary directory in the sandbox. */
Path tmpDirInSandbox;
#if 0
/* File descriptor for the log file. */
AutoCloseFD fdLogFile;
std::shared_ptr<BufferedSink> logFileSink, logSink;
@ -105,6 +74,7 @@ struct DerivationGoal : public Goal
size_t currentLogLinePos = 0; // to handle carriage return
std::string currentHookLine;
#endif
/* Pipe for the builder's standard output/error. */
Pipe builderOut;
@ -120,8 +90,10 @@ struct DerivationGoal : public Goal
namespace. */
bool usingUserNamespace = true;
#if 0
/* The build hook. */
std::unique_ptr<HookInstance> hook;
#endif
/* Whether we're currently doing a chroot build. */
bool useChroot = false;
@ -131,14 +103,18 @@ struct DerivationGoal : public Goal
/* RAII object to delete the chroot directory. */
std::shared_ptr<AutoDelete> autoDelChroot;
#if 0
/* The sort of derivation we are building. */
DerivationType derivationType;
#endif
/* Whether to run the build in a private network namespace. */
bool privateNetwork = false;
#if 0
typedef void (DerivationGoal::*GoalState)();
GoalState state;
#endif
/* Stuff we need to pass to initChild(). */
struct ChrootPath {
@ -179,6 +155,7 @@ struct DerivationGoal : public Goal
*/
OutputPathMap scratchOutputs;
#if 0
/* The final output paths of the build.
- For input-addressed derivations, always the precomputed paths
@ -190,18 +167,21 @@ struct DerivationGoal : public Goal
OutputPathMap finalOutputs;
BuildMode buildMode;
#endif
/* If we're repairing without a chroot, there may be outputs that
are valid but corrupt. So we redirect these outputs to
temporary paths. */
StorePathSet redirectedBadOutputs;
#if 0
BuildResult result;
/* The current round, if we're building multiple times. */
size_t curRound = 1;
size_t nrRounds;
#endif
/* Path registration info from the previous round, if we're
building multiple times. Since this contains the hash, it
@ -214,6 +194,7 @@ struct DerivationGoal : public Goal
const static Path homeDir;
#if 0
std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds, mcRunningBuilds;
std::unique_ptr<Activity> act;
@ -225,6 +206,7 @@ struct DerivationGoal : public Goal
/* The remote machine on which we're building. */
std::string machineName;
#endif
/* The recursive Nix daemon socket. */
AutoCloseFD daemonSocket;
@ -249,17 +231,14 @@ struct DerivationGoal : public Goal
friend struct RestrictedStore;
DerivationGoal(const StorePath & drvPath,
const StringSet & wantedOutputs, Worker & worker,
BuildMode buildMode = bmNormal);
DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
const StringSet & wantedOutputs, Worker & worker,
BuildMode buildMode = bmNormal);
~DerivationGoal();
using DerivationGoal::DerivationGoal;
virtual ~LocalDerivationGoal() override;
/* Whether we need to perform hash rewriting if there are valid output paths. */
bool needsHashRewrite();
#if 0
void timedOut(Error && ex) override;
string key() override;
@ -280,13 +259,16 @@ struct DerivationGoal : public Goal
void closureRepaired();
void inputsRealised();
void tryToBuild();
void tryLocalBuild();
#endif
void tryLocalBuild() override;
#if 0
void buildDone();
void resolvedFinished();
/* Is the build hook willing to perform the build? */
HookReply tryBuildHook();
#endif
/* Start building a derivation. */
void startBuilder();
@ -311,27 +293,46 @@ struct DerivationGoal : public Goal
/* Make a file owned by the builder. */
void chownToBuilder(const Path & path);
int getChildStatus() override;
/* Run the builder's process. */
void runChild();
/* Check that the derivation outputs all exist and register them
as valid. */
void registerOutputs();
void registerOutputs() override;
/* Check that an output meets the requirements specified by the
'outputChecks' attribute (or the legacy
'{allowed,disallowed}{References,Requisites}' attributes). */
void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs);
#if 0
/* Open a log file and a pipe to it. */
Path openLogFile();
/* Close the log file. */
void closeLogFile();
#endif
/* Close the read side of the logger pipe. */
void closeReadPipes() override;
/* Cleanup hooks for buildDone() */
void cleanupHookFinally() override;
void cleanupPreChildKill() override;
void cleanupPostChildKill() override;
bool cleanupDecideWhetherDiskFull() override;
void cleanupPostOutputsRegisteredModeCheck() override;
void cleanupPostOutputsRegisteredModeNonCheck() override;
bool isReadDesc(int fd) override;
/* Delete the temporary directory, if we have one. */
void deleteTmpDir(bool force);
#if 0
/* Callback used by the worker to write to the log. */
void handleChildOutput(int fd, const string & data) override;
void handleEOF(int fd) override;
@ -345,9 +346,10 @@ struct DerivationGoal : public Goal
/* Return the set of (in)valid paths. */
void checkPathValidity();
#endif
/* Forcibly kill the child process, if any. */
void killChild();
void killChild() override;
/* Create alternative path calculated from but distinct from the
input, so we can avoid overwriting outputs (or other store paths)
@ -359,6 +361,7 @@ struct DerivationGoal : public Goal
rewrites caught everything */
StorePath makeFallbackPath(std::string_view outputName);
#if 0
void repairClosure();
void started();
@ -368,6 +371,7 @@ struct DerivationGoal : public Goal
std::optional<Error> ex = {});
StorePathSet exportReferences(const StorePathSet & storePaths);
#endif
};
}

View file

@ -1,7 +1,7 @@
#include "machines.hh"
#include "worker.hh"
#include "substitution-goal.hh"
#include "derivation-goal.hh"
#include "local-derivation-goal.hh"
#include "hook-instance.hh"
#include <poll.h>
@ -59,8 +59,10 @@ std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon(
std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drvPath,
const StringSet & wantedOutputs, BuildMode buildMode)
{
return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() {
return std::make_shared<DerivationGoal>(drvPath, wantedOutputs, *this, buildMode);
return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr<DerivationGoal> {
return !dynamic_cast<LocalStore *>(&store)
? std::make_shared</* */DerivationGoal>(drvPath, wantedOutputs, *this, buildMode)
: std::make_shared<LocalDerivationGoal>(drvPath, wantedOutputs, *this, buildMode);
});
}
@ -68,8 +70,10 @@ std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drv
std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath & drvPath,
const BasicDerivation & drv, const StringSet & wantedOutputs, BuildMode buildMode)
{
return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() {
return std::make_shared<DerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode);
return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr<DerivationGoal> {
return !dynamic_cast<LocalStore *>(&store)
? std::make_shared</* */DerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode)
: std::make_shared<LocalDerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode);
});
}

View file

@ -280,7 +280,7 @@ private:
void createUser(const std::string & userName, uid_t userId) override;
friend struct DerivationGoal;
friend struct LocalDerivationGoal;
friend struct SubstitutionGoal;
};