lix-project/lix
20
58
Fork 31
Code Issues 359 Code Review (Gerrit) Projects 3 Releases Wiki Activity

Trim derivation-goal.hh

Browse source
This commit is contained in:
John Ericson 2020-10-12 17:15:32 +00:00
parent 4eb8c69853
commit 2ce726947a
1 changed files with 2 additions and 407 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

409
src/libstore/build/derivation-goal.hh
View file

@ -1,312 +1,18 @@
#pragma once #pragma once
#include "machines.hh"
#include "parsed-derivations.hh" #include "parsed-derivations.hh"
#include "lock.hh" #include "lock.hh"
#include "local-store.hh" #include "local-store.hh"
#include "goal.hh"
#include <memory>
#include <algorithm>
#include <iostream>
#include <map>
#include <sstream>
#include <thread>
#include <future>
#include <chrono>
#include <regex>
#include <queue>
namespace nix { namespace nix {
using std::map; using std::map;
/* Forward definition. */
class Worker;
struct HookInstance; struct HookInstance;
/* A pointer to a goal. */
struct Goal;
class DerivationGoal;
typedef std::shared_ptr<Goal> GoalPtr;
typedef std::weak_ptr<Goal> WeakGoalPtr;
struct CompareGoalPtrs {
bool operator() (const GoalPtr & a, const GoalPtr & b) const;
};
/* Set of goals. */
typedef set<GoalPtr, CompareGoalPtrs> Goals;
typedef list<WeakGoalPtr> WeakGoals;
/* A map of paths to goals (and the other way around). */
typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap;
struct Goal : public std::enable_shared_from_this<Goal>
{
typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode;
/* Backlink to the worker. */
Worker & worker;
/* Goals that this goal is waiting for. */
Goals waitees;
/* Goals waiting for this one to finish. Must use weak pointers
here to prevent cycles. */
WeakGoals waiters;
/* Number of goals we are/were waiting for that have failed. */
unsigned int nrFailed;
/* Number of substitution goals we are/were waiting for that
failed because there are no substituters. */
unsigned int nrNoSubstituters;
/* Number of substitution goals we are/were waiting for that
failed because othey had unsubstitutable references. */
unsigned int nrIncompleteClosure;
/* Name of this goal for debugging purposes. */
string name;
/* Whether the goal is finished. */
ExitCode exitCode;
/* Exception containing an error message, if any. */
std::optional<Error> ex;
Goal(Worker & worker) : worker(worker)
{
nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
exitCode = ecBusy;
}
virtual ~Goal()
{
trace("goal destroyed");
}
virtual void work() = 0;
void addWaitee(GoalPtr waitee);
virtual void waiteeDone(GoalPtr waitee, ExitCode result);
virtual void handleChildOutput(int fd, const string & data)
{
abort();
}
virtual void handleEOF(int fd)
{
abort();
}
void trace(const FormatOrString & fs);
string getName()
{
return name;
}
/* Callback in case of a timeout. It should wake up its waiters,
get rid of any running child processes that are being monitored
by the worker (important!), etc. */
virtual void timedOut(Error && ex) = 0;
virtual string key() = 0;
void amDone(ExitCode result, std::optional<Error> ex = {});
};
typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
/* A mapping used to remember for each child process to what goal it
belongs, and file descriptors for receiving log data and output
path creation commands. */
struct Child
{
WeakGoalPtr goal;
Goal * goal2; // ugly hackery
set<int> fds;
bool respectTimeouts;
bool inBuildSlot;
steady_time_point lastOutput; /* time we last got output on stdout/stderr */
steady_time_point timeStarted;
};
/* The worker class. */
class Worker
{
private:
/* Note: the worker should only have strong pointers to the
top-level goals. */
/* The top-level goals of the worker. */
Goals topGoals;
/* Goals that are ready to do some work. */
WeakGoals awake;
/* Goals waiting for a build slot. */
WeakGoals wantingToBuild;
/* Child processes currently running. */
std::list<Child> children;
/* Number of build slots occupied. This includes local builds and
substitutions but not remote builds via the build hook. */
unsigned int nrLocalBuilds;
/* Maps used to prevent multiple instantiations of a goal for the
same derivation / path. */
WeakGoalMap derivationGoals;
WeakGoalMap substitutionGoals;
/* Goals waiting for busy paths to be unlocked. */
WeakGoals waitingForAnyGoal;
/* Goals sleeping for a few seconds (polling a lock). */
WeakGoals waitingForAWhile;
/* Last time the goals in `waitingForAWhile' where woken up. */
steady_time_point lastWokenUp;
/* Cache for pathContentsGood(). */
std::map<StorePath, bool> pathContentsGoodCache;
public:
const Activity act;
const Activity actDerivations;
const Activity actSubstitutions;
/* Set if at least one derivation had a BuildError (i.e. permanent
failure). */
bool permanentFailure;
/* Set if at least one derivation had a timeout. */
bool timedOut;
/* Set if at least one derivation fails with a hash mismatch. */
bool hashMismatch;
/* Set if at least one derivation is not deterministic in check mode. */
bool checkMismatch;
LocalStore & store;
std::unique_ptr<HookInstance> hook;
uint64_t expectedBuilds = 0;
uint64_t doneBuilds = 0;
uint64_t failedBuilds = 0;
uint64_t runningBuilds = 0;
uint64_t expectedSubstitutions = 0;
uint64_t doneSubstitutions = 0;
uint64_t failedSubstitutions = 0;
uint64_t runningSubstitutions = 0;
uint64_t expectedDownloadSize = 0;
uint64_t doneDownloadSize = 0;
uint64_t expectedNarSize = 0;
uint64_t doneNarSize = 0;
/* Whether to ask the build hook if it can build a derivation. If
it answers with "decline-permanently", we don't try again. */
bool tryBuildHook = true;
Worker(LocalStore & store);
~Worker();
/* Make a goal (with caching). */
/* derivation goal */
private:
std::shared_ptr<DerivationGoal> makeDerivationGoalCommon(
const StorePath & drvPath, const StringSet & wantedOutputs,
std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal);
public:
std::shared_ptr<DerivationGoal> makeDerivationGoal(
const StorePath & drvPath,
const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(
const StorePath & drvPath, const BasicDerivation & drv,
const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
/* substitution goal */
GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
/* Remove a dead goal. */
void removeGoal(GoalPtr goal);
/* Wake up a goal (i.e., there is something for it to do). */
void wakeUp(GoalPtr goal);
/* Return the number of local build and substitution processes
currently running (but not remote builds via the build
hook). */
unsigned int getNrLocalBuilds();
/* Registers a running child process. `inBuildSlot' means that
the process counts towards the jobs limit. */
void childStarted(GoalPtr goal, const set<int> & fds,
bool inBuildSlot, bool respectTimeouts);
/* Unregisters a running child process. `wakeSleepers' should be
false if there is no sense in waking up goals that are sleeping
because they can't run yet (e.g., there is no free build slot,
or the hook would still say `postpone'). */
void childTerminated(Goal * goal, bool wakeSleepers = true);
/* Put `goal' to sleep until a build slot becomes available (which
might be right away). */
void waitForBuildSlot(GoalPtr goal);
/* Wait for any goal to finish. Pretty indiscriminate way to
wait for some resource that some other goal is holding. */
void waitForAnyGoal(GoalPtr goal);
/* Wait for a few seconds and then retry this goal. Used when
waiting for a lock held by another process. This kind of
polling is inefficient, but POSIX doesn't really provide a way
to wait for multiple locks in the main select() loop. */
void waitForAWhile(GoalPtr goal);
/* Loop until the specified top-level goals have finished. */
void run(const Goals & topGoals);
/* Wait for input to become available. */
void waitForInput();
unsigned int exitStatus();
/* Check whether the given valid path exists and has the right
contents. */
bool pathContentsGood(const StorePath & path);
void markContentsGood(const StorePath & path);
void updateProgress()
{
actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds);
actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions);
act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize);
act.setExpected(actCopyPath, expectedNarSize + doneNarSize);
}
};
typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
class SubstitutionGoal;
/* Unless we are repairing, we don't both to test validity and just assume it, /* Unless we are repairing, we don't both to test validity and just assume it,
so the choices are `Absent` or `Valid`. */ so the choices are `Absent` or `Valid`. */
enum struct PathStatus { enum struct PathStatus {
@ -554,14 +260,7 @@ public:
void timedOut(Error && ex) override; void timedOut(Error && ex) override;
string key() override string key() override;
{
/* Ensure that derivations get built in order of their name,
i.e. a derivation named "aardvark" always comes before
"baboon". And substitution goals always happen before
derivation goals (due to "b$"). */
return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath);
}
void work() override; void work() override;
@ -677,108 +376,4 @@ private:
StorePathSet exportReferences(const StorePathSet & storePaths); StorePathSet exportReferences(const StorePathSet & storePaths);
}; };
class SubstitutionGoal : public Goal
{
friend class Worker;
private:
/* The store path that should be realised through a substitute. */
StorePath storePath;
/* The path the substituter refers to the path as. This will be
* different when the stores have different names. */
std::optional<StorePath> subPath;
/* The remaining substituters. */
std::list<ref<Store>> subs;
/* The current substituter. */
std::shared_ptr<Store> sub;
/* Whether a substituter failed. */
bool substituterFailed = false;
/* Path info returned by the substituter's query info operation. */
std::shared_ptr<const ValidPathInfo> info;
/* Pipe for the substituter's standard output. */
Pipe outPipe;
/* The substituter thread. */
std::thread thr;
std::promise<void> promise;
/* Whether to try to repair a valid path. */
RepairFlag repair;
/* Location where we're downloading the substitute. Differs from
storePath when doing a repair. */
Path destPath;
std::unique_ptr<MaintainCount<uint64_t>> maintainExpectedSubstitutions,
maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload;
typedef void (SubstitutionGoal::*GoalState)();
GoalState state;
/* Content address for recomputing store path */
std::optional<ContentAddress> ca;
public:
SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
~SubstitutionGoal();
void timedOut(Error && ex) override { abort(); };
string key() override
{
/* "a$" ensures substitution goals happen before derivation
goals. */
return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath);
}
void work() override;
/* The states. */
void init();
void tryNext();
void gotInfo();
void referencesValid();
void tryToRun();
void finished();
/* Callback used by the worker to write to the log. */
void handleChildOutput(int fd, const string & data) override;
void handleEOF(int fd) override;
StorePath getStorePath() { return storePath; }
};
struct HookInstance
{
/* Pipes for talking to the build hook. */
Pipe toHook;
/* Pipe for the hook's standard output/error. */
Pipe fromHook;
/* Pipe for the builder's standard output/error. */
Pipe builderOut;
/* The process ID of the hook. */
Pid pid;
FdSink sink;
std::map<ActivityId, Activity> activities;
HookInstance();
~HookInstance();
};
void addToWeakGoals(WeakGoals & goals, GoalPtr p);
} }