Merge remote-tracking branch 'upstream/master' into fix-and-ci-static-builds
This commit is contained in:
John Ericson
commit
fccef6a7fa
|
|
@ -4,13 +4,14 @@ function _complete_nix {
|
|||
_get_comp_words_by_ref -n ':=&' words cword cur
|
||||
local have_type
|
||||
while IFS= read -r line; do
|
||||
local completion=${line%% *}
|
||||
if [[ -z $have_type ]]; then
|
||||
have_type=1
|
||||
if [[ $line = filenames ]]; then
|
||||
if [[ $completion = filenames ]]; then
|
||||
compopt -o filenames
|
||||
fi
|
||||
else
|
||||
COMPREPLY+=("$line")
|
||||
COMPREPLY+=("$completion")
|
||||
fi
|
||||
done < <(NIX_GET_COMPLETIONS=$cword "${words[@]}")
|
||||
__ltrim_colon_completions "$cur"
|
||||
|
|
|
|||
21
misc/zsh/completion.zsh
Normal file
21
misc/zsh/completion.zsh
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
function _nix() {
|
||||
local ifs_bk="$IFS"
|
||||
local input=("${(Q)words[@]}")
|
||||
IFS=$'\n'
|
||||
local res=($(NIX_GET_COMPLETIONS=$((CURRENT - 1)) "$input[@]"))
|
||||
IFS="$ifs_bk"
|
||||
local tpe="${${res[1]}%%> *}"
|
||||
local -a suggestions
|
||||
declare -a suggestions
|
||||
for suggestion in ${res:1}; do
|
||||
# FIXME: This doesn't work properly if the suggestion word contains a `:`
|
||||
# itself
|
||||
suggestions+="${suggestion/ /:}"
|
||||
done
|
||||
if [[ "$tpe" == filenames ]]; then
|
||||
compadd -f
|
||||
fi
|
||||
_describe 'nix' suggestions
|
||||
}
|
||||
|
||||
compdef _nix nix
|
||||
|
|
@ -3,6 +3,7 @@
|
|||
#include "util.hh"
|
||||
#include "globals.hh"
|
||||
#include "store-api.hh"
|
||||
#include "local-fs-store.hh"
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ MixCommonArgs::MixCommonArgs(const string & programName)
|
|||
globalConfig.getSettings(settings);
|
||||
for (auto & s : settings)
|
||||
if (hasPrefix(s.first, prefix))
|
||||
completions->insert(s.first);
|
||||
completions->add(s.first, s.second.description);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
379
src/libstore/build/derivation-goal.hh
Normal file
379
src/libstore/build/derivation-goal.hh
Normal file
|
|
@ -0,0 +1,379 @@
|
|||
#pragma once
|
||||
|
||||
#include "parsed-derivations.hh"
|
||||
#include "lock.hh"
|
||||
#include "local-store.hh"
|
||||
#include "goal.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
using std::map;
|
||||
|
||||
struct HookInstance;
|
||||
|
||||
typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
|
||||
|
||||
/* Unless we are repairing, we don't both to test validity and just assume it,
|
||||
so the choices are `Absent` or `Valid`. */
|
||||
enum struct PathStatus {
|
||||
Corrupt,
|
||||
Absent,
|
||||
Valid,
|
||||
};
|
||||
|
||||
struct InitialOutputStatus {
|
||||
StorePath path;
|
||||
PathStatus status;
|
||||
/* Valid in the store, and additionally non-corrupt if we are repairing */
|
||||
bool isValid() const {
|
||||
return status == PathStatus::Valid;
|
||||
}
|
||||
/* Merely present, allowed to be corrupt */
|
||||
bool isPresent() const {
|
||||
return status == PathStatus::Corrupt
|
||||
|| status == PathStatus::Valid;
|
||||
}
|
||||
};
|
||||
|
||||
struct InitialOutput {
|
||||
bool wanted;
|
||||
std::optional<InitialOutputStatus> known;
|
||||
};
|
||||
|
||||
class DerivationGoal : public Goal
|
||||
{
|
||||
private:
|
||||
/* Whether to use an on-disk .drv file. */
|
||||
bool useDerivation;
|
||||
|
||||
/* The path of the derivation. */
|
||||
StorePath drvPath;
|
||||
|
||||
/* The specific outputs that we need to build. Empty means all of
|
||||
them. */
|
||||
StringSet wantedOutputs;
|
||||
|
||||
/* Whether additional wanted outputs have been added. */
|
||||
bool needRestart = false;
|
||||
|
||||
/* Whether to retry substituting the outputs after building the
|
||||
inputs. */
|
||||
bool retrySubstitution;
|
||||
|
||||
/* The derivation stored at drvPath. */
|
||||
std::unique_ptr<BasicDerivation> drv;
|
||||
|
||||
std::unique_ptr<ParsedDerivation> parsedDrv;
|
||||
|
||||
/* The remainder is state held during the build. */
|
||||
|
||||
/* Locks on (fixed) output paths. */
|
||||
PathLocks outputLocks;
|
||||
|
||||
/* All input paths (that is, the union of FS closures of the
|
||||
immediate input paths). */
|
||||
StorePathSet inputPaths;
|
||||
|
||||
std::map<std::string, InitialOutput> initialOutputs;
|
||||
|
||||
/* User selected for running the builder. */
|
||||
std::unique_ptr<UserLock> buildUser;
|
||||
|
||||
/* The process ID of the builder. */
|
||||
Pid pid;
|
||||
|
||||
/* The temporary directory. */
|
||||
Path tmpDir;
|
||||
|
||||
/* The path of the temporary directory in the sandbox. */
|
||||
Path tmpDirInSandbox;
|
||||
|
||||
/* File descriptor for the log file. */
|
||||
AutoCloseFD fdLogFile;
|
||||
std::shared_ptr<BufferedSink> logFileSink, logSink;
|
||||
|
||||
/* Number of bytes received from the builder's stdout/stderr. */
|
||||
unsigned long logSize;
|
||||
|
||||
/* The most recent log lines. */
|
||||
std::list<std::string> logTail;
|
||||
|
||||
std::string currentLogLine;
|
||||
size_t currentLogLinePos = 0; // to handle carriage return
|
||||
|
||||
std::string currentHookLine;
|
||||
|
||||
/* Pipe for the builder's standard output/error. */
|
||||
Pipe builderOut;
|
||||
|
||||
/* Pipe for synchronising updates to the builder namespaces. */
|
||||
Pipe userNamespaceSync;
|
||||
|
||||
/* The mount namespace of the builder, used to add additional
|
||||
paths to the sandbox as a result of recursive Nix calls. */
|
||||
AutoCloseFD sandboxMountNamespace;
|
||||
|
||||
/* On Linux, whether we're doing the build in its own user
|
||||
namespace. */
|
||||
bool usingUserNamespace = true;
|
||||
|
||||
/* The build hook. */
|
||||
std::unique_ptr<HookInstance> hook;
|
||||
|
||||
/* Whether we're currently doing a chroot build. */
|
||||
bool useChroot = false;
|
||||
|
||||
Path chrootRootDir;
|
||||
|
||||
/* RAII object to delete the chroot directory. */
|
||||
std::shared_ptr<AutoDelete> autoDelChroot;
|
||||
|
||||
/* The sort of derivation we are building. */
|
||||
DerivationType derivationType;
|
||||
|
||||
/* Whether to run the build in a private network namespace. */
|
||||
bool privateNetwork = false;
|
||||
|
||||
typedef void (DerivationGoal::*GoalState)();
|
||||
GoalState state;
|
||||
|
||||
/* Stuff we need to pass to initChild(). */
|
||||
struct ChrootPath {
|
||||
Path source;
|
||||
bool optional;
|
||||
ChrootPath(Path source = "", bool optional = false)
|
||||
: source(source), optional(optional)
|
||||
{ }
|
||||
};
|
||||
typedef map<Path, ChrootPath> DirsInChroot; // maps target path to source path
|
||||
DirsInChroot dirsInChroot;
|
||||
|
||||
typedef map<string, string> Environment;
|
||||
Environment env;
|
||||
|
||||
#if __APPLE__
|
||||
typedef string SandboxProfile;
|
||||
SandboxProfile additionalSandboxProfile;
|
||||
#endif
|
||||
|
||||
/* Hash rewriting. */
|
||||
StringMap inputRewrites, outputRewrites;
|
||||
typedef map<StorePath, StorePath> RedirectedOutputs;
|
||||
RedirectedOutputs redirectedOutputs;
|
||||
|
||||
/* The outputs paths used during the build.
|
||||
|
||||
- Input-addressed derivations or fixed content-addressed outputs are
|
||||
sometimes built when some of their outputs already exist, and can not
|
||||
be hidden via sandboxing. We use temporary locations instead and
|
||||
rewrite after the build. Otherwise the regular predetermined paths are
|
||||
put here.
|
||||
|
||||
- Floating content-addressed derivations do not know their final build
|
||||
output paths until the outputs are hashed, so random locations are
|
||||
used, and then renamed. The randomness helps guard against hidden
|
||||
self-references.
|
||||
*/
|
||||
OutputPathMap scratchOutputs;
|
||||
|
||||
/* The final output paths of the build.
|
||||
|
||||
- For input-addressed derivations, always the precomputed paths
|
||||
|
||||
- For content-addressed derivations, calcuated from whatever the hash
|
||||
ends up being. (Note that fixed outputs derivations that produce the
|
||||
"wrong" output still install that data under its true content-address.)
|
||||
*/
|
||||
OutputPathMap finalOutputs;
|
||||
|
||||
BuildMode buildMode;
|
||||
|
||||
/* If we're repairing without a chroot, there may be outputs that
|
||||
are valid but corrupt. So we redirect these outputs to
|
||||
temporary paths. */
|
||||
StorePathSet redirectedBadOutputs;
|
||||
|
||||
BuildResult result;
|
||||
|
||||
/* The current round, if we're building multiple times. */
|
||||
size_t curRound = 1;
|
||||
|
||||
size_t nrRounds;
|
||||
|
||||
/* Path registration info from the previous round, if we're
|
||||
building multiple times. Since this contains the hash, it
|
||||
allows us to compare whether two rounds produced the same
|
||||
result. */
|
||||
std::map<Path, ValidPathInfo> prevInfos;
|
||||
|
||||
uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); }
|
||||
gid_t sandboxGid() { return usingUserNamespace ? 100 : buildUser->getGID(); }
|
||||
|
||||
const static Path homeDir;
|
||||
|
||||
std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds, mcRunningBuilds;
|
||||
|
||||
std::unique_ptr<Activity> act;
|
||||
|
||||
/* Activity that denotes waiting for a lock. */
|
||||
std::unique_ptr<Activity> actLock;
|
||||
|
||||
std::map<ActivityId, Activity> builderActivities;
|
||||
|
||||
/* The remote machine on which we're building. */
|
||||
std::string machineName;
|
||||
|
||||
/* The recursive Nix daemon socket. */
|
||||
AutoCloseFD daemonSocket;
|
||||
|
||||
/* The daemon main thread. */
|
||||
std::thread daemonThread;
|
||||
|
||||
/* The daemon worker threads. */
|
||||
std::vector<std::thread> daemonWorkerThreads;
|
||||
|
||||
/* Paths that were added via recursive Nix calls. */
|
||||
StorePathSet addedPaths;
|
||||
|
||||
/* Recursive Nix calls are only allowed to build or realize paths
|
||||
in the original input closure or added via a recursive Nix call
|
||||
(so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
|
||||
/nix/store/<bla> is some arbitrary path in a binary cache). */
|
||||
bool isAllowed(const StorePath & path)
|
||||
{
|
||||
return inputPaths.count(path) || addedPaths.count(path);
|
||||
}
|
||||
|
||||
friend struct RestrictedStore;
|
||||
|
||||
public:
|
||||
DerivationGoal(const StorePath & drvPath,
|
||||
const StringSet & wantedOutputs, Worker & worker,
|
||||
BuildMode buildMode = bmNormal);
|
||||
DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
|
||||
const StringSet & wantedOutputs, Worker & worker,
|
||||
BuildMode buildMode = bmNormal);
|
||||
~DerivationGoal();
|
||||
|
||||
/* Whether we need to perform hash rewriting if there are valid output paths. */
|
||||
bool needsHashRewrite();
|
||||
|
||||
void timedOut(Error && ex) override;
|
||||
|
||||
string key() override;
|
||||
|
||||
void work() override;
|
||||
|
||||
StorePath getDrvPath()
|
||||
{
|
||||
return drvPath;
|
||||
}
|
||||
|
||||
/* Add wanted outputs to an already existing derivation goal. */
|
||||
void addWantedOutputs(const StringSet & outputs);
|
||||
|
||||
BuildResult getResult() { return result; }
|
||||
|
||||
private:
|
||||
/* The states. */
|
||||
void getDerivation();
|
||||
void loadDerivation();
|
||||
void haveDerivation();
|
||||
void outputsSubstitutionTried();
|
||||
void gaveUpOnSubstitution();
|
||||
void closureRepaired();
|
||||
void inputsRealised();
|
||||
void tryToBuild();
|
||||
void tryLocalBuild();
|
||||
void buildDone();
|
||||
|
||||
void resolvedFinished();
|
||||
|
||||
/* Is the build hook willing to perform the build? */
|
||||
HookReply tryBuildHook();
|
||||
|
||||
/* Start building a derivation. */
|
||||
void startBuilder();
|
||||
|
||||
/* Fill in the environment for the builder. */
|
||||
void initEnv();
|
||||
|
||||
/* Setup tmp dir location. */
|
||||
void initTmpDir();
|
||||
|
||||
/* Write a JSON file containing the derivation attributes. */
|
||||
void writeStructuredAttrs();
|
||||
|
||||
void startDaemon();
|
||||
|
||||
void stopDaemon();
|
||||
|
||||
/* Add 'path' to the set of paths that may be referenced by the
|
||||
outputs, and make it appear in the sandbox. */
|
||||
void addDependency(const StorePath & path);
|
||||
|
||||
/* Make a file owned by the builder. */
|
||||
void chownToBuilder(const Path & path);
|
||||
|
||||
/* Run the builder's process. */
|
||||
void runChild();
|
||||
|
||||
friend int childEntry(void *);
|
||||
|
||||
/* Check that the derivation outputs all exist and register them
|
||||
as valid. */
|
||||
void registerOutputs();
|
||||
|
||||
/* Check that an output meets the requirements specified by the
|
||||
'outputChecks' attribute (or the legacy
|
||||
'{allowed,disallowed}{References,Requisites}' attributes). */
|
||||
void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs);
|
||||
|
||||
/* Open a log file and a pipe to it. */
|
||||
Path openLogFile();
|
||||
|
||||
/* Close the log file. */
|
||||
void closeLogFile();
|
||||
|
||||
/* Delete the temporary directory, if we have one. */
|
||||
void deleteTmpDir(bool force);
|
||||
|
||||
/* Callback used by the worker to write to the log. */
|
||||
void handleChildOutput(int fd, const string & data) override;
|
||||
void handleEOF(int fd) override;
|
||||
void flushLine();
|
||||
|
||||
/* Wrappers around the corresponding Store methods that first consult the
|
||||
derivation. This is currently needed because when there is no drv file
|
||||
there also is no DB entry. */
|
||||
std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap();
|
||||
OutputPathMap queryDerivationOutputMap();
|
||||
|
||||
/* Return the set of (in)valid paths. */
|
||||
void checkPathValidity();
|
||||
|
||||
/* Forcibly kill the child process, if any. */
|
||||
void killChild();
|
||||
|
||||
/* Create alternative path calculated from but distinct from the
|
||||
input, so we can avoid overwriting outputs (or other store paths)
|
||||
that already exist. */
|
||||
StorePath makeFallbackPath(const StorePath & path);
|
||||
/* Make a path to another based on the output name along with the
|
||||
derivation hash. */
|
||||
/* FIXME add option to randomize, so we can audit whether our
|
||||
rewrites caught everything */
|
||||
StorePath makeFallbackPath(std::string_view outputName);
|
||||
|
||||
void repairClosure();
|
||||
|
||||
void started();
|
||||
|
||||
void done(
|
||||
BuildResult::Status status,
|
||||
std::optional<Error> ex = {});
|
||||
|
||||
StorePathSet exportReferences(const StorePathSet & storePaths);
|
||||
};
|
||||
|
||||
}
|
||||
89
src/libstore/build/goal.cc
Normal file
89
src/libstore/build/goal.cc
Normal file
|
|
@ -0,0 +1,89 @@
|
|||
#include "goal.hh"
|
||||
#include "worker.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
|
||||
bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const {
|
||||
string s1 = a->key();
|
||||
string s2 = b->key();
|
||||
return s1 < s2;
|
||||
}
|
||||
|
||||
|
||||
void addToWeakGoals(WeakGoals & goals, GoalPtr p)
|
||||
{
|
||||
// FIXME: necessary?
|
||||
// FIXME: O(n)
|
||||
for (auto & i : goals)
|
||||
if (i.lock() == p) return;
|
||||
goals.push_back(p);
|
||||
}
|
||||
|
||||
|
||||
void Goal::addWaitee(GoalPtr waitee)
|
||||
{
|
||||
waitees.insert(waitee);
|
||||
addToWeakGoals(waitee->waiters, shared_from_this());
|
||||
}
|
||||
|
||||
|
||||
void Goal::waiteeDone(GoalPtr waitee, ExitCode result)
|
||||
{
|
||||
assert(waitees.find(waitee) != waitees.end());
|
||||
waitees.erase(waitee);
|
||||
|
||||
trace(fmt("waitee '%s' done; %d left", waitee->name, waitees.size()));
|
||||
|
||||
if (result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure) ++nrFailed;
|
||||
|
||||
if (result == ecNoSubstituters) ++nrNoSubstituters;
|
||||
|
||||
if (result == ecIncompleteClosure) ++nrIncompleteClosure;
|
||||
|
||||
if (waitees.empty() || (result == ecFailed && !settings.keepGoing)) {
|
||||
|
||||
/* If we failed and keepGoing is not set, we remove all
|
||||
remaining waitees. */
|
||||
for (auto & goal : waitees) {
|
||||
WeakGoals waiters2;
|
||||
for (auto & j : goal->waiters)
|
||||
if (j.lock() != shared_from_this()) waiters2.push_back(j);
|
||||
goal->waiters = waiters2;
|
||||
}
|
||||
waitees.clear();
|
||||
|
||||
worker.wakeUp(shared_from_this());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void Goal::amDone(ExitCode result, std::optional<Error> ex)
|
||||
{
|
||||
trace("done");
|
||||
assert(exitCode == ecBusy);
|
||||
assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure);
|
||||
exitCode = result;
|
||||
|
||||
if (ex) {
|
||||
if (!waiters.empty())
|
||||
logError(ex->info());
|
||||
else
|
||||
this->ex = std::move(*ex);
|
||||
}
|
||||
|
||||
for (auto & i : waiters) {
|
||||
GoalPtr goal = i.lock();
|
||||
if (goal) goal->waiteeDone(shared_from_this(), result);
|
||||
}
|
||||
waiters.clear();
|
||||
worker.removeGoal(shared_from_this());
|
||||
}
|
||||
|
||||
|
||||
void Goal::trace(const FormatOrString & fs)
|
||||
{
|
||||
debug("%1%: %2%", name, fs.s);
|
||||
}
|
||||
|
||||
}
|
||||
107
src/libstore/build/goal.hh
Normal file
107
src/libstore/build/goal.hh
Normal file
|
|
@ -0,0 +1,107 @@
|
|||
#pragma once
|
||||
|
||||
#include "types.hh"
|
||||
#include "store-api.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
/* Forward definition. */
|
||||
struct Goal;
|
||||
struct Worker;
|
||||
|
||||
/* A pointer to a goal. */
|
||||
typedef std::shared_ptr<Goal> GoalPtr;
|
||||
typedef std::weak_ptr<Goal> WeakGoalPtr;
|
||||
|
||||
struct CompareGoalPtrs {
|
||||
bool operator() (const GoalPtr & a, const GoalPtr & b) const;
|
||||
};
|
||||
|
||||
/* Set of goals. */
|
||||
typedef set<GoalPtr, CompareGoalPtrs> Goals;
|
||||
typedef list<WeakGoalPtr> WeakGoals;
|
||||
|
||||
/* A map of paths to goals (and the other way around). */
|
||||
typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap;
|
||||
|
||||
struct Goal : public std::enable_shared_from_this<Goal>
|
||||
{
|
||||
typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode;
|
||||
|
||||
/* Backlink to the worker. */
|
||||
Worker & worker;
|
||||
|
||||
/* Goals that this goal is waiting for. */
|
||||
Goals waitees;
|
||||
|
||||
/* Goals waiting for this one to finish. Must use weak pointers
|
||||
here to prevent cycles. */
|
||||
WeakGoals waiters;
|
||||
|
||||
/* Number of goals we are/were waiting for that have failed. */
|
||||
unsigned int nrFailed;
|
||||
|
||||
/* Number of substitution goals we are/were waiting for that
|
||||
failed because there are no substituters. */
|
||||
unsigned int nrNoSubstituters;
|
||||
|
||||
/* Number of substitution goals we are/were waiting for that
|
||||
failed because othey had unsubstitutable references. */
|
||||
unsigned int nrIncompleteClosure;
|
||||
|
||||
/* Name of this goal for debugging purposes. */
|
||||
string name;
|
||||
|
||||
/* Whether the goal is finished. */
|
||||
ExitCode exitCode;
|
||||
|
||||
/* Exception containing an error message, if any. */
|
||||
std::optional<Error> ex;
|
||||
|
||||
Goal(Worker & worker) : worker(worker)
|
||||
{
|
||||
nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
|
||||
exitCode = ecBusy;
|
||||
}
|
||||
|
||||
virtual ~Goal()
|
||||
{
|
||||
trace("goal destroyed");
|
||||
}
|
||||
|
||||
virtual void work() = 0;
|
||||
|
||||
void addWaitee(GoalPtr waitee);
|
||||
|
||||
virtual void waiteeDone(GoalPtr waitee, ExitCode result);
|
||||
|
||||
virtual void handleChildOutput(int fd, const string & data)
|
||||
{
|
||||
abort();
|
||||
}
|
||||
|
||||
virtual void handleEOF(int fd)
|
||||
{
|
||||
abort();
|
||||
}
|
||||
|
||||
void trace(const FormatOrString & fs);
|
||||
|
||||
string getName()
|
||||
{
|
||||
return name;
|
||||
}
|
||||
|
||||
/* Callback in case of a timeout. It should wake up its waiters,
|
||||
get rid of any running child processes that are being monitored
|
||||
by the worker (important!), etc. */
|
||||
virtual void timedOut(Error && ex) = 0;
|
||||
|
||||
virtual string key() = 0;
|
||||
|
||||
void amDone(ExitCode result, std::optional<Error> ex = {});
|
||||
};
|
||||
|
||||
void addToWeakGoals(WeakGoals & goals, GoalPtr p);
|
||||
|
||||
}
|
||||
72
src/libstore/build/hook-instance.cc
Normal file
72
src/libstore/build/hook-instance.cc
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
#include "globals.hh"
|
||||
#include "hook-instance.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
HookInstance::HookInstance()
|
||||
{
|
||||
debug("starting build hook '%s'", settings.buildHook);
|
||||
|
||||
/* Create a pipe to get the output of the child. */
|
||||
fromHook.create();
|
||||
|
||||
/* Create the communication pipes. */
|
||||
toHook.create();
|
||||
|
||||
/* Create a pipe to get the output of the builder. */
|
||||
builderOut.create();
|
||||
|
||||
/* Fork the hook. */
|
||||
pid = startProcess([&]() {
|
||||
|
||||
commonChildInit(fromHook);
|
||||
|
||||
if (chdir("/") == -1) throw SysError("changing into /");
|
||||
|
||||
/* Dup the communication pipes. */
|
||||
if (dup2(toHook.readSide.get(), STDIN_FILENO) == -1)
|
||||
throw SysError("dupping to-hook read side");
|
||||
|
||||
/* Use fd 4 for the builder's stdout/stderr. */
|
||||
if (dup2(builderOut.writeSide.get(), 4) == -1)
|
||||
throw SysError("dupping builder's stdout/stderr");
|
||||
|
||||
/* Hack: pass the read side of that fd to allow build-remote
|
||||
to read SSH error messages. */
|
||||
if (dup2(builderOut.readSide.get(), 5) == -1)
|
||||
throw SysError("dupping builder's stdout/stderr");
|
||||
|
||||
Strings args = {
|
||||
std::string(baseNameOf(settings.buildHook.get())),
|
||||
std::to_string(verbosity),
|
||||
};
|
||||
|
||||
execv(settings.buildHook.get().c_str(), stringsToCharPtrs(args).data());
|
||||
|
||||
throw SysError("executing '%s'", settings.buildHook);
|
||||
});
|
||||
|
||||
pid.setSeparatePG(true);
|
||||
fromHook.writeSide = -1;
|
||||
toHook.readSide = -1;
|
||||
|
||||
sink = FdSink(toHook.writeSide.get());
|
||||
std::map<std::string, Config::SettingInfo> settings;
|
||||
globalConfig.getSettings(settings);
|
||||
for (auto & setting : settings)
|
||||
sink << 1 << setting.first << setting.second.value;
|
||||
sink << 0;
|
||||
}
|
||||
|
||||
|
||||
HookInstance::~HookInstance()
|
||||
{
|
||||
try {
|
||||
toHook.writeSide = -1;
|
||||
if (pid != -1) pid.kill();
|
||||
} catch (...) {
|
||||
ignoreException();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
31
src/libstore/build/hook-instance.hh
Normal file
31
src/libstore/build/hook-instance.hh
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
#pragma once
|
||||
|
||||
#include "logging.hh"
|
||||
#include "serialise.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
struct HookInstance
|
||||
{
|
||||
/* Pipes for talking to the build hook. */
|
||||
Pipe toHook;
|
||||
|
||||
/* Pipe for the hook's standard output/error. */
|
||||
Pipe fromHook;
|
||||
|
||||
/* Pipe for the builder's standard output/error. */
|
||||
Pipe builderOut;
|
||||
|
||||
/* The process ID of the hook. */
|
||||
Pid pid;
|
||||
|
||||
FdSink sink;
|
||||
|
||||
std::map<ActivityId, Activity> activities;
|
||||
|
||||
HookInstance();
|
||||
|
||||
~HookInstance();
|
||||
};
|
||||
|
||||
}
|
||||
126
src/libstore/build/local-store-build.cc
Normal file
126
src/libstore/build/local-store-build.cc
Normal file
|
|
@ -0,0 +1,126 @@
|
|||
#include "machines.hh"
|
||||
#include "worker.hh"
|
||||
#include "substitution-goal.hh"
|
||||
#include "derivation-goal.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
static void primeCache(Store & store, const std::vector<StorePathWithOutputs> & paths)
|
||||
{
|
||||
StorePathSet willBuild, willSubstitute, unknown;
|
||||
uint64_t downloadSize, narSize;
|
||||
store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize);
|
||||
|
||||
if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty())
|
||||
throw Error(
|
||||
"%d derivations need to be built, but neither local builds ('--max-jobs') "
|
||||
"nor remote builds ('--builders') are enabled", willBuild.size());
|
||||
}
|
||||
|
||||
|
||||
void LocalStore::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, BuildMode buildMode)
|
||||
{
|
||||
Worker worker(*this);
|
||||
|
||||
primeCache(*this, drvPaths);
|
||||
|
||||
Goals goals;
|
||||
for (auto & path : drvPaths) {
|
||||
if (path.path.isDerivation())
|
||||
goals.insert(worker.makeDerivationGoal(path.path, path.outputs, buildMode));
|
||||
else
|
||||
goals.insert(worker.makeSubstitutionGoal(path.path, buildMode == bmRepair ? Repair : NoRepair));
|
||||
}
|
||||
|
||||
worker.run(goals);
|
||||
|
||||
StorePathSet failed;
|
||||
std::optional<Error> ex;
|
||||
for (auto & i : goals) {
|
||||
if (i->ex) {
|
||||
if (ex)
|
||||
logError(i->ex->info());
|
||||
else
|
||||
ex = i->ex;
|
||||
}
|
||||
if (i->exitCode != Goal::ecSuccess) {
|
||||
DerivationGoal * i2 = dynamic_cast<DerivationGoal *>(i.get());
|
||||
if (i2) failed.insert(i2->getDrvPath());
|
||||
else failed.insert(dynamic_cast<SubstitutionGoal *>(i.get())->getStorePath());
|
||||
}
|
||||
}
|
||||
|
||||
if (failed.size() == 1 && ex) {
|
||||
ex->status = worker.exitStatus();
|
||||
throw *ex;
|
||||
} else if (!failed.empty()) {
|
||||
if (ex) logError(ex->info());
|
||||
throw Error(worker.exitStatus(), "build of %s failed", showPaths(failed));
|
||||
}
|
||||
}
|
||||
|
||||
BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
|
||||
BuildMode buildMode)
|
||||
{
|
||||
Worker worker(*this);
|
||||
auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode);
|
||||
|
||||
BuildResult result;
|
||||
|
||||
try {
|
||||
worker.run(Goals{goal});
|
||||
result = goal->getResult();
|
||||
} catch (Error & e) {
|
||||
result.status = BuildResult::MiscFailure;
|
||||
result.errorMsg = e.msg();
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
void LocalStore::ensurePath(const StorePath & path)
|
||||
{
|
||||
/* If the path is already valid, we're done. */
|
||||
if (isValidPath(path)) return;
|
||||
|
||||
primeCache(*this, {{path}});
|
||||
|
||||
Worker worker(*this);
|
||||
GoalPtr goal = worker.makeSubstitutionGoal(path);
|
||||
Goals goals = {goal};
|
||||
|
||||
worker.run(goals);
|
||||
|
||||
if (goal->exitCode != Goal::ecSuccess) {
|
||||
if (goal->ex) {
|
||||
goal->ex->status = worker.exitStatus();
|
||||
throw *goal->ex;
|
||||
} else
|
||||
throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void LocalStore::repairPath(const StorePath & path)
|
||||
{
|
||||
Worker worker(*this);
|
||||
GoalPtr goal = worker.makeSubstitutionGoal(path, Repair);
|
||||
Goals goals = {goal};
|
||||
|
||||
worker.run(goals);
|
||||
|
||||
if (goal->exitCode != Goal::ecSuccess) {
|
||||
/* Since substituting the path didn't work, if we have a valid
|
||||
deriver, then rebuild the deriver. */
|
||||
auto info = queryPathInfo(path);
|
||||
if (info->deriver && isValidPath(*info->deriver)) {
|
||||
goals.clear();
|
||||
goals.insert(worker.makeDerivationGoal(*info->deriver, StringSet(), bmRepair));
|
||||
worker.run(goals);
|
||||
} else
|
||||
throw Error(worker.exitStatus(), "cannot repair path '%s'", printStorePath(path));
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
296
src/libstore/build/substitution-goal.cc
Normal file
296
src/libstore/build/substitution-goal.cc
Normal file
|
|
@ -0,0 +1,296 @@
|
|||
#include "worker.hh"
|
||||
#include "substitution-goal.hh"
|
||||
#include "nar-info.hh"
|
||||
#include "finally.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
|
||||
: Goal(worker)
|
||||
, storePath(storePath)
|
||||
, repair(repair)
|
||||
, ca(ca)
|
||||
{
|
||||
state = &SubstitutionGoal::init;
|
||||
name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath));
|
||||
trace("created");
|
||||
maintainExpectedSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.expectedSubstitutions);
|
||||
}
|
||||
|
||||
|
||||
SubstitutionGoal::~SubstitutionGoal()
|
||||
{
|
||||
try {
|
||||
if (thr.joinable()) {
|
||||
// FIXME: signal worker thread to quit.
|
||||
thr.join();
|
||||
worker.childTerminated(this);
|
||||
}
|
||||
} catch (...) {
|
||||
ignoreException();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void SubstitutionGoal::work()
|
||||
{
|
||||
(this->*state)();
|
||||
}
|
||||
|
||||
|
||||
void SubstitutionGoal::init()
|
||||
{
|
||||
trace("init");
|
||||
|
||||
worker.store.addTempRoot(storePath);
|
||||
|
||||
/* If the path already exists we're done. */
|
||||
if (!repair && worker.store.isValidPath(storePath)) {
|
||||
amDone(ecSuccess);
|
||||
return;
|
||||
}
|
||||
|
||||
if (settings.readOnlyMode)
|
||||
throw Error("cannot substitute path '%s' - no write access to the Nix store", worker.store.printStorePath(storePath));
|
||||
|
||||
subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
|
||||
|
||||
tryNext();
|
||||
}
|
||||
|
||||
|
||||
void SubstitutionGoal::tryNext()
|
||||
{
|
||||
trace("trying next substituter");
|
||||
|
||||
if (subs.size() == 0) {
|
||||
/* None left. Terminate this goal and let someone else deal
|
||||
with it. */
|
||||
debug("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath));
|
||||
|
||||
/* Hack: don't indicate failure if there were no substituters.
|
||||
In that case the calling derivation should just do a
|
||||
build. */
|
||||
amDone(substituterFailed ? ecFailed : ecNoSubstituters);
|
||||
|
||||
if (substituterFailed) {
|
||||
worker.failedSubstitutions++;
|
||||
worker.updateProgress();
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
sub = subs.front();
|
||||
subs.pop_front();
|
||||
|
||||
if (ca) {
|
||||
subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca);
|
||||
if (sub->storeDir == worker.store.storeDir)
|
||||
assert(subPath == storePath);
|
||||
} else if (sub->storeDir != worker.store.storeDir) {
|
||||
tryNext();
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
// FIXME: make async
|
||||
info = sub->queryPathInfo(subPath ? *subPath : storePath);
|
||||
} catch (InvalidPath &) {
|
||||
tryNext();
|
||||
return;
|
||||
} catch (SubstituterDisabled &) {
|
||||
if (settings.tryFallback) {
|
||||
tryNext();
|
||||
return;
|
||||
}
|
||||
throw;
|
||||
} catch (Error & e) {
|
||||
if (settings.tryFallback) {
|
||||
logError(e.info());
|
||||
tryNext();
|
||||
return;
|
||||
}
|
||||
throw;
|
||||
}
|
||||
|
||||
if (info->path != storePath) {
|
||||
if (info->isContentAddressed(*sub) && info->references.empty()) {
|
||||
auto info2 = std::make_shared<ValidPathInfo>(*info);
|
||||
info2->path = storePath;
|
||||
info = info2;
|
||||
} else {
|
||||
printError("asked '%s' for '%s' but got '%s'",
|
||||
sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path));
|
||||
tryNext();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
/* Update the total expected download size. */
|
||||
auto narInfo = std::dynamic_pointer_cast<const NarInfo>(info);
|
||||
|
||||
maintainExpectedNar = std::make_unique<MaintainCount<uint64_t>>(worker.expectedNarSize, info->narSize);
|
||||
|
||||
maintainExpectedDownload =
|
||||
narInfo && narInfo->fileSize
|
||||
? std::make_unique<MaintainCount<uint64_t>>(worker.expectedDownloadSize, narInfo->fileSize)
|
||||
: nullptr;
|
||||
|
||||
worker.updateProgress();
|
||||
|
||||
/* Bail out early if this substituter lacks a valid
|
||||
signature. LocalStore::addToStore() also checks for this, but
|
||||
only after we've downloaded the path. */
|
||||
if (worker.store.requireSigs
|
||||
&& !sub->isTrusted
|
||||
&& !info->checkSignatures(worker.store, worker.store.getPublicKeys()))
|
||||
{
|
||||
logWarning({
|
||||
.name = "Invalid path signature",
|
||||
.hint = hintfmt("substituter '%s' does not have a valid signature for path '%s'",
|
||||
sub->getUri(), worker.store.printStorePath(storePath))
|
||||
});
|
||||
tryNext();
|
||||
return;
|
||||
}
|
||||
|
||||
/* To maintain the closure invariant, we first have to realise the
|
||||
paths referenced by this one. */
|
||||
for (auto & i : info->references)
|
||||
if (i != storePath) /* ignore self-references */
|
||||
addWaitee(worker.makeSubstitutionGoal(i));
|
||||
|
||||
if (waitees.empty()) /* to prevent hang (no wake-up event) */
|
||||
referencesValid();
|
||||
else
|
||||
state = &SubstitutionGoal::referencesValid;
|
||||
}
|
||||
|
||||
|
||||
void SubstitutionGoal::referencesValid()
|
||||
{
|
||||
trace("all references realised");
|
||||
|
||||
if (nrFailed > 0) {
|
||||
debug("some references of path '%s' could not be realised", worker.store.printStorePath(storePath));
|
||||
amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed);
|
||||
return;
|
||||
}
|
||||
|
||||
for (auto & i : info->references)
|
||||
if (i != storePath) /* ignore self-references */
|
||||
assert(worker.store.isValidPath(i));
|
||||
|
||||
state = &SubstitutionGoal::tryToRun;
|
||||
worker.wakeUp(shared_from_this());
|
||||
}
|
||||
|
||||
|
||||
void SubstitutionGoal::tryToRun()
|
||||
{
|
||||
trace("trying to run");
|
||||
|
||||
/* Make sure that we are allowed to start a build. Note that even
|
||||
if maxBuildJobs == 0 (no local builds allowed), we still allow
|
||||
a substituter to run. This is because substitutions cannot be
|
||||
distributed to another machine via the build hook. */
|
||||
if (worker.getNrLocalBuilds() >= std::max(1U, (unsigned int) settings.maxBuildJobs)) {
|
||||
worker.waitForBuildSlot(shared_from_this());
|
||||
return;
|
||||
}
|
||||
|
||||
maintainRunningSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.runningSubstitutions);
|
||||
worker.updateProgress();
|
||||
|
||||
outPipe.create();
|
||||
|
||||
promise = std::promise<void>();
|
||||
|
||||
thr = std::thread([this]() {
|
||||
try {
|
||||
/* Wake up the worker loop when we're done. */
|
||||
Finally updateStats([this]() { outPipe.writeSide = -1; });
|
||||
|
||||
Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()});
|
||||
PushActivity pact(act.id);
|
||||
|
||||
copyStorePath(ref<Store>(sub), ref<Store>(worker.store.shared_from_this()),
|
||||
subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs);
|
||||
|
||||
promise.set_value();
|
||||
} catch (...) {
|
||||
promise.set_exception(std::current_exception());
|
||||
}
|
||||
});
|
||||
|
||||
worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false);
|
||||
|
||||
state = &SubstitutionGoal::finished;
|
||||
}
|
||||
|
||||
|
||||
void SubstitutionGoal::finished()
|
||||
{
|
||||
trace("substitute finished");
|
||||
|
||||
thr.join();
|
||||
worker.childTerminated(this);
|
||||
|
||||
try {
|
||||
promise.get_future().get();
|
||||
} catch (std::exception & e) {
|
||||
printError(e.what());
|
||||
|
||||
/* Cause the parent build to fail unless --fallback is given,
|
||||
or the substitute has disappeared. The latter case behaves
|
||||
the same as the substitute never having existed in the
|
||||
first place. */
|
||||
try {
|
||||
throw;
|
||||
} catch (SubstituteGone &) {
|
||||
} catch (...) {
|
||||
substituterFailed = true;
|
||||
}
|
||||
|
||||
/* Try the next substitute. */
|
||||
state = &SubstitutionGoal::tryNext;
|
||||
worker.wakeUp(shared_from_this());
|
||||
return;
|
||||
}
|
||||
|
||||
worker.markContentsGood(storePath);
|
||||
|
||||
printMsg(lvlChatty, "substitution of path '%s' succeeded", worker.store.printStorePath(storePath));
|
||||
|
||||
maintainRunningSubstitutions.reset();
|
||||
|
||||
maintainExpectedSubstitutions.reset();
|
||||
worker.doneSubstitutions++;
|
||||
|
||||
if (maintainExpectedDownload) {
|
||||
auto fileSize = maintainExpectedDownload->delta;
|
||||
maintainExpectedDownload.reset();
|
||||
worker.doneDownloadSize += fileSize;
|
||||
}
|
||||
|
||||
worker.doneNarSize += maintainExpectedNar->delta;
|
||||
maintainExpectedNar.reset();
|
||||
|
||||
worker.updateProgress();
|
||||
|
||||
amDone(ecSuccess);
|
||||
}
|
||||
|
||||
|
||||
void SubstitutionGoal::handleChildOutput(int fd, const string & data)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
void SubstitutionGoal::handleEOF(int fd)
|
||||
{
|
||||
if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this());
|
||||
}
|
||||
|
||||
}
|
||||
89
src/libstore/build/substitution-goal.hh
Normal file
89
src/libstore/build/substitution-goal.hh
Normal file
|
|
@ -0,0 +1,89 @@
|
|||
#pragma once
|
||||
|
||||
#include "lock.hh"
|
||||
#include "store-api.hh"
|
||||
#include "goal.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
class Worker;
|
||||
|
||||
class SubstitutionGoal : public Goal
|
||||
{
|
||||
friend class Worker;
|
||||
|
||||
private:
|
||||
/* The store path that should be realised through a substitute. */
|
||||
StorePath storePath;
|
||||
|
||||
/* The path the substituter refers to the path as. This will be
|
||||
* different when the stores have different names. */
|
||||
std::optional<StorePath> subPath;
|
||||
|
||||
/* The remaining substituters. */
|
||||
std::list<ref<Store>> subs;
|
||||
|
||||
/* The current substituter. */
|
||||
std::shared_ptr<Store> sub;
|
||||
|
||||
/* Whether a substituter failed. */
|
||||
bool substituterFailed = false;
|
||||
|
||||
/* Path info returned by the substituter's query info operation. */
|
||||
std::shared_ptr<const ValidPathInfo> info;
|
||||
|
||||
/* Pipe for the substituter's standard output. */
|
||||
Pipe outPipe;
|
||||
|
||||
/* The substituter thread. */
|
||||
std::thread thr;
|
||||
|
||||
std::promise<void> promise;
|
||||
|
||||
/* Whether to try to repair a valid path. */
|
||||
RepairFlag repair;
|
||||
|
||||
/* Location where we're downloading the substitute. Differs from
|
||||
storePath when doing a repair. */
|
||||
Path destPath;
|
||||
|
||||
std::unique_ptr<MaintainCount<uint64_t>> maintainExpectedSubstitutions,
|
||||
maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload;
|
||||
|
||||
typedef void (SubstitutionGoal::*GoalState)();
|
||||
GoalState state;
|
||||
|
||||
/* Content address for recomputing store path */
|
||||
std::optional<ContentAddress> ca;
|
||||
|
||||
public:
|
||||
SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
|
||||
~SubstitutionGoal();
|
||||
|
||||
void timedOut(Error && ex) override { abort(); };
|
||||
|
||||
string key() override
|
||||
{
|
||||
/* "a$" ensures substitution goals happen before derivation
|
||||
goals. */
|
||||
return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath);
|
||||
}
|
||||
|
||||
void work() override;
|
||||
|
||||
/* The states. */
|
||||
void init();
|
||||
void tryNext();
|
||||
void gotInfo();
|
||||
void referencesValid();
|
||||
void tryToRun();
|
||||
void finished();
|
||||
|
||||
/* Callback used by the worker to write to the log. */
|
||||
void handleChildOutput(int fd, const string & data) override;
|
||||
void handleEOF(int fd) override;
|
||||
|
||||
StorePath getStorePath() { return storePath; }
|
||||
};
|
||||
|
||||
}
|
||||
455
src/libstore/build/worker.cc
Normal file
455
src/libstore/build/worker.cc
Normal file
|
|
@ -0,0 +1,455 @@
|
|||
#include "machines.hh"
|
||||
#include "worker.hh"
|
||||
#include "substitution-goal.hh"
|
||||
#include "derivation-goal.hh"
|
||||
#include "hook-instance.hh"
|
||||
|
||||
#include <poll.h>
|
||||
|
||||
namespace nix {
|
||||
|
||||
Worker::Worker(LocalStore & store)
|
||||
: act(*logger, actRealise)
|
||||
, actDerivations(*logger, actBuilds)
|
||||
, actSubstitutions(*logger, actCopyPaths)
|
||||
, store(store)
|
||||
{
|
||||
/* Debugging: prevent recursive workers. */
|
||||
nrLocalBuilds = 0;
|
||||
lastWokenUp = steady_time_point::min();
|
||||
permanentFailure = false;
|
||||
timedOut = false;
|
||||
hashMismatch = false;
|
||||
checkMismatch = false;
|
||||
}
|
||||
|
||||
|
||||
Worker::~Worker()
|
||||
{
|
||||
/* Explicitly get rid of all strong pointers now. After this all
|
||||
goals that refer to this worker should be gone. (Otherwise we
|
||||
are in trouble, since goals may call childTerminated() etc. in
|
||||
their destructors). */
|
||||
topGoals.clear();
|
||||
|
||||
assert(expectedSubstitutions == 0);
|
||||
assert(expectedDownloadSize == 0);
|
||||
assert(expectedNarSize == 0);
|
||||
}
|
||||
|
||||
|
||||
std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon(
|
||||
const StorePath & drvPath,
|
||||
const StringSet & wantedOutputs,
|
||||
std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal)
|
||||
{
|
||||
WeakGoalPtr & abstract_goal_weak = derivationGoals[drvPath];
|
||||
GoalPtr abstract_goal = abstract_goal_weak.lock(); // FIXME
|
||||
std::shared_ptr<DerivationGoal> goal;
|
||||
if (!abstract_goal) {
|
||||
goal = mkDrvGoal();
|
||||
abstract_goal_weak = goal;
|
||||
wakeUp(goal);
|
||||
} else {
|
||||
goal = std::dynamic_pointer_cast<DerivationGoal>(abstract_goal);
|
||||
assert(goal);
|
||||
goal->addWantedOutputs(wantedOutputs);
|
||||
}
|
||||
return goal;
|
||||
}
|
||||
|
||||
|
||||
std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drvPath,
|
||||
const StringSet & wantedOutputs, BuildMode buildMode)
|
||||
{
|
||||
return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() {
|
||||
return std::make_shared<DerivationGoal>(drvPath, wantedOutputs, *this, buildMode);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath & drvPath,
|
||||
const BasicDerivation & drv, const StringSet & wantedOutputs, BuildMode buildMode)
|
||||
{
|
||||
return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() {
|
||||
return std::make_shared<DerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca)
|
||||
{
|
||||
WeakGoalPtr & goal_weak = substitutionGoals[path];
|
||||
GoalPtr goal = goal_weak.lock(); // FIXME
|
||||
if (!goal) {
|
||||
goal = std::make_shared<SubstitutionGoal>(path, *this, repair, ca);
|
||||
goal_weak = goal;
|
||||
wakeUp(goal);
|
||||
}
|
||||
return goal;
|
||||
}
|
||||
|
||||
|
||||
static void removeGoal(GoalPtr goal, WeakGoalMap & goalMap)
|
||||
{
|
||||
/* !!! inefficient */
|
||||
for (WeakGoalMap::iterator i = goalMap.begin();
|
||||
i != goalMap.end(); )
|
||||
if (i->second.lock() == goal) {
|
||||
WeakGoalMap::iterator j = i; ++j;
|
||||
goalMap.erase(i);
|
||||
i = j;
|
||||
}
|
||||
else ++i;
|
||||
}
|
||||
|
||||
|
||||
void Worker::removeGoal(GoalPtr goal)
|
||||
{
|
||||
nix::removeGoal(goal, derivationGoals);
|
||||
nix::removeGoal(goal, substitutionGoals);
|
||||
if (topGoals.find(goal) != topGoals.end()) {
|
||||
topGoals.erase(goal);
|
||||
/* If a top-level goal failed, then kill all other goals
|
||||
(unless keepGoing was set). */
|
||||
if (goal->exitCode == Goal::ecFailed && !settings.keepGoing)
|
||||
topGoals.clear();
|
||||
}
|
||||
|
||||
/* Wake up goals waiting for any goal to finish. */
|
||||
for (auto & i : waitingForAnyGoal) {
|
||||
GoalPtr goal = i.lock();
|
||||
if (goal) wakeUp(goal);
|
||||
}
|
||||
|
||||
waitingForAnyGoal.clear();
|
||||
}
|
||||
|
||||
|
||||
void Worker::wakeUp(GoalPtr goal)
|
||||
{
|
||||
goal->trace("woken up");
|
||||
addToWeakGoals(awake, goal);
|
||||
}
|
||||
|
||||
|
||||
unsigned Worker::getNrLocalBuilds()
|
||||
{
|
||||
return nrLocalBuilds;
|
||||
}
|
||||
|
||||
|
||||
void Worker::childStarted(GoalPtr goal, const set<int> & fds,
|
||||
bool inBuildSlot, bool respectTimeouts)
|
||||
{
|
||||
Child child;
|
||||
child.goal = goal;
|
||||
child.goal2 = goal.get();
|
||||
child.fds = fds;
|
||||
child.timeStarted = child.lastOutput = steady_time_point::clock::now();
|
||||
child.inBuildSlot = inBuildSlot;
|
||||
child.respectTimeouts = respectTimeouts;
|
||||
children.emplace_back(child);
|
||||
if (inBuildSlot) nrLocalBuilds++;
|
||||
}
|
||||
|
||||
|
||||
void Worker::childTerminated(Goal * goal, bool wakeSleepers)
|
||||
{
|
||||
auto i = std::find_if(children.begin(), children.end(),
|
||||
[&](const Child & child) { return child.goal2 == goal; });
|
||||
if (i == children.end()) return;
|
||||
|
||||
if (i->inBuildSlot) {
|
||||
assert(nrLocalBuilds > 0);
|
||||
nrLocalBuilds--;
|
||||
}
|
||||
|
||||
children.erase(i);
|
||||
|
||||
if (wakeSleepers) {
|
||||
|
||||
/* Wake up goals waiting for a build slot. */
|
||||
for (auto & j : wantingToBuild) {
|
||||
GoalPtr goal = j.lock();
|
||||
if (goal) wakeUp(goal);
|
||||
}
|
||||
|
||||
wantingToBuild.clear();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void Worker::waitForBuildSlot(GoalPtr goal)
|
||||
{
|
||||
debug("wait for build slot");
|
||||
if (getNrLocalBuilds() < settings.maxBuildJobs)
|
||||
wakeUp(goal); /* we can do it right away */
|
||||
else
|
||||
addToWeakGoals(wantingToBuild, goal);
|
||||
}
|
||||
|
||||
|
||||
void Worker::waitForAnyGoal(GoalPtr goal)
|
||||
{
|
||||
debug("wait for any goal");
|
||||
addToWeakGoals(waitingForAnyGoal, goal);
|
||||
}
|
||||
|
||||
|
||||
void Worker::waitForAWhile(GoalPtr goal)
|
||||
{
|
||||
debug("wait for a while");
|
||||
addToWeakGoals(waitingForAWhile, goal);
|
||||
}
|
||||
|
||||
|
||||
void Worker::run(const Goals & _topGoals)
|
||||
{
|
||||
for (auto & i : _topGoals) topGoals.insert(i);
|
||||
|
||||
debug("entered goal loop");
|
||||
|
||||
while (1) {
|
||||
|
||||
checkInterrupt();
|
||||
|
||||
store.autoGC(false);
|
||||
|
||||
/* Call every wake goal (in the ordering established by
|
||||
CompareGoalPtrs). */
|
||||
while (!awake.empty() && !topGoals.empty()) {
|
||||
Goals awake2;
|
||||
for (auto & i : awake) {
|
||||
GoalPtr goal = i.lock();
|
||||
if (goal) awake2.insert(goal);
|
||||
}
|
||||
awake.clear();
|
||||
for (auto & goal : awake2) {
|
||||
checkInterrupt();
|
||||
goal->work();
|
||||
if (topGoals.empty()) break; // stuff may have been cancelled
|
||||
}
|
||||
}
|
||||
|
||||
if (topGoals.empty()) break;
|
||||
|
||||
/* Wait for input. */
|
||||
if (!children.empty() || !waitingForAWhile.empty())
|
||||
waitForInput();
|
||||
else {
|
||||
if (awake.empty() && 0 == settings.maxBuildJobs)
|
||||
{
|
||||
if (getMachines().empty())
|
||||
throw Error("unable to start any build; either increase '--max-jobs' "
|
||||
"or enable remote builds."
|
||||
"\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
|
||||
else
|
||||
throw Error("unable to start any build; remote machines may not have "
|
||||
"all required system features."
|
||||
"\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
|
||||
|
||||
}
|
||||
assert(!awake.empty());
|
||||
}
|
||||
}
|
||||
|
||||
/* If --keep-going is not set, it's possible that the main goal
|
||||
exited while some of its subgoals were still active. But if
|
||||
--keep-going *is* set, then they must all be finished now. */
|
||||
assert(!settings.keepGoing || awake.empty());
|
||||
assert(!settings.keepGoing || wantingToBuild.empty());
|
||||
assert(!settings.keepGoing || children.empty());
|
||||
}
|
||||
|
||||
void Worker::waitForInput()
|
||||
{
|
||||
printMsg(lvlVomit, "waiting for children");
|
||||
|
||||
/* Process output from the file descriptors attached to the
|
||||
children, namely log output and output path creation commands.
|
||||
We also use this to detect child termination: if we get EOF on
|
||||
the logger pipe of a build, we assume that the builder has
|
||||
terminated. */
|
||||
|
||||
bool useTimeout = false;
|
||||
long timeout = 0;
|
||||
auto before = steady_time_point::clock::now();
|
||||
|
||||
/* If we're monitoring for silence on stdout/stderr, or if there
|
||||
is a build timeout, then wait for input until the first
|
||||
deadline for any child. */
|
||||
auto nearest = steady_time_point::max(); // nearest deadline
|
||||
if (settings.minFree.get() != 0)
|
||||
// Periodicallty wake up to see if we need to run the garbage collector.
|
||||
nearest = before + std::chrono::seconds(10);
|
||||
for (auto & i : children) {
|
||||
if (!i.respectTimeouts) continue;
|
||||
if (0 != settings.maxSilentTime)
|
||||
nearest = std::min(nearest, i.lastOutput + std::chrono::seconds(settings.maxSilentTime));
|
||||
if (0 != settings.buildTimeout)
|
||||
nearest = std::min(nearest, i.timeStarted + std::chrono::seconds(settings.buildTimeout));
|
||||
}
|
||||
if (nearest != steady_time_point::max()) {
|
||||
timeout = std::max(1L, (long) std::chrono::duration_cast<std::chrono::seconds>(nearest - before).count());
|
||||
useTimeout = true;
|
||||
}
|
||||
|
||||
/* If we are polling goals that are waiting for a lock, then wake
|
||||
up after a few seconds at most. */
|
||||
if (!waitingForAWhile.empty()) {
|
||||
useTimeout = true;
|
||||
if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) lastWokenUp = before;
|
||||
timeout = std::max(1L,
|
||||
(long) std::chrono::duration_cast<std::chrono::seconds>(
|
||||
lastWokenUp + std::chrono::seconds(settings.pollInterval) - before).count());
|
||||
} else lastWokenUp = steady_time_point::min();
|
||||
|
||||
if (useTimeout)
|
||||
vomit("sleeping %d seconds", timeout);
|
||||
|
||||
/* Use select() to wait for the input side of any logger pipe to
|
||||
become `available'. Note that `available' (i.e., non-blocking)
|
||||
includes EOF. */
|
||||
std::vector<struct pollfd> pollStatus;
|
||||
std::map <int, int> fdToPollStatus;
|
||||
for (auto & i : children) {
|
||||
for (auto & j : i.fds) {
|
||||
pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN });
|
||||
fdToPollStatus[j] = pollStatus.size() - 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (poll(pollStatus.data(), pollStatus.size(),
|
||||
useTimeout ? timeout * 1000 : -1) == -1) {
|
||||
if (errno == EINTR) return;
|
||||
throw SysError("waiting for input");
|
||||
}
|
||||
|
||||
auto after = steady_time_point::clock::now();
|
||||
|
||||
/* Process all available file descriptors. FIXME: this is
|
||||
O(children * fds). */
|
||||
decltype(children)::iterator i;
|
||||
for (auto j = children.begin(); j != children.end(); j = i) {
|
||||
i = std::next(j);
|
||||
|
||||
checkInterrupt();
|
||||
|
||||
GoalPtr goal = j->goal.lock();
|
||||
assert(goal);
|
||||
|
||||
set<int> fds2(j->fds);
|
||||
std::vector<unsigned char> buffer(4096);
|
||||
for (auto & k : fds2) {
|
||||
if (pollStatus.at(fdToPollStatus.at(k)).revents) {
|
||||
ssize_t rd = ::read(k, buffer.data(), buffer.size());
|
||||
// FIXME: is there a cleaner way to handle pt close
|
||||
// than EIO? Is this even standard?
|
||||
if (rd == 0 || (rd == -1 && errno == EIO)) {
|
||||
debug("%1%: got EOF", goal->getName());
|
||||
goal->handleEOF(k);
|
||||
j->fds.erase(k);
|
||||
} else if (rd == -1) {
|
||||
if (errno != EINTR)
|
||||
throw SysError("%s: read failed", goal->getName());
|
||||
} else {
|
||||
printMsg(lvlVomit, "%1%: read %2% bytes",
|
||||
goal->getName(), rd);
|
||||
string data((char *) buffer.data(), rd);
|
||||
j->lastOutput = after;
|
||||
goal->handleChildOutput(k, data);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (goal->exitCode == Goal::ecBusy &&
|
||||
0 != settings.maxSilentTime &&
|
||||
j->respectTimeouts &&
|
||||
after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime))
|
||||
{
|
||||
goal->timedOut(Error(
|
||||
"%1% timed out after %2% seconds of silence",
|
||||
goal->getName(), settings.maxSilentTime));
|
||||
}
|
||||
|
||||
else if (goal->exitCode == Goal::ecBusy &&
|
||||
0 != settings.buildTimeout &&
|
||||
j->respectTimeouts &&
|
||||
after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout))
|
||||
{
|
||||
goal->timedOut(Error(
|
||||
"%1% timed out after %2% seconds",
|
||||
goal->getName(), settings.buildTimeout));
|
||||
}
|
||||
}
|
||||
|
||||
if (!waitingForAWhile.empty() && lastWokenUp + std::chrono::seconds(settings.pollInterval) <= after) {
|
||||
lastWokenUp = after;
|
||||
for (auto & i : waitingForAWhile) {
|
||||
GoalPtr goal = i.lock();
|
||||
if (goal) wakeUp(goal);
|
||||
}
|
||||
waitingForAWhile.clear();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
unsigned int Worker::exitStatus()
|
||||
{
|
||||
/*
|
||||
* 1100100
|
||||
* ^^^^
|
||||
* |||`- timeout
|
||||
* ||`-- output hash mismatch
|
||||
* |`--- build failure
|
||||
* `---- not deterministic
|
||||
*/
|
||||
unsigned int mask = 0;
|
||||
bool buildFailure = permanentFailure || timedOut || hashMismatch;
|
||||
if (buildFailure)
|
||||
mask |= 0x04; // 100
|
||||
if (timedOut)
|
||||
mask |= 0x01; // 101
|
||||
if (hashMismatch)
|
||||
mask |= 0x02; // 102
|
||||
if (checkMismatch) {
|
||||
mask |= 0x08; // 104
|
||||
}
|
||||
|
||||
if (mask)
|
||||
mask |= 0x60;
|
||||
return mask ? mask : 1;
|
||||
}
|
||||
|
||||
|
||||
bool Worker::pathContentsGood(const StorePath & path)
|
||||
{
|
||||
auto i = pathContentsGoodCache.find(path);
|
||||
if (i != pathContentsGoodCache.end()) return i->second;
|
||||
printInfo("checking path '%s'...", store.printStorePath(path));
|
||||
auto info = store.queryPathInfo(path);
|
||||
bool res;
|
||||
if (!pathExists(store.printStorePath(path)))
|
||||
res = false;
|
||||
else {
|
||||
HashResult current = hashPath(info->narHash.type, store.printStorePath(path));
|
||||
Hash nullHash(htSHA256);
|
||||
res = info->narHash == nullHash || info->narHash == current.first;
|
||||
}
|
||||
pathContentsGoodCache.insert_or_assign(path, res);
|
||||
if (!res)
|
||||
logError({
|
||||
.name = "Corrupted path",
|
||||
.hint = hintfmt("path '%s' is corrupted or missing!", store.printStorePath(path))
|
||||
});
|
||||
return res;
|
||||
}
|
||||
|
||||
|
||||
void Worker::markContentsGood(const StorePath & path)
|
||||
{
|
||||
pathContentsGoodCache.insert_or_assign(path, true);
|
||||
}
|
||||
|
||||
}
|
||||
195
src/libstore/build/worker.hh
Normal file
195
src/libstore/build/worker.hh
Normal file
|
|
@ -0,0 +1,195 @@
|
|||
#pragma once
|
||||
|
||||
#include "types.hh"
|
||||
#include "lock.hh"
|
||||
#include "local-store.hh"
|
||||
#include "goal.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
/* Forward definition. */
|
||||
class DerivationGoal;
|
||||
|
||||
typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
|
||||
|
||||
|
||||
/* A mapping used to remember for each child process to what goal it
|
||||
belongs, and file descriptors for receiving log data and output
|
||||
path creation commands. */
|
||||
struct Child
|
||||
{
|
||||
WeakGoalPtr goal;
|
||||
Goal * goal2; // ugly hackery
|
||||
set<int> fds;
|
||||
bool respectTimeouts;
|
||||
bool inBuildSlot;
|
||||
steady_time_point lastOutput; /* time we last got output on stdout/stderr */
|
||||
steady_time_point timeStarted;
|
||||
};
|
||||
|
||||
/* Forward definition. */
|
||||
struct HookInstance;
|
||||
|
||||
/* The worker class. */
|
||||
class Worker
|
||||
{
|
||||
private:
|
||||
|
||||
/* Note: the worker should only have strong pointers to the
|
||||
top-level goals. */
|
||||
|
||||
/* The top-level goals of the worker. */
|
||||
Goals topGoals;
|
||||
|
||||
/* Goals that are ready to do some work. */
|
||||
WeakGoals awake;
|
||||
|
||||
/* Goals waiting for a build slot. */
|
||||
WeakGoals wantingToBuild;
|
||||
|
||||
/* Child processes currently running. */
|
||||
std::list<Child> children;
|
||||
|
||||
/* Number of build slots occupied. This includes local builds and
|
||||
substitutions but not remote builds via the build hook. */
|
||||
unsigned int nrLocalBuilds;
|
||||
|
||||
/* Maps used to prevent multiple instantiations of a goal for the
|
||||
same derivation / path. */
|
||||
WeakGoalMap derivationGoals;
|
||||
WeakGoalMap substitutionGoals;
|
||||
|
||||
/* Goals waiting for busy paths to be unlocked. */
|
||||
WeakGoals waitingForAnyGoal;
|
||||
|
||||
/* Goals sleeping for a few seconds (polling a lock). */
|
||||
WeakGoals waitingForAWhile;
|
||||
|
||||
/* Last time the goals in `waitingForAWhile' where woken up. */
|
||||
steady_time_point lastWokenUp;
|
||||
|
||||
/* Cache for pathContentsGood(). */
|
||||
std::map<StorePath, bool> pathContentsGoodCache;
|
||||
|
||||
public:
|
||||
|
||||
const Activity act;
|
||||
const Activity actDerivations;
|
||||
const Activity actSubstitutions;
|
||||
|
||||
/* Set if at least one derivation had a BuildError (i.e. permanent
|
||||
failure). */
|
||||
bool permanentFailure;
|
||||
|
||||
/* Set if at least one derivation had a timeout. */
|
||||
bool timedOut;
|
||||
|
||||
/* Set if at least one derivation fails with a hash mismatch. */
|
||||
bool hashMismatch;
|
||||
|
||||
/* Set if at least one derivation is not deterministic in check mode. */
|
||||
bool checkMismatch;
|
||||
|
||||
LocalStore & store;
|
||||
|
||||
std::unique_ptr<HookInstance> hook;
|
||||
|
||||
uint64_t expectedBuilds = 0;
|
||||
uint64_t doneBuilds = 0;
|
||||
uint64_t failedBuilds = 0;
|
||||
uint64_t runningBuilds = 0;
|
||||
|
||||
uint64_t expectedSubstitutions = 0;
|
||||
uint64_t doneSubstitutions = 0;
|
||||
uint64_t failedSubstitutions = 0;
|
||||
uint64_t runningSubstitutions = 0;
|
||||
uint64_t expectedDownloadSize = 0;
|
||||
uint64_t doneDownloadSize = 0;
|
||||
uint64_t expectedNarSize = 0;
|
||||
uint64_t doneNarSize = 0;
|
||||
|
||||
/* Whether to ask the build hook if it can build a derivation. If
|
||||
it answers with "decline-permanently", we don't try again. */
|
||||
bool tryBuildHook = true;
|
||||
|
||||
Worker(LocalStore & store);
|
||||
~Worker();
|
||||
|
||||
/* Make a goal (with caching). */
|
||||
|
||||
/* derivation goal */
|
||||
private:
|
||||
std::shared_ptr<DerivationGoal> makeDerivationGoalCommon(
|
||||
const StorePath & drvPath, const StringSet & wantedOutputs,
|
||||
std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal);
|
||||
public:
|
||||
std::shared_ptr<DerivationGoal> makeDerivationGoal(
|
||||
const StorePath & drvPath,
|
||||
const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
|
||||
std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(
|
||||
const StorePath & drvPath, const BasicDerivation & drv,
|
||||
const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
|
||||
|
||||
/* substitution goal */
|
||||
GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
|
||||
|
||||
/* Remove a dead goal. */
|
||||
void removeGoal(GoalPtr goal);
|
||||
|
||||
/* Wake up a goal (i.e., there is something for it to do). */
|
||||
void wakeUp(GoalPtr goal);
|
||||
|
||||
/* Return the number of local build and substitution processes
|
||||
currently running (but not remote builds via the build
|
||||
hook). */
|
||||
unsigned int getNrLocalBuilds();
|
||||
|
||||
/* Registers a running child process. `inBuildSlot' means that
|
||||
the process counts towards the jobs limit. */
|
||||
void childStarted(GoalPtr goal, const set<int> & fds,
|
||||
bool inBuildSlot, bool respectTimeouts);
|
||||
|
||||
/* Unregisters a running child process. `wakeSleepers' should be
|
||||
false if there is no sense in waking up goals that are sleeping
|
||||
because they can't run yet (e.g., there is no free build slot,
|
||||
or the hook would still say `postpone'). */
|
||||
void childTerminated(Goal * goal, bool wakeSleepers = true);
|
||||
|
||||
/* Put `goal' to sleep until a build slot becomes available (which
|
||||
might be right away). */
|
||||
void waitForBuildSlot(GoalPtr goal);
|
||||
|
||||
/* Wait for any goal to finish. Pretty indiscriminate way to
|
||||
wait for some resource that some other goal is holding. */
|
||||
void waitForAnyGoal(GoalPtr goal);
|
||||
|
||||
/* Wait for a few seconds and then retry this goal. Used when
|
||||
waiting for a lock held by another process. This kind of
|
||||
polling is inefficient, but POSIX doesn't really provide a way
|
||||
to wait for multiple locks in the main select() loop. */
|
||||
void waitForAWhile(GoalPtr goal);
|
||||
|
||||
/* Loop until the specified top-level goals have finished. */
|
||||
void run(const Goals & topGoals);
|
||||
|
||||
/* Wait for input to become available. */
|
||||
void waitForInput();
|
||||
|
||||
unsigned int exitStatus();
|
||||
|
||||
/* Check whether the given valid path exists and has the right
|
||||
contents. */
|
||||
bool pathContentsGood(const StorePath & path);
|
||||
|
||||
void markContentsGood(const StorePath & path);
|
||||
|
||||
void updateProgress()
|
||||
{
|
||||
actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds);
|
||||
actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions);
|
||||
act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize);
|
||||
act.setExpected(actCopyPath, expectedNarSize + doneNarSize);
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
#include "derivations.hh"
|
||||
#include "globals.hh"
|
||||
#include "local-store.hh"
|
||||
#include "local-fs-store.hh"
|
||||
#include "finally.hh"
|
||||
|
||||
#include <functional>
|
||||
|
|
@ -682,7 +683,7 @@ void LocalStore::removeUnusedLinks(const GCState & state)
|
|||
struct stat st;
|
||||
if (stat(linksDir.c_str(), &st) == -1)
|
||||
throw SysError("statting '%1%'", linksDir);
|
||||
auto overhead = st.st_blocks * 512ULL;
|
||||
int64_t overhead = st.st_blocks * 512ULL;
|
||||
|
||||
printInfo("note: currently hard linking saves %.2f MiB",
|
||||
((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0)));
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
#include "archive.hh"
|
||||
#include "fs-accessor.hh"
|
||||
#include "store-api.hh"
|
||||
#include "local-fs-store.hh"
|
||||
#include "globals.hh"
|
||||
#include "compression.hh"
|
||||
#include "derivations.hh"
|
||||
|
|
|
|||
48
src/libstore/local-fs-store.hh
Normal file
48
src/libstore/local-fs-store.hh
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
#pragma once
|
||||
|
||||
#include "store-api.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
struct LocalFSStoreConfig : virtual StoreConfig
|
||||
{
|
||||
using StoreConfig::StoreConfig;
|
||||
// FIXME: the (StoreConfig*) cast works around a bug in gcc that causes
|
||||
// it to omit the call to the Setting constructor. Clang works fine
|
||||
// either way.
|
||||
const PathSetting rootDir{(StoreConfig*) this, true, "",
|
||||
"root", "directory prefixed to all other paths"};
|
||||
const PathSetting stateDir{(StoreConfig*) this, false,
|
||||
rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir,
|
||||
"state", "directory where Nix will store state"};
|
||||
const PathSetting logDir{(StoreConfig*) this, false,
|
||||
rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir,
|
||||
"log", "directory where Nix will store state"};
|
||||
};
|
||||
|
||||
class LocalFSStore : public virtual Store, public virtual LocalFSStoreConfig
|
||||
{
|
||||
public:
|
||||
|
||||
const static string drvsLogDir;
|
||||
|
||||
LocalFSStore(const Params & params);
|
||||
|
||||
void narFromPath(const StorePath & path, Sink & sink) override;
|
||||
ref<FSAccessor> getFSAccessor() override;
|
||||
|
||||
/* Register a permanent GC root. */
|
||||
Path addPermRoot(const StorePath & storePath, const Path & gcRoot);
|
||||
|
||||
virtual Path getRealStoreDir() { return storeDir; }
|
||||
|
||||
Path toRealPath(const Path & storePath) override
|
||||
{
|
||||
assert(isInStore(storePath));
|
||||
return getRealStoreDir() + "/" + std::string(storePath, storeDir.size() + 1);
|
||||
}
|
||||
|
||||
std::shared_ptr<std::string> getBuildLog(const StorePath & path) override;
|
||||
};
|
||||
|
||||
}
|
||||
|
|
@ -4,6 +4,7 @@
|
|||
|
||||
#include "pathlocks.hh"
|
||||
#include "store-api.hh"
|
||||
#include "local-fs-store.hh"
|
||||
#include "sync.hh"
|
||||
#include "util.hh"
|
||||
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ libstore_NAME = libnixstore
|
|||
|
||||
libstore_DIR := $(d)
|
||||
|
||||
libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc)
|
||||
libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc $(d)/build/*.cc)
|
||||
|
||||
libstore_LIBS = libutil
|
||||
|
||||
|
|
@ -32,7 +32,7 @@ ifeq ($(HAVE_SECCOMP), 1)
|
|||
endif
|
||||
|
||||
libstore_CXXFLAGS += \
|
||||
-I src/libutil -I src/libstore \
|
||||
-I src/libutil -I src/libstore -I src/libstore/build \
|
||||
-DNIX_PREFIX=\"$(prefix)\" \
|
||||
-DNIX_STORE_DIR=\"$(storedir)\" \
|
||||
-DNIX_DATA_DIR=\"$(datadir)\" \
|
||||
|
|
@ -64,3 +64,6 @@ $(eval $(call install-file-in, $(d)/nix-store.pc, $(prefix)/lib/pkgconfig, 0644)
|
|||
|
||||
$(foreach i, $(wildcard src/libstore/builtins/*.hh), \
|
||||
$(eval $(call install-file-in, $(i), $(includedir)/nix/builtins, 0644)))
|
||||
|
||||
$(foreach i, $(wildcard src/libstore/build/*.hh), \
|
||||
$(eval $(call install-file-in, $(i), $(includedir)/nix/build, 0644)))
|
||||
|
|
|
|||
93
src/libstore/lock.cc
Normal file
93
src/libstore/lock.cc
Normal file
|
|
@ -0,0 +1,93 @@
|
|||
#include "lock.hh"
|
||||
#include "globals.hh"
|
||||
#include "pathlocks.hh"
|
||||
|
||||
#include <grp.h>
|
||||
#include <pwd.h>
|
||||
|
||||
#include <fcntl.h>
|
||||
#include <unistd.h>
|
||||
|
||||
namespace nix {
|
||||
|
||||
UserLock::UserLock()
|
||||
{
|
||||
assert(settings.buildUsersGroup != "");
|
||||
createDirs(settings.nixStateDir + "/userpool");
|
||||
}
|
||||
|
||||
bool UserLock::findFreeUser() {
|
||||
if (enabled()) return true;
|
||||
|
||||
/* Get the members of the build-users-group. */
|
||||
struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str());
|
||||
if (!gr)
|
||||
throw Error("the group '%1%' specified in 'build-users-group' does not exist",
|
||||
settings.buildUsersGroup);
|
||||
gid = gr->gr_gid;
|
||||
|
||||
/* Copy the result of getgrnam. */
|
||||
Strings users;
|
||||
for (char * * p = gr->gr_mem; *p; ++p) {
|
||||
debug("found build user '%1%'", *p);
|
||||
users.push_back(*p);
|
||||
}
|
||||
|
||||
if (users.empty())
|
||||
throw Error("the build users group '%1%' has no members",
|
||||
settings.buildUsersGroup);
|
||||
|
||||
/* Find a user account that isn't currently in use for another
|
||||
build. */
|
||||
for (auto & i : users) {
|
||||
debug("trying user '%1%'", i);
|
||||
|
||||
struct passwd * pw = getpwnam(i.c_str());
|
||||
if (!pw)
|
||||
throw Error("the user '%1%' in the group '%2%' does not exist",
|
||||
i, settings.buildUsersGroup);
|
||||
|
||||
|
||||
fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str();
|
||||
|
||||
AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
|
||||
if (!fd)
|
||||
throw SysError("opening user lock '%1%'", fnUserLock);
|
||||
|
||||
if (lockFile(fd.get(), ltWrite, false)) {
|
||||
fdUserLock = std::move(fd);
|
||||
user = i;
|
||||
uid = pw->pw_uid;
|
||||
|
||||
/* Sanity check... */
|
||||
if (uid == getuid() || uid == geteuid())
|
||||
throw Error("the Nix user should not be a member of '%1%'",
|
||||
settings.buildUsersGroup);
|
||||
|
||||
#if __linux__
|
||||
/* Get the list of supplementary groups of this build user. This
|
||||
is usually either empty or contains a group such as "kvm". */
|
||||
supplementaryGIDs.resize(10);
|
||||
int ngroups = supplementaryGIDs.size();
|
||||
int err = getgrouplist(pw->pw_name, pw->pw_gid,
|
||||
supplementaryGIDs.data(), &ngroups);
|
||||
if (err == -1)
|
||||
throw Error("failed to get list of supplementary groups for '%1%'", pw->pw_name);
|
||||
|
||||
supplementaryGIDs.resize(ngroups);
|
||||
#endif
|
||||
|
||||
isEnabled = true;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
void UserLock::kill()
|
||||
{
|
||||
killUser(uid);
|
||||
}
|
||||
|
||||
}
|
||||
37
src/libstore/lock.hh
Normal file
37
src/libstore/lock.hh
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
#pragma once
|
||||
|
||||
#include "sync.hh"
|
||||
#include "types.hh"
|
||||
#include "util.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
class UserLock
|
||||
{
|
||||
private:
|
||||
Path fnUserLock;
|
||||
AutoCloseFD fdUserLock;
|
||||
|
||||
bool isEnabled = false;
|
||||
string user;
|
||||
uid_t uid = 0;
|
||||
gid_t gid = 0;
|
||||
std::vector<gid_t> supplementaryGIDs;
|
||||
|
||||
public:
|
||||
UserLock();
|
||||
|
||||
void kill();
|
||||
|
||||
string getUser() { return user; }
|
||||
uid_t getUID() { assert(uid); return uid; }
|
||||
uid_t getGID() { assert(gid); return gid; }
|
||||
std::vector<gid_t> getSupplementaryGIDs() { return supplementaryGIDs; }
|
||||
|
||||
bool findFreeUser();
|
||||
|
||||
bool enabled() { return isEnabled; }
|
||||
|
||||
};
|
||||
|
||||
}
|
||||
|
|
@ -1,5 +1,6 @@
|
|||
#include "profiles.hh"
|
||||
#include "store-api.hh"
|
||||
#include "local-fs-store.hh"
|
||||
#include "util.hh"
|
||||
|
||||
#include <sys/types.h>
|
||||
|
|
|
|||
|
|
@ -12,16 +12,6 @@
|
|||
#include "logging.hh"
|
||||
#include "callback.hh"
|
||||
|
||||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/socket.h>
|
||||
#include <sys/un.h>
|
||||
#include <errno.h>
|
||||
#include <fcntl.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include <cstring>
|
||||
|
||||
namespace nix {
|
||||
|
||||
namespace worker_proto {
|
||||
|
|
@ -125,69 +115,6 @@ ref<RemoteStore::Connection> RemoteStore::openConnectionWrapper()
|
|||
}
|
||||
|
||||
|
||||
UDSRemoteStore::UDSRemoteStore(const Params & params)
|
||||
: StoreConfig(params)
|
||||
, Store(params)
|
||||
, LocalFSStore(params)
|
||||
, RemoteStore(params)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
UDSRemoteStore::UDSRemoteStore(
|
||||
const std::string scheme,
|
||||
std::string socket_path,
|
||||
const Params & params)
|
||||
: UDSRemoteStore(params)
|
||||
{
|
||||
path.emplace(socket_path);
|
||||
}
|
||||
|
||||
|
||||
std::string UDSRemoteStore::getUri()
|
||||
{
|
||||
if (path) {
|
||||
return std::string("unix://") + *path;
|
||||
} else {
|
||||
return "daemon";
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
ref<RemoteStore::Connection> UDSRemoteStore::openConnection()
|
||||
{
|
||||
auto conn = make_ref<Connection>();
|
||||
|
||||
/* Connect to a daemon that does the privileged work for us. */
|
||||
conn->fd = socket(PF_UNIX, SOCK_STREAM
|
||||
#ifdef SOCK_CLOEXEC
|
||||
| SOCK_CLOEXEC
|
||||
#endif
|
||||
, 0);
|
||||
if (!conn->fd)
|
||||
throw SysError("cannot create Unix domain socket");
|
||||
closeOnExec(conn->fd.get());
|
||||
|
||||
string socketPath = path ? *path : settings.nixDaemonSocketFile;
|
||||
|
||||
struct sockaddr_un addr;
|
||||
addr.sun_family = AF_UNIX;
|
||||
if (socketPath.size() + 1 >= sizeof(addr.sun_path))
|
||||
throw Error("socket path '%1%' is too long", socketPath);
|
||||
strcpy(addr.sun_path, socketPath.c_str());
|
||||
|
||||
if (::connect(conn->fd.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1)
|
||||
throw SysError("cannot connect to daemon at '%1%'", socketPath);
|
||||
|
||||
conn->from.fd = conn->fd.get();
|
||||
conn->to.fd = conn->fd.get();
|
||||
|
||||
conn->startTime = std::chrono::steady_clock::now();
|
||||
|
||||
return conn;
|
||||
}
|
||||
|
||||
|
||||
void RemoteStore::initConnection(Connection & conn)
|
||||
{
|
||||
/* Send the magic greeting, check for the reply. */
|
||||
|
|
@ -1012,6 +939,4 @@ void ConnectionHandle::withFramedSink(std::function<void(Sink &sink)> fun)
|
|||
|
||||
}
|
||||
|
||||
static RegisterStoreImplementation<UDSRemoteStore, UDSRemoteStoreConfig> regUDSRemoteStore;
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -155,49 +155,5 @@ private:
|
|||
|
||||
};
|
||||
|
||||
struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreConfig
|
||||
{
|
||||
UDSRemoteStoreConfig(const Store::Params & params)
|
||||
: StoreConfig(params)
|
||||
, LocalFSStoreConfig(params)
|
||||
, RemoteStoreConfig(params)
|
||||
{
|
||||
}
|
||||
|
||||
UDSRemoteStoreConfig()
|
||||
: UDSRemoteStoreConfig(Store::Params({}))
|
||||
{
|
||||
}
|
||||
|
||||
const std::string name() override { return "Local Daemon Store"; }
|
||||
};
|
||||
|
||||
class UDSRemoteStore : public LocalFSStore, public RemoteStore, public virtual UDSRemoteStoreConfig
|
||||
{
|
||||
public:
|
||||
|
||||
UDSRemoteStore(const Params & params);
|
||||
UDSRemoteStore(const std::string scheme, std::string path, const Params & params);
|
||||
|
||||
std::string getUri() override;
|
||||
|
||||
static std::set<std::string> uriSchemes()
|
||||
{ return {"unix"}; }
|
||||
|
||||
bool sameMachine() override
|
||||
{ return true; }
|
||||
|
||||
ref<FSAccessor> getFSAccessor() override
|
||||
{ return LocalFSStore::getFSAccessor(); }
|
||||
|
||||
void narFromPath(const StorePath & path, Sink & sink) override
|
||||
{ LocalFSStore::narFromPath(path, sink); }
|
||||
|
||||
private:
|
||||
|
||||
ref<RemoteStore::Connection> openConnection() override;
|
||||
std::optional<std::string> path;
|
||||
};
|
||||
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1011,7 +1011,7 @@ Derivation Store::readDerivation(const StorePath & drvPath)
|
|||
|
||||
|
||||
#include "local-store.hh"
|
||||
#include "remote-store.hh"
|
||||
#include "uds-remote-store.hh"
|
||||
|
||||
|
||||
namespace nix {
|
||||
|
|
|
|||
|
|
@ -715,47 +715,6 @@ protected:
|
|||
|
||||
};
|
||||
|
||||
struct LocalFSStoreConfig : virtual StoreConfig
|
||||
{
|
||||
using StoreConfig::StoreConfig;
|
||||
// FIXME: the (StoreConfig*) cast works around a bug in gcc that causes
|
||||
// it to omit the call to the Setting constructor. Clang works fine
|
||||
// either way.
|
||||
const PathSetting rootDir{(StoreConfig*) this, true, "",
|
||||
"root", "directory prefixed to all other paths"};
|
||||
const PathSetting stateDir{(StoreConfig*) this, false,
|
||||
rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir,
|
||||
"state", "directory where Nix will store state"};
|
||||
const PathSetting logDir{(StoreConfig*) this, false,
|
||||
rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir,
|
||||
"log", "directory where Nix will store state"};
|
||||
};
|
||||
|
||||
class LocalFSStore : public virtual Store, public virtual LocalFSStoreConfig
|
||||
{
|
||||
public:
|
||||
|
||||
const static string drvsLogDir;
|
||||
|
||||
LocalFSStore(const Params & params);
|
||||
|
||||
void narFromPath(const StorePath & path, Sink & sink) override;
|
||||
ref<FSAccessor> getFSAccessor() override;
|
||||
|
||||
/* Register a permanent GC root. */
|
||||
Path addPermRoot(const StorePath & storePath, const Path & gcRoot);
|
||||
|
||||
virtual Path getRealStoreDir() { return storeDir; }
|
||||
|
||||
Path toRealPath(const Path & storePath) override
|
||||
{
|
||||
assert(isInStore(storePath));
|
||||
return getRealStoreDir() + "/" + std::string(storePath, storeDir.size() + 1);
|
||||
}
|
||||
|
||||
std::shared_ptr<std::string> getBuildLog(const StorePath & path) override;
|
||||
};
|
||||
|
||||
|
||||
/* Copy a path from one store to another. */
|
||||
void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
|
||||
|
|
|
|||
81
src/libstore/uds-remote-store.cc
Normal file
81
src/libstore/uds-remote-store.cc
Normal file
|
|
@ -0,0 +1,81 @@
|
|||
#include "uds-remote-store.hh"
|
||||
|
||||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/socket.h>
|
||||
#include <sys/un.h>
|
||||
#include <errno.h>
|
||||
#include <fcntl.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include <cstring>
|
||||
|
||||
|
||||
namespace nix {
|
||||
|
||||
UDSRemoteStore::UDSRemoteStore(const Params & params)
|
||||
: StoreConfig(params)
|
||||
, Store(params)
|
||||
, LocalFSStore(params)
|
||||
, RemoteStore(params)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
UDSRemoteStore::UDSRemoteStore(
|
||||
const std::string scheme,
|
||||
std::string socket_path,
|
||||
const Params & params)
|
||||
: UDSRemoteStore(params)
|
||||
{
|
||||
path.emplace(socket_path);
|
||||
}
|
||||
|
||||
|
||||
std::string UDSRemoteStore::getUri()
|
||||
{
|
||||
if (path) {
|
||||
return std::string("unix://") + *path;
|
||||
} else {
|
||||
return "daemon";
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
ref<RemoteStore::Connection> UDSRemoteStore::openConnection()
|
||||
{
|
||||
auto conn = make_ref<Connection>();
|
||||
|
||||
/* Connect to a daemon that does the privileged work for us. */
|
||||
conn->fd = socket(PF_UNIX, SOCK_STREAM
|
||||
#ifdef SOCK_CLOEXEC
|
||||
| SOCK_CLOEXEC
|
||||
#endif
|
||||
, 0);
|
||||
if (!conn->fd)
|
||||
throw SysError("cannot create Unix domain socket");
|
||||
closeOnExec(conn->fd.get());
|
||||
|
||||
string socketPath = path ? *path : settings.nixDaemonSocketFile;
|
||||
|
||||
struct sockaddr_un addr;
|
||||
addr.sun_family = AF_UNIX;
|
||||
if (socketPath.size() + 1 >= sizeof(addr.sun_path))
|
||||
throw Error("socket path '%1%' is too long", socketPath);
|
||||
strcpy(addr.sun_path, socketPath.c_str());
|
||||
|
||||
if (::connect(conn->fd.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1)
|
||||
throw SysError("cannot connect to daemon at '%1%'", socketPath);
|
||||
|
||||
conn->from.fd = conn->fd.get();
|
||||
conn->to.fd = conn->fd.get();
|
||||
|
||||
conn->startTime = std::chrono::steady_clock::now();
|
||||
|
||||
return conn;
|
||||
}
|
||||
|
||||
|
||||
static RegisterStoreImplementation<UDSRemoteStore, UDSRemoteStoreConfig> regUDSRemoteStore;
|
||||
|
||||
}
|
||||
52
src/libstore/uds-remote-store.hh
Normal file
52
src/libstore/uds-remote-store.hh
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
#pragma once
|
||||
|
||||
#include "remote-store.hh"
|
||||
#include "local-fs-store.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreConfig
|
||||
{
|
||||
UDSRemoteStoreConfig(const Store::Params & params)
|
||||
: StoreConfig(params)
|
||||
, LocalFSStoreConfig(params)
|
||||
, RemoteStoreConfig(params)
|
||||
{
|
||||
}
|
||||
|
||||
UDSRemoteStoreConfig()
|
||||
: UDSRemoteStoreConfig(Store::Params({}))
|
||||
{
|
||||
}
|
||||
|
||||
const std::string name() override { return "Local Daemon Store"; }
|
||||
};
|
||||
|
||||
class UDSRemoteStore : public LocalFSStore, public RemoteStore, public virtual UDSRemoteStoreConfig
|
||||
{
|
||||
public:
|
||||
|
||||
UDSRemoteStore(const Params & params);
|
||||
UDSRemoteStore(const std::string scheme, std::string path, const Params & params);
|
||||
|
||||
std::string getUri() override;
|
||||
|
||||
static std::set<std::string> uriSchemes()
|
||||
{ return {"unix"}; }
|
||||
|
||||
bool sameMachine() override
|
||||
{ return true; }
|
||||
|
||||
ref<FSAccessor> getFSAccessor() override
|
||||
{ return LocalFSStore::getFSAccessor(); }
|
||||
|
||||
void narFromPath(const StorePath & path, Sink & sink) override
|
||||
{ LocalFSStore::narFromPath(path, sink); }
|
||||
|
||||
private:
|
||||
|
||||
ref<RemoteStore::Connection> openConnection() override;
|
||||
std::optional<std::string> path;
|
||||
};
|
||||
|
||||
}
|
||||
|
|
@ -17,8 +17,20 @@ void Args::addFlag(Flag && flag_)
|
|||
if (flag->shortName) shortFlags[flag->shortName] = flag;
|
||||
}
|
||||
|
||||
void Completions::add(std::string completion, std::string description)
|
||||
{
|
||||
assert(description.find('\n') == std::string::npos);
|
||||
insert(Completion {
|
||||
.completion = completion,
|
||||
.description = description
|
||||
});
|
||||
}
|
||||
|
||||
bool Completion::operator<(const Completion & other) const
|
||||
{ return completion < other.completion || (completion == other.completion && description < other.description); }
|
||||
|
||||
bool pathCompletions = false;
|
||||
std::shared_ptr<std::set<std::string>> completions;
|
||||
std::shared_ptr<Completions> completions;
|
||||
|
||||
std::string completionMarker = "___COMPLETE___";
|
||||
|
||||
|
|
@ -148,7 +160,7 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
|
|||
for (auto & [name, flag] : longFlags) {
|
||||
if (!hiddenCategories.count(flag->category)
|
||||
&& hasPrefix(name, std::string(*prefix, 2)))
|
||||
completions->insert("--" + name);
|
||||
completions->add("--" + name, flag->description);
|
||||
}
|
||||
}
|
||||
auto i = longFlags.find(string(*pos, 2));
|
||||
|
|
@ -165,9 +177,9 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
|
|||
|
||||
if (auto prefix = needsCompletion(*pos)) {
|
||||
if (prefix == "-") {
|
||||
completions->insert("--");
|
||||
for (auto & [flag, _] : shortFlags)
|
||||
completions->insert(std::string("-") + flag);
|
||||
completions->add("--");
|
||||
for (auto & [flagName, flag] : shortFlags)
|
||||
completions->add(std::string("-") + flagName, flag->description);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -248,7 +260,7 @@ static void hashTypeCompleter(size_t index, std::string_view prefix)
|
|||
{
|
||||
for (auto & type : hashTypes)
|
||||
if (hasPrefix(type, prefix))
|
||||
completions->insert(type);
|
||||
completions->add(type);
|
||||
}
|
||||
|
||||
Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashType * ht)
|
||||
|
|
@ -292,7 +304,7 @@ static void _completePath(std::string_view prefix, bool onlyDirs)
|
|||
auto st = lstat(globbuf.gl_pathv[i]);
|
||||
if (!S_ISDIR(st.st_mode)) continue;
|
||||
}
|
||||
completions->insert(globbuf.gl_pathv[i]);
|
||||
completions->add(globbuf.gl_pathv[i]);
|
||||
}
|
||||
globfree(&globbuf);
|
||||
}
|
||||
|
|
@ -385,7 +397,7 @@ MultiCommand::MultiCommand(const Commands & commands)
|
|||
if (auto prefix = needsCompletion(s)) {
|
||||
for (auto & [name, command] : commands)
|
||||
if (hasPrefix(name, *prefix))
|
||||
completions->insert(name);
|
||||
completions->add(name);
|
||||
}
|
||||
auto i = commands.find(s);
|
||||
if (i == commands.end())
|
||||
|
|
|
|||
|
|
@ -283,7 +283,17 @@ typedef std::vector<std::pair<std::string, std::string>> Table2;
|
|||
|
||||
void printTable(std::ostream & out, const Table2 & table);
|
||||
|
||||
extern std::shared_ptr<std::set<std::string>> completions;
|
||||
struct Completion {
|
||||
std::string completion;
|
||||
std::string description;
|
||||
|
||||
bool operator<(const Completion & other) const;
|
||||
};
|
||||
class Completions : public std::set<Completion> {
|
||||
public:
|
||||
void add(std::string completion, std::string description = "");
|
||||
};
|
||||
extern std::shared_ptr<Completions> completions;
|
||||
extern bool pathCompletions;
|
||||
|
||||
std::optional<std::string> needsCompletion(std::string_view s);
|
||||
|
|
|
|||
|
|
@ -1660,4 +1660,33 @@ string showBytes(uint64_t bytes)
|
|||
}
|
||||
|
||||
|
||||
void commonChildInit(Pipe & logPipe)
|
||||
{
|
||||
const static string pathNullDevice = "/dev/null";
|
||||
restoreSignals();
|
||||
|
||||
/* Put the child in a separate session (and thus a separate
|
||||
process group) so that it has no controlling terminal (meaning
|
||||
that e.g. ssh cannot open /dev/tty) and it doesn't receive
|
||||
terminal signals. */
|
||||
if (setsid() == -1)
|
||||
throw SysError("creating a new session");
|
||||
|
||||
/* Dup the write side of the logger pipe into stderr. */
|
||||
if (dup2(logPipe.writeSide.get(), STDERR_FILENO) == -1)
|
||||
throw SysError("cannot pipe standard error into log file");
|
||||
|
||||
/* Dup stderr to stdout. */
|
||||
if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1)
|
||||
throw SysError("cannot dup stderr into stdout");
|
||||
|
||||
/* Reroute stdin to /dev/null. */
|
||||
int fdDevNull = open(pathNullDevice.c_str(), O_RDWR);
|
||||
if (fdDevNull == -1)
|
||||
throw SysError("cannot open '%1%'", pathNullDevice);
|
||||
if (dup2(fdDevNull, STDIN_FILENO) == -1)
|
||||
throw SysError("cannot dup null device into stdin");
|
||||
close(fdDevNull);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -536,6 +536,8 @@ typedef std::function<bool(const Path & path)> PathFilter;
|
|||
|
||||
extern PathFilter defaultPathFilter;
|
||||
|
||||
/* Common initialisation performed in child processes. */
|
||||
void commonChildInit(Pipe & logPipe);
|
||||
|
||||
/* Create a Unix domain socket in listen mode. */
|
||||
AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode);
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
#include <vector>
|
||||
|
||||
#include "store-api.hh"
|
||||
#include "local-fs-store.hh"
|
||||
#include "globals.hh"
|
||||
#include "derivations.hh"
|
||||
#include "affinity.hh"
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@
|
|||
#include "profiles.hh"
|
||||
#include "shared.hh"
|
||||
#include "store-api.hh"
|
||||
#include "local-fs-store.hh"
|
||||
#include "user-env.hh"
|
||||
#include "util.hh"
|
||||
#include "json.hh"
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
#include "util.hh"
|
||||
#include "derivations.hh"
|
||||
#include "store-api.hh"
|
||||
#include "local-fs-store.hh"
|
||||
#include "globals.hh"
|
||||
#include "shared.hh"
|
||||
#include "eval.hh"
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@
|
|||
#include "value-to-json.hh"
|
||||
#include "util.hh"
|
||||
#include "store-api.hh"
|
||||
#include "local-fs-store.hh"
|
||||
#include "common-eval-args.hh"
|
||||
#include "../nix/legacy.hh"
|
||||
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
#include "common-args.hh"
|
||||
#include "shared.hh"
|
||||
#include "store-api.hh"
|
||||
#include "local-fs-store.hh"
|
||||
|
||||
using namespace nix;
|
||||
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
#include "common-args.hh"
|
||||
#include "shared.hh"
|
||||
#include "store-api.hh"
|
||||
#include "local-fs-store.hh"
|
||||
#include "fs-accessor.hh"
|
||||
|
||||
using namespace nix;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
#include "command.hh"
|
||||
#include "store-api.hh"
|
||||
#include "local-fs-store.hh"
|
||||
#include "derivations.hh"
|
||||
#include "nixexpr.hh"
|
||||
#include "profiles.hh"
|
||||
|
|
|
|||
|
|
@ -164,6 +164,7 @@ struct Common : InstallableCommand, MixProfile
|
|||
"BASHOPTS",
|
||||
"EUID",
|
||||
"HOME", // FIXME: don't ignore in pure mode?
|
||||
"HOSTNAME",
|
||||
"NIX_BUILD_TOP",
|
||||
"NIX_ENFORCE_PURITY",
|
||||
"NIX_LOG_FD",
|
||||
|
|
@ -377,6 +378,10 @@ struct CmdDevelop : Common, MixEnvironment
|
|||
script += fmt("exec %s\n", concatStringsSep(" ", args));
|
||||
}
|
||||
|
||||
else {
|
||||
script += "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc;\n";
|
||||
}
|
||||
|
||||
writeFull(rcFileFd.get(), script);
|
||||
|
||||
stopProgressBar();
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@
|
|||
#include "serve-protocol.hh"
|
||||
#include "shared.hh"
|
||||
#include "store-api.hh"
|
||||
#include "local-fs-store.hh"
|
||||
#include "util.hh"
|
||||
#include "worker-protocol.hh"
|
||||
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ void completeFlakeInputPath(
|
|||
auto flake = flake::getFlake(*evalState, flakeRef, true);
|
||||
for (auto & input : flake.inputs)
|
||||
if (hasPrefix(input.first, prefix))
|
||||
completions->insert(input.first);
|
||||
completions->add(input.first);
|
||||
}
|
||||
|
||||
MixFlakeOptions::MixFlakeOptions()
|
||||
|
|
@ -211,7 +211,7 @@ void completeFlakeRefWithFragment(
|
|||
auto attrPath2 = attr->getAttrPath(attr2);
|
||||
/* Strip the attrpath prefix. */
|
||||
attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size());
|
||||
completions->insert(flakeRefS + "#" + concatStringsSep(".", attrPath2));
|
||||
completions->add(flakeRefS + "#" + concatStringsSep(".", attrPath2));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -222,7 +222,7 @@ void completeFlakeRefWithFragment(
|
|||
for (auto & attrPath : defaultFlakeAttrPaths) {
|
||||
auto attr = root->findAlongAttrPath(parseAttrPath(*evalState, attrPath));
|
||||
if (!attr) continue;
|
||||
completions->insert(flakeRefS + "#");
|
||||
completions->add(flakeRefS + "#");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -243,7 +243,7 @@ ref<EvalState> EvalCommand::getEvalState()
|
|||
void completeFlakeRef(ref<Store> store, std::string_view prefix)
|
||||
{
|
||||
if (prefix == "")
|
||||
completions->insert(".");
|
||||
completions->add(".");
|
||||
|
||||
completeDir(0, prefix);
|
||||
|
||||
|
|
@ -254,10 +254,10 @@ void completeFlakeRef(ref<Store> store, std::string_view prefix)
|
|||
if (!hasPrefix(prefix, "flake:") && hasPrefix(from, "flake:")) {
|
||||
std::string from2(from, 6);
|
||||
if (hasPrefix(from2, prefix))
|
||||
completions->insert(from2);
|
||||
completions->add(from2);
|
||||
} else {
|
||||
if (hasPrefix(from, prefix))
|
||||
completions->insert(from);
|
||||
completions->add(from);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -208,7 +208,7 @@ void mainWrapped(int argc, char * * argv)
|
|||
if (completions) {
|
||||
std::cout << (pathCompletions ? "filenames\n" : "no-filenames\n");
|
||||
for (auto & s : *completions)
|
||||
std::cout << s << "\n";
|
||||
std::cout << s.completion << "\t" << s.description << "\n";
|
||||
}
|
||||
});
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue