Compare commits

..

No commits in common. "74589afdc92553dd939e37cbc98cfb356550b045" and "03cbc0ecb9402fe7bbe1a2acd4643995003d7bb2" have entirely different histories.

23 changed files with 132 additions and 118 deletions

View file

@ -1,13 +0,0 @@
---
synopsis: Ctrl-C stops Nix commands much more reliably and responsively
issues: [7245, fj#393]
cls: [2016]
prs: [11618]
category: Fixes
credits: [roberth, 9999years]
---
CTRL-C will now stop Nix commands much more reliably and responsively. While
there are still some cases where a Nix command can be slow or unresponsive
following a `SIGINT` (please report these as issues!), the vast majority of
signals will now cause the Nix command to quit quickly and consistently.

View file

@ -50,9 +50,10 @@ project('lix', 'cpp', 'rust',
meson_version : '>=1.4.0',
version : run_command('bash', '-c', 'echo -n $(jq -r .version < ./version.json)$VERSION_SUFFIX', check : true).stdout().strip(),
default_options : [
'cpp_std=c++23',
'cpp_std=c++2a',
'rust_std=2021',
'warning_level=2',
# TODO(Qyriad): increase the warning level
'warning_level=1',
'debug=true',
'optimization=2',
'errorlogs=true', # Please print logs for tests that fail
@ -484,7 +485,6 @@ add_project_arguments(
# TODO(Qyriad): Yes this is how the autoconf+Make system did it.
# It would be nice for our headers to be idempotent instead.
'-include', 'config.h',
'-Wno-unused-parameter',
'-Wno-deprecated-declarations',
'-Wimplicit-fallthrough',
'-Werror=switch',

View file

@ -21,14 +21,6 @@ std::ostream & operator <<(std::ostream & str, const SymbolStr & symbol)
return printIdentifier(str, s);
}
AttrName::AttrName(Symbol s) : symbol(s)
{
}
AttrName::AttrName(std::unique_ptr<Expr> e) : expr(std::move(e))
{
}
void Expr::show(const SymbolTable & symbols, std::ostream & str) const
{
abort();

View file

@ -30,8 +30,8 @@ struct AttrName
{
Symbol symbol;
std::unique_ptr<Expr> expr;
AttrName(Symbol s);
AttrName(std::unique_ptr<Expr> e);
AttrName(Symbol s) : symbol(s) {};
AttrName(std::unique_ptr<Expr> e) : expr(std::move(e)) {};
};
typedef std::vector<AttrName> AttrPath;

View file

@ -9,7 +9,7 @@ namespace nix::parser {
struct StringToken
{
std::string_view s;
bool hasIndentation = false;
bool hasIndentation;
operator std::string_view() const { return s; }
};

View file

@ -47,7 +47,7 @@ struct BuildResult
* @todo This should be an entire ErrorInfo object, not just a
* string, for richer information.
*/
std::string errorMsg = {};
std::string errorMsg;
std::string toString() const {
auto strStatus = [&]() {
@ -90,7 +90,7 @@ struct BuildResult
* For derivations, a mapping from the names of the wanted outputs
* to actual paths.
*/
SingleDrvOutputs builtOutputs = {};
SingleDrvOutputs builtOutputs;
/**
* The start/stop times of the build (or one of the rounds, if it

View file

@ -63,7 +63,7 @@ struct InitialOutputStatus {
struct InitialOutput {
bool wanted;
Hash outputHash;
std::optional<InitialOutputStatus> known = {};
std::optional<InitialOutputStatus> known;
};
/**

View file

@ -26,6 +26,7 @@ try {
trace("done");
notify->fulfill(result);
cleanup();
co_return std::move(result);

View file

@ -82,14 +82,19 @@ struct Goal
*/
std::string name;
struct WorkResult;
// for use by Worker and Goal only. will go away once work() is a promise.
kj::Own<kj::PromiseFulfiller<Result<WorkResult>>> notify;
protected:
AsyncSemaphore::Token slotToken;
public:
struct [[nodiscard]] WorkResult {
ExitCode exitCode;
BuildResult result = {};
std::shared_ptr<Error> ex = {};
BuildResult result;
std::shared_ptr<Error> ex;
bool permanentFailure = false;
bool timedOut = false;
bool hashMismatch = false;

View file

@ -48,10 +48,6 @@ Worker::~Worker()
their destructors). */
children.clear();
derivationGoals.clear();
drvOutputSubstitutionGoals.clear();
substitutionGoals.clear();
assert(expectedSubstitutions == 0);
assert(expectedDownloadSize == 0);
assert(expectedNarSize == 0);
@ -71,45 +67,25 @@ std::pair<std::shared_ptr<G>, kj::Promise<Result<Goal::WorkResult>>> Worker::mak
// and then we only want to recreate the goal *once*. concurrent accesses
// to the worker are not sound, we want to catch them if at all possible.
for ([[maybe_unused]] auto _attempt : {1, 2}) {
auto & cachedGoal = it->second;
auto & goal = cachedGoal.goal;
auto & goal_weak = it->second;
auto goal = goal_weak.goal.lock();
if (!goal) {
goal = create();
goal->notify = std::move(goal_weak.fulfiller);
goal_weak.goal = goal;
// do not start working immediately. if we are not yet running we
// may create dependencies as though they were toplevel goals, in
// which case the dependencies will not report build errors. when
// we are running we may be called for this same goal more times,
// and then we want to modify rather than recreate when possible.
auto removeWhenDone = [goal, &map, it] {
// c++ lambda coroutine capture semantics are *so* fucked up.
return [](auto goal, auto & map, auto it) -> kj::Promise<Result<Goal::WorkResult>> {
auto result = co_await goal->work();
// a concurrent call to makeGoalCommon may have reset our
// cached goal and replaced it with a new instance. don't
// remove the goal in this case, otherwise we will crash.
if (goal == it->second.goal) {
map.erase(it);
}
co_return result;
}(goal, map, it);
};
cachedGoal.promise = kj::evalLater(std::move(removeWhenDone)).fork();
children.add(cachedGoal.promise.addBranch().then([this](auto _result) {
if (_result.has_value()) {
auto & result = _result.value();
permanentFailure |= result.permanentFailure;
timedOut |= result.timedOut;
hashMismatch |= result.hashMismatch;
checkMismatch |= result.checkMismatch;
}
}));
childStarted(goal, kj::evalLater([goal] { return goal->work(); }));
} else {
if (!modify(*goal)) {
cachedGoal = {};
goal_weak = {};
continue;
}
}
return {goal, cachedGoal.promise.addBranch()};
return {goal, goal_weak.promise->addBranch()};
}
assert(false && "could not make a goal. possible concurrent worker access");
}
@ -203,6 +179,58 @@ std::pair<GoalPtr, kj::Promise<Result<Goal::WorkResult>>> Worker::makeGoal(const
}, req.raw());
}
template<typename G>
static void removeGoal(std::shared_ptr<G> goal, auto & goalMap)
{
/* !!! inefficient */
for (auto i = goalMap.begin();
i != goalMap.end(); )
if (i->second.goal.lock() == goal) {
auto j = i; ++j;
goalMap.erase(i);
i = j;
}
else ++i;
}
void Worker::goalFinished(GoalPtr goal, Goal::WorkResult & f)
{
permanentFailure |= f.permanentFailure;
timedOut |= f.timedOut;
hashMismatch |= f.hashMismatch;
checkMismatch |= f.checkMismatch;
removeGoal(goal);
}
void Worker::removeGoal(GoalPtr goal)
{
if (auto drvGoal = std::dynamic_pointer_cast<DerivationGoal>(goal))
nix::removeGoal(drvGoal, derivationGoals);
else if (auto subGoal = std::dynamic_pointer_cast<PathSubstitutionGoal>(goal))
nix::removeGoal(subGoal, substitutionGoals);
else if (auto subGoal = std::dynamic_pointer_cast<DrvOutputSubstitutionGoal>(goal))
nix::removeGoal(subGoal, drvOutputSubstitutionGoals);
else
assert(false);
}
void Worker::childStarted(GoalPtr goal, kj::Promise<Result<Goal::WorkResult>> promise)
{
children.add(promise
.then([this, goal](auto result) {
if (result.has_value()) {
goalFinished(goal, result.assume_value());
} else {
goal->notify->fulfill(result.assume_error());
}
}));
}
kj::Promise<Result<Worker::Results>> Worker::updateStatistics()
try {
while (true) {
@ -247,7 +275,7 @@ Worker::Results Worker::run(std::function<Targets (GoalFactory &)> req)
.exclusiveJoin(std::move(onInterrupt.promise));
// TODO GC interface?
if (auto localStore = dynamic_cast<LocalStore *>(&store); localStore && settings.minFree != 0u) {
if (auto localStore = dynamic_cast<LocalStore *>(&store); localStore && settings.minFree != 0) {
// Periodically wake up to see if we need to run the garbage collector.
promise = promise.exclusiveJoin(boopGC(*localStore));
}

View file

@ -95,8 +95,16 @@ private:
template<typename G>
struct CachedGoal
{
std::shared_ptr<G> goal;
kj::ForkedPromise<Result<Goal::WorkResult>> promise{nullptr};
std::weak_ptr<G> goal;
kj::Own<kj::ForkedPromise<Result<Goal::WorkResult>>> promise;
kj::Own<kj::PromiseFulfiller<Result<Goal::WorkResult>>> fulfiller;
CachedGoal()
{
auto pf = kj::newPromiseAndFulfiller<Result<Goal::WorkResult>>();
promise = kj::heap(pf.promise.fork());
fulfiller = std::move(pf.fulfiller);
}
};
/**
* Maps used to prevent multiple instantiations of a goal for the
@ -132,6 +140,18 @@ private:
*/
bool checkMismatch = false;
void goalFinished(GoalPtr goal, Goal::WorkResult & f);
/**
* Remove a dead goal.
*/
void removeGoal(GoalPtr goal);
/**
* Registers a running child process.
*/
void childStarted(GoalPtr goal, kj::Promise<Result<Goal::WorkResult>> promise);
/**
* Pass current stats counters to the logger for progress bar updates.
*/

View file

@ -6,7 +6,6 @@
#include "signals.hh"
#include "compression.hh"
#include "strings.hh"
#include <cstddef>
#if ENABLE_S3
#include <aws/core/client/ClientConfiguration.h>
@ -785,10 +784,8 @@ struct curlFileTransfer : public FileTransfer
size_t read(char * data, size_t len) override
{
auto readPartial = [this](char * data, size_t len) -> size_t {
auto readPartial = [this](char * data, size_t len) {
const auto available = std::min(len, buffered.size());
if (available == 0u) return 0u;
memcpy(data, buffered.data(), available);
buffered.remove_prefix(available);
return available;

View file

@ -20,10 +20,10 @@ struct NarMember
file in the NAR. */
uint64_t start = 0, size = 0;
std::string target = {};
std::string target;
/* If this is a directory, all the children of the directory. */
std::map<std::string, NarMember> children = {};
std::map<std::string, NarMember> children;
};
struct NarAccessor : public FSAccessor

View file

@ -17,7 +17,7 @@ namespace nix {
struct StorePathWithOutputs
{
StorePath path;
std::set<std::string> outputs = {};
std::set<std::string> outputs;
std::string to_string(const Store & store) const;

View file

@ -50,7 +50,7 @@ struct Realisation {
DrvOutput id;
StorePath outPath;
StringSet signatures = {};
StringSet signatures;
/**
* The realisations that are required for the current one to be valid.
@ -58,7 +58,7 @@ struct Realisation {
* When importing this realisation, the store will first check that all its
* dependencies exist, and map to the correct output path
*/
std::map<DrvOutput, StorePath> dependentRealisations = {};
std::map<DrvOutput, StorePath> dependentRealisations;
nlohmann::json toJSON() const;
static Realisation fromJSON(const nlohmann::json& json, const std::string& whence);

View file

@ -829,7 +829,7 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m
{
size_t left;
StorePathSet valid;
std::exception_ptr exc = {};
std::exception_ptr exc;
};
Sync<State> state_(State{paths.size(), StorePathSet()});

View file

@ -70,17 +70,17 @@ inline bool operator<=(const Trace& lhs, const Trace& rhs);
inline bool operator>=(const Trace& lhs, const Trace& rhs);
struct ErrorInfo {
Verbosity level = Verbosity::lvlError;
Verbosity level;
HintFmt msg;
std::shared_ptr<Pos> pos;
std::list<Trace> traces = {};
std::list<Trace> traces;
/**
* Exit status.
*/
unsigned int status = 1;
Suggestions suggestions = {};
Suggestions suggestions;
static std::optional<std::string> programName;
};

View file

@ -78,11 +78,11 @@ struct RunOptions
{
Path program;
bool searchPath = true;
Strings args = {};
std::optional<uid_t> uid = {};
std::optional<uid_t> gid = {};
std::optional<Path> chdir = {};
std::optional<std::map<std::string, std::string>> environment = {};
Strings args;
std::optional<uid_t> uid;
std::optional<uid_t> gid;
std::optional<Path> chdir;
std::optional<std::map<std::string, std::string>> environment;
bool captureStdout = false;
bool mergeStderrToStdout = false;
bool isInteractive = false;

View file

@ -8,8 +8,6 @@ clearStore
# See https://github.com/NixOS/nix/issues/6195
repo=$TEST_ROOT/./git
default_branch="$(git config init.defaultBranch)"
export _NIX_FORCE_HTTP=1
rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix $TEST_ROOT/worktree $TEST_ROOT/shallow $TEST_ROOT/minimal
@ -49,7 +47,7 @@ git -C $repo checkout -b devtest
echo "different file" >> $TEST_ROOT/git/differentbranch
git -C $repo add differentbranch
git -C $repo commit -m 'Test2'
git -C $repo checkout "$default_branch"
git -C $repo checkout master
devrev=$(git -C $repo rev-parse devtest)
out=$(nix eval --impure --raw --expr "builtins.fetchGit { url = \"file://$repo\"; rev = \"$devrev\"; }" 2>&1) || status=$?
[[ $status == 1 ]]
@ -120,7 +118,7 @@ path2=$(nix eval --impure --raw --expr "(builtins.fetchGit $repo).outPath")
[[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).dirtyShortRev") = "${rev2:0:7}-dirty" ]]
# ... unless we're using an explicit ref or rev.
path3=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = \"$default_branch\"; }).outPath")
path3=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = \"master\"; }).outPath")
[[ $path = $path3 ]]
path3=$(nix eval --raw --expr "(builtins.fetchGit { url = $repo; rev = \"$rev2\"; }).outPath")

View file

@ -6,8 +6,6 @@ clearStore
repo="$TEST_ROOT/git"
default_branch="$(git config init.defaultBranch)"
rm -rf "$repo" "${repo}-tmp" "$TEST_HOME/.cache/nix"
git init "$repo"
@ -18,7 +16,7 @@ echo utrecht > "$repo"/hello
git -C "$repo" add hello
git -C "$repo" commit -m 'Bla1'
path=$(nix eval --raw --impure --expr "(builtins.fetchGit { url = $repo; ref = \"$default_branch\"; }).outPath")
path=$(nix eval --raw --impure --expr "(builtins.fetchGit { url = $repo; ref = \"master\"; }).outPath")
# Test various combinations of ref names
# (taken from the git project)
@ -40,7 +38,7 @@ path=$(nix eval --raw --impure --expr "(builtins.fetchGit { url = $repo; ref = \
valid_ref() {
{ set +x; printf >&2 '\n>>>>>>>>>> valid_ref %s\b <<<<<<<<<<\n' $(printf %s "$1" | sed -n -e l); set -x; }
git check-ref-format --branch "$1" >/dev/null
git -C "$repo" branch "$1" "$default_branch" >/dev/null
git -C "$repo" branch "$1" master >/dev/null
path1=$(nix eval --raw --impure --expr "(builtins.fetchGit { url = $repo; ref = ''$1''; }).outPath")
[[ $path1 = $path ]]
git -C "$repo" branch -D "$1" >/dev/null

View file

@ -3,9 +3,6 @@ source ./common.sh
requireGit
clearStore
default_branch="$(git config init.defaultBranch)"
rm -rf $TEST_HOME/.cache $TEST_HOME/.config
flake1Dir=$TEST_ROOT/flake1
@ -18,10 +15,10 @@ badFlakeDir=$TEST_ROOT/badFlake
flakeGitBare=$TEST_ROOT/flakeGitBare
for repo in $flake1Dir $flake2Dir $flake3Dir $flake7Dir $nonFlakeDir; do
# Give one repo a non-default initial branch.
# Give one repo a non-main initial branch.
extraArgs=
if [[ $repo == $flake2Dir ]]; then
extraArgs="--initial-branch=notdefault"
extraArgs="--initial-branch=main"
fi
createGitRepo "$repo" "$extraArgs"
@ -155,11 +152,11 @@ nix build -o $TEST_ROOT/result $flake2Dir#bar --no-write-lock-file
expect 1 nix build -o $TEST_ROOT/result $flake2Dir#bar --no-update-lock-file 2>&1 | grep 'requires lock file changes'
nix build -o $TEST_ROOT/result $flake2Dir#bar --commit-lock-file
[[ -e $flake2Dir/flake.lock ]]
[[ -z $(git -C $flake2Dir diff notdefault || echo failed) ]]
[[ -z $(git -C $flake2Dir diff main || echo failed) ]]
# Rerunning the build should not change the lockfile.
nix build -o $TEST_ROOT/result $flake2Dir#bar
[[ -z $(git -C $flake2Dir diff notdefault || echo failed) ]]
[[ -z $(git -C $flake2Dir diff main || echo failed) ]]
# Building with a lockfile should not require a fetch of the registry.
nix build -o $TEST_ROOT/result --flake-registry file:///no-registry.json $flake2Dir#bar --refresh
@ -168,7 +165,7 @@ nix build -o $TEST_ROOT/result --no-use-registries $flake2Dir#bar --refresh
# Updating the flake should not change the lockfile.
nix flake lock $flake2Dir
[[ -z $(git -C $flake2Dir diff notdefault || echo failed) ]]
[[ -z $(git -C $flake2Dir diff main || echo failed) ]]
# Now we should be able to build the flake in pure mode.
nix build -o $TEST_ROOT/result flake2#bar
@ -203,7 +200,7 @@ nix build -o $TEST_ROOT/result $flake3Dir#"sth sth"
nix build -o $TEST_ROOT/result $flake3Dir#"sth%20sth"
# Check whether it saved the lockfile
[[ -n $(git -C $flake3Dir diff "$default_branch") ]]
[[ -n $(git -C $flake3Dir diff master) ]]
git -C $flake3Dir add flake.lock
@ -289,7 +286,7 @@ nix build -o $TEST_ROOT/result $flake3Dir#sth --commit-lock-file
Flake lock file updates:
"?" Added input 'nonFlake':
'git+file://"*"/flakes/flakes/nonFlake?ref=refs/heads/$default_branch&rev="*"' "*"
'git+file://"*"/flakes/flakes/nonFlake?ref=refs/heads/master&rev="*"' "*"
"?" Added input 'nonFlakeFile':
'path:"*"/flakes/flakes/nonFlake/README.md?lastModified="*"&narHash=sha256-cPh6hp48IOdRxVV3xGd0PDgSxgzj5N/2cK0rMPNaR4o%3D' "*"
"?" Added input 'nonFlakeFile2':
@ -316,10 +313,10 @@ nix build -o $TEST_ROOT/result flake4#xyzzy
# Test 'nix flake update' and --override-flake.
nix flake lock $flake3Dir
[[ -z $(git -C $flake3Dir diff "$default_branch" || echo failed) ]]
[[ -z $(git -C $flake3Dir diff master || echo failed) ]]
nix flake update --flake "$flake3Dir" --override-flake flake2 nixpkgs
[[ ! -z $(git -C "$flake3Dir" diff "$default_branch" || echo failed) ]]
[[ ! -z $(git -C "$flake3Dir" diff master || echo failed) ]]
# Make branch "removeXyzzy" where flake3 doesn't have xyzzy anymore
git -C $flake3Dir checkout -b removeXyzzy
@ -353,7 +350,7 @@ EOF
nix flake lock $flake3Dir
git -C $flake3Dir add flake.nix flake.lock
git -C $flake3Dir commit -m 'Remove packages.xyzzy'
git -C $flake3Dir checkout "$default_branch"
git -C $flake3Dir checkout master
# Test whether fuzzy-matching works for registry entries.
(! nix build -o $TEST_ROOT/result flake4/removeXyzzy#xyzzy)
@ -502,7 +499,7 @@ nix flake lock $flake3Dir --override-input flake2/flake1 file://$TEST_ROOT/flake
nix flake lock $flake3Dir --override-input flake2/flake1 flake1
[[ $(jq -r .nodes.flake1_2.locked.rev $flake3Dir/flake.lock) =~ $hash2 ]]
nix flake lock $flake3Dir --override-input flake2/flake1 "flake1/$default_branch/$hash1"
nix flake lock $flake3Dir --override-input flake2/flake1 flake1/master/$hash1
[[ $(jq -r .nodes.flake1_2.locked.rev $flake3Dir/flake.lock) =~ $hash1 ]]
nix flake lock $flake3Dir
@ -513,8 +510,8 @@ nix flake update flake2/flake1 --flake "$flake3Dir"
[[ $(jq -r .nodes.flake1_2.locked.rev "$flake3Dir/flake.lock") =~ $hash2 ]]
# Test updating multiple inputs.
nix flake lock "$flake3Dir" --override-input flake1 "flake1/$default_branch/$hash1"
nix flake lock "$flake3Dir" --override-input flake2/flake1 "flake1/$default_branch/$hash1"
nix flake lock "$flake3Dir" --override-input flake1 flake1/master/$hash1
nix flake lock "$flake3Dir" --override-input flake2/flake1 flake1/master/$hash1
[[ $(jq -r .nodes.flake1.locked.rev "$flake3Dir/flake.lock") =~ $hash1 ]]
[[ $(jq -r .nodes.flake1_2.locked.rev "$flake3Dir/flake.lock") =~ $hash1 ]]

View file

@ -150,14 +150,6 @@ TEST(FileTransfer, exceptionAbortsDownload)
}
}
TEST(FileTransfer, exceptionAbortsRead)
{
auto [port, srv] = serveHTTP("200 ok", "content-length: 0\r\n", [] { return ""; });
auto ft = makeFileTransfer();
char buf[10] = "";
ASSERT_THROW(ft->download(FileTransferRequest(fmt("http://[::1]:%d/index", port)))->read(buf, 10), EndOfFile);
}
TEST(FileTransfer, NOT_ON_DARWIN(reportsSetupErrors))
{
auto [port, srv] = serveHTTP("404 not found", "", [] { return ""; });

View file

@ -1,5 +1,4 @@
#include "compression.hh"
#include <cstddef>
#include <gtest/gtest.h>
namespace nix {
@ -148,7 +147,7 @@ TEST_P(PerTypeNonNullCompressionTest, truncatedValidInput)
/* n.b. This also tests zero-length input, which is also invalid.
* As of the writing of this comment, it returns empty output, but is
* allowed to throw a compression error instead. */
for (size_t i = 0u; i < compressed.length(); ++i) {
for (int i = 0; i < compressed.length(); ++i) {
auto newCompressed = compressed.substr(compressed.length() - i);
try {
decompress(method, newCompressed);