From d8730fb86facadbef22d3df7f8a743a56e7ed53c Mon Sep 17 00:00:00 2001 From: Ersin Akinci Date: Sun, 20 Oct 2019 19:08:05 -0700 Subject: [PATCH 01/23] Document import syntax --- doc/manual/expressions/builtins.xml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index 465fa1e0b..21da8b87b 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -733,13 +733,21 @@ builtins.genList (x: x * x) 5 import path + import + <path> builtins.import path + builtins.import + <path> Load, parse and return the Nix expression in the file path. If path is a directory, the file default.nix - in that directory is loaded. Evaluation aborts if the + in that directory is loaded. If the + <path> syntax is used, the path will be resolved + relative to those listed in the NIX_PATH environment + variable (see the section on NIX_PATH + for details on how the resolution works). Evaluation aborts if the file doesn’t exist or contains an incorrect Nix expression. import implements Nix’s module system: you can put any Nix expression (such as a set or a function) in a From 9be7787ec04d604a4e65a22a6b9ae56567aaf618 Mon Sep 17 00:00:00 2001 From: Ersin Akinci Date: Mon, 21 Oct 2019 13:12:41 -0700 Subject: [PATCH 02/23] Revert "Document import syntax" This reverts commit d8730fb86facadbef22d3df7f8a743a56e7ed53c. --- doc/manual/expressions/builtins.xml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index 21da8b87b..465fa1e0b 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -733,21 +733,13 @@ builtins.genList (x: x * x) 5 import path - import - <path> builtins.import path - builtins.import - <path> Load, parse and return the Nix expression in the file path. If path is a directory, the file default.nix - in that directory is loaded. If the - <path> syntax is used, the path will be resolved - relative to those listed in the NIX_PATH environment - variable (see the section on NIX_PATH - for details on how the resolution works). Evaluation aborts if the + in that directory is loaded. Evaluation aborts if the file doesn’t exist or contains an incorrect Nix expression. import implements Nix’s module system: you can put any Nix expression (such as a set or a function) in a From b7a936224ee6e1ad9c842f618bbd5bc1864ebd54 Mon Sep 17 00:00:00 2001 From: Ersin Akinci Date: Mon, 21 Oct 2019 14:11:26 -0700 Subject: [PATCH 03/23] Add hint about path in builtins.import --- doc/manual/expressions/builtins.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index 465fa1e0b..8b53cd578 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -746,6 +746,10 @@ builtins.genList (x: x * x) 5 separate file, and use it from Nix expressions in other files. + Unlike some languages, import is a regular + function in Nix. Paths using the angle bracket syntax (e.g., import <foo>) are normal + values (see ). + A Nix expression loaded by import must not contain any free variables (identifiers that are not defined in the Nix expression itself and are not From f107a270026596e525bdf8df0e2d331783ec4a10 Mon Sep 17 00:00:00 2001 From: Ersin Akinci Date: Mon, 21 Oct 2019 14:16:55 -0700 Subject: [PATCH 04/23] Tweak path hint --- doc/manual/expressions/builtins.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index 8b53cd578..f39b393bf 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -747,7 +747,8 @@ builtins.genList (x: x * x) 5 files. Unlike some languages, import is a regular - function in Nix. Paths using the angle bracket syntax (e.g., import <foo>) are normal + function in Nix. Paths using the angle bracket syntax (e.g., + import <foo>) are normal path values (see ). A Nix expression loaded by import must From c92ea927e508f3c06b6b3ee68e1f0565264cac2c Mon Sep 17 00:00:00 2001 From: Chaz Schlarp Date: Tue, 22 Oct 2019 16:04:49 -0700 Subject: [PATCH 05/23] Remove superfluous IAM action for S3 cache `s3:ListObjects` isn't a real IAM action, but _is_ the name of an S3 API method. `s3:ListBucket` is the relevant action for that method. https://docs.aws.amazon.com/IAM/latest/UserGuide/list_amazons3.html --- doc/manual/packages/s3-substituter.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/manual/packages/s3-substituter.xml b/doc/manual/packages/s3-substituter.xml index 1722090ef..868b5a66d 100644 --- a/doc/manual/packages/s3-substituter.xml +++ b/doc/manual/packages/s3-substituter.xml @@ -159,7 +159,6 @@ the S3 URL: "s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:ListMultipartUploadParts", - "s3:ListObjects", "s3:PutObject" ], "Resource": [ From 2f96a89646c6e55e2f1bbb80805dcbbe60fa94ae Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 23 Oct 2019 21:24:00 +0200 Subject: [PATCH 06/23] install-multi-user.sh: Remove unused variables https://hydra.nixos.org/build/104119659 --- scripts/install-multi-user.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index d060e5165..13762cba3 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -19,9 +19,6 @@ readonly BLUE_UL='\033[38;4;34m' readonly GREEN='\033[38;32m' readonly GREEN_UL='\033[38;4;32m' readonly RED='\033[38;31m' -readonly RED_UL='\033[38;4;31m' -readonly YELLOW='\033[38;33m' -readonly YELLOW_UL='\033[38;4;33m' readonly NIX_USER_COUNT="32" readonly NIX_BUILD_GROUP_ID="30000" From e583df52800b4baa1564b027fe3b83a21756c2cc Mon Sep 17 00:00:00 2001 From: Robin Gloster Date: Sun, 27 Oct 2019 10:15:51 +0100 Subject: [PATCH 07/23] builtins.toJSON: fix __toString usage --- src/libexpr/eval.cc | 23 +++++++++++++++++------ src/libexpr/eval.hh | 4 ++++ src/libexpr/value-to-json.cc | 7 ++++++- tests/lang/eval-okay-tojson.exp | 2 +- tests/lang/eval-okay-tojson.nix | 1 + 5 files changed, 29 insertions(+), 8 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index d8e10d9f2..8f728b906 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1565,6 +1565,19 @@ bool EvalState::isDerivation(Value & v) } +std::optional EvalState::tryAttrsToString(const Pos & pos, Value & v, + PathSet & context, bool coerceMore, bool copyToStore) +{ + auto i = v.attrs->find(sToString); + if (i != v.attrs->end()) { + Value v1; + callFunction(*i->value, v, v1, pos); + return coerceToString(pos, v1, context, coerceMore, copyToStore); + } + + return {}; +} + string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, bool coerceMore, bool copyToStore) { @@ -1583,13 +1596,11 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, } if (v.type == tAttrs) { - auto i = v.attrs->find(sToString); - if (i != v.attrs->end()) { - Value v1; - callFunction(*i->value, v, v1, pos); - return coerceToString(pos, v1, context, coerceMore, copyToStore); + auto maybeString = tryAttrsToString(pos, v, context, coerceMore, copyToStore); + if (maybeString) { + return *maybeString; } - i = v.attrs->find(sOutPath); + auto i = v.attrs->find(sOutPath); if (i == v.attrs->end()) throwTypeError("cannot coerce a set to a string, at %1%", pos); return coerceToString(pos, *i->value, context, coerceMore, copyToStore); } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index a314e01e0..82e6ca877 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -8,6 +8,7 @@ #include "config.hh" #include +#include #include @@ -195,6 +196,9 @@ public: set with attribute `type = "derivation"'). */ bool isDerivation(Value & v); + std::optional tryAttrsToString(const Pos & pos, Value & v, + PathSet & context, bool coerceMore = false, bool copyToStore = true); + /* String coercion. Converts strings, paths and derivations to a string. If `coerceMore' is set, also converts nulls, integers, booleans and lists to a string. If `copyToStore' is set, diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 72e413e44..5fe8570ad 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -40,7 +40,12 @@ void printValueAsJSON(EvalState & state, bool strict, break; case tAttrs: { - Bindings::iterator i = v.attrs->find(state.sOutPath); + auto maybeString = state.tryAttrsToString(noPos, v, context, false, false); + if (maybeString) { + out.write(*maybeString); + break; + } + auto i = v.attrs->find(state.sOutPath); if (i == v.attrs->end()) { auto obj(out.object()); StringSet names; diff --git a/tests/lang/eval-okay-tojson.exp b/tests/lang/eval-okay-tojson.exp index 33588493f..e92aae323 100644 --- a/tests/lang/eval-okay-tojson.exp +++ b/tests/lang/eval-okay-tojson.exp @@ -1 +1 @@ -"{\"a\":123,\"b\":-456,\"c\":\"foo\",\"d\":\"foo\\n\\\"bar\\\"\",\"e\":true,\"f\":false,\"g\":[1,2,3],\"h\":[\"a\",[\"b\",{\"foo\\nbar\":{}}]],\"i\":3,\"j\":1.44}" +"{\"a\":123,\"b\":-456,\"c\":\"foo\",\"d\":\"foo\\n\\\"bar\\\"\",\"e\":true,\"f\":false,\"g\":[1,2,3],\"h\":[\"a\",[\"b\",{\"foo\\nbar\":{}}]],\"i\":3,\"j\":1.44,\"k\":\"foo\"}" diff --git a/tests/lang/eval-okay-tojson.nix b/tests/lang/eval-okay-tojson.nix index c046ba4ae..ce67943be 100644 --- a/tests/lang/eval-okay-tojson.nix +++ b/tests/lang/eval-okay-tojson.nix @@ -9,4 +9,5 @@ builtins.toJSON h = [ "a" [ "b" { "foo\nbar" = {}; } ] ]; i = 1 + 2; j = 1.44; + k = { __toString = self: self.a; a = "foo"; }; } From 9d612c393abc3a73590650d24bcfe2ee57792872 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 27 Oct 2019 09:34:33 +0100 Subject: [PATCH 08/23] Add inNixShell = true to nix-shell auto-call This is an alternative to the IN_NIX_SHELL environment variable, allowing the expression to adapt itself to nix-shell without triggering those adaptations when used as a dependency of another shell. Closes #3147 --- src/nix-build/nix-build.cc | 16 +++++++++++++++- tests/nix-shell.sh | 8 ++++---- tests/shell.nix | 3 ++- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 085650e4f..357986980 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -245,7 +245,21 @@ static void _main(int argc, char * * argv) auto state = std::make_unique(myArgs.searchPath, store); state->repair = repair; - Bindings & autoArgs = *myArgs.getAutoArgs(*state); + Bindings & autoArgs = *[&](){ + Bindings *userAutoArgs = myArgs.getAutoArgs(*state); + if (runEnv) { + Bindings * res = state->allocBindings(userAutoArgs->size() + 1); + Value * tru = state->allocValue(); + mkBool(*tru, true); + res->push_back(Attr(state->symbols.create("inNixShell"), tru)); + for (auto & i : *userAutoArgs) { + res->push_back(i); + } + res->sort(); + return res; + } + else return userAutoArgs; + }(); if (packages) { std::ostringstream joined; diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh index ee502dddb..235e2a5ff 100644 --- a/tests/nix-shell.sh +++ b/tests/nix-shell.sh @@ -7,9 +7,9 @@ export IMPURE_VAR=foo export SELECTED_IMPURE_VAR=baz export NIX_BUILD_SHELL=$SHELL output=$(nix-shell --pure shell.nix -A shellDrv --run \ - 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') + 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX - $TEST_inNixShell"') -[ "$output" = " - foo - bar" ] +[ "$output" = " - foo - bar - true" ] # Test --keep output=$(nix-shell --pure --keep SELECTED_IMPURE_VAR shell.nix -A shellDrv --run \ @@ -19,10 +19,10 @@ output=$(nix-shell --pure --keep SELECTED_IMPURE_VAR shell.nix -A shellDrv --run # Test nix-shell on a .drv [[ $(nix-shell --pure $(nix-instantiate shell.nix -A shellDrv) --run \ - 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]] + 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX - $TEST_inNixShell"') = " - foo - bar - false" ]] [[ $(nix-shell --pure $(nix-instantiate shell.nix -A shellDrv) --run \ - 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]] + 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX - $TEST_inNixShell"') = " - foo - bar - false" ]] # Test nix-shell on a .drv symlink diff --git a/tests/shell.nix b/tests/shell.nix index eb39f9039..6cb4f082b 100644 --- a/tests/shell.nix +++ b/tests/shell.nix @@ -1,4 +1,4 @@ -{ }: +{ inNixShell ? false }: with import ./config.nix; @@ -22,6 +22,7 @@ let pkgs = rec { name = "shellDrv"; builder = "/does/not/exist"; VAR_FROM_NIX = "bar"; + TEST_inNixShell = if inNixShell then "true" else "false"; inherit stdenv; }; From 3913afdd69a5bd065d6096dd569397a836ac6857 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 27 Oct 2019 17:58:25 +0100 Subject: [PATCH 09/23] Simplification --- src/nix-build/nix-build.cc | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 357986980..1c5d71681 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -245,21 +245,17 @@ static void _main(int argc, char * * argv) auto state = std::make_unique(myArgs.searchPath, store); state->repair = repair; - Bindings & autoArgs = *[&](){ - Bindings *userAutoArgs = myArgs.getAutoArgs(*state); - if (runEnv) { - Bindings * res = state->allocBindings(userAutoArgs->size() + 1); - Value * tru = state->allocValue(); - mkBool(*tru, true); - res->push_back(Attr(state->symbols.create("inNixShell"), tru)); - for (auto & i : *userAutoArgs) { - res->push_back(i); - } - res->sort(); - return res; - } - else return userAutoArgs; - }(); + auto autoArgs = myArgs.getAutoArgs(*state); + + if (runEnv) { + auto newArgs = state->allocBindings(autoArgs->size() + 1); + auto tru = state->allocValue(); + mkBool(*tru, true); + newArgs->push_back(Attr(state->symbols.create("inNixShell"), tru)); + for (auto & i : *autoArgs) newArgs->push_back(i); + newArgs->sort(); + autoArgs = newArgs; + } if (packages) { std::ostringstream joined; @@ -313,9 +309,9 @@ static void _main(int argc, char * * argv) state->eval(e, vRoot); for (auto & i : attrPaths) { - Value & v(*findAlongAttrPath(*state, i, autoArgs, vRoot)); + Value & v(*findAlongAttrPath(*state, i, *autoArgs, vRoot)); state->forceValue(v); - getDerivations(*state, v, "", autoArgs, drvs, false); + getDerivations(*state, v, "", *autoArgs, drvs, false); } } From f1c0b2c0e16154e738f342f2defa42c4d9348c4c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 1 Oct 2018 17:05:55 +0200 Subject: [PATCH 10/23] Add O(1)-memory copyPath() function --- src/libutil/archive.cc | 9 +++++++++ src/libutil/archive.hh | 2 ++ 2 files changed, 11 insertions(+) diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 3aa120270..db544a212 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -375,4 +375,13 @@ void copyNAR(Source & source, Sink & sink) } +void copyPath(const Path & from, const Path & to) +{ + auto source = sinkToSource([&](Sink & sink) { + dumpPath(from, sink); + }); + restorePath(to, *source); +} + + } diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index 25be426c1..768fe2536 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -77,6 +77,8 @@ void restorePath(const Path & path, Source & source); /* Read a NAR from 'source' and write it to 'sink'. */ void copyNAR(Source & source, Sink & sink); +void copyPath(const Path & from, const Path & to); + extern const std::string narVersionMagic1; From f7ce80f90ab364058e28989af68ba993fd8180a9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 2 Oct 2018 11:22:13 +0200 Subject: [PATCH 11/23] Factor out linkOrCopy() --- src/libstore/build.cc | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 68c2f2ce3..886c864a3 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1874,6 +1874,21 @@ static void preloadNSS() { }); } + +void linkOrCopy(const Path & from, const Path & to) +{ + if (link(from.c_str(), to.c_str()) == -1) { + /* Hard-linking fails if we exceed the maximum link count on a + file (e.g. 32000 of ext3), which is quite possible after a + 'nix-store --optimise'. FIXME: actually, why don't we just + bind-mount in this case? */ + if (errno != EMLINK) + throw SysError("linking '%s' to '%s'", to, from); + copyPath(from, to); + } +} + + void DerivationGoal::startBuilder() { /* Right platform? */ @@ -2117,22 +2132,8 @@ void DerivationGoal::startBuilder() throw SysError(format("getting attributes of path '%1%'") % i); if (S_ISDIR(st.st_mode)) dirsInChroot[i] = r; - else { - Path p = chrootRootDir + i; - debug("linking '%1%' to '%2%'", p, r); - if (link(r.c_str(), p.c_str()) == -1) { - /* Hard-linking fails if we exceed the maximum - link count on a file (e.g. 32000 of ext3), - which is quite possible after a `nix-store - --optimise'. */ - if (errno != EMLINK) - throw SysError(format("linking '%1%' to '%2%'") % p % i); - StringSink sink; - dumpPath(r, sink); - StringSource source(*sink.s); - restorePath(p, source); - } - } + else + linkOrCopy(r, chrootRootDir + i); } /* If we're repairing, checking or rebuilding part of a From 95c727caef110c03900a77133eb3e873fca7f019 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 2 Oct 2018 11:24:54 +0200 Subject: [PATCH 12/23] Remove the check against concurrent builds in the same process --- src/libstore/build.cc | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 886c864a3..dfae8024a 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -4138,9 +4138,6 @@ void SubstitutionGoal::handleEOF(int fd) ////////////////////////////////////////////////////////////////////// -static bool working = false; - - Worker::Worker(LocalStore & store) : act(*logger, actRealise) , actDerivations(*logger, actBuilds) @@ -4148,8 +4145,6 @@ Worker::Worker(LocalStore & store) , store(store) { /* Debugging: prevent recursive workers. */ - if (working) abort(); - working = true; nrLocalBuilds = 0; lastWokenUp = steady_time_point::min(); permanentFailure = false; @@ -4161,8 +4156,6 @@ Worker::Worker(LocalStore & store) Worker::~Worker() { - working = false; - /* Explicitly get rid of all strong pointers now. After this all goals that refer to this worker should be gone. (Otherwise we are in trouble, since goals may call childTerminated() etc. in From 2d37e88319441958b7eed876042a106e79c7c85d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 24 Sep 2018 13:53:44 +0200 Subject: [PATCH 13/23] Move most of the daemon implementation to libstore --- src/libstore/daemon.cc | 798 +++++++++++++++++++++++++++++++ src/libstore/daemon.hh | 12 + src/libstore/worker-protocol.hh | 3 + src/nix-daemon/nix-daemon.cc | 802 +------------------------------- 4 files changed, 821 insertions(+), 794 deletions(-) create mode 100644 src/libstore/daemon.cc create mode 100644 src/libstore/daemon.hh diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc new file mode 100644 index 000000000..2e1d10bcc --- /dev/null +++ b/src/libstore/daemon.cc @@ -0,0 +1,798 @@ +#include "daemon.hh" +#include "monitor-fd.hh" +#include "worker-protocol.hh" +#include "store-api.hh" +#include "local-store.hh" +#include "finally.hh" +#include "affinity.hh" +#include "archive.hh" +#include "derivations.hh" +#include "args.hh" + +namespace nix::daemon { + +Sink & operator << (Sink & sink, const Logger::Fields & fields) +{ + sink << fields.size(); + for (auto & f : fields) { + sink << f.type; + if (f.type == Logger::Field::tInt) + sink << f.i; + else if (f.type == Logger::Field::tString) + sink << f.s; + else abort(); + } + return sink; +} + +/* Logger that forwards log messages to the client, *if* we're in a + state where the protocol allows it (i.e., when canSendStderr is + true). */ +struct TunnelLogger : public Logger +{ + FdSink & to; + + struct State + { + bool canSendStderr = false; + std::vector pendingMsgs; + }; + + Sync state_; + + unsigned int clientVersion; + + TunnelLogger(FdSink & to, unsigned int clientVersion) + : to(to), clientVersion(clientVersion) { } + + void enqueueMsg(const std::string & s) + { + auto state(state_.lock()); + + if (state->canSendStderr) { + assert(state->pendingMsgs.empty()); + try { + to(s); + to.flush(); + } catch (...) { + /* Write failed; that means that the other side is + gone. */ + state->canSendStderr = false; + throw; + } + } else + state->pendingMsgs.push_back(s); + } + + void log(Verbosity lvl, const FormatOrString & fs) override + { + if (lvl > verbosity) return; + + StringSink buf; + buf << STDERR_NEXT << (fs.s + "\n"); + enqueueMsg(*buf.s); + } + + /* startWork() means that we're starting an operation for which we + want to send out stderr to the client. */ + void startWork() + { + auto state(state_.lock()); + state->canSendStderr = true; + + for (auto & msg : state->pendingMsgs) + to(msg); + + state->pendingMsgs.clear(); + + to.flush(); + } + + /* stopWork() means that we're done; stop sending stderr to the + client. */ + void stopWork(bool success = true, const string & msg = "", unsigned int status = 0) + { + auto state(state_.lock()); + + state->canSendStderr = false; + + if (success) + to << STDERR_LAST; + else { + to << STDERR_ERROR << msg; + if (status != 0) to << status; + } + } + + void startActivity(ActivityId act, Verbosity lvl, ActivityType type, + const std::string & s, const Fields & fields, ActivityId parent) override + { + if (GET_PROTOCOL_MINOR(clientVersion) < 20) { + if (!s.empty()) + log(lvl, s + "..."); + return; + } + + StringSink buf; + buf << STDERR_START_ACTIVITY << act << lvl << type << s << fields << parent; + enqueueMsg(*buf.s); + } + + void stopActivity(ActivityId act) override + { + if (GET_PROTOCOL_MINOR(clientVersion) < 20) return; + StringSink buf; + buf << STDERR_STOP_ACTIVITY << act; + enqueueMsg(*buf.s); + } + + void result(ActivityId act, ResultType type, const Fields & fields) override + { + if (GET_PROTOCOL_MINOR(clientVersion) < 20) return; + StringSink buf; + buf << STDERR_RESULT << act << type << fields; + enqueueMsg(*buf.s); + } +}; + +struct TunnelSink : Sink +{ + Sink & to; + TunnelSink(Sink & to) : to(to) { } + virtual void operator () (const unsigned char * data, size_t len) + { + to << STDERR_WRITE; + writeString(data, len, to); + } +}; + +struct TunnelSource : BufferedSource +{ + Source & from; + BufferedSink & to; + TunnelSource(Source & from, BufferedSink & to) : from(from), to(to) { } + size_t readUnbuffered(unsigned char * data, size_t len) override + { + to << STDERR_READ << len; + to.flush(); + size_t n = readString(data, len, from); + if (n == 0) throw EndOfFile("unexpected end-of-file"); + return n; + } +}; + +/* If the NAR archive contains a single file at top-level, then save + the contents of the file to `s'. Otherwise barf. */ +struct RetrieveRegularNARSink : ParseSink +{ + bool regular; + string s; + + RetrieveRegularNARSink() : regular(true) { } + + void createDirectory(const Path & path) + { + regular = false; + } + + void receiveContents(unsigned char * data, unsigned int len) + { + s.append((const char *) data, len); + } + + void createSymlink(const Path & path, const string & target) + { + regular = false; + } +}; + +static void performOp(TunnelLogger * logger, ref store, + bool trusted, unsigned int clientVersion, + Source & from, BufferedSink & to, unsigned int op) +{ + switch (op) { + + case wopIsValidPath: { + /* 'readStorePath' could raise an error leading to the connection + being closed. To be able to recover from an invalid path error, + call 'startWork' early, and do 'assertStorePath' afterwards so + that the 'Error' exception handler doesn't close the + connection. */ + Path path = readString(from); + logger->startWork(); + store->assertStorePath(path); + bool result = store->isValidPath(path); + logger->stopWork(); + to << result; + break; + } + + case wopQueryValidPaths: { + PathSet paths = readStorePaths(*store, from); + logger->startWork(); + PathSet res = store->queryValidPaths(paths); + logger->stopWork(); + to << res; + break; + } + + case wopHasSubstitutes: { + Path path = readStorePath(*store, from); + logger->startWork(); + PathSet res = store->querySubstitutablePaths({path}); + logger->stopWork(); + to << (res.find(path) != res.end()); + break; + } + + case wopQuerySubstitutablePaths: { + PathSet paths = readStorePaths(*store, from); + logger->startWork(); + PathSet res = store->querySubstitutablePaths(paths); + logger->stopWork(); + to << res; + break; + } + + case wopQueryPathHash: { + Path path = readStorePath(*store, from); + logger->startWork(); + auto hash = store->queryPathInfo(path)->narHash; + logger->stopWork(); + to << hash.to_string(Base16, false); + break; + } + + case wopQueryReferences: + case wopQueryReferrers: + case wopQueryValidDerivers: + case wopQueryDerivationOutputs: { + Path path = readStorePath(*store, from); + logger->startWork(); + PathSet paths; + if (op == wopQueryReferences) + paths = store->queryPathInfo(path)->references; + else if (op == wopQueryReferrers) + store->queryReferrers(path, paths); + else if (op == wopQueryValidDerivers) + paths = store->queryValidDerivers(path); + else paths = store->queryDerivationOutputs(path); + logger->stopWork(); + to << paths; + break; + } + + case wopQueryDerivationOutputNames: { + Path path = readStorePath(*store, from); + logger->startWork(); + StringSet names; + names = store->queryDerivationOutputNames(path); + logger->stopWork(); + to << names; + break; + } + + case wopQueryDeriver: { + Path path = readStorePath(*store, from); + logger->startWork(); + auto deriver = store->queryPathInfo(path)->deriver; + logger->stopWork(); + to << deriver; + break; + } + + case wopQueryPathFromHashPart: { + string hashPart = readString(from); + logger->startWork(); + Path path = store->queryPathFromHashPart(hashPart); + logger->stopWork(); + to << path; + break; + } + + case wopAddToStore: { + bool fixed, recursive; + std::string s, baseName; + from >> baseName >> fixed /* obsolete */ >> recursive >> s; + /* Compatibility hack. */ + if (!fixed) { + s = "sha256"; + recursive = true; + } + HashType hashAlgo = parseHashType(s); + + TeeSource savedNAR(from); + RetrieveRegularNARSink savedRegular; + + if (recursive) { + /* Get the entire NAR dump from the client and save it to + a string so that we can pass it to + addToStoreFromDump(). */ + ParseSink sink; /* null sink; just parse the NAR */ + parseDump(sink, savedNAR); + } else + parseDump(savedRegular, from); + + logger->startWork(); + if (!savedRegular.regular) throw Error("regular file expected"); + + auto store2 = store.dynamic_pointer_cast(); + if (!store2) throw Error("operation is only supported by LocalStore"); + + Path path = store2->addToStoreFromDump(recursive ? *savedNAR.data : savedRegular.s, baseName, recursive, hashAlgo); + logger->stopWork(); + + to << path; + break; + } + + case wopAddTextToStore: { + string suffix = readString(from); + string s = readString(from); + PathSet refs = readStorePaths(*store, from); + logger->startWork(); + Path path = store->addTextToStore(suffix, s, refs, NoRepair); + logger->stopWork(); + to << path; + break; + } + + case wopExportPath: { + Path path = readStorePath(*store, from); + readInt(from); // obsolete + logger->startWork(); + TunnelSink sink(to); + store->exportPath(path, sink); + logger->stopWork(); + to << 1; + break; + } + + case wopImportPaths: { + logger->startWork(); + TunnelSource source(from, to); + Paths paths = store->importPaths(source, nullptr, + trusted ? NoCheckSigs : CheckSigs); + logger->stopWork(); + to << paths; + break; + } + + case wopBuildPaths: { + PathSet drvs = readStorePaths(*store, from); + BuildMode mode = bmNormal; + if (GET_PROTOCOL_MINOR(clientVersion) >= 15) { + mode = (BuildMode) readInt(from); + + /* Repairing is not atomic, so disallowed for "untrusted" + clients. */ + if (mode == bmRepair && !trusted) + throw Error("repairing is not allowed because you are not in 'trusted-users'"); + } + logger->startWork(); + store->buildPaths(drvs, mode); + logger->stopWork(); + to << 1; + break; + } + + case wopBuildDerivation: { + Path drvPath = readStorePath(*store, from); + BasicDerivation drv; + readDerivation(from, *store, drv); + BuildMode buildMode = (BuildMode) readInt(from); + logger->startWork(); + if (!trusted) + throw Error("you are not privileged to build derivations"); + auto res = store->buildDerivation(drvPath, drv, buildMode); + logger->stopWork(); + to << res.status << res.errorMsg; + break; + } + + case wopEnsurePath: { + Path path = readStorePath(*store, from); + logger->startWork(); + store->ensurePath(path); + logger->stopWork(); + to << 1; + break; + } + + case wopAddTempRoot: { + Path path = readStorePath(*store, from); + logger->startWork(); + store->addTempRoot(path); + logger->stopWork(); + to << 1; + break; + } + + case wopAddIndirectRoot: { + Path path = absPath(readString(from)); + logger->startWork(); + store->addIndirectRoot(path); + logger->stopWork(); + to << 1; + break; + } + + case wopSyncWithGC: { + logger->startWork(); + store->syncWithGC(); + logger->stopWork(); + to << 1; + break; + } + + case wopFindRoots: { + logger->startWork(); + Roots roots = store->findRoots(!trusted); + logger->stopWork(); + + size_t size = 0; + for (auto & i : roots) + size += i.second.size(); + + to << size; + + for (auto & [target, links] : roots) + for (auto & link : links) + to << link << target; + + break; + } + + case wopCollectGarbage: { + GCOptions options; + options.action = (GCOptions::GCAction) readInt(from); + options.pathsToDelete = readStorePaths(*store, from); + from >> options.ignoreLiveness >> options.maxFreed; + // obsolete fields + readInt(from); + readInt(from); + readInt(from); + + GCResults results; + + logger->startWork(); + if (options.ignoreLiveness) + throw Error("you are not allowed to ignore liveness"); + store->collectGarbage(options, results); + logger->stopWork(); + + to << results.paths << results.bytesFreed << 0 /* obsolete */; + + break; + } + + case wopSetOptions: { + settings.keepFailed = readInt(from); + settings.keepGoing = readInt(from); + settings.tryFallback = readInt(from); + verbosity = (Verbosity) readInt(from); + settings.maxBuildJobs.assign(readInt(from)); + settings.maxSilentTime = readInt(from); + readInt(from); // obsolete useBuildHook + settings.verboseBuild = lvlError == (Verbosity) readInt(from); + readInt(from); // obsolete logType + readInt(from); // obsolete printBuildTrace + settings.buildCores = readInt(from); + settings.useSubstitutes = readInt(from); + + StringMap overrides; + if (GET_PROTOCOL_MINOR(clientVersion) >= 12) { + unsigned int n = readInt(from); + for (unsigned int i = 0; i < n; i++) { + string name = readString(from); + string value = readString(from); + overrides.emplace(name, value); + } + } + + logger->startWork(); + + for (auto & i : overrides) { + auto & name(i.first); + auto & value(i.second); + + auto setSubstituters = [&](Setting & res) { + if (name != res.name && res.aliases.count(name) == 0) + return false; + StringSet trusted = settings.trustedSubstituters; + for (auto & s : settings.substituters.get()) + trusted.insert(s); + Strings subs; + auto ss = tokenizeString(value); + for (auto & s : ss) + if (trusted.count(s)) + subs.push_back(s); + else + warn("ignoring untrusted substituter '%s'", s); + res = subs; + return true; + }; + + try { + if (name == "ssh-auth-sock") // obsolete + ; + else if (trusted + || name == settings.buildTimeout.name + || name == "connect-timeout" + || (name == "builders" && value == "")) + settings.set(name, value); + else if (setSubstituters(settings.substituters)) + ; + else if (setSubstituters(settings.extraSubstituters)) + ; + else + warn("ignoring the user-specified setting '%s', because it is a restricted setting and you are not a trusted user", name); + } catch (UsageError & e) { + warn(e.what()); + } + } + + logger->stopWork(); + break; + } + + case wopQuerySubstitutablePathInfo: { + Path path = absPath(readString(from)); + logger->startWork(); + SubstitutablePathInfos infos; + store->querySubstitutablePathInfos({path}, infos); + logger->stopWork(); + SubstitutablePathInfos::iterator i = infos.find(path); + if (i == infos.end()) + to << 0; + else { + to << 1 << i->second.deriver << i->second.references << i->second.downloadSize << i->second.narSize; + } + break; + } + + case wopQuerySubstitutablePathInfos: { + PathSet paths = readStorePaths(*store, from); + logger->startWork(); + SubstitutablePathInfos infos; + store->querySubstitutablePathInfos(paths, infos); + logger->stopWork(); + to << infos.size(); + for (auto & i : infos) { + to << i.first << i.second.deriver << i.second.references + << i.second.downloadSize << i.second.narSize; + } + break; + } + + case wopQueryAllValidPaths: { + logger->startWork(); + PathSet paths = store->queryAllValidPaths(); + logger->stopWork(); + to << paths; + break; + } + + case wopQueryPathInfo: { + Path path = readStorePath(*store, from); + std::shared_ptr info; + logger->startWork(); + try { + info = store->queryPathInfo(path); + } catch (InvalidPath &) { + if (GET_PROTOCOL_MINOR(clientVersion) < 17) throw; + } + logger->stopWork(); + if (info) { + if (GET_PROTOCOL_MINOR(clientVersion) >= 17) + to << 1; + to << info->deriver << info->narHash.to_string(Base16, false) << info->references + << info->registrationTime << info->narSize; + if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { + to << info->ultimate + << info->sigs + << info->ca; + } + } else { + assert(GET_PROTOCOL_MINOR(clientVersion) >= 17); + to << 0; + } + break; + } + + case wopOptimiseStore: + logger->startWork(); + store->optimiseStore(); + logger->stopWork(); + to << 1; + break; + + case wopVerifyStore: { + bool checkContents, repair; + from >> checkContents >> repair; + logger->startWork(); + if (repair && !trusted) + throw Error("you are not privileged to repair paths"); + bool errors = store->verifyStore(checkContents, (RepairFlag) repair); + logger->stopWork(); + to << errors; + break; + } + + case wopAddSignatures: { + Path path = readStorePath(*store, from); + StringSet sigs = readStrings(from); + logger->startWork(); + if (!trusted) + throw Error("you are not privileged to add signatures"); + store->addSignatures(path, sigs); + logger->stopWork(); + to << 1; + break; + } + + case wopNarFromPath: { + auto path = readStorePath(*store, from); + logger->startWork(); + logger->stopWork(); + dumpPath(path, to); + break; + } + + case wopAddToStoreNar: { + bool repair, dontCheckSigs; + ValidPathInfo info; + info.path = readStorePath(*store, from); + from >> info.deriver; + if (!info.deriver.empty()) + store->assertStorePath(info.deriver); + info.narHash = Hash(readString(from), htSHA256); + info.references = readStorePaths(*store, from); + from >> info.registrationTime >> info.narSize >> info.ultimate; + info.sigs = readStrings(from); + from >> info.ca >> repair >> dontCheckSigs; + if (!trusted && dontCheckSigs) + dontCheckSigs = false; + if (!trusted) + info.ultimate = false; + + std::string saved; + std::unique_ptr source; + if (GET_PROTOCOL_MINOR(clientVersion) >= 21) + source = std::make_unique(from, to); + else { + TeeSink tee(from); + parseDump(tee, tee.source); + saved = std::move(*tee.source.data); + source = std::make_unique(saved); + } + + logger->startWork(); + + // FIXME: race if addToStore doesn't read source? + store->addToStore(info, *source, (RepairFlag) repair, + dontCheckSigs ? NoCheckSigs : CheckSigs, nullptr); + + logger->stopWork(); + break; + } + + case wopQueryMissing: { + PathSet targets = readStorePaths(*store, from); + logger->startWork(); + PathSet willBuild, willSubstitute, unknown; + unsigned long long downloadSize, narSize; + store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize); + logger->stopWork(); + to << willBuild << willSubstitute << unknown << downloadSize << narSize; + break; + } + + default: + throw Error(format("invalid operation %1%") % op); + } +} + +void processConnection( + FdSource & from, + FdSink & to, + bool trusted, + const std::string & userName, + uid_t userId) +{ + MonitorFdHup monitor(from.fd); + + /* Exchange the greeting. */ + unsigned int magic = readInt(from); + if (magic != WORKER_MAGIC_1) throw Error("protocol mismatch"); + to << WORKER_MAGIC_2 << PROTOCOL_VERSION; + to.flush(); + unsigned int clientVersion = readInt(from); + + if (clientVersion < 0x10a) + throw Error("the Nix client version is too old"); + + auto tunnelLogger = new TunnelLogger(to, clientVersion); + auto prevLogger = nix::logger; + logger = tunnelLogger; + + unsigned int opCount = 0; + + Finally finally([&]() { + _isInterrupted = false; + prevLogger->log(lvlDebug, fmt("%d operations", opCount)); + }); + + if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from)) + setAffinityTo(readInt(from)); + + readInt(from); // obsolete reserveSpace + + /* Send startup error messages to the client. */ + tunnelLogger->startWork(); + + try { + + /* If we can't accept clientVersion, then throw an error + *here* (not above). */ + +#if 0 + /* Prevent users from doing something very dangerous. */ + if (geteuid() == 0 && + querySetting("build-users-group", "") == "") + throw Error("if you run 'nix-daemon' as root, then you MUST set 'build-users-group'!"); +#endif + + /* Open the store. */ + Store::Params params; // FIXME: get params from somewhere + // Disable caching since the client already does that. + params["path-info-cache-size"] = "0"; + auto store = openStore(settings.storeUri, params); + + store->createUser(userName, userId); + + tunnelLogger->stopWork(); + to.flush(); + + /* Process client requests. */ + while (true) { + WorkerOp op; + try { + op = (WorkerOp) readInt(from); + } catch (Interrupted & e) { + break; + } catch (EndOfFile & e) { + break; + } + + opCount++; + + try { + performOp(tunnelLogger, store, trusted, clientVersion, from, to, op); + } catch (Error & e) { + /* If we're not in a state where we can send replies, then + something went wrong processing the input of the + client. This can happen especially if I/O errors occur + during addTextToStore() / importPath(). If that + happens, just send the error message and exit. */ + bool errorAllowed = tunnelLogger->state_.lock()->canSendStderr; + tunnelLogger->stopWork(false, e.msg(), e.status); + if (!errorAllowed) throw; + } catch (std::bad_alloc & e) { + tunnelLogger->stopWork(false, "Nix daemon out of memory", 1); + throw; + } + + to.flush(); + + assert(!tunnelLogger->state_.lock()->canSendStderr); + }; + + } catch (std::exception & e) { + tunnelLogger->stopWork(false, e.what(), 1); + to.flush(); + return; + } +} + +} diff --git a/src/libstore/daemon.hh b/src/libstore/daemon.hh new file mode 100644 index 000000000..6d4015e34 --- /dev/null +++ b/src/libstore/daemon.hh @@ -0,0 +1,12 @@ +#include "serialise.hh" + +namespace nix::daemon { + +void processConnection( + FdSource & from, + FdSink & to, + bool trusted, + const std::string & userName, + uid_t userId); + +} diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 5ebdfaf13..a03edd10f 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -62,6 +62,9 @@ typedef enum { #define STDERR_RESULT 0x52534c54 +class Store; +class Source; + Path readStorePath(Store & store, Source & from); template T readStorePaths(Store & store, Source & from); diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index cd18489b0..799d2bacf 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -2,14 +2,12 @@ #include "local-store.hh" #include "util.hh" #include "serialise.hh" -#include "worker-protocol.hh" #include "archive.hh" -#include "affinity.hh" #include "globals.hh" -#include "monitor-fd.hh" #include "derivations.hh" #include "finally.hh" #include "legacy.hh" +#include "daemon.hh" #include @@ -32,6 +30,7 @@ #endif using namespace nix; +using namespace nix::daemon; #ifndef __linux__ #define SPLICE_F_MOVE 0 @@ -53,793 +52,6 @@ static ssize_t splice(int fd_in, void *off_in, int fd_out, void *off_out, size_t } #endif -static FdSource from(STDIN_FILENO); -static FdSink to(STDOUT_FILENO); - - -Sink & operator << (Sink & sink, const Logger::Fields & fields) -{ - sink << fields.size(); - for (auto & f : fields) { - sink << f.type; - if (f.type == Logger::Field::tInt) - sink << f.i; - else if (f.type == Logger::Field::tString) - sink << f.s; - else abort(); - } - return sink; -} - - -/* Logger that forwards log messages to the client, *if* we're in a - state where the protocol allows it (i.e., when canSendStderr is - true). */ -struct TunnelLogger : public Logger -{ - struct State - { - bool canSendStderr = false; - std::vector pendingMsgs; - }; - - Sync state_; - - unsigned int clientVersion; - - TunnelLogger(unsigned int clientVersion) : clientVersion(clientVersion) { } - - void enqueueMsg(const std::string & s) - { - auto state(state_.lock()); - - if (state->canSendStderr) { - assert(state->pendingMsgs.empty()); - try { - to(s); - to.flush(); - } catch (...) { - /* Write failed; that means that the other side is - gone. */ - state->canSendStderr = false; - throw; - } - } else - state->pendingMsgs.push_back(s); - } - - void log(Verbosity lvl, const FormatOrString & fs) override - { - if (lvl > verbosity) return; - - StringSink buf; - buf << STDERR_NEXT << (fs.s + "\n"); - enqueueMsg(*buf.s); - } - - /* startWork() means that we're starting an operation for which we - want to send out stderr to the client. */ - void startWork() - { - auto state(state_.lock()); - state->canSendStderr = true; - - for (auto & msg : state->pendingMsgs) - to(msg); - - state->pendingMsgs.clear(); - - to.flush(); - } - - /* stopWork() means that we're done; stop sending stderr to the - client. */ - void stopWork(bool success = true, const string & msg = "", unsigned int status = 0) - { - auto state(state_.lock()); - - state->canSendStderr = false; - - if (success) - to << STDERR_LAST; - else { - to << STDERR_ERROR << msg; - if (status != 0) to << status; - } - } - - void startActivity(ActivityId act, Verbosity lvl, ActivityType type, - const std::string & s, const Fields & fields, ActivityId parent) override - { - if (GET_PROTOCOL_MINOR(clientVersion) < 20) { - if (!s.empty()) - log(lvl, s + "..."); - return; - } - - StringSink buf; - buf << STDERR_START_ACTIVITY << act << lvl << type << s << fields << parent; - enqueueMsg(*buf.s); - } - - void stopActivity(ActivityId act) override - { - if (GET_PROTOCOL_MINOR(clientVersion) < 20) return; - StringSink buf; - buf << STDERR_STOP_ACTIVITY << act; - enqueueMsg(*buf.s); - } - - void result(ActivityId act, ResultType type, const Fields & fields) override - { - if (GET_PROTOCOL_MINOR(clientVersion) < 20) return; - StringSink buf; - buf << STDERR_RESULT << act << type << fields; - enqueueMsg(*buf.s); - } -}; - - -struct TunnelSink : Sink -{ - Sink & to; - TunnelSink(Sink & to) : to(to) { } - virtual void operator () (const unsigned char * data, size_t len) - { - to << STDERR_WRITE; - writeString(data, len, to); - } -}; - - -struct TunnelSource : BufferedSource -{ - Source & from; - TunnelSource(Source & from) : from(from) { } -protected: - size_t readUnbuffered(unsigned char * data, size_t len) override - { - to << STDERR_READ << len; - to.flush(); - size_t n = readString(data, len, from); - if (n == 0) throw EndOfFile("unexpected end-of-file"); - return n; - } -}; - - -/* If the NAR archive contains a single file at top-level, then save - the contents of the file to `s'. Otherwise barf. */ -struct RetrieveRegularNARSink : ParseSink -{ - bool regular; - string s; - - RetrieveRegularNARSink() : regular(true) { } - - void createDirectory(const Path & path) - { - regular = false; - } - - void receiveContents(unsigned char * data, unsigned int len) - { - s.append((const char *) data, len); - } - - void createSymlink(const Path & path, const string & target) - { - regular = false; - } -}; - - -static void performOp(TunnelLogger * logger, ref store, - bool trusted, unsigned int clientVersion, - Source & from, Sink & to, unsigned int op) -{ - switch (op) { - - case wopIsValidPath: { - /* 'readStorePath' could raise an error leading to the connection - being closed. To be able to recover from an invalid path error, - call 'startWork' early, and do 'assertStorePath' afterwards so - that the 'Error' exception handler doesn't close the - connection. */ - Path path = readString(from); - logger->startWork(); - store->assertStorePath(path); - bool result = store->isValidPath(path); - logger->stopWork(); - to << result; - break; - } - - case wopQueryValidPaths: { - PathSet paths = readStorePaths(*store, from); - logger->startWork(); - PathSet res = store->queryValidPaths(paths); - logger->stopWork(); - to << res; - break; - } - - case wopHasSubstitutes: { - Path path = readStorePath(*store, from); - logger->startWork(); - PathSet res = store->querySubstitutablePaths({path}); - logger->stopWork(); - to << (res.find(path) != res.end()); - break; - } - - case wopQuerySubstitutablePaths: { - PathSet paths = readStorePaths(*store, from); - logger->startWork(); - PathSet res = store->querySubstitutablePaths(paths); - logger->stopWork(); - to << res; - break; - } - - case wopQueryPathHash: { - Path path = readStorePath(*store, from); - logger->startWork(); - auto hash = store->queryPathInfo(path)->narHash; - logger->stopWork(); - to << hash.to_string(Base16, false); - break; - } - - case wopQueryReferences: - case wopQueryReferrers: - case wopQueryValidDerivers: - case wopQueryDerivationOutputs: { - Path path = readStorePath(*store, from); - logger->startWork(); - PathSet paths; - if (op == wopQueryReferences) - paths = store->queryPathInfo(path)->references; - else if (op == wopQueryReferrers) - store->queryReferrers(path, paths); - else if (op == wopQueryValidDerivers) - paths = store->queryValidDerivers(path); - else paths = store->queryDerivationOutputs(path); - logger->stopWork(); - to << paths; - break; - } - - case wopQueryDerivationOutputNames: { - Path path = readStorePath(*store, from); - logger->startWork(); - StringSet names; - names = store->queryDerivationOutputNames(path); - logger->stopWork(); - to << names; - break; - } - - case wopQueryDeriver: { - Path path = readStorePath(*store, from); - logger->startWork(); - auto deriver = store->queryPathInfo(path)->deriver; - logger->stopWork(); - to << deriver; - break; - } - - case wopQueryPathFromHashPart: { - string hashPart = readString(from); - logger->startWork(); - Path path = store->queryPathFromHashPart(hashPart); - logger->stopWork(); - to << path; - break; - } - - case wopAddToStore: { - bool fixed, recursive; - std::string s, baseName; - from >> baseName >> fixed /* obsolete */ >> recursive >> s; - /* Compatibility hack. */ - if (!fixed) { - s = "sha256"; - recursive = true; - } - HashType hashAlgo = parseHashType(s); - - TeeSource savedNAR(from); - RetrieveRegularNARSink savedRegular; - - if (recursive) { - /* Get the entire NAR dump from the client and save it to - a string so that we can pass it to - addToStoreFromDump(). */ - ParseSink sink; /* null sink; just parse the NAR */ - parseDump(sink, savedNAR); - } else - parseDump(savedRegular, from); - - logger->startWork(); - if (!savedRegular.regular) throw Error("regular file expected"); - - auto store2 = store.dynamic_pointer_cast(); - if (!store2) throw Error("operation is only supported by LocalStore"); - - Path path = store2->addToStoreFromDump(recursive ? *savedNAR.data : savedRegular.s, baseName, recursive, hashAlgo); - logger->stopWork(); - - to << path; - break; - } - - case wopAddTextToStore: { - string suffix = readString(from); - string s = readString(from); - PathSet refs = readStorePaths(*store, from); - logger->startWork(); - Path path = store->addTextToStore(suffix, s, refs, NoRepair); - logger->stopWork(); - to << path; - break; - } - - case wopExportPath: { - Path path = readStorePath(*store, from); - readInt(from); // obsolete - logger->startWork(); - TunnelSink sink(to); - store->exportPath(path, sink); - logger->stopWork(); - to << 1; - break; - } - - case wopImportPaths: { - logger->startWork(); - TunnelSource source(from); - Paths paths = store->importPaths(source, nullptr, - trusted ? NoCheckSigs : CheckSigs); - logger->stopWork(); - to << paths; - break; - } - - case wopBuildPaths: { - PathSet drvs = readStorePaths(*store, from); - BuildMode mode = bmNormal; - if (GET_PROTOCOL_MINOR(clientVersion) >= 15) { - mode = (BuildMode) readInt(from); - - /* Repairing is not atomic, so disallowed for "untrusted" - clients. */ - if (mode == bmRepair && !trusted) - throw Error("repairing is not allowed because you are not in 'trusted-users'"); - } - logger->startWork(); - store->buildPaths(drvs, mode); - logger->stopWork(); - to << 1; - break; - } - - case wopBuildDerivation: { - Path drvPath = readStorePath(*store, from); - BasicDerivation drv; - readDerivation(from, *store, drv); - BuildMode buildMode = (BuildMode) readInt(from); - logger->startWork(); - if (!trusted) - throw Error("you are not privileged to build derivations"); - auto res = store->buildDerivation(drvPath, drv, buildMode); - logger->stopWork(); - to << res.status << res.errorMsg; - break; - } - - case wopEnsurePath: { - Path path = readStorePath(*store, from); - logger->startWork(); - store->ensurePath(path); - logger->stopWork(); - to << 1; - break; - } - - case wopAddTempRoot: { - Path path = readStorePath(*store, from); - logger->startWork(); - store->addTempRoot(path); - logger->stopWork(); - to << 1; - break; - } - - case wopAddIndirectRoot: { - Path path = absPath(readString(from)); - logger->startWork(); - store->addIndirectRoot(path); - logger->stopWork(); - to << 1; - break; - } - - case wopSyncWithGC: { - logger->startWork(); - store->syncWithGC(); - logger->stopWork(); - to << 1; - break; - } - - case wopFindRoots: { - logger->startWork(); - Roots roots = store->findRoots(!trusted); - logger->stopWork(); - - size_t size = 0; - for (auto & i : roots) - size += i.second.size(); - - to << size; - - for (auto & [target, links] : roots) - for (auto & link : links) - to << link << target; - - break; - } - - case wopCollectGarbage: { - GCOptions options; - options.action = (GCOptions::GCAction) readInt(from); - options.pathsToDelete = readStorePaths(*store, from); - from >> options.ignoreLiveness >> options.maxFreed; - // obsolete fields - readInt(from); - readInt(from); - readInt(from); - - GCResults results; - - logger->startWork(); - if (options.ignoreLiveness) - throw Error("you are not allowed to ignore liveness"); - store->collectGarbage(options, results); - logger->stopWork(); - - to << results.paths << results.bytesFreed << 0 /* obsolete */; - - break; - } - - case wopSetOptions: { - settings.keepFailed = readInt(from); - settings.keepGoing = readInt(from); - settings.tryFallback = readInt(from); - verbosity = (Verbosity) readInt(from); - settings.maxBuildJobs.assign(readInt(from)); - settings.maxSilentTime = readInt(from); - readInt(from); // obsolete useBuildHook - settings.verboseBuild = lvlError == (Verbosity) readInt(from); - readInt(from); // obsolete logType - readInt(from); // obsolete printBuildTrace - settings.buildCores = readInt(from); - settings.useSubstitutes = readInt(from); - - StringMap overrides; - if (GET_PROTOCOL_MINOR(clientVersion) >= 12) { - unsigned int n = readInt(from); - for (unsigned int i = 0; i < n; i++) { - string name = readString(from); - string value = readString(from); - overrides.emplace(name, value); - } - } - - logger->startWork(); - - for (auto & i : overrides) { - auto & name(i.first); - auto & value(i.second); - - auto setSubstituters = [&](Setting & res) { - if (name != res.name && res.aliases.count(name) == 0) - return false; - StringSet trusted = settings.trustedSubstituters; - for (auto & s : settings.substituters.get()) - trusted.insert(s); - Strings subs; - auto ss = tokenizeString(value); - for (auto & s : ss) - if (trusted.count(s)) - subs.push_back(s); - else - warn("ignoring untrusted substituter '%s'", s); - res = subs; - return true; - }; - - try { - if (name == "ssh-auth-sock") // obsolete - ; - else if (trusted - || name == settings.buildTimeout.name - || name == "connect-timeout" - || (name == "builders" && value == "")) - settings.set(name, value); - else if (setSubstituters(settings.substituters)) - ; - else if (setSubstituters(settings.extraSubstituters)) - ; - else - warn("ignoring the user-specified setting '%s', because it is a restricted setting and you are not a trusted user", name); - } catch (UsageError & e) { - warn(e.what()); - } - } - - logger->stopWork(); - break; - } - - case wopQuerySubstitutablePathInfo: { - Path path = absPath(readString(from)); - logger->startWork(); - SubstitutablePathInfos infos; - store->querySubstitutablePathInfos({path}, infos); - logger->stopWork(); - SubstitutablePathInfos::iterator i = infos.find(path); - if (i == infos.end()) - to << 0; - else { - to << 1 << i->second.deriver << i->second.references << i->second.downloadSize << i->second.narSize; - } - break; - } - - case wopQuerySubstitutablePathInfos: { - PathSet paths = readStorePaths(*store, from); - logger->startWork(); - SubstitutablePathInfos infos; - store->querySubstitutablePathInfos(paths, infos); - logger->stopWork(); - to << infos.size(); - for (auto & i : infos) { - to << i.first << i.second.deriver << i.second.references - << i.second.downloadSize << i.second.narSize; - } - break; - } - - case wopQueryAllValidPaths: { - logger->startWork(); - PathSet paths = store->queryAllValidPaths(); - logger->stopWork(); - to << paths; - break; - } - - case wopQueryPathInfo: { - Path path = readStorePath(*store, from); - std::shared_ptr info; - logger->startWork(); - try { - info = store->queryPathInfo(path); - } catch (InvalidPath &) { - if (GET_PROTOCOL_MINOR(clientVersion) < 17) throw; - } - logger->stopWork(); - if (info) { - if (GET_PROTOCOL_MINOR(clientVersion) >= 17) - to << 1; - to << info->deriver << info->narHash.to_string(Base16, false) << info->references - << info->registrationTime << info->narSize; - if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { - to << info->ultimate - << info->sigs - << info->ca; - } - } else { - assert(GET_PROTOCOL_MINOR(clientVersion) >= 17); - to << 0; - } - break; - } - - case wopOptimiseStore: - logger->startWork(); - store->optimiseStore(); - logger->stopWork(); - to << 1; - break; - - case wopVerifyStore: { - bool checkContents, repair; - from >> checkContents >> repair; - logger->startWork(); - if (repair && !trusted) - throw Error("you are not privileged to repair paths"); - bool errors = store->verifyStore(checkContents, (RepairFlag) repair); - logger->stopWork(); - to << errors; - break; - } - - case wopAddSignatures: { - Path path = readStorePath(*store, from); - StringSet sigs = readStrings(from); - logger->startWork(); - if (!trusted) - throw Error("you are not privileged to add signatures"); - store->addSignatures(path, sigs); - logger->stopWork(); - to << 1; - break; - } - - case wopNarFromPath: { - auto path = readStorePath(*store, from); - logger->startWork(); - logger->stopWork(); - dumpPath(path, to); - break; - } - - case wopAddToStoreNar: { - bool repair, dontCheckSigs; - ValidPathInfo info; - info.path = readStorePath(*store, from); - from >> info.deriver; - if (!info.deriver.empty()) - store->assertStorePath(info.deriver); - info.narHash = Hash(readString(from), htSHA256); - info.references = readStorePaths(*store, from); - from >> info.registrationTime >> info.narSize >> info.ultimate; - info.sigs = readStrings(from); - from >> info.ca >> repair >> dontCheckSigs; - if (!trusted && dontCheckSigs) - dontCheckSigs = false; - if (!trusted) - info.ultimate = false; - - std::string saved; - std::unique_ptr source; - if (GET_PROTOCOL_MINOR(clientVersion) >= 21) - source = std::make_unique(from); - else { - TeeSink tee(from); - parseDump(tee, tee.source); - saved = std::move(*tee.source.data); - source = std::make_unique(saved); - } - - logger->startWork(); - - // FIXME: race if addToStore doesn't read source? - store->addToStore(info, *source, (RepairFlag) repair, - dontCheckSigs ? NoCheckSigs : CheckSigs, nullptr); - - logger->stopWork(); - break; - } - - case wopQueryMissing: { - PathSet targets = readStorePaths(*store, from); - logger->startWork(); - PathSet willBuild, willSubstitute, unknown; - unsigned long long downloadSize, narSize; - store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize); - logger->stopWork(); - to << willBuild << willSubstitute << unknown << downloadSize << narSize; - break; - } - - default: - throw Error(format("invalid operation %1%") % op); - } -} - - -static void processConnection(bool trusted, - const std::string & userName, uid_t userId) -{ - MonitorFdHup monitor(from.fd); - - /* Exchange the greeting. */ - unsigned int magic = readInt(from); - if (magic != WORKER_MAGIC_1) throw Error("protocol mismatch"); - to << WORKER_MAGIC_2 << PROTOCOL_VERSION; - to.flush(); - unsigned int clientVersion = readInt(from); - - if (clientVersion < 0x10a) - throw Error("the Nix client version is too old"); - - auto tunnelLogger = new TunnelLogger(clientVersion); - auto prevLogger = nix::logger; - logger = tunnelLogger; - - unsigned int opCount = 0; - - Finally finally([&]() { - _isInterrupted = false; - prevLogger->log(lvlDebug, fmt("%d operations", opCount)); - }); - - if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from)) - setAffinityTo(readInt(from)); - - readInt(from); // obsolete reserveSpace - - /* Send startup error messages to the client. */ - tunnelLogger->startWork(); - - try { - - /* If we can't accept clientVersion, then throw an error - *here* (not above). */ - -#if 0 - /* Prevent users from doing something very dangerous. */ - if (geteuid() == 0 && - querySetting("build-users-group", "") == "") - throw Error("if you run 'nix-daemon' as root, then you MUST set 'build-users-group'!"); -#endif - - /* Open the store. */ - Store::Params params; // FIXME: get params from somewhere - // Disable caching since the client already does that. - params["path-info-cache-size"] = "0"; - auto store = openStore(settings.storeUri, params); - - store->createUser(userName, userId); - - tunnelLogger->stopWork(); - to.flush(); - - /* Process client requests. */ - while (true) { - WorkerOp op; - try { - op = (WorkerOp) readInt(from); - } catch (Interrupted & e) { - break; - } catch (EndOfFile & e) { - break; - } - - opCount++; - - try { - performOp(tunnelLogger, store, trusted, clientVersion, from, to, op); - } catch (Error & e) { - /* If we're not in a state where we can send replies, then - something went wrong processing the input of the - client. This can happen especially if I/O errors occur - during addTextToStore() / importPath(). If that - happens, just send the error message and exit. */ - bool errorAllowed = tunnelLogger->state_.lock()->canSendStderr; - tunnelLogger->stopWork(false, e.msg(), e.status); - if (!errorAllowed) throw; - } catch (std::bad_alloc & e) { - tunnelLogger->stopWork(false, "Nix daemon out of memory", 1); - throw; - } - - to.flush(); - - assert(!tunnelLogger->state_.lock()->canSendStderr); - }; - - } catch (std::exception & e) { - tunnelLogger->stopWork(false, e.what(), 1); - to.flush(); - return; - } -} - static void sigChldHandler(int sigNo) { @@ -1054,9 +266,9 @@ static void daemonLoop(char * * argv) } /* Handle the connection. */ - from.fd = remote.get(); - to.fd = remote.get(); - processConnection(trusted, user, peer.uid); + FdSource from(remote.get()); + FdSink to(remote.get()); + processConnection(from, to, trusted, user, peer.uid); exit(0); }, options); @@ -1136,7 +348,9 @@ static int _main(int argc, char * * argv) } } } else { - processConnection(true, "root", 0); + FdSource from(STDIN_FILENO); + FdSink to(STDOUT_FILENO); + processConnection(from, to, true, "root", 0); } } else { daemonLoop(argv); From 63b99af85a1e280876da3d807e4bc96a5c5dde39 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 25 Sep 2018 12:36:11 +0200 Subject: [PATCH 14/23] Move Unix domain socket creation to libutil Also drop multithread-unfriendly hacks like doing a temporary chmod/umask. --- src/libutil/util.cc | 31 +++++++++++++++++++++++++ src/libutil/util.hh | 4 ++++ src/nix-daemon/nix-daemon.cc | 44 +++--------------------------------- 3 files changed, 38 insertions(+), 41 deletions(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 2e416edef..2a1272885 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -21,7 +21,9 @@ #include #include #include +#include #include +#include #include #ifdef __APPLE__ @@ -1562,4 +1564,33 @@ std::unique_ptr createInterruptCallback(std::function return std::unique_ptr(res.release()); } + +AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode) +{ + AutoCloseFD fdSocket = socket(PF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0); + if (!fdSocket) + throw SysError("cannot create Unix domain socket"); + + closeOnExec(fdSocket.get()); + + struct sockaddr_un addr; + addr.sun_family = AF_UNIX; + if (path.size() >= sizeof(addr.sun_path)) + throw Error("socket path '%1%' is too long", path); + strcpy(addr.sun_path, path.c_str()); + + unlink(path.c_str()); + + if (bind(fdSocket.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1) + throw SysError("cannot bind to socket '%1%'", path); + + if (chmod(path.c_str(), mode) == -1) + throw SysError("changing permissions on '%1%'", path); + + if (listen(fdSocket.get(), 5) == -1) + throw SysError("cannot listen on socket '%1%'", path); + + return fdSocket; +} + } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 3493e80b5..0bb171268 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -553,4 +553,8 @@ typedef std::function PathFilter; extern PathFilter defaultPathFilter; +/* Create a Unix domain socket in listen mode. */ +AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode); + + } diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 799d2bacf..67bf4dfeb 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -156,53 +156,15 @@ static void daemonLoop(char * * argv) if (getEnv("LISTEN_PID") != std::to_string(getpid()) || getEnv("LISTEN_FDS") != "1") throw Error("unexpected systemd environment variables"); fdSocket = SD_LISTEN_FDS_START; + closeOnExec(fdSocket.get()); } /* Otherwise, create and bind to a Unix domain socket. */ else { - - /* Create and bind to a Unix domain socket. */ - fdSocket = socket(PF_UNIX, SOCK_STREAM, 0); - if (!fdSocket) - throw SysError("cannot create Unix domain socket"); - - string socketPath = settings.nixDaemonSocketFile; - - createDirs(dirOf(socketPath)); - - /* Urgh, sockaddr_un allows path names of only 108 characters. - So chdir to the socket directory so that we can pass a - relative path name. */ - if (chdir(dirOf(socketPath).c_str()) == -1) - throw SysError("cannot change current directory"); - Path socketPathRel = "./" + baseNameOf(socketPath); - - struct sockaddr_un addr; - addr.sun_family = AF_UNIX; - if (socketPathRel.size() >= sizeof(addr.sun_path)) - throw Error(format("socket path '%1%' is too long") % socketPathRel); - strcpy(addr.sun_path, socketPathRel.c_str()); - - unlink(socketPath.c_str()); - - /* Make sure that the socket is created with 0666 permission - (everybody can connect --- provided they have access to the - directory containing the socket). */ - mode_t oldMode = umask(0111); - int res = bind(fdSocket.get(), (struct sockaddr *) &addr, sizeof(addr)); - umask(oldMode); - if (res == -1) - throw SysError(format("cannot bind to socket '%1%'") % socketPath); - - if (chdir("/") == -1) /* back to the root */ - throw SysError("cannot change current directory"); - - if (listen(fdSocket.get(), 5) == -1) - throw SysError(format("cannot listen on socket '%1%'") % socketPath); + createDirs(dirOf(settings.nixDaemonSocketFile)); + fdSocket = createUnixDomainSocket(settings.nixDaemonSocketFile, 0666); } - closeOnExec(fdSocket.get()); - /* Loop accepting connections. */ while (1) { From 05819d013fb39eb9902d952978fff7f69e0e326e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 25 Sep 2018 12:49:20 +0200 Subject: [PATCH 15/23] Don't create a Store in processConnection() --- src/libstore/daemon.cc | 7 +------ src/libstore/daemon.hh | 2 ++ src/nix-daemon/nix-daemon.cc | 13 +++++++++++-- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 2e1d10bcc..cc1436f90 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -694,6 +694,7 @@ static void performOp(TunnelLogger * logger, ref store, } void processConnection( + ref store, FdSource & from, FdSink & to, bool trusted, @@ -743,12 +744,6 @@ void processConnection( throw Error("if you run 'nix-daemon' as root, then you MUST set 'build-users-group'!"); #endif - /* Open the store. */ - Store::Params params; // FIXME: get params from somewhere - // Disable caching since the client already does that. - params["path-info-cache-size"] = "0"; - auto store = openStore(settings.storeUri, params); - store->createUser(userName, userId); tunnelLogger->stopWork(); diff --git a/src/libstore/daemon.hh b/src/libstore/daemon.hh index 6d4015e34..fa26ba9c2 100644 --- a/src/libstore/daemon.hh +++ b/src/libstore/daemon.hh @@ -1,8 +1,10 @@ #include "serialise.hh" +#include "store-api.hh" namespace nix::daemon { void processConnection( + ref store, FdSource & from, FdSink & to, bool trusted, diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 67bf4dfeb..da17ddcba 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -140,6 +140,15 @@ static PeerInfo getPeerInfo(int remote) #define SD_LISTEN_FDS_START 3 +static ref openUncachedStore() +{ + Store::Params params; // FIXME: get params from somewhere + // Disable caching since the client already does that. + params["path-info-cache-size"] = "0"; + return openStore(settings.storeUri, params); +} + + static void daemonLoop(char * * argv) { if (chdir("/") == -1) @@ -230,7 +239,7 @@ static void daemonLoop(char * * argv) /* Handle the connection. */ FdSource from(remote.get()); FdSink to(remote.get()); - processConnection(from, to, trusted, user, peer.uid); + processConnection(openUncachedStore(), from, to, trusted, user, peer.uid); exit(0); }, options); @@ -312,7 +321,7 @@ static int _main(int argc, char * * argv) } else { FdSource from(STDIN_FILENO); FdSink to(STDOUT_FILENO); - processConnection(from, to, true, "root", 0); + processConnection(openUncachedStore(), from, to, true, "root", 0); } } else { daemonLoop(argv); From 992a2ad4757f43c02f3d06771c632a408d90ded0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 25 Sep 2018 18:55:37 +0200 Subject: [PATCH 16/23] Move addToStoreFromDump to Store --- src/libstore/daemon.cc | 5 +---- src/libstore/local-store.hh | 2 +- src/libstore/store-api.hh | 7 +++++++ 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index cc1436f90..64147027b 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -316,10 +316,7 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); if (!savedRegular.regular) throw Error("regular file expected"); - auto store2 = store.dynamic_pointer_cast(); - if (!store2) throw Error("operation is only supported by LocalStore"); - - Path path = store2->addToStoreFromDump(recursive ? *savedNAR.data : savedRegular.s, baseName, recursive, hashAlgo); + Path path = store->addToStoreFromDump(recursive ? *savedNAR.data : savedRegular.s, baseName, recursive, hashAlgo); logger->stopWork(); to << path; diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 379a06af8..77253fa4a 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -157,7 +157,7 @@ public: true) or simply the contents of a regular file (if recursive == false). */ Path addToStoreFromDump(const string & dump, const string & name, - bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair); + bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override; Path addTextToStore(const string & name, const string & s, const PathSet & references, RepairFlag repair) override; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 250e1a4bc..115b23cbf 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -422,6 +422,13 @@ public: bool recursive = true, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0; + // FIXME: remove? + virtual Path addToStoreFromDump(const string & dump, const string & name, + bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) + { + throw Error("addToStoreFromDump() is not supported by this store"); + } + /* Like addToStore, but the contents written to the output path is a regular file containing the given string. */ virtual Path addTextToStore(const string & name, const string & s, From e5319a87ce75bbd2dd88f57c3b470a396195e849 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 25 Sep 2018 18:54:16 +0200 Subject: [PATCH 17/23] queryPathInfoUncached(): Return const ValidPathInfo --- src/libstore/binary-cache-store.cc | 2 +- src/libstore/binary-cache-store.hh | 2 +- src/libstore/legacy-ssh-store.cc | 2 +- src/libstore/local-store.cc | 2 +- src/libstore/local-store.hh | 2 +- src/libstore/misc.cc | 2 +- src/libstore/nar-info-disk-cache.cc | 4 ++-- src/libstore/nar-info-disk-cache.hh | 2 +- src/libstore/remote-store.cc | 2 +- src/libstore/remote-store.hh | 2 +- src/libstore/store-api.cc | 16 ++++++++-------- src/libstore/store-api.hh | 6 +++--- 12 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 8e6f1f55d..7e3e5ff88 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -296,7 +296,7 @@ void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink) } void BinaryCacheStore::queryPathInfoUncached(const Path & storePath, - Callback> callback) noexcept + Callback> callback) noexcept { auto uri = getUri(); auto act = std::make_shared(*logger, lvlTalkative, actQueryPathInfo, diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index c77292294..2d7cd1947 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -74,7 +74,7 @@ public: bool isValidPathUncached(const Path & path) override; void queryPathInfoUncached(const Path & path, - Callback> callback) noexcept override; + Callback> callback) noexcept override; Path queryPathFromHashPart(const string & hashPart) override { unsupported("queryPathFromHashPart"); } diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index d5fbdd25a..1b8b5908c 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -88,7 +88,7 @@ struct LegacySSHStore : public Store } void queryPathInfoUncached(const Path & path, - Callback> callback) noexcept override + Callback> callback) noexcept override { try { auto conn(connections->get()); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 49061f27d..6bbe5433c 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -632,7 +632,7 @@ uint64_t LocalStore::addValidPath(State & state, void LocalStore::queryPathInfoUncached(const Path & path, - Callback> callback) noexcept + Callback> callback) noexcept { try { auto info = std::make_shared(); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 77253fa4a..5aa6b0519 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -127,7 +127,7 @@ public: PathSet queryAllValidPaths() override; void queryPathInfoUncached(const Path & path, - Callback> callback) noexcept override; + Callback> callback) noexcept override; void queryReferrers(const Path & path, PathSet & referrers) override; diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 05b93d4c9..2fceb9b9a 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -33,7 +33,7 @@ void Store::computeFSClosure(const PathSet & startPaths, state->pending++; } - queryPathInfo(path, {[&, path](std::future> fut) { + queryPathInfo(path, {[&, path](std::future> fut) { // FIXME: calls to isValidPath() should be async try { diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 32ad7f2b2..5bf982195 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -219,7 +219,7 @@ public: void upsertNarInfo( const std::string & uri, const std::string & hashPart, - std::shared_ptr info) override + std::shared_ptr info) override { retrySQLite([&]() { auto state(_state.lock()); @@ -228,7 +228,7 @@ public: if (info) { - auto narInfo = std::dynamic_pointer_cast(info); + auto narInfo = std::dynamic_pointer_cast(info); assert(hashPart == storePathToHash(info->path)); diff --git a/src/libstore/nar-info-disk-cache.hh b/src/libstore/nar-info-disk-cache.hh index 88d909732..11e6c55ca 100644 --- a/src/libstore/nar-info-disk-cache.hh +++ b/src/libstore/nar-info-disk-cache.hh @@ -21,7 +21,7 @@ public: virtual void upsertNarInfo( const std::string & uri, const std::string & hashPart, - std::shared_ptr info) = 0; + std::shared_ptr info) = 0; }; /* Return a singleton cache object that can be used concurrently by diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index f34369d8f..2a89b7c98 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -350,7 +350,7 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths, void RemoteStore::queryPathInfoUncached(const Path & path, - Callback> callback) noexcept + Callback> callback) noexcept { try { std::shared_ptr info; diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 1f375dd71..84de4c460 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -43,7 +43,7 @@ public: PathSet queryAllValidPaths() override; void queryPathInfoUncached(const Path & path, - Callback> callback) noexcept override; + Callback> callback) noexcept override; void queryReferrers(const Path & path, PathSet & referrers) override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 1a1b24e3b..54430d3ba 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -320,10 +320,10 @@ bool Store::isValidPathUncached(const Path & path) ref Store::queryPathInfo(const Path & storePath) { - std::promise> promise; + std::promise> promise; queryPathInfo(storePath, - {[&](std::future> result) { + {[&](std::future> result) { try { promise.set_value(result.get()); } catch (...) { @@ -336,7 +336,7 @@ ref Store::queryPathInfo(const Path & storePath) void Store::queryPathInfo(const Path & storePath, - Callback> callback) noexcept + Callback> callback) noexcept { std::string hashPart; @@ -351,7 +351,7 @@ void Store::queryPathInfo(const Path & storePath, stats.narInfoReadAverted++; if (!*res) throw InvalidPath(format("path '%s' is not valid") % storePath); - return callback(ref(*res)); + return callback(ref(*res)); } } @@ -367,7 +367,7 @@ void Store::queryPathInfo(const Path & storePath, (res.second->path != storePath && storePathToName(storePath) != "")) throw InvalidPath(format("path '%s' is not valid") % storePath); } - return callback(ref(res.second)); + return callback(ref(res.second)); } } @@ -376,7 +376,7 @@ void Store::queryPathInfo(const Path & storePath, auto callbackPtr = std::make_shared(std::move(callback)); queryPathInfoUncached(storePath, - {[this, storePath, hashPart, callbackPtr](std::future> fut) { + {[this, storePath, hashPart, callbackPtr](std::future> fut) { try { auto info = fut.get(); @@ -396,7 +396,7 @@ void Store::queryPathInfo(const Path & storePath, throw InvalidPath("path '%s' is not valid", storePath); } - (*callbackPtr)(ref(info)); + (*callbackPtr)(ref(info)); } catch (...) { callbackPtr->rethrow(); } }}); } @@ -418,7 +418,7 @@ PathSet Store::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubsti auto doQuery = [&](const Path & path ) { checkInterrupt(); - queryPathInfo(path, {[path, &state_, &wakeup](std::future> fut) { + queryPathInfo(path, {[path, &state_, &wakeup](std::future> fut) { auto state(state_.lock()); try { auto info = fut.get(); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 115b23cbf..e9e6e0dd2 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -258,7 +258,7 @@ protected: struct State { - LRUCache> pathInfoCache; + LRUCache> pathInfoCache; }; Sync state; @@ -361,12 +361,12 @@ public: /* Asynchronous version of queryPathInfo(). */ void queryPathInfo(const Path & path, - Callback> callback) noexcept; + Callback> callback) noexcept; protected: virtual void queryPathInfoUncached(const Path & path, - Callback> callback) noexcept = 0; + Callback> callback) noexcept = 0; public: From 0e9b72e097e027ff7f78e0f596a3ade606df9253 Mon Sep 17 00:00:00 2001 From: Will Dietz Date: Wed, 30 Oct 2019 14:38:02 -0500 Subject: [PATCH 18/23] minor: fix mismatch of struct/class forward decl of 'Source' Fixes the following warning and the indicate potential issue: src/libstore/worker-protocol.hh:66:1: warning: class 'Source' was previously declared as a struct; this is valid, but may result in linker errors under the Microsoft C++ ABI [-Wmismatched-tags] (cherry picked from commit 6e1bb04870b1b723282d32182af286646f13bf3c) --- src/libstore/worker-protocol.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index a03edd10f..6762b609d 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -63,7 +63,7 @@ typedef enum { class Store; -class Source; +struct Source; Path readStorePath(Store & store, Source & from); template T readStorePaths(Store & store, Source & from); From 80d5ec6ff4388b258278ee507b94694dc34d19f5 Mon Sep 17 00:00:00 2001 From: Benjamin Hipple Date: Sun, 27 Oct 2019 13:45:02 -0400 Subject: [PATCH 19/23] Minor updates to inline comments Add missing docstring on InstallableCommand. Also, some of these were wrapped when they're right next to a line longer than the unwrapped line, so we can just unwrap them to save vertical space. --- src/libexpr/primops/fetchGit.cc | 4 +--- src/libstore/build.cc | 12 ++++-------- src/libstore/local-store.cc | 3 +-- src/nix-store/nix-store.cc | 6 ++---- src/nix/command.hh | 3 ++- 5 files changed, 10 insertions(+), 18 deletions(-) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 90f600284..7ef3b3823 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -45,9 +45,7 @@ GitInfo exportGit(ref store, const std::string & uri, if (!clean) { - /* This is an unclean working tree. So copy all tracked - files. */ - + /* This is an unclean working tree. So copy all tracked files. */ GitInfo gitInfo; gitInfo.rev = "0000000000000000000000000000000000000000"; gitInfo.shortRev = std::string(gitInfo.rev, 0, 7); diff --git a/src/libstore/build.cc b/src/libstore/build.cc index dfae8024a..ada1cf8ec 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3264,8 +3264,7 @@ void DerivationGoal::registerOutputs() i.second.parseHashInfo(recursive, h); if (!recursive) { - /* The output path should be a regular file without - execute permission. */ + /* The output path should be a regular file without execute permission. */ if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) throw BuildError( format("output path '%1%' should be a non-executable regular file") % path); @@ -3343,8 +3342,7 @@ void DerivationGoal::registerOutputs() % drvPath % path); } - /* Since we verified the build, it's now ultimately - trusted. */ + /* Since we verified the build, it's now ultimately trusted. */ if (!info.ultimate) { info.ultimate = true; worker.store.signPathInfo(info); @@ -3354,8 +3352,7 @@ void DerivationGoal::registerOutputs() continue; } - /* For debugging, print out the referenced and unreferenced - paths. */ + /* For debugging, print out the referenced and unreferenced paths. */ for (auto & i : inputPaths) { PathSet::iterator j = references.find(i); if (j == references.end()) @@ -3413,8 +3410,7 @@ void DerivationGoal::registerOutputs() } } - /* If this is the first round of several, then move the output out - of the way. */ + /* If this is the first round of several, then move the output out of the way. */ if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) { for (auto & i : drv->outputs) { Path prev = i.second.path + checkSuffix; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 6bbe5433c..b5b0f1ba7 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -937,8 +937,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) not be valid yet. */ for (auto & i : infos) if (isDerivation(i.path)) { - // FIXME: inefficient; we already loaded the - // derivation in addValidPath(). + // FIXME: inefficient; we already loaded the derivation in addValidPath(). Derivation drv = readDerivation(realStoreDir + "/" + baseNameOf(i.path)); checkDerivationOutputs(i.path, drv); } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 19384499d..dfa473db5 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -632,8 +632,7 @@ static void opDelete(Strings opFlags, Strings opArgs) } -/* Dump a path as a Nix archive. The archive is written to standard - output. */ +/* Dump a path as a Nix archive. The archive is written to stdout */ static void opDump(Strings opFlags, Strings opArgs) { if (!opFlags.empty()) throw UsageError("unknown flag"); @@ -646,8 +645,7 @@ static void opDump(Strings opFlags, Strings opArgs) } -/* Restore a value from a Nix archive. The archive is read from - standard input. */ +/* Restore a value from a Nix archive. The archive is read from stdin. */ static void opRestore(Strings opFlags, Strings opArgs) { if (!opFlags.empty()) throw UsageError("unknown flag"); diff --git a/src/nix/command.hh b/src/nix/command.hh index 45ad1cd2a..fad404c73 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -34,7 +34,7 @@ struct Command : virtual Args class Store; -/* A command that require a Nix store. */ +/* A command that requires a Nix store. */ struct StoreCommand : virtual Command { StoreCommand(); @@ -115,6 +115,7 @@ private: std::vector _installables; }; +/* A command that operates on exactly one "installable" */ struct InstallableCommand : virtual Args, SourceExprCommand { std::shared_ptr installable; From f1782642d363330b9ce8607d942dcbb47414bf82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 31 Oct 2019 16:34:59 +0000 Subject: [PATCH 20/23] travis: enable linux builds Also disable email to not notify the whole NixOS community about build failures --- .travis.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 99218a963..ee4ea1ac6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,2 +1,8 @@ -os: osx -script: ./tests/install-darwin.sh +matrix: + include: + - language: osx + script: ./tests/install-darwin.sh + - language: nix + script: nix-build release.nix -A build.x86_64-linux +notifications: + email: false From b811bd2172bc9796f42b87996f455d4e5e4382ba Mon Sep 17 00:00:00 2001 From: ng0 Date: Fri, 1 Nov 2019 14:09:42 +0000 Subject: [PATCH 21/23] include netinet/in.h in src/nix/main.cc Fixes #3186 --- src/nix/main.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nix/main.cc b/src/nix/main.cc index 22f5e8c4c..1c9d909d8 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -15,6 +15,7 @@ #include #include #include +#include extern std::string chrootHelperName; From dcd7a260639c5130368e5969a34aa9bb2365df17 Mon Sep 17 00:00:00 2001 From: kraem Date: Sun, 3 Nov 2019 18:47:28 +0100 Subject: [PATCH 22/23] Update nix eval --help msg to not include deprecated command --- src/nix/eval.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/eval.cc b/src/nix/eval.cc index b7058361c..daefea757 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -41,7 +41,7 @@ struct CmdEval : MixJSON, InstallableCommand }, Example{ "To get the current version of Nixpkgs:", - "nix eval --raw nixpkgs.lib.nixpkgsVersion" + "nix eval --raw nixpkgs.lib.version" }, Example{ "To print the store path of the Hello package:", From c935ad3f025d5c3d8026711a1eb50b2917b61d59 Mon Sep 17 00:00:00 2001 From: Harald van Dijk Date: Sun, 3 Nov 2019 21:46:59 +0000 Subject: [PATCH 23/23] Fix progress bar when nix-prefetch-url is piped. The intent of the code was that if the window size cannot be determined, it would be treated as having the maximum possible size. Because of a missing assignment, it was actually treated as having a width of 0. The reason the width could not be determined was because it was obtained from stdout, not stderr, even though the printing was done to stderr. This commit addresses both issues. --- src/libutil/util.cc | 2 +- src/nix/progress-bar.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 2a1272885..d29802bcd 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1467,7 +1467,7 @@ static Sync> windowSize{{0, 0}}; static void updateWindowSize() { struct winsize ws; - if (ioctl(1, TIOCGWINSZ, &ws) == 0) { + if (ioctl(2, TIOCGWINSZ, &ws) == 0) { auto windowSize_(windowSize.lock()); windowSize_->first = ws.ws_row; windowSize_->second = ws.ws_col; diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc index 5c05d6b22..661966733 100644 --- a/src/nix/progress-bar.cc +++ b/src/nix/progress-bar.cc @@ -341,7 +341,7 @@ public: } auto width = getWindowSize().second; - if (width <= 0) std::numeric_limits::max(); + if (width <= 0) width = std::numeric_limits::max(); writeToStderr("\r" + filterANSIEscapes(line, false, width) + "\e[K"); }