From 6d30f9e6fea7d451033653f8f167aef58f7f5347 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 13 Dec 2021 17:16:46 +0100 Subject: [PATCH 1/4] Explicitly make GC roots client sockets blocking On macOS / BSD, these sockets inherit the non-blocking flag of the server soocket, which is not what we want. https://github.com/dotnet/runtime/issues/25069 https://bugs.python.org/issue7995 https://hydra.nixos.org/build/161439304 --- src/libstore/gc.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 7a414da6b..ee123d84e 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -535,6 +535,12 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) } }); + /* On macOS, accepted sockets inherit the + non-blocking flag from the server socket, so + explicitly make it blocking. */ + if (fcntl(fdServer.get(), F_SETFL, fcntl(fdServer.get(), F_GETFL) & ~O_NONBLOCK) == -1) + abort(); + while (true) { try { auto path = readLine(fdClient.get()); From 05fcc681ac07c4ea04b365e6b03a76f3e6c54c94 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 13 Dec 2021 16:14:57 +0100 Subject: [PATCH 2/4] Retry on ECONNREFUSED https://hydra.nixos.org/build/161439235 --- src/libstore/gc.cc | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index ee123d84e..fa68cd8f3 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -126,7 +126,17 @@ void LocalStore::addTempRoot(const StorePath & path) auto socketPath = stateDir.get() + gcSocketPath; debug("connecting to '%s'", socketPath); state->fdRootsSocket = createUnixDomainSocket(); - nix::connect(state->fdRootsSocket.get(), socketPath); + try { + nix::connect(state->fdRootsSocket.get(), socketPath); + } catch (SysError & e) { + /* The garbage collector may have exited, so we need to + restart. */ + if (e.errNo == ECONNREFUSED) { + debug("GC socket connection refused"); + state->fdRootsSocket.close(); + goto restart; + } + } } try { From df1edd143fcb2532e32bbcb28f774ced1402cda4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 13 Dec 2021 16:24:25 +0100 Subject: [PATCH 3/4] gc-non-blocking.sh: Don't hang indefinitely if the GC roots server crashes --- tests/gc-non-blocking.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/gc-non-blocking.sh b/tests/gc-non-blocking.sh index 8b21c6f1c..0d781485d 100644 --- a/tests/gc-non-blocking.sh +++ b/tests/gc-non-blocking.sh @@ -19,7 +19,7 @@ pid=$! sleep 2 -outPath=$(nix-build -o "$TEST_ROOT/result" -E " +outPath=$(nix-build --max-silent-time 60 -o "$TEST_ROOT/result" -E " with import ./config.nix; mkDerivation { name = \"non-blocking\"; From d17d46cfc2e38efb63565270a1fba7b03b37de16 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 13 Dec 2021 17:12:32 +0100 Subject: [PATCH 4/4] More debug info --- src/libstore/gc.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index fa68cd8f3..e35199b3d 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -533,6 +533,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) AutoCloseFD fdClient = accept(fdServer.get(), nullptr, nullptr); if (!fdClient) continue; + debug("GC roots server accepted new client"); + /* Process the connection in a separate thread. */ auto fdClient_ = fdClient.get(); std::thread clientThread([&, fdClient = std::move(fdClient)]() { @@ -575,7 +577,10 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) } else printError("received garbage instead of a root from client"); writeFull(fdClient.get(), "1", false); - } catch (Error &) { break; } + } catch (Error & e) { + debug("reading GC root from client: %s", e.msg()); + break; + } } });