diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc
index 83167b0f3..04be0da99 100644
--- a/src/libstore/build/worker.cc
+++ b/src/libstore/build/worker.cc
@@ -429,6 +429,28 @@ void Worker::waitForInput()
         GoalPtr goal = j->goal.lock();
         assert(goal);
 
+        if (goal->exitCode == Goal::ecBusy &&
+            0 != settings.maxSilentTime &&
+            j->respectTimeouts &&
+            after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime))
+        {
+            goal->timedOut(Error(
+                    "%1% timed out after %2% seconds of silence",
+                    goal->getName(), settings.maxSilentTime));
+            continue;
+        }
+
+        else if (goal->exitCode == Goal::ecBusy &&
+            0 != settings.buildTimeout &&
+            j->respectTimeouts &&
+            after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout))
+        {
+            goal->timedOut(Error(
+                    "%1% timed out after %2% seconds",
+                    goal->getName(), settings.buildTimeout));
+            continue;
+        }
+
         std::set<int> fds2(j->fds);
         std::vector<unsigned char> buffer(4096);
         for (auto & k : fds2) {
@@ -455,26 +477,6 @@ void Worker::waitForInput()
                 }
             }
         }
-
-        if (goal->exitCode == Goal::ecBusy &&
-            0 != settings.maxSilentTime &&
-            j->respectTimeouts &&
-            after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime))
-        {
-            goal->timedOut(Error(
-                    "%1% timed out after %2% seconds of silence",
-                    goal->getName(), settings.maxSilentTime));
-        }
-
-        else if (goal->exitCode == Goal::ecBusy &&
-            0 != settings.buildTimeout &&
-            j->respectTimeouts &&
-            after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout))
-        {
-            goal->timedOut(Error(
-                    "%1% timed out after %2% seconds",
-                    goal->getName(), settings.buildTimeout));
-        }
     }
 
     if (!waitingForAWhile.empty() && lastWokenUp + std::chrono::seconds(settings.pollInterval) <= after) {