diff --git a/src/script/hydra-queue-runner b/src/script/hydra-queue-runner
index 270b4d1c..4caae3c8 100755
--- a/src/script/hydra-queue-runner
+++ b/src/script/hydra-queue-runner
@@ -148,7 +148,14 @@ sub checkBuilds {
                     if (!defined $timeSpent) {
                         $timeSpent = $jobset->builds->search(
                             { },
-                            { where => \ ("(finished = 0 or (me.stoptime >= " . (time() - $windowSize) . "))")
+                            { where => \ ("(finished = 0)")
+                            , join => 'buildsteps'
+                            , select => \ "sum(coalesce(buildsteps.stoptime, ${\time}) - buildsteps.starttime)"
+                            , as => "sum" })->single->get_column("sum") // 0;
+
+                        $timeSpent += $jobset->builds->search(
+                            { },
+                            { where => \ ("(me.stoptime >= " . (time() - $windowSize) . ")")
                             , join => 'buildsteps'
                             , select => \ "sum(coalesce(buildsteps.stoptime, ${\time}) - buildsteps.starttime)"
                             , as => "sum" })->single->get_column("sum") // 0;