From 3e73a2f7098ebf568c827b9f3ef9638fd5f4bf13 Mon Sep 17 00:00:00 2001
From: Kevin Quick <kquick@galois.com>
Date: Mon, 27 Jul 2020 12:46:57 -0700
Subject: [PATCH] =?UTF-8?q?Fix=20issue=20#614:=20restart=20queue/evaluator?=
 =?UTF-8?q?=20on=20sufficient=20disk=20space=20avai=E2=80=A6=20(#777)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Fix issue #614: restart queue/evaluator on sufficient disk space available.

* Only try to stop the service if it is currently running.

* Use named variables and added restarting message.
---
 hydra-module.nix | 32 ++++++++++++++++++++++++--------
 1 file changed, 24 insertions(+), 8 deletions(-)

diff --git a/hydra-module.nix b/hydra-module.nix
index 81df15d6..08d623a0 100644
--- a/hydra-module.nix
+++ b/hydra-module.nix
@@ -395,17 +395,33 @@ in
 
     # If there is less than a certain amount of free disk space, stop
     # the queue/evaluator to prevent builds from failing or aborting.
+    # Leaves a tag file indicating this reason; if the tag file exists
+    # and disk space is above the threshold + 10GB, the queue/evaluator will be
+    # restarted; starting it if it is already started is not harmful.
     systemd.services.hydra-check-space =
       { script =
           ''
-            if [ $(($(stat -f -c '%a' /nix/store) * $(stat -f -c '%S' /nix/store))) -lt $((${toString cfg.minimumDiskFree} * 1024**3)) ]; then
-                echo "stopping Hydra queue runner due to lack of free space..."
-                systemctl stop hydra-queue-runner
-            fi
-            if [ $(($(stat -f -c '%a' /nix/store) * $(stat -f -c '%S' /nix/store))) -lt $((${toString cfg.minimumDiskFreeEvaluator} * 1024**3)) ]; then
-                echo "stopping Hydra evaluator due to lack of free space..."
-                systemctl stop hydra-evaluator
-            fi
+            spaceleft=$(($(stat -f -c '%a' /nix/store) * $(stat -f -c '%S' /nix/store)))
+            spacestopstart() {
+              service=$1
+              minFreeGB=$2
+              if [ $spaceleft -lt $(($minFreeGB * 1024**3)) ]; then
+                if [ $(systemctl is-active $service) == active ]; then
+                  echo "stopping $service due to lack of free space..."
+                  systemctl stop $service
+                  date > /var/lib/hydra/.$service-stopped-minspace
+                fi
+              else
+                if [ $spaceleft -gt $(( ($minFreeGB + 10) * 1024**3)) -a \
+                     -r /var/lib/hydra/.$service-stopped-minspace ] ; then
+                  rm /var/lib/hydra/.$service-stopped-minspace
+                  echo "restarting $service due to newly available free space..."
+                  systemctl start $service
+                fi
+              fi
+            }
+            spacestopstart hydra-queue-runner ${toString cfg.minimumDiskFree}
+            spacestopstart hydra-evaluator ${toString cfg.minimumDiskFreeEvaluator}
           '';
         startAt = "*:0/5";
       };