snaakey/hydra
0
0
Fork 0
forked from lix-project/hydra
Code Pull requests Activity

Fix issue #614: restart queue/evaluator on sufficient disk space avai… (#777)

Browse source 
* Fix issue #614: restart queue/evaluator on sufficient disk space available.

* Only try to stop the service if it is currently running.

* Use named variables and added restarting message.
This commit is contained in:
Kevin Quick 2020-07-27 12:46:57 -07:00 committed by GitHub
parent 63188667b8
commit 3e73a2f709
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 24 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
hydra-module.nix
View file

@ -395,17 +395,33 @@ in
# If there is less than a certain amount of free disk space, stop # If there is less than a certain amount of free disk space, stop
# the queue/evaluator to prevent builds from failing or aborting. # the queue/evaluator to prevent builds from failing or aborting.
# Leaves a tag file indicating this reason; if the tag file exists
# and disk space is above the threshold + 10GB, the queue/evaluator will be
# restarted; starting it if it is already started is not harmful.
systemd.services.hydra-check-space = systemd.services.hydra-check-space =
{ script = { script =
'' ''
if [ $(($(stat -f -c '%a' /nix/store) * $(stat -f -c '%S' /nix/store))) -lt $((${toString cfg.minimumDiskFree} * 1024**3)) ]; then spaceleft=$(($(stat -f -c '%a' /nix/store) * $(stat -f -c '%S' /nix/store)))
echo "stopping Hydra queue runner due to lack of free space..." spacestopstart() {
systemctl stop hydra-queue-runner service=$1
minFreeGB=$2
if [ $spaceleft -lt $(($minFreeGB * 1024**3)) ]; then
if [ $(systemctl is-active $service) == active ]; then
echo "stopping $service due to lack of free space..."
systemctl stop $service
date > /var/lib/hydra/.$service-stopped-minspace
fi fi
if [ $(($(stat -f -c '%a' /nix/store) * $(stat -f -c '%S' /nix/store))) -lt $((${toString cfg.minimumDiskFreeEvaluator} * 1024**3)) ]; then else
echo "stopping Hydra evaluator due to lack of free space..." if [ $spaceleft -gt $(( ($minFreeGB + 10) * 1024**3)) -a \
systemctl stop hydra-evaluator -r /var/lib/hydra/.$service-stopped-minspace ] ; then
rm /var/lib/hydra/.$service-stopped-minspace
echo "restarting $service due to newly available free space..."
systemctl start $service
fi fi
fi
}
spacestopstart hydra-queue-runner ${toString cfg.minimumDiskFree}
spacestopstart hydra-evaluator ${toString cfg.minimumDiskFreeEvaluator}
''; '';
startAt = "*:0/5"; startAt = "*:0/5";
}; };