From 70ad3a924affd56d57cdced7b31c5b36a7181701 Mon Sep 17 00:00:00 2001
From: Casey Ransom <casey@dailykos.com>
Date: Wed, 26 Oct 2022 16:13:40 -0400
Subject: [PATCH] exit with error if database connectivity lost

There's currently no automatic recovery for disconnected databases in
the evaluator. This means if the database is ever temporarily
unavailable, hydra-evaluator will sit and spin with no work
accomplished.

If this condition is caught, the daemon will exit and systemd will be
responsible for resuming the service.
---
 src/hydra-evaluator/hydra-evaluator.cc | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/hydra-evaluator/hydra-evaluator.cc b/src/hydra-evaluator/hydra-evaluator.cc
index 2d7e68d9..a1ccf047 100644
--- a/src/hydra-evaluator/hydra-evaluator.cc
+++ b/src/hydra-evaluator/hydra-evaluator.cc
@@ -366,6 +366,9 @@ struct Evaluator
                     printInfo("received jobset event");
                 }
 
+            } catch (pqxx::broken_connection & e) {
+                printError("Database connection broken: %s", e.what());
+                std::_Exit(1);
             } catch (std::exception & e) {
                 printError("exception in database monitor thread: %s", e.what());
                 sleep(30);
@@ -473,6 +476,9 @@ struct Evaluator
         while (true) {
             try {
                 loop();
+            } catch (pqxx::broken_connection & e) {
+                printError("Database connection broken: %s", e.what());
+                std::_Exit(1);
             } catch (std::exception & e) {
                 printError("exception in main loop: %s", e.what());
                 sleep(30);