Make the github event intake endpoint retry connections to rabbitmq until it works

README.md

@@ -158,6 +158,9 @@ config.known-users.json, run `./scripts/update-known-users.sh`.
 
 ## old php stuff...
 
+Only Graham needs to do this, since I run the only remaining PHP
+components.
+
 ```php
 <?php
 
@@ -165,26 +168,25 @@ require_once __DIR__ . '/vendor/autoload.php';
 use PhpAmqpLib\Connection\AMQPSSLConnection;
 use PhpAmqpLib\Message\AMQPMessage;
 
-define("NIX_SYSTEM", "x86_64-linux");
-define("WORKING_DIR", "/home/grahamc/.nix-test");
-
-function rabbitmq_conn() {
+function rabbitmq_conn($timeout = 3) {
+    $host = 'events.nix.gsc.io';
     $connection = new AMQPSSLConnection(
-        'events.nix.gsc.io', 5671,
-        eventsuser, eventspasswordd, '/', array(
+        $host, 5671,
+        'eventsuser, eventspassword, '/',
+        array(
             'verify_peer' => true,
             'verify_peer_name' => true,
-            'peer_name' => 'events.nix.gsc.io',
+            'peer_name' => $host,
             'verify_depth' => 10,
-            'ca_file' => '/etc/ssl/certs/ca-certificates.crt'
+            'ca_file' => '/etc/ssl/certs/ca-certificates.crt',
+        ), array(
+            'connection_timeout' => $timeout,
         )
     );
 
     return $connection;
 }
 
-/*
-# Only leader machines (ie: graham's) need this:
 function gh_client() {
     $client = new \Github\Client();
     $client->authenticate('githubusername',
@@ -193,5 +195,5 @@ function gh_client() {
 
     return $client;
 }
-*/
+
 ```

php/web/index.php

@@ -1,5 +1,8 @@
 <?php
 
+ini_set("display_errors", 0);
+error_reporting(-1);
+
 ob_start();
 
 require_once __DIR__ . '/../config.php';
@@ -12,6 +15,25 @@ class InvalidEventTypeException extends DumpableException {}
 class ValidationFailureException extends DumpableException {}
 class ExecutionFailureException extends DumpableException {}
 
+function retry_rabbitmq_conn() {
+    $maximum_time = 25;
+    $delay = 1;
+    $timeout = 0.5;
+
+    for ($i = 0.0; $i < $maximum_time; $i += ($timeout + $delay)) {
+        try {
+            return rabbitmq_conn($timeout);
+        } catch (ErrorException $e) {
+            trigger_error(print_r($e, true), E_USER_WARNING);
+        }
+        sleep($delay);
+    }
+    trigger_error("Failed to connect to RabbitMQ", E_USER_WARNING);
+    echo "rabbit failure";
+    exit(1);
+}
+
+
 function payload() {
     if (!isset($_SERVER)) {
         throw new InvalidPayloadException('_SERVER undefined');
@@ -117,7 +139,7 @@ try {
     $name = strtolower($input->repository->full_name);
     $eventtype = event_type();
 
-    $connection = rabbitmq_conn();
+    $connection = retry_rabbitmq_conn();
     $channel = $connection->channel();
 
     $dec = $channel->exchange_declare('github-events', 'topic', false, true, false);