Merge pull request #456 from NixOS/retryon-bad-creds

Retry on bad creds

ofborg/src/commitstatus.rs

@@ -71,23 +71,20 @@ impl<'a> CommitStatus<'a> {
 
 #[derive(Debug)]
 pub enum CommitStatusError {
+    ExpiredCreds(hubcaps::Error),
     MissingSHA(hubcaps::Error),
     Error(hubcaps::Error),
 }
 
-impl CommitStatusError {
-    pub fn is_internal_error(&self) -> bool {
-        match self {
-            CommitStatusError::MissingSHA(_) => false,
-            CommitStatusError::Error(_) => true,
-        }
-    }
-}
-
 impl From<hubcaps::Error> for CommitStatusError {
     fn from(e: hubcaps::Error) -> CommitStatusError {
         use hyper::status::StatusCode;
         match e.kind() {
+            hubcaps::ErrorKind::Fault { code, error }
+                if code == &StatusCode::Unauthorized && error.message == "Bad credentials" =>
+            {
+                CommitStatusError::ExpiredCreds(e)
+            }
             hubcaps::ErrorKind::Fault { code, error }
                 if code == &StatusCode::UnprocessableEntity
                     && error.message.starts_with("No commit found for SHA:") =>

ofborg/src/message/evaluationjob.rs

@@ -20,6 +20,10 @@ impl EvaluationJob {
 pub struct Actions {}
 
 impl Actions {
+    pub fn retry_later(&mut self, _job: &EvaluationJob) -> worker::Actions {
+        vec![worker::Action::NackRequeue]
+    }
+
     pub fn skip(&mut self, _job: &EvaluationJob) -> worker::Actions {
         vec![worker::Action::Ack]
     }

ofborg/src/tasks/evaluate.rs

@@ -226,13 +226,19 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> {
 
                 self.actions().skip(&self.job)
             }
-            Err(Err(cswerr)) if !cswerr.is_internal_error() => {
+            Err(Err(CommitStatusError::ExpiredCreds(e))) => {
-                error!("Ignorable error writing commit status: {:?}", cswerr);
+                error!("Failed writing commit status: creds expired: {:?}", e);
-
+                self.actions().retry_later(&self.job)
+            }
+            Err(Err(CommitStatusError::MissingSHA(e))) => {
+                error!(
+                    "Failed writing commit status: commit sha was force-pushed away: {:?}",
+                    e
+                );
                 self.actions().skip(&self.job)
             }
 
-            Err(Err(cswerr)) => {
+            Err(Err(CommitStatusError::Error(cswerr))) => {
                 error!(
                     "Internal error writing commit status: {:?}, marking internal error",
                     cswerr