retry a job if we get an expired creds error

ofborg/src/commitstatus.rs

@@ -71,6 +71,7 @@ impl<'a> CommitStatus<'a> {
 
 #[derive(Debug)]
 pub enum CommitStatusError {
+    ExpiredCreds(hubcaps::Error),
     MissingSHA(hubcaps::Error),
     Error(hubcaps::Error),
 }
@@ -79,6 +80,11 @@ impl From<hubcaps::Error> for CommitStatusError {
     fn from(e: hubcaps::Error) -> CommitStatusError {
         use hyper::status::StatusCode;
         match e.kind() {
+            hubcaps::ErrorKind::Fault { code, error }
+                if code == &StatusCode::Unauthorized && error.message == "Bad credentials" =>
+            {
+                CommitStatusError::ExpiredCreds(e)
+            }
             hubcaps::ErrorKind::Fault { code, error }
                 if code == &StatusCode::UnprocessableEntity
                     && error.message.starts_with("No commit found for SHA:") =>

ofborg/src/message/evaluationjob.rs

@@ -20,6 +20,10 @@ impl EvaluationJob {
 pub struct Actions {}
 
 impl Actions {
+    pub fn retry_later(&mut self, _job: &EvaluationJob) -> worker::Actions {
+        vec![worker::Action::NackRequeue]
+    }
+
     pub fn skip(&mut self, _job: &EvaluationJob) -> worker::Actions {
         vec![worker::Action::Ack]
     }

ofborg/src/tasks/evaluate.rs

@@ -226,6 +226,10 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> {
 
                 self.actions().skip(&self.job)
             }
+            Err(Err(CommitStatusError::ExpiredCreds(e))) => {
+                error!("Failed writing commit status: creds expired: {:?}", e);
+                self.actions().retry_later(&self.job)
+            }
             Err(Err(CommitStatusError::MissingSHA(e))) => {
                 error!(
                     "Failed writing commit status: commit sha was force-pushed away: {:?}",