danderson/ofborg
0
0
Fork 0
forked from the-distro/ofborg
Code Pull requests Activity

Merge pull request #206 from LnL7/detect-timedout

Browse source 
detect timedout builds
This commit is contained in:
Graham Christensen 2018-08-07 15:52:34 -04:00 committed by GitHub
parent 2e3b8d0fd3 9d85f62835
commit 67406a7a18
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 229 additions and 57 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

40
ofborg/src/asynccmd.rs
View file

@ -3,9 +3,10 @@ use std::thread;
use std::collections::HashMap;
use std::process::Stdio;
use std::process::ExitStatus;
use std::sync::mpsc::sync_channel;
use std::process::Command;
use std::io::Read;
use std::sync::mpsc;
use std::sync::mpsc::sync_channel;
use std::sync::mpsc::{SyncSender, Receiver};
use std::io::BufReader;
use std::io::BufRead;
@ -188,12 +189,15 @@ impl AsyncCmd {
impl SpawnedAsyncCmd {
pub fn lines<'a>(&'a mut self) -> &'a Receiver<String> {
&self.rx
pub fn lines<'a>(&'a mut self) -> mpsc::Iter<'a, String> {
self.rx.iter()
}
pub fn wait(self) -> thread::Result<Option<Result<ExitStatus, io::Error>>> {
pub fn wait(self) -> Result<ExitStatus, io::Error> {
self.waiter.join()
.map_err(|_err| io::Error::new(io::ErrorKind::Other, "Couldn't join thread."))
.and_then(|opt| opt.ok_or(io::Error::new(io::ErrorKind::Other, "Thread didn't return an exit status.")))
.and_then(|res| res)
}
}
@ -214,9 +218,9 @@ mod tests {
let acmd = AsyncCmd::new(cmd);
let mut spawned = acmd.spawn();
let lines: Vec<String> = spawned.lines().into_iter().collect();
let lines: Vec<String> = spawned.lines().collect();
assert_eq!(lines, vec!["hi"]);
let ret = spawned.wait().unwrap().unwrap().unwrap().success();
let ret = spawned.wait().unwrap().success();
assert_eq!(true, ret);
}
@ -234,9 +238,9 @@ mod tests {
let acmd = AsyncCmd::new(cmd);
let mut spawned = acmd.spawn();
let lines: Vec<String> = spawned.lines().into_iter().collect();
let lines: Vec<String> = spawned.lines().collect();
assert_eq!(lines, vec!["stdout", "stderr", "stdout2", "stderr2"]);
let ret = spawned.wait().unwrap().unwrap().unwrap().success();
let ret = spawned.wait().unwrap().success();
assert_eq!(true, ret);
}
@ -250,14 +254,10 @@ mod tests {
let acmd = AsyncCmd::new(cmd);
let mut spawned = acmd.spawn();
let lines: Vec<String> = spawned.lines().into_iter().collect();
let lines: Vec<String> = spawned.lines().collect();
assert_eq!(lines.len(), 20000);
let thread_result = spawned.wait();
let child_result_opt = thread_result.expect("Thread should exit correctly");
let child_result = child_result_opt.expect(
"Thread should have properly properly returned the child's status",
);
let exit_status = child_result.expect("The child should have no problem exiting");
let exit_status = thread_result.expect("Thread should exit correctly");
assert_eq!(true, exit_status.success());
}
@ -270,14 +270,10 @@ mod tests {
let acmd = AsyncCmd::new(cmd);
let mut spawned = acmd.spawn();
let lines: Vec<String> = spawned.lines().into_iter().collect();
let lines: Vec<String> = spawned.lines().collect();
assert_eq!(lines.len(), 200000);
let thread_result = spawned.wait();
let child_result_opt = thread_result.expect("Thread should exit correctly");
let child_result = child_result_opt.expect(
"Thread should have properly properly returned the child's status",
);
let exit_status = child_result.expect("The child should have no problem exiting");
let exit_status = thread_result.expect("Thread should exit correctly");
assert_eq!(true, exit_status.success());
}
@ -293,12 +289,12 @@ mod tests {
let acmd = AsyncCmd::new(cmd);
let mut spawned = acmd.spawn();
let lines: Vec<String> = spawned.lines().into_iter().collect();
let lines: Vec<String> = spawned.lines().collect();
assert_eq!(
lines,
vec!["hi", "Non-UTF8 data omitted from the log.", "there"]
);
let ret = spawned.wait().unwrap().unwrap().unwrap().success();
let ret = spawned.wait().unwrap().success();
assert_eq!(true, ret);
}
}

13
ofborg/src/message/buildresult.rs
View file

@ -1,5 +1,15 @@
use ofborg::message::{Pr, Repo};
#[derive(Serialize, Deserialize, Debug)]
pub enum BuildStatus {
Skipped,
Success,
Failure,
TimedOut,
UnexpectedError { err: String },
}
#[derive(Serialize, Deserialize, Debug)]
pub struct BuildResult {
pub repo: Repo,
@ -8,7 +18,8 @@ pub struct BuildResult {
pub output: Vec<String>,
pub attempt_id: String,
pub request_id: Option<String>,
pub success: Option<bool>,
pub success: Option<bool>, // replaced by status
pub status: Option<BuildStatus>,
pub skipped_attrs: Option<Vec<String>>,
pub attempted_attrs: Option<Vec<String>>,
}

16
ofborg/src/nix.rs
View file

@ -8,8 +8,10 @@ use std::process::{Command, Stdio};
use tempfile::tempfile;
use std::io::BufReader;
use std::io::BufRead;
use ofborg::asynccmd::{AsyncCmd, SpawnedAsyncCmd};
use ofborg::partition_result;
#[derive(Clone, Debug)]
pub enum Operation {
Instantiate,
@ -162,7 +164,17 @@ impl Nix {
return self.run(cmd, true);
}
pub fn safely_build_attrs_cmd(
pub fn safely_build_attrs_async(
&self,
nixpkgs: &Path,
file: &str,
attrs: Vec<String>,
) -> SpawnedAsyncCmd {
AsyncCmd::new(self.safely_build_attrs_cmd(nixpkgs, file, attrs))
.spawn()
}
fn safely_build_attrs_cmd(
&self,
nixpkgs: &Path,
file: &str,
@ -175,7 +187,7 @@ impl Nix {
attrargs.push(attr);
}
return self.safe_command(Operation::Build, nixpkgs, attrargs);
self.safe_command(Operation::Build, nixpkgs, attrargs)
}
pub fn safely(

51
ofborg/src/tasks/build.rs
View file

@ -5,10 +5,9 @@ extern crate env_logger;
use uuid::Uuid;
use std::collections::VecDeque;
use ofborg::asynccmd::AsyncCmd;
use ofborg::checkout;
use ofborg::message::buildjob;
use ofborg::message::buildresult;
use ofborg::message::buildresult::{BuildResult, BuildStatus};
use ofborg::message::buildlogmsg;
use ofborg::nix;
use ofborg::commentparser;
@ -113,7 +112,7 @@ impl<'a, 'b> JobActions<'a, 'b> {
}
pub fn merge_failed(&mut self) {
let msg = buildresult::BuildResult {
let msg = BuildResult {
repo: self.job.repo.clone(),
pr: self.job.pr.clone(),
system: self.system.clone(),
@ -122,6 +121,7 @@ impl<'a, 'b> JobActions<'a, 'b> {
request_id: Some(self.job.request_id.clone()),
attempted_attrs: None,
skipped_attrs: None,
status: Some(BuildStatus::Failure),
success: Some(false),
};
@ -196,7 +196,7 @@ impl<'a, 'b> JobActions<'a, 'b> {
pub fn build_not_attempted(&mut self, not_attempted_attrs: Vec<String>,
) {
let msg = buildresult::BuildResult {
let msg = BuildResult {
repo: self.job.repo.clone(),
pr: self.job.pr.clone(),
system: self.system.clone(),
@ -205,6 +205,7 @@ impl<'a, 'b> JobActions<'a, 'b> {
request_id: Some(self.job.request_id.clone()),
skipped_attrs: Some(not_attempted_attrs),
attempted_attrs: None,
status: None,
success: None,
};
@ -227,18 +228,23 @@ impl<'a, 'b> JobActions<'a, 'b> {
self.tell(worker::Action::Ack);
}
pub fn build_finished(&mut self, success: bool,
pub fn build_finished(&mut self, status: BuildStatus,
attempted_attrs: Vec<String>,
not_attempted_attrs: Vec<String>,
) {
let msg = buildresult::BuildResult {
let success = match status {
BuildStatus::Success => true,
_ => false,
};
let msg = BuildResult {
repo: self.job.repo.clone(),
pr: self.job.pr.clone(),
system: self.system.clone(),
output: self.log_snippet(),
attempt_id: self.attempt_id.clone(),
request_id: Some(self.job.request_id.clone()),
status: Some(status),
success: Some(success),
attempted_attrs: Some(attempted_attrs),
skipped_attrs: Some(not_attempted_attrs),
@ -364,35 +370,40 @@ impl notifyworker::SimpleNotifyWorker for BuildWorker {
return;
}
let cmd = self.nix.safely_build_attrs_cmd(
let mut spawned = self.nix.safely_build_attrs_async(
refpath.as_ref(),
buildfile,
can_build.clone(),
);
println!("About to execute {:?}", cmd);
let acmd = AsyncCmd::new(cmd);
let mut spawned = acmd.spawn();
for line in spawned.lines().iter() {
for line in spawned.lines() {
actions.log_line(&line);
}
let success = match spawned.wait() {
Ok(Some(Ok(status))) => status.success(),
e => {
println!("Failed on the interior command: {:?}", e);
false
// TODO: this belongs in the nix module.
let status = match spawned.wait() {
Ok(s) => match s.code() {
Some(0) => BuildStatus::Success,
Some(100) => BuildStatus::Failure, // nix permanent failure
Some(101) => BuildStatus::TimedOut, // nix build timedout
Some(i) => BuildStatus::UnexpectedError {
err: format!("command failed with exit code {}", i),
},
None => BuildStatus::UnexpectedError {
err: "unexpected build failure".into(),
},
},
e => BuildStatus::UnexpectedError {
err: format!("failed on interior command {:?}", e),
}
};
println!("ok built ({:?}), building", success);
println!("ok built ({:?}), building", status);
println!("Lines:\n-----8<-----");
actions.log_snippet().iter().inspect(|x| println!("{}", x)).last();
println!("----->8-----");
actions.build_finished(success, can_build, cannot_build_attrs);
actions.build_finished(status, can_build, cannot_build_attrs);
println!("Done!");
}
}

159
ofborg/src/tasks/githubcommentposter.rs
View file

@ -4,7 +4,7 @@ extern crate env_logger;
use serde_json;
use hubcaps;
use ofborg::message::buildresult::BuildResult;
use ofborg::message::buildresult::{BuildStatus, BuildResult};
use ofborg::worker;
use amqp::protocol::basic::{Deliver, BasicProperties};
@ -86,12 +86,26 @@ fn result_to_comment(result: &BuildResult) -> String {
"".to_owned()
};
let status = match result.status {
None => {
// Fallback for old format.
match result.success {
None => &BuildStatus::Skipped,
Some(true) => &BuildStatus::Success,
Some(false) => &BuildStatus::Failure,
}
},
Some(ref s) => s,
};
reply.push(format!(
"{} on {}{}",
(match result.success {
Some(true) => "Success",
Some(false) => "Failure",
None => "No attempt",
(match *status {
BuildStatus::Skipped => "No attempt".into(),
BuildStatus::Success => "Success".into(),
BuildStatus::Failure => "Failure".into(),
BuildStatus::TimedOut => "Timed out, unknown build status".into(),
BuildStatus::UnexpectedError { ref err } => format!("Unexpected error: {}", err),
}),
result.system,
log_link
@ -185,7 +199,8 @@ mod tests {
system: "x86_64-linux".to_owned(),
attempted_attrs: Some(vec!["foo".to_owned()]),
skipped_attrs: Some(vec!["bar".to_owned()]),
success: Some(true),
status: Some(BuildStatus::Success),
success: None,
};
assert_eq!(
@ -247,7 +262,8 @@ patching script interpreter paths in /nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29
system: "x86_64-linux".to_owned(),
attempted_attrs: Some(vec!["foo".to_owned()]),
skipped_attrs: None,
success: Some(false),
status: Some(BuildStatus::Failure),
success: None,
};
assert_eq!(
@ -276,6 +292,64 @@ patching script interpreter paths in /nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29
);
}
#[test]
pub fn test_timedout_build() {
let result = BuildResult {
repo: Repo {
clone_url: "https://github.com/nixos/nixpkgs.git".to_owned(),
full_name: "NixOS/nixpkgs".to_owned(),
owner: "NixOS".to_owned(),
name: "nixpkgs".to_owned(),
},
pr: Pr {
head_sha: "abc123".to_owned(),
number: 2345,
target_branch: Some("master".to_owned()),
},
output: vec![
"make[2]: Entering directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline'".to_owned(),
"make[2]: Nothing to be done for 'install'.".to_owned(),
"make[2]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline'".to_owned(),
"make[1]: Nothing to be done for 'install-target'.".to_owned(),
"make[1]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1'".to_owned(),
"removed '/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1/share/info/bfd.info'".to_owned(),
"post-installation fixup".to_owned(),
"building of '/nix/store/l1limh50lx2cx45yb2gqpv7k8xl1mik2-gdb-8.1.drv' timed out after 1 seconds".to_owned(),
"error: build of '/nix/store/l1limh50lx2cx45yb2gqpv7k8xl1mik2-gdb-8.1.drv' failed".to_owned(),
],
attempt_id: "neatattemptid".to_owned(),
request_id: Some("bogus-request-id".to_owned()),
system: "x86_64-linux".to_owned(),
attempted_attrs: Some(vec!["foo".to_owned()]),
skipped_attrs: None,
status: Some(BuildStatus::TimedOut),
success: None,
};
assert_eq!(
&result_to_comment(&result),
"Timed out, unknown build status on x86_64-linux [(full log)](https://logs.nix.ci/?key=nixos/nixpkgs.2345&attempt_id=neatattemptid)
Attempted: foo
<details><summary>Partial log (click to expand)</summary><p>
```
make[2]: Entering directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline'
make[2]: Nothing to be done for 'install'.
make[2]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline'
make[1]: Nothing to be done for 'install-target'.
make[1]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1'
removed '/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1/share/info/bfd.info'
post-installation fixup
building of '/nix/store/l1limh50lx2cx45yb2gqpv7k8xl1mik2-gdb-8.1.drv' timed out after 1 seconds
error: build of '/nix/store/l1limh50lx2cx45yb2gqpv7k8xl1mik2-gdb-8.1.drv' failed
```
</p></details>
"
);
}
#[test]
pub fn test_passing_build_unspecified_attributes() {
@ -308,7 +382,8 @@ patching script interpreter paths in /nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29
system: "x86_64-linux".to_owned(),
attempted_attrs: None,
skipped_attrs: None,
success: Some(true),
status: Some(BuildStatus::Success),
success: None,
};
assert_eq!(
@ -366,7 +441,8 @@ patching script interpreter paths in /nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29
system: "x86_64-linux".to_owned(),
attempted_attrs: None,
skipped_attrs: None,
success: Some(false),
status: Some(BuildStatus::Failure),
success: None,
};
assert_eq!(
@ -413,6 +489,7 @@ patching script interpreter paths in /nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29
system: "x86_64-linux".to_owned(),
attempted_attrs: None,
skipped_attrs: Some(vec!["not-attempted".to_owned()]),
status: Some(BuildStatus::Skipped),
success: None,
};
@ -453,6 +530,7 @@ foo
system: "x86_64-linux".to_owned(),
attempted_attrs: None,
skipped_attrs: Some(vec!["not-attempted".to_owned()]),
status: Some(BuildStatus::Skipped),
success: None,
};
@ -463,6 +541,69 @@ foo
The following builds were skipped because they don't evaluate on x86_64-linux: not-attempted
No partial log is available.
"
);
}
#[test]
pub fn test_no_status_compatibility() {
let result = BuildResult {
repo: Repo {
clone_url: "https://github.com/nixos/nixpkgs.git".to_owned(),
full_name: "NixOS/nixpkgs".to_owned(),
owner: "NixOS".to_owned(),
name: "nixpkgs".to_owned(),
},
pr: Pr {
head_sha: "abc123".to_owned(),
number: 2345,
target_branch: Some("master".to_owned()),
},
output: vec![
"make[2]: Entering directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline'".to_owned(),
"make[2]: Nothing to be done for 'install'.".to_owned(),
"make[2]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline'".to_owned(),
"make[1]: Nothing to be done for 'install-target'.".to_owned(),
"make[1]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1'".to_owned(),
"removed '/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1/share/info/bfd.info'".to_owned(),
"post-installation fixup".to_owned(),
"strip is /nix/store/5a88zk3jgimdmzg8rfhvm93kxib3njf9-cctools-binutils-darwin/bin/strip".to_owned(),
"patching script interpreter paths in /nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1".to_owned(),
"/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1".to_owned(),
],
attempt_id: "neatattemptid".to_owned(),
request_id: Some("bogus-request-id".to_owned()),
system: "x86_64-linux".to_owned(),
attempted_attrs: Some(vec!["foo".to_owned()]),
skipped_attrs: Some(vec!["bar".to_owned()]),
status: None,
success: Some(true),
};
assert_eq!(
&result_to_comment(&result),
"Success on x86_64-linux [(full log)](https://logs.nix.ci/?key=nixos/nixpkgs.2345&attempt_id=neatattemptid)
Attempted: foo
The following builds were skipped because they don't evaluate on x86_64-linux: bar
<details><summary>Partial log (click to expand)</summary><p>
```
make[2]: Entering directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline'
make[2]: Nothing to be done for 'install'.
make[2]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline'
make[1]: Nothing to be done for 'install-target'.
make[1]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1'
removed '/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1/share/info/bfd.info'
post-installation fixup
strip is /nix/store/5a88zk3jgimdmzg8rfhvm93kxib3njf9-cctools-binutils-darwin/bin/strip
patching script interpreter paths in /nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1
/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1
```
</p></details>
"
);
}

5
ofborg/src/tasks/log_message_collector.rs
View file

@ -9,7 +9,7 @@ use std::io::Write;
use ofborg::writetoline::LineWriter;
use ofborg::message::buildlogmsg::{BuildLogStart, BuildLogMsg};
use ofborg::message::buildresult::BuildResult;
use ofborg::message::buildresult::{BuildStatus, BuildResult};
use ofborg::worker;
use amqp::protocol::basic::{Deliver, BasicProperties};
@ -445,6 +445,7 @@ mod tests {
output: vec![],
attempt_id: "attempt-id-foo".to_owned(),
request_id: Some("bogus-request-id".to_owned()),
status: Some(BuildStatus::Success),
success: Some(true),
attempted_attrs: Some(vec!["foo".to_owned()]),
skipped_attrs: Some(vec!["bar".to_owned()]),
@ -478,6 +479,6 @@ mod tests {
let mut s = String::new();
pr.push("routing-key-foo/attempt-id-foo.result.json");
File::open(pr).unwrap().read_to_string(&mut s).unwrap();
assert_eq!(&s, "{\"repo\":{\"owner\":\"NixOS\",\"name\":\"ofborg\",\"full_name\":\"NixOS/ofborg\",\"clone_url\":\"https://github.com/nixos/ofborg.git\"},\"pr\":{\"target_branch\":\"scratch\",\"number\":42,\"head_sha\":\"6dd9f0265d52b946dd13daf996f30b64e4edb446\"},\"system\":\"x86_64-linux\",\"output\":[],\"attempt_id\":\"attempt-id-foo\",\"request_id\":\"bogus-request-id\",\"success\":true,\"skipped_attrs\":[\"bar\"],\"attempted_attrs\":[\"foo\"]}");
assert_eq!(&s, "{\"repo\":{\"owner\":\"NixOS\",\"name\":\"ofborg\",\"full_name\":\"NixOS/ofborg\",\"clone_url\":\"https://github.com/nixos/ofborg.git\"},\"pr\":{\"target_branch\":\"scratch\",\"number\":42,\"head_sha\":\"6dd9f0265d52b946dd13daf996f30b64e4edb446\"},\"system\":\"x86_64-linux\",\"output\":[],\"attempt_id\":\"attempt-id-foo\",\"request_id\":\"bogus-request-id\",\"success\":true,\"status\":\"Success\",\"skipped_attrs\":[\"bar\"],\"attempted_attrs\":[\"foo\"]}");
}
}