danderson/ofborg
0
0
Fork 0
forked from the-distro/ofborg
Code Pull requests Activity

Merge pull request #295 from NixOS/clippy

Browse source 
Clippy and cargo fmt
This commit is contained in:
Graham Christensen 2019-01-02 21:48:14 -05:00 committed by GitHub
parent f81796e522 f6192a7f35
commit 830f376699
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
54 changed files with 1387 additions and 2948 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
.travis.yml
View file

@ -3,9 +3,11 @@ nix: 2.1
sudo: true
matrix:
include:
- name: cargo pedantry
script: nix-shell --run checkPhase --arg useNix1 false -A mozilla-rust-overlay
- name: checkPhase - Nix 2
script: nix-shell --run checkPhase --arg useNix1 false
- name: checkPhase - Nix 2
script: nix-shell --run checkPhase --arg useNix1 false
- name: checkPhase - Nix 1
script: nix-shell --run checkPhase --arg useNix1 true
- name: nix-build
script: nix-build -A ofborg.rs

1
nix/default.nix
View file

@ -15,7 +15,6 @@ in import (hostpkgs.stdenv.mkDerivation {
};
patches = [
./nixpkgs-pr50452.patch
];
moveToOut = ''

1442
nix/nixpkgs-pr50452.patch
View file

File diff suppressed because it is too large Load diff

6
nix/nixpkgs.json
View file

@ -1,7 +1,7 @@
{
"url": "https://github.com/nixos/nixpkgs-channels.git",
"rev": "80738ed9dc0ce48d7796baed5364eef8072c794d",
"date": "2018-11-17T11:18:10+01:00",
"sha256": "0anmvr6b47gbbyl9v2fn86mfkcwgpbd5lf0yf3drgm8pbv57c1dc",
"rev": "201d739b0ffbebceb444864d1856babcd1a666a8",
"date": "2018-12-30T01:29:37+00:00",
"sha256": "0mfkzmylglpw84w85zs3djpspcx45bg3s62hk4j44dxl2p0fvggj",
"fetchSubmodules": false
}

264
ofborg/build.rs
View file

@ -1,4 +1,3 @@
use std::env;
use std::fs::File;
use std::io::Write;
@ -12,12 +11,8 @@ enum MetricType {
impl MetricType {
fn collector_type(&self) -> String {
match self {
&MetricType::Ticker(_) => {
String::from("u64")
}
&MetricType::Counter(_) => {
String::from("u64")
}
&MetricType::Ticker(_) => String::from("u64"),
&MetricType::Counter(_) => String::from("u64"),
}
}
@ -33,45 +28,29 @@ impl MetricType {
fn variant(&self) -> String {
match self {
&MetricType::Ticker(ref event) => {
event.variant.clone()
}
&MetricType::Counter(ref event) => {
event.variant.clone()
}
&MetricType::Ticker(ref event) => event.variant.clone(),
&MetricType::Counter(ref event) => event.variant.clone(),
}
}
fn metric_type(&self) -> String {
match self {
&MetricType::Ticker(_) => {
String::from("counter")
}
&MetricType::Counter(_) => {
String::from("counter")
}
&MetricType::Ticker(_) => String::from("counter"),
&MetricType::Counter(_) => String::from("counter"),
}
}
fn metric_name(&self) -> String {
match self {
&MetricType::Ticker(ref event) => {
event.metric_name.clone()
}
&MetricType::Counter(ref event) => {
event.metric_name.clone()
}
&MetricType::Ticker(ref event) => event.metric_name.clone(),
&MetricType::Counter(ref event) => event.metric_name.clone(),
}
}
fn description(&self) -> String {
match self {
&MetricType::Ticker(ref event) => {
event.description.clone()
}
&MetricType::Counter(ref event) => {
event.description.clone()
}
&MetricType::Ticker(ref event) => event.description.clone(),
&MetricType::Counter(ref event) => event.description.clone(),
}
}
@ -87,12 +66,13 @@ impl MetricType {
}
}
let fields: Vec<String> = event.fields
let fields: Vec<String> = event
.fields
.iter()
.map(|&(ref _fieldname, ref fieldtype)| fieldtype.clone())
.collect();
return fields
return fields;
}
fn enum_field_types(&self) -> Vec<String> {
@ -108,7 +88,7 @@ impl MetricType {
let mut fields: Vec<String> = self.enum_index_types();
fields.append(&mut extra_fields);
return fields
return fields;
}
fn enum_index_names(&self) -> Vec<String> {
@ -123,12 +103,13 @@ impl MetricType {
}
}
let fields: Vec<String> = event.fields
let fields: Vec<String> = event
.fields
.iter()
.map(|&(ref fieldname, ref _fieldtype)| fieldname.clone())
.collect();
return fields
return fields;
}
fn enum_field_names(&self) -> Vec<String> {
@ -144,29 +125,24 @@ impl MetricType {
let mut fields: Vec<String> = self.enum_index_names();
fields.append(&mut extra_fields);
return fields
return fields;
}
fn record_value(&self) -> String {
match self {
&MetricType::Ticker(_) => {
String::from("1")
}
&MetricType::Counter(_) => {
String::from("value")
}
&MetricType::Ticker(_) => String::from("1"),
&MetricType::Counter(_) => String::from("value"),
}
}
}
struct Metric {
variant: String,
fields: Vec<(String,String)>, // Vec because it is sorted
fields: Vec<(String, String)>, // Vec because it is sorted
metric_name: String,
description: String,
}
fn name_to_parts(name: &str) -> Vec<String> {
let mut parts: Vec<String> = vec![];
let mut buf = String::from("");
@ -182,41 +158,38 @@ fn name_to_parts(name: &str) -> Vec<String> {
std::mem::drop(buf);
}
return parts;
}
impl Metric {
pub fn ticker(name: &str, desc: &str, fields: Option<Vec<(&str,&str)>>) -> MetricType {
pub fn ticker(name: &str, desc: &str, fields: Option<Vec<(&str, &str)>>) -> MetricType {
let parts = name_to_parts(name);
MetricType::Ticker(Metric {
variant: parts
.iter()
.map(|f| f.clone().to_owned())
.collect(),
variant: parts.iter().map(|f| f.clone().to_owned()).collect(),
fields: fields
.unwrap_or(vec![])
.iter()
.map(|&(ref fieldname, ref fieldtype)| (fieldname.clone().to_owned(), fieldtype.clone().to_owned()))
.map(|&(ref fieldname, ref fieldtype)| {
(fieldname.clone().to_owned(), fieldtype.clone().to_owned())
})
.collect(),
metric_name: parts.join("_").to_lowercase(),
description: desc.to_owned(),
})
}
pub fn counter(name: &str, desc: &str, fields: Option<Vec<(&str,&str)>>) -> MetricType {
pub fn counter(name: &str, desc: &str, fields: Option<Vec<(&str, &str)>>) -> MetricType {
let parts = name_to_parts(name);
MetricType::Counter(Metric {
variant: parts
.iter()
.map(|f| f.clone().to_owned())
.collect(),
variant: parts.iter().map(|f| f.clone().to_owned()).collect(),
fields: fields
.unwrap_or(vec![])
.iter()
.map(|&(ref fieldname, ref fieldtype)| (fieldname.clone().to_owned(), fieldtype.clone().to_owned()))
.map(|&(ref fieldname, ref fieldtype)| {
(fieldname.clone().to_owned(), fieldtype.clone().to_owned())
})
.collect(),
metric_name: parts.join("_").to_lowercase(),
description: desc.to_owned(),
@ -236,31 +209,21 @@ fn events() -> Vec<MetricType> {
"Number of received unparseable events",
None,
),
Metric::ticker(
"JobReceived",
"Number of received worker jobs",
None,
),
Metric::ticker("JobReceived", "Number of received worker jobs", None),
Metric::counter(
"EvaluationDuration",
"Amount of time spent running evaluations",
Some(vec![
("branch", "String"),
]),
Some(vec![("branch", "String")]),
),
Metric::ticker(
"EvaluationDurationCount",
"Number of timed evaluations performed",
Some(vec![
("branch", "String"),
]),
Some(vec![("branch", "String")]),
),
Metric::ticker(
"TargetBranchFailsEvaluation",
"Number of PR evaluations which failed because the target branch failed",
Some(vec![
("branch", "String"),
]),
Some(vec![("branch", "String")]),
),
Metric::ticker(
"JobDecodeSuccess",
@ -410,63 +373,68 @@ fn main() {
println!("cargo:rerun-if-changed=build.rs");
// Write the Event enum, which contains all possible event types
f.write_all(b"
f.write_all(
b"
use std::collections::HashMap;
use std::sync::Arc;
use std::sync::Mutex;
#[derive(Serialize, Deserialize, Debug, Clone)]
#[serde(rename_all=\"kebab-case\")]
pub enum Event {
").unwrap();
",
)
.unwrap();
let variants: Vec<String> = events()
.iter()
.map(|mtype| format!(" {}", mtype.enum_matcher_types()) )
.map(|mtype| format!(" {}", mtype.enum_matcher_types()))
.collect();
f.write_all(variants.join(",\n").as_bytes()).unwrap();
f.write_all("\n}\n\n".as_bytes()).unwrap();
f.write_all(b"pub fn event_metric_name(event: &Event) -> String {
match event {
").unwrap();
f.write_all(
b"pub fn event_metric_name(event: &Event) -> String {
match *event {
",
)
.unwrap();
let variants: Vec<String> = events()
.iter()
.map(|mtype| {
let fields: Vec<String> = mtype.enum_field_names()
let fields: Vec<String> = mtype
.enum_field_names()
.iter()
.map(|_| String::from("_"))
.collect();
let variant_match: String;
if fields.len() > 0 {
variant_match = format!(
"{}({})",
&mtype.variant(),
fields
.join(", "));
variant_match = format!("{}({})", &mtype.variant(), fields.join(", "));
} else {
variant_match = format!("{}", &mtype.variant());
}
format!(" &Event::{} => String::from(\"{}\")",
&variant_match,
&mtype.metric_name(),
format!(
" Event::{} => String::from(\"{}\")",
&variant_match,
&mtype.metric_name(),
)
}).collect();
})
.collect();
f.write_all(variants.join(",\n").as_bytes()).unwrap();
f.write_all("}\n }".as_bytes()).unwrap();
// Create a struct to hold all the possible metrics
f.write_all(b"
#[derive(Debug, Clone)]
f.write_all(
b"
#[derive(Default, Debug, Clone)]
pub struct MetricCollector {
").unwrap();
",
)
.unwrap();
let variants: Vec<String> = events()
.iter()
@ -474,44 +442,32 @@ pub struct MetricCollector {
let mut fields: Vec<String> = mtype.enum_index_types();
fields.push("String".to_owned()); // Instance
format!(" {}: Arc<Mutex<HashMap<({}),{}>>>",
mtype.metric_name(),
fields.join(", "),
mtype.collector_type(),
format!(
" {}: Arc<Mutex<HashMap<({}),{}>>>",
mtype.metric_name(),
fields.join(", "),
mtype.collector_type(),
)
}).collect();
})
.collect();
f.write_all(variants.join(",\n").as_bytes()).unwrap();
f.write_all("\n}\n\n".as_bytes()).unwrap();
// Create a struct to hold all the possible metrics
f.write_all(b"
f.write_all(
b"
impl MetricCollector {
pub fn new() -> MetricCollector {
MetricCollector {
").unwrap();
Default::default()
}
let variants: Vec<String> = events()
.iter()
.map(|mtype| {
let mut fields: Vec<String> = mtype.enum_field_types();
fields.push("String".to_owned()); // Instance
format!(" {}: Arc::new(Mutex::new(HashMap::new()))",
&mtype.metric_name(),
)
}).collect();
f.write_all(variants.join(",\n").as_bytes()).unwrap();
f.write_all("\n }\n".as_bytes()).unwrap();
f.write_all("\n }\n".as_bytes()).unwrap();
f.write_all(b"
pub fn record(&self, instance: String, event: Event) {
match event {
").unwrap();
",
)
.unwrap();
let variants: Vec<String> = events()
.iter()
@ -532,7 +488,8 @@ impl MetricCollector {
index_fields = format!("({})", index_fields);
}
format!("
format!(
"
Event::{} => {{
let mut accum_table = self.{}
.lock()
@ -543,38 +500,37 @@ impl MetricCollector {
*accum += {};
}}
",
variant_match,
&mtype.metric_name(),
&mtype.metric_name(),
index_fields,
&mtype.record_value(),
variant_match,
&mtype.metric_name(),
&mtype.metric_name(),
index_fields,
&mtype.record_value(),
)
}).collect();
})
.collect();
f.write_all(variants.join(",\n").as_bytes()).unwrap();
f.write_all("\n }\n".as_bytes()).unwrap();
f.write_all("\n }\n".as_bytes()).unwrap();
f.write_all(b"pub fn prometheus_output(&self) -> String {
f.write_all(
b"pub fn prometheus_output(&self) -> String {
let mut output = String::new();
").unwrap();
",
)
.unwrap();
let variants: Vec<String> = events()
let variants: Vec<String> = events()
.iter()
.map(|mtype| {
let mut index_fields: Vec<String> = mtype.enum_index_names();
index_fields.push("instance".to_owned());
let ref_index_fields: Vec<String> = index_fields
.iter()
.map(|m| format!("ref {}", m))
.collect();
let ref_index_fields: Vec<String> =
index_fields.iter().map(|m| format!("{}", m)).collect();
let for_matcher: String;
if index_fields.len() > 1 {
for_matcher = format!("({})",
ref_index_fields.join(", "));
for_matcher = format!("({})", ref_index_fields.join(", "));
} else {
for_matcher = ref_index_fields.join(", ");
}
@ -583,7 +539,8 @@ impl MetricCollector {
.iter()
.map(|name| format!(" format!(\"{}=\\\"{{}}\\\"\", {})", &name, &name))
.collect();
format!("
format!(
"
output.push_str(\"# HELP ofborg_{} {}\n\");
output.push_str(\"# TYPE ofborg_{} {}\n\");
@ -591,7 +548,7 @@ impl MetricCollector {
.expect(\"Failed to unwrap metric mutex for {}\");
let values: Vec<String> = (*table)
.iter()
.map(|(&{}, value)| {{
.map(|({}, value)| {{
let kvs: Vec<String> = vec![
{}
];
@ -601,21 +558,20 @@ impl MetricCollector {
output.push_str(&values.join(\"\n\"));
output.push_str(\"\n\");
",
&mtype.metric_name(),
&mtype.description(),
&mtype.metric_name(),
&mtype.metric_type(),
&mtype.metric_name(),
&mtype.metric_name(),
for_matcher,
&key_value_pairs.join(",\n"),
&mtype.metric_name(),
&mtype.metric_name(),
&mtype.description(),
&mtype.metric_name(),
&mtype.metric_type(),
&mtype.metric_name(),
&mtype.metric_name(),
for_matcher,
&key_value_pairs.join(",\n"),
&mtype.metric_name(),
)
}).collect();
})
.collect();
f.write_all(variants.join("\n").as_bytes()).unwrap();
f.write_all("return output;\n }".as_bytes()).unwrap();
f.write_all("output\n }".as_bytes()).unwrap();
f.write_all("\n}".as_bytes()).unwrap();
}

27
ofborg/src/acl.rs
View file

@ -1,4 +1,3 @@
pub struct ACL {
trusted_users: Vec<String>,
known_users: Vec<String>,
@ -9,15 +8,19 @@ impl ACL {
pub fn new(
repos: Vec<String>,
mut trusted_users: Vec<String>,
mut known_users: Vec<String>
mut known_users: Vec<String>,
) -> ACL {
trusted_users.iter_mut().map(|x| *x = x.to_lowercase()).last();
trusted_users
.iter_mut()
.map(|x| *x = x.to_lowercase())
.last();
known_users.iter_mut().map(|x| *x = x.to_lowercase()).last();
return ACL {
trusted_users: trusted_users,
known_users: known_users,
repos: repos,
};
ACL {
trusted_users,
known_users,
repos,
}
}
pub fn is_repo_eligible(&self, name: &str) -> bool {
@ -46,16 +49,14 @@ impl ACL {
return false;
}
return self.known_users.contains(&user.to_lowercase());
self.known_users.contains(&user.to_lowercase())
}
pub fn can_build_unrestricted(&self, user: &str, repo: &str) -> bool {
if repo.to_lowercase() == "nixos/nixpkgs" {
return self.trusted_users.contains(&user.to_lowercase());
} else if user == "grahamc" {
return true;
self.trusted_users.contains(&user.to_lowercase())
} else {
return false;
user == "grahamc"
}
}
}

92
ofborg/src/asynccmd.rs
View file

@ -1,17 +1,17 @@
use std::thread;
use std::collections::HashMap;
use std::process::Stdio;
use std::process::ExitStatus;
use std::process::Command;
use std::io;
use std::io::BufRead;
use std::io::BufReader;
use std::io::Read;
use std::process::Child;
use std::process::Command;
use std::process::ExitStatus;
use std::process::Stdio;
use std::sync::mpsc;
use std::sync::mpsc::sync_channel;
use std::sync::mpsc::{SyncSender, Receiver};
use std::io::BufReader;
use std::io::BufRead;
use std::io;
use std::process::Child;
use std::sync::mpsc::{Receiver, SyncSender};
use std::thread::JoinHandle;
// Specifically set to fall under 1/2 of the AMQP library's
@ -48,35 +48,32 @@ enum WaitResult<T> {
fn reader_tx<R: 'static + Read + Send>(read: R, tx: SyncSender<String>) -> thread::JoinHandle<()> {
let read = BufReader::new(read);
thread::spawn(move || for line in read.lines() {
let to_send: String = match line {
Ok(line) => line,
Err(e) => {
error!("Error reading data in reader_tx: {:?}", e);
"Non-UTF8 data omitted from the log.".to_owned()
}
};
thread::spawn(move || {
for line in read.lines() {
let to_send: String = match line {
Ok(line) => line,
Err(e) => {
error!("Error reading data in reader_tx: {:?}", e);
"Non-UTF8 data omitted from the log.".to_owned()
}
};
if let Err(e) = tx.send(to_send) {
error!("Failed to send log line: {:?}", e);
if let Err(e) = tx.send(to_send) {
error!("Failed to send log line: {:?}", e);
}
}
})
}
fn spawn_join<T: Send + 'static>(
id: WaitTarget,
tx: SyncSender<(WaitTarget, WaitResult<T>)>,
waiting_on: thread::JoinHandle<T>,
) -> thread::JoinHandle<()> {
thread::spawn(move || if let Err(e) = tx.send((
id,
WaitResult::Thread(
waiting_on.join(),
),
))
{
error!("Failed to send message to the thread waiter: {:?}", e);
thread::spawn(move || {
if let Err(e) = tx.send((id, WaitResult::Thread(waiting_on.join()))) {
error!("Failed to send message to the thread waiter: {:?}", e);
}
})
}
@ -85,14 +82,10 @@ fn child_wait<T: Send + 'static>(
tx: SyncSender<(WaitTarget, WaitResult<T>)>,
mut waiting_on: Child,
) -> thread::JoinHandle<()> {
thread::spawn(move || if let Err(e) = tx.send((
id,
WaitResult::Process(
waiting_on.wait(),
),
))
{
error!("Failed to send message to the thread waiter: {:?}", e);
thread::spawn(move || {
if let Err(e) = tx.send((id, WaitResult::Process(waiting_on.wait()))) {
error!("Failed to send message to the thread waiter: {:?}", e);
}
})
}
@ -102,7 +95,8 @@ impl AsyncCmd {
}
pub fn spawn(mut self) -> SpawnedAsyncCmd {
let mut child = self.command
let mut child = self
.command
.stdin(Stdio::null())
.stderr(Stdio::piped())
.stdout(Stdio::piped())
@ -156,7 +150,6 @@ impl AsyncCmd {
return_status = Some(t);
}
}
}
None => {
error!(
@ -166,7 +159,7 @@ impl AsyncCmd {
}
}
if waiters.len() == 0 {
if waiters.is_empty() {
debug!("Closing up the waiter receiver thread, no more waiters.");
break;
}
@ -177,7 +170,7 @@ impl AsyncCmd {
waiters.len()
);
return return_status;
return_status
});
SpawnedAsyncCmd {
@ -187,28 +180,30 @@ impl AsyncCmd {
}
}
impl SpawnedAsyncCmd {
pub fn lines<'a>(&'a mut self) -> mpsc::Iter<'a, String> {
pub fn lines(&mut self) -> mpsc::Iter<'_, String> {
self.rx.iter()
}
pub fn wait(self) -> Result<ExitStatus, io::Error> {
self.waiter.join()
self.waiter
.join()
.map_err(|_err| io::Error::new(io::ErrorKind::Other, "Couldn't join thread."))
.and_then(|opt| opt.ok_or(io::Error::new(io::ErrorKind::Other, "Thread didn't return an exit status.")))
.and_then(|opt| {
opt.ok_or_else(|| {
io::Error::new(io::ErrorKind::Other, "Thread didn't return an exit status.")
})
})
.and_then(|res| res)
}
}
#[cfg(test)]
mod tests {
use super::AsyncCmd;
use std::process::Command;
use std::ffi::{OsStr, OsString};
use std::os::unix::ffi::OsStrExt;
use std::process::Command;
#[test]
fn basic_echo_test() {
@ -248,9 +243,7 @@ mod tests {
fn lots_of_small_ios_test() {
let mut cmd = Command::new("/bin/sh");
cmd.arg("-c");
cmd.arg(
"for i in `seq 1 100`; do (seq 1 100)& (seq 1 100 >&2)& wait; wait; done",
);
cmd.arg("for i in `seq 1 100`; do (seq 1 100)& (seq 1 100 >&2)& wait; wait; done");
let acmd = AsyncCmd::new(cmd);
let mut spawned = acmd.spawn();
@ -261,7 +254,6 @@ mod tests {
assert_eq!(true, exit_status.success());
}
#[test]
fn lots_of_io_test() {
let mut cmd = Command::new("/bin/sh");

17
ofborg/src/bin/build-faker.rs
View file

@ -1,22 +1,21 @@
extern crate ofborg;
extern crate amqp;
extern crate env_logger;
extern crate ofborg;
extern crate hyper;
extern crate hubcaps;
extern crate hyper;
extern crate hyper_native_tls;
use std::env;
use ofborg::commentparser;
use ofborg::config;
use ofborg::worker;
use ofborg::easyamqp;
use ofborg::message::buildjob;
use ofborg::message::{Pr, Repo};
use ofborg::notifyworker;
use ofborg::notifyworker::NotificationReceiver;
use ofborg::commentparser;
use ofborg::message::buildjob;
use ofborg::easyamqp;
use ofborg::message::{Pr, Repo};
use ofborg::worker;
fn main() {
let cfg = config::load(env::args().nth(1).unwrap().as_ref());
@ -24,11 +23,9 @@ fn main() {
println!("Hello, world!");
let mut session = easyamqp::session_from_config(&cfg.rabbitmq).unwrap();
println!("Connected to rabbitmq");
let mut channel = session.open_channel(1).unwrap();
let repo_msg = Repo {

26
ofborg/src/bin/builder.rs
View file

@ -1,21 +1,20 @@
extern crate ofborg;
extern crate amqp;
extern crate env_logger;
extern crate ofborg;
#[macro_use]
extern crate log;
use std::env;
use std::path::Path;
use amqp::Basic;
use ofborg::config;
use ofborg::checkout;
use ofborg::notifyworker;
use ofborg::tasks;
use ofborg::config;
use ofborg::easyamqp;
use ofborg::easyamqp::TypedWrappers;
use ofborg::notifyworker;
use ofborg::tasks;
use std::path::Path;
fn main() {
let cfg = config::load(env::args().nth(1).unwrap().as_ref());
@ -51,9 +50,8 @@ fn main() {
})
.unwrap();
let queue_name: String;
if cfg.runner.build_all_jobs != Some(true) {
queue_name = channel
let queue_name: String = if cfg.runner.build_all_jobs != Some(true) {
channel
.declare_queue(easyamqp::QueueConfig {
queue: format!("build-inputs-{}", cfg.nix.system.clone()),
passive: false,
@ -63,11 +61,12 @@ fn main() {
no_wait: false,
arguments: None,
})
.unwrap().queue;
.unwrap()
.queue
} else {
warn!("Building all jobs, please don't use this unless you're");
warn!("developing and have Graham's permission!");
queue_name = channel
channel
.declare_queue(easyamqp::QueueConfig {
queue: "".to_owned(),
passive: false,
@ -77,8 +76,9 @@ fn main() {
no_wait: false,
arguments: None,
})
.unwrap().queue;
}
.unwrap()
.queue
};
channel
.bind_queue(easyamqp::BindQueueConfig {

11
ofborg/src/bin/evaluation-filter.rs
View file

@ -1,22 +1,20 @@
extern crate ofborg;
extern crate amqp;
extern crate env_logger;
extern crate ofborg;
extern crate hyper;
extern crate hubcaps;
extern crate hyper;
extern crate hyper_native_tls;
use std::env;
use amqp::Basic;
use ofborg::config;
use ofborg::worker;
use ofborg::tasks;
use ofborg::easyamqp;
use ofborg::easyamqp::TypedWrappers;
use ofborg::tasks;
use ofborg::worker;
fn main() {
let cfg = config::load(env::args().nth(1).unwrap().as_ref());
@ -24,7 +22,6 @@ fn main() {
println!("Hello, world!");
let mut session = easyamqp::session_from_config(&cfg.rabbitmq).unwrap();
println!("Connected to rabbitmq");

11
ofborg/src/bin/github-comment-filter.rs
View file

@ -1,22 +1,20 @@
extern crate ofborg;
extern crate amqp;
extern crate env_logger;
extern crate ofborg;
extern crate hyper;
extern crate hubcaps;
extern crate hyper;
extern crate hyper_native_tls;
use std::env;
use amqp::Basic;
use ofborg::config;
use ofborg::worker;
use ofborg::tasks;
use ofborg::easyamqp;
use ofborg::easyamqp::TypedWrappers;
use ofborg::tasks;
use ofborg::worker;
fn main() {
let cfg = config::load(env::args().nth(1).unwrap().as_ref());
@ -24,7 +22,6 @@ fn main() {
println!("Hello, world!");
let mut session = easyamqp::session_from_config(&cfg.rabbitmq).unwrap();
println!("Connected to rabbitmq");

10
ofborg/src/bin/github-comment-poster.rs
View file

@ -1,22 +1,20 @@
extern crate ofborg;
extern crate amqp;
extern crate env_logger;
extern crate ofborg;
extern crate hyper;
extern crate hubcaps;
extern crate hyper;
extern crate hyper_native_tls;
use std::env;
use amqp::Basic;
use ofborg::config;
use ofborg::worker;
use ofborg::tasks;
use ofborg::easyamqp;
use ofborg::easyamqp::TypedWrappers;
use ofborg::tasks;
use ofborg::worker;
fn main() {
let cfg = config::load(env::args().nth(1).unwrap().as_ref());

9
ofborg/src/bin/log-message-collector.rs
View file

@ -1,16 +1,15 @@
extern crate ofborg;
extern crate amqp;
extern crate env_logger;
extern crate ofborg;
use std::env;
use std::path::PathBuf;
use ofborg::config;
use ofborg::worker;
use ofborg::tasks;
use ofborg::easyamqp;
use ofborg::easyamqp::TypedWrappers;
use ofborg::tasks;
use ofborg::worker;
fn main() {
let cfg = config::load(env::args().nth(1).unwrap().as_ref());
@ -75,7 +74,6 @@ fn main() {
)
.unwrap();
channel.start_consuming();
println!("Finished consuming?");
@ -84,5 +82,4 @@ fn main() {
println!("Closed the channel");
session.close(200, "Good Bye");
println!("Closed the session... EOF");
}

13
ofborg/src/bin/log-message-generator.rs
View file

@ -1,18 +1,18 @@
extern crate ofborg;
extern crate amqp;
extern crate env_logger;
extern crate ofborg;
use std::env;
use std::time::Duration;
use std::thread;
use std::time::Duration;
use ofborg::message::{Pr, Repo};
use ofborg::config;
use ofborg::easyamqp;
use ofborg::message::buildjob;
use ofborg::notifyworker;
use ofborg::tasks::build;
use ofborg::message::buildjob;
use ofborg::easyamqp;
fn main() {
let cfg = config::load(env::args().nth(1).unwrap().as_ref());
@ -39,10 +39,7 @@ fn main() {
owner: "ofborg-test".to_owned(),
},
subset: None,
logs: Some((
Some(String::from("logs")),
Some(String::from("build.log")),
)),
logs: Some((Some(String::from("logs")), Some(String::from("build.log")))),
statusreport: Some((Some(String::from("build-results")), None)),
request_id: "bogus-request-id".to_owned(),
};

24
ofborg/src/bin/mass-rebuilder.rs
View file

@ -1,26 +1,30 @@
extern crate ofborg;
extern crate amqp;
extern crate env_logger;
extern crate ofborg;
extern crate sys_info;
use ofborg::checkout;
use ofborg::config;
use ofborg::tasks;
use std::env;
use std::path::Path;
use std::process;
use ofborg::tasks;
use ofborg::config;
use ofborg::checkout;
use ofborg::stats;
use ofborg::worker;
use amqp::Basic;
use ofborg::easyamqp;
use ofborg::easyamqp::TypedWrappers;
use ofborg::stats;
use ofborg::worker;
fn main() {
let memory_info = sys_info::mem_info().expect("Unable to get memory information from OS");
if memory_info.avail < 8 * 1024 * 1024 { // seems this stuff is in kilobytes?
println!("Less than 8Gb of memory available (got {:.2}Gb). Aborting.", (memory_info.avail as f32) / 1024.0 / 1024.0 );
if memory_info.avail < 8 * 1024 * 1024 {
// seems this stuff is in kilobytes?
println!(
"Less than 8Gb of memory available (got {:.2}Gb). Aborting.",
(memory_info.avail as f32) / 1024.0 / 1024.0
);
process::exit(1);
};
@ -40,12 +44,12 @@ fn main() {
let events = stats::RabbitMQ::new(
&format!("{}-{}", cfg.runner.identity.clone(), cfg.nix.system.clone()),
session.open_channel(3).unwrap()
session.open_channel(3).unwrap(),
);
let mrw = tasks::massrebuilder::MassRebuildWorker::new(
cloner,
nix,
&nix,
cfg.github(),
cfg.acl(),
cfg.runner.identity.clone(),

9
ofborg/src/bin/simple-build.rs
View file

@ -1,14 +1,13 @@
extern crate ofborg;
extern crate amqp;
extern crate env_logger;
extern crate ofborg;
use std::env;
use std::path::Path;
use ofborg::config;
use std::fs::File;
use std::io::Read;
use ofborg::config;
use std::path::Path;
fn main() {
let cfg = config::load(env::args().nth(1).unwrap().as_ref());
@ -32,5 +31,5 @@ fn main() {
fn file_to_str(f: &mut File) -> String {
let mut buffer = Vec::new();
f.read_to_end(&mut buffer).expect("Reading eval output");
return String::from(String::from_utf8_lossy(&buffer));
String::from(String::from_utf8_lossy(&buffer))
}

18
ofborg/src/bin/stats.rs
View file

@ -1,13 +1,13 @@
extern crate hyper;
extern crate amqp;
extern crate hyper;
extern crate ofborg;
use ofborg::{config, easyamqp, stats, tasks, worker};
use std::env;
use ofborg::{easyamqp, tasks, worker, config, stats};
use amqp::Basic;
use ofborg::easyamqp::TypedWrappers;
use hyper::server::{Request, Response, Server};
use ofborg::easyamqp::TypedWrappers;
use std::thread;
@ -17,21 +17,17 @@ fn main() {
println!("Hello, world!");
let mut session = easyamqp::session_from_config(&cfg.rabbitmq).unwrap();
println!("Connected to rabbitmq");
let events = stats::RabbitMQ::new(
&format!("{}-{}", cfg.runner.identity.clone(), cfg.nix.system.clone()),
session.open_channel(3).unwrap()
session.open_channel(3).unwrap(),
);
let metrics = stats::MetricCollector::new();
let collector = tasks::statscollector::StatCollectorWorker::new(
events,
metrics.clone(),
);
let collector = tasks::statscollector::StatCollectorWorker::new(events, metrics.clone());
let mut channel = session.open_channel(1).unwrap();
channel
@ -85,8 +81,7 @@ fn main() {
)
.unwrap();
thread::spawn(||{
thread::spawn(|| {
let addr = "0.0.0.0:9898";
println!("listening addr {:?}", addr);
Server::http(addr)
@ -97,7 +92,6 @@ fn main() {
.unwrap();
});
channel.start_consuming();
println!("Finished consuming?");

107
ofborg/src/checkout.rs
View file

@ -1,11 +1,11 @@
use std::path::{Path, PathBuf};
use md5;
use std::fs;
use std::io::{Error, ErrorKind};
use ofborg::clone;
use ofborg::clone::GitClonable;
use std::ffi::OsStr;
use std::ffi::OsString;
use std::fs;
use std::io::{Error, ErrorKind};
use std::path::{Path, PathBuf};
use std::process::Command;
pub struct CachedCloner {
@ -13,7 +13,9 @@ pub struct CachedCloner {
}
pub fn cached_cloner(path: &Path) -> CachedCloner {
return CachedCloner { root: path.to_path_buf() };
CachedCloner {
root: path.to_path_buf(),
}
}
pub struct CachedProject {
@ -29,7 +31,7 @@ pub struct CachedProjectCo {
}
impl CachedCloner {
pub fn project(&self, name: String, clone_url: String) -> CachedProject {
pub fn project(&self, name: &str, clone_url: String) -> CachedProject {
// <root>/repo/<hash>/clone
// <root>/repo/<hash>/clone.lock
// <root>/repo/<hash>/<type>/<id>
@ -39,10 +41,10 @@ impl CachedCloner {
new_root.push("repo");
new_root.push(format!("{:x}", md5::compute(&name)));
return CachedProject {
CachedProject {
root: new_root,
clone_url: clone_url,
};
clone_url,
}
}
}
@ -53,12 +55,12 @@ impl CachedProject {
let mut new_root = self.root.clone();
new_root.push(use_category);
return Ok(CachedProjectCo {
Ok(CachedProjectCo {
root: new_root,
id: id,
id,
clone_url: self.clone_from().clone(),
local_reference: self.clone_to().clone(),
});
})
}
fn prefetch_cache(&self) -> Result<PathBuf, Error> {
@ -67,7 +69,7 @@ impl CachedProject {
self.clone_repo()?;
self.fetch_repo()?;
return Ok(self.clone_to());
Ok(self.clone_to())
}
}
@ -89,7 +91,7 @@ impl CachedProjectCo {
// let build_dir = self.build_dir();
return Ok(self.clone_to().to_str().unwrap().to_string());
Ok(self.clone_to().to_str().unwrap().to_string())
}
pub fn fetch_pr(&self, pr_id: u64) -> Result<(), Error> {
@ -105,9 +107,9 @@ impl CachedProjectCo {
lock.unlock();
if result.success() {
return Ok(());
Ok(())
} else {
return Err(Error::new(ErrorKind::Other, "Failed to fetch PR"));
Err(Error::new(ErrorKind::Other, "Failed to fetch PR"))
}
}
@ -124,7 +126,7 @@ impl CachedProjectCo {
lock.unlock();
return result.success();
result.success()
}
pub fn merge_commit(&self, commit: &OsStr) -> Result<(), Error> {
@ -142,9 +144,9 @@ impl CachedProjectCo {
lock.unlock();
if result.success() {
return Ok(());
Ok(())
} else {
return Err(Error::new(ErrorKind::Other, "Failed to merge"));
Err(Error::new(ErrorKind::Other, "Failed to merge"))
}
}
@ -161,17 +163,15 @@ impl CachedProjectCo {
lock.unlock();
if result.status.success() {
return Ok(
String::from_utf8_lossy(&result.stdout)
.lines()
.map(|l| l.to_owned())
.collect(),
);
Ok(String::from_utf8_lossy(&result.stdout)
.lines()
.map(|l| l.to_owned())
.collect())
} else {
return Err(Error::new(
Err(Error::new(
ErrorKind::Other,
String::from_utf8_lossy(&result.stderr).to_lowercase(),
));
))
}
}
@ -188,77 +188,74 @@ impl CachedProjectCo {
lock.unlock();
if result.status.success() {
return Ok(
String::from_utf8_lossy(&result.stdout)
.lines()
.map(|l| l.to_owned())
.collect(),
);
Ok(String::from_utf8_lossy(&result.stdout)
.lines()
.map(|l| l.to_owned())
.collect())
} else {
return Err(Error::new(
Err(Error::new(
ErrorKind::Other,
String::from_utf8_lossy(&result.stderr).to_lowercase(),
));
))
}
}
}
impl clone::GitClonable for CachedProjectCo {
fn clone_from(&self) -> String {
return self.clone_url.clone();
self.clone_url.clone()
}
fn clone_to(&self) -> PathBuf {
let mut clone_path = self.root.clone();
clone_path.push(&self.id);
return clone_path;
clone_path
}
fn lock_path(&self) -> PathBuf {
let mut lock_path = self.root.clone();
lock_path.push(format!("{}.lock", self.id));
return lock_path;
lock_path
}
fn extra_clone_args(&self) -> Vec<&OsStr> {
let local_ref = self.local_reference.as_ref();
return vec![
vec![
OsStr::new("--shared"),
OsStr::new("--reference-if-able"),
local_ref,
];
]
}
}
impl clone::GitClonable for CachedProject {
fn clone_from(&self) -> String {
return self.clone_url.clone();
self.clone_url.clone()
}
fn clone_to(&self) -> PathBuf {
let mut clone_path = self.root.clone();
clone_path.push("clone");
return clone_path;
clone_path
}
fn lock_path(&self) -> PathBuf {
let mut clone_path = self.root.clone();
clone_path.push("clone.lock");
return clone_path;
clone_path
}
fn extra_clone_args(&self) -> Vec<&OsStr> {
return vec![OsStr::new("--bare")];
vec![OsStr::new("--bare")]
}
}
#[cfg(test)]
mod tests {
use super::*;
use ofborg::test_scratch::TestScratch;
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use ofborg::test_scratch::TestScratch;
fn tpath(component: &str) -> PathBuf {
return Path::new(env!("CARGO_MANIFEST_DIR")).join(component);
@ -273,8 +270,8 @@ mod tests {
.output()
.expect("building the test PR failed");
let stderr = String::from_utf8(output.stderr)
.unwrap_or_else(|err| format!("warning: {}", err));
let stderr =
String::from_utf8(output.stderr).unwrap_or_else(|err| format!("warning: {}", err));
println!("{}", stderr);
let hash = String::from_utf8(output.stdout).expect("Should just be a hash");
@ -290,7 +287,7 @@ mod tests {
let hash = make_pr_repo(&bare.path(), &mk_co.path());
let cloner = cached_cloner(&workingdir.path());
let project = cloner.project("commit-msg-list".to_owned(), bare.string());
let project = cloner.project("commit-msg-list", bare.string());
let working_co = project
.clone_for("testing-commit-msgs".to_owned(), "123".to_owned())
.expect("clone should work");
@ -301,9 +298,9 @@ mod tests {
let expect: Vec<String> = vec!["check out this cool PR".to_owned()];
assert_eq!(
working_co.commit_messages_from_head(&hash).expect(
"fetching messages should work",
),
working_co
.commit_messages_from_head(&hash)
.expect("fetching messages should work",),
expect
);
}
@ -317,7 +314,7 @@ mod tests {
let hash = make_pr_repo(&bare.path(), &mk_co.path());
let cloner = cached_cloner(&workingdir.path());
let project = cloner.project("commit-files-changed-list".to_owned(), bare.string());
let project = cloner.project("commit-files-changed-list", bare.string());
let working_co = project
.clone_for("testing-files-changed".to_owned(), "123".to_owned())
.expect("clone should work");
@ -328,9 +325,9 @@ mod tests {
let expect: Vec<String> = vec!["default.nix".to_owned(), "hi another file".to_owned()];
assert_eq!(
working_co.files_changed_from_head(&hash).expect(
"fetching files changed should work",
),
working_co
.files_changed_from_head(&hash)
.expect("fetching files changed should work",),
expect
);
}

56
ofborg/src/clone.rs
View file

@ -1,9 +1,9 @@
use std::path::PathBuf;
use fs2::FileExt;
use std::ffi::OsStr;
use std::fs;
use std::io::{Error, ErrorKind};
use std::path::PathBuf;
use std::process::Command;
use std::ffi::OsStr;
pub struct Lock {
lock: Option<fs::File>,
@ -28,25 +28,22 @@ pub trait GitClonable {
match fs::File::create(self.lock_path()) {
Err(e) => {
warn!("Failed to create lock file {:?}: {}", self.lock_path(), e);
return Err(e);
Err(e)
}
Ok(lock) => {
match lock.lock_exclusive() {
Err(e) => {
warn!(
"Failed to get exclusive lock on file {:?}: {}",
self.lock_path(),
e
);
return Err(e);
}
Ok(_) => {
debug!("Got lock on {:?}", self.lock_path());
return Ok(Lock { lock: Some(lock) });
}
Ok(lock) => match lock.lock_exclusive() {
Err(e) => {
warn!(
"Failed to get exclusive lock on file {:?}: {}",
self.lock_path(),
e
);
Err(e)
}
}
Ok(_) => {
debug!("Got lock on {:?}", self.lock_path());
Ok(Lock { lock: Some(lock) })
}
},
}
}
@ -74,9 +71,16 @@ pub trait GitClonable {
lock.unlock();
if result.success() {
return Ok(());
Ok(())
} else {
return Err(Error::new(ErrorKind::Other, format!("Failed to clone from {:?} to {:?}", self.clone_from(), self.clone_to())));
Err(Error::new(
ErrorKind::Other,
format!(
"Failed to clone from {:?} to {:?}",
self.clone_from(),
self.clone_to()
),
))
}
}
@ -93,9 +97,9 @@ pub trait GitClonable {
lock.unlock();
if result.success() {
return Ok(());
Ok(())
} else {
return Err(Error::new(ErrorKind::Other, "Failed to fetch"));
Err(Error::new(ErrorKind::Other, "Failed to fetch"))
}
}
@ -125,7 +129,7 @@ pub trait GitClonable {
lock.unlock();
return Ok(());
Ok(())
}
fn checkout(&self, git_ref: &OsStr) -> Result<(), Error> {
@ -142,9 +146,9 @@ pub trait GitClonable {
lock.unlock();
if result.success() {
return Ok(());
Ok(())
} else {
return Err(Error::new(ErrorKind::Other, "Failed to checkout"));
Err(Error::new(ErrorKind::Other, "Failed to checkout"))
}
}
}

127
ofborg/src/commentparser.rs
View file

@ -1,17 +1,18 @@
use nom::types::CompleteStr;
pub fn parse(text: &str) -> Option<Vec<Instruction>> {
let instructions: Vec<Instruction> = text.lines()
let instructions: Vec<Instruction> = text
.lines()
.flat_map(|s| match parse_line(s) {
Some(instructions) => instructions.into_iter(),
None => Vec::new().into_iter(),
})
.collect();
if instructions.len() == 0 {
return None;
if instructions.is_empty() {
None
} else {
return Some(instructions);
Some(instructions)
}
}
@ -50,10 +51,11 @@ named!(parse_line_impl(CompleteStr) -> Option<Vec<Instruction>>, alt!(
pub fn parse_line(text: &str) -> Option<Vec<Instruction>> {
match parse_line_impl(CompleteStr(text)) {
Ok((_, res)) => res,
Err(e) => { // This should likely never happen thanks to the | value!(None), but well...
Err(e) => {
// This should likely never happen thanks to the | value!(None), but well...
warn!("Failed parsing string ‘{}’: result was {:?}", text, e);
None
},
}
}
}
@ -110,10 +112,7 @@ mod tests {
assert_eq!(
Some(vec![
Instruction::Eval,
Instruction::Build(
Subset::Nixpkgs,
vec![String::from("foo")]
),
Instruction::Build(Subset::Nixpkgs, vec![String::from("foo")]),
]),
parse("@grahamcofborg eval @grahamcofborg build foo")
);
@ -123,15 +122,9 @@ mod tests {
fn build_and_eval_and_build_comment() {
assert_eq!(
Some(vec![
Instruction::Build(
Subset::Nixpkgs,
vec![String::from("bar")]
),
Instruction::Build(Subset::Nixpkgs, vec![String::from("bar")]),
Instruction::Eval,
Instruction::Build(
Subset::Nixpkgs,
vec![String::from("foo")]
),
Instruction::Build(Subset::Nixpkgs, vec![String::from("foo")]),
]),
parse(
"
@ -146,15 +139,9 @@ mod tests {
fn complex_comment_with_paragraphs() {
assert_eq!(
Some(vec![
Instruction::Build(
Subset::Nixpkgs,
vec![String::from("bar")]
),
Instruction::Build(Subset::Nixpkgs, vec![String::from("bar")]),
Instruction::Eval,
Instruction::Build(
Subset::Nixpkgs,
vec![String::from("foo")]
),
Instruction::Build(Subset::Nixpkgs, vec![String::from("foo")]),
]),
parse(
"
@ -172,15 +159,11 @@ Also, just in case, let's try foo
);
}
#[test]
fn build_and_eval_comment() {
assert_eq!(
Some(vec![
Instruction::Build(
Subset::Nixpkgs,
vec![String::from("foo")]
),
Instruction::Build(Subset::Nixpkgs, vec![String::from("foo")]),
Instruction::Eval,
]),
parse("@grahamcofborg build foo @grahamcofborg eval")
@ -190,12 +173,10 @@ Also, just in case, let's try foo
#[test]
fn build_comment() {
assert_eq!(
Some(vec![
Instruction::Build(
Subset::Nixpkgs,
vec![String::from("foo"), String::from("bar")]
),
]),
Some(vec![Instruction::Build(
Subset::Nixpkgs,
vec![String::from("foo"), String::from("bar")]
),]),
parse(
"@GrahamCOfBorg build foo bar
@ -207,16 +188,14 @@ baz",
#[test]
fn test_comment() {
assert_eq!(
Some(vec![
Instruction::Build(
Subset::NixOS,
vec![
String::from("tests.foo"),
String::from("tests.bar"),
String::from("tests.baz"),
]
),
]),
Some(vec![Instruction::Build(
Subset::NixOS,
vec![
String::from("tests.foo"),
String::from("tests.bar"),
String::from("tests.baz"),
]
),]),
parse("@GrahamCOfBorg test foo bar baz")
);
}
@ -224,16 +203,14 @@ baz",
#[test]
fn build_comment_newlines() {
assert_eq!(
Some(vec![
Instruction::Build(
Subset::Nixpkgs,
vec![
String::from("foo"),
String::from("bar"),
String::from("baz"),
]
),
]),
Some(vec![Instruction::Build(
Subset::Nixpkgs,
vec![
String::from("foo"),
String::from("bar"),
String::from("baz"),
]
),]),
parse("@GrahamCOfBorg build foo bar baz")
);
}
@ -241,16 +218,14 @@ baz",
#[test]
fn build_comment_lower() {
assert_eq!(
Some(vec![
Instruction::Build(
Subset::Nixpkgs,
vec![
String::from("foo"),
String::from("bar"),
String::from("baz"),
]
),
]),
Some(vec![Instruction::Build(
Subset::Nixpkgs,
vec![
String::from("foo"),
String::from("bar"),
String::from("baz"),
]
),]),
parse("@grahamcofborg build foo bar baz")
);
}
@ -258,16 +233,14 @@ baz",
#[test]
fn build_comment_lower_package_case_retained() {
assert_eq!(
Some(vec![
Instruction::Build(
Subset::Nixpkgs,
vec![
String::from("foo"),
String::from("bar"),
String::from("baz.Baz"),
]
),
]),
Some(vec![Instruction::Build(
Subset::Nixpkgs,
vec![
String::from("foo"),
String::from("bar"),
String::from("baz.Baz"),
]
),]),
parse("@grahamcofborg build foo bar baz.Baz")
);
}

12
ofborg/src/commitstatus.rs
View file

@ -20,20 +20,20 @@ impl<'a> CommitStatus<'a> {
url: Option<String>,
) -> CommitStatus<'a> {
let mut stat = CommitStatus {
api: api,
sha: sha,
context: context,
description: description,
api,
sha,
context,
description,
url: "".to_owned(),
};
stat.set_url(url);
return stat;
stat
}
pub fn set_url(&mut self, url: Option<String>) {
self.url = url.unwrap_or(String::from(""))
self.url = url.unwrap_or_else(|| String::from(""))
}
pub fn set_with_description(&mut self, description: &str, state: hubcaps::statuses::State) {

71
ofborg/src/config.rs
View file

@ -1,14 +1,13 @@
use serde_json;
use std::fs::File;
use std::path::{Path, PathBuf};
use std::io::Read;
use hyper::Client;
use hyper::net::HttpsConnector;
use hyper_native_tls::NativeTlsClient;
use hubcaps::{Credentials, Github, InstallationTokenGenerator, JWTCredentials};
use hyper::net::HttpsConnector;
use hyper::Client;
use hyper_native_tls::NativeTlsClient;
use nix::Nix;
use serde_json;
use std::collections::HashMap;
use std::fs::File;
use std::io::Read;
use std::path::{Path, PathBuf};
use ofborg::acl;
@ -44,7 +43,7 @@ pub struct NixConfig {
pub system: String,
pub remote: String,
pub build_timeout_seconds: u16,
pub initial_heap_size: Option<String>
pub initial_heap_size: Option<String>,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
@ -77,7 +76,7 @@ pub struct RunnerConfig {
/// architecture.
///
/// This should only be turned on for development.
pub build_all_jobs: Option<bool>
pub build_all_jobs: Option<bool>,
}
#[derive(Serialize, Deserialize, Debug)]
@ -87,21 +86,24 @@ pub struct CheckoutConfig {
impl Config {
pub fn whoami(&self) -> String {
return format!("{}-{}", self.runner.identity, self.nix.system);
format!("{}-{}", self.runner.identity, self.nix.system)
}
pub fn acl(&self) -> acl::ACL {
return acl::ACL::new(
self.runner.repos.clone().expect(
"fetching config's runner.repos",
),
self.runner.trusted_users.clone().expect(
"fetching config's runner.trusted_users",
),
self.runner.known_users.clone().expect(
"fetching config's runner.known_users",
),
);
acl::ACL::new(
self.runner
.repos
.clone()
.expect("fetching config's runner.repos"),
self.runner
.trusted_users
.clone()
.expect("fetching config's runner.trusted_users"),
self.runner
.known_users
.clone()
.expect("fetching config's runner.known_users"),
)
}
pub fn github(&self) -> Github {
@ -119,15 +121,10 @@ impl Config {
"github.com/grahamc/ofborg (app)",
// tls configured hyper client
Client::with_connector(HttpsConnector::new(NativeTlsClient::new().unwrap())),
Credentials::InstallationToken(
InstallationTokenGenerator::new(
conf.installation_id,
JWTCredentials::new(
conf.app_id,
conf.private_key
)
)
)
Credentials::InstallationToken(InstallationTokenGenerator::new(
conf.installation_id,
JWTCredentials::new(conf.app_id, conf.private_key),
)),
)
}
@ -141,25 +138,25 @@ impl Config {
panic!();
}
return Nix::new(
Nix::new(
self.nix.system.clone(),
self.nix.remote.clone(),
self.nix.build_timeout_seconds,
self.nix.initial_heap_size.clone(),
);
)
}
}
impl RabbitMQConfig {
pub fn as_uri(&self) -> String {
return format!(
format!(
"{}://{}:{}@{}/{}",
if self.ssl { "amqps" } else { "amqp" },
self.username,
self.password,
self.host,
self.virtualhost.clone().unwrap_or("/".to_owned()),
);
self.virtualhost.clone().unwrap_or_else(|| "/".to_owned()),
)
}
}
@ -170,5 +167,5 @@ pub fn load(filename: &Path) -> Config {
let deserialized: Config = serde_json::from_str(&contents).unwrap();
return deserialized;
deserialized
}

24
ofborg/src/easyamqp.rs
View file

@ -1,8 +1,7 @@
use ofborg;
use ofborg::config::RabbitMQConfig;
use amqp;
use amqp::Basic;
use ofborg;
use ofborg::config::RabbitMQConfig;
pub struct ConsumeConfig {
/// Specifies the name of the queue to consume from.
@ -301,18 +300,18 @@ pub fn session_from_config(config: &RabbitMQConfig) -> Result<amqp::Session, amq
amqp::AMQPScheme::AMQPS => 5671,
amqp::AMQPScheme::AMQP => 5672,
},
vhost: config.virtualhost.clone().unwrap_or("/".to_owned()),
vhost: config.virtualhost.clone().unwrap_or_else(|| "/".to_owned()),
login: config.username.clone(),
password: config.password.clone(),
scheme: scheme,
properties: properties,
scheme,
properties,
..amqp::Options::default()
};
let session = try!(amqp::Session::new(options));
info!("Connected to {}", &config.host);
return Ok(session);
Ok(session)
}
pub trait TypedWrappers {
@ -349,7 +348,7 @@ impl TypedWrappers for amqp::Channel {
config.no_ack,
config.exclusive,
config.no_wait,
config.arguments.unwrap_or(amqp::Table::new()),
config.arguments.unwrap_or_else(amqp::Table::new),
)
}
@ -365,11 +364,10 @@ impl TypedWrappers for amqp::Channel {
config.auto_delete,
config.internal,
config.no_wait,
config.arguments.unwrap_or(amqp::Table::new()),
config.arguments.unwrap_or_else(amqp::Table::new),
)
}
fn declare_queue(
&mut self,
config: QueueConfig,
@ -381,7 +379,7 @@ impl TypedWrappers for amqp::Channel {
config.exclusive,
config.auto_delete,
config.no_wait,
config.arguments.unwrap_or(amqp::Table::new()),
config.arguments.unwrap_or_else(amqp::Table::new),
)
}
@ -392,9 +390,9 @@ impl TypedWrappers for amqp::Channel {
self.queue_bind(
config.queue,
config.exchange,
config.routing_key.unwrap_or("".to_owned()),
config.routing_key.unwrap_or_else(|| "".to_owned()),
config.no_wait,
config.arguments.unwrap_or(amqp::Table::new()),
config.arguments.unwrap_or_else(amqp::Table::new),
)
}
}

12
ofborg/src/evalchecker.rs
View file

@ -1,9 +1,9 @@
extern crate amqp;
extern crate env_logger;
use ofborg::nix;
use std::fs::File;
use std::path::Path;
use ofborg::nix;
pub struct EvalChecker {
name: String,
@ -16,9 +16,9 @@ impl EvalChecker {
pub fn new(name: &str, op: nix::Operation, args: Vec<String>, nix: nix::Nix) -> EvalChecker {
EvalChecker {
name: name.to_owned(),
op: op,
args: args,
nix: nix,
op,
args,
nix,
}
}
@ -27,12 +27,12 @@ impl EvalChecker {
}
pub fn execute(&self, path: &Path) -> Result<File, File> {
self.nix.safely(self.op.clone(), path, self.args.clone(), false)
self.nix.safely(&self.op, path, self.args.clone(), false)
}
pub fn cli_cmd(&self) -> String {
let mut cli = vec![self.op.to_string()];
cli.append(&mut self.args.clone());
return cli.join(" ");
cli.join(" ")
}
}

4
ofborg/src/files.rs
View file

@ -1,8 +1,8 @@
use std::io::Read;
use std::fs::File;
use std::io::Read;
pub fn file_to_str(f: &mut File) -> String {
let mut buffer = Vec::new();
f.read_to_end(&mut buffer).expect("Reading eval output");
return String::from(String::from_utf8_lossy(&buffer));
String::from(String::from_utf8_lossy(&buffer))
}

4
ofborg/src/ghevent/issuecomment.rs
View file

@ -1,4 +1,4 @@
use ofborg::ghevent::{Comment, Repository, Issue};
use ofborg::ghevent::{Comment, Issue, Repository};
#[derive(Serialize, Deserialize, Debug)]
pub struct IssueComment {
@ -9,7 +9,7 @@ pub struct IssueComment {
}
#[derive(Serialize, Deserialize, Debug, PartialEq)]
#[serde(rename_all="snake_case")]
#[serde(rename_all = "snake_case")]
pub enum IssueCommentAction {
Created,
Edited,

8
ofborg/src/ghevent/mod.rs
View file

@ -2,6 +2,8 @@ mod common;
mod issuecomment;
mod pullrequestevent;
pub use self::issuecomment::{IssueComment,IssueCommentAction};
pub use self::pullrequestevent::{PullRequest, PullRequestEvent, PullRequestAction, PullRequestState};
pub use self::common::{Issue, Repository, User, Comment};
pub use self::common::{Comment, Issue, Repository, User};
pub use self::issuecomment::{IssueComment, IssueCommentAction};
pub use self::pullrequestevent::{
PullRequest, PullRequestAction, PullRequestEvent, PullRequestState,
};

14
ofborg/src/ghevent/pullrequestevent.rs
View file

@ -1,4 +1,4 @@
use ofborg::ghevent::{Repository};
use ofborg::ghevent::Repository;
#[derive(Serialize, Deserialize)]
pub struct PullRequestEvent {
@ -16,7 +16,7 @@ pub struct PullRequestChanges {
#[derive(Serialize, Deserialize, Debug)]
pub struct BaseChange {
#[serde(rename="ref")]
#[serde(rename = "ref")]
pub git_ref: ChangeWas,
pub sha: ChangeWas,
}
@ -27,14 +27,14 @@ pub struct ChangeWas {
}
#[derive(Serialize, Deserialize, Debug, PartialEq)]
#[serde(rename_all="snake_case")]
#[serde(rename_all = "snake_case")]
pub enum PullRequestState {
Open,
Closed,
}
#[derive(Serialize, Deserialize, Debug, PartialEq)]
#[serde(rename_all="snake_case")]
#[serde(rename_all = "snake_case")]
pub enum PullRequestAction {
Assigned,
Unassigned,
@ -51,7 +51,7 @@ pub enum PullRequestAction {
#[derive(Serialize, Deserialize, Debug)]
pub struct PullRequestRef {
#[serde(rename="ref")]
#[serde(rename = "ref")]
pub git_ref: String,
pub sha: String,
}
@ -63,7 +63,6 @@ pub struct PullRequest {
pub head: PullRequestRef,
}
#[cfg(test)]
mod tests {
use super::*;
@ -74,7 +73,6 @@ mod tests {
let data = include_str!("../../test-srcs/events/pr-changed-base.json");
let _p: PullRequestEvent =
serde_json::from_str(&data.to_string())
.expect("Should properly deserialize");
serde_json::from_str(&data.to_string()).expect("Should properly deserialize");
}
}

85
ofborg/src/lib.rs
View file

@ -1,5 +1,4 @@
#![recursion_limit="512"]
#![recursion_limit = "512"]
#[macro_use]
extern crate serde_derive;
@ -13,77 +12,81 @@ extern crate log;
#[macro_use]
extern crate nom;
extern crate amqp;
extern crate chrono;
extern crate either;
extern crate env_logger;
extern crate fs2;
extern crate hubcaps;
extern crate hyper;
extern crate hyper_native_tls;
extern crate either;
extern crate lru_cache;
extern crate tempfile;
extern crate amqp;
extern crate fs2;
extern crate md5;
extern crate tempfile;
extern crate uuid;
extern crate env_logger;
extern crate chrono;
use std::env;
pub mod acl;
pub mod asynccmd;
pub mod checkout;
pub mod locks;
pub mod clone;
pub mod worker;
pub mod config;
pub mod message;
pub mod tasks;
pub mod evalchecker;
pub mod files;
pub mod nix;
pub mod stats;
pub mod ghevent;
pub mod commentparser;
pub mod commitstatus;
pub mod outpathdiff;
pub mod tagger;
pub mod asynccmd;
pub mod notifyworker;
pub mod writetoline;
pub mod test_scratch;
pub mod config;
pub mod easyamqp;
pub mod evalchecker;
pub mod files;
pub mod ghevent;
pub mod locks;
pub mod message;
pub mod nix;
pub mod notifyworker;
pub mod outpathdiff;
pub mod stats;
pub mod tagger;
pub mod tasks;
pub mod test_scratch;
pub mod worker;
pub mod writetoline;
pub mod ofborg {
pub use acl;
pub use asynccmd;
pub use stats;
pub use config;
pub use checkout;
pub use locks;
pub use clone;
pub use worker;
pub use notifyworker;
pub use message;
pub use tasks;
pub use commentparser;
pub use commitstatus;
pub use config;
pub use easyamqp;
pub use evalchecker;
pub use files;
pub use commitstatus;
pub use ghevent;
pub use locks;
pub use message;
pub use nix;
pub use acl;
pub use commentparser;
pub use notifyworker;
pub use outpathdiff;
pub use stats;
pub use tagger;
pub use writetoline;
pub use tasks;
pub use test_scratch;
pub use easyamqp;
pub use worker;
pub use writetoline;
pub const VERSION: &'static str = env!("CARGO_PKG_VERSION");
pub const VERSION: &str = env!("CARGO_PKG_VERSION");
pub fn partition_result<A,B>(results: Vec<Result<A,B>>) -> (Vec<A>, Vec<B>) {
pub fn partition_result<A, B>(results: Vec<Result<A, B>>) -> (Vec<A>, Vec<B>) {
let mut ok = Vec::new();
let mut err = Vec::new();
for result in results.into_iter() {
match result {
Ok(x) => { ok.push(x); }
Err(x) => { err.push(x); }
Ok(x) => {
ok.push(x);
}
Err(x) => {
err.push(x);
}
}
}
@ -92,7 +95,7 @@ pub mod ofborg {
}
pub fn setup_log() {
if let Err(_) = env::var("RUST_LOG") {
if env::var("RUST_LOG").is_err() {
env::set_var("RUST_LOG", "info");
env_logger::init().unwrap();
info!("Defaulting RUST_LOG environment variable to info");

5
ofborg/src/locks.rs
View file

@ -1,8 +1,7 @@
use std::path::PathBuf;
use fs2::FileExt;
use std::fs;
use std::io::Error;
use std::path::PathBuf;
pub trait Lockable {
fn lock_path(&self) -> PathBuf;
@ -10,7 +9,7 @@ pub trait Lockable {
fn lock(&self) -> Result<Lock, Error> {
let lock = fs::File::create(self.lock_path())?;
lock.lock_exclusive()?;
return Ok(Lock { lock: Some(lock) });
Ok(Lock { lock: Some(lock) })
}
}

25
ofborg/src/message/buildjob.rs
View file

@ -1,5 +1,5 @@
use ofborg::message::{Pr, Repo};
use ofborg::commentparser::Subset;
use ofborg::message::{Pr, Repo};
use serde_json;
#[derive(Serialize, Deserialize, Debug)]
@ -27,29 +27,22 @@ impl BuildJob {
statusreport: Option<ExchangeQueue>,
request_id: String,
) -> BuildJob {
let logbackrk = format!(
"{}.{}",
repo.full_name.clone(),
pr.number,
).to_lowercase();
let logbackrk = format!("{}.{}", repo.full_name.clone(), pr.number,).to_lowercase();
BuildJob {
repo: repo,
pr: pr,
repo,
pr,
subset: Some(subset),
attrs: attrs,
attrs,
logs: Some(logs.unwrap_or((Some("logs".to_owned()), Some(logbackrk)))),
statusreport: Some(statusreport.unwrap_or(
(Some("build-results".to_owned()), None),
)),
request_id: request_id,
statusreport: Some(statusreport.unwrap_or((Some("build-results".to_owned()), None))),
request_id,
}
}
}
pub fn from(data: &Vec<u8>) -> Result<BuildJob, serde_json::error::Error> {
return serde_json::from_slice(&data);
pub fn from(data: &[u8]) -> Result<BuildJob, serde_json::error::Error> {
serde_json::from_slice(&data)
}
pub struct Actions {

1
ofborg/src/message/buildlogmsg.rs
View file

@ -1,4 +1,3 @@
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct BuildLogMsg {
pub system: String,

101
ofborg/src/message/buildresult.rs
View file

@ -1,6 +1,5 @@
use ofborg::message::{Pr, Repo};
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
pub enum BuildStatus {
Skipped,
@ -10,7 +9,6 @@ pub enum BuildStatus {
UnexpectedError { err: String },
}
pub struct LegacyBuildResult {
pub repo: Repo,
pub pr: Pr,
@ -25,14 +23,14 @@ pub struct LegacyBuildResult {
#[derive(Serialize, Deserialize, Debug)]
pub enum V1Tag {
V1
V1,
}
#[derive(Serialize, Deserialize, Debug)]
#[serde(untagged)]
pub enum BuildResult {
V1 {
tag: V1Tag, // use serde once all enum variants have a tag
tag: V1Tag, // use serde once all enum variants have a tag
repo: Repo,
pr: Pr,
system: String,
@ -63,52 +61,71 @@ impl BuildResult {
// TODO: replace this with simpler structs for specific usecases, since
// it's decouples the structs from serialization. These can be changed
// as long as we can translate all enum variants.
match self {
&BuildResult::Legacy { ref repo, ref pr, ref system, ref output, ref attempt_id, ref request_id, ref attempted_attrs, ref skipped_attrs, .. } =>
LegacyBuildResult {
repo: repo.to_owned(),
pr: pr.to_owned(),
system: system.to_owned(),
output: output.to_owned(),
attempt_id: attempt_id.to_owned(),
request_id: request_id.to_owned(),
status: self.status(),
attempted_attrs: attempted_attrs.to_owned(),
skipped_attrs: skipped_attrs.to_owned(),
},
&BuildResult::V1 { ref repo, ref pr, ref system, ref output, ref attempt_id, ref request_id, ref attempted_attrs, ref skipped_attrs, .. } =>
LegacyBuildResult {
repo: repo.to_owned(),
pr: pr.to_owned(),
system: system.to_owned(),
output: output.to_owned(),
attempt_id: attempt_id.to_owned(),
request_id: request_id.to_owned(),
status: self.status(),
attempted_attrs: attempted_attrs.to_owned(),
skipped_attrs: skipped_attrs.to_owned(),
},
match *self {
BuildResult::Legacy {
ref repo,
ref pr,
ref system,
ref output,
ref attempt_id,
ref request_id,
ref attempted_attrs,
ref skipped_attrs,
..
} => LegacyBuildResult {
repo: repo.to_owned(),
pr: pr.to_owned(),
system: system.to_owned(),
output: output.to_owned(),
attempt_id: attempt_id.to_owned(),
request_id: request_id.to_owned(),
status: self.status(),
attempted_attrs: attempted_attrs.to_owned(),
skipped_attrs: skipped_attrs.to_owned(),
},
BuildResult::V1 {
ref repo,
ref pr,
ref system,
ref output,
ref attempt_id,
ref request_id,
ref attempted_attrs,
ref skipped_attrs,
..
} => LegacyBuildResult {
repo: repo.to_owned(),
pr: pr.to_owned(),
system: system.to_owned(),
output: output.to_owned(),
attempt_id: attempt_id.to_owned(),
request_id: request_id.to_owned(),
status: self.status(),
attempted_attrs: attempted_attrs.to_owned(),
skipped_attrs: skipped_attrs.to_owned(),
},
}
}
pub fn status(&self) -> BuildStatus {
match self {
&BuildResult::Legacy { ref status, ref success, .. } =>
status.to_owned().unwrap_or_else(|| {
// Fallback for old format.
match success {
&None => BuildStatus::Skipped,
&Some(true) => BuildStatus::Success,
&Some(false) => BuildStatus::Failure,
}
}),
&BuildResult::V1 { ref status, .. } =>
status.to_owned(),
match *self {
BuildResult::Legacy {
ref status,
ref success,
..
} => status.to_owned().unwrap_or_else(|| {
// Fallback for old format.
match *success {
None => BuildStatus::Skipped,
Some(true) => BuildStatus::Success,
Some(false) => BuildStatus::Failure,
}
}),
BuildResult::V1 { ref status, .. } => status.to_owned(),
}
}
}
#[cfg(test)]
mod tests {
use super::*;

1
ofborg/src/message/common.rs
View file

@ -1,4 +1,3 @@
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct Repo {
pub owner: String,

9
ofborg/src/message/massrebuildjob.rs
View file

@ -2,9 +2,8 @@ use ofborg::message::{Pr, Repo};
use ofborg::worker;
use serde_json;
pub fn from(data: &Vec<u8>) -> Result<MassRebuildJob, serde_json::error::Error> {
return serde_json::from_slice(&data);
pub fn from(data: &[u8]) -> Result<MassRebuildJob, serde_json::error::Error> {
serde_json::from_slice(&data)
}
#[derive(Serialize, Deserialize, Debug)]
@ -17,7 +16,7 @@ pub struct Actions {}
impl Actions {
pub fn skip(&mut self, _job: &MassRebuildJob) -> worker::Actions {
return vec![worker::Action::Ack];
vec![worker::Action::Ack]
}
pub fn done(
@ -26,6 +25,6 @@ impl Actions {
mut response: worker::Actions,
) -> worker::Actions {
response.push(worker::Action::Ack);
return response;
response
}
}

6
ofborg/src/message/mod.rs
View file

@ -1,7 +1,7 @@
mod common;
pub mod buildjob;
pub mod buildresult;
pub mod massrebuildjob;
pub mod buildlogmsg;
pub mod buildresult;
mod common;
pub mod massrebuildjob;
pub use self::common::{Pr, Repo};

312
ofborg/src/nix.rs
View file

@ -1,16 +1,15 @@
use ofborg::asynccmd::{AsyncCmd, SpawnedAsyncCmd};
use ofborg::partition_result;
use std::env;
use std::fmt;
use std::fs::File;
use std::io::BufRead;
use std::io::BufReader;
use std::io::Seek;
use std::io::SeekFrom;
use std::path::Path;
use std::process::{Command, Stdio};
use tempfile::tempfile;
use std::io::BufReader;
use std::io::BufRead;
use ofborg::asynccmd::{AsyncCmd, SpawnedAsyncCmd};
use ofborg::partition_result;
#[derive(Clone, Debug)]
pub enum Operation {
@ -29,7 +28,7 @@ impl Operation {
Operation::Build => Command::new("nix-build"),
Operation::QueryPackagesJSON => Command::new("nix-env"),
Operation::QueryPackagesOutputs => Command::new("nix-env"),
Operation::NoOp { operation: _ } => Command::new("echo"),
Operation::NoOp { .. } => Command::new("echo"),
Operation::Unknown { ref program } => Command::new(program),
}
}
@ -38,14 +37,22 @@ impl Operation {
match *self {
Operation::Build => {
command.args(&["--no-out-link", "--keep-going"]);
},
}
Operation::QueryPackagesJSON => {
command.args(&["--query", "--available", "--json"]);
},
}
Operation::QueryPackagesOutputs => {
command.args(&["--query", "--available", "--no-name", "--attr-path", "--out-path"]);
},
Operation::NoOp { ref operation } => { operation.args(command); },
command.args(&[
"--query",
"--available",
"--no-name",
"--attr-path",
"--out-path",
]);
}
Operation::NoOp { ref operation } => {
operation.args(command);
}
_ => (),
};
}
@ -54,10 +61,10 @@ impl Operation {
impl fmt::Display for Operation {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match *self {
Operation::Build => write!(f, "{}", "nix-build"),
Operation::Instantiate => write!(f, "{}", "nix-instantiate"),
Operation::QueryPackagesJSON => write!(f, "{}", "nix-env -qa --json"),
Operation::QueryPackagesOutputs => write!(f, "{}", "nix-env -qaP --no-name --out-path"),
Operation::Build => write!(f, "nix-build"),
Operation::Instantiate => write!(f, "nix-instantiate"),
Operation::QueryPackagesJSON => write!(f, "nix-env -qa --json"),
Operation::QueryPackagesOutputs => write!(f, "nix-env -qaP --no-name --out-path"),
Operation::NoOp { ref operation } => operation.fmt(f),
Operation::Unknown { ref program } => write!(f, "{}", program),
}
@ -74,32 +81,37 @@ pub struct Nix {
}
impl Nix {
pub fn new(system: String, remote: String, build_timeout: u16, initial_heap_size: Option<String>) -> Nix {
return Nix {
system: system,
remote: remote,
build_timeout: build_timeout,
initial_heap_size: initial_heap_size,
pub fn new(
system: String,
remote: String,
build_timeout: u16,
initial_heap_size: Option<String>,
) -> Nix {
Nix {
system,
remote,
build_timeout,
initial_heap_size,
limit_supported_systems: true,
};
}
}
pub fn with_system(&self, system: String) -> Nix {
let mut n = self.clone();
n.system = system;
return n;
n
}
pub fn with_limited_supported_systems(&self) -> Nix {
let mut n = self.clone();
n.limit_supported_systems = true;
return n;
n
}
pub fn without_limited_supported_systems(&self) -> Nix {
let mut n = self.clone();
n.limit_supported_systems = false;
return n;
n
}
pub fn safely_partition_instantiable_attrs(
@ -107,19 +119,14 @@ impl Nix {
nixpkgs: &Path,
file: &str,
attrs: Vec<String>,
) -> (Vec<String>, Vec<(String,Vec<String>)>) {
let attr_instantiations: Vec<Result<String, (String, Vec<String>)>> =
attrs
) -> (Vec<String>, Vec<(String, Vec<String>)>) {
let attr_instantiations: Vec<Result<String, (String, Vec<String>)>> = attrs
.into_iter()
.map(|attr|
match self.safely_instantiate_attrs(
nixpkgs,
file,
vec![attr.clone()]
) {
Ok(_) => Ok(attr.clone()),
Err(f) => Err((attr.clone(), lines_from_file(f)))
}
.map(
|attr| match self.safely_instantiate_attrs(nixpkgs, file, vec![attr.clone()]) {
Ok(_) => Ok(attr.clone()),
Err(f) => Err((attr.clone(), lines_from_file(f))),
},
)
.collect();
@ -134,7 +141,7 @@ impl Nix {
) -> Result<File, File> {
let cmd = self.safely_instantiate_attrs_cmd(nixpkgs, file, attrs);
return self.run(cmd, true);
self.run(cmd, true)
}
pub fn safely_instantiate_attrs_cmd(
@ -150,7 +157,7 @@ impl Nix {
attrargs.push(attr);
}
return self.safe_command(Operation::Instantiate, nixpkgs, attrargs);
self.safe_command(&Operation::Instantiate, nixpkgs, attrargs)
}
pub fn safely_build_attrs(
@ -161,7 +168,7 @@ impl Nix {
) -> Result<File, File> {
let cmd = self.safely_build_attrs_cmd(nixpkgs, file, attrs);
return self.run(cmd, true);
self.run(cmd, true)
}
pub fn safely_build_attrs_async(
@ -170,16 +177,10 @@ impl Nix {
file: &str,
attrs: Vec<String>,
) -> SpawnedAsyncCmd {
AsyncCmd::new(self.safely_build_attrs_cmd(nixpkgs, file, attrs))
.spawn()
AsyncCmd::new(self.safely_build_attrs_cmd(nixpkgs, file, attrs)).spawn()
}
fn safely_build_attrs_cmd(
&self,
nixpkgs: &Path,
file: &str,
attrs: Vec<String>,
) -> Command {
fn safely_build_attrs_cmd(&self, nixpkgs: &Path, file: &str, attrs: Vec<String>) -> Command {
let mut attrargs: Vec<String> = Vec::with_capacity(3 + (attrs.len() * 2));
attrargs.push(file.to_owned());
for attr in attrs {
@ -187,49 +188,47 @@ impl Nix {
attrargs.push(attr);
}
self.safe_command(Operation::Build, nixpkgs, attrargs)
self.safe_command(&Operation::Build, nixpkgs, attrargs)
}
pub fn safely(
&self,
op: Operation,
op: &Operation,
nixpkgs: &Path,
args: Vec<String>,
keep_stdout: bool,
) -> Result<File, File> {
return self.run(self.safe_command(op, nixpkgs, args), keep_stdout);
self.run(self.safe_command(&op, nixpkgs, args), keep_stdout)
}
pub fn run(&self, mut cmd: Command, keep_stdout: bool) -> Result<File, File> {
let stderr = tempfile().expect("Fetching a stderr tempfile");
let mut reader = stderr.try_clone().expect("Cloning stderr to the reader");
let stdout: Stdio;
if keep_stdout {
let stdout_fd = stderr.try_clone().expect("Cloning stderr for stdout");
stdout = Stdio::from(stdout_fd);
let stdout: Stdio = if keep_stdout {
Stdio::from(stderr.try_clone().expect("Cloning stderr for stdout"))
} else {
stdout = Stdio::null();
}
Stdio::null()
};
let status = cmd.stdout(Stdio::from(stdout))
let status = cmd
.stdout(stdout)
.stderr(Stdio::from(stderr))
.status()
.expect(format!("Running a program ...").as_ref());
.expect("Running a program ...");
reader.seek(SeekFrom::Start(0)).expect(
"Seeking to Start(0)",
);
reader
.seek(SeekFrom::Start(0))
.expect("Seeking to Start(0)");
if status.success() {
return Ok(reader);
Ok(reader)
} else {
return Err(reader);
Err(reader)
}
}
pub fn safe_command(&self, op: Operation, nixpkgs: &Path, args: Vec<String>) -> Command {
pub fn safe_command(&self, op: &Operation, nixpkgs: &Path, args: Vec<String>) -> Command {
let nixpath = format!("nixpkgs={}", nixpkgs.display());
let mut command = op.command();
@ -250,35 +249,30 @@ impl Nix {
command.args(&["--show-trace"]);
command.args(&["--option", "restrict-eval", "true"]);
command.args(
&[
"--option",
"build-timeout",
&format!("{}", self.build_timeout),
],
);
command.args(&[
"--option",
"build-timeout",
&format!("{}", self.build_timeout),
]);
command.args(&["--argstr", "system", &self.system]);
if self.limit_supported_systems {
command.args(
&[
"--arg",
"supportedSystems",
&format!("[\"{}\"]", &self.system),
],
);
command.args(&[
"--arg",
"supportedSystems",
&format!("[\"{}\"]", &self.system),
]);
}
command.args(args);
return command;
command
}
}
fn lines_from_file(file: File) -> Vec<String> {
BufReader::new(file)
.lines()
.into_iter()
.filter(|line| line.is_ok())
.map(|line| line.unwrap())
.collect()
@ -292,29 +286,33 @@ mod tests {
}
fn noop(operation: Operation) -> Operation {
Operation::NoOp { operation: Box::new(operation) }
Operation::NoOp {
operation: Box::new(operation),
}
}
fn env_noop() -> Operation {
Operation::Unknown { program: "./environment.sh".to_owned() }
Operation::Unknown {
program: "./environment.sh".to_owned(),
}
}
fn build_path() -> PathBuf {
let mut cwd = env::current_dir().unwrap();
cwd.push(Path::new("./test-srcs/build"));
return cwd;
cwd
}
fn passing_eval_path() -> PathBuf {
let mut cwd = env::current_dir().unwrap();
cwd.push(Path::new("./test-srcs/eval"));
return cwd;
cwd
}
fn individual_eval_path() -> PathBuf {
let mut cwd = env::current_dir().unwrap();
cwd.push(Path::new("./test-srcs/eval-mixed-failure"));
return cwd;
cwd
}
fn strip_ansi(string: &str) -> String {
@ -356,11 +354,13 @@ mod tests {
let requirements_held: Vec<Result<String, String>> = require
.into_iter()
.map(|line| line.to_owned())
.map(|line| if buildlog.contains(&line) {
Ok(line)
} else {
missed_requirements += 1;
Err(line)
.map(|line| {
if buildlog.contains(&line) {
Ok(line)
} else {
missed_requirements += 1;
Err(line)
}
})
.collect();
@ -409,8 +409,8 @@ mod tests {
}
use super::*;
use std::path::PathBuf;
use std::env;
use std::path::PathBuf;
#[test]
fn test_build_operations() {
@ -418,11 +418,10 @@ mod tests {
let op = noop(Operation::Build);
assert_eq!(op.to_string(), "nix-build");
let ret: Result<File, File> =
nix.run(
nix.safe_command(op, build_path().as_path(), vec![String::from("--version")]),
true,
);
let ret: Result<File, File> = nix.run(
nix.safe_command(&op, build_path().as_path(), vec![String::from("--version")]),
true,
);
assert_run(
ret,
@ -437,17 +436,12 @@ mod tests {
let op = noop(Operation::Instantiate);
assert_eq!(op.to_string(), "nix-instantiate");
let ret: Result<File, File> =
nix.run(
nix.safe_command(op, build_path().as_path(), vec![String::from("--version")]),
true,
);
assert_run(
ret,
Expect::Pass,
vec!["--version"],
let ret: Result<File, File> = nix.run(
nix.safe_command(&op, build_path().as_path(), vec![String::from("--version")]),
true,
);
assert_run(ret, Expect::Pass, vec!["--version"]);
}
#[test]
@ -456,11 +450,10 @@ mod tests {
let op = noop(Operation::QueryPackagesJSON);
assert_eq!(op.to_string(), "nix-env -qa --json");
let ret: Result<File, File> =
nix.run(
nix.safe_command(op, build_path().as_path(), vec![String::from("--version")]),
true,
);
let ret: Result<File, File> = nix.run(
nix.safe_command(&op, build_path().as_path(), vec![String::from("--version")]),
true,
);
assert_run(
ret,
@ -475,18 +468,17 @@ mod tests {
let op = noop(Operation::QueryPackagesOutputs);
assert_eq!(op.to_string(), "nix-env -qaP --no-name --out-path");
let ret: Result<File, File> =
nix.run(
nix.safe_command(op, build_path().as_path(), vec![String::from("--version")]),
true,
);
let ret: Result<File, File> = nix.run(
nix.safe_command(&op, build_path().as_path(), vec![String::from("--version")]),
true,
);
assert_run(
ret,
Expect::Pass,
vec![
"--query --available --no-name --attr-path --out-path",
"--version"
"--version",
],
);
}
@ -495,11 +487,10 @@ mod tests {
fn safe_command_environment() {
let nix = nix();
let ret: Result<File, File> =
nix.run(
nix.safe_command(env_noop(), build_path().as_path(), vec![]),
true,
);
let ret: Result<File, File> = nix.run(
nix.safe_command(&env_noop(), build_path().as_path(), vec![]),
true,
);
assert_run(
ret,
@ -516,13 +507,17 @@ mod tests {
#[test]
fn safe_command_custom_gc() {
let remote = env::var("NIX_REMOTE").unwrap_or("".to_owned());
let nix = Nix::new("x86_64-linux".to_owned(), remote, 1800, Some("4g".to_owned()));
let nix = Nix::new(
"x86_64-linux".to_owned(),
remote,
1800,
Some("4g".to_owned()),
);
let ret: Result<File, File> =
nix.run(
nix.safe_command(env_noop(), build_path().as_path(), vec![]),
true,
);
let ret: Result<File, File> = nix.run(
nix.safe_command(&env_noop(), build_path().as_path(), vec![]),
true,
);
assert_run(
ret,
@ -542,10 +537,8 @@ mod tests {
let nix = nix();
let op = noop(Operation::Build);
let ret: Result<File, File> = nix.run(
nix.safe_command(op, build_path().as_path(), vec![]),
true,
);
let ret: Result<File, File> =
nix.run(nix.safe_command(&op, build_path().as_path(), vec![]), true);
assert_run(
ret,
@ -597,23 +590,30 @@ mod tests {
fn partition_instantiable_attributes() {
let nix = nix();
let ret: (Vec<String>, Vec<(String, Vec<String>)>) = nix.safely_partition_instantiable_attrs(
individual_eval_path().as_path(),
"default.nix",
vec![
String::from("fails-instantiation"),
String::from("passes-instantiation"),
String::from("missing-attr"),
],
);
let ret: (Vec<String>, Vec<(String, Vec<String>)>) = nix
.safely_partition_instantiable_attrs(
individual_eval_path().as_path(),
"default.nix",
vec![
String::from("fails-instantiation"),
String::from("passes-instantiation"),
String::from("missing-attr"),
],
);
assert_eq!(ret.0, vec!["passes-instantiation"]);
assert_eq!(ret.1[0].0, "fails-instantiation");
assert_eq!(ret.1[0].1[0], "trace: You just can't frooble the frozz on this particular system.");
assert_eq!(
ret.1[0].1[0],
"trace: You just can't frooble the frozz on this particular system."
);
assert_eq!(ret.1[1].0, "missing-attr");
assert_eq!(strip_ansi(&ret.1[1].1[0]), "error: attribute 'missing-attr' in selection path 'missing-attr' not found");
assert_eq!(
strip_ansi(&ret.1[1].1[0]),
"error: attribute 'missing-attr' in selection path 'missing-attr' not found"
);
}
#[test]
@ -629,10 +629,7 @@ mod tests {
assert_run(
ret,
Expect::Fail,
vec![
"You just can't",
"assertion failed",
],
vec!["You just can't", "assertion failed"],
);
}
@ -646,13 +643,7 @@ mod tests {
vec![String::from("passes-instantiation")],
);
assert_run(
ret,
Expect::Pass,
vec![
"-passes-instantiation.drv"
],
);
assert_run(ret, Expect::Pass, vec!["-passes-instantiation.drv"]);
}
#[test]
@ -677,7 +668,7 @@ mod tests {
#[test]
fn instantiation_success() {
let ret: Result<File, File> = nix().safely(
Operation::Instantiate,
&Operation::Instantiate,
passing_eval_path().as_path(),
vec![],
true,
@ -697,7 +688,7 @@ mod tests {
#[test]
fn instantiation_nixpkgs_restricted_mode() {
let ret: Result<File, File> = nix().safely(
Operation::Instantiate,
&Operation::Instantiate,
individual_eval_path().as_path(),
vec![String::from("-A"), String::from("nixpkgs-restricted-mode")],
true,
@ -706,10 +697,7 @@ mod tests {
assert_run(
ret,
Expect::Fail,
vec![
"access to path '/fake'",
"is forbidden in restricted mode",
],
vec!["access to path '/fake'", "is forbidden in restricted mode"],
);
}
}

31
ofborg/src/notifyworker.rs
View file

@ -1,6 +1,6 @@
use amqp::protocol::basic::{BasicProperties, Deliver};
use amqp::Basic;
use amqp::{Consumer, Channel};
use amqp::protocol::basic::{Deliver, BasicProperties};
use amqp::{Channel, Consumer};
use std::marker::Send;
use worker::Action;
@ -17,7 +17,7 @@ pub trait SimpleNotifyWorker {
&self,
method: &Deliver,
headers: &BasicProperties,
body: &Vec<u8>,
body: &[u8],
) -> Result<Self::J, String>;
}
@ -25,13 +25,14 @@ pub trait NotificationReceiver {
fn tell(&mut self, action: Action);
}
#[derive(Default)]
pub struct DummyNotificationReceiver {
pub actions: Vec<Action>,
}
impl DummyNotificationReceiver {
pub fn new() -> DummyNotificationReceiver {
DummyNotificationReceiver { actions: vec![] }
Default::default()
}
}
@ -48,10 +49,10 @@ pub struct ChannelNotificationReceiver<'a> {
impl<'a> ChannelNotificationReceiver<'a> {
pub fn new(channel: &'a mut Channel, delivery_tag: u64) -> ChannelNotificationReceiver<'a> {
return ChannelNotificationReceiver {
channel: channel,
delivery_tag: delivery_tag,
};
ChannelNotificationReceiver {
channel,
delivery_tag,
}
}
}
@ -71,13 +72,13 @@ impl<'a> NotificationReceiver for ChannelNotificationReceiver<'a> {
.basic_nack(self.delivery_tag, false, false)
.unwrap();
}
Action::Publish(msg) => {
let exch = msg.exchange.clone().unwrap_or("".to_owned());
let key = msg.routing_key.clone().unwrap_or("".to_owned());
Action::Publish(mut msg) => {
let exch = msg.exchange.take().unwrap_or_else(|| "".to_owned());
let key = msg.routing_key.take().unwrap_or_else(|| "".to_owned());
let props = msg.properties.unwrap_or(
BasicProperties { ..Default::default() },
);
let props = msg.properties.take().unwrap_or(BasicProperties {
..Default::default()
});
self.channel
.basic_publish(exch, key, msg.mandatory, msg.immediate, props, msg.content)
.unwrap();
@ -87,7 +88,7 @@ impl<'a> NotificationReceiver for ChannelNotificationReceiver<'a> {
}
pub fn new<T: SimpleNotifyWorker>(worker: T) -> NotifyWorker<T> {
return NotifyWorker { internal: worker };
NotifyWorker { internal: worker }
}
impl<T: SimpleNotifyWorker + Send> Consumer for NotifyWorker<T> {

58
ofborg/src/outpathdiff.rs
View file

@ -1,14 +1,14 @@
extern crate amqp;
extern crate env_logger;
use std::collections::{HashSet, HashMap};
use std::fs::File;
use ofborg::nix;
use std::collections::{HashMap, HashSet};
use std::fs;
use std::fs::File;
use std::io::BufRead;
use std::io::BufReader;
use std::path::PathBuf;
use ofborg::nix;
use std::io::Write;
use std::path::PathBuf;
pub struct OutPathDiff {
calculator: OutPaths,
@ -30,11 +30,11 @@ impl OutPathDiff {
match x {
Ok(f) => {
self.original = Some(f);
return Ok(true);
Ok(true)
}
Err(e) => {
info!("Failed to find Before list");
return Err(e);
Err(e)
}
}
}
@ -49,11 +49,11 @@ impl OutPathDiff {
match x {
Ok(f) => {
self.current = Some(f);
return Ok(true);
Ok(true)
}
Err(e) => {
info!("Failed to find After list");
return Err(e);
Err(e)
}
}
}
@ -64,14 +64,20 @@ impl OutPathDiff {
let orig_set: HashSet<&PackageArch> = orig.keys().collect();
let cur_set: HashSet<&PackageArch> = cur.keys().collect();
let removed: Vec<PackageArch> = orig_set.difference(&cur_set).map(|ref p| (**p).clone()).collect();
let added: Vec<PackageArch> = cur_set.difference(&orig_set).map(|ref p| (**p).clone()).collect();
return Some((removed, added));
let removed: Vec<PackageArch> = orig_set
.difference(&cur_set)
.map(|ref p| (**p).clone())
.collect();
let added: Vec<PackageArch> = cur_set
.difference(&orig_set)
.map(|ref p| (**p).clone())
.collect();
Some((removed, added))
} else {
return None;
None
}
} else {
return None;
None
}
}
@ -94,7 +100,7 @@ impl OutPathDiff {
}
}
return None;
None
}
fn run(&mut self) -> Result<PackageOutPaths, File> {
@ -122,9 +128,9 @@ pub struct OutPaths {
impl OutPaths {
pub fn new(nix: nix::Nix, path: PathBuf, check_meta: bool) -> OutPaths {
OutPaths {
nix: nix,
path: path,
check_meta: check_meta,
nix,
path,
check_meta,
}
}
@ -145,7 +151,7 @@ impl OutPaths {
fn place_nix(&self) {
let mut file = File::create(self.nix_path()).expect("Failed to create nix out path check");
file.write_all(include_str!("outpaths.nix").as_bytes())
file.write_all(include_bytes!("outpaths.nix"))
.expect("Failed to place outpaths.nix");
}
@ -161,16 +167,14 @@ impl OutPaths {
}
fn execute(&self) -> Result<File, File> {
let check_meta: String;
if self.check_meta {
check_meta = String::from("true");
let check_meta: String = if self.check_meta {
String::from("true")
} else {
check_meta = String::from("false");
}
String::from("false")
};
self.nix.safely(
nix::Operation::QueryPackagesOutputs,
&nix::Operation::QueryPackagesOutputs,
&self.path,
vec![
String::from("-f"),
@ -184,7 +188,6 @@ impl OutPaths {
}
}
fn parse_lines(data: &mut BufRead) -> PackageOutPaths {
data.lines()
.filter_map(|line| match line {
@ -196,7 +199,7 @@ fn parse_lines(data: &mut BufRead) -> PackageOutPaths {
if split.len() == 2 {
let outpaths = String::from(split[1]);
let path: Vec<&str> = split[0].rsplitn(2, ".").collect();
let path: Vec<&str> = split[0].rsplitn(2, '.').collect();
if path.len() == 2 {
Some((
PackageArch {
@ -213,7 +216,6 @@ fn parse_lines(data: &mut BufRead) -> PackageOutPaths {
info!("Warning: not 2 word segments in {:?}", split);
None
}
})
.collect()
}

18
ofborg/src/stats.rs
View file

@ -1,7 +1,7 @@
use serde_json;
use amqp::Channel;
use amqp::protocol::basic::BasicProperties;
use amqp::Basic;
use amqp::Channel;
use serde_json;
include!(concat!(env!("OUT_DIR"), "/events.rs"));
@ -15,7 +15,6 @@ pub trait SysEvents: Send {
fn notify(&mut self, event: Event);
}
#[derive(Serialize, Deserialize, Debug)]
pub struct EventMessage {
pub sender: String,
@ -29,13 +28,18 @@ pub struct RabbitMQ {
impl RabbitMQ {
pub fn new(identity: &str, channel: Channel) -> RabbitMQ {
RabbitMQ { identity: identity.to_owned(), channel: channel }
RabbitMQ {
identity: identity.to_owned(),
channel,
}
}
}
impl SysEvents for RabbitMQ {
fn notify(&mut self, event: Event) {
let props = BasicProperties { ..Default::default() };
let props = BasicProperties {
..Default::default()
};
self.channel
.basic_publish(
String::from("stats"),
@ -46,7 +50,9 @@ impl SysEvents for RabbitMQ {
serde_json::to_string(&EventMessage {
sender: self.identity.clone(),
events: vec![event],
}).unwrap().into_bytes(),
})
.unwrap()
.into_bytes(),
)
.unwrap();
}

85
ofborg/src/tagger.rs
View file

@ -1,5 +1,5 @@
use ofborg::tasks;
use ofborg::outpathdiff::PackageArch;
use ofborg::tasks;
use std::collections::HashMap;
pub struct StdenvTagger {
@ -7,8 +7,8 @@ pub struct StdenvTagger {
selected: Vec<String>,
}
impl StdenvTagger {
pub fn new() -> StdenvTagger {
impl Default for StdenvTagger {
fn default() -> StdenvTagger {
let mut t = StdenvTagger {
possible: vec![
String::from("10.rebuild-linux-stdenv"),
@ -18,7 +18,13 @@ impl StdenvTagger {
};
t.possible.sort();
return t;
t
}
}
impl StdenvTagger {
pub fn new() -> StdenvTagger {
Default::default()
}
pub fn changed(&mut self, systems: Vec<tasks::eval::stdenvs::System>) {
@ -54,7 +60,7 @@ impl StdenvTagger {
remove.remove(pos);
}
return remove;
remove
}
}
@ -63,8 +69,8 @@ pub struct PkgsAddedRemovedTagger {
selected: Vec<String>,
}
impl PkgsAddedRemovedTagger {
pub fn new() -> PkgsAddedRemovedTagger {
impl Default for PkgsAddedRemovedTagger {
fn default() -> PkgsAddedRemovedTagger {
let mut t = PkgsAddedRemovedTagger {
possible: vec![
String::from("8.has: package (new)"),
@ -74,15 +80,21 @@ impl PkgsAddedRemovedTagger {
};
t.possible.sort();
return t;
t
}
}
impl PkgsAddedRemovedTagger {
pub fn new() -> PkgsAddedRemovedTagger {
Default::default()
}
pub fn changed(&mut self, removed: Vec<PackageArch>, added: Vec<PackageArch>) {
if removed.len() > 0 {
pub fn changed(&mut self, removed: &[PackageArch], added: &[PackageArch]) {
if !removed.is_empty() {
self.selected.push(String::from("8.has: clean-up"));
}
if added.len() > 0 {
if !added.is_empty() {
self.selected.push(String::from("8.has: package (new)"));
}
}
@ -93,7 +105,7 @@ impl PkgsAddedRemovedTagger {
pub fn tags_to_remove(&self) -> Vec<String> {
// The cleanup tag is too vague to automatically remove.
return vec![];
vec![]
}
}
@ -102,8 +114,8 @@ pub struct RebuildTagger {
selected: Vec<String>,
}
impl RebuildTagger {
pub fn new() -> RebuildTagger {
impl Default for RebuildTagger {
fn default() -> RebuildTagger {
let mut t = RebuildTagger {
possible: vec![
String::from("10.rebuild-linux: 501+"),
@ -111,7 +123,6 @@ impl RebuildTagger {
String::from("10.rebuild-linux: 11-100"),
String::from("10.rebuild-linux: 1-10"),
String::from("10.rebuild-linux: 0"),
String::from("10.rebuild-darwin: 501+"),
String::from("10.rebuild-darwin: 101-500"),
String::from("10.rebuild-darwin: 11-100"),
@ -122,7 +133,13 @@ impl RebuildTagger {
};
t.possible.sort();
return t;
t
}
}
impl RebuildTagger {
pub fn new() -> RebuildTagger {
Default::default()
}
pub fn parse_attrs(&mut self, attrs: Vec<PackageArch>) {
@ -145,14 +162,10 @@ impl RebuildTagger {
}
}
self.selected =
vec![
String::from(format!("10.rebuild-linux: {}", self.bucket(counter_linux))),
String::from(format!(
"10.rebuild-darwin: {}",
self.bucket(counter_darwin)
)),
];
self.selected = vec![
format!("10.rebuild-linux: {}", self.bucket(counter_linux)),
format!("10.rebuild-darwin: {}", self.bucket(counter_darwin)),
];
for tag in &self.selected {
if !self.possible.contains(&tag) {
@ -175,22 +188,21 @@ impl RebuildTagger {
remove.remove(pos);
}
return remove;
remove
}
fn bucket(&self, count: u64) -> &str {
if count > 500 {
return "501+";
"501+"
} else if count > 100 {
return "101-500";
"101-500"
} else if count > 10 {
return "11-100";
"11-100"
} else if count > 0 {
return "1-10";
"1-10"
} else {
return "0";
"0"
}
}
}
@ -208,12 +220,11 @@ impl PathsTagger {
}
pub fn path_changed(&mut self, path: &str) {
let mut tags_to_add: Vec<String> = self.possible
let mut tags_to_add: Vec<String> = self
.possible
.iter()
.filter(|&(ref tag, ref _paths)| !self.selected.contains(&tag))
.filter(|&(ref _tag, ref paths)| {
paths.iter().any(|tp| path.contains(tp))
})
.filter(|&(ref _tag, ref paths)| paths.iter().any(|tp| path.contains(tp)))
.map(|(tag, _paths)| tag.clone())
.collect();
self.selected.append(&mut tags_to_add);
@ -232,7 +243,7 @@ impl PathsTagger {
remove.remove(pos);
}
return remove;
remove
}
}
@ -267,7 +278,6 @@ mod tests {
vec!["topic: python".to_owned(), "topic: ruby".to_owned()]
);
tagger.path_changed("pkgs/development/interpreters/ruby/default.nix");
assert_eq!(tagger.tags_to_add(), vec!["topic: ruby".to_owned()]);
assert_eq!(tagger.tags_to_remove(), vec!["topic: python".to_owned()]);
@ -276,7 +286,6 @@ mod tests {
assert_eq!(tagger.tags_to_add(), vec!["topic: ruby".to_owned()]);
assert_eq!(tagger.tags_to_remove(), vec!["topic: python".to_owned()]);
tagger.path_changed("pkgs/top-level/python-packages.nix");
assert_eq!(
tagger.tags_to_add(),

169
ofborg/src/tasks/build.rs
View file

@ -1,21 +1,20 @@
extern crate amqp;
extern crate uuid;
extern crate env_logger;
extern crate uuid;
use uuid::Uuid;
use std::collections::VecDeque;
use ofborg::checkout;
use ofborg::message::buildjob;
use ofborg::message::buildresult::{BuildResult, BuildStatus, V1Tag};
use ofborg::message::buildlogmsg;
use ofborg::nix;
use ofborg::commentparser;
use ofborg::message::buildjob;
use ofborg::message::buildlogmsg;
use ofborg::message::buildresult::{BuildResult, BuildStatus, V1Tag};
use ofborg::nix;
use std::collections::VecDeque;
use ofborg::worker;
use amqp::protocol::basic::{BasicProperties, Deliver};
use ofborg::notifyworker;
use amqp::protocol::basic::{Deliver, BasicProperties};
use ofborg::worker;
pub struct BuildWorker {
cloner: checkout::CachedCloner,
@ -31,12 +30,12 @@ impl BuildWorker {
system: String,
identity: String,
) -> BuildWorker {
return BuildWorker {
cloner: cloner,
nix: nix,
system: system,
identity: identity,
};
BuildWorker {
cloner,
nix,
system,
identity,
}
}
fn actions<'a, 'b>(
@ -69,30 +68,29 @@ impl<'a, 'b> JobActions<'a, 'b> {
job: &'b buildjob::BuildJob,
receiver: &'a mut notifyworker::NotificationReceiver,
) -> JobActions<'a, 'b> {
let (log_exchange, log_routing_key) = job.logs.clone().unwrap_or((
Some(String::from("logs")),
Some(String::from("build.log")),
));
let (log_exchange, log_routing_key) = job
.logs
.clone()
.unwrap_or((Some(String::from("logs")), Some(String::from("build.log"))));
let (result_exchange, result_routing_key) =
job.statusreport.clone().unwrap_or((
Some(String::from("build-results")),
None,
));
let (result_exchange, result_routing_key) = job
.statusreport
.clone()
.unwrap_or((Some(String::from("build-results")), None));
return JobActions {
JobActions {
system: system.to_owned(),
identity: identity.to_owned(),
receiver: receiver,
job: job,
receiver,
job,
line_counter: 0,
snippet_log: VecDeque::with_capacity(10),
attempt_id: format!("{}", Uuid::new_v4()),
log_exchange: log_exchange,
log_routing_key: log_routing_key,
result_exchange: result_exchange,
result_routing_key: result_routing_key,
};
log_exchange,
log_routing_key,
result_exchange,
result_routing_key,
}
}
pub fn log_snippet(&self) -> Vec<String> {
@ -128,7 +126,6 @@ impl<'a, 'b> JobActions<'a, 'b> {
let result_exchange = self.result_exchange.clone();
let result_routing_key = self.result_routing_key.clone();
self.tell(worker::publish_serde_action(
result_exchange,
result_routing_key,
@ -193,9 +190,7 @@ impl<'a, 'b> JobActions<'a, 'b> {
));
}
pub fn build_not_attempted(&mut self, not_attempted_attrs: Vec<String>,
) {
pub fn build_not_attempted(&mut self, not_attempted_attrs: Vec<String>) {
let msg = BuildResult::V1 {
tag: V1Tag::V1,
repo: self.job.repo.clone(),
@ -228,10 +223,11 @@ impl<'a, 'b> JobActions<'a, 'b> {
self.tell(worker::Action::Ack);
}
pub fn build_finished(&mut self, status: BuildStatus,
attempted_attrs: Vec<String>,
not_attempted_attrs: Vec<String>,
pub fn build_finished(
&mut self,
status: BuildStatus,
attempted_attrs: Vec<String>,
not_attempted_attrs: Vec<String>,
) {
let msg = BuildResult::V1 {
tag: V1Tag::V1,
@ -241,7 +237,7 @@ impl<'a, 'b> JobActions<'a, 'b> {
output: self.log_snippet(),
attempt_id: self.attempt_id.clone(),
request_id: self.job.request_id.clone(),
status: status,
status,
attempted_attrs: Some(attempted_attrs),
skipped_attrs: Some(not_attempted_attrs),
};
@ -273,20 +269,15 @@ impl<'a, 'b> JobActions<'a, 'b> {
impl notifyworker::SimpleNotifyWorker for BuildWorker {
type J = buildjob::BuildJob;
fn msg_to_job(
&self,
_: &Deliver,
_: &BasicProperties,
body: &Vec<u8>,
) -> Result<Self::J, String> {
fn msg_to_job(&self, _: &Deliver, _: &BasicProperties, body: &[u8]) -> Result<Self::J, String> {
println!("lmao I got a job?");
return match buildjob::from(body) {
match buildjob::from(body) {
Ok(e) => Ok(e),
Err(e) => {
println!("{:?}", String::from_utf8(body.clone()));
println!("{:?}", String::from_utf8(body.to_vec()));
panic!("{:?}", e);
}
};
}
}
fn consumer(
@ -296,16 +287,15 @@ impl notifyworker::SimpleNotifyWorker for BuildWorker {
) {
let mut actions = self.actions(&job, notifier);
if job.attrs.len() == 0 {
if job.attrs.is_empty() {
actions.nothing_to_do();
return;
}
info!("Working on {}", job.pr.number);
let project = self.cloner.project(
job.repo.full_name.clone(),
job.repo.clone_url.clone(),
);
let project = self
.cloner
.project(&job.repo.full_name, job.repo.clone_url.clone());
let co = project
.clone_for("builder".to_string(), self.identity.clone())
.unwrap();
@ -334,13 +324,16 @@ impl notifyworker::SimpleNotifyWorker for BuildWorker {
return;
}
if let Err(_) = co.merge_commit(job.pr.head_sha.as_ref()) {
if co.merge_commit(job.pr.head_sha.as_ref()).is_err() {
info!("Failed to merge {}", job.pr.head_sha);
actions.merge_failed();
return;
}
println!("Got path: {:?}, determining which ones we can build ", refpath);
println!(
"Got path: {:?}, determining which ones we can build ",
refpath
);
let (can_build, cannot_build) = self.nix.safely_partition_instantiable_attrs(
refpath.as_ref(),
buildfile,
@ -350,27 +343,26 @@ impl notifyworker::SimpleNotifyWorker for BuildWorker {
let cannot_build_attrs: Vec<String> = cannot_build
.clone()
.into_iter()
.map(|(attr,_)| attr)
.map(|(attr, _)| attr)
.collect();
println!("Can build: '{}', Cannot build: '{}'",
can_build.join(", "),
cannot_build_attrs.join(", "));
println!(
"Can build: '{}', Cannot build: '{}'",
can_build.join(", "),
cannot_build_attrs.join(", ")
);
actions.log_started(can_build.clone(), cannot_build_attrs.clone());
actions.log_instantiation_errors(cannot_build);
if can_build.len() == 0 {
if can_build.is_empty() {
actions.build_not_attempted(cannot_build_attrs);
return;
}
let mut spawned = self.nix.safely_build_attrs_async(
refpath.as_ref(),
buildfile,
can_build.clone(),
);
let mut spawned =
self.nix
.safely_build_attrs_async(refpath.as_ref(), buildfile, can_build.clone());
for line in spawned.lines() {
actions.log_line(&line);
@ -391,12 +383,16 @@ impl notifyworker::SimpleNotifyWorker for BuildWorker {
},
e => BuildStatus::UnexpectedError {
err: format!("failed on interior command {:?}", e),
}
},
};
println!("ok built ({:?}), building", status);
println!("Lines:\n-----8<-----");
actions.log_snippet().iter().inspect(|x| println!("{}", x)).last();
actions
.log_snippet()
.iter()
.inspect(|x| println!("{}", x))
.last();
println!("----->8-----");
actions.build_finished(status, can_build, cannot_build_attrs);
@ -407,13 +403,13 @@ impl notifyworker::SimpleNotifyWorker for BuildWorker {
#[cfg(test)]
mod tests {
use super::*;
use notifyworker::SimpleNotifyWorker;
use ofborg::message::{Pr, Repo};
use ofborg::test_scratch::TestScratch;
use std::env;
use std::path::{Path, PathBuf};
use ofborg::message::{Pr, Repo};
use notifyworker::SimpleNotifyWorker;
use std::process::{Command, Stdio};
use std::vec::IntoIter;
use ofborg::test_scratch::TestScratch;
fn nix() -> nix::Nix {
let remote = env::var("NIX_REMOTE").unwrap_or("".to_owned());
@ -421,7 +417,7 @@ mod tests {
}
fn tpath(component: &str) -> PathBuf {
return Path::new(env!("CARGO_MANIFEST_DIR")).join(component);
Path::new(env!("CARGO_MANIFEST_DIR")).join(component)
}
fn make_worker(path: &Path) -> BuildWorker {
@ -434,7 +430,7 @@ mod tests {
"cargo-test-build".to_owned(),
);
return worker;
worker
}
fn make_pr_repo(bare: &Path, co: &Path) -> String {
@ -447,7 +443,8 @@ mod tests {
.output()
.expect("building the test PR failed");
let hash = String::from_utf8(output.stdout).expect("Should just be a hash");
return hash.trim().to_owned();
hash.trim().to_owned()
}
fn strip_escaped_ansi(string: &str) -> String {
@ -496,7 +493,7 @@ mod tests {
let job = buildjob::BuildJob {
attrs: vec!["success".to_owned()],
pr: Pr {
head_sha: head_sha,
head_sha,
number: 1,
target_branch: Some("master".to_owned()),
},
@ -507,10 +504,7 @@ mod tests {
owner: "ofborg-test".to_owned(),
},
subset: None,
logs: Some((
Some(String::from("logs")),
Some(String::from("build.log")),
)),
logs: Some((Some(String::from("logs")), Some(String::from("build.log")))),
statusreport: Some((Some(String::from("build-results")), None)),
request_id: "bogus-request-id".to_owned(),
};
@ -532,7 +526,6 @@ mod tests {
assert_eq!(actions.next(), Some(worker::Action::Ack));
}
#[test]
pub fn test_all_jobs_skipped() {
let p = TestScratch::new_dir("no-attempt");
@ -545,7 +538,7 @@ mod tests {
let job = buildjob::BuildJob {
attrs: vec!["not-real".to_owned()],
pr: Pr {
head_sha: head_sha,
head_sha,
number: 1,
target_branch: Some("master".to_owned()),
},
@ -556,10 +549,7 @@ mod tests {
owner: "ofborg-test".to_owned(),
},
subset: None,
logs: Some((
Some(String::from("logs")),
Some(String::from("build.log")),
)),
logs: Some((Some(String::from("logs")), Some(String::from("build.log")))),
statusreport: Some((Some(String::from("build-results")), None)),
request_id: "bogus-request-id".to_owned(),
};
@ -570,7 +560,10 @@ mod tests {
println!("Total actions: {:?}", dummyreceiver.actions.len());
let mut actions = dummyreceiver.actions.into_iter();
assert_contains_job(&mut actions, "\"line_number\":1,\"output\":\"Cannot nix-instantiate `not-real\' because:\"");
assert_contains_job(
&mut actions,
"\"line_number\":1,\"output\":\"Cannot nix-instantiate `not-real\' because:\"",
);
assert_contains_job(&mut actions, "\"line_number\":2,\"output\":\"error: attribute 'not-real' in selection path 'not-real' not found\"}");
assert_contains_job(&mut actions, "skipped_attrs\":[\"not-real"); // First one to the github poster
assert_contains_job(&mut actions, "skipped_attrs\":[\"not-real"); // This one to the logs

33
ofborg/src/tasks/eval/stdenvs.rs
View file

@ -1,7 +1,6 @@
use std::path::PathBuf;
use ofborg::nix;
use ofborg::files::file_to_str;
use ofborg::nix;
use std::path::PathBuf;
enum StdenvFrom {
Before,
@ -28,16 +27,16 @@ pub struct Stdenvs {
impl Stdenvs {
pub fn new(nix: nix::Nix, co: PathBuf) -> Stdenvs {
return Stdenvs {
nix: nix,
co: co,
Stdenvs {
nix,
co,
linux_stdenv_before: None,
linux_stdenv_after: None,
darwin_stdenv_before: None,
darwin_stdenv_after: None,
};
}
}
pub fn identify_before(&mut self) {
@ -51,7 +50,7 @@ impl Stdenvs {
}
pub fn are_same(&self) -> bool {
return self.changed().len() == 0;
self.changed().is_empty()
}
pub fn changed(&self) -> Vec<System> {
@ -65,8 +64,7 @@ impl Stdenvs {
changed.push(System::X8664Darwin);
}
return changed;
changed
}
fn identify(&mut self, system: System, from: StdenvFrom) {
@ -91,7 +89,7 @@ impl Stdenvs {
/// given system.
fn evalstdenv(&self, system: &str) -> Option<String> {
let result = self.nix.with_system(system.to_owned()).safely(
nix::Operation::QueryPackagesOutputs,
&nix::Operation::QueryPackagesOutputs,
&self.co,
vec![
String::from("-f"),
@ -104,13 +102,13 @@ impl Stdenvs {
println!("{:?}", result);
return match result {
match result {
Ok(mut out) => Some(file_to_str(&mut out)),
Err(mut out) => {
println!("{:?}", file_to_str(&mut out));
None
}
};
}
}
}
@ -128,16 +126,11 @@ mod tests {
.output()
.expect("nix-instantiate required");
let nixpkgs = String::from_utf8(output.stdout)
.expect("nixpkgs required");
let nixpkgs = String::from_utf8(output.stdout).expect("nixpkgs required");
let remote = env::var("NIX_REMOTE").unwrap_or("".to_owned());
let nix = nix::Nix::new(String::from("x86_64-linux"), remote, 1200, None);
let mut stdenv =
Stdenvs::new(
nix.clone(),
PathBuf::from(nixpkgs.trim_right()),
);
let mut stdenv = Stdenvs::new(nix.clone(), PathBuf::from(nixpkgs.trim_right()));
stdenv.identify(System::X8664Linux, StdenvFrom::Before);
stdenv.identify(System::X8664Darwin, StdenvFrom::Before);

67
ofborg/src/tasks/evaluationfilter.rs
View file

@ -1,24 +1,21 @@
extern crate amqp;
extern crate env_logger;
use ofborg::ghevent;
use ofborg::acl;
use ofborg::ghevent;
use serde_json;
use ofborg::message::{Repo, Pr, massrebuildjob};
use amqp::protocol::basic::{BasicProperties, Deliver};
use ofborg::message::{massrebuildjob, Pr, Repo};
use ofborg::worker;
use amqp::protocol::basic::{Deliver, BasicProperties};
pub struct EvaluationFilterWorker {
acl: acl::ACL,
}
impl EvaluationFilterWorker {
impl EvaluationFilterWorker {
pub fn new(acl: acl::ACL) -> EvaluationFilterWorker {
return EvaluationFilterWorker {
acl: acl,
};
EvaluationFilterWorker { acl }
}
}
@ -29,18 +26,16 @@ impl worker::SimpleWorker for EvaluationFilterWorker {
&mut self,
_: &Deliver,
_: &BasicProperties,
body: &Vec<u8>,
body: &[u8],
) -> Result<Self::J, String> {
return match serde_json::from_slice(body) {
match serde_json::from_slice(body) {
Ok(e) => Ok(e),
Err(e) => {
Err(format!(
"Failed to deserialize job {:?}: {:?}",
e,
String::from_utf8(body.clone())
))
}
};
Err(e) => Err(format!(
"Failed to deserialize job {:?}: {:?}",
e,
String::from_utf8(body.to_vec())
)),
}
}
fn consumer(&mut self, job: &ghevent::PullRequestEvent) -> worker::Actions {
@ -50,7 +45,10 @@ impl worker::SimpleWorker for EvaluationFilterWorker {
}
if job.pull_request.state != ghevent::PullRequestState::Open {
info!("PR is not open ({}#{})", job.repository.full_name, job.number);
info!(
"PR is not open ({}#{})",
job.repository.full_name, job.number
);
return vec![worker::Action::Ack];
}
@ -64,22 +62,22 @@ impl worker::SimpleWorker for EvaluationFilterWorker {
} else {
false
}
},
}
_ => false,
};
if !interesting {
info!("Not interesting: {}#{} because of {:?}",
job.repository.full_name, job.number, job.action
info!(
"Not interesting: {}#{} because of {:?}",
job.repository.full_name, job.number, job.action
);
return vec![
worker::Action::Ack
];
return vec![worker::Action::Ack];
}
info!("Found {}#{} to be interesting because of {:?}",
job.repository.full_name, job.number, job.action
info!(
"Found {}#{} to be interesting because of {:?}",
job.repository.full_name, job.number, job.action
);
let repo_msg = Repo {
clone_url: job.repository.clone_url.clone(),
@ -89,7 +87,7 @@ impl worker::SimpleWorker for EvaluationFilterWorker {
};
let pr_msg = Pr {
number: job.number.clone(),
number: job.number,
head_sha: job.pull_request.head.sha.clone(),
target_branch: Some(job.pull_request.base.git_ref.clone()),
};
@ -100,28 +98,23 @@ impl worker::SimpleWorker for EvaluationFilterWorker {
};
return vec![
worker::publish_serde_action(
None,
Some("mass-rebuild-check-jobs".to_owned()),
&msg
),
worker::Action::Ack
worker::publish_serde_action(None, Some("mass-rebuild-check-jobs".to_owned()), &msg),
worker::Action::Ack,
];
}
}
#[cfg(test)]
mod tests {
use worker::SimpleWorker;
use super::*;
use worker::SimpleWorker;
#[test]
fn changed_base() {
let data = include_str!("../../test-srcs/events/pr-changed-base.json");
let job: ghevent::PullRequestEvent =
serde_json::from_str(&data.to_string())
.expect("Should properly deserialize");
serde_json::from_str(&data.to_string()).expect("Should properly deserialize");
let mut worker = EvaluationFilterWorker::new(acl::ACL::new(
vec!["nixos/nixpkgs".to_owned()],

38
ofborg/src/tasks/githubcommentfilter.rs
View file

@ -2,17 +2,16 @@ extern crate amqp;
extern crate env_logger;
extern crate uuid;
use uuid::Uuid;
use ofborg::ghevent;
use ofborg::acl;
use ofborg::ghevent;
use serde_json;
use uuid::Uuid;
use amqp::protocol::basic::{BasicProperties, Deliver};
use hubcaps;
use ofborg::message::{Repo, Pr, buildjob, massrebuildjob};
use ofborg::worker;
use ofborg::commentparser;
use amqp::protocol::basic::{Deliver, BasicProperties};
use ofborg::message::{buildjob, massrebuildjob, Pr, Repo};
use ofborg::worker;
pub struct GitHubCommentWorker {
acl: acl::ACL,
@ -21,10 +20,7 @@ pub struct GitHubCommentWorker {
impl GitHubCommentWorker {
pub fn new(acl: acl::ACL, github: hubcaps::Github) -> GitHubCommentWorker {
return GitHubCommentWorker {
acl: acl,
github: github,
};
GitHubCommentWorker { acl, github }
}
}
@ -35,18 +31,18 @@ impl worker::SimpleWorker for GitHubCommentWorker {
&mut self,
_: &Deliver,
_: &BasicProperties,
body: &Vec<u8>,
body: &[u8],
) -> Result<Self::J, String> {
return match serde_json::from_slice(body) {
match serde_json::from_slice(body) {
Ok(e) => Ok(e),
Err(e) => {
println!(
"Failed to deserialize IsssueComment: {:?}",
String::from_utf8(body.clone())
String::from_utf8(body.to_vec())
);
panic!("{:?}", e);
}
};
}
}
fn consumer(&mut self, job: &ghevent::IssueComment) -> worker::Actions {
@ -64,7 +60,7 @@ impl worker::SimpleWorker for GitHubCommentWorker {
&job.repository.full_name,
);
if build_destinations.len() == 0 {
if build_destinations.is_empty() {
println!("No build destinations for: {:?}", job);
// Don't process comments if they can't build anything
return vec![worker::Action::Ack];
@ -75,7 +71,8 @@ impl worker::SimpleWorker for GitHubCommentWorker {
let instructions = commentparser::parse(&job.comment.body);
println!("Instructions: {:?}", instructions);
let pr = self.github
let pr = self
.github
.repo(
job.repository.owner.login.clone(),
job.repository.name.clone(),
@ -87,9 +84,7 @@ impl worker::SimpleWorker for GitHubCommentWorker {
if let Err(x) = pr {
info!(
"fetching PR {}#{} from GitHub yielded error {}",
job.repository.full_name,
job.issue.number,
x
job.repository.full_name, job.issue.number, x
);
return vec![worker::Action::Ack];
}
@ -104,7 +99,7 @@ impl worker::SimpleWorker for GitHubCommentWorker {
};
let pr_msg = Pr {
number: job.issue.number.clone(),
number: job.issue.number,
head_sha: pr.head.sha.clone(),
target_branch: Some(pr.base.commit_ref.clone()),
};
@ -140,12 +135,11 @@ impl worker::SimpleWorker for GitHubCommentWorker {
&msg,
));
}
}
}
}
response.push(worker::Action::Ack);
return response;
response
}
}

192
ofborg/src/tasks/githubcommentposter.rs
View file

@ -3,13 +3,12 @@ extern crate env_logger;
use serde_json;
use hubcaps;
use hubcaps::checks::{CheckRunOptions, Output, Conclusion, CheckRunState};
use ofborg::message::buildresult::{BuildStatus, BuildResult, LegacyBuildResult};
use ofborg::worker;
use amqp::protocol::basic::{Deliver, BasicProperties};
use amqp::protocol::basic::{BasicProperties, Deliver};
use chrono::{DateTime, Utc};
use hubcaps;
use hubcaps::checks::{CheckRunOptions, CheckRunState, Conclusion, Output};
use ofborg::message::buildresult::{BuildResult, BuildStatus, LegacyBuildResult};
use ofborg::worker;
pub struct GitHubCommentPoster {
github: hubcaps::Github,
@ -17,7 +16,7 @@ pub struct GitHubCommentPoster {
impl GitHubCommentPoster {
pub fn new(github: hubcaps::Github) -> GitHubCommentPoster {
return GitHubCommentPoster { github: github };
GitHubCommentPoster { github }
}
}
@ -28,64 +27,52 @@ impl worker::SimpleWorker for GitHubCommentPoster {
&mut self,
_: &Deliver,
_: &BasicProperties,
body: &Vec<u8>,
body: &[u8],
) -> Result<Self::J, String> {
return match serde_json::from_slice(body) {
match serde_json::from_slice(body) {
Ok(e) => Ok(e),
Err(e) => {
Err(format!(
"Failed to deserialize BuildResult: {:?}, err: {:}",
String::from_utf8_lossy(&body.clone()),
e
))
}
};
Err(e) => Err(format!(
"Failed to deserialize BuildResult: {:?}, err: {:}",
String::from_utf8_lossy(&body.to_vec()),
e
)),
}
}
fn consumer(&mut self, job: &BuildResult) -> worker::Actions {
let result = job.legacy();
let comment = hubcaps::comments::CommentOptions { body: result_to_comment(&result) };
let comment = hubcaps::comments::CommentOptions {
body: result_to_comment(&result),
};
let check = result_to_check(&result, Utc::now());
println!(":{:?}", check);
println!(":{:?}", comment);
let check_create_attempt = self.github
let check_create_attempt = self
.github
.repo(result.repo.owner.clone(), result.repo.name.clone())
.checkruns()
.create(&check);
match check_create_attempt {
Ok(comment) => {
info!("Successfully sent {:?} to {}",
comment,
result.pr.number,
)
}
Err(err) => {
info!(
"Failed to send check {:?} to {}",
err,
result.pr.number,
)
}
Ok(comment) => info!("Successfully sent {:?} to {}", comment, result.pr.number,),
Err(err) => info!("Failed to send check {:?} to {}", err, result.pr.number,),
}
return vec![worker::Action::Ack];
vec![worker::Action::Ack]
}
}
fn result_to_check(result: &LegacyBuildResult, timestamp: DateTime<Utc>) -> CheckRunOptions {
let mut all_attrs: Vec<String> = vec![
result.attempted_attrs.clone(),
result.skipped_attrs.clone()
]
.into_iter()
.map(|opt| opt.unwrap_or(vec![]))
.flat_map(|list| list.into_iter().map(|attr| format!("-A {}", attr)))
.collect();
let mut all_attrs: Vec<String> =
vec![result.attempted_attrs.clone(), result.skipped_attrs.clone()]
.into_iter()
.map(|opt| opt.unwrap_or_else(|| vec![]))
.flat_map(|list| list.into_iter().map(|attr| format!("-A {}", attr)))
.collect();
all_attrs.sort();
if all_attrs.len() == 0{
if all_attrs.is_empty() {
all_attrs = vec![String::from("(unknown attributes)")];
}
@ -94,12 +81,12 @@ fn result_to_check(result: &LegacyBuildResult, timestamp: DateTime<Utc>) -> Chec
BuildStatus::Success => Conclusion::Success,
BuildStatus::Failure => Conclusion::Neutral,
BuildStatus::TimedOut => Conclusion::Neutral,
BuildStatus::UnexpectedError { err: _ } => Conclusion::Neutral,
BuildStatus::UnexpectedError { .. } => Conclusion::Neutral,
};
let mut summary: Vec<String> = vec![];
if let Some(ref attempted) = result.attempted_attrs {
summary.extend(list_segment("Attempted", attempted.clone()));
summary.extend(list_segment("Attempted", &attempted));
}
if result.status == BuildStatus::TimedOut {
@ -112,12 +99,11 @@ fn result_to_check(result: &LegacyBuildResult, timestamp: DateTime<Utc>) -> Chec
"The following builds were skipped because they don't evaluate on {}",
result.system
),
skipped.clone()));
&skipped,
));
}
let text: String;
if result.output.len() > 0 {
let text: String = if !result.output.is_empty() {
let mut reply: Vec<String> = vec![];
reply.push("## Partial log".to_owned());
@ -126,13 +112,12 @@ fn result_to_check(result: &LegacyBuildResult, timestamp: DateTime<Utc>) -> Chec
reply.extend(result.output.clone());
reply.push("```".to_owned());
text = reply.join("\n");
reply.join("\n")
} else {
text = String::from("No partial log is available.");
}
String::from("No partial log is available.")
};
CheckRunOptions{
CheckRunOptions {
name: format!(
"nix-build {} --argstr system {}",
all_attrs.join(" "),
@ -157,7 +142,7 @@ fn result_to_check(result: &LegacyBuildResult, timestamp: DateTime<Utc>) -> Chec
images: None,
summary: summary.join("\n"),
text: Some(text),
title: "Build Results".to_string()
title: "Build Results".to_string(),
}),
status: Some(CheckRunState::Completed),
}
@ -166,7 +151,7 @@ fn result_to_check(result: &LegacyBuildResult, timestamp: DateTime<Utc>) -> Chec
fn result_to_comment(result: &LegacyBuildResult) -> String {
let mut reply: Vec<String> = vec![];
let log_link = if result.output.len() > 0 {
let log_link = if !result.output.is_empty() {
format!(
" [(full log)](https://logs.nix.ci/?key={}/{}.{}&attempt_id={})",
&result.repo.owner.to_lowercase(),
@ -179,7 +164,8 @@ fn result_to_comment(result: &LegacyBuildResult) -> String {
};
reply.push(format!("<!--REQUEST_ID={}-->", result.request_id));
reply.push(format!("{} on {}{}",
reply.push(format!(
"{} on {}{}",
(match result.status {
BuildStatus::Skipped => "No attempt".into(),
BuildStatus::Success => "Success".into(),
@ -193,7 +179,7 @@ fn result_to_comment(result: &LegacyBuildResult) -> String {
reply.push("".to_owned());
if let Some(ref attempted) = result.attempted_attrs {
reply.extend(list_segment("Attempted", attempted.clone()));
reply.extend(list_segment("Attempted", &attempted));
}
if let Some(ref skipped) = result.skipped_attrs {
@ -202,10 +188,11 @@ fn result_to_comment(result: &LegacyBuildResult) -> String {
"The following builds were skipped because they don't evaluate on {}",
result.system
),
skipped.clone()));
&skipped,
));
}
if result.output.len() > 0 {
if !result.output.is_empty() {
reply.extend(partial_log_segment(&result.output));
reply.push("".to_owned());
reply.push("".to_owned());
@ -217,37 +204,35 @@ fn result_to_comment(result: &LegacyBuildResult) -> String {
reply.join("\n")
}
fn list_segment(name: &str, things: Vec<String>) -> Vec<String> {
fn list_segment(name: &str, things: &[String]) -> Vec<String> {
let mut reply: Vec<String> = vec![];
if things.len() > 0 {
if !things.is_empty() {
reply.push(format!("{}: {}", name, things.join(", ")));
reply.push("".to_owned());
}
return reply;
reply
}
fn partial_log_segment(output: &Vec<String>) -> Vec<String> {
fn partial_log_segment(output: &[String]) -> Vec<String> {
let mut reply: Vec<String> = vec![];
reply.push(
"<details><summary>Partial log (click to expand)</summary><p>".to_owned(),
);
reply.push("<details><summary>Partial log (click to expand)</summary><p>".to_owned());
reply.push("".to_owned());
reply.push("```".to_owned());
reply.extend(output.clone());
reply.extend(output.to_vec());
reply.push("```".to_owned());
reply.push("</p></details>".to_owned());
return reply;
reply
}
#[cfg(test)]
mod tests {
use super::*;
use message::{Pr, Repo};
use chrono::TimeZone;
use message::{Pr, Repo};
#[test]
pub fn test_passing_build() {
@ -671,7 +656,10 @@ No partial log is available.
completed_at: Some("2023-04-20T13:37:42Z".to_string()),
status: Some(CheckRunState::Completed),
conclusion: Some(Conclusion::Success),
details_url: Some("https://logs.nix.ci/?key=nixos/nixpkgs.2345&attempt_id=neatattemptid".to_string()),
details_url: Some(
"https://logs.nix.ci/?key=nixos/nixpkgs.2345&attempt_id=neatattemptid"
.to_string()
),
external_id: Some("neatattemptid".to_string()),
head_sha: "abc123".to_string(),
output: Some(Output {
@ -679,8 +667,10 @@ No partial log is available.
summary: "Attempted: foo
The following builds were skipped because they don't evaluate on x86_64-linux: bar
".to_string(),
text: Some("## Partial log
"
.to_string(),
text: Some(
"## Partial log
```
make[2]: Entering directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline'
@ -693,7 +683,9 @@ post-installation fixup
strip is /nix/store/5a88zk3jgimdmzg8rfhvm93kxib3njf9-cctools-binutils-darwin/bin/strip
patching script interpreter paths in /nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1
/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1
```".to_string()),
```"
.to_string()
),
annotations: None,
images: None,
})
@ -746,14 +738,19 @@ patching script interpreter paths in /nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29
completed_at: Some("2023-04-20T13:37:42Z".to_string()),
status: Some(CheckRunState::Completed),
conclusion: Some(Conclusion::Neutral),
details_url: Some("https://logs.nix.ci/?key=nixos/nixpkgs.2345&attempt_id=neatattemptid".to_string()),
details_url: Some(
"https://logs.nix.ci/?key=nixos/nixpkgs.2345&attempt_id=neatattemptid"
.to_string()
),
external_id: Some("neatattemptid".to_string()),
head_sha: "abc123".to_string(),
output: Some(Output {
title: "Build Results".to_string(),
summary: "Attempted: foo
".to_string(),
text: Some("## Partial log
"
.to_string(),
text: Some(
"## Partial log
```
make[2]: Entering directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline'
@ -766,7 +763,9 @@ post-installation fixup
strip is /nix/store/5a88zk3jgimdmzg8rfhvm93kxib3njf9-cctools-binutils-darwin/bin/strip
patching script interpreter paths in /nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1
/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1
```".to_string()),
```"
.to_string()
),
annotations: None,
images: None,
})
@ -818,15 +817,20 @@ patching script interpreter paths in /nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29
completed_at: Some("2023-04-20T13:37:42Z".to_string()),
status: Some(CheckRunState::Completed),
conclusion: Some(Conclusion::Neutral),
details_url: Some("https://logs.nix.ci/?key=nixos/nixpkgs.2345&attempt_id=neatattemptid".to_string()),
details_url: Some(
"https://logs.nix.ci/?key=nixos/nixpkgs.2345&attempt_id=neatattemptid"
.to_string()
),
external_id: Some("neatattemptid".to_string()),
head_sha: "abc123".to_string(),
output: Some(Output {
title: "Build Results".to_string(),
summary: "Attempted: foo
Build timed out.".to_string(),
text: Some("## Partial log
Build timed out."
.to_string(),
text: Some(
"## Partial log
```
make[2]: Entering directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline'
@ -838,7 +842,9 @@ removed '/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1/share/info/bfd.info
post-installation fixup
building of '/nix/store/l1limh50lx2cx45yb2gqpv7k8xl1mik2-gdb-8.1.drv' timed out after 1 seconds
error: build of '/nix/store/l1limh50lx2cx45yb2gqpv7k8xl1mik2-gdb-8.1.drv' failed
```".to_string()),
```"
.to_string()
),
annotations: None,
images: None,
})
@ -891,13 +897,17 @@ error: build of '/nix/store/l1limh50lx2cx45yb2gqpv7k8xl1mik2-gdb-8.1.drv' failed
completed_at: Some("2023-04-20T13:37:42Z".to_string()),
status: Some(CheckRunState::Completed),
conclusion: Some(Conclusion::Success),
details_url: Some("https://logs.nix.ci/?key=nixos/nixpkgs.2345&attempt_id=neatattemptid".to_string()),
details_url: Some(
"https://logs.nix.ci/?key=nixos/nixpkgs.2345&attempt_id=neatattemptid"
.to_string()
),
external_id: Some("neatattemptid".to_string()),
head_sha: "abc123".to_string(),
output: Some(Output {
title: "Build Results".to_string(),
summary: "".to_string(),
text: Some("## Partial log
text: Some(
"## Partial log
```
make[2]: Entering directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline'
@ -910,7 +920,9 @@ post-installation fixup
strip is /nix/store/5a88zk3jgimdmzg8rfhvm93kxib3njf9-cctools-binutils-darwin/bin/strip
patching script interpreter paths in /nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1
/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1
```".to_string()),
```"
.to_string()
),
annotations: None,
images: None,
})
@ -963,13 +975,17 @@ patching script interpreter paths in /nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29
completed_at: Some("2023-04-20T13:37:42Z".to_string()),
status: Some(CheckRunState::Completed),
conclusion: Some(Conclusion::Neutral),
details_url: Some("https://logs.nix.ci/?key=nixos/nixpkgs.2345&attempt_id=neatattemptid".to_string()),
details_url: Some(
"https://logs.nix.ci/?key=nixos/nixpkgs.2345&attempt_id=neatattemptid"
.to_string()
),
external_id: Some("neatattemptid".to_string()),
head_sha: "abc123".to_string(),
output: Some(Output {
title: "Build Results".to_string(),
summary: "".to_string(),
text: Some("## Partial log
text: Some(
"## Partial log
```
make[2]: Entering directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline'
@ -982,7 +998,9 @@ post-installation fixup
strip is /nix/store/5a88zk3jgimdmzg8rfhvm93kxib3njf9-cctools-binutils-darwin/bin/strip
patching script interpreter paths in /nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1
/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1
```".to_string()),
```"
.to_string()
),
annotations: None,
images: None,
})

226
ofborg/src/tasks/log_message_collector.rs
View file

@ -3,15 +3,15 @@ extern crate env_logger;
use lru_cache::LruCache;
use serde_json;
use std::fs;
use std::fs::{OpenOptions, File};
use std::path::{Component, PathBuf};
use std::fs::{File, OpenOptions};
use std::io::Write;
use std::path::{Component, PathBuf};
use ofborg::writetoline::LineWriter;
use ofborg::message::buildlogmsg::{BuildLogStart, BuildLogMsg};
use amqp::protocol::basic::{BasicProperties, Deliver};
use ofborg::message::buildlogmsg::{BuildLogMsg, BuildLogStart};
use ofborg::message::buildresult::BuildResult;
use ofborg::worker;
use amqp::protocol::basic::{Deliver, BasicProperties};
use ofborg::writetoline::LineWriter;
#[derive(Eq, PartialEq, Hash, Debug, Clone)]
pub struct LogFrom {
@ -28,13 +28,13 @@ pub struct LogMessageCollector {
enum MsgType {
Start(BuildLogStart),
Msg(BuildLogMsg),
Finish(BuildResult),
Finish(Box<BuildResult>),
}
#[derive(Debug)]
pub struct LogMessage {
from: LogFrom,
message: MsgType
message: MsgType,
}
fn validate_path_segment(segment: &PathBuf) -> Result<(), String> {
@ -50,25 +50,24 @@ fn validate_path_segment(segment: &PathBuf) -> Result<(), String> {
println!("Invalid path component: {:?}", e);
false
}
})
{
return Ok(());
}) {
Ok(())
} else {
return Err(String::from("Path contained invalid components"));
Err(String::from("Path contained invalid components"))
}
}
impl LogMessageCollector {
pub fn new(log_root: PathBuf, max_open: usize) -> LogMessageCollector {
return LogMessageCollector {
LogMessageCollector {
handles: LruCache::new(max_open),
log_root: log_root,
};
log_root,
}
}
pub fn write_metadata(&mut self, from: &LogFrom, data: &BuildLogStart) -> Result<(), String>{
pub fn write_metadata(&mut self, from: &LogFrom, data: &BuildLogStart) -> Result<(), String> {
let metapath = self.path_for_metadata(&from)?;
let mut fp = self.open_file(metapath)?;
let mut fp = self.open_file(&metapath)?;
match serde_json::to_string(data) {
Ok(data) => {
@ -77,17 +76,14 @@ impl LogMessageCollector {
} else {
Ok(())
}
},
Err(e) => {
Err(format!("Failed to stringify metadata: {:?}", e))
}
Err(e) => Err(format!("Failed to stringify metadata: {:?}", e)),
}
}
pub fn write_result(&mut self, from: &LogFrom, data: &BuildResult) -> Result<(), String>{
pub fn write_result(&mut self, from: &LogFrom, data: &BuildResult) -> Result<(), String> {
let path = self.path_for_result(&from)?;
let mut fp = self.open_file(path)?;
let mut fp = self.open_file(&path)?;
match serde_json::to_string(data) {
Ok(data) => {
@ -96,29 +92,28 @@ impl LogMessageCollector {
} else {
Ok(())
}
},
Err(e) => {
Err(format!("Failed to stringify result: {:?}", e))
}
Err(e) => Err(format!("Failed to stringify result: {:?}", e)),
}
}
pub fn handle_for(&mut self, from: &LogFrom) -> Result<&mut LineWriter, String> {
if self.handles.contains_key(&from) {
return Ok(self.handles.get_mut(&from).expect(
"handles just contained the key",
));
Ok(self
.handles
.get_mut(&from)
.expect("handles just contained the key"))
} else {
let logpath = self.path_for_log(&from)?;
let fp = self.open_file(logpath)?;
let fp = self.open_file(&logpath)?;
let writer = LineWriter::new(fp);
self.handles.insert(from.clone(), writer);
if let Some(handle) = self.handles.get_mut(&from) {
return Ok(handle);
Ok(handle)
} else {
return Err(String::from(
Err(String::from(
"A just-inserted value should already be there",
));
))
}
}
}
@ -126,13 +121,13 @@ impl LogMessageCollector {
fn path_for_metadata(&self, from: &LogFrom) -> Result<PathBuf, String> {
let mut path = self.path_for_log(from)?;
path.set_extension("metadata.json");
return Ok(path);
Ok(path)
}
fn path_for_result(&self, from: &LogFrom) -> Result<PathBuf, String> {
let mut path = self.path_for_log(from)?;
path.set_extension("result.json");
return Ok(path);
Ok(path)
}
fn path_for_log(&self, from: &LogFrom) -> Result<PathBuf, String> {
@ -147,17 +142,16 @@ impl LogMessageCollector {
location.push(attempt_id);
if location.starts_with(&self.log_root) {
return Ok(location);
Ok(location)
} else {
return Err(format!(
Err(format!(
"Calculating the log location for {:?} resulted in an invalid path {:?}",
from,
location
));
from, location
))
}
}
fn open_file(&self, path: PathBuf) -> Result<File, String> {
fn open_file(&self, path: &PathBuf) -> Result<File, String> {
let dir = path.parent().unwrap();
fs::create_dir_all(dir).unwrap();
@ -172,8 +166,7 @@ impl LogMessageCollector {
Ok(handle) => Ok(handle),
Err(e) => Err(format!(
"Failed to open the file for {:?}, err: {:?}",
&path,
e
&path, e
)),
}
}
@ -186,9 +179,8 @@ impl worker::SimpleWorker for LogMessageCollector {
&mut self,
deliver: &Deliver,
_: &BasicProperties,
body: &Vec<u8>,
body: &[u8],
) -> Result<Self::J, String> {
let message: MsgType;
let attempt_id: String;
@ -205,51 +197,52 @@ impl worker::SimpleWorker for LogMessageCollector {
let decode_msg: Result<BuildResult, _> = serde_json::from_slice(body);
if let Ok(msg) = decode_msg {
attempt_id = msg.legacy().attempt_id.clone();
message = MsgType::Finish(msg);
message = MsgType::Finish(Box::new(msg));
} else {
return Err(format!("failed to decode job: {:?}", decode_msg));
}
}
}
return Ok(LogMessage {
Ok(LogMessage {
from: LogFrom {
routing_key: deliver.routing_key.clone(),
attempt_id: attempt_id,
attempt_id,
},
message: message
});
message,
})
}
fn consumer(&mut self, job: &LogMessage) -> worker::Actions {
match job.message {
MsgType::Start(ref start) => {
self.write_metadata(&job.from, &start).expect("failed to write metadata");
},
self.write_metadata(&job.from, &start)
.expect("failed to write metadata");
}
MsgType::Msg(ref message) => {
let handle = self.handle_for(&job.from).unwrap();
handle.write_to_line((message.line_number - 1) as usize,
&message.output);
},
handle.write_to_line((message.line_number - 1) as usize, &message.output);
}
MsgType::Finish(ref finish) => {
self.write_result(&job.from, &finish).expect("failed to write result");
},
self.write_result(&job.from, &finish)
.expect("failed to write result");
}
}
return vec![worker::Action::Ack];
vec![worker::Action::Ack]
}
}
#[cfg(test)]
mod tests {
use super::*;
use ofborg::message::buildresult::{BuildStatus, V1Tag};
use ofborg::message::{Pr, Repo};
use ofborg::test_scratch::TestScratch;
use ofborg::worker::SimpleWorker;
use std::io::Read;
use std::path::PathBuf;
use ofborg::worker::SimpleWorker;
use ofborg::test_scratch::TestScratch;
use ofborg::message::buildresult::{BuildStatus, V1Tag};
use ofborg::message::{Pr,Repo};
fn make_worker(path: PathBuf) -> LogMessageCollector {
LogMessageCollector::new(path, 3)
@ -291,12 +284,13 @@ mod tests {
})
.expect("the path should be valid");
assert!(path.starts_with(p.path()));
assert!(path.as_os_str().to_string_lossy().ends_with("my-routing-key/my-attempt-id.metadata.json"));
assert!(path
.as_os_str()
.to_string_lossy()
.ends_with("my-routing-key/my-attempt-id.metadata.json"));
}
#[test]
fn test_path_for_result() {
let p = TestScratch::new_dir("log-message-collector-path_for_result");
@ -309,9 +303,11 @@ mod tests {
})
.expect("the path should be valid");
assert!(path.starts_with(p.path()));
assert!(path.as_os_str().to_string_lossy().ends_with("my-routing-key/my-attempt-id.result.json"));
assert!(path
.as_os_str()
.to_string_lossy()
.ends_with("my-routing-key/my-attempt-id.result.json"));
}
#[test]
@ -326,7 +322,6 @@ mod tests {
})
.expect("the path should be valid");
assert!(path.starts_with(p.path()));
assert!(path.ends_with("my-routing-key/my-attempt-id"));
}
@ -361,22 +356,17 @@ mod tests {
assert!(validate_path_segment(&PathBuf::from("/foo")).is_err());
}
#[test]
fn test_open_file() {
let p = TestScratch::new_dir("log-message-collector-open_file");
let worker = make_worker(p.path());
assert!(
worker
.open_file(worker.path_for_log(&make_from("a")).unwrap())
.is_ok()
);
assert!(
worker
.open_file(worker.path_for_log(&make_from("b.foo/123")).unwrap())
.is_ok()
);
assert!(worker
.open_file(&worker.path_for_log(&make_from("a")).unwrap())
.is_ok());
assert!(worker
.open_file(&worker.path_for_log(&make_from("b.foo/123")).unwrap())
.is_ok());
}
#[test]
@ -397,19 +387,18 @@ mod tests {
{
let mut worker = make_worker(p.path());
assert_eq!(vec![worker::Action::Ack],
worker.consumer(&
LogMessage {
from: make_from("foo"),
message: MsgType::Start(BuildLogStart {
attempt_id: String::from("my-attempt-id"),
identity: String::from("my-identity"),
system: String::from("foobar-x8664"),
attempted_attrs: Some(vec!["foo".to_owned()]),
skipped_attrs: Some(vec!["bar".to_owned()]),
})
}
)
assert_eq!(
vec![worker::Action::Ack],
worker.consumer(&LogMessage {
from: make_from("foo"),
message: MsgType::Start(BuildLogStart {
attempt_id: String::from("my-attempt-id"),
identity: String::from("my-identity"),
system: String::from("foobar-x8664"),
attempted_attrs: Some(vec!["foo".to_owned()]),
skipped_attrs: Some(vec!["bar".to_owned()]),
})
})
);
assert_eq!(vec![worker::Action::Ack], worker.consumer(&job));
@ -426,33 +415,32 @@ mod tests {
job.message = MsgType::Msg(logmsg.clone());
assert_eq!(vec![worker::Action::Ack], worker.consumer(&job));
assert_eq!(vec![worker::Action::Ack],
worker.consumer(&
LogMessage {
from: make_from("foo"),
message: MsgType::Finish(BuildResult::V1 {
tag: V1Tag::V1,
repo: Repo {
clone_url: "https://github.com/nixos/ofborg.git".to_owned(),
full_name: "NixOS/ofborg".to_owned(),
owner: "NixOS".to_owned(),
name: "ofborg".to_owned(),
},
pr: Pr {
number: 42,
head_sha: "6dd9f0265d52b946dd13daf996f30b64e4edb446".to_owned(),
target_branch: Some("scratch".to_owned()),
},
system: "x86_64-linux".to_owned(),
output: vec![],
attempt_id: "attempt-id-foo".to_owned(),
request_id: "bogus-request-id".to_owned(),
status: BuildStatus::Success,
attempted_attrs: Some(vec!["foo".to_owned()]),
skipped_attrs: Some(vec!["bar".to_owned()]),
})
}
)
assert_eq!(
vec![worker::Action::Ack],
worker.consumer(&LogMessage {
from: make_from("foo"),
message: MsgType::Finish(Box::new(BuildResult::V1 {
tag: V1Tag::V1,
repo: Repo {
clone_url: "https://github.com/nixos/ofborg.git".to_owned(),
full_name: "NixOS/ofborg".to_owned(),
owner: "NixOS".to_owned(),
name: "ofborg".to_owned(),
},
pr: Pr {
number: 42,
head_sha: "6dd9f0265d52b946dd13daf996f30b64e4edb446".to_owned(),
target_branch: Some("scratch".to_owned()),
},
system: "x86_64-linux".to_owned(),
output: vec![],
attempt_id: "attempt-id-foo".to_owned(),
request_id: "bogus-request-id".to_owned(),
status: BuildStatus::Success,
attempted_attrs: Some(vec!["foo".to_owned()]),
skipped_attrs: Some(vec!["bar".to_owned()]),
}))
})
);
}
@ -462,14 +450,12 @@ mod tests {
File::open(prm).unwrap().read_to_string(&mut sm).unwrap();
assert_eq!(&sm, "{\"system\":\"foobar-x8664\",\"identity\":\"my-identity\",\"attempt_id\":\"my-attempt-id\",\"attempted_attrs\":[\"foo\"],\"skipped_attrs\":[\"bar\"]}");
let mut prf = p.path();
let mut sf = String::new();
prf.push("routing-key-foo/attempt-id-foo");
File::open(prf).unwrap().read_to_string(&mut sf).unwrap();
assert_eq!(&sf, "line-1\n\n\n\nline-5\n");
let mut pr = p.path();
let mut s = String::new();
pr.push("routing-key-foo/my-other-attempt");

296
ofborg/src/tasks/massrebuilder.rs
View file

@ -3,27 +3,27 @@ extern crate amqp;
extern crate env_logger;
extern crate uuid;
use tasks::eval;
use uuid::Uuid;
use amqp::protocol::basic::{BasicProperties, Deliver};
use hubcaps;
use ofborg::acl::ACL;
use ofborg::checkout;
use ofborg::commentparser::Subset;
use ofborg::commitstatus::CommitStatus;
use ofborg::evalchecker::EvalChecker;
use ofborg::files::file_to_str;
use ofborg::message::{buildjob, massrebuildjob};
use ofborg::nix;
use ofborg::outpathdiff::{OutPathDiff, OutPaths};
use ofborg::stats;
use ofborg::stats::Event;
use ofborg::tagger::{PathsTagger, PkgsAddedRemovedTagger, RebuildTagger, StdenvTagger};
use ofborg::worker;
use std::collections::HashMap;
use std::path::Path;
use std::path::PathBuf;
use ofborg::checkout;
use ofborg::message::{massrebuildjob, buildjob};
use std::time::Instant;
use ofborg::files::file_to_str;
use ofborg::nix;
use ofborg::acl::ACL;
use ofborg::stats;
use ofborg::stats::Event;
use ofborg::worker;
use ofborg::tagger::{StdenvTagger, RebuildTagger, PathsTagger, PkgsAddedRemovedTagger};
use ofborg::outpathdiff::{OutPaths, OutPathDiff};
use ofborg::evalchecker::EvalChecker;
use ofborg::commitstatus::CommitStatus;
use ofborg::commentparser::Subset;
use amqp::protocol::basic::{Deliver, BasicProperties};
use hubcaps;
use tasks::eval;
use uuid::Uuid;
pub struct MassRebuildWorker<E> {
cloner: checkout::CachedCloner,
@ -38,42 +38,39 @@ pub struct MassRebuildWorker<E> {
impl<E: stats::SysEvents> MassRebuildWorker<E> {
pub fn new(
cloner: checkout::CachedCloner,
nix: nix::Nix,
nix: &nix::Nix,
github: hubcaps::Github,
acl: ACL,
identity: String,
events: E,
tag_paths: HashMap<String, Vec<String>>,
) -> MassRebuildWorker<E> {
return MassRebuildWorker {
cloner: cloner,
MassRebuildWorker {
cloner,
nix: nix.without_limited_supported_systems(),
github: github,
acl: acl,
identity: identity,
events: events,
tag_paths: tag_paths
};
github,
acl,
identity,
events,
tag_paths,
}
}
fn actions(&self) -> massrebuildjob::Actions {
return massrebuildjob::Actions {};
massrebuildjob::Actions {}
}
fn tag_from_title(&self, issue: &hubcaps::issues::IssueRef) {
let darwin = issue.get()
let darwin = issue
.get()
.map(|iss| {
iss.title.to_lowercase().contains("darwin") ||
iss.title.to_lowercase().contains("macos")
iss.title.to_lowercase().contains("darwin")
|| iss.title.to_lowercase().contains("macos")
})
.unwrap_or(false);
if darwin {
update_labels(
&issue,
vec![String::from("6.topic: darwin")],
vec![],
);
update_labels(&issue, &[String::from("6.topic: darwin")], &[]);
}
}
@ -84,11 +81,7 @@ impl<E: stats::SysEvents> MassRebuildWorker<E> {
tagger.path_changed(&path);
}
update_labels(
&issue,
tagger.tags_to_add(),
tagger.tags_to_remove(),
);
update_labels(&issue, &tagger.tags_to_add(), &tagger.tags_to_remove());
}
}
@ -99,10 +92,10 @@ impl<E: stats::SysEvents + 'static> worker::SimpleWorker for MassRebuildWorker<E
&mut self,
_: &Deliver,
_: &BasicProperties,
body: &Vec<u8>,
body: &[u8],
) -> Result<Self::J, String> {
self.events.notify(Event::JobReceived);
return match massrebuildjob::from(body) {
match massrebuildjob::from(body) {
Ok(e) => {
self.events.notify(Event::JobDecodeSuccess);
Ok(e)
@ -111,19 +104,18 @@ impl<E: stats::SysEvents + 'static> worker::SimpleWorker for MassRebuildWorker<E
self.events.notify(Event::JobDecodeFailure);
error!(
"Failed to decode message: {:?}, Err: {:?}",
String::from_utf8(body.clone()),
String::from_utf8(body.to_vec()),
e
);
Err("Failed to decode message".to_owned())
}
};
}
}
fn consumer(&mut self, job: &massrebuildjob::MassRebuildJob) -> worker::Actions {
let repo = self.github.repo(
job.repo.owner.clone(),
job.repo.name.clone(),
);
let repo = self
.github
.repo(job.repo.owner.clone(), job.repo.name.clone());
let gists = self.github.gists();
let issue = repo.issue(job.pr.number);
@ -140,10 +132,9 @@ impl<E: stats::SysEvents + 'static> worker::SimpleWorker for MassRebuildWorker<E
if issue_is_wip(&iss) {
auto_schedule_build_archs = vec![];
} else {
auto_schedule_build_archs = self.acl.build_job_destinations_for_user_repo(
&iss.user.login,
&job.repo.full_name,
);
auto_schedule_build_archs = self
.acl
.build_job_destinations_for_user_repo(&iss.user.login, &job.repo.full_name);
}
}
Err(e) => {
@ -166,10 +157,9 @@ impl<E: stats::SysEvents + 'static> worker::SimpleWorker for MassRebuildWorker<E
overall_status.set_with_description("Starting", hubcaps::statuses::State::Pending);
let project = self.cloner.project(
job.repo.full_name.clone(),
job.repo.clone_url.clone(),
);
let project = self
.cloner
.project(&job.repo.full_name, job.repo.clone_url.clone());
overall_status.set_with_description("Cloning project", hubcaps::statuses::State::Pending);
@ -195,7 +185,6 @@ impl<E: stats::SysEvents + 'static> worker::SimpleWorker for MassRebuildWorker<E
hubcaps::statuses::State::Pending,
);
let mut stdenvs = eval::Stdenvs::new(self.nix.clone(), PathBuf::from(&refpath));
stdenvs.identify_before();
@ -211,12 +200,13 @@ impl<E: stats::SysEvents + 'static> worker::SimpleWorker for MassRebuildWorker<E
if let Err(mut output) = rebuildsniff.find_before() {
overall_status.set_url(make_gist(
&gists,
"Output path comparison".to_owned(),
"Output path comparison",
Some("".to_owned()),
file_to_str(&mut output),
));
self.events.notify(Event::TargetBranchFailsEvaluation(target_branch.clone()));
self.events
.notify(Event::TargetBranchFailsEvaluation(target_branch.clone()));
overall_status.set_with_description(
format!("Target branch {} doesn't evaluate!", &target_branch).as_ref(),
hubcaps::statuses::State::Failure,
@ -224,83 +214,63 @@ impl<E: stats::SysEvents + 'static> worker::SimpleWorker for MassRebuildWorker<E
return self.actions().skip(&job);
}
self.events.notify(
Event::EvaluationDuration(
target_branch.clone(),
target_branch_rebuild_sniff_start.elapsed().as_secs(),
)
);
self.events.notify(
Event::EvaluationDurationCount(
target_branch.clone()
)
);
self.events.notify(Event::EvaluationDuration(
target_branch.clone(),
target_branch_rebuild_sniff_start.elapsed().as_secs(),
));
self.events
.notify(Event::EvaluationDurationCount(target_branch.clone()));
overall_status.set_with_description("Fetching PR", hubcaps::statuses::State::Pending);
co.fetch_pr(job.pr.number).unwrap();
if !co.commit_exists(job.pr.head_sha.as_ref()) {
overall_status.set_with_description(
"Commit not found",
hubcaps::statuses::State::Error,
);
overall_status
.set_with_description("Commit not found", hubcaps::statuses::State::Error);
info!("Commit {} doesn't exist", job.pr.head_sha);
return self.actions().skip(&job);
}
let possibly_touched_packages =
parse_commit_messages(co.commit_messages_from_head(&job.pr.head_sha).unwrap_or(
vec!["".to_owned()],
));
let possibly_touched_packages = parse_commit_messages(
&co.commit_messages_from_head(&job.pr.head_sha)
.unwrap_or_else(|_| vec!["".to_owned()]),
);
self.tag_from_paths(
&issue,
co.files_changed_from_head(&job.pr.head_sha).unwrap_or(vec![])
co.files_changed_from_head(&job.pr.head_sha)
.unwrap_or_else(|_| vec![]),
);
overall_status.set_with_description("Merging PR", hubcaps::statuses::State::Pending);
if let Err(_) = co.merge_commit(job.pr.head_sha.as_ref()) {
overall_status.set_with_description(
"Failed to merge",
hubcaps::statuses::State::Failure,
);
if co.merge_commit(job.pr.head_sha.as_ref()).is_err() {
overall_status
.set_with_description("Failed to merge", hubcaps::statuses::State::Failure);
info!("Failed to merge {}", job.pr.head_sha);
update_labels(
&issue,
vec!["2.status: merge conflict".to_owned()],
vec![],
);
update_labels(&issue, &["2.status: merge conflict".to_owned()], &[]);
return self.actions().skip(&job);
} else {
update_labels(
&issue,
vec![],
vec!["2.status: merge conflict".to_owned()],
);
update_labels(&issue, &[], &["2.status: merge conflict".to_owned()]);
}
overall_status.set_with_description(
"Checking new stdenvs",
hubcaps::statuses::State::Pending,
);
overall_status
.set_with_description("Checking new stdenvs", hubcaps::statuses::State::Pending);
stdenvs.identify_after();
overall_status.set_with_description(
"Checking new out paths",
hubcaps::statuses::State::Pending,
);
overall_status
.set_with_description("Checking new out paths", hubcaps::statuses::State::Pending);
if let Err(mut output) = rebuildsniff.find_after() {
overall_status.set_url(make_gist(
&gists,
"Output path comparison".to_owned(),
"Output path comparison",
Some("".to_owned()),
file_to_str(&mut output),
));
@ -308,29 +278,24 @@ impl<E: stats::SysEvents + 'static> worker::SimpleWorker for MassRebuildWorker<E
format!(
"Failed to enumerate outputs after merging to {}",
&target_branch
).as_ref(),
)
.as_ref(),
hubcaps::statuses::State::Failure,
);
return self.actions().skip(&job);
}
println!("Got path: {:?}, building", refpath);
overall_status.set_with_description(
"Beginning Evaluations",
hubcaps::statuses::State::Pending,
);
overall_status
.set_with_description("Beginning Evaluations", hubcaps::statuses::State::Pending);
let eval_checks = vec![
EvalChecker::new(
"package-list",
nix::Operation::QueryPackagesJSON,
vec![
String::from("--file"),
String::from("."),
],
self.nix.clone()
vec![String::from("--file"), String::from(".")],
self.nix.clone(),
),
EvalChecker::new(
"package-list-no-aliases",
nix::Operation::QueryPackagesJSON,
@ -341,9 +306,8 @@ impl<E: stats::SysEvents + 'static> worker::SimpleWorker for MassRebuildWorker<E
String::from("config"),
String::from("{ allowAliases = false; }"),
],
self.nix.clone()
self.nix.clone(),
),
EvalChecker::new(
"nixos-options",
nix::Operation::Instantiate,
@ -355,9 +319,8 @@ impl<E: stats::SysEvents + 'static> worker::SimpleWorker for MassRebuildWorker<E
String::from("-A"),
String::from("options"),
],
self.nix.clone()
self.nix.clone(),
),
EvalChecker::new(
"nixos-manual",
nix::Operation::Instantiate,
@ -369,9 +332,8 @@ impl<E: stats::SysEvents + 'static> worker::SimpleWorker for MassRebuildWorker<E
String::from("-A"),
String::from("manual"),
],
self.nix.clone()
self.nix.clone(),
),
EvalChecker::new(
"nixpkgs-manual",
nix::Operation::Instantiate,
@ -383,9 +345,8 @@ impl<E: stats::SysEvents + 'static> worker::SimpleWorker for MassRebuildWorker<E
String::from("-A"),
String::from("manual"),
],
self.nix.clone()
self.nix.clone(),
),
EvalChecker::new(
"nixpkgs-tarball",
nix::Operation::Instantiate,
@ -397,9 +358,8 @@ impl<E: stats::SysEvents + 'static> worker::SimpleWorker for MassRebuildWorker<E
String::from("-A"),
String::from("tarball"),
],
self.nix.clone()
self.nix.clone(),
),
EvalChecker::new(
"nixpkgs-unstable-jobset",
nix::Operation::Instantiate,
@ -411,7 +371,7 @@ impl<E: stats::SysEvents + 'static> worker::SimpleWorker for MassRebuildWorker<E
String::from("-A"),
String::from("unstable"),
],
self.nix.clone()
self.nix.clone(),
),
];
@ -439,7 +399,7 @@ impl<E: stats::SysEvents + 'static> worker::SimpleWorker for MassRebuildWorker<E
state = hubcaps::statuses::State::Failure;
gist_url = make_gist(
&gists,
check.name(),
&check.name(),
Some(format!("{:?}", state)),
file_to_str(&mut out),
);
@ -450,14 +410,13 @@ impl<E: stats::SysEvents + 'static> worker::SimpleWorker for MassRebuildWorker<E
status.set(state.clone());
if state == hubcaps::statuses::State::Success {
return Ok(());
Ok(())
} else {
return Err(());
Err(())
}
})
.all(|status| status == Ok(()));
let mut response: worker::Actions = vec![];
if eval_results {
@ -480,14 +439,15 @@ impl<E: stats::SysEvents + 'static> worker::SimpleWorker for MassRebuildWorker<E
state = hubcaps::statuses::State::Success;
gist_url = None;
let mut try_build: Vec<String> = pkgs.keys()
let mut try_build: Vec<String> = pkgs
.keys()
.map(|pkgarch| pkgarch.package.clone())
.filter(|pkg| possibly_touched_packages.contains(&pkg))
.collect();
try_build.sort();
try_build.dedup();
if try_build.len() > 0 && try_build.len() <= 10 {
if !try_build.is_empty() && try_build.len() <= 10 {
// In the case of trying to merge master in to
// a stable branch, we don't want to do this.
// Therefore, only schedule builds if there
@ -511,7 +471,7 @@ impl<E: stats::SysEvents + 'static> worker::SimpleWorker for MassRebuildWorker<E
state = hubcaps::statuses::State::Failure;
gist_url = make_gist(
&gists,
String::from("Meta Check"),
"Meta Check",
Some(format!("{:?}", state)),
file_to_str(&mut out),
);
@ -534,26 +494,26 @@ impl<E: stats::SysEvents + 'static> worker::SimpleWorker for MassRebuildWorker<E
}
update_labels(
&issue,
stdenvtagger.tags_to_add(),
stdenvtagger.tags_to_remove(),
&stdenvtagger.tags_to_add(),
&stdenvtagger.tags_to_remove(),
);
if let Some((removed, added)) = rebuildsniff.package_diff() {
let mut addremovetagger = PkgsAddedRemovedTagger::new();
addremovetagger.changed(removed, added);
let mut addremovetagger = PkgsAddedRemovedTagger::new();
addremovetagger.changed(&removed, &added);
update_labels(
&issue,
addremovetagger.tags_to_add(),
addremovetagger.tags_to_remove(),
&addremovetagger.tags_to_add(),
&addremovetagger.tags_to_remove(),
);
}
let mut rebuild_tags = RebuildTagger::new();
if let Some(attrs) = rebuildsniff.calculate_rebuild() {
if attrs.len() > 0 {
if !attrs.is_empty() {
let gist_url = make_gist(
&gists,
String::from("Changed Paths"),
"Changed Paths",
Some("".to_owned()),
attrs
.iter()
@ -570,55 +530,50 @@ impl<E: stats::SysEvents + 'static> worker::SimpleWorker for MassRebuildWorker<E
update_labels(
&issue,
rebuild_tags.tags_to_add(),
rebuild_tags.tags_to_remove(),
&rebuild_tags.tags_to_add(),
&rebuild_tags.tags_to_remove(),
);
overall_status.set_with_description("^.^!", hubcaps::statuses::State::Success);
} else {
overall_status.set_with_description(
"Complete, with errors",
hubcaps::statuses::State::Failure,
);
overall_status
.set_with_description("Complete, with errors", hubcaps::statuses::State::Failure);
}
self.events.notify(Event::TaskEvaluationCheckComplete);
return self.actions().done(&job, response);
self.actions().done(&job, response)
}
}
fn make_gist<'a>(
gists: &hubcaps::gists::Gists<'a>,
name: String,
name: &str,
description: Option<String>,
contents: String,
) -> Option<String> {
let mut files = HashMap::new();
let mut files: HashMap<String, hubcaps::gists::Content> = HashMap::new();
files.insert(
name.clone(),
name.to_string(),
hubcaps::gists::Content {
filename: Some(name.clone()),
filename: Some(name.to_string()),
content: contents,
},
);
return Some(
Some(
gists
.create(&hubcaps::gists::GistOptions {
description: description,
description,
public: Some(true),
files: files,
files,
})
.expect("Failed to create gist!")
.html_url,
);
)
}
pub fn update_labels(issue: &hubcaps::issues::IssueRef, add: Vec<String>, remove: Vec<String>) {
pub fn update_labels(issue: &hubcaps::issues::IssueRef, add: &[String], remove: &[String]) {
let l = issue.labels();
let existing: Vec<String> = issue
@ -632,13 +587,14 @@ pub fn update_labels(issue: &hubcaps::issues::IssueRef, add: Vec<String>, remove
let to_add = add
.iter()
.filter(|l| !existing.contains(l)) // Remove labels already on the issue
.map(|l| l.as_ref()).collect();
.map(|l| l.as_ref())
.collect();
info!("Adding labels: {:?}", to_add);
let to_remove: Vec<String> = remove
.iter()
.filter(|l| existing.contains(l)) // Remove labels already on the issue
.map(|l| l.clone())
.cloned()
.collect();
info!("Removing labels: {:?}", to_remove);
@ -649,12 +605,12 @@ pub fn update_labels(issue: &hubcaps::issues::IssueRef, add: Vec<String>, remove
}
}
fn parse_commit_messages(messages: Vec<String>) -> Vec<String> {
fn parse_commit_messages(messages: &[String]) -> Vec<String> {
messages
.iter()
.filter_map(|line| {
// Convert "foo: some notes" in to "foo"
let parts: Vec<&str> = line.splitn(2, ":").collect();
let parts: Vec<&str> = line.splitn(2, ':').collect();
if parts.len() == 2 {
Some(parts[0])
} else {
@ -662,7 +618,7 @@ fn parse_commit_messages(messages: Vec<String>) -> Vec<String> {
}
})
.flat_map(|line| {
let pkgs: Vec<&str> = line.split(",").collect();
let pkgs: Vec<&str> = line.split(',').collect();
pkgs
})
.map(|line| line.trim().to_owned())
@ -678,7 +634,7 @@ mod tests {
fn test_parse_commit_messages() {
let expect: Vec<&str> = vec![
"firefox{-esr", // don't support such fancy syntax
"}", // Don't support such fancy syntax
"}", // Don't support such fancy syntax
"firefox",
"buildkite-agent",
"python.pkgs.ptyprocess",
@ -689,7 +645,7 @@ mod tests {
];
assert_eq!(
parse_commit_messages(
"
&"
firefox{-esr,}: fix failing build due to the google-api-key
Merge pull request #34483 from andir/dovecot-cve-2017-15132
firefox: enable official branding
@ -703,9 +659,9 @@ mod tests {
Merge pull request #34414 from dotlambda/postfix
foo,bar: something here: yeah
"
.lines()
.map(|l| l.to_owned())
.collect(),
.lines()
.map(|l| l.to_owned())
.collect::<Vec<String>>(),
),
expect
);
@ -735,5 +691,5 @@ fn indicates_wip(text: &str) -> bool {
return true;
}
return false;
false
}

9
ofborg/src/tasks/mod.rs
View file

@ -1,9 +1,8 @@
pub mod eval;
pub mod build;
pub mod massrebuilder;
pub mod eval;
pub mod evaluationfilter;
pub mod githubcommentfilter;
pub mod githubcommentposter;
pub mod statscollector;
pub mod log_message_collector;
pub mod evaluationfilter;
pub mod massrebuilder;
pub mod statscollector;

34
ofborg/src/tasks/statscollector.rs
View file

@ -1,10 +1,10 @@
extern crate amqp;
extern crate env_logger;
use serde_json;
use ofborg::worker;
use amqp::protocol::basic::{BasicProperties, Deliver};
use ofborg::stats;
use amqp::protocol::basic::{Deliver, BasicProperties};
use ofborg::worker;
use serde_json;
pub struct StatCollectorWorker<E> {
events: E,
@ -13,10 +13,7 @@ pub struct StatCollectorWorker<E> {
impl<E: stats::SysEvents + 'static> StatCollectorWorker<E> {
pub fn new(events: E, collector: stats::MetricCollector) -> StatCollectorWorker<E> {
StatCollectorWorker {
events: events,
collector: collector,
}
StatCollectorWorker { events, collector }
}
}
@ -27,44 +24,45 @@ impl<E: stats::SysEvents + 'static> worker::SimpleWorker for StatCollectorWorker
&mut self,
_: &Deliver,
_: &BasicProperties,
body: &Vec<u8>,
body: &[u8],
) -> Result<Self::J, String> {
return match serde_json::from_slice(body) {
match serde_json::from_slice(body) {
Ok(e) => Ok(e),
Err(_) => {
let mut modified_body: Vec<u8> = vec!["\"".as_bytes()[0]];
modified_body.append(&mut body.clone());
modified_body.push("\"".as_bytes()[0]);
let mut modified_body: Vec<u8> = vec![b"\""[0]];
modified_body.append(&mut body.to_vec());
modified_body.push(b"\""[0]);
match serde_json::from_slice(&modified_body) {
Ok(e) => {
self.events.notify(stats::Event::StatCollectorLegacyEvent(stats::event_metric_name(&e)));
self.events.notify(stats::Event::StatCollectorLegacyEvent(
stats::event_metric_name(&e),
));
Ok(stats::EventMessage {
sender: "".to_owned(),
events: vec![e],
})
},
}
Err(e) => {
self.events.notify(stats::Event::StatCollectorBogusEvent);
error!(
"Failed to decode message: {:?}, Err: {:?}",
String::from_utf8(body.clone()),
String::from_utf8(body.to_vec()),
e
);
Err("Failed to decode message".to_owned())
}
}
}
};
}
}
fn consumer(&mut self, job: &stats::EventMessage) -> worker::Actions {
let sender = job.sender.clone();
for event in job.events.iter() {
self.collector.record(sender.clone(), event.clone());
}
return vec![worker::Action::Ack];
vec![worker::Action::Ack]
}
}

4
ofborg/src/test_scratch.rs
View file

@ -17,7 +17,7 @@ impl TestScratch {
TestScratch::create_dir(&scratch);
return scratch;
scratch
}
pub fn new_file(ident: &str) -> TestScratch {
@ -29,7 +29,7 @@ impl TestScratch {
};
TestScratch::create_dir(&scratch);
return scratch;
scratch
}
fn create_dir(path: &TestScratch) {

34
ofborg/src/worker.rs
View file

@ -1,9 +1,9 @@
use amqp::protocol::basic::{BasicProperties, Deliver};
use amqp::Basic;
use amqp::{Consumer, Channel};
use amqp::protocol::basic::{Deliver, BasicProperties};
use std::marker::Send;
use amqp::{Channel, Consumer};
use serde::Serialize;
use serde_json;
use std::marker::Send;
pub struct Worker<T: SimpleWorker> {
internal: T,
@ -18,7 +18,7 @@ pub enum Action {
Ack,
NackRequeue,
NackDump,
Publish(QueueMsg),
Publish(Box<QueueMsg>),
}
#[derive(Debug, PartialEq)]
@ -44,14 +44,14 @@ where
..Default::default()
};
return Action::Publish(QueueMsg {
exchange: exchange,
routing_key: routing_key,
Action::Publish(Box::new(QueueMsg {
exchange,
routing_key,
mandatory: false,
immediate: false,
properties: Some(props),
content: serde_json::to_string(&msg).unwrap().into_bytes(),
});
}))
}
pub trait SimpleWorker: Send + 'static {
@ -63,16 +63,14 @@ pub trait SimpleWorker: Send + 'static {
&mut self,
method: &Deliver,
headers: &BasicProperties,
body: &Vec<u8>,
body: &[u8],
) -> Result<Self::J, String>;
}
pub fn new<T: SimpleWorker>(worker: T) -> Worker<T> {
return Worker { internal: worker };
Worker { internal: worker }
}
impl<T: SimpleWorker + Send> Consumer for Worker<T> {
fn handle_delivery(
&mut self,
@ -104,13 +102,13 @@ impl<T: SimpleWorker + Send> Consumer for Worker<T> {
.basic_nack(method.delivery_tag, false, false)
.unwrap();
}
Action::Publish(msg) => {
let exch = msg.exchange.clone().unwrap_or("".to_owned());
let key = msg.routing_key.clone().unwrap_or("".to_owned());
Action::Publish(mut msg) => {
let exch = msg.exchange.take().unwrap_or_else(|| "".to_owned());
let key = msg.routing_key.take().unwrap_or_else(|| "".to_owned());
let props = msg.properties.unwrap_or(
BasicProperties { ..Default::default() },
);
let props = msg.properties.take().unwrap_or(BasicProperties {
..Default::default()
});
channel
.basic_publish(exch, key, msg.mandatory, msg.immediate, props, msg.content)
.unwrap();

27
ofborg/src/writetoline.rs
View file

@ -1,9 +1,9 @@
use std::io::BufReader;
use std::fs::File;
use std::io::BufRead;
use std::io::Write;
use std::io::BufReader;
use std::io::Seek;
use std::io::SeekFrom;
use std::fs::File;
use std::io::Write;
pub struct LineWriter {
file: File,
@ -16,13 +16,11 @@ impl LineWriter {
let buf = LineWriter::load_buffer(&mut rw);
let len = buf.len();
let writer = LineWriter {
LineWriter {
file: rw,
buffer: buf,
last_line: len,
};
return writer;
}
}
fn load_buffer(file: &mut File) -> Vec<String> {
@ -56,7 +54,7 @@ impl LineWriter {
self.file
.write_all(self.buffer.join("\n").as_bytes())
.unwrap();
self.file.write("\n".as_bytes()).unwrap();
self.file.write_all(b"\n").unwrap();
} else {
// println!("taking the append option");
// println!("Writing {:?} to line {}", data, line);
@ -71,8 +69,8 @@ impl LineWriter {
// we have to use one more than the range we want for the
// end
// println!("selected buffer: {:?}", to_write);
self.file.write(to_write.as_bytes()).unwrap();
self.file.write("\n".as_bytes()).unwrap();
self.file.write_all(to_write.as_bytes()).unwrap();
self.file.write_all(b"\n").unwrap();
}
self.last_line = line;
@ -83,15 +81,14 @@ impl LineWriter {
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::path::Path;
use std::fs::File;
use std::io::Read;
use std::fs::OpenOptions;
use ofborg::test_scratch::TestScratch;
use std::fs::File;
use std::fs::OpenOptions;
use std::io::Read;
use std::path::Path;
use std::time::Instant;
fn testfile(path: &Path) -> File {

44
shell.nix
View file

@ -1,4 +1,8 @@
{ pkgs ? import ./nix {}, useNix1 ? false }:
{ pkgs ? import ./nix {
overlays = [
(import (builtins.fetchTarball https://github.com/mozilla/nixpkgs-mozilla/archive/master.tar.gz))
];
}, useNix1 ? false }:
let
# A random Nixpkgs revision *before* the default glibc
@ -31,6 +35,43 @@ let
# HISTFILE = "${src}/.bash_hist";
};
mozilla-rust-overlay = stdenv.mkDerivation (rec {
name = "mozilla-rust-overlay";
buildInputs = with pkgs; [
latest.rustChannels.stable.rust
git
pkgconfig
openssl.dev
]
++ stdenv.lib.optional stdenv.isDarwin pkgs.darwin.Security;
postHook = ''
checkPhase() (
cd "${builtins.toString ./.}/ofborg"
set -x
cargo fmt
git diff --exit-code
cargofmtexit=$?
cargo clippy
cargoclippyexit=$?
sum=$((cargofmtexit + cargoclippyexit))
exit $sum
)
'';
RUSTFLAGS = "-D warnings";
RUST_BACKTRACE = "1";
NIX_PATH = "nixpkgs=${pkgs.path}";
}
// stdenv.lib.optionalAttrs stdenv.isLinux {
LOCALE_ARCHIVE_2_21 = "${oldpkgs.glibcLocales}/lib/locale/locale-archive";
LOCALE_ARCHIVE_2_27 = "${pkgs.glibcLocales}/lib/locale/locale-archive";
});
rustEnv = stdenv.mkDerivation (rec {
name = "gh-event-forwarder";
buildInputs = with pkgs; [
@ -59,6 +100,7 @@ let
RUST_LOG = "ofborg=debug";
NIX_PATH = "nixpkgs=${pkgs.path}";
passthru.phpEnv = phpEnv;
passthru.mozilla-rust-overlay = mozilla-rust-overlay;
}
// stdenv.lib.optionalAttrs stdenv.isLinux {
LOCALE_ARCHIVE_2_21 = "${oldpkgs.glibcLocales}/lib/locale/locale-archive";