Compare commits

...

2 commits

Author SHA1 Message Date
Ilya K 6348541982 fix: delete unused macro 2024-11-16 23:52:05 +03:00
Ilya K 43ec489263 refactor: asyncify everything
Tracing actually works now. Me and clippy have signed a non-aggression pact.
2024-11-16 23:49:18 +03:00
59 changed files with 1561 additions and 1325 deletions

31
Cargo.lock generated
View file

@ -1833,6 +1833,7 @@ name = "ofborg"
version = "0.90.0" version = "0.90.0"
dependencies = [ dependencies = [
"async-stream", "async-stream",
"async-trait",
"axum", "axum",
"base64", "base64",
"brace-expand", "brace-expand",
@ -1854,6 +1855,7 @@ dependencies = [
"opentelemetry", "opentelemetry",
"opentelemetry-otlp", "opentelemetry-otlp",
"opentelemetry-semantic-conventions", "opentelemetry-semantic-conventions",
"opentelemetry-stdout",
"opentelemetry_sdk", "opentelemetry_sdk",
"regex", "regex",
"rustls-pemfile", "rustls-pemfile",
@ -1880,6 +1882,7 @@ version = "0.1.0"
dependencies = [ dependencies = [
"log", "log",
"ofborg", "ofborg",
"tokio",
] ]
[[package]] [[package]]
@ -1986,6 +1989,23 @@ version = "0.27.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc1b6902ff63b32ef6c489e8048c5e253e2e4a803ea3ea7e783914536eb15c52" checksum = "bc1b6902ff63b32ef6c489e8048c5e253e2e4a803ea3ea7e783914536eb15c52"
[[package]]
name = "opentelemetry-stdout"
version = "0.27.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc8a298402aa5c260be90d10dc54b5a7d4e1025c354848f8e2c976d761351049"
dependencies = [
"async-trait",
"chrono",
"futures-util",
"opentelemetry",
"opentelemetry_sdk",
"ordered-float",
"serde",
"serde_json",
"thiserror 1.0.69",
]
[[package]] [[package]]
name = "opentelemetry_sdk" name = "opentelemetry_sdk"
version = "0.27.0" version = "0.27.0"
@ -2003,6 +2023,8 @@ dependencies = [
"rand", "rand",
"serde_json", "serde_json",
"thiserror 1.0.69", "thiserror 1.0.69",
"tokio",
"tokio-stream",
"tracing", "tracing",
] ]
@ -2012,6 +2034,15 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"
[[package]]
name = "ordered-float"
version = "4.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c65ee1f9701bf938026630b455d5315f490640234259037edb259798b3bcf85e"
dependencies = [
"num-traits",
]
[[package]] [[package]]
name = "overload" name = "overload"
version = "0.1.1" version = "0.1.1"

View file

@ -7,4 +7,5 @@ edition = "2018"
[dependencies] [dependencies]
ofborg = { path = "../ofborg" } ofborg = { path = "../ofborg" }
tokio = "*"
log = "0.4.22" log = "0.4.22"

View file

@ -8,7 +8,8 @@ use std::path::Path;
use ofborg::config; use ofborg::config;
use ofborg::nix; use ofborg::nix;
fn main() { #[tokio::main]
async fn main() {
ofborg::setup_log(); ofborg::setup_log();
log::info!("Loading config..."); log::info!("Loading config...");
@ -16,11 +17,14 @@ fn main() {
let nix = cfg.nix(); let nix = cfg.nix();
log::info!("Running build..."); log::info!("Running build...");
match nix.safely_build_attrs( match nix
.safely_build_attrs(
Path::new("./"), Path::new("./"),
nix::File::DefaultNixpkgs, nix::File::DefaultNixpkgs,
vec![String::from("hello")], vec![String::from("hello")],
) { )
.await
{
Ok(mut out) => { Ok(mut out) => {
print!("{}", file_to_str(&mut out)); print!("{}", file_to_str(&mut out));
} }

View file

@ -10,6 +10,7 @@ edition = "2018"
[dependencies] [dependencies]
async-stream = "0.3.6" async-stream = "0.3.6"
async-trait = "0.1.83"
axum = "0.7.8" axum = "0.7.8"
base64 = "0.22.1" base64 = "0.22.1"
brace-expand = "0.1.0" brace-expand = "0.1.0"
@ -26,12 +27,13 @@ jfs = "0.9.0"
lapin = "2.5.0" lapin = "2.5.0"
lru-cache = "0.1.2" lru-cache = "0.1.2"
md5 = "0.7.0" md5 = "0.7.0"
nom = "4,<5" nom = "4,<5" # FIXME: update
openssh = { version = "0.11.3", features = ["process-mux"], default-features = false } openssh = { version = "0.11.3", features = ["process-mux"], default-features = false }
opentelemetry = "0.27.0" opentelemetry = "0.27.0"
opentelemetry_sdk = { version = "0.27.0", features = ["rt-tokio"] }
opentelemetry-otlp = { version = "0.27.0", features = ["http-json", "reqwest-client", "reqwest-rustls"] }
opentelemetry-semantic-conventions = "0.27.0" opentelemetry-semantic-conventions = "0.27.0"
opentelemetry_sdk = "0.27.0" opentelemetry-stdout = "0.27.0"
opentelemetry-otlp = { version = "0.27.0", features = ["http-json", "reqwest-blocking-client", "reqwest-rustls"] }
regex = "1.11.1" regex = "1.11.1"
rustls-pemfile = "2.2.0" rustls-pemfile = "2.2.0"
separator = "0.4.1" separator = "0.4.1"

View file

@ -6,6 +6,7 @@ pub struct Acl {
} }
impl Acl { impl Acl {
#[must_use]
pub fn new(repos: Vec<String>, mut trusted_users: Option<Vec<String>>) -> Acl { pub fn new(repos: Vec<String>, mut trusted_users: Option<Vec<String>>) -> Acl {
if let Some(ref mut users) = trusted_users { if let Some(ref mut users) = trusted_users {
users.iter_mut().map(|x| *x = x.to_lowercase()).last(); users.iter_mut().map(|x| *x = x.to_lowercase()).last();
@ -17,10 +18,12 @@ impl Acl {
} }
} }
#[must_use]
pub fn is_repo_eligible(&self, name: &str) -> bool { pub fn is_repo_eligible(&self, name: &str) -> bool {
self.repos.contains(&name.to_lowercase()) self.repos.contains(&name.to_lowercase())
} }
#[must_use]
pub fn build_job_architectures_for_user_repo(&self, user: &str, repo: &str) -> Vec<System> { pub fn build_job_architectures_for_user_repo(&self, user: &str, repo: &str) -> Vec<System> {
if self.can_build_unrestricted(user, repo) { if self.can_build_unrestricted(user, repo) {
vec![ vec![
@ -35,6 +38,7 @@ impl Acl {
} }
} }
#[must_use]
pub fn build_job_destinations_for_user_repo( pub fn build_job_destinations_for_user_repo(
&self, &self,
user: &str, user: &str,
@ -42,10 +46,11 @@ impl Acl {
) -> Vec<(Option<String>, Option<String>)> { ) -> Vec<(Option<String>, Option<String>)> {
self.build_job_architectures_for_user_repo(user, repo) self.build_job_architectures_for_user_repo(user, repo)
.iter() .iter()
.map(|system| system.as_build_destination()) .map(super::systems::System::as_build_destination)
.collect() .collect()
} }
#[must_use]
pub fn can_build_unrestricted(&self, user: &str, repo: &str) -> bool { pub fn can_build_unrestricted(&self, user: &str, repo: &str) -> bool {
if let Some(ref users) = self.trusted_users { if let Some(ref users) = self.trusted_users {
if repo.to_lowercase() == "nixos/nixpkgs" { if repo.to_lowercase() == "nixos/nixpkgs" {

View file

@ -82,10 +82,12 @@ fn child_wait<T: Send + 'static>(
} }
impl AsyncCmd { impl AsyncCmd {
#[must_use]
pub fn new(cmd: Command) -> AsyncCmd { pub fn new(cmd: Command) -> AsyncCmd {
AsyncCmd { command: cmd } AsyncCmd { command: cmd }
} }
#[must_use]
pub fn spawn(mut self) -> SpawnedAsyncCmd { pub fn spawn(mut self) -> SpawnedAsyncCmd {
let mut child = self let mut child = self
.command .command
@ -122,7 +124,7 @@ impl AsyncCmd {
child_wait(WaitTarget::Child, monitor_tx, child), child_wait(WaitTarget::Child, monitor_tx, child),
); );
let head_waiter = thread::spawn(move || block_on_waiters(monitor_rx, waiters)); let head_waiter = thread::spawn(move || block_on_waiters(&monitor_rx, waiters));
SpawnedAsyncCmd { SpawnedAsyncCmd {
waiter: head_waiter, waiter: head_waiter,
@ -152,12 +154,12 @@ impl SpawnedAsyncCmd {
// FIXME: remove with rust/cargo update // FIXME: remove with rust/cargo update
#[allow(clippy::cognitive_complexity)] #[allow(clippy::cognitive_complexity)]
fn block_on_waiters( fn block_on_waiters(
monitor_rx: mpsc::Receiver<(WaitTarget, WaitResult<()>)>, monitor_rx: &mpsc::Receiver<(WaitTarget, WaitResult<()>)>,
mut waiters: HashMap<WaitTarget, thread::JoinHandle<()>>, mut waiters: HashMap<WaitTarget, thread::JoinHandle<()>>,
) -> Option<Result<ExitStatus, io::Error>> { ) -> Option<Result<ExitStatus, io::Error>> {
let mut status = None; let mut status = None;
for (id, interior_result) in monitor_rx.iter() { for (id, interior_result) in monitor_rx {
match waiters.remove(&id) { match waiters.remove(&id) {
Some(handle) => { Some(handle) => {
info!("Received notice that {:?} finished", id); info!("Received notice that {:?} finished", id);
@ -262,7 +264,7 @@ mod tests {
let mut spawned = acmd.spawn(); let mut spawned = acmd.spawn();
let lines: Vec<String> = spawned.lines().collect(); let lines: Vec<String> = spawned.lines().collect();
assert_eq!(lines.len(), 200000); assert_eq!(lines.len(), 200_000);
let thread_result = spawned.wait(); let thread_result = spawned.wait();
let exit_status = thread_result.expect("Thread should exit correctly"); let exit_status = thread_result.expect("Thread should exit correctly");
assert!(exit_status.success()); assert!(exit_status.success());

View file

@ -1,7 +1,6 @@
use std::env; use std::env;
use std::error::Error; use std::error::Error;
use futures::executor::block_on;
use lapin::message::Delivery; use lapin::message::Delivery;
use lapin::BasicProperties; use lapin::BasicProperties;
@ -12,14 +11,15 @@ use ofborg::message::{buildjob, Change, Repo};
use ofborg::notifyworker::NotificationReceiver; use ofborg::notifyworker::NotificationReceiver;
use ofborg::worker; use ofborg::worker;
fn main() -> Result<(), Box<dyn Error>> { #[tokio::main]
async fn main() -> Result<(), Box<dyn Error>> {
ofborg::setup_log(); ofborg::setup_log();
let arg = env::args().nth(1).expect("usage: build-faker <config>"); let arg = env::args().nth(1).expect("usage: build-faker <config>");
let cfg = config::load(arg.as_ref()); let cfg = config::load(arg.as_ref());
let conn = easylapin::from_config(&cfg.rabbitmq)?; let conn = easylapin::from_config(&cfg.rabbitmq).await?;
let mut chan = block_on(conn.create_channel())?; let mut chan = conn.create_channel().await?;
let repo_msg = Repo { let repo_msg = Repo {
clone_url: "https://github.com/nixos/ofborg.git".to_owned(), clone_url: "https://github.com/nixos/ofborg.git".to_owned(),
@ -60,10 +60,11 @@ fn main() -> Result<(), Box<dyn Error>> {
for _i in 1..2 { for _i in 1..2 {
recv.tell(worker::publish_serde_action( recv.tell(worker::publish_serde_action(
None, &None,
Some("build-inputs-x86_64-darwin".to_owned()), &Some("build-inputs-x86_64-darwin".to_owned()),
&msg, &msg,
)); ))
.await;
} }
} }

View file

@ -2,9 +2,7 @@ use std::env;
use std::error::Error; use std::error::Error;
use std::path::Path; use std::path::Path;
use futures::executor::block_on;
use futures_util::future; use futures_util::future;
use tokio::spawn;
use tokio::task::JoinHandle; use tokio::task::JoinHandle;
use tracing::{info, warn}; use tracing::{info, warn};
@ -14,7 +12,8 @@ use ofborg::{checkout, config, tasks};
// FIXME: remove with rust/cargo update // FIXME: remove with rust/cargo update
#[allow(clippy::cognitive_complexity)] #[allow(clippy::cognitive_complexity)]
fn main() -> Result<(), Box<dyn Error>> { #[tokio::main]
async fn main() -> Result<(), Box<dyn Error>> {
ofborg::setup_log(); ofborg::setup_log();
let arg = env::args().nth(1).expect("usage: builder <config>"); let arg = env::args().nth(1).expect("usage: builder <config>");
@ -30,27 +29,27 @@ fn main() -> Result<(), Box<dyn Error>> {
panic!(); panic!();
}; };
let conn = easylapin::from_config(&cfg.rabbitmq)?; let conn = easylapin::from_config(&cfg.rabbitmq).await?;
let mut handles = Vec::new(); let mut handles = Vec::new();
for system in &cfg.nix.system { for system in &cfg.nix.system {
let handle_ext = self::create_handle(&conn, &cfg, system.to_string())?; let handle_ext = self::create_handle(&conn, &cfg, system.to_string());
handles.push(handle_ext); handles.push(handle_ext);
} }
block_on(future::join_all(handles)); future::join_all(handles).await;
drop(conn); // Close connection. drop(conn); // Close connection.
info!("Closed the session... EOF"); info!("Closed the session... EOF");
Ok(()) Ok(())
} }
fn create_handle( async fn create_handle(
conn: &lapin::Connection, conn: &lapin::Connection,
cfg: &config::Config, cfg: &config::Config,
system: String, system: String,
) -> Result<JoinHandle<()>, Box<dyn Error>> { ) -> Result<JoinHandle<Result<(), lapin::Error>>, Box<dyn Error>> {
let mut chan = block_on(conn.create_channel())?; let mut chan = conn.create_channel().await?;
let cloner = checkout::cached_cloner(Path::new(&cfg.checkout.root)); let cloner = checkout::cached_cloner(Path::new(&cfg.checkout.root));
let nix = cfg.nix().with_system(system.clone()); let nix = cfg.nix().with_system(system.clone());
@ -63,7 +62,8 @@ fn create_handle(
auto_delete: false, auto_delete: false,
no_wait: false, no_wait: false,
internal: false, internal: false,
})?; })
.await?;
let queue_name = if cfg.runner.build_all_jobs != Some(true) { let queue_name = if cfg.runner.build_all_jobs != Some(true) {
let queue_name = format!("build-inputs-{}", system); let queue_name = format!("build-inputs-{}", system);
@ -74,7 +74,8 @@ fn create_handle(
exclusive: false, exclusive: false,
auto_delete: false, auto_delete: false,
no_wait: false, no_wait: false,
})?; })
.await?;
queue_name queue_name
} else { } else {
warn!("Building all jobs, please don't use this unless you're"); warn!("Building all jobs, please don't use this unless you're");
@ -87,7 +88,8 @@ fn create_handle(
exclusive: true, exclusive: true,
auto_delete: true, auto_delete: true,
no_wait: false, no_wait: false,
})?; })
.await?;
queue_name queue_name
}; };
@ -96,9 +98,12 @@ fn create_handle(
exchange: "build-jobs".to_owned(), exchange: "build-jobs".to_owned(),
routing_key: None, routing_key: None,
no_wait: false, no_wait: false,
})?; })
.await?;
let handle = easylapin::NotifyChannel(chan).consume( info!("Fetching jobs from {}", &queue_name);
let fut = easylapin::NotifyChannel(chan).consume(
tasks::build::BuildWorker::new(cloner, nix, system, cfg.runner.identity.clone()), tasks::build::BuildWorker::new(cloner, nix, system, cfg.runner.identity.clone()),
easyamqp::ConsumeConfig { easyamqp::ConsumeConfig {
queue: queue_name.clone(), queue: queue_name.clone(),
@ -108,8 +113,7 @@ fn create_handle(
no_wait: false, no_wait: false,
exclusive: false, exclusive: false,
}, },
)?; );
info!("Fetching jobs from {}", &queue_name); Ok(tokio::spawn(fut))
Ok(spawn(handle))
} }

View file

@ -1,7 +1,6 @@
use std::env; use std::env;
use std::error::Error; use std::error::Error;
use futures::executor::block_on;
use tracing::info; use tracing::info;
use ofborg::config; use ofborg::config;
@ -9,7 +8,8 @@ use ofborg::easyamqp::{self, ChannelExt, ConsumerExt};
use ofborg::easylapin; use ofborg::easylapin;
use ofborg::tasks; use ofborg::tasks;
fn main() -> Result<(), Box<dyn Error>> { #[tokio::main]
async fn main() -> Result<(), Box<dyn Error>> {
ofborg::setup_log(); ofborg::setup_log();
let arg = env::args() let arg = env::args()
@ -17,8 +17,8 @@ fn main() -> Result<(), Box<dyn Error>> {
.expect("usage: evaluation-filter <config>"); .expect("usage: evaluation-filter <config>");
let cfg = config::load(arg.as_ref()); let cfg = config::load(arg.as_ref());
let conn = easylapin::from_config(&cfg.rabbitmq)?; let conn = easylapin::from_config(&cfg.rabbitmq).await?;
let mut chan = block_on(conn.create_channel())?; let mut chan = conn.create_channel().await?;
chan.declare_exchange(easyamqp::ExchangeConfig { chan.declare_exchange(easyamqp::ExchangeConfig {
exchange: "github-events".to_owned(), exchange: "github-events".to_owned(),
@ -28,7 +28,8 @@ fn main() -> Result<(), Box<dyn Error>> {
auto_delete: false, auto_delete: false,
no_wait: false, no_wait: false,
internal: false, internal: false,
})?; })
.await?;
chan.declare_queue(easyamqp::QueueConfig { chan.declare_queue(easyamqp::QueueConfig {
queue: "mass-rebuild-check-jobs".to_owned(), queue: "mass-rebuild-check-jobs".to_owned(),
@ -37,7 +38,8 @@ fn main() -> Result<(), Box<dyn Error>> {
exclusive: false, exclusive: false,
auto_delete: false, auto_delete: false,
no_wait: false, no_wait: false,
})?; })
.await?;
let queue_name = String::from("mass-rebuild-check-inputs"); let queue_name = String::from("mass-rebuild-check-inputs");
chan.declare_queue(easyamqp::QueueConfig { chan.declare_queue(easyamqp::QueueConfig {
@ -47,16 +49,20 @@ fn main() -> Result<(), Box<dyn Error>> {
exclusive: false, exclusive: false,
auto_delete: false, auto_delete: false,
no_wait: false, no_wait: false,
})?; })
.await?;
chan.bind_queue(easyamqp::BindQueueConfig { chan.bind_queue(easyamqp::BindQueueConfig {
queue: queue_name.clone(), queue: queue_name.clone(),
exchange: "github-events".to_owned(), exchange: "github-events".to_owned(),
routing_key: Some("pull_request.nixos/nixpkgs".to_owned()), routing_key: Some("pull_request.nixos/nixpkgs".to_owned()),
no_wait: false, no_wait: false,
})?; })
.await?;
let handle = easylapin::WorkerChannel(chan).consume( info!("Fetching jobs from {}", &queue_name);
easylapin::WorkerChannel(chan)
.consume(
tasks::evaluationfilter::EvaluationFilterWorker::new(cfg.acl()), tasks::evaluationfilter::EvaluationFilterWorker::new(cfg.acl()),
easyamqp::ConsumeConfig { easyamqp::ConsumeConfig {
queue: queue_name.clone(), queue: queue_name.clone(),
@ -66,10 +72,8 @@ fn main() -> Result<(), Box<dyn Error>> {
no_wait: false, no_wait: false,
exclusive: false, exclusive: false,
}, },
)?; )
.await?;
info!("Fetching jobs from {}", &queue_name);
block_on(handle);
drop(conn); // Close connection. drop(conn); // Close connection.
info!("Closed the session... EOF"); info!("Closed the session... EOF");

View file

@ -4,7 +4,6 @@
use std::env; use std::env;
use std::error::Error; use std::error::Error;
use futures::executor::block_on;
use futures::{pin_mut, StreamExt}; use futures::{pin_mut, StreamExt};
use lapin::options::BasicPublishOptions; use lapin::options::BasicPublishOptions;
use lapin::BasicProperties; use lapin::BasicProperties;
@ -16,7 +15,8 @@ use ofborg::config;
use ofborg::easyamqp::{self, ChannelExt}; use ofborg::easyamqp::{self, ChannelExt};
use ofborg::easylapin; use ofborg::easylapin;
fn main() -> Result<(), Box<dyn Error>> { #[tokio::main]
async fn main() -> Result<(), Box<dyn Error>> {
ofborg::setup_log(); ofborg::setup_log();
let arg = env::args() let arg = env::args()
@ -24,8 +24,8 @@ fn main() -> Result<(), Box<dyn Error>> {
.expect("usage: gerrit-events-streamer <config>"); .expect("usage: gerrit-events-streamer <config>");
let cfg = config::load(arg.as_ref()); let cfg = config::load(arg.as_ref());
let conn = easylapin::from_config(&cfg.rabbitmq)?; let conn = easylapin::from_config(&cfg.rabbitmq).await?;
let mut chan = block_on(conn.create_channel())?; let mut chan = conn.create_channel().await?;
let exchange_name = "gerrit-events"; let exchange_name = "gerrit-events";
@ -37,7 +37,8 @@ fn main() -> Result<(), Box<dyn Error>> {
auto_delete: false, auto_delete: false,
no_wait: false, no_wait: false,
internal: false, internal: false,
})?; })
.await?;
info!("Publishing events from Gerrit into {}", &exchange_name); info!("Publishing events from Gerrit into {}", &exchange_name);
@ -47,11 +48,11 @@ fn main() -> Result<(), Box<dyn Error>> {
let mut gerrit_api = GerritSSHApi::new( let mut gerrit_api = GerritSSHApi::new(
gerrit_cfg.ssh_private_key_file, gerrit_cfg.ssh_private_key_file,
&format!("ssh://{}:{}", gerrit_cfg.instance_uri, gerrit_cfg.ssh_port), &format!("ssh://{}:{}", gerrit_cfg.instance_uri, gerrit_cfg.ssh_port),
); )
.await;
let routing_key = "abc"; let routing_key = "abc";
block_on(async {
let event_stream = gerrit_api.stream_events().await.unwrap(); let event_stream = gerrit_api.stream_events().await.unwrap();
pin_mut!(event_stream); pin_mut!(event_stream);
loop { loop {
@ -93,11 +94,4 @@ fn main() -> Result<(), Box<dyn Error>> {
} }
} }
} }
});
//task::block_on(handle);
drop(conn); // Close connection.
info!("Closed the session... EOF");
Ok(())
} }

View file

@ -3,13 +3,14 @@
use std::env; use std::env;
use std::error::Error; use std::error::Error;
use futures::{executor::block_on, pin_mut, StreamExt}; use futures::{pin_mut, StreamExt};
use ofborg::vcs::gerrit::ssh::GerritSSHApi; use ofborg::vcs::gerrit::ssh::GerritSSHApi;
use tracing::info; use tracing::info;
use ofborg::config; use ofborg::config;
fn main() -> Result<(), Box<dyn Error>> { #[tokio::main]
async fn main() -> Result<(), Box<dyn Error>> {
ofborg::setup_log(); ofborg::setup_log();
let arg = env::args() let arg = env::args()
@ -22,9 +23,8 @@ fn main() -> Result<(), Box<dyn Error>> {
.expect("Gerrit event streaming requires a Gerrit configuration"); .expect("Gerrit event streaming requires a Gerrit configuration");
let gerrit_ssh_uri = format!("ssh://{}:{}", gerrit_cfg.instance_uri, gerrit_cfg.ssh_port); let gerrit_ssh_uri = format!("ssh://{}:{}", gerrit_cfg.instance_uri, gerrit_cfg.ssh_port);
info!("Listening events from Gerrit on {}", gerrit_ssh_uri); info!("Listening events from Gerrit on {}", gerrit_ssh_uri);
let mut gerrit_api = GerritSSHApi::new(gerrit_cfg.ssh_private_key_file, &gerrit_ssh_uri); let mut gerrit_api = GerritSSHApi::new(gerrit_cfg.ssh_private_key_file, &gerrit_ssh_uri).await;
block_on(async {
let event_stream = gerrit_api.stream_events().await.unwrap(); let event_stream = gerrit_api.stream_events().await.unwrap();
pin_mut!(event_stream); pin_mut!(event_stream);
loop { loop {
@ -34,7 +34,4 @@ fn main() -> Result<(), Box<dyn Error>> {
println!("{:#?}", event); println!("{:#?}", event);
} }
} }
});
Ok(())
} }

View file

@ -2,7 +2,6 @@ use std::env;
use std::error::Error; use std::error::Error;
use std::path::PathBuf; use std::path::PathBuf;
use futures::executor::block_on;
use tracing::info; use tracing::info;
use ofborg::config; use ofborg::config;
@ -10,7 +9,8 @@ use ofborg::easyamqp::{self, ChannelExt, ConsumerExt};
use ofborg::easylapin; use ofborg::easylapin;
use ofborg::tasks; use ofborg::tasks;
fn main() -> Result<(), Box<dyn Error>> { #[tokio::main]
async fn main() -> Result<(), Box<dyn Error>> {
ofborg::setup_log(); ofborg::setup_log();
let arg = env::args() let arg = env::args()
@ -18,8 +18,8 @@ fn main() -> Result<(), Box<dyn Error>> {
.expect("usage: log-message-collector <config>"); .expect("usage: log-message-collector <config>");
let cfg = config::load(arg.as_ref()); let cfg = config::load(arg.as_ref());
let conn = easylapin::from_config(&cfg.rabbitmq)?; let conn = easylapin::from_config(&cfg.rabbitmq).await?;
let mut chan = block_on(conn.create_channel())?; let mut chan = conn.create_channel().await?;
chan.declare_exchange(easyamqp::ExchangeConfig { chan.declare_exchange(easyamqp::ExchangeConfig {
exchange: "logs".to_owned(), exchange: "logs".to_owned(),
@ -29,7 +29,8 @@ fn main() -> Result<(), Box<dyn Error>> {
auto_delete: false, auto_delete: false,
no_wait: false, no_wait: false,
internal: false, internal: false,
})?; })
.await?;
let queue_name = "".to_owned(); let queue_name = "".to_owned();
chan.declare_queue(easyamqp::QueueConfig { chan.declare_queue(easyamqp::QueueConfig {
@ -39,17 +40,21 @@ fn main() -> Result<(), Box<dyn Error>> {
exclusive: true, exclusive: true,
auto_delete: true, auto_delete: true,
no_wait: false, no_wait: false,
})?; })
.await?;
chan.bind_queue(easyamqp::BindQueueConfig { chan.bind_queue(easyamqp::BindQueueConfig {
queue: queue_name.clone(), queue: queue_name.clone(),
exchange: "logs".to_owned(), exchange: "logs".to_owned(),
routing_key: Some("*.*".to_owned()), routing_key: Some("*.*".to_owned()),
no_wait: false, no_wait: false,
})?; })
.await?;
info!("Fetching jobs from {}", &queue_name);
// Regular channel, we want prefetching here. // Regular channel, we want prefetching here.
let handle = chan.consume( chan.consume(
tasks::log_message_collector::LogMessageCollector::new( tasks::log_message_collector::LogMessageCollector::new(
PathBuf::from(cfg.log_storage.clone().unwrap().path), PathBuf::from(cfg.log_storage.clone().unwrap().path),
100, 100,
@ -62,10 +67,8 @@ fn main() -> Result<(), Box<dyn Error>> {
no_wait: false, no_wait: false,
exclusive: false, exclusive: false,
}, },
)?; )
.await?;
info!("Fetching jobs from {}", &queue_name);
block_on(handle);
drop(conn); // Close connection. drop(conn); // Close connection.
info!("Closed the session... EOF"); info!("Closed the session... EOF");

View file

@ -3,7 +3,6 @@ use std::error::Error;
use std::path::Path; use std::path::Path;
use std::process; use std::process;
use futures::executor::block_on;
use ofborg::config::VCSConfig; use ofborg::config::VCSConfig;
use ofborg::tasks::evaluate::SupportedVCS; use ofborg::tasks::evaluate::SupportedVCS;
use tracing::{error, info}; use tracing::{error, info};
@ -17,7 +16,8 @@ use ofborg::tasks;
// FIXME: remove with rust/cargo update // FIXME: remove with rust/cargo update
#[allow(clippy::cognitive_complexity)] #[allow(clippy::cognitive_complexity)]
fn main() -> Result<(), Box<dyn Error>> { #[tokio::main]
async fn main() -> Result<(), Box<dyn Error>> {
ofborg::setup_log(); ofborg::setup_log();
let arg = env::args().nth(1).expect("usage: mass-rebuilder <config>"); let arg = env::args().nth(1).expect("usage: mass-rebuilder <config>");
@ -34,14 +34,14 @@ fn main() -> Result<(), Box<dyn Error>> {
process::exit(1); process::exit(1);
}; };
let conn = easylapin::from_config(&cfg.rabbitmq)?; let conn = easylapin::from_config(&cfg.rabbitmq).await?;
let mut chan = block_on(conn.create_channel())?; let mut chan = conn.create_channel().await?;
let root = Path::new(&cfg.checkout.root); let root = Path::new(&cfg.checkout.root);
let cloner = checkout::cached_cloner(&root.join(cfg.runner.instance.to_string())); let cloner = checkout::cached_cloner(&root.join(cfg.runner.instance.to_string()));
let nix = cfg.nix(); let nix = cfg.nix();
let events = stats::RabbitMq::from_lapin(&cfg.whoami(), block_on(conn.create_channel())?); let events = stats::RabbitMq::from_lapin(&cfg.whoami(), conn.create_channel().await?);
let queue_name = String::from("mass-rebuild-check-jobs"); let queue_name = String::from("mass-rebuild-check-jobs");
chan.declare_queue(easyamqp::QueueConfig { chan.declare_queue(easyamqp::QueueConfig {
@ -51,13 +51,17 @@ fn main() -> Result<(), Box<dyn Error>> {
exclusive: false, exclusive: false,
auto_delete: false, auto_delete: false,
no_wait: false, no_wait: false,
})?; })
.await?;
let vcs_data = match cfg.vcs { let vcs_data = match cfg.vcs {
VCSConfig::Gerrit => SupportedVCS::Gerrit, VCSConfig::Gerrit => SupportedVCS::Gerrit,
}; };
let handle = easylapin::WorkerChannel(chan).consume( info!("Fetching jobs from {}", queue_name);
easylapin::WorkerChannel(chan)
.consume(
tasks::evaluate::EvaluationWorker::new( tasks::evaluate::EvaluationWorker::new(
cloner, cloner,
&nix, &nix,
@ -74,10 +78,8 @@ fn main() -> Result<(), Box<dyn Error>> {
no_wait: false, no_wait: false,
exclusive: false, exclusive: false,
}, },
)?; )
.await?;
info!("Fetching jobs from {}", queue_name);
block_on(handle);
drop(conn); // Close connection. drop(conn); // Close connection.
info!("Closed the session... EOF"); info!("Closed the session... EOF");

View file

@ -1,7 +1,6 @@
use std::error::Error; use std::error::Error;
use std::io::Read; use std::io::Read;
use futures::executor::block_on;
use lapin::message::Delivery; use lapin::message::Delivery;
use lapin::BasicProperties; use lapin::BasicProperties;
@ -102,7 +101,7 @@ impl<'a> Publisher<'a> {
} }
} }
fn publish_serde_action<T>( async fn publish_serde_action<T>(
&mut self, &mut self,
exchange: Option<String>, exchange: Option<String>,
routing_key: Option<String>, routing_key: Option<String>,
@ -111,18 +110,20 @@ impl<'a> Publisher<'a> {
T: Serialize + ?Sized, T: Serialize + ?Sized,
{ {
self.recv self.recv
.tell(worker::publish_serde_action(exchange, routing_key, msg)); .tell(worker::publish_serde_action(&exchange, &routing_key, msg))
.await;
} }
} }
fn main() -> Result<(), Box<dyn Error>> { #[tokio::main]
async fn main() -> Result<(), Box<dyn Error>> {
ofborg::setup_log(); ofborg::setup_log();
let args = Cli::parse(); let args = Cli::parse();
let cfg = config::load(args.config.as_ref()); let cfg = config::load(args.config.as_ref());
let conn = easylapin::from_config(&cfg.rabbitmq)?; let conn = easylapin::from_config(&cfg.rabbitmq).await?;
let mut chan = block_on(conn.create_channel())?; let mut chan = conn.create_channel().await?;
match args.command { match args.command {
Commands::Pastebin { action } => match action { Commands::Pastebin { action } => match action {
@ -133,11 +134,13 @@ fn main() -> Result<(), Box<dyn Error>> {
.read_to_string(&mut contents) .read_to_string(&mut contents)
.expect("Failed to read pastebin contents"); .expect("Failed to read pastebin contents");
publisher.publish_serde_action( publisher
.publish_serde_action(
None, None,
Some("pastebin-log".to_owned()), Some("pastebin-log".to_owned()),
&Pastebin { title, contents }, &Pastebin { title, contents },
); )
.await;
} }
}, },

View file

@ -1,7 +1,6 @@
use std::env; use std::env;
use std::error::Error; use std::error::Error;
use futures::executor::block_on;
use ofborg::config; use ofborg::config;
use ofborg::easyamqp; use ofborg::easyamqp;
use ofborg::easyamqp::ChannelExt; use ofborg::easyamqp::ChannelExt;
@ -10,14 +9,15 @@ use ofborg::easylapin;
use ofborg::tasks; use ofborg::tasks;
use tracing::info; use tracing::info;
fn main() -> Result<(), Box<dyn Error>> { #[tokio::main]
async fn main() -> Result<(), Box<dyn Error>> {
ofborg::setup_log(); ofborg::setup_log();
let arg = env::args().nth(1).expect("usage: pastebin-worker <config>"); let arg = env::args().nth(1).expect("usage: pastebin-worker <config>");
let cfg = config::load(arg.as_ref()); let cfg = config::load(arg.as_ref());
let conn = easylapin::from_config(&cfg.rabbitmq)?; let conn = easylapin::from_config(&cfg.rabbitmq).await?;
let mut chan = block_on(conn.create_channel())?; let mut chan = conn.create_channel().await?;
let queue_name = "pastebin-log".to_owned(); let queue_name = "pastebin-log".to_owned();
@ -29,7 +29,8 @@ fn main() -> Result<(), Box<dyn Error>> {
auto_delete: false, auto_delete: false,
no_wait: false, no_wait: false,
internal: false, internal: false,
})?; })
.await?;
chan.declare_queue(easyamqp::QueueConfig { chan.declare_queue(easyamqp::QueueConfig {
queue: queue_name.clone(), queue: queue_name.clone(),
@ -38,16 +39,21 @@ fn main() -> Result<(), Box<dyn Error>> {
exclusive: false, exclusive: false,
auto_delete: false, auto_delete: false,
no_wait: false, no_wait: false,
})?; })
.await?;
chan.bind_queue(easyamqp::BindQueueConfig { chan.bind_queue(easyamqp::BindQueueConfig {
queue: queue_name.clone(), queue: queue_name.clone(),
exchange: "pastebin-log".to_owned(), exchange: "pastebin-log".to_owned(),
routing_key: None, routing_key: None,
no_wait: false, no_wait: false,
})?; })
.await?;
let handle = easylapin::WorkerChannel(chan).consume( info!("Fetching jobs from {}", &queue_name);
easylapin::WorkerChannel(chan)
.consume(
tasks::pastebin_collector::PastebinCollector::new( tasks::pastebin_collector::PastebinCollector::new(
cfg.pastebin.clone().root, cfg.pastebin.clone().root,
cfg.pastebin.clone().db, cfg.pastebin.clone().db,
@ -60,10 +66,8 @@ fn main() -> Result<(), Box<dyn Error>> {
no_wait: false, no_wait: false,
exclusive: false, exclusive: false,
}, },
)?; )
.await?;
info!("Fetching jobs from {}", &queue_name);
block_on(handle);
drop(conn); // Close connection. drop(conn); // Close connection.
info!("Closed the session... EOF"); info!("Closed the session... EOF");

View file

@ -6,7 +6,6 @@
use std::env; use std::env;
use std::error::Error; use std::error::Error;
use futures::executor::block_on;
use ofborg::config; use ofborg::config;
use ofborg::easyamqp; use ofborg::easyamqp;
use ofborg::easyamqp::ChannelExt; use ofborg::easyamqp::ChannelExt;
@ -15,7 +14,8 @@ use ofborg::easylapin;
use ofborg::tasks; use ofborg::tasks;
use tracing::info; use tracing::info;
fn main() -> Result<(), Box<dyn Error>> { #[tokio::main]
async fn main() -> Result<(), Box<dyn Error>> {
ofborg::setup_log(); ofborg::setup_log();
let arg = env::args() let arg = env::args()
@ -23,8 +23,8 @@ fn main() -> Result<(), Box<dyn Error>> {
.expect("usage: statcheck-worker <config>"); .expect("usage: statcheck-worker <config>");
let cfg = config::load(arg.as_ref()); let cfg = config::load(arg.as_ref());
let conn = easylapin::from_config(&cfg.rabbitmq)?; let conn = easylapin::from_config(&cfg.rabbitmq).await?;
let mut chan = block_on(conn.create_channel())?; let mut chan = conn.create_channel().await?;
// an RPC queue for verbs // an RPC queue for verbs
let api_queue_name = "statcheck-api".to_owned(); let api_queue_name = "statcheck-api".to_owned();
@ -39,7 +39,8 @@ fn main() -> Result<(), Box<dyn Error>> {
auto_delete: false, auto_delete: false,
no_wait: false, no_wait: false,
internal: false, internal: false,
})?; })
.await?;
chan.declare_queue(easyamqp::QueueConfig { chan.declare_queue(easyamqp::QueueConfig {
queue: api_queue_name.clone(), queue: api_queue_name.clone(),
@ -48,16 +49,22 @@ fn main() -> Result<(), Box<dyn Error>> {
exclusive: false, exclusive: false,
auto_delete: false, auto_delete: false,
no_wait: false, no_wait: false,
})?; })
.await?;
chan.bind_queue(easyamqp::BindQueueConfig { chan.bind_queue(easyamqp::BindQueueConfig {
queue: api_queue_name.clone(), queue: api_queue_name.clone(),
exchange: api_queue_name.clone(), exchange: api_queue_name.clone(),
routing_key: None, routing_key: None,
no_wait: false, no_wait: false,
})?; })
.await?;
let handle = easylapin::WorkerChannel(chan).consume( info!("Waiting for API calls on {}", api_queue_name);
info!("Notifying of new changes on {}", event_queue_name);
easylapin::WorkerChannel(chan)
.consume(
tasks::status_check_collector::StatusCheckCollector::new(cfg.statcheck.clone().db), tasks::status_check_collector::StatusCheckCollector::new(cfg.statcheck.clone().db),
easyamqp::ConsumeConfig { easyamqp::ConsumeConfig {
queue: api_queue_name.clone(), queue: api_queue_name.clone(),
@ -67,11 +74,8 @@ fn main() -> Result<(), Box<dyn Error>> {
no_wait: false, no_wait: false,
exclusive: false, exclusive: false,
}, },
)?; )
.await?;
info!("Waiting for API calls on {}", api_queue_name);
info!("Notifying of new changes on {}", event_queue_name);
block_on(handle);
drop(conn); // Close connection. drop(conn); // Close connection.
info!("Closed the session... EOF"); info!("Closed the session... EOF");

View file

@ -1,26 +1,24 @@
use std::env; use std::env;
use std::error::Error; use std::error::Error;
use std::thread;
use axum::routing::get; use axum::routing::get;
use futures::executor::block_on;
use axum::Router; use axum::Router;
use tokio::runtime::Runtime;
use tracing::info; use tracing::info;
use ofborg::easyamqp::{ChannelExt, ConsumerExt}; use ofborg::easyamqp::{ChannelExt, ConsumerExt};
use ofborg::{config, easyamqp, easylapin, stats, tasks}; use ofborg::{config, easyamqp, easylapin, stats, tasks};
fn main() -> Result<(), Box<dyn Error>> { #[tokio::main]
async fn main() -> Result<(), Box<dyn Error>> {
ofborg::setup_log(); ofborg::setup_log();
let arg = env::args().nth(1).expect("usage: stats <config>"); let arg = env::args().nth(1).expect("usage: stats <config>");
let cfg = config::load(arg.as_ref()); let cfg = config::load(arg.as_ref());
let conn = easylapin::from_config(&cfg.rabbitmq)?; let conn = easylapin::from_config(&cfg.rabbitmq).await?;
let mut chan = block_on(conn.create_channel())?; let mut chan = conn.create_channel().await?;
let events = stats::RabbitMq::from_lapin(&cfg.whoami(), block_on(conn.create_channel())?); let events = stats::RabbitMq::from_lapin(&cfg.whoami(), conn.create_channel().await?);
let metrics = stats::MetricCollector::new(); let metrics = stats::MetricCollector::new();
let collector = tasks::statscollector::StatCollectorWorker::new(events, metrics.clone()); let collector = tasks::statscollector::StatCollectorWorker::new(events, metrics.clone());
@ -33,7 +31,8 @@ fn main() -> Result<(), Box<dyn Error>> {
auto_delete: false, auto_delete: false,
no_wait: false, no_wait: false,
internal: false, internal: false,
})?; })
.await?;
let queue_name = String::from("stats-events"); let queue_name = String::from("stats-events");
chan.declare_queue(easyamqp::QueueConfig { chan.declare_queue(easyamqp::QueueConfig {
@ -43,16 +42,30 @@ fn main() -> Result<(), Box<dyn Error>> {
exclusive: false, exclusive: false,
auto_delete: false, auto_delete: false,
no_wait: false, no_wait: false,
})?; })
.await?;
chan.bind_queue(easyamqp::BindQueueConfig { chan.bind_queue(easyamqp::BindQueueConfig {
queue: queue_name.clone(), queue: queue_name.clone(),
exchange: "stats".to_owned(), exchange: "stats".to_owned(),
routing_key: None, routing_key: None,
no_wait: false, no_wait: false,
})?; })
.await?;
let handle = chan.consume( tokio::spawn(async {
let addr = "0.0.0.0:9898";
info!("listening addr {:?}", addr);
let app = Router::new().route(
"/metrics",
get(|| async move { metrics.prometheus_output() }),
);
let listener = tokio::net::TcpListener::bind(addr).await.unwrap();
axum::serve(listener, app).await.unwrap()
});
info!("Fetching jobs from {}", &queue_name);
chan.consume(
collector, collector,
easyamqp::ConsumeConfig { easyamqp::ConsumeConfig {
queue: "stats-events".to_owned(), queue: "stats-events".to_owned(),
@ -62,22 +75,8 @@ fn main() -> Result<(), Box<dyn Error>> {
no_wait: false, no_wait: false,
exclusive: false, exclusive: false,
}, },
)?; )
.await?;
thread::spawn(|| {
Runtime::new().unwrap().block_on(async {
let addr = "0.0.0.0:9898";
info!("listening addr {:?}", addr);
let app = Router::new().route("/metrics", get(|| async move {
metrics.prometheus_output()
}));
let listener= tokio::net::TcpListener::bind(addr).await.unwrap();
axum::serve(listener, app).await.unwrap()
});
});
info!("Fetching jobs from {}", &queue_name);
block_on(handle);
drop(conn); // Close connection. drop(conn); // Close connection.
info!("Closed the session... EOF"); info!("Closed the session... EOF");

View file

@ -12,6 +12,7 @@ pub struct CachedCloner {
root: PathBuf, root: PathBuf,
} }
#[must_use]
pub fn cached_cloner(path: &Path) -> CachedCloner { pub fn cached_cloner(path: &Path) -> CachedCloner {
CachedCloner { CachedCloner {
root: path.to_path_buf(), root: path.to_path_buf(),
@ -31,6 +32,7 @@ pub struct CachedProjectCo {
} }
impl CachedCloner { impl CachedCloner {
#[must_use]
pub fn project(&self, name: &str, clone_url: String) -> CachedProject { pub fn project(&self, name: &str, clone_url: String) -> CachedProject {
// <root>/repo/<hash>/clone // <root>/repo/<hash>/clone
// <root>/repo/<hash>/clone.lock // <root>/repo/<hash>/clone.lock
@ -101,7 +103,7 @@ impl CachedProjectCo {
let result = Command::new("git") let result = Command::new("git")
.arg("fetch") .arg("fetch")
.arg("origin") .arg("origin")
.arg(format!("+refs/pull/{}/head:pr", pr_id)) .arg(format!("+refs/pull/{pr_id}/head:pr"))
.current_dir(self.clone_to()) .current_dir(self.clone_to())
.stdout(Stdio::null()) .stdout(Stdio::null())
.status()?; .status()?;
@ -162,7 +164,7 @@ impl CachedProjectCo {
let result = Command::new("git") let result = Command::new("git")
.arg("log") .arg("log")
.arg("--format=format:%s") .arg("--format=format:%s")
.arg(format!("HEAD..{}", commit)) .arg(format!("HEAD..{commit}"))
.current_dir(self.clone_to()) .current_dir(self.clone_to())
.output()?; .output()?;
@ -171,7 +173,7 @@ impl CachedProjectCo {
if result.status.success() { if result.status.success() {
Ok(String::from_utf8_lossy(&result.stdout) Ok(String::from_utf8_lossy(&result.stdout)
.lines() .lines()
.map(|l| l.to_owned()) .map(std::borrow::ToOwned::to_owned)
.collect()) .collect())
} else { } else {
Err(Error::new( Err(Error::new(
@ -187,7 +189,7 @@ impl CachedProjectCo {
let result = Command::new("git") let result = Command::new("git")
.arg("diff") .arg("diff")
.arg("--name-only") .arg("--name-only")
.arg(format!("HEAD...{}", commit)) .arg(format!("HEAD...{commit}"))
.current_dir(self.clone_to()) .current_dir(self.clone_to())
.output()?; .output()?;
@ -196,7 +198,7 @@ impl CachedProjectCo {
if result.status.success() { if result.status.success() {
Ok(String::from_utf8_lossy(&result.stdout) Ok(String::from_utf8_lossy(&result.stdout)
.lines() .lines()
.map(|l| l.to_owned()) .map(std::borrow::ToOwned::to_owned)
.collect()) .collect())
} else { } else {
Err(Error::new( Err(Error::new(
@ -278,8 +280,8 @@ mod tests {
.expect("building the test PR failed"); .expect("building the test PR failed");
let stderr = let stderr =
String::from_utf8(output.stderr).unwrap_or_else(|err| format!("warning: {}", err)); String::from_utf8(output.stderr).unwrap_or_else(|err| format!("warning: {err}"));
println!("{}", stderr); println!("{stderr}");
let hash = String::from_utf8(output.stdout).expect("Should just be a hash"); let hash = String::from_utf8(output.stdout).expect("Should just be a hash");
return hash.trim().to_owned(); return hash.trim().to_owned();

View file

@ -14,7 +14,7 @@ pub struct Lock {
impl Lock { impl Lock {
pub fn unlock(&mut self) { pub fn unlock(&mut self) {
self.lock = None self.lock = None;
} }
} }
@ -33,20 +33,19 @@ pub trait GitClonable {
warn!("Failed to create lock file {:?}: {}", self.lock_path(), e); warn!("Failed to create lock file {:?}: {}", self.lock_path(), e);
Err(e) Err(e)
} }
Ok(lock) => match lock.lock_exclusive() { Ok(lock) => {
Err(e) => { if let Err(e) = lock.lock_exclusive() {
warn!( warn!(
"Failed to get exclusive lock on file {:?}: {}", "Failed to get exclusive lock on file {:?}: {}",
self.lock_path(), self.lock_path(),
e e
); );
Err(e) Err(e)
} } else {
Ok(_) => {
debug!("Got lock on {:?}", self.lock_path()); debug!("Got lock on {:?}", self.lock_path());
Ok(Lock { lock: Some(lock) }) Ok(Lock { lock: Some(lock) })
} }
}, }
} }
} }

View file

@ -1,6 +1,7 @@
use nom::types::CompleteStr; use nom::types::CompleteStr;
use tracing::warn; use tracing::warn;
#[must_use]
pub fn parse(text: &str) -> Option<Vec<Instruction>> { pub fn parse(text: &str) -> Option<Vec<Instruction>> {
let instructions: Vec<Instruction> = text let instructions: Vec<Instruction> = text
.lines() .lines()

View file

@ -119,10 +119,12 @@ pub struct CheckoutConfig {
} }
impl Config { impl Config {
#[must_use]
pub fn whoami(&self) -> String { pub fn whoami(&self) -> String {
format!("{}-{}", self.runner.identity, self.nix.system.join(",")) format!("{}-{}", self.runner.identity, self.nix.system.join(","))
} }
#[must_use]
pub fn acl(&self) -> acl::Acl { pub fn acl(&self) -> acl::Acl {
let repos = self let repos = self
.runner .runner
@ -172,7 +174,7 @@ impl RabbitMqConfig {
.expect("Failed to expand the password-file configuration string"); .expect("Failed to expand the password-file configuration string");
password = std::fs::read_to_string(expanded_password_file.as_ref())?; password = std::fs::read_to_string(expanded_password_file.as_ref())?;
} else { } else {
password = "".to_owned(); password = String::new();
} }
let uri = format!( let uri = format!(
"{}://{}:{}@{}/{}", "{}://{}:{}@{}/{}",
@ -186,6 +188,7 @@ impl RabbitMqConfig {
} }
} }
#[must_use]
pub fn load(filename: &Path) -> Config { pub fn load(filename: &Path) -> Config {
let mut file = File::open(filename).unwrap(); let mut file = File::open(filename).unwrap();
let mut contents = String::new(); let mut contents = String::new();

View file

@ -1,5 +1,10 @@
use std::fmt::Debug;
use async_trait::async_trait;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
#[derive(Debug)]
#[allow(clippy::struct_excessive_bools)]
pub struct ConsumeConfig { pub struct ConsumeConfig {
/// Specifies the name of the queue to consume from. /// Specifies the name of the queue to consume from.
pub queue: String, pub queue: String,
@ -40,6 +45,7 @@ pub struct ConsumeConfig {
pub no_wait: bool, pub no_wait: bool,
} }
#[derive(Debug)]
pub struct BindQueueConfig { pub struct BindQueueConfig {
/// Specifies the name of the queue to bind. /// Specifies the name of the queue to bind.
/// ///
@ -83,6 +89,7 @@ pub struct BindQueueConfig {
pub no_wait: bool, pub no_wait: bool,
} }
#[derive(Debug)]
pub enum ExchangeType { pub enum ExchangeType {
Topic, Topic,
Headers, Headers,
@ -103,6 +110,8 @@ impl From<ExchangeType> for String {
} }
} }
#[derive(Debug)]
#[allow(clippy::struct_excessive_bools)]
pub struct ExchangeConfig { pub struct ExchangeConfig {
/// Exchange names starting with "amq." are reserved for /// Exchange names starting with "amq." are reserved for
/// pre-declared and standardised exchanges. The client MAY /// pre-declared and standardised exchanges. The client MAY
@ -183,6 +192,8 @@ pub struct ExchangeConfig {
pub no_wait: bool, pub no_wait: bool,
} }
#[derive(Debug)]
#[allow(clippy::struct_excessive_bools)]
pub struct QueueConfig { pub struct QueueConfig {
/// The queue name MAY be empty, in which case the server MUST /// The queue name MAY be empty, in which case the server MUST
/// create a new queue with a unique generated name and return /// create a new queue with a unique generated name and return
@ -260,21 +271,27 @@ pub struct QueueConfig {
pub no_wait: bool, pub no_wait: bool,
} }
#[async_trait]
pub trait ChannelExt { pub trait ChannelExt {
type Error; type Error;
fn declare_exchange(&mut self, config: ExchangeConfig) -> Result<(), Self::Error>; async fn declare_exchange(&mut self, config: ExchangeConfig) -> Result<(), Self::Error>;
fn declare_queue(&mut self, config: QueueConfig) -> Result<(), Self::Error>; async fn declare_queue(&mut self, config: QueueConfig) -> Result<(), Self::Error>;
fn bind_queue(&mut self, config: BindQueueConfig) -> Result<(), Self::Error>; async fn bind_queue(&mut self, config: BindQueueConfig) -> Result<(), Self::Error>;
fn send_request<T: ?Sized + Serialize + std::marker::Sync, U: for<'a> Deserialize<'a>>( async fn send_request<
T: ?Sized + Serialize + Sync + Send + Debug,
U: for<'a> Deserialize<'a> + Debug,
>(
&mut self, &mut self,
exchange: Option<&str>, exchange: Option<&str>,
routing_key: Option<&str>, routing_key: Option<&str>,
msg: &T, msg: &T,
) -> impl std::future::Future<Output = Result<U, Self::Error>> + Send; ) -> Result<U, Self::Error>;
} }
pub trait ConsumerExt<'a, C> { #[async_trait]
pub trait ConsumerExt<C> {
type Error; type Error;
type Handle; async fn consume(self, callback: C, config: ConsumeConfig) -> Result<(), Self::Error>
fn consume(self, callback: C, config: ConsumeConfig) -> Result<Self::Handle, Self::Error>; where
C: 'async_trait;
} }

View file

@ -1,6 +1,3 @@
use std::future::Future;
use std::pin::Pin;
use crate::config::RabbitMqConfig; use crate::config::RabbitMqConfig;
use crate::easyamqp::{ use crate::easyamqp::{
BindQueueConfig, ChannelExt, ConsumeConfig, ConsumerExt, ExchangeConfig, ExchangeType, BindQueueConfig, ChannelExt, ConsumeConfig, ConsumerExt, ExchangeConfig, ExchangeType,
@ -10,7 +7,9 @@ use crate::notifyworker::{NotificationReceiver, SimpleNotifyWorker};
use crate::ofborg; use crate::ofborg;
use crate::worker::{prepare_queue_message, Action, SimpleWorker}; use crate::worker::{prepare_queue_message, Action, SimpleWorker};
use futures::executor::block_on; use std::fmt::Debug;
use async_trait::async_trait;
use futures::StreamExt; use futures::StreamExt;
use lapin::message::Delivery; use lapin::message::Delivery;
use lapin::options::{ use lapin::options::{
@ -22,7 +21,7 @@ use lapin::{BasicProperties, Channel, Connection, ConnectionProperties, Exchange
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use tracing::{debug, trace}; use tracing::{debug, trace};
pub fn from_config(cfg: &RabbitMqConfig) -> Result<Connection, lapin::Error> { pub async fn from_config(cfg: &RabbitMqConfig) -> Result<Connection, lapin::Error> {
let mut props = FieldTable::default(); let mut props = FieldTable::default();
props.insert( props.insert(
"ofborg_version".into(), "ofborg_version".into(),
@ -32,13 +31,15 @@ pub fn from_config(cfg: &RabbitMqConfig) -> Result<Connection, lapin::Error> {
client_properties: props, client_properties: props,
..Default::default() ..Default::default()
}; };
block_on(Connection::connect(&cfg.as_uri()?, opts)) Connection::connect(&cfg.as_uri()?, opts).await
} }
#[async_trait]
impl ChannelExt for Channel { impl ChannelExt for Channel {
type Error = lapin::Error; type Error = lapin::Error;
fn declare_exchange(&mut self, config: ExchangeConfig) -> Result<(), Self::Error> { #[tracing::instrument(ret)]
async fn declare_exchange(&mut self, config: ExchangeConfig) -> Result<(), Self::Error> {
let opts = ExchangeDeclareOptions { let opts = ExchangeDeclareOptions {
passive: config.passive, passive: config.passive,
durable: config.durable, durable: config.durable,
@ -52,11 +53,13 @@ impl ChannelExt for Channel {
ExchangeType::Fanout => ExchangeKind::Fanout, ExchangeType::Fanout => ExchangeKind::Fanout,
_ => panic!("exchange kind"), _ => panic!("exchange kind"),
}; };
block_on(self.exchange_declare(&config.exchange, kind, opts, FieldTable::default()))?; self.exchange_declare(&config.exchange, kind, opts, FieldTable::default())
.await?;
Ok(()) Ok(())
} }
fn declare_queue(&mut self, config: QueueConfig) -> Result<(), Self::Error> { #[tracing::instrument(ret)]
async fn declare_queue(&mut self, config: QueueConfig) -> Result<(), Self::Error> {
let opts = QueueDeclareOptions { let opts = QueueDeclareOptions {
passive: config.passive, passive: config.passive,
durable: config.durable, durable: config.durable,
@ -65,26 +68,33 @@ impl ChannelExt for Channel {
nowait: config.no_wait, nowait: config.no_wait,
}; };
block_on(self.queue_declare(&config.queue, opts, FieldTable::default()))?; self.queue_declare(&config.queue, opts, FieldTable::default())
.await?;
Ok(()) Ok(())
} }
fn bind_queue(&mut self, config: BindQueueConfig) -> Result<(), Self::Error> { #[tracing::instrument(ret)]
async fn bind_queue(&mut self, config: BindQueueConfig) -> Result<(), Self::Error> {
let opts = QueueBindOptions { let opts = QueueBindOptions {
nowait: config.no_wait, nowait: config.no_wait,
}; };
block_on(self.queue_bind( self.queue_bind(
&config.queue, &config.queue,
&config.exchange, &config.exchange,
&config.routing_key.unwrap_or_else(|| "".into()), &config.routing_key.unwrap_or_default(),
opts, opts,
FieldTable::default(), FieldTable::default(),
))?; )
.await?;
Ok(()) Ok(())
} }
async fn send_request<T: ?Sized + Serialize, U: for<'a> Deserialize<'a>>( #[tracing::instrument(ret)]
async fn send_request<
T: ?Sized + Serialize + Sync + Send + Debug,
U: for<'a> Deserialize<'a> + Debug,
>(
&mut self, &mut self,
exchange: Option<&str>, exchange: Option<&str>,
routing_key: Option<&str>, routing_key: Option<&str>,
@ -105,8 +115,8 @@ impl ChannelExt for Channel {
trace!(?exchange, ?routing_key, "sending a RPC request"); trace!(?exchange, ?routing_key, "sending a RPC request");
let confirmation = self let confirmation = self
.basic_publish( .basic_publish(
&msg.exchange.take().unwrap_or_else(|| "".to_owned()), &msg.exchange.take().unwrap_or_default(),
&msg.routing_key.take().unwrap_or_else(|| "".to_owned()), &msg.routing_key.take().unwrap_or_default(),
BasicPublishOptions::default(), BasicPublishOptions::default(),
&msg.content, &msg.content,
props, props,
@ -139,55 +149,64 @@ impl ChannelExt for Channel {
} }
} }
impl<'a, W: SimpleWorker + 'a> ConsumerExt<'a, W> for Channel { #[async_trait]
impl<W: SimpleWorker> ConsumerExt<W> for Channel {
type Error = lapin::Error; type Error = lapin::Error;
type Handle = Pin<Box<dyn Future<Output = ()> + 'a>>;
fn consume( #[tracing::instrument(skip(worker), ret)]
mut self, async fn consume(mut self, mut worker: W, config: ConsumeConfig) -> Result<(), Self::Error>
mut worker: W, where
config: ConsumeConfig, W: 'async_trait,
) -> Result<Self::Handle, Self::Error> { {
let mut consumer = block_on(self.basic_consume( let mut consumer = self
.basic_consume(
&config.queue, &config.queue,
&config.consumer_tag, &config.consumer_tag,
BasicConsumeOptions::default(), BasicConsumeOptions::default(),
FieldTable::default(), FieldTable::default(),
))?; )
Ok(Box::pin(async move { .await?;
while let Some(Ok(deliver)) = consumer.next().await { while let Some(Ok(deliver)) = consumer.next().await {
debug!(?deliver.delivery_tag, "consumed delivery"); debug!(?deliver.delivery_tag, "consumed delivery");
let content_type = deliver.properties.content_type(); let content_type = deliver.properties.content_type();
let job = worker let job = worker
.msg_to_job( .msg_to_job(
deliver.routing_key.as_str(), deliver.routing_key.as_str(),
&content_type.as_ref().map(|s| s.to_string()), &content_type.as_ref().map(std::string::ToString::to_string),
&deliver.data, &deliver.data,
) )
.await
.expect("worker unexpected message consumed"); .expect("worker unexpected message consumed");
for action in worker.consumer(&mut self, &job) { for action in worker.consumer(&mut self, &job).await {
action_deliver(&self, &deliver, action) action_deliver(&self, &deliver, action)
.await .await
.expect("action deliver failure"); .expect("action deliver failure");
} }
debug!(?deliver.delivery_tag, "done"); debug!(?deliver.delivery_tag, "done");
} }
}))
Ok(())
} }
} }
/// Same as a regular channel, but without prefetching, /// Same as a regular channel, but without prefetching,
/// used for services with multiple instances. /// used for services with multiple instances.
#[derive(Debug)]
pub struct WorkerChannel(pub Channel); pub struct WorkerChannel(pub Channel);
impl<'a, W: SimpleWorker + 'a> ConsumerExt<'a, W> for WorkerChannel { #[async_trait]
impl<W: SimpleWorker> ConsumerExt<W> for WorkerChannel {
type Error = lapin::Error; type Error = lapin::Error;
type Handle = Pin<Box<dyn Future<Output = ()> + 'a>>;
fn consume(self, worker: W, config: ConsumeConfig) -> Result<Self::Handle, Self::Error> { #[tracing::instrument(skip(worker), ret)]
block_on(self.0.basic_qos(1, BasicQosOptions::default()))?; async fn consume(self, worker: W, config: ConsumeConfig) -> Result<(), Self::Error>
self.0.consume(worker, config) where
W: 'async_trait,
{
self.0.basic_qos(1, BasicQosOptions::default()).await?;
Ok(self.0.consume(worker, config).await?)
} }
} }
@ -202,32 +221,42 @@ impl<'a> ChannelNotificationReceiver<'a> {
} }
} }
#[async_trait]
impl<'a> NotificationReceiver for ChannelNotificationReceiver<'a> { impl<'a> NotificationReceiver for ChannelNotificationReceiver<'a> {
fn tell(&mut self, action: Action) { async fn tell(&mut self, action: Action) {
block_on(action_deliver(self.channel, self.deliver, action)) action_deliver(self.channel, self.deliver, action)
.await
.expect("action deliver failure"); .expect("action deliver failure");
} }
} }
// FIXME the consumer trait for SimpleWorker and SimpleNotifyWorker conflict, // FIXME the consumer trait for SimpleWorker and SimpleNotifyWorker conflict,
// but one could probably be implemented in terms of the other instead. // but one could probably be implemented in terms of the other instead.
#[derive(Debug)]
pub struct NotifyChannel(pub Channel); pub struct NotifyChannel(pub Channel);
impl<'a, W: SimpleNotifyWorker + 'a + Send> ConsumerExt<'a, W> for NotifyChannel { #[async_trait]
impl<W: SimpleNotifyWorker> ConsumerExt<W> for NotifyChannel {
type Error = lapin::Error; type Error = lapin::Error;
type Handle = Pin<Box<dyn Future<Output = ()> + 'a + Send>>;
fn consume(self, worker: W, config: ConsumeConfig) -> Result<Self::Handle, Self::Error> { #[tracing::instrument(skip(worker), ret)]
block_on(self.0.basic_qos(1, BasicQosOptions::default()))?; async fn consume(self, worker: W, config: ConsumeConfig) -> Result<(), Self::Error>
where
W: 'async_trait,
{
self.0.basic_qos(1, BasicQosOptions::default()).await?;
let mut consumer = block_on(self.0.basic_consume( let mut consumer = self
.0
.basic_consume(
&config.queue, &config.queue,
&config.consumer_tag, &config.consumer_tag,
BasicConsumeOptions::default(), BasicConsumeOptions::default(),
FieldTable::default(), FieldTable::default(),
))?; )
.await?;
let mut chan = self.0; let mut chan = self.0;
Ok(Box::pin(async move {
while let Some(Ok(deliver)) = consumer.next().await { while let Some(Ok(deliver)) = consumer.next().await {
debug!(?deliver.delivery_tag, "consumed delivery"); debug!(?deliver.delivery_tag, "consumed delivery");
let mut receiver = ChannelNotificationReceiver { let mut receiver = ChannelNotificationReceiver {
@ -239,15 +268,17 @@ impl<'a, W: SimpleNotifyWorker + 'a + Send> ConsumerExt<'a, W> for NotifyChannel
let job = worker let job = worker
.msg_to_job( .msg_to_job(
deliver.routing_key.as_str(), deliver.routing_key.as_str(),
&content_type.as_ref().map(|s| s.to_string()), &content_type.as_ref().map(std::string::ToString::to_string),
&deliver.data, &deliver.data,
) )
.await
.expect("worker unexpected message consumed"); .expect("worker unexpected message consumed");
worker.consumer(&job, &mut receiver); worker.consumer(&job, &mut receiver).await;
debug!(?deliver.delivery_tag, "done"); debug!(?deliver.delivery_tag, "done");
} }
}))
Ok(())
} }
} }
@ -276,8 +307,8 @@ async fn action_deliver(
.await .await
} }
Action::Publish(mut msg) => { Action::Publish(mut msg) => {
let exch = msg.exchange.take().unwrap_or_else(|| "".to_owned()); let exch = msg.exchange.take().unwrap_or_else(String::new);
let key = msg.routing_key.take().unwrap_or_else(|| "".to_owned()); let key = msg.routing_key.take().unwrap_or_else(String::new);
trace!(?exch, ?key, "action publish"); trace!(?exch, ?key, "action publish");
let mut props = BasicProperties::default().with_delivery_mode(2); // persistent. let mut props = BasicProperties::default().with_delivery_mode(2); // persistent.

View file

@ -11,6 +11,7 @@ pub struct EvalChecker {
} }
impl EvalChecker { impl EvalChecker {
#[must_use]
pub fn new(name: &str, op: nix::Operation, args: Vec<String>, nix: nix::Nix) -> EvalChecker { pub fn new(name: &str, op: nix::Operation, args: Vec<String>, nix: nix::Nix) -> EvalChecker {
EvalChecker { EvalChecker {
name: name.to_owned(), name: name.to_owned(),
@ -20,14 +21,18 @@ impl EvalChecker {
} }
} }
#[must_use]
pub fn name(&self) -> &str { pub fn name(&self) -> &str {
&self.name &self.name
} }
pub fn execute(&self, path: &Path) -> Result<File, File> { pub async fn execute(&self, path: &Path) -> Result<File, File> {
self.nix.safely(&self.op, path, self.args.clone(), false) self.nix
.safely(&self.op, path, self.args.clone(), false)
.await
} }
#[must_use]
pub fn cli_cmd(&self) -> String { pub fn cli_cmd(&self) -> String {
let mut cli = vec![self.op.to_string()]; let mut cli = vec![self.op.to_string()];
cli.append(&mut self.args.clone()); cli.append(&mut self.args.clone());

View file

@ -1,9 +1,9 @@
#![recursion_limit = "512"] #![recursion_limit = "512"]
// Replacing .map(|arch| arch.to_string()) #![deny(clippy::pedantic)]
// with .map(systems::System::to_string) #![allow(clippy::missing_errors_doc)]
// #![allow(clippy::missing_panics_doc)]
// seems much less clear and I just don't like it :) #![allow(clippy::module_name_repetitions)]
#![allow(clippy::redundant_closure)] #![allow(clippy::similar_names)]
#[macro_use] #[macro_use]
extern crate serde_derive; extern crate serde_derive;
@ -14,15 +14,14 @@ extern crate nom;
use std::env; use std::env;
use opentelemetry::global; use opentelemetry::global;
use opentelemetry::trace::TracerProvider;
use opentelemetry::KeyValue; use opentelemetry::KeyValue;
use opentelemetry_otlp::HasHttpConfig;
use opentelemetry_sdk::Resource; use opentelemetry_sdk::Resource;
use opentelemetry_semantic_conventions::resource::SERVICE_NAME; use opentelemetry_semantic_conventions::resource::SERVICE_NAME;
use opentelemetry_semantic_conventions::resource::SERVICE_VERSION; use opentelemetry_semantic_conventions::resource::SERVICE_VERSION;
use opentelemetry_semantic_conventions::SCHEMA_URL; use opentelemetry_semantic_conventions::SCHEMA_URL;
use tracing_subscriber::prelude::*; use tracing_subscriber::prelude::*;
use tracing_subscriber::EnvFilter; use tracing_subscriber::EnvFilter;
use opentelemetry::trace::TracerProvider;
pub mod acl; pub mod acl;
pub mod asynccmd; pub mod asynccmd;
@ -80,10 +79,11 @@ pub mod ofborg {
pub const VERSION: &str = env!("CARGO_PKG_VERSION"); pub const VERSION: &str = env!("CARGO_PKG_VERSION");
#[must_use]
pub fn partition_result<A, B>(results: Vec<Result<A, B>>) -> (Vec<A>, Vec<B>) { pub fn partition_result<A, B>(results: Vec<Result<A, B>>) -> (Vec<A>, Vec<B>) {
let mut ok = Vec::new(); let mut ok = Vec::new();
let mut err = Vec::new(); let mut err = Vec::new();
for result in results.into_iter() { for result in results {
match result { match result {
Ok(x) => { Ok(x) => {
ok.push(x); ok.push(x);
@ -121,12 +121,15 @@ pub fn setup_log() {
tracing_subscriber::fmt::layer().boxed() tracing_subscriber::fmt::layer().boxed()
}; };
let exporter = opentelemetry_otlp::SpanExporter::builder().with_http().build().unwrap();
let provider = opentelemetry_sdk::trace::TracerProvider::builder() let provider = opentelemetry_sdk::trace::TracerProvider::builder()
.with_simple_exporter(exporter) .with_batch_exporter(
.with_config( opentelemetry_otlp::SpanExporter::builder()
opentelemetry_sdk::trace::Config::default().with_resource(resource()) .with_http()
.build()
.unwrap(),
opentelemetry_sdk::runtime::Tokio,
) )
.with_config(opentelemetry_sdk::trace::Config::default().with_resource(resource()))
.build(); .build();
global::set_tracer_provider(provider.clone()); global::set_tracer_provider(provider.clone());

View file

@ -20,6 +20,6 @@ pub struct Lock {
impl Lock { impl Lock {
pub fn unlock(&mut self) { pub fn unlock(&mut self) {
self.lock = None self.lock = None;
} }
} }

View file

@ -48,7 +48,7 @@ impl From<std::string::FromUtf8Error> for CalculationError {
} }
impl ImpactedMaintainers { impl ImpactedMaintainers {
pub fn calculate( pub async fn calculate(
nix: &Nix, nix: &Nix,
checkout: &Path, checkout: &Path,
paths: &[String], paths: &[String],
@ -56,11 +56,11 @@ impl ImpactedMaintainers {
) -> Result<ImpactedMaintainers, CalculationError> { ) -> Result<ImpactedMaintainers, CalculationError> {
let mut path_file = NamedTempFile::new()?; let mut path_file = NamedTempFile::new()?;
let pathstr = serde_json::to_string(&paths)?; let pathstr = serde_json::to_string(&paths)?;
write!(path_file, "{}", pathstr)?; write!(path_file, "{pathstr}")?;
let mut attr_file = NamedTempFile::new()?; let mut attr_file = NamedTempFile::new()?;
let attrstr = serde_json::to_string(&attributes)?; let attrstr = serde_json::to_string(&attributes)?;
write!(attr_file, "{}", attrstr)?; write!(attr_file, "{attrstr}")?;
let mut argstrs: HashMap<&str, &str> = HashMap::new(); let mut argstrs: HashMap<&str, &str> = HashMap::new();
argstrs.insert("changedattrsjson", attr_file.path().to_str().unwrap()); argstrs.insert("changedattrsjson", attr_file.path().to_str().unwrap());
@ -73,21 +73,25 @@ impl ImpactedMaintainers {
&[path_file.path(), attr_file.path()], &[path_file.path(), attr_file.path()],
); );
let ret = cmd.output()?; let ret = cmd.output().await?;
Ok(serde_json::from_str(&String::from_utf8(ret.stdout)?)?) Ok(serde_json::from_str(&String::from_utf8(ret.stdout)?)?)
} }
#[must_use]
pub fn maintainers(&self) -> Vec<String> { pub fn maintainers(&self) -> Vec<String> {
self.0.keys().map(|maintainer| maintainer.0.clone()) self.0
.keys()
.map(|maintainer| maintainer.0.clone())
.collect() .collect()
} }
#[must_use]
pub fn maintainers_by_package(&self) -> MaintainersByPackage { pub fn maintainers_by_package(&self) -> MaintainersByPackage {
let mut bypkg = MaintainersByPackage(HashMap::new()); let mut bypkg = MaintainersByPackage(HashMap::new());
for (maintainer, packages) in self.0.iter() { for (maintainer, packages) in &self.0 {
for package in packages.iter() { for package in packages {
bypkg bypkg
.0 .0
.entry(package.clone()) .entry(package.clone())
@ -118,7 +122,7 @@ impl std::fmt::Display for ImpactedMaintainers {
}) })
.collect::<Vec<String>>() .collect::<Vec<String>>()
.join("\n"); .join("\n");
write!(f, "{}", d) write!(f, "{d}")
} }
} }
@ -154,15 +158,15 @@ mod tests {
.expect("building the test PR failed"); .expect("building the test PR failed");
let stderr = let stderr =
String::from_utf8(output.stderr).unwrap_or_else(|err| format!("warning: {}", err)); String::from_utf8(output.stderr).unwrap_or_else(|err| format!("warning: {err}"));
println!("{}", stderr); println!("{stderr}");
let hash = String::from_utf8(output.stdout).expect("Should just be a hash"); let hash = String::from_utf8(output.stdout).expect("Should just be a hash");
return hash.trim().to_owned(); return hash.trim().to_owned();
} }
#[test] #[tokio::test]
fn example() { async fn example() {
let workingdir = TestScratch::new_dir("test-maintainers-example"); let workingdir = TestScratch::new_dir("test-maintainers-example");
let bare = TestScratch::new_dir("test-maintainers-example-bare"); let bare = TestScratch::new_dir("test-maintainers-example-bare");
@ -186,11 +190,11 @@ mod tests {
working_co.checkout_ref(OsStr::new(&hash)).unwrap(); working_co.checkout_ref(OsStr::new(&hash)).unwrap();
let remote = env::var("NIX_REMOTE").unwrap_or("".to_owned()); let remote = env::var("NIX_REMOTE").unwrap_or_default();
let nix = Nix::new(SYSTEM.to_owned(), remote, 1800, None); let nix = Nix::new(SYSTEM.to_owned(), remote, 1800, None);
let parsed = let parsed =
ImpactedMaintainers::calculate(&nix, &working_co.clone_to(), &paths, &attributes); ImpactedMaintainers::calculate(&nix, &working_co.clone_to(), &paths, &attributes).await;
let mut expect = ImpactedMaintainers(HashMap::new()); let mut expect = ImpactedMaintainers(HashMap::new());
expect.0.insert( expect.0.insert(

View file

@ -23,6 +23,7 @@ type Exchange = String;
type RoutingKey = String; type RoutingKey = String;
impl BuildJob { impl BuildJob {
#[must_use]
pub fn new( pub fn new(
repo: Repo, repo: Repo,
change: Change, change: Change,

View file

@ -3,7 +3,7 @@ pub struct BuildLogMsg {
pub system: String, pub system: String,
pub identity: String, pub identity: String,
pub attempt_id: String, pub attempt_id: String,
pub line_number: u64, pub line_number: usize,
pub output: String, pub output: String,
} }

View file

@ -23,7 +23,7 @@ impl From<BuildStatus> for String {
BuildStatus::Failure => "Failure".into(), BuildStatus::Failure => "Failure".into(),
BuildStatus::HashMismatch => "A fixed output derivation's hash was incorrect".into(), BuildStatus::HashMismatch => "A fixed output derivation's hash was incorrect".into(),
BuildStatus::TimedOut => "Timed out, unknown build status".into(), BuildStatus::TimedOut => "Timed out, unknown build status".into(),
BuildStatus::UnexpectedError { ref err } => format!("Unexpected error: {}", err), BuildStatus::UnexpectedError { ref err } => format!("Unexpected error: {err}"),
} }
} }
} }
@ -96,6 +96,7 @@ pub enum BuildResult {
} }
impl BuildResult { impl BuildResult {
#[must_use]
pub fn legacy(&self) -> LegacyBuildResult { pub fn legacy(&self) -> LegacyBuildResult {
// TODO: replace this with simpler structs for specific usecases, since // TODO: replace this with simpler structs for specific usecases, since
// it's decouples the structs from serialization. These can be changed // it's decouples the structs from serialization. These can be changed
@ -111,18 +112,8 @@ impl BuildResult {
ref attempted_attrs, ref attempted_attrs,
ref skipped_attrs, ref skipped_attrs,
.. ..
} => LegacyBuildResult { }
repo: repo.to_owned(), | BuildResult::V1 {
pr: pr.to_owned(),
system: system.to_owned(),
output: output.to_owned(),
attempt_id: attempt_id.to_owned(),
request_id: request_id.to_owned(),
status: self.status(),
attempted_attrs: attempted_attrs.to_owned(),
skipped_attrs: skipped_attrs.to_owned(),
},
BuildResult::V1 {
ref repo, ref repo,
ref pr, ref pr,
ref system, ref system,
@ -167,14 +158,15 @@ impl BuildResult {
} }
} }
#[must_use]
pub fn change(&self) -> Change { pub fn change(&self) -> Change {
match self { match self {
BuildResult::Legacy { pr, .. } => pr.to_owned(), BuildResult::Legacy { pr, .. } | BuildResult::V1 { pr, .. } => pr.to_owned(),
BuildResult::V1 { pr, .. } => pr.to_owned(),
BuildResult::V2 { change, .. } => change.to_owned(), BuildResult::V2 { change, .. } => change.to_owned(),
} }
} }
#[must_use]
pub fn status(&self) -> BuildStatus { pub fn status(&self) -> BuildStatus {
match *self { match *self {
BuildResult::Legacy { BuildResult::Legacy {
@ -210,8 +202,7 @@ mod tests {
assert_eq!( assert_eq!(
output, output,
r#"{"tag":"V1","repo":{"owner":"NixOS","name":"nixpkgs","full_name":"NixOS/nixpkgs","clone_url":"https://github.com/nixos/nixpkgs.git"},"pr":{"target_branch":"master","number":42,"head_sha":"0000000000000000000000000000000000000000"},"system":"x86_64-linux","output":["unpacking sources"],"attempt_id":"attempt-id-foo","request_id":"bogus-request-id","status":"Success","skipped_attrs":["AAAAAASomeThingsFailToEvaluate"],"attempted_attrs":["hello"]}"#, r#"{"tag":"V1","repo":{"owner":"NixOS","name":"nixpkgs","full_name":"NixOS/nixpkgs","clone_url":"https://github.com/nixos/nixpkgs.git"},"pr":{"target_branch":"master","number":42,"head_sha":"0000000000000000000000000000000000000000"},"system":"x86_64-linux","output":["unpacking sources"],"attempt_id":"attempt-id-foo","request_id":"bogus-request-id","status":"Success","skipped_attrs":["AAAAAASomeThingsFailToEvaluate"],"attempted_attrs":["hello"]}"#,
"json of: {:?}", "json of: {result:?}"
result
); );
} }
@ -224,8 +215,7 @@ mod tests {
assert_eq!( assert_eq!(
output, output,
r#"{"repo":{"owner":"NixOS","name":"nixpkgs","full_name":"NixOS/nixpkgs","clone_url":"https://github.com/nixos/nixpkgs.git"},"pr":{"target_branch":"master","number":42,"head_sha":"0000000000000000000000000000000000000000"},"system":"x86_64-linux","output":["unpacking sources"],"attempt_id":"attempt-id-foo","request_id":"bogus-request-id","success":true,"status":"Success","skipped_attrs":["AAAAAASomeThingsFailToEvaluate"],"attempted_attrs":["hello"]}"#, r#"{"repo":{"owner":"NixOS","name":"nixpkgs","full_name":"NixOS/nixpkgs","clone_url":"https://github.com/nixos/nixpkgs.git"},"pr":{"target_branch":"master","number":42,"head_sha":"0000000000000000000000000000000000000000"},"system":"x86_64-linux","output":["unpacking sources"],"attempt_id":"attempt-id-foo","request_id":"bogus-request-id","success":true,"status":"Success","skipped_attrs":["AAAAAASomeThingsFailToEvaluate"],"attempted_attrs":["hello"]}"#,
"json of: {:?}", "json of: {result:?}"
result
); );
} }
@ -238,8 +228,7 @@ mod tests {
assert_eq!( assert_eq!(
output, output,
r#"{"repo":{"owner":"NixOS","name":"nixpkgs","full_name":"NixOS/nixpkgs","clone_url":"https://github.com/nixos/nixpkgs.git"},"pr":{"target_branch":"master","number":42,"head_sha":"0000000000000000000000000000000000000000"},"system":"x86_64-linux","output":[],"attempt_id":"attempt-id-foo","request_id":"bogus-request-id","success":null,"status":null,"skipped_attrs":null,"attempted_attrs":null}"#, r#"{"repo":{"owner":"NixOS","name":"nixpkgs","full_name":"NixOS/nixpkgs","clone_url":"https://github.com/nixos/nixpkgs.git"},"pr":{"target_branch":"master","number":42,"head_sha":"0000000000000000000000000000000000000000"},"system":"x86_64-linux","output":[],"attempt_id":"attempt-id-foo","request_id":"bogus-request-id","success":null,"status":null,"skipped_attrs":null,"attempted_attrs":null}"#,
"json of: {:?}", "json of: {result:?}"
result
); );
} }
@ -252,8 +241,7 @@ mod tests {
assert_eq!( assert_eq!(
output, output,
r#"{"repo":{"owner":"NixOS","name":"nixpkgs","full_name":"NixOS/nixpkgs","clone_url":"https://github.com/nixos/nixpkgs.git"},"pr":{"target_branch":"master","number":42,"head_sha":"0000000000000000000000000000000000000000"},"system":"x86_64-linux","output":["unpacking sources"],"attempt_id":"attempt-id-foo","request_id":"bogus-request-id","success":true,"status":null,"skipped_attrs":["AAAAAASomeThingsFailToEvaluate"],"attempted_attrs":["hello"]}"#, r#"{"repo":{"owner":"NixOS","name":"nixpkgs","full_name":"NixOS/nixpkgs","clone_url":"https://github.com/nixos/nixpkgs.git"},"pr":{"target_branch":"master","number":42,"head_sha":"0000000000000000000000000000000000000000"},"system":"x86_64-linux","output":["unpacking sources"],"attempt_id":"attempt-id-foo","request_id":"bogus-request-id","success":true,"status":null,"skipped_attrs":["AAAAAASomeThingsFailToEvaluate"],"attempted_attrs":["hello"]}"#,
"json of: {:?}", "json of: {result:?}"
result
); );
} }
} }

View file

@ -12,6 +12,7 @@ pub struct EvaluationJob {
} }
impl EvaluationJob { impl EvaluationJob {
#[must_use]
pub fn is_nixpkgs(&self) -> bool { pub fn is_nixpkgs(&self) -> bool {
self.repo.name == "nixpkgs" self.repo.name == "nixpkgs"
} }
@ -20,15 +21,18 @@ impl EvaluationJob {
pub struct Actions {} pub struct Actions {}
impl Actions { impl Actions {
pub fn retry_later(&mut self, _job: &EvaluationJob) -> worker::Actions { #[must_use]
pub fn retry_later(_job: &EvaluationJob) -> worker::Actions {
vec![worker::Action::NackRequeue] vec![worker::Action::NackRequeue]
} }
pub fn skip(&mut self, _job: &EvaluationJob) -> worker::Actions { #[must_use]
pub fn skip(_job: &EvaluationJob) -> worker::Actions {
vec![worker::Action::Ack] vec![worker::Action::Ack]
} }
pub fn done(&mut self, _job: &EvaluationJob, mut response: worker::Actions) -> worker::Actions { #[must_use]
pub fn done(_job: &EvaluationJob, mut response: worker::Actions) -> worker::Actions {
response.push(worker::Action::Ack); response.push(worker::Action::Ack);
response response
} }

View file

@ -1,4 +1,3 @@
use crate::asynccmd::{AsyncCmd, SpawnedAsyncCmd};
use crate::message::buildresult::BuildStatus; use crate::message::buildresult::BuildStatus;
use crate::ofborg::partition_result; use crate::ofborg::partition_result;
@ -9,9 +8,10 @@ use std::fmt;
use std::fs; use std::fs;
use std::io::{BufRead, BufReader, Seek, SeekFrom}; use std::io::{BufRead, BufReader, Seek, SeekFrom};
use std::path::Path; use std::path::Path;
use std::process::{Command, Stdio}; use std::process::Stdio;
use tempfile::tempfile; use tempfile::tempfile;
use tokio::process::{Child, Command};
#[allow(clippy::upper_case_acronyms)] #[allow(clippy::upper_case_acronyms)]
#[derive(Clone, Copy, Debug, PartialEq, Eq)] #[derive(Clone, Copy, Debug, PartialEq, Eq)]
@ -43,11 +43,11 @@ pub enum Operation {
impl Operation { impl Operation {
fn command(&self) -> Command { fn command(&self) -> Command {
match *self { match *self {
Operation::Evaluate => Command::new("nix-instantiate"), Operation::Evaluate | Operation::Instantiate => Command::new("nix-instantiate"),
Operation::Instantiate => Command::new("nix-instantiate"),
Operation::Build => Command::new("nix-build"), Operation::Build => Command::new("nix-build"),
Operation::QueryPackagesJson => Command::new("nix-env"), Operation::QueryPackagesJson | Operation::QueryPackagesOutputs => {
Operation::QueryPackagesOutputs => Command::new("nix-env"), Command::new("nix-env")
}
Operation::NoOp { .. } => Command::new("echo"), Operation::NoOp { .. } => Command::new("echo"),
Operation::Unknown { ref program } => Command::new(program), Operation::Unknown { ref program } => Command::new(program),
} }
@ -102,7 +102,7 @@ impl Operation {
Operation::Instantiate => { Operation::Instantiate => {
command.args(["--option", "extra-experimental-features", "no-url-literals"]); command.args(["--option", "extra-experimental-features", "no-url-literals"]);
} }
_ => (), Operation::Unknown { .. } => (),
}; };
} }
} }
@ -115,7 +115,7 @@ impl fmt::Display for Operation {
Operation::QueryPackagesJson => write!(f, "nix-env -qa --json"), Operation::QueryPackagesJson => write!(f, "nix-env -qa --json"),
Operation::QueryPackagesOutputs => write!(f, "nix-env -qaP --no-name --out-path"), Operation::QueryPackagesOutputs => write!(f, "nix-env -qaP --no-name --out-path"),
Operation::NoOp { ref operation } => operation.fmt(f), Operation::NoOp { ref operation } => operation.fmt(f),
Operation::Unknown { ref program } => write!(f, "{}", program), Operation::Unknown { ref program } => write!(f, "{program}"),
Operation::Evaluate => write!(f, "nix-instantiate --strict --json ..."), Operation::Evaluate => write!(f, "nix-instantiate --strict --json ..."),
} }
} }
@ -131,6 +131,7 @@ pub struct Nix {
} }
impl Nix { impl Nix {
#[must_use]
pub fn new( pub fn new(
system: String, system: String,
remote: String, remote: String,
@ -146,54 +147,62 @@ impl Nix {
} }
} }
#[must_use]
pub fn with_system(&self, system: String) -> Nix { pub fn with_system(&self, system: String) -> Nix {
let mut n = self.clone(); let mut n = self.clone();
n.system = system; n.system = system;
n n
} }
#[must_use]
pub fn with_limited_supported_systems(&self) -> Nix { pub fn with_limited_supported_systems(&self) -> Nix {
let mut n = self.clone(); let mut n = self.clone();
n.limit_supported_systems = true; n.limit_supported_systems = true;
n n
} }
#[must_use]
pub fn without_limited_supported_systems(&self) -> Nix { pub fn without_limited_supported_systems(&self) -> Nix {
let mut n = self.clone(); let mut n = self.clone();
n.limit_supported_systems = false; n.limit_supported_systems = false;
n n
} }
pub fn safely_partition_instantiable_attrs( pub async fn safely_partition_instantiable_attrs(
&self, &self,
nixpkgs: &Path, nixpkgs: &Path,
file: File, file: File,
attrs: Vec<String>, attrs: Vec<String>,
) -> (Vec<String>, Vec<(String, Vec<String>)>) { ) -> (Vec<String>, Vec<(String, Vec<String>)>) {
let attr_instantiations: Vec<Result<String, (String, Vec<String>)>> = attrs let mut attr_instantiations: Vec<Result<String, (String, Vec<String>)>> = vec![];
.into_iter()
.map( for attr in attrs {
|attr| match self.safely_instantiate_attrs(nixpkgs, file, vec![attr.clone()]) { attr_instantiations.push(
match self
.safely_instantiate_attrs(nixpkgs, file, vec![attr.clone()])
.await
{
Ok(_) => Ok(attr), Ok(_) => Ok(attr),
Err(f) => Err((attr, lines_from_file(f))), Err(f) => Err((attr, lines_from_file(f))),
}, },
) );
.collect(); }
partition_result(attr_instantiations) partition_result(attr_instantiations)
} }
pub fn safely_instantiate_attrs( pub async fn safely_instantiate_attrs(
&self, &self,
nixpkgs: &Path, nixpkgs: &Path,
file: File, file: File,
attrs: Vec<String>, attrs: Vec<String>,
) -> Result<fs::File, fs::File> { ) -> Result<fs::File, fs::File> {
let mut command = self.safe_command::<&OsStr>(&Operation::Instantiate, nixpkgs, &[], &[]); let mut command = self.safe_command::<&OsStr>(&Operation::Instantiate, nixpkgs, &[], &[]);
self.set_attrs_command(&mut command, file, attrs); Self::set_attrs_command(&mut command, file, attrs);
self.run(command, true) self.run(command, true).await
} }
#[must_use]
pub fn safely_evaluate_expr_cmd( pub fn safely_evaluate_expr_cmd(
&self, &self,
nixpkgs: &Path, nixpkgs: &Path,
@ -213,31 +222,32 @@ impl Nix {
self.safe_command(&Operation::Evaluate, nixpkgs, &attrargs, extra_paths) self.safe_command(&Operation::Evaluate, nixpkgs, &attrargs, extra_paths)
} }
pub fn safely_build_attrs( pub async fn safely_build_attrs(
&self, &self,
nixpkgs: &Path, nixpkgs: &Path,
file: File, file: File,
attrs: Vec<String>, attrs: Vec<String>,
) -> Result<fs::File, fs::File> { ) -> Result<fs::File, fs::File> {
let mut command = self.safe_command::<&OsStr>(&Operation::Build, nixpkgs, &[], &[]); let mut command = self.safe_command::<&OsStr>(&Operation::Build, nixpkgs, &[], &[]);
self.set_attrs_command(&mut command, file, attrs); Self::set_attrs_command(&mut command, file, attrs);
self.run(command, true) self.run(command, true).await
} }
#[must_use]
pub fn safely_build_attrs_async( pub fn safely_build_attrs_async(
&self, &self,
nixpkgs: &Path, nixpkgs: &Path,
file: File, file: File,
attrs: Vec<String>, attrs: Vec<String>,
) -> SpawnedAsyncCmd { ) -> Child {
let mut command = self.safe_command::<&OsStr>(&Operation::Build, nixpkgs, &[], &[]); let mut command = self.safe_command::<&OsStr>(&Operation::Build, nixpkgs, &[], &[]);
self.set_attrs_command(&mut command, file, attrs); Nix::set_attrs_command(&mut command, file, attrs);
AsyncCmd::new(command).spawn() command.spawn().expect("Failed to run Nix")
} }
fn set_attrs_command(&self, command: &mut Command, file: File, attrs: Vec<String>) { fn set_attrs_command(command: &mut Command, file: File, attrs: Vec<String>) {
let mut args: Vec<String> = Vec::with_capacity(3 + (attrs.len() * 2)); let mut args: Vec<String> = Vec::with_capacity(3 + (attrs.len() * 2));
args.push(format!("{}", file)); args.push(format!("{file}"));
for attr in attrs { for attr in attrs {
args.push(String::from("-A")); args.push(String::from("-A"));
args.push(attr); args.push(attr);
@ -252,7 +262,7 @@ impl Nix {
command.args(args); command.args(args);
} }
pub fn safely( pub async fn safely(
&self, &self,
op: &Operation, op: &Operation,
nixpkgs: &Path, nixpkgs: &Path,
@ -260,9 +270,10 @@ impl Nix {
keep_stdout: bool, keep_stdout: bool,
) -> Result<fs::File, fs::File> { ) -> Result<fs::File, fs::File> {
self.run(self.safe_command(op, nixpkgs, &args, &[]), keep_stdout) self.run(self.safe_command(op, nixpkgs, &args, &[]), keep_stdout)
.await
} }
pub fn run(&self, mut cmd: Command, keep_stdout: bool) -> Result<fs::File, fs::File> { pub async fn run(&self, mut cmd: Command, keep_stdout: bool) -> Result<fs::File, fs::File> {
let stderr = tempfile().expect("Fetching a stderr tempfile"); let stderr = tempfile().expect("Fetching a stderr tempfile");
let mut reader = stderr.try_clone().expect("Cloning stderr to the reader"); let mut reader = stderr.try_clone().expect("Cloning stderr to the reader");
@ -276,6 +287,7 @@ impl Nix {
.stdout(stdout) .stdout(stdout)
.stderr(Stdio::from(stderr)) .stderr(Stdio::from(stderr))
.status() .status()
.await
.expect("Running a program ..."); .expect("Running a program ...");
reader reader
@ -289,7 +301,7 @@ impl Nix {
} }
} }
pub fn run_stderr_stdout(&self, mut cmd: Command) -> (bool, fs::File, fs::File) { pub async fn run_stderr_stdout(&self, mut cmd: Command) -> (bool, fs::File, fs::File) {
let stdout_file = tempfile().expect("Fetching a stdout tempfile"); let stdout_file = tempfile().expect("Fetching a stdout tempfile");
let mut stdout_reader = stdout_file let mut stdout_reader = stdout_file
.try_clone() .try_clone()
@ -304,6 +316,7 @@ impl Nix {
.stdout(Stdio::from(stdout_file)) .stdout(Stdio::from(stdout_file))
.stderr(Stdio::from(stderr_file)) .stderr(Stdio::from(stderr_file))
.status() .status()
.await
.expect("Running a program ..."); .expect("Running a program ...");
stdout_reader stdout_reader
@ -379,28 +392,29 @@ fn lines_from_file(file: fs::File) -> Vec<String> {
.collect() .collect()
} }
#[must_use]
pub fn is_user_setting_warning(line: &str) -> bool { pub fn is_user_setting_warning(line: &str) -> bool {
let line = line.trim(); let line = line.trim();
line.starts_with("warning: ignoring the user-specified setting '") line.starts_with("warning: ignoring the user-specified setting '")
&& line.ends_with("because it is a restricted setting and you are not a trusted user") && line.ends_with("because it is a restricted setting and you are not a trusted user")
} }
pub fn wait_for_build_status(spawned: SpawnedAsyncCmd) -> BuildStatus { pub async fn wait_for_build_status(mut spawned: Child) -> BuildStatus {
match spawned.wait() { match spawned.wait().await {
Ok(s) => match s.code() { Ok(s) => match s.code() {
Some(0) => BuildStatus::Success, Some(0) => BuildStatus::Success,
Some(100) => BuildStatus::Failure, // nix permanent failure Some(100) => BuildStatus::Failure, // nix permanent failure
Some(101) => BuildStatus::TimedOut, // nix build timedout Some(101) => BuildStatus::TimedOut, // nix build timedout
Some(102) => BuildStatus::HashMismatch, // Fixed Output Derivation's hash was wrong Some(102) => BuildStatus::HashMismatch, // Fixed Output Derivation's hash was wrong
Some(i) => BuildStatus::UnexpectedError { Some(i) => BuildStatus::UnexpectedError {
err: format!("command failed with exit code {}", i), err: format!("command failed with exit code {i}"),
}, },
None => BuildStatus::UnexpectedError { None => BuildStatus::UnexpectedError {
err: "unexpected build failure".into(), err: "unexpected build failure".into(),
}, },
}, },
e => BuildStatus::UnexpectedError { e => BuildStatus::UnexpectedError {
err: format!("failed on interior command {:?}", e), err: format!("failed on interior command {e:?}"),
}, },
} }
} }
@ -420,7 +434,7 @@ mod tests {
let path = env::var("PATH").unwrap(); let path = env::var("PATH").unwrap();
let test_path = format!("{}/test-nix/bin:{}", env!("CARGO_MANIFEST_DIR"), path); let test_path = format!("{}/test-nix/bin:{}", env!("CARGO_MANIFEST_DIR"), path);
env::set_var("PATH", test_path); env::set_var("PATH", test_path);
let remote = env::var("NIX_REMOTE").unwrap_or("".to_owned()); let remote = env::var("NIX_REMOTE").unwrap_or_default();
Nix::new(SYSTEM.to_owned(), remote, 1800, None) Nix::new(SYSTEM.to_owned(), remote, 1800, None)
} }
@ -467,15 +481,14 @@ mod tests {
Fail, Fail,
} }
fn assert_run(res: Result<fs::File, fs::File>, expected: Expect, require: Vec<&str>) { fn assert_run(res: Result<fs::File, fs::File>, expected: &Expect, require: Vec<&str>) {
let expectation_held: bool = match expected { let expectation_held: bool = match expected {
Expect::Pass => res.is_ok(), Expect::Pass => res.is_ok(),
Expect::Fail => res.is_err(), Expect::Fail => res.is_err(),
}; };
let file: fs::File = match res { let file: fs::File = match res {
Ok(file) => file, Ok(file) | Err(file) => file,
Err(file) => file,
}; };
let lines = lines_from_file(file); let lines = lines_from_file(file);
@ -483,7 +496,7 @@ mod tests {
let buildlog = lines let buildlog = lines
.into_iter() .into_iter()
.map(|line| strip_ansi(&line)) .map(|line| strip_ansi(&line))
.map(|line| format!(" | {}", line)) .map(|line| format!(" | {line}"))
.collect::<Vec<String>>() .collect::<Vec<String>>()
.join("\n"); .join("\n");
@ -491,7 +504,7 @@ mod tests {
let mut missed_requirements: usize = 0; let mut missed_requirements: usize = 0;
let requirements_held: Vec<Result<String, String>> = require let requirements_held: Vec<Result<String, String>> = require
.into_iter() .into_iter()
.map(|line| line.to_owned()) .map(std::borrow::ToOwned::to_owned)
.map(|line| { .map(|line| {
if buildlog.contains(&line) { if buildlog.contains(&line) {
Ok(line) Ok(line)
@ -502,36 +515,34 @@ mod tests {
}) })
.collect(); .collect();
let mut prefixes: Vec<String> = vec!["".to_owned(), "".to_owned()]; let mut prefixes: Vec<String> = vec![String::new(), String::new()];
if !expectation_held { if expectation_held {
prefixes.push(format!( prefixes.push(format!("The run was expected to {expected:?}, and did."));
"The run was expected to {:?}, but did not.",
expected
));
prefixes.push("".to_owned());
} else { } else {
prefixes.push(format!("The run was expected to {:?}, and did.", expected)); prefixes.push(format!(
prefixes.push("".to_owned()); "The run was expected to {expected:?}, but did not."
));
} }
prefixes.push(String::new());
let mut suffixes = vec![ let mut suffixes = vec![
"".to_owned(), String::new(),
format!( format!(
"{} out of {} required lines matched.", "{} out of {} required lines matched.",
(total_requirements - missed_requirements), (total_requirements - missed_requirements),
total_requirements total_requirements
), ),
"".to_owned(), String::new(),
]; ];
for expected_line in requirements_held { for expected_line in requirements_held {
suffixes.push(format!(" - {:?}", expected_line)); suffixes.push(format!(" - {expected_line:?}"));
} }
suffixes.push("".to_owned()); suffixes.push(String::new());
let output_blocks: Vec<Vec<String>> = let output_blocks: Vec<Vec<String>> =
vec![prefixes, vec![buildlog, "".to_owned()], suffixes]; vec![prefixes, vec![buildlog, String::new()], suffixes];
let output_blocks_strings: Vec<String> = output_blocks let output_blocks_strings: Vec<String> = output_blocks
.into_iter() .into_iter()
@ -546,70 +557,78 @@ mod tests {
} }
} }
#[test] #[tokio::test]
fn test_build_operations() { async fn test_build_operations() {
let nix = nix(); let nix = nix();
let op = noop(Operation::Build); let op = noop(Operation::Build);
assert_eq!(op.to_string(), "nix-build"); assert_eq!(op.to_string(), "nix-build");
let ret: Result<fs::File, fs::File> = nix.run( let ret: Result<fs::File, fs::File> = nix
.run(
nix.safe_command(&op, build_path().as_path(), &["--version"], &[]), nix.safe_command(&op, build_path().as_path(), &["--version"], &[]),
true, true,
); )
.await;
assert_run( assert_run(
ret, ret,
Expect::Pass, &Expect::Pass,
vec!["--no-out-link --keep-going", "--version"], vec!["--no-out-link --keep-going", "--version"],
); );
} }
#[test] #[tokio::test]
fn test_instantiate_operation() { async fn test_instantiate_operation() {
let nix = nix(); let nix = nix();
let op = noop(Operation::Instantiate); let op = noop(Operation::Instantiate);
assert_eq!(op.to_string(), "nix-instantiate"); assert_eq!(op.to_string(), "nix-instantiate");
let ret: Result<fs::File, fs::File> = nix.run( let ret: Result<fs::File, fs::File> = nix
.run(
nix.safe_command(&op, build_path().as_path(), &["--version"], &[]), nix.safe_command(&op, build_path().as_path(), &["--version"], &[]),
true, true,
); )
.await;
assert_run(ret, Expect::Pass, vec!["--version"]); assert_run(ret, &Expect::Pass, vec!["--version"]);
} }
#[test] #[tokio::test]
fn test_query_packages_json() { async fn test_query_packages_json() {
let nix = nix(); let nix = nix();
let op = noop(Operation::QueryPackagesJson); let op = noop(Operation::QueryPackagesJson);
assert_eq!(op.to_string(), "nix-env -qa --json"); assert_eq!(op.to_string(), "nix-env -qa --json");
let ret: Result<fs::File, fs::File> = nix.run( let ret: Result<fs::File, fs::File> = nix
.run(
nix.safe_command(&op, build_path().as_path(), &["--version"], &[]), nix.safe_command(&op, build_path().as_path(), &["--version"], &[]),
true, true,
); )
.await;
assert_run( assert_run(
ret, ret,
Expect::Pass, &Expect::Pass,
vec!["--query --available --json", "--version"], vec!["--query --available --json", "--version"],
); );
} }
#[test] #[tokio::test]
fn test_query_packages_outputs() { async fn test_query_packages_outputs() {
let nix = nix(); let nix = nix();
let op = noop(Operation::QueryPackagesOutputs); let op = noop(Operation::QueryPackagesOutputs);
assert_eq!(op.to_string(), "nix-env -qaP --no-name --out-path"); assert_eq!(op.to_string(), "nix-env -qaP --no-name --out-path");
let ret: Result<fs::File, fs::File> = nix.run( let ret: Result<fs::File, fs::File> = nix
.run(
nix.safe_command(&op, build_path().as_path(), &["--version"], &[]), nix.safe_command(&op, build_path().as_path(), &["--version"], &[]),
true, true,
); )
.await;
assert_run( assert_run(
ret, ret,
Expect::Pass, &Expect::Pass,
vec![ vec![
"--query --available --no-name --attr-path --out-path", "--query --available --no-name --attr-path --out-path",
"--version", "--version",
@ -617,18 +636,20 @@ mod tests {
); );
} }
#[test] #[tokio::test]
fn safe_command_environment() { async fn safe_command_environment() {
let nix = nix(); let nix = nix();
let ret: Result<fs::File, fs::File> = nix.run( let ret: Result<fs::File, fs::File> = nix
.run(
nix.safe_command::<&OsStr>(&env_noop(), build_path().as_path(), &[], &[]), nix.safe_command::<&OsStr>(&env_noop(), build_path().as_path(), &[], &[]),
true, true,
); )
.await;
assert_run( assert_run(
ret, ret,
Expect::Pass, &Expect::Pass,
vec![ vec![
"HOME=/homeless-shelter", "HOME=/homeless-shelter",
"NIX_PATH=ofborg-nixpkgs-pr=", "NIX_PATH=ofborg-nixpkgs-pr=",
@ -638,19 +659,21 @@ mod tests {
); );
} }
#[test] #[tokio::test]
fn safe_command_custom_gc() { async fn safe_command_custom_gc() {
let remote = env::var("NIX_REMOTE").unwrap_or("".to_owned()); let remote = env::var("NIX_REMOTE").unwrap_or_default();
let nix = Nix::new(SYSTEM.to_owned(), remote, 1800, Some("4g".to_owned())); let nix = Nix::new(SYSTEM.to_owned(), remote, 1800, Some("4g".to_owned()));
let ret: Result<fs::File, fs::File> = nix.run( let ret: Result<fs::File, fs::File> = nix
.run(
nix.safe_command::<&OsStr>(&env_noop(), build_path().as_path(), &[], &[]), nix.safe_command::<&OsStr>(&env_noop(), build_path().as_path(), &[], &[]),
true, true,
); )
.await;
assert_run( assert_run(
ret, ret,
Expect::Pass, &Expect::Pass,
vec![ vec![
"HOME=/homeless-shelter", "HOME=/homeless-shelter",
"NIX_PATH=ofborg-nixpkgs-pr=", "NIX_PATH=ofborg-nixpkgs-pr=",
@ -661,57 +684,59 @@ mod tests {
); );
} }
#[test] #[tokio::test]
fn safe_command_options() { async fn safe_command_options() {
let nix = nix(); let nix = nix();
let op = noop(Operation::Build); let op = noop(Operation::Build);
let ret: Result<fs::File, fs::File> = nix.run( let ret: Result<fs::File, fs::File> = nix
.run(
nix.safe_command::<&OsStr>(&op, build_path().as_path(), &[], &[]), nix.safe_command::<&OsStr>(&op, build_path().as_path(), &[], &[]),
true, true,
); )
.await;
assert_run( assert_run(
ret, ret,
Expect::Pass, &Expect::Pass,
vec!["--option restrict-eval true", "--option build-timeout 1800"], vec!["--option restrict-eval true", "--option build-timeout 1800"],
); );
} }
#[test] #[tokio::test]
fn set_attrs_nixpkgs() { async fn set_attrs_nixpkgs() {
let nix = nix(); let nix = nix();
let op = noop(Operation::Build); let op = noop(Operation::Build);
let mut command = nix.safe_command::<&OsStr>(&op, build_path().as_path(), &[], &[]); let mut command = nix.safe_command::<&OsStr>(&op, build_path().as_path(), &[], &[]);
nix.set_attrs_command( Nix::set_attrs_command(
&mut command, &mut command,
File::DefaultNixpkgs, File::DefaultNixpkgs,
vec!["foo".into(), "bar".into()], vec!["foo".into(), "bar".into()],
); );
let ret: Result<fs::File, fs::File> = nix.run(command, true); let ret: Result<fs::File, fs::File> = nix.run(command, true).await;
assert_run(ret, Expect::Pass, vec!["./default.nix", "-A foo -A bar"]); assert_run(ret, &Expect::Pass, vec!["./default.nix", "-A foo -A bar"]);
} }
#[test] #[tokio::test]
fn set_attrs_nixos() { async fn set_attrs_nixos() {
let nix = nix(); let nix = nix();
let op = noop(Operation::Instantiate); let op = noop(Operation::Instantiate);
let mut command = nix.safe_command::<&OsStr>(&op, build_path().as_path(), &[], &[]); let mut command = nix.safe_command::<&OsStr>(&op, build_path().as_path(), &[], &[]);
nix.set_attrs_command( Nix::set_attrs_command(
&mut command, &mut command,
File::ReleaseNixOS, File::ReleaseNixOS,
vec!["foo".into(), "bar".into()], vec!["foo".into(), "bar".into()],
); );
let ret: Result<fs::File, fs::File> = nix.run(command, true); let ret: Result<fs::File, fs::File> = nix.run(command, true).await;
assert_run( assert_run(
ret, ret,
Expect::Pass, &Expect::Pass,
vec![ vec![
"./nixos/release.nix", "./nixos/release.nix",
"--arg nixpkgs { outPath=./.; revCount=999999; shortRev=\"ofborg\"; rev=\"0000000000000000000000000000000000000000\"; }", "--arg nixpkgs { outPath=./.; revCount=999999; shortRev=\"ofborg\"; rev=\"0000000000000000000000000000000000000000\"; }",
@ -719,36 +744,40 @@ mod tests {
); );
} }
#[test] #[tokio::test]
fn safely_build_attrs_success() { async fn safely_build_attrs_success() {
let nix = nix(); let nix = nix();
let ret: Result<fs::File, fs::File> = nix.safely_build_attrs( let ret: Result<fs::File, fs::File> = nix
.safely_build_attrs(
build_path().as_path(), build_path().as_path(),
File::DefaultNixpkgs, File::DefaultNixpkgs,
vec![String::from("success")], vec![String::from("success")],
); )
.await;
assert_run( assert_run(
ret, ret,
Expect::Pass, &Expect::Pass,
vec!["-success.drv", "building ", "hi", "-success"], vec!["-success.drv", "building ", "hi", "-success"],
); );
} }
#[test] #[tokio::test]
fn safely_build_attrs_failure() { async fn safely_build_attrs_failure() {
let nix = nix(); let nix = nix();
let ret: Result<fs::File, fs::File> = nix.safely_build_attrs( let ret: Result<fs::File, fs::File> = nix
.safely_build_attrs(
build_path().as_path(), build_path().as_path(),
File::DefaultNixpkgs, File::DefaultNixpkgs,
vec![String::from("failed")], vec![String::from("failed")],
); )
.await;
assert_run( assert_run(
ret, ret,
Expect::Fail, &Expect::Fail,
vec![ vec![
"-failed.drv", "-failed.drv",
"building ", "building ",
@ -758,8 +787,8 @@ mod tests {
); );
} }
#[test] #[tokio::test]
fn partition_instantiable_attributes() { async fn partition_instantiable_attributes() {
let nix = nix(); let nix = nix();
let ret: (Vec<String>, Vec<(String, Vec<String>)>) = nix let ret: (Vec<String>, Vec<(String, Vec<String>)>) = nix
@ -771,7 +800,8 @@ mod tests {
String::from("passes-instantiation"), String::from("passes-instantiation"),
String::from("missing-attr"), String::from("missing-attr"),
], ],
); )
.await;
assert_eq!(ret.0, vec!["passes-instantiation"]); assert_eq!(ret.0, vec!["passes-instantiation"]);
@ -790,80 +820,90 @@ mod tests {
); );
} }
#[test] #[tokio::test]
fn safely_instantiate_attrs_failure() { async fn safely_instantiate_attrs_failure() {
let nix = nix(); let nix = nix();
let ret: Result<fs::File, fs::File> = nix.safely_instantiate_attrs( let ret: Result<fs::File, fs::File> = nix
.safely_instantiate_attrs(
individual_eval_path().as_path(), individual_eval_path().as_path(),
File::DefaultNixpkgs, File::DefaultNixpkgs,
vec![String::from("fails-instantiation")], vec![String::from("fails-instantiation")],
); )
.await;
assert_run( assert_run(
ret, ret,
Expect::Fail, &Expect::Fail,
vec!["You just can't", "assertion", "failed"], vec!["You just can't", "assertion", "failed"],
); );
} }
#[test] #[tokio::test]
fn safely_instantiate_attrs_success() { async fn safely_instantiate_attrs_success() {
let nix = nix(); let nix = nix();
let ret: Result<fs::File, fs::File> = nix.safely_instantiate_attrs( let ret: Result<fs::File, fs::File> = nix
.safely_instantiate_attrs(
individual_eval_path().as_path(), individual_eval_path().as_path(),
File::DefaultNixpkgs, File::DefaultNixpkgs,
vec![String::from("passes-instantiation")], vec![String::from("passes-instantiation")],
); )
.await;
assert_run(ret, Expect::Pass, vec!["-passes-instantiation.drv"]); assert_run(ret, &Expect::Pass, vec!["-passes-instantiation.drv"]);
} }
#[test] #[tokio::test]
fn safely_evaluate_expr_success() { async fn safely_evaluate_expr_success() {
let nix = nix(); let nix = nix();
let ret: Result<fs::File, fs::File> = nix.run( let ret: Result<fs::File, fs::File> = nix
.run(
nix.safely_evaluate_expr_cmd( nix.safely_evaluate_expr_cmd(
individual_eval_path().as_path(), individual_eval_path().as_path(),
r#"{ foo ? "bar" }: "The magic value is ${foo}""#, r#"{ foo ? "bar" }: "The magic value is ${foo}""#,
[("foo", "tux")].iter().cloned().collect(), [("foo", "tux")].iter().copied().collect(),
&[], &[],
), ),
true, true,
); )
.await;
assert_run(ret, Expect::Pass, vec!["The magic value is tux"]); assert_run(ret, &Expect::Pass, vec!["The magic value is tux"]);
} }
#[test] #[tokio::test]
fn strict_sandboxing() { async fn strict_sandboxing() {
let ret: Result<fs::File, fs::File> = nix().safely_build_attrs( let ret: Result<fs::File, fs::File> = nix()
.safely_build_attrs(
build_path().as_path(), build_path().as_path(),
File::DefaultNixpkgs, File::DefaultNixpkgs,
vec![String::from("sandbox-violation")], vec![String::from("sandbox-violation")],
); )
.await;
assert_run( assert_run(
ret, ret,
Expect::Fail, &Expect::Fail,
vec!["access to absolute path", "is forbidden in restricted mode"], vec!["access to absolute path", "is forbidden in restricted mode"],
); );
} }
#[test] #[tokio::test]
fn instantiation_success() { async fn instantiation_success() {
let ret: Result<fs::File, fs::File> = nix().safely( let ret: Result<fs::File, fs::File> = nix()
.safely(
&Operation::Instantiate, &Operation::Instantiate,
passing_eval_path().as_path(), passing_eval_path().as_path(),
vec![], vec![],
true, true,
); )
.await;
assert_run( assert_run(
ret, ret,
Expect::Pass, &Expect::Pass,
vec![ vec![
"the result might be removed by the garbage collector", "the result might be removed by the garbage collector",
"-failed.drv", "-failed.drv",
@ -872,18 +912,20 @@ mod tests {
); );
} }
#[test] #[tokio::test]
fn instantiation_nixpkgs_restricted_mode() { async fn instantiation_nixpkgs_restricted_mode() {
let ret: Result<fs::File, fs::File> = nix().safely( let ret: Result<fs::File, fs::File> = nix()
.safely(
&Operation::Instantiate, &Operation::Instantiate,
individual_eval_path().as_path(), individual_eval_path().as_path(),
vec![String::from("-A"), String::from("nixpkgs-restricted-mode")], vec![String::from("-A"), String::from("nixpkgs-restricted-mode")],
true, true,
); )
.await;
assert_run( assert_run(
ret, ret,
Expect::Fail, &Expect::Fail,
vec![ vec![
"access to absolute path '/fake'", "access to absolute path '/fake'",
"is forbidden in restricted mode", "is forbidden in restricted mode",

View file

@ -17,6 +17,7 @@ pub struct HydraNixEnv {
} }
impl HydraNixEnv { impl HydraNixEnv {
#[must_use]
pub fn new(nix: nix::Nix, path: PathBuf, check_meta: bool) -> HydraNixEnv { pub fn new(nix: nix::Nix, path: PathBuf, check_meta: bool) -> HydraNixEnv {
HydraNixEnv { HydraNixEnv {
path, path,
@ -25,11 +26,11 @@ impl HydraNixEnv {
} }
} }
pub fn execute_with_stats( pub async fn execute_with_stats(
&self, &self,
) -> Result<(outpathdiff::PackageOutPaths, EvaluationStats), Error> { ) -> Result<(outpathdiff::PackageOutPaths, EvaluationStats), Error> {
self.place_nix()?; self.place_nix()?;
let (status, stdout, stderr, stats) = self.run_nix_env(); let (status, stdout, stderr, stats) = self.run_nix_env().await;
self.remove_nix()?; self.remove_nix()?;
if status { if status {
@ -79,7 +80,7 @@ impl HydraNixEnv {
// when it fails to evaluate something. In this case, we can ignore (but // when it fails to evaluate something. In this case, we can ignore (but
// warn about) the error. // warn about) the error.
if let Err(e) = fs::remove_file(&outpath_stats) { if let Err(e) = fs::remove_file(&outpath_stats) {
warn!("Failed to remove file {:?}: {:?}", outpath_stats, e) warn!("Failed to remove file {:?}: {:?}", outpath_stats, e);
} }
Ok(()) Ok(())
@ -93,7 +94,7 @@ impl HydraNixEnv {
self.path.join(".gc-of-borg-stats.json") self.path.join(".gc-of-borg-stats.json")
} }
fn run_nix_env(&self) -> (bool, File, File, Result<File, io::Error>) { async fn run_nix_env(&self) -> (bool, File, File, Result<File, io::Error>) {
let check_meta = if self.check_meta { "true" } else { "false" }; let check_meta = if self.check_meta { "true" } else { "false" };
let mut cmd = self.nix.safe_command( let mut cmd = self.nix.safe_command(
@ -111,7 +112,7 @@ impl HydraNixEnv {
cmd.env("NIX_SHOW_STATS", "1"); cmd.env("NIX_SHOW_STATS", "1");
cmd.env("NIX_SHOW_STATS_PATH", self.outpath_stats_path()); cmd.env("NIX_SHOW_STATS_PATH", self.outpath_stats_path());
let (status, stdout, stderr) = self.nix.run_stderr_stdout(cmd); let (status, stdout, stderr) = self.nix.run_stderr_stdout(cmd).await;
let stats = File::open(self.outpath_stats_path()); let stats = File::open(self.outpath_stats_path());
(status, stdout, stderr, stats) (status, stdout, stderr, stats)
@ -135,13 +136,14 @@ impl From<io::Error> for Error {
} }
impl Error { impl Error {
#[must_use]
pub fn display(self) -> String { pub fn display(self) -> String {
match self { match self {
Error::Io(e) => format!("Failed during the setup of executing nix-env: {:?}", e), Error::Io(e) => format!("Failed during the setup of executing nix-env: {e:?}"),
Error::CreateFile(path, err) => format!("Failed to create file {:?}: {:?}", path, err), Error::CreateFile(path, err) => format!("Failed to create file {path:?}: {err:?}"),
Error::RemoveFile(path, err) => format!("Failed to remove file {:?}: {:?}", path, err), Error::RemoveFile(path, err) => format!("Failed to remove file {path:?}: {err:?}"),
Error::WriteFile(file, err) => { Error::WriteFile(file, err) => {
format!("Failed to write to file '{:?}': {:?}", file, err) format!("Failed to write to file '{file:?}': {err:?}")
} }
Error::CommandFailed(mut fd) => { Error::CommandFailed(mut fd) => {
let mut buffer = Vec::new(); let mut buffer = Vec::new();
@ -149,15 +151,14 @@ impl Error {
let bufstr = String::from_utf8_lossy(&buffer); let bufstr = String::from_utf8_lossy(&buffer);
match read_result { match read_result {
Ok(_) => format!("nix-env failed:\n{}", bufstr), Ok(_) => format!("nix-env failed:\n{bufstr}"),
Err(e) => format!( Err(e) => format!(
"nix-env failed and loading the error result caused a new error {:?}\n\n{}", "nix-env failed and loading the error result caused a new error {e:?}\n\n{bufstr}"
e, bufstr
), ),
} }
} }
Error::UncleanEvaluation(warnings) => { Error::UncleanEvaluation(warnings) => {
format!("nix-env did not evaluate cleanly:\n {:?}", warnings) format!("nix-env did not evaluate cleanly:\n {warnings:?}")
} }
Error::StatsParse(mut fd, seek, parse_err) => { Error::StatsParse(mut fd, seek, parse_err) => {
let mut buffer = Vec::new(); let mut buffer = Vec::new();
@ -169,21 +170,19 @@ impl Error {
if let Err(seek_err) = seek { if let Err(seek_err) = seek {
lines.push_str(&format!( lines.push_str(&format!(
"Additionally, resetting to the beginning of the output failed with:\n{:?}\n\n", "Additionally, resetting to the beginning of the output failed with:\n{seek_err:?}\n\n"
seek_err
)); ));
} }
if let Err(read_err) = read_result { if let Err(read_err) = read_result {
lines.push_str(&format!( lines.push_str(&format!(
"Additionally, loading the output failed with:\n{:?}\n\n", "Additionally, loading the output failed with:\n{read_err:?}\n\n"
read_err
)); ));
} }
lines.push_str(&format!("Parse error:\n{:?}\n\n", parse_err)); lines.push_str(&format!("Parse error:\n{parse_err:?}\n\n"));
lines.push_str(&format!("Evaluation output:\n{}", bufstr)); lines.push_str(&format!("Evaluation output:\n{bufstr}"));
lines lines
} }

View file

@ -1,4 +1,4 @@
//! Statistics emitted by Nix when NIX_SHOW_STATS=1 //! Statistics emitted by Nix when `NIX_SHOW_STATS=1`
use separator::Separatable; use separator::Separatable;
use std::collections::HashMap; use std::collections::HashMap;
@ -102,23 +102,14 @@ pub struct EvaluationStatsDiff<'a> {
right: &'a EvaluationStats, right: &'a EvaluationStats,
} }
impl<'a> EvaluationStatsDiff<'a> { struct Row {
pub fn compare(
left: &'a EvaluationStats,
right: &'a EvaluationStats,
) -> EvaluationStatsDiff<'a> {
EvaluationStatsDiff { left, right }
}
pub fn markdown(&self) -> String {
struct Row {
before: String, before: String,
after: String, after: String,
diff: String, diff: String,
diff_pct: String, diff_pct: String,
} }
impl Row { impl Row {
fn from_u64(left: u64, right: u64) -> Row { fn from_u64(left: u64, right: u64) -> Row {
let (diff, direction): (u64, _) = match left.cmp(&right) { let (diff, direction): (u64, _) = match left.cmp(&right) {
std::cmp::Ordering::Greater => (left - right, ""), std::cmp::Ordering::Greater => (left - right, ""),
@ -126,13 +117,14 @@ impl<'a> EvaluationStatsDiff<'a> {
std::cmp::Ordering::Equal => (0, ""), std::cmp::Ordering::Equal => (0, ""),
}; };
#[allow(clippy::cast_precision_loss)]
let diff_pct: String = if diff > 0 { let diff_pct: String = if diff > 0 {
format!( format!(
"{:.2}%", "{:.2}%",
((right as f64) - (left as f64)) / (left as f64) * 100.0 ((right as f64) - (left as f64)) / (left as f64) * 100.0
) )
} else { } else {
String::from("") String::new()
}; };
Row { Row {
@ -150,27 +142,39 @@ impl<'a> EvaluationStatsDiff<'a> {
{ {
std::cmp::Ordering::Greater => (left - right, ""), std::cmp::Ordering::Greater => (left - right, ""),
std::cmp::Ordering::Less => (right - left, ""), std::cmp::Ordering::Less => (right - left, ""),
std::cmp::Ordering::Equal => (0 as f32, ""), std::cmp::Ordering::Equal => (0.0, ""),
}; };
let diff_pct: String = if diff > 0 as _ { let diff_pct: String = if diff > 0.0 {
format!( format!(
"{:.2}%", "{:.2}%",
((right as f64) - (left as f64)) / (left as f64) * 100.0 (f64::from(right) - f64::from(left)) / f64::from(left) * 100.0
) )
} else { } else {
String::from("") String::new()
}; };
Row { Row {
before: format!("{:.2}", left), before: format!("{left:.2}"),
after: format!("{:.2}", right), after: format!("{right:.2}"),
diff: format!("{}{:.2}", direction, diff), diff: format!("{direction}{diff:.2}"),
diff_pct, diff_pct,
} }
} }
}
impl<'a> EvaluationStatsDiff<'a> {
#[must_use]
pub fn compare(
left: &'a EvaluationStats,
right: &'a EvaluationStats,
) -> EvaluationStatsDiff<'a> {
EvaluationStatsDiff { left, right }
} }
#[must_use]
#[allow(clippy::too_many_lines)]
pub fn markdown(&self) -> String {
let mut data: HashMap<&str, Row> = HashMap::new(); let mut data: HashMap<&str, Row> = HashMap::new();
data.insert( data.insert(
"cpuTime", "cpuTime",
@ -310,7 +314,7 @@ impl<'a> EvaluationStatsDiff<'a> {
}, },
); );
let mut keys = data.keys().cloned().collect::<Vec<&str>>(); let mut keys = data.keys().copied().collect::<Vec<&str>>();
keys.sort_unstable(); keys.sort_unstable();
let rows = keys let rows = keys
@ -442,53 +446,54 @@ mod tests {
"#; "#;
#[test] #[test]
#[allow(clippy::float_cmp)] // FIXME
fn verify_load() { fn verify_load() {
let load: EvaluationStats = serde_json::from_str(EXAMPLE).unwrap(); let load: EvaluationStats = serde_json::from_str(EXAMPLE).unwrap();
assert_eq!(load.cpu_time, 135.2); assert_eq!(load.cpu_time, 135.2);
assert_eq!(load.envs.number, 130714125); assert_eq!(load.envs.number, 130_714_125);
assert_eq!(load.envs.elements, 183953876); assert_eq!(load.envs.elements, 183_953_876);
assert_eq!(load.envs.bytes, 3563057008); assert_eq!(load.envs.bytes, 3_563_057_008);
assert_eq!(load.list.elements, 207421516); assert_eq!(load.list.elements, 207_421_516);
assert_eq!(load.list.bytes, 1659372128); assert_eq!(load.list.bytes, 1_659_372_128);
assert_eq!(load.list.concats, 7194150); assert_eq!(load.list.concats, 7_194_150);
assert_eq!(load.values.number, 260454370); assert_eq!(load.values.number, 260_454_370);
assert_eq!(load.values.bytes, 6250904880); assert_eq!(load.values.bytes, 6_250_904_880);
assert_eq!(load.symbols.number, 372918); assert_eq!(load.symbols.number, 372_918);
assert_eq!(load.symbols.bytes, 16324262); assert_eq!(load.symbols.bytes, 16_324_262);
assert_eq!(load.sets.number, 27310541); assert_eq!(load.sets.number, 27_310_541);
assert_eq!(load.sets.bytes, 7134676648); assert_eq!(load.sets.bytes, 7_134_676_648);
assert_eq!(load.sets.elements, 288174680); assert_eq!(load.sets.elements, 288_174_680);
assert_eq!(load.sizes.env, 16); assert_eq!(load.sizes.env, 16);
assert_eq!(load.sizes.value, 24); assert_eq!(load.sizes.value, 24);
assert_eq!(load.sizes.bindings, 8); assert_eq!(load.sizes.bindings, 8);
assert_eq!(load.sizes.attr, 24); assert_eq!(load.sizes.attr, 24);
assert_eq!(load.nr_op_updates, 11883339); assert_eq!(load.nr_op_updates, 11_883_339);
assert_eq!(load.nr_op_update_values_copied, 208834564); assert_eq!(load.nr_op_update_values_copied, 208_834_564);
assert_eq!(load.nr_thunks, 173325665); assert_eq!(load.nr_thunks, 173_325_665);
assert_eq!(load.nr_avoided, 177840681); assert_eq!(load.nr_avoided, 177_840_681);
assert_eq!(load.nr_lookups, 75292052); assert_eq!(load.nr_lookups, 75_292_052);
assert_eq!(load.nr_prim_op_calls, 85571252); assert_eq!(load.nr_prim_op_calls, 85_571_252);
assert_eq!(load.nr_function_calls, 115193164); assert_eq!(load.nr_function_calls, 115_193_164);
assert_eq!(load.gc.heap_size, 12104687616); assert_eq!(load.gc.heap_size, 12_104_687_616);
assert_eq!(load.gc.total_bytes, 24191819392); assert_eq!(load.gc.total_bytes, 24_191_819_392);
} }
fn diff_text(left: &str, right: &str) { fn diff_text(left: &str, right: &str) {
println!("left:\n{}", left); println!("left:\n{left}");
println!("right:\n{}", right); println!("right:\n{right}");
let lines = left.split('\n').zip(right.split('\n')); let lines = left.split('\n').zip(right.split('\n'));
for (idx, (linea, lineb)) in lines.enumerate() { for (idx, (linea, lineb)) in lines.enumerate() {
assert_eq!(linea, lineb, "Line {}", idx); assert_eq!(linea, lineb, "Line {idx}");
} }
} }
@ -499,7 +504,7 @@ mod tests {
diff_text( diff_text(
&EvaluationStatsDiff::compare(&left, &right).markdown(), &EvaluationStatsDiff::compare(&left, &right).markdown(),
r#" r"
| stat | before | after | Δ | Δ% | | stat | before | after | Δ | Δ% |
|:---------------------------|---------------:|---------------:|:--------------|-------:| |:---------------------------|---------------:|---------------:|:--------------|-------:|
| **cpuTime** | 135.20 | 132.90 | 2.30 | -1.70% | | **cpuTime** | 135.20 | 132.90 | 2.30 | -1.70% |
@ -529,7 +534,7 @@ mod tests {
| **symbols-number** | 372,918 | 372,917 | 1 | -0.00% | | **symbols-number** | 372,918 | 372,917 | 1 | -0.00% |
| **values-bytes** | 6,250,904,880 | 5,869,027,296 | 381,877,584 | -6.11% | | **values-bytes** | 6,250,904,880 | 5,869,027,296 | 381,877,584 | -6.11% |
| **values-number** | 260,454,370 | 244,542,804 | 15,911,566 | -6.11% | | **values-number** | 260,454,370 | 244,542,804 | 15,911,566 | -6.11% |
"# "
.trim_start(), .trim_start(),
); );
} }

View file

@ -1,11 +1,13 @@
use crate::worker::Action; use crate::worker::Action;
use async_trait::async_trait;
pub trait SimpleNotifyWorker { #[async_trait]
type J; pub trait SimpleNotifyWorker: Send + Sync {
type J: Send + Sync;
fn consumer(&self, job: &Self::J, notifier: &mut dyn NotificationReceiver); async fn consumer(&self, job: &Self::J, notifier: &mut dyn NotificationReceiver);
fn msg_to_job( async fn msg_to_job(
&self, &self,
routing_key: &str, routing_key: &str,
content_type: &Option<String>, content_type: &Option<String>,
@ -13,8 +15,9 @@ pub trait SimpleNotifyWorker {
) -> Result<Self::J, String>; ) -> Result<Self::J, String>;
} }
pub trait NotificationReceiver { #[async_trait]
fn tell(&mut self, action: Action); pub trait NotificationReceiver: Send + Sync {
async fn tell(&mut self, action: Action);
} }
#[derive(Default)] #[derive(Default)]
@ -23,13 +26,15 @@ pub struct DummyNotificationReceiver {
} }
impl DummyNotificationReceiver { impl DummyNotificationReceiver {
#[must_use]
pub fn new() -> DummyNotificationReceiver { pub fn new() -> DummyNotificationReceiver {
Default::default() DummyNotificationReceiver::default()
} }
} }
#[async_trait]
impl NotificationReceiver for DummyNotificationReceiver { impl NotificationReceiver for DummyNotificationReceiver {
fn tell(&mut self, action: Action) { async fn tell(&mut self, action: Action) {
self.actions.push(action); self.actions.push(action);
} }
} }

View file

@ -15,6 +15,7 @@ pub struct OutPathDiff {
} }
impl OutPathDiff { impl OutPathDiff {
#[must_use]
pub fn new(nix: nix::Nix, path: PathBuf) -> OutPathDiff { pub fn new(nix: nix::Nix, path: PathBuf) -> OutPathDiff {
OutPathDiff { OutPathDiff {
calculator: HydraNixEnv::new(nix, path, false), calculator: HydraNixEnv::new(nix, path, false),
@ -23,21 +24,22 @@ impl OutPathDiff {
} }
} }
pub fn find_before(&mut self) -> Result<(), NixEnvError> { pub async fn find_before(&mut self) -> Result<(), NixEnvError> {
self.original = Some(self.run()?); self.original = Some(self.run().await?);
Ok(()) Ok(())
} }
pub fn find_after(&mut self) -> Result<(), NixEnvError> { pub async fn find_after(&mut self) -> Result<(), NixEnvError> {
if self.original.is_none() { if self.original.is_none() {
debug!("Before is None, not bothering with After"); debug!("Before is None, not bothering with After");
return Ok(()); return Ok(());
} }
self.current = Some(self.run()?); self.current = Some(self.run().await?);
Ok(()) Ok(())
} }
#[must_use]
pub fn performance_diff(&self) -> Option<EvaluationStatsDiff> { pub fn performance_diff(&self) -> Option<EvaluationStatsDiff> {
if let Some((_, ref cur)) = self.current { if let Some((_, ref cur)) = self.current {
if let Some((_, ref orig)) = self.original { if let Some((_, ref orig)) = self.original {
@ -50,6 +52,7 @@ impl OutPathDiff {
} }
} }
#[must_use]
pub fn package_diff(&self) -> Option<(Vec<PackageArch>, Vec<PackageArch>)> { pub fn package_diff(&self) -> Option<(Vec<PackageArch>, Vec<PackageArch>)> {
if let Some((ref cur, _)) = self.current { if let Some((ref cur, _)) = self.current {
if let Some((ref orig, _)) = self.original { if let Some((ref orig, _)) = self.original {
@ -80,11 +83,11 @@ impl OutPathDiff {
if let Some((ref orig, _)) = self.original { if let Some((ref orig, _)) = self.original {
for key in cur.keys() { for key in cur.keys() {
trace!("Checking out {:?}", key); trace!("Checking out {:?}", key);
if cur.get(key) != orig.get(key) { if cur.get(key) == orig.get(key) {
trace!(" {:?} != {:?}", cur.get(key), orig.get(key));
rebuild.push(key.clone())
} else {
trace!(" {:?} == {:?}", cur.get(key), orig.get(key)); trace!(" {:?} == {:?}", cur.get(key), orig.get(key));
} else {
trace!(" {:?} != {:?}", cur.get(key), orig.get(key));
rebuild.push(key.clone());
} }
} }
@ -95,8 +98,8 @@ impl OutPathDiff {
None None
} }
fn run(&mut self) -> Result<(PackageOutPaths, EvaluationStats), NixEnvError> { async fn run(&mut self) -> Result<(PackageOutPaths, EvaluationStats), NixEnvError> {
self.calculator.execute_with_stats() self.calculator.execute_with_stats().await
} }
} }

View file

@ -1,16 +1,15 @@
use futures::executor::block_on; use async_trait::async_trait;
use lapin::options::BasicPublishOptions; use lapin::options::BasicPublishOptions;
include!(concat!(env!("OUT_DIR"), "/events.rs")); #[allow(clippy::all, clippy::pedantic)]
mod events {
#[macro_use] include!(concat!(env!("OUT_DIR"), "/events.rs"));
mod macros {
#[macro_export]
macro_rules! my_macro(() => (FooBar));
} }
pub use events::*;
#[async_trait]
pub trait SysEvents: Send { pub trait SysEvents: Send {
fn notify(&mut self, event: Event); async fn notify(&mut self, event: Event);
} }
#[derive(Serialize, Deserialize, Debug)] #[derive(Serialize, Deserialize, Debug)]
@ -25,6 +24,7 @@ pub struct RabbitMq<C> {
} }
impl RabbitMq<lapin::Channel> { impl RabbitMq<lapin::Channel> {
#[must_use]
pub fn from_lapin(identity: &str, channel: lapin::Channel) -> Self { pub fn from_lapin(identity: &str, channel: lapin::Channel) -> Self {
RabbitMq { RabbitMq {
identity: identity.to_owned(), identity: identity.to_owned(),
@ -33,10 +33,10 @@ impl RabbitMq<lapin::Channel> {
} }
} }
#[async_trait]
impl SysEvents for RabbitMq<lapin::Channel> { impl SysEvents for RabbitMq<lapin::Channel> {
fn notify(&mut self, event: Event) { async fn notify(&mut self, event: Event) {
let props = lapin::BasicProperties::default().with_content_type("application/json".into()); let props = lapin::BasicProperties::default().with_content_type("application/json".into());
block_on(async {
let _confirmaton = self let _confirmaton = self
.channel .channel
.basic_publish( .basic_publish(
@ -55,6 +55,5 @@ impl SysEvents for RabbitMq<lapin::Channel> {
.unwrap() .unwrap()
.await .await
.unwrap(); .unwrap();
});
} }
} }

View file

@ -18,16 +18,16 @@ impl std::fmt::Display for System {
} }
impl System { impl System {
#[must_use]
pub fn as_build_destination(&self) -> (Option<String>, Option<String>) { pub fn as_build_destination(&self) -> (Option<String>, Option<String>) {
(None, Some(format!("build-inputs-{}", self))) (None, Some(format!("build-inputs-{self}")))
} }
#[must_use]
pub fn can_run_nixos_tests(&self) -> bool { pub fn can_run_nixos_tests(&self) -> bool {
match self { match self {
System::X8664Linux => true, System::X8664Linux | System::Aarch64Linux => true,
System::Aarch64Linux => true, System::X8664Darwin | System::Aarch64Darwin => false,
System::X8664Darwin => false,
System::Aarch64Darwin => false,
} }
} }
} }

View file

@ -25,8 +25,9 @@ impl Default for StdenvTagger {
} }
impl StdenvTagger { impl StdenvTagger {
#[must_use]
pub fn new() -> StdenvTagger { pub fn new() -> StdenvTagger {
Default::default() StdenvTagger::default()
} }
pub fn changed(&mut self, systems: Vec<tasks::eval::stdenvs::System>) { pub fn changed(&mut self, systems: Vec<tasks::eval::stdenvs::System>) {
@ -42,19 +43,20 @@ impl StdenvTagger {
} }
for tag in &self.selected { for tag in &self.selected {
if !self.possible.contains(tag) { assert!(
panic!( self.possible.contains(tag),
"Tried to add label {} but it isn't in the possible list!", "Tried to add label {} but it isn't in the possible list!",
tag tag
); );
} }
} }
}
#[must_use]
pub fn tags_to_add(&self) -> Vec<String> { pub fn tags_to_add(&self) -> Vec<String> {
self.selected.clone() self.selected.clone()
} }
#[must_use]
pub fn tags_to_remove(&self) -> Vec<String> { pub fn tags_to_remove(&self) -> Vec<String> {
let mut remove = self.possible.clone(); let mut remove = self.possible.clone();
for tag in &self.selected { for tag in &self.selected {
@ -87,8 +89,9 @@ impl Default for PkgsAddedRemovedTagger {
} }
impl PkgsAddedRemovedTagger { impl PkgsAddedRemovedTagger {
#[must_use]
pub fn new() -> PkgsAddedRemovedTagger { pub fn new() -> PkgsAddedRemovedTagger {
Default::default() PkgsAddedRemovedTagger::default()
} }
pub fn changed(&mut self, removed: &[PackageArch], added: &[PackageArch]) { pub fn changed(&mut self, removed: &[PackageArch], added: &[PackageArch]) {
@ -101,10 +104,12 @@ impl PkgsAddedRemovedTagger {
} }
} }
#[must_use]
pub fn tags_to_add(&self) -> Vec<String> { pub fn tags_to_add(&self) -> Vec<String> {
self.selected.clone() self.selected.clone()
} }
#[must_use]
pub fn tags_to_remove(&self) -> Vec<String> { pub fn tags_to_remove(&self) -> Vec<String> {
// The cleanup tag is too vague to automatically remove. // The cleanup tag is too vague to automatically remove.
vec![] vec![]
@ -147,8 +152,9 @@ impl Default for RebuildTagger {
} }
impl RebuildTagger { impl RebuildTagger {
#[must_use]
pub fn new() -> RebuildTagger { pub fn new() -> RebuildTagger {
Default::default() RebuildTagger::default()
} }
pub fn parse_attrs(&mut self, attrs: Vec<PackageArch>) { pub fn parse_attrs(&mut self, attrs: Vec<PackageArch>) {
@ -160,12 +166,10 @@ impl RebuildTagger {
"x86_64-darwin" => { "x86_64-darwin" => {
counter_darwin += 1; counter_darwin += 1;
} }
"aarch64-darwin" => {}
"x86_64-linux" => { "x86_64-linux" => {
counter_linux += 1; counter_linux += 1;
} }
"aarch64-linux" => {} "aarch64-darwin" | "aarch64-linux" | "i686-linux" => {}
"i686-linux" => {}
arch => { arch => {
info!("Unknown arch: {:?}", arch); info!("Unknown arch: {:?}", arch);
} }
@ -176,35 +180,36 @@ impl RebuildTagger {
self.selected.extend( self.selected.extend(
RebuildTagger::bucket(counter_darwin) RebuildTagger::bucket(counter_darwin)
.iter() .iter()
.map(|bucket| format!("10.rebuild-darwin: {}", bucket)) .map(|bucket| format!("10.rebuild-darwin: {bucket}"))
.collect::<Vec<String>>(), .collect::<Vec<String>>(),
); );
self.selected.extend( self.selected.extend(
RebuildTagger::bucket(counter_linux) RebuildTagger::bucket(counter_linux)
.iter() .iter()
.map(|bucket| format!("10.rebuild-linux: {}", bucket)) .map(|bucket| format!("10.rebuild-linux: {bucket}"))
.collect::<Vec<String>>(), .collect::<Vec<String>>(),
); );
for tag in &self.selected { for tag in &self.selected {
if !self.possible.contains(tag) { assert!(
panic!( self.possible.contains(tag),
"Tried to add label {} but it isn't in the possible list!", "Tried to add label {} but it isn't in the possible list!",
tag tag
); );
} }
} }
}
#[must_use]
pub fn tags_to_add(&self) -> Vec<String> { pub fn tags_to_add(&self) -> Vec<String> {
self.selected.clone() self.selected.clone()
} }
#[must_use]
pub fn tags_to_remove(&self) -> Vec<String> { pub fn tags_to_remove(&self) -> Vec<String> {
let mut remove = vec![]; let mut remove = vec![];
for tag in self.possible.clone().into_iter() { for tag in self.possible.clone() {
if !self.selected.contains(&tag) { if !self.selected.contains(&tag) {
remove.push(tag); remove.push(tag);
} }
@ -254,8 +259,9 @@ impl Default for MaintainerPrTagger {
} }
impl MaintainerPrTagger { impl MaintainerPrTagger {
#[must_use]
pub fn new() -> MaintainerPrTagger { pub fn new() -> MaintainerPrTagger {
Default::default() MaintainerPrTagger::default()
} }
pub fn record_maintainer( pub fn record_maintainer(
@ -270,7 +276,7 @@ impl MaintainerPrTagger {
return; return;
} }
for (_package, maintainers) in identified_maintainers.0.iter() { for maintainers in identified_maintainers.0.values() {
if !maintainers.contains(&submitter) { if !maintainers.contains(&submitter) {
// One of the packages is not maintained by this submitter // One of the packages is not maintained by this submitter
return; return;
@ -281,10 +287,12 @@ impl MaintainerPrTagger {
.push(String::from("11.by: package-maintainer")); .push(String::from("11.by: package-maintainer"));
} }
#[must_use]
pub fn tags_to_add(&self) -> Vec<String> { pub fn tags_to_add(&self) -> Vec<String> {
self.selected.clone() self.selected.clone()
} }
#[must_use]
pub fn tags_to_remove(&self) -> Vec<String> { pub fn tags_to_remove(&self) -> Vec<String> {
// The cleanup tag is too vague to automatically remove. // The cleanup tag is too vague to automatically remove.
vec![] vec![]
@ -333,6 +341,7 @@ mod tests {
} }
#[test] #[test]
#[allow(clippy::too_many_lines)]
pub fn test_packages_changed() { pub fn test_packages_changed() {
let mut tagger = RebuildTagger::new(); let mut tagger = RebuildTagger::new();
tagger.parse_attrs(PackageArchSrc::linux(0).and_darwin(0).into()); tagger.parse_attrs(PackageArchSrc::linux(0).and_darwin(0).into());

View file

@ -8,6 +8,9 @@ use crate::worker;
use std::collections::VecDeque; use std::collections::VecDeque;
use async_trait::async_trait;
use tokio::io::AsyncBufReadExt;
use tokio::io::BufReader;
use tracing::{debug, debug_span, error, info}; use tracing::{debug, debug_span, error, info};
use uuid::Uuid; use uuid::Uuid;
@ -19,6 +22,7 @@ pub struct BuildWorker {
} }
impl BuildWorker { impl BuildWorker {
#[must_use]
pub fn new( pub fn new(
cloner: checkout::CachedCloner, cloner: checkout::CachedCloner,
nix: nix::Nix, nix: nix::Nix,
@ -47,7 +51,7 @@ pub struct JobActions<'a, 'b> {
identity: String, identity: String,
receiver: &'a mut dyn notifyworker::NotificationReceiver, receiver: &'a mut dyn notifyworker::NotificationReceiver,
job: &'b buildjob::BuildJob, job: &'b buildjob::BuildJob,
line_counter: u64, line_counter: usize,
snippet_log: VecDeque<String>, snippet_log: VecDeque<String>,
attempt_id: String, attempt_id: String,
log_exchange: Option<String>, log_exchange: Option<String>,
@ -88,23 +92,24 @@ impl<'a, 'b> JobActions<'a, 'b> {
} }
} }
#[must_use]
pub fn log_snippet(&self) -> Vec<String> { pub fn log_snippet(&self) -> Vec<String> {
self.snippet_log.clone().into() self.snippet_log.clone().into()
} }
pub fn pr_head_missing(&mut self) { pub async fn pr_head_missing(&mut self) {
self.tell(worker::Action::Ack); self.tell(worker::Action::Ack).await;
} }
pub fn commit_missing(&mut self) { pub async fn commit_missing(&mut self) {
self.tell(worker::Action::Ack); self.tell(worker::Action::Ack).await;
} }
pub fn nothing_to_do(&mut self) { pub async fn nothing_to_do(&mut self) {
self.tell(worker::Action::Ack); self.tell(worker::Action::Ack).await;
} }
pub fn merge_failed(&mut self) { pub async fn merge_failed(&mut self) {
let msg = BuildResult::V1 { let msg = BuildResult::V1 {
tag: V1Tag::V1, tag: V1Tag::V1,
repo: self.job.repo.clone(), repo: self.job.repo.clone(),
@ -122,14 +127,15 @@ impl<'a, 'b> JobActions<'a, 'b> {
let result_routing_key = self.result_routing_key.clone(); let result_routing_key = self.result_routing_key.clone();
self.tell(worker::publish_serde_action( self.tell(worker::publish_serde_action(
result_exchange, &result_exchange,
result_routing_key, &result_routing_key,
&msg, &msg,
)); ))
self.tell(worker::Action::Ack); .await;
self.tell(worker::Action::Ack).await;
} }
pub fn log_started(&mut self, can_build: Vec<String>, cannot_build: Vec<String>) { pub async fn log_started(&mut self, can_build: Vec<String>, cannot_build: Vec<String>) {
let msg = buildlogmsg::BuildLogStart { let msg = buildlogmsg::BuildLogStart {
identity: self.identity.clone(), identity: self.identity.clone(),
system: self.system.clone(), system: self.system.clone(),
@ -142,24 +148,26 @@ impl<'a, 'b> JobActions<'a, 'b> {
let log_routing_key = self.log_routing_key.clone(); let log_routing_key = self.log_routing_key.clone();
self.tell(worker::publish_serde_action( self.tell(worker::publish_serde_action(
log_exchange, &log_exchange,
log_routing_key, &log_routing_key,
&msg, &msg,
)); ))
.await;
} }
pub fn log_instantiation_errors(&mut self, cannot_build: Vec<(String, Vec<String>)>) { pub async fn log_instantiation_errors(&mut self, cannot_build: Vec<(String, Vec<String>)>) {
for (attr, log) in cannot_build { for (attr, log) in cannot_build {
self.log_line(&format!("Cannot nix-instantiate `{}' because:", &attr)); self.log_line(&format!("Cannot nix-instantiate `{}' because:", &attr))
.await;
for line in log { for line in log {
self.log_line(&line); self.log_line(&line).await;
} }
self.log_line(""); self.log_line("").await;
} }
} }
pub fn log_line(&mut self, line: &str) { pub async fn log_line(&mut self, line: &str) {
self.line_counter += 1; self.line_counter += 1;
if self.snippet_log.len() >= 10 { if self.snippet_log.len() >= 10 {
@ -179,13 +187,14 @@ impl<'a, 'b> JobActions<'a, 'b> {
let log_routing_key = self.log_routing_key.clone(); let log_routing_key = self.log_routing_key.clone();
self.tell(worker::publish_serde_action( self.tell(worker::publish_serde_action(
log_exchange, &log_exchange,
log_routing_key, &log_routing_key,
&msg, &msg,
)); ))
.await;
} }
pub fn build_not_attempted(&mut self, not_attempted_attrs: Vec<String>) { pub async fn build_not_attempted(&mut self, not_attempted_attrs: Vec<String>) {
let msg = BuildResult::V1 { let msg = BuildResult::V1 {
tag: V1Tag::V1, tag: V1Tag::V1,
repo: self.job.repo.clone(), repo: self.job.repo.clone(),
@ -202,23 +211,25 @@ impl<'a, 'b> JobActions<'a, 'b> {
let result_exchange = self.result_exchange.clone(); let result_exchange = self.result_exchange.clone();
let result_routing_key = self.result_routing_key.clone(); let result_routing_key = self.result_routing_key.clone();
self.tell(worker::publish_serde_action( self.tell(worker::publish_serde_action(
result_exchange, &result_exchange,
result_routing_key, &result_routing_key,
&msg, &msg,
)); ))
.await;
let log_exchange = self.log_exchange.clone(); let log_exchange = self.log_exchange.clone();
let log_routing_key = self.log_routing_key.clone(); let log_routing_key = self.log_routing_key.clone();
self.tell(worker::publish_serde_action( self.tell(worker::publish_serde_action(
log_exchange, &log_exchange,
log_routing_key, &log_routing_key,
&msg, &msg,
)); ))
.await;
self.tell(worker::Action::Ack); self.tell(worker::Action::Ack).await;
} }
pub fn build_finished( pub async fn build_finished(
&mut self, &mut self,
status: BuildStatus, status: BuildStatus,
attempted_attrs: Vec<String>, attempted_attrs: Vec<String>,
@ -240,31 +251,39 @@ impl<'a, 'b> JobActions<'a, 'b> {
let result_exchange = self.result_exchange.clone(); let result_exchange = self.result_exchange.clone();
let result_routing_key = self.result_routing_key.clone(); let result_routing_key = self.result_routing_key.clone();
self.tell(worker::publish_serde_action( self.tell(worker::publish_serde_action(
result_exchange, &result_exchange,
result_routing_key, &result_routing_key,
&msg, &msg,
)); ))
.await;
let log_exchange = self.log_exchange.clone(); let log_exchange = self.log_exchange.clone();
let log_routing_key = self.log_routing_key.clone(); let log_routing_key = self.log_routing_key.clone();
self.tell(worker::publish_serde_action( self.tell(worker::publish_serde_action(
log_exchange, &log_exchange,
log_routing_key, &log_routing_key,
&msg, &msg,
)); ))
.await;
self.tell(worker::Action::Ack); self.tell(worker::Action::Ack).await;
} }
fn tell(&mut self, action: worker::Action) { async fn tell(&mut self, action: worker::Action) {
self.receiver.tell(action); self.receiver.tell(action).await;
} }
} }
#[async_trait]
impl notifyworker::SimpleNotifyWorker for BuildWorker { impl notifyworker::SimpleNotifyWorker for BuildWorker {
type J = buildjob::BuildJob; type J = buildjob::BuildJob;
fn msg_to_job(&self, _: &str, _: &Option<String>, body: &[u8]) -> Result<Self::J, String> { async fn msg_to_job(
&self,
_: &str,
_: &Option<String>,
body: &[u8],
) -> Result<Self::J, String> {
info!("lmao I got a job?"); info!("lmao I got a job?");
match buildjob::from(body) { match buildjob::from(body) {
Ok(e) => Ok(e), Ok(e) => Ok(e),
@ -277,7 +296,7 @@ impl notifyworker::SimpleNotifyWorker for BuildWorker {
// FIXME: remove with rust/cargo update // FIXME: remove with rust/cargo update
#[allow(clippy::cognitive_complexity)] #[allow(clippy::cognitive_complexity)]
fn consumer( async fn consumer(
&self, &self,
job: &buildjob::BuildJob, job: &buildjob::BuildJob,
notifier: &mut dyn notifyworker::NotificationReceiver, notifier: &mut dyn notifyworker::NotificationReceiver,
@ -289,7 +308,7 @@ impl notifyworker::SimpleNotifyWorker for BuildWorker {
if job.attrs.is_empty() { if job.attrs.is_empty() {
debug!("No attrs to build"); debug!("No attrs to build");
actions.nothing_to_do(); actions.nothing_to_do().await;
return; return;
} }
@ -318,19 +337,19 @@ impl notifyworker::SimpleNotifyWorker for BuildWorker {
if co.fetch_pr(job.change.number).is_err() { if co.fetch_pr(job.change.number).is_err() {
info!("Failed to fetch {}", job.change.number); info!("Failed to fetch {}", job.change.number);
actions.pr_head_missing(); actions.pr_head_missing().await;
return; return;
} }
if !co.commit_exists(job.change.head_sha.as_ref()) { if !co.commit_exists(job.change.head_sha.as_ref()) {
info!("Commit {} doesn't exist", job.change.head_sha); info!("Commit {} doesn't exist", job.change.head_sha);
actions.commit_missing(); actions.commit_missing().await;
return; return;
} }
if co.merge_commit(job.change.head_sha.as_ref()).is_err() { if co.merge_commit(job.change.head_sha.as_ref()).is_err() {
info!("Failed to merge {}", job.change.head_sha); info!("Failed to merge {}", job.change.head_sha);
actions.merge_failed(); actions.merge_failed().await;
return; return;
} }
@ -338,11 +357,10 @@ impl notifyworker::SimpleNotifyWorker for BuildWorker {
"Got path: {:?}, determining which ones we can build ", "Got path: {:?}, determining which ones we can build ",
refpath refpath
); );
let (can_build, cannot_build) = self.nix.safely_partition_instantiable_attrs( let (can_build, cannot_build) = self
refpath.as_ref(), .nix
buildfile, .safely_partition_instantiable_attrs(refpath.as_ref(), buildfile, job.attrs.clone())
job.attrs.clone(), .await;
);
let cannot_build_attrs: Vec<String> = cannot_build let cannot_build_attrs: Vec<String> = cannot_build
.clone() .clone()
@ -356,11 +374,13 @@ impl notifyworker::SimpleNotifyWorker for BuildWorker {
cannot_build_attrs.join(", ") cannot_build_attrs.join(", ")
); );
actions.log_started(can_build.clone(), cannot_build_attrs.clone()); actions
actions.log_instantiation_errors(cannot_build); .log_started(can_build.clone(), cannot_build_attrs.clone())
.await;
actions.log_instantiation_errors(cannot_build).await;
if can_build.is_empty() { if can_build.is_empty() {
actions.build_not_attempted(cannot_build_attrs); actions.build_not_attempted(cannot_build_attrs).await;
return; return;
} }
@ -368,11 +388,12 @@ impl notifyworker::SimpleNotifyWorker for BuildWorker {
self.nix self.nix
.safely_build_attrs_async(refpath.as_ref(), buildfile, can_build.clone()); .safely_build_attrs_async(refpath.as_ref(), buildfile, can_build.clone());
for line in spawned.lines() { let mut reader = BufReader::new(spawned.stdout.take().unwrap()).lines();
actions.log_line(&line); while let Ok(Some(line)) = reader.next_line().await {
actions.log_line(&line).await;
} }
let status = nix::wait_for_build_status(spawned); let status = nix::wait_for_build_status(spawned).await;
info!("ok built ({:?}), building", status); info!("ok built ({:?}), building", status);
info!("Lines:"); info!("Lines:");
@ -384,7 +405,9 @@ impl notifyworker::SimpleNotifyWorker for BuildWorker {
.last(); .last();
info!("----->8-----"); info!("----->8-----");
actions.build_finished(status, can_build, cannot_build_attrs); actions
.build_finished(status, can_build, cannot_build_attrs)
.await;
info!("Build done!"); info!("Build done!");
} }
} }
@ -406,7 +429,7 @@ mod tests {
const SYSTEM: &str = "x86_64-darwin"; const SYSTEM: &str = "x86_64-darwin";
fn nix() -> nix::Nix { fn nix() -> nix::Nix {
let remote = env::var("NIX_REMOTE").unwrap_or("".to_owned()); let remote = env::var("NIX_REMOTE").unwrap_or_default();
nix::Nix::new("x86_64-linux".to_owned(), remote, 1800, None) nix::Nix::new("x86_64-linux".to_owned(), remote, 1800, None)
} }
@ -438,8 +461,7 @@ mod tests {
assert!(cmd.status().unwrap().success()); assert!(cmd.status().unwrap().success());
let output = cmd.output() let output = cmd.output().expect("building the test PR failed");
.expect("building the test PR failed");
let hash = String::from_utf8(output.stdout).expect("Should just be a hash"); let hash = String::from_utf8(output.stdout).expect("Should just be a hash");
hash.trim().to_owned() hash.trim().to_owned()
@ -453,23 +475,23 @@ mod tests {
} }
fn assert_contains_job(actions: &mut IntoIter<worker::Action>, text_to_match: &str) { fn assert_contains_job(actions: &mut IntoIter<worker::Action>, text_to_match: &str) {
println!("\n\n Searching: {:?}", text_to_match); println!("\n\n Searching: {text_to_match:?}");
actions actions
.position(|job| match job { .position(|job| match job {
worker::Action::Publish(ref body) => { worker::Action::Publish(ref body) => {
let content = String::from_utf8(body.content.clone()).unwrap(); let content = String::from_utf8(body.content.clone()).unwrap();
let text = strip_escaped_ansi(&content); let text = strip_escaped_ansi(&content);
eprintln!("{}", text); eprintln!("{text}");
if text.contains(text_to_match) { if text.contains(text_to_match) {
println!(" ok"); println!(" ok");
true true
} else { } else {
println!(" notContains: {}", text); println!(" notContains: {text}");
false false
} }
} }
e => { e => {
println!(" notPublish: {:?}", e); println!(" notPublish: {e:?}");
false false
} }
}) })
@ -481,8 +503,8 @@ mod tests {
}); });
} }
#[test] #[tokio::test]
pub fn test_simple_build() { pub async fn test_simple_build() {
let p = TestScratch::new_dir("build-simple-build-working"); let p = TestScratch::new_dir("build-simple-build-working");
let bare_repo = TestScratch::new_dir("build-simple-build-bare"); let bare_repo = TestScratch::new_dir("build-simple-build-bare");
let co_repo = TestScratch::new_dir("build-simple-build-co"); let co_repo = TestScratch::new_dir("build-simple-build-co");
@ -511,7 +533,7 @@ mod tests {
let mut dummyreceiver = notifyworker::DummyNotificationReceiver::new(); let mut dummyreceiver = notifyworker::DummyNotificationReceiver::new();
worker.consumer(&job, &mut dummyreceiver); worker.consumer(&job, &mut dummyreceiver).await;
println!("Total actions: {:?}", dummyreceiver.actions.len()); println!("Total actions: {:?}", dummyreceiver.actions.len());
let mut actions = dummyreceiver.actions.into_iter(); let mut actions = dummyreceiver.actions.into_iter();
@ -526,8 +548,8 @@ mod tests {
assert_eq!(actions.next(), Some(worker::Action::Ack)); assert_eq!(actions.next(), Some(worker::Action::Ack));
} }
#[test] #[tokio::test]
pub fn test_all_jobs_skipped() { pub async fn test_all_jobs_skipped() {
let p = TestScratch::new_dir("no-attempt"); let p = TestScratch::new_dir("no-attempt");
let bare_repo = TestScratch::new_dir("no-attempt-bare"); let bare_repo = TestScratch::new_dir("no-attempt-bare");
let co_repo = TestScratch::new_dir("no-attempt-co"); let co_repo = TestScratch::new_dir("no-attempt-co");
@ -556,7 +578,7 @@ mod tests {
let mut dummyreceiver = notifyworker::DummyNotificationReceiver::new(); let mut dummyreceiver = notifyworker::DummyNotificationReceiver::new();
worker.consumer(&job, &mut dummyreceiver); worker.consumer(&job, &mut dummyreceiver).await;
println!("Total actions: {:?}", dummyreceiver.actions.len()); println!("Total actions: {:?}", dummyreceiver.actions.len());
let mut actions = dummyreceiver.actions.into_iter(); let mut actions = dummyreceiver.actions.into_iter();

View file

@ -1,46 +0,0 @@
use crate::checkout::CachedProjectCo;
use crate::evalchecker::EvalChecker;
use crate::tasks::eval::{EvaluationComplete, EvaluationStrategy, StepResult};
use crate::vcs::commit_status::CommitStatus;
use std::path::Path;
#[derive(Default)]
pub struct GenericStrategy {}
impl GenericStrategy {
pub fn new() -> GenericStrategy {
Self {}
}
}
impl EvaluationStrategy for GenericStrategy {
fn pre_clone(&mut self) -> StepResult<()> {
Ok(())
}
fn on_target_branch(&mut self, _co: &Path, _status: &mut CommitStatus) -> StepResult<()> {
Ok(())
}
fn after_fetch(&mut self, _co: &CachedProjectCo) -> StepResult<()> {
Ok(())
}
fn merge_conflict(&mut self) {}
fn after_merge(&mut self, _status: &mut CommitStatus) -> StepResult<()> {
Ok(())
}
fn evaluation_checks(&self) -> Vec<EvalChecker> {
vec![]
}
fn all_evaluations_passed(
&mut self,
_co: &Path,
_status: &mut CommitStatus,
) -> StepResult<EvaluationComplete> {
Ok(Default::default())
}
}

View file

@ -1,33 +1,12 @@
mod generic;
mod nixpkgs; mod nixpkgs;
pub mod stdenvs; pub mod stdenvs;
pub use self::generic::GenericStrategy;
pub use self::nixpkgs::NixpkgsStrategy; pub use self::nixpkgs::NixpkgsStrategy;
pub use self::stdenvs::Stdenvs; pub use self::stdenvs::Stdenvs;
use crate::checkout::CachedProjectCo;
use crate::evalchecker::EvalChecker;
use crate::message::buildjob::BuildJob; use crate::message::buildjob::BuildJob;
use crate::vcs::commit_status::{CommitStatus, CommitStatusError}; use crate::vcs::commit_status::CommitStatusError;
use crate::vcs::generic::CheckRunOptions; use crate::vcs::generic::CheckRunOptions;
use std::path::Path;
pub trait EvaluationStrategy {
fn pre_clone(&mut self) -> StepResult<()>;
fn on_target_branch(&mut self, co: &Path, status: &mut CommitStatus) -> StepResult<()>;
fn after_fetch(&mut self, co: &CachedProjectCo) -> StepResult<()>;
fn merge_conflict(&mut self);
fn after_merge(&mut self, status: &mut CommitStatus) -> StepResult<()>;
fn evaluation_checks(&self) -> Vec<EvalChecker>;
fn all_evaluations_passed(
&mut self,
co: &Path,
status: &mut CommitStatus,
) -> StepResult<EvaluationComplete>;
}
pub type StepResult<T> = Result<T, Error>; pub type StepResult<T> = Result<T, Error>;
#[derive(Default)] #[derive(Default)]

View file

@ -9,19 +9,16 @@ use crate::nix::{self, Nix};
use crate::nixenv::HydraNixEnv; use crate::nixenv::HydraNixEnv;
use crate::outpathdiff::{OutPathDiff, PackageArch}; use crate::outpathdiff::{OutPathDiff, PackageArch};
use crate::tagger::{MaintainerPrTagger, PkgsAddedRemovedTagger, RebuildTagger, StdenvTagger}; use crate::tagger::{MaintainerPrTagger, PkgsAddedRemovedTagger, RebuildTagger, StdenvTagger};
use crate::tasks::eval::{ use crate::tasks::eval::{stdenvs::Stdenvs, Error, EvaluationComplete, StepResult};
stdenvs::Stdenvs, Error, EvaluationComplete, EvaluationStrategy, StepResult,
};
use crate::vcs::commit_status::CommitStatus; use crate::vcs::commit_status::CommitStatus;
use crate::vcs::generic::{ use crate::vcs::generic::{
CheckRunOptions, CheckRunState, Conclusion, State, VersionControlSystemAPI, CheckRunOptions, CheckRunState, Conclusion, State, VersionControlSystemAPI,
}; };
use std::path::Path; use std::path::Path;
use std::rc::Rc; use std::sync::Arc;
use chrono::Utc; use chrono::Utc;
use futures::executor::block_on;
use regex::Regex; use regex::Regex;
use tracing::{info, warn}; use tracing::{info, warn};
use uuid::Uuid; use uuid::Uuid;
@ -39,7 +36,7 @@ fn label_from_title(title: &str) -> Vec<String> {
let labels: Vec<_> = TITLE_LABELS let labels: Vec<_> = TITLE_LABELS
.iter() .iter()
.filter(|(word, _label)| { .filter(|(word, _label)| {
let re = Regex::new(&format!("\\b{}\\b", word)).unwrap(); let re = Regex::new(&format!("\\b{word}\\b")).unwrap();
re.is_match(title) re.is_match(title)
}) })
.map(|(_word, label)| (*label).into()) .map(|(_word, label)| (*label).into())
@ -51,7 +48,7 @@ fn label_from_title(title: &str) -> Vec<String> {
pub struct NixpkgsStrategy<'a> { pub struct NixpkgsStrategy<'a> {
chan: lapin::Channel, chan: lapin::Channel,
job: &'a EvaluationJob, job: &'a EvaluationJob,
vcs_api: Rc<dyn VersionControlSystemAPI>, vcs_api: Arc<dyn VersionControlSystemAPI>,
change: &'a Change, change: &'a Change,
repo: &'a Repo, repo: &'a Repo,
nix: Nix, nix: Nix,
@ -66,7 +63,7 @@ impl<'a> NixpkgsStrategy<'a> {
pub fn new( pub fn new(
chan: lapin::Channel, chan: lapin::Channel,
job: &'a EvaluationJob, job: &'a EvaluationJob,
vcs_api: Rc<dyn VersionControlSystemAPI>, vcs_api: Arc<dyn VersionControlSystemAPI>,
repo: &'a Repo, repo: &'a Repo,
change: &'a Change, change: &'a Change,
nix: Nix, nix: Nix,
@ -85,12 +82,9 @@ impl<'a> NixpkgsStrategy<'a> {
} }
} }
fn tag_from_title(&self) { async fn tag_from_title(&self) {
let issue = let Ok(issue) = self.vcs_api.get_issue(self.repo, self.change.number).await else {
match block_on(self.vcs_api.get_issue(self.repo, self.change.number)) return;
{
Ok(issue) => issue,
Err(_) => return,
}; };
let labels = label_from_title(&issue.title); let labels = label_from_title(&issue.title);
@ -98,34 +92,28 @@ impl<'a> NixpkgsStrategy<'a> {
return; return;
} }
block_on(self.vcs_api.update_labels( self.vcs_api
self.repo, .update_labels(self.repo, self.change.number, &labels, &[])
self.change.number, .await;
&labels,
&[],
));
} }
fn update_labels(&self, to_add: &[String], to_remove: &[String]) { async fn update_labels(&self, to_add: &[String], to_remove: &[String]) {
block_on(self.vcs_api.update_labels( self.vcs_api
self.repo, .update_labels(self.repo, self.change.number, to_add, to_remove)
self.change.number, .await;
to_add,
to_remove,
));
} }
fn request_reviews(&self, impacted_maintainers: &maintainers::ImpactedMaintainers) { async fn request_reviews(&self, impacted_maintainers: &maintainers::ImpactedMaintainers) {
info!( info!(
"Impact maintainers: {:?}", "Impact maintainers: {:?}",
impacted_maintainers.maintainers() impacted_maintainers.maintainers()
); );
if impacted_maintainers.maintainers().len() < 10 { if impacted_maintainers.maintainers().len() < 10 {
let existing_reviewers = block_on( let existing_reviewers = self
self.vcs_api .vcs_api
.get_existing_reviewers(self.repo, self.change.number), .get_existing_reviewers(self.repo, self.change.number)
); .await;
// Normalize both sides, compute the IM - ER set // Normalize both sides, compute the IM - ER set
let new_reviewers: Vec<String> = impacted_maintainers let new_reviewers: Vec<String> = impacted_maintainers
@ -136,12 +124,9 @@ impl<'a> NixpkgsStrategy<'a> {
.collect(); .collect();
// Add them as reviewers. // Add them as reviewers.
block_on(self.vcs_api.request_reviewers( self.vcs_api
self.repo, .request_reviewers(self.repo, self.change.number, new_reviewers, vec![])
self.change.number, .await;
new_reviewers,
vec![],
));
} else { } else {
warn!( warn!(
"Too many reviewers ({}), skipping review requests", "Too many reviewers ({}), skipping review requests",
@ -150,32 +135,33 @@ impl<'a> NixpkgsStrategy<'a> {
} }
} }
fn check_stdenvs_before(&mut self, dir: &Path) { async fn check_stdenvs_before(&mut self, dir: &Path) {
let mut stdenvs = Stdenvs::new(self.nix.clone(), dir.to_path_buf()); let mut stdenvs = Stdenvs::new(self.nix.clone(), dir.to_path_buf());
stdenvs.identify_before(); stdenvs.identify_before().await;
self.stdenv_diff = Some(stdenvs); self.stdenv_diff = Some(stdenvs);
} }
fn check_stdenvs_after(&mut self) { async fn check_stdenvs_after(&mut self) {
if let Some(ref mut stdenvs) = self.stdenv_diff { if let Some(ref mut stdenvs) = self.stdenv_diff {
stdenvs.identify_after(); stdenvs.identify_after().await;
} }
} }
fn update_stdenv_labels(&self) { async fn update_stdenv_labels(&self) {
if let Some(ref stdenvs) = self.stdenv_diff { if let Some(ref stdenvs) = self.stdenv_diff {
let mut stdenvtagger = StdenvTagger::new(); let mut stdenvtagger = StdenvTagger::new();
if !stdenvs.are_same() { if !stdenvs.are_same() {
stdenvtagger.changed(stdenvs.changed()); stdenvtagger.changed(stdenvs.changed());
} }
self.update_labels(&stdenvtagger.tags_to_add(), &stdenvtagger.tags_to_remove()); self.update_labels(&stdenvtagger.tags_to_add(), &stdenvtagger.tags_to_remove())
.await;
} }
} }
fn check_outpaths_before(&mut self, dir: &Path) -> StepResult<()> { async fn check_outpaths_before(&mut self, dir: &Path) -> StepResult<()> {
let mut rebuildsniff = OutPathDiff::new(self.nix.clone(), dir.to_path_buf()); let mut rebuildsniff = OutPathDiff::new(self.nix.clone(), dir.to_path_buf());
if let Err(err) = rebuildsniff.find_before() { if let Err(err) = rebuildsniff.find_before().await {
/* /*
self.events self.events
.notify(Event::TargetBranchFailsEvaluation(target_branch.clone())); .notify(Event::TargetBranchFailsEvaluation(target_branch.clone()));
@ -192,9 +178,9 @@ impl<'a> NixpkgsStrategy<'a> {
} }
} }
fn check_outpaths_after(&mut self) -> StepResult<()> { async fn check_outpaths_after(&mut self) -> StepResult<()> {
if let Some(ref mut rebuildsniff) = self.outpath_diff { if let Some(ref mut rebuildsniff) = self.outpath_diff {
if let Err(err) = rebuildsniff.find_after() { if let Err(err) = rebuildsniff.find_after().await {
Err(Error::FailWithPastebin( Err(Error::FailWithPastebin(
String::from("This PR does not cleanly list package outputs after merging."), String::from("This PR does not cleanly list package outputs after merging."),
String::from("Output path comparison"), String::from("Output path comparison"),
@ -238,7 +224,7 @@ impl<'a> NixpkgsStrategy<'a> {
vec![] vec![]
} }
fn update_new_package_labels(&self) { async fn update_new_package_labels(&self) {
if let Some(ref rebuildsniff) = self.outpath_diff { if let Some(ref rebuildsniff) = self.outpath_diff {
if let Some((removed, added)) = rebuildsniff.package_diff() { if let Some((removed, added)) = rebuildsniff.package_diff() {
let mut addremovetagger = PkgsAddedRemovedTagger::new(); let mut addremovetagger = PkgsAddedRemovedTagger::new();
@ -246,12 +232,13 @@ impl<'a> NixpkgsStrategy<'a> {
self.update_labels( self.update_labels(
&addremovetagger.tags_to_add(), &addremovetagger.tags_to_add(),
&addremovetagger.tags_to_remove(), &addremovetagger.tags_to_remove(),
); )
.await;
} }
} }
} }
fn update_rebuild_labels( async fn update_rebuild_labels(
&mut self, &mut self,
dir: &Path, dir: &Path,
overall_status: &mut CommitStatus, overall_status: &mut CommitStatus,
@ -261,20 +248,21 @@ impl<'a> NixpkgsStrategy<'a> {
if let Some(attrs) = rebuildsniff.calculate_rebuild() { if let Some(attrs) = rebuildsniff.calculate_rebuild() {
if !attrs.is_empty() { if !attrs.is_empty() {
overall_status.set_url(self.gist_changed_paths(&attrs)); overall_status.set_url(self.gist_changed_paths(&attrs).await);
self.record_impacted_maintainers(dir, &attrs)?; self.record_impacted_maintainers(dir, &attrs).await?;
} }
rebuild_tags.parse_attrs(attrs); rebuild_tags.parse_attrs(attrs);
} }
self.update_labels(&rebuild_tags.tags_to_add(), &rebuild_tags.tags_to_remove()); self.update_labels(&rebuild_tags.tags_to_add(), &rebuild_tags.tags_to_remove())
.await;
} }
Ok(()) Ok(())
} }
fn gist_changed_paths(&mut self, attrs: &[PackageArch]) -> Option<String> { async fn gist_changed_paths(&mut self, attrs: &[PackageArch]) -> Option<String> {
block_on(crate::utils::pastebin::make_pastebin( crate::utils::pastebin::make_pastebin(
&mut self.chan, &mut self.chan,
"Changed Paths", "Changed Paths",
attrs attrs
@ -282,12 +270,13 @@ impl<'a> NixpkgsStrategy<'a> {
.map(|attr| format!("{}\t{}", &attr.architecture, &attr.package)) .map(|attr| format!("{}\t{}", &attr.architecture, &attr.package))
.collect::<Vec<String>>() .collect::<Vec<String>>()
.join("\n"), .join("\n"),
)) )
.await
.ok() .ok()
.map(|pp| pp.uri) .map(|pp| pp.uri)
} }
fn record_impacted_maintainers( async fn record_impacted_maintainers(
&mut self, &mut self,
dir: &Path, dir: &Path,
attrs: &[PackageArch], attrs: &[PackageArch],
@ -299,16 +288,18 @@ impl<'a> NixpkgsStrategy<'a> {
if let Some(ref changed_paths) = self.changed_paths { if let Some(ref changed_paths) = self.changed_paths {
let m = let m =
ImpactedMaintainers::calculate(&self.nix, dir, changed_paths, &changed_attributes); ImpactedMaintainers::calculate(&self.nix, dir, changed_paths, &changed_attributes)
.await;
let gist_url = block_on(crate::utils::pastebin::make_pastebin( let gist_url = crate::utils::pastebin::make_pastebin(
&mut self.chan, &mut self.chan,
"Potential Maintainers", "Potential Maintainers",
match m { match m {
Ok(ref maintainers) => format!("Maintainers:\n{}", maintainers), Ok(ref maintainers) => format!("Maintainers:\n{maintainers}"),
Err(ref e) => format!("Ignorable calculation error:\n{:?}", e), Err(ref e) => format!("Ignorable calculation error:\n{e:?}"),
}, },
)) )
.await
.ok() .ok()
.map(|pp| pp.uri); .map(|pp| pp.uri);
@ -325,7 +316,7 @@ impl<'a> NixpkgsStrategy<'a> {
String::from("large change, skipping automatic review requests"), String::from("large change, skipping automatic review requests"),
gist_url, gist_url,
); );
status.set(State::Success)?; status.set(State::Success).await?;
return Ok(()); return Ok(());
} }
@ -337,26 +328,28 @@ impl<'a> NixpkgsStrategy<'a> {
String::from("matching changed paths to changed attrs..."), String::from("matching changed paths to changed attrs..."),
gist_url, gist_url,
); );
status.set(State::Success)?; status.set(State::Success).await?;
if let Ok(ref maint) = m { if let Ok(ref maint) = m {
self.request_reviews(maint); self.request_reviews(maint).await;
let mut maint_tagger = MaintainerPrTagger::new(); let mut maint_tagger = MaintainerPrTagger::new();
// TODO: this is really weird. // TODO: this is really weird.
let issue = block_on( let issue = self
self.vcs_api.get_issue(self.repo, self.change.number), .vcs_api
) .get_issue(self.repo, self.change.number)
.await
.expect("Failed to obtain the issue"); .expect("Failed to obtain the issue");
maint_tagger maint_tagger
.record_maintainer(&issue.created_by.username, &maint.maintainers_by_package()); .record_maintainer(&issue.created_by.username, &maint.maintainers_by_package());
self.update_labels(&maint_tagger.tags_to_add(), &maint_tagger.tags_to_remove()); self.update_labels(&maint_tagger.tags_to_add(), &maint_tagger.tags_to_remove())
.await;
} }
} }
Ok(()) Ok(())
} }
fn check_meta_queue_builds(&mut self, dir: &Path) -> StepResult<Vec<BuildJob>> { async fn check_meta_queue_builds(&mut self, dir: &Path) -> StepResult<Vec<BuildJob>> {
if let Some(ref possibly_touched_packages) = self.touched_packages { if let Some(ref possibly_touched_packages) = self.touched_packages {
let mut status = CommitStatus::new( let mut status = CommitStatus::new(
self.vcs_api.clone(), self.vcs_api.clone(),
@ -366,10 +359,10 @@ impl<'a> NixpkgsStrategy<'a> {
String::from("config.nix: checkMeta = true"), String::from("config.nix: checkMeta = true"),
None, None,
); );
status.set(State::Pending)?; status.set(State::Pending).await?;
let nixenv = HydraNixEnv::new(self.nix.clone(), dir.to_path_buf(), true); let nixenv = HydraNixEnv::new(self.nix.clone(), dir.to_path_buf(), true);
match nixenv.execute_with_stats() { match nixenv.execute_with_stats().await {
Ok((pkgs, _stats)) => { Ok((pkgs, _stats)) => {
let mut try_build: Vec<String> = pkgs let mut try_build: Vec<String> = pkgs
.keys() .keys()
@ -381,7 +374,7 @@ impl<'a> NixpkgsStrategy<'a> {
try_build.dedup(); try_build.dedup();
status.set_url(None); status.set_url(None);
status.set(State::Success)?; status.set(State::Success).await?;
if !try_build.is_empty() && try_build.len() <= 20 { if !try_build.is_empty() && try_build.len() <= 20 {
// In the case of trying to merge master in to // In the case of trying to merge master in to
@ -403,15 +396,16 @@ impl<'a> NixpkgsStrategy<'a> {
} }
Err(out) => { Err(out) => {
status.set_url( status.set_url(
block_on(crate::utils::pastebin::make_pastebin( crate::utils::pastebin::make_pastebin(
&mut self.chan, &mut self.chan,
"Meta Check", "Meta Check",
out.display(), out.display(),
)) )
.await
.ok() .ok()
.map(|pp| pp.uri), .map(|pp| pp.uri),
); );
status.set(State::Failure)?; status.set(State::Failure).await?;
Err(Error::Fail(String::from( Err(Error::Fail(String::from(
"Failed to validate package metadata.", "Failed to validate package metadata.",
))) )))
@ -421,25 +415,31 @@ impl<'a> NixpkgsStrategy<'a> {
Ok(vec![]) Ok(vec![])
} }
} }
}
impl<'a> EvaluationStrategy for NixpkgsStrategy<'a> { pub(crate) async fn pre_clone(&self) -> StepResult<()> {
fn pre_clone(&mut self) -> StepResult<()> { self.tag_from_title().await;
self.tag_from_title();
Ok(()) Ok(())
} }
fn on_target_branch(&mut self, dir: &Path, status: &mut CommitStatus) -> StepResult<()> { pub(crate) async fn on_target_branch(
status.set_with_description("Checking original stdenvs", State::Pending)?; &mut self,
self.check_stdenvs_before(dir); dir: &Path,
status: &mut CommitStatus,
) -> StepResult<()> {
status
.set_with_description("Checking original stdenvs", State::Pending)
.await?;
self.check_stdenvs_before(dir).await;
status.set_with_description("Checking original out paths", State::Pending)?; status
self.check_outpaths_before(dir)?; .set_with_description("Checking original out paths", State::Pending)
.await?;
self.check_outpaths_before(dir).await?;
Ok(()) Ok(())
} }
fn after_fetch(&mut self, co: &CachedProjectCo) -> StepResult<()> { pub(crate) fn after_fetch(&mut self, co: &CachedProjectCo) {
let changed_paths = co let changed_paths = co
.files_changed_from_head(&self.job.change.head_sha) .files_changed_from_head(&self.job.change.head_sha)
.unwrap_or_else(|_| vec![]); .unwrap_or_else(|_| vec![]);
@ -447,29 +447,34 @@ impl<'a> EvaluationStrategy for NixpkgsStrategy<'a> {
self.touched_packages = Some(parse_commit_messages( self.touched_packages = Some(parse_commit_messages(
&co.commit_messages_from_head(&self.job.change.head_sha) &co.commit_messages_from_head(&self.job.change.head_sha)
.unwrap_or_else(|_| vec!["".to_owned()]), .unwrap_or_else(|_| vec![String::new()]),
)); ));
}
pub(crate) async fn merge_conflict(&self) {
self.update_labels(&["2.status: merge conflict".to_owned()], &[])
.await;
}
pub(crate) async fn after_merge(&mut self, status: &mut CommitStatus) -> StepResult<()> {
self.update_labels(&[], &["2.status: merge conflict".to_owned()])
.await;
status
.set_with_description("Checking new stdenvs", State::Pending)
.await?;
self.check_stdenvs_after().await;
status
.set_with_description("Checking new out paths", State::Pending)
.await?;
self.check_outpaths_after().await?;
Ok(()) Ok(())
} }
fn merge_conflict(&mut self) { #[allow(clippy::too_many_lines)]
self.update_labels(&["2.status: merge conflict".to_owned()], &[]); pub(crate) fn evaluation_checks(&self) -> Vec<EvalChecker> {
}
fn after_merge(&mut self, status: &mut CommitStatus) -> StepResult<()> {
self.update_labels(&[], &["2.status: merge conflict".to_owned()]);
status.set_with_description("Checking new stdenvs", State::Pending)?;
self.check_stdenvs_after();
status.set_with_description("Checking new out paths", State::Pending)?;
self.check_outpaths_after()?;
Ok(())
}
fn evaluation_checks(&self) -> Vec<EvalChecker> {
// the value that's passed as the nixpkgs arg // the value that's passed as the nixpkgs arg
let nixpkgs_arg_value = format!( let nixpkgs_arg_value = format!(
"{{ outPath=./.; revCount=999999; shortRev=\"{}\"; rev=\"{}\"; }}", "{{ outPath=./.; revCount=999999; shortRev=\"{}\"; rev=\"{}\"; }}",
@ -600,20 +605,22 @@ impl<'a> EvaluationStrategy for NixpkgsStrategy<'a> {
] ]
} }
fn all_evaluations_passed( pub(crate) async fn all_evaluations_passed(
&mut self, &mut self,
dir: &Path, dir: &Path,
status: &mut CommitStatus, status: &mut CommitStatus,
) -> StepResult<EvaluationComplete> { ) -> StepResult<EvaluationComplete> {
self.update_stdenv_labels(); self.update_stdenv_labels().await;
status.set_with_description("Calculating Changed Outputs", State::Pending)?; status
.set_with_description("Calculating Changed Outputs", State::Pending)
.await?;
self.update_new_package_labels(); self.update_new_package_labels().await;
self.update_rebuild_labels(dir, status)?; self.update_rebuild_labels(dir, status).await?;
let checks = self.performance_stats(); let checks = self.performance_stats();
let builds = self.check_meta_queue_builds(dir)?; let builds = self.check_meta_queue_builds(dir).await?;
Ok(EvaluationComplete { builds, checks }) Ok(EvaluationComplete { builds, checks })
} }
} }
@ -628,7 +635,7 @@ fn parse_commit_messages(messages: &[String]) -> Vec<String> {
// NOTE: This transforms `{foo,bar}` into `{{foo,bar}}` and `foo,bar` into `{foo,bar}`, // NOTE: This transforms `{foo,bar}` into `{{foo,bar}}` and `foo,bar` into `{foo,bar}`,
// which allows both the old style (`foo,bar`) and the new style (`{foo,bar}`) to expand to // which allows both the old style (`foo,bar`) and the new style (`{foo,bar}`) to expand to
// `foo` and `bar`. // `foo` and `bar`.
.flat_map(|line| brace_expand::brace_expand(&format!("{{{}}}", line))) .flat_map(|line| brace_expand::brace_expand(&format!("{{{line}}}")))
.map(|line| line.trim().to_owned()) .map(|line| line.trim().to_owned())
.collect() .collect()
} }
@ -674,7 +681,7 @@ mod tests {
firefox{,-beta}{,-bin}, librewolf: blah blah blah firefox{,-beta}{,-bin}, librewolf: blah blah blah
" "
.lines() .lines()
.map(|l| l.to_owned()) .map(std::borrow::ToOwned::to_owned)
.collect::<Vec<String>>(), .collect::<Vec<String>>(),
), ),
expect expect
@ -709,7 +716,10 @@ mod tests {
); );
assert_eq!( assert_eq!(
label_from_title("fix build on bsd and darwin"), label_from_title("fix build on bsd and darwin"),
[String::from("6.topic: bsd"), String::from("6.topic: darwin")] [
String::from("6.topic: bsd"),
String::from("6.topic: darwin")
]
); );
assert_eq!( assert_eq!(
label_from_title("pkg: fix cross"), label_from_title("pkg: fix cross"),

View file

@ -29,6 +29,7 @@ pub struct Stdenvs {
} }
impl Stdenvs { impl Stdenvs {
#[must_use]
pub fn new(nix: nix::Nix, co: PathBuf) -> Stdenvs { pub fn new(nix: nix::Nix, co: PathBuf) -> Stdenvs {
Stdenvs { Stdenvs {
nix, nix,
@ -42,20 +43,22 @@ impl Stdenvs {
} }
} }
pub fn identify_before(&mut self) { pub async fn identify_before(&mut self) {
self.identify(System::X8664Linux, StdenvFrom::Before); self.identify(System::X8664Linux, StdenvFrom::Before).await;
self.identify(System::X8664Darwin, StdenvFrom::Before); self.identify(System::X8664Darwin, StdenvFrom::Before).await;
} }
pub fn identify_after(&mut self) { pub async fn identify_after(&mut self) {
self.identify(System::X8664Linux, StdenvFrom::After); self.identify(System::X8664Linux, StdenvFrom::After).await;
self.identify(System::X8664Darwin, StdenvFrom::After); self.identify(System::X8664Darwin, StdenvFrom::After).await;
} }
#[must_use]
pub fn are_same(&self) -> bool { pub fn are_same(&self) -> bool {
self.changed().is_empty() self.changed().is_empty()
} }
#[must_use]
pub fn changed(&self) -> Vec<System> { pub fn changed(&self) -> Vec<System> {
let mut changed: Vec<System> = vec![]; let mut changed: Vec<System> = vec![];
@ -70,29 +73,31 @@ impl Stdenvs {
changed changed
} }
fn identify(&mut self, system: System, from: StdenvFrom) { async fn identify(&mut self, system: System, from: StdenvFrom) {
match (system, from) { match (system, from) {
(System::X8664Linux, StdenvFrom::Before) => { (System::X8664Linux, StdenvFrom::Before) => {
self.linux_stdenv_before = self.evalstdenv("x86_64-linux"); self.linux_stdenv_before = self.evalstdenv("x86_64-linux").await;
} }
(System::X8664Linux, StdenvFrom::After) => { (System::X8664Linux, StdenvFrom::After) => {
self.linux_stdenv_after = self.evalstdenv("x86_64-linux"); self.linux_stdenv_after = self.evalstdenv("x86_64-linux").await;
} }
(System::X8664Darwin, StdenvFrom::Before) => { (System::X8664Darwin, StdenvFrom::Before) => {
self.darwin_stdenv_before = self.evalstdenv("x86_64-darwin"); self.darwin_stdenv_before = self.evalstdenv("x86_64-darwin").await;
} }
(System::X8664Darwin, StdenvFrom::After) => { (System::X8664Darwin, StdenvFrom::After) => {
self.darwin_stdenv_after = self.evalstdenv("x86_64-darwin"); self.darwin_stdenv_after = self.evalstdenv("x86_64-darwin").await;
} }
} }
} }
/// This is used to find out what the output path of the stdenv for the /// This is used to find out what the output path of the stdenv for the
/// given system. /// given system.
fn evalstdenv(&self, system: &str) -> Option<String> { async fn evalstdenv(&self, system: &str) -> Option<String> {
info!(?system, "query stdenv output"); info!(?system, "query stdenv output");
let result = self.nix.with_system(system.to_owned()).safely( let nix = self.nix.with_system(system.to_owned());
let result = nix
.safely(
&nix::Operation::QueryPackagesOutputs, &nix::Operation::QueryPackagesOutputs,
&self.co, &self.co,
vec![ vec![
@ -102,7 +107,8 @@ impl Stdenvs {
String::from("stdenv"), String::from("stdenv"),
], ],
true, true,
); )
.await;
match result { match result {
Ok(mut out) => Some(file_to_str(&mut out)), Ok(mut out) => Some(file_to_str(&mut out)),
@ -121,8 +127,8 @@ mod tests {
use std::env; use std::env;
use std::process::Command; use std::process::Command;
#[test] #[tokio::test]
fn stdenv_checking() { async fn stdenv_checking() {
let output = Command::new("nix-instantiate") let output = Command::new("nix-instantiate")
.args(["--eval", "-E", "<nixpkgs>"]) .args(["--eval", "-E", "<nixpkgs>"])
.output() .output()
@ -130,14 +136,20 @@ mod tests {
let nixpkgs = String::from_utf8(output.stdout).expect("nixpkgs required"); let nixpkgs = String::from_utf8(output.stdout).expect("nixpkgs required");
let remote = env::var("NIX_REMOTE").unwrap_or("".to_owned()); let remote = env::var("NIX_REMOTE").unwrap_or_default();
let nix = nix::Nix::new(String::from("x86_64-linux"), remote, 1200, None); let nix = nix::Nix::new(String::from("x86_64-linux"), remote, 1200, None);
let mut stdenv = Stdenvs::new(nix, PathBuf::from(nixpkgs.trim_end())); let mut stdenv = Stdenvs::new(nix, PathBuf::from(nixpkgs.trim_end()));
stdenv.identify(System::X8664Linux, StdenvFrom::Before); stdenv
stdenv.identify(System::X8664Darwin, StdenvFrom::Before); .identify(System::X8664Linux, StdenvFrom::Before)
.await;
stdenv
.identify(System::X8664Darwin, StdenvFrom::Before)
.await;
stdenv.identify(System::X8664Linux, StdenvFrom::After); stdenv.identify(System::X8664Linux, StdenvFrom::After).await;
stdenv.identify(System::X8664Darwin, StdenvFrom::After); stdenv
.identify(System::X8664Darwin, StdenvFrom::After)
.await;
assert!(stdenv.are_same()); assert!(stdenv.are_same());
} }

View file

@ -2,6 +2,7 @@
use crate::acl::Acl; use crate::acl::Acl;
use crate::checkout; use crate::checkout;
use crate::files::file_to_str; use crate::files::file_to_str;
use crate::message::evaluationjob::Actions;
use crate::message::{buildjob, evaluationjob}; use crate::message::{buildjob, evaluationjob};
use crate::nix; use crate::nix;
use crate::stats::{self, Event}; use crate::stats::{self, Event};
@ -14,10 +15,10 @@ use crate::vcs::gerrit::http::GerritHTTPApi;
use crate::worker; use crate::worker;
use std::path::Path; use std::path::Path;
use std::rc::Rc; use std::sync::Arc;
use std::time::Instant; use std::time::Instant;
use futures::executor::block_on; use async_trait::async_trait;
use tracing::{debug_span, error, info, warn}; use tracing::{debug_span, error, info, warn};
pub enum SupportedVCS { pub enum SupportedVCS {
@ -54,18 +55,24 @@ impl<E: stats::SysEvents> EvaluationWorker<E> {
} }
} }
impl<E: stats::SysEvents + 'static> worker::SimpleWorker for EvaluationWorker<E> { #[async_trait]
impl<E: stats::SysEvents + 'static + Sync + Send> worker::SimpleWorker for EvaluationWorker<E> {
type J = evaluationjob::EvaluationJob; type J = evaluationjob::EvaluationJob;
fn msg_to_job(&mut self, _: &str, _: &Option<String>, body: &[u8]) -> Result<Self::J, String> { async fn msg_to_job(
self.events.notify(Event::JobReceived); &mut self,
_: &str,
_: &Option<String>,
body: &[u8],
) -> Result<Self::J, String> {
self.events.notify(Event::JobReceived).await;
match evaluationjob::from(body) { match evaluationjob::from(body) {
Ok(e) => { Ok(e) => {
self.events.notify(Event::JobDecodeSuccess); self.events.notify(Event::JobDecodeSuccess).await;
Ok(e) Ok(e)
} }
Err(e) => { Err(e) => {
self.events.notify(Event::JobDecodeFailure); self.events.notify(Event::JobDecodeFailure).await;
error!( error!(
"Failed to decode message: {:?}, Err: {:?}", "Failed to decode message: {:?}, Err: {:?}",
String::from_utf8(body.to_vec()), String::from_utf8(body.to_vec()),
@ -76,16 +83,16 @@ impl<E: stats::SysEvents + 'static> worker::SimpleWorker for EvaluationWorker<E>
} }
} }
fn consumer( async fn consumer(
&mut self, &mut self,
chan: &mut lapin::Channel, chan: &mut lapin::Channel,
job: &evaluationjob::EvaluationJob, job: &evaluationjob::EvaluationJob,
) -> worker::Actions { ) -> worker::Actions {
let span = debug_span!("job", change_id = ?job.change.number); let span: tracing::Span = debug_span!("job", change_id = ?job.change.number);
let _enter = span.enter(); let _enter = span.enter();
let vcs_api: Rc<dyn VersionControlSystemAPI> = match self.vcs { let vcs_api: Arc<dyn VersionControlSystemAPI> = match self.vcs {
SupportedVCS::Gerrit => Rc::new(GerritHTTPApi), SupportedVCS::Gerrit => Arc::new(GerritHTTPApi),
}; };
OneEval::new( OneEval::new(
@ -98,11 +105,12 @@ impl<E: stats::SysEvents + 'static> worker::SimpleWorker for EvaluationWorker<E>
job, job,
) )
.worker_actions(chan) .worker_actions(chan)
.await
} }
} }
struct OneEval<'a, E> { struct OneEval<'a, E> {
vcs_api: Rc<dyn VersionControlSystemAPI>, vcs_api: Arc<dyn VersionControlSystemAPI>,
nix: &'a nix::Nix, nix: &'a nix::Nix,
acl: &'a Acl, acl: &'a Acl,
events: &'a mut E, events: &'a mut E,
@ -114,7 +122,7 @@ struct OneEval<'a, E> {
impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> { impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> {
#[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)]
fn new( fn new(
vcs_api: Rc<dyn VersionControlSystemAPI>, vcs_api: Arc<dyn VersionControlSystemAPI>,
nix: &'a nix::Nix, nix: &'a nix::Nix,
acl: &'a Acl, acl: &'a Acl,
events: &'a mut E, events: &'a mut E,
@ -133,11 +141,7 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> {
} }
} }
fn actions(&self) -> evaluationjob::Actions { async fn update_status(
evaluationjob::Actions {}
}
fn update_status(
&self, &self,
description: String, description: String,
url: Option<String>, url: Option<String>,
@ -158,42 +162,50 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> {
&self.job.change.number, &self.job.change.head_sha, &description &self.job.change.number, &self.job.change.head_sha, &description
); );
block_on(self.vcs_api.create_commit_statuses( self.vcs_api
.create_commit_statuses(
&self.job.repo, &self.job.repo,
self.job.change.head_sha.clone(), self.job.change.head_sha.clone(),
state, state,
"ofborg-eval".to_owned(), "ofborg-eval".to_owned(),
description, description,
// TODO: make this an option // TODO: make this an option
url.unwrap_or_else(|| "".to_owned()), url.unwrap_or_default(),
)) )
.await
} }
fn make_pastebin( async fn make_pastebin(
&mut self, &mut self,
chan: &mut lapin::Channel, chan: &mut lapin::Channel,
title: &str, title: &str,
contents: String, contents: String,
) -> Option<PersistedPastebin> { ) -> Option<PersistedPastebin> {
block_on(crate::utils::pastebin::make_pastebin(chan, title, contents)).ok() crate::utils::pastebin::make_pastebin(chan, title, contents)
.await
.ok()
} }
fn worker_actions(&mut self, chan: &mut lapin::Channel) -> worker::Actions { async fn worker_actions(&mut self, chan: &mut lapin::Channel) -> worker::Actions {
let eval_result = self let eval_result = match self.evaluate_job(chan).await {
.evaluate_job(chan) Ok(r) => Ok(r),
.map_err(|eval_error| match eval_error {
// Handle error cases which expect us to post statuses // Handle error cases which expect us to post statuses
// to github. Convert Eval Errors in to Result<_, CommitStatusWrite> // to github. Convert Eval Errors in to Result<_, CommitStatusWrite>
EvalWorkerError::EvalError(eval::Error::Fail(msg)) => { Err(EvalWorkerError::EvalError(eval::Error::Fail(msg))) => {
self.update_status(msg, None, State::Failure) Err(self.update_status(msg, None, State::Failure).await)
} }
EvalWorkerError::EvalError(eval::Error::FailWithPastebin(msg, title, content)) => { Err(EvalWorkerError::EvalError(eval::Error::FailWithPastebin(msg, title, content))) => {
let pastebin = self.make_pastebin(chan, &title, content).map(|pp| pp.uri); let pastebin = self
self.update_status(msg, pastebin, State::Failure) .make_pastebin(chan, &title, content)
.await
.map(|pp| pp.uri);
Err(self.update_status(msg, pastebin, State::Failure).await)
} }
EvalWorkerError::EvalError(eval::Error::CommitStatusWrite(e)) => Err(e), Err(
EvalWorkerError::CommitStatusWrite(e) => Err(e), EvalWorkerError::EvalError(eval::Error::CommitStatusWrite(e))
}); | EvalWorkerError::CommitStatusWrite(e),
) => Err(Err(e)),
};
match eval_result { match eval_result {
Ok(eval_actions) => eval_actions, Ok(eval_actions) => eval_actions,
@ -201,18 +213,18 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> {
// There was an error during eval, but we successfully // There was an error during eval, but we successfully
// updated the PR. // updated the PR.
self.actions().skip(self.job) Actions::skip(self.job)
} }
Err(Err(CommitStatusError::ExpiredCreds(e))) => { Err(Err(CommitStatusError::ExpiredCreds(e))) => {
error!("Failed writing commit status: creds expired: {:?}", e); error!("Failed writing commit status: creds expired: {:?}", e);
self.actions().retry_later(self.job) Actions::retry_later(self.job)
} }
Err(Err(CommitStatusError::MissingSha(e))) => { Err(Err(CommitStatusError::MissingSha(e))) => {
error!( error!(
"Failed writing commit status: commit sha was force-pushed away: {:?}", "Failed writing commit status: commit sha was force-pushed away: {:?}",
e e
); );
self.actions().skip(self.job) Actions::skip(self.job)
} }
Err(Err(CommitStatusError::Error(cswerr))) => { Err(Err(CommitStatusError::Error(cswerr))) => {
@ -221,38 +233,41 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> {
cswerr cswerr
); );
block_on(self.vcs_api.update_labels( self.vcs_api
.update_labels(
&self.job.repo, &self.job.repo,
self.job.change.number, self.job.change.number,
&[String::from("ofborg-internal-error")], &[String::from("ofborg-internal-error")],
&[], &[],
)); )
.await;
self.actions().skip(self.job) Actions::skip(self.job)
} }
} }
} }
// FIXME: remove with rust/cargo update // FIXME: remove with rust/cargo update
#[allow(clippy::cognitive_complexity)] #[allow(clippy::cognitive_complexity)]
fn evaluate_job( #[allow(clippy::too_many_lines)]
async fn evaluate_job(
&mut self, &mut self,
chan: &mut lapin::Channel, chan: &mut lapin::Channel,
) -> Result<worker::Actions, EvalWorkerError> { ) -> Result<worker::Actions, EvalWorkerError> {
let job = self.job; let job = self.job;
let issue_ref = block_on(self.vcs_api.get_issue(&job.repo, job.change.number)); let issue_ref = self.vcs_api.get_issue(&job.repo, job.change.number).await;
let auto_schedule_build_archs: Vec<systems::System>; let auto_schedule_build_archs: Vec<systems::System>;
let _issue: Issue = match issue_ref { let _issue: Issue = match issue_ref {
Ok(iss) => { Ok(iss) => {
if matches!(iss.state, IssueState::Closed) { if matches!(iss.state, IssueState::Closed) {
self.events.notify(Event::IssueAlreadyClosed); self.events.notify(Event::IssueAlreadyClosed).await;
info!("Skipping {} because it is closed", job.change.number); info!("Skipping {} because it is closed", job.change.number);
return Ok(self.actions().skip(job)); return Ok(Actions::skip(job));
} }
if iss.is_wip() { if iss.is_wip() {
self.events.notify(Event::CurrentlyWorkInProgress); self.events.notify(Event::CurrentlyWorkInProgress).await;
auto_schedule_build_archs = vec![]; auto_schedule_build_archs = vec![];
} else { } else {
auto_schedule_build_archs = self.acl.build_job_architectures_for_user_repo( auto_schedule_build_archs = self.acl.build_job_architectures_for_user_repo(
@ -265,25 +280,21 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> {
} }
Err(e) => { Err(e) => {
self.events.notify(Event::IssueFetchFailed); self.events.notify(Event::IssueFetchFailed).await;
error!("Error fetching {}!", job.change.number); error!("Error fetching {}!", job.change.number);
error!("E: {:?}", e); error!("E: {:?}", e);
return Ok(self.actions().skip(job)); return Ok(Actions::skip(job));
} }
}; };
let mut evaluation_strategy: Box<dyn eval::EvaluationStrategy> = if job.is_nixpkgs() { let mut evaluation_strategy = eval::NixpkgsStrategy::new(
Box::new(eval::NixpkgsStrategy::new(
chan.clone(), chan.clone(),
job, job,
self.vcs_api.clone(), self.vcs_api.clone(),
&job.repo, &job.repo,
&job.change, &job.change,
self.nix.clone(), self.nix.clone(),
)) );
} else {
Box::new(eval::GenericStrategy::new())
};
let mut overall_status = CommitStatus::new( let mut overall_status = CommitStatus::new(
self.vcs_api.clone(), self.vcs_api.clone(),
@ -294,15 +305,19 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> {
None, None,
); );
overall_status.set_with_description("Starting", State::Pending)?; overall_status
.set_with_description("Starting", State::Pending)
.await?;
evaluation_strategy.pre_clone()?; evaluation_strategy.pre_clone().await?;
let project = self let project = self
.cloner .cloner
.project(&job.repo.full_name, job.repo.clone_url.clone()); .project(&job.repo.full_name, job.repo.clone_url.clone());
overall_status.set_with_description("Cloning project", State::Pending)?; overall_status
.set_with_description("Cloning project", State::Pending)
.await?;
info!("Working on {}", job.change.number); info!("Working on {}", job.change.number);
let co = project let co = project
@ -317,69 +332,86 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> {
// TODO: this is a preflight check, encode it as such. // TODO: this is a preflight check, encode it as such.
if target_branch.starts_with("nixos-") || target_branch.starts_with("nixpkgs-") { if target_branch.starts_with("nixos-") || target_branch.starts_with("nixpkgs-") {
overall_status.set_with_description( overall_status
.set_with_description(
"The branch you have targeted is a read-only mirror for channels. \ "The branch you have targeted is a read-only mirror for channels. \
Please target release-* or master.", Please target release-* or master.",
State::Error, State::Error,
)?; )
.await?;
info!("PR targets a nixos-* or nixpkgs-* branch"); info!("PR targets a nixos-* or nixpkgs-* branch");
return Ok(self.actions().skip(job)); return Ok(Actions::skip(job));
}; };
overall_status.set_with_description( overall_status
.set_with_description(
format!("Checking out {}", &target_branch).as_ref(), format!("Checking out {}", &target_branch).as_ref(),
State::Pending, State::Pending,
)?; )
.await?;
info!("Checking out target branch {}", &target_branch); info!("Checking out target branch {}", &target_branch);
let refpath = co.checkout_origin_ref(target_branch.as_ref()).unwrap(); let refpath = co.checkout_origin_ref(target_branch.as_ref()).unwrap();
evaluation_strategy.on_target_branch(Path::new(&refpath), &mut overall_status)?; evaluation_strategy
.on_target_branch(Path::new(&refpath), &mut overall_status)
.await?;
let target_branch_rebuild_sniff_start = Instant::now(); let target_branch_rebuild_sniff_start = Instant::now();
self.events.notify(Event::EvaluationDuration( self.events
.notify(Event::EvaluationDuration(
target_branch.clone(), target_branch.clone(),
target_branch_rebuild_sniff_start.elapsed().as_secs(), target_branch_rebuild_sniff_start.elapsed().as_secs(),
)); ))
.await;
self.events self.events
.notify(Event::EvaluationDurationCount(target_branch)); .notify(Event::EvaluationDurationCount(target_branch))
.await;
overall_status.set_with_description("Fetching PR", State::Pending)?; overall_status
.set_with_description("Fetching PR", State::Pending)
.await?;
// TODO: generalize fetch change // TODO: generalize fetch change
co.fetch_pr(job.change.number).unwrap(); co.fetch_pr(job.change.number).unwrap();
if !co.commit_exists(job.change.head_sha.as_ref()) { if !co.commit_exists(job.change.head_sha.as_ref()) {
overall_status.set_with_description("Commit not found", State::Error)?; overall_status
.set_with_description("Commit not found", State::Error)
.await?;
info!("Commit {} doesn't exist", job.change.head_sha); info!("Commit {} doesn't exist", job.change.head_sha);
return Ok(self.actions().skip(job)); return Ok(Actions::skip(job));
} }
evaluation_strategy.after_fetch(&co)?; evaluation_strategy.after_fetch(&co);
overall_status.set_with_description("Merging PR", State::Pending)?; overall_status
.set_with_description("Merging PR", State::Pending)
.await?;
if co.merge_commit(job.change.head_sha.as_ref()).is_err() { if co.merge_commit(job.change.head_sha.as_ref()).is_err() {
overall_status.set_with_description("Failed to merge", State::Failure)?; overall_status
.set_with_description("Failed to merge", State::Failure)
.await?;
info!("Failed to merge {}", job.change.head_sha); info!("Failed to merge {}", job.change.head_sha);
evaluation_strategy.merge_conflict(); evaluation_strategy.merge_conflict().await;
return Ok(self.actions().skip(job)); return Ok(Actions::skip(job));
} }
evaluation_strategy.after_merge(&mut overall_status)?; evaluation_strategy.after_merge(&mut overall_status).await?;
info!("Got path: {:?}, building", refpath); info!("Got path: {:?}, building", refpath);
overall_status.set_with_description("Beginning Evaluations", State::Pending)?; overall_status
.set_with_description("Beginning Evaluations", State::Pending)
.await?;
let eval_results: bool = evaluation_strategy let mut all_good = true;
.evaluation_checks() for check in evaluation_strategy.evaluation_checks() {
.into_iter()
.map(|check| {
let mut status = CommitStatus::new( let mut status = CommitStatus::new(
self.vcs_api.clone(), self.vcs_api.clone(),
job.repo.clone(), job.repo.clone(),
@ -391,11 +423,12 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> {
status status
.set(State::Pending) .set(State::Pending)
.await
.expect("Failed to set status on eval strategy"); .expect("Failed to set status on eval strategy");
let state: State; let state: State;
let gist_url: Option<String>; let gist_url: Option<String>;
match check.execute(Path::new(&refpath)) { match check.execute(Path::new(&refpath)).await {
Ok(_) => { Ok(_) => {
state = State::Success; state = State::Success;
gist_url = None; gist_url = None;
@ -408,6 +441,7 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> {
&format!("[ofborg] Evaluation of {}", check.name()), &format!("[ofborg] Evaluation of {}", check.name()),
file_to_str(&mut out), file_to_str(&mut out),
) )
.await
.map(|pp| pp.uri); .map(|pp| pp.uri);
} }
} }
@ -415,44 +449,46 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> {
status.set_url(gist_url); status.set_url(gist_url);
status status
.set(state) .set(state)
.await
.expect("Failed to set status on eval strategy"); .expect("Failed to set status on eval strategy");
if state == State::Success { if state != State::Success {
Ok(()) all_good = false;
} else { }
Err(())
} }
})
.all(|status| status == Ok(()));
info!("Finished evaluations"); info!("Finished evaluations");
let mut response: worker::Actions = vec![]; let mut response: worker::Actions = vec![];
if eval_results { if all_good {
let complete = evaluation_strategy let complete = evaluation_strategy
.all_evaluations_passed(Path::new(&refpath), &mut overall_status)?; .all_evaluations_passed(Path::new(&refpath), &mut overall_status)
.await?;
block_on(
self.vcs_api self.vcs_api
.create_check_statuses(&job.repo, complete.checks), .create_check_statuses(&job.repo, complete.checks)
); .await;
response.extend(schedule_builds(complete.builds, auto_schedule_build_archs)); response.extend(schedule_builds(complete.builds, &auto_schedule_build_archs));
overall_status.set_with_description("^.^!", State::Success)?; overall_status
.set_with_description("^.^!", State::Success)
.await?;
} else { } else {
overall_status.set_with_description("Complete, with errors", State::Failure)?; overall_status
.set_with_description("Complete, with errors", State::Failure)
.await?;
} }
self.events.notify(Event::TaskEvaluationCheckComplete); self.events.notify(Event::TaskEvaluationCheckComplete).await;
info!("Evaluations done!"); info!("Evaluations done!");
Ok(self.actions().done(job, response)) Ok(Actions::done(job, response))
} }
} }
fn schedule_builds( fn schedule_builds(
builds: Vec<buildjob::BuildJob>, builds: Vec<buildjob::BuildJob>,
auto_schedule_build_archs: Vec<systems::System>, auto_schedule_build_archs: &[systems::System],
) -> Vec<worker::Action> { ) -> Vec<worker::Action> {
let mut response = vec![]; let mut response = vec![];
info!( info!(
@ -460,20 +496,22 @@ fn schedule_builds(
builds, auto_schedule_build_archs builds, auto_schedule_build_archs
); );
for buildjob in builds { for buildjob in builds {
for arch in auto_schedule_build_archs.iter() { for arch in auto_schedule_build_archs {
let (exchange, routingkey) = arch.as_build_destination(); let (exchange, routingkey) = arch.as_build_destination();
response.push(worker::publish_serde_action( response.push(worker::publish_serde_action(
exchange, routingkey, &buildjob, &exchange,
&routingkey,
&buildjob,
)); ));
} }
response.push(worker::publish_serde_action( response.push(worker::publish_serde_action(
Some("build-results".to_string()), &Some("build-results".to_string()),
None, &None,
&buildjob::QueuedBuildJobs { &buildjob::QueuedBuildJobs {
job: buildjob, job: buildjob,
architectures: auto_schedule_build_archs architectures: auto_schedule_build_archs
.iter() .iter()
.map(|arch| arch.to_string()) .map(std::string::ToString::to_string)
.collect(), .collect(),
}, },
)); ));

View file

@ -3,6 +3,7 @@ use crate::ghevent;
use crate::message::{evaluationjob, Change, Repo}; use crate::message::{evaluationjob, Change, Repo};
use crate::worker; use crate::worker;
use async_trait::async_trait;
use tracing::{debug_span, info}; use tracing::{debug_span, info};
pub struct EvaluationFilterWorker { pub struct EvaluationFilterWorker {
@ -10,15 +11,22 @@ pub struct EvaluationFilterWorker {
} }
impl EvaluationFilterWorker { impl EvaluationFilterWorker {
#[must_use]
pub fn new(acl: acl::Acl) -> EvaluationFilterWorker { pub fn new(acl: acl::Acl) -> EvaluationFilterWorker {
EvaluationFilterWorker { acl } EvaluationFilterWorker { acl }
} }
} }
#[async_trait]
impl worker::SimpleWorker for EvaluationFilterWorker { impl worker::SimpleWorker for EvaluationFilterWorker {
type J = ghevent::PullRequestEvent; type J = ghevent::PullRequestEvent;
fn msg_to_job(&mut self, _: &str, _: &Option<String>, body: &[u8]) -> Result<Self::J, String> { async fn msg_to_job(
&mut self,
_: &str,
_: &Option<String>,
body: &[u8],
) -> Result<Self::J, String> {
match serde_json::from_slice(body) { match serde_json::from_slice(body) {
Ok(e) => Ok(e), Ok(e) => Ok(e),
Err(e) => Err(format!( Err(e) => Err(format!(
@ -29,7 +37,7 @@ impl worker::SimpleWorker for EvaluationFilterWorker {
} }
} }
fn consumer( async fn consumer(
&mut self, &mut self,
_chan: &mut lapin::Channel, _chan: &mut lapin::Channel,
job: &ghevent::PullRequestEvent, job: &ghevent::PullRequestEvent,
@ -51,9 +59,9 @@ impl worker::SimpleWorker for EvaluationFilterWorker {
} }
let interesting: bool = match job.action { let interesting: bool = match job.action {
ghevent::PullRequestAction::Opened => true, ghevent::PullRequestAction::Opened
ghevent::PullRequestAction::Synchronize => true, | ghevent::PullRequestAction::Synchronize
ghevent::PullRequestAction::Reopened => true, | ghevent::PullRequestAction::Reopened => true,
ghevent::PullRequestAction::Edited => { ghevent::PullRequestAction::Edited => {
if let Some(ref changes) = job.changes { if let Some(ref changes) = job.changes {
changes.base.is_some() changes.base.is_some()
@ -61,7 +69,7 @@ impl worker::SimpleWorker for EvaluationFilterWorker {
false false
} }
} }
_ => false, ghevent::PullRequestAction::Unknown => false,
}; };
if !interesting { if !interesting {
@ -96,7 +104,7 @@ impl worker::SimpleWorker for EvaluationFilterWorker {
}; };
vec![ vec![
worker::publish_serde_action(None, Some("mass-rebuild-check-jobs".to_owned()), &msg), worker::publish_serde_action(&None, &Some("mass-rebuild-check-jobs".to_owned()), &msg),
worker::Action::Ack, worker::Action::Ack,
] ]
} }

View file

@ -3,6 +3,7 @@ use crate::message::buildresult::BuildResult;
use crate::worker; use crate::worker;
use crate::writetoline::LineWriter; use crate::writetoline::LineWriter;
use async_trait::async_trait;
use std::fs::{self, File, OpenOptions}; use std::fs::{self, File, OpenOptions};
use std::io::Write; use std::io::Write;
use std::path::{Component, Path, PathBuf}; use std::path::{Component, Path, PathBuf};
@ -55,6 +56,7 @@ fn validate_path_segment(segment: &Path) -> Result<(), String> {
} }
impl LogMessageCollector { impl LogMessageCollector {
#[must_use]
pub fn new(log_root: PathBuf, max_open: usize) -> LogMessageCollector { pub fn new(log_root: PathBuf, max_open: usize) -> LogMessageCollector {
LogMessageCollector { LogMessageCollector {
handles: LruCache::new(max_open), handles: LruCache::new(max_open),
@ -64,33 +66,33 @@ impl LogMessageCollector {
pub fn write_metadata(&mut self, from: &LogFrom, data: &BuildLogStart) -> Result<(), String> { pub fn write_metadata(&mut self, from: &LogFrom, data: &BuildLogStart) -> Result<(), String> {
let metapath = self.path_for_metadata(from)?; let metapath = self.path_for_metadata(from)?;
let mut fp = self.open_file(&metapath)?; let mut fp = Self::open_file(&metapath)?;
match serde_json::to_string(data) { match serde_json::to_string(data) {
Ok(data) => { Ok(data) => {
if let Err(e) = fp.write(data.as_bytes()) { if let Err(e) = fp.write(data.as_bytes()) {
Err(format!("Failed to write metadata: {:?}", e)) Err(format!("Failed to write metadata: {e:?}"))
} else { } else {
Ok(()) Ok(())
} }
} }
Err(e) => Err(format!("Failed to stringify metadata: {:?}", e)), Err(e) => Err(format!("Failed to stringify metadata: {e:?}")),
} }
} }
pub fn write_result(&mut self, from: &LogFrom, data: &BuildResult) -> Result<(), String> { pub fn write_result(&mut self, from: &LogFrom, data: &BuildResult) -> Result<(), String> {
let path = self.path_for_result(from)?; let path = self.path_for_result(from)?;
let mut fp = self.open_file(&path)?; let mut fp = Self::open_file(&path)?;
match serde_json::to_string(data) { match serde_json::to_string(data) {
Ok(data) => { Ok(data) => {
if let Err(e) = fp.write(data.as_bytes()) { if let Err(e) = fp.write(data.as_bytes()) {
Err(format!("Failed to write result: {:?}", e)) Err(format!("Failed to write result: {e:?}"))
} else { } else {
Ok(()) Ok(())
} }
} }
Err(e) => Err(format!("Failed to stringify result: {:?}", e)), Err(e) => Err(format!("Failed to stringify result: {e:?}")),
} }
} }
@ -102,7 +104,7 @@ impl LogMessageCollector {
.expect("handles just contained the key")) .expect("handles just contained the key"))
} else { } else {
let logpath = self.path_for_log(from)?; let logpath = self.path_for_log(from)?;
let fp = self.open_file(&logpath)?; let fp = Self::open_file(&logpath)?;
let writer = LineWriter::new(fp); let writer = LineWriter::new(fp);
self.handles.insert(from.clone(), writer); self.handles.insert(from.clone(), writer);
if let Some(handle) = self.handles.get_mut(from) { if let Some(handle) = self.handles.get_mut(from) {
@ -142,13 +144,12 @@ impl LogMessageCollector {
Ok(location) Ok(location)
} else { } else {
Err(format!( Err(format!(
"Calculating the log location for {:?} resulted in an invalid path {:?}", "Calculating the log location for {from:?} resulted in an invalid path {location:?}"
from, location
)) ))
} }
} }
fn open_file(&self, path: &Path) -> Result<File, String> { fn open_file(path: &Path) -> Result<File, String> {
let dir = path.parent().unwrap(); let dir = path.parent().unwrap();
fs::create_dir_all(dir).unwrap(); fs::create_dir_all(dir).unwrap();
@ -168,10 +169,11 @@ impl LogMessageCollector {
} }
} }
#[async_trait]
impl worker::SimpleWorker for LogMessageCollector { impl worker::SimpleWorker for LogMessageCollector {
type J = LogMessage; type J = LogMessage;
fn msg_to_job( async fn msg_to_job(
&mut self, &mut self,
routing_key: &str, routing_key: &str,
_: &Option<String>, _: &Option<String>,
@ -195,7 +197,7 @@ impl worker::SimpleWorker for LogMessageCollector {
attempt_id = msg.legacy().attempt_id; attempt_id = msg.legacy().attempt_id;
message = MsgType::Finish(Box::new(msg)); message = MsgType::Finish(Box::new(msg));
} else { } else {
return Err(format!("failed to decode job: {:?}", decode_msg)); return Err(format!("failed to decode job: {decode_msg:?}"));
} }
} }
} }
@ -209,7 +211,7 @@ impl worker::SimpleWorker for LogMessageCollector {
}) })
} }
fn consumer(&mut self, _chan: &mut lapin::Channel, job: &LogMessage) -> worker::Actions { async fn consumer(&mut self, _chan: &mut lapin::Channel, job: &LogMessage) -> worker::Actions {
match job.message { match job.message {
MsgType::Start(ref start) => { MsgType::Start(ref start) => {
self.write_metadata(&job.from, start) self.write_metadata(&job.from, start)
@ -223,7 +225,7 @@ impl worker::SimpleWorker for LogMessageCollector {
MsgType::Msg(ref message) => { MsgType::Msg(ref message) => {
let handle = self.handle_for(&job.from).unwrap(); let handle = self.handle_for(&job.from).unwrap();
handle.write_to_line((message.line_number - 1) as usize, &message.output); handle.write_to_line(message.line_number - 1, &message.output);
} }
MsgType::Finish(ref finish) => { MsgType::Finish(ref finish) => {
self.write_result(&job.from, finish) self.write_result(&job.from, finish)

View file

@ -4,6 +4,7 @@ use std::{
path::PathBuf, path::PathBuf,
}; };
use async_trait::async_trait;
use tracing::{debug_span, error, warn}; use tracing::{debug_span, error, warn};
use zstd::stream::write::Encoder; use zstd::stream::write::Encoder;
@ -18,6 +19,7 @@ pub struct PastebinCollector {
} }
impl PastebinCollector { impl PastebinCollector {
#[must_use]
pub fn new(pastebin_root: PathBuf, db_path: PathBuf) -> Self { pub fn new(pastebin_root: PathBuf, db_path: PathBuf) -> Self {
Self { Self {
pastebin_root, pastebin_root,
@ -26,10 +28,11 @@ impl PastebinCollector {
} }
} }
#[async_trait]
impl worker::SimpleWorker for PastebinCollector { impl worker::SimpleWorker for PastebinCollector {
type J = Pastebin; type J = Pastebin;
fn msg_to_job( async fn msg_to_job(
&mut self, &mut self,
_routing_key: &str, _routing_key: &str,
_: &Option<String>, _: &Option<String>,
@ -47,7 +50,7 @@ impl worker::SimpleWorker for PastebinCollector {
} }
} }
fn consumer(&mut self, _chan: &mut lapin::Channel, job: &Self::J) -> worker::Actions { async fn consumer(&mut self, _chan: &mut lapin::Channel, job: &Self::J) -> worker::Actions {
let span = debug_span!("pastebin", title = ?job.title); let span = debug_span!("pastebin", title = ?job.title);
let _enter = span.enter(); let _enter = span.enter();

View file

@ -1,6 +1,7 @@
use crate::stats; use crate::stats;
use crate::worker; use crate::worker;
use async_trait::async_trait;
use tracing::error; use tracing::error;
pub struct StatCollectorWorker<E> { pub struct StatCollectorWorker<E> {
@ -14,29 +15,39 @@ impl<E: stats::SysEvents + 'static> StatCollectorWorker<E> {
} }
} }
#[async_trait]
impl<E: stats::SysEvents + 'static> worker::SimpleWorker for StatCollectorWorker<E> { impl<E: stats::SysEvents + 'static> worker::SimpleWorker for StatCollectorWorker<E> {
type J = stats::EventMessage; type J = stats::EventMessage;
fn msg_to_job(&mut self, _: &str, _: &Option<String>, body: &[u8]) -> Result<Self::J, String> { async fn msg_to_job(
match serde_json::from_slice(body) { &mut self,
Ok(e) => Ok(e), _: &str,
Err(_) => { _: &Option<String>,
body: &[u8],
) -> Result<Self::J, String> {
if let Ok(e) = serde_json::from_slice(body) {
Ok(e)
} else {
let mut modified_body: Vec<u8> = vec![b"\""[0]]; let mut modified_body: Vec<u8> = vec![b"\""[0]];
modified_body.append(&mut body.to_vec()); modified_body.append(&mut body.to_vec());
modified_body.push(b"\""[0]); modified_body.push(b"\""[0]);
match serde_json::from_slice(&modified_body) { match serde_json::from_slice(&modified_body) {
Ok(e) => { Ok(e) => {
self.events.notify(stats::Event::StatCollectorLegacyEvent( self.events
.notify(stats::Event::StatCollectorLegacyEvent(
stats::event_metric_name(&e), stats::event_metric_name(&e),
)); ))
.await;
Ok(stats::EventMessage { Ok(stats::EventMessage {
sender: "".to_owned(), sender: String::new(),
events: vec![e], events: vec![e],
}) })
} }
Err(e) => { Err(e) => {
self.events.notify(stats::Event::StatCollectorBogusEvent); self.events
.notify(stats::Event::StatCollectorBogusEvent)
.await;
error!( error!(
"Failed to decode message: {:?}, Err: {:?}", "Failed to decode message: {:?}, Err: {:?}",
String::from_utf8(body.to_vec()), String::from_utf8(body.to_vec()),
@ -47,15 +58,14 @@ impl<E: stats::SysEvents + 'static> worker::SimpleWorker for StatCollectorWorker
} }
} }
} }
}
fn consumer( async fn consumer(
&mut self, &mut self,
_chan: &mut lapin::Channel, _chan: &mut lapin::Channel,
job: &stats::EventMessage, job: &stats::EventMessage,
) -> worker::Actions { ) -> worker::Actions {
let sender = job.sender.clone(); let sender = job.sender.clone();
for event in job.events.iter() { for event in &job.events {
self.collector.record(sender.clone(), event.clone()); self.collector.record(sender.clone(), event.clone());
} }

View file

@ -1,5 +1,6 @@
use std::path::PathBuf; use std::path::PathBuf;
use async_trait::async_trait;
use tracing::{debug_span, error}; use tracing::{debug_span, error};
use crate::worker; use crate::worker;
@ -16,16 +17,18 @@ pub struct StatusCheckCollector {
} }
impl StatusCheckCollector { impl StatusCheckCollector {
#[must_use]
pub fn new(db_path: PathBuf) -> Self { pub fn new(db_path: PathBuf) -> Self {
Self { db_path } Self { db_path }
} }
} }
// RPC API worker // RPC API worker
#[async_trait]
impl worker::SimpleWorker for StatusCheckCollector { impl worker::SimpleWorker for StatusCheckCollector {
type J = StatusCheckRPCMessage; type J = StatusCheckRPCMessage;
fn msg_to_job( async fn msg_to_job(
&mut self, &mut self,
_method: &str, _method: &str,
_headers: &Option<String>, _headers: &Option<String>,
@ -43,7 +46,7 @@ impl worker::SimpleWorker for StatusCheckCollector {
} }
} }
fn consumer(&mut self, _chan: &mut lapin::Channel, _job: &Self::J) -> worker::Actions { async fn consumer(&mut self, _chan: &mut lapin::Channel, _job: &Self::J) -> worker::Actions {
let span = debug_span!("command"); let span = debug_span!("command");
let _enter = span.enter(); let _enter = span.enter();

View file

@ -9,12 +9,13 @@ pub struct TestScratch {
} }
impl TestScratch { impl TestScratch {
#[must_use]
pub fn new_dir(ident: &str) -> TestScratch { pub fn new_dir(ident: &str) -> TestScratch {
let scratch = TestScratch { let scratch = TestScratch {
root: Path::new(env!("CARGO_MANIFEST_DIR")) root: Path::new(env!("CARGO_MANIFEST_DIR"))
.join("test-scratch") .join("test-scratch")
.join("dirs") .join("dirs")
.join(format!("dir-{}", ident)), .join(format!("dir-{ident}")),
}; };
TestScratch::create_dir(&scratch); TestScratch::create_dir(&scratch);
@ -22,12 +23,13 @@ impl TestScratch {
scratch scratch
} }
#[must_use]
pub fn new_file(ident: &str) -> TestScratch { pub fn new_file(ident: &str) -> TestScratch {
let scratch = TestScratch { let scratch = TestScratch {
root: Path::new(env!("CARGO_MANIFEST_DIR")) root: Path::new(env!("CARGO_MANIFEST_DIR"))
.join("test-scratch") .join("test-scratch")
.join("files") .join("files")
.join(format!("file-{}", ident)), .join(format!("file-{ident}")),
}; };
TestScratch::create_dir(&scratch); TestScratch::create_dir(&scratch);
@ -40,10 +42,12 @@ impl TestScratch {
fs::create_dir_all(target).unwrap(); fs::create_dir_all(target).unwrap();
} }
#[must_use]
pub fn path(&self) -> PathBuf { pub fn path(&self) -> PathBuf {
self.root.clone() self.root.clone()
} }
#[must_use]
pub fn string(&self) -> String { pub fn string(&self) -> String {
self.path().to_str().unwrap().to_owned() self.path().to_str().unwrap().to_owned()
} }

View file

@ -1,6 +1,5 @@
use std::rc::Rc; use std::sync::Arc;
use futures::executor::block_on;
use tracing::warn; use tracing::warn;
use crate::vcs::generic::State; use crate::vcs::generic::State;
@ -8,7 +7,7 @@ use crate::vcs::generic::State;
use super::generic::VersionControlSystemAPI; use super::generic::VersionControlSystemAPI;
pub struct CommitStatus { pub struct CommitStatus {
api: Rc<dyn VersionControlSystemAPI>, api: Arc<dyn VersionControlSystemAPI>,
repo: crate::message::Repo, repo: crate::message::Repo,
sha: String, sha: String,
context: String, context: String,
@ -18,7 +17,7 @@ pub struct CommitStatus {
impl CommitStatus { impl CommitStatus {
pub fn new( pub fn new(
api: Rc<dyn VersionControlSystemAPI>, api: Arc<dyn VersionControlSystemAPI>,
repo: crate::message::Repo, repo: crate::message::Repo,
sha: String, sha: String,
context: String, context: String,
@ -31,7 +30,7 @@ impl CommitStatus {
sha, sha,
context, context,
description, description,
url: "".to_owned(), url: String::new(),
}; };
stat.set_url(url); stat.set_url(url);
@ -40,23 +39,23 @@ impl CommitStatus {
} }
pub fn set_url(&mut self, url: Option<String>) { pub fn set_url(&mut self, url: Option<String>) {
self.url = url.unwrap_or_else(|| String::from("")) self.url = url.unwrap_or_default();
} }
pub fn set_with_description( pub async fn set_with_description(
&mut self, &mut self,
description: &str, description: &str,
state: State, state: State,
) -> Result<(), CommitStatusError> { ) -> Result<(), CommitStatusError> {
self.set_description(description.to_owned()); self.set_description(description.to_owned());
self.set(state) self.set(state).await
} }
pub fn set_description(&mut self, description: String) { pub fn set_description(&mut self, description: String) {
self.description = description; self.description = description;
} }
pub fn set(&self, state: State) -> Result<(), CommitStatusError> { pub async fn set(&self, state: State) -> Result<(), CommitStatusError> {
let desc = if self.description.len() >= 140 { let desc = if self.description.len() >= 140 {
warn!( warn!(
"description is over 140 char; truncating: {:?}", "description is over 140 char; truncating: {:?}",
@ -67,14 +66,16 @@ impl CommitStatus {
self.description.clone() self.description.clone()
}; };
block_on(self.api.create_commit_statuses( self.api
.create_commit_statuses(
&self.repo, &self.repo,
self.sha.clone(), self.sha.clone(),
state, state,
self.context.clone(), self.context.clone(),
desc, desc,
self.url.clone(), self.url.clone(),
)) )
.await
} }
} }

View file

@ -34,12 +34,13 @@ pub struct ChangeReviewers {
} }
impl Issue { impl Issue {
#[must_use]
pub fn is_wip(&self) -> bool { pub fn is_wip(&self) -> bool {
false false
} }
} }
pub trait VersionControlSystemAPI { pub trait VersionControlSystemAPI: Sync + Send {
fn get_repository(&self, repo: &crate::message::Repo) -> Repository; fn get_repository(&self, repo: &crate::message::Repo) -> Repository;
fn get_changes(&self, repo: &crate::message::Repo) -> BoxFuture<Vec<Change>>; fn get_changes(&self, repo: &crate::message::Repo) -> BoxFuture<Vec<Change>>;
fn get_change(&self, repo: &crate::message::Repo, number: u64) -> BoxFuture<Option<Change>>; fn get_change(&self, repo: &crate::message::Repo, number: u64) -> BoxFuture<Option<Change>>;

View file

@ -1,7 +1,7 @@
use crate::vcs::generic::CheckRunState; use crate::vcs::generic::CheckRunState;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
/// Port from https://gerrit.googlesource.com/gerrit/+/master/polygerrit-ui/app/api/checks.ts /// Port from <https://gerrit.googlesource.com/gerrit/+/master/polygerrit-ui/app/api/checks.ts>
#[derive(Debug, Serialize, Deserialize, PartialEq)] #[derive(Debug, Serialize, Deserialize, PartialEq)]
#[serde(rename_all = "UPPERCASE")] #[serde(rename_all = "UPPERCASE")]

View file

@ -7,22 +7,23 @@ use super::data_structures::{Account, Change};
pub struct GerritHTTPApi; pub struct GerritHTTPApi;
#[allow(clippy::unused_async)] // FIXME
impl GerritHTTPApi { impl GerritHTTPApi {
// async fn get_project(&self, project_name: &str) -> Project {} // async fn get_project(&self, project_name: &str) -> Project {}
/// Fetches all changes according to the query and the given limit. /// Fetches all changes according to the query and the given limit.
/// This will default to 60 changes by default. /// This will default to 60 changes by default.
pub(crate) async fn list_changes(&self, _query: &str, _limit: Option<u64>) -> Vec<Change> { pub(crate) async fn list_changes(&self, _query: &str, _limit: Option<u64>) -> Vec<Change> {
Default::default() vec![]
} }
/// Fetch the latest change ID for a given project and CL number. /// Fetch the latest change ID for a given project and CL number.
pub(crate) async fn get_change_id(&self, _project_name: &str, _cl_number: u64) -> String { pub(crate) async fn get_change_id(&self, _project_name: &str, _cl_number: u64) -> String {
"".to_owned() String::new()
} }
/// Fetch a given change according to the change ID (not the CL number). /// Fetch a given change according to the change ID (not the CL number).
pub(crate) async fn get_change(&self, _change_id: &str) -> Option<Change> { pub(crate) async fn get_change(&self, _change_id: &str) -> Option<Change> {
Default::default() None
} }
/// Set additional and remove certain hashtags for a given change ID (not the CL number). /// Set additional and remove certain hashtags for a given change ID (not the CL number).
@ -32,11 +33,11 @@ impl GerritHTTPApi {
_add: &[String], _add: &[String],
_remove: &[String], _remove: &[String],
) -> Vec<String> { ) -> Vec<String> {
Default::default() vec![]
} }
/// List all reviewers on a given change ID (not the CL number). /// List all reviewers on a given change ID (not the CL number).
pub(crate) async fn list_reviewers(&self, _change_id: &str) -> Vec<Account> { pub(crate) async fn list_reviewers(&self, _change_id: &str) -> Vec<Account> {
Default::default() vec![]
} }
/// Set reviewers and a message on a given change ID (not the CL number). /// Set reviewers and a message on a given change ID (not the CL number).
pub(crate) async fn set_reviewers( pub(crate) async fn set_reviewers(

View file

@ -50,12 +50,12 @@ impl VersionControlSystemAPI for GerritHTTPApi {
&self, &self,
repo: &crate::message::Repo, repo: &crate::message::Repo,
) -> futures_util::future::BoxFuture<Vec<crate::message::Change>> { ) -> futures_util::future::BoxFuture<Vec<crate::message::Change>> {
let repo_name = repo.name.to_owned(); let repo_name = repo.name.clone();
async move { async move {
self.list_changes(&format!("project:{}", &repo_name), None) self.list_changes(&format!("project:{}", &repo_name), None)
.await .await
.into_iter() .into_iter()
.map(|c| c.into()) .map(std::convert::Into::into)
.collect() .collect()
} }
.boxed() .boxed()
@ -66,12 +66,12 @@ impl VersionControlSystemAPI for GerritHTTPApi {
repo: &crate::message::Repo, repo: &crate::message::Repo,
number: u64, number: u64,
) -> futures_util::future::BoxFuture<Option<crate::message::Change>> { ) -> futures_util::future::BoxFuture<Option<crate::message::Change>> {
let repo_name = repo.name.to_owned(); let repo_name = repo.name.clone();
async move { async move {
let change_id = self.get_change_id(&repo_name, number).await; let change_id = self.get_change_id(&repo_name, number).await;
GerritHTTPApi::get_change(self, &change_id) GerritHTTPApi::get_change(self, &change_id)
.await .await
.map(|c| c.into()) .map(std::convert::Into::into)
} }
.boxed() .boxed()
} }
@ -85,7 +85,7 @@ impl VersionControlSystemAPI for GerritHTTPApi {
) -> futures_util::future::BoxFuture<()> { ) -> futures_util::future::BoxFuture<()> {
let add = add.to_owned(); let add = add.to_owned();
let remove = remove.to_owned(); let remove = remove.to_owned();
let repo_name = repo.name.to_owned(); let repo_name = repo.name.clone();
async move { async move {
let change_id = self.get_change_id(&repo_name, number).await; let change_id = self.get_change_id(&repo_name, number).await;
@ -99,7 +99,7 @@ impl VersionControlSystemAPI for GerritHTTPApi {
repo: &crate::message::Repo, repo: &crate::message::Repo,
number: u64, number: u64,
) -> futures_util::future::BoxFuture<crate::vcs::generic::ChangeReviewers> { ) -> futures_util::future::BoxFuture<crate::vcs::generic::ChangeReviewers> {
let repo_name = repo.name.to_owned(); let repo_name = repo.name.clone();
async move { async move {
let change_id = self.get_change_id(&repo_name, number).await; let change_id = self.get_change_id(&repo_name, number).await;
self.list_reviewers(&change_id).await.into() self.list_reviewers(&change_id).await.into()
@ -115,7 +115,7 @@ impl VersionControlSystemAPI for GerritHTTPApi {
// FIXME: support group reviews // FIXME: support group reviews
_team_reviewers: Vec<String>, _team_reviewers: Vec<String>,
) -> futures_util::future::BoxFuture<()> { ) -> futures_util::future::BoxFuture<()> {
let repo_name = repo.name.to_owned(); let repo_name = repo.name.clone();
async move { async move {
let change_id = self.get_change_id(&repo_name, number).await; let change_id = self.get_change_id(&repo_name, number).await;
self.set_reviewers( self.set_reviewers(
@ -130,7 +130,7 @@ impl VersionControlSystemAPI for GerritHTTPApi {
}) })
.collect(), .collect(),
) )
.await .await;
} }
.boxed() .boxed()
} }

View file

@ -1,7 +1,7 @@
use std::{error::Error, path::PathBuf}; use std::{error::Error, path::PathBuf};
use async_stream::stream; use async_stream::stream;
use futures::{executor::block_on, StreamExt}; use futures::StreamExt;
use futures_util::Stream; use futures_util::Stream;
use openssh::{Session, SessionBuilder, Stdio}; use openssh::{Session, SessionBuilder, Stdio};
use tokio::io::AsyncBufReadExt; use tokio::io::AsyncBufReadExt;
@ -13,10 +13,13 @@ pub struct GerritSSHApi {
} }
impl GerritSSHApi { impl GerritSSHApi {
pub fn new(private_key_file: PathBuf, uri: &str) -> Self { pub async fn new(private_key_file: PathBuf, uri: &str) -> Self {
let mut builder = SessionBuilder::default(); let mut builder = SessionBuilder::default();
let (builder, destination) = builder.keyfile(&private_key_file).resolve(uri); let (builder, destination) = builder.keyfile(&private_key_file).resolve(uri);
let tempdir = block_on(builder.launch_master(destination)).unwrap_or_else(|_| panic!("Failed to launch SSH master to destination '{}'", uri)); let tempdir = builder
.launch_master(destination)
.await
.unwrap_or_else(|_| panic!("Failed to launch SSH master to destination '{}'", uri));
Self { Self {
session: Session::new_process_mux(tempdir), session: Session::new_process_mux(tempdir),
} }

View file

@ -1,3 +1,4 @@
use async_trait::async_trait;
use std::marker::Send; use std::marker::Send;
use futures::Stream; use futures::Stream;
@ -34,8 +35,8 @@ where
T: Serialize + ?Sized, T: Serialize + ?Sized,
{ {
QueueMsg { QueueMsg {
exchange: exchange.map(|s| s.to_owned()), exchange: exchange.map(std::borrow::ToOwned::to_owned),
routing_key: routing_key.map(|s| s.to_owned()), routing_key: routing_key.map(std::borrow::ToOwned::to_owned),
mandatory: false, mandatory: false,
immediate: false, immediate: false,
content_type: Some("application/json".to_owned()), content_type: Some("application/json".to_owned()),
@ -44,8 +45,8 @@ where
} }
pub fn publish_serde_action<T>( pub fn publish_serde_action<T>(
exchange: Option<String>, exchange: &Option<String>,
routing_key: Option<String>, routing_key: &Option<String>,
msg: &T, msg: &T,
) -> Action ) -> Action
where where
@ -58,11 +59,13 @@ where
))) )))
} }
#[async_trait]
pub trait SimpleWorker: Send { pub trait SimpleWorker: Send {
type J: Send; type J: Send;
fn consumer(&mut self, chan: &mut lapin::Channel, job: &Self::J) -> Actions;
fn msg_to_job( async fn consumer(&mut self, chan: &mut lapin::Channel, job: &Self::J) -> Actions;
async fn msg_to_job(
&mut self, &mut self,
method: &str, method: &str,
headers: &Option<String>, headers: &Option<String>,

View file

@ -8,6 +8,7 @@ pub struct LineWriter {
} }
impl LineWriter { impl LineWriter {
#[must_use]
pub fn new(mut rw: File) -> LineWriter { pub fn new(mut rw: File) -> LineWriter {
let buf = LineWriter::load_buffer(&mut rw); let buf = LineWriter::load_buffer(&mut rw);
let len = buf.len(); let len = buf.len();
@ -27,7 +28,7 @@ impl LineWriter {
.lines() .lines()
.map(|line| match line { .map(|line| match line {
Ok(s) => s, Ok(s) => s,
Err(e) => format!("UTF-8 Decode err: {:?}", e), Err(e) => format!("UTF-8 Decode err: {e:?}"),
}) })
.collect() .collect()
} }
@ -35,7 +36,7 @@ impl LineWriter {
pub fn write_to_line(&mut self, line: usize, data: &str) { pub fn write_to_line(&mut self, line: usize, data: &str) {
let original_len = self.buffer.len(); let original_len = self.buffer.len();
while self.buffer.len() <= line { while self.buffer.len() <= line {
self.buffer.push("".to_owned()); self.buffer.push(String::new());
} }
self.buffer.remove(line); self.buffer.remove(line);
@ -72,6 +73,7 @@ impl LineWriter {
self.last_line = line; self.last_line = line;
} }
#[must_use]
pub fn inner(self) -> File { pub fn inner(self) -> File {
self.file self.file
} }