the-distro/ofborg
2
0
Fork 1
Code Issues 1 Pull requests 1 Packages Projects Releases Wiki Activity

Compare commits

base: the-distro:b561c15dcfac8276b04f8c6ee42a28bb841f75df
Branches Tags
the-distro:main
the-distro:vcs-generalization
the-distro:systemd-creds
...
compare: the-distro:63485419826e2c489a3d1129927780b322874964
Branches Tags
the-distro:vcs-generalization
the-distro:main
the-distro:systemd-creds

2 commits
b561c15dcf ... 6348541982

Author SHA1 Message Date
Ilya K 6348541982 fix: delete unused macro 2024-11-16 23:52:05 +03:00
Ilya K 43ec489263 refactor: asyncify everything 
Tracing actually works now. Me and clippy have signed a non-aggression pact.
 2024-11-16 23:49:18 +03:00
59 changed files with 1561 additions and 1325 deletions
Show stats Expand all files Collapse all files

31
Cargo.lock generated
View file

@ -1833,6 +1833,7 @@ name = "ofborg"
version = "0.90.0"
dependencies = [
"async-stream",
"async-trait",
"axum",
"base64",
"brace-expand",
@ -1854,6 +1855,7 @@ dependencies = [
"opentelemetry",
"opentelemetry-otlp",
"opentelemetry-semantic-conventions",
"opentelemetry-stdout",
"opentelemetry_sdk",
"regex",
"rustls-pemfile",
@ -1880,6 +1882,7 @@ version = "0.1.0"
dependencies = [
"log",
"ofborg",
"tokio",
]
[[package]]
@ -1986,6 +1989,23 @@ version = "0.27.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc1b6902ff63b32ef6c489e8048c5e253e2e4a803ea3ea7e783914536eb15c52"
[[package]]
name = "opentelemetry-stdout"
version = "0.27.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc8a298402aa5c260be90d10dc54b5a7d4e1025c354848f8e2c976d761351049"
dependencies = [
"async-trait",
"chrono",
"futures-util",
"opentelemetry",
"opentelemetry_sdk",
"ordered-float",
"serde",
"serde_json",
"thiserror 1.0.69",
]
[[package]]
name = "opentelemetry_sdk"
version = "0.27.0"
@ -2003,6 +2023,8 @@ dependencies = [
"rand",
"serde_json",
"thiserror 1.0.69",
"tokio",
"tokio-stream",
"tracing",
]
@ -2012,6 +2034,15 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"
[[package]]
name = "ordered-float"
version = "4.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c65ee1f9701bf938026630b455d5315f490640234259037edb259798b3bcf85e"
dependencies = [
"num-traits",
]
[[package]]
name = "overload"
version = "0.1.1"

1
ofborg-simple-build/Cargo.toml
View file

@ -7,4 +7,5 @@ edition = "2018"
[dependencies]
ofborg = { path = "../ofborg" }
tokio = "*"
log = "0.4.22"

16
ofborg-simple-build/src/main.rs
View file

@ -8,7 +8,8 @@ use std::path::Path;
use ofborg::config;
use ofborg::nix;
fn main() {
#[tokio::main]
async fn main() {
ofborg::setup_log();
log::info!("Loading config...");
@ -16,11 +17,14 @@ fn main() {
let nix = cfg.nix();
log::info!("Running build...");
match nix.safely_build_attrs(
Path::new("./"),
nix::File::DefaultNixpkgs,
vec![String::from("hello")],
) {
match nix
.safely_build_attrs(
Path::new("./"),
nix::File::DefaultNixpkgs,
vec![String::from("hello")],
)
.await
{
Ok(mut out) => {
print!("{}", file_to_str(&mut out));
}

8
ofborg/Cargo.toml
View file

@ -10,6 +10,7 @@ edition = "2018"
[dependencies]
async-stream = "0.3.6"
async-trait = "0.1.83"
axum = "0.7.8"
base64 = "0.22.1"
brace-expand = "0.1.0"
@ -26,12 +27,13 @@ jfs = "0.9.0"
lapin = "2.5.0"
lru-cache = "0.1.2"
md5 = "0.7.0"
nom = "4,<5"
nom = "4,<5" # FIXME: update
openssh = { version = "0.11.3", features = ["process-mux"], default-features = false }
opentelemetry = "0.27.0"
opentelemetry_sdk = { version = "0.27.0", features = ["rt-tokio"] }
opentelemetry-otlp = { version = "0.27.0", features = ["http-json", "reqwest-client", "reqwest-rustls"] }
opentelemetry-semantic-conventions = "0.27.0"
opentelemetry_sdk = "0.27.0"
opentelemetry-otlp = { version = "0.27.0", features = ["http-json", "reqwest-blocking-client", "reqwest-rustls"] }
opentelemetry-stdout = "0.27.0"
regex = "1.11.1"
rustls-pemfile = "2.2.0"
separator = "0.4.1"

7
ofborg/src/acl.rs
View file

@ -6,6 +6,7 @@ pub struct Acl {
}
impl Acl {
#[must_use]
pub fn new(repos: Vec<String>, mut trusted_users: Option<Vec<String>>) -> Acl {
if let Some(ref mut users) = trusted_users {
users.iter_mut().map(|x| *x = x.to_lowercase()).last();
@ -17,10 +18,12 @@ impl Acl {
}
}
#[must_use]
pub fn is_repo_eligible(&self, name: &str) -> bool {
self.repos.contains(&name.to_lowercase())
}
#[must_use]
pub fn build_job_architectures_for_user_repo(&self, user: &str, repo: &str) -> Vec<System> {
if self.can_build_unrestricted(user, repo) {
vec![
@ -35,6 +38,7 @@ impl Acl {
}
}
#[must_use]
pub fn build_job_destinations_for_user_repo(
&self,
user: &str,
@ -42,10 +46,11 @@ impl Acl {
) -> Vec<(Option<String>, Option<String>)> {
self.build_job_architectures_for_user_repo(user, repo)
.iter()
.map(|system| system.as_build_destination())
.map(super::systems::System::as_build_destination)
.collect()
}
#[must_use]
pub fn can_build_unrestricted(&self, user: &str, repo: &str) -> bool {
if let Some(ref users) = self.trusted_users {
if repo.to_lowercase() == "nixos/nixpkgs" {

10
ofborg/src/asynccmd.rs
View file

@ -82,10 +82,12 @@ fn child_wait<T: Send + 'static>(
}
impl AsyncCmd {
#[must_use]
pub fn new(cmd: Command) -> AsyncCmd {
AsyncCmd { command: cmd }
}
#[must_use]
pub fn spawn(mut self) -> SpawnedAsyncCmd {
let mut child = self
.command
@ -122,7 +124,7 @@ impl AsyncCmd {
child_wait(WaitTarget::Child, monitor_tx, child),
);
let head_waiter = thread::spawn(move || block_on_waiters(monitor_rx, waiters));
let head_waiter = thread::spawn(move || block_on_waiters(&monitor_rx, waiters));
SpawnedAsyncCmd {
waiter: head_waiter,
@ -152,12 +154,12 @@ impl SpawnedAsyncCmd {
// FIXME: remove with rust/cargo update
#[allow(clippy::cognitive_complexity)]
fn block_on_waiters(
monitor_rx: mpsc::Receiver<(WaitTarget, WaitResult<()>)>,
monitor_rx: &mpsc::Receiver<(WaitTarget, WaitResult<()>)>,
mut waiters: HashMap<WaitTarget, thread::JoinHandle<()>>,
) -> Option<Result<ExitStatus, io::Error>> {
let mut status = None;
for (id, interior_result) in monitor_rx.iter() {
for (id, interior_result) in monitor_rx {
match waiters.remove(&id) {
Some(handle) => {
info!("Received notice that {:?} finished", id);
@ -262,7 +264,7 @@ mod tests {
let mut spawned = acmd.spawn();
let lines: Vec<String> = spawned.lines().collect();
assert_eq!(lines.len(), 200000);
assert_eq!(lines.len(), 200_000);
let thread_result = spawned.wait();
let exit_status = thread_result.expect("Thread should exit correctly");
assert!(exit_status.success());

15
ofborg/src/bin/build-faker.rs
View file

@ -1,7 +1,6 @@
use std::env;
use std::error::Error;
use futures::executor::block_on;
use lapin::message::Delivery;
use lapin::BasicProperties;
@ -12,14 +11,15 @@ use ofborg::message::{buildjob, Change, Repo};
use ofborg::notifyworker::NotificationReceiver;
use ofborg::worker;
fn main() -> Result<(), Box<dyn Error>> {
#[tokio::main]
async fn main() -> Result<(), Box<dyn Error>> {
ofborg::setup_log();
let arg = env::args().nth(1).expect("usage: build-faker <config>");
let cfg = config::load(arg.as_ref());
let conn = easylapin::from_config(&cfg.rabbitmq)?;
let mut chan = block_on(conn.create_channel())?;
let conn = easylapin::from_config(&cfg.rabbitmq).await?;
let mut chan = conn.create_channel().await?;
let repo_msg = Repo {
clone_url: "https://github.com/nixos/ofborg.git".to_owned(),
@ -60,10 +60,11 @@ fn main() -> Result<(), Box<dyn Error>> {
for _i in 1..2 {
recv.tell(worker::publish_serde_action(
None,
Some("build-inputs-x86_64-darwin".to_owned()),
&None,
&Some("build-inputs-x86_64-darwin".to_owned()),
&msg,
));
))
.await;
}
}

38
ofborg/src/bin/builder.rs
View file

@ -2,9 +2,7 @@ use std::env;
use std::error::Error;
use std::path::Path;
use futures::executor::block_on;
use futures_util::future;
use tokio::spawn;
use tokio::task::JoinHandle;
use tracing::{info, warn};
@ -14,7 +12,8 @@ use ofborg::{checkout, config, tasks};
// FIXME: remove with rust/cargo update
#[allow(clippy::cognitive_complexity)]
fn main() -> Result<(), Box<dyn Error>> {
#[tokio::main]
async fn main() -> Result<(), Box<dyn Error>> {
ofborg::setup_log();
let arg = env::args().nth(1).expect("usage: builder <config>");
@ -30,27 +29,27 @@ fn main() -> Result<(), Box<dyn Error>> {
panic!();
};
let conn = easylapin::from_config(&cfg.rabbitmq)?;
let conn = easylapin::from_config(&cfg.rabbitmq).await?;
let mut handles = Vec::new();
for system in &cfg.nix.system {
let handle_ext = self::create_handle(&conn, &cfg, system.to_string())?;
let handle_ext = self::create_handle(&conn, &cfg, system.to_string());
handles.push(handle_ext);
}
block_on(future::join_all(handles));
future::join_all(handles).await;
drop(conn); // Close connection.
info!("Closed the session... EOF");
Ok(())
}
fn create_handle(
async fn create_handle(
conn: &lapin::Connection,
cfg: &config::Config,
system: String,
) -> Result<JoinHandle<()>, Box<dyn Error>> {
let mut chan = block_on(conn.create_channel())?;
) -> Result<JoinHandle<Result<(), lapin::Error>>, Box<dyn Error>> {
let mut chan = conn.create_channel().await?;
let cloner = checkout::cached_cloner(Path::new(&cfg.checkout.root));
let nix = cfg.nix().with_system(system.clone());
@ -63,7 +62,8 @@ fn create_handle(
auto_delete: false,
no_wait: false,
internal: false,
})?;
})
.await?;
let queue_name = if cfg.runner.build_all_jobs != Some(true) {
let queue_name = format!("build-inputs-{}", system);
@ -74,7 +74,8 @@ fn create_handle(
exclusive: false,
auto_delete: false,
no_wait: false,
})?;
})
.await?;
queue_name
} else {
warn!("Building all jobs, please don't use this unless you're");
@ -87,7 +88,8 @@ fn create_handle(
exclusive: true,
auto_delete: true,
no_wait: false,
})?;
})
.await?;
queue_name
};
@ -96,9 +98,12 @@ fn create_handle(
exchange: "build-jobs".to_owned(),
routing_key: None,
no_wait: false,
})?;
})
.await?;
let handle = easylapin::NotifyChannel(chan).consume(
info!("Fetching jobs from {}", &queue_name);
let fut = easylapin::NotifyChannel(chan).consume(
tasks::build::BuildWorker::new(cloner, nix, system, cfg.runner.identity.clone()),
easyamqp::ConsumeConfig {
queue: queue_name.clone(),
@ -108,8 +113,7 @@ fn create_handle(
no_wait: false,
exclusive: false,
},
)?;
);
info!("Fetching jobs from {}", &queue_name);
Ok(spawn(handle))
Ok(tokio::spawn(fut))
}

46
ofborg/src/bin/evaluation-filter.rs
View file

@ -1,7 +1,6 @@
use std::env;
use std::error::Error;
use futures::executor::block_on;
use tracing::info;
use ofborg::config;
@ -9,7 +8,8 @@ use ofborg::easyamqp::{self, ChannelExt, ConsumerExt};
use ofborg::easylapin;
use ofborg::tasks;
fn main() -> Result<(), Box<dyn Error>> {
#[tokio::main]
async fn main() -> Result<(), Box<dyn Error>> {
ofborg::setup_log();
let arg = env::args()
@ -17,8 +17,8 @@ fn main() -> Result<(), Box<dyn Error>> {
.expect("usage: evaluation-filter <config>");
let cfg = config::load(arg.as_ref());
let conn = easylapin::from_config(&cfg.rabbitmq)?;
let mut chan = block_on(conn.create_channel())?;
let conn = easylapin::from_config(&cfg.rabbitmq).await?;
let mut chan = conn.create_channel().await?;
chan.declare_exchange(easyamqp::ExchangeConfig {
exchange: "github-events".to_owned(),
@ -28,7 +28,8 @@ fn main() -> Result<(), Box<dyn Error>> {
auto_delete: false,
no_wait: false,
internal: false,
})?;
})
.await?;
chan.declare_queue(easyamqp::QueueConfig {
queue: "mass-rebuild-check-jobs".to_owned(),
@ -37,7 +38,8 @@ fn main() -> Result<(), Box<dyn Error>> {
exclusive: false,
auto_delete: false,
no_wait: false,
})?;
})
.await?;
let queue_name = String::from("mass-rebuild-check-inputs");
chan.declare_queue(easyamqp::QueueConfig {
@ -47,29 +49,31 @@ fn main() -> Result<(), Box<dyn Error>> {
exclusive: false,
auto_delete: false,
no_wait: false,
})?;
})
.await?;
chan.bind_queue(easyamqp::BindQueueConfig {
queue: queue_name.clone(),
exchange: "github-events".to_owned(),
routing_key: Some("pull_request.nixos/nixpkgs".to_owned()),
no_wait: false,
})?;
let handle = easylapin::WorkerChannel(chan).consume(
tasks::evaluationfilter::EvaluationFilterWorker::new(cfg.acl()),
easyamqp::ConsumeConfig {
queue: queue_name.clone(),
consumer_tag: format!("{}-evaluation-filter", cfg.whoami()),
no_local: false,
no_ack: false,
no_wait: false,
exclusive: false,
},
)?;
})
.await?;
info!("Fetching jobs from {}", &queue_name);
block_on(handle);
easylapin::WorkerChannel(chan)
.consume(
tasks::evaluationfilter::EvaluationFilterWorker::new(cfg.acl()),
easyamqp::ConsumeConfig {
queue: queue_name.clone(),
consumer_tag: format!("{}-evaluation-filter", cfg.whoami()),
no_local: false,
no_ack: false,
no_wait: false,
exclusive: false,
},
)
.await?;
drop(conn); // Close connection.
info!("Closed the session... EOF");

92
ofborg/src/bin/gerrit-event-streamer.rs
View file

@ -4,7 +4,6 @@
use std::env;
use std::error::Error;
use futures::executor::block_on;
use futures::{pin_mut, StreamExt};
use lapin::options::BasicPublishOptions;
use lapin::BasicProperties;
@ -16,7 +15,8 @@ use ofborg::config;
use ofborg::easyamqp::{self, ChannelExt};
use ofborg::easylapin;
fn main() -> Result<(), Box<dyn Error>> {
#[tokio::main]
async fn main() -> Result<(), Box<dyn Error>> {
ofborg::setup_log();
let arg = env::args()
@ -24,8 +24,8 @@ fn main() -> Result<(), Box<dyn Error>> {
.expect("usage: gerrit-events-streamer <config>");
let cfg = config::load(arg.as_ref());
let conn = easylapin::from_config(&cfg.rabbitmq)?;
let mut chan = block_on(conn.create_channel())?;
let conn = easylapin::from_config(&cfg.rabbitmq).await?;
let mut chan = conn.create_channel().await?;
let exchange_name = "gerrit-events";
@ -37,7 +37,8 @@ fn main() -> Result<(), Box<dyn Error>> {
auto_delete: false,
no_wait: false,
internal: false,
})?;
})
.await?;
info!("Publishing events from Gerrit into {}", &exchange_name);
@ -47,57 +48,50 @@ fn main() -> Result<(), Box<dyn Error>> {
let mut gerrit_api = GerritSSHApi::new(
gerrit_cfg.ssh_private_key_file,
&format!("ssh://{}:{}", gerrit_cfg.instance_uri, gerrit_cfg.ssh_port),
);
)
.await;
let routing_key = "abc";
block_on(async {
let event_stream = gerrit_api.stream_events().await.unwrap();
pin_mut!(event_stream);
loop {
let raw_evt = event_stream.next().await;
tracing::debug!("{:?}", raw_evt);
let event_stream = gerrit_api.stream_events().await.unwrap();
pin_mut!(event_stream);
loop {
let raw_evt = event_stream.next().await;
tracing::debug!("{:?}", raw_evt);
match raw_evt {
Some(Ok(event)) => {
println!("{:#?}", event);
let queue_message =
prepare_queue_message(Some(exchange_name), Some(routing_key), &event);
let props = BasicProperties::default()
.with_delivery_mode(2)
.with_content_type("application/json".into());
match raw_evt {
Some(Ok(event)) => {
println!("{:#?}", event);
let queue_message =
prepare_queue_message(Some(exchange_name), Some(routing_key), &event);
let props = BasicProperties::default()
.with_delivery_mode(2)
.with_content_type("application/json".into());
match chan
.basic_publish(
exchange_name,
routing_key,
BasicPublishOptions::default(),
&queue_message.content,
props,
)
.await
{
Ok(_confirmation) => {
tracing::debug!("Gerrit event published in the exchange");
}
Err(err) => {
tracing::error!("Failed to publish gerrit event: {}", err);
}
match chan
.basic_publish(
exchange_name,
routing_key,
BasicPublishOptions::default(),
&queue_message.content,
props,
)
.await
{
Ok(_confirmation) => {
tracing::debug!("Gerrit event published in the exchange");
}
Err(err) => {
tracing::error!("Failed to publish gerrit event: {}", err);
}
}
Some(Err(_err)) => {
// notify the event
}
None => {
// notify the event
}
}
Some(Err(_err)) => {
// notify the event
}
None => {
// notify the event
}
}
});
//task::block_on(handle);
drop(conn); // Close connection.
info!("Closed the session... EOF");
Ok(())
}
}

27
ofborg/src/bin/listen-gerrit-events.rs
View file

@ -3,13 +3,14 @@
use std::env;
use std::error::Error;
use futures::{executor::block_on, pin_mut, StreamExt};
use futures::{pin_mut, StreamExt};
use ofborg::vcs::gerrit::ssh::GerritSSHApi;
use tracing::info;
use ofborg::config;
fn main() -> Result<(), Box<dyn Error>> {
#[tokio::main]
async fn main() -> Result<(), Box<dyn Error>> {
ofborg::setup_log();
let arg = env::args()
@ -22,19 +23,15 @@ fn main() -> Result<(), Box<dyn Error>> {
.expect("Gerrit event streaming requires a Gerrit configuration");
let gerrit_ssh_uri = format!("ssh://{}:{}", gerrit_cfg.instance_uri, gerrit_cfg.ssh_port);
info!("Listening events from Gerrit on {}", gerrit_ssh_uri);
let mut gerrit_api = GerritSSHApi::new(gerrit_cfg.ssh_private_key_file, &gerrit_ssh_uri);
let mut gerrit_api = GerritSSHApi::new(gerrit_cfg.ssh_private_key_file, &gerrit_ssh_uri).await;
block_on(async {
let event_stream = gerrit_api.stream_events().await.unwrap();
pin_mut!(event_stream);
loop {
let thing = event_stream.next().await;
println!("{:?}", thing);
if let Some(Ok(event)) = thing {
println!("{:#?}", event);
}
let event_stream = gerrit_api.stream_events().await.unwrap();
pin_mut!(event_stream);
loop {
let thing = event_stream.next().await;
println!("{:?}", thing);
if let Some(Ok(event)) = thing {
println!("{:#?}", event);
}
});
Ok(())
}
}

27
ofborg/src/bin/log-message-collector.rs
View file

@ -2,7 +2,6 @@ use std::env;
use std::error::Error;
use std::path::PathBuf;
use futures::executor::block_on;
use tracing::info;
use ofborg::config;
@ -10,7 +9,8 @@ use ofborg::easyamqp::{self, ChannelExt, ConsumerExt};
use ofborg::easylapin;
use ofborg::tasks;
fn main() -> Result<(), Box<dyn Error>> {
#[tokio::main]
async fn main() -> Result<(), Box<dyn Error>> {
ofborg::setup_log();
let arg = env::args()
@ -18,8 +18,8 @@ fn main() -> Result<(), Box<dyn Error>> {
.expect("usage: log-message-collector <config>");
let cfg = config::load(arg.as_ref());
let conn = easylapin::from_config(&cfg.rabbitmq)?;
let mut chan = block_on(conn.create_channel())?;
let conn = easylapin::from_config(&cfg.rabbitmq).await?;
let mut chan = conn.create_channel().await?;
chan.declare_exchange(easyamqp::ExchangeConfig {
exchange: "logs".to_owned(),
@ -29,7 +29,8 @@ fn main() -> Result<(), Box<dyn Error>> {
auto_delete: false,
no_wait: false,
internal: false,
})?;
})
.await?;
let queue_name = "".to_owned();
chan.declare_queue(easyamqp::QueueConfig {
@ -39,17 +40,21 @@ fn main() -> Result<(), Box<dyn Error>> {
exclusive: true,
auto_delete: true,
no_wait: false,
})?;
})
.await?;
chan.bind_queue(easyamqp::BindQueueConfig {
queue: queue_name.clone(),
exchange: "logs".to_owned(),
routing_key: Some("*.*".to_owned()),
no_wait: false,
})?;
})
.await?;
info!("Fetching jobs from {}", &queue_name);
// Regular channel, we want prefetching here.
let handle = chan.consume(
chan.consume(
tasks::log_message_collector::LogMessageCollector::new(
PathBuf::from(cfg.log_storage.clone().unwrap().path),
100,
@ -62,10 +67,8 @@ fn main() -> Result<(), Box<dyn Error>> {
no_wait: false,
exclusive: false,
},
)?;
info!("Fetching jobs from {}", &queue_name);
block_on(handle);
)
.await?;
drop(conn); // Close connection.
info!("Closed the session... EOF");

54
ofborg/src/bin/mass-rebuilder.rs
View file

@ -3,7 +3,6 @@ use std::error::Error;
use std::path::Path;
use std::process;
use futures::executor::block_on;
use ofborg::config::VCSConfig;
use ofborg::tasks::evaluate::SupportedVCS;
use tracing::{error, info};
@ -17,7 +16,8 @@ use ofborg::tasks;
// FIXME: remove with rust/cargo update
#[allow(clippy::cognitive_complexity)]
fn main() -> Result<(), Box<dyn Error>> {
#[tokio::main]
async fn main() -> Result<(), Box<dyn Error>> {
ofborg::setup_log();
let arg = env::args().nth(1).expect("usage: mass-rebuilder <config>");
@ -34,14 +34,14 @@ fn main() -> Result<(), Box<dyn Error>> {
process::exit(1);
};
let conn = easylapin::from_config(&cfg.rabbitmq)?;
let mut chan = block_on(conn.create_channel())?;
let conn = easylapin::from_config(&cfg.rabbitmq).await?;
let mut chan = conn.create_channel().await?;
let root = Path::new(&cfg.checkout.root);
let cloner = checkout::cached_cloner(&root.join(cfg.runner.instance.to_string()));
let nix = cfg.nix();
let events = stats::RabbitMq::from_lapin(&cfg.whoami(), block_on(conn.create_channel())?);
let events = stats::RabbitMq::from_lapin(&cfg.whoami(), conn.create_channel().await?);
let queue_name = String::from("mass-rebuild-check-jobs");
chan.declare_queue(easyamqp::QueueConfig {
@ -51,33 +51,35 @@ fn main() -> Result<(), Box<dyn Error>> {
exclusive: false,
auto_delete: false,
no_wait: false,
})?;
})
.await?;
let vcs_data = match cfg.vcs {
VCSConfig::Gerrit => SupportedVCS::Gerrit,
};
let handle = easylapin::WorkerChannel(chan).consume(
tasks::evaluate::EvaluationWorker::new(
cloner,
&nix,
vcs_data,
cfg.acl(),
cfg.runner.identity.clone(),
events,
),
easyamqp::ConsumeConfig {
queue: queue_name.clone(),
consumer_tag: format!("{}-mass-rebuild-checker", cfg.whoami()),
no_local: false,
no_ack: false,
no_wait: false,
exclusive: false,
},
)?;
info!("Fetching jobs from {}", queue_name);
block_on(handle);
easylapin::WorkerChannel(chan)
.consume(
tasks::evaluate::EvaluationWorker::new(
cloner,
&nix,
vcs_data,
cfg.acl(),
cfg.runner.identity.clone(),
events,
),
easyamqp::ConsumeConfig {
queue: queue_name.clone(),
consumer_tag: format!("{}-mass-rebuild-checker", cfg.whoami()),
no_local: false,
no_ack: false,
no_wait: false,
exclusive: false,
},
)
.await?;
drop(conn); // Close connection.
info!("Closed the session... EOF");

25
ofborg/src/bin/ofborgctl.rs
View file

@ -1,7 +1,6 @@
use std::error::Error;
use std::io::Read;
use futures::executor::block_on;
use lapin::message::Delivery;
use lapin::BasicProperties;
@ -102,7 +101,7 @@ impl<'a> Publisher<'a> {
}
}
fn publish_serde_action<T>(
async fn publish_serde_action<T>(
&mut self,
exchange: Option<String>,
routing_key: Option<String>,
@ -111,18 +110,20 @@ impl<'a> Publisher<'a> {
T: Serialize + ?Sized,
{
self.recv
.tell(worker::publish_serde_action(exchange, routing_key, msg));
.tell(worker::publish_serde_action(&exchange, &routing_key, msg))
.await;
}
}
fn main() -> Result<(), Box<dyn Error>> {
#[tokio::main]
async fn main() -> Result<(), Box<dyn Error>> {
ofborg::setup_log();
let args = Cli::parse();
let cfg = config::load(args.config.as_ref());
let conn = easylapin::from_config(&cfg.rabbitmq)?;
let mut chan = block_on(conn.create_channel())?;
let conn = easylapin::from_config(&cfg.rabbitmq).await?;
let mut chan = conn.create_channel().await?;
match args.command {
Commands::Pastebin { action } => match action {
@ -133,11 +134,13 @@ fn main() -> Result<(), Box<dyn Error>> {
.read_to_string(&mut contents)
.expect("Failed to read pastebin contents");
publisher.publish_serde_action(
None,
Some("pastebin-log".to_owned()),
&Pastebin { title, contents },
);
publisher
.publish_serde_action(
None,
Some("pastebin-log".to_owned()),
&Pastebin { title, contents },
)
.await;
}
},

50
ofborg/src/bin/pastebin-worker.rs
View file

@ -1,7 +1,6 @@
use std::env;
use std::error::Error;
use futures::executor::block_on;
use ofborg::config;
use ofborg::easyamqp;
use ofborg::easyamqp::ChannelExt;
@ -10,14 +9,15 @@ use ofborg::easylapin;
use ofborg::tasks;
use tracing::info;
fn main() -> Result<(), Box<dyn Error>> {
#[tokio::main]
async fn main() -> Result<(), Box<dyn Error>> {
ofborg::setup_log();
let arg = env::args().nth(1).expect("usage: pastebin-worker <config>");
let cfg = config::load(arg.as_ref());
let conn = easylapin::from_config(&cfg.rabbitmq)?;
let mut chan = block_on(conn.create_channel())?;
let conn = easylapin::from_config(&cfg.rabbitmq).await?;
let mut chan = conn.create_channel().await?;
let queue_name = "pastebin-log".to_owned();
@ -29,7 +29,8 @@ fn main() -> Result<(), Box<dyn Error>> {
auto_delete: false,
no_wait: false,
internal: false,
})?;
})
.await?;
chan.declare_queue(easyamqp::QueueConfig {
queue: queue_name.clone(),
@ -38,32 +39,35 @@ fn main() -> Result<(), Box<dyn Error>> {
exclusive: false,
auto_delete: false,
no_wait: false,
})?;
})
.await?;
chan.bind_queue(easyamqp::BindQueueConfig {
queue: queue_name.clone(),
exchange: "pastebin-log".to_owned(),
routing_key: None,
no_wait: false,
})?;
let handle = easylapin::WorkerChannel(chan).consume(
tasks::pastebin_collector::PastebinCollector::new(
cfg.pastebin.clone().root,
cfg.pastebin.clone().db,
),
easyamqp::ConsumeConfig {
queue: queue_name.clone(),
consumer_tag: format!("{}-pastebin-worker", cfg.whoami()),
no_local: false,
no_ack: false,
no_wait: false,
exclusive: false,
},
)?;
})
.await?;
info!("Fetching jobs from {}", &queue_name);
block_on(handle);
easylapin::WorkerChannel(chan)
.consume(
tasks::pastebin_collector::PastebinCollector::new(
cfg.pastebin.clone().root,
cfg.pastebin.clone().db,
),
easyamqp::ConsumeConfig {
queue: queue_name.clone(),
consumer_tag: format!("{}-pastebin-worker", cfg.whoami()),
no_local: false,
no_ack: false,
no_wait: false,
exclusive: false,
},
)
.await?;
drop(conn); // Close connection.
info!("Closed the session... EOF");

44
ofborg/src/bin/statcheck-worker.rs
View file

@ -6,7 +6,6 @@
use std::env;
use std::error::Error;
use futures::executor::block_on;
use ofborg::config;
use ofborg::easyamqp;
use ofborg::easyamqp::ChannelExt;
@ -15,7 +14,8 @@ use ofborg::easylapin;
use ofborg::tasks;
use tracing::info;
fn main() -> Result<(), Box<dyn Error>> {
#[tokio::main]
async fn main() -> Result<(), Box<dyn Error>> {
ofborg::setup_log();
let arg = env::args()
@ -23,8 +23,8 @@ fn main() -> Result<(), Box<dyn Error>> {
.expect("usage: statcheck-worker <config>");
let cfg = config::load(arg.as_ref());
let conn = easylapin::from_config(&cfg.rabbitmq)?;
let mut chan = block_on(conn.create_channel())?;
let conn = easylapin::from_config(&cfg.rabbitmq).await?;
let mut chan = conn.create_channel().await?;
// an RPC queue for verbs
let api_queue_name = "statcheck-api".to_owned();
@ -39,7 +39,8 @@ fn main() -> Result<(), Box<dyn Error>> {
auto_delete: false,
no_wait: false,
internal: false,
})?;
})
.await?;
chan.declare_queue(easyamqp::QueueConfig {
queue: api_queue_name.clone(),
@ -48,30 +49,33 @@ fn main() -> Result<(), Box<dyn Error>> {
exclusive: false,
auto_delete: false,
no_wait: false,
})?;
})
.await?;
chan.bind_queue(easyamqp::BindQueueConfig {
queue: api_queue_name.clone(),
exchange: api_queue_name.clone(),
routing_key: None,
no_wait: false,
})?;
let handle = easylapin::WorkerChannel(chan).consume(
tasks::status_check_collector::StatusCheckCollector::new(cfg.statcheck.clone().db),
easyamqp::ConsumeConfig {
queue: api_queue_name.clone(),
consumer_tag: format!("{}-{}", cfg.whoami(), api_queue_name),
no_local: false,
no_ack: false,
no_wait: false,
exclusive: false,
},
)?;
})
.await?;
info!("Waiting for API calls on {}", api_queue_name);
info!("Notifying of new changes on {}", event_queue_name);
block_on(handle);
easylapin::WorkerChannel(chan)
.consume(
tasks::status_check_collector::StatusCheckCollector::new(cfg.statcheck.clone().db),
easyamqp::ConsumeConfig {
queue: api_queue_name.clone(),
consumer_tag: format!("{}-{}", cfg.whoami(), api_queue_name),
no_local: false,
no_ack: false,
no_wait: false,
exclusive: false,
},
)
.await?;
drop(conn); // Close connection.
info!("Closed the session... EOF");

53
ofborg/src/bin/stats.rs
View file

@ -1,26 +1,24 @@
use std::env;
use std::error::Error;
use std::thread;
use axum::routing::get;
use futures::executor::block_on;
use axum::Router;
use tokio::runtime::Runtime;
use tracing::info;
use ofborg::easyamqp::{ChannelExt, ConsumerExt};
use ofborg::{config, easyamqp, easylapin, stats, tasks};
fn main() -> Result<(), Box<dyn Error>> {
#[tokio::main]
async fn main() -> Result<(), Box<dyn Error>> {
ofborg::setup_log();
let arg = env::args().nth(1).expect("usage: stats <config>");
let cfg = config::load(arg.as_ref());
let conn = easylapin::from_config(&cfg.rabbitmq)?;
let mut chan = block_on(conn.create_channel())?;
let conn = easylapin::from_config(&cfg.rabbitmq).await?;
let mut chan = conn.create_channel().await?;
let events = stats::RabbitMq::from_lapin(&cfg.whoami(), block_on(conn.create_channel())?);
let events = stats::RabbitMq::from_lapin(&cfg.whoami(), conn.create_channel().await?);
let metrics = stats::MetricCollector::new();
let collector = tasks::statscollector::StatCollectorWorker::new(events, metrics.clone());
@ -33,7 +31,8 @@ fn main() -> Result<(), Box<dyn Error>> {
auto_delete: false,
no_wait: false,
internal: false,
})?;
})
.await?;
let queue_name = String::from("stats-events");
chan.declare_queue(easyamqp::QueueConfig {
@ -43,16 +42,30 @@ fn main() -> Result<(), Box<dyn Error>> {
exclusive: false,
auto_delete: false,
no_wait: false,
})?;
})
.await?;
chan.bind_queue(easyamqp::BindQueueConfig {
queue: queue_name.clone(),
exchange: "stats".to_owned(),
routing_key: None,
no_wait: false,
})?;
})
.await?;
let handle = chan.consume(
tokio::spawn(async {
let addr = "0.0.0.0:9898";
info!("listening addr {:?}", addr);
let app = Router::new().route(
"/metrics",
get(|| async move { metrics.prometheus_output() }),
);
let listener = tokio::net::TcpListener::bind(addr).await.unwrap();
axum::serve(listener, app).await.unwrap()
});
info!("Fetching jobs from {}", &queue_name);
chan.consume(
collector,
easyamqp::ConsumeConfig {
queue: "stats-events".to_owned(),
@ -62,22 +75,8 @@ fn main() -> Result<(), Box<dyn Error>> {
no_wait: false,
exclusive: false,
},
)?;
thread::spawn(|| {
Runtime::new().unwrap().block_on(async {
let addr = "0.0.0.0:9898";
info!("listening addr {:?}", addr);
let app = Router::new().route("/metrics", get(|| async move {
metrics.prometheus_output()
}));
let listener= tokio::net::TcpListener::bind(addr).await.unwrap();
axum::serve(listener, app).await.unwrap()
});
});
info!("Fetching jobs from {}", &queue_name);
block_on(handle);
)
.await?;
drop(conn); // Close connection.
info!("Closed the session... EOF");

16
ofborg/src/checkout.rs
View file

@ -12,6 +12,7 @@ pub struct CachedCloner {
root: PathBuf,
}
#[must_use]
pub fn cached_cloner(path: &Path) -> CachedCloner {
CachedCloner {
root: path.to_path_buf(),
@ -31,6 +32,7 @@ pub struct CachedProjectCo {
}
impl CachedCloner {
#[must_use]
pub fn project(&self, name: &str, clone_url: String) -> CachedProject {
// <root>/repo/<hash>/clone
// <root>/repo/<hash>/clone.lock
@ -101,7 +103,7 @@ impl CachedProjectCo {
let result = Command::new("git")
.arg("fetch")
.arg("origin")
.arg(format!("+refs/pull/{}/head:pr", pr_id))
.arg(format!("+refs/pull/{pr_id}/head:pr"))
.current_dir(self.clone_to())
.stdout(Stdio::null())
.status()?;
@ -162,7 +164,7 @@ impl CachedProjectCo {
let result = Command::new("git")
.arg("log")
.arg("--format=format:%s")
.arg(format!("HEAD..{}", commit))
.arg(format!("HEAD..{commit}"))
.current_dir(self.clone_to())
.output()?;
@ -171,7 +173,7 @@ impl CachedProjectCo {
if result.status.success() {
Ok(String::from_utf8_lossy(&result.stdout)
.lines()
.map(|l| l.to_owned())
.map(std::borrow::ToOwned::to_owned)
.collect())
} else {
Err(Error::new(
@ -187,7 +189,7 @@ impl CachedProjectCo {
let result = Command::new("git")
.arg("diff")
.arg("--name-only")
.arg(format!("HEAD...{}", commit))
.arg(format!("HEAD...{commit}"))
.current_dir(self.clone_to())
.output()?;
@ -196,7 +198,7 @@ impl CachedProjectCo {
if result.status.success() {
Ok(String::from_utf8_lossy(&result.stdout)
.lines()
.map(|l| l.to_owned())
.map(std::borrow::ToOwned::to_owned)
.collect())
} else {
Err(Error::new(
@ -278,8 +280,8 @@ mod tests {
.expect("building the test PR failed");
let stderr =
String::from_utf8(output.stderr).unwrap_or_else(|err| format!("warning: {}", err));
println!("{}", stderr);
String::from_utf8(output.stderr).unwrap_or_else(|err| format!("warning: {err}"));
println!("{stderr}");
let hash = String::from_utf8(output.stdout).expect("Should just be a hash");
return hash.trim().to_owned();

11
ofborg/src/clone.rs
View file

@ -14,7 +14,7 @@ pub struct Lock {
impl Lock {
pub fn unlock(&mut self) {
self.lock = None
self.lock = None;
}
}
@ -33,20 +33,19 @@ pub trait GitClonable {
warn!("Failed to create lock file {:?}: {}", self.lock_path(), e);
Err(e)
}
Ok(lock) => match lock.lock_exclusive() {
Err(e) => {
Ok(lock) => {
if let Err(e) = lock.lock_exclusive() {
warn!(
"Failed to get exclusive lock on file {:?}: {}",
self.lock_path(),
e
);
Err(e)
}
Ok(_) => {
} else {
debug!("Got lock on {:?}", self.lock_path());
Ok(Lock { lock: Some(lock) })
}
},
}
}
}

1
ofborg/src/commentparser.rs
View file

@ -1,6 +1,7 @@
use nom::types::CompleteStr;
use tracing::warn;
#[must_use]
pub fn parse(text: &str) -> Option<Vec<Instruction>> {
let instructions: Vec<Instruction> = text
.lines()

5
ofborg/src/config.rs
View file

@ -119,10 +119,12 @@ pub struct CheckoutConfig {
}
impl Config {
#[must_use]
pub fn whoami(&self) -> String {
format!("{}-{}", self.runner.identity, self.nix.system.join(","))
}
#[must_use]
pub fn acl(&self) -> acl::Acl {
let repos = self
.runner
@ -172,7 +174,7 @@ impl RabbitMqConfig {
.expect("Failed to expand the password-file configuration string");
password = std::fs::read_to_string(expanded_password_file.as_ref())?;
} else {
password = "".to_owned();
password = String::new();
}
let uri = format!(
"{}://{}:{}@{}/{}",
@ -186,6 +188,7 @@ impl RabbitMqConfig {
}
}
#[must_use]
pub fn load(filename: &Path) -> Config {
let mut file = File::open(filename).unwrap();
let mut contents = String::new();

33
ofborg/src/easyamqp.rs
View file

@ -1,5 +1,10 @@
use std::fmt::Debug;
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
#[derive(Debug)]
#[allow(clippy::struct_excessive_bools)]
pub struct ConsumeConfig {
/// Specifies the name of the queue to consume from.
pub queue: String,
@ -40,6 +45,7 @@ pub struct ConsumeConfig {
pub no_wait: bool,
}
#[derive(Debug)]
pub struct BindQueueConfig {
/// Specifies the name of the queue to bind.
///
@ -83,6 +89,7 @@ pub struct BindQueueConfig {
pub no_wait: bool,
}
#[derive(Debug)]
pub enum ExchangeType {
Topic,
Headers,
@ -103,6 +110,8 @@ impl From<ExchangeType> for String {
}
}
#[derive(Debug)]
#[allow(clippy::struct_excessive_bools)]
pub struct ExchangeConfig {
/// Exchange names starting with "amq." are reserved for
/// pre-declared and standardised exchanges. The client MAY
@ -183,6 +192,8 @@ pub struct ExchangeConfig {
pub no_wait: bool,
}
#[derive(Debug)]
#[allow(clippy::struct_excessive_bools)]
pub struct QueueConfig {
/// The queue name MAY be empty, in which case the server MUST
/// create a new queue with a unique generated name and return
@ -260,21 +271,27 @@ pub struct QueueConfig {
pub no_wait: bool,
}
#[async_trait]
pub trait ChannelExt {
type Error;
fn declare_exchange(&mut self, config: ExchangeConfig) -> Result<(), Self::Error>;
fn declare_queue(&mut self, config: QueueConfig) -> Result<(), Self::Error>;
fn bind_queue(&mut self, config: BindQueueConfig) -> Result<(), Self::Error>;
fn send_request<T: ?Sized + Serialize + std::marker::Sync, U: for<'a> Deserialize<'a>>(
async fn declare_exchange(&mut self, config: ExchangeConfig) -> Result<(), Self::Error>;
async fn declare_queue(&mut self, config: QueueConfig) -> Result<(), Self::Error>;
async fn bind_queue(&mut self, config: BindQueueConfig) -> Result<(), Self::Error>;
async fn send_request<
T: ?Sized + Serialize + Sync + Send + Debug,
U: for<'a> Deserialize<'a> + Debug,
>(
&mut self,
exchange: Option<&str>,
routing_key: Option<&str>,
msg: &T,
) -> impl std::future::Future<Output = Result<U, Self::Error>> + Send;
) -> Result<U, Self::Error>;
}
pub trait ConsumerExt<'a, C> {
#[async_trait]
pub trait ConsumerExt<C> {
type Error;
type Handle;
fn consume(self, callback: C, config: ConsumeConfig) -> Result<Self::Handle, Self::Error>;
async fn consume(self, callback: C, config: ConsumeConfig) -> Result<(), Self::Error>
where
C: 'async_trait;
}

203
ofborg/src/easylapin.rs
View file

@ -1,6 +1,3 @@
use std::future::Future;
use std::pin::Pin;
use crate::config::RabbitMqConfig;
use crate::easyamqp::{
BindQueueConfig, ChannelExt, ConsumeConfig, ConsumerExt, ExchangeConfig, ExchangeType,
@ -10,7 +7,9 @@ use crate::notifyworker::{NotificationReceiver, SimpleNotifyWorker};
use crate::ofborg;
use crate::worker::{prepare_queue_message, Action, SimpleWorker};
use futures::executor::block_on;
use std::fmt::Debug;
use async_trait::async_trait;
use futures::StreamExt;
use lapin::message::Delivery;
use lapin::options::{
@ -22,7 +21,7 @@ use lapin::{BasicProperties, Channel, Connection, ConnectionProperties, Exchange
use serde::{Deserialize, Serialize};
use tracing::{debug, trace};
pub fn from_config(cfg: &RabbitMqConfig) -> Result<Connection, lapin::Error> {
pub async fn from_config(cfg: &RabbitMqConfig) -> Result<Connection, lapin::Error> {
let mut props = FieldTable::default();
props.insert(
"ofborg_version".into(),
@ -32,13 +31,15 @@ pub fn from_config(cfg: &RabbitMqConfig) -> Result<Connection, lapin::Error> {
client_properties: props,
..Default::default()
};
block_on(Connection::connect(&cfg.as_uri()?, opts))
Connection::connect(&cfg.as_uri()?, opts).await
}
#[async_trait]
impl ChannelExt for Channel {
type Error = lapin::Error;
fn declare_exchange(&mut self, config: ExchangeConfig) -> Result<(), Self::Error> {
#[tracing::instrument(ret)]
async fn declare_exchange(&mut self, config: ExchangeConfig) -> Result<(), Self::Error> {
let opts = ExchangeDeclareOptions {
passive: config.passive,
durable: config.durable,
@ -52,11 +53,13 @@ impl ChannelExt for Channel {
ExchangeType::Fanout => ExchangeKind::Fanout,
_ => panic!("exchange kind"),
};
block_on(self.exchange_declare(&config.exchange, kind, opts, FieldTable::default()))?;
self.exchange_declare(&config.exchange, kind, opts, FieldTable::default())
.await?;
Ok(())
}
fn declare_queue(&mut self, config: QueueConfig) -> Result<(), Self::Error> {
#[tracing::instrument(ret)]
async fn declare_queue(&mut self, config: QueueConfig) -> Result<(), Self::Error> {
let opts = QueueDeclareOptions {
passive: config.passive,
durable: config.durable,
@ -65,26 +68,33 @@ impl ChannelExt for Channel {
nowait: config.no_wait,
};
block_on(self.queue_declare(&config.queue, opts, FieldTable::default()))?;
self.queue_declare(&config.queue, opts, FieldTable::default())
.await?;
Ok(())
}
fn bind_queue(&mut self, config: BindQueueConfig) -> Result<(), Self::Error> {
#[tracing::instrument(ret)]
async fn bind_queue(&mut self, config: BindQueueConfig) -> Result<(), Self::Error> {
let opts = QueueBindOptions {
nowait: config.no_wait,
};
block_on(self.queue_bind(
self.queue_bind(
&config.queue,
&config.exchange,
&config.routing_key.unwrap_or_else(|| "".into()),
&config.routing_key.unwrap_or_default(),
opts,
FieldTable::default(),
))?;
)
.await?;
Ok(())
}
async fn send_request<T: ?Sized + Serialize, U: for<'a> Deserialize<'a>>(
#[tracing::instrument(ret)]
async fn send_request<
T: ?Sized + Serialize + Sync + Send + Debug,
U: for<'a> Deserialize<'a> + Debug,
>(
&mut self,
exchange: Option<&str>,
routing_key: Option<&str>,
@ -105,8 +115,8 @@ impl ChannelExt for Channel {
trace!(?exchange, ?routing_key, "sending a RPC request");
let confirmation = self
.basic_publish(
&msg.exchange.take().unwrap_or_else(|| "".to_owned()),
&msg.routing_key.take().unwrap_or_else(|| "".to_owned()),
&msg.exchange.take().unwrap_or_default(),
&msg.routing_key.take().unwrap_or_default(),
BasicPublishOptions::default(),
&msg.content,
props,
@ -139,55 +149,64 @@ impl ChannelExt for Channel {
}
}
impl<'a, W: SimpleWorker + 'a> ConsumerExt<'a, W> for Channel {
#[async_trait]
impl<W: SimpleWorker> ConsumerExt<W> for Channel {
type Error = lapin::Error;
type Handle = Pin<Box<dyn Future<Output = ()> + 'a>>;
fn consume(
mut self,
mut worker: W,
config: ConsumeConfig,
) -> Result<Self::Handle, Self::Error> {
let mut consumer = block_on(self.basic_consume(
&config.queue,
&config.consumer_tag,
BasicConsumeOptions::default(),
FieldTable::default(),
))?;
Ok(Box::pin(async move {
while let Some(Ok(deliver)) = consumer.next().await {
debug!(?deliver.delivery_tag, "consumed delivery");
let content_type = deliver.properties.content_type();
let job = worker
.msg_to_job(
deliver.routing_key.as_str(),
&content_type.as_ref().map(|s| s.to_string()),
&deliver.data,
)
.expect("worker unexpected message consumed");
#[tracing::instrument(skip(worker), ret)]
async fn consume(mut self, mut worker: W, config: ConsumeConfig) -> Result<(), Self::Error>
where
W: 'async_trait,
{
let mut consumer = self
.basic_consume(
&config.queue,
&config.consumer_tag,
BasicConsumeOptions::default(),
FieldTable::default(),
)
.await?;
for action in worker.consumer(&mut self, &job) {
action_deliver(&self, &deliver, action)
.await
.expect("action deliver failure");
}
debug!(?deliver.delivery_tag, "done");
while let Some(Ok(deliver)) = consumer.next().await {
debug!(?deliver.delivery_tag, "consumed delivery");
let content_type = deliver.properties.content_type();
let job = worker
.msg_to_job(
deliver.routing_key.as_str(),
&content_type.as_ref().map(std::string::ToString::to_string),
&deliver.data,
)
.await
.expect("worker unexpected message consumed");
for action in worker.consumer(&mut self, &job).await {
action_deliver(&self, &deliver, action)
.await
.expect("action deliver failure");
}
}))
debug!(?deliver.delivery_tag, "done");
}
Ok(())
}
}
/// Same as a regular channel, but without prefetching,
/// used for services with multiple instances.
#[derive(Debug)]
pub struct WorkerChannel(pub Channel);
impl<'a, W: SimpleWorker + 'a> ConsumerExt<'a, W> for WorkerChannel {
#[async_trait]
impl<W: SimpleWorker> ConsumerExt<W> for WorkerChannel {
type Error = lapin::Error;
type Handle = Pin<Box<dyn Future<Output = ()> + 'a>>;
fn consume(self, worker: W, config: ConsumeConfig) -> Result<Self::Handle, Self::Error> {
block_on(self.0.basic_qos(1, BasicQosOptions::default()))?;
self.0.consume(worker, config)
#[tracing::instrument(skip(worker), ret)]
async fn consume(self, worker: W, config: ConsumeConfig) -> Result<(), Self::Error>
where
W: 'async_trait,
{
self.0.basic_qos(1, BasicQosOptions::default()).await?;
Ok(self.0.consume(worker, config).await?)
}
}
@ -202,52 +221,64 @@ impl<'a> ChannelNotificationReceiver<'a> {
}
}
#[async_trait]
impl<'a> NotificationReceiver for ChannelNotificationReceiver<'a> {
fn tell(&mut self, action: Action) {
block_on(action_deliver(self.channel, self.deliver, action))
async fn tell(&mut self, action: Action) {
action_deliver(self.channel, self.deliver, action)
.await
.expect("action deliver failure");
}
}
// FIXME the consumer trait for SimpleWorker and SimpleNotifyWorker conflict,
// but one could probably be implemented in terms of the other instead.
#[derive(Debug)]
pub struct NotifyChannel(pub Channel);
impl<'a, W: SimpleNotifyWorker + 'a + Send> ConsumerExt<'a, W> for NotifyChannel {
#[async_trait]
impl<W: SimpleNotifyWorker> ConsumerExt<W> for NotifyChannel {
type Error = lapin::Error;
type Handle = Pin<Box<dyn Future<Output = ()> + 'a + Send>>;
fn consume(self, worker: W, config: ConsumeConfig) -> Result<Self::Handle, Self::Error> {
block_on(self.0.basic_qos(1, BasicQosOptions::default()))?;
#[tracing::instrument(skip(worker), ret)]
async fn consume(self, worker: W, config: ConsumeConfig) -> Result<(), Self::Error>
where
W: 'async_trait,
{
self.0.basic_qos(1, BasicQosOptions::default()).await?;
let mut consumer = block_on(self.0.basic_consume(
&config.queue,
&config.consumer_tag,
BasicConsumeOptions::default(),
FieldTable::default(),
))?;
let mut consumer = self
.0
.basic_consume(
&config.queue,
&config.consumer_tag,
BasicConsumeOptions::default(),
FieldTable::default(),
)
.await?;
let mut chan = self.0;
Ok(Box::pin(async move {
while let Some(Ok(deliver)) = consumer.next().await {
debug!(?deliver.delivery_tag, "consumed delivery");
let mut receiver = ChannelNotificationReceiver {
channel: &mut chan,
deliver: &deliver,
};
let content_type = deliver.properties.content_type();
let job = worker
.msg_to_job(
deliver.routing_key.as_str(),
&content_type.as_ref().map(|s| s.to_string()),
&deliver.data,
)
.expect("worker unexpected message consumed");
while let Some(Ok(deliver)) = consumer.next().await {
debug!(?deliver.delivery_tag, "consumed delivery");
let mut receiver = ChannelNotificationReceiver {
channel: &mut chan,
deliver: &deliver,
};
worker.consumer(&job, &mut receiver);
debug!(?deliver.delivery_tag, "done");
}
}))
let content_type = deliver.properties.content_type();
let job = worker
.msg_to_job(
deliver.routing_key.as_str(),
&content_type.as_ref().map(std::string::ToString::to_string),
&deliver.data,
)
.await
.expect("worker unexpected message consumed");
worker.consumer(&job, &mut receiver).await;
debug!(?deliver.delivery_tag, "done");
}
Ok(())
}
}
@ -276,8 +307,8 @@ async fn action_deliver(
.await
}
Action::Publish(mut msg) => {
let exch = msg.exchange.take().unwrap_or_else(|| "".to_owned());
let key = msg.routing_key.take().unwrap_or_else(|| "".to_owned());
let exch = msg.exchange.take().unwrap_or_else(String::new);
let key = msg.routing_key.take().unwrap_or_else(String::new);
trace!(?exch, ?key, "action publish");
let mut props = BasicProperties::default().with_delivery_mode(2); // persistent.

9
ofborg/src/evalchecker.rs
View file

@ -11,6 +11,7 @@ pub struct EvalChecker {
}
impl EvalChecker {
#[must_use]
pub fn new(name: &str, op: nix::Operation, args: Vec<String>, nix: nix::Nix) -> EvalChecker {
EvalChecker {
name: name.to_owned(),
@ -20,14 +21,18 @@ impl EvalChecker {
}
}
#[must_use]
pub fn name(&self) -> &str {
&self.name
}
pub fn execute(&self, path: &Path) -> Result<File, File> {
self.nix.safely(&self.op, path, self.args.clone(), false)
pub async fn execute(&self, path: &Path) -> Result<File, File> {
self.nix
.safely(&self.op, path, self.args.clone(), false)
.await
}
#[must_use]
pub fn cli_cmd(&self) -> String {
let mut cli = vec![self.op.to_string()];
cli.append(&mut self.args.clone());

27
ofborg/src/lib.rs
View file

@ -1,9 +1,9 @@
#![recursion_limit = "512"]
// Replacing .map(|arch| arch.to_string())
// with .map(systems::System::to_string)
//
// seems much less clear and I just don't like it :)
#![allow(clippy::redundant_closure)]
#![deny(clippy::pedantic)]
#![allow(clippy::missing_errors_doc)]
#![allow(clippy::missing_panics_doc)]
#![allow(clippy::module_name_repetitions)]
#![allow(clippy::similar_names)]
#[macro_use]
extern crate serde_derive;
@ -14,15 +14,14 @@ extern crate nom;
use std::env;
use opentelemetry::global;
use opentelemetry::trace::TracerProvider;
use opentelemetry::KeyValue;
use opentelemetry_otlp::HasHttpConfig;
use opentelemetry_sdk::Resource;
use opentelemetry_semantic_conventions::resource::SERVICE_NAME;
use opentelemetry_semantic_conventions::resource::SERVICE_VERSION;
use opentelemetry_semantic_conventions::SCHEMA_URL;
use tracing_subscriber::prelude::*;
use tracing_subscriber::EnvFilter;
use opentelemetry::trace::TracerProvider;
pub mod acl;
pub mod asynccmd;
@ -80,10 +79,11 @@ pub mod ofborg {
pub const VERSION: &str = env!("CARGO_PKG_VERSION");
#[must_use]
pub fn partition_result<A, B>(results: Vec<Result<A, B>>) -> (Vec<A>, Vec<B>) {
let mut ok = Vec::new();
let mut err = Vec::new();
for result in results.into_iter() {
for result in results {
match result {
Ok(x) => {
ok.push(x);
@ -121,12 +121,15 @@ pub fn setup_log() {
tracing_subscriber::fmt::layer().boxed()
};
let exporter = opentelemetry_otlp::SpanExporter::builder().with_http().build().unwrap();
let provider = opentelemetry_sdk::trace::TracerProvider::builder()
.with_simple_exporter(exporter)
.with_config(
opentelemetry_sdk::trace::Config::default().with_resource(resource())
.with_batch_exporter(
opentelemetry_otlp::SpanExporter::builder()
.with_http()
.build()
.unwrap(),
opentelemetry_sdk::runtime::Tokio,
)
.with_config(opentelemetry_sdk::trace::Config::default().with_resource(resource()))
.build();
global::set_tracer_provider(provider.clone());

2
ofborg/src/locks.rs
View file

@ -20,6 +20,6 @@ pub struct Lock {
impl Lock {
pub fn unlock(&mut self) {
self.lock = None
self.lock = None;
}
}

32
ofborg/src/maintainers.rs
View file

@ -48,7 +48,7 @@ impl From<std::string::FromUtf8Error> for CalculationError {
}
impl ImpactedMaintainers {
pub fn calculate(
pub async fn calculate(
nix: &Nix,
checkout: &Path,
paths: &[String],
@ -56,11 +56,11 @@ impl ImpactedMaintainers {
) -> Result<ImpactedMaintainers, CalculationError> {
let mut path_file = NamedTempFile::new()?;
let pathstr = serde_json::to_string(&paths)?;
write!(path_file, "{}", pathstr)?;
write!(path_file, "{pathstr}")?;
let mut attr_file = NamedTempFile::new()?;
let attrstr = serde_json::to_string(&attributes)?;
write!(attr_file, "{}", attrstr)?;
write!(attr_file, "{attrstr}")?;
let mut argstrs: HashMap<&str, &str> = HashMap::new();
argstrs.insert("changedattrsjson", attr_file.path().to_str().unwrap());
@ -73,21 +73,25 @@ impl ImpactedMaintainers {
&[path_file.path(), attr_file.path()],
);
let ret = cmd.output()?;
let ret = cmd.output().await?;
Ok(serde_json::from_str(&String::from_utf8(ret.stdout)?)?)
}
#[must_use]
pub fn maintainers(&self) -> Vec<String> {
self.0.keys().map(|maintainer| maintainer.0.clone())
self.0
.keys()
.map(|maintainer| maintainer.0.clone())
.collect()
}
#[must_use]
pub fn maintainers_by_package(&self) -> MaintainersByPackage {
let mut bypkg = MaintainersByPackage(HashMap::new());
for (maintainer, packages) in self.0.iter() {
for package in packages.iter() {
for (maintainer, packages) in &self.0 {
for package in packages {
bypkg
.0
.entry(package.clone())
@ -118,7 +122,7 @@ impl std::fmt::Display for ImpactedMaintainers {
})
.collect::<Vec<String>>()
.join("\n");
write!(f, "{}", d)
write!(f, "{d}")
}
}
@ -154,15 +158,15 @@ mod tests {
.expect("building the test PR failed");
let stderr =
String::from_utf8(output.stderr).unwrap_or_else(|err| format!("warning: {}", err));
println!("{}", stderr);
String::from_utf8(output.stderr).unwrap_or_else(|err| format!("warning: {err}"));
println!("{stderr}");
let hash = String::from_utf8(output.stdout).expect("Should just be a hash");
return hash.trim().to_owned();
}
#[test]
fn example() {
#[tokio::test]
async fn example() {
let workingdir = TestScratch::new_dir("test-maintainers-example");
let bare = TestScratch::new_dir("test-maintainers-example-bare");
@ -186,11 +190,11 @@ mod tests {
working_co.checkout_ref(OsStr::new(&hash)).unwrap();
let remote = env::var("NIX_REMOTE").unwrap_or("".to_owned());
let remote = env::var("NIX_REMOTE").unwrap_or_default();
let nix = Nix::new(SYSTEM.to_owned(), remote, 1800, None);
let parsed =
ImpactedMaintainers::calculate(&nix, &working_co.clone_to(), &paths, &attributes);
ImpactedMaintainers::calculate(&nix, &working_co.clone_to(), &paths, &attributes).await;
let mut expect = ImpactedMaintainers(HashMap::new());
expect.0.insert(

1
ofborg/src/message/buildjob.rs
View file

@ -23,6 +23,7 @@ type Exchange = String;
type RoutingKey = String;
impl BuildJob {
#[must_use]
pub fn new(
repo: Repo,
change: Change,

2
ofborg/src/message/buildlogmsg.rs
View file

@ -3,7 +3,7 @@ pub struct BuildLogMsg {
pub system: String,
pub identity: String,
pub attempt_id: String,
pub line_number: u64,
pub line_number: usize,
pub output: String,
}

34
ofborg/src/message/buildresult.rs
View file

@ -23,7 +23,7 @@ impl From<BuildStatus> for String {
BuildStatus::Failure => "Failure".into(),
BuildStatus::HashMismatch => "A fixed output derivation's hash was incorrect".into(),
BuildStatus::TimedOut => "Timed out, unknown build status".into(),
BuildStatus::UnexpectedError { ref err } => format!("Unexpected error: {}", err),
BuildStatus::UnexpectedError { ref err } => format!("Unexpected error: {err}"),
}
}
}
@ -96,6 +96,7 @@ pub enum BuildResult {
}
impl BuildResult {
#[must_use]
pub fn legacy(&self) -> LegacyBuildResult {
// TODO: replace this with simpler structs for specific usecases, since
// it's decouples the structs from serialization. These can be changed
@ -111,18 +112,8 @@ impl BuildResult {
ref attempted_attrs,
ref skipped_attrs,
..
} => LegacyBuildResult {
repo: repo.to_owned(),
pr: pr.to_owned(),
system: system.to_owned(),
output: output.to_owned(),
attempt_id: attempt_id.to_owned(),
request_id: request_id.to_owned(),
status: self.status(),
attempted_attrs: attempted_attrs.to_owned(),
skipped_attrs: skipped_attrs.to_owned(),
},
BuildResult::V1 {
}
| BuildResult::V1 {
ref repo,
ref pr,
ref system,
@ -167,14 +158,15 @@ impl BuildResult {
}
}
#[must_use]
pub fn change(&self) -> Change {
match self {
BuildResult::Legacy { pr, .. } => pr.to_owned(),
BuildResult::V1 { pr, .. } => pr.to_owned(),
BuildResult::Legacy { pr, .. } | BuildResult::V1 { pr, .. } => pr.to_owned(),
BuildResult::V2 { change, .. } => change.to_owned(),
}
}
#[must_use]
pub fn status(&self) -> BuildStatus {
match *self {
BuildResult::Legacy {
@ -210,8 +202,7 @@ mod tests {
assert_eq!(
output,
r#"{"tag":"V1","repo":{"owner":"NixOS","name":"nixpkgs","full_name":"NixOS/nixpkgs","clone_url":"https://github.com/nixos/nixpkgs.git"},"pr":{"target_branch":"master","number":42,"head_sha":"0000000000000000000000000000000000000000"},"system":"x86_64-linux","output":["unpacking sources"],"attempt_id":"attempt-id-foo","request_id":"bogus-request-id","status":"Success","skipped_attrs":["AAAAAASomeThingsFailToEvaluate"],"attempted_attrs":["hello"]}"#,
"json of: {:?}",
result
"json of: {result:?}"
);
}
@ -224,8 +215,7 @@ mod tests {
assert_eq!(
output,
r#"{"repo":{"owner":"NixOS","name":"nixpkgs","full_name":"NixOS/nixpkgs","clone_url":"https://github.com/nixos/nixpkgs.git"},"pr":{"target_branch":"master","number":42,"head_sha":"0000000000000000000000000000000000000000"},"system":"x86_64-linux","output":["unpacking sources"],"attempt_id":"attempt-id-foo","request_id":"bogus-request-id","success":true,"status":"Success","skipped_attrs":["AAAAAASomeThingsFailToEvaluate"],"attempted_attrs":["hello"]}"#,
"json of: {:?}",
result
"json of: {result:?}"
);
}
@ -238,8 +228,7 @@ mod tests {
assert_eq!(
output,
r#"{"repo":{"owner":"NixOS","name":"nixpkgs","full_name":"NixOS/nixpkgs","clone_url":"https://github.com/nixos/nixpkgs.git"},"pr":{"target_branch":"master","number":42,"head_sha":"0000000000000000000000000000000000000000"},"system":"x86_64-linux","output":[],"attempt_id":"attempt-id-foo","request_id":"bogus-request-id","success":null,"status":null,"skipped_attrs":null,"attempted_attrs":null}"#,
"json of: {:?}",
result
"json of: {result:?}"
);
}
@ -252,8 +241,7 @@ mod tests {
assert_eq!(
output,
r#"{"repo":{"owner":"NixOS","name":"nixpkgs","full_name":"NixOS/nixpkgs","clone_url":"https://github.com/nixos/nixpkgs.git"},"pr":{"target_branch":"master","number":42,"head_sha":"0000000000000000000000000000000000000000"},"system":"x86_64-linux","output":["unpacking sources"],"attempt_id":"attempt-id-foo","request_id":"bogus-request-id","success":true,"status":null,"skipped_attrs":["AAAAAASomeThingsFailToEvaluate"],"attempted_attrs":["hello"]}"#,
"json of: {:?}",
result
"json of: {result:?}"
);
}
}

10
ofborg/src/message/evaluationjob.rs
View file

@ -12,6 +12,7 @@ pub struct EvaluationJob {
}
impl EvaluationJob {
#[must_use]
pub fn is_nixpkgs(&self) -> bool {
self.repo.name == "nixpkgs"
}
@ -20,15 +21,18 @@ impl EvaluationJob {
pub struct Actions {}
impl Actions {
pub fn retry_later(&mut self, _job: &EvaluationJob) -> worker::Actions {
#[must_use]
pub fn retry_later(_job: &EvaluationJob) -> worker::Actions {
vec![worker::Action::NackRequeue]
}
pub fn skip(&mut self, _job: &EvaluationJob) -> worker::Actions {
#[must_use]
pub fn skip(_job: &EvaluationJob) -> worker::Actions {
vec![worker::Action::Ack]
}
pub fn done(&mut self, _job: &EvaluationJob, mut response: worker::Actions) -> worker::Actions {
#[must_use]
pub fn done(_job: &EvaluationJob, mut response: worker::Actions) -> worker::Actions {
response.push(worker::Action::Ack);
response
}

422
ofborg/src/nix.rs
View file

@ -1,4 +1,3 @@
use crate::asynccmd::{AsyncCmd, SpawnedAsyncCmd};
use crate::message::buildresult::BuildStatus;
use crate::ofborg::partition_result;
@ -9,9 +8,10 @@ use std::fmt;
use std::fs;
use std::io::{BufRead, BufReader, Seek, SeekFrom};
use std::path::Path;
use std::process::{Command, Stdio};
use std::process::Stdio;
use tempfile::tempfile;
use tokio::process::{Child, Command};
#[allow(clippy::upper_case_acronyms)]
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
@ -43,11 +43,11 @@ pub enum Operation {
impl Operation {
fn command(&self) -> Command {
match *self {
Operation::Evaluate => Command::new("nix-instantiate"),
Operation::Instantiate => Command::new("nix-instantiate"),
Operation::Evaluate | Operation::Instantiate => Command::new("nix-instantiate"),
Operation::Build => Command::new("nix-build"),
Operation::QueryPackagesJson => Command::new("nix-env"),
Operation::QueryPackagesOutputs => Command::new("nix-env"),
Operation::QueryPackagesJson | Operation::QueryPackagesOutputs => {
Command::new("nix-env")
}
Operation::NoOp { .. } => Command::new("echo"),
Operation::Unknown { ref program } => Command::new(program),
}
@ -102,7 +102,7 @@ impl Operation {
Operation::Instantiate => {
command.args(["--option", "extra-experimental-features", "no-url-literals"]);
}
_ => (),
Operation::Unknown { .. } => (),
};
}
}
@ -115,7 +115,7 @@ impl fmt::Display for Operation {
Operation::QueryPackagesJson => write!(f, "nix-env -qa --json"),
Operation::QueryPackagesOutputs => write!(f, "nix-env -qaP --no-name --out-path"),
Operation::NoOp { ref operation } => operation.fmt(f),
Operation::Unknown { ref program } => write!(f, "{}", program),
Operation::Unknown { ref program } => write!(f, "{program}"),
Operation::Evaluate => write!(f, "nix-instantiate --strict --json ..."),
}
}
@ -131,6 +131,7 @@ pub struct Nix {
}
impl Nix {
#[must_use]
pub fn new(
system: String,
remote: String,
@ -146,54 +147,62 @@ impl Nix {
}
}
#[must_use]
pub fn with_system(&self, system: String) -> Nix {
let mut n = self.clone();
n.system = system;
n
}
#[must_use]
pub fn with_limited_supported_systems(&self) -> Nix {
let mut n = self.clone();
n.limit_supported_systems = true;
n
}
#[must_use]
pub fn without_limited_supported_systems(&self) -> Nix {
let mut n = self.clone();
n.limit_supported_systems = false;
n
}
pub fn safely_partition_instantiable_attrs(
pub async fn safely_partition_instantiable_attrs(
&self,
nixpkgs: &Path,
file: File,
attrs: Vec<String>,
) -> (Vec<String>, Vec<(String, Vec<String>)>) {
let attr_instantiations: Vec<Result<String, (String, Vec<String>)>> = attrs
.into_iter()
.map(
|attr| match self.safely_instantiate_attrs(nixpkgs, file, vec![attr.clone()]) {
let mut attr_instantiations: Vec<Result<String, (String, Vec<String>)>> = vec![];
for attr in attrs {
attr_instantiations.push(
match self
.safely_instantiate_attrs(nixpkgs, file, vec![attr.clone()])
.await
{
Ok(_) => Ok(attr),
Err(f) => Err((attr, lines_from_file(f))),
},
)
.collect();
);
}
partition_result(attr_instantiations)
}
pub fn safely_instantiate_attrs(
pub async fn safely_instantiate_attrs(
&self,
nixpkgs: &Path,
file: File,
attrs: Vec<String>,
) -> Result<fs::File, fs::File> {
let mut command = self.safe_command::<&OsStr>(&Operation::Instantiate, nixpkgs, &[], &[]);
self.set_attrs_command(&mut command, file, attrs);
self.run(command, true)
Self::set_attrs_command(&mut command, file, attrs);
self.run(command, true).await
}
#[must_use]
pub fn safely_evaluate_expr_cmd(
&self,
nixpkgs: &Path,
@ -213,31 +222,32 @@ impl Nix {
self.safe_command(&Operation::Evaluate, nixpkgs, &attrargs, extra_paths)
}
pub fn safely_build_attrs(
pub async fn safely_build_attrs(
&self,
nixpkgs: &Path,
file: File,
attrs: Vec<String>,
) -> Result<fs::File, fs::File> {
let mut command = self.safe_command::<&OsStr>(&Operation::Build, nixpkgs, &[], &[]);
self.set_attrs_command(&mut command, file, attrs);
self.run(command, true)
Self::set_attrs_command(&mut command, file, attrs);
self.run(command, true).await
}
#[must_use]
pub fn safely_build_attrs_async(
&self,
nixpkgs: &Path,
file: File,
attrs: Vec<String>,
) -> SpawnedAsyncCmd {
) -> Child {
let mut command = self.safe_command::<&OsStr>(&Operation::Build, nixpkgs, &[], &[]);
self.set_attrs_command(&mut command, file, attrs);
AsyncCmd::new(command).spawn()
Nix::set_attrs_command(&mut command, file, attrs);
command.spawn().expect("Failed to run Nix")
}
fn set_attrs_command(&self, command: &mut Command, file: File, attrs: Vec<String>) {
fn set_attrs_command(command: &mut Command, file: File, attrs: Vec<String>) {
let mut args: Vec<String> = Vec::with_capacity(3 + (attrs.len() * 2));
args.push(format!("{}", file));
args.push(format!("{file}"));
for attr in attrs {
args.push(String::from("-A"));
args.push(attr);
@ -252,7 +262,7 @@ impl Nix {
command.args(args);
}
pub fn safely(
pub async fn safely(
&self,
op: &Operation,
nixpkgs: &Path,
@ -260,9 +270,10 @@ impl Nix {
keep_stdout: bool,
) -> Result<fs::File, fs::File> {
self.run(self.safe_command(op, nixpkgs, &args, &[]), keep_stdout)
.await
}
pub fn run(&self, mut cmd: Command, keep_stdout: bool) -> Result<fs::File, fs::File> {
pub async fn run(&self, mut cmd: Command, keep_stdout: bool) -> Result<fs::File, fs::File> {
let stderr = tempfile().expect("Fetching a stderr tempfile");
let mut reader = stderr.try_clone().expect("Cloning stderr to the reader");
@ -276,6 +287,7 @@ impl Nix {
.stdout(stdout)
.stderr(Stdio::from(stderr))
.status()
.await
.expect("Running a program ...");
reader
@ -289,7 +301,7 @@ impl Nix {
}
}
pub fn run_stderr_stdout(&self, mut cmd: Command) -> (bool, fs::File, fs::File) {
pub async fn run_stderr_stdout(&self, mut cmd: Command) -> (bool, fs::File, fs::File) {
let stdout_file = tempfile().expect("Fetching a stdout tempfile");
let mut stdout_reader = stdout_file
.try_clone()
@ -304,6 +316,7 @@ impl Nix {
.stdout(Stdio::from(stdout_file))
.stderr(Stdio::from(stderr_file))
.status()
.await
.expect("Running a program ...");
stdout_reader
@ -379,28 +392,29 @@ fn lines_from_file(file: fs::File) -> Vec<String> {
.collect()
}
#[must_use]
pub fn is_user_setting_warning(line: &str) -> bool {
let line = line.trim();
line.starts_with("warning: ignoring the user-specified setting '")
&& line.ends_with("because it is a restricted setting and you are not a trusted user")
}
pub fn wait_for_build_status(spawned: SpawnedAsyncCmd) -> BuildStatus {
match spawned.wait() {
pub async fn wait_for_build_status(mut spawned: Child) -> BuildStatus {
match spawned.wait().await {
Ok(s) => match s.code() {
Some(0) => BuildStatus::Success,
Some(100) => BuildStatus::Failure, // nix permanent failure
Some(101) => BuildStatus::TimedOut, // nix build timedout
Some(102) => BuildStatus::HashMismatch, // Fixed Output Derivation's hash was wrong
Some(i) => BuildStatus::UnexpectedError {
err: format!("command failed with exit code {}", i),
err: format!("command failed with exit code {i}"),
},
None => BuildStatus::UnexpectedError {
err: "unexpected build failure".into(),
},
},
e => BuildStatus::UnexpectedError {
err: format!("failed on interior command {:?}", e),
err: format!("failed on interior command {e:?}"),
},
}
}
@ -420,7 +434,7 @@ mod tests {
let path = env::var("PATH").unwrap();
let test_path = format!("{}/test-nix/bin:{}", env!("CARGO_MANIFEST_DIR"), path);
env::set_var("PATH", test_path);
let remote = env::var("NIX_REMOTE").unwrap_or("".to_owned());
let remote = env::var("NIX_REMOTE").unwrap_or_default();
Nix::new(SYSTEM.to_owned(), remote, 1800, None)
}
@ -467,15 +481,14 @@ mod tests {
Fail,
}
fn assert_run(res: Result<fs::File, fs::File>, expected: Expect, require: Vec<&str>) {
fn assert_run(res: Result<fs::File, fs::File>, expected: &Expect, require: Vec<&str>) {
let expectation_held: bool = match expected {
Expect::Pass => res.is_ok(),
Expect::Fail => res.is_err(),
};
let file: fs::File = match res {
Ok(file) => file,
Err(file) => file,
Ok(file) | Err(file) => file,
};
let lines = lines_from_file(file);
@ -483,7 +496,7 @@ mod tests {
let buildlog = lines
.into_iter()
.map(|line| strip_ansi(&line))
.map(|line| format!(" | {}", line))
.map(|line| format!(" | {line}"))
.collect::<Vec<String>>()
.join("\n");
@ -491,7 +504,7 @@ mod tests {
let mut missed_requirements: usize = 0;
let requirements_held: Vec<Result<String, String>> = require
.into_iter()
.map(|line| line.to_owned())
.map(std::borrow::ToOwned::to_owned)
.map(|line| {
if buildlog.contains(&line) {
Ok(line)
@ -502,36 +515,34 @@ mod tests {
})
.collect();
let mut prefixes: Vec<String> = vec!["".to_owned(), "".to_owned()];
let mut prefixes: Vec<String> = vec![String::new(), String::new()];
if !expectation_held {
prefixes.push(format!(
"The run was expected to {:?}, but did not.",
expected
));
prefixes.push("".to_owned());
if expectation_held {
prefixes.push(format!("The run was expected to {expected:?}, and did."));
} else {
prefixes.push(format!("The run was expected to {:?}, and did.", expected));
prefixes.push("".to_owned());
prefixes.push(format!(
"The run was expected to {expected:?}, but did not."
));
}
prefixes.push(String::new());
let mut suffixes = vec![
"".to_owned(),
String::new(),
format!(
"{} out of {} required lines matched.",
(total_requirements - missed_requirements),
total_requirements
),
"".to_owned(),
String::new(),
];
for expected_line in requirements_held {
suffixes.push(format!(" - {:?}", expected_line));
suffixes.push(format!(" - {expected_line:?}"));
}
suffixes.push("".to_owned());
suffixes.push(String::new());
let output_blocks: Vec<Vec<String>> =
vec![prefixes, vec![buildlog, "".to_owned()], suffixes];
vec![prefixes, vec![buildlog, String::new()], suffixes];
let output_blocks_strings: Vec<String> = output_blocks
.into_iter()
@ -546,70 +557,78 @@ mod tests {
}
}
#[test]
fn test_build_operations() {
#[tokio::test]
async fn test_build_operations() {
let nix = nix();
let op = noop(Operation::Build);
assert_eq!(op.to_string(), "nix-build");
let ret: Result<fs::File, fs::File> = nix.run(
nix.safe_command(&op, build_path().as_path(), &["--version"], &[]),
true,
);
let ret: Result<fs::File, fs::File> = nix
.run(
nix.safe_command(&op, build_path().as_path(), &["--version"], &[]),
true,
)
.await;
assert_run(
ret,
Expect::Pass,
&Expect::Pass,
vec!["--no-out-link --keep-going", "--version"],
);
}
#[test]
fn test_instantiate_operation() {
#[tokio::test]
async fn test_instantiate_operation() {
let nix = nix();
let op = noop(Operation::Instantiate);
assert_eq!(op.to_string(), "nix-instantiate");
let ret: Result<fs::File, fs::File> = nix.run(
nix.safe_command(&op, build_path().as_path(), &["--version"], &[]),
true,
);
let ret: Result<fs::File, fs::File> = nix
.run(
nix.safe_command(&op, build_path().as_path(), &["--version"], &[]),
true,
)
.await;
assert_run(ret, Expect::Pass, vec!["--version"]);
assert_run(ret, &Expect::Pass, vec!["--version"]);
}
#[test]
fn test_query_packages_json() {
#[tokio::test]
async fn test_query_packages_json() {
let nix = nix();
let op = noop(Operation::QueryPackagesJson);
assert_eq!(op.to_string(), "nix-env -qa --json");
let ret: Result<fs::File, fs::File> = nix.run(
nix.safe_command(&op, build_path().as_path(), &["--version"], &[]),
true,
);
let ret: Result<fs::File, fs::File> = nix
.run(
nix.safe_command(&op, build_path().as_path(), &["--version"], &[]),
true,
)
.await;
assert_run(
ret,
Expect::Pass,
&Expect::Pass,
vec!["--query --available --json", "--version"],
);
}
#[test]
fn test_query_packages_outputs() {
#[tokio::test]
async fn test_query_packages_outputs() {
let nix = nix();
let op = noop(Operation::QueryPackagesOutputs);
assert_eq!(op.to_string(), "nix-env -qaP --no-name --out-path");
let ret: Result<fs::File, fs::File> = nix.run(
nix.safe_command(&op, build_path().as_path(), &["--version"], &[]),
true,
);
let ret: Result<fs::File, fs::File> = nix
.run(
nix.safe_command(&op, build_path().as_path(), &["--version"], &[]),
true,
)
.await;
assert_run(
ret,
Expect::Pass,
&Expect::Pass,
vec![
"--query --available --no-name --attr-path --out-path",
"--version",
@ -617,18 +636,20 @@ mod tests {
);
}
#[test]
fn safe_command_environment() {
#[tokio::test]
async fn safe_command_environment() {
let nix = nix();
let ret: Result<fs::File, fs::File> = nix.run(
nix.safe_command::<&OsStr>(&env_noop(), build_path().as_path(), &[], &[]),
true,
);
let ret: Result<fs::File, fs::File> = nix
.run(
nix.safe_command::<&OsStr>(&env_noop(), build_path().as_path(), &[], &[]),
true,
)
.await;
assert_run(
ret,
Expect::Pass,
&Expect::Pass,
vec![
"HOME=/homeless-shelter",
"NIX_PATH=ofborg-nixpkgs-pr=",
@ -638,19 +659,21 @@ mod tests {
);
}
#[test]
fn safe_command_custom_gc() {
let remote = env::var("NIX_REMOTE").unwrap_or("".to_owned());
#[tokio::test]
async fn safe_command_custom_gc() {
let remote = env::var("NIX_REMOTE").unwrap_or_default();
let nix = Nix::new(SYSTEM.to_owned(), remote, 1800, Some("4g".to_owned()));
let ret: Result<fs::File, fs::File> = nix.run(
nix.safe_command::<&OsStr>(&env_noop(), build_path().as_path(), &[], &[]),
true,
);
let ret: Result<fs::File, fs::File> = nix
.run(
nix.safe_command::<&OsStr>(&env_noop(), build_path().as_path(), &[], &[]),
true,
)
.await;
assert_run(
ret,
Expect::Pass,
&Expect::Pass,
vec![
"HOME=/homeless-shelter",
"NIX_PATH=ofborg-nixpkgs-pr=",
@ -661,57 +684,59 @@ mod tests {
);
}
#[test]
fn safe_command_options() {
#[tokio::test]
async fn safe_command_options() {
let nix = nix();
let op = noop(Operation::Build);
let ret: Result<fs::File, fs::File> = nix.run(
nix.safe_command::<&OsStr>(&op, build_path().as_path(), &[], &[]),
true,
);
let ret: Result<fs::File, fs::File> = nix
.run(
nix.safe_command::<&OsStr>(&op, build_path().as_path(), &[], &[]),
true,
)
.await;
assert_run(
ret,
Expect::Pass,
&Expect::Pass,
vec!["--option restrict-eval true", "--option build-timeout 1800"],
);
}
#[test]
fn set_attrs_nixpkgs() {
#[tokio::test]
async fn set_attrs_nixpkgs() {
let nix = nix();
let op = noop(Operation::Build);
let mut command = nix.safe_command::<&OsStr>(&op, build_path().as_path(), &[], &[]);
nix.set_attrs_command(
Nix::set_attrs_command(
&mut command,
File::DefaultNixpkgs,
vec!["foo".into(), "bar".into()],
);
let ret: Result<fs::File, fs::File> = nix.run(command, true);
let ret: Result<fs::File, fs::File> = nix.run(command, true).await;
assert_run(ret, Expect::Pass, vec!["./default.nix", "-A foo -A bar"]);
assert_run(ret, &Expect::Pass, vec!["./default.nix", "-A foo -A bar"]);
}
#[test]
fn set_attrs_nixos() {
#[tokio::test]
async fn set_attrs_nixos() {
let nix = nix();
let op = noop(Operation::Instantiate);
let mut command = nix.safe_command::<&OsStr>(&op, build_path().as_path(), &[], &[]);
nix.set_attrs_command(
Nix::set_attrs_command(
&mut command,
File::ReleaseNixOS,
vec!["foo".into(), "bar".into()],
);
let ret: Result<fs::File, fs::File> = nix.run(command, true);
let ret: Result<fs::File, fs::File> = nix.run(command, true).await;
assert_run(
ret,
Expect::Pass,
&Expect::Pass,
vec![
"./nixos/release.nix",
"--arg nixpkgs { outPath=./.; revCount=999999; shortRev=\"ofborg\"; rev=\"0000000000000000000000000000000000000000\"; }",
@ -719,36 +744,40 @@ mod tests {
);
}
#[test]
fn safely_build_attrs_success() {
#[tokio::test]
async fn safely_build_attrs_success() {
let nix = nix();
let ret: Result<fs::File, fs::File> = nix.safely_build_attrs(
build_path().as_path(),
File::DefaultNixpkgs,
vec![String::from("success")],
);
let ret: Result<fs::File, fs::File> = nix
.safely_build_attrs(
build_path().as_path(),
File::DefaultNixpkgs,
vec![String::from("success")],
)
.await;
assert_run(
ret,
Expect::Pass,
&Expect::Pass,
vec!["-success.drv", "building ", "hi", "-success"],
);
}
#[test]
fn safely_build_attrs_failure() {
#[tokio::test]
async fn safely_build_attrs_failure() {
let nix = nix();
let ret: Result<fs::File, fs::File> = nix.safely_build_attrs(
build_path().as_path(),
File::DefaultNixpkgs,
vec![String::from("failed")],
);
let ret: Result<fs::File, fs::File> = nix
.safely_build_attrs(
build_path().as_path(),
File::DefaultNixpkgs,
vec![String::from("failed")],
)
.await;
assert_run(
ret,
Expect::Fail,
&Expect::Fail,
vec![
"-failed.drv",
"building ",
@ -758,8 +787,8 @@ mod tests {
);
}
#[test]
fn partition_instantiable_attributes() {
#[tokio::test]
async fn partition_instantiable_attributes() {
let nix = nix();
let ret: (Vec<String>, Vec<(String, Vec<String>)>) = nix
@ -771,7 +800,8 @@ mod tests {
String::from("passes-instantiation"),
String::from("missing-attr"),
],
);
)
.await;
assert_eq!(ret.0, vec!["passes-instantiation"]);
@ -790,80 +820,90 @@ mod tests {
);
}
#[test]
fn safely_instantiate_attrs_failure() {
#[tokio::test]
async fn safely_instantiate_attrs_failure() {
let nix = nix();
let ret: Result<fs::File, fs::File> = nix.safely_instantiate_attrs(
individual_eval_path().as_path(),
File::DefaultNixpkgs,
vec![String::from("fails-instantiation")],
);
let ret: Result<fs::File, fs::File> = nix
.safely_instantiate_attrs(
individual_eval_path().as_path(),
File::DefaultNixpkgs,
vec![String::from("fails-instantiation")],
)
.await;
assert_run(
ret,
Expect::Fail,
&Expect::Fail,
vec!["You just can't", "assertion", "failed"],
);
}
#[test]
fn safely_instantiate_attrs_success() {
#[tokio::test]
async fn safely_instantiate_attrs_success() {
let nix = nix();
let ret: Result<fs::File, fs::File> = nix.safely_instantiate_attrs(
individual_eval_path().as_path(),
File::DefaultNixpkgs,
vec![String::from("passes-instantiation")],
);
assert_run(ret, Expect::Pass, vec!["-passes-instantiation.drv"]);
}
#[test]
fn safely_evaluate_expr_success() {
let nix = nix();
let ret: Result<fs::File, fs::File> = nix.run(
nix.safely_evaluate_expr_cmd(
let ret: Result<fs::File, fs::File> = nix
.safely_instantiate_attrs(
individual_eval_path().as_path(),
r#"{ foo ? "bar" }: "The magic value is ${foo}""#,
[("foo", "tux")].iter().cloned().collect(),
&[],
),
true,
);
File::DefaultNixpkgs,
vec![String::from("passes-instantiation")],
)
.await;
assert_run(ret, Expect::Pass, vec!["The magic value is tux"]);
assert_run(ret, &Expect::Pass, vec!["-passes-instantiation.drv"]);
}
#[test]
fn strict_sandboxing() {
let ret: Result<fs::File, fs::File> = nix().safely_build_attrs(
build_path().as_path(),
File::DefaultNixpkgs,
vec![String::from("sandbox-violation")],
);
#[tokio::test]
async fn safely_evaluate_expr_success() {
let nix = nix();
let ret: Result<fs::File, fs::File> = nix
.run(
nix.safely_evaluate_expr_cmd(
individual_eval_path().as_path(),
r#"{ foo ? "bar" }: "The magic value is ${foo}""#,
[("foo", "tux")].iter().copied().collect(),
&[],
),
true,
)
.await;
assert_run(ret, &Expect::Pass, vec!["The magic value is tux"]);
}
#[tokio::test]
async fn strict_sandboxing() {
let ret: Result<fs::File, fs::File> = nix()
.safely_build_attrs(
build_path().as_path(),
File::DefaultNixpkgs,
vec![String::from("sandbox-violation")],
)
.await;
assert_run(
ret,
Expect::Fail,
&Expect::Fail,
vec!["access to absolute path", "is forbidden in restricted mode"],
);
}
#[test]
fn instantiation_success() {
let ret: Result<fs::File, fs::File> = nix().safely(
&Operation::Instantiate,
passing_eval_path().as_path(),
vec![],
true,
);
#[tokio::test]
async fn instantiation_success() {
let ret: Result<fs::File, fs::File> = nix()
.safely(
&Operation::Instantiate,
passing_eval_path().as_path(),
vec![],
true,
)
.await;
assert_run(
ret,
Expect::Pass,
&Expect::Pass,
vec![
"the result might be removed by the garbage collector",
"-failed.drv",
@ -872,18 +912,20 @@ mod tests {
);
}
#[test]
fn instantiation_nixpkgs_restricted_mode() {
let ret: Result<fs::File, fs::File> = nix().safely(
&Operation::Instantiate,
individual_eval_path().as_path(),
vec![String::from("-A"), String::from("nixpkgs-restricted-mode")],
true,
);
#[tokio::test]
async fn instantiation_nixpkgs_restricted_mode() {
let ret: Result<fs::File, fs::File> = nix()
.safely(
&Operation::Instantiate,
individual_eval_path().as_path(),
vec![String::from("-A"), String::from("nixpkgs-restricted-mode")],
true,
)
.await;
assert_run(
ret,
Expect::Fail,
&Expect::Fail,
vec![
"access to absolute path '/fake'",
"is forbidden in restricted mode",

37
ofborg/src/nixenv.rs
View file

@ -17,6 +17,7 @@ pub struct HydraNixEnv {
}
impl HydraNixEnv {
#[must_use]
pub fn new(nix: nix::Nix, path: PathBuf, check_meta: bool) -> HydraNixEnv {
HydraNixEnv {
path,
@ -25,11 +26,11 @@ impl HydraNixEnv {
}
}
pub fn execute_with_stats(
pub async fn execute_with_stats(
&self,
) -> Result<(outpathdiff::PackageOutPaths, EvaluationStats), Error> {
self.place_nix()?;
let (status, stdout, stderr, stats) = self.run_nix_env();
let (status, stdout, stderr, stats) = self.run_nix_env().await;
self.remove_nix()?;
if status {
@ -79,7 +80,7 @@ impl HydraNixEnv {
// when it fails to evaluate something. In this case, we can ignore (but
// warn about) the error.
if let Err(e) = fs::remove_file(&outpath_stats) {
warn!("Failed to remove file {:?}: {:?}", outpath_stats, e)
warn!("Failed to remove file {:?}: {:?}", outpath_stats, e);
}
Ok(())
@ -93,7 +94,7 @@ impl HydraNixEnv {
self.path.join(".gc-of-borg-stats.json")
}
fn run_nix_env(&self) -> (bool, File, File, Result<File, io::Error>) {
async fn run_nix_env(&self) -> (bool, File, File, Result<File, io::Error>) {
let check_meta = if self.check_meta { "true" } else { "false" };
let mut cmd = self.nix.safe_command(
@ -111,7 +112,7 @@ impl HydraNixEnv {
cmd.env("NIX_SHOW_STATS", "1");
cmd.env("NIX_SHOW_STATS_PATH", self.outpath_stats_path());
let (status, stdout, stderr) = self.nix.run_stderr_stdout(cmd);
let (status, stdout, stderr) = self.nix.run_stderr_stdout(cmd).await;
let stats = File::open(self.outpath_stats_path());
(status, stdout, stderr, stats)
@ -135,13 +136,14 @@ impl From<io::Error> for Error {
}
impl Error {
#[must_use]
pub fn display(self) -> String {
match self {
Error::Io(e) => format!("Failed during the setup of executing nix-env: {:?}", e),
Error::CreateFile(path, err) => format!("Failed to create file {:?}: {:?}", path, err),
Error::RemoveFile(path, err) => format!("Failed to remove file {:?}: {:?}", path, err),
Error::Io(e) => format!("Failed during the setup of executing nix-env: {e:?}"),
Error::CreateFile(path, err) => format!("Failed to create file {path:?}: {err:?}"),
Error::RemoveFile(path, err) => format!("Failed to remove file {path:?}: {err:?}"),
Error::WriteFile(file, err) => {
format!("Failed to write to file '{:?}': {:?}", file, err)
format!("Failed to write to file '{file:?}': {err:?}")
}
Error::CommandFailed(mut fd) => {
let mut buffer = Vec::new();
@ -149,15 +151,14 @@ impl Error {
let bufstr = String::from_utf8_lossy(&buffer);
match read_result {
Ok(_) => format!("nix-env failed:\n{}", bufstr),
Ok(_) => format!("nix-env failed:\n{bufstr}"),
Err(e) => format!(
"nix-env failed and loading the error result caused a new error {:?}\n\n{}",
e, bufstr
"nix-env failed and loading the error result caused a new error {e:?}\n\n{bufstr}"
),
}
}
Error::UncleanEvaluation(warnings) => {
format!("nix-env did not evaluate cleanly:\n {:?}", warnings)
format!("nix-env did not evaluate cleanly:\n {warnings:?}")
}
Error::StatsParse(mut fd, seek, parse_err) => {
let mut buffer = Vec::new();
@ -169,21 +170,19 @@ impl Error {
if let Err(seek_err) = seek {
lines.push_str(&format!(
"Additionally, resetting to the beginning of the output failed with:\n{:?}\n\n",
seek_err
"Additionally, resetting to the beginning of the output failed with:\n{seek_err:?}\n\n"
));
}
if let Err(read_err) = read_result {
lines.push_str(&format!(
"Additionally, loading the output failed with:\n{:?}\n\n",
read_err
"Additionally, loading the output failed with:\n{read_err:?}\n\n"
));
}
lines.push_str(&format!("Parse error:\n{:?}\n\n", parse_err));
lines.push_str(&format!("Parse error:\n{parse_err:?}\n\n"));
lines.push_str(&format!("Evaluation output:\n{}", bufstr));
lines.push_str(&format!("Evaluation output:\n{bufstr}"));
lines
}

183
ofborg/src/nixstats.rs
View file

@ -1,4 +1,4 @@
//! Statistics emitted by Nix when NIX_SHOW_STATS=1
//! Statistics emitted by Nix when `NIX_SHOW_STATS=1`
use separator::Separatable;
use std::collections::HashMap;
@ -102,7 +102,69 @@ pub struct EvaluationStatsDiff<'a> {
right: &'a EvaluationStats,
}
struct Row {
before: String,
after: String,
diff: String,
diff_pct: String,
}
impl Row {
fn from_u64(left: u64, right: u64) -> Row {
let (diff, direction): (u64, _) = match left.cmp(&right) {
std::cmp::Ordering::Greater => (left - right, "↘ "),
std::cmp::Ordering::Less => (right - left, "↗ "),
std::cmp::Ordering::Equal => (0, ""),
};
#[allow(clippy::cast_precision_loss)]
let diff_pct: String = if diff > 0 {
format!(
"{:.2}%",
((right as f64) - (left as f64)) / (left as f64) * 100.0
)
} else {
String::new()
};
Row {
before: left.separated_string(),
after: right.separated_string(),
diff: format!("{}{}", direction, diff.separated_string()),
diff_pct,
}
}
fn from_f32(left: f32, right: f32) -> Row {
let (diff, direction): (f32, _) = match left
.partial_cmp(&right)
.unwrap_or(std::cmp::Ordering::Equal)
{
std::cmp::Ordering::Greater => (left - right, "↘ "),
std::cmp::Ordering::Less => (right - left, "↗ "),
std::cmp::Ordering::Equal => (0.0, ""),
};
let diff_pct: String = if diff > 0.0 {
format!(
"{:.2}%",
(f64::from(right) - f64::from(left)) / f64::from(left) * 100.0
)
} else {
String::new()
};
Row {
before: format!("{left:.2}"),
after: format!("{right:.2}"),
diff: format!("{direction}{diff:.2}"),
diff_pct,
}
}
}
impl<'a> EvaluationStatsDiff<'a> {
#[must_use]
pub fn compare(
left: &'a EvaluationStats,
right: &'a EvaluationStats,
@ -110,67 +172,9 @@ impl<'a> EvaluationStatsDiff<'a> {
EvaluationStatsDiff { left, right }
}
#[must_use]
#[allow(clippy::too_many_lines)]
pub fn markdown(&self) -> String {
struct Row {
before: String,
after: String,
diff: String,
diff_pct: String,
}
impl Row {
fn from_u64(left: u64, right: u64) -> Row {
let (diff, direction): (u64, _) = match left.cmp(&right) {
std::cmp::Ordering::Greater => (left - right, "↘ "),
std::cmp::Ordering::Less => (right - left, "↗ "),
std::cmp::Ordering::Equal => (0, ""),
};
let diff_pct: String = if diff > 0 {
format!(
"{:.2}%",
((right as f64) - (left as f64)) / (left as f64) * 100.0
)
} else {
String::from("")
};
Row {
before: left.separated_string(),
after: right.separated_string(),
diff: format!("{}{}", direction, diff.separated_string()),
diff_pct,
}
}
fn from_f32(left: f32, right: f32) -> Row {
let (diff, direction): (f32, _) = match left
.partial_cmp(&right)
.unwrap_or(std::cmp::Ordering::Equal)
{
std::cmp::Ordering::Greater => (left - right, "↘ "),
std::cmp::Ordering::Less => (right - left, "↗ "),
std::cmp::Ordering::Equal => (0 as f32, ""),
};
let diff_pct: String = if diff > 0 as _ {
format!(
"{:.2}%",
((right as f64) - (left as f64)) / (left as f64) * 100.0
)
} else {
String::from("")
};
Row {
before: format!("{:.2}", left),
after: format!("{:.2}", right),
diff: format!("{}{:.2}", direction, diff),
diff_pct,
}
}
}
let mut data: HashMap<&str, Row> = HashMap::new();
data.insert(
"cpuTime",
@ -310,7 +314,7 @@ impl<'a> EvaluationStatsDiff<'a> {
},
);
let mut keys = data.keys().cloned().collect::<Vec<&str>>();
let mut keys = data.keys().copied().collect::<Vec<&str>>();
keys.sort_unstable();
let rows = keys
@ -442,53 +446,54 @@ mod tests {
"#;
#[test]
#[allow(clippy::float_cmp)] // FIXME
fn verify_load() {
let load: EvaluationStats = serde_json::from_str(EXAMPLE).unwrap();
assert_eq!(load.cpu_time, 135.2);
assert_eq!(load.envs.number, 130714125);
assert_eq!(load.envs.elements, 183953876);
assert_eq!(load.envs.bytes, 3563057008);
assert_eq!(load.envs.number, 130_714_125);
assert_eq!(load.envs.elements, 183_953_876);
assert_eq!(load.envs.bytes, 3_563_057_008);
assert_eq!(load.list.elements, 207421516);
assert_eq!(load.list.bytes, 1659372128);
assert_eq!(load.list.concats, 7194150);
assert_eq!(load.list.elements, 207_421_516);
assert_eq!(load.list.bytes, 1_659_372_128);
assert_eq!(load.list.concats, 7_194_150);
assert_eq!(load.values.number, 260454370);
assert_eq!(load.values.bytes, 6250904880);
assert_eq!(load.values.number, 260_454_370);
assert_eq!(load.values.bytes, 6_250_904_880);
assert_eq!(load.symbols.number, 372918);
assert_eq!(load.symbols.bytes, 16324262);
assert_eq!(load.symbols.number, 372_918);
assert_eq!(load.symbols.bytes, 16_324_262);
assert_eq!(load.sets.number, 27310541);
assert_eq!(load.sets.bytes, 7134676648);
assert_eq!(load.sets.elements, 288174680);
assert_eq!(load.sets.number, 27_310_541);
assert_eq!(load.sets.bytes, 7_134_676_648);
assert_eq!(load.sets.elements, 288_174_680);
assert_eq!(load.sizes.env, 16);
assert_eq!(load.sizes.value, 24);
assert_eq!(load.sizes.bindings, 8);
assert_eq!(load.sizes.attr, 24);
assert_eq!(load.nr_op_updates, 11883339);
assert_eq!(load.nr_op_update_values_copied, 208834564);
assert_eq!(load.nr_thunks, 173325665);
assert_eq!(load.nr_avoided, 177840681);
assert_eq!(load.nr_lookups, 75292052);
assert_eq!(load.nr_prim_op_calls, 85571252);
assert_eq!(load.nr_function_calls, 115193164);
assert_eq!(load.nr_op_updates, 11_883_339);
assert_eq!(load.nr_op_update_values_copied, 208_834_564);
assert_eq!(load.nr_thunks, 173_325_665);
assert_eq!(load.nr_avoided, 177_840_681);
assert_eq!(load.nr_lookups, 75_292_052);
assert_eq!(load.nr_prim_op_calls, 85_571_252);
assert_eq!(load.nr_function_calls, 115_193_164);
assert_eq!(load.gc.heap_size, 12104687616);
assert_eq!(load.gc.total_bytes, 24191819392);
assert_eq!(load.gc.heap_size, 12_104_687_616);
assert_eq!(load.gc.total_bytes, 24_191_819_392);
}
fn diff_text(left: &str, right: &str) {
println!("left:\n{}", left);
println!("right:\n{}", right);
println!("left:\n{left}");
println!("right:\n{right}");
let lines = left.split('\n').zip(right.split('\n'));
for (idx, (linea, lineb)) in lines.enumerate() {
assert_eq!(linea, lineb, "Line {}", idx);
assert_eq!(linea, lineb, "Line {idx}");
}
}
@ -499,7 +504,7 @@ mod tests {
diff_text(
&EvaluationStatsDiff::compare(&left, &right).markdown(),
r#"
r"
| stat | before | after | Δ | Δ% |
|:---------------------------|---------------:|---------------:|:--------------|-------:|
| **cpuTime** | 135.20 | 132.90 | ↘ 2.30 | -1.70% |
@ -529,7 +534,7 @@ mod tests {
| **symbols-number** | 372,918 | 372,917 | ↘ 1 | -0.00% |
| **values-bytes** | 6,250,904,880 | 5,869,027,296 | ↘ 381,877,584 | -6.11% |
| **values-number** | 260,454,370 | 244,542,804 | ↘ 15,911,566 | -6.11% |
"#
"
.trim_start(),
);
}

21
ofborg/src/notifyworker.rs
View file

@ -1,11 +1,13 @@
use crate::worker::Action;
use async_trait::async_trait;
pub trait SimpleNotifyWorker {
type J;
#[async_trait]
pub trait SimpleNotifyWorker: Send + Sync {
type J: Send + Sync;
fn consumer(&self, job: &Self::J, notifier: &mut dyn NotificationReceiver);
async fn consumer(&self, job: &Self::J, notifier: &mut dyn NotificationReceiver);
fn msg_to_job(
async fn msg_to_job(
&self,
routing_key: &str,
content_type: &Option<String>,
@ -13,8 +15,9 @@ pub trait SimpleNotifyWorker {
) -> Result<Self::J, String>;
}
pub trait NotificationReceiver {
fn tell(&mut self, action: Action);
#[async_trait]
pub trait NotificationReceiver: Send + Sync {
async fn tell(&mut self, action: Action);
}
#[derive(Default)]
@ -23,13 +26,15 @@ pub struct DummyNotificationReceiver {
}
impl DummyNotificationReceiver {
#[must_use]
pub fn new() -> DummyNotificationReceiver {
Default::default()
DummyNotificationReceiver::default()
}
}
#[async_trait]
impl NotificationReceiver for DummyNotificationReceiver {
fn tell(&mut self, action: Action) {
async fn tell(&mut self, action: Action) {
self.actions.push(action);
}
}

23
ofborg/src/outpathdiff.rs
View file

@ -15,6 +15,7 @@ pub struct OutPathDiff {
}
impl OutPathDiff {
#[must_use]
pub fn new(nix: nix::Nix, path: PathBuf) -> OutPathDiff {
OutPathDiff {
calculator: HydraNixEnv::new(nix, path, false),
@ -23,21 +24,22 @@ impl OutPathDiff {
}
}
pub fn find_before(&mut self) -> Result<(), NixEnvError> {
self.original = Some(self.run()?);
pub async fn find_before(&mut self) -> Result<(), NixEnvError> {
self.original = Some(self.run().await?);
Ok(())
}
pub fn find_after(&mut self) -> Result<(), NixEnvError> {
pub async fn find_after(&mut self) -> Result<(), NixEnvError> {
if self.original.is_none() {
debug!("Before is None, not bothering with After");
return Ok(());
}
self.current = Some(self.run()?);
self.current = Some(self.run().await?);
Ok(())
}
#[must_use]
pub fn performance_diff(&self) -> Option<EvaluationStatsDiff> {
if let Some((_, ref cur)) = self.current {
if let Some((_, ref orig)) = self.original {
@ -50,6 +52,7 @@ impl OutPathDiff {
}
}
#[must_use]
pub fn package_diff(&self) -> Option<(Vec<PackageArch>, Vec<PackageArch>)> {
if let Some((ref cur, _)) = self.current {
if let Some((ref orig, _)) = self.original {
@ -80,11 +83,11 @@ impl OutPathDiff {
if let Some((ref orig, _)) = self.original {
for key in cur.keys() {
trace!("Checking out {:?}", key);
if cur.get(key) != orig.get(key) {
trace!(" {:?} != {:?}", cur.get(key), orig.get(key));
rebuild.push(key.clone())
} else {
if cur.get(key) == orig.get(key) {
trace!(" {:?} == {:?}", cur.get(key), orig.get(key));
} else {
trace!(" {:?} != {:?}", cur.get(key), orig.get(key));
rebuild.push(key.clone());
}
}
@ -95,8 +98,8 @@ impl OutPathDiff {
None
}
fn run(&mut self) -> Result<(PackageOutPaths, EvaluationStats), NixEnvError> {
self.calculator.execute_with_stats()
async fn run(&mut self) -> Result<(PackageOutPaths, EvaluationStats), NixEnvError> {
self.calculator.execute_with_stats().await
}
}

55
ofborg/src/stats.rs
View file

@ -1,16 +1,15 @@
use futures::executor::block_on;
use async_trait::async_trait;
use lapin::options::BasicPublishOptions;
include!(concat!(env!("OUT_DIR"), "/events.rs"));
#[macro_use]
mod macros {
#[macro_export]
macro_rules! my_macro(() => (FooBar));
#[allow(clippy::all, clippy::pedantic)]
mod events {
include!(concat!(env!("OUT_DIR"), "/events.rs"));
}
pub use events::*;
#[async_trait]
pub trait SysEvents: Send {
fn notify(&mut self, event: Event);
async fn notify(&mut self, event: Event);
}
#[derive(Serialize, Deserialize, Debug)]
@ -25,6 +24,7 @@ pub struct RabbitMq<C> {
}
impl RabbitMq<lapin::Channel> {
#[must_use]
pub fn from_lapin(identity: &str, channel: lapin::Channel) -> Self {
RabbitMq {
identity: identity.to_owned(),
@ -33,28 +33,27 @@ impl RabbitMq<lapin::Channel> {
}
}
#[async_trait]
impl SysEvents for RabbitMq<lapin::Channel> {
fn notify(&mut self, event: Event) {
async fn notify(&mut self, event: Event) {
let props = lapin::BasicProperties::default().with_content_type("application/json".into());
block_on(async {
let _confirmaton = self
.channel
.basic_publish(
&String::from("stats"),
"",
BasicPublishOptions::default(),
&serde_json::to_string(&EventMessage {
sender: self.identity.clone(),
events: vec![event],
})
.unwrap()
.into_bytes(),
props,
)
.await
let _confirmaton = self
.channel
.basic_publish(
&String::from("stats"),
"",
BasicPublishOptions::default(),
&serde_json::to_string(&EventMessage {
sender: self.identity.clone(),
events: vec![event],
})
.unwrap()
.await
.unwrap();
});
.into_bytes(),
props,
)
.await
.unwrap()
.await
.unwrap();
}
}

10
ofborg/src/systems.rs
View file

@ -18,16 +18,16 @@ impl std::fmt::Display for System {
}
impl System {
#[must_use]
pub fn as_build_destination(&self) -> (Option<String>, Option<String>) {
(None, Some(format!("build-inputs-{}", self)))
(None, Some(format!("build-inputs-{self}")))
}
#[must_use]
pub fn can_run_nixos_tests(&self) -> bool {
match self {
System::X8664Linux => true,
System::Aarch64Linux => true,
System::X8664Darwin => false,
System::Aarch64Darwin => false,
System::X8664Linux | System::Aarch64Linux => true,
System::X8664Darwin | System::Aarch64Darwin => false,
}
}
}

55
ofborg/src/tagger.rs
View file

@ -25,8 +25,9 @@ impl Default for StdenvTagger {
}
impl StdenvTagger {
#[must_use]
pub fn new() -> StdenvTagger {
Default::default()
StdenvTagger::default()
}
pub fn changed(&mut self, systems: Vec<tasks::eval::stdenvs::System>) {
@ -42,19 +43,20 @@ impl StdenvTagger {
}
for tag in &self.selected {
if !self.possible.contains(tag) {
panic!(
"Tried to add label {} but it isn't in the possible list!",
tag
);
}
assert!(
self.possible.contains(tag),
"Tried to add label {} but it isn't in the possible list!",
tag
);
}
}
#[must_use]
pub fn tags_to_add(&self) -> Vec<String> {
self.selected.clone()
}
#[must_use]
pub fn tags_to_remove(&self) -> Vec<String> {
let mut remove = self.possible.clone();
for tag in &self.selected {
@ -87,8 +89,9 @@ impl Default for PkgsAddedRemovedTagger {
}
impl PkgsAddedRemovedTagger {
#[must_use]
pub fn new() -> PkgsAddedRemovedTagger {
Default::default()
PkgsAddedRemovedTagger::default()
}
pub fn changed(&mut self, removed: &[PackageArch], added: &[PackageArch]) {
@ -101,10 +104,12 @@ impl PkgsAddedRemovedTagger {
}
}
#[must_use]
pub fn tags_to_add(&self) -> Vec<String> {
self.selected.clone()
}
#[must_use]
pub fn tags_to_remove(&self) -> Vec<String> {
// The cleanup tag is too vague to automatically remove.
vec![]
@ -147,8 +152,9 @@ impl Default for RebuildTagger {
}
impl RebuildTagger {
#[must_use]
pub fn new() -> RebuildTagger {
Default::default()
RebuildTagger::default()
}
pub fn parse_attrs(&mut self, attrs: Vec<PackageArch>) {
@ -160,12 +166,10 @@ impl RebuildTagger {
"x86_64-darwin" => {
counter_darwin += 1;
}
"aarch64-darwin" => {}
"x86_64-linux" => {
counter_linux += 1;
}
"aarch64-linux" => {}
"i686-linux" => {}
"aarch64-darwin" | "aarch64-linux" | "i686-linux" => {}
arch => {
info!("Unknown arch: {:?}", arch);
}
@ -176,35 +180,36 @@ impl RebuildTagger {
self.selected.extend(
RebuildTagger::bucket(counter_darwin)
.iter()
.map(|bucket| format!("10.rebuild-darwin: {}", bucket))
.map(|bucket| format!("10.rebuild-darwin: {bucket}"))
.collect::<Vec<String>>(),
);
self.selected.extend(
RebuildTagger::bucket(counter_linux)
.iter()
.map(|bucket| format!("10.rebuild-linux: {}", bucket))
.map(|bucket| format!("10.rebuild-linux: {bucket}"))
.collect::<Vec<String>>(),
);
for tag in &self.selected {
if !self.possible.contains(tag) {
panic!(
"Tried to add label {} but it isn't in the possible list!",
tag
);
}
assert!(
self.possible.contains(tag),
"Tried to add label {} but it isn't in the possible list!",
tag
);
}
}
#[must_use]
pub fn tags_to_add(&self) -> Vec<String> {
self.selected.clone()
}
#[must_use]
pub fn tags_to_remove(&self) -> Vec<String> {
let mut remove = vec![];
for tag in self.possible.clone().into_iter() {
for tag in self.possible.clone() {
if !self.selected.contains(&tag) {
remove.push(tag);
}
@ -254,8 +259,9 @@ impl Default for MaintainerPrTagger {
}
impl MaintainerPrTagger {
#[must_use]
pub fn new() -> MaintainerPrTagger {
Default::default()
MaintainerPrTagger::default()
}
pub fn record_maintainer(
@ -270,7 +276,7 @@ impl MaintainerPrTagger {
return;
}
for (_package, maintainers) in identified_maintainers.0.iter() {
for maintainers in identified_maintainers.0.values() {
if !maintainers.contains(&submitter) {
// One of the packages is not maintained by this submitter
return;
@ -281,10 +287,12 @@ impl MaintainerPrTagger {
.push(String::from("11.by: package-maintainer"));
}
#[must_use]
pub fn tags_to_add(&self) -> Vec<String> {
self.selected.clone()
}
#[must_use]
pub fn tags_to_remove(&self) -> Vec<String> {
// The cleanup tag is too vague to automatically remove.
vec![]
@ -333,6 +341,7 @@ mod tests {
}
#[test]
#[allow(clippy::too_many_lines)]
pub fn test_packages_changed() {
let mut tagger = RebuildTagger::new();
tagger.parse_attrs(PackageArchSrc::linux(0).and_darwin(0).into());

168
ofborg/src/tasks/build.rs
View file

@ -8,6 +8,9 @@ use crate::worker;
use std::collections::VecDeque;
use async_trait::async_trait;
use tokio::io::AsyncBufReadExt;
use tokio::io::BufReader;
use tracing::{debug, debug_span, error, info};
use uuid::Uuid;
@ -19,6 +22,7 @@ pub struct BuildWorker {
}
impl BuildWorker {
#[must_use]
pub fn new(
cloner: checkout::CachedCloner,
nix: nix::Nix,
@ -47,7 +51,7 @@ pub struct JobActions<'a, 'b> {
identity: String,
receiver: &'a mut dyn notifyworker::NotificationReceiver,
job: &'b buildjob::BuildJob,
line_counter: u64,
line_counter: usize,
snippet_log: VecDeque<String>,
attempt_id: String,
log_exchange: Option<String>,
@ -88,23 +92,24 @@ impl<'a, 'b> JobActions<'a, 'b> {
}
}
#[must_use]
pub fn log_snippet(&self) -> Vec<String> {
self.snippet_log.clone().into()
}
pub fn pr_head_missing(&mut self) {
self.tell(worker::Action::Ack);
pub async fn pr_head_missing(&mut self) {
self.tell(worker::Action::Ack).await;
}
pub fn commit_missing(&mut self) {
self.tell(worker::Action::Ack);
pub async fn commit_missing(&mut self) {
self.tell(worker::Action::Ack).await;
}
pub fn nothing_to_do(&mut self) {
self.tell(worker::Action::Ack);
pub async fn nothing_to_do(&mut self) {
self.tell(worker::Action::Ack).await;
}
pub fn merge_failed(&mut self) {
pub async fn merge_failed(&mut self) {
let msg = BuildResult::V1 {
tag: V1Tag::V1,
repo: self.job.repo.clone(),
@ -122,14 +127,15 @@ impl<'a, 'b> JobActions<'a, 'b> {
let result_routing_key = self.result_routing_key.clone();
self.tell(worker::publish_serde_action(
result_exchange,
result_routing_key,
&result_exchange,
&result_routing_key,
&msg,
));
self.tell(worker::Action::Ack);
))
.await;
self.tell(worker::Action::Ack).await;
}
pub fn log_started(&mut self, can_build: Vec<String>, cannot_build: Vec<String>) {
pub async fn log_started(&mut self, can_build: Vec<String>, cannot_build: Vec<String>) {
let msg = buildlogmsg::BuildLogStart {
identity: self.identity.clone(),
system: self.system.clone(),
@ -142,24 +148,26 @@ impl<'a, 'b> JobActions<'a, 'b> {
let log_routing_key = self.log_routing_key.clone();
self.tell(worker::publish_serde_action(
log_exchange,
log_routing_key,
&log_exchange,
&log_routing_key,
&msg,
));
))
.await;
}
pub fn log_instantiation_errors(&mut self, cannot_build: Vec<(String, Vec<String>)>) {
pub async fn log_instantiation_errors(&mut self, cannot_build: Vec<(String, Vec<String>)>) {
for (attr, log) in cannot_build {
self.log_line(&format!("Cannot nix-instantiate `{}' because:", &attr));
self.log_line(&format!("Cannot nix-instantiate `{}' because:", &attr))
.await;
for line in log {
self.log_line(&line);
self.log_line(&line).await;
}
self.log_line("");
self.log_line("").await;
}
}
pub fn log_line(&mut self, line: &str) {
pub async fn log_line(&mut self, line: &str) {
self.line_counter += 1;
if self.snippet_log.len() >= 10 {
@ -179,13 +187,14 @@ impl<'a, 'b> JobActions<'a, 'b> {
let log_routing_key = self.log_routing_key.clone();
self.tell(worker::publish_serde_action(
log_exchange,
log_routing_key,
&log_exchange,
&log_routing_key,
&msg,
));
))
.await;
}
pub fn build_not_attempted(&mut self, not_attempted_attrs: Vec<String>) {
pub async fn build_not_attempted(&mut self, not_attempted_attrs: Vec<String>) {
let msg = BuildResult::V1 {
tag: V1Tag::V1,
repo: self.job.repo.clone(),
@ -202,23 +211,25 @@ impl<'a, 'b> JobActions<'a, 'b> {
let result_exchange = self.result_exchange.clone();
let result_routing_key = self.result_routing_key.clone();
self.tell(worker::publish_serde_action(
result_exchange,
result_routing_key,
&result_exchange,
&result_routing_key,
&msg,
));
))
.await;
let log_exchange = self.log_exchange.clone();
let log_routing_key = self.log_routing_key.clone();
self.tell(worker::publish_serde_action(
log_exchange,
log_routing_key,
&log_exchange,
&log_routing_key,
&msg,
));
))
.await;
self.tell(worker::Action::Ack);
self.tell(worker::Action::Ack).await;
}
pub fn build_finished(
pub async fn build_finished(
&mut self,
status: BuildStatus,
attempted_attrs: Vec<String>,
@ -240,31 +251,39 @@ impl<'a, 'b> JobActions<'a, 'b> {
let result_exchange = self.result_exchange.clone();
let result_routing_key = self.result_routing_key.clone();
self.tell(worker::publish_serde_action(
result_exchange,
result_routing_key,
&result_exchange,
&result_routing_key,
&msg,
));
))
.await;
let log_exchange = self.log_exchange.clone();
let log_routing_key = self.log_routing_key.clone();
self.tell(worker::publish_serde_action(
log_exchange,
log_routing_key,
&log_exchange,
&log_routing_key,
&msg,
));
))
.await;
self.tell(worker::Action::Ack);
self.tell(worker::Action::Ack).await;
}
fn tell(&mut self, action: worker::Action) {
self.receiver.tell(action);
async fn tell(&mut self, action: worker::Action) {
self.receiver.tell(action).await;
}
}
#[async_trait]
impl notifyworker::SimpleNotifyWorker for BuildWorker {
type J = buildjob::BuildJob;
fn msg_to_job(&self, _: &str, _: &Option<String>, body: &[u8]) -> Result<Self::J, String> {
async fn msg_to_job(
&self,
_: &str,
_: &Option<String>,
body: &[u8],
) -> Result<Self::J, String> {
info!("lmao I got a job?");
match buildjob::from(body) {
Ok(e) => Ok(e),
@ -277,7 +296,7 @@ impl notifyworker::SimpleNotifyWorker for BuildWorker {
// FIXME: remove with rust/cargo update
#[allow(clippy::cognitive_complexity)]
fn consumer(
async fn consumer(
&self,
job: &buildjob::BuildJob,
notifier: &mut dyn notifyworker::NotificationReceiver,
@ -289,7 +308,7 @@ impl notifyworker::SimpleNotifyWorker for BuildWorker {
if job.attrs.is_empty() {
debug!("No attrs to build");
actions.nothing_to_do();
actions.nothing_to_do().await;
return;
}
@ -318,19 +337,19 @@ impl notifyworker::SimpleNotifyWorker for BuildWorker {
if co.fetch_pr(job.change.number).is_err() {
info!("Failed to fetch {}", job.change.number);
actions.pr_head_missing();
actions.pr_head_missing().await;
return;
}
if !co.commit_exists(job.change.head_sha.as_ref()) {
info!("Commit {} doesn't exist", job.change.head_sha);
actions.commit_missing();
actions.commit_missing().await;
return;
}
if co.merge_commit(job.change.head_sha.as_ref()).is_err() {
info!("Failed to merge {}", job.change.head_sha);
actions.merge_failed();
actions.merge_failed().await;
return;
}
@ -338,11 +357,10 @@ impl notifyworker::SimpleNotifyWorker for BuildWorker {
"Got path: {:?}, determining which ones we can build ",
refpath
);
let (can_build, cannot_build) = self.nix.safely_partition_instantiable_attrs(
refpath.as_ref(),
buildfile,
job.attrs.clone(),
);
let (can_build, cannot_build) = self
.nix
.safely_partition_instantiable_attrs(refpath.as_ref(), buildfile, job.attrs.clone())
.await;
let cannot_build_attrs: Vec<String> = cannot_build
.clone()
@ -356,11 +374,13 @@ impl notifyworker::SimpleNotifyWorker for BuildWorker {
cannot_build_attrs.join(", ")
);
actions.log_started(can_build.clone(), cannot_build_attrs.clone());
actions.log_instantiation_errors(cannot_build);
actions
.log_started(can_build.clone(), cannot_build_attrs.clone())
.await;
actions.log_instantiation_errors(cannot_build).await;
if can_build.is_empty() {
actions.build_not_attempted(cannot_build_attrs);
actions.build_not_attempted(cannot_build_attrs).await;
return;
}
@ -368,11 +388,12 @@ impl notifyworker::SimpleNotifyWorker for BuildWorker {
self.nix
.safely_build_attrs_async(refpath.as_ref(), buildfile, can_build.clone());
for line in spawned.lines() {
actions.log_line(&line);
let mut reader = BufReader::new(spawned.stdout.take().unwrap()).lines();
while let Ok(Some(line)) = reader.next_line().await {
actions.log_line(&line).await;
}
let status = nix::wait_for_build_status(spawned);
let status = nix::wait_for_build_status(spawned).await;
info!("ok built ({:?}), building", status);
info!("Lines:");
@ -384,7 +405,9 @@ impl notifyworker::SimpleNotifyWorker for BuildWorker {
.last();
info!("----->8-----");
actions.build_finished(status, can_build, cannot_build_attrs);
actions
.build_finished(status, can_build, cannot_build_attrs)
.await;
info!("Build done!");
}
}
@ -406,7 +429,7 @@ mod tests {
const SYSTEM: &str = "x86_64-darwin";
fn nix() -> nix::Nix {
let remote = env::var("NIX_REMOTE").unwrap_or("".to_owned());
let remote = env::var("NIX_REMOTE").unwrap_or_default();
nix::Nix::new("x86_64-linux".to_owned(), remote, 1800, None)
}
@ -438,8 +461,7 @@ mod tests {
assert!(cmd.status().unwrap().success());
let output = cmd.output()
.expect("building the test PR failed");
let output = cmd.output().expect("building the test PR failed");
let hash = String::from_utf8(output.stdout).expect("Should just be a hash");
hash.trim().to_owned()
@ -453,23 +475,23 @@ mod tests {
}
fn assert_contains_job(actions: &mut IntoIter<worker::Action>, text_to_match: &str) {
println!("\n\n Searching: {:?}", text_to_match);
println!("\n\n Searching: {text_to_match:?}");
actions
.position(|job| match job {
worker::Action::Publish(ref body) => {
let content = String::from_utf8(body.content.clone()).unwrap();
let text = strip_escaped_ansi(&content);
eprintln!("{}", text);
eprintln!("{text}");
if text.contains(text_to_match) {
println!(" ok");
true
} else {
println!(" notContains: {}", text);
println!(" notContains: {text}");
false
}
}
e => {
println!(" notPublish: {:?}", e);
println!(" notPublish: {e:?}");
false
}
})
@ -481,8 +503,8 @@ mod tests {
});
}
#[test]
pub fn test_simple_build() {
#[tokio::test]
pub async fn test_simple_build() {
let p = TestScratch::new_dir("build-simple-build-working");
let bare_repo = TestScratch::new_dir("build-simple-build-bare");
let co_repo = TestScratch::new_dir("build-simple-build-co");
@ -511,7 +533,7 @@ mod tests {
let mut dummyreceiver = notifyworker::DummyNotificationReceiver::new();
worker.consumer(&job, &mut dummyreceiver);
worker.consumer(&job, &mut dummyreceiver).await;
println!("Total actions: {:?}", dummyreceiver.actions.len());
let mut actions = dummyreceiver.actions.into_iter();
@ -526,8 +548,8 @@ mod tests {
assert_eq!(actions.next(), Some(worker::Action::Ack));
}
#[test]
pub fn test_all_jobs_skipped() {
#[tokio::test]
pub async fn test_all_jobs_skipped() {
let p = TestScratch::new_dir("no-attempt");
let bare_repo = TestScratch::new_dir("no-attempt-bare");
let co_repo = TestScratch::new_dir("no-attempt-co");
@ -556,7 +578,7 @@ mod tests {
let mut dummyreceiver = notifyworker::DummyNotificationReceiver::new();
worker.consumer(&job, &mut dummyreceiver);
worker.consumer(&job, &mut dummyreceiver).await;
println!("Total actions: {:?}", dummyreceiver.actions.len());
let mut actions = dummyreceiver.actions.into_iter();

46
ofborg/src/tasks/eval/generic.rs
View file

@ -1,46 +0,0 @@
use crate::checkout::CachedProjectCo;
use crate::evalchecker::EvalChecker;
use crate::tasks::eval::{EvaluationComplete, EvaluationStrategy, StepResult};
use crate::vcs::commit_status::CommitStatus;
use std::path::Path;
#[derive(Default)]
pub struct GenericStrategy {}
impl GenericStrategy {
pub fn new() -> GenericStrategy {
Self {}
}
}
impl EvaluationStrategy for GenericStrategy {
fn pre_clone(&mut self) -> StepResult<()> {
Ok(())
}
fn on_target_branch(&mut self, _co: &Path, _status: &mut CommitStatus) -> StepResult<()> {
Ok(())
}
fn after_fetch(&mut self, _co: &CachedProjectCo) -> StepResult<()> {
Ok(())
}
fn merge_conflict(&mut self) {}
fn after_merge(&mut self, _status: &mut CommitStatus) -> StepResult<()> {
Ok(())
}
fn evaluation_checks(&self) -> Vec<EvalChecker> {
vec![]
}
fn all_evaluations_passed(
&mut self,
_co: &Path,
_status: &mut CommitStatus,
) -> StepResult<EvaluationComplete> {
Ok(Default::default())
}
}

23
ofborg/src/tasks/eval/mod.rs
View file

@ -1,33 +1,12 @@
mod generic;
mod nixpkgs;
pub mod stdenvs;
pub use self::generic::GenericStrategy;
pub use self::nixpkgs::NixpkgsStrategy;
pub use self::stdenvs::Stdenvs;
use crate::checkout::CachedProjectCo;
use crate::evalchecker::EvalChecker;
use crate::message::buildjob::BuildJob;
use crate::vcs::commit_status::{CommitStatus, CommitStatusError};
use crate::vcs::commit_status::CommitStatusError;
use crate::vcs::generic::CheckRunOptions;
use std::path::Path;
pub trait EvaluationStrategy {
fn pre_clone(&mut self) -> StepResult<()>;
fn on_target_branch(&mut self, co: &Path, status: &mut CommitStatus) -> StepResult<()>;
fn after_fetch(&mut self, co: &CachedProjectCo) -> StepResult<()>;
fn merge_conflict(&mut self);
fn after_merge(&mut self, status: &mut CommitStatus) -> StepResult<()>;
fn evaluation_checks(&self) -> Vec<EvalChecker>;
fn all_evaluations_passed(
&mut self,
co: &Path,
status: &mut CommitStatus,
) -> StepResult<EvaluationComplete>;
}
pub type StepResult<T> = Result<T, Error>;
#[derive(Default)]

242
ofborg/src/tasks/eval/nixpkgs.rs
View file

@ -9,19 +9,16 @@ use crate::nix::{self, Nix};
use crate::nixenv::HydraNixEnv;
use crate::outpathdiff::{OutPathDiff, PackageArch};
use crate::tagger::{MaintainerPrTagger, PkgsAddedRemovedTagger, RebuildTagger, StdenvTagger};
use crate::tasks::eval::{
stdenvs::Stdenvs, Error, EvaluationComplete, EvaluationStrategy, StepResult,
};
use crate::tasks::eval::{stdenvs::Stdenvs, Error, EvaluationComplete, StepResult};
use crate::vcs::commit_status::CommitStatus;
use crate::vcs::generic::{
CheckRunOptions, CheckRunState, Conclusion, State, VersionControlSystemAPI,
};
use std::path::Path;
use std::rc::Rc;
use std::sync::Arc;
use chrono::Utc;
use futures::executor::block_on;
use regex::Regex;
use tracing::{info, warn};
use uuid::Uuid;
@ -39,7 +36,7 @@ fn label_from_title(title: &str) -> Vec<String> {
let labels: Vec<_> = TITLE_LABELS
.iter()
.filter(|(word, _label)| {
let re = Regex::new(&format!("\\b{}\\b", word)).unwrap();
let re = Regex::new(&format!("\\b{word}\\b")).unwrap();
re.is_match(title)
})
.map(|(_word, label)| (*label).into())
@ -51,7 +48,7 @@ fn label_from_title(title: &str) -> Vec<String> {
pub struct NixpkgsStrategy<'a> {
chan: lapin::Channel,
job: &'a EvaluationJob,
vcs_api: Rc<dyn VersionControlSystemAPI>,
vcs_api: Arc<dyn VersionControlSystemAPI>,
change: &'a Change,
repo: &'a Repo,
nix: Nix,
@ -66,7 +63,7 @@ impl<'a> NixpkgsStrategy<'a> {
pub fn new(
chan: lapin::Channel,
job: &'a EvaluationJob,
vcs_api: Rc<dyn VersionControlSystemAPI>,
vcs_api: Arc<dyn VersionControlSystemAPI>,
repo: &'a Repo,
change: &'a Change,
nix: Nix,
@ -85,47 +82,38 @@ impl<'a> NixpkgsStrategy<'a> {
}
}
fn tag_from_title(&self) {
let issue =
match block_on(self.vcs_api.get_issue(self.repo, self.change.number))
{
Ok(issue) => issue,
Err(_) => return,
};
async fn tag_from_title(&self) {
let Ok(issue) = self.vcs_api.get_issue(self.repo, self.change.number).await else {
return;
};
let labels = label_from_title(&issue.title);
if labels.is_empty() {
return;
}
block_on(self.vcs_api.update_labels(
self.repo,
self.change.number,
&labels,
&[],
));
self.vcs_api
.update_labels(self.repo, self.change.number, &labels, &[])
.await;
}
fn update_labels(&self, to_add: &[String], to_remove: &[String]) {
block_on(self.vcs_api.update_labels(
self.repo,
self.change.number,
to_add,
to_remove,
));
async fn update_labels(&self, to_add: &[String], to_remove: &[String]) {
self.vcs_api
.update_labels(self.repo, self.change.number, to_add, to_remove)
.await;
}
fn request_reviews(&self, impacted_maintainers: &maintainers::ImpactedMaintainers) {
async fn request_reviews(&self, impacted_maintainers: &maintainers::ImpactedMaintainers) {
info!(
"Impact maintainers: {:?}",
impacted_maintainers.maintainers()
);
if impacted_maintainers.maintainers().len() < 10 {
let existing_reviewers = block_on(
self.vcs_api
.get_existing_reviewers(self.repo, self.change.number),
);
let existing_reviewers = self
.vcs_api
.get_existing_reviewers(self.repo, self.change.number)
.await;
// Normalize both sides, compute the IM - ER set
let new_reviewers: Vec<String> = impacted_maintainers
@ -136,12 +124,9 @@ impl<'a> NixpkgsStrategy<'a> {
.collect();
// Add them as reviewers.
block_on(self.vcs_api.request_reviewers(
self.repo,
self.change.number,
new_reviewers,
vec![],
));
self.vcs_api
.request_reviewers(self.repo, self.change.number, new_reviewers, vec![])
.await;
} else {
warn!(
"Too many reviewers ({}), skipping review requests",
@ -150,32 +135,33 @@ impl<'a> NixpkgsStrategy<'a> {
}
}
fn check_stdenvs_before(&mut self, dir: &Path) {
async fn check_stdenvs_before(&mut self, dir: &Path) {
let mut stdenvs = Stdenvs::new(self.nix.clone(), dir.to_path_buf());
stdenvs.identify_before();
stdenvs.identify_before().await;
self.stdenv_diff = Some(stdenvs);
}
fn check_stdenvs_after(&mut self) {
async fn check_stdenvs_after(&mut self) {
if let Some(ref mut stdenvs) = self.stdenv_diff {
stdenvs.identify_after();
stdenvs.identify_after().await;
}
}
fn update_stdenv_labels(&self) {
async fn update_stdenv_labels(&self) {
if let Some(ref stdenvs) = self.stdenv_diff {
let mut stdenvtagger = StdenvTagger::new();
if !stdenvs.are_same() {
stdenvtagger.changed(stdenvs.changed());
}
self.update_labels(&stdenvtagger.tags_to_add(), &stdenvtagger.tags_to_remove());
self.update_labels(&stdenvtagger.tags_to_add(), &stdenvtagger.tags_to_remove())
.await;
}
}
fn check_outpaths_before(&mut self, dir: &Path) -> StepResult<()> {
async fn check_outpaths_before(&mut self, dir: &Path) -> StepResult<()> {
let mut rebuildsniff = OutPathDiff::new(self.nix.clone(), dir.to_path_buf());
if let Err(err) = rebuildsniff.find_before() {
if let Err(err) = rebuildsniff.find_before().await {
/*
self.events
.notify(Event::TargetBranchFailsEvaluation(target_branch.clone()));
@ -192,9 +178,9 @@ impl<'a> NixpkgsStrategy<'a> {
}
}
fn check_outpaths_after(&mut self) -> StepResult<()> {
async fn check_outpaths_after(&mut self) -> StepResult<()> {
if let Some(ref mut rebuildsniff) = self.outpath_diff {
if let Err(err) = rebuildsniff.find_after() {
if let Err(err) = rebuildsniff.find_after().await {
Err(Error::FailWithPastebin(
String::from("This PR does not cleanly list package outputs after merging."),
String::from("Output path comparison"),
@ -238,7 +224,7 @@ impl<'a> NixpkgsStrategy<'a> {
vec![]
}
fn update_new_package_labels(&self) {
async fn update_new_package_labels(&self) {
if let Some(ref rebuildsniff) = self.outpath_diff {
if let Some((removed, added)) = rebuildsniff.package_diff() {
let mut addremovetagger = PkgsAddedRemovedTagger::new();
@ -246,12 +232,13 @@ impl<'a> NixpkgsStrategy<'a> {
self.update_labels(
&addremovetagger.tags_to_add(),
&addremovetagger.tags_to_remove(),
);
)
.await;
}
}
}
fn update_rebuild_labels(
async fn update_rebuild_labels(
&mut self,
dir: &Path,
overall_status: &mut CommitStatus,
@ -261,20 +248,21 @@ impl<'a> NixpkgsStrategy<'a> {
if let Some(attrs) = rebuildsniff.calculate_rebuild() {
if !attrs.is_empty() {
overall_status.set_url(self.gist_changed_paths(&attrs));
self.record_impacted_maintainers(dir, &attrs)?;
overall_status.set_url(self.gist_changed_paths(&attrs).await);
self.record_impacted_maintainers(dir, &attrs).await?;
}
rebuild_tags.parse_attrs(attrs);
}
self.update_labels(&rebuild_tags.tags_to_add(), &rebuild_tags.tags_to_remove());
self.update_labels(&rebuild_tags.tags_to_add(), &rebuild_tags.tags_to_remove())
.await;
}
Ok(())
}
fn gist_changed_paths(&mut self, attrs: &[PackageArch]) -> Option<String> {
block_on(crate::utils::pastebin::make_pastebin(
async fn gist_changed_paths(&mut self, attrs: &[PackageArch]) -> Option<String> {
crate::utils::pastebin::make_pastebin(
&mut self.chan,
"Changed Paths",
attrs
@ -282,12 +270,13 @@ impl<'a> NixpkgsStrategy<'a> {
.map(|attr| format!("{}\t{}", &attr.architecture, &attr.package))
.collect::<Vec<String>>()
.join("\n"),
))
)
.await
.ok()
.map(|pp| pp.uri)
}
fn record_impacted_maintainers(
async fn record_impacted_maintainers(
&mut self,
dir: &Path,
attrs: &[PackageArch],
@ -299,16 +288,18 @@ impl<'a> NixpkgsStrategy<'a> {
if let Some(ref changed_paths) = self.changed_paths {
let m =
ImpactedMaintainers::calculate(&self.nix, dir, changed_paths, &changed_attributes);
ImpactedMaintainers::calculate(&self.nix, dir, changed_paths, &changed_attributes)
.await;
let gist_url = block_on(crate::utils::pastebin::make_pastebin(
let gist_url = crate::utils::pastebin::make_pastebin(
&mut self.chan,
"Potential Maintainers",
match m {
Ok(ref maintainers) => format!("Maintainers:\n{}", maintainers),
Err(ref e) => format!("Ignorable calculation error:\n{:?}", e),
Ok(ref maintainers) => format!("Maintainers:\n{maintainers}"),
Err(ref e) => format!("Ignorable calculation error:\n{e:?}"),
},
))
)
.await
.ok()
.map(|pp| pp.uri);
@ -325,7 +316,7 @@ impl<'a> NixpkgsStrategy<'a> {
String::from("large change, skipping automatic review requests"),
gist_url,
);
status.set(State::Success)?;
status.set(State::Success).await?;
return Ok(());
}
@ -337,26 +328,28 @@ impl<'a> NixpkgsStrategy<'a> {
String::from("matching changed paths to changed attrs..."),
gist_url,
);
status.set(State::Success)?;
status.set(State::Success).await?;
if let Ok(ref maint) = m {
self.request_reviews(maint);
self.request_reviews(maint).await;
let mut maint_tagger = MaintainerPrTagger::new();
// TODO: this is really weird.
let issue = block_on(
self.vcs_api.get_issue(self.repo, self.change.number),
)
.expect("Failed to obtain the issue");
let issue = self
.vcs_api
.get_issue(self.repo, self.change.number)
.await
.expect("Failed to obtain the issue");
maint_tagger
.record_maintainer(&issue.created_by.username, &maint.maintainers_by_package());
self.update_labels(&maint_tagger.tags_to_add(), &maint_tagger.tags_to_remove());
self.update_labels(&maint_tagger.tags_to_add(), &maint_tagger.tags_to_remove())
.await;
}
}
Ok(())
}
fn check_meta_queue_builds(&mut self, dir: &Path) -> StepResult<Vec<BuildJob>> {
async fn check_meta_queue_builds(&mut self, dir: &Path) -> StepResult<Vec<BuildJob>> {
if let Some(ref possibly_touched_packages) = self.touched_packages {
let mut status = CommitStatus::new(
self.vcs_api.clone(),
@ -366,10 +359,10 @@ impl<'a> NixpkgsStrategy<'a> {
String::from("config.nix: checkMeta = true"),
None,
);
status.set(State::Pending)?;
status.set(State::Pending).await?;
let nixenv = HydraNixEnv::new(self.nix.clone(), dir.to_path_buf(), true);
match nixenv.execute_with_stats() {
match nixenv.execute_with_stats().await {
Ok((pkgs, _stats)) => {
let mut try_build: Vec<String> = pkgs
.keys()
@ -381,7 +374,7 @@ impl<'a> NixpkgsStrategy<'a> {
try_build.dedup();
status.set_url(None);
status.set(State::Success)?;
status.set(State::Success).await?;
if !try_build.is_empty() && try_build.len() <= 20 {
// In the case of trying to merge master in to
@ -403,15 +396,16 @@ impl<'a> NixpkgsStrategy<'a> {
}
Err(out) => {
status.set_url(
block_on(crate::utils::pastebin::make_pastebin(
crate::utils::pastebin::make_pastebin(
&mut self.chan,
"Meta Check",
out.display(),
))
)
.await
.ok()
.map(|pp| pp.uri),
);
status.set(State::Failure)?;
status.set(State::Failure).await?;
Err(Error::Fail(String::from(
"Failed to validate package metadata.",
)))
@ -421,25 +415,31 @@ impl<'a> NixpkgsStrategy<'a> {
Ok(vec![])
}
}
}
impl<'a> EvaluationStrategy for NixpkgsStrategy<'a> {
fn pre_clone(&mut self) -> StepResult<()> {
self.tag_from_title();
pub(crate) async fn pre_clone(&self) -> StepResult<()> {
self.tag_from_title().await;
Ok(())
}
fn on_target_branch(&mut self, dir: &Path, status: &mut CommitStatus) -> StepResult<()> {
status.set_with_description("Checking original stdenvs", State::Pending)?;
self.check_stdenvs_before(dir);
pub(crate) async fn on_target_branch(
&mut self,
dir: &Path,
status: &mut CommitStatus,
) -> StepResult<()> {
status
.set_with_description("Checking original stdenvs", State::Pending)
.await?;
self.check_stdenvs_before(dir).await;
status.set_with_description("Checking original out paths", State::Pending)?;
self.check_outpaths_before(dir)?;
status
.set_with_description("Checking original out paths", State::Pending)
.await?;
self.check_outpaths_before(dir).await?;
Ok(())
}
fn after_fetch(&mut self, co: &CachedProjectCo) -> StepResult<()> {
pub(crate) fn after_fetch(&mut self, co: &CachedProjectCo) {
let changed_paths = co
.files_changed_from_head(&self.job.change.head_sha)
.unwrap_or_else(|_| vec![]);
@ -447,29 +447,34 @@ impl<'a> EvaluationStrategy for NixpkgsStrategy<'a> {
self.touched_packages = Some(parse_commit_messages(
&co.commit_messages_from_head(&self.job.change.head_sha)
.unwrap_or_else(|_| vec!["".to_owned()]),
.unwrap_or_else(|_| vec![String::new()]),
));
}
pub(crate) async fn merge_conflict(&self) {
self.update_labels(&["2.status: merge conflict".to_owned()], &[])
.await;
}
pub(crate) async fn after_merge(&mut self, status: &mut CommitStatus) -> StepResult<()> {
self.update_labels(&[], &["2.status: merge conflict".to_owned()])
.await;
status
.set_with_description("Checking new stdenvs", State::Pending)
.await?;
self.check_stdenvs_after().await;
status
.set_with_description("Checking new out paths", State::Pending)
.await?;
self.check_outpaths_after().await?;
Ok(())
}
fn merge_conflict(&mut self) {
self.update_labels(&["2.status: merge conflict".to_owned()], &[]);
}
fn after_merge(&mut self, status: &mut CommitStatus) -> StepResult<()> {
self.update_labels(&[], &["2.status: merge conflict".to_owned()]);
status.set_with_description("Checking new stdenvs", State::Pending)?;
self.check_stdenvs_after();
status.set_with_description("Checking new out paths", State::Pending)?;
self.check_outpaths_after()?;
Ok(())
}
fn evaluation_checks(&self) -> Vec<EvalChecker> {
#[allow(clippy::too_many_lines)]
pub(crate) fn evaluation_checks(&self) -> Vec<EvalChecker> {
// the value that's passed as the nixpkgs arg
let nixpkgs_arg_value = format!(
"{{ outPath=./.; revCount=999999; shortRev=\"{}\"; rev=\"{}\"; }}",
@ -600,20 +605,22 @@ impl<'a> EvaluationStrategy for NixpkgsStrategy<'a> {
]
}
fn all_evaluations_passed(
pub(crate) async fn all_evaluations_passed(
&mut self,
dir: &Path,
status: &mut CommitStatus,
) -> StepResult<EvaluationComplete> {
self.update_stdenv_labels();
self.update_stdenv_labels().await;
status.set_with_description("Calculating Changed Outputs", State::Pending)?;
status
.set_with_description("Calculating Changed Outputs", State::Pending)
.await?;
self.update_new_package_labels();
self.update_rebuild_labels(dir, status)?;
self.update_new_package_labels().await;
self.update_rebuild_labels(dir, status).await?;
let checks = self.performance_stats();
let builds = self.check_meta_queue_builds(dir)?;
let builds = self.check_meta_queue_builds(dir).await?;
Ok(EvaluationComplete { builds, checks })
}
}
@ -628,7 +635,7 @@ fn parse_commit_messages(messages: &[String]) -> Vec<String> {
// NOTE: This transforms `{foo,bar}` into `{{foo,bar}}` and `foo,bar` into `{foo,bar}`,
// which allows both the old style (`foo,bar`) and the new style (`{foo,bar}`) to expand to
// `foo` and `bar`.
.flat_map(|line| brace_expand::brace_expand(&format!("{{{}}}", line)))
.flat_map(|line| brace_expand::brace_expand(&format!("{{{line}}}")))
.map(|line| line.trim().to_owned())
.collect()
}
@ -674,7 +681,7 @@ mod tests {
firefox{,-beta}{,-bin}, librewolf: blah blah blah
"
.lines()
.map(|l| l.to_owned())
.map(std::borrow::ToOwned::to_owned)
.collect::<Vec<String>>(),
),
expect
@ -709,7 +716,10 @@ mod tests {
);
assert_eq!(
label_from_title("fix build on bsd and darwin"),
[String::from("6.topic: bsd"), String::from("6.topic: darwin")]
[
String::from("6.topic: bsd"),
String::from("6.topic: darwin")
]
);
assert_eq!(
label_from_title("pkg: fix cross"),

72
ofborg/src/tasks/eval/stdenvs.rs
View file

@ -29,6 +29,7 @@ pub struct Stdenvs {
}
impl Stdenvs {
#[must_use]
pub fn new(nix: nix::Nix, co: PathBuf) -> Stdenvs {
Stdenvs {
nix,
@ -42,20 +43,22 @@ impl Stdenvs {
}
}
pub fn identify_before(&mut self) {
self.identify(System::X8664Linux, StdenvFrom::Before);
self.identify(System::X8664Darwin, StdenvFrom::Before);
pub async fn identify_before(&mut self) {
self.identify(System::X8664Linux, StdenvFrom::Before).await;
self.identify(System::X8664Darwin, StdenvFrom::Before).await;
}
pub fn identify_after(&mut self) {
self.identify(System::X8664Linux, StdenvFrom::After);
self.identify(System::X8664Darwin, StdenvFrom::After);
pub async fn identify_after(&mut self) {
self.identify(System::X8664Linux, StdenvFrom::After).await;
self.identify(System::X8664Darwin, StdenvFrom::After).await;
}
#[must_use]
pub fn are_same(&self) -> bool {
self.changed().is_empty()
}
#[must_use]
pub fn changed(&self) -> Vec<System> {
let mut changed: Vec<System> = vec![];
@ -70,39 +73,42 @@ impl Stdenvs {
changed
}
fn identify(&mut self, system: System, from: StdenvFrom) {
async fn identify(&mut self, system: System, from: StdenvFrom) {
match (system, from) {
(System::X8664Linux, StdenvFrom::Before) => {
self.linux_stdenv_before = self.evalstdenv("x86_64-linux");
self.linux_stdenv_before = self.evalstdenv("x86_64-linux").await;
}
(System::X8664Linux, StdenvFrom::After) => {
self.linux_stdenv_after = self.evalstdenv("x86_64-linux");
self.linux_stdenv_after = self.evalstdenv("x86_64-linux").await;
}
(System::X8664Darwin, StdenvFrom::Before) => {
self.darwin_stdenv_before = self.evalstdenv("x86_64-darwin");
self.darwin_stdenv_before = self.evalstdenv("x86_64-darwin").await;
}
(System::X8664Darwin, StdenvFrom::After) => {
self.darwin_stdenv_after = self.evalstdenv("x86_64-darwin");
self.darwin_stdenv_after = self.evalstdenv("x86_64-darwin").await;
}
}
}
/// This is used to find out what the output path of the stdenv for the
/// given system.
fn evalstdenv(&self, system: &str) -> Option<String> {
async fn evalstdenv(&self, system: &str) -> Option<String> {
info!(?system, "query stdenv output");
let result = self.nix.with_system(system.to_owned()).safely(
&nix::Operation::QueryPackagesOutputs,
&self.co,
vec![
String::from("-f"),
String::from("."),
String::from("-A"),
String::from("stdenv"),
],
true,
);
let nix = self.nix.with_system(system.to_owned());
let result = nix
.safely(
&nix::Operation::QueryPackagesOutputs,
&self.co,
vec![
String::from("-f"),
String::from("."),
String::from("-A"),
String::from("stdenv"),
],
true,
)
.await;
match result {
Ok(mut out) => Some(file_to_str(&mut out)),
@ -121,8 +127,8 @@ mod tests {
use std::env;
use std::process::Command;
#[test]
fn stdenv_checking() {
#[tokio::test]
async fn stdenv_checking() {
let output = Command::new("nix-instantiate")
.args(["--eval", "-E", "<nixpkgs>"])
.output()
@ -130,14 +136,20 @@ mod tests {
let nixpkgs = String::from_utf8(output.stdout).expect("nixpkgs required");
let remote = env::var("NIX_REMOTE").unwrap_or("".to_owned());
let remote = env::var("NIX_REMOTE").unwrap_or_default();
let nix = nix::Nix::new(String::from("x86_64-linux"), remote, 1200, None);
let mut stdenv = Stdenvs::new(nix, PathBuf::from(nixpkgs.trim_end()));
stdenv.identify(System::X8664Linux, StdenvFrom::Before);
stdenv.identify(System::X8664Darwin, StdenvFrom::Before);
stdenv
.identify(System::X8664Linux, StdenvFrom::Before)
.await;
stdenv
.identify(System::X8664Darwin, StdenvFrom::Before)
.await;
stdenv.identify(System::X8664Linux, StdenvFrom::After);
stdenv.identify(System::X8664Darwin, StdenvFrom::After);
stdenv.identify(System::X8664Linux, StdenvFrom::After).await;
stdenv
.identify(System::X8664Darwin, StdenvFrom::After)
.await;
assert!(stdenv.are_same());
}

364
ofborg/src/tasks/evaluate.rs
View file

@ -2,6 +2,7 @@
use crate::acl::Acl;
use crate::checkout;
use crate::files::file_to_str;
use crate::message::evaluationjob::Actions;
use crate::message::{buildjob, evaluationjob};
use crate::nix;
use crate::stats::{self, Event};
@ -14,10 +15,10 @@ use crate::vcs::gerrit::http::GerritHTTPApi;
use crate::worker;
use std::path::Path;
use std::rc::Rc;
use std::sync::Arc;
use std::time::Instant;
use futures::executor::block_on;
use async_trait::async_trait;
use tracing::{debug_span, error, info, warn};
pub enum SupportedVCS {
@ -54,18 +55,24 @@ impl<E: stats::SysEvents> EvaluationWorker<E> {
}
}
impl<E: stats::SysEvents + 'static> worker::SimpleWorker for EvaluationWorker<E> {
#[async_trait]
impl<E: stats::SysEvents + 'static + Sync + Send> worker::SimpleWorker for EvaluationWorker<E> {
type J = evaluationjob::EvaluationJob;
fn msg_to_job(&mut self, _: &str, _: &Option<String>, body: &[u8]) -> Result<Self::J, String> {
self.events.notify(Event::JobReceived);
async fn msg_to_job(
&mut self,
_: &str,
_: &Option<String>,
body: &[u8],
) -> Result<Self::J, String> {
self.events.notify(Event::JobReceived).await;
match evaluationjob::from(body) {
Ok(e) => {
self.events.notify(Event::JobDecodeSuccess);
self.events.notify(Event::JobDecodeSuccess).await;
Ok(e)
}
Err(e) => {
self.events.notify(Event::JobDecodeFailure);
self.events.notify(Event::JobDecodeFailure).await;
error!(
"Failed to decode message: {:?}, Err: {:?}",
String::from_utf8(body.to_vec()),
@ -76,16 +83,16 @@ impl<E: stats::SysEvents + 'static> worker::SimpleWorker for EvaluationWorker<E>
}
}
fn consumer(
async fn consumer(
&mut self,
chan: &mut lapin::Channel,
job: &evaluationjob::EvaluationJob,
) -> worker::Actions {
let span = debug_span!("job", change_id = ?job.change.number);
let span: tracing::Span = debug_span!("job", change_id = ?job.change.number);
let _enter = span.enter();
let vcs_api: Rc<dyn VersionControlSystemAPI> = match self.vcs {
SupportedVCS::Gerrit => Rc::new(GerritHTTPApi),
let vcs_api: Arc<dyn VersionControlSystemAPI> = match self.vcs {
SupportedVCS::Gerrit => Arc::new(GerritHTTPApi),
};
OneEval::new(
@ -98,11 +105,12 @@ impl<E: stats::SysEvents + 'static> worker::SimpleWorker for EvaluationWorker<E>
job,
)
.worker_actions(chan)
.await
}
}
struct OneEval<'a, E> {
vcs_api: Rc<dyn VersionControlSystemAPI>,
vcs_api: Arc<dyn VersionControlSystemAPI>,
nix: &'a nix::Nix,
acl: &'a Acl,
events: &'a mut E,
@ -114,7 +122,7 @@ struct OneEval<'a, E> {
impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> {
#[allow(clippy::too_many_arguments)]
fn new(
vcs_api: Rc<dyn VersionControlSystemAPI>,
vcs_api: Arc<dyn VersionControlSystemAPI>,
nix: &'a nix::Nix,
acl: &'a Acl,
events: &'a mut E,
@ -133,11 +141,7 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> {
}
}
fn actions(&self) -> evaluationjob::Actions {
evaluationjob::Actions {}
}
fn update_status(
async fn update_status(
&self,
description: String,
url: Option<String>,
@ -158,42 +162,50 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> {
&self.job.change.number, &self.job.change.head_sha, &description
);
block_on(self.vcs_api.create_commit_statuses(
&self.job.repo,
self.job.change.head_sha.clone(),
state,
"ofborg-eval".to_owned(),
description,
// TODO: make this an option
url.unwrap_or_else(|| "".to_owned()),
))
self.vcs_api
.create_commit_statuses(
&self.job.repo,
self.job.change.head_sha.clone(),
state,
"ofborg-eval".to_owned(),
description,
// TODO: make this an option
url.unwrap_or_default(),
)
.await
}
fn make_pastebin(
async fn make_pastebin(
&mut self,
chan: &mut lapin::Channel,
title: &str,
contents: String,
) -> Option<PersistedPastebin> {
block_on(crate::utils::pastebin::make_pastebin(chan, title, contents)).ok()
crate::utils::pastebin::make_pastebin(chan, title, contents)
.await
.ok()
}
fn worker_actions(&mut self, chan: &mut lapin::Channel) -> worker::Actions {
let eval_result = self
.evaluate_job(chan)
.map_err(|eval_error| match eval_error {
// Handle error cases which expect us to post statuses
// to github. Convert Eval Errors in to Result<_, CommitStatusWrite>
EvalWorkerError::EvalError(eval::Error::Fail(msg)) => {
self.update_status(msg, None, State::Failure)
}
EvalWorkerError::EvalError(eval::Error::FailWithPastebin(msg, title, content)) => {
let pastebin = self.make_pastebin(chan, &title, content).map(|pp| pp.uri);
self.update_status(msg, pastebin, State::Failure)
}
EvalWorkerError::EvalError(eval::Error::CommitStatusWrite(e)) => Err(e),
EvalWorkerError::CommitStatusWrite(e) => Err(e),
});
async fn worker_actions(&mut self, chan: &mut lapin::Channel) -> worker::Actions {
let eval_result = match self.evaluate_job(chan).await {
Ok(r) => Ok(r),
// Handle error cases which expect us to post statuses
// to github. Convert Eval Errors in to Result<_, CommitStatusWrite>
Err(EvalWorkerError::EvalError(eval::Error::Fail(msg))) => {
Err(self.update_status(msg, None, State::Failure).await)
}
Err(EvalWorkerError::EvalError(eval::Error::FailWithPastebin(msg, title, content))) => {
let pastebin = self
.make_pastebin(chan, &title, content)
.await
.map(|pp| pp.uri);
Err(self.update_status(msg, pastebin, State::Failure).await)
}
Err(
EvalWorkerError::EvalError(eval::Error::CommitStatusWrite(e))
| EvalWorkerError::CommitStatusWrite(e),
) => Err(Err(e)),
};
match eval_result {
Ok(eval_actions) => eval_actions,
@ -201,18 +213,18 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> {
// There was an error during eval, but we successfully
// updated the PR.
self.actions().skip(self.job)
Actions::skip(self.job)
}
Err(Err(CommitStatusError::ExpiredCreds(e))) => {
error!("Failed writing commit status: creds expired: {:?}", e);
self.actions().retry_later(self.job)
Actions::retry_later(self.job)
}
Err(Err(CommitStatusError::MissingSha(e))) => {
error!(
"Failed writing commit status: commit sha was force-pushed away: {:?}",
e
);
self.actions().skip(self.job)
Actions::skip(self.job)
}
Err(Err(CommitStatusError::Error(cswerr))) => {
@ -221,38 +233,41 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> {
cswerr
);
block_on(self.vcs_api.update_labels(
&self.job.repo,
self.job.change.number,
&[String::from("ofborg-internal-error")],
&[],
));
self.vcs_api
.update_labels(
&self.job.repo,
self.job.change.number,
&[String::from("ofborg-internal-error")],
&[],
)
.await;
self.actions().skip(self.job)
Actions::skip(self.job)
}
}
}
// FIXME: remove with rust/cargo update
#[allow(clippy::cognitive_complexity)]
fn evaluate_job(
#[allow(clippy::too_many_lines)]
async fn evaluate_job(
&mut self,
chan: &mut lapin::Channel,
) -> Result<worker::Actions, EvalWorkerError> {
let job = self.job;
let issue_ref = block_on(self.vcs_api.get_issue(&job.repo, job.change.number));
let issue_ref = self.vcs_api.get_issue(&job.repo, job.change.number).await;
let auto_schedule_build_archs: Vec<systems::System>;
let _issue: Issue = match issue_ref {
Ok(iss) => {
if matches!(iss.state, IssueState::Closed) {
self.events.notify(Event::IssueAlreadyClosed);
self.events.notify(Event::IssueAlreadyClosed).await;
info!("Skipping {} because it is closed", job.change.number);
return Ok(self.actions().skip(job));
return Ok(Actions::skip(job));
}
if iss.is_wip() {
self.events.notify(Event::CurrentlyWorkInProgress);
self.events.notify(Event::CurrentlyWorkInProgress).await;
auto_schedule_build_archs = vec![];
} else {
auto_schedule_build_archs = self.acl.build_job_architectures_for_user_repo(
@ -265,25 +280,21 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> {
}
Err(e) => {
self.events.notify(Event::IssueFetchFailed);
self.events.notify(Event::IssueFetchFailed).await;
error!("Error fetching {}!", job.change.number);
error!("E: {:?}", e);
return Ok(self.actions().skip(job));
return Ok(Actions::skip(job));
}
};
let mut evaluation_strategy: Box<dyn eval::EvaluationStrategy> = if job.is_nixpkgs() {
Box::new(eval::NixpkgsStrategy::new(
chan.clone(),
job,
self.vcs_api.clone(),
&job.repo,
&job.change,
self.nix.clone(),
))
} else {
Box::new(eval::GenericStrategy::new())
};
let mut evaluation_strategy = eval::NixpkgsStrategy::new(
chan.clone(),
job,
self.vcs_api.clone(),
&job.repo,
&job.change,
self.nix.clone(),
);
let mut overall_status = CommitStatus::new(
self.vcs_api.clone(),
@ -294,15 +305,19 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> {
None,
);
overall_status.set_with_description("Starting", State::Pending)?;
overall_status
.set_with_description("Starting", State::Pending)
.await?;
evaluation_strategy.pre_clone()?;
evaluation_strategy.pre_clone().await?;
let project = self
.cloner
.project(&job.repo.full_name, job.repo.clone_url.clone());
overall_status.set_with_description("Cloning project", State::Pending)?;
overall_status
.set_with_description("Cloning project", State::Pending)
.await?;
info!("Working on {}", job.change.number);
let co = project
@ -317,142 +332,163 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> {
// TODO: this is a preflight check, encode it as such.
if target_branch.starts_with("nixos-") || target_branch.starts_with("nixpkgs-") {
overall_status.set_with_description(
"The branch you have targeted is a read-only mirror for channels. \
overall_status
.set_with_description(
"The branch you have targeted is a read-only mirror for channels. \
Please target release-* or master.",
State::Error,
)?;
State::Error,
)
.await?;
info!("PR targets a nixos-* or nixpkgs-* branch");
return Ok(self.actions().skip(job));
return Ok(Actions::skip(job));
};
overall_status.set_with_description(
format!("Checking out {}", &target_branch).as_ref(),
State::Pending,
)?;
overall_status
.set_with_description(
format!("Checking out {}", &target_branch).as_ref(),
State::Pending,
)
.await?;
info!("Checking out target branch {}", &target_branch);
let refpath = co.checkout_origin_ref(target_branch.as_ref()).unwrap();
evaluation_strategy.on_target_branch(Path::new(&refpath), &mut overall_status)?;
evaluation_strategy
.on_target_branch(Path::new(&refpath), &mut overall_status)
.await?;
let target_branch_rebuild_sniff_start = Instant::now();
self.events.notify(Event::EvaluationDuration(
target_branch.clone(),
target_branch_rebuild_sniff_start.elapsed().as_secs(),
));
self.events
.notify(Event::EvaluationDurationCount(target_branch));
.notify(Event::EvaluationDuration(
target_branch.clone(),
target_branch_rebuild_sniff_start.elapsed().as_secs(),
))
.await;
self.events
.notify(Event::EvaluationDurationCount(target_branch))
.await;
overall_status.set_with_description("Fetching PR", State::Pending)?;
overall_status
.set_with_description("Fetching PR", State::Pending)
.await?;
// TODO: generalize fetch change
co.fetch_pr(job.change.number).unwrap();
if !co.commit_exists(job.change.head_sha.as_ref()) {
overall_status.set_with_description("Commit not found", State::Error)?;
overall_status
.set_with_description("Commit not found", State::Error)
.await?;
info!("Commit {} doesn't exist", job.change.head_sha);
return Ok(self.actions().skip(job));
return Ok(Actions::skip(job));
}
evaluation_strategy.after_fetch(&co)?;
evaluation_strategy.after_fetch(&co);
overall_status.set_with_description("Merging PR", State::Pending)?;
overall_status
.set_with_description("Merging PR", State::Pending)
.await?;
if co.merge_commit(job.change.head_sha.as_ref()).is_err() {
overall_status.set_with_description("Failed to merge", State::Failure)?;
overall_status
.set_with_description("Failed to merge", State::Failure)
.await?;
info!("Failed to merge {}", job.change.head_sha);
evaluation_strategy.merge_conflict();
evaluation_strategy.merge_conflict().await;
return Ok(self.actions().skip(job));
return Ok(Actions::skip(job));
}
evaluation_strategy.after_merge(&mut overall_status)?;
evaluation_strategy.after_merge(&mut overall_status).await?;
info!("Got path: {:?}, building", refpath);
overall_status.set_with_description("Beginning Evaluations", State::Pending)?;
overall_status
.set_with_description("Beginning Evaluations", State::Pending)
.await?;
let eval_results: bool = evaluation_strategy
.evaluation_checks()
.into_iter()
.map(|check| {
let mut status = CommitStatus::new(
self.vcs_api.clone(),
job.repo.clone(),
job.change.head_sha.clone(),
format!("ofborg-eval-{}", check.name()),
check.cli_cmd(),
None,
);
let mut all_good = true;
for check in evaluation_strategy.evaluation_checks() {
let mut status = CommitStatus::new(
self.vcs_api.clone(),
job.repo.clone(),
job.change.head_sha.clone(),
format!("ofborg-eval-{}", check.name()),
check.cli_cmd(),
None,
);
status
.set(State::Pending)
.expect("Failed to set status on eval strategy");
status
.set(State::Pending)
.await
.expect("Failed to set status on eval strategy");
let state: State;
let gist_url: Option<String>;
match check.execute(Path::new(&refpath)) {
Ok(_) => {
state = State::Success;
gist_url = None;
}
Err(mut out) => {
state = State::Failure;
gist_url = self
.make_pastebin(
chan,
&format!("[ofborg] Evaluation of {}", check.name()),
file_to_str(&mut out),
)
.map(|pp| pp.uri);
}
let state: State;
let gist_url: Option<String>;
match check.execute(Path::new(&refpath)).await {
Ok(_) => {
state = State::Success;
gist_url = None;
}
status.set_url(gist_url);
status
.set(state)
.expect("Failed to set status on eval strategy");
if state == State::Success {
Ok(())
} else {
Err(())
Err(mut out) => {
state = State::Failure;
gist_url = self
.make_pastebin(
chan,
&format!("[ofborg] Evaluation of {}", check.name()),
file_to_str(&mut out),
)
.await
.map(|pp| pp.uri);
}
})
.all(|status| status == Ok(()));
}
status.set_url(gist_url);
status
.set(state)
.await
.expect("Failed to set status on eval strategy");
if state != State::Success {
all_good = false;
}
}
info!("Finished evaluations");
let mut response: worker::Actions = vec![];
if eval_results {
if all_good {
let complete = evaluation_strategy
.all_evaluations_passed(Path::new(&refpath), &mut overall_status)?;
.all_evaluations_passed(Path::new(&refpath), &mut overall_status)
.await?;
block_on(
self.vcs_api
.create_check_statuses(&job.repo, complete.checks),
);
response.extend(schedule_builds(complete.builds, auto_schedule_build_archs));
self.vcs_api
.create_check_statuses(&job.repo, complete.checks)
.await;
response.extend(schedule_builds(complete.builds, &auto_schedule_build_archs));
overall_status.set_with_description("^.^!", State::Success)?;
overall_status
.set_with_description("^.^!", State::Success)
.await?;
} else {
overall_status.set_with_description("Complete, with errors", State::Failure)?;
overall_status
.set_with_description("Complete, with errors", State::Failure)
.await?;
}
self.events.notify(Event::TaskEvaluationCheckComplete);
self.events.notify(Event::TaskEvaluationCheckComplete).await;
info!("Evaluations done!");
Ok(self.actions().done(job, response))
Ok(Actions::done(job, response))
}
}
fn schedule_builds(
builds: Vec<buildjob::BuildJob>,
auto_schedule_build_archs: Vec<systems::System>,
auto_schedule_build_archs: &[systems::System],
) -> Vec<worker::Action> {
let mut response = vec![];
info!(
@ -460,20 +496,22 @@ fn schedule_builds(
builds, auto_schedule_build_archs
);
for buildjob in builds {
for arch in auto_schedule_build_archs.iter() {
for arch in auto_schedule_build_archs {
let (exchange, routingkey) = arch.as_build_destination();
response.push(worker::publish_serde_action(
exchange, routingkey, &buildjob,
&exchange,
&routingkey,
&buildjob,
));
}
response.push(worker::publish_serde_action(
Some("build-results".to_string()),
None,
&Some("build-results".to_string()),
&None,
&buildjob::QueuedBuildJobs {
job: buildjob,
architectures: auto_schedule_build_archs
.iter()
.map(|arch| arch.to_string())
.map(std::string::ToString::to_string)
.collect(),
},
));

22
ofborg/src/tasks/evaluationfilter.rs
View file

@ -3,6 +3,7 @@ use crate::ghevent;
use crate::message::{evaluationjob, Change, Repo};
use crate::worker;
use async_trait::async_trait;
use tracing::{debug_span, info};
pub struct EvaluationFilterWorker {
@ -10,15 +11,22 @@ pub struct EvaluationFilterWorker {
}
impl EvaluationFilterWorker {
#[must_use]
pub fn new(acl: acl::Acl) -> EvaluationFilterWorker {
EvaluationFilterWorker { acl }
}
}
#[async_trait]
impl worker::SimpleWorker for EvaluationFilterWorker {
type J = ghevent::PullRequestEvent;
fn msg_to_job(&mut self, _: &str, _: &Option<String>, body: &[u8]) -> Result<Self::J, String> {
async fn msg_to_job(
&mut self,
_: &str,
_: &Option<String>,
body: &[u8],
) -> Result<Self::J, String> {
match serde_json::from_slice(body) {
Ok(e) => Ok(e),
Err(e) => Err(format!(
@ -29,7 +37,7 @@ impl worker::SimpleWorker for EvaluationFilterWorker {
}
}
fn consumer(
async fn consumer(
&mut self,
_chan: &mut lapin::Channel,
job: &ghevent::PullRequestEvent,
@ -51,9 +59,9 @@ impl worker::SimpleWorker for EvaluationFilterWorker {
}
let interesting: bool = match job.action {
ghevent::PullRequestAction::Opened => true,
ghevent::PullRequestAction::Synchronize => true,
ghevent::PullRequestAction::Reopened => true,
ghevent::PullRequestAction::Opened
| ghevent::PullRequestAction::Synchronize
| ghevent::PullRequestAction::Reopened => true,
ghevent::PullRequestAction::Edited => {
if let Some(ref changes) = job.changes {
changes.base.is_some()
@ -61,7 +69,7 @@ impl worker::SimpleWorker for EvaluationFilterWorker {
false
}
}
_ => false,
ghevent::PullRequestAction::Unknown => false,
};
if !interesting {
@ -96,7 +104,7 @@ impl worker::SimpleWorker for EvaluationFilterWorker {
};
vec![
worker::publish_serde_action(None, Some("mass-rebuild-check-jobs".to_owned()), &msg),
worker::publish_serde_action(&None, &Some("mass-rebuild-check-jobs".to_owned()), &msg),
worker::Action::Ack,
]
}

30
ofborg/src/tasks/log_message_collector.rs
View file

@ -3,6 +3,7 @@ use crate::message::buildresult::BuildResult;
use crate::worker;
use crate::writetoline::LineWriter;
use async_trait::async_trait;
use std::fs::{self, File, OpenOptions};
use std::io::Write;
use std::path::{Component, Path, PathBuf};
@ -55,6 +56,7 @@ fn validate_path_segment(segment: &Path) -> Result<(), String> {
}
impl LogMessageCollector {
#[must_use]
pub fn new(log_root: PathBuf, max_open: usize) -> LogMessageCollector {
LogMessageCollector {
handles: LruCache::new(max_open),
@ -64,33 +66,33 @@ impl LogMessageCollector {
pub fn write_metadata(&mut self, from: &LogFrom, data: &BuildLogStart) -> Result<(), String> {
let metapath = self.path_for_metadata(from)?;
let mut fp = self.open_file(&metapath)?;
let mut fp = Self::open_file(&metapath)?;
match serde_json::to_string(data) {
Ok(data) => {
if let Err(e) = fp.write(data.as_bytes()) {
Err(format!("Failed to write metadata: {:?}", e))
Err(format!("Failed to write metadata: {e:?}"))
} else {
Ok(())
}
}
Err(e) => Err(format!("Failed to stringify metadata: {:?}", e)),
Err(e) => Err(format!("Failed to stringify metadata: {e:?}")),
}
}
pub fn write_result(&mut self, from: &LogFrom, data: &BuildResult) -> Result<(), String> {
let path = self.path_for_result(from)?;
let mut fp = self.open_file(&path)?;
let mut fp = Self::open_file(&path)?;
match serde_json::to_string(data) {
Ok(data) => {
if let Err(e) = fp.write(data.as_bytes()) {
Err(format!("Failed to write result: {:?}", e))
Err(format!("Failed to write result: {e:?}"))
} else {
Ok(())
}
}
Err(e) => Err(format!("Failed to stringify result: {:?}", e)),
Err(e) => Err(format!("Failed to stringify result: {e:?}")),
}
}
@ -102,7 +104,7 @@ impl LogMessageCollector {
.expect("handles just contained the key"))
} else {
let logpath = self.path_for_log(from)?;
let fp = self.open_file(&logpath)?;
let fp = Self::open_file(&logpath)?;
let writer = LineWriter::new(fp);
self.handles.insert(from.clone(), writer);
if let Some(handle) = self.handles.get_mut(from) {
@ -142,13 +144,12 @@ impl LogMessageCollector {
Ok(location)
} else {
Err(format!(
"Calculating the log location for {:?} resulted in an invalid path {:?}",
from, location
"Calculating the log location for {from:?} resulted in an invalid path {location:?}"
))
}
}
fn open_file(&self, path: &Path) -> Result<File, String> {
fn open_file(path: &Path) -> Result<File, String> {
let dir = path.parent().unwrap();
fs::create_dir_all(dir).unwrap();
@ -168,10 +169,11 @@ impl LogMessageCollector {
}
}
#[async_trait]
impl worker::SimpleWorker for LogMessageCollector {
type J = LogMessage;
fn msg_to_job(
async fn msg_to_job(
&mut self,
routing_key: &str,
_: &Option<String>,
@ -195,7 +197,7 @@ impl worker::SimpleWorker for LogMessageCollector {
attempt_id = msg.legacy().attempt_id;
message = MsgType::Finish(Box::new(msg));
} else {
return Err(format!("failed to decode job: {:?}", decode_msg));
return Err(format!("failed to decode job: {decode_msg:?}"));
}
}
}
@ -209,7 +211,7 @@ impl worker::SimpleWorker for LogMessageCollector {
})
}
fn consumer(&mut self, _chan: &mut lapin::Channel, job: &LogMessage) -> worker::Actions {
async fn consumer(&mut self, _chan: &mut lapin::Channel, job: &LogMessage) -> worker::Actions {
match job.message {
MsgType::Start(ref start) => {
self.write_metadata(&job.from, start)
@ -223,7 +225,7 @@ impl worker::SimpleWorker for LogMessageCollector {
MsgType::Msg(ref message) => {
let handle = self.handle_for(&job.from).unwrap();
handle.write_to_line((message.line_number - 1) as usize, &message.output);
handle.write_to_line(message.line_number - 1, &message.output);
}
MsgType::Finish(ref finish) => {
self.write_result(&job.from, finish)

7
ofborg/src/tasks/pastebin_collector.rs
View file

@ -4,6 +4,7 @@ use std::{
path::PathBuf,
};
use async_trait::async_trait;
use tracing::{debug_span, error, warn};
use zstd::stream::write::Encoder;
@ -18,6 +19,7 @@ pub struct PastebinCollector {
}
impl PastebinCollector {
#[must_use]
pub fn new(pastebin_root: PathBuf, db_path: PathBuf) -> Self {
Self {
pastebin_root,
@ -26,10 +28,11 @@ impl PastebinCollector {
}
}
#[async_trait]
impl worker::SimpleWorker for PastebinCollector {
type J = Pastebin;
fn msg_to_job(
async fn msg_to_job(
&mut self,
_routing_key: &str,
_: &Option<String>,
@ -47,7 +50,7 @@ impl worker::SimpleWorker for PastebinCollector {
}
}
fn consumer(&mut self, _chan: &mut lapin::Channel, job: &Self::J) -> worker::Actions {
async fn consumer(&mut self, _chan: &mut lapin::Channel, job: &Self::J) -> worker::Actions {
let span = debug_span!("pastebin", title = ?job.title);
let _enter = span.enter();

64
ofborg/src/tasks/statscollector.rs
View file

@ -1,6 +1,7 @@
use crate::stats;
use crate::worker;
use async_trait::async_trait;
use tracing::error;
pub struct StatCollectorWorker<E> {
@ -14,48 +15,57 @@ impl<E: stats::SysEvents + 'static> StatCollectorWorker<E> {
}
}
#[async_trait]
impl<E: stats::SysEvents + 'static> worker::SimpleWorker for StatCollectorWorker<E> {
type J = stats::EventMessage;
fn msg_to_job(&mut self, _: &str, _: &Option<String>, body: &[u8]) -> Result<Self::J, String> {
match serde_json::from_slice(body) {
Ok(e) => Ok(e),
Err(_) => {
let mut modified_body: Vec<u8> = vec![b"\""[0]];
modified_body.append(&mut body.to_vec());
modified_body.push(b"\""[0]);
async fn msg_to_job(
&mut self,
_: &str,
_: &Option<String>,
body: &[u8],
) -> Result<Self::J, String> {
if let Ok(e) = serde_json::from_slice(body) {
Ok(e)
} else {
let mut modified_body: Vec<u8> = vec![b"\""[0]];
modified_body.append(&mut body.to_vec());
modified_body.push(b"\""[0]);
match serde_json::from_slice(&modified_body) {
Ok(e) => {
self.events.notify(stats::Event::StatCollectorLegacyEvent(
match serde_json::from_slice(&modified_body) {
Ok(e) => {
self.events
.notify(stats::Event::StatCollectorLegacyEvent(
stats::event_metric_name(&e),
));
Ok(stats::EventMessage {
sender: "".to_owned(),
events: vec![e],
})
}
Err(e) => {
self.events.notify(stats::Event::StatCollectorBogusEvent);
error!(
"Failed to decode message: {:?}, Err: {:?}",
String::from_utf8(body.to_vec()),
e
);
Err("Failed to decode message".to_owned())
}
))
.await;
Ok(stats::EventMessage {
sender: String::new(),
events: vec![e],
})
}
Err(e) => {
self.events
.notify(stats::Event::StatCollectorBogusEvent)
.await;
error!(
"Failed to decode message: {:?}, Err: {:?}",
String::from_utf8(body.to_vec()),
e
);
Err("Failed to decode message".to_owned())
}
}
}
}
fn consumer(
async fn consumer(
&mut self,
_chan: &mut lapin::Channel,
job: &stats::EventMessage,
) -> worker::Actions {
let sender = job.sender.clone();
for event in job.events.iter() {
for event in &job.events {
self.collector.record(sender.clone(), event.clone());
}

7
ofborg/src/tasks/status_check_collector.rs
View file

@ -1,5 +1,6 @@
use std::path::PathBuf;
use async_trait::async_trait;
use tracing::{debug_span, error};
use crate::worker;
@ -16,16 +17,18 @@ pub struct StatusCheckCollector {
}
impl StatusCheckCollector {
#[must_use]
pub fn new(db_path: PathBuf) -> Self {
Self { db_path }
}
}
// RPC API worker
#[async_trait]
impl worker::SimpleWorker for StatusCheckCollector {
type J = StatusCheckRPCMessage;
fn msg_to_job(
async fn msg_to_job(
&mut self,
_method: &str,
_headers: &Option<String>,
@ -43,7 +46,7 @@ impl worker::SimpleWorker for StatusCheckCollector {
}
}
fn consumer(&mut self, _chan: &mut lapin::Channel, _job: &Self::J) -> worker::Actions {
async fn consumer(&mut self, _chan: &mut lapin::Channel, _job: &Self::J) -> worker::Actions {
let span = debug_span!("command");
let _enter = span.enter();

8
ofborg/src/test_scratch.rs
View file

@ -9,12 +9,13 @@ pub struct TestScratch {
}
impl TestScratch {
#[must_use]
pub fn new_dir(ident: &str) -> TestScratch {
let scratch = TestScratch {
root: Path::new(env!("CARGO_MANIFEST_DIR"))
.join("test-scratch")
.join("dirs")
.join(format!("dir-{}", ident)),
.join(format!("dir-{ident}")),
};
TestScratch::create_dir(&scratch);
@ -22,12 +23,13 @@ impl TestScratch {
scratch
}
#[must_use]
pub fn new_file(ident: &str) -> TestScratch {
let scratch = TestScratch {
root: Path::new(env!("CARGO_MANIFEST_DIR"))
.join("test-scratch")
.join("files")
.join(format!("file-{}", ident)),
.join(format!("file-{ident}")),
};
TestScratch::create_dir(&scratch);
@ -40,10 +42,12 @@ impl TestScratch {
fs::create_dir_all(target).unwrap();
}
#[must_use]
pub fn path(&self) -> PathBuf {
self.root.clone()
}
#[must_use]
pub fn string(&self) -> String {
self.path().to_str().unwrap().to_owned()
}

35
ofborg/src/vcs/commit_status.rs
View file

@ -1,6 +1,5 @@
use std::rc::Rc;
use std::sync::Arc;
use futures::executor::block_on;
use tracing::warn;
use crate::vcs::generic::State;
@ -8,7 +7,7 @@ use crate::vcs::generic::State;
use super::generic::VersionControlSystemAPI;
pub struct CommitStatus {
api: Rc<dyn VersionControlSystemAPI>,
api: Arc<dyn VersionControlSystemAPI>,
repo: crate::message::Repo,
sha: String,
context: String,
@ -18,7 +17,7 @@ pub struct CommitStatus {
impl CommitStatus {
pub fn new(
api: Rc<dyn VersionControlSystemAPI>,
api: Arc<dyn VersionControlSystemAPI>,
repo: crate::message::Repo,
sha: String,
context: String,
@ -31,7 +30,7 @@ impl CommitStatus {
sha,
context,
description,
url: "".to_owned(),
url: String::new(),
};
stat.set_url(url);
@ -40,23 +39,23 @@ impl CommitStatus {
}
pub fn set_url(&mut self, url: Option<String>) {
self.url = url.unwrap_or_else(|| String::from(""))
self.url = url.unwrap_or_default();
}
pub fn set_with_description(
pub async fn set_with_description(
&mut self,
description: &str,
state: State,
) -> Result<(), CommitStatusError> {
self.set_description(description.to_owned());
self.set(state)
self.set(state).await
}
pub fn set_description(&mut self, description: String) {
self.description = description;
}
pub fn set(&self, state: State) -> Result<(), CommitStatusError> {
pub async fn set(&self, state: State) -> Result<(), CommitStatusError> {
let desc = if self.description.len() >= 140 {
warn!(
"description is over 140 char; truncating: {:?}",
@ -67,14 +66,16 @@ impl CommitStatus {
self.description.clone()
};
block_on(self.api.create_commit_statuses(
&self.repo,
self.sha.clone(),
state,
self.context.clone(),
desc,
self.url.clone(),
))
self.api
.create_commit_statuses(
&self.repo,
self.sha.clone(),
state,
self.context.clone(),
desc,
self.url.clone(),
)
.await
}
}

3
ofborg/src/vcs/generic.rs
View file

@ -34,12 +34,13 @@ pub struct ChangeReviewers {
}
impl Issue {
#[must_use]
pub fn is_wip(&self) -> bool {
false
}
}
pub trait VersionControlSystemAPI {
pub trait VersionControlSystemAPI: Sync + Send {
fn get_repository(&self, repo: &crate::message::Repo) -> Repository;
fn get_changes(&self, repo: &crate::message::Repo) -> BoxFuture<Vec<Change>>;
fn get_change(&self, repo: &crate::message::Repo, number: u64) -> BoxFuture<Option<Change>>;

2
ofborg/src/vcs/gerrit/checks.rs
View file

@ -1,7 +1,7 @@
use crate::vcs::generic::CheckRunState;
use serde::{Deserialize, Serialize};
/// Port from https://gerrit.googlesource.com/gerrit/+/master/polygerrit-ui/app/api/checks.ts
/// Port from <https://gerrit.googlesource.com/gerrit/+/master/polygerrit-ui/app/api/checks.ts>
#[derive(Debug, Serialize, Deserialize, PartialEq)]
#[serde(rename_all = "UPPERCASE")]

11
ofborg/src/vcs/gerrit/http.rs
View file

@ -7,22 +7,23 @@ use super::data_structures::{Account, Change};
pub struct GerritHTTPApi;
#[allow(clippy::unused_async)] // FIXME
impl GerritHTTPApi {
// async fn get_project(&self, project_name: &str) -> Project {}
/// Fetches all changes according to the query and the given limit.
/// This will default to 60 changes by default.
pub(crate) async fn list_changes(&self, _query: &str, _limit: Option<u64>) -> Vec<Change> {
Default::default()
vec![]
}
/// Fetch the latest change ID for a given project and CL number.
pub(crate) async fn get_change_id(&self, _project_name: &str, _cl_number: u64) -> String {
"".to_owned()
String::new()
}
/// Fetch a given change according to the change ID (not the CL number).
pub(crate) async fn get_change(&self, _change_id: &str) -> Option<Change> {
Default::default()
None
}
/// Set additional and remove certain hashtags for a given change ID (not the CL number).
@ -32,11 +33,11 @@ impl GerritHTTPApi {
_add: &[String],
_remove: &[String],
) -> Vec<String> {
Default::default()
vec![]
}
/// List all reviewers on a given change ID (not the CL number).
pub(crate) async fn list_reviewers(&self, _change_id: &str) -> Vec<Account> {
Default::default()
vec![]
}
/// Set reviewers and a message on a given change ID (not the CL number).
pub(crate) async fn set_reviewers(

16
ofborg/src/vcs/gerrit/impl.rs
View file

@ -50,12 +50,12 @@ impl VersionControlSystemAPI for GerritHTTPApi {
&self,
repo: &crate::message::Repo,
) -> futures_util::future::BoxFuture<Vec<crate::message::Change>> {
let repo_name = repo.name.to_owned();
let repo_name = repo.name.clone();
async move {
self.list_changes(&format!("project:{}", &repo_name), None)
.await
.into_iter()
.map(|c| c.into())
.map(std::convert::Into::into)
.collect()
}
.boxed()
@ -66,12 +66,12 @@ impl VersionControlSystemAPI for GerritHTTPApi {
repo: &crate::message::Repo,
number: u64,
) -> futures_util::future::BoxFuture<Option<crate::message::Change>> {
let repo_name = repo.name.to_owned();
let repo_name = repo.name.clone();
async move {
let change_id = self.get_change_id(&repo_name, number).await;
GerritHTTPApi::get_change(self, &change_id)
.await
.map(|c| c.into())
.map(std::convert::Into::into)
}
.boxed()
}
@ -85,7 +85,7 @@ impl VersionControlSystemAPI for GerritHTTPApi {
) -> futures_util::future::BoxFuture<()> {
let add = add.to_owned();
let remove = remove.to_owned();
let repo_name = repo.name.to_owned();
let repo_name = repo.name.clone();
async move {
let change_id = self.get_change_id(&repo_name, number).await;
@ -99,7 +99,7 @@ impl VersionControlSystemAPI for GerritHTTPApi {
repo: &crate::message::Repo,
number: u64,
) -> futures_util::future::BoxFuture<crate::vcs::generic::ChangeReviewers> {
let repo_name = repo.name.to_owned();
let repo_name = repo.name.clone();
async move {
let change_id = self.get_change_id(&repo_name, number).await;
self.list_reviewers(&change_id).await.into()
@ -115,7 +115,7 @@ impl VersionControlSystemAPI for GerritHTTPApi {
// FIXME: support group reviews
_team_reviewers: Vec<String>,
) -> futures_util::future::BoxFuture<()> {
let repo_name = repo.name.to_owned();
let repo_name = repo.name.clone();
async move {
let change_id = self.get_change_id(&repo_name, number).await;
self.set_reviewers(
@ -130,7 +130,7 @@ impl VersionControlSystemAPI for GerritHTTPApi {
})
.collect(),
)
.await
.await;
}
.boxed()
}

9
ofborg/src/vcs/gerrit/ssh.rs
View file

@ -1,7 +1,7 @@
use std::{error::Error, path::PathBuf};
use async_stream::stream;
use futures::{executor::block_on, StreamExt};
use futures::StreamExt;
use futures_util::Stream;
use openssh::{Session, SessionBuilder, Stdio};
use tokio::io::AsyncBufReadExt;
@ -13,10 +13,13 @@ pub struct GerritSSHApi {
}
impl GerritSSHApi {
pub fn new(private_key_file: PathBuf, uri: &str) -> Self {
pub async fn new(private_key_file: PathBuf, uri: &str) -> Self {
let mut builder = SessionBuilder::default();
let (builder, destination) = builder.keyfile(&private_key_file).resolve(uri);
let tempdir = block_on(builder.launch_master(destination)).unwrap_or_else(|_| panic!("Failed to launch SSH master to destination '{}'", uri));
let tempdir = builder
.launch_master(destination)
.await
.unwrap_or_else(|_| panic!("Failed to launch SSH master to destination '{}'", uri));
Self {
session: Session::new_process_mux(tempdir),
}

15
ofborg/src/worker.rs
View file

@ -1,3 +1,4 @@
use async_trait::async_trait;
use std::marker::Send;
use futures::Stream;
@ -34,8 +35,8 @@ where
T: Serialize + ?Sized,
{
QueueMsg {
exchange: exchange.map(|s| s.to_owned()),
routing_key: routing_key.map(|s| s.to_owned()),
exchange: exchange.map(std::borrow::ToOwned::to_owned),
routing_key: routing_key.map(std::borrow::ToOwned::to_owned),
mandatory: false,
immediate: false,
content_type: Some("application/json".to_owned()),
@ -44,8 +45,8 @@ where
}
pub fn publish_serde_action<T>(
exchange: Option<String>,
routing_key: Option<String>,
exchange: &Option<String>,
routing_key: &Option<String>,
msg: &T,
) -> Action
where
@ -58,11 +59,13 @@ where
)))
}
#[async_trait]
pub trait SimpleWorker: Send {
type J: Send;
fn consumer(&mut self, chan: &mut lapin::Channel, job: &Self::J) -> Actions;
fn msg_to_job(
async fn consumer(&mut self, chan: &mut lapin::Channel, job: &Self::J) -> Actions;
async fn msg_to_job(
&mut self,
method: &str,
headers: &Option<String>,

6
ofborg/src/writetoline.rs
View file

@ -8,6 +8,7 @@ pub struct LineWriter {
}
impl LineWriter {
#[must_use]
pub fn new(mut rw: File) -> LineWriter {
let buf = LineWriter::load_buffer(&mut rw);
let len = buf.len();
@ -27,7 +28,7 @@ impl LineWriter {
.lines()
.map(|line| match line {
Ok(s) => s,
Err(e) => format!("UTF-8 Decode err: {:?}", e),
Err(e) => format!("UTF-8 Decode err: {e:?}"),
})
.collect()
}
@ -35,7 +36,7 @@ impl LineWriter {
pub fn write_to_line(&mut self, line: usize, data: &str) {
let original_len = self.buffer.len();
while self.buffer.len() <= line {
self.buffer.push("".to_owned());
self.buffer.push(String::new());
}
self.buffer.remove(line);
@ -72,6 +73,7 @@ impl LineWriter {
self.last_line = line;
}
#[must_use]
pub fn inner(self) -> File {
self.file
}