Compare commits
2 commits
b561c15dcf
...
6348541982
Author | SHA1 | Date | |
---|---|---|---|
Ilya K | 6348541982 | ||
Ilya K | 43ec489263 |
31
Cargo.lock
generated
31
Cargo.lock
generated
|
@ -1833,6 +1833,7 @@ name = "ofborg"
|
||||||
version = "0.90.0"
|
version = "0.90.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-stream",
|
"async-stream",
|
||||||
|
"async-trait",
|
||||||
"axum",
|
"axum",
|
||||||
"base64",
|
"base64",
|
||||||
"brace-expand",
|
"brace-expand",
|
||||||
|
@ -1854,6 +1855,7 @@ dependencies = [
|
||||||
"opentelemetry",
|
"opentelemetry",
|
||||||
"opentelemetry-otlp",
|
"opentelemetry-otlp",
|
||||||
"opentelemetry-semantic-conventions",
|
"opentelemetry-semantic-conventions",
|
||||||
|
"opentelemetry-stdout",
|
||||||
"opentelemetry_sdk",
|
"opentelemetry_sdk",
|
||||||
"regex",
|
"regex",
|
||||||
"rustls-pemfile",
|
"rustls-pemfile",
|
||||||
|
@ -1880,6 +1882,7 @@ version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"log",
|
"log",
|
||||||
"ofborg",
|
"ofborg",
|
||||||
|
"tokio",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
@ -1986,6 +1989,23 @@ version = "0.27.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "bc1b6902ff63b32ef6c489e8048c5e253e2e4a803ea3ea7e783914536eb15c52"
|
checksum = "bc1b6902ff63b32ef6c489e8048c5e253e2e4a803ea3ea7e783914536eb15c52"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "opentelemetry-stdout"
|
||||||
|
version = "0.27.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "bc8a298402aa5c260be90d10dc54b5a7d4e1025c354848f8e2c976d761351049"
|
||||||
|
dependencies = [
|
||||||
|
"async-trait",
|
||||||
|
"chrono",
|
||||||
|
"futures-util",
|
||||||
|
"opentelemetry",
|
||||||
|
"opentelemetry_sdk",
|
||||||
|
"ordered-float",
|
||||||
|
"serde",
|
||||||
|
"serde_json",
|
||||||
|
"thiserror 1.0.69",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "opentelemetry_sdk"
|
name = "opentelemetry_sdk"
|
||||||
version = "0.27.0"
|
version = "0.27.0"
|
||||||
|
@ -2003,6 +2023,8 @@ dependencies = [
|
||||||
"rand",
|
"rand",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"thiserror 1.0.69",
|
"thiserror 1.0.69",
|
||||||
|
"tokio",
|
||||||
|
"tokio-stream",
|
||||||
"tracing",
|
"tracing",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
@ -2012,6 +2034,15 @@ version = "0.2.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"
|
checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "ordered-float"
|
||||||
|
version = "4.5.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "c65ee1f9701bf938026630b455d5315f490640234259037edb259798b3bcf85e"
|
||||||
|
dependencies = [
|
||||||
|
"num-traits",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "overload"
|
name = "overload"
|
||||||
version = "0.1.1"
|
version = "0.1.1"
|
||||||
|
|
|
@ -7,4 +7,5 @@ edition = "2018"
|
||||||
[dependencies]
|
[dependencies]
|
||||||
ofborg = { path = "../ofborg" }
|
ofborg = { path = "../ofborg" }
|
||||||
|
|
||||||
|
tokio = "*"
|
||||||
log = "0.4.22"
|
log = "0.4.22"
|
||||||
|
|
|
@ -8,7 +8,8 @@ use std::path::Path;
|
||||||
use ofborg::config;
|
use ofborg::config;
|
||||||
use ofborg::nix;
|
use ofborg::nix;
|
||||||
|
|
||||||
fn main() {
|
#[tokio::main]
|
||||||
|
async fn main() {
|
||||||
ofborg::setup_log();
|
ofborg::setup_log();
|
||||||
|
|
||||||
log::info!("Loading config...");
|
log::info!("Loading config...");
|
||||||
|
@ -16,11 +17,14 @@ fn main() {
|
||||||
let nix = cfg.nix();
|
let nix = cfg.nix();
|
||||||
|
|
||||||
log::info!("Running build...");
|
log::info!("Running build...");
|
||||||
match nix.safely_build_attrs(
|
match nix
|
||||||
Path::new("./"),
|
.safely_build_attrs(
|
||||||
nix::File::DefaultNixpkgs,
|
Path::new("./"),
|
||||||
vec![String::from("hello")],
|
nix::File::DefaultNixpkgs,
|
||||||
) {
|
vec![String::from("hello")],
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
Ok(mut out) => {
|
Ok(mut out) => {
|
||||||
print!("{}", file_to_str(&mut out));
|
print!("{}", file_to_str(&mut out));
|
||||||
}
|
}
|
||||||
|
|
|
@ -10,6 +10,7 @@ edition = "2018"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
async-stream = "0.3.6"
|
async-stream = "0.3.6"
|
||||||
|
async-trait = "0.1.83"
|
||||||
axum = "0.7.8"
|
axum = "0.7.8"
|
||||||
base64 = "0.22.1"
|
base64 = "0.22.1"
|
||||||
brace-expand = "0.1.0"
|
brace-expand = "0.1.0"
|
||||||
|
@ -26,12 +27,13 @@ jfs = "0.9.0"
|
||||||
lapin = "2.5.0"
|
lapin = "2.5.0"
|
||||||
lru-cache = "0.1.2"
|
lru-cache = "0.1.2"
|
||||||
md5 = "0.7.0"
|
md5 = "0.7.0"
|
||||||
nom = "4,<5"
|
nom = "4,<5" # FIXME: update
|
||||||
openssh = { version = "0.11.3", features = ["process-mux"], default-features = false }
|
openssh = { version = "0.11.3", features = ["process-mux"], default-features = false }
|
||||||
opentelemetry = "0.27.0"
|
opentelemetry = "0.27.0"
|
||||||
|
opentelemetry_sdk = { version = "0.27.0", features = ["rt-tokio"] }
|
||||||
|
opentelemetry-otlp = { version = "0.27.0", features = ["http-json", "reqwest-client", "reqwest-rustls"] }
|
||||||
opentelemetry-semantic-conventions = "0.27.0"
|
opentelemetry-semantic-conventions = "0.27.0"
|
||||||
opentelemetry_sdk = "0.27.0"
|
opentelemetry-stdout = "0.27.0"
|
||||||
opentelemetry-otlp = { version = "0.27.0", features = ["http-json", "reqwest-blocking-client", "reqwest-rustls"] }
|
|
||||||
regex = "1.11.1"
|
regex = "1.11.1"
|
||||||
rustls-pemfile = "2.2.0"
|
rustls-pemfile = "2.2.0"
|
||||||
separator = "0.4.1"
|
separator = "0.4.1"
|
||||||
|
|
|
@ -6,6 +6,7 @@ pub struct Acl {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Acl {
|
impl Acl {
|
||||||
|
#[must_use]
|
||||||
pub fn new(repos: Vec<String>, mut trusted_users: Option<Vec<String>>) -> Acl {
|
pub fn new(repos: Vec<String>, mut trusted_users: Option<Vec<String>>) -> Acl {
|
||||||
if let Some(ref mut users) = trusted_users {
|
if let Some(ref mut users) = trusted_users {
|
||||||
users.iter_mut().map(|x| *x = x.to_lowercase()).last();
|
users.iter_mut().map(|x| *x = x.to_lowercase()).last();
|
||||||
|
@ -17,10 +18,12 @@ impl Acl {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub fn is_repo_eligible(&self, name: &str) -> bool {
|
pub fn is_repo_eligible(&self, name: &str) -> bool {
|
||||||
self.repos.contains(&name.to_lowercase())
|
self.repos.contains(&name.to_lowercase())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub fn build_job_architectures_for_user_repo(&self, user: &str, repo: &str) -> Vec<System> {
|
pub fn build_job_architectures_for_user_repo(&self, user: &str, repo: &str) -> Vec<System> {
|
||||||
if self.can_build_unrestricted(user, repo) {
|
if self.can_build_unrestricted(user, repo) {
|
||||||
vec![
|
vec![
|
||||||
|
@ -35,6 +38,7 @@ impl Acl {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub fn build_job_destinations_for_user_repo(
|
pub fn build_job_destinations_for_user_repo(
|
||||||
&self,
|
&self,
|
||||||
user: &str,
|
user: &str,
|
||||||
|
@ -42,10 +46,11 @@ impl Acl {
|
||||||
) -> Vec<(Option<String>, Option<String>)> {
|
) -> Vec<(Option<String>, Option<String>)> {
|
||||||
self.build_job_architectures_for_user_repo(user, repo)
|
self.build_job_architectures_for_user_repo(user, repo)
|
||||||
.iter()
|
.iter()
|
||||||
.map(|system| system.as_build_destination())
|
.map(super::systems::System::as_build_destination)
|
||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub fn can_build_unrestricted(&self, user: &str, repo: &str) -> bool {
|
pub fn can_build_unrestricted(&self, user: &str, repo: &str) -> bool {
|
||||||
if let Some(ref users) = self.trusted_users {
|
if let Some(ref users) = self.trusted_users {
|
||||||
if repo.to_lowercase() == "nixos/nixpkgs" {
|
if repo.to_lowercase() == "nixos/nixpkgs" {
|
||||||
|
|
|
@ -82,10 +82,12 @@ fn child_wait<T: Send + 'static>(
|
||||||
}
|
}
|
||||||
|
|
||||||
impl AsyncCmd {
|
impl AsyncCmd {
|
||||||
|
#[must_use]
|
||||||
pub fn new(cmd: Command) -> AsyncCmd {
|
pub fn new(cmd: Command) -> AsyncCmd {
|
||||||
AsyncCmd { command: cmd }
|
AsyncCmd { command: cmd }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub fn spawn(mut self) -> SpawnedAsyncCmd {
|
pub fn spawn(mut self) -> SpawnedAsyncCmd {
|
||||||
let mut child = self
|
let mut child = self
|
||||||
.command
|
.command
|
||||||
|
@ -122,7 +124,7 @@ impl AsyncCmd {
|
||||||
child_wait(WaitTarget::Child, monitor_tx, child),
|
child_wait(WaitTarget::Child, monitor_tx, child),
|
||||||
);
|
);
|
||||||
|
|
||||||
let head_waiter = thread::spawn(move || block_on_waiters(monitor_rx, waiters));
|
let head_waiter = thread::spawn(move || block_on_waiters(&monitor_rx, waiters));
|
||||||
|
|
||||||
SpawnedAsyncCmd {
|
SpawnedAsyncCmd {
|
||||||
waiter: head_waiter,
|
waiter: head_waiter,
|
||||||
|
@ -152,12 +154,12 @@ impl SpawnedAsyncCmd {
|
||||||
// FIXME: remove with rust/cargo update
|
// FIXME: remove with rust/cargo update
|
||||||
#[allow(clippy::cognitive_complexity)]
|
#[allow(clippy::cognitive_complexity)]
|
||||||
fn block_on_waiters(
|
fn block_on_waiters(
|
||||||
monitor_rx: mpsc::Receiver<(WaitTarget, WaitResult<()>)>,
|
monitor_rx: &mpsc::Receiver<(WaitTarget, WaitResult<()>)>,
|
||||||
mut waiters: HashMap<WaitTarget, thread::JoinHandle<()>>,
|
mut waiters: HashMap<WaitTarget, thread::JoinHandle<()>>,
|
||||||
) -> Option<Result<ExitStatus, io::Error>> {
|
) -> Option<Result<ExitStatus, io::Error>> {
|
||||||
let mut status = None;
|
let mut status = None;
|
||||||
|
|
||||||
for (id, interior_result) in monitor_rx.iter() {
|
for (id, interior_result) in monitor_rx {
|
||||||
match waiters.remove(&id) {
|
match waiters.remove(&id) {
|
||||||
Some(handle) => {
|
Some(handle) => {
|
||||||
info!("Received notice that {:?} finished", id);
|
info!("Received notice that {:?} finished", id);
|
||||||
|
@ -262,7 +264,7 @@ mod tests {
|
||||||
|
|
||||||
let mut spawned = acmd.spawn();
|
let mut spawned = acmd.spawn();
|
||||||
let lines: Vec<String> = spawned.lines().collect();
|
let lines: Vec<String> = spawned.lines().collect();
|
||||||
assert_eq!(lines.len(), 200000);
|
assert_eq!(lines.len(), 200_000);
|
||||||
let thread_result = spawned.wait();
|
let thread_result = spawned.wait();
|
||||||
let exit_status = thread_result.expect("Thread should exit correctly");
|
let exit_status = thread_result.expect("Thread should exit correctly");
|
||||||
assert!(exit_status.success());
|
assert!(exit_status.success());
|
||||||
|
|
|
@ -1,7 +1,6 @@
|
||||||
use std::env;
|
use std::env;
|
||||||
use std::error::Error;
|
use std::error::Error;
|
||||||
|
|
||||||
use futures::executor::block_on;
|
|
||||||
use lapin::message::Delivery;
|
use lapin::message::Delivery;
|
||||||
use lapin::BasicProperties;
|
use lapin::BasicProperties;
|
||||||
|
|
||||||
|
@ -12,14 +11,15 @@ use ofborg::message::{buildjob, Change, Repo};
|
||||||
use ofborg::notifyworker::NotificationReceiver;
|
use ofborg::notifyworker::NotificationReceiver;
|
||||||
use ofborg::worker;
|
use ofborg::worker;
|
||||||
|
|
||||||
fn main() -> Result<(), Box<dyn Error>> {
|
#[tokio::main]
|
||||||
|
async fn main() -> Result<(), Box<dyn Error>> {
|
||||||
ofborg::setup_log();
|
ofborg::setup_log();
|
||||||
|
|
||||||
let arg = env::args().nth(1).expect("usage: build-faker <config>");
|
let arg = env::args().nth(1).expect("usage: build-faker <config>");
|
||||||
let cfg = config::load(arg.as_ref());
|
let cfg = config::load(arg.as_ref());
|
||||||
|
|
||||||
let conn = easylapin::from_config(&cfg.rabbitmq)?;
|
let conn = easylapin::from_config(&cfg.rabbitmq).await?;
|
||||||
let mut chan = block_on(conn.create_channel())?;
|
let mut chan = conn.create_channel().await?;
|
||||||
|
|
||||||
let repo_msg = Repo {
|
let repo_msg = Repo {
|
||||||
clone_url: "https://github.com/nixos/ofborg.git".to_owned(),
|
clone_url: "https://github.com/nixos/ofborg.git".to_owned(),
|
||||||
|
@ -60,10 +60,11 @@ fn main() -> Result<(), Box<dyn Error>> {
|
||||||
|
|
||||||
for _i in 1..2 {
|
for _i in 1..2 {
|
||||||
recv.tell(worker::publish_serde_action(
|
recv.tell(worker::publish_serde_action(
|
||||||
None,
|
&None,
|
||||||
Some("build-inputs-x86_64-darwin".to_owned()),
|
&Some("build-inputs-x86_64-darwin".to_owned()),
|
||||||
&msg,
|
&msg,
|
||||||
));
|
))
|
||||||
|
.await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -2,9 +2,7 @@ use std::env;
|
||||||
use std::error::Error;
|
use std::error::Error;
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
|
|
||||||
use futures::executor::block_on;
|
|
||||||
use futures_util::future;
|
use futures_util::future;
|
||||||
use tokio::spawn;
|
|
||||||
use tokio::task::JoinHandle;
|
use tokio::task::JoinHandle;
|
||||||
use tracing::{info, warn};
|
use tracing::{info, warn};
|
||||||
|
|
||||||
|
@ -14,7 +12,8 @@ use ofborg::{checkout, config, tasks};
|
||||||
|
|
||||||
// FIXME: remove with rust/cargo update
|
// FIXME: remove with rust/cargo update
|
||||||
#[allow(clippy::cognitive_complexity)]
|
#[allow(clippy::cognitive_complexity)]
|
||||||
fn main() -> Result<(), Box<dyn Error>> {
|
#[tokio::main]
|
||||||
|
async fn main() -> Result<(), Box<dyn Error>> {
|
||||||
ofborg::setup_log();
|
ofborg::setup_log();
|
||||||
|
|
||||||
let arg = env::args().nth(1).expect("usage: builder <config>");
|
let arg = env::args().nth(1).expect("usage: builder <config>");
|
||||||
|
@ -30,27 +29,27 @@ fn main() -> Result<(), Box<dyn Error>> {
|
||||||
panic!();
|
panic!();
|
||||||
};
|
};
|
||||||
|
|
||||||
let conn = easylapin::from_config(&cfg.rabbitmq)?;
|
let conn = easylapin::from_config(&cfg.rabbitmq).await?;
|
||||||
let mut handles = Vec::new();
|
let mut handles = Vec::new();
|
||||||
|
|
||||||
for system in &cfg.nix.system {
|
for system in &cfg.nix.system {
|
||||||
let handle_ext = self::create_handle(&conn, &cfg, system.to_string())?;
|
let handle_ext = self::create_handle(&conn, &cfg, system.to_string());
|
||||||
handles.push(handle_ext);
|
handles.push(handle_ext);
|
||||||
}
|
}
|
||||||
|
|
||||||
block_on(future::join_all(handles));
|
future::join_all(handles).await;
|
||||||
|
|
||||||
drop(conn); // Close connection.
|
drop(conn); // Close connection.
|
||||||
info!("Closed the session... EOF");
|
info!("Closed the session... EOF");
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn create_handle(
|
async fn create_handle(
|
||||||
conn: &lapin::Connection,
|
conn: &lapin::Connection,
|
||||||
cfg: &config::Config,
|
cfg: &config::Config,
|
||||||
system: String,
|
system: String,
|
||||||
) -> Result<JoinHandle<()>, Box<dyn Error>> {
|
) -> Result<JoinHandle<Result<(), lapin::Error>>, Box<dyn Error>> {
|
||||||
let mut chan = block_on(conn.create_channel())?;
|
let mut chan = conn.create_channel().await?;
|
||||||
|
|
||||||
let cloner = checkout::cached_cloner(Path::new(&cfg.checkout.root));
|
let cloner = checkout::cached_cloner(Path::new(&cfg.checkout.root));
|
||||||
let nix = cfg.nix().with_system(system.clone());
|
let nix = cfg.nix().with_system(system.clone());
|
||||||
|
@ -63,7 +62,8 @@ fn create_handle(
|
||||||
auto_delete: false,
|
auto_delete: false,
|
||||||
no_wait: false,
|
no_wait: false,
|
||||||
internal: false,
|
internal: false,
|
||||||
})?;
|
})
|
||||||
|
.await?;
|
||||||
|
|
||||||
let queue_name = if cfg.runner.build_all_jobs != Some(true) {
|
let queue_name = if cfg.runner.build_all_jobs != Some(true) {
|
||||||
let queue_name = format!("build-inputs-{}", system);
|
let queue_name = format!("build-inputs-{}", system);
|
||||||
|
@ -74,7 +74,8 @@ fn create_handle(
|
||||||
exclusive: false,
|
exclusive: false,
|
||||||
auto_delete: false,
|
auto_delete: false,
|
||||||
no_wait: false,
|
no_wait: false,
|
||||||
})?;
|
})
|
||||||
|
.await?;
|
||||||
queue_name
|
queue_name
|
||||||
} else {
|
} else {
|
||||||
warn!("Building all jobs, please don't use this unless you're");
|
warn!("Building all jobs, please don't use this unless you're");
|
||||||
|
@ -87,7 +88,8 @@ fn create_handle(
|
||||||
exclusive: true,
|
exclusive: true,
|
||||||
auto_delete: true,
|
auto_delete: true,
|
||||||
no_wait: false,
|
no_wait: false,
|
||||||
})?;
|
})
|
||||||
|
.await?;
|
||||||
queue_name
|
queue_name
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -96,9 +98,12 @@ fn create_handle(
|
||||||
exchange: "build-jobs".to_owned(),
|
exchange: "build-jobs".to_owned(),
|
||||||
routing_key: None,
|
routing_key: None,
|
||||||
no_wait: false,
|
no_wait: false,
|
||||||
})?;
|
})
|
||||||
|
.await?;
|
||||||
|
|
||||||
let handle = easylapin::NotifyChannel(chan).consume(
|
info!("Fetching jobs from {}", &queue_name);
|
||||||
|
|
||||||
|
let fut = easylapin::NotifyChannel(chan).consume(
|
||||||
tasks::build::BuildWorker::new(cloner, nix, system, cfg.runner.identity.clone()),
|
tasks::build::BuildWorker::new(cloner, nix, system, cfg.runner.identity.clone()),
|
||||||
easyamqp::ConsumeConfig {
|
easyamqp::ConsumeConfig {
|
||||||
queue: queue_name.clone(),
|
queue: queue_name.clone(),
|
||||||
|
@ -108,8 +113,7 @@ fn create_handle(
|
||||||
no_wait: false,
|
no_wait: false,
|
||||||
exclusive: false,
|
exclusive: false,
|
||||||
},
|
},
|
||||||
)?;
|
);
|
||||||
|
|
||||||
info!("Fetching jobs from {}", &queue_name);
|
Ok(tokio::spawn(fut))
|
||||||
Ok(spawn(handle))
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,7 +1,6 @@
|
||||||
use std::env;
|
use std::env;
|
||||||
use std::error::Error;
|
use std::error::Error;
|
||||||
|
|
||||||
use futures::executor::block_on;
|
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
|
|
||||||
use ofborg::config;
|
use ofborg::config;
|
||||||
|
@ -9,7 +8,8 @@ use ofborg::easyamqp::{self, ChannelExt, ConsumerExt};
|
||||||
use ofborg::easylapin;
|
use ofborg::easylapin;
|
||||||
use ofborg::tasks;
|
use ofborg::tasks;
|
||||||
|
|
||||||
fn main() -> Result<(), Box<dyn Error>> {
|
#[tokio::main]
|
||||||
|
async fn main() -> Result<(), Box<dyn Error>> {
|
||||||
ofborg::setup_log();
|
ofborg::setup_log();
|
||||||
|
|
||||||
let arg = env::args()
|
let arg = env::args()
|
||||||
|
@ -17,8 +17,8 @@ fn main() -> Result<(), Box<dyn Error>> {
|
||||||
.expect("usage: evaluation-filter <config>");
|
.expect("usage: evaluation-filter <config>");
|
||||||
let cfg = config::load(arg.as_ref());
|
let cfg = config::load(arg.as_ref());
|
||||||
|
|
||||||
let conn = easylapin::from_config(&cfg.rabbitmq)?;
|
let conn = easylapin::from_config(&cfg.rabbitmq).await?;
|
||||||
let mut chan = block_on(conn.create_channel())?;
|
let mut chan = conn.create_channel().await?;
|
||||||
|
|
||||||
chan.declare_exchange(easyamqp::ExchangeConfig {
|
chan.declare_exchange(easyamqp::ExchangeConfig {
|
||||||
exchange: "github-events".to_owned(),
|
exchange: "github-events".to_owned(),
|
||||||
|
@ -28,7 +28,8 @@ fn main() -> Result<(), Box<dyn Error>> {
|
||||||
auto_delete: false,
|
auto_delete: false,
|
||||||
no_wait: false,
|
no_wait: false,
|
||||||
internal: false,
|
internal: false,
|
||||||
})?;
|
})
|
||||||
|
.await?;
|
||||||
|
|
||||||
chan.declare_queue(easyamqp::QueueConfig {
|
chan.declare_queue(easyamqp::QueueConfig {
|
||||||
queue: "mass-rebuild-check-jobs".to_owned(),
|
queue: "mass-rebuild-check-jobs".to_owned(),
|
||||||
|
@ -37,7 +38,8 @@ fn main() -> Result<(), Box<dyn Error>> {
|
||||||
exclusive: false,
|
exclusive: false,
|
||||||
auto_delete: false,
|
auto_delete: false,
|
||||||
no_wait: false,
|
no_wait: false,
|
||||||
})?;
|
})
|
||||||
|
.await?;
|
||||||
|
|
||||||
let queue_name = String::from("mass-rebuild-check-inputs");
|
let queue_name = String::from("mass-rebuild-check-inputs");
|
||||||
chan.declare_queue(easyamqp::QueueConfig {
|
chan.declare_queue(easyamqp::QueueConfig {
|
||||||
|
@ -47,29 +49,31 @@ fn main() -> Result<(), Box<dyn Error>> {
|
||||||
exclusive: false,
|
exclusive: false,
|
||||||
auto_delete: false,
|
auto_delete: false,
|
||||||
no_wait: false,
|
no_wait: false,
|
||||||
})?;
|
})
|
||||||
|
.await?;
|
||||||
|
|
||||||
chan.bind_queue(easyamqp::BindQueueConfig {
|
chan.bind_queue(easyamqp::BindQueueConfig {
|
||||||
queue: queue_name.clone(),
|
queue: queue_name.clone(),
|
||||||
exchange: "github-events".to_owned(),
|
exchange: "github-events".to_owned(),
|
||||||
routing_key: Some("pull_request.nixos/nixpkgs".to_owned()),
|
routing_key: Some("pull_request.nixos/nixpkgs".to_owned()),
|
||||||
no_wait: false,
|
no_wait: false,
|
||||||
})?;
|
})
|
||||||
|
.await?;
|
||||||
let handle = easylapin::WorkerChannel(chan).consume(
|
|
||||||
tasks::evaluationfilter::EvaluationFilterWorker::new(cfg.acl()),
|
|
||||||
easyamqp::ConsumeConfig {
|
|
||||||
queue: queue_name.clone(),
|
|
||||||
consumer_tag: format!("{}-evaluation-filter", cfg.whoami()),
|
|
||||||
no_local: false,
|
|
||||||
no_ack: false,
|
|
||||||
no_wait: false,
|
|
||||||
exclusive: false,
|
|
||||||
},
|
|
||||||
)?;
|
|
||||||
|
|
||||||
info!("Fetching jobs from {}", &queue_name);
|
info!("Fetching jobs from {}", &queue_name);
|
||||||
block_on(handle);
|
easylapin::WorkerChannel(chan)
|
||||||
|
.consume(
|
||||||
|
tasks::evaluationfilter::EvaluationFilterWorker::new(cfg.acl()),
|
||||||
|
easyamqp::ConsumeConfig {
|
||||||
|
queue: queue_name.clone(),
|
||||||
|
consumer_tag: format!("{}-evaluation-filter", cfg.whoami()),
|
||||||
|
no_local: false,
|
||||||
|
no_ack: false,
|
||||||
|
no_wait: false,
|
||||||
|
exclusive: false,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
drop(conn); // Close connection.
|
drop(conn); // Close connection.
|
||||||
info!("Closed the session... EOF");
|
info!("Closed the session... EOF");
|
||||||
|
|
|
@ -4,7 +4,6 @@
|
||||||
use std::env;
|
use std::env;
|
||||||
use std::error::Error;
|
use std::error::Error;
|
||||||
|
|
||||||
use futures::executor::block_on;
|
|
||||||
use futures::{pin_mut, StreamExt};
|
use futures::{pin_mut, StreamExt};
|
||||||
use lapin::options::BasicPublishOptions;
|
use lapin::options::BasicPublishOptions;
|
||||||
use lapin::BasicProperties;
|
use lapin::BasicProperties;
|
||||||
|
@ -16,7 +15,8 @@ use ofborg::config;
|
||||||
use ofborg::easyamqp::{self, ChannelExt};
|
use ofborg::easyamqp::{self, ChannelExt};
|
||||||
use ofborg::easylapin;
|
use ofborg::easylapin;
|
||||||
|
|
||||||
fn main() -> Result<(), Box<dyn Error>> {
|
#[tokio::main]
|
||||||
|
async fn main() -> Result<(), Box<dyn Error>> {
|
||||||
ofborg::setup_log();
|
ofborg::setup_log();
|
||||||
|
|
||||||
let arg = env::args()
|
let arg = env::args()
|
||||||
|
@ -24,8 +24,8 @@ fn main() -> Result<(), Box<dyn Error>> {
|
||||||
.expect("usage: gerrit-events-streamer <config>");
|
.expect("usage: gerrit-events-streamer <config>");
|
||||||
let cfg = config::load(arg.as_ref());
|
let cfg = config::load(arg.as_ref());
|
||||||
|
|
||||||
let conn = easylapin::from_config(&cfg.rabbitmq)?;
|
let conn = easylapin::from_config(&cfg.rabbitmq).await?;
|
||||||
let mut chan = block_on(conn.create_channel())?;
|
let mut chan = conn.create_channel().await?;
|
||||||
|
|
||||||
let exchange_name = "gerrit-events";
|
let exchange_name = "gerrit-events";
|
||||||
|
|
||||||
|
@ -37,7 +37,8 @@ fn main() -> Result<(), Box<dyn Error>> {
|
||||||
auto_delete: false,
|
auto_delete: false,
|
||||||
no_wait: false,
|
no_wait: false,
|
||||||
internal: false,
|
internal: false,
|
||||||
})?;
|
})
|
||||||
|
.await?;
|
||||||
|
|
||||||
info!("Publishing events from Gerrit into {}", &exchange_name);
|
info!("Publishing events from Gerrit into {}", &exchange_name);
|
||||||
|
|
||||||
|
@ -47,57 +48,50 @@ fn main() -> Result<(), Box<dyn Error>> {
|
||||||
let mut gerrit_api = GerritSSHApi::new(
|
let mut gerrit_api = GerritSSHApi::new(
|
||||||
gerrit_cfg.ssh_private_key_file,
|
gerrit_cfg.ssh_private_key_file,
|
||||||
&format!("ssh://{}:{}", gerrit_cfg.instance_uri, gerrit_cfg.ssh_port),
|
&format!("ssh://{}:{}", gerrit_cfg.instance_uri, gerrit_cfg.ssh_port),
|
||||||
);
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
let routing_key = "abc";
|
let routing_key = "abc";
|
||||||
|
|
||||||
block_on(async {
|
let event_stream = gerrit_api.stream_events().await.unwrap();
|
||||||
let event_stream = gerrit_api.stream_events().await.unwrap();
|
pin_mut!(event_stream);
|
||||||
pin_mut!(event_stream);
|
loop {
|
||||||
loop {
|
let raw_evt = event_stream.next().await;
|
||||||
let raw_evt = event_stream.next().await;
|
tracing::debug!("{:?}", raw_evt);
|
||||||
tracing::debug!("{:?}", raw_evt);
|
|
||||||
|
|
||||||
match raw_evt {
|
match raw_evt {
|
||||||
Some(Ok(event)) => {
|
Some(Ok(event)) => {
|
||||||
println!("{:#?}", event);
|
println!("{:#?}", event);
|
||||||
let queue_message =
|
let queue_message =
|
||||||
prepare_queue_message(Some(exchange_name), Some(routing_key), &event);
|
prepare_queue_message(Some(exchange_name), Some(routing_key), &event);
|
||||||
let props = BasicProperties::default()
|
let props = BasicProperties::default()
|
||||||
.with_delivery_mode(2)
|
.with_delivery_mode(2)
|
||||||
.with_content_type("application/json".into());
|
.with_content_type("application/json".into());
|
||||||
|
|
||||||
match chan
|
match chan
|
||||||
.basic_publish(
|
.basic_publish(
|
||||||
exchange_name,
|
exchange_name,
|
||||||
routing_key,
|
routing_key,
|
||||||
BasicPublishOptions::default(),
|
BasicPublishOptions::default(),
|
||||||
&queue_message.content,
|
&queue_message.content,
|
||||||
props,
|
props,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
Ok(_confirmation) => {
|
Ok(_confirmation) => {
|
||||||
tracing::debug!("Gerrit event published in the exchange");
|
tracing::debug!("Gerrit event published in the exchange");
|
||||||
}
|
}
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
tracing::error!("Failed to publish gerrit event: {}", err);
|
tracing::error!("Failed to publish gerrit event: {}", err);
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Some(Err(_err)) => {
|
}
|
||||||
// notify the event
|
Some(Err(_err)) => {
|
||||||
}
|
// notify the event
|
||||||
None => {
|
}
|
||||||
// notify the event
|
None => {
|
||||||
}
|
// notify the event
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
});
|
}
|
||||||
|
|
||||||
//task::block_on(handle);
|
|
||||||
|
|
||||||
drop(conn); // Close connection.
|
|
||||||
info!("Closed the session... EOF");
|
|
||||||
Ok(())
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,13 +3,14 @@
|
||||||
use std::env;
|
use std::env;
|
||||||
use std::error::Error;
|
use std::error::Error;
|
||||||
|
|
||||||
use futures::{executor::block_on, pin_mut, StreamExt};
|
use futures::{pin_mut, StreamExt};
|
||||||
use ofborg::vcs::gerrit::ssh::GerritSSHApi;
|
use ofborg::vcs::gerrit::ssh::GerritSSHApi;
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
|
|
||||||
use ofborg::config;
|
use ofborg::config;
|
||||||
|
|
||||||
fn main() -> Result<(), Box<dyn Error>> {
|
#[tokio::main]
|
||||||
|
async fn main() -> Result<(), Box<dyn Error>> {
|
||||||
ofborg::setup_log();
|
ofborg::setup_log();
|
||||||
|
|
||||||
let arg = env::args()
|
let arg = env::args()
|
||||||
|
@ -22,19 +23,15 @@ fn main() -> Result<(), Box<dyn Error>> {
|
||||||
.expect("Gerrit event streaming requires a Gerrit configuration");
|
.expect("Gerrit event streaming requires a Gerrit configuration");
|
||||||
let gerrit_ssh_uri = format!("ssh://{}:{}", gerrit_cfg.instance_uri, gerrit_cfg.ssh_port);
|
let gerrit_ssh_uri = format!("ssh://{}:{}", gerrit_cfg.instance_uri, gerrit_cfg.ssh_port);
|
||||||
info!("Listening events from Gerrit on {}", gerrit_ssh_uri);
|
info!("Listening events from Gerrit on {}", gerrit_ssh_uri);
|
||||||
let mut gerrit_api = GerritSSHApi::new(gerrit_cfg.ssh_private_key_file, &gerrit_ssh_uri);
|
let mut gerrit_api = GerritSSHApi::new(gerrit_cfg.ssh_private_key_file, &gerrit_ssh_uri).await;
|
||||||
|
|
||||||
block_on(async {
|
let event_stream = gerrit_api.stream_events().await.unwrap();
|
||||||
let event_stream = gerrit_api.stream_events().await.unwrap();
|
pin_mut!(event_stream);
|
||||||
pin_mut!(event_stream);
|
loop {
|
||||||
loop {
|
let thing = event_stream.next().await;
|
||||||
let thing = event_stream.next().await;
|
println!("{:?}", thing);
|
||||||
println!("{:?}", thing);
|
if let Some(Ok(event)) = thing {
|
||||||
if let Some(Ok(event)) = thing {
|
println!("{:#?}", event);
|
||||||
println!("{:#?}", event);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
});
|
}
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,7 +2,6 @@ use std::env;
|
||||||
use std::error::Error;
|
use std::error::Error;
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
|
|
||||||
use futures::executor::block_on;
|
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
|
|
||||||
use ofborg::config;
|
use ofborg::config;
|
||||||
|
@ -10,7 +9,8 @@ use ofborg::easyamqp::{self, ChannelExt, ConsumerExt};
|
||||||
use ofborg::easylapin;
|
use ofborg::easylapin;
|
||||||
use ofborg::tasks;
|
use ofborg::tasks;
|
||||||
|
|
||||||
fn main() -> Result<(), Box<dyn Error>> {
|
#[tokio::main]
|
||||||
|
async fn main() -> Result<(), Box<dyn Error>> {
|
||||||
ofborg::setup_log();
|
ofborg::setup_log();
|
||||||
|
|
||||||
let arg = env::args()
|
let arg = env::args()
|
||||||
|
@ -18,8 +18,8 @@ fn main() -> Result<(), Box<dyn Error>> {
|
||||||
.expect("usage: log-message-collector <config>");
|
.expect("usage: log-message-collector <config>");
|
||||||
let cfg = config::load(arg.as_ref());
|
let cfg = config::load(arg.as_ref());
|
||||||
|
|
||||||
let conn = easylapin::from_config(&cfg.rabbitmq)?;
|
let conn = easylapin::from_config(&cfg.rabbitmq).await?;
|
||||||
let mut chan = block_on(conn.create_channel())?;
|
let mut chan = conn.create_channel().await?;
|
||||||
|
|
||||||
chan.declare_exchange(easyamqp::ExchangeConfig {
|
chan.declare_exchange(easyamqp::ExchangeConfig {
|
||||||
exchange: "logs".to_owned(),
|
exchange: "logs".to_owned(),
|
||||||
|
@ -29,7 +29,8 @@ fn main() -> Result<(), Box<dyn Error>> {
|
||||||
auto_delete: false,
|
auto_delete: false,
|
||||||
no_wait: false,
|
no_wait: false,
|
||||||
internal: false,
|
internal: false,
|
||||||
})?;
|
})
|
||||||
|
.await?;
|
||||||
|
|
||||||
let queue_name = "".to_owned();
|
let queue_name = "".to_owned();
|
||||||
chan.declare_queue(easyamqp::QueueConfig {
|
chan.declare_queue(easyamqp::QueueConfig {
|
||||||
|
@ -39,17 +40,21 @@ fn main() -> Result<(), Box<dyn Error>> {
|
||||||
exclusive: true,
|
exclusive: true,
|
||||||
auto_delete: true,
|
auto_delete: true,
|
||||||
no_wait: false,
|
no_wait: false,
|
||||||
})?;
|
})
|
||||||
|
.await?;
|
||||||
|
|
||||||
chan.bind_queue(easyamqp::BindQueueConfig {
|
chan.bind_queue(easyamqp::BindQueueConfig {
|
||||||
queue: queue_name.clone(),
|
queue: queue_name.clone(),
|
||||||
exchange: "logs".to_owned(),
|
exchange: "logs".to_owned(),
|
||||||
routing_key: Some("*.*".to_owned()),
|
routing_key: Some("*.*".to_owned()),
|
||||||
no_wait: false,
|
no_wait: false,
|
||||||
})?;
|
})
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
info!("Fetching jobs from {}", &queue_name);
|
||||||
|
|
||||||
// Regular channel, we want prefetching here.
|
// Regular channel, we want prefetching here.
|
||||||
let handle = chan.consume(
|
chan.consume(
|
||||||
tasks::log_message_collector::LogMessageCollector::new(
|
tasks::log_message_collector::LogMessageCollector::new(
|
||||||
PathBuf::from(cfg.log_storage.clone().unwrap().path),
|
PathBuf::from(cfg.log_storage.clone().unwrap().path),
|
||||||
100,
|
100,
|
||||||
|
@ -62,10 +67,8 @@ fn main() -> Result<(), Box<dyn Error>> {
|
||||||
no_wait: false,
|
no_wait: false,
|
||||||
exclusive: false,
|
exclusive: false,
|
||||||
},
|
},
|
||||||
)?;
|
)
|
||||||
|
.await?;
|
||||||
info!("Fetching jobs from {}", &queue_name);
|
|
||||||
block_on(handle);
|
|
||||||
|
|
||||||
drop(conn); // Close connection.
|
drop(conn); // Close connection.
|
||||||
info!("Closed the session... EOF");
|
info!("Closed the session... EOF");
|
||||||
|
|
|
@ -3,7 +3,6 @@ use std::error::Error;
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use std::process;
|
use std::process;
|
||||||
|
|
||||||
use futures::executor::block_on;
|
|
||||||
use ofborg::config::VCSConfig;
|
use ofborg::config::VCSConfig;
|
||||||
use ofborg::tasks::evaluate::SupportedVCS;
|
use ofborg::tasks::evaluate::SupportedVCS;
|
||||||
use tracing::{error, info};
|
use tracing::{error, info};
|
||||||
|
@ -17,7 +16,8 @@ use ofborg::tasks;
|
||||||
|
|
||||||
// FIXME: remove with rust/cargo update
|
// FIXME: remove with rust/cargo update
|
||||||
#[allow(clippy::cognitive_complexity)]
|
#[allow(clippy::cognitive_complexity)]
|
||||||
fn main() -> Result<(), Box<dyn Error>> {
|
#[tokio::main]
|
||||||
|
async fn main() -> Result<(), Box<dyn Error>> {
|
||||||
ofborg::setup_log();
|
ofborg::setup_log();
|
||||||
|
|
||||||
let arg = env::args().nth(1).expect("usage: mass-rebuilder <config>");
|
let arg = env::args().nth(1).expect("usage: mass-rebuilder <config>");
|
||||||
|
@ -34,14 +34,14 @@ fn main() -> Result<(), Box<dyn Error>> {
|
||||||
process::exit(1);
|
process::exit(1);
|
||||||
};
|
};
|
||||||
|
|
||||||
let conn = easylapin::from_config(&cfg.rabbitmq)?;
|
let conn = easylapin::from_config(&cfg.rabbitmq).await?;
|
||||||
let mut chan = block_on(conn.create_channel())?;
|
let mut chan = conn.create_channel().await?;
|
||||||
|
|
||||||
let root = Path::new(&cfg.checkout.root);
|
let root = Path::new(&cfg.checkout.root);
|
||||||
let cloner = checkout::cached_cloner(&root.join(cfg.runner.instance.to_string()));
|
let cloner = checkout::cached_cloner(&root.join(cfg.runner.instance.to_string()));
|
||||||
let nix = cfg.nix();
|
let nix = cfg.nix();
|
||||||
|
|
||||||
let events = stats::RabbitMq::from_lapin(&cfg.whoami(), block_on(conn.create_channel())?);
|
let events = stats::RabbitMq::from_lapin(&cfg.whoami(), conn.create_channel().await?);
|
||||||
|
|
||||||
let queue_name = String::from("mass-rebuild-check-jobs");
|
let queue_name = String::from("mass-rebuild-check-jobs");
|
||||||
chan.declare_queue(easyamqp::QueueConfig {
|
chan.declare_queue(easyamqp::QueueConfig {
|
||||||
|
@ -51,33 +51,35 @@ fn main() -> Result<(), Box<dyn Error>> {
|
||||||
exclusive: false,
|
exclusive: false,
|
||||||
auto_delete: false,
|
auto_delete: false,
|
||||||
no_wait: false,
|
no_wait: false,
|
||||||
})?;
|
})
|
||||||
|
.await?;
|
||||||
|
|
||||||
let vcs_data = match cfg.vcs {
|
let vcs_data = match cfg.vcs {
|
||||||
VCSConfig::Gerrit => SupportedVCS::Gerrit,
|
VCSConfig::Gerrit => SupportedVCS::Gerrit,
|
||||||
};
|
};
|
||||||
|
|
||||||
let handle = easylapin::WorkerChannel(chan).consume(
|
|
||||||
tasks::evaluate::EvaluationWorker::new(
|
|
||||||
cloner,
|
|
||||||
&nix,
|
|
||||||
vcs_data,
|
|
||||||
cfg.acl(),
|
|
||||||
cfg.runner.identity.clone(),
|
|
||||||
events,
|
|
||||||
),
|
|
||||||
easyamqp::ConsumeConfig {
|
|
||||||
queue: queue_name.clone(),
|
|
||||||
consumer_tag: format!("{}-mass-rebuild-checker", cfg.whoami()),
|
|
||||||
no_local: false,
|
|
||||||
no_ack: false,
|
|
||||||
no_wait: false,
|
|
||||||
exclusive: false,
|
|
||||||
},
|
|
||||||
)?;
|
|
||||||
|
|
||||||
info!("Fetching jobs from {}", queue_name);
|
info!("Fetching jobs from {}", queue_name);
|
||||||
block_on(handle);
|
|
||||||
|
easylapin::WorkerChannel(chan)
|
||||||
|
.consume(
|
||||||
|
tasks::evaluate::EvaluationWorker::new(
|
||||||
|
cloner,
|
||||||
|
&nix,
|
||||||
|
vcs_data,
|
||||||
|
cfg.acl(),
|
||||||
|
cfg.runner.identity.clone(),
|
||||||
|
events,
|
||||||
|
),
|
||||||
|
easyamqp::ConsumeConfig {
|
||||||
|
queue: queue_name.clone(),
|
||||||
|
consumer_tag: format!("{}-mass-rebuild-checker", cfg.whoami()),
|
||||||
|
no_local: false,
|
||||||
|
no_ack: false,
|
||||||
|
no_wait: false,
|
||||||
|
exclusive: false,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
drop(conn); // Close connection.
|
drop(conn); // Close connection.
|
||||||
info!("Closed the session... EOF");
|
info!("Closed the session... EOF");
|
||||||
|
|
|
@ -1,7 +1,6 @@
|
||||||
use std::error::Error;
|
use std::error::Error;
|
||||||
use std::io::Read;
|
use std::io::Read;
|
||||||
|
|
||||||
use futures::executor::block_on;
|
|
||||||
use lapin::message::Delivery;
|
use lapin::message::Delivery;
|
||||||
use lapin::BasicProperties;
|
use lapin::BasicProperties;
|
||||||
|
|
||||||
|
@ -102,7 +101,7 @@ impl<'a> Publisher<'a> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn publish_serde_action<T>(
|
async fn publish_serde_action<T>(
|
||||||
&mut self,
|
&mut self,
|
||||||
exchange: Option<String>,
|
exchange: Option<String>,
|
||||||
routing_key: Option<String>,
|
routing_key: Option<String>,
|
||||||
|
@ -111,18 +110,20 @@ impl<'a> Publisher<'a> {
|
||||||
T: Serialize + ?Sized,
|
T: Serialize + ?Sized,
|
||||||
{
|
{
|
||||||
self.recv
|
self.recv
|
||||||
.tell(worker::publish_serde_action(exchange, routing_key, msg));
|
.tell(worker::publish_serde_action(&exchange, &routing_key, msg))
|
||||||
|
.await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn main() -> Result<(), Box<dyn Error>> {
|
#[tokio::main]
|
||||||
|
async fn main() -> Result<(), Box<dyn Error>> {
|
||||||
ofborg::setup_log();
|
ofborg::setup_log();
|
||||||
|
|
||||||
let args = Cli::parse();
|
let args = Cli::parse();
|
||||||
let cfg = config::load(args.config.as_ref());
|
let cfg = config::load(args.config.as_ref());
|
||||||
|
|
||||||
let conn = easylapin::from_config(&cfg.rabbitmq)?;
|
let conn = easylapin::from_config(&cfg.rabbitmq).await?;
|
||||||
let mut chan = block_on(conn.create_channel())?;
|
let mut chan = conn.create_channel().await?;
|
||||||
|
|
||||||
match args.command {
|
match args.command {
|
||||||
Commands::Pastebin { action } => match action {
|
Commands::Pastebin { action } => match action {
|
||||||
|
@ -133,11 +134,13 @@ fn main() -> Result<(), Box<dyn Error>> {
|
||||||
.read_to_string(&mut contents)
|
.read_to_string(&mut contents)
|
||||||
.expect("Failed to read pastebin contents");
|
.expect("Failed to read pastebin contents");
|
||||||
|
|
||||||
publisher.publish_serde_action(
|
publisher
|
||||||
None,
|
.publish_serde_action(
|
||||||
Some("pastebin-log".to_owned()),
|
None,
|
||||||
&Pastebin { title, contents },
|
Some("pastebin-log".to_owned()),
|
||||||
);
|
&Pastebin { title, contents },
|
||||||
|
)
|
||||||
|
.await;
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
|
|
|
@ -1,7 +1,6 @@
|
||||||
use std::env;
|
use std::env;
|
||||||
use std::error::Error;
|
use std::error::Error;
|
||||||
|
|
||||||
use futures::executor::block_on;
|
|
||||||
use ofborg::config;
|
use ofborg::config;
|
||||||
use ofborg::easyamqp;
|
use ofborg::easyamqp;
|
||||||
use ofborg::easyamqp::ChannelExt;
|
use ofborg::easyamqp::ChannelExt;
|
||||||
|
@ -10,14 +9,15 @@ use ofborg::easylapin;
|
||||||
use ofborg::tasks;
|
use ofborg::tasks;
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
|
|
||||||
fn main() -> Result<(), Box<dyn Error>> {
|
#[tokio::main]
|
||||||
|
async fn main() -> Result<(), Box<dyn Error>> {
|
||||||
ofborg::setup_log();
|
ofborg::setup_log();
|
||||||
|
|
||||||
let arg = env::args().nth(1).expect("usage: pastebin-worker <config>");
|
let arg = env::args().nth(1).expect("usage: pastebin-worker <config>");
|
||||||
let cfg = config::load(arg.as_ref());
|
let cfg = config::load(arg.as_ref());
|
||||||
|
|
||||||
let conn = easylapin::from_config(&cfg.rabbitmq)?;
|
let conn = easylapin::from_config(&cfg.rabbitmq).await?;
|
||||||
let mut chan = block_on(conn.create_channel())?;
|
let mut chan = conn.create_channel().await?;
|
||||||
|
|
||||||
let queue_name = "pastebin-log".to_owned();
|
let queue_name = "pastebin-log".to_owned();
|
||||||
|
|
||||||
|
@ -29,7 +29,8 @@ fn main() -> Result<(), Box<dyn Error>> {
|
||||||
auto_delete: false,
|
auto_delete: false,
|
||||||
no_wait: false,
|
no_wait: false,
|
||||||
internal: false,
|
internal: false,
|
||||||
})?;
|
})
|
||||||
|
.await?;
|
||||||
|
|
||||||
chan.declare_queue(easyamqp::QueueConfig {
|
chan.declare_queue(easyamqp::QueueConfig {
|
||||||
queue: queue_name.clone(),
|
queue: queue_name.clone(),
|
||||||
|
@ -38,32 +39,35 @@ fn main() -> Result<(), Box<dyn Error>> {
|
||||||
exclusive: false,
|
exclusive: false,
|
||||||
auto_delete: false,
|
auto_delete: false,
|
||||||
no_wait: false,
|
no_wait: false,
|
||||||
})?;
|
})
|
||||||
|
.await?;
|
||||||
|
|
||||||
chan.bind_queue(easyamqp::BindQueueConfig {
|
chan.bind_queue(easyamqp::BindQueueConfig {
|
||||||
queue: queue_name.clone(),
|
queue: queue_name.clone(),
|
||||||
exchange: "pastebin-log".to_owned(),
|
exchange: "pastebin-log".to_owned(),
|
||||||
routing_key: None,
|
routing_key: None,
|
||||||
no_wait: false,
|
no_wait: false,
|
||||||
})?;
|
})
|
||||||
|
.await?;
|
||||||
let handle = easylapin::WorkerChannel(chan).consume(
|
|
||||||
tasks::pastebin_collector::PastebinCollector::new(
|
|
||||||
cfg.pastebin.clone().root,
|
|
||||||
cfg.pastebin.clone().db,
|
|
||||||
),
|
|
||||||
easyamqp::ConsumeConfig {
|
|
||||||
queue: queue_name.clone(),
|
|
||||||
consumer_tag: format!("{}-pastebin-worker", cfg.whoami()),
|
|
||||||
no_local: false,
|
|
||||||
no_ack: false,
|
|
||||||
no_wait: false,
|
|
||||||
exclusive: false,
|
|
||||||
},
|
|
||||||
)?;
|
|
||||||
|
|
||||||
info!("Fetching jobs from {}", &queue_name);
|
info!("Fetching jobs from {}", &queue_name);
|
||||||
block_on(handle);
|
|
||||||
|
easylapin::WorkerChannel(chan)
|
||||||
|
.consume(
|
||||||
|
tasks::pastebin_collector::PastebinCollector::new(
|
||||||
|
cfg.pastebin.clone().root,
|
||||||
|
cfg.pastebin.clone().db,
|
||||||
|
),
|
||||||
|
easyamqp::ConsumeConfig {
|
||||||
|
queue: queue_name.clone(),
|
||||||
|
consumer_tag: format!("{}-pastebin-worker", cfg.whoami()),
|
||||||
|
no_local: false,
|
||||||
|
no_ack: false,
|
||||||
|
no_wait: false,
|
||||||
|
exclusive: false,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
drop(conn); // Close connection.
|
drop(conn); // Close connection.
|
||||||
info!("Closed the session... EOF");
|
info!("Closed the session... EOF");
|
||||||
|
|
|
@ -6,7 +6,6 @@
|
||||||
use std::env;
|
use std::env;
|
||||||
use std::error::Error;
|
use std::error::Error;
|
||||||
|
|
||||||
use futures::executor::block_on;
|
|
||||||
use ofborg::config;
|
use ofborg::config;
|
||||||
use ofborg::easyamqp;
|
use ofborg::easyamqp;
|
||||||
use ofborg::easyamqp::ChannelExt;
|
use ofborg::easyamqp::ChannelExt;
|
||||||
|
@ -15,7 +14,8 @@ use ofborg::easylapin;
|
||||||
use ofborg::tasks;
|
use ofborg::tasks;
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
|
|
||||||
fn main() -> Result<(), Box<dyn Error>> {
|
#[tokio::main]
|
||||||
|
async fn main() -> Result<(), Box<dyn Error>> {
|
||||||
ofborg::setup_log();
|
ofborg::setup_log();
|
||||||
|
|
||||||
let arg = env::args()
|
let arg = env::args()
|
||||||
|
@ -23,8 +23,8 @@ fn main() -> Result<(), Box<dyn Error>> {
|
||||||
.expect("usage: statcheck-worker <config>");
|
.expect("usage: statcheck-worker <config>");
|
||||||
let cfg = config::load(arg.as_ref());
|
let cfg = config::load(arg.as_ref());
|
||||||
|
|
||||||
let conn = easylapin::from_config(&cfg.rabbitmq)?;
|
let conn = easylapin::from_config(&cfg.rabbitmq).await?;
|
||||||
let mut chan = block_on(conn.create_channel())?;
|
let mut chan = conn.create_channel().await?;
|
||||||
|
|
||||||
// an RPC queue for verbs
|
// an RPC queue for verbs
|
||||||
let api_queue_name = "statcheck-api".to_owned();
|
let api_queue_name = "statcheck-api".to_owned();
|
||||||
|
@ -39,7 +39,8 @@ fn main() -> Result<(), Box<dyn Error>> {
|
||||||
auto_delete: false,
|
auto_delete: false,
|
||||||
no_wait: false,
|
no_wait: false,
|
||||||
internal: false,
|
internal: false,
|
||||||
})?;
|
})
|
||||||
|
.await?;
|
||||||
|
|
||||||
chan.declare_queue(easyamqp::QueueConfig {
|
chan.declare_queue(easyamqp::QueueConfig {
|
||||||
queue: api_queue_name.clone(),
|
queue: api_queue_name.clone(),
|
||||||
|
@ -48,30 +49,33 @@ fn main() -> Result<(), Box<dyn Error>> {
|
||||||
exclusive: false,
|
exclusive: false,
|
||||||
auto_delete: false,
|
auto_delete: false,
|
||||||
no_wait: false,
|
no_wait: false,
|
||||||
})?;
|
})
|
||||||
|
.await?;
|
||||||
|
|
||||||
chan.bind_queue(easyamqp::BindQueueConfig {
|
chan.bind_queue(easyamqp::BindQueueConfig {
|
||||||
queue: api_queue_name.clone(),
|
queue: api_queue_name.clone(),
|
||||||
exchange: api_queue_name.clone(),
|
exchange: api_queue_name.clone(),
|
||||||
routing_key: None,
|
routing_key: None,
|
||||||
no_wait: false,
|
no_wait: false,
|
||||||
})?;
|
})
|
||||||
|
.await?;
|
||||||
let handle = easylapin::WorkerChannel(chan).consume(
|
|
||||||
tasks::status_check_collector::StatusCheckCollector::new(cfg.statcheck.clone().db),
|
|
||||||
easyamqp::ConsumeConfig {
|
|
||||||
queue: api_queue_name.clone(),
|
|
||||||
consumer_tag: format!("{}-{}", cfg.whoami(), api_queue_name),
|
|
||||||
no_local: false,
|
|
||||||
no_ack: false,
|
|
||||||
no_wait: false,
|
|
||||||
exclusive: false,
|
|
||||||
},
|
|
||||||
)?;
|
|
||||||
|
|
||||||
info!("Waiting for API calls on {}", api_queue_name);
|
info!("Waiting for API calls on {}", api_queue_name);
|
||||||
info!("Notifying of new changes on {}", event_queue_name);
|
info!("Notifying of new changes on {}", event_queue_name);
|
||||||
block_on(handle);
|
|
||||||
|
easylapin::WorkerChannel(chan)
|
||||||
|
.consume(
|
||||||
|
tasks::status_check_collector::StatusCheckCollector::new(cfg.statcheck.clone().db),
|
||||||
|
easyamqp::ConsumeConfig {
|
||||||
|
queue: api_queue_name.clone(),
|
||||||
|
consumer_tag: format!("{}-{}", cfg.whoami(), api_queue_name),
|
||||||
|
no_local: false,
|
||||||
|
no_ack: false,
|
||||||
|
no_wait: false,
|
||||||
|
exclusive: false,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
drop(conn); // Close connection.
|
drop(conn); // Close connection.
|
||||||
info!("Closed the session... EOF");
|
info!("Closed the session... EOF");
|
||||||
|
|
|
@ -1,26 +1,24 @@
|
||||||
use std::env;
|
use std::env;
|
||||||
use std::error::Error;
|
use std::error::Error;
|
||||||
use std::thread;
|
|
||||||
|
|
||||||
use axum::routing::get;
|
use axum::routing::get;
|
||||||
use futures::executor::block_on;
|
|
||||||
use axum::Router;
|
use axum::Router;
|
||||||
use tokio::runtime::Runtime;
|
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
|
|
||||||
use ofborg::easyamqp::{ChannelExt, ConsumerExt};
|
use ofborg::easyamqp::{ChannelExt, ConsumerExt};
|
||||||
use ofborg::{config, easyamqp, easylapin, stats, tasks};
|
use ofborg::{config, easyamqp, easylapin, stats, tasks};
|
||||||
|
|
||||||
fn main() -> Result<(), Box<dyn Error>> {
|
#[tokio::main]
|
||||||
|
async fn main() -> Result<(), Box<dyn Error>> {
|
||||||
ofborg::setup_log();
|
ofborg::setup_log();
|
||||||
|
|
||||||
let arg = env::args().nth(1).expect("usage: stats <config>");
|
let arg = env::args().nth(1).expect("usage: stats <config>");
|
||||||
let cfg = config::load(arg.as_ref());
|
let cfg = config::load(arg.as_ref());
|
||||||
|
|
||||||
let conn = easylapin::from_config(&cfg.rabbitmq)?;
|
let conn = easylapin::from_config(&cfg.rabbitmq).await?;
|
||||||
let mut chan = block_on(conn.create_channel())?;
|
let mut chan = conn.create_channel().await?;
|
||||||
|
|
||||||
let events = stats::RabbitMq::from_lapin(&cfg.whoami(), block_on(conn.create_channel())?);
|
let events = stats::RabbitMq::from_lapin(&cfg.whoami(), conn.create_channel().await?);
|
||||||
|
|
||||||
let metrics = stats::MetricCollector::new();
|
let metrics = stats::MetricCollector::new();
|
||||||
let collector = tasks::statscollector::StatCollectorWorker::new(events, metrics.clone());
|
let collector = tasks::statscollector::StatCollectorWorker::new(events, metrics.clone());
|
||||||
|
@ -33,7 +31,8 @@ fn main() -> Result<(), Box<dyn Error>> {
|
||||||
auto_delete: false,
|
auto_delete: false,
|
||||||
no_wait: false,
|
no_wait: false,
|
||||||
internal: false,
|
internal: false,
|
||||||
})?;
|
})
|
||||||
|
.await?;
|
||||||
|
|
||||||
let queue_name = String::from("stats-events");
|
let queue_name = String::from("stats-events");
|
||||||
chan.declare_queue(easyamqp::QueueConfig {
|
chan.declare_queue(easyamqp::QueueConfig {
|
||||||
|
@ -43,16 +42,30 @@ fn main() -> Result<(), Box<dyn Error>> {
|
||||||
exclusive: false,
|
exclusive: false,
|
||||||
auto_delete: false,
|
auto_delete: false,
|
||||||
no_wait: false,
|
no_wait: false,
|
||||||
})?;
|
})
|
||||||
|
.await?;
|
||||||
|
|
||||||
chan.bind_queue(easyamqp::BindQueueConfig {
|
chan.bind_queue(easyamqp::BindQueueConfig {
|
||||||
queue: queue_name.clone(),
|
queue: queue_name.clone(),
|
||||||
exchange: "stats".to_owned(),
|
exchange: "stats".to_owned(),
|
||||||
routing_key: None,
|
routing_key: None,
|
||||||
no_wait: false,
|
no_wait: false,
|
||||||
})?;
|
})
|
||||||
|
.await?;
|
||||||
|
|
||||||
let handle = chan.consume(
|
tokio::spawn(async {
|
||||||
|
let addr = "0.0.0.0:9898";
|
||||||
|
info!("listening addr {:?}", addr);
|
||||||
|
let app = Router::new().route(
|
||||||
|
"/metrics",
|
||||||
|
get(|| async move { metrics.prometheus_output() }),
|
||||||
|
);
|
||||||
|
let listener = tokio::net::TcpListener::bind(addr).await.unwrap();
|
||||||
|
axum::serve(listener, app).await.unwrap()
|
||||||
|
});
|
||||||
|
|
||||||
|
info!("Fetching jobs from {}", &queue_name);
|
||||||
|
chan.consume(
|
||||||
collector,
|
collector,
|
||||||
easyamqp::ConsumeConfig {
|
easyamqp::ConsumeConfig {
|
||||||
queue: "stats-events".to_owned(),
|
queue: "stats-events".to_owned(),
|
||||||
|
@ -62,22 +75,8 @@ fn main() -> Result<(), Box<dyn Error>> {
|
||||||
no_wait: false,
|
no_wait: false,
|
||||||
exclusive: false,
|
exclusive: false,
|
||||||
},
|
},
|
||||||
)?;
|
)
|
||||||
|
.await?;
|
||||||
thread::spawn(|| {
|
|
||||||
Runtime::new().unwrap().block_on(async {
|
|
||||||
let addr = "0.0.0.0:9898";
|
|
||||||
info!("listening addr {:?}", addr);
|
|
||||||
let app = Router::new().route("/metrics", get(|| async move {
|
|
||||||
metrics.prometheus_output()
|
|
||||||
}));
|
|
||||||
let listener= tokio::net::TcpListener::bind(addr).await.unwrap();
|
|
||||||
axum::serve(listener, app).await.unwrap()
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
info!("Fetching jobs from {}", &queue_name);
|
|
||||||
block_on(handle);
|
|
||||||
|
|
||||||
drop(conn); // Close connection.
|
drop(conn); // Close connection.
|
||||||
info!("Closed the session... EOF");
|
info!("Closed the session... EOF");
|
||||||
|
|
|
@ -12,6 +12,7 @@ pub struct CachedCloner {
|
||||||
root: PathBuf,
|
root: PathBuf,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub fn cached_cloner(path: &Path) -> CachedCloner {
|
pub fn cached_cloner(path: &Path) -> CachedCloner {
|
||||||
CachedCloner {
|
CachedCloner {
|
||||||
root: path.to_path_buf(),
|
root: path.to_path_buf(),
|
||||||
|
@ -31,6 +32,7 @@ pub struct CachedProjectCo {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl CachedCloner {
|
impl CachedCloner {
|
||||||
|
#[must_use]
|
||||||
pub fn project(&self, name: &str, clone_url: String) -> CachedProject {
|
pub fn project(&self, name: &str, clone_url: String) -> CachedProject {
|
||||||
// <root>/repo/<hash>/clone
|
// <root>/repo/<hash>/clone
|
||||||
// <root>/repo/<hash>/clone.lock
|
// <root>/repo/<hash>/clone.lock
|
||||||
|
@ -101,7 +103,7 @@ impl CachedProjectCo {
|
||||||
let result = Command::new("git")
|
let result = Command::new("git")
|
||||||
.arg("fetch")
|
.arg("fetch")
|
||||||
.arg("origin")
|
.arg("origin")
|
||||||
.arg(format!("+refs/pull/{}/head:pr", pr_id))
|
.arg(format!("+refs/pull/{pr_id}/head:pr"))
|
||||||
.current_dir(self.clone_to())
|
.current_dir(self.clone_to())
|
||||||
.stdout(Stdio::null())
|
.stdout(Stdio::null())
|
||||||
.status()?;
|
.status()?;
|
||||||
|
@ -162,7 +164,7 @@ impl CachedProjectCo {
|
||||||
let result = Command::new("git")
|
let result = Command::new("git")
|
||||||
.arg("log")
|
.arg("log")
|
||||||
.arg("--format=format:%s")
|
.arg("--format=format:%s")
|
||||||
.arg(format!("HEAD..{}", commit))
|
.arg(format!("HEAD..{commit}"))
|
||||||
.current_dir(self.clone_to())
|
.current_dir(self.clone_to())
|
||||||
.output()?;
|
.output()?;
|
||||||
|
|
||||||
|
@ -171,7 +173,7 @@ impl CachedProjectCo {
|
||||||
if result.status.success() {
|
if result.status.success() {
|
||||||
Ok(String::from_utf8_lossy(&result.stdout)
|
Ok(String::from_utf8_lossy(&result.stdout)
|
||||||
.lines()
|
.lines()
|
||||||
.map(|l| l.to_owned())
|
.map(std::borrow::ToOwned::to_owned)
|
||||||
.collect())
|
.collect())
|
||||||
} else {
|
} else {
|
||||||
Err(Error::new(
|
Err(Error::new(
|
||||||
|
@ -187,7 +189,7 @@ impl CachedProjectCo {
|
||||||
let result = Command::new("git")
|
let result = Command::new("git")
|
||||||
.arg("diff")
|
.arg("diff")
|
||||||
.arg("--name-only")
|
.arg("--name-only")
|
||||||
.arg(format!("HEAD...{}", commit))
|
.arg(format!("HEAD...{commit}"))
|
||||||
.current_dir(self.clone_to())
|
.current_dir(self.clone_to())
|
||||||
.output()?;
|
.output()?;
|
||||||
|
|
||||||
|
@ -196,7 +198,7 @@ impl CachedProjectCo {
|
||||||
if result.status.success() {
|
if result.status.success() {
|
||||||
Ok(String::from_utf8_lossy(&result.stdout)
|
Ok(String::from_utf8_lossy(&result.stdout)
|
||||||
.lines()
|
.lines()
|
||||||
.map(|l| l.to_owned())
|
.map(std::borrow::ToOwned::to_owned)
|
||||||
.collect())
|
.collect())
|
||||||
} else {
|
} else {
|
||||||
Err(Error::new(
|
Err(Error::new(
|
||||||
|
@ -278,8 +280,8 @@ mod tests {
|
||||||
.expect("building the test PR failed");
|
.expect("building the test PR failed");
|
||||||
|
|
||||||
let stderr =
|
let stderr =
|
||||||
String::from_utf8(output.stderr).unwrap_or_else(|err| format!("warning: {}", err));
|
String::from_utf8(output.stderr).unwrap_or_else(|err| format!("warning: {err}"));
|
||||||
println!("{}", stderr);
|
println!("{stderr}");
|
||||||
|
|
||||||
let hash = String::from_utf8(output.stdout).expect("Should just be a hash");
|
let hash = String::from_utf8(output.stdout).expect("Should just be a hash");
|
||||||
return hash.trim().to_owned();
|
return hash.trim().to_owned();
|
||||||
|
|
|
@ -14,7 +14,7 @@ pub struct Lock {
|
||||||
|
|
||||||
impl Lock {
|
impl Lock {
|
||||||
pub fn unlock(&mut self) {
|
pub fn unlock(&mut self) {
|
||||||
self.lock = None
|
self.lock = None;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -33,20 +33,19 @@ pub trait GitClonable {
|
||||||
warn!("Failed to create lock file {:?}: {}", self.lock_path(), e);
|
warn!("Failed to create lock file {:?}: {}", self.lock_path(), e);
|
||||||
Err(e)
|
Err(e)
|
||||||
}
|
}
|
||||||
Ok(lock) => match lock.lock_exclusive() {
|
Ok(lock) => {
|
||||||
Err(e) => {
|
if let Err(e) = lock.lock_exclusive() {
|
||||||
warn!(
|
warn!(
|
||||||
"Failed to get exclusive lock on file {:?}: {}",
|
"Failed to get exclusive lock on file {:?}: {}",
|
||||||
self.lock_path(),
|
self.lock_path(),
|
||||||
e
|
e
|
||||||
);
|
);
|
||||||
Err(e)
|
Err(e)
|
||||||
}
|
} else {
|
||||||
Ok(_) => {
|
|
||||||
debug!("Got lock on {:?}", self.lock_path());
|
debug!("Got lock on {:?}", self.lock_path());
|
||||||
Ok(Lock { lock: Some(lock) })
|
Ok(Lock { lock: Some(lock) })
|
||||||
}
|
}
|
||||||
},
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
use nom::types::CompleteStr;
|
use nom::types::CompleteStr;
|
||||||
use tracing::warn;
|
use tracing::warn;
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub fn parse(text: &str) -> Option<Vec<Instruction>> {
|
pub fn parse(text: &str) -> Option<Vec<Instruction>> {
|
||||||
let instructions: Vec<Instruction> = text
|
let instructions: Vec<Instruction> = text
|
||||||
.lines()
|
.lines()
|
||||||
|
|
|
@ -119,10 +119,12 @@ pub struct CheckoutConfig {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Config {
|
impl Config {
|
||||||
|
#[must_use]
|
||||||
pub fn whoami(&self) -> String {
|
pub fn whoami(&self) -> String {
|
||||||
format!("{}-{}", self.runner.identity, self.nix.system.join(","))
|
format!("{}-{}", self.runner.identity, self.nix.system.join(","))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub fn acl(&self) -> acl::Acl {
|
pub fn acl(&self) -> acl::Acl {
|
||||||
let repos = self
|
let repos = self
|
||||||
.runner
|
.runner
|
||||||
|
@ -172,7 +174,7 @@ impl RabbitMqConfig {
|
||||||
.expect("Failed to expand the password-file configuration string");
|
.expect("Failed to expand the password-file configuration string");
|
||||||
password = std::fs::read_to_string(expanded_password_file.as_ref())?;
|
password = std::fs::read_to_string(expanded_password_file.as_ref())?;
|
||||||
} else {
|
} else {
|
||||||
password = "".to_owned();
|
password = String::new();
|
||||||
}
|
}
|
||||||
let uri = format!(
|
let uri = format!(
|
||||||
"{}://{}:{}@{}/{}",
|
"{}://{}:{}@{}/{}",
|
||||||
|
@ -186,6 +188,7 @@ impl RabbitMqConfig {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub fn load(filename: &Path) -> Config {
|
pub fn load(filename: &Path) -> Config {
|
||||||
let mut file = File::open(filename).unwrap();
|
let mut file = File::open(filename).unwrap();
|
||||||
let mut contents = String::new();
|
let mut contents = String::new();
|
||||||
|
|
|
@ -1,5 +1,10 @@
|
||||||
|
use std::fmt::Debug;
|
||||||
|
|
||||||
|
use async_trait::async_trait;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
#[allow(clippy::struct_excessive_bools)]
|
||||||
pub struct ConsumeConfig {
|
pub struct ConsumeConfig {
|
||||||
/// Specifies the name of the queue to consume from.
|
/// Specifies the name of the queue to consume from.
|
||||||
pub queue: String,
|
pub queue: String,
|
||||||
|
@ -40,6 +45,7 @@ pub struct ConsumeConfig {
|
||||||
pub no_wait: bool,
|
pub no_wait: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
pub struct BindQueueConfig {
|
pub struct BindQueueConfig {
|
||||||
/// Specifies the name of the queue to bind.
|
/// Specifies the name of the queue to bind.
|
||||||
///
|
///
|
||||||
|
@ -83,6 +89,7 @@ pub struct BindQueueConfig {
|
||||||
pub no_wait: bool,
|
pub no_wait: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
pub enum ExchangeType {
|
pub enum ExchangeType {
|
||||||
Topic,
|
Topic,
|
||||||
Headers,
|
Headers,
|
||||||
|
@ -103,6 +110,8 @@ impl From<ExchangeType> for String {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
#[allow(clippy::struct_excessive_bools)]
|
||||||
pub struct ExchangeConfig {
|
pub struct ExchangeConfig {
|
||||||
/// Exchange names starting with "amq." are reserved for
|
/// Exchange names starting with "amq." are reserved for
|
||||||
/// pre-declared and standardised exchanges. The client MAY
|
/// pre-declared and standardised exchanges. The client MAY
|
||||||
|
@ -183,6 +192,8 @@ pub struct ExchangeConfig {
|
||||||
pub no_wait: bool,
|
pub no_wait: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
#[allow(clippy::struct_excessive_bools)]
|
||||||
pub struct QueueConfig {
|
pub struct QueueConfig {
|
||||||
/// The queue name MAY be empty, in which case the server MUST
|
/// The queue name MAY be empty, in which case the server MUST
|
||||||
/// create a new queue with a unique generated name and return
|
/// create a new queue with a unique generated name and return
|
||||||
|
@ -260,21 +271,27 @@ pub struct QueueConfig {
|
||||||
pub no_wait: bool,
|
pub no_wait: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
pub trait ChannelExt {
|
pub trait ChannelExt {
|
||||||
type Error;
|
type Error;
|
||||||
fn declare_exchange(&mut self, config: ExchangeConfig) -> Result<(), Self::Error>;
|
async fn declare_exchange(&mut self, config: ExchangeConfig) -> Result<(), Self::Error>;
|
||||||
fn declare_queue(&mut self, config: QueueConfig) -> Result<(), Self::Error>;
|
async fn declare_queue(&mut self, config: QueueConfig) -> Result<(), Self::Error>;
|
||||||
fn bind_queue(&mut self, config: BindQueueConfig) -> Result<(), Self::Error>;
|
async fn bind_queue(&mut self, config: BindQueueConfig) -> Result<(), Self::Error>;
|
||||||
fn send_request<T: ?Sized + Serialize + std::marker::Sync, U: for<'a> Deserialize<'a>>(
|
async fn send_request<
|
||||||
|
T: ?Sized + Serialize + Sync + Send + Debug,
|
||||||
|
U: for<'a> Deserialize<'a> + Debug,
|
||||||
|
>(
|
||||||
&mut self,
|
&mut self,
|
||||||
exchange: Option<&str>,
|
exchange: Option<&str>,
|
||||||
routing_key: Option<&str>,
|
routing_key: Option<&str>,
|
||||||
msg: &T,
|
msg: &T,
|
||||||
) -> impl std::future::Future<Output = Result<U, Self::Error>> + Send;
|
) -> Result<U, Self::Error>;
|
||||||
}
|
}
|
||||||
|
|
||||||
pub trait ConsumerExt<'a, C> {
|
#[async_trait]
|
||||||
|
pub trait ConsumerExt<C> {
|
||||||
type Error;
|
type Error;
|
||||||
type Handle;
|
async fn consume(self, callback: C, config: ConsumeConfig) -> Result<(), Self::Error>
|
||||||
fn consume(self, callback: C, config: ConsumeConfig) -> Result<Self::Handle, Self::Error>;
|
where
|
||||||
|
C: 'async_trait;
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,6 +1,3 @@
|
||||||
use std::future::Future;
|
|
||||||
use std::pin::Pin;
|
|
||||||
|
|
||||||
use crate::config::RabbitMqConfig;
|
use crate::config::RabbitMqConfig;
|
||||||
use crate::easyamqp::{
|
use crate::easyamqp::{
|
||||||
BindQueueConfig, ChannelExt, ConsumeConfig, ConsumerExt, ExchangeConfig, ExchangeType,
|
BindQueueConfig, ChannelExt, ConsumeConfig, ConsumerExt, ExchangeConfig, ExchangeType,
|
||||||
|
@ -10,7 +7,9 @@ use crate::notifyworker::{NotificationReceiver, SimpleNotifyWorker};
|
||||||
use crate::ofborg;
|
use crate::ofborg;
|
||||||
use crate::worker::{prepare_queue_message, Action, SimpleWorker};
|
use crate::worker::{prepare_queue_message, Action, SimpleWorker};
|
||||||
|
|
||||||
use futures::executor::block_on;
|
use std::fmt::Debug;
|
||||||
|
|
||||||
|
use async_trait::async_trait;
|
||||||
use futures::StreamExt;
|
use futures::StreamExt;
|
||||||
use lapin::message::Delivery;
|
use lapin::message::Delivery;
|
||||||
use lapin::options::{
|
use lapin::options::{
|
||||||
|
@ -22,7 +21,7 @@ use lapin::{BasicProperties, Channel, Connection, ConnectionProperties, Exchange
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use tracing::{debug, trace};
|
use tracing::{debug, trace};
|
||||||
|
|
||||||
pub fn from_config(cfg: &RabbitMqConfig) -> Result<Connection, lapin::Error> {
|
pub async fn from_config(cfg: &RabbitMqConfig) -> Result<Connection, lapin::Error> {
|
||||||
let mut props = FieldTable::default();
|
let mut props = FieldTable::default();
|
||||||
props.insert(
|
props.insert(
|
||||||
"ofborg_version".into(),
|
"ofborg_version".into(),
|
||||||
|
@ -32,13 +31,15 @@ pub fn from_config(cfg: &RabbitMqConfig) -> Result<Connection, lapin::Error> {
|
||||||
client_properties: props,
|
client_properties: props,
|
||||||
..Default::default()
|
..Default::default()
|
||||||
};
|
};
|
||||||
block_on(Connection::connect(&cfg.as_uri()?, opts))
|
Connection::connect(&cfg.as_uri()?, opts).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
impl ChannelExt for Channel {
|
impl ChannelExt for Channel {
|
||||||
type Error = lapin::Error;
|
type Error = lapin::Error;
|
||||||
|
|
||||||
fn declare_exchange(&mut self, config: ExchangeConfig) -> Result<(), Self::Error> {
|
#[tracing::instrument(ret)]
|
||||||
|
async fn declare_exchange(&mut self, config: ExchangeConfig) -> Result<(), Self::Error> {
|
||||||
let opts = ExchangeDeclareOptions {
|
let opts = ExchangeDeclareOptions {
|
||||||
passive: config.passive,
|
passive: config.passive,
|
||||||
durable: config.durable,
|
durable: config.durable,
|
||||||
|
@ -52,11 +53,13 @@ impl ChannelExt for Channel {
|
||||||
ExchangeType::Fanout => ExchangeKind::Fanout,
|
ExchangeType::Fanout => ExchangeKind::Fanout,
|
||||||
_ => panic!("exchange kind"),
|
_ => panic!("exchange kind"),
|
||||||
};
|
};
|
||||||
block_on(self.exchange_declare(&config.exchange, kind, opts, FieldTable::default()))?;
|
self.exchange_declare(&config.exchange, kind, opts, FieldTable::default())
|
||||||
|
.await?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn declare_queue(&mut self, config: QueueConfig) -> Result<(), Self::Error> {
|
#[tracing::instrument(ret)]
|
||||||
|
async fn declare_queue(&mut self, config: QueueConfig) -> Result<(), Self::Error> {
|
||||||
let opts = QueueDeclareOptions {
|
let opts = QueueDeclareOptions {
|
||||||
passive: config.passive,
|
passive: config.passive,
|
||||||
durable: config.durable,
|
durable: config.durable,
|
||||||
|
@ -65,26 +68,33 @@ impl ChannelExt for Channel {
|
||||||
nowait: config.no_wait,
|
nowait: config.no_wait,
|
||||||
};
|
};
|
||||||
|
|
||||||
block_on(self.queue_declare(&config.queue, opts, FieldTable::default()))?;
|
self.queue_declare(&config.queue, opts, FieldTable::default())
|
||||||
|
.await?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn bind_queue(&mut self, config: BindQueueConfig) -> Result<(), Self::Error> {
|
#[tracing::instrument(ret)]
|
||||||
|
async fn bind_queue(&mut self, config: BindQueueConfig) -> Result<(), Self::Error> {
|
||||||
let opts = QueueBindOptions {
|
let opts = QueueBindOptions {
|
||||||
nowait: config.no_wait,
|
nowait: config.no_wait,
|
||||||
};
|
};
|
||||||
|
|
||||||
block_on(self.queue_bind(
|
self.queue_bind(
|
||||||
&config.queue,
|
&config.queue,
|
||||||
&config.exchange,
|
&config.exchange,
|
||||||
&config.routing_key.unwrap_or_else(|| "".into()),
|
&config.routing_key.unwrap_or_default(),
|
||||||
opts,
|
opts,
|
||||||
FieldTable::default(),
|
FieldTable::default(),
|
||||||
))?;
|
)
|
||||||
|
.await?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn send_request<T: ?Sized + Serialize, U: for<'a> Deserialize<'a>>(
|
#[tracing::instrument(ret)]
|
||||||
|
async fn send_request<
|
||||||
|
T: ?Sized + Serialize + Sync + Send + Debug,
|
||||||
|
U: for<'a> Deserialize<'a> + Debug,
|
||||||
|
>(
|
||||||
&mut self,
|
&mut self,
|
||||||
exchange: Option<&str>,
|
exchange: Option<&str>,
|
||||||
routing_key: Option<&str>,
|
routing_key: Option<&str>,
|
||||||
|
@ -105,8 +115,8 @@ impl ChannelExt for Channel {
|
||||||
trace!(?exchange, ?routing_key, "sending a RPC request");
|
trace!(?exchange, ?routing_key, "sending a RPC request");
|
||||||
let confirmation = self
|
let confirmation = self
|
||||||
.basic_publish(
|
.basic_publish(
|
||||||
&msg.exchange.take().unwrap_or_else(|| "".to_owned()),
|
&msg.exchange.take().unwrap_or_default(),
|
||||||
&msg.routing_key.take().unwrap_or_else(|| "".to_owned()),
|
&msg.routing_key.take().unwrap_or_default(),
|
||||||
BasicPublishOptions::default(),
|
BasicPublishOptions::default(),
|
||||||
&msg.content,
|
&msg.content,
|
||||||
props,
|
props,
|
||||||
|
@ -139,55 +149,64 @@ impl ChannelExt for Channel {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<'a, W: SimpleWorker + 'a> ConsumerExt<'a, W> for Channel {
|
#[async_trait]
|
||||||
|
impl<W: SimpleWorker> ConsumerExt<W> for Channel {
|
||||||
type Error = lapin::Error;
|
type Error = lapin::Error;
|
||||||
type Handle = Pin<Box<dyn Future<Output = ()> + 'a>>;
|
|
||||||
|
|
||||||
fn consume(
|
#[tracing::instrument(skip(worker), ret)]
|
||||||
mut self,
|
async fn consume(mut self, mut worker: W, config: ConsumeConfig) -> Result<(), Self::Error>
|
||||||
mut worker: W,
|
where
|
||||||
config: ConsumeConfig,
|
W: 'async_trait,
|
||||||
) -> Result<Self::Handle, Self::Error> {
|
{
|
||||||
let mut consumer = block_on(self.basic_consume(
|
let mut consumer = self
|
||||||
&config.queue,
|
.basic_consume(
|
||||||
&config.consumer_tag,
|
&config.queue,
|
||||||
BasicConsumeOptions::default(),
|
&config.consumer_tag,
|
||||||
FieldTable::default(),
|
BasicConsumeOptions::default(),
|
||||||
))?;
|
FieldTable::default(),
|
||||||
Ok(Box::pin(async move {
|
)
|
||||||
while let Some(Ok(deliver)) = consumer.next().await {
|
.await?;
|
||||||
debug!(?deliver.delivery_tag, "consumed delivery");
|
|
||||||
let content_type = deliver.properties.content_type();
|
|
||||||
let job = worker
|
|
||||||
.msg_to_job(
|
|
||||||
deliver.routing_key.as_str(),
|
|
||||||
&content_type.as_ref().map(|s| s.to_string()),
|
|
||||||
&deliver.data,
|
|
||||||
)
|
|
||||||
.expect("worker unexpected message consumed");
|
|
||||||
|
|
||||||
for action in worker.consumer(&mut self, &job) {
|
while let Some(Ok(deliver)) = consumer.next().await {
|
||||||
action_deliver(&self, &deliver, action)
|
debug!(?deliver.delivery_tag, "consumed delivery");
|
||||||
.await
|
let content_type = deliver.properties.content_type();
|
||||||
.expect("action deliver failure");
|
let job = worker
|
||||||
}
|
.msg_to_job(
|
||||||
debug!(?deliver.delivery_tag, "done");
|
deliver.routing_key.as_str(),
|
||||||
|
&content_type.as_ref().map(std::string::ToString::to_string),
|
||||||
|
&deliver.data,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("worker unexpected message consumed");
|
||||||
|
|
||||||
|
for action in worker.consumer(&mut self, &job).await {
|
||||||
|
action_deliver(&self, &deliver, action)
|
||||||
|
.await
|
||||||
|
.expect("action deliver failure");
|
||||||
}
|
}
|
||||||
}))
|
debug!(?deliver.delivery_tag, "done");
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Same as a regular channel, but without prefetching,
|
/// Same as a regular channel, but without prefetching,
|
||||||
/// used for services with multiple instances.
|
/// used for services with multiple instances.
|
||||||
|
#[derive(Debug)]
|
||||||
pub struct WorkerChannel(pub Channel);
|
pub struct WorkerChannel(pub Channel);
|
||||||
|
|
||||||
impl<'a, W: SimpleWorker + 'a> ConsumerExt<'a, W> for WorkerChannel {
|
#[async_trait]
|
||||||
|
impl<W: SimpleWorker> ConsumerExt<W> for WorkerChannel {
|
||||||
type Error = lapin::Error;
|
type Error = lapin::Error;
|
||||||
type Handle = Pin<Box<dyn Future<Output = ()> + 'a>>;
|
|
||||||
|
|
||||||
fn consume(self, worker: W, config: ConsumeConfig) -> Result<Self::Handle, Self::Error> {
|
#[tracing::instrument(skip(worker), ret)]
|
||||||
block_on(self.0.basic_qos(1, BasicQosOptions::default()))?;
|
async fn consume(self, worker: W, config: ConsumeConfig) -> Result<(), Self::Error>
|
||||||
self.0.consume(worker, config)
|
where
|
||||||
|
W: 'async_trait,
|
||||||
|
{
|
||||||
|
self.0.basic_qos(1, BasicQosOptions::default()).await?;
|
||||||
|
Ok(self.0.consume(worker, config).await?)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -202,52 +221,64 @@ impl<'a> ChannelNotificationReceiver<'a> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
impl<'a> NotificationReceiver for ChannelNotificationReceiver<'a> {
|
impl<'a> NotificationReceiver for ChannelNotificationReceiver<'a> {
|
||||||
fn tell(&mut self, action: Action) {
|
async fn tell(&mut self, action: Action) {
|
||||||
block_on(action_deliver(self.channel, self.deliver, action))
|
action_deliver(self.channel, self.deliver, action)
|
||||||
|
.await
|
||||||
.expect("action deliver failure");
|
.expect("action deliver failure");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// FIXME the consumer trait for SimpleWorker and SimpleNotifyWorker conflict,
|
// FIXME the consumer trait for SimpleWorker and SimpleNotifyWorker conflict,
|
||||||
// but one could probably be implemented in terms of the other instead.
|
// but one could probably be implemented in terms of the other instead.
|
||||||
|
#[derive(Debug)]
|
||||||
pub struct NotifyChannel(pub Channel);
|
pub struct NotifyChannel(pub Channel);
|
||||||
|
|
||||||
impl<'a, W: SimpleNotifyWorker + 'a + Send> ConsumerExt<'a, W> for NotifyChannel {
|
#[async_trait]
|
||||||
|
impl<W: SimpleNotifyWorker> ConsumerExt<W> for NotifyChannel {
|
||||||
type Error = lapin::Error;
|
type Error = lapin::Error;
|
||||||
type Handle = Pin<Box<dyn Future<Output = ()> + 'a + Send>>;
|
|
||||||
|
|
||||||
fn consume(self, worker: W, config: ConsumeConfig) -> Result<Self::Handle, Self::Error> {
|
#[tracing::instrument(skip(worker), ret)]
|
||||||
block_on(self.0.basic_qos(1, BasicQosOptions::default()))?;
|
async fn consume(self, worker: W, config: ConsumeConfig) -> Result<(), Self::Error>
|
||||||
|
where
|
||||||
|
W: 'async_trait,
|
||||||
|
{
|
||||||
|
self.0.basic_qos(1, BasicQosOptions::default()).await?;
|
||||||
|
|
||||||
let mut consumer = block_on(self.0.basic_consume(
|
let mut consumer = self
|
||||||
&config.queue,
|
.0
|
||||||
&config.consumer_tag,
|
.basic_consume(
|
||||||
BasicConsumeOptions::default(),
|
&config.queue,
|
||||||
FieldTable::default(),
|
&config.consumer_tag,
|
||||||
))?;
|
BasicConsumeOptions::default(),
|
||||||
|
FieldTable::default(),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
let mut chan = self.0;
|
let mut chan = self.0;
|
||||||
Ok(Box::pin(async move {
|
|
||||||
while let Some(Ok(deliver)) = consumer.next().await {
|
|
||||||
debug!(?deliver.delivery_tag, "consumed delivery");
|
|
||||||
let mut receiver = ChannelNotificationReceiver {
|
|
||||||
channel: &mut chan,
|
|
||||||
deliver: &deliver,
|
|
||||||
};
|
|
||||||
|
|
||||||
let content_type = deliver.properties.content_type();
|
while let Some(Ok(deliver)) = consumer.next().await {
|
||||||
let job = worker
|
debug!(?deliver.delivery_tag, "consumed delivery");
|
||||||
.msg_to_job(
|
let mut receiver = ChannelNotificationReceiver {
|
||||||
deliver.routing_key.as_str(),
|
channel: &mut chan,
|
||||||
&content_type.as_ref().map(|s| s.to_string()),
|
deliver: &deliver,
|
||||||
&deliver.data,
|
};
|
||||||
)
|
|
||||||
.expect("worker unexpected message consumed");
|
|
||||||
|
|
||||||
worker.consumer(&job, &mut receiver);
|
let content_type = deliver.properties.content_type();
|
||||||
debug!(?deliver.delivery_tag, "done");
|
let job = worker
|
||||||
}
|
.msg_to_job(
|
||||||
}))
|
deliver.routing_key.as_str(),
|
||||||
|
&content_type.as_ref().map(std::string::ToString::to_string),
|
||||||
|
&deliver.data,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("worker unexpected message consumed");
|
||||||
|
|
||||||
|
worker.consumer(&job, &mut receiver).await;
|
||||||
|
debug!(?deliver.delivery_tag, "done");
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -276,8 +307,8 @@ async fn action_deliver(
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
Action::Publish(mut msg) => {
|
Action::Publish(mut msg) => {
|
||||||
let exch = msg.exchange.take().unwrap_or_else(|| "".to_owned());
|
let exch = msg.exchange.take().unwrap_or_else(String::new);
|
||||||
let key = msg.routing_key.take().unwrap_or_else(|| "".to_owned());
|
let key = msg.routing_key.take().unwrap_or_else(String::new);
|
||||||
trace!(?exch, ?key, "action publish");
|
trace!(?exch, ?key, "action publish");
|
||||||
|
|
||||||
let mut props = BasicProperties::default().with_delivery_mode(2); // persistent.
|
let mut props = BasicProperties::default().with_delivery_mode(2); // persistent.
|
||||||
|
|
|
@ -11,6 +11,7 @@ pub struct EvalChecker {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl EvalChecker {
|
impl EvalChecker {
|
||||||
|
#[must_use]
|
||||||
pub fn new(name: &str, op: nix::Operation, args: Vec<String>, nix: nix::Nix) -> EvalChecker {
|
pub fn new(name: &str, op: nix::Operation, args: Vec<String>, nix: nix::Nix) -> EvalChecker {
|
||||||
EvalChecker {
|
EvalChecker {
|
||||||
name: name.to_owned(),
|
name: name.to_owned(),
|
||||||
|
@ -20,14 +21,18 @@ impl EvalChecker {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub fn name(&self) -> &str {
|
pub fn name(&self) -> &str {
|
||||||
&self.name
|
&self.name
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn execute(&self, path: &Path) -> Result<File, File> {
|
pub async fn execute(&self, path: &Path) -> Result<File, File> {
|
||||||
self.nix.safely(&self.op, path, self.args.clone(), false)
|
self.nix
|
||||||
|
.safely(&self.op, path, self.args.clone(), false)
|
||||||
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub fn cli_cmd(&self) -> String {
|
pub fn cli_cmd(&self) -> String {
|
||||||
let mut cli = vec![self.op.to_string()];
|
let mut cli = vec![self.op.to_string()];
|
||||||
cli.append(&mut self.args.clone());
|
cli.append(&mut self.args.clone());
|
||||||
|
|
|
@ -1,9 +1,9 @@
|
||||||
#![recursion_limit = "512"]
|
#![recursion_limit = "512"]
|
||||||
// Replacing .map(|arch| arch.to_string())
|
#![deny(clippy::pedantic)]
|
||||||
// with .map(systems::System::to_string)
|
#![allow(clippy::missing_errors_doc)]
|
||||||
//
|
#![allow(clippy::missing_panics_doc)]
|
||||||
// seems much less clear and I just don't like it :)
|
#![allow(clippy::module_name_repetitions)]
|
||||||
#![allow(clippy::redundant_closure)]
|
#![allow(clippy::similar_names)]
|
||||||
|
|
||||||
#[macro_use]
|
#[macro_use]
|
||||||
extern crate serde_derive;
|
extern crate serde_derive;
|
||||||
|
@ -14,15 +14,14 @@ extern crate nom;
|
||||||
use std::env;
|
use std::env;
|
||||||
|
|
||||||
use opentelemetry::global;
|
use opentelemetry::global;
|
||||||
|
use opentelemetry::trace::TracerProvider;
|
||||||
use opentelemetry::KeyValue;
|
use opentelemetry::KeyValue;
|
||||||
use opentelemetry_otlp::HasHttpConfig;
|
|
||||||
use opentelemetry_sdk::Resource;
|
use opentelemetry_sdk::Resource;
|
||||||
use opentelemetry_semantic_conventions::resource::SERVICE_NAME;
|
use opentelemetry_semantic_conventions::resource::SERVICE_NAME;
|
||||||
use opentelemetry_semantic_conventions::resource::SERVICE_VERSION;
|
use opentelemetry_semantic_conventions::resource::SERVICE_VERSION;
|
||||||
use opentelemetry_semantic_conventions::SCHEMA_URL;
|
use opentelemetry_semantic_conventions::SCHEMA_URL;
|
||||||
use tracing_subscriber::prelude::*;
|
use tracing_subscriber::prelude::*;
|
||||||
use tracing_subscriber::EnvFilter;
|
use tracing_subscriber::EnvFilter;
|
||||||
use opentelemetry::trace::TracerProvider;
|
|
||||||
|
|
||||||
pub mod acl;
|
pub mod acl;
|
||||||
pub mod asynccmd;
|
pub mod asynccmd;
|
||||||
|
@ -80,10 +79,11 @@ pub mod ofborg {
|
||||||
|
|
||||||
pub const VERSION: &str = env!("CARGO_PKG_VERSION");
|
pub const VERSION: &str = env!("CARGO_PKG_VERSION");
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub fn partition_result<A, B>(results: Vec<Result<A, B>>) -> (Vec<A>, Vec<B>) {
|
pub fn partition_result<A, B>(results: Vec<Result<A, B>>) -> (Vec<A>, Vec<B>) {
|
||||||
let mut ok = Vec::new();
|
let mut ok = Vec::new();
|
||||||
let mut err = Vec::new();
|
let mut err = Vec::new();
|
||||||
for result in results.into_iter() {
|
for result in results {
|
||||||
match result {
|
match result {
|
||||||
Ok(x) => {
|
Ok(x) => {
|
||||||
ok.push(x);
|
ok.push(x);
|
||||||
|
@ -121,12 +121,15 @@ pub fn setup_log() {
|
||||||
tracing_subscriber::fmt::layer().boxed()
|
tracing_subscriber::fmt::layer().boxed()
|
||||||
};
|
};
|
||||||
|
|
||||||
let exporter = opentelemetry_otlp::SpanExporter::builder().with_http().build().unwrap();
|
|
||||||
let provider = opentelemetry_sdk::trace::TracerProvider::builder()
|
let provider = opentelemetry_sdk::trace::TracerProvider::builder()
|
||||||
.with_simple_exporter(exporter)
|
.with_batch_exporter(
|
||||||
.with_config(
|
opentelemetry_otlp::SpanExporter::builder()
|
||||||
opentelemetry_sdk::trace::Config::default().with_resource(resource())
|
.with_http()
|
||||||
|
.build()
|
||||||
|
.unwrap(),
|
||||||
|
opentelemetry_sdk::runtime::Tokio,
|
||||||
)
|
)
|
||||||
|
.with_config(opentelemetry_sdk::trace::Config::default().with_resource(resource()))
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
global::set_tracer_provider(provider.clone());
|
global::set_tracer_provider(provider.clone());
|
||||||
|
|
|
@ -20,6 +20,6 @@ pub struct Lock {
|
||||||
|
|
||||||
impl Lock {
|
impl Lock {
|
||||||
pub fn unlock(&mut self) {
|
pub fn unlock(&mut self) {
|
||||||
self.lock = None
|
self.lock = None;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -48,7 +48,7 @@ impl From<std::string::FromUtf8Error> for CalculationError {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ImpactedMaintainers {
|
impl ImpactedMaintainers {
|
||||||
pub fn calculate(
|
pub async fn calculate(
|
||||||
nix: &Nix,
|
nix: &Nix,
|
||||||
checkout: &Path,
|
checkout: &Path,
|
||||||
paths: &[String],
|
paths: &[String],
|
||||||
|
@ -56,11 +56,11 @@ impl ImpactedMaintainers {
|
||||||
) -> Result<ImpactedMaintainers, CalculationError> {
|
) -> Result<ImpactedMaintainers, CalculationError> {
|
||||||
let mut path_file = NamedTempFile::new()?;
|
let mut path_file = NamedTempFile::new()?;
|
||||||
let pathstr = serde_json::to_string(&paths)?;
|
let pathstr = serde_json::to_string(&paths)?;
|
||||||
write!(path_file, "{}", pathstr)?;
|
write!(path_file, "{pathstr}")?;
|
||||||
|
|
||||||
let mut attr_file = NamedTempFile::new()?;
|
let mut attr_file = NamedTempFile::new()?;
|
||||||
let attrstr = serde_json::to_string(&attributes)?;
|
let attrstr = serde_json::to_string(&attributes)?;
|
||||||
write!(attr_file, "{}", attrstr)?;
|
write!(attr_file, "{attrstr}")?;
|
||||||
|
|
||||||
let mut argstrs: HashMap<&str, &str> = HashMap::new();
|
let mut argstrs: HashMap<&str, &str> = HashMap::new();
|
||||||
argstrs.insert("changedattrsjson", attr_file.path().to_str().unwrap());
|
argstrs.insert("changedattrsjson", attr_file.path().to_str().unwrap());
|
||||||
|
@ -73,21 +73,25 @@ impl ImpactedMaintainers {
|
||||||
&[path_file.path(), attr_file.path()],
|
&[path_file.path(), attr_file.path()],
|
||||||
);
|
);
|
||||||
|
|
||||||
let ret = cmd.output()?;
|
let ret = cmd.output().await?;
|
||||||
|
|
||||||
Ok(serde_json::from_str(&String::from_utf8(ret.stdout)?)?)
|
Ok(serde_json::from_str(&String::from_utf8(ret.stdout)?)?)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub fn maintainers(&self) -> Vec<String> {
|
pub fn maintainers(&self) -> Vec<String> {
|
||||||
self.0.keys().map(|maintainer| maintainer.0.clone())
|
self.0
|
||||||
|
.keys()
|
||||||
|
.map(|maintainer| maintainer.0.clone())
|
||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub fn maintainers_by_package(&self) -> MaintainersByPackage {
|
pub fn maintainers_by_package(&self) -> MaintainersByPackage {
|
||||||
let mut bypkg = MaintainersByPackage(HashMap::new());
|
let mut bypkg = MaintainersByPackage(HashMap::new());
|
||||||
|
|
||||||
for (maintainer, packages) in self.0.iter() {
|
for (maintainer, packages) in &self.0 {
|
||||||
for package in packages.iter() {
|
for package in packages {
|
||||||
bypkg
|
bypkg
|
||||||
.0
|
.0
|
||||||
.entry(package.clone())
|
.entry(package.clone())
|
||||||
|
@ -118,7 +122,7 @@ impl std::fmt::Display for ImpactedMaintainers {
|
||||||
})
|
})
|
||||||
.collect::<Vec<String>>()
|
.collect::<Vec<String>>()
|
||||||
.join("\n");
|
.join("\n");
|
||||||
write!(f, "{}", d)
|
write!(f, "{d}")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -154,15 +158,15 @@ mod tests {
|
||||||
.expect("building the test PR failed");
|
.expect("building the test PR failed");
|
||||||
|
|
||||||
let stderr =
|
let stderr =
|
||||||
String::from_utf8(output.stderr).unwrap_or_else(|err| format!("warning: {}", err));
|
String::from_utf8(output.stderr).unwrap_or_else(|err| format!("warning: {err}"));
|
||||||
println!("{}", stderr);
|
println!("{stderr}");
|
||||||
|
|
||||||
let hash = String::from_utf8(output.stdout).expect("Should just be a hash");
|
let hash = String::from_utf8(output.stdout).expect("Should just be a hash");
|
||||||
return hash.trim().to_owned();
|
return hash.trim().to_owned();
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[tokio::test]
|
||||||
fn example() {
|
async fn example() {
|
||||||
let workingdir = TestScratch::new_dir("test-maintainers-example");
|
let workingdir = TestScratch::new_dir("test-maintainers-example");
|
||||||
|
|
||||||
let bare = TestScratch::new_dir("test-maintainers-example-bare");
|
let bare = TestScratch::new_dir("test-maintainers-example-bare");
|
||||||
|
@ -186,11 +190,11 @@ mod tests {
|
||||||
|
|
||||||
working_co.checkout_ref(OsStr::new(&hash)).unwrap();
|
working_co.checkout_ref(OsStr::new(&hash)).unwrap();
|
||||||
|
|
||||||
let remote = env::var("NIX_REMOTE").unwrap_or("".to_owned());
|
let remote = env::var("NIX_REMOTE").unwrap_or_default();
|
||||||
let nix = Nix::new(SYSTEM.to_owned(), remote, 1800, None);
|
let nix = Nix::new(SYSTEM.to_owned(), remote, 1800, None);
|
||||||
|
|
||||||
let parsed =
|
let parsed =
|
||||||
ImpactedMaintainers::calculate(&nix, &working_co.clone_to(), &paths, &attributes);
|
ImpactedMaintainers::calculate(&nix, &working_co.clone_to(), &paths, &attributes).await;
|
||||||
|
|
||||||
let mut expect = ImpactedMaintainers(HashMap::new());
|
let mut expect = ImpactedMaintainers(HashMap::new());
|
||||||
expect.0.insert(
|
expect.0.insert(
|
||||||
|
|
|
@ -23,6 +23,7 @@ type Exchange = String;
|
||||||
type RoutingKey = String;
|
type RoutingKey = String;
|
||||||
|
|
||||||
impl BuildJob {
|
impl BuildJob {
|
||||||
|
#[must_use]
|
||||||
pub fn new(
|
pub fn new(
|
||||||
repo: Repo,
|
repo: Repo,
|
||||||
change: Change,
|
change: Change,
|
||||||
|
|
|
@ -3,7 +3,7 @@ pub struct BuildLogMsg {
|
||||||
pub system: String,
|
pub system: String,
|
||||||
pub identity: String,
|
pub identity: String,
|
||||||
pub attempt_id: String,
|
pub attempt_id: String,
|
||||||
pub line_number: u64,
|
pub line_number: usize,
|
||||||
pub output: String,
|
pub output: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -23,7 +23,7 @@ impl From<BuildStatus> for String {
|
||||||
BuildStatus::Failure => "Failure".into(),
|
BuildStatus::Failure => "Failure".into(),
|
||||||
BuildStatus::HashMismatch => "A fixed output derivation's hash was incorrect".into(),
|
BuildStatus::HashMismatch => "A fixed output derivation's hash was incorrect".into(),
|
||||||
BuildStatus::TimedOut => "Timed out, unknown build status".into(),
|
BuildStatus::TimedOut => "Timed out, unknown build status".into(),
|
||||||
BuildStatus::UnexpectedError { ref err } => format!("Unexpected error: {}", err),
|
BuildStatus::UnexpectedError { ref err } => format!("Unexpected error: {err}"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -96,6 +96,7 @@ pub enum BuildResult {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl BuildResult {
|
impl BuildResult {
|
||||||
|
#[must_use]
|
||||||
pub fn legacy(&self) -> LegacyBuildResult {
|
pub fn legacy(&self) -> LegacyBuildResult {
|
||||||
// TODO: replace this with simpler structs for specific usecases, since
|
// TODO: replace this with simpler structs for specific usecases, since
|
||||||
// it's decouples the structs from serialization. These can be changed
|
// it's decouples the structs from serialization. These can be changed
|
||||||
|
@ -111,18 +112,8 @@ impl BuildResult {
|
||||||
ref attempted_attrs,
|
ref attempted_attrs,
|
||||||
ref skipped_attrs,
|
ref skipped_attrs,
|
||||||
..
|
..
|
||||||
} => LegacyBuildResult {
|
}
|
||||||
repo: repo.to_owned(),
|
| BuildResult::V1 {
|
||||||
pr: pr.to_owned(),
|
|
||||||
system: system.to_owned(),
|
|
||||||
output: output.to_owned(),
|
|
||||||
attempt_id: attempt_id.to_owned(),
|
|
||||||
request_id: request_id.to_owned(),
|
|
||||||
status: self.status(),
|
|
||||||
attempted_attrs: attempted_attrs.to_owned(),
|
|
||||||
skipped_attrs: skipped_attrs.to_owned(),
|
|
||||||
},
|
|
||||||
BuildResult::V1 {
|
|
||||||
ref repo,
|
ref repo,
|
||||||
ref pr,
|
ref pr,
|
||||||
ref system,
|
ref system,
|
||||||
|
@ -167,14 +158,15 @@ impl BuildResult {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub fn change(&self) -> Change {
|
pub fn change(&self) -> Change {
|
||||||
match self {
|
match self {
|
||||||
BuildResult::Legacy { pr, .. } => pr.to_owned(),
|
BuildResult::Legacy { pr, .. } | BuildResult::V1 { pr, .. } => pr.to_owned(),
|
||||||
BuildResult::V1 { pr, .. } => pr.to_owned(),
|
|
||||||
BuildResult::V2 { change, .. } => change.to_owned(),
|
BuildResult::V2 { change, .. } => change.to_owned(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub fn status(&self) -> BuildStatus {
|
pub fn status(&self) -> BuildStatus {
|
||||||
match *self {
|
match *self {
|
||||||
BuildResult::Legacy {
|
BuildResult::Legacy {
|
||||||
|
@ -210,8 +202,7 @@ mod tests {
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
output,
|
output,
|
||||||
r#"{"tag":"V1","repo":{"owner":"NixOS","name":"nixpkgs","full_name":"NixOS/nixpkgs","clone_url":"https://github.com/nixos/nixpkgs.git"},"pr":{"target_branch":"master","number":42,"head_sha":"0000000000000000000000000000000000000000"},"system":"x86_64-linux","output":["unpacking sources"],"attempt_id":"attempt-id-foo","request_id":"bogus-request-id","status":"Success","skipped_attrs":["AAAAAASomeThingsFailToEvaluate"],"attempted_attrs":["hello"]}"#,
|
r#"{"tag":"V1","repo":{"owner":"NixOS","name":"nixpkgs","full_name":"NixOS/nixpkgs","clone_url":"https://github.com/nixos/nixpkgs.git"},"pr":{"target_branch":"master","number":42,"head_sha":"0000000000000000000000000000000000000000"},"system":"x86_64-linux","output":["unpacking sources"],"attempt_id":"attempt-id-foo","request_id":"bogus-request-id","status":"Success","skipped_attrs":["AAAAAASomeThingsFailToEvaluate"],"attempted_attrs":["hello"]}"#,
|
||||||
"json of: {:?}",
|
"json of: {result:?}"
|
||||||
result
|
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -224,8 +215,7 @@ mod tests {
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
output,
|
output,
|
||||||
r#"{"repo":{"owner":"NixOS","name":"nixpkgs","full_name":"NixOS/nixpkgs","clone_url":"https://github.com/nixos/nixpkgs.git"},"pr":{"target_branch":"master","number":42,"head_sha":"0000000000000000000000000000000000000000"},"system":"x86_64-linux","output":["unpacking sources"],"attempt_id":"attempt-id-foo","request_id":"bogus-request-id","success":true,"status":"Success","skipped_attrs":["AAAAAASomeThingsFailToEvaluate"],"attempted_attrs":["hello"]}"#,
|
r#"{"repo":{"owner":"NixOS","name":"nixpkgs","full_name":"NixOS/nixpkgs","clone_url":"https://github.com/nixos/nixpkgs.git"},"pr":{"target_branch":"master","number":42,"head_sha":"0000000000000000000000000000000000000000"},"system":"x86_64-linux","output":["unpacking sources"],"attempt_id":"attempt-id-foo","request_id":"bogus-request-id","success":true,"status":"Success","skipped_attrs":["AAAAAASomeThingsFailToEvaluate"],"attempted_attrs":["hello"]}"#,
|
||||||
"json of: {:?}",
|
"json of: {result:?}"
|
||||||
result
|
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -238,8 +228,7 @@ mod tests {
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
output,
|
output,
|
||||||
r#"{"repo":{"owner":"NixOS","name":"nixpkgs","full_name":"NixOS/nixpkgs","clone_url":"https://github.com/nixos/nixpkgs.git"},"pr":{"target_branch":"master","number":42,"head_sha":"0000000000000000000000000000000000000000"},"system":"x86_64-linux","output":[],"attempt_id":"attempt-id-foo","request_id":"bogus-request-id","success":null,"status":null,"skipped_attrs":null,"attempted_attrs":null}"#,
|
r#"{"repo":{"owner":"NixOS","name":"nixpkgs","full_name":"NixOS/nixpkgs","clone_url":"https://github.com/nixos/nixpkgs.git"},"pr":{"target_branch":"master","number":42,"head_sha":"0000000000000000000000000000000000000000"},"system":"x86_64-linux","output":[],"attempt_id":"attempt-id-foo","request_id":"bogus-request-id","success":null,"status":null,"skipped_attrs":null,"attempted_attrs":null}"#,
|
||||||
"json of: {:?}",
|
"json of: {result:?}"
|
||||||
result
|
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -252,8 +241,7 @@ mod tests {
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
output,
|
output,
|
||||||
r#"{"repo":{"owner":"NixOS","name":"nixpkgs","full_name":"NixOS/nixpkgs","clone_url":"https://github.com/nixos/nixpkgs.git"},"pr":{"target_branch":"master","number":42,"head_sha":"0000000000000000000000000000000000000000"},"system":"x86_64-linux","output":["unpacking sources"],"attempt_id":"attempt-id-foo","request_id":"bogus-request-id","success":true,"status":null,"skipped_attrs":["AAAAAASomeThingsFailToEvaluate"],"attempted_attrs":["hello"]}"#,
|
r#"{"repo":{"owner":"NixOS","name":"nixpkgs","full_name":"NixOS/nixpkgs","clone_url":"https://github.com/nixos/nixpkgs.git"},"pr":{"target_branch":"master","number":42,"head_sha":"0000000000000000000000000000000000000000"},"system":"x86_64-linux","output":["unpacking sources"],"attempt_id":"attempt-id-foo","request_id":"bogus-request-id","success":true,"status":null,"skipped_attrs":["AAAAAASomeThingsFailToEvaluate"],"attempted_attrs":["hello"]}"#,
|
||||||
"json of: {:?}",
|
"json of: {result:?}"
|
||||||
result
|
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -12,6 +12,7 @@ pub struct EvaluationJob {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl EvaluationJob {
|
impl EvaluationJob {
|
||||||
|
#[must_use]
|
||||||
pub fn is_nixpkgs(&self) -> bool {
|
pub fn is_nixpkgs(&self) -> bool {
|
||||||
self.repo.name == "nixpkgs"
|
self.repo.name == "nixpkgs"
|
||||||
}
|
}
|
||||||
|
@ -20,15 +21,18 @@ impl EvaluationJob {
|
||||||
pub struct Actions {}
|
pub struct Actions {}
|
||||||
|
|
||||||
impl Actions {
|
impl Actions {
|
||||||
pub fn retry_later(&mut self, _job: &EvaluationJob) -> worker::Actions {
|
#[must_use]
|
||||||
|
pub fn retry_later(_job: &EvaluationJob) -> worker::Actions {
|
||||||
vec![worker::Action::NackRequeue]
|
vec![worker::Action::NackRequeue]
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn skip(&mut self, _job: &EvaluationJob) -> worker::Actions {
|
#[must_use]
|
||||||
|
pub fn skip(_job: &EvaluationJob) -> worker::Actions {
|
||||||
vec![worker::Action::Ack]
|
vec![worker::Action::Ack]
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn done(&mut self, _job: &EvaluationJob, mut response: worker::Actions) -> worker::Actions {
|
#[must_use]
|
||||||
|
pub fn done(_job: &EvaluationJob, mut response: worker::Actions) -> worker::Actions {
|
||||||
response.push(worker::Action::Ack);
|
response.push(worker::Action::Ack);
|
||||||
response
|
response
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,4 +1,3 @@
|
||||||
use crate::asynccmd::{AsyncCmd, SpawnedAsyncCmd};
|
|
||||||
use crate::message::buildresult::BuildStatus;
|
use crate::message::buildresult::BuildStatus;
|
||||||
use crate::ofborg::partition_result;
|
use crate::ofborg::partition_result;
|
||||||
|
|
||||||
|
@ -9,9 +8,10 @@ use std::fmt;
|
||||||
use std::fs;
|
use std::fs;
|
||||||
use std::io::{BufRead, BufReader, Seek, SeekFrom};
|
use std::io::{BufRead, BufReader, Seek, SeekFrom};
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use std::process::{Command, Stdio};
|
use std::process::Stdio;
|
||||||
|
|
||||||
use tempfile::tempfile;
|
use tempfile::tempfile;
|
||||||
|
use tokio::process::{Child, Command};
|
||||||
|
|
||||||
#[allow(clippy::upper_case_acronyms)]
|
#[allow(clippy::upper_case_acronyms)]
|
||||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||||
|
@ -43,11 +43,11 @@ pub enum Operation {
|
||||||
impl Operation {
|
impl Operation {
|
||||||
fn command(&self) -> Command {
|
fn command(&self) -> Command {
|
||||||
match *self {
|
match *self {
|
||||||
Operation::Evaluate => Command::new("nix-instantiate"),
|
Operation::Evaluate | Operation::Instantiate => Command::new("nix-instantiate"),
|
||||||
Operation::Instantiate => Command::new("nix-instantiate"),
|
|
||||||
Operation::Build => Command::new("nix-build"),
|
Operation::Build => Command::new("nix-build"),
|
||||||
Operation::QueryPackagesJson => Command::new("nix-env"),
|
Operation::QueryPackagesJson | Operation::QueryPackagesOutputs => {
|
||||||
Operation::QueryPackagesOutputs => Command::new("nix-env"),
|
Command::new("nix-env")
|
||||||
|
}
|
||||||
Operation::NoOp { .. } => Command::new("echo"),
|
Operation::NoOp { .. } => Command::new("echo"),
|
||||||
Operation::Unknown { ref program } => Command::new(program),
|
Operation::Unknown { ref program } => Command::new(program),
|
||||||
}
|
}
|
||||||
|
@ -102,7 +102,7 @@ impl Operation {
|
||||||
Operation::Instantiate => {
|
Operation::Instantiate => {
|
||||||
command.args(["--option", "extra-experimental-features", "no-url-literals"]);
|
command.args(["--option", "extra-experimental-features", "no-url-literals"]);
|
||||||
}
|
}
|
||||||
_ => (),
|
Operation::Unknown { .. } => (),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -115,7 +115,7 @@ impl fmt::Display for Operation {
|
||||||
Operation::QueryPackagesJson => write!(f, "nix-env -qa --json"),
|
Operation::QueryPackagesJson => write!(f, "nix-env -qa --json"),
|
||||||
Operation::QueryPackagesOutputs => write!(f, "nix-env -qaP --no-name --out-path"),
|
Operation::QueryPackagesOutputs => write!(f, "nix-env -qaP --no-name --out-path"),
|
||||||
Operation::NoOp { ref operation } => operation.fmt(f),
|
Operation::NoOp { ref operation } => operation.fmt(f),
|
||||||
Operation::Unknown { ref program } => write!(f, "{}", program),
|
Operation::Unknown { ref program } => write!(f, "{program}"),
|
||||||
Operation::Evaluate => write!(f, "nix-instantiate --strict --json ..."),
|
Operation::Evaluate => write!(f, "nix-instantiate --strict --json ..."),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -131,6 +131,7 @@ pub struct Nix {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Nix {
|
impl Nix {
|
||||||
|
#[must_use]
|
||||||
pub fn new(
|
pub fn new(
|
||||||
system: String,
|
system: String,
|
||||||
remote: String,
|
remote: String,
|
||||||
|
@ -146,54 +147,62 @@ impl Nix {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub fn with_system(&self, system: String) -> Nix {
|
pub fn with_system(&self, system: String) -> Nix {
|
||||||
let mut n = self.clone();
|
let mut n = self.clone();
|
||||||
n.system = system;
|
n.system = system;
|
||||||
n
|
n
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub fn with_limited_supported_systems(&self) -> Nix {
|
pub fn with_limited_supported_systems(&self) -> Nix {
|
||||||
let mut n = self.clone();
|
let mut n = self.clone();
|
||||||
n.limit_supported_systems = true;
|
n.limit_supported_systems = true;
|
||||||
n
|
n
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub fn without_limited_supported_systems(&self) -> Nix {
|
pub fn without_limited_supported_systems(&self) -> Nix {
|
||||||
let mut n = self.clone();
|
let mut n = self.clone();
|
||||||
n.limit_supported_systems = false;
|
n.limit_supported_systems = false;
|
||||||
n
|
n
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn safely_partition_instantiable_attrs(
|
pub async fn safely_partition_instantiable_attrs(
|
||||||
&self,
|
&self,
|
||||||
nixpkgs: &Path,
|
nixpkgs: &Path,
|
||||||
file: File,
|
file: File,
|
||||||
attrs: Vec<String>,
|
attrs: Vec<String>,
|
||||||
) -> (Vec<String>, Vec<(String, Vec<String>)>) {
|
) -> (Vec<String>, Vec<(String, Vec<String>)>) {
|
||||||
let attr_instantiations: Vec<Result<String, (String, Vec<String>)>> = attrs
|
let mut attr_instantiations: Vec<Result<String, (String, Vec<String>)>> = vec![];
|
||||||
.into_iter()
|
|
||||||
.map(
|
for attr in attrs {
|
||||||
|attr| match self.safely_instantiate_attrs(nixpkgs, file, vec![attr.clone()]) {
|
attr_instantiations.push(
|
||||||
|
match self
|
||||||
|
.safely_instantiate_attrs(nixpkgs, file, vec![attr.clone()])
|
||||||
|
.await
|
||||||
|
{
|
||||||
Ok(_) => Ok(attr),
|
Ok(_) => Ok(attr),
|
||||||
Err(f) => Err((attr, lines_from_file(f))),
|
Err(f) => Err((attr, lines_from_file(f))),
|
||||||
},
|
},
|
||||||
)
|
);
|
||||||
.collect();
|
}
|
||||||
|
|
||||||
partition_result(attr_instantiations)
|
partition_result(attr_instantiations)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn safely_instantiate_attrs(
|
pub async fn safely_instantiate_attrs(
|
||||||
&self,
|
&self,
|
||||||
nixpkgs: &Path,
|
nixpkgs: &Path,
|
||||||
file: File,
|
file: File,
|
||||||
attrs: Vec<String>,
|
attrs: Vec<String>,
|
||||||
) -> Result<fs::File, fs::File> {
|
) -> Result<fs::File, fs::File> {
|
||||||
let mut command = self.safe_command::<&OsStr>(&Operation::Instantiate, nixpkgs, &[], &[]);
|
let mut command = self.safe_command::<&OsStr>(&Operation::Instantiate, nixpkgs, &[], &[]);
|
||||||
self.set_attrs_command(&mut command, file, attrs);
|
Self::set_attrs_command(&mut command, file, attrs);
|
||||||
self.run(command, true)
|
self.run(command, true).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub fn safely_evaluate_expr_cmd(
|
pub fn safely_evaluate_expr_cmd(
|
||||||
&self,
|
&self,
|
||||||
nixpkgs: &Path,
|
nixpkgs: &Path,
|
||||||
|
@ -213,31 +222,32 @@ impl Nix {
|
||||||
self.safe_command(&Operation::Evaluate, nixpkgs, &attrargs, extra_paths)
|
self.safe_command(&Operation::Evaluate, nixpkgs, &attrargs, extra_paths)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn safely_build_attrs(
|
pub async fn safely_build_attrs(
|
||||||
&self,
|
&self,
|
||||||
nixpkgs: &Path,
|
nixpkgs: &Path,
|
||||||
file: File,
|
file: File,
|
||||||
attrs: Vec<String>,
|
attrs: Vec<String>,
|
||||||
) -> Result<fs::File, fs::File> {
|
) -> Result<fs::File, fs::File> {
|
||||||
let mut command = self.safe_command::<&OsStr>(&Operation::Build, nixpkgs, &[], &[]);
|
let mut command = self.safe_command::<&OsStr>(&Operation::Build, nixpkgs, &[], &[]);
|
||||||
self.set_attrs_command(&mut command, file, attrs);
|
Self::set_attrs_command(&mut command, file, attrs);
|
||||||
self.run(command, true)
|
self.run(command, true).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub fn safely_build_attrs_async(
|
pub fn safely_build_attrs_async(
|
||||||
&self,
|
&self,
|
||||||
nixpkgs: &Path,
|
nixpkgs: &Path,
|
||||||
file: File,
|
file: File,
|
||||||
attrs: Vec<String>,
|
attrs: Vec<String>,
|
||||||
) -> SpawnedAsyncCmd {
|
) -> Child {
|
||||||
let mut command = self.safe_command::<&OsStr>(&Operation::Build, nixpkgs, &[], &[]);
|
let mut command = self.safe_command::<&OsStr>(&Operation::Build, nixpkgs, &[], &[]);
|
||||||
self.set_attrs_command(&mut command, file, attrs);
|
Nix::set_attrs_command(&mut command, file, attrs);
|
||||||
AsyncCmd::new(command).spawn()
|
command.spawn().expect("Failed to run Nix")
|
||||||
}
|
}
|
||||||
|
|
||||||
fn set_attrs_command(&self, command: &mut Command, file: File, attrs: Vec<String>) {
|
fn set_attrs_command(command: &mut Command, file: File, attrs: Vec<String>) {
|
||||||
let mut args: Vec<String> = Vec::with_capacity(3 + (attrs.len() * 2));
|
let mut args: Vec<String> = Vec::with_capacity(3 + (attrs.len() * 2));
|
||||||
args.push(format!("{}", file));
|
args.push(format!("{file}"));
|
||||||
for attr in attrs {
|
for attr in attrs {
|
||||||
args.push(String::from("-A"));
|
args.push(String::from("-A"));
|
||||||
args.push(attr);
|
args.push(attr);
|
||||||
|
@ -252,7 +262,7 @@ impl Nix {
|
||||||
command.args(args);
|
command.args(args);
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn safely(
|
pub async fn safely(
|
||||||
&self,
|
&self,
|
||||||
op: &Operation,
|
op: &Operation,
|
||||||
nixpkgs: &Path,
|
nixpkgs: &Path,
|
||||||
|
@ -260,9 +270,10 @@ impl Nix {
|
||||||
keep_stdout: bool,
|
keep_stdout: bool,
|
||||||
) -> Result<fs::File, fs::File> {
|
) -> Result<fs::File, fs::File> {
|
||||||
self.run(self.safe_command(op, nixpkgs, &args, &[]), keep_stdout)
|
self.run(self.safe_command(op, nixpkgs, &args, &[]), keep_stdout)
|
||||||
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn run(&self, mut cmd: Command, keep_stdout: bool) -> Result<fs::File, fs::File> {
|
pub async fn run(&self, mut cmd: Command, keep_stdout: bool) -> Result<fs::File, fs::File> {
|
||||||
let stderr = tempfile().expect("Fetching a stderr tempfile");
|
let stderr = tempfile().expect("Fetching a stderr tempfile");
|
||||||
let mut reader = stderr.try_clone().expect("Cloning stderr to the reader");
|
let mut reader = stderr.try_clone().expect("Cloning stderr to the reader");
|
||||||
|
|
||||||
|
@ -276,6 +287,7 @@ impl Nix {
|
||||||
.stdout(stdout)
|
.stdout(stdout)
|
||||||
.stderr(Stdio::from(stderr))
|
.stderr(Stdio::from(stderr))
|
||||||
.status()
|
.status()
|
||||||
|
.await
|
||||||
.expect("Running a program ...");
|
.expect("Running a program ...");
|
||||||
|
|
||||||
reader
|
reader
|
||||||
|
@ -289,7 +301,7 @@ impl Nix {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn run_stderr_stdout(&self, mut cmd: Command) -> (bool, fs::File, fs::File) {
|
pub async fn run_stderr_stdout(&self, mut cmd: Command) -> (bool, fs::File, fs::File) {
|
||||||
let stdout_file = tempfile().expect("Fetching a stdout tempfile");
|
let stdout_file = tempfile().expect("Fetching a stdout tempfile");
|
||||||
let mut stdout_reader = stdout_file
|
let mut stdout_reader = stdout_file
|
||||||
.try_clone()
|
.try_clone()
|
||||||
|
@ -304,6 +316,7 @@ impl Nix {
|
||||||
.stdout(Stdio::from(stdout_file))
|
.stdout(Stdio::from(stdout_file))
|
||||||
.stderr(Stdio::from(stderr_file))
|
.stderr(Stdio::from(stderr_file))
|
||||||
.status()
|
.status()
|
||||||
|
.await
|
||||||
.expect("Running a program ...");
|
.expect("Running a program ...");
|
||||||
|
|
||||||
stdout_reader
|
stdout_reader
|
||||||
|
@ -379,28 +392,29 @@ fn lines_from_file(file: fs::File) -> Vec<String> {
|
||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub fn is_user_setting_warning(line: &str) -> bool {
|
pub fn is_user_setting_warning(line: &str) -> bool {
|
||||||
let line = line.trim();
|
let line = line.trim();
|
||||||
line.starts_with("warning: ignoring the user-specified setting '")
|
line.starts_with("warning: ignoring the user-specified setting '")
|
||||||
&& line.ends_with("because it is a restricted setting and you are not a trusted user")
|
&& line.ends_with("because it is a restricted setting and you are not a trusted user")
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn wait_for_build_status(spawned: SpawnedAsyncCmd) -> BuildStatus {
|
pub async fn wait_for_build_status(mut spawned: Child) -> BuildStatus {
|
||||||
match spawned.wait() {
|
match spawned.wait().await {
|
||||||
Ok(s) => match s.code() {
|
Ok(s) => match s.code() {
|
||||||
Some(0) => BuildStatus::Success,
|
Some(0) => BuildStatus::Success,
|
||||||
Some(100) => BuildStatus::Failure, // nix permanent failure
|
Some(100) => BuildStatus::Failure, // nix permanent failure
|
||||||
Some(101) => BuildStatus::TimedOut, // nix build timedout
|
Some(101) => BuildStatus::TimedOut, // nix build timedout
|
||||||
Some(102) => BuildStatus::HashMismatch, // Fixed Output Derivation's hash was wrong
|
Some(102) => BuildStatus::HashMismatch, // Fixed Output Derivation's hash was wrong
|
||||||
Some(i) => BuildStatus::UnexpectedError {
|
Some(i) => BuildStatus::UnexpectedError {
|
||||||
err: format!("command failed with exit code {}", i),
|
err: format!("command failed with exit code {i}"),
|
||||||
},
|
},
|
||||||
None => BuildStatus::UnexpectedError {
|
None => BuildStatus::UnexpectedError {
|
||||||
err: "unexpected build failure".into(),
|
err: "unexpected build failure".into(),
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
e => BuildStatus::UnexpectedError {
|
e => BuildStatus::UnexpectedError {
|
||||||
err: format!("failed on interior command {:?}", e),
|
err: format!("failed on interior command {e:?}"),
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -420,7 +434,7 @@ mod tests {
|
||||||
let path = env::var("PATH").unwrap();
|
let path = env::var("PATH").unwrap();
|
||||||
let test_path = format!("{}/test-nix/bin:{}", env!("CARGO_MANIFEST_DIR"), path);
|
let test_path = format!("{}/test-nix/bin:{}", env!("CARGO_MANIFEST_DIR"), path);
|
||||||
env::set_var("PATH", test_path);
|
env::set_var("PATH", test_path);
|
||||||
let remote = env::var("NIX_REMOTE").unwrap_or("".to_owned());
|
let remote = env::var("NIX_REMOTE").unwrap_or_default();
|
||||||
Nix::new(SYSTEM.to_owned(), remote, 1800, None)
|
Nix::new(SYSTEM.to_owned(), remote, 1800, None)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -467,15 +481,14 @@ mod tests {
|
||||||
Fail,
|
Fail,
|
||||||
}
|
}
|
||||||
|
|
||||||
fn assert_run(res: Result<fs::File, fs::File>, expected: Expect, require: Vec<&str>) {
|
fn assert_run(res: Result<fs::File, fs::File>, expected: &Expect, require: Vec<&str>) {
|
||||||
let expectation_held: bool = match expected {
|
let expectation_held: bool = match expected {
|
||||||
Expect::Pass => res.is_ok(),
|
Expect::Pass => res.is_ok(),
|
||||||
Expect::Fail => res.is_err(),
|
Expect::Fail => res.is_err(),
|
||||||
};
|
};
|
||||||
|
|
||||||
let file: fs::File = match res {
|
let file: fs::File = match res {
|
||||||
Ok(file) => file,
|
Ok(file) | Err(file) => file,
|
||||||
Err(file) => file,
|
|
||||||
};
|
};
|
||||||
|
|
||||||
let lines = lines_from_file(file);
|
let lines = lines_from_file(file);
|
||||||
|
@ -483,7 +496,7 @@ mod tests {
|
||||||
let buildlog = lines
|
let buildlog = lines
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.map(|line| strip_ansi(&line))
|
.map(|line| strip_ansi(&line))
|
||||||
.map(|line| format!(" | {}", line))
|
.map(|line| format!(" | {line}"))
|
||||||
.collect::<Vec<String>>()
|
.collect::<Vec<String>>()
|
||||||
.join("\n");
|
.join("\n");
|
||||||
|
|
||||||
|
@ -491,7 +504,7 @@ mod tests {
|
||||||
let mut missed_requirements: usize = 0;
|
let mut missed_requirements: usize = 0;
|
||||||
let requirements_held: Vec<Result<String, String>> = require
|
let requirements_held: Vec<Result<String, String>> = require
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.map(|line| line.to_owned())
|
.map(std::borrow::ToOwned::to_owned)
|
||||||
.map(|line| {
|
.map(|line| {
|
||||||
if buildlog.contains(&line) {
|
if buildlog.contains(&line) {
|
||||||
Ok(line)
|
Ok(line)
|
||||||
|
@ -502,36 +515,34 @@ mod tests {
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
let mut prefixes: Vec<String> = vec!["".to_owned(), "".to_owned()];
|
let mut prefixes: Vec<String> = vec![String::new(), String::new()];
|
||||||
|
|
||||||
if !expectation_held {
|
if expectation_held {
|
||||||
prefixes.push(format!(
|
prefixes.push(format!("The run was expected to {expected:?}, and did."));
|
||||||
"The run was expected to {:?}, but did not.",
|
|
||||||
expected
|
|
||||||
));
|
|
||||||
prefixes.push("".to_owned());
|
|
||||||
} else {
|
} else {
|
||||||
prefixes.push(format!("The run was expected to {:?}, and did.", expected));
|
prefixes.push(format!(
|
||||||
prefixes.push("".to_owned());
|
"The run was expected to {expected:?}, but did not."
|
||||||
|
));
|
||||||
}
|
}
|
||||||
|
prefixes.push(String::new());
|
||||||
|
|
||||||
let mut suffixes = vec![
|
let mut suffixes = vec![
|
||||||
"".to_owned(),
|
String::new(),
|
||||||
format!(
|
format!(
|
||||||
"{} out of {} required lines matched.",
|
"{} out of {} required lines matched.",
|
||||||
(total_requirements - missed_requirements),
|
(total_requirements - missed_requirements),
|
||||||
total_requirements
|
total_requirements
|
||||||
),
|
),
|
||||||
"".to_owned(),
|
String::new(),
|
||||||
];
|
];
|
||||||
|
|
||||||
for expected_line in requirements_held {
|
for expected_line in requirements_held {
|
||||||
suffixes.push(format!(" - {:?}", expected_line));
|
suffixes.push(format!(" - {expected_line:?}"));
|
||||||
}
|
}
|
||||||
suffixes.push("".to_owned());
|
suffixes.push(String::new());
|
||||||
|
|
||||||
let output_blocks: Vec<Vec<String>> =
|
let output_blocks: Vec<Vec<String>> =
|
||||||
vec![prefixes, vec![buildlog, "".to_owned()], suffixes];
|
vec![prefixes, vec![buildlog, String::new()], suffixes];
|
||||||
|
|
||||||
let output_blocks_strings: Vec<String> = output_blocks
|
let output_blocks_strings: Vec<String> = output_blocks
|
||||||
.into_iter()
|
.into_iter()
|
||||||
|
@ -546,70 +557,78 @@ mod tests {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[tokio::test]
|
||||||
fn test_build_operations() {
|
async fn test_build_operations() {
|
||||||
let nix = nix();
|
let nix = nix();
|
||||||
let op = noop(Operation::Build);
|
let op = noop(Operation::Build);
|
||||||
assert_eq!(op.to_string(), "nix-build");
|
assert_eq!(op.to_string(), "nix-build");
|
||||||
|
|
||||||
let ret: Result<fs::File, fs::File> = nix.run(
|
let ret: Result<fs::File, fs::File> = nix
|
||||||
nix.safe_command(&op, build_path().as_path(), &["--version"], &[]),
|
.run(
|
||||||
true,
|
nix.safe_command(&op, build_path().as_path(), &["--version"], &[]),
|
||||||
);
|
true,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
assert_run(
|
assert_run(
|
||||||
ret,
|
ret,
|
||||||
Expect::Pass,
|
&Expect::Pass,
|
||||||
vec!["--no-out-link --keep-going", "--version"],
|
vec!["--no-out-link --keep-going", "--version"],
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[tokio::test]
|
||||||
fn test_instantiate_operation() {
|
async fn test_instantiate_operation() {
|
||||||
let nix = nix();
|
let nix = nix();
|
||||||
let op = noop(Operation::Instantiate);
|
let op = noop(Operation::Instantiate);
|
||||||
assert_eq!(op.to_string(), "nix-instantiate");
|
assert_eq!(op.to_string(), "nix-instantiate");
|
||||||
|
|
||||||
let ret: Result<fs::File, fs::File> = nix.run(
|
let ret: Result<fs::File, fs::File> = nix
|
||||||
nix.safe_command(&op, build_path().as_path(), &["--version"], &[]),
|
.run(
|
||||||
true,
|
nix.safe_command(&op, build_path().as_path(), &["--version"], &[]),
|
||||||
);
|
true,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
assert_run(ret, Expect::Pass, vec!["--version"]);
|
assert_run(ret, &Expect::Pass, vec!["--version"]);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[tokio::test]
|
||||||
fn test_query_packages_json() {
|
async fn test_query_packages_json() {
|
||||||
let nix = nix();
|
let nix = nix();
|
||||||
let op = noop(Operation::QueryPackagesJson);
|
let op = noop(Operation::QueryPackagesJson);
|
||||||
assert_eq!(op.to_string(), "nix-env -qa --json");
|
assert_eq!(op.to_string(), "nix-env -qa --json");
|
||||||
|
|
||||||
let ret: Result<fs::File, fs::File> = nix.run(
|
let ret: Result<fs::File, fs::File> = nix
|
||||||
nix.safe_command(&op, build_path().as_path(), &["--version"], &[]),
|
.run(
|
||||||
true,
|
nix.safe_command(&op, build_path().as_path(), &["--version"], &[]),
|
||||||
);
|
true,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
assert_run(
|
assert_run(
|
||||||
ret,
|
ret,
|
||||||
Expect::Pass,
|
&Expect::Pass,
|
||||||
vec!["--query --available --json", "--version"],
|
vec!["--query --available --json", "--version"],
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[tokio::test]
|
||||||
fn test_query_packages_outputs() {
|
async fn test_query_packages_outputs() {
|
||||||
let nix = nix();
|
let nix = nix();
|
||||||
let op = noop(Operation::QueryPackagesOutputs);
|
let op = noop(Operation::QueryPackagesOutputs);
|
||||||
assert_eq!(op.to_string(), "nix-env -qaP --no-name --out-path");
|
assert_eq!(op.to_string(), "nix-env -qaP --no-name --out-path");
|
||||||
|
|
||||||
let ret: Result<fs::File, fs::File> = nix.run(
|
let ret: Result<fs::File, fs::File> = nix
|
||||||
nix.safe_command(&op, build_path().as_path(), &["--version"], &[]),
|
.run(
|
||||||
true,
|
nix.safe_command(&op, build_path().as_path(), &["--version"], &[]),
|
||||||
);
|
true,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
assert_run(
|
assert_run(
|
||||||
ret,
|
ret,
|
||||||
Expect::Pass,
|
&Expect::Pass,
|
||||||
vec![
|
vec![
|
||||||
"--query --available --no-name --attr-path --out-path",
|
"--query --available --no-name --attr-path --out-path",
|
||||||
"--version",
|
"--version",
|
||||||
|
@ -617,18 +636,20 @@ mod tests {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[tokio::test]
|
||||||
fn safe_command_environment() {
|
async fn safe_command_environment() {
|
||||||
let nix = nix();
|
let nix = nix();
|
||||||
|
|
||||||
let ret: Result<fs::File, fs::File> = nix.run(
|
let ret: Result<fs::File, fs::File> = nix
|
||||||
nix.safe_command::<&OsStr>(&env_noop(), build_path().as_path(), &[], &[]),
|
.run(
|
||||||
true,
|
nix.safe_command::<&OsStr>(&env_noop(), build_path().as_path(), &[], &[]),
|
||||||
);
|
true,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
assert_run(
|
assert_run(
|
||||||
ret,
|
ret,
|
||||||
Expect::Pass,
|
&Expect::Pass,
|
||||||
vec![
|
vec![
|
||||||
"HOME=/homeless-shelter",
|
"HOME=/homeless-shelter",
|
||||||
"NIX_PATH=ofborg-nixpkgs-pr=",
|
"NIX_PATH=ofborg-nixpkgs-pr=",
|
||||||
|
@ -638,19 +659,21 @@ mod tests {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[tokio::test]
|
||||||
fn safe_command_custom_gc() {
|
async fn safe_command_custom_gc() {
|
||||||
let remote = env::var("NIX_REMOTE").unwrap_or("".to_owned());
|
let remote = env::var("NIX_REMOTE").unwrap_or_default();
|
||||||
let nix = Nix::new(SYSTEM.to_owned(), remote, 1800, Some("4g".to_owned()));
|
let nix = Nix::new(SYSTEM.to_owned(), remote, 1800, Some("4g".to_owned()));
|
||||||
|
|
||||||
let ret: Result<fs::File, fs::File> = nix.run(
|
let ret: Result<fs::File, fs::File> = nix
|
||||||
nix.safe_command::<&OsStr>(&env_noop(), build_path().as_path(), &[], &[]),
|
.run(
|
||||||
true,
|
nix.safe_command::<&OsStr>(&env_noop(), build_path().as_path(), &[], &[]),
|
||||||
);
|
true,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
assert_run(
|
assert_run(
|
||||||
ret,
|
ret,
|
||||||
Expect::Pass,
|
&Expect::Pass,
|
||||||
vec![
|
vec![
|
||||||
"HOME=/homeless-shelter",
|
"HOME=/homeless-shelter",
|
||||||
"NIX_PATH=ofborg-nixpkgs-pr=",
|
"NIX_PATH=ofborg-nixpkgs-pr=",
|
||||||
|
@ -661,57 +684,59 @@ mod tests {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[tokio::test]
|
||||||
fn safe_command_options() {
|
async fn safe_command_options() {
|
||||||
let nix = nix();
|
let nix = nix();
|
||||||
let op = noop(Operation::Build);
|
let op = noop(Operation::Build);
|
||||||
|
|
||||||
let ret: Result<fs::File, fs::File> = nix.run(
|
let ret: Result<fs::File, fs::File> = nix
|
||||||
nix.safe_command::<&OsStr>(&op, build_path().as_path(), &[], &[]),
|
.run(
|
||||||
true,
|
nix.safe_command::<&OsStr>(&op, build_path().as_path(), &[], &[]),
|
||||||
);
|
true,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
assert_run(
|
assert_run(
|
||||||
ret,
|
ret,
|
||||||
Expect::Pass,
|
&Expect::Pass,
|
||||||
vec!["--option restrict-eval true", "--option build-timeout 1800"],
|
vec!["--option restrict-eval true", "--option build-timeout 1800"],
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[tokio::test]
|
||||||
fn set_attrs_nixpkgs() {
|
async fn set_attrs_nixpkgs() {
|
||||||
let nix = nix();
|
let nix = nix();
|
||||||
let op = noop(Operation::Build);
|
let op = noop(Operation::Build);
|
||||||
|
|
||||||
let mut command = nix.safe_command::<&OsStr>(&op, build_path().as_path(), &[], &[]);
|
let mut command = nix.safe_command::<&OsStr>(&op, build_path().as_path(), &[], &[]);
|
||||||
nix.set_attrs_command(
|
Nix::set_attrs_command(
|
||||||
&mut command,
|
&mut command,
|
||||||
File::DefaultNixpkgs,
|
File::DefaultNixpkgs,
|
||||||
vec!["foo".into(), "bar".into()],
|
vec!["foo".into(), "bar".into()],
|
||||||
);
|
);
|
||||||
|
|
||||||
let ret: Result<fs::File, fs::File> = nix.run(command, true);
|
let ret: Result<fs::File, fs::File> = nix.run(command, true).await;
|
||||||
|
|
||||||
assert_run(ret, Expect::Pass, vec!["./default.nix", "-A foo -A bar"]);
|
assert_run(ret, &Expect::Pass, vec!["./default.nix", "-A foo -A bar"]);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[tokio::test]
|
||||||
fn set_attrs_nixos() {
|
async fn set_attrs_nixos() {
|
||||||
let nix = nix();
|
let nix = nix();
|
||||||
let op = noop(Operation::Instantiate);
|
let op = noop(Operation::Instantiate);
|
||||||
|
|
||||||
let mut command = nix.safe_command::<&OsStr>(&op, build_path().as_path(), &[], &[]);
|
let mut command = nix.safe_command::<&OsStr>(&op, build_path().as_path(), &[], &[]);
|
||||||
nix.set_attrs_command(
|
Nix::set_attrs_command(
|
||||||
&mut command,
|
&mut command,
|
||||||
File::ReleaseNixOS,
|
File::ReleaseNixOS,
|
||||||
vec!["foo".into(), "bar".into()],
|
vec!["foo".into(), "bar".into()],
|
||||||
);
|
);
|
||||||
|
|
||||||
let ret: Result<fs::File, fs::File> = nix.run(command, true);
|
let ret: Result<fs::File, fs::File> = nix.run(command, true).await;
|
||||||
|
|
||||||
assert_run(
|
assert_run(
|
||||||
ret,
|
ret,
|
||||||
Expect::Pass,
|
&Expect::Pass,
|
||||||
vec![
|
vec![
|
||||||
"./nixos/release.nix",
|
"./nixos/release.nix",
|
||||||
"--arg nixpkgs { outPath=./.; revCount=999999; shortRev=\"ofborg\"; rev=\"0000000000000000000000000000000000000000\"; }",
|
"--arg nixpkgs { outPath=./.; revCount=999999; shortRev=\"ofborg\"; rev=\"0000000000000000000000000000000000000000\"; }",
|
||||||
|
@ -719,36 +744,40 @@ mod tests {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[tokio::test]
|
||||||
fn safely_build_attrs_success() {
|
async fn safely_build_attrs_success() {
|
||||||
let nix = nix();
|
let nix = nix();
|
||||||
|
|
||||||
let ret: Result<fs::File, fs::File> = nix.safely_build_attrs(
|
let ret: Result<fs::File, fs::File> = nix
|
||||||
build_path().as_path(),
|
.safely_build_attrs(
|
||||||
File::DefaultNixpkgs,
|
build_path().as_path(),
|
||||||
vec![String::from("success")],
|
File::DefaultNixpkgs,
|
||||||
);
|
vec![String::from("success")],
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
assert_run(
|
assert_run(
|
||||||
ret,
|
ret,
|
||||||
Expect::Pass,
|
&Expect::Pass,
|
||||||
vec!["-success.drv", "building ", "hi", "-success"],
|
vec!["-success.drv", "building ", "hi", "-success"],
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[tokio::test]
|
||||||
fn safely_build_attrs_failure() {
|
async fn safely_build_attrs_failure() {
|
||||||
let nix = nix();
|
let nix = nix();
|
||||||
|
|
||||||
let ret: Result<fs::File, fs::File> = nix.safely_build_attrs(
|
let ret: Result<fs::File, fs::File> = nix
|
||||||
build_path().as_path(),
|
.safely_build_attrs(
|
||||||
File::DefaultNixpkgs,
|
build_path().as_path(),
|
||||||
vec![String::from("failed")],
|
File::DefaultNixpkgs,
|
||||||
);
|
vec![String::from("failed")],
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
assert_run(
|
assert_run(
|
||||||
ret,
|
ret,
|
||||||
Expect::Fail,
|
&Expect::Fail,
|
||||||
vec![
|
vec![
|
||||||
"-failed.drv",
|
"-failed.drv",
|
||||||
"building ",
|
"building ",
|
||||||
|
@ -758,8 +787,8 @@ mod tests {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[tokio::test]
|
||||||
fn partition_instantiable_attributes() {
|
async fn partition_instantiable_attributes() {
|
||||||
let nix = nix();
|
let nix = nix();
|
||||||
|
|
||||||
let ret: (Vec<String>, Vec<(String, Vec<String>)>) = nix
|
let ret: (Vec<String>, Vec<(String, Vec<String>)>) = nix
|
||||||
|
@ -771,7 +800,8 @@ mod tests {
|
||||||
String::from("passes-instantiation"),
|
String::from("passes-instantiation"),
|
||||||
String::from("missing-attr"),
|
String::from("missing-attr"),
|
||||||
],
|
],
|
||||||
);
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
assert_eq!(ret.0, vec!["passes-instantiation"]);
|
assert_eq!(ret.0, vec!["passes-instantiation"]);
|
||||||
|
|
||||||
|
@ -790,80 +820,90 @@ mod tests {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[tokio::test]
|
||||||
fn safely_instantiate_attrs_failure() {
|
async fn safely_instantiate_attrs_failure() {
|
||||||
let nix = nix();
|
let nix = nix();
|
||||||
|
|
||||||
let ret: Result<fs::File, fs::File> = nix.safely_instantiate_attrs(
|
let ret: Result<fs::File, fs::File> = nix
|
||||||
individual_eval_path().as_path(),
|
.safely_instantiate_attrs(
|
||||||
File::DefaultNixpkgs,
|
individual_eval_path().as_path(),
|
||||||
vec![String::from("fails-instantiation")],
|
File::DefaultNixpkgs,
|
||||||
);
|
vec![String::from("fails-instantiation")],
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
assert_run(
|
assert_run(
|
||||||
ret,
|
ret,
|
||||||
Expect::Fail,
|
&Expect::Fail,
|
||||||
vec!["You just can't", "assertion", "failed"],
|
vec!["You just can't", "assertion", "failed"],
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[tokio::test]
|
||||||
fn safely_instantiate_attrs_success() {
|
async fn safely_instantiate_attrs_success() {
|
||||||
let nix = nix();
|
let nix = nix();
|
||||||
|
|
||||||
let ret: Result<fs::File, fs::File> = nix.safely_instantiate_attrs(
|
let ret: Result<fs::File, fs::File> = nix
|
||||||
individual_eval_path().as_path(),
|
.safely_instantiate_attrs(
|
||||||
File::DefaultNixpkgs,
|
|
||||||
vec![String::from("passes-instantiation")],
|
|
||||||
);
|
|
||||||
|
|
||||||
assert_run(ret, Expect::Pass, vec!["-passes-instantiation.drv"]);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn safely_evaluate_expr_success() {
|
|
||||||
let nix = nix();
|
|
||||||
|
|
||||||
let ret: Result<fs::File, fs::File> = nix.run(
|
|
||||||
nix.safely_evaluate_expr_cmd(
|
|
||||||
individual_eval_path().as_path(),
|
individual_eval_path().as_path(),
|
||||||
r#"{ foo ? "bar" }: "The magic value is ${foo}""#,
|
File::DefaultNixpkgs,
|
||||||
[("foo", "tux")].iter().cloned().collect(),
|
vec![String::from("passes-instantiation")],
|
||||||
&[],
|
)
|
||||||
),
|
.await;
|
||||||
true,
|
|
||||||
);
|
|
||||||
|
|
||||||
assert_run(ret, Expect::Pass, vec!["The magic value is tux"]);
|
assert_run(ret, &Expect::Pass, vec!["-passes-instantiation.drv"]);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[tokio::test]
|
||||||
fn strict_sandboxing() {
|
async fn safely_evaluate_expr_success() {
|
||||||
let ret: Result<fs::File, fs::File> = nix().safely_build_attrs(
|
let nix = nix();
|
||||||
build_path().as_path(),
|
|
||||||
File::DefaultNixpkgs,
|
let ret: Result<fs::File, fs::File> = nix
|
||||||
vec![String::from("sandbox-violation")],
|
.run(
|
||||||
);
|
nix.safely_evaluate_expr_cmd(
|
||||||
|
individual_eval_path().as_path(),
|
||||||
|
r#"{ foo ? "bar" }: "The magic value is ${foo}""#,
|
||||||
|
[("foo", "tux")].iter().copied().collect(),
|
||||||
|
&[],
|
||||||
|
),
|
||||||
|
true,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
assert_run(ret, &Expect::Pass, vec!["The magic value is tux"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn strict_sandboxing() {
|
||||||
|
let ret: Result<fs::File, fs::File> = nix()
|
||||||
|
.safely_build_attrs(
|
||||||
|
build_path().as_path(),
|
||||||
|
File::DefaultNixpkgs,
|
||||||
|
vec![String::from("sandbox-violation")],
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
assert_run(
|
assert_run(
|
||||||
ret,
|
ret,
|
||||||
Expect::Fail,
|
&Expect::Fail,
|
||||||
vec!["access to absolute path", "is forbidden in restricted mode"],
|
vec!["access to absolute path", "is forbidden in restricted mode"],
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[tokio::test]
|
||||||
fn instantiation_success() {
|
async fn instantiation_success() {
|
||||||
let ret: Result<fs::File, fs::File> = nix().safely(
|
let ret: Result<fs::File, fs::File> = nix()
|
||||||
&Operation::Instantiate,
|
.safely(
|
||||||
passing_eval_path().as_path(),
|
&Operation::Instantiate,
|
||||||
vec![],
|
passing_eval_path().as_path(),
|
||||||
true,
|
vec![],
|
||||||
);
|
true,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
assert_run(
|
assert_run(
|
||||||
ret,
|
ret,
|
||||||
Expect::Pass,
|
&Expect::Pass,
|
||||||
vec![
|
vec![
|
||||||
"the result might be removed by the garbage collector",
|
"the result might be removed by the garbage collector",
|
||||||
"-failed.drv",
|
"-failed.drv",
|
||||||
|
@ -872,18 +912,20 @@ mod tests {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[tokio::test]
|
||||||
fn instantiation_nixpkgs_restricted_mode() {
|
async fn instantiation_nixpkgs_restricted_mode() {
|
||||||
let ret: Result<fs::File, fs::File> = nix().safely(
|
let ret: Result<fs::File, fs::File> = nix()
|
||||||
&Operation::Instantiate,
|
.safely(
|
||||||
individual_eval_path().as_path(),
|
&Operation::Instantiate,
|
||||||
vec![String::from("-A"), String::from("nixpkgs-restricted-mode")],
|
individual_eval_path().as_path(),
|
||||||
true,
|
vec![String::from("-A"), String::from("nixpkgs-restricted-mode")],
|
||||||
);
|
true,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
assert_run(
|
assert_run(
|
||||||
ret,
|
ret,
|
||||||
Expect::Fail,
|
&Expect::Fail,
|
||||||
vec![
|
vec![
|
||||||
"access to absolute path '/fake'",
|
"access to absolute path '/fake'",
|
||||||
"is forbidden in restricted mode",
|
"is forbidden in restricted mode",
|
||||||
|
|
|
@ -17,6 +17,7 @@ pub struct HydraNixEnv {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl HydraNixEnv {
|
impl HydraNixEnv {
|
||||||
|
#[must_use]
|
||||||
pub fn new(nix: nix::Nix, path: PathBuf, check_meta: bool) -> HydraNixEnv {
|
pub fn new(nix: nix::Nix, path: PathBuf, check_meta: bool) -> HydraNixEnv {
|
||||||
HydraNixEnv {
|
HydraNixEnv {
|
||||||
path,
|
path,
|
||||||
|
@ -25,11 +26,11 @@ impl HydraNixEnv {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn execute_with_stats(
|
pub async fn execute_with_stats(
|
||||||
&self,
|
&self,
|
||||||
) -> Result<(outpathdiff::PackageOutPaths, EvaluationStats), Error> {
|
) -> Result<(outpathdiff::PackageOutPaths, EvaluationStats), Error> {
|
||||||
self.place_nix()?;
|
self.place_nix()?;
|
||||||
let (status, stdout, stderr, stats) = self.run_nix_env();
|
let (status, stdout, stderr, stats) = self.run_nix_env().await;
|
||||||
self.remove_nix()?;
|
self.remove_nix()?;
|
||||||
|
|
||||||
if status {
|
if status {
|
||||||
|
@ -79,7 +80,7 @@ impl HydraNixEnv {
|
||||||
// when it fails to evaluate something. In this case, we can ignore (but
|
// when it fails to evaluate something. In this case, we can ignore (but
|
||||||
// warn about) the error.
|
// warn about) the error.
|
||||||
if let Err(e) = fs::remove_file(&outpath_stats) {
|
if let Err(e) = fs::remove_file(&outpath_stats) {
|
||||||
warn!("Failed to remove file {:?}: {:?}", outpath_stats, e)
|
warn!("Failed to remove file {:?}: {:?}", outpath_stats, e);
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
|
@ -93,7 +94,7 @@ impl HydraNixEnv {
|
||||||
self.path.join(".gc-of-borg-stats.json")
|
self.path.join(".gc-of-borg-stats.json")
|
||||||
}
|
}
|
||||||
|
|
||||||
fn run_nix_env(&self) -> (bool, File, File, Result<File, io::Error>) {
|
async fn run_nix_env(&self) -> (bool, File, File, Result<File, io::Error>) {
|
||||||
let check_meta = if self.check_meta { "true" } else { "false" };
|
let check_meta = if self.check_meta { "true" } else { "false" };
|
||||||
|
|
||||||
let mut cmd = self.nix.safe_command(
|
let mut cmd = self.nix.safe_command(
|
||||||
|
@ -111,7 +112,7 @@ impl HydraNixEnv {
|
||||||
cmd.env("NIX_SHOW_STATS", "1");
|
cmd.env("NIX_SHOW_STATS", "1");
|
||||||
cmd.env("NIX_SHOW_STATS_PATH", self.outpath_stats_path());
|
cmd.env("NIX_SHOW_STATS_PATH", self.outpath_stats_path());
|
||||||
|
|
||||||
let (status, stdout, stderr) = self.nix.run_stderr_stdout(cmd);
|
let (status, stdout, stderr) = self.nix.run_stderr_stdout(cmd).await;
|
||||||
let stats = File::open(self.outpath_stats_path());
|
let stats = File::open(self.outpath_stats_path());
|
||||||
|
|
||||||
(status, stdout, stderr, stats)
|
(status, stdout, stderr, stats)
|
||||||
|
@ -135,13 +136,14 @@ impl From<io::Error> for Error {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Error {
|
impl Error {
|
||||||
|
#[must_use]
|
||||||
pub fn display(self) -> String {
|
pub fn display(self) -> String {
|
||||||
match self {
|
match self {
|
||||||
Error::Io(e) => format!("Failed during the setup of executing nix-env: {:?}", e),
|
Error::Io(e) => format!("Failed during the setup of executing nix-env: {e:?}"),
|
||||||
Error::CreateFile(path, err) => format!("Failed to create file {:?}: {:?}", path, err),
|
Error::CreateFile(path, err) => format!("Failed to create file {path:?}: {err:?}"),
|
||||||
Error::RemoveFile(path, err) => format!("Failed to remove file {:?}: {:?}", path, err),
|
Error::RemoveFile(path, err) => format!("Failed to remove file {path:?}: {err:?}"),
|
||||||
Error::WriteFile(file, err) => {
|
Error::WriteFile(file, err) => {
|
||||||
format!("Failed to write to file '{:?}': {:?}", file, err)
|
format!("Failed to write to file '{file:?}': {err:?}")
|
||||||
}
|
}
|
||||||
Error::CommandFailed(mut fd) => {
|
Error::CommandFailed(mut fd) => {
|
||||||
let mut buffer = Vec::new();
|
let mut buffer = Vec::new();
|
||||||
|
@ -149,15 +151,14 @@ impl Error {
|
||||||
let bufstr = String::from_utf8_lossy(&buffer);
|
let bufstr = String::from_utf8_lossy(&buffer);
|
||||||
|
|
||||||
match read_result {
|
match read_result {
|
||||||
Ok(_) => format!("nix-env failed:\n{}", bufstr),
|
Ok(_) => format!("nix-env failed:\n{bufstr}"),
|
||||||
Err(e) => format!(
|
Err(e) => format!(
|
||||||
"nix-env failed and loading the error result caused a new error {:?}\n\n{}",
|
"nix-env failed and loading the error result caused a new error {e:?}\n\n{bufstr}"
|
||||||
e, bufstr
|
|
||||||
),
|
),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Error::UncleanEvaluation(warnings) => {
|
Error::UncleanEvaluation(warnings) => {
|
||||||
format!("nix-env did not evaluate cleanly:\n {:?}", warnings)
|
format!("nix-env did not evaluate cleanly:\n {warnings:?}")
|
||||||
}
|
}
|
||||||
Error::StatsParse(mut fd, seek, parse_err) => {
|
Error::StatsParse(mut fd, seek, parse_err) => {
|
||||||
let mut buffer = Vec::new();
|
let mut buffer = Vec::new();
|
||||||
|
@ -169,21 +170,19 @@ impl Error {
|
||||||
|
|
||||||
if let Err(seek_err) = seek {
|
if let Err(seek_err) = seek {
|
||||||
lines.push_str(&format!(
|
lines.push_str(&format!(
|
||||||
"Additionally, resetting to the beginning of the output failed with:\n{:?}\n\n",
|
"Additionally, resetting to the beginning of the output failed with:\n{seek_err:?}\n\n"
|
||||||
seek_err
|
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Err(read_err) = read_result {
|
if let Err(read_err) = read_result {
|
||||||
lines.push_str(&format!(
|
lines.push_str(&format!(
|
||||||
"Additionally, loading the output failed with:\n{:?}\n\n",
|
"Additionally, loading the output failed with:\n{read_err:?}\n\n"
|
||||||
read_err
|
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
lines.push_str(&format!("Parse error:\n{:?}\n\n", parse_err));
|
lines.push_str(&format!("Parse error:\n{parse_err:?}\n\n"));
|
||||||
|
|
||||||
lines.push_str(&format!("Evaluation output:\n{}", bufstr));
|
lines.push_str(&format!("Evaluation output:\n{bufstr}"));
|
||||||
|
|
||||||
lines
|
lines
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
//! Statistics emitted by Nix when NIX_SHOW_STATS=1
|
//! Statistics emitted by Nix when `NIX_SHOW_STATS=1`
|
||||||
use separator::Separatable;
|
use separator::Separatable;
|
||||||
|
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
|
@ -102,7 +102,69 @@ pub struct EvaluationStatsDiff<'a> {
|
||||||
right: &'a EvaluationStats,
|
right: &'a EvaluationStats,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
struct Row {
|
||||||
|
before: String,
|
||||||
|
after: String,
|
||||||
|
diff: String,
|
||||||
|
diff_pct: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Row {
|
||||||
|
fn from_u64(left: u64, right: u64) -> Row {
|
||||||
|
let (diff, direction): (u64, _) = match left.cmp(&right) {
|
||||||
|
std::cmp::Ordering::Greater => (left - right, "↘ "),
|
||||||
|
std::cmp::Ordering::Less => (right - left, "↗ "),
|
||||||
|
std::cmp::Ordering::Equal => (0, ""),
|
||||||
|
};
|
||||||
|
|
||||||
|
#[allow(clippy::cast_precision_loss)]
|
||||||
|
let diff_pct: String = if diff > 0 {
|
||||||
|
format!(
|
||||||
|
"{:.2}%",
|
||||||
|
((right as f64) - (left as f64)) / (left as f64) * 100.0
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
String::new()
|
||||||
|
};
|
||||||
|
|
||||||
|
Row {
|
||||||
|
before: left.separated_string(),
|
||||||
|
after: right.separated_string(),
|
||||||
|
diff: format!("{}{}", direction, diff.separated_string()),
|
||||||
|
diff_pct,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn from_f32(left: f32, right: f32) -> Row {
|
||||||
|
let (diff, direction): (f32, _) = match left
|
||||||
|
.partial_cmp(&right)
|
||||||
|
.unwrap_or(std::cmp::Ordering::Equal)
|
||||||
|
{
|
||||||
|
std::cmp::Ordering::Greater => (left - right, "↘ "),
|
||||||
|
std::cmp::Ordering::Less => (right - left, "↗ "),
|
||||||
|
std::cmp::Ordering::Equal => (0.0, ""),
|
||||||
|
};
|
||||||
|
|
||||||
|
let diff_pct: String = if diff > 0.0 {
|
||||||
|
format!(
|
||||||
|
"{:.2}%",
|
||||||
|
(f64::from(right) - f64::from(left)) / f64::from(left) * 100.0
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
String::new()
|
||||||
|
};
|
||||||
|
|
||||||
|
Row {
|
||||||
|
before: format!("{left:.2}"),
|
||||||
|
after: format!("{right:.2}"),
|
||||||
|
diff: format!("{direction}{diff:.2}"),
|
||||||
|
diff_pct,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
impl<'a> EvaluationStatsDiff<'a> {
|
impl<'a> EvaluationStatsDiff<'a> {
|
||||||
|
#[must_use]
|
||||||
pub fn compare(
|
pub fn compare(
|
||||||
left: &'a EvaluationStats,
|
left: &'a EvaluationStats,
|
||||||
right: &'a EvaluationStats,
|
right: &'a EvaluationStats,
|
||||||
|
@ -110,67 +172,9 @@ impl<'a> EvaluationStatsDiff<'a> {
|
||||||
EvaluationStatsDiff { left, right }
|
EvaluationStatsDiff { left, right }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
|
#[allow(clippy::too_many_lines)]
|
||||||
pub fn markdown(&self) -> String {
|
pub fn markdown(&self) -> String {
|
||||||
struct Row {
|
|
||||||
before: String,
|
|
||||||
after: String,
|
|
||||||
diff: String,
|
|
||||||
diff_pct: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Row {
|
|
||||||
fn from_u64(left: u64, right: u64) -> Row {
|
|
||||||
let (diff, direction): (u64, _) = match left.cmp(&right) {
|
|
||||||
std::cmp::Ordering::Greater => (left - right, "↘ "),
|
|
||||||
std::cmp::Ordering::Less => (right - left, "↗ "),
|
|
||||||
std::cmp::Ordering::Equal => (0, ""),
|
|
||||||
};
|
|
||||||
|
|
||||||
let diff_pct: String = if diff > 0 {
|
|
||||||
format!(
|
|
||||||
"{:.2}%",
|
|
||||||
((right as f64) - (left as f64)) / (left as f64) * 100.0
|
|
||||||
)
|
|
||||||
} else {
|
|
||||||
String::from("")
|
|
||||||
};
|
|
||||||
|
|
||||||
Row {
|
|
||||||
before: left.separated_string(),
|
|
||||||
after: right.separated_string(),
|
|
||||||
diff: format!("{}{}", direction, diff.separated_string()),
|
|
||||||
diff_pct,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn from_f32(left: f32, right: f32) -> Row {
|
|
||||||
let (diff, direction): (f32, _) = match left
|
|
||||||
.partial_cmp(&right)
|
|
||||||
.unwrap_or(std::cmp::Ordering::Equal)
|
|
||||||
{
|
|
||||||
std::cmp::Ordering::Greater => (left - right, "↘ "),
|
|
||||||
std::cmp::Ordering::Less => (right - left, "↗ "),
|
|
||||||
std::cmp::Ordering::Equal => (0 as f32, ""),
|
|
||||||
};
|
|
||||||
|
|
||||||
let diff_pct: String = if diff > 0 as _ {
|
|
||||||
format!(
|
|
||||||
"{:.2}%",
|
|
||||||
((right as f64) - (left as f64)) / (left as f64) * 100.0
|
|
||||||
)
|
|
||||||
} else {
|
|
||||||
String::from("")
|
|
||||||
};
|
|
||||||
|
|
||||||
Row {
|
|
||||||
before: format!("{:.2}", left),
|
|
||||||
after: format!("{:.2}", right),
|
|
||||||
diff: format!("{}{:.2}", direction, diff),
|
|
||||||
diff_pct,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let mut data: HashMap<&str, Row> = HashMap::new();
|
let mut data: HashMap<&str, Row> = HashMap::new();
|
||||||
data.insert(
|
data.insert(
|
||||||
"cpuTime",
|
"cpuTime",
|
||||||
|
@ -310,7 +314,7 @@ impl<'a> EvaluationStatsDiff<'a> {
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
let mut keys = data.keys().cloned().collect::<Vec<&str>>();
|
let mut keys = data.keys().copied().collect::<Vec<&str>>();
|
||||||
keys.sort_unstable();
|
keys.sort_unstable();
|
||||||
|
|
||||||
let rows = keys
|
let rows = keys
|
||||||
|
@ -442,53 +446,54 @@ mod tests {
|
||||||
"#;
|
"#;
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
#[allow(clippy::float_cmp)] // FIXME
|
||||||
fn verify_load() {
|
fn verify_load() {
|
||||||
let load: EvaluationStats = serde_json::from_str(EXAMPLE).unwrap();
|
let load: EvaluationStats = serde_json::from_str(EXAMPLE).unwrap();
|
||||||
|
|
||||||
assert_eq!(load.cpu_time, 135.2);
|
assert_eq!(load.cpu_time, 135.2);
|
||||||
assert_eq!(load.envs.number, 130714125);
|
assert_eq!(load.envs.number, 130_714_125);
|
||||||
assert_eq!(load.envs.elements, 183953876);
|
assert_eq!(load.envs.elements, 183_953_876);
|
||||||
assert_eq!(load.envs.bytes, 3563057008);
|
assert_eq!(load.envs.bytes, 3_563_057_008);
|
||||||
|
|
||||||
assert_eq!(load.list.elements, 207421516);
|
assert_eq!(load.list.elements, 207_421_516);
|
||||||
assert_eq!(load.list.bytes, 1659372128);
|
assert_eq!(load.list.bytes, 1_659_372_128);
|
||||||
assert_eq!(load.list.concats, 7194150);
|
assert_eq!(load.list.concats, 7_194_150);
|
||||||
|
|
||||||
assert_eq!(load.values.number, 260454370);
|
assert_eq!(load.values.number, 260_454_370);
|
||||||
assert_eq!(load.values.bytes, 6250904880);
|
assert_eq!(load.values.bytes, 6_250_904_880);
|
||||||
|
|
||||||
assert_eq!(load.symbols.number, 372918);
|
assert_eq!(load.symbols.number, 372_918);
|
||||||
assert_eq!(load.symbols.bytes, 16324262);
|
assert_eq!(load.symbols.bytes, 16_324_262);
|
||||||
|
|
||||||
assert_eq!(load.sets.number, 27310541);
|
assert_eq!(load.sets.number, 27_310_541);
|
||||||
assert_eq!(load.sets.bytes, 7134676648);
|
assert_eq!(load.sets.bytes, 7_134_676_648);
|
||||||
assert_eq!(load.sets.elements, 288174680);
|
assert_eq!(load.sets.elements, 288_174_680);
|
||||||
|
|
||||||
assert_eq!(load.sizes.env, 16);
|
assert_eq!(load.sizes.env, 16);
|
||||||
assert_eq!(load.sizes.value, 24);
|
assert_eq!(load.sizes.value, 24);
|
||||||
assert_eq!(load.sizes.bindings, 8);
|
assert_eq!(load.sizes.bindings, 8);
|
||||||
assert_eq!(load.sizes.attr, 24);
|
assert_eq!(load.sizes.attr, 24);
|
||||||
|
|
||||||
assert_eq!(load.nr_op_updates, 11883339);
|
assert_eq!(load.nr_op_updates, 11_883_339);
|
||||||
assert_eq!(load.nr_op_update_values_copied, 208834564);
|
assert_eq!(load.nr_op_update_values_copied, 208_834_564);
|
||||||
assert_eq!(load.nr_thunks, 173325665);
|
assert_eq!(load.nr_thunks, 173_325_665);
|
||||||
assert_eq!(load.nr_avoided, 177840681);
|
assert_eq!(load.nr_avoided, 177_840_681);
|
||||||
assert_eq!(load.nr_lookups, 75292052);
|
assert_eq!(load.nr_lookups, 75_292_052);
|
||||||
assert_eq!(load.nr_prim_op_calls, 85571252);
|
assert_eq!(load.nr_prim_op_calls, 85_571_252);
|
||||||
assert_eq!(load.nr_function_calls, 115193164);
|
assert_eq!(load.nr_function_calls, 115_193_164);
|
||||||
|
|
||||||
assert_eq!(load.gc.heap_size, 12104687616);
|
assert_eq!(load.gc.heap_size, 12_104_687_616);
|
||||||
assert_eq!(load.gc.total_bytes, 24191819392);
|
assert_eq!(load.gc.total_bytes, 24_191_819_392);
|
||||||
}
|
}
|
||||||
|
|
||||||
fn diff_text(left: &str, right: &str) {
|
fn diff_text(left: &str, right: &str) {
|
||||||
println!("left:\n{}", left);
|
println!("left:\n{left}");
|
||||||
println!("right:\n{}", right);
|
println!("right:\n{right}");
|
||||||
|
|
||||||
let lines = left.split('\n').zip(right.split('\n'));
|
let lines = left.split('\n').zip(right.split('\n'));
|
||||||
|
|
||||||
for (idx, (linea, lineb)) in lines.enumerate() {
|
for (idx, (linea, lineb)) in lines.enumerate() {
|
||||||
assert_eq!(linea, lineb, "Line {}", idx);
|
assert_eq!(linea, lineb, "Line {idx}");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -499,7 +504,7 @@ mod tests {
|
||||||
|
|
||||||
diff_text(
|
diff_text(
|
||||||
&EvaluationStatsDiff::compare(&left, &right).markdown(),
|
&EvaluationStatsDiff::compare(&left, &right).markdown(),
|
||||||
r#"
|
r"
|
||||||
| stat | before | after | Δ | Δ% |
|
| stat | before | after | Δ | Δ% |
|
||||||
|:---------------------------|---------------:|---------------:|:--------------|-------:|
|
|:---------------------------|---------------:|---------------:|:--------------|-------:|
|
||||||
| **cpuTime** | 135.20 | 132.90 | ↘ 2.30 | -1.70% |
|
| **cpuTime** | 135.20 | 132.90 | ↘ 2.30 | -1.70% |
|
||||||
|
@ -529,7 +534,7 @@ mod tests {
|
||||||
| **symbols-number** | 372,918 | 372,917 | ↘ 1 | -0.00% |
|
| **symbols-number** | 372,918 | 372,917 | ↘ 1 | -0.00% |
|
||||||
| **values-bytes** | 6,250,904,880 | 5,869,027,296 | ↘ 381,877,584 | -6.11% |
|
| **values-bytes** | 6,250,904,880 | 5,869,027,296 | ↘ 381,877,584 | -6.11% |
|
||||||
| **values-number** | 260,454,370 | 244,542,804 | ↘ 15,911,566 | -6.11% |
|
| **values-number** | 260,454,370 | 244,542,804 | ↘ 15,911,566 | -6.11% |
|
||||||
"#
|
"
|
||||||
.trim_start(),
|
.trim_start(),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,11 +1,13 @@
|
||||||
use crate::worker::Action;
|
use crate::worker::Action;
|
||||||
|
use async_trait::async_trait;
|
||||||
|
|
||||||
pub trait SimpleNotifyWorker {
|
#[async_trait]
|
||||||
type J;
|
pub trait SimpleNotifyWorker: Send + Sync {
|
||||||
|
type J: Send + Sync;
|
||||||
|
|
||||||
fn consumer(&self, job: &Self::J, notifier: &mut dyn NotificationReceiver);
|
async fn consumer(&self, job: &Self::J, notifier: &mut dyn NotificationReceiver);
|
||||||
|
|
||||||
fn msg_to_job(
|
async fn msg_to_job(
|
||||||
&self,
|
&self,
|
||||||
routing_key: &str,
|
routing_key: &str,
|
||||||
content_type: &Option<String>,
|
content_type: &Option<String>,
|
||||||
|
@ -13,8 +15,9 @@ pub trait SimpleNotifyWorker {
|
||||||
) -> Result<Self::J, String>;
|
) -> Result<Self::J, String>;
|
||||||
}
|
}
|
||||||
|
|
||||||
pub trait NotificationReceiver {
|
#[async_trait]
|
||||||
fn tell(&mut self, action: Action);
|
pub trait NotificationReceiver: Send + Sync {
|
||||||
|
async fn tell(&mut self, action: Action);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Default)]
|
#[derive(Default)]
|
||||||
|
@ -23,13 +26,15 @@ pub struct DummyNotificationReceiver {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl DummyNotificationReceiver {
|
impl DummyNotificationReceiver {
|
||||||
|
#[must_use]
|
||||||
pub fn new() -> DummyNotificationReceiver {
|
pub fn new() -> DummyNotificationReceiver {
|
||||||
Default::default()
|
DummyNotificationReceiver::default()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
impl NotificationReceiver for DummyNotificationReceiver {
|
impl NotificationReceiver for DummyNotificationReceiver {
|
||||||
fn tell(&mut self, action: Action) {
|
async fn tell(&mut self, action: Action) {
|
||||||
self.actions.push(action);
|
self.actions.push(action);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -15,6 +15,7 @@ pub struct OutPathDiff {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl OutPathDiff {
|
impl OutPathDiff {
|
||||||
|
#[must_use]
|
||||||
pub fn new(nix: nix::Nix, path: PathBuf) -> OutPathDiff {
|
pub fn new(nix: nix::Nix, path: PathBuf) -> OutPathDiff {
|
||||||
OutPathDiff {
|
OutPathDiff {
|
||||||
calculator: HydraNixEnv::new(nix, path, false),
|
calculator: HydraNixEnv::new(nix, path, false),
|
||||||
|
@ -23,21 +24,22 @@ impl OutPathDiff {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn find_before(&mut self) -> Result<(), NixEnvError> {
|
pub async fn find_before(&mut self) -> Result<(), NixEnvError> {
|
||||||
self.original = Some(self.run()?);
|
self.original = Some(self.run().await?);
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn find_after(&mut self) -> Result<(), NixEnvError> {
|
pub async fn find_after(&mut self) -> Result<(), NixEnvError> {
|
||||||
if self.original.is_none() {
|
if self.original.is_none() {
|
||||||
debug!("Before is None, not bothering with After");
|
debug!("Before is None, not bothering with After");
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
self.current = Some(self.run()?);
|
self.current = Some(self.run().await?);
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub fn performance_diff(&self) -> Option<EvaluationStatsDiff> {
|
pub fn performance_diff(&self) -> Option<EvaluationStatsDiff> {
|
||||||
if let Some((_, ref cur)) = self.current {
|
if let Some((_, ref cur)) = self.current {
|
||||||
if let Some((_, ref orig)) = self.original {
|
if let Some((_, ref orig)) = self.original {
|
||||||
|
@ -50,6 +52,7 @@ impl OutPathDiff {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub fn package_diff(&self) -> Option<(Vec<PackageArch>, Vec<PackageArch>)> {
|
pub fn package_diff(&self) -> Option<(Vec<PackageArch>, Vec<PackageArch>)> {
|
||||||
if let Some((ref cur, _)) = self.current {
|
if let Some((ref cur, _)) = self.current {
|
||||||
if let Some((ref orig, _)) = self.original {
|
if let Some((ref orig, _)) = self.original {
|
||||||
|
@ -80,11 +83,11 @@ impl OutPathDiff {
|
||||||
if let Some((ref orig, _)) = self.original {
|
if let Some((ref orig, _)) = self.original {
|
||||||
for key in cur.keys() {
|
for key in cur.keys() {
|
||||||
trace!("Checking out {:?}", key);
|
trace!("Checking out {:?}", key);
|
||||||
if cur.get(key) != orig.get(key) {
|
if cur.get(key) == orig.get(key) {
|
||||||
trace!(" {:?} != {:?}", cur.get(key), orig.get(key));
|
|
||||||
rebuild.push(key.clone())
|
|
||||||
} else {
|
|
||||||
trace!(" {:?} == {:?}", cur.get(key), orig.get(key));
|
trace!(" {:?} == {:?}", cur.get(key), orig.get(key));
|
||||||
|
} else {
|
||||||
|
trace!(" {:?} != {:?}", cur.get(key), orig.get(key));
|
||||||
|
rebuild.push(key.clone());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -95,8 +98,8 @@ impl OutPathDiff {
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
fn run(&mut self) -> Result<(PackageOutPaths, EvaluationStats), NixEnvError> {
|
async fn run(&mut self) -> Result<(PackageOutPaths, EvaluationStats), NixEnvError> {
|
||||||
self.calculator.execute_with_stats()
|
self.calculator.execute_with_stats().await
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1,16 +1,15 @@
|
||||||
use futures::executor::block_on;
|
use async_trait::async_trait;
|
||||||
use lapin::options::BasicPublishOptions;
|
use lapin::options::BasicPublishOptions;
|
||||||
|
|
||||||
include!(concat!(env!("OUT_DIR"), "/events.rs"));
|
#[allow(clippy::all, clippy::pedantic)]
|
||||||
|
mod events {
|
||||||
#[macro_use]
|
include!(concat!(env!("OUT_DIR"), "/events.rs"));
|
||||||
mod macros {
|
|
||||||
#[macro_export]
|
|
||||||
macro_rules! my_macro(() => (FooBar));
|
|
||||||
}
|
}
|
||||||
|
pub use events::*;
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
pub trait SysEvents: Send {
|
pub trait SysEvents: Send {
|
||||||
fn notify(&mut self, event: Event);
|
async fn notify(&mut self, event: Event);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Serialize, Deserialize, Debug)]
|
#[derive(Serialize, Deserialize, Debug)]
|
||||||
|
@ -25,6 +24,7 @@ pub struct RabbitMq<C> {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl RabbitMq<lapin::Channel> {
|
impl RabbitMq<lapin::Channel> {
|
||||||
|
#[must_use]
|
||||||
pub fn from_lapin(identity: &str, channel: lapin::Channel) -> Self {
|
pub fn from_lapin(identity: &str, channel: lapin::Channel) -> Self {
|
||||||
RabbitMq {
|
RabbitMq {
|
||||||
identity: identity.to_owned(),
|
identity: identity.to_owned(),
|
||||||
|
@ -33,28 +33,27 @@ impl RabbitMq<lapin::Channel> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
impl SysEvents for RabbitMq<lapin::Channel> {
|
impl SysEvents for RabbitMq<lapin::Channel> {
|
||||||
fn notify(&mut self, event: Event) {
|
async fn notify(&mut self, event: Event) {
|
||||||
let props = lapin::BasicProperties::default().with_content_type("application/json".into());
|
let props = lapin::BasicProperties::default().with_content_type("application/json".into());
|
||||||
block_on(async {
|
let _confirmaton = self
|
||||||
let _confirmaton = self
|
.channel
|
||||||
.channel
|
.basic_publish(
|
||||||
.basic_publish(
|
&String::from("stats"),
|
||||||
&String::from("stats"),
|
"",
|
||||||
"",
|
BasicPublishOptions::default(),
|
||||||
BasicPublishOptions::default(),
|
&serde_json::to_string(&EventMessage {
|
||||||
&serde_json::to_string(&EventMessage {
|
sender: self.identity.clone(),
|
||||||
sender: self.identity.clone(),
|
events: vec![event],
|
||||||
events: vec![event],
|
})
|
||||||
})
|
|
||||||
.unwrap()
|
|
||||||
.into_bytes(),
|
|
||||||
props,
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.await
|
.into_bytes(),
|
||||||
.unwrap();
|
props,
|
||||||
});
|
)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -18,16 +18,16 @@ impl std::fmt::Display for System {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl System {
|
impl System {
|
||||||
|
#[must_use]
|
||||||
pub fn as_build_destination(&self) -> (Option<String>, Option<String>) {
|
pub fn as_build_destination(&self) -> (Option<String>, Option<String>) {
|
||||||
(None, Some(format!("build-inputs-{}", self)))
|
(None, Some(format!("build-inputs-{self}")))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub fn can_run_nixos_tests(&self) -> bool {
|
pub fn can_run_nixos_tests(&self) -> bool {
|
||||||
match self {
|
match self {
|
||||||
System::X8664Linux => true,
|
System::X8664Linux | System::Aarch64Linux => true,
|
||||||
System::Aarch64Linux => true,
|
System::X8664Darwin | System::Aarch64Darwin => false,
|
||||||
System::X8664Darwin => false,
|
|
||||||
System::Aarch64Darwin => false,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -25,8 +25,9 @@ impl Default for StdenvTagger {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl StdenvTagger {
|
impl StdenvTagger {
|
||||||
|
#[must_use]
|
||||||
pub fn new() -> StdenvTagger {
|
pub fn new() -> StdenvTagger {
|
||||||
Default::default()
|
StdenvTagger::default()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn changed(&mut self, systems: Vec<tasks::eval::stdenvs::System>) {
|
pub fn changed(&mut self, systems: Vec<tasks::eval::stdenvs::System>) {
|
||||||
|
@ -42,19 +43,20 @@ impl StdenvTagger {
|
||||||
}
|
}
|
||||||
|
|
||||||
for tag in &self.selected {
|
for tag in &self.selected {
|
||||||
if !self.possible.contains(tag) {
|
assert!(
|
||||||
panic!(
|
self.possible.contains(tag),
|
||||||
"Tried to add label {} but it isn't in the possible list!",
|
"Tried to add label {} but it isn't in the possible list!",
|
||||||
tag
|
tag
|
||||||
);
|
);
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub fn tags_to_add(&self) -> Vec<String> {
|
pub fn tags_to_add(&self) -> Vec<String> {
|
||||||
self.selected.clone()
|
self.selected.clone()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub fn tags_to_remove(&self) -> Vec<String> {
|
pub fn tags_to_remove(&self) -> Vec<String> {
|
||||||
let mut remove = self.possible.clone();
|
let mut remove = self.possible.clone();
|
||||||
for tag in &self.selected {
|
for tag in &self.selected {
|
||||||
|
@ -87,8 +89,9 @@ impl Default for PkgsAddedRemovedTagger {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl PkgsAddedRemovedTagger {
|
impl PkgsAddedRemovedTagger {
|
||||||
|
#[must_use]
|
||||||
pub fn new() -> PkgsAddedRemovedTagger {
|
pub fn new() -> PkgsAddedRemovedTagger {
|
||||||
Default::default()
|
PkgsAddedRemovedTagger::default()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn changed(&mut self, removed: &[PackageArch], added: &[PackageArch]) {
|
pub fn changed(&mut self, removed: &[PackageArch], added: &[PackageArch]) {
|
||||||
|
@ -101,10 +104,12 @@ impl PkgsAddedRemovedTagger {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub fn tags_to_add(&self) -> Vec<String> {
|
pub fn tags_to_add(&self) -> Vec<String> {
|
||||||
self.selected.clone()
|
self.selected.clone()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub fn tags_to_remove(&self) -> Vec<String> {
|
pub fn tags_to_remove(&self) -> Vec<String> {
|
||||||
// The cleanup tag is too vague to automatically remove.
|
// The cleanup tag is too vague to automatically remove.
|
||||||
vec![]
|
vec![]
|
||||||
|
@ -147,8 +152,9 @@ impl Default for RebuildTagger {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl RebuildTagger {
|
impl RebuildTagger {
|
||||||
|
#[must_use]
|
||||||
pub fn new() -> RebuildTagger {
|
pub fn new() -> RebuildTagger {
|
||||||
Default::default()
|
RebuildTagger::default()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn parse_attrs(&mut self, attrs: Vec<PackageArch>) {
|
pub fn parse_attrs(&mut self, attrs: Vec<PackageArch>) {
|
||||||
|
@ -160,12 +166,10 @@ impl RebuildTagger {
|
||||||
"x86_64-darwin" => {
|
"x86_64-darwin" => {
|
||||||
counter_darwin += 1;
|
counter_darwin += 1;
|
||||||
}
|
}
|
||||||
"aarch64-darwin" => {}
|
|
||||||
"x86_64-linux" => {
|
"x86_64-linux" => {
|
||||||
counter_linux += 1;
|
counter_linux += 1;
|
||||||
}
|
}
|
||||||
"aarch64-linux" => {}
|
"aarch64-darwin" | "aarch64-linux" | "i686-linux" => {}
|
||||||
"i686-linux" => {}
|
|
||||||
arch => {
|
arch => {
|
||||||
info!("Unknown arch: {:?}", arch);
|
info!("Unknown arch: {:?}", arch);
|
||||||
}
|
}
|
||||||
|
@ -176,35 +180,36 @@ impl RebuildTagger {
|
||||||
self.selected.extend(
|
self.selected.extend(
|
||||||
RebuildTagger::bucket(counter_darwin)
|
RebuildTagger::bucket(counter_darwin)
|
||||||
.iter()
|
.iter()
|
||||||
.map(|bucket| format!("10.rebuild-darwin: {}", bucket))
|
.map(|bucket| format!("10.rebuild-darwin: {bucket}"))
|
||||||
.collect::<Vec<String>>(),
|
.collect::<Vec<String>>(),
|
||||||
);
|
);
|
||||||
|
|
||||||
self.selected.extend(
|
self.selected.extend(
|
||||||
RebuildTagger::bucket(counter_linux)
|
RebuildTagger::bucket(counter_linux)
|
||||||
.iter()
|
.iter()
|
||||||
.map(|bucket| format!("10.rebuild-linux: {}", bucket))
|
.map(|bucket| format!("10.rebuild-linux: {bucket}"))
|
||||||
.collect::<Vec<String>>(),
|
.collect::<Vec<String>>(),
|
||||||
);
|
);
|
||||||
|
|
||||||
for tag in &self.selected {
|
for tag in &self.selected {
|
||||||
if !self.possible.contains(tag) {
|
assert!(
|
||||||
panic!(
|
self.possible.contains(tag),
|
||||||
"Tried to add label {} but it isn't in the possible list!",
|
"Tried to add label {} but it isn't in the possible list!",
|
||||||
tag
|
tag
|
||||||
);
|
);
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub fn tags_to_add(&self) -> Vec<String> {
|
pub fn tags_to_add(&self) -> Vec<String> {
|
||||||
self.selected.clone()
|
self.selected.clone()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub fn tags_to_remove(&self) -> Vec<String> {
|
pub fn tags_to_remove(&self) -> Vec<String> {
|
||||||
let mut remove = vec![];
|
let mut remove = vec![];
|
||||||
|
|
||||||
for tag in self.possible.clone().into_iter() {
|
for tag in self.possible.clone() {
|
||||||
if !self.selected.contains(&tag) {
|
if !self.selected.contains(&tag) {
|
||||||
remove.push(tag);
|
remove.push(tag);
|
||||||
}
|
}
|
||||||
|
@ -254,8 +259,9 @@ impl Default for MaintainerPrTagger {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl MaintainerPrTagger {
|
impl MaintainerPrTagger {
|
||||||
|
#[must_use]
|
||||||
pub fn new() -> MaintainerPrTagger {
|
pub fn new() -> MaintainerPrTagger {
|
||||||
Default::default()
|
MaintainerPrTagger::default()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn record_maintainer(
|
pub fn record_maintainer(
|
||||||
|
@ -270,7 +276,7 @@ impl MaintainerPrTagger {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
for (_package, maintainers) in identified_maintainers.0.iter() {
|
for maintainers in identified_maintainers.0.values() {
|
||||||
if !maintainers.contains(&submitter) {
|
if !maintainers.contains(&submitter) {
|
||||||
// One of the packages is not maintained by this submitter
|
// One of the packages is not maintained by this submitter
|
||||||
return;
|
return;
|
||||||
|
@ -281,10 +287,12 @@ impl MaintainerPrTagger {
|
||||||
.push(String::from("11.by: package-maintainer"));
|
.push(String::from("11.by: package-maintainer"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub fn tags_to_add(&self) -> Vec<String> {
|
pub fn tags_to_add(&self) -> Vec<String> {
|
||||||
self.selected.clone()
|
self.selected.clone()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub fn tags_to_remove(&self) -> Vec<String> {
|
pub fn tags_to_remove(&self) -> Vec<String> {
|
||||||
// The cleanup tag is too vague to automatically remove.
|
// The cleanup tag is too vague to automatically remove.
|
||||||
vec![]
|
vec![]
|
||||||
|
@ -333,6 +341,7 @@ mod tests {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
#[allow(clippy::too_many_lines)]
|
||||||
pub fn test_packages_changed() {
|
pub fn test_packages_changed() {
|
||||||
let mut tagger = RebuildTagger::new();
|
let mut tagger = RebuildTagger::new();
|
||||||
tagger.parse_attrs(PackageArchSrc::linux(0).and_darwin(0).into());
|
tagger.parse_attrs(PackageArchSrc::linux(0).and_darwin(0).into());
|
||||||
|
|
|
@ -8,6 +8,9 @@ use crate::worker;
|
||||||
|
|
||||||
use std::collections::VecDeque;
|
use std::collections::VecDeque;
|
||||||
|
|
||||||
|
use async_trait::async_trait;
|
||||||
|
use tokio::io::AsyncBufReadExt;
|
||||||
|
use tokio::io::BufReader;
|
||||||
use tracing::{debug, debug_span, error, info};
|
use tracing::{debug, debug_span, error, info};
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
@ -19,6 +22,7 @@ pub struct BuildWorker {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl BuildWorker {
|
impl BuildWorker {
|
||||||
|
#[must_use]
|
||||||
pub fn new(
|
pub fn new(
|
||||||
cloner: checkout::CachedCloner,
|
cloner: checkout::CachedCloner,
|
||||||
nix: nix::Nix,
|
nix: nix::Nix,
|
||||||
|
@ -47,7 +51,7 @@ pub struct JobActions<'a, 'b> {
|
||||||
identity: String,
|
identity: String,
|
||||||
receiver: &'a mut dyn notifyworker::NotificationReceiver,
|
receiver: &'a mut dyn notifyworker::NotificationReceiver,
|
||||||
job: &'b buildjob::BuildJob,
|
job: &'b buildjob::BuildJob,
|
||||||
line_counter: u64,
|
line_counter: usize,
|
||||||
snippet_log: VecDeque<String>,
|
snippet_log: VecDeque<String>,
|
||||||
attempt_id: String,
|
attempt_id: String,
|
||||||
log_exchange: Option<String>,
|
log_exchange: Option<String>,
|
||||||
|
@ -88,23 +92,24 @@ impl<'a, 'b> JobActions<'a, 'b> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub fn log_snippet(&self) -> Vec<String> {
|
pub fn log_snippet(&self) -> Vec<String> {
|
||||||
self.snippet_log.clone().into()
|
self.snippet_log.clone().into()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn pr_head_missing(&mut self) {
|
pub async fn pr_head_missing(&mut self) {
|
||||||
self.tell(worker::Action::Ack);
|
self.tell(worker::Action::Ack).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn commit_missing(&mut self) {
|
pub async fn commit_missing(&mut self) {
|
||||||
self.tell(worker::Action::Ack);
|
self.tell(worker::Action::Ack).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn nothing_to_do(&mut self) {
|
pub async fn nothing_to_do(&mut self) {
|
||||||
self.tell(worker::Action::Ack);
|
self.tell(worker::Action::Ack).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn merge_failed(&mut self) {
|
pub async fn merge_failed(&mut self) {
|
||||||
let msg = BuildResult::V1 {
|
let msg = BuildResult::V1 {
|
||||||
tag: V1Tag::V1,
|
tag: V1Tag::V1,
|
||||||
repo: self.job.repo.clone(),
|
repo: self.job.repo.clone(),
|
||||||
|
@ -122,14 +127,15 @@ impl<'a, 'b> JobActions<'a, 'b> {
|
||||||
let result_routing_key = self.result_routing_key.clone();
|
let result_routing_key = self.result_routing_key.clone();
|
||||||
|
|
||||||
self.tell(worker::publish_serde_action(
|
self.tell(worker::publish_serde_action(
|
||||||
result_exchange,
|
&result_exchange,
|
||||||
result_routing_key,
|
&result_routing_key,
|
||||||
&msg,
|
&msg,
|
||||||
));
|
))
|
||||||
self.tell(worker::Action::Ack);
|
.await;
|
||||||
|
self.tell(worker::Action::Ack).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn log_started(&mut self, can_build: Vec<String>, cannot_build: Vec<String>) {
|
pub async fn log_started(&mut self, can_build: Vec<String>, cannot_build: Vec<String>) {
|
||||||
let msg = buildlogmsg::BuildLogStart {
|
let msg = buildlogmsg::BuildLogStart {
|
||||||
identity: self.identity.clone(),
|
identity: self.identity.clone(),
|
||||||
system: self.system.clone(),
|
system: self.system.clone(),
|
||||||
|
@ -142,24 +148,26 @@ impl<'a, 'b> JobActions<'a, 'b> {
|
||||||
let log_routing_key = self.log_routing_key.clone();
|
let log_routing_key = self.log_routing_key.clone();
|
||||||
|
|
||||||
self.tell(worker::publish_serde_action(
|
self.tell(worker::publish_serde_action(
|
||||||
log_exchange,
|
&log_exchange,
|
||||||
log_routing_key,
|
&log_routing_key,
|
||||||
&msg,
|
&msg,
|
||||||
));
|
))
|
||||||
|
.await;
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn log_instantiation_errors(&mut self, cannot_build: Vec<(String, Vec<String>)>) {
|
pub async fn log_instantiation_errors(&mut self, cannot_build: Vec<(String, Vec<String>)>) {
|
||||||
for (attr, log) in cannot_build {
|
for (attr, log) in cannot_build {
|
||||||
self.log_line(&format!("Cannot nix-instantiate `{}' because:", &attr));
|
self.log_line(&format!("Cannot nix-instantiate `{}' because:", &attr))
|
||||||
|
.await;
|
||||||
|
|
||||||
for line in log {
|
for line in log {
|
||||||
self.log_line(&line);
|
self.log_line(&line).await;
|
||||||
}
|
}
|
||||||
self.log_line("");
|
self.log_line("").await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn log_line(&mut self, line: &str) {
|
pub async fn log_line(&mut self, line: &str) {
|
||||||
self.line_counter += 1;
|
self.line_counter += 1;
|
||||||
|
|
||||||
if self.snippet_log.len() >= 10 {
|
if self.snippet_log.len() >= 10 {
|
||||||
|
@ -179,13 +187,14 @@ impl<'a, 'b> JobActions<'a, 'b> {
|
||||||
let log_routing_key = self.log_routing_key.clone();
|
let log_routing_key = self.log_routing_key.clone();
|
||||||
|
|
||||||
self.tell(worker::publish_serde_action(
|
self.tell(worker::publish_serde_action(
|
||||||
log_exchange,
|
&log_exchange,
|
||||||
log_routing_key,
|
&log_routing_key,
|
||||||
&msg,
|
&msg,
|
||||||
));
|
))
|
||||||
|
.await;
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn build_not_attempted(&mut self, not_attempted_attrs: Vec<String>) {
|
pub async fn build_not_attempted(&mut self, not_attempted_attrs: Vec<String>) {
|
||||||
let msg = BuildResult::V1 {
|
let msg = BuildResult::V1 {
|
||||||
tag: V1Tag::V1,
|
tag: V1Tag::V1,
|
||||||
repo: self.job.repo.clone(),
|
repo: self.job.repo.clone(),
|
||||||
|
@ -202,23 +211,25 @@ impl<'a, 'b> JobActions<'a, 'b> {
|
||||||
let result_exchange = self.result_exchange.clone();
|
let result_exchange = self.result_exchange.clone();
|
||||||
let result_routing_key = self.result_routing_key.clone();
|
let result_routing_key = self.result_routing_key.clone();
|
||||||
self.tell(worker::publish_serde_action(
|
self.tell(worker::publish_serde_action(
|
||||||
result_exchange,
|
&result_exchange,
|
||||||
result_routing_key,
|
&result_routing_key,
|
||||||
&msg,
|
&msg,
|
||||||
));
|
))
|
||||||
|
.await;
|
||||||
|
|
||||||
let log_exchange = self.log_exchange.clone();
|
let log_exchange = self.log_exchange.clone();
|
||||||
let log_routing_key = self.log_routing_key.clone();
|
let log_routing_key = self.log_routing_key.clone();
|
||||||
self.tell(worker::publish_serde_action(
|
self.tell(worker::publish_serde_action(
|
||||||
log_exchange,
|
&log_exchange,
|
||||||
log_routing_key,
|
&log_routing_key,
|
||||||
&msg,
|
&msg,
|
||||||
));
|
))
|
||||||
|
.await;
|
||||||
|
|
||||||
self.tell(worker::Action::Ack);
|
self.tell(worker::Action::Ack).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn build_finished(
|
pub async fn build_finished(
|
||||||
&mut self,
|
&mut self,
|
||||||
status: BuildStatus,
|
status: BuildStatus,
|
||||||
attempted_attrs: Vec<String>,
|
attempted_attrs: Vec<String>,
|
||||||
|
@ -240,31 +251,39 @@ impl<'a, 'b> JobActions<'a, 'b> {
|
||||||
let result_exchange = self.result_exchange.clone();
|
let result_exchange = self.result_exchange.clone();
|
||||||
let result_routing_key = self.result_routing_key.clone();
|
let result_routing_key = self.result_routing_key.clone();
|
||||||
self.tell(worker::publish_serde_action(
|
self.tell(worker::publish_serde_action(
|
||||||
result_exchange,
|
&result_exchange,
|
||||||
result_routing_key,
|
&result_routing_key,
|
||||||
&msg,
|
&msg,
|
||||||
));
|
))
|
||||||
|
.await;
|
||||||
|
|
||||||
let log_exchange = self.log_exchange.clone();
|
let log_exchange = self.log_exchange.clone();
|
||||||
let log_routing_key = self.log_routing_key.clone();
|
let log_routing_key = self.log_routing_key.clone();
|
||||||
self.tell(worker::publish_serde_action(
|
self.tell(worker::publish_serde_action(
|
||||||
log_exchange,
|
&log_exchange,
|
||||||
log_routing_key,
|
&log_routing_key,
|
||||||
&msg,
|
&msg,
|
||||||
));
|
))
|
||||||
|
.await;
|
||||||
|
|
||||||
self.tell(worker::Action::Ack);
|
self.tell(worker::Action::Ack).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
fn tell(&mut self, action: worker::Action) {
|
async fn tell(&mut self, action: worker::Action) {
|
||||||
self.receiver.tell(action);
|
self.receiver.tell(action).await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
impl notifyworker::SimpleNotifyWorker for BuildWorker {
|
impl notifyworker::SimpleNotifyWorker for BuildWorker {
|
||||||
type J = buildjob::BuildJob;
|
type J = buildjob::BuildJob;
|
||||||
|
|
||||||
fn msg_to_job(&self, _: &str, _: &Option<String>, body: &[u8]) -> Result<Self::J, String> {
|
async fn msg_to_job(
|
||||||
|
&self,
|
||||||
|
_: &str,
|
||||||
|
_: &Option<String>,
|
||||||
|
body: &[u8],
|
||||||
|
) -> Result<Self::J, String> {
|
||||||
info!("lmao I got a job?");
|
info!("lmao I got a job?");
|
||||||
match buildjob::from(body) {
|
match buildjob::from(body) {
|
||||||
Ok(e) => Ok(e),
|
Ok(e) => Ok(e),
|
||||||
|
@ -277,7 +296,7 @@ impl notifyworker::SimpleNotifyWorker for BuildWorker {
|
||||||
|
|
||||||
// FIXME: remove with rust/cargo update
|
// FIXME: remove with rust/cargo update
|
||||||
#[allow(clippy::cognitive_complexity)]
|
#[allow(clippy::cognitive_complexity)]
|
||||||
fn consumer(
|
async fn consumer(
|
||||||
&self,
|
&self,
|
||||||
job: &buildjob::BuildJob,
|
job: &buildjob::BuildJob,
|
||||||
notifier: &mut dyn notifyworker::NotificationReceiver,
|
notifier: &mut dyn notifyworker::NotificationReceiver,
|
||||||
|
@ -289,7 +308,7 @@ impl notifyworker::SimpleNotifyWorker for BuildWorker {
|
||||||
|
|
||||||
if job.attrs.is_empty() {
|
if job.attrs.is_empty() {
|
||||||
debug!("No attrs to build");
|
debug!("No attrs to build");
|
||||||
actions.nothing_to_do();
|
actions.nothing_to_do().await;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -318,19 +337,19 @@ impl notifyworker::SimpleNotifyWorker for BuildWorker {
|
||||||
|
|
||||||
if co.fetch_pr(job.change.number).is_err() {
|
if co.fetch_pr(job.change.number).is_err() {
|
||||||
info!("Failed to fetch {}", job.change.number);
|
info!("Failed to fetch {}", job.change.number);
|
||||||
actions.pr_head_missing();
|
actions.pr_head_missing().await;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if !co.commit_exists(job.change.head_sha.as_ref()) {
|
if !co.commit_exists(job.change.head_sha.as_ref()) {
|
||||||
info!("Commit {} doesn't exist", job.change.head_sha);
|
info!("Commit {} doesn't exist", job.change.head_sha);
|
||||||
actions.commit_missing();
|
actions.commit_missing().await;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if co.merge_commit(job.change.head_sha.as_ref()).is_err() {
|
if co.merge_commit(job.change.head_sha.as_ref()).is_err() {
|
||||||
info!("Failed to merge {}", job.change.head_sha);
|
info!("Failed to merge {}", job.change.head_sha);
|
||||||
actions.merge_failed();
|
actions.merge_failed().await;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -338,11 +357,10 @@ impl notifyworker::SimpleNotifyWorker for BuildWorker {
|
||||||
"Got path: {:?}, determining which ones we can build ",
|
"Got path: {:?}, determining which ones we can build ",
|
||||||
refpath
|
refpath
|
||||||
);
|
);
|
||||||
let (can_build, cannot_build) = self.nix.safely_partition_instantiable_attrs(
|
let (can_build, cannot_build) = self
|
||||||
refpath.as_ref(),
|
.nix
|
||||||
buildfile,
|
.safely_partition_instantiable_attrs(refpath.as_ref(), buildfile, job.attrs.clone())
|
||||||
job.attrs.clone(),
|
.await;
|
||||||
);
|
|
||||||
|
|
||||||
let cannot_build_attrs: Vec<String> = cannot_build
|
let cannot_build_attrs: Vec<String> = cannot_build
|
||||||
.clone()
|
.clone()
|
||||||
|
@ -356,11 +374,13 @@ impl notifyworker::SimpleNotifyWorker for BuildWorker {
|
||||||
cannot_build_attrs.join(", ")
|
cannot_build_attrs.join(", ")
|
||||||
);
|
);
|
||||||
|
|
||||||
actions.log_started(can_build.clone(), cannot_build_attrs.clone());
|
actions
|
||||||
actions.log_instantiation_errors(cannot_build);
|
.log_started(can_build.clone(), cannot_build_attrs.clone())
|
||||||
|
.await;
|
||||||
|
actions.log_instantiation_errors(cannot_build).await;
|
||||||
|
|
||||||
if can_build.is_empty() {
|
if can_build.is_empty() {
|
||||||
actions.build_not_attempted(cannot_build_attrs);
|
actions.build_not_attempted(cannot_build_attrs).await;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -368,11 +388,12 @@ impl notifyworker::SimpleNotifyWorker for BuildWorker {
|
||||||
self.nix
|
self.nix
|
||||||
.safely_build_attrs_async(refpath.as_ref(), buildfile, can_build.clone());
|
.safely_build_attrs_async(refpath.as_ref(), buildfile, can_build.clone());
|
||||||
|
|
||||||
for line in spawned.lines() {
|
let mut reader = BufReader::new(spawned.stdout.take().unwrap()).lines();
|
||||||
actions.log_line(&line);
|
while let Ok(Some(line)) = reader.next_line().await {
|
||||||
|
actions.log_line(&line).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
let status = nix::wait_for_build_status(spawned);
|
let status = nix::wait_for_build_status(spawned).await;
|
||||||
|
|
||||||
info!("ok built ({:?}), building", status);
|
info!("ok built ({:?}), building", status);
|
||||||
info!("Lines:");
|
info!("Lines:");
|
||||||
|
@ -384,7 +405,9 @@ impl notifyworker::SimpleNotifyWorker for BuildWorker {
|
||||||
.last();
|
.last();
|
||||||
info!("----->8-----");
|
info!("----->8-----");
|
||||||
|
|
||||||
actions.build_finished(status, can_build, cannot_build_attrs);
|
actions
|
||||||
|
.build_finished(status, can_build, cannot_build_attrs)
|
||||||
|
.await;
|
||||||
info!("Build done!");
|
info!("Build done!");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -406,7 +429,7 @@ mod tests {
|
||||||
const SYSTEM: &str = "x86_64-darwin";
|
const SYSTEM: &str = "x86_64-darwin";
|
||||||
|
|
||||||
fn nix() -> nix::Nix {
|
fn nix() -> nix::Nix {
|
||||||
let remote = env::var("NIX_REMOTE").unwrap_or("".to_owned());
|
let remote = env::var("NIX_REMOTE").unwrap_or_default();
|
||||||
nix::Nix::new("x86_64-linux".to_owned(), remote, 1800, None)
|
nix::Nix::new("x86_64-linux".to_owned(), remote, 1800, None)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -438,8 +461,7 @@ mod tests {
|
||||||
|
|
||||||
assert!(cmd.status().unwrap().success());
|
assert!(cmd.status().unwrap().success());
|
||||||
|
|
||||||
let output = cmd.output()
|
let output = cmd.output().expect("building the test PR failed");
|
||||||
.expect("building the test PR failed");
|
|
||||||
let hash = String::from_utf8(output.stdout).expect("Should just be a hash");
|
let hash = String::from_utf8(output.stdout).expect("Should just be a hash");
|
||||||
|
|
||||||
hash.trim().to_owned()
|
hash.trim().to_owned()
|
||||||
|
@ -453,23 +475,23 @@ mod tests {
|
||||||
}
|
}
|
||||||
|
|
||||||
fn assert_contains_job(actions: &mut IntoIter<worker::Action>, text_to_match: &str) {
|
fn assert_contains_job(actions: &mut IntoIter<worker::Action>, text_to_match: &str) {
|
||||||
println!("\n\n Searching: {:?}", text_to_match);
|
println!("\n\n Searching: {text_to_match:?}");
|
||||||
actions
|
actions
|
||||||
.position(|job| match job {
|
.position(|job| match job {
|
||||||
worker::Action::Publish(ref body) => {
|
worker::Action::Publish(ref body) => {
|
||||||
let content = String::from_utf8(body.content.clone()).unwrap();
|
let content = String::from_utf8(body.content.clone()).unwrap();
|
||||||
let text = strip_escaped_ansi(&content);
|
let text = strip_escaped_ansi(&content);
|
||||||
eprintln!("{}", text);
|
eprintln!("{text}");
|
||||||
if text.contains(text_to_match) {
|
if text.contains(text_to_match) {
|
||||||
println!(" ok");
|
println!(" ok");
|
||||||
true
|
true
|
||||||
} else {
|
} else {
|
||||||
println!(" notContains: {}", text);
|
println!(" notContains: {text}");
|
||||||
false
|
false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
e => {
|
e => {
|
||||||
println!(" notPublish: {:?}", e);
|
println!(" notPublish: {e:?}");
|
||||||
false
|
false
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
@ -481,8 +503,8 @@ mod tests {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[tokio::test]
|
||||||
pub fn test_simple_build() {
|
pub async fn test_simple_build() {
|
||||||
let p = TestScratch::new_dir("build-simple-build-working");
|
let p = TestScratch::new_dir("build-simple-build-working");
|
||||||
let bare_repo = TestScratch::new_dir("build-simple-build-bare");
|
let bare_repo = TestScratch::new_dir("build-simple-build-bare");
|
||||||
let co_repo = TestScratch::new_dir("build-simple-build-co");
|
let co_repo = TestScratch::new_dir("build-simple-build-co");
|
||||||
|
@ -511,7 +533,7 @@ mod tests {
|
||||||
|
|
||||||
let mut dummyreceiver = notifyworker::DummyNotificationReceiver::new();
|
let mut dummyreceiver = notifyworker::DummyNotificationReceiver::new();
|
||||||
|
|
||||||
worker.consumer(&job, &mut dummyreceiver);
|
worker.consumer(&job, &mut dummyreceiver).await;
|
||||||
|
|
||||||
println!("Total actions: {:?}", dummyreceiver.actions.len());
|
println!("Total actions: {:?}", dummyreceiver.actions.len());
|
||||||
let mut actions = dummyreceiver.actions.into_iter();
|
let mut actions = dummyreceiver.actions.into_iter();
|
||||||
|
@ -526,8 +548,8 @@ mod tests {
|
||||||
assert_eq!(actions.next(), Some(worker::Action::Ack));
|
assert_eq!(actions.next(), Some(worker::Action::Ack));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[tokio::test]
|
||||||
pub fn test_all_jobs_skipped() {
|
pub async fn test_all_jobs_skipped() {
|
||||||
let p = TestScratch::new_dir("no-attempt");
|
let p = TestScratch::new_dir("no-attempt");
|
||||||
let bare_repo = TestScratch::new_dir("no-attempt-bare");
|
let bare_repo = TestScratch::new_dir("no-attempt-bare");
|
||||||
let co_repo = TestScratch::new_dir("no-attempt-co");
|
let co_repo = TestScratch::new_dir("no-attempt-co");
|
||||||
|
@ -556,7 +578,7 @@ mod tests {
|
||||||
|
|
||||||
let mut dummyreceiver = notifyworker::DummyNotificationReceiver::new();
|
let mut dummyreceiver = notifyworker::DummyNotificationReceiver::new();
|
||||||
|
|
||||||
worker.consumer(&job, &mut dummyreceiver);
|
worker.consumer(&job, &mut dummyreceiver).await;
|
||||||
|
|
||||||
println!("Total actions: {:?}", dummyreceiver.actions.len());
|
println!("Total actions: {:?}", dummyreceiver.actions.len());
|
||||||
let mut actions = dummyreceiver.actions.into_iter();
|
let mut actions = dummyreceiver.actions.into_iter();
|
||||||
|
|
|
@ -1,46 +0,0 @@
|
||||||
use crate::checkout::CachedProjectCo;
|
|
||||||
use crate::evalchecker::EvalChecker;
|
|
||||||
use crate::tasks::eval::{EvaluationComplete, EvaluationStrategy, StepResult};
|
|
||||||
use crate::vcs::commit_status::CommitStatus;
|
|
||||||
|
|
||||||
use std::path::Path;
|
|
||||||
|
|
||||||
#[derive(Default)]
|
|
||||||
pub struct GenericStrategy {}
|
|
||||||
impl GenericStrategy {
|
|
||||||
pub fn new() -> GenericStrategy {
|
|
||||||
Self {}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl EvaluationStrategy for GenericStrategy {
|
|
||||||
fn pre_clone(&mut self) -> StepResult<()> {
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn on_target_branch(&mut self, _co: &Path, _status: &mut CommitStatus) -> StepResult<()> {
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn after_fetch(&mut self, _co: &CachedProjectCo) -> StepResult<()> {
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn merge_conflict(&mut self) {}
|
|
||||||
|
|
||||||
fn after_merge(&mut self, _status: &mut CommitStatus) -> StepResult<()> {
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn evaluation_checks(&self) -> Vec<EvalChecker> {
|
|
||||||
vec![]
|
|
||||||
}
|
|
||||||
|
|
||||||
fn all_evaluations_passed(
|
|
||||||
&mut self,
|
|
||||||
_co: &Path,
|
|
||||||
_status: &mut CommitStatus,
|
|
||||||
) -> StepResult<EvaluationComplete> {
|
|
||||||
Ok(Default::default())
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,33 +1,12 @@
|
||||||
mod generic;
|
|
||||||
mod nixpkgs;
|
mod nixpkgs;
|
||||||
pub mod stdenvs;
|
pub mod stdenvs;
|
||||||
|
|
||||||
pub use self::generic::GenericStrategy;
|
|
||||||
pub use self::nixpkgs::NixpkgsStrategy;
|
pub use self::nixpkgs::NixpkgsStrategy;
|
||||||
pub use self::stdenvs::Stdenvs;
|
pub use self::stdenvs::Stdenvs;
|
||||||
use crate::checkout::CachedProjectCo;
|
|
||||||
use crate::evalchecker::EvalChecker;
|
|
||||||
use crate::message::buildjob::BuildJob;
|
use crate::message::buildjob::BuildJob;
|
||||||
use crate::vcs::commit_status::{CommitStatus, CommitStatusError};
|
use crate::vcs::commit_status::CommitStatusError;
|
||||||
use crate::vcs::generic::CheckRunOptions;
|
use crate::vcs::generic::CheckRunOptions;
|
||||||
|
|
||||||
use std::path::Path;
|
|
||||||
|
|
||||||
pub trait EvaluationStrategy {
|
|
||||||
fn pre_clone(&mut self) -> StepResult<()>;
|
|
||||||
|
|
||||||
fn on_target_branch(&mut self, co: &Path, status: &mut CommitStatus) -> StepResult<()>;
|
|
||||||
fn after_fetch(&mut self, co: &CachedProjectCo) -> StepResult<()>;
|
|
||||||
fn merge_conflict(&mut self);
|
|
||||||
fn after_merge(&mut self, status: &mut CommitStatus) -> StepResult<()>;
|
|
||||||
fn evaluation_checks(&self) -> Vec<EvalChecker>;
|
|
||||||
fn all_evaluations_passed(
|
|
||||||
&mut self,
|
|
||||||
co: &Path,
|
|
||||||
status: &mut CommitStatus,
|
|
||||||
) -> StepResult<EvaluationComplete>;
|
|
||||||
}
|
|
||||||
|
|
||||||
pub type StepResult<T> = Result<T, Error>;
|
pub type StepResult<T> = Result<T, Error>;
|
||||||
|
|
||||||
#[derive(Default)]
|
#[derive(Default)]
|
||||||
|
|
|
@ -9,19 +9,16 @@ use crate::nix::{self, Nix};
|
||||||
use crate::nixenv::HydraNixEnv;
|
use crate::nixenv::HydraNixEnv;
|
||||||
use crate::outpathdiff::{OutPathDiff, PackageArch};
|
use crate::outpathdiff::{OutPathDiff, PackageArch};
|
||||||
use crate::tagger::{MaintainerPrTagger, PkgsAddedRemovedTagger, RebuildTagger, StdenvTagger};
|
use crate::tagger::{MaintainerPrTagger, PkgsAddedRemovedTagger, RebuildTagger, StdenvTagger};
|
||||||
use crate::tasks::eval::{
|
use crate::tasks::eval::{stdenvs::Stdenvs, Error, EvaluationComplete, StepResult};
|
||||||
stdenvs::Stdenvs, Error, EvaluationComplete, EvaluationStrategy, StepResult,
|
|
||||||
};
|
|
||||||
use crate::vcs::commit_status::CommitStatus;
|
use crate::vcs::commit_status::CommitStatus;
|
||||||
use crate::vcs::generic::{
|
use crate::vcs::generic::{
|
||||||
CheckRunOptions, CheckRunState, Conclusion, State, VersionControlSystemAPI,
|
CheckRunOptions, CheckRunState, Conclusion, State, VersionControlSystemAPI,
|
||||||
};
|
};
|
||||||
|
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use std::rc::Rc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use chrono::Utc;
|
use chrono::Utc;
|
||||||
use futures::executor::block_on;
|
|
||||||
use regex::Regex;
|
use regex::Regex;
|
||||||
use tracing::{info, warn};
|
use tracing::{info, warn};
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
@ -39,7 +36,7 @@ fn label_from_title(title: &str) -> Vec<String> {
|
||||||
let labels: Vec<_> = TITLE_LABELS
|
let labels: Vec<_> = TITLE_LABELS
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|(word, _label)| {
|
.filter(|(word, _label)| {
|
||||||
let re = Regex::new(&format!("\\b{}\\b", word)).unwrap();
|
let re = Regex::new(&format!("\\b{word}\\b")).unwrap();
|
||||||
re.is_match(title)
|
re.is_match(title)
|
||||||
})
|
})
|
||||||
.map(|(_word, label)| (*label).into())
|
.map(|(_word, label)| (*label).into())
|
||||||
|
@ -51,7 +48,7 @@ fn label_from_title(title: &str) -> Vec<String> {
|
||||||
pub struct NixpkgsStrategy<'a> {
|
pub struct NixpkgsStrategy<'a> {
|
||||||
chan: lapin::Channel,
|
chan: lapin::Channel,
|
||||||
job: &'a EvaluationJob,
|
job: &'a EvaluationJob,
|
||||||
vcs_api: Rc<dyn VersionControlSystemAPI>,
|
vcs_api: Arc<dyn VersionControlSystemAPI>,
|
||||||
change: &'a Change,
|
change: &'a Change,
|
||||||
repo: &'a Repo,
|
repo: &'a Repo,
|
||||||
nix: Nix,
|
nix: Nix,
|
||||||
|
@ -66,7 +63,7 @@ impl<'a> NixpkgsStrategy<'a> {
|
||||||
pub fn new(
|
pub fn new(
|
||||||
chan: lapin::Channel,
|
chan: lapin::Channel,
|
||||||
job: &'a EvaluationJob,
|
job: &'a EvaluationJob,
|
||||||
vcs_api: Rc<dyn VersionControlSystemAPI>,
|
vcs_api: Arc<dyn VersionControlSystemAPI>,
|
||||||
repo: &'a Repo,
|
repo: &'a Repo,
|
||||||
change: &'a Change,
|
change: &'a Change,
|
||||||
nix: Nix,
|
nix: Nix,
|
||||||
|
@ -85,47 +82,38 @@ impl<'a> NixpkgsStrategy<'a> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn tag_from_title(&self) {
|
async fn tag_from_title(&self) {
|
||||||
let issue =
|
let Ok(issue) = self.vcs_api.get_issue(self.repo, self.change.number).await else {
|
||||||
match block_on(self.vcs_api.get_issue(self.repo, self.change.number))
|
return;
|
||||||
{
|
};
|
||||||
Ok(issue) => issue,
|
|
||||||
Err(_) => return,
|
|
||||||
};
|
|
||||||
let labels = label_from_title(&issue.title);
|
let labels = label_from_title(&issue.title);
|
||||||
|
|
||||||
if labels.is_empty() {
|
if labels.is_empty() {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
block_on(self.vcs_api.update_labels(
|
self.vcs_api
|
||||||
self.repo,
|
.update_labels(self.repo, self.change.number, &labels, &[])
|
||||||
self.change.number,
|
.await;
|
||||||
&labels,
|
|
||||||
&[],
|
|
||||||
));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn update_labels(&self, to_add: &[String], to_remove: &[String]) {
|
async fn update_labels(&self, to_add: &[String], to_remove: &[String]) {
|
||||||
block_on(self.vcs_api.update_labels(
|
self.vcs_api
|
||||||
self.repo,
|
.update_labels(self.repo, self.change.number, to_add, to_remove)
|
||||||
self.change.number,
|
.await;
|
||||||
to_add,
|
|
||||||
to_remove,
|
|
||||||
));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn request_reviews(&self, impacted_maintainers: &maintainers::ImpactedMaintainers) {
|
async fn request_reviews(&self, impacted_maintainers: &maintainers::ImpactedMaintainers) {
|
||||||
info!(
|
info!(
|
||||||
"Impact maintainers: {:?}",
|
"Impact maintainers: {:?}",
|
||||||
impacted_maintainers.maintainers()
|
impacted_maintainers.maintainers()
|
||||||
);
|
);
|
||||||
|
|
||||||
if impacted_maintainers.maintainers().len() < 10 {
|
if impacted_maintainers.maintainers().len() < 10 {
|
||||||
let existing_reviewers = block_on(
|
let existing_reviewers = self
|
||||||
self.vcs_api
|
.vcs_api
|
||||||
.get_existing_reviewers(self.repo, self.change.number),
|
.get_existing_reviewers(self.repo, self.change.number)
|
||||||
);
|
.await;
|
||||||
|
|
||||||
// Normalize both sides, compute the IM - ER set
|
// Normalize both sides, compute the IM - ER set
|
||||||
let new_reviewers: Vec<String> = impacted_maintainers
|
let new_reviewers: Vec<String> = impacted_maintainers
|
||||||
|
@ -136,12 +124,9 @@ impl<'a> NixpkgsStrategy<'a> {
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
// Add them as reviewers.
|
// Add them as reviewers.
|
||||||
block_on(self.vcs_api.request_reviewers(
|
self.vcs_api
|
||||||
self.repo,
|
.request_reviewers(self.repo, self.change.number, new_reviewers, vec![])
|
||||||
self.change.number,
|
.await;
|
||||||
new_reviewers,
|
|
||||||
vec![],
|
|
||||||
));
|
|
||||||
} else {
|
} else {
|
||||||
warn!(
|
warn!(
|
||||||
"Too many reviewers ({}), skipping review requests",
|
"Too many reviewers ({}), skipping review requests",
|
||||||
|
@ -150,32 +135,33 @@ impl<'a> NixpkgsStrategy<'a> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn check_stdenvs_before(&mut self, dir: &Path) {
|
async fn check_stdenvs_before(&mut self, dir: &Path) {
|
||||||
let mut stdenvs = Stdenvs::new(self.nix.clone(), dir.to_path_buf());
|
let mut stdenvs = Stdenvs::new(self.nix.clone(), dir.to_path_buf());
|
||||||
stdenvs.identify_before();
|
stdenvs.identify_before().await;
|
||||||
self.stdenv_diff = Some(stdenvs);
|
self.stdenv_diff = Some(stdenvs);
|
||||||
}
|
}
|
||||||
|
|
||||||
fn check_stdenvs_after(&mut self) {
|
async fn check_stdenvs_after(&mut self) {
|
||||||
if let Some(ref mut stdenvs) = self.stdenv_diff {
|
if let Some(ref mut stdenvs) = self.stdenv_diff {
|
||||||
stdenvs.identify_after();
|
stdenvs.identify_after().await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn update_stdenv_labels(&self) {
|
async fn update_stdenv_labels(&self) {
|
||||||
if let Some(ref stdenvs) = self.stdenv_diff {
|
if let Some(ref stdenvs) = self.stdenv_diff {
|
||||||
let mut stdenvtagger = StdenvTagger::new();
|
let mut stdenvtagger = StdenvTagger::new();
|
||||||
if !stdenvs.are_same() {
|
if !stdenvs.are_same() {
|
||||||
stdenvtagger.changed(stdenvs.changed());
|
stdenvtagger.changed(stdenvs.changed());
|
||||||
}
|
}
|
||||||
self.update_labels(&stdenvtagger.tags_to_add(), &stdenvtagger.tags_to_remove());
|
self.update_labels(&stdenvtagger.tags_to_add(), &stdenvtagger.tags_to_remove())
|
||||||
|
.await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn check_outpaths_before(&mut self, dir: &Path) -> StepResult<()> {
|
async fn check_outpaths_before(&mut self, dir: &Path) -> StepResult<()> {
|
||||||
let mut rebuildsniff = OutPathDiff::new(self.nix.clone(), dir.to_path_buf());
|
let mut rebuildsniff = OutPathDiff::new(self.nix.clone(), dir.to_path_buf());
|
||||||
|
|
||||||
if let Err(err) = rebuildsniff.find_before() {
|
if let Err(err) = rebuildsniff.find_before().await {
|
||||||
/*
|
/*
|
||||||
self.events
|
self.events
|
||||||
.notify(Event::TargetBranchFailsEvaluation(target_branch.clone()));
|
.notify(Event::TargetBranchFailsEvaluation(target_branch.clone()));
|
||||||
|
@ -192,9 +178,9 @@ impl<'a> NixpkgsStrategy<'a> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn check_outpaths_after(&mut self) -> StepResult<()> {
|
async fn check_outpaths_after(&mut self) -> StepResult<()> {
|
||||||
if let Some(ref mut rebuildsniff) = self.outpath_diff {
|
if let Some(ref mut rebuildsniff) = self.outpath_diff {
|
||||||
if let Err(err) = rebuildsniff.find_after() {
|
if let Err(err) = rebuildsniff.find_after().await {
|
||||||
Err(Error::FailWithPastebin(
|
Err(Error::FailWithPastebin(
|
||||||
String::from("This PR does not cleanly list package outputs after merging."),
|
String::from("This PR does not cleanly list package outputs after merging."),
|
||||||
String::from("Output path comparison"),
|
String::from("Output path comparison"),
|
||||||
|
@ -238,7 +224,7 @@ impl<'a> NixpkgsStrategy<'a> {
|
||||||
vec![]
|
vec![]
|
||||||
}
|
}
|
||||||
|
|
||||||
fn update_new_package_labels(&self) {
|
async fn update_new_package_labels(&self) {
|
||||||
if let Some(ref rebuildsniff) = self.outpath_diff {
|
if let Some(ref rebuildsniff) = self.outpath_diff {
|
||||||
if let Some((removed, added)) = rebuildsniff.package_diff() {
|
if let Some((removed, added)) = rebuildsniff.package_diff() {
|
||||||
let mut addremovetagger = PkgsAddedRemovedTagger::new();
|
let mut addremovetagger = PkgsAddedRemovedTagger::new();
|
||||||
|
@ -246,12 +232,13 @@ impl<'a> NixpkgsStrategy<'a> {
|
||||||
self.update_labels(
|
self.update_labels(
|
||||||
&addremovetagger.tags_to_add(),
|
&addremovetagger.tags_to_add(),
|
||||||
&addremovetagger.tags_to_remove(),
|
&addremovetagger.tags_to_remove(),
|
||||||
);
|
)
|
||||||
|
.await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn update_rebuild_labels(
|
async fn update_rebuild_labels(
|
||||||
&mut self,
|
&mut self,
|
||||||
dir: &Path,
|
dir: &Path,
|
||||||
overall_status: &mut CommitStatus,
|
overall_status: &mut CommitStatus,
|
||||||
|
@ -261,20 +248,21 @@ impl<'a> NixpkgsStrategy<'a> {
|
||||||
|
|
||||||
if let Some(attrs) = rebuildsniff.calculate_rebuild() {
|
if let Some(attrs) = rebuildsniff.calculate_rebuild() {
|
||||||
if !attrs.is_empty() {
|
if !attrs.is_empty() {
|
||||||
overall_status.set_url(self.gist_changed_paths(&attrs));
|
overall_status.set_url(self.gist_changed_paths(&attrs).await);
|
||||||
self.record_impacted_maintainers(dir, &attrs)?;
|
self.record_impacted_maintainers(dir, &attrs).await?;
|
||||||
}
|
}
|
||||||
|
|
||||||
rebuild_tags.parse_attrs(attrs);
|
rebuild_tags.parse_attrs(attrs);
|
||||||
}
|
}
|
||||||
|
|
||||||
self.update_labels(&rebuild_tags.tags_to_add(), &rebuild_tags.tags_to_remove());
|
self.update_labels(&rebuild_tags.tags_to_add(), &rebuild_tags.tags_to_remove())
|
||||||
|
.await;
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn gist_changed_paths(&mut self, attrs: &[PackageArch]) -> Option<String> {
|
async fn gist_changed_paths(&mut self, attrs: &[PackageArch]) -> Option<String> {
|
||||||
block_on(crate::utils::pastebin::make_pastebin(
|
crate::utils::pastebin::make_pastebin(
|
||||||
&mut self.chan,
|
&mut self.chan,
|
||||||
"Changed Paths",
|
"Changed Paths",
|
||||||
attrs
|
attrs
|
||||||
|
@ -282,12 +270,13 @@ impl<'a> NixpkgsStrategy<'a> {
|
||||||
.map(|attr| format!("{}\t{}", &attr.architecture, &attr.package))
|
.map(|attr| format!("{}\t{}", &attr.architecture, &attr.package))
|
||||||
.collect::<Vec<String>>()
|
.collect::<Vec<String>>()
|
||||||
.join("\n"),
|
.join("\n"),
|
||||||
))
|
)
|
||||||
|
.await
|
||||||
.ok()
|
.ok()
|
||||||
.map(|pp| pp.uri)
|
.map(|pp| pp.uri)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn record_impacted_maintainers(
|
async fn record_impacted_maintainers(
|
||||||
&mut self,
|
&mut self,
|
||||||
dir: &Path,
|
dir: &Path,
|
||||||
attrs: &[PackageArch],
|
attrs: &[PackageArch],
|
||||||
|
@ -299,16 +288,18 @@ impl<'a> NixpkgsStrategy<'a> {
|
||||||
|
|
||||||
if let Some(ref changed_paths) = self.changed_paths {
|
if let Some(ref changed_paths) = self.changed_paths {
|
||||||
let m =
|
let m =
|
||||||
ImpactedMaintainers::calculate(&self.nix, dir, changed_paths, &changed_attributes);
|
ImpactedMaintainers::calculate(&self.nix, dir, changed_paths, &changed_attributes)
|
||||||
|
.await;
|
||||||
|
|
||||||
let gist_url = block_on(crate::utils::pastebin::make_pastebin(
|
let gist_url = crate::utils::pastebin::make_pastebin(
|
||||||
&mut self.chan,
|
&mut self.chan,
|
||||||
"Potential Maintainers",
|
"Potential Maintainers",
|
||||||
match m {
|
match m {
|
||||||
Ok(ref maintainers) => format!("Maintainers:\n{}", maintainers),
|
Ok(ref maintainers) => format!("Maintainers:\n{maintainers}"),
|
||||||
Err(ref e) => format!("Ignorable calculation error:\n{:?}", e),
|
Err(ref e) => format!("Ignorable calculation error:\n{e:?}"),
|
||||||
},
|
},
|
||||||
))
|
)
|
||||||
|
.await
|
||||||
.ok()
|
.ok()
|
||||||
.map(|pp| pp.uri);
|
.map(|pp| pp.uri);
|
||||||
|
|
||||||
|
@ -325,7 +316,7 @@ impl<'a> NixpkgsStrategy<'a> {
|
||||||
String::from("large change, skipping automatic review requests"),
|
String::from("large change, skipping automatic review requests"),
|
||||||
gist_url,
|
gist_url,
|
||||||
);
|
);
|
||||||
status.set(State::Success)?;
|
status.set(State::Success).await?;
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -337,26 +328,28 @@ impl<'a> NixpkgsStrategy<'a> {
|
||||||
String::from("matching changed paths to changed attrs..."),
|
String::from("matching changed paths to changed attrs..."),
|
||||||
gist_url,
|
gist_url,
|
||||||
);
|
);
|
||||||
status.set(State::Success)?;
|
status.set(State::Success).await?;
|
||||||
|
|
||||||
if let Ok(ref maint) = m {
|
if let Ok(ref maint) = m {
|
||||||
self.request_reviews(maint);
|
self.request_reviews(maint).await;
|
||||||
let mut maint_tagger = MaintainerPrTagger::new();
|
let mut maint_tagger = MaintainerPrTagger::new();
|
||||||
// TODO: this is really weird.
|
// TODO: this is really weird.
|
||||||
let issue = block_on(
|
let issue = self
|
||||||
self.vcs_api.get_issue(self.repo, self.change.number),
|
.vcs_api
|
||||||
)
|
.get_issue(self.repo, self.change.number)
|
||||||
.expect("Failed to obtain the issue");
|
.await
|
||||||
|
.expect("Failed to obtain the issue");
|
||||||
maint_tagger
|
maint_tagger
|
||||||
.record_maintainer(&issue.created_by.username, &maint.maintainers_by_package());
|
.record_maintainer(&issue.created_by.username, &maint.maintainers_by_package());
|
||||||
self.update_labels(&maint_tagger.tags_to_add(), &maint_tagger.tags_to_remove());
|
self.update_labels(&maint_tagger.tags_to_add(), &maint_tagger.tags_to_remove())
|
||||||
|
.await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn check_meta_queue_builds(&mut self, dir: &Path) -> StepResult<Vec<BuildJob>> {
|
async fn check_meta_queue_builds(&mut self, dir: &Path) -> StepResult<Vec<BuildJob>> {
|
||||||
if let Some(ref possibly_touched_packages) = self.touched_packages {
|
if let Some(ref possibly_touched_packages) = self.touched_packages {
|
||||||
let mut status = CommitStatus::new(
|
let mut status = CommitStatus::new(
|
||||||
self.vcs_api.clone(),
|
self.vcs_api.clone(),
|
||||||
|
@ -366,10 +359,10 @@ impl<'a> NixpkgsStrategy<'a> {
|
||||||
String::from("config.nix: checkMeta = true"),
|
String::from("config.nix: checkMeta = true"),
|
||||||
None,
|
None,
|
||||||
);
|
);
|
||||||
status.set(State::Pending)?;
|
status.set(State::Pending).await?;
|
||||||
|
|
||||||
let nixenv = HydraNixEnv::new(self.nix.clone(), dir.to_path_buf(), true);
|
let nixenv = HydraNixEnv::new(self.nix.clone(), dir.to_path_buf(), true);
|
||||||
match nixenv.execute_with_stats() {
|
match nixenv.execute_with_stats().await {
|
||||||
Ok((pkgs, _stats)) => {
|
Ok((pkgs, _stats)) => {
|
||||||
let mut try_build: Vec<String> = pkgs
|
let mut try_build: Vec<String> = pkgs
|
||||||
.keys()
|
.keys()
|
||||||
|
@ -381,7 +374,7 @@ impl<'a> NixpkgsStrategy<'a> {
|
||||||
try_build.dedup();
|
try_build.dedup();
|
||||||
|
|
||||||
status.set_url(None);
|
status.set_url(None);
|
||||||
status.set(State::Success)?;
|
status.set(State::Success).await?;
|
||||||
|
|
||||||
if !try_build.is_empty() && try_build.len() <= 20 {
|
if !try_build.is_empty() && try_build.len() <= 20 {
|
||||||
// In the case of trying to merge master in to
|
// In the case of trying to merge master in to
|
||||||
|
@ -403,15 +396,16 @@ impl<'a> NixpkgsStrategy<'a> {
|
||||||
}
|
}
|
||||||
Err(out) => {
|
Err(out) => {
|
||||||
status.set_url(
|
status.set_url(
|
||||||
block_on(crate::utils::pastebin::make_pastebin(
|
crate::utils::pastebin::make_pastebin(
|
||||||
&mut self.chan,
|
&mut self.chan,
|
||||||
"Meta Check",
|
"Meta Check",
|
||||||
out.display(),
|
out.display(),
|
||||||
))
|
)
|
||||||
|
.await
|
||||||
.ok()
|
.ok()
|
||||||
.map(|pp| pp.uri),
|
.map(|pp| pp.uri),
|
||||||
);
|
);
|
||||||
status.set(State::Failure)?;
|
status.set(State::Failure).await?;
|
||||||
Err(Error::Fail(String::from(
|
Err(Error::Fail(String::from(
|
||||||
"Failed to validate package metadata.",
|
"Failed to validate package metadata.",
|
||||||
)))
|
)))
|
||||||
|
@ -421,25 +415,31 @@ impl<'a> NixpkgsStrategy<'a> {
|
||||||
Ok(vec![])
|
Ok(vec![])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
impl<'a> EvaluationStrategy for NixpkgsStrategy<'a> {
|
pub(crate) async fn pre_clone(&self) -> StepResult<()> {
|
||||||
fn pre_clone(&mut self) -> StepResult<()> {
|
self.tag_from_title().await;
|
||||||
self.tag_from_title();
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn on_target_branch(&mut self, dir: &Path, status: &mut CommitStatus) -> StepResult<()> {
|
pub(crate) async fn on_target_branch(
|
||||||
status.set_with_description("Checking original stdenvs", State::Pending)?;
|
&mut self,
|
||||||
self.check_stdenvs_before(dir);
|
dir: &Path,
|
||||||
|
status: &mut CommitStatus,
|
||||||
|
) -> StepResult<()> {
|
||||||
|
status
|
||||||
|
.set_with_description("Checking original stdenvs", State::Pending)
|
||||||
|
.await?;
|
||||||
|
self.check_stdenvs_before(dir).await;
|
||||||
|
|
||||||
status.set_with_description("Checking original out paths", State::Pending)?;
|
status
|
||||||
self.check_outpaths_before(dir)?;
|
.set_with_description("Checking original out paths", State::Pending)
|
||||||
|
.await?;
|
||||||
|
self.check_outpaths_before(dir).await?;
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn after_fetch(&mut self, co: &CachedProjectCo) -> StepResult<()> {
|
pub(crate) fn after_fetch(&mut self, co: &CachedProjectCo) {
|
||||||
let changed_paths = co
|
let changed_paths = co
|
||||||
.files_changed_from_head(&self.job.change.head_sha)
|
.files_changed_from_head(&self.job.change.head_sha)
|
||||||
.unwrap_or_else(|_| vec![]);
|
.unwrap_or_else(|_| vec![]);
|
||||||
|
@ -447,29 +447,34 @@ impl<'a> EvaluationStrategy for NixpkgsStrategy<'a> {
|
||||||
|
|
||||||
self.touched_packages = Some(parse_commit_messages(
|
self.touched_packages = Some(parse_commit_messages(
|
||||||
&co.commit_messages_from_head(&self.job.change.head_sha)
|
&co.commit_messages_from_head(&self.job.change.head_sha)
|
||||||
.unwrap_or_else(|_| vec!["".to_owned()]),
|
.unwrap_or_else(|_| vec![String::new()]),
|
||||||
));
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn merge_conflict(&self) {
|
||||||
|
self.update_labels(&["2.status: merge conflict".to_owned()], &[])
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn after_merge(&mut self, status: &mut CommitStatus) -> StepResult<()> {
|
||||||
|
self.update_labels(&[], &["2.status: merge conflict".to_owned()])
|
||||||
|
.await;
|
||||||
|
|
||||||
|
status
|
||||||
|
.set_with_description("Checking new stdenvs", State::Pending)
|
||||||
|
.await?;
|
||||||
|
self.check_stdenvs_after().await;
|
||||||
|
|
||||||
|
status
|
||||||
|
.set_with_description("Checking new out paths", State::Pending)
|
||||||
|
.await?;
|
||||||
|
self.check_outpaths_after().await?;
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn merge_conflict(&mut self) {
|
#[allow(clippy::too_many_lines)]
|
||||||
self.update_labels(&["2.status: merge conflict".to_owned()], &[]);
|
pub(crate) fn evaluation_checks(&self) -> Vec<EvalChecker> {
|
||||||
}
|
|
||||||
|
|
||||||
fn after_merge(&mut self, status: &mut CommitStatus) -> StepResult<()> {
|
|
||||||
self.update_labels(&[], &["2.status: merge conflict".to_owned()]);
|
|
||||||
|
|
||||||
status.set_with_description("Checking new stdenvs", State::Pending)?;
|
|
||||||
self.check_stdenvs_after();
|
|
||||||
|
|
||||||
status.set_with_description("Checking new out paths", State::Pending)?;
|
|
||||||
self.check_outpaths_after()?;
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn evaluation_checks(&self) -> Vec<EvalChecker> {
|
|
||||||
// the value that's passed as the nixpkgs arg
|
// the value that's passed as the nixpkgs arg
|
||||||
let nixpkgs_arg_value = format!(
|
let nixpkgs_arg_value = format!(
|
||||||
"{{ outPath=./.; revCount=999999; shortRev=\"{}\"; rev=\"{}\"; }}",
|
"{{ outPath=./.; revCount=999999; shortRev=\"{}\"; rev=\"{}\"; }}",
|
||||||
|
@ -600,20 +605,22 @@ impl<'a> EvaluationStrategy for NixpkgsStrategy<'a> {
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
||||||
fn all_evaluations_passed(
|
pub(crate) async fn all_evaluations_passed(
|
||||||
&mut self,
|
&mut self,
|
||||||
dir: &Path,
|
dir: &Path,
|
||||||
status: &mut CommitStatus,
|
status: &mut CommitStatus,
|
||||||
) -> StepResult<EvaluationComplete> {
|
) -> StepResult<EvaluationComplete> {
|
||||||
self.update_stdenv_labels();
|
self.update_stdenv_labels().await;
|
||||||
|
|
||||||
status.set_with_description("Calculating Changed Outputs", State::Pending)?;
|
status
|
||||||
|
.set_with_description("Calculating Changed Outputs", State::Pending)
|
||||||
|
.await?;
|
||||||
|
|
||||||
self.update_new_package_labels();
|
self.update_new_package_labels().await;
|
||||||
self.update_rebuild_labels(dir, status)?;
|
self.update_rebuild_labels(dir, status).await?;
|
||||||
let checks = self.performance_stats();
|
let checks = self.performance_stats();
|
||||||
|
|
||||||
let builds = self.check_meta_queue_builds(dir)?;
|
let builds = self.check_meta_queue_builds(dir).await?;
|
||||||
Ok(EvaluationComplete { builds, checks })
|
Ok(EvaluationComplete { builds, checks })
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -628,7 +635,7 @@ fn parse_commit_messages(messages: &[String]) -> Vec<String> {
|
||||||
// NOTE: This transforms `{foo,bar}` into `{{foo,bar}}` and `foo,bar` into `{foo,bar}`,
|
// NOTE: This transforms `{foo,bar}` into `{{foo,bar}}` and `foo,bar` into `{foo,bar}`,
|
||||||
// which allows both the old style (`foo,bar`) and the new style (`{foo,bar}`) to expand to
|
// which allows both the old style (`foo,bar`) and the new style (`{foo,bar}`) to expand to
|
||||||
// `foo` and `bar`.
|
// `foo` and `bar`.
|
||||||
.flat_map(|line| brace_expand::brace_expand(&format!("{{{}}}", line)))
|
.flat_map(|line| brace_expand::brace_expand(&format!("{{{line}}}")))
|
||||||
.map(|line| line.trim().to_owned())
|
.map(|line| line.trim().to_owned())
|
||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
|
@ -674,7 +681,7 @@ mod tests {
|
||||||
firefox{,-beta}{,-bin}, librewolf: blah blah blah
|
firefox{,-beta}{,-bin}, librewolf: blah blah blah
|
||||||
"
|
"
|
||||||
.lines()
|
.lines()
|
||||||
.map(|l| l.to_owned())
|
.map(std::borrow::ToOwned::to_owned)
|
||||||
.collect::<Vec<String>>(),
|
.collect::<Vec<String>>(),
|
||||||
),
|
),
|
||||||
expect
|
expect
|
||||||
|
@ -709,7 +716,10 @@ mod tests {
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
label_from_title("fix build on bsd and darwin"),
|
label_from_title("fix build on bsd and darwin"),
|
||||||
[String::from("6.topic: bsd"), String::from("6.topic: darwin")]
|
[
|
||||||
|
String::from("6.topic: bsd"),
|
||||||
|
String::from("6.topic: darwin")
|
||||||
|
]
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
label_from_title("pkg: fix cross"),
|
label_from_title("pkg: fix cross"),
|
||||||
|
|
|
@ -29,6 +29,7 @@ pub struct Stdenvs {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Stdenvs {
|
impl Stdenvs {
|
||||||
|
#[must_use]
|
||||||
pub fn new(nix: nix::Nix, co: PathBuf) -> Stdenvs {
|
pub fn new(nix: nix::Nix, co: PathBuf) -> Stdenvs {
|
||||||
Stdenvs {
|
Stdenvs {
|
||||||
nix,
|
nix,
|
||||||
|
@ -42,20 +43,22 @@ impl Stdenvs {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn identify_before(&mut self) {
|
pub async fn identify_before(&mut self) {
|
||||||
self.identify(System::X8664Linux, StdenvFrom::Before);
|
self.identify(System::X8664Linux, StdenvFrom::Before).await;
|
||||||
self.identify(System::X8664Darwin, StdenvFrom::Before);
|
self.identify(System::X8664Darwin, StdenvFrom::Before).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn identify_after(&mut self) {
|
pub async fn identify_after(&mut self) {
|
||||||
self.identify(System::X8664Linux, StdenvFrom::After);
|
self.identify(System::X8664Linux, StdenvFrom::After).await;
|
||||||
self.identify(System::X8664Darwin, StdenvFrom::After);
|
self.identify(System::X8664Darwin, StdenvFrom::After).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub fn are_same(&self) -> bool {
|
pub fn are_same(&self) -> bool {
|
||||||
self.changed().is_empty()
|
self.changed().is_empty()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub fn changed(&self) -> Vec<System> {
|
pub fn changed(&self) -> Vec<System> {
|
||||||
let mut changed: Vec<System> = vec![];
|
let mut changed: Vec<System> = vec![];
|
||||||
|
|
||||||
|
@ -70,39 +73,42 @@ impl Stdenvs {
|
||||||
changed
|
changed
|
||||||
}
|
}
|
||||||
|
|
||||||
fn identify(&mut self, system: System, from: StdenvFrom) {
|
async fn identify(&mut self, system: System, from: StdenvFrom) {
|
||||||
match (system, from) {
|
match (system, from) {
|
||||||
(System::X8664Linux, StdenvFrom::Before) => {
|
(System::X8664Linux, StdenvFrom::Before) => {
|
||||||
self.linux_stdenv_before = self.evalstdenv("x86_64-linux");
|
self.linux_stdenv_before = self.evalstdenv("x86_64-linux").await;
|
||||||
}
|
}
|
||||||
(System::X8664Linux, StdenvFrom::After) => {
|
(System::X8664Linux, StdenvFrom::After) => {
|
||||||
self.linux_stdenv_after = self.evalstdenv("x86_64-linux");
|
self.linux_stdenv_after = self.evalstdenv("x86_64-linux").await;
|
||||||
}
|
}
|
||||||
|
|
||||||
(System::X8664Darwin, StdenvFrom::Before) => {
|
(System::X8664Darwin, StdenvFrom::Before) => {
|
||||||
self.darwin_stdenv_before = self.evalstdenv("x86_64-darwin");
|
self.darwin_stdenv_before = self.evalstdenv("x86_64-darwin").await;
|
||||||
}
|
}
|
||||||
(System::X8664Darwin, StdenvFrom::After) => {
|
(System::X8664Darwin, StdenvFrom::After) => {
|
||||||
self.darwin_stdenv_after = self.evalstdenv("x86_64-darwin");
|
self.darwin_stdenv_after = self.evalstdenv("x86_64-darwin").await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// This is used to find out what the output path of the stdenv for the
|
/// This is used to find out what the output path of the stdenv for the
|
||||||
/// given system.
|
/// given system.
|
||||||
fn evalstdenv(&self, system: &str) -> Option<String> {
|
async fn evalstdenv(&self, system: &str) -> Option<String> {
|
||||||
info!(?system, "query stdenv output");
|
info!(?system, "query stdenv output");
|
||||||
let result = self.nix.with_system(system.to_owned()).safely(
|
let nix = self.nix.with_system(system.to_owned());
|
||||||
&nix::Operation::QueryPackagesOutputs,
|
let result = nix
|
||||||
&self.co,
|
.safely(
|
||||||
vec![
|
&nix::Operation::QueryPackagesOutputs,
|
||||||
String::from("-f"),
|
&self.co,
|
||||||
String::from("."),
|
vec![
|
||||||
String::from("-A"),
|
String::from("-f"),
|
||||||
String::from("stdenv"),
|
String::from("."),
|
||||||
],
|
String::from("-A"),
|
||||||
true,
|
String::from("stdenv"),
|
||||||
);
|
],
|
||||||
|
true,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
match result {
|
match result {
|
||||||
Ok(mut out) => Some(file_to_str(&mut out)),
|
Ok(mut out) => Some(file_to_str(&mut out)),
|
||||||
|
@ -121,8 +127,8 @@ mod tests {
|
||||||
use std::env;
|
use std::env;
|
||||||
use std::process::Command;
|
use std::process::Command;
|
||||||
|
|
||||||
#[test]
|
#[tokio::test]
|
||||||
fn stdenv_checking() {
|
async fn stdenv_checking() {
|
||||||
let output = Command::new("nix-instantiate")
|
let output = Command::new("nix-instantiate")
|
||||||
.args(["--eval", "-E", "<nixpkgs>"])
|
.args(["--eval", "-E", "<nixpkgs>"])
|
||||||
.output()
|
.output()
|
||||||
|
@ -130,14 +136,20 @@ mod tests {
|
||||||
|
|
||||||
let nixpkgs = String::from_utf8(output.stdout).expect("nixpkgs required");
|
let nixpkgs = String::from_utf8(output.stdout).expect("nixpkgs required");
|
||||||
|
|
||||||
let remote = env::var("NIX_REMOTE").unwrap_or("".to_owned());
|
let remote = env::var("NIX_REMOTE").unwrap_or_default();
|
||||||
let nix = nix::Nix::new(String::from("x86_64-linux"), remote, 1200, None);
|
let nix = nix::Nix::new(String::from("x86_64-linux"), remote, 1200, None);
|
||||||
let mut stdenv = Stdenvs::new(nix, PathBuf::from(nixpkgs.trim_end()));
|
let mut stdenv = Stdenvs::new(nix, PathBuf::from(nixpkgs.trim_end()));
|
||||||
stdenv.identify(System::X8664Linux, StdenvFrom::Before);
|
stdenv
|
||||||
stdenv.identify(System::X8664Darwin, StdenvFrom::Before);
|
.identify(System::X8664Linux, StdenvFrom::Before)
|
||||||
|
.await;
|
||||||
|
stdenv
|
||||||
|
.identify(System::X8664Darwin, StdenvFrom::Before)
|
||||||
|
.await;
|
||||||
|
|
||||||
stdenv.identify(System::X8664Linux, StdenvFrom::After);
|
stdenv.identify(System::X8664Linux, StdenvFrom::After).await;
|
||||||
stdenv.identify(System::X8664Darwin, StdenvFrom::After);
|
stdenv
|
||||||
|
.identify(System::X8664Darwin, StdenvFrom::After)
|
||||||
|
.await;
|
||||||
|
|
||||||
assert!(stdenv.are_same());
|
assert!(stdenv.are_same());
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,6 +2,7 @@
|
||||||
use crate::acl::Acl;
|
use crate::acl::Acl;
|
||||||
use crate::checkout;
|
use crate::checkout;
|
||||||
use crate::files::file_to_str;
|
use crate::files::file_to_str;
|
||||||
|
use crate::message::evaluationjob::Actions;
|
||||||
use crate::message::{buildjob, evaluationjob};
|
use crate::message::{buildjob, evaluationjob};
|
||||||
use crate::nix;
|
use crate::nix;
|
||||||
use crate::stats::{self, Event};
|
use crate::stats::{self, Event};
|
||||||
|
@ -14,10 +15,10 @@ use crate::vcs::gerrit::http::GerritHTTPApi;
|
||||||
use crate::worker;
|
use crate::worker;
|
||||||
|
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use std::rc::Rc;
|
use std::sync::Arc;
|
||||||
use std::time::Instant;
|
use std::time::Instant;
|
||||||
|
|
||||||
use futures::executor::block_on;
|
use async_trait::async_trait;
|
||||||
use tracing::{debug_span, error, info, warn};
|
use tracing::{debug_span, error, info, warn};
|
||||||
|
|
||||||
pub enum SupportedVCS {
|
pub enum SupportedVCS {
|
||||||
|
@ -54,18 +55,24 @@ impl<E: stats::SysEvents> EvaluationWorker<E> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<E: stats::SysEvents + 'static> worker::SimpleWorker for EvaluationWorker<E> {
|
#[async_trait]
|
||||||
|
impl<E: stats::SysEvents + 'static + Sync + Send> worker::SimpleWorker for EvaluationWorker<E> {
|
||||||
type J = evaluationjob::EvaluationJob;
|
type J = evaluationjob::EvaluationJob;
|
||||||
|
|
||||||
fn msg_to_job(&mut self, _: &str, _: &Option<String>, body: &[u8]) -> Result<Self::J, String> {
|
async fn msg_to_job(
|
||||||
self.events.notify(Event::JobReceived);
|
&mut self,
|
||||||
|
_: &str,
|
||||||
|
_: &Option<String>,
|
||||||
|
body: &[u8],
|
||||||
|
) -> Result<Self::J, String> {
|
||||||
|
self.events.notify(Event::JobReceived).await;
|
||||||
match evaluationjob::from(body) {
|
match evaluationjob::from(body) {
|
||||||
Ok(e) => {
|
Ok(e) => {
|
||||||
self.events.notify(Event::JobDecodeSuccess);
|
self.events.notify(Event::JobDecodeSuccess).await;
|
||||||
Ok(e)
|
Ok(e)
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
self.events.notify(Event::JobDecodeFailure);
|
self.events.notify(Event::JobDecodeFailure).await;
|
||||||
error!(
|
error!(
|
||||||
"Failed to decode message: {:?}, Err: {:?}",
|
"Failed to decode message: {:?}, Err: {:?}",
|
||||||
String::from_utf8(body.to_vec()),
|
String::from_utf8(body.to_vec()),
|
||||||
|
@ -76,16 +83,16 @@ impl<E: stats::SysEvents + 'static> worker::SimpleWorker for EvaluationWorker<E>
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn consumer(
|
async fn consumer(
|
||||||
&mut self,
|
&mut self,
|
||||||
chan: &mut lapin::Channel,
|
chan: &mut lapin::Channel,
|
||||||
job: &evaluationjob::EvaluationJob,
|
job: &evaluationjob::EvaluationJob,
|
||||||
) -> worker::Actions {
|
) -> worker::Actions {
|
||||||
let span = debug_span!("job", change_id = ?job.change.number);
|
let span: tracing::Span = debug_span!("job", change_id = ?job.change.number);
|
||||||
let _enter = span.enter();
|
let _enter = span.enter();
|
||||||
|
|
||||||
let vcs_api: Rc<dyn VersionControlSystemAPI> = match self.vcs {
|
let vcs_api: Arc<dyn VersionControlSystemAPI> = match self.vcs {
|
||||||
SupportedVCS::Gerrit => Rc::new(GerritHTTPApi),
|
SupportedVCS::Gerrit => Arc::new(GerritHTTPApi),
|
||||||
};
|
};
|
||||||
|
|
||||||
OneEval::new(
|
OneEval::new(
|
||||||
|
@ -98,11 +105,12 @@ impl<E: stats::SysEvents + 'static> worker::SimpleWorker for EvaluationWorker<E>
|
||||||
job,
|
job,
|
||||||
)
|
)
|
||||||
.worker_actions(chan)
|
.worker_actions(chan)
|
||||||
|
.await
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
struct OneEval<'a, E> {
|
struct OneEval<'a, E> {
|
||||||
vcs_api: Rc<dyn VersionControlSystemAPI>,
|
vcs_api: Arc<dyn VersionControlSystemAPI>,
|
||||||
nix: &'a nix::Nix,
|
nix: &'a nix::Nix,
|
||||||
acl: &'a Acl,
|
acl: &'a Acl,
|
||||||
events: &'a mut E,
|
events: &'a mut E,
|
||||||
|
@ -114,7 +122,7 @@ struct OneEval<'a, E> {
|
||||||
impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> {
|
impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> {
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
fn new(
|
fn new(
|
||||||
vcs_api: Rc<dyn VersionControlSystemAPI>,
|
vcs_api: Arc<dyn VersionControlSystemAPI>,
|
||||||
nix: &'a nix::Nix,
|
nix: &'a nix::Nix,
|
||||||
acl: &'a Acl,
|
acl: &'a Acl,
|
||||||
events: &'a mut E,
|
events: &'a mut E,
|
||||||
|
@ -133,11 +141,7 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn actions(&self) -> evaluationjob::Actions {
|
async fn update_status(
|
||||||
evaluationjob::Actions {}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn update_status(
|
|
||||||
&self,
|
&self,
|
||||||
description: String,
|
description: String,
|
||||||
url: Option<String>,
|
url: Option<String>,
|
||||||
|
@ -158,42 +162,50 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> {
|
||||||
&self.job.change.number, &self.job.change.head_sha, &description
|
&self.job.change.number, &self.job.change.head_sha, &description
|
||||||
);
|
);
|
||||||
|
|
||||||
block_on(self.vcs_api.create_commit_statuses(
|
self.vcs_api
|
||||||
&self.job.repo,
|
.create_commit_statuses(
|
||||||
self.job.change.head_sha.clone(),
|
&self.job.repo,
|
||||||
state,
|
self.job.change.head_sha.clone(),
|
||||||
"ofborg-eval".to_owned(),
|
state,
|
||||||
description,
|
"ofborg-eval".to_owned(),
|
||||||
// TODO: make this an option
|
description,
|
||||||
url.unwrap_or_else(|| "".to_owned()),
|
// TODO: make this an option
|
||||||
))
|
url.unwrap_or_default(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
fn make_pastebin(
|
async fn make_pastebin(
|
||||||
&mut self,
|
&mut self,
|
||||||
chan: &mut lapin::Channel,
|
chan: &mut lapin::Channel,
|
||||||
title: &str,
|
title: &str,
|
||||||
contents: String,
|
contents: String,
|
||||||
) -> Option<PersistedPastebin> {
|
) -> Option<PersistedPastebin> {
|
||||||
block_on(crate::utils::pastebin::make_pastebin(chan, title, contents)).ok()
|
crate::utils::pastebin::make_pastebin(chan, title, contents)
|
||||||
|
.await
|
||||||
|
.ok()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn worker_actions(&mut self, chan: &mut lapin::Channel) -> worker::Actions {
|
async fn worker_actions(&mut self, chan: &mut lapin::Channel) -> worker::Actions {
|
||||||
let eval_result = self
|
let eval_result = match self.evaluate_job(chan).await {
|
||||||
.evaluate_job(chan)
|
Ok(r) => Ok(r),
|
||||||
.map_err(|eval_error| match eval_error {
|
// Handle error cases which expect us to post statuses
|
||||||
// Handle error cases which expect us to post statuses
|
// to github. Convert Eval Errors in to Result<_, CommitStatusWrite>
|
||||||
// to github. Convert Eval Errors in to Result<_, CommitStatusWrite>
|
Err(EvalWorkerError::EvalError(eval::Error::Fail(msg))) => {
|
||||||
EvalWorkerError::EvalError(eval::Error::Fail(msg)) => {
|
Err(self.update_status(msg, None, State::Failure).await)
|
||||||
self.update_status(msg, None, State::Failure)
|
}
|
||||||
}
|
Err(EvalWorkerError::EvalError(eval::Error::FailWithPastebin(msg, title, content))) => {
|
||||||
EvalWorkerError::EvalError(eval::Error::FailWithPastebin(msg, title, content)) => {
|
let pastebin = self
|
||||||
let pastebin = self.make_pastebin(chan, &title, content).map(|pp| pp.uri);
|
.make_pastebin(chan, &title, content)
|
||||||
self.update_status(msg, pastebin, State::Failure)
|
.await
|
||||||
}
|
.map(|pp| pp.uri);
|
||||||
EvalWorkerError::EvalError(eval::Error::CommitStatusWrite(e)) => Err(e),
|
Err(self.update_status(msg, pastebin, State::Failure).await)
|
||||||
EvalWorkerError::CommitStatusWrite(e) => Err(e),
|
}
|
||||||
});
|
Err(
|
||||||
|
EvalWorkerError::EvalError(eval::Error::CommitStatusWrite(e))
|
||||||
|
| EvalWorkerError::CommitStatusWrite(e),
|
||||||
|
) => Err(Err(e)),
|
||||||
|
};
|
||||||
|
|
||||||
match eval_result {
|
match eval_result {
|
||||||
Ok(eval_actions) => eval_actions,
|
Ok(eval_actions) => eval_actions,
|
||||||
|
@ -201,18 +213,18 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> {
|
||||||
// There was an error during eval, but we successfully
|
// There was an error during eval, but we successfully
|
||||||
// updated the PR.
|
// updated the PR.
|
||||||
|
|
||||||
self.actions().skip(self.job)
|
Actions::skip(self.job)
|
||||||
}
|
}
|
||||||
Err(Err(CommitStatusError::ExpiredCreds(e))) => {
|
Err(Err(CommitStatusError::ExpiredCreds(e))) => {
|
||||||
error!("Failed writing commit status: creds expired: {:?}", e);
|
error!("Failed writing commit status: creds expired: {:?}", e);
|
||||||
self.actions().retry_later(self.job)
|
Actions::retry_later(self.job)
|
||||||
}
|
}
|
||||||
Err(Err(CommitStatusError::MissingSha(e))) => {
|
Err(Err(CommitStatusError::MissingSha(e))) => {
|
||||||
error!(
|
error!(
|
||||||
"Failed writing commit status: commit sha was force-pushed away: {:?}",
|
"Failed writing commit status: commit sha was force-pushed away: {:?}",
|
||||||
e
|
e
|
||||||
);
|
);
|
||||||
self.actions().skip(self.job)
|
Actions::skip(self.job)
|
||||||
}
|
}
|
||||||
|
|
||||||
Err(Err(CommitStatusError::Error(cswerr))) => {
|
Err(Err(CommitStatusError::Error(cswerr))) => {
|
||||||
|
@ -221,38 +233,41 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> {
|
||||||
cswerr
|
cswerr
|
||||||
);
|
);
|
||||||
|
|
||||||
block_on(self.vcs_api.update_labels(
|
self.vcs_api
|
||||||
&self.job.repo,
|
.update_labels(
|
||||||
self.job.change.number,
|
&self.job.repo,
|
||||||
&[String::from("ofborg-internal-error")],
|
self.job.change.number,
|
||||||
&[],
|
&[String::from("ofborg-internal-error")],
|
||||||
));
|
&[],
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
self.actions().skip(self.job)
|
Actions::skip(self.job)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// FIXME: remove with rust/cargo update
|
// FIXME: remove with rust/cargo update
|
||||||
#[allow(clippy::cognitive_complexity)]
|
#[allow(clippy::cognitive_complexity)]
|
||||||
fn evaluate_job(
|
#[allow(clippy::too_many_lines)]
|
||||||
|
async fn evaluate_job(
|
||||||
&mut self,
|
&mut self,
|
||||||
chan: &mut lapin::Channel,
|
chan: &mut lapin::Channel,
|
||||||
) -> Result<worker::Actions, EvalWorkerError> {
|
) -> Result<worker::Actions, EvalWorkerError> {
|
||||||
let job = self.job;
|
let job = self.job;
|
||||||
let issue_ref = block_on(self.vcs_api.get_issue(&job.repo, job.change.number));
|
let issue_ref = self.vcs_api.get_issue(&job.repo, job.change.number).await;
|
||||||
let auto_schedule_build_archs: Vec<systems::System>;
|
let auto_schedule_build_archs: Vec<systems::System>;
|
||||||
|
|
||||||
let _issue: Issue = match issue_ref {
|
let _issue: Issue = match issue_ref {
|
||||||
Ok(iss) => {
|
Ok(iss) => {
|
||||||
if matches!(iss.state, IssueState::Closed) {
|
if matches!(iss.state, IssueState::Closed) {
|
||||||
self.events.notify(Event::IssueAlreadyClosed);
|
self.events.notify(Event::IssueAlreadyClosed).await;
|
||||||
info!("Skipping {} because it is closed", job.change.number);
|
info!("Skipping {} because it is closed", job.change.number);
|
||||||
return Ok(self.actions().skip(job));
|
return Ok(Actions::skip(job));
|
||||||
}
|
}
|
||||||
|
|
||||||
if iss.is_wip() {
|
if iss.is_wip() {
|
||||||
self.events.notify(Event::CurrentlyWorkInProgress);
|
self.events.notify(Event::CurrentlyWorkInProgress).await;
|
||||||
auto_schedule_build_archs = vec![];
|
auto_schedule_build_archs = vec![];
|
||||||
} else {
|
} else {
|
||||||
auto_schedule_build_archs = self.acl.build_job_architectures_for_user_repo(
|
auto_schedule_build_archs = self.acl.build_job_architectures_for_user_repo(
|
||||||
|
@ -265,25 +280,21 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> {
|
||||||
}
|
}
|
||||||
|
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
self.events.notify(Event::IssueFetchFailed);
|
self.events.notify(Event::IssueFetchFailed).await;
|
||||||
error!("Error fetching {}!", job.change.number);
|
error!("Error fetching {}!", job.change.number);
|
||||||
error!("E: {:?}", e);
|
error!("E: {:?}", e);
|
||||||
return Ok(self.actions().skip(job));
|
return Ok(Actions::skip(job));
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
let mut evaluation_strategy: Box<dyn eval::EvaluationStrategy> = if job.is_nixpkgs() {
|
let mut evaluation_strategy = eval::NixpkgsStrategy::new(
|
||||||
Box::new(eval::NixpkgsStrategy::new(
|
chan.clone(),
|
||||||
chan.clone(),
|
job,
|
||||||
job,
|
self.vcs_api.clone(),
|
||||||
self.vcs_api.clone(),
|
&job.repo,
|
||||||
&job.repo,
|
&job.change,
|
||||||
&job.change,
|
self.nix.clone(),
|
||||||
self.nix.clone(),
|
);
|
||||||
))
|
|
||||||
} else {
|
|
||||||
Box::new(eval::GenericStrategy::new())
|
|
||||||
};
|
|
||||||
|
|
||||||
let mut overall_status = CommitStatus::new(
|
let mut overall_status = CommitStatus::new(
|
||||||
self.vcs_api.clone(),
|
self.vcs_api.clone(),
|
||||||
|
@ -294,15 +305,19 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> {
|
||||||
None,
|
None,
|
||||||
);
|
);
|
||||||
|
|
||||||
overall_status.set_with_description("Starting", State::Pending)?;
|
overall_status
|
||||||
|
.set_with_description("Starting", State::Pending)
|
||||||
|
.await?;
|
||||||
|
|
||||||
evaluation_strategy.pre_clone()?;
|
evaluation_strategy.pre_clone().await?;
|
||||||
|
|
||||||
let project = self
|
let project = self
|
||||||
.cloner
|
.cloner
|
||||||
.project(&job.repo.full_name, job.repo.clone_url.clone());
|
.project(&job.repo.full_name, job.repo.clone_url.clone());
|
||||||
|
|
||||||
overall_status.set_with_description("Cloning project", State::Pending)?;
|
overall_status
|
||||||
|
.set_with_description("Cloning project", State::Pending)
|
||||||
|
.await?;
|
||||||
|
|
||||||
info!("Working on {}", job.change.number);
|
info!("Working on {}", job.change.number);
|
||||||
let co = project
|
let co = project
|
||||||
|
@ -317,142 +332,163 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> {
|
||||||
|
|
||||||
// TODO: this is a preflight check, encode it as such.
|
// TODO: this is a preflight check, encode it as such.
|
||||||
if target_branch.starts_with("nixos-") || target_branch.starts_with("nixpkgs-") {
|
if target_branch.starts_with("nixos-") || target_branch.starts_with("nixpkgs-") {
|
||||||
overall_status.set_with_description(
|
overall_status
|
||||||
"The branch you have targeted is a read-only mirror for channels. \
|
.set_with_description(
|
||||||
|
"The branch you have targeted is a read-only mirror for channels. \
|
||||||
Please target release-* or master.",
|
Please target release-* or master.",
|
||||||
State::Error,
|
State::Error,
|
||||||
)?;
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
info!("PR targets a nixos-* or nixpkgs-* branch");
|
info!("PR targets a nixos-* or nixpkgs-* branch");
|
||||||
return Ok(self.actions().skip(job));
|
return Ok(Actions::skip(job));
|
||||||
};
|
};
|
||||||
|
|
||||||
overall_status.set_with_description(
|
overall_status
|
||||||
format!("Checking out {}", &target_branch).as_ref(),
|
.set_with_description(
|
||||||
State::Pending,
|
format!("Checking out {}", &target_branch).as_ref(),
|
||||||
)?;
|
State::Pending,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
info!("Checking out target branch {}", &target_branch);
|
info!("Checking out target branch {}", &target_branch);
|
||||||
let refpath = co.checkout_origin_ref(target_branch.as_ref()).unwrap();
|
let refpath = co.checkout_origin_ref(target_branch.as_ref()).unwrap();
|
||||||
|
|
||||||
evaluation_strategy.on_target_branch(Path::new(&refpath), &mut overall_status)?;
|
evaluation_strategy
|
||||||
|
.on_target_branch(Path::new(&refpath), &mut overall_status)
|
||||||
|
.await?;
|
||||||
|
|
||||||
let target_branch_rebuild_sniff_start = Instant::now();
|
let target_branch_rebuild_sniff_start = Instant::now();
|
||||||
|
|
||||||
self.events.notify(Event::EvaluationDuration(
|
|
||||||
target_branch.clone(),
|
|
||||||
target_branch_rebuild_sniff_start.elapsed().as_secs(),
|
|
||||||
));
|
|
||||||
self.events
|
self.events
|
||||||
.notify(Event::EvaluationDurationCount(target_branch));
|
.notify(Event::EvaluationDuration(
|
||||||
|
target_branch.clone(),
|
||||||
|
target_branch_rebuild_sniff_start.elapsed().as_secs(),
|
||||||
|
))
|
||||||
|
.await;
|
||||||
|
self.events
|
||||||
|
.notify(Event::EvaluationDurationCount(target_branch))
|
||||||
|
.await;
|
||||||
|
|
||||||
overall_status.set_with_description("Fetching PR", State::Pending)?;
|
overall_status
|
||||||
|
.set_with_description("Fetching PR", State::Pending)
|
||||||
|
.await?;
|
||||||
|
|
||||||
// TODO: generalize fetch change
|
// TODO: generalize fetch change
|
||||||
co.fetch_pr(job.change.number).unwrap();
|
co.fetch_pr(job.change.number).unwrap();
|
||||||
|
|
||||||
if !co.commit_exists(job.change.head_sha.as_ref()) {
|
if !co.commit_exists(job.change.head_sha.as_ref()) {
|
||||||
overall_status.set_with_description("Commit not found", State::Error)?;
|
overall_status
|
||||||
|
.set_with_description("Commit not found", State::Error)
|
||||||
|
.await?;
|
||||||
|
|
||||||
info!("Commit {} doesn't exist", job.change.head_sha);
|
info!("Commit {} doesn't exist", job.change.head_sha);
|
||||||
return Ok(self.actions().skip(job));
|
return Ok(Actions::skip(job));
|
||||||
}
|
}
|
||||||
|
|
||||||
evaluation_strategy.after_fetch(&co)?;
|
evaluation_strategy.after_fetch(&co);
|
||||||
|
|
||||||
overall_status.set_with_description("Merging PR", State::Pending)?;
|
overall_status
|
||||||
|
.set_with_description("Merging PR", State::Pending)
|
||||||
|
.await?;
|
||||||
|
|
||||||
if co.merge_commit(job.change.head_sha.as_ref()).is_err() {
|
if co.merge_commit(job.change.head_sha.as_ref()).is_err() {
|
||||||
overall_status.set_with_description("Failed to merge", State::Failure)?;
|
overall_status
|
||||||
|
.set_with_description("Failed to merge", State::Failure)
|
||||||
|
.await?;
|
||||||
|
|
||||||
info!("Failed to merge {}", job.change.head_sha);
|
info!("Failed to merge {}", job.change.head_sha);
|
||||||
|
|
||||||
evaluation_strategy.merge_conflict();
|
evaluation_strategy.merge_conflict().await;
|
||||||
|
|
||||||
return Ok(self.actions().skip(job));
|
return Ok(Actions::skip(job));
|
||||||
}
|
}
|
||||||
|
|
||||||
evaluation_strategy.after_merge(&mut overall_status)?;
|
evaluation_strategy.after_merge(&mut overall_status).await?;
|
||||||
|
|
||||||
info!("Got path: {:?}, building", refpath);
|
info!("Got path: {:?}, building", refpath);
|
||||||
overall_status.set_with_description("Beginning Evaluations", State::Pending)?;
|
overall_status
|
||||||
|
.set_with_description("Beginning Evaluations", State::Pending)
|
||||||
|
.await?;
|
||||||
|
|
||||||
let eval_results: bool = evaluation_strategy
|
let mut all_good = true;
|
||||||
.evaluation_checks()
|
for check in evaluation_strategy.evaluation_checks() {
|
||||||
.into_iter()
|
let mut status = CommitStatus::new(
|
||||||
.map(|check| {
|
self.vcs_api.clone(),
|
||||||
let mut status = CommitStatus::new(
|
job.repo.clone(),
|
||||||
self.vcs_api.clone(),
|
job.change.head_sha.clone(),
|
||||||
job.repo.clone(),
|
format!("ofborg-eval-{}", check.name()),
|
||||||
job.change.head_sha.clone(),
|
check.cli_cmd(),
|
||||||
format!("ofborg-eval-{}", check.name()),
|
None,
|
||||||
check.cli_cmd(),
|
);
|
||||||
None,
|
|
||||||
);
|
|
||||||
|
|
||||||
status
|
status
|
||||||
.set(State::Pending)
|
.set(State::Pending)
|
||||||
.expect("Failed to set status on eval strategy");
|
.await
|
||||||
|
.expect("Failed to set status on eval strategy");
|
||||||
|
|
||||||
let state: State;
|
let state: State;
|
||||||
let gist_url: Option<String>;
|
let gist_url: Option<String>;
|
||||||
match check.execute(Path::new(&refpath)) {
|
match check.execute(Path::new(&refpath)).await {
|
||||||
Ok(_) => {
|
Ok(_) => {
|
||||||
state = State::Success;
|
state = State::Success;
|
||||||
gist_url = None;
|
gist_url = None;
|
||||||
}
|
|
||||||
Err(mut out) => {
|
|
||||||
state = State::Failure;
|
|
||||||
gist_url = self
|
|
||||||
.make_pastebin(
|
|
||||||
chan,
|
|
||||||
&format!("[ofborg] Evaluation of {}", check.name()),
|
|
||||||
file_to_str(&mut out),
|
|
||||||
)
|
|
||||||
.map(|pp| pp.uri);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
Err(mut out) => {
|
||||||
status.set_url(gist_url);
|
state = State::Failure;
|
||||||
status
|
gist_url = self
|
||||||
.set(state)
|
.make_pastebin(
|
||||||
.expect("Failed to set status on eval strategy");
|
chan,
|
||||||
|
&format!("[ofborg] Evaluation of {}", check.name()),
|
||||||
if state == State::Success {
|
file_to_str(&mut out),
|
||||||
Ok(())
|
)
|
||||||
} else {
|
.await
|
||||||
Err(())
|
.map(|pp| pp.uri);
|
||||||
}
|
}
|
||||||
})
|
}
|
||||||
.all(|status| status == Ok(()));
|
|
||||||
|
status.set_url(gist_url);
|
||||||
|
status
|
||||||
|
.set(state)
|
||||||
|
.await
|
||||||
|
.expect("Failed to set status on eval strategy");
|
||||||
|
|
||||||
|
if state != State::Success {
|
||||||
|
all_good = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
info!("Finished evaluations");
|
info!("Finished evaluations");
|
||||||
let mut response: worker::Actions = vec![];
|
let mut response: worker::Actions = vec![];
|
||||||
|
|
||||||
if eval_results {
|
if all_good {
|
||||||
let complete = evaluation_strategy
|
let complete = evaluation_strategy
|
||||||
.all_evaluations_passed(Path::new(&refpath), &mut overall_status)?;
|
.all_evaluations_passed(Path::new(&refpath), &mut overall_status)
|
||||||
|
.await?;
|
||||||
|
|
||||||
block_on(
|
self.vcs_api
|
||||||
self.vcs_api
|
.create_check_statuses(&job.repo, complete.checks)
|
||||||
.create_check_statuses(&job.repo, complete.checks),
|
.await;
|
||||||
);
|
response.extend(schedule_builds(complete.builds, &auto_schedule_build_archs));
|
||||||
response.extend(schedule_builds(complete.builds, auto_schedule_build_archs));
|
|
||||||
|
|
||||||
overall_status.set_with_description("^.^!", State::Success)?;
|
overall_status
|
||||||
|
.set_with_description("^.^!", State::Success)
|
||||||
|
.await?;
|
||||||
} else {
|
} else {
|
||||||
overall_status.set_with_description("Complete, with errors", State::Failure)?;
|
overall_status
|
||||||
|
.set_with_description("Complete, with errors", State::Failure)
|
||||||
|
.await?;
|
||||||
}
|
}
|
||||||
|
|
||||||
self.events.notify(Event::TaskEvaluationCheckComplete);
|
self.events.notify(Event::TaskEvaluationCheckComplete).await;
|
||||||
|
|
||||||
info!("Evaluations done!");
|
info!("Evaluations done!");
|
||||||
Ok(self.actions().done(job, response))
|
Ok(Actions::done(job, response))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn schedule_builds(
|
fn schedule_builds(
|
||||||
builds: Vec<buildjob::BuildJob>,
|
builds: Vec<buildjob::BuildJob>,
|
||||||
auto_schedule_build_archs: Vec<systems::System>,
|
auto_schedule_build_archs: &[systems::System],
|
||||||
) -> Vec<worker::Action> {
|
) -> Vec<worker::Action> {
|
||||||
let mut response = vec![];
|
let mut response = vec![];
|
||||||
info!(
|
info!(
|
||||||
|
@ -460,20 +496,22 @@ fn schedule_builds(
|
||||||
builds, auto_schedule_build_archs
|
builds, auto_schedule_build_archs
|
||||||
);
|
);
|
||||||
for buildjob in builds {
|
for buildjob in builds {
|
||||||
for arch in auto_schedule_build_archs.iter() {
|
for arch in auto_schedule_build_archs {
|
||||||
let (exchange, routingkey) = arch.as_build_destination();
|
let (exchange, routingkey) = arch.as_build_destination();
|
||||||
response.push(worker::publish_serde_action(
|
response.push(worker::publish_serde_action(
|
||||||
exchange, routingkey, &buildjob,
|
&exchange,
|
||||||
|
&routingkey,
|
||||||
|
&buildjob,
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
response.push(worker::publish_serde_action(
|
response.push(worker::publish_serde_action(
|
||||||
Some("build-results".to_string()),
|
&Some("build-results".to_string()),
|
||||||
None,
|
&None,
|
||||||
&buildjob::QueuedBuildJobs {
|
&buildjob::QueuedBuildJobs {
|
||||||
job: buildjob,
|
job: buildjob,
|
||||||
architectures: auto_schedule_build_archs
|
architectures: auto_schedule_build_archs
|
||||||
.iter()
|
.iter()
|
||||||
.map(|arch| arch.to_string())
|
.map(std::string::ToString::to_string)
|
||||||
.collect(),
|
.collect(),
|
||||||
},
|
},
|
||||||
));
|
));
|
||||||
|
|
|
@ -3,6 +3,7 @@ use crate::ghevent;
|
||||||
use crate::message::{evaluationjob, Change, Repo};
|
use crate::message::{evaluationjob, Change, Repo};
|
||||||
use crate::worker;
|
use crate::worker;
|
||||||
|
|
||||||
|
use async_trait::async_trait;
|
||||||
use tracing::{debug_span, info};
|
use tracing::{debug_span, info};
|
||||||
|
|
||||||
pub struct EvaluationFilterWorker {
|
pub struct EvaluationFilterWorker {
|
||||||
|
@ -10,15 +11,22 @@ pub struct EvaluationFilterWorker {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl EvaluationFilterWorker {
|
impl EvaluationFilterWorker {
|
||||||
|
#[must_use]
|
||||||
pub fn new(acl: acl::Acl) -> EvaluationFilterWorker {
|
pub fn new(acl: acl::Acl) -> EvaluationFilterWorker {
|
||||||
EvaluationFilterWorker { acl }
|
EvaluationFilterWorker { acl }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
impl worker::SimpleWorker for EvaluationFilterWorker {
|
impl worker::SimpleWorker for EvaluationFilterWorker {
|
||||||
type J = ghevent::PullRequestEvent;
|
type J = ghevent::PullRequestEvent;
|
||||||
|
|
||||||
fn msg_to_job(&mut self, _: &str, _: &Option<String>, body: &[u8]) -> Result<Self::J, String> {
|
async fn msg_to_job(
|
||||||
|
&mut self,
|
||||||
|
_: &str,
|
||||||
|
_: &Option<String>,
|
||||||
|
body: &[u8],
|
||||||
|
) -> Result<Self::J, String> {
|
||||||
match serde_json::from_slice(body) {
|
match serde_json::from_slice(body) {
|
||||||
Ok(e) => Ok(e),
|
Ok(e) => Ok(e),
|
||||||
Err(e) => Err(format!(
|
Err(e) => Err(format!(
|
||||||
|
@ -29,7 +37,7 @@ impl worker::SimpleWorker for EvaluationFilterWorker {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn consumer(
|
async fn consumer(
|
||||||
&mut self,
|
&mut self,
|
||||||
_chan: &mut lapin::Channel,
|
_chan: &mut lapin::Channel,
|
||||||
job: &ghevent::PullRequestEvent,
|
job: &ghevent::PullRequestEvent,
|
||||||
|
@ -51,9 +59,9 @@ impl worker::SimpleWorker for EvaluationFilterWorker {
|
||||||
}
|
}
|
||||||
|
|
||||||
let interesting: bool = match job.action {
|
let interesting: bool = match job.action {
|
||||||
ghevent::PullRequestAction::Opened => true,
|
ghevent::PullRequestAction::Opened
|
||||||
ghevent::PullRequestAction::Synchronize => true,
|
| ghevent::PullRequestAction::Synchronize
|
||||||
ghevent::PullRequestAction::Reopened => true,
|
| ghevent::PullRequestAction::Reopened => true,
|
||||||
ghevent::PullRequestAction::Edited => {
|
ghevent::PullRequestAction::Edited => {
|
||||||
if let Some(ref changes) = job.changes {
|
if let Some(ref changes) = job.changes {
|
||||||
changes.base.is_some()
|
changes.base.is_some()
|
||||||
|
@ -61,7 +69,7 @@ impl worker::SimpleWorker for EvaluationFilterWorker {
|
||||||
false
|
false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
_ => false,
|
ghevent::PullRequestAction::Unknown => false,
|
||||||
};
|
};
|
||||||
|
|
||||||
if !interesting {
|
if !interesting {
|
||||||
|
@ -96,7 +104,7 @@ impl worker::SimpleWorker for EvaluationFilterWorker {
|
||||||
};
|
};
|
||||||
|
|
||||||
vec![
|
vec![
|
||||||
worker::publish_serde_action(None, Some("mass-rebuild-check-jobs".to_owned()), &msg),
|
worker::publish_serde_action(&None, &Some("mass-rebuild-check-jobs".to_owned()), &msg),
|
||||||
worker::Action::Ack,
|
worker::Action::Ack,
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,6 +3,7 @@ use crate::message::buildresult::BuildResult;
|
||||||
use crate::worker;
|
use crate::worker;
|
||||||
use crate::writetoline::LineWriter;
|
use crate::writetoline::LineWriter;
|
||||||
|
|
||||||
|
use async_trait::async_trait;
|
||||||
use std::fs::{self, File, OpenOptions};
|
use std::fs::{self, File, OpenOptions};
|
||||||
use std::io::Write;
|
use std::io::Write;
|
||||||
use std::path::{Component, Path, PathBuf};
|
use std::path::{Component, Path, PathBuf};
|
||||||
|
@ -55,6 +56,7 @@ fn validate_path_segment(segment: &Path) -> Result<(), String> {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl LogMessageCollector {
|
impl LogMessageCollector {
|
||||||
|
#[must_use]
|
||||||
pub fn new(log_root: PathBuf, max_open: usize) -> LogMessageCollector {
|
pub fn new(log_root: PathBuf, max_open: usize) -> LogMessageCollector {
|
||||||
LogMessageCollector {
|
LogMessageCollector {
|
||||||
handles: LruCache::new(max_open),
|
handles: LruCache::new(max_open),
|
||||||
|
@ -64,33 +66,33 @@ impl LogMessageCollector {
|
||||||
|
|
||||||
pub fn write_metadata(&mut self, from: &LogFrom, data: &BuildLogStart) -> Result<(), String> {
|
pub fn write_metadata(&mut self, from: &LogFrom, data: &BuildLogStart) -> Result<(), String> {
|
||||||
let metapath = self.path_for_metadata(from)?;
|
let metapath = self.path_for_metadata(from)?;
|
||||||
let mut fp = self.open_file(&metapath)?;
|
let mut fp = Self::open_file(&metapath)?;
|
||||||
|
|
||||||
match serde_json::to_string(data) {
|
match serde_json::to_string(data) {
|
||||||
Ok(data) => {
|
Ok(data) => {
|
||||||
if let Err(e) = fp.write(data.as_bytes()) {
|
if let Err(e) = fp.write(data.as_bytes()) {
|
||||||
Err(format!("Failed to write metadata: {:?}", e))
|
Err(format!("Failed to write metadata: {e:?}"))
|
||||||
} else {
|
} else {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Err(e) => Err(format!("Failed to stringify metadata: {:?}", e)),
|
Err(e) => Err(format!("Failed to stringify metadata: {e:?}")),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn write_result(&mut self, from: &LogFrom, data: &BuildResult) -> Result<(), String> {
|
pub fn write_result(&mut self, from: &LogFrom, data: &BuildResult) -> Result<(), String> {
|
||||||
let path = self.path_for_result(from)?;
|
let path = self.path_for_result(from)?;
|
||||||
let mut fp = self.open_file(&path)?;
|
let mut fp = Self::open_file(&path)?;
|
||||||
|
|
||||||
match serde_json::to_string(data) {
|
match serde_json::to_string(data) {
|
||||||
Ok(data) => {
|
Ok(data) => {
|
||||||
if let Err(e) = fp.write(data.as_bytes()) {
|
if let Err(e) = fp.write(data.as_bytes()) {
|
||||||
Err(format!("Failed to write result: {:?}", e))
|
Err(format!("Failed to write result: {e:?}"))
|
||||||
} else {
|
} else {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Err(e) => Err(format!("Failed to stringify result: {:?}", e)),
|
Err(e) => Err(format!("Failed to stringify result: {e:?}")),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -102,7 +104,7 @@ impl LogMessageCollector {
|
||||||
.expect("handles just contained the key"))
|
.expect("handles just contained the key"))
|
||||||
} else {
|
} else {
|
||||||
let logpath = self.path_for_log(from)?;
|
let logpath = self.path_for_log(from)?;
|
||||||
let fp = self.open_file(&logpath)?;
|
let fp = Self::open_file(&logpath)?;
|
||||||
let writer = LineWriter::new(fp);
|
let writer = LineWriter::new(fp);
|
||||||
self.handles.insert(from.clone(), writer);
|
self.handles.insert(from.clone(), writer);
|
||||||
if let Some(handle) = self.handles.get_mut(from) {
|
if let Some(handle) = self.handles.get_mut(from) {
|
||||||
|
@ -142,13 +144,12 @@ impl LogMessageCollector {
|
||||||
Ok(location)
|
Ok(location)
|
||||||
} else {
|
} else {
|
||||||
Err(format!(
|
Err(format!(
|
||||||
"Calculating the log location for {:?} resulted in an invalid path {:?}",
|
"Calculating the log location for {from:?} resulted in an invalid path {location:?}"
|
||||||
from, location
|
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn open_file(&self, path: &Path) -> Result<File, String> {
|
fn open_file(path: &Path) -> Result<File, String> {
|
||||||
let dir = path.parent().unwrap();
|
let dir = path.parent().unwrap();
|
||||||
fs::create_dir_all(dir).unwrap();
|
fs::create_dir_all(dir).unwrap();
|
||||||
|
|
||||||
|
@ -168,10 +169,11 @@ impl LogMessageCollector {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
impl worker::SimpleWorker for LogMessageCollector {
|
impl worker::SimpleWorker for LogMessageCollector {
|
||||||
type J = LogMessage;
|
type J = LogMessage;
|
||||||
|
|
||||||
fn msg_to_job(
|
async fn msg_to_job(
|
||||||
&mut self,
|
&mut self,
|
||||||
routing_key: &str,
|
routing_key: &str,
|
||||||
_: &Option<String>,
|
_: &Option<String>,
|
||||||
|
@ -195,7 +197,7 @@ impl worker::SimpleWorker for LogMessageCollector {
|
||||||
attempt_id = msg.legacy().attempt_id;
|
attempt_id = msg.legacy().attempt_id;
|
||||||
message = MsgType::Finish(Box::new(msg));
|
message = MsgType::Finish(Box::new(msg));
|
||||||
} else {
|
} else {
|
||||||
return Err(format!("failed to decode job: {:?}", decode_msg));
|
return Err(format!("failed to decode job: {decode_msg:?}"));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -209,7 +211,7 @@ impl worker::SimpleWorker for LogMessageCollector {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
fn consumer(&mut self, _chan: &mut lapin::Channel, job: &LogMessage) -> worker::Actions {
|
async fn consumer(&mut self, _chan: &mut lapin::Channel, job: &LogMessage) -> worker::Actions {
|
||||||
match job.message {
|
match job.message {
|
||||||
MsgType::Start(ref start) => {
|
MsgType::Start(ref start) => {
|
||||||
self.write_metadata(&job.from, start)
|
self.write_metadata(&job.from, start)
|
||||||
|
@ -223,7 +225,7 @@ impl worker::SimpleWorker for LogMessageCollector {
|
||||||
MsgType::Msg(ref message) => {
|
MsgType::Msg(ref message) => {
|
||||||
let handle = self.handle_for(&job.from).unwrap();
|
let handle = self.handle_for(&job.from).unwrap();
|
||||||
|
|
||||||
handle.write_to_line((message.line_number - 1) as usize, &message.output);
|
handle.write_to_line(message.line_number - 1, &message.output);
|
||||||
}
|
}
|
||||||
MsgType::Finish(ref finish) => {
|
MsgType::Finish(ref finish) => {
|
||||||
self.write_result(&job.from, finish)
|
self.write_result(&job.from, finish)
|
||||||
|
|
|
@ -4,6 +4,7 @@ use std::{
|
||||||
path::PathBuf,
|
path::PathBuf,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
use async_trait::async_trait;
|
||||||
use tracing::{debug_span, error, warn};
|
use tracing::{debug_span, error, warn};
|
||||||
use zstd::stream::write::Encoder;
|
use zstd::stream::write::Encoder;
|
||||||
|
|
||||||
|
@ -18,6 +19,7 @@ pub struct PastebinCollector {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl PastebinCollector {
|
impl PastebinCollector {
|
||||||
|
#[must_use]
|
||||||
pub fn new(pastebin_root: PathBuf, db_path: PathBuf) -> Self {
|
pub fn new(pastebin_root: PathBuf, db_path: PathBuf) -> Self {
|
||||||
Self {
|
Self {
|
||||||
pastebin_root,
|
pastebin_root,
|
||||||
|
@ -26,10 +28,11 @@ impl PastebinCollector {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
impl worker::SimpleWorker for PastebinCollector {
|
impl worker::SimpleWorker for PastebinCollector {
|
||||||
type J = Pastebin;
|
type J = Pastebin;
|
||||||
|
|
||||||
fn msg_to_job(
|
async fn msg_to_job(
|
||||||
&mut self,
|
&mut self,
|
||||||
_routing_key: &str,
|
_routing_key: &str,
|
||||||
_: &Option<String>,
|
_: &Option<String>,
|
||||||
|
@ -47,7 +50,7 @@ impl worker::SimpleWorker for PastebinCollector {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn consumer(&mut self, _chan: &mut lapin::Channel, job: &Self::J) -> worker::Actions {
|
async fn consumer(&mut self, _chan: &mut lapin::Channel, job: &Self::J) -> worker::Actions {
|
||||||
let span = debug_span!("pastebin", title = ?job.title);
|
let span = debug_span!("pastebin", title = ?job.title);
|
||||||
let _enter = span.enter();
|
let _enter = span.enter();
|
||||||
|
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
use crate::stats;
|
use crate::stats;
|
||||||
use crate::worker;
|
use crate::worker;
|
||||||
|
|
||||||
|
use async_trait::async_trait;
|
||||||
use tracing::error;
|
use tracing::error;
|
||||||
|
|
||||||
pub struct StatCollectorWorker<E> {
|
pub struct StatCollectorWorker<E> {
|
||||||
|
@ -14,48 +15,57 @@ impl<E: stats::SysEvents + 'static> StatCollectorWorker<E> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
impl<E: stats::SysEvents + 'static> worker::SimpleWorker for StatCollectorWorker<E> {
|
impl<E: stats::SysEvents + 'static> worker::SimpleWorker for StatCollectorWorker<E> {
|
||||||
type J = stats::EventMessage;
|
type J = stats::EventMessage;
|
||||||
|
|
||||||
fn msg_to_job(&mut self, _: &str, _: &Option<String>, body: &[u8]) -> Result<Self::J, String> {
|
async fn msg_to_job(
|
||||||
match serde_json::from_slice(body) {
|
&mut self,
|
||||||
Ok(e) => Ok(e),
|
_: &str,
|
||||||
Err(_) => {
|
_: &Option<String>,
|
||||||
let mut modified_body: Vec<u8> = vec![b"\""[0]];
|
body: &[u8],
|
||||||
modified_body.append(&mut body.to_vec());
|
) -> Result<Self::J, String> {
|
||||||
modified_body.push(b"\""[0]);
|
if let Ok(e) = serde_json::from_slice(body) {
|
||||||
|
Ok(e)
|
||||||
|
} else {
|
||||||
|
let mut modified_body: Vec<u8> = vec![b"\""[0]];
|
||||||
|
modified_body.append(&mut body.to_vec());
|
||||||
|
modified_body.push(b"\""[0]);
|
||||||
|
|
||||||
match serde_json::from_slice(&modified_body) {
|
match serde_json::from_slice(&modified_body) {
|
||||||
Ok(e) => {
|
Ok(e) => {
|
||||||
self.events.notify(stats::Event::StatCollectorLegacyEvent(
|
self.events
|
||||||
|
.notify(stats::Event::StatCollectorLegacyEvent(
|
||||||
stats::event_metric_name(&e),
|
stats::event_metric_name(&e),
|
||||||
));
|
))
|
||||||
Ok(stats::EventMessage {
|
.await;
|
||||||
sender: "".to_owned(),
|
Ok(stats::EventMessage {
|
||||||
events: vec![e],
|
sender: String::new(),
|
||||||
})
|
events: vec![e],
|
||||||
}
|
})
|
||||||
Err(e) => {
|
}
|
||||||
self.events.notify(stats::Event::StatCollectorBogusEvent);
|
Err(e) => {
|
||||||
error!(
|
self.events
|
||||||
"Failed to decode message: {:?}, Err: {:?}",
|
.notify(stats::Event::StatCollectorBogusEvent)
|
||||||
String::from_utf8(body.to_vec()),
|
.await;
|
||||||
e
|
error!(
|
||||||
);
|
"Failed to decode message: {:?}, Err: {:?}",
|
||||||
Err("Failed to decode message".to_owned())
|
String::from_utf8(body.to_vec()),
|
||||||
}
|
e
|
||||||
|
);
|
||||||
|
Err("Failed to decode message".to_owned())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn consumer(
|
async fn consumer(
|
||||||
&mut self,
|
&mut self,
|
||||||
_chan: &mut lapin::Channel,
|
_chan: &mut lapin::Channel,
|
||||||
job: &stats::EventMessage,
|
job: &stats::EventMessage,
|
||||||
) -> worker::Actions {
|
) -> worker::Actions {
|
||||||
let sender = job.sender.clone();
|
let sender = job.sender.clone();
|
||||||
for event in job.events.iter() {
|
for event in &job.events {
|
||||||
self.collector.record(sender.clone(), event.clone());
|
self.collector.record(sender.clone(), event.clone());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1,5 +1,6 @@
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
|
|
||||||
|
use async_trait::async_trait;
|
||||||
use tracing::{debug_span, error};
|
use tracing::{debug_span, error};
|
||||||
|
|
||||||
use crate::worker;
|
use crate::worker;
|
||||||
|
@ -16,16 +17,18 @@ pub struct StatusCheckCollector {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl StatusCheckCollector {
|
impl StatusCheckCollector {
|
||||||
|
#[must_use]
|
||||||
pub fn new(db_path: PathBuf) -> Self {
|
pub fn new(db_path: PathBuf) -> Self {
|
||||||
Self { db_path }
|
Self { db_path }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// RPC API worker
|
// RPC API worker
|
||||||
|
#[async_trait]
|
||||||
impl worker::SimpleWorker for StatusCheckCollector {
|
impl worker::SimpleWorker for StatusCheckCollector {
|
||||||
type J = StatusCheckRPCMessage;
|
type J = StatusCheckRPCMessage;
|
||||||
|
|
||||||
fn msg_to_job(
|
async fn msg_to_job(
|
||||||
&mut self,
|
&mut self,
|
||||||
_method: &str,
|
_method: &str,
|
||||||
_headers: &Option<String>,
|
_headers: &Option<String>,
|
||||||
|
@ -43,7 +46,7 @@ impl worker::SimpleWorker for StatusCheckCollector {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn consumer(&mut self, _chan: &mut lapin::Channel, _job: &Self::J) -> worker::Actions {
|
async fn consumer(&mut self, _chan: &mut lapin::Channel, _job: &Self::J) -> worker::Actions {
|
||||||
let span = debug_span!("command");
|
let span = debug_span!("command");
|
||||||
let _enter = span.enter();
|
let _enter = span.enter();
|
||||||
|
|
||||||
|
|
|
@ -9,12 +9,13 @@ pub struct TestScratch {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl TestScratch {
|
impl TestScratch {
|
||||||
|
#[must_use]
|
||||||
pub fn new_dir(ident: &str) -> TestScratch {
|
pub fn new_dir(ident: &str) -> TestScratch {
|
||||||
let scratch = TestScratch {
|
let scratch = TestScratch {
|
||||||
root: Path::new(env!("CARGO_MANIFEST_DIR"))
|
root: Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||||
.join("test-scratch")
|
.join("test-scratch")
|
||||||
.join("dirs")
|
.join("dirs")
|
||||||
.join(format!("dir-{}", ident)),
|
.join(format!("dir-{ident}")),
|
||||||
};
|
};
|
||||||
|
|
||||||
TestScratch::create_dir(&scratch);
|
TestScratch::create_dir(&scratch);
|
||||||
|
@ -22,12 +23,13 @@ impl TestScratch {
|
||||||
scratch
|
scratch
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub fn new_file(ident: &str) -> TestScratch {
|
pub fn new_file(ident: &str) -> TestScratch {
|
||||||
let scratch = TestScratch {
|
let scratch = TestScratch {
|
||||||
root: Path::new(env!("CARGO_MANIFEST_DIR"))
|
root: Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||||
.join("test-scratch")
|
.join("test-scratch")
|
||||||
.join("files")
|
.join("files")
|
||||||
.join(format!("file-{}", ident)),
|
.join(format!("file-{ident}")),
|
||||||
};
|
};
|
||||||
|
|
||||||
TestScratch::create_dir(&scratch);
|
TestScratch::create_dir(&scratch);
|
||||||
|
@ -40,10 +42,12 @@ impl TestScratch {
|
||||||
fs::create_dir_all(target).unwrap();
|
fs::create_dir_all(target).unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub fn path(&self) -> PathBuf {
|
pub fn path(&self) -> PathBuf {
|
||||||
self.root.clone()
|
self.root.clone()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub fn string(&self) -> String {
|
pub fn string(&self) -> String {
|
||||||
self.path().to_str().unwrap().to_owned()
|
self.path().to_str().unwrap().to_owned()
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,6 +1,5 @@
|
||||||
use std::rc::Rc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use futures::executor::block_on;
|
|
||||||
use tracing::warn;
|
use tracing::warn;
|
||||||
|
|
||||||
use crate::vcs::generic::State;
|
use crate::vcs::generic::State;
|
||||||
|
@ -8,7 +7,7 @@ use crate::vcs::generic::State;
|
||||||
use super::generic::VersionControlSystemAPI;
|
use super::generic::VersionControlSystemAPI;
|
||||||
|
|
||||||
pub struct CommitStatus {
|
pub struct CommitStatus {
|
||||||
api: Rc<dyn VersionControlSystemAPI>,
|
api: Arc<dyn VersionControlSystemAPI>,
|
||||||
repo: crate::message::Repo,
|
repo: crate::message::Repo,
|
||||||
sha: String,
|
sha: String,
|
||||||
context: String,
|
context: String,
|
||||||
|
@ -18,7 +17,7 @@ pub struct CommitStatus {
|
||||||
|
|
||||||
impl CommitStatus {
|
impl CommitStatus {
|
||||||
pub fn new(
|
pub fn new(
|
||||||
api: Rc<dyn VersionControlSystemAPI>,
|
api: Arc<dyn VersionControlSystemAPI>,
|
||||||
repo: crate::message::Repo,
|
repo: crate::message::Repo,
|
||||||
sha: String,
|
sha: String,
|
||||||
context: String,
|
context: String,
|
||||||
|
@ -31,7 +30,7 @@ impl CommitStatus {
|
||||||
sha,
|
sha,
|
||||||
context,
|
context,
|
||||||
description,
|
description,
|
||||||
url: "".to_owned(),
|
url: String::new(),
|
||||||
};
|
};
|
||||||
|
|
||||||
stat.set_url(url);
|
stat.set_url(url);
|
||||||
|
@ -40,23 +39,23 @@ impl CommitStatus {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn set_url(&mut self, url: Option<String>) {
|
pub fn set_url(&mut self, url: Option<String>) {
|
||||||
self.url = url.unwrap_or_else(|| String::from(""))
|
self.url = url.unwrap_or_default();
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn set_with_description(
|
pub async fn set_with_description(
|
||||||
&mut self,
|
&mut self,
|
||||||
description: &str,
|
description: &str,
|
||||||
state: State,
|
state: State,
|
||||||
) -> Result<(), CommitStatusError> {
|
) -> Result<(), CommitStatusError> {
|
||||||
self.set_description(description.to_owned());
|
self.set_description(description.to_owned());
|
||||||
self.set(state)
|
self.set(state).await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn set_description(&mut self, description: String) {
|
pub fn set_description(&mut self, description: String) {
|
||||||
self.description = description;
|
self.description = description;
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn set(&self, state: State) -> Result<(), CommitStatusError> {
|
pub async fn set(&self, state: State) -> Result<(), CommitStatusError> {
|
||||||
let desc = if self.description.len() >= 140 {
|
let desc = if self.description.len() >= 140 {
|
||||||
warn!(
|
warn!(
|
||||||
"description is over 140 char; truncating: {:?}",
|
"description is over 140 char; truncating: {:?}",
|
||||||
|
@ -67,14 +66,16 @@ impl CommitStatus {
|
||||||
self.description.clone()
|
self.description.clone()
|
||||||
};
|
};
|
||||||
|
|
||||||
block_on(self.api.create_commit_statuses(
|
self.api
|
||||||
&self.repo,
|
.create_commit_statuses(
|
||||||
self.sha.clone(),
|
&self.repo,
|
||||||
state,
|
self.sha.clone(),
|
||||||
self.context.clone(),
|
state,
|
||||||
desc,
|
self.context.clone(),
|
||||||
self.url.clone(),
|
desc,
|
||||||
))
|
self.url.clone(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -34,12 +34,13 @@ pub struct ChangeReviewers {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Issue {
|
impl Issue {
|
||||||
|
#[must_use]
|
||||||
pub fn is_wip(&self) -> bool {
|
pub fn is_wip(&self) -> bool {
|
||||||
false
|
false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub trait VersionControlSystemAPI {
|
pub trait VersionControlSystemAPI: Sync + Send {
|
||||||
fn get_repository(&self, repo: &crate::message::Repo) -> Repository;
|
fn get_repository(&self, repo: &crate::message::Repo) -> Repository;
|
||||||
fn get_changes(&self, repo: &crate::message::Repo) -> BoxFuture<Vec<Change>>;
|
fn get_changes(&self, repo: &crate::message::Repo) -> BoxFuture<Vec<Change>>;
|
||||||
fn get_change(&self, repo: &crate::message::Repo, number: u64) -> BoxFuture<Option<Change>>;
|
fn get_change(&self, repo: &crate::message::Repo, number: u64) -> BoxFuture<Option<Change>>;
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
use crate::vcs::generic::CheckRunState;
|
use crate::vcs::generic::CheckRunState;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
/// Port from https://gerrit.googlesource.com/gerrit/+/master/polygerrit-ui/app/api/checks.ts
|
/// Port from <https://gerrit.googlesource.com/gerrit/+/master/polygerrit-ui/app/api/checks.ts>
|
||||||
|
|
||||||
#[derive(Debug, Serialize, Deserialize, PartialEq)]
|
#[derive(Debug, Serialize, Deserialize, PartialEq)]
|
||||||
#[serde(rename_all = "UPPERCASE")]
|
#[serde(rename_all = "UPPERCASE")]
|
||||||
|
|
|
@ -7,22 +7,23 @@ use super::data_structures::{Account, Change};
|
||||||
|
|
||||||
pub struct GerritHTTPApi;
|
pub struct GerritHTTPApi;
|
||||||
|
|
||||||
|
#[allow(clippy::unused_async)] // FIXME
|
||||||
impl GerritHTTPApi {
|
impl GerritHTTPApi {
|
||||||
// async fn get_project(&self, project_name: &str) -> Project {}
|
// async fn get_project(&self, project_name: &str) -> Project {}
|
||||||
/// Fetches all changes according to the query and the given limit.
|
/// Fetches all changes according to the query and the given limit.
|
||||||
/// This will default to 60 changes by default.
|
/// This will default to 60 changes by default.
|
||||||
pub(crate) async fn list_changes(&self, _query: &str, _limit: Option<u64>) -> Vec<Change> {
|
pub(crate) async fn list_changes(&self, _query: &str, _limit: Option<u64>) -> Vec<Change> {
|
||||||
Default::default()
|
vec![]
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Fetch the latest change ID for a given project and CL number.
|
/// Fetch the latest change ID for a given project and CL number.
|
||||||
pub(crate) async fn get_change_id(&self, _project_name: &str, _cl_number: u64) -> String {
|
pub(crate) async fn get_change_id(&self, _project_name: &str, _cl_number: u64) -> String {
|
||||||
"".to_owned()
|
String::new()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Fetch a given change according to the change ID (not the CL number).
|
/// Fetch a given change according to the change ID (not the CL number).
|
||||||
pub(crate) async fn get_change(&self, _change_id: &str) -> Option<Change> {
|
pub(crate) async fn get_change(&self, _change_id: &str) -> Option<Change> {
|
||||||
Default::default()
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Set additional and remove certain hashtags for a given change ID (not the CL number).
|
/// Set additional and remove certain hashtags for a given change ID (not the CL number).
|
||||||
|
@ -32,11 +33,11 @@ impl GerritHTTPApi {
|
||||||
_add: &[String],
|
_add: &[String],
|
||||||
_remove: &[String],
|
_remove: &[String],
|
||||||
) -> Vec<String> {
|
) -> Vec<String> {
|
||||||
Default::default()
|
vec![]
|
||||||
}
|
}
|
||||||
/// List all reviewers on a given change ID (not the CL number).
|
/// List all reviewers on a given change ID (not the CL number).
|
||||||
pub(crate) async fn list_reviewers(&self, _change_id: &str) -> Vec<Account> {
|
pub(crate) async fn list_reviewers(&self, _change_id: &str) -> Vec<Account> {
|
||||||
Default::default()
|
vec![]
|
||||||
}
|
}
|
||||||
/// Set reviewers and a message on a given change ID (not the CL number).
|
/// Set reviewers and a message on a given change ID (not the CL number).
|
||||||
pub(crate) async fn set_reviewers(
|
pub(crate) async fn set_reviewers(
|
||||||
|
|
|
@ -50,12 +50,12 @@ impl VersionControlSystemAPI for GerritHTTPApi {
|
||||||
&self,
|
&self,
|
||||||
repo: &crate::message::Repo,
|
repo: &crate::message::Repo,
|
||||||
) -> futures_util::future::BoxFuture<Vec<crate::message::Change>> {
|
) -> futures_util::future::BoxFuture<Vec<crate::message::Change>> {
|
||||||
let repo_name = repo.name.to_owned();
|
let repo_name = repo.name.clone();
|
||||||
async move {
|
async move {
|
||||||
self.list_changes(&format!("project:{}", &repo_name), None)
|
self.list_changes(&format!("project:{}", &repo_name), None)
|
||||||
.await
|
.await
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.map(|c| c.into())
|
.map(std::convert::Into::into)
|
||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
.boxed()
|
.boxed()
|
||||||
|
@ -66,12 +66,12 @@ impl VersionControlSystemAPI for GerritHTTPApi {
|
||||||
repo: &crate::message::Repo,
|
repo: &crate::message::Repo,
|
||||||
number: u64,
|
number: u64,
|
||||||
) -> futures_util::future::BoxFuture<Option<crate::message::Change>> {
|
) -> futures_util::future::BoxFuture<Option<crate::message::Change>> {
|
||||||
let repo_name = repo.name.to_owned();
|
let repo_name = repo.name.clone();
|
||||||
async move {
|
async move {
|
||||||
let change_id = self.get_change_id(&repo_name, number).await;
|
let change_id = self.get_change_id(&repo_name, number).await;
|
||||||
GerritHTTPApi::get_change(self, &change_id)
|
GerritHTTPApi::get_change(self, &change_id)
|
||||||
.await
|
.await
|
||||||
.map(|c| c.into())
|
.map(std::convert::Into::into)
|
||||||
}
|
}
|
||||||
.boxed()
|
.boxed()
|
||||||
}
|
}
|
||||||
|
@ -85,7 +85,7 @@ impl VersionControlSystemAPI for GerritHTTPApi {
|
||||||
) -> futures_util::future::BoxFuture<()> {
|
) -> futures_util::future::BoxFuture<()> {
|
||||||
let add = add.to_owned();
|
let add = add.to_owned();
|
||||||
let remove = remove.to_owned();
|
let remove = remove.to_owned();
|
||||||
let repo_name = repo.name.to_owned();
|
let repo_name = repo.name.clone();
|
||||||
|
|
||||||
async move {
|
async move {
|
||||||
let change_id = self.get_change_id(&repo_name, number).await;
|
let change_id = self.get_change_id(&repo_name, number).await;
|
||||||
|
@ -99,7 +99,7 @@ impl VersionControlSystemAPI for GerritHTTPApi {
|
||||||
repo: &crate::message::Repo,
|
repo: &crate::message::Repo,
|
||||||
number: u64,
|
number: u64,
|
||||||
) -> futures_util::future::BoxFuture<crate::vcs::generic::ChangeReviewers> {
|
) -> futures_util::future::BoxFuture<crate::vcs::generic::ChangeReviewers> {
|
||||||
let repo_name = repo.name.to_owned();
|
let repo_name = repo.name.clone();
|
||||||
async move {
|
async move {
|
||||||
let change_id = self.get_change_id(&repo_name, number).await;
|
let change_id = self.get_change_id(&repo_name, number).await;
|
||||||
self.list_reviewers(&change_id).await.into()
|
self.list_reviewers(&change_id).await.into()
|
||||||
|
@ -115,7 +115,7 @@ impl VersionControlSystemAPI for GerritHTTPApi {
|
||||||
// FIXME: support group reviews
|
// FIXME: support group reviews
|
||||||
_team_reviewers: Vec<String>,
|
_team_reviewers: Vec<String>,
|
||||||
) -> futures_util::future::BoxFuture<()> {
|
) -> futures_util::future::BoxFuture<()> {
|
||||||
let repo_name = repo.name.to_owned();
|
let repo_name = repo.name.clone();
|
||||||
async move {
|
async move {
|
||||||
let change_id = self.get_change_id(&repo_name, number).await;
|
let change_id = self.get_change_id(&repo_name, number).await;
|
||||||
self.set_reviewers(
|
self.set_reviewers(
|
||||||
|
@ -130,7 +130,7 @@ impl VersionControlSystemAPI for GerritHTTPApi {
|
||||||
})
|
})
|
||||||
.collect(),
|
.collect(),
|
||||||
)
|
)
|
||||||
.await
|
.await;
|
||||||
}
|
}
|
||||||
.boxed()
|
.boxed()
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
use std::{error::Error, path::PathBuf};
|
use std::{error::Error, path::PathBuf};
|
||||||
|
|
||||||
use async_stream::stream;
|
use async_stream::stream;
|
||||||
use futures::{executor::block_on, StreamExt};
|
use futures::StreamExt;
|
||||||
use futures_util::Stream;
|
use futures_util::Stream;
|
||||||
use openssh::{Session, SessionBuilder, Stdio};
|
use openssh::{Session, SessionBuilder, Stdio};
|
||||||
use tokio::io::AsyncBufReadExt;
|
use tokio::io::AsyncBufReadExt;
|
||||||
|
@ -13,10 +13,13 @@ pub struct GerritSSHApi {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl GerritSSHApi {
|
impl GerritSSHApi {
|
||||||
pub fn new(private_key_file: PathBuf, uri: &str) -> Self {
|
pub async fn new(private_key_file: PathBuf, uri: &str) -> Self {
|
||||||
let mut builder = SessionBuilder::default();
|
let mut builder = SessionBuilder::default();
|
||||||
let (builder, destination) = builder.keyfile(&private_key_file).resolve(uri);
|
let (builder, destination) = builder.keyfile(&private_key_file).resolve(uri);
|
||||||
let tempdir = block_on(builder.launch_master(destination)).unwrap_or_else(|_| panic!("Failed to launch SSH master to destination '{}'", uri));
|
let tempdir = builder
|
||||||
|
.launch_master(destination)
|
||||||
|
.await
|
||||||
|
.unwrap_or_else(|_| panic!("Failed to launch SSH master to destination '{}'", uri));
|
||||||
Self {
|
Self {
|
||||||
session: Session::new_process_mux(tempdir),
|
session: Session::new_process_mux(tempdir),
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,3 +1,4 @@
|
||||||
|
use async_trait::async_trait;
|
||||||
use std::marker::Send;
|
use std::marker::Send;
|
||||||
|
|
||||||
use futures::Stream;
|
use futures::Stream;
|
||||||
|
@ -34,8 +35,8 @@ where
|
||||||
T: Serialize + ?Sized,
|
T: Serialize + ?Sized,
|
||||||
{
|
{
|
||||||
QueueMsg {
|
QueueMsg {
|
||||||
exchange: exchange.map(|s| s.to_owned()),
|
exchange: exchange.map(std::borrow::ToOwned::to_owned),
|
||||||
routing_key: routing_key.map(|s| s.to_owned()),
|
routing_key: routing_key.map(std::borrow::ToOwned::to_owned),
|
||||||
mandatory: false,
|
mandatory: false,
|
||||||
immediate: false,
|
immediate: false,
|
||||||
content_type: Some("application/json".to_owned()),
|
content_type: Some("application/json".to_owned()),
|
||||||
|
@ -44,8 +45,8 @@ where
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn publish_serde_action<T>(
|
pub fn publish_serde_action<T>(
|
||||||
exchange: Option<String>,
|
exchange: &Option<String>,
|
||||||
routing_key: Option<String>,
|
routing_key: &Option<String>,
|
||||||
msg: &T,
|
msg: &T,
|
||||||
) -> Action
|
) -> Action
|
||||||
where
|
where
|
||||||
|
@ -58,11 +59,13 @@ where
|
||||||
)))
|
)))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
pub trait SimpleWorker: Send {
|
pub trait SimpleWorker: Send {
|
||||||
type J: Send;
|
type J: Send;
|
||||||
fn consumer(&mut self, chan: &mut lapin::Channel, job: &Self::J) -> Actions;
|
|
||||||
|
|
||||||
fn msg_to_job(
|
async fn consumer(&mut self, chan: &mut lapin::Channel, job: &Self::J) -> Actions;
|
||||||
|
|
||||||
|
async fn msg_to_job(
|
||||||
&mut self,
|
&mut self,
|
||||||
method: &str,
|
method: &str,
|
||||||
headers: &Option<String>,
|
headers: &Option<String>,
|
||||||
|
|
|
@ -8,6 +8,7 @@ pub struct LineWriter {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl LineWriter {
|
impl LineWriter {
|
||||||
|
#[must_use]
|
||||||
pub fn new(mut rw: File) -> LineWriter {
|
pub fn new(mut rw: File) -> LineWriter {
|
||||||
let buf = LineWriter::load_buffer(&mut rw);
|
let buf = LineWriter::load_buffer(&mut rw);
|
||||||
let len = buf.len();
|
let len = buf.len();
|
||||||
|
@ -27,7 +28,7 @@ impl LineWriter {
|
||||||
.lines()
|
.lines()
|
||||||
.map(|line| match line {
|
.map(|line| match line {
|
||||||
Ok(s) => s,
|
Ok(s) => s,
|
||||||
Err(e) => format!("UTF-8 Decode err: {:?}", e),
|
Err(e) => format!("UTF-8 Decode err: {e:?}"),
|
||||||
})
|
})
|
||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
|
@ -35,7 +36,7 @@ impl LineWriter {
|
||||||
pub fn write_to_line(&mut self, line: usize, data: &str) {
|
pub fn write_to_line(&mut self, line: usize, data: &str) {
|
||||||
let original_len = self.buffer.len();
|
let original_len = self.buffer.len();
|
||||||
while self.buffer.len() <= line {
|
while self.buffer.len() <= line {
|
||||||
self.buffer.push("".to_owned());
|
self.buffer.push(String::new());
|
||||||
}
|
}
|
||||||
|
|
||||||
self.buffer.remove(line);
|
self.buffer.remove(line);
|
||||||
|
@ -72,6 +73,7 @@ impl LineWriter {
|
||||||
self.last_line = line;
|
self.last_line = line;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub fn inner(self) -> File {
|
pub fn inner(self) -> File {
|
||||||
self.file
|
self.file
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue