danderson/ofborg
0
0
Fork 0
forked from the-distro/ofborg
Code Pull requests Activity

Getting closer on the eval

Browse source
This commit is contained in:
Graham Christensen 2017-11-19 16:12:43 -05:00
parent 154b3d4621
commit 2cfdc39d3e
No known key found for this signature in database
GPG key ID: ACA1C1D120C83D5C
8 changed files with 339 additions and 90 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
ofborg/src/bin/mass-rebuilder.rs
View file

@ -51,10 +51,15 @@ fn main() {
let nix = nix::new(cfg.nix.system.clone(), cfg.nix.remote.clone());
let mrw = tasks::massrebuilder::MassRebuildWorker::new(cloner, nix, cfg.github());
let mrw = tasks::massrebuilder::MassRebuildWorker::new(
cloner,
nix,
cfg.github(),
cfg.checkout.root.clone()
);
println!("{:?}", mrw.consumer(&message::massrebuildjob::MassRebuildJob{
pr: ofborg::message::Pr {
head_sha: String::from("e82a34e55cc52e0eace0d9b5d4452c7359038a19"),
head_sha: String::from("85589b80e81d5839cc91eb6be2cc3f7c041b760a"),
number: 30777,
target_branch: Some(String::from("master")),
},
@ -70,7 +75,7 @@ fn main() {
channel.basic_consume(
worker::new(tasks::massrebuilder::MassRebuildWorker::new(cloner, nix, cfg.github())),
worker::new(mrw),
"mass-rebuild-check-jobs",
format!("{}-mass-rebuild-checker", cfg.whoami()).as_ref(),
false,

52
ofborg/src/commitstatus.rs Normal file
View file

@ -0,0 +1,52 @@
extern crate amqp;
extern crate env_logger;
use hubcaps;
pub struct CommitStatus<'a> {
api: hubcaps::statuses::Statuses<'a>,
sha: String,
context: String,
description: String,
url: String,
}
impl <'a> CommitStatus<'a> {
pub fn new(api: hubcaps::statuses::Statuses<'a>, sha: String, context: String, description: String, url: Option<String>) -> CommitStatus<'a> {
let mut stat = CommitStatus {
api: api,
sha: sha,
context: context,
description: description,
url: "".to_owned(),
};
stat.set_url(url);
return stat
}
pub fn set_url(&mut self, url: Option<String>) {
self.url = url.unwrap_or(String::from(""))
}
pub fn set_with_description(&mut self, description: &str, state: hubcaps::statuses::State) {
self.set_description(description.to_owned());
self.set(state);
}
pub fn set_description(&mut self, description: String) {
self.description = description;
}
pub fn set(&self, state: hubcaps::statuses::State) {
self.api.create(
self.sha.as_ref(),
&hubcaps::statuses::StatusOptions::builder(state)
.context(self.context.clone())
.description(self.description.clone())
.target_url(self.url.clone())
.build()
).expect("Failed to mark final status on commit");
}
}

39
ofborg/src/evalchecker.rs Normal file
View file

@ -0,0 +1,39 @@
extern crate amqp;
extern crate env_logger;
use std::fs::File;
use std::path::Path;
use ofborg::nix;
pub struct EvalChecker {
name: String,
cmd: String,
args: Vec<String>,
nix: nix::Nix,
}
impl EvalChecker {
pub fn new(name: &str, cmd: &str, args: Vec<String>, nix: nix::Nix) -> EvalChecker {
EvalChecker{
name: name.to_owned(),
cmd: cmd.to_owned(),
args: args,
nix: nix,
}
}
pub fn name(&self) -> String {
format!("grahamcofborg-eval-{}", self.name)
}
pub fn execute(&self, path: &Path) -> Result<File, File> {
self.nix.safely(&self.cmd, path, self.args.clone())
}
pub fn cli_cmd(&self) -> String {
let mut cli = vec![self.cmd.clone()];
cli.append(&mut self.args.clone());
return cli.join(" ");
}
}

7
ofborg/src/lib.rs
View file

@ -23,9 +23,13 @@ pub mod worker;
pub mod config;
pub mod message;
pub mod tasks;
pub mod evalchecker;
pub mod nix;
pub mod ghevent;
pub mod commentparser;
pub mod commitstatus;
pub mod outpathdiff;
pub mod ofborg {
pub use config;
@ -35,8 +39,11 @@ pub mod ofborg {
pub use worker;
pub use message;
pub use tasks;
pub use evalchecker;
pub use commitstatus;
pub use ghevent;
pub use nix;
pub use acl;
pub use commentparser;
pub use outpathdiff;
}

152
ofborg/src/outpathdiff.rs Normal file
View file

@ -0,0 +1,152 @@
extern crate amqp;
extern crate env_logger;
use std::collections::HashMap;
use std::fs::File;
use std::fs;
use std::io::Read;
use std::io::BufRead;
use std::io::BufReader;
use std::path::Path;
use std::path::PathBuf;
use ofborg::checkout;
use ofborg::message::massrebuildjob;
use ofborg::nix;
use std::io::Write;
use ofborg::worker;
use ofborg::evalchecker::EvalChecker;
use ofborg::commitstatus::CommitStatus;
use amqp::protocol::basic::{Deliver,BasicProperties};
use hubcaps;
pub struct OutPathDiff {
path: PathBuf,
nix: nix::Nix,
pub original: Option<HashMap<String, String>>,
pub current: Option<HashMap<String, String>>,
}
impl OutPathDiff {
pub fn new(nix: nix::Nix, path: PathBuf) -> OutPathDiff {
OutPathDiff {
nix: nix,
path: path,
original: None,
current: None,
}
}
fn parse(&self, f: File) -> HashMap<String, String> {
let mut result: HashMap<String,String>;
result = HashMap::new();
{
BufReader::new(f)
.lines()
.filter_map(|line| match line {
Ok(line) => Some(line),
Err(_) => None
})
.map(|x| {
let split: Vec<&str> = x.split_whitespace().collect();
if split.len() == 2 {
result.insert(String::from(split[0]), String::from(split[1]));
} else {
info!("Warning: not 2 word segments in {:?}", split);
}
}).count();
}
return result;
}
pub fn find_before(&mut self) {
let x = self.run();
match x {
Ok(f) => {
self.original = Some(self.parse(f))
}
Err(_) => {
info!("Failed to find Before list");
}
}
}
pub fn find_after(&mut self) {
if self.original == None {
debug!("Before is None, not bothering with After");
return
}
let x = self.run();
match x {
Ok(f) => {
self.current = Some(self.parse(f))
}
Err(_) => {
info!("Failed to find After list");
}
}
}
pub fn calculate_rebuild(self) -> Option<Vec<String>> {
let mut rebuild: Vec<String> = vec![];
if let Some(cur) = self.current {
if let Some(orig) = self.original {
for key in cur.keys() {
debug!("Checking out {}", key);
if cur.get(key) != orig.get(key) {
debug!(" {:?} != {:?}", cur.get(key), orig.get(key));
rebuild.push(key.clone())
} else {
debug!(" {:?} == {:?}", cur.get(key), orig.get(key));
}
}
return Some(rebuild);
}
}
return None;
}
fn run(&self) -> Result<File, File> {
self.place_nix();
let ret = self.execute();
self.remove_nix();
return ret
}
fn place_nix(&self) {
let mut file = File::create(self.nix_path()).expect("Failed to create nix out path check");
file.write_all(include_str!("rebuild-amount.nix").as_bytes()).expect("");
}
fn remove_nix(&self) {
fs::remove_file(self.nix_path()).expect(":)");
}
fn nix_path(&self) -> PathBuf {
let mut dest = self.path.clone();
dest.push(".gc-of-borg-out-list.nix");
dest
}
fn execute(&self) -> Result<File, File>{
let checker = EvalChecker::new("out-paths",
"nix-env",
vec![
String::from("-f"),
String::from(".gc-of-borg-out-list.nix"),
String::from("-qaP"),
String::from("--no-name"),
String::from("--out-path"),
String::from("--show-trace"),
],
self.nix.clone()
);
checker.execute(&self.path)
}
}

27
ofborg/src/rebuild-amount.nix Normal file
View file

@ -0,0 +1,27 @@
let
lib = import ./lib;
hydraJobs = import ./pkgs/top-level/release.nix
# Compromise: accuracy vs. resources needed for evaluation.
{ supportedSystems = [ "x86_64-linux" "x86_64-darwin" ]; };
recurseIntoAttrs = attrs: attrs // { recurseForDerivations = true; };
# hydraJobs leaves recurseForDerivations as empty attrmaps;
# that would break nix-env and we also need to recurse everywhere.
tweak = lib.mapAttrs
(name: val:
if name == "recurseForDerivations" then true
else if lib.isAttrs val && val.type or null != "derivation"
then recurseIntoAttrs (tweak val)
else val
);
# Some of these contain explicit references to platform(s) we want to avoid;
# some even (transitively) depend on ~/.nixpkgs/config.nix (!)
blacklist = [
"tarball" "metrics" "manual"
"darwin-tested" "unstable" "stdenvBootstrapTools"
"moduleSystem" "lib-tests" # these just confuse the output
];
in
tweak (builtins.removeAttrs hydraJobs blacklist)

9
ofborg/src/tagger.rs Normal file
View file

@ -0,0 +1,9 @@
struct Tagger {
possible: Vec<String>,
selected: Vec<String>,
}
impl Tagger {
}

132
ofborg/src/tasks/massrebuilder.rs
View file

@ -13,6 +13,9 @@ use ofborg::message::massrebuildjob;
use ofborg::nix;
use ofborg::worker;
use ofborg::outpathdiff::OutPathDiff;
use ofborg::evalchecker::EvalChecker;
use ofborg::commitstatus::CommitStatus;
use amqp::protocol::basic::{Deliver,BasicProperties};
use hubcaps;
@ -20,14 +23,16 @@ pub struct MassRebuildWorker {
cloner: checkout::CachedCloner,
nix: nix::Nix,
github: hubcaps::Github,
tmp_root: String,
}
impl MassRebuildWorker {
pub fn new(cloner: checkout::CachedCloner, nix: nix::Nix, github: hubcaps::Github) -> MassRebuildWorker {
pub fn new(cloner: checkout::CachedCloner, nix: nix::Nix, github: hubcaps::Github, tmp_root: String) -> MassRebuildWorker {
return MassRebuildWorker{
cloner: cloner,
nix: nix,
github: github,
tmp_root: tmp_root,
};
}
@ -76,24 +81,37 @@ impl worker::SimpleWorker for MassRebuildWorker {
let target_branch = match job.pr.target_branch.clone() {
Some(x) => { x }
None => { String::from("origin/master") }
None => { String::from("master") }
};
overall_status.set_with_description(
format!("Checking out {}", target_branch),
format!("Checking out {}", &target_branch).as_ref(),
hubcaps::statuses::State::Pending
);
let refpath = co.checkout_ref(target_branch.as_ref()).unwrap();
info!("Checking out target branch {}", &target_branch);
let refpath = co.checkout_origin_ref(target_branch.as_ref()).unwrap();
overall_status.set_with_description(
"Checking original stdenvs",
hubcaps::statuses::State::Pending
);
let mut stdenvs = Stdenvs::new(self.nix.clone(), PathBuf::from(&refpath));
stdenvs.identify_before();
let mut rebuildsniff = OutPathDiff::new(
self.nix.clone(),
PathBuf::from(&refpath)
);
overall_status.set_with_description(
"Checking original out paths",
hubcaps::statuses::State::Pending
);
rebuildsniff.find_before();
overall_status.set_with_description(
"Fetching PR",
hubcaps::statuses::State::Pending
@ -133,6 +151,13 @@ impl worker::SimpleWorker for MassRebuildWorker {
stdenvs.identify_after();
overall_status.set_with_description(
"Checking new out paths",
hubcaps::statuses::State::Pending
);
rebuildsniff.find_after();
println!("Got path: {:?}, building", refpath);
overall_status.set_with_description(
"Begining Evaluations",
@ -218,7 +243,7 @@ impl worker::SimpleWorker for MassRebuildWorker {
let state: hubcaps::statuses::State;
let mut out: File;
match check.execute((&refpath).to_owned()) {
match check.execute(Path::new(&refpath)) {
Ok(o) => {
out = o;
state = hubcaps::statuses::State::Success;
@ -263,98 +288,31 @@ impl worker::SimpleWorker for MassRebuildWorker {
hubcaps::statuses::State::Pending
);
tagger = StdenvTagger::new();
if !stdenvs.are_same() {
println!("Stdenvs changed? {:?}", stdenvs.changed());
}
if let Some(attrs) = rebuildsniff.calculate_rebuild() {
bucketize_attrs(attrs)
}
overall_status.set_with_description(
"^.^!",
hubcaps::statuses::State::Success
);
} else {
overall_status.set_with_description(
"Complete, with errors",
hubcaps::statuses::State::Failed
);
}
return vec![];
}
}
struct CommitStatus<'a> {
api: hubcaps::statuses::Statuses<'a>,
sha: String,
context: String,
description: String,
url: String,
}
impl <'a> CommitStatus<'a> {
fn new(api: hubcaps::statuses::Statuses<'a>, sha: String, context: String, description: String, url: Option<String>) -> CommitStatus<'a> {
let mut stat = CommitStatus {
api: api,
sha: sha,
context: context,
description: description,
url: "".to_owned(),
};
stat.set_url(url);
return stat
}
fn set_url(&mut self, url: Option<String>) {
self.url = url.unwrap_or(String::from(""))
}
fn set_with_description(&mut self, description: &str, state: hubcaps::statuses::State) {
self.set_description(description.to_owned());
self.set(state);
}
fn set_description(&mut self, description: String) {
self.description = description;
}
fn set(&self, state: hubcaps::statuses::State) {
self.api.create(
self.sha.as_ref(),
&hubcaps::statuses::StatusOptions::builder(state)
.context(self.context.clone())
.description(self.description.clone())
.target_url(self.url.clone())
.build()
).expect("Failed to mark final status on commit");
}
}
struct EvalChecker {
name: String,
cmd: String,
args: Vec<String>,
nix: nix::Nix,
}
impl EvalChecker {
fn new(name: &str, cmd: &str, args: Vec<String>, nix: nix::Nix) -> EvalChecker {
EvalChecker{
name: name.to_owned(),
cmd: cmd.to_owned(),
args: args,
nix: nix,
}
}
fn name(&self) -> String {
format!("grahamcofborg-eval-{}", self.name)
}
fn execute(&self, path: String) -> Result<File, File> {
self.nix.safely(&self.cmd, &Path::new(&path), self.args.clone())
}
fn cli_cmd(&self) -> String {
let mut cli = vec![self.cmd.clone()];
cli.append(&mut self.args.clone());
return cli.join(" ");
}
}
enum StdenvFrom {
Before,
After