Merge pull request #234 from NixOS/move-mass-rebuilder

Move mass rebuilder

.travis.yml

@@ -1,4 +1,5 @@
 language: nix
+nix: 2.1
 matrix:
   include:
     - name: checkPhase - Nix 2

nix/nixpkgs.json

@@ -1,7 +1,7 @@
 {
   "url": "https://github.com/nixos/nixpkgs-channels.git",
-  "rev": "2e855dc6b0cd88767a8a5df2faff9e66e1cd7f18",
-  "date": "2018-04-11T10:16:39+08:00",
-  "sha256": "08s6mfh5a05kd2qs3hmza50ng3pyhd3qha4nanwwk0s8fjzqnv4a",
+  "rev": "4477cf04b6779a537cdb5f0bd3dd30e75aeb4a3b",
+  "date": "2018-08-31T21:37:07-04:00",
+  "sha256": "1i39wsfwkvj9yryj8di3jibpdg3b3j86ych7s9rb6z79k08yaaxc",
   "fetchSubmodules": true
 }

ofborg/src/files.rs

@@ -0,0 +1,8 @@
+use std::io::Read;
+use std::fs::File;
+
+pub fn file_to_str(f: &mut File) -> String {
+    let mut buffer = Vec::new();
+    f.read_to_end(&mut buffer).expect("Reading eval output");
+    return String::from(String::from_utf8_lossy(&buffer));
+}

ofborg/src/lib.rs

@@ -36,6 +36,7 @@ pub mod config;
 pub mod message;
 pub mod tasks;
 pub mod evalchecker;
+pub mod files;
 pub mod nix;
 pub mod stats;
 pub mod ghevent;
@@ -61,6 +62,7 @@ pub mod ofborg {
     pub use message;
     pub use tasks;
     pub use evalchecker;
+    pub use files;
     pub use commitstatus;
     pub use ghevent;
     pub use nix;

ofborg/src/tagger.rs

@@ -21,13 +21,13 @@ impl StdenvTagger {
         return t;
     }
 
-    pub fn changed(&mut self, systems: Vec<tasks::massrebuilder::System>) {
+    pub fn changed(&mut self, systems: Vec<tasks::eval::stdenvs::System>) {
         for system in systems {
             match system {
-                tasks::massrebuilder::System::X8664Darwin => {
+                tasks::eval::stdenvs::System::X8664Darwin => {
                     self.selected.push(String::from("10.rebuild-darwin-stdenv"));
                 }
-                tasks::massrebuilder::System::X8664Linux => {
+                tasks::eval::stdenvs::System::X8664Linux => {
                     self.selected.push(String::from("10.rebuild-linux-stdenv"));
                 }
             }

ofborg/src/tasks/eval/mod.rs

@@ -0,0 +1,2 @@
+pub mod stdenvs;
+pub use self::stdenvs::Stdenvs;

ofborg/src/tasks/eval/stdenvs.rs

@@ -0,0 +1,149 @@
+
+use std::path::PathBuf;
+use ofborg::nix;
+use ofborg::files::file_to_str;
+
+enum StdenvFrom {
+    Before,
+    After,
+}
+
+#[derive(Debug)]
+pub enum System {
+    X8664Darwin,
+    X8664Linux,
+}
+
+#[derive(Debug, PartialEq)]
+pub struct Stdenvs {
+    nix: nix::Nix,
+    co: PathBuf,
+
+    linux_stdenv_before: Option<String>,
+    linux_stdenv_after: Option<String>,
+
+    darwin_stdenv_before: Option<String>,
+    darwin_stdenv_after: Option<String>,
+}
+
+impl Stdenvs {
+    pub fn new(nix: nix::Nix, co: PathBuf) -> Stdenvs {
+        return Stdenvs {
+            nix: nix,
+            co: co,
+
+            linux_stdenv_before: None,
+            linux_stdenv_after: None,
+
+            darwin_stdenv_before: None,
+            darwin_stdenv_after: None,
+        };
+    }
+
+    pub fn identify_before(&mut self) {
+        self.identify(System::X8664Linux, StdenvFrom::Before);
+        self.identify(System::X8664Darwin, StdenvFrom::Before);
+    }
+
+    pub fn identify_after(&mut self) {
+        self.identify(System::X8664Linux, StdenvFrom::After);
+        self.identify(System::X8664Darwin, StdenvFrom::After);
+    }
+
+    pub fn are_same(&self) -> bool {
+        return self.changed().len() == 0;
+    }
+
+    pub fn changed(&self) -> Vec<System> {
+        let mut changed: Vec<System> = vec![];
+
+        if self.linux_stdenv_before != self.linux_stdenv_after {
+            changed.push(System::X8664Linux);
+        }
+
+        if self.darwin_stdenv_before != self.darwin_stdenv_after {
+            changed.push(System::X8664Darwin);
+        }
+
+
+        return changed;
+    }
+
+    fn identify(&mut self, system: System, from: StdenvFrom) {
+        match (system, from) {
+            (System::X8664Linux, StdenvFrom::Before) => {
+                self.linux_stdenv_before = self.evalstdenv("x86_64-linux");
+            }
+            (System::X8664Linux, StdenvFrom::After) => {
+                self.linux_stdenv_after = self.evalstdenv("x86_64-linux");
+            }
+
+            (System::X8664Darwin, StdenvFrom::Before) => {
+                self.darwin_stdenv_before = self.evalstdenv("x86_64-darwin");
+            }
+            (System::X8664Darwin, StdenvFrom::After) => {
+                self.darwin_stdenv_after = self.evalstdenv("x86_64-darwin");
+            }
+        }
+    }
+
+    /// This is used to find out what the output path of the stdenv for the
+    /// given system.
+    fn evalstdenv(&self, system: &str) -> Option<String> {
+        let result = self.nix.with_system(system.to_owned()).safely(
+            nix::Operation::QueryPackagesOutputs,
+            &self.co,
+            vec![
+                String::from("-f"),
+                String::from("."),
+                String::from("-A"),
+                String::from("stdenv"),
+            ],
+            true,
+        );
+
+        println!("{:?}", result);
+
+        return match result {
+            Ok(mut out) => Some(file_to_str(&mut out)),
+            Err(mut out) => {
+                println!("{:?}", file_to_str(&mut out));
+                None
+            }
+        };
+    }
+}
+
+#[cfg(test)]
+mod tests {
+
+    use super::*;
+    use std::env;
+    use std::process::Command;
+
+    #[test]
+    fn stdenv_checking() {
+        let output = Command::new("nix-instantiate")
+            .args(&["--eval", "-E", "<nixpkgs>"])
+            .output()
+            .expect("nix-instantiate required");
+
+        let nixpkgs = String::from_utf8(output.stdout)
+            .expect("nixpkgs required");
+
+        let remote = env::var("NIX_REMOTE").unwrap_or("".to_owned());
+        let nix = nix::Nix::new(String::from("x86_64-linux"), remote, 1200, None);
+        let mut stdenv =
+            Stdenvs::new(
+                nix.clone(),
+                PathBuf::from(nixpkgs.trim_right()),
+            );
+        stdenv.identify(System::X8664Linux, StdenvFrom::Before);
+        stdenv.identify(System::X8664Darwin, StdenvFrom::Before);
+
+        stdenv.identify(System::X8664Linux, StdenvFrom::After);
+        stdenv.identify(System::X8664Darwin, StdenvFrom::After);
+
+        assert!(stdenv.are_same());
+    }
+}

ofborg/src/tasks/log_message_collector.rs

@@ -393,7 +393,7 @@ mod tests {
             message: MsgType::Msg(logmsg.clone()),
         };
 
-        let p = TestScratch::new_dir("log-message-collector-path_for_log");
+        let p = TestScratch::new_dir("log-message-collector-logs_collector");
 
         {
             let mut worker = make_worker(p.path());

ofborg/src/tasks/massrebuilder.rs

@@ -3,15 +3,15 @@ extern crate amqp;
 extern crate env_logger;
 extern crate uuid;
 
+use tasks::eval;
 use uuid::Uuid;
 use std::collections::HashMap;
-use std::fs::File;
-use std::io::Read;
 use std::path::Path;
 use std::path::PathBuf;
 use ofborg::checkout;
 use ofborg::message::{massrebuildjob, buildjob};
 use std::time::Instant;
+use ofborg::files::file_to_str;
 use ofborg::nix;
 use ofborg::acl::ACL;
 use ofborg::stats;
@@ -196,7 +196,7 @@ impl<E: stats::SysEvents + 'static> worker::SimpleWorker for MassRebuildWorker<E
         );
 
 
-        let mut stdenvs = Stdenvs::new(self.nix.clone(), PathBuf::from(&refpath));
+        let mut stdenvs = eval::Stdenvs::new(self.nix.clone(), PathBuf::from(&refpath));
         stdenvs.identify_before();
 
         let mut rebuildsniff = OutPathDiff::new(self.nix.clone(), PathBuf::from(&refpath));
@@ -561,116 +561,7 @@ impl<E: stats::SysEvents + 'static> worker::SimpleWorker for MassRebuildWorker<E
     }
 }
 
-enum StdenvFrom {
-    Before,
-    After,
-}
 
-#[derive(Debug)]
-pub enum System {
-    X8664Darwin,
-    X8664Linux,
-}
-
-#[derive(Debug, PartialEq)]
-struct Stdenvs {
-    nix: nix::Nix,
-    co: PathBuf,
-
-    linux_stdenv_before: Option<String>,
-    linux_stdenv_after: Option<String>,
-
-    darwin_stdenv_before: Option<String>,
-    darwin_stdenv_after: Option<String>,
-}
-
-impl Stdenvs {
-    fn new(nix: nix::Nix, co: PathBuf) -> Stdenvs {
-        return Stdenvs {
-            nix: nix,
-            co: co,
-
-            linux_stdenv_before: None,
-            linux_stdenv_after: None,
-
-            darwin_stdenv_before: None,
-            darwin_stdenv_after: None,
-        };
-    }
-
-    fn identify_before(&mut self) {
-        self.identify(System::X8664Linux, StdenvFrom::Before);
-        self.identify(System::X8664Darwin, StdenvFrom::Before);
-    }
-
-    fn identify_after(&mut self) {
-        self.identify(System::X8664Linux, StdenvFrom::After);
-        self.identify(System::X8664Darwin, StdenvFrom::After);
-    }
-
-    fn are_same(&self) -> bool {
-        return self.changed().len() == 0;
-    }
-
-    fn changed(&self) -> Vec<System> {
-        let mut changed: Vec<System> = vec![];
-
-        if self.linux_stdenv_before != self.linux_stdenv_after {
-            changed.push(System::X8664Linux);
-        }
-
-        if self.darwin_stdenv_before != self.darwin_stdenv_after {
-            changed.push(System::X8664Darwin);
-        }
-
-
-        return changed;
-    }
-
-    fn identify(&mut self, system: System, from: StdenvFrom) {
-        match (system, from) {
-            (System::X8664Linux, StdenvFrom::Before) => {
-                self.linux_stdenv_before = self.evalstdenv("x86_64-linux");
-            }
-            (System::X8664Linux, StdenvFrom::After) => {
-                self.linux_stdenv_after = self.evalstdenv("x86_64-linux");
-            }
-
-            (System::X8664Darwin, StdenvFrom::Before) => {
-                self.darwin_stdenv_before = self.evalstdenv("x86_64-darwin");
-            }
-            (System::X8664Darwin, StdenvFrom::After) => {
-                self.darwin_stdenv_after = self.evalstdenv("x86_64-darwin");
-            }
-        }
-    }
-
-    /// This is used to find out what the output path of the stdenv for the
-    /// given system.
-    fn evalstdenv(&self, system: &str) -> Option<String> {
-        let result = self.nix.with_system(system.to_owned()).safely(
-            nix::Operation::QueryPackagesOutputs,
-            &self.co,
-            vec![
-                String::from("-f"),
-                String::from("."),
-                String::from("-A"),
-                String::from("stdenv"),
-            ],
-            true,
-        );
-
-        println!("{:?}", result);
-
-        return match result {
-            Ok(mut out) => Some(file_to_str(&mut out)),
-            Err(mut out) => {
-                println!("{:?}", file_to_str(&mut out));
-                None
-            }
-        };
-    }
-}
 
 fn make_gist<'a>(
     gists: &hubcaps::gists::Gists<'a>,
@@ -730,12 +621,6 @@ pub fn update_labels(issue: &hubcaps::issues::IssueRef, add: Vec<String>, remove
     }
 }
 
-fn file_to_str(f: &mut File) -> String {
-    let mut buffer = Vec::new();
-    f.read_to_end(&mut buffer).expect("Reading eval output");
-    return String::from(String::from_utf8_lossy(&buffer));
-}
-
 fn parse_commit_messages(messages: Vec<String>) -> Vec<String> {
     messages
         .iter()
@@ -760,34 +645,6 @@ fn parse_commit_messages(messages: Vec<String>) -> Vec<String> {
 mod tests {
 
     use super::*;
-    use std::env;
-    use std::process::Command;
-
-    #[test]
-    fn stdenv_checking() {
-        let output = Command::new("nix-instantiate")
-            .args(&["--eval", "-E", "<nixpkgs>"])
-            .output()
-            .expect("nix-instantiate required");
-
-        let nixpkgs = String::from_utf8(output.stdout)
-            .expect("nixpkgs required");
-
-        let remote = env::var("NIX_REMOTE").unwrap_or("".to_owned());
-        let nix = nix::Nix::new(String::from("x86_64-linux"), remote, 1200, None);
-        let mut stdenv =
-            Stdenvs::new(
-                nix.clone(),
-                PathBuf::from(nixpkgs.trim_right()),
-            );
-        stdenv.identify(System::X8664Linux, StdenvFrom::Before);
-        stdenv.identify(System::X8664Darwin, StdenvFrom::Before);
-
-        stdenv.identify(System::X8664Linux, StdenvFrom::After);
-        stdenv.identify(System::X8664Darwin, StdenvFrom::After);
-
-        assert!(stdenv.are_same());
-    }
 
     #[test]
     fn test_parse_commit_messages() {

ofborg/src/tasks/mod.rs

@@ -1,4 +1,5 @@
 
+pub mod eval;
 pub mod build;
 pub mod massrebuilder;
 pub mod githubcommentfilter;

shell.nix

@@ -1,6 +1,17 @@
 { pkgs ? import ./nix {}, useNix1 ? false }:
 
 let
+  # A random Nixpkgs revision *before* the default glibc
+  # was switched to version 2.27.x.
+  oldpkgsSrc = pkgs.fetchFromGitHub {
+    owner = "nixos";
+    repo = "nixpkgs";
+    rev = "0252e6ca31c98182e841df494e6c9c4fb022c676";
+    sha256 = "1sr5a11sb26rgs1hmlwv5bxynw2pl5w4h5ic0qv3p2ppcpmxwykz";
+  };
+
+  oldpkgs = import oldpkgsSrc {};
+
   inherit (pkgs) stdenv;
 
   phpEnv = stdenv.mkDerivation rec {
@@ -23,6 +34,7 @@ let
   rustEnv = stdenv.mkDerivation rec {
     name = "gh-event-forwarder";
     buildInputs = with pkgs; [
+      bash
       nix-prefetch-git
       rust.rustc
       rust.cargo
@@ -43,7 +55,10 @@ let
 
     HISTFILE = "${toString ./.}/.bash_hist";
     RUSTFLAGS = "-D warnings";
-
+    LOCALE_ARCHIVE_2_21 = "${oldpkgs.glibcLocales}/lib/locale/locale-archive";
+    LOCALE_ARCHIVE_2_27 = "${pkgs.glibcLocales}/lib/locale/locale-archive";
+    RUST_BACKTRACE = "1";
+    RUST_LOG = "ofborg=debug";
     passthru.phpEnv = phpEnv;
   };