Merge branch 'released' of github.com:grahamc/ofborg into released

.gitignore

@@ -4,6 +4,9 @@ vendor
 test.php
 config.json
 .bash_hist
-/config*
+config.private.json
+config.prod.json
+config.local.json
+config.*irc*.json
 result
 target

README.md

@@ -144,6 +144,17 @@ Run
 ```
 
 
+Note the config.public.json for the public pieces of how I run ofborg,
+which is merged with config.known-users.json and a third private
+config file of credentials. These files contain some special keys like
+
+ - known users
+ - authorized users
+ - log storage
+
+they are only used in the backend processing tasks, and there is no
+need for them on builders. However, to update the list in
+config.known-users.json, run `./scripts/update-known-users.sh`.
 
 ## old php stuff...
 

config.known-users.json

@@ -0,0 +1,111 @@
+{
+  "runner": {
+    "known_users": [
+      "7c6f434c",
+      "abbradar",
+      "adisbladis",
+      "aforemny",
+      "amiddelk",
+      "aminechikhaoui",
+      "andersontorres",
+      "andir",
+      "antono",
+      "aristidb",
+      "armijnhemel",
+      "astsmtl",
+      "aszlig",
+      "aycanirican",
+      "bendlas",
+      "benley",
+      "bennofs",
+      "bjornfor",
+      "bluescreen303",
+      "c0bw3b",
+      "chaoflow",
+      "cillianderoiste",
+      "civodul",
+      "copumpkin",
+      "cpages",
+      "cstrahan",
+      "damiencassou",
+      "dezgeg",
+      "dguibert",
+      "disassembler",
+      "domenkozar",
+      "edolstra",
+      "edwtjo",
+      "ehmry",
+      "ericson2314",
+      "errge",
+      "falsifian",
+      "fpletz",
+      "fridh",
+      "fuuzetsu",
+      "garbas",
+      "gebner",
+      "globin",
+      "grahamc",
+      "grahamcofborg",
+      "gridaphobe",
+      "hrdinka",
+      "jagajaga",
+      "jgeerds",
+      "joachifm",
+      "jtojnar",
+      "jwiegley",
+      "kevincox",
+      "kosmikus",
+      "lethalman",
+      "lnl7",
+      "lovek323",
+      "lsix",
+      "madjar",
+      "maggesi",
+      "matejc",
+      "matthewbauer",
+      "mic92",
+      "mornfall",
+      "mp2e",
+      "nbp",
+      "nckx",
+      "ndowens",
+      "nequissimus",
+      "nicolaspetton",
+      "obadz",
+      "ocharles",
+      "offlinehacker",
+      "orivej",
+      "peterhoeg",
+      "peti",
+      "phreedom",
+      "pikajude",
+      "primeos",
+      "profpatsch",
+      "psub",
+      "qknight",
+      "rasendubi",
+      "rbvermaa",
+      "rickynils",
+      "roconnor",
+      "rushmorem",
+      "ryantrinkle",
+      "rycee",
+      "shlevy",
+      "srhb",
+      "svanderburg",
+      "the-kenny",
+      "thoughtpolice",
+      "ts468",
+      "ttuegel",
+      "vbgl",
+      "vcunat",
+      "viric",
+      "vrthra",
+      "wizeman",
+      "wkennington",
+      "wmertens",
+      "yegortimoshenko",
+      "zimbatm"
+    ]
+  }
+}

config.public.json

@@ -0,0 +1,50 @@
+{
+    "feedback": {
+        "full_logs": true
+    },
+    "log_storage": {
+        "path": "/var/lib/nginx/ofborg/logs/"
+    },
+    "runner": {
+        "trusted_users": [
+            "7c6f434c",
+            "adisbladis",
+            "andir",
+            "ankhers",
+            "aneeshusa",
+            "aszlig",
+            "copumpkin",
+            "disassembler",
+            "domenkozar",
+            "fpletz",
+            "fridh",
+            "garbas",
+            "globin",
+            "grahamc",
+            "jb55",
+            "joachifm",
+            "jtojnar",
+            "lheckemann",
+            "lnl7",
+            "mic92",
+            "nequissimus",
+            "orivej",
+            "peti",
+            "rbvermaa",
+            "shlevy",
+            "srhb",
+            "veprbl",
+            "vcunat",
+            "yegortimoshenko",
+            "zimbatm"
+        ]
+    },
+    "checkout": {
+        "root": "/var/lib/gc-of-borg/.nix-test-rs"
+    },
+    "nix": {
+        "system": "x86_64-linux",
+        "remote": "daemon",
+        "build_timeout_seconds": 3600
+    }
+}

ofborg/.gitignore

@@ -2,3 +2,4 @@ target
 rust-amqp
 test-scratch
 *.bk
+rust-amq-proto

ofborg/src/acl.rs

@@ -1,18 +1,30 @@
 
 pub struct ACL {
-    authorized_users: Vec<String>,
+    trusted_users: Vec<String>,
+    known_users: Vec<String>,
 }
 
 impl ACL {
-    pub fn new(authorized_users: Vec<String>) -> ACL {
+    pub fn new(trusted_users: Vec<String>, known_users: Vec<String>) -> ACL {
-        return ACL { authorized_users: authorized_users };
+        return ACL {
+            trusted_users: trusted_users,
+            known_users: known_users,
+        };
     }
 
-    pub fn can_build(&self, user: &str, repo: &str) -> bool {
+    pub fn can_build_restricted(&self, user: &str, repo: &str) -> bool {
         if repo.to_lowercase() != "nixos/nixpkgs" {
             return false;
         }
 
-        return self.authorized_users.contains(&user.to_lowercase());
+        return self.known_users.contains(&user.to_lowercase());
+    }
+
+    pub fn can_build_unrestricted(&self, user: &str, repo: &str) -> bool {
+        if repo.to_lowercase() != "nixos/nixpkgs" {
+            return false;
+        }
+
+        return self.trusted_users.contains(&user.to_lowercase());
     }
 }

ofborg/src/config.rs

@@ -55,7 +55,8 @@ pub struct LogStorage {
 #[derive(Serialize, Deserialize, Debug)]
 pub struct RunnerConfig {
     pub identity: String,
-    pub authorized_users: Option<Vec<String>>,
+    pub trusted_users: Option<Vec<String>>,
+    pub known_users: Option<Vec<String>>,
 }
 
 #[derive(Serialize, Deserialize, Debug)]
@@ -69,9 +70,14 @@ impl Config {
     }
 
     pub fn acl(&self) -> acl::ACL {
-        return acl::ACL::new(self.runner.authorized_users.clone().expect(
+        return acl::ACL::new(
-            "fetching config's runner.authorized_users",
+            self.runner.trusted_users.clone().expect(
-        ));
+                "fetching config's runner.trusted_users",
+            ),
+            self.runner.known_users.clone().expect(
+                "fetching config's runner.known_users",
+            ),
+        );
     }
 
     pub fn github(&self) -> Github {

ofborg/src/tasks/githubcommentfilter.rs

@@ -53,11 +53,25 @@ impl worker::SimpleWorker for GitHubCommentWorker {
             return vec![worker::Action::Ack];
         }
 
-        if !self.acl.can_build(
+        let build_destinations: Vec<(Option<String>,Option<String>)>;
+
+        if self.acl.can_build_unrestricted(
+            &job.comment.user.login,
+            &job.repository.full_name,
+        ) {
+            build_destinations = vec![
+                (Some("build-jobs".to_owned()), None)
+            ];
+        } else if self.acl.can_build_restricted(
             &job.comment.user.login,
             &job.repository.full_name,
         )
         {
+            build_destinations = vec![
+                (None, Some("build-inputs-x86_64-linux".to_owned())),
+                (None, Some("build-inputs-aarch64-linux".to_owned())),
+            ];
+        } else {
             println!(
                 "ACL prohibits {} from building {:?} for {}",
                 job.comment.user.login,
@@ -125,11 +139,13 @@ impl worker::SimpleWorker for GitHubCommentWorker {
                             statusreport: Some((Some("build-results".to_owned()), None)),
                         };
 
-                        response.push(worker::publish_serde_action(
+                        for (exch, rk) in build_destinations.clone() {
-                            Some("build-jobs".to_owned()),
+                            response.push(worker::publish_serde_action(
-                            None,
+                                exch,
-                            &msg,
+                                rk,
-                        ));
+                                &msg,
+                            ));
+                        }
                     }
                     commentparser::Instruction::Eval => {
                         let msg = massrebuildjob::MassRebuildJob {

scripts/merge-config.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -p bash -p jq -p curl -i bash
+
+jq -s '.[0] * .[1] * .[2]' ./config.public.json ./config.known-users.json ./config.private.json > ./config.prod.json

scripts/update-known-users.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -p bash -p jq -p curl -i bash
+
+readonly token=$(jq -r '.github.token' ./config.private.json)
+
+readonly dest=config.known-users.json
+readonly scratch=user-list.scratch
+readonly accumulator=user-list.accumulator
+readonly result=user-list.result
+
+function fetch_users() {
+    curl \
+        -H "Authorization: token $token" \
+        "https://api.github.com/orgs/NixOS/members?page=$1" \
+        | jq 'map(.login | ascii_downcase)'
+}
+
+echo '[]' > "$accumulator"
+
+page=0
+while true; do
+    page=$((page + 1))
+    fetch_users "$page" > "$scratch"
+
+    jq -s '.[0] + .[1]' "$accumulator" "$scratch" > "$result"
+    mv "$result" "$accumulator"
+
+    if [ $(jq -r 'length' "$scratch") -eq 0 ]; then
+        break
+    fi
+done
+
+jq -s '{ "runner": { "known_users": .[0]}}' "$accumulator" > "$dest"
+
+rm -f "$result" "$scratch" "$accumulator"