diff --git a/Cargo.lock b/Cargo.lock index 31e4d2d..2670f9c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -85,39 +85,45 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" [[package]] -name = "async-std" -version = "1.5.0" +name = "async-channel" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "538ecb01eb64eecd772087e5b6f7540cbc917f047727339a472dafed2185b267" +checksum = "2114d64672151c0c5eaa5e131ec84a74f06e1e559830dabba01ca30605d66319" dependencies = [ - "async-task 1.3.1", - "crossbeam-channel 0.4.4", - "crossbeam-deque", + "concurrent-queue", + "event-listener", + "futures-core", +] + +[[package]] +name = "async-std" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a45cee2749d880d7066e328a7e161c7470ced883b2fd000ca4643e9f1dd5083a" +dependencies = [ + "async-task 3.0.0", "crossbeam-utils 0.7.2", + "futures-channel", "futures-core", "futures-io", "futures-timer", "kv-log-macro", "log 0.4.14", "memchr", - "mio 0.6.23", - "mio-uds", "num_cpus", "once_cell", "pin-project-lite 0.1.12", "pin-utils", "slab", + "smol", + "wasm-bindgen-futures", ] [[package]] name = "async-task" -version = "1.3.1" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ac2c016b079e771204030951c366db398864f5026f84a44dafb0ff20f02085d" -dependencies = [ - "libc", - "winapi 0.3.9", -] +checksum = "c17772156ef2829aadc587461c7753af20b7e8db1529bc66855add962a3b35d3" [[package]] name = "async-task" @@ -125,6 +131,12 @@ version = "4.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e91831deabf0d6d7ec49552e489aed63b7456a7a3c46cff62adad428110b0af0" +[[package]] +name = "atomic-waker" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "065374052e7df7ee4047b1160cca5e1467a12351a40b3da123c870ba0b8eda2a" + [[package]] name = "autocfg" version = "1.0.1" @@ -189,12 +201,38 @@ dependencies = [ "wyz", ] +[[package]] +name = "blocking" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2468ff7bf85066b4a3678fede6fe66db31846d753ff0adfbfab2c6a6e81612b" +dependencies = [ + "async-channel", + "atomic-waker", + "futures-lite", + "once_cell", + "parking 1.0.6", + "waker-fn", +] + +[[package]] +name = "bumpalo" +version = "3.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899" + [[package]] name = "byteorder" version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +[[package]] +name = "cache-padded" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1db59621ec70f09c5e9b597b220c7a2b43611f4710dc03ceb8748637775692c" + [[package]] name = "cc" version = "1.0.68" @@ -226,6 +264,15 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "concurrent-queue" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30ed07550be01594c6026cff2a1d7fe9c8f683caa798e12b68694ac9e88286a3" +dependencies = [ + "cache-padded", +] + [[package]] name = "cookie-factory" version = "0.3.2" @@ -248,16 +295,6 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b" -[[package]] -name = "crossbeam-channel" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b153fe7cbef478c567df0f972e02e6d736db11affe43dfc9c56a9374d1adfb87" -dependencies = [ - "crossbeam-utils 0.7.2", - "maybe-uninit", -] - [[package]] name = "crossbeam-channel" version = "0.5.1" @@ -268,32 +305,6 @@ dependencies = [ "crossbeam-utils 0.8.5", ] -[[package]] -name = "crossbeam-deque" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f02af974daeee82218205558e51ec8768b48cf524bd01d550abe5573a608285" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils 0.7.2", - "maybe-uninit", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" -dependencies = [ - "autocfg", - "cfg-if 0.1.10", - "crossbeam-utils 0.7.2", - "lazy_static", - "maybe-uninit", - "memoffset", - "scopeguard", -] - [[package]] name = "crossbeam-utils" version = "0.7.2" @@ -346,6 +357,21 @@ dependencies = [ "backtrace", ] +[[package]] +name = "event-listener" +version = "2.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77f3309417938f28bf8228fcff79a4a37103981e3e186d2ccd19c74b38f4eb71" + +[[package]] +name = "fastrand" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3fcf0cee53519c866c09b5de1f6c56ff9d647101f81c1964fa632e148896cdf" +dependencies = [ + "instant", +] + [[package]] name = "foreign-types" version = "0.3.2" @@ -399,22 +425,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" -[[package]] -name = "fuchsia-zircon" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" -dependencies = [ - "bitflags", - "fuchsia-zircon-sys", -] - -[[package]] -name = "fuchsia-zircon-sys" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" - [[package]] name = "funty" version = "1.1.0" @@ -422,22 +432,83 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fed34cd105917e91daa4da6b3728c47b068749d6a62c59811f06ed2ac71d9da7" [[package]] -name = "futures-core" +name = "futures-channel" version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0402f765d8a89a26043b889b26ce3c4679d268fa6bb22cd7c6aad98340e179d1" +checksum = "e682a68b29a882df0545c143dc3646daefe80ba479bcdede94d5a703de2871e2" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c09fd04b7e4073ac7156a9539b57a484a8ea920f79c7c675d05d289ab6110d3" [[package]] name = "futures-io" -version = "0.3.15" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acc499defb3b348f8d8f3f66415835a9131856ff7714bf10dadfc4ec4bdb29a1" +checksum = "fc4045962a5a5e935ee2fdedaa4e08284547402885ab326734432bed5d12966b" + +[[package]] +name = "futures-lite" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97999970129b808f0ccba93211201d431fcc12d7e1ffae03a61b5cedd1a7ced2" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "memchr", + "parking 2.0.0", + "pin-project-lite 0.1.12", + "waker-fn", +] + +[[package]] +name = "futures-macro" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33c1e13800337f4d4d7a316bf45a567dbcb6ffe087f16424852d97e97a91f512" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-task" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c66a976bf5909d801bbef33416c41372779507e7a6b3a5e25e4749c58f776a" [[package]] name = "futures-timer" -version = "2.0.2" +version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1de7508b218029b0f01662ed8f61b1c964b3ae99d6f25462d0f55a595109df6" +checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" +dependencies = [ + "gloo-timers", + "send_wrapper", +] + +[[package]] +name = "futures-util" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8b7abd5d659d9b90c8cba917f6ec750a74e2dc23902ef9cd4cc8c8b22e6036a" +dependencies = [ + "futures-core", + "futures-io", + "futures-macro", + "futures-task", + "memchr", + "pin-project-lite 0.2.6", + "pin-utils", + "slab", +] [[package]] name = "getrandom" @@ -456,6 +527,18 @@ version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e4075386626662786ddb0ec9081e7c7eeb1ba31951f447ca780ef9f5d568189" +[[package]] +name = "gloo-timers" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d12a7f4e95cfe710f1d624fb1210b7d961a5fb05c4fd942f4feab06e61f590e" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "hermit-abi" version = "0.1.18" @@ -547,21 +630,21 @@ dependencies = [ "cfg-if 1.0.0", ] -[[package]] -name = "iovec" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" -dependencies = [ - "libc", -] - [[package]] name = "itoa" version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" +[[package]] +name = "js-sys" +version = "0.3.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a38fc24e30fd564ce974c02bf1d337caddff65be6cc4735a1f7eab22a7440f04" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "kernel32-sys" version = "0.2.2" @@ -595,10 +678,10 @@ checksum = "5b9d21a790e85496a97d9b82821aa39c59579433e9ad29d7c1ab54d9807ebfcc" dependencies = [ "amq-protocol", "async-task 4.0.3", - "crossbeam-channel 0.5.1", + "crossbeam-channel", "futures-core", "log 0.4.14", - "mio 0.7.11", + "mio", "parking_lot", "pinky-swear", "serde", @@ -687,12 +770,6 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" -[[package]] -name = "maybe-uninit" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" - [[package]] name = "md5" version = "0.3.8" @@ -705,15 +782,6 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc" -[[package]] -name = "memoffset" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa" -dependencies = [ - "autocfg", -] - [[package]] name = "mime" version = "0.2.6" @@ -733,25 +801,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "mio" -version = "0.6.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4afd66f5b91bf2a3bc13fad0e21caedac168ca4c707504e75585648ae80e4cc4" -dependencies = [ - "cfg-if 0.1.10", - "fuchsia-zircon", - "fuchsia-zircon-sys", - "iovec", - "kernel32-sys", - "libc", - "log 0.4.14", - "miow 0.2.2", - "net2", - "slab", - "winapi 0.2.8", -] - [[package]] name = "mio" version = "0.7.11" @@ -760,34 +809,11 @@ checksum = "cf80d3e903b34e0bd7282b218398aec54e082c840d9baf8339e0080a0c542956" dependencies = [ "libc", "log 0.4.14", - "miow 0.3.7", + "miow", "ntapi", "winapi 0.3.9", ] -[[package]] -name = "mio-uds" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afcb699eb26d4332647cc848492bbc15eafb26f08d0304550d5aa1f612e066f0" -dependencies = [ - "iovec", - "libc", - "mio 0.6.23", -] - -[[package]] -name = "miow" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebd808424166322d4a38da87083bfddd3ac4c131334ed55856112eb06d46944d" -dependencies = [ - "kernel32-sys", - "net2", - "winapi 0.2.8", - "ws2_32-sys", -] - [[package]] name = "miow" version = "0.3.7" @@ -815,17 +841,6 @@ dependencies = [ "tempfile 3.2.0", ] -[[package]] -name = "net2" -version = "0.2.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "391630d12b68002ae1e25e8f974306474966550ad82dac6886fb8910c19568ae" -dependencies = [ - "cfg-if 0.1.10", - "libc", - "winapi 0.3.9", -] - [[package]] name = "nom" version = "4.2.3" @@ -904,6 +919,7 @@ dependencies = [ "chrono", "either", "fs2", + "futures-util", "hubcaps", "hyper", "hyper-native-tls", @@ -969,6 +985,18 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "parking" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb300f271742d4a2a66c01b6b2fa0c83dfebd2e0bf11addb879a3547b4ed87c" + +[[package]] +name = "parking" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" + [[package]] name = "parking_lot" version = "0.11.1" @@ -1245,6 +1273,12 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "scoped-tls" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea6a9290e3c9cf0f18145ef7ffa62d68ee0bf5fcd651017e586dc7fd5da448c2" + [[package]] name = "scopeguard" version = "1.1.0" @@ -1274,6 +1308,12 @@ dependencies = [ "libc", ] +[[package]] +name = "send_wrapper" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f638d531eccd6e23b980caf34876660d38e265409d8e99b397ab71eb3612fad0" + [[package]] name = "separator" version = "0.4.1" @@ -1332,6 +1372,38 @@ version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" +[[package]] +name = "smol" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "620cbb3c6e34da57d3a248cda0cd01cd5848164dc062e764e65d06fe3ea7aed5" +dependencies = [ + "async-task 3.0.0", + "blocking", + "concurrent-queue", + "fastrand", + "futures-io", + "futures-util", + "libc", + "once_cell", + "scoped-tls", + "slab", + "socket2", + "wepoll-sys-stjepang", + "winapi 0.3.9", +] + +[[package]] +name = "socket2" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "122e570113d28d773067fab24266b66753f6ea915758651696b6e35e49f88d6e" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "winapi 0.3.9", +] + [[package]] name = "static_assertions" version = "1.1.0" @@ -1372,7 +1444,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a29af643fc448ecb2d9b8a74aa2c60c72c36cb447bcf490b866797b58f00f13" dependencies = [ "cfg-if 1.0.0", - "mio 0.7.11", + "mio", "native-tls", "pem", ] @@ -1618,12 +1690,103 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" +[[package]] +name = "waker-fn" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" + [[package]] name = "wasi" version = "0.10.2+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" +[[package]] +name = "wasm-bindgen" +version = "0.2.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25f1af7423d8588a3d840681122e72e6a24ddbcb3f0ec385cac0d12d24256c06" +dependencies = [ + "cfg-if 1.0.0", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b21c0df030f5a177f3cba22e9bc4322695ec43e7257d865302900290bcdedca" +dependencies = [ + "bumpalo", + "lazy_static", + "log 0.4.14", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eb6ec270a31b1d3c7e266b999739109abce8b6c87e4b31fcfcd788b65267395" +dependencies = [ + "cfg-if 1.0.0", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4203d69e40a52ee523b2529a773d5ffc1dc0071801c87b3d270b471b80ed01" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa8a30d46208db204854cadbb5d4baf5fcf8071ba5bf48190c3e59937962ebc" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d958d035c4438e28c70e4321a2911302f10135ce78a9c7834c0cab4123d06a2" + +[[package]] +name = "web-sys" +version = "0.3.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c060b319f29dd25724f09a2ba1418f142f539b2be99fbf4d2d5a8f7330afb8eb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wepoll-sys-stjepang" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fdfbb03f290ca0b27922e8d48a0997b4ceea12df33269b9f75e713311eb178d" +dependencies = [ + "cc", +] + [[package]] name = "winapi" version = "0.2.8" @@ -1658,16 +1821,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "ws2_32-sys" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" -dependencies = [ - "winapi 0.2.8", - "winapi-build", -] - [[package]] name = "wyz" version = "0.2.0" diff --git a/ofborg/Cargo.toml b/ofborg/Cargo.toml index f7f5c7d..cc0fdcb 100644 --- a/ofborg/Cargo.toml +++ b/ofborg/Cargo.toml @@ -6,10 +6,11 @@ build = "build.rs" edition = "2018" [dependencies] -async-std = "=1.5.0" +async-std = { version = "=1.6.0", features = ["unstable"] } chrono = "0.4.6" either = "1.4.0" fs2 = "0.4.2" +futures-util = "0.3.21" hubcaps = { git = "https://github.com/grahamc/hubcaps.git" } #hubcaps = { path = "../../../grahamc/hubcaps/" } # for testing patches hyper = "0.10.*" diff --git a/ofborg/src/bin/builder.rs b/ofborg/src/bin/builder.rs index b0e45a0..f6a0e4d 100644 --- a/ofborg/src/bin/builder.rs +++ b/ofborg/src/bin/builder.rs @@ -2,7 +2,8 @@ use std::env; use std::error::Error; use std::path::Path; -use async_std::task; +use async_std::task::{self, JoinHandle}; +use futures_util::future; use tracing::{info, warn}; use ofborg::easyamqp::{self, ChannelExt, ConsumerExt}; @@ -17,9 +18,6 @@ fn main() -> Result<(), Box> { let arg = env::args().nth(1).expect("usage: builder "); let cfg = config::load(arg.as_ref()); - let cloner = checkout::cached_cloner(Path::new(&cfg.checkout.root)); - let nix = cfg.nix(); - if !cfg.feedback.full_logs { warn!("Please define feedback.full_logs in your configuration to true!"); warn!("feedback.full_logs when true will cause the full build log to be sent back"); @@ -31,8 +29,30 @@ fn main() -> Result<(), Box> { }; let conn = easylapin::from_config(&cfg.rabbitmq)?; + let mut handles = Vec::new(); + + for system in &cfg.nix.system { + let handle_ext = self::create_handle(&conn, &cfg, system.to_string())?; + handles.push(handle_ext); + } + + task::block_on(future::join_all(handles)); + + drop(conn); // Close connection. + info!("Closed the session... EOF"); + Ok(()) +} + +fn create_handle( + conn: &lapin::Connection, + cfg: &config::Config, + system: String, +) -> Result, Box> { let mut chan = task::block_on(conn.create_channel())?; + let cloner = checkout::cached_cloner(Path::new(&cfg.checkout.root)); + let nix = cfg.nix().with_system(system.clone()); + chan.declare_exchange(easyamqp::ExchangeConfig { exchange: "build-jobs".to_owned(), exchange_type: easyamqp::ExchangeType::Fanout, @@ -44,7 +64,7 @@ fn main() -> Result<(), Box> { })?; let queue_name = if cfg.runner.build_all_jobs != Some(true) { - let queue_name = format!("build-inputs-{}", cfg.nix.system); + let queue_name = format!("build-inputs-{}", system); chan.declare_queue(easyamqp::QueueConfig { queue: queue_name.clone(), passive: false, @@ -77,12 +97,7 @@ fn main() -> Result<(), Box> { })?; let handle = easylapin::NotifyChannel(chan).consume( - tasks::build::BuildWorker::new( - cloner, - nix, - cfg.nix.system.clone(), - cfg.runner.identity.clone(), - ), + tasks::build::BuildWorker::new(cloner, nix, system, cfg.runner.identity.clone()), easyamqp::ConsumeConfig { queue: queue_name.clone(), consumer_tag: format!("{}-builder", cfg.whoami()), @@ -94,9 +109,5 @@ fn main() -> Result<(), Box> { )?; info!("Fetching jobs from {}", &queue_name); - task::block_on(handle); - - drop(conn); // Close connection. - info!("Closed the session... EOF"); - Ok(()) + Ok(task::spawn(handle)) } diff --git a/ofborg/src/bin/log-message-generator.rs b/ofborg/src/bin/log-message-generator.rs deleted file mode 100644 index 4f36bf7..0000000 --- a/ofborg/src/bin/log-message-generator.rs +++ /dev/null @@ -1,69 +0,0 @@ -use std::env; -use std::error::Error; -use std::thread; -use std::time::Duration; - -use async_std::task; -use lapin::message::Delivery; -use lapin::BasicProperties; -use tracing::info; - -use ofborg::config; -use ofborg::easylapin; -use ofborg::message::{buildjob, Pr, Repo}; -use ofborg::tasks::build; - -fn main() -> Result<(), Box> { - ofborg::setup_log(); - - let arg = env::args() - .nth(1) - .expect("usage: log-message-generator "); - let cfg = config::load(arg.as_ref()); - - let conn = easylapin::from_config(&cfg.rabbitmq)?; - let mut chan = task::block_on(conn.create_channel())?; - - let deliver = Delivery { - delivery_tag: 0, - exchange: "no-exchange".into(), - routing_key: "".into(), - redelivered: false, - properties: BasicProperties::default(), - data: vec![], - acker: Default::default(), - }; - let mut receiver = easylapin::ChannelNotificationReceiver::new(&mut chan, &deliver); - let job = buildjob::BuildJob { - attrs: vec![], - pr: Pr { - head_sha: String::from("bogus"), - number: 1, - target_branch: Some("master".to_owned()), - }, - repo: Repo { - clone_url: String::from("bogus"), - full_name: "test-git".to_owned(), - name: "nixos".to_owned(), - owner: "ofborg-test".to_owned(), - }, - subset: None, - logs: Some((Some(String::from("logs")), Some(String::from("build.log")))), - statusreport: Some((Some(String::from("build-results")), None)), - request_id: "bogus-request-id".to_owned(), - }; - - loop { - info!("Starting a new build simulation"); - let mut actions = - build::JobActions::new(&cfg.nix.system, &cfg.runner.identity, &job, &mut receiver); - actions.log_started(vec![], vec![]); - - for i in 1..51 { - actions.log_line(&format!("Bogus message #{:?}/50", i)); - thread::sleep(Duration::from_secs(3)) - } - - thread::sleep(Duration::from_secs(10)) - } -} diff --git a/ofborg/src/bin/mass-rebuilder.rs b/ofborg/src/bin/mass-rebuilder.rs index ebd5339..5d118c3 100644 --- a/ofborg/src/bin/mass-rebuilder.rs +++ b/ofborg/src/bin/mass-rebuilder.rs @@ -38,10 +38,7 @@ fn main() -> Result<(), Box> { let cloner = checkout::cached_cloner(Path::new(&cfg.checkout.root)); let nix = cfg.nix(); - let events = stats::RabbitMq::from_lapin( - &format!("{}-{}", cfg.runner.identity, cfg.nix.system), - task::block_on(conn.create_channel())?, - ); + let events = stats::RabbitMq::from_lapin(&cfg.whoami(), task::block_on(conn.create_channel())?); let queue_name = String::from("mass-rebuild-check-jobs"); chan.declare_queue(easyamqp::QueueConfig { diff --git a/ofborg/src/bin/stats.rs b/ofborg/src/bin/stats.rs index 7717927..3072d78 100644 --- a/ofborg/src/bin/stats.rs +++ b/ofborg/src/bin/stats.rs @@ -18,10 +18,7 @@ fn main() -> Result<(), Box> { let conn = easylapin::from_config(&cfg.rabbitmq)?; let mut chan = task::block_on(conn.create_channel())?; - let events = stats::RabbitMq::from_lapin( - &format!("{}-{}", cfg.runner.identity, cfg.nix.system), - task::block_on(conn.create_channel())?, - ); + let events = stats::RabbitMq::from_lapin(&cfg.whoami(), task::block_on(conn.create_channel())?); let metrics = stats::MetricCollector::new(); let collector = tasks::statscollector::StatCollectorWorker::new(events, metrics.clone()); diff --git a/ofborg/src/config.rs b/ofborg/src/config.rs index 30ba8d4..8fd791e 100644 --- a/ofborg/src/config.rs +++ b/ofborg/src/config.rs @@ -2,14 +2,17 @@ use crate::acl; use crate::nix::Nix; use std::collections::HashMap; +use std::fmt; use std::fs::File; use std::io::Read; +use std::marker::PhantomData; use std::path::{Path, PathBuf}; use hubcaps::{Credentials, Github, InstallationTokenGenerator, JWTCredentials}; use hyper::net::HttpsConnector; use hyper::Client; use hyper_native_tls::NativeTlsClient; +use serde::de::{self, Deserialize, Deserializer}; use tracing::{debug, error, info, warn}; #[derive(Serialize, Deserialize, Debug)] @@ -40,7 +43,8 @@ pub struct RabbitMqConfig { #[derive(Serialize, Deserialize, Debug)] pub struct NixConfig { - pub system: String, + #[serde(deserialize_with = "deserialize_one_or_many")] + pub system: Vec, pub remote: String, pub build_timeout_seconds: u16, pub initial_heap_size: Option, @@ -87,7 +91,7 @@ pub struct CheckoutConfig { impl Config { pub fn whoami(&self) -> String { - format!("{}-{}", self.runner.identity, self.nix.system) + format!("{}-{}", self.runner.identity, self.nix.system.join(",")) } pub fn acl(&self) -> acl::Acl { @@ -135,7 +139,11 @@ impl Config { } Nix::new( - self.nix.system.clone(), + self.nix + .system + .first() + .expect("expected at least one system") + .clone(), self.nix.remote.clone(), self.nix.build_timeout_seconds, self.nix.initial_heap_size.clone(), @@ -223,3 +231,35 @@ impl GithubAppVendingMachine { })) } } + +// Copied from https://stackoverflow.com/a/43627388 +fn deserialize_one_or_many<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + struct StringOrVec(PhantomData>); + + impl<'de> de::Visitor<'de> for StringOrVec { + type Value = Vec; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("string or list of strings") + } + + fn visit_str(self, value: &str) -> Result + where + E: de::Error, + { + Ok(vec![value.to_owned()]) + } + + fn visit_seq(self, visitor: S) -> Result + where + S: de::SeqAccess<'de>, + { + Deserialize::deserialize(de::value::SeqAccessDeserializer::new(visitor)) + } + } + + deserializer.deserialize_any(StringOrVec(PhantomData)) +} diff --git a/ofborg/src/easylapin.rs b/ofborg/src/easylapin.rs index 38ef5fb..31d2496 100644 --- a/ofborg/src/easylapin.rs +++ b/ofborg/src/easylapin.rs @@ -155,9 +155,9 @@ impl<'a> NotificationReceiver for ChannelNotificationReceiver<'a> { // but one could probably be implemented in terms of the other instead. pub struct NotifyChannel(pub Channel); -impl<'a, W: SimpleNotifyWorker + 'a> ConsumerExt<'a, W> for NotifyChannel { +impl<'a, W: SimpleNotifyWorker + 'a + Send> ConsumerExt<'a, W> for NotifyChannel { type Error = lapin::Error; - type Handle = Pin + 'a>>; + type Handle = Pin + 'a + Send>>; fn consume(self, worker: W, config: ConsumeConfig) -> Result { task::block_on(self.0.basic_qos(1, BasicQosOptions::default()))?; diff --git a/ofborg/src/nix.rs b/ofborg/src/nix.rs index 54cda60..1437835 100644 --- a/ofborg/src/nix.rs +++ b/ofborg/src/nix.rs @@ -97,7 +97,7 @@ impl fmt::Display for Operation { #[derive(Clone, Debug, PartialEq)] pub struct Nix { - system: String, + pub system: String, remote: String, build_timeout: u16, limit_supported_systems: bool, diff --git a/ofborg/src/tasks/evaluate.rs b/ofborg/src/tasks/evaluate.rs index 7891a1f..083a609 100644 --- a/ofborg/src/tasks/evaluate.rs +++ b/ofborg/src/tasks/evaluate.rs @@ -298,8 +298,8 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> { Err(e) => { self.events.notify(Event::IssueFetchFailed); - info!("Error fetching {}!", job.pr.number); - info!("E: {:?}", e); + error!("Error fetching {}!", job.pr.number); + error!("E: {:?}", e); return Ok(self.actions().skip(&job)); } };