diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..3322b5d --- /dev/null +++ b/.pylintrc @@ -0,0 +1,9 @@ +[MESSAGES CONTROL] +disable=C0330, C0114, R0903 + +[BASIC] +no-docstring-rgx=(__.*__)|(_.*) + +[FORMAT] +indent-string=' ' +good-names=i,f diff --git a/LICENSE b/LICENSE index a8be45b..0f529da 100644 --- a/LICENSE +++ b/LICENSE @@ -1,201 +1,283 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ +``` + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ -TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION -1. Definitions. + 1. Definitions. - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. -2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. -3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. -4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. -5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. -6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. -7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. -8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. -9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. -END OF TERMS AND CONDITIONS + END OF TERMS AND CONDITIONS -APPENDIX: How to apply the Apache License to your work. + APPENDIX: How to apply the Apache License to your work. - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. -Copyright 2020 The Android Open Source Project + Copyright 2020 The Android Open Source Project -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` + +## Subcomponents + +This project includes the following subcomponents that are subject to separate +license terms. Your use of these subcomponents is subject to the separate +license terms applicable to each subcomponent. + +Passlib \ +https://bitbucket.org/ecollins/passlib/wiki/Home \ +Copyright (c) 2008-2017 Assurance Technologies, LLC. +All rights reserved. \ +3-Clause BSD License (https://passlib.readthedocs.io/en/stable/copyright.html) + +Python-GnuPG \ +https://bitbucket.org/vinay.sajip/python-gnupg \ +Copyright (c) 2008-2014 by Vinay Sajip. +All rights reserved. \ +3-Clause BSD License (https://bitbucket.org/vinay.sajip/python-gnupg/src/master/LICENSE.txt) + +PyYaml \ +https://github.com/yaml/pyyaml \ +Copyright (c) 2017-2019 Ingy döt Net \ +Copyright (c) 2006-2016 Kirill Simonov \ +MIT License (https://github.com/yaml/pyyaml/blob/master/LICENSE) + +--- +## The MIT License (MIT) + +``` +Copyright + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +``` + +## 3-Clause BSD License + +``` +Copyright + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors +may be used to endorse or promote products derived from this software without +specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +``` diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..3568467 --- /dev/null +++ b/Pipfile @@ -0,0 +1,17 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] +typed-ast = "~=1.4.1" +black = "==19.10b0" +pylint = "==2.4.4" + +[packages] +pyyaml = "~=5.3" +passlib = "~=1.7.2" +python-gnupg = "~=0.4.5" + +[requires] +python_version = "3.8" diff --git a/Pipfile.lock b/Pipfile.lock new file mode 100644 index 0000000..b00c5cd --- /dev/null +++ b/Pipfile.lock @@ -0,0 +1,219 @@ +{ + "_meta": { + "hash": { + "sha256": "ac147eba23c329eeb6cf858f428692964ba3951ce309c64c121fa504b9198abc" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.8" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "passlib": { + "hashes": [ + "sha256:68c35c98a7968850e17f1b6892720764cc7eed0ef2b7cb3116a89a28e43fe177", + "sha256:8d666cef936198bc2ab47ee9b0410c94adf2ba798e5a84bf220be079ae7ab6a8" + ], + "index": "pypi", + "version": "==1.7.2" + }, + "python-gnupg": { + "hashes": [ + "sha256:3353e59949cd2c15efbf1fca45e347d8a22f4bed0d93e9b89b2657bda19cec05", + "sha256:c095a41f310ad7a4fd393406660ac9bd6c175ccaa0f072f9c18f33be8130a27a" + ], + "index": "pypi", + "version": "==0.4.5" + }, + "pyyaml": { + "hashes": [ + "sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97", + "sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76", + "sha256:4f4b913ca1a7319b33cfb1369e91e50354d6f07a135f3b901aca02aa95940bd2", + "sha256:69f00dca373f240f842b2931fb2c7e14ddbacd1397d57157a9b005a6a9942648", + "sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf", + "sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f", + "sha256:7739fc0fa8205b3ee8808aea45e968bc90082c10aef6ea95e855e10abf4a37b2", + "sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee", + "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d", + "sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c", + "sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a" + ], + "index": "pypi", + "version": "==5.3.1" + } + }, + "develop": { + "appdirs": { + "hashes": [ + "sha256:9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92", + "sha256:d8b24664561d0d34ddfaec54636d502d7cea6e29c3eaf68f3df6180863e2166e" + ], + "version": "==1.4.3" + }, + "astroid": { + "hashes": [ + "sha256:71ea07f44df9568a75d0f354c49143a4575d90645e9fead6dfb52c26a85ed13a", + "sha256:840947ebfa8b58f318d42301cf8c0a20fd794a33b61cc4638e28e9e61ba32f42" + ], + "version": "==2.3.3" + }, + "attrs": { + "hashes": [ + "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c", + "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72" + ], + "version": "==19.3.0" + }, + "black": { + "hashes": [ + "sha256:1b30e59be925fafc1ee4565e5e08abef6b03fe455102883820fe5ee2e4734e0b", + "sha256:c2edb73a08e9e0e6f65a0e6af18b059b8b1cdd5bef997d7a0b181df93dc81539" + ], + "index": "pypi", + "version": "==19.10b0" + }, + "click": { + "hashes": [ + "sha256:8a18b4ea89d8820c5d0c7da8a64b2c324b4dabb695804dbfea19b9be9d88c0cc", + "sha256:e345d143d80bf5ee7534056164e5e112ea5e22716bbb1ce727941f4c8b471b9a" + ], + "version": "==7.1.1" + }, + "isort": { + "hashes": [ + "sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1", + "sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd" + ], + "version": "==4.3.21" + }, + "lazy-object-proxy": { + "hashes": [ + "sha256:0c4b206227a8097f05c4dbdd323c50edf81f15db3b8dc064d08c62d37e1a504d", + "sha256:194d092e6f246b906e8f70884e620e459fc54db3259e60cf69a4d66c3fda3449", + "sha256:1be7e4c9f96948003609aa6c974ae59830a6baecc5376c25c92d7d697e684c08", + "sha256:4677f594e474c91da97f489fea5b7daa17b5517190899cf213697e48d3902f5a", + "sha256:48dab84ebd4831077b150572aec802f303117c8cc5c871e182447281ebf3ac50", + "sha256:5541cada25cd173702dbd99f8e22434105456314462326f06dba3e180f203dfd", + "sha256:59f79fef100b09564bc2df42ea2d8d21a64fdcda64979c0fa3db7bdaabaf6239", + "sha256:8d859b89baf8ef7f8bc6b00aa20316483d67f0b1cbf422f5b4dc56701c8f2ffb", + "sha256:9254f4358b9b541e3441b007a0ea0764b9d056afdeafc1a5569eee1cc6c1b9ea", + "sha256:9651375199045a358eb6741df3e02a651e0330be090b3bc79f6d0de31a80ec3e", + "sha256:97bb5884f6f1cdce0099f86b907aa41c970c3c672ac8b9c8352789e103cf3156", + "sha256:9b15f3f4c0f35727d3a0fba4b770b3c4ebbb1fa907dbcc046a1d2799f3edd142", + "sha256:a2238e9d1bb71a56cd710611a1614d1194dc10a175c1e08d75e1a7bcc250d442", + "sha256:a6ae12d08c0bf9909ce12385803a543bfe99b95fe01e752536a60af2b7797c62", + "sha256:ca0a928a3ddbc5725be2dd1cf895ec0a254798915fb3a36af0964a0a4149e3db", + "sha256:cb2c7c57005a6804ab66f106ceb8482da55f5314b7fcb06551db1edae4ad1531", + "sha256:d74bb8693bf9cf75ac3b47a54d716bbb1a92648d5f781fc799347cfc95952383", + "sha256:d945239a5639b3ff35b70a88c5f2f491913eb94871780ebfabb2568bd58afc5a", + "sha256:eba7011090323c1dadf18b3b689845fd96a61ba0a1dfbd7f24b921398affc357", + "sha256:efa1909120ce98bbb3777e8b6f92237f5d5c8ea6758efea36a473e1d38f7d3e4", + "sha256:f3900e8a5de27447acbf900b4750b0ddfd7ec1ea7fbaf11dfa911141bc522af0" + ], + "version": "==1.4.3" + }, + "mccabe": { + "hashes": [ + "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", + "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" + ], + "version": "==0.6.1" + }, + "pathspec": { + "hashes": [ + "sha256:7d91249d21749788d07a2d0f94147accd8f845507400749ea19c1ec9054a12b0", + "sha256:da45173eb3a6f2a5a487efba21f050af2b41948be6ab52b6a1e3ff22bb8b7061" + ], + "version": "==0.8.0" + }, + "pylint": { + "hashes": [ + "sha256:3db5468ad013380e987410a8d6956226963aed94ecb5f9d3a28acca6d9ac36cd", + "sha256:886e6afc935ea2590b462664b161ca9a5e40168ea99e5300935f6591ad467df4" + ], + "index": "pypi", + "version": "==2.4.4" + }, + "regex": { + "hashes": [ + "sha256:08119f707f0ebf2da60d2f24c2f39ca616277bb67ef6c92b72cbf90cbe3a556b", + "sha256:0ce9537396d8f556bcfc317c65b6a0705320701e5ce511f05fc04421ba05b8a8", + "sha256:1cbe0fa0b7f673400eb29e9ef41d4f53638f65f9a2143854de6b1ce2899185c3", + "sha256:2294f8b70e058a2553cd009df003a20802ef75b3c629506be20687df0908177e", + "sha256:23069d9c07e115537f37270d1d5faea3e0bdded8279081c4d4d607a2ad393683", + "sha256:24f4f4062eb16c5bbfff6a22312e8eab92c2c99c51a02e39b4eae54ce8255cd1", + "sha256:295badf61a51add2d428a46b8580309c520d8b26e769868b922750cf3ce67142", + "sha256:2a3bf8b48f8e37c3a40bb3f854bf0121c194e69a650b209628d951190b862de3", + "sha256:4385f12aa289d79419fede43f979e372f527892ac44a541b5446617e4406c468", + "sha256:5635cd1ed0a12b4c42cce18a8d2fb53ff13ff537f09de5fd791e97de27b6400e", + "sha256:5bfed051dbff32fd8945eccca70f5e22b55e4148d2a8a45141a3b053d6455ae3", + "sha256:7e1037073b1b7053ee74c3c6c0ada80f3501ec29d5f46e42669378eae6d4405a", + "sha256:90742c6ff121a9c5b261b9b215cb476eea97df98ea82037ec8ac95d1be7a034f", + "sha256:a58dd45cb865be0ce1d5ecc4cfc85cd8c6867bea66733623e54bd95131f473b6", + "sha256:c087bff162158536387c53647411db09b6ee3f9603c334c90943e97b1052a156", + "sha256:c162a21e0da33eb3d31a3ac17a51db5e634fc347f650d271f0305d96601dc15b", + "sha256:c9423a150d3a4fc0f3f2aae897a59919acd293f4cb397429b120a5fcd96ea3db", + "sha256:ccccdd84912875e34c5ad2d06e1989d890d43af6c2242c6fcfa51556997af6cd", + "sha256:e91ba11da11cf770f389e47c3f5c30473e6d85e06d7fd9dcba0017d2867aab4a", + "sha256:ea4adf02d23b437684cd388d557bf76e3afa72f7fed5bbc013482cc00c816948", + "sha256:fb95debbd1a824b2c4376932f2216cc186912e389bdb0e27147778cf6acb3f89" + ], + "version": "==2020.4.4" + }, + "six": { + "hashes": [ + "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a", + "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c" + ], + "version": "==1.14.0" + }, + "toml": { + "hashes": [ + "sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c", + "sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e" + ], + "version": "==0.10.0" + }, + "typed-ast": { + "hashes": [ + "sha256:0666aa36131496aed8f7be0410ff974562ab7eeac11ef351def9ea6fa28f6355", + "sha256:0c2c07682d61a629b68433afb159376e24e5b2fd4641d35424e462169c0a7919", + "sha256:249862707802d40f7f29f6e1aad8d84b5aa9e44552d2cc17384b209f091276aa", + "sha256:24995c843eb0ad11a4527b026b4dde3da70e1f2d8806c99b7b4a7cf491612652", + "sha256:269151951236b0f9a6f04015a9004084a5ab0d5f19b57de779f908621e7d8b75", + "sha256:4083861b0aa07990b619bd7ddc365eb7fa4b817e99cf5f8d9cf21a42780f6e01", + "sha256:498b0f36cc7054c1fead3d7fc59d2150f4d5c6c56ba7fb150c013fbc683a8d2d", + "sha256:4e3e5da80ccbebfff202a67bf900d081906c358ccc3d5e3c8aea42fdfdfd51c1", + "sha256:6daac9731f172c2a22ade6ed0c00197ee7cc1221aa84cfdf9c31defeb059a907", + "sha256:715ff2f2df46121071622063fc7543d9b1fd19ebfc4f5c8895af64a77a8c852c", + "sha256:73d785a950fc82dd2a25897d525d003f6378d1cb23ab305578394694202a58c3", + "sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b", + "sha256:8ce678dbaf790dbdb3eba24056d5364fb45944f33553dd5869b7580cdbb83614", + "sha256:aaee9905aee35ba5905cfb3c62f3e83b3bec7b39413f0a7f19be4e547ea01ebb", + "sha256:bcd3b13b56ea479b3650b82cabd6b5343a625b0ced5429e4ccad28a8973f301b", + "sha256:c9e348e02e4d2b4a8b2eedb48210430658df6951fa484e59de33ff773fbd4b41", + "sha256:d205b1b46085271b4e15f670058ce182bd1199e56b317bf2ec004b6a44f911f6", + "sha256:d43943ef777f9a1c42bf4e552ba23ac77a6351de620aa9acf64ad54933ad4d34", + "sha256:d5d33e9e7af3b34a40dc05f498939f0ebf187f07c385fd58d591c533ad8562fe", + "sha256:fc0fea399acb12edbf8a628ba8d2312f583bdbdb3335635db062fa98cf71fca4", + "sha256:fe460b922ec15dd205595c9b5b99e2f056fd98ae8f9f56b888e7a17dc2b757e7" + ], + "index": "pypi", + "version": "==1.4.1" + }, + "wrapt": { + "hashes": [ + "sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1" + ], + "version": "==1.11.2" + } + } +} diff --git a/README.md b/README.md index 241f29d..e5b63f0 100644 --- a/README.md +++ b/README.md @@ -41,9 +41,11 @@ ytt is a templating tool for yaml-files. It is required for some last moment configuration. Installation instructions can be found [here](https://k14s.io/#install-from-github-release). -- yq \ -yq is a commandline processor for yaml-files. Installation instructions can be -found [here](https://mikefarah.gitbook.io/yq/). +- Pipenv \ +Pipenv sets up a virtual python environment and installs required python packages +based on a lock-file, ensuring a deterministic Python environment. Instruction on +how Pipenv can be installed, can be found +[here](https://github.com/pypa/pipenv#installation) ### Infrastructure @@ -122,32 +124,35 @@ The configuration file contains secrets. Thus, to be able to share the configura e.g. with the CI-system, it is meant to be encrypted. The encryption is explained [here](./documentation/config-management.md). -The `./install.sh`-script will decrypt the file before templating, if it was -encrypted with `sops`. +The `gerrit-monitoring.py install`-command will decrypt the file before templating, +if it was encrypted with `sops`. ## Installation -Before beginning with the installation, ensure that the local helm repository is -up-to-date: +Before using the script, set up a python environment using `pipenv install`. -```sh -helm repo add loki https://grafana.github.io/loki/charts -helm repo update -``` +The installation will use the environment of the current shell. Thus, make sure +that the path for `ytt`, `kubectl`and `helm` are set. Also the `KUBECONFIG`-variable +has to be set to point to the kubeconfig of the target Kubernetes cluster. This project provides a script to quickly install the monitoring setup. To use it, run: ```sh -./install.sh \ +pipenv run python ./gerrit-monitoring.py \ + --config config.yaml \ + install \ [--output ./dist] \ [--dryrun] \ - config.yaml + [--update-repo] ``` -The command will use the given configuration to create the final -files in the directory given by `--output` (default `./dist`) and install/update -the Kubernetes resources and charts, if the `--dryrun` flag is not set. +The command will use the given configuration (`--config`/`-c`) to create the +final files in the directory given by `--output`/`-o` (default `./dist`) and +install/update the Kubernetes resources and charts, if the `--dryrun`/`-d` flag +is not set. If the `--update-repo`-flag is used, the helm repository will be updated +before installing the helm charts. This is for example required, if a chart version +was updated. ## Configure Promtail @@ -202,10 +207,11 @@ Remove the namespace: kubectl delete -f ./dist/namespace.yaml ``` -The `./uninstall.sh`-script will automatically remove the charts installed in -by the `./install.sh`-script from the configured namespace and delete the -namespace as well: +The `./gerrit-monitoring.py uninstall`-script will automatically remove the +charts installed in the configured namespace and delete the namespace as well: ```sh -./uninstall.sh config.yaml +pipenv run python ./gerrit-monitoring.py \ + --config config.yaml \ + uninstall ``` diff --git a/cfgmgr/__init__.py b/cfgmgr/__init__.py new file mode 100644 index 0000000..fa8ce9b --- /dev/null +++ b/cfgmgr/__init__.py @@ -0,0 +1,36 @@ +# Copyright (C) 2020 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import yaml + +from .default import DefaultConfigManager +from .sops import SopsConfigManager + + +def get_config_manager(config_path): + """Decide which ConfigManager is required to parse the config file. + + Arguments: + config_path {string} -- Path to config file + + Returns: + AbstractConfigManager -- ConfigManager that can parse the given config file + """ + with open(config_path, "r") as f: + config = yaml.load(f, Loader=yaml.SafeLoader) + + if "sops" in config.keys(): + return SopsConfigManager(config_path) + + return DefaultConfigManager(config_path) diff --git a/cfgmgr/abstract.py b/cfgmgr/abstract.py new file mode 100644 index 0000000..4e5cf61 --- /dev/null +++ b/cfgmgr/abstract.py @@ -0,0 +1,59 @@ +# Copyright (C) 2020 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import abc + +from passlib.apache import HtpasswdFile + + +class AbstractConfigManager(abc.ABC): + """Provide abstract base class to implement config + managers that can e.g. handle different encryption methods. + """ + + def __init__(self, config_path): + self.config_path = config_path + + self.requires_htpasswd = [ + ["loki"], + ["prometheus", "server"], + ] + + def get_config(self): + """Parse the configuration and return it as a dictionary. + + Returns: + dict -- Dictionary containing the unencrypted configuration as parsed + from the file + """ + + config = self._parse() + for component in self.requires_htpasswd: + section = config + for i in component: + section = section[i] + section["htpasswd"] = self._create_htpasswd_entry( + section["username"], section["password"] + ) + return config + + @staticmethod + def _create_htpasswd_entry(username, password): + htpasswd = HtpasswdFile() + htpasswd.set_password(username, password) + return htpasswd.to_string()[:-1] + + @abc.abstractmethod + def _parse(self): + pass diff --git a/cfgmgr/default.py b/cfgmgr/default.py new file mode 100644 index 0000000..9ebecea --- /dev/null +++ b/cfgmgr/default.py @@ -0,0 +1,27 @@ +# Copyright (C) 2020 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import yaml + +from .abstract import AbstractConfigManager + + +class DefaultConfigManager(AbstractConfigManager): + """Config manager for unencrypted files.""" + + def _parse(self): + with open(self.config_path, "r") as f: + config = yaml.load(f, Loader=yaml.SafeLoader) + + return config diff --git a/cfgmgr/sops.py b/cfgmgr/sops.py new file mode 100644 index 0000000..8129f22 --- /dev/null +++ b/cfgmgr/sops.py @@ -0,0 +1,27 @@ +# Copyright (C) 2020 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import subprocess +import yaml + +from .abstract import AbstractConfigManager + + +class SopsConfigManager(AbstractConfigManager): + """Config manager for config file encrypted with sops.""" + + def _parse(self): + command = ["sops", "-d", self.config_path] + output = subprocess.check_output(command) + return yaml.load(output, Loader=yaml.SafeLoader) diff --git a/documentation/config-management.md b/documentation/config-management.md index 4a6c5e8..537c240 100644 --- a/documentation/config-management.md +++ b/documentation/config-management.md @@ -59,19 +59,19 @@ sops \ `$EMAIL` refers to the email used during the creation of the GPG key. -Alternatively, the `./encrypt.sh`-script can be used to encrypt the file: +Alternatively, the `gerrit-monitoring.py encrypt`-script can be used to encrypt +the file: ```sh -./encrypt.sh \ - [--email $EMAIL] \ - [--fingerprint $FINGERPRINT] \ - $FILE_TO_ENCODE +pipenv run python ./gerrit-monitoring.py \ + --config config.yaml \ + encrypt \ + --pgp "abcde1234" ``` -The gpg-key used to encrypt the file can be selected by directly giving the key's -fingerprint using the `--fingerprint` option or giving the email used to identify -the key using the `--email` option. The `--fingerprint` option will have preference. -At least one of these options has to be set. +The gpg-key used to encrypt the file can be selected by giving the fingerprint, +key ID or part of the unique ID to the `--pgp`-argument. This identifier has to +be unique among the keys in the GPG keystore. ## Decrypt file diff --git a/encrypt.sh b/encrypt.sh deleted file mode 100755 index 1ae94e4..0000000 --- a/encrypt.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -e - -# Copyright (C) 2020 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -usage() { - me=`basename "$0"` - echo >&2 "Usage: $me [--email EMAIL] [--fingerprint FINGERPRINT] CONFIG" - exit 1 -} - -while test $# -gt 0 ; do - case "$1" in - --email) - shift - EMAIL=$1 - shift - ;; - - --fingerprint) - shift - FINGERPRINT=$1 - shift - ;; - - *) - break - esac -done - -CONFIG=$1 -test -z "$CONFIG" && usage - -if test -z $FINGERPRINT; then - test -z $EMAIL && usage - FINGERPRINT=$(gpg --fingerprint "$EMAIL" | \ - grep pub -A 1 | \ - grep -v pub | \ - sed s/\ //g) -fi - -sops \ - --encrypt \ - --in-place \ - --encrypted-regex '(password|htpasswd|cert|key|apiUrl|caCert|secret|accessToken)$' \ - --pgp $FINGERPRINT \ - $CONFIG diff --git a/gerrit_monitoring.py b/gerrit_monitoring.py new file mode 100644 index 0000000..f8a98a1 --- /dev/null +++ b/gerrit_monitoring.py @@ -0,0 +1,103 @@ +# Copyright (C) 2020 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os.path + +from cfgmgr import get_config_manager +from subcommands import encrypt, install, uninstall + + +def _run_encrypt(args): + encrypt(args.pgp_identifier, os.path.abspath(args.config)) + + +def _run_install(args): + install( + get_config_manager(os.path.abspath(args.config)), + os.path.abspath(args.output_dir), + args.dryrun, + args.update_repo, + ) + + +def _run_uninstall(args): + uninstall(get_config_manager(args.config)) + + +def main(): + """Argument parser for the gerrit monitoring installer.""" + parser = argparse.ArgumentParser() + + parser.add_argument( + "-c", + "--config", + help="Path to configuration file.", + dest="config", + action="store", + required=True, + ) + + subparsers = parser.add_subparsers() + + parser_install = subparsers.add_parser("install", help="Install Gerrit monitoring") + parser_install.set_defaults(func=_run_install) + + parser_install.add_argument( + "-o", + "--output", + help="Output directory for generated files.", + dest="output_dir", + action="store", + default="./dist", + ) + + parser_install.add_argument( + "-d", + "--dryrun", + help="Only generate files, but do not install them.", + dest="dryrun", + action="store_true", + ) + + parser.add_argument( + "--update-repo", + help="Update the helm repositories.", + dest="update_repo", + action="store_true", + ) + + parser_uninstall = subparsers.add_parser( + "uninstall", help="Uninstall Gerrit monitoring" + ) + parser_uninstall.set_defaults(func=_run_uninstall) + + parser_encrypt = subparsers.add_parser("encrypt", help="Encrypt config") + parser_encrypt.set_defaults(func=_run_encrypt) + + parser_encrypt.add_argument( + "-p", + "--pgp", + help="PGP fingerpint or associated email.", + dest="pgp_identifier", + action="store", + required=True, + ) + + args = parser.parse_args() + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/install.sh b/install.sh deleted file mode 100755 index fc1bd3b..0000000 --- a/install.sh +++ /dev/null @@ -1,158 +0,0 @@ -#!/bin/bash -e - -# Copyright (C) 2020 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -usage() { - me=`basename "$0"` - echo >&2 "Usage: $me [--output OUTPUT] [--dryrun] CONFIG" - exit 1 -} - -while test $# -gt 0 ; do - case "$1" in - --output) - shift - OUTPUT=$1 - shift - ;; - - --dryrun) - DRYRUN="true" - shift - ;; - - *) - break - esac -done - -test -z "$OUTPUT" && OUTPUT="$(dirname $0)/dist" - -CONFIG=$1 -test -z "$CONFIG" && usage - -NAMESPACE=$(yq r $CONFIG namespace) -TMP_CONFIG=$OUTPUT/$(basename $CONFIG) - -function updateOrInstall() { - if test -n "$(helm ls -n $NAMESPACE --short | grep $1)"; then - echo "upgrade" - else - echo "install" - fi -} - -function addHtpasswdEntryUnencrypted() { - local COMPONENT=$1 - - local HTPASSWD=$(htpasswd -nb \ - $(yq r $TMP_CONFIG $COMPONENT.username) \ - $(yq r $TMP_CONFIG $COMPONENT.password)) - - yq w -i $TMP_CONFIG $COMPONENT.htpasswd $HTPASSWD -} - -function addHtpasswdEntryEncrypted() { - local COMPONENT=$1 - - local HTPASSWD=$(htpasswd -nb \ - $(sops -d --extract "$COMPONENT['username']" $TMP_CONFIG) \ - $(sops -d --extract "$COMPONENT['password']" $TMP_CONFIG)) - - sops --set "$COMPONENT['htpasswd'] \"$HTPASSWD\"" $TMP_CONFIG -} - -function addDashboards() { - for dashboard in dashboards/*; do - local DASHBOARD_NAME="${dashboard%.json}" - local DASHBOARD_NAME="${DASHBOARD_NAME#"dashboards/"}" - - kubectl create configmap $DASHBOARD_NAME \ - --from-file=$dashboard \ - --dry-run=true \ - --namespace=$NAMESPACE \ - -o yaml > $OUTPUT/dashboards/$DASHBOARD_NAME.dashboard.yaml - - yq w -i $OUTPUT/dashboards/$DASHBOARD_NAME.dashboard.yaml \ - metadata.labels.grafana_dashboard $DASHBOARD_NAME - done -} - - -function runYtt() { - ytt \ - -f charts/namespace.yaml \ - -f charts/prometheus/ \ - -f charts/loki/ \ - -f charts/grafana/ \ - -f promtail/ \ - --output-directory $OUTPUT \ - --ignore-unknown-comments \ - -f $1 -} - -mkdir -p $OUTPUT -cp $CONFIG $TMP_CONFIG - -# Fill in templates -if test -z "$(grep -o '^sops:$' $TMP_CONFIG)"; then - addHtpasswdEntryUnencrypted loki - addHtpasswdEntryUnencrypted prometheus.server - echo -e "#@data/values\n---\n$(cat $TMP_CONFIG)" | runYtt - -else - addHtpasswdEntryEncrypted "['loki']" $TMP_CONFIG - addHtpasswdEntryEncrypted "['prometheus']['server']" $TMP_CONFIG - echo -e "#@data/values\n---\n$(sops -d $TMP_CONFIG)" | runYtt - -fi - -# Create configmaps with dashboards -mkdir -p $OUTPUT/dashboards -addDashboards - -test -n "$DRYRUN" && exit 0 - -# Install loose components -kubectl apply -f $OUTPUT/namespace.yaml -kubectl apply -f $OUTPUT/configuration -kubectl apply -f $OUTPUT/dashboards -kubectl apply -f $OUTPUT/storage - -# Add Loki helm repository -helm repo add loki https://grafana.github.io/loki/charts -helm repo update - -# Install Prometheus -PROMETHEUS_CHART_NAME=prometheus-$NAMESPACE -helm $(updateOrInstall $PROMETHEUS_CHART_NAME) $PROMETHEUS_CHART_NAME \ - stable/prometheus \ - --version $(cat ./charts/prometheus/VERSION) \ - --values $OUTPUT/prometheus.yaml \ - --namespace $NAMESPACE - -# Install Loki -LOKI_CHART_NAME=loki-$NAMESPACE -helm $(updateOrInstall $LOKI_CHART_NAME) $LOKI_CHART_NAME \ - loki/loki \ - --version $(cat ./charts/loki/VERSION) \ - --values $OUTPUT/loki.yaml \ - --namespace $NAMESPACE - -# Install Grafana -GRAFANA_CHART_NAME=grafana-$NAMESPACE -helm $(updateOrInstall $GRAFANA_CHART_NAME) $GRAFANA_CHART_NAME \ - stable/grafana \ - --version $(cat ./charts/grafana/VERSION) \ - --values $OUTPUT/grafana.yaml \ - --namespace $NAMESPACE diff --git a/subcommands/__init__.py b/subcommands/__init__.py new file mode 100644 index 0000000..9509bd5 --- /dev/null +++ b/subcommands/__init__.py @@ -0,0 +1,17 @@ +# Copyright (C) 2020 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .encrypt import encrypt +from .install import install +from .uninstall import uninstall diff --git a/subcommands/_globals.py b/subcommands/_globals.py new file mode 100644 index 0000000..43f5ffe --- /dev/null +++ b/subcommands/_globals.py @@ -0,0 +1,19 @@ +# Copyright (C) 2020 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +HELM_CHARTS = { + "grafana": "stable/grafana", + "loki": "loki/loki", + "prometheus": "stable/prometheus", +} diff --git a/subcommands/encrypt.py b/subcommands/encrypt.py new file mode 100644 index 0000000..fcc6cb6 --- /dev/null +++ b/subcommands/encrypt.py @@ -0,0 +1,71 @@ +# Copyright (C) 2020 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import subprocess + +import gnupg + + +ENCRYPTED_KEYS = [ + "accessToken", + "apiUrl", + "caCert", + "cert", + "htpasswd", + "key", + "password", + "secret", +] + + +def encrypt(pgp_identifier, config_path): + """Encrypt the config file using sops and a PGP key. + + Arguments: + pgp_identifier {string} -- A unique identifier of the PGP key to be used. + This can be the fingerprint, keyid or part of the uid (e.g. the email + address) + config_path {string} -- The path to the config file to be encrypted + + Raises: + ValueError: Error, if no (unique) PGP key could be found + """ + gpg = gnupg.GPG() + gpg_keys = gpg.list_keys() + selected_keys = list( + filter( + lambda k: pgp_identifier in k["fingerprint"] + or pgp_identifier in k["keyid"] + or len([v for v in k["uids"] if pgp_identifier in v]) > 0, + gpg_keys, + ) + ) + + if not selected_keys: + raise ValueError("PGP key not found.") + + if len(selected_keys) > 1: + raise ValueError("Identifier of PGP not unique.") + + command = [ + "sops", + "--encrypt", + "--in-place", + "--encrypted-regex", + f"({'|'.join(ENCRYPTED_KEYS)})", + "--pgp", + selected_keys[0]["fingerprint"], + config_path, + ] + subprocess.check_output(command) diff --git a/subcommands/install.py b/subcommands/install.py new file mode 100644 index 0000000..c2db390 --- /dev/null +++ b/subcommands/install.py @@ -0,0 +1,175 @@ +# Copyright (C) 2020 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os.path +import subprocess +import yaml + +from ._globals import HELM_CHARTS + + +TEMPLATES = [ + "charts/namespace.yaml", + "charts/prometheus", + "charts/loki", + "charts/grafana", + "promtail", +] + +HELM_REPOS = { + "stable": "https://kubernetes-charts.storage.googleapis.com", + "loki": "https://grafana.github.io/loki/charts", +} + +LOOSE_RESOURCES = [ + "namespace.yaml", + "configuration", + "dashboards", + "storage", +] + + +def _create_dashboard_configmaps(output_dir, namespace): + dashboards_dir = os.path.abspath("./dashboards") + + output_dir = os.path.join(output_dir, "dashboards") + if not os.path.exists(output_dir): + os.mkdir(output_dir) + + for dashboard in os.listdir(dashboards_dir): + dashboard_path = os.path.join(dashboards_dir, dashboard) + dashboard_name = os.path.splitext(dashboard)[0] + output_file = f"{output_dir}/{dashboard_name}.dashboard.yaml" + command = ( + f"kubectl create configmap {dashboard_name} -o yaml " + f"--from-file={dashboard_path} --dry-run=client --namespace={namespace} " + f"> {output_file}" + ) + + try: + subprocess.check_output(command, shell=True) + except subprocess.CalledProcessError as err: + print(err.output) + + with open(output_file, "r") as f: + dashboard_cm = yaml.load(f, Loader=yaml.SafeLoader) + dashboard_cm["metadata"]["labels"] = dict() + dashboard_cm["metadata"]["labels"]["grafana_dashboard"] = dashboard_name + + with open(output_file, "w") as f: + yaml.dump(dashboard_cm, f) + + +def _run_ytt(config, output_dir): + config_string = "#@data/values\n---\n" + config_string += yaml.dump(config) + + command = [ + "ytt", + ] + + for template in TEMPLATES: + command += ["-f", template] + + command += [ + "--output-directory", + output_dir, + "--ignore-unknown-comments", + "-f", + "-", + ] + + try: + # pylint: disable=E1123 + print(subprocess.check_output(command, input=config_string, text=True)) + except subprocess.CalledProcessError as err: + print(err.output) + + +def _update_helm_repos(): + for repo, url in HELM_REPOS.items(): + command = ["helm", "repo", "add", repo, url] + try: + subprocess.check_output(" ".join(command), shell=True) + except subprocess.CalledProcessError as err: + print(err.output) + try: + print(subprocess.check_output(["helm", "repo", "update"]).decode("utf-8")) + except subprocess.CalledProcessError as err: + print(err.output) + + +def _deploy_loose_resources(output_dir): + for resource in LOOSE_RESOURCES: + command = [ + "kubectl", + "apply", + "-f", + f"{output_dir}/{resource}", + ] + print(subprocess.check_output(command).decode("utf-8")) + + +def _get_installed_charts_in_namespace(namespace): + command = ["helm", "ls", "-n", namespace, "--short"] + return subprocess.check_output(command).decode("utf-8").split("\n") + + +def _install_or_update_charts(output_dir, namespace): + installed_charts = _get_installed_charts_in_namespace(namespace) + charts_path = os.path.abspath("./charts") + for chart, repo in HELM_CHARTS.items(): + chart_name = chart + "-" + namespace + with open(f"{charts_path}/{chart}/VERSION", "r") as f: + chart_version = f.readlines()[0].strip() + command = ["helm"] + command.append("upgrade" if chart_name in installed_charts else "install") + command += [ + chart_name, + repo, + "--version", + chart_version, + "--values", + f"{output_dir}/{chart}.yaml", + "--namespace", + namespace, + ] + try: + print(subprocess.check_output(command).decode("utf-8")) + except subprocess.CalledProcessError as err: + print(err.output) + + +def install(config_manager, output_dir, dryrun, update_repo): + """Create the final configuration for the helm charts and Kubernetes resources + and install them to Kubernetes, if not run in --dryrun mode. + + Arguments: + config_manager {AbstractConfigManager} -- ConfigManager that contains the + configuration of the monitoring setup to be uninstalled. + output_dir {string} -- Path to the directory where the generated files + should be safed in + dryrun {boolean} -- Whether the installation will be run in dryrun mode + update_repo {boolean} -- Whether to update the helm repositories locally + """ + _run_ytt(config_manager.get_config(), output_dir) + + namespace = config_manager.get_config()["namespace"] + _create_dashboard_configmaps(output_dir, namespace) + + if not dryrun: + if update_repo: + _update_helm_repos() + _deploy_loose_resources(output_dir) + _install_or_update_charts(output_dir, namespace) diff --git a/subcommands/uninstall.py b/subcommands/uninstall.py new file mode 100644 index 0000000..386e5bf --- /dev/null +++ b/subcommands/uninstall.py @@ -0,0 +1,59 @@ +# Copyright (C) 2020 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import subprocess + +from ._globals import HELM_CHARTS + + +def _get_yn_response(message): + while True: + response = input(message) + if response == "y": + return True + + if response == "n": + return False + + print("Unknown input.") + + +def _remove_helm_deployment(chart, namespace): + deployment_name = f"{chart}-{namespace}" + if _get_yn_response( + f"This will remove the deployment {deployment_name}. Continue (y/n)? " + ): + command = ["helm", "uninstall", deployment_name, "-n", namespace] + subprocess.check_output(command) + + +def _delete_namespace(namespace): + if _get_yn_response( + f"This will remove the namespace {namespace}. Continue (y/n)? " + ): + command = ["kubectl", "delete", "ns", namespace] + subprocess.check_output(command) + + +def uninstall(config_manager): + """Uninstall the monitoring setup. + + Arguments: + config_manager {AbstractConfigManager} -- ConfigManager that contains the + configuration of the monitoring setup to be uninstalled. + """ + namespace = config_manager.get_config()["namespace"] + for chart in HELM_CHARTS: + _remove_helm_deployment(chart, namespace) + _delete_namespace(namespace) diff --git a/uninstall.sh b/uninstall.sh deleted file mode 100755 index de5226a..0000000 --- a/uninstall.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -e - -# Copyright (C) 2020 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -usage() { - me=`basename "$0"` - echo >&2 "Usage: $me CONFIG" - exit 1 -} - -test -z "$1" && usage -CONFIG=$1 - -NAMESPACE=$(yq r $CONFIG namespace) - -function removeHelmDeployment() { - read -p "This will remove the deployment $1-$NAMESPACE. Continue (y/n)? " response - if [[ "$response" == "y" ]]; then - helm uninstall $1-$NAMESPACE -n $NAMESPACE || true - fi -} - -removeHelmDeployment grafana -removeHelmDeployment loki -removeHelmDeployment prometheus - -read -p "This will remove the namespace $NAMESPACE. Continue (y/n)? " response -if [[ "$response" == "y" ]]; then - kubectl delete ns $NAMESPACE -fi