├── .github ├── ISSUE_TEMPLATE │ ├── bug-report.md │ ├── contribution-request.md │ ├── feature-request.md │ └── question.md └── workflows │ └── ci.yml ├── .gitignore ├── .whitesource ├── CLA.md ├── LICENSE ├── README.md ├── ToDo.md ├── requirements.txt ├── setup.py └── ws_bulk_report_generator ├── __init__.py ├── _version.py └── bulk_report_generator.py /.github/ISSUE_TEMPLATE/bug-report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug Report 3 | about: Create a report to help us improve 4 | title: "[BUG] [ws-bulk-report-generator] Issue Short Description" 5 | labels: bug 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Bug Description** 11 | A clear and concise description of what the bug is. 12 | 13 | **Steps to Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected Behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Environment Details** 27 | - OS: [e.g. Ubuntu 18.04] 28 | - Browser [e.g. chrome, safari] 29 | - Version [e.g. 22] 30 | 31 | **Additional Context** 32 | Add any other context about the problem here. 33 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/contribution-request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Contribution Request 3 | about: Discuss potential changes you wish to contribute 4 | title: "[CR] [ws-bulk-report-generator] Contribution Request Topic" 5 | labels: enhancement 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Tool:** ws-tool-name 11 | 12 | **Planned Changes:** 13 | Describe the changes you wish to contribute, to initiate a discussion with WhiteSource-PS team. 14 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature-request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature Request 3 | about: Suggest an idea for this project 4 | title: "[FR] [ws-bulk-report-generator] Feature Short Description" 5 | labels: feature request 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/question.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Question 3 | about: General question/how-to 4 | title: "[Question] [ws-bulk-report-generator] Question Topic" 5 | labels: question 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Question** 11 | Ask your question here. Please be as specific as possible. 12 | 13 | **Environment Details** 14 | - OS: [e.g. Ubuntu 18.04] 15 | - Browser [e.g. chrome, safari] 16 | - Version [e.g. 22] 17 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | on: 3 | push: 4 | branches: 5 | - '**' 6 | - '!ws-iac-scan-results/**' 7 | - '!whitesource-remediate/master-all**' 8 | - '!whitesource/migrate-configuration**' 9 | tags: 10 | - '*' 11 | jobs: 12 | build-and-publish: 13 | runs-on: ubuntu-latest 14 | strategy: 15 | matrix: 16 | python-version: [3.8, 3.9] 17 | steps: 18 | - name: Set Environment Variables 19 | run: | 20 | TOOL_NAME=$(echo '${{ github.repository }}' |awk -F '/' '{gsub(/-/, "_", $0) ;print $NF}') 21 | echo "TOOL_NAME=$TOOL_NAME" >> $GITHUB_ENV 22 | echo "RELEASE=false" >> $GITHUB_ENV 23 | echo "VERSION=0.0.0.ci0" >> $GITHUB_ENV 24 | echo "TOOL_DIR=$TOOL_NAME" >> $GITHUB_ENV 25 | if [[ $GITHUB_REF == refs/tags/v* ]]; then 26 | echo "VERSION=$(echo ${{github.ref}} | sed -r 's/^[\/a-zA-z-]+//')" >> $GITHUB_ENV 27 | if [[ $VERSION != *@(a|b)* ]]; then 28 | echo "RELEASE=true" >> $GITHUB_ENV 29 | fi 30 | fi 31 | - uses: actions/checkout@v3 32 | - name: Set release version 33 | run: | 34 | sed -E -i "s/^__version__ = \"[a-z0-9\.]+\"/__version__ = \"$VERSION\"/g" ${{ env.TOOL_DIR }}/_version.py 35 | # - uses: UnicornGlobal/trufflehog-actions-scan@master 36 | # with: 37 | # branch: ${{ github.head_ref }} 38 | - name: Set up Python ${{ matrix.python-version }} 39 | uses: actions/setup-python@v4 40 | with: 41 | python-version: ${{ matrix.python-version }} 42 | - name: Change to last ws-sdk version on odd days (Mon, Wed, Fri, Sun) 43 | run: | 44 | sdk_c_ver=$(grep "ws-sdk" requirements.txt | awk -F '=' '{print $NF}') 45 | sdk_t_ver=$(curl -sL https://pypi.org/pypi/ws-sdk/json | jq -r '.releases| keys[]' | sort -t. -k 1,1n -k 2,2n -k 3,3n | tail -n1) 46 | 47 | if (( $(date +"%u") % 2 )) ; then 48 | echo "Odd day - Replacing current ws-sdk version: ${sdk_c_ver} with latest release: ${sdk_t_ver}" 49 | sed -E -i "s/^ws-sdk.+/ws-sdk==${sdk_t_ver}/g" requirements.txt 50 | else 51 | echo "Even day" 52 | fi 53 | - name: Install dependencies 54 | run: | 55 | python -m pip install --upgrade pip 56 | pip install wheel pytest flake8 57 | pip install -r requirements.txt 58 | - name: Lint with flake8 59 | run: | 60 | # stop the build if there are Python syntax errors or undefined names 61 | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --ignore=E501,F841 62 | # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide 63 | flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics 64 | - name: Create Wheel Package 65 | run: python setup.py bdist_wheel 66 | - name: Install Wheel package 67 | run: pip install dist/${{ env.TOOL_DIR }}-${{ env.VERSION }}-py3-none-any.whl[xlsx] 68 | - name: Full Test (due diligence seperate files) 69 | run: ${{ env.TOOL_NAME }} -u ${{ secrets.WS_USER_KEY }} -k ${{ secrets.WS_ORG_TOKEN }} -r due_diligence 70 | - name: Full Test (vulnerability report filtered with single CVE on product level unifed json) 71 | run: ${{ env.TOOL_NAME }} -u ${{ secrets.WS_USER_KEY }} -k ${{ secrets.WS_ORG_TOKEN }} -r vulnerability -t unified_json -x vulnerability_names=CVE-2021-45046 72 | - name: Full Test (inventory report product level unifed xlsx) 73 | run: ${{ env.TOOL_NAME }} -u ${{ secrets.WS_USER_KEY }} -k ${{ secrets.WS_ORG_TOKEN }} -s product -r inventory -t unified_xlsx 74 | - name: Full Test (inventory asyncr report) 75 | run: ${{ env.TOOL_NAME }} -u ${{ secrets.WS_USER_KEY }} -k ${{ secrets.WS_ORG_TOKEN }} -i ${{ secrets.IBM_PRODUCT_TOKEN }} -r inventory -t binary -c True 76 | - name: Create Release 77 | if: startsWith(github.ref, 'refs/tags/v') 78 | uses: ncipollo/release-action@v1 79 | with: 80 | artifacts: dist/${{ env.TOOL_DIR }}-${{ env.VERSION }}-py3-none-any.whl 81 | allowUpdates: true 82 | token: ${{ secrets.GITHUB_TOKEN }} 83 | prerelease: env.RELEASE != 'true' 84 | - name: Publish to Test PyPI 85 | if: startsWith(github.ref, 'refs/tags/test-v') 86 | uses: pypa/gh-action-pypi-publish@release/v1 87 | with: 88 | skip_existing: true 89 | user: __token__ 90 | password: ${{ secrets.TEST_PYPI_API_TOKEN }} 91 | repository_url: https://test.pypi.org/legacy/ 92 | - name: Publish to PyPI 93 | if: startsWith(github.ref, 'refs/tags/v') 94 | uses: pypa/gh-action-pypi-publish@release/v1 95 | with: 96 | skip_existing: true 97 | user: __token__ 98 | password: ${{ secrets.PYPI_API_TOKEN }} 99 | 100 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Excluded IDE Directories 2 | .idea/ 3 | .vs/ 4 | .vscode/ 5 | .ws/ 6 | 7 | # Excluded Local/User Directories 8 | _archive/ 9 | _misc/ 10 | local/ 11 | log/ 12 | target/ 13 | 14 | # Excluded Files - Extensions 15 | *.iml 16 | *.png 17 | *.tar.gz 18 | *.url 19 | 20 | # Excluded Files - Naming Convention 21 | local-env.* 22 | /ws_bulk_reports_generator/conf/ 23 | /venv/ 24 | /ws_bulk_reports_generator/reports/ 25 | /dist/ 26 | /ws_bulk_report_generator.egg-info/ 27 | /reports/ 28 | /build/ 29 | /ws_bulk_report_generator/reports/ 30 | /build/ 31 | /ws_bulk_report_generator/bulk_report_generator.log 32 | ws_bulk_report_generator/__pycache__/ -------------------------------------------------------------------------------- /.whitesource: -------------------------------------------------------------------------------- 1 | { 2 | "settingsInheritedFrom": "whitesource-ps/whitesource-config@main" 3 | } -------------------------------------------------------------------------------- /CLA.md: -------------------------------------------------------------------------------- 1 | # WhiteSource Software Contributor License Agreement 2 | Thank you for your interest in contributing to the open source software projects (the "Projects") made available by WhiteSource Software or its affiliates (the “Company”) 3 | By Submitting Your Contribution (as these terms are defined below), You agree to the terms and conditions of this Contributor License Agreement (the "Agreement"). In case You are an entity, the individual submitting the Contribution for the entity confirms that they have the proper authority to legally bind the entity to this Agreement. They also confirm they agree, on behalf of that entity, to be contractually bound by this Agreement. 4 | If You have any questions regarding this Agreement, please contact support@whitesourcesoftware.com. 5 | ### 1. Definitions 6 | “Contribution” means any source code, bug fixes, configuration changes, tools, documentation, data, materials, feedback, information or other works of authorship that You Submit or have Submitted, in any form and in any manner, to the Company in connection with any Project(s), excluding communication that is conspicuously marked or otherwise designated in writing by You as "Not a Contribution." 7 | "You" means any individual or legal entity that Submits Contributions to the Project. 8 | "Source Code" means human-readable software code (including any comments therein), which is the preferred way for modifying a software. 9 | "Submit" means to upload a Contribution to the Project's Source Code management system, to distribute a Contribution under the Open Source License or to otherwise make a Contribution available to the Company – e.g., by email, electronic communication or by any other way. 10 | "Open Source License" means the [please complete] license or any other open source license that the Company may designate to the Project from time to time. 11 | “Patents” means all patent claims that You own or control that would be infringed by any manner of using the Contribution in accordance with this Agreement. 12 | ### 2. Representations 13 | - 2.1. You represent and warrants that - 14 | - 2.1.1. Your Contributions is an original work that You created. 15 | - 2.1.2. Your Contribution is free from any third party rights or claims (including copyright, patents, trademarks, trade secrets and other intellectual property rights) – unless You included the details of any such rights clearly and conspicuously in Your Contribution by including the phrase “Submission containing materials of a third party:” followed by the names of the third party and any licenses or other restrictions of which You are aware. 16 | - 2.1.3. You are legally entitled to Submit the Contribution to the Company and grant the Company the rights and licenses mentioned in section 3 below. 17 | - 2.1.4. If Your Contribution was created in the course of Your employment or under any other agreement according to which not You but rather a third party is the copyright owner in Your Contribution, including (but not limited to) as a work made for hire, then Your employer – or, as the case may be, the copyright owner - has given You all licenses and permissions required to enable You to lawfully Submit the Contributions to the Company under this Agreement. 18 | ### 3. License Grant 19 | - 3.1. Copyright License 20 | Subject to the terms and conditions of this Agreement, You hereby grant the Company a worldwide, royalty-free, non-exclusive, transferrable, sublicensable, assignable, perpetual and irrevocable license to reproduce, prepare derivative works of, display and perform publicly, sublicense, make available to the public and distribute Your Contribution and any work derived from it. 21 | - 3.2. Patent License 22 | You grant the Company a free-of-charge and royalty free, non-exclusive, worldwide, perpetual, irrevocable, transferrable, assignable license under the Patents, with the right to sublicense through multiple tiers, to use, make, have made, sell, have sold, develop, manufacture and produce, have developed, manufactured and produced, in connection with the Project. 23 | - 3.3. Moral Rights 24 | Moral Rights remain unaffected to the extent they are recognized and not waivable by applicable law. To the extent permitted under applicable law, You hereby waive, and agree not to assert, all of Your “moral rights” in or relating to Your Contributions for the Company’s benefit. If not waivable, You hereby agree not to bring a claim against the Company in connection thereto. Notwithstanding, You may add your name to the attribution mechanism customary used in the materials You Contribute to, such as the header of the Source Code files of Your Contribution, and the Company will make a good-faith effort, but will not be obligated, to include Your name in the copyright notices in the headers of every file that contains Your Contribution. 25 | - 3.4. Licensing the Project and Your Contribution 26 | The Company will license the Project under the Open Source License. However, as long as the Company makes the Project available under an Open Source License, the Company may also license the Project, including Your Contribution, under commercial licenses. 27 | ### 4. Disclaimer 28 | You are not expected to provide support for Your Contributions. You may provide support for free, for a fee, or not at all. The Contribution is provided "as is". More particularly, all express or implied warranties including, without limitation, any implied warranty of satisfactory quality, fitness for a particular purpose and non-infringement are expressly disclaimed by You to the Company and by the Company to You. To the extent that any such warranties cannot be disclaimed, such warranty is limited in duration and extent to the minimum period and extent permitted by law. 29 | ### 5. No Obligation to Use 30 | The Company is under no obligation to use Your Contribution and it retains the sole discretion to decide whether or not to use Your Contribution or incorporate it in the Project. 31 | ### 6. Miscellaneous 32 | - 6.1. Governing Law and Jurisdiction. This Agreement and all disputes, claims, actions, suits or other proceedings arising out of this Agreement or relating in any way to it shall be governed by the laws of Israel excluding its conflict of law provisions. You and the Company each consent to the sole and exclusive personal jurisdiction and venue for any legal proceedings in connection with this Agreement, in the competent courts in the District of Tel-Aviv-Jaffa, Israel, and waive any objections related thereto. 33 | - 6.2. Entire Agreement. This Agreement sets out the entire agreement between You and the Company regarding Your Contribution and overrides all other agreements or understandings. 34 | - 6.3. Severability. If any provision of this Agreement is found void or unenforceable, such provision will be replaced to the extent possible with a provision that comes closest to the meaning of the original provision and that is enforceable. 35 | - 6.4. Status. This Agreement does not establish a partnership, joint venture, agency or employment relationship between the You and the Company. 36 | - 6.5. Notification. You agree to notify the Company of any facts or circumstances of which you become aware that would make Your representations in this Agreement inaccurate in any respect. 37 | 38 | 39 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Logo](https://resources.mend.io/mend-sig/logo/mend-dark-logo-horizontal.png)](https://www.mend.io/) 2 | [![License](https://img.shields.io/badge/License-Apache%202.0-yellowgreen.svg)](https://opensource.org/licenses/Apache-2.0) 3 | [![CI](https://github.com/whitesource-ps/ws-bulk-report-generator/actions/workflows/ci.yml/badge.svg)](https://github.com/whitesource-ps/ws-bulk-report-generator/actions/workflows/ci.yml) 4 | [![PyPI](https://img.shields.io/pypi/v/ws-bulk-report-generator?style=plastic)](https://pypi.org/project/ws-bulk-report-generator/) 5 | # [Mend Bulk Report Generator](https://github.com/whitesource-ps/ws-bulk-report-generator) 6 | CLI Tool to generate reports on multiple products or projects. 7 | * The tool allows including and excluding scopes by stating their tokens. 8 | * Report scope (`-s, --ReportScope`) determines whether reports run on projects or products. 9 | * If included scopes (via `-i, --includedTokens`) are not specified, the tool runs reports on **all** scopes. 10 | * Report data exported by default in binary format (i.e., Excel or PDF) or JSON. 11 | * Recommended using -o parameter for specifying output-dir. Otherwise, the output stored in the running dir (which is pip install cmd work with, and it depends on your OS) 12 | 13 | ## Supported Operating Systems 14 | - **Linux (Bash):** CentOS, Debian, Ubuntu, RedHat 15 | - **Windows (PowerShell):** 10, 2012, 2016 16 | 17 | ## Prerequisites 18 | * Python 3.6+ 19 | 20 | ## Installation and Execution by pulling package from PyPi: 21 | 1. Execute `pip install ws-bulk-report-generator` 22 | 2. Run report: `ws_bulk_report_generator -u -k -r -o ` 23 | >* *Note: If installing packages as a non-root user, insure to include the path to the executables within the Operating System paths.* 24 | 25 | ## Examples: 26 | Generate Due Diligence Reports (file per product) on all products within an organization in JSON format: 27 | `ws_bulk_report_generator -u -k -o -r due_diligence -t json` 28 | 29 | --- 30 | 31 | Generate Due Diligence Reports (file per project) on all projects within an organization in JSON format: 32 | `ws_bulk_report_generator -u -k -o -s project -r due_diligence -t json` 33 | 34 | --- 35 | 36 | Generate Risk Reports (PDF format) on all products (file per product) within an organization: 37 | `ws_bulk_report_generator -a app-eu -u -k -o -r risk` 38 | 39 | --- 40 | 41 | Search for log4j 3 recent vulnerabilities in the entire organization and get output in a single unified JSON: 42 | `ws_bulk_report_generator -a di.whitesourcesoftware.com -u -k -o -r vulnerability -t unified_json -x vulnerability_names="CVE-2021-45046,CVE-2021-44228,CVE-2021-4104"` 43 | * *Note: The output produces only if the specified CVEs were discovered.* 44 | 45 | --- 46 | 47 | Generate Inventory report filtered by 'libwebp-dev_0.6.1-2_amd64.deb' and get a unified JSON for the entire organization: 48 | `ws_bulk_report_generator -u -k -o -r inventory -t unified_json -x lib_name=libwebp-dev_0.6.1-2_amd64.deb` 49 | 50 | --- 51 | 52 | Generate Security Alerts report and get a unified JSON for all organizations within a Global organization (Note: user must be defined in each organization): 53 | `ws_bulk_report_generator -u -k -o -r inventory -t unified_json -y globalOrganization` 54 | 55 | --- 56 | 57 | Generate Vulnerability report and get a unified Excel report on 2 specific products in the organization (-s project means the API calls run on the project level behind the scenes, used when timeouts in the API response): 58 | `ws_bulk_report_generator -u -k -o -r vulnerability -t unified_xlsx -i " , -s project"` 59 | 60 | --- 61 | 62 | 63 | >**NEW!** USING ASYNCHRONOUS API for large organizations. 64 | Supported reports: `inventory`, `vulnerability`, `alerts`, `plugin request history` 65 | 66 | The TIMEOUT is 6 minutes, meaning it's checking for the report status READY for 6 min before moving to the next one. If it's not ready in 6 minutes, the tool logs the reportStatusId for manually checking the status (and downloading), and moves along. The timeout will be configurable in the following releases. 67 | 68 | Generate Vulnerability report using asynchronous API calls in Excel format: 69 | `ws_bulk_report_generator -u -k -o -r vulnerability -t binary -c True` 70 | 71 | --- 72 | 73 | Search for log4j 3 recent vulnerabilities in the entire organization using asynchronous API call and get output per each Product in JSON format: 74 | `ws_bulk_report_generator -u -k -o -r vulnerability -t json -x vulnerability_names="CVE-2021-45046,CVE-2021-44228,CVE-2021-4104" -c True` 75 | * *Note: The output produces only if the specified CVEs were discovered.* 76 | 77 | --- 78 | 79 | Generate Security Alerts report using asynchronous API calls in Excel format: 80 | `ws_bulk_report_generator -u -k -o -r alerts -t binary -c True` 81 | 82 | --- 83 | 84 | Generate Plugin Request history report using asynchronous API calls in Excel format (unlimited results): 85 | `ws_bulk_report_generator -u -k -o -r request_history -t binary -c True -x plugin=True` 86 | 87 | --- 88 | 89 | Generate Inventory report using asynchronous API calls in Excel format: 90 | `ws_bulk_report_generator -u -k -o -r inventory -t binary -c True` 91 | 92 |
93 | 94 | # Full Usage: 95 | ```shell 96 | usage: ws_bulk_report_generator [-h] -u WS_USER_KEY -k WS_TOKEN [-y {organization,globalOrganization}] -r 97 | {alerts,ignored_alerts,resolved_alerts,inventory,lib_dependencies,vulnerability,container_vulnerability,source_files,source_file_inventory,in_house_libraries,in_house,risk,library_location,license_compatibility,due_diligence,attributes,attribution,effective_licenses,bugs,request_history(-x plugin=true)} 98 | [-t {unified_json,unified_xlsx,binary,json}] [-s {project,product}] [-a WS_URL] [-o DIR] [-x EXTRA_REPORT_ARGS] [-i INC_TOKENS] [-e EXC_TOKENS] [-c {True,False}] 99 | 100 | Mend Bulk Reports Generator 101 | 102 | optional arguments: 103 | -h, --help show this help message and exit 104 | -u WS_USER_KEY, --userKey WS_USER_KEY 105 | WS User Key 106 | -k WS_TOKEN, --token WS_TOKEN 107 | WS Token 108 | -y {organization,globalOrganization}, --token_type 109 | WS Token Type 110 | -r {alerts,ignored_alerts,resolved_alerts,inventory,lib_dependencies,vulnerability,container_vulnerability,source_files,source_file_inventory,in_house_libraries,in_house,risk,library_location,license_compatibility,due_diligence,at 111 | tributes,attribution,effective_licenses,bugs,request_history}, --report 112 | Report Type to produce 113 | -t {unified_json,unified_xlsx,binary(i.e., Excel or PDF),json}, --outputType 114 | Type of output 115 | -s {project,product}, --ReportScope 116 | Scope of report 117 | -a WS_URL, --wsUrl WS_URL 118 | WS URL 119 | -o DIR, --reportDir DIR 120 | Report Dir 121 | -x EXTRA_REPORT_ARGS, --extraReportArguments EXTRA_REPORT_ARGS 122 | Extra arguments (key=value) to pass the report 123 | -i INC_TOKENS, --includedTokens INC_TOKENS 124 | Included token (Default: All) 125 | -e EXC_TOKENS, --excludedTokens EXC_TOKENS 126 | Excluded token (Default: None) 127 | -c ASYNCR, --asynchronousCalls ASYNCR 128 | Asynchronous API (Default: False) 129 | ``` 130 | -------------------------------------------------------------------------------- /ToDo.md: -------------------------------------------------------------------------------- 1 | # To Do List 2 | - Item 1 3 | - Item 2 4 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | ws-sdk==23.12.2 2 | DateTime==4.9 3 | requests==2.31.0 4 | XlsxWriter==3.0.3 5 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | from ws_bulk_report_generator._version import __version__, __tool_name__, __description__ 3 | 4 | ws_name = f"ws_{__tool_name__}" 5 | 6 | setuptools.setup( 7 | name=ws_name, 8 | entry_points={ 9 | 'console_scripts': [ 10 | f'{ws_name}={ws_name}.{__tool_name__}:main' 11 | ]}, 12 | version=__version__, 13 | author="WhiteSource Professional Services", 14 | author_email="ps@whitesourcesoftware.com", 15 | description=__description__, 16 | url=f"https://github.com/whitesource-ps/{ws_name.replace('_', '-')}", 17 | license='LICENSE.txt', 18 | packages=setuptools.find_packages(), 19 | python_requires='>=3.7', 20 | install_requires=[line.strip() for line in open("requirements.txt").readlines()], 21 | # extras_require={"xlsx": ["XlsxWriter~=3.0.2"]}, 22 | long_description=open("README.md").read(), 23 | long_description_content_type="text/markdown", 24 | classifiers=[ 25 | "Programming Language :: Python :: 3.8", 26 | "Programming Language :: Python :: 3.9", 27 | "License :: OSI Approved :: Apache Software License", 28 | "Operating System :: OS Independent", 29 | ], 30 | ) 31 | -------------------------------------------------------------------------------- /ws_bulk_report_generator/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/whitesource-ps/ws-bulk-report-generator/a617c6cf341645084c9e97c9d52df77f1a19089d/ws_bulk_report_generator/__init__.py -------------------------------------------------------------------------------- /ws_bulk_report_generator/_version.py: -------------------------------------------------------------------------------- 1 | __version__ = "0.0.0.dev0" 2 | __tool_name__ = "bulk_report_generator" 3 | __description__ = "WS Bulk Report Generator" 4 | -------------------------------------------------------------------------------- /ws_bulk_report_generator/bulk_report_generator.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import concurrent 3 | import json 4 | import logging 5 | import os 6 | from concurrent.futures import ThreadPoolExecutor 7 | from copy import copy 8 | from datetime import datetime 9 | from multiprocessing.pool import ThreadPool 10 | from typing import Tuple, List 11 | 12 | import xlsxwriter 13 | from ws_sdk import WS, ws_constants, ws_errors 14 | 15 | from ws_bulk_report_generator._version import __tool_name__, __version__, __description__ 16 | 17 | is_debug = logging.DEBUG if bool(os.environ.get("DEBUG", 0)) else logging.INFO 18 | 19 | logger = logging.getLogger(__tool_name__) 20 | logger.setLevel(logging.DEBUG) 21 | 22 | sdk_logger = logging.getLogger(WS.__module__) 23 | sdk_logger.setLevel(is_debug) 24 | 25 | formatter = logging.Formatter('%(levelname)s %(asctime)s %(thread)d %(name)s: %(message)s') 26 | s_handler = logging.StreamHandler() 27 | s_handler.setFormatter(formatter) 28 | s_handler.setLevel(is_debug) 29 | logger.addHandler(s_handler) 30 | sdk_logger.addHandler(s_handler) 31 | sdk_logger.propagate = False 32 | logger.propagate = False 33 | 34 | PROJECT_PARALLELISM_LEVEL = int(os.environ.get("PROJECT_PARALLELISM_LEVEL", "10")) 35 | conf = args = None 36 | JSON = 'json' 37 | BINARY = 'binary' 38 | UNIFIED_JSON = "unified_json" 39 | UNIFIED_XLSX = "unified_xlsx" 40 | UNIFIED = [UNIFIED_JSON, UNIFIED_XLSX] 41 | ALL_OUTPUT_TYPES = UNIFIED + [BINARY, JSON] 42 | 43 | 44 | def parse_args(): 45 | parser = argparse.ArgumentParser(description=__description__) 46 | parser.add_argument('-u', '--userKey', help="WS User Key", dest='ws_user_key', type=str, required=True) 47 | parser.add_argument('-k', '--token', help="WS Token", dest='ws_token', type=str, required=True) 48 | parser.add_argument('-y', '--token_type', help="WS Token Type", dest='ws_token_type', choices=[ws_constants.ScopeTypes.ORGANIZATION, ws_constants.ScopeTypes.GLOBAL], type=str, default=None) 49 | parser.add_argument('-r', '--report', help="Report Type to produce", type=str, choices=WS.get_report_types(), dest='report', required=True) 50 | parser.add_argument('-t', '--outputType', help="Type of output", choices=ALL_OUTPUT_TYPES, dest='output_type', default=BINARY) 51 | parser.add_argument('-s', '--ReportScope', help="Scope of report", type=str, choices=[ws_constants.ScopeTypes.PROJECT, ws_constants.ScopeTypes.PRODUCT], dest='report_scope_type', default=ws_constants.ScopeTypes.PRODUCT) 52 | parser.add_argument('-a', '--wsUrl', help="WS URL", dest='ws_url', type=str, default="saas") 53 | parser.add_argument('-o', '--reportDir', help="Report Dir", dest='dir', default="reports", type=str) 54 | parser.add_argument('-x', '--extraReportArguments', help="Extra arguments (key=value) to pass the report", dest='extra_report_args', type=str) 55 | parser.add_argument('-i', '--includedTokens', help="Included token (Default: All)", dest='inc_tokens', default=[]) 56 | parser.add_argument('-e', '--excludedTokens', help="Excluded token (Default: None)", dest='exc_tokens', default=[]) 57 | parser.add_argument('-c', '--asynchronousCalls', help="Asynchronous API (Default: False)", dest='asyncr', default=False, type=str2bool) 58 | 59 | return parser.parse_args() 60 | 61 | 62 | def str2bool(s): 63 | if isinstance(s, str): 64 | return strtobool(s) 65 | return bool(s) 66 | 67 | 68 | def strtobool(val): 69 | """Convert a string representation of truth to true (1) or false (0). 70 | 71 | True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values 72 | are 'n', 'no', 'f', 'false', 'off', and '0'. Raises ValueError if 73 | 'val' is anything else. 74 | """ 75 | val = val.lower() 76 | if val in ('y', 'yes', 't', 'true', 'on', '1'): 77 | return 1 78 | elif val in ('n', 'no', 'f', 'false', 'off', '0'): 79 | return 0 80 | else: 81 | raise ValueError("invalid truth value %r" % (val,)) 82 | 83 | 84 | def init(): 85 | def get_extra_report_args(extra_report_args: str) -> dict: 86 | """ 87 | Function to extract extra report argument and parse it to key value dictionary where value can be a string or a list (comma seperated). 88 | :param extra_report_args: string that of key=val or key=val1,val2... 89 | :return: dictionary 90 | """ 91 | ret = {} 92 | if extra_report_args: 93 | extra_report_args_l = extra_report_args.split("=") 94 | report_args_val_l = extra_report_args_l[1].split(',') 95 | 96 | if len(report_args_val_l) > 1: 97 | extra_report_args_l[1] = [value.strip() for value in report_args_val_l] 98 | ret = {extra_report_args_l[0]: extra_report_args_l[1]} 99 | logger.debug(f"Extra arguments passed to report: {ret}") 100 | 101 | return ret 102 | 103 | global conf, args 104 | if args.ws_token_type is None: 105 | # args.ws_token_type = WS.discover_token_type(user_key=args.ws_user_key, token=args.ws_user_key) # TBD 106 | args.ws_token_type = ws_constants.ScopeTypes.ORGANIZATION 107 | 108 | args.ws_conn = WS(url=args.ws_url, 109 | user_key=args.ws_user_key, 110 | token=args.ws_token, 111 | token_type=args.ws_token_type, 112 | tool_details=(f"ps-{__tool_name__.replace('_', '-')}", __version__), 113 | timeout=3600) 114 | 115 | args.report_method = f"get_{args.report}" 116 | try: 117 | args.report_method = getattr(WS, args.report_method) 118 | except AttributeError: 119 | logger.error(f"report: {args.report} was not found") 120 | 121 | if not os.path.exists(args.dir): 122 | logger.info(f"Creating directory: {args.dir}") 123 | os.makedirs(args.dir) 124 | 125 | args.extra_report_args_d = get_extra_report_args(args.extra_report_args) 126 | args.is_binary = True if args.output_type == BINARY else False 127 | args.write_mode = 'bw' if args.is_binary else 'w' 128 | args.reports_error = [] 129 | 130 | async_list = ['inventory', 'vulnerability', 'alerts', 'request_history'] 131 | if args.asyncr and args.report not in async_list: 132 | logger.error(f"asynchronous report mode is only supported for {async_list}") 133 | exit() 134 | 135 | 136 | def get_reports_scopes() -> List[dict]: 137 | if args.ws_token_type == ws_constants.ScopeTypes.GLOBAL: 138 | orgs = args.ws_conn.get_organizations() 139 | logger.info(f"Found: {len(orgs)} Organizations under Global Organization token: '{args.ws_token}'") 140 | else: 141 | orgs = [args.ws_conn.get_organization_details()] 142 | scopes, errors = generic_thread_pool_m(orgs, get_reports_scopes_from_org_w) 143 | if args.exc_tokens: 144 | scopes = [s for s in scopes if s['token'] not in args.exc_tokens] 145 | 146 | logger.info(f"Found {len(scopes)} Scopes on") 147 | 148 | return scopes 149 | 150 | 151 | def generic_thread_pool_m(ent_l: list, worker: callable) -> Tuple[list, list]: 152 | data = [] 153 | errors = [] 154 | 155 | with ThreadPoolExecutor(max_workers=PROJECT_PARALLELISM_LEVEL) as executer: 156 | futures = [executer.submit(worker, ent) for ent in ent_l] 157 | 158 | for future in concurrent.futures.as_completed(futures): 159 | try: 160 | temp_l = future.result() 161 | if temp_l: 162 | data.extend(temp_l) 163 | except Exception as e: 164 | errors.append(e) 165 | logger.error(f"Error on future: {future.result()}") 166 | SystemExit() 167 | 168 | return data, errors 169 | 170 | 171 | def get_reports_scopes_from_org_w(org: dict) -> List[dict]: 172 | def replace_invalid_chars(directory: str) -> str: 173 | for char in ws_constants.INVALID_FS_CHARS: 174 | directory = directory.replace(char, "_") 175 | 176 | return directory 177 | 178 | def prep_scope(report_scopes: list, o: dict): 179 | for s in report_scopes: 180 | args.report_extension = JSON if args.output_type.endswith(JSON) else args.report_method(WS, ws_constants.ReportsMetaData.REPORT_BIN_TYPE) 181 | report_name = f"{s['name']}_{s.get('productName')}" if s['type'] == ws_constants.PROJECT else s['name'] 182 | filename = f"{s['type']}_{replace_invalid_chars(report_name)}_{args.report}_org_{o['name']}.{args.report_extension}" 183 | s['report_full_name'] = os.path.join(args.dir, filename) 184 | s['ws_conn'] = org_conn 185 | s['org_name'] = o['name'] 186 | 187 | def replace_invalid_chars(directory: str) -> str: 188 | for char in ws_constants.INVALID_FS_CHARS: 189 | directory = directory.replace(char, "_") 190 | 191 | return directory 192 | 193 | global args 194 | org_conn = copy(args.ws_conn) 195 | org_conn.token_type = ws_constants.ScopeTypes.ORGANIZATION 196 | org_conn.token = org['token'] 197 | scopes = [] 198 | pre_scopes = [] 199 | 200 | if args.inc_tokens: 201 | inc_tokens_l = [t.strip() for t in args.inc_tokens.split(',')] 202 | for token in inc_tokens_l: 203 | pre_scopes.append(org_conn.get_scope_by_token(token=token, token_type=ws_constants.ScopeTypes.ORGANIZATION)) 204 | if ws_constants.ScopeTypes.PROJECT in args.report_scope_type: 205 | for scope in pre_scopes: 206 | scopes.extend(list(org_conn.get_scopes(scope_type=args.report_scope_type, include_prod_proj_names=True, 207 | product_token=scope.get("token")))) 208 | else: 209 | scopes = pre_scopes 210 | else: 211 | try: 212 | if args.extra_report_args_d.get('plugin'): 213 | if strtobool(args.extra_report_args_d.get('plugin')): 214 | scopes = org_conn.get_scopes(scope_type=ws_constants.ScopeTypes.ORGANIZATION) 215 | args.report_scope_type = ws_constants.ScopeTypes.ORGANIZATION 216 | else: 217 | scopes = org_conn.get_scopes(scope_type=args.report_scope_type, include_prod_proj_names=False) 218 | except ws_errors.WsSdkServerInactiveOrg: 219 | logger.warning(f"Organization: '{org['name']}' is disabled and will be skipped") 220 | 221 | prep_scope(scopes, org) 222 | 223 | return scopes 224 | 225 | 226 | def generate_unified_report_w(report_desc: dict) -> list: 227 | ret = None 228 | logger.info(f"Running '{args.report}' report on {report_desc['type']}: '{report_desc['name']}' on organization: '{report_desc['org_name']}'") 229 | 230 | output = args.report_method(report_desc['ws_conn'], 231 | token=(report_desc['token'], args.report_scope_type), 232 | report=args.is_binary, 233 | **args.extra_report_args_d) 234 | 235 | if output: 236 | for item in output: 237 | item.update({"org_name": report_desc.get("org_name")}) 238 | ret = output 239 | else: 240 | logger.debug(f"Report '{args.report}' returned empty on {report_desc['type']}: '{report_desc['name']}' on organization: '{report_desc['org_name']}'") 241 | 242 | return ret 243 | 244 | 245 | def generate_xlsx(output, full_path) -> List[dict]: 246 | def generate_row_data(col_names: list, d: dict) -> list: 247 | row_data_l = [] 248 | for c in col_names: 249 | cell_val = d.get(c) 250 | if isinstance(cell_val, (list, dict)): 251 | cell_val = json.dumps(cell_val) 252 | row_data_l.append(cell_val) 253 | 254 | return row_data_l 255 | 256 | def generate_table_labels(o: list) -> List[str]: 257 | col_names = args.report_method(WS, ws_constants.ReportsMetaData.COLUMN_NAMES) 258 | if not col_names: 259 | col_names = o[0].keys() 260 | 261 | for c_num, c_name in enumerate(col_names): 262 | worksheet.write(0, c_num, c_name, cell_format) 263 | 264 | return col_names 265 | 266 | options = None 267 | with xlsxwriter.Workbook(full_path, options=options) as workbook: 268 | worksheet = workbook.add_worksheet() 269 | cell_format = workbook.add_format({'bold': True, 'italic': False}) 270 | column_names = generate_table_labels(output) 271 | 272 | for row_num, row_data in enumerate(output): 273 | worksheet.write_row(row_num + 1, 0, generate_row_data(column_names, row_data)) 274 | 275 | logger.debug(f"Total number of Excel rows: {row_num}") 276 | 277 | 278 | def write_unified_file(output: list): 279 | report_name = f"{args.ws_conn.get_name()} - {args.report} report" 280 | filename = f"{report_name}.{args.report_extension}" 281 | full_path = os.path.join(args.dir, filename) 282 | 283 | start_time = datetime.now() 284 | if args.output_type == UNIFIED_XLSX: 285 | logger.info("Converting output to Excel") 286 | generate_xlsx(output, full_path) 287 | else: 288 | with open(full_path, args.write_mode) as fp: 289 | json.dump(output, fp) 290 | 291 | logger.info(f"Finished writing filename: '{full_path}'. Total time: {datetime.now() - start_time}") 292 | 293 | 294 | def generate_unified_reports(report_scopes: list): 295 | return generic_thread_pool_m(ent_l=report_scopes, worker=generate_unified_report_w) 296 | 297 | 298 | def generate_reports(report_scopes: list): 299 | 300 | def generate_report_w(report_desc: dict, args) -> list: 301 | logger.info(f"Running '{args.report}' report on {report_desc['type']}: '{report_desc['name']}' on organization: '{report_desc['org_name']}'") 302 | 303 | output = args.report_method(report_desc['ws_conn'], 304 | token=(report_desc['token'], args.report_scope_type), 305 | asyncr=args.asyncr, 306 | report=args.is_binary, 307 | **args.extra_report_args_d) 308 | if isinstance(output, dict): 309 | for k, v in output.items(): 310 | if 'asyncReport' in k: 311 | handle_async_reports_names(output, report_desc) 312 | elif 'Failed' in k: 313 | return 314 | else: 315 | write_report(output, report_desc) 316 | 317 | def handle_async_reports_names(output, report_desc): 318 | for key, value in output.items(): 319 | line = report_desc['type'] + '_' + key.split("asyncReport: ", 1)[1] 320 | index = line.find('.') 321 | name = line[:index] + f"_org_{report_desc['org_name']}" + line[index:] 322 | report_desc['report_full_name'] = os.path.join(args.dir, name) 323 | output = value 324 | write_report(output, report_desc) 325 | 326 | def write_report(output, report_desc): 327 | if output: 328 | logger.debug(f"Saving report in: {report_desc['report_full_name']}") 329 | f = open(report_desc['report_full_name'], args.write_mode) 330 | report = output if args.is_binary else json.dumps(output) 331 | f.write(report) 332 | f.close() 333 | 334 | global PROJECT_PARALLELISM_LEVEL 335 | if args.asyncr: 336 | PROJECT_PARALLELISM_LEVEL = 1 337 | with ThreadPool(processes=PROJECT_PARALLELISM_LEVEL) as thread_pool: 338 | thread_pool.starmap(generate_report_w, [(comp, args) for comp in report_scopes]) 339 | 340 | 341 | def handle_unified_report(output: list): 342 | if output: 343 | write_unified_file(output) 344 | else: 345 | logger.info("No data returned. No report will be saved") 346 | 347 | 348 | def main(): 349 | global args, conf 350 | start_time = datetime.now() 351 | args = parse_args() 352 | logger.info(f"Start running {__description__} Version {__version__} on token {args.ws_token}. Parallelism level: {PROJECT_PARALLELISM_LEVEL}") 353 | init() 354 | report_scopes = get_reports_scopes() 355 | 356 | if args.output_type in UNIFIED: 357 | ret, errors = generate_unified_reports(report_scopes) 358 | handle_unified_report(ret) 359 | else: 360 | generate_reports(report_scopes) 361 | 362 | logger.info(f"Finished running {__description__}. Run time: {datetime.now() - start_time}") 363 | 364 | 365 | if __name__ == '__main__': 366 | main() 367 | --------------------------------------------------------------------------------