├── .github └── workflows │ ├── pypi.yml │ └── unittests.yml ├── .gitignore ├── CHANGES.md ├── LICENSE ├── README.md ├── docs ├── introduction.md ├── recipes.md ├── tools.md └── usage.md ├── images ├── mussels-500.png └── mussels.png ├── mussels ├── __init__.py ├── __main__.py ├── bookshelf.py ├── mussels.py ├── recipe.py ├── tool.py └── utils │ ├── __init__.py │ ├── click.py │ └── versions.py ├── setup.py └── tests ├── tool_variables_test.py ├── versions__compare_versions_test.py └── versions__get_item_version_test.py /.github/workflows/pypi.yml: -------------------------------------------------------------------------------- 1 | name: Publish Python 🐍 distributions 📦 to PyPI 2 | 3 | on: release 4 | 5 | jobs: 6 | build-n-publish: 7 | name: Build and publish Python 🐍 distributions 📦 to PyPI 8 | runs-on: ubuntu-latest 9 | 10 | steps: 11 | - uses: actions/checkout@master 12 | 13 | - name: Set up Python 3.12 14 | uses: actions/setup-python@v1 15 | with: 16 | python-version: 3.12 17 | 18 | - name: Install pypa/build 19 | run: >- 20 | python -m 21 | pip install 22 | build 23 | --user 24 | 25 | - name: Build a binary wheel and a source tarball 26 | run: >- 27 | python -m 28 | build 29 | --sdist 30 | --wheel 31 | --outdir dist/ 32 | . 33 | 34 | - name: Publish distribution 📦 to PyPI 35 | uses: pypa/gh-action-pypi-publish@master 36 | with: 37 | password: ${{ secrets.PYPI_API_TOKEN }} 38 | -------------------------------------------------------------------------------- /.github/workflows/unittests.yml: -------------------------------------------------------------------------------- 1 | name: Unit Tests 2 | 3 | on: 4 | push: 5 | branches: [master] 6 | pull_request: 7 | branches: [master] 8 | 9 | jobs: 10 | build: 11 | runs-on: ubuntu-latest 12 | 13 | steps: 14 | - uses: actions/checkout@v1 15 | - name: Set up Python 3.12 16 | uses: actions/setup-python@v1 17 | with: 18 | python-version: 3.12 19 | - name: Install dependencies 20 | run: | 21 | python -m pip install --upgrade pip 22 | pip install . 23 | - name: Lint with flake8 24 | run: | 25 | pip install flake8 26 | # stop the build if there are Python syntax errors or undefined names 27 | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics 28 | # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide 29 | flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics 30 | - name: Test with pytest 31 | run: | 32 | pip install pytest 33 | pytest -vvs 34 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | .mypy_cache 3 | .pytest_cache 4 | .vscode 5 | mussels.egg-info 6 | -------------------------------------------------------------------------------- /CHANGES.md: -------------------------------------------------------------------------------- 1 | # Notable Changes 2 | 3 | > _Note_: Changes should be grouped by release and use these icons: 4 | > - Added: ➕ 5 | > - Changed: 🌌 6 | > - Deprecated: 👇 7 | > - Removed: ❌ 8 | > - Fixed: 🐛 9 | > - Security: 🛡 10 | 11 | ## Version 0.4.1 12 | 13 | 🐛 Fixed an issue using pkg_resources package when setuptools is not installed. The fix adds setuptools as a dependency. 14 | 15 | ## Version 0.4.0 16 | 17 | ### Added 18 | 19 | ➕ Added support for extracting `.tar.xz` source archives, in addition to `.tar.gz` and `.zip` archives. 20 | 21 | ## Version 0.3.0 22 | 23 | ### Added 24 | 25 | ➕ Added the ability to define variables in tools that can be accessed in recipes. 26 | 27 | To define variables, add a `variables` list for each platform. 28 | 29 | This feature was inspired by a need to make recipes that have strings in them specific to a given tool version. Specifically we'll be using this to define CMake generator names in Visual Studio tool YAML files. The correct generator name for a given Visual Studio version will then be available for use in recipes that use Visual Studio. 30 | 31 | Example tool, `visualstudio-2017.yaml`: 32 | 33 | ```yaml 34 | name: visualstudio 35 | version: "2017" 36 | mussels_version: "0.3" 37 | type: tool 38 | platforms: 39 | Windows: 40 | file_checks: 41 | - C:\/Program Files (x86)/Microsoft Visual Studio/2017/Professional/VC/Auxiliary/Build/vcvarsall.bat 42 | - C:\/Program Files (x86)/Microsoft Visual Studio/2017/Enterprise/VC/Auxiliary/Build/vcvarsall.bat 43 | - C:\/Program Files (x86)/Microsoft Visual Studio/2017/Community/VC/Auxiliary/Build/vcvarsall.bat 44 | variables: 45 | cmake_generator: Visual Studio 15 2017 46 | ``` 47 | 48 | Example recipe, `libz.yaml`: 49 | 50 | ```yaml 51 | name: libz 52 | version: "1.2.11" 53 | url: https://www.zlib.net/zlib-1.2.11.tar.gz 54 | mussels_version: "0.3" 55 | type: recipe 56 | platforms: 57 | Windows: 58 | x64: 59 | build_script: 60 | configure: | 61 | CALL cmake.exe -G "{visualstudio.cmake_generator} Win64" 62 | make: | 63 | CALL cmake.exe --build . --config Release 64 | dependencies: [] 65 | install_paths: 66 | include: 67 | - zlib.h 68 | - zconf.h 69 | lib: 70 | - Release/zlibstatic.lib 71 | license/zlib: 72 | - README 73 | required_tools: 74 | - cmake 75 | - visualstudio>=2017 76 | ``` 77 | 78 | 🐛 Fixed a crash caused by a failure to properly decode command output text from some locales to utf8. 79 | 80 | ## Version 0.2.1 81 | 82 | ➕ Added the Mussels version string to the `msl --help` output. 83 | 84 | ➕ Added `msl build` options allowing users to customize the work, logs, and downloads directories: 85 | - `-w`, `--work-dir TEXT` Work directory. The default is: ~/.mussels/cache/work 86 | - `-l`, `--log-dir TEXT` Log directory. The default is: ~/.mussels/logs 87 | - `-D`, `--download-dir TEXT` Downloads directory. The default is: ~/.mussels/cache/downloads 88 | 89 | 🐛 The `msl build -c` shorthand for `--cookbook` was overloaded by the `--clean` (`-c`) shorthand. Because `--cookbook` (`-c` ) is also used elsewhere, the `--clean` option was renamed to `--rebuild` (`-r`). 90 | 91 | 🐛 Fixed an issue where the list of available tools was being limited based on _all_ recipe tool version requirements rather than just those found in the dependency chain. 92 | 93 | ## Version 0.2.0 94 | 95 | ➕ Added the `msl build` `--install`/`-i` option, allowing builds to install directly to a directory of the user's choosing. 96 | 97 | 🌌 The `{install}` build script variable now points to the full install prefix, including the target architecture directory when building with the default install directory (eg: `host`, `x86`, `x64`, etc). 98 | 99 | This was necessary in order for the `--install` option to make sense. This also simplifies recipes because they no longer have to specify "`{install}/{target}`" to reference the install directory. This, unfortunately, also makes it a breaking change. All recipes will have to remove the "`/{target}`" to remain compatible. 100 | 101 | ## Version 0.1.0 102 | 103 | ➕ First release! 104 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | Mussels 3 |

4 | 5 |

A tool to download, build, and assemble application dependencies. 6 |
Brought to you by the Clam AntiVirus Team. 7 |

Copyright (C) 2019-2021 Cisco Systems, Inc. and/or its affiliates. All rights reserved.

8 | 9 |

10 | 11 | PyPI version 12 | 13 | PyPI - Python Version 14 | 15 | Unit Tests 16 | 17 |

18 | 19 | ## About 20 | 21 | Mussels is a cross-platform and general-purpose dependency build automation tool. 22 | 23 | Mussels helps automate the building of applications _and_ their versioned dependency chains using the original build systems intended by the software authors. 24 | 25 | For a more in depth explanation, see the [Mussels Introduction](docs/introduction.md). 26 | 27 | ## Requirements 28 | 29 | - Python 3.6 or newer. 30 | - Git (must be added to your PATH environment variable). 31 | 32 | An internet connection is required to use the public Mussels cookbooks. Some form of internet or intranet is required to download source archives from the URLs defined in each recipe. 33 | 34 | Every recipe will require tools in order to run. If you don't have the required tools, you'll get an error message stating that you're missing a required tool. It will be up to you to install the tool in order for that recipe to build. 35 | 36 | ### Common Tools Requirements for building C/C++ software 37 | 38 | Mussels was born out of the ClamAV project so you can find some good example recipe and tool definitions here: https://github.com/Cisco-Talos/clamav-mussels-cookbook/ 39 | 40 | The ClamAV recipes build C libraries for the most part. When using them, you'll probably the following compiler toolchain software installed on your system for the build to work. If these are missing, the Mussels build will fail and tell you as much. 41 | 42 | Linux: 43 | 44 | - gcc 45 | - Make 46 | - CMake 47 | - patchelf 48 | 49 | MacOS (Darwin): 50 | 51 | - Clang (comes with XCode) 52 | - Make 53 | - CMake 54 | 55 | Windows: 56 | 57 | - Visual Studio 2017+ 58 | - CMake 59 | 60 | ## Installation 61 | 62 | You may install Mussels from PyPI using `pip`, or you may clone the Mussels Git repository and use `pip` to install it locally. 63 | 64 | Install Mussels from PyPI: 65 | 66 | > `python3 -m pip install --user mussels` 67 | 68 | ## Usage 69 | 70 | Use the `--help` option to get information about any Mussels command. 71 | 72 | > `mussels` 73 | > 74 | > `mussels --help` 75 | > 76 | > `mussels build --help` 77 | 78 | When performing a build, the intermediate build files are placed into the `~/.mussels/cache/work/` directory and the final installed files into the `~/.mussels/install/` directory. This default behavior can be overridden using `msl build --work-dir ` to specify a different work directory, and `msl build --install ` to specify a different install directory. 79 | 80 | _Tip_: Use the `msl` shortcut, instead of `mussels` to save keystrokes. 81 | 82 | _Tip_: You may not be able to run `mussels` or the `msl` shortcut directly if your Python Scripts directory is not in your `PATH` environment variable. If you run into this issue, and do not wish to add the Python Scripts directory to your path, you can run Mussels like this: 83 | 84 | > `python -m mussels` 85 | 86 | Learn more about how use Mussels in our [documentation](docs/usage.md). 87 | 88 | ## How it works 89 | 90 | Any time you run Mussels, Mussels will search your current directory to load Mussels-YAML files that define "recipes" and "tools". Recipes define instructions to build a program or library. Tools describe a way to verify that your computer has a program needed to build a recipe. Recipes may depend any number of tools and on other recipes. Mussels will fail to build a recipe if a circular dependency is detected. 91 | 92 | > _Tip_: Don't run Mussels in a big directory like your home directory. The recursive search for Mussels-YAML files will make startup sluggish, and the current version may throw some errors if it encounters non-Mussels YAML files. 93 | 94 | All Mussels-YAML files have a type field that may be either "tool", "recipe", or "collection". A collection is just a special purpose recipe that only contains dependencies for other recipes. Each YAML file also includes the minimum Mussels version required to use the recipe. 95 | 96 | Inside of each recipe YAML there is: 97 | - A recipe name. 98 | - A recipe version number. 99 | - A URL for downloading the source code, 100 | - For each `platform` (OS): 101 | - For each `target` supported on that OS: 102 | - Recipe dependencies that must be built before this recipe. 103 | - Tools dependencies that must be present to build this recipe. 104 | - (*optional*) Build scripts to "configure", "make", and "install" for this target. 105 | - (*optional*) A list of additional files that will be copied from the work directory to the install directory after the "install" script has run. 106 | - (*optional*) A directory name next to the recipe file that contains patches that that will be applied to the source code before running the "configure" script. 107 | 108 | On Linux/UNIX systems, the default target is `host`. But you can define custom targets for variants of the recipe like `host-static`, or `host-static-debug`. On Windows the default target is either `x86` or `x64` depending on your current OS architecture. But you're welcome to use something custom here as well. 109 | 110 | Inside of each tool YAML there is: 111 | - A tool name. 112 | - (*optional*) A tool version number. 113 | - Ways to check if the tool exists for each `platform`: 114 | - (*optional*) Check if one or more names is in the `$PATH`. 115 | - (*optional*) Check if one or more filepaths exists on disk. 116 | - (*optional*) Check if the output of running one or more commands (like `clang --version`) matches expectations. 117 | 118 | Each `tool` definition is good for one (1) program check. Lets say you wanted to check if a suite of programs exists. You may check for just one of those tools, but if you need to verify that all of the programs exist, you would need to have a `tool` YAML file for each program. 119 | 120 | When assembling the dependency chain, Mussels will only evaluate recipes that all have the same platform and target. That is to say, that if you want to build your recipe with a `host-static` target, then each of your recipe dependencies must also have a `host-static` target defined. 121 | 122 | At build time, Mussels will evaluate the dependency chain and verify that (A) all of the recipes in the chain exist for the given platform and architecture and that (B) all of the tools required by the recipes can be found on the system, using methods defined in each tool's YAML file. You can use the `msl build --dry-run` option to do this check without performing an actual build. When not doing a "dry run", the build will proceed to build the dependency chain in reverse order, building the things with no dependencies first, followed by the things that depend on them. The `msl build --dry-run` option will show you that chain if you're curious what it looks like. 123 | 124 | Each recipe has 3 stages: "configure", "make", and "install". On Linux/Unix these stages are instructions for bash scripts and on Windows they're instructions for batch scripts. These scripts are written to the `~/.mussels/cache/work/{dependency}` directory and executed. 125 | 126 | - `configure`: This is used for Autotools' `./configure` step, or CMake's first call to `cmake .` build system generation step. 127 | - The "configure" script is only run the first time you build the recipe. 128 | 129 | > _Tip_: If something goes wrong during this stage, like you canceled the build early, all subsequent attempts to build will fail. You can force Mussels to rebuild from scratch using the `--rebuild` (`-r`) option. This will re-build ALL recipes in the dependency chain. 130 | 131 | - `make`: This is used for Autotools' `make` step, or CMake's `cmake --build .` 132 | - Mussels will re-run this step for every dependency in the chain every time you build a recipe. This is usually pretty fast because if the scripts use CMake, Autotools, Meson, etc to do the build... those tools will verify what _actually_ needs to be recompiled. 133 | 134 | - `install`: This is used for Autotools' `make install` step, or CMake's `cmake --build . --target install`. 135 | - Mussels will re-run this step for every dependency in the chain every time you build a recipe. 136 | 137 | ## Contribute 138 | 139 | Mussels is open source and we'd love your help. There are many ways to contribute! 140 | 141 | ### Community 142 | 143 | Join the ClamAV / Mussels community on the [ClamAV Discord chat server](https://discord.gg/6vNAqWnVgw). 144 | 145 | ### Contribute Recipes 146 | 147 | You can contribute to the Mussels community by creating new recipes or improving on existing recipes in the ["scrapbook"](https://github.com/Cisco-Talos/mussels-recipe-scrapbook). Do this by submitting a pull request to that Git repository. 148 | 149 | If your project is willing to make your project-specific recipes available to the public, we'd also be happy to add your cookbook repository to the Mussels [bookshelf](mussels/bookshelf.py). Do this submitting a pull request to this Git repository. As noted above, each cookbook's license must be compatible with the Apache v2.0 license used by Mussels in order to be included in the bookshelf. 150 | 151 | To learn more about how to read & write Mussels recipe and tool definitions, check out the following: 152 | 153 | - [Recipe guide](docs/recipes.md) 154 | - [Tool guide](docs/tools.md) 155 | 156 | ### Report issues 157 | 158 | If you find an issue with Mussels or the Mussels documentation, please submit an issue to our [GitHub issue tracker](https://github.com/Cisco-Talos/Mussels/issues). Before you submit, please check to if someone else has already reported the issue. 159 | 160 | ### Mussels Development 161 | 162 | If you find a bug and you're able to craft a fix yourself, consider submitting the fix in a [pull request](https://github.com/Cisco-Talos/Mussels/pulls). Your help will be greatly appreciated. 163 | 164 | If you want to contribute to the project and don't have anything specific in mind, please check out our [issue tracker](https://github.com/Cisco-Talos/Mussels/issues). Perhaps you'll be able to fix a bug or add a cool new feature. 165 | 166 | _By submitting a contribution to the Mussels project, you acknowledge and agree to assign Cisco Systems, Inc the copyright for the contribution. If you submit a significant contribution such as a new feature or capability or a large amount of code, you may be asked to sign a contributors license agreement comfirming that Cisco will have copyright license and patent license and that you are authorized to contribute the code._ 167 | 168 | #### Mussels Development Set-up 169 | 170 | The following steps are intended to help users that wish to contribute to development of the Mussels project get started. 171 | 172 | 1. Create a fork of the [Mussels git repository](https://github.com/Cisco-Talos/Mussels), and then clone your fork to a local directory. 173 | 174 | For example: 175 | 176 | > `git clone https://github.com//Mussels.git` 177 | 178 | 2. Make user Mussels is not already installed. If it is, remove it. 179 | 180 | > `python3 -m pip uninstall mussels` 181 | 182 | 3. Use pip to install Mussels in "edit" mode. 183 | 184 | > `python3 -m pip install -e --user ./Mussels` 185 | 186 | Once installed in "edit" mode, any changes you make to your clone of the Mussels code will be immediately usable simply by running the `mussels` / `msl` commands. 187 | 188 | ### Conduct 189 | 190 | This project has not selected a specific Code-of-Conduct document at this time. However, contributors are expected to behave in professional and respectful manner. Disrespectful or inappropriate behavior will not be tolerated. 191 | 192 | ## License 193 | 194 | Mussels is licensed under the Apache License, Version 2.0 (the "License"). You may not use the Mussels project except in compliance with the License. 195 | 196 | A copy of the license is located [here](LICENSE), and is also available online at [apache.org](http://www.apache.org/licenses/LICENSE-2.0). 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /docs/introduction.md: -------------------------------------------------------------------------------- 1 | # Mussels 2 | 3 | - [Mussels](#mussels) 4 | - [Introduction](#introduction) 5 | - [But really, what is Mussels?](#but-really-what-is-mussels) 6 | - [Terminology](#terminology) 7 | - [`recipe`](#recipe) 8 | - [`tool`](#tool) 9 | - [`cookbook`](#cookbook) 10 | - [`platform`](#platform) 11 | - [`target`](#target) 12 | 13 | ## Introduction 14 | 15 | Mussels can help you build software on macOS, Linux/Unix, and Windows operating systems. Though current recipes and tools are written to build C-based software for the host machine, Mussels provides the flexibility to extend recipes to other architectures, and to build software written in other languages. 16 | 17 | At its core, a Mussels recipe defines the following for each build platform and each target architecture: 18 | 19 | - A list of other recipes (dependencies) that must be built first. 20 | - A list of tools needed to perform the build. 21 | - A patch-set directory. (optional) 22 | - Build scripts. (optional) 23 | - Extra install files. (optional) 24 | 25 | Building a recipe performs the following: 26 | 27 | 1. Download and extract the source archive for your build, using a URL defined by the recipe. 28 | 2. Apply patches to the extracted source, if needed. 29 | 3. Run scripts, defined by the recipe: 30 | a. Configure the build, only run the first time a build is run. 31 | b. Build (make) a build. 32 | c. Install the build. 33 | 4. Copy additional files declared by the recipe to specified directories. 34 | 35 | ## But really, what is Mussels? 36 | 37 | Mussels... 38 | 39 | - Is all about sharing. Place your recipes in a public Git repository or "cookbook". Anyone else can then use your recipes as templates for their own project. 40 | 41 | _IMPORTANT_: Users aren't intended to blindly build recipes from other users' cookbooks, which would amount to blinding downloading and executing untrusted code. Yikes! Users ARE, however, encouraged to copy each others recipes and tweak them to suite their projects' needs. 42 | 43 | To this end, Mussels provides the `cookbook add` and `cookbook update` commands to find new recipes, along with the `recipe clone` and `tool clone` commands to copy any recipe or tool definition into the current working directory. 44 | 45 | In addition, you have the option to explicitly "trust" a cookbook with the `cookbook trust` command, allowing you to run builds directly from that cookbook. Mussels will not let you build recipes from untrusted cookbooks. 46 | 47 | - Is cross-platform! Mussels is designed to work on Windows, macOS, Linux, and other forms of UNIX. 48 | 49 | - Provides dependency chaining and dependency versioning. 50 | 51 | Each recipe must identify the names of its depependencies and may optionally specify which dependency versions are compatible using `>` (greater than), `<` (less than), or `=` (equal to). 52 | 53 | Recipes for dependencies must either be provided in the same location, or a "cookbook" may be specified to indicate where to find the recipe. 54 | 55 | For example, a recipe depends on the `zlib` recipe, version greather than `1.2.8`, provided by the `scrapbook`^ cookbook would state its "dependencies" as follows: 56 | 57 | ```yaml 58 | dependencies: 59 | - scrapbook:zlib>1.2.8 60 | ``` 61 | 62 | - Provides build tool detection and build tool version selection. 63 | 64 | Similar to how recipes identify their recipe dependencies, recipes must identify the tools required to build a recipe. Developers may create custom tool definitions if existing tool definitions don't suit their needs. 65 | 66 | A recipe may depend on specific build tools, and may tool versions as needed. As with recipes, a curated list of tools is provided in the Mussels "scrapbook" cookbook, but users are welcome to define their own to suit their needs. 67 | 68 | Example recipe "required_tools" definition using tool definitions provided by the scrapbook: 69 | 70 | ```yaml 71 | required_tools: 72 | - scrapbook:nasm 73 | - scrapbook:perl 74 | - scrapbook:visualstudio>=2017 75 | ``` 76 | 77 | - Does _NOT_ require changes to your project source code. 78 | 79 | Unlike other dependency management tools, there is no requirement to add any Mussels files to your project repository. 80 | 81 | If you need to define your own recipes, and you probably will, you're encouraged to place them in a "cookbook" repository. We suggest that you make this separate from your main project repository, although you could also place them in a sub-directory in your project if you so desire. 82 | 83 | - Does _NOT_ insert itself as a new dependency for your project. 84 | 85 | Mussels exists make your life easier by providing a framework to automate your existing build processes. 86 | 87 | - Does _NOT_ require you to write all new build tooling. Unlike other dependency management tools, there are no custom CMakelists, Makefiles, or Visual Studio project files required for your code or your dependencies. 88 | 89 | Mussels recipes are, at their core, simple bash or batch scripts written to build your dependencies using the tools and commands that the library authors expected you to use. 90 | 91 | - Is _NOT_ a replacement for traditional build tools. 92 | 93 | Your project will still require a traditional build system and compiler such a Make, CMake, Meson, Bazel, Visual Studio, gcc, Clang, _etc_. 94 | 95 | - Is _NOT_ a package manager. Mussels is not intended to compete with or replace application distribution tools such as DNF/Yum, Homebrew, Chocolatey, apt-get, Snapcraft, Flatpak, the Windows App Store, _etc_. 96 | 97 | - Is intended to enable application developers to build their own dependencies to be distributed with their applications on systems where the system cannot or should not be relied upon to provide application dependencies. 98 | 99 | ^_Nota bene_: The [scrapbook](https://github.com/Cisco-Talos/mussels-recipe-scrapbook) is a curated collection of general purpose recipes. If you would like to provide a recipe or two for public use and don't want to maintain your own cookbook repository, consider submitting your recipes in a pull-request to the scrapbook. 100 | 101 | ## Terminology 102 | 103 | ### `recipe` 104 | 105 | A YAML file defining how to build a specific library or application. It defines the following: 106 | 107 | - name 108 | - version 109 | - download URL 110 | - build scripts, each of which are optional: 111 | - configure 112 | - This will only run the first time a recipe is built, unless you build with the `--rebuild` option, or delete the build directory from the `~/.mussels/cache` 113 | - make 114 | - install 115 | - dependencies (other recipes) required for the build 116 | - required tools needed to perform the build 117 | - files and directories to be copied to the `~/.mussels/install/{target}` directory after install 118 | - (optional) A recipe may be a collection; that is - a list of recipe "dependencies" with no download URL or and no build instructions. 119 | 120 | For more detail about recipes, please view the [recipe specification](docs/recipes.md). 121 | 122 | ### `tool` 123 | 124 | A YAML file defining how to find tools used to build recipes. It defines the following: 125 | 126 | - name 127 | - version 128 | - how to identify that the tool is installed. May check: 129 | - if a file exists in the PATH directories 130 | - if a command executes 131 | - with an option to check if the output includes some text, such as a specific version string 132 | - if a filepath exists 133 | - this will add the directory where the file is found to the PATH variable at build time automatically. 134 | - file or directory paths that, if they exist, indicate that the tool is installed 135 | 136 | For more detail about tools, pleases view the [tool specification](docs/tools.md). 137 | 138 | ### `cookbook` 139 | 140 | A git repository containing recipe and tool definitions. 141 | 142 | Mussels maintains a [an index of cookbooks](mussels/bookshelf.py). To register your cookbook in the index so that others may more easily reference your recipes, please submit a [pull-request on GitHub](https://github.com/Cisco-Talos/Mussels/pulls). 143 | 144 | When Mussels is run in a local directory containing recipe and/or tool definitions, it will detect^ these and make them available as being provided by the "local" cookbook. 145 | 146 | ^_Caution_: Mussels recursively indexes your current working directory looking for YAML files containing the `mussels_version` field. This is how it identifies local recipes and tools. **There is no recursion depth limit at this time**, so this process can take a while in very large directory trees. _You are advised to run Mussels in a small directory tree or empty directory._ 147 | 148 | ### `platform` 149 | 150 | The host operating system. The `platform` options extend Python's `platform.system()` function, and may be one of the following^: 151 | 152 | - Darwin / macOS / OSX 153 | - Windows 154 | - Linux 155 | - Unix ( Darwin, FreeBSD, OpenBSD, SunOS, AIX, HP-UX ) 156 | - Posix ( Unix and Linux ) 157 | - FreeBSD 158 | - OpenBSD 159 | - SunOS 160 | - AIX 161 | - HP-UX 162 | 163 | ^_Disclaimer_: Mussels has really only been tested on macOS, Windows, and Linux so far but is intended to work on any OS that supports Python, and Git. 164 | 165 | ### `target` 166 | 167 | The target architecture for a build. The default on Posix systems is `host`, and the default on Windows is the host architecture (`x64` or `x86`). You're welcome to define any `target` name that you wish in your recipes, so long as all of your recipe dependencies also support that `target`. 168 | -------------------------------------------------------------------------------- /docs/recipes.md: -------------------------------------------------------------------------------- 1 | # Recipes 2 | 3 | Recipes are simple YAML files that must adhere to the following format: 4 | 5 | ```yaml 6 | name: template 7 | version: "0.2" 8 | url: "hxxps://www.example.com/releases/v0.2.tar.gz" 9 | archive_name_change: # Optional; delete if not needed. 10 | - v0.2 # search pattern 11 | - template-0.2 # replace pattern 12 | mussels_version: "0.2" 13 | type: recipe 14 | platforms: 15 | : 16 | : 17 | build_script: 18 | configure: | 19 | 20 | make: | 21 | 22 | install: | 23 | 24 | dependencies: [] 25 | install_paths: 26 | : 27 | - 28 | - 29 | - 30 | patches: # Optional; delete if not needed. 31 | required_tools: 32 | - 33 | - 34 | - 35 | ``` 36 | 37 | ## Recipe Fields, in Detail 38 | 39 | ### `name` 40 | 41 | The name of the library or application this recipe builds. 42 | 43 | _Tip_: Mussels recipe names may not include the following characters: `-`, `=`, `@`, `>`, `<`, `:`. Instead, consider the humble underscore `_`. 44 | 45 | ### `version` 46 | 47 | The recipe version _string_ is generally expected to follow traditional semantic versioning practices (i.e `".."`), though any alpha-numeric version string should be fine. So long as the format is consistent across multiple versions, Mussels should be able to compare version strings for a given recipe. 48 | 49 | ### `url` 50 | 51 | The URL to be used to download a TAR or ZIP archive containing the source code to be built. 52 | 53 | In the future, we would like to add support for local paths and Git repositories, but for the moment this must be a URL ending in `.tar.gz` or `.zip`. 54 | 55 | ### `archive_name_change` (optional) 56 | 57 | This _optional_ field exists because some software packages are provided in zips or tarballs that have been renamed after creation, meaning that after extraction the resulting director does not have the same prefix as the original archive. 58 | 59 | The `archive_name_change` list field should have 2 items: 60 | 61 | - The "search pattern" 62 | - The "replace pattern" 63 | 64 | When downloaded, an archive will be renamed to replace the "search pattern" with the "replace pattern", thereby reverting the archive to it's true/original name so that when extracted - the resulting directory name will match the archive. 65 | 66 | ### `mussels_version` 67 | 68 | This version string defines which version of Musssels the recipe is written to work with. It is also the key used by Mussels to differentiate Mussels YAML files from any other YAML file. 69 | 70 | The value must be `"0.1"` 71 | 72 | ### `type` 73 | 74 | Recipe type can either be one of: 75 | 76 | - `recipe` 77 | - `collection` 78 | 79 | What is the difference between a `recipe` and a `collection`? Recipes have the `url` field, and include the fields `build_script`, `install_paths`, `patches`, and `required_tools`. Collections don't include any of the above. Collections just provide the `dependencies` lists. 80 | 81 | ### `platforms` 82 | 83 | The platforms dictionary allows you to define build instructions for different host platforms all in one file. 84 | 85 | The `` keys under `platforms` may be one of the following^: 86 | 87 | - Darwin / macOS / OSX 88 | - Windows 89 | - Linux 90 | - Unix ( Darwin, FreeBSD, OpenBSD, SunOS, AIX, HP-UX ) 91 | - Posix ( Unix and Linux ) 92 | - FreeBSD 93 | - OpenBSD 94 | - SunOS 95 | - AIX 96 | - HP-UX 97 | 98 | ### `target` 99 | 100 | Each `platform` under `platforms` is itself also a dictionary and may contain one or more `` keys. 101 | 102 | Each `target` represents the built target architecture. On Posix systems, Mussels' default target is `host` and on Windows, it is the current architecture (either `x64` or `x86`). 103 | 104 | However, you may define the `` name to be anything you like, representing the architecture of the machine intended to run the built software. 105 | 106 | The `target` name must be provided by every dependency of your recipes for the given `platform`. 107 | 108 | ### `build_script` 109 | 110 | The `build_script` dictionary provides up to three (3) scripts used to build the software. On Posix systems these are each written to a shell script and executed, and on Windows these are written to a batch (.bat) scripts and executed. 111 | 112 | These three scripts are each optional, but must be named as follows:: 113 | 114 | - `configure` - Configure the build, only run the first time a build is run. This script is only run the first time you build a package and is skipped in subsequent builds so as to save compile time. If you need to re-run this step, build the recipe with the `msl build --rebuild` option. 115 | 116 | In many recipes, this step sets the "prefix" to the `{install}/{target}` directory, so the resulting binaries are automatically copied there when the `install` step runs `make install`. 117 | 118 | - `make` - Build the software. 119 | 120 | - `install` - Install the software to the `.mussels/install/{target}` directory 121 | 122 | Within the scripts, curly braces are used to identify a few special variables that you may use to reference dependencies or the install path. 123 | 124 | Variables available in Mussels 0.1 include: 125 | 126 | - `{target}` - The name of the build `target` (i.e. `host` / `x64` / `x86` or whatever you named it.) 127 | 128 | - `{build}` 129 | 130 | The build directory, found under `.mussels/cache/work/{target}/` 131 | 132 | - `{install}` 133 | 134 | The `.mussels/install/{target}` directory (default), or the directory specified when building with the `-i`/`--install` option. 135 | 136 | - `{includes}` 137 | 138 | Shorthand for the `{install}/include` directory. 139 | 140 | - `{libs}` 141 | 142 | Shorthand for the `{install}/lib` directory. 143 | 144 | ### `dependencies` 145 | 146 | The `dependencies` list may either be empty (`[]`), meaning no dependencies, or may be a list of other recipes names with version numbers and even cookbooks specified if so desired. 147 | 148 | Some fictional examples: 149 | 150 | ```yaml 151 | dependencies: 152 | - meepioux 153 | - blarghus>=1.2.3 154 | - wheeple@0.2.0 155 | - pyplo==5.1.0g 156 | - "scrapbook:sasquatch<2.0.0" 157 | - "scrapbook:minnow<0.1.12" 158 | ``` 159 | 160 | ### `install_paths` 161 | 162 | The `install_paths` provides lists of files and directories to be copied to a specific path under `{install}`. 163 | 164 | ### `patches` (optional) 165 | 166 | This optional field provides the name of a directory provided alongside the recipe YAML file that contains a patch set to be applied to the source before any of the build scripts are run. 167 | 168 | ### `required_tools` 169 | 170 | This list of required tools defines which tools much be present on the host platform in order to do the build. Like the `dependencies`, these lists may include version numbers and cookbook names in the format: 171 | 172 | `[cookbook:]name[>=,<=,>,<,(==|=|@)version]` 173 | 174 | Some fictional examples: 175 | 176 | ```yaml 177 | required_tools: 178 | - "scrapbook:pkgfiend<1.1.1" 179 | - appmaker 180 | ``` 181 | 182 | ## Example Recipe Definition 183 | 184 | This recipe, copypasted from the `clamav` cookbook defines how to build libpcre2. 185 | 186 | Several notable things about this recipe... 187 | 188 | The recipe provides instructions for 3 platforms: Darwin, Linux, and Windows. 189 | 190 | For Windows, it provides build instructions targeting both `x64` and `x86`, but for the others, it simply provides `host`. 191 | 192 | In the `configure` script, the install "prefix" is set for the package in the Linux and Mac recipes. This is often necessary to configure the software to be used from the install directory. 193 | 194 | The Windows instructions omit the `install` script. All scripts are optional, but the `install` step is often not requires for Windows. The Darwin and Linux recipes also make use of `install_name_tool` (provided by Clang), and `patchelf` (an additional required tool) to set the RPATH for each executable so that dynamic libraries may be found at runtime. 195 | 196 | ```yaml 197 | name: pcre2 198 | version: "10.33" 199 | url: https://ftp.pcre.org/pub/pcre/pcre2-10.33.tar.gz 200 | mussels_version: "0.2" 201 | type: recipe 202 | platforms: 203 | Darwin: 204 | host: 205 | build_script: 206 | configure: | 207 | ./configure --prefix="{install}/{target}" --disable-dependency-tracking 208 | make: | 209 | make 210 | install: | 211 | make install 212 | install_name_tool -add_rpath @executable_path/../lib "{install}/lib/libpcre2-8.dylib" 213 | dependencies: 214 | - bzip2 215 | - zlib 216 | install_paths: 217 | license/pcre2: 218 | - COPYING 219 | required_tools: 220 | - make 221 | - clang 222 | Linux: 223 | host: 224 | build_script: 225 | configure: | 226 | chmod +x ./configure ./install-sh 227 | ./configure --prefix="{install}/{target}" --disable-dependency-tracking 228 | make: | 229 | make 230 | install: | 231 | make install 232 | patchelf --set-rpath '$ORIGIN/../lib' "{install}/lib/libpcre2-8.so" 233 | dependencies: 234 | - bzip2 235 | - zlib 236 | install_paths: 237 | license/pcre2: 238 | - COPYING 239 | required_tools: 240 | - make 241 | - gcc 242 | - patchelf 243 | Windows: 244 | x64: 245 | build_script: 246 | configure: | 247 | CALL cmake.exe -G "Visual Studio 15 2017 Win64" \ 248 | -DBUILD_SHARED_LIBS=ON \ 249 | -DZLIB_INCLUDE_DIR="{includes}" \ 250 | -DZLIB_LIBRARY_RELEASE="{libs}/zlibstatic.lib" \ 251 | -DBZIP2_INCLUDE_DIR="{includes}" \ 252 | -DBZIP2_LIBRARY_RELEASE="{libs}/libbz2.lib" 253 | make: | 254 | CALL cmake.exe --build . --config Release 255 | dependencies: 256 | - bzip2<1.1.0 257 | - zlib 258 | install_paths: 259 | include: 260 | - pcre2.h 261 | lib: 262 | - Release/pcre2-8.dll 263 | - Release/pcre2-8.lib 264 | license/openssl: 265 | - COPYING 266 | patches: pcre2-10.33-patches 267 | required_tools: 268 | - cmake 269 | - visualstudio>=2017 270 | x86: 271 | build_script: 272 | configure: | 273 | CALL cmake.exe -G "Visual Studio 15 2017" \ 274 | -DBUILD_SHARED_LIBS=ON \ 275 | -DZLIB_INCLUDE_DIR="{includes}" \ 276 | -DZLIB_LIBRARY_RELEASE="{libs}/zlibstatic.lib" \ 277 | -DBZIP2_INCLUDE_DIR="{includes}" \ 278 | -DBZIP2_LIBRARY_RELEASE="{libs}/libbz2.lib" 279 | make: | 280 | CALL cmake.exe --build . --config Release 281 | dependencies: 282 | - bzip2<1.1.0 283 | - zlib 284 | install_paths: 285 | include: 286 | - pcre2.h 287 | lib: 288 | - Release/pcre2-8.dll 289 | - Release/pcre2-8.lib 290 | license/openssl: 291 | - COPYING 292 | patches: pcre2-10.33-patches 293 | required_tools: 294 | - cmake 295 | - visualstudio>=2017 296 | ``` 297 | -------------------------------------------------------------------------------- /docs/tools.md: -------------------------------------------------------------------------------- 1 | # Tools 2 | 3 | Tools are simple YAML files that must adhere to the following format: 4 | 5 | ```yaml 6 | name: template_tool 7 | version: "0.1" 8 | mussels_version: "0.1" 9 | type: tool 10 | platforms: 11 | : 12 | path_checks: 13 | - template_tool 14 | command_checks: 15 | - command: "" 16 | output_has: "" 17 | file_checks: 18 | - 19 | - 20 | ``` 21 | 22 | ## Tool Fields, in Detail 23 | 24 | ### `name` 25 | 26 | The name of the program this tool detects. 27 | 28 | _Tip_: Mussels tool names may not include the following characters: `-`, `=`, `@`, `>`, `<`, `:`. Instead, consider the humble underscore `_`. 29 | 30 | ### `version` 31 | 32 | The tool version _string_ is generally expected to follow traditional semantic versioning practices (i.e `".."`), though any alpha-numeric version string should be fine. So long as the format is consistent across multiple versions, Mussels should be able to compare version strings for a given tool. 33 | 34 | Tools, unlike recipes, may omit the version number if no specific version is needed. 35 | 36 | ### `mussels_version` 37 | 38 | This version string defines which version of Musssels the tool is written to work with. It is also the key used by Mussels to differentiate Mussels YAML files from any other YAML file. 39 | 40 | The value must be `"0.1"` 41 | 42 | ### `type` 43 | 44 | Tool type must be set to `tool`: 45 | 46 | ### `platforms` 47 | 48 | The platforms dictionary allows you to define instructions to identify tools for different host platforms all in one file. 49 | 50 | The `` keys under `platforms` may be one of the following^: 51 | 52 | - Darwin / macOS / OSX 53 | - Windows 54 | - Linux 55 | - Unix ( Darwin, FreeBSD, OpenBSD, SunOS, AIX, HP-UX ) 56 | - Posix ( Unix and Linux ) 57 | - FreeBSD 58 | - OpenBSD 59 | - SunOS 60 | - AIX 61 | - HP-UX 62 | 63 | ### Checks 64 | 65 | There are 3 ways to check if a tool exists. If any one of these pass, then the tool is "detected": 66 | 67 | - `path_checks`: 68 | 69 | The `path_checks` are a way to check if an executable exists in the PATH. This check will look in the pATH directories for the executable name. If found, the check passes. 70 | 71 | - `command_checks` 72 | 73 | The `command_checks` are a way to run a `command` and verify the result. If the `command` exit code is `0` and the `output_has` string is found within the `command` output, then the check passes. 74 | 75 | `output_has` may be an empty string, in which case only the exit code needs to be `0` for the check to pass. 76 | 77 | - `file_checks` 78 | 79 | The `file_checks` are a list of absolute paths to executables. If any one of these exist, then the tool will be "detected". At build time, the path where the executable was found will be added to the PATH environment variable. 80 | 81 | ## Example Tool Definition 82 | 83 | This tool, copypasted from the `scrapbook` defines how to find CMake. 84 | 85 | Several notable things about this tool... 86 | 87 | The tool provides instructions for 2 platforms: Posix, and Windows. Posix was chosen rather than something more specific because the means to identify CMake on POSIX systems are generally the same. 88 | 89 | For Windows, it this tool definition merely checks for the existance of the `cmake.exe` program. If found, the directory will be added to the `%PATH%` environment variable at build time. 90 | 91 | For Posix systems, this tool will check first if `cmake` is in the `$PATH` already. If that fails, it will check if the executable `cmake` exists in either `/usr/local/bin` or `/usr/bin`. As with the Windows platform, it would add the directories to the `$PATH` at build time, if found - though in this case those directories will probably already be in the `$PATH`. 92 | 93 | ```yaml 94 | name: cmake 95 | version: "" 96 | mussels_version: "0.1" 97 | type: tool 98 | platforms: 99 | Posix: 100 | path_checks: 101 | - cmake 102 | file_checks: 103 | - /usr/local/bin/cmake 104 | - /usr/bin/cmake 105 | Windows: 106 | file_checks: 107 | - C:\/Program Files/CMake/bin/cmake.exe 108 | ``` 109 | 110 | If a specific version of CMake was needed, a tool definition could be written to identify it using the `command_checks` field. 111 | 112 | For example: 113 | 114 | ```yaml 115 | name: cmake 116 | version: "3.14" 117 | mussels_version: "0.1" 118 | type: tool 119 | platforms: 120 | Posix: 121 | command_checks: 122 | - command: "cmake --version" 123 | output_has: "cmake version 3.14" 124 | Windows: 125 | command_checks: 126 | - command: "cmake.exe --version" 127 | output_has: "cmake version 3.14" 128 | - command: "C:\/Program Files/CMake/bin/cmake.exe --version" 129 | output_has: "cmake version 3.14" 130 | ``` 131 | -------------------------------------------------------------------------------- /docs/usage.md: -------------------------------------------------------------------------------- 1 | # Usage 2 | 3 | - [Usage](#usage) 4 | - [Search for recipes](#search-for-recipes) 5 | - [Build a recipe](#build-a-recipe) 6 | - [Create your own recipes](#create-your-own-recipes) 7 | - [Create your own cookbook](#create-your-own-cookbook) 8 | - [To use a local cookbook directory](#to-use-a-local-cookbook-directory) 9 | - [To use a private cookbook repository](#to-use-a-private-cookbook-repository) 10 | 11 | ## Search for recipes 12 | 13 | Download recipes from public sources: 14 | 15 | > `mussels update` 16 | 17 | or: 18 | 19 | > `msl update` 20 | 21 | View recipes available for your current platform: 22 | 23 | > `msl list` 24 | > 25 | > `msl list -V` (verbose) 26 | 27 | Many Mussels commands may be shortened to save keystrokes. For example, the following are all equivalent: 28 | 29 | > `msl list` 30 | > 31 | > `msl lis` 32 | > 33 | > `msl li` 34 | > 35 | > `msl l` 36 | 37 | View recipes for available for _all_ platforms: 38 | 39 | > `msl list -a` 40 | > 41 | > `msl list -a -V` 42 | 43 | Show details about a specific recipe: 44 | 45 | > `msl show openssl` 46 | > 47 | > `msl show openssl -V` (verbose) 48 | 49 | ## Build a recipe 50 | 51 | Perform a dry-run to view order in which dependency graph will be build a specific recipe: 52 | 53 | > `msl build openssl -d` 54 | 55 | Build a specific version of a recipe from a specific cookbook: 56 | 57 | > `msl build openssl -v 1.1.0j -c clamav` 58 | 59 | ## Create your own recipes 60 | 61 | A recipe is just a YAML file containing metadata about where to find, and how to build, a specific version of a given project. The easiest way to create your own recipe is to copy an existing recipe. 62 | 63 | Use the `list` command to find a recipe you would like to use as a starting point: 64 | 65 | > `msl list -a -V` 66 | 67 | Once you've chosen a recipe, copy it to your current working directory with the `clone` command. For example: 68 | 69 | > `msl clone nghttp2` 70 | > 71 | > `ls -la` 72 | 73 | _Tip_: If the recipe requires one or more patch-sets to build, the corresponding patch directories will also be copied to your current working directory. 74 | 75 | Now rename the cloned recipe to whatever you like and start making changes! So long as you keep the `.yaml` extension, Mussels will still recognize it. 76 | 77 | _Tip_: When testing your recipes, the recipes must be in, or in a subdirectory of, your current working directory in order for Mussels to find them. Use `msl list -a -V` to display all current recipes. Recipes found in the current working directory will show up as being provided by the "local" cookbook. Use `msl show -V` to view more information about a specific recipe. 78 | 79 | ## Create your own cookbook 80 | 81 | Simply put, a cookbook is a Git repository that contains Mussels recipe files and/or Mussels tool files. The structure of the cookbook is up to the project owners as is the naming convention for recipe and tool files. To identify recipes and tools, Mussels will search every YAML file in the repository for files containing `mussels_version` key. 82 | 83 | Cookbooks are a way for users to curate recipes to build their project without relying on recipes provided by others where changes may inadvertantly break their build. As cookbooks are privately owned, their owners are free to copyright and license the recipe and tool definitions within as they see fit. 84 | 85 | The Mussels project maintains [an index](mussels/bookshelf.py) of cookbooks provided by third-parties. Cookbook authors are encouraged to add their cookbook to the index by submitting a pull-request to the Mussels project. However, we ask that each cookbook's license must be compatible with the Apache v2.0 license used by Mussels in order to be included in the index. 86 | 87 | You don't need to add your cookbook to the public index in order to use it. 88 | 89 | ### To use a local cookbook directory 90 | 91 | Simply `cd` to your cookbook directory and execute `mussels` commands in that directory for it to detect the "local" cookbook. 92 | 93 | ### To use a private cookbook repository 94 | 95 | Run: 96 | 97 | > `msl cookbook add private ` 98 | 99 | This will add the Git URL for your cookbook to your global Mussels config. Mussels will record your cookbook in the index on your local machine. 100 | 101 | In the above example we used the name `private` but you're free to choose any name for your cookbook. 102 | 103 | Then run: 104 | 105 | > `msl update` 106 | 107 | Mussels will clone the repository in your `~/.mussels` directory and the recipes will be available for use. 108 | 109 | Now you should be able to the recipes and tools provided by your cookbook with: 110 | 111 | > `msl cook show private -V` 112 | 113 | and reference your cookbook's recipes with: 114 | 115 | > `msl show my_recipe -c private` 116 | > 117 | > `msl build my_recipe -c private -d` 118 | -------------------------------------------------------------------------------- /images/mussels-500.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cisco-Talos/Mussels/8621c1c2a7de9a012f1a71e1c97d3bb7f6ad63d8/images/mussels-500.png -------------------------------------------------------------------------------- /images/mussels.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cisco-Talos/Mussels/8621c1c2a7de9a012f1a71e1c97d3bb7f6ad63d8/images/mussels.png -------------------------------------------------------------------------------- /mussels/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cisco-Talos/Mussels/8621c1c2a7de9a012f1a71e1c97d3bb7f6ad63d8/mussels/__init__.py -------------------------------------------------------------------------------- /mussels/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | r""" 4 | __ __ __ __ ______ ______ ______ __ ______ 5 | /\ "-./ \ /\ \/\ \ /\ ___\ /\ ___\ /\ ___\ /\ \ /\ ___\ 6 | \ \ \-./\ \ \ \ \_\ \ \ \___ \ \ \___ \ \ \ __\ \ \ \____ \ \___ \ 7 | \ \_\ \ \_\ \ \_____\ \/\_____\ \/\_____\ \ \_____\ \ \_____\ \/\_____\ 8 | \/_/ \/_/ \/_____/ \/_____/ \/_____/ \/_____/ \/_____/ \/_____/ 9 | """ 10 | 11 | _description = """ 12 | A tool to download, build, and assemble application dependencies. 13 | Brought to you by the Clam AntiVirus Team. 14 | """ 15 | _copyright = """ 16 | Copyright (C) 2019-2020 Cisco Systems, Inc. and/or its affiliates. All rights reserved. 17 | """ 18 | 19 | """ 20 | Author: Micah Snyder 21 | 22 | Licensed under the Apache License, Version 2.0 (the "License"); 23 | you may not use this file except in compliance with the License. 24 | You may obtain a copy of the License at 25 | 26 | http://www.apache.org/licenses/LICENSE-2.0 27 | 28 | Unless required by applicable law or agreed to in writing, software 29 | distributed under the License is distributed on an "AS IS" BASIS, 30 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 31 | See the License for the specific language governing permissions and 32 | limitations under the License. 33 | """ 34 | 35 | import logging 36 | import os 37 | import sys 38 | 39 | import click 40 | import coloredlogs 41 | 42 | import pkg_resources 43 | from mussels.mussels import Mussels 44 | from mussels.utils.click import MusselsModifier, ShortNames 45 | 46 | logging.basicConfig() 47 | module_logger = logging.getLogger("mussels") 48 | coloredlogs.install(level="DEBUG", fmt="%(asctime)s %(name)s %(levelname)s %(message)s") 49 | module_logger.setLevel(logging.DEBUG) 50 | 51 | from colorama import Fore, Back, Style 52 | 53 | # 54 | # CLI Interface 55 | # 56 | @click.group( 57 | cls=MusselsModifier, 58 | epilog=Fore.BLUE 59 | + __doc__ 60 | + Fore.GREEN 61 | + _description 62 | + f"\nVersion {pkg_resources.get_distribution('mussels').version}\n" 63 | + Style.RESET_ALL 64 | + _copyright, 65 | ) 66 | def cli(): 67 | pass 68 | 69 | 70 | @cli.group(cls=ShortNames, help="Commands that operate on cookbooks.") 71 | def cookbook(): 72 | pass 73 | 74 | 75 | @cookbook.command("list") 76 | @click.option( 77 | "--verbose", "-V", is_flag=True, default=False, help="Verbose output. [optional]" 78 | ) 79 | def cookbook_list(verbose: bool): 80 | """ 81 | Print the list of all known cookbooks. 82 | """ 83 | my_mussels = Mussels(load_all_recipes=True) 84 | 85 | my_mussels.list_cookbooks(verbose) 86 | 87 | 88 | @cookbook.command("show") 89 | @click.argument("cookbook", required=True) 90 | @click.option( 91 | "--verbose", "-V", is_flag=True, default=False, help="Verbose output. [optional]" 92 | ) 93 | def cookbook_show(cookbook: str, verbose: bool): 94 | """ 95 | Show details about a specific cookbook. 96 | """ 97 | my_mussels = Mussels(load_all_recipes=True) 98 | 99 | my_mussels.show_cookbook(cookbook, verbose) 100 | 101 | 102 | @cookbook.command("update") 103 | def cookbook_update(): 104 | """ 105 | Update the cookbooks from the internet. 106 | """ 107 | my_mussels = Mussels(load_all_recipes=True) 108 | 109 | my_mussels.update_cookbooks() 110 | 111 | 112 | @cookbook.command("trust") 113 | @click.argument("cookbook", required=True) 114 | @click.option( 115 | "--yes", 116 | "-y", 117 | is_flag=True, 118 | default=False, 119 | help="Confirm trust. [required for non-interactive modes]", 120 | ) 121 | def cookbook_trust(cookbook, yes): 122 | """ 123 | Trust a cookbook. 124 | """ 125 | my_mussels = Mussels(load_all_recipes=True) 126 | 127 | if yes != True: 128 | print( 129 | f"\nDisclaimer: There is a non-zero risk when running code downloaded from the internet.\n" 130 | ) 131 | response = input( 132 | f"Are you sure you would like to trust recipes from cookbook '{cookbook}'? [N/y] " 133 | ) 134 | response = response.strip().lower() 135 | 136 | if response != "y" and response != "yes": 137 | return 138 | 139 | my_mussels.config_trust_cookbook(cookbook) 140 | 141 | 142 | @cookbook.command("add") 143 | @click.argument("cookbook", required=True) 144 | @click.option("--author", "-a", default="", help="Author or company name") 145 | @click.option("--url", "-u", default="", help="Git repository URL") 146 | @click.option( 147 | "--trust", 148 | "-t", 149 | is_flag=True, 150 | default=False, 151 | help="Add as a trusted cookbook, enabling you to build directly from this cookbook.", 152 | ) 153 | def cookbook_add(cookbook, author, url, trust): 154 | """ 155 | Add a cookbook to the list of known cookbooks. 156 | """ 157 | my_mussels = Mussels(load_all_recipes=True) 158 | 159 | my_mussels.config_add_cookbook(cookbook, author, url, trust=trust) 160 | 161 | 162 | @cookbook.command("remove") 163 | @click.argument("cookbook", required=True) 164 | def cookbook_remove(cookbook): 165 | """ 166 | Remove a cookbook from the list of known cookbooks. 167 | """ 168 | my_mussels = Mussels(load_all_recipes=True) 169 | 170 | my_mussels.config_remove_cookbook(cookbook) 171 | 172 | 173 | @cli.group(cls=ShortNames, help="Commands that operate on recipes.") 174 | def recipe(): 175 | pass 176 | 177 | 178 | @recipe.command("list") 179 | @click.option( 180 | "--verbose", "-V", is_flag=True, default=False, help="Verbose output. [optional]" 181 | ) 182 | @click.option( 183 | "--all", 184 | "-a", 185 | is_flag=True, 186 | default=False, 187 | help="List all recipes, including those for other platforms. [optional]", 188 | ) 189 | def recipe_list(verbose: bool, all: bool): 190 | """ 191 | Print the list of all known recipes. 192 | An asterisk indicates default (highest) version. 193 | """ 194 | my_mussels = Mussels(load_all_recipes=all) 195 | 196 | my_mussels.list_recipes(verbose) 197 | 198 | 199 | @recipe.command("show") 200 | @click.argument("recipe", required=True) 201 | @click.option("--version", "-v", default="", help="Version. [optional]") 202 | @click.option( 203 | "--verbose", "-V", is_flag=True, default=False, help="Verbose output. [optional]" 204 | ) 205 | @click.option( 206 | "--all", 207 | "-a", 208 | is_flag=True, 209 | default=False, 210 | help="Show all recipe variants, including those for other platforms. [optional]", 211 | ) 212 | def recipe_show(recipe: str, version: str, verbose: bool, all: bool): 213 | """ 214 | Show details about a specific recipe. 215 | """ 216 | my_mussels = Mussels(load_all_recipes=all) 217 | 218 | my_mussels.show_recipe(recipe, version, verbose) 219 | 220 | 221 | @recipe.command("clone") 222 | @click.argument("recipe", required=True) 223 | @click.option( 224 | "--version", "-v", default="", help="Specific version to clone. [optional]" 225 | ) 226 | @click.option( 227 | "--cookbook", "-c", default="", help="Specific cookbook to clone. [optional]" 228 | ) 229 | @click.option("--dest", "-d", default="", help="Destination directory. [optional]") 230 | def recipe_clone(recipe: str, version: str, cookbook: str, dest: str): 231 | """ 232 | Copy a recipe to the current directory or to a specific directory. 233 | """ 234 | my_mussels = Mussels(load_all_recipes=True) 235 | 236 | my_mussels.clone_recipe(recipe, version, cookbook, dest) 237 | 238 | 239 | @recipe.command("build") 240 | @click.argument("recipe", required=True) 241 | @click.option( 242 | "--version", 243 | "-v", 244 | default="", 245 | help="Version of recipe to build. May not be combined with @version in recipe name. [optional]", 246 | ) 247 | @click.option( 248 | "--cookbook", "-c", default="", help="Specific cookbook to use. [optional]" 249 | ) 250 | @click.option("--target", "-t", default="", help="Target architecture. [optional]") 251 | @click.option( 252 | "--dry-run", 253 | "-d", 254 | is_flag=True, 255 | help="Print out the version dependency graph without actually doing a build. [optional]", 256 | ) 257 | @click.option( 258 | "--rebuild", 259 | "-r", 260 | is_flag=True, 261 | help="Re-build a recipe, even if already built. [optional]", 262 | ) 263 | @click.option( 264 | "--install", "-i", default="", help="Install directory. [optional] Default is: ~/.mussels/install/" 265 | ) 266 | @click.option( 267 | "--work-dir", "-w", default="", help="Work directory. [optional] Default is: ~/.mussels/cache/work" 268 | ) 269 | @click.option( 270 | "--log-dir", "-l", default="", help="Log directory. [optional] Default is: ~/.mussels/logs" 271 | ) 272 | @click.option( 273 | "--download-dir", "-D", default="", help="Downloads directory. [optional] Default is: ~/.mussels/cache/downloads" 274 | ) 275 | def recipe_build( 276 | recipe: str, 277 | version: str, 278 | cookbook: str, 279 | target: str, 280 | dry_run: bool, 281 | rebuild: bool, 282 | install: str, 283 | work_dir: str, 284 | log_dir: str, 285 | download_dir: str, 286 | ): 287 | """ 288 | Download, extract, build, and install a recipe. 289 | """ 290 | 291 | my_mussels = Mussels( 292 | install_dir=install, 293 | work_dir=work_dir, 294 | log_dir=log_dir, 295 | download_dir=download_dir, 296 | ) 297 | 298 | results = [] 299 | 300 | success = my_mussels.build_recipe( 301 | recipe, version, cookbook, target, results, dry_run, rebuild 302 | ) 303 | if success == False: 304 | sys.exit(1) 305 | 306 | sys.exit(0) 307 | 308 | 309 | @cli.group(cls=ShortNames, help="Commands that operate on tools.") 310 | def tool(): 311 | pass 312 | 313 | 314 | @tool.command("list") 315 | @click.option( 316 | "--verbose", "-V", is_flag=True, default=False, help="Verbose output. [optional]" 317 | ) 318 | @click.option( 319 | "--all", 320 | "-a", 321 | is_flag=True, 322 | default=False, 323 | help="List all tools, including those for other platforms. [optional]", 324 | ) 325 | def tool_list(verbose: bool, all: bool): 326 | """ 327 | Print the list of all known tools. 328 | An asterisk indicates default (highest) version. 329 | """ 330 | my_mussels = Mussels(load_all_recipes=all) 331 | 332 | my_mussels.list_tools(verbose) 333 | 334 | 335 | @tool.command("show") 336 | @click.argument("tool", required=True) 337 | @click.option("--version", "-v", default="", help="Version. [optional]") 338 | @click.option( 339 | "--verbose", "-V", is_flag=True, default=False, help="Verbose output. [optional]" 340 | ) 341 | @click.option( 342 | "--all", 343 | "-a", 344 | is_flag=True, 345 | default=False, 346 | help="Show all tool variants, including those for other platforms. [optional]", 347 | ) 348 | def tool_show(tool: str, version: str, verbose: bool, all: bool): 349 | """ 350 | Show details about a specific tool. 351 | """ 352 | my_mussels = Mussels(load_all_recipes=all) 353 | 354 | my_mussels.show_tool(tool, version, verbose) 355 | 356 | 357 | @tool.command("clone") 358 | @click.argument("tool", required=True) 359 | @click.option( 360 | "--version", "-v", default="", help="Specific version to clone. [optional]" 361 | ) 362 | @click.option( 363 | "--cookbook", "-c", default="", help="Specific cookbook to clone. [optional]" 364 | ) 365 | @click.option("--dest", "-d", default="", help="Destination directory. [optional]") 366 | def tool_clone(tool: str, version: str, cookbook: str, dest: str): 367 | """ 368 | Copy a tool to the current directory or to a specific directory. 369 | """ 370 | my_mussels = Mussels(load_all_recipes=True) 371 | 372 | my_mussels.clone_tool(tool, version, cookbook, dest) 373 | 374 | 375 | @tool.command("check") 376 | @click.argument("tool", required=False, default="") 377 | @click.option( 378 | "--version", 379 | "-v", 380 | default="", 381 | help="Version of tool to check. May not be combined with @version in tool name. [optional]", 382 | ) 383 | @click.option( 384 | "--cookbook", "-c", default="", help="Specific cookbook to use. [optional]" 385 | ) 386 | def tool_check(tool: str, version: str, cookbook: str): 387 | """ 388 | Check if a tool is installed. 389 | """ 390 | 391 | my_mussels = Mussels() 392 | 393 | results = [] 394 | 395 | success = my_mussels.check_tool(tool, version, cookbook, results) 396 | if success == False: 397 | sys.exit(1) 398 | 399 | sys.exit(0) 400 | 401 | 402 | @cli.group(cls=ShortNames, help="Commands to clean up.") 403 | def clean(): 404 | pass 405 | 406 | 407 | @clean.command("cache") 408 | def clean_cache(): 409 | """ 410 | Clear the cache files. 411 | """ 412 | my_mussels = Mussels(load_all_recipes=True) 413 | 414 | my_mussels.clean_cache() 415 | 416 | 417 | @clean.command("install") 418 | def clean_install(): 419 | """ 420 | Clear the install files. 421 | """ 422 | my_mussels = Mussels(load_all_recipes=True) 423 | 424 | my_mussels.clean_install() 425 | 426 | 427 | @clean.command("logs") 428 | def clean_logs(): 429 | """ 430 | Clear the logs files. 431 | """ 432 | my_mussels = Mussels(load_all_recipes=True) 433 | 434 | my_mussels.clean_logs() 435 | 436 | 437 | @clean.command("all") 438 | def clean_all(): 439 | """ 440 | Clear the all files. 441 | """ 442 | my_mussels = Mussels(load_all_recipes=True) 443 | 444 | my_mussels.clean_all() 445 | 446 | 447 | # 448 | # Command Aliases 449 | # 450 | @cli.command("build") 451 | @click.argument("recipe", required=True) 452 | @click.option( 453 | "--version", 454 | "-v", 455 | default="", 456 | help="Version of recipe to build. May not be combined with @version in recipe name. [optional]", 457 | ) 458 | @click.option( 459 | "--cookbook", "-c", default="", help="Specific cookbook to use. [optional]" 460 | ) 461 | @click.option("--target", "-t", default="", help="Target architecture. [optional]") 462 | @click.option( 463 | "--dry-run", 464 | "-d", 465 | is_flag=True, 466 | help="Print out the version dependency graph without actually doing a build. [optional]", 467 | ) 468 | @click.option( 469 | "--rebuild", 470 | "-r", 471 | is_flag=True, 472 | help="Re-build a recipe, even if already built. [optional]", 473 | ) 474 | @click.option( 475 | "--install", "-i", default="", help="Install directory. [optional] Default is: ~/.mussels/install/" 476 | ) 477 | @click.option( 478 | "--work-dir", "-w", default="", help="Work directory. [optional] Default is: ~/.mussels/cache/work" 479 | ) 480 | @click.option( 481 | "--log-dir", "-l", default="", help="Log directory. [optional] Default is: ~/.mussels/logs" 482 | ) 483 | @click.option( 484 | "--download-dir", "-D", default="", help="Downloads directory. [optional] Default is: ~/.mussels/cache/downloads" 485 | ) 486 | @click.pass_context 487 | def build_alias( 488 | ctx, 489 | recipe: str, 490 | version: str, 491 | cookbook: str, 492 | target: str, 493 | dry_run: bool, 494 | rebuild: bool, 495 | install: str, 496 | work_dir: str, 497 | log_dir: str, 498 | download_dir: str, 499 | ): 500 | """ 501 | Download, extract, build, and install a recipe. 502 | 503 | This is just an alias for `recipe build`. 504 | """ 505 | ctx.forward(recipe_build) 506 | 507 | 508 | @cli.command("list") 509 | @click.pass_context 510 | @click.option( 511 | "--verbose", "-V", is_flag=True, default=False, help="Verbose output. [optional]" 512 | ) 513 | @click.option( 514 | "--all", 515 | "-a", 516 | is_flag=True, 517 | default=False, 518 | help="List all recipes, including those for other platforms. [optional]", 519 | ) 520 | def list_alias(ctx, verbose: bool, all: bool): 521 | """ 522 | List a list of recipes you can build on this platform. 523 | 524 | This is just an alias for `recipe list`. 525 | """ 526 | ctx.forward(recipe_list) 527 | 528 | 529 | @cli.command("show") 530 | @click.pass_context 531 | @click.argument("recipe", required=True) 532 | @click.option("--version", "-v", default="", help="Version. [optional]") 533 | @click.option( 534 | "--verbose", "-V", is_flag=True, default=False, help="Verbose output. [optional]" 535 | ) 536 | @click.option( 537 | "--all", 538 | "-a", 539 | is_flag=True, 540 | default=False, 541 | help="Show all recipe variants, including those for other platforms. [optional]", 542 | ) 543 | def show_alias(ctx, recipe: str, version: str, verbose: bool, all: bool): 544 | """ 545 | Show details about a specific recipe. 546 | 547 | This is just an alias for `recipe show`. 548 | """ 549 | ctx.forward(recipe_show) 550 | 551 | 552 | @cli.command("update") 553 | @click.pass_context 554 | def update_alias(ctx): 555 | """ 556 | Update local copy of cookbooks (using Git). 557 | 558 | This is just an alias for `recipe show`. 559 | """ 560 | ctx.forward(cookbook_update) 561 | 562 | 563 | if __name__ == "__main__": 564 | sys.argv[0] = "mussels" 565 | cli(sys.argv[1:]) 566 | -------------------------------------------------------------------------------- /mussels/bookshelf.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2019-2020 Cisco Systems, Inc. and/or its affiliates. All rights reserved. 3 | 4 | This module provides a list of public Mussels recipe cookbooks. 5 | To have your cookbook added to the list, create an issue here: 6 | 7 | https://github.com/Cisco-Talos/Mussels/issues 8 | 9 | Licensed under the Apache License, Version 2.0 (the "License"); 10 | you may not use this file except in compliance with the License. 11 | You may obtain a copy of the License at 12 | 13 | http://www.apache.org/licenses/LICENSE-2.0 14 | 15 | Unless required by applicable law or agreed to in writing, software 16 | distributed under the License is distributed on an "AS IS" BASIS, 17 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | See the License for the specific language governing permissions and 19 | limitations under the License. 20 | """ 21 | 22 | cookbooks = { 23 | "scrapbook": { 24 | "author": "Cisco", 25 | "url": "https://github.com/Cisco-Talos/mussels-recipe-scrapbook.git", 26 | }, 27 | "clamav": { 28 | "author": "Cisco", 29 | "url": "https://github.com/Cisco-Talos/clamav-mussels-cookbook.git", 30 | }, 31 | } 32 | -------------------------------------------------------------------------------- /mussels/mussels.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2019-2020 Cisco Systems, Inc. and/or its affiliates. All rights reserved. 3 | 4 | This module provides the core Mussels class, used by the CLI interface defined in __main__.py 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | """ 18 | 19 | from collections import defaultdict 20 | from pathlib import Path 21 | 22 | import datetime 23 | import fnmatch 24 | import json 25 | import logging 26 | import os 27 | import platform 28 | import shutil 29 | import sys 30 | import time 31 | from typing import * 32 | 33 | if platform.system() == "Windows": 34 | if not r"c:\program files\git\cmd" in os.environ["PATH"].lower(): 35 | os.environ["PATH"] = os.environ["PATH"] + r";C:\Program Files\Git\cmd" 36 | if not r"c:\program files\git\mingw64\bin" in os.environ["PATH"].lower(): 37 | os.environ["PATH"] = os.environ["PATH"] + r";C:\Program Files\Git\mingw64\bin" 38 | if not r"c:\program files\git\usr\bin" in os.environ["PATH"].lower(): 39 | os.environ["PATH"] = os.environ["PATH"] + r";C:\Program Files\Git\usr\bin" 40 | if not r"c:\program files\git\bin" in os.environ["PATH"].lower(): 41 | os.environ["PATH"] = os.environ["PATH"] + r";C:\Program Files\Git\bin" 42 | import git 43 | import yaml 44 | 45 | import mussels.bookshelf 46 | import mussels.recipe 47 | import mussels.tool 48 | from mussels.utils.versions import ( 49 | NVC, 50 | nvc_str, 51 | sort_cookbook_by_version, 52 | version_keys, 53 | get_item_version, 54 | platform_is, 55 | platform_matches, 56 | pick_platform, 57 | ) 58 | 59 | 60 | class Mussels: 61 | config: dict = {} 62 | cookbooks: defaultdict = defaultdict(dict) 63 | 64 | recipes: defaultdict = defaultdict(dict) 65 | sorted_recipes: dict = {} 66 | 67 | tools: defaultdict = defaultdict(dict) 68 | sorted_tools: dict = {} 69 | 70 | log_level: str 71 | 72 | def __init__( 73 | self, 74 | load_all_recipes: bool = False, 75 | data_dir: str = os.path.join(str(Path.home()), ".mussels"), 76 | install_dir: str = "", 77 | work_dir: str = "", 78 | log_dir: str = "", 79 | download_dir: str = "", 80 | log_level: str = "DEBUG", 81 | ) -> None: 82 | """ 83 | Mussels class. 84 | 85 | Args: 86 | data_dir: path where ClamAV should be installed. 87 | log_file: path output log. 88 | log_level: log level ("DEBUG", "INFO", "WARNING", "ERROR"). 89 | """ 90 | if log_dir != "": 91 | self.log_file = os.path.join(log_dir, "mussels.log") 92 | else: 93 | self.log_file = os.path.join(data_dir, "logs", "mussels.log") 94 | self.log_level = log_level 95 | self._init_logging(log_level) 96 | 97 | self.app_data_dir = data_dir 98 | if install_dir == "": 99 | self.install_dir = os.path.join(self.app_data_dir, "install") 100 | self.custom_install_dir = False 101 | else: 102 | self.install_dir = os.path.abspath(install_dir) 103 | self.custom_install_dir = True 104 | 105 | self.work_dir = "" if work_dir == "" else os.path.abspath(work_dir) 106 | self.log_dir = "" if log_dir == "" else os.path.abspath(log_dir) 107 | self.download_dir = "" if download_dir == "" else os.path.abspath(download_dir) 108 | 109 | self._load_config("cookbooks.json", self.cookbooks) 110 | self._load_recipes(all=load_all_recipes) 111 | 112 | def _init_logging(self, level="DEBUG"): 113 | """ 114 | Initializes the logging parameters 115 | 116 | Returns: nothing 117 | """ 118 | levels = { 119 | "DEBUG": logging.DEBUG, 120 | "INFO": logging.INFO, 121 | "WARN": logging.WARNING, 122 | "WARNING": logging.WARNING, 123 | "ERROR": logging.ERROR, 124 | } 125 | 126 | self.logger = logging.getLogger("Mussels") 127 | self.logger.setLevel(levels[level]) 128 | 129 | formatter = logging.Formatter( 130 | fmt="%(asctime)s - %(levelname)s: %(message)s", 131 | datefmt="%m/%d/%Y %I:%M:%S %p", 132 | ) 133 | 134 | if not os.path.exists(os.path.split(self.log_file)[0]): 135 | os.makedirs(os.path.split(self.log_file)[0]) 136 | self.filehandler = logging.FileHandler(filename=self.log_file) 137 | self.filehandler.setLevel(levels[level]) 138 | self.filehandler.setFormatter(formatter) 139 | 140 | self.logger.addHandler(self.filehandler) 141 | 142 | def _load_config(self, filename, config) -> bool: 143 | """ 144 | Load the cache. 145 | """ 146 | # load config, if exists. 147 | try: 148 | with open( 149 | os.path.join(self.app_data_dir, "config", filename), "r" 150 | ) as config_file: 151 | config.update(json.load(config_file)) 152 | except Exception: 153 | # No existing config to load, that's probaby ok, but return false to indicate the failure. 154 | return False 155 | 156 | return True 157 | 158 | def _store_config(self, filename, config) -> bool: 159 | """ 160 | Update the cache. 161 | """ 162 | try: 163 | if not os.path.isdir(os.path.join(self.app_data_dir, "config")): 164 | os.makedirs(os.path.join(self.app_data_dir, "config")) 165 | except Exception as exc: 166 | self.logger.warning(f"Failed to create config directory. Exception: {exc}") 167 | return False 168 | 169 | try: 170 | with open( 171 | os.path.join(self.app_data_dir, "config", filename), "w" 172 | ) as config_file: 173 | json.dump(config, config_file, indent=4) 174 | except Exception as exc: 175 | self.logger.warning(f"Failed to update config. Exception: {exc}") 176 | return False 177 | 178 | return True 179 | 180 | def load_directory(self, cookbook: str, load_path: str) -> tuple: 181 | """ 182 | Load all recipes and tools in a directory. 183 | This function reads in YAML files and assigns each to a new Recipe or Tool class, accordingly. 184 | The classes are returned in a tuple. 185 | """ 186 | minimum_version = "0.1" 187 | recipes = defaultdict(dict) 188 | tools = defaultdict(dict) 189 | 190 | if not os.path.exists(load_path): 191 | return recipes, tools 192 | 193 | for root, dirs, filenames in os.walk(load_path): 194 | for fname in filenames: 195 | if not fname.endswith(".yaml"): 196 | continue 197 | fpath = os.path.abspath(os.path.join(root, fname)) 198 | with open(fpath, "r") as fd: 199 | try: 200 | yaml_file = yaml.load(fd.read(), Loader=yaml.SafeLoader) 201 | except Exception as exc: 202 | self.logger.warning(f"Failed to load YAML file: {fpath}") 203 | self.logger.warning(f"Exception occured: \n{exc}") 204 | continue 205 | if yaml_file == None: 206 | continue 207 | 208 | if ( 209 | "mussels_version" in yaml_file 210 | and yaml_file["mussels_version"] >= minimum_version 211 | ): 212 | if not "type" in yaml_file: 213 | self.logger.warning(f"Failed to load recipe: {fpath}") 214 | self.logger.warning(f"Missing required 'type' field.") 215 | continue 216 | 217 | if ( 218 | yaml_file["type"] == "recipe" 219 | or yaml_file["type"] == "collection" 220 | ): 221 | if not "name" in yaml_file: 222 | self.logger.warning(f"Failed to load recipe: {fpath}") 223 | self.logger.warning(f"Missing required 'name' field.") 224 | continue 225 | name = f"{cookbook}__{yaml_file['name']}" 226 | 227 | if not "version" in yaml_file: 228 | self.logger.warning(f"Failed to load recipe: {fpath}") 229 | self.logger.warning( 230 | f"Missing required 'version' field." 231 | ) 232 | continue 233 | else: 234 | name = f"{name}_{yaml_file['version']}" 235 | 236 | recipe_class = type( 237 | name, 238 | (mussels.recipe.BaseRecipe,), 239 | {"__doc__": f"{yaml_file['name']} recipe class."}, 240 | ) 241 | 242 | recipe_class.module_file = fpath 243 | 244 | recipe_class.name = yaml_file["name"] 245 | 246 | recipe_class.version = yaml_file["version"] 247 | 248 | if yaml_file["type"] == "collection": 249 | recipe_class.is_collection = True 250 | else: 251 | recipe_class.is_collection = False 252 | 253 | if not "url" in yaml_file: 254 | self.logger.warning( 255 | f"Failed to load recipe: {fpath}" 256 | ) 257 | self.logger.warning( 258 | f"Missing required 'url' field." 259 | ) 260 | continue 261 | else: 262 | recipe_class.url = yaml_file["url"] 263 | 264 | if "archive_name_change" in yaml_file: 265 | recipe_class.archive_name_change = ( 266 | yaml_file["archive_name_change"][0], 267 | yaml_file["archive_name_change"][1], 268 | ) 269 | 270 | if not "platforms" in yaml_file: 271 | self.logger.warning(f"Failed to load recipe: {fpath}") 272 | self.logger.warning( 273 | f"Missing required 'platforms' field." 274 | ) 275 | continue 276 | else: 277 | recipe_class.platforms = yaml_file["platforms"] 278 | 279 | recipes[recipe_class.name][ 280 | recipe_class.version 281 | ] = recipe_class 282 | 283 | elif yaml_file["type"] == "tool": 284 | if not "name" in yaml_file: 285 | self.logger.warning(f"Failed to load tool: {fpath}") 286 | self.logger.warning(f"Missing required 'name' field.") 287 | continue 288 | name = f"{cookbook}__{yaml_file['name']}" 289 | 290 | if "version" in yaml_file: 291 | name = f"{name}_{yaml_file['version']}" 292 | 293 | tool_class = type( 294 | name, 295 | (mussels.tool.BaseTool,), 296 | {"__doc__": f"{yaml_file['name']} tool class."}, 297 | ) 298 | 299 | tool_class.module_file = fpath 300 | 301 | tool_class.name = yaml_file["name"] 302 | 303 | if "version" in yaml_file: 304 | tool_class.version = yaml_file["version"] 305 | 306 | if not "platforms" in yaml_file: 307 | self.logger.warning(f"Failed to load tool: {fpath}") 308 | self.logger.warning( 309 | f"Missing required 'platforms' field." 310 | ) 311 | continue 312 | else: 313 | tool_class.platforms = yaml_file["platforms"] 314 | 315 | tools[tool_class.name][tool_class.version] = tool_class 316 | 317 | return recipes, tools 318 | 319 | def _read_cookbook(self, cookbook: str, cookbook_path: str) -> bool: 320 | """ 321 | Load the recipes and tools from a single cookbook. 322 | """ 323 | 324 | sorted_recipes: defaultdict = defaultdict(list) 325 | sorted_tools: defaultdict = defaultdict(list) 326 | 327 | # Load the recipes and the tools 328 | recipes, tools = self.load_directory( 329 | cookbook=cookbook, load_path=os.path.join(cookbook_path) 330 | ) 331 | 332 | # Sort the recipes 333 | sorted_recipes = sort_cookbook_by_version(recipes) 334 | 335 | if len(sorted_recipes) > 0: 336 | self.cookbooks[cookbook]["recipes"] = sorted_recipes 337 | for recipe in recipes.keys(): 338 | for version in recipes[recipe]: 339 | if version not in self.recipes[recipe].keys(): 340 | self.recipes[recipe][version] = {} 341 | self.recipes[recipe][version][cookbook] = recipes[recipe][version] 342 | 343 | # Sort the tools 344 | sorted_tools = sort_cookbook_by_version(tools) 345 | 346 | if len(sorted_tools) > 0: 347 | self.cookbooks[cookbook]["tools"] = sorted_tools 348 | for tool in tools.keys(): 349 | for version in tools[tool]: 350 | if version not in self.tools[tool].keys(): 351 | self.tools[tool][version] = {} 352 | self.tools[tool][version][cookbook] = tools[tool][version] 353 | 354 | if len(recipes) == 0 and len(tools) == 0: 355 | return False 356 | 357 | if "trusted" not in self.cookbooks[cookbook]: 358 | self.cookbooks[cookbook]["trusted"] = False 359 | 360 | return True 361 | 362 | def _read_bookshelf(self) -> bool: 363 | """ 364 | Load the recipes and tools from cookbooks in ~/.mussels/cookbooks 365 | """ 366 | bookshelf = os.path.join(self.app_data_dir, "cookbooks") 367 | if os.path.isdir(bookshelf): 368 | for cookbook in os.listdir(bookshelf): 369 | cookbook_path = os.path.join( 370 | os.path.join(self.app_data_dir, "cookbooks"), cookbook 371 | ) 372 | if os.path.isdir(cookbook_path): 373 | if not self._read_cookbook(cookbook, cookbook_path): 374 | self.logger.warning( 375 | f"Failed to read any recipes or tools from cookbook: {cookbook}" 376 | ) 377 | 378 | self._store_config("cookbooks.json", self.cookbooks) 379 | 380 | return True 381 | 382 | def _read_local_recipes(self) -> bool: 383 | """ 384 | Load the recipes and tools from local "mussels" directory 385 | """ 386 | # Load recipes and tools from `cwd` directory, if any exist. 387 | local_recipes = os.path.join(os.getcwd()) 388 | if os.path.isdir(local_recipes): 389 | if not self._read_cookbook("local", local_recipes): 390 | return False 391 | 392 | self.cookbooks["local"]["url"] = "" 393 | self.cookbooks["local"]["path"] = local_recipes 394 | self.cookbooks["local"]["trusted"] = True 395 | 396 | return True 397 | 398 | def _sort_items_by_version( 399 | self, items: defaultdict, all: bool, has_target: bool = False 400 | ) -> dict: 401 | """ 402 | Sort recipes, and determine the highest versions. 403 | Only includes trusted recipes for current platform. 404 | """ 405 | sorted_items: dict = {} 406 | 407 | for item in items: 408 | versions_list = list(items[item].keys()) 409 | versions_list.sort(key=version_keys) 410 | versions_list.reverse() 411 | 412 | sorted_item_list = [] 413 | 414 | for version in versions_list: 415 | found_good_version = False 416 | item_version = {"version": version, "cookbooks": {}} 417 | 418 | for each_cookbook in items[item][version].keys(): 419 | if not all and not self.cookbooks[each_cookbook]["trusted"]: 420 | continue 421 | 422 | cookbook: dict = {} 423 | 424 | for each_platform in items[item][version][each_cookbook].platforms: 425 | if not all and not platform_is(each_platform): 426 | continue 427 | 428 | if has_target: 429 | cookbook[each_platform] = [ 430 | target 431 | for target in items[item][version][each_cookbook] 432 | .platforms[each_platform] 433 | .keys() 434 | ] 435 | else: 436 | cookbook[each_platform] = [] 437 | 438 | found_good_version = True 439 | 440 | item_version["cookbooks"][each_cookbook] = cookbook 441 | 442 | if found_good_version: 443 | sorted_item_list.append(item_version) 444 | 445 | if len(sorted_item_list) > 0: 446 | sorted_items[item] = sorted_item_list 447 | 448 | return sorted_items 449 | 450 | def _load_recipes(self, all: bool = False) -> bool: 451 | """ 452 | Load the recipes and tools. 453 | """ 454 | # If the cache is empty, try reading from the local bookshelf. 455 | if len(self.recipes) == 0 or len(self.tools) == 0: 456 | self._read_bookshelf() 457 | 458 | # Load recipes from the local mussels directory, if those exists. 459 | if not self._read_local_recipes() and "local" in self.cookbooks: 460 | self.cookbooks.pop("local") 461 | 462 | if len(self.recipes) == 0: 463 | return False 464 | 465 | self.sorted_recipes = self._sort_items_by_version( 466 | self.recipes, all=all, has_target=True 467 | ) 468 | self.sorted_tools = self._sort_items_by_version(self.tools, all=all) 469 | 470 | if len(self.sorted_recipes) == 0 or len(self.sorted_tools) == 0: 471 | return False 472 | 473 | return True 474 | 475 | def _build_recipe( 476 | self, 477 | recipe: str, 478 | version: str, 479 | cookbook: str, 480 | platform: str, 481 | target: str, 482 | toolchain: dict, 483 | rebuild: bool = False, 484 | ) -> dict: 485 | """ 486 | Build a specific recipe. 487 | 488 | Args: 489 | recipe: The recipe name with no version information. 490 | version: The recipe version. 491 | 492 | Returns: A dictionary of build results 493 | """ 494 | result = {"name": recipe, "version": version, "success": False} 495 | 496 | if not self.cookbooks[cookbook]["trusted"]: 497 | self.logger.error( 498 | f"Unable to build {recipe}={version} from '{cookbook}'. You have not elected to trust '{cookbook}'" 499 | ) 500 | self.logger.error( 501 | f"Building recipes involve downloading and executing code from the internet, which carries some risk." 502 | ) 503 | self.logger.error( 504 | f"Please review the recipes provided by '{cookbook}' at: {self.cookbooks[cookbook]['url']}." 505 | ) 506 | self.logger.error( 507 | f"If you're comfortable with the level of risk, run the following command to trust all recipes from '{cookbook}':" 508 | ) 509 | self.logger.error(f"") 510 | self.logger.error(f" mussels cookbook trust {cookbook}") 511 | self.logger.error(f"") 512 | self.logger.error( 513 | f"Alternatively, you may consider cloning only the recipe you need for your own cookbook." 514 | ) 515 | self.logger.error( 516 | f"This is a safer option, though you are still encouraged to review the recipe before using it." 517 | ) 518 | self.logger.error( 519 | f"To clone the recipe {recipe}={version} from '{cookbook}', run the following command:" 520 | ) 521 | self.logger.error(f"") 522 | self.logger.error( 523 | f" mussels recipe clone {recipe} -v {version} -c {cookbook}" 524 | ) 525 | return result 526 | 527 | start = time.time() 528 | 529 | self.logger.info(f"Attempting to build {recipe}...") 530 | 531 | if version == "": 532 | # Use the default (highest) version 533 | try: 534 | version = self.sorted_recipes[recipe][0] 535 | except KeyError: 536 | self.logger.error(f"FAILED to find recipe: {recipe}!") 537 | result["time elapsed"] = time.time() - start 538 | return result 539 | 540 | try: 541 | recipe_class = self.recipes[recipe][version][cookbook] 542 | except KeyError: 543 | self.logger.error(f"FAILED to find recipe: {nvc_str(recipe, version)}!") 544 | result["time elapsed"] = time.time() - start 545 | return result 546 | 547 | # If the user specified a custom install directory, then don't add the target arch subdirectory. 548 | if self.custom_install_dir == True: 549 | install_dir = self.install_dir 550 | else: 551 | install_dir = os.path.join(self.install_dir, target) 552 | 553 | recipe_object = recipe_class( 554 | toolchain=toolchain, 555 | platform=platform, 556 | target=target, 557 | data_dir=self.app_data_dir, 558 | install_dir=install_dir, 559 | work_dir=self.work_dir, 560 | log_dir=self.log_dir, 561 | download_dir=self.download_dir, 562 | log_level=self.log_level, 563 | ) 564 | 565 | if not recipe_object.build(rebuild): 566 | self.logger.error(f"FAILURE: {nvc_str(recipe, version)} build failed!\n") 567 | else: 568 | self.logger.info( 569 | f"Success: {nvc_str(recipe, version)} build succeeded. :)\n" 570 | ) 571 | result["success"] = True 572 | 573 | result["time elapsed"] = time.time() - start 574 | 575 | return result 576 | 577 | def _get_recipe_version(self, recipe: str, platform: str, target: str) -> NVC: 578 | """ 579 | Select recipe version based on version requirements. 580 | Eliminate recipe versions and sorted tools versions based on 581 | these requirements, and the required_tools requirements of remaining recipes. 582 | 583 | Args: 584 | recipe: A specific recipe string, which may include version information. 585 | cookbook: The preferred cookbook to select the recipe from. 586 | 587 | :return: named tuple describing the highest qualified version: 588 | NVC( 589 | "name"->str, 590 | "version"->str, 591 | "cookbook"->str, 592 | ) 593 | """ 594 | # Select the recipe 595 | nvc = get_item_version(recipe, self.sorted_recipes, target, logger=self.logger) 596 | 597 | # Use "get_item_version()" to prune the list of sorted_tools based on the required tools for the selected recipe. 598 | recipe_class = self.recipes[nvc.name][nvc.version][nvc.cookbook] 599 | 600 | for each_platform in recipe_class.platforms: 601 | if platform_matches(each_platform, platform): 602 | variant = recipe_class.platforms[each_platform] 603 | if target in variant.keys(): 604 | build_target = variant[target] 605 | 606 | if "required_tools" in build_target.keys(): 607 | for tool in build_target["required_tools"]: 608 | try: 609 | get_item_version(tool, self.sorted_tools, logger=self.logger) 610 | except Exception as exc: 611 | raise Exception(f"The {tool} tool, required by {nvc_str(nvc.name, nvc.version, nvc.cookbook)} is not available...\n{exc}") 612 | break 613 | return nvc 614 | 615 | def _identify_build_recipes( 616 | self, recipe: str, chain: list, platform: str, target: str 617 | ) -> list: 618 | """ 619 | Identify all recipes that must be built given a specific recipe. 620 | 621 | Args: 622 | recipe: A specific recipe to build. 623 | chain: (in,out) A dependency chain starting from the first 624 | recursive call used to identify circular dependencies. 625 | """ 626 | recipe_nvc = self._get_recipe_version(recipe, platform, target) 627 | 628 | if (len(chain) > 0) and (recipe_nvc.name == chain[0]): 629 | raise ValueError(f"Circular dependencies found! {chain}") 630 | chain.append(recipe_nvc.name) 631 | 632 | recipes = [] 633 | 634 | recipes.append(recipe) 635 | 636 | # Verify that recipe supports current platform. 637 | platform_options = self.recipes[recipe_nvc.name][recipe_nvc.version][ 638 | recipe_nvc.cookbook 639 | ].platforms.keys() 640 | matching_platform = pick_platform(platform, platform_options) 641 | if matching_platform == "": 642 | # recipe doesn't support current platform. 643 | # TODO: see if next recipe does. 644 | pass 645 | 646 | # verify that recipe supports requested target architecture 647 | 648 | if ( 649 | "dependencies" 650 | in self.recipes[recipe_nvc.name][recipe_nvc.version][ 651 | recipe_nvc.cookbook 652 | ].platforms[matching_platform][target] 653 | ): 654 | dependencies = self.recipes[recipe_nvc.name][recipe_nvc.version][ 655 | recipe_nvc.cookbook 656 | ].platforms[matching_platform][target]["dependencies"] 657 | for dependency in dependencies: 658 | if ":" not in dependency: 659 | # If the cookbook isn't explicitly specified for the dependency, 660 | # select the recipe from the current cookbook. 661 | dependency = f"{recipe_nvc.cookbook}:{dependency}" 662 | 663 | try: 664 | recipes += self._identify_build_recipes( 665 | dependency, chain, platform, target 666 | ) 667 | except Exception as exc: 668 | raise Exception(f"The {dependency} recipe, required by {nvc_str(recipe_nvc.name, recipe_nvc.version, recipe_nvc.cookbook)} has dependency issues...\n{exc}") 669 | 670 | return recipes 671 | 672 | def _get_build_batches(self, recipe: str, platform: str, target: str) -> list: 673 | """ 674 | Get list of build batches that can be built concurrently. 675 | 676 | Args: 677 | recipe: A recipes string in the format [cookbook:]recipe[==version]. 678 | """ 679 | # Identify all recipes that must be built given list of desired builds. 680 | try: 681 | all_recipes = set(self._identify_build_recipes(recipe, [], platform, target)) 682 | except Exception as exc: 683 | raise Exception(f"Failed to assemble dependency chain for {recipe} on {platform} ({target}):\n{exc}") 684 | 685 | # Build a map of recipes (name,version) tuples to sets of dependency (name,version,cookbook) tuples 686 | nvc_to_deps = {} 687 | for recipe in all_recipes: 688 | recipe_nvc = self._get_recipe_version(recipe, platform, target) 689 | platform_options = self.recipes[recipe_nvc.name][recipe_nvc.version][ 690 | recipe_nvc.cookbook 691 | ].platforms.keys() 692 | matching_platform = pick_platform(platform, platform_options) 693 | 694 | if ( 695 | "dependencies" 696 | in self.recipes[recipe_nvc.name][recipe_nvc.version][ 697 | recipe_nvc.cookbook 698 | ].platforms[matching_platform][target] 699 | ): 700 | dependencies = self.recipes[recipe_nvc.name][recipe_nvc.version][ 701 | recipe_nvc.cookbook 702 | ].platforms[matching_platform][target]["dependencies"] 703 | else: 704 | dependencies = [] 705 | nvc_to_deps[recipe_nvc] = set( 706 | [ 707 | self._get_recipe_version(dependency, platform, target) 708 | for dependency in dependencies 709 | ] 710 | ) 711 | 712 | batches = [] 713 | 714 | # While there are dependencies to solve... 715 | while nvc_to_deps: 716 | 717 | # Get all recipes with no dependencies 718 | ready = {recipe for recipe, deps in nvc_to_deps.items() if not deps} 719 | 720 | # If there aren't any, we have a loop in the graph 721 | if not ready: 722 | msg = "Circular dependencies found!\n" 723 | msg += json.dumps(nvc_to_deps, indent=4) 724 | raise ValueError(msg) 725 | 726 | # Remove them from the dependency graph 727 | for recipe in ready: 728 | del nvc_to_deps[recipe] 729 | for deps in nvc_to_deps.values(): 730 | deps.difference_update(ready) 731 | 732 | # Add the batch to the list 733 | batches.append(ready) 734 | 735 | # Return the list of batches 736 | return batches 737 | 738 | def _select_cookbook( 739 | self, recipe: str, recipe_version: dict, preferred_book: str = "" 740 | ) -> str: 741 | """ 742 | Return the cookbook name, if only one cookbook provides the recipe-version. 743 | If more then one cookbook provides the recipe-version, explain the options and return an empty string. 744 | """ 745 | cookbook = "" 746 | 747 | num_cookbooks = len(recipe_version["cookbooks"].keys()) 748 | if num_cookbooks == 0: 749 | self.logger.error( 750 | f"Recipe {nvc_str(recipe, recipe_version['version'])} not provided by any cookbook!(?!)" 751 | ) 752 | 753 | elif num_cookbooks == 1: 754 | cookbook = next(iter(recipe_version["cookbooks"])) 755 | 756 | else: 757 | if "local" in recipe_version["cookbooks"]: 758 | # Always prefer to use a local recipe. 759 | cookbook = "local" 760 | elif preferred_book != "" and preferred_book in recipe_version["cookbooks"]: 761 | # 2nd choice is the "preferred" cookbook, which was probably the same cookbook as the target recipe. 762 | cookbook = preferred_book 763 | else: 764 | # More than one option exists, but no good excuse to choose one over another. 765 | # Bail out and ask for more specific instructions. 766 | self.logger.error( 767 | f'Failed to select a cookbook for {nvc_str(recipe, recipe_version["version"])}' 768 | ) 769 | self.logger.error( 770 | f"No cookbook specified, no local recipe exists, and no recipe exists in the same cookbook as the primary build target recipe." 771 | ) 772 | self.logger.error( 773 | f"However, multiple cookbooks do provide the recipe. Please retry with a specific cookbook using the `-c` / `--cookbook` options" 774 | ) 775 | self.logger.info(f"") 776 | 777 | self.print_recipe_details( 778 | recipe, recipe_version, verbose=True, all=True 779 | ) 780 | 781 | return cookbook 782 | 783 | def check_tool( 784 | self, 785 | tool: str, 786 | version: str, 787 | cookbook: str, 788 | results: list, 789 | ) -> bool: 790 | """ 791 | Check if a tool exists. Will check all tools if tool arg is "". 792 | 793 | Args: 794 | recipe: The recipe to build. 795 | version: A specific version to build. Leave empty ("") to build the newest. 796 | cookbook: A specific cookbook to use. Leave empty ("") if there's probably only one. 797 | results: (out) A list of dictionaries describing the results of the build. 798 | """ 799 | found_tool = False 800 | 801 | for each_tool in self.sorted_tools: 802 | if tool == "" or tool == each_tool: 803 | for each_version in self.sorted_tools[each_tool]: 804 | if version == "" or version == each_version["version"]: 805 | for each_cookbook in each_version["cookbooks"]: 806 | if cookbook == "" or cookbook == each_cookbook: 807 | found_tool = True 808 | 809 | tool_class = self.tools[each_tool][each_version["version"]][each_cookbook] 810 | tool_object = tool_class( 811 | self.app_data_dir, 812 | log_level=self.log_level, 813 | ) 814 | 815 | if tool_object.detect(): 816 | # Found! 817 | self.logger.warning( 818 | f" {nvc_str(each_tool, each_version['version'], each_cookbook)} FOUND." 819 | ) 820 | else: 821 | # Not found. 822 | self.logger.error( 823 | f" {nvc_str(each_tool, each_version['version'], each_cookbook)} NOT found." 824 | ) 825 | if not found_tool: 826 | self.logger.warning( 827 | f" Unable to find tool definition matching: {nvc_str(tool, version, cookbook)}." 828 | ) 829 | 830 | def build_recipe( 831 | self, 832 | recipe: str, 833 | version: str, 834 | cookbook: str, 835 | target: str, 836 | results: list, 837 | dry_run: bool = False, 838 | rebuild: bool = False, 839 | ) -> bool: 840 | """ 841 | Execute a build of a recipe. 842 | 843 | Args: 844 | recipe: The recipe to build. 845 | version: A specific version to build. Leave empty ("") to build the newest. 846 | cookbook: A specific cookbook to use. Leave empty ("") if there's probably only one. 847 | target: The target architecture to build. 848 | results: (out) A list of dictionaries describing the results of the build. 849 | dry_run: (optional) Don't actually build, just print the build chain. 850 | rebuild: (optional) Rebuild the entire dependency chain. 851 | """ 852 | 853 | def print_results(results: list): 854 | """ 855 | Print the build results in a pretty way. 856 | 857 | Args: 858 | results: (out) A list of dictionaries describing the results of the build. 859 | """ 860 | for result in results: 861 | if result["success"]: 862 | self.logger.info( 863 | f"Successful build of {nvc_str(result['name'], result['version'])} completed in {datetime.timedelta(0, result['time elapsed'])}." 864 | ) 865 | else: 866 | self.logger.error( 867 | f"Failure building {nvc_str(result['name'], result['version'])}, terminated after {datetime.timedelta(0, result['time elapsed'])}" 868 | ) 869 | 870 | if not recipe in self.sorted_recipes: 871 | self.logger.error(f"The recipe does not exist, or at least does not exist for the current platform ({platform.system()}") 872 | self.logger.error(f"To available recipes for your platform, run: msl list") 873 | self.logger.error(f"To all recipes for all platforms, run: msl list -a") 874 | self.logger.error(f"To download the latest recipes, run: msl update") 875 | return False 876 | 877 | 878 | batches: List[dict] = [] 879 | 880 | recipe_str = nvc_str(recipe, version, cookbook) 881 | 882 | if target == "": 883 | if platform.system() == "Windows": 884 | target = ( 885 | "x64" if os.environ["PROCESSOR_ARCHITECTURE"] == "AMD64" else "x86" 886 | ) 887 | else: 888 | target = "host" 889 | 890 | try: 891 | batches = self._get_build_batches( 892 | recipe_str, platform=platform.system(), target=target 893 | ) 894 | except Exception as exc: 895 | self.logger.error(f"{recipe_str} build failed!") 896 | for line in str(exc).split('\n'): 897 | self.logger.warning(f"{line}") 898 | return False 899 | 900 | # 901 | # Validate toolchain 902 | # 903 | # Collect set of required tools for entire build. 904 | toolchain = {} 905 | preferred_tool_versions = set() 906 | for i, bundle in enumerate(batches): 907 | for j, recipe_nvc in enumerate(bundle): 908 | recipe_class = self.recipes[recipe_nvc.name][recipe_nvc.version][ 909 | recipe_nvc.cookbook 910 | ] 911 | 912 | for each_platform in recipe_class.platforms: 913 | if platform_is(each_platform): 914 | if ( 915 | "required_tools" 916 | in recipe_class.platforms[each_platform][target].keys() 917 | ): 918 | for tool in recipe_class.platforms[each_platform][target][ 919 | "required_tools" 920 | ]: 921 | tool_nvc = get_item_version(tool, self.sorted_tools, logger=self.logger) 922 | preferred_tool_versions.add(tool_nvc) 923 | 924 | # Check if required tools are installed 925 | missing_tools = [] 926 | for tool_nvc in preferred_tool_versions: 927 | tool_found = False 928 | preferred_tool = self.tools[tool_nvc.name][tool_nvc.version][ 929 | tool_nvc.cookbook 930 | ](self.app_data_dir) 931 | 932 | if preferred_tool.detect(): 933 | # Preferred tool version is available. 934 | tool_found = True 935 | toolchain[tool_nvc.name] = preferred_tool 936 | self.logger.info( 937 | f" {nvc_str(tool_nvc.name, tool_nvc.version, tool_nvc.cookbook)} found." 938 | ) 939 | else: 940 | # Check if non-preferred (older, but compatible) version is available. 941 | self.logger.debug( 942 | f" {nvc_str(tool_nvc.name, tool_nvc.version, tool_nvc.cookbook)} not found." 943 | ) 944 | 945 | if len(self.sorted_tools[tool_nvc.name]) > 1: 946 | self.logger.debug(f" Checking for alternative versions...") 947 | alt_versions = self.sorted_tools[tool_nvc.name][1:] 948 | 949 | for alt_version in alt_versions: 950 | alt_version_cookbook = self._select_cookbook( 951 | tool_nvc.name, alt_version, cookbook 952 | ) 953 | alt_tool = self.tools[tool_nvc.name][alt_version["version"]][ 954 | alt_version_cookbook 955 | ](self.app_data_dir) 956 | 957 | if alt_tool.detect(): 958 | # Found a compatible version to use. 959 | tool_found = True 960 | toolchain[tool_nvc.name] = alt_tool 961 | 962 | # Select the exact version (pruning all other options) so it will be the default. 963 | get_item_version( 964 | f"{nvc_str(tool_nvc.name, alt_version['version'], alt_version_cookbook)}", 965 | self.sorted_tools, 966 | logger=self.logger 967 | ) 968 | self.logger.info( 969 | f" Alternative version {nvc_str(tool_nvc.name, alt_version['version'], alt_version_cookbook)} found." 970 | ) 971 | else: 972 | self.logger.debug( 973 | f" Alternative version {nvc_str(tool_nvc.name, alt_version['version'], alt_version_cookbook)} not found." 974 | ) 975 | 976 | if not tool_found: 977 | # Tool is missing. Build will fail. 978 | missing_tools.append(tool_nvc) 979 | 980 | if len(missing_tools) > 0: 981 | self.logger.warning("") 982 | self.logger.warning( 983 | "The following tools are missing and must be installed for this build to continue:" 984 | ) 985 | for tool_version in missing_tools: 986 | self.logger.warning(f" {nvc_str(tool_version.name, tool_version.version)}") 987 | 988 | sys.exit(1) 989 | 990 | self.logger.info("Toolchain:") 991 | for tool in toolchain: 992 | self.logger.info(f" {nvc_str(tool, toolchain[tool].version)}") 993 | 994 | #FF 995 | # Perform Build 996 | # 997 | if dry_run: 998 | self.logger.warning("") 999 | self.logger.warning(r" ___ ___ _ ___ _ _ ") 1000 | self.logger.warning(r" | | \ | |_) \ \_/ | |_) | | | | |\ |") 1001 | self.logger.warning(r" |_|_/ |_| \ |_| |_| \ \_\_/ |_| \|") 1002 | self.logger.warning("") 1003 | self.logger.info("Build-order of requested recipes:") 1004 | 1005 | idx = 0 1006 | failure = False 1007 | for i, bundle in enumerate(batches): 1008 | for j, recipe_nvc in enumerate(bundle): 1009 | idx += 1 1010 | 1011 | platform_options = self.recipes[recipe_nvc.name][recipe_nvc.version][ 1012 | recipe_nvc.cookbook 1013 | ].platforms.keys() 1014 | matching_platform = pick_platform(platform.system(), platform_options) 1015 | 1016 | if dry_run: 1017 | self.logger.info( 1018 | f" {idx:2} [{i}:{j:2}]: {nvc_str(recipe_nvc.name, recipe_nvc.version, recipe_nvc.cookbook)}" 1019 | ) 1020 | if ( 1021 | "required_tools" 1022 | in self.recipes[recipe_nvc.name][recipe_nvc.version][ 1023 | recipe_nvc.cookbook 1024 | ].platforms[matching_platform][target] 1025 | ): 1026 | self.logger.debug(f" Tool(s):") 1027 | for tool in self.recipes[recipe_nvc.name][recipe_nvc.version][ 1028 | recipe_nvc.cookbook 1029 | ].platforms[matching_platform][target]["required_tools"]: 1030 | tool_nvc = get_item_version(tool, self.sorted_tools, logger=self.logger) 1031 | self.logger.debug( 1032 | f" {nvc_str(tool_nvc.name, tool_nvc.version, tool_nvc.cookbook)}" 1033 | ) 1034 | continue 1035 | 1036 | if failure: 1037 | self.logger.warning( 1038 | f"Skipping {nvc_str(recipe_nvc.name, recipe_nvc.version, recipe_nvc.cookbook)} build due to prior failure." 1039 | ) 1040 | else: 1041 | result = self._build_recipe( 1042 | recipe_nvc.name, 1043 | recipe_nvc.version, 1044 | recipe_nvc.cookbook, 1045 | matching_platform, 1046 | target, 1047 | toolchain, 1048 | rebuild, 1049 | ) 1050 | results.append(result) 1051 | if not result["success"]: 1052 | failure = True 1053 | 1054 | if not dry_run: 1055 | print_results(results) 1056 | 1057 | if failure: 1058 | return False 1059 | return True 1060 | 1061 | def print_recipe_details( 1062 | self, recipe: str, version: dict, verbose: bool, all: bool 1063 | ): 1064 | """ 1065 | Print recipe information. 1066 | """ 1067 | version_num = version["version"] 1068 | cookbooks = version["cookbooks"].keys() 1069 | self.logger.info(f" {nvc_str(recipe, version_num)}; provided by cookbook(s): {list(cookbooks)}") 1070 | 1071 | if verbose: 1072 | self.logger.info("") 1073 | for cookbook in cookbooks: 1074 | self.logger.info(f" Cookbook: {cookbook}") 1075 | 1076 | book_recipe = self.recipes[recipe][version_num][cookbook] 1077 | 1078 | if book_recipe.is_collection: 1079 | self.logger.info(f" Collection: Yes") 1080 | else: 1081 | self.logger.info(f" Collection: No") 1082 | 1083 | self.logger.info(f" Platforms:") 1084 | for each_platform in book_recipe.platforms: 1085 | if all or platform_is(each_platform): 1086 | self.logger.info(f" Host platform: {each_platform}") 1087 | 1088 | variant = book_recipe.platforms[each_platform] 1089 | for arch in variant.keys(): 1090 | self.logger.info(f" Target architecture: {arch}") 1091 | self.logger.info( 1092 | f" Dependencies: {', '.join(variant[arch]['dependencies'])}" 1093 | ) 1094 | self.logger.info( 1095 | f" Required tools: {', '.join(variant[arch]['required_tools'])}" 1096 | ) 1097 | 1098 | if not all: 1099 | break 1100 | self.logger.info("") 1101 | 1102 | def show_recipe(self, recipe_match: str, version_match: str, verbose: bool = False): 1103 | """ 1104 | Search recipes for a specific recipe and print recipe details. 1105 | """ 1106 | 1107 | found = False 1108 | 1109 | if version_match == "": 1110 | self.logger.info(f'Searching for recipe matching name: "{recipe_match}"...') 1111 | else: 1112 | self.logger.info( 1113 | f'Searching for recipe matching name: "{recipe_match}", version: "{version_match}"...' 1114 | ) 1115 | # Attempt to match the recipe name 1116 | for recipe in self.sorted_recipes: 1117 | if fnmatch.fnmatch(recipe, recipe_match): 1118 | if version_match == "": 1119 | found = True 1120 | 1121 | # Show info for every version 1122 | for version in self.sorted_recipes[recipe]: 1123 | self.print_recipe_details(recipe, version, verbose, all) 1124 | break 1125 | else: 1126 | # Attempt to match the version too 1127 | for version in self.sorted_recipes[recipe]: 1128 | if fnmatch.fnmatch(version, version_match): 1129 | found = True 1130 | 1131 | self.print_recipe_details(recipe, version, verbose, all) 1132 | break 1133 | if found: 1134 | break 1135 | if not found: 1136 | if version_match == "": 1137 | self.logger.warning(f'No recipe matching name: "{recipe_match}"') 1138 | else: 1139 | self.logger.warning( 1140 | f'No recipe matching name: "{recipe_match}", version: "{version_match}"' 1141 | ) 1142 | 1143 | def clone_recipe(self, recipe: str, version: str, cookbook: str, destination: str): 1144 | """ 1145 | Search recipes for a specific recipe and copy the file to the CWD. 1146 | """ 1147 | 1148 | def get_cookbook(recipe: str, recipe_version: dict) -> str: 1149 | """ 1150 | Return the cookbook name, if only one cookbook provides the recipe-version. 1151 | If more then one cookbook provides the recipe-version, explain the options and return an empty string. 1152 | """ 1153 | cookbook = "" 1154 | 1155 | num_cookbooks = len(recipe_version["cookbooks"].keys()) 1156 | if num_cookbooks == 0: 1157 | self.logger.error( 1158 | f"Recipe {nvc_str(recipe, version)} not provided by any cookbook!(?!)" 1159 | ) 1160 | 1161 | elif num_cookbooks == 1: 1162 | cookbook = next(iter(recipe_version["cookbooks"])) 1163 | 1164 | else: 1165 | self.logger.error( 1166 | f'Clone failed: No cookbook specified, and multiple cookbooks provide recipe "{nvc_str(recipe, recipe_version["version"])}"' 1167 | ) 1168 | self.logger.error( 1169 | f"Please retry with a specific cookbook using the `-c` or `--cookbook` option:" 1170 | ) 1171 | self.logger.info(f"") 1172 | 1173 | self.print_recipe_details( 1174 | recipe, recipe_version, verbose=True, all=True 1175 | ) 1176 | 1177 | return cookbook 1178 | 1179 | found = False 1180 | 1181 | self.logger.info( 1182 | f'Attempting to clone recipe: "{nvc_str(recipe, version, cookbook)}"...' 1183 | ) 1184 | 1185 | try: 1186 | recipe_versions = self.sorted_recipes[recipe] 1187 | except KeyError: 1188 | self.logger.error(f'Clone failed: No such recipe "{recipe}"') 1189 | return False 1190 | 1191 | # Identify highest available version, for future reference. 1192 | highest_recipe_version = recipe_versions[0] 1193 | 1194 | # 1195 | # Now repeat the above if/else logic to select the exact recipe requested. 1196 | # 1197 | 1198 | if version == "": 1199 | if cookbook == "": 1200 | # neither version nor cookbook was specified. 1201 | self.logger.info( 1202 | f"No version or cookbook specified, will select highest available version." 1203 | ) 1204 | version = highest_recipe_version["version"] 1205 | 1206 | cookbook = get_cookbook(recipe, highest_recipe_version) 1207 | 1208 | if cookbook == "": 1209 | return False 1210 | 1211 | else: 1212 | # cookbook specified, but version wasn't. 1213 | self.logger.info( 1214 | f'No version specified, will select highest version provided by cookbook: "{cookbook}".' 1215 | ) 1216 | 1217 | selected_recipe_version = {} 1218 | 1219 | for recipe_version in recipe_versions: 1220 | if cookbook in recipe_version["cookbooks"].keys(): 1221 | selected_recipe_version = recipe_version 1222 | break 1223 | 1224 | if selected_recipe_version == {}: 1225 | self.logger.error( 1226 | f'Clone failed: Requested recipe "{recipe}" could not be found in cookbook: "{cookbook}".' 1227 | ) 1228 | return False 1229 | 1230 | version = selected_recipe_version["version"] 1231 | 1232 | if ( 1233 | selected_recipe_version["version"] 1234 | != highest_recipe_version["version"] 1235 | ): 1236 | self.logger.warning( 1237 | f'The version selected from cookbook "{cookbook}" is not the highest version available.' 1238 | ) 1239 | self.logger.warning( 1240 | f"A newer version appears to be available from other sources:" 1241 | ) 1242 | self.logger.info(f"") 1243 | self.print_recipe_details( 1244 | recipe, highest_recipe_version, verbose=True, all=True 1245 | ) 1246 | 1247 | else: 1248 | # version specified 1249 | if cookbook == "": 1250 | self.logger.info( 1251 | f"No cookbook specified, will select recipe only if version is provided by only one cookbook." 1252 | ) 1253 | 1254 | selected_recipe_version = {} 1255 | 1256 | for recipe_version in recipe_versions: 1257 | if version == recipe_version["version"]: 1258 | 1259 | cookbook = get_cookbook(recipe, recipe_version) 1260 | break 1261 | 1262 | if cookbook == "": 1263 | return False 1264 | 1265 | else: 1266 | # version and cookbook specified. 1267 | pass 1268 | 1269 | if destination == "": 1270 | destination = os.getcwd() 1271 | 1272 | try: 1273 | recipe_class = self.recipes[recipe][version][cookbook] 1274 | except KeyError: 1275 | self.logger.error( 1276 | f'Clone failed: Requested recipe "{nvc_str(recipe, version, cookbook)}" could not be found.' 1277 | ) 1278 | return False 1279 | 1280 | recipe_basename = os.path.basename(recipe_class.module_file) 1281 | clone_path = os.path.join(destination, recipe_basename) 1282 | 1283 | try: 1284 | shutil.copyfile( 1285 | recipe_class.module_file, os.path.join(destination, recipe_basename) 1286 | ) 1287 | 1288 | patch_dirs_copied: list = [] 1289 | for each_platform in recipe_class.platforms: 1290 | for target in recipe_class.platforms[each_platform]: 1291 | if ( 1292 | "patches" in recipe_class.platforms[each_platform][target] 1293 | and recipe_class.platforms[each_platform][target]["patches"] != "" 1294 | ): 1295 | patch_dir = os.path.join( 1296 | os.path.split(recipe_class.module_file)[0], 1297 | recipe_class.platforms[each_platform][target]["patches"], 1298 | ) 1299 | if patch_dir in patch_dirs_copied: 1300 | # Already got this one, 1301 | continue 1302 | 1303 | if not os.path.exists(patch_dir): 1304 | self.logger.warning( 1305 | f"Unable to clone referenced patch directory: {patch_dir}" 1306 | ) 1307 | self.logger.warning(f"Directory does not exist.") 1308 | else: 1309 | patches_basename = os.path.basename(patch_dir) 1310 | shutil.copytree( 1311 | patch_dir, os.path.join(destination, patches_basename) 1312 | ) 1313 | patch_dirs_copied.append(patch_dir) 1314 | except Exception as exc: 1315 | self.logger.error(f"Clone failed. Exception: {exc}") 1316 | return False 1317 | 1318 | self.logger.info( 1319 | f'Successfully cloned recipe "{nvc_str(recipe, version, cookbook)}" to:' 1320 | ) 1321 | self.logger.info(f" {clone_path}") 1322 | 1323 | return True 1324 | 1325 | def list_recipes(self, verbose: bool = False): 1326 | """ 1327 | Print out a list of all recipes and all collections. 1328 | """ 1329 | has_collections = False 1330 | 1331 | if len(self.sorted_recipes) == 0: 1332 | if len(self.cookbooks) > 0: 1333 | self.logger.warning(f"No recipes available from trusted cookbooks.") 1334 | self.logger.warning(f"Recipes from \"untrusted\" cookbooks are hidden by default.\n") 1335 | self.logger.info(f"Run the this to view that which cookbooks are available:") 1336 | self.logger.info(f" mussels cookbook list -V\n") 1337 | self.logger.info(f"Run this to view recipes & tools from a specific cookbook:") 1338 | self.logger.info(f" mussels cookbook show -V\n") 1339 | self.logger.info(f"To view ALL recipes, use:") 1340 | self.logger.info(f" mussels recipe list -a") 1341 | return 1342 | else: 1343 | self.logger.warning(f"Failed to load any recipes.\n") 1344 | self.logger.info(f"Re-run Mussels from a directory containing Mussels recipe & tool definitions,") 1345 | self.logger.info(f" or use `mussels update` to download recipes from the public cookbooks.") 1346 | return 1347 | 1348 | self.logger.info("Recipes:") 1349 | for recipe in self.sorted_recipes: 1350 | newest_version = self.sorted_recipes[recipe][0]["version"] 1351 | cookbooks = list(self.recipes[recipe][newest_version].keys()) 1352 | if not self.recipes[recipe][newest_version][cookbooks[0]].is_collection: 1353 | if not verbose: 1354 | outline = f" {recipe:10} " 1355 | for i, version in enumerate(self.sorted_recipes[recipe]): 1356 | if i == 0: 1357 | outline += f" {version['version']}" 1358 | if len(self.sorted_recipes[recipe]) > 1: 1359 | outline += "*" 1360 | else: 1361 | outline += f", {version['version']}" 1362 | outline += "" 1363 | self.logger.info(outline) 1364 | else: 1365 | outline = f" {recipe:10} " 1366 | for i, version in enumerate(self.sorted_recipes[recipe]): 1367 | if i == 0: 1368 | outline += f" {version['version']} {version['cookbooks']}" 1369 | if len(self.sorted_recipes[recipe]) > 1: 1370 | outline += "*" 1371 | else: 1372 | outline += f", {version['version']} {version['cookbooks']}" 1373 | outline += "" 1374 | self.logger.info(outline) 1375 | 1376 | for recipe in self.sorted_recipes: 1377 | newest_version = self.sorted_recipes[recipe][0]["version"] 1378 | cookbooks = list(self.recipes[recipe][newest_version].keys()) 1379 | if self.recipes[recipe][newest_version][cookbooks[0]].is_collection: 1380 | if not has_collections: 1381 | self.logger.info("") 1382 | self.logger.info("Collections:") 1383 | has_collections = True 1384 | 1385 | if not verbose: 1386 | outline = f" {recipe:10} " 1387 | for i, version in enumerate(self.sorted_recipes[recipe]): 1388 | if i == 0: 1389 | outline += f" {version['version']}" 1390 | if len(self.sorted_recipes[recipe]) > 1: 1391 | outline += "*" 1392 | else: 1393 | outline += f", {version['version']}" 1394 | outline += "" 1395 | self.logger.info(outline) 1396 | else: 1397 | outline = f" {recipe:10} " 1398 | for i, version in enumerate(self.sorted_recipes[recipe]): 1399 | if i == 0: 1400 | outline += f" {version['version']} {version['cookbooks']}" 1401 | if len(self.sorted_recipes[recipe]) > 1: 1402 | outline += "*" 1403 | else: 1404 | outline += f", {version['version']} {version['cookbooks']}" 1405 | outline += "" 1406 | self.logger.info(outline) 1407 | 1408 | def print_tool_details( 1409 | self, tool: str, version: dict, verbose: bool, all: bool 1410 | ): 1411 | """ 1412 | Print tool information. 1413 | """ 1414 | version_num = version["version"] 1415 | cookbooks = version["cookbooks"].keys() 1416 | self.logger.info(f" {nvc_str(tool, version_num)}; provided by cookbook(s): {list(cookbooks)}") 1417 | 1418 | if verbose: 1419 | self.logger.info("") 1420 | for cookbook in cookbooks: 1421 | self.logger.info(f" Cookbook: {cookbook}") 1422 | 1423 | book_tool = self.tools[tool][version_num][cookbook] 1424 | 1425 | self.logger.info(f" Platforms:") 1426 | for each_platform in book_tool.platforms: 1427 | if all or platform_is(each_platform): 1428 | self.logger.info(f" Host platform: {each_platform}") 1429 | 1430 | details = yaml.dump(book_tool.platforms[each_platform], indent=4) 1431 | for detail in details.split('\n'): 1432 | self.logger.info(" " + detail) 1433 | 1434 | if not all: 1435 | break 1436 | self.logger.info("") 1437 | 1438 | def show_tool(self, tool_match: str, version_match: str, verbose: bool = False): 1439 | """ 1440 | Search tools for a specific tool and print tool details. 1441 | """ 1442 | 1443 | found = False 1444 | 1445 | if version_match == "": 1446 | self.logger.info(f'Searching for tool matching name: "{tool_match}"...') 1447 | else: 1448 | self.logger.info( 1449 | f'Searching for tool matching name: "{tool_match}", version: "{version_match}"...' 1450 | ) 1451 | # Attempt to match the tool name 1452 | for tool in self.sorted_tools: 1453 | if fnmatch.fnmatch(tool, tool_match): 1454 | if version_match == "": 1455 | found = True 1456 | 1457 | # Show info for every version 1458 | for version in self.sorted_tools[tool]: 1459 | self.print_tool_details(tool, version, verbose, all) 1460 | break 1461 | else: 1462 | # Attempt to match the version too 1463 | for version in self.sorted_tools[tool]: 1464 | if fnmatch.fnmatch(version, version_match): 1465 | found = True 1466 | 1467 | self.print_tool_details(tool, version, verbose, all) 1468 | break 1469 | if found: 1470 | break 1471 | if not found: 1472 | if version_match == "": 1473 | self.logger.warning(f'No tool matching name: "{tool_match}"') 1474 | else: 1475 | self.logger.warning( 1476 | f'No tool matching name: "{tool_match}", version: "{version_match}"' 1477 | ) 1478 | 1479 | def clone_tool(self, tool: str, version: str, cookbook: str, destination: str): 1480 | """ 1481 | Search tools for a specific tool and copy the file to the CWD. 1482 | """ 1483 | 1484 | def get_cookbook(tool: str, tool_version: dict) -> str: 1485 | """ 1486 | Return the cookbook name, if only one cookbook provides the tool-version. 1487 | If more then one cookbook provides the tool-version, explain the options and return an empty string. 1488 | """ 1489 | cookbook = "" 1490 | 1491 | num_cookbooks = len(tool_version["cookbooks"].keys()) 1492 | if num_cookbooks == 0: 1493 | self.logger.error( 1494 | f"tool {nvc_str(tool, version)} not provided by any cookbook!(?!)" 1495 | ) 1496 | 1497 | elif num_cookbooks == 1: 1498 | cookbook = next(iter(tool_version["cookbooks"])) 1499 | 1500 | else: 1501 | self.logger.error( 1502 | f'Clone failed: No cookbook specified, and multiple cookbooks provide tool "{nvc_str(tool, tool_version["version"])}"' 1503 | ) 1504 | self.logger.error( 1505 | f"Please retry with a specific cookbook using the `-c` or `--cookbook` option:" 1506 | ) 1507 | self.logger.info(f"") 1508 | 1509 | self.print_tool_details( 1510 | tool, tool_version, verbose=True, all=True 1511 | ) 1512 | 1513 | return cookbook 1514 | 1515 | found = False 1516 | 1517 | self.logger.info( 1518 | f'Attempting to clone tool: "{nvc_str(tool, version, cookbook)}"...' 1519 | ) 1520 | 1521 | try: 1522 | tool_versions = self.sorted_tools[tool] 1523 | except KeyError: 1524 | self.logger.error(f'Clone failed: No such tool "{tool}"') 1525 | return False 1526 | 1527 | # Identify highest available version, for future reference. 1528 | highest_tool_version = tool_versions[0] 1529 | 1530 | # 1531 | # Now repeat the above if/else logic to select the exact tool requested. 1532 | # 1533 | 1534 | if version == "": 1535 | if cookbook == "": 1536 | # neither version nor cookbook was specified. 1537 | self.logger.info( 1538 | f"No version or cookbook specified, will select highest available version." 1539 | ) 1540 | version = highest_tool_version["version"] 1541 | 1542 | cookbook = get_cookbook(version, highest_tool_version) 1543 | 1544 | if cookbook == "": 1545 | return False 1546 | 1547 | else: 1548 | # cookbook specified, but version wasn't. 1549 | self.logger.info( 1550 | f'No version specified, will select highest version provided by cookbook: "{cookbook}".' 1551 | ) 1552 | 1553 | selected_tool_version = {} 1554 | 1555 | for tool_version in tool_versions: 1556 | if cookbook in tool_version["cookbooks"].keys(): 1557 | selected_tool_version = tool_version 1558 | break 1559 | 1560 | if selected_tool_version == {}: 1561 | self.logger.error( 1562 | f'Clone failed: Requested tool "{tool}" could not be found in cookbook: "{cookbook}".' 1563 | ) 1564 | return False 1565 | 1566 | version = selected_tool_version["version"] 1567 | 1568 | if ( 1569 | selected_tool_version["version"] 1570 | != highest_tool_version["version"] 1571 | ): 1572 | self.logger.warning( 1573 | f'The version selected from cookbook "{cookbook}" is not the highest version available.' 1574 | ) 1575 | self.logger.warning( 1576 | f"A newer version appears to be available from other sources:" 1577 | ) 1578 | self.logger.info(f"") 1579 | self.print_tool_details( 1580 | tool, highest_tool_version, verbose=True, all=True 1581 | ) 1582 | 1583 | else: 1584 | # version specified 1585 | if cookbook == "": 1586 | self.logger.info( 1587 | f"No cookbook specified, will select tool only if version is provided by only one cookbook." 1588 | ) 1589 | 1590 | selected_tool_version = {} 1591 | 1592 | for tool_version in tool_versions: 1593 | if version == tool_version["version"]: 1594 | 1595 | cookbook = get_cookbook(tool, tool_version) 1596 | break 1597 | 1598 | if cookbook == "": 1599 | return False 1600 | 1601 | else: 1602 | # version and cookbook specified. 1603 | pass 1604 | 1605 | if destination == "": 1606 | destination = os.getcwd() 1607 | 1608 | try: 1609 | tool_class = self.tools[tool][version][cookbook] 1610 | except KeyError: 1611 | self.logger.error( 1612 | f'Clone failed: Requested tool "{nvc_str(tool, version, cookbook)}" could not be found.' 1613 | ) 1614 | return False 1615 | 1616 | tool_basename = os.path.basename(tool_class.module_file) 1617 | clone_path = os.path.join(destination, tool_basename) 1618 | 1619 | try: 1620 | shutil.copyfile( 1621 | tool_class.module_file, os.path.join(destination, tool_basename) 1622 | ) 1623 | except Exception as exc: 1624 | self.logger.error(f"Clone failed. Exception: {exc}") 1625 | return False 1626 | 1627 | self.logger.info( 1628 | f'Successfully cloned tool "{nvc_str(tool, version, cookbook)}" to:' 1629 | ) 1630 | self.logger.info(f" {clone_path}") 1631 | 1632 | return True 1633 | 1634 | def list_tools(self, verbose: bool = False): 1635 | """ 1636 | Print out a list of all tools and all collections. 1637 | """ 1638 | has_collections = False 1639 | 1640 | if len(self.sorted_tools) == 0: 1641 | if len(self.cookbooks) > 0: 1642 | self.logger.warning(f"No tools available from trusted cookbooks.") 1643 | self.logger.warning(f"Tools from \"untrusted\" cookbooks are hidden by default.\n") 1644 | self.logger.info(f"Run the this to view that which cookbooks are available:") 1645 | self.logger.info(f" mussels cookbook list -V\n") 1646 | self.logger.info(f"Run this to view recipes & tools from a specific cookbook:") 1647 | self.logger.info(f" mussels cookbook show -V\n") 1648 | self.logger.info(f"To view ALL tools, use:") 1649 | self.logger.info(f" mussels tool list -a") 1650 | return 1651 | else: 1652 | self.logger.warning(f"Failed to load any tools.\n") 1653 | self.logger.info(f"Re-run Mussels from a directory containing Mussels recipe & tool definitions,") 1654 | self.logger.info(f" or use `mussels update` to download recipes from the public cookbooks.") 1655 | return 1656 | 1657 | self.logger.info("Tools:") 1658 | for tool in self.sorted_tools: 1659 | newest_version = self.sorted_tools[tool][0]["version"] 1660 | cookbooks = list(self.tools[tool][newest_version].keys()) 1661 | 1662 | if not verbose: 1663 | outline = f" {tool:10} " 1664 | for i, version in enumerate(self.sorted_tools[tool]): 1665 | if i == 0: 1666 | outline += f" {version['version']}" 1667 | if len(self.sorted_tools[tool]) > 1: 1668 | outline += "*" 1669 | else: 1670 | outline += f", {version['version']}" 1671 | outline += "" 1672 | self.logger.info(outline) 1673 | else: 1674 | outline = f" {tool:10} " 1675 | for i, version in enumerate(self.sorted_tools[tool]): 1676 | if i == 0: 1677 | outline += f" {version['version']} {version['cookbooks']}" 1678 | if len(self.sorted_tools[tool]) > 1: 1679 | outline += "*" 1680 | else: 1681 | outline += f", {version['version']} {version['cookbooks']}" 1682 | outline += "" 1683 | self.logger.info(outline) 1684 | 1685 | def update_cookbooks(self) -> None: 1686 | """ 1687 | Attempt to update each cookbook in using Git to clone or pull each repo. 1688 | If git isn't available, warn the user they should probably install Git and add it to their PATH. 1689 | """ 1690 | # Create ~/.mussels/bookshelf if it doesn't already exist. 1691 | os.makedirs(os.path.join(self.app_data_dir, "cookbooks"), exist_ok=True) 1692 | 1693 | # Get url for each cookbook from the mussels bookshelf. 1694 | for book in mussels.bookshelf.cookbooks: 1695 | repo_dir = os.path.join(self.app_data_dir, "cookbooks", book) 1696 | self.cookbooks[book]["path"] = repo_dir 1697 | self.cookbooks[book]["url"] = mussels.bookshelf.cookbooks[book]["url"] 1698 | if "trusted" not in self.cookbooks[book]: 1699 | self.cookbooks[book]["trusted"] = False 1700 | 1701 | for book in self.cookbooks: 1702 | repo_dir = os.path.join(self.app_data_dir, "cookbooks", book) 1703 | 1704 | if "url" in self.cookbooks[book] and self.cookbooks[book]["url"] != "": 1705 | if not os.path.isdir(repo_dir): 1706 | repo = git.Repo.clone_from(self.cookbooks[book]["url"], repo_dir) 1707 | else: 1708 | repo = git.Repo(repo_dir) 1709 | repo.git.pull() 1710 | 1711 | self._read_cookbook(book, repo_dir) 1712 | 1713 | self._store_config("cookbooks.json", self.cookbooks) 1714 | 1715 | def list_cookbooks(self, verbose: bool = False): 1716 | """ 1717 | Print out a list of all cookbooks. 1718 | """ 1719 | 1720 | if len(self.cookbooks) == 0: 1721 | self.logger.warning(f"Failed to load any cookbooks.\n") 1722 | self.logger.info(f"Re-run Mussels from a directory containing Mussels recipe & tool definitions,") 1723 | self.logger.info(f" or use `mussels update` to download recipes from the public cookbooks.") 1724 | return 1725 | 1726 | self.logger.info("Cookbooks:") 1727 | for cookbook in self.cookbooks: 1728 | self.logger.info(f" {cookbook}") 1729 | 1730 | if verbose: 1731 | if cookbook == "local": 1732 | self.logger.info(f" url: n/a") 1733 | else: 1734 | self.logger.info( 1735 | f" url: {self.cookbooks[cookbook]['url']}" 1736 | ) 1737 | self.logger.info(f" path: {self.cookbooks[cookbook]['path']}") 1738 | self.logger.info( 1739 | f" trusted: {self.cookbooks[cookbook]['trusted']}" 1740 | ) 1741 | self.logger.info(f"") 1742 | 1743 | def show_cookbook(self, cookbook_match: str, verbose: bool): 1744 | """ 1745 | Search cookbooks for a specific cookbook and print the details. 1746 | """ 1747 | found = False 1748 | 1749 | self.logger.info(f'Searching for cookbook matching name: "{cookbook_match}"...') 1750 | 1751 | # Attempt to match the cookbook name 1752 | for cookbook in self.cookbooks: 1753 | if fnmatch.fnmatch(cookbook, cookbook_match): 1754 | found = True 1755 | 1756 | self.logger.info(f" {cookbook}") 1757 | if cookbook == "local": 1758 | self.logger.info(f" url: n/a") 1759 | else: 1760 | self.logger.info( 1761 | f" url: {self.cookbooks[cookbook]['url']}" 1762 | ) 1763 | self.logger.info(f" path: {self.cookbooks[cookbook]['path']}") 1764 | self.logger.info( 1765 | f" trusted: {self.cookbooks[cookbook]['trusted']}" 1766 | ) 1767 | 1768 | if verbose: 1769 | self.logger.info(f"") 1770 | if len(self.cookbooks[cookbook]["recipes"].keys()) > 0: 1771 | self.logger.info(f" Recipes:") 1772 | for recipe in self.cookbooks[cookbook]["recipes"]: 1773 | self.logger.info( 1774 | f" {recipe} : {self.cookbooks[cookbook]['recipes'][recipe]}" 1775 | ) 1776 | self.logger.info(f"") 1777 | if len(self.cookbooks[cookbook]["tools"].keys()) > 0: 1778 | self.logger.info(f" Tools:") 1779 | for tool in self.cookbooks[cookbook]["tools"]: 1780 | self.logger.info( 1781 | f" {tool} : {self.cookbooks[cookbook]['tools'][tool]}" 1782 | ) 1783 | 1784 | if not found: 1785 | self.logger.warning(f'No cookbook matching name: "{cookbook_match}"') 1786 | 1787 | def clean_cache(self): 1788 | """ 1789 | Clear the cache files. 1790 | """ 1791 | self.logger.info( 1792 | f"Clearing cache directory ( {os.path.join(self.app_data_dir, 'cache')} )..." 1793 | ) 1794 | 1795 | if os.path.exists(os.path.join(self.app_data_dir, "cache")): 1796 | shutil.rmtree(os.path.join(self.app_data_dir, "cache")) 1797 | self.logger.info(f"Cache directory cleared.") 1798 | else: 1799 | self.logger.info(f"No cache directory to clear.") 1800 | 1801 | def clean_install(self): 1802 | """ 1803 | Clear the install files. 1804 | """ 1805 | self.logger.info( 1806 | f"Clearing install directory ( {os.path.join(self.app_data_dir, 'install')} )..." 1807 | ) 1808 | 1809 | if os.path.exists(os.path.join(self.install_dir)): 1810 | shutil.rmtree(os.path.join(self.install_dir)) 1811 | self.logger.info(f"Install directory cleared.") 1812 | else: 1813 | self.logger.info(f"No install directory to clear.") 1814 | 1815 | def clean_logs(self): 1816 | """ 1817 | Clear the log files. 1818 | """ 1819 | self.logger.info( 1820 | f"Clearing logs directory ( {os.path.join(self.app_data_dir, 'logs')} )..." 1821 | ) 1822 | 1823 | self.filehandler.close() 1824 | self.logger.removeHandler(self.filehandler) 1825 | 1826 | if os.path.exists(os.path.join(self.app_data_dir, "logs")): 1827 | shutil.rmtree(os.path.join(self.app_data_dir, "logs")) 1828 | self.logger.info(f"Logs directory cleared.") 1829 | else: 1830 | self.logger.info(f"No logs directory to clear.") 1831 | 1832 | def clean_all(self): 1833 | """ 1834 | Clear all Mussels files. 1835 | """ 1836 | self.clean_cache() 1837 | self.clean_install() 1838 | self.clean_logs() 1839 | 1840 | self.logger.info( 1841 | f"Clearing Mussels directory ( {os.path.join(self.app_data_dir)} )..." 1842 | ) 1843 | 1844 | if os.path.exists(os.path.join(self.app_data_dir)): 1845 | shutil.rmtree(os.path.join(self.app_data_dir)) 1846 | self.logger.info(f"Mussels directory cleared.") 1847 | else: 1848 | self.logger.info(f"No Mussels directory to clear.") 1849 | 1850 | def config_trust_cookbook(self, cookbook): 1851 | """ 1852 | Update config to indicate that a given cookbook is trusted. 1853 | """ 1854 | if cookbook not in self.cookbooks: 1855 | self.logger.error( 1856 | f"Can't trust cookbook '{cookbook}'. Cookbook is unknown." 1857 | ) 1858 | 1859 | self.logger.info(f"'{cookbook}' cookbook is now trusted.") 1860 | 1861 | self.cookbooks[cookbook]["trusted"] = True 1862 | 1863 | self._store_config("cookbooks.json", self.cookbooks) 1864 | 1865 | def config_add_cookbook(self, cookbook, author, url, trust=False): 1866 | """ 1867 | Update config to indicate that a given cookbook is trusted. 1868 | """ 1869 | self.cookbooks[cookbook]["author"] = author 1870 | self.cookbooks[cookbook]["url"] = url 1871 | self.cookbooks[cookbook]["trusted"] = trust 1872 | 1873 | self._store_config("cookbooks.json", self.cookbooks) 1874 | 1875 | def config_remove_cookbook(self, cookbook): 1876 | self.cookbooks.pop(cookbook) 1877 | 1878 | self._store_config("cookbooks.json", self.cookbooks) 1879 | -------------------------------------------------------------------------------- /mussels/recipe.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2019-2020 Cisco Systems, Inc. and/or its affiliates. All rights reserved. 3 | 4 | This module provides the base class for every Recipe. 5 | 6 | This includes the logic for how to perform a build and how to relocate (or "install") 7 | select files to a directory structure where other recipes can find & depend on them. 8 | 9 | Licensed under the Apache License, Version 2.0 (the "License"); 10 | you may not use this file except in compliance with the License. 11 | You may obtain a copy of the License at 12 | 13 | http://www.apache.org/licenses/LICENSE-2.0 14 | 15 | Unless required by applicable law or agreed to in writing, software 16 | distributed under the License is distributed on an "AS IS" BASIS, 17 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | See the License for the specific language governing permissions and 19 | limitations under the License. 20 | """ 21 | 22 | import datetime 23 | from distutils import dir_util 24 | import glob 25 | import inspect 26 | from io import StringIO 27 | import logging 28 | import os 29 | import platform 30 | import shutil 31 | import stat 32 | import subprocess 33 | import sys 34 | import tarfile 35 | import time 36 | import zipfile 37 | 38 | import requests 39 | import urllib.request 40 | import patch 41 | 42 | from mussels.utils.versions import pick_platform, nvc_str 43 | 44 | 45 | class BaseRecipe(object): 46 | """ 47 | Base class for Mussels recipe. 48 | """ 49 | 50 | name = "sample" 51 | version = "1.2.3" 52 | 53 | # Set collection to True if this is just a collection of recipes to build, and not an actual recipe. 54 | # If True, only `dependencies` and `required_tools` matter. Everything else should be omited. 55 | is_collection = False 56 | 57 | url = "https://sample.com/sample.tar.gz" # URL of project release materials. 58 | 59 | # archive_name_change is a tuple of strings to replace. 60 | # For example: 61 | # ("v", "nghttp2-") 62 | # will change: 63 | # v2.3.4 to nghttp2-2.3.4. 64 | # This hack is necessary because archives with changed names will extract to their original directory name. 65 | archive_name_change: tuple = ("", "") 66 | 67 | platforms: dict = {} # Dictionary of recipe instructions for each platform. 68 | 69 | builds: dict = {} # Dictionary of build paths. 70 | 71 | module_file: str = "" 72 | 73 | variables: dict = {} # variables that will be evaluated in build scripts 74 | 75 | def __init__( 76 | self, 77 | toolchain: dict, 78 | platform: str, 79 | target: str, 80 | install_dir: str = "", 81 | data_dir: str = "", 82 | work_dir: str = "", 83 | log_dir: str = "", 84 | download_dir: str = "", 85 | log_level: str = "DEBUG", 86 | ): 87 | """ 88 | Download the archive (if necessary) to the Downloads directory. 89 | Extract the archive to the temp directory so it is ready to build. 90 | """ 91 | self.toolchain = toolchain 92 | self.platform = platform 93 | self.target = target 94 | 95 | if data_dir == "": 96 | # No temp dir provided, build in the current working directory. 97 | self.data_dir = os.getcwd() 98 | else: 99 | self.data_dir = os.path.abspath(data_dir) 100 | 101 | self.install_dir = install_dir 102 | 103 | if download_dir != "": 104 | self.download_dir = download_dir 105 | else: 106 | self.download_dir = os.path.join(self.data_dir, "cache", "downloads") 107 | 108 | if log_dir != "": 109 | self.log_dir = log_dir 110 | else: 111 | self.log_dir = os.path.join(self.data_dir, "logs", "recipes") 112 | 113 | if work_dir != "": 114 | self.work_dir = work_dir 115 | else: 116 | self.work_dir = os.path.join(self.data_dir, "cache", "work") 117 | 118 | self.module_dir = os.path.split(self.module_file)[0] 119 | 120 | if "patches" in self.platforms[self.platform][self.target]: 121 | self.patch_dir = os.path.join( 122 | self.module_dir, self.platforms[self.platform][self.target]["patches"] 123 | ) 124 | else: 125 | self.patch_dir = "" 126 | 127 | self._init_logging(log_level) 128 | 129 | def _init_logging(self, level="DEBUG"): 130 | """ 131 | Initializes the logging parameters 132 | """ 133 | levels = { 134 | "DEBUG": logging.DEBUG, 135 | "INFO": logging.INFO, 136 | "WARN": logging.WARNING, 137 | "WARNING": logging.WARNING, 138 | "ERROR": logging.ERROR, 139 | } 140 | 141 | os.makedirs(self.log_dir, exist_ok=True) 142 | 143 | self.logger = logging.getLogger(f"{nvc_str(self.name, self.version)}") 144 | 145 | formatter = logging.Formatter( 146 | fmt="%(asctime)s - %(levelname)s: %(message)s", 147 | datefmt="%m/%d/%Y %I:%M:%S %p", 148 | ) 149 | 150 | self.log_file = os.path.join( 151 | self.log_dir, 152 | f"{nvc_str(self.name, self.version)}.{datetime.datetime.now()}.log".replace( 153 | ":", "_" 154 | ), 155 | ) 156 | filehandler = logging.FileHandler(filename=self.log_file) 157 | filehandler.setLevel(logging.DEBUG) 158 | filehandler.setFormatter(formatter) 159 | 160 | self.logger.addHandler(filehandler) 161 | self.logger.setLevel(levels[os.environ.get("LOG_LEVEL", level)]) 162 | 163 | def _download_archive(self) -> bool: 164 | """ 165 | Use the URL to download the archive if it doesn't already exist in the Downloads directory. 166 | """ 167 | os.makedirs(self.download_dir, exist_ok=True) 168 | 169 | # Determine download path from URL & possible archive name change. 170 | self.archive = self.url.split("/")[-1] 171 | if self.archive_name_change[0] != "": 172 | self.archive = self.archive.replace( 173 | self.archive_name_change[0], self.archive_name_change[1] 174 | ) 175 | self.download_path = os.path.join( 176 | self.download_dir, self.archive 177 | ) 178 | 179 | # Exit early if we already have the archive. 180 | if os.path.exists(self.download_path): 181 | self.logger.debug(f"Archive already downloaded.") 182 | return True 183 | 184 | self.logger.info(f"Downloading {self.url}") 185 | self.logger.info(f" to {self.download_path} ...") 186 | 187 | if self.url.startswith("ftp"): 188 | try: 189 | urllib.request.urlretrieve(self.url, self.download_path) 190 | except Exception as exc: 191 | self.logger.info(f"Failed to download archive from {self.url}, {exc}!") 192 | return False 193 | else: 194 | try: 195 | r = requests.get(self.url) 196 | with open(self.download_path, "wb") as f: 197 | f.write(r.content) 198 | except Exception: 199 | self.logger.info(f"Failed to download archive from {self.url}!") 200 | return False 201 | 202 | return True 203 | 204 | def _extract_archive(self, rebuild: bool) -> bool: 205 | """ 206 | Extract the archive found in Downloads directory, if necessary. 207 | """ 208 | if self.archive.endswith(".tar.gz"): 209 | self.builds[self.target] = os.path.join( 210 | self.work_dir, self.target, f"{self.archive[:-len('.tar.gz')]}" 211 | ) 212 | elif self.archive.endswith(".zip"): 213 | self.builds[self.target] = os.path.join( 214 | self.work_dir, self.target, f"{self.archive[:-len('.zip')]}" 215 | ) 216 | elif self.archive.endswith(".tar.xz"): 217 | self.builds[self.target] = os.path.join( 218 | self.work_dir, self.target, f"{self.archive[:-len('.tar.xz')]}" 219 | ) 220 | else: 221 | self.logger.error( 222 | f"Unexpected archive extension. Currently only supports .tar.gz and .zip!" 223 | ) 224 | return False 225 | 226 | self.prior_build_exists = os.path.exists(self.builds[self.target]) 227 | 228 | if self.prior_build_exists: 229 | if not rebuild: 230 | # Build directory already exists. We're good. 231 | return True 232 | 233 | # Remove previous built, start over. 234 | self.logger.info( 235 | f"--rebuild: Removing previous {self.target} build directory:" 236 | ) 237 | self.logger.info(f" {self.builds[self.target]}") 238 | shutil.rmtree(self.builds[self.target]) 239 | self.prior_build_exists = False 240 | 241 | os.makedirs(os.path.join(self.work_dir, self.target), exist_ok=True) 242 | 243 | # Make our own copy of the extracted source so we don't dirty the original. 244 | self.logger.debug(f"Preparing {self.target} build directory:") 245 | self.logger.debug(f" {self.builds[self.target]}") 246 | 247 | if self.archive.endswith(".tar.gz"): 248 | # Un-tar 249 | self.logger.info( 250 | f"Extracting tarball archive {self.archive} to {self.builds[self.target]} ..." 251 | ) 252 | 253 | tar = tarfile.open(self.download_path, "r:gz") 254 | tar.extractall(os.path.join(self.work_dir, self.target)) 255 | tar.close() 256 | elif self.archive.endswith(".zip"): 257 | # Un-zip 258 | self.logger.info( 259 | f"Extracting zip archive {self.archive} to {self.builds[self.target]} ..." 260 | ) 261 | 262 | zip_ref = zipfile.ZipFile(self.download_path, "r") 263 | zip_ref.extractall(os.path.join(self.work_dir, self.target)) 264 | zip_ref.close() 265 | elif self.archive.endswith(".tar.xz"): 266 | # Un-tar 267 | self.logger.info( 268 | f"Extracting tarball archive {self.archive} to {self.builds[self.target]} ..." 269 | ) 270 | 271 | tar = tarfile.open(self.download_path, "r:xz") 272 | tar.extractall(os.path.join(self.work_dir, self.target)) 273 | tar.close() 274 | 275 | return True 276 | 277 | def _run_script(self, target, name, script) -> bool: 278 | """ 279 | Run a script in the current working directory. 280 | """ 281 | # Create a build script. 282 | if platform.system() == "Windows": 283 | script_name = f"_{name}.bat" 284 | newline = "\r\n" 285 | else: 286 | script_name = f"_{name}.sh" 287 | newline = "\n" 288 | 289 | with open(os.path.join(os.getcwd(), script_name), "w", newline=newline) as fd: 290 | # Evaluate "".format() syntax in the build script 291 | script = script.format(**self.variables) 292 | 293 | # Write the build commands to a file 294 | build_lines = script.splitlines() 295 | for line in build_lines: 296 | line = line.strip() 297 | if platform.system() == "Windows" and line.endswith("\\"): 298 | fd.write(line.rstrip("\\") + " ") 299 | else: 300 | fd.write(line + "\n") 301 | 302 | if platform.system() != "Windows": 303 | st = os.stat(script_name) 304 | os.chmod(script_name, st.st_mode | stat.S_IEXEC) 305 | 306 | # Run the build script. 307 | process = subprocess.Popen( 308 | os.path.join(os.getcwd(), script_name), 309 | shell=True, 310 | stdout=subprocess.PIPE, 311 | stderr=subprocess.STDOUT, 312 | ) 313 | with process.stdout: 314 | for line in iter(process.stdout.readline, b""): 315 | self.logger.debug(line.decode("utf-8", errors='replace').strip()) 316 | process.wait() 317 | if process.returncode != 0: 318 | self.logger.warning( 319 | f"{nvc_str(self.name, self.version)} {target} build failed!" 320 | ) 321 | self.logger.warning(f"Command:") 322 | for line in script.splitlines(): 323 | self.logger.warning(line) 324 | self.logger.warning(f"Exit code: {process.returncode}") 325 | self.logger.error(f'"{name}" script failed for {target} build') 326 | return False 327 | 328 | return True 329 | 330 | def build(self, rebuild: bool = False) -> bool: 331 | """ 332 | Patch source materials if not already patched. 333 | Then, for each architecture, run the build commands if the output files don't already exist. 334 | """ 335 | if self.is_collection: 336 | self.logger.debug( 337 | f"Build completed for recipe collection {nvc_str(self.name, self.version)}" 338 | ) 339 | return True 340 | 341 | os.makedirs(self.work_dir, exist_ok=True) 342 | 343 | # Download and build if necessary. 344 | if not self._download_archive(): 345 | self.logger.error( 346 | f"Failed to download source archive for {nvc_str(self.name, self.version)}" 347 | ) 348 | return False 349 | 350 | # Extract to the work_dir. 351 | if not self._extract_archive(rebuild): 352 | self.logger.error( 353 | f"Failed to extract source archive for {nvc_str(self.name, self.version)}" 354 | ) 355 | return False 356 | 357 | if not os.path.isdir(self.patch_dir): 358 | self.logger.debug(f"No patch directory found.") 359 | else: 360 | # Patches exists for this recipe. 361 | self.logger.debug( 362 | f"Patch directory found for {nvc_str(self.name, self.version)}." 363 | ) 364 | if not os.path.exists( 365 | os.path.join(self.builds[self.target], "_mussles.patched") 366 | ): 367 | # Not yet patched. Apply patches. 368 | self.logger.info( 369 | f"Applying patches to {nvc_str(self.name, self.version)} ({self.target}) build directory ..." 370 | ) 371 | for patchfile in os.listdir(self.patch_dir): 372 | if patchfile.endswith(".diff") or patchfile.endswith(".patch"): 373 | self.logger.info(f"Attempting to apply patch: {patchfile}") 374 | pset = patch.fromfile(os.path.join(self.patch_dir, patchfile)) 375 | patched = pset.apply(1, root=self.builds[self.target]) 376 | if not patched: 377 | self.logger.error(f"Patch failed!") 378 | return False 379 | else: 380 | self.logger.info( 381 | f"Copying new file {patchfile} to {nvc_str(self.name, self.version)} ({self.target}) build directory ..." 382 | ) 383 | shutil.copyfile( 384 | os.path.join(self.patch_dir, patchfile), 385 | os.path.join(self.builds[self.target], patchfile), 386 | ) 387 | 388 | with open( 389 | os.path.join(self.builds[self.target], "_mussles.patched"), "w" 390 | ) as patchmark: 391 | patchmark.write("patched") 392 | 393 | build_scripts = self.platforms[self.platform][self.target]["build_script"] 394 | 395 | self.logger.info( 396 | f"Attempting to build {nvc_str(self.name, self.version)} for {self.target}" 397 | ) 398 | 399 | self.variables["includes"] = os.path.join(self.install_dir, "include").replace("\\", "/") 400 | self.variables["libs"] = os.path.join(self.install_dir, "lib").replace("\\", "/") 401 | self.variables["install"] = os.path.join(self.install_dir).replace("\\", "/") 402 | self.variables["build"] = os.path.join(self.builds[self.target]).replace("\\", "/") 403 | self.variables["target"] = self.target 404 | 405 | for tool in self.toolchain: 406 | # Add each tool from the toolchain to the PATH environment variable. 407 | if self.toolchain[tool].tool_path != "": 408 | self.logger.debug( 409 | f"Adding tool {tool} path {self.toolchain[tool].tool_path} to PATH" 410 | ) 411 | os.environ["PATH"] = ( 412 | self.toolchain[tool].tool_path + os.pathsep + os.environ["PATH"] 413 | ) 414 | 415 | # Collect tool variables for use in the build. 416 | platform_options = self.toolchain[tool].platforms.keys() 417 | matching_platform = pick_platform(platform.system(), platform_options) 418 | 419 | if "variables" in self.toolchain[tool].platforms[matching_platform]: 420 | # The following will allow us to format strings like "echo {tool.variable}" 421 | tool_vars = lambda: None 422 | for variable in self.toolchain[tool].platforms[matching_platform]["variables"]: 423 | setattr(tool_vars, variable, self.toolchain[tool].platforms[matching_platform]["variables"][variable]) 424 | self.variables[tool] = tool_vars 425 | 426 | cwd = os.getcwd() 427 | os.chdir(self.builds[self.target]) 428 | 429 | if not self.prior_build_exists: 430 | # Run "configure" script, if exists. 431 | if "configure" in build_scripts.keys(): 432 | if not self._run_script( 433 | self.target, "configure", build_scripts["configure"] 434 | ): 435 | self.logger.error( 436 | f"{nvc_str(self.name, self.version)} {self.target} build failed." 437 | ) 438 | os.chdir(cwd) 439 | return False 440 | 441 | else: 442 | os.chdir(self.builds[self.target]) 443 | 444 | # Run "make" script, if exists. 445 | if "make" in build_scripts.keys(): 446 | if not self._run_script(self.target, "make", build_scripts["make"]): 447 | self.logger.error( 448 | f"{nvc_str(self.name, self.version)} {self.target} build failed." 449 | ) 450 | os.chdir(cwd) 451 | return False 452 | 453 | # Run "install" script, if exists. 454 | if "install" in build_scripts.keys(): 455 | if not self._run_script(self.target, "install", build_scripts["install"]): 456 | self.logger.error( 457 | f"{nvc_str(self.name, self.version)} {self.target} build failed." 458 | ) 459 | os.chdir(cwd) 460 | return False 461 | 462 | self.logger.info( 463 | f"{nvc_str(self.name, self.version)} {self.target} build succeeded." 464 | ) 465 | os.chdir(cwd) 466 | 467 | if not self._install(): 468 | return False 469 | 470 | return True 471 | 472 | def _install(self): 473 | """ 474 | Copy the headers and libs to an install directory. 475 | """ 476 | os.makedirs(self.install_dir, exist_ok=True) 477 | 478 | self.logger.info( 479 | f"Copying {nvc_str(self.name, self.version)} install files to: {self.install_dir}." 480 | ) 481 | 482 | if 'install_paths' not in self.platforms[self.platform][self.target]: 483 | self.logger.info( 484 | f"{nvc_str(self.name, self.version)} {self.target} nothing additional to install." 485 | ) 486 | 487 | else: 488 | install_paths = self.platforms[self.platform][self.target]["install_paths"] 489 | 490 | for install_path in install_paths: 491 | 492 | for install_item in install_paths[install_path]: 493 | item_installed = False 494 | src_path = os.path.join(self.builds[self.target], install_item) 495 | 496 | for src_filepath in glob.glob(src_path): 497 | dst_path = os.path.join( 498 | self.install_dir, install_path, os.path.basename(src_filepath) 499 | ) 500 | 501 | # Remove prior installation, if exists. 502 | if os.path.isdir(dst_path): 503 | shutil.rmtree(dst_path) 504 | elif os.path.isfile(dst_path): 505 | os.remove(dst_path) 506 | 507 | # Create the target install paths, if it doesn't already exist. 508 | os.makedirs(os.path.split(dst_path)[0], exist_ok=True) 509 | 510 | self.logger.debug(f"Copying: {src_filepath}") 511 | self.logger.debug(f" to: {dst_path}") 512 | 513 | # Now copy the file or directory. 514 | if os.path.isdir(src_filepath): 515 | dir_util.copy_tree(src_filepath, dst_path) 516 | else: 517 | shutil.copyfile(src_filepath, dst_path) 518 | 519 | item_installed = True 520 | 521 | # Globbing only shows us files that actually exists. 522 | # It's possible we didn't install anything at all. 523 | # Verify that we installed at least one item. 524 | if not item_installed: 525 | self.logger.error( 526 | f"Required target install files do not exist: {src_path}" 527 | ) 528 | return False 529 | 530 | self.logger.info( 531 | f"{nvc_str(self.name, self.version)} {self.target} install succeeded." 532 | ) 533 | return True 534 | -------------------------------------------------------------------------------- /mussels/tool.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2019-2020 Cisco Systems, Inc. and/or its affiliates. All rights reserved. 3 | 4 | This module provides the base class for every Tool. A tool is anything that a Recipe 5 | might depend on from the system environment. 6 | 7 | The base class provides the logic for detecting tools. 8 | 9 | There is work-in-progress logic to [optionally] download and install missing tools 10 | on behalf of the user. 11 | 12 | Licensed under the Apache License, Version 2.0 (the "License"); 13 | you may not use this file except in compliance with the License. 14 | You may obtain a copy of the License at 15 | 16 | http://www.apache.org/licenses/LICENSE-2.0 17 | 18 | Unless required by applicable law or agreed to in writing, software 19 | distributed under the License is distributed on an "AS IS" BASIS, 20 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 21 | See the License for the specific language governing permissions and 22 | limitations under the License. 23 | """ 24 | 25 | import datetime 26 | from distutils import dir_util, spawn 27 | import inspect 28 | import logging 29 | import os 30 | import platform 31 | import requests 32 | import shutil 33 | import stat 34 | import subprocess 35 | import tarfile 36 | import zipfile 37 | 38 | from io import StringIO 39 | 40 | from mussels.utils.versions import platform_is, nvc_str 41 | 42 | 43 | class BaseTool(object): 44 | """ 45 | Base class for Mussels tool detection. 46 | """ 47 | 48 | name = "sample" 49 | version = "" 50 | platforms: dict = {} 51 | logs_dir = "" 52 | tool_path = "" 53 | 54 | def __init__(self, 55 | data_dir: str = "", 56 | log_level: str = "DEBUG", 57 | ): 58 | """ 59 | Download the archive (if necessary) to the Downloads directory. 60 | Extract the archive to the temp directory so it is ready to build. 61 | """ 62 | if data_dir == "": 63 | # No temp dir provided, build in the current working directory. 64 | self.logs_dir = os.path.join(os.getcwd(), "logs", "tools") 65 | else: 66 | self.logs_dir = os.path.join(os.path.abspath(data_dir), "logs", "tools") 67 | os.makedirs(self.logs_dir, exist_ok=True) 68 | 69 | self.name_version = nvc_str(self.name, self.version) 70 | 71 | self._init_logging(log_level) 72 | 73 | def _init_logging(self, level="DEBUG"): 74 | """ 75 | Initializes the logging parameters 76 | """ 77 | levels = { 78 | "DEBUG": logging.DEBUG, 79 | "INFO": logging.INFO, 80 | "WARN": logging.WARNING, 81 | "WARNING": logging.WARNING, 82 | "ERROR": logging.ERROR, 83 | } 84 | 85 | self.logger = logging.getLogger(f"{self.name_version}") 86 | 87 | formatter = logging.Formatter( 88 | fmt="%(asctime)s - %(levelname)s: %(message)s", 89 | datefmt="%m/%d/%Y %I:%M:%S %p", 90 | ) 91 | 92 | self.log_file = os.path.join( 93 | self.logs_dir, 94 | f"{self.name_version}.{datetime.datetime.now()}.log".replace(":", "_"), 95 | ) 96 | filehandler = logging.FileHandler(filename=self.log_file) 97 | filehandler.setLevel(logging.DEBUG) 98 | filehandler.setFormatter(formatter) 99 | 100 | self.logger.addHandler(filehandler) 101 | self.logger.setLevel(levels[os.environ.get("LOG_LEVEL", level)]) 102 | 103 | def _run_command(self, command: str, expected_output: str) -> bool: 104 | """ 105 | Run a command. 106 | """ 107 | found_expected_output = False 108 | 109 | cmd = command.split() 110 | 111 | # Run the build script. 112 | try: 113 | process = subprocess.Popen( 114 | cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT 115 | ) 116 | with process.stdout: 117 | for line in iter(process.stdout.readline, b""): 118 | if expected_output in line.decode("utf-8", errors='replace'): 119 | found_expected_output = True 120 | 121 | process.wait() 122 | if process.returncode != 0: 123 | self.logger.debug(f"Command failed!") 124 | return False 125 | except FileNotFoundError: 126 | self.logger.debug(f"Command failed; File not found!") 127 | return False 128 | 129 | return found_expected_output 130 | 131 | def detect(self) -> bool: 132 | """ 133 | Determine if tool is available in expected locations. 134 | """ 135 | found = False 136 | 137 | self.logger.info(f"Detecting tool: {self.name_version}...") 138 | 139 | for each_platform in self.platforms: 140 | if platform_is(each_platform): 141 | if "path_checks" in self.platforms[each_platform]: 142 | for path_check in self.platforms[each_platform]["path_checks"]: 143 | self.logger.info(f" Checking for {path_check} in PATH") 144 | install_location = spawn.find_executable(path_check) 145 | if install_location == None: 146 | self.logger.info(f" {path_check} not found") 147 | else: 148 | self.logger.info( 149 | f" {path_check} found, at: {install_location}" 150 | ) 151 | found = True 152 | break 153 | 154 | if found: 155 | break 156 | 157 | if "command_checks" in self.platforms[each_platform]: 158 | for script_check in self.platforms[each_platform]["command_checks"]: 159 | found = self._run_command( 160 | command=script_check["command"], 161 | expected_output=script_check["output_has"], 162 | ) 163 | if not found: 164 | self.logger.info(f" {script_check['command']} failed.") 165 | else: 166 | self.logger.info(f" {script_check['command']} passed!") 167 | break 168 | 169 | if found: 170 | break 171 | 172 | if "file_checks" in self.platforms[each_platform]: 173 | for filepath in self.platforms[each_platform]["file_checks"]: 174 | if not os.path.exists(filepath): 175 | self.logger.info( 176 | f'{self.name_version} file "{filepath}" not found' 177 | ) 178 | else: 179 | self.logger.info( 180 | f'{self.name_version} file "{filepath}" found' 181 | ) 182 | self.tool_path = os.path.dirname(filepath) 183 | found = True 184 | break 185 | break 186 | 187 | if not found: 188 | self.logger.debug(f"Failed to detect {self.name_version}.") 189 | return False 190 | 191 | return True 192 | -------------------------------------------------------------------------------- /mussels/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cisco-Talos/Mussels/8621c1c2a7de9a012f1a71e1c97d3bb7f6ad63d8/mussels/utils/__init__.py -------------------------------------------------------------------------------- /mussels/utils/click.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2019-2020 Cisco Systems, Inc. and/or its affiliates. All rights reserved. 3 | 4 | A collections of helpers for Click 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | """ 18 | 19 | import click 20 | 21 | # 22 | # Click command-group modifiers 23 | # 24 | class MusselsModifier(click.Group): 25 | def get_command(self, ctx, cmd_name): 26 | rv = click.Group.get_command(self, ctx, cmd_name) 27 | if rv is not None: 28 | return rv 29 | matches = [x for x in self.list_commands(ctx) if x.startswith(cmd_name)] 30 | if not matches: 31 | return None 32 | elif len(matches) == 1: 33 | return click.Group.get_command(self, ctx, matches[0]) 34 | ctx.fail("Too many matches: %s" % ", ".join(sorted(matches))) 35 | 36 | def format_epilog(self, ctx, formatter): 37 | if self.epilog: 38 | print(self.epilog) 39 | 40 | 41 | class ShortNames(click.Group): 42 | def get_command(self, ctx, cmd_name): 43 | rv = click.Group.get_command(self, ctx, cmd_name) 44 | if rv is not None: 45 | return rv 46 | matches = [x for x in self.list_commands(ctx) if x.startswith(cmd_name)] 47 | if not matches: 48 | return None 49 | elif len(matches) == 1: 50 | return click.Group.get_command(self, ctx, matches[0]) 51 | ctx.fail("Too many matches: %s" % ", ".join(sorted(matches))) 52 | -------------------------------------------------------------------------------- /mussels/utils/versions.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2019-2020 Cisco Systems, Inc. and/or its affiliates. All rights reserved. 3 | 4 | This module provides an assortment of helper functions that Mussels depends on. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | """ 18 | 19 | from collections import defaultdict, namedtuple 20 | import platform 21 | 22 | NVC = namedtuple("NVC", "name version cookbook") 23 | 24 | 25 | def version_keys(s): 26 | """ 27 | `key` function enabling python's `sort` function to sort version strings. 28 | """ 29 | import re 30 | 31 | keys = [] 32 | for u in s.split("."): 33 | for v in re.split(r"(\d+)", u): 34 | try: 35 | val = int(v) 36 | except: 37 | val = str(v) 38 | keys.append(val) 39 | return keys 40 | 41 | 42 | def sort_cookbook_by_version(items) -> defaultdict: 43 | """ 44 | Sort items, and determine the highest versions. 45 | """ 46 | sorted_items: defaultdict = defaultdict(list) 47 | 48 | for item in items: 49 | versions_list = list(items[item].keys()) 50 | versions_list.sort(key=version_keys) 51 | versions_list.reverse() 52 | for version in versions_list: 53 | sorted_items[item].append(version) 54 | 55 | return sorted_items 56 | 57 | 58 | PLATFORMS = { 59 | "posix": [ 60 | "linux", 61 | "darwin", 62 | "macos", 63 | "osx", 64 | "freebsd", 65 | "openbsd", 66 | "sunos", 67 | "aix", 68 | "hp-ux", 69 | ], 70 | "unix": ["darwin", "macos", "osx", "freebsd", "openbsd", "sunos", "aix", "hp-ux"], 71 | } 72 | 73 | 74 | def platform_matches(requested_platform: str, specific_platform) -> bool: 75 | """ 76 | Compare two platforms. 77 | Common platforms: 78 | - Windows 79 | - macos / darwin / osx 80 | - linux 81 | - unix (macos, sunos, bsd unix's) 82 | - *nix / posix (not windows) 83 | :return: True if current platform matches requested platform. 84 | :return: False otherwise. 85 | """ 86 | specific_platform = specific_platform.lower() 87 | requested_platform = requested_platform.lower() 88 | 89 | if requested_platform == specific_platform: 90 | return True 91 | 92 | elif ( 93 | requested_platform == "mac" 94 | or requested_platform == "macos" 95 | or requested_platform == "osx" 96 | ) and specific_platform == "darwin": 97 | return True 98 | 99 | if (requested_platform == "unix") and ( 100 | specific_platform == "darwin" 101 | or specific_platform == "sunos" 102 | or "bsd" in specific_platform 103 | ): 104 | return True 105 | 106 | elif requested_platform == "*nix" or requested_platform == "posix": 107 | if specific_platform != "windows": 108 | return True 109 | 110 | else: 111 | return False 112 | 113 | return False 114 | 115 | 116 | def platform_is(requested_platform: str) -> bool: 117 | """ 118 | Compare requested platform with current platform. 119 | Common platforms: 120 | - Win / Windows 121 | - Mac / macOS / Darwin 122 | - Linux 123 | - Unix (Mac, SunOS, BSD unix's) 124 | - *nix / posix (Not Windows) 125 | :return: True if current platform matches requested platform. 126 | :return: False otherwise. 127 | """ 128 | return platform_matches(requested_platform, platform.system()) 129 | 130 | 131 | def pick_platform(requested_platform: str, platform_options: list) -> str: 132 | """ 133 | Given a list of platforms, pick the one that most closely matches the current platform. 134 | Prefer exact, allow superset. 135 | :return: string name of selected platform. 136 | """ 137 | if requested_platform in platform_options: 138 | return requested_platform 139 | 140 | for option in platform_options: 141 | if platform_matches(option, requested_platform): 142 | return option 143 | 144 | return "" 145 | 146 | 147 | def compare_versions(version_a: str, version_b: str) -> int: 148 | """ 149 | Evaluate version strings of two versions. 150 | Compare if version A against version B. 151 | :return: -1 if A < B 152 | :return: 0 if A == B 153 | :return: 1 if A > B 154 | """ 155 | if version_a == version_b: 156 | return 0 157 | 158 | versions_list = [version_a, version_b] 159 | 160 | versions_list.sort(key=version_keys) 161 | 162 | if versions_list[0] == version_a: 163 | return -1 164 | else: 165 | return 1 166 | 167 | 168 | def get_item_version(item_name: str, sorted_items: dict, target: str = "", logger = None) -> NVC: 169 | """ 170 | Convert a item name in the below format to a (name, version) tuple: 171 | 172 | [cookbook:]name[>=,<=,>,<,(==|=|@)version] 173 | 174 | Examples: 175 | - meepioux 176 | - blarghus>=1.2.3 177 | - wheeple@0.2.0 178 | - pyplo==5.1.0g 179 | - scrapbook:sasquatch<2.0.0 180 | - scrapbook: minnow < 0.1.12 181 | 182 | The highest available version will be selected if one is not specified. 183 | Version requirements will whittle down the list of available versions 184 | in the sorted_items list. 185 | 186 | If a specific version is specified, all other versions will be disqualified (pruned). 187 | 188 | If no versions remain that satisfy build qualifications, an exception will be raised. 189 | 190 | :return: named tuple describing the highest qualified version: 191 | NVC( 192 | "name"->str, 193 | "version"->str, 194 | "cookbook"->str, 195 | ) 196 | """ 197 | 198 | def select_cookbook_version(nvc, item_version, target: str = "") -> bool: 199 | cookbook_selected = False 200 | 201 | def cookbook_has_build_target(cookbooks_item: dict, target) -> bool: 202 | if target == "": 203 | return True 204 | 205 | for each_platform in cookbooks_item: 206 | # Note: sorted_items has been filtered down to compatible platform. 207 | # No need to check with platform_is() 208 | if target in cookbooks_item[each_platform]: 209 | return True 210 | return False 211 | 212 | # Prefer local over all else, enabling monkey-patching of recipes. 213 | if "local" in item_version["cookbooks"] and cookbook_has_build_target( 214 | item_version["cookbooks"]["local"], target 215 | ): 216 | if nvc["cookbook"] != "" and nvc["cookbook"] != "local": 217 | if logger: 218 | logger.debug(f"Overriding {nvc_str(nvc['name'], nvc['version'], nvc['cookbook'])} with {nvc_str(nvc['name'], item_version['version'], 'local')}") 219 | nvc["version"] = item_version["version"] 220 | nvc["cookbook"] = "local" 221 | cookbook_selected = True 222 | else: 223 | if nvc["cookbook"] == "": 224 | # Any cookbook will do. 225 | for cookbook in item_version["cookbooks"]: 226 | if cookbook_has_build_target( 227 | item_version["cookbooks"][cookbook], target 228 | ): 229 | nvc["version"] = item_version["version"] 230 | nvc["cookbook"] = cookbook 231 | cookbook_selected = True 232 | break 233 | else: 234 | # Check for requested cookbook. 235 | for cookbook in item_version["cookbooks"]: 236 | if cookbook == nvc["cookbook"] and cookbook_has_build_target( 237 | item_version["cookbooks"][cookbook], target 238 | ): 239 | nvc["version"] = item_version["version"] 240 | cookbook_selected = True 241 | break 242 | 243 | # Remove all other cookbooks for this item version. 244 | if cookbook_selected: 245 | item_version["cookbooks"] = { 246 | nvc["cookbook"]: item_version["cookbooks"][nvc["cookbook"]] 247 | } 248 | return cookbook_selected 249 | 250 | nvc = {"name": "", "version": "", "cookbook": ""} 251 | 252 | requested_item = item_name 253 | item_selected = False 254 | 255 | # Identify cookbook name, if provided. 256 | if ":" in item_name: 257 | cookbook, item = item_name.split(":") 258 | nvc["cookbook"] = cookbook.strip() 259 | item_name = item.strip() 260 | 261 | if ">=" in item_name: 262 | # GTE requirement found. 263 | name, version = item_name.split(">=") 264 | nvc["name"] = name.strip() 265 | version = version.strip() 266 | for i, item_version in enumerate(sorted_items[name]): 267 | cmp = compare_versions(item_version["version"], version) 268 | if cmp >= 0: 269 | # Version is good. 270 | if item_selected != True: 271 | item_selected = select_cookbook_version(nvc, item_version, target) 272 | else: 273 | # Version is too low. Remove it, and subsequent versions. 274 | if logger != None and len(sorted_items[name][:i]) < len(sorted_items[name]): 275 | logger.debug(f"{name} limited to version: {', '.join([item['version'] for item in sorted_items[name][:i]])}") 276 | 277 | sorted_items[name] = sorted_items[name][:i] 278 | break 279 | 280 | elif ">" in item_name: 281 | # GT requirement found. 282 | name, version = item_name.split(">") 283 | nvc["name"] = name.strip() 284 | version = version.strip() 285 | for i, item_version in enumerate(sorted_items[name]): 286 | cmp = compare_versions(item_version["version"], version) 287 | if cmp > 0: 288 | # Version is good. 289 | if item_selected != True: 290 | item_selected = select_cookbook_version(nvc, item_version, target) 291 | else: 292 | # Version is too low. Remove it, and subsequent versions. 293 | if logger != None and len(sorted_items[name][:i]) < len(sorted_items[name]): 294 | logger.debug(f"{name} limited to version: {', '.join([item['version'] for item in sorted_items[name][:i]])}") 295 | 296 | sorted_items[name] = sorted_items[name][:i] 297 | break 298 | 299 | elif "<=" in item_name: 300 | # LTE requirement found. 301 | name, version = item_name.split("<=") 302 | nvc["name"] = name.strip() 303 | version = version.strip() 304 | 305 | pruned = False 306 | # First, prune down to highest tolerable version 307 | if len(sorted_items[name]) > 0: 308 | 309 | while ( 310 | len(sorted_items[name]) > 0 311 | and compare_versions(sorted_items[name][0]["version"], version) > 0 312 | ): 313 | # Remove a version from the sorted_items. 314 | sorted_items[name].remove(sorted_items[name][0]) 315 | pruned = True 316 | 317 | # Then, prune down to the highest version provided by a the requested cookbook 318 | if len(sorted_items[name]) > 0: 319 | while len(sorted_items[name]) > 0 and not item_selected: 320 | item_selected = select_cookbook_version( 321 | nvc, sorted_items[name][0], target 322 | ) 323 | 324 | if not item_selected: 325 | # Remove a version from the sorted_items. 326 | sorted_items[name].remove(sorted_items[name][0]) 327 | pruned = True 328 | 329 | if logger != None and pruned: 330 | logger.debug(f"{name} limited to version: {', '.join([item['version'] for item in sorted_items[name]])}") 331 | 332 | elif "<" in item_name: 333 | # LT requirement found. 334 | name, version = item_name.split("<") 335 | nvc["name"] = name.strip() 336 | version = version.strip() 337 | 338 | pruned = False 339 | # First, prune down to highest tolerable version 340 | if len(sorted_items[name]) > 0: 341 | 342 | while ( 343 | len(sorted_items[name]) > 0 344 | and compare_versions(sorted_items[name][0]["version"], version) >= 0 345 | ): 346 | # Remove a version from the sorted_items. 347 | sorted_items[name].remove(sorted_items[name][0]) 348 | pruned = True 349 | 350 | # Then, prune down to the highest version provided by a the requested cookbook 351 | if len(sorted_items[name]) > 0: 352 | while len(sorted_items[name]) > 0 and not item_selected: 353 | item_selected = select_cookbook_version( 354 | nvc, sorted_items[name][0], target 355 | ) 356 | 357 | if not item_selected: 358 | # Remove a version from the sorted_items. 359 | sorted_items[name].remove(sorted_items[name][0]) 360 | pruned = True 361 | 362 | if logger != None and pruned: 363 | logger.debug(f"{name} limited to version: {', '.join([item['version'] for item in sorted_items[name]])}") 364 | 365 | else: 366 | eq_cond = False 367 | if "==" in item_name: 368 | name, version = item_name.split("==") 369 | eq_cond = True 370 | elif "=" in item_name: 371 | name, version = item_name.split("=") 372 | eq_cond = True 373 | elif "-" in item_name: 374 | name, version = item_name.split("-") 375 | eq_cond = True 376 | elif "@" in item_name: 377 | name, version = item_name.split("@") 378 | eq_cond = True 379 | 380 | if eq_cond == True: 381 | nvc["name"] = name.strip() 382 | nvc["version"] = version.strip() 383 | # EQ requirement found. 384 | # Try to find the specific version, and remove all others. 385 | item_selected = False 386 | for item_version in sorted_items[nvc["name"]]: 387 | if version == item_version["version"]: 388 | item_selected = select_cookbook_version(nvc, item_version, target) 389 | if item_selected: 390 | if logger != None and len(sorted_items[nvc["name"]]) > 1: 391 | logger.debug(f"{name} limited to version: {version}") 392 | 393 | sorted_items[nvc["name"]] = [item_version] 394 | break 395 | 396 | else: 397 | # No version requirement found. 398 | nvc["name"] = item_name.strip() 399 | 400 | for item_version in sorted_items[nvc["name"]]: 401 | item_selected = select_cookbook_version(nvc, item_version, target) 402 | if item_selected: 403 | break 404 | 405 | if not item_selected: 406 | if target == "": 407 | raise Exception( 408 | f"No versions available to satisfy requirement for {requested_item}.\nThe requested version may have been filtered out by requirements for another recipe." 409 | ) 410 | else: 411 | raise Exception( 412 | f"No versions available to satisfy requirement for {requested_item} ({target}).\nThe requested version may have been filtered out by requirements for another recipe." 413 | ) 414 | 415 | return NVC(nvc["name"], nvc["version"], nvc["cookbook"]) 416 | 417 | 418 | def nvc_str(name, version, cookbook: str = ""): 419 | def nv_str(name, version): 420 | if version != "": 421 | return f"{name}-{version}" 422 | else: 423 | return name 424 | 425 | if cookbook != "": 426 | return f"{cookbook}:{nv_str(name, version)}" 427 | else: 428 | return nv_str(name, version) 429 | 430 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | with open("README.md", "r") as fh: 4 | long_description = fh.read() 5 | 6 | setuptools.setup( 7 | name="mussels", 8 | version="0.4.1", 9 | author="Micah Snyder", 10 | author_email="micasnyd@cisco.com", 11 | copyright="Copyright (C) 2025 Cisco Systems, Inc. and/or its affiliates. All rights reserved.", 12 | description="Mussels Dependency Build Automation Tool", 13 | long_description=long_description, 14 | long_description_content_type="text/markdown", 15 | url="https://github.com/Cisco-Talos/mussels", 16 | packages=setuptools.find_packages(), 17 | entry_points={ 18 | "console_scripts": [ 19 | "mussels = mussels.__main__:cli", 20 | "msl = mussels.__main__:cli", 21 | ] 22 | }, 23 | install_requires=[ 24 | "click>=7.0", 25 | "coloredlogs>=10.0", 26 | "colorama", 27 | "requests", 28 | "patch", 29 | "gitpython", 30 | "pyyaml", 31 | "setuptools", 32 | ], 33 | classifiers=[ 34 | "Programming Language :: Python :: 3", 35 | "License :: OSI Approved :: Apache Software License", 36 | "Operating System :: OS Independent", 37 | ], 38 | ) 39 | -------------------------------------------------------------------------------- /tests/tool_variables_test.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2021 Cisco Systems, Inc. and/or its affiliates. All rights reserved. 3 | 4 | Tests for versions.py utility functions 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | """ 18 | 19 | import json 20 | import logging 21 | import multiprocessing 22 | import os 23 | import unittest 24 | import tempfile 25 | import shutil 26 | import sys 27 | import platform 28 | from pathlib import Path 29 | import http.server 30 | import socketserver 31 | from multiprocessing import Process, Pipe 32 | import tarfile 33 | import time 34 | 35 | import pytest 36 | 37 | from mussels.mussels import Mussels 38 | 39 | def execute_build(stdout, stderr): 40 | sys.stdout = stdout.fileno() 41 | sys.stderr = stderr.fileno() 42 | 43 | my_mussels = Mussels( 44 | install_dir=os.getcwd(), 45 | work_dir=os.getcwd(), 46 | log_dir=os.getcwd(), 47 | download_dir=os.getcwd(), 48 | ) 49 | 50 | results = [] 51 | 52 | success = my_mussels.build_recipe( 53 | recipe="foobar", 54 | version="", 55 | cookbook="", 56 | target="host", 57 | results=results, 58 | dry_run=False, 59 | rebuild=False 60 | ) 61 | 62 | stdout.send("The End") 63 | stdout.close() 64 | stderr.close() 65 | 66 | if success == False: 67 | sys.exit(1) 68 | 69 | sys.exit(0) 70 | 71 | def host_server(): 72 | ''' 73 | Host files in the current directory on port 8000. 74 | ''' 75 | PORT = 8000 76 | Handler = http.server.SimpleHTTPRequestHandler 77 | with socketserver.TCPServer(("", PORT), Handler) as httpd: 78 | print("serving at port", PORT) 79 | httpd.serve_forever() 80 | 81 | 82 | class TC(unittest.TestCase): 83 | @classmethod 84 | def setUpClass(cls): 85 | TC.path_tmp = Path(tempfile.mkdtemp(prefix="msl-test-")) 86 | TC.savedir = os.getcwd() 87 | os.chdir(str(TC.path_tmp)) 88 | 89 | (TC.path_tmp / "foo.yaml").write_text(f''' 90 | name: python 91 | version: "3" 92 | mussels_version: "0.3" 93 | type: tool 94 | platforms: 95 | Posix: 96 | file_checks: 97 | - {sys.executable} 98 | variables: 99 | foo: Hello World! 100 | Windows: 101 | file_checks: 102 | - {sys.executable} 103 | variables: 104 | foo: Hello World! 105 | ''') 106 | 107 | # Make a foo.tar.gz package that the recipe build can download & extract. 108 | (TC.path_tmp / 'foo').mkdir() 109 | (TC.path_tmp / 'foo' / "foo.txt").write_text("Hello, World!\n") 110 | with tarfile.open(str(TC.path_tmp / "foo.tar.gz"), "w:gz") as tar: 111 | tar.add("foo") 112 | 113 | TC.p = Process(target=host_server) 114 | TC.p.start() 115 | 116 | @classmethod 117 | def tearDownClass(cls): 118 | TC.p.terminate() 119 | TC.p.kill() 120 | os.chdir(TC.savedir) 121 | shutil.rmtree(str(TC.path_tmp)) 122 | 123 | def setUp(self): 124 | pass 125 | 126 | def tearDown(self): 127 | pass 128 | 129 | def test_0_basic_variable(self): 130 | (TC.path_tmp / "foobar.yaml").write_text(''' 131 | name: foobar 132 | version: "1.2.3" 133 | url: https://www.example.com/foo.tar.gz 134 | mussels_version: "0.3" 135 | type: recipe 136 | platforms: 137 | Posix: 138 | host: 139 | build_script: 140 | make: | 141 | echo "{python.foo}" 142 | dependencies: [] 143 | required_tools: 144 | - python 145 | Windows: 146 | host: 147 | build_script: 148 | make: | 149 | echo "{python.foo}" 150 | dependencies: [] 151 | required_tools: 152 | - python 153 | ''') 154 | 155 | my_mussels = Mussels( 156 | install_dir=os.getcwd(), 157 | work_dir=os.getcwd(), 158 | log_dir=os.getcwd(), 159 | download_dir=os.getcwd(), 160 | log_level="DEBUG" 161 | ) 162 | 163 | results = [] 164 | 165 | success = my_mussels.build_recipe( 166 | recipe="foobar", 167 | version="", 168 | cookbook="", 169 | target="host", 170 | results=results, 171 | dry_run=False, 172 | rebuild=False 173 | ) 174 | 175 | logging.shutdown() 176 | 177 | # Find the foobar log 178 | for i in os.listdir(str(TC.path_tmp)): 179 | if (TC.path_tmp / i).is_file() and i.startswith('foobar-1.2.3'): 180 | text = (TC.path_tmp / i).read_text() 181 | break 182 | 183 | print(f"Checking for 'Hello World!' in:\n\n{text}") 184 | 185 | assert "Hello World!" in text 186 | 187 | if __name__ == "__main__": 188 | pytest.main(args=["-v", os.path.abspath(__file__)]) 189 | -------------------------------------------------------------------------------- /tests/versions__compare_versions_test.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2019-2020 Cisco Systems, Inc. and/or its affiliates. All rights reserved. 3 | 4 | Tests for versions.py utility functions 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | """ 18 | 19 | import json 20 | import os 21 | import unittest 22 | 23 | import pytest 24 | 25 | from mussels.utils.versions import * 26 | 27 | 28 | class TestClass(unittest.TestCase): 29 | @classmethod 30 | def setUpClass(cls): 31 | pass 32 | 33 | @classmethod 34 | def tearDownClass(cls): 35 | pass 36 | 37 | def setUp(self): 38 | pass 39 | 40 | def tearDown(self): 41 | pass 42 | 43 | def test_compare_versions_lt_0(self): 44 | assert compare_versions("0.0.1", "0.0.2") == -1 45 | 46 | def test_compare_versions_eq_0(self): 47 | assert compare_versions("0.0.1", "0.0.1") == 0 48 | 49 | def test_compare_versions_gt_0(self): 50 | assert compare_versions("0.0.2", "0.0.1") == 1 51 | 52 | def test_compare_versions_lt_1(self): 53 | assert compare_versions("0.0.2", "0.2.1") == -1 54 | 55 | def test_compare_versions_eq_1(self): 56 | assert compare_versions("0.2.1", "0.2.1") == 0 57 | 58 | def test_compare_versions_gt_1(self): 59 | assert compare_versions("0.2.2", "0.0.3") == 1 60 | 61 | def test_compare_versions_lt_2(self): 62 | assert compare_versions("1.0.2g", "1.1.1c") == -1 63 | 64 | def test_compare_versions_eq_2(self): 65 | assert compare_versions("1.0.2g", "1.0.2g") == 0 66 | 67 | def test_compare_versions_gt_2(self): 68 | assert compare_versions("1.1.1a", "1.0.2s") == 1 69 | 70 | def test_compare_versions_lt_3(self): 71 | assert compare_versions("0.101.0_1", "0.102.0_0") == -1 72 | 73 | def test_compare_versions_eq_3(self): 74 | assert compare_versions("0.101.0_1", "0.101.0_1") == 0 75 | 76 | def test_compare_versions_gt_3(self): 77 | assert compare_versions("0.102.0_1", "0.101.0") == 1 78 | 79 | def test_compare_versions_gt_beta_0(self): 80 | assert compare_versions("0.102.0", "0.101.0-beta") == 1 81 | 82 | def test_compare_versions_gt_beta_1(self): 83 | assert compare_versions("0.102.0-beta", "0.101.0") == 1 84 | 85 | 86 | if __name__ == "__main__": 87 | pytest.main(args=["-v", os.path.abspath(__file__)]) 88 | -------------------------------------------------------------------------------- /tests/versions__get_item_version_test.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2019-2020 Cisco Systems, Inc. and/or its affiliates. All rights reserved. 3 | 4 | Tests for versions.py utility functions 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | """ 18 | 19 | import json 20 | import os 21 | import unittest 22 | 23 | import pytest 24 | 25 | from mussels.utils.versions import * 26 | 27 | 28 | class TestClass(unittest.TestCase): 29 | @classmethod 30 | def setUpClass(cls): 31 | pass 32 | 33 | @classmethod 34 | def tearDownClass(cls): 35 | pass 36 | 37 | def setUp(self): 38 | self.sorted_items = { 39 | "wheeple": [ 40 | {"version": "2.0.0", "cookbooks": {"tectonic": None}}, 41 | {"version": "1.0.1", "cookbooks": {"tectonic": None}}, 42 | {"version": "1.0.0", "cookbooks": {"tectonic": None}}, 43 | ] 44 | } 45 | 46 | def tearDown(self): 47 | pass 48 | 49 | def test_get_item_version(self): 50 | 51 | nvc = get_item_version(item_name="wheeple", sorted_items=self.sorted_items) 52 | 53 | print("nvc:") 54 | print(json.dumps(nvc, indent=4)) 55 | 56 | assert nvc.name == "wheeple" 57 | assert nvc.version == "2.0.0" 58 | assert nvc.cookbook == "tectonic" 59 | 60 | def test_get_item_version_gt(self): 61 | 62 | nvc = get_item_version( 63 | item_name="wheeple>1.0.0", sorted_items=self.sorted_items 64 | ) 65 | 66 | print("nvc:") 67 | print(json.dumps(nvc, indent=4)) 68 | 69 | assert nvc.name == "wheeple" 70 | assert nvc.version == "2.0.0" 71 | assert nvc.cookbook == "tectonic" 72 | 73 | def test_get_item_version_gt_nexists(self): 74 | 75 | no_item = False 76 | try: 77 | nvc = get_item_version( 78 | item_name="wheeple>2.0.0", sorted_items=self.sorted_items 79 | ) 80 | except: 81 | no_item = True 82 | 83 | assert no_item 84 | 85 | def test_get_item_version_gte(self): 86 | 87 | nvc = get_item_version( 88 | item_name="wheeple>=1.0.1", sorted_items=self.sorted_items 89 | ) 90 | 91 | print("nvc:") 92 | print(json.dumps(nvc, indent=4)) 93 | 94 | assert nvc.name == "wheeple" 95 | assert nvc.version == "2.0.0" 96 | assert nvc.cookbook == "tectonic" 97 | 98 | def test_get_item_version_gte_nexists(self): 99 | 100 | no_item = False 101 | try: 102 | nvc = get_item_version( 103 | item_name="wheeple>=2.0.1", sorted_items=self.sorted_items 104 | ) 105 | except: 106 | no_item = True 107 | 108 | assert no_item 109 | 110 | def test_get_item_version_lt(self): 111 | 112 | nvc = get_item_version( 113 | item_name="wheeple<1.0.1", sorted_items=self.sorted_items 114 | ) 115 | 116 | print("nvc:") 117 | print(json.dumps(nvc, indent=4)) 118 | 119 | assert nvc.name == "wheeple" 120 | assert nvc.version == "1.0.0" 121 | assert nvc.cookbook == "tectonic" 122 | 123 | def test_get_item_version_lt_nexists(self): 124 | 125 | no_item = False 126 | try: 127 | nvc = get_item_version( 128 | item_name="wheeple<1.0.0", sorted_items=self.sorted_items 129 | ) 130 | except: 131 | no_item = True 132 | 133 | assert no_item 134 | 135 | def test_get_item_version_eq_0(self): 136 | 137 | nvc = get_item_version( 138 | item_name="wheeple=1.0.0", sorted_items=self.sorted_items 139 | ) 140 | 141 | print("nvc:") 142 | print(json.dumps(nvc, indent=4)) 143 | 144 | assert nvc.name == "wheeple" 145 | assert nvc.version == "1.0.0" 146 | assert nvc.cookbook == "tectonic" 147 | 148 | def test_get_item_version_eq_1(self): 149 | 150 | nvc = get_item_version( 151 | item_name="wheeple@1.0.0", sorted_items=self.sorted_items 152 | ) 153 | 154 | print("nvc:") 155 | print(json.dumps(nvc, indent=4)) 156 | 157 | assert nvc.name == "wheeple" 158 | assert nvc.version == "1.0.0" 159 | assert nvc.cookbook == "tectonic" 160 | 161 | def test_get_item_version_eq_2(self): 162 | 163 | nvc = get_item_version( 164 | item_name="wheeple==1.0.0", sorted_items=self.sorted_items 165 | ) 166 | 167 | print("nvc:") 168 | print(json.dumps(nvc, indent=4)) 169 | 170 | assert nvc.name == "wheeple" 171 | assert nvc.version == "1.0.0" 172 | assert nvc.cookbook == "tectonic" 173 | 174 | def test_get_item_version_eq_3(self): 175 | 176 | nvc = get_item_version( 177 | item_name="wheeple-1.0.0", sorted_items=self.sorted_items 178 | ) 179 | 180 | print("nvc:") 181 | print(json.dumps(nvc, indent=4)) 182 | 183 | assert nvc.name == "wheeple" 184 | assert nvc.version == "1.0.0" 185 | assert nvc.cookbook == "tectonic" 186 | 187 | def test_get_item_version_eq_nexists(self): 188 | 189 | no_item = False 190 | try: 191 | nvc = get_item_version( 192 | item_name="wheeple<1.0.0", sorted_items=self.sorted_items 193 | ) 194 | except: 195 | no_item = True 196 | 197 | assert no_item 198 | 199 | def test_get_item_version_book_eq(self): 200 | 201 | nvc = get_item_version( 202 | item_name="tectonic:wheeple", sorted_items=self.sorted_items 203 | ) 204 | 205 | print("nvc:") 206 | print(json.dumps(nvc, indent=4)) 207 | 208 | assert nvc.name == "wheeple" 209 | assert nvc.version == "2.0.0" 210 | assert nvc.cookbook == "tectonic" 211 | 212 | def test_get_item_version_book_eq_nexists(self): 213 | 214 | no_item = False 215 | try: 216 | nvc = get_item_version( 217 | item_name="timtim:wheeple", sorted_items=self.sorted_items 218 | ) 219 | except: 220 | no_item = True 221 | 222 | assert no_item 223 | 224 | def test_get_item_version_prune_version_0(self): 225 | 226 | nvc = get_item_version( 227 | item_name="tectonic:wheeple=1.0.0", sorted_items=self.sorted_items 228 | ) 229 | 230 | print("nvc:") 231 | print(json.dumps(nvc, indent=4)) 232 | print("sorted_items:") 233 | print(json.dumps(self.sorted_items["wheeple"], indent=4)) 234 | 235 | assert len(self.sorted_items["wheeple"]) == 1 236 | 237 | def test_get_item_version_prune_version_1(self): 238 | 239 | nvc = get_item_version( 240 | item_name="tectonic:wheeple>1.0.0", sorted_items=self.sorted_items 241 | ) 242 | 243 | print("nvc:") 244 | print(json.dumps(nvc, indent=4)) 245 | print("sorted_items:") 246 | print(json.dumps(self.sorted_items["wheeple"], indent=4)) 247 | 248 | assert len(self.sorted_items["wheeple"]) == 2 249 | 250 | def test_get_item_version_prune_version_2(self): 251 | 252 | nvc = get_item_version( 253 | item_name="tectonic:wheeple>=1.0.1", sorted_items=self.sorted_items 254 | ) 255 | 256 | print("nvc:") 257 | print(json.dumps(nvc, indent=4)) 258 | print("sorted_items:") 259 | print(json.dumps(self.sorted_items["wheeple"], indent=4)) 260 | 261 | assert len(self.sorted_items["wheeple"]) == 2 262 | 263 | def test_get_item_version_prune_version_3(self): 264 | 265 | nvc = get_item_version( 266 | item_name="tectonic:wheeple<2.0.0", sorted_items=self.sorted_items 267 | ) 268 | 269 | print("nvc:") 270 | print(json.dumps(nvc, indent=4)) 271 | print("sorted_items:") 272 | print(json.dumps(self.sorted_items["wheeple"], indent=4)) 273 | 274 | assert len(self.sorted_items["wheeple"]) == 2 275 | 276 | def test_get_item_version_prune_version_4(self): 277 | 278 | nvc = get_item_version( 279 | item_name="tectonic:wheeple<=1.0.1", sorted_items=self.sorted_items 280 | ) 281 | 282 | print("nvc:") 283 | print(json.dumps(nvc, indent=4)) 284 | print("sorted_items:") 285 | print(json.dumps(self.sorted_items["wheeple"], indent=4)) 286 | 287 | assert len(self.sorted_items["wheeple"]) == 2 288 | 289 | def test_get_item_version_no_prune_version(self): 290 | 291 | nvc = get_item_version( 292 | item_name="tectonic:wheeple", sorted_items=self.sorted_items 293 | ) 294 | 295 | print("nvc:") 296 | print(json.dumps(nvc, indent=4)) 297 | print("sorted_items:") 298 | print(json.dumps(self.sorted_items["wheeple"], indent=4)) 299 | 300 | assert len(self.sorted_items["wheeple"]) == 3 301 | 302 | 303 | if __name__ == "__main__": 304 | pytest.main(args=["-v", os.path.abspath(__file__)]) 305 | --------------------------------------------------------------------------------