├── .github └── workflows │ └── main.yml ├── .gitignore ├── LICENSE ├── README.md ├── bin ├── whisk └── whiskenv ├── ci ├── dummy-init └── test.py ├── images └── whisk.svg ├── init-build-env ├── meta-whisk ├── classes │ ├── build-alias.bbclass │ ├── deploy-alias.bbclass │ └── product-build-alias.bbclass ├── conf │ └── layer.conf └── recipes-core │ └── targets │ └── all-targets.bb ├── requirements.txt ├── setup-venv ├── whisk.example.yaml ├── whisk.py ├── whisk.schema.json └── yamllint.yaml /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: build 2 | on: 3 | - push 4 | - pull_request 5 | 6 | jobs: 7 | check: 8 | runs-on: ubuntu-22.04 9 | steps: 10 | - uses: actions/checkout@v2 11 | 12 | - uses: actions/setup-python@v2 13 | with: 14 | python-version: 3.8 15 | 16 | - name: Install python packages 17 | run: | 18 | pip3 install black 19 | 20 | - name: Run black 21 | run: | 22 | black --check $(git ls-files '*.py') 23 | 24 | build: 25 | strategy: 26 | fail-fast: false 27 | matrix: 28 | python-version: 29 | - "3.7" 30 | - "3.8" 31 | - "3.9" 32 | - "3.10" 33 | - "3.11" 34 | - "3.12" 35 | 36 | runs-on: ubuntu-22.04 37 | 38 | steps: 39 | - uses: actions/checkout@v2 40 | 41 | - name: Set up Python ${{ matrix.python-version }} 42 | uses: actions/setup-python@v2 43 | with: 44 | python-version: ${{ matrix.python-version }} 45 | 46 | - name: Install dependencies 47 | run: | 48 | python -m pip install --upgrade pip 49 | pip install pyyaml 50 | 51 | - name: Prepare environment 52 | run: | 53 | ./setup-venv 54 | 55 | - name: Test 56 | run: | 57 | ./ci/test.py -vb 58 | 59 | 60 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .config.yaml 2 | build/ 3 | layers/ 4 | test/ 5 | Pipfile.lock 6 | venv/ 7 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://github.com/garmin/whisk/workflows/build/badge.svg?branch=master&event=push)](https://github.com/garmin/whisk/actions?query=workflow%3Abuild+event%3Apush+branch%3Amaster) 2 | 3 | # Whisk 4 | 5 | 6 | Organize OpenEmbedded products 7 | 8 | ## What is Whisk? 9 | 10 | Whisk is a tool for managing complex product configurations when using 11 | OpenEmbedded and the Yocto project. The key features are: 12 | 1. **Single source tree** Whisk is designed to allow multiple products with 13 | vastly different configurations to co-exist in the same repository (and the 14 | same branch) 15 | 2. **Multiple axes of configuration** You can configure what you want to build 16 | (products), how you want to build them (modes) and where you are build them 17 | (sites) 18 | 3. **Multiple products builds** Whisk sets up each product in it's own 19 | [multiconfig][]. This means that you can configure and build multiple 20 | products with the same invocation of bitbake, and that each products has 21 | its own isolated bitbake environment. 22 | 4. **Isolated layer configuration** Each product may define which layers it 23 | needs to build. If multiple products are configured and the set of layers 24 | required for each product is not equal, Whisk will use [BBMASK][] to mask 25 | off the unused layers independently for each product. See [Product Layer Masking][] 26 | 5. **Multiple versions** Whisk lets you define multiple different versions of 27 | layers to target for your product. Each product can define a default version 28 | to use if unspecified, but users can override the default. This allows 29 | several use cases, such as testing products with more recent Yocto releases, 30 | or maintaining similar builds across multiple Yocto versions for testing. 31 | 32 | [Requirements]: #requirements 33 | ## Requirements 34 | 35 | Whisk is primarily written in Python, with a small initialization shell script 36 | to setup the build environment. By default, the Python code runs under 37 | [virtualenv](https://virtualenv.pypa.io/en/latest/) and the initialization 38 | script will use it to automatically install the required dependencies. See the 39 | virtualenv documentation for installation instructions. 40 | 41 | If you wish to avoid the usage of virtualenv, for instance because you run the 42 | initialization shell script in a restricted environment where downloading 43 | dependencies is not possible, you may set the `WHISK_NO_VENV` environment 44 | variable. The value does not matter, only that it is set. In that case, you 45 | yourself must ensure that the required dependencies are available. See 46 | [requirements.txt](/requirements.txt) for the list of dependencies. 47 | 48 | [Using Whisk]: #using-whisk 49 | ## Using Whisk 50 | 51 | _NOTE: This is the end users guide to using Whisk. If you are looking for 52 | instructions on how to setup and configure Whisk for your project, see 53 | [Project Setup][]_ 54 | 55 | Building a product inside a repo using Whisk is a fairly straight forward 56 | process. Whisk itself lets you configure 4 attributes about what you want to 57 | build: 58 | 1. **The product** This is the actual thing you want to build. You may choose 59 | to build one or more products in a given environment 60 | 2. **The mode** This allows you to choose how you want to build the product(s). 61 | For example, there may be a mode to build the product(s) for internal use, 62 | and a mode to build the product(s) for external (public) consumption. 63 | 3. **The site** This is where you are building from. There may be build options 64 | that are affected by your physical location, such as mirror setups, use of 65 | distributed compilers, etc. 66 | 4. **The version** This defines what version of Yocto you want to build the 67 | product(s) against. Allowing this to be defined as a build parameter allows 68 | quickly testing if a product is compatible with multiple versions of Yocto 69 | and having different products use different versions independently of each 70 | other. If you don't really care about this, you may specify the version 71 | as `default` to choose the default version defined for the product(s) 72 | 73 | ### Initializing the environment 74 | 75 | To get started, you must first initialize the environment by sourcing the 76 | initialization script, usually located in your project root. When you 77 | initialize, you may specify any of the above items to be configured. If you do 78 | not specify one, the default specified in the projects `whisk.yaml` file will 79 | be used (if no default is specified there, you will be forced to provide a 80 | value). If you would like a list of options that can be specified when 81 | initializing the environment, use the `--help` option, like so: 82 | 83 | . init-build-env --help 84 | 85 | If you would like a complete list of options that may be specified for 86 | `--products`, `--mode`, `--site`, or `--version`, use the `--list` option: 87 | 88 | . init-build-env --list 89 | 90 | Once you have decided what options you would like to use, initialize your build 91 | environment, e.g. 92 | 93 | . init-build-env --product=foo --mode=debug --site=here --version=default 94 | 95 | This will setup the build environment and change your working directory to the 96 | build directory 97 | 98 | *Note:* If you choose `default` for the version and specify multiple products 99 | to be configured, whisk will fail if all specified products do not use the same 100 | version. 101 | 102 | ### Reconfiguring 103 | 104 | At any time after the environment has been initialized, you may change certain 105 | configuration options such as what product, mode, or site you are using without 106 | re-initializing the environment. To do this, run the `configure` command 107 | defined by whisk. This command takes all of the same arguments as the 108 | `init-build-env` script, so for example 109 | 110 | configure --list 111 | 112 | Will show all the possible options for products, modes, sites, and versions. 113 | 114 | *Note:* While most things can be changed when reconfiguring, there are some 115 | options that whisk can't change without re-initializing the environment, such 116 | as the version and build directory. 117 | 118 | ### Building 119 | 120 | After configuring the build environment, you should have all of the normal 121 | bitbake tools at your disposal to build products. The simplest thing to do is 122 | to run the command: 123 | 124 | bitbake all-targets 125 | 126 | This will build all of the default targets for all of the currently selected 127 | products. 128 | 129 | If you want to build a specific recipe for a specific product, be aware that 130 | whisk puts each product into it's own [multiconfig][]. So, if you want to build 131 | `core-image-minimal` for product `foo`, you would need to run: 132 | 133 | bitbake mc:product-foo:core-image-minimal 134 | 135 | Likewise, to dump the base environment for product `foo`, you would run: 136 | 137 | bitbake -e mc:product-foo 138 | 139 | ### Build output 140 | 141 | Whisk splits each build into its own build directory to ensure that they do not 142 | interfere with each other. By default, each build has [TMPDIR][] set to 143 | `${TOPDIR}/tmp/${WHISK_MODE}/${WHISK_ACTUAL_VERSION}/${WHISK_PRODUCT}` (see 144 | [Build Variables][]). 145 | 146 | In addition, each build also has [DEPLOY_DIR][] set to 147 | `${TOPDIR}/deploy/${WHISK_MODE}/${WHISK_ACUTAL_VERSION}/${WHISK_PRODUCT}` 148 | 149 | [Project Setup]: #project-setup 150 | ## Project Setup 151 | 152 | A typical project would integrate Whisk with the following steps: 153 | 1. Add Whisk to the project. We recommend pulling it in as a git submodule, but 154 | you may use whatever module composition tool you like (or even just copy the 155 | source) 156 | 2. Link the whisk [init-build-env script](./init-build-env) into the project 157 | root, for example with the command: 158 | ```sh 159 | ln -s whisk/init-build-env ./init-build-env 160 | ``` 161 | 162 | 3. Write a `whisk.yaml` file in the project root along side the 163 | `init-build-env` symlink. See [Project Configuration][] 164 | 165 | [Project Configuration]: #project-configuration 166 | ## Project Configuration 167 | 168 | The project is primarily configured through the `whisk.yaml` file, usually 169 | located in the project root. Extensive documentation on the options and their 170 | values is available in the [example configuration][]. 171 | 172 | To help validate your configuration, Whisk includes a `validate` command that 173 | can be run on your whisk.yaml file to validate it is correctly formatted. 174 | 175 | [Building in a Container]: #building-in-a-container 176 | ## Building in a Container 177 | 178 | Whisk supports building your products inside a 179 | [Pyrex](https://github.com/garmin/pyrex) container as a first-class option. To 180 | enable this support, generate a Pyrex configuration file and add the `pyrex` section 181 | to your versions as shown in the [example configuration][]. When this is 182 | enabled, Whisk will automatically setup the correct environment variables to 183 | use pyrex when invoking your build commands. 184 | 185 | [Build Variables]: #build-variables 186 | ## Build Variables 187 | 188 | Whisk sets a number of bitbake variables that can be used in recipes to make 189 | they aware of the current user configuration. These are: 190 | 191 | | Variable | Description | 192 | |----------|-------------| 193 | | `WHISK_PROJECT_ROOT` | The absolute path the to the project root | 194 | | `WHISK_PRODUCT` | The current product. When evaluated in a products multiconfig, it will be the name of the product. In the base environment, it will be `"core"` | 195 | | `WHISK_PRODUCTS` | The list of products the user has currently selected to be built | 196 | | `WHISK_MODE` | The name of the mode the user has currently selected | 197 | | `WHISK_SITE` | The name of the site the user has currently selected | 198 | | `WHISK_VERSION` | The name of the version the user has currently selected. May be `"default"` if the user specified that | 199 | | `WHISK_ACTUAL_VERSION` | the name of the version the user has specified, resolved to an actual name (e.g. will never be `"default"` | 200 | | `WHISK_TARGETS` | The combined set of all default build targets for all user configured products | 201 | 202 | In addition, some variables are set for each defined product. In this table 203 | `${product}` will be replaced with the actual name of the product: 204 | 205 | | Variable | Description | 206 | |----------|-------------| 207 | | `WHISK_TARGET_${product}` | The default targets for this product | 208 | | `WHISK_DEPLOY_DIR_${product}` | The [DEPLOY_DIR][] for this product (see [Sharing Files Between Products][]) | 209 | | `DEPLOY_DIR_${product}` | *Deprecated* Alias for `WHISK_DEPLOY_DIR_${product}`, only available when config file version is `1`. New products should not use this variable as it can cause problems with overrides. | 210 | 211 | Finally, some variables are also set in the shell environment when the hook 212 | scripts are run. These include `WHISK_PROJECT_ROOT`, `WHISK_PRODUCTS`, 213 | `WHISK_MODE`, `WHISK_SITE`, `WHISK_VERSION`, `WHISK_ACTUAL_VERSION`, and the 214 | variables in the following tables: 215 | 216 | | Variable | Description | 217 | |----------|-------------| 218 | | `WHISK_BUILD_DIR` | The absolute path to the user-specified build directory | 219 | | `WHISK_INIT` | This will have the value `"true"` if the hook is being invoked during the first initialization of the environment, and `"false"` during a reconfigure | 220 | 221 | 222 | [Sharing Files Between Products]: #sharing-files-between-products 223 | ## Sharing Files Between Products 224 | 225 | A common need that arises when building with [multiconfig][] is how to share 226 | files between different multiconfigs (or the base environment). The easiest 227 | answer to that a multiconfig that wants to share something with another 228 | multiconfig should [deploy][] the files it wants to share from a given recipe. 229 | Then, another multiconfig can [mcdepends][] on that source recipes `do_deploy` 230 | task and pull the files out of the source multiconfigs [DEPLOY_DIR][]. However, 231 | in order for this to work, the `DEPLOY_DIR` of each source multiconfig must be 232 | at a known location. To aid in this discovery, Whisk creates a 233 | `WHISK_DEPLOY_DIR` variable for each defined product, so that all products can 234 | easily reference each others deployed files. 235 | 236 | For example, assume we have two products, `source` and `dest`. `source` has a 237 | recipe called "hello.bb" that contains: 238 | 239 | inherit deploy 240 | 241 | do_deploy { 242 | echo "Hello" > ${DEPLOYDIR}/hello.txt 243 | } 244 | addtask do_deploy before do_build 245 | 246 | `dest` wants to bring in this file in another recipe, so it does: 247 | 248 | do_install() { 249 | cp ${WHISK_DEPLOY_DIR_source}/hello.txt ${D}/hello_source.txt 250 | } 251 | do_install[mcdepends] = "mc:source:dest:hello:do_deploy" 252 | 253 | 254 | Note that for this to work properly, the `source` product would have to have 255 | been configured by the user, which Whisk doesn't check. 256 | 257 | *As of this writing, it's not possible to query another multiconfigs variables, 258 | although it's been discussed. This would eliminate the need for publishing the 259 | per-product `WHISK_DEPLOY_DIR` variables, because one could simply query what 260 | `DEPLOY_DIR` is set to in the source multiconfig* 261 | 262 | [Product Layer Masking]: #product-layer-masking 263 | ## Product Layer Masking 264 | 265 | Product layer masking requires Yocto 3.2 (gatesgarth) or later, as this is the 266 | first version to support separate [BBMASK][] per multiconfig. 267 | 268 | [Layer Fetching]: #layer-fetching 269 | ## Layer Fetching 270 | 271 | Whisk has the ability to automatically fetch the layers required to build a 272 | given set of products when configuring. This allows a user to fetch only the 273 | required subset, instead of being forced to checkout all layers. This can 274 | significantly cut down on the amount of fetching, particularly since whisk 275 | encourages the same remote module to be present in the source tree multiple 276 | times for multiple versions (e.g. you will probably have oe-core or poky 277 | present multiple times in your source tree; one for each version). This can be 278 | particularly help for CI builds where the extra fetching wastes computation 279 | time. 280 | 281 | Fetching is controlled by `fetch` objects in the `whisk.yaml` file. The top 282 | level, version objects, and layer sets can all have a fetch object, see the 283 | [example configuration][] for more information. 284 | 285 | A detailed example for using fetch commands with `git submodules` will now be 286 | explained. Other methods of fetching can be used, but whisk fetching pairs 287 | particularly well with submodules. 288 | 289 | The example will focus on an example repository with a `.gitmodules` file that 290 | looks like this: 291 | 292 | ``` 293 | [submodule "whisk"] 294 | path = whisk 295 | branch = master 296 | url = https://github.com/garmin/whisk.git 297 | [submodule "yocto-3.0/zeus"] 298 | path = yocto-3.0/poky 299 | branch = zeus 300 | url = https://git.yoctoproject.org/git/poky 301 | [submodule "yocto-3.0/meta-mingw"] 302 | path = yocto-3.1/meta-mingw 303 | branch = zeus 304 | url = https://git.yoctoproject.org/git/meta-mingw 305 | [submodule "yocto-3.1/poky"] 306 | path = yocto-3.1/poky 307 | branch = dunfell 308 | url = https://git.yoctoproject.org/git/poky 309 | [submodule "yocto-3.1/meta-mingw"] 310 | path = yocto-3.1/meta-mingw 311 | branch = dunfell 312 | url = https://git.yoctoproject.org/git/meta-mingw 313 | ``` 314 | 315 | And a `whisk.yaml` file that looks like this: 316 | 317 | ```yaml 318 | version: 1 319 | 320 | defaults: 321 | mode: default 322 | site: default 323 | 324 | versions: 325 | dunfell: 326 | description: Yocto 3.1 327 | oeinit: "%{WHISK_PROJECT_ROOT}/yocto-3.1/poky/oe-init-build-env" 328 | 329 | layers: 330 | - name: core 331 | paths: 332 | - "%{WHISK_PROJECT_ROOT}/yocto-3.1/poky/meta" 333 | fetch: 334 | commands: 335 | - git submodule update --init yocto-3.1/poky 336 | 337 | - name: mingw 338 | paths: 339 | - "%{WHISK_PROJECT_ROOT}/yocto-3.1/meta-mingw" 340 | fetch: 341 | commands: 342 | - git submodule update --init yocto-3.1/meta-mingw 343 | 344 | zeus: 345 | description: Yocto 3.0 346 | oeinit: "%{WHISK_PROJECT_ROOT}/yocto-3.0/poky/oe-init-build-env" 347 | 348 | layers: 349 | - name: core 350 | paths: 351 | - "%{WHISK_PROJECT_ROOT}/yocto-3.0/poky/meta" 352 | fetch: 353 | commands: 354 | - git submodule update --init yocto-3.0/poky 355 | 356 | - name: mingw 357 | paths: 358 | - "%{WHISK_PROJECT_ROOT}/yocto-3.0/meta-mingw" 359 | fetch: 360 | commands: 361 | - git submodule update --init yocto-3.0/meta-mingw 362 | 363 | modes: 364 | default: 365 | desription: Default mode 366 | 367 | sites: 368 | default: 369 | description: Default site 370 | 371 | core: 372 | layers: 373 | - core 374 | 375 | conf: | 376 | MACHINE ?= "qemux86-64" 377 | DISTRO ?= "poky" 378 | 379 | products: 380 | albatross: 381 | default_verison: dunfell 382 | layers: 383 | - core 384 | 385 | typhoon: 386 | default_version: dunfell 387 | layers: 388 | - core 389 | - mingw 390 | 391 | phoenix: 392 | default_version: zeus 393 | layers: 394 | - core 395 | 396 | eagle: 397 | default_version: zeus 398 | layers: 399 | - core 400 | - mingw 401 | ``` 402 | 403 | Now, when a product is configured with the `--fetch` argument, whisk will 404 | automatically run `git submodule update --init ` for layers the product 405 | requires, but users who want all layers can still easily fetch everything with 406 | a simple `git submodule update --init` command. If you wanted to ensure that 407 | your CI jobs only fetch the minimum number of required layers, you might use a 408 | script like this: 409 | 410 | ```shell 411 | #! /bin/sh 412 | set -e 413 | 414 | # First fetch whisk 415 | git submodule update --init whisk 416 | 417 | # Configure whisk, instructing it to fetch the required product layers 418 | . init-build-env -n --product=$PRODUCT --fetch 419 | 420 | # Build default targets 421 | bitbake all-targets 422 | ``` 423 | 424 | [multiconfig]: https://www.yoctoproject.org/docs/3.1/bitbake-user-manual/bitbake-user-manual.html#executing-a-multiple-configuration-build 425 | [BBMASK]: https://www.yoctoproject.org/docs/3.1/bitbake-user-manual/bitbake-user-manual.html#var-bb-BBMASK 426 | [TMPDIR]: https://www.yoctoproject.org/docs/3.1/mega-manual/mega-manual.html#var-TMPDIR 427 | [DEPLOY_DIR]: https://www.yoctoproject.org/docs/3.1/mega-manual/mega-manual.html#var-DEPLOY_DIR 428 | [example configuration]: ./whisk.example.yaml 429 | [deploy]: https://www.yoctoproject.org/docs/3.1/mega-manual/mega-manual.html#ref-classes-deploy 430 | [mcdepends]: https://www.yoctoproject.org/docs/3.1/mega-manual/mega-manual.html#bb-enabling-multiple-configuration-build-dependencies 431 | -------------------------------------------------------------------------------- /bin/whisk: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | # 3 | # 2020 Garmin Ltd. or its subsidiaries 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | THIS_DIR="$(dirname $0)" 18 | WHISKENV="$THIS_DIR/whiskenv" 19 | 20 | if [ ! -z "${WHISK_NO_VENV}" ]; then 21 | echo "Not using python venv, make sure you have the necessary packages installed as system packages." 22 | set -eu 23 | exec "$THIS_DIR/../whisk.py" "$@" 24 | else 25 | set -eu 26 | exec ${WHISKENV} "$THIS_DIR/../whisk.py" "$@" 27 | fi 28 | -------------------------------------------------------------------------------- /bin/whiskenv: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | # 3 | # 2020 Garmin Ltd. or its subsidiaries 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -eu 18 | 19 | THIS_DIR="$(dirname $0)" 20 | TOP_DIR="$THIS_DIR/.." 21 | VENV_DIR="$TOP_DIR/venv" 22 | 23 | $TOP_DIR/setup-venv 24 | 25 | set +eu 26 | if [ -n "$1" ]; then 27 | . "$VENV_DIR/bin/activate" 28 | exec "$@" 29 | fi 30 | 31 | -------------------------------------------------------------------------------- /ci/dummy-init: -------------------------------------------------------------------------------- 1 | # A dummy initialization script used for testing 2 | -------------------------------------------------------------------------------- /ci/test.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | # 3 | # 2020 Garmin Ltd. or its subsidiaries 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import os 18 | import pathlib 19 | import shutil 20 | import subprocess 21 | import tempfile 22 | import textwrap 23 | import unittest 24 | import yaml 25 | 26 | ROOT = pathlib.Path(__file__).parent.parent.absolute() 27 | 28 | 29 | class WhiskTests(object): 30 | def setUp(self): 31 | self.project_root = ROOT / "test" / ("%d" % os.getpid()) / self.id() 32 | 33 | def cleanup_project(): 34 | if self.project_root.is_dir(): 35 | shutil.rmtree(self.project_root) 36 | 37 | self.addCleanup(cleanup_project) 38 | cleanup_project() 39 | self.project_root.mkdir(parents=True) 40 | 41 | oldcwd = os.getcwd() 42 | os.chdir(self.project_root) 43 | self.addCleanup(os.chdir, oldcwd) 44 | 45 | os.symlink(ROOT / "init-build-env", self.project_root / "init-build-env") 46 | 47 | self.conf_file = self.project_root / "whisk.yaml" 48 | 49 | def write_conf(self, conf): 50 | with self.conf_file.open("w") as f: 51 | f.write(textwrap.dedent(conf)) 52 | 53 | def append_conf(self, conf): 54 | with self.conf_file.open("a+") as f: 55 | f.write(textwrap.dedent(conf)) 56 | 57 | def assertShellCode(self, fragment, expected_capture={}, env=None, success=True): 58 | (fd, log_file) = tempfile.mkstemp() 59 | self.addCleanup(lambda: os.unlink(log_file)) 60 | os.close(fd) 61 | 62 | (fd, capture_file) = tempfile.mkstemp() 63 | self.addCleanup(lambda: os.unlink(capture_file)) 64 | os.close(fd) 65 | 66 | with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8") as f: 67 | f.write("set -eo pipefail\n") 68 | f.write(textwrap.dedent(fragment)) 69 | f.write("\n") 70 | for v in expected_capture.keys(): 71 | f.write('echo "%s=$%s" >> %s\n' % (v, v, capture_file)) 72 | f.flush() 73 | 74 | if env is None: 75 | env = os.environ 76 | 77 | with open(log_file, "w") as log: 78 | p = subprocess.run( 79 | ["/bin/bash", f.name], 80 | stdout=log, 81 | stderr=subprocess.STDOUT, 82 | cwd=self.project_root, 83 | env=env, 84 | ) 85 | 86 | with open(log_file, "r") as log: 87 | if success: 88 | self.assertEqual( 89 | p.returncode, 90 | 0, 91 | "Process exited with non-zero exit code. Output:\n%s" % log.read(), 92 | ) 93 | else: 94 | self.assertNotEqual( 95 | p.returncode, 96 | 0, 97 | "Process exited with zero exit code. Output:\n%s" % log.read(), 98 | ) 99 | 100 | actual_captured = {} 101 | with open(capture_file, "r") as f: 102 | for line in f: 103 | line = line.rstrip() 104 | key, val = line.split("=", 1) 105 | actual_captured[key] = val 106 | 107 | for key, value in expected_capture.items(): 108 | if isinstance(value, set): 109 | # If the input capture variable set is a set, convert the 110 | # actual captured variables to a set (split by whitespace) also 111 | if key in actual_captured: 112 | actual_captured[key] = set(actual_captured[key].split()) 113 | 114 | self.assertDictEqual(actual_captured, expected_capture) 115 | 116 | def assertConfigVar(self, name, value): 117 | with self.conf_file.open("r") as f: 118 | data = yaml.load(f.read(), Loader=yaml.Loader) 119 | cache_file = pathlib.Path( 120 | data.get("cache", self.project_root / ".config.yaml") 121 | ) 122 | 123 | with cache_file.open("r") as f: 124 | data = yaml.load(f.read(), Loader=yaml.Loader) 125 | 126 | self.assertIn(name, data) 127 | self.assertEqual(data[name], value) 128 | 129 | 130 | class WhiskExampleConfTests(unittest.TestCase): 131 | def test_validate_example(self): 132 | p = subprocess.run( 133 | [ROOT / "bin" / "whisk", "validate", ROOT / "whisk.example.yaml"], 134 | stdout=subprocess.PIPE, 135 | stderr=subprocess.STDOUT, 136 | ) 137 | self.assertEqual( 138 | p.returncode, 0, "Validation failed with:\n%s" % p.stdout.decode("utf-8") 139 | ) 140 | 141 | 142 | class WhiskCommandTests(WhiskTests, unittest.TestCase): 143 | def test_relative_invocation(self): 144 | p = subprocess.run( 145 | [os.path.relpath(ROOT / "bin" / "whisk", self.project_root), "--help"], 146 | stdout=subprocess.PIPE, 147 | stderr=subprocess.STDOUT, 148 | ) 149 | self.assertEqual( 150 | p.returncode, 0, "Unable to invoke whisk:\n%s" % p.stdout.decode("utf-8") 151 | ) 152 | 153 | def test_absolute_invocation(self): 154 | p = subprocess.run( 155 | [ROOT / "bin" / "whisk", "--help"], 156 | stdout=subprocess.PIPE, 157 | stderr=subprocess.STDOUT, 158 | ) 159 | self.assertEqual( 160 | p.returncode, 0, "Unable to invoke whisk:\n%s" % p.stdout.decode("utf-8") 161 | ) 162 | 163 | 164 | class WhiskConfParseTests(WhiskTests, unittest.TestCase): 165 | def setUp(self): 166 | super().setUp() 167 | 168 | self.write_conf( 169 | """ 170 | version: 2 171 | defaults: 172 | products: 173 | - test-dunfell 174 | mode: mode 175 | site: site 176 | 177 | versions: 178 | dunfell: 179 | oeinit: {ROOT}/ci/dummy-init 180 | 181 | products: 182 | test-dunfell: 183 | default_version: dunfell 184 | 185 | modes: 186 | mode: {{}} 187 | 188 | sites: 189 | site: {{}} 190 | 191 | """.format( 192 | ROOT=ROOT 193 | ) 194 | ) 195 | 196 | def test_project_root_expansion(self): 197 | temp_root = self.project_root / "temp_root" 198 | temp_root.mkdir() 199 | 200 | self.append_conf( 201 | """\ 202 | project_root: temp_root 203 | hooks: 204 | pre_init: | 205 | MY_PROJECT_ROOT=%{WHISK_PROJECT_ROOT} 206 | """ 207 | ) 208 | 209 | self.assertShellCode( 210 | """\ 211 | . init-build-env 212 | """, 213 | { 214 | "WHISK_PROJECT_ROOT": str(temp_root.absolute()), 215 | "MY_PROJECT_ROOT": str(temp_root.absolute()), 216 | }, 217 | ) 218 | 219 | def test_env_var_expansion(self): 220 | self.append_conf( 221 | """\ 222 | hooks: 223 | pre_init: | 224 | MY_VAR=%{TEST_VAR} 225 | """ 226 | ) 227 | 228 | env = os.environ.copy() 229 | env["TEST_VAR"] = "FOOBAR" 230 | 231 | self.assertShellCode( 232 | """\ 233 | . init-build-env 234 | """, 235 | { 236 | "MY_VAR": "FOOBAR", 237 | }, 238 | env=env, 239 | ) 240 | 241 | # A missing variable causes a failure 242 | self.assertShellCode( 243 | """\ 244 | . init-build-env 245 | """, 246 | success=False, 247 | ) 248 | 249 | def test_maintainers(self): 250 | self.write_conf( 251 | """ 252 | version: 2 253 | defaults: 254 | products: 255 | - test-dunfell 256 | mode: mode 257 | site: site 258 | 259 | versions: 260 | dunfell: 261 | oeinit: {ROOT}/ci/dummy-init 262 | 263 | products: 264 | test-dunfell: 265 | maintainers: 266 | - name: John Doe 267 | email: john.doe@company.com 268 | - name: Jane Doe 269 | email: jane.doe@company.com 270 | default_version: dunfell 271 | 272 | modes: 273 | mode: {{}} 274 | 275 | sites: 276 | site: {{}} 277 | 278 | """.format( 279 | ROOT=ROOT 280 | ) 281 | ) 282 | 283 | self.assertShellCode( 284 | """\ 285 | . init-build-env 286 | """ 287 | ) 288 | 289 | def test_tags(self): 290 | self.write_conf( 291 | """\ 292 | --- 293 | version: 2 294 | defaults: 295 | products: 296 | - test-dunfell 297 | mode: mode 298 | site: site 299 | 300 | versions: 301 | dunfell: 302 | oeinit: {ROOT}/ci/dummy-init 303 | tags: 304 | string: "bar" 305 | number: 123 306 | array: 307 | - 1 308 | - 2 309 | - 3 310 | dictionary: 311 | key1: "A" 312 | key2: "B" 313 | 314 | products: 315 | test-dunfell: 316 | default_version: dunfell 317 | tags: 318 | string: "bar" 319 | number: 123 320 | array: 321 | - 1 322 | - 2 323 | - 3 324 | dictionary: 325 | key1: "A" 326 | key2: "B" 327 | 328 | 329 | modes: 330 | mode: 331 | tags: 332 | string: "bar" 333 | number: 123 334 | array: 335 | - 1 336 | - 2 337 | - 3 338 | dictionary: 339 | key1: "A" 340 | key2: "B" 341 | 342 | sites: 343 | site: 344 | tags: 345 | string: "bar" 346 | number: 123 347 | array: 348 | - 1 349 | - 2 350 | - 3 351 | dictionary: 352 | key1: "A" 353 | key2: "B" 354 | 355 | """.format( 356 | ROOT=ROOT 357 | ) 358 | ) 359 | 360 | self.assertShellCode( 361 | """\ 362 | . init-build-env 363 | """ 364 | ) 365 | 366 | 367 | class WhiskFetchTests(WhiskTests, unittest.TestCase): 368 | def setUp(self): 369 | super().setUp() 370 | self.write_conf( 371 | """\ 372 | version: 2 373 | defaults: 374 | mode: mode 375 | site: site 376 | 377 | fetch: 378 | commands: 379 | - echo main > %{{WHISK_PROJECT_ROOT}}/fetch 380 | 381 | versions: 382 | dunfell: 383 | oeinit: {ROOT}/ci/dummy-init 384 | fetch: 385 | commands: 386 | - echo dunfell >> %{{WHISK_PROJECT_ROOT}}/fetch 387 | 388 | layers: 389 | - name: core 390 | fetch: 391 | commands: 392 | - echo core >> %{{WHISK_PROJECT_ROOT}}/fetch 393 | - name: A 394 | fetch: 395 | commands: 396 | - echo A >> %{{WHISK_PROJECT_ROOT}}/fetch 397 | - name: test-environment 398 | fetch: 399 | commands: 400 | - pwd >> %{{WHISK_PROJECT_ROOT}}/fetch 401 | - echo ${{WHISK_PROJECT_ROOT}} >> %{{WHISK_PROJECT_ROOT}}/fetch 402 | - echo ${{FOO}} >> %{{WHISK_PROJECT_ROOT}}/fetch 403 | 404 | zeus: 405 | oeinit: {ROOT}/ci/dummy-init 406 | fetch: 407 | commands: 408 | - echo zeus >> %{{WHISK_PROJECT_ROOT}}/fetch 409 | 410 | layers: 411 | - name: core 412 | fetch: 413 | commands: 414 | - echo core >> %{{WHISK_PROJECT_ROOT}}/fetch 415 | - name: A 416 | fetch: 417 | commands: 418 | - echo A >> %{{WHISK_PROJECT_ROOT}}/fetch 419 | 420 | modes: 421 | mode: {{}} 422 | 423 | sites: 424 | site: {{}} 425 | 426 | """.format( 427 | ROOT=ROOT 428 | ) 429 | ) 430 | 431 | def assertFetches(self, code, fetches, **kwargs): 432 | self.assertShellCode(code, **kwargs) 433 | 434 | with (self.project_root / "fetch").open("r") as f: 435 | lines = [l.rstrip() for l in f.readlines()] 436 | 437 | self.assertEqual(lines, fetches) 438 | 439 | def test_fetch(self): 440 | self.append_conf( 441 | """\ 442 | products: 443 | test-dunfell: 444 | default_version: dunfell 445 | 446 | test-dunfell-core: 447 | default_version: dunfell 448 | layers: 449 | - core 450 | 451 | test-dunfell-A: 452 | default_version: dunfell 453 | layers: 454 | - A 455 | 456 | test-dunfell-core-A: 457 | default_version: dunfell 458 | layers: 459 | - A 460 | - core 461 | 462 | test-zeus: 463 | default_version: zeus 464 | 465 | test-zeus-core: 466 | default_version: zeus 467 | layers: 468 | - core 469 | 470 | test-zeus-A: 471 | default_version: zeus 472 | layers: 473 | - A 474 | 475 | test-zeus-core-A: 476 | default_version: zeus 477 | layers: 478 | - A 479 | - core 480 | """ 481 | ) 482 | 483 | for v in ("dunfell", "zeus"): 484 | with self.subTest(v): 485 | self.assertFetches( 486 | """\ 487 | . init-build-env --product=test-%s --fetch 488 | """ 489 | % v, 490 | ["main", v], 491 | ) 492 | 493 | for l in (("core",), ("A",), ("core", "A")): 494 | name = "%s-%s" % (v, "-".join(l)) 495 | with self.subTest(name): 496 | self.assertFetches( 497 | """\ 498 | . init-build-env --product=test-%s --fetch 499 | """ 500 | % name, 501 | ["main", v] + list(l), 502 | ) 503 | 504 | def test_fetch_env(self): 505 | self.append_conf( 506 | """\ 507 | products: 508 | test-environment: 509 | default_version: dunfell 510 | layers: 511 | - test-environment 512 | """ 513 | ) 514 | 515 | env = os.environ.copy() 516 | env["FOO"] = "BAR" 517 | 518 | self.assertFetches( 519 | """ 520 | . init-build-env --product=test-environment --fetch 521 | """, 522 | [ 523 | "main", 524 | "dunfell", 525 | str(self.project_root), 526 | str(self.project_root), 527 | "BAR", 528 | ], 529 | env=env, 530 | ) 531 | 532 | 533 | class WhiskVersionTests(WhiskTests, unittest.TestCase): 534 | def setUp(self): 535 | super().setUp() 536 | self.write_conf( 537 | """\ 538 | version: 2 539 | defaults: 540 | mode: mode 541 | site: site 542 | 543 | hooks: 544 | env_passthrough_vars: 545 | - TEST_VAR 546 | 547 | versions: 548 | kirkstone: 549 | compat: kirkstone 550 | oeinit: {ROOT}/ci/dummy-init 551 | dunfell: 552 | compat: dunfell 553 | oeinit: {ROOT}/ci/dummy-init 554 | zeus: 555 | oeinit: {ROOT}/ci/dummy-init 556 | 557 | kirkstone-auto: 558 | oeinit: {ROOT}/ci/dummy-init 559 | layers: 560 | - name: core 561 | paths: 562 | - {PROJECT_ROOT}/kirkstone/ 563 | 564 | dunfell-auto: 565 | oeinit: {ROOT}/ci/dummy-init 566 | layers: 567 | - name: core 568 | paths: 569 | - {PROJECT_ROOT}/dunfell/ 570 | 571 | pyro-auto: 572 | oeinit: {ROOT}/ci/dummy-init 573 | # No layers, so "auto" will resolve to the newest version 574 | # without LAYERSERIES_CORENAMES (pyro) 575 | 576 | 577 | future-auto: 578 | oeinit: {ROOT}/ci/dummy-init 579 | layers: 580 | - name: core 581 | paths: 582 | - {PROJECT_ROOT}/future/ 583 | 584 | future: 585 | oeinit: {ROOT}/ci/dummy-init 586 | compat: future 587 | 588 | products: 589 | test-kirkstone: 590 | default_version: kirkstone 591 | test-dunfell: 592 | default_version: dunfell 593 | test-dunfell2: 594 | default_version: dunfell 595 | test-zeus: 596 | default_version: zeus 597 | 598 | test-kirkstone-auto: 599 | default_version: kirkstone-auto 600 | 601 | test-dunfell-auto: 602 | default_version: dunfell-auto 603 | 604 | test-pyro-auto: 605 | default_version: pyro-auto 606 | 607 | test-future-auto: 608 | default_version: future-auto 609 | 610 | test-future: 611 | default_version: future 612 | 613 | modes: 614 | mode: {{}} 615 | 616 | sites: 617 | site: {{}} 618 | 619 | """.format( 620 | ROOT=ROOT, 621 | PROJECT_ROOT=self.project_root, 622 | ) 623 | ) 624 | 625 | for c in ("dunfell", "kirkstone", "future"): 626 | conf_dir = self.project_root / c / "conf" 627 | conf_dir.mkdir(parents=True, exist_ok=True) 628 | 629 | with (conf_dir / "layer.conf").open("w") as f: 630 | f.write(f'LAYERSERIES_CORENAMES = "{c}"\n') 631 | 632 | def test_default_version(self): 633 | self.assertShellCode( 634 | """\ 635 | . init-build-env --product=test-dunfell --version=default 636 | """, 637 | { 638 | "WHISK_VERSION": "default", 639 | "WHISK_ACTUAL_VERSION": "dunfell", 640 | }, 641 | ) 642 | 643 | self.assertShellCode( 644 | """\ 645 | . init-build-env --product=test-dunfell 646 | """, 647 | { 648 | "WHISK_VERSION": "default", 649 | "WHISK_ACTUAL_VERSION": "dunfell", 650 | }, 651 | ) 652 | 653 | self.assertShellCode( 654 | """\ 655 | . init-build-env --product=test-zeus --version=default 656 | """, 657 | { 658 | "WHISK_VERSION": "default", 659 | "WHISK_ACTUAL_VERSION": "zeus", 660 | }, 661 | ) 662 | 663 | self.assertShellCode( 664 | """\ 665 | . init-build-env --product=test-zeus 666 | """, 667 | { 668 | "WHISK_VERSION": "default", 669 | "WHISK_ACTUAL_VERSION": "zeus", 670 | }, 671 | ) 672 | 673 | def test_explicit_version(self): 674 | self.assertShellCode( 675 | """\ 676 | . init-build-env --product=test-dunfell --version=zeus 677 | """, 678 | { 679 | "WHISK_VERSION": "zeus", 680 | "WHISK_ACTUAL_VERSION": "zeus", 681 | }, 682 | ) 683 | 684 | self.assertShellCode( 685 | """\ 686 | . init-build-env --product=test-dunfell --product=test-zeus --version=zeus 687 | """, 688 | { 689 | "WHISK_VERSION": "zeus", 690 | "WHISK_ACTUAL_VERSION": "zeus", 691 | }, 692 | ) 693 | 694 | def test_mixed_product_implicit_default(self): 695 | # Mixing products with different versions using an implicit default 696 | # should fail 697 | self.assertShellCode( 698 | """\ 699 | . init-build-env --product=test-dunfell --product=test-zeus 700 | """, 701 | success=False, 702 | ) 703 | 704 | def test_mixed_product_explicit_default(self): 705 | # Different version products with explicit default should fail 706 | self.assertShellCode( 707 | """\ 708 | . init-build-env --product=test-dunfell --product=test-zeus --version=default 709 | """, 710 | success=False, 711 | ) 712 | 713 | def test_changing_compatible_version_when_default(self): 714 | # Changing to a product with the same version after configuring 715 | self.assertShellCode( 716 | """\ 717 | . init-build-env --product=test-dunfell --version=default 718 | configure --product=test-dunfell2 719 | """, 720 | { 721 | "WHISK_PRODUCTS": "test-dunfell2", 722 | "WHISK_VERSION": "default", 723 | "WHISK_ACTUAL_VERSION": "dunfell", 724 | }, 725 | ) 726 | 727 | def test_changing_incompatible_version_when_default(self): 728 | # Changing to a product with a different version after configuring with 729 | # default should fail 730 | self.assertShellCode( 731 | """\ 732 | . init-build-env --product=test-dunfell --version=default 733 | configure --product=test-zeus 734 | """, 735 | success=False, 736 | ) 737 | self.assertConfigVar("version", "default") 738 | 739 | def test_changing_incompatible_version_with_explicit_version(self): 740 | # Changing to a product with explicit version 741 | self.assertShellCode( 742 | """\ 743 | . init-build-env --product=test-dunfell --version=dunfell 744 | configure --product=test-zeus 745 | """, 746 | { 747 | "WHISK_VERSION": "dunfell", 748 | "WHISK_ACTUAL_VERSION": "dunfell", 749 | }, 750 | ) 751 | 752 | def test_default_presists(self): 753 | # Default value persists between shell instances 754 | config_vars = { 755 | "WHISK_VERSION": "default", 756 | "WHISK_ACTUAL_VERSION": "dunfell", 757 | "WHISK_PRODUCTS": "test-dunfell", 758 | } 759 | self.assertShellCode( 760 | """\ 761 | . init-build-env --product=test-dunfell --version=default 762 | """, 763 | config_vars, 764 | ) 765 | self.assertShellCode( 766 | """\ 767 | . init-build-env 768 | """, 769 | config_vars, 770 | ) 771 | 772 | def test_explicit_version_persists(self): 773 | config_vars = { 774 | "WHISK_VERSION": "zeus", 775 | "WHISK_ACTUAL_VERSION": "zeus", 776 | "WHISK_PRODUCTS": "test-dunfell", 777 | } 778 | self.assertShellCode( 779 | """\ 780 | . init-build-env --product=test-dunfell --version=zeus 781 | """, 782 | config_vars, 783 | ) 784 | self.assertShellCode( 785 | """\ 786 | . init-build-env 787 | """, 788 | config_vars, 789 | ) 790 | 791 | def test_default_persists_across_versions(self): 792 | # Tests that when the default version persists, it means whatever version 793 | # is supported by the configured products 794 | self.assertShellCode( 795 | """\ 796 | . init-build-env --product=test-dunfell --version=default 797 | """, 798 | { 799 | "WHISK_VERSION": "default", 800 | "WHISK_ACTUAL_VERSION": "dunfell", 801 | "WHISK_PRODUCTS": "test-dunfell", 802 | }, 803 | ) 804 | self.assertShellCode( 805 | """\ 806 | . init-build-env --product=test-zeus 807 | """, 808 | { 809 | "WHISK_VERSION": "default", 810 | "WHISK_ACTUAL_VERSION": "zeus", 811 | "WHISK_PRODUCTS": "test-zeus", 812 | }, 813 | ) 814 | 815 | def test_changing_saved_explicit_with_default(self): 816 | # Test that the version is allowed to be changed to default after it 817 | # was explicitly set to a version not compatible with the new default 818 | self.assertShellCode( 819 | """\ 820 | . init-build-env --product=test-dunfell --version=dunfell 821 | """, 822 | { 823 | "WHISK_VERSION": "dunfell", 824 | "WHISK_ACTUAL_VERSION": "dunfell", 825 | "WHISK_PRODUCTS": "test-dunfell", 826 | }, 827 | ) 828 | self.assertShellCode( 829 | """\ 830 | . init-build-env --product=test-zeus --version=default 831 | """, 832 | { 833 | "WHISK_VERSION": "default", 834 | "WHISK_ACTUAL_VERSION": "zeus", 835 | "WHISK_PRODUCTS": "test-zeus", 836 | }, 837 | ) 838 | 839 | def test_keeping_explicit_verison(self): 840 | # Tests that if an explicit version is set, it is preserved even when 841 | # switching to a product that would be incompatible by default 842 | self.assertShellCode( 843 | """\ 844 | . init-build-env --product=test-dunfell --version=dunfell 845 | """, 846 | { 847 | "WHISK_VERSION": "dunfell", 848 | "WHISK_ACTUAL_VERSION": "dunfell", 849 | }, 850 | ) 851 | self.assertShellCode( 852 | """\ 853 | . init-build-env --product=test-zeus 854 | configure | grep "dunfell" 855 | """, 856 | { 857 | "WHISK_VERSION": "dunfell", 858 | "WHISK_ACTUAL_VERSION": "dunfell", 859 | }, 860 | ) 861 | 862 | def test_compat_explicit(self): 863 | # Test that using a version with an explicit compat option reports that 864 | # version 865 | for version in "dunfell", "kirkstone", "future": 866 | with self.subTest(version=version): 867 | self.assertShellCode( 868 | f"""\ 869 | . init-build-env --product=test-{version} 870 | """, 871 | { 872 | "WHISK_COMPAT": version, 873 | }, 874 | ) 875 | 876 | def test_compat_future_auto(self): 877 | # Test that auto detection of version by reading layer.conf works 878 | for version in "pyro", "dunfell", "kirkstone": 879 | with self.subTest(version=version): 880 | self.assertShellCode( 881 | f"""\ 882 | . init-build-env --product=test-{version}-auto 883 | """, 884 | { 885 | "WHISK_COMPAT": version, 886 | }, 887 | ) 888 | 889 | def test_dunfell_passthrough(self): 890 | self.assertShellCode( 891 | """\ 892 | . init-build-env --product=test-dunfell 893 | """, 894 | { 895 | "WHISK_COMPAT": "dunfell", 896 | "BB_ENV_EXTRAWHITE": { 897 | "TEST_VAR", 898 | "WHISK_ACTUAL_VERSION", 899 | "WHISK_MODE", 900 | "WHISK_PRODUCTS", 901 | "WHISK_PROJECT_ROOT", 902 | "WHISK_SITE", 903 | }, 904 | "BB_ENV_PASSTHROUGH_ADDITIONS": "", 905 | }, 906 | ) 907 | 908 | def test_kirkstone_passthrough(self): 909 | self.assertShellCode( 910 | """\ 911 | . init-build-env --product=test-kirkstone 912 | """, 913 | { 914 | "WHISK_COMPAT": "kirkstone", 915 | "BB_ENV_EXTRAWHITE": "", 916 | "BB_ENV_PASSTHROUGH_ADDITIONS": { 917 | "TEST_VAR", 918 | "WHISK_ACTUAL_VERSION", 919 | "WHISK_MODE", 920 | "WHISK_PRODUCTS", 921 | "WHISK_PROJECT_ROOT", 922 | "WHISK_SITE", 923 | }, 924 | }, 925 | ) 926 | 927 | 928 | class WhiskInitTests(WhiskTests, unittest.TestCase): 929 | def setUp(self): 930 | super().setUp() 931 | self.write_conf( 932 | """\ 933 | version: 2 934 | 935 | versions: 936 | dunfell: 937 | oeinit: {ROOT}/ci/dummy-init 938 | zeus: 939 | oeinit: {ROOT}/ci/dummy-init 940 | 941 | products: 942 | test-dunfell: 943 | default_version: dunfell 944 | test-zeus: 945 | default_version: zeus 946 | 947 | modes: 948 | modeA: {{}} 949 | modeB: {{}} 950 | 951 | sites: 952 | siteA: {{}} 953 | siteB: {{}} 954 | 955 | """.format( 956 | ROOT=ROOT 957 | ) 958 | ) 959 | 960 | def test_required_mode(self): 961 | self.assertShellCode( 962 | """\ 963 | . init-build-env --product=test-dunfell --site=siteA 964 | """, 965 | success=False, 966 | ) 967 | 968 | def test_required_site(self): 969 | self.assertShellCode( 970 | """\ 971 | . init-build-env --product=test-dunfell --mode=modeA 972 | """, 973 | success=False, 974 | ) 975 | 976 | def test_required_product(self): 977 | self.assertShellCode( 978 | """\ 979 | . init-build-env --site=siteA --mode=modeA 980 | """, 981 | success=False, 982 | ) 983 | 984 | def test_multiple_products_joined(self): 985 | self.assertShellCode( 986 | """\ 987 | . init-build-env --products="test-dunfell test-zeus" --version=dunfell --mode=modeA --site=siteA 988 | """, 989 | { 990 | "WHISK_PRODUCTS": "test-dunfell test-zeus", 991 | }, 992 | ) 993 | 994 | def test_multiple_products_split(self): 995 | self.assertShellCode( 996 | """\ 997 | . init-build-env --product=test-dunfell --product=test-zeus --version=dunfell --mode=modeA --site=siteA 998 | """, 999 | { 1000 | "WHISK_PRODUCTS": "test-dunfell test-zeus", 1001 | }, 1002 | ) 1003 | 1004 | def test_defaults(self): 1005 | self.append_conf( 1006 | """\ 1007 | defaults: 1008 | mode: modeA 1009 | site: siteA 1010 | products: 1011 | - test-dunfell 1012 | """ 1013 | ) 1014 | 1015 | self.assertShellCode( 1016 | """\ 1017 | . init-build-env 1018 | """, 1019 | { 1020 | "WHISK_SITE": "siteA", 1021 | "WHISK_MODE": "modeA", 1022 | "WHISK_PRODUCTS": "test-dunfell", 1023 | }, 1024 | ) 1025 | 1026 | # Test defaults can be overridden 1027 | self.assertShellCode( 1028 | """\ 1029 | . init-build-env --mode=modeB --site=siteB --product=test-zeus 1030 | """, 1031 | { 1032 | "WHISK_SITE": "siteB", 1033 | "WHISK_MODE": "modeB", 1034 | "WHISK_PRODUCTS": "test-zeus", 1035 | }, 1036 | ) 1037 | 1038 | # Test overrides are remembered in a subsequent environment initialization 1039 | self.assertShellCode( 1040 | """\ 1041 | . init-build-env 1042 | """, 1043 | { 1044 | "WHISK_SITE": "siteB", 1045 | "WHISK_MODE": "modeB", 1046 | "WHISK_PRODUCTS": "test-zeus", 1047 | }, 1048 | ) 1049 | 1050 | def test_ignore_cache(self): 1051 | self.append_conf( 1052 | """\ 1053 | defaults: 1054 | mode: modeA 1055 | site: siteA 1056 | products: 1057 | - test-dunfell 1058 | """ 1059 | ) 1060 | 1061 | self.assertShellCode( 1062 | """\ 1063 | . init-build-env --mode=modeB --site=siteB --product=test-zeus 1064 | """, 1065 | { 1066 | "WHISK_SITE": "siteB", 1067 | "WHISK_MODE": "modeB", 1068 | "WHISK_PRODUCTS": "test-zeus", 1069 | }, 1070 | ) 1071 | 1072 | # --no-config causes the cache to be ignored 1073 | self.assertShellCode( 1074 | """\ 1075 | . init-build-env --no-config 1076 | """, 1077 | { 1078 | "WHISK_SITE": "siteA", 1079 | "WHISK_MODE": "modeA", 1080 | "WHISK_PRODUCTS": "test-dunfell", 1081 | }, 1082 | ) 1083 | 1084 | # Cache wasn't overwritten by --no-config and continues to have the 1085 | # originally configured values 1086 | self.assertShellCode( 1087 | """\ 1088 | . init-build-env 1089 | """, 1090 | { 1091 | "WHISK_SITE": "siteB", 1092 | "WHISK_MODE": "modeB", 1093 | "WHISK_PRODUCTS": "test-zeus", 1094 | }, 1095 | ) 1096 | 1097 | 1098 | class WhiskNonMulticonfigTests(WhiskTests, unittest.TestCase): 1099 | def setUp(self): 1100 | super().setUp() 1101 | self.write_conf( 1102 | """\ 1103 | version: 2 1104 | 1105 | versions: 1106 | dunfell: 1107 | oeinit: {ROOT}/ci/dummy-init 1108 | 1109 | products: 1110 | 1111 | test-non-mc1: 1112 | default_version: dunfell 1113 | multiconfig_enabled: false 1114 | 1115 | test-non-mc2: 1116 | default_version: dunfell 1117 | multiconfig_enabled: false 1118 | 1119 | test-non-mc3: 1120 | default_version: dunfell 1121 | multiconfig_enabled: false 1122 | multiconfigs: ["test-mc1", "test-mc2"] 1123 | 1124 | test-mc1: 1125 | default_version: dunfell 1126 | multiconfig_enabled: true 1127 | 1128 | test-mc2: 1129 | default_version: dunfell 1130 | # multiconfig_enabled is true by default 1131 | 1132 | modes: 1133 | modeA: {{}} 1134 | 1135 | sites: 1136 | siteA: {{}} 1137 | 1138 | defaults: 1139 | mode: modeA 1140 | site: siteA 1141 | products: 1142 | - test-non-mc1 1143 | 1144 | """.format( 1145 | ROOT=ROOT 1146 | ) 1147 | ) 1148 | 1149 | def test_single_non_multiconfig_product(self): 1150 | # Using a single non-multiconfig product is okay. 1151 | self.assertShellCode( 1152 | """\ 1153 | . init-build-env --product=test-non-mc1 1154 | """, 1155 | success=True, 1156 | ) 1157 | 1158 | def test_multiple_non_multiconfig_products(self): 1159 | # Attempting to use two non-multiconfig products at the same time 1160 | # should fail. 1161 | self.assertShellCode( 1162 | """\ 1163 | . init-build-env --product=test-non-mc1 --product=test-non-mc2 1164 | """, 1165 | success=False, 1166 | ) 1167 | 1168 | def test_multiple_multiconfig_products(self): 1169 | # Attempting to use any multiconfig product in conjunction with a 1170 | # non-multiconfig product should fail. 1171 | self.assertShellCode( 1172 | """\ 1173 | . init-build-env --product=test-mc1 --product=test-mc2 1174 | """, 1175 | success=True, 1176 | ) 1177 | 1178 | def test_non_multiconfig_product_with_other_multiconfigs_enabled(self): 1179 | # Attempting to enable additional multiconfigs which should implicitly 1180 | # be enabled when a non-multiconfig product is selected, should fail. 1181 | self.assertShellCode( 1182 | """\ 1183 | . init-build-env --product=test-non-mc3 1184 | """, 1185 | success=False, 1186 | ) 1187 | 1188 | 1189 | class WhiskBbmaskTests(WhiskTests, unittest.TestCase): 1190 | def setUp(self): 1191 | super().setUp() 1192 | self.write_conf( 1193 | """\ 1194 | version: 2 1195 | 1196 | versions: 1197 | dunfell: 1198 | oeinit: {ROOT}/ci/dummy-init 1199 | 1200 | layers: 1201 | 1202 | - name: layer1 1203 | paths: 1204 | - "%{{WHISK_PROJECT_ROOT}}/layers/meta-layer1" 1205 | 1206 | - name: layer2 1207 | paths: 1208 | - "%{{WHISK_PROJECT_ROOT}}/layers/meta-layer2" 1209 | bbmask: 1210 | - "%{{WHISK_PROJECT_ROOT}}/layers/meta-layer2/recipes-bad/bad.bb" 1211 | 1212 | products: 1213 | 1214 | using-collection1: 1215 | default_version: dunfell 1216 | layers: 1217 | - layer1 1218 | 1219 | using-collection2: 1220 | default_version: dunfell 1221 | layers: 1222 | - layer2 1223 | 1224 | modes: 1225 | modeA: {{}} 1226 | 1227 | sites: 1228 | siteA: {{}} 1229 | 1230 | defaults: 1231 | mode: modeA 1232 | site: siteA 1233 | products: 1234 | - using-collection1 1235 | 1236 | """.format( 1237 | ROOT=ROOT 1238 | ) 1239 | ) 1240 | 1241 | def readBbconfLines(self): 1242 | with open( 1243 | os.path.join(self.project_root, "build", "conf", "bblayers.conf"), "r" 1244 | ) as bblayers_file: 1245 | return bblayers_file.readlines() 1246 | 1247 | def assertInBbconf(self, line): 1248 | self.assertIn(line, self.readBbconfLines()) 1249 | 1250 | def assertNotInBbconf(self, line): 1251 | self.assertNotIn(line, self.readBbconfLines()) 1252 | 1253 | def test_layer_bbmask(self): 1254 | # Check that the basic formulation of per-product masks for layer 1255 | # collections is working. 1256 | self.assertShellCode( 1257 | """\ 1258 | . init-build-env --product=using-collection1 --product=using-collection2 1259 | """, 1260 | success=True, 1261 | ) 1262 | 1263 | self.assertInBbconf( 1264 | 'BBMASK_using-collection1 += "{PROJECT_ROOT}/layers/meta-layer2"\n'.format( 1265 | PROJECT_ROOT=self.project_root 1266 | ) 1267 | ) 1268 | self.assertInBbconf( 1269 | 'BBMASK_using-collection2 += "{PROJECT_ROOT}/layers/meta-layer1"\n'.format( 1270 | PROJECT_ROOT=self.project_root 1271 | ) 1272 | ) 1273 | self.assertNotInBbconf( 1274 | 'BBMASK_using-collection1 += "{PROJECT_ROOT}/layers/meta-layer1"\n'.format( 1275 | PROJECT_ROOT=self.project_root 1276 | ) 1277 | ) 1278 | self.assertNotInBbconf( 1279 | 'BBMASK_using-collection2 += "{PROJECT_ROOT}/layers/meta-layer2"\n'.format( 1280 | PROJECT_ROOT=self.project_root 1281 | ) 1282 | ) 1283 | 1284 | def test_active_layer_collection_bbmask(self): 1285 | # Check that laye-specific bbmasks are applied on the product using the layers. 1286 | self.assertShellCode( 1287 | """\ 1288 | . init-build-env --product=using-collection1 --product=using-collection2 1289 | """, 1290 | success=True, 1291 | ) 1292 | 1293 | self.assertInBbconf( 1294 | 'BBMASK_using-collection2 += "{PROJECT_ROOT}/layers/meta-layer2/recipes-bad/bad.bb"\n'.format( 1295 | PROJECT_ROOT=self.project_root 1296 | ) 1297 | ) 1298 | 1299 | def test_inactive_layer_collection_bbmask(self): 1300 | # Check that laye-specific bbmasks are applied on the product using the layers. 1301 | self.assertShellCode( 1302 | """\ 1303 | . init-build-env --product=using-collection1 --product=using-collection2 1304 | """, 1305 | success=True, 1306 | ) 1307 | 1308 | self.assertNotInBbconf( 1309 | 'BBMASK_using-collection1 += "{PROJECT_ROOT}/layers/meta-layer2/recipes-bad/bad.bb"\n'.format( 1310 | PROJECT_ROOT=self.project_root 1311 | ) 1312 | ) 1313 | 1314 | 1315 | if __name__ == "__main__": 1316 | unittest.main() 1317 | -------------------------------------------------------------------------------- /images/whisk.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | 7 | 10 | 11 | 12 | 13 | 14 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | -------------------------------------------------------------------------------- /init-build-env: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | # 3 | # 2020 Garmin Ltd. or its subsidiaries 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | if [ -n "$BASH_SOURCE" ]; then 18 | THIS_SCRIPT=$BASH_SOURCE 19 | elif [ -n "$ZSH_NAME" ]; then 20 | THIS_SCRIPT=$0 21 | else 22 | THIS_SCRIPT="$(pwd)/init-build-env" 23 | fi 24 | 25 | if [ -z "$ZSH_NAME" ] && [ "$0" = "$THIS_SCRIPT" ]; then 26 | echo "Error: This script needs to be sourced. Please run as '. $THIS_SCRIPT'" 27 | exit 1 28 | fi 29 | 30 | if [ -z "$WHISK_ROOT" ]; then 31 | WHISK_ROOT=$(dirname $(readlink -f $THIS_SCRIPT)) 32 | if [ -z "$WHISK_ROOT" ]; then 33 | WHISK_ROOT="$(pwd)" 34 | fi 35 | fi 36 | 37 | if [ -z "$WHISKCONF" ]; then 38 | WHISKCONF="$(readlink -f $(dirname $THIS_SCRIPT))/whisk.yaml" 39 | fi 40 | 41 | whisk_cleanup() { 42 | unset whisk_cleanup WHISK_EXTRA_CONF 43 | } 44 | 45 | configure() { 46 | local TEMP_ENV_FILE=$(mktemp -t whisk-env.XXXXXX) 47 | 48 | $WHISK_ROOT/bin/whisk configure \ 49 | --env $TEMP_ENV_FILE \ 50 | --conf $WHISKCONF \ 51 | $WHISK_EXTRA_CONF \ 52 | -- "$@" 53 | local ERR=$? 54 | 55 | if [ "$WHISK_CAPTURE_ENV" = "-" ]; then 56 | cat $TEMP_ENV_FILE 57 | elif [ -n "$WHISK_CAPTURE_ENV" ]; then 58 | cat $TEMP_ENV_FILE > "$WHISK_CAPTURE_ENV" 59 | fi 60 | 61 | if [ $ERR != 0 ]; then 62 | rm $TEMP_ENV_FILE 63 | return $ERR 64 | fi 65 | 66 | . $TEMP_ENV_FILE 67 | rm $TEMP_ENV_FILE 68 | return 0 69 | } 70 | 71 | WHISK_EXTRA_CONF="--init" configure "$@" 72 | if [ $? -ne 0 ]; then 73 | whisk_cleanup 74 | return 1 75 | fi 76 | 77 | whisk_cleanup 78 | # vim: noexpandtab 79 | -------------------------------------------------------------------------------- /meta-whisk/classes/build-alias.bbclass: -------------------------------------------------------------------------------- 1 | # 2020 Garmin Ltd. or its subsidiaries 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | # 16 | # This class lets you define a "build alias"; that is a command that doesn't 17 | # build anything itself, but depends on a set of other targets and causes them 18 | # to build 19 | 20 | LICENSE = "MIT" 21 | 22 | TARGETS ?= "" 23 | 24 | INHIBIT_DEFAULT_DEPS = "1" 25 | 26 | PACKAGE_ARCH = "all" 27 | 28 | DEPCHAIN_DBGDEFAULTDEPS = "1" 29 | 30 | inherit nopackages allarch 31 | 32 | deltask do_fetch 33 | deltask do_unpack 34 | deltask do_patch 35 | deltask do_configure 36 | deltask do_compile 37 | deltask do_install 38 | deltask do_populate_sysroot 39 | deltask do_populate_lic 40 | 41 | python() { 42 | depends = [] 43 | mcdepends = [] 44 | 45 | for t in (d.getVar('TARGETS') or '').split(): 46 | if t.startswith('mc:'): 47 | _, mc, recipe = t.split(':') 48 | mcdepends.append("mc::%s:%s:do_build" % (mc, recipe)) 49 | else: 50 | depends.append("%s:do_build" % t) 51 | 52 | # uniquify and sort 53 | depends = sorted(list(set(depends))) 54 | mcdepends = sorted(list(set(mcdepends))) 55 | 56 | d.setVarFlags("do_build", { 57 | "mcdepends": " ".join(mcdepends), 58 | "depends": " ".join(depends), 59 | "noexec": "1", 60 | "nostamp": "1", 61 | }) 62 | } 63 | 64 | -------------------------------------------------------------------------------- /meta-whisk/classes/deploy-alias.bbclass: -------------------------------------------------------------------------------- 1 | # 2020 Garmin Ltd. or its subsidiaries 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | # 16 | # This class lets you define a "deploy alias"; that is a command that doesn't 17 | # build anything itself, but cross-links a deliverable from one multiconfig 18 | # deploy directory to another 19 | # 20 | # The class is controlled by the DEPLOY_ALIASES variable. It specifies 21 | # a space separated list of what deployed artifact aliases should be copied. 22 | # Each item is in the form: 23 | # 24 | # mc:MULTICONFIG:RECIPE:TASK:PATH 25 | # 26 | # Where: 27 | # 28 | # MULTICONFIG - The name of the source multiconfig 29 | # RECIPE - The source recipe that produces the artifact 30 | # TASK - The source task that produces the artifact (e.g. do_deploy, 31 | # do_image_complete, etc.) 32 | # PATH - The path to the artifact that should be copied 33 | 34 | 35 | LICENSE = "MIT" 36 | 37 | DEPLOY_ALIASES ?= "" 38 | 39 | INHIBIT_DEFAULT_DEPS = "1" 40 | 41 | PACKAGE_ARCH = "all" 42 | 43 | DEPCHAIN_DBGDEFAULTDEPS = "1" 44 | 45 | inherit nopackages allarch deploy 46 | 47 | deltask do_fetch 48 | deltask do_unpack 49 | deltask do_patch 50 | deltask do_configure 51 | deltask do_compile 52 | deltask do_install 53 | deltask do_populate_sysroot 54 | deltask do_populate_lic 55 | 56 | def get_mcdepends(d): 57 | aliases = d.getVar('DEPLOY_ALIASES') 58 | 59 | mcdepends = [] 60 | for a in aliases.split(): 61 | if not a.startswith('mc:'): 62 | continue 63 | 64 | (_, mc, pn, task, _) = a.split(':') 65 | mcdepends.append("mc::%s:%s:%s" % (mc, pn, task)) 66 | 67 | return ' '.join(sorted(mcdepends)) 68 | 69 | python do_deploy() { 70 | aliases = d.getVar('DEPLOY_ALIASES') 71 | for a in aliases.split(): 72 | (_, _, _, _, src) = a.split(':') 73 | 74 | filename = os.path.basename(src) 75 | dst = "%s/%s" % (d.getVar('DEPLOYDIR'), filename) 76 | 77 | if os.path.islink(src): 78 | real_src = os.path.realpath(src) 79 | real_filename = os.path.basename(real_src) 80 | real_dst = "%s/%s" % (d.getVar('DEPLOYDIR'), real_filename) 81 | 82 | oe.path.copyhardlink(real_src, real_dst) 83 | os.symlink(real_filename, dst) 84 | else: 85 | oe.path.copyhardlink(src, dst) 86 | } 87 | do_deploy[mcdepends] = "${@get_mcdepends(d)}" 88 | do_deploy[cleandirs] = "${DEPLOYDIR}" 89 | addtask do_deploy after do_compile before do_build 90 | -------------------------------------------------------------------------------- /meta-whisk/classes/product-build-alias.bbclass: -------------------------------------------------------------------------------- 1 | # 2020 Garmin Ltd. or its subsidiaries 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | # Just like build-alias, but skips the recipe in the event that the 16 | # product is not configured 17 | 18 | inherit build-alias 19 | 20 | PRODUCT ?= "" 21 | 22 | python() { 23 | product = d.getVar('PRODUCT') 24 | if not product: 25 | raise bb.parse.SkipRecipe('PRODUCT not defined') 26 | 27 | if not product in (d.getVar('WHISK_PRODUCTS') or '').split(): 28 | raise bb.parse.SkipRecipe('Product %s not configured' % product) 29 | } 30 | -------------------------------------------------------------------------------- /meta-whisk/conf/layer.conf: -------------------------------------------------------------------------------- 1 | # We have a conf directory, add to BBPATH 2 | BBPATH .= ":${LAYERDIR}" 3 | 4 | # We have recipes, add them to BBFILES 5 | BBFILES += "\ 6 | ${LAYERDIR}/recipes-*/*/*.bb \ 7 | ${LAYERDIR}/recipes-*/*/*.bbappend \ 8 | " 9 | 10 | BBFILE_COLLECTIONS += "whisk" 11 | BBFILE_PATTERN_whisk = "^${LAYERDIR}/" 12 | BBFILE_PRIORITY_whisk = "6" 13 | 14 | LAYERSERIES_COMPAT_whisk = "\ 15 | thud \ 16 | zeus \ 17 | dunfell \ 18 | gatesgarth \ 19 | hardknott \ 20 | kirkstone \ 21 | mickledore \ 22 | langdale \ 23 | nanbield \ 24 | scarthgap \ 25 | " 26 | -------------------------------------------------------------------------------- /meta-whisk/recipes-core/targets/all-targets.bb: -------------------------------------------------------------------------------- 1 | SUMMARY = "Main build target" 2 | 3 | inherit build-alias 4 | 5 | TARGETS = "${WHISK_TARGETS_core}" 6 | 7 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | PyYAML==6.0; python_version < '3.8' 2 | PyYAML==6.0.2; python_version >= '3.8' 3 | jsonschema==3.2.0; python_version < '3.7' 4 | jsonschema==4.17.3; python_version == '3.7' 5 | jsonschema==4.23.0; python_version >= '3.8' 6 | tabulate==0.8.10; python_version < '3.7' 7 | tabulate==0.9.0; python_version >= '3.7' 8 | tqdm==4.64.1; python_version < '3.7' 9 | tqdm==4.66.5; python_version >= '3.7' 10 | yamllint==1.28.0; python_version < '3.7' 11 | yamllint==1.32.0; python_version == '3.7' 12 | yamllint==1.35.1; python_version >= '3.8' 13 | -------------------------------------------------------------------------------- /setup-venv: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | # 3 | # 2020 Garmin Ltd. or its subsidiaries 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -eu 18 | 19 | THIS_DIR="$(dirname $0)" 20 | 21 | cd $THIS_DIR 22 | 23 | REV="$(git rev-parse HEAD):$(shasum "requirements.txt")" 24 | 25 | if [ ! -e "venv/rev.txt" ] || [ "$(cat venv/rev.txt)" != "$REV" ]; then 26 | python3 -m venv "venv" 27 | . "venv/bin/activate" 28 | pip install -U pip 29 | pip install -r requirements.txt 30 | echo "$REV" > "venv/rev.txt" 31 | fi 32 | 33 | -------------------------------------------------------------------------------- /whisk.example.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # This file describes how your project is configured. Any variables of the 3 | # %NAME or %{NAME} will be expanded with the environment variable of the same 4 | # name. In addition, the following variables will also be expanded: 5 | # 6 | # %WHISK_PROJECT_ROOT - The absolute path of the project root 7 | # 8 | # Whisk will fail if any undefined variables are encountered 9 | # 10 | # The version of this config file. Must be 1 or 2. 11 | # Version changes: 12 | # 2: No differences in the config file format. If this version is specified, 13 | # whisk will not define DEPLOY_DIR_ variable aliases, since they 14 | # can cause problems if you product name happens to be an override 15 | version: 2 16 | 17 | # The location of the project root, relative to the parent directory of this 18 | # configuration file. This value is evaluated before any variable expansion is 19 | # done. Defaults to ".", meaning that this config file is in the root of the 20 | # project 21 | project_root: . 22 | 23 | # The location where the user's locally configured cache will be stored 24 | cache: "%{WHISK_PROJECT_ROOT}/.config.yaml" 25 | 26 | defaults: 27 | # The default values if the user has never configured before. If any are 28 | # omitted, the user *must* specify them the first time they configure. Note 29 | # that the default version is not configurable because it defaults to 30 | # "default". 31 | products: [] 32 | mode: release 33 | site: roaming 34 | # Default build directory, or "build" if unspecified 35 | build_dir: build 36 | 37 | # Shell code fragments that will be inserted into the shell code executed by 38 | # the initialization script. All are optional. Whisk will set the following 39 | # extra shell variables which may be used in these fragments: 40 | # 41 | # $WHISK_BUILD_DIR - The absolute path to the user specified build directory 42 | # $WHISK_INIT - This variable will have the value "true" if this is the 43 | # first time the environment is being initialized, and 44 | # "false" if the user is re-configuring 45 | hooks: 46 | # Shell code executed before the version initialization script 47 | pre_init: | 48 | # placeholder 49 | 50 | # Shell code executed after the version initialization script 51 | post_init: | 52 | # placeholder 53 | 54 | # A list of environment variables to pass through to bitbake (e.g. 55 | # BB_ENV_EXTRAWHITE/BB_ENV_PASSTHROUGH_ADDITIONS) 56 | env_passthrough_vars: [] 57 | 58 | # Commands to run if the user specifies --fetch to fetch layers. 59 | # 60 | # Fetch commands are always run with the current working directory set to the 61 | # project root. Multiple commands may be specified, and each will execute in 62 | # turn. If any command fails, fetching will stop and the configuration will 63 | # fail 64 | fetch: 65 | commands: 66 | - fetch one 67 | - fetch two 68 | 69 | # The version of Yocto that are supported 70 | versions: 71 | # Each version is named here. Any name can be chosen, but the release name 72 | # (e.g. zeus, dunfell, gatesgarth), is recommended 73 | dunfell: 74 | # a short description of the version (optional) 75 | description: Yocto 3.1 76 | 77 | # What Yocto release this version is compatible with, as the codename of 78 | # the release (e.g. "zeus", "dunfell", "gatesgarth", etc.). If set to 79 | # "auto" (the default if unspecified), whisk will scan all layers in this 80 | # version to try and automatically figure out an appropriate version 81 | compat: auto 82 | 83 | # The path to the OE initialization script for this version 84 | oeinit: "%{WHISK_PROJECT_ROOT}/layers/dunfell/poky/oe-init-build-env" 85 | 86 | # Commands to run if the user specifies --fetch to fetch layers and this 87 | # version is active. 88 | fetch: 89 | commands: 90 | - fetch one 91 | - fetch two 92 | 93 | # The pyrex configuration for this version. If omitted, pyrex will not be 94 | # used 95 | pyrex: 96 | # The path to the pyrex root for this version. In general, you can share 97 | # the same version of pyrex across multiple different versions and change 98 | # the config file to pull in the pyrex image that matches one of the 99 | # SANITY_TESTED_DISTROS 100 | root: "%{WHISK_PROJECT_ROOT}/layers/meta-pyrex" 101 | 102 | # The path to the pyrex configuration file to use for this version 103 | conf: "%{WHISK_PROJECT_ROOT}/layers/dunfell/pyrex.ini" 104 | 105 | # A dictionary where additional annotations may be placed. Whisk 106 | # ignores anything in this field (optional) 107 | tags: {} 108 | 109 | # The layer collections supported by this version. The order is preserved, 110 | # so layers will added to BBLAYERS in the same order as listed here. 111 | layers: 112 | # The name of the layer collection, as referenced by a product. You 113 | # should keep the names of similar layer collections the same across 114 | # multiple different Yocto versions, as it makes it easier to migrate a 115 | # product to a newer version 116 | - name: oe-core 117 | # The list of layers in this collection. Note that a layer collection 118 | # is allowed to have multiple layers 119 | paths: 120 | - "%{WHISK_PROJECT_ROOT}/layers/dunfell/poky/meta" 121 | 122 | # Commands to run if the user specifies --fetch to fetch layers and 123 | # this layer is required by a configured product. 124 | fetch: 125 | commands: 126 | - git submodule update --init layers/dunfell/poky 127 | 128 | # A list of individual strings which should be inserted into BBMASK 129 | # for any product which puts this layer collection's name into its 130 | # layers list. 131 | # 132 | # Customarily, each element in this list would be expected to be 133 | # phrased starting from the project root. 134 | bbmask: 135 | - "%{WHISK_PROJECT_ROOT}/layers/dunfell/poky/meta-poky/recipes-core/tiny-init" 136 | 137 | # A dictionary where additional annotations may be placed. Whisk 138 | # ignores anything in this field (optional) 139 | tags: {} 140 | 141 | - name: mingw 142 | paths: 143 | - "%{WHISK_PROJECT_ROOT}/layers/dunfell/meta-mingw" 144 | fetch: 145 | commands: 146 | - git submodule update --init layers/dunfell/meta-mingw 147 | 148 | zeus: 149 | description: Yocto 3.0 150 | 151 | oeinit: "%{WHISK_PROJECT_ROOT}/layers/zeus/poky/oe-init-build-env" 152 | 153 | pyrex: 154 | root: "%{WHISK_PROJECT_ROOT}/layers/meta-pyrex" 155 | conf: "%{WHISK_PROJECT_ROOT}/layers/dunfell/pyrex.ini" 156 | 157 | layers: 158 | - name: oe-core 159 | paths: 160 | - "%{WHISK_PROJECT_ROOT}/layers/zeus/poky/meta" 161 | 162 | # The build modes that are supported 163 | modes: 164 | internal: 165 | # A short description of the build mode (optional) 166 | description: Engineering build for internal development 167 | 168 | # A fragment that will be directly written into 'site.conf' when this mode 169 | # is used (optional) 170 | conf: | 171 | # Mode placeholder 172 | 173 | # A dictionary where additional annotations may be placed. Whisk 174 | # ignores anything in this field (optional) 175 | tags: {} 176 | 177 | release: 178 | description: Release build for public release 179 | conf: | 180 | # Mode placeholder 181 | 182 | sites: 183 | roaming: 184 | # A short description of the build site (optional) 185 | description: Roaming outside an office 186 | 187 | # A fragment that will be directly written into 'site.conf' when this site 188 | # is used (optional) 189 | conf: | 190 | # Site placeholder 191 | 192 | # The core configuration. This defines how the base bitbake configuration (e.g. 193 | # outside of any multiconfigs) behaves 194 | core: 195 | # The list of layer collections that should be included in the base 196 | # configuration. Note that all versions *must* provide a layer collection 197 | # with the name of each collection listed here, or else it would be 198 | # impossible to use the base configuration (and thus any configuration). 199 | layers: 200 | - oe-core 201 | 202 | # A fragment that will be directly written into 'bblayers.conf' (optional) 203 | layerconf: | 204 | # placeholder 205 | 206 | # A fragment that will be directly written into 'site.conf' (optional) 207 | conf: | 208 | MACHINE ?= "qemux86" 209 | DISTRO ?= "poky" 210 | 211 | 212 | # Defines products that can be built 213 | products: 214 | qemux86-64: 215 | # A short description of the product. This is what shows up when user asks 216 | # for help, so it should try and help the distinguish products, 217 | # particularly if you use a lot of codenames (optional) 218 | description: A test qemux86-64 product 219 | 220 | # The list of maintainers for the product. The name of the maintainer is 221 | # required, but the email is optional. 222 | maintainers: 223 | - name: John Doe 224 | email: john.doe@company.com 225 | 226 | # The default version that this product will build with if unspecified, or 227 | # if the user specifies the default should be used. The user can force a 228 | # product to use a different version if they are feeling adventurous 229 | default_version: dunfell 230 | 231 | # The list of layer collections that should be used for this this product 232 | layers: 233 | - oe-core 234 | 235 | # Indicates whether the generated Yocto configuration should use a 236 | # traditional layout based only on local.conf/site.conf, or a configuration 237 | # based on multiconfig. Default value is true. 238 | multiconfig_enabled: true 239 | 240 | # The list of additional multiconfigs that should be enabled when this 241 | # product is configured. The product multiconfig (i.e. "product-$NAME") is 242 | # always enabled when the product is included, but you may specify 243 | # additional ones to activate when the product is configured (e.g. if the 244 | # product has firmware built with a different multiconfig). (optional) 245 | # 246 | # This list must not have any elements if multiconfig_enabled is set to 247 | # false. 248 | multiconfigs: [] 249 | 250 | # The list of default build targets that should be built when this product 251 | # is configured. Note that you are not restricted to building a target from 252 | # any specific multiconfig, so you must use the "mc:..." bitbake syntax to 253 | # build from a specific multiconfig. In particular, you will probably want 254 | # most of your entries to to be prefixed with "mc:product-$NAME" to make 255 | # them build in this products multiconfig (optional) 256 | targets: 257 | - "mc:product-qemux86-64:core-image-minimal" 258 | 259 | # A fragment that will be directly written into the product multiconfig 260 | # (optional) 261 | conf: | 262 | MACHINE = "qemux86-64" 263 | DISTRO = "poky" 264 | 265 | # A dictionary where additional annotations may be placed. Whisk 266 | # ignores anything in this field (optional) 267 | tags: {} 268 | 269 | qemuarm: 270 | description: A test qemuarm product 271 | default_version: dunfell 272 | layers: 273 | - oe-core 274 | 275 | targets: 276 | - mc:product-qemuarm:core-image-minimal 277 | 278 | conf: | 279 | MACHINE = "qemuarm" 280 | DISTRO = "poky" 281 | 282 | qemuarm-without-multiconfig: 283 | description: A test qemuarm product, built without using multiconfig 284 | default_version: dunfell 285 | layers: 286 | - oe-core 287 | 288 | multiconfig_enabled: false 289 | 290 | targets: 291 | - core-image-minimal 292 | 293 | conf: | 294 | MACHINE = "qemuarm" 295 | DISTRO = "poky" 296 | -------------------------------------------------------------------------------- /whisk.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | # 3 | # 2020 Garmin Ltd. or its subsidiaries 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import argparse 18 | import itertools 19 | import json 20 | import jsonschema 21 | import os 22 | import pathlib 23 | import string 24 | import subprocess 25 | import sys 26 | import tabulate 27 | import textwrap 28 | import tqdm 29 | import yaml 30 | import re 31 | from enum import Enum 32 | from collections import namedtuple 33 | 34 | tabulate.PRESERVE_WHITESPACE = True 35 | 36 | THIS_DIR = pathlib.Path(__file__).parent.absolute() 37 | SCHEMA_FILE = THIS_DIR / "whisk.schema.json" 38 | 39 | CACHE_VERSION = 1 40 | 41 | 42 | class OEVersion(object): 43 | def __init__( 44 | self, 45 | conf_version, 46 | *, 47 | append_sep=":", 48 | basehash_ignore_vars="BB_BASEHASH_IGNORE_VARS", 49 | env_passthrough_var="BB_ENV_PASSTHROUGH_ADDITIONS", 50 | ): 51 | self.conf_version = conf_version 52 | self.append_sep = append_sep 53 | self.basehash_ignore_vars = basehash_ignore_vars 54 | self.env_passthrough_var = env_passthrough_var 55 | 56 | 57 | VERSION_COMPAT = { 58 | "morty": OEVersion( 59 | 1, 60 | append_sep="_", 61 | basehash_ignore_vars="BB_HASHBASE_WHITELIST", 62 | env_passthrough_var="BB_ENV_EXTRAWHITE", 63 | ), 64 | "pyro": OEVersion( 65 | 1, 66 | append_sep="_", 67 | basehash_ignore_vars="BB_HASHBASE_WHITELIST", 68 | env_passthrough_var="BB_ENV_EXTRAWHITE", 69 | ), 70 | "rocko": OEVersion( 71 | 1, 72 | append_sep="_", 73 | basehash_ignore_vars="BB_HASHBASE_WHITELIST", 74 | env_passthrough_var="BB_ENV_EXTRAWHITE", 75 | ), 76 | "sumo": OEVersion( 77 | 1, 78 | append_sep="_", 79 | basehash_ignore_vars="BB_HASHBASE_WHITELIST", 80 | env_passthrough_var="BB_ENV_EXTRAWHITE", 81 | ), 82 | "thud": OEVersion( 83 | 1, 84 | append_sep="_", 85 | basehash_ignore_vars="BB_HASHBASE_WHITELIST", 86 | env_passthrough_var="BB_ENV_EXTRAWHITE", 87 | ), 88 | "warrior": OEVersion( 89 | 1, 90 | append_sep="_", 91 | basehash_ignore_vars="BB_HASHBASE_WHITELIST", 92 | env_passthrough_var="BB_ENV_EXTRAWHITE", 93 | ), 94 | "zeus": OEVersion( 95 | 1, 96 | append_sep="_", 97 | basehash_ignore_vars="BB_HASHBASE_WHITELIST", 98 | env_passthrough_var="BB_ENV_EXTRAWHITE", 99 | ), 100 | "dunfell": OEVersion( 101 | 1, 102 | append_sep="_", 103 | basehash_ignore_vars="BB_HASHBASE_WHITELIST", 104 | env_passthrough_var="BB_ENV_EXTRAWHITE", 105 | ), 106 | "gatesgarth": OEVersion( 107 | 1, 108 | append_sep="_", 109 | basehash_ignore_vars="BB_HASHBASE_WHITELIST", 110 | env_passthrough_var="BB_ENV_EXTRAWHITE", 111 | ), 112 | "hardknott": OEVersion( 113 | 1, 114 | append_sep="_", 115 | basehash_ignore_vars="BB_HASHBASE_WHITELIST", 116 | env_passthrough_var="BB_ENV_EXTRAWHITE", 117 | ), 118 | "honister": OEVersion( 119 | 1, 120 | basehash_ignore_vars="BB_HASHBASE_WHITELIST", 121 | env_passthrough_var="BB_ENV_EXTRAWHITE", 122 | ), 123 | "kirkstone": OEVersion(2), 124 | "langdale": OEVersion(2), 125 | } 126 | 127 | LATEST_COMPAT = OEVersion(2) 128 | 129 | 130 | class ConfTemplate(string.Template): 131 | delimiter = r"%" 132 | 133 | 134 | def get_layerseries_corenames(version): 135 | for layer in version.get("layers", []): 136 | for layer_path in [pathlib.Path(p) for p in layer.get("paths", [])]: 137 | conf_file = layer_path / "conf" / "layer.conf" 138 | if conf_file.exists(): 139 | with conf_file.open("r") as f: 140 | for line in f: 141 | m = re.match(r'^LAYERSERIES_CORENAMES *= *"(.*)"', line) 142 | if m is not None: 143 | return m.group(1).split() 144 | 145 | return [] 146 | 147 | 148 | def print_items(items, is_current, extra=[]): 149 | def get_current(i): 150 | if is_current(i): 151 | return " *" 152 | return " " 153 | 154 | print( 155 | tabulate.tabulate( 156 | [ 157 | ( 158 | get_current(i), 159 | i, 160 | items[i].get("description", ""), 161 | ) 162 | for i in sorted(items) 163 | ] 164 | + [(get_current(e), e, "") for e in extra], 165 | tablefmt="plain", 166 | ) 167 | ) 168 | 169 | 170 | def print_modes(conf, cur_mode): 171 | print_items(conf["modes"], lambda m: m == cur_mode) 172 | 173 | 174 | def print_sites(conf, cur_site): 175 | print_items(conf["sites"], lambda s: s == cur_site) 176 | 177 | 178 | def print_products(conf, cur_products): 179 | print_items(conf["products"], lambda p: p in cur_products) 180 | 181 | 182 | def print_versions(conf, cur_version): 183 | print_items(conf["versions"], lambda v: v == cur_version, extra=["default"]) 184 | 185 | 186 | def write_hook(f, conf, hook): 187 | f.write(conf.get("hooks", {}).get(hook, "")) 188 | f.write("\n") 189 | 190 | 191 | def parse_conf_file(path): 192 | with path.open("r") as f: 193 | conf_str = f.read() 194 | 195 | conf = yaml.load(conf_str, Loader=yaml.Loader) 196 | 197 | if not "version" in conf: 198 | print("Config file '%s' missing version" % sys_args.conf) 199 | return (None, None) 200 | 201 | if conf["version"] < 1 or conf["version"] > 2: 202 | print("Bad version %r in config file '%s'" % (conf["version"], path)) 203 | return (None, None) 204 | 205 | project_root = path.parent / conf.get("project_root", ".") 206 | 207 | # Re-parse, expanding variables 208 | env = os.environ.copy() 209 | env["WHISK_PROJECT_ROOT"] = project_root.absolute() 210 | conf = yaml.load(conf_str, Loader=yaml.Loader) 211 | 212 | # Recursively do environment substitution on all strings 213 | def substitute(item): 214 | if isinstance(item, dict): 215 | for k in item: 216 | item[k] = substitute(item[k]) 217 | return item 218 | elif isinstance(item, list): 219 | for i in range(0, len(item)): 220 | item[i] = substitute(item[i]) 221 | return item 222 | elif isinstance(item, str): 223 | # Throws a KeyError (handled by the caller) if the string contains 224 | # an attempted dereference to an undefined environment variable. 225 | return ConfTemplate(item).substitute(**env) 226 | else: 227 | return item 228 | 229 | try: 230 | conf = substitute(conf) 231 | except KeyError as e: 232 | print("Couldn't substitute all environment variables (%s is undefined)." % e) 233 | return (None, None) 234 | 235 | try: 236 | with SCHEMA_FILE.open("r") as f: 237 | jsonschema.validate(conf, json.load(f)) 238 | except jsonschema.ValidationError as e: 239 | print("Error validating %s: %s" % (path, e.message)) 240 | return (None, None) 241 | 242 | return (conf, project_root) 243 | 244 | 245 | def configure(sys_args): 246 | parser = argparse.ArgumentParser(description="Configure build") 247 | parser.add_argument( 248 | "--products", action="append", default=[], help="Change build product(s)" 249 | ) 250 | parser.add_argument("--mode", help="Change build mode") 251 | parser.add_argument("--site", help="Change build site") 252 | parser.add_argument("--version", help="Set Yocto version") 253 | parser.add_argument("--build-dir", help="Set build directory") 254 | parser.add_argument("--list", action="store_true", help="List options") 255 | parser.add_argument( 256 | "--write", 257 | action="store_true", 258 | help="Write out new config files (useful if product configuration has changed)", 259 | ) 260 | parser.add_argument( 261 | "--no-config", 262 | "-n", 263 | action="store_true", 264 | help="Ignore cached user configuration", 265 | ) 266 | parser.add_argument( 267 | "--quiet", "-q", action="store_true", help="Suppress non-error output" 268 | ) 269 | parser.add_argument("--fetch", action="store_true", help="Fetch required layers") 270 | parser.add_argument("--no-pyrex", action="store_true", help="Do not use Pyrex") 271 | 272 | user_args = parser.parse_args(sys_args.user_args) 273 | 274 | (conf, project_root) = parse_conf_file(sys_args.conf) 275 | if not conf: 276 | return 1 277 | 278 | def get_product(name): 279 | nonlocal conf 280 | if name == "core": 281 | return conf.get("core", {}) 282 | return conf["products"][name] 283 | 284 | cache_path = pathlib.Path(conf.get("cache", project_root / ".config.yaml")) 285 | cache = {} 286 | if not user_args.no_config: 287 | try: 288 | with cache_path.open("r") as f: 289 | cache = yaml.load(f, Loader=yaml.Loader) 290 | except OSError: 291 | pass 292 | 293 | try: 294 | if cache.get("cache_version") != CACHE_VERSION: 295 | cache = {} 296 | except AttributeError: 297 | cache = {} 298 | 299 | defaults = conf.get("defaults", {}) 300 | 301 | cur_mode = cache.get("mode", defaults.get("mode")) 302 | cur_products = cache.get("products", defaults.get("products", [])) 303 | cur_site = cache.get("site", defaults.get("site")) 304 | cur_version = cache.get("version", "default") 305 | cur_actual_version = cache.get("actual_version", "") 306 | build_dir = pathlib.Path(cache.get("build_dir", defaults.get("build_dir", "build"))) 307 | 308 | write = user_args.write or sys_args.init 309 | 310 | if user_args.list: 311 | print("Possible products:") 312 | print_products(conf, cur_products) 313 | print("Possible modes:") 314 | print_modes(conf, cur_mode) 315 | print("Possible sites:") 316 | print_sites(conf, cur_site) 317 | print("Possible versions:") 318 | print_versions(conf, cur_version) 319 | return 0 320 | 321 | if user_args.products: 322 | write = True 323 | user_products = sorted( 324 | set(itertools.chain(*(a.split() for a in user_args.products))) 325 | ) 326 | for p in user_products: 327 | if not p in conf.get("products", {}): 328 | print("Unknown product '%s'. Please choose from:" % p) 329 | print_products(conf, cur_products) 330 | return 1 331 | cur_products = user_products 332 | 333 | if user_args.mode: 334 | write = True 335 | if user_args.mode not in conf["modes"]: 336 | print("Unknown mode '%s'. Please choose from:" % user_args.mode) 337 | print_modes(conf, cur_mode) 338 | return 1 339 | cur_mode = user_args.mode 340 | 341 | if user_args.site: 342 | write = True 343 | if user_args.site not in conf["sites"]: 344 | print("Unknown site '%s'. Please choose from:" % user_args.site) 345 | print_sites(conf, cur_site) 346 | return 1 347 | cur_site = user_args.site 348 | 349 | if user_args.version: 350 | write = True 351 | if sys_args.init: 352 | if ( 353 | user_args.version != "default" 354 | and user_args.version not in conf["versions"] 355 | ): 356 | print("Unknown version '%s'. Please choose from:" % user_args.version) 357 | print_versions(conf, cur_version) 358 | return 1 359 | 360 | cur_version = user_args.version 361 | elif user_args.version != cur_version: 362 | print( 363 | "The version cannot be changed after the environment is initialized. Please initialize a new environment with '--version=%s'" 364 | % user_args.version 365 | ) 366 | return 1 367 | 368 | if user_args.build_dir: 369 | if not sys_args.init: 370 | print( 371 | "Build directory cannot be changed after environment is initialized. Please initialize a new environment with '--build-dir=%s'" 372 | % user_args.build_dir 373 | ) 374 | return 1 375 | build_dir = pathlib.Path(user_args.build_dir) 376 | 377 | if not cur_products: 378 | print("One or more products must be specified with --product") 379 | return 1 380 | 381 | if not cur_mode: 382 | print("A build mode must be specified with --mode") 383 | return 1 384 | 385 | if not cur_site: 386 | print("A site must be specified with --site") 387 | return 1 388 | 389 | # Set the actual version 390 | if cur_version == "default": 391 | product_versions = {} 392 | 393 | for p in cur_products: 394 | v = conf["products"][p]["default_version"] 395 | product_versions.setdefault(v, []).append(p) 396 | 397 | keys = list(product_versions) 398 | if len(keys) == 1: 399 | if sys_args.init or keys[0] == cur_actual_version: 400 | # Environment hasn't been initialized or it's not changing, so 401 | # it can be set 402 | cur_actual_version = keys[0] 403 | else: 404 | print( 405 | "Build environment is configured to build version '{actual}' and cannot be changed to version '{v}' required to build products: {products}. Please initialize a new environment with `--product='{products}' --version=default`".format( 406 | actual=cur_actual_version, 407 | v=keys[0], 408 | products=" ".join(product_versions[keys[0]]), 409 | ) 410 | ) 411 | return 1 412 | else: 413 | print( 414 | "Multiple products with different default versions were chosen. They are:" 415 | ) 416 | print( 417 | tabulate.tabulate( 418 | [(k, " ".join(v)) for k, v in product_versions.items()], 419 | tablefmt="plain", 420 | ) 421 | ) 422 | return 1 423 | 424 | else: 425 | cur_actual_version = cur_version 426 | 427 | version = conf["versions"][cur_actual_version] 428 | 429 | cur_layers_paths = { 430 | l["name"]: l.get("paths", []) for l in version.get("layers", []) 431 | } 432 | cur_layers_bbmasks = { 433 | l["name"]: l.get("bbmask", []) for l in version.get("layers", []) 434 | } 435 | 436 | using_multiconfig = True 437 | 438 | # Sanity check that if any selected product does not support multiconfig, 439 | # it is the only selected product. 440 | for p in cur_products: 441 | if not conf["products"][p].get("multiconfig_enabled", True): 442 | using_multiconfig = False 443 | 444 | if len(cur_products) > 1: 445 | print( 446 | "Product '{product}' does not support multiconfig, but more than one product is chosen.".format( 447 | product=p 448 | ) 449 | ) 450 | return 1 451 | 452 | multiconfigs = conf["products"][p].get("multiconfigs", []) 453 | if len(multiconfigs) > 0: 454 | print( 455 | "Product '{product}' does not support multiconfig, but its 'multiconfig' attribute has some elements: {multiconfigs}.".format( 456 | product=p, multiconfigs=multiconfigs 457 | ) 458 | ) 459 | return 1 460 | 461 | # Sanity check that all configured products have layers 462 | for p in ["core"] + cur_products: 463 | missing = set( 464 | l for l in get_product(p).get("layers", []) if not l in cur_layers_paths 465 | ) 466 | if missing: 467 | print( 468 | "Product '{product}' requires layer collection(s) '{layers}' which is not present in version '{version}'".format( 469 | product=p, layers=" ".join(missing), version=cur_actual_version 470 | ) 471 | ) 472 | return 1 473 | 474 | requested_layers = set() 475 | for name in ["core"] + cur_products: 476 | requested_layers.update(get_product(name).get("layers", [])) 477 | 478 | if user_args.fetch: 479 | fetch_commands = [] 480 | for o in [conf, version] + [ 481 | l for l in version.get("layers", []) if l["name"] in requested_layers 482 | ]: 483 | fetch_commands.extend(o.get("fetch", {}).get("commands", [])) 484 | 485 | env = os.environ.copy() 486 | env["WHISK_PROJECT_ROOT"] = project_root.absolute() 487 | 488 | for c in tqdm.tqdm( 489 | fetch_commands, 490 | desc="Fetching", 491 | bar_format="{l_bar}{bar}| {n_fmt}/{total_fmt} [{elapsed}<{remaining}]", 492 | disable=user_args.quiet, 493 | ): 494 | r = subprocess.run( 495 | c, 496 | shell=True, 497 | cwd=project_root, 498 | stdout=subprocess.PIPE, 499 | stderr=subprocess.STDOUT, 500 | env=env, 501 | ) 502 | if r.returncode: 503 | print("Fetch command '%s' failed:\n%s" % (c, r.stdout)) 504 | return 1 505 | 506 | compat_name = version.get("compat", "auto") 507 | if compat_name == "auto": 508 | corenames = get_layerseries_corenames(version) 509 | if corenames: 510 | for c in corenames: 511 | if c in VERSION_COMPAT: 512 | compat_name = c 513 | break 514 | else: 515 | # Version unknown. Assume the latest (no compat flags) 516 | compat_name = "" 517 | else: 518 | # No LAYERSERIES_CORENAMES. Assume "pyro" since it is was the last 519 | # version to not have one 520 | compat_name = "pyro" 521 | 522 | compat = VERSION_COMPAT.get(compat_name, LATEST_COMPAT) 523 | 524 | with sys_args.env.open("w") as f: 525 | f.write( 526 | textwrap.dedent( 527 | f"""\ 528 | export WHISK_PRODUCTS="{' '.join(cur_products)}" 529 | export WHISK_MODE="{cur_mode}" 530 | export WHISK_SITE="{cur_site}" 531 | export WHISK_VERSION="{cur_version}" 532 | export WHISK_ACTUAL_VERSION="{cur_actual_version}" 533 | 534 | export WHISK_BUILD_DIR={str(build_dir.absolute())} 535 | export WHISK_INIT={'true' if sys_args.init else 'false'} 536 | export WHISK_COMPAT="{compat_name}" 537 | """ 538 | ) 539 | ) 540 | 541 | write_hook(f, conf, "pre_init") 542 | if sys_args.init: 543 | bitbake_dir = version.get("bitbakedir") 544 | if bitbake_dir: 545 | f.write('export BITBAKEDIR="%s"\n' % bitbake_dir) 546 | 547 | f.write( 548 | textwrap.dedent( 549 | """\ 550 | export WHISK_PROJECT_ROOT="{root}" 551 | export {env_passthrough_var}="${{{env_passthrough_var}}} {passthrough_vars} WHISK_PROJECT_ROOT WHISK_PRODUCTS WHISK_MODE WHISK_SITE WHISK_ACTUAL_VERSION" 552 | PATH="{this_dir}/bin:$PATH" 553 | """ 554 | ).format( 555 | passthrough_vars=" ".join( 556 | conf.get("hooks", {}).get("env_passthrough_vars", []) 557 | ), 558 | env_passthrough_var=compat.env_passthrough_var, 559 | root=project_root.absolute(), 560 | this_dir=THIS_DIR, 561 | ) 562 | ) 563 | 564 | if version.get("pyrex") and not user_args.no_pyrex: 565 | f.write( 566 | textwrap.dedent( 567 | """\ 568 | PYREX_CONFIG_BIND="{root}" 569 | PYREX_ROOT="{version[pyrex][root]}" 570 | PYREX_OEINIT="{version[oeinit]}" 571 | PYREXCONFFILE="{version[pyrex][conf]}" 572 | 573 | . {version[pyrex][root]}/pyrex-init-build-env $WHISK_BUILD_DIR 574 | """ 575 | ).format( 576 | root=project_root.absolute(), 577 | version=version, 578 | ) 579 | ) 580 | 581 | else: 582 | f.write( 583 | ". {version[oeinit]} $WHISK_BUILD_DIR\n".format(version=version) 584 | ) 585 | 586 | write_hook(f, conf, "post_init") 587 | 588 | f.write("unset WHISK_BUILD_DIR WHISK_INIT\n") 589 | 590 | if not user_args.no_config: 591 | with cache_path.open("w") as f: 592 | f.write( 593 | yaml.dump( 594 | { 595 | "cache_version": CACHE_VERSION, 596 | "mode": cur_mode, 597 | "products": cur_products, 598 | "site": cur_site, 599 | "version": cur_version, 600 | "actual_version": cur_actual_version, 601 | "build_dir": str(build_dir.absolute()), 602 | }, 603 | Dumper=yaml.Dumper, 604 | ) 605 | ) 606 | 607 | if write: 608 | (build_dir / "conf").mkdir(parents=True, exist_ok=True) 609 | 610 | with (build_dir / "conf" / "site.conf").open("w") as f: 611 | f.write("# This file was dynamically generated by whisk\n") 612 | f.write(f'CONF_VERSION = "{compat.conf_version}"\n\n') 613 | 614 | f.write(conf["sites"][cur_site].get("conf", "")) 615 | f.write("\n") 616 | f.write(conf["modes"][cur_mode].get("conf", "")) 617 | f.write("\n") 618 | 619 | if conf["version"] < 2: 620 | f.write( 621 | textwrap.dedent( 622 | """\ 623 | DEPLOY_DIR_BASE ?= "${TOPDIR}/deploy/${WHISK_MODE}/${WHISK_ACTUAL_VERSION}" 624 | WHISK_DEPLOY_DIR_BASE ?= "${DEPLOY_DIR_BASE}" 625 | 626 | WHISK_DEPLOY_DIR_core = "${WHISK_DEPLOY_DIR_BASE}/core" 627 | DEPLOY_DIR_core = "${WHISK_DEPLOY_DIR_core}" 628 | """ 629 | ) 630 | ) 631 | else: 632 | f.write( 633 | textwrap.dedent( 634 | """\ 635 | WHISK_DEPLOY_DIR_BASE ?= "${TOPDIR}/deploy/${WHISK_MODE}/${WHISK_ACTUAL_VERSION}" 636 | 637 | WHISK_DEPLOY_DIR_core = "${WHISK_DEPLOY_DIR_BASE}/core" 638 | """ 639 | ) 640 | ) 641 | 642 | f.write( 643 | textwrap.dedent( 644 | """\ 645 | BBPATH .= ":${TOPDIR}/whisk" 646 | 647 | WHISK_PRODUCT ?= "core" 648 | 649 | # Set TMPDIR to a version specific location 650 | TMPDIR_BASE ?= "${TOPDIR}/tmp/${WHISK_MODE}/${WHISK_ACTUAL_VERSION}" 651 | 652 | TMPDIR = "${TMPDIR_BASE}/${WHISK_PRODUCT}" 653 | 654 | # Set the deploy directory to output to a well-known location 655 | DEPLOY_DIR = "${WHISK_DEPLOY_DIR_${WHISK_PRODUCT}}" 656 | DEPLOY_DIR_IMAGE = "${DEPLOY_DIR}/images" 657 | """ 658 | ) 659 | ) 660 | f.write( 661 | 'WHISK_TARGETS_core = "%s"\n' 662 | % (" ".join("${WHISK_TARGETS_%s}" % p for p in cur_products)) 663 | ) 664 | 665 | for p in sorted(conf["products"]): 666 | if conf["version"] < 2: 667 | f.write( 668 | 'DEPLOY_DIR_{p} = "${{WHISK_DEPLOY_DIR_{p}}}"\n'.format(p=p) 669 | ) 670 | 671 | f.write( 672 | textwrap.dedent( 673 | """\ 674 | WHISK_DEPLOY_DIR_{p} = "${{WHISK_DEPLOY_DIR_BASE}}/{p}" 675 | WHISK_TARGETS_{p} = "{targets}" 676 | """ 677 | ).format( 678 | p=p, 679 | targets=" ".join( 680 | sorted(conf["products"][p].get("targets", [])) 681 | ), 682 | ) 683 | ) 684 | 685 | f.write("\n") 686 | 687 | # Only set up Bitbake multiconfig if the selected Whisk products 688 | # support it. 689 | if using_multiconfig: 690 | multiconfigs = set("product-%s" % p for p in cur_products) 691 | for p in cur_products: 692 | multiconfigs |= set(conf["products"][p].get("multiconfigs", [])) 693 | 694 | f.write( 695 | textwrap.dedent( 696 | """\ 697 | BBMULTICONFIG = "{multiconfigs}" 698 | """ 699 | ).format(multiconfigs=" ".join(sorted(multiconfigs))) 700 | ) 701 | 702 | f.write('BBMASK += "${BBMASK_${WHISK_PRODUCT}}"\n') 703 | f.write( 704 | f'{compat.basehash_ignore_vars}{compat.append_sep}append = " WHISK_PROJECT_ROOT"\n' 705 | ) 706 | 707 | f.write(conf.get("core", {}).get("conf", "")) 708 | f.write("\n") 709 | 710 | # If the selected whisk product is not using multiconfig, then 711 | # force all the configuration bits Whisk would normally assign 712 | # to the product MC, into the base conf. 713 | # 714 | # Do this by forcibly injecting the product-.conf file 715 | # in Whisk's conf/multiconfig directory, at the very bottom 716 | # of site.conf. 717 | if not using_multiconfig: 718 | assert len(cur_products) == 1 719 | 720 | f.write( 721 | textwrap.dedent( 722 | """\ 723 | # Multiconfig not enabled. Insert all product- 724 | # specific info into base configuration. 725 | require conf/multiconfig/product-{product}.conf 726 | """ 727 | ).format(product=cur_products[0]) 728 | ) 729 | f.write("\n") 730 | 731 | mc_dir = build_dir / "whisk" / "conf" / "multiconfig" 732 | mc_dir.mkdir(parents=True, exist_ok=True) 733 | for name, p in conf["products"].items(): 734 | with (mc_dir / ("product-%s.conf" % name)).open("w") as f: 735 | f.write( 736 | textwrap.dedent( 737 | """\ 738 | # This file was dynamically generated by whisk 739 | WHISK_PRODUCT = "{product}" 740 | WHISK_PRODUCT_DESCRIPTION = "{description}" 741 | 742 | """ 743 | ).format( 744 | product=name, 745 | description=p.get("description", ""), 746 | ) 747 | ) 748 | 749 | f.write(p.get("conf", "")) 750 | f.write("\n") 751 | 752 | with (build_dir / "conf" / "bblayers.conf").open("w") as f: 753 | f.write( 754 | textwrap.dedent( 755 | """\ 756 | # This file was dynamically generated by whisk 757 | BBPATH = "${TOPDIR}" 758 | BBFILES ?= "" 759 | 760 | """ 761 | ) 762 | ) 763 | 764 | for name in ["core"] + cur_products: 765 | for l, paths in cur_layers_paths.items(): 766 | if not l in get_product(name).get("layers", []): 767 | for p in paths: 768 | f.write('BBMASK_%s += "%s"\n' % (name, p)) 769 | for l, masks in cur_layers_bbmasks.items(): 770 | if l in get_product(name).get("layers", []): 771 | for m in masks: 772 | f.write('BBMASK_%s += "%s"\n' % (name, m)) 773 | f.write("\n") 774 | 775 | for l in version.get("layers", []): 776 | if l["name"] in requested_layers: 777 | for p in l.get("paths", []): 778 | f.write('BBLAYERS += "%s"\n' % p) 779 | 780 | f.write('BBLAYERS += "%s/meta-whisk"\n\n' % THIS_DIR) 781 | 782 | f.write("%s\n" % conf.get("core", {}).get("layerconf", "")) 783 | 784 | f.write( 785 | textwrap.dedent( 786 | """\ 787 | # This line gives devtool a place to add its layers 788 | BBLAYERS += "" 789 | """ 790 | ) 791 | ) 792 | 793 | if write and not sys_args.init: 794 | return 0 795 | 796 | if not user_args.quiet: 797 | print("PRODUCT = %s" % " ".join(cur_products)) 798 | print("MODE = %s" % cur_mode) 799 | print("SITE = %s" % cur_site) 800 | print("VERSION = %s" % cur_version, end="") 801 | if cur_version != cur_actual_version: 802 | print(" (%s)" % cur_actual_version) 803 | else: 804 | print() 805 | 806 | return 0 807 | 808 | 809 | def validate(args): 810 | import yamllint.linter 811 | import yamllint.config 812 | 813 | ret = 0 814 | 815 | try: 816 | with args.conf.open("r") as f, SCHEMA_FILE.open("r") as schema: 817 | jsonschema.validate(yaml.load(f, Loader=yaml.Loader), json.load(schema)) 818 | except jsonschema.ValidationError as e: 819 | print(e) 820 | ret = 1 821 | 822 | if parse_conf_file(args.conf) == (None, None): 823 | ret = 1 824 | 825 | config = yamllint.config.YamlLintConfig( 826 | textwrap.dedent( 827 | """\ 828 | extends: default 829 | rules: 830 | # Long lines are fine 831 | line-length: disable 832 | """ 833 | ) 834 | ) 835 | 836 | ret = 0 837 | with args.conf.open("r") as f: 838 | for p in yamllint.linter.run(f, config, str(args.conf)): 839 | print("%r" % p) 840 | ret = 1 841 | 842 | return ret 843 | 844 | 845 | def main(): 846 | parser = argparse.ArgumentParser(description="Whisk product manager") 847 | 848 | subparser = parser.add_subparsers(dest="command") 849 | subparser.required = True 850 | 851 | configure_parser = subparser.add_parser( 852 | "configure", help="Configure build environment" 853 | ) 854 | 855 | configure_parser.add_argument( 856 | "--conf", help="Project configuration file", type=pathlib.Path 857 | ) 858 | configure_parser.add_argument( 859 | "--init", action="store_true", help="Run first-time initialization" 860 | ) 861 | configure_parser.add_argument( 862 | "--env", help="Path to environment output file", type=pathlib.Path 863 | ) 864 | configure_parser.add_argument("user_args", nargs="*", help="User arguments") 865 | configure_parser.set_defaults(func=configure) 866 | 867 | validate_parser = subparser.add_parser( 868 | "validate", help="Validate configuration file" 869 | ) 870 | validate_parser.add_argument( 871 | "conf", help="configuration file to validate", type=pathlib.Path 872 | ) 873 | validate_parser.set_defaults(func=validate) 874 | 875 | args = parser.parse_args() 876 | 877 | return args.func(args) 878 | 879 | 880 | if __name__ == "__main__": 881 | sys.exit(main()) 882 | -------------------------------------------------------------------------------- /whisk.schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-07/schema#", 3 | "$id": "http://www.garmin.com/schemas/whisk.json", 4 | 5 | "definitions": { 6 | "fetch": { 7 | "type": "object", 8 | "properties": { 9 | "commands": { 10 | "type": "array", 11 | "items": { 12 | "type": "string" 13 | } 14 | } 15 | }, 16 | "additionalProperties": false 17 | } 18 | }, 19 | 20 | "type": "object", 21 | "properties": { 22 | "version": { 23 | "type": "number" 24 | }, 25 | 26 | "project_root": { 27 | "type": "string" 28 | }, 29 | 30 | "cache": { 31 | "type": "string" 32 | }, 33 | 34 | "defaults": { 35 | "type": "object", 36 | "properties": { 37 | "products": { 38 | "type": "array", 39 | "items": { 40 | "type": "string" 41 | } 42 | }, 43 | "mode": { 44 | "type": "string" 45 | }, 46 | "site": { 47 | "type": "string" 48 | }, 49 | "build_dir": { 50 | "type": "string" 51 | } 52 | }, 53 | "additionalProperties": false 54 | }, 55 | 56 | "hooks": { 57 | "type": "object", 58 | "properties": { 59 | "pre_init": { 60 | "type": "string" 61 | }, 62 | "env_passthrough_vars": { 63 | "type": "array", 64 | "items": { 65 | "type": "string" 66 | } 67 | }, 68 | "post_init": { 69 | "type": "string" 70 | } 71 | }, 72 | "additionalProperties": false 73 | }, 74 | 75 | "fetch": { 76 | "$ref": "#/definitions/fetch" 77 | }, 78 | 79 | "versions": { 80 | "type": "object", 81 | "patternProperties": { 82 | "^(?!default$)[a-zA-Z0-9_-]+$": { 83 | "type": "object", 84 | "properties": { 85 | "description": { 86 | "type": "string" 87 | }, 88 | "oeinit": { 89 | "type": "string" 90 | }, 91 | "compat": { 92 | "type": "string" 93 | }, 94 | "fetch": { 95 | "$ref": "#/definitions/fetch" 96 | }, 97 | "pyrex": { 98 | "type": "object", 99 | "properties": { 100 | "root": { 101 | "type": "string" 102 | }, 103 | "conf": { 104 | "type": "string" 105 | } 106 | }, 107 | "additionalProperties": false, 108 | "required": [ 109 | "root", 110 | "conf" 111 | ] 112 | }, 113 | "layers": { 114 | "type": "array", 115 | "items": { 116 | "type": "object", 117 | "properties": { 118 | "name": { 119 | "type": "string" 120 | }, 121 | "paths": { 122 | "type": "array", 123 | "items": { 124 | "type": "string" 125 | } 126 | }, 127 | "bbmask": { 128 | "type": "array", 129 | "items": { 130 | "type": "string" 131 | } 132 | }, 133 | "fetch": { 134 | "$ref": "#/definitions/fetch" 135 | } 136 | }, 137 | "additionalProperties": false, 138 | "required": [ 139 | "name" 140 | ] 141 | } 142 | }, 143 | "tags": { 144 | "type": "object" 145 | } 146 | }, 147 | "additionalProperties": false 148 | } 149 | }, 150 | "additionalProperties": false 151 | }, 152 | 153 | "modes": { 154 | "type": "object", 155 | "patternProperties": { 156 | "^[a-zA-Z0-9_-]+$": { 157 | "type": "object", 158 | "properties": { 159 | "description": { 160 | "type": "string" 161 | }, 162 | "conf": { 163 | "type": "string" 164 | }, 165 | "tags": { 166 | "type": "object" 167 | } 168 | }, 169 | "additionalProperties": false 170 | } 171 | }, 172 | "additionalProperties": false 173 | }, 174 | 175 | "sites": { 176 | "type": "object", 177 | "patternProperties": { 178 | "^[a-zA-Z0-9_-]+$": { 179 | "type": "object", 180 | "properties": { 181 | "description": { 182 | "type": "string" 183 | }, 184 | "conf": { 185 | "type": "string" 186 | }, 187 | "tags": { 188 | "type": "object" 189 | } 190 | }, 191 | "additionalProperties": false 192 | } 193 | }, 194 | "additionalProperties": false 195 | }, 196 | 197 | "core": { 198 | "type": "object", 199 | "properties": { 200 | "layers": { 201 | "type": "array", 202 | "items": { 203 | "type": "string" 204 | } 205 | }, 206 | "layerconf": { 207 | "type": "string" 208 | }, 209 | "conf": { 210 | "type": "string" 211 | } 212 | }, 213 | "additionalProperties": false 214 | }, 215 | 216 | "products": { 217 | "type": "object", 218 | "patternProperties": { 219 | "^(?!core$)[a-zA-Z0-9_-]+$": { 220 | "type": "object", 221 | "properties": { 222 | "description": { 223 | "type": "string" 224 | }, 225 | "maintainers": { 226 | "type": "array", 227 | "items": { 228 | "type": "object", 229 | "properties": { 230 | "name": { 231 | "type": "string" 232 | }, 233 | "email": { 234 | "type": "string" 235 | } 236 | } 237 | }, 238 | "additionalProperties": false, 239 | "required": [ 240 | "name" 241 | ] 242 | }, 243 | "default_version": { 244 | "type": "string" 245 | }, 246 | "layers": { 247 | "type": "array", 248 | "items": { 249 | "type": "string" 250 | } 251 | }, 252 | "targets": { 253 | "type": "array", 254 | "items": { 255 | "type": "string" 256 | } 257 | }, 258 | "multiconfig_enabled": { 259 | "type": "boolean" 260 | }, 261 | "multiconfigs": { 262 | "type": "array", 263 | "items": { 264 | "type": "string" 265 | } 266 | }, 267 | "conf": { 268 | "type": "string" 269 | }, 270 | "tags": { 271 | "type": "object" 272 | } 273 | }, 274 | "additionalProperties": false, 275 | "required": [ 276 | "default_version" 277 | ] 278 | } 279 | }, 280 | "additionalProperties": false 281 | } 282 | }, 283 | "additionalProperties": false, 284 | "required": [ 285 | "version", 286 | "versions" 287 | ] 288 | } 289 | -------------------------------------------------------------------------------- /yamllint.yaml: -------------------------------------------------------------------------------- 1 | extends: default 2 | 3 | rules: 4 | # Long lines are fine 5 | line-length: disable 6 | --------------------------------------------------------------------------------