├── .coveragerc ├── .github ├── CODEOWNERS └── workflows │ ├── autoapprove.yml │ ├── automerge.yml │ ├── e2e_test.yml │ ├── pip_e2e_test.yml │ ├── shellcheck.yml │ └── test.yml ├── .gitignore ├── .readthedocs.yml ├── .vale.ini ├── CONTRIBUTING.md ├── DEVELOPING.md ├── LICENSE ├── MANIFEST.in ├── README.md ├── docs ├── Makefile ├── make.bat ├── requirements.txt ├── source │ ├── conf.py │ └── index.md └── sphinx_build_symlink │ ├── __init__.py │ └── setup.py ├── ros_cross_compile ├── __init__.py ├── __main__.py ├── builders.py ├── data_collector.py ├── dependencies.py ├── docker │ ├── build_workspace.sh │ ├── gather_rosdeps.sh │ ├── rosdep.Dockerfile │ ├── rosdep_focal.Dockerfile │ ├── runtime.Dockerfile │ └── sysroot.Dockerfile ├── docker_client.py ├── mixins │ ├── cross-compile.mixin │ └── index.yaml ├── pipeline_stages.py ├── platform.py ├── qemu │ ├── qemu-aarch64-static │ └── qemu-arm-static ├── ros_cross_compile.py ├── runtime.py └── sysroot_creator.py ├── setup.py ├── test ├── __init__.py ├── custom-setup-with-data.sh ├── custom-setup.sh ├── data │ └── arbitrary.txt ├── dummy_pkg │ ├── CMakeLists.txt │ ├── main.cpp │ └── package.xml ├── dummy_pkg_ros2_cpp │ ├── CMakeLists.txt │ ├── main.cpp │ └── package.xml ├── dummy_pkg_ros2_py │ ├── __init__.py │ ├── dummy_pkg_ros2_py │ │ └── __init__.py │ ├── package.xml │ ├── resource │ │ └── dummy_pkg_ros2_py │ └── setup.py ├── run_e2e_test.sh ├── test_builders.py ├── test_colcon_mixins.py ├── test_copyright.py ├── test_data_collector.py ├── test_dependencies.py ├── test_docker_client.py ├── test_entrypoint.py ├── test_flake8.py ├── test_mypy.py ├── test_pep257.py ├── test_pipeline_stages.py ├── test_platform.py ├── test_sysroot_creator.py ├── user-custom-setup └── utilities.py └── tox.ini /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | omit = .eggs/* 3 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # Reviewers (and by extension approvers) will be the default owners for everything in the repo. 2 | # Unless a later match takes precedence, @reviewers will be requested for review when someone opens a pull request. 3 | * @ros-tooling/reviewers 4 | -------------------------------------------------------------------------------- /.github/workflows/autoapprove.yml: -------------------------------------------------------------------------------- 1 | name: Auto approve 2 | 3 | on: 4 | pull_request_target: 5 | types: [labeled] 6 | 7 | jobs: 8 | # Auto-approve dependabot PRs since this repo requires at least one approving review. 9 | # Dependabot will automatically merge minor version upgrades 10 | # (see .dependabot/config.yml for more info). 11 | auto-approve-dependabot: 12 | runs-on: ubuntu-latest 13 | if: github.actor == 'dependabot[bot]' && contains(github.event.pull_request.labels.*.name, 'dependencies') 14 | steps: 15 | - uses: hmarr/auto-approve-action@v3.1.0 16 | with: 17 | github-token: "${{ secrets.GITHUB_TOKEN }}" 18 | -------------------------------------------------------------------------------- /.github/workflows/automerge.yml: -------------------------------------------------------------------------------- 1 | name: Auto merge 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | master 7 | pull_request_review: 8 | types: 9 | - submitted 10 | check_suite: 11 | types: 12 | - completed 13 | status: {} 14 | jobs: 15 | # Automatically merge approved and green dependabot PRs. 16 | auto-merge-dependabot: 17 | runs-on: ubuntu-latest 18 | if: github.actor == 'dependabot[bot]' || github.actor == 'dependabot-preview[bot]' 19 | steps: 20 | - uses: pascalgn/automerge-action@v0.15.2 21 | env: 22 | GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" 23 | MERGE_LABELS: "dependencies" 24 | MERGE_METHOD: "squash" # Sqush and merge 25 | MERGE_COMMIT_MESSAGE: "pull-request-title-and-description" 26 | MERGE_RETRY_SLEEP: "1200000" # Retry after 20m, enough time for check suites to run 27 | UPDATE_RETRIES: "6" 28 | UPDATE_METHOD: "rebase" # Rebase PR on base branch 29 | UPDATE_RETRY_SLEEP: "300000" 30 | -------------------------------------------------------------------------------- /.github/workflows/e2e_test.yml: -------------------------------------------------------------------------------- 1 | name: End-to-end Testing (Nightly) 2 | on: 3 | pull_request: 4 | # Running on pull requests to catch breaking changes as early as possible. 5 | # Waiting for this test to pass is recommended, but contributors can use their discretion whether they want to or not. 6 | 7 | jobs: 8 | build_and_test_20_04: 9 | runs-on: ubuntu-20.04 10 | strategy: 11 | fail-fast: false 12 | matrix: 13 | target_arch: [aarch64, armhf, x86_64] 14 | target_os: [ubuntu, debian] 15 | rosdistro: [foxy, galactic] 16 | env: 17 | METRICS_OUT_DIR: /tmp/collected_metrics 18 | steps: 19 | - name: Checkout sources 20 | uses: actions/checkout@v3 21 | - name: Setup python 22 | uses: actions/setup-python@v4 23 | with: 24 | # NOTE: doing a python version matrix on top of the 25 | # already huge target matrix would take forever, 26 | # so here we just target the minimum supported version 27 | python-version: 3.7 28 | - name: Install dependencies 29 | run: | 30 | sudo apt update && sudo apt install -y qemu-user-static 31 | - name: Install cross-compile 32 | run: pip install -e . 33 | - run: mkdir -p ${{ env.METRICS_OUT_DIR }} 34 | - name: Run end-to-end test 35 | run: | 36 | ./test/run_e2e_test.sh -a "${{ matrix.target_arch }}" -o "${{ matrix.target_os }}" -d "${{ matrix.rosdistro }}" 37 | - uses: actions/upload-artifact@v3 38 | with: 39 | name: performance_metrics 40 | path: ${{ env.METRICS_OUT_DIR }} 41 | 42 | build_and_test_22_04: 43 | runs-on: ubuntu-22.04 44 | strategy: 45 | fail-fast: false 46 | matrix: 47 | target_arch: [aarch64, armhf, x86_64] 48 | target_os: [ubuntu, debian] 49 | rosdistro: [humble, rolling] 50 | env: 51 | METRICS_OUT_DIR: /tmp/collected_metrics 52 | steps: 53 | - name: Checkout sources 54 | uses: actions/checkout@v3 55 | - name: Setup python 56 | uses: actions/setup-python@v4 57 | with: 58 | # NOTE: doing a python version matrix on top of the 59 | # already huge target matrix would take forever, 60 | # so here we just target the minimum supported version 61 | python-version: 3.8 62 | - name: Install dependencies 63 | run: | 64 | sudo apt update && sudo apt install -y qemu-user-static 65 | - name: Install cross-compile 66 | run: pip install -e . 67 | - run: mkdir -p ${{ env.METRICS_OUT_DIR }} 68 | - name: Run end-to-end test 69 | run: | 70 | ./test/run_e2e_test.sh -a "${{ matrix.target_arch }}" -o "${{ matrix.target_os }}" -d "${{ matrix.rosdistro }}" 71 | - uses: actions/upload-artifact@v3 72 | with: 73 | name: performance_metrics 74 | path: ${{ env.METRICS_OUT_DIR }} 75 | -------------------------------------------------------------------------------- /.github/workflows/pip_e2e_test.yml: -------------------------------------------------------------------------------- 1 | name: End-to-end testing for ros-cross-compile using PIP (Nightly Canary) 2 | on: 3 | pull_request: 4 | # Running on pull requests to catch breaking changes as early as possible. 5 | # Waiting for this test to pass is recommended, but contributors can use their discretion whether they want to or not. 6 | 7 | jobs: 8 | build_and_test_20_04: 9 | runs-on: ubuntu-20.04 10 | strategy: 11 | fail-fast: false 12 | matrix: 13 | target_arch: [aarch64, armhf] 14 | target_os: [ubuntu] 15 | rosdistro: [foxy, galactic] 16 | install_type: 17 | - test 18 | - prod 19 | steps: 20 | - name: Checkout sources 21 | uses: actions/checkout@v3 22 | - name: Setup python 23 | uses: actions/setup-python@v4 24 | with: 25 | python-version: 3.7 26 | - name: Install dependencies 27 | run: | 28 | sudo apt update && sudo apt install -y qemu-user-static 29 | - name: Install ros-cross-compile from Test PyPi 30 | if: matrix.install_type == 'test' 31 | run: | 32 | pip install 'docker>=2,<3' 33 | pip install -i https://test.pypi.org/simple/ ros-cross-compile 34 | - name: Install ros-cross-compile from Prod PyPi 35 | if: matrix.install_type == 'prod' 36 | run: pip install ros-cross-compile 37 | - name: Run end-to-end test 38 | run: | 39 | ./test/run_e2e_test.sh -a "${{ matrix.target_arch }}" -o "${{ matrix.target_os }}" -d "${{ matrix.rosdistro }}" 40 | 41 | build_and_test_22_04: 42 | runs-on: ubuntu-22.04 43 | strategy: 44 | fail-fast: false 45 | matrix: 46 | target_arch: [aarch64, armhf] 47 | target_os: [ubuntu] 48 | rosdistro: [humble, rolling] 49 | install_type: 50 | - test 51 | - prod 52 | steps: 53 | - name: Checkout sources 54 | uses: actions/checkout@v3 55 | - name: Setup python 56 | uses: actions/setup-python@v4 57 | with: 58 | python-version: 3.8 59 | - name: Install dependencies 60 | run: | 61 | sudo apt update && sudo apt install -y qemu-user-static 62 | - name: Install ros-cross-compile from Test PyPi 63 | if: matrix.install_type == 'test' 64 | run: | 65 | pip install 'docker>=2,<3' 66 | pip install -i https://test.pypi.org/simple/ ros-cross-compile 67 | - name: Install ros-cross-compile from Prod PyPi 68 | if: matrix.install_type == 'prod' 69 | run: pip install ros-cross-compile 70 | - name: Run end-to-end test 71 | run: | 72 | ./test/run_e2e_test.sh -a "${{ matrix.target_arch }}" -o "${{ matrix.target_os }}" -d "${{ matrix.rosdistro }}" 73 | -------------------------------------------------------------------------------- /.github/workflows/shellcheck.yml: -------------------------------------------------------------------------------- 1 | name: Run shellcheck linter 2 | on: 3 | pull_request: 4 | push: 5 | branches: 6 | - master 7 | jobs: 8 | lint: 9 | runs-on: ubuntu-20.04 10 | steps: 11 | - uses: actions/checkout@v1 12 | - name: Run Shellcheck on shell scripts 13 | uses: reviewdog/action-shellcheck@v1 14 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test cross_compile 2 | on: 3 | pull_request: 4 | push: 5 | branches: 6 | - master 7 | 8 | jobs: 9 | test_macOS: 10 | runs-on: macOS-latest 11 | strategy: 12 | fail-fast: false 13 | matrix: 14 | python-version: [3.7, 3.8] 15 | steps: 16 | - name: Checkout sources 17 | uses: actions/checkout@v3 18 | - uses: actions/setup-python@v4 19 | with: 20 | python-version: ${{ matrix.python-version }} 21 | architecture: x64 22 | - name: Install Tox for testing 23 | run: pip install tox 24 | - name: Run tests 25 | run: tox -e py 26 | 27 | test_ubuntu: 28 | runs-on: ubuntu-latest 29 | strategy: 30 | fail-fast: false 31 | matrix: 32 | python-version: [3.7, 3.8] 33 | steps: 34 | - name: Checkout sources 35 | uses: actions/checkout@v3 36 | - name: Install dependencies 37 | run: sudo apt-get update && sudo apt-get install -y qemu-user-static 38 | - uses: actions/setup-python@v4 39 | with: 40 | python-version: ${{ matrix.python-version }} 41 | architecture: x64 42 | - name: Install Tox for testing 43 | run: pip3 install tox 44 | - name: Run tests 45 | run: tox -e py 46 | - uses: codecov/codecov-action@v3.1.1 47 | with: 48 | file: coverage.xml 49 | flags: unittests 50 | name: codecov-umbrella 51 | yml: ./codecov.yml 52 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | *.pyc 3 | *.egg-info/ 4 | .eggs/ 5 | .tox/ 6 | /.coverage 7 | /coverage.xml 8 | 9 | .idea/ 10 | 11 | sysroot/ 12 | 13 | .DS_Store 14 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | sphinx: 4 | configuration: docs/source/conf.py 5 | 6 | python: 7 | version: 3.7 8 | install: 9 | # Install requirements as system packages using pip 10 | - requirements: docs/requirements.txt 11 | # Ensure sphinx-build is in PATH 12 | - method: setuptools 13 | path: docs/sphinx_build_symlink/ 14 | # Install ros_cross_compile using setuptools 15 | - method: setuptools 16 | path: . 17 | -------------------------------------------------------------------------------- /.vale.ini: -------------------------------------------------------------------------------- 1 | StylesPath = styles 2 | MinAlertLevel = suggestion 3 | 4 | [*.md] 5 | BasedOnStyles = Google, Vale, write-good 6 | 7 | Vale.Spelling = YES 8 | 9 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional 3 | documentation, we greatly value feedback and contributions from our community. 4 | 5 | Please read through this document before submitting any issues or pull requests to ensure we have all the necessary 6 | information to effectively respond to your bug report or contribution. 7 | 8 | 9 | ## Reporting Bugs/Feature Requests 10 | We welcome you to use the GitHub issue tracker to report bugs or suggest features. 11 | 12 | When filing an issue, please check [existing open](https://github.com/ros-tooling/cross_compile/issues), or [recently closed](https://github.com/ros-tooling/cross_compile/issues?utf8=%E2%9C%93&q=is%3Aissue%20is%3Aclosed%20), issues to make sure somebody else hasn't already 13 | reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: 14 | 15 | * A reproducible test case or series of steps 16 | * The version of our code being used 17 | * Any modifications you've made relevant to the bug 18 | * Anything unusual about your environment or deployment 19 | 20 | 21 | ## Contributing via Pull Requests 22 | Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: 23 | 24 | 1. You are working against the latest source on the *master* branch. 25 | 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. 26 | 3. You open an issue to discuss any significant work - we would hate for your time to be wasted. 27 | 28 | To send us a pull request, please: 29 | 30 | 1. Fork the repository. 31 | 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. 32 | 3. Ensure local tests pass. 33 | 4. Commit to your fork using clear commit messages. 34 | 5. Send us a pull request, answering any default questions in the pull request interface. 35 | 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. 36 | 37 | GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and 38 | [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). 39 | 40 | 41 | ## Finding contributions to work on 42 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any ['help wanted'](https://github.com/ros-tooling/cross_compile/labels/help%20wanted) issues is a great place to start. 43 | 44 | 45 | ## Code of Conduct 46 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 47 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 48 | opensource-codeofconduct@amazon.com with any additional questions or comments. 49 | 50 | 51 | ## Security issue notifications 52 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. 53 | 54 | 55 | ## Licensing 56 | Any contribution that you make to this repository will 57 | be under the Apache 2 License, as dictated by that 58 | [license](http://www.apache.org/licenses/LICENSE-2.0.html): 59 | 60 | ~~~ 61 | 5. Submission of Contributions. Unless You explicitly state otherwise, 62 | any Contribution intentionally submitted for inclusion in the Work 63 | by You to the Licensor shall be under the terms and conditions of 64 | this License, without any additional terms or conditions. 65 | Notwithstanding the above, nothing herein shall supersede or modify 66 | the terms of any separate license agreement you may have executed 67 | with Licensor regarding such Contributions. 68 | ~~~ 69 | 70 | We may ask you to sign a [Contributor License Agreement (CLA)](http://en.wikipedia.org/wiki/Contributor_License_Agreement) for larger changes. 71 | -------------------------------------------------------------------------------- /DEVELOPING.md: -------------------------------------------------------------------------------- 1 | # Developing ros-cross-compile 2 | 3 | This document contains instructions, guidelines, and requirements for developing this tool. 4 | 5 | ## Writing Code 6 | 7 | For easiest development, use `virtualenv` to manage the dependency environment. 8 | 9 | ``` 10 | # create a virtualenv 11 | virtualenv venv 12 | # use it 13 | source venv/bin/activate 14 | # install this package in the virtualenv to get dependencies 15 | pip3 install -e . 16 | # now you can run your work 17 | ros_cross_compile 18 | ``` 19 | 20 | ## Tests 21 | 22 | These are testing entrypoints: 23 | 24 | * Unit tests via `tox` 25 | * run `tox -e py` 26 | * End-to-end shell script 27 | * run `./test/run_e2e_test.sh` 28 | 29 | ## Host Platforms 30 | 31 | The target host platforms for this tool are the Tier 1 platforms specified by non-EOL ROS 2 LTS distributions, on x86_64. 32 | Cross-compiling on ARM host platforms is out of scope. 33 | 34 | See [REP 2000](https://www.ros.org/reps/rep-2000.html) for this list, which is now: 35 | * Foxy Fitzroy 36 | * Ubuntu Focal 20.04 37 | * support installation via `pip3` and `apt` 38 | * MacOS Sierra (10.12) 39 | * support installation via `pip3` 40 | * Windows 10 (VS2019) 41 | * support installation via `pip3` 42 | 43 | Though not all of these targets may be fully supported yet, design decisions may not be made that rule out support for those platforms in the future. 44 | 45 | ## Releasing ros-cross-compile 46 | 47 | This just a simple reminder for the process to release: 48 | 49 | 1. `pip3 install -U twine setuptools` 50 | 1. Update version number in `setup.py` 51 | 1. Create a new git tag associated with that commit 52 | * Note, this needs to be the _actual commit_ that goes on `master` - so if you do a PR, wait until after the PR is merged to tag the new HEAD (in case of e.g. squash and merge) 53 | 1. `python3 setup.py sdist bdist_wheel` 54 | 1. Probably try testpypi first: `twine upload -r testpypi dist/*` - then test it 55 | 1. Upload to pypi: `twine upload dist/*` 56 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include CONTRIBUTING.md 2 | include LICENSE 3 | include README.md 4 | 5 | exclude .coveragerc 6 | exclude .readthedocs.yml 7 | exclude .vale.ini 8 | exclude tox.ini 9 | 10 | prune .github 11 | prune docs/build 12 | prune test 13 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Notice: This repository is deprecated. It is outdated and will not be supported anymore. 2 | It is recommended to use a native cross-compilation method, which can be up to 8 times faster. 3 | See details in this ROS Discourse discussion: 4 | https://discourse.ros.org/t/call-for-help-maintainership-of-the-ros-cross-compile-tool/26511 5 | 6 | # ROS / ROS 2 Cross Compile Tool 7 | 8 | ![License](https://img.shields.io/github/license/ros-tooling/cross_compile) 9 | [![Documentation Status](https://readthedocs.org/projects/cross_compile/badge/?version=latest)](https://cross_compile.readthedocs.io/en/latest/?badge=latest) 10 | 11 | A tool to automate compiling ROS and ROS 2 workspaces to non-native architectures. 12 | 13 | :construction: `ros_cross_compile` relies on running emulated builds 14 | using QEmu, #69 tracks progress toward enabling cross-compilation. 15 | 16 | 17 | ## Supported targets 18 | 19 | This tool supports compiling a workspace for all combinations of the following: 20 | 21 | * Architecture: `armhf`, `aarch64`, `x86_64` 22 | * ROS Distro 23 | * ROS: `melodic`, `noetic` 24 | * ROS 2: `foxy`, `galactic`, `humble`, `rolling` 25 | * OS: `Ubuntu`, `Debian` 26 | 27 | NOTE: ROS 2 supports Debian only as a Tier 3 platform. 28 | This means that there are not `apt` repositories available for the ROS 2 Core on this platform. 29 | Because of that, when targeting Debian for a ROS 2 workspace, you must also include the source for the core as well. 30 | It is recommended to use a release branch of `ros2.repos` from https://github.com/ros2/ros2 to do so, rather than `master`, so that you are not affected by development branch bugs and API changes. 31 | 32 | ## Supported hosts 33 | 34 | This tool officially supports running on the following host systems. 35 | Note that many others likely work, but these are being thoroughly tested. 36 | 37 | * Ubuntu 20.04 Focal 38 | * OSX Mojave 39 | 40 | ## Installation 41 | 42 | ### Prerequisites 43 | 44 | This tool requires that you have already installed 45 | * [Docker](https://docs.docker.com/install/) 46 | * Follow the instructions to add yourself to the `docker` group as well, so you can run containers as a non-root user 47 | * Python 3.7 or higher 48 | 49 | If you are using a Linux host, you must also install QEmu (Docker for OSX performs emulation automatically): 50 | 51 | ```sh 52 | sudo apt-get install qemu-user-static 53 | ``` 54 | 55 | ### Installing ros_cross_compile 56 | 57 | To install the stable release, 58 | 59 | ```sh 60 | pip3 install ros_cross_compile 61 | ``` 62 | 63 | If you would like the latest nightly build, you can get it from Test PyPI 64 | 65 | ```sh 66 | pip3 install --index-url https://test.pypi.org/simple/ ros_cross_compile 67 | ``` 68 | 69 | ## How it works, high level 70 | 71 | 1. Collect dependencies 72 | 1. Create a Docker image that has `rosdep` 73 | 1. Run the `rosdep` image against your target workspace to output a script that describes how to install its dependencies 74 | 1. Create "sysroot image" that has everything needed for building target workspace 75 | 1. Use a base image for the target architecture (aarch64, armhf, ...) 76 | 1. Install build tools (compilers, cmake, colcon, etc) 77 | 1. Run the dependency installer script collected in Step 1 (if dependency list hasn't changed since last run, this uses the Docker cache) 78 | 1. Build 79 | 1. Runs the "sysroot image" using QEmu emulation 80 | 1. `colcon build` 81 | 1. (Optional) Create runtime image 82 | 1. Creates a docker image that can be used on the target platform to run the build. See "Runtime Image" section. 83 | 84 | ## Usage 85 | 86 | This package installs the `ros_cross_compile` command. 87 | The command's first argument is the path to your ROS workspace. 88 | 89 | Here is a simple invocation for a standard workflow. 90 | 91 | ```bash 92 | ros_cross_compile /path/to/my/workspace --arch aarch64 --os ubuntu --rosdistro foxy 93 | ``` 94 | 95 | For information on all available options, run `ros_cross_compile -h`. 96 | See the following sections for information on the more complex options. 97 | 98 | ### Package Selection and Build Customization 99 | 100 | To choose which packages to install dependencies for, this tool runs `colcon list` on your workspace. 101 | To build, it runs `colcon build`. 102 | 103 | You can provide arbitrary arguments to these commands via the [colcon `defaults.yaml`](https://colcon.readthedocs.io/en/released/user/configuration.html#defaults-yaml). 104 | 105 | You can either specify the name of this file via `ros_cross_compile --colcon-defaults /path/to/defaults.yaml`, or if not specified, a file called `defaults.yaml` will be used if present. 106 | 107 | For example, there are repositories checked out in your workspace that contain packages that are not needed for your application - some repos provide many packages and you may only want one! 108 | In this scenario there is a "bringup" package that acts as the entry point to your application: 109 | 110 | ```yaml 111 | # my_workspace/defaults.yaml 112 | list: 113 | # only install dependencies for source packages that my package depends on 114 | packages-up-to: [my_application_bringup] 115 | build: 116 | # only build up to my package 117 | packages-up-to: [my_application_bringup] 118 | # example of a boolean commandline argument 119 | merge-install: true 120 | ``` 121 | 122 | Other configurations can be passed and used as command line args. Examples are CMake build arguments, like the build type or the verb configurations for the event handlers: 123 | 124 | ```yaml 125 | # my_workspace/defaults.yaml 126 | build: 127 | cmake-args: ["-DCMAKE_BUILD_TYPE=Release"] 128 | event-handlers: ["console_direct+"] 129 | ``` 130 | 131 | ### Custom rosdep script 132 | 133 | Your ROS application may need nonstandard rosdep rules. 134 | If so, you have the option to provide a script to be run before the `rosdep install` command collects keys. 135 | 136 | This script has access to the "Custom data directory" same as the "Custom setup script", see the following sections. If you need any extra files for setting up rosdep, they can be accessed via this custom data directory. 137 | 138 | Note that: 139 | 1. Rosdeps for melodic collected in an Ubuntu focal container for all other ROS distros 140 | rosdeps collected in an Ubuntu Focal container, so scripts must be compatible with that 141 | 142 | Here is an example script for an application that adds extra rosdep source lists 143 | 144 | ```bash 145 | cp ./custom-data/rosdep-rules/raspicam-node.yaml /etc/ros/rosdep/custom-rules/raspicam-node.yaml 146 | echo "yaml file:/etc/ros/rosdep/custom-rules/raspicam-node.yaml" > /etc/ros/rosdep/sources.list.d/22-raspicam-node.list 147 | ``` 148 | 149 | Tool invocation for this example: 150 | 151 | ```bash 152 | ros_cross_compile /path/to/my/workspace --arch aarch64 --os ubuntu \ 153 | --custom-rosdep-script /path/to/rosdep-script.sh \ 154 | --custom-data-dir /arbitrary/local/directory 155 | ``` 156 | 157 | ### Custom setup script 158 | 159 | Your ROS application may have build needs that aren't covered by `rosdep install`. 160 | If this is the case (for example you need to add extra apt repos), use the option `--custom-setup-script` to execute arbitrary code in the sysroot container. 161 | 162 | The path provided may be absolute, or relative to the current directory. 163 | 164 | Keep in mind 165 | * It's up to the user to determine whether the script is compatible with chosen base platform 166 | * Make sure to specify non-interactive versions of commands, for example `apt-get install -y`, or the script may hang waiting for input 167 | * You cannot make any assumptions about the state of the apt cache, so run `apt-get update` before installing packages 168 | * The script runs as root user in the container, so you don't need `sudo` 169 | 170 | Below is an example script for an application that installs some custom Raspberry Pi libraries. 171 | 172 | ```bash 173 | apt-get update 174 | apt-get install -y software-properties-common 175 | 176 | # Install Raspberry Pi library that we have not provided a rosdep rule for 177 | add-apt-repository ppa:rpi-distro/ppa 178 | apt-get install -y pigpio 179 | ``` 180 | 181 | Additionally, a custom setup script may control the build environment by populating the `/custom-data/setup.bash` file which will be sourced before building. 182 | 183 | ### Custom post-build script 184 | 185 | You may want to perform arbitrary post-processing on your build outputs, in the event of a sucessful build - use `--custom-post-build-script` for this. 186 | Keep in mind that it is run at the root of the built workspace. 187 | 188 | Following is an example setup that allows a user to run [colcon bundle](https://github.com/colcon/colcon-bundle) to create a portable bundle of the cross-compiled application. 189 | 190 | Here are the contents of `./postbuild.sh` 191 | 192 | ```bash 193 | #!/bin/bash 194 | set -eux 195 | 196 | apt-get update 197 | apt-get install -y wget 198 | wget http://packages.osrfoundation.org/gazebo.key -O - | apt-key add - 199 | 200 | apt-get install -y python3-apt 201 | pip3 install -u setuptools pip 202 | pip3 install -U colcon-ros-bundle 203 | 204 | colcon bundle \ 205 | --build-base build_"${TARGET_ARCH}" \ 206 | --install-base install_"${TARGET_ARCH}" \ 207 | --bundle-base bundle_"${TARGET_ARCH}" 208 | ``` 209 | 210 | Now, run 211 | 212 | ``` 213 | ros_cross_compile /path/to/my/workspace --arch aarch64 --os ubuntu \ 214 | --custom-post-build-script ./postbuild.sh 215 | ``` 216 | 217 | After the build completes, you should see the bundle outputs in `bundle_aarch64` 218 | 219 | 220 | ### Custom data directory 221 | 222 | Your custom setup or rosdep script (see preceding sections) may need some data that is not otherwise accessible. 223 | For example, you need to copy some precompiled vendor binaries to a specific location, or provide custom rosdep rules files. 224 | For this use case, you can use the option `--custom-data-dir` to point to an arbitrary path. 225 | The sysroot build copies this directory into the build environment, where it's available for use by your custom setup script at `./custom-data/`. 226 | 227 | **Example:** 228 | 229 | Custom data directory (`/arbitrary/local/directory`) 230 | ``` 231 | /arbitrary/local/directory/ 232 | +-- my-data/ 233 | | +-- something.txt 234 | ``` 235 | 236 | Setup Script (`/path/to/custom-setup.sh`) 237 | 238 | ```bash 239 | #!/bin/bash 240 | cat custom-data/something.txt 241 | ``` 242 | 243 | Tool invocation: 244 | 245 | ```bash 246 | ros_cross_compile /path/to/my/workspace --arch aarch64 --os ubuntu \ 247 | --custom-setup-script /path/to/custom-setup.sh \ 248 | --custom-data-dir /arbitrary/local/directory 249 | ``` 250 | 251 | Now, during the sysroot creation process, you should see the contents of `something.txt` printed during the execution of the custom script. 252 | 253 | NOTE: for trivial text files, as in the preceding example, you could have created those files fully within the `--custom-setup-script`. But for large or binary data such as precompiled libraries, this feature comes to the rescue. 254 | 255 | 256 | ### Runtime Image 257 | 258 | `ros_cross_compile` can optionally create and tag a Docker image that contains the build output and its runtime dependencies. 259 | 260 | The argument `--runtime-tag` takes a single value, which is the tag used for the output image. 261 | 262 | ``` 263 | OUTPUT_IMAGE=my_registry/image_name:image_tag 264 | ros_cross_compile $workspace --runtime-tag $OUTPUT_IMAGE 265 | ``` 266 | 267 | One way to deploy this image is to push it to a registry, from where it can be pulled onto a target platform 268 | 269 | ``` 270 | docker push $OUTPUT_IMAGE 271 | ``` 272 | 273 | The image contains any necessary emulation binaries to run locally if desired for smoke testing. 274 | 275 | ``` 276 | docker run -it $OUTPUT_IMAGE 277 | # In the shell inside the running container, the setup is already sourced for the default entrypoint 278 | ros2 launch my_package my.launch.py 279 | ``` 280 | 281 | Note: Currently this feature is a thin layer on top of the image used for building, so it is not a fully minimal image - it contains build tools, build dependencies, and test dependencies in addition to the necessary runtime dependencies. 282 | Future work is planned to slim down this output image to a properly minimal runtime. 283 | This work is tracked in https://github.com/ros-tooling/cross_compile/issues/263. 284 | 285 | 286 | ## Tutorial 287 | 288 | For a new user, this section walks you through a representative use case, step by step. 289 | 290 | This tutorial demonstrates how to cross-compile the [ROS 2 Demo Nodes](https://github.com/ros-tooling/demos) against ROS 2 Foxy, to run on an ARM64 Ubuntu system. 291 | You can generalize this workflow to use on any workspace for your project. 292 | 293 | NOTE: this tutorial assumes a Debian-based (including Ubuntu) Linux distribution as the host platform. 294 | 295 | ### Creating a simple source workspace 296 | 297 | Create a directory for your workspace and checkout the sources 298 | 299 | ``` 300 | mkdir -p cross_compile_ws/src 301 | cd cross_compile_ws 302 | git clone -b foxy https://github.com/ros2/demos src/demos 303 | ``` 304 | 305 | Create a file `defaults.yaml` in this directory with the following contents. This file narrows down the set of built packages, rather than building every single package in the source repository. This file is optional - see preceding section "Package Selection and Build Customization 306 | " for more information. 307 | 308 | ``` 309 | build: 310 | # only build the demo_nodes_cpp package, to save time building all of the demos 311 | packages-up-to: 312 | - demo_nodes_cpp 313 | # make a merged install space, which is easier to distribute 314 | merge-install: true 315 | # add some output for readability 316 | event-handlers: 317 | - console_cohesion+ 318 | - console_package_list+ 319 | 320 | ``` 321 | 322 | ### Running the cross-compilation 323 | 324 | ```bash 325 | ros_cross_compile . --rosdistro foxy --arch aarch64 --os ubuntu --colcon-defaults ./defaults.yaml 326 | ``` 327 | 328 | Here is a detailed look at the arguments passed to the script (`ros_cross_compile -h` will print all valid choices for each option): 329 | 330 | * `.` 331 | * The first argument to `ros_cross_compile` is the directory of the workspace to be built. This could be any relative or absolute path, in this case it's just `.`, the current working directory. 332 | * `--rosdistro foxy` 333 | * You may specify either a ROS and ROS 2 distribution by name, for example `noetic` (ROS) or `galactic` (ROS 2). 334 | * `--arch aarch64` 335 | * Target the ARMv8 / ARM64 / aarch64 architecture (which are different names for effectively the same thing). 336 | * `--os ubuntu` 337 | * The target OS is Ubuntu - the tool chooses the OS version automatically based on the ROS Distro's target OS. In this case for ROS 2 Foxy - Ubuntu 20.04 Focal Fossa. 338 | 339 | ### Outputs of the build 340 | 341 | Run the following command 342 | 343 | ```bash 344 | ls cross_compile_ws 345 | ``` 346 | 347 | If the build succeeded, the directory looks like this: 348 | 349 | ``` 350 | build_aarch64/ 351 | cc_internals/ 352 | defaults.yaml 353 | install_aarch64/ 354 | log/ 355 | src/ 356 | ``` 357 | 358 | * The created directory `install_aarch64` is the installation of your ROS workspace for your target architecture. 359 | * `cc_internals` is used by `ros_cross_compile` to cache artifacts between builds - as a user you will not need to inspect it 360 | 361 | You can verify that the build created binaries for the target architecture (note "ARM aarch64" in below output. Your `sha1` may differ): 362 | 363 | ```bash 364 | $ file install_aarch64/demo_nodes_cpp/lib/demo_nodes_cpp/talker 365 | install_aarch64/demo_nodes_cpp/lib/demo_nodes_cpp/talker: ELF 64-bit LSB shared object, ARM aarch64, version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux-aarch64.so.1, BuildID[sha1]=f086db477d6f5f919414d63911366077f1051b80, for GNU/Linux 3.7.0, not stripped 366 | ``` 367 | 368 | ### Using the build on a target platform 369 | 370 | Copy `install_aarch64` onto the target system into a location of your choosing. It contains the binaries for _your_ workspace. 371 | 372 | If your workspace has any dependencies that are outside the source tree - that is, if `rosdep` had anything to install during the build - then you still need to install these dependencies on the target system. 373 | 374 | Note first: if you need `rosdep` to install packages via the package manager, then your system will need its package manager sources (APT for Ubuntu). See [Setup Sources](https://docs.ros.org/en/foxy/Installation/Ubuntu-Install-Debians.html#setup-sources) portion of ROS 2 installation instructions for an example of how to do this on Ubuntu. 375 | 376 | ```bash 377 | # Run this on the target system, which must have rosdep already installed 378 | # remember `rosdep init`, `rosdep update`, `apt-get update` if you need them 379 | rosdep install --from-paths install_aarch64/share --ignore-src --rosdistro foxy -y 380 | ``` 381 | 382 | Now you may use the ROS installation as you would on any other system 383 | 384 | ```bash 385 | source install_aarch64/setup.bash 386 | ros2 run demo_nodes_cpp talker 387 | 388 | # and in a different shell 389 | ros2 run demo_nodes_cpp listener 390 | ``` 391 | 392 | ## Troubleshooting 393 | 394 | If you are running in docker with `/var/run/docker.sock` mounted and see the following error: 395 | > No src/ directory found at /ws, did you remember to mount your workspace? 396 | 397 | You may need to try running in docker-in-docker. This approach is demonstrated to work in gitlab-ci with a privileged runner and the following `gitlab.yml` as an example: 398 | 399 | ```yaml 400 | image: teracy/ubuntu:18.04-dind-19.03.3 401 | 402 | services: 403 | - docker:19.03.3-dind 404 | 405 | variables: 406 | # Disable TLS or we get SSLv1 errors. We shouldn't need this since we mount the /certs volume. 407 | # We also need to connect to the docker daemon via DOCKER_HOST. 408 | DOCKER_TLS_CERTDIR: "" 409 | DOCKER_HOST: tcp://docker:2375 410 | 411 | build-stuff: 412 | stage: build 413 | tags: 414 | - ros 415 | before_script: 416 | # Install packages 417 | - apt update 418 | - apt install -qq -y qemu-user-static python3-pip rsync 419 | 420 | # Set up the workspace 421 | - cd ${CI_PROJECT_DIR}/.. 422 | - rm -rf cross_compile_ws/src 423 | - mkdir -p cross_compile_ws/src 424 | - cp -r ${CI_PROJECT_DIR} cross_compile_ws/src/ 425 | - rsync -a ${CI_PROJECT_DIR}/../cross_compile_ws ${CI_PROJECT_DIR} 426 | - cd ${CI_PROJECT_DIR} 427 | 428 | # Install ros_cross_compile 429 | - pip3 install ros_cross_compile 430 | script: 431 | - ros_cross_compile cross_compile_ws --arch aarch64 --os ubuntu --rosdistro melodic 432 | artifacts: 433 | paths: 434 | - $CI_PROJECT_DIR/cross_compile_ws/install_aarch64 435 | expire_in: 1 week 436 | ``` 437 | 438 | ## License 439 | 440 | This library is licensed under the Apache 2.0 License. 441 | 442 | ## Build status 443 | 444 | | ROS 2 Release | Branch Name | Development | Source Debian Package | X86-64 Debian Package | ARM64 Debian Package | ARMHF Debian package | 445 | | ------------- | --------------- | ----------- | --------------------- | --------------------- | -------------------- | -------------------- | 446 | | Latest | `master` | [![Test Pipeline Status](https://github.com/ros-tooling/cross_compile/workflows/Test%20cross_compile/badge.svg)](https://github.com/ros-tooling/cross_compile/actions) | N/A | N/A | N/A | N/A | 447 | | Foxy | `foxy-devel` | [![Build Status](http://build.ros2.org/buildStatus/icon?job=Ddev__cross_compile__ubuntu_focal_amd64)](http://build.ros2.org/job/Ddev__cross_compile__ubuntu_focal_amd64) | [![Build Status](http://build.ros2.org/buildStatus/icon?job=Dsrc_uB__cross_compile__ubuntu_focal__source)](http://build.ros2.org/job/Dsrc_uB__cross_compile__ubuntu_focal__source) | [![Build Status](http://build.ros2.org/buildStatus/icon?job=Dbin_uB64__cross_compile__ubuntu_focal_amd64__binary)](http://build.ros2.org/job/Dbin_uB64__cross_compile__ubuntu_focal_amd64__binary) | N/A | N/A | 448 | 449 | 450 | [ros2_dev_setup]: https://index.ros.org/doc/ros2/Installation/Latest-Development-Setup/ 451 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SPHINXPROJ = ros_cross_compile 8 | SOURCEDIR = source 9 | BUILDDIR = build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=source 11 | set BUILDDIR=build 12 | set SPHINXPROJ=ros_cross_compile 13 | 14 | if "%1" == "" goto help 15 | 16 | %SPHINXBUILD% >NUL 2>NUL 17 | if errorlevel 9009 ( 18 | echo. 19 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 20 | echo.installed, then set the SPHINXBUILD environment variable to point 21 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 22 | echo.may add the Sphinx directory to PATH. 23 | echo. 24 | echo.If you don't have Sphinx installed, grab it from 25 | echo.http://sphinx-doc.org/ 26 | exit /b 1 27 | ) 28 | 29 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 30 | goto end 31 | 32 | :help 33 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 34 | 35 | :end 36 | popd 37 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | # Install ROS 2 dependencies using pip to help sphinx generate documentation API 2 | # This should *not* be used with the intent of actually running the software, colcon should 3 | # be used in this case. 4 | git+https://github.com/ament/ament_index.git#egg=ament_index_python&subdirectory=ament_index_python 5 | git+https://github.com/ros2/launch.git#egg=launch&subdirectory=launch 6 | sphinx-markdown-tables 7 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # -*- coding: utf-8 -*- 16 | # 17 | # ros_cross_compile documentation build configuration file, created by 18 | # sphinx-quickstart on Thu Sep 19 08:53:28 2019. 19 | # 20 | # This file is execfile()d with the current directory set to its 21 | # containing dir. 22 | # 23 | # Note that not all possible configuration values are present in this 24 | # autogenerated file. 25 | # 26 | # All configuration values have a default; values that are commented out 27 | # serve to show the default. 28 | 29 | # If extensions (or modules to document with autodoc) are in another directory, 30 | # add these directories to sys.path here. If the directory is relative to the 31 | # documentation root, use os.path.abspath to make it absolute, like shown here. 32 | # 33 | import os 34 | import sys 35 | from typing import Dict 36 | from typing import List 37 | sys.path.insert(0, os.path.abspath('../..')) 38 | 39 | 40 | # -- General configuration ------------------------------------------------ 41 | 42 | # If your documentation needs a minimal Sphinx version, state it here. 43 | # 44 | # needs_sphinx = '1.0' 45 | 46 | # Add any Sphinx extension module names here, as strings. They can be 47 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 48 | # ones. 49 | extensions = [ 50 | 'recommonmark', 51 | 'sphinx.ext.autodoc', 52 | 'sphinx.ext.coverage', 53 | 'sphinx.ext.doctest', 54 | 'sphinx.ext.githubpages', 55 | 'sphinx.ext.todo', 56 | 'sphinx.ext.viewcode', 57 | 'sphinx_markdown_tables', 58 | ] 59 | 60 | # Add any paths that contain templates here, relative to this directory. 61 | templates_path = ['_templates'] 62 | 63 | # The suffix(es) of source filenames. 64 | # You can specify multiple suffix as a list of string: 65 | # 66 | source_suffix = ['.rst', '.md'] 67 | # source_suffix = '.rst' 68 | 69 | # The master toctree document. 70 | master_doc = 'index' 71 | 72 | # General information about the project. 73 | project = 'ros_cross_compile' 74 | copyright = '2019, ROS 2 Tooling Working Group' # NOQA 75 | author = 'AWS Robomaker' 76 | 77 | # The version info for the project you're documenting, acts as replacement for 78 | # |version| and |release|, also used in various other places throughout the 79 | # built documents. 80 | # 81 | # The short X.Y version. 82 | version = '0.0.1' 83 | # The full version, including alpha/beta/rc tags. 84 | release = '0.0.1' 85 | 86 | # The language for content autogenerated by Sphinx. Refer to documentation 87 | # for a list of supported languages. 88 | # 89 | # This is also used if you do content translation via gettext catalogs. 90 | # Usually you set "language" from the command line for these cases. 91 | language = None 92 | 93 | # List of patterns, relative to source directory, that match files and 94 | # directories to ignore when looking for source files. 95 | # This patterns also effect to html_static_path and html_extra_path 96 | exclude_patterns = [] # type: List[str] 97 | 98 | # The name of the Pygments (syntax highlighting) style to use. 99 | pygments_style = 'sphinx' 100 | 101 | # If true, `todo` and `todoList` produce output, else they produce nothing. 102 | todo_include_todos = True 103 | 104 | 105 | # -- Options for HTML output ---------------------------------------------- 106 | 107 | # The theme to use for HTML and HTML Help pages. See the documentation for 108 | # a list of builtin themes. 109 | # 110 | html_theme = 'alabaster' 111 | 112 | # Theme options are theme-specific and customize the look and feel of a theme 113 | # further. For a list of options available for each theme, see the 114 | # documentation. 115 | # 116 | html_theme_options = { 117 | 'description': 'A cross-compiation tool for ROS 2', 118 | 'fixed_sidebar': True 119 | } 120 | 121 | # Add any paths that contain custom static files (such as style sheets) here, 122 | # relative to this directory. They are copied after the builtin static files, 123 | # so a file named "default.css" will overwrite the builtin "default.css". 124 | # html_static_path = ['_static'] 125 | 126 | # Custom sidebar templates, must be a dictionary that maps document names 127 | # to template names. 128 | # 129 | # This is required for the alabaster theme 130 | # refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars 131 | html_sidebars = { 132 | '**': [ 133 | 'about.html', 134 | 'searchbox.html', 135 | 'navigation.html', 136 | ] 137 | } 138 | 139 | 140 | # -- Options for HTMLHelp output ------------------------------------------ 141 | 142 | # Output file base name for HTML help builder. 143 | htmlhelp_basename = 'ros_cross_compiledoc' 144 | 145 | 146 | # -- Options for LaTeX output --------------------------------------------- 147 | 148 | latex_elements = { 149 | # The paper size ('letterpaper' or 'a4paper'). 150 | # 151 | # 'papersize': 'letterpaper', 152 | 153 | # The font size ('10pt', '11pt' or '12pt'). 154 | # 155 | # 'pointsize': '10pt', 156 | 157 | # Additional stuff for the LaTeX preamble. 158 | # 159 | # 'preamble': '', 160 | 161 | # Latex figure (float) alignment 162 | # 163 | # 'figure_align': 'htbp', 164 | } # type: Dict[str, str] 165 | 166 | # Grouping the document tree into LaTeX files. List of tuples 167 | # (source start file, target name, title, 168 | # author, documentclass [howto, manual, or own class]). 169 | latex_documents = [ 170 | (master_doc, 'ros_cross_compile.tex', 'ros_cross_compile Documentation', 171 | 'AWS Robomaker', 'manual'), 172 | ] 173 | 174 | 175 | # -- Options for manual page output --------------------------------------- 176 | 177 | # One entry per manual page. List of tuples 178 | # (source start file, name, description, authors, manual section). 179 | man_pages = [ 180 | (master_doc, 'ros_cross_compile', 'ros_cross_compile Documentation', 181 | [author], 1) 182 | ] 183 | 184 | 185 | # -- Options for Texinfo output ------------------------------------------- 186 | 187 | # Grouping the document tree into Texinfo files. List of tuples 188 | # (source start file, target name, title, author, 189 | # dir menu entry, description, category) 190 | texinfo_documents = [ 191 | (master_doc, 'ros_cross_compile', 'ros_cross_compile Documentation', 192 | author, 'ros_cross_compile', 'One line description of project.', 193 | 'Miscellaneous'), 194 | ] 195 | -------------------------------------------------------------------------------- /docs/source/index.md: -------------------------------------------------------------------------------- 1 | ../../README.md -------------------------------------------------------------------------------- /docs/sphinx_build_symlink/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ros-tooling/cross_compile/1cedd079950bb184b8ae76198a1b92a8a8229912/docs/sphinx_build_symlink/__init__.py -------------------------------------------------------------------------------- /docs/sphinx_build_symlink/setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | """ 17 | Looks for sphinx-build in lib/ and creates a symlink in bin/. 18 | 19 | This workaround mitigates an issue in readthedocs with sphinx where the 20 | sphinx-build binary is installed in a directory which is not listed in PATH, 21 | causing the documentation generation to fail (failed to open sphinx-build). 22 | """ 23 | 24 | import os 25 | import pathlib 26 | 27 | from setuptools import setup 28 | 29 | 30 | sphinx_build = next(pathlib.Path('/').glob('**/sphinx-build')) 31 | 32 | # Looking for the venv bin directory. The home directory is 33 | # chosen as a starting point, because it excludes system bin 34 | # packages, and the venv is a subdirectory of the home dir. 35 | bin_directory = next(pathlib.Path.home().glob('**/bin')) 36 | 37 | try: 38 | os.symlink(str(sphinx_build), str(bin_directory / 'sphinx-build')) 39 | except FileExistsError: 40 | # readthedocs re-use venvs acrosss multiple builds, which means that, 41 | # sometimes, the sphinx-build will have been previously created. 42 | pass 43 | 44 | package_name = 'sphinx_build_symlink' 45 | 46 | setup( 47 | name=package_name, 48 | version='0.1.0', 49 | ) 50 | -------------------------------------------------------------------------------- /ros_cross_compile/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Cross Compile package.""" 15 | -------------------------------------------------------------------------------- /ros_cross_compile/__main__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | from __future__ import absolute_import 15 | 16 | import sys 17 | 18 | from ros_cross_compile.ros_cross_compile import main as _main 19 | 20 | if __name__ == '__main__': 21 | sys.exit(_main()) 22 | -------------------------------------------------------------------------------- /ros_cross_compile/builders.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import logging 15 | import os 16 | from pathlib import Path 17 | 18 | from ros_cross_compile.data_collector import DataCollector 19 | from ros_cross_compile.docker_client import DockerClient 20 | from ros_cross_compile.pipeline_stages import PipelineStage 21 | from ros_cross_compile.pipeline_stages import PipelineStageOptions 22 | from ros_cross_compile.platform import Platform 23 | 24 | logging.basicConfig(level=logging.INFO) 25 | logger = logging.getLogger(__name__) 26 | 27 | 28 | def run_emulated_docker_build( 29 | docker_client: DockerClient, 30 | platform: Platform, 31 | workspace_path: Path 32 | ) -> None: 33 | """ 34 | Spin up a sysroot docker container and run an emulated build inside. 35 | 36 | :param docker_client: Preconfigured to run Docker images. 37 | :param platform: Information about the target platform. 38 | :param workspace: Absolute path to the user's source workspace. 39 | """ 40 | docker_client.run_container( 41 | image_name=platform.sysroot_image_tag, 42 | environment={ 43 | 'OWNER_USER': str(os.getuid()), 44 | 'ROS_DISTRO': platform.ros_distro, 45 | 'TARGET_ARCH': platform.arch, 46 | }, 47 | volumes={ 48 | workspace_path: '/ros_ws', 49 | }, 50 | ) 51 | 52 | 53 | class EmulatedDockerBuildStage(PipelineStage): 54 | """ 55 | This stage spins up a docker container and runs the emulated build with it. 56 | 57 | Uses the sysroot image from the previous stage. 58 | """ 59 | 60 | def __init__(self): 61 | super().__init__('emulated_build') 62 | 63 | def __call__( 64 | self, 65 | platform: Platform, 66 | docker_client: DockerClient, 67 | ros_workspace_dir: Path, 68 | options: PipelineStageOptions, 69 | data_collector: DataCollector 70 | ): 71 | run_emulated_docker_build(docker_client, platform, ros_workspace_dir) 72 | -------------------------------------------------------------------------------- /ros_cross_compile/data_collector.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Classes for time series data collection and writing said data to a file.""" 16 | 17 | from contextlib import contextmanager 18 | from datetime import datetime 19 | from enum import Enum 20 | import json 21 | from pathlib import Path 22 | import time 23 | from typing import Dict, List, NamedTuple, Union 24 | 25 | from ros_cross_compile.platform import Platform 26 | 27 | 28 | INTERNALS_DIR = 'cc_internals' 29 | 30 | 31 | Datum = NamedTuple('Datum', [('name', str), 32 | ('value', Union[int, float]), 33 | ('unit', str), 34 | ('timestamp', float), 35 | ('complete', bool)]) 36 | 37 | 38 | class Units(Enum): 39 | Seconds = 'Seconds' 40 | Bytes = 'Bytes' 41 | 42 | 43 | class DataCollector: 44 | """Provides an interface to collect time series data.""" 45 | 46 | def __init__(self): 47 | self._data = [] 48 | 49 | @property 50 | def data(self): 51 | return self._data 52 | 53 | def add_datum(self, new_datum: Datum): 54 | self._data.append(new_datum) 55 | 56 | @contextmanager 57 | def timer(self, name: str): 58 | """Provide an interface to time a statement's duration with a 'with'.""" 59 | start = time.monotonic() 60 | complete = False 61 | try: 62 | yield 63 | complete = True 64 | finally: 65 | elapsed = time.monotonic() - start 66 | time_metric = Datum('{}-time'.format(name), elapsed, 67 | Units.Seconds.value, time.time(), complete) 68 | self.add_datum(time_metric) 69 | 70 | def add_size(self, name: str, size: int): 71 | """Provide an interface to add collected Docker image sizes.""" 72 | size_metric = Datum('{}-size'.format(name), size, 73 | Units.Bytes.value, time.time(), True) 74 | self.add_datum(size_metric) 75 | 76 | 77 | class DataWriter: 78 | """Provides an interface to write collected data to a file.""" 79 | 80 | def __init__(self, ros_workspace_dir: Path, 81 | output_file): 82 | """Configure path for writing data.""" 83 | self._write_path = Path(str(ros_workspace_dir)) / Path(INTERNALS_DIR) / Path('metrics') 84 | self._write_path.mkdir(parents=True, exist_ok=True) 85 | self.write_file = self._write_path / output_file 86 | 87 | def print_helper(self, data_to_print: List[Datum]): 88 | print('--------------------------------- Collected Data ---------------------------------') 89 | print('=================================================================================') 90 | for datum in data_to_print: 91 | # readable_time = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(datum['timestamp'])) 92 | readable_time = datetime.utcfromtimestamp(datum.timestamp).isoformat() 93 | if datum.unit == Units.Seconds.value: 94 | print('{:>12} | {:>35}: {:.2f} {}'.format(readable_time, datum.name, 95 | datum.value, datum.unit), 96 | end='') 97 | else: 98 | print('{:>12} | {:>35}: {} {}'.format(readable_time, datum.name, 99 | datum.value, datum.unit), 100 | end='') 101 | if datum.complete: 102 | print('\n') 103 | else: 104 | print(' {}'.format('incomplete')) 105 | 106 | def serialize_to_cloudwatch(self, data: List[Datum], platform: Platform) -> List[Dict]: 107 | """ 108 | Serialize collected datums to fit the AWS Cloudwatch publishing format. 109 | 110 | To get an idea of what the Cloudwatch formatting is like, refer to 111 | https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ 112 | cloudwatch/put-metric-data.html. 113 | """ 114 | def serialize_helper(datum: Datum) -> Dict: 115 | return { 116 | 'MetricName': datum.name, 117 | 'Value': datum.value, 118 | 'Unit': datum.unit, 119 | 'Timestamp': datum.timestamp, 120 | 'Dimensions': [{'Name': 'Complete', 'Value': str(datum.complete)}, 121 | {'Name': 'Architecture', 'Value': platform.arch}, 122 | {'Name': 'OS', 'Value': platform.os_name}, 123 | {'Name': 'ROS Distro', 'Value': platform.ros_distro}] 124 | } 125 | 126 | return [serialize_helper(d) for d in data] 127 | 128 | def write(self, data_collector: DataCollector, platform: Platform, print_data: bool): 129 | """ 130 | Write collected datums to a file. 131 | 132 | Before writing, however, we convert each datum to a dictionary, 133 | so that they are conveniently 'dumpable' into a JSON file. 134 | """ 135 | if print_data: 136 | self.print_helper(data_collector.data) 137 | with self.write_file.open('w') as f: 138 | data_to_dump = self.serialize_to_cloudwatch(data_collector.data, platform) 139 | json.dump(list(data_to_dump), f, sort_keys=True, indent=4) 140 | -------------------------------------------------------------------------------- /ros_cross_compile/dependencies.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import logging 16 | import os 17 | from pathlib import Path 18 | from typing import List 19 | from typing import Optional 20 | 21 | from ros_cross_compile.data_collector import DataCollector 22 | from ros_cross_compile.docker_client import DockerClient 23 | from ros_cross_compile.pipeline_stages import PipelineStage 24 | from ros_cross_compile.pipeline_stages import PipelineStageOptions 25 | from ros_cross_compile.platform import Platform 26 | from ros_cross_compile.sysroot_creator import build_internals_dir 27 | 28 | logging.basicConfig(level=logging.INFO) 29 | logger = logging.getLogger('Rosdep Gatherer') 30 | 31 | 32 | CUSTOM_SETUP = '/usercustom/rosdep_setup' 33 | CUSTOM_DATA = '/usercustom/custom-data' 34 | _IMG_NAME = 'ros_cross_compile:rosdep' 35 | 36 | 37 | def rosdep_install_script(platform: Platform) -> Path: 38 | """Construct relative path of the script that installs rosdeps into the sysroot image.""" 39 | return build_internals_dir(platform) / 'install_rosdeps.sh' 40 | 41 | 42 | def gather_rosdeps( 43 | docker_client: DockerClient, 44 | platform: Platform, 45 | workspace: Path, 46 | skip_rosdep_keys: List[str] = [], 47 | custom_script: Optional[Path] = None, 48 | custom_data_dir: Optional[Path] = None, 49 | ) -> None: 50 | """ 51 | Run the rosdep Docker image, which outputs a script for dependency installation. 52 | 53 | :param docker_client: Will be used to run the container 54 | :param platform: The name of the image produced by `build_rosdep_image` 55 | :param workspace: Absolute path to the colcon source workspace. 56 | :param custom_script: Optional absolute path of a script that does custom setup for rosdep 57 | :param custom_data_dir: Optional absolute path of a directory containing custom data for setup 58 | :return None 59 | """ 60 | out_path = rosdep_install_script(platform) 61 | 62 | logger.info('Building rosdep collector image: %s', _IMG_NAME) 63 | if platform.ros_distro == 'melodic': 64 | docker_client.build_image( 65 | dockerfile_name='rosdep.Dockerfile', 66 | tag=_IMG_NAME, 67 | ) 68 | else: 69 | docker_client.build_image( 70 | dockerfile_name='rosdep_focal.Dockerfile', 71 | tag=_IMG_NAME, 72 | ) 73 | 74 | logger.info('Running rosdep collector image on workspace {}'.format(workspace)) 75 | volumes = { 76 | workspace: '/ws', 77 | } 78 | if custom_script: 79 | volumes[custom_script] = CUSTOM_SETUP 80 | if custom_data_dir: 81 | volumes[custom_data_dir] = CUSTOM_DATA 82 | 83 | docker_client.run_container( 84 | image_name=_IMG_NAME, 85 | environment={ 86 | 'CUSTOM_SETUP': CUSTOM_SETUP, 87 | 'OUT_PATH': str(out_path), 88 | 'OWNER_USER': str(os.getuid()), 89 | 'ROSDISTRO': platform.ros_distro, 90 | 'SKIP_ROSDEP_KEYS': ' '.join(skip_rosdep_keys), 91 | 'COLCON_DEFAULTS_FILE': 'defaults.yaml', 92 | 'TARGET_OS': '{}:{}'.format(platform.os_name, platform.os_distro), 93 | }, 94 | volumes=volumes, 95 | ) 96 | 97 | 98 | def assert_install_rosdep_script_exists( 99 | ros_workspace_dir: Path, 100 | platform: Platform 101 | ) -> bool: 102 | install_rosdep_script_path = ros_workspace_dir / rosdep_install_script(platform) 103 | if not install_rosdep_script_path.is_file(): 104 | raise RuntimeError( 105 | 'Rosdep installation script has never been created, you need to run this without ' 106 | 'skipping rosdep collection at least once.') 107 | return True 108 | 109 | 110 | class CollectDependencyListStage(PipelineStage): 111 | """ 112 | This stage determines what external dependencies are needed for building. 113 | 114 | It outputs a script into the internals directory that will install those 115 | dependencies for the target platform. 116 | """ 117 | 118 | def __init__(self): 119 | super().__init__('gather_rosdeps') 120 | 121 | def __call__( 122 | self, 123 | platform: Platform, 124 | docker_client: DockerClient, 125 | ros_workspace_dir: Path, 126 | options: PipelineStageOptions, 127 | data_collector: DataCollector 128 | ): 129 | """ 130 | Run the inspection and output the dependency installation script. 131 | 132 | Also recovers the size of the docker image generated. 133 | 134 | :raises RuntimeError if the step was skipped when no dependency script has been 135 | previously generated 136 | """ 137 | gather_rosdeps( 138 | docker_client=docker_client, 139 | platform=platform, 140 | workspace=ros_workspace_dir, 141 | skip_rosdep_keys=options.skip_rosdep_keys, 142 | custom_script=options.custom_script, 143 | custom_data_dir=options.custom_data_dir) 144 | assert_install_rosdep_script_exists(ros_workspace_dir, platform) 145 | 146 | img_size = docker_client.get_image_size(_IMG_NAME) 147 | data_collector.add_size(self.name, img_size) 148 | -------------------------------------------------------------------------------- /ros_cross_compile/docker/build_workspace.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euxo pipefail 3 | 4 | cleanup() { 5 | chown -R "${OWNER_USER}" . 6 | } 7 | 8 | trap 'cleanup' EXIT 9 | 10 | mkdir -p /opt/ros/"${ROS_DISTRO}" 11 | touch /opt/ros/"${ROS_DISTRO}"/setup.bash 12 | 13 | set +ux 14 | # shellcheck source=/dev/null 15 | source /opt/ros/"${ROS_DISTRO}"/setup.bash 16 | if [ -f /custom-data/setup.bash ]; then 17 | # shellcheck source=/dev/null 18 | source /custom-data/setup.bash 19 | fi 20 | set -ux 21 | colcon build --mixin "${TARGET_ARCH}"-docker \ 22 | --build-base build_"${TARGET_ARCH}" \ 23 | --install-base install_"${TARGET_ARCH}" 24 | 25 | # Runs user-provided post-build logic (file is present and empty if it wasn't specified) 26 | /user-custom-post-build 27 | -------------------------------------------------------------------------------- /ros_cross_compile/docker/gather_rosdeps.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euxo pipefail 3 | 4 | if [ ! -d ./src ]; then 5 | echo "No src/ directory found at $(pwd), did you remember to mount your workspace?" 6 | exit 1 7 | fi 8 | 9 | if [ -f "${CUSTOM_SETUP}" ]; then 10 | chmod +x "${CUSTOM_SETUP}" 11 | pushd "$(dirname "${CUSTOM_SETUP}")" 12 | "${CUSTOM_SETUP}" 13 | popd 14 | fi 15 | 16 | out_dir=$(dirname "${OUT_PATH}") 17 | mkdir -p "${out_dir}" 18 | 19 | # Note: users are allowed to build against EOL distros! There are just no updates to them, 20 | # but they should continue working 21 | rosdep update --include-eol-distros 22 | 23 | cat > "${OUT_PATH}" <> /tmp/all-deps.sh 40 | 41 | # Grep returns nonzero when there are no matches and we exit on error in this script, 42 | # but it's ok if there are no matches, so use "|| true" to always return 0 43 | # Find the non-apt lines and move them as-is to the final script 44 | grep -v "apt-get install -y" /tmp/all-deps.sh >> "${OUT_PATH}" || true 45 | 46 | # Find all apt-get lines from the rosdep output 47 | # As an optimization, we will combine all such commands into a single command to save time 48 | grep "apt-get install -y" /tmp/all-deps.sh > /tmp/apt-deps.sh || true 49 | # awk notes: 50 | # "apt-get", "install", "-y", package_name is the fourth column 51 | # ORS=' ' makes the output space-separated instead of newline-separated output 52 | echo "apt-get install -y $(awk '{print $4}' ORS=' ' < /tmp/apt-deps.sh)" >> "${OUT_PATH}" 53 | 54 | chmod +x "${OUT_PATH}" 55 | chown -R "${OWNER_USER}" "${out_dir}" 56 | -------------------------------------------------------------------------------- /ros_cross_compile/docker/rosdep.Dockerfile: -------------------------------------------------------------------------------- 1 | # This Dockerfile describes a simple image with rosdep installed. 2 | # When `run`, it outputs a script for installing dependencies for a given workspace 3 | # Requirements: 4 | # * mount a colcon workspace at /ws 5 | # * see gather_rosdeps.sh for all-caps required input environment 6 | FROM ubuntu:bionic 7 | 8 | RUN apt-get update && apt-get install --no-install-recommends -y \ 9 | gnupg \ 10 | && rm -rf /var/lib/apt/lists/* 11 | RUN apt-key adv --keyserver 'hkp://keyserver.ubuntu.com:80' --recv-key C1CF6E31E6BADE8868B172B4F42ED6FBAB17C654 12 | RUN echo "deb http://packages.ros.org/ros/ubuntu bionic main" > /etc/apt/sources.list.d/ros-latest.list 13 | RUN apt-get update && apt-get install --no-install-recommends -y \ 14 | python-rosdep \ 15 | python3-colcon-common-extensions \ 16 | && rm -rf /var/lib/apt/lists/* 17 | 18 | RUN rosdep init 19 | COPY gather_rosdeps.sh /root/ 20 | RUN mkdir -p /ws 21 | WORKDIR /ws 22 | ENTRYPOINT ["/root/gather_rosdeps.sh"] 23 | -------------------------------------------------------------------------------- /ros_cross_compile/docker/rosdep_focal.Dockerfile: -------------------------------------------------------------------------------- 1 | # This Dockerfile describes a simple image with rosdep installed. 2 | # When `run`, it outputs a script for installing dependencies for a given workspace 3 | # Requirements: 4 | # * mount a colcon workspace at /ws 5 | # * see gather_rosdeps.sh for all-caps required input environment 6 | FROM ubuntu:focal 7 | 8 | RUN apt-get update && apt-get install --no-install-recommends -y \ 9 | gnupg \ 10 | && rm -rf /var/lib/apt/lists/* 11 | RUN apt-key adv --keyserver 'hkp://keyserver.ubuntu.com:80' --recv-key C1CF6E31E6BADE8868B172B4F42ED6FBAB17C654 12 | RUN echo "deb http://packages.ros.org/ros/ubuntu focal main" > /etc/apt/sources.list.d/ros-latest.list 13 | RUN apt-get update && apt-get install --no-install-recommends -y \ 14 | python3-rosdep \ 15 | python3-colcon-common-extensions \ 16 | && rm -rf /var/lib/apt/lists/* 17 | 18 | RUN rosdep init 19 | COPY gather_rosdeps.sh /root/ 20 | RUN mkdir -p /ws 21 | WORKDIR /ws 22 | ENTRYPOINT ["/root/gather_rosdeps.sh"] -------------------------------------------------------------------------------- /ros_cross_compile/docker/runtime.Dockerfile: -------------------------------------------------------------------------------- 1 | # This dockerfile creates an image containing the runtime environment for the workspace. 2 | # For now this is a thin layer on top of the sysroot/buildroot environment used to build. 3 | # It re-sets the entrypoint to a shell instead of a build script, and copies in the install. 4 | # The eventual plan for it is to only contain `` of the workspace 5 | 6 | ARG BASE_IMAGE 7 | FROM $BASE_IMAGE 8 | 9 | WORKDIR /ros_ws 10 | 11 | ARG INSTALL_PATH 12 | COPY $INSTALL_PATH/ install 13 | 14 | RUN echo "source /ros_ws/install/setup.bash" >> ~/.bashrc 15 | ENTRYPOINT /bin/bash 16 | -------------------------------------------------------------------------------- /ros_cross_compile/docker/sysroot.Dockerfile: -------------------------------------------------------------------------------- 1 | # This file describes an image that has everything necessary installed to build a target ROS workspace 2 | # It uses QEmu user-mode emulation to perform dependency installation and build 3 | # Assumptions: qemu-user-static directory is present in docker build context 4 | 5 | ARG BASE_IMAGE 6 | FROM ${BASE_IMAGE} 7 | 8 | ARG ROS_VERSION 9 | 10 | SHELL ["/bin/bash", "-c"] 11 | 12 | COPY bin/* /usr/bin/ 13 | 14 | # Set timezone 15 | RUN echo 'Etc/UTC' > /etc/timezone && \ 16 | ln -sf /usr/share/zoneinfo/Etc/UTC /etc/localtime 17 | 18 | RUN apt-get update && apt-get install --no-install-recommends -y \ 19 | tzdata \ 20 | locales \ 21 | && rm -rf /var/lib/apt/lists/* 22 | 23 | # Set locale 24 | RUN echo 'en_US.UTF-8 UTF-8' >> /etc/locale.gen && \ 25 | locale-gen && \ 26 | update-locale LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 27 | ENV LANG en_US.UTF-8 28 | ENV LC_ALL C.UTF-8 29 | 30 | # Add the ros apt repo 31 | RUN apt-get update && apt-get install --no-install-recommends -y \ 32 | ca-certificates \ 33 | curl \ 34 | gnupg \ 35 | lsb-release \ 36 | && rm -rf /var/lib/apt/lists/* 37 | RUN if [[ "${ROS_VERSION}" == "ros2" ]]; then \ 38 | curl -sSL https://raw.githubusercontent.com/ros/rosdistro/master/ros.key -o /usr/share/keyrings/ros-archive-keyring.gpg; \ 39 | echo "deb [signed-by=/usr/share/keyrings/ros-archive-keyring.gpg] http://packages.ros.org/${ROS_VERSION}/ubuntu `lsb_release -cs` main" | \ 40 | tee /etc/apt/sources.list.d/ros2.list > /dev/null; \ 41 | else \ 42 | curl -s https://raw.githubusercontent.com/ros/rosdistro/master/ros.asc | apt-key add - ; \ 43 | echo "deb http://packages.ros.org/${ROS_VERSION}/ubuntu `lsb_release -cs` main" \ 44 | > /etc/apt/sources.list.d/${ROS_VERSION}-latest.list; \ 45 | fi 46 | 47 | # ROS dependencies 48 | RUN apt-get update && apt-get install --no-install-recommends -y \ 49 | build-essential \ 50 | cmake \ 51 | python3-colcon-common-extensions \ 52 | python3-colcon-mixin \ 53 | python3-dev \ 54 | python3-pip \ 55 | && rm -rf /var/lib/apt/lists/* 56 | 57 | RUN python3 -m pip install -U \ 58 | setuptools 59 | 60 | # Install some pip packages needed for testing ROS 2 61 | RUN if [[ "${ROS_VERSION}" == "ros2" ]]; then \ 62 | python3 -m pip install -U \ 63 | flake8 \ 64 | flake8-blind-except \ 65 | flake8-builtins \ 66 | flake8-class-newline \ 67 | flake8-comprehensions \ 68 | flake8-deprecated \ 69 | flake8-docstrings \ 70 | flake8-import-order \ 71 | flake8-quotes \ 72 | pytest-repeat \ 73 | pytest-rerunfailures \ 74 | pytest \ 75 | pytest-cov \ 76 | pytest-runner \ 77 | ; fi 78 | 79 | # Install Fast-RTPS dependencies for ROS 2 80 | RUN if [[ "${ROS_VERSION}" == "ros2" ]]; then \ 81 | apt-get update && apt-get install --no-install-recommends -y \ 82 | libasio-dev \ 83 | libtinyxml2-dev \ 84 | && rm -rf /var/lib/apt/lists/* \ 85 | ; fi 86 | 87 | # Run arbitrary user setup (copy data and run script) 88 | COPY user-custom-data/ custom-data/ 89 | COPY user-custom-setup . 90 | RUN chmod +x ./user-custom-setup && \ 91 | ./user-custom-setup && \ 92 | rm -rf /var/lib/apt/lists/* 93 | 94 | # Use generated rosdep installation script 95 | COPY install_rosdeps.sh . 96 | RUN chmod +x install_rosdeps.sh 97 | RUN export DEBIAN_FRONTEND=noninteractive && apt-get update && \ 98 | ./install_rosdeps.sh && \ 99 | rm -rf /var/lib/apt/lists/* 100 | 101 | # Copy colcon defaults config and set COLCON_DEFAULTS_FILE 102 | COPY defaults.yaml /root 103 | ENV COLCON_DEFAULTS_FILE=/root/defaults.yaml 104 | 105 | # Set up build tools for the workspace 106 | COPY mixins/ mixins/ 107 | RUN colcon mixin add cc_mixin file://$(pwd)/mixins/index.yaml && colcon mixin update cc_mixin 108 | # In case the workspace did not actually install any dependencies, add these for uniformity 109 | COPY build_workspace.sh /root 110 | WORKDIR /ros_ws 111 | COPY user-custom-post-build / 112 | RUN chmod +x /user-custom-post-build 113 | ENTRYPOINT ["/root/build_workspace.sh"] 114 | -------------------------------------------------------------------------------- /ros_cross_compile/docker_client.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import logging 15 | from pathlib import Path 16 | from typing import Dict 17 | from typing import Optional 18 | 19 | import docker 20 | from docker.utils import kwargs_from_env as docker_kwargs_from_env 21 | 22 | 23 | logging.basicConfig(level=logging.INFO) 24 | logger = logging.getLogger('Docker Client') 25 | 26 | 27 | class DockerClient: 28 | """Simplified Docker API for this package's usage patterns.""" 29 | 30 | def __init__( 31 | self, 32 | disable_cache: bool = False, 33 | default_docker_dir: Optional[Path] = None 34 | ): 35 | """ 36 | Construct the DockerClient. 37 | 38 | :param disable_cache: If True, disable the Docker cache when building images. 39 | """ 40 | self._client = docker.from_env() 41 | self._disable_cache = disable_cache 42 | self._default_docker_dir = str(default_docker_dir or Path(__file__).parent / 'docker') 43 | 44 | def build_image( 45 | self, 46 | dockerfile_name: str, 47 | tag: str, 48 | dockerfile_dir: Optional[Path] = None, 49 | buildargs: Optional[Dict[str, str]] = None, 50 | ) -> None: 51 | """ 52 | Build a Docker image from a Dockerfile. 53 | 54 | :param dockerfile_dir: Absolute path to directory where Dockerfile can be found, 55 | defaults to the 'docker' directory in this package. 56 | :param dockerfile_name: The name of the Dockerfile to build. 57 | :param tag: What tag to give the created image. 58 | :param buildargs: Optional dictionary of str->str to set arguments for the build. 59 | :return None 60 | :raises docker.errors.BuildError: on build error 61 | """ 62 | # Use low-level API to expose logs for image building 63 | docker_api = docker.APIClient(**docker_kwargs_from_env()) 64 | log_generator = docker_api.build( 65 | path=str(dockerfile_dir) if dockerfile_dir else self._default_docker_dir, 66 | dockerfile=dockerfile_name, 67 | tag=tag, 68 | buildargs=buildargs, 69 | quiet=False, 70 | nocache=self._disable_cache, 71 | decode=True, 72 | network_mode='host' 73 | ) 74 | self._process_build_log(log_generator) 75 | 76 | def _process_build_log(self, log_generator) -> None: 77 | for chunk in log_generator: 78 | # There are two outputs we want to capture, stream and error. 79 | # We also process line breaks. 80 | error_line = chunk.get('error', None) 81 | if error_line: 82 | logger.exception( 83 | 'Error building Docker image. The follow error was caught:\n' + error_line) 84 | raise docker.errors.BuildError(error_line) 85 | line = chunk.get('stream', '') 86 | line = line.rstrip() 87 | if line: 88 | logger.info(line) 89 | 90 | def run_container( 91 | self, 92 | image_name: str, 93 | command: Optional[str] = None, 94 | environment: Dict[str, str] = {}, 95 | volumes: Dict[Path, str] = {}, 96 | container_name: Optional[str] = None, 97 | ) -> None: 98 | """ 99 | Run a container of an existing image. 100 | 101 | :param image_name: Name of the image to run. 102 | :param command: Optional command to run on the container 103 | :param environment: Map of environment variable names to values. 104 | :param volumes: Map of absolute path to a host directory, to str of mount destination. 105 | :raises docker.errors.ContainerError if container run has nonzero exit code 106 | :return None 107 | """ 108 | docker_volumes = { 109 | str(src): { 110 | 'bind': dest, 111 | 'mode': 'rw', 112 | } 113 | for src, dest in volumes.items() 114 | } 115 | # Note that the `run` kwarg `stream` is not available 116 | # in the version of dockerpy that we are using, so we must detach to live-stream logs 117 | # Do not `remove` so that the container can be queried for its exit code after finishing 118 | container = self._client.containers.run( 119 | image=image_name, 120 | name=container_name, 121 | command=command, 122 | environment=environment, 123 | volumes=docker_volumes, 124 | detach=True, 125 | network_mode='host', 126 | ) 127 | try: 128 | logs = container.logs(stream=True) 129 | for line in logs: 130 | logger.info(line.decode('utf-8').rstrip()) 131 | exit_code = container.wait() 132 | finally: 133 | container.stop() 134 | container.remove() 135 | 136 | if docker.version_info[0] >= 3: 137 | exit_code = exit_code['StatusCode'] 138 | 139 | if exit_code: 140 | raise docker.errors.ContainerError( 141 | image_name, exit_code, '', image_name, 'See above ^') 142 | 143 | def get_image_size(self, img_name: str) -> int: 144 | return self._client.images.get(img_name).attrs['Size'] 145 | -------------------------------------------------------------------------------- /ros_cross_compile/mixins/cross-compile.mixin: -------------------------------------------------------------------------------- 1 | build: 2 | aarch64-docker: {} 3 | aarch64-linux: 4 | cmake-args: 5 | - "-DCMAKE_SYSTEM_NAME=Linux" 6 | - "-DCMAKE_SYSTEM_VERSION=1" 7 | - "-DCMAKE_SYSTEM_PROCESSOR=aarch64" 8 | - "-DCMAKE_C_COMPILER=/usr/bin/aarch64-linux-gnu-gcc" 9 | - "-DCMAKE_CXX_COMPILER=/usr/bin/aarch64-linux-gnu-g++" 10 | - "-DCMAKE_SYSROOT=$CC_ROOT/sysroot" 11 | - "-DCMAKE_TRY_COMPILE_TARGET_TYPE=STATIC_LIBRARY" 12 | - "-DCMAKE_FIND_ROOT_PATH=$CC_ROOT/sysroot/root_path:$CC_ROOT/install" 13 | - "-DCMAKE_INSTALL_RPATH=$CC_ROOT/sysroot/opt/ros/$ROS_DISTRO/lib" 14 | - "-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=TRUE" 15 | - "-DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER" 16 | - "-DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=ONLY" 17 | - "-DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY" 18 | - "-DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY" 19 | - "-DPYTHON_SOABI=cpython-36m-aarch64-linux-gnu" 20 | - "-DTHREADS_PTHREAD_ARG=0" 21 | - "--no-warn-unused-cli" 22 | arm-linux: 23 | cmake-args: 24 | - "-DCMAKE_SYSTEM_NAME=Linux" 25 | - "-DCMAKE_SYSTEM_VERSION=1" 26 | - "-DCMAKE_SYSTEM_PROCESSOR=armv7l" 27 | - "-DCMAKE_C_COMPILER=/usr/bin/arm-linux-gnueabi-gcc" 28 | - "-DCMAKE_CXX_COMPILER=/usr/bin/arm-linux-gnueabi-g++" 29 | - "-DCMAKE_SYSROOT=$CC_ROOT/sysroot" 30 | - "-DCMAKE_FIND_ROOT_PATH=$CC_ROOT/sysroot/root_path:$CC_ROOT/install" 31 | - "-DCMAKE_INSTALL_RPATH=$CC_ROOT/sysroot/opt/ros/$ROS_DISTRO/lib" 32 | - "-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=TRUE" 33 | - "-DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER" 34 | - "-DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=ONLY" 35 | - "-DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY" 36 | - "-DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY" 37 | - "-DPYTHON_SOABI=cpython-36m-arm-linux-gnueabi" 38 | - "-DTHREADS_PTHREAD_ARG=0" 39 | - "--no-warn-unused-cli" 40 | armhf-docker: 41 | cmake-args: 42 | - "-DCMAKE_C_FLAGS=-Wno-psabi" 43 | - "-DCMAKE_CXX_FLAGS=-Wno-psabi" 44 | - "--no-warn-unused-cli" 45 | armhf-linux: 46 | cmake-args: 47 | - "-DCMAKE_SYSTEM_NAME=Linux" 48 | - "-DCMAKE_SYSTEM_VERSION=1" 49 | - "-DCMAKE_SYSTEM_PROCESSOR=arm" 50 | - "-DCMAKE_C_COMPILER=/usr/bin/arm-linux-gnueabihf-gcc" 51 | - "-DCMAKE_CXX_COMPILER=/usr/bin/arm-linux-gnueabihf-g++" 52 | - "-DCMAKE_SYSROOT=$CC_ROOT/sysroot" 53 | - "-DCMAKE_C_FLAGS=-Wno-psabi" 54 | - "-DCMAKE_CXX_FLAGS=-Wno-psabi" 55 | - "-DCMAKE_FIND_ROOT_PATH=$CC_ROOT/sysroot/root_path:$CC_ROOT/install" 56 | - "-DCMAKE_INSTALL_RPATH=$CC_ROOT/sysroot/opt/ros/$ROS_DISTRO/lib" 57 | - "-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=TRUE" 58 | - "-DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER" 59 | - "-DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=ONLY" 60 | - "-DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY" 61 | - "-DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY" 62 | - "-DPYTHON_SOABI=cpython-36m-arm-linux-gnueabihf" 63 | - "-DTHREADS_PTHREAD_ARG=0" 64 | - "--no-warn-unused-cli" 65 | x86_64-docker: {} 66 | x86_64-linux: {} 67 | -------------------------------------------------------------------------------- /ros_cross_compile/mixins/index.yaml: -------------------------------------------------------------------------------- 1 | mixin: 2 | - cross-compile.mixin 3 | -------------------------------------------------------------------------------- /ros_cross_compile/pipeline_stages.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from abc import ABC, abstractmethod 16 | from pathlib import Path 17 | from typing import List, NamedTuple, Optional 18 | 19 | from ros_cross_compile.data_collector import DataCollector 20 | from ros_cross_compile.docker_client import DockerClient 21 | from ros_cross_compile.platform import Platform 22 | 23 | 24 | """ 25 | A NamedTuple that collects the customizations for each stage passed in by the user. 26 | 27 | As such, the documentation for each customization can be found by looking at the 28 | argparse options in ros_cross_compile.py. 29 | """ 30 | PipelineStageOptions = NamedTuple( 31 | 'PipelineStageOptions', 32 | [ 33 | ('skip_rosdep_keys', List[str]), 34 | ('custom_script', Optional[Path]), 35 | ('custom_data_dir', Optional[Path]), 36 | ('custom_setup_script', Optional[Path]), 37 | ('runtime_tag', Optional[str]), 38 | ]) 39 | 40 | 41 | class PipelineStage(ABC): 42 | """Interface to represent a stage of the cross compile pipeline.""" 43 | 44 | @abstractmethod 45 | def __init__(self, name): 46 | self._name = name 47 | 48 | @property 49 | def name(self): 50 | return self._name 51 | 52 | @abstractmethod 53 | def __call__( 54 | self, 55 | platform: Platform, 56 | docker_client: DockerClient, 57 | ros_workspace_dir: Path, 58 | options: PipelineStageOptions, 59 | data_collector: DataCollector 60 | ): 61 | raise NotImplementedError 62 | -------------------------------------------------------------------------------- /ros_cross_compile/platform.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import getpass 15 | from typing import NamedTuple 16 | from typing import Optional 17 | 18 | ArchNameMapping = NamedTuple('ArchNameMapping', [('docker', str), ('qemu', str)]) 19 | 20 | 21 | # NOTE: when changing any following values, update README.md Supported Targets section 22 | ARCHITECTURE_NAME_MAP = { 23 | 'armhf': ArchNameMapping(docker='arm32v7', qemu='arm'), 24 | 'aarch64': ArchNameMapping(docker='arm64v8', qemu='aarch64'), 25 | 'x86_64': ArchNameMapping(docker='', qemu='x86_64'), 26 | } 27 | SUPPORTED_ARCHITECTURES = tuple(ARCHITECTURE_NAME_MAP.keys()) 28 | 29 | SUPPORTED_ROS2_DISTROS = ('foxy', 'galactic', 'humble', 'rolling') 30 | SUPPORTED_ROS_DISTROS = ('melodic', 'noetic') 31 | 32 | ROSDISTRO_OS_MAP = { 33 | 'melodic': { 34 | 'ubuntu': 'bionic', 35 | 'debian': 'stretch', 36 | }, 37 | 'noetic': { 38 | 'ubuntu': 'focal', 39 | 'debian': 'buster', 40 | }, 41 | 'foxy': { 42 | 'ubuntu': 'focal', 43 | 'debian': 'buster', 44 | }, 45 | 'galactic': { 46 | 'ubuntu': 'focal', 47 | 'debian': 'buster', 48 | }, 49 | 'humble': { 50 | 'ubuntu': 'jammy', 51 | 'debian': 'bullseye', 52 | }, 53 | 'rolling': { 54 | 'ubuntu': 'jammy', 55 | 'debian': 'bullseye', 56 | }, 57 | } 58 | # NOTE: when changing any preceding values, update README.md Supported Targets section 59 | 60 | 61 | class Platform: 62 | """ 63 | Represents the target platform for cross compiling. 64 | 65 | Includes: 66 | * Target architecture 67 | * Target operating system 68 | * Target ROS distribution 69 | """ 70 | 71 | def __init__( 72 | self, arch: str, os_name: str, ros_distro: str, override_base_image: Optional[str] = None, 73 | ): 74 | """Initialize platform parameters.""" 75 | self._arch = arch 76 | self._ros_distro = ros_distro 77 | self._os_name = os_name 78 | 79 | try: 80 | docker_org = ARCHITECTURE_NAME_MAP[arch].docker 81 | except KeyError: 82 | raise ValueError('Unknown target architecture "{}" specified'.format(arch)) 83 | 84 | if self.ros_distro in SUPPORTED_ROS2_DISTROS: 85 | self._ros_version = 'ros2' 86 | elif self.ros_distro in SUPPORTED_ROS_DISTROS: 87 | self._ros_version = 'ros' 88 | else: 89 | raise ValueError('Unknown ROS distribution "{}" specified'.format(ros_distro)) 90 | 91 | os_map = ROSDISTRO_OS_MAP[self.ros_distro] 92 | if self.os_name not in os_map: 93 | raise ValueError( 94 | 'OS "{}" not supported for ROS distro "{}". Supported values: {} ' 95 | '(note that these are case sensitive)'.format( 96 | os_name, ros_distro, list(os_map.keys()))) 97 | 98 | if override_base_image: 99 | self._docker_target_base = override_base_image 100 | else: 101 | self._os_distro = ROSDISTRO_OS_MAP[self.ros_distro][self.os_name] 102 | native_base = '{}:{}'.format(self.os_name, self.os_distro) 103 | if docker_org: 104 | self._docker_target_base = '{}/{}'.format(docker_org, native_base) 105 | else: 106 | self._docker_target_base = native_base 107 | 108 | @property 109 | def arch(self): 110 | return self._arch 111 | 112 | @property 113 | def qemu_arch(self): 114 | return ARCHITECTURE_NAME_MAP[self.arch].qemu 115 | 116 | @property 117 | def ros_distro(self): 118 | return self._ros_distro 119 | 120 | @property 121 | def os_name(self): 122 | return self._os_name 123 | 124 | @property 125 | def os_distro(self): 126 | return self._os_distro 127 | 128 | @property 129 | def ros_version(self): 130 | return self._ros_version 131 | 132 | def __str__(self): 133 | """Return string representation of platform parameters.""" 134 | return '-'.join((self.arch, self.os_name, self.ros_distro)) 135 | 136 | @property 137 | def sysroot_image_tag(self) -> str: 138 | """Generate docker image name and tag.""" 139 | return getpass.getuser() + '/' + str(self) + ':latest' 140 | 141 | @property 142 | def target_base_image(self) -> str: 143 | """Name of the base OS Docker image for the target architecture.""" 144 | return self._docker_target_base 145 | -------------------------------------------------------------------------------- /ros_cross_compile/qemu/qemu-aarch64-static: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ros-tooling/cross_compile/1cedd079950bb184b8ae76198a1b92a8a8229912/ros_cross_compile/qemu/qemu-aarch64-static -------------------------------------------------------------------------------- /ros_cross_compile/qemu/qemu-arm-static: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ros-tooling/cross_compile/1cedd079950bb184b8ae76198a1b92a8a8229912/ros_cross_compile/qemu/qemu-arm-static -------------------------------------------------------------------------------- /ros_cross_compile/ros_cross_compile.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | """Executable for cross-compiling ROS and ROS 2 packages.""" 18 | 19 | import argparse 20 | from datetime import datetime 21 | import logging 22 | from pathlib import Path 23 | import sys 24 | from typing import List 25 | from typing import Optional 26 | 27 | from ros_cross_compile.builders import EmulatedDockerBuildStage 28 | from ros_cross_compile.data_collector import DataCollector 29 | from ros_cross_compile.data_collector import DataWriter 30 | from ros_cross_compile.dependencies import CollectDependencyListStage 31 | from ros_cross_compile.docker_client import DockerClient 32 | from ros_cross_compile.pipeline_stages import PipelineStageOptions 33 | from ros_cross_compile.platform import Platform 34 | from ros_cross_compile.platform import SUPPORTED_ARCHITECTURES 35 | from ros_cross_compile.platform import SUPPORTED_ROS2_DISTROS 36 | from ros_cross_compile.platform import SUPPORTED_ROS_DISTROS 37 | from ros_cross_compile.runtime import PackageRuntimeImageStage 38 | from ros_cross_compile.sysroot_creator import CreateSysrootStage 39 | from ros_cross_compile.sysroot_creator import prepare_docker_build_environment 40 | 41 | logging.basicConfig(level=logging.INFO) 42 | logger = logging.getLogger(__name__) 43 | 44 | _PIPELINE = [ 45 | CollectDependencyListStage(), 46 | CreateSysrootStage(), 47 | EmulatedDockerBuildStage(), 48 | PackageRuntimeImageStage(), 49 | ] 50 | 51 | 52 | def _path_if(path: Optional[str] = None) -> Optional[Path]: 53 | return Path(path) if path else None 54 | 55 | 56 | def _resolve_ros_workspace(ros_workspace_input: str) -> Path: 57 | """Allow for relative paths to be passed in as a ros workspace dir.""" 58 | ros_workspace_dir = Path(ros_workspace_input).resolve() 59 | if not (ros_workspace_dir / 'src').is_dir(): 60 | raise ValueError( 61 | 'specified workspace "{}" does not look like a colcon workspace ' 62 | '(there is no "src/" directory). cannot continue'.format(ros_workspace_dir)) 63 | 64 | return ros_workspace_dir 65 | 66 | 67 | def parse_args(args: List[str]) -> argparse.Namespace: 68 | """Parse command line arguments.""" 69 | parser = argparse.ArgumentParser( 70 | prog='ros_cross_compile', # this can be invoked from __main__.py, so be explicit 71 | description='Sysroot creator for cross compilation workflows.') 72 | parser.add_argument( 73 | 'ros_workspace', type=str, 74 | help='Path of the colcon workspace to be cross-compiled. Contains "src" directory.') 75 | parser.add_argument( 76 | '-a', '--arch', 77 | required=True, 78 | type=str, 79 | choices=SUPPORTED_ARCHITECTURES, 80 | help='Target architecture') 81 | parser.add_argument( 82 | '-d', '--rosdistro', 83 | required=False, 84 | type=str, 85 | default='foxy', 86 | choices=SUPPORTED_ROS_DISTROS + SUPPORTED_ROS2_DISTROS, 87 | help='Target ROS distribution') 88 | parser.add_argument( 89 | '-o', '--os', 90 | required=True, 91 | type=str, 92 | # NOTE: not specifying choices here, as different distros may support different lists 93 | help='Target OS') 94 | parser.add_argument( 95 | '--sysroot-base-image', 96 | required=False, 97 | type=str, 98 | help='Override the default base Docker image to use for building the sysroot. ' 99 | 'Ex. "arm64v8/ubuntu:focal"') 100 | parser.add_argument( 101 | '--sysroot-nocache', 102 | action='store_true', 103 | required=False, 104 | help="When set to true, this disables Docker's cache when building " 105 | 'the Docker image for the workspace') 106 | parser.add_argument( 107 | '--custom-rosdep-script', 108 | required=False, 109 | default=None, 110 | type=str, 111 | help='Provide a path to a shell script that will be executed right before collecting ' 112 | 'the list of dependencies for the target workspace. This allows you to install ' 113 | 'extra rosdep rules/sources that are not in the standard "rosdistro" set. See the ' 114 | 'section "Custom Rosdep Script" in README.md for more details.') 115 | parser.add_argument( 116 | '--custom-setup-script', 117 | required=False, 118 | default=None, 119 | type=str, 120 | help='Provide a path to a shell script that will be executed in the build container ' 121 | 'right before running "rosdep install" for your ROS workspace. This allows for ' 122 | 'adding extra apt sources that rosdep may not handle, or other arbitrary setup that ' 123 | 'is specific to your application build. See the section on "Custom Setup Script" ' 124 | 'in README.md for more details.') 125 | parser.add_argument( 126 | '--custom-post-build-script', 127 | required=False, 128 | default=None, 129 | type=str, 130 | help='Provide a path to a shell script that will be executed in the build container ' 131 | 'after the build successfully completes. ' 132 | 'See "Custom post-build script" in README.md for more information.') 133 | parser.add_argument( 134 | '--custom-data-dir', 135 | required=False, 136 | default=None, 137 | type=str, 138 | help='Provide a path to a custom arbitrary directory to copy into the sysroot container. ' 139 | 'You may use this data in your --custom-setup-script, it will be available as ' 140 | '"./custom_data/" in the current working directory when the script is run.') 141 | parser.add_argument( 142 | '--colcon-defaults', 143 | required=False, 144 | default=None, 145 | type=str, 146 | help='Provide a path to a configuration file that provides colcon arguments. ' 147 | 'See "Package Selection and Build Customization" in README.md for more details.') 148 | parser.add_argument( 149 | '--skip-rosdep-keys', 150 | default=[], 151 | nargs='+', 152 | help='Skip specified rosdep keys when collecting dependencies for the workspace.') 153 | parser.add_argument( 154 | '--custom-metric-file', 155 | required=False, 156 | default='{}.json'.format(datetime.now().strftime('%s')), 157 | type=str, 158 | help='Provide a filename to store the collected metrics. If no name is provided, ' 159 | 'then the filename will be the current time in UNIX Epoch format. ') 160 | parser.add_argument( 161 | '--print-metrics', 162 | action='store_true', 163 | required=False, 164 | help='All collected metrics will be printed to stdout via the logging framework.') 165 | parser.add_argument( 166 | '--stages-skip', 167 | nargs='+', 168 | choices=[stage.name for stage in _PIPELINE], 169 | default=[], 170 | help='Skip the given stages of the build pipeline to save time. Use at your own risk, ' 171 | 'as this could cause undefined behavior if some stage has not been previously run.') 172 | parser.add_argument( 173 | '--runtime-tag', 174 | type=str, 175 | required=False, 176 | default=None, 177 | help='Package the resulting build as a runnable Docker image, with all runtime ' 178 | 'dependencies installed. The image will use the tag provided to this argument. ' 179 | 'Example: "my_registry/image_name:image_tag"') 180 | 181 | return parser.parse_args(args) 182 | 183 | 184 | def cross_compile_pipeline( 185 | args: argparse.Namespace, 186 | data_collector: DataCollector, 187 | platform: Platform, 188 | ): 189 | ros_workspace_dir = _resolve_ros_workspace(args.ros_workspace) 190 | skip_rosdep_keys = args.skip_rosdep_keys 191 | custom_data_dir = _path_if(args.custom_data_dir) 192 | custom_rosdep_script = _path_if(args.custom_rosdep_script) 193 | custom_setup_script = _path_if(args.custom_setup_script) 194 | custom_post_build_script = _path_if(args.custom_post_build_script) 195 | colcon_defaults_file = _path_if(args.colcon_defaults) 196 | 197 | sysroot_build_context = prepare_docker_build_environment( 198 | platform=platform, 199 | ros_workspace=ros_workspace_dir, 200 | custom_setup_script=custom_setup_script, 201 | custom_post_build_script=custom_post_build_script, 202 | colcon_defaults_file=colcon_defaults_file, 203 | custom_data_dir=custom_data_dir) 204 | docker_client = DockerClient( 205 | args.sysroot_nocache, 206 | default_docker_dir=sysroot_build_context) 207 | 208 | options = PipelineStageOptions( 209 | skip_rosdep_keys, 210 | custom_rosdep_script, 211 | custom_data_dir, 212 | custom_setup_script, 213 | args.runtime_tag) 214 | 215 | skip = set(args.stages_skip) 216 | 217 | # Only package the runtime image if the user has specified a tag for it 218 | if not args.runtime_tag: 219 | skip.add(PackageRuntimeImageStage.NAME) 220 | 221 | for stage in _PIPELINE: 222 | if stage.name not in skip: 223 | with data_collector.timer('{}'.format(stage.name)): 224 | stage(platform, docker_client, ros_workspace_dir, options, data_collector) 225 | 226 | 227 | def main(): 228 | """Start the cross-compilation workflow.""" 229 | args = parse_args(sys.argv[1:]) 230 | platform = Platform(args.arch, args.os, args.rosdistro, args.sysroot_base_image) 231 | ros_workspace_dir = _resolve_ros_workspace(args.ros_workspace) 232 | data_collector = DataCollector() 233 | data_writer = DataWriter(ros_workspace_dir, args.custom_metric_file) 234 | 235 | try: 236 | with data_collector.timer('end_to_end'): 237 | cross_compile_pipeline(args, data_collector, platform) 238 | finally: 239 | data_writer.write(data_collector, platform, args.print_metrics) 240 | 241 | 242 | if __name__ == '__main__': 243 | if sys.version_info < (3, 7): 244 | logger.warning('You are using an unsupported version of Python.' 245 | 'Cross-compile only supports Python >= 3.7 per the ROS2 REP 2000.') 246 | main() 247 | -------------------------------------------------------------------------------- /ros_cross_compile/runtime.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import logging 16 | import os 17 | from pathlib import Path 18 | 19 | from ros_cross_compile.data_collector import DataCollector 20 | from ros_cross_compile.docker_client import DockerClient 21 | from ros_cross_compile.pipeline_stages import PipelineStage 22 | from ros_cross_compile.pipeline_stages import PipelineStageOptions 23 | from ros_cross_compile.platform import Platform 24 | 25 | logging.basicConfig(level=logging.INFO) 26 | logger = logging.getLogger(__name__) 27 | 28 | 29 | def create_runtime_image( 30 | docker_client: DockerClient, platform: Platform, workspace_dir: Path, image_tag: str 31 | ): 32 | logger.info('Building runtime image: {}'.format(image_tag)) 33 | docker_client.build_image( 34 | dockerfile_name=os.path.join(docker_client._default_docker_dir, 'runtime.Dockerfile'), 35 | dockerfile_dir=workspace_dir, 36 | tag=image_tag, 37 | buildargs={ 38 | 'BASE_IMAGE': platform.sysroot_image_tag, 39 | 'INSTALL_PATH': 'install_{}'.format(platform.arch), 40 | }, 41 | ) 42 | 43 | 44 | class PackageRuntimeImageStage(PipelineStage): 45 | """ 46 | This stage determines what external dependencies are needed for building. 47 | 48 | It outputs a script into the internals directory that will install those 49 | dependencies for the target platform. 50 | """ 51 | 52 | NAME = 'runtime' 53 | 54 | def __init__(self): 55 | super().__init__(self.NAME) 56 | 57 | def __call__( 58 | self, platform: Platform, docker_client: DockerClient, ros_workspace_dir: Path, 59 | options: PipelineStageOptions, 60 | data_collector: DataCollector 61 | ): 62 | tag = options.runtime_tag 63 | assert tag is not None 64 | create_runtime_image(docker_client, platform, ros_workspace_dir, tag) 65 | img_size = docker_client.get_image_size(tag) 66 | data_collector.add_size(self.name, img_size) 67 | -------------------------------------------------------------------------------- /ros_cross_compile/sysroot_creator.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from distutils.dir_util import copy_tree 16 | import logging 17 | from pathlib import Path 18 | import platform as py_platform 19 | import shutil 20 | from typing import Optional 21 | 22 | from ros_cross_compile.data_collector import DataCollector 23 | from ros_cross_compile.data_collector import INTERNALS_DIR 24 | from ros_cross_compile.docker_client import DockerClient 25 | from ros_cross_compile.pipeline_stages import PipelineStage 26 | from ros_cross_compile.pipeline_stages import PipelineStageOptions 27 | from ros_cross_compile.platform import Platform 28 | 29 | logging.basicConfig(level=logging.INFO) 30 | logger = logging.getLogger(__name__) 31 | 32 | 33 | def build_internals_dir(platform: Platform) -> Path: 34 | """Construct a relative path for this build, for storing intermediate artifacts.""" 35 | return Path(INTERNALS_DIR) / str(platform) 36 | 37 | 38 | def _copytree(src: Path, dest: Path) -> None: 39 | """Copy contents of directory 'src' into 'dest'.""" 40 | copy_tree(str(src), str(dest)) 41 | 42 | 43 | def _copyfile(src: Path, dest: Path) -> None: 44 | """Copy a single file to a destination location.""" 45 | shutil.copy(str(src), str(dest)) 46 | 47 | 48 | def _copy_or_touch(src: Optional[Path], dest: Path) -> None: 49 | """ 50 | Copy a single file, if specified, to a destination. 51 | 52 | If the file src is not provided, create an empty file at dest. 53 | """ 54 | if src: 55 | _copyfile(src, dest) 56 | else: 57 | dest.touch() 58 | 59 | 60 | def setup_emulator(arch: str, output_dir: Path) -> None: 61 | """Copy the appropriate emulator binary to the output location.""" 62 | emulator_name = 'qemu-{}-static'.format(arch) 63 | bin_dir = output_dir / 'bin' 64 | bin_dir.mkdir(parents=True, exist_ok=True) 65 | needs_emulator = (py_platform.system() != 'Darwin') and (py_platform.machine() != arch) 66 | if needs_emulator: 67 | """ 68 | Using the same qemu binaries as the ones provided in 69 | https://github.com/osrf/multiarch-docker-image-generation in order to 70 | work around https://bugs.launchpad.net/qemu/+bug/1805913 and so qemu 71 | supports renameat2() syscall. 72 | """ 73 | emulator_path = Path(__file__).parent / 'qemu' / emulator_name 74 | if not emulator_path.is_file(): 75 | raise RuntimeError('Could not find the expected QEmu emulator binary "{}"'.format( 76 | emulator_path)) 77 | _copyfile(emulator_path, bin_dir) 78 | else: 79 | (bin_dir / emulator_name).touch() 80 | 81 | 82 | def prepare_docker_build_environment( 83 | platform: Platform, 84 | ros_workspace: Path, 85 | custom_setup_script: Optional[Path] = None, 86 | custom_post_build_script: Optional[Path] = None, 87 | colcon_defaults_file: Optional[Path] = None, 88 | custom_data_dir: Optional[Path] = None, 89 | ) -> Path: 90 | """ 91 | Prepare the directory for the sysroot.Docker build context. 92 | 93 | :param platform Information about the target platform 94 | :param ros_workspace Location of the ROS source workspace 95 | :param custom_setup_script Optional arbitrary script 96 | :param colcon_defaults_file Optional colcon configuration file 97 | :param custom_data_dir Optional arbitrary directory for use by custom_setup_script 98 | :return The directory that was created. 99 | """ 100 | package_dir = Path(__file__).parent 101 | docker_build_dir = ros_workspace / build_internals_dir(platform) 102 | docker_build_dir.mkdir(parents=True, exist_ok=True) 103 | 104 | _copytree(package_dir / 'docker', docker_build_dir) 105 | _copytree(package_dir / 'mixins', docker_build_dir / 'mixins') 106 | 107 | custom_data_dest = docker_build_dir / 'user-custom-data' 108 | if custom_data_dir: 109 | _copytree(custom_data_dir, custom_data_dest) 110 | else: 111 | custom_data_dest.mkdir(exist_ok=True) 112 | 113 | _copy_or_touch(custom_setup_script, docker_build_dir / 'user-custom-setup') 114 | _copy_or_touch(custom_post_build_script, docker_build_dir / 'user-custom-post-build') 115 | 116 | if colcon_defaults_file: 117 | _copyfile(colcon_defaults_file, docker_build_dir / 'defaults.yaml') 118 | else: 119 | (docker_build_dir / 'defaults.yaml').write_text(""" 120 | build: 121 | event-handlers: ["console_cohesion+","console_package_list+"] 122 | """) 123 | 124 | setup_emulator(platform.qemu_arch, docker_build_dir) 125 | 126 | return docker_build_dir 127 | 128 | 129 | def create_workspace_sysroot_image( 130 | docker_client: DockerClient, 131 | platform: Platform, 132 | ) -> None: 133 | """ 134 | Create the target platform sysroot image. 135 | 136 | :param docker_client Docker client to use for building 137 | :param platform Information about the target platform 138 | :param build_context Directory containing all assets needed by sysroot.Dockerfile 139 | """ 140 | image_tag = platform.sysroot_image_tag 141 | 142 | logger.info('Building sysroot image: %s', image_tag) 143 | docker_client.build_image( 144 | dockerfile_name='sysroot.Dockerfile', 145 | tag=image_tag, 146 | buildargs={ 147 | 'BASE_IMAGE': platform.target_base_image, 148 | 'ROS_VERSION': platform.ros_version, 149 | } 150 | ) 151 | logger.info('Successfully created sysroot docker image: %s', image_tag) 152 | 153 | 154 | class CreateSysrootStage(PipelineStage): 155 | """ 156 | This stage creates the target platform Docker sysroot image. 157 | 158 | It will output the sysroot image. 159 | """ 160 | 161 | def __init__(self): 162 | super().__init__('sysroot') 163 | 164 | def __call__( 165 | self, 166 | platform: Platform, 167 | docker_client: DockerClient, 168 | ros_workspace_dir: Path, 169 | options: PipelineStageOptions, 170 | data_collector: DataCollector 171 | ): 172 | create_workspace_sysroot_image(docker_client, platform) 173 | 174 | img_size = docker_client.get_image_size(platform.sysroot_image_tag) 175 | data_collector.add_size(self.name, img_size) 176 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | from os import path 15 | 16 | from setuptools import find_packages 17 | from setuptools import setup 18 | 19 | package_name = 'ros_cross_compile' 20 | 21 | this_directory = path.abspath(path.dirname(__file__)) 22 | with open(path.join(this_directory, 'README.md'), encoding='utf-8') as f: 23 | long_description = f.read() 24 | 25 | setup( 26 | name=package_name, 27 | version='0.10.0', 28 | packages=find_packages(exclude=['test']), 29 | author='ROS Tooling Working Group', 30 | author_email='ros-tooling@googlegroups.com', 31 | maintainer='ROS Tooling Working Group', 32 | maintainer_email='ros-tooling@googlegroups.com', 33 | url='https://github.com/ros-tooling/cross_compile', 34 | download_url='https://github.com/ros-tooling/cross_compile/releases', 35 | keywords=['ROS', 'ROS2'], 36 | classifiers=[ 37 | 'Development Status :: 4 - Beta', 38 | 'Environment :: Console', 39 | 'Intended Audience :: Developers', 40 | 'License :: OSI Approved :: Apache Software License', 41 | 'Programming Language :: Python :: 3.7', 42 | 'Programming Language :: Python :: 3.8', 43 | 'Topic :: Software Development', 44 | ], 45 | description='A tool to build ROS workspaces for various target architectures and platforms.', 46 | long_description=long_description, 47 | long_description_content_type='text/markdown', 48 | license='Apache License, Version 2.0', 49 | package_data={ 50 | package_name: ['docker/*.*', 'mixins/*.*', 'qemu/*'], 51 | }, 52 | install_requires=[ 53 | 'docker>=2,<3', 54 | 'setuptools', 55 | ], 56 | zip_safe=True, 57 | entry_points={ 58 | 'console_scripts': [ 59 | 'ros_cross_compile = ros_cross_compile.ros_cross_compile:main' 60 | ] 61 | }, 62 | python_requires='>=3.7', 63 | ) 64 | -------------------------------------------------------------------------------- /test/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /test/custom-setup-with-data.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo "This custom setup script uses the custom data dir." 3 | cat custom-data/arbitrary.txt 4 | -------------------------------------------------------------------------------- /test/custom-setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo "This is a custom setup script for the sysroot environment." 3 | -------------------------------------------------------------------------------- /test/data/arbitrary.txt: -------------------------------------------------------------------------------- 1 | This is a test file with text in it. 2 | -------------------------------------------------------------------------------- /test/dummy_pkg/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.5) 2 | project(dummy_pkg) 3 | add_executable(dummy_binary main.cpp) 4 | install(TARGETS dummy_binary DESTINATION bin) 5 | -------------------------------------------------------------------------------- /test/dummy_pkg/main.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | int main() {} 15 | -------------------------------------------------------------------------------- /test/dummy_pkg/package.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | dummy_pkg 5 | 0.0.0 6 | dummy package 7 | dummy package 8 | dummy package 9 | 10 | 11 | ament_cmake 12 | 13 | 14 | -------------------------------------------------------------------------------- /test/dummy_pkg_ros2_cpp/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.5) 2 | project(dummy_pkg_ros2_cpp) 3 | 4 | find_package(ament_cmake REQUIRED) 5 | find_package(rclcpp REQUIRED) 6 | 7 | add_executable(dummy_binary main.cpp) 8 | ament_target_dependencies(dummy_binary "rclcpp") 9 | install(TARGETS dummy_binary DESTINATION bin) 10 | ament_package() 11 | -------------------------------------------------------------------------------- /test/dummy_pkg_ros2_cpp/main.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #include "rclcpp/rclcpp.hpp" 15 | 16 | int main(int argc, char** argv) { 17 | rclcpp::init(argc, argv); 18 | auto node = std::make_unique("dummy_node"); 19 | rclcpp::shutdown(); 20 | return 0; 21 | } 22 | -------------------------------------------------------------------------------- /test/dummy_pkg_ros2_cpp/package.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | dummy_pkg_ros2_cpp 5 | 0.0.0 6 | end to end ROS2 C++ test package for cross compiling 7 | Nobody 8 | Apache 2.0 9 | 10 | ament_cmake 11 | rclcpp 12 | 13 | 14 | ament_cmake 15 | 16 | 17 | -------------------------------------------------------------------------------- /test/dummy_pkg_ros2_py/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ros-tooling/cross_compile/1cedd079950bb184b8ae76198a1b92a8a8229912/test/dummy_pkg_ros2_py/__init__.py -------------------------------------------------------------------------------- /test/dummy_pkg_ros2_py/dummy_pkg_ros2_py/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ros-tooling/cross_compile/1cedd079950bb184b8ae76198a1b92a8a8229912/test/dummy_pkg_ros2_py/dummy_pkg_ros2_py/__init__.py -------------------------------------------------------------------------------- /test/dummy_pkg_ros2_py/package.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | dummy_pkg_ros2_py 5 | 0.0.0 6 | end to end ROS2 Python test package for cross compiling 7 | Nobody 8 | Apache 2.0 9 | 10 | rclpy 11 | 12 | 13 | ament_python 14 | 15 | 16 | -------------------------------------------------------------------------------- /test/dummy_pkg_ros2_py/resource/dummy_pkg_ros2_py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ros-tooling/cross_compile/1cedd079950bb184b8ae76198a1b92a8a8229912/test/dummy_pkg_ros2_py/resource/dummy_pkg_ros2_py -------------------------------------------------------------------------------- /test/dummy_pkg_ros2_py/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | package_name = 'dummy_pkg_ros2_py' 4 | 5 | setup( 6 | name=package_name, 7 | version='0.0.0', 8 | packages=[package_name], 9 | data_files=[ 10 | ('share/ament_index/resource_index/packages', 11 | ['resource/' + package_name]), 12 | ('share/' + package_name, ['package.xml']), 13 | ], 14 | install_requires=['setuptools'], 15 | zip_safe=True, 16 | maintainer='nobody', 17 | maintainer_email='nobody@example.com', 18 | description='package description', 19 | license='none', 20 | ) 21 | -------------------------------------------------------------------------------- /test/run_e2e_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # This end-to-end test runs the entire cross-compilation pipeline for this tool 3 | # This test does the following: 4 | # 1. Mocks a sysroot directory in a temporary directory. 5 | # 2. Runs the cross compilation script against the mock sysroot, compiling all the packages. 6 | # 3. Runs a Docker container using the image built from cross compilation script. 7 | # 4. Executes a C++ example demo node in the container. 8 | # 5. Closes the Docker container. 9 | # The test passes if none of the above actions fail to complete. 10 | 11 | set -euxo pipefail 12 | 13 | # Terminal output colors 14 | readonly RED='\033[0;31m' 15 | readonly GREEN='\033[0;32m' 16 | readonly YELLOW='\033[0;33m' 17 | readonly CYAN='\033[0;36m' 18 | readonly NORM='\033[0m' 19 | # Defaults 20 | arch=aarch64 # or "armhf" 21 | os=ubuntu # or "debian" 22 | distro=foxy 23 | result=1 # Default to failure 24 | 25 | RUNTIME_IMAGE_TAG="$(whoami)/$arch-$os-$distro:e2e-runtime" 26 | readonly RUNTIME_IMAGE_TAG 27 | 28 | # Loggers 29 | log(){ 30 | printf "%b%s%b\n" "$CYAN" "$1" "$NORM" 31 | } 32 | 33 | warning(){ 34 | printf "%b%s%b\n" "$YELLOW" "$1" "$NORM" 35 | } 36 | 37 | error(){ 38 | printf "%b%s%b\n" "$RED" "$1" "$NORM" 39 | } 40 | 41 | panic() { 42 | error "$1" 43 | result=1 44 | exit 1 45 | } 46 | 47 | success(){ 48 | printf "%b%s%b\n" "$GREEN" "$1" "$NORM" 49 | } 50 | 51 | # Utilities 52 | cleanup(){ 53 | # Usage: cleanup RESULT 54 | if [[ "$1" -eq 0 ]]; then 55 | success PASS 56 | else 57 | error FAIL 58 | fi 59 | rm -rf "$test_sysroot_dir"; 60 | } 61 | 62 | setup(){ 63 | if [[ "$distro" =~ ^(melodic|noetic)$ ]]; then 64 | ros_version=ros 65 | else 66 | ros_version=ros2 67 | fi 68 | 69 | test_sysroot_dir=$(mktemp -d) 70 | mkdir -p "$test_sysroot_dir/src" 71 | # Get full directory name of the script no matter where it is being called from 72 | dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 73 | 74 | # Copy correct dummy test pkg for the current argument set 75 | if [ "$ros_version" == "ros2" ] && [ "$os" == "ubuntu" ]; then 76 | # ROS2 + Debian info 77 | # Debian is a Tier 3 package for current ROS 2 distributions, so it doesn't have apt releases 78 | # Therefore we can't resolve the rclcpp dependency of the ros2 package, so we build the empty project 79 | cp -r "$dir/dummy_pkg_ros2_cpp" "$test_sysroot_dir/src" 80 | cp -r "$dir/dummy_pkg_ros2_py" "$test_sysroot_dir/src" 81 | target_package="dummy_pkg_ros2_cpp" 82 | fi 83 | 84 | cp -r "$dir/dummy_pkg" "$test_sysroot_dir/src" 85 | target_package="dummy_pkg" 86 | 87 | custom_setup_script=${test_sysroot_dir}/custom-setup.bash 88 | echo "#!/bin/bash" > "$custom_setup_script" 89 | if [ "$arch" == "armhf" ]; then 90 | if [ "$distro" == "foxy" ] || \ 91 | [ "$distro" == "galactic" ] || \ 92 | [ "$distro" == "humble" ] || \ 93 | [ "$distro" == "rolling" ]; then 94 | error "Foxy, Galactic, Humble and Rolling do not have armhf binaries available" 95 | exit 0 96 | fi 97 | if [ "$os" == "debian" ]; then 98 | if [ "$distro" == "noetic" ]; then 99 | error "Noetic does not have armhf binaries available to debian" 100 | exit 0 101 | fi 102 | fi 103 | fi 104 | } 105 | 106 | # Argparser 107 | while [[ $# -gt 0 ]] 108 | do 109 | key="$1" 110 | 111 | case $key in 112 | -a|--arch) 113 | arch="$2" 114 | shift 2 115 | ;; 116 | -o|--os) 117 | os="$2" 118 | shift 2 119 | ;; 120 | -d|--rosdistro) 121 | distro="$2" 122 | shift 2 123 | ;; 124 | *) 125 | panic "Unrecognized option $1" 126 | ;; 127 | esac 128 | done 129 | 130 | # Create trap to make sure all artifacts are removed on exit 131 | trap 'cleanup $result' EXIT 132 | 133 | # Testing starts here 134 | setup 135 | 136 | # Run the cross compilation script 137 | log "Executing cross compilation script..." 138 | python3 -m ros_cross_compile "$test_sysroot_dir" \ 139 | --arch "$arch" --os "$os" --rosdistro "$distro" \ 140 | --runtime-tag "$RUNTIME_IMAGE_TAG" \ 141 | --custom-setup-script "$custom_setup_script" --custom-metric-file "${arch}_${os}_${distro}_a" 142 | CC_SCRIPT_STATUS=$? 143 | if [[ "$CC_SCRIPT_STATUS" -ne 0 ]]; then 144 | panic "Failed to run cross compile script." 145 | fi 146 | 147 | install_dir=$test_sysroot_dir/install_$arch 148 | build_dir=$test_sysroot_dir/build_$arch 149 | 150 | log "Checking that install directory was output to the correct place..." 151 | if [ ! -d "$install_dir" ]; then 152 | panic "The install directory was not where it should have been output" 153 | fi 154 | 155 | log "Checking that extraction didn't overwrite our workspace" 156 | if [ ! -d "$test_sysroot_dir/src" ]; then 157 | panic "The user's source tree got deleted by the build" 158 | fi 159 | 160 | log "Checking that the binary output is in the correct architecture..." 161 | if [ "$arch" = 'armhf' ]; then 162 | expected_binary_bits='ELF 32-bit' 163 | elif [ "$arch" = 'aarch64' ]; then 164 | expected_binary_bits='ELF 64-bit' 165 | elif [ "$arch" = 'x86_64' ]; then 166 | expected_binary_bits='ELF 64-bit' 167 | fi 168 | 169 | if [ "$arch" = 'armhf' ]; then 170 | expected_binary_architecture=', ARM' 171 | elif [ "$arch" = 'aarch64' ]; then 172 | expected_binary_architecture=', ARM aarch64' 173 | elif [ "$arch" = 'x86_64' ]; then 174 | expected_binary_architecture=', x86-64' 175 | fi 176 | binary_file_info=$(file "$install_dir/$target_package/bin/dummy_binary") 177 | if [[ "$binary_file_info" != *"$expected_binary_bits"*"$expected_binary_architecture"* ]]; then 178 | panic "The binary output was not of the expected architecture" 179 | fi 180 | 181 | log "Running example node in the runtime container..." 182 | docker run --rm \ 183 | "$RUNTIME_IMAGE_TAG" \ 184 | -c "source /root/.bashrc && dummy_binary" 185 | RUN_RESULT=$? 186 | if [[ "$RUN_RESULT" -ne 0 ]]; then 187 | panic "Failed to run the dummy binary in the Docker container." 188 | fi 189 | 190 | log "Rerunning build with package selection..." 191 | rm -rf "$install_dir" 192 | rm -rf "$build_dir" 193 | cp -r "$dir/dummy_pkg_ros2_cpp" "$test_sysroot_dir/src" 194 | cat > "$test_sysroot_dir/defaults.yaml" < 111 | bad_package 112 | 113 | ament_cmake 114 | 115 | 116 | """) 117 | (bad_package / 'CMakeLists.txt').write_text('Invalid CMakeLists content.') 118 | 119 | # Expect the build to fail 120 | with pytest.raises(Exception): 121 | EmulatedDockerBuildStage()( 122 | buildable_env.platform, 123 | buildable_env.docker, 124 | ros_workspace, 125 | buildable_env.options, 126 | buildable_env.data_collector) 127 | 128 | # make sure all files are owned by the current user 129 | user = getpass.getuser() 130 | for p in buildable_env.ros_workspace.rglob('*'): 131 | assert user == p.owner() 132 | 133 | 134 | @uses_docker 135 | def test_custom_post_build_script(tmpdir): 136 | created_filename = 'file-created-by-post-build' 137 | platform = Platform('aarch64', 'ubuntu', 'foxy') 138 | ros_workspace = Path(str(tmpdir)) / 'ros_ws' 139 | _touch_anywhere(ros_workspace / rosdep_install_script(platform)) 140 | post_build_script = ros_workspace / 'post_build' 141 | post_build_script.write_text(""" 142 | #!/bin/bash 143 | echo "success" > {} 144 | """.format(created_filename)) 145 | build_context = prepare_docker_build_environment( 146 | platform, 147 | ros_workspace, 148 | custom_post_build_script=post_build_script) 149 | docker = DockerClient(disable_cache=False, default_docker_dir=build_context) 150 | options = default_pipeline_options() 151 | data_collector = DataCollector() 152 | 153 | CreateSysrootStage()( 154 | platform, docker, ros_workspace, options, data_collector) 155 | EmulatedDockerBuildStage()( 156 | platform, docker, ros_workspace, options, data_collector) 157 | 158 | assert (ros_workspace / created_filename).is_file() 159 | -------------------------------------------------------------------------------- /test/test_colcon_mixins.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | """ 17 | Lint the index.yaml file as well as all files ending with .mixin. 18 | 19 | Inspired by lint.py in https://github.com/colcon/colcon-mixin-repository. 20 | """ 21 | 22 | import os 23 | 24 | import pkg_resources 25 | import pytest 26 | from yamllint.cli import run 27 | 28 | 29 | @pytest.mark.linter 30 | def test_colcon_mixins(): 31 | any_error = False 32 | mixins_dir = pkg_resources.resource_filename('ros_cross_compile', 'mixins') 33 | for name in sorted(os.listdir(mixins_dir)): 34 | if name != 'index.yaml' and not name.endswith('.mixin'): 35 | continue 36 | 37 | try: 38 | run([ 39 | '--config-data', 40 | '{' 41 | 'extends: default, ' 42 | 'rules: {' 43 | 'document-start: {present: false}, ' 44 | 'empty-lines: {max: 0}, ' 45 | 'key-ordering: {}, ' 46 | 'line-length: {max: 999}' 47 | '}' 48 | '}', 49 | '--strict', 50 | os.path.join(mixins_dir, name), 51 | ]) 52 | except SystemExit as e: 53 | any_error |= bool(e.code) 54 | continue 55 | 56 | assert not any_error, 'Should not have seen any errors' 57 | -------------------------------------------------------------------------------- /test/test_copyright.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from ament_copyright.main import main 16 | import pytest 17 | 18 | 19 | @pytest.mark.copyright 20 | @pytest.mark.linter 21 | def test_copyright(): 22 | rc = main(argv=['.', 'test']) 23 | assert rc == 0, 'Found errors' 24 | -------------------------------------------------------------------------------- /test/test_data_collector.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import json 16 | from json import JSONDecodeError 17 | from pathlib import Path 18 | 19 | import pytest 20 | 21 | from ros_cross_compile.data_collector import DataCollector 22 | from ros_cross_compile.data_collector import DataWriter 23 | from ros_cross_compile.data_collector import Datum 24 | from ros_cross_compile.platform import Platform 25 | 26 | 27 | def test_datum_construction(): 28 | test_datum = Datum('test_stat', 3, 'tests', 130.222, True) 29 | assert test_datum 30 | 31 | 32 | def test_collector_construction(): 33 | test_collector = DataCollector() 34 | assert test_collector 35 | 36 | 37 | def test_data_collection(): 38 | test_collector = DataCollector() 39 | 40 | test_datum_a = Datum('test_stat_1', 3, 'tests', 130.452, True) 41 | test_datum_b = Datum('test_stat_2', 4, 'tests', 130.455, True) 42 | 43 | test_collector.add_datum(test_datum_a) 44 | test_collector.add_datum(test_datum_b) 45 | 46 | to_test_data = test_collector.data 47 | 48 | assert to_test_data[0].name == 'test_stat_1' 49 | assert to_test_data[1].name == 'test_stat_2' 50 | assert to_test_data[0].value == 3 51 | assert to_test_data[0].unit == 'tests' 52 | assert abs(to_test_data[0].timestamp - 130.452) < 0.1 53 | assert to_test_data[0].complete 54 | 55 | 56 | def test_timer_can_time(): 57 | test_collector = DataCollector() 58 | with test_collector.timer('test_time'): 59 | pass 60 | 61 | assert test_collector._data[0].complete 62 | assert test_collector._data[0].value > 0 63 | 64 | 65 | def test_timer_error_handling(): 66 | test_collector = DataCollector() 67 | # The timer should not hide the exception, we expect it to add the datum value 68 | with pytest.raises(Exception): 69 | with test_collector.timer('test_time_fail'): 70 | raise Exception 71 | 72 | assert len(test_collector._data) > 0 73 | assert test_collector._data[0].complete is False 74 | 75 | 76 | def test_data_writing(tmp_path): 77 | def load_json_validation(filename: Path) -> bool: 78 | try: 79 | with filename.open() as f: 80 | json.load(f) 81 | return True 82 | except JSONDecodeError: 83 | return False 84 | 85 | platform = Platform('aarch64', 'ubuntu', 'foxy') 86 | test_collector = DataCollector() 87 | 88 | test_datum_a = Datum('test_stat_1', 3, 'tests', 130.243, True) 89 | test_datum_b = Datum('test_stat_2', 4, 'tests', 130.244, True) 90 | 91 | test_collector.add_datum(test_datum_a) 92 | test_collector.add_datum(test_datum_b) 93 | 94 | test_writer = DataWriter(tmp_path, 'test.json') 95 | 96 | test_writer.write(test_collector, platform, False) 97 | 98 | assert test_writer.write_file.exists() 99 | assert load_json_validation(test_writer.write_file) 100 | 101 | 102 | def test_data_printing(tmp_path, capfd): 103 | platform = Platform('aarch64', 'ubuntu', 'foxy') 104 | test_collector = DataCollector() 105 | test_datum_a = Datum('test_stat_1', 3, 'tests', 130.243, True) 106 | test_collector.add_datum(test_datum_a) 107 | 108 | test_writer = DataWriter(tmp_path, 'test.json') 109 | test_writer.write(test_collector, platform, True) 110 | 111 | out, err = capfd.readouterr() 112 | test_name = '------------' 113 | 114 | assert test_name in out 115 | -------------------------------------------------------------------------------- /test/test_dependencies.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | from pathlib import Path 15 | import shutil 16 | 17 | import docker 18 | import pytest 19 | 20 | from ros_cross_compile.data_collector import DataCollector 21 | from ros_cross_compile.dependencies import CollectDependencyListStage 22 | from ros_cross_compile.dependencies import gather_rosdeps 23 | from ros_cross_compile.dependencies import rosdep_install_script 24 | from ros_cross_compile.docker_client import DockerClient 25 | from ros_cross_compile.platform import Platform 26 | 27 | from .utilities import default_pipeline_options 28 | from .utilities import uses_docker 29 | 30 | 31 | def make_pkg_xml(contents): 32 | return """ 33 | dummy 34 | 0.0.0 35 | Test package 36 | Example 37 | None 38 | {} 39 | 40 | """.format(contents) 41 | 42 | 43 | RCLCPP_PKG_XML = make_pkg_xml(""" 44 | ament_cmake 45 | rclcpp 46 | """) 47 | 48 | CUSTOM_KEY_PKG_XML = make_pkg_xml(""" 49 | definitely_does_not_exist 50 | """) 51 | 52 | 53 | @uses_docker 54 | def test_dummy_ros2_pkg(tmpdir): 55 | ws = Path(str(tmpdir)) 56 | pkg_xml = ws / 'src' / 'dummy' / 'package.xml' 57 | pkg_xml.parent.mkdir(parents=True) 58 | pkg_xml.write_text(RCLCPP_PKG_XML) 59 | 60 | client = DockerClient() 61 | platform = Platform(arch='aarch64', os_name='ubuntu', ros_distro='foxy') 62 | out_script = ws / rosdep_install_script(platform) 63 | test_collector = DataCollector() 64 | 65 | stage = CollectDependencyListStage() 66 | stage(platform, client, ws, default_pipeline_options(), test_collector) 67 | 68 | result = out_script.read_text() 69 | assert 'ros-foxy-ament-cmake' in result 70 | assert 'ros-foxy-rclcpp' in result 71 | 72 | 73 | @uses_docker 74 | def test_rosdep_bad_key(tmpdir): 75 | ws = Path(str(tmpdir)) 76 | pkg_xml = Path(ws) / 'src' / 'dummy' / 'package.xml' 77 | pkg_xml.parent.mkdir(parents=True) 78 | pkg_xml.write_text(CUSTOM_KEY_PKG_XML) 79 | client = DockerClient() 80 | platform = Platform(arch='aarch64', os_name='ubuntu', ros_distro='foxy') 81 | with pytest.raises(docker.errors.ContainerError): 82 | gather_rosdeps(client, platform, workspace=ws) 83 | 84 | 85 | @uses_docker 86 | def test_custom_rosdep_no_data_dir(tmpdir): 87 | script_contents = """ 88 | cat > /test_rules.yaml < /etc/ros/rosdep/sources.list.d/22-test-rules.list 94 | """ 95 | ws = Path(str(tmpdir)) 96 | pkg_xml = Path(ws) / 'src' / 'dummy' / 'package.xml' 97 | pkg_xml.parent.mkdir(parents=True) 98 | pkg_xml.write_text(CUSTOM_KEY_PKG_XML) 99 | client = DockerClient() 100 | platform = Platform(arch='aarch64', os_name='ubuntu', ros_distro='foxy') 101 | 102 | rosdep_setup = ws / 'rosdep_setup.sh' 103 | rosdep_setup.write_text(script_contents) 104 | 105 | gather_rosdeps(client, platform, workspace=ws, custom_script=rosdep_setup) 106 | out_script = ws / rosdep_install_script(platform) 107 | result = out_script.read_text() 108 | assert 'successful_test' in result 109 | 110 | 111 | @uses_docker 112 | def test_custom_rosdep_melodic_no_data_dir(tmpdir): 113 | script_contents = """ 114 | cat > /test_rules.yaml < /etc/ros/rosdep/sources.list.d/22-test-rules.list 120 | """ 121 | ws = Path(str(tmpdir)) 122 | pkg_xml = Path(ws) / 'src' / 'dummy' / 'package.xml' 123 | pkg_xml.parent.mkdir(parents=True) 124 | pkg_xml.write_text(CUSTOM_KEY_PKG_XML) 125 | client = DockerClient() 126 | platform = Platform(arch='aarch64', os_name='ubuntu', ros_distro='melodic') 127 | 128 | rosdep_setup = ws / 'rosdep_setup.sh' 129 | rosdep_setup.write_text(script_contents) 130 | 131 | gather_rosdeps(client, platform, workspace=ws, custom_script=rosdep_setup) 132 | out_script = ws / rosdep_install_script(platform) 133 | result = out_script.read_text() 134 | assert 'successful_test' in result 135 | 136 | 137 | @uses_docker 138 | def test_custom_rosdep_with_data_dir(tmpdir): 139 | rule_contents = """ 140 | definitely_does_not_exist: 141 | ubuntu: 142 | focal: [successful_test] 143 | """ 144 | 145 | script_contents = """ 146 | #!/bin/bash 147 | set -euxo 148 | cp ./custom-data/test_rules.yaml /test_rules.yaml 149 | echo "yaml file:/test_rules.yaml" > /etc/ros/rosdep/sources.list.d/22-test-rules.list 150 | """ 151 | ws = Path(str(tmpdir)) 152 | pkg_xml = Path(ws) / 'src' / 'dummy' / 'package.xml' 153 | pkg_xml.parent.mkdir(parents=True) 154 | pkg_xml.write_text(CUSTOM_KEY_PKG_XML) 155 | client = DockerClient() 156 | platform = Platform(arch='aarch64', os_name='ubuntu', ros_distro='foxy') 157 | 158 | rosdep_setup = ws / 'rosdep_setup.sh' 159 | rosdep_setup.write_text(script_contents) 160 | 161 | data_dir = ws / 'custom_data' 162 | data_file = data_dir / 'test_rules.yaml' 163 | data_file.parent.mkdir(parents=True) 164 | data_file.write_text(rule_contents) 165 | 166 | gather_rosdeps( 167 | client, platform, workspace=ws, custom_script=rosdep_setup, custom_data_dir=data_dir) 168 | 169 | out_script = ws / rosdep_install_script(platform) 170 | result = out_script.read_text() 171 | assert 'successful_test' in result 172 | 173 | 174 | @uses_docker 175 | def test_custom_rosdep_melodic_with_data_dir(tmpdir): 176 | rule_contents = """ 177 | definitely_does_not_exist: 178 | ubuntu: 179 | bionic: [successful_test] 180 | """ 181 | 182 | script_contents = """ 183 | #!/bin/bash 184 | set -euxo 185 | cp ./custom-data/test_rules.yaml /test_rules.yaml 186 | echo "yaml file:/test_rules.yaml" > /etc/ros/rosdep/sources.list.d/22-test-rules.list 187 | """ 188 | ws = Path(str(tmpdir)) 189 | pkg_xml = Path(ws) / 'src' / 'dummy' / 'package.xml' 190 | pkg_xml.parent.mkdir(parents=True) 191 | pkg_xml.write_text(CUSTOM_KEY_PKG_XML) 192 | client = DockerClient() 193 | platform = Platform(arch='aarch64', os_name='ubuntu', ros_distro='melodic') 194 | 195 | rosdep_setup = ws / 'rosdep_setup.sh' 196 | rosdep_setup.write_text(script_contents) 197 | 198 | data_dir = ws / 'custom_data' 199 | data_file = data_dir / 'test_rules.yaml' 200 | data_file.parent.mkdir(parents=True) 201 | data_file.write_text(rule_contents) 202 | 203 | gather_rosdeps( 204 | client, platform, workspace=ws, custom_script=rosdep_setup, custom_data_dir=data_dir) 205 | 206 | out_script = ws / rosdep_install_script(platform) 207 | result = out_script.read_text() 208 | assert 'successful_test' in result 209 | 210 | 211 | @uses_docker 212 | def test_colcon_defaults(tmpdir): 213 | ws = Path(str(tmpdir)) 214 | this_dir = Path(__file__).parent 215 | src = ws / 'src' 216 | src.mkdir() 217 | shutil.copytree(str(this_dir / 'dummy_pkg_ros2_cpp'), str(src / 'dummy_pkg_ros2_cpp')) 218 | shutil.copytree(str(this_dir / 'dummy_pkg_ros2_py'), str(src / 'dummy_pkg_ros2_py')) 219 | 220 | client = DockerClient() 221 | platform = Platform(arch='aarch64', os_name='ubuntu', ros_distro='foxy') 222 | out_script = ws / rosdep_install_script(platform) 223 | 224 | # none or default config should get everything 225 | gather_rosdeps(client, platform, workspace=ws) 226 | 227 | result = out_script.read_text() 228 | assert 'ros-foxy-ament-cmake' in result 229 | assert 'ros-foxy-rclcpp' in result 230 | assert 'ros-foxy-rclpy' in result 231 | 232 | # write defaults file and expect selective dependency output 233 | (ws / 'defaults.yaml').write_text(""" 234 | list: 235 | packages-select: [dummy_pkg_ros2_py] 236 | """) 237 | gather_rosdeps(client, platform, workspace=ws) 238 | result = out_script.read_text() 239 | assert 'ros-foxy-ament-cmake' not in result 240 | assert 'ros-foxy-rclcpp' not in result 241 | assert 'ros-foxy-rclpy' in result 242 | 243 | 244 | @uses_docker 245 | def test_dummy_skip_rosdep_keys_doesnt_exist_pkg(tmpdir): 246 | ws = Path(str(tmpdir)) 247 | pkg_xml = Path(ws) / 'src' / 'dummy' / 'package.xml' 248 | pkg_xml.parent.mkdir(parents=True) 249 | pkg_xml.write_text(CUSTOM_KEY_PKG_XML) 250 | client = DockerClient() 251 | platform = Platform(arch='aarch64', os_name='ubuntu', ros_distro='foxy') 252 | skip_keys = ['definitely_does_not_exist'] 253 | try: 254 | gather_rosdeps(client, platform, workspace=ws, skip_rosdep_keys=skip_keys) 255 | except docker.errors.ContainerError: 256 | assert False 257 | 258 | 259 | @uses_docker 260 | def test_dummy_skip_rosdep_multiple_keys_pkg(tmpdir): 261 | ws = Path(str(tmpdir)) 262 | pkg_xml = ws / 'src' / 'dummy' / 'package.xml' 263 | pkg_xml.parent.mkdir(parents=True) 264 | pkg_xml.write_text(RCLCPP_PKG_XML) 265 | client = DockerClient() 266 | platform = Platform(arch='aarch64', os_name='ubuntu', ros_distro='foxy') 267 | out_script = ws / rosdep_install_script(platform) 268 | 269 | skip_keys = ['ament_cmake', 'rclcpp'] 270 | gather_rosdeps(client, platform, workspace=ws, skip_rosdep_keys=skip_keys) 271 | result = out_script.read_text() 272 | assert 'ros-foxy-ament-cmake' not in result 273 | assert 'ros-foxy-rclcpp' not in result 274 | 275 | 276 | def test_dependencies_stage_creation(): 277 | temp_stage = CollectDependencyListStage() 278 | assert temp_stage 279 | 280 | 281 | def test_dependencies_stage_name(): 282 | temp_stage = CollectDependencyListStage() 283 | assert temp_stage._name == gather_rosdeps.__name__ 284 | -------------------------------------------------------------------------------- /test/test_docker_client.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import unittest 16 | from unittest.mock import patch 17 | 18 | import docker 19 | import pytest 20 | 21 | from ros_cross_compile.docker_client import DockerClient 22 | 23 | from .utilities import uses_docker 24 | 25 | 26 | class DockerClientTest(unittest.TestCase): 27 | 28 | def remove_container(self, client: docker.client.DockerClient, name: str) -> None: 29 | try: 30 | preexisting_container = client.containers.get(name) 31 | preexisting_container.remove() 32 | except docker.errors.NotFound: 33 | pass 34 | 35 | def test_parse_docker_build_output(self): 36 | """Test the SysrootCreator constructor assuming valid path setup.""" 37 | # Create mock directories and files 38 | client = DockerClient() 39 | log_generator_without_errors = [ 40 | {'stream': ' ---\\u003e a9eb17255234\\n'}, 41 | {'stream': 'Step 1 : VOLUME /data\\n'}, 42 | {'stream': ' ---\\u003e Running in abdc1e6896c6\\n'}, 43 | {'stream': ' ---\\u003e 713bca62012e\\n'}, 44 | {'stream': 'Removing intermediate container abdc1e6896c6\\n'}, 45 | {'stream': 'Step 2 : CMD [\\"/bin/sh\\"]\\n'}, 46 | {'stream': ' ---\\u003e Running in dba30f2a1a7e\\n'}, 47 | {'stream': ' ---\\u003e 032b8b2855fc\\n'}, 48 | {'stream': 'Removing intermediate container dba30f2a1a7e\\n'}, 49 | {'stream': 'Successfully built 032b8b2855fc\\n'}, 50 | ] 51 | # Just expect it not to raise 52 | client._process_build_log(log_generator_without_errors) 53 | 54 | log_generator_with_errors = [ 55 | {'stream': ' ---\\u003e a9eb17255234\\n'}, 56 | {'stream': 'Step 1 : VOLUME /data\\n'}, 57 | {'stream': ' ---\\u003e Running in abdc1e6896c6\\n'}, 58 | {'stream': ' ---\\u003e 713bca62012e\\n'}, 59 | {'stream': 'Removing intermediate container abdc1e6896c6\\n'}, 60 | {'stream': 'Step 2 : CMD [\\"/bin/sh\\"]\\n'}, 61 | {'error': ' ---\\COMMAND NOT FOUND\\n'}, 62 | ] 63 | with pytest.raises(docker.errors.BuildError): 64 | client._process_build_log(log_generator_with_errors) 65 | 66 | @uses_docker 67 | def test_fail_docker_run(self): 68 | client = DockerClient() 69 | with pytest.raises(docker.errors.ContainerError): 70 | client.run_container('ubuntu:18.04', command='/bin/sh -c "exit 1"') 71 | 72 | @uses_docker 73 | def test_stream(self): 74 | client = DockerClient() 75 | test_command = 'echo message1 && sleep 2 && echo message2' 76 | with self.assertLogs('Docker Client', level='INFO') as cm: 77 | client.run_container('ubuntu:18.04', command='/bin/sh -c "{}"'.format(test_command)) 78 | 79 | timestamps = [r.created for r in cm.records] 80 | assert len(timestamps) == 2, 'Did not receive all expected messages' 81 | 82 | # we know for sure that the logs were streaming if we printed them with a gap between 83 | # we do not check for the full 2 seconds because sleep is not that precise, 84 | # but even on the slowest test environment consecutive prints will not take a full second 85 | timediff = timestamps[1] - timestamps[0] 86 | assert timediff >= 1 87 | 88 | @uses_docker 89 | def test_removal(self): 90 | client = DockerClient() 91 | api_client = client._client 92 | name = 'test_removing_run_container' 93 | self.remove_container(api_client, name) 94 | 95 | client.run_container( 96 | 'ubuntu:18.04', command='/bin/sh -c "echo hello"', container_name=name) 97 | 98 | containers = api_client.containers.list(all=True, filters={'name': name}) 99 | assert len(containers) == 0 100 | 101 | @uses_docker 102 | def test_removal_on_failure(self): 103 | client = DockerClient() 104 | api_client = client._client 105 | name = 'test_removing_failed_container' 106 | self.remove_container(api_client, name) 107 | 108 | def mock_logs(*args, **kwargs): 109 | raise docker.errors.APIError('Logs raises an exception in this test') 110 | 111 | with patch('docker.models.containers.Container.logs', side_effect=mock_logs): 112 | with pytest.raises(docker.errors.APIError): 113 | client.run_container( 114 | 'ubuntu:18.04', command='/bin/sh -c "echo hello"', container_name=name) 115 | 116 | containers = client._client.containers.list(all=True, filters={'name': name}) 117 | assert len(containers) == 0 118 | -------------------------------------------------------------------------------- /test/test_entrypoint.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import contextlib 16 | import os 17 | from pathlib import Path 18 | from unittest.mock import Mock 19 | from unittest.mock import patch 20 | 21 | import pytest 22 | 23 | # from ros_cross_compile.docker_client import DockerClient 24 | from ros_cross_compile.data_collector import DataCollector 25 | from ros_cross_compile.dependencies import assert_install_rosdep_script_exists 26 | from ros_cross_compile.dependencies import rosdep_install_script 27 | from ros_cross_compile.platform import Platform 28 | from ros_cross_compile.ros_cross_compile import cross_compile_pipeline 29 | from ros_cross_compile.ros_cross_compile import parse_args 30 | 31 | 32 | @contextlib.contextmanager 33 | def chdir(dirname: str): 34 | """Provide a "with" statement for changing the working directory.""" 35 | curdir = os.getcwd() 36 | try: 37 | os.chdir(dirname) 38 | yield 39 | finally: 40 | os.chdir(curdir) 41 | 42 | 43 | def test_trivial_argparse(): 44 | args = parse_args(['somepath', '-a', 'aarch64', '-o', 'ubuntu']) 45 | assert args 46 | 47 | 48 | def test_bad_workspace(tmpdir): 49 | args = parse_args([str(tmpdir), '-a', 'aarch64', '-o', 'ubuntu', '-d', 'foxy']) 50 | test_collector = DataCollector() 51 | platform = Platform(args.arch, args.os, args.rosdistro) 52 | with pytest.raises(ValueError): 53 | cross_compile_pipeline(args, test_collector, platform) 54 | 55 | 56 | def test_relative_workspace(tmpdir): 57 | # Change directory to the tmp dir and invoke using '.' as the 58 | # workspace to check if relative paths work 59 | tmp = Path(str(tmpdir)) 60 | test_collector = DataCollector() 61 | (tmp / 'src').mkdir() 62 | relative_dir = '.' 63 | args = parse_args([relative_dir, '-a', 'aarch64', '-o', 'ubuntu', '-d', 'foxy']) 64 | platform = Platform(args.arch, args.os, args.rosdistro) 65 | with chdir(str(tmp)), patch( 66 | 'ros_cross_compile.ros_cross_compile.DockerClient', Mock() 67 | ), patch( 68 | 'ros_cross_compile.dependencies.assert_install_rosdep_script_exists' 69 | ): 70 | # should not raise an exception 71 | cross_compile_pipeline(args, test_collector, platform) 72 | 73 | 74 | def test_mocked_cc_pipeline(tmpdir): 75 | tmp = Path(str(tmpdir)) 76 | test_collector = DataCollector() 77 | (tmp / 'src').mkdir() 78 | args = parse_args([str(tmpdir), '-a', 'aarch64', '-o', 'ubuntu']) 79 | platform = Platform(args.arch, args.os, args.rosdistro) 80 | with patch( 81 | 'ros_cross_compile.ros_cross_compile.DockerClient', Mock() 82 | ) as docker_mock, patch( 83 | 'ros_cross_compile.dependencies.assert_install_rosdep_script_exists' 84 | ) as script_mock: 85 | cross_compile_pipeline(args, test_collector, platform) 86 | assert script_mock.called 87 | assert docker_mock.called 88 | assert docker_mock().build_image.call_count == 2 89 | assert docker_mock().run_container.call_count == 2 90 | 91 | 92 | def test_install_rosdep_script_exist(tmpdir): 93 | ws = Path(str(tmpdir)) 94 | platform = Platform('aarch64', 'ubuntu', 'foxy') 95 | data_file = ws / rosdep_install_script(platform) 96 | data_file.parent.mkdir(parents=True) 97 | data_file.touch() 98 | check_script = assert_install_rosdep_script_exists(ws, platform) 99 | assert check_script 100 | 101 | 102 | def test_install_rosdep_script_doesnot_exist(tmpdir): 103 | ws = Path(str(tmpdir)) 104 | platform = Platform('aarch64', 'ubuntu', 'foxy') 105 | data_file = ws / rosdep_install_script(platform) 106 | data_file.parent.mkdir(parents=True) 107 | with pytest.raises(RuntimeError): 108 | assert_install_rosdep_script_exists(ws, platform) 109 | -------------------------------------------------------------------------------- /test/test_flake8.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import logging 16 | 17 | from ament_flake8.main import main 18 | import pytest 19 | 20 | 21 | @pytest.mark.flake8 22 | @pytest.mark.linter 23 | def test_flake8(): 24 | logging.getLogger('flake8').setLevel(logging.INFO) 25 | rc = main(argv=[]) 26 | assert rc == 0, 'Found errors' 27 | -------------------------------------------------------------------------------- /test/test_mypy.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from ament_mypy.main import main 16 | import pytest 17 | 18 | 19 | @pytest.mark.mypy 20 | @pytest.mark.linter 21 | def test_mypy(): 22 | assert main(argv=[]) == 0, 'Found errors' 23 | -------------------------------------------------------------------------------- /test/test_pep257.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from ament_pep257.main import main 16 | import pytest 17 | 18 | 19 | @pytest.mark.linter 20 | @pytest.mark.pep257 21 | def test_pep257(): 22 | rc = main(argv=['.']) 23 | assert rc == 0, 'Found code style errors / warnings' 24 | -------------------------------------------------------------------------------- /test/test_pipeline_stages.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from pathlib import Path 16 | 17 | from ros_cross_compile.pipeline_stages import PipelineStageOptions 18 | 19 | 20 | def test_config_tuple_creation(): 21 | test_options = PipelineStageOptions( 22 | skip_rosdep_keys=['abc', 'def'], 23 | custom_script=Path('./'), 24 | custom_data_dir=Path('./'), 25 | custom_setup_script=Path('./'), 26 | runtime_tag='foo') 27 | assert test_options 28 | -------------------------------------------------------------------------------- /test/test_platform.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | """Unit tests for the `create_cc_sysroot.py` script.""" 17 | 18 | import getpass 19 | 20 | import docker 21 | import pytest 22 | 23 | from ros_cross_compile.sysroot_creator import Platform 24 | 25 | 26 | @pytest.fixture 27 | def platform_config() -> Platform: 28 | return Platform( 29 | arch='aarch64', 30 | os_name='ubuntu', 31 | ros_distro='foxy') 32 | 33 | 34 | def test_platform_argument_validation(): 35 | p = Platform('armhf', 'ubuntu', 'foxy') 36 | assert p 37 | 38 | with pytest.raises(ValueError): 39 | # invalid arch 40 | p = Platform('mips', 'ubuntu', 'foxy') 41 | 42 | with pytest.raises(ValueError): 43 | # invalid distro 44 | p = Platform('armhf', 'ubuntu', 'ardent') 45 | 46 | with pytest.raises(ValueError): 47 | # invalid OS 48 | p = Platform('armhf', 'rhel', 'foxy') 49 | 50 | 51 | def test_construct_x86(): 52 | p = Platform('x86_64', 'ubuntu', 'foxy') 53 | assert p 54 | 55 | 56 | def test_sysroot_image_tag(platform_config): 57 | """Make sure the image tag is created correctly.""" 58 | image_tag = platform_config.sysroot_image_tag 59 | test_tag = '{}/{}:latest'.format(getpass.getuser(), str(platform_config)) 60 | assert isinstance(image_tag, str) 61 | assert image_tag == test_tag 62 | 63 | 64 | def verify_base_docker_images(arch, os_name, rosdistro, image_name): 65 | """Assert correct base image is generated.""" 66 | platform = Platform(arch, os_name, rosdistro) 67 | assert platform.target_base_image == image_name 68 | 69 | 70 | def test_get_docker_base_image(): 71 | """Test that the correct base docker image is used for all arguments.""" 72 | verify_base_docker_images('aarch64', 'ubuntu', 'foxy', 'arm64v8/ubuntu:focal') 73 | verify_base_docker_images('aarch64', 'ubuntu', 'melodic', 'arm64v8/ubuntu:bionic') 74 | verify_base_docker_images('aarch64', 'ubuntu', 'noetic', 'arm64v8/ubuntu:focal') 75 | 76 | verify_base_docker_images('aarch64', 'debian', 'foxy', 'arm64v8/debian:buster') 77 | verify_base_docker_images('aarch64', 'debian', 'melodic', 'arm64v8/debian:stretch') 78 | verify_base_docker_images('aarch64', 'debian', 'noetic', 'arm64v8/debian:buster') 79 | 80 | verify_base_docker_images('armhf', 'ubuntu', 'foxy', 'arm32v7/ubuntu:focal') 81 | verify_base_docker_images('armhf', 'ubuntu', 'melodic', 'arm32v7/ubuntu:bionic') 82 | verify_base_docker_images('armhf', 'ubuntu', 'noetic', 'arm32v7/ubuntu:focal') 83 | 84 | verify_base_docker_images('armhf', 'debian', 'foxy', 'arm32v7/debian:buster') 85 | verify_base_docker_images('armhf', 'debian', 'melodic', 'arm32v7/debian:stretch') 86 | verify_base_docker_images('armhf', 'debian', 'noetic', 'arm32v7/debian:buster') 87 | 88 | 89 | def test_docker_py_version(): 90 | # Explicitly check a known difference between apt and pip versions 91 | with pytest.raises(TypeError): 92 | # 1.20 (from pip, which we are not using) API has named arguments 93 | err = docker.errors.BuildError(reason='problem', build_log='stuff that happened') 94 | 95 | # 1.10 API (from apt which we are using) does not 96 | err = docker.errors.BuildError('problem') 97 | assert err 98 | 99 | 100 | def test_ros_version_map(): 101 | platform = Platform('armhf', 'ubuntu', 'foxy') 102 | assert platform.ros_version == 'ros2' 103 | platform = Platform('x86_64', 'ubuntu', 'foxy') 104 | assert platform.ros_version == 'ros2' 105 | platform = Platform('armhf', 'ubuntu', 'melodic') 106 | assert platform.ros_version == 'ros' 107 | platform = Platform('aarch64', 'ubuntu', 'noetic') 108 | assert platform.ros_version == 'ros' 109 | 110 | 111 | def test_override_base(): 112 | override = 'arm128v12/ubuntu:quintessential' 113 | platform = Platform('aarch64', 'ubuntu', 'foxy', override) 114 | assert platform.target_base_image == override 115 | -------------------------------------------------------------------------------- /test/test_sysroot_creator.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | """Unit tests for the `create_cc_sysroot.py` script.""" 17 | 18 | import os 19 | from pathlib import Path 20 | from platform import system 21 | from unittest.mock import Mock 22 | from unittest.mock import patch 23 | 24 | import pytest 25 | 26 | from ros_cross_compile.platform import Platform 27 | from ros_cross_compile.sysroot_creator import CreateSysrootStage 28 | from ros_cross_compile.sysroot_creator import prepare_docker_build_environment 29 | from ros_cross_compile.sysroot_creator import setup_emulator 30 | 31 | from .utilities import default_pipeline_options 32 | 33 | THIS_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) 34 | 35 | 36 | @patch('ros_cross_compile.sysroot_creator.py_platform.system', side_effect=lambda: 'Linux') 37 | def test_emulator_not_installed(system_mock, tmpdir): 38 | with pytest.raises(RuntimeError): 39 | setup_emulator('not-an-arch', Path(str(tmpdir))) 40 | 41 | 42 | @patch('ros_cross_compile.sysroot_creator.py_platform.system', side_effect=lambda: 'Darwin') 43 | def test_emulator_touch(system_mock, tmpdir): 44 | setup_emulator('aarch64', Path(str(tmpdir))) 45 | 46 | 47 | def test_prepare_docker_build_basic(tmpdir): 48 | platform = Platform('armhf', 'debian', 'melodic') 49 | tmp = Path(str(tmpdir)) 50 | out_dir = prepare_docker_build_environment(platform, tmp, None, None) 51 | 52 | if system() != 'Darwin': 53 | assert (out_dir / 'bin' / 'qemu-arm-static').exists() 54 | assert (out_dir / 'rosdep.Dockerfile').exists() 55 | assert (out_dir / 'sysroot.Dockerfile').exists() 56 | 57 | 58 | def test_run_twice(tmpdir): 59 | # The test is that this doesn't throw an exception for already existing paths 60 | platform = Platform('armhf', 'debian', 'noetic') 61 | tmp = Path(str(tmpdir)) 62 | prepare_docker_build_environment(platform, tmp, None, None) 63 | prepare_docker_build_environment(platform, tmp, None, None) 64 | 65 | 66 | def test_prepare_docker_build_with_user_custom(tmpdir): 67 | platform = Platform('aarch64', 'ubuntu', 'foxy') 68 | tmp = Path(str(tmpdir)) 69 | this_dir = Path(__file__).parent 70 | out_dir = prepare_docker_build_environment( 71 | platform, tmp, 72 | custom_data_dir=this_dir / 'data', 73 | custom_setup_script=this_dir / 'user-custom-setup', 74 | ) 75 | 76 | assert (out_dir / 'bin' / 'qemu-aarch64-static').exists() 77 | assert (out_dir / 'rosdep_focal.Dockerfile').exists() 78 | assert (out_dir / 'sysroot.Dockerfile').exists() 79 | assert (out_dir / 'custom-data' / 'arbitrary.txt') 80 | assert (out_dir / 'user-custom-setup') 81 | 82 | 83 | def test_basic_sysroot_creation(tmpdir): 84 | """Very simple smoke test to validate that syntax is correct.""" 85 | # Very simple smoke test to validate that all internal syntax is correct 86 | 87 | mock_docker_client = Mock() 88 | mock_data_collector = Mock() 89 | platform = Platform('aarch64', 'ubuntu', 'foxy') 90 | 91 | stage = CreateSysrootStage() 92 | stage( 93 | platform, 94 | mock_docker_client, 95 | Path('dummy_path'), 96 | default_pipeline_options(), 97 | mock_data_collector) 98 | assert mock_docker_client.build_image.call_count == 1 99 | 100 | 101 | def test_create_sysroot_stage_creation(): 102 | temp_stage = CreateSysrootStage() 103 | assert temp_stage 104 | -------------------------------------------------------------------------------- /test/user-custom-setup: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ros-tooling/cross_compile/1cedd079950bb184b8ae76198a1b92a8a8229912/test/user-custom-setup -------------------------------------------------------------------------------- /test/utilities.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import functools 16 | import platform 17 | 18 | import pytest 19 | 20 | from ros_cross_compile.pipeline_stages import PipelineStageOptions 21 | 22 | 23 | def uses_docker(func): 24 | """Decorate test to be skipped on platforms that don't have Docker for testing.""" 25 | NO_MAC_REASON = 'CI environment cannot install Docker on Mac OS hosts.' 26 | IS_MAC = platform.system() == 'Darwin' 27 | 28 | @functools.wraps(func) 29 | @pytest.mark.skipif(IS_MAC, reason=NO_MAC_REASON) 30 | def wrapper(*args, **kwargs): 31 | return func(*args, **kwargs) 32 | return wrapper 33 | 34 | 35 | def default_pipeline_options() -> PipelineStageOptions: 36 | return PipelineStageOptions( 37 | skip_rosdep_keys=[], 38 | custom_script=None, 39 | custom_data_dir=None, 40 | custom_setup_script=None, 41 | runtime_tag=None) 42 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py37 3 | 4 | [testenv] 5 | deps = 6 | git+https://github.com/ament/ament_lint.git@0.8.1#egg=ament-copyright&subdirectory=ament_copyright 7 | git+https://github.com/ament/ament_lint.git@0.8.1#egg=ament-flake8&subdirectory=ament_flake8 8 | git+https://github.com/ament/ament_lint.git@0.8.1#egg=ament-lint&subdirectory=ament_lint 9 | git+https://github.com/ament/ament_lint.git@0.8.1#egg=ament-mypy&subdirectory=ament_mypy 10 | git+https://github.com/ament/ament_lint.git@0.8.1#egg=ament-pep257&subdirectory=ament_pep257 11 | flake8<3.8 12 | flake8-blind-except 13 | flake8-builtins 14 | flake8-class-newline 15 | flake8-comprehensions 16 | flake8-deprecated 17 | flake8-docstrings 18 | flake8-import-order 19 | flake8-quotes 20 | mypy 21 | pydocstyle 22 | pytest 23 | pytest-cov 24 | pytest-repeat 25 | pytest-runner 26 | types-pkg_resources 27 | yamllint 28 | commands = 29 | pytest --basetemp="{envtmpdir}" --cov=ros_cross_compile --cov-report=xml test/ {posargs} 30 | 31 | [pytest] 32 | log_cli = true 33 | log_cli_level = DEBUG 34 | --------------------------------------------------------------------------------