├── .doxygen └── Doxyfile ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── custom.md │ └── feature_request.md ├── actions │ ├── generate-doxybook │ │ └── action.yml │ └── generate-doxygen │ │ └── action.yml └── workflows │ ├── build_docs.yml │ └── build_test.yml ├── .gitignore ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── autoware_reference_system ├── CHANGELOG.rst ├── CMakeLists.txt ├── README.md ├── autoware_reference_system.dot ├── cmake │ └── test_requirements.cmake ├── include │ └── autoware_reference_system │ │ ├── autoware_system_builder.hpp │ │ ├── priorities.hpp │ │ └── system │ │ └── timing │ │ ├── benchmark.hpp │ │ └── default.hpp ├── package.xml ├── scripts │ └── benchmark.py ├── src │ ├── number_cruncher_benchmark.cpp │ ├── priorities.cpp │ └── ros2 │ │ └── executor │ │ ├── autoware_default_cbg.cpp │ │ ├── autoware_default_multithreaded.cpp │ │ ├── autoware_default_prioritized.cpp │ │ ├── autoware_default_singlethreaded.cpp │ │ └── autoware_default_staticsinglethreaded.cpp └── test │ ├── conftest.py │ ├── test_autoware_reference_system.cpp │ ├── test_platform.py │ └── test_requirements.py ├── content └── img │ └── autoware_reference_system.svg ├── docs ├── .pages ├── doxybook2_config.json ├── index.md ├── overrides │ └── main.html ├── reports │ └── index.md ├── requirements.txt ├── the-autoware-reference-system.md └── the-reference-system.md ├── mkdocs.yml ├── reference_interfaces ├── CHANGELOG.rst ├── CMakeLists.txt ├── msg │ ├── Message4kb.idl │ └── TransmissionStats.idl └── package.xml ├── reference_system ├── CHANGELOG.rst ├── CMakeLists.txt ├── README.md ├── cfg │ ├── memory_report_template.md │ └── std_report_template.md ├── include │ └── reference_system │ │ ├── msg_types.hpp │ │ ├── nodes │ │ ├── rclcpp │ │ │ ├── command.hpp │ │ │ ├── cyclic.hpp │ │ │ ├── fusion.hpp │ │ │ ├── intersection.hpp │ │ │ ├── sensor.hpp │ │ │ └── transform.hpp │ │ └── settings.hpp │ │ ├── number_cruncher.hpp │ │ ├── sample_management.hpp │ │ └── system │ │ └── type │ │ └── rclcpp_system.hpp ├── package.xml ├── pyproject.toml ├── reference_system_py │ ├── __init__.py │ ├── benchmark.py │ ├── callback_duration.py │ ├── constants.py │ ├── dropped_messages.py │ ├── memory_usage.py │ ├── plot_utils.py │ ├── report.py │ ├── std_latency.py │ └── trace_utils.py ├── resource │ └── reference_system_py ├── setup.cfg ├── setup.py └── test │ ├── gtest_main.cpp │ ├── test_fixtures.hpp │ ├── test_number_cruncher.cpp │ ├── test_reference_system_rclcpp.cpp │ └── test_sample_management.cpp └── requirements.txt /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Desktop (please complete the following information):** 27 | - OS: [e.g. iOS] 28 | - Browser [e.g. chrome, safari] 29 | - Version [e.g. 22] 30 | 31 | **Smartphone (please complete the following information):** 32 | - Device: [e.g. iPhone6] 33 | - OS: [e.g. iOS8.1] 34 | - Browser [e.g. stock browser, safari] 35 | - Version [e.g. 22] 36 | 37 | **Additional context** 38 | Add any other context about the problem here. 39 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/custom.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Custom issue template 3 | about: Describe this issue template's purpose here. 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | 11 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.github/actions/generate-doxybook/action.yml: -------------------------------------------------------------------------------- 1 | name: Generate doxybook2 2 | description: Generate markdown using doxybook2 with given configuration 3 | inputs: 4 | input-doxygen-artifact: 5 | required: true 6 | description: Name of input doxygen artifact to download 7 | doxygen-artifact-extraction-path: 8 | required: true 9 | description: Path to extract input doxygen artifact to 10 | doxybook2-version: 11 | required: true # TODO: make this optional 12 | description: Version of doxybook2 to download and use 13 | output-path: 14 | required: true 15 | description: The path to generate the doxybook2 markdown to 16 | doxybook2-config-path: 17 | required: true # TODO: make this optional with smart default 18 | description: Path to the doxybook2 configuration file (including .json extension) 19 | base-url: 20 | required: true # TODO: make this optional by adding logic below 21 | description: Base URL to overwrite the default with 22 | artifact-path: 23 | required: true 24 | description: Path to directory to upload as artifact 25 | artifact-name: 26 | required: true 27 | description: Name of the artifact to upload 28 | artifact-retention-days: 29 | required: true 30 | description: Number of days to keep the artifact for 31 | runs: 32 | using: composite 33 | steps: 34 | - name: Download Doxygen XML 35 | uses: actions/download-artifact@v3 36 | with: 37 | name: ${{ inputs.input-doxygen-artifact }} 38 | path: ${{ inputs.doxygen-artifact-extraction-path }} 39 | - name: Build API Reference 40 | shell: bash 41 | run: | 42 | # ensure output directory exists 43 | mkdir -p ${{ inputs.output-path }} 44 | export DOXYBOOK_URL=https://github.com/matusnovak/doxybook2/releases/download 45 | export DOXYBOOK_URL=${DOXYBOOK_URL}/${{ inputs.doxybook2-version }} 46 | export DOXYBOOK_ZIP=doxybook2-linux-amd64-${{ inputs.doxybook2-version }}.zip 47 | wget ${DOXYBOOK_URL}/${DOXYBOOK_ZIP} 48 | sudo apt-get install unzip 49 | unzip ${DOXYBOOK_ZIP} 50 | ./bin/doxybook2 --input ${{ inputs.doxygen-artifact-extraction-path }} \ 51 | --output ${{ inputs.output-path }} \ 52 | --config ${{ inputs.doxybook2-config-path }} \ 53 | --config-data '{"baseUrl": "${{ inputs.base-url }}"}' 54 | - name: Upload API reference 55 | uses: actions/upload-artifact@v3 56 | with: 57 | name: ${{ inputs.artifact-name }} 58 | path: ${{ inputs.artifact-path }} 59 | retention-days: ${{ inputs.artifact-retention-days }} -------------------------------------------------------------------------------- /.github/actions/generate-doxygen/action.yml: -------------------------------------------------------------------------------- 1 | name: Generate doxygen documentation and upload as artifact 2 | description: Build doxygen documentation with given configuration 3 | inputs: 4 | working-directory: 5 | required: true 6 | description: Working directory to run doxygen from 7 | doxyfile-path: 8 | required: true 9 | description: The path to checkout the given repository too 10 | artifact-path: 11 | required: true 12 | description: Path to directory to upload as artifact 13 | artifact-name: 14 | required: true 15 | description: Name of the artifact to upload 16 | artifact-retention-days: 17 | required: true 18 | description: Number of days to keep the artifact for 19 | runs: 20 | using: composite 21 | steps: 22 | - name: Run doxygen on source 23 | uses: mattnotmitt/doxygen-action@v1 24 | with: 25 | working-directory: ${{ inputs.working-directory }} 26 | doxyfile-path: ${{ inputs.doxyfile-path }} 27 | - name: Upload Doxygen XML 28 | uses: actions/upload-artifact@v3 29 | with: 30 | name: ${{ inputs.artifact-name }} 31 | path: ${{ inputs.artifact-path }} 32 | retention-days: ${{ inputs.artifact-retention-days }} -------------------------------------------------------------------------------- /.github/workflows/build_docs.yml: -------------------------------------------------------------------------------- 1 | name: Build Documentation 2 | 3 | on: 4 | workflow_run: 5 | workflows: [Build and test] 6 | types: [completed] 7 | branches: [main] 8 | pull_request: 9 | branches: 10 | - main 11 | workflow_dispatch: 12 | 13 | env: 14 | WORKSPACE_PATH: ros_ws/src/reference_system 15 | DOXYGEN_ARTIFACT: doxygen_xml 16 | DOXYBOOK_ARTIFACT: api_reference 17 | DOXYBOOK_VERSION: v1.4.0 18 | 19 | # only run one build doc workflow at a time, cancel any running ones 20 | concurrency: 21 | group: ${{ github.workflow }}-${{ github.ref }} 22 | cancel-in-progress: true 23 | 24 | jobs: 25 | generate-doxygen: 26 | runs-on: ubuntu-latest 27 | container: 28 | image: rostooling/setup-ros-docker:ubuntu-focal-ros-galactic-ros-base-latest 29 | steps: 30 | - name: Make sure output directory exists 31 | run: mkdir -p ${{ env.WORKSPACE_PATH }} 32 | - name: Checkout repository 33 | uses: actions/checkout@v2 34 | with: 35 | path: ${{ env.WORKSPACE_PATH }} 36 | - name: Generate doxygen and upload as artifact 37 | # TODO: figure out way to use WORKSPACE_PATH var here 38 | uses: ./ros_ws/src/reference_system/.github/actions/generate-doxygen 39 | with: 40 | working-directory: ${{ env.WORKSPACE_PATH }} 41 | doxyfile-path: '.doxygen/Doxyfile' 42 | artifact-path: ${{ env.WORKSPACE_PATH }}/docs/xml 43 | artifact-name: ${{ env.DOXYGEN_ARTIFACT }} 44 | artifact-retention-days: 30 45 | generate-doxybook2: 46 | runs-on: ubuntu-latest 47 | needs: generate-doxygen 48 | steps: 49 | - name: Make sure output directory exists 50 | run: mkdir -p ${{ env.WORKSPACE_PATH }} 51 | - name: Checkout repository 52 | uses: actions/checkout@v2 53 | with: 54 | path: ${{ env.WORKSPACE_PATH }} 55 | - name: Generate API reference and upload as artifact 56 | # TODO: figure out way to use WORKSPACE_PATH var here 57 | uses: ./ros_ws/src/reference_system/.github/actions/generate-doxybook 58 | with: 59 | input-doxygen-artifact: ${{ env.DOXYGEN_ARTIFACT }} 60 | doxygen-artifact-extraction-path: ${{ env.WORKSPACE_PATH }}/docs/xml 61 | doxybook2-version: ${{ env.DOXYBOOK_VERSION }} 62 | doxybook2-config-path: ${{ env.WORKSPACE_PATH }}/docs/doxybook2_config.json 63 | output-path: ${{ env.WORKSPACE_PATH }}/docs/api-reference 64 | base-url: /reference-system/latest/api-reference/ 65 | artifact-path: ${{ env.WORKSPACE_PATH }}/docs/api-reference 66 | artifact-name: ${{ env.DOXYBOOK_ARTIFACT }} 67 | artifact-retention-days: 30 68 | build-mkdocs: 69 | runs-on: ubuntu-latest 70 | if: github.ref != 'refs/heads/main' 71 | needs: generate-doxybook2 72 | steps: 73 | - name: Make sure output directory exists 74 | run: mkdir -p ${{ env.WORKSPACE_PATH }} 75 | - name: Checkout repository 76 | uses: actions/checkout@v2 77 | with: 78 | path: ${{ env.WORKSPACE_PATH }} 79 | - name: Download API Reference 80 | uses: actions/download-artifact@v3 81 | with: 82 | name: ${{ env.DOXYBOOK_ARTIFACT }} 83 | path: ${{ env.WORKSPACE_PATH }}/docs/api-reference 84 | - name: Build mkdocs site 85 | run: | 86 | cd ${{ env.WORKSPACE_PATH }} 87 | # ensure gh-pages git history is fetched 88 | git fetch origin gh-pages --depth=1 89 | sudo apt-get update -y 90 | # install mkdocs dependencies 91 | python3 -m pip install -r docs/requirements.txt 92 | # build site 93 | mkdocs build 94 | - name: Upload docs site 95 | uses: actions/upload-artifact@v3 96 | with: 97 | name: reference_system_site 98 | path: ${{ env.WORKSPACE_PATH }}/site 99 | run_benchmarks: 100 | runs-on: ubuntu-latest 101 | if: github.ref == 'refs/heads/main' # Only run benchmarks on `main` 102 | continue-on-error: true 103 | strategy: 104 | fail-fast: false 105 | matrix: 106 | ros_distribution: 107 | - humble 108 | - iron 109 | - rolling 110 | include: 111 | # Humble Hawksbill (May 2022 - May 2027) 112 | - ros_distribution: humble 113 | docker_image: rostooling/setup-ros-docker:ubuntu-jammy-ros-humble-ros-base-latest 114 | # Iron Irwini (May 2023 - November 2024) 115 | - ros_distribution: iron 116 | docker_image: rostooling/setup-ros-docker:ubuntu-jammy-ros-iron-ros-base-latest 117 | # Rolling Ridley (June 2020 - Present) 118 | - ros_distribution: rolling 119 | docker_image: rostooling/setup-ros-docker:ubuntu-jammy-ros-rolling-ros-base-latest 120 | container: 121 | image: ${{ matrix.docker_image }} 122 | # if: github.ref == 'refs/heads/main' # Only run benchmarks on `main` 123 | steps: 124 | - name: Make sure output directory exists 125 | run: mkdir -p ${{ env.WORKSPACE_PATH }} 126 | - name: Checkout repository 127 | uses: actions/checkout@v2 128 | with: 129 | path: ${{ env.WORKSPACE_PATH }} 130 | - name: Install benchmark dependencies 131 | shell: bash 132 | run: | 133 | sudo apt-get update -y 134 | python3 -m pip install -r ${{ env.WORKSPACE_PATH }}/requirements.txt 135 | - name: Build and run benchmarks 136 | uses: ros-tooling/action-ros-ci@v0.3 137 | with: 138 | package-name: autoware_reference_system reference_system reference_interfaces 139 | target-ros2-distro: ${{ matrix.ros_distribution }} 140 | vcs-repo-file-url: "" 141 | colcon-defaults: | 142 | { 143 | "build": { 144 | "cmake-args": [ 145 | "-DRUN_BENCHMARK=ON", 146 | "-DALL_RMWS=ON" 147 | ] 148 | } 149 | } 150 | - name: Upload Autoware Benchmark Reports 151 | continue-on-error: true # continue even if report gen failsS 152 | uses: actions/upload-artifact@v3 153 | with: 154 | name: autoware_benchmark_reports_${{ matrix.ros_distribution }} 155 | path: '~/.ros/benchmark_autoware_reference_system' 156 | retention-days: 60 157 | deploy_docs: 158 | runs-on: ubuntu-latest 159 | # only run on main branch after jobs listed in `needs` have finished (successful or not) 160 | if: github.ref == 'refs/heads/main' && always() 161 | needs: [build-mkdocs, run_benchmarks] 162 | steps: 163 | - name: Make sure output directory exists 164 | run: mkdir -p ${{ env.WORKSPACE_PATH }} 165 | - name: Checkout repository 166 | uses: actions/checkout@v2 167 | with: 168 | path: ${{ env.WORKSPACE_PATH }} 169 | - name: Download API reference 170 | uses: actions/download-artifact@v3 171 | with: 172 | name: ${{ env.DOXYBOOK_ARTIFACT }} 173 | path: ${{ env.WORKSPACE_PATH }}/docs/api-reference 174 | - name: Download Foxy reports 175 | continue-on-error: true # Still publish docs even if reports are not available 176 | uses: actions/download-artifact@v3 177 | with: 178 | name: autoware_benchmark_reports_foxy 179 | path: ${{ env.WORKSPACE_PATH }}/docs/reports/foxy 180 | - name: Download Galactic reports 181 | continue-on-error: true # Still publish docs even if reports are not available 182 | uses: actions/download-artifact@v3 183 | with: 184 | name: autoware_benchmark_reports_galactic 185 | path: ${{ env.WORKSPACE_PATH }}/docs/reports/galactic 186 | - name: Download Humble reports 187 | continue-on-error: true # Still publish docs even if reports are not available 188 | uses: actions/download-artifact@v3 189 | with: 190 | name: autoware_benchmark_reports_humble 191 | path: ${{ env.WORKSPACE_PATH }}/docs/reports/humble 192 | - name: Download Rolling reports 193 | continue-on-error: true # Still publish docs even if reports are not available 194 | uses: actions/download-artifact@v3 195 | with: 196 | name: autoware_benchmark_reports_rolling 197 | path: ${{ env.WORKSPACE_PATH }}/docs/reports/rolling 198 | - name: Deploy mkdocs site 199 | shell: bash 200 | run: | 201 | cd ${{ env.WORKSPACE_PATH }} 202 | # ensure gh-pages git history is fetched 203 | git fetch origin gh-pages --depth=1 204 | sudo apt-get update -y 205 | # install docs dependencies 206 | python3 -m pip install -r docs/requirements.txt 207 | # TODO: mike rebuilds entire site, instead we should 208 | # skip the build and download site artifact from previous workflow 209 | if [ -z ${{ github.event.release.tag_name }}]; then 210 | export NEW_VERSION=main 211 | else 212 | export NEW_VERSION=${{ github.event.release.tag_name }} 213 | fi 214 | git config user.name doc-bot 215 | git config user.email doc-bot@ros-realtime.com 216 | mike deploy --push --update-aliases $NEW_VERSION latest 217 | 218 | -------------------------------------------------------------------------------- /.github/workflows/build_test.yml: -------------------------------------------------------------------------------- 1 | name: Build and test 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | tags: 8 | - '**' 9 | pull_request: 10 | branches: 11 | - main 12 | # Allows you to run this workflow manually from the Actions tab 13 | workflow_dispatch: 14 | 15 | env: 16 | WORKSPACE_PATH: ros_ws 17 | CLONE_PATH: ros_ws/src/reference_system 18 | 19 | # only run one build doc workflow at a time, cancel any running ones 20 | concurrency: 21 | group: ${{ github.workflow }}-${{ github.ref }} 22 | cancel-in-progress: true 23 | 24 | jobs: 25 | build-and-test: 26 | runs-on: ubuntu-latest 27 | strategy: 28 | fail-fast: false 29 | matrix: 30 | ros_distribution: 31 | - humble 32 | - iron 33 | - rolling 34 | include: 35 | # Humble Hawksbill (May 2022 - May 2027) 36 | - ros_distribution: humble 37 | docker_image: rostooling/setup-ros-docker:ubuntu-jammy-ros-humble-ros-base-latest 38 | # Iron Irwini (May 2023 - November 2024) 39 | - ros_distribution: iron 40 | docker_image: rostooling/setup-ros-docker:ubuntu-jammy-ros-iron-ros-base-latest 41 | # Rolling Ridley (June 2020 - Present) 42 | - ros_distribution: rolling 43 | docker_image: rostooling/setup-ros-docker:ubuntu-jammy-ros-rolling-ros-base-latest 44 | container: 45 | image: ${{ matrix.docker_image }} 46 | # Steps represent a sequence of tasks that will be executed as part of the job 47 | steps: 48 | - name: Setup workspace 49 | run: mkdir -p ${{ env.CLONE_PATH }} 50 | - name: checkout 51 | uses: actions/checkout@v2 52 | with: 53 | path: ${{ env.CLONE_PATH }} 54 | - name: Build and test 55 | uses: ros-tooling/action-ros-ci@v0.3 56 | with: 57 | package-name: autoware_reference_system reference_system reference_interfaces 58 | target-ros2-distro: ${{ matrix.ros_distribution }} 59 | vcs-repo-file-url: "" 60 | - name: Upload logs as artifacts 61 | uses: actions/upload-artifact@v3 62 | with: 63 | name: build_and_test_logs_${{ matrix.ros_distribution }} 64 | path: ${{ env.WORKSPACE_PATH }}/log -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | **__pycache__ 2 | docs/xml 3 | docs/api-reference 4 | docs/site 5 | 6 | # ignore the doxybook2 stuff if installed 7 | doxybook2** 8 | bin 9 | include 10 | lib 11 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Any contribution that you make to this repository will 2 | be under the Apache 2 License, as dictated by that 3 | [license](http://www.apache.org/licenses/LICENSE-2.0.html): 4 | 5 | ~~~ 6 | 5. Submission of Contributions. Unless You explicitly state otherwise, 7 | any Contribution intentionally submitted for inclusion in the Work 8 | by You to the Licensor shall be under the terms and conditions of 9 | this License, without any additional terms or conditions. 10 | Notwithstanding the above, nothing herein shall supersede or modify 11 | the terms of any separate license agreement you may have executed 12 | with Licensor regarding such Contributions. 13 | ~~~ 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # The reference system 2 | 3 | With the distributed development of ROS across many different organizations it is sometimes hard to 4 | benchmark and concretely show how a certain change to a certain system improves or 5 | reduces the performance of that system. 6 | For example did a change from one executor to another actually 7 | reduce the CPU or was it something else entirely? 8 | 9 | [The `reference_system` package](reference_system/README.md) was developed to provide the 10 | fundamental building blocks to create complex systems that then can be used to 11 | evaluate features or performance in a standardized and repeatable way. 12 | 13 | The first project to use this `reference_system` is [the `autoware_reference_system`.]( 14 | autoware_reference_system/README.md) 15 | 16 | Future _reference systems_ could be proposed that are more complex using the same 17 | basic node building blocks within the `reference_system` package. 18 | 19 | ## Defining a reference system 20 | 21 | A _reference system_ is defined by: 22 | 23 | - A fixed number of nodes 24 | - each node with: 25 | - a fixed number of publishers and subscribers 26 | - a fixed _processing time_ or a fixed _publishing rate_ 27 | - the `reference_system` base package defines [reusable base node types]( 28 | reference_system/README.md#base-node-types) to be used in various reference systems 29 | - A fixed _message type_ of fixed size to be used for every _node_ 30 | - For simplicity and ease of benchmarking, **all nodes must run on a single process** 31 | 32 | Reference systems can run on what is referred to as a [platform](#supported-platforms). 33 | A platform is defined by: 34 | - Hardware (e.g. an off-the-shelf single-board computer, embedded ECU, etc.) 35 | - if there are multiple configurations available for such hardware, ensure it is specified 36 | - Operating System (OS) like RT Linux, QNX, etc. along with any special configurations made 37 | 38 | With these defined attributes the reference system can be replicated across many different possible 39 | configurations to be used to benchmark each configuration against the other in a reliable and fair 40 | manner. 41 | 42 | With this approach [portable and repeatable tests](#testing) can also be defined to reliably confirm 43 | if a given _reference system_ meets the requirements. 44 | 45 | ## Supported Platforms 46 | 47 | To enable as many people as possible to replicate this reference system, the platform(s) were chosen 48 | to be easily accessible (inexpensive, high volume), have lots of documentation, 49 | large community use and will be supported well into the future. 50 | 51 | Platforms were not chosen for performance of the reference system - we know we could run “faster” 52 | with a more powerful CPU or GPU but then it would be harder for others to validate findings 53 | and test their own configurations. 54 | Accessibility is the key here and will be considered if 55 | more platforms want to be added to this benchmark list. 56 | 57 | **Platforms:** 58 | 59 | - [Raspberry Pi 4B](https://www.raspberrypi.org/products/raspberry-pi-4-model-b/): 60 | - 4 GB RAM version is the assumed default 61 | - other versions could also be tested / added by the community 62 | - [real-time Linux kernel](https://github.com/ros-realtime/rt-kernel-docker-builder) 63 | 64 | _Note: create an [issue](https://github.com/ros-realtime/reference-system-autoware/issues/)_ 65 | _to add more platforms to the list, keeping in mind the above criteria_ 66 | 67 | !!! warning 68 | Each reference system can be run on other targets as well 69 | however the results will change drastically depending on the 70 | specifications of the target hardware. 71 | 72 | ## Implemented reference systems 73 | 74 | The first reference system benchmark proposed is based on the *Autoware.Auto* LiDAR data pipeline 75 | as stated above and shown in the node graph image above as well. 76 | 77 | 1. **Autoware Reference System** 78 | - ROS2 79 | - Executors 80 | - Single Threaded 81 | - Static Single Threaded 82 | - Multithreaded 83 | - Callback Group 84 | - Prioritized 85 | 86 | Results below show various characteristics of the same simulated system (Autoware.Auto). 87 | 88 | ## Setup Raspberry Pi 4 for the test 89 | 90 | The goal is to provide a clean computation environment for the test 91 | avoiding an interference of other Ubuntu components. 92 | 93 | ### Setup a constant CPU frequency 94 | 95 | Frequency is setup to 1.50 GHz for all CPUs 96 | 97 | ```console 98 | # run it as root 99 | sudo su 100 | 101 | echo -n "setup constant CPU frequency to 1.50 GHz ... " 102 | # disable ondemand governor 103 | systemctl disable ondemand 104 | 105 | # set performance governor for all cpus 106 | echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor >/dev/null 107 | 108 | # set constant frequency 109 | echo 1500000 | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_min_freq >/dev/null 110 | echo 1500000 | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_max_freq >/dev/null 111 | 112 | # reset frequency counters 113 | echo 1 | tee /sys/devices/system/cpu/cpu*/cpufreq/stats/reset >/dev/null 114 | 115 | echo done 116 | 117 | sleep 1 118 | # get freq info 119 | echo `cpufreq-info | grep stats | cut -d ' ' -f 23-25` 120 | ``` 121 | 122 | ### Isolate CPUs 123 | 124 | CPU 2,3 are isolated to run tests. 125 | 126 | ```console 127 | sudo apt install -y sysstat u-boot-tools 128 | ``` 129 | 130 | ```console 131 | # modify kernel cmdline 132 | cd ~ 133 | dd if=/boot/firmware/boot.scr of=boot.script bs=72 skip=1 134 | 135 | # edit boot.script and modify bootargs to 136 | ubuntu@ubuntu:~$ cat boot.script | grep "setenv bootargs" | head -1 137 | setenv bootargs " ${bootargs} rcu_nocbs=2,3 nohz_full=2,3 isolcpus=2,3 irqaffinity=0,1 audit=0 watchdog=0 skew_tick=1 quiet splash" 138 | 139 | # generate boot.scr 140 | mkimage -A arm64 -O linux -T script -C none -d boot.script boot.scr 141 | 142 | # replace boot.scr 143 | sudo cp boot.scr /boot/firmware/boot.scr 144 | 145 | sudo reboot 146 | 147 | # check cmdline 148 | ubuntu@ubuntu:~$ cat /proc/cmdline 149 | coherent_pool=1M 8250.nr_uarts=1 snd_bcm2835.enable_compat_alsa=0 snd_bcm2835.enable_hdmi=1 bcm2708_fb.fbwidth=0 bcm2708_fb.fbheight=0 bcm2708_fb.fbswap=1 smsc95xx.macaddr=DC:A6:32:2E:5 150 | 4:97 vc_mem.mem_base=0x3ec00000 vc_mem.mem_size=0x40000000 net.ifnames=0 dwc_otg.lpm_enable=0 console=ttyS0,115200 console=tty1 root=LABEL=writable rootfstype=ext4 elevator=deadline roo 151 | twait fixrtc rcu_nocbs=2,3 nohz_full=2,3 isolcpus=2,3 irqaffinity=0,1 audit=0 watchdog=0 skew_tick=1 quiet splash 152 | 153 | # check interrupts 154 | # Only the number of interrupts handled by CPU 0,1 increases. 155 | watch -n1 cat /proc/interrupts 156 | 157 | # check soft interrupts 158 | watch -n1 cat /proc/softirqs 159 | 160 | # check isolated CPUs 161 | cat /sys/devices/system/cpu/isolated 162 | 2-3 163 | cat /sys/devices/system/cpu/present 164 | 0-3 165 | 166 | # run reference system on CPU2 167 | taskset -c 2 install/autoware_reference_system/lib/autoware_reference_system/autoware_default_singlethreaded > /dev/null 168 | 169 | # get pid 170 | RF_PID=`pidof autoware_default_singlethreaded` && cat /proc/$RF_PID/status | grep ^Cpu 171 | 172 | # check how many threads are running 173 | ps -aL | grep $RF_PID 174 | 3835 3835 ttyS0 00:03:46 autoware_defaul 175 | 3835 3836 ttyS0 00:00:00 autoware_defaul 176 | 3835 3837 ttyS0 00:00:00 autoware_defaul 177 | 3835 3838 ttyS0 00:00:00 autoware_defaul 178 | 3835 3839 ttyS0 00:00:00 gc 179 | 3835 3840 ttyS0 00:00:00 dq.builtins 180 | 3835 3841 ttyS0 00:00:00 dq.user 181 | 3835 3842 ttyS0 00:00:00 tev 182 | 3835 3843 ttyS0 00:00:00 recv 183 | 3835 3844 ttyS0 00:00:00 recvMC 184 | 3835 3845 ttyS0 00:00:00 recvUC 185 | 3835 3846 ttyS0 00:00:00 autoware_defaul 186 | ``` 187 | 188 | ## Hints 189 | 190 | - If you run `colcon build` on a Raspberry Pi 4 with little memory, use `export MAKEFLAGS="-j 1"` 191 | to inhibit parallelism. Otherwise, the system could hang due to memory swapping. 192 | -------------------------------------------------------------------------------- /autoware_reference_system/CHANGELOG.rst: -------------------------------------------------------------------------------- 1 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 2 | Changelog for package autoware_reference_system 3 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 4 | 5 | v1.1.0 6 | ------ 7 | * Add Iron ROS 2 distribution 8 | * Remove EoL distributions Foxy and Galactic 9 | * Remove legacy hack for rosidl_geneartor_py 10 | 11 | v1.0.0 12 | ----------- 13 | * Add first changelog 14 | * Bump version of reference_system packages to 1.0.0 15 | * Skip callback group exe if distro is Foxy 16 | * Update reference_system docs, logic in various places 17 | * Migrate benchmark scripts to python 18 | * clean up reporting code, adjust title and label sizes for figures in reports 19 | * [91] add unit and integration tests for the reference system, fix some bugs found by tests 20 | * Added note on super user privileges. 21 | * Adding autoware_default_prioritized and autoware_default_cbg only to test set if super user rights available. 22 | Signed-off-by: Ralph Lange 23 | * Fixed uncrustify finding. 24 | Signed-off-by: Ralph Lange 25 | * Do not exit but print warning if thread prioritization fails. 26 | Signed-off-by: Ralph Lange 27 | * add skip_tracing cmake arg to readme 28 | * update memory individual report text sizes as well 29 | * increase label sizes for figures 30 | * Under Foxy, exclude executable using callback-group interface of Executor. 31 | Signed-off-by: Ralph Lange 32 | * Added executables for prioritized and callback-group-level Executor. 33 | Signed-off-by: Ralph Lange 34 | * default to not run benchmark tests 35 | * Make cpu benchmark timings consistent 36 | * switch to use cmake options 37 | * patch version bump to include mutex for prints 38 | * remove extra line 39 | * fix flake8 errors 40 | * return none not no 41 | * handle case where log file line is incomplete 42 | * initial release for each package 43 | * sort axis labels along with data for latency plots 44 | * only run tests for 5s by default 45 | * update dependency list, add warnings to test section 46 | * update node graph 47 | * clean up reports 48 | * add behavior planner jitter 49 | * use candlesticks to show min, max, and std dev 50 | * add std trace type, generate summary report 51 | * fix dropped message count for now 52 | * apply feedback from pr 53 | * fix flake8 errors 54 | * create node graph from list of tuples 55 | * fix flake8 errors 56 | * rebase, refactor report gen, fix dropped msg count 57 | * clean up report generation code 58 | * add prototype latency figure to report 59 | * fix flake8 errors 60 | * clean up report, add expected_count to hovertool 61 | * begin to add dropped message report 62 | * Add advanced dropped sample statistics for each node 63 | * Add dropped samples statistics 64 | * Add sequence number, behavior planner period, timestamp relates to callback startup 65 | * update node graph and svg 66 | * Lint and uncrustify 67 | * Add intersection node docu 68 | * Add intersection node 69 | * add all_rmw flag to readme 70 | * move cmake functions to separate files, use default rmw 71 | * fix small grammer mistakes in readme 72 | * adjust timings for more typcial run times 73 | * switch each executor to use default timing config 74 | * Reduce list of enabled tracepoints down to minimum 75 | * add default timeout variable to cmake 76 | * add goals/kpis to readme, rearrange testing sections 77 | * add cmake arg to skip lttng tests 78 | * add title to summary reports, fix callback report gen 79 | * fix cpp header guards, timing bug 80 | * add mem and cpu usage summary report 81 | * add cpu/mem usage reports, add hovertool to plots 82 | * use psrecord for mem and cpu usage 83 | * add onto memory_usage report generation 84 | * clean up memory_usage report generation 85 | * fix env var setting for userspace traces, add memory usage events 86 | * fix ROS_HOME usage in cmake 87 | * change name of generated reports to match parent directory 88 | * clean up cmake variable names 89 | * generate callback_duration report 90 | * Uncrustify and adjust number cruncher 91 | * Cyclic node with cyclic trigger implemented 92 | * Rename Reactor node into Cyclic node 93 | * Remove time diff check from fusion node 94 | * Number cruncher uses upper limit instead of timeout, add number cruncher benchmark to help find the best limits for a system 95 | fix generating trace files 96 | * fix remaing lint errors 97 | * fix flake8 errors 98 | * use same timing config for all executors 99 | * fix timeout for generating traces 100 | * fix requirements test, update readme 101 | * add report generation from trace files 102 | * add fusion max input time diff to requirements 103 | * Add Fusion node max input timediff and fail when it is exceeded 104 | * add generate_tracing test 105 | * update README, regenerate svg, add graphviz section to README 106 | * fix lint errors, rename tests 107 | * list exes in cmake and loop over them 108 | * add initial pub/sub test 109 | * rename nodes, update docs, add platform tes 110 | * fix readme links after reorg 111 | * remove cpu and msg tests for now 112 | seperate out reference_system vs autoware specifics 113 | * clean up cmake, reorg READMEs 114 | * seperate out reference_system vs autoware specifics 115 | * Contributors: Christian, Christian Eltzschig, Christophe Bedard, Evan Flynn, Lander Usategui, Ralph Lange 116 | -------------------------------------------------------------------------------- /autoware_reference_system/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.5) 2 | project(autoware_reference_system) 3 | 4 | option(RUN_BENCHMARK "Run the full benchmark tests" OFF) # default to off for CI purposes 5 | option(SKIP_TRACING "Skip the ros2_tracing (LTTng) tests" ON) # default to off until ros-realtime/reference_system#35 is closed 6 | option(TEST_PLATFORM "Test if running on a supported platform" OFF) # default to off for development purposes 7 | option(ALL_RMWS "Run tests for all available RMWs" OFF) # default to off so only one RMW is tested by default 8 | set(FRAMEWORK ros CACHE STRING "The framework to build for. Currently supported is ROS 2 (\"ros\").") 9 | set(AVAILABLE_FRAMEWORKS ros) 10 | 11 | if(NOT ${FRAMEWORK} IN_LIST AVAILABLE_FRAMEWORKS) 12 | message(FATAL_ERROR "Unsupported framework: ${FRAMEWORK}") 13 | endif() 14 | 15 | if(NOT CMAKE_CXX_STANDARD) 16 | set(CMAKE_CXX_STANDARD 17) 17 | endif() 18 | 19 | if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") 20 | add_compile_options(-Wall -Wextra -Wpedantic) 21 | endif() 22 | 23 | find_package(ament_cmake_auto REQUIRED) 24 | ament_auto_find_build_dependencies() 25 | 26 | # Number Cruncher Benchmark 27 | ament_auto_add_executable(number_cruncher_benchmark 28 | src/number_cruncher_benchmark.cpp) 29 | 30 | set(BENCHMARK_EXECUTABLES "") 31 | macro(add_benchmark_executable target src) 32 | ament_auto_add_executable(${target} ${src} src/priorities.cpp) 33 | list(APPEND BENCHMARK_EXECUTABLES ${target}) 34 | endmacro() 35 | 36 | if(${FRAMEWORK} STREQUAL ros) 37 | # Single Threaded Executor 38 | add_benchmark_executable(autoware_default_singlethreaded 39 | src/ros2/executor/autoware_default_singlethreaded.cpp) 40 | # Multi Threaded Executor 41 | add_benchmark_executable(autoware_default_multithreaded 42 | src/ros2/executor/autoware_default_multithreaded.cpp) 43 | 44 | # Static Single Threaded Executor 45 | add_benchmark_executable(autoware_default_staticsinglethreaded 46 | src/ros2/executor/autoware_default_staticsinglethreaded.cpp) 47 | 48 | # Prioritized Executor 49 | add_benchmark_executable(autoware_default_prioritized 50 | src/ros2/executor/autoware_default_prioritized.cpp) 51 | 52 | # API `add_callback_group` not available in Foxy 53 | if(NOT $ENV{ROS_DISTRO} MATCHES "foxy") 54 | # Multiple executors on callback-group-level with suitable prioritization of critical path. 55 | add_benchmark_executable(autoware_default_cbg 56 | src/ros2/executor/autoware_default_cbg.cpp) 57 | endif() 58 | endif() 59 | 60 | # Add new executors to test here 61 | #add_benchmark_executable(autoware_default_custom 62 | # src/ros2/executor/autoware_default_custom.cpp 63 | #) 64 | 65 | if(${BUILD_TESTING}) 66 | find_package(ament_lint_auto REQUIRED) 67 | ament_lint_auto_find_test_dependencies() 68 | 69 | if(${TEST_PLATFORM}) 70 | # check current platform 71 | ament_add_pytest_test(platform_test 72 | test/test_platform.py 73 | TIMEOUT 5 74 | ) 75 | endif() 76 | 77 | if(${RUN_BENCHMARK}) 78 | # Test all executables by default. 79 | # Modify this variable to test only a subset of executables 80 | set(TEST_TARGETS ${BENCHMARK_EXECUTABLES}) 81 | 82 | # Add more run times here (time to run traces for) 83 | set(RUN_TIMES 84 | 5 85 | # 10 86 | # 30 87 | # 60 88 | # 120 89 | ) 90 | 91 | # Add more trace types here 92 | # The CPU trace type is disabled since it produces too many events 93 | # see https://github.com/ros-realtime/reference-system/pull/33#issuecomment-928264240 94 | set(TRACE_TYPES 95 | callback # uses ros2_tracing, LTTng 96 | memory # uses psrecord 97 | std # parses the log files that include the prints from std::cout 98 | # cpu # built into memory tests using psrecord 99 | ) 100 | 101 | # remove ros2_tracing trace_types if SKIP_TRACING is ON 102 | if(${SKIP_TRACING}) 103 | message(STATUS "SKIP_TRACING is ON") 104 | message(STATUS "Removing callback trace tests") 105 | list(REMOVE_ITEM TRACE_TYPES "callback") 106 | endif() 107 | 108 | find_package(ros_testing REQUIRED) 109 | 110 | # get available rmw implementations 111 | find_package(rmw_implementation_cmake REQUIRED) 112 | get_available_rmw_implementations(rmws_available) 113 | 114 | # only use default RMW by default 115 | if(${ALL_RMWS} MATCHES OFF) 116 | list(REVERSE rmws_available) 117 | foreach(rmw ${rmws_available}) 118 | list(LENGTH rmws_available COUNT) 119 | if(NOT COUNT MATCHES 1) 120 | # message("Removing ${rmw} from tests") 121 | list(REMOVE_AT rmws_available COUNT) 122 | endif() 123 | endforeach() 124 | endif() 125 | 126 | # include cmake functions to use later on 127 | include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/test_requirements.cmake) 128 | 129 | # check each executable matches the system requirements 130 | foreach(exe ${TEST_TARGETS}) 131 | test_requirements(${exe} 3) 132 | endforeach() 133 | 134 | list(JOIN rmws_available "," RMWS_COMMA_LIST) 135 | list(JOIN TEST_TARGETS "," EXES_COMMA_LIST) 136 | list(JOIN TRACE_TYPES "," TRACE_TYPES_COMMA_LIST) 137 | list(JOIN RUN_TIMES "," RUN_TIMES_COMMA_LIST) 138 | 139 | # generate traces for each executable 140 | set(TEST_NAME benchmark) 141 | include(FindPython3) 142 | find_package(Python3 REQUIRED COMPONENTS Interpreter) 143 | add_test( 144 | NAME ${TEST_NAME} 145 | COMMAND ${Python3_EXECUTABLE} ${CMAKE_INSTALL_PREFIX}/share/${PROJECT_NAME}/scripts/benchmark.py 146 | --trace_types ${TRACE_TYPES_COMMA_LIST} 147 | --rmws ${RMWS_COMMA_LIST} 148 | ${RUN_TIMES_COMMA_LIST} ${EXES_COMMA_LIST}) 149 | endif() 150 | endif() 151 | 152 | ament_auto_package( 153 | INSTALL_TO_SHARE test scripts 154 | ) 155 | -------------------------------------------------------------------------------- /autoware_reference_system/autoware_reference_system.dot: -------------------------------------------------------------------------------- 1 | digraph G { 2 | labelloc="b"; 3 | labeljust="l"; 4 | label="autoware_reference_system"; 5 | nodesep=0.5; 6 | node [shape=box, style=filled, penwidth=2, ordering=in]; 7 | edge [weight=8]; 8 | /* Declare all nodes and style them */ 9 | node [fillcolor="#FCD975"]; 10 | "Front Lidar Driver"; 11 | "Rear Lidar Driver"; 12 | "Point Cloud Map" [shape=cylinder]; 13 | "Visualizer" [shape=box3d]; 14 | "Lanelet2 Map" [shape=cylinder]; 15 | 16 | node [fillcolor="#469986"]; 17 | "Front Points Transformer"; 18 | "Rear Points Transformer"; 19 | "Point Cloud Map Loader"; 20 | "Voxel Grid Downsampler"; 21 | "Ray Ground Filter"; 22 | "Object Collision Estimator"; 23 | "MPC Controller"; 24 | "Lane Planner"; 25 | "Parking Planner"; 26 | 27 | node [fillcolor="#FF6961"]; 28 | "Euclidean Cluster Settings" [shape=box]; 29 | "Intersection Output" [shape=box]; 30 | "Euclidean Cluster Detector" [shape=octagon]; 31 | 32 | node [fillcolor="#9D813B"]; 33 | "Point Cloud Fusion"; 34 | "NDT Localizer"; 35 | "Vehicle Interface"; 36 | "Lanelet2 Map Loader"; 37 | "Lanelet2 Global Planner"; 38 | 39 | node [fillcolor="#2576E8"]; 40 | "Behavior Planner"; 41 | 42 | node [fillcolor="#ffaa66"]; 43 | "Vehicle DBW System"; 44 | 45 | /* Declare all edges and style them */ 46 | /* rank 1 */ 47 | { 48 | rank = same; 49 | "Front Lidar Driver"; 50 | "Rear Lidar Driver"; 51 | "Point Cloud Map"; 52 | "Visualizer"; 53 | "Lanelet2 Map"; 54 | } 55 | "Front Lidar Driver" -> "Front Points Transformer"; 56 | "Rear Lidar Driver" -> "Rear Points Transformer"; 57 | "Point Cloud Map" -> "Point Cloud Map Loader"; 58 | "Visualizer" -> "Lanelet2 Global Planner"; 59 | "Lanelet2 Map" -> "Lanelet2 Map Loader"; 60 | 61 | /* rank 2 */ 62 | { 63 | rank = same; 64 | "Front Points Transformer"; 65 | "Rear Points Transformer"; 66 | "Point Cloud Map Loader"; 67 | "Lanelet2 Global Planner"; 68 | "Lanelet2 Map Loader"; 69 | "Parking Planner"; 70 | } 71 | "Front Points Transformer" -> "Point Cloud Fusion"; 72 | "Rear Points Transformer" -> "Point Cloud Fusion"; 73 | "Point Cloud Map Loader" -> "NDT Localizer"; 74 | "Lanelet2 Global Planner" -> "Lanelet2 Map Loader"; 75 | "Lanelet2 Global Planner" -> "Behavior Planner"; 76 | "Lanelet2 Map Loader" -> "Behavior Planner"; 77 | "Lanelet2 Map Loader" -> "Parking Planner"; 78 | "Parking Planner" -> "Behavior Planner"; 79 | "Lanelet2 Map Loader" -> "Lane Planner" [constraint=false]; 80 | 81 | /* rank 3 */ 82 | { 83 | rank = same; 84 | "Euclidean Cluster Settings"; 85 | "Point Cloud Fusion"; 86 | "Voxel Grid Downsampler"; 87 | "NDT Localizer"; 88 | "Behavior Planner"; 89 | "Lane Planner"; 90 | }; 91 | "Euclidean Cluster Settings" -> "Euclidean Cluster Detector"; 92 | "Point Cloud Fusion" -> "Ray Ground Filter"; 93 | "Point Cloud Fusion" -> "Voxel Grid Downsampler"; 94 | "Voxel Grid Downsampler" -> "NDT Localizer"; 95 | "NDT Localizer" -> "Behavior Planner"; 96 | "NDT Localizer" -> "Lanelet2 Global Planner"; 97 | 98 | "Behavior Planner" -> "MPC Controller"; 99 | "Behavior Planner" -> "Vehicle Interface"; 100 | "Lane Planner" -> "Behavior Planner" [constraint=false]; 101 | { 102 | rank = same; 103 | edge[style=invis]; 104 | "Euclidean Cluster Settings" -> "Point Cloud Fusion" -> "Voxel Grid Downsampler" -> "NDT Localizer"; 105 | rankdir = LR; 106 | } 107 | 108 | { 109 | rank = same; 110 | edge[style=invis]; 111 | "Intersection Output" -> "Ray Ground Filter" -> "Euclidean Cluster Detector" -> "Object Collision Estimator" -> "MPC Controller" -> "Vehicle Interface"; 112 | rankdir = LR; 113 | } 114 | /* rank 4 */ 115 | { 116 | rank = same; 117 | "Intersection Output"; 118 | "Ray Ground Filter"; 119 | "Euclidean Cluster Detector"; 120 | "Object Collision Estimator"; 121 | "MPC Controller"; 122 | "Vehicle Interface"; 123 | "Vehicle DBW System" [constraint=false]; 124 | } 125 | "Euclidean Cluster Detector" -> "Intersection Output" [constraint=false]; 126 | "Ray Ground Filter" -> "Euclidean Cluster Detector"; 127 | "Euclidean Cluster Detector" -> "Object Collision Estimator"; 128 | "Object Collision Estimator" -> "Behavior Planner"; 129 | "MPC Controller" -> "Vehicle Interface"; 130 | "Vehicle Interface" -> "Vehicle DBW System"; 131 | { 132 | rank = same; 133 | edge[style=invis]; 134 | "Intersection Output" -> "Ray Ground Filter" -> "Euclidean Cluster Detector" -> "Object Collision Estimator" -> "MPC Controller" -> "Vehicle Interface"; 135 | rankdir = LR; 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /autoware_reference_system/cmake/test_requirements.cmake: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Apex.AI, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # make sure executable matches system requirements 16 | function(test_requirements target) 17 | set(TEST_EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/${target}) 18 | set(TEST_EXECUTABLE_NAME test_requirements_${target}) 19 | # replaces all @var@ and ${var} within input file 20 | configure_file( 21 | test/test_requirements.py 22 | test_requirements_${target}.py 23 | @ONLY 24 | ) 25 | add_ros_test( 26 | ${CMAKE_CURRENT_BINARY_DIR}/test_requirements_${target}.py 27 | TIMEOUT ${DEFAULT_TIMEOUT} # seconds 28 | ) 29 | if(TARGET ${target}) 30 | ament_target_dependencies(${target} 31 | "rclcpp" "reference_interfaces" "reference_system") 32 | endif() 33 | endfunction() -------------------------------------------------------------------------------- /autoware_reference_system/include/autoware_reference_system/autoware_system_builder.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Apex.AI, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #ifndef AUTOWARE_REFERENCE_SYSTEM__AUTOWARE_SYSTEM_BUILDER_HPP_ 15 | #define AUTOWARE_REFERENCE_SYSTEM__AUTOWARE_SYSTEM_BUILDER_HPP_ 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #include "reference_system/nodes/settings.hpp" 22 | #include "reference_system/sample_management.hpp" 23 | 24 | using namespace std::chrono_literals; // NOLINT 25 | 26 | template 27 | auto create_autoware_nodes() 28 | ->std::vector> 29 | { 30 | std::vector> nodes; 31 | 32 | SampleManagementSettings::get().set_hot_path( 33 | {"FrontLidarDriver", 34 | "RearLidarDriver", 35 | "PointsTransformerFront", 36 | "PointsTransformerRear", 37 | "PointCloudFusion", 38 | "RayGroundFilter", 39 | "EuclideanClusterDetector", 40 | "ObjectCollisionEstimator"}, 41 | {"FrontLidarDriver", "RearLidarDriver"}, 42 | "ObjectCollisionEstimator"); 43 | 44 | // ignore the warning about designated initializers - they make the code much 45 | // more readable 46 | #pragma GCC diagnostic push 47 | #pragma GCC diagnostic ignored "-Wpedantic" 48 | 49 | // setup communication graph 50 | // sensor nodes 51 | nodes.emplace_back( 52 | std::make_shared( 53 | nodes::SensorSettings{.node_name = "FrontLidarDriver", 54 | .topic_name = "FrontLidarDriver", 55 | .cycle_time = TimingConfig::FRONT_LIDAR_DRIVER})); 56 | 57 | nodes.emplace_back( 58 | std::make_shared( 59 | nodes::SensorSettings{.node_name = "RearLidarDriver", 60 | .topic_name = "RearLidarDriver", 61 | .cycle_time = TimingConfig::REAR_LIDAR_DRIVER})); 62 | 63 | nodes.emplace_back( 64 | std::make_shared( 65 | nodes::SensorSettings{.node_name = "PointCloudMap", 66 | .topic_name = "PointCloudMap", 67 | .cycle_time = TimingConfig::POINT_CLOUD_MAP})); 68 | 69 | nodes.emplace_back( 70 | std::make_shared( 71 | nodes::SensorSettings{.node_name = "Visualizer", 72 | .topic_name = "Visualizer", 73 | .cycle_time = TimingConfig::VISUALIZER})); 74 | 75 | nodes.emplace_back( 76 | std::make_shared( 77 | nodes::SensorSettings{.node_name = "Lanelet2Map", 78 | .topic_name = "Lanelet2Map", 79 | .cycle_time = TimingConfig::LANELET2MAP})); 80 | 81 | nodes.emplace_back( 82 | std::make_shared( 83 | nodes::SensorSettings{ 84 | .node_name = "EuclideanClusterSettings", 85 | .topic_name = "EuclideanClusterSettings", 86 | .cycle_time = TimingConfig::EUCLIDEAN_CLUSTER_SETTINGS})); 87 | 88 | // transform nodes 89 | nodes.emplace_back( 90 | std::make_shared( 91 | nodes::TransformSettings{ 92 | .node_name = "PointsTransformerFront", 93 | .input_topic = "FrontLidarDriver", 94 | .output_topic = "PointsTransformerFront", 95 | .number_crunch_limit = TimingConfig::POINTS_TRANSFORMER_FRONT})); 96 | 97 | nodes.emplace_back( 98 | std::make_shared( 99 | nodes::TransformSettings{ 100 | .node_name = "PointsTransformerRear", 101 | .input_topic = "RearLidarDriver", 102 | .output_topic = "PointsTransformerRear", 103 | .number_crunch_limit = TimingConfig::POINTS_TRANSFORMER_REAR})); 104 | 105 | nodes.emplace_back( 106 | std::make_shared( 107 | nodes::TransformSettings{ 108 | .node_name = "VoxelGridDownsampler", 109 | .input_topic = "PointCloudFusion", 110 | .output_topic = "VoxelGridDownsampler", 111 | .number_crunch_limit = TimingConfig::VOXEL_GRID_DOWNSAMPLER})); 112 | 113 | nodes.emplace_back( 114 | std::make_shared( 115 | nodes::TransformSettings{ 116 | .node_name = "PointCloudMapLoader", 117 | .input_topic = "PointCloudMap", 118 | .output_topic = "PointCloudMapLoader", 119 | .number_crunch_limit = TimingConfig::POINT_CLOUD_MAP_LOADER})); 120 | 121 | nodes.emplace_back( 122 | std::make_shared( 123 | nodes::TransformSettings{ 124 | .node_name = "RayGroundFilter", 125 | .input_topic = "PointCloudFusion", 126 | .output_topic = "RayGroundFilter", 127 | .number_crunch_limit = TimingConfig::RAY_GROUND_FILTER})); 128 | 129 | nodes.emplace_back( 130 | std::make_shared( 131 | nodes::TransformSettings{ 132 | .node_name = "ObjectCollisionEstimator", 133 | .input_topic = "EuclideanClusterDetector", 134 | .output_topic = "ObjectCollisionEstimator", 135 | .number_crunch_limit = TimingConfig::OBJECT_COLLISION_ESTIMATOR})); 136 | 137 | nodes.emplace_back( 138 | std::make_shared( 139 | nodes::TransformSettings{ 140 | .node_name = "MPCController", 141 | .input_topic = "BehaviorPlanner", 142 | .output_topic = "MPCController", 143 | .number_crunch_limit = TimingConfig::MPC_CONTROLLER})); 144 | 145 | nodes.emplace_back( 146 | std::make_shared( 147 | nodes::TransformSettings{ 148 | .node_name = "ParkingPlanner", 149 | .input_topic = "Lanelet2MapLoader", 150 | .output_topic = "ParkingPlanner", 151 | .number_crunch_limit = TimingConfig::PARKING_PLANNER})); 152 | 153 | nodes.emplace_back( 154 | std::make_shared( 155 | nodes::TransformSettings{ 156 | .node_name = "LanePlanner", 157 | .input_topic = "Lanelet2MapLoader", 158 | .output_topic = "LanePlanner", 159 | .number_crunch_limit = TimingConfig::LANE_PLANNER})); 160 | 161 | // fusion nodes 162 | nodes.emplace_back( 163 | std::make_shared( 164 | nodes::FusionSettings{ 165 | .node_name = "PointCloudFusion", 166 | .input_0 = "PointsTransformerFront", 167 | .input_1 = "PointsTransformerRear", 168 | .output_topic = "PointCloudFusion", 169 | .number_crunch_limit = TimingConfig::POINT_CLOUD_FUSION})); 170 | 171 | nodes.emplace_back( 172 | std::make_shared( 173 | nodes::FusionSettings{ 174 | .node_name = "NDTLocalizer", 175 | .input_0 = "VoxelGridDownsampler", 176 | .input_1 = "PointCloudMapLoader", 177 | .output_topic = "NDTLocalizer", 178 | .number_crunch_limit = TimingConfig::NDT_LOCALIZER})); 179 | 180 | nodes.emplace_back( 181 | std::make_shared( 182 | nodes::FusionSettings{ 183 | .node_name = "VehicleInterface", 184 | .input_0 = "MPCController", 185 | .input_1 = "BehaviorPlanner", 186 | .output_topic = "VehicleInterface", 187 | .number_crunch_limit = TimingConfig::VEHICLE_INTERFACE})); 188 | 189 | nodes.emplace_back( 190 | std::make_shared( 191 | nodes::FusionSettings{ 192 | .node_name = "Lanelet2GlobalPlanner", 193 | .input_0 = "Visualizer", 194 | .input_1 = "NDTLocalizer", 195 | .output_topic = "Lanelet2GlobalPlanner", 196 | .number_crunch_limit = TimingConfig::LANELET_2_GLOBAL_PLANNER})); 197 | 198 | nodes.emplace_back( 199 | std::make_shared( 200 | nodes::FusionSettings{ 201 | .node_name = "Lanelet2MapLoader", 202 | .input_0 = "Lanelet2Map", 203 | .input_1 = "Lanelet2GlobalPlanner", 204 | .output_topic = "Lanelet2MapLoader", 205 | .number_crunch_limit = TimingConfig::LANELET_2_MAP_LOADER})); 206 | 207 | // cyclic node 208 | nodes.emplace_back( 209 | std::make_shared( 210 | nodes::CyclicSettings{ 211 | .node_name = "BehaviorPlanner", 212 | .inputs = {"ObjectCollisionEstimator", "NDTLocalizer", 213 | "Lanelet2GlobalPlanner", "Lanelet2MapLoader", 214 | "ParkingPlanner", "LanePlanner"}, 215 | .output_topic = "BehaviorPlanner", 216 | .number_crunch_limit = TimingConfig::BEHAVIOR_PLANNER, 217 | .cycle_time = TimingConfig::BEHAVIOR_PLANNER_CYCLE})); 218 | 219 | // intersection node 220 | nodes.emplace_back( 221 | std::make_shared( 222 | nodes::IntersectionSettings{ 223 | .node_name = "EuclideanClusterDetector", 224 | .connections = { 225 | {.input_topic = "RayGroundFilter", 226 | .output_topic = "EuclideanClusterDetector", 227 | .number_crunch_limit = TimingConfig::EUCLIDEAN_CLUSTER_DETECTOR}, 228 | {.input_topic = "EuclideanClusterSettings", 229 | .output_topic = "EuclideanIntersection", 230 | .number_crunch_limit = TimingConfig::EUCLIDEAN_INTERSECTION}}})); 231 | 232 | // command node 233 | nodes.emplace_back( 234 | std::make_shared( 235 | nodes::CommandSettings{ 236 | .node_name = "VehicleDBWSystem", .input_topic = "VehicleInterface"})); 237 | 238 | nodes.emplace_back( 239 | std::make_shared( 240 | nodes::CommandSettings{.node_name = "IntersectionOutput", 241 | .input_topic = "EuclideanIntersection"})); 242 | #pragma GCC diagnostic pop 243 | 244 | return nodes; 245 | } 246 | 247 | template 248 | std::shared_ptr get_node( 249 | const std::string & name, 250 | const std::vector> & v) 251 | { 252 | for (const auto & n : v) { 253 | if (n->get_name() == std::string(name)) { 254 | return n; 255 | } 256 | } 257 | 258 | return std::shared_ptr(); 259 | } 260 | 261 | #endif // AUTOWARE_REFERENCE_SYSTEM__AUTOWARE_SYSTEM_BUILDER_HPP_ 262 | -------------------------------------------------------------------------------- /autoware_reference_system/include/autoware_reference_system/priorities.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Apex.AI, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef AUTOWARE_REFERENCE_SYSTEM__PRIORITIES_HPP_ 16 | #define AUTOWARE_REFERENCE_SYSTEM__PRIORITIES_HPP_ 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | 24 | extern const std::set hotpath_nodes; 25 | extern const std::set planner_nodes; 26 | 27 | constexpr int hotpath_prio = 1; 28 | #define HOTPATH_AFFINITY {1, 2, 3} 29 | 30 | 31 | constexpr int planner_prio = 30; 32 | #define PLANNER_AFFINITY {0} 33 | 34 | #endif // AUTOWARE_REFERENCE_SYSTEM__PRIORITIES_HPP_ 35 | -------------------------------------------------------------------------------- /autoware_reference_system/include/autoware_reference_system/system/timing/benchmark.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Apex.AI, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #ifndef AUTOWARE_REFERENCE_SYSTEM__SYSTEM__TIMING__BENCHMARK_HPP_ 15 | #define AUTOWARE_REFERENCE_SYSTEM__SYSTEM__TIMING__BENCHMARK_HPP_ 16 | #include 17 | 18 | #include "default.hpp" 19 | 20 | namespace nodes 21 | { 22 | namespace timing 23 | { 24 | 25 | struct BenchmarkThroughput 26 | { 27 | using time_t = std::chrono::nanoseconds; 28 | using milliseconds = std::chrono::milliseconds; 29 | using seconds = std::chrono::seconds; 30 | 31 | // sensors 32 | static constexpr time_t FRONT_LIDAR_DRIVER = milliseconds(0); 33 | static constexpr time_t REAR_LIDAR_DRIVER = milliseconds(0); 34 | static constexpr time_t POINT_CLOUD_MAP = milliseconds(0); 35 | static constexpr time_t VISUALIZER = milliseconds(0); 36 | static constexpr time_t LANELET2MAP = milliseconds(0); 37 | static constexpr time_t EUCLIDEAN_CLUSTER_SETTINGS = milliseconds(0); 38 | 39 | // the following values are used as the number_cruncher_limit 40 | // to search for primes up to starting at 3 41 | // for your platform, run the `number_cruncher_benchmark` executable 42 | // to figure out what values to place here corresponding to the run_time 43 | // you would like to run each node for 44 | // processing 45 | static constexpr uint64_t POINTS_TRANSFORMER_FRONT = 0; 46 | static constexpr uint64_t POINTS_TRANSFORMER_REAR = 0; 47 | static constexpr uint64_t VOXEL_GRID_DOWNSAMPLER = 0; 48 | static constexpr uint64_t POINT_CLOUD_MAP_LOADER = 0; 49 | static constexpr uint64_t RAY_GROUND_FILTER = 0; 50 | static constexpr uint64_t EUCLIDEAN_CLUSTER_DETECTOR = 0; 51 | static constexpr uint64_t EUCLIDEAN_INTERSECTION = 0; 52 | static constexpr uint64_t OBJECT_COLLISION_ESTIMATOR = 0; 53 | static constexpr uint64_t MPC_CONTROLLER = 0; 54 | static constexpr uint64_t PARKING_PLANNER = 0; 55 | static constexpr uint64_t LANE_PLANNER = 0; 56 | 57 | // fusion 58 | static constexpr uint64_t POINT_CLOUD_FUSION = 0; 59 | static constexpr uint64_t NDT_LOCALIZER = 0; 60 | static constexpr uint64_t VEHICLE_INTERFACE = 0; 61 | static constexpr uint64_t LANELET_2_GLOBAL_PLANNER = 0; 62 | static constexpr uint64_t LANELET_2_MAP_LOADER = 0; 63 | 64 | // cyclic 65 | static constexpr uint64_t BEHAVIOR_PLANNER = 0; 66 | static constexpr time_t BEHAVIOR_PLANNER_CYCLE = milliseconds(0); 67 | }; 68 | 69 | constexpr BenchmarkThroughput::time_t BenchmarkThroughput::FRONT_LIDAR_DRIVER; 70 | constexpr BenchmarkThroughput::time_t BenchmarkThroughput::REAR_LIDAR_DRIVER; 71 | constexpr BenchmarkThroughput::time_t BenchmarkThroughput::POINT_CLOUD_MAP; 72 | constexpr BenchmarkThroughput::time_t BenchmarkThroughput::VISUALIZER; 73 | constexpr BenchmarkThroughput::time_t BenchmarkThroughput::LANELET2MAP; 74 | constexpr BenchmarkThroughput::time_t 75 | BenchmarkThroughput::EUCLIDEAN_CLUSTER_SETTINGS; 76 | constexpr uint64_t BenchmarkThroughput::POINTS_TRANSFORMER_FRONT; 77 | constexpr uint64_t BenchmarkThroughput::POINTS_TRANSFORMER_REAR; 78 | constexpr uint64_t BenchmarkThroughput::VOXEL_GRID_DOWNSAMPLER; 79 | constexpr uint64_t BenchmarkThroughput::POINT_CLOUD_MAP_LOADER; 80 | constexpr uint64_t BenchmarkThroughput::RAY_GROUND_FILTER; 81 | constexpr uint64_t BenchmarkThroughput::EUCLIDEAN_CLUSTER_DETECTOR; 82 | constexpr uint64_t BenchmarkThroughput::EUCLIDEAN_INTERSECTION; 83 | constexpr uint64_t BenchmarkThroughput::OBJECT_COLLISION_ESTIMATOR; 84 | constexpr uint64_t BenchmarkThroughput::MPC_CONTROLLER; 85 | constexpr uint64_t BenchmarkThroughput::PARKING_PLANNER; 86 | constexpr uint64_t BenchmarkThroughput::LANE_PLANNER; 87 | constexpr uint64_t BenchmarkThroughput::POINT_CLOUD_FUSION; 88 | constexpr uint64_t BenchmarkThroughput::NDT_LOCALIZER; 89 | constexpr uint64_t BenchmarkThroughput::VEHICLE_INTERFACE; 90 | constexpr uint64_t BenchmarkThroughput::LANELET_2_GLOBAL_PLANNER; 91 | constexpr uint64_t BenchmarkThroughput::LANELET_2_MAP_LOADER; 92 | constexpr uint64_t BenchmarkThroughput::BEHAVIOR_PLANNER; 93 | constexpr BenchmarkThroughput::time_t 94 | BenchmarkThroughput::BEHAVIOR_PLANNER_CYCLE; 95 | 96 | struct BenchmarkCPUUsage 97 | { 98 | using time_t = std::chrono::nanoseconds; 99 | using milliseconds = std::chrono::milliseconds; 100 | using seconds = std::chrono::seconds; 101 | 102 | // sensors 103 | static constexpr time_t FRONT_LIDAR_DRIVER = Default::FRONT_LIDAR_DRIVER; 104 | static constexpr time_t REAR_LIDAR_DRIVER = Default::REAR_LIDAR_DRIVER; 105 | static constexpr time_t POINT_CLOUD_MAP = Default::POINT_CLOUD_MAP; 106 | static constexpr time_t VISUALIZER = Default::VISUALIZER; 107 | static constexpr time_t LANELET2MAP = Default::LANELET2MAP; 108 | static constexpr time_t EUCLIDEAN_CLUSTER_SETTINGS = 109 | Default::EUCLIDEAN_CLUSTER_SETTINGS; 110 | 111 | // the following values are used as the number_cruncher_limit 112 | // to search for primes up to starting at 3 113 | // for your platform, run the `number_cruncher_benchmark` executable 114 | // to figure out what values to place here corresponding to the run_time 115 | // you would like to run each node for 116 | // processing 117 | static constexpr uint64_t POINTS_TRANSFORMER_FRONT = 0; 118 | static constexpr uint64_t POINTS_TRANSFORMER_REAR = 0; 119 | static constexpr uint64_t VOXEL_GRID_DOWNSAMPLER = 0; 120 | static constexpr uint64_t POINT_CLOUD_MAP_LOADER = 0; 121 | static constexpr uint64_t RAY_GROUND_FILTER = 0; 122 | static constexpr uint64_t EUCLIDEAN_CLUSTER_DETECTOR = 0; 123 | static constexpr uint64_t EUCLIDEAN_INTERSECTION = 0; 124 | static constexpr uint64_t OBJECT_COLLISION_ESTIMATOR = 0; 125 | static constexpr uint64_t MPC_CONTROLLER = 0; 126 | static constexpr uint64_t PARKING_PLANNER = 0; 127 | static constexpr uint64_t LANE_PLANNER = 0; 128 | 129 | // fusion 130 | static constexpr uint64_t POINT_CLOUD_FUSION = 0; 131 | static constexpr uint64_t NDT_LOCALIZER = 0; 132 | static constexpr uint64_t VEHICLE_INTERFACE = 0; 133 | static constexpr uint64_t LANELET_2_GLOBAL_PLANNER = 0; 134 | static constexpr uint64_t LANELET_2_MAP_LOADER = 0; 135 | 136 | // cyclic 137 | static constexpr uint64_t BEHAVIOR_PLANNER = 0; 138 | static constexpr time_t BEHAVIOR_PLANNER_CYCLE = 139 | Default::BEHAVIOR_PLANNER_CYCLE; 140 | }; 141 | 142 | constexpr BenchmarkCPUUsage::time_t BenchmarkCPUUsage::FRONT_LIDAR_DRIVER; 143 | constexpr BenchmarkCPUUsage::time_t BenchmarkCPUUsage::REAR_LIDAR_DRIVER; 144 | constexpr BenchmarkCPUUsage::time_t BenchmarkCPUUsage::POINT_CLOUD_MAP; 145 | constexpr BenchmarkCPUUsage::time_t BenchmarkCPUUsage::VISUALIZER; 146 | constexpr BenchmarkCPUUsage::time_t BenchmarkCPUUsage::LANELET2MAP; 147 | constexpr BenchmarkCPUUsage::time_t 148 | BenchmarkCPUUsage::EUCLIDEAN_CLUSTER_SETTINGS; 149 | constexpr uint64_t BenchmarkCPUUsage::POINTS_TRANSFORMER_FRONT; 150 | constexpr uint64_t BenchmarkCPUUsage::POINTS_TRANSFORMER_REAR; 151 | constexpr uint64_t BenchmarkCPUUsage::VOXEL_GRID_DOWNSAMPLER; 152 | constexpr uint64_t BenchmarkCPUUsage::POINT_CLOUD_MAP_LOADER; 153 | constexpr uint64_t BenchmarkCPUUsage::RAY_GROUND_FILTER; 154 | constexpr uint64_t BenchmarkCPUUsage::EUCLIDEAN_CLUSTER_DETECTOR; 155 | constexpr uint64_t BenchmarkCPUUsage::EUCLIDEAN_INTERSECTION; 156 | constexpr uint64_t BenchmarkCPUUsage::OBJECT_COLLISION_ESTIMATOR; 157 | constexpr uint64_t BenchmarkCPUUsage::MPC_CONTROLLER; 158 | constexpr uint64_t BenchmarkCPUUsage::PARKING_PLANNER; 159 | constexpr uint64_t BenchmarkCPUUsage::LANE_PLANNER; 160 | constexpr uint64_t BenchmarkCPUUsage::POINT_CLOUD_FUSION; 161 | constexpr uint64_t BenchmarkCPUUsage::NDT_LOCALIZER; 162 | constexpr uint64_t BenchmarkCPUUsage::VEHICLE_INTERFACE; 163 | constexpr uint64_t BenchmarkCPUUsage::LANELET_2_GLOBAL_PLANNER; 164 | constexpr uint64_t BenchmarkCPUUsage::LANELET_2_MAP_LOADER; 165 | constexpr uint64_t BenchmarkCPUUsage::BEHAVIOR_PLANNER; 166 | constexpr BenchmarkCPUUsage::time_t BenchmarkCPUUsage::BEHAVIOR_PLANNER_CYCLE; 167 | 168 | } // namespace timing 169 | } // namespace nodes 170 | #endif // AUTOWARE_REFERENCE_SYSTEM__SYSTEM__TIMING__BENCHMARK_HPP_ 171 | -------------------------------------------------------------------------------- /autoware_reference_system/include/autoware_reference_system/system/timing/default.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Apex.AI, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #ifndef AUTOWARE_REFERENCE_SYSTEM__SYSTEM__TIMING__DEFAULT_HPP_ 15 | #define AUTOWARE_REFERENCE_SYSTEM__SYSTEM__TIMING__DEFAULT_HPP_ 16 | #include 17 | 18 | namespace nodes 19 | { 20 | namespace timing 21 | { 22 | 23 | struct Default 24 | { 25 | using time_t = std::chrono::nanoseconds; 26 | using milliseconds = std::chrono::milliseconds; 27 | 28 | // sensors 29 | static constexpr time_t FRONT_LIDAR_DRIVER = milliseconds(100); 30 | static constexpr time_t REAR_LIDAR_DRIVER = milliseconds(100); 31 | static constexpr time_t POINT_CLOUD_MAP = milliseconds(120); 32 | static constexpr time_t VISUALIZER = milliseconds(60); 33 | static constexpr time_t LANELET2MAP = milliseconds(100); 34 | static constexpr time_t EUCLIDEAN_CLUSTER_SETTINGS = milliseconds(25); 35 | 36 | // the following values are used as the number_cruncher_limit 37 | // to search for primes up to starting at 3 38 | // for your platform, run the `number_cruncher_benchmark` executable 39 | // to figure out what values to place here corresponding to the run_time 40 | // you would like to run each node for 41 | // processing 42 | static constexpr uint64_t POINTS_TRANSFORMER_FRONT = 4096; 43 | static constexpr uint64_t POINTS_TRANSFORMER_REAR = 4096; 44 | static constexpr uint64_t VOXEL_GRID_DOWNSAMPLER = 4096; 45 | static constexpr uint64_t POINT_CLOUD_MAP_LOADER = 4096; 46 | static constexpr uint64_t RAY_GROUND_FILTER = 4096; 47 | static constexpr uint64_t EUCLIDEAN_CLUSTER_DETECTOR = 4096; 48 | static constexpr uint64_t EUCLIDEAN_INTERSECTION = 4096; 49 | static constexpr uint64_t OBJECT_COLLISION_ESTIMATOR = 4096; 50 | static constexpr uint64_t MPC_CONTROLLER = 4096; 51 | static constexpr uint64_t PARKING_PLANNER = 4096; 52 | static constexpr uint64_t LANE_PLANNER = 4096; 53 | 54 | // fusion 55 | static constexpr uint64_t POINT_CLOUD_FUSION = 4096; 56 | static constexpr uint64_t NDT_LOCALIZER = 4096; 57 | static constexpr uint64_t VEHICLE_INTERFACE = 4096; 58 | static constexpr uint64_t LANELET_2_GLOBAL_PLANNER = 4096; 59 | static constexpr uint64_t LANELET_2_MAP_LOADER = 4096; 60 | 61 | // cyclic 62 | static constexpr uint64_t BEHAVIOR_PLANNER = 4096; 63 | static constexpr time_t BEHAVIOR_PLANNER_CYCLE = milliseconds(100); 64 | }; 65 | 66 | constexpr Default::time_t Default::FRONT_LIDAR_DRIVER; 67 | constexpr Default::time_t Default::REAR_LIDAR_DRIVER; 68 | constexpr Default::time_t Default::POINT_CLOUD_MAP; 69 | constexpr Default::time_t Default::VISUALIZER; 70 | constexpr Default::time_t Default::LANELET2MAP; 71 | constexpr Default::time_t Default::EUCLIDEAN_CLUSTER_SETTINGS; 72 | constexpr uint64_t Default::POINTS_TRANSFORMER_FRONT; 73 | constexpr uint64_t Default::POINTS_TRANSFORMER_REAR; 74 | constexpr uint64_t Default::VOXEL_GRID_DOWNSAMPLER; 75 | constexpr uint64_t Default::POINT_CLOUD_MAP_LOADER; 76 | constexpr uint64_t Default::RAY_GROUND_FILTER; 77 | constexpr uint64_t Default::EUCLIDEAN_CLUSTER_DETECTOR; 78 | constexpr uint64_t Default::EUCLIDEAN_INTERSECTION; 79 | constexpr uint64_t Default::OBJECT_COLLISION_ESTIMATOR; 80 | constexpr uint64_t Default::MPC_CONTROLLER; 81 | constexpr uint64_t Default::PARKING_PLANNER; 82 | constexpr uint64_t Default::LANE_PLANNER; 83 | constexpr uint64_t Default::POINT_CLOUD_FUSION; 84 | constexpr uint64_t Default::NDT_LOCALIZER; 85 | constexpr uint64_t Default::VEHICLE_INTERFACE; 86 | constexpr uint64_t Default::LANELET_2_GLOBAL_PLANNER; 87 | constexpr uint64_t Default::LANELET_2_MAP_LOADER; 88 | constexpr uint64_t Default::BEHAVIOR_PLANNER; 89 | constexpr Default::time_t Default::BEHAVIOR_PLANNER_CYCLE; 90 | 91 | } // namespace timing 92 | } // namespace nodes 93 | #endif // AUTOWARE_REFERENCE_SYSTEM__SYSTEM__TIMING__DEFAULT_HPP_ 94 | -------------------------------------------------------------------------------- /autoware_reference_system/package.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | autoware_reference_system 5 | 1.1.0 6 | Autoware Reference System for the ROScon workshop 7 | Evan Flynn 8 | Apache License 2.0 9 | 10 | Christian Eltzschig 11 | 12 | ament_cmake 13 | ament_cmake_auto 14 | 15 | reference_system 16 | 17 | rclcpp 18 | reference_interfaces 19 | rcl_interfaces 20 | 21 | ament_cmake_pytest 22 | ament_cmake_gtest 23 | ament_lint_auto 24 | ament_lint_common 25 | ros_testing 26 | launch 27 | launch_ros 28 | tracetools_analysis 29 | tracetools_launch 30 | tracetools_trace 31 | python3-bokeh-pip 32 | python3-selenium 33 | 34 | 35 | ament_cmake 36 | 37 | 38 | -------------------------------------------------------------------------------- /autoware_reference_system/scripts/benchmark.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright 2021 Apex.AI, Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import argparse 17 | import itertools 18 | 19 | # Generates traces for specified executables and RMWs 20 | from reference_system_py.benchmark import available_executables, generate_trace 21 | from reference_system_py.benchmark import ROS_HOME, setup_benchmark_directory 22 | from reference_system_py.report import generate_report, generate_summary_report 23 | 24 | 25 | if __name__ == '__main__': 26 | parser = argparse.ArgumentParser(description='Benchmark an executor implementation.') 27 | parser.add_argument('runtimes', 28 | help='comma-separated list of runtimes (in seconds)') 29 | parser.add_argument('executables', 30 | help='comma-separated list of target executables') 31 | parser.add_argument('--trace_types', 32 | help='comma-separated list of trace types (default: memory,std)', 33 | default='memory,std') 34 | parser.add_argument('--rmws', 35 | default='rmw_cyclonedds_cpp', 36 | help='comma-separated list of rmw implementations') 37 | parser.add_argument('--logdir', 38 | default=None, 39 | help=('The directory where traces and results are placed (default: ' + 40 | f'{ROS_HOME}/benchmark_/)')) 41 | parser.add_argument('--plot_only', 42 | default=False, 43 | help='Only plot existing data, do not run new experiments.', 44 | action='store_true') 45 | parser.add_argument('--template_file', 46 | default='cfg/report_template.md', 47 | help='Path to template report file to fill in') 48 | 49 | cmdline_args = parser.parse_args() 50 | common_args = {'pkg': 'autoware_reference_system', 51 | 'directory': cmdline_args.logdir} 52 | 53 | if common_args['directory'] is None: 54 | create_dir = (not cmdline_args.plot_only) 55 | common_args['directory'] = str(setup_benchmark_directory(pkg=common_args['pkg'], 56 | create=create_dir)) 57 | 58 | runtimes = [int(runtime) for runtime in cmdline_args.runtimes.split(',')] 59 | exe_patterns, rmws, trace_types = (lst.split(',') 60 | for lst in [cmdline_args.executables, 61 | cmdline_args.rmws, 62 | cmdline_args.trace_types]) 63 | 64 | exes = [exe for pattern in exe_patterns 65 | for exe in available_executables(pattern=pattern, pkg=common_args['pkg'])] 66 | for runtime, exe, rmw, trace_type in itertools.product(runtimes, exes, rmws, trace_types): 67 | if not cmdline_args.plot_only: 68 | generate_trace(trace_type, exe, rmw=rmw, runtime_sec=runtime, **common_args) 69 | generate_report(trace_type, exe, rmw=rmw, runtime_sec=runtime, **common_args) 70 | 71 | for trace_type, runtime in itertools.product(trace_types, runtimes): 72 | generate_summary_report( 73 | trace_type=trace_type, 74 | runtime_sec=runtime, 75 | template_file=cmdline_args.template_file, 76 | **common_args) 77 | -------------------------------------------------------------------------------- /autoware_reference_system/src/number_cruncher_benchmark.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Apex.AI, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | #include "reference_system/number_cruncher.hpp" 16 | 17 | #include 18 | #include 19 | 20 | 21 | int main() 22 | { 23 | long double crunch_time = 0.0; 24 | std::cout << "maximum_number run time" << std::endl; 25 | for (uint64_t i = 64; crunch_time < 1000.0; i *= 2) { 26 | crunch_time = get_crunch_time_in_ms(i); 27 | std::cout << std::setfill(' ') << std::setw(12) << i << " " << 28 | crunch_time << "ms" << std::endl; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /autoware_reference_system/src/priorities.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Apex.AI, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "autoware_reference_system/priorities.hpp" 16 | #include 17 | #include 18 | 19 | const std::set hotpath_nodes = {"FrontLidarDriver", "PointsTransformerFront", 20 | "RearLidarDriver", "PointsTransformerRear", 21 | "PointCloudFusion", "RayGroundFilter", 22 | "EuclideanClusterDetector", "ObjectCollisionEstimator"}; 23 | const std::set planner_nodes {"BehaviorPlanner"}; 24 | -------------------------------------------------------------------------------- /autoware_reference_system/src/ros2/executor/autoware_default_cbg.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Robert Bosch GmbH 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | 22 | #include "rclcpp/rclcpp.hpp" 23 | 24 | #include "reference_system/system/type/rclcpp_system.hpp" 25 | 26 | #include "autoware_reference_system/autoware_system_builder.hpp" 27 | #include "autoware_reference_system/system/timing/benchmark.hpp" 28 | #include "autoware_reference_system/system/timing/default.hpp" 29 | #include "autoware_reference_system/priorities.hpp" 30 | 31 | void set_rt_properties(int prio, const std::unordered_set & affinity) 32 | { 33 | struct sched_param sched_param = { 0 }; 34 | sched_param.sched_priority = prio; 35 | sched_setscheduler(0, SCHED_RR, &sched_param); 36 | 37 | cpu_set_t cpuset; 38 | CPU_ZERO(&cpuset); 39 | for (const auto cpu : affinity) { 40 | CPU_SET(cpu, &cpuset); 41 | } 42 | sched_setaffinity(0, sizeof(cpuset), &cpuset); 43 | } 44 | 45 | int main(int argc, char ** argv) 46 | { 47 | rclcpp::init(argc, argv); 48 | 49 | using TimeConfig = nodes::timing::Default; 50 | // uncomment for benchmarking 51 | // using TimeConfig = nodes::timing::BenchmarkCPUUsage; 52 | // set_benchmark_mode(true); 53 | 54 | auto nodes_vec = create_autoware_nodes(); 55 | using NodeMap = std::unordered_map>; 57 | 58 | NodeMap nodes; 59 | for (const auto & node : nodes_vec) { 60 | nodes.emplace(node->get_name(), node); 61 | } 62 | 63 | rclcpp::executors::SingleThreadedExecutor 64 | front_exe, 65 | rear_exe, 66 | fusion_exe, 67 | planner_exe, 68 | other_exe; 69 | 70 | std::set front_nodes = {"FrontLidarDriver", "PointsTransformerFront"}; 71 | std::set rear_nodes = {"RearLidarDriver", "PointsTransformerRear"}; 72 | std::set fusion_nodes = {"PointCloudFusion", 73 | "RayGroundFilter", 74 | // "EuclideanClusterDetector" is a special case and 75 | // prioritized on the callback-group level. 76 | "ObjectCollisionEstimator"}; 77 | std::set planner_nodes = {"BehaviorPlanner"}; 78 | std::set other_nodes = {"PointCloudMap", 79 | "Visualizer", 80 | "Lanelet2Map", 81 | "EuclideanClusterSettings", 82 | "PointCloudMapLoader", 83 | "MPCController", 84 | "VehicleInterface", 85 | "VehicleDBWSystem", 86 | "NDTLocalizer", 87 | "Lanelet2GlobalPlanner", 88 | "Lanelet2MapLoader", 89 | "ParkingPlanner", 90 | "LanePlanner", 91 | "IntersectionOutput", 92 | "VoxelGridDownsampler"}; 93 | 94 | for (const auto & node : front_nodes) { 95 | front_exe.add_node(nodes.at(node)); 96 | } 97 | for (const auto & node : rear_nodes) { 98 | rear_exe.add_node(nodes.at(node)); 99 | } 100 | for (const auto & node : planner_nodes) { 101 | planner_exe.add_node(nodes.at(node)); 102 | } 103 | for (const auto & node : fusion_nodes) { 104 | fusion_exe.add_node(nodes.at(node)); 105 | } 106 | 107 | auto euclidean_node = std::dynamic_pointer_cast( 108 | nodes.at("EuclideanClusterDetector")); 109 | assert(euclidean_node); 110 | fusion_exe.add_callback_group( 111 | euclidean_node->get_callback_group_of_subscription("/RayGroundFilter"), 112 | euclidean_node->get_node_base_interface()); 113 | 114 | other_exe.add_node(nodes.at("EuclideanClusterDetector")); 115 | for (const auto & node : other_nodes) { 116 | other_exe.add_node(nodes.at(node)); 117 | } 118 | 119 | std::thread front_thread {[&]() { 120 | set_rt_properties(hotpath_prio, HOTPATH_AFFINITY); 121 | front_exe.spin(); 122 | }}; 123 | std::thread rear_thread {[&]() { 124 | set_rt_properties(hotpath_prio, HOTPATH_AFFINITY); 125 | rear_exe.spin(); 126 | }}; 127 | std::thread fusion_thread {[&]() { 128 | set_rt_properties(hotpath_prio, HOTPATH_AFFINITY); 129 | fusion_exe.spin(); 130 | }}; 131 | std::thread planner_thread {[&]() { 132 | set_rt_properties(planner_prio, PLANNER_AFFINITY); 133 | planner_exe.spin(); 134 | }}; 135 | std::thread other_thread {[&]() { 136 | other_exe.spin(); 137 | }}; 138 | 139 | front_thread.join(); 140 | rear_thread.join(); 141 | fusion_thread.join(); 142 | planner_thread.join(); 143 | 144 | rclcpp::shutdown(); 145 | } 146 | -------------------------------------------------------------------------------- /autoware_reference_system/src/ros2/executor/autoware_default_multithreaded.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Apex.AI, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #include 15 | #include 16 | 17 | #include "rclcpp/rclcpp.hpp" 18 | 19 | #include "reference_system/system/type/rclcpp_system.hpp" 20 | 21 | #include "autoware_reference_system/autoware_system_builder.hpp" 22 | #include "autoware_reference_system/system/timing/benchmark.hpp" 23 | #include "autoware_reference_system/system/timing/default.hpp" 24 | 25 | int main(int argc, char * argv[]) 26 | { 27 | rclcpp::init(argc, argv); 28 | 29 | std::vector> nodes; 30 | if (argc == 2 && strncmp(argv[1], "benchmark", 9) == 0) { 31 | nodes = 32 | create_autoware_nodes(); 33 | } else if (argc == 2 && strncmp(argv[1], "quietbenchmark", 14) == 0) { 34 | set_benchmark_mode(true); 35 | nodes = 36 | create_autoware_nodes(); 37 | } else { 38 | nodes = create_autoware_nodes(); 39 | } 40 | 41 | rclcpp::executors::MultiThreadedExecutor executor; 42 | for (auto & node : nodes) { 43 | executor.add_node(node); 44 | } 45 | executor.spin(); 46 | 47 | nodes.clear(); 48 | rclcpp::shutdown(); 49 | 50 | return 0; 51 | } 52 | -------------------------------------------------------------------------------- /autoware_reference_system/src/ros2/executor/autoware_default_prioritized.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Robert Bosch GmbH 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | 22 | #include "rclcpp/rclcpp.hpp" 23 | 24 | #include "reference_system/system/type/rclcpp_system.hpp" 25 | 26 | #include "autoware_reference_system/autoware_system_builder.hpp" 27 | #include "autoware_reference_system/system/timing/benchmark.hpp" 28 | #include "autoware_reference_system/system/timing/default.hpp" 29 | #include "autoware_reference_system/priorities.hpp" 30 | 31 | void set_rt_properties(int prio, const std::unordered_set & affinity) 32 | { 33 | struct sched_param sched_param = { 0 }; 34 | sched_param.sched_priority = prio; 35 | sched_setscheduler(0, SCHED_RR, &sched_param); 36 | 37 | cpu_set_t cpuset; 38 | CPU_ZERO(&cpuset); 39 | for (const auto cpu : affinity) { 40 | CPU_SET(cpu, &cpuset); 41 | } 42 | sched_setaffinity(0, sizeof(cpuset), &cpuset); 43 | } 44 | 45 | int main(int argc, char ** argv) 46 | { 47 | rclcpp::init(argc, argv); 48 | 49 | using TimeConfig = nodes::timing::Default; 50 | // uncomment for benchmarking 51 | // using TimeConfig = nodes::timing::BenchmarkCPUUsage; 52 | // set_benchmark_mode(true); 53 | 54 | auto nodes_vec = create_autoware_nodes(); 55 | using NodeMap = std::unordered_map>; 57 | 58 | NodeMap nodes; 59 | for (const auto & node : nodes_vec) { 60 | nodes.emplace(node->get_name(), node); 61 | std::cout << node->get_name() << "\n"; 62 | } 63 | 64 | rclcpp::executors::SingleThreadedExecutor 65 | front_exe, 66 | rear_exe, 67 | fusion_exe, 68 | planner_exe, 69 | other_exe; 70 | 71 | std::set front_nodes = {"FrontLidarDriver", "PointsTransformerFront"}; 72 | std::set rear_nodes = {"RearLidarDriver", "PointsTransformerRear"}; 73 | std::set fusion_nodes = {"PointCloudFusion", 74 | "RayGroundFilter", 75 | "EuclideanClusterDetector", 76 | "ObjectCollisionEstimator"}; 77 | std::set planner_nodes = {"BehaviorPlanner"}; 78 | std::set other_nodes = {"PointCloudMap", 79 | "Visualizer", 80 | "Lanelet2Map", 81 | "EuclideanClusterSettings", 82 | "PointCloudMapLoader", 83 | "MPCController", 84 | "VehicleInterface", 85 | "VehicleDBWSystem", 86 | "NDTLocalizer", 87 | "Lanelet2GlobalPlanner", 88 | "Lanelet2MapLoader", 89 | "ParkingPlanner", 90 | "LanePlanner", 91 | "IntersectionOutput", 92 | "VoxelGridDownsampler"}; 93 | 94 | for (const auto & node : front_nodes) { 95 | front_exe.add_node(nodes.at(node)); 96 | } 97 | for (const auto & node : rear_nodes) { 98 | rear_exe.add_node(nodes.at(node)); 99 | } 100 | for (const auto & node : fusion_nodes) { 101 | std::cout << node << "\n"; 102 | fusion_exe.add_node(nodes.at(node)); 103 | } 104 | for (const auto & node : planner_nodes) { 105 | planner_exe.add_node(nodes.at(node)); 106 | } 107 | for (const auto & node : other_nodes) { 108 | other_exe.add_node(nodes.at(node)); 109 | } 110 | 111 | std::thread front_thread {[&]() { 112 | set_rt_properties(hotpath_prio, HOTPATH_AFFINITY); 113 | front_exe.spin(); 114 | }}; 115 | std::thread rear_thread {[&]() { 116 | set_rt_properties(hotpath_prio, HOTPATH_AFFINITY); 117 | rear_exe.spin(); 118 | }}; 119 | std::thread fusion_thread {[&]() { 120 | set_rt_properties(hotpath_prio, HOTPATH_AFFINITY); 121 | fusion_exe.spin(); 122 | }}; 123 | std::thread planner_thread {[&]() { 124 | set_rt_properties(planner_prio, PLANNER_AFFINITY); 125 | planner_exe.spin(); 126 | }}; 127 | std::thread other_thread {[&]() { 128 | other_exe.spin(); 129 | }}; 130 | 131 | front_thread.join(); 132 | rear_thread.join(); 133 | fusion_thread.join(); 134 | planner_thread.join(); 135 | 136 | rclcpp::shutdown(); 137 | } 138 | -------------------------------------------------------------------------------- /autoware_reference_system/src/ros2/executor/autoware_default_singlethreaded.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Apex.AI, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "rclcpp/rclcpp.hpp" 16 | 17 | #include "reference_system/system/type/rclcpp_system.hpp" 18 | 19 | #include "autoware_reference_system/autoware_system_builder.hpp" 20 | #include "autoware_reference_system/system/timing/benchmark.hpp" 21 | #include "autoware_reference_system/system/timing/default.hpp" 22 | 23 | int main(int argc, char * argv[]) 24 | { 25 | rclcpp::init(argc, argv); 26 | 27 | using TimeConfig = nodes::timing::Default; 28 | // uncomment for benchmarking 29 | // using TimeConfig = nodes::timing::BenchmarkCPUUsage; 30 | // set_benchmark_mode(true); 31 | 32 | auto nodes = create_autoware_nodes(); 33 | 34 | rclcpp::executors::SingleThreadedExecutor executor; 35 | for (auto & node : nodes) { 36 | executor.add_node(node); 37 | } 38 | executor.spin(); 39 | 40 | nodes.clear(); 41 | rclcpp::shutdown(); 42 | 43 | return 0; 44 | } 45 | -------------------------------------------------------------------------------- /autoware_reference_system/src/ros2/executor/autoware_default_staticsinglethreaded.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Apex.AI, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "rclcpp/rclcpp.hpp" 16 | 17 | #include "reference_system/system/type/rclcpp_system.hpp" 18 | 19 | #include "autoware_reference_system/autoware_system_builder.hpp" 20 | #include "autoware_reference_system/system/timing/benchmark.hpp" 21 | #include "autoware_reference_system/system/timing/default.hpp" 22 | 23 | int main(int argc, char * argv[]) 24 | { 25 | rclcpp::init(argc, argv); 26 | 27 | using TimeConfig = nodes::timing::Default; 28 | // uncomment for benchmarking 29 | // using TimeConfig = nodes::timing::BenchmarkCPUUsage; 30 | // set_benchmark_mode(true); 31 | 32 | auto nodes = create_autoware_nodes(); 33 | 34 | rclcpp::executors::StaticSingleThreadedExecutor executor; 35 | for (auto & node : nodes) { 36 | executor.add_node(node); 37 | } 38 | executor.spin(); 39 | 40 | nodes.clear(); 41 | rclcpp::shutdown(); 42 | 43 | return 0; 44 | } 45 | -------------------------------------------------------------------------------- /autoware_reference_system/test/conftest.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Apex.AI, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | def pytest_configure(config): 17 | """Document pytest.mark.TEST_ID to avoid pytest warning.""" 18 | config.addinivalue_line('markers', 'TEST_ID: Append a test ID to a test case') 19 | 20 | 21 | def pytest_collection_modifyitems(items): 22 | for item in items: 23 | for marker in item.iter_markers(name='TEST_ID'): 24 | test_id = marker.args[0] 25 | item.user_properties.append(('TEST_ID', test_id)) 26 | -------------------------------------------------------------------------------- /autoware_reference_system/test/test_autoware_reference_system.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Apex.AI, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "gtest/gtest.h" 16 | 17 | TEST(TestReferenceSystemAutoware, DummyTest) { 18 | EXPECT_EQ(1, 1); 19 | } 20 | -------------------------------------------------------------------------------- /autoware_reference_system/test/test_platform.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Apex.AI, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import multiprocessing 16 | import platform 17 | 18 | # this file has @variables@ that are meant to be automatically replaced 19 | # by values using the `configure_file` CMake function during the build 20 | 21 | platforms = {} 22 | 23 | # TODO(flynneva): move this to its own file for portability 24 | # can add more supported platforms here 25 | platforms['rpi4-linux-rt'] = { 26 | 'common-name': 'raspberrypi4', 27 | 'machine': 'aarch64', 28 | 'processor': 'aarch64', 29 | 'system': 'Linux', 30 | 'flavor': 'ubuntu', 31 | 'cores': 4, 32 | 'realtime': True 33 | } 34 | 35 | 36 | def test_platform(record_property): 37 | # get current system information 38 | system, node, release, version, machine, processor = platform.uname() 39 | platform_supported = False 40 | for pform in platforms: 41 | if(platforms[pform]['system'] == system): 42 | if(platforms[pform]['processor'] == processor): 43 | platform_supported = True 44 | assert multiprocessing.cpu_count() == platforms[pform]['cores'] 45 | if(platforms[pform]['realtime']): 46 | assert 'PREEMPT_RT' in version 47 | if platform_supported: 48 | print('platform supported') 49 | assert True 50 | else: 51 | print('platform unsupported') 52 | assert False 53 | -------------------------------------------------------------------------------- /autoware_reference_system/test/test_requirements.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Apex.AI, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import os 15 | import time 16 | import unittest 17 | 18 | from launch import LaunchDescription 19 | from launch.actions import ExecuteProcess 20 | from launch_testing.actions import ReadyToTest 21 | 22 | import rclpy.context 23 | 24 | from ros2cli.node.direct import DirectNode 25 | import ros2topic.api 26 | 27 | # Tests to check if executable complies with the requirements for 28 | # the autoware_reference_system by checking number of nodes, publishers, 29 | # subscribers and frequency of some topics 30 | 31 | # this file has @variables@ that are meant to be automatically replaced 32 | # by values using the `configure_file` CMake function during the build 33 | 34 | checks = {'topic_exists': False, 'pubs_match': False, 'subs_match': False} 35 | 36 | # define autoware_reference_system requirements for each topic 37 | # NOTE: the pub/sub counts are for the topic, not the node itself 38 | reference_system = { 39 | '/FrontLidarDriver': {'pub_count': 1, 'sub_count': 1, 'checks': checks.copy()}, 40 | '/RearLidarDriver': {'pub_count': 1, 'sub_count': 1, 'checks': checks.copy()}, 41 | '/PointCloudMap': {'pub_count': 1, 'sub_count': 1, 'checks': checks.copy()}, 42 | '/Visualizer': {'pub_count': 1, 'sub_count': 1, 'checks': checks.copy()}, 43 | '/Lanelet2Map': {'pub_count': 1, 'sub_count': 1, 'checks': checks.copy()}, 44 | '/PointsTransformerFront': {'pub_count': 1, 'sub_count': 1, 'checks': checks.copy()}, 45 | '/PointsTransformerRear': {'pub_count': 1, 'sub_count': 1, 'checks': checks.copy()}, 46 | '/VoxelGridDownsampler': {'pub_count': 1, 'sub_count': 1, 'checks': checks.copy()}, 47 | '/PointCloudMapLoader': {'pub_count': 1, 'sub_count': 1, 'checks': checks.copy()}, 48 | '/EuclideanClusterDetector': {'pub_count': 1, 'sub_count': 1, 'checks': checks.copy()}, 49 | '/ObjectCollisionEstimator': {'pub_count': 1, 'sub_count': 1, 'checks': checks.copy()}, 50 | '/MPCController': {'pub_count': 1, 'sub_count': 1, 'checks': checks.copy()}, 51 | '/ParkingPlanner': {'pub_count': 1, 'sub_count': 1, 'checks': checks.copy()}, 52 | '/LanePlanner': {'pub_count': 1, 'sub_count': 1, 'checks': checks.copy()}, 53 | '/PointCloudFusion': {'pub_count': 1, 'sub_count': 2, 'checks': checks.copy()}, 54 | '/NDTLocalizer': {'pub_count': 1, 'sub_count': 2, 'checks': checks.copy()}, 55 | '/VehicleInterface': {'pub_count': 1, 'sub_count': 1, 'checks': checks.copy()}, 56 | '/Lanelet2GlobalPlanner': {'pub_count': 1, 'sub_count': 2, 'checks': checks.copy()}, 57 | '/Lanelet2MapLoader': {'pub_count': 1, 'sub_count': 3, 'checks': checks.copy()}, 58 | '/BehaviorPlanner': {'pub_count': 1, 'sub_count': 2, 'checks': checks.copy()} 59 | # does not exist as topic but only as a node 60 | # '/VehicleDBWSystem': {'pub_count': 0, 'sub_count': 0, 'checks': checks.copy()} 61 | } 62 | 63 | 64 | def generate_test_description(): 65 | env = os.environ.copy() 66 | env['RCUTILS_CONSOLE_OUTPUT_FORMAT'] = '[{severity}] [{name}]: {message}' 67 | 68 | launch_description = LaunchDescription() 69 | 70 | context = rclpy.context.Context() 71 | rclpy.init(context=context) 72 | launch_description.add_action(ReadyToTest()) 73 | 74 | proc_under_test = ExecuteProcess( 75 | cmd=['@TEST_EXECUTABLE@'], 76 | name='@TEST_EXECUTABLE_NAME@', 77 | output='screen', 78 | env=env, 79 | ) 80 | launch_description.add_action(proc_under_test) 81 | return launch_description, locals() 82 | 83 | 84 | class TestRequirementsAutowareReferenceSystem(unittest.TestCase): 85 | 86 | def test_pubs_and_subs(self): 87 | with DirectNode([]) as node: 88 | seen_topics = {} 89 | try: 90 | while True: 91 | print('topic_monitor looping:') 92 | for name in ros2topic.api.get_topic_names(node=node): 93 | if name not in seen_topics: 94 | seen_topics[name] = {'pub_count': 0, 'sub_count': 0} 95 | publishers = node.count_publishers(name) 96 | subscribers = node.count_subscribers(name) 97 | 98 | if seen_topics[name]['pub_count'] < publishers: 99 | seen_topics[name]['pub_count'] = publishers 100 | 101 | if seen_topics[name]['sub_count'] < subscribers: 102 | seen_topics[name]['sub_count'] = subscribers 103 | 104 | if len(seen_topics) > 0: 105 | print('Topic monitor data collected') 106 | for name in reference_system: 107 | if name in seen_topics.keys(): 108 | reference_system[name]['checks']['topic_exists'] = True 109 | 110 | if(reference_system[name]['pub_count'] == 111 | seen_topics[name]['pub_count']): 112 | reference_system[name]['checks']['pubs_match'] = True 113 | else: 114 | print('[' + name + '::pubs] EXPECTED: ' + 115 | str(reference_system[name]['pub_count'])) 116 | print('[' + name + '::pubs] RECEIVED: ' + 117 | str(seen_topics[name]['pub_count'])) 118 | 119 | if(reference_system[name]['sub_count'] == 120 | seen_topics[name]['sub_count']): 121 | reference_system[name]['checks']['subs_match'] = True 122 | else: 123 | print('[' + name + '::subs] EXPECTED: ' + 124 | str(reference_system[name]['sub_count'])) 125 | print('[' + name + '::subs] RECEIVED: ' + 126 | str(seen_topics[name]['sub_count'])) 127 | 128 | self.assertTrue(all(reference_system[name]['checks'].values())) 129 | print( 130 | f'\t\t{name}: ' 131 | f"['topic_exists'=" 132 | f"{reference_system[name]['checks']['topic_exists']}," 133 | f" 'pubs_match'=" 134 | f"{reference_system[name]['checks']['pubs_match']}," 135 | f" 'subs_match'=" 136 | f"{reference_system[name]['checks']['subs_match']}]") 137 | # exit while loop, data was collected 138 | return 139 | # slow down while loop 140 | time.sleep(0.5) 141 | except SystemError: 142 | pass 143 | except KeyboardInterrupt: 144 | pass 145 | -------------------------------------------------------------------------------- /docs/.pages: -------------------------------------------------------------------------------- 1 | nav: 2 | - Home: index.md 3 | - Reference System: the-reference-system.md 4 | - Autoware Reference System: the-autoware-reference-system.md 5 | - Reports: reports 6 | - API Reference: api-reference 7 | - ... -------------------------------------------------------------------------------- /docs/doxybook2_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "baseUrl": "/reference-system/api-reference/", 3 | "copyImages": true, 4 | "fileExt": "md", 5 | "filesFilter": [], 6 | "folderClassesName": "Classes", 7 | "folderGroupsName": "Modules", 8 | "folderNamespacesName": "Namespaces", 9 | "foldersToGenerate": [ 10 | "modules", 11 | "classes", 12 | "namespaces" 13 | ], 14 | "formulaBlockEnd": "\\]", 15 | "formulaBlockStart": "\\[", 16 | "formulaInlineEnd": "\\)", 17 | "formulaInlineStart": "\\(", 18 | "imagesFolder": "images", 19 | "indexClassesName": "index", 20 | "indexClassesTitle": "Classes", 21 | "indexExamplesName": "index", 22 | "indexGroupsName": "index", 23 | "indexGroupsTitle": "Modules", 24 | "indexInFolders": true, 25 | "indexNamespacesName": "index", 26 | "indexNamespacesTitle": "Namespaces", 27 | "linkAndInlineCodeAsHTML": false, 28 | "linkLowercase": false, 29 | "linkSuffix": "/", 30 | "mainPageInRoot": true, 31 | "mainPageName": "index", 32 | "sort": true, 33 | "templateIndexClasses": "index_classes", 34 | "templateIndexGroups": "index_groups", 35 | "templateIndexNamespaces": "index_namespaces", 36 | "templateKindClass": "kind_class", 37 | "templateKindDir": "kind_file", 38 | "templateKindGroup": "kind_nonclass", 39 | "templateKindInterface": "kind_class", 40 | "templateKindNamespace": "kind_nonclass", 41 | "templateKindStruct": "kind_class", 42 | "templateKindUnion": "kind_class", 43 | "useFolders": true 44 | } 45 | -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | ../README.md -------------------------------------------------------------------------------- /docs/overrides/main.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | 3 | {% block outdated %} 4 | You're not viewing the latest version. 5 | 6 | 7 | Click here to go to latest. 8 | 9 | {% endblock %} 10 | -------------------------------------------------------------------------------- /docs/reports/index.md: -------------------------------------------------------------------------------- 1 | # Reference system reports 2 | 3 | Eventually this page will hold CI-run benchmarks that are updated regularly. 4 | 5 | Still a work in progress but please check back soon. 6 | 7 | ### Disclaimer regarding the reports 8 | 9 | The reports hosted here are ran on a vanilla Ubuntu CI system and should not be taken as 10 | the true performance of the system if it was running on a real-time embedded system. They are 11 | meant as a rough approximation for anyone looking for some quick data. 12 | 13 | For reports actually ran on a real-time benchmark system please see 14 | [the ROS 2 real-time benchmarks](https://ros-realtime.github.io/ros2_realtime_benchmarks/) site. 15 | 16 | If neither of the above solutions are appropriate run the reference system manually on the desired 17 | platform and generate the reports [following the instructions](../the-autoware-reference-system.md). 18 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | mkdocs==1.2.3 2 | mkdocs-material==7.3.6 3 | mkdocs-awesome-pages-plugin==2.7.0 4 | mike==1.1.2 -------------------------------------------------------------------------------- /docs/the-autoware-reference-system.md: -------------------------------------------------------------------------------- 1 | ../autoware_reference_system/README.md -------------------------------------------------------------------------------- /docs/the-reference-system.md: -------------------------------------------------------------------------------- 1 | ../reference_system/README.md -------------------------------------------------------------------------------- /mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: Reference System Documentation 2 | site_url: https://ros-realtime.github.io/reference-system/ 3 | site_author: Evan Flynn 4 | repo_url: https://github.com/ros-realtime/reference-system.git 5 | theme: 6 | name: 'material' 7 | features: 8 | - navigation.indexes 9 | - navigation.tracking 10 | plugins: 11 | - search 12 | - awesome-pages 13 | extra: 14 | version: 15 | provider: mike 16 | -------------------------------------------------------------------------------- /reference_interfaces/CHANGELOG.rst: -------------------------------------------------------------------------------- 1 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 2 | Changelog for package reference_interfaces 3 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 4 | 5 | v1.1.0 6 | ------ 7 | * Add Iron ROS 2 distribution 8 | * Remove EoL distributions Foxy and Galactic 9 | * Remove legacy hack for rosidl_geneartor_py 10 | 11 | v1.0.0 12 | ----------- 13 | * Add first changelog 14 | * Bump version of reference_system packages to 1.0.0 15 | * Add test_depend on ament_lint\_{auto,common} for reference_interfaces 16 | * patch version bump to include mutex for prints 17 | * initial release for each package 18 | * Add sequence number, behavior planner period, timestamp relates to callback startup 19 | * Number cruncher uses upper limit instead of timeout, add number cruncher benchmark to help find the best limits for a system 20 | * remove cpu and msg tests for now 21 | * switch target name to wait for idl fix 22 | * add warning and fix for generated idl file bug 23 | * add idl bug work-around as cmake post-build cmd 24 | * seperate out reference_system vs autoware specifics 25 | * Apply suggestions from code review 26 | * Sample tracing implemented 27 | * remove header from msg 28 | * move msgs to their own package, use ament auto 29 | * Contributors: Christian, Christian Eltzschig, Christophe Bedard, Evan Flynn, Lander Usategui 30 | -------------------------------------------------------------------------------- /reference_interfaces/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.5) 2 | project(reference_interfaces) 3 | 4 | # Default to C++17 5 | if(NOT CMAKE_CXX_STANDARD) 6 | set(CMAKE_CXX_STANDARD 17) 7 | endif() 8 | 9 | if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") 10 | add_compile_options(-Wall -Wextra -Wpedantic) 11 | endif() 12 | 13 | find_package(ament_cmake_auto REQUIRED) 14 | ament_auto_find_build_dependencies() 15 | 16 | # add additional messages here 17 | set(msg_files 18 | "msg/TransmissionStats.idl" 19 | "msg/Message4kb.idl" 20 | ) 21 | 22 | # add additional message dependencies here 23 | #set(msg_dependencies 24 | # "std_msgs" 25 | #) 26 | 27 | rosidl_generate_interfaces(${PROJECT_NAME} 28 | ${msg_files} 29 | DEPENDENCIES 30 | ${msg_dependencies} 31 | ADD_LINTER_TESTS 32 | ) 33 | 34 | ament_auto_package() 35 | -------------------------------------------------------------------------------- /reference_interfaces/msg/Message4kb.idl: -------------------------------------------------------------------------------- 1 | #include "reference_interfaces/msg/TransmissionStats.idl" 2 | 3 | module reference_interfaces { 4 | module msg { 5 | module Message4kb_Constants { 6 | const uint64 STATS_CAPACITY = 63; 7 | }; 8 | struct Message4kb { 9 | uint64 size; // 8 10 | reference_interfaces::msg::TransmissionStats stats[63]; // + 4032 = 63 * 64 11 | int64 data[7]; // + 56 = 7 * 8 12 | //----------------- 13 | // 4096 14 | }; 15 | }; 16 | }; 17 | -------------------------------------------------------------------------------- /reference_interfaces/msg/TransmissionStats.idl: -------------------------------------------------------------------------------- 1 | module reference_interfaces { 2 | module msg { 3 | module TransmissionStats_Constants { 4 | const uint64 NODE_NAME_LENGTH = 48; 5 | }; 6 | struct TransmissionStats { // 64 bytes 7 | uint64 timestamp; 8 | uint32 sequence_number; 9 | uint32 dropped_samples; 10 | char node_name[48]; 11 | }; 12 | }; 13 | }; 14 | -------------------------------------------------------------------------------- /reference_interfaces/package.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | reference_interfaces 5 | 1.1.0 6 | Reference system for the ROScon workshop 7 | Christian Eltzschig 8 | Apache License 2.0 9 | 10 | ament_cmake_auto 11 | rosidl_default_generators 12 | 13 | rosidl_default_runtime 14 | 15 | ament_lint_auto 16 | ament_lint_common 17 | 18 | rosidl_interface_packages 19 | 20 | 21 | ament_cmake 22 | 23 | 24 | -------------------------------------------------------------------------------- /reference_system/CHANGELOG.rst: -------------------------------------------------------------------------------- 1 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 2 | Changelog for package reference_system 3 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 4 | 5 | v1.1.0 6 | ------ 7 | * Add Iron ROS 2 distribution 8 | * Remove EoL distributions Foxy and Galactic 9 | * Remove legacy hack for rosidl_geneartor_py 10 | 11 | v1.0.0 12 | ----------- 13 | * Add first changelog 14 | * Bump version of reference_system packages to 1.0.0 15 | * Update reference_system docs, logic in various places 16 | * Migrate benchmark scripts to python 17 | * [91] skip node graph in reference_system tests for now 18 | * [91] add unit and integration tests for the reference system, fix some bugs found by tests 19 | * Added callback group per subscription in Intersection node. 20 | * patch version bump to include mutex for prints 21 | * Guard cout with mutex 22 | * initial release for each package 23 | * Add missing const keywords 24 | * add std trace type, generate summary report 25 | * Fix latency calculation error, adjust table header 26 | * Uncrustify code 27 | * Add advanced dropped sample statistics for each node 28 | * Add dropped samples statistics 29 | * Add sequence number, behavior planner period, timestamp relates to callback startup 30 | * Add more advanced statistics for latency 31 | * Lint and uncrustify 32 | * Add intersection node docu 33 | * Add intersection node 34 | * fix cpp header guards, timing bug 35 | * Uncrustify and adjust number cruncher 36 | * Cyclic node has number crunching 37 | * Cyclic node with cyclic trigger implemented 38 | * Rename Reactor node into Cyclic node 39 | * Remove time diff check from fusion node 40 | * Number cruncher uses upper limit instead of timeout, add number cruncher benchmark to help find the best limits for a system 41 | * Add Fusion node max input timediff and fail when it is exceeded 42 | * rename nodes, update docs, add platform test 43 | * clean up cmake, reorg READMEs 44 | * seperate out reference_system vs autoware specifics 45 | * fix uncrustify errors 46 | * Add more fine grained timing configuration 47 | * fix README links, add requirements doc, linter cleanups 48 | * fix lint errors 49 | * Apply suggestions from code review 50 | * Add benchmark comments and add custom timings in system builder 51 | * Add benchmark timing 52 | * Add benchmark mode 53 | * Uncomment code for single threaded executors 54 | * update README, clean up system naming 55 | * reorg, lint and clean up 56 | * Move processing times into config 57 | * Remove warnings 58 | * Write number crunch result at first index 59 | * Rename SampleType to SampleTypePointer to make it clear that we expect a pointer 60 | * Config is stored in struct to allow multiple configurations in parallel 61 | * Remove pub/sub port alias 62 | * Rename reference_system_autoware into reference_system 63 | * Contributors: Christian, Christian Eltzschig, Christophe Bedard, Evan Flynn, Lander Usategui, Ralph Lange 64 | -------------------------------------------------------------------------------- /reference_system/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.5) 2 | project(reference_system) 3 | 4 | set(FRAMEWORK ros CACHE STRING "The framework to build for. Currently supported values are ROS 2 (\"ros\").") 5 | set(AVAILABLE_FRAMEWORKS ros) 6 | 7 | if(NOT ${FRAMEWORK} IN_LIST AVAILABLE_FRAMEWORKS) 8 | message(FATAL_ERROR "Unsupported framework: ${FRAMEWORK}") 9 | endif() 10 | 11 | if(NOT CMAKE_CXX_STANDARD) 12 | set(CMAKE_CXX_STANDARD 17) 13 | endif() 14 | 15 | if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") 16 | add_compile_options(-Wall -Wextra -Wpedantic) 17 | endif() 18 | 19 | find_package(ament_cmake_auto REQUIRED) 20 | ament_auto_find_build_dependencies() 21 | 22 | # Add header-only library 23 | add_library(${PROJECT_NAME} INTERFACE) 24 | 25 | target_include_directories(${PROJECT_NAME} INTERFACE 26 | $ 27 | $ 28 | ) 29 | 30 | if(${BUILD_TESTING}) 31 | find_package(ament_lint_auto REQUIRED) 32 | ament_lint_auto_find_test_dependencies() 33 | 34 | find_package(ament_cmake_gtest) 35 | 36 | # unit tests 37 | ament_add_gtest(test_sample_management 38 | test/test_sample_management.cpp) 39 | target_link_libraries(test_sample_management ${PROJECT_NAME}) 40 | ament_target_dependencies(test_sample_management reference_interfaces) 41 | 42 | ament_add_gtest(test_number_cruncher 43 | test/test_number_cruncher.cpp) 44 | target_link_libraries(test_number_cruncher ${PROJECT_NAME}) 45 | 46 | # integration tests 47 | if(${FRAMEWORK} STREQUAL ros) 48 | ament_add_gtest(test_reference_system_rclcpp 49 | test/test_fixtures.hpp 50 | test/test_reference_system_rclcpp.cpp) 51 | target_link_libraries(test_reference_system_rclcpp ${PROJECT_NAME}) 52 | ament_target_dependencies(test_reference_system_rclcpp reference_interfaces rclcpp) 53 | endif() 54 | endif() 55 | 56 | # Install 57 | install(TARGETS ${PROJECT_NAME} 58 | EXPORT "export_${PROJECT_NAME}" 59 | ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} 60 | LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} 61 | RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} 62 | INCLUDES DESTINATION include 63 | ) 64 | 65 | ament_python_install_package(reference_system_py) 66 | 67 | ament_auto_package( 68 | INSTALL_TO_SHARE cfg 69 | ) 70 | -------------------------------------------------------------------------------- /reference_system/README.md: -------------------------------------------------------------------------------- 1 | # The `reference_system` package 2 | 3 | The `reference_system` package provides reusable base components (nodes) that allow users 4 | to construct any number of "reference systems" to replicate real-world scenarios on 5 | real-world hardware. 6 | 7 | The purpose of these components are not to provide the most efficient way of writing a node 8 | but rather a reusable and repeatible way of writing a node so that it can be used to 9 | reliably test various key performance indicators later on. 10 | 11 | ## Base node types 12 | 13 | Most real-world systems can be boiled down to only a handful of base node "types" that are then 14 | repeated to make the real-world system. 15 | This does not cover _all_ possible node types, however it allows 16 | for numerous complicated systems to be developed using the same base building blocks. 17 | 18 | 1. **Sensor Node** 19 | - input node to system 20 | - one publisher, zero subscribers 21 | - publishes message cyclically at some fixed frequency 22 | 2. **Transform Node** 23 | - one subscriber, one publisher 24 | - starts processing for N milliseconds after a message is received 25 | - publishes message after processing is complete 26 | 3. **Fusion Node** 27 | - 2 subscribers, one publisher 28 | - starts processing for N milliseconds after a message is received **from all** subscriptions 29 | - publishes message after processing is complete 30 | 4. **Cyclic Node** 31 | - N subscribers, one publisher 32 | - cyclically processes all received messages since the last cycle for N milliseconds 33 | - publishes message after processing is complete 34 | 5. **Command Node** 35 | - prints output stats everytime a message is received 36 | 6. **Intersection Node** 37 | - behaves like N transform nodes 38 | - N subscribers, N publisher bundled together in one-to-one connections 39 | - starts processing on connection where sample was received 40 | - publishes message after processing is complete 41 | 42 | These basic building-block nodes can be mixed-and-matched to create quite complex systems that 43 | replicate real-world scenarios to benchmark different configurations against each other. 44 | 45 | New base node types can be added if necessary. 46 | 47 | ## Testing and Dependencies 48 | 49 | Common benchmarking scripts are provided within the `reference_system/reference_system_py` 50 | directory which is a python module itself. The methods and tools provided there can assist 51 | with running standardized benchmarking tests and with generating reports as well. See 52 | [the `autoware_reference_system` for an example](../autoware_reference_system/scripts/benchmark.py) 53 | 54 | Unit and integration tests have also been written for the `reference_system` package and can be found 55 | within [the `test` directory](test/test_reference_system_rclcpp.cpp). If a new system type is 56 | to be added, new unit and integration tests should also be added as well. -------------------------------------------------------------------------------- /reference_system/cfg/memory_report_template.md: -------------------------------------------------------------------------------- 1 | # Memory and CPU usage report 2 | 3 | 4 | 5 | 6 | {{ bokeh_script|safe }} 7 | 8 | {{ cpu_table|safe }} 9 | {{ cpu_fig|safe }} 10 | 11 | {{ real_mem_table|safe }} 12 | {{ real_mem_fig|safe }} 13 | 14 | {{ virtual_mem_table|safe }} 15 | {{ virtual_mem_fig|safe }} -------------------------------------------------------------------------------- /reference_system/cfg/std_report_template.md: -------------------------------------------------------------------------------- 1 | # Key performance indicator report 2 | 3 | 4 | 5 | 6 | {{ bokeh_script|safe }} 7 | 8 | {{ latency_table|safe }} 9 | {{ latency_fig|safe }} 10 | 11 | {{ dropped_table|safe }} 12 | {{ dropped_fig|safe }} 13 | 14 | {{ period_table|safe }} 15 | {{ period_fig|safe }} -------------------------------------------------------------------------------- /reference_system/include/reference_system/msg_types.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Apex.AI, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #ifndef REFERENCE_SYSTEM__MSG_TYPES_HPP_ 15 | #define REFERENCE_SYSTEM__MSG_TYPES_HPP_ 16 | 17 | #include "reference_interfaces/msg/message4kb.hpp" 18 | 19 | using message_t = reference_interfaces::msg::Message4kb; 20 | 21 | #endif // REFERENCE_SYSTEM__MSG_TYPES_HPP_ 22 | -------------------------------------------------------------------------------- /reference_system/include/reference_system/nodes/rclcpp/command.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Apex.AI, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #ifndef REFERENCE_SYSTEM__NODES__RCLCPP__COMMAND_HPP_ 15 | #define REFERENCE_SYSTEM__NODES__RCLCPP__COMMAND_HPP_ 16 | 17 | #include 18 | #include 19 | 20 | #include "rclcpp/rclcpp.hpp" 21 | #include "reference_system/msg_types.hpp" 22 | #include "reference_system/nodes/settings.hpp" 23 | #include "reference_system/sample_management.hpp" 24 | 25 | namespace nodes 26 | { 27 | namespace rclcpp_system 28 | { 29 | 30 | class Command : public rclcpp::Node 31 | { 32 | public: 33 | explicit Command(const CommandSettings & settings) 34 | : Node(settings.node_name) 35 | { 36 | subscription_ = this->create_subscription( 37 | settings.input_topic, 10, 38 | [this](const message_t::SharedPtr msg) {input_callback(msg);}); 39 | } 40 | 41 | private: 42 | void input_callback(const message_t::SharedPtr input_message) 43 | { 44 | uint32_t missed_samples = 45 | get_missed_samples_and_update_seq_nr(input_message, sequence_number_); 46 | print_sample_path(this->get_name(), missed_samples, input_message); 47 | } 48 | 49 | private: 50 | rclcpp::Subscription::SharedPtr subscription_; 51 | uint32_t sequence_number_ = 0; 52 | }; 53 | } // namespace rclcpp_system 54 | } // namespace nodes 55 | #endif // REFERENCE_SYSTEM__NODES__RCLCPP__COMMAND_HPP_ 56 | -------------------------------------------------------------------------------- /reference_system/include/reference_system/nodes/rclcpp/cyclic.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Apex.AI, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #ifndef REFERENCE_SYSTEM__NODES__RCLCPP__CYCLIC_HPP_ 15 | #define REFERENCE_SYSTEM__NODES__RCLCPP__CYCLIC_HPP_ 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #include "rclcpp/rclcpp.hpp" 22 | #include "reference_system/msg_types.hpp" 23 | #include "reference_system/nodes/settings.hpp" 24 | #include "reference_system/number_cruncher.hpp" 25 | #include "reference_system/sample_management.hpp" 26 | 27 | namespace nodes 28 | { 29 | namespace rclcpp_system 30 | { 31 | 32 | class Cyclic : public rclcpp::Node 33 | { 34 | public: 35 | explicit Cyclic(const CyclicSettings & settings) 36 | : Node(settings.node_name), 37 | number_crunch_limit_(settings.number_crunch_limit) 38 | { 39 | uint64_t input_number = 0U; 40 | for (const auto & input_topic : settings.inputs) { 41 | subscriptions_.emplace_back( 42 | subscription_t{ 43 | this->create_subscription( 44 | input_topic, 1, 45 | [this, input_number](const message_t::SharedPtr msg) { 46 | input_callback(input_number, msg); 47 | }), 48 | 0, message_t::SharedPtr()}); 49 | ++input_number; 50 | } 51 | publisher_ = this->create_publisher(settings.output_topic, 1); 52 | timer_ = this->create_wall_timer( 53 | settings.cycle_time, 54 | [this] {timer_callback();}); 55 | } 56 | 57 | private: 58 | void input_callback( 59 | const uint64_t input_number, 60 | const message_t::SharedPtr input_message) 61 | { 62 | subscriptions_[input_number].cache = input_message; 63 | } 64 | 65 | void timer_callback() 66 | { 67 | uint64_t timestamp = now_as_int(); 68 | auto number_cruncher_result = number_cruncher(number_crunch_limit_); 69 | 70 | auto output_message = publisher_->borrow_loaned_message(); 71 | output_message.get().size = 0; 72 | 73 | uint32_t missed_samples = 0; 74 | for (auto & s : subscriptions_) { 75 | if (!s.cache) { 76 | continue; 77 | } 78 | 79 | missed_samples += 80 | get_missed_samples_and_update_seq_nr(s.cache, s.sequence_number); 81 | 82 | merge_history_into_sample(output_message.get(), s.cache); 83 | s.cache.reset(); 84 | } 85 | set_sample( 86 | this->get_name(), sequence_number_++, missed_samples, timestamp, 87 | output_message.get()); 88 | 89 | output_message.get().data[0] = number_cruncher_result; 90 | publisher_->publish(std::move(output_message)); 91 | } 92 | 93 | private: 94 | rclcpp::Publisher::SharedPtr publisher_; 95 | rclcpp::TimerBase::SharedPtr timer_; 96 | 97 | struct subscription_t 98 | { 99 | rclcpp::Subscription::SharedPtr subscription; 100 | uint32_t sequence_number = 0; 101 | message_t::SharedPtr cache; 102 | }; 103 | 104 | std::vector subscriptions_; 105 | uint64_t number_crunch_limit_; 106 | uint32_t sequence_number_ = 0; 107 | }; 108 | } // namespace rclcpp_system 109 | } // namespace nodes 110 | #endif // REFERENCE_SYSTEM__NODES__RCLCPP__CYCLIC_HPP_ 111 | -------------------------------------------------------------------------------- /reference_system/include/reference_system/nodes/rclcpp/fusion.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Apex.AI, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #ifndef REFERENCE_SYSTEM__NODES__RCLCPP__FUSION_HPP_ 15 | #define REFERENCE_SYSTEM__NODES__RCLCPP__FUSION_HPP_ 16 | #include 17 | #include 18 | #include 19 | 20 | #include "rclcpp/rclcpp.hpp" 21 | #include "reference_system/msg_types.hpp" 22 | #include "reference_system/nodes/settings.hpp" 23 | #include "reference_system/number_cruncher.hpp" 24 | #include "reference_system/sample_management.hpp" 25 | 26 | namespace nodes 27 | { 28 | namespace rclcpp_system 29 | { 30 | 31 | class Fusion : public rclcpp::Node 32 | { 33 | public: 34 | explicit Fusion(const FusionSettings & settings) 35 | : Node(settings.node_name), 36 | number_crunch_limit_(settings.number_crunch_limit) 37 | { 38 | subscriptions_[0].subscription = this->create_subscription( 39 | settings.input_0, 1, 40 | [this](const message_t::SharedPtr msg) {input_callback(0U, msg);}); 41 | 42 | subscriptions_[1].subscription = this->create_subscription( 43 | settings.input_1, 1, 44 | [this](const message_t::SharedPtr msg) {input_callback(1U, msg);}); 45 | publisher_ = this->create_publisher(settings.output_topic, 1); 46 | } 47 | 48 | private: 49 | void input_callback( 50 | const uint64_t input_number, 51 | const message_t::SharedPtr input_message) 52 | { 53 | uint64_t timestamp = now_as_int(); 54 | subscriptions_[input_number].cache = input_message; 55 | 56 | // only process and publish when we can perform an actual fusion, this means 57 | // we have received a sample from each subscription 58 | if (!subscriptions_[0].cache || !subscriptions_[1].cache) { 59 | return; 60 | } 61 | 62 | auto number_cruncher_result = number_cruncher(number_crunch_limit_); 63 | 64 | auto output_message = publisher_->borrow_loaned_message(); 65 | 66 | uint32_t missed_samples = 67 | get_missed_samples_and_update_seq_nr( 68 | subscriptions_[0].cache, subscriptions_[0].sequence_number) + 69 | get_missed_samples_and_update_seq_nr( 70 | subscriptions_[1].cache, 71 | subscriptions_[1].sequence_number); 72 | 73 | output_message.get().size = 0; 74 | merge_history_into_sample(output_message.get(), subscriptions_[0].cache); 75 | merge_history_into_sample(output_message.get(), subscriptions_[1].cache); 76 | set_sample( 77 | this->get_name(), sequence_number_++, missed_samples, timestamp, 78 | output_message.get()); 79 | 80 | output_message.get().data[0] = number_cruncher_result; 81 | publisher_->publish(std::move(output_message)); 82 | 83 | subscriptions_[0].cache.reset(); 84 | subscriptions_[1].cache.reset(); 85 | } 86 | 87 | private: 88 | struct subscription_t 89 | { 90 | rclcpp::Subscription::SharedPtr subscription; 91 | uint32_t sequence_number = 0; 92 | message_t::SharedPtr cache; 93 | }; 94 | rclcpp::Publisher::SharedPtr publisher_; 95 | 96 | subscription_t subscriptions_[2]; 97 | 98 | uint64_t number_crunch_limit_; 99 | uint32_t sequence_number_ = 0; 100 | }; 101 | } // namespace rclcpp_system 102 | } // namespace nodes 103 | #endif // REFERENCE_SYSTEM__NODES__RCLCPP__FUSION_HPP_ 104 | -------------------------------------------------------------------------------- /reference_system/include/reference_system/nodes/rclcpp/intersection.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Apex.AI, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #ifndef REFERENCE_SYSTEM__NODES__RCLCPP__INTERSECTION_HPP_ 15 | #define REFERENCE_SYSTEM__NODES__RCLCPP__INTERSECTION_HPP_ 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | 22 | #include "rclcpp/rclcpp.hpp" 23 | #include "reference_system/msg_types.hpp" 24 | #include "reference_system/nodes/settings.hpp" 25 | #include "reference_system/number_cruncher.hpp" 26 | #include "reference_system/sample_management.hpp" 27 | 28 | namespace nodes 29 | { 30 | namespace rclcpp_system 31 | { 32 | 33 | class Intersection : public rclcpp::Node 34 | { 35 | public: 36 | explicit Intersection(const IntersectionSettings & settings) 37 | : Node(settings.node_name) 38 | { 39 | for (auto & connection : settings.connections) { 40 | rclcpp::SubscriptionOptionsWithAllocator> options; 41 | rclcpp::CallbackGroup::SharedPtr callback_group = this->create_callback_group( 42 | rclcpp::CallbackGroupType::MutuallyExclusive); 43 | options.callback_group = callback_group; 44 | connections_.emplace_back( 45 | Connection{ 46 | this->create_publisher(connection.output_topic, 1), 47 | this->create_subscription( 48 | connection.input_topic, 1, 49 | [this, id = connections_.size()](const message_t::SharedPtr msg) { 50 | input_callback(msg, id); 51 | }), 52 | callback_group, 53 | connection.number_crunch_limit}); 54 | } 55 | } 56 | rclcpp::CallbackGroup::SharedPtr get_callback_group_of_subscription( 57 | const std::string & input_topic) 58 | { 59 | for (auto & connection : connections_) { 60 | if (input_topic == connection.subscription->get_topic_name()) { 61 | return connection.callback_group; 62 | } 63 | } 64 | RCLCPP_FATAL(get_logger(), "Subscription for topic '%s' not found!", input_topic.c_str()); 65 | std::exit(1); 66 | } 67 | 68 | private: 69 | void input_callback( 70 | const message_t::SharedPtr input_message, 71 | const uint64_t id) 72 | { 73 | uint64_t timestamp = now_as_int(); 74 | auto number_cruncher_result = 75 | number_cruncher(connections_[id].number_crunch_limit); 76 | 77 | auto output_message = connections_[id].publisher->borrow_loaned_message(); 78 | output_message.get().size = 0; 79 | merge_history_into_sample(output_message.get(), input_message); 80 | 81 | uint32_t missed_samples = get_missed_samples_and_update_seq_nr( 82 | input_message, connections_[id].input_sequence_number); 83 | 84 | set_sample( 85 | this->get_name(), connections_[id].sequence_number++, 86 | missed_samples, timestamp, output_message.get()); 87 | 88 | // use result so that it is not optimizied away by some clever compiler 89 | output_message.get().data[0] = number_cruncher_result; 90 | connections_[id].publisher->publish(std::move(output_message)); 91 | } 92 | 93 | private: 94 | struct Connection 95 | { 96 | rclcpp::Publisher::SharedPtr publisher; 97 | rclcpp::Subscription::SharedPtr subscription; 98 | rclcpp::CallbackGroup::SharedPtr callback_group; 99 | uint64_t number_crunch_limit; 100 | uint32_t sequence_number = 0; 101 | uint32_t input_sequence_number = 0; 102 | }; 103 | std::vector connections_; 104 | }; 105 | } // namespace rclcpp_system 106 | } // namespace nodes 107 | #endif // REFERENCE_SYSTEM__NODES__RCLCPP__INTERSECTION_HPP_ 108 | -------------------------------------------------------------------------------- /reference_system/include/reference_system/nodes/rclcpp/sensor.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Apex.AI, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #ifndef REFERENCE_SYSTEM__NODES__RCLCPP__SENSOR_HPP_ 15 | #define REFERENCE_SYSTEM__NODES__RCLCPP__SENSOR_HPP_ 16 | #include 17 | #include 18 | #include 19 | 20 | #include "rclcpp/rclcpp.hpp" 21 | #include "reference_system/msg_types.hpp" 22 | #include "reference_system/nodes/settings.hpp" 23 | #include "reference_system/sample_management.hpp" 24 | 25 | namespace nodes 26 | { 27 | namespace rclcpp_system 28 | { 29 | 30 | class Sensor : public rclcpp::Node 31 | { 32 | public: 33 | explicit Sensor(const SensorSettings & settings) 34 | : Node(settings.node_name) 35 | { 36 | publisher_ = this->create_publisher(settings.topic_name, 1); 37 | timer_ = this->create_wall_timer( 38 | settings.cycle_time, 39 | [this] {timer_callback();}); 40 | } 41 | 42 | private: 43 | void timer_callback() 44 | { 45 | uint64_t timestamp = now_as_int(); 46 | auto message = publisher_->borrow_loaned_message(); 47 | message.get().size = 0; 48 | 49 | set_sample( 50 | this->get_name(), sequence_number_++, 0, timestamp, 51 | message.get()); 52 | 53 | publisher_->publish(std::move(message)); 54 | } 55 | 56 | private: 57 | rclcpp::Publisher::SharedPtr publisher_; 58 | rclcpp::TimerBase::SharedPtr timer_; 59 | uint32_t sequence_number_ = 0; 60 | }; 61 | } // namespace rclcpp_system 62 | } // namespace nodes 63 | #endif // REFERENCE_SYSTEM__NODES__RCLCPP__SENSOR_HPP_ 64 | -------------------------------------------------------------------------------- /reference_system/include/reference_system/nodes/rclcpp/transform.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Apex.AI, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #ifndef REFERENCE_SYSTEM__NODES__RCLCPP__TRANSFORM_HPP_ 15 | #define REFERENCE_SYSTEM__NODES__RCLCPP__TRANSFORM_HPP_ 16 | #include 17 | #include 18 | #include 19 | 20 | #include "rclcpp/rclcpp.hpp" 21 | #include "reference_system/msg_types.hpp" 22 | #include "reference_system/nodes/settings.hpp" 23 | #include "reference_system/number_cruncher.hpp" 24 | #include "reference_system/sample_management.hpp" 25 | 26 | namespace nodes 27 | { 28 | namespace rclcpp_system 29 | { 30 | 31 | class Transform : public rclcpp::Node 32 | { 33 | public: 34 | explicit Transform(const TransformSettings & settings) 35 | : Node(settings.node_name), 36 | number_crunch_limit_(settings.number_crunch_limit) 37 | { 38 | subscription_ = this->create_subscription( 39 | settings.input_topic, 1, 40 | [this](const message_t::SharedPtr msg) {input_callback(msg);}); 41 | publisher_ = this->create_publisher(settings.output_topic, 1); 42 | } 43 | 44 | private: 45 | void input_callback(const message_t::SharedPtr input_message) 46 | { 47 | uint64_t timestamp = now_as_int(); 48 | auto number_cruncher_result = number_cruncher(number_crunch_limit_); 49 | 50 | auto output_message = publisher_->borrow_loaned_message(); 51 | output_message.get().size = 0; 52 | merge_history_into_sample(output_message.get(), input_message); 53 | 54 | uint32_t missed_samples = get_missed_samples_and_update_seq_nr( 55 | input_message, input_sequence_number_); 56 | 57 | set_sample( 58 | this->get_name(), sequence_number_++, missed_samples, timestamp, 59 | output_message.get()); 60 | 61 | // use result so that it is not optimizied away by some clever compiler 62 | output_message.get().data[0] = number_cruncher_result; 63 | publisher_->publish(std::move(output_message)); 64 | } 65 | 66 | private: 67 | rclcpp::Publisher::SharedPtr publisher_; 68 | rclcpp::Subscription::SharedPtr subscription_; 69 | uint64_t number_crunch_limit_; 70 | uint32_t sequence_number_ = 0; 71 | uint32_t input_sequence_number_ = 0; 72 | }; 73 | } // namespace rclcpp_system 74 | } // namespace nodes 75 | #endif // REFERENCE_SYSTEM__NODES__RCLCPP__TRANSFORM_HPP_ 76 | -------------------------------------------------------------------------------- /reference_system/include/reference_system/nodes/settings.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Apex.AI, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #ifndef REFERENCE_SYSTEM__NODES__SETTINGS_HPP_ 15 | #define REFERENCE_SYSTEM__NODES__SETTINGS_HPP_ 16 | 17 | #include 18 | #include 19 | #include 20 | 21 | namespace nodes 22 | { 23 | 24 | struct CommandSettings 25 | { 26 | std::string node_name; 27 | std::string input_topic; 28 | }; 29 | 30 | struct FusionSettings 31 | { 32 | std::string node_name; 33 | std::string input_0; 34 | std::string input_1; 35 | std::string output_topic; 36 | uint64_t number_crunch_limit; 37 | }; 38 | 39 | struct TransformSettings 40 | { 41 | std::string node_name; 42 | std::string input_topic; 43 | std::string output_topic; 44 | uint64_t number_crunch_limit; 45 | }; 46 | 47 | struct IntersectionSettings 48 | { 49 | struct Connection 50 | { 51 | std::string input_topic; 52 | std::string output_topic; 53 | uint64_t number_crunch_limit; 54 | }; 55 | 56 | std::string node_name; 57 | std::vector connections; 58 | }; 59 | 60 | struct CyclicSettings 61 | { 62 | std::string node_name; 63 | std::vector inputs; 64 | std::string output_topic; 65 | uint64_t number_crunch_limit; 66 | std::chrono::nanoseconds cycle_time; 67 | }; 68 | 69 | struct SensorSettings 70 | { 71 | std::string node_name; 72 | std::string topic_name; 73 | std::chrono::nanoseconds cycle_time; 74 | }; 75 | } // namespace nodes 76 | #endif // REFERENCE_SYSTEM__NODES__SETTINGS_HPP_ 77 | -------------------------------------------------------------------------------- /reference_system/include/reference_system/number_cruncher.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Apex.AI, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #ifndef REFERENCE_SYSTEM__NUMBER_CRUNCHER_HPP_ 15 | #define REFERENCE_SYSTEM__NUMBER_CRUNCHER_HPP_ 16 | 17 | #include 18 | #include 19 | #include 20 | 21 | // prevents the compiler from optimizing access to value. 22 | // Taken from http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0412r0.html 23 | template 24 | inline void escape(Tp const & value) 25 | { 26 | asm volatile ("" : : "g" (value) : "memory"); 27 | } 28 | // Computes an expensive function (count number of primes below maximum_number) 29 | // This serves as a scalable dummy-workload for the various nodes. 30 | static inline int64_t number_cruncher(const uint64_t maximum_number) 31 | { 32 | int64_t number_of_primes = 0; 33 | uint64_t initial_value = 2; 34 | // edge case where max number is too low 35 | if (maximum_number <= initial_value) { 36 | return 2; 37 | } 38 | for (uint64_t i = initial_value; i <= maximum_number; ++i) { 39 | bool is_prime = true; 40 | for (uint64_t n = initial_value; n < i; ++n) { 41 | if (i % n == 0) { 42 | is_prime = false; 43 | break; 44 | } 45 | } 46 | escape(is_prime); 47 | if (is_prime) { 48 | // number_of_primes cannot overflow since there are fewer than 2**63 49 | // primes in [0, 2**64). 50 | ++number_of_primes; 51 | } 52 | } 53 | return number_of_primes; 54 | } 55 | 56 | // Returns the time (in ms of wall-clock time) needed to compute number_cruncher(maximum_number) 57 | static inline long double get_crunch_time_in_ms(const uint64_t maximum_number) 58 | { 59 | auto start = std::chrono::system_clock::now(); 60 | number_cruncher(maximum_number); 61 | auto stop = std::chrono::system_clock::now(); 62 | 63 | using milliseconds_ld = std::chrono::duration; 64 | return milliseconds_ld(stop - start).count(); 65 | } 66 | 67 | #endif // REFERENCE_SYSTEM__NUMBER_CRUNCHER_HPP_ 68 | -------------------------------------------------------------------------------- /reference_system/include/reference_system/system/type/rclcpp_system.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Apex.AI, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #ifndef REFERENCE_SYSTEM__SYSTEM__TYPE__RCLCPP_SYSTEM_HPP_ 15 | #define REFERENCE_SYSTEM__SYSTEM__TYPE__RCLCPP_SYSTEM_HPP_ 16 | #include "reference_system/nodes/rclcpp/command.hpp" 17 | #include "reference_system/nodes/rclcpp/fusion.hpp" 18 | #include "reference_system/nodes/rclcpp/transform.hpp" 19 | #include "reference_system/nodes/rclcpp/cyclic.hpp" 20 | #include "reference_system/nodes/rclcpp/sensor.hpp" 21 | #include "reference_system/nodes/rclcpp/intersection.hpp" 22 | 23 | struct RclcppSystem 24 | { 25 | using NodeBaseType = rclcpp::Node; 26 | 27 | using Command = nodes::rclcpp_system::Command; 28 | using Cyclic = nodes::rclcpp_system::Cyclic; 29 | using Fusion = nodes::rclcpp_system::Fusion; 30 | using Intersection = nodes::rclcpp_system::Intersection; 31 | using Sensor = nodes::rclcpp_system::Sensor; 32 | using Transform = nodes::rclcpp_system::Transform; 33 | }; 34 | 35 | #endif // REFERENCE_SYSTEM__SYSTEM__TYPE__RCLCPP_SYSTEM_HPP_ 36 | -------------------------------------------------------------------------------- /reference_system/package.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | reference_system 5 | 1.1.0 6 | Reference system for the ROScon workshop 7 | Evan Flynn 8 | Apache License 2.0 9 | 10 | Christian Eltzschig 11 | 12 | ament_cmake 13 | ament_cmake_auto 14 | 15 | rclcpp 16 | reference_interfaces 17 | rcl_interfaces 18 | 19 | ament_cmake_gtest 20 | ament_lint_auto 21 | ament_lint_common 22 | 23 | 24 | ament_cmake 25 | 26 | 27 | -------------------------------------------------------------------------------- /reference_system/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["flit_core >=3.2,<4"] 3 | build-backend = "flit_core.buildapi" 4 | 5 | [project] 6 | name = "evaluation_scripts" 7 | authors = [{name = "Tobias Stark", email = "tobias.stark@apex.ai"}] 8 | dynamic = ["version", "description"] 9 | -------------------------------------------------------------------------------- /reference_system/reference_system_py/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ros-realtime/reference-system/de8f3bc06f399ac99d6073460b50c5b59fc73a9c/reference_system/reference_system_py/__init__.py -------------------------------------------------------------------------------- /reference_system/reference_system_py/benchmark.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright 2021 Apex.AI, Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # Generates traces for specified executables and RMWs 17 | 18 | import contextlib 19 | import fcntl 20 | import os 21 | from pathlib import Path 22 | import subprocess 23 | import time 24 | 25 | from ament_index_python import get_package_prefix 26 | from ament_index_python.packages import get_package_share_directory 27 | 28 | import psutil 29 | 30 | try: 31 | from tracetools_trace.tools.names import DEFAULT_EVENTS_ROS 32 | from trace_utils import initDataModel # noqa: F401 33 | tracetools_available = True 34 | except ModuleNotFoundError: 35 | tracetools_available = False 36 | 37 | 38 | ROS_HOME = Path(os.environ.get('ROS_HOME', os.environ['HOME']+'/.ros')) 39 | REFERENCE_SYSTEM_SHARE_DIR = Path(get_package_share_directory('reference_system')) 40 | 41 | 42 | def available_executables(pkg, pattern='*'): 43 | prefix = get_package_prefix(pkg) 44 | return [path.stem for path in (Path(prefix)/'lib'/pkg).glob(pattern)] 45 | 46 | 47 | def get_benchmark_directory(base_directory, executable, runtime_sec, rmw, create=False): 48 | """ 49 | Return the directory to place measurements and reports for the given experiment. 50 | 51 | If `create` is True, the directory is created if it does not exist yet. 52 | """ 53 | # Note: memory_usage.py and std_latency.py make assumptions about the directory format. 54 | # Do not change this without also changing these other files. 55 | directory = Path(base_directory)/f'{runtime_sec}s/{rmw}/{executable}/' 56 | if create: 57 | directory.mkdir(parents=True, exist_ok=True) 58 | return directory 59 | 60 | 61 | def get_benchmark_directories_below(base_directory, runtime_sec=None): 62 | """Return all benchmark directories found below `base_directory`.""" 63 | runtime_re = '*[0-9]' if runtime_sec is None else str(runtime_sec) 64 | return [str(directory) for directory in Path(base_directory).glob(f'*{runtime_re}s/rmw_*/*')] 65 | 66 | 67 | @contextlib.contextmanager 68 | def terminatingRos2Run(pkg, executable, rmw, env=os.environ, args=[], **kwargs): 69 | """ 70 | Run the given executable (part of the given package) under the given rmw. 71 | 72 | The executable is automatically terminated upon exit from the context 73 | """ 74 | env = env.copy() 75 | env['RCUTILS_CONSOLE_OUTPUT_FORMAT'] = '[{severity}] [{name}]: {message}' 76 | env['RMW_IMPLEMENTATION'] = rmw 77 | env['RCL_ASSERT_RMW_ID_MATCHES'] = rmw 78 | 79 | assert 'timeout' not in kwargs, ('terminatingRos2Run does not support the timeout argument;' + 80 | 'use time.sleep in the with block instead') 81 | 82 | ros_executable = Path(get_package_prefix(pkg))/'lib'/pkg/executable 83 | cmdline = f'{ros_executable} {" ".join(args)}' 84 | process = subprocess.Popen(cmdline, 85 | shell=True, 86 | env=env, 87 | **kwargs) 88 | shellproc = psutil.Process(process.pid) 89 | try: 90 | yield process 91 | finally: 92 | if process.poll() not in (None, 0): 93 | # Process terminated with an error 94 | raise RuntimeError(f'Command "{cmdline}" terminated with error: {process.returncode}') 95 | 96 | # The process returned by subprocess.Popen is the shell process, not the 97 | # ROS process. Terminating the former will not necessarily terminate the latter. 98 | # Terminate all the *children* of the shell process instead. 99 | try: 100 | children = shellproc.children() 101 | except psutil.NoSuchProcess: 102 | children = [] 103 | 104 | assert len(children) <= 1 105 | if children: 106 | rosproc = children[0] 107 | rosproc.terminate() 108 | 109 | 110 | @contextlib.contextmanager 111 | def roudi_daemon(env=os.environ, roudi_config_path=None): 112 | """ 113 | Context manager that runs a RouDi instance for the duration of the context. 114 | 115 | The `env` parameter specifies environment variables for the RouDi process. 116 | The `roudi_config_path` parameter can be used to provide a RouDi toml configuration file. 117 | """ 118 | if 'ICEORYX_HOME' not in env: 119 | raise RuntimeError('Cannot find ICEORYX_HOME in environment. ' + 120 | 'Is the iceoryx environment set up?') 121 | try: 122 | with open('/tmp/roudi.lock') as roudi_lock: 123 | try: 124 | fcntl.flock(roudi_lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) 125 | except BlockingIOError as exception: 126 | raise RuntimeError('RouDi already running.') from exception 127 | finally: 128 | fcntl.flock(roudi_lock.fileno(), fcntl.LOCK_UN) 129 | except FileNotFoundError: 130 | pass # If the file does not exist, everything is fine; roudi is not running. 131 | 132 | roudi_shell = subprocess.Popen(('$ICEORYX_HOME/bin/iox-roudi ' + 133 | ('' if roudi_config_path is None 134 | else f"-c '{roudi_config_path}'")), 135 | shell=True, 136 | stdout=subprocess.PIPE, 137 | stderr=subprocess.PIPE, 138 | env=env) 139 | 140 | try: 141 | yield roudi_shell 142 | finally: 143 | if roudi_shell.poll() is not None: 144 | raise RuntimeError(f'Roudi terminated with return code {roudi_shell.returncode}:\n' + 145 | f'Output: {roudi_shell.stderr.read()}') 146 | 147 | roudi_process, = psutil.Process(roudi_shell.pid).children() 148 | roudi_process.terminate() 149 | roudi_process.wait() 150 | 151 | 152 | def generate_callback_trace(executable, pkg, directory, runtime_sec, rmw): 153 | """ 154 | Generate a tracefile for the given executable using the 'callback' method. 155 | 156 | The 'callback' method measures the executable using 'ros2 trace' 157 | """ 158 | raise NotImplementedError('lttng currently does not work within ADE') 159 | 160 | if not tracetools_available: 161 | raise RuntimeError('Unable to import tracetools_trace. Are the tracetools installed?') 162 | 163 | log_directory = get_benchmark_directory(directory, executable, runtime_sec, rmw, create=True) 164 | 165 | kernel_events = [] 166 | user_events = DEFAULT_EVENTS_ROS 167 | session = f'callback_trace_{pkg}_{executable}_{rmw}_{runtime_sec}s' 168 | tracer = subprocess.Popen(f'ros2 trace -s {session} ' + 169 | f'-p {log_directory/"callback_trace"} ' + 170 | f'-k {" ".join(kernel_events)} ' + 171 | f'-u {" ".join(user_events)} ', 172 | shell=True, 173 | text=True, 174 | stdin=subprocess.PIPE) 175 | try: 176 | with terminatingRos2Run(pkg, executable, rmw, 177 | stdout=subprocess.DEVNULL): 178 | # transmit the key press that ros2 trace requires to start 179 | tracer.stdin.write('\n') 180 | time.sleep(runtime_sec) 181 | finally: 182 | tracer.terminate() 183 | 184 | 185 | def generate_std_trace(executable, pkg, directory, runtime_sec, rmw): 186 | """ 187 | Generate a tracefile for the given executable using the 'std' method. 188 | 189 | The 'std' method logs stdout of the executable. 190 | """ 191 | log_directory = get_benchmark_directory(directory, executable, runtime_sec, rmw, create=True) 192 | 193 | logfile = log_directory/'std_output.log' 194 | with logfile.open('w', encoding='utf8') as logfd: 195 | with terminatingRos2Run(pkg, executable, rmw, 196 | stdout=logfd, 197 | text=True): 198 | time.sleep(runtime_sec) 199 | 200 | 201 | def generate_memory_trace(executable, pkg, directory, runtime_sec, rmw): 202 | """ 203 | Generate a tracefile for the given executable using the 'memory' method. 204 | 205 | The 'memory' method uses `psrecord` to profile memory and CPU usage. 206 | """ 207 | log_directory = get_benchmark_directory(directory, executable, runtime_sec, rmw, create=True) 208 | logfile = log_directory/f'memory_log.txt' 209 | plotfile = log_directory/f'memory_log.png' 210 | 211 | psrecord_cmd = subprocess.run('command -v psrecord', shell=True, stdout=subprocess.DEVNULL) 212 | if psrecord_cmd.returncode != 0: 213 | raise RuntimeError('psrecord is not installed; install it with pip install -U psrecord') 214 | 215 | with terminatingRos2Run(pkg, executable, rmw, 216 | stdout=subprocess.DEVNULL) as rosprocess: 217 | # Note: psrecord does provide a duration argument, but it just kills its subprocess 218 | # using SIGKILL, which does not reliably terminate ROS programs. 219 | # We therefore run psrecord inside the terminatingRos2Run instead. 220 | tracerprocess = subprocess.Popen(f'psrecord --include-children ' + 221 | f'--log {logfile} ' + 222 | f'--plot {plotfile} ' + 223 | f'{rosprocess.pid} ', 224 | shell=True) 225 | 226 | time.sleep(runtime_sec+0.5) 227 | tracerprocess.wait(20) 228 | 229 | 230 | def generate_trace(trace_type, *args, **kwargs): 231 | if trace_type == 'memory': 232 | return generate_memory_trace(*args, **kwargs) 233 | elif trace_type == 'callback': 234 | return generate_callback_trace(*args, **kwargs) 235 | elif trace_type == 'std': 236 | return generate_std_trace(*args, **kwargs) 237 | else: 238 | raise ValueError(f'Invalid trace_type: {trace_type}') 239 | 240 | 241 | def setup_benchmark_directory(pkg, create=False): 242 | base_dir = ROS_HOME/f'benchmark_{pkg}' 243 | if not create: 244 | latest = base_dir/'latest' 245 | if not latest.exists(): 246 | raise FileNotFoundError(f'Benchmark directory {base_dir} does not exist') 247 | return latest 248 | 249 | base_dir.mkdir(exist_ok=True, parents=True) 250 | # Create a subdirectory with a timestamp and link 'latest' to it. 251 | timestamp = time.strftime('%Y-%m-%d-%H-%M-%S') 252 | benchmark_dir = base_dir/timestamp 253 | latest_symlink = base_dir/'latest' 254 | latest_symlink.unlink(missing_ok=True) 255 | latest_symlink.symlink_to(benchmark_dir) 256 | return benchmark_dir 257 | -------------------------------------------------------------------------------- /reference_system/reference_system_py/callback_duration.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Apex.AI, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import random 16 | 17 | from bokeh.models import ColumnDataSource 18 | from bokeh.models import DatetimeTickFormatter 19 | from bokeh.models import Legend 20 | from bokeh.models.tools import HoverTool 21 | from bokeh.plotting import figure 22 | 23 | import numpy as np 24 | import pandas as pd 25 | 26 | 27 | def summary(data_model, size): 28 | callback_symbols = data_model.get_callback_symbols() 29 | colours = [] # Adds random colours for each callback 30 | colour_i = 0 31 | earliest_date = None 32 | fname = '' 33 | for obj, symbol in callback_symbols.items(): 34 | duration_df = data_model.get_callback_durations(obj) 35 | thedate = duration_df.loc[:, 'timestamp'].iloc[0] 36 | if earliest_date is None or thedate <= earliest_date: 37 | earliest_date = thedate 38 | 39 | starttime = earliest_date.strftime('%Y-%m-%d %H:%M') 40 | duration = figure( 41 | title='Callback Durations Summary', 42 | x_axis_label=f'start ({starttime})', 43 | y_axis_label='duration (ms)', 44 | plot_width=int(size * 2.0), 45 | plot_height=size, 46 | margin=(10, 10, 10, 10), 47 | sizing_mode='scale_width' 48 | ) 49 | 50 | legend_it = [] 51 | for obj, symbol in callback_symbols.items(): 52 | 53 | # Filter out internal subscriptions and get node information 54 | owner_info = data_model.get_callback_owner_info(obj) 55 | if not owner_info or '/parameter_events' in owner_info: 56 | continue 57 | 58 | if(len(colours) <= colour_i): 59 | colours.append('#%06X' % random.randint(0, 256**3-1)) 60 | 61 | duration_df = data_model.get_callback_durations(obj) 62 | source = ColumnDataSource(duration_df) 63 | duration.title.align = 'center' 64 | 65 | substr = 'node: ' 66 | index = str.find(owner_info, substr) 67 | if index >= 0: 68 | index += len(substr) 69 | fname = 'node_' + owner_info[index:(str.find(owner_info, ','))] 70 | substr = 'topic: /' 71 | index = str.find(owner_info, substr) 72 | if index >= 0: 73 | index += len(substr) 74 | fname += '_topic_' + owner_info[index:] 75 | 76 | c = duration.line( 77 | x='timestamp', 78 | y='duration', 79 | line_width=2, 80 | source=source, 81 | line_color=colours[colour_i], 82 | alpha=0.8, 83 | muted_color=colours[colour_i], 84 | muted_alpha=0.2, 85 | name=fname 86 | ) 87 | legend_it.append((fname, [c])) 88 | colour_i += 1 89 | # duration.legend.label_text_font_size = '11px' 90 | duration.xaxis[0].formatter = DatetimeTickFormatter(seconds=['%Ss']) 91 | 92 | legend = Legend( 93 | items=legend_it, 94 | label_text_font_size='8pt', 95 | label_standoff=1, 96 | padding=1, 97 | spacing=1 98 | ) 99 | legend.click_policy = 'hide' 100 | 101 | duration.add_layout(legend, 'right') 102 | 103 | # add hover tool 104 | hover = HoverTool() 105 | hover.tooltips = [ 106 | ('Callback', '$name'), 107 | ('Duration', '@duration{0.000}' + 's'), 108 | ('Timestamp', '@timestamp{%S.%3Ns}') 109 | ] 110 | hover.formatters = { 111 | '@timestamp': 'datetime' 112 | } 113 | duration.add_tools(hover) 114 | 115 | return duration 116 | # show(duration) 117 | # export_png(duration, filename=path + 'callback_duration_summary.png') 118 | 119 | 120 | def individual(data_model, size): 121 | # returns a list of individual plots for each callback symbol 122 | callback_symbols = data_model.get_callback_symbols() 123 | colours = [] # Adds random colours for each callback 124 | colour_i = 0 125 | fname = '' 126 | figs = [] 127 | for obj, symbol in callback_symbols.items(): 128 | owner_info = data_model.get_callback_owner_info(obj) 129 | if owner_info is None: 130 | owner_info = '[unknown]' 131 | 132 | # Filter out internal subscriptions 133 | if '/parameter_events' in owner_info: 134 | continue 135 | 136 | substr = 'node: ' 137 | index = str.find(owner_info, substr) 138 | if index >= 0: 139 | index += len(substr) 140 | fname = 'node_' + owner_info[index:(str.find(owner_info, ','))] 141 | substr = 'topic: /' 142 | index = str.find(owner_info, substr) 143 | if index >= 0: 144 | index += len(substr) 145 | fname += '_topic_' + owner_info[index:] 146 | 147 | # Duration 148 | duration_df = data_model.get_callback_durations(obj) 149 | starttime = duration_df.loc[:, 'timestamp'].iloc[0].strftime('%Y-%m-%d %H:%M') 150 | source = ColumnDataSource(duration_df) 151 | duration = figure( 152 | title='Callback Duration Over time', 153 | x_axis_label=f'start ({starttime})', 154 | y_axis_label='duration (ms)', 155 | plot_width=int(size * 1.2), 156 | plot_height=size, 157 | margin=(10, 10, 10, 175), # Top, R, Bottom, L 158 | sizing_mode='scale_width' 159 | ) 160 | duration.title.align = 'center' 161 | 162 | if(len(colours) <= colour_i): 163 | colours.append('#%06X' % random.randint(0, 256**3-1)) 164 | 165 | duration.line( 166 | x='timestamp', 167 | y='duration', 168 | legend_label=fname, 169 | line_width=2, 170 | source=source, 171 | line_color=colours[colour_i] 172 | ) 173 | # duration.legend_label.text_font_size = '11px' 174 | duration.xaxis[0].formatter = DatetimeTickFormatter(seconds=['%Ss']) 175 | 176 | # add hover tool 177 | hover = HoverTool() 178 | hover.tooltips = [ 179 | ('Duration', '@duration{0.000}' + 's'), 180 | ('Timestamp', '@timestamp{%S.%3Ns}') 181 | ] 182 | hover.formatters = { 183 | '@timestamp': 'datetime' 184 | } 185 | duration.add_tools(hover) 186 | 187 | # Histogram 188 | # (convert to milliseconds) 189 | dur_hist, edges = np.histogram(duration_df['duration'] * 1000 / np.timedelta64(1, 's')) 190 | duration_hist = pd.DataFrame({ 191 | 'duration': dur_hist, 192 | 'left': edges[:-1], 193 | 'right': edges[1:], 194 | }) 195 | hist = figure( 196 | title='Frequency of Callback Duration', 197 | x_axis_label='duration (ms)', 198 | y_axis_label='frequency', 199 | plot_width=int(size * 1.2), 200 | plot_height=size, 201 | margin=(10, 10, 10, 25), # Top, R, Bottom, L 202 | sizing_mode='scale_width' 203 | ) 204 | hist.title.align = 'center' 205 | hist.quad( 206 | bottom=0, 207 | top=duration_hist['duration'], 208 | left=duration_hist['left'], 209 | right=duration_hist['right'], 210 | fill_color=colours[colour_i], 211 | line_color=colours[colour_i], 212 | legend_label=fname 213 | ) 214 | 215 | colour_i += 1 216 | figs.append([duration, hist]) 217 | return figs 218 | -------------------------------------------------------------------------------- /reference_system/reference_system_py/constants.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Apex.AI, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | TRACE_CALLBACK = 'tracing' 16 | TRACE_MEMORY = 'memory' 17 | TRACE_STD = 'log' 18 | 19 | # TODO(flynneva): support path as just the `tracing` directory and loop over 20 | # all subdirectories that have tracing data in them 21 | TRACE_DIRECTORY = 'tracing' 22 | 23 | SIZE_SUMMARY = 800 24 | SIZE_SUBPLOT = 500 25 | 26 | SIZE_TABLE_ROW = 50 27 | SIZE_TABLE_WIDTH = 1250 28 | 29 | SIZE_TITLE = '22px' 30 | SIZE_AXIS_LABEL = '20px' 31 | SIZE_CATEGORY_LABEL = '15px' 32 | SIZE_MAJOR_LABEL = '14px' 33 | -------------------------------------------------------------------------------- /reference_system/reference_system_py/memory_usage.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Apex.AI, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import os 15 | import re 16 | 17 | from bokeh.models import ColumnDataSource 18 | from bokeh.models.axes import LinearAxis 19 | from bokeh.models.ranges import FactorRange, Range1d 20 | from bokeh.models.tools import HoverTool 21 | from bokeh.models.widgets.markups import Div 22 | from bokeh.models.widgets.tables import DataTable, TableColumn 23 | from bokeh.palettes import cividis 24 | from bokeh.plotting import figure 25 | from bokeh.transform import factor_cmap 26 | 27 | import pandas as pd 28 | 29 | from .constants import SIZE_AXIS_LABEL, SIZE_MAJOR_LABEL, SIZE_TITLE 30 | from .constants import SIZE_TABLE_ROW, SIZE_TABLE_WIDTH 31 | from .plot_utils import plot_barplot 32 | 33 | 34 | def summary_from_directories(dirs, duration, size): 35 | data = [] 36 | x = [] 37 | raw_df = [] 38 | summary_data = { 39 | 'exe': [], 40 | 'rmw': [], 41 | 'type': [], 42 | 'low': [], 43 | 'mean': [], 44 | 'high': [], 45 | 'std_dev': [], 46 | 'box_top': [], 47 | 'box_bottom': [] 48 | } 49 | dir_re = re.compile('([0-9]+)s/(rmw_.*)/([^/]+)$') 50 | for idx, directory in enumerate(dirs): 51 | match = dir_re.search(directory) 52 | if not match: 53 | raise ValueError(f'Given directory {directory} does not match naming requirements') 54 | extracted_duration, rmw, exe = match.groups() 55 | if int(extracted_duration) != duration: 56 | raise ValueError( 57 | f'Given directory {directory} does not have expected duration {duration}') 58 | fpath = directory + '/memory_log.txt' 59 | # open datafile and parse it into a usable structure 60 | data.append(open(fpath).read().splitlines()[1:]) 61 | data[idx] = [[float(element) for element in line.split()] for line in data[idx]] 62 | # add raw data to dataframe 63 | raw_df.append( 64 | pd.DataFrame( 65 | data=data[idx], 66 | columns=[ 67 | 'time', 68 | 'cpu', 69 | 'real', 70 | 'virtual'])) 71 | # calculate statics from raw data 72 | df_summary = raw_df[idx].describe().T.reset_index() 73 | 74 | # add data to dictionary 75 | for index, row in df_summary.iterrows(): 76 | summary_data['exe'].append(exe) 77 | summary_data['rmw'].append(rmw) 78 | summary_data['type'].append(row['index']) 79 | summary_data['low'].append(row['min']) 80 | summary_data['mean'].append(row['mean']) 81 | summary_data['high'].append(row['max']) 82 | summary_data['box_top'].append(row['mean'] + row['std']) 83 | summary_data['box_bottom'].append(row['mean'] - row['std']) 84 | summary_data['std_dev'].append(row['std']) 85 | 86 | df = pd.DataFrame.from_records( 87 | summary_data, columns=[ 88 | 'exe', 'rmw', 'type', 'low', 'mean', 'high', 'box_top', 'box_bottom', 'std_dev']) 89 | # sort by exe and rmw 90 | df = df.sort_values(['exe', 'rmw'], ascending=True) 91 | 92 | rmws = list(df.rmw.drop_duplicates()) 93 | x = [tuple(x) for x in df[['rmw', 'exe']].drop_duplicates().to_records(index=False)] 94 | 95 | cpu = df.type == 'cpu' 96 | real = df.type == 'real' 97 | virtual = df.type == 'virtual' 98 | 99 | cpu_source = ColumnDataSource(df[cpu]) 100 | real_source = ColumnDataSource(df[real]) 101 | virtual_source = ColumnDataSource(df[virtual]) 102 | # add exe and rmw list of tuples for x axis 103 | cpu_source.data['x'] = x 104 | real_source.data['x'] = x 105 | virtual_source.data['x'] = x 106 | fill_color = factor_cmap( 107 | 'x', palette=cividis(len(rmws)), factors=list(rmws), start=0, end=1) 108 | 109 | # initialize cpu figure 110 | cpu_fig = figure( 111 | title='CPU Usage Over Time ' + str(duration) + 's', 112 | x_axis_label=f'Executors (with RMW)', 113 | y_axis_label='CPU (%)', 114 | x_range=FactorRange(*x), 115 | plot_width=int(size * 2.0), 116 | plot_height=size, 117 | margin=(10, 10, 10, 10), 118 | sizing_mode='scale_width' 119 | ) 120 | plot_barplot(cpu_fig, cpu_source, fill_color=fill_color) 121 | 122 | # initialize real memory figure 123 | real_fig = figure( 124 | title='Real Memory Usage Over Time ' + str(duration) + 's', 125 | x_axis_label=f'Executors (with RMW)', 126 | y_axis_label='Real Memory Usage (MB)', 127 | x_range=FactorRange(*x), 128 | plot_width=int(size * 2.0), 129 | plot_height=size, 130 | margin=(10, 10, 10, 10), 131 | sizing_mode='scale_width' 132 | ) 133 | plot_barplot(real_fig, real_source, fill_color=fill_color) 134 | 135 | # initialize virtual memory figure 136 | virtual_fig = figure( 137 | title='Virtual Memory Usage Over Time ' + str(duration) + 's', 138 | x_axis_label=f'Executors (with RMW)', 139 | y_axis_label='Virtual Memory Usage (MB)', 140 | x_range=FactorRange(*x), 141 | plot_width=int(size * 2.0), 142 | plot_height=size, 143 | margin=(10, 10, 10, 10), 144 | sizing_mode='scale_width' 145 | ) 146 | plot_barplot(virtual_fig, virtual_source, fill_color=fill_color) 147 | 148 | # columns to use in all the tables below 149 | columns = [TableColumn(field=field, title=title) 150 | for field, title in [('exe', 'Benchmark'), 151 | ('rmw', 'RMW'), 152 | ('low', 'Min'), 153 | ('mean', 'Mean'), 154 | ('high', 'Max'), 155 | ('std_dev', 'Std. Dev.')]] 156 | 157 | # create CPU table 158 | cpu_table = DataTable( 159 | columns=columns, 160 | source=ColumnDataSource(df[cpu].round(decimals=2)), 161 | autosize_mode='fit_viewport', 162 | margin=(0, 10, 10, 10), 163 | height=(len(df[cpu].exe.values.tolist()) * SIZE_TABLE_ROW), 164 | width=SIZE_TABLE_WIDTH) 165 | 166 | # create real memory table 167 | real_table = DataTable( 168 | columns=columns, 169 | source=ColumnDataSource(df[real]), 170 | margin=(0, 10, 10, 10), 171 | height=(len(df[real].exe.values.tolist()) * SIZE_TABLE_ROW), 172 | width=SIZE_TABLE_WIDTH) 173 | 174 | # create virtual memory table 175 | virtual_table = DataTable( 176 | columns=columns, 177 | source=ColumnDataSource(df[virtual]), 178 | margin=(0, 10, 10, 10), 179 | height=(len(df[virtual].exe.values.tolist()) * SIZE_TABLE_ROW), 180 | width=SIZE_TABLE_WIDTH) 181 | # add figures and tables to output 182 | memory_figs = { 183 | 'cpu_table': cpu_table, 184 | 'cpu_fig': cpu_fig, 185 | 'real_mem_table': real_table, 186 | 'real_mem_fig': real_fig, 187 | 'virtual_mem_table': virtual_table, 188 | 'virtual_mem_fig': virtual_fig 189 | } 190 | return memory_figs 191 | 192 | 193 | def individual(path, size): 194 | basename = os.path.basename(path)[:-4] 195 | # open file 196 | data = open(path).read().splitlines()[1:] 197 | data = [[float(element) for element in line.split()] for line in data] 198 | df = pd.DataFrame(data=data, columns=['Elapsed Time', 'CPU (%)', 'Real (MB)', 'Virtual (MB)']) 199 | # add summary stats 200 | df_summary = df.describe().T.reset_index() 201 | # initialize list of figures 202 | memory = [] 203 | source = ColumnDataSource(df) 204 | # colors 205 | colors = [ 206 | '#158171', 207 | '#286f80', 208 | '#1bab78' 209 | ] 210 | # initialize raw data figure 211 | raw_data_fig = figure( 212 | title='Memory and CPU Usage Data [' + basename + ']', 213 | x_axis_label=f'Time (sec)', 214 | y_axis_label='CPU (%)', 215 | plot_width=int(size * 2.0), 216 | plot_height=size, 217 | margin=(10, 10, 10, 10), 218 | sizing_mode='scale_width' 219 | ) 220 | # add CPU usage to figure 221 | raw_data_fig.line( 222 | x='Elapsed Time', 223 | y='CPU (%)', 224 | line_width=1, 225 | source=source, 226 | line_color=colors[0], 227 | alpha=0.8, 228 | legend_label='CPU (%)', 229 | muted_color=colors[0], 230 | muted_alpha=0.2 231 | ) 232 | # legend attributes 233 | raw_data_fig.legend.location = 'top_right' 234 | raw_data_fig.legend.click_policy = 'hide' 235 | # add extra y ranges 236 | raw_data_fig.extra_y_ranges = { 237 | 'Real (MB)': Range1d( 238 | start=0, 239 | end=df_summary.loc[2, 'max'] + 25 240 | ) 241 | } 242 | raw_data_fig.add_layout( 243 | LinearAxis( 244 | y_range_name='Real (MB)', 245 | axis_label='Real (MB)'), 246 | 'right' 247 | ) 248 | # add Real memory usage to figure 249 | raw_data_fig.line( 250 | x='Elapsed Time', 251 | y='Real (MB)', 252 | y_range_name='Real (MB)', 253 | line_width=1, 254 | source=source, 255 | line_color=colors[1], 256 | alpha=0.8, 257 | legend_label='Real (MB)', 258 | muted_color=colors[1], 259 | muted_alpha=0.2 260 | ) 261 | raw_data_fig.title.text_font_size = SIZE_TITLE 262 | raw_data_fig.xaxis.axis_label_text_font_size = SIZE_AXIS_LABEL 263 | raw_data_fig.yaxis.axis_label_text_font_size = SIZE_AXIS_LABEL 264 | raw_data_fig.xaxis.major_label_text_font_size = SIZE_MAJOR_LABEL 265 | raw_data_fig.yaxis.major_label_text_font_size = SIZE_MAJOR_LABEL 266 | 267 | # add hover tool 268 | hover = HoverTool() 269 | hover.tooltips = [ 270 | ('CPU (%)', '@{CPU (%)}{0.00}'), 271 | ('Real (MB)', '@{Real (MB)}{0.00}'), 272 | ('Virtual (MB)', '@{Virtual (MB)}{0.00}') 273 | ] 274 | raw_data_fig.add_tools(hover) 275 | # create summary table 276 | columns = [TableColumn(field=col, title=col) for col in df_summary.columns] 277 | table_title = Div( 278 | text='Memory and CPU Usage Data: ' + basename + '', 279 | width=1000, 280 | height=20, 281 | style={ 282 | 'font-size': SIZE_MAJOR_LABEL 283 | } 284 | ) 285 | summary_fig = [ 286 | table_title, 287 | DataTable( 288 | columns=columns, 289 | source=ColumnDataSource(df_summary), 290 | margin=(10, 10, 10, 10), 291 | height=140)] 292 | # add figure and table to output 293 | memory = [[summary_fig], [raw_data_fig]] 294 | return memory 295 | -------------------------------------------------------------------------------- /reference_system/reference_system_py/plot_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Apex.AI, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import math 15 | 16 | from bokeh.models.tools import HoverTool 17 | 18 | from .constants import SIZE_AXIS_LABEL, SIZE_CATEGORY_LABEL, SIZE_MAJOR_LABEL, SIZE_TITLE 19 | 20 | 21 | def plot_barplot(fig, data_source, fill_color='gray'): 22 | """ 23 | Plot a barplot in `fig` using data from `data_source`. 24 | 25 | `data_source` is assumed to contain the following columns: 26 | - 'x' contains the x-coordinate 27 | - 'low' and 'high' contain the minimum/maximum 28 | - 'box_bottom' and 'box_top' contain the beginning/end of the box (mean +- stddev) 29 | - 'mean' contains the mean value 30 | """ 31 | fig.segment( 32 | 'x', 'box_bottom', 'x', 'low', color='black', line_width=2, 33 | source=data_source) 34 | fig.segment( 35 | 'x', 'box_top', 'x', 'high', color='black', line_width=2, 36 | source=data_source) 37 | fig.vbar( 38 | width=0.2, 39 | x='x', 40 | top='box_top', 41 | bottom='box_bottom', 42 | line_color='black', 43 | line_width=1, 44 | source=data_source, 45 | fill_color=fill_color 46 | ) 47 | fig.scatter( 48 | size=25, 49 | x='x', 50 | y='high', 51 | source=data_source, 52 | line_color='black', 53 | line_width=2, 54 | marker='dash', 55 | fill_color=fill_color 56 | ) 57 | fig.scatter( 58 | size=25, 59 | x='x', 60 | y='low', 61 | source=data_source, 62 | line_color='black', 63 | line_width=2, 64 | marker='dash', 65 | fill_color=fill_color 66 | ) 67 | fig.y_range.start = 0 68 | fig.x_range.range_padding = 0.1 69 | fig.title.text_font_size = SIZE_TITLE 70 | fig.xaxis.axis_label_text_font_size = SIZE_AXIS_LABEL 71 | fig.yaxis.axis_label_text_font_size = SIZE_AXIS_LABEL 72 | fig.yaxis.major_label_text_font_size = SIZE_MAJOR_LABEL 73 | fig.below[0].group_text_font_size = SIZE_CATEGORY_LABEL 74 | fig.below[0].major_label_text_font_size = SIZE_MAJOR_LABEL 75 | fig.xaxis.major_label_orientation = math.pi/16 76 | 77 | yaxis_label = fig.yaxis[0].axis_label 78 | # Add hover tool 79 | hover = HoverTool() 80 | hover.tooltips = [ 81 | ('Benchmark', '@{exe} [@{rmw}]'), 82 | ('Average '+yaxis_label, '@{mean}{0.00}'), 83 | ('Minimum '+yaxis_label, '@{low}{0.00}'), 84 | ('Maximum '+yaxis_label, '@{high}{0.00}'), 85 | ] 86 | fig.add_tools(hover) 87 | -------------------------------------------------------------------------------- /reference_system/reference_system_py/report.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Apex.AI, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from pathlib import Path 16 | 17 | from bokeh.embed import components 18 | from bokeh.io import output_file as bokeh_output_file 19 | from bokeh.layouts import layout as bokeh_layout 20 | from bokeh.plotting import save as bokeh_save 21 | import bokeh.util.version 22 | 23 | from jinja2 import Environment, FileSystemLoader, select_autoescape 24 | 25 | from . import callback_duration 26 | from . import dropped_messages 27 | from . import memory_usage 28 | from . import std_latency 29 | from .benchmark import get_benchmark_directories_below, get_benchmark_directory 30 | from .benchmark import REFERENCE_SYSTEM_SHARE_DIR 31 | from .constants import SIZE_SUBPLOT, SIZE_SUMMARY 32 | 33 | 34 | try: 35 | from tracetools_trace.tools.names import DEFAULT_EVENTS_ROS # noqa: F401 36 | from trace_utils import initDataModel 37 | tracetools_available = True 38 | except ModuleNotFoundError: 39 | tracetools_available = False 40 | 41 | 42 | def generate_callback_report(executable, pkg, directory, runtime_sec, rmw): 43 | """Generate a per-executable report from the 'callback' trace file.""" 44 | if not tracetools_available: 45 | raise RuntimeError('Unable to import tracetools_trace. Are the tracetools installed?') 46 | 47 | log_directory = get_benchmark_directory(directory, executable, runtime_sec, rmw) 48 | duration_output = log_directory/'callback_duration_report.html' 49 | dropped_msgs_output = log_directory/'tracing_latency_and_dropped_messages_report.html' 50 | data_model = initDataModel(log_directory/'callback_trace') 51 | 52 | bokeh_output_file(filename=duration_output, 53 | title='Callback Duration Report') 54 | print('Output report to', duration_output) 55 | duration_summary = callback_duration.summary( 56 | data_model=data_model, 57 | size=SIZE_SUMMARY) 58 | duration_individual = callback_duration.individual( 59 | data_model=data_model, 60 | size=SIZE_SUBPLOT) 61 | report = bokeh_layout([[duration_summary], *duration_individual]) 62 | bokeh_save(report) 63 | 64 | bokeh_output_file(filename=dropped_msgs_output, 65 | title='ROS 2 Tracing Latency and Dropped Messages Report') 66 | print('Output report to', dropped_msgs_output) 67 | dropped_msgs = dropped_messages.individual( 68 | data_model=data_model, 69 | size=SIZE_SUMMARY) 70 | report = bokeh_layout([[dropped_msgs]]) 71 | bokeh_save(report) 72 | 73 | 74 | def generate_memory_report(executable, pkg, directory, runtime_sec, rmw): 75 | """Generate a per-executable report from the 'memory' trace file.""" 76 | log_directory = get_benchmark_directory(directory, executable, runtime_sec, rmw) 77 | output = log_directory/'memory_and_cpu_usage_report.html' 78 | input_path = log_directory/'memory_log.txt' 79 | 80 | bokeh_output_file(filename=output, 81 | title='Memory and CPU Usage Report') 82 | print('Output report to', output) 83 | mem_individual = memory_usage.individual(input_path, 84 | size=SIZE_SUMMARY) 85 | report = bokeh_layout([*mem_individual]) 86 | bokeh_save(report) 87 | 88 | 89 | def generate_report(trace_type, *args, **kwargs): 90 | if trace_type == 'memory': 91 | return generate_memory_report(*args, **kwargs) 92 | elif trace_type == 'callback': 93 | return generate_callback_report(*args, **kwargs) 94 | elif trace_type == 'std': 95 | return None # No postprocessing needed for std trace 96 | else: 97 | raise ValueError(f'Invalid trace_type: {trace_type}') 98 | 99 | 100 | def fill_in_template(template_file: Path, report_title: str, figures: dict): 101 | """ 102 | Fill in the given tempalte file with the supplied figures. 103 | 104 | template_file: Full path to template file, including template file name 105 | report_title: Title of report 106 | figures: Figures to insert into report 107 | """ 108 | # get template directory 109 | template_directory = template_file.parents[0] 110 | env = Environment( 111 | loader=FileSystemLoader(str(template_directory)), 112 | autoescape=select_autoescape() 113 | ) 114 | template = env.get_template(str(template_file.name)) 115 | # get html and script from figures 116 | jinja_components = {} 117 | script, div = components(figures) 118 | jinja_components['bokeh_script'] = script 119 | jinja_components = {**jinja_components, **div} 120 | # add current bokeh CDN version to template variables 121 | jinja_components['bokeh_version'] = bokeh.util.version.base_version() 122 | 123 | return template.render(jinja_components, title=report_title) 124 | 125 | 126 | def generate_summary_report(trace_type, pkg, directory, runtime_sec, template_file): 127 | """Generate a summary report for the given `trace_type`, using all traces under `directory`.""" 128 | trace_dirs = get_benchmark_directories_below(directory, runtime_sec=runtime_sec) 129 | template_file_extension = template_file.split('.')[-1] 130 | 131 | if trace_type == 'memory': 132 | # use same filetype extension as input template file 133 | output_file = \ 134 | f'{directory}/memory_and_cpu_usage_summary_report_{runtime_sec}s.' + \ 135 | f'{template_file_extension}' 136 | report_title = 'Memory and CPU Usage Summary Report' 137 | mem_summary_figs = memory_usage.summary_from_directories(trace_dirs, 138 | duration=runtime_sec, 139 | size=SIZE_SUMMARY) 140 | 141 | # confirm template_file exists 142 | if not Path(template_file).exists(): 143 | print(f'Template file was not found: {template_file}') 144 | template_file = REFERENCE_SYSTEM_SHARE_DIR / 'cfg/memory_report_template.md' 145 | template_file_extension = '.md' 146 | print(f'Falling back to default: {template_file}') 147 | output_report = fill_in_template( 148 | template_file, report_title, mem_summary_figs) 149 | 150 | with open(output_file, 'w') as report: 151 | report.write(output_report) 152 | 153 | elif trace_type == 'std': 154 | output_file = \ 155 | f'{directory}/executor_kpi_summary_report_{runtime_sec}s.' + \ 156 | f'{template_file_extension}' 157 | report_title = 'Executor Key Performance Indicator (KPI) Report' 158 | 159 | std_summary_figs = std_latency.summary_from_directories(trace_dirs, 160 | duration=runtime_sec, 161 | size=SIZE_SUMMARY) 162 | # confirm template_file exists 163 | if not Path(template_file).exists(): 164 | print(f'Template file was not found: {template_file}') 165 | template_file = REFERENCE_SYSTEM_SHARE_DIR / 'cfg/std_report_template.md' 166 | template_file_extension = '.md' 167 | print(f'Falling back to default: {template_file}') 168 | output_report = fill_in_template( 169 | template_file, report_title, std_summary_figs) 170 | 171 | with open(output_file, 'w') as report: 172 | report.write(output_report) 173 | else: 174 | raise NotImplementedError(f'Unsupported trace type {trace_type}') 175 | 176 | print('Output report to', output_file) 177 | -------------------------------------------------------------------------------- /reference_system/reference_system_py/std_latency.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Apex.AI, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | from collections import defaultdict 15 | import re 16 | 17 | from bokeh.models import ColumnDataSource 18 | from bokeh.models.ranges import FactorRange 19 | from bokeh.models.widgets.tables import DataTable, TableColumn 20 | from bokeh.palettes import cividis 21 | from bokeh.plotting import figure 22 | from bokeh.transform import factor_cmap 23 | 24 | import pandas as pd 25 | 26 | from .constants import SIZE_TABLE_ROW, SIZE_TABLE_WIDTH 27 | from .plot_utils import plot_barplot 28 | 29 | 30 | def summary_from_directories(dirs, duration, size): 31 | data, hot_path_name = parseLogSummaryFromFiles( 32 | [directory+'/std_output.log' for directory in dirs], duration) 33 | 34 | df_dict = { 35 | 'exe': [], 36 | 'rmw': [], 37 | 'type': [], 38 | 'low': [], 39 | 'mean': [], 40 | 'high': [], 41 | 'std_dev': [], 42 | 'box_top': [], 43 | 'box_bottom': [] 44 | } 45 | 46 | def add_row(exe, rmw, data_type, stats): 47 | df_dict['exe'].append(exe) 48 | df_dict['rmw'].append(rmw) 49 | df_dict['type'].append(data_type) 50 | for stat, value in stats.items(): 51 | df_dict[stat].append(value) 52 | df_dict['box_top'].append(stats['mean'] + stats['std_dev']) 53 | df_dict['box_bottom'].append(stats['mean'] - stats['std_dev']) 54 | 55 | for (exe, rmw), results in data.items(): 56 | stats = results['hot_path']['latency'][-1] 57 | add_row(exe, rmw, 'latency', stats) 58 | stats = results['hot_path']['dropped'][-1] 59 | add_row(exe, rmw, 'dropped', stats) 60 | 61 | stats_list = results['behavior_planner']['period'] 62 | if stats_list: 63 | add_row(exe, rmw, 'period', stats_list[-1]) 64 | 65 | df = pd.DataFrame.from_dict(df_dict) 66 | # sort by exe and rmw 67 | df = df.sort_values(['exe', 'rmw'], ascending=True) 68 | rmws = list(df.rmw.drop_duplicates()) 69 | x = [tuple(x) for x in df[['rmw', 'exe']].drop_duplicates().to_records(index=False)] 70 | fill_color = factor_cmap( 71 | 'x', palette=cividis(len(rmws)), factors=list(rmws), start=0, end=1) 72 | 73 | latency = df[df.type == 'latency'] 74 | dropped = df[df.type == 'dropped'] 75 | period = df[df.type == 'period'] 76 | latency_source = ColumnDataSource(latency) 77 | dropped_source = ColumnDataSource(dropped) 78 | period_source = ColumnDataSource(period) 79 | # add exe and rmw list of tuples for x axis 80 | latency_source.data['x'] = x 81 | dropped_source.data['x'] = x 82 | period_source.data['x'] = x 83 | # initialize dict of figures 84 | std_figs = {} 85 | # initialize latency figure 86 | test_info = str(duration) + 's [' + hot_path_name + ']' 87 | columns = [TableColumn(field=field, title=title) 88 | for field, title in [('exe', 'Benchmark'), 89 | ('rmw', 'RMW'), 90 | ('low', 'Min'), 91 | ('mean', 'Mean'), 92 | ('high', 'Max'), 93 | ('std_dev', 'Std. Dev.')]] 94 | 95 | if not latency.empty: 96 | latency_fig = figure( 97 | title='Latency Summary ' + test_info, 98 | x_axis_label=f'Executors (with RMW)', 99 | y_axis_label='Latency (ms)', 100 | x_range=FactorRange(*x), 101 | plot_width=int(size * 2.0), 102 | plot_height=size, 103 | margin=(10, 10, 10, 10), 104 | sizing_mode='scale_width' 105 | ) 106 | plot_barplot(latency_fig, latency_source, fill_color=fill_color) 107 | 108 | latency_table = DataTable( 109 | columns=columns, 110 | source=ColumnDataSource(latency.round(decimals=3)), 111 | autosize_mode='fit_viewport', 112 | margin=(0, 10, 10, 10), 113 | height=(len(latency.exe.values.tolist()) * SIZE_TABLE_ROW), 114 | width=SIZE_TABLE_WIDTH) 115 | std_figs['latency_table'] = latency_table 116 | std_figs['latency_fig'] = latency_fig 117 | 118 | if not dropped.empty: 119 | dropped_fig = figure( 120 | title='Dropped Messages Summary ' + test_info, 121 | x_axis_label=f'Executors (with RMW)', 122 | y_axis_label='Dropped Messages', 123 | x_range=FactorRange(*x), 124 | plot_width=int(size * 2.0), 125 | plot_height=size, 126 | margin=(10, 10, 10, 10), 127 | sizing_mode='scale_width' 128 | ) 129 | plot_barplot(dropped_fig, dropped_source, fill_color=fill_color) 130 | 131 | dropped_table = DataTable( 132 | columns=columns, 133 | source=ColumnDataSource(dropped.round(decimals=1)), 134 | autosize_mode='fit_viewport', 135 | margin=(0, 10, 10, 10), 136 | height=(len(dropped.exe.values.tolist()) * SIZE_TABLE_ROW), 137 | width=SIZE_TABLE_WIDTH) 138 | std_figs['dropped_table'] = dropped_table 139 | std_figs['dropped_fig'] = dropped_fig 140 | 141 | if not period.empty: 142 | period_fig = figure( 143 | title='Behavior Planner Jitter Summary ' + str(duration) + 's', 144 | x_axis_label=f'Executors (with RMW)', 145 | y_axis_label='Period (ms)', 146 | x_range=FactorRange(*x), 147 | plot_width=int(size * 2.0), 148 | plot_height=size, 149 | margin=(10, 10, 10, 10), 150 | sizing_mode='scale_width' 151 | ) 152 | plot_barplot(period_fig, period_source, fill_color=fill_color) 153 | 154 | period_table = DataTable( 155 | columns=columns, 156 | source=ColumnDataSource(period.round(decimals=3)), 157 | autosize_mode='fit_viewport', 158 | margin=(0, 10, 10, 10), 159 | height=(len(period.exe.values.tolist()) * SIZE_TABLE_ROW), 160 | width=SIZE_TABLE_WIDTH) 161 | std_figs['period_table'] = period_table 162 | std_figs['period_fig'] = period_fig 163 | 164 | return std_figs 165 | 166 | 167 | def parse_stats_from_values(latency_, min_, max_, average_, deviation_): 168 | stats = { 169 | 'low': float(min_), 170 | 'high': float(max_), 171 | 'mean': float(average_), 172 | 'std_dev': float(deviation_)} 173 | 174 | return stats 175 | 176 | 177 | def parseLogSummaryFromFiles(files, duration): 178 | hot_path_name = None 179 | 180 | # result maps each pair (exe, rmw) to lists of results corresponding to the runs 181 | results = defaultdict(lambda: {'hot_path': {'latency': [], 182 | 'dropped': []}, 183 | 'behavior_planner': {'period': []}}) 184 | 185 | hot_path_name_regex = re.compile(r'^ *hot path: *(.*)$') 186 | hot_path_latency_regex = re.compile(r'^ *hot path latency: *(.+)ms \[min=(.+)ms, ' + 187 | r'max=(.+)ms, average=(.+)ms, deviation=(.+)ms\]$') 188 | hot_path_drops_regex = re.compile(r'^ *hot path drops: *(.+) \[min=(.+), max=(.+), ' + 189 | r'average=(.+), deviation=(.+)\]$') 190 | behavior_planner_period_regex = re.compile(r'^ *behavior planner period: *(.+)ms \[' + 191 | r'min=(.+)ms, max=(.+)ms, average=(.+)ms, ' + 192 | r'deviation=(.+)ms\]$') 193 | 194 | rmw_regex = re.compile(r'^RMW Implementation: (rmw_.*)') 195 | filename_regex = re.compile(r'.*/([0-9]+)s/(rmw_.*)/(.*)/std_output.log') 196 | for count, file in enumerate(files): 197 | match = filename_regex.match(file) 198 | if not match: 199 | raise ValueError(f'File {file} does not conform to the naming scheme') 200 | 201 | extracted_duration, rmw, exe = match.groups() 202 | if int(extracted_duration) != duration: 203 | raise ValueError(f'File {file} does not match expected duration {duration}') 204 | with open(file) as fp: 205 | rmw_line, *data = fp.read().splitlines() 206 | 207 | match = rmw_regex.match(rmw_line) 208 | if match and rmw != match.groups()[0]: 209 | raise ValueError((f'{file}: mismatch between filename-rmw ("{rmw}")' + 210 | f'and content-rmw("{match.groups()[0]}")')) 211 | 212 | if rmw not in file: 213 | raise ValueError(f'File {file} contains data from RMW {rmw}, contradicting its name') 214 | 215 | for line in data: 216 | match = hot_path_name_regex.match(line) 217 | if match: 218 | name, = match.groups() 219 | if hot_path_name is not None and hot_path_name != name: 220 | raise ValueError('Two different hotpaths in a single summary: ' + 221 | f'{name} {hot_path_name}') 222 | hot_path_name = name 223 | continue 224 | match = hot_path_latency_regex.match(line) 225 | if match: 226 | results[(exe, rmw)]['hot_path']['latency'].append( 227 | parse_stats_from_values(*match.groups())) 228 | continue 229 | match = hot_path_drops_regex.match(line) 230 | if match: 231 | results[(exe, rmw)]['hot_path']['dropped'].append( 232 | parse_stats_from_values(*match.groups())) 233 | continue 234 | match = behavior_planner_period_regex.match(line) 235 | if match: 236 | results[(exe, rmw)]['behavior_planner']['period'].append( 237 | parse_stats_from_values(*match.groups())) 238 | continue 239 | 240 | if hot_path_name is None: 241 | raise RuntimeError('No hot_path defined in experiment.') 242 | return results, hot_path_name 243 | -------------------------------------------------------------------------------- /reference_system/reference_system_py/trace_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Apex.AI, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | from tracetools_analysis.loading import load_file 15 | from tracetools_analysis.processor.ros2 import Ros2Handler 16 | from tracetools_analysis.utils.ros2 import Ros2DataModelUtil 17 | 18 | 19 | def initDataModel(path): 20 | events = load_file(path) 21 | handler = Ros2Handler.process(events) 22 | # handler.data.print_data() 23 | 24 | return Ros2DataModelUtil(handler.data) 25 | -------------------------------------------------------------------------------- /reference_system/resource/reference_system_py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ros-realtime/reference-system/de8f3bc06f399ac99d6073460b50c5b59fc73a9c/reference_system/resource/reference_system_py -------------------------------------------------------------------------------- /reference_system/setup.cfg: -------------------------------------------------------------------------------- 1 | [develop] 2 | script-dir=$base/lib/reference_system_py 3 | [install] 4 | install-scripts=$base/lib/reference_system_py -------------------------------------------------------------------------------- /reference_system/setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from setuptools import setup 4 | 5 | package_name = 'reference_system' 6 | 7 | setup( 8 | name=package_name, 9 | version='1.0.0', 10 | description='Scripts and utilities for the evaluation of executor reference systems', 11 | license='Apache 2.0 License', 12 | author='Tobias Stark', 13 | author_email='tobias.stark@apex.ai', 14 | data_files=[ 15 | f'share/{package_name}/package.xml', 16 | ('share/ament_index/resource_index/packages', 17 | [f'resource/{package_name}'])], 18 | packages=[package_name], 19 | tests_require=[], 20 | entry_points={}) 21 | -------------------------------------------------------------------------------- /reference_system/test/gtest_main.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Apex.AI, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #include "gtest/gtest.h" 15 | #include 16 | 17 | 18 | int main(int argc, char * argv[]) 19 | { 20 | ::testing::InitGoogleTest(&argc, argv); 21 | return RUN_ALL_TESTS(); 22 | } 23 | -------------------------------------------------------------------------------- /reference_system/test/test_fixtures.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Apex.AI, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #ifndef TEST_FIXTURES_HPP_ 15 | #define TEST_FIXTURES_HPP_ 16 | 17 | #include 18 | 19 | #include 20 | 21 | #include "rclcpp/rclcpp.hpp" 22 | 23 | 24 | class TestNodeGraph : public ::testing::Test 25 | { 26 | public: 27 | void SetUp() 28 | { 29 | rclcpp::init(0, nullptr); 30 | } 31 | 32 | void TearDown() 33 | { 34 | rclcpp::shutdown(); 35 | } 36 | }; 37 | 38 | using milliseconds = std::chrono::milliseconds; 39 | static constexpr uint64_t CRUNCH = 65536; 40 | 41 | template 42 | auto create_node(SettingsType settings) 43 | ->std::shared_ptr 44 | { 45 | auto node = 46 | std::make_shared(settings); 47 | return node; 48 | } 49 | 50 | #endif // TEST_FIXTURES_HPP_ 51 | -------------------------------------------------------------------------------- /reference_system/test/test_number_cruncher.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Apex.AI, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #include 15 | 16 | #include 17 | 18 | #include "reference_system/number_cruncher.hpp" 19 | 20 | 21 | TEST(test_number_cruncher, number_cruncher) { 22 | auto primes = number_cruncher(10); 23 | // 2, 3, 5, 7 24 | EXPECT_EQ(primes, 4); 25 | 26 | primes = number_cruncher(20); 27 | // 11, 13, 17, 19 28 | EXPECT_EQ(primes, 8); 29 | 30 | primes = number_cruncher(30); 31 | // 23, 29 32 | EXPECT_EQ(primes, 10); 33 | } 34 | 35 | TEST(test_number_cruncher, crunch_time) { 36 | auto expected_fast = get_crunch_time_in_ms(100); 37 | auto expected_slow = get_crunch_time_in_ms(65536); 38 | // lower maximum number should result in faster crunch times 39 | EXPECT_LT(expected_fast, expected_slow); 40 | } 41 | -------------------------------------------------------------------------------- /reference_system/test/test_reference_system_rclcpp.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Apex.AI, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #include 15 | 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | 22 | #include "test_fixtures.hpp" 23 | #include "rclcpp/node_interfaces/node_graph.hpp" 24 | #include "reference_system/system/type/rclcpp_system.hpp" 25 | 26 | 27 | // set the system to use 28 | using SystemType = RclcppSystem; 29 | 30 | void sleep_for_sec(uint32_t secs) 31 | { 32 | std::this_thread::sleep_for(std::chrono::seconds(secs)); 33 | } 34 | 35 | // remove /rosout and /parameter_events topics from map if they exist 36 | auto remove_default_topics(std::map> topic_map) 37 | ->std::map> 38 | { 39 | std::vector topics_to_remove{}; 40 | topics_to_remove.push_back("/rosout"); 41 | topics_to_remove.push_back("/parameter_events"); 42 | 43 | for (auto topic : topics_to_remove) { 44 | auto topic_iter = topic_map.find(topic); 45 | if (topic_iter != topic_map.end()) { 46 | std::cout << "Removing default topic: " << topic << std::endl; 47 | topic_map.erase(topic_iter); 48 | } 49 | } 50 | 51 | return topic_map; 52 | } 53 | 54 | TEST_F(TestNodeGraph, rclcpp_sensor_node) { 55 | ::testing::Test::RecordProperty("TEST_ID", "31c600f9-f6dc-407f-a487-b563178836ce"); 56 | auto settings = nodes::SensorSettings(); 57 | settings.node_name = "SensorNode"; 58 | settings.topic_name = settings.node_name; 59 | settings.cycle_time = milliseconds(100); 60 | // create node 61 | auto node = 62 | create_node(settings); 63 | sleep_for_sec(1); 64 | // confirm node was initialized with settings 65 | EXPECT_EQ(node->get_name(), settings.node_name); 66 | // get node graph of node 67 | auto * node_graph = node->get_node_graph_interface().get(); 68 | ASSERT_NE(nullptr, node_graph); 69 | auto topic_names_and_types = remove_default_topics(node_graph->get_topic_names_and_types(false)); 70 | // sensor nodes should publish one topic 71 | EXPECT_EQ(1, topic_names_and_types.size()); 72 | EXPECT_EQ(1, node_graph->count_publishers(settings.topic_name)); 73 | } 74 | 75 | TEST_F(TestNodeGraph, rclcpp_transform_node) { 76 | ::testing::Test::RecordProperty("TEST_ID", "76a88488-bf7b-414e-865b-92770f38cce2"); 77 | auto settings = nodes::TransformSettings(); 78 | settings.node_name = "TransformNode"; 79 | settings.input_topic = settings.node_name + "1"; 80 | settings.output_topic = settings.node_name; 81 | settings.number_crunch_limit = CRUNCH; 82 | // create node 83 | auto node = 84 | create_node(settings); 85 | sleep_for_sec(1); 86 | // confirm node was initialized with settings 87 | EXPECT_EQ(node->get_name(), settings.node_name); 88 | // get node graph of node 89 | auto * node_graph = node->get_node_graph_interface().get(); 90 | ASSERT_NE(nullptr, node_graph); 91 | auto topic_names_and_types = remove_default_topics(node_graph->get_topic_names_and_types(false)); 92 | // transform nodes should publish one topic and subscribe to one topic 93 | auto pubs = 1; 94 | auto subs = 1; 95 | auto total_pubs_and_subs = pubs + subs; 96 | EXPECT_EQ(total_pubs_and_subs, topic_names_and_types.size()); 97 | EXPECT_EQ(pubs, node_graph->count_publishers(settings.output_topic)); 98 | EXPECT_EQ(subs, node_graph->count_subscribers(settings.input_topic)); 99 | } 100 | 101 | TEST_F(TestNodeGraph, rclcpp_intersection_node) { 102 | ::testing::Test::RecordProperty("TEST_ID", "f2d0485c-c608-446f-8ee8-2dbf1b04399d"); 103 | auto settings = nodes::IntersectionSettings(); 104 | settings.node_name = "IntersectionNode"; 105 | std::string input_topic = settings.node_name + "_in_"; 106 | settings.connections.emplace_back(nodes::IntersectionSettings::Connection()); 107 | settings.connections.emplace_back(nodes::IntersectionSettings::Connection()); 108 | settings.connections[0].input_topic = input_topic + "1"; 109 | settings.connections[0].output_topic = settings.node_name + "1"; 110 | settings.connections[0].number_crunch_limit = CRUNCH; 111 | settings.connections[1].input_topic = input_topic + "2"; 112 | settings.connections[1].output_topic = settings.node_name + "2"; 113 | settings.connections[1].number_crunch_limit = CRUNCH; 114 | // create node 115 | auto node = 116 | create_node(settings); 117 | sleep_for_sec(1); 118 | // confirm node was initialized with settings 119 | EXPECT_EQ(node->get_name(), settings.node_name); 120 | // get node graph of node 121 | auto * node_graph = node->get_node_graph_interface().get(); 122 | ASSERT_NE(nullptr, node_graph); 123 | auto topic_names_and_types = remove_default_topics(node_graph->get_topic_names_and_types(false)); 124 | // intersection nodes should publish two topics and subscribe to two topics 125 | auto pubs = settings.connections.size(); 126 | auto subs = settings.connections.size(); 127 | auto total_pubs_and_subs = pubs + subs; 128 | EXPECT_EQ(total_pubs_and_subs, topic_names_and_types.size()); 129 | for (auto connection : settings.connections) { 130 | EXPECT_EQ(1, node_graph->count_publishers(connection.output_topic)); 131 | EXPECT_EQ(1, node_graph->count_subscribers(connection.input_topic)); 132 | } 133 | } 134 | 135 | TEST_F(TestNodeGraph, rclcpp_fusion_node) { 136 | ::testing::Test::RecordProperty("TEST_ID", "0a389fbd-d87b-42fb-8abb-189425958264"); 137 | auto settings = nodes::FusionSettings(); 138 | settings.node_name = "FusionNode"; 139 | settings.input_0 = settings.node_name + "1"; 140 | settings.input_1 = settings.node_name + "2"; 141 | settings.number_crunch_limit = CRUNCH; 142 | settings.output_topic = settings.node_name; 143 | // create node 144 | auto node = 145 | create_node(settings); 146 | sleep_for_sec(1); 147 | // confirm node was initialized with settings 148 | EXPECT_EQ(node->get_name(), settings.node_name); 149 | // get node graph of node 150 | auto * node_graph = node->get_node_graph_interface().get(); 151 | ASSERT_NE(nullptr, node_graph); 152 | auto topic_names_and_types = remove_default_topics(node_graph->get_topic_names_and_types(false)); 153 | // intersection nodes should publish two topics and subscribe to two topics 154 | auto pubs = 1; 155 | auto subs = 2; 156 | auto total_pubs_and_subs = pubs + subs; 157 | EXPECT_EQ(total_pubs_and_subs, topic_names_and_types.size()); 158 | EXPECT_EQ(1, node_graph->count_publishers(settings.node_name)); 159 | EXPECT_EQ(1, node_graph->count_subscribers(settings.input_0)); 160 | EXPECT_EQ(1, node_graph->count_subscribers(settings.input_1)); 161 | } 162 | 163 | TEST_F(TestNodeGraph, rclcpp_cyclic_node) { 164 | ::testing::Test::RecordProperty("TEST_ID", "46542da9-cc40-4f2f-93b1-a82577ee90ab"); 165 | auto settings = nodes::CyclicSettings(); 166 | settings.node_name = "CyclicNode"; 167 | settings.inputs.emplace_back(settings.node_name + "1"); 168 | settings.inputs.emplace_back(settings.node_name + "2"); 169 | settings.inputs.emplace_back(settings.node_name + "3"); 170 | settings.number_crunch_limit = CRUNCH; 171 | settings.output_topic = settings.node_name; 172 | // create node 173 | auto node = 174 | create_node(settings); 175 | sleep_for_sec(1); 176 | // confirm node was initialized with settings 177 | EXPECT_EQ(node->get_name(), settings.node_name); 178 | // get node graph of node 179 | auto * node_graph = node->get_node_graph_interface().get(); 180 | ASSERT_NE(nullptr, node_graph); 181 | auto topic_names_and_types = remove_default_topics(node_graph->get_topic_names_and_types(false)); 182 | // intersection nodes should publish two topics and subscribe to two topics 183 | auto pubs = 1; 184 | auto subs = 3; 185 | auto total_pubs_and_subs = pubs + subs; 186 | EXPECT_EQ(total_pubs_and_subs, topic_names_and_types.size()); 187 | EXPECT_EQ(1, node_graph->count_publishers(settings.node_name)); 188 | EXPECT_EQ(1, node_graph->count_subscribers(settings.inputs[0])); 189 | EXPECT_EQ(1, node_graph->count_subscribers(settings.inputs[1])); 190 | EXPECT_EQ(1, node_graph->count_subscribers(settings.inputs[2])); 191 | } 192 | 193 | TEST_F(TestNodeGraph, rclcpp_command_node) { 194 | ::testing::Test::RecordProperty("TEST_ID", "7507e0fb-0374-48f1-8d97-535af2e57bb2"); 195 | auto settings = nodes::CommandSettings(); 196 | settings.node_name = "CommandNode"; 197 | settings.input_topic = settings.node_name + "_in"; 198 | // create node 199 | auto node = 200 | create_node(settings); 201 | sleep_for_sec(1); 202 | // confirm node was initialized with settings 203 | EXPECT_EQ(node->get_name(), settings.node_name); 204 | // get node graph of node 205 | auto * node_graph = node->get_node_graph_interface().get(); 206 | ASSERT_NE(nullptr, node_graph); 207 | auto topic_names_and_types = remove_default_topics(node_graph->get_topic_names_and_types(false)); 208 | // intersection nodes should publish two topics and subscribe to two topics 209 | auto pubs = 0; 210 | auto subs = 1; 211 | auto total_pubs_and_subs = pubs + subs; 212 | EXPECT_EQ(total_pubs_and_subs, topic_names_and_types.size()); 213 | EXPECT_EQ(0, node_graph->count_publishers(settings.node_name)); 214 | EXPECT_EQ(1, node_graph->count_subscribers(settings.input_topic)); 215 | } 216 | -------------------------------------------------------------------------------- /reference_system/test/test_sample_management.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Apex.AI, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #include 15 | 16 | #include 17 | 18 | #include "reference_system/sample_management.hpp" 19 | #include "reference_system/msg_types.hpp" 20 | 21 | 22 | TEST(test_sample_management, set_benchmark_mode) { 23 | ::testing::Test::RecordProperty("TEST_ID", "41781862-6454-4d85-a3ee-bf82b1872763"); 24 | EXPECT_FALSE(is_in_benchmark_mode()); 25 | set_benchmark_mode(true); 26 | EXPECT_TRUE(is_in_benchmark_mode()); 27 | set_benchmark_mode(false); 28 | EXPECT_FALSE(is_in_benchmark_mode()); 29 | } 30 | 31 | TEST(test_sample_management, sample_helpers) { 32 | ::testing::Test::RecordProperty("TEST_ID", "6245be8e-3050-432f-9f61-a73dd33ff2c4"); 33 | message_t sample; 34 | std::string node_name = "test_node"; 35 | uint32_t sequence_number = 10; 36 | uint32_t dropped_samples = 5; 37 | uint64_t timestamp = 8675309; 38 | set_sample(node_name, sequence_number, dropped_samples, timestamp, sample); 39 | // EXPECT_EQ(sample.stats[0].node_name.data(), node_name.data()); 40 | // see reference_interfaces for more details 41 | // 4kb msg = 4032 = 63 * 64 bytes, 64 bytes = TransmissionStats length 42 | EXPECT_EQ(sample.stats.size(), 63u); 43 | EXPECT_EQ(sample.stats[0].sequence_number, sequence_number); 44 | EXPECT_EQ(sample.stats[0].dropped_samples, dropped_samples); 45 | EXPECT_EQ(sample.stats[0].timestamp, timestamp); 46 | 47 | auto retrieved_stamp = get_sample_timestamp(&sample); 48 | EXPECT_EQ(retrieved_stamp, timestamp); 49 | 50 | auto retrieved_sequence_number = get_sample_sequence_number(&sample); 51 | EXPECT_EQ(retrieved_sequence_number, sequence_number); 52 | 53 | auto retrieved_dropped_samples = 54 | get_missed_samples_and_update_seq_nr(&sample, sequence_number); 55 | // should be zero based on sequence number 56 | EXPECT_EQ(retrieved_dropped_samples, 0u); 57 | } 58 | 59 | TEST(test_sample_management, statistic_value_struct) { 60 | auto stats = statistic_value_t(); 61 | 62 | stats.suffix = "the_suffix"; 63 | // simulate multiple messages coming in 64 | stats.set(1); 65 | stats.set(4); 66 | stats.set(5); 67 | stats.set(7); 68 | 69 | EXPECT_EQ(stats.average, 4.25); 70 | EXPECT_EQ(stats.deviation, 2.5); 71 | EXPECT_EQ(stats.min, 1u); 72 | EXPECT_EQ(stats.max, 7u); 73 | EXPECT_EQ(stats.current, 7u); 74 | EXPECT_EQ(stats.total_number, 4u); 75 | EXPECT_EQ(stats.suffix, "the_suffix"); 76 | EXPECT_EQ(stats.adjustment, 0.0); 77 | } 78 | 79 | TEST(test_sample_management, sample_statistic_struct) { 80 | auto stats = sample_statistic_t(); 81 | stats.timepoint_of_first_received_sample = 1; 82 | stats.previous_behavior_planner_sequence = 2; 83 | stats.previous_behavior_planner_time_stamp = 3; 84 | 85 | EXPECT_EQ(stats.timepoint_of_first_received_sample, uint64_t(1)); 86 | EXPECT_EQ(stats.previous_behavior_planner_sequence, uint32_t(2)); 87 | EXPECT_EQ(stats.previous_behavior_planner_time_stamp, uint64_t(3)); 88 | } 89 | 90 | TEST(test_sample_management, print_sample_path) { 91 | message_t sample; 92 | 93 | std::string node_name = "test_node"; 94 | 95 | uint32_t sample_size = 105; 96 | // TODO(evan.flynn): add test for operator<< print function used within print_sample_path 97 | set_benchmark_mode(false); 98 | uint64_t timestamp = 1; 99 | for (uint32_t i = 0; i < sample_size; i++) { 100 | set_sample(node_name, i, 0, timestamp, sample); 101 | timestamp += 1; 102 | } 103 | // message sample size will always be 63 if message type is set to 4kb 104 | // see reference_interfaces package for more details 105 | EXPECT_EQ(sample.size, 63u); 106 | print_sample_path("test_node", uint32_t(1), &sample); 107 | } 108 | 109 | TEST(test_sample_management, set_sample_terminates_node_name) { 110 | message_t sample; 111 | const std::string short_name("name_with_few_characters"); 112 | const std::string long_name(sample.stats[0].node_name.size(), 'A'); 113 | 114 | // Fill the stats buffer with '%'-characters. These characters must no 115 | // longer be visible in the resulting buffer 116 | sample.size = 0; 117 | sample.stats[0].node_name.fill('%'); 118 | set_sample(short_name, 1, 0, 0, sample); 119 | EXPECT_STREQ( 120 | short_name.c_str(), 121 | reinterpret_cast(sample.stats[0].node_name.data())); 122 | 123 | sample.size = 0; 124 | sample.stats[0].node_name.fill('%'); 125 | set_sample(long_name, 1, 0, 0, sample); 126 | ASSERT_EQ(sample.stats[0].node_name.back(), '\0'); 127 | auto long_name_without_last_char = long_name.substr(0, long_name.size() - 1); 128 | EXPECT_STREQ( 129 | long_name_without_last_char.c_str(), 130 | reinterpret_cast(sample.stats[0].node_name.data())); 131 | } 132 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | bokeh==2.4.1 2 | psrecord==1.2 3 | --------------------------------------------------------------------------------