├── .github └── workflows │ └── ci.yaml ├── .gitignore ├── CMakeLists.txt ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── buildfarm_perf_tests ├── __init__.py ├── generate_config_yaml.py ├── launch.py └── test_results.py ├── img ├── agregate_latency.png ├── cpu.png ├── latency.png ├── lost_packets.png ├── received_packets.png ├── sent_packets.png └── size.png ├── include ├── linux_cpu_process_measurement.hpp ├── linux_cpu_system_measurement.hpp ├── linux_memory_process_measurement.hpp ├── linux_memory_system_measurement.hpp └── utilities │ └── utilities.hpp ├── package.xml ├── scripts └── generate_config_yaml.py ├── src ├── linux_cpu_process_measurement.cpp ├── linux_cpu_system_measurement.cpp ├── linux_memory_process_measurement.cpp ├── linux_memory_system_measurement.cpp ├── main_measurements.cpp ├── node_spinning.cpp └── utilities │ └── utilities.cpp ├── templates ├── node_spinning.txt ├── overhead_cpu_usage.txt ├── overhead_lost_messages.txt ├── overhead_physical_memory.txt ├── overhead_received_messages.txt ├── overhead_resident_anonymous_memory.txt ├── overhead_round_trip.txt ├── overhead_sent_messages.txt ├── overhead_virtual_memory.txt ├── performance_test_1p_1k.txt ├── performance_test_1p_multi.txt ├── performance_test_2p_1k.txt └── performance_test_2p_multi.txt ├── test ├── CMakeLists.txt ├── DEFAULT_FASTRTPS_PROFILES.xml ├── DEFAULT_FASTRTPS_PROFILES_TWO_PROCESSES.xml ├── add_performance_tests.cmake ├── test_performance.py.in ├── test_pub_sub.py.in └── test_spinning.py.in └── tools └── ros2_dependencies.repos /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | name: Continuous Integration 2 | on: 3 | pull_request: 4 | push: 5 | branches: 6 | - master 7 | 8 | jobs: 9 | build_and_test: 10 | name: Build and Test 11 | runs-on: ubuntu-latest 12 | container: ${{matrix.container_image}} 13 | strategy: 14 | matrix: 15 | container_image: ['osrf/ros2:nightly-rmw-nonfree'] 16 | 17 | steps: 18 | - name: Clone buildfarm_perf_tests 19 | uses: actions/checkout@v2 20 | with: 21 | path: src/buildfarm_perf_tests 22 | - name: Update, build, and test 23 | run: | 24 | echo ::group::Update packages 25 | apt-get update 26 | apt-get upgrade -y 27 | apt-get install -y python3-pip 28 | rosdep update 29 | echo ::endgroup:: 30 | . /opt/ros/*/setup.sh 31 | echo ::group::Install dependencies 32 | rosdep install -y --from-path src --ignore-src --skip-keys 'performance_test' 33 | pip3 install flake8_builtins flake8_quotes 34 | echo ::endgroup:: 35 | echo ::group::Build package 36 | colcon build --event-handler console_direct+ --cmake-args -DCMAKE_CXX_FLAGS=-Werror -Werror=dev -Werror=deprecated 37 | echo ::endgroup:: 38 | echo ::group::Test package 39 | colcon test --event-handler console_direct+ --ctest-args --label-exclude performance 40 | echo ::endgroup:: 41 | colcon test-result 42 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.5) 2 | project(buildfarm_perf_tests) 3 | 4 | # Default to C++14 5 | if(NOT CMAKE_CXX_STANDARD) 6 | set(CMAKE_CXX_STANDARD 14) 7 | endif() 8 | 9 | if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") 10 | add_compile_options(-Wall -Wextra -Wpedantic) 11 | endif() 12 | 13 | # find dependencies 14 | find_package(ament_cmake REQUIRED) 15 | find_package(ament_cmake_python REQUIRED) 16 | find_package(rclcpp REQUIRED) 17 | 18 | add_executable(node_spinning src/node_spinning.cpp) 19 | ament_target_dependencies( 20 | node_spinning 21 | "rclcpp" 22 | ) 23 | 24 | install(TARGETS node_spinning 25 | EXPORT export_${PROJECT_NAME} 26 | DESTINATION lib/${PROJECT_NAME}) 27 | 28 | option(ENABLE_SYSTEM_METRIC_COLLECTOR 29 | "Enables the system metric collector and tests which use it" ${UNIX}) 30 | if(ENABLE_SYSTEM_METRIC_COLLECTOR) 31 | find_package(Threads REQUIRED) 32 | 33 | add_executable(system_metric_collector 34 | src/linux_cpu_process_measurement.cpp 35 | src/linux_cpu_system_measurement.cpp 36 | src/linux_memory_process_measurement.cpp 37 | src/linux_memory_system_measurement.cpp 38 | src/main_measurements.cpp 39 | src/utilities/utilities.cpp 40 | ) 41 | target_include_directories(system_metric_collector PUBLIC 42 | $ 43 | $) 44 | target_link_libraries(system_metric_collector 45 | "stdc++fs" 46 | ${CMAKE_THREAD_LIBS_INIT} 47 | ) 48 | 49 | install(TARGETS system_metric_collector 50 | EXPORT export_${PROJECT_NAME} 51 | DESTINATION lib/${PROJECT_NAME}) 52 | endif() 53 | 54 | ament_python_install_package(${PROJECT_NAME}) 55 | 56 | install(DIRECTORY templates 57 | DESTINATION share/${PROJECT_NAME}) 58 | 59 | install(PROGRAMS scripts/generate_config_yaml.py 60 | DESTINATION lib/${PROJECT_NAME}) 61 | 62 | if(BUILD_TESTING) 63 | find_package(ament_lint_auto REQUIRED) 64 | ament_lint_auto_find_test_dependencies() 65 | 66 | add_subdirectory(test) 67 | endif() 68 | 69 | ament_package() 70 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Any contribution that you make to this repository will 2 | be under the Apache 2 License, as dictated by that 3 | [license](http://www.apache.org/licenses/LICENSE-2.0.html): 4 | 5 | ~~~ 6 | 5. Submission of Contributions. Unless You explicitly state otherwise, 7 | any Contribution intentionally submitted for inclusion in the Work 8 | by You to the Licensor shall be under the terms and conditions of 9 | this License, without any additional terms or conditions. 10 | Notwithstanding the above, nothing herein shall supersede or modify 11 | the terms of any separate license agreement you may have executed 12 | with Licensor regarding such Contributions. 13 | ~~~ 14 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # buildfarm perf tests 2 | 3 | [![Continuous Integration](https://github.com/ros2/buildfarm_perf_tests/workflows/Continuous%20Integration/badge.svg?branch=master&event=push)](https://github.com/ros2/buildfarm_perf_tests/actions?query=branch%3Amaster+event%3Apush) 4 | 5 | ## Purpose 6 | 7 | This package defines some tests. 8 | On one hand it invokes `perf_test` from Apex.AI's [performance_test](https://gitlab.com/ApexAI/performance_test) package. 9 | This allows you to test performance and latency of several ROS 2 RMW implementations. 10 | On the other hand we are evaluating the additional overhead caused by a single pub/sub topic or one process spinning and detect potential leaks related to theses activities. 11 | 12 | * There is a test for each RMW: 13 | 14 | - CycloneDDS (standalone test) 15 | - FastRTPS (standalone test) 16 | - rmw_connext_cpp 17 | - rmw_cyclonedds_cpp 18 | - rmw_fastrtps_cpp 19 | - rmw_fastrtps_dynamic_cpp 20 | - rmw_opensplice_cpp 21 | 22 | ### Test 1 - Performance Test (Apex.AI) 23 | 24 | In this test we are running the Performance Test provided by Apex.AI. 25 | Right now we have [our own fork](https://github.com/ros2/performance_test) because there are some pending pull requests in the official gitlab repository. 26 | 27 | In this test we are measurement: 28 | - Average latency 29 | - CPU usage (provided by Apex.AI tool) 30 | - Sent/Received messages per second 31 | - Total lost messages 32 | - Max resident set size 33 | 34 | We are generating two plots per measurement 35 | - [One per-build](http://build.ros2.org/view/Rci/job/Rci__nightly-performance_ubuntu_focal_amd64/lastBuild/) 36 | - [Other over time](http://build.ros2.org/view/Rci/job/Rci__nightly-performance_ubuntu_focal_amd64/plot/) 37 | 38 | **The test only measures the latency between the same RMW implementation** 39 | 40 | | Publisher/Subscriber | rmw_fastrtps_cpp | rmw_opensplice_cpp | rmw_cyclonedds_cpp | rmw_fastrtps_dynamic_cpp | rmw_connext_cpp | 41 | |--------------------------|--------------------------|--------------------------|--------------------------|--------------------------|--------------------------| 42 | | rmw_fastrtps_cpp | :heavy_check_mark: | :heavy_multiplication_x: | :heavy_multiplication_x: | :heavy_multiplication_x: | :heavy_multiplication_x: | 43 | | rmw_opensplice_cpp | :heavy_multiplication_x: | :heavy_check_mark: | :heavy_multiplication_x: | :heavy_multiplication_x: | :heavy_multiplication_x: | 44 | | rmw_cyclonedds_cpp | :heavy_multiplication_x: | :heavy_multiplication_x: | :heavy_check_mark: | :heavy_multiplication_x: | :heavy_multiplication_x: | 45 | | rmw_fastrtps_dynamic_cpp | :heavy_multiplication_x: | :heavy_multiplication_x: | :heavy_multiplication_x: | :heavy_check_mark: | :heavy_multiplication_x: | 46 | | rmw_connext_cpp | :heavy_multiplication_x: | :heavy_multiplication_x: | :heavy_multiplication_x: | :heavy_multiplication_x: | :heavy_check_mark: | 47 | 48 | ## Test 2 - Simple pub/sub 49 | 50 | In this case we are testing one publisher and one subscriber **in different processes** sending a 1kArray at 5Hz. 51 | This will allow us to evaluate additional overhead caused by a single pub/sub topic and detect leaks related to this activity. 52 | 53 | We measure for both publisher and subscriber: 54 | 55 | - Average round trip 56 | - CPU usage ( readed from the filesystem ) 57 | - Total lost messages 58 | - Received/Sent messages per second 59 | - Physical memory 60 | - Resident anonymous memory 61 | - Virtual memory 62 | 63 | Again we plot measurement: 64 | - [One per-build](http://build.ros2.org/view/Rci/job/Rci__nightly-performance_ubuntu_focal_amd64/lastBuild/) 65 | - [Other over time](http://build.ros2.org/view/Rci/job/Rci__nightly-performance_ubuntu_focal_amd64/plot/) 66 | 67 | | Publisher/Subscriber | rmw_fastrtps_cpp | rmw_opensplice_cpp | rmw_cyclonedds_cpp | rmw_fastrtps_dynamic_cpp | rmw_connext_cpp | 68 | |--------------------------|--------------------|--------------------|--------------------|--------------------------|--------------------| 69 | | rmw_fastrtps_cpp | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | 70 | | rmw_opensplice_cpp | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | 71 | | rmw_cyclonedds_cpp | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | 72 | | rmw_fastrtps_dynamic_cpp | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | 73 | | rmw_connext_cpp | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | 74 | 75 | ## Test 3 - Node spinning 76 | 77 | This test creates one process and spins for 1 minute to evaluate ROS 2 overhead and detect obvious leaks. 78 | 79 | We measure: 80 | 81 | - CPU usage ( readed from file system ) 82 | - Physical memory 83 | - Resident anonymous memory 84 | - Virtual memory 85 | 86 | Again we plot measurement: 87 | - [One per-build](http://build.ros2.org/view/Rci/job/Rci__nightly-performance_ubuntu_focal_amd64/lastBuild/) 88 | - [Other over time](http://build.ros2.org/view/Rci/job/Rci__nightly-performance_ubuntu_focal_amd64/plot/Node%20Spinning%20Results/) 89 | 90 | | DDS | Process 1 | 91 | |--------------------------|-----------| 92 | | rmw_fastrtps_cpp | :heavy_check_mark: | 93 | | rmw_opensplice_cpp | :heavy_check_mark: | 94 | | rmw_cyclonedds_cpp | :heavy_check_mark: | 95 | | rmw_fastrtps_dynamic_cpp | :heavy_check_mark: | 96 | | rmw_connext_cpp | :heavy_check_mark: | 97 | 98 | ## Build 99 | 100 | 1. Install ROS 2 (https://index.ros.org/doc/ros2/Installation/). 101 | 1. Source the ROS 2 installation (either `/opt/ros//setup.bash` if installing from binaries, or `~/ros2_ws/install/setup.bash` if building from source): 102 | 1. `source /opt/ros//setup.bash` or `source ~/ros2_ws/install/setup.bash` 103 | 1. Make a new workspace and clone this repository into it: 104 | 1. `mkdir -p ~/performance_ws/src` 105 | 1. `cd ~/performance_ws` 106 | 1. `wget https://github.com/ros2/buildfarm_perf_tests/raw/master/tools/ros2_dependencies.repos` 107 | 1. `vcs import src < ros2_dependencies.repos` 108 | 1. `rosdep install --from-path src --ignore-src` 109 | 1. Build the local workspace: 110 | 1. `colcon build` 111 | 1. Source the local workspace: 112 | 1. `source install/local_setup.bash` 113 | 114 | ### Run 115 | 116 | ```bash 117 | colcon test --packages-select buildfarm_perf_tests --event-handlers console_direct+ 118 | ``` 119 | 120 | ## Details 121 | 122 | ***Note: the graphs presented here are for demonstration purposes only. The data in the graphs are not meant to be accurate or current.*** 123 | 124 | * Each test runs for **30** seconds with a **1k payload**, but this can be changed using CMake variables. 125 | - `PERF_TEST_RUNTIME`: Maximum number of seconds to run before exiting. Zero runs forever. 126 | - `PERF_TEST_TOPICS`: Topics to use. These are all available topics: `Array1k`, `Array4k`, `Array16k`, `Array32k`, `Array60k`, `Array1m`, `Array2m`, `Struct16`, `Struct256`, `Struct4k`, `Struct32k`, `PointCloud512k`, `PointCloud1m`, `PointCloud2m`, `PointCloud4m`, `Range`, `NavSatFix`, `RadarDetection` and `RadarTrack`. 127 | 128 | For example, If we want to run the test during `30` seconds using the topic `Array1k`: 129 | 130 | ```bash 131 | colcon build --packages-select buildfarm_perf_tests --cmake-args -DPERF_TEST_RUNTIME="30" -DPERF_TEST_TOPICS="Array1k;Array4k" --no-warn-unused-cli 132 | ``` 133 | 134 | * Each test produces a PNG plot of [various measures](http://build.ros2.org/view/Eci/job/Eci__nightly-performance_ubuntu_bionic_amd64/) across time, displayed in Jenkins using the image gallery plugin. 135 | - These plots are displayed on the build's summary page, and are part of the output artifacts for the build. 136 | ![](img/latency.png) 137 | 138 | * Each test also produces a couple of [aggregated measures](http://build.ros2.org/view/Eci/job/Eci__nightly-performance_ubuntu_bionic_amd64/plot/) in a small csv file, used to plot build-over-build using the Jenkins plot plugin. 139 | 140 | - To see these plots, click the "Plots" link on the left side of the JOB summary (not a build summary) 141 | - You should be able to click one of those points to jump to the aforementioned PNG plot that produced that aggregated point. 142 | ![](img/agregate_latency.png) 143 | ![](img/size.png) 144 | ![](img/cpu.png) 145 | ![](img/lost_packets.png) 146 | ![](img/received_packets.png) 147 | ![](img/sent_packets.png) 148 | 149 | ## System metrics collector tool 150 | 151 | This tool allows to create statistics based on the name of the process and arguments. This tool allows to collect the following statistics: 152 | 153 | - CPU usage (%): This information is fetched from `/proc/stat`. 154 | - CPU memory virtual, ResidentAnonymousMemory and physical: This information is fetched from `/proc/meminfo`. 155 | - Process usage (%): This information is fetched from `/proc/stat`. 156 | - Process memory: a) virtual, b) resident anonymous memory and c) physical: This information is fetched from `/proc/ None: 45 | """ 46 | Construct a SystemMetricCollector action. 47 | 48 | Parameters 49 | ---------- 50 | target_action : ExecuteProcess 51 | ExecuteProcess (or Node) instance to collect metrics on 52 | log_file : str or LaunchSubstitutionsType 53 | Path to where the collected metrics should be written to 54 | timeout : int or LaunchSubstitutionsType 55 | Maximum time to run the metrics collector if the target process does 56 | not exit 57 | 58 | """ 59 | # These Node/ExecuteProcess arguments are invalid in this context 60 | # because we implicitly set them right here. 61 | assert 'arguments' not in kwargs 62 | assert 'package' not in kwargs 63 | assert 'executable' not in kwargs 64 | assert 'executable' not in kwargs 65 | 66 | self.__pid_var_name = '__PROCESS_ID_%d' % id(self) 67 | 68 | kwargs['package'] = 'buildfarm_perf_tests' 69 | kwargs['executable'] = 'system_metric_collector' 70 | kwargs['arguments'] = [ 71 | '--log', log_file, 72 | '--process_pid', LocalSubstitution(self.__pid_var_name)] 73 | if timeout is not None: 74 | kwargs['arguments'] += [ 75 | '--timeout', 76 | str(timeout) if isinstance(timeout, int) else timeout 77 | ] 78 | 79 | super().__init__(**kwargs) 80 | 81 | self.__target_start_handler = OnProcessStart( 82 | target_action=target_action, on_start=self.__on_target_start) 83 | self.__target_exit_handler = OnProcessExit( 84 | target_action=target_action, on_exit=EmitEvent( 85 | event=ShutdownProcess( 86 | process_matcher=matches_action(self)))) 87 | 88 | def execute(self, context): 89 | context.register_event_handler(self.__target_start_handler) 90 | 91 | def __on_target_start(self, event, context): 92 | context.extend_locals({self.__pid_var_name: str(event.pid)}) 93 | context.register_event_handler(self.__target_exit_handler) 94 | super().execute(context) 95 | -------------------------------------------------------------------------------- /buildfarm_perf_tests/test_results.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Open Source Robotics Foundation, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import json 16 | 17 | from pandas import read_csv 18 | 19 | 20 | def read_performance_test_csv(csv_path, start_marker='---EXPERIMENT-START---\n'): 21 | """ 22 | Read CSV data from a file on disk, optionall skipping header data. 23 | 24 | Parameters 25 | ---------- 26 | csv_path : str 27 | Path to the *.csv file to read data from. 28 | start_marker : str, optional 29 | Full line contents which mark the end of the header. 30 | 31 | """ 32 | with open(csv_path, 'r') as csv: 33 | if start_marker: 34 | while csv.readline() not in [start_marker, '']: 35 | pass 36 | return read_csv(csv, sep='[ \t]*,[ \t]*', engine='python') 37 | 38 | 39 | def write_jenkins_plot_csv(csv_path, column_name, values): 40 | """ 41 | Write some values to a CSV file for the Jenkins plot plugin to use. 42 | 43 | Parameters 44 | ---------- 45 | csv_path : str 46 | Path to where the *.csv file should be written to. 47 | column_name : str 48 | Name of this data series, such as the RMW or COMM value. 49 | values : list 50 | List of string to be written to the file. 51 | 52 | """ 53 | with open(csv_path, 'w') as csv: 54 | csv.write(','.join([column_name] * len(values)) + '\n') 55 | csv.write(','.join([str(v) for v in values]) + '\n') 56 | 57 | 58 | def write_jenkins_benchmark_json(json_path, sub_group, values): 59 | """ 60 | Write some values to a JSON file for the Jenkins benchmark plugin to use. 61 | 62 | Parameters 63 | ---------- 64 | json_path : str 65 | Path to where the *.json file should be written to. 66 | sub_group : str 67 | Optional supplementary identifier for the test group name. 68 | values : dict 69 | Mapping from measurement name to benchmark result to be written to the file. 70 | 71 | """ 72 | group_name = 'buildfarm_perf_tests.' + sub_group 73 | out_data = { 74 | group_name: values, 75 | } 76 | with open(json_path, 'w') as json_file: 77 | json.dump(out_data, json_file) 78 | -------------------------------------------------------------------------------- /img/agregate_latency.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ros2/buildfarm_perf_tests/0b9dba249ec581b11dd6cf688ed640d55085b103/img/agregate_latency.png -------------------------------------------------------------------------------- /img/cpu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ros2/buildfarm_perf_tests/0b9dba249ec581b11dd6cf688ed640d55085b103/img/cpu.png -------------------------------------------------------------------------------- /img/latency.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ros2/buildfarm_perf_tests/0b9dba249ec581b11dd6cf688ed640d55085b103/img/latency.png -------------------------------------------------------------------------------- /img/lost_packets.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ros2/buildfarm_perf_tests/0b9dba249ec581b11dd6cf688ed640d55085b103/img/lost_packets.png -------------------------------------------------------------------------------- /img/received_packets.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ros2/buildfarm_perf_tests/0b9dba249ec581b11dd6cf688ed640d55085b103/img/received_packets.png -------------------------------------------------------------------------------- /img/sent_packets.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ros2/buildfarm_perf_tests/0b9dba249ec581b11dd6cf688ed640d55085b103/img/sent_packets.png -------------------------------------------------------------------------------- /img/size.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ros2/buildfarm_perf_tests/0b9dba249ec581b11dd6cf688ed640d55085b103/img/size.png -------------------------------------------------------------------------------- /include/linux_cpu_process_measurement.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Open Source Robotics Foundation, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #ifndef LINUX_CPU_PROCESS_MEASUREMENT_HPP_ 15 | #define LINUX_CPU_PROCESS_MEASUREMENT_HPP_ 16 | 17 | #include 18 | #include 19 | #include 20 | #include 21 | 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include "utilities/utilities.hpp" 27 | 28 | class LinuxCPUProcessMeasurement 29 | { 30 | public: 31 | explicit LinuxCPUProcessMeasurement(std::string pid); 32 | double getCPUCurrentlyUsedByCurrentProcess(); 33 | int gettimesinceboot(); 34 | double getUptime(); 35 | 36 | private: 37 | void initCPUProcess(); 38 | 39 | int numProcessors_; 40 | std::string pid_; 41 | 42 | int16_t tickspersec_; 43 | double cpu_usage_; 44 | }; 45 | 46 | #endif // LINUX_CPU_PROCESS_MEASUREMENT_HPP_ 47 | -------------------------------------------------------------------------------- /include/linux_cpu_system_measurement.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Open Source Robotics Foundation, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #ifndef LINUX_CPU_SYSTEM_MEASUREMENT_HPP_ 15 | #define LINUX_CPU_SYSTEM_MEASUREMENT_HPP_ 16 | 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | 26 | enum CPUStates 27 | { 28 | S_USER = 0, 29 | S_NICE, 30 | S_SYSTEM, 31 | S_IDLE, 32 | S_IOWAIT, 33 | S_IRQ, 34 | S_SOFTIRQ, 35 | S_STEAL, 36 | S_GUEST, 37 | S_GUEST_NICE, 38 | }; 39 | 40 | const int NUM_CPU_STATES = 10; 41 | 42 | typedef struct CPUData 43 | { 44 | std::string cpu; 45 | size_t times[NUM_CPU_STATES]; 46 | } CPUData; 47 | 48 | class LinuxCPUSystemMeasurement 49 | { 50 | public: 51 | LinuxCPUSystemMeasurement(); 52 | void ReadStatsCPU(std::vector & entries); 53 | float getCPUSystemCurrentlyUsed(); 54 | 55 | private: 56 | size_t GetIdleTime(const CPUData & e); 57 | size_t GetActiveTime(const CPUData & e); 58 | void initCPUProcess(); 59 | void makeReading(); 60 | 61 | std::vector entries1_; 62 | std::vector entries2_; 63 | int numProcessors_; 64 | }; 65 | 66 | #endif // LINUX_CPU_SYSTEM_MEASUREMENT_HPP_ 67 | -------------------------------------------------------------------------------- /include/linux_memory_process_measurement.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Open Source Robotics Foundation, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #ifndef LINUX_MEMORY_PROCESS_MEASUREMENT_HPP_ 15 | #define LINUX_MEMORY_PROCESS_MEASUREMENT_HPP_ 16 | 17 | #include 18 | #include 19 | #include 20 | #include 21 | 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include "utilities/utilities.hpp" 27 | 28 | class LinuxMemoryProcessMeasurement 29 | { 30 | public: 31 | explicit LinuxMemoryProcessMeasurement(std::string pid); 32 | void makeReading(); 33 | 34 | inline double getTotalVirtualMem() {return this->totalVirtualMem_;} 35 | inline double getVirtualMemUsed() {return this->virtualMemUsed_;} 36 | inline double getVirtualMemUsedProcess() {return this->virtualMemUsedProcess_;} 37 | inline double getTotalPhysMem() {return this->totalPhysMem_;} 38 | inline double getPhysMemUsed() {return this->physMemUsed_;} 39 | inline double getPhysMemUsedProcess() {return this->physMemUsedProcess_;} 40 | inline double getResidentAnonymousMemory() {return this->residentAnonymousMemory_;} 41 | 42 | private: 43 | int getResidentAnonymousMemoryCurrentlyUsedByCurrentProcess(); 44 | int getVirtualMemoryCurrentlyUsedByCurrentProcess(); 45 | int getPhysicalMemoryCurrentlyUsedByCurrentProcess(); 46 | 47 | double totalVirtualMem_; 48 | double virtualMemUsed_; 49 | double residentAnonymousMemory_; 50 | double virtualMemUsedProcess_; 51 | double totalPhysMem_; 52 | double physMemUsed_; 53 | double physMemUsedProcess_; 54 | std::string pid_; 55 | }; 56 | 57 | #endif // LINUX_MEMORY_PROCESS_MEASUREMENT_HPP_ 58 | -------------------------------------------------------------------------------- /include/linux_memory_system_measurement.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Open Source Robotics Foundation, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #ifndef LINUX_MEMORY_SYSTEM_MEASUREMENT_HPP_ 15 | #define LINUX_MEMORY_SYSTEM_MEASUREMENT_HPP_ 16 | 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | class LinuxMemorySystemMeasurement 25 | { 26 | public: 27 | LinuxMemorySystemMeasurement(); 28 | double getTotalMemorySystem(); 29 | double getFreeMemorySystem(); 30 | double getAvailableMemorySystem(); 31 | }; 32 | 33 | #endif // LINUX_MEMORY_SYSTEM_MEASUREMENT_HPP_ 34 | -------------------------------------------------------------------------------- /include/utilities/utilities.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Open Source Robotics Foundation, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #ifndef UTILITIES__UTILITIES_HPP_ 15 | #define UTILITIES__UTILITIES_HPP_ 16 | 17 | #include 18 | #include 19 | #include 20 | #include 21 | 22 | #if __cplusplus == 201402L 23 | #include // C++14 24 | namespace fs = std::experimental::filesystem; // C++14 25 | #elif __cplusplus > 201402L 26 | #include // C++17 27 | namespace fs = std::filesystem; 28 | #endif 29 | 30 | std::string getPIDByName(std::string process_name, std::string my_own_name, std::string arguments); 31 | std::vector split(const std::string & s, char delimiter); 32 | 33 | #endif // UTILITIES__UTILITIES_HPP_ 34 | -------------------------------------------------------------------------------- /package.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | buildfarm_perf_tests 5 | 0.0.0 6 | Performance tests which run regularly on the buildfarm 7 | Scott K Logan 8 | Apache License 2.0 9 | 10 | ament_cmake 11 | ament_cmake_python 12 | 13 | rclcpp 14 | 15 | ament_index_python 16 | launch 17 | launch_ros 18 | python3-pandas 19 | 20 | ament_lint_auto 21 | ament_lint_common 22 | launch_testing_ament_cmake 23 | performance_test 24 | rmw_implementation_cmake 25 | 26 | 27 | python3-matplotlib 28 | python3-numpy 29 | 30 | 31 | ament_cmake 32 | 33 | 34 | -------------------------------------------------------------------------------- /scripts/generate_config_yaml.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2020 Open Source Robotics Foundation, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import argparse 18 | import sys 19 | 20 | from buildfarm_perf_tests.generate_config_yaml import node_spinnnig 21 | from buildfarm_perf_tests.generate_config_yaml import overhead_simple_publisher_and_subscriber 22 | from buildfarm_perf_tests.generate_config_yaml import performance_test 23 | 24 | 25 | def main(): 26 | """Call main function.""" 27 | parser = argparse.ArgumentParser(description='Create ros buildfarm config.') 28 | parser.add_argument('-f', '--filename', dest='filename', required=True, 29 | help='output file', metavar='FILE') 30 | parser.add_argument('-c', '--ci_name', dest='ci_name', required=True, 31 | help='ci_name job', metavar='ci_name',) 32 | parser.add_argument('-rmw', '--rmw_implementations', metavar='rmw_implementations', nargs='+', 33 | help='This will integrate the rmw_implementation in the plots') 34 | args = parser.parse_args() 35 | 36 | with open(args.filename, 'w') as f: 37 | overhead_simple_publisher_and_subscriber(f, args.rmw_implementations, args.ci_name) 38 | node_spinnnig(f, args.ci_name) 39 | performance_test(f, args.ci_name) 40 | 41 | 42 | if __name__ == '__main__': 43 | sys.exit(main()) 44 | -------------------------------------------------------------------------------- /src/linux_cpu_process_measurement.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Open Source Robotics Foundation, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #include "linux_cpu_process_measurement.hpp" 15 | #include 16 | #include 17 | 18 | LinuxCPUProcessMeasurement::LinuxCPUProcessMeasurement(std::string pid) 19 | : numProcessors_(0), 20 | pid_(pid), 21 | tickspersec_(sysconf(_SC_CLK_TCK)) 22 | { 23 | initCPUProcess(); 24 | } 25 | 26 | void LinuxCPUProcessMeasurement::initCPUProcess() 27 | { 28 | std::string line; 29 | std::ifstream myfile(std::string("/proc/cpuinfo")); 30 | if (myfile.is_open()) { 31 | while (std::getline(myfile, line) ) { 32 | if (line.find("processor") != std::string::npos) { 33 | this->numProcessors_++; 34 | } 35 | } 36 | } 37 | myfile.close(); 38 | } 39 | 40 | int LinuxCPUProcessMeasurement::gettimesinceboot() 41 | { 42 | double result = 0; 43 | std::string line; 44 | std::ifstream myfile(std::string("/proc/uptime")); 45 | if (myfile.is_open()) { 46 | while (std::getline(myfile, line) ) { 47 | std::vector tokens = split(line, ' '); 48 | result = atof(tokens[0].c_str()) * tickspersec_ + atof(tokens[1].c_str()); 49 | } 50 | } 51 | myfile.close(); 52 | return result; 53 | } 54 | 55 | double LinuxCPUProcessMeasurement::getUptime() 56 | { 57 | double uptime = 0; 58 | std::string line; 59 | std::ifstream myfile(std::string("/proc/uptime")); 60 | if (myfile.is_open()) { 61 | while (std::getline(myfile, line) ) { 62 | std::vector tokens = split(line, ' '); 63 | uptime = atof(tokens[0].c_str()); 64 | } 65 | } 66 | myfile.close(); 67 | return uptime; 68 | } 69 | 70 | double LinuxCPUProcessMeasurement::getCPUCurrentlyUsedByCurrentProcess() 71 | { 72 | int64_t pid; 73 | std::string tcomm_string; 74 | char state; 75 | 76 | int64_t ppid; 77 | int64_t pgid; 78 | int64_t sid; 79 | int64_t tty_nr; 80 | int64_t tty_pgrp; 81 | 82 | int64_t flags; 83 | int64_t min_flt; 84 | int64_t cmin_flt; 85 | int64_t maj_flt; 86 | int64_t cmaj_flt; 87 | int64_t utime; 88 | int64_t stimev; 89 | 90 | int64_t cutime; 91 | int64_t cstime; 92 | int64_t priority; 93 | int64_t nicev; 94 | int64_t num_threads; 95 | int64_t it_real_value; 96 | 97 | uint64_t start_time; 98 | 99 | int64_t vsize; 100 | int64_t rss; 101 | int64_t rsslim; 102 | int64_t start_code; 103 | int64_t end_code; 104 | int64_t start_stack; 105 | int64_t esp; 106 | int64_t eip; 107 | 108 | int64_t pending; 109 | int64_t blocked; 110 | int64_t sigign; 111 | int64_t sigcatch; 112 | int64_t wchan; 113 | int64_t zero1; 114 | int64_t zero2; 115 | int64_t exit_signal; 116 | int64_t cpu; 117 | int64_t rt_priority; 118 | int64_t policy; 119 | 120 | double uptime = getUptime(); 121 | 122 | std::ifstream inputFile(std::string("/proc/") + this->pid_ + std::string("/stat")); 123 | 124 | if (!inputFile.good()) { 125 | return 0; 126 | } 127 | 128 | inputFile >> pid >> tcomm_string >> state >> ppid >> pgid >> sid >> tty_nr >> 129 | tty_pgrp >> flags >> min_flt >> cmin_flt >> maj_flt >> cmaj_flt >> 130 | utime >> stimev >> cutime >> cstime >> priority >> nicev >> num_threads >> 131 | it_real_value >> start_time >> vsize >> rss >> rsslim >> start_code >> 132 | end_code >> start_stack >> esp >> eip >> pending >> blocked >> sigign >> 133 | sigcatch >> wchan >> zero1 >> zero2 >> exit_signal >> cpu >> rt_priority >> 134 | policy; 135 | 136 | double total_time = static_cast(utime) + static_cast(stimev); 137 | total_time += static_cast(cstime) + static_cast(cutime); 138 | 139 | double seconds = uptime - (start_time / this->tickspersec_); 140 | 141 | this->cpu_usage_ = 100 * ((total_time / this->tickspersec_) / seconds) / this->numProcessors_; 142 | 143 | return this->cpu_usage_; 144 | } 145 | -------------------------------------------------------------------------------- /src/linux_cpu_system_measurement.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Open Source Robotics Foundation, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #include "linux_cpu_system_measurement.hpp" 15 | 16 | #include 17 | #include 18 | 19 | LinuxCPUSystemMeasurement::LinuxCPUSystemMeasurement() 20 | : numProcessors_(0) 21 | { 22 | initCPUProcess(); 23 | makeReading(); 24 | } 25 | 26 | void LinuxCPUSystemMeasurement::initCPUProcess() 27 | { 28 | std::string line; 29 | std::ifstream myfile(std::string("/proc/cpuinfo")); 30 | if (myfile.is_open()) { 31 | while (std::getline(myfile, line) ) { 32 | if (line.find("processor") != std::string::npos) { 33 | this->numProcessors_++; 34 | } 35 | } 36 | } 37 | myfile.close(); 38 | } 39 | 40 | size_t LinuxCPUSystemMeasurement::GetIdleTime(const CPUData & e) 41 | { 42 | return e.times[S_IDLE] + 43 | e.times[S_IOWAIT]; 44 | } 45 | size_t LinuxCPUSystemMeasurement::GetActiveTime(const CPUData & e) 46 | { 47 | return e.times[S_USER] + 48 | e.times[S_NICE] + 49 | e.times[S_SYSTEM] + 50 | e.times[S_IRQ] + 51 | e.times[S_SOFTIRQ] + 52 | e.times[S_STEAL] + 53 | e.times[S_GUEST] + 54 | e.times[S_GUEST_NICE]; 55 | } 56 | 57 | void LinuxCPUSystemMeasurement::makeReading() 58 | { 59 | ReadStatsCPU(entries1_); 60 | } 61 | 62 | float LinuxCPUSystemMeasurement::getCPUSystemCurrentlyUsed() 63 | { 64 | ReadStatsCPU(entries2_); 65 | 66 | const size_t NUM_ENTRIES = entries1_.size(); 67 | 68 | double total = 0; 69 | 70 | for (size_t i = 0; i < NUM_ENTRIES; ++i) { 71 | const CPUData & e1 = entries1_[i]; 72 | const CPUData & e2 = entries2_[i]; 73 | 74 | const float ACTIVE_TIME = static_cast(GetActiveTime(e2) - GetActiveTime(e1)); 75 | const float IDLE_TIME = static_cast(GetIdleTime(e2) - GetIdleTime(e1)); 76 | const float TOTAL_TIME = ACTIVE_TIME + IDLE_TIME; 77 | 78 | if (TOTAL_TIME > 0.0) { 79 | total += (100.f * ACTIVE_TIME / TOTAL_TIME); 80 | } 81 | } 82 | 83 | entries1_ = entries2_; 84 | 85 | return total / this->numProcessors_; 86 | } 87 | 88 | void LinuxCPUSystemMeasurement::ReadStatsCPU(std::vector & entries) 89 | { 90 | entries.clear(); 91 | std::ifstream fileStat("/proc/stat"); 92 | 93 | std::string line; 94 | 95 | const std::string STR_CPU("cpu"); 96 | const std::size_t LEN_STR_CPU = STR_CPU.size(); 97 | 98 | while (std::getline(fileStat, line)) { 99 | // cpu stats line found 100 | if (!line.compare(0, LEN_STR_CPU, STR_CPU)) { 101 | std::istringstream ss(line); 102 | 103 | // read cpu label 104 | std::string label; 105 | ss >> label; 106 | 107 | if (label.size() > LEN_STR_CPU) { 108 | label.erase(0, LEN_STR_CPU); 109 | } else { 110 | continue; 111 | } 112 | 113 | // store entry 114 | entries.emplace_back(CPUData()); 115 | CPUData & entry = entries.back(); 116 | entry.cpu = label; 117 | 118 | // read times 119 | for (int i = 0; i < NUM_CPU_STATES; ++i) { 120 | ss >> entry.times[i]; 121 | } 122 | } 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /src/linux_memory_process_measurement.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Open Source Robotics Foundation, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #include "linux_memory_process_measurement.hpp" 15 | 16 | #include 17 | #include 18 | 19 | LinuxMemoryProcessMeasurement::LinuxMemoryProcessMeasurement(std::string pid) 20 | : totalVirtualMem_(0), 21 | virtualMemUsed_(0), 22 | virtualMemUsedProcess_(0), 23 | totalPhysMem_(0), 24 | physMemUsed_(0), 25 | physMemUsedProcess_(0), 26 | pid_(pid) 27 | { 28 | } 29 | 30 | int LinuxMemoryProcessMeasurement::getVirtualMemoryCurrentlyUsedByCurrentProcess() 31 | { // Note: this value is in KB! 32 | std::ifstream inputFile(std::string("/proc/") + this->pid_ + std::string("/statm")); 33 | if (!inputFile.good()) { 34 | return 0; 35 | } 36 | int process_memory_used; 37 | int process_memory_used_resident; 38 | inputFile >> process_memory_used >> process_memory_used_resident; 39 | return process_memory_used; 40 | } 41 | 42 | int LinuxMemoryProcessMeasurement::getResidentAnonymousMemoryCurrentlyUsedByCurrentProcess() 43 | { // Note: this value is in KB! 44 | std::ifstream inputFile(std::string("/proc/") + this->pid_ + std::string("/statm")); 45 | if (!inputFile.good()) { 46 | return 0; 47 | } 48 | int process_memory_used; 49 | int process_memory_used_resident; 50 | inputFile >> process_memory_used >> process_memory_used_resident; 51 | return process_memory_used_resident; 52 | } 53 | 54 | int LinuxMemoryProcessMeasurement::getPhysicalMemoryCurrentlyUsedByCurrentProcess() 55 | { // Note: this value is in KB! 56 | std::string line; 57 | std::ifstream myfile(std::string("/proc/") + this->pid_.c_str() + std::string("/status")); 58 | if (myfile.is_open()) { 59 | while (std::getline(myfile, line) ) { 60 | if (line.find("VmRSS:") != std::string::npos) { 61 | std::vector tokens = split(line, ' '); 62 | myfile.close(); 63 | return atoi(tokens[1].c_str()); 64 | } 65 | } 66 | } 67 | return 0; 68 | } 69 | 70 | void LinuxMemoryProcessMeasurement::makeReading() 71 | { 72 | struct sysinfo memInfo; 73 | sysinfo(&memInfo); 74 | int64_t totalVirtualMem = memInfo.totalram; 75 | // Add other values in next statement to avoid int overflow on right hand side... 76 | totalVirtualMem += memInfo.totalswap; 77 | totalVirtualMem *= memInfo.mem_unit; 78 | this->totalVirtualMem_ = totalVirtualMem / 1024.0 / 1024.0; // Mb 79 | 80 | int64_t virtualMemUsed = memInfo.totalram - memInfo.freeram; 81 | // Add other values in next statement to avoid int overflow on right hand side... 82 | virtualMemUsed += memInfo.totalswap - memInfo.freeswap; 83 | virtualMemUsed *= memInfo.mem_unit; 84 | this->virtualMemUsed_ = virtualMemUsed / 1024.0 / 1024.0; // Mb 85 | 86 | this->virtualMemUsedProcess_ = getVirtualMemoryCurrentlyUsedByCurrentProcess() / 1024.0; // Mb 87 | 88 | this->residentAnonymousMemory_ = getResidentAnonymousMemoryCurrentlyUsedByCurrentProcess() / 89 | 1024.0; // Mb 90 | 91 | int64_t totalPhysMem = memInfo.totalram; 92 | // Multiply in next statement to avoid int overflow on right hand side... 93 | totalPhysMem *= memInfo.mem_unit; 94 | this->totalPhysMem_ = totalPhysMem / 1024.0 / 1024.0; // Mb 95 | 96 | int64_t physMemUsed = memInfo.totalram - memInfo.freeram; 97 | physMemUsed *= memInfo.mem_unit; 98 | this->physMemUsed_ = physMemUsed / 1024.0 / 1024.0; 99 | 100 | this->physMemUsedProcess_ = getPhysicalMemoryCurrentlyUsedByCurrentProcess() / 1024.0; // Mb 101 | } 102 | -------------------------------------------------------------------------------- /src/linux_memory_system_measurement.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Open Source Robotics Foundation, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #include "linux_memory_system_measurement.hpp" 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | LinuxMemorySystemMeasurement::LinuxMemorySystemMeasurement() {} 21 | 22 | double LinuxMemorySystemMeasurement::getTotalMemorySystem() 23 | { 24 | std::string token; 25 | std::ifstream file("/proc/meminfo"); 26 | while (file >> token) { 27 | if (token == "MemTotal:") { 28 | uint64_t mem; 29 | if (file >> mem) { 30 | return mem / 1024.0; // Mb 31 | } else { 32 | return 0; 33 | } 34 | } 35 | // ignore rest of the line 36 | file.ignore(std::numeric_limits::max(), '\n'); 37 | } 38 | return 0; 39 | } 40 | 41 | double LinuxMemorySystemMeasurement::getFreeMemorySystem() 42 | { 43 | std::string token; 44 | std::ifstream file("/proc/meminfo"); 45 | while (file >> token) { 46 | if (token == "MemFree:") { 47 | uint64_t mem; 48 | if (file >> mem) { 49 | return mem / 1024.0; // Mb 50 | } else { 51 | return 0; 52 | } 53 | } 54 | // ignore rest of the line 55 | file.ignore(std::numeric_limits::max(), '\n'); 56 | } 57 | return 0; 58 | } 59 | 60 | double LinuxMemorySystemMeasurement::getAvailableMemorySystem() 61 | { 62 | std::string token; 63 | std::ifstream file("/proc/meminfo"); 64 | while (file >> token) { 65 | if (token == "MemAvailable:") { 66 | uint64_t mem; 67 | if (file >> mem) { 68 | return mem / 1024.0; // Mb 69 | } else { 70 | return 0; 71 | } 72 | } 73 | // ignore rest of the line 74 | file.ignore(std::numeric_limits::max(), '\n'); 75 | } 76 | return 0; 77 | } 78 | -------------------------------------------------------------------------------- /src/main_measurements.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Open Source Robotics Foundation, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include "linux_memory_process_measurement.hpp" 20 | #include "linux_cpu_process_measurement.hpp" 21 | #include "linux_memory_system_measurement.hpp" 22 | #include "linux_cpu_system_measurement.hpp" 23 | 24 | static sem_t sentinel; 25 | 26 | static void post_sentinel(int signum) 27 | { 28 | (void)signum; 29 | const char msg[] = "\nGot signal - shutting down\n"; 30 | (void)!write(STDERR_FILENO, msg, sizeof(msg)); 31 | sem_post(&sentinel); 32 | } 33 | 34 | void show_usage() 35 | { 36 | std::cerr << "Usage: system_metric_collector [OPTION [ARG]] ...\n" << 37 | "Options:\n" << 38 | "\t-h,--help\t\tShow this help message\n" << 39 | "\t--timeout arg (=60)\tTest duration\n" << 40 | "\t--log arg (=out.csv)\tLog filename\n" << 41 | "\t--process_pid arg\tProcess PID\n" << 42 | std::endl; 43 | } 44 | 45 | int main(int argc, char * argv[]) 46 | { 47 | float timeout = 60; 48 | std::ofstream m_os; 49 | std::string process_pid; 50 | struct timespec wait_until; 51 | 52 | signal(SIGINT, post_sentinel); 53 | 54 | for (int i = 1; i < argc; ++i) { 55 | std::string arg = argv[i]; 56 | if ((arg == "-h") || (arg == "--help")) { 57 | show_usage(); 58 | return 0; 59 | } else if (arg == "--process_pid") { 60 | if (i + 1 < argc) { // Make sure we aren't at the end of argv! 61 | // Increment 'i' so we don't get the argument as the next argv[i]. 62 | process_pid = argv[++i]; 63 | } else { // Uh-oh, there was no argument to the destination option. 64 | std::cerr << "--process_pid option requires one argument." << std::endl; 65 | return 1; 66 | } 67 | } else if (arg == "--timeout") { 68 | if (i + 1 < argc) { // Make sure we aren't at the end of argv! 69 | // Increment 'i' so we don't get the argument as the next argv[i]. 70 | timeout = std::atoi(argv[++i]); 71 | } else { // Uh-oh, there was no argument to the destination option. 72 | std::cerr << "--timeout option requires one argument." << std::endl; 73 | return 1; 74 | } 75 | } else if (arg == "--log") { 76 | if (i + 1 < argc) { // Make sure we aren't at the end of argv! 77 | std::string filename = argv[++i]; 78 | m_os.open(filename, std::ofstream::out); 79 | std::cout << "file_name " << filename << std::endl; 80 | if (m_os.is_open()) { 81 | m_os << "T_experiment" << ",cpu_usage (%)" << ",virtual memory (Mb)" << 82 | ",physical memory (Mb)" << ",resident anonymous memory (Mb)" << 83 | ",system_cpu_usage (%)" << ",system virtual memory (Mb)" << std::endl; 84 | } 85 | } else { // Uh-oh, there was no argument to the destination option. 86 | std::cerr << "--log option requires one argument." << std::endl; 87 | return 1; 88 | } 89 | } 90 | } 91 | 92 | auto start = std::chrono::high_resolution_clock::now(); 93 | 94 | LinuxMemoryProcessMeasurement linux_memory_process_measurement(process_pid); 95 | LinuxCPUProcessMeasurement linux_cpu_process_measurement(process_pid); 96 | 97 | LinuxCPUSystemMeasurement linux_cpu_system_measurement; 98 | LinuxMemorySystemMeasurement linux_memory_system_measurement; 99 | 100 | while (true) { 101 | linux_memory_process_measurement.makeReading(); 102 | 103 | double cpu_process_percentage = 104 | linux_cpu_process_measurement.getCPUCurrentlyUsedByCurrentProcess(); 105 | double virtual_mem_process_usage = linux_memory_process_measurement.getVirtualMemUsedProcess(); 106 | double phy_mem_process_usage = linux_memory_process_measurement.getPhysMemUsedProcess(); 107 | double resident_anonymousMemory_process_usage = 108 | linux_memory_process_measurement.getResidentAnonymousMemory(); 109 | double cpu_system_percentage = linux_cpu_system_measurement.getCPUSystemCurrentlyUsed(); 110 | double phy_mem_system_usage = linux_memory_system_measurement.getTotalMemorySystem() - 111 | linux_memory_system_measurement.getAvailableMemorySystem(); 112 | 113 | std::cout << "virtualMemUsed Process: " << virtual_mem_process_usage << " Mb" << std::endl; 114 | std::cout << "ResidentAnonymousMemory Process: " << resident_anonymousMemory_process_usage << 115 | " Mb" << 116 | std::endl; 117 | std::cout << "PhysMemUsedProcess: " << phy_mem_process_usage << " Mb" << std::endl; 118 | std::cout << "CPU Usage Process: " << cpu_process_percentage << " (%)" << std::endl; 119 | std::cout << "CPU Usage system: " << cpu_system_percentage << " (%)" << std::endl; 120 | std::cout << "PhysMemUsedsystem: " << phy_mem_system_usage << " (Mb)" << std::endl; 121 | 122 | clock_gettime(CLOCK_REALTIME, &wait_until); 123 | wait_until.tv_sec += 1; 124 | if (sem_timedwait(&sentinel, &wait_until) == 0 || errno != ETIMEDOUT) { 125 | break; 126 | } 127 | 128 | auto finish = std::chrono::high_resolution_clock::now(); 129 | std::chrono::duration elapsed = finish - start; 130 | float seconds_running = elapsed.count() / 1000; 131 | std::cout << "------------------------- " << std::endl; 132 | 133 | if (seconds_running > timeout) { 134 | break; 135 | } 136 | 137 | if (m_os.is_open()) { 138 | m_os << seconds_running << ", " << cpu_process_percentage << ", " << 139 | virtual_mem_process_usage << ", " << phy_mem_process_usage << ", " << 140 | resident_anonymousMemory_process_usage << ", " << cpu_system_percentage << 141 | ", " << phy_mem_system_usage << std::endl; 142 | } 143 | } 144 | 145 | return 0; 146 | } 147 | -------------------------------------------------------------------------------- /src/node_spinning.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Open Source Robotics Foundation, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | 17 | #include 18 | 19 | class MinimalNode : public rclcpp::Node 20 | { 21 | public: 22 | MinimalNode() 23 | : Node("minimal_node") {} 24 | }; 25 | 26 | int main(int argc, char * argv[]) 27 | { 28 | rclcpp::init(argc, argv); 29 | rclcpp::spin(std::make_shared()); 30 | rclcpp::shutdown(); 31 | return 0; 32 | } 33 | -------------------------------------------------------------------------------- /src/utilities/utilities.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Open Source Robotics Foundation, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #include "utilities/utilities.hpp" 15 | 16 | #include 17 | #include 18 | 19 | std::vector split(const std::string & s, char delimiter) 20 | { 21 | std::vector tokens; 22 | std::string token; 23 | std::istringstream tokenStream(s); 24 | while (std::getline(tokenStream, token, delimiter)) { 25 | if (!token.empty()) { 26 | tokens.push_back(token); 27 | } 28 | } 29 | 30 | return tokens; 31 | } 32 | -------------------------------------------------------------------------------- /templates/node_spinning.txt: -------------------------------------------------------------------------------- 1 | - title: Node Spinning Virtual Memory 2 | description: "The figure shown above shows the virtual memory usage in Mb used by a single node spinning. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • Runtime 30 seconds
  • N process 2
" 3 | y_axis_label: Mb 4 | master_csv_name: plot-${random_number}.csv 5 | style: line 6 | num_builds: 10 7 | y_axis_exclude_zero: true 8 | y_axis_minimum: 0 9 | y_axis_maximum: 1024 10 | data_series: 11 | - data_file: ws/test_results/buildfarm_perf_tests/overhead_node_test_results_*.csv 12 | data_type: csv 13 | selection_flag: INCLUDE_BY_COLUMN 14 | selection_value: 2 15 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/overhead_node_test_results_%name%_virtual_memory.png 16 | - title: Node Spinning CPU Usage 17 | description: "The figure shown above shows the CPU usage in % used by a single node spinning. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • Runtime 30 seconds
  • N process 2
" 18 | y_axis_label: Utilization (%) 19 | master_csv_name: plot-${random_number}-1.csv 20 | style: line 21 | num_builds: 10 22 | y_axis_minimum: 0 23 | y_axis_maximum: 100 24 | y_axis_exclude_zero: true 25 | data_series: 26 | - data_file: ws/test_results/buildfarm_perf_tests/overhead_node_test_results_*.csv 27 | data_type: csv 28 | selection_flag: INCLUDE_BY_COLUMN 29 | selection_value: 5 30 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/overhead_node_test_results_%name%_cpu_usage.png 31 | - title: Node Spinning Physical Memory 32 | description: "The figure shown above shows the physical memory in Mb used by a single node spinning. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • Runtime 30 seconds
  • N process 2
" 33 | y_axis_label: Mb 34 | master_csv_name: plot-${random_number}-2.csv 35 | style: line 36 | num_builds: 10 37 | y_axis_exclude_zero: true 38 | y_axis_minimum: 0 39 | y_axis_maximum: 100 40 | data_series: 41 | - data_file: ws/test_results/buildfarm_perf_tests/overhead_node_test_results_*.csv 42 | data_type: csv 43 | selection_flag: INCLUDE_BY_COLUMN 44 | selection_value: 8 45 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/overhead_node_test_results_%name%_physical_memory.png 46 | - title: Node Spinning Resident Anonymous Memory 47 | description: "The figure shown above shows the resident anonymous memory in Mb used by a single node spinning. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • Runtime 30 seconds
  • N process 2
" 48 | y_axis_label: Mb 49 | master_csv_name: plot-${random_number}-3.csv 50 | style: line 51 | num_builds: 10 52 | y_axis_minimum: 0 53 | y_axis_maximum: 100 54 | y_axis_exclude_zero: true 55 | data_series: 56 | - data_file: ws/test_results/buildfarm_perf_tests/overhead_node_test_results_*.csv 57 | data_type: csv 58 | selection_flag: INCLUDE_BY_COLUMN 59 | selection_value: 11 60 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/overhead_node_test_results_%name%_resident_anonymous_memory.png 61 | -------------------------------------------------------------------------------- /templates/overhead_cpu_usage.txt: -------------------------------------------------------------------------------- 1 | - title: Simple Pub ${rmw_implementation} CPU Usage 2 | description: "The figure shown above shows the CPU usage in % used by the publisher. The publisher is set to ${rmw_implementation} but the subscriber will vary between the avaiables. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 5
  • Runtime 30 seconds
  • N process 2
  • Message size 1K
" 3 | y_axis_label: Utilization (%) 4 | master_csv_name: plot-${random_number}.csv 5 | style: line 6 | num_builds: 10 7 | y_axis_exclude_zero: true 8 | y_axis_minimum: 0 9 | y_axis_maximum: 100 10 | data_series: 11 | - data_file: ws/test_results/buildfarm_perf_tests/overhead_test_results_${rmw_implementation}_*_pub.csv 12 | data_type: csv 13 | selection_flag: INCLUDE_BY_COLUMN 14 | selection_value: 5 15 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/overhead_test_results_%name%_publisher_cpu_usage.png 16 | - title: Simple Sub ${rmw_implementation} CPU Usage 17 | description: "The figure shown above shows the CPU usage in % used by the subscriber. The publisher is set to ${rmw_implementation} but the subscriber will vary between the avaiables. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 5
  • Runtime 30 seconds
  • N process 2
  • Message size 1K
" 18 | y_axis_label: Utilization (%) 19 | master_csv_name: plot-${random_number}-1.csv 20 | style: line 21 | num_builds: 10 22 | y_axis_exclude_zero: true 23 | y_axis_minimum: 0 24 | y_axis_maximum: 100 25 | data_series: 26 | - data_file: ws/test_results/buildfarm_perf_tests/overhead_test_results_${rmw_implementation}_*_sub.csv 27 | data_type: csv 28 | selection_flag: INCLUDE_BY_COLUMN 29 | selection_value: 5 30 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/overhead_test_results_%name%_subscriber_cpu_usage.png 31 | -------------------------------------------------------------------------------- /templates/overhead_lost_messages.txt: -------------------------------------------------------------------------------- 1 | - title: Simple Pub ${rmw_implementation} Lost messages 2 | description: "The figure shown above shows the lost messages. The publisher is set to ${rmw_implementation} but the subscriber will vary between the avaiables. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 5
  • Runtime 30 seconds
  • N process 2
  • Message size 1K
" 3 | y_axis_label: Total Count 4 | master_csv_name: plot-${random_number}.csv 5 | style: line 6 | num_builds: 10 7 | y_axis_minimum: 0 8 | y_axis_maximum: 10 9 | data_series: 10 | - data_file: ws/test_results/buildfarm_perf_tests/overhead_test_results_${rmw_implementation}_*_pub.csv 11 | data_type: csv 12 | selection_flag: INCLUDE_BY_COLUMN 13 | selection_value: 18 14 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/overhead_test_results_%name%_publisher_histogram.png 15 | -------------------------------------------------------------------------------- /templates/overhead_physical_memory.txt: -------------------------------------------------------------------------------- 1 | - title: Simple Pub ${rmw_implementation} Physical Memory 2 | description: "The figure shown above shows the physical memory in Mb used by the publisher. The publisher is set to ${rmw_implementation} but the subscriber will vary between the avaiables. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 5
  • Runtime 30 seconds
  • N process 2
  • Message size 1K
" 3 | y_axis_label: Mb 4 | master_csv_name: plot-${random_number}.csv 5 | style: line 6 | num_builds: 10 7 | y_axis_exclude_zero: true 8 | y_axis_minimum: 0 9 | y_axis_maximum: 100 10 | data_series: 11 | - data_file: ws/test_results/buildfarm_perf_tests/overhead_test_results_${rmw_implementation}_*_pub.csv 12 | data_type: csv 13 | selection_flag: INCLUDE_BY_COLUMN 14 | selection_value: 8 15 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/overhead_test_results_%name%_publisher_physical_memory.png 16 | - title: Simple Sub ${rmw_implementation} Physical Memory 17 | description: "The figure shown above shows the physical memory in Mb used by the subscriber. The publisher is set to ${rmw_implementation} but the subscriber will vary between the avaiables. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 5
  • Runtime 30 seconds
  • N process 2
  • Message size 1K
" 18 | y_axis_label: Mb 19 | master_csv_name: plot-${random_number}-1.csv 20 | style: line 21 | num_builds: 10 22 | y_axis_exclude_zero: true 23 | y_axis_minimum: 0 24 | y_axis_maximum: 100 25 | data_series: 26 | - data_file: ws/test_results/buildfarm_perf_tests/overhead_test_results_${rmw_implementation}_*_sub.csv 27 | data_type: csv 28 | selection_flag: INCLUDE_BY_COLUMN 29 | selection_value: 8 30 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/overhead_test_results_%name%_subscriber_physical_memory.png 31 | -------------------------------------------------------------------------------- /templates/overhead_received_messages.txt: -------------------------------------------------------------------------------- 1 | - title: Simple Pub ${rmw_implementation} Received messages per second 2 | description: "The figure shown above shows the received messages per second. The publisher is set to ${rmw_implementation} but the subscriber will vary between the avaiables. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 5
  • Runtime 30 seconds
  • N process 2
  • Message size 1K
" 3 | y_axis_label: Messages per second 4 | master_csv_name: plot-${random_number}.csv 5 | style: line 6 | num_builds: 10 7 | y_axis_exclude_zero: true 8 | y_axis_minimum: 0 9 | y_axis_maximum: 10 10 | data_series: 11 | - data_file: ws/test_results/buildfarm_perf_tests/overhead_test_results_${rmw_implementation}_*_pub.csv 12 | data_type: csv 13 | selection_flag: INCLUDE_BY_COLUMN 14 | selection_value: 16 15 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/overhead_test_results_%name%_publisher_histogram.png 16 | -------------------------------------------------------------------------------- /templates/overhead_resident_anonymous_memory.txt: -------------------------------------------------------------------------------- 1 | - title: Simple Pub ${rmw_implementation} Resident Anonymous Memory 2 | description: "The figure shown above shows the resident anonymous memory in Mb used by the publisher. The publisher is set to ${rmw_implementation} but the subscriber will vary between the avaiables. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 5
  • Runtime 30 seconds
  • N process 2
  • Message size 1K
" 3 | y_axis_label: Mb 4 | master_csv_name: plot-${random_number}.csv 5 | style: line 6 | num_builds: 10 7 | y_axis_exclude_zero: true 8 | data_series: 9 | - data_file: ws/test_results/buildfarm_perf_tests/overhead_test_results_${rmw_implementation}_*_pub.csv 10 | data_type: csv 11 | selection_flag: INCLUDE_BY_COLUMN 12 | selection_value: 11 13 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/overhead_test_results_%name%_publisher_resident_anonymous_memory.png 14 | - title: Simple Sub ${rmw_implementation} Resident Anonymous Memory 15 | description: "The figure shown above shows the resident anonymous memory in Mb used by the subscriber. The publisher is set to ${rmw_implementation} but the subscriber will vary between the avaiables. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 5
  • Runtime 30 seconds
  • N process 2
  • Message size 1K
" 16 | y_axis_label: Mb 17 | master_csv_name: plot-${random_number}-1.csv 18 | style: line 19 | num_builds: 10 20 | y_axis_exclude_zero: true 21 | data_series: 22 | - data_file: ws/test_results/buildfarm_perf_tests/overhead_test_results_${rmw_implementation}_*_sub.csv 23 | data_type: csv 24 | selection_flag: INCLUDE_BY_COLUMN 25 | selection_value: 11 26 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/overhead_test_results_%name%_subscriber_resident_anonymous_memory.png 27 | -------------------------------------------------------------------------------- /templates/overhead_round_trip.txt: -------------------------------------------------------------------------------- 1 | - title: Simple Pub ${rmw_implementation} Average Round-Trip Time 2 | description: "The figure shown above shows the average round-trip time in milisecond. The publisher is set to ${rmw_implementation} but the subscriber will vary between the avaiables. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 5
  • Runtime 30 seconds
  • N process 2
  • Message size 1K
" 3 | y_axis_label: Milliseconds 4 | master_csv_name: plot-${random_number}.csv 5 | style: line 6 | num_builds: 10 7 | exclZero: false 8 | data_series: 9 | - data_file: ws/test_results/buildfarm_perf_tests/overhead_test_results_${rmw_implementation}_*_pub.csv 10 | data_type: csv 11 | selection_flag: INCLUDE_BY_COLUMN 12 | selection_value: 14 13 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/overhead_test_results_%name%_publisher_latency.png 14 | -------------------------------------------------------------------------------- /templates/overhead_sent_messages.txt: -------------------------------------------------------------------------------- 1 | - title: Simple Pub ${rmw_implementation} Sent messages per second 2 | description: "The figure shown above shows the sent messages per second. The publisher is set to ${rmw_implementation} but the subscriber will vary between the avaiables. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 5
  • Runtime 30 seconds
  • N process 2
  • Message size 1K
" 3 | y_axis_label: Messages per second 4 | master_csv_name: plot-${random_number}.csv 5 | style: line 6 | num_builds: 10 7 | y_axis_exclude_zero: true 8 | y_axis_minimum: 0 9 | y_axis_maximum: 10 10 | data_series: 11 | - data_file: ws/test_results/buildfarm_perf_tests/overhead_test_results_${rmw_implementation}_*_pub.csv 12 | data_type: csv 13 | selection_flag: INCLUDE_BY_COLUMN 14 | selection_value: 17 15 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/overhead_test_results_%name%_publisher_histogram.png 16 | -------------------------------------------------------------------------------- /templates/overhead_virtual_memory.txt: -------------------------------------------------------------------------------- 1 | - title: Simple Pub ${rmw_implementation} Virtual Memory 2 | description: "The figure shown above shows the virtual memory in Mb used by the publisher. The publisher is set to ${rmw_implementation} but the subscriber will vary between the avaiables. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 5
  • Runtime 30 seconds
  • N process 2
  • Message size 1K
" 3 | y_axis_label: Mb 4 | master_csv_name: plot-${random_number}.csv 5 | style: line 6 | num_builds: 10 7 | y_axis_exclude_zero: true 8 | y_axis_minimum: 0 9 | y_axis_maximum: 1000 10 | data_series: 11 | - data_file: ws/test_results/buildfarm_perf_tests/overhead_test_results_${rmw_implementation}_*_pub.csv 12 | data_type: csv 13 | selection_flag: INCLUDE_BY_COLUMN 14 | selection_value: 2 15 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/overhead_test_results_%name%_publisher_virtual_memory.png 16 | - title: Simple Sub ${rmw_implementation} Virtual Memory 17 | description: "The figure shown above shows the virtual memory in Mb used by the subscriber. The publisher is set to ${rmw_implementation} but the subscriber will vary between the avaiables. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 5
  • Runtime 30 seconds
  • N process 2
  • Message size 1K
" 18 | y_axis_label: Mb 19 | master_csv_name: plot-${random_number}-1.csv 20 | style: line 21 | num_builds: 10 22 | y_axis_exclude_zero: true 23 | y_axis_minimum: 0 24 | y_axis_maximum: 1000 25 | data_series: 26 | - data_file: ws/test_results/buildfarm_perf_tests/overhead_test_results_${rmw_implementation}_*_sub.csv 27 | data_type: csv 28 | selection_flag: INCLUDE_BY_COLUMN 29 | selection_value: 2 30 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/overhead_test_results_%name%_subscriber_virtual_memory.png 31 | -------------------------------------------------------------------------------- /templates/performance_test_1p_1k.txt: -------------------------------------------------------------------------------- 1 | - title: Average Single-Trip Time 2 | description: "The figure shown above shows the average single-trip time in millisecond for different DDS vendors using a 1K array message. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 1K
" 3 | y_axis_label: Milliseconds 4 | master_csv_name: plot-${random_number}.csv 5 | style: line 6 | num_builds: 10 7 | y_axis_minimum: 0 8 | y_axis_maximum: 0.2 9 | y_axis_exclude_zero: false 10 | data_series: 11 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_results_*_Array1k.csv 12 | data_type: csv 13 | selection_flag: INCLUDE_BY_COLUMN 14 | selection_value: 0 15 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_results_%name%.png 16 | - title: Throughtput 17 | description: "The figure shown above shows the throughput in Mbits/s for different DDS vendors using a 1K array message. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 1K
" 18 | y_axis_label: Mbits/s (mean) 19 | master_csv_name: plot-${random_number}-1.csv 20 | style: line 21 | num_builds: 10 22 | y_axis_minimum: 0 23 | y_axis_maximum: 1.05 24 | y_axis_exclude_zero: false 25 | data_series: 26 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_results_*_Array1k.csv 27 | data_type: csv 28 | selection_flag: INCLUDE_BY_COLUMN 29 | selection_value: 10 30 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_results_%name%_throughtput.png 31 | - title: Max Resident Set Size 32 | description: "The figure shown above shows the max resident size Megabytes for different DDS vendors using a 1K array message. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 1K
" 33 | y_axis_label: Megabytes 34 | master_csv_name: plot-${random_number}-2.csv 35 | style: line 36 | num_builds: 10 37 | y_axis_exclude_zero: true 38 | data_series: 39 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_results_*_Array1k.csv 40 | data_type: csv 41 | selection_flag: INCLUDE_BY_COLUMN 42 | selection_value: 3 43 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_results_%name%.png 44 | - title: Received messages 45 | description: "The figure shown above shows the received messages for different DDS vendors using a 1K array message. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 1K
" 46 | y_axis_label: Number 47 | master_csv_name: plot-${random_number}-3.csv 48 | style: line 49 | num_builds: 10 50 | y_axis_exclude_zero: true 51 | data_series: 52 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_results_*_Array1k.csv 53 | data_type: csv 54 | selection_flag: INCLUDE_BY_COLUMN 55 | selection_value: 4 56 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_results_%name%_histogram.png 57 | - title: Sent messages 58 | description: "The figure shown above shows the sent messages for different DDS vendors using a 1K array message. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 1K
" 59 | y_axis_label: Number 60 | master_csv_name: plot-${random_number}-4.csv 61 | style: line 62 | num_builds: 10 63 | y_axis_exclude_zero: true 64 | data_series: 65 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_results_*_Array1k.csv 66 | data_type: csv 67 | selection_flag: INCLUDE_BY_COLUMN 68 | selection_value: 5 69 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_results_%name%_histogram.png 70 | - title: Lost messages 71 | description: "The figure shown above shows the lost messages for different DDS vendors using a 1K array message. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 1K
" 72 | y_axis_label: Number 73 | master_csv_name: plot-${random_number}-5.csv 74 | style: line 75 | num_builds: 10 76 | y_axis_exclude_zero: true 77 | data_series: 78 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_results_*_Array1k.csv 79 | data_type: csv 80 | selection_flag: INCLUDE_BY_COLUMN 81 | selection_value: 6 82 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_results_%name%_histogram.png 83 | - title: CPU usage (%) 84 | description: "The figure shown above shows the cpu usage in % for different DDS vendors using a 1K array message. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 1K
" 85 | y_axis_label: Number 86 | master_csv_name: plot-${random_number}-6.csv 87 | style: line 88 | num_builds: 10 89 | y_axis_minimum: 0 90 | y_axis_maximum: 100 91 | y_axis_exclude_zero: true 92 | data_series: 93 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_results_*_Array1k.csv 94 | data_type: csv 95 | selection_flag: INCLUDE_BY_COLUMN 96 | selection_value: 8 97 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_results_%name%_cpu_usage.png 98 | -------------------------------------------------------------------------------- /templates/performance_test_1p_multi.txt: -------------------------------------------------------------------------------- 1 | - title: Average Single-Trip Time (Array1k) 2 | description: "The figure shown above shows the average single-trip time in millisecond for different DDS vendors using a 1K array message. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 1K
" 3 | y_axis_label: Milliseconds 4 | master_csv_name: plot-${random_number}.csv 5 | style: line 6 | num_builds: 10 7 | exclZero: false 8 | data_series: 9 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_results_*_Array1k.csv 10 | data_type: csv 11 | selection_flag: INCLUDE_BY_COLUMN 12 | selection_value: 0 13 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_results_%name%.png 14 | - title: Average Single-Trip Time (Array4k) 15 | description: "The figure shown above shows the average single-trip time in millisecond for different DDS vendors using a 4K array message. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 4K
" 16 | y_axis_label: Milliseconds 17 | master_csv_name: plot-${random_number}-1.csv 18 | style: line 19 | num_builds: 10 20 | exclZero: false 21 | data_series: 22 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_results_*_Array4k.csv 23 | data_type: csv 24 | selection_flag: INCLUDE_BY_COLUMN 25 | selection_value: 0 26 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_results_%name%.png 27 | - title: Average Single-Trip Time (Array16k) 28 | description: "The figure shown above shows the average single-trip time in millisecond for different DDS vendors using a 16K array message. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 16K
" 29 | y_axis_label: Milliseconds 30 | master_csv_name: plot-${random_number}-2.csv 31 | style: line 32 | num_builds: 10 33 | exclZero: false 34 | data_series: 35 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_results_*_Array16k.csv 36 | data_type: csv 37 | selection_flag: INCLUDE_BY_COLUMN 38 | selection_value: 0 39 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_results_%name%.png 40 | - title: Average Single-Trip Time (Array32k) 41 | description: "The figure shown above shows the average single-trip time in millisecond for different DDS vendors using a 32K array message. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 32K
" 42 | y_axis_label: Milliseconds 43 | master_csv_name: plot-${random_number}-3.csv 44 | style: line 45 | num_builds: 10 46 | exclZero: false 47 | data_series: 48 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_results_*_Array32k.csv 49 | data_type: csv 50 | selection_flag: INCLUDE_BY_COLUMN 51 | selection_value: 0 52 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_results_%name%.png 53 | - title: Average Single-Trip Time (Array60k) 54 | description: "The figure shown above shows the average single-trip time in millisecond for different DDS vendors using a 60K array message. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 60K
" 55 | y_axis_label: Milliseconds 56 | master_csv_name: plot-${random_number}-4.csv 57 | style: line 58 | num_builds: 10 59 | exclZero: false 60 | data_series: 61 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_results_*_Array60k.csv 62 | data_type: csv 63 | selection_flag: INCLUDE_BY_COLUMN 64 | selection_value: 0 65 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_results_%name%.png 66 | - title: Average Single-Trip Time (PointCloud512k) 67 | description: "The figure shown above shows the average single-trip time in millisecond for different DDS vendors using a 512K array message. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 512K
" 68 | y_axis_label: Milliseconds 69 | master_csv_name: plot-${random_number}-5.csv 70 | style: line 71 | num_builds: 10 72 | exclZero: false 73 | data_series: 74 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_results_*_PointCloud512k.csv 75 | data_type: csv 76 | selection_flag: INCLUDE_BY_COLUMN 77 | selection_value: 0 78 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_results_%name%.png 79 | - title: Average Single-Trip Time (Array1m) 80 | description: "The figure shown above shows the average single-trip time in millisecond for different DDS vendors using a 1M array message. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 1M
" 81 | y_axis_label: Milliseconds 82 | master_csv_name: plot-${random_number}-6.csv 83 | style: line 84 | num_builds: 10 85 | exclZero: false 86 | data_series: 87 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_results_*_Array1m.csv 88 | data_type: csv 89 | selection_flag: INCLUDE_BY_COLUMN 90 | selection_value: 0 91 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_results_%name%.png 92 | - title: Average Single-Trip Time (Array2m) 93 | description: "The figure shown above shows the average single-trip time in millisecond for different DDS vendors using a 2M array message. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 2M
" 94 | y_axis_label: Milliseconds 95 | master_csv_name: plot-${random_number}-7.csv 96 | style: line 97 | num_builds: 10 98 | exclZero: false 99 | data_series: 100 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_results_*_Array2m.csv 101 | data_type: csv 102 | selection_flag: INCLUDE_BY_COLUMN 103 | selection_value: 0 104 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_results_%name%.png 105 | - title: Average Single-Trip Time (Array4m) 106 | description: "The figure shown above shows the average single-trip time in milisecond for different DDS vendors using a 4M array message. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 4M
" 107 | y_axis_label: Milliseconds 108 | master_csv_name: plot-${random_number}-24.csv 109 | style: line 110 | num_builds: 10 111 | exclZero: false 112 | data_series: 113 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_results_*_Array4m.csv 114 | data_type: csv 115 | selection_flag: INCLUDE_BY_COLUMN 116 | selection_value: 0 117 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_results_%name%.png 118 | - title: Average Single-Trip Time (Array8m) 119 | description: "The figure shown above shows the average single-trip time in milisecond for different DDS vendors using a 8M array message. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 8M
" 120 | y_axis_label: Milliseconds 121 | master_csv_name: plot-${random_number}-25.csv 122 | style: line 123 | num_builds: 10 124 | exclZero: false 125 | data_series: 126 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_results_*_Array8m.csv 127 | data_type: csv 128 | selection_flag: INCLUDE_BY_COLUMN 129 | selection_value: 0 130 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_results_%name%.png 131 | - title: Average Single-Trip Time (PointCloud8m) 132 | description: "The figure shown above shows the average single-trip time in milisecond for different DDS vendors using a 8M point cloud message. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 8M
" 133 | y_axis_label: Milliseconds 134 | master_csv_name: plot-${random_number}-26.csv 135 | style: line 136 | num_builds: 10 137 | exclZero: false 138 | data_series: 139 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_results_*_PointCloud8m.csv 140 | data_type: csv 141 | selection_flag: INCLUDE_BY_COLUMN 142 | selection_value: 0 143 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_results_%name%.png 144 | - title: Throughput (Array1k) 145 | description: "The figure shown above shows the throughput in Mbits/s for different DDS vendors using a 1K array message. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 1K
" 146 | y_axis_label: Mbits/s 147 | master_csv_name: plot-${random_number}-8.csv 148 | style: line 149 | num_builds: 10 150 | exclZero: false 151 | data_series: 152 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_results_*_Array1k.csv 153 | data_type: csv 154 | selection_flag: INCLUDE_BY_COLUMN 155 | selection_value: 10 156 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_results_%name%_throughput.png 157 | - title: Throughput (Array4k) 158 | description: "The figure shown above shows the throughput in Mbits/s for different DDS vendors using a 4K array message. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 4K
" 159 | y_axis_label: Mbits/s 160 | master_csv_name: plot-${random_number}-9.csv 161 | style: line 162 | num_builds: 10 163 | exclZero: false 164 | data_series: 165 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_results_*_Array4k.csv 166 | data_type: csv 167 | selection_flag: INCLUDE_BY_COLUMN 168 | selection_value: 10 169 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_results_%name%_throughput.png 170 | - title: Throughput (Array16k) 171 | description: "The figure shown above shows the throughput in Mbits/s for different DDS vendors using a 16K array message. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 16K
" 172 | y_axis_label: Mbits/s 173 | master_csv_name: plot-${random_number}-10.csv 174 | style: line 175 | num_builds: 10 176 | exclZero: false 177 | data_series: 178 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_results_*_Array16k.csv 179 | data_type: csv 180 | selection_flag: INCLUDE_BY_COLUMN 181 | selection_value: 10 182 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_results_%name%_throughput.png 183 | - title: Throughput (Array32k) 184 | description: "The figure shown above shows the throughput in Mbits/s for different DDS vendors using a 32K array message. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 32K
" 185 | y_axis_label: Mbits/s 186 | master_csv_name: plot-${random_number}-11.csv 187 | style: line 188 | num_builds: 10 189 | exclZero: false 190 | data_series: 191 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_results_*_Array32k.csv 192 | data_type: csv 193 | selection_flag: INCLUDE_BY_COLUMN 194 | selection_value: 10 195 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_results_%name%_throughput.png 196 | - title: Throughput (Array60k) 197 | description: "The figure shown above shows the throughput in Mbits/s for different DDS vendors using a 60K array message. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 60K
" 198 | y_axis_label: Mbits/s 199 | master_csv_name: plot-${random_number}-12.csv 200 | style: line 201 | num_builds: 10 202 | exclZero: false 203 | data_series: 204 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_results_*_Array60k.csv 205 | data_type: csv 206 | selection_flag: INCLUDE_BY_COLUMN 207 | selection_value: 10 208 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_results_%name%_throughput.png 209 | - title: Throughput (PointCloud512k) 210 | description: "The figure shown above shows the throughput in Mbits/s for different DDS vendors using a 512K array message. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 512K
" 211 | y_axis_label: Mbits/s 212 | master_csv_name: plot-${random_number}-13.csv 213 | style: line 214 | num_builds: 10 215 | exclZero: false 216 | data_series: 217 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_results_*_PointCloud512k.csv 218 | data_type: csv 219 | selection_flag: INCLUDE_BY_COLUMN 220 | selection_value: 10 221 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_results_%name%_throughput.png 222 | - title: Throughput (Array1m) 223 | description: "The figure shown above shows the throughput in Mbits/s for different DDS vendors using a 1M array message. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 1M
" 224 | y_axis_label: Mbits/s 225 | master_csv_name: plot-${random_number}-14.csv 226 | style: line 227 | num_builds: 10 228 | exclZero: false 229 | data_series: 230 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_results_*_Array1m.csv 231 | data_type: csv 232 | selection_flag: INCLUDE_BY_COLUMN 233 | selection_value: 10 234 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_results_%name%_throughput.png 235 | - title: Throughput (Array2m) 236 | description: "The figure shown above shows the throughput in Mbits/s for different DDS vendors using a 2M array message. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 2M
" 237 | y_axis_label: Mbits/s 238 | master_csv_name: plot-${random_number}-15.csv 239 | style: line 240 | num_builds: 10 241 | exclZero: false 242 | data_series: 243 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_results_*_Array2m.csv 244 | data_type: csv 245 | selection_flag: INCLUDE_BY_COLUMN 246 | selection_value: 10 247 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_results_%name%_throughput.png 248 | - title: Throughput (Array4m) 249 | description: "The figure shown above shows the throughput in Mbits/s for different DDS vendors using a 4M array message. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 4M
" 250 | y_axis_label: Mbits/s 251 | master_csv_name: plot-${random_number}-27.csv 252 | style: line 253 | num_builds: 10 254 | exclZero: false 255 | data_series: 256 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_results_*_Array4m.csv 257 | data_type: csv 258 | selection_flag: INCLUDE_BY_COLUMN 259 | selection_value: 10 260 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_results_%name%_throughput.png 261 | - title: Throughput (Array8m) 262 | description: "The figure shown above shows the throughput in Mbits/s for different DDS vendors using a 8M array message. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 8M
" 263 | y_axis_label: Mbits/s 264 | master_csv_name: plot-${random_number}-28.csv 265 | style: line 266 | num_builds: 10 267 | exclZero: false 268 | data_series: 269 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_results_*_Array8m.csv 270 | data_type: csv 271 | selection_flag: INCLUDE_BY_COLUMN 272 | selection_value: 10 273 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_results_%name%_throughput.png 274 | - title: Throughput (PointCloud8m) 275 | description: "The figure shown above shows the throughput in Mbits/s for different DDS vendors using a 8M point cloud message. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 8M
" 276 | y_axis_label: Mbits/s 277 | master_csv_name: plot-${random_number}-29.csv 278 | style: line 279 | num_builds: 10 280 | exclZero: false 281 | data_series: 282 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_results_*_PointCloud8m.csv 283 | data_type: csv 284 | selection_flag: INCLUDE_BY_COLUMN 285 | selection_value: 10 286 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_results_%name%_throughput.png 287 | - title: Lost messages (Array1k) 288 | description: "The figure shown above shows the lost messages for different DDS vendors using a 1K array message. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 1K
" 289 | y_axis_label: Number 290 | master_csv_name: plot-${random_number}-16.csv 291 | style: line 292 | num_builds: 10 293 | y_axis_exclude_zero: true 294 | data_series: 295 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_results_*_Array1k.csv 296 | data_type: csv 297 | selection_flag: INCLUDE_BY_COLUMN 298 | selection_value: 6 299 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_results_%name%_histogram.png 300 | - title: Lost messages (Array4k) 301 | description: "The figure shown above shows the lost messages for different DDS vendors using a 4K array message. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 4K
" 302 | y_axis_label: Number 303 | master_csv_name: plot-${random_number}-17.csv 304 | style: line 305 | num_builds: 10 306 | y_axis_exclude_zero: true 307 | data_series: 308 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_results_*_Array4k.csv 309 | data_type: csv 310 | selection_flag: INCLUDE_BY_COLUMN 311 | selection_value: 6 312 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_results_%name%_histogram.png 313 | - title: Lost messages (Array16k) 314 | description: "The figure shown above shows the lost messages for different DDS vendors using a 16K array message. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 16K
" 315 | y_axis_label: Number 316 | master_csv_name: plot-${random_number}-18.csv 317 | style: line 318 | num_builds: 10 319 | y_axis_exclude_zero: true 320 | data_series: 321 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_results_*_Array16k.csv 322 | data_type: csv 323 | selection_flag: INCLUDE_BY_COLUMN 324 | selection_value: 6 325 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_results_%name%_histogram.png 326 | - title: Lost messages (Array32k) 327 | description: "The figure shown above shows the lost messages for different DDS vendors using a 32K array message. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 32K
" 328 | y_axis_label: Number 329 | master_csv_name: plot-${random_number}-19.csv 330 | style: line 331 | num_builds: 10 332 | y_axis_exclude_zero: true 333 | data_series: 334 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_results_*_Array32k.csv 335 | data_type: csv 336 | selection_flag: INCLUDE_BY_COLUMN 337 | selection_value: 6 338 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_results_%name%_histogram.png 339 | - title: Lost messages (Array60k) 340 | description: "The figure shown above shows the lost messages for different DDS vendors using a 64K array message. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 60K
" 341 | y_axis_label: Number 342 | master_csv_name: plot-${random_number}-20.csv 343 | style: line 344 | num_builds: 10 345 | y_axis_exclude_zero: true 346 | data_series: 347 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_results_*_Array60k.csv 348 | data_type: csv 349 | selection_flag: INCLUDE_BY_COLUMN 350 | selection_value: 6 351 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_results_%name%_histogram.png 352 | - title: Lost messages (PointCloud512k) 353 | description: "The figure shown above shows the lost messages for different DDS vendors using a 512K point cloud message. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 512k
" 354 | y_axis_label: Number 355 | master_csv_name: plot-${random_number}-21.csv 356 | style: line 357 | num_builds: 10 358 | y_axis_exclude_zero: true 359 | data_series: 360 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_results_*_PointCloud512k.csv 361 | data_type: csv 362 | selection_flag: INCLUDE_BY_COLUMN 363 | selection_value: 6 364 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_results_%name%_histogram.png 365 | - title: Lost messages (Array1m) 366 | description: "The figure shown above shows the lost messages for different DDS vendors using a 1m array message. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 1m
" 367 | y_axis_label: Number 368 | master_csv_name: plot-${random_number}-22.csv 369 | style: line 370 | num_builds: 10 371 | y_axis_exclude_zero: true 372 | data_series: 373 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_results_*_Array1m.csv 374 | data_type: csv 375 | selection_flag: INCLUDE_BY_COLUMN 376 | selection_value: 6 377 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_results_%name%_histogram.png 378 | - title: Lost messages (Array2m) 379 | description: "The figure shown above shows the lost messages for different DDS vendors using a 2m array message. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 2m
" 380 | y_axis_label: Number 381 | master_csv_name: plot-${random_number}-23.csv 382 | style: line 383 | num_builds: 10 384 | y_axis_exclude_zero: true 385 | data_series: 386 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_results_*_Array2m.csv 387 | data_type: csv 388 | selection_flag: INCLUDE_BY_COLUMN 389 | selection_value: 6 390 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_results_%name%_histogram.png 391 | - title: Lost messages (Array4m) 392 | description: "The figure shown above shows the lost messages for different DDS vendors using a 4m array message. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 4m
" 393 | y_axis_label: Number 394 | master_csv_name: plot-${random_number}-30.csv 395 | style: line 396 | num_builds: 10 397 | y_axis_exclude_zero: true 398 | data_series: 399 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_results_*_Array4m.csv 400 | data_type: csv 401 | selection_flag: INCLUDE_BY_COLUMN 402 | selection_value: 6 403 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_results_%name%_histogram.png 404 | - title: Lost messages (Array8m) 405 | description: "The figure shown above shows the lost messages for different DDS vendors using a 8m array message. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 8m
" 406 | y_axis_label: Number 407 | master_csv_name: plot-${random_number}-31.csv 408 | style: line 409 | num_builds: 10 410 | y_axis_exclude_zero: true 411 | data_series: 412 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_results_*_Array8m.csv 413 | data_type: csv 414 | selection_flag: INCLUDE_BY_COLUMN 415 | selection_value: 6 416 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_results_%name%_histogram.png 417 | - title: Lost messages (PointCloud8m) 418 | description: "The figure shown above shows the lost messages for different DDS vendors using a 8m point cloud message. All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 8m
" 419 | y_axis_label: Number 420 | master_csv_name: plot-${random_number}-32.csv 421 | style: line 422 | num_builds: 10 423 | y_axis_exclude_zero: true 424 | data_series: 425 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_results_*_PointCloud8m.csv 426 | data_type: csv 427 | selection_flag: INCLUDE_BY_COLUMN 428 | selection_value: 6 429 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_results_%name%_histogram.png 430 | -------------------------------------------------------------------------------- /templates/performance_test_2p_1k.txt: -------------------------------------------------------------------------------- 1 | - title: Average Single-Trip Time 2 | description: "The figure shown above shows the average single-trip time in millisecond for different DDS vendors using a 1K array message and two processes (one process to create the publisher and another process for the subscriber). All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 2
  • Message size 1K
" 3 | y_axis_label: Milliseconds 4 | master_csv_name: plot-${random_number}.csv 5 | style: line 6 | num_builds: 10 7 | y_axis_minimum: 0 8 | y_axis_maximum: 0.2 9 | y_axis_exclude_zero: false 10 | data_series: 11 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_*_Array1k.csv 12 | data_type: csv 13 | selection_flag: INCLUDE_BY_COLUMN 14 | selection_value: 0 15 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_%name%.png 16 | - title: Throughtput 17 | description: "The figure shown above shows the throughput in Mbits/s for different DDS vendors using a 1K array message and two processes (one process to create the publisher and another process for the subscriber). All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 2
  • Message size 1k
" 18 | y_axis_label: Mbits/s (mean) 19 | master_csv_name: plot-${random_number}-1.csv 20 | style: line 21 | num_builds: 10 22 | y_axis_minimum: 0 23 | y_axis_maximum: 1.05 24 | y_axis_exclude_zero: false 25 | data_series: 26 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_*_Array1k.csv 27 | data_type: csv 28 | selection_flag: INCLUDE_BY_COLUMN 29 | selection_value: 10 30 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_%name%_throughtput.png 31 | - title: Max Resident Set Size 32 | description: "The figure shown above shows the max resident set size in Megabytes for different DDS vendors using a 1K array message and two processes (one process to create the publisher and another process for the subscriber). All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 2
  • Message size 1k
" 33 | y_axis_label: Megabytes 34 | master_csv_name: plot-${random_number}-2.csv 35 | style: line 36 | num_builds: 10 37 | y_axis_exclude_zero: true 38 | data_series: 39 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_*_Array1k.csv 40 | data_type: csv 41 | selection_flag: INCLUDE_BY_COLUMN 42 | selection_value: 3 43 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_%name%.png 44 | - title: Received messages 45 | description: "The figure shown above shows the received messages per second for different DDS vendors using a 1K array message and two processes (one process to create the publisher and another process for the subscriber). All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 2
  • Message size 1k
" 46 | y_axis_label: Number 47 | master_csv_name: plot-${random_number}-3.csv 48 | style: line 49 | num_builds: 10 50 | y_axis_exclude_zero: true 51 | data_series: 52 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_*_Array1k.csv 53 | data_type: csv 54 | selection_flag: INCLUDE_BY_COLUMN 55 | selection_value: 4 56 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_%name%_histogram.png 57 | - title: Sent messages 58 | description: "The figure shown above shows the sent messages per second for different DDS vendors using a 1K array message and two processes (one process to create the publisher and another process for the subscriber). All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 2
  • Message size 1k
" 59 | y_axis_label: Number 60 | master_csv_name: plot-${random_number}-4.csv 61 | style: line 62 | num_builds: 10 63 | y_axis_exclude_zero: true 64 | data_series: 65 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_*_Array1k.csv 66 | data_type: csv 67 | selection_flag: INCLUDE_BY_COLUMN 68 | selection_value: 5 69 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_%name%_histogram.png 70 | - title: Lost messages 71 | description: "The figure shown above shows the total lost messages for different DDS vendors using a 1K array message and two processes (one process to create the publisher and another process for the subscriber). All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 2
  • Message size 1k
" 72 | y_axis_label: Number 73 | master_csv_name: plot-${random_number}-5.csv 74 | style: line 75 | num_builds: 10 76 | y_axis_exclude_zero: true 77 | data_series: 78 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_*_Array1k.csv 79 | data_type: csv 80 | selection_flag: INCLUDE_BY_COLUMN 81 | selection_value: 6 82 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_%name%_histogram.png 83 | - title: CPU usage (%) 84 | description: "The figure shown above shows the CPU usage in % for different DDS vendors using a 1K array message and two processes (one process to create the publisher and another process for the subscriber). All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 2
  • Message size 1k
" 85 | y_axis_label: Number 86 | master_csv_name: plot-${random_number}-6.csv 87 | style: line 88 | num_builds: 10 89 | y_axis_minimum: 0 90 | y_axis_maximum: 100 91 | y_axis_exclude_zero: true 92 | data_series: 93 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_*_Array1k.csv 94 | data_type: csv 95 | selection_flag: INCLUDE_BY_COLUMN 96 | selection_value: 8 97 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_%name%_cpu_usage.png 98 | -------------------------------------------------------------------------------- /templates/performance_test_2p_multi.txt: -------------------------------------------------------------------------------- 1 | - title: Average Single-Trip Time (Array1k) 2 | description: "The figure shown above shows the average single-trip time in millisecond for different DDS vendors using a 1K array message and two processes (one process to create the publisher and another process for the subscriber). All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 2
  • Message size 1K
" 3 | y_axis_label: Milliseconds 4 | master_csv_name: plot-${random_number}.csv 5 | style: line 6 | num_builds: 10 7 | exclZero: false 8 | data_series: 9 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_*_Array1k.csv 10 | data_type: csv 11 | selection_flag: INCLUDE_BY_COLUMN 12 | selection_value: 0 13 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_%name%.png 14 | - title: Average Single-Trip Time (Array4k) 15 | description: "The figure shown above shows the average single-trip time in millisecond for different DDS vendors using a 4K array message and two processes (one process to create the publisher and another process for the subscriber). All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 2
  • Message size 4K
" 16 | y_axis_label: Milliseconds 17 | master_csv_name: plot-${random_number}-1.csv 18 | style: line 19 | num_builds: 10 20 | exclZero: false 21 | data_series: 22 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_*_Array4k.csv 23 | data_type: csv 24 | selection_flag: INCLUDE_BY_COLUMN 25 | selection_value: 0 26 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_%name%.png 27 | - title: Average Single-Trip Time (Array16k) 28 | description: "The figure shown above shows the average single-trip time in millisecond for different DDS vendors using a 16K array message and two processes (one process to create the publisher and another process for the subscriber). All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 2
  • Message size 16K
" 29 | y_axis_label: Milliseconds 30 | master_csv_name: plot-${random_number}-2.csv 31 | style: line 32 | num_builds: 10 33 | exclZero: false 34 | data_series: 35 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_*_Array16k.csv 36 | data_type: csv 37 | selection_flag: INCLUDE_BY_COLUMN 38 | selection_value: 0 39 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_%name%.png 40 | - title: Average Single-Trip Time (Array32k) 41 | description: "The figure shown above shows the average single-trip time in millisecond for different DDS vendors using a 32K array message and two processes (one process to create the publisher and another process for the subscriber). All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 2
  • Message size 32K
" 42 | y_axis_label: Milliseconds 43 | master_csv_name: plot-${random_number}-3.csv 44 | style: line 45 | num_builds: 10 46 | exclZero: false 47 | data_series: 48 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_*_Array32k.csv 49 | data_type: csv 50 | selection_flag: INCLUDE_BY_COLUMN 51 | selection_value: 0 52 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_%name%.png 53 | - title: Average Single-Trip Time (Array60k) 54 | description: "The figure shown above shows the average single-trip time in millisecond for different DDS vendors using a 60K array message and two processes (one process to create the publisher and another process for the subscriber). All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 2
  • Message size 60K
" 55 | y_axis_label: Milliseconds 56 | master_csv_name: plot-${random_number}-4.csv 57 | style: line 58 | num_builds: 10 59 | exclZero: false 60 | data_series: 61 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_*_Array60k.csv 62 | data_type: csv 63 | selection_flag: INCLUDE_BY_COLUMN 64 | selection_value: 0 65 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_%name%.png 66 | - title: Average Single-Trip Time (PointCloud512k) 67 | description: "The figure shown above shows the average single-trip time in milisecond for different DDS vendors using a 512K point cloud message and two processes (one process to create the publisher and another process for the subscriber). All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 2
  • Message size 512K
" 68 | y_axis_label: Milliseconds 69 | master_csv_name: plot-${random_number}-5.csv 70 | style: line 71 | num_builds: 10 72 | exclZero: false 73 | data_series: 74 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_*_PointCloud512k.csv 75 | data_type: csv 76 | selection_flag: INCLUDE_BY_COLUMN 77 | selection_value: 0 78 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_%name%.png 79 | - title: Average Single-Trip Time (Array1m) 80 | description: "The figure shown above shows the average single-trip time in millisecond for different DDS vendors using a 1M array message and two processes (one process to create the publisher and another process for the subscriber). All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 2
  • Message size 1M
" 81 | y_axis_label: Milliseconds 82 | master_csv_name: plot-${random_number}-6.csv 83 | style: line 84 | num_builds: 10 85 | exclZero: false 86 | data_series: 87 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_*_Array1m.csv 88 | data_type: csv 89 | selection_flag: INCLUDE_BY_COLUMN 90 | selection_value: 0 91 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_%name%.png 92 | - title: Average Single-Trip Time (Array2m) 93 | description: "The figure shown above shows the average single-trip time in millisecond for different DDS vendors using a 2M array message and two processes (one process to create the publisher and another process for the subscriber). All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 2
  • Message size 2M
" 94 | y_axis_label: Milliseconds 95 | master_csv_name: plot-${random_number}-7.csv 96 | style: line 97 | num_builds: 10 98 | exclZero: false 99 | data_series: 100 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_*_Array2m.csv 101 | data_type: csv 102 | selection_flag: INCLUDE_BY_COLUMN 103 | selection_value: 0 104 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_%name%.png 105 | - title: Average Single-Trip Time (Array4m) 106 | description: "The figure shown above shows the average single-trip time in milisecond for different DDS vendors using a 4M array message and two processes (one process to create the publisher and another process for the subscriber). All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 2
  • Message size 4M
" 107 | y_axis_label: Milliseconds 108 | master_csv_name: plot-${random_number}-24.csv 109 | style: line 110 | num_builds: 10 111 | exclZero: false 112 | data_series: 113 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_*_Array4m.csv 114 | data_type: csv 115 | selection_flag: INCLUDE_BY_COLUMN 116 | selection_value: 0 117 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_%name%.png 118 | - title: Average Single-Trip Time (Array8m) 119 | description: "The figure shown above shows the average single-trip time in milisecond for different DDS vendors using a 8M array message and two processes (one process to create the publisher and another process for the subscriber). All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 2
  • Message size 8M
" 120 | y_axis_label: Milliseconds 121 | master_csv_name: plot-${random_number}-25.csv 122 | style: line 123 | num_builds: 10 124 | exclZero: false 125 | data_series: 126 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_*_Array8m.csv 127 | data_type: csv 128 | selection_flag: INCLUDE_BY_COLUMN 129 | selection_value: 0 130 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_%name%.png 131 | - title: Average Single-Trip Time (PointCloud8m) 132 | description: "The figure shown above shows the average single-trip time in milisecond for different DDS vendors using a 8M point cloud message and two processes (one process to create the publisher and another process for the subscriber). All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 2
  • Message size 8M
" 133 | y_axis_label: Milliseconds 134 | master_csv_name: plot-${random_number}-26.csv 135 | style: line 136 | num_builds: 10 137 | exclZero: false 138 | data_series: 139 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_*_PointCloud8m.csv 140 | data_type: csv 141 | selection_flag: INCLUDE_BY_COLUMN 142 | selection_value: 0 143 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_%name%.png 144 | - title: Throughput (Array1k) 145 | description: "The figure shown above shows the throughput in Mbits/s for different DDS vendors using a 1K array message and two processes (one process to create the publisher and another process for the subscriber). All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 1K
" 146 | y_axis_label: Mbits/s 147 | master_csv_name: plot-${random_number}-8.csv 148 | style: line 149 | num_builds: 10 150 | exclZero: false 151 | data_series: 152 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_*_Array1k.csv 153 | data_type: csv 154 | selection_flag: INCLUDE_BY_COLUMN 155 | selection_value: 10 156 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_%name%_throughput.png 157 | - title: Throughput (Array4k) 158 | description: "The figure shown above shows the throughput in Mbits/s for different DDS vendors using a 4K array message and two processes (one process to create the publisher and another process for the subscriber). All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 4K
" 159 | y_axis_label: Mbits/s 160 | master_csv_name: plot-${random_number}-9.csv 161 | style: line 162 | num_builds: 10 163 | exclZero: false 164 | data_series: 165 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_*_Array4k.csv 166 | data_type: csv 167 | selection_flag: INCLUDE_BY_COLUMN 168 | selection_value: 10 169 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_%name%_throughput.png 170 | - title: Throughput (Array16k) 171 | description: "The figure shown above shows the throughput in Mbits/s for different DDS vendors using a 16K array message and two processes (one process to create the publisher and another process for the subscriber). All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 16K
" 172 | y_axis_label: Mbits/s 173 | master_csv_name: plot-${random_number}-10.csv 174 | style: line 175 | num_builds: 10 176 | exclZero: false 177 | data_series: 178 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_*_Array16k.csv 179 | data_type: csv 180 | selection_flag: INCLUDE_BY_COLUMN 181 | selection_value: 10 182 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_%name%_throughput.png 183 | - title: Throughput (Array32k) 184 | description: "The figure shown above shows the throughput in Mbits/s for different DDS vendors using a 32K array message and two processes (one process to create the publisher and another process for the subscriber). All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 32K
" 185 | y_axis_label: Mbits/s 186 | master_csv_name: plot-${random_number}-11.csv 187 | style: line 188 | num_builds: 10 189 | exclZero: false 190 | data_series: 191 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_*_Array32k.csv 192 | data_type: csv 193 | selection_flag: INCLUDE_BY_COLUMN 194 | selection_value: 10 195 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_%name%_throughput.png 196 | - title: Throughput (Array60k) 197 | description: "The figure shown above shows the throughput in Mbits/s for different DDS vendors using a 60K array message and two processes (one process to create the publisher and another process for the subscriber). All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 60K
" 198 | y_axis_label: Mbits/s 199 | master_csv_name: plot-${random_number}-12.csv 200 | style: line 201 | num_builds: 10 202 | exclZero: false 203 | data_series: 204 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_*_Array60k.csv 205 | data_type: csv 206 | selection_flag: INCLUDE_BY_COLUMN 207 | selection_value: 10 208 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_%name%_throughput.png 209 | - title: Throughput (PointCloud512k) 210 | description: "The figure shown above shows the throughput in Mbits/s for different DDS vendors using a 512K point cloud message and two processes (one process to create the publisher and another process for the subscriber). All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 512K
" 211 | y_axis_label: Mbits/s 212 | master_csv_name: plot-${random_number}-13.csv 213 | style: line 214 | num_builds: 10 215 | exclZero: false 216 | data_series: 217 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_*_PointCloud512k.csv 218 | data_type: csv 219 | selection_flag: INCLUDE_BY_COLUMN 220 | selection_value: 10 221 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_%name%_throughput.png 222 | - title: Throughput (Array1m) 223 | description: "The figure shown above shows the throughput in Mbits/s for different DDS vendors using a 1M array message and two processes (one process to create the publisher and another process for the subscriber). All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 1M
" 224 | y_axis_label: Mbits/s 225 | master_csv_name: plot-${random_number}-14.csv 226 | style: line 227 | num_builds: 10 228 | exclZero: false 229 | data_series: 230 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_*_Array1m.csv 231 | data_type: csv 232 | selection_flag: INCLUDE_BY_COLUMN 233 | selection_value: 10 234 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_%name%_throughput.png 235 | - title: Throughput (Array2m) 236 | description: "The figure shown above shows the throughput in Mbits/s for different DDS vendors using a 2M array message and two processes (one process to create the publisher and another process for the subscriber). All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 2M
" 237 | y_axis_label: Mbits/s 238 | master_csv_name: plot-${random_number}-15.csv 239 | style: line 240 | num_builds: 10 241 | exclZero: false 242 | data_series: 243 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_*_Array2m.csv 244 | data_type: csv 245 | selection_flag: INCLUDE_BY_COLUMN 246 | selection_value: 10 247 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_%name%_throughput.png 248 | - title: Throughput (Array4m) 249 | description: "The figure shown above shows the throughput in Mbits/s for different DDS vendors using a 4M array message and two processes (one process to create the publisher and another process for the subscriber). All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 4M
" 250 | y_axis_label: Mbits/s 251 | master_csv_name: plot-${random_number}-27.csv 252 | style: line 253 | num_builds: 10 254 | exclZero: false 255 | data_series: 256 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_*_Array4m.csv 257 | data_type: csv 258 | selection_flag: INCLUDE_BY_COLUMN 259 | selection_value: 10 260 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_%name%_throughput.png 261 | - title: Throughput (Array8m) 262 | description: "The figure shown above shows the throughput in Mbits/s for different DDS vendors using a 8M array message and two processes (one process to create the publisher and another process for the subscriber). All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 8M
" 263 | y_axis_label: Mbits/s 264 | master_csv_name: plot-${random_number}-28.csv 265 | style: line 266 | num_builds: 10 267 | exclZero: false 268 | data_series: 269 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_*_Array8m.csv 270 | data_type: csv 271 | selection_flag: INCLUDE_BY_COLUMN 272 | selection_value: 10 273 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_%name%_throughput.png 274 | - title: Throughput (PointCloud8m) 275 | description: "The figure shown above shows the throughput in Mbits/s for different DDS vendors using a 8M point cloud message and two processes (one process to create the publisher and another process for the subscriber). All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 8M
" 276 | y_axis_label: Mbits/s 277 | master_csv_name: plot-${random_number}-29.csv 278 | style: line 279 | num_builds: 10 280 | exclZero: false 281 | data_series: 282 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_*_PointCloud8m.csv 283 | data_type: csv 284 | selection_flag: INCLUDE_BY_COLUMN 285 | selection_value: 10 286 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_%name%_throughput.png 287 | - title: Lost messages (Array1k) 288 | description: "The figure shown above shows the lost messages for different DDS vendors using a 1K array message and two processes (one process to create the publisher and another process for the subscriber). All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 1K
" 289 | y_axis_label: Number 290 | master_csv_name: plot-${random_number}-16.csv 291 | style: line 292 | num_builds: 10 293 | y_axis_exclude_zero: true 294 | data_series: 295 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_*_Array1k.csv 296 | data_type: csv 297 | selection_flag: INCLUDE_BY_COLUMN 298 | selection_value: 6 299 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_%name%_histogram.png 300 | - title: Lost messages (Array4k) 301 | description: "The figure shown above shows the lost messages for different DDS vendors using a 4K array message and two processes (one process to create the publisher and another process for the subscriber). All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 4K
" 302 | y_axis_label: Number 303 | master_csv_name: plot-${random_number}-17.csv 304 | style: line 305 | num_builds: 10 306 | y_axis_exclude_zero: true 307 | data_series: 308 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_*_Array4k.csv 309 | data_type: csv 310 | selection_flag: INCLUDE_BY_COLUMN 311 | selection_value: 6 312 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_%name%_histogram.png 313 | - title: Lost messages (Array16k) 314 | description: "The figure shown above shows the lost messages for different DDS vendors using a 16K array message and two processes (one process to create the publisher and another process for the subscriber). All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 16K
" 315 | y_axis_label: Number 316 | master_csv_name: plot-${random_number}-18.csv 317 | style: line 318 | num_builds: 10 319 | y_axis_exclude_zero: true 320 | data_series: 321 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_*_Array16k.csv 322 | data_type: csv 323 | selection_flag: INCLUDE_BY_COLUMN 324 | selection_value: 6 325 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_%name%_histogram.png 326 | - title: Lost messages (Array32k) 327 | description: "The figure shown above shows the lost messages for different DDS vendors using a 32K array message and two processes (one process to create the publisher and another process for the subscriber). All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 32K
" 328 | y_axis_label: Number 329 | master_csv_name: plot-${random_number}-19.csv 330 | style: line 331 | num_builds: 10 332 | y_axis_exclude_zero: true 333 | data_series: 334 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_*_Array32k.csv 335 | data_type: csv 336 | selection_flag: INCLUDE_BY_COLUMN 337 | selection_value: 6 338 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_%name%_histogram.png 339 | - title: Lost messages (Array60k) 340 | description: "The figure shown above shows the lost messages for different DDS vendors using a 64K array message and two processes (one process to create the publisher and another process for the subscriber). All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 60K
" 341 | y_axis_label: Number 342 | master_csv_name: plot-${random_number}-20.csv 343 | style: line 344 | num_builds: 10 345 | y_axis_exclude_zero: true 346 | data_series: 347 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_*_Array60k.csv 348 | data_type: csv 349 | selection_flag: INCLUDE_BY_COLUMN 350 | selection_value: 6 351 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_%name%_histogram.png 352 | - title: Lost messages (PointCloud512k) 353 | description: "The figure shown above shows the lost messages for different DDS vendors using a 512K point cloud message and two processes (one process to create the publisher and another process for the subscriber). All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 512k
" 354 | y_axis_label: Number 355 | master_csv_name: plot-${random_number}-21.csv 356 | style: line 357 | num_builds: 10 358 | y_axis_exclude_zero: true 359 | data_series: 360 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_*_PointCloud512k.csv 361 | data_type: csv 362 | selection_flag: INCLUDE_BY_COLUMN 363 | selection_value: 6 364 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_%name%_histogram.png 365 | - title: Lost messages (Array1m) 366 | description: "The figure shown above shows the lost messages for different DDS vendors using a 1m array message and two processes (one process to create the publisher and another process for the subscriber). All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 1m
" 367 | y_axis_label: Number 368 | master_csv_name: plot-${random_number}-22.csv 369 | style: line 370 | num_builds: 10 371 | y_axis_exclude_zero: true 372 | data_series: 373 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_*_Array1m.csv 374 | data_type: csv 375 | selection_flag: INCLUDE_BY_COLUMN 376 | selection_value: 6 377 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_%name%_histogram.png 378 | - title: Lost messages (Array2m) 379 | description: "The figure shown above shows the lost messages for different DDS vendors using a 2m array message and two processes (one process to create the publisher and another process for the subscriber). All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 2m
" 380 | y_axis_label: Number 381 | master_csv_name: plot-${random_number}-23.csv 382 | style: line 383 | num_builds: 10 384 | y_axis_exclude_zero: true 385 | data_series: 386 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_*_Array2m.csv 387 | data_type: csv 388 | selection_flag: INCLUDE_BY_COLUMN 389 | selection_value: 6 390 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_%name%_histogram.png 391 | - title: Lost messages (Array4m) 392 | description: "The figure shown above shows the lost messages for different DDS vendors using a 4m array message and two processes (one process to create the publisher and another process for the subscriber). All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 4m
" 393 | y_axis_label: Number 394 | master_csv_name: plot-${random_number}-30.csv 395 | style: line 396 | num_builds: 10 397 | y_axis_exclude_zero: true 398 | data_series: 399 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_*_Array4m.csv 400 | data_type: csv 401 | selection_flag: INCLUDE_BY_COLUMN 402 | selection_value: 6 403 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_%name%_histogram.png 404 | - title: Lost messages (Array8m) 405 | description: "The figure shown above shows the lost messages for different DDS vendors using a 8m array message and two processes (one process to create the publisher and another process for the subscriber). All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 8m
" 406 | y_axis_label: Number 407 | master_csv_name: plot-${random_number}-31.csv 408 | style: line 409 | num_builds: 10 410 | y_axis_exclude_zero: true 411 | data_series: 412 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_*_Array8m.csv 413 | data_type: csv 414 | selection_flag: INCLUDE_BY_COLUMN 415 | selection_value: 6 416 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_%name%_histogram.png 417 | - title: Lost messages (PointCloud8m) 418 | description: "The figure shown above shows the lost messages for different DDS vendors using a 8m point cloud message and two processes (one process to create the publisher and another process for the subscriber). All DDS vendors are configured in asynchronous mode except cycloneDDS which is configured in syncronous mode.

  • QoS Best effort
  • Rate 1000
  • Runtime 30 seconds
  • N process 1
  • Message size 8m
" 419 | y_axis_label: Number 420 | master_csv_name: plot-${random_number}-32.csv 421 | style: line 422 | num_builds: 10 423 | y_axis_exclude_zero: true 424 | data_series: 425 | - data_file: ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_*_PointCloud8m.csv 426 | data_type: csv 427 | selection_flag: INCLUDE_BY_COLUMN 428 | selection_value: 6 429 | url: /job/${ci_name}/%build%/artifact/ws/test_results/buildfarm_perf_tests/performance_test_two_process_results_%name%_histogram.png 430 | -------------------------------------------------------------------------------- /test/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # 2 | # Global performance test settings 3 | # 4 | 5 | set(PERF_TEST_RUNTIME 30 6 | CACHE STRING "Duration for each performance test run") 7 | set(PERF_TEST_TOPIC_PUB_SUB Array60k 8 | CACHE STRING "perf_test topic used by pub/sub tests") 9 | set(PERF_TEST_TOPICS 10 | Array1k 11 | Array4k 12 | Array16k 13 | Array32k 14 | Array60k 15 | Array1m 16 | Array2m 17 | Array4m 18 | Array8m 19 | PointCloud512k 20 | PointCloud8m 21 | CACHE STRING "List of perf_test topics to run tests with" 22 | ) 23 | 24 | # 25 | # Standalone (non-ROS) performance test settings 26 | # 27 | 28 | set(PERF_TEST_COMM_TYPE_rmw_cyclonedds_cpp "CycloneDDS" 29 | CACHE STRING "Standalone COMM type for rmw_cyclonedds_cpp") 30 | set(PERF_TEST_COMM_TYPE_rmw_fastrtps_cpp "FastRTPS" 31 | CACHE STRING "Standalone COMM type for rmw_fastrtps_cpp") 32 | 33 | # 34 | # Special environment variable settings 35 | # 36 | 37 | # Fast-RTPS RMWs need some special env vars for sync mode 38 | get_filename_component(FASTRTPS_DEFAULT_PROFILES_FILE 39 | "${CMAKE_CURRENT_SOURCE_DIR}/DEFAULT_FASTRTPS_PROFILES.xml" ABSOLUTE) 40 | set(PERF_TEST_ENV_rmw_fastrtps_cpp_sync 41 | "RMW_FASTRTPS_USE_QOS_FROM_XML=1" 42 | "FASTRTPS_DEFAULT_PROFILES_FILE=${FASTRTPS_DEFAULT_PROFILES_FILE}" 43 | ) 44 | set(PERF_TEST_ENV_rmw_fastrtps_dynamic_cpp_sync 45 | "RMW_FASTRTPS_USE_QOS_FROM_XML=1" 46 | "FASTRTPS_DEFAULT_PROFILES_FILE=${FASTRTPS_DEFAULT_PROFILES_FILE}" 47 | ) 48 | 49 | # ...and some EXTRA special env vars for multi-process sync mode 50 | get_filename_component(FASTRTPS_DEFAULT_PROFILES_TWO_PROCESSES_FILE 51 | "${CMAKE_CURRENT_SOURCE_DIR}/DEFAULT_FASTRTPS_PROFILES_TWO_PROCESSES.xml" ABSOLUTE) 52 | set(PERF_TEST_ENV_two_processes_rmw_fastrtps_cpp_sync 53 | "FASTRTPS_DEFAULT_PROFILES_FILE=${FASTRTPS_DEFAULT_PROFILES_TWO_PROCESSES_FILE}" 54 | ) 55 | set(PERF_TEST_ENV_two_processes_rmw_fastrtps_dynamic_cpp_sync 56 | "FASTRTPS_DEFAULT_PROFILES_FILE=${FASTRTPS_DEFAULT_PROFILES_TWO_PROCESSES_FILE}" 57 | ) 58 | set(PERF_TEST_ENV_pub_sub_rmw_fastrtps_cpp_sync 59 | "FASTRTPS_DEFAULT_PROFILES_FILE=${FASTRTPS_DEFAULT_PROFILES_TWO_PROCESSES_FILE}" 60 | ) 61 | set(PERF_TEST_ENV_pub_sub_rmw_fastrtps_dynamic_cpp_sync 62 | "FASTRTPS_DEFAULT_PROFILES_FILE=${FASTRTPS_DEFAULT_PROFILES_TWO_PROCESSES_FILE}" 63 | ) 64 | 65 | # 66 | # Skipped test settings 67 | # 68 | 69 | # Some DDS/RMWs don't support async/sync 70 | set(PERF_TEST_SKIP_CycloneDDS_async ON 71 | CACHE BOOL "Skip CycloneDDS async tests") 72 | set(PERF_TEST_SKIP_rmw_cyclonedds_cpp_async ON 73 | CACHE BOOL "Skip rmw_cyclonedds_cpp async tests") 74 | set(PERF_TEST_SKIP_rmw_opensplice_cpp_sync ON 75 | CACHE BOOL "Skip rmw_opensplice_cpp sync tests") 76 | set(PERF_TEST_SKIP_rmw_connext_cpp_sync ON 77 | CACHE BOOL "Skip rmw_connext_cpp sync tests") 78 | 79 | # Fast-RTPS ros2-eloquent branch (1.9.3 + bugfixes) does not support synchronous 80 | # sending of fragments. This means that in that branch, topics Array1m, 81 | # Array2m, Array4m, Array8m, PointCloud512k, PointCloud8m cannot be sent synchronously in 82 | # ROS 2 Eloquent (either from standalone Fast-RTPS, or from rmw_fastrtps_*) 83 | # https://github.com/ros2/buildfarm_perf_tests/pull/26#discussion_r393229707 84 | set(FASTRTPS_SYNC_SKIP_ROS_DISTROS 85 | "dashing" 86 | "eloquent" 87 | ) 88 | if("$ENV{ROS_DISTRO}" IN_LIST FASTRTPS_SYNC_SKIP_ROS_DISTROS) 89 | foreach(skip_rmw rmw_fastrtps_cpp rmw_fastrtps_dynamic_cpp FastRTPS) 90 | foreach(skip_topic Array1m Array2m Array4m Array8m PointCloud512k PointCloud8m) 91 | set(PERF_TEST_SKIP_${skip_rmw}_sync_${skip_topic} ON 92 | CACHE BOOL "Skip ${skip_rmw} sync ${skip_topic} tests") 93 | endforeach() 94 | endforeach() 95 | endif() 96 | 97 | # TODO (ahcorde): perf_test is not working with CycloneDDS when 98 | # processes are splitted in two 99 | # what(): Round trip mode is not implemented for Cyclone DDS! 100 | set(PERF_TEST_SKIP_two_process_CycloneDDS_async ON 101 | CACHE BOOL "Skip two-process CycloneDDS async tests") 102 | set(PERF_TEST_SKIP_two_process_CycloneDDS_sync ON 103 | CACHE BOOL "Skip two-process CycloneDDS sync tests") 104 | 105 | # 106 | # Add performance tests 107 | # 108 | 109 | find_package(ament_cmake_test REQUIRED) 110 | find_package(ament_cmake_flake8 REQUIRED) 111 | find_package(launch_testing_ament_cmake REQUIRED) 112 | find_package(rmw_implementation_cmake REQUIRED) 113 | 114 | ament_flake8(TESTNAME "flake8_generated" 115 | ${CMAKE_CURRENT_BINARY_DIR}/test 116 | ) 117 | 118 | include(add_performance_tests.cmake) 119 | 120 | get_available_rmw_implementations(_RMW_IMPLEMENTATIONS) 121 | set(PERF_TEST_RMW_IMPLEMENTATIONS "${_RMW_IMPLEMENTATIONS}" 122 | CACHE STRING "List of RMW implementations to test with") 123 | 124 | add_performance_tests("${PERF_TEST_RMW_IMPLEMENTATIONS}") 125 | 126 | set(GENERATE_CONFIG_YAML_CMD 127 | ${PROJECT_SOURCE_DIR}/scripts/generate_config_yaml.py -rmw foo -c bar -f config_generator_smoke_test.yaml) 128 | ament_add_test("config_generator_smoke_test" 129 | COMMAND ${GENERATE_CONFIG_YAML_CMD} 130 | GENERATE_RESULT_FOR_RETURN_CODE_ZERO 131 | ) 132 | -------------------------------------------------------------------------------- /test/DEFAULT_FASTRTPS_PROFILES.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | FULL 6 | 7 | 8 | 9 | 10 | 11 | SYNCHRONOUS 12 | 13 | 14 | PREALLOCATED_WITH_REALLOC 15 | 16 | 17 | 18 | PREALLOCATED_WITH_REALLOC 19 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /test/DEFAULT_FASTRTPS_PROFILES_TWO_PROCESSES.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | SYNCHRONOUS 8 | 9 | 10 | PREALLOCATED_WITH_REALLOC 11 | 12 | 13 | 14 | PREALLOCATED_WITH_REALLOC 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /test/add_performance_tests.cmake: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Open Source Robotics Foundation, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # 16 | # Add performance tests for a single COMM/RMW type. 17 | # 18 | # Adds the following tests: 19 | # 1. Apex.AI perf_test for every topic defined in ``PERF_TEST_TOPICS`` 20 | # 2. Same as (1.), but using separate publisher and subscriber processes 21 | # 3. Standalone spinner/overhead test (Only for ``ROS2`` COMM) 22 | # 4. Cross-vendor publisher/subscriber tests for each other RMW implementation 23 | # (Only for ``ROS2`` COMM) 24 | # 25 | # :param TEST_NAME: The name the test, used in the resulting test report, output 26 | # artifacts and behavior augmentation variable names. 27 | # :type TEST_NAME: string 28 | # :param COMM: The Apex.AI ``performance_test`` ``COMM`` type (e.x. ``ROS2``). 29 | # :type COMM: string 30 | # :param RMW_IMPLEMENTATION: The RMW implementation for use when ``COMM`` is set 31 | # to ``ROS2``. 32 | # :type RMW_IMPLEMENTATION: string 33 | # :param SYNC_MODE: One of ``async`` or ``sync``. 34 | # :type SYNC_MODE: string 35 | # :param RMW_IMPLMENETATIONS: The list of RMW implementations to test against 36 | # in publisher/subscriber tests (only used when ``COMM`` is set to ``ROS2``). 37 | # :type RMW_IMPLEMENTATIONS: semicolon-separated string 38 | # 39 | # All of the behavior augmentations documented for 40 | # :cmake:macro:`add_performance_tests` also apply to this function. 41 | # 42 | function(add_performance_test 43 | TEST_NAME COMM RMW_IMPLEMENTATION SYNC_MODE RMW_IMPLEMENTATIONS) 44 | 45 | # 46 | # Setup for performance sub-tests 47 | # 48 | 49 | set(TEST_NAME_SYNC "${TEST_NAME}_${SYNC_MODE}") 50 | set(SKIP_TEST "") 51 | if(PERF_TEST_SKIP_${TEST_NAME_SYNC}) 52 | message(STATUS "Skipping performance tests: ${TEST_NAME_SYNC}") 53 | set(SKIP_TEST "SKIP_TEST") 54 | endif() 55 | 56 | set(TEST_ENV "") 57 | if(COMM STREQUAL "ROS2") 58 | list(APPEND TEST_ENV "RMW_IMPLEMENTATION=${RMW_IMPLEMENTATION}") 59 | endif() 60 | list(APPEND TEST_ENV ${PERF_TEST_ENV_${TEST_NAME_SYNC}}) 61 | 62 | # Have CTest kill the test if it runs for more than 3x what it should 63 | math(EXPR PERF_TEST_TIMEOUT "${PERF_TEST_RUNTIME} * 3") 64 | 65 | # 66 | # Setup for per-topic tests 67 | # 68 | 69 | foreach(PERF_TEST_TOPIC ${PERF_TEST_TOPICS}) 70 | # Start with single-process tests 71 | 72 | set(TEST_NAME_SYNC_TOPIC "${TEST_NAME_SYNC}_${PERF_TEST_TOPIC}") 73 | set(SKIP_TEST_TOPIC "${SKIP_TEST}") 74 | if(NOT SKIP_TEST_TOPIC) 75 | if(PERF_TEST_SKIP_${TEST_NAME_SYNC_TOPIC} OR PERF_TEST_SKIP_apex_ai_suite) 76 | message(STATUS "Skipping performance tests: ${TEST_NAME_SYNC_TOPIC}") 77 | set(SKIP_TEST_TOPIC "SKIP_TEST") 78 | endif() 79 | endif() 80 | set(TEST_ENV_TOPIC ${TEST_ENV}) 81 | list(APPEND TEST_ENV_TOPIC ${PERF_TEST_ENV_${TEST_NAME_SYNC_TOPIC}}) 82 | 83 | set(NUMBER_PROCESS "1") 84 | 85 | get_filename_component( 86 | PERF_TEST_RESULTS_BASE_PATH 87 | "${AMENT_TEST_RESULTS_DIR}/${PROJECT_NAME}/performance_test_results_${TEST_NAME_SYNC_TOPIC}" 88 | ABSOLUTE 89 | ) 90 | list(APPEND TEST_ENV_TOPIC 91 | "PERF_TEST_RESULTS_BASE_PATH=${PERF_TEST_RESULTS_BASE_PATH}" 92 | ) 93 | 94 | configure_file( 95 | "test_performance.py.in" 96 | "${CMAKE_CURRENT_BINARY_DIR}/test_performance_${TEST_NAME_SYNC_TOPIC}.py.configure" 97 | @ONLY 98 | ) 99 | file(GENERATE 100 | OUTPUT 101 | "${CMAKE_CURRENT_BINARY_DIR}/test/test_performance_${TEST_NAME_SYNC_TOPIC}.py" 102 | INPUT 103 | "${CMAKE_CURRENT_BINARY_DIR}/test_performance_${TEST_NAME_SYNC_TOPIC}.py.configure" 104 | ) 105 | add_launch_test( 106 | "${CMAKE_CURRENT_BINARY_DIR}/test/test_performance_${TEST_NAME_SYNC_TOPIC}.py" 107 | TARGET test_performance_${TEST_NAME_SYNC_TOPIC} 108 | ENV ${TEST_ENV_TOPIC} 109 | TIMEOUT ${PERF_TEST_TIMEOUT} 110 | ${SKIP_TEST_TOPIC} 111 | ) 112 | set_tests_properties(test_performance_${TEST_NAME_SYNC_TOPIC} PROPERTIES 113 | RUN_SERIAL TRUE 114 | LABELS "performance" 115 | ) 116 | 117 | # Inherit settings from single-process to run two-process tests 118 | 119 | if(NOT SKIP_TEST_TOPIC) 120 | if(PERF_TEST_SKIP_two_process_${TEST_NAME_SYNC_TOPIC} OR 121 | PERF_TEST_SKIP_two_process_${TEST_NAME_SYNC}) 122 | message(STATUS 123 | "Skipping performance tests: two_process_${TEST_NAME_SYNC_TOPIC}") 124 | set(SKIP_TEST_TOPIC "SKIP_TEST") 125 | endif() 126 | endif() 127 | list(APPEND TEST_ENV_TOPIC 128 | ${PERF_TEST_ENV_two_process_${TEST_NAME_SYNC_TOPIC}} 129 | ) 130 | 131 | set(NUMBER_PROCESS "2") 132 | 133 | get_filename_component( 134 | PERF_TEST_RESULTS_BASE_PATH 135 | "${AMENT_TEST_RESULTS_DIR}/${PROJECT_NAME}/performance_test_two_process_results_${TEST_NAME_SYNC_TOPIC}" 136 | ABSOLUTE 137 | ) 138 | list(APPEND TEST_ENV_TOPIC 139 | "PERF_TEST_RESULTS_BASE_PATH=${PERF_TEST_RESULTS_BASE_PATH}" 140 | ) 141 | 142 | configure_file( 143 | "test_performance.py.in" 144 | "${CMAKE_CURRENT_BINARY_DIR}/test_performance_two_process_${TEST_NAME_SYNC_TOPIC}.py.configure" 145 | @ONLY 146 | ) 147 | file(GENERATE 148 | OUTPUT 149 | "${CMAKE_CURRENT_BINARY_DIR}/test/test_performance_two_process_${TEST_NAME_SYNC_TOPIC}.py" 150 | INPUT 151 | "${CMAKE_CURRENT_BINARY_DIR}/test_performance_two_process_${TEST_NAME_SYNC_TOPIC}.py.configure" 152 | ) 153 | 154 | add_launch_test( 155 | "${CMAKE_CURRENT_BINARY_DIR}/test/test_performance_two_process_${TEST_NAME_SYNC_TOPIC}.py" 156 | TARGET test_performance_two_process_${TEST_NAME_SYNC_TOPIC} 157 | ENV ${TEST_ENV_TOPIC} 158 | TIMEOUT ${PERF_TEST_TIMEOUT} 159 | ${SKIP_TEST_TOPIC} 160 | ) 161 | set_tests_properties( 162 | test_performance_two_process_${TEST_NAME_SYNC_TOPIC} PROPERTIES 163 | RUN_SERIAL TRUE 164 | LABELS "performance" 165 | ) 166 | endforeach() 167 | 168 | # 169 | # All remaining tests only apply to RMWs 170 | # 171 | 172 | if(NOT "${COMM}" STREQUAL "ROS2") 173 | # Since standalone DDS can never run correctly, 174 | # exclude instead of skipping the tests. 175 | return() 176 | endif() 177 | 178 | # 179 | # All remaining tests require the system_metric_collector 180 | # 181 | 182 | if(NOT ENABLE_SYSTEM_METRIC_COLLECTOR) 183 | set(SKIP_TEST "SKIP_TEST") 184 | message(STATUS 185 | "Skipping performance tests: ${TEST_NAME_SYNC}" 186 | " (no system_metric_collector)") 187 | endif() 188 | 189 | # 190 | # Setup for spinning tests 191 | # 192 | 193 | set(SKIP_TEST_SPINNING "${SKIP_TEST}") 194 | if(NOT SKIP_TEST_SPINNING) 195 | if(PERF_TEST_SKIP_spinning_${TEST_NAME_SYNC} OR 196 | PERF_TEST_SKIP_spinning OR NOT ENABLE_SYSTEM_METRIC_COLLECTOR) 197 | message(STATUS "Skipping performance tests: spinning_${TEST_NAME_SYNC}") 198 | set(SKIP_TEST_SPINNING "SKIP_TEST") 199 | endif() 200 | endif() 201 | set(TEST_ENV_SPINNING ${TEST_ENV}) 202 | list(APPEND TEST_ENV_SPINNING ${PERF_TEST_ENV_spinning_${TEST_NAME_SYNC}}) 203 | 204 | get_filename_component( 205 | PERF_TEST_RESULTS_BASE_PATH 206 | "${AMENT_TEST_RESULTS_DIR}/${PROJECT_NAME}/overhead_node_test_results_${TEST_NAME_SYNC}" 207 | ABSOLUTE 208 | ) 209 | list(APPEND TEST_ENV_SPINNING 210 | "PERF_TEST_RESULTS_BASE_PATH=${PERF_TEST_RESULTS_BASE_PATH}" 211 | ) 212 | 213 | configure_file( 214 | "test_spinning.py.in" 215 | ${CMAKE_CURRENT_BINARY_DIR}/test_spinning_${TEST_NAME_SYNC}.py.configure 216 | @ONLY 217 | ) 218 | file(GENERATE 219 | OUTPUT 220 | "${CMAKE_CURRENT_BINARY_DIR}/test/test_spinning_${TEST_NAME_SYNC}.py" 221 | INPUT 222 | "${CMAKE_CURRENT_BINARY_DIR}/test_spinning_${TEST_NAME_SYNC}.py.configure" 223 | ) 224 | 225 | add_launch_test( 226 | "${CMAKE_CURRENT_BINARY_DIR}/test/test_spinning_${TEST_NAME_SYNC}.py" 227 | TARGET test_spinning_${TEST_NAME_SYNC} 228 | ENV ${TEST_ENV_SPINNING} 229 | TIMEOUT ${PERF_TEST_TIMEOUT} 230 | ${SKIP_TEST_SPINNING} 231 | ) 232 | set_tests_properties(test_spinning_${TEST_NAME_SYNC} PROPERTIES 233 | RUN_SERIAL TRUE 234 | LABELS "performance" 235 | ) 236 | 237 | # 238 | # Setup for cross-vendor tests 239 | # 240 | 241 | set(PERF_TEST_TOPIC ${PERF_TEST_TOPIC_PUB_SUB}) 242 | foreach(RMW_IMPLEMENTATION_SUB ${RMW_IMPLEMENTATIONS}) 243 | # TODO: Why is ${COMM} on the end here? It's always ROS2. 244 | set(TEST_NAME_PUB_SUB "${TEST_NAME_SYNC}_${RMW_IMPLEMENTATION_SUB}_${COMM}") 245 | set(SKIP_TEST_PUB_SUB "${SKIP_TEST}") 246 | if(NOT SKIP_TEST_PUB_SUB) 247 | if(PERF_TEST_SKIP_${TEST_NAME_PUB_SUB} OR 248 | PERF_TEST_SKIP_pub_sub_${TEST_NAME_SYNC} OR 249 | PERF_TEST_SKIP_pub_sub OR PERF_TEST_SKIP_apex_ai_suite OR 250 | NOT ENABLE_SYSTEM_METRIC_COLLECTOR) 251 | message(STATUS "Skipping performance tests: ${TEST_NAME_PUB_SUB}") 252 | set(SKIP_TEST_PUB_SUB "SKIP_TEST") 253 | endif() 254 | endif() 255 | set(TEST_ENV_PUB_SUB ${TEST_ENV}) 256 | list(APPEND TEST_ENV_PUB_SUB ${PERF_TEST_ENV_pub_sub_${TEST_NAME_SYNC}}) 257 | list(APPEND TEST_ENV_PUB_SUB ${PERF_TEST_ENV_${TEST_NAME_PUB_SUB}}) 258 | 259 | get_filename_component( 260 | PERF_TEST_RESULTS_BASE_PATH 261 | "${AMENT_TEST_RESULTS_DIR}/${PROJECT_NAME}/overhead_test_results_${TEST_NAME_PUB_SUB}" 262 | ABSOLUTE 263 | ) 264 | # TODO: Why doesn't this file name have a suffix? 265 | get_filename_component( 266 | PERFORMANCE_OVERHEAD_PNG 267 | "${AMENT_TEST_RESULTS_DIR}/${PROJECT_NAME}/overhead_test_results" 268 | ABSOLUTE 269 | ) 270 | 271 | list(APPEND TEST_ENV_PUB_SUB 272 | "PERF_TEST_RESULTS_BASE_PATH=${PERF_TEST_RESULTS_BASE_PATH}" 273 | "PERFORMANCE_OVERHEAD_PNG=${PERFORMANCE_OVERHEAD_PNG}" 274 | ) 275 | 276 | configure_file( 277 | "test_pub_sub.py.in" 278 | ${CMAKE_CURRENT_BINARY_DIR}/test_pub_sub_${TEST_NAME_PUB_SUB}.py.configure 279 | @ONLY 280 | ) 281 | file(GENERATE 282 | OUTPUT 283 | "${CMAKE_CURRENT_BINARY_DIR}/test/test_pub_sub_${TEST_NAME_PUB_SUB}.py" 284 | INPUT 285 | "${CMAKE_CURRENT_BINARY_DIR}/test_pub_sub_${TEST_NAME_PUB_SUB}.py.configure" 286 | ) 287 | 288 | add_launch_test( 289 | "${CMAKE_CURRENT_BINARY_DIR}/test/test_pub_sub_${TEST_NAME_PUB_SUB}.py" 290 | TARGET test_pub_sub_${TEST_NAME_PUB_SUB} 291 | ENV ${TEST_ENV_PUB_SUB} 292 | TIMEOUT ${PERF_TEST_TIMEOUT} 293 | ${SKIP_TEST_PUB_SUB} 294 | ) 295 | set_tests_properties(test_pub_sub_${TEST_NAME_PUB_SUB} PROPERTIES 296 | RUN_SERIAL TRUE 297 | LABELS "performance" 298 | ) 299 | endforeach() 300 | 301 | endfunction() 302 | 303 | # 304 | # Add performance tests for all available RMW implementations and all sandalone 305 | # DDS implmenetations associated with those RMWs by invoking 306 | # :cmake:macro:`add_performance_test` on each of them. 307 | # 308 | # :param RMW_IMPLEMENTATIONS: List of RMW implementations to test. 309 | # :type RMW_IMPLEMENTATIONS: semicolon-separated string 310 | # 311 | # The following values, when defined prior to invocation, will augment the 312 | # behavior of the tests: 313 | # 314 | # - **PERF_TEST_COMM_TYPE_${RMW}** *(string)* – Specifies a standalone 315 | # ``COMM`` type that is associated with the given ``RMW``. If undefined or set 316 | # to an empty string, no standalone test is added. 317 | # - **PERF_TEST_SKIP[_${TYPE}]_${COMM_OR_RMW}[_${SYNC_MODE}[_${TOPIC_NAME}]]** 318 | # *(bool)* – Specifies that matching tests should be automatically skipped. 319 | # - **PERF_TEST_ENV[_${TYPE}]_${COMM_OR_RMW}[_${SYNC_MODE}[_${TOPIC_NAME}]]** 320 | # *(list of "key=value")* – Specifies additional environment variables to 321 | # add to matching tests. 322 | # 323 | # Examples: 324 | # ``PERF_TEST_COMM_TYPE_rmw_cyclonedds_cpp=CycloneDDS`` 325 | # An additional standalone ``COMM`` type called ``CycloneDDS`` should be run 326 | # if ``rmw_cyclonedds_cpp`` is found and run. 327 | # ``PERF_TEST_SKIP_rmw_connext_cpp_sync=TRUE`` 328 | # Automatically skip all synchronous tests for ``rmw_cyclonedds_cpp``. 329 | # ``PERF_TEST_ENV_rmw_fastrtps_cpp_sync="RMW_FASTRTPS_USE_QOS_FROM_XML=1"`` 330 | # Set the environment variable ``RMW_FASTRTPS_USE_QOS_FROM_XML`` to ``1`` 331 | # whenever synchronous tests are run for ``rmw_fastrtps_cpp``. 332 | # 333 | function(add_performance_tests RMW_IMPLEMENTATIONS) 334 | foreach(RMW_IMPLEMENTATION ${RMW_IMPLEMENTATIONS}) 335 | foreach(PERF_TEST_SYNC "async" "sync") 336 | add_performance_test( 337 | ${RMW_IMPLEMENTATION} 338 | "ROS2" 339 | ${RMW_IMPLEMENTATION} 340 | ${PERF_TEST_SYNC} 341 | "${RMW_IMPLEMENTATIONS}" 342 | ) 343 | endforeach() 344 | 345 | if(PERF_TEST_COMM_TYPE_${RMW_IMPLEMENTATION}) 346 | foreach(PERF_TEST_SYNC "async" "sync") 347 | add_performance_test( 348 | ${PERF_TEST_COMM_TYPE_${RMW_IMPLEMENTATION}} 349 | ${PERF_TEST_COMM_TYPE_${RMW_IMPLEMENTATION}} 350 | ${RMW_IMPLEMENTATION} 351 | ${PERF_TEST_SYNC} 352 | "${RMW_IMPLEMENTATIONS}" 353 | ) 354 | endforeach() 355 | else() 356 | message(STATUS "No standalone DDS support for RMW: ${RMW_IMPLEMENTATION}") 357 | endif() 358 | endforeach() 359 | endfunction() 360 | -------------------------------------------------------------------------------- /test/test_performance.py.in: -------------------------------------------------------------------------------- 1 | # generated from buildfarm_perf_tests/test/test_performance.py.in 2 | # generated code does not contain a copyright notice 3 | 4 | from glob import glob 5 | import os 6 | import sys 7 | import tempfile 8 | import unittest 9 | 10 | from buildfarm_perf_tests.launch import assert_wait_for_successful_exit 11 | from buildfarm_perf_tests.test_results import read_performance_test_csv 12 | from buildfarm_perf_tests.test_results import write_jenkins_benchmark_json 13 | from buildfarm_perf_tests.test_results import write_jenkins_plot_csv 14 | from launch import LaunchDescription 15 | from launch_ros.actions import Node 16 | import launch_testing.markers 17 | 18 | import matplotlib # noqa: F401 19 | import matplotlib.pyplot as plt 20 | import numpy as np # noqa: F401 21 | import pandas as pd 22 | 23 | from rclpy import get_rmw_implementation_identifier 24 | 25 | plt.switch_backend('agg') 26 | 27 | BYTE_TO_MBYTE = 1 / (1024*1024) 28 | BYTE_TO_KBYTE = 1 / (1024) 29 | 30 | 31 | def _cleanUpLogs(log_pattern): 32 | for log in glob(log_pattern): 33 | os.remove(log) 34 | 35 | 36 | def _raw_to_jenkins(dataframe, csv_path): 37 | """ 38 | Convert from the raw csv data to formats consumble by Jenkins plugins. 39 | 40 | Do not change the order of the columns in the csv file. The plot plugin 41 | indexes into the csv using the column number instead of the column name, 42 | because we're using the columns to identify which test produced the data. 43 | 44 | Changing the column names here will change the name of the line that 45 | appears on the plot. 46 | """ 47 | dataframe_agg = dataframe.agg(['max', 'mean', 'sum', 'median']) 48 | 49 | values = [ 50 | dataframe_agg.loc['mean', 'latency_mean (ms)'], 51 | dataframe_agg.loc['median', 'latency_mean (ms)'], 52 | dataframe['latency_mean (ms)'].describe(percentiles=[0.95]).iloc[5], 53 | dataframe_agg.loc['max', 'ru_maxrss'], 54 | dataframe_agg.loc['mean', 'received'], 55 | dataframe_agg.loc['mean', 'sent'], 56 | dataframe_agg.loc['sum', 'lost'], 57 | dataframe_agg.loc['mean', 'cpu_usage (%)'], 58 | dataframe['cpu_usage (%)'].describe(percentiles=[0.95]).iloc[5], 59 | dataframe_agg.loc['median', 'cpu_usage (%)'], 60 | dataframe_agg.loc['mean', 'data_received'] * BYTE_TO_MBYTE, 61 | dataframe_agg.loc['median', 'data_received'] * BYTE_TO_MBYTE, 62 | dataframe['data_received'].describe(percentiles=[0.95]).iloc[5] * BYTE_TO_MBYTE, 63 | ] 64 | 65 | write_jenkins_plot_csv(csv_path, '@TEST_NAME@_@PERF_TEST_TOPIC@', values) 66 | 67 | json_path = os.path.splitext(csv_path)[0] + '.benchmark.json' 68 | json_values = { 69 | 'parameters': { 70 | 'runtime': { 71 | 'value': '@PERF_TEST_RUNTIME@', 72 | 'unit': 's', 73 | }, 74 | 'process_count': { 75 | 'value': '@NUMBER_PROCESS@', 76 | }, 77 | 'message_type': { 78 | 'value': '@PERF_TEST_TOPIC@', 79 | }, 80 | }, 81 | 'average_single_trip_time': { 82 | 'dblValue': values[0], 83 | 'unit': 'ms', 84 | }, 85 | 'throughput': { 86 | 'dblValue': values[10], 87 | 'unit': 'Mbit/s', 88 | }, 89 | 'max_resident_set_size': { 90 | 'dblValue': values[3], 91 | 'unit': 'MB', 92 | }, 93 | 'received_messages': { 94 | 'dblValue': values[4], 95 | }, 96 | 'sent_messages': { 97 | 'dblValue': values[5], 98 | }, 99 | 'lost_messages': { 100 | 'intValue': values[6], 101 | }, 102 | 'cpu_usage': { 103 | 'dblValue': values[8], 104 | 'unit': 'percent', 105 | }, 106 | } 107 | 108 | group_name = 'buildfarm_perf_tests.performance' 109 | if '@NUMBER_PROCESS@' == '2': 110 | group_name += '_two_process' 111 | 112 | write_jenkins_benchmark_json( 113 | json_path, 114 | group_name, 115 | {'@TEST_NAME@_@PERF_TEST_TOPIC@': json_values}) 116 | 117 | 118 | def _raw_to_png(dataframe, png_path): 119 | pd.options.display.float_format = '{:.4f}'.format 120 | 121 | dataframe = dataframe.copy() 122 | 123 | dataframe['maxrss (Mb)'] = dataframe['ru_maxrss'] * BYTE_TO_KBYTE 124 | dataframe.drop(list(dataframe.filter(regex='ru_')), axis=1, inplace=True) 125 | dataframe['latency_variance (ms) * 100'] = 100.0 * dataframe['latency_variance (ms)'] 126 | percentils_latency = dataframe['latency_mean (ms)'].describe(percentiles=[0.95]) 127 | dataframe['latency_p95 (ms)'] = percentils_latency.iloc[5] 128 | ax = dataframe[['T_experiment', 129 | 'latency_min (ms)', 130 | 'latency_max (ms)', 131 | 'latency_mean (ms)', 132 | 'latency_p95 (ms)', 133 | 'latency_variance (ms) * 100', 134 | 'maxrss (Mb)']].plot(x='T_experiment', secondary_y=['maxrss (Mb)']) 135 | 136 | plot_name = 'One Process' 137 | 138 | if '@NUMBER_PROCESS@' == '2': 139 | plot_name = 'Two Processes' 140 | 141 | plt.title('Performance ' + plot_name + 142 | ' tests latency\n@TEST_NAME@ @SYNC_MODE@ @PERF_TEST_TOPIC@') 143 | ax.set_ylabel('ms') 144 | plt.savefig(png_path) 145 | 146 | dataframe['data_received Mbits/s'] = dataframe['data_received'] * BYTE_TO_MBYTE 147 | percentils_data_received = dataframe['data_received Mbits/s'].describe(percentiles=[0.95]) 148 | dataframe['data_received Mbits/s (P95)'] = percentils_data_received.iloc[5] 149 | ax = dataframe[['T_experiment', 150 | 'data_received Mbits/s', 151 | 'data_received Mbits/s (P95)']].plot(x='T_experiment') 152 | plt.title('Performance ' + plot_name 153 | + ' Throughput (Mbits/s) Tests\n@TEST_NAME@ @SYNC_MODE@ @PERF_TEST_TOPIC@') 154 | ax.set_ylabel('Mbits/s') 155 | plt.savefig(png_path[:-4] + '_throughput.png') 156 | 157 | ax = dataframe[['T_experiment', 'cpu_usage (%)']].plot(x='T_experiment') 158 | plt.title('Performance ' + plot_name 159 | + ' tests CPU usage (%)\n@TEST_NAME@ @SYNC_MODE@ @PERF_TEST_TOPIC@') 160 | ax.set_ylabel('%') 161 | plt.savefig(png_path[:-4] + '_cpu_usage.png') 162 | 163 | ax = dataframe.plot(kind='bar', y=['received', 'sent', 'lost']) 164 | plt.title(plot_name 165 | + ' Received/Sent messages per second and Lost messages\n' 166 | + '@TEST_NAME@ @SYNC_MODE@ @PERF_TEST_TOPIC@') 167 | ax.set_ylabel('Number of messages') 168 | plt.savefig(png_path[:-4] + '_histogram.png') 169 | 170 | 171 | @launch_testing.markers.keep_alive 172 | def generate_test_description(): 173 | if '@COMM@' == 'ROS2': 174 | expected_rmw_implementation = '@RMW_IMPLEMENTATION@' 175 | rmw_implementation_identifier = get_rmw_implementation_identifier() 176 | assert rmw_implementation_identifier == expected_rmw_implementation, \ 177 | f'{rmw_implementation_identifier} != {expected_rmw_implementation}' 178 | 179 | performance_log_prefix = tempfile.mkstemp( 180 | prefix='performance_test_@TEST_NAME@_', text=True)[1] 181 | 182 | # perf_test will create its own logs with this prefix 183 | # we just need the prefix to be unique 184 | os.remove(performance_log_prefix) 185 | 186 | arguments = [ 187 | '-c', '@COMM@', 188 | '-t', '@PERF_TEST_TOPIC@', 189 | '--max_runtime', '@PERF_TEST_RUNTIME@', 190 | '--ignore', '3', 191 | '--keep_last', 192 | '--history_depth', '10', 193 | ] + ([ 194 | '--disable-async', 195 | ] if '@SYNC_MODE@' == 'sync' and '@COMM@' != 'ROS2' else []) 196 | 197 | nodes = [] 198 | 199 | if @NUMBER_PROCESS@ == 2: 200 | # Publisher node. It has 0 subscribers 201 | node_pub = Node( 202 | package='performance_test', 203 | executable='perf_test', 204 | output='log', 205 | arguments=arguments + [ 206 | '-s', '0', 207 | ], 208 | ) 209 | nodes.append(node_pub) 210 | 211 | # Add 0-publishers flag to arguments for node_under_test, which in the 212 | # case of 2 processes will be a subscriber. 213 | arguments += ['-p', '0'] 214 | 215 | # node_under_test is the node in charge of logging the results, since the 216 | # measurements are always taken in the subscriber. This node behaves 217 | # differently depending on the number of processes involved: 218 | # - 1 process: The node has a publisher and a subscriber 219 | # - 2 processes: The node only has a subscriber 220 | node_under_test = Node( 221 | package='performance_test', 222 | executable='perf_test', 223 | output='log', 224 | arguments=arguments + [ 225 | '-l', performance_log_prefix, 226 | ], 227 | ) 228 | nodes.append(node_under_test) 229 | 230 | return ( 231 | LaunchDescription(nodes + [launch_testing.actions.ReadyToTest()]), 232 | locals()) 233 | 234 | 235 | class PerformanceTestResults(unittest.TestCase): 236 | 237 | def test_results_@TEST_NAME@( 238 | self, proc_info, nodes, performance_log_prefix): 239 | self.addCleanup(_cleanUpLogs, performance_log_prefix + '*') 240 | 241 | timeout = @PERF_TEST_RUNTIME@ * 2 242 | for process in nodes: 243 | assert_wait_for_successful_exit(proc_info, process, timeout) 244 | 245 | results_base_path = os.environ.get('PERF_TEST_RESULTS_BASE_PATH') 246 | if results_base_path: 247 | performance_logs = glob(performance_log_prefix + '*') 248 | assert len(performance_logs) == 1 249 | performance_data = read_performance_test_csv(performance_logs[0]) 250 | _raw_to_jenkins(performance_data, results_base_path + '.csv') 251 | _raw_to_png(performance_data, results_base_path + '.png') 252 | else: 253 | print( 254 | 'No results reports written - set PERF_TEST_RESULTS_BASE_PATH to write reports', 255 | file=sys.stderr) 256 | -------------------------------------------------------------------------------- /test/test_pub_sub.py.in: -------------------------------------------------------------------------------- 1 | # generated from buildfarm_perf_tests/test/test_pub_sub.py.in 2 | # generated code does not contain a copyright notice 3 | 4 | from glob import glob 5 | import os 6 | import sys 7 | import tempfile 8 | import unittest 9 | 10 | from buildfarm_perf_tests.launch import assert_wait_for_successful_exit 11 | from buildfarm_perf_tests.launch import SystemMetricCollector 12 | from buildfarm_perf_tests.test_results import read_performance_test_csv 13 | from buildfarm_perf_tests.test_results import write_jenkins_benchmark_json 14 | from buildfarm_perf_tests.test_results import write_jenkins_plot_csv 15 | from launch import LaunchDescription 16 | from launch_ros.actions import Node 17 | import launch_testing.markers 18 | 19 | import matplotlib # noqa: F401 20 | import matplotlib.pyplot as plt 21 | import numpy as np # noqa: F401 22 | import pandas as pd 23 | 24 | plt.switch_backend('agg') 25 | 26 | 27 | def _cleanUpLogs(log_pattern): 28 | for log in glob(log_pattern): 29 | os.remove(log) 30 | 31 | 32 | def _raw_to_jenkins(dataframe, dataframe_perf, csv_path, mode): 33 | """ 34 | Convert from the raw csv data to formats consumble by Jenkins plugins. 35 | 36 | Do not change the order of the columns in the csv file. The plot plugin 37 | indexes into the csv using the column number instead of the column name, 38 | because we're using the columns to identify which test produced the data. 39 | 40 | Changing the column names here will change the name of the line that 41 | appears on the plot. 42 | """ 43 | dataframe_agg = dataframe.agg(['median', 'mean']) 44 | dataframe_agg_perf = dataframe_perf.agg(['max', 'mean', 'sum', 'median']) 45 | 46 | values = [ 47 | dataframe_agg.loc['mean', 'virtual memory (Mb)'], 48 | dataframe_agg.loc['median', 'virtual memory (Mb)'], 49 | dataframe['virtual memory (Mb)'].describe(percentiles=[0.95]).iloc[5], 50 | dataframe_agg.loc['mean', 'cpu_usage (%)'], 51 | dataframe_agg.loc['median', 'cpu_usage (%)'], 52 | dataframe['cpu_usage (%)'].describe(percentiles=[0.95]).iloc[5], 53 | dataframe_agg.loc['mean', 'physical memory (Mb)'], 54 | dataframe_agg.loc['median', 'physical memory (Mb)'], 55 | dataframe['physical memory (Mb)'].describe(percentiles=[0.95]).iloc[5], 56 | dataframe_agg.loc['mean', 'resident anonymous memory (Mb)'], 57 | dataframe_agg.loc['median', 'resident anonymous memory (Mb)'], 58 | dataframe['resident anonymous memory (Mb)'].describe(percentiles=[0.95]).iloc[5], 59 | dataframe_agg_perf.loc['mean', 'latency_mean (ms)'], 60 | dataframe_agg_perf.loc['median', 'latency_mean (ms)'], 61 | dataframe_agg_perf['latency_mean (ms)'].describe(percentiles=[0.95]).iloc[5], 62 | dataframe_agg_perf.loc['max', 'ru_maxrss'], 63 | dataframe_agg_perf.loc['mean', 'received'], 64 | dataframe_agg_perf.loc['mean', 'sent'], 65 | dataframe_agg_perf.loc['sum', 'lost'], 66 | dataframe_agg.loc['mean', 'system_cpu_usage (%)'], 67 | dataframe_agg.loc['mean', 'system virtual memory (Mb)'], 68 | ] 69 | 70 | full_csv_path = '%s_%s.csv' % (csv_path[:-4], mode) 71 | write_jenkins_plot_csv( 72 | full_csv_path, 73 | 'Publisher-@TEST_NAME@_Subscriber-@RMW_IMPLEMENTATION_SUB@', values) 74 | 75 | json_path = os.path.splitext(full_csv_path)[0] + '.benchmark.json' 76 | json_values = { 77 | 'parameters': { 78 | 'runtime': { 79 | 'value': '@PERF_TEST_RUNTIME@', 80 | 'unit': 's', 81 | }, 82 | 'process_count': { 83 | 'value': '2', 84 | }, 85 | 'message_type': { 86 | 'value': '@PERF_TEST_TOPIC@', 87 | }, 88 | }, 89 | 'cpu_usage_' + mode: { 90 | 'dblValue': values[5], 91 | 'unit': 'percent', 92 | }, 93 | 'memory_physical_' + mode: { 94 | 'dblValue': values[8], 95 | 'unit': 'MB', 96 | }, 97 | 'resident_anonymous_memory_' + mode: { 98 | 'dblValue': values[11], 99 | 'unit': 'MB', 100 | }, 101 | 'virtual_memory_' + mode: { 102 | 'dblValue': values[2], 103 | 'unit': 'MB', 104 | }, 105 | } 106 | 107 | if mode == 'pub': 108 | json_values.update({ 109 | 'lost_messages': { 110 | 'intValue': values[18], 111 | }, 112 | 'received_messages': { 113 | 'dblValue': values[16], 114 | 'unit': 'msg/s', 115 | }, 116 | 'sent_messages': { 117 | 'dblValue': values[17], 118 | 'unit': 'msg/s', 119 | }, 120 | 'average_round_trip_time': { 121 | 'dblValue': values[14], 122 | 'unit': 'ms', 123 | }, 124 | }) 125 | 126 | write_jenkins_benchmark_json( 127 | json_path, 128 | 'buildfarm_perf_tests.pub_sub', 129 | {'Publisher-@TEST_NAME@_Subscriber-@RMW_IMPLEMENTATION_SUB@': json_values}) 130 | 131 | 132 | def _raw_to_png(dataframe, dataframe_perf, png_path, mode, rmw_impl): 133 | pd.options.display.float_format = '{:.4f}'.format 134 | 135 | dataframe[['T_experiment', 'virtual memory (Mb)']].plot(x='T_experiment') 136 | plt.title('Simple Pub/Sub virtual memory usage\n' + mode + ': ' + rmw_impl) 137 | plt.ylim(0, 1024) 138 | plt.savefig(png_path + '_' 139 | + 'Publisher-@RMW_IMPLEMENTATION@_Subscriber-@RMW_IMPLEMENTATION_SUB@_' 140 | + mode + '_virtual_memory.png') 141 | 142 | dataframe[['T_experiment', 'cpu_usage (%)']].plot(x='T_experiment') 143 | plt.title('Simple Pub/Sub CPU usage\n' + mode + ': ' + rmw_impl) 144 | plt.ylim(0, 100) 145 | plt.savefig(png_path + '_' 146 | + 'Publisher-@RMW_IMPLEMENTATION@_Subscriber-@RMW_IMPLEMENTATION_SUB@_' 147 | + mode + '_cpu_usage.png') 148 | 149 | dataframe[['T_experiment', 'physical memory (Mb)']].plot(x='T_experiment') 150 | plt.title('Simple Pub/Sub physical memory usage\n' + mode + ': ' + rmw_impl) 151 | plt.ylim(0, 100) 152 | plt.savefig(png_path + '_' 153 | + 'Publisher-@RMW_IMPLEMENTATION@_Subscriber-@RMW_IMPLEMENTATION_SUB@_' 154 | + mode + '_physical_memory.png') 155 | 156 | dataframe[['T_experiment', 'resident anonymous memory (Mb)']].plot(x='T_experiment') 157 | plt.title('Simple Pub/Sub resident anonymous memory\n' + mode + ': ' + rmw_impl) 158 | plt.ylim(0, 100) 159 | plt.savefig(png_path + '_' 160 | + 'Publisher-@RMW_IMPLEMENTATION@_Subscriber-@RMW_IMPLEMENTATION_SUB@_' 161 | + mode + '_resident_anonymous_memory.png') 162 | 163 | if mode == 'subscriber': 164 | return 165 | 166 | dataframe = dataframe_perf.copy() 167 | pd.options.display.float_format = '{:.4f}'.format 168 | dataframe.plot(kind='bar', y=['received', 'sent', 'lost']) 169 | plt.title('Simple Pub/Sub Received/Sent/Lost messages\n' + mode + ': ' + rmw_impl) 170 | plt.savefig(png_path + '_' 171 | + 'Publisher-@RMW_IMPLEMENTATION@_Subscriber-@RMW_IMPLEMENTATION_SUB@_' 172 | + mode + '_histogram.png') 173 | 174 | dataframe['maxrss (Mb)'] = dataframe['ru_maxrss'] / 1e3 175 | dataframe.drop(list(dataframe.filter(regex='ru_')), axis=1, inplace=True) 176 | dataframe['latency_variance (ms) * 100'] = 100.0 * dataframe['latency_variance (ms)'] 177 | dataframe[['T_experiment', 178 | 'latency_min (ms)', 179 | 'latency_max (ms)', 180 | 'latency_mean (ms)', 181 | 'latency_variance (ms) * 100', 182 | 'maxrss (Mb)']].plot(x='T_experiment', secondary_y=['maxrss (Mb)']) 183 | plt.title('Simple Pub/Sub latency\n' + mode + ': ' + '@RMW_IMPLEMENTATION@' 184 | + '\nsubscriber: ' + '@RMW_IMPLEMENTATION_SUB@') 185 | plt.savefig(png_path + '_' 186 | + 'Publisher-@RMW_IMPLEMENTATION@_Subscriber-@RMW_IMPLEMENTATION_SUB@_' 187 | + mode + '_latency.png') 188 | 189 | 190 | @launch_testing.markers.keep_alive 191 | def generate_test_description(): 192 | performance_log_cpumem_pub = tempfile.mkstemp( 193 | prefix='overhead_cpumem_pub_test_results_' 194 | + '@TEST_NAME_PUB_SUB@_', 195 | suffix='.csv', text=True)[1] 196 | performance_log_cpumem_sub = tempfile.mkstemp( 197 | prefix='overhead_cpumem_sub_test_results_' 198 | + '@TEST_NAME_PUB_SUB@_', 199 | suffix='.csv', text=True)[1] 200 | performance_log_prefix_pub = tempfile.mkstemp( 201 | prefix='overhead_pub_test_results_' 202 | + '@TEST_NAME_PUB_SUB@_', 203 | text=True)[1] 204 | performance_log_prefix_sub = tempfile.mkstemp( 205 | prefix='overhead_sub_test_results_' 206 | + '@TEST_NAME_PUB_SUB@_', 207 | text=True)[1] 208 | 209 | # perf_test will create its own logs with this prefix 210 | # we just need the prefix to be unique 211 | os.remove(performance_log_prefix_pub) 212 | os.remove(performance_log_prefix_sub) 213 | 214 | node_main_test = Node( 215 | package='performance_test', executable='perf_test', output='log', 216 | arguments=[ 217 | '-c', 'ROS2', 218 | '-t', '@PERF_TEST_TOPIC@', 219 | '--max_runtime', '@PERF_TEST_RUNTIME@', 220 | '-l', performance_log_prefix_pub, 221 | '--rate', '5', 222 | '--roundtrip_mode', 'Main', 223 | '--ignore', '3', 224 | ] + ([ 225 | '--disable-async', 226 | ] if '@SYNC_MODE@' == 'sync' and '@COMM@' != 'ROS2' else []), 227 | ) 228 | 229 | system_metric_collector_pub = SystemMetricCollector( 230 | target_action=node_main_test, 231 | log_file=performance_log_cpumem_pub, 232 | timeout=int(@PERF_TEST_RUNTIME@ / (1 / 1.5))) 233 | 234 | node_relay_test = Node( 235 | package='performance_test', executable='perf_test', output='log', 236 | arguments=[ 237 | '-c', 'ROS2', 238 | '-t', '@PERF_TEST_TOPIC@', 239 | '--max_runtime', '@PERF_TEST_RUNTIME@', 240 | '-l', performance_log_prefix_sub, 241 | '--rate', '5', 242 | '--roundtrip_mode', 'Relay', 243 | '--ignore', '3', 244 | ] + ([ 245 | '--disable-async', 246 | ] if '@SYNC_MODE@' == 'sync' and '@COMM@' != 'ROS2' else []), 247 | additional_env={'RMW_IMPLEMENTATION': '@RMW_IMPLEMENTATION_SUB@'}, 248 | ) 249 | 250 | system_metric_collector_sub = SystemMetricCollector( 251 | target_action=node_relay_test, 252 | log_file=performance_log_cpumem_sub, 253 | timeout=int(@PERF_TEST_RUNTIME@ / (1 / 1.5))) 254 | 255 | nodes = [ 256 | system_metric_collector_pub, 257 | system_metric_collector_sub, 258 | node_main_test, 259 | node_relay_test, 260 | ] 261 | 262 | return ( 263 | LaunchDescription(nodes + [launch_testing.actions.ReadyToTest()]), 264 | locals()) 265 | 266 | 267 | class PerformanceTestResults(unittest.TestCase): 268 | 269 | def test_results_@TEST_NAME_PUB_SUB@( 270 | self, proc_info, nodes, 271 | performance_log_cpumem_pub, performance_log_cpumem_sub, 272 | performance_log_prefix_pub, performance_log_prefix_sub): 273 | self.addCleanup(_cleanUpLogs, performance_log_prefix_pub + '*') 274 | self.addCleanup(_cleanUpLogs, performance_log_prefix_sub + '*') 275 | self.addCleanup(_cleanUpLogs, performance_log_cpumem_pub) 276 | self.addCleanup(_cleanUpLogs, performance_log_cpumem_sub) 277 | 278 | timeout = @PERF_TEST_RUNTIME@ * 2 279 | for process in nodes: 280 | assert_wait_for_successful_exit(proc_info, process, timeout) 281 | 282 | results_base_path = os.environ.get('PERF_TEST_RESULTS_BASE_PATH') 283 | performance_overhead_png = os.environ.get('PERFORMANCE_OVERHEAD_PNG') 284 | if results_base_path: 285 | performance_logs = glob(performance_log_prefix_pub + '*') 286 | assert len(performance_logs) == 1 287 | performance_data = read_performance_test_csv(performance_logs[0]) 288 | performance_data_cpumem = read_performance_test_csv( 289 | performance_log_cpumem_pub, start_marker=None) 290 | _raw_to_jenkins( 291 | performance_data_cpumem, performance_data, 292 | results_base_path + '.csv', 'pub') 293 | if performance_overhead_png: 294 | _raw_to_png( 295 | performance_data_cpumem, performance_data, 296 | performance_overhead_png, 'publisher', '@RMW_IMPLEMENTATION@') 297 | 298 | performance_logs = glob(performance_log_prefix_sub + '*') 299 | assert len(performance_logs) == 1 300 | performance_data = read_performance_test_csv(performance_logs[0]) 301 | performance_data_cpumem = read_performance_test_csv( 302 | performance_log_cpumem_sub, start_marker=None) 303 | _raw_to_jenkins( 304 | performance_data_cpumem, performance_data, 305 | results_base_path + '.csv', 'sub') 306 | if performance_overhead_png: 307 | _raw_to_png( 308 | performance_data_cpumem, performance_data, 309 | performance_overhead_png, 'subscriber', '@RMW_IMPLEMENTATION_SUB@') 310 | else: 311 | print( 312 | 'No results reports written - set PERF_TEST_RESULTS_BASE_PATH to write reports', 313 | file=sys.stderr) 314 | -------------------------------------------------------------------------------- /test/test_spinning.py.in: -------------------------------------------------------------------------------- 1 | # generated from buildfarm_perf_tests/test/test_spinning.py.in 2 | # generated code does not contain a copyright notice 3 | 4 | from glob import glob 5 | import os 6 | import sys 7 | import tempfile 8 | import unittest 9 | 10 | from buildfarm_perf_tests.launch import assert_wait_for_successful_exit 11 | from buildfarm_perf_tests.launch import SystemMetricCollector 12 | from buildfarm_perf_tests.test_results import read_performance_test_csv 13 | from buildfarm_perf_tests.test_results import write_jenkins_benchmark_json 14 | from buildfarm_perf_tests.test_results import write_jenkins_plot_csv 15 | from launch import LaunchDescription 16 | from launch.actions import EmitEvent 17 | from launch.actions import RegisterEventHandler 18 | from launch.actions import TimerAction 19 | from launch.event_handlers import OnProcessStart 20 | from launch.events import matches_action 21 | from launch.events.process import ShutdownProcess 22 | from launch_ros.actions import Node 23 | import launch_testing.markers 24 | 25 | import matplotlib # noqa: F401 26 | import matplotlib.pyplot as plt 27 | import numpy as np # noqa: F401 28 | import pandas as pd 29 | 30 | plt.switch_backend('agg') 31 | 32 | 33 | def _cleanUpLogs(log_pattern): 34 | for log in glob(log_pattern): 35 | os.remove(log) 36 | 37 | 38 | def _raw_to_jenkins(dataframe, csv_path): 39 | """ 40 | Convert from the raw csv data to formats consumble by Jenkins plugins. 41 | 42 | Do not change the order of the columns in the csv file. The plot plugin 43 | indexes into the csv using the column number instead of the column name, 44 | because we're using the columns to identify which test produced the data. 45 | 46 | Changing the column names here will change the name of the line that 47 | appears on the plot. 48 | """ 49 | dataframe_agg = dataframe.agg(['median', 'mean']) 50 | 51 | values = [ 52 | dataframe_agg.loc['mean', 'virtual memory (Mb)'], 53 | dataframe_agg.loc['median', 'virtual memory (Mb)'], 54 | dataframe['virtual memory (Mb)'].describe(percentiles=[0.95]).iloc[5], 55 | dataframe_agg.loc['mean', 'cpu_usage (%)'], 56 | dataframe_agg.loc['median', 'cpu_usage (%)'], 57 | dataframe['cpu_usage (%)'].describe(percentiles=[0.95]).iloc[5], 58 | dataframe_agg.loc['mean', 'physical memory (Mb)'], 59 | dataframe_agg.loc['median', 'physical memory (Mb)'], 60 | dataframe['physical memory (Mb)'].describe(percentiles=[0.95]).iloc[5], 61 | dataframe_agg.loc['mean', 'resident anonymous memory (Mb)'], 62 | dataframe_agg.loc['median', 'resident anonymous memory (Mb)'], 63 | dataframe['resident anonymous memory (Mb)'].describe(percentiles=[0.95]).iloc[5], 64 | ] 65 | 66 | write_jenkins_plot_csv(csv_path, '@TEST_NAME@', values) 67 | 68 | json_path = os.path.splitext(csv_path)[0] + '.benchmark.json' 69 | json_values = { 70 | 'parameters': { 71 | 'runtime': { 72 | 'value': '@PERF_TEST_RUNTIME@', 73 | 'unit': 's', 74 | }, 75 | 'process_count': { 76 | 'value': '1', 77 | }, 78 | }, 79 | 'virtual_memory': { 80 | 'dblValue': values[2], 81 | 'unit': 'MB', 82 | }, 83 | 'cpu_usage': { 84 | 'dblValue': values[5], 85 | 'unit': 'percent', 86 | }, 87 | 'physical_memory': { 88 | 'dblValue': values[8], 89 | 'unit': 'MB', 90 | }, 91 | 'resident_anonymous_memory': { 92 | 'dblValue': values[11], 93 | 'unit': 'MB', 94 | }, 95 | } 96 | 97 | write_jenkins_benchmark_json( 98 | json_path, 99 | 'buildfarm_perf_tests.spinning', 100 | {'@TEST_NAME@': json_values}) 101 | 102 | 103 | def _raw_to_png(dataframe, png_path): 104 | png_base, png_ext = os.path.splitext(png_path) 105 | pd.options.display.float_format = '{:.4f}'.format 106 | dataframe[['T_experiment', 'virtual memory (Mb)']].plot(x='T_experiment') 107 | 108 | plt.title('Node spinning virtual memory usage\n@TEST_NAME@') 109 | plt.ylim(0, 1024) 110 | plt.savefig(png_base + '_virtual_memory' + png_ext) 111 | 112 | dataframe[['T_experiment', 'cpu_usage (%)']].plot(x='T_experiment') 113 | plt.title('Node spinning CPU usage\n@TEST_NAME@') 114 | plt.ylim(0, 100) 115 | plt.savefig(png_base + '_cpu_usage' + png_ext) 116 | 117 | dataframe[['T_experiment', 'physical memory (Mb)']].plot(x='T_experiment') 118 | plt.ylim(0, 100) 119 | plt.title('Node spinning physical memory usage\n@TEST_NAME@') 120 | plt.savefig(png_base + '_physical_memory' + png_ext) 121 | 122 | dataframe[['T_experiment', 'resident anonymous memory (Mb)']].plot(x='T_experiment') 123 | plt.title('Node spinning resident anonymous memory\n@TEST_NAME@') 124 | plt.ylim(0, 100) 125 | plt.savefig(png_base + '_resident_anonymous_memory' + png_ext) 126 | 127 | 128 | @launch_testing.markers.keep_alive 129 | def generate_test_description(): 130 | performance_log_cpumem = tempfile.mkstemp( 131 | prefix='overhead_node_test_results_@TEST_NAME@_', 132 | suffix='.csv', text=True)[1] 133 | 134 | node_spinning_test = Node( 135 | package='buildfarm_perf_tests', 136 | executable='node_spinning', 137 | output='log', 138 | ) 139 | node_spinning_timer = RegisterEventHandler(OnProcessStart( 140 | target_action=node_spinning_test, on_start=TimerAction( 141 | period=float(@PERF_TEST_RUNTIME@), actions=[ 142 | EmitEvent( 143 | event=ShutdownProcess( 144 | process_matcher=matches_action(node_spinning_test))) 145 | ]))) 146 | 147 | node_metrics_collector = SystemMetricCollector( 148 | target_action=node_spinning_test, 149 | log_file=performance_log_cpumem, 150 | timeout=int(@PERF_TEST_RUNTIME@ / (1 / 1.5))) 151 | 152 | nodes = [ 153 | node_metrics_collector, 154 | node_spinning_test, 155 | ] 156 | 157 | return LaunchDescription(nodes + [ 158 | node_spinning_timer, 159 | launch_testing.actions.ReadyToTest()]), locals() 160 | 161 | 162 | class NodeSpinningTestResults(unittest.TestCase): 163 | 164 | def test_results_@TEST_NAME@( 165 | self, proc_info, nodes, performance_log_cpumem): 166 | self.addCleanup(_cleanUpLogs, performance_log_cpumem) 167 | 168 | timeout = @PERF_TEST_RUNTIME@ * 2 169 | for process in nodes: 170 | assert_wait_for_successful_exit(proc_info, process, timeout) 171 | 172 | results_base_path = os.environ.get('PERF_TEST_RESULTS_BASE_PATH') 173 | if results_base_path: 174 | performance_data_cpumem = read_performance_test_csv( 175 | performance_log_cpumem, start_marker=None) 176 | _raw_to_jenkins(performance_data_cpumem, results_base_path + '.csv') 177 | _raw_to_png(performance_data_cpumem, results_base_path + '.png') 178 | else: 179 | print( 180 | 'No results reports written - set PERF_TEST_RESULTS_BASE_PATH to write reports', 181 | file=sys.stderr) 182 | -------------------------------------------------------------------------------- /tools/ros2_dependencies.repos: -------------------------------------------------------------------------------- 1 | repositories: 2 | buildfarm_perf_tests: 3 | type: git 4 | url: https://github.com/ros2/buildfarm_perf_tests.git 5 | version: master 6 | performance_test: 7 | type: git 8 | url: https://github.com/ros2/performance_test.git 9 | version: run_on_buildfarm 10 | --------------------------------------------------------------------------------