├── .gitignore ├── VSOpenFileFromDirFilters.json ├── test ├── README.md ├── multi_cpp │ ├── picobench_configured.hpp │ ├── suite_a.cpp │ ├── suite_b.cpp │ ├── suite_b_cont.cpp │ └── main.cpp ├── CMakeLists.txt ├── get_cpm.cmake └── basic.cpp ├── tools ├── CMakeLists.txt ├── README.md └── picobench.cpp ├── example ├── CMakeLists.txt ├── basic.cpp └── locks.cpp ├── LICENSE.txt ├── dev.cmake ├── CMakeLists.txt ├── .github └── workflows │ └── test.yml ├── README.md └── include └── picobench └── picobench.hpp /.gitignore: -------------------------------------------------------------------------------- 1 | build/ 2 | out/ 3 | 4 | # ides 5 | .vs/ 6 | .vscode/ 7 | -------------------------------------------------------------------------------- /VSOpenFileFromDirFilters.json: -------------------------------------------------------------------------------- 1 | { 2 | "dirs": [ 3 | ".git", 4 | ".vs", 5 | "build", 6 | "out" 7 | ] 8 | } 9 | -------------------------------------------------------------------------------- /test/README.md: -------------------------------------------------------------------------------- 1 | ## picobench Unit Tests 2 | 3 | The unit tests use [doctest](https://github.com/onqtam/doctest) which is a submodule of this repo. -------------------------------------------------------------------------------- /test/multi_cpp/picobench_configured.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #define PICOBENCH_DEBUG 4 | #define PICOBENCH_TEST 5 | #define PICOBENCH_STD_FUNCTION_BENCHMARKS 6 | #define PICOBENCH_NAMESPACE pb 7 | #include 8 | -------------------------------------------------------------------------------- /tools/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.5) 2 | 3 | project(picobench-tools) 4 | 5 | if(MSVC) 6 | add_definitions(-D_CRT_SECURE_NO_WARNINGS=1) 7 | endif() 8 | 9 | add_executable(picobench-cli picobench.cpp) 10 | target_link_libraries(picobench-cli picobench) 11 | set_target_properties(picobench-cli PROPERTIES OUTPUT_NAME picobench) 12 | set_target_properties(picobench-cli PROPERTIES FOLDER tools) 13 | -------------------------------------------------------------------------------- /test/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.14 FATAL_ERROR) 2 | 3 | include(./get_cpm.cmake) 4 | CPMAddPackage(gh:iboB/doctest-util@0.1.2) 5 | 6 | macro(pb_test test) 7 | add_doctest_lib_test(${test} picobench ${ARGN}) 8 | endmacro() 9 | 10 | pb_test(basic basic.cpp) 11 | pb_test(multi_cpp 12 | multi_cpp/main.cpp 13 | multi_cpp/suite_a.cpp 14 | multi_cpp/suite_b.cpp 15 | multi_cpp/suite_b_cont.cpp 16 | ) 17 | -------------------------------------------------------------------------------- /example/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | macro(pb_example name) 2 | set(tgt picobench-example-${name}) 3 | add_executable(${tgt} ${ARGN}) 4 | target_link_libraries(${tgt} picobench::picobench) 5 | set_target_properties(${tgt} PROPERTIES FOLDER example) 6 | add_custom_target( 7 | picobench-run-example-${name} 8 | COMMAND ${tgt} 9 | ) 10 | endmacro() 11 | 12 | pb_example(basic basic.cpp) 13 | pb_example(locks locks.cpp) 14 | -------------------------------------------------------------------------------- /test/multi_cpp/suite_a.cpp: -------------------------------------------------------------------------------- 1 | #include "picobench_configured.hpp" 2 | 3 | PICOBENCH_SUITE("suite a"); 4 | 5 | static void a_a(pb::state& s) 6 | { 7 | for (auto _ : s) 8 | { 9 | pb::test::this_thread_sleep_for_ns(10); 10 | } 11 | } 12 | PICOBENCH(a_a); 13 | 14 | static void a_b(pb::state& s) 15 | { 16 | for (auto _ : s) 17 | { 18 | pb::test::this_thread_sleep_for_ns(15); 19 | } 20 | } 21 | PICOBENCH(a_b); 22 | -------------------------------------------------------------------------------- /test/multi_cpp/suite_b.cpp: -------------------------------------------------------------------------------- 1 | #include "picobench_configured.hpp" 2 | 3 | PICOBENCH_SUITE("suite b"); 4 | 5 | static void a_a(pb::state& s) 6 | { 7 | for (auto _ : s) 8 | { 9 | pb::test::this_thread_sleep_for_ns(15); 10 | } 11 | } 12 | PICOBENCH(a_a); 13 | 14 | static void a_b(size_t stime, pb::state& s) 15 | { 16 | s.start_timer(); 17 | pb::test::this_thread_sleep_for_ns(s.iterations() * size_t(stime)); 18 | s.stop_timer(); 19 | } 20 | PICOBENCH([](pb::state& s) { a_b(30, s); }).label("a_b").baseline(); 21 | -------------------------------------------------------------------------------- /test/multi_cpp/suite_b_cont.cpp: -------------------------------------------------------------------------------- 1 | #include "picobench_configured.hpp" 2 | #include 3 | 4 | extern std::map g_num_samples; 5 | 6 | PICOBENCH_SUITE("suite b"); 7 | 8 | static void b_a(pb::state& s) 9 | { 10 | ++g_num_samples[s.iterations()]; 11 | for (auto _ : s) 12 | { 13 | pb::test::this_thread_sleep_for_ns(20); 14 | } 15 | } 16 | PICOBENCH(b_a); 17 | 18 | static void b_b(pb::state& s) 19 | { 20 | s.start_timer(); 21 | pb::test::this_thread_sleep_for_ns(s.iterations() * size_t(25)); 22 | s.stop_timer(); 23 | } 24 | PICOBENCH(b_b); 25 | -------------------------------------------------------------------------------- /example/basic.cpp: -------------------------------------------------------------------------------- 1 | #define PICOBENCH_DEBUG 2 | #define PICOBENCH_IMPLEMENT_WITH_MAIN 3 | #include "picobench/picobench.hpp" 4 | 5 | #include 6 | #include 7 | #include 8 | 9 | void rand_vector(picobench::state& s) 10 | { 11 | std::vector v; 12 | for (auto _ : s) 13 | { 14 | v.push_back(rand()); 15 | } 16 | } 17 | PICOBENCH(rand_vector); 18 | 19 | void rand_vector_reserve(picobench::state& s) 20 | { 21 | std::vector v; 22 | v.reserve(s.iterations()); 23 | for (auto _ : s) 24 | { 25 | v.push_back(rand()); 26 | } 27 | } 28 | PICOBENCH(rand_vector_reserve); 29 | 30 | void rand_deque(picobench::state& s) 31 | { 32 | std::deque v; 33 | for (auto _ : s) 34 | { 35 | v.push_back(rand()); 36 | } 37 | } 38 | PICOBENCH(rand_deque); 39 | -------------------------------------------------------------------------------- /tools/README.md: -------------------------------------------------------------------------------- 1 | ## picobench Tools 2 | 3 | ### picobench.cpp 4 | 5 | An executable which allows the user to benchmark commands. 6 | 7 | Usage: 8 | 9 | `$ picobench [ ... ] [args]` 10 | 11 | The default number of iterations is one. And the default number of samples is two. 12 | 13 | It supports the command-line arguments for the picobench library plus an additional one: 14 | 15 | `--bfile=` - Sets a filename which lists the commands to test as benchmarks 16 | 17 | The benchmark file format is: 18 | 19 | ``` 20 | title for baseline 21 | command line for baseline 22 | 23 | title for other benchmark 1 24 | command file for benchmark 2 25 | 26 | [...] 27 | 28 | title for benchmark N 29 | command line for benchmark N 30 | ``` 31 | 32 | Empty lines are ignored. 33 | 34 | Examples: 35 | 36 | * `$ picobench "sleep 1" "sleep 1.2"` 37 | * `$ picobench --bfile=benchmarks.txt --samples=10 --output=data.csv --out-fmt=csv` 38 | 39 | 40 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2017-2025 Borislav Stanimirov 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. 20 | -------------------------------------------------------------------------------- /dev.cmake: -------------------------------------------------------------------------------- 1 | # Copyright (c) Borislav Stanimirov 2 | # SPDX-License-Identifier: MIT 3 | # 4 | set(CMAKE_CXX_STANDARD 11) 5 | set(CMAKE_CXX_EXTENSIONS OFF) 6 | set(CMAKE_CXX_STANDARD_REQUIRED ON) 7 | 8 | if(MSVC) 9 | set(icm_compiler_flags "-D_CRT_SECURE_NO_WARNINGS /Zc:__cplusplus /permissive-\ 10 | /w34100 /w34701 /w34702 /w34703 /w34706 /w34714 /w34913\ 11 | /wd4251 /wd4275" 12 | ) 13 | else() 14 | set(icm_compiler_flags "-Wall -Wextra") 15 | endif() 16 | 17 | if(SAN_ADDR) 18 | if(MSVC) 19 | set(icm_san_flags "/fsanitize=address") 20 | elseif(APPLE) 21 | # apple clang doesn't support the leak sanitizer 22 | set(icm_san_flags "-fsanitize=address,undefined -pthread -g") 23 | else() 24 | set(icm_san_flags "-fsanitize=address,undefined,leak -pthread -g") 25 | endif() 26 | endif() 27 | 28 | set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${icm_compiler_flags} ${icm_san_flags}") 29 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${icm_compiler_flags} ${icm_san_flags}") 30 | set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${icm_san_flags}") 31 | set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${icm_san_flags}") 32 | -------------------------------------------------------------------------------- /test/get_cpm.cmake: -------------------------------------------------------------------------------- 1 | set(CPM_DOWNLOAD_VERSION 0.38.1) 2 | 3 | if(CPM_SOURCE_CACHE) 4 | set(CPM_DOWNLOAD_LOCATION "${CPM_SOURCE_CACHE}/cpm/CPM_${CPM_DOWNLOAD_VERSION}.cmake") 5 | elseif(DEFINED ENV{CPM_SOURCE_CACHE}) 6 | set(CPM_DOWNLOAD_LOCATION "$ENV{CPM_SOURCE_CACHE}/cpm/CPM_${CPM_DOWNLOAD_VERSION}.cmake") 7 | else() 8 | set(CPM_DOWNLOAD_LOCATION "${CMAKE_BINARY_DIR}/cmake/CPM_${CPM_DOWNLOAD_VERSION}.cmake") 9 | endif() 10 | 11 | # Expand relative path. This is important if the provided path contains a tilde (~) 12 | get_filename_component(CPM_DOWNLOAD_LOCATION ${CPM_DOWNLOAD_LOCATION} ABSOLUTE) 13 | 14 | function(download_cpm) 15 | message(STATUS "Downloading CPM.cmake to ${CPM_DOWNLOAD_LOCATION}") 16 | file(DOWNLOAD 17 | https://github.com/cpm-cmake/CPM.cmake/releases/download/v${CPM_DOWNLOAD_VERSION}/CPM.cmake 18 | ${CPM_DOWNLOAD_LOCATION} 19 | ) 20 | endfunction() 21 | 22 | if(NOT (EXISTS ${CPM_DOWNLOAD_LOCATION})) 23 | download_cpm() 24 | else() 25 | # resume download if it previously failed 26 | file(READ ${CPM_DOWNLOAD_LOCATION} check) 27 | if("${check}" STREQUAL "") 28 | download_cpm() 29 | endif() 30 | unset(check) 31 | endif() 32 | 33 | include(${CPM_DOWNLOAD_LOCATION}) 34 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (c) Borislav Stanimirov 2 | # SPDX-License-Identifier: MIT 3 | # 4 | cmake_minimum_required(VERSION 3.10) 5 | 6 | project(picobench) 7 | 8 | if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) 9 | # dev_mode is used below to make life easier for developers 10 | # it enables some configurations and the defaults for building tests and 11 | # examples which typically wouldn't be built if xmem is a subdirectory of 12 | # another project 13 | set(dev_mode ON) 14 | else() 15 | set(dev_mode OFF) 16 | endif() 17 | 18 | option(PICOBENCH_BUILD_TOOLS "picobench: build tools" ${dev_mode}) 19 | option(PICOBENCH_BUILD_TESTS "picobench: build tests" ${dev_mode}) 20 | option(PICOBENCH_BUILD_EXAMPLES "picobench: build examples" ${dev_mode}) 21 | mark_as_advanced(PICOBENCH_BUILD_TOOLS PICOBENCH_BUILD_TESTS PICOBENCH_BUILD_EXAMPLES) 22 | 23 | if(dev_mode) 24 | include(./dev.cmake) 25 | endif() 26 | 27 | add_library(picobench INTERFACE) 28 | add_library(picobench::picobench ALIAS picobench) 29 | target_include_directories(picobench INTERFACE include) 30 | 31 | if(PICOBENCH_BUILD_TOOLS) 32 | add_subdirectory(tools) 33 | endif() 34 | 35 | if(PICOBENCH_BUILD_TESTS) 36 | enable_testing() 37 | add_subdirectory(test) 38 | endif() 39 | 40 | if(PICOBENCH_BUILD_EXAMPLES) 41 | add_subdirectory(example) 42 | endif() 43 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | # Copyright (c) Borislav Stanimirov 2 | # SPDX-License-Identifier: MIT 3 | # 4 | name: Test 5 | on: 6 | push: 7 | branches: [master] 8 | pull_request: 9 | branches: [master] 10 | jobs: 11 | build-and-test: 12 | runs-on: ${{ matrix.os }} 13 | strategy: 14 | matrix: 15 | os: [ubuntu-latest, windows-latest, macos-latest] 16 | type: [Debug, RelWithDebInfo] 17 | steps: 18 | - name: Clone 19 | uses: actions/checkout@v3 20 | - name: Install Ninja 21 | uses: seanmiddleditch/gha-setup-ninja@v4 22 | - name: VC Vars 23 | # Setup vcvars on Windows 24 | # MSVC's address sanitizer attaches dependencies to several DLLs which are not in PATH 25 | # vcvars will add them to PATH and allow msvc asan executables to run 26 | if: matrix.os == 'windows-latest' 27 | uses: ilammy/msvc-dev-cmd@v1 28 | - name: Configure 29 | run: cmake . -G Ninja -DCMAKE_BUILD_TYPE=${{ matrix.type }} -DSAN_ADDR=1 30 | - name: Build 31 | run: cmake --build . --config ${{ matrix.type }} 32 | - name: Test 33 | run: ctest -C ${{ matrix.type }} --output-on-failure 34 | - name: Example 35 | run: cmake --build . --config Release --target=picobench-run-example-basic 36 | # TODO: ./picobench "sleep 0.3" "sleep 0.22" 37 | -------------------------------------------------------------------------------- /example/locks.cpp: -------------------------------------------------------------------------------- 1 | #define PICOBENCH_DEBUG 2 | #define PICOBENCH_IMPLEMENT_WITH_MAIN 3 | #define PICOBENCH_DEFAULT_ITERATIONS {1000, 10000, 100000, 1000000} 4 | #include "picobench/picobench.hpp" 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | volatile int sum; 12 | 13 | template 14 | int calc_sum(bool inc, const int n, Locker& lock) 15 | { 16 | for(int i=0; i guard(lock); 19 | if (inc) sum += 2; 20 | else sum -= 3; 21 | } 22 | return n; 23 | } 24 | 25 | template 26 | void bench(picobench::state& s) 27 | { 28 | Locker lock; 29 | sum = 0; 30 | picobench::scope time(s); 31 | auto f = std::async(std::launch::async, std::bind(calc_sum, true, s.iterations(), std::ref(lock))); 32 | calc_sum(false, s.iterations(), lock); 33 | f.wait(); 34 | s.set_result(picobench::result_t(sum)); 35 | } 36 | 37 | template 38 | struct spinlock 39 | { 40 | void lock() 41 | { 42 | while(std::atomic_flag_test_and_set_explicit( 43 | &spin_flag, 44 | std::memory_order_acquire)) 45 | Yield(); 46 | } 47 | 48 | void unlock() 49 | { 50 | std::atomic_flag_clear_explicit( 51 | &spin_flag, 52 | std::memory_order_release); 53 | } 54 | 55 | std::atomic_flag spin_flag = ATOMIC_FLAG_INIT; 56 | }; 57 | 58 | inline void noop() {} 59 | 60 | using noop_spin = spinlock; 61 | using yield_spin = spinlock; 62 | 63 | using namespace std; 64 | 65 | PICOBENCH(bench); 66 | PICOBENCH(bench); 67 | PICOBENCH(bench); 68 | 69 | #if defined(__X86_64__) || defined(__x86_64) || defined(_M_X64) 70 | #include 71 | inline void pause() { _mm_pause(); } 72 | using pause_spin = spinlock; 73 | PICOBENCH(bench); 74 | #endif 75 | -------------------------------------------------------------------------------- /test/multi_cpp/main.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #define PICOBENCH_IMPLEMENT 4 | #include "picobench_configured.hpp" 5 | 6 | #include 7 | #include 8 | 9 | using namespace std; 10 | 11 | std::map g_num_samples; 12 | 13 | const pb::report::suite& find_suite(const char* s, const pb::report& r) 14 | { 15 | auto suite = r.find_suite(s); 16 | REQUIRE(suite); 17 | return *suite; 18 | } 19 | 20 | TEST_CASE("[picobench] multi cpp test") 21 | { 22 | using namespace pb; 23 | runner r; 24 | 25 | const vector iters = { 100, 2000, 5000 }; 26 | r.set_default_state_iterations(iters); 27 | 28 | const int samples = 13; 29 | r.set_default_samples(samples); 30 | 31 | r.run_benchmarks(); 32 | auto report = r.generate_report(); 33 | CHECK(report.suites.size() == 2); 34 | 35 | CHECK(g_num_samples.size() == iters.size()); 36 | size_t i = 0; 37 | for (auto& elem : g_num_samples) 38 | { 39 | CHECK(elem.first == iters[i]); 40 | CHECK(elem.second == samples); 41 | ++i; 42 | } 43 | 44 | auto& a = find_suite("suite a", report); 45 | CHECK(strcmp(a.name, "suite a") == 0); 46 | CHECK(a.benchmarks.size() == 2); 47 | 48 | auto& aa = a.benchmarks[0]; 49 | CHECK(strcmp(aa.name, "a_a") == 0); 50 | CHECK(aa.is_baseline); 51 | CHECK(aa.data.size() == iters.size()); 52 | 53 | for (size_t i = 0; i 2 | 3 | #if defined(_WIN32) 4 | #include 5 | 6 | template 7 | void zm(T& data) 8 | { 9 | ZeroMemory(&data, sizeof(T)); 10 | } 11 | 12 | int exec(const char* cmd) 13 | { 14 | const short lim = _MAX_PATH + 10; 15 | char cmd_line[lim]; 16 | strcpy(cmd_line, "cmd /c "); 17 | strncat(cmd_line, cmd, _MAX_PATH); 18 | 19 | STARTUPINFO s_info; 20 | zm(s_info); 21 | s_info.cb = sizeof(STARTUPINFO); 22 | s_info.dwFlags = STARTF_USESTDHANDLES; 23 | 24 | PROCESS_INFORMATION proc_info; 25 | zm(proc_info); 26 | 27 | auto success = CreateProcessA( 28 | nullptr, 29 | &cmd_line[0], 30 | nullptr, 31 | nullptr, 32 | TRUE, 33 | 0, 34 | nullptr, 35 | nullptr, 36 | &s_info, 37 | &proc_info); 38 | 39 | if (!success) return -1; 40 | 41 | // spin lock? doesn't seem to do anything different 42 | // while (WAIT_TIMEOUT == WaitForSingleObject(proc_info.hProcess, 0)); 43 | WaitForSingleObject(proc_info.hProcess, INFINITE); 44 | 45 | DWORD exit_code; 46 | success = GetExitCodeProcess(proc_info.hProcess, &exit_code); 47 | 48 | if (!success) return -1; 49 | 50 | CloseHandle(proc_info.hProcess); 51 | CloseHandle(proc_info.hThread); 52 | 53 | return int(exit_code); 54 | } 55 | 56 | #else 57 | 58 | #include 59 | 60 | int exec(const char* cmd) 61 | { 62 | auto s = popen(cmd, "r"); 63 | if (!s) return -1; 64 | return pclose(s); 65 | } 66 | 67 | #endif 68 | 69 | #include 70 | #include 71 | #include 72 | 73 | #define PICOBENCH_DEBUG 74 | #define PICOBENCH_IMPLEMENT 75 | #include "picobench/picobench.hpp" 76 | 77 | using namespace picobench; 78 | using namespace std; 79 | 80 | // calculate nanoseconds to spawn an empty process 81 | // by running some empty commands and taking the minimum 82 | int64_t calc_spawn_time() 83 | { 84 | const int lim = 50; 85 | int64_t min_sample = LLONG_MAX; 86 | for (int i = 0; i < lim; ++i) 87 | { 88 | auto start = high_res_clock::now(); 89 | exec(""); 90 | auto duration = high_res_clock::now() - start; 91 | auto ns = std::chrono::duration_cast(duration).count(); 92 | if (ns < min_sample) min_sample = ns; 93 | } 94 | return min_sample; 95 | } 96 | 97 | struct bench 98 | { 99 | string name; 100 | string cmd; 101 | }; 102 | 103 | vector benchmarks; 104 | static int64_t spawn_time; 105 | 106 | void bench_proc(state& s) 107 | { 108 | const char* cmd = benchmarks[s.user_data()].cmd.c_str(); 109 | for (auto _ : s) 110 | { 111 | exec(cmd); 112 | } 113 | 114 | s.add_custom_duration(-spawn_time); 115 | }; 116 | 117 | bool parse_bfile(uintptr_t, const char* file) 118 | { 119 | if (!*file) 120 | { 121 | cerr << "Error: bfile missing filename\n"; 122 | return false; 123 | } 124 | 125 | ifstream fin(file); 126 | 127 | if (!fin) 128 | { 129 | cerr << "Error: Cannot open " << file << "\n"; 130 | return false; 131 | } 132 | 133 | int iline = 0; 134 | string line; 135 | string name; 136 | while (!fin.eof()) 137 | { 138 | getline(fin, line); 139 | bool empty = true; 140 | for (auto& c : line) 141 | { 142 | if (!isspace(c)) 143 | { 144 | empty = false; 145 | break; 146 | } 147 | } 148 | 149 | if (empty) continue; 150 | 151 | ++iline; 152 | // odd lines are benchmark names 153 | // even lines are commands 154 | if (iline & 1) 155 | { 156 | name = line; 157 | } 158 | else 159 | { 160 | benchmarks.push_back({ name, line }); 161 | } 162 | } 163 | return true; 164 | } 165 | 166 | int main(int argc, char* argv[]) 167 | { 168 | if (argc == 1) 169 | { 170 | cout << "picobench " PICOBENCH_VERSION_STR "\n"; 171 | cout << "Usage: picobench \n"; 172 | cout << "Type 'picobench --help' for help.\n"; 173 | return 0; 174 | } 175 | 176 | for (int i = 1; i < argc; ++i) 177 | { 178 | if (argv[i][0] != '-') 179 | { 180 | benchmarks.push_back({ argv[i], argv[i] }); 181 | } 182 | } 183 | 184 | runner r; 185 | r.set_default_state_iterations({ 1 }); 186 | r.set_default_samples(1); 187 | 188 | r.add_cmd_opt("-bfile=", "", "Set a file which lists benchmarks", parse_bfile); 189 | 190 | r.parse_cmd_line(argc, argv); 191 | 192 | if (!r.should_run()) return r.error(); 193 | 194 | for (size_t i = 0; i < benchmarks.size(); ++i) 195 | { 196 | auto& b = benchmarks[i]; 197 | r.add_benchmark(b.name.c_str(), bench_proc).user_data(i); 198 | } 199 | 200 | spawn_time = calc_spawn_time(); 201 | 202 | r.run_benchmarks(); 203 | auto report = r.generate_report(); 204 | std::ostream* out = &std::cout; 205 | std::ofstream fout; 206 | if (r.preferred_output_filename()) 207 | { 208 | fout.open(r.preferred_output_filename()); 209 | if (!fout.is_open()) 210 | { 211 | std::cerr << "Error: Could not open output file `" << r.preferred_output_filename() << "`\n"; 212 | return 1; 213 | } 214 | out = &fout; 215 | } 216 | 217 | switch (r.preferred_output_format()) 218 | { 219 | case picobench::report_output_format::text: 220 | report.to_text(*out); 221 | break; 222 | case picobench::report_output_format::concise_text: 223 | report.to_text_concise(*out); 224 | break; 225 | case picobench::report_output_format::csv: 226 | report.to_csv(*out); 227 | break; 228 | } 229 | 230 | return r.error(); 231 | } 232 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # picobench 3 | [![Language](https://img.shields.io/badge/language-C++-blue.svg)](https://isocpp.org/) [![Standard](https://img.shields.io/badge/C%2B%2B-11-blue.svg)](https://en.wikipedia.org/wiki/C%2B%2B#Standardization) [![License](https://img.shields.io/badge/license-MIT-blue.svg)](https://opensource.org/licenses/MIT) 4 | 5 | [![Test](https://github.com/iboB/picobench/actions/workflows/test.yml/badge.svg)](https://github.com/iboB/picobench/actions/workflows/test.yml) 6 | 7 | picobench is a tiny (micro) microbenchmarking library in a single header file. 8 | 9 | It's designed to be easy to use and integrate and fast to compile while covering the most common features of a microbenchmarking library. 10 | 11 | ## Example usage 12 | 13 | Here's the complete code of a microbenchmark which compares adding elements to a `std::vector` with and without using `reserve`: 14 | 15 | ```c++ 16 | #define PICOBENCH_IMPLEMENT_WITH_MAIN 17 | #include "picobench/picobench.hpp" 18 | 19 | #include 20 | #include // for rand 21 | 22 | // Benchmarking function written by the user: 23 | static void rand_vector(picobench::state& s) 24 | { 25 | std::vector v; 26 | for (auto _ : s) 27 | { 28 | v.push_back(rand()); 29 | } 30 | } 31 | PICOBENCH(rand_vector); // Register the above function with picobench 32 | 33 | // Another benchmarking function: 34 | static void rand_vector_reserve(picobench::state& s) 35 | { 36 | std::vector v; 37 | v.reserve(s.iterations()); 38 | for (auto _ : s) 39 | { 40 | v.push_back(rand()); 41 | } 42 | } 43 | PICOBENCH(rand_vector_reserve); 44 | ``` 45 | 46 | The output of this benchmark might look like this: 47 | 48 | ``` 49 | Name (* = baseline) | Dim | Total ms | ns/op |Baseline| Ops/second 50 | --------------------------|--------:|----------:|--------:|-------:|----------: 51 | rand_vector * | 8 | 0.001 | 167 | - | 5974607.9 52 | rand_vector_reserve | 8 | 0.000 | 55 | 0.329 | 18181818.1 53 | rand_vector * | 64 | 0.004 | 69 | - | 14343343.8 54 | rand_vector_reserve | 64 | 0.002 | 27 | 0.400 | 35854341.7 55 | rand_vector * | 512 | 0.017 | 33 | - | 30192239.7 56 | rand_vector_reserve | 512 | 0.012 | 23 | 0.710 | 42496679.9 57 | rand_vector * | 4096 | 0.181 | 44 | - | 22607850.9 58 | rand_vector_reserve | 4096 | 0.095 | 23 | 0.527 | 42891848.9 59 | rand_vector * | 8196 | 0.266 | 32 | - | 30868196.3 60 | rand_vector_reserve | 8196 | 0.207 | 25 | 0.778 | 39668749.5 61 | ``` 62 | 63 | ...which tells us that we see a noticeable performance gain when we use `reserve` but the effect gets less prominent for bigger numbers of elements inserted. 64 | 65 | ## Documentation 66 | 67 | To use picobench, you need to include `picobench.hpp` by either copying it inside your project or adding this repo as a submodule to yours. 68 | 69 | In one compilation unit (.cpp file) in the module (typically the benchmark executable) in which you use picobench, you need to define `PICOBENCH_IMPLEMENT_WITH_MAIN` (or `PICOBENCH_IMPLEMENT` if you want to write your own `main` function). 70 | 71 | ### Creating benchmarks 72 | 73 | A benchmark is a function which you're written with the signature `void (picobench::state& s)`. You need to register the function with the macro `PICOBENCH(func_name)` where the only argument is the function's name as shown in the example above. 74 | 75 | The library will run the benchmark function several times with different numbers of iterations, to simulate different problem spaces, then collect the results in a report. 76 | 77 | Typically a benchmark has a loop. To run the loop, use the `picobench::state` argument in a range-based for loop in your function. The time spent looping is measured for the benchmark. You can have initialization/deinitialization code outside of the loop and it won't be measured. 78 | 79 | You can have multiple benchmarks in multiple files. All of them will be run when the executable starts. 80 | 81 | Use `state::iterations` as shown in the example to make initialization based on how many iterations the loop will make. 82 | 83 | If you don't want the automatic time measurement, you can use `state::start_timer` and `state::stop_timer` to manually measure it, or use the RAII class `picobench::scope` for semi-automatic measurement. 84 | 85 | Here's an example of a couple of benchmarks, which does not use the range-based for loop for time measurement: 86 | 87 | ```c++ 88 | void my_func(); // Function you want to benchmark 89 | static void benchmark_my_func(picobench::state& s) 90 | { 91 | s.start_timer(); // Manual start 92 | for (int i=0; i(new my_vector2(result))); 202 | } 203 | 204 | bool compare_vectors(result_t a, result_t b) 205 | { 206 | auto v1 = reinterpret_cast(a); 207 | auto v2 = reinterpret_cast(b); 208 | return v1->x == v2->x && v1->y == v2->y; 209 | } 210 | 211 | ... 212 | 213 | auto report = runner.generate_report(compare_vectors); 214 | 215 | ``` 216 | 217 | 218 | ### Other options 219 | 220 | Other characteristics of a benchmark are: 221 | 222 | * **Iterations**: (or "problem spaces") a vector of integers describing the set of iterations to be made for a benchmark. Set with `.iterations({i1, i2, i3...})`. The default is {8, 64, 512, 4096, 8196}. 223 | * **Label**: a string which is used for this benchmark in the report instead of the function name. Set with `.label("my label")` 224 | * **User data**: a user defined number (`uintptr_t`) assinged to a benchmark which can be accessed by `state::user_data` 225 | 226 | You can combine the options by concatenating them like this: `PICOBENCH(my_func).label("My Function").samples(2).iterations({1000, 10000, 50000});` 227 | 228 | If you write your own main function, you can set the default iterations and samples for all benchmarks with `runner::set_default_state_iterations` and `runner::set_default_samples` *before* calling `runner::run_benchmarks`. 229 | 230 | If you parse the command line or use the library-provided `main` function you can also set the iterations and samples with command line args: 231 | * `--iters=1000,5000,10000` will set the iterations for benchmarks which don't explicitly override them 232 | * `--samples=5` will set the samples for benchmarks which don't explicitly override them 233 | 234 | ### Other command line arguments 235 | 236 | If you're using the library-provided `main` function, it will also handle the following command line arguments: 237 | * `--out-fmt=` - sets the output report format to either full text, concise text or csv. 238 | * `--output=` - writes the output report to a given file 239 | * `--compare-results` - will compare results from benchmarks and trigger an error if they don't match. 240 | 241 | ### Misc 242 | 243 | * The runner randomizes the benchmarks. To have the same order on every run and every platform, set an integer seed to `runner::run_benchmarks`. 244 | 245 | Here's another example of a custom main function incporporating the above: 246 | 247 | ```c++ 248 | #define PICOBENCH_IMPLEMENT 249 | #include "picobench/picobench.hpp" 250 | ... 251 | int main() 252 | { 253 | // User-defined code which makes global initializations 254 | custom_global_init(); 255 | 256 | picobench::runner runner; 257 | // Disregard command-line for simplicity 258 | 259 | // Two sets of iterations 260 | runner.set_default_state_iterations({10000, 50000}); 261 | 262 | // One sample per benchmark because the huge numbers are expected to compensate 263 | // for external factors 264 | runner.set_default_samples(1); 265 | 266 | // Run the benchmarks with some seed which guarantees the same order every time 267 | auto report = runner.run_benchmarks(123); 268 | 269 | // Output to some file 270 | report.to_csv(ofstream("my.csv")); 271 | 272 | return 0; 273 | } 274 | ``` 275 | 276 | ## Contributing 277 | 278 | Contributions in the form of issues and pull requests are welcome. 279 | 280 | ## License 281 | 282 | This software is distributed under the MIT Software License. 283 | 284 | See accompanying file LICENSE.txt or copy [here](https://opensource.org/licenses/MIT). 285 | 286 | Copyright © 2017-2025 [Borislav Stanimirov](http://github.com/iboB) 287 | -------------------------------------------------------------------------------- /test/basic.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #define PICOBENCH_DEBUG 4 | #define PICOBENCH_TEST 5 | #define PICOBENCH_IMPLEMENT 6 | #include 7 | 8 | #include 9 | #include 10 | 11 | using namespace picobench; 12 | using namespace std; 13 | 14 | PICOBENCH_SUITE("test a"); 15 | 16 | static void a_a(picobench::state& s) 17 | { 18 | for (auto _ : s) 19 | { 20 | test::this_thread_sleep_for_ns(10); 21 | } 22 | s.set_result(s.iterations() * 2); 23 | } 24 | PICOBENCH(a_a); 25 | 26 | map a_b_samples; 27 | static void a_b(picobench::state& s) 28 | { 29 | uint64_t time = 11; 30 | if (a_b_samples.find(s.iterations()) == a_b_samples.end()) 31 | { 32 | // slower first time 33 | time = 32; 34 | } 35 | 36 | ++a_b_samples[s.iterations()]; 37 | for (auto _ : s) 38 | { 39 | test::this_thread_sleep_for_ns(time); 40 | } 41 | s.set_result(s.iterations() * 2); 42 | } 43 | PICOBENCH(a_b); 44 | 45 | static void a_c(picobench::state& s) 46 | { 47 | s.start_timer(); 48 | test::this_thread_sleep_for_ns((s.iterations() - 1) * 20); 49 | s.stop_timer(); 50 | 51 | s.add_custom_duration(20); 52 | s.set_result(s.iterations() * 2); 53 | } 54 | PICOBENCH(a_c); 55 | 56 | PICOBENCH_SUITE("test empty"); 57 | 58 | PICOBENCH_SUITE("test b"); 59 | 60 | static void b_a(picobench::state& s) 61 | { 62 | CHECK(s.user_data() == 9088); 63 | for (auto _ : s) 64 | { 65 | test::this_thread_sleep_for_ns(75); 66 | } 67 | } 68 | PICOBENCH(b_a) 69 | .iterations({20, 30, 50}) 70 | .user_data(9088); 71 | 72 | map b_b_samples; 73 | 74 | static void b_b(picobench::state& s) 75 | { 76 | uint64_t time = 111; 77 | if (b_b_samples.find(s.iterations()) == b_b_samples.end()) 78 | { 79 | // faster first time 80 | time = 100; 81 | } 82 | 83 | ++b_b_samples[s.iterations()]; 84 | picobench::scope scope(s); 85 | test::this_thread_sleep_for_ns(s.iterations() * time); 86 | } 87 | PICOBENCH(b_b) 88 | .baseline() 89 | .label("something else") 90 | .samples(15) 91 | .iterations({10, 20, 30}); 92 | 93 | const picobench::report::suite& find_suite(const char* s, const picobench::report& r) 94 | { 95 | auto suite = r.find_suite(s); 96 | REQUIRE(suite); 97 | return *suite; 98 | } 99 | 100 | #define cntof(ar) (sizeof(ar) / sizeof(ar[0])) 101 | 102 | TEST_CASE("[picobench] test utils") 103 | { 104 | const char* ar[] = {"test", "123", "asdf"}; 105 | CHECK(cntof(ar) == 3); 106 | 107 | auto start = high_res_clock::now(); 108 | test::this_thread_sleep_for_ns(1234); 109 | auto end = high_res_clock::now(); 110 | 111 | auto duration = end - start; 112 | CHECK(duration == std::chrono::nanoseconds(1234)); 113 | 114 | start = high_res_clock::now(); 115 | test::this_thread_sleep_for(std::chrono::milliseconds(987)); 116 | end = high_res_clock::now(); 117 | duration = end - start; 118 | CHECK(duration == std::chrono::milliseconds(987)); 119 | } 120 | 121 | TEST_CASE("[picobench] picostring") 122 | { 123 | picostring str("test"); 124 | CHECK(str == "test"); 125 | CHECK(str.len == 4); 126 | CHECK(!str.is_start_of("tes")); 127 | CHECK(str.is_start_of("test")); 128 | CHECK(str.is_start_of("test123")); 129 | } 130 | 131 | TEST_CASE("[picobench] state") 132 | { 133 | state s0(3); 134 | CHECK(s0.iterations() == 3); 135 | CHECK(s0.user_data() == 0); 136 | 137 | int i = 0; 138 | for (auto _ : s0) 139 | { 140 | CHECK(_ == i); 141 | ++i; 142 | test::this_thread_sleep_for_ns(1); 143 | } 144 | CHECK(s0.duration_ns() == 3); 145 | s0.add_custom_duration(5); 146 | CHECK(s0.duration_ns() == 8); 147 | 148 | state s(2, 123); 149 | CHECK(s.iterations() == 2); 150 | CHECK(s.user_data() == 123); 151 | 152 | i = 0; 153 | for (auto it = s.begin(); it != s.end(); ++it) 154 | { 155 | CHECK(*it == i); 156 | ++i; 157 | test::this_thread_sleep_for_ns(2); 158 | } 159 | CHECK(s.duration_ns() == 4); 160 | } 161 | 162 | const vector default_iters = { 8, 64, 512, 4096, 8192 }; 163 | const int default_samples = 2; 164 | 165 | TEST_CASE("[picobench] cmd line") 166 | { 167 | { 168 | local_runner r; 169 | bool b = r.parse_cmd_line(0, {}); 170 | CHECK(b); 171 | CHECK(r.should_run()); 172 | CHECK(r.error() == 0); 173 | CHECK(r.default_state_iterations() == default_iters); 174 | CHECK(r.default_samples() == default_samples); 175 | CHECK(!r.preferred_output_filename()); 176 | CHECK(r.preferred_output_format() == report_output_format::text); 177 | CHECK(!r.compare_results_across_benchmarks()); 178 | CHECK(!r.compare_results_across_samples()); 179 | } 180 | 181 | { 182 | local_runner r; 183 | ostringstream sout, serr; 184 | r.set_output_streams(sout, serr); 185 | const char* cmd_line[] = { "", "-asdf" }; 186 | bool b = r.parse_cmd_line(cntof(cmd_line), cmd_line, "-"); 187 | CHECK(sout.str().empty()); 188 | CHECK(serr.str() == "Error: Unknown command-line argument: -asdf\n"); 189 | CHECK(!b); 190 | CHECK(!r.should_run()); 191 | CHECK(r.error() == error_unknown_cmd_line_argument); 192 | } 193 | 194 | { 195 | local_runner r; 196 | const char* cmd_line[] = { "", "--no-run", "--iters=1,2,3", "--samples=54", "--out-fmt=con", "--output=stdout" }; 197 | bool b = r.parse_cmd_line(cntof(cmd_line), cmd_line); 198 | CHECK(b); 199 | CHECK(!r.should_run()); 200 | CHECK(r.error() == 0); 201 | CHECK(r.default_samples() == 54); 202 | CHECK(r.default_state_iterations() == vector({ 1, 2, 3 })); 203 | CHECK(!r.preferred_output_filename()); 204 | CHECK(r.preferred_output_format() == report_output_format::concise_text); 205 | CHECK(!r.compare_results_across_benchmarks()); 206 | CHECK(!r.compare_results_across_samples()); 207 | } 208 | 209 | { 210 | local_runner r; 211 | const char* cmd_line[] = { "", "--pb-no-run", "--pb-iters=1000,2000,3000", "-other-cmd1", "--pb-samples=54", 212 | "-other-cmd2", "--pb-out-fmt=csv", "--pb-output=foo.csv", "--pb-compare-results" }; 213 | bool b = r.parse_cmd_line(cntof(cmd_line), cmd_line, "--pb"); 214 | CHECK(b); 215 | CHECK(!r.should_run()); 216 | CHECK(r.error() == 0); 217 | CHECK(r.default_samples() == 54); 218 | CHECK(r.default_state_iterations() == vector({ 1000, 2000, 3000 })); 219 | CHECK(strcmp(r.preferred_output_filename(), "foo.csv") == 0); 220 | CHECK(r.preferred_output_format() == report_output_format::csv); 221 | CHECK(r.compare_results_across_benchmarks()); 222 | CHECK(r.compare_results_across_samples()); 223 | 224 | } 225 | 226 | { 227 | local_runner r; 228 | ostringstream sout, serr; 229 | r.set_output_streams(sout, serr); 230 | const char* cmd_line[] = { "", "--samples=xxx" }; 231 | bool b = r.parse_cmd_line(cntof(cmd_line), cmd_line, "-"); 232 | CHECK(sout.str().empty()); 233 | CHECK(serr.str() == "Error: Bad command-line argument: --samples=xxx\n"); 234 | CHECK(!b); 235 | CHECK(!r.should_run()); 236 | CHECK(r.error() == error_bad_cmd_line_argument); 237 | CHECK(r.default_samples() == default_samples); 238 | } 239 | 240 | { 241 | local_runner r; 242 | ostringstream sout, serr; 243 | r.set_output_streams(sout, serr); 244 | const char* cmd_line[] = { "", "--iters=1,xxx,2" }; 245 | bool b = r.parse_cmd_line(cntof(cmd_line), cmd_line, "-"); 246 | CHECK(sout.str().empty()); 247 | CHECK(serr.str() == "Error: Bad command-line argument: --iters=1,xxx,2\n"); 248 | CHECK(!b); 249 | CHECK(!r.should_run()); 250 | CHECK(r.error() == error_bad_cmd_line_argument); 251 | CHECK(r.default_state_iterations() == default_iters); 252 | } 253 | 254 | { 255 | local_runner r; 256 | ostringstream sout, serr; 257 | r.set_output_streams(sout, serr); 258 | const char* cmd_line[] = { "", "--out-fmt=asdf" }; 259 | bool b = r.parse_cmd_line(cntof(cmd_line), cmd_line, "-"); 260 | CHECK(sout.str().empty()); 261 | CHECK(serr.str() == "Error: Bad command-line argument: --out-fmt=asdf\n"); 262 | CHECK(!b); 263 | CHECK(!r.should_run()); 264 | CHECK(r.error() == error_bad_cmd_line_argument); 265 | CHECK(r.preferred_output_format() == report_output_format::text); 266 | } 267 | 268 | #define PB_VERSION_INFO "picobench " PICOBENCH_VERSION_STR "\n" 269 | 270 | { 271 | const char* v = PB_VERSION_INFO; 272 | 273 | local_runner r; 274 | ostringstream sout, serr; 275 | r.set_output_streams(sout, serr); 276 | const char* cmd_line[] = { "", "--pb-version" }; 277 | bool b = r.parse_cmd_line(cntof(cmd_line), cmd_line, "--pb"); 278 | CHECK(sout.str() == v); 279 | CHECK(serr.str().empty()); 280 | CHECK(b); 281 | CHECK(!r.should_run()); 282 | CHECK(r.error() == 0); 283 | } 284 | 285 | #define PB_HELP \ 286 | " --pb-iters= Sets default iterations for benchmarks\n" \ 287 | " --pb-samples= Sets default number of samples for benchmarks\n" \ 288 | " --pb-out-fmt= Outputs text or concise or csv\n" \ 289 | " --pb-output= Sets output filename or `stdout`\n" \ 290 | " --pb-compare-results Compare benchmark results\n" \ 291 | " --pb-no-run Doesn't run benchmarks\n" \ 292 | " --pb-run-suite= Runs only benchmarks from suite\n" \ 293 | " --pb-run-only= Runs only selected benchmarks\n" \ 294 | " --pb-list Lists available benchmarks\n" \ 295 | " --pb-version Show version info\n" \ 296 | " --pb-help Prints help\n" 297 | 298 | { 299 | const char* help = 300 | PB_VERSION_INFO 301 | PB_HELP; 302 | 303 | local_runner r; 304 | ostringstream sout, serr; 305 | r.set_output_streams(sout, serr); 306 | const char* cmd_line[] = { "", "--pb-help" }; 307 | bool b = r.parse_cmd_line(cntof(cmd_line), cmd_line, "--pb"); 308 | CHECK(sout.str() == help); 309 | CHECK(serr.str().empty()); 310 | CHECK(b); 311 | CHECK(!r.should_run()); 312 | CHECK(r.error() == 0); 313 | } 314 | 315 | { 316 | const char* help = 317 | PB_VERSION_INFO 318 | " --pb-cmd-hi Custom help\n" 319 | " --pb-cmd-bi=123 More custom help\n" 320 | PB_HELP; 321 | 322 | local_runner r; 323 | 324 | auto handler_hi = [](uintptr_t data, const char* cmd) -> bool { 325 | CHECK(data == 123); 326 | CHECK(*cmd == 0); 327 | return true; 328 | }; 329 | 330 | r.add_cmd_opt("-cmd-hi", "", "Custom help", handler_hi, 123); 331 | 332 | auto handler_bi = [](uintptr_t data, const char* cmd) -> bool { 333 | CHECK(data == 98); 334 | CHECK(strcmp(cmd, "123") == 0); 335 | return true; 336 | }; 337 | 338 | r.add_cmd_opt("-cmd-bi=", "123", "More custom help", handler_bi, 98); 339 | 340 | ostringstream sout, serr; 341 | r.set_output_streams(sout, serr); 342 | const char* cmd_line[] = { "", "--pb-help" }; 343 | bool b = r.parse_cmd_line(cntof(cmd_line), cmd_line, "--pb"); 344 | CHECK(sout.str() == help); 345 | CHECK(serr.str().empty()); 346 | CHECK(b); 347 | CHECK(!r.should_run()); 348 | CHECK(r.error() == 0); 349 | 350 | sout.str(std::string()); 351 | serr.str(std::string()); 352 | 353 | const char* cmd_line2[] = { "", "--zz-cmd-bi=123", "--zz-cmd-hi" }; 354 | b = r.parse_cmd_line(cntof(cmd_line2), cmd_line2, "--zz"); 355 | 356 | CHECK(sout.str().empty()); 357 | CHECK(serr.str().empty()); 358 | CHECK(b); 359 | CHECK(r.error() == 0); 360 | } 361 | } 362 | 363 | TEST_CASE("[picobench] test") 364 | { 365 | runner r; 366 | CHECK(r.default_state_iterations() == default_iters); 367 | CHECK(r.default_samples() == default_samples); 368 | 369 | r.set_compare_results_across_benchmarks(true); 370 | r.set_compare_results_across_samples(true); 371 | 372 | ostringstream sout; 373 | ostringstream serr; 374 | r.set_output_streams(sout, serr); 375 | 376 | r.run_benchmarks(); 377 | auto report = r.generate_report(); 378 | 379 | CHECK(serr.str().empty()); 380 | 381 | const char* warnings = 382 | "Warning: Benchmark something else @10 has a single instance and cannot be compared to others.\n" 383 | "Warning: Benchmark b_a @50 has a single instance and cannot be compared to others.\n"; 384 | CHECK(sout.str() == warnings); 385 | 386 | CHECK(report.suites.size() == 2); 387 | CHECK(!report.find_suite("asdf")); 388 | 389 | auto& a = find_suite("test a", report); 390 | CHECK(strcmp(a.name, "test a") == 0); 391 | CHECK(a.benchmarks.size() == 3); 392 | CHECK(!a.find_benchmark("b_a")); 393 | 394 | auto& aa = a.benchmarks[0]; 395 | CHECK(a.find_baseline() == &aa); 396 | CHECK(a.find_benchmark("a_a") == &aa); 397 | CHECK(strcmp(aa.name, "a_a") == 0); 398 | CHECK(aa.is_baseline); 399 | CHECK(aa.data.size() == r.default_state_iterations().size()); 400 | 401 | for (size_t i = 0; i 127 | #include 128 | #include 129 | 130 | #if defined(PICOBENCH_STD_FUNCTION_BENCHMARKS) 131 | # include 132 | #endif 133 | 134 | #define PICOBENCH_VERSION 2.8.0 135 | #define PICOBENCH_VERSION_STR "2.8.0" 136 | 137 | #if defined(PICOBENCH_DEBUG) 138 | # include 139 | # define I_PICOBENCH_ASSERT assert 140 | #else 141 | # define I_PICOBENCH_ASSERT(...) 142 | #endif 143 | 144 | #if defined(__GNUC__) 145 | # define PICOBENCH_INLINE __attribute__((always_inline)) 146 | #elif defined(_MSC_VER) 147 | # define PICOBENCH_INLINE __forceinline 148 | #else 149 | # define PICOBENCH_INLINE inline 150 | #endif 151 | 152 | #if !defined(PICOBENCH_NAMESPACE) 153 | # define PICOBENCH_NAMESPACE picobench 154 | #endif 155 | 156 | namespace PICOBENCH_NAMESPACE 157 | { 158 | 159 | #if defined(_MSC_VER) || defined(__MINGW32__) || defined(PICOBENCH_TEST) 160 | struct high_res_clock 161 | { 162 | typedef long long rep; 163 | typedef std::nano period; 164 | typedef std::chrono::duration duration; 165 | typedef std::chrono::time_point time_point; 166 | static const bool is_steady = true; 167 | 168 | static time_point now(); 169 | }; 170 | #else 171 | using high_res_clock = std::chrono::high_resolution_clock; 172 | #endif 173 | 174 | using result_t = intptr_t; 175 | 176 | class state 177 | { 178 | public: 179 | explicit state(int num_iterations, uintptr_t user_data = 0) 180 | : _user_data(user_data) 181 | , _iterations(num_iterations) 182 | { 183 | I_PICOBENCH_ASSERT(_iterations > 0); 184 | } 185 | 186 | int iterations() const { return _iterations; } 187 | 188 | int64_t duration_ns() const { return _duration_ns; } 189 | void add_custom_duration(int64_t duration_ns) { _duration_ns += duration_ns; } 190 | 191 | uintptr_t user_data() const { return _user_data; } 192 | 193 | // optionally set result of benchmark 194 | // this can be used as a value sync to prevent optimizations 195 | // or a way to check whether benchmarks produce the same results 196 | void set_result(uintptr_t data) { _result = data; } 197 | result_t result() const { return _result; } 198 | 199 | PICOBENCH_INLINE 200 | void start_timer() 201 | { 202 | _start = high_res_clock::now(); 203 | } 204 | 205 | PICOBENCH_INLINE 206 | void stop_timer() 207 | { 208 | auto duration = high_res_clock::now() - _start; 209 | _duration_ns = std::chrono::duration_cast(duration).count(); 210 | } 211 | 212 | struct iterator 213 | { 214 | PICOBENCH_INLINE 215 | iterator(state* parent) 216 | : _counter(0) 217 | , _lim(parent->iterations()) 218 | , _state(parent) 219 | { 220 | I_PICOBENCH_ASSERT(_counter < _lim); 221 | } 222 | 223 | PICOBENCH_INLINE 224 | iterator() 225 | : _counter(0) 226 | , _lim(0) 227 | , _state(nullptr) 228 | {} 229 | 230 | PICOBENCH_INLINE 231 | iterator& operator++() 232 | { 233 | I_PICOBENCH_ASSERT(_counter < _lim); 234 | ++_counter; 235 | return *this; 236 | } 237 | 238 | PICOBENCH_INLINE 239 | bool operator!=(const iterator&) const 240 | { 241 | if (_counter < _lim) return true; 242 | _state->stop_timer(); 243 | return false; 244 | } 245 | 246 | PICOBENCH_INLINE 247 | int operator*() const 248 | { 249 | return _counter; 250 | } 251 | 252 | private: 253 | int _counter; 254 | const int _lim; 255 | state* _state; 256 | }; 257 | 258 | PICOBENCH_INLINE 259 | iterator begin() 260 | { 261 | start_timer(); 262 | return iterator(this); 263 | } 264 | 265 | PICOBENCH_INLINE 266 | iterator end() 267 | { 268 | return iterator(); 269 | } 270 | 271 | private: 272 | high_res_clock::time_point _start; 273 | int64_t _duration_ns = 0; 274 | uintptr_t _user_data; 275 | int _iterations; 276 | result_t _result = 0; 277 | }; 278 | 279 | // this can be used for manual measurement 280 | class scope 281 | { 282 | public: 283 | PICOBENCH_INLINE 284 | scope(state& s) 285 | : _state(s) 286 | { 287 | _state.start_timer(); 288 | } 289 | 290 | PICOBENCH_INLINE 291 | ~scope() 292 | { 293 | _state.stop_timer(); 294 | } 295 | private: 296 | state& _state; 297 | }; 298 | 299 | #if defined(PICOBENCH_STD_FUNCTION_BENCHMARKS) 300 | using benchmark_proc = std::function; 301 | #else 302 | using benchmark_proc = void(*)(state&); 303 | #endif 304 | 305 | class benchmark 306 | { 307 | public: 308 | const char* name() const { return _name; } 309 | 310 | benchmark& iterations(std::vector data) { _state_iterations = std::move(data); return *this; } 311 | benchmark& samples(int n) { _samples = n; return *this; } 312 | benchmark& label(const char* label) { _name = label; return *this; } 313 | benchmark& baseline(bool b = true) { _baseline = b; return *this; } 314 | benchmark& user_data(uintptr_t data) { _user_data = data; return *this; } 315 | 316 | protected: 317 | friend class runner; 318 | 319 | benchmark(const char* name, benchmark_proc proc); 320 | 321 | const char* _name; 322 | const benchmark_proc _proc; 323 | bool _baseline = false; 324 | 325 | uintptr_t _user_data = 0; 326 | std::vector _state_iterations; 327 | int _samples = 0; 328 | }; 329 | 330 | // used for globally functions 331 | // note that you can instantiate a runner and register local benchmarks for it alone 332 | class global_registry 333 | { 334 | public: 335 | static int set_bench_suite(const char* name); 336 | static benchmark& new_benchmark(const char* name, benchmark_proc proc); 337 | }; 338 | 339 | } 340 | 341 | // Optionally define PICOBENCH_UNIQUE_SYM_SUFFIX to replace __LINE__ with something 342 | // non standard like __COUNTER__ in case you need multiple PICOBENCH macros in a 343 | // macro of yours 344 | #if !defined(PICOBENCH_UNIQUE_SYM_SUFFIX) 345 | #define PICOBENCH_UNIQUE_SYM_SUFFIX __LINE__ 346 | #endif 347 | 348 | #define I_PICOBENCH_PP_CAT(a, b) I_PICOBENCH_PP_INTERNAL_CAT(a, b) 349 | #define I_PICOBENCH_PP_INTERNAL_CAT(a, b) a##b 350 | 351 | #define PICOBENCH_SUITE(name) \ 352 | static int I_PICOBENCH_PP_CAT(picobench_suite, PICOBENCH_UNIQUE_SYM_SUFFIX) = \ 353 | PICOBENCH_NAMESPACE::global_registry::set_bench_suite(name) 354 | 355 | #define PICOBENCH(func) \ 356 | static auto& I_PICOBENCH_PP_CAT(picobench, PICOBENCH_UNIQUE_SYM_SUFFIX) = \ 357 | PICOBENCH_NAMESPACE::global_registry::new_benchmark(#func, func) 358 | 359 | #if defined(PICOBENCH_IMPLEMENT_WITH_MAIN) 360 | # define PICOBENCH_IMPLEMENT 361 | # define PICOBENCH_IMPLEMENT_MAIN 362 | #endif 363 | 364 | #endif // PICOBENCH_HPP_INCLUDED 365 | 366 | #if defined(PICOBENCH_IMPLEMENT) 367 | 368 | #include 369 | #include 370 | #include 371 | #include 372 | #include 373 | #include 374 | #include 375 | #include 376 | #include 377 | 378 | #if defined(_WIN32) 379 | # define WIN32_LEAN_AND_MEAN 380 | # include 381 | #else 382 | # if !defined(PICOBENCH_DONT_BIND_TO_ONE_CORE) 383 | # if defined(__APPLE__) 384 | # include 385 | # else 386 | # include 387 | # endif 388 | # endif 389 | #endif 390 | 391 | namespace PICOBENCH_NAMESPACE 392 | { 393 | 394 | // namespace 395 | // { 396 | 397 | enum error_t 398 | { 399 | no_error, 400 | error_bad_cmd_line_argument, // ill-formed command-line argument 401 | error_unknown_cmd_line_argument, // command argument looks like a picobench one, but isn't 402 | error_sample_compare, // benchmark produced different results across samples 403 | error_benchmark_compare, // two benchmarks of the same suite and dimension produced different results 404 | }; 405 | 406 | class report 407 | { 408 | public: 409 | struct benchmark_problem_space 410 | { 411 | int dimension; // number of iterations for the problem space 412 | int samples; // number of samples taken 413 | int64_t total_time_ns; // fastest sample!!! 414 | result_t result; // result of fastest sample 415 | }; 416 | struct benchmark 417 | { 418 | const char* name; 419 | bool is_baseline; 420 | std::vector data; 421 | }; 422 | 423 | struct suite 424 | { 425 | const char* name; 426 | std::vector benchmarks; // benchmark view 427 | 428 | const benchmark* find_benchmark(const char* bname) const 429 | { 430 | for (auto& b : benchmarks) 431 | { 432 | if (strcmp(b.name, bname) == 0) 433 | return &b; 434 | } 435 | 436 | return nullptr; 437 | } 438 | 439 | const benchmark* find_baseline() const 440 | { 441 | for (auto& b : benchmarks) 442 | { 443 | if (b.is_baseline) 444 | return &b; 445 | } 446 | 447 | return nullptr; 448 | } 449 | }; 450 | 451 | std::vector suites; 452 | error_t error = no_error; 453 | 454 | const suite* find_suite(const char* name) const 455 | { 456 | for (auto& s : suites) 457 | { 458 | if (strcmp(s.name, name) == 0) 459 | return &s; 460 | } 461 | 462 | return nullptr; 463 | } 464 | 465 | void to_text(std::ostream& out) const 466 | { 467 | using namespace std; 468 | for (auto& suite : suites) 469 | { 470 | if (suite.name) 471 | { 472 | out << "## " << suite.name << ":\n"; 473 | } 474 | 475 | out.put('\n'); 476 | out << 477 | " Name (* = baseline) | Dim | Total ms | ns/op |Baseline| Ops/second\n"; 478 | out << 479 | "--------------------------|--------:|----------:|--------:|-------:|----------:\n"; 480 | 481 | auto problem_space_view = get_problem_space_view(suite); 482 | for (auto& ps : problem_space_view) 483 | { 484 | const problem_space_benchmark* baseline = nullptr; 485 | for (auto& bm : ps.second) 486 | { 487 | if (bm.is_baseline) 488 | { 489 | baseline = &bm; 490 | break; 491 | } 492 | } 493 | 494 | for (auto& bm : ps.second) 495 | { 496 | out << ' ' << bm.name; 497 | auto pad = 24 - int(strlen(bm.name)); 498 | if (bm.is_baseline) 499 | { 500 | out << " *"; 501 | pad -= 2; 502 | } 503 | for (int i = 0; i < pad; ++i) { 504 | out.put(' '); 505 | } 506 | 507 | out << " |" 508 | << setw(8) << ps.first << " |" 509 | << setw(10) << fixed << setprecision(3) << double(bm.total_time_ns) / 1000000.0 << " |"; 510 | 511 | auto ns_op = (bm.total_time_ns / ps.first); 512 | if (ns_op > 99999999) 513 | { 514 | int e = 0; 515 | while (ns_op > 999999) 516 | { 517 | ++e; 518 | ns_op /= 10; 519 | } 520 | out << ns_op << 'e' << e; 521 | } 522 | else 523 | { 524 | out << setw(8) << ns_op; 525 | } 526 | 527 | out << " |"; 528 | 529 | if (baseline == &bm) 530 | { 531 | out << " - |"; 532 | } 533 | else if (baseline) 534 | { 535 | out << setw(7) << fixed << setprecision(3) 536 | << double(bm.total_time_ns) / double(baseline->total_time_ns) << " |"; 537 | } 538 | else 539 | { 540 | // no baseline to compare to 541 | out << " ??? |"; 542 | } 543 | 544 | auto ops_per_sec = ps.first * (1000000000.0 / double(bm.total_time_ns)); 545 | out << setw(11) << fixed << setprecision(1) << ops_per_sec << "\n"; 546 | } 547 | } 548 | out.put('\n'); 549 | } 550 | } 551 | 552 | void to_text_concise(std::ostream& out) 553 | { 554 | using namespace std; 555 | for (auto& suite : suites) 556 | { 557 | if (suite.name) 558 | { 559 | out << "## " << suite.name << ":\n"; 560 | } 561 | 562 | out.put('\n'); 563 | out << 564 | " Name (* = baseline) | ns/op | Baseline | Ops/second\n"; 565 | out << 566 | "--------------------------|--------:|---------:|-----------:\n"; 567 | 568 | const benchmark* baseline = nullptr; 569 | for (auto& bm : suite.benchmarks) 570 | { 571 | if (bm.is_baseline) 572 | { 573 | baseline = &bm; 574 | break; 575 | } 576 | } 577 | I_PICOBENCH_ASSERT(baseline); 578 | int64_t baseline_total_time = 0; 579 | int baseline_total_iterations = 0; 580 | for (auto& d : baseline->data) 581 | { 582 | baseline_total_time += d.total_time_ns; 583 | baseline_total_iterations += d.dimension; 584 | } 585 | int64_t baseline_ns_per_op = baseline_total_time / baseline_total_iterations; 586 | 587 | for (auto& bm : suite.benchmarks) 588 | { 589 | out << ' ' << bm.name; 590 | auto pad = 24 - int(strlen(bm.name)); 591 | if (bm.is_baseline) 592 | { 593 | out << " *"; 594 | pad -= 2; 595 | } 596 | for (int i = 0; i < pad; ++i) { 597 | out.put(' '); 598 | } 599 | 600 | int64_t total_time = 0; 601 | int total_iterations = 0; 602 | for (auto& d : bm.data) 603 | { 604 | total_time += d.total_time_ns; 605 | total_iterations += d.dimension; 606 | } 607 | int64_t ns_per_op = total_time / total_iterations; 608 | 609 | out << " |" << setw(8) << ns_per_op << " |"; 610 | 611 | if (&bm == baseline) 612 | { 613 | out << " - |"; 614 | } 615 | else 616 | { 617 | out << setw(9) << fixed << setprecision(3) 618 | << double(ns_per_op) / double(baseline_ns_per_op) << " |"; 619 | } 620 | 621 | auto ops_per_sec = total_iterations * (1000000000.0 / double(total_time)); 622 | out << setw(12) << fixed << setprecision(1) << ops_per_sec << "\n"; 623 | } 624 | 625 | out.put('\n'); 626 | } 627 | } 628 | 629 | void to_csv(std::ostream& out, bool header = true) const 630 | { 631 | using namespace std; 632 | 633 | if (header) 634 | { 635 | out << "Suite,Benchmark,b,D,S,\"Total ns\",Result,\"ns/op\",Baseline\n"; 636 | } 637 | 638 | for (auto& suite : suites) 639 | { 640 | const benchmark* baseline = nullptr; 641 | for (auto& bm : suite.benchmarks) 642 | { 643 | if (bm.is_baseline) 644 | { 645 | baseline = &bm; 646 | break; 647 | } 648 | } 649 | I_PICOBENCH_ASSERT(baseline); 650 | 651 | for (auto& bm : suite.benchmarks) 652 | { 653 | for (auto& d : bm.data) 654 | { 655 | if (suite.name) 656 | { 657 | out << '"' << suite.name << '"';; 658 | } 659 | out << ",\"" << bm.name << "\","; 660 | if (&bm == baseline) 661 | { 662 | out << '*'; 663 | } 664 | out << ',' 665 | << d.dimension << ',' 666 | << d.samples << ',' 667 | << d.total_time_ns << ',' 668 | << d.result << ',' 669 | << (d.total_time_ns / d.dimension) << ','; 670 | 671 | if (baseline) 672 | { 673 | for (auto& bd : baseline->data) 674 | { 675 | if (bd.dimension == d.dimension) 676 | { 677 | out << fixed << setprecision(3) << (double(d.total_time_ns) / double(bd.total_time_ns)); 678 | } 679 | } 680 | } 681 | 682 | out << '\n'; 683 | } 684 | } 685 | } 686 | } 687 | 688 | struct problem_space_benchmark 689 | { 690 | const char* name; 691 | bool is_baseline; 692 | int64_t total_time_ns; // fastest sample!!! 693 | result_t result; // result of fastest sample 694 | }; 695 | 696 | static std::map> get_problem_space_view(const suite& s) 697 | { 698 | std::map> res; 699 | for (auto& bm : s.benchmarks) 700 | { 701 | for (auto& d : bm.data) 702 | { 703 | auto& pvbs = res[d.dimension]; 704 | pvbs.push_back({ bm.name, bm.is_baseline, d.total_time_ns, d.result }); 705 | } 706 | } 707 | return res; 708 | } 709 | 710 | private: 711 | }; 712 | 713 | class benchmark_impl : public benchmark 714 | { 715 | public: 716 | benchmark_impl(const char* name, benchmark_proc proc) 717 | : benchmark(name, proc) 718 | {} 719 | 720 | private: 721 | friend class runner; 722 | 723 | // state 724 | std::vector _states; // length is _samples * _state_iterations.size() 725 | std::vector::iterator _istate; 726 | }; 727 | 728 | class picostring 729 | { 730 | public: 731 | picostring() = default; 732 | explicit picostring(const char* text) 733 | : str(text) 734 | , len(int(strlen(text))) 735 | {} 736 | picostring(const char* text, int len) 737 | : str(text) 738 | , len(len) 739 | {} 740 | 741 | const char* str; 742 | int len = 0; 743 | 744 | // checks whether other begins with this string 745 | bool is_start_of(const char* other) const 746 | { 747 | return strncmp(str, other, size_t(len)) == 0; 748 | } 749 | 750 | bool operator==(const picostring& other) const 751 | { 752 | if (len != other.len) return false; 753 | return strncmp(str, other.str, size_t(len)) == 0; 754 | } 755 | 756 | bool operator==(const char* other) const 757 | { 758 | return operator==(picostring(other)); 759 | } 760 | }; 761 | 762 | class null_streambuf : public std::streambuf 763 | { 764 | public: 765 | virtual int overflow(int c) override { return c; } 766 | }; 767 | 768 | struct null_stream : public std::ostream 769 | { 770 | null_stream() : std::ostream(&_buf) {} 771 | private: 772 | null_streambuf _buf; 773 | } cnull; 774 | 775 | enum class report_output_format 776 | { 777 | text, 778 | concise_text, 779 | csv, 780 | }; 781 | 782 | #if !defined(PICOBENCH_DEFAULT_ITERATIONS) 783 | # define PICOBENCH_DEFAULT_ITERATIONS { 8, 64, 512, 4096, 8192 } 784 | #endif 785 | 786 | #if !defined(PICOBENCH_DEFAULT_SAMPLES) 787 | # define PICOBENCH_DEFAULT_SAMPLES 2 788 | #endif 789 | 790 | using benchmarks_vector = std::vector>; 791 | struct rsuite 792 | { 793 | const char* name; 794 | benchmarks_vector benchmarks; 795 | }; 796 | 797 | class registry 798 | { 799 | public: 800 | benchmark& add_benchmark(const char* name, benchmark_proc proc) 801 | { 802 | auto b = new benchmark_impl(name, proc); 803 | benchmarks_for_current_suite().emplace_back(b); 804 | return *b; 805 | } 806 | 807 | void set_suite(const char* name) 808 | { 809 | _current_suite_name = name; 810 | } 811 | 812 | const char*& current_suite_name() 813 | { 814 | return _current_suite_name; 815 | } 816 | 817 | benchmarks_vector& benchmarks_for_current_suite() 818 | { 819 | for (auto& s : _suites) 820 | { 821 | if (s.name == _current_suite_name) 822 | return s.benchmarks; 823 | 824 | if (s.name && _current_suite_name && strcmp(s.name, _current_suite_name) == 0) 825 | return s.benchmarks; 826 | } 827 | _suites.push_back({ _current_suite_name, {} }); 828 | return _suites.back().benchmarks; 829 | } 830 | 831 | protected: 832 | friend class runner; 833 | const char* _current_suite_name = nullptr; 834 | std::vector _suites; 835 | }; 836 | 837 | registry& g_registry() 838 | { 839 | static registry r; 840 | return r; 841 | } 842 | 843 | class runner : public registry 844 | { 845 | public: 846 | runner(bool local = false) 847 | : _default_state_iterations(PICOBENCH_DEFAULT_ITERATIONS) 848 | , _default_samples(PICOBENCH_DEFAULT_SAMPLES) 849 | { 850 | if (!local) 851 | { 852 | _suites = std::move(g_registry()._suites); 853 | } 854 | } 855 | 856 | int run(int benchmark_random_seed = -1) 857 | { 858 | if (should_run()) 859 | { 860 | run_benchmarks(benchmark_random_seed); 861 | auto report = generate_report(); 862 | std::ostream* out = _stdout; 863 | std::ofstream fout; 864 | if (preferred_output_filename()) 865 | { 866 | fout.open(preferred_output_filename()); 867 | if (!fout.is_open()) 868 | { 869 | std::cerr << "Error: Could not open output file `" << preferred_output_filename() << "`\n"; 870 | return 1; 871 | } 872 | out = &fout; 873 | } 874 | 875 | switch (preferred_output_format()) 876 | { 877 | case report_output_format::text: 878 | report.to_text(*out); 879 | break; 880 | case report_output_format::concise_text: 881 | report.to_text_concise(*out); 882 | break; 883 | case report_output_format::csv: 884 | report.to_csv(*out); 885 | break; 886 | } 887 | } 888 | return error(); 889 | } 890 | 891 | void run_benchmarks(int random_seed = -1) 892 | { 893 | I_PICOBENCH_ASSERT(_error == no_error && _should_run); 894 | 895 | if (random_seed == -1) 896 | { 897 | random_seed = int(std::random_device()()); 898 | } 899 | 900 | std::minstd_rand rnd(random_seed); 901 | 902 | // vector of all benchmarks 903 | std::vector benchmarks; 904 | for (auto& suite : _suites) 905 | { 906 | // also identify a baseline in this loop 907 | // if there is no explicit one, set the first one as a baseline 908 | bool found_baseline = false; 909 | for (auto irb = suite.benchmarks.begin(); irb != suite.benchmarks.end(); ++irb) 910 | { 911 | auto& rb = *irb; 912 | rb->_states.clear(); // clear states so we can safely call run_benchmarks multiple times 913 | benchmarks.push_back(rb.get()); 914 | if (rb->_baseline) 915 | { 916 | found_baseline = true; 917 | } 918 | 919 | #if !defined(PICOBENCH_STD_FUNCTION_BENCHMARKS) 920 | // check for same func 921 | for (auto ib = irb+1; ib != suite.benchmarks.end(); ++ib) 922 | { 923 | auto& b = *ib; 924 | if (rb->_proc == b->_proc && rb->_user_data == b->_user_data) 925 | { 926 | *_stdwarn << "Warning: " << rb->name() << " and " << b->name() 927 | << " are benchmarks of the same function.\n"; 928 | } 929 | } 930 | #endif 931 | } 932 | 933 | if (!found_baseline && !suite.benchmarks.empty()) 934 | { 935 | suite.benchmarks.front()->_baseline = true; 936 | } 937 | } 938 | 939 | // initialize benchmarks 940 | for (auto b : benchmarks) 941 | { 942 | const std::vector& state_iterations = 943 | b->_state_iterations.empty() ? 944 | _default_state_iterations : 945 | b->_state_iterations; 946 | 947 | if (b->_samples == 0) 948 | b->_samples = _default_samples; 949 | 950 | b->_states.reserve(state_iterations.size() * size_t(b->_samples)); 951 | 952 | // fill states while random shuffling them 953 | for (auto iters : state_iterations) 954 | { 955 | for (int i = 0; i < b->_samples; ++i) 956 | { 957 | auto index = rnd() % (b->_states.size() + 1); 958 | auto pos = b->_states.begin() + long(index); 959 | b->_states.emplace(pos, iters, b->_user_data); 960 | } 961 | } 962 | 963 | b->_istate = b->_states.begin(); 964 | } 965 | 966 | #if !defined(PICOBENCH_DONT_BIND_TO_ONE_CORE) 967 | // set thread affinity to first cpu 968 | // so the high resolution clock doesn't miss cycles 969 | { 970 | #if defined(_WIN32) 971 | SetThreadAffinityMask(GetCurrentThread(), 1); 972 | #elif defined(__APPLE__) 973 | thread_affinity_policy_data_t policy = {0}; 974 | thread_policy_set( 975 | pthread_mach_thread_np(pthread_self()), 976 | THREAD_AFFINITY_POLICY, 977 | (thread_policy_t)&policy, 1); 978 | #else 979 | cpu_set_t cpuset; 980 | CPU_ZERO(&cpuset); 981 | CPU_SET(0, &cpuset); 982 | 983 | sched_setaffinity(0, sizeof(cpu_set_t), &cpuset); 984 | #endif 985 | } 986 | #endif 987 | 988 | // we run a random benchmark from it incrementing _istate for each 989 | // when _istate reaches _states.end(), we erase the benchmark 990 | // when the vector becomes empty, we're done 991 | while (!benchmarks.empty()) 992 | { 993 | auto i = benchmarks.begin() + long(rnd() % benchmarks.size()); 994 | auto& b = *i; 995 | 996 | b->_proc(*b->_istate); 997 | 998 | ++b->_istate; 999 | 1000 | if (b->_istate == b->_states.end()) 1001 | { 1002 | benchmarks.erase(i); 1003 | } 1004 | } 1005 | } 1006 | 1007 | // function to compare results 1008 | template > 1009 | report generate_report(CompareResult cmp = std::equal_to()) const 1010 | { 1011 | report rpt; 1012 | 1013 | rpt.suites.resize(_suites.size()); 1014 | auto rpt_suite = rpt.suites.begin(); 1015 | 1016 | for (auto& suite : _suites) 1017 | { 1018 | rpt_suite->name = suite.name; 1019 | 1020 | // build benchmark view 1021 | rpt_suite->benchmarks.resize(suite.benchmarks.size()); 1022 | auto rpt_benchmark = rpt_suite->benchmarks.begin(); 1023 | 1024 | for (auto& b : suite.benchmarks) 1025 | { 1026 | rpt_benchmark->name = b->_name; 1027 | rpt_benchmark->is_baseline = b->_baseline; 1028 | 1029 | const std::vector& state_iterations = 1030 | b->_state_iterations.empty() ? 1031 | _default_state_iterations : 1032 | b->_state_iterations; 1033 | 1034 | rpt_benchmark->data.reserve(state_iterations.size()); 1035 | for (auto d : state_iterations) 1036 | { 1037 | rpt_benchmark->data.push_back({d, 0, 0ll, result_t(0)}); 1038 | } 1039 | 1040 | for (auto& state : b->_states) 1041 | { 1042 | for (auto& d : rpt_benchmark->data) 1043 | { 1044 | if (state.iterations() == d.dimension) 1045 | { 1046 | if (d.total_time_ns == 0 || d.total_time_ns > state.duration_ns()) 1047 | { 1048 | d.total_time_ns = state.duration_ns(); 1049 | d.result = state.result(); 1050 | } 1051 | 1052 | if (_compare_results_across_samples) 1053 | { 1054 | if (d.result != state.result() && !cmp(d.result, state.result())) 1055 | { 1056 | *_stderr << "Error: Two samples of " << b->name() << " @" << d.dimension << " produced different results: " 1057 | << d.result << " and " << state.result() << '\n'; 1058 | _error = error_sample_compare; 1059 | } 1060 | } 1061 | 1062 | ++d.samples; 1063 | } 1064 | } 1065 | } 1066 | 1067 | #if defined(PICOBENCH_DEBUG) 1068 | for (auto& d : rpt_benchmark->data) 1069 | { 1070 | I_PICOBENCH_ASSERT(d.samples == b->_samples); 1071 | } 1072 | #endif 1073 | 1074 | ++rpt_benchmark; 1075 | } 1076 | 1077 | ++rpt_suite; 1078 | } 1079 | 1080 | if (_compare_results_across_benchmarks) 1081 | { 1082 | for(auto& suite : rpt.suites) 1083 | { 1084 | auto psview = report::get_problem_space_view(suite); 1085 | 1086 | for (auto& space : psview) 1087 | { 1088 | I_PICOBENCH_ASSERT(!space.second.empty()); 1089 | 1090 | if (space.second.size() == 1) 1091 | { 1092 | auto& b = space.second.front(); 1093 | *_stdwarn << "Warning: Benchmark " << b.name << " @" << space.first 1094 | << " has a single instance and cannot be compared to others.\n"; 1095 | continue; 1096 | } 1097 | 1098 | auto result0 = space.second.front().result; 1099 | 1100 | for (auto& b : space.second) 1101 | { 1102 | if (result0 != b.result && !cmp(result0, b.result)) 1103 | { 1104 | auto& f = space.second.front(); 1105 | *_stderr << "Error: Benchmarks " << f.name << " and " << b.name 1106 | << " @" << space.first << " produce different results: " 1107 | << result0 << " and " << b.result << '\n'; 1108 | _error = error_benchmark_compare; 1109 | } 1110 | } 1111 | } 1112 | } 1113 | } 1114 | 1115 | return rpt; 1116 | } 1117 | 1118 | void set_default_state_iterations(const std::vector& data) 1119 | { 1120 | _default_state_iterations = data; 1121 | } 1122 | 1123 | const std::vector& default_state_iterations() const 1124 | { 1125 | return _default_state_iterations; 1126 | } 1127 | 1128 | void set_default_samples(int n) 1129 | { 1130 | _default_samples = n; 1131 | } 1132 | 1133 | int default_samples() const 1134 | { 1135 | return _default_samples; 1136 | } 1137 | 1138 | void add_cmd_opt(const char* cmd, const char* arg_desc, const char* cmd_desc, bool(*handler)(uintptr_t, const char*), uintptr_t user_data = 0) 1139 | { 1140 | cmd_line_option opt; 1141 | opt.cmd = picostring(cmd); 1142 | opt.arg_desc = picostring(arg_desc); 1143 | opt.desc = cmd_desc; 1144 | opt.handler = nullptr; 1145 | opt.user_data = user_data; 1146 | opt.user_handler = handler; 1147 | _opts.push_back(opt); 1148 | } 1149 | 1150 | // returns false if there were errors parsing the command line 1151 | // all args starting with prefix are parsed 1152 | // the others are ignored 1153 | bool parse_cmd_line(int argc, const char* const argv[], const char* cmd_prefix = "-") 1154 | { 1155 | _cmd_prefix = picostring(cmd_prefix); 1156 | 1157 | if (!_has_opts) 1158 | { 1159 | _opts.emplace_back("-iters=", "", 1160 | "Sets default iterations for benchmarks", 1161 | &runner::cmd_iters); 1162 | _opts.emplace_back("-samples=", "", 1163 | "Sets default number of samples for benchmarks", 1164 | &runner::cmd_samples); 1165 | _opts.emplace_back("-out-fmt=", "", 1166 | "Outputs text or concise or csv", 1167 | &runner::cmd_out_fmt); 1168 | _opts.emplace_back("-output=", "", 1169 | "Sets output filename or `stdout`", 1170 | &runner::cmd_output); 1171 | _opts.emplace_back("-compare-results", "", 1172 | "Compare benchmark results", 1173 | &runner::cmd_compare_results); 1174 | _opts.emplace_back("-no-run", "", 1175 | "Doesn't run benchmarks", 1176 | &runner::cmd_no_run); 1177 | _opts.emplace_back("-run-suite=", "", 1178 | "Runs only benchmarks from suite", 1179 | &runner::cmd_run_suite); 1180 | _opts.emplace_back("-run-only=", "", 1181 | "Runs only selected benchmarks", 1182 | &runner::cmd_run_only); 1183 | _opts.emplace_back("-list", "", 1184 | "Lists available benchmarks", 1185 | &runner::cmd_list); 1186 | _opts.emplace_back("-version", "", 1187 | "Show version info", 1188 | &runner::cmd_version); 1189 | _opts.emplace_back("-help", "", 1190 | "Prints help", 1191 | &runner::cmd_help); 1192 | _has_opts = true; 1193 | } 1194 | 1195 | for (int i = 1; i < argc; ++i) 1196 | { 1197 | if (!_cmd_prefix.is_start_of(argv[i])) 1198 | continue; 1199 | 1200 | auto arg = argv[i] + _cmd_prefix.len; 1201 | 1202 | bool found = false; 1203 | for (auto& opt : _opts) 1204 | { 1205 | if (opt.cmd.is_start_of(arg)) 1206 | { 1207 | found = true; 1208 | bool success = false; 1209 | if (opt.handler) 1210 | { 1211 | success = (this->*opt.handler)(arg + opt.cmd.len); 1212 | } 1213 | else 1214 | { 1215 | I_PICOBENCH_ASSERT(opt.user_handler); 1216 | success = opt.user_handler(opt.user_data, arg + opt.cmd.len); 1217 | } 1218 | 1219 | if (!success) 1220 | { 1221 | *_stderr << "Error: Bad command-line argument: " << argv[i] << "\n"; 1222 | _error = error_bad_cmd_line_argument; 1223 | return false; 1224 | } 1225 | break; 1226 | } 1227 | } 1228 | 1229 | if (!found) 1230 | { 1231 | *_stderr << "Error: Unknown command-line argument: " << argv[i] << "\n"; 1232 | _error = error_unknown_cmd_line_argument; 1233 | return false; 1234 | } 1235 | } 1236 | 1237 | return true; 1238 | } 1239 | 1240 | void set_should_run(bool set) { _should_run = set; } 1241 | bool should_run() const { return _error == no_error && _should_run; } 1242 | void set_error(error_t e) { _error = e; } 1243 | error_t error() const { return _error; } 1244 | 1245 | void set_output_streams(std::ostream& out, std::ostream& err) 1246 | { 1247 | _stdout = &out; 1248 | _stderr = &err; 1249 | _stdwarn = &out; 1250 | } 1251 | 1252 | void set_preferred_output_format(report_output_format fmt) { _output_format = fmt; } 1253 | report_output_format preferred_output_format() const { return _output_format; } 1254 | 1255 | // can be nullptr (run will interpret it as stdout) 1256 | void set_preferred_output_filename(const char* path) { _output_file = path; } 1257 | const char* preferred_output_filename() const { return _output_file; } 1258 | 1259 | void set_compare_results_across_samples(bool b) { _compare_results_across_samples = b; } 1260 | bool compare_results_across_samples() const { return _compare_results_across_samples; } 1261 | 1262 | void set_compare_results_across_benchmarks(bool b) { _compare_results_across_benchmarks = b; } 1263 | bool compare_results_across_benchmarks() const { return _compare_results_across_benchmarks; } 1264 | 1265 | private: 1266 | // runner's suites and benchmarks come from its parent: registry 1267 | 1268 | // state and configuration 1269 | mutable error_t _error = no_error; 1270 | bool _should_run = true; 1271 | 1272 | bool _compare_results_across_samples = false; 1273 | bool _compare_results_across_benchmarks = false; 1274 | 1275 | report_output_format _output_format = report_output_format::text; 1276 | const char* _output_file = nullptr; // nullptr means stdout 1277 | 1278 | std::ostream* _stdout = &std::cout; 1279 | std::ostream* _stderr = &std::cerr; 1280 | std::ostream* _stdwarn = &std::cout; 1281 | 1282 | // default data 1283 | 1284 | // default iterations per state per benchmark 1285 | std::vector _default_state_iterations; 1286 | 1287 | // default samples per benchmark 1288 | int _default_samples; 1289 | 1290 | // command line parsing 1291 | picostring _cmd_prefix; 1292 | typedef bool (runner::*cmd_handler)(const char*); // internal handler 1293 | typedef bool(*ext_handler)(uintptr_t user_data, const char* cmd_line); // external (user) handler 1294 | struct cmd_line_option 1295 | { 1296 | cmd_line_option() = default; 1297 | cmd_line_option(const char* c, const char* a, const char* d, cmd_handler h) 1298 | : cmd(c) 1299 | , arg_desc(a) 1300 | , desc(d) 1301 | , handler(h) 1302 | , user_data(0) 1303 | , user_handler(nullptr) 1304 | {} 1305 | picostring cmd; 1306 | picostring arg_desc; 1307 | const char* desc; 1308 | cmd_handler handler; // may be nullptr for external handlers 1309 | uintptr_t user_data; // passed as an argument to user handlers 1310 | ext_handler user_handler; 1311 | }; 1312 | bool _has_opts = false; // have opts been added to list 1313 | std::vector _opts; 1314 | 1315 | bool cmd_iters(const char* line) 1316 | { 1317 | std::vector iters; 1318 | auto p = line; 1319 | while (true) 1320 | { 1321 | auto i = int(strtoul(p, nullptr, 10)); 1322 | if (i <= 0) return false; 1323 | iters.push_back(i); 1324 | p = strchr(p + 1, ','); 1325 | if (!p) break; 1326 | ++p; 1327 | } 1328 | if (iters.empty()) return false; 1329 | _default_state_iterations = iters; 1330 | return true; 1331 | } 1332 | 1333 | bool cmd_samples(const char* line) 1334 | { 1335 | int samples = int(strtol(line, nullptr, 10)); 1336 | if (samples <= 0) return false; 1337 | _default_samples = samples; 1338 | return true; 1339 | } 1340 | 1341 | bool cmd_no_run(const char* line) 1342 | { 1343 | if (*line) return false; 1344 | _should_run = false; 1345 | return true; 1346 | } 1347 | 1348 | bool cmd_run_suite(const char* line) 1349 | { 1350 | auto new_end = std::remove_if(_suites.begin(), _suites.end(), [line](const rsuite& s) { 1351 | return !s.name || strcmp(s.name, line) != 0; 1352 | }); 1353 | _suites.erase(new_end, _suites.end()); 1354 | return true; 1355 | } 1356 | 1357 | bool cmd_run_only(const char* line) 1358 | { 1359 | std::vector names; 1360 | 1361 | auto p = line; 1362 | while (true) 1363 | { 1364 | const char* q = strchr(p, ','); 1365 | if (!q) q = p + strlen(p); 1366 | names.emplace_back(p, int(q - p)); 1367 | if (!*q) break; 1368 | p = q + 1; 1369 | } 1370 | 1371 | for (auto& s : _suites) 1372 | { 1373 | auto new_end = std::remove_if(s.benchmarks.begin(), s.benchmarks.end(), [&names](const std::unique_ptr& b) { 1374 | auto f = std::find(names.begin(), names.end(), b->name()); 1375 | return f == names.end(); 1376 | }); 1377 | s.benchmarks.erase(new_end, s.benchmarks.end()); 1378 | } 1379 | return true; 1380 | } 1381 | 1382 | bool cmd_list(const char* line) 1383 | { 1384 | if (*line) return false; 1385 | _should_run = false; 1386 | for (auto& suite : _suites) 1387 | { 1388 | if (suite.name) 1389 | { 1390 | *_stdout << " " << suite.name << ":\n"; 1391 | } 1392 | else 1393 | { 1394 | *_stdout << " :\n"; 1395 | } 1396 | for (auto& bench : suite.benchmarks) 1397 | { 1398 | *_stdout << " " << bench->name() << "\n"; 1399 | } 1400 | } 1401 | _should_run = false; 1402 | return true; 1403 | } 1404 | 1405 | bool cmd_version(const char* line) 1406 | { 1407 | if (*line) return false; 1408 | *_stdout << "picobench " PICOBENCH_VERSION_STR << "\n"; 1409 | _should_run = false; 1410 | return true; 1411 | } 1412 | 1413 | bool cmd_help(const char* line) 1414 | { 1415 | if (*line) return false; 1416 | cmd_version(line); 1417 | auto& cout = *_stdout; 1418 | for (auto& opt : _opts) 1419 | { 1420 | cout << ' ' << _cmd_prefix.str << opt.cmd.str << opt.arg_desc.str; 1421 | int w = 27 - (_cmd_prefix.len + opt.cmd.len + opt.arg_desc.len); 1422 | for (int i = 0; i < w; ++i) 1423 | { 1424 | cout.put(' '); 1425 | } 1426 | cout << opt.desc << "\n"; 1427 | } 1428 | _should_run = false; 1429 | return true; 1430 | } 1431 | 1432 | bool cmd_out_fmt(const char* line) 1433 | { 1434 | if (strcmp(line, "txt") == 0) 1435 | { 1436 | _output_format = report_output_format::text; 1437 | } 1438 | else if (strcmp(line, "con") == 0) 1439 | { 1440 | _output_format = report_output_format::concise_text; 1441 | } 1442 | else if (strcmp(line, "csv") == 0) 1443 | { 1444 | _output_format = report_output_format::csv; 1445 | } 1446 | else 1447 | { 1448 | return false; 1449 | } 1450 | return true; 1451 | } 1452 | 1453 | bool cmd_output(const char* line) 1454 | { 1455 | if (strcmp(line, "stdout") != 0) 1456 | { 1457 | _output_file = line; 1458 | } 1459 | else 1460 | { 1461 | _output_file = nullptr; 1462 | } 1463 | return true; 1464 | } 1465 | 1466 | bool cmd_compare_results(const char* line) 1467 | { 1468 | if (*line) return false; 1469 | _compare_results_across_samples = true; 1470 | _compare_results_across_benchmarks = true; 1471 | return true; 1472 | } 1473 | }; 1474 | 1475 | class local_runner : public runner 1476 | { 1477 | public: 1478 | local_runner() : runner(true) 1479 | {} 1480 | }; 1481 | 1482 | // } // anonymous namespace 1483 | 1484 | benchmark::benchmark(const char* name, benchmark_proc proc) 1485 | : _name(name) 1486 | , _proc(proc) 1487 | {} 1488 | 1489 | benchmark& global_registry::new_benchmark(const char* name, benchmark_proc proc) 1490 | { 1491 | return g_registry().add_benchmark(name, proc); 1492 | } 1493 | 1494 | int global_registry::set_bench_suite(const char* name) 1495 | { 1496 | g_registry().current_suite_name() = name; 1497 | return 0; 1498 | } 1499 | 1500 | #if (defined(_MSC_VER) || defined(__MINGW32__)) && !defined(PICOBENCH_TEST) 1501 | 1502 | static const long long high_res_clock_freq = []() -> long long 1503 | { 1504 | LARGE_INTEGER frequency; 1505 | QueryPerformanceFrequency(&frequency); 1506 | return frequency.QuadPart; 1507 | }(); 1508 | 1509 | high_res_clock::time_point high_res_clock::now() 1510 | { 1511 | LARGE_INTEGER t; 1512 | QueryPerformanceCounter(&t); 1513 | return time_point(duration((t.QuadPart * rep(period::den)) / high_res_clock_freq)); 1514 | } 1515 | #endif 1516 | } 1517 | 1518 | #endif 1519 | 1520 | #if defined(PICOBENCH_IMPLEMENT_MAIN) 1521 | int main(int argc, char* argv[]) 1522 | { 1523 | PICOBENCH_NAMESPACE::runner r; 1524 | r.parse_cmd_line(argc, argv); 1525 | return r.run(); 1526 | } 1527 | #endif 1528 | 1529 | #if defined(PICOBENCH_TEST) 1530 | 1531 | // fake time keeping functions for the tests 1532 | namespace PICOBENCH_NAMESPACE 1533 | { 1534 | namespace test 1535 | { 1536 | 1537 | 1538 | void this_thread_sleep_for_ns(uint64_t ns); 1539 | 1540 | template 1541 | void this_thread_sleep_for(const std::chrono::duration& duration) 1542 | { 1543 | this_thread_sleep_for_ns(std::chrono::duration_cast(duration).count()); 1544 | } 1545 | 1546 | #if defined(PICOBENCH_IMPLEMENT) 1547 | static struct fake_time 1548 | { 1549 | uint64_t now; 1550 | } the_time; 1551 | 1552 | void this_thread_sleep_for_ns(uint64_t ns) 1553 | { 1554 | the_time.now += ns; 1555 | } 1556 | 1557 | } // namespace test 1558 | 1559 | high_res_clock::time_point high_res_clock::now() 1560 | { 1561 | auto ret = time_point(duration(test::the_time.now)); 1562 | return ret; 1563 | #endif 1564 | } // dual purpose closing brace 1565 | } 1566 | #endif 1567 | --------------------------------------------------------------------------------