├── VERSION ├── cmake └── third-party │ ├── COMMON_LIB.version │ ├── YAML_CPP.version │ ├── API_BRIDGE.version │ ├── API_BRIDGE_check.cmake │ ├── API_BRIDGE.cmake │ ├── COMMON_LIB.cmake │ └── YAML_CPP.cmake ├── CODEOWNERS ├── HEADER ├── docsrc ├── doxyinputs.in ├── images │ ├── architecture.png │ ├── function_pipeline.png │ ├── architecture_diagram.png │ ├── apibridge_load_diagram.png │ ├── apibridge_decode_diagram.png │ ├── apibridge_decrypt_diagram.png │ ├── apibridge_encode_diagram.png │ ├── apibridge_encrypt_diagram.png │ ├── apibridge_operate_diagram.png │ ├── apibridge_store_diagram.png │ ├── architecture_diagram_backend_1.png │ └── architecture_diagram_frontend_1.png ├── examples │ ├── generic_wl_example │ │ └── generic_wl_tutorial │ │ │ ├── generic_wl_config.md │ │ │ ├── generic_wl_dataset.md │ │ │ ├── generic_wl_cmakelists.md │ │ │ ├── dataset.csv │ │ │ ├── include │ │ │ ├── generic_wl_engine.h │ │ │ └── generic_wl_benchmark.h │ │ │ ├── config.yaml │ │ │ ├── src │ │ │ └── generic_wl_engine.cpp │ │ │ └── CMakeLists.txt │ └── backend_tutorials │ │ ├── seal │ │ ├── tutorial_backend │ │ │ ├── include │ │ │ │ ├── tutorial_error_seal.h │ │ │ │ └── tutorial_engine_seal.h │ │ │ ├── src │ │ │ │ └── tutorial_engine_seal.cpp │ │ │ └── CMakeLists.txt │ │ ├── backend_tutorial_files_seal.md │ │ └── backend_tutorial_seal.md │ │ ├── palisade │ │ ├── tutorial_backend │ │ │ ├── include │ │ │ │ ├── tutorial_error_palisade.h │ │ │ │ └── tutorial_engine_palisade.h │ │ │ ├── src │ │ │ │ └── tutorial_engine_palisade.cpp │ │ │ └── CMakeLists.txt │ │ ├── backend_tutorial_files_palisade.md │ │ └── backend_tutorial_palisade.md │ │ └── backend_tutorial.md ├── categories_overview.md ├── tests_overview.md ├── dataset_loader_overview.md ├── function_pipeline.md ├── report_compiler_overview.md ├── frontend_overview.md ├── intro_workload_add.md ├── backend_overview.md ├── test_harness_overview.md ├── categories │ └── cat_latency.md ├── overview_file_format.md └── tests │ ├── elementwise_mult.md │ └── elementwise_add.md ├── report_gen ├── CMakeLists.txt ├── report_compiler │ └── CMakeLists.txt ├── report_gen_ar │ └── CMakeLists.txt ├── report_gen_lib │ └── CMakeLists.txt ├── report_compiler_lib │ ├── CMakeLists.txt │ └── include │ │ ├── hebench_report_overview_header.h │ │ └── hebench_report_compiler.h └── include │ └── hebench_report_types.h ├── .github ├── validation-ci │ ├── invalid_so.sh │ ├── backend_symlink_validation.sh │ ├── invalid_yml.sh │ ├── config_symlink_validation.sh │ ├── backend_tmp_validation.sh │ ├── config_tmp_validation.sh │ ├── dasetloader_invalid_tag.sh │ ├── dasetloader_invalid_int_0.sh │ ├── dasetloader_invalid_int_1.sh │ ├── dasetloader_missing_type.sh │ ├── dasetloader_sample0.sh │ ├── dasetloader_file_not_found.sh │ ├── dasetloader_invalid_type.sh │ ├── dasetloader_sample1.sh │ └── dasetloader_sample2.sh ├── ISSUE_TEMPLATE │ ├── feature_request.md │ └── bug_report.md ├── workflows │ ├── validate_testharness_output.sh │ ├── cmake.yml │ ├── documentation.yml │ └── install-requirements.sh ├── PULL_REQUEST_TEMPLATE.md └── validation-ci-ssi │ ├── testharness_gen_PSI_X_bigger_than_Y.sh │ ├── testharness_gen_PSI_Y_bigger_than_X.sh │ ├── testharness_dsl_PSI_X_bigger_than_Y.sh │ ├── testharness_dsl_PSI_k_smaller_than_biggest_element.sh │ ├── testharness_dsl_PSI_k_mismatch_between_config_and_dataset.sh │ ├── testharness_dsl_PSI_X_in_Y.sh │ ├── testharness_dsl_PSI_Y_bigger_than_X.sh │ ├── testharness_dsl_PSI_Y_in_X.sh │ └── testharness_dsl_PSI_X_intersect_Y_is_empty.sh ├── scripts ├── doxygen_1.9.1_install.sh └── read_version.sh ├── dataset_loader ├── CMakeLists.txt ├── tests │ └── CMakeLists.txt ├── src │ ├── main.cpp │ └── hebench_dataset_loader_functional_test.cpp └── include │ ├── hebench_dataset_loader_functional_test.h │ └── hebench_dataset_loader.h ├── test_harness ├── include │ ├── hebench_types_harness.h │ ├── hebench_version.h.in │ ├── hebench_benchmark_factory.h │ └── inl │ │ └── hebench_utilities_harness.inl ├── benchmarks │ ├── categories │ │ └── include │ │ │ ├── hebench_benchmark_latency.h │ │ │ └── hebench_benchmark_offline.h │ ├── DotProduct │ │ ├── latency │ │ │ └── include │ │ │ │ └── hebench_dotproduct_l.h │ │ └── offline │ │ │ └── include │ │ │ └── hebench_dotproduct_o.h │ ├── EltwiseAdd │ │ ├── latency │ │ │ └── include │ │ │ │ └── hebench_eltwiseadd_l.h │ │ └── offline │ │ │ └── include │ │ │ └── hebench_eltwiseadd_o.h │ ├── EltwiseMult │ │ ├── latency │ │ │ └── include │ │ │ │ └── hebench_eltwisemult_l.h │ │ └── offline │ │ │ └── include │ │ │ └── hebench_eltwisemult_o.h │ ├── MatrixMultiply │ │ ├── latency │ │ │ └── include │ │ │ │ └── hebench_matmult_l.h │ │ └── offline │ │ │ └── include │ │ │ └── hebench_matmult_o.h │ ├── Generic │ │ ├── latency │ │ │ └── include │ │ │ │ └── hebench_genericwl_l.h │ │ └── offline │ │ │ └── include │ │ │ └── hebench_genericwl_o.h │ ├── LogisticRegression │ │ ├── latency │ │ │ └── include │ │ │ │ └── hebench_logreg_l.h │ │ └── offline │ │ │ └── include │ │ │ └── hebench_logreg_o.h │ └── SimpleSetIntersection │ │ ├── latency │ │ └── include │ │ │ └── hebench_simple_set_intersection_l.h │ │ └── offline │ │ └── include │ │ └── hebench_simple_set_intersection_o.h └── src │ └── hebench_utilities_harness.cpp ├── .pre-commit-config.yaml ├── .clang-format └── CMakeLists.txt /VERSION: -------------------------------------------------------------------------------- 1 | v 2 | 0 3 | 9 4 | 0 5 | beta 6 | -------------------------------------------------------------------------------- /cmake/third-party/COMMON_LIB.version: -------------------------------------------------------------------------------- 1 | v0.1.5 2 | -------------------------------------------------------------------------------- /cmake/third-party/YAML_CPP.version: -------------------------------------------------------------------------------- 1 | yaml-cpp-0.6.3 2 | -------------------------------------------------------------------------------- /cmake/third-party/API_BRIDGE.version: -------------------------------------------------------------------------------- 1 | v 2 | 0 3 | 8 4 | 0 5 | beta 6 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | # Default codeowners for all files 2 | * @sidezrw @kylanerace @jobottle 3 | -------------------------------------------------------------------------------- /HEADER: -------------------------------------------------------------------------------- 1 | Copyright (C) 2021 Intel Corporation 2 | SPDX-License-Identifier: Apache-2.0 3 | -------------------------------------------------------------------------------- /docsrc/doxyinputs.in: -------------------------------------------------------------------------------- 1 | ../test_harness/ 2 | ../dataset_loader/ 3 | ../report_gen/ 4 | ../docsrc/ 5 | -------------------------------------------------------------------------------- /docsrc/images/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hebench/frontend/HEAD/docsrc/images/architecture.png -------------------------------------------------------------------------------- /docsrc/images/function_pipeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hebench/frontend/HEAD/docsrc/images/function_pipeline.png -------------------------------------------------------------------------------- /docsrc/images/architecture_diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hebench/frontend/HEAD/docsrc/images/architecture_diagram.png -------------------------------------------------------------------------------- /docsrc/images/apibridge_load_diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hebench/frontend/HEAD/docsrc/images/apibridge_load_diagram.png -------------------------------------------------------------------------------- /docsrc/images/apibridge_decode_diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hebench/frontend/HEAD/docsrc/images/apibridge_decode_diagram.png -------------------------------------------------------------------------------- /docsrc/images/apibridge_decrypt_diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hebench/frontend/HEAD/docsrc/images/apibridge_decrypt_diagram.png -------------------------------------------------------------------------------- /docsrc/images/apibridge_encode_diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hebench/frontend/HEAD/docsrc/images/apibridge_encode_diagram.png -------------------------------------------------------------------------------- /docsrc/images/apibridge_encrypt_diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hebench/frontend/HEAD/docsrc/images/apibridge_encrypt_diagram.png -------------------------------------------------------------------------------- /docsrc/images/apibridge_operate_diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hebench/frontend/HEAD/docsrc/images/apibridge_operate_diagram.png -------------------------------------------------------------------------------- /docsrc/images/apibridge_store_diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hebench/frontend/HEAD/docsrc/images/apibridge_store_diagram.png -------------------------------------------------------------------------------- /docsrc/images/architecture_diagram_backend_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hebench/frontend/HEAD/docsrc/images/architecture_diagram_backend_1.png -------------------------------------------------------------------------------- /docsrc/images/architecture_diagram_frontend_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hebench/frontend/HEAD/docsrc/images/architecture_diagram_frontend_1.png -------------------------------------------------------------------------------- /docsrc/examples/generic_wl_example/generic_wl_tutorial/generic_wl_config.md: -------------------------------------------------------------------------------- 1 | config.yaml {#generic_wl_config_yaml} 2 | --------- 3 | 4 | \verbinclude docsrc/examples/generic_wl_example/generic_wl_tutorial/config.yaml 5 | -------------------------------------------------------------------------------- /docsrc/examples/generic_wl_example/generic_wl_tutorial/generic_wl_dataset.md: -------------------------------------------------------------------------------- 1 | dataset.csv {#generic_wl_dataset_csv} 2 | --------- 3 | 4 | \verbinclude docsrc/examples/generic_wl_example/generic_wl_tutorial/dataset.csv 5 | -------------------------------------------------------------------------------- /docsrc/examples/generic_wl_example/generic_wl_tutorial/generic_wl_cmakelists.md: -------------------------------------------------------------------------------- 1 | CMakeLists.txt {#generic_wl_cmakelists} 2 | --------- 3 | 4 | \verbinclude docsrc/examples/generic_wl_example/generic_wl_tutorial/CMakeLists.txt 5 | -------------------------------------------------------------------------------- /docsrc/examples/backend_tutorials/seal/tutorial_backend/include/tutorial_error_seal.h: -------------------------------------------------------------------------------- 1 | 2 | // Copyright (C) 2021 Intel Corporation 3 | // SPDX-License-Identifier: Apache-2.0 4 | 5 | #pragma once 6 | 7 | /// @cond 8 | 9 | #define TUTORIAL_ECODE_SEAL_ERROR 2 10 | 11 | /// @endcond 12 | -------------------------------------------------------------------------------- /report_gen/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Intel Corporation 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | project(report_gen) 5 | 6 | add_subdirectory(report_gen_lib) 7 | add_subdirectory(report_gen_ar) 8 | add_subdirectory(report_compiler_lib) 9 | add_subdirectory(report_compiler) 10 | -------------------------------------------------------------------------------- /docsrc/examples/backend_tutorials/palisade/tutorial_backend/include/tutorial_error_palisade.h: -------------------------------------------------------------------------------- 1 | 2 | // Copyright (C) 2021 Intel Corporation 3 | // SPDX-License-Identifier: Apache-2.0 4 | 5 | #pragma once 6 | 7 | /// @cond 8 | 9 | #define TUTORIAL_ECODE_PALISADE_ERROR 2 10 | 11 | /// @endcond 12 | -------------------------------------------------------------------------------- /docsrc/categories_overview.md: -------------------------------------------------------------------------------- 1 | HEBench Supported Benchmark Categories {#category_overview} 2 | ======================== 3 | 4 | ##Description 5 | This page lists all of the categories currently supported by Test Harness. 6 | 7 | ##Full Category List 8 | - @ref category_latency 9 | - @ref category_offline 10 | 11 | -------------------------------------------------------------------------------- /docsrc/tests_overview.md: -------------------------------------------------------------------------------- 1 | HEBench Supported Workloads {#tests_overview} 2 | ======================== 3 | 4 | ##Description 5 | This page lists all of the tests currently implemented as part of Test Harness. 6 | 7 | ##Full Test List 8 | - @ref generic_workload 9 | - @ref dot_product 10 | - @ref elementwise_add 11 | - @ref elementwise_mult 12 | - @ref logistic_regression 13 | - @ref matrix_multiplication 14 | - @ref simple_set_intersection 15 | -------------------------------------------------------------------------------- /.github/validation-ci/invalid_so.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright (C) 2021 Intel Corporation 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | <> $BADBACKEND 15 | ./bin/test_harness --backend_lib_path $BADBACKEND 16 | 17 | if $? 18 | then 19 | echo "Invalid .so validation Failed" 20 | exit 1 21 | fi 22 | 23 | echo "Invalid .so validation PASSED" 24 | exit 0 25 | -------------------------------------------------------------------------------- /docsrc/examples/generic_wl_example/generic_wl_tutorial/dataset.csv: -------------------------------------------------------------------------------- 1 | input, 0, 2, local 2 | 1, 2 3 | -1, -1 4 | 5 | input, 1, 5, local 6 | -1, 0 7 | 1, 1 8 | 2, -1 9 | -2, 1 10 | 1, -1 11 | 12 | output, 0, 10, local 13 | 0, 2 14 | 2, 3 15 | 3, 1 16 | -1, 3 17 | 2, 1 18 | -2, -1 19 | 0, 0 20 | 1, -2 21 | -3, 0 22 | 0, -2 23 | 24 | output, 1, 10, local 25 | 2, 2 26 | 0, 1 27 | -1, 3 28 | 3, 1 29 | 0, 3 30 | 0, -1 31 | -2, -2 32 | -3, 0 33 | 1, -2 34 | -2, 0 35 | 36 | output, 2, 10, local 37 | -1 38 | 3 39 | 0 40 | 0 41 | -1 42 | 1 43 | -2 44 | -1 45 | 1 46 | 0 47 | 48 | -------------------------------------------------------------------------------- /.github/validation-ci/backend_symlink_validation.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright (C) 2021 Intel Corporation 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | <> $BADCONFIG 17 | ./bin/test_harness --config_file $BADCONFIG --backend_lib_path $BACKEND 18 | 19 | if $? 20 | then 21 | echo "Invalid yaml validation Failed" 22 | exit 1 23 | fi 24 | 25 | echo "Invalid yaml validation PASSED" 26 | exit 0 27 | -------------------------------------------------------------------------------- /.github/validation-ci/config_symlink_validation.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright (C) 2021 Intel Corporation 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | < 11 | 12 | #define HEBENCH_HE_SCHEME_PLAIN 0 13 | #define HEBENCH_HE_SECURITY_NONE 0 14 | #define TUTORIAL_HE_SECURITY_128 1 15 | 16 | class TutorialEngine : public hebench::cpp::BaseEngine 17 | { 18 | public: 19 | HEBERROR_DECLARE_CLASS_NAME(ExampleEngine) 20 | 21 | public: 22 | static TutorialEngine *create(); 23 | static void destroy(TutorialEngine *p); 24 | 25 | ~TutorialEngine() override; 26 | 27 | protected: 28 | TutorialEngine(); 29 | 30 | void init() override; 31 | }; 32 | 33 | /// @endcond 34 | -------------------------------------------------------------------------------- /docsrc/examples/backend_tutorials/palisade/tutorial_backend/include/tutorial_engine_palisade.h: -------------------------------------------------------------------------------- 1 | 2 | // Copyright (C) 2021 Intel Corporation 3 | // SPDX-License-Identifier: Apache-2.0 4 | 5 | #pragma once 6 | 7 | /// @cond 8 | 9 | #include "hebench/api_bridge/cpp/hebench.hpp" 10 | #include 11 | 12 | #define HEBENCH_HE_SCHEME_PLAIN 0 13 | #define HEBENCH_HE_SECURITY_NONE 0 14 | #define TUTORIAL_HE_SECURITY_128 1 15 | 16 | class TutorialEngine : public hebench::cpp::BaseEngine 17 | { 18 | public: 19 | HEBERROR_DECLARE_CLASS_NAME(ExampleEngine) 20 | 21 | public: 22 | static TutorialEngine *create(); 23 | static void destroy(TutorialEngine *p); 24 | 25 | ~TutorialEngine() override; 26 | 27 | protected: 28 | TutorialEngine(); 29 | 30 | void init() override; 31 | }; 32 | 33 | /// @endcond 34 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. _Ex. I'm always frustrated when [...]_ 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | 22 | **If the feature request is approved, would you be willing to submit a PR** 23 | Yes / No _(Help can be provided if you need assistance submitting a PR)_ 24 | 25 | -------------------------------------------------------------------------------- /docsrc/dataset_loader_overview.md: -------------------------------------------------------------------------------- 1 | 2 | External Dataset Loader Overview {#dataset_loader_overview} 3 | ======================== 4 | 5 | [Benchmark configuration files](@ref config_file_reference) can specify an external file containing the dataset to use for each benchmark via the `dataset` field. This dataset must contain the input data, and optional ground truths for the benchmark workload operation evaluated on each possible combination of the specified input data (possible combinations are defined in @ref results_order ). 6 | 7 | The input and output data must match the workload specifications. This is, the input and output dimensions, the number of elements per input/output sample, and the number of samples, must match what the workload specification requires. 8 | 9 | The following are the supported data formats: 10 | - [Format-specific Comma-separated values (CSV)](@ref dataset_csv_reference) 11 | 12 | -------------------------------------------------------------------------------- /report_gen/report_gen_ar/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Intel Corporation 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | cmake_minimum_required(VERSION 2.9) 5 | project(hebench_reportgen_lib) 6 | 7 | set(${PROJECT_NAME}_SOURCES 8 | "${CMAKE_CURRENT_SOURCE_DIR}/src/hebench_report_cpp.cpp" 9 | ) 10 | set(${PROJECT_NAME}_HEADERS 11 | "${CMAKE_CURRENT_SOURCE_DIR}/../include/hebench_report_cpp.h" 12 | "${CMAKE_CURRENT_SOURCE_DIR}/../include/hebench_report_types.h" 13 | "${CMAKE_CURRENT_SOURCE_DIR}/../include/hebench_report.h" 14 | ) 15 | 16 | add_library(${PROJECT_NAME} STATIC ${${PROJECT_NAME}_SOURCES} ${${PROJECT_NAME}_HEADERS}) 17 | 18 | target_include_directories(${PROJECT_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include) 19 | target_include_directories(${PROJECT_NAME} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../include) 20 | 21 | target_compile_options(${PROJECT_NAME} PRIVATE -Wall -Wextra -fPIC) 22 | -------------------------------------------------------------------------------- /cmake/third-party/API_BRIDGE.cmake: -------------------------------------------------------------------------------- 1 | set(CONTENT_NAME api_bridge) 2 | 3 | set(CMAKE_CXX_FLAGS_CURRENT "${CMAKE_CXX_FLAGS}") 4 | 5 | FetchContent_Declare( 6 | ${CONTENT_NAME} 7 | GIT_REPOSITORY https://github.com/hebench/api-bridge.git 8 | GIT_TAG ${${_COMPONENT_NAME}_TAG} 9 | SUBBUILD_DIR ${FETCHCONTENT_BASE_DIR}/${CONTENT_NAME}/${CONTENT_NAME}-subbuild 10 | SOURCE_DIR ${FETCHCONTENT_BASE_DIR}/${CONTENT_NAME}/${CONTENT_NAME}-src 11 | BINARY_DIR ${FETCHCONTENT_BASE_DIR}/${CONTENT_NAME}/${CONTENT_NAME}-build 12 | ) 13 | 14 | FetchContent_GetProperties(${CONTENT_NAME}) 15 | if(NOT ${CONTENT_NAME}_POPULATED) 16 | FetchContent_Populate(${CONTENT_NAME}) 17 | if (${HIDE_EXT_WARNINGS}) 18 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w") 19 | endif() 20 | add_subdirectory(${${CONTENT_NAME}_SOURCE_DIR} ${${CONTENT_NAME}_BINARY_DIR}) 21 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS_CURRENT}") 22 | endif() 23 | 24 | -------------------------------------------------------------------------------- /cmake/third-party/COMMON_LIB.cmake: -------------------------------------------------------------------------------- 1 | set(CONTENT_NAME common-lib) 2 | 3 | set(CMAKE_CXX_FLAGS_CURRENT "${CMAKE_CXX_FLAGS}") 4 | 5 | FetchContent_Declare( 6 | ${CONTENT_NAME} 7 | GIT_REPOSITORY https://github.com/hebench/common-lib 8 | GIT_TAG ${${_COMPONENT_NAME}_TAG} 9 | SUBBUILD_DIR ${FETCHCONTENT_BASE_DIR}/${CONTENT_NAME}/${CONTENT_NAME}-subbuild 10 | SOURCE_DIR ${FETCHCONTENT_BASE_DIR}/${CONTENT_NAME}/${CONTENT_NAME}-src 11 | BINARY_DIR ${FETCHCONTENT_BASE_DIR}/${CONTENT_NAME}/${CONTENT_NAME}-build 12 | ) 13 | 14 | FetchContent_GetProperties(${CONTENT_NAME}) 15 | if(NOT ${CONTENT_NAME}_POPULATED) 16 | FetchContent_Populate(${CONTENT_NAME}) 17 | if (${HIDE_EXT_WARNINGS}) 18 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w") 19 | endif() 20 | add_subdirectory(${${CONTENT_NAME}_SOURCE_DIR} ${${CONTENT_NAME}_BINARY_DIR} EXCLUDE_FROM_ALL) 21 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS_CURRENT}") 22 | endif() 23 | 24 | -------------------------------------------------------------------------------- /dataset_loader/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Intel Corporation 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | cmake_minimum_required(VERSION 3.12) 5 | 6 | project(hebench_dataset_loader LANGUAGES C CXX) 7 | 8 | set(${PROJECT_NAME}_SOURCES 9 | "${CMAKE_CURRENT_SOURCE_DIR}/src/hebench_dataset_loader.cpp" 10 | ) 11 | set(${PROJECT_NAME}_HEADERS 12 | "${CMAKE_CURRENT_SOURCE_DIR}/include/hebench_dataset_loader.h" 13 | ) 14 | 15 | add_library(${PROJECT_NAME} STATIC ${${PROJECT_NAME}_SOURCES} ${${PROJECT_NAME}_HEADERS}) 16 | 17 | target_include_directories(${PROJECT_NAME} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include) 18 | 19 | # set compiler properties 20 | set_target_properties(${PROJECT_NAME} PROPERTIES 21 | CXX_STANDARD 17 22 | CXX_STANDARD_REQUIRED YES 23 | CXX_EXTENSIONS NO 24 | ) 25 | 26 | # add lib-common-lib.a dependency 27 | target_link_libraries(${PROJECT_NAME} PRIVATE hebench_common-lib) 28 | 29 | #target_link_libraries(${PROJECT_NAME} PRIVATE dl) 30 | target_compile_options(${PROJECT_NAME} PRIVATE -Wall -Wextra) 31 | -------------------------------------------------------------------------------- /.github/validation-ci/dasetloader_invalid_tag.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright (C) 2021 Intel Corporation 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | < "$file" 34 | 35 | for data_type in i32 i64 f32 f64; do 36 | "$DATA_LOADER" "$file" "$data_type" 37 | if [ $? -eq 0 ]; then 38 | echo "Failed to run the dataset_loader with $file and $data_type" 39 | RESULT=1 40 | fi 41 | done 42 | 43 | rm "$file" 44 | 45 | if [ $RESULT -eq 0 ]; then 46 | echo "Successfully tested Dataset Loader with: $0" 47 | fi 48 | 49 | exit $RESULT 50 | -------------------------------------------------------------------------------- /.github/validation-ci/dasetloader_invalid_int_0.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright (C) 2021 Intel Corporation 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | < "$file" 34 | 35 | for data_type in i32 i64 f32 f64; do 36 | "$DATA_LOADER" "$file" "$data_type" 37 | if [ $? -eq 0 ]; then 38 | echo "Failed to run the dataset_loader with $file and $data_type" 39 | RESULT=1 40 | fi 41 | done 42 | 43 | rm "$file" 44 | 45 | if [ $RESULT -eq 0 ]; then 46 | echo "Successfully tested Dataset Loader with: $0" 47 | fi 48 | 49 | exit $RESULT 50 | -------------------------------------------------------------------------------- /.github/validation-ci/dasetloader_invalid_int_1.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright (C) 2021 Intel Corporation 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | < "$file" 34 | 35 | for data_type in i32 i64 f32 f64; do 36 | "$DATA_LOADER" "$file" "$data_type" 37 | if [ $? -eq 0 ]; then 38 | echo "Failed to run the dataset_loader with $file and $data_type" 39 | RESULT=1 40 | fi 41 | done 42 | 43 | rm "$file" 44 | 45 | if [ $RESULT -eq 0 ]; then 46 | echo "Successfully tested Dataset Loader with: $0" 47 | fi 48 | 49 | exit $RESULT 50 | -------------------------------------------------------------------------------- /.github/validation-ci/dasetloader_missing_type.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright (C) 2021 Intel Corporation 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | < "$file" 34 | 35 | for data_type in i32 i64 f32 f64; do 36 | "$DATA_LOADER" "$file" "$data_type" 37 | if [ $? -eq 0 ]; then 38 | echo "Failed to run the dataset_loader with $file and $data_type" 39 | RESULT=1 40 | fi 41 | done 42 | 43 | rm "$file" 44 | 45 | if [ $RESULT -eq 0 ]; then 46 | echo "Successfully tested Dataset Loader with: $0" 47 | fi 48 | 49 | exit $RESULT 50 | -------------------------------------------------------------------------------- /.github/validation-ci/dasetloader_sample0.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright (C) 2021 Intel Corporation 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | < "$file" 34 | 35 | for data_type in i32 i64 f32 f64; do 36 | "$DATA_LOADER" "$file" "$data_type" 37 | if [ $? -eq 1 ]; then 38 | echo "Failed to run the dataset_loader with $file and $data_type" 39 | RESULT=1 40 | fi 41 | done 42 | 43 | rm "$file" 44 | 45 | if [ $RESULT -eq 0 ]; then 46 | echo "Successfully tested Dataset Loader with: $0" 47 | fi 48 | 49 | exit $RESULT -------------------------------------------------------------------------------- /.github/validation-ci/dasetloader_file_not_found.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright (C) 2021 Intel Corporation 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | < "$file" 33 | 34 | for data_type in i32 i64 f32 f64; do 35 | "$DATA_LOADER" "$file" "$data_type" 36 | if [ $? -eq 0 ]; then 37 | echo "Failed to run the dataset_loader with $file and $data_type" 38 | RESULT=1 39 | fi 40 | done 41 | 42 | rm "$file" 43 | 44 | if [ $RESULT -eq 0 ]; then 45 | echo "Successfully tested Dataset Loader with: $0" 46 | fi 47 | 48 | exit $RESULT 49 | -------------------------------------------------------------------------------- /.github/validation-ci/dasetloader_invalid_type.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright (C) 2021 Intel Corporation 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | < "$file" 34 | 35 | for data_type in i32 i64 f32 f64; do 36 | "$DATA_LOADER" "$file" "$data_type" 37 | if [ $? -eq 0 ]; then 38 | echo "Failed to run the dataset_loader with $file and $data_type" 39 | RESULT=1 40 | fi 41 | done 42 | 43 | rm "$file" 44 | 45 | if [ $RESULT -eq 0 ]; then 46 | echo "Successfully tested Dataset Loader with: $0" 47 | fi 48 | 49 | exit $RESULT 50 | -------------------------------------------------------------------------------- /cmake/third-party/YAML_CPP.cmake: -------------------------------------------------------------------------------- 1 | set(CONTENT_NAME yaml-cpp) 2 | 3 | set(CMAKE_CXX_FLAGS_CURRENT "${CMAKE_CXX_FLAGS}") 4 | 5 | FetchContent_Declare( 6 | ${CONTENT_NAME} 7 | GIT_REPOSITORY https://github.com/jbeder/yaml-cpp 8 | GIT_TAG ${${_COMPONENT_NAME}_TAG} 9 | SUBBUILD_DIR ${FETCHCONTENT_BASE_DIR}/${CONTENT_NAME}/${CONTENT_NAME}-subbuild 10 | SOURCE_DIR ${FETCHCONTENT_BASE_DIR}/${CONTENT_NAME}/${CONTENT_NAME}-src 11 | BINARY_DIR ${FETCHCONTENT_BASE_DIR}/${CONTENT_NAME}/${CONTENT_NAME}-build 12 | ) 13 | 14 | FetchContent_GetProperties(${CONTENT_NAME}) 15 | if(NOT ${CONTENT_NAME}_POPULATED) 16 | FetchContent_Populate(${CONTENT_NAME}) 17 | set(YAML_CPP_BUILD_TESTS OFF CACHE BOOL "Build/Install Google Test for Yaml-CPP unit tests") 18 | set(YAML_CPP_INSTALL OFF CACHE BOOL "Disable generation of install target") 19 | if (${HIDE_EXT_WARNINGS}) 20 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w") 21 | endif() 22 | add_subdirectory(${${CONTENT_NAME}_SOURCE_DIR} ${${CONTENT_NAME}_BINARY_DIR} EXCLUDE_FROM_ALL) 23 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS_CURRENT}") 24 | endif() 25 | 26 | -------------------------------------------------------------------------------- /report_gen/report_gen_lib/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Intel Corporation 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | cmake_minimum_required(VERSION 2.9) 5 | project(hebench_reportgen) 6 | 7 | set(${PROJECT_NAME}_SOURCES 8 | "${CMAKE_CURRENT_SOURCE_DIR}/src/hebench_report_impl.cpp" 9 | "${CMAKE_CURRENT_SOURCE_DIR}/src/hebench_report.cpp" 10 | ) 11 | set(${PROJECT_NAME}_HEADERS 12 | "${CMAKE_CURRENT_SOURCE_DIR}/include/hebench_report_impl.h" 13 | "${CMAKE_CURRENT_SOURCE_DIR}/../include/hebench_report_types.h" 14 | "${CMAKE_CURRENT_SOURCE_DIR}/../include/hebench_report.h" 15 | ) 16 | 17 | add_library(${PROJECT_NAME} SHARED ${${PROJECT_NAME}_SOURCES} ${${PROJECT_NAME}_HEADERS}) 18 | 19 | # add lib-common-lib.a dependency 20 | target_link_libraries(${PROJECT_NAME} PRIVATE hebench_common-lib) 21 | 22 | target_include_directories(${PROJECT_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include) 23 | target_include_directories(${PROJECT_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../include) 24 | target_compile_options(${PROJECT_NAME} PRIVATE -Wall -Wextra) 25 | 26 | install(TARGETS ${PROJECT_NAME} DESTINATION lib) 27 | -------------------------------------------------------------------------------- /.github/validation-ci/dasetloader_sample1.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright (C) 2021 Intel Corporation 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | < "$file" 42 | 43 | for data_type in i32 i64 f32 f64; do 44 | "$DATA_LOADER" "$file" "$data_type" 45 | if [ $? -eq 1 ]; then 46 | echo "Failed to run the dataset_loader with $file and $data_type" 47 | RESULT=1 48 | fi 49 | done 50 | 51 | rm "$file" 52 | 53 | if [ $RESULT -eq 0 ]; then 54 | echo "Successfully tested Dataset Loader with: $0" 55 | fi 56 | 57 | exit $RESULT -------------------------------------------------------------------------------- /docsrc/function_pipeline.md: -------------------------------------------------------------------------------- 1 | API Bridge Function Pipeline Chart {#function_pipeline_chart} 2 | ======================== 3 | 4 | API Bridge functions are called by the frontend's Test Harness as part of the testing procedure. Each function will recieve some parameters as input, perform some expected operation in the backend, and then pass the results back to the Test Harness which will use the returned results as input to later functions in the flow. 5 | 6 | This chart depicts the pipeline. 7 | 8 |
9 |
10 | Figure 1. : API Bridge Function pipeline flow chart. 11 |
12 | 13 | Arrows indicate the data flow: the tail of an arrow indicates that the output of that function may be used as input to the function at the head of the arrow, and thus, they should be compatible. 14 | 15 | Note that green arrows specify the default flow for a standard benchmark and must be supported by all backends. 16 | 17 | Yellow arrows show optional dependencies, and unless specified by a workload or categories they do not need to be implemented by backends. 18 | 19 | More detailed information can be found in each specific API Bridge function documentation. 20 | -------------------------------------------------------------------------------- /docsrc/examples/backend_tutorials/backend_tutorial.md: -------------------------------------------------------------------------------- 1 | Quickstart C++ Wrapper Backend Tutorial {#simple_cpp_example} 2 | ======================== 3 | This tutorial details the steps involved in implementing a new backend engine utilizing the [C++ wrapper](@ref CPP_overview) for use with the testing framework. 4 | 5 | Refer to @ref glossary for a list of terms that will be mentioned throughout this tutorial. 6 | 7 | This guide is intended to be a beginner's guide for people who are looking to get a backend with a single test implemented quickly. It is based on using the `api_bridge_example_backend` as a starting point for a new backend implementation which uses the C++ wrapper. 8 | 9 | For the purposes of this guide, we will be creating a very basic backend implementing a benchmark for the **vector element-wise addition** workload, under the **offline** category, with **first operand in plain text and second encrypted**. 10 | 11 |
12 | Select your preferred library below. The tutorial will follow up using that library to perform Homomorphic Encryption (HE) operations. 13 | 14 | [C++ Wrapper Backend Tutorial using PALISADE lattice encryption library]( @ref simple_cpp_example_palisade)
15 | [C++ Wrapper Backend Tutorial using Microsoft SEAL library](@ref simple_cpp_example_seal) 16 | -------------------------------------------------------------------------------- /dataset_loader/tests/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Intel Corporation 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | cmake_minimum_required(VERSION 3.12) 5 | 6 | project(hebench_dataset_loader_test LANGUAGES C CXX) 7 | 8 | set(${PROJECT_NAME}_SOURCES 9 | "${CMAKE_CURRENT_SOURCE_DIR}/../src/hebench_dataset_loader.cpp" 10 | "${CMAKE_CURRENT_SOURCE_DIR}/../src/hebench_dataset_loader_functional_test.cpp" 11 | "${CMAKE_CURRENT_SOURCE_DIR}/../src/main.cpp" 12 | ) 13 | set(${PROJECT_NAME}_HEADERS 14 | "${CMAKE_CURRENT_SOURCE_DIR}/../include/hebench_dataset_loader.h" 15 | "${CMAKE_CURRENT_SOURCE_DIR}/../include/hebench_dataset_loader_functional_test.h" 16 | ) 17 | 18 | add_executable(${PROJECT_NAME} ${${PROJECT_NAME}_SOURCES} ${${PROJECT_NAME}_HEADERS}) 19 | 20 | target_include_directories(${PROJECT_NAME} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../include) 21 | 22 | # set compiler properties 23 | set_target_properties(${PROJECT_NAME} PROPERTIES 24 | CXX_STANDARD 17 25 | CXX_STANDARD_REQUIRED YES 26 | CXX_EXTENSIONS NO 27 | ) 28 | 29 | add_subdirectory(../../common-lib common-lib/build) 30 | 31 | target_link_libraries(${PROJECT_NAME} PRIVATE dl) 32 | target_link_libraries(${PROJECT_NAME} PRIVATE hebench_common-lib) 33 | 34 | install(TARGETS ${PROJECT_NAME} DESTINATION bin) 35 | -------------------------------------------------------------------------------- /.github/validation-ci/dasetloader_sample2.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright (C) 2021 Intel Corporation 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | < "$file" 39 | 40 | for data_type in i32 i64 f32 f64; do 41 | "$DATA_LOADER" "$file" "$data_type" 42 | if [ $? -eq 1 ]; then 43 | echo "Failed to run the dataset_loader with $file and $data_type" 44 | RESULT=1 45 | fi 46 | done 47 | 48 | rm "$file" 49 | 50 | if [ $RESULT -eq 0 ]; then 51 | echo "Successfully tested Dataset Loader with: $0" 52 | fi 53 | 54 | exit $RESULT 55 | -------------------------------------------------------------------------------- /.github/workflows/validate_testharness_output.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright (C) 2021 Intel Corporation 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | set -e 7 | 8 | mkdir validation-test && pushd validation-test 9 | echo "Cloning Latest Cleartext Backend..." 10 | git clone https://github.com/hebench/backend-cpu-cleartext.git 11 | pushd ./backend-cpu-cleartext 12 | 13 | mkdir build && pushd build 14 | 15 | echo "Configuring & Building Cleartext Backend..." 16 | cmake -DCMAKE_INSTALL_PREFIX=./ -DCMAKE_BUILD_TYPE=Release -DAPI_BRIDGE_INSTALL_DIR=../../../ -DCMAKE_C_COMPILER=gcc-9 -DCMAKE_CXX_COMPILER=g++-9 .. 17 | make -j install 18 | 19 | CLEARTEXTLIB=$(realpath ./lib/libhebench_cleartext_backend.so) 20 | 21 | popd 22 | popd 23 | popd 24 | 25 | pushd bin 26 | ./test_harness --config_file ./config.yaml --random_seed 1234 --report_delay 0 --backend_lib_path $CLEARTEXTLIB --dump 27 | sed -i 's/default_min_test_time: 0/default_min_test_time: 50/g' config.yaml 28 | ./test_harness --config_file ./config.yaml --random_seed 1234 --report_delay 0 --backend_lib_path $CLEARTEXTLIB |& tee test_harness_output.log 29 | 30 | echo "Checking output from Test Harness..." 31 | if ! grep -Fxq "[ Info ] Failed: 0" test_harness_output.log 32 | then 33 | echo "Validation Failed" 34 | exit 1 35 | fi 36 | echo "Validation PASSED" 37 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Version Information (Put N/A for Not Applicable)** 11 | | Software | Version(s) | 12 | | ---------------------------| --------------- | 13 | | Linux | | 14 | | Git | | 15 | | CMake | | 16 | | Compiler (GCC, Clang, etc.)| | 17 | | GLIBC (ldd) | | 18 | | Doxygen | | 19 | | HEBench Frontend | | 20 | | HEBench API Bridge | | 21 | | Reference Backend | | 22 | 23 | **Describe the bug** 24 | A clear and concise description of what the bug is. 25 | 26 | **To Reproduce** 27 | Steps to reproduce the behavior: 28 | 1. Go to '...' 29 | 2. Click on '....' 30 | 3. Scroll down to '....' 31 | 4. etc. 32 | 5. See error 33 | 34 | **Expected behavior** 35 | A clear and concise description of what you expected to happen. 36 | 37 | **Screenshots** 38 | If applicable, add screenshots to help explain your problem. 39 | 40 | **Additional context** 41 | Add any other context about the problem here. 42 | -------------------------------------------------------------------------------- /docsrc/examples/backend_tutorials/seal/backend_tutorial_files_seal.md: -------------------------------------------------------------------------------- 1 | Quickstart C++ Wrapper Backend Tutorial - File Reference {#be_tutorial_files_seal} 2 | ======================== 3 | 4 | The code snippets used in this tutorial are available as part of example documentation. The complete source for the example is located in the following files. Note that these files are intended only as an example to teach core concepts of how to implement a backend using the C++ wrapper. They are missing a lot of features like error checking that would be expected in a robust backend. 5 | 6 | Example File | Description 7 | -------------- | ------------ 8 | @ref original_flow_seal.cpp | Original workflow to benchmark 9 | @ref tutorial_engine_seal.h | Tutorial engine header 10 | @ref tutorial_engine_seal.cpp | Tutorial engine implementation 11 | @ref tutorial_error_seal.h | Tutorial error definition file 12 | @ref tutorial_eltwiseadd_benchmark_seal.h | Tutorial benchmark header 13 | @ref tutorial_eltwiseadd_benchmark_seal.cpp | Tutorial benchmark implementation 14 | 15 | ## Tutorial steps 16 | 17 | [Tutorial Home](backend_tutorial_seal.md)
18 | [Preparation](backend_tutorial_preparation_seal.md)
19 | [Engine Initialization and Benchmark Description](backend_tutorial_init_seal.md)
20 | [Benchmark Implementation](backend_tutorial_impl_seal.md)
21 | File References 22 | -------------------------------------------------------------------------------- /test_harness/include/hebench_types_harness.h: -------------------------------------------------------------------------------- 1 | 2 | // Copyright (C) 2021 Intel Corporation 3 | // SPDX-License-Identifier: Apache-2.0 4 | 5 | #ifndef _HEBench_Harness_Types_H_0596d40a3cce4b108a81595c50eb286d 6 | #define _HEBench_Harness_Types_H_0596d40a3cce4b108a81595c50eb286d 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #include "hebench/api_bridge/types.h" 17 | 18 | #define IOS_MSG_DONE "[ DONE ] " 19 | #define IOS_MSG_WARNING "[ WARNING ] " 20 | #define IOS_MSG_ERROR "[ ERROR ] " 21 | #define IOS_MSG_FAILED "[ FAIL ] " 22 | #define IOS_MSG_INFO "[ Info ] " 23 | #define IOS_MSG_OK "[ OK ] " 24 | 25 | namespace hebench { 26 | namespace TestHarness { 27 | 28 | using DefaultTimeInterval = std::micro; 29 | 30 | template 31 | using unique_ptr_custom_deleter = std::unique_ptr>; 32 | 33 | constexpr const char *FileNameNoExtReport = "report"; 34 | constexpr const char *FileNameNoExtSummary = "summary"; 35 | 36 | // '\\' needs to be taken into account in case Windows is supported. 37 | constexpr char separator = '/'; 38 | constexpr char hyphen = '-'; 39 | 40 | } // namespace TestHarness 41 | } // namespace hebench 42 | 43 | #endif // defined _HEBench_Harness_Types_H_0596d40a3cce4b108a81595c50eb286d 44 | -------------------------------------------------------------------------------- /docsrc/examples/backend_tutorials/palisade/backend_tutorial_files_palisade.md: -------------------------------------------------------------------------------- 1 | Quickstart C++ Wrapper Backend Tutorial - File Reference {#be_tutorial_files_palisade} 2 | ======================== 3 | 4 | The code snippets used in this tutorial are available as part of example documentation. The complete source for the example is located in the following files. Note that these files are intended only as an example to teach core concepts of how to implement a backend using the C++ wrapper. They are missing a lot of features like error checking that would be expected in a robust backend. 5 | 6 | Example File | Description 7 | -------------- | ------------ 8 | @ref original_flow_palisade.cpp | Original workflow to benchmark 9 | @ref tutorial_engine_palisade.h | Tutorial engine header 10 | @ref tutorial_engine_palisade.cpp | Tutorial engine implementation 11 | @ref tutorial_error_palisade.h | Tutorial error definition file 12 | @ref tutorial_eltwiseadd_benchmark_palisade.h | Tutorial benchmark header 13 | @ref tutorial_eltwiseadd_benchmark_palisade.cpp | Tutorial benchmark implementation 14 | 15 | ## Tutorial steps 16 | 17 | [Tutorial Home](backend_tutorial_palisade.md)
18 | [Preparation](backend_tutorial_preparation_palisade.md)
19 | [Engine Initialization and Benchmark Description](backend_tutorial_init_palisade.md)
20 | [Benchmark Implementation](backend_tutorial_impl_palisade.md)
21 | File References 22 | -------------------------------------------------------------------------------- /.github/workflows/cmake.yml: -------------------------------------------------------------------------------- 1 | name: CMake 2 | 3 | on: 4 | push: 5 | branches: 6 | - development 7 | pull_request: 8 | branches: 9 | - development 10 | - main 11 | 12 | env: 13 | BUILD_TYPE: Release 14 | 15 | jobs: 16 | build: 17 | runs-on: ubuntu-20.04 18 | environment: Workflow Approval 19 | 20 | steps: 21 | - uses: actions/checkout@v3 22 | with: 23 | fetch-depth: 0 24 | 25 | - name: Setup 26 | working-directory: ${{github.workspace}} 27 | run: echo yes | .github/workflows/install-requirements.sh 28 | 29 | - name: Format 30 | run: | 31 | pre-commit install 32 | pre-commit run --all-files 33 | 34 | - name: Configure CMake 35 | run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build -DCMAKE_C_COMPILER=gcc-9 -DCMAKE_CXX_COMPILER=g++-9 36 | 37 | - name: Build 38 | run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} -j 39 | 40 | - name: Install 41 | run: cmake --install ${{github.workspace}}/build 42 | 43 | - name: Test 44 | working-directory: ${{github.workspace}}/build/bin 45 | run: ./test_harness --backend_lib_path ../lib/libhebench_example_backend.so 46 | 47 | - name: Validation 48 | working-directory: ${{github.workspace}}/build 49 | run: ../.github/workflows/validate_testharness_output.sh 50 | -------------------------------------------------------------------------------- /docsrc/report_compiler_overview.md: -------------------------------------------------------------------------------- 1 | 2 | HEBench Report Compiler Overview {#report_compiler_overview} 3 | ======================== 4 | 5 | [TOC] 6 | 7 | ## Description 8 | 9 | The Report Compiler is a component of the HEBench frontend. For other frontend components refer to @ref frontend_overview . 10 | 11 | The Report Compiler is designed to aid Test Harness in the generation of human-readable summaries, computation of in-depth statistics from benchmark reports, and an overview of a Test Harness run. 12 | 13 | Users should use the report compiler to convert Test Harness run reports into human-readable data and easy-to-parse statistics and overview files. In a default run, Test Harness will produce compiled reports using default report compiler options; however, if users require a different configuration, they can run the report compiler manually through its command line interface to produce the desired data. See the user guide below for information on using the report compiler. 14 | 15 | ## User Guides 16 | 17 | The documents and pages listed in this section detail how to use the report compiler to generate summaries, statistics and run overviews from Test Harness benchmark reports. 18 | 19 | - @ref report_compiler_usage_guide 20 | - @ref stats_n_summary_format 21 | - @ref overview_file_format 22 | 23 | ## Namespace 24 | 25 | Report generation namespace: `hebench::ReportGen` 26 | 27 | Report compiler library: `hebench::ReportGen::Compiler` 28 | 29 | -------------------------------------------------------------------------------- /report_gen/report_compiler_lib/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Intel Corporation 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | cmake_minimum_required(VERSION 2.9) 5 | project(hebench_report_compiler) 6 | 7 | # Shared Library to expose the report compiler capabilities. 8 | 9 | set(${PROJECT_NAME}_SOURCES 10 | "${CMAKE_CURRENT_SOURCE_DIR}/src/hebench_report_compiler.cpp" 11 | "${CMAKE_CURRENT_SOURCE_DIR}/src/hebench_report_stats.cpp" 12 | "${CMAKE_CURRENT_SOURCE_DIR}/src/hebench_report_overview_header.cpp" 13 | ) 14 | 15 | set(${PROJECT_NAME}_HEADERS 16 | "${CMAKE_CURRENT_SOURCE_DIR}/include/hebench_report_compiler.h" 17 | "${CMAKE_CURRENT_SOURCE_DIR}/include/hebench_report_stats.h" 18 | "${CMAKE_CURRENT_SOURCE_DIR}/include/hebench_report_overview_header.h" 19 | ) 20 | 21 | add_library(${PROJECT_NAME} SHARED ${${PROJECT_NAME}_SOURCES} ${${PROJECT_NAME}_HEADERS}) 22 | 23 | # add lib-common-lib.a dependency 24 | target_link_libraries(${PROJECT_NAME} PRIVATE hebench_common-lib) 25 | 26 | # add the report generation libraries dependency 27 | 28 | target_include_directories(${PROJECT_NAME} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include) 29 | target_include_directories(${PROJECT_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../include) 30 | 31 | target_link_libraries(${PROJECT_NAME} PRIVATE hebench_reportgen_lib) 32 | target_link_libraries(${PROJECT_NAME} PRIVATE hebench_reportgen) 33 | target_compile_options(${PROJECT_NAME} PRIVATE -Wall -Wextra) 34 | 35 | install(TARGETS ${PROJECT_NAME} DESTINATION lib) 36 | -------------------------------------------------------------------------------- /.github/workflows/documentation.yml: -------------------------------------------------------------------------------- 1 | name: Documentation 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | 8 | env: 9 | # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.) 10 | BUILD_TYPE: Release 11 | 12 | jobs: 13 | documentation: 14 | runs-on: ubuntu-20.04 15 | 16 | steps: 17 | - uses: actions/checkout@v3 18 | with: 19 | ref: documentation 20 | token: ${{ secrets.TOKEN_DOCS }} 21 | 22 | - name: Documentation 23 | working-directory: ${{github.workspace}} 24 | run: | 25 | git pull 26 | git reset --hard 210fb7c819a79eeb75163eee3bf7ccf7b66fcdd5 27 | git clean -fdx 28 | API_BRIDGE_TAG="main" 29 | tmp_dir=$(mktemp -d) 30 | DOCTARGETDIR=$PWD 31 | cd $tmp_dir 32 | git clone https://github.com/hebench/frontend.git 33 | cd frontend 34 | if [ -f cmake/third-party/API_BRIDGE.version ]; then 35 | API_BRIDGE_TAG=$(scripts/read_version.sh cmake/third-party/API_BRIDGE.version) 36 | fi 37 | yes | scripts/build_docs.sh $tmp_dir "${API_BRIDGE_TAG}" 38 | mv $tmp_dir/docs/html $DOCTARGETDIR/docs 39 | cd $DOCTARGETDIR 40 | rm -rf $tmp_dir 41 | git add . --force 42 | git config user.email "support@hebench.org" 43 | git config user.name "Documentation" 44 | PRE_COMMIT_ALLOW_NO_CONFIG=1 git commit -m "Documentation" --author "Documentation " 45 | git push origin documentation --force 46 | -------------------------------------------------------------------------------- /docsrc/frontend_overview.md: -------------------------------------------------------------------------------- 1 | HEBench frontend overview {#frontend_overview} 2 | ======================== 3 | 4 | [TOC] 5 | 6 | ## Description 7 | 8 | The HEBench frontend consists of the components relevant to users running benchmarks, extending the test harness with their own benchmarks, and developers who wish to contribute new tests and workloads to the framework. 9 | 10 |
11 |
12 | Figure 1. : HEBench frontend component diagram 13 |
14 | 15 | The main component of the frontend is the Test Harness. It is the main executable that benchmarks the performance of supported HE workloads implemented in backends. All HEBench users should become familiar with the Test Harness features and functionality. 16 | 17 | ## Components 18 | The frontend is comprised of the following components: 19 | - @ref test_harness_overview : benchmarks solutions to supported HE workloads provided on backends which implement the API Bridge. 20 | - @ref report_compiler_overview : compiles reports generated by Test Harness, producing human readable summaries and in-depth statistics. 21 | - @ref tests_overview : list of all currently implemented tests with detailed description for each one. 22 | 23 | ## Tutorials 24 | 25 | **NOTE**: Extending the Test Harness is an advanced topic. The primary usage for HEBench is expected to be the creation of backends and benchmarking them with the existing Test Harness. 26 | 27 | - @ref extend_test_harness 28 | -------------------------------------------------------------------------------- /docsrc/intro_workload_add.md: -------------------------------------------------------------------------------- 1 | Working with Unsupported Workloads {#intro_workload_addition} 2 | ======================== 3 | 4 | [TOC] 5 | 6 | At some point a user may need to benchmark a workload that is not currently supported by Test Harness. While creating a backend for and benchmarking currently supported workloads is easy, and it is the intended way of using HEBench, benchmarking an unsupported workload is still possible. 7 | 8 | There are two ways to benchmark an unsupported workload: 9 | 10 | - [Create a custom workload using the Generic Workload operation](@ref generic_workload) 11 | 12 | This does not require changing the code base for the framework. Users need to configure the custom workload using a configuration file, and supply a custom dataset that defines the workload inputs and expected outputs, (the dataset must be specified as the data should be laid out in memory; i.e. ordering, padding, etc.). Once configured, users can provide backends that operate on the custom workload as if it were any other supported workload. Test Harness uses the configuration and dataset to communicate information to the backend. 13 | - [Add support for the new workload into Test Harness](@ref extend_test_harness) 14 | 15 | This method requires modifying the framework to extend Test Harness in order to provide the implementation of the new workload. This is useful to add workloads that may be difficult to match to the expected structure of the Generic Workload. However, this method requires knowledge of the framework code base and it is an advanced topic. 16 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v2.5.0 4 | hooks: 5 | - id: trailing-whitespace 6 | files: \.(c|cc|cxx|cpp|h|hpp|hxx|inl|h.in|py)$ 7 | - id: end-of-file-fixer 8 | files: \.(c|cc|cxx|cpp|h|hpp|hxx|inl|h.in|py)$ 9 | - id: check-merge-conflict 10 | files: \.(c|cc|cxx|cpp|h|hpp|hxx|inl|h.in|py)$ 11 | - id: mixed-line-ending 12 | files: \.(c|cc|cxx|cpp|h|hpp|hxx|inl|h.in|py)$ 13 | - id: check-byte-order-marker 14 | files: \.(c|cc|cxx|cpp|h|hpp|hxx|inl|h.in|py)$ 15 | - repo: https://github.com/Lucas-C/pre-commit-hooks 16 | rev: v1.1.10 17 | hooks: 18 | - id: insert-license 19 | files: (CMakeLists\.txt)|\.(sh|py)$ 20 | args: 21 | - --license-filepath 22 | # defaults to: LICENSE.txt 23 | - HEADER 24 | - repo: https://github.com/Lucas-C/pre-commit-hooks 25 | rev: v1.1.10 26 | hooks: 27 | - id: insert-license 28 | files: \.(c|cc|cxx|cpp|h|hpp|hxx|inl|h.in)$ 29 | args: 30 | - --license-filepath 31 | # defaults to: LICENSE.txt 32 | - HEADER 33 | - --comment-style 34 | - // # defaults to: # 35 | - repo: local 36 | hooks: 37 | - id: clang-format-9 38 | name: clang-format-9 39 | entry: clang-format-9 40 | language: system 41 | files: \.(c|cc|cxx|cpp|h|hpp|hxx|inl)$ 42 | args: ["-i", "--style=file"] 43 | 44 | -------------------------------------------------------------------------------- /docsrc/examples/generic_wl_example/generic_wl_tutorial/config.yaml: -------------------------------------------------------------------------------- 1 | default_min_test_time: 0 2 | default_sample_size: 0 3 | random_seed: 1234 4 | 5 | benchmark: 6 | 7 | # Benchmark with workload parameters: 8 | # Generic 2 Inputs, 3 Outputs 9 | # Descriptor: 10 | # wp_2_3_2_2_2_2_1 | offline | float64 | default | all_plain | plain | none | 0 11 | - ID: 1 12 | dataset: dataset.csv 13 | default_min_test_time: 1000 14 | default_sample_sizes: 15 | 0: 2 16 | 1: 5 17 | params: 18 | 0: 19 | name: n 20 | type: UInt64 21 | value: 22 | from: 2 23 | to: 2 24 | step: 0 25 | 1: 26 | name: m 27 | type: UInt64 28 | value: 29 | from: 3 30 | to: 3 31 | step: 0 32 | 2: 33 | name: length_InputParam0 34 | type: UInt64 35 | value: 36 | from: 2 37 | to: 2 38 | step: 0 39 | 3: 40 | name: length_InputParam1 41 | type: UInt64 42 | value: 43 | from: 2 44 | to: 2 45 | step: 0 46 | 4: 47 | name: length_ResultComponent0 48 | type: UInt64 49 | value: 50 | from: 2 51 | to: 2 52 | step: 0 53 | 5: 54 | name: length_ResultComponent1 55 | type: UInt64 56 | value: 57 | from: 2 58 | to: 2 59 | step: 0 60 | 6: 61 | name: length_ResultComponent2 62 | type: UInt64 63 | value: 64 | from: 1 65 | to: 1 66 | step: 0 67 | -------------------------------------------------------------------------------- /test_harness/include/hebench_version.h.in: -------------------------------------------------------------------------------- 1 | 2 | // Copyright (C) 2021 Intel Corporation 3 | // SPDX-License-Identifier: Apache-2.0 4 | 5 | #ifndef _HEBench_Harness_Version_H_0596d40a3cce4b108a81595c50eb286d 6 | #define _HEBench_Harness_Version_H_0596d40a3cce4b108a81595c50eb286d 7 | 8 | #include "hebench/api_bridge/version.h" 9 | 10 | #define HEBENCH_TEST_HARNESS_APP_NAME "HEBench Test Harness" 11 | 12 | #ifndef HEBENCH_TEST_HARNESS_VERSION_MAJOR 13 | #define HEBENCH_TEST_HARNESS_VERSION_MAJOR @@CMAKE_PROJECT_NAME@_VERSION_MAJOR@ 14 | #endif 15 | #ifndef HEBENCH_TEST_HARNESS_VERSION_MINOR 16 | #define HEBENCH_TEST_HARNESS_VERSION_MINOR @@CMAKE_PROJECT_NAME@_VERSION_MINOR@ 17 | #endif 18 | #ifndef HEBENCH_TEST_HARNESS_VERSION_REVISION 19 | #define HEBENCH_TEST_HARNESS_VERSION_REVISION @@CMAKE_PROJECT_NAME@_VERSION_REVISION@ 20 | #endif 21 | #ifndef HEBENCH_TEST_HARNESS_VERSION_BUILD 22 | #define HEBENCH_TEST_HARNESS_VERSION_BUILD "@@CMAKE_PROJECT_NAME@_VERSION_BUILD@" 23 | #endif 24 | 25 | #define HEBENCH_TEST_HARNESS_API_REQUIRED_VERSION_MAJOR @API_BRIDGE_REQUIRED_VERSION_MAJOR@ 26 | #define HEBENCH_TEST_HARNESS_API_REQUIRED_VERSION_MINOR @API_BRIDGE_REQUIRED_VERSION_MINOR@ 27 | #define HEBENCH_TEST_HARNESS_API_MIN_REQUIRED_VERSION_REVISION @API_BRIDGE_REQUIRED_VERSION_REVISION@ 28 | #ifndef API_BRIDGE_NOT_REQUIRED_VERSION_BUILD 29 | #define HEBENCH_TEST_HARNESS_API_REQUIRED_VERSION_BUILD "@API_BRIDGE_REQUIRED_VERSION_BUILD@" 30 | #else 31 | #define HEBENCH_TEST_HARNESS_API_REQUIRED_VERSION_BUILD HEBENCH_API_VERSION_BUILD 32 | #endif 33 | 34 | #endif // defined _HEBench_Harness_Version_H_0596d40a3cce4b108a81595c50eb286d 35 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## Proposed changes 2 | 3 | Describe the big picture of your changes here to communicate to the maintainers why we should accept this pull request. If it fixes a bug or resolves a feature request, be sure to link to that issue. 4 | 5 | ## Types of changes 6 | 7 | What types of changes does your code introduce to the HEBench Frontend? 8 | _Put an `x` in the boxes that apply_ 9 | 10 | - [ ] Bugfix (non-breaking change which fixes an issue) 11 | - [ ] New feature (non-breaking change which adds functionality) 12 | - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) 13 | - [ ] Documentation Update (if none of the other choices apply) 14 | 15 | ## Checklist 16 | 17 | _Put an `x` in the boxes that apply. You can also fill these out after creating the PR. If you're unsure about any of them, don't hesitate to ask. We're here to help! This is simply a reminder of what we are going to look for before merging your code._ 18 | 19 | - [ ] I have read the [CONTRIBUTING](https://github.com/hebench/frontend/blob/main/CONTRIBUTING.md) doc 20 | - [ ] Current formatting and unit tests / base functionality passes locally with my changes 21 | - [ ] I have added tests that prove my fix is effective or that my feature works (if appropriate) 22 | - [ ] I have added necessary documentation (if appropriate) 23 | - [ ] Any dependent changes have been merged and published in downstream modules 24 | 25 | ## Further comments 26 | 27 | If this is a relatively large or complex change, kick off the discussion by explaining why you chose the solution you did, what alternatives you considered, etc. 28 | -------------------------------------------------------------------------------- /dataset_loader/src/main.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2021 Intel Corporation 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | #include "hebench_dataset_loader_functional_test.h" 5 | 6 | int main(int argc, char **argv) 7 | { 8 | if (argc < min_args_allowed) 9 | { 10 | signal_error("Missing CSV file path.\n", true); 11 | } 12 | else if (argc == min_args_allowed) 13 | { 14 | if (strcmp(argv[help_arg], help_cstr_short) == 0 || strcmp(argv[help_arg], help_cstr_long) == 0) 15 | { 16 | std::cout << help_msg << std::endl; 17 | } 18 | else 19 | { 20 | signal_error(check_usage); 21 | } 22 | } 23 | else if (argc > max_args_allowed) 24 | { 25 | signal_error("The amount of arguments exceed the required.\n", true); 26 | } 27 | else 28 | { 29 | std::filesystem::path file_path = argv[file_path_arg]; 30 | if (!std::filesystem::exists(file_path)) 31 | { 32 | signal_error("The provided file does not exist."); 33 | } 34 | if (file_path.extension() != ".csv") 35 | { 36 | signal_error("The provided file's extension is not CSV related."); 37 | } 38 | else 39 | { 40 | std::uint64_t max_loaded_size = argc == max_args_allowed ? (std::uint64_t)std::stoul(argv[max_loaded_size_arg], nullptr, 0) : 0; 41 | std::string data_load_type = argv[type_arg]; 42 | try 43 | { 44 | load_from_csv(data_load_type, file_path, max_loaded_size); 45 | } 46 | catch (const std::exception &exc) 47 | { 48 | signal_error(exc.what()); 49 | } 50 | } 51 | } 52 | return EXIT_SUCCESS; 53 | } 54 | -------------------------------------------------------------------------------- /docsrc/backend_overview.md: -------------------------------------------------------------------------------- 1 | HEBench Backend Overview {#backend_overview} 2 | ======================== 3 | 4 | [TOC] 5 | 6 | ## Overview 7 | 8 | The HEBench components that enable backends consist of the API Bridge and the C++ wrapper. These enable the creation of backends to implement the benchmarks for [supported workloads](@ref tests_overview). 9 | 10 | Backend engines are expected to be shared libraries (.so in Linux, .dll in Windows) that expose the [API Bridge](@ref APIBridge_overview) functionality. The functionality can be implemented by directly defining the required functions specified in the @ref hebench::APIBridge or extending the [C++ wrapper](@ref CPP_overview). 11 | 12 | Backends are free to implement any combination of test and datatype combinations and register these tests as part of their implementation. 13 | 14 | Creating backends for different hardware and software libraries is the expected primary way in which submitters will interface with HEBench. 15 | 16 |
17 |
18 | Figure 1. : HEBench backend component diagram 19 |
20 | 21 | The tutorial at the end of this page is the fastest way for users to get started in the process of creating a backend to benchmark their implementation of one or more of the supported workloads. 22 | 23 | ## Components 24 | 25 | - @ref APIBridge_overview : Describes the details of the backends C API. 26 | - @ref CPP_overview : Describes the details of the backends C++ API wrapper. 27 | - @ref tests_overview : list of all currently implemented tests with detailed description for each one. 28 | 29 | ## Tutorials 30 | 31 | - @ref simple_cpp_example : A quick start example showing how to implement a simple backend by extending the C++ wrapper. 32 | -------------------------------------------------------------------------------- /docsrc/test_harness_overview.md: -------------------------------------------------------------------------------- 1 | HEBench Test Harness Overview {#test_harness_overview} 2 | ======================== 3 | 4 | [TOC] 5 | 6 | ##Description 7 | 8 | The Test Harness is the core frontend component and user interaction point for the HEBench project. It is built as part of the HEBench frontend and it is responsible for orchestrating the benchmarking operations, submiting and timing requests to backends, parsing user arguments and configurations, and outputting benchmark reports. It provides support for dynamically loading compatible HEBench backends and benchmarking supported tests. 9 | 10 | For other frontend components refer to @ref frontend_overview . 11 | 12 | ##User Guides 13 | 14 | The documents and pages listed in this section detail how to use the test harness, including how to run workloads for different backends, configure workloads, available command line options, etc. All HEBench users should become familiar with the features and functionality offered by Test Harness in order to facilitate the collection of benchmark results. 15 | 16 | - @ref test_harness_usage_guide 17 | - @ref config_file_reference 18 | - Dataset loader: 19 | - @ref dataset_csv_reference 20 | 21 | ##Namespace 22 | 23 | [Test Harness namespace](@ref hebench::TestHarness) 24 | 25 | ##Tutorials 26 | 27 | The documents and guides in this section detail how to extend the test harness. Including how to add new workloads, test categories, etc. 28 | 29 | **NOTE**: Extending the Test Harness is an advanced topic. The primary usage for HEBench is expected to be the creation of backends and benchmarking them with the existing Test Harness. 30 | 31 | This section is for advanced users that require other tests, workloads, or categories not currently supported by the Test Harness and API Bridge. 32 | 33 | - @ref extend_test_harness 34 | -------------------------------------------------------------------------------- /report_gen/report_compiler_lib/include/hebench_report_overview_header.h: -------------------------------------------------------------------------------- 1 | 2 | // Copyright (C) 2021 Intel Corporation 3 | // SPDX-License-Identifier: Apache-2.0 4 | 5 | #ifndef _HEBench_ReportHeader_H_0596d40a3cce4b108a81595c50eb286d 6 | #define _HEBench_ReportHeader_H_0596d40a3cce4b108a81595c50eb286d 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | namespace hebench { 15 | namespace ReportGen { 16 | 17 | struct OverviewHeader 18 | { 19 | public: 20 | static constexpr const char *EndStateOK = "OK"; 21 | static constexpr const char *EndStateGeneralFailure = "Failed"; 22 | 23 | OverviewHeader() : 24 | other(0) {} 25 | 26 | std::string workload_name; 27 | std::string end_state; 28 | std::string report_file; 29 | std::string category; 30 | std::string data_type; 31 | std::string cipher_text; 32 | std::string scheme; 33 | std::string security; 34 | std::int64_t other; 35 | std::vector w_params; 36 | 37 | void parseHeader(const std::string &filename, const std::string &s_header, const std::string &s_end_state); 38 | 39 | // outputs header without workload parameters 40 | void outputHeader(std::ostream &os, bool new_line = true); 41 | 42 | private: 43 | static std::vector extractInfoFromCSVLine(std::string_view s_row, 44 | const std::string_view &s_tag, 45 | std::size_t num_values); 46 | static std::string extractCiphertextBitset(std::vector s_indices); 47 | }; 48 | 49 | } // namespace ReportGen 50 | } // namespace hebench 51 | 52 | #endif // defined _HEBench_ReportHeader_H_0596d40a3cce4b108a81595c50eb286d 53 | -------------------------------------------------------------------------------- /docsrc/examples/backend_tutorials/seal/backend_tutorial_seal.md: -------------------------------------------------------------------------------- 1 | Quickstart C++ Wrapper Backend Tutorial - SEAL {#simple_cpp_example_seal} 2 | ======================== 3 | 4 | [TOC] 5 | 6 | ## Backend Creation Tutorial 7 | This tutorial details the steps involved in implementing a new backend utilizing the [C++ wrapper](@ref CPP_overview) for use with the testing framework. 8 | 9 | Refer to @ref glossary for a list of terms that will be mentioned throughout this tutorial. 10 | 11 | This guide is intended to be a beginners guide for people who are looking to get a backend with a single test implemented quickly. It is based on using the `api_bridge_example_backend` as a starting point for a new backend implementation which uses the C++ wrapper. 12 | 13 | For the purposes of this guide we will be creating a very basic backend implementing a benchmark for the **vector element-wise addition** workload, under the **offline** category, with **first operand in plain text and second encrypted**, using Microsoft SEAL 3.6 to perform Homomorphic Encryption (HE) operations. 14 | 15 | Assume that we already have a functional workflow for our workload as listed below, and we want to benchmark it using HEBench. 16 | 17 | \snippet{lineno} docsrc/examples/backend_tutorials/seal/tutorial_backend/original_flow_seal.cpp tutorial original_flow seal 18 | 19 | What we have to do is extract the stages of the testing pipeline from our workflow and integrate them into the C++ wrapper. 20 | 21 | Follow the tutorial steps in order to complete it. 22 | 23 | ## Tutorial steps 24 | 25 | Tutorial Home
26 | [Preparation](backend_tutorial_preparation_seal.md)
27 | [Engine Initialization and Benchmark Description](backend_tutorial_init_seal.md)
28 | [Benchmark Implementation](backend_tutorial_impl_seal.md)
29 | [File References](backend_tutorial_files_seal.md) 30 | -------------------------------------------------------------------------------- /docsrc/examples/backend_tutorials/palisade/backend_tutorial_palisade.md: -------------------------------------------------------------------------------- 1 | Quickstart C++ Wrapper Backend Tutorial - PALISADE {#simple_cpp_example_palisade} 2 | ======================== 3 | 4 | [TOC] 5 | 6 | ## Backend Creation Tutorial 7 | This tutorial details the steps involved in implementing a new backend utilizing the [C++ wrapper](@ref CPP_overview) for use with the testing framework. 8 | 9 | Refer to @ref glossary for a list of terms that will be mentioned throughout this tutorial. 10 | 11 | This guide is intended to be a beginners guide for people who are looking to get a backend with a single test implemented quickly. It is based on using the `api_bridge_example_backend` as a starting point for a new backend implementation which uses the C++ wrapper. 12 | 13 | For the purposes of this guide we will be creating a very basic backend implementing a benchmark for the **vector element-wise addition** workload, under the **offline** category, with **first operand in plain text and second encrypted**, using PALISADE v1.11.3 to perform Homomorphic Encryption (HE) operations. 14 | 15 | Assume that we already have a functional workflow for our workload as listed below, and we want to benchmark it using HEBench. 16 | 17 | \snippet{lineno} docsrc/examples/backend_tutorials/palisade/tutorial_backend/original_flow_palisade.cpp tutorial original_flow palisade 18 | 19 | What we have to do is extract the stages of the testing pipeline from our workflow and integrate them into the C++ wrapper. 20 | 21 | Follow the tutorial steps in order to complete it. 22 | 23 | ## Tutorial steps 24 | 25 | Tutorial Home
26 | [Preparation](backend_tutorial_preparation_palisade.md)
27 | [Engine Initialization and Benchmark Description](backend_tutorial_init_palisade.md)
28 | [Benchmark Implementation](backend_tutorial_impl_palisade.md)
29 | [File References](backend_tutorial_files_palisade.md) 30 | -------------------------------------------------------------------------------- /dataset_loader/include/hebench_dataset_loader_functional_test.h: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2021 Intel Corporation 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | #ifndef __HEBENCH_DATASET_LOADER_FUNCTIONAL_TEST__H__ 5 | #define __HEBENCH_DATASET_LOADER_FUNCTIONAL_TEST__H__ 6 | 7 | #include "hebench_dataset_loader.h" 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | enum DataType 16 | { 17 | i32, 18 | i64, 19 | f32, 20 | f64, 21 | data_fail 22 | }; 23 | 24 | constexpr std::uint8_t help_arg = 1; 25 | constexpr const char *help_msg = "NAME\n\tdata_loader - launches the data loader module.\n\n" 26 | "SYNOPSYS\n\tdata_loader [OPTION] [FILE] [DATA_TYPE] [MAX_LOAD]\n\n" 27 | "DESCRIPTION\n\tIs a launcher app to test the data loader module.\n" 28 | "\tWith no MAX_LOAD, it will be set to 0.\n\n" 29 | "\t-h, --help displays help mesage and exits"; 30 | constexpr const char *check_usage = "Please check the correct usage by typing \"data_loader -h\" or" 31 | " \"data_loader --help\"."; 32 | constexpr const char *help_cstr_short = "-h"; 33 | constexpr const char *help_cstr_long = "--help"; 34 | 35 | constexpr std::uint8_t file_path_arg = 1; 36 | constexpr std::uint8_t type_arg = 2; 37 | constexpr std::uint8_t max_loaded_size_arg = 3; 38 | 39 | constexpr std::uint8_t max_args_allowed = 4; 40 | constexpr std::uint8_t min_args_allowed = 2; 41 | 42 | void signal_error(const char *msg, bool usage = false); 43 | std::uint8_t get_type_from_data(std::string); 44 | void load_from_csv(std::string data, std::filesystem::path file_path, std::uint64_t max_loaded_size); 45 | 46 | #endif //!__HEBENCH_DATASET_LOADER_FUNCTIONAL_TEST__H__ 47 | -------------------------------------------------------------------------------- /docsrc/examples/generic_wl_example/generic_wl_tutorial/src/generic_wl_engine.cpp: -------------------------------------------------------------------------------- 1 | 2 | // Copyright (C) 2021 Intel Corporation 3 | // SPDX-License-Identifier: Apache-2.0 4 | 5 | /// @cond 6 | 7 | #include "../include/generic_wl_engine.h" 8 | #include 9 | 10 | // include all benchmarks 11 | #include "../include/generic_wl_benchmark.h" 12 | 13 | //----------------- 14 | // Engine creation 15 | //----------------- 16 | 17 | namespace hebench { 18 | namespace cpp { 19 | 20 | BaseEngine *createEngine() 21 | { 22 | return ExampleEngine::create(); 23 | } 24 | 25 | void destroyEngine(BaseEngine *p) 26 | { 27 | ExampleEngine *_p = dynamic_cast(p); 28 | ExampleEngine::destroy(_p); 29 | } 30 | 31 | } // namespace cpp 32 | } // namespace hebench 33 | 34 | //--------------------- 35 | // class ExampleEngine 36 | //--------------------- 37 | 38 | ExampleEngine *ExampleEngine::create() 39 | { 40 | ExampleEngine *p_retval = new ExampleEngine(); 41 | p_retval->init(); 42 | return p_retval; 43 | } 44 | 45 | void ExampleEngine::destroy(ExampleEngine *p) 46 | { 47 | if (p) 48 | delete p; 49 | } 50 | 51 | ExampleEngine::ExampleEngine() 52 | { 53 | } 54 | 55 | ExampleEngine::~ExampleEngine() 56 | { 57 | } 58 | 59 | void ExampleEngine::init() 60 | { 61 | // add any new error codes: use 62 | 63 | // this->addErrorCode(code, "generic description"); 64 | 65 | // add supported schemes 66 | 67 | addSchemeName(HEBENCH_HE_SCHEME_PLAIN, "Plain"); 68 | 69 | // add supported security 70 | 71 | addSecurityName(HEBENCH_HE_SECURITY_NONE, "None"); 72 | 73 | // add the all benchmark descriptors 74 | addBenchmarkDescription(std::make_shared(hebench::APIBridge::Category::Latency)); 75 | addBenchmarkDescription(std::make_shared(hebench::APIBridge::Category::Offline)); 76 | } 77 | 78 | /// @endcond 79 | -------------------------------------------------------------------------------- /.github/workflows/install-requirements.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright (C) 2021 Intel Corporation 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | set -e 7 | 8 | echo "This script will install the minimum requirements to run." 9 | echo "It may override previously installed instances of the dependencies therein." 10 | read -r -p "Are you sure you want to continue with this? [y/N] " response 11 | response=${response,,} 12 | if [[ ! "$response" =~ ^(yes|y)$ ]]; then 13 | echo "User confirmation required for installation." 14 | echo "Ending script execution now" 15 | exit 1 16 | fi 17 | 18 | echo -e "\nSTARTING REQUIREMENT INSTALLATION...\n" 19 | 20 | # Formatting 21 | sudo apt-get update 22 | sudo apt-get -y install python3-pip clang-format-9 23 | pip install pre-commit 24 | pre-commit --version 25 | 26 | # CMake 27 | which cmake 28 | mkdir cmake-tmp 29 | pushd cmake-tmp 30 | VERSION=3.15.3 # Use any version 3.13+ (3.15+ required for cmake --install vs make install) 31 | wget https://github.com/Kitware/CMake/releases/download/v$VERSION/cmake-$VERSION-linux-x86_64.sh 32 | sudo chmod +x cmake-$VERSION-linux-x86_64.sh 33 | sudo ./cmake-$VERSION-linux-x86_64.sh --skip-license --prefix=/usr/local # Will overwrite a cmake installation 34 | popd 35 | sudo rm cmake-tmp -rf 36 | echo "Global CMake: $(cmake --version)" 37 | if [ "$(cmake --version | head -n 1)" != "cmake version $VERSION" ]; then 38 | echo "Wrong CMake version installed here: "; 39 | which cmake; 40 | exit 1; 41 | else 42 | echo "Global CMake: $(cmake --version)"; 43 | fi 44 | 45 | # GCC 46 | sudo apt -y install build-essential 47 | sudo apt -y install gcc-9 g++-9 48 | sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 9 49 | sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-9 9 50 | gcc --version 51 | g++ --version 52 | 53 | echo 54 | echo -e "\nCOMPELTED REQUIREMENT INSTALLATION\n" 55 | 56 | exit 0 57 | 58 | -------------------------------------------------------------------------------- /scripts/read_version.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright (C) 2021 Intel Corporation 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | # Reads a version file in the HEBench version format and outputs the 7 | # full formatted version to stdout. 8 | 9 | #exit on any command failure 10 | set -e 11 | tmp_dir="" 12 | 13 | error_exit() 14 | { 15 | if [ -d $tmp_dir ]; then 16 | rm -rf $tmp_dir 17 | fi 18 | if [ "$1" != "0" ]; then 19 | echo "Unexpected error occurred during last operation." 20 | fi 21 | } 22 | 23 | # call exit function on exit to display success/error message 24 | trap 'error_exit $?' EXIT 25 | 26 | if [ -z $1 ] || [ "${1}" == "-h" ] || [ "${1}" == "--help" ]; then 27 | echo "HEBench version parser." 28 | echo " Usage:" 29 | echo " script " 30 | echo 31 | echo " Positional arguments:" 32 | echo " - version_filename: file from which to parse the version tag." 33 | echo 34 | if [ -z $1 ]; then 35 | exit 1 36 | else 37 | exit 0 38 | fi 39 | fi 40 | 41 | VERSION_FILE=$1 42 | 43 | if [ ! -f "${VERSION_FILE}" ]; then 44 | echo "File not found: ${VERSION_FILE}" 45 | exit 1 46 | fi 47 | 48 | mapfile -t VERSION_LINES < ${VERSION_FILE} 49 | if [ ${#VERSION_LINES[@]} -lt 1 ]; then 50 | echo "Invalid version file format." 51 | exit 1 52 | fi 53 | 54 | if [ ${#VERSION_LINES[@]} -gt 1 ] && [ "${VERSION_LINES[0]}" == "v" ]; then 55 | #v2 56 | TAG="v${VERSION_LINES[1]}" 57 | max_lines=${#VERSION_LINES[@]} 58 | if [ $max_lines -gt 4 ]; then 59 | max_lines=4 60 | fi 61 | #v2.5.3 62 | for i in $(seq 2 $((max_lines - 1))); do 63 | TAG="${TAG}.${VERSION_LINES[${i}]}" 64 | done 65 | #v2.5.3-rc 66 | if [ ${#VERSION_LINES[@]} -gt 4 ]; then 67 | TAG="${TAG}-${VERSION_LINES[4]}" 68 | fi 69 | echo $TAG 70 | else 71 | echo "${VERSION_LINES[0]}" 72 | fi 73 | 74 | exit 0 75 | 76 | -------------------------------------------------------------------------------- /docsrc/examples/generic_wl_example/generic_wl_tutorial/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Intel Corporation 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | project(generic_wl_backend) 5 | 6 | set(generic_wl_SOURCES 7 | "${CMAKE_CURRENT_SOURCE_DIR}/src/generic_wl_engine.cpp" 8 | "${CMAKE_CURRENT_SOURCE_DIR}/src/generic_wl_benchmark.cpp" 9 | ) 10 | set(generic_wl_HEADERS 11 | "${CMAKE_CURRENT_SOURCE_DIR}/include/generic_wl_engine.h" 12 | "${CMAKE_CURRENT_SOURCE_DIR}/include/generic_wl_benchmark.h" 13 | ) 14 | 15 | add_library(${PROJECT_NAME} SHARED ${generic_wl_SOURCES} ${generic_wl_HEADERS}) 16 | target_include_directories(${PROJECT_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include) 17 | 18 | # find the hebench_cpp archive 19 | set(HEBENCH_API_BRIDGE_INSTAL_DIR "/install/directory/for/api_bridge") 20 | set(HEBENCH_API_INCLUDE_DIR "${HEBENCH_API_BRIDGE_INSTAL_DIR}/include") 21 | target_include_directories(${PROJECT_NAME} PRIVATE ${HEBENCH_API_INCLUDE_DIR}) # point to include for api_bridge 22 | find_library(hebench_cpp_FOUND NAMES libhebench_cpp.a HINTS "${HEBENCH_API_BRIDGE_INSTAL_DIR}/lib") 23 | if(hebench_cpp_FOUND) 24 | add_library(hebench_cpp UNKNOWN IMPORTED) 25 | # populate the found library with its properties 26 | set_property(TARGET hebench_cpp PROPERTY IMPORTED_LOCATION ${hebench_cpp_FOUND}) 27 | set_property(TARGET hebench_cpp APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${HEBENCH_API_INCLUDE_DIR}) 28 | message(STATUS "libhebench_cpp.a found") 29 | else() 30 | message(FATAL_ERROR "libhebench_cpp.a not found.") 31 | endif() 32 | 33 | # link found library 34 | target_link_libraries(${PROJECT_NAME} PUBLIC "-Wl,--whole-archive" hebench_cpp "-Wl,--no-whole-archive") 35 | 36 | # extra compile options 37 | target_compile_options(${PROJECT_NAME} PRIVATE -Wall -Wextra) # show warnings 38 | 39 | # install options 40 | include(GNUInstallDirs) 41 | set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib") 42 | install(TARGETS ${PROJECT_NAME} DESTINATION lib) 43 | 44 | -------------------------------------------------------------------------------- /dataset_loader/src/hebench_dataset_loader_functional_test.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2021 Intel Corporation 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | #include "hebench_dataset_loader_functional_test.h" 5 | 6 | using namespace hebench::DataLoader; 7 | 8 | void signal_error(const char *msg, bool usage) 9 | { 10 | std::cerr << msg; 11 | if (usage) 12 | { 13 | std::cerr << check_usage; 14 | } 15 | std::cerr << std::endl; 16 | exit(EXIT_FAILURE); 17 | } 18 | 19 | std::uint8_t get_type_from_data(std::string data) 20 | { 21 | std::map dataTypeMap = { { "i32", i32 }, { "i64", i64 }, { "f32", f32 }, { "f64", f64 } }; 22 | try 23 | { 24 | return dataTypeMap.at(data); 25 | } 26 | catch (const std::out_of_range &missing_key) 27 | { 28 | return data_fail; 29 | } 30 | } 31 | 32 | void load_from_csv(std::string data, std::filesystem::path file_path, std::uint64_t max_loaded_size) 33 | { 34 | std::uint8_t data_load_type_evaluated = get_type_from_data(data); 35 | switch (data_load_type_evaluated) 36 | { 37 | case i32: 38 | { 39 | ExternalDataset data_set_i32 = 40 | ExternalDatasetLoader::loadFromCSV(file_path.string(), max_loaded_size); 41 | break; 42 | } 43 | case i64: 44 | { 45 | ExternalDataset data_set_i64 = 46 | ExternalDatasetLoader::loadFromCSV(file_path.string(), max_loaded_size); 47 | break; 48 | } 49 | case f32: 50 | { 51 | ExternalDataset data_set_f32 = 52 | ExternalDatasetLoader::loadFromCSV(file_path.string(), max_loaded_size); 53 | break; 54 | } 55 | case f64: 56 | { 57 | ExternalDataset data_set_f64 = 58 | ExternalDatasetLoader::loadFromCSV(file_path.string(), max_loaded_size); 59 | break; 60 | } 61 | case data_fail: 62 | { 63 | signal_error("Unknown 'data_type' has been specified."); 64 | break; 65 | } 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /.github/validation-ci-ssi/testharness_gen_PSI_X_bigger_than_Y.sh: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Intel Corporation 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | < |Y|. 9 | com 10 | 11 | # server paths 12 | CLEARTEXTLIB=$(realpath ./lib/libhebench_cleartext_backend.so) 13 | TESTHARNESS=$(realpath ./bin/test_harness) 14 | 15 | # IDs relevant for offline and latency, covering all the types. 16 | for id in 48 49 50 51 52 53 54 55; do 17 | # testing even and odd k 18 | for k in 14 15; do 19 | 20 | data="default_min_test_time: 0 21 | default_sample_size: 0 22 | random_seed: 1234 23 | benchmark: 24 | - ID: $id 25 | dataset: ~ 26 | default_min_test_time: 0 27 | default_sample_sizes: 28 | 0: 0 29 | 1: 0 30 | params: 31 | 0: 32 | name: n 33 | type: UInt64 34 | value: 35 | from: 5 36 | to: 5 37 | step: 0 38 | 1: 39 | name: m 40 | type: UInt64 41 | value: 42 | from: 3 43 | to: 3 44 | step: 0 45 | 2: 46 | name: k 47 | type: UInt64 48 | value: 49 | from: $k 50 | to: $k 51 | step: 0" 52 | 53 | file=psi.yaml 54 | echo "$data" > "$(pwd)/$file" 55 | 56 | "$TESTHARNESS" --backend_lib_path "$CLEARTEXTLIB" --config_file "$(pwd)/$file" | tee test_harness_output.log 57 | if ! grep -Fxq "[ Info ] Failed: 0" test_harness_output.log 58 | then 59 | echo "Failed to test the PSI's worload with: $0" 60 | echo "Workload ID: $id" 61 | echo "k: $k" 62 | # clean-up 63 | rm "$(pwd)/$file" 64 | exit 1 65 | else 66 | echo "Successfully tested Test Harness' PSI with: $0" 67 | echo "Workload ID: $id" 68 | echo "k: $k" 69 | fi 70 | done # k related for 71 | done # ID related for 72 | 73 | # clean-up 74 | rm "$(pwd)/$file" 75 | 76 | exit 0 -------------------------------------------------------------------------------- /.github/validation-ci-ssi/testharness_gen_PSI_Y_bigger_than_X.sh: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Intel Corporation 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | < |X|. 9 | 10 | com 11 | 12 | # server paths 13 | CLEARTEXTLIB=$(realpath ./lib/libhebench_cleartext_backend.so) 14 | TESTHARNESS=$(realpath ./bin/test_harness) 15 | 16 | # IDs relevant for offline and latency, covering all the types. 17 | for id in 48 49 50 51 52 53 54 55; do 18 | # testing even and odd k values 19 | for k in 14 15; do 20 | 21 | data="default_min_test_time: 0 22 | default_sample_size: 0 23 | random_seed: 1234 24 | benchmark: 25 | - ID: $id 26 | dataset: ~ 27 | default_min_test_time: 0 28 | default_sample_sizes: 29 | 0: 0 30 | 1: 0 31 | params: 32 | 0: 33 | name: n 34 | type: UInt64 35 | value: 36 | from: 3 37 | to: 3 38 | step: 0 39 | 1: 40 | name: m 41 | type: UInt64 42 | value: 43 | from: 5 44 | to: 5 45 | step: 0 46 | 2: 47 | name: k 48 | type: UInt64 49 | value: 50 | from: $k 51 | to: $k 52 | step: 0" 53 | 54 | file=psi.yaml 55 | echo "$data" > "$(pwd)/$file" 56 | 57 | "$TESTHARNESS" --backend_lib_path "$CLEARTEXTLIB" --config_file "$(pwd)/$file" | tee test_harness_output.log 58 | if ! grep -Fxq "[ Info ] Failed: 0" test_harness_output.log 59 | then 60 | echo "Failed to test the PSI's worload with: $0" 61 | echo "Workload ID: $id" 62 | echo "k: $k" 63 | # clean-up 64 | rm "$(pwd)/$file" 65 | exit 1 66 | else 67 | echo "Successfully tested Test Harness' PSI with: $0" 68 | echo "Workload ID: $id" 69 | echo "k: $k" 70 | fi 71 | done # k related for 72 | done # ID related for 73 | 74 | # clean-up 75 | rm "$(pwd)/$file" 76 | 77 | exit 0 -------------------------------------------------------------------------------- /.github/validation-ci-ssi/testharness_dsl_PSI_X_bigger_than_Y.sh: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Intel Corporation 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | < |Y|. 9 | com 10 | 11 | # server paths 12 | CLEARTEXTLIB=$(realpath ./lib/libhebench_cleartext_backend.so) 13 | TESTHARNESS=$(realpath ./bin/test_harness) 14 | 15 | # IDs relevant for offline and latency, covering all the types. 16 | for id in 48 49 50 51 52 53 54 55; do 17 | # testing even and odd k 18 | for k in 14 15; do 19 | 20 | dataset="input, 0, 1, local, $k, 0 21 | Brazil, , Canada, , Colombia, , Mexico, , United States 22 | 23 | input, 1, 1, local, $k, 0 24 | Canada, , United States, , Ivory Coast" 25 | 26 | dataset_file=countries.csv 27 | echo "$dataset" > "$(pwd)/$dataset_file" 28 | 29 | data="default_min_test_time: 0 30 | default_sample_size: 0 31 | random_seed: 1234 32 | benchmark: 33 | - ID: $id 34 | dataset: $(pwd)/$dataset_file 35 | default_min_test_time: 0 36 | default_sample_sizes: 37 | 0: 0 38 | 1: 0 39 | params: 40 | 0: 41 | name: n 42 | type: UInt64 43 | value: 44 | from: 5 45 | to: 5 46 | step: 0 47 | 1: 48 | name: m 49 | type: UInt64 50 | value: 51 | from: 3 52 | to: 3 53 | step: 0 54 | 2: 55 | name: k 56 | type: UInt64 57 | value: 58 | from: $k 59 | to: $k 60 | step: 0" 61 | 62 | file=psi.yaml 63 | echo "$data" > "$(pwd)/$file" 64 | 65 | "$TESTHARNESS" --backend_lib_path "$CLEARTEXTLIB" --config_file "$(pwd)/$file" | tee test_harness_output.log 66 | if ! grep -Fxq "[ Info ] Failed: 0" test_harness_output.log 67 | then 68 | echo "Failed to test the PSI's worload with: $0" 69 | echo "Workload ID: $id" 70 | echo "k: $k" 71 | # clean-up 72 | rm "$(pwd)/$file" 73 | rm "$(pwd)/$dataset_file" 74 | exit 1 75 | else 76 | echo "Successfully tested Test Harness' PSI with: $0" 77 | echo "Workload ID: $id" 78 | echo "k: $k" 79 | fi 80 | done # k related for 81 | done # ID related for 82 | 83 | # clean-up 84 | rm "$(pwd)/$file" 85 | rm "$(pwd)/$dataset_file" 86 | 87 | exit 0 -------------------------------------------------------------------------------- /.github/validation-ci-ssi/testharness_dsl_PSI_k_smaller_than_biggest_element.sh: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Intel Corporation 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | < "$(pwd)/$dataset_file" 28 | 29 | data="default_min_test_time: 0 30 | default_sample_size: 0 31 | random_seed: 1234 32 | benchmark: 33 | - ID: $id 34 | dataset: $(pwd)/$dataset_file 35 | default_min_test_time: 0 36 | default_sample_sizes: 37 | 0: 0 38 | 1: 0 39 | params: 40 | 0: 41 | name: n 42 | type: UInt64 43 | value: 44 | from: 5 45 | to: 5 46 | step: 0 47 | 1: 48 | name: m 49 | type: UInt64 50 | value: 51 | from: 3 52 | to: 3 53 | step: 0 54 | 2: 55 | name: k 56 | type: UInt64 57 | value: 58 | from: $k 59 | to: $k 60 | step: 0" 61 | 62 | file=psi.yaml 63 | echo "$data" > "$(pwd)/$file" 64 | 65 | "$TESTHARNESS" --backend_lib_path "$CLEARTEXTLIB" --config_file "$(pwd)/$file" 66 | if [ $? -ne 255 ]; 67 | then 68 | echo "Failed to test the PSI's worload with: $0" 69 | echo "Workload ID: $id" 70 | echo "k: $k" 71 | # clean-up 72 | rm "$(pwd)/$file" 73 | rm "$(pwd)/$dataset_file" 74 | exit 1 75 | else 76 | echo "Successfully tested Test Harness' PSI with: $0" 77 | echo "Workload ID: $id" 78 | echo "k: $k" 79 | fi 80 | done # k related for 81 | done # ID related for 82 | 83 | # clean-up 84 | rm "$(pwd)/$file" 85 | rm "$(pwd)/$dataset_file" 86 | 87 | exit 0 -------------------------------------------------------------------------------- /.github/validation-ci-ssi/testharness_dsl_PSI_k_mismatch_between_config_and_dataset.sh: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Intel Corporation 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | < "$(pwd)/$dataset_file" 24 | 25 | # IDs relevant for offline and latency, covering all the types. 26 | for id in 48 49 50 51 52 53 54 55; do 27 | # testing k even and odd, 20 does not match with the sequence 28 | for k in 14 15; do 29 | 30 | data="default_min_test_time: 0 31 | default_sample_size: 0 32 | random_seed: 1234 33 | benchmark: 34 | - ID: $id 35 | dataset: $(pwd)/$dataset_file 36 | default_min_test_time: 0 37 | default_sample_sizes: 38 | 0: 0 39 | 1: 0 40 | params: 41 | 0: 42 | name: n 43 | type: UInt64 44 | value: 45 | from: 5 46 | to: 5 47 | step: 0 48 | 1: 49 | name: m 50 | type: UInt64 51 | value: 52 | from: 3 53 | to: 3 54 | step: 0 55 | 2: 56 | name: k 57 | type: UInt64 58 | value: 59 | from: $k 60 | to: $k 61 | step: 0" 62 | 63 | file=psi.yaml 64 | echo "$data" > "$(pwd)/$file" 65 | 66 | "$TESTHARNESS" --backend_lib_path "$CLEARTEXTLIB" --config_file "$(pwd)/$file" 67 | if [ $? -ne 255 ]; 68 | then 69 | echo "Failed to test the PSI's worload with: $0" 70 | echo "Workload ID: $id" 71 | echo "k: $k" 72 | # clean-up 73 | rm "$(pwd)/$file" 74 | rm "$(pwd)/$dataset_file" 75 | exit 1 76 | else 77 | echo "Successfully tested Test Harness' PSI with: $0" 78 | echo "Workload ID: $id" 79 | echo "k: $k" 80 | fi 81 | done # k related for 82 | done # ID related for 83 | 84 | # clean-up 85 | rm "$(pwd)/$file" 86 | rm "$(pwd)/$dataset_file" 87 | 88 | exit 0 -------------------------------------------------------------------------------- /.github/validation-ci-ssi/testharness_dsl_PSI_X_in_Y.sh: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Intel Corporation 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | < "$(pwd)/$dataset_file" 28 | 29 | 30 | data="default_min_test_time: 0 31 | default_sample_size: 0 32 | random_seed: 1234 33 | benchmark: 34 | - ID: $id 35 | dataset: $(pwd)/$dataset_file 36 | default_min_test_time: 0 37 | default_sample_sizes: 38 | 0: 0 39 | 1: 0 40 | params: 41 | 0: 42 | name: n 43 | type: UInt64 44 | value: 45 | from: 2 46 | to: 2 47 | step: 0 48 | 1: 49 | name: m 50 | type: UInt64 51 | value: 52 | from: 5 53 | to: 5 54 | step: 0 55 | 2: 56 | name: k 57 | type: UInt64 58 | value: 59 | from: $k 60 | to: $k 61 | step: 0" 62 | 63 | file=psi.yaml 64 | echo "$data" > "$(pwd)/$file" 65 | 66 | "$TESTHARNESS" --backend_lib_path "$CLEARTEXTLIB" --config_file "$(pwd)/$file" | tee test_harness_output.log 67 | if ! grep -Fxq "[ Info ] Failed: 0" test_harness_output.log 68 | then 69 | echo "Failed to test the PSI's worload with: $0" 70 | echo "Workload ID: $id" 71 | echo "k: $k" 72 | # clean-up 73 | rm "$(pwd)/$file" 74 | rm "$(pwd)/$dataset_file" 75 | exit 1 76 | else 77 | echo "Successfully tested Test Harness' PSI with: $0" 78 | echo "Workload ID: $id" 79 | echo "k: $k" 80 | fi 81 | done # k related for 82 | done # ID related for 83 | 84 | # clean-up 85 | rm "$(pwd)/$file" 86 | rm "$(pwd)/$dataset_file" 87 | 88 | exit 0 -------------------------------------------------------------------------------- /.github/validation-ci-ssi/testharness_dsl_PSI_Y_bigger_than_X.sh: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Intel Corporation 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | < |X|. 9 | 10 | com 11 | 12 | # server paths 13 | CLEARTEXTLIB=$(realpath ./lib/libhebench_cleartext_backend.so) 14 | TESTHARNESS=$(realpath ./bin/test_harness) 15 | 16 | # IDs relevant for offline and latency, covering all the types. 17 | for id in 48 49 50 51 52 53 54 55; do 18 | # testing even and odd k values 19 | for k in 14 15; do 20 | dataset="input, 0, 1, local, $k, 0 21 | Canada, , United States, , Ivory Coast 22 | 23 | input, 1, 1, local, $k, 0 24 | Brazil, , Canada, , Colombia, , Mexico, , United States" 25 | 26 | dataset_file=countries.csv 27 | echo "$dataset" > "$(pwd)/$dataset_file" 28 | 29 | data="default_min_test_time: 0 30 | default_sample_size: 0 31 | random_seed: 1234 32 | benchmark: 33 | - ID: $id 34 | dataset: $(pwd)/$dataset_file 35 | default_min_test_time: 0 36 | default_sample_sizes: 37 | 0: 0 38 | 1: 0 39 | params: 40 | 0: 41 | name: n 42 | type: UInt64 43 | value: 44 | from: 3 45 | to: 3 46 | step: 0 47 | 1: 48 | name: m 49 | type: UInt64 50 | value: 51 | from: 5 52 | to: 5 53 | step: 0 54 | 2: 55 | name: k 56 | type: UInt64 57 | value: 58 | from: $k 59 | to: $k 60 | step: 0" 61 | 62 | file=psi.yaml 63 | echo "$data" > "$(pwd)/$file" 64 | 65 | "$TESTHARNESS" --backend_lib_path "$CLEARTEXTLIB" --config_file "$(pwd)/$file" | tee test_harness_output.log 66 | if ! grep -Fxq "[ Info ] Failed: 0" test_harness_output.log 67 | then 68 | echo "Failed to test the PSI's worload with: $0" 69 | echo "Workload ID: $id" 70 | echo "k: $k" 71 | # clean-up 72 | rm "$(pwd)/$file" 73 | rm "$(pwd)/$dataset_file" 74 | exit 1 75 | else 76 | echo "Successfully tested Test Harness' PSI with: $0" 77 | echo "Workload ID: $id" 78 | echo "k: $k" 79 | fi 80 | done # k related for 81 | done # ID related for 82 | 83 | # clean-up 84 | rm "$(pwd)/$file" 85 | rm "$(pwd)/$dataset_file" 86 | 87 | exit 0 -------------------------------------------------------------------------------- /.github/validation-ci-ssi/testharness_dsl_PSI_Y_in_X.sh: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Intel Corporation 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | < "$(pwd)/$dataset_file" 29 | 30 | data="default_min_test_time: 0 31 | default_sample_size: 0 32 | random_seed: 1234 33 | benchmark: 34 | - ID: $id 35 | dataset: $(pwd)/$dataset_file 36 | default_min_test_time: 0 37 | default_sample_sizes: 38 | 0: 0 39 | 1: 0 40 | params: 41 | 0: 42 | name: n 43 | type: UInt64 44 | value: 45 | from: 5 46 | to: 5 47 | step: 0 48 | 1: 49 | name: m 50 | type: UInt64 51 | value: 52 | from: 2 53 | to: 2 54 | step: 0 55 | 2: 56 | name: k 57 | type: UInt64 58 | value: 59 | from: $k 60 | to: $k 61 | step: 0" 62 | 63 | file=psi.yaml 64 | echo "$data" > "$(pwd)/$file" 65 | 66 | "$TESTHARNESS" --backend_lib_path "$CLEARTEXTLIB" --config_file "$(pwd)/$file" | tee test_harness_output.log 67 | if ! grep -Fxq "[ Info ] Failed: 0" test_harness_output.log 68 | then 69 | echo "Failed to test the PSI's worload with: $0" 70 | echo "Workload ID: $id" 71 | echo "k: $k" 72 | # clean-up 73 | rm "$(pwd)/$file" 74 | rm "$(pwd)/$dataset_file" 75 | exit 1 76 | else 77 | echo "Successfully tested Test Harness' PSI with: $0" 78 | echo "Workload ID: $id" 79 | echo "k: $k" 80 | fi 81 | done # k related for 82 | done # ID related for 83 | 84 | # clean-up 85 | rm "$(pwd)/$file" 86 | rm "$(pwd)/$dataset_file" 87 | 88 | exit 0 -------------------------------------------------------------------------------- /.github/validation-ci-ssi/testharness_dsl_PSI_X_intersect_Y_is_empty.sh: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Intel Corporation 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | < "$(pwd)/$dataset_file" 29 | 30 | data="default_min_test_time: 0 31 | default_sample_size: 0 32 | random_seed: 1234 33 | benchmark: 34 | - ID: $id 35 | dataset: $(pwd)/$dataset_file 36 | default_min_test_time: 0 37 | default_sample_sizes: 38 | 0: 0 39 | 1: 0 40 | params: 41 | 0: 42 | name: n 43 | type: UInt64 44 | value: 45 | from: 4 46 | to: 4 47 | step: 0 48 | 1: 49 | name: m 50 | type: UInt64 51 | value: 52 | from: 2 53 | to: 2 54 | step: 0 55 | 2: 56 | name: k 57 | type: UInt64 58 | value: 59 | from: $k 60 | to: $k 61 | step: 0" 62 | 63 | file=psi.yaml 64 | echo "$data" > "$(pwd)/$file" 65 | 66 | "$TESTHARNESS" --backend_lib_path "$CLEARTEXTLIB" --config_file "$(pwd)/$file" | tee test_harness_output.log 67 | if ! grep -Fxq "[ Info ] Failed: 0" test_harness_output.log 68 | then 69 | echo "Failed to test the PSI's worload with: $0" 70 | echo "Workload ID: $id" 71 | echo "k: $k" 72 | # clean-up 73 | rm "$(pwd)/$file" 74 | rm "$(pwd)/$dataset_file" 75 | exit 1 76 | else 77 | echo "Successfully tested Test Harness' PSI with: $0" 78 | echo "Workload ID: $id" 79 | echo "k: $k" 80 | fi 81 | done # k related for 82 | done # ID related for 83 | 84 | # clean-up 85 | rm "$(pwd)/$file" 86 | rm "$(pwd)/$dataset_file" 87 | 88 | exit 0 -------------------------------------------------------------------------------- /docsrc/examples/backend_tutorials/seal/tutorial_backend/src/tutorial_engine_seal.cpp: -------------------------------------------------------------------------------- 1 | 2 | // Copyright (C) 2021 Intel Corporation 3 | // SPDX-License-Identifier: Apache-2.0 4 | 5 | /// @cond 6 | 7 | #include 8 | 9 | #include "tutorial_engine_seal.h" 10 | #include "tutorial_error_seal.h" 11 | 12 | // include all benchmarks 13 | #include "tutorial_eltwiseadd_benchmark_seal.h" 14 | 15 | //----------------- 16 | // Engine creation 17 | //----------------- 18 | 19 | namespace hebench { 20 | namespace cpp { 21 | 22 | BaseEngine *createEngine() 23 | { 24 | // It is a good idea to check here if the API Bridge version is correct for 25 | // our backend by checking against the constants defined in `hebench/api_bridge/version_seal.h` 26 | // HEBENCH_API_VERSION_* 27 | // For simplicity purposes, no check is performed in this tutorial. 28 | 29 | return TutorialEngine::create(); 30 | } 31 | 32 | void destroyEngine(BaseEngine *p) 33 | { 34 | TutorialEngine *_p = dynamic_cast(p); 35 | TutorialEngine::destroy(_p); 36 | } 37 | 38 | } // namespace cpp 39 | } // namespace hebench 40 | 41 | //--------------------- 42 | // class ExampleEngine 43 | //--------------------- 44 | 45 | TutorialEngine *TutorialEngine::create() 46 | { 47 | TutorialEngine *p_retval = new TutorialEngine(); 48 | p_retval->init(); 49 | return p_retval; 50 | } 51 | 52 | void TutorialEngine::destroy(TutorialEngine *p) 53 | { 54 | if (p) 55 | delete p; 56 | } 57 | 58 | TutorialEngine::TutorialEngine() 59 | { 60 | } 61 | 62 | TutorialEngine::~TutorialEngine() 63 | { 64 | } 65 | 66 | //! [engine init] 67 | void TutorialEngine::init() 68 | { 69 | //! [engine init error codes] 70 | // add any new error codes 71 | 72 | addErrorCode(TUTORIAL_ECODE_SEAL_ERROR, "SEAL error."); 73 | //! [engine init error codes] 74 | 75 | //! [engine init schemes] 76 | // add supported schemes 77 | 78 | //addSchemeName(HEBENCH_HE_SCHEME_PLAIN, "Plain"); 79 | addSchemeName(HEBENCH_HE_SCHEME_BFV, "BFV"); 80 | //! [engine init schemes] 81 | 82 | //! [engine init security] 83 | // add supported security 84 | 85 | //addSecurityName(HEBENCH_HE_SECURITY_NONE, "None"); 86 | addSecurityName(TUTORIAL_HE_SECURITY_128, "128 bits"); 87 | //! [engine init security] 88 | 89 | //! [engine init benchmarks] 90 | // add the all benchmark descriptors 91 | addBenchmarkDescription(std::make_shared()); 92 | //! [engine init benchmarks] 93 | } 94 | //! [engine init] 95 | 96 | /// @endcond 97 | -------------------------------------------------------------------------------- /.clang-format: -------------------------------------------------------------------------------- 1 | --- 2 | --- 3 | Language: Cpp 4 | BasedOnStyle: Microsoft 5 | AccessModifierOffset: -4 6 | AlignAfterOpenBracket: Align 7 | AlignConsecutiveAssignments: true 8 | AlignConsecutiveMacros: true 9 | AlignOperands: true 10 | AlignTrailingComments: false 11 | AllowAllArgumentsOnNextLine: true 12 | AllowAllConstructorInitializersOnNextLine: true 13 | AllowAllParametersOfDeclarationOnNextLine: true 14 | AllowShortCaseLabelsOnASingleLine: false 15 | AllowShortFunctionsOnASingleLine: Inline 16 | AllowShortIfStatementsOnASingleLine: false 17 | AllowShortLoopsOnASingleLine: false 18 | AlwaysBreakAfterDefinitionReturnType: None 19 | AlwaysBreakAfterReturnType: None 20 | AlwaysBreakTemplateDeclarations: true 21 | BinPackArguments: true 22 | BinPackParameters: true 23 | BraceWrapping: 24 | AfterCaseLabel: true 25 | AfterClass: true 26 | AfterControlStatement: true 27 | AfterEnum: true 28 | AfterFunction: true 29 | AfterNamespace: false 30 | AfterObjCDeclaration: true 31 | AfterStruct: true 32 | AfterUnion: true 33 | BeforeCatch: true 34 | BeforeElse: true 35 | IndentBraces: false 36 | BreakBeforeBinaryOperators: NonAssignment 37 | BreakBeforeBraces: Custom 38 | BreakBeforeTernaryOperators: false 39 | BreakConstructorInitializers: AfterColon 40 | ColumnLimit: 0 41 | CompactNamespaces: false 42 | ConstructorInitializerAllOnOneLineOrOnePerLine: false 43 | ConstructorInitializerIndentWidth: 4 44 | ContinuationIndentWidth: 4 45 | Cpp11BracedListStyle: false 46 | DerivePointerAlignment: false 47 | FixNamespaceComments: true 48 | IndentCaseLabels: false 49 | IndentPPDirectives: None 50 | IndentWidth: 4 51 | NamespaceIndentation: None 52 | PenaltyBreakBeforeFirstCallParameter: 19 53 | PenaltyBreakComment: 300 54 | PenaltyBreakFirstLessLess: 120 55 | PenaltyBreakString: 1000 56 | PenaltyBreakTemplateDeclaration: 10 57 | PenaltyExcessCharacter: 1000000 58 | PenaltyReturnTypeOnItsOwnLine: 1000 59 | PointerAlignment: Right 60 | SortIncludes: true 61 | SortUsingDeclarations: true 62 | SpaceAfterCStyleCast: false 63 | SpaceAfterLogicalNot: false 64 | SpaceBeforeAssignmentOperators: true 65 | SpaceBeforeCpp11BracedList: false 66 | SpaceBeforeCtorInitializerColon: true 67 | SpaceBeforeInheritanceColon: true 68 | SpaceBeforeParens: ControlStatements 69 | SpaceBeforeRangeBasedForLoopColon: true 70 | SpaceInEmptyParentheses: false 71 | SpacesBeforeTrailingComments: 1 72 | SpacesInAngles: false 73 | SpacesInContainerLiterals: true 74 | SpacesInCStyleCastParentheses: false 75 | SpacesInParentheses: false 76 | SpacesInSquareBrackets: false 77 | ReflowComments: false 78 | UseTab: Never 79 | -------------------------------------------------------------------------------- /docsrc/examples/backend_tutorials/palisade/tutorial_backend/src/tutorial_engine_palisade.cpp: -------------------------------------------------------------------------------- 1 | 2 | // Copyright (C) 2021 Intel Corporation 3 | // SPDX-License-Identifier: Apache-2.0 4 | 5 | /// @cond 6 | 7 | #include 8 | 9 | #include "tutorial_engine_palisade.h" 10 | #include "tutorial_error_palisade.h" 11 | 12 | // include all benchmarks 13 | #include "tutorial_eltwiseadd_benchmark_palisade.h" 14 | 15 | //----------------- 16 | // Engine creation 17 | //----------------- 18 | 19 | namespace hebench { 20 | namespace cpp { 21 | 22 | BaseEngine *createEngine() 23 | { 24 | // It is a good idea to check here if the API Bridge version is correct for 25 | // our backend by checking against the constants defined in `hebench/api_bridge/version_palisade.h` 26 | // HEBENCH_API_VERSION_* 27 | // For simplicity purposes, no check is performed in this tutorial. 28 | 29 | return TutorialEngine::create(); 30 | } 31 | 32 | void destroyEngine(BaseEngine *p) 33 | { 34 | TutorialEngine *_p = dynamic_cast(p); 35 | TutorialEngine::destroy(_p); 36 | } 37 | 38 | } // namespace cpp 39 | } // namespace hebench 40 | 41 | //--------------------- 42 | // class ExampleEngine 43 | //--------------------- 44 | 45 | TutorialEngine *TutorialEngine::create() 46 | { 47 | TutorialEngine *p_retval = new TutorialEngine(); 48 | p_retval->init(); 49 | return p_retval; 50 | } 51 | 52 | void TutorialEngine::destroy(TutorialEngine *p) 53 | { 54 | if (p) 55 | delete p; 56 | } 57 | 58 | TutorialEngine::TutorialEngine() 59 | { 60 | } 61 | 62 | TutorialEngine::~TutorialEngine() 63 | { 64 | } 65 | 66 | //! [engine init] 67 | void TutorialEngine::init() 68 | { 69 | //! [engine init error codes] 70 | // add any new error codes 71 | 72 | addErrorCode(TUTORIAL_ECODE_PALISADE_ERROR, "PALISADE error."); 73 | //! [engine init error codes] 74 | 75 | //! [engine init schemes] 76 | // add supported schemes 77 | 78 | //addSchemeName(HEBENCH_HE_SCHEME_PLAIN, "Plain"); 79 | addSchemeName(HEBENCH_HE_SCHEME_BFV, "BFV"); 80 | //! [engine init schemes] 81 | 82 | //! [engine init security] 83 | // add supported security 84 | 85 | //addSecurityName(HEBENCH_HE_SECURITY_NONE, "None"); 86 | addSecurityName(TUTORIAL_HE_SECURITY_128, "128 bits"); 87 | //! [engine init security] 88 | 89 | //! [engine init benchmarks] 90 | // add the all benchmark descriptors 91 | addBenchmarkDescription(std::make_shared()); 92 | //! [engine init benchmarks] 93 | } 94 | //! [engine init] 95 | 96 | /// @endcond 97 | -------------------------------------------------------------------------------- /test_harness/benchmarks/categories/include/hebench_benchmark_latency.h: -------------------------------------------------------------------------------- 1 | 2 | // Copyright (C) 2021 Intel Corporation 3 | // SPDX-License-Identifier: Apache-2.0 4 | 5 | #ifndef _HEBench_Harness_Benchmark_Latency_H_0596d40a3cce4b108a81595c50eb286d 6 | #define _HEBench_Harness_Benchmark_Latency_H_0596d40a3cce4b108a81595c50eb286d 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | #include "hebench/modules/general/include/nocopy.h" 13 | #include "hebench/modules/logging/include/logging.h" 14 | 15 | #include "hebench_benchmark_category.h" 16 | 17 | namespace hebench { 18 | namespace TestHarness { 19 | 20 | class Engine; 21 | 22 | /** 23 | * @brief Base class for workload benchmarks in the latency category. 24 | * @details This class offers an implementation to latency category benchmarking. 25 | * 26 | * To extend this class, first clients must understand the execution flow as specified 27 | * in IBenchmark interface. Then, see \ref extend_test_harness_l . 28 | */ 29 | class BenchmarkLatency : public PartialBenchmarkCategory 30 | { 31 | public: 32 | DISABLE_COPY(BenchmarkLatency) 33 | DISABLE_MOVE(BenchmarkLatency) 34 | private: 35 | IL_DECLARE_CLASS_NAME(BenchmarkLatency) 36 | 37 | public: 38 | typedef std::shared_ptr Ptr; 39 | 40 | ~BenchmarkLatency() override; 41 | 42 | /** 43 | * @brief Executes the benchmark latency test. 44 | * @param out_report Object where to append the report of the operation. 45 | * @param config Specifies configuration parameters for the run. 46 | * @returns true if benchmark succeeded and operation results were valid. 47 | * @returns false if benchmark operation results were not valid. 48 | * @details 49 | * In this implementation, validation failure is reported on the first 50 | * result that fails validation. 51 | * 52 | * - Returns when no errors (`true`), or validation failed (`false`). 53 | * - hebench::Common::ErrorException with backend error number attached if 54 | * backend errors occur. 55 | * - Throws std::exception (or derived exception other than hebench::Common::ErrorException) 56 | * on any other errors. 57 | */ 58 | bool run(hebench::Utilities::TimingReportEx &out_report, IBenchmark::RunConfig &config) override; 59 | 60 | protected: 61 | BenchmarkLatency(std::shared_ptr p_engine, 62 | const IBenchmarkDescriptor::DescriptionToken &description_token); 63 | 64 | private: 65 | bool run(hebench::Utilities::TimingReportEx &out_report, 66 | IDataLoader::Ptr p_dataset, 67 | IBenchmark::RunConfig &run_config); 68 | }; 69 | 70 | } // namespace TestHarness 71 | } // namespace hebench 72 | 73 | #endif // defined _HEBench_Harness_Benchmark_Latency_H_0596d40a3cce4b108a81595c50eb286d 74 | -------------------------------------------------------------------------------- /test_harness/benchmarks/categories/include/hebench_benchmark_offline.h: -------------------------------------------------------------------------------- 1 | 2 | // Copyright (C) 2021 Intel Corporation 3 | // SPDX-License-Identifier: Apache-2.0 4 | 5 | #ifndef _HEBench_Harness_Benchmark_Offline_H_0596d40a3cce4b108a81595c50eb286d 6 | #define _HEBench_Harness_Benchmark_Offline_H_0596d40a3cce4b108a81595c50eb286d 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | #include "hebench/modules/general/include/nocopy.h" 13 | #include "hebench/modules/logging/include/logging.h" 14 | 15 | #include "hebench_benchmark_category.h" 16 | 17 | namespace hebench { 18 | namespace TestHarness { 19 | 20 | class Engine; 21 | 22 | /** 23 | * @brief Base class for workload benchmarks in the offline category. 24 | * @details This class offers an implementation to offline category benchmarking. 25 | * 26 | * To extend this class, first clients must understand the execution flow as specified 27 | * in IBenchmark interface. Then, see \ref extend_test_harness_o . 28 | */ 29 | class BenchmarkOffline : public PartialBenchmarkCategory 30 | { 31 | public: 32 | DISABLE_COPY(BenchmarkOffline) 33 | DISABLE_MOVE(BenchmarkOffline) 34 | private: 35 | IL_DECLARE_CLASS_NAME(BenchmarkOffline) 36 | 37 | public: 38 | typedef std::shared_ptr Ptr; 39 | 40 | ~BenchmarkOffline() override; 41 | 42 | /** 43 | * @brief Executes the benchmark offline test. 44 | * @param out_report Object where to append the report of the operation. 45 | * @param config Specifies configuration parameters for the run. 46 | * @returns true if benchmark succeeded and operation results were valid. 47 | * @returns false if benchmark operation results were not valid. 48 | * @details 49 | * In this implementation, validation failure is reported on the first 50 | * result that fails validation. 51 | * 52 | * - Returns when no errors (`true`), or validation failed (`false`). 53 | * - hebench::Common::ErrorException with backend error number attached if 54 | * backend errors occur. 55 | * - Throws std::exception (or derived exception other than hebench::Common::ErrorException) 56 | * on any other errors. 57 | */ 58 | bool run(hebench::Utilities::TimingReportEx &out_report, IBenchmark::RunConfig &config) override; 59 | 60 | protected: 61 | BenchmarkOffline(std::shared_ptr p_engine, 62 | const IBenchmarkDescriptor::DescriptionToken &description_token); 63 | 64 | private: 65 | bool run(hebench::Utilities::TimingReportEx &out_report, 66 | IDataLoader::Ptr p_dataset, 67 | IBenchmark::RunConfig &run_config); 68 | }; 69 | 70 | } // namespace TestHarness 71 | } // namespace hebench 72 | 73 | #endif // defined _HEBench_Harness_Benchmark_Offline_H_0596d40a3cce4b108a81595c50eb286d 74 | -------------------------------------------------------------------------------- /report_gen/report_compiler_lib/include/hebench_report_compiler.h: -------------------------------------------------------------------------------- 1 | 2 | // Copyright (C) 2021 Intel Corporation 3 | // SPDX-License-Identifier: Apache-2.0 4 | 5 | #ifndef _HEBench_ReportCompiler_H_0596d40a3cce4b108a81595c50eb286d 6 | #define _HEBench_ReportCompiler_H_0596d40a3cce4b108a81595c50eb286d 7 | 8 | #include "stddef.h" 9 | #include "stdint.h" 10 | 11 | extern "C" 12 | { 13 | 14 | namespace hebench { 15 | namespace ReportGen { 16 | namespace Compiler { 17 | 18 | /** 19 | * @brief Configuration for a compiler run. 20 | */ 21 | struct ReportCompilerConfigC 22 | { 23 | /** 24 | * @brief C-string containing the input file name. 25 | */ 26 | const char *input_file; 27 | /** 28 | * @brief If non-zero, the compiled overview will be output to stdout. 29 | */ 30 | int32_t b_show_overview; 31 | /** 32 | * @brief If non-zero, the run details will be omited. Any warning, error, 33 | * or important messages are directed to stderr. 34 | */ 35 | int32_t b_silent; 36 | /** 37 | * @brief Fallback time unit when no time unit is specified for a specific output. 38 | * @details One of: 39 | * `0` - (default): an appropriate time unit will be used to keep the values between 1 and 1000. 40 | * `s` - seconds 41 | * `ms` - milliseconds 42 | * `us` - microseconds 43 | * `ns` - nanoseconds 44 | */ 45 | char time_unit; 46 | /** 47 | * @brief Time unit for report statistics. If `0`, the fallback `time_unit` will be used. 48 | */ 49 | char time_unit_stats; 50 | /** 51 | * @brief Time unit for report overview. If `0`, the fallback `time_unit` will be used. 52 | */ 53 | char time_unit_overview; 54 | /** 55 | * @brief Time unit for report summaries. If `0`, the fallback `time_unit` will be used. 56 | */ 57 | char time_unit_summary; 58 | }; 59 | 60 | /** 61 | * @brief Runs the compiler using the specified configuration. 62 | * @param[in] p_config Configuration for the compiler execution. 63 | * @param[out] s_error Pointer to buffer to receive any error message. If `null`, error messages 64 | * are ignored. 65 | * @param[in] s_error_size Size, in bytes, of the memory pointed to by \p s_error. If zero, error messages 66 | * are ignored. 67 | * @return true on success. 68 | * @details If an error occurs, the error message will be saved in the provided buffer \p s_error. 69 | * This function will attempt to copy as many bytes from the error message into the buffer. Otherwise, 70 | * the contents of the buffer will remain unchanged. The buffer can `null` or the number of bytes 71 | * pointed to can be zero to indicate that caller is ignoring any error messages. 72 | */ 73 | int32_t compile(const ReportCompilerConfigC *p_config, char *s_error, size_t s_error_size); 74 | 75 | } // namespace Compiler 76 | } // namespace ReportGen 77 | } // namespace hebench 78 | } 79 | 80 | #endif // defined _HEBench_ReportCompiler_H_0596d40a3cce4b108a81595c50eb286d 81 | -------------------------------------------------------------------------------- /docsrc/examples/generic_wl_example/generic_wl_tutorial/include/generic_wl_benchmark.h: -------------------------------------------------------------------------------- 1 | 2 | // Copyright (C) 2021 Intel Corporation 3 | // SPDX-License-Identifier: Apache-2.0 4 | 5 | #pragma once 6 | 7 | /// @cond 8 | 9 | #include 10 | 11 | #include "hebench/api_bridge/cpp/hebench.hpp" 12 | 13 | class ExampleEngine; 14 | 15 | class ExampleBenchmarkDescription : public hebench::cpp::BenchmarkDescription 16 | { 17 | public: 18 | HEBERROR_DECLARE_CLASS_NAME(ExampleBenchmarkDescription) 19 | 20 | public: 21 | // This workload requires 7 workload parameters 22 | static constexpr std::uint64_t NumWorkloadParams = 7; 23 | ExampleBenchmarkDescription(hebench::APIBridge::Category category); 24 | ~ExampleBenchmarkDescription() override; 25 | 26 | hebench::cpp::BaseBenchmark *createBenchmark(hebench::cpp::BaseEngine &engine, 27 | const hebench::APIBridge::WorkloadParams *p_params) override; 28 | void destroyBenchmark(hebench::cpp::BaseBenchmark *p_bench) override; 29 | }; 30 | 31 | class ExampleBenchmark : public hebench::cpp::BaseBenchmark 32 | { 33 | public: 34 | HEBERROR_DECLARE_CLASS_NAME(ExampleBenchmark) 35 | 36 | public: 37 | static constexpr std::int64_t tag = 0x1; 38 | 39 | ExampleBenchmark(ExampleEngine &engine, 40 | const hebench::APIBridge::BenchmarkDescriptor &bench_desc, 41 | const hebench::APIBridge::WorkloadParams &bench_params); 42 | ~ExampleBenchmark() override; 43 | 44 | hebench::APIBridge::Handle encode(const hebench::APIBridge::DataPackCollection *p_parameters) override; 45 | void decode(hebench::APIBridge::Handle encoded_data, hebench::APIBridge::DataPackCollection *p_native) override; 46 | hebench::APIBridge::Handle encrypt(hebench::APIBridge::Handle encoded_data) override; 47 | hebench::APIBridge::Handle decrypt(hebench::APIBridge::Handle encrypted_data) override; 48 | 49 | hebench::APIBridge::Handle load(const hebench::APIBridge::Handle *p_local_data, std::uint64_t count) override; 50 | void store(hebench::APIBridge::Handle remote_data, 51 | hebench::APIBridge::Handle *p_local_data, std::uint64_t count) override; 52 | 53 | hebench::APIBridge::Handle operate(hebench::APIBridge::Handle h_remote_packed, 54 | const hebench::APIBridge::ParameterIndexer *p_param_indexers) override; 55 | 56 | std::int64_t classTag() const override { return BaseBenchmark::classTag() | ExampleBenchmark::tag; } 57 | 58 | private: 59 | static constexpr std::uint64_t ParametersCount = 2; // number of parameters for this operation 60 | static constexpr std::uint64_t ResultComponentsCount = 3; // number of components of result for this operation 61 | 62 | // InternalInputData[input_param_index][input_sample][element] 63 | typedef std::array>, ParametersCount> InternalInputData; 64 | // InternalResultData[result_sample][result_component][element] 65 | typedef std::vector, ResultComponentsCount>> InternalResultData; 66 | }; 67 | 68 | /// @endcond 69 | -------------------------------------------------------------------------------- /docsrc/examples/backend_tutorials/seal/tutorial_backend/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Intel Corporation 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | cmake_minimum_required(VERSION 3.12) 5 | project(be_tutorial LANGUAGES C CXX) 6 | 7 | # C++ version (SEAL requires 17) 8 | set(CMAKE_CXX_STANDARD 17) # C++ standard C++17 9 | set(CMAKE_CXX_STANDARD_REQUIRED ON) 10 | set(CMAKE_CXX_EXTENSIONS OFF) 11 | 12 | set(${PROJECT_NAME}_SOURCES 13 | "${CMAKE_CURRENT_SOURCE_DIR}/src/tutorial_engine_seal.cpp" 14 | "${CMAKE_CURRENT_SOURCE_DIR}/src/tutorial_eltwiseadd_benchmark_seal.cpp" 15 | ) 16 | set(${PROJECT_NAME}_HEADERS 17 | "${CMAKE_CURRENT_SOURCE_DIR}/include/tutorial_error_seal.h" 18 | "${CMAKE_CURRENT_SOURCE_DIR}/include/tutorial_engine_seal.h" 19 | "${CMAKE_CURRENT_SOURCE_DIR}/include/tutorial_eltwiseadd_benchmark_seal.h" 20 | ) 21 | 22 | add_library(${PROJECT_NAME} SHARED ${${PROJECT_NAME}_SOURCES} ${${PROJECT_NAME}_HEADERS}) 23 | target_include_directories(${PROJECT_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include) 24 | 25 | # All external libraries are assumed to be pre-compiled for tutorial simplicity. 26 | 27 | # find the hebench_cpp archive 28 | set(HEBENCH_API_INCLUDE_DIR "/include/directory/for/api_bridge") 29 | target_include_directories(${PROJECT_NAME} PRIVATE ${HEBENCH_API_INCLUDE_DIR}) # point to include for api_bridge 30 | find_library(hebench_cpp_FOUND NAMES libhebench_cpp.a HINTS "/directory/containing/libhebench_cpp.a") 31 | if(hebench_cpp_FOUND) 32 | add_library(hebench_cpp UNKNOWN IMPORTED) 33 | # populate the found library with its properties 34 | set_property(TARGET hebench_cpp PROPERTY IMPORTED_LOCATION ${hebench_cpp_FOUND}) 35 | set_property(TARGET hebench_cpp APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${HEBENCH_API_INCLUDE_DIR}) 36 | message(STATUS "libhebench_cpp.a found") 37 | else() 38 | message(FATAL_ERROR "libhebench_cpp.a not found.") 39 | endif() 40 | 41 | # link found library 42 | target_link_libraries(${PROJECT_NAME} PUBLIC "-Wl,--whole-archive" hebench_cpp "-Wl,--no-whole-archive") 43 | 44 | # find other third party dependencies 45 | # pre-built Microsoft SEAL 3.6 46 | set(SEAL_INCLUDE_DIR "/usr/local/include/SEAL-3.6") 47 | target_include_directories(${PROJECT_NAME} PRIVATE ${SEAL_INCLUDE_DIR}) # point to include for SEAL 48 | find_library(SEAL_FOUND NAMES libseal-3.6.a HINTS "/usr/local/lib") 49 | if(SEAL_FOUND) 50 | add_library(seal-3.6 UNKNOWN IMPORTED) 51 | # populate the found library with its properties 52 | set_property(TARGET seal-3.6 PROPERTY IMPORTED_LOCATION ${SEAL_FOUND}) 53 | set_property(TARGET seal-3.6 APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${SEAL_INCLUDE_DIR}) 54 | else() 55 | message(FATAL_ERROR "libseal-3.6.a not found.") 56 | endif() 57 | 58 | # link found library 59 | target_link_libraries(${PROJECT_NAME} PUBLIC "-Wl,--whole-archive" seal-3.6 "-Wl,--no-whole-archive") 60 | 61 | # extra compile options 62 | target_compile_options(${PROJECT_NAME} PRIVATE -Wall -Wextra) # show warnings 63 | 64 | # install options 65 | include(GNUInstallDirs) 66 | set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib") 67 | install(TARGETS ${PROJECT_NAME} DESTINATION lib) 68 | -------------------------------------------------------------------------------- /docsrc/examples/backend_tutorials/palisade/tutorial_backend/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Intel Corporation 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | cmake_minimum_required(VERSION 3.12) 5 | project(be_tutorial LANGUAGES C CXX) 6 | 7 | # C++ version 8 | set(CMAKE_CXX_STANDARD 17) # C++ standard C++17 9 | set(CMAKE_CXX_STANDARD_REQUIRED ON) 10 | set(CMAKE_CXX_EXTENSIONS OFF) 11 | 12 | set(${PROJECT_NAME}_SOURCES 13 | "${CMAKE_CURRENT_SOURCE_DIR}/src/tutorial_engine_palisade.cpp" 14 | "${CMAKE_CURRENT_SOURCE_DIR}/src/tutorial_eltwiseadd_benchmark_palisade.cpp" 15 | ) 16 | set(${PROJECT_NAME}_HEADERS 17 | "${CMAKE_CURRENT_SOURCE_DIR}/include/tutorial_error_palisade.h" 18 | "${CMAKE_CURRENT_SOURCE_DIR}/include/tutorial_engine_palisade.h" 19 | "${CMAKE_CURRENT_SOURCE_DIR}/include/tutorial_eltwiseadd_benchmark_palisade.h" 20 | ) 21 | 22 | add_library(${PROJECT_NAME} SHARED ${${PROJECT_NAME}_SOURCES} ${${PROJECT_NAME}_HEADERS}) 23 | target_include_directories(${PROJECT_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include) 24 | 25 | # All external libraries are assumed to be pre-compiled for tutorial simplicity. 26 | 27 | # find the hebench_cpp archive 28 | set(HEBENCH_API_INCLUDE_DIR "/include/directory/for/api_bridge") 29 | target_include_directories(${PROJECT_NAME} PRIVATE ${HEBENCH_API_INCLUDE_DIR}) # point to include for api_bridge 30 | find_library(hebench_cpp_FOUND NAMES libhebench_cpp.a HINTS "/directory/containing/libhebench_cpp.a") 31 | if(hebench_cpp_FOUND) 32 | add_library(hebench_cpp UNKNOWN IMPORTED) 33 | # populate the found library with its properties 34 | set_property(TARGET hebench_cpp PROPERTY IMPORTED_LOCATION ${hebench_cpp_FOUND}) 35 | set_property(TARGET hebench_cpp APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${HEBENCH_API_INCLUDE_DIR}) 36 | else() 37 | message(FATAL_ERROR "libhebench_cpp.a not found.") 38 | endif() 39 | 40 | # link found library 41 | target_link_libraries(${PROJECT_NAME} PUBLIC "-Wl,--whole-archive" hebench_cpp "-Wl,--no-whole-archive") 42 | 43 | # find other third party dependencies 44 | # pre-built PALISADE 45 | set(PALISADE_HINT_DIR "/usr/local/") 46 | 47 | find_package(Palisade HINTS ${PALISADE_HINT_DIR} REQUIRED) 48 | add_library(libpalisade INTERFACE) 49 | target_include_directories(libpalisade INTERFACE 50 | ${OPENMP_INCLUDES} # Will be empty if PALISADE is built single-threaded 51 | ${PALISADE_INCLUDE} 52 | ${PALISADE_INCLUDE}/third-party/include 53 | ${PALISADE_INCLUDE}/core 54 | ${PALISADE_INCLUDE}/pke 55 | ${PALISADE_INCLUDE}/binfhe) 56 | target_link_directories(libpalisade INTERFACE ${PALISADE_LIBDIR}) 57 | target_link_libraries(libpalisade INTERFACE ${PALISADE_LIBRARIES} ${OPENMP_LIBRARIES}) 58 | 59 | # Ignore errors from PALISADE 60 | add_compile_options(-Wno-error=unused-parameter -Wno-error=ignored-qualifiers -Wno-error=deprecated-copy) 61 | 62 | # link found library 63 | target_link_libraries(${PROJECT_NAME} PUBLIC libpalisade) 64 | 65 | # extra compile options 66 | target_compile_options(${PROJECT_NAME} PRIVATE -Wall -Wextra) # show warnings 67 | 68 | # install options 69 | include(GNUInstallDirs) 70 | set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib") 71 | install(TARGETS ${PROJECT_NAME} DESTINATION lib) 72 | -------------------------------------------------------------------------------- /report_gen/include/hebench_report_types.h: -------------------------------------------------------------------------------- 1 | 2 | // Copyright (C) 2021 Intel Corporation 3 | // SPDX-License-Identifier: Apache-2.0 4 | 5 | #ifndef _HEBench_Harness_Report_Types_H_0596d40a3cce4b108a81595c50eb286d 6 | #define _HEBench_Harness_Report_Types_H_0596d40a3cce4b108a81595c50eb286d 7 | 8 | #include 9 | 10 | namespace hebench { 11 | namespace ReportGen { 12 | 13 | extern "C" 14 | { 15 | 16 | #define MAX_DESCRIPTION_BUFFER_SIZE 256 17 | #define MAX_SYMBOL_BUFFER_SIZE 4 18 | #define MAX_TIME_REPORT_EVENT_DESCRIPTION_SIZE 256 19 | 20 | struct _TimingReportEventC 21 | { 22 | /** 23 | * @brief ID specifying the event type. 24 | * @details During summary report, events with the same ID will be 25 | * grouped together for statistical computations. Users must make sure 26 | * that related events are tagged with the correct ID. 27 | */ 28 | uint32_t event_type_id; 29 | double cpu_time_start; 30 | double cpu_time_end; 31 | double wall_time_start; 32 | double wall_time_end; 33 | /** 34 | * @brief Scale of time interval used for this event. 35 | * @details Timing values in this instance are given in this scale. 36 | * 37 | * This represents the ratio of this scale with respect to `1` second. 38 | * For example, `1` millisecond is `0.001` seconds, and thus, its 39 | * scale ratio is `1:1000`. So, the time, in seconds for wall time is 40 | * @code 41 | * (wall_time_end - wall_time_start) * time_interval_ratio_num / time_interval_ratio_den 42 | * @endcode 43 | * Similarly for CPU time. 44 | */ 45 | int64_t time_interval_ratio_num; 46 | /** 47 | * @brief Scale of time interval used for this event. 48 | * @sa time_interval_ratio_num 49 | */ 50 | int64_t time_interval_ratio_den; 51 | uint64_t input_sample_count; 52 | /** 53 | * @brief Description attached to this event. 54 | * @details Set to empty string if no description. 55 | */ 56 | char description[MAX_TIME_REPORT_EVENT_DESCRIPTION_SIZE]; 57 | }; 58 | typedef struct _TimingReportEventC TimingReportEventC; 59 | 60 | struct _UnitPrefix 61 | { 62 | /** 63 | * @brief Value in the specified unit. 64 | */ 65 | double value; 66 | /** 67 | * @brief Denominator of timing scale ratio with respect to a unit. 68 | * @details Numerator is always 1. 69 | */ 70 | int64_t time_interval_ratio_den; 71 | /** 72 | * @brief Symbol for the prefix. 73 | * @details m for milli, u micro, n for nano 74 | */ 75 | char symbol[MAX_SYMBOL_BUFFER_SIZE]; 76 | /** 77 | * @brief Full name for the prefix (or empty string if no prefix). 78 | * @details milli, micro, nano 79 | */ 80 | char prefix[MAX_DESCRIPTION_BUFFER_SIZE]; 81 | }; 82 | typedef struct _UnitPrefix TimingPrefixedSeconds; 83 | } 84 | 85 | } // namespace ReportGen 86 | } // namespace hebench 87 | 88 | #endif // defined _HEBench_Harness_Report_Types_H_0596d40a3cce4b108a81595c50eb286d 89 | -------------------------------------------------------------------------------- /test_harness/src/hebench_utilities_harness.cpp: -------------------------------------------------------------------------------- 1 | 2 | // Copyright (C) 2021 Intel Corporation 3 | // SPDX-License-Identifier: Apache-2.0 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include "include/hebench_utilities_harness.h" 12 | 13 | namespace hebench { 14 | namespace Utilities { 15 | 16 | std::mt19937 RandomGenerator::m_rand; 17 | 18 | std::string convertToDirectoryName(const std::string &s, bool to_lowercase) 19 | { 20 | std::stringstream ss_retval; 21 | std::string s_tmp = s; 22 | 23 | // convert all non-alnum to underscores and, if requested, alnum to lowercase 24 | std::transform(s_tmp.begin(), s_tmp.end(), s_tmp.begin(), 25 | [to_lowercase](unsigned char c) { return (std::isalnum(c) || c == '.' ? 26 | (to_lowercase ? std::tolower(c) : c) : 27 | '_'); }); 28 | // remove leading and trailing underscores 29 | s_tmp.erase(0, s_tmp.find_first_not_of('_')); 30 | s_tmp.erase(s_tmp.find_last_not_of('_') + 1); 31 | 32 | // remove contiguous underscore repetitions 33 | std::size_t pos = 0; 34 | while (pos < s_tmp.size()) 35 | { 36 | std::size_t i = s_tmp.find_first_of('_', pos); 37 | if (i == std::string::npos) 38 | ss_retval << s_tmp.substr(pos); 39 | else 40 | { 41 | if (i < s_tmp.size()) 42 | ++i; 43 | ss_retval << s_tmp.substr(pos, i - pos); 44 | i = s_tmp.find_first_not_of('_', i); 45 | } // end if 46 | pos = i; 47 | } // end while 48 | 49 | return ss_retval.str(); 50 | } 51 | 52 | void printArraysAsColumns(std::ostream &os, 53 | const hebench::APIBridge::NativeDataBuffer **p_buffers, std::size_t count, 54 | hebench::APIBridge::DataType data_type, 55 | bool output_row_index, 56 | const char *separator) 57 | { 58 | switch (data_type) 59 | { 60 | case hebench::APIBridge::DataType::Int32: 61 | printArraysAsColumns(os, p_buffers, count, output_row_index, separator); 62 | break; 63 | 64 | case hebench::APIBridge::DataType::Int64: 65 | printArraysAsColumns(os, p_buffers, count, output_row_index, separator); 66 | break; 67 | 68 | case hebench::APIBridge::DataType::Float32: 69 | printArraysAsColumns(os, p_buffers, count, output_row_index, separator); 70 | break; 71 | 72 | case hebench::APIBridge::DataType::Float64: 73 | printArraysAsColumns(os, p_buffers, count, output_row_index, separator); 74 | break; 75 | 76 | default: 77 | printArraysAsColumns(os, p_buffers, count, output_row_index, separator); 78 | break; 79 | } // end switch 80 | } 81 | 82 | //----------------------- 83 | // class RandomGenerator 84 | //----------------------- 85 | 86 | void RandomGenerator::setRandomSeed(std::uint64_t seed) 87 | { 88 | m_rand.seed(seed); 89 | } 90 | 91 | void RandomGenerator::setRandomSeed() 92 | { 93 | setRandomSeed(std::chrono::system_clock::now().time_since_epoch().count()); 94 | } 95 | 96 | } // namespace Utilities 97 | } // namespace hebench 98 | -------------------------------------------------------------------------------- /test_harness/benchmarks/DotProduct/latency/include/hebench_dotproduct_l.h: -------------------------------------------------------------------------------- 1 | 2 | // Copyright (C) 2021 Intel Corporation 3 | // SPDX-License-Identifier: Apache-2.0 4 | 5 | #ifndef _HEBench_Harness_DotProduct_L_H_0596d40a3cce4b108a81595c50eb286d 6 | #define _HEBench_Harness_DotProduct_L_H_0596d40a3cce4b108a81595c50eb286d 7 | 8 | #include 9 | 10 | #include "hebench/modules/general/include/nocopy.h" 11 | #include "hebench/modules/logging/include/logging.h" 12 | 13 | #include "benchmarks/DotProduct/include/hebench_dotproduct.h" 14 | #include "benchmarks/categories/include/hebench_benchmark_latency.h" 15 | #include "hebench/api_bridge/types.h" 16 | 17 | namespace hebench { 18 | namespace TestHarness { 19 | namespace DotProduct { 20 | namespace Latency { 21 | 22 | class BenchmarkDescriptor final : public hebench::TestHarness::DotProduct::BenchmarkDescriptorCategory 23 | { 24 | public: 25 | DISABLE_COPY(BenchmarkDescriptor) 26 | DISABLE_MOVE(BenchmarkDescriptor) 27 | private: 28 | IL_DECLARE_CLASS_NAME(DotProduct::Latency::BenchmarkDescriptor) 29 | public: 30 | static constexpr std::uint32_t BenchmarkID = 601; 31 | static constexpr std::uint64_t DefaultBatchSize = 1; // batch size for a parameter (always 1 for latency) 32 | 33 | public: 34 | BenchmarkDescriptor() = default; 35 | ~BenchmarkDescriptor() override = default; 36 | 37 | hebench::TestHarness::PartialBenchmark *createBenchmark(std::shared_ptr p_engine, 38 | const DescriptionToken &description_token) override; 39 | void destroyBenchmark(hebench::TestHarness::PartialBenchmark *p_bench) override; 40 | 41 | protected: 42 | bool matchBenchmarkDescriptor(const hebench::APIBridge::BenchmarkDescriptor &bench_desc, 43 | const std::vector &w_params) const override; 44 | void completeWorkloadDescription(WorkloadDescriptionOutput &output, 45 | const Engine &engine, 46 | const BenchmarkDescription::Backend &backend_desc, 47 | const BenchmarkDescription::Configuration &config) const override; 48 | 49 | private: 50 | static bool m_b_registered; 51 | }; 52 | 53 | class Benchmark final : public hebench::TestHarness::BenchmarkLatency 54 | { 55 | public: 56 | DISABLE_COPY(Benchmark) 57 | DISABLE_MOVE(Benchmark) 58 | private: 59 | IL_DECLARE_CLASS_NAME(DotProduct::Latency::Benchmark) 60 | 61 | public: 62 | friend class hebench::TestHarness::DotProduct::Latency::BenchmarkDescriptor; 63 | 64 | ~Benchmark() override = default; 65 | 66 | protected: 67 | void init() override; 68 | IDataLoader::Ptr getDataset() const override { return m_data; } 69 | std::uint32_t getEventIDStart() const override { return BenchmarkDescriptor::BenchmarkID; } 70 | bool validateResult(IDataLoader::Ptr dataset, 71 | const std::uint64_t *param_data_pack_indices, 72 | const std::vector &p_outputs, 73 | hebench::APIBridge::DataType data_type) const override; 74 | 75 | private: 76 | DataLoader::Ptr m_data; 77 | 78 | Benchmark(std::shared_ptr p_engine, 79 | const IBenchmarkDescriptor::DescriptionToken &description_token); 80 | }; 81 | 82 | } // namespace Latency 83 | } // namespace DotProduct 84 | } // namespace TestHarness 85 | } // namespace hebench 86 | 87 | #endif // defined _HEBench_Harness_DotProduct_L_H_0596d40a3cce4b108a81595c50eb286d 88 | -------------------------------------------------------------------------------- /test_harness/benchmarks/EltwiseAdd/latency/include/hebench_eltwiseadd_l.h: -------------------------------------------------------------------------------- 1 | 2 | // Copyright (C) 2021 Intel Corporation 3 | // SPDX-License-Identifier: Apache-2.0 4 | 5 | #ifndef _HEBench_Harness_EltwiseAdd_L_H_0596d40a3cce4b108a81595c50eb286d 6 | #define _HEBench_Harness_EltwiseAdd_L_H_0596d40a3cce4b108a81595c50eb286d 7 | 8 | #include 9 | 10 | #include "hebench/modules/general/include/nocopy.h" 11 | #include "hebench/modules/logging/include/logging.h" 12 | 13 | #include "benchmarks/EltwiseAdd/include/hebench_eltwiseadd.h" 14 | #include "benchmarks/categories/include/hebench_benchmark_latency.h" 15 | #include "hebench/api_bridge/types.h" 16 | 17 | namespace hebench { 18 | namespace TestHarness { 19 | namespace EltwiseAdd { 20 | namespace Latency { 21 | 22 | class BenchmarkDescriptor final : public hebench::TestHarness::EltwiseAdd::BenchmarkDescriptorCategory 23 | { 24 | public: 25 | DISABLE_COPY(BenchmarkDescriptor) 26 | DISABLE_MOVE(BenchmarkDescriptor) 27 | private: 28 | IL_DECLARE_CLASS_NAME(EltwiseAdd::Latency::BenchmarkDescriptor) 29 | public: 30 | static constexpr std::uint32_t BenchmarkID = 301; 31 | static constexpr std::uint64_t DefaultBatchSize = 1; // batch size for a parameter (always 1 for latency) 32 | 33 | public: 34 | BenchmarkDescriptor() = default; 35 | ~BenchmarkDescriptor() override = default; 36 | 37 | hebench::TestHarness::PartialBenchmark *createBenchmark(std::shared_ptr p_engine, 38 | const DescriptionToken &description_token) override; 39 | void destroyBenchmark(hebench::TestHarness::PartialBenchmark *p_bench) override; 40 | 41 | protected: 42 | bool matchBenchmarkDescriptor(const hebench::APIBridge::BenchmarkDescriptor &bench_desc, 43 | const std::vector &w_params) const override; 44 | void completeWorkloadDescription(WorkloadDescriptionOutput &output, 45 | const Engine &engine, 46 | const BenchmarkDescription::Backend &backend_desc, 47 | const BenchmarkDescription::Configuration &config) const override; 48 | 49 | private: 50 | static bool m_b_registered; 51 | }; 52 | 53 | class Benchmark final : public hebench::TestHarness::BenchmarkLatency 54 | { 55 | public: 56 | DISABLE_COPY(Benchmark) 57 | DISABLE_MOVE(Benchmark) 58 | private: 59 | IL_DECLARE_CLASS_NAME(EltwiseAdd::Latency::Benchmark) 60 | 61 | public: 62 | friend class hebench::TestHarness::EltwiseAdd::Latency::BenchmarkDescriptor; 63 | 64 | ~Benchmark() override = default; 65 | 66 | protected: 67 | void init() override; 68 | IDataLoader::Ptr getDataset() const override { return m_data; } 69 | std::uint32_t getEventIDStart() const override { return BenchmarkDescriptor::BenchmarkID; } 70 | bool validateResult(IDataLoader::Ptr dataset, 71 | const std::uint64_t *param_data_pack_indices, 72 | const std::vector &p_outputs, 73 | hebench::APIBridge::DataType data_type) const override; 74 | 75 | private: 76 | DataLoader::Ptr m_data; 77 | 78 | Benchmark(std::shared_ptr p_engine, 79 | const IBenchmarkDescriptor::DescriptionToken &description_token); 80 | }; 81 | 82 | } // namespace Latency 83 | } // namespace EltwiseAdd 84 | } // namespace TestHarness 85 | } // namespace hebench 86 | 87 | #endif // defined _HEBench_Harness_EltwiseAdd_L_H_0596d40a3cce4b108a81595c50eb286d 88 | -------------------------------------------------------------------------------- /test_harness/benchmarks/EltwiseMult/latency/include/hebench_eltwisemult_l.h: -------------------------------------------------------------------------------- 1 | 2 | // Copyright (C) 2021 Intel Corporation 3 | // SPDX-License-Identifier: Apache-2.0 4 | 5 | #ifndef _HEBench_Harness_EltwiseMult_L_H_0596d40a3cce4b108a81595c50eb286d 6 | #define _HEBench_Harness_EltwiseMult_L_H_0596d40a3cce4b108a81595c50eb286d 7 | 8 | #include 9 | 10 | #include "hebench/modules/general/include/nocopy.h" 11 | #include "hebench/modules/logging/include/logging.h" 12 | 13 | #include "benchmarks/EltwiseMult/include/hebench_eltwisemult.h" 14 | #include "benchmarks/categories/include/hebench_benchmark_latency.h" 15 | #include "hebench/api_bridge/types.h" 16 | 17 | namespace hebench { 18 | namespace TestHarness { 19 | namespace EltwiseMult { 20 | namespace Latency { 21 | 22 | class BenchmarkDescriptor final : public hebench::TestHarness::EltwiseMult::BenchmarkDescriptorCategory 23 | { 24 | public: 25 | DISABLE_COPY(BenchmarkDescriptor) 26 | DISABLE_MOVE(BenchmarkDescriptor) 27 | private: 28 | IL_DECLARE_CLASS_NAME(EltwiseMult::Latency::BenchmarkDescriptor) 29 | public: 30 | static constexpr std::uint32_t BenchmarkID = 501; 31 | static constexpr std::uint64_t DefaultBatchSize = 1; // batch size for a parameter (always 1 for latency) 32 | 33 | public: 34 | BenchmarkDescriptor() = default; 35 | ~BenchmarkDescriptor() override = default; 36 | 37 | hebench::TestHarness::PartialBenchmark *createBenchmark(std::shared_ptr p_engine, 38 | const DescriptionToken &description_token) override; 39 | void destroyBenchmark(hebench::TestHarness::PartialBenchmark *p_bench) override; 40 | 41 | protected: 42 | bool matchBenchmarkDescriptor(const hebench::APIBridge::BenchmarkDescriptor &bench_desc, 43 | const std::vector &w_params) const override; 44 | void completeWorkloadDescription(WorkloadDescriptionOutput &output, 45 | const Engine &engine, 46 | const BenchmarkDescription::Backend &backend_desc, 47 | const BenchmarkDescription::Configuration &config) const override; 48 | 49 | private: 50 | static bool m_b_registered; 51 | }; 52 | 53 | class Benchmark : public hebench::TestHarness::BenchmarkLatency 54 | { 55 | public: 56 | DISABLE_COPY(Benchmark) 57 | DISABLE_MOVE(Benchmark) 58 | private: 59 | IL_DECLARE_CLASS_NAME(EltwiseMult::Latency::Benchmark) 60 | 61 | public: 62 | friend class hebench::TestHarness::EltwiseMult::Latency::BenchmarkDescriptor; 63 | 64 | ~Benchmark() override = default; 65 | 66 | protected: 67 | void init() override; 68 | IDataLoader::Ptr getDataset() const override { return m_data; } 69 | std::uint32_t getEventIDStart() const override { return BenchmarkDescriptor::BenchmarkID; } 70 | bool validateResult(IDataLoader::Ptr dataset, 71 | const std::uint64_t *param_data_pack_indices, 72 | const std::vector &p_outputs, 73 | hebench::APIBridge::DataType data_type) const override; 74 | 75 | private: 76 | DataLoader::Ptr m_data; 77 | 78 | Benchmark(std::shared_ptr p_engine, 79 | const IBenchmarkDescriptor::DescriptionToken &description_token); 80 | }; 81 | 82 | } // namespace Latency 83 | } // namespace EltwiseMult 84 | } // namespace TestHarness 85 | } // namespace hebench 86 | 87 | #endif // defined _HEBench_Harness_EltwiseMult_L_H_0596d40a3cce4b108a81595c50eb286d 88 | -------------------------------------------------------------------------------- /test_harness/benchmarks/DotProduct/offline/include/hebench_dotproduct_o.h: -------------------------------------------------------------------------------- 1 | 2 | // Copyright (C) 2021 Intel Corporation 3 | // SPDX-License-Identifier: Apache-2.0 4 | 5 | #ifndef _HEBench_Harness_DotProduct_O_H_0596d40a3cce4b108a81595c50eb286d 6 | #define _HEBench_Harness_DotProduct_O_H_0596d40a3cce4b108a81595c50eb286d 7 | 8 | #include 9 | 10 | #include "hebench/modules/general/include/nocopy.h" 11 | #include "hebench/modules/logging/include/logging.h" 12 | 13 | #include "benchmarks/DotProduct/include/hebench_dotproduct.h" 14 | #include "benchmarks/categories/include/hebench_benchmark_offline.h" 15 | #include "hebench/api_bridge/types.h" 16 | 17 | namespace hebench { 18 | namespace TestHarness { 19 | namespace DotProduct { 20 | namespace Offline { 21 | 22 | class BenchmarkDescriptor final : public hebench::TestHarness::DotProduct::BenchmarkDescriptorCategory 23 | { 24 | public: 25 | DISABLE_COPY(BenchmarkDescriptor) 26 | DISABLE_MOVE(BenchmarkDescriptor) 27 | private: 28 | IL_DECLARE_CLASS_NAME(DotProduct::Offline::BenchmarkDescriptor) 29 | public: 30 | static constexpr std::uint32_t BenchmarkID = 701; 31 | // default number of samples for an operand 32 | // (based on docs definition of the workload operation) 33 | static constexpr std::uint64_t DefaultBatchSize = 5; 34 | 35 | public: 36 | BenchmarkDescriptor() = default; 37 | ~BenchmarkDescriptor() override = default; 38 | 39 | hebench::TestHarness::PartialBenchmark *createBenchmark(std::shared_ptr p_engine, 40 | const DescriptionToken &description_token) override; 41 | void destroyBenchmark(hebench::TestHarness::PartialBenchmark *p_bench) override; 42 | 43 | protected: 44 | bool matchBenchmarkDescriptor(const hebench::APIBridge::BenchmarkDescriptor &bench_desc, 45 | const std::vector &w_params) const override; 46 | void completeWorkloadDescription(WorkloadDescriptionOutput &output, 47 | const Engine &engine, 48 | const BenchmarkDescription::Backend &backend_desc, 49 | const BenchmarkDescription::Configuration &config) const override; 50 | 51 | private: 52 | static bool m_b_registered; 53 | }; 54 | 55 | class Benchmark : public hebench::TestHarness::BenchmarkOffline 56 | { 57 | public: 58 | DISABLE_COPY(Benchmark) 59 | DISABLE_MOVE(Benchmark) 60 | private: 61 | IL_DECLARE_CLASS_NAME(DotProduct::Offline::Benchmark) 62 | 63 | public: 64 | friend class hebench::TestHarness::DotProduct::Offline::BenchmarkDescriptor; 65 | 66 | ~Benchmark() override = default; 67 | 68 | protected: 69 | void init() override; 70 | IDataLoader::Ptr getDataset() const override { return m_data; } 71 | std::uint32_t getEventIDStart() const override { return BenchmarkDescriptor::BenchmarkID; } 72 | bool validateResult(IDataLoader::Ptr dataset, 73 | const std::uint64_t *param_data_pack_indices, 74 | const std::vector &p_outputs, 75 | hebench::APIBridge::DataType data_type) const override; 76 | 77 | private: 78 | DataLoader::Ptr m_data; 79 | 80 | Benchmark(std::shared_ptr p_engine, const IBenchmarkDescriptor::DescriptionToken &description_token); 81 | }; 82 | 83 | } // namespace Offline 84 | } // namespace DotProduct 85 | } // namespace TestHarness 86 | } // namespace hebench 87 | 88 | #endif // defined _HEBench_Harness_DotProduct_O_H_0596d40a3cce4b108a81595c50eb286d 89 | -------------------------------------------------------------------------------- /test_harness/benchmarks/MatrixMultiply/latency/include/hebench_matmult_l.h: -------------------------------------------------------------------------------- 1 | 2 | // Copyright (C) 2021 Intel Corporation 3 | // SPDX-License-Identifier: Apache-2.0 4 | 5 | #ifndef _HEBench_Harness_MatrixMultiply_L_H_0596d40a3cce4b108a81595c50eb286d 6 | #define _HEBench_Harness_MatrixMultiply_L_H_0596d40a3cce4b108a81595c50eb286d 7 | 8 | #include 9 | 10 | #include "hebench/modules/general/include/nocopy.h" 11 | #include "hebench/modules/logging/include/logging.h" 12 | 13 | #include "benchmarks/MatrixMultiply/include/hebench_matmult.h" 14 | #include "benchmarks/categories/include/hebench_benchmark_latency.h" 15 | #include "hebench/api_bridge/types.h" 16 | 17 | namespace hebench { 18 | namespace TestHarness { 19 | namespace MatrixMultiply { 20 | namespace Latency { 21 | 22 | class BenchmarkDescriptor final : public hebench::TestHarness::MatrixMultiply::BenchmarkDescriptorCategory 23 | { 24 | public: 25 | DISABLE_COPY(BenchmarkDescriptor) 26 | DISABLE_MOVE(BenchmarkDescriptor) 27 | private: 28 | IL_DECLARE_CLASS_NAME(MatrixMultiply::Latency::BenchmarkDescriptor) 29 | public: 30 | static constexpr std::uint32_t BenchmarkID = 101; 31 | static constexpr std::uint64_t DefaultBatchSize = 1; // batch size for a parameter (always 1 for latency) 32 | 33 | public: 34 | BenchmarkDescriptor() = default; 35 | ~BenchmarkDescriptor() override = default; 36 | 37 | hebench::TestHarness::PartialBenchmark *createBenchmark(std::shared_ptr p_engine, 38 | const DescriptionToken &description_token) override; 39 | void destroyBenchmark(hebench::TestHarness::PartialBenchmark *p_bench) override; 40 | 41 | protected: 42 | bool matchBenchmarkDescriptor(const hebench::APIBridge::BenchmarkDescriptor &bench_desc, 43 | const std::vector &w_params) const override; 44 | void completeWorkloadDescription(WorkloadDescriptionOutput &output, 45 | const Engine &engine, 46 | const BenchmarkDescription::Backend &backend_desc, 47 | const BenchmarkDescription::Configuration &config) const override; 48 | 49 | private: 50 | static bool m_b_registered; 51 | }; 52 | 53 | class Benchmark final : public hebench::TestHarness::BenchmarkLatency 54 | { 55 | public: 56 | DISABLE_COPY(Benchmark) 57 | DISABLE_MOVE(Benchmark) 58 | private: 59 | IL_DECLARE_CLASS_NAME(MatrixMultiply::Latency::Benchmark) 60 | 61 | public: 62 | friend class hebench::TestHarness::MatrixMultiply::Latency::BenchmarkDescriptor; 63 | 64 | ~Benchmark() override = default; 65 | 66 | protected: 67 | void init() override; 68 | IDataLoader::Ptr getDataset() const override { return m_data; } 69 | std::uint32_t getEventIDStart() const override { return BenchmarkDescriptor::BenchmarkID; } 70 | bool validateResult(IDataLoader::Ptr dataset, 71 | const std::uint64_t *param_data_pack_indices, 72 | const std::vector &p_outputs, 73 | hebench::APIBridge::DataType data_type) const override; 74 | 75 | private: 76 | DataLoader::Ptr m_data; 77 | 78 | Benchmark(std::shared_ptr p_engine, 79 | const IBenchmarkDescriptor::DescriptionToken &description_token); 80 | }; 81 | 82 | } // namespace Latency 83 | } // namespace MatrixMultiply 84 | } // namespace TestHarness 85 | } // namespace hebench 86 | 87 | #endif // defined _HEBench_Harness_MatrixMultiply_L_H_0596d40a3cce4b108a81595c50eb286d 88 | -------------------------------------------------------------------------------- /test_harness/benchmarks/EltwiseAdd/offline/include/hebench_eltwiseadd_o.h: -------------------------------------------------------------------------------- 1 | 2 | // Copyright (C) 2021 Intel Corporation 3 | // SPDX-License-Identifier: Apache-2.0 4 | 5 | #ifndef _HEBench_Harness_EltwiseAdd_O_H_0596d40a3cce4b108a81595c50eb286d 6 | #define _HEBench_Harness_EltwiseAdd_O_H_0596d40a3cce4b108a81595c50eb286d 7 | 8 | #include 9 | 10 | #include "hebench/modules/general/include/nocopy.h" 11 | #include "hebench/modules/logging/include/logging.h" 12 | 13 | #include "benchmarks/EltwiseAdd/include/hebench_eltwiseadd.h" 14 | #include "benchmarks/categories/include/hebench_benchmark_offline.h" 15 | #include "hebench/api_bridge/types.h" 16 | 17 | namespace hebench { 18 | namespace TestHarness { 19 | namespace EltwiseAdd { 20 | namespace Offline { 21 | 22 | class BenchmarkDescriptor final : public hebench::TestHarness::EltwiseAdd::BenchmarkDescriptorCategory 23 | { 24 | public: 25 | DISABLE_COPY(BenchmarkDescriptor) 26 | DISABLE_MOVE(BenchmarkDescriptor) 27 | private: 28 | IL_DECLARE_CLASS_NAME(EltwiseAdd::Offline::BenchmarkDescriptor) 29 | public: 30 | static constexpr std::uint32_t BenchmarkID = 401; 31 | // default number of samples for an operand 32 | // (based on docs definition of the workload operation) 33 | static constexpr std::uint64_t DefaultBatchSize = 5; 34 | 35 | public: 36 | BenchmarkDescriptor() = default; 37 | ~BenchmarkDescriptor() override = default; 38 | 39 | hebench::TestHarness::PartialBenchmark *createBenchmark(std::shared_ptr p_engine, 40 | const DescriptionToken &description_token) override; 41 | void destroyBenchmark(hebench::TestHarness::PartialBenchmark *p_bench) override; 42 | 43 | protected: 44 | bool matchBenchmarkDescriptor(const hebench::APIBridge::BenchmarkDescriptor &bench_desc, 45 | const std::vector &w_params) const override; 46 | void completeWorkloadDescription(WorkloadDescriptionOutput &output, 47 | const Engine &engine, 48 | const BenchmarkDescription::Backend &backend_desc, 49 | const BenchmarkDescription::Configuration &config) const override; 50 | 51 | private: 52 | static bool m_b_registered; 53 | }; 54 | 55 | class Benchmark : public hebench::TestHarness::BenchmarkOffline 56 | { 57 | public: 58 | DISABLE_COPY(Benchmark) 59 | DISABLE_MOVE(Benchmark) 60 | private: 61 | IL_DECLARE_CLASS_NAME(EltwiseAdd::Offline::Benchmark) 62 | 63 | public: 64 | friend class hebench::TestHarness::EltwiseAdd::Offline::BenchmarkDescriptor; 65 | 66 | ~Benchmark() override = default; 67 | 68 | protected: 69 | void init() override; 70 | IDataLoader::Ptr getDataset() const override { return m_data; } 71 | std::uint32_t getEventIDStart() const override { return BenchmarkDescriptor::BenchmarkID; } 72 | bool validateResult(IDataLoader::Ptr dataset, 73 | const std::uint64_t *param_data_pack_indices, 74 | const std::vector &p_outputs, 75 | hebench::APIBridge::DataType data_type) const override; 76 | 77 | private: 78 | DataLoader::Ptr m_data; 79 | 80 | Benchmark(std::shared_ptr p_engine, 81 | const IBenchmarkDescriptor::DescriptionToken &description_token); 82 | }; 83 | 84 | } // namespace Offline 85 | } // namespace EltwiseAdd 86 | } // namespace TestHarness 87 | } // namespace hebench 88 | 89 | #endif // defined _HEBench_Harness_EltwiseAdd_O_H_0596d40a3cce4b108a81595c50eb286d 90 | -------------------------------------------------------------------------------- /docsrc/categories/cat_latency.md: -------------------------------------------------------------------------------- 1 | Latency Category {#category_latency} 2 | ======================== 3 | 4 | [TOC] 5 | 6 | Latency is one of the HEBench supported benchmarking categories. 7 | 8 | See the complete [list of benchmarking categories](@ref category_overview) for other options. 9 | 10 | ## Category Flag 11 | 12 | ```cpp 13 | hebench:APIBridge::Category::Latency 14 | ``` 15 | 16 | ## Category Parameters 17 | 18 | - `std::uint64_t min_test_time_ms`: supported by all categories. 19 | Specifies the minimum time, in milliseconds, to run the test. 20 | 21 | - `std::uint64_t warmup_iterations_count`: 22 | Specifies the number of warmup iterations to perform before running the actual test. 23 | 24 | Note that a backend implementation that sets any field in `CategoryParams` to `0` or `null` values (as corresponding to the field data type) is indicating that it accepts any valid value in the supported range for said field (according to the valid ranges as specified in the documentation for each field). In this case, these values are user-configurable using Test Harness features such as [benchmark configuration files](@ref config_file_reference). 25 | 26 | ## Description 27 | 28 | The Latency benchmarking category measures the end-to-end time to complete a workload operation with a single input. 29 | 30 | Latency is a deterministic computation category: workloads that support this category must be able to produce the exact same results given the same input values. 31 | 32 | During latency test, Test Harness passes a single input sample to the backend. When testing, more than one call will be issued to `hebench::APIBridge::operate()`. Calls will occur until the test time specified during benchmark configuration has elapsed (at least two calls will always occur). The same input sample is used during every call; therefore, the backend should create a copy of the input before operating when the implementation modifies or destructs the values in the input. The objective is for Test Harness to measure how fast can the implementation perform the operation on a single input sample. The repetition is intended to collect statistically significant results. 33 | 34 | To correctly measure latency, the implementation of the backend `hebench::APIBridge::operate()` function should not cache results of previous calls, intermediate, final, or any other kind. This is, results must be computed every time, regardless of whether the input sample are the same as inputs from previous calls. The backend implementation must make sure that the operation actually starts running when the `operate()` function is called, not when the `hebench::APIBridge::load()` function completes. Function `hebench::APIBridge::store()` is provided to retrieve the results of the operation; however, the backend implementation must also ensure that `operate()` does not return until the operation completes and results are ready to be retrieved, not before. For example, if the operation will execute asynchronously, it should start only when `operate()` function is called, and the implementation of `operate()` should block until the asynchronous operation completes. 35 | 36 | If the implementation requires warm up iterations, it can request so via benchmark configuration file, or enforce it with the `warmup_iterations_count` field from `hebench::APIBridge::CategoryParams` when filling out the `hebench::APIBridge::BenchmarkDescriptor` structure in the backend. Warmup iterations are calls to `hebench::APIBridge::operate()` exactly as if they were being tested, but they do not affect the measured results. Note that Test Harness still measures the time for the warmup; it is just reported in its own entry in the benchmark report. 37 | -------------------------------------------------------------------------------- /test_harness/benchmarks/EltwiseMult/offline/include/hebench_eltwisemult_o.h: -------------------------------------------------------------------------------- 1 | 2 | // Copyright (C) 2021 Intel Corporation 3 | // SPDX-License-Identifier: Apache-2.0 4 | 5 | #ifndef _HEBench_Harness_EltwiseMult_O_H_0596d40a3cce4b108a81595c50eb286d 6 | #define _HEBench_Harness_EltwiseMult_O_H_0596d40a3cce4b108a81595c50eb286d 7 | 8 | #include 9 | 10 | #include "hebench/modules/general/include/nocopy.h" 11 | #include "hebench/modules/logging/include/logging.h" 12 | 13 | #include "benchmarks/EltwiseMult/include/hebench_eltwisemult.h" 14 | #include "benchmarks/categories/include/hebench_benchmark_offline.h" 15 | #include "hebench/api_bridge/types.h" 16 | 17 | namespace hebench { 18 | namespace TestHarness { 19 | namespace EltwiseMult { 20 | namespace Offline { 21 | 22 | class BenchmarkDescriptor final : public hebench::TestHarness::EltwiseMult::BenchmarkDescriptorCategory 23 | { 24 | public: 25 | DISABLE_COPY(BenchmarkDescriptor) 26 | DISABLE_MOVE(BenchmarkDescriptor) 27 | private: 28 | IL_DECLARE_CLASS_NAME(EltwiseMult::Offline::BenchmarkDescriptor) 29 | public: 30 | static constexpr std::uint32_t BenchmarkID = 601; 31 | // default number of samples for an operand 32 | // (based on docs definition of the workload operation) 33 | static constexpr std::uint64_t DefaultBatchSize = 5; 34 | 35 | public: 36 | BenchmarkDescriptor() = default; 37 | ~BenchmarkDescriptor() override = default; 38 | 39 | hebench::TestHarness::PartialBenchmark *createBenchmark(std::shared_ptr p_engine, 40 | const DescriptionToken &description_token) override; 41 | void destroyBenchmark(hebench::TestHarness::PartialBenchmark *p_bench) override; 42 | 43 | protected: 44 | bool matchBenchmarkDescriptor(const hebench::APIBridge::BenchmarkDescriptor &bench_desc, 45 | const std::vector &w_params) const override; 46 | void completeWorkloadDescription(WorkloadDescriptionOutput &output, 47 | const Engine &engine, 48 | const BenchmarkDescription::Backend &backend_desc, 49 | const BenchmarkDescription::Configuration &config) const override; 50 | 51 | private: 52 | static bool m_b_registered; 53 | }; 54 | 55 | class Benchmark : public hebench::TestHarness::BenchmarkOffline 56 | { 57 | public: 58 | DISABLE_COPY(Benchmark) 59 | DISABLE_MOVE(Benchmark) 60 | private: 61 | IL_DECLARE_CLASS_NAME(EltwiseMult::Offline::Benchmark) 62 | 63 | public: 64 | friend class hebench::TestHarness::EltwiseMult::Offline::BenchmarkDescriptor; 65 | 66 | ~Benchmark() override = default; 67 | 68 | protected: 69 | void init() override; 70 | IDataLoader::Ptr getDataset() const override { return m_data; } 71 | std::uint32_t getEventIDStart() const override { return BenchmarkDescriptor::BenchmarkID; } 72 | bool validateResult(IDataLoader::Ptr dataset, 73 | const std::uint64_t *param_data_pack_indices, 74 | const std::vector &p_outputs, 75 | hebench::APIBridge::DataType data_type) const override; 76 | 77 | private: 78 | DataLoader::Ptr m_data; 79 | 80 | Benchmark(std::shared_ptr p_engine, 81 | const IBenchmarkDescriptor::DescriptionToken &description_token); 82 | }; 83 | 84 | } // namespace Offline 85 | } // namespace EltwiseMult 86 | } // namespace TestHarness 87 | } // namespace hebench 88 | 89 | #endif // defined _HEBench_Harness_EltwiseMult_O_H_0596d40a3cce4b108a81595c50eb286d 90 | -------------------------------------------------------------------------------- /test_harness/benchmarks/Generic/latency/include/hebench_genericwl_l.h: -------------------------------------------------------------------------------- 1 | 2 | // Copyright (C) 2021 Intel Corporation 3 | // SPDX-License-Identifier: Apache-2.0 4 | 5 | #ifndef _HEBench_Harness_GenericWL_L_H_0596d40a3cce4b108a81595c50eb286d 6 | #define _HEBench_Harness_GenericWL_L_H_0596d40a3cce4b108a81595c50eb286d 7 | 8 | #include 9 | 10 | #include "hebench/modules/general/include/nocopy.h" 11 | #include "hebench/modules/logging/include/logging.h" 12 | 13 | #include "benchmarks/Generic/include/hebench_genericwl.h" 14 | #include "benchmarks/categories/include/hebench_benchmark_latency.h" 15 | #include "hebench/api_bridge/types.h" 16 | 17 | namespace hebench { 18 | namespace TestHarness { 19 | namespace GenericWL { 20 | namespace Latency { 21 | 22 | class BenchmarkDescriptor final : public hebench::TestHarness::GenericWL::BenchmarkDescriptorCategory 23 | { 24 | public: 25 | DISABLE_COPY(BenchmarkDescriptor) 26 | DISABLE_MOVE(BenchmarkDescriptor) 27 | private: 28 | IL_DECLARE_CLASS_NAME(GenericWL::Latency::BenchmarkDescriptor) 29 | public: 30 | static constexpr std::uint32_t BenchmarkID = 1001; 31 | static constexpr std::uint64_t DefaultBatchSize = 1; // batch size for a parameter (always 1 for latency) 32 | 33 | public: 34 | BenchmarkDescriptor() = default; 35 | ~BenchmarkDescriptor() override = default; 36 | 37 | hebench::TestHarness::PartialBenchmark *createBenchmark(std::shared_ptr p_engine, 38 | const DescriptionToken &description_token) override; 39 | void destroyBenchmark(hebench::TestHarness::PartialBenchmark *p_bench) override; 40 | 41 | protected: 42 | bool matchBenchmarkDescriptor(const hebench::APIBridge::BenchmarkDescriptor &bench_desc, 43 | const std::vector &w_params) const override; 44 | void completeWorkloadDescription(WorkloadDescriptionOutput &output, 45 | const Engine &engine, 46 | const BenchmarkDescription::Backend &backend_desc, 47 | const BenchmarkDescription::Configuration &config) const override; 48 | 49 | private: 50 | static bool m_b_registered; 51 | }; 52 | 53 | class Benchmark final : public hebench::TestHarness::BenchmarkLatency 54 | { 55 | public: 56 | DISABLE_COPY(Benchmark) 57 | DISABLE_MOVE(Benchmark) 58 | private: 59 | IL_DECLARE_CLASS_NAME(GenericWL::Latency::Benchmark) 60 | 61 | public: 62 | friend class hebench::TestHarness::GenericWL::Latency::BenchmarkDescriptor; 63 | 64 | ~Benchmark() override = default; 65 | 66 | protected: 67 | void init() override; 68 | IDataLoader::Ptr getDataset() const override { return m_data; } 69 | std::uint32_t getEventIDStart() const override { return BenchmarkDescriptor::BenchmarkID; } 70 | bool validateResult(IDataLoader::Ptr dataset, 71 | const std::uint64_t *param_data_pack_indices, 72 | const std::vector &p_outputs, 73 | hebench::APIBridge::DataType data_type) const override; 74 | 75 | private: 76 | std::uint64_t m_op_input_count; 77 | std::uint64_t m_op_output_count; 78 | DataLoader::Ptr m_data; 79 | 80 | Benchmark(std::shared_ptr p_engine, 81 | const IBenchmarkDescriptor::DescriptionToken &description_token); 82 | }; 83 | 84 | } // namespace Latency 85 | } // namespace GenericWL 86 | } // namespace TestHarness 87 | } // namespace hebench 88 | 89 | #endif // defined _HEBench_Harness_GenericWL_L_H_0596d40a3cce4b108a81595c50eb286d 90 | -------------------------------------------------------------------------------- /test_harness/benchmarks/LogisticRegression/latency/include/hebench_logreg_l.h: -------------------------------------------------------------------------------- 1 | 2 | // Copyright (C) 2021 Intel Corporation 3 | // SPDX-License-Identifier: Apache-2.0 4 | 5 | #ifndef _HEBench_Harness_LogisticRegression_L_H_0596d40a3cce4b108a81595c50eb286d 6 | #define _HEBench_Harness_LogisticRegression_L_H_0596d40a3cce4b108a81595c50eb286d 7 | 8 | #include 9 | 10 | #include "hebench/modules/general/include/nocopy.h" 11 | #include "hebench/modules/logging/include/logging.h" 12 | 13 | #include "benchmarks/LogisticRegression/include/hebench_logreg.h" 14 | #include "benchmarks/categories/include/hebench_benchmark_latency.h" 15 | #include "hebench/api_bridge/types.h" 16 | 17 | namespace hebench { 18 | namespace TestHarness { 19 | namespace LogisticRegression { 20 | namespace Latency { 21 | 22 | class BenchmarkDescriptor final : public hebench::TestHarness::LogisticRegression::BenchmarkDescriptorCategory 23 | { 24 | public: 25 | DISABLE_COPY(BenchmarkDescriptor) 26 | DISABLE_MOVE(BenchmarkDescriptor) 27 | private: 28 | IL_DECLARE_CLASS_NAME(LogisticRegression::Latency::BenchmarkDescriptor) 29 | public: 30 | static constexpr std::uint32_t BenchmarkID = 801; 31 | static constexpr std::uint64_t DefaultBatchSize = 1; // batch size for a parameter (always 1 for latency) 32 | 33 | public: 34 | BenchmarkDescriptor() = default; 35 | ~BenchmarkDescriptor() override = default; 36 | 37 | hebench::TestHarness::PartialBenchmark *createBenchmark(std::shared_ptr p_engine, 38 | const DescriptionToken &description_token) override; 39 | void destroyBenchmark(hebench::TestHarness::PartialBenchmark *p_bench) override; 40 | 41 | protected: 42 | bool matchBenchmarkDescriptor(const hebench::APIBridge::BenchmarkDescriptor &bench_desc, 43 | const std::vector &w_params) const override; 44 | void completeWorkloadDescription(WorkloadDescriptionOutput &output, 45 | const Engine &engine, 46 | const BenchmarkDescription::Backend &backend_desc, 47 | const BenchmarkDescription::Configuration &config) const override; 48 | 49 | private: 50 | static bool m_b_registered; 51 | }; 52 | 53 | class Benchmark final : public hebench::TestHarness::BenchmarkLatency 54 | { 55 | public: 56 | DISABLE_COPY(Benchmark) 57 | DISABLE_MOVE(Benchmark) 58 | private: 59 | IL_DECLARE_CLASS_NAME(LogisticRegression::Latency::Benchmark) 60 | 61 | public: 62 | friend class hebench::TestHarness::LogisticRegression::Latency::BenchmarkDescriptor; 63 | 64 | ~Benchmark() override = default; 65 | 66 | protected: 67 | void init() override; 68 | IDataLoader::Ptr getDataset() const override { return m_data; } 69 | std::uint32_t getEventIDStart() const override { return BenchmarkDescriptor::BenchmarkID; } 70 | bool validateResult(IDataLoader::Ptr dataset, 71 | const std::uint64_t *param_data_pack_indices, 72 | const std::vector &p_outputs, 73 | hebench::APIBridge::DataType data_type) const override; 74 | 75 | private: 76 | DataLoader::Ptr m_data; 77 | 78 | Benchmark(std::shared_ptr p_engine, 79 | const IBenchmarkDescriptor::DescriptionToken &description_token); 80 | }; 81 | 82 | } // namespace Latency 83 | } // namespace LogisticRegression 84 | } // namespace TestHarness 85 | } // namespace hebench 86 | 87 | #endif // defined _HEBench_Harness_LogisticRegression_L_H_0596d40a3cce4b108a81595c50eb286d 88 | -------------------------------------------------------------------------------- /test_harness/benchmarks/MatrixMultiply/offline/include/hebench_matmult_o.h: -------------------------------------------------------------------------------- 1 | 2 | // Copyright (C) 2021 Intel Corporation 3 | // SPDX-License-Identifier: Apache-2.0 4 | 5 | #ifndef _HEBench_Harness_MatrixMultiply_O_H_0596d40a3cce4b108a81595c50eb286d 6 | #define _HEBench_Harness_MatrixMultiply_O_H_0596d40a3cce4b108a81595c50eb286d 7 | 8 | #include 9 | 10 | #include "hebench/modules/general/include/nocopy.h" 11 | #include "hebench/modules/logging/include/logging.h" 12 | 13 | #include "benchmarks/MatrixMultiply/include/hebench_matmult.h" 14 | #include "benchmarks/categories/include/hebench_benchmark_offline.h" 15 | #include "hebench/api_bridge/types.h" 16 | 17 | namespace hebench { 18 | namespace TestHarness { 19 | namespace MatrixMultiply { 20 | namespace Offline { 21 | 22 | class BenchmarkDescriptor final : public hebench::TestHarness::MatrixMultiply::BenchmarkDescriptorCategory 23 | { 24 | public: 25 | DISABLE_COPY(BenchmarkDescriptor) 26 | DISABLE_MOVE(BenchmarkDescriptor) 27 | private: 28 | IL_DECLARE_CLASS_NAME(MatrixMultiply::Latency::BenchmarkDescriptor) 29 | public: 30 | static constexpr std::uint32_t BenchmarkID = 201; 31 | // default number of samples for an operand 32 | // (based on docs definition of the workload operation) 33 | static constexpr std::uint64_t DefaultBatchSize = 100; 34 | 35 | public: 36 | BenchmarkDescriptor() = default; 37 | ~BenchmarkDescriptor() override = default; 38 | 39 | hebench::TestHarness::PartialBenchmark *createBenchmark(std::shared_ptr p_engine, 40 | const DescriptionToken &description_token) override; 41 | void destroyBenchmark(hebench::TestHarness::PartialBenchmark *p_bench) override; 42 | 43 | protected: 44 | bool matchBenchmarkDescriptor(const hebench::APIBridge::BenchmarkDescriptor &bench_desc, 45 | const std::vector &w_params) const override; 46 | void completeWorkloadDescription(WorkloadDescriptionOutput &output, 47 | const Engine &engine, 48 | const BenchmarkDescription::Backend &backend_desc, 49 | const BenchmarkDescription::Configuration &config) const override; 50 | 51 | private: 52 | static bool m_b_registered; 53 | }; 54 | 55 | class Benchmark final : public hebench::TestHarness::BenchmarkOffline 56 | { 57 | public: 58 | DISABLE_COPY(Benchmark) 59 | DISABLE_MOVE(Benchmark) 60 | private: 61 | IL_DECLARE_CLASS_NAME(MatrixMultiply::Offline::Benchmark) 62 | 63 | public: 64 | friend class hebench::TestHarness::MatrixMultiply::Offline::BenchmarkDescriptor; 65 | 66 | ~Benchmark() override = default; 67 | 68 | protected: 69 | void init() override; 70 | IDataLoader::Ptr getDataset() const override { return m_data; } 71 | std::uint32_t getEventIDStart() const override { return BenchmarkDescriptor::BenchmarkID; } 72 | bool validateResult(IDataLoader::Ptr dataset, 73 | const std::uint64_t *param_data_pack_indices, 74 | const std::vector &p_outputs, 75 | hebench::APIBridge::DataType data_type) const override; 76 | 77 | private: 78 | DataLoader::Ptr m_data; 79 | 80 | Benchmark(std::shared_ptr p_engine, 81 | const IBenchmarkDescriptor::DescriptionToken &description_token); 82 | }; 83 | 84 | } // namespace Offline 85 | } // namespace MatrixMultiply 86 | } // namespace TestHarness 87 | } // namespace hebench 88 | 89 | #endif // defined _HEBench_Harness_MatrixMultiply_O_H_0596d40a3cce4b108a81595c50eb286d 90 | -------------------------------------------------------------------------------- /docsrc/overview_file_format.md: -------------------------------------------------------------------------------- 1 | 2 | Run Overview Output Format {#overview_file_format} 3 | ======================== 4 | 5 | [TOC] 6 | 7 | The following details the format and description of the run overview file produced by [Report Compiler](@ref report_compiler_overview). 8 | 9 | The overview file is a highlight of the main event of each report processed during the report compiler run. Its main purpose is to summarize and offer an easy way to see the big picture of the run at a glance. It is also intended to simplify parsing of the reports where statistical information for each main event is aggregated and conveniently available for further use. If more details are required from a report, the overview file points to each file from which the data was extracted. On a successful run, statistics and summary files are located in the same location as the report files. 10 | 11 | ## Overview example output 12 | 13 | This example shows an example of the contents of an overview file. 14 | 15 | ```csv 16 | ,,,,,,,,,,,,,,Wall Time,,,,,,,,,,,,,CPU Time 17 | Workload,End State,Filename,Category,Data type,Cipher text,Scheme,Security,Extra,ID,Event,Total Wall Time,Samples per sec,Samples per sec trimmed,Average,Standard Deviation,Time Unit,Time Factor,Min,Max,Median,Trimmed Average,Trimmed Standard Deviation,1-th percentile,10-th percentile,90-th percentile,99-th percentile,Average,Standard Deviation,Time Unit,Time Factor,Min,Max,Median,Trimmed Average,Trimmed Standard Deviation,1-th percentile,10-th percentile,90-th percentile,99-th percentile,Input Samples,wp0,wp1,wp2 18 | Matrix Multiplication (100x100) x (100x100),OK,"/linux-storage/storage/documents/repos/hebench/frontend/build/report/matrix_multiplication_0-wp_100_100_100-latency-float64-20001-all_plain-plain-none-0-report.csv",Latency,Float64,0,Plain,None,0,108,Operation,2019.196494,50.019896676782,50.98381115765,19.992044495049,2.661381013492,ms,0.001,19.537712,45.72633,19.609865,19.614069197531,0.034799752167,19.543096,19.563912,19.71045,20.344244,19.99102970297,2.661467916228,ms,0.001,19.538,45.726,19.609,19.613,0.034760250287,19.543,19.563,19.711,20.343,101,100,100,100 19 | ``` 20 | 21 | ## File Contents 22 | 23 | This file features a table where each row contains the statistical data compiled for the corresponding report's main event. The table shares many columns with a statistics file. 24 | 25 | Here are listed the columns that are unique to the overview file. For details on the other columns, see @ref stats_n_summary_format . 26 | 27 | | Column name | Description | 28 | |-|-| 29 | | Workload | Name of the workload as extracted from the header of the report file
or
"Failed" if the corresponding report belongs to a failed benchmark. | 30 | | End State | Shows the completion status per benchmark: whether it completed successfully (`OK`) or failed (`Failed`).
See benchmark report file for cause of failure. | 31 | | Filename | Full path to the report file used to generate these statistics. If the report is not a failed benchmark, the statistics and summary files should be located in the same location as the report after a successful compilation. | 32 | | Category | Testing category
or
Reason for the benchmark failure, if the corresponding report belongs to a failed benchmark. | 33 | | Data type | Data type used for input and output in the test. | 34 | | Cipher text | Binary representation of which operation parameters were encrypted. Each bit position corresponds to the operation parameter index. | 35 | | Scheme | Homomorphic Encryption scheme used. | 36 | | Security | Encryption security level. | 37 | | Extra | Extra identifier of the benchmark according to the report. | 38 | 39 | Most fields will be blank if the corresponding report belongs to a failed benchmark. 40 | -------------------------------------------------------------------------------- /test_harness/benchmarks/Generic/offline/include/hebench_genericwl_o.h: -------------------------------------------------------------------------------- 1 | 2 | // Copyright (C) 2021 Intel Corporation 3 | // SPDX-License-Identifier: Apache-2.0 4 | 5 | #ifndef _HEBench_Harness_GenericWL_O_H_0596d40a3cce4b108a81595c50eb286d 6 | #define _HEBench_Harness_GenericWL_O_H_0596d40a3cce4b108a81595c50eb286d 7 | 8 | #include 9 | 10 | #include "hebench/modules/general/include/nocopy.h" 11 | #include "hebench/modules/logging/include/logging.h" 12 | 13 | #include "benchmarks/Generic/include/hebench_genericwl.h" 14 | #include "benchmarks/categories/include/hebench_benchmark_offline.h" 15 | #include "hebench/api_bridge/types.h" 16 | 17 | namespace hebench { 18 | namespace TestHarness { 19 | namespace GenericWL { 20 | namespace Offline { 21 | 22 | class BenchmarkDescriptor final : public hebench::TestHarness::GenericWL::BenchmarkDescriptorCategory 23 | { 24 | public: 25 | DISABLE_COPY(BenchmarkDescriptor) 26 | DISABLE_MOVE(BenchmarkDescriptor) 27 | private: 28 | IL_DECLARE_CLASS_NAME(GenericWL::Offline::BenchmarkDescriptor) 29 | public: 30 | static constexpr std::uint32_t BenchmarkID = 1101; 31 | // default number of samples for an operand 32 | // (based on docs definition of the workload operation) 33 | static constexpr std::uint64_t DefaultBatchSize = 1; 34 | 35 | public: 36 | BenchmarkDescriptor() = default; 37 | ~BenchmarkDescriptor() override = default; 38 | 39 | hebench::TestHarness::PartialBenchmark *createBenchmark(std::shared_ptr p_engine, 40 | const DescriptionToken &description_token) override; 41 | void destroyBenchmark(hebench::TestHarness::PartialBenchmark *p_bench) override; 42 | 43 | protected: 44 | bool matchBenchmarkDescriptor(const hebench::APIBridge::BenchmarkDescriptor &bench_desc, 45 | const std::vector &w_params) const override; 46 | void completeWorkloadDescription(WorkloadDescriptionOutput &output, 47 | const Engine &engine, 48 | const BenchmarkDescription::Backend &backend_desc, 49 | const BenchmarkDescription::Configuration &config) const override; 50 | 51 | private: 52 | static bool m_b_registered; 53 | }; 54 | 55 | class Benchmark : public hebench::TestHarness::BenchmarkOffline 56 | { 57 | public: 58 | DISABLE_COPY(Benchmark) 59 | DISABLE_MOVE(Benchmark) 60 | private: 61 | IL_DECLARE_CLASS_NAME(GenericWL::Offline::Benchmark) 62 | 63 | public: 64 | friend class hebench::TestHarness::GenericWL::Offline::BenchmarkDescriptor; 65 | 66 | ~Benchmark() override = default; 67 | 68 | protected: 69 | void init() override; 70 | IDataLoader::Ptr getDataset() const override { return m_data; } 71 | std::uint32_t getEventIDStart() const override { return BenchmarkDescriptor::BenchmarkID; } 72 | bool validateResult(IDataLoader::Ptr dataset, 73 | const std::uint64_t *param_data_pack_indices, 74 | const std::vector &p_outputs, 75 | hebench::APIBridge::DataType data_type) const override; 76 | 77 | private: 78 | std::uint64_t m_op_input_count; 79 | std::uint64_t m_op_output_count; 80 | DataLoader::Ptr m_data; 81 | 82 | Benchmark(std::shared_ptr p_engine, 83 | const IBenchmarkDescriptor::DescriptionToken &description_token); 84 | }; 85 | 86 | } // namespace Offline 87 | } // namespace GenericWL 88 | } // namespace TestHarness 89 | } // namespace hebench 90 | 91 | #endif // defined _HEBench_Harness_GenericWL_O_H_0596d40a3cce4b108a81595c50eb286d 92 | -------------------------------------------------------------------------------- /dataset_loader/include/hebench_dataset_loader.h: -------------------------------------------------------------------------------- 1 | 2 | // Copyright (C) 2021 Intel Corporation 3 | // SPDX-License-Identifier: Apache-2.0 4 | 5 | #ifndef _HEBench_Harness_DatasetLoader_H_0596d40a3cce4b108a81595c50eb286d 6 | #define _HEBench_Harness_DatasetLoader_H_0596d40a3cce4b108a81595c50eb286d 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | namespace hebench { 14 | namespace DataLoader { 15 | 16 | template 17 | struct ExternalDataset 18 | { 19 | /** 20 | * @brief Contains the samples for each input parameter as loaded from external source. 21 | * @details `inputs[i]` contains all samples for input parameter `i`. 22 | * `inputs[i][j]` is sample `j` in input parameter `i`. The following must be true: 23 | * `inputs[i][j0].size() == inputs[i][j1].size()` for all `0 < j0, j1 < inputs[i].size()`. 24 | * Every sample is a vector of scalars of type `T`. 25 | */ 26 | std::vector>> inputs; 27 | /** 28 | * @brief Contains the samples for each result component as loaded from external source. 29 | * @details `outputs[i]` contains all samples for output component `i`. 30 | * `outputs[i][j]` is sample `j` in output component `i`. The following must be true: 31 | * `outputs[i][j0].size() == outputs[i][j1].size()` for all `0 < j0, j1 < outputs[i].size()`. 32 | * Every sample is a vector of scalars of type `T`. 33 | */ 34 | std::vector>> outputs; 35 | }; 36 | 37 | template || std::is_same_v || std::is_same_v || std::is_same_v>> 40 | class ExternalDatasetLoader 41 | { 42 | public: 43 | /** 44 | * @brief Loads a dataset from an external csv file that follows the defined structure. 45 | * @param[in] filename Name of the file to load. 46 | * @param[in] max_loaded_size Maximum size, in bytes, allowed for the data to occupy after 47 | * loading. If 0, there is no limit to how much memory the data can occupy. 48 | * @throws Standard C++ exception derived from std::exception on error describing the 49 | * failure. 50 | * @return An `ExternalDataset` structure containing the data loaded where the loaded scalars 51 | * are of type `T`. 52 | * @details Contents of the file loaded will be casted to the type indicated for the template 53 | * parameter specification during the method call. Samples containing strings will be converted 54 | * into equivalent ASCII characters and cast into template type `T`. 55 | * 56 | * If \p max_loaded_size is not zero, then, the total number of bytes required to hold all 57 | * the values loaded in the returned `ExternalDataset` must be less than or equal to value of 58 | * \p max_loaded_size . Otherwise, an exception is thrown because the size of the dataset in 59 | * the file specified after loading and converting to `ExternalDataset` is larger than 60 | * non-zero \p max_loaded_size . 61 | */ 62 | static ExternalDataset loadFromCSV(const std::string &filename, 63 | std::uint64_t max_loaded_size = 0); 64 | }; 65 | 66 | // template implementations 67 | 68 | template class ExternalDatasetLoader; 69 | template class ExternalDatasetLoader; 70 | template class ExternalDatasetLoader; 71 | template class ExternalDatasetLoader; 72 | 73 | } // namespace DataLoader 74 | } // namespace hebench 75 | 76 | #endif // defined _HEBench_Harness_DatasetLoader_H_0596d40a3cce4b108a81595c50eb286d 77 | -------------------------------------------------------------------------------- /test_harness/benchmarks/LogisticRegression/offline/include/hebench_logreg_o.h: -------------------------------------------------------------------------------- 1 | 2 | // Copyright (C) 2021 Intel Corporation 3 | // SPDX-License-Identifier: Apache-2.0 4 | 5 | #ifndef _HEBench_Harness_LogisticRegression_O_H_0596d40a3cce4b108a81595c50eb286d 6 | #define _HEBench_Harness_LogisticRegression_O_H_0596d40a3cce4b108a81595c50eb286d 7 | 8 | #include 9 | 10 | #include "hebench/modules/general/include/nocopy.h" 11 | #include "hebench/modules/logging/include/logging.h" 12 | 13 | #include "benchmarks/LogisticRegression/include/hebench_logreg.h" 14 | #include "benchmarks/categories/include/hebench_benchmark_offline.h" 15 | #include "hebench/api_bridge/types.h" 16 | 17 | namespace hebench { 18 | namespace TestHarness { 19 | namespace LogisticRegression { 20 | namespace Offline { 21 | 22 | class BenchmarkDescriptor final : public hebench::TestHarness::LogisticRegression::BenchmarkDescriptorCategory 23 | { 24 | public: 25 | DISABLE_COPY(BenchmarkDescriptor) 26 | DISABLE_MOVE(BenchmarkDescriptor) 27 | private: 28 | IL_DECLARE_CLASS_NAME(LogisticRegression::Offline::BenchmarkDescriptor) 29 | public: 30 | static constexpr std::uint32_t BenchmarkID = 901; 31 | // default number of samples for an operand 32 | // (based on docs definition of the workload operation) 33 | static constexpr std::uint64_t DefaultBatchSize = 100; 34 | 35 | public: 36 | BenchmarkDescriptor() = default; 37 | ~BenchmarkDescriptor() override = default; 38 | 39 | hebench::TestHarness::PartialBenchmark *createBenchmark(std::shared_ptr p_engine, 40 | const DescriptionToken &description_token) override; 41 | void destroyBenchmark(hebench::TestHarness::PartialBenchmark *p_bench) override; 42 | 43 | protected: 44 | bool matchBenchmarkDescriptor(const hebench::APIBridge::BenchmarkDescriptor &bench_desc, 45 | const std::vector &w_params) const override; 46 | void completeWorkloadDescription(WorkloadDescriptionOutput &output, 47 | const Engine &engine, 48 | const BenchmarkDescription::Backend &backend_desc, 49 | const BenchmarkDescription::Configuration &config) const override; 50 | 51 | private: 52 | static bool m_b_registered; 53 | }; 54 | 55 | class Benchmark : public hebench::TestHarness::BenchmarkOffline 56 | { 57 | public: 58 | DISABLE_COPY(Benchmark) 59 | DISABLE_MOVE(Benchmark) 60 | private: 61 | IL_DECLARE_CLASS_NAME(LogisticRegression::Offline::Benchmark) 62 | 63 | public: 64 | friend class hebench::TestHarness::LogisticRegression::Offline::BenchmarkDescriptor; 65 | 66 | ~Benchmark() override = default; 67 | 68 | protected: 69 | void init() override; 70 | IDataLoader::Ptr getDataset() const override { return m_data; } 71 | std::uint32_t getEventIDStart() const override { return BenchmarkDescriptor::BenchmarkID; } 72 | bool validateResult(IDataLoader::Ptr dataset, 73 | const std::uint64_t *param_data_pack_indices, 74 | const std::vector &p_outputs, 75 | hebench::APIBridge::DataType data_type) const override; 76 | 77 | private: 78 | DataLoader::Ptr m_data; 79 | 80 | Benchmark(std::shared_ptr p_engine, 81 | const IBenchmarkDescriptor::DescriptionToken &description_token); 82 | }; 83 | 84 | } // namespace Offline 85 | } // namespace LogisticRegression 86 | } // namespace TestHarness 87 | } // namespace hebench 88 | 89 | #endif // defined _HEBench_Harness_LogisticRegression_O_H_0596d40a3cce4b108a81595c50eb286d 90 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Intel Corporation 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | cmake_minimum_required(VERSION 3.13) 5 | project(frontend LANGUAGES C CXX) 6 | set(CMAKE_PROJECT_NAME ${PROJECT_NAME}) 7 | 8 | if(CMAKE_BUILD_TYPE) 9 | set(RELEASE_TYPES 10 | Debug 11 | Release 12 | RelWithDebInfo 13 | MinSizeRel) 14 | list(FIND RELEASE_TYPES ${CMAKE_BUILD_TYPE} INDEX_FOUND) 15 | if(${INDEX_FOUND} EQUAL -1) 16 | message( 17 | FATAL_ERROR 18 | "CMAKE_BUILD_TYPE must be one of Debug, Release, RelWithDebInfo, or MinSizeRel" 19 | ) 20 | endif() 21 | else() 22 | set(CMAKE_BUILD_TYPE "Debug" CACHE STRING "Choose the type of Build" FORCE) 23 | endif() 24 | 25 | message(STATUS "CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}") 26 | 27 | # versioning 28 | configure_file("VERSION" "VERSION") 29 | file(STRINGS "VERSION" ${CMAKE_PROJECT_NAME}_VERSION) 30 | list(LENGTH ${CMAKE_PROJECT_NAME}_VERSION ${CMAKE_PROJECT_NAME}_VERSION_LEN) 31 | if(${CMAKE_PROJECT_NAME}_VERSION_LEN LESS 4) 32 | message(FATAL_ERROR "Invalid project version file format.") 33 | endif() 34 | list(GET ${CMAKE_PROJECT_NAME}_VERSION 0 ${CMAKE_PROJECT_NAME}_VERSION_PREFIX) 35 | if(NOT ${CMAKE_PROJECT_NAME}_VERSION_PREFIX STREQUAL "v") 36 | message(FATAL_ERROR "Invalid project version prefix format read from file.") 37 | endif() 38 | list(GET ${CMAKE_PROJECT_NAME}_VERSION 1 ${CMAKE_PROJECT_NAME}_VERSION_MAJOR) 39 | list(GET ${CMAKE_PROJECT_NAME}_VERSION 2 ${CMAKE_PROJECT_NAME}_VERSION_MINOR) 40 | list(GET ${CMAKE_PROJECT_NAME}_VERSION 3 ${CMAKE_PROJECT_NAME}_VERSION_REVISION) 41 | set(VERSION_TXT "v${${CMAKE_PROJECT_NAME}_VERSION_MAJOR}.${${CMAKE_PROJECT_NAME}_VERSION_MINOR}.${${CMAKE_PROJECT_NAME}_VERSION_REVISION}") 42 | if(${CMAKE_PROJECT_NAME}_VERSION_LEN GREATER 4) 43 | list(GET ${CMAKE_PROJECT_NAME}_VERSION 4 ${CMAKE_PROJECT_NAME}_VERSION_BUILD) 44 | set(VERSION_TXT "${VERSION_TXT}-${${CMAKE_PROJECT_NAME}_VERSION_BUILD}") 45 | else() 46 | set(${CMAKE_PROJECT_NAME}_VERSION_BUILD "") 47 | endif() 48 | message(STATUS "Project Version: ${VERSION_TXT}") 49 | 50 | set(CMAKE_CXX_STANDARD 17) 51 | set(CMAKE_CXX_STANDARD_REQUIRED ON) 52 | set(CMAKE_CXX_EXTENSIONS OFF) 53 | 54 | cmake_policy(SET CMP0079 NEW) 55 | 56 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") 57 | 58 | set(THREADS_PREFER_PTHREAD_FLAG ON) 59 | find_package(Threads REQUIRED) 60 | 61 | option(HIDE_EXT_WARNINGS "Attempts to hide all warnings encountered by third-party projects" OFF) 62 | message(STATUS "HIDE_EXT_WARNINGS: ${HIDE_EXT_WARNINGS}") 63 | 64 | include(GNUInstallDirs) 65 | set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib") 66 | 67 | message(STATUS "CMAKE_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX}") 68 | 69 | # third party 70 | include(FetchContent) 71 | set(FETCHCONTENT_BASE_DIR ${CMAKE_BINARY_DIR}/third-party) 72 | 73 | # common-lib 74 | set(_COMPONENT_NAME "COMMON_LIB") 75 | set(_COMPONENT_HEADER "hebench/modules") 76 | set(_COMPONENT_LIB_NAME "hebench_common-lib") 77 | set(_HEADER_ONLY FALSE) 78 | include(cmake/utils/import-library.cmake) 79 | 80 | ## APIBridge 81 | set(_COMPONENT_NAME "API_BRIDGE") 82 | set(_COMPONENT_HEADER "hebench/api_bridge/api.h") 83 | set(_COMPONENT_LIB_NAME "hebench_dynamic_lib_load" "api_bridge") 84 | set(_HEADER_ONLY FALSE) 85 | include(cmake/utils/import-library.cmake) 86 | include(cmake/third-party/${_COMPONENT_NAME}_check.cmake) 87 | 88 | # subprojects 89 | add_subdirectory(report_gen) 90 | add_subdirectory(dataset_loader) 91 | add_subdirectory(test_harness) 92 | 93 | ## yaml-cpp 94 | set(_COMPONENT_NAME "YAML_CPP") 95 | set(_COMPONENT_HEADER "yaml-cpp/yaml.h") 96 | set(_COMPONENT_LIB_NAME "yaml-cpp") 97 | set(_HEADER_ONLY FALSE) 98 | set(_LINK_TO "test_harness") 99 | include(cmake/utils/import-library.cmake) 100 | -------------------------------------------------------------------------------- /test_harness/benchmarks/SimpleSetIntersection/latency/include/hebench_simple_set_intersection_l.h: -------------------------------------------------------------------------------- 1 | 2 | // Copyright (C) 2021 Intel Corporation 3 | // SPDX-License-Identifier: Apache-2.0 4 | 5 | #ifndef _HEBench_Harness_SimpleSetIntersection_L_H_0596d40a3cce4b108a81595c50eb286d 6 | #define _HEBench_Harness_SimpleSetIntersection_L_H_0596d40a3cce4b108a81595c50eb286d 7 | 8 | #include 9 | 10 | #include "hebench/modules/general/include/nocopy.h" 11 | #include "hebench/modules/logging/include/logging.h" 12 | 13 | #include "benchmarks/SimpleSetIntersection/include/hebench_simple_set_intersection.h" 14 | #include "benchmarks/categories/include/hebench_benchmark_latency.h" 15 | #include "hebench/api_bridge/types.h" 16 | 17 | namespace hebench { 18 | namespace TestHarness { 19 | namespace SimpleSetIntersection { 20 | namespace Latency { 21 | 22 | class BenchmarkDescriptor final : public hebench::TestHarness::SimpleSetIntersection::BenchmarkDescriptorCategory 23 | { 24 | public: 25 | DISABLE_COPY(BenchmarkDescriptor) 26 | DISABLE_MOVE(BenchmarkDescriptor) 27 | private: 28 | IL_DECLARE_CLASS_NAME(SimpleSetIntersection::Latency::BenchmarkDescriptor) 29 | public: 30 | // TODO: Ask for ID selection 31 | static constexpr std::uint32_t BenchmarkID = 1201; 32 | static constexpr std::uint64_t DefaultBatchSize = 1; // batch size for a parameter (always 1 for latency) 33 | 34 | public: 35 | BenchmarkDescriptor() = default; 36 | ~BenchmarkDescriptor() override = default; 37 | 38 | hebench::TestHarness::PartialBenchmark *createBenchmark(std::shared_ptr p_engine, 39 | const DescriptionToken &description_token) override; 40 | void destroyBenchmark(hebench::TestHarness::PartialBenchmark *p_bench) override; 41 | 42 | protected: 43 | bool matchBenchmarkDescriptor(const hebench::APIBridge::BenchmarkDescriptor &bench_desc, 44 | const std::vector &w_params) const override; 45 | void completeWorkloadDescription(WorkloadDescriptionOutput &output, 46 | const Engine &engine, 47 | const BenchmarkDescription::Backend &backend_desc, 48 | const BenchmarkDescription::Configuration &config) const override; 49 | 50 | private: 51 | static bool m_b_registered; 52 | }; 53 | 54 | class Benchmark final : public hebench::TestHarness::BenchmarkLatency 55 | { 56 | public: 57 | DISABLE_COPY(Benchmark) 58 | DISABLE_MOVE(Benchmark) 59 | private: 60 | IL_DECLARE_CLASS_NAME(SimpleSetIntersection::Latency::Benchmark) 61 | 62 | public: 63 | friend class hebench::TestHarness::SimpleSetIntersection::Latency::BenchmarkDescriptor; 64 | 65 | ~Benchmark() override = default; 66 | 67 | protected: 68 | void init() override; 69 | IDataLoader::Ptr getDataset() const override { return m_data; } 70 | std::uint32_t getEventIDStart() const override { return BenchmarkDescriptor::BenchmarkID; } 71 | bool validateResult(IDataLoader::Ptr dataset, 72 | const std::uint64_t *param_data_pack_indices, 73 | const std::vector &p_outputs, 74 | hebench::APIBridge::DataType data_type) const override; 75 | 76 | private: 77 | DataLoader::Ptr m_data; 78 | std::uint64_t m_k_count; 79 | 80 | Benchmark(std::shared_ptr p_engine, 81 | const IBenchmarkDescriptor::DescriptionToken &description_token); 82 | }; 83 | 84 | } // namespace Latency 85 | } // namespace SimpleSetIntersection 86 | } // namespace TestHarness 87 | } // namespace hebench 88 | 89 | #endif // defined _HEBench_Harness_SimpleSetIntersection_L_H_0596d40a3cce4b108a81595c50eb286d 90 | -------------------------------------------------------------------------------- /test_harness/benchmarks/SimpleSetIntersection/offline/include/hebench_simple_set_intersection_o.h: -------------------------------------------------------------------------------- 1 | 2 | // Copyright (C) 2021 Intel Corporation 3 | // SPDX-License-Identifier: Apache-2.0 4 | 5 | #ifndef _HEBench_Harness_SimpleSetIntersection_O_H_0596d40a3cce4b108a81595c50eb286d 6 | #define _HEBench_Harness_SimpleSetIntersection_O_H_0596d40a3cce4b108a81595c50eb286d 7 | 8 | #include 9 | 10 | #include "hebench/modules/general/include/nocopy.h" 11 | #include "hebench/modules/logging/include/logging.h" 12 | 13 | #include "benchmarks/SimpleSetIntersection/include/hebench_simple_set_intersection.h" 14 | #include "benchmarks/categories/include/hebench_benchmark_offline.h" 15 | #include "hebench/api_bridge/types.h" 16 | 17 | namespace hebench { 18 | namespace TestHarness { 19 | namespace SimpleSetIntersection { 20 | namespace Offline { 21 | 22 | class BenchmarkDescriptor final : public hebench::TestHarness::SimpleSetIntersection::BenchmarkDescriptorCategory 23 | { 24 | public: 25 | DISABLE_COPY(BenchmarkDescriptor) 26 | DISABLE_MOVE(BenchmarkDescriptor) 27 | private: 28 | IL_DECLARE_CLASS_NAME(SimpleSetIntersection::Offline::BenchmarkDescriptor) 29 | public: 30 | // TODO: Ask for ID selection 31 | static constexpr std::uint32_t BenchmarkID = 1301; 32 | // default number of samples for an operand 33 | // (based on docs definition of the workload operation) 34 | static constexpr std::uint64_t DefaultBatchSize = 5; 35 | 36 | public: 37 | BenchmarkDescriptor() = default; 38 | ~BenchmarkDescriptor() override = default; 39 | 40 | hebench::TestHarness::PartialBenchmark *createBenchmark(std::shared_ptr p_engine, 41 | const DescriptionToken &description_token) override; 42 | void destroyBenchmark(hebench::TestHarness::PartialBenchmark *p_bench) override; 43 | 44 | protected: 45 | bool matchBenchmarkDescriptor(const hebench::APIBridge::BenchmarkDescriptor &bench_desc, 46 | const std::vector &w_params) const override; 47 | void completeWorkloadDescription(WorkloadDescriptionOutput &output, 48 | const Engine &engine, 49 | const BenchmarkDescription::Backend &backend_desc, 50 | const BenchmarkDescription::Configuration &config) const override; 51 | 52 | private: 53 | static bool m_b_registered; 54 | }; 55 | 56 | class Benchmark : public hebench::TestHarness::BenchmarkOffline 57 | { 58 | public: 59 | DISABLE_COPY(Benchmark) 60 | DISABLE_MOVE(Benchmark) 61 | private: 62 | IL_DECLARE_CLASS_NAME(SimpleSetIntersection::Offline::Benchmark) 63 | 64 | public: 65 | friend class hebench::TestHarness::SimpleSetIntersection::Offline::BenchmarkDescriptor; 66 | 67 | ~Benchmark() override = default; 68 | 69 | protected: 70 | void init() override; 71 | IDataLoader::Ptr getDataset() const override { return m_data; } 72 | std::uint32_t getEventIDStart() const override { return BenchmarkDescriptor::BenchmarkID; } 73 | bool validateResult(IDataLoader::Ptr dataset, 74 | const std::uint64_t *param_data_pack_indices, 75 | const std::vector &p_outputs, 76 | hebench::APIBridge::DataType data_type) const override; 77 | 78 | private: 79 | DataLoader::Ptr m_data; 80 | std::uint64_t m_k_count; 81 | 82 | Benchmark(std::shared_ptr p_engine, const IBenchmarkDescriptor::DescriptionToken &description_token); 83 | }; 84 | 85 | } // namespace Offline 86 | } // namespace SimpleSetIntersection 87 | } // namespace TestHarness 88 | } // namespace hebench 89 | 90 | #endif // defined _HEBench_Harness_SimpleSetIntersection_O_H_0596d40a3cce4b108a81595c50eb286d 91 | -------------------------------------------------------------------------------- /test_harness/include/hebench_benchmark_factory.h: -------------------------------------------------------------------------------- 1 | 2 | // Copyright (C) 2021 Intel Corporation 3 | // SPDX-License-Identifier: Apache-2.0 4 | 5 | #ifndef _HEBench_Harness_Benchmark_Factory_H_0596d40a3cce4b108a81595c50eb286d 6 | #define _HEBench_Harness_Benchmark_Factory_H_0596d40a3cce4b108a81595c50eb286d 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | #include "hebench/modules/general/include/nocopy.h" 13 | #include "hebench/modules/logging/include/logging.h" 14 | 15 | #include "hebench/api_bridge/types.h" 16 | #include "hebench_ibenchmark.h" 17 | #include "hebench_utilities_harness.h" 18 | 19 | namespace hebench { 20 | namespace TestHarness { 21 | 22 | class Engine; 23 | 24 | class BenchmarkFactory final 25 | { 26 | public: 27 | DISABLE_COPY(BenchmarkFactory) 28 | DISABLE_MOVE(BenchmarkFactory) 29 | private: 30 | IL_DECLARE_CLASS_NAME(BenchmarkFactory) 31 | public: 32 | friend class Engine; 33 | 34 | /** 35 | * @brief Returns a token representing a benchmark that can perform the described 36 | * workload with specified parameters, if any. 37 | * @param[in] engine Engine requesting the matching. 38 | * @param[in] backend_desc Backend descriptor to match. 39 | * @param[in] config Configuration of benchmark to match. 40 | * @returns A valid token representing the matched benchmark. 41 | * @returns `null` if no match was found. 42 | */ 43 | static IBenchmarkDescriptor::DescriptionToken::Ptr matchBenchmarkDescriptor(const Engine &engine, 44 | const BenchmarkDescription::Backend &backend_desc, 45 | const BenchmarkDescription::Configuration &config); 46 | /** 47 | * @brief Registers a benchmark description object that represents one of the 48 | * supported workloads. 49 | * @param[in] p_desc_obj Smart pointer to benchmark description object to register. 50 | * @returns `true` if registration is successful. 51 | * @returns `false` otherwise. 52 | * @details 53 | * This method should be called for each benchmark during static initialization to 54 | * register the benchmark description before the application main entry-point is reached. 55 | * 56 | * This method does not throw exceptions. 57 | */ 58 | static bool registerSupportedBenchmark(std::shared_ptr p_desc_obj); 59 | 60 | private: 61 | static std::vector> &getRegisteredBenchmarks(); 62 | /** 63 | * @brief Instantiates the benchmark represented by the specified token. 64 | * @param p_engine Calling engine object. 65 | * @param[in] p_token Token representing the benchmark to instantiate, as returned by 66 | * matchBenchmarkDescriptor(). 67 | * @return The benchmark object for which test harness can issue calls to 68 | * execute the benchmark. 69 | * @throws std::invalid_argument if any argument is `null` or in an invalid state. 70 | * @throws std::runtime_error if an error occurs during creation of the benchmark. 71 | * @throws any exception that the underlying benchmark generation system throws. 72 | * @details Only Engine can create a benchmark, thus, this method is private, 73 | * accessible only by friend class Engine. 74 | */ 75 | static IBenchmark::Ptr createBenchmark(std::shared_ptr p_engine, 76 | IBenchmarkDescriptor::DescriptionToken::Ptr p_token, 77 | hebench::Utilities::TimingReportEx &out_report); 78 | 79 | BenchmarkFactory() = default; 80 | }; 81 | 82 | } // namespace TestHarness 83 | } // namespace hebench 84 | 85 | #endif // defined _HEBench_Harness_Benchmark_Factory_H_0596d40a3cce4b108a81595c50eb286d 86 | -------------------------------------------------------------------------------- /test_harness/include/inl/hebench_utilities_harness.inl: -------------------------------------------------------------------------------- 1 | 2 | // Copyright (C) 2021 Intel Corporation 3 | // SPDX-License-Identifier: Apache-2.0 4 | 5 | #ifndef _HEBench_Harness_Utilities_SRC_0596d40a3cce4b108a81595c50eb286d 6 | #define _HEBench_Harness_Utilities_SRC_0596d40a3cce4b108a81595c50eb286d 7 | 8 | #include 9 | 10 | #include "../hebench_utilities_harness.h" 11 | #include "hebench/modules/general/include/hebench_utilities.h" 12 | 13 | namespace hebench { 14 | namespace Utilities { 15 | 16 | template 17 | void printArraysAsColumns(std::ostream &os, 18 | const hebench::APIBridge::NativeDataBuffer **p_buffers, std::size_t count, 19 | bool output_row_index, 20 | const char *separator) 21 | { 22 | std::uint64_t max_rows = 0; 23 | if (p_buffers && count > 0) 24 | { 25 | auto it = std::max_element(p_buffers, p_buffers + count, 26 | [](const hebench::APIBridge::NativeDataBuffer *lhs, const hebench::APIBridge::NativeDataBuffer *rhs) { 27 | return lhs->size < rhs->size; 28 | }); 29 | max_rows = (*it)->size / sizeof(T); 30 | } // end if 31 | 32 | for (std::uint64_t row_i = 0; row_i < max_rows; ++row_i) 33 | { 34 | // print next row 35 | if (output_row_index) 36 | os << row_i << separator; 37 | for (std::size_t buffer_i = 0; buffer_i < count; ++buffer_i) 38 | { 39 | if (buffer_i > 0) 40 | os << separator; 41 | if (p_buffers[buffer_i]->p && row_i < p_buffers[buffer_i]->size / sizeof(T)) 42 | os << reinterpret_cast(p_buffers[buffer_i]->p)[row_i]; 43 | } // end for 44 | os << std::endl; 45 | } // end for 46 | } 47 | 48 | template 49 | inline void TimingReportEx::addEvent(hebench::Common::TimingReportEvent::Ptr p_event) 50 | { 51 | this->addEvent(p_event, nullptr); 52 | } 53 | 54 | template 55 | inline void TimingReportEx::addEvent(hebench::Common::TimingReportEvent::Ptr p_event, 56 | const std::string &event_type_name) 57 | { 58 | this->addEvent(p_event, event_type_name.c_str()); 59 | } 60 | 61 | template 62 | inline void TimingReportEx::addEvent(hebench::Common::TimingReportEvent::Ptr p_event, 63 | const char *event_type_name) 64 | { 65 | if (event_type_name) 66 | this->addEventType(p_event->id, event_type_name); 67 | hebench::ReportGen::TimingReportEventC tre_c = 68 | convert2C(*p_event); 69 | this->addEvent(tre_c); 70 | } 71 | 72 | template 73 | inline hebench::ReportGen::TimingReportEventC 74 | TimingReportEx::convert2C(const hebench::Common::TimingReportEvent &timing_event) 75 | { 76 | hebench::ReportGen::TimingReportEventC retval; 77 | retval.time_interval_ratio_num = TimeInterval::num; 78 | retval.time_interval_ratio_den = TimeInterval::den; 79 | retval.event_type_id = timing_event.id; 80 | retval.cpu_time_start = timing_event.timeStartCPU(); 81 | retval.cpu_time_end = timing_event.timeEndCPU(); 82 | retval.wall_time_start = timing_event.timeStartWall(); 83 | retval.wall_time_end = timing_event.timeEndWall(); 84 | retval.input_sample_count = timing_event.iterations(); 85 | copyString(retval.description, MAX_TIME_REPORT_EVENT_DESCRIPTION_SIZE, timing_event.description); 86 | 87 | return retval; 88 | } 89 | 90 | } // namespace Utilities 91 | } // namespace hebench 92 | 93 | #endif // defined _HEBench_Harness_Utilities_SRC_0596d40a3cce4b108a81595c50eb286d 94 | -------------------------------------------------------------------------------- /docsrc/tests/elementwise_mult.md: -------------------------------------------------------------------------------- 1 | Vector Element-wise Multiplication Workload {#elementwise_mult} 2 | ======================== 3 | 4 | [TOC] 5 | 6 | ## Operation Description 7 | 8 | ### Summary 9 | This operation is defined as: 10 | 11 | ```cpp 12 | C = op(A, B) 13 | ``` 14 | 15 | where `op` is the standard vector element-wise multiplication. 16 | 17 | ### Operation Parameters 18 | 19 | Input: `2` parameters 20 | 21 | | Parameter | Description | 22 | |-|-| 23 | | `0` | `A` is a vector with `n` components. | 24 | | `1` | `B` is a vector with `n` components. | 25 | 26 | ### Result: 27 | 28 | Output: `1` output 29 | 30 | | Output | Description | 31 | |-|-| 32 | | `0` | `C` is a vector with `n` components. It is the result of the component-wise multiplication of `A` and `B`. | 33 | 34 | ### Details 35 | 36 | If `Z[i]` denotes the element at component `i` in vector `Z`, then, the standard element-wise multiplication operation is defined as: 37 | 38 | ``` 39 | C[i] = A[i] * B[i] 40 | ``` 41 | 42 | ## Workloads 43 | 44 | This document applies to the following workloads: 45 | 46 | ```cpp 47 | // From hebench/api_bridge/types.h 48 | 49 | hebench:APIBridge::Workload::EltwiseMultiply 50 | ``` 51 | 52 | ### Workload Parameters 53 | 54 | | Index | Name | Type | Description | 55 | |-|-|-|-| 56 | | `0` | `n` | `uint64_t` | Number of components in a vector. | 57 | 58 | Above parameters are required for the workload in the specified order. A backend must specify, at least, a set of default arguments for these parameters. 59 | 60 | Backends can require extra parameters beyond the base requirements. If a backend requires extra parameters, these must have default values in every set of default arguments for the workload parameters. 61 | 62 | ## Categories 63 | This workload supports the following categories: 64 | 65 | ```cpp 66 | hebench:APIBridge::Category::Latency 67 | hebench:APIBridge::Category::Offline 68 | ``` 69 | 70 | ### Category Parameters 71 | #### Latency 72 | See @ref category_latency . 73 | 74 | #### Offline 75 | See @ref category_offline . 76 | 77 | Value ranges for elements in `CategoryParams::offline::data_count`. Default value is used when the backend implementation sets the `data_count` for the corresponding operand to `0`, but user specified `0` or no value at run-time. 78 | 79 | | Parameter | Lower bound | Upper bound | Default | 80 | |-|-|-|-| 81 | | `0` | `1` | none |`5` | 82 | | `1` | `1` | none |`5` | 83 | 84 | ## Data Type 85 | 86 | This workload is defined for the following data types: 87 | 88 | ```cpp 89 | hebench::APIBridge::DataType::Int32 90 | hebench::APIBridge::DataType::Int64 91 | hebench::APIBridge::DataType::Float32 92 | hebench::APIBridge::DataType::Float64 93 | ``` 94 | 95 | ## Data Layout 96 | All scalar elements in a vector with elements of type `T` (where `T` is any of the supported types) lie contiguous in memory. 97 | 98 | For example, given the following vector `A` with `n = 3` components: 99 | 100 | ```cpp 101 | A = [ a0, a1, a2 ] 102 | ``` 103 | 104 | The elements will be stored in memory as: 105 | 106 | | Offset: | 0 | 1 | 2 | 107 | |-|-|-|-| 108 | |A| `a0` | `a1` | `a2` | 109 | 110 | Backends should expect this layout for their raw, clear text inputs, and must generate this layout for their decoded outputs. 111 | 112 | #### Notes 113 | If several vectors will be pointed at by a single pointer, consecutive vectors will follow each other in memory. 114 | 115 | ## Dataset 116 | Supported modes: 117 | 118 | | Generate | External | 119 | |-|-| 120 | | yes | yes | 121 | 122 | Data generation for vectors used as input for this workload occurs during workload initialization by Test Harness. Ground truths are pre-computed during data generation. There is no standard dataset. 123 | 124 | During data generation, all vector elements are extracted from a pseudo-random uniform distribution between `-10` and `10`: `u(-10, 10)`. 125 | -------------------------------------------------------------------------------- /docsrc/tests/elementwise_add.md: -------------------------------------------------------------------------------- 1 | Vector Element-wise Addition Workload {#elementwise_add} 2 | ======================== 3 | 4 | [TOC] 5 | 6 | ## Operation Description 7 | 8 | ### Summary 9 | This operation is defined as: 10 | 11 | ```cpp 12 | C = op(A, B) 13 | ``` 14 | 15 | where 16 | 17 | ```cpp 18 | op(A, B) = A + B 19 | ``` 20 | 21 | is the standard vector element-wise addition. 22 | 23 | ### Operation Parameters 24 | 25 | Input: `2` parameters 26 | 27 | | Parameter | Description | 28 | |-|-| 29 | | `0` | `A` is a vector with `n` components. | 30 | | `1` | `B` is a vector with `n` components. | 31 | 32 | ### Result: 33 | 34 | Output: `1` output 35 | 36 | | Output | Description | 37 | |-|-| 38 | | `0` | `C` is a vector with `n` components. It is the result of adding `A` and `B`. | 39 | 40 | ### Details 41 | 42 | If `Z[i]` denotes the element at component `i` in vector `Z`, then, the standard element-wise addition operation is defined as: 43 | 44 | ``` 45 | C[i] = A[i] + B[i] 46 | ``` 47 | 48 | ## Workloads 49 | 50 | This document applies to the following workloads: 51 | 52 | ```cpp 53 | // From hebench/api_bridge/types.h 54 | 55 | hebench:APIBridge::Workload::EltwiseAdd 56 | ``` 57 | 58 | ### Workload Parameters 59 | 60 | Required workload parameters: `1` 61 | 62 | | Index | Name | Type | Description | 63 | |-|-|-|-| 64 | | `0` | `n` | `uint64_t` | Number of components in a vector. | 65 | 66 | Above parameters are required for the workload in the specified order. A backend must specify, at least, a set of default arguments for these parameters. 67 | 68 | Backends can require extra parameters beyond the base requirements. If a backend requires extra parameters, these must have default values in every set of default arguments for the workload parameters. 69 | 70 | ## Categories 71 | This workload supports the following categories: 72 | 73 | ```cpp 74 | hebench:APIBridge::Category::Latency 75 | hebench:APIBridge::Category::Offline 76 | ``` 77 | 78 | ### Category Parameters 79 | #### Latency 80 | See @ref category_latency . 81 | 82 | #### Offline 83 | See @ref category_offline . 84 | 85 | Value ranges for elements in `CategoryParams::offline::data_count`. Default value is used when the backend implementation sets the `data_count` for the corresponding operand to `0`, but user specified `0` or no value at run-time. 86 | 87 | | Parameter | Lower bound | Upper bound | Default | 88 | |-|-|-|-| 89 | | `0` | `1` | none |`5` | 90 | | `1` | `1` | none |`5` | 91 | 92 | ## Data Type 93 | 94 | This workload is defined for the following data types: 95 | 96 | ```cpp 97 | hebench::APIBridge::DataType::Int32 98 | hebench::APIBridge::DataType::Int64 99 | hebench::APIBridge::DataType::Float32 100 | hebench::APIBridge::DataType::Float64 101 | ``` 102 | 103 | ## Data Layout 104 | All scalar elements in a vector with elements of type `T` (where `T` is any of the supported types) lie contiguous in memory. 105 | 106 | For example, given the following vector `A` with `n = 3` components: 107 | 108 | ```cpp 109 | A = [ a0, a1, a2 ] 110 | ``` 111 | 112 | The elements will be stored in memory as: 113 | 114 | | Offset: | 0 | 1 | 2 | 115 | |-|-|-|-| 116 | |A| `a0` | `a1` | `a2` | 117 | 118 | Backends should expect this layout for their raw, clear text inputs, and must generate this layout for their decoded outputs. 119 | 120 | #### Notes 121 | If several vectors will be pointed at by a single pointer, consecutive vectors will follow each other in memory. 122 | 123 | ## Dataset 124 | Supported modes: 125 | 126 | | Generate | External | 127 | |-|-| 128 | | yes | yes | 129 | 130 | Data generation for vectors used as input for this workload occurs during workload initialization by Test Harness. Ground truths are pre-computed during data generation. There is no standard dataset. 131 | 132 | During data generation, all vector elements are extracted from a pseudo-random uniform distribution between `-10` and `10`: `u(-10, 10)`. 133 | --------------------------------------------------------------------------------