├── .clang-format ├── .github └── workflows │ ├── ci-test.yml │ └── deploy-api-doc.yml ├── .gitignore ├── .gitlab-ci.yml ├── .readthedocs.yaml ├── CITATION.cff ├── CMakeLists.txt ├── COPYRIGHT ├── LICENSE-APACHE ├── LICENSE-MIT ├── MetallConfig.h.in ├── NOTICE ├── README.md ├── bench ├── CMakeLists.txt ├── adjacency_list │ ├── CMakeLists.txt │ ├── bench_driver.hpp │ ├── edge_generator │ │ ├── CMakeLists.txt │ │ ├── generate_rmat_edge_list.cpp │ │ └── rmat_edge_generator.hpp │ ├── kernel.hpp │ ├── numa_aware_kernel.hpp │ ├── run_adj_list_bench_bip.cpp │ ├── run_adj_list_bench_bip_extend.cpp │ ├── run_adj_list_bench_metall.cpp │ ├── run_adj_list_bench_pmem.cpp │ ├── run_adj_list_bench_reflink_snapshot.cpp │ ├── run_adj_list_bench_stl.cpp │ ├── run_bench.sh │ └── test │ │ ├── CMakeLists.txt │ │ ├── compare_key_value_lists.cpp │ │ ├── data │ │ ├── CMakeLists.txt │ │ ├── edge_list_rmat_s10_0_of_4 │ │ ├── edge_list_rmat_s10_1_of_4 │ │ ├── edge_list_rmat_s10_2_of_4 │ │ └── edge_list_rmat_s10_3_of_4 │ │ ├── destroy_metall.cpp │ │ ├── extend_metall.cpp │ │ ├── open_metall.cpp │ │ ├── test.sh │ │ └── test_large.sh ├── bfs │ ├── CMakeLists.txt │ ├── bench_driver.hpp │ ├── kernel.hpp │ ├── run_bench.sh │ ├── run_bfs_bench_bip.cpp │ ├── run_bfs_bench_metall.cpp │ └── run_bfs_bench_metall_multiple.cpp ├── container │ ├── CMakeLists.txt │ ├── bench_common.hpp │ ├── run_map_bench.cpp │ ├── run_unordered_map_bench.cpp │ └── run_vector_bench.cpp ├── data_structure │ ├── adjacency_list.hpp │ ├── multithread_adjacency_list.hpp │ └── partitioned_multithread_adjacency_list.hpp ├── mapping │ ├── CMakeLists.txt │ └── run_mapping_bench.cpp ├── offset_ptr │ ├── CMakeLists.txt │ └── run_offset_ptr_bench.cpp ├── rand_engine │ ├── CMakeLists.txt │ └── run_rand_engine_bench.cpp ├── simple_alloc │ ├── CMakeLists.txt │ ├── kernel.hpp │ ├── run_bench.sh │ ├── run_simple_allocation_bench_bip.cpp │ ├── run_simple_allocation_bench_metall.cpp │ └── run_simple_allocation_bench_stl.cpp └── utility │ ├── jemalloc_allocator.hpp │ ├── numa.hpp │ ├── numa_allocator.hpp │ └── pair_reader.hpp ├── cmake ├── MetallConfig.cmake.in ├── build_doc.cmake ├── check_cxx_filesystem_library.cmake ├── find_boost_headers.cmake ├── setup_mpi.cmake └── setup_omp.cmake ├── docs ├── Doxyfile.in ├── publications │ └── metall_101.pdf └── readthedocs │ ├── about │ ├── license_notice.md │ └── publication.md │ ├── advanced_build │ ├── cmake.md │ ├── test.md │ └── umap.md │ ├── basics │ ├── compile_time_options.md │ └── getting_started.md │ ├── css │ └── custom.css │ ├── detail │ ├── api.md │ ├── example.md │ ├── internal_architecture.md │ ├── introduction.md │ ├── ls.md │ ├── persistence_policy.md │ ├── pointer.md │ └── snapshot.md │ ├── external │ ├── list.md │ ├── ripples-metall.patch │ └── ripples.md │ ├── img │ ├── metall_architecture.png │ └── metall_overview.png │ └── index.md ├── example ├── CMakeLists.txt ├── README.md ├── adjacency_list_graph.cpp ├── allocator_aware_type.cpp ├── c_api.c ├── cmake │ ├── FetchContent │ │ ├── CMakeLists.txt │ │ └── README.md │ ├── README.md │ ├── find_package │ │ ├── CMakeLists.txt │ │ └── README.md │ └── src │ │ ├── c_example.c │ │ └── cpp_example.cpp ├── complex_map.cpp ├── concurrent.cpp ├── concurrent_map.cpp ├── container │ ├── CMakeLists.txt │ └── string_key_store.cpp ├── csr_graph.cpp ├── custom_logger.cpp ├── datastore_description.cpp ├── fallback_allocator.cpp ├── graph_data_structure │ ├── adjacency_list.hpp │ ├── csr.hpp │ └── csr_using_vector.hpp ├── json │ ├── CMakeLists.txt │ ├── jgraph.cpp │ ├── json_create.cpp │ ├── json_open.cpp │ └── vector_of_json.cpp ├── logger.cpp ├── metall_containers.cpp ├── mpi_create.cpp ├── mpi_open.cpp ├── multilevel_containers.cpp ├── object_attribute.cpp ├── object_attribute_api_list.cpp ├── offset_pointer.cpp ├── simple.cpp ├── snapshot.cpp ├── static_mutex.cpp ├── string.cpp ├── string_map.cpp └── vector_of_vectors.cpp ├── include └── metall │ ├── basic_manager.hpp │ ├── c_api │ └── metall.h │ ├── container │ ├── concurrent_map.hpp │ ├── deque.hpp │ ├── experimental │ │ ├── jgraph │ │ │ └── jgraph.hpp │ │ └── json │ │ │ └── json.hpp │ ├── fallback_allocator.hpp │ ├── list.hpp │ ├── map.hpp │ ├── priority_queue.hpp │ ├── queue.hpp │ ├── scoped_allocator.hpp │ ├── set.hpp │ ├── stack.hpp │ ├── string.hpp │ ├── string_key_store.hpp │ ├── string_key_store_locator.hpp │ ├── unordered_flat_map.hpp │ ├── unordered_flat_set.hpp │ ├── unordered_map.hpp │ ├── unordered_node_map.hpp │ ├── unordered_node_set.hpp │ ├── unordered_set.hpp │ └── vector.hpp │ ├── defs.hpp │ ├── detail │ ├── array_construct.hpp │ ├── bitset.hpp │ ├── builtin_functions.hpp │ ├── char_ptr_holder.hpp │ ├── file.hpp │ ├── file_clone.hpp │ ├── hash.hpp │ ├── memory.hpp │ ├── mmap.hpp │ ├── mutex.hpp │ ├── named_proxy.hpp │ ├── parameter_pack.hpp │ ├── proc.hpp │ ├── ptree.hpp │ ├── soft_dirty_page.hpp │ ├── static_bitset.hpp │ ├── time.hpp │ ├── utilities.hpp │ └── uuid.hpp │ ├── ext │ ├── privateer.hpp │ └── umap.hpp │ ├── json │ ├── array.hpp │ ├── details │ │ ├── compact_object.hpp │ │ └── indexed_object.hpp │ ├── equal.hpp │ ├── json.hpp │ ├── json_fwd.hpp │ ├── key_value_pair.hpp │ ├── object.hpp │ ├── parse.hpp │ ├── pretty_print.hpp │ ├── serialize.hpp │ ├── string.hpp │ ├── value.hpp │ ├── value_from.hpp │ └── value_to.hpp │ ├── kernel │ ├── attributed_object_directory.hpp │ ├── bin_directory.hpp │ ├── bin_number_manager.hpp │ ├── chunk_directory.hpp │ ├── manager_kernel.hpp │ ├── manager_kernel_fwd.hpp │ ├── manager_kernel_impl.ipp │ ├── manager_kernel_profile_impl.ipp │ ├── multilayer_bitset.hpp │ ├── object_cache.hpp │ ├── object_size_manager.hpp │ ├── segment_allocator.hpp │ ├── segment_header.hpp │ ├── segment_storage.hpp │ ├── segment_storage │ │ └── umap_sparse_segment_storage.hpp │ └── storage.hpp │ ├── logger.hpp │ ├── logger_interface.h │ ├── metall.hpp │ ├── object_attribute_accessor.hpp │ ├── offset_ptr.hpp │ ├── stl_allocator.hpp │ ├── tags.hpp │ ├── utility │ ├── container_of_containers_iterator_adaptor.hpp │ ├── datastore_ls.hpp │ ├── fallback_allocator_adaptor.hpp │ ├── filesystem.hpp │ ├── hash.hpp │ ├── metall_mpi_adaptor.hpp │ ├── metall_mpi_datastore.hpp │ ├── mpi.hpp │ ├── mutex.hpp │ ├── open_mp.hpp │ └── random.hpp │ └── version.hpp ├── mkdocs.yml ├── scripts ├── CI │ └── build_and_test.sh ├── release_test │ ├── full_build_and_test.sh │ └── run_tests.sh ├── sanitizer │ └── run_sanitizers.sh ├── test_kernel.sh └── test_utility.sh ├── src ├── CMakeLists.txt ├── datastore_ls.cpp ├── metall_c.cpp └── mpi_datastore_ls.cpp ├── test ├── CMakeLists.txt ├── CMakeListsGTest.txt.in ├── container │ ├── CMakeLists.txt │ ├── concurrent_map_test.cpp │ ├── fallback_allocator_test.cpp │ ├── json │ │ ├── CMakeLists.txt │ │ ├── json_array.cpp │ │ ├── json_object.cpp │ │ └── json_value.cpp │ ├── stl_allocator_test.cpp │ └── string_key_store_test.cpp ├── gtest_main.cpp ├── kernel │ ├── CMakeLists.txt │ ├── attributed_object_directory_test.cpp │ ├── bin_directory_test.cpp │ ├── bin_manager_test.cpp │ ├── chunk_directory_test.cpp │ ├── copy_datastore_test.cpp │ ├── manager_multithread_test.cpp │ ├── manager_test.cpp │ ├── multilayer_bitset_test.cpp │ ├── multimanager_test.cpp │ ├── object_attribute_accessor_test.cpp │ ├── object_cache_test.cpp │ ├── segment_storage_test.cpp │ └── snapshot_test.cpp ├── test_utility.hpp └── utility │ ├── CMakeLists.txt │ └── bitset_test.cpp ├── tool └── waste_memory.sh ├── tutorial ├── hands_on │ ├── README.md │ ├── open_terminal.png │ ├── t0.cpp │ ├── t1-1.cpp │ ├── t1-2.cpp │ ├── t2-1.cpp │ ├── t2-2.cpp │ ├── t3.cpp │ ├── t4-1.cpp │ ├── t4-1.hpp │ ├── t4-2.cpp │ ├── t4-2.hpp │ ├── t5-1.cpp │ ├── t5-2_create.cpp │ ├── t5-2_open.cpp │ ├── tutorial_flow.png │ └── use_googleclould.pdf ├── ics21 │ ├── README.md │ └── metall_101.pdf └── nvmw21 │ ├── CMakeLists.txt │ ├── README.md │ ├── metall_101.pdf │ ├── t0.cpp │ ├── t1-1.cpp │ ├── t1-2.cpp │ ├── t2-1.cpp │ ├── t2-2.cpp │ ├── t3.cpp │ ├── t4-1.cpp │ ├── t4-1.hpp │ ├── t4-2.cpp │ ├── t4-2.hpp │ ├── t5-1.cpp │ ├── t5-2_create.cpp │ ├── t5-2_open.cpp │ ├── tutorial_flow.png │ └── use_googleclould.pdf └── verification ├── CMakeLists.txt ├── file_clone ├── CMakeLists.txt └── verify_file_clone.cpp ├── file_io ├── CMakeLists.txt └── verify_64bits_file_io.cpp ├── free_mmap_region ├── CMakeLists.txt ├── free_mmap_region.hpp ├── run_free_file_space_bench.cpp └── verify_mmap_region_free.cpp ├── logger ├── CMakeLists.txt └── verify_logger.cpp ├── mmap ├── CMakeLists.txt ├── run_many_files_mmap_bench.cpp ├── verify_mmap.cpp └── verify_mmap_openmp.cpp ├── object_size ├── CMakeLists.txt └── show_object_sizes.cpp ├── soft_dirty ├── CMakeLists.txt └── verify_soft_dirty.cpp └── sparse_copy ├── CMakeLists.txt ├── verify_sparse_copy.cpp └── verify_sparse_copy_syscalls.cpp /.github/workflows/ci-test.yml: -------------------------------------------------------------------------------- 1 | name: CI Test 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | bst1-87-0: 7 | runs-on: ubuntu-latest 8 | env: 9 | METALL_LIMIT_MAKE_PARALLELS: 8 10 | steps: 11 | - uses: actions/checkout@v3 12 | - name: Test 13 | run: | 14 | pushd /dev/shm 15 | wget -q https://archives.boost.io/release/1.87.0/source/boost_1_87_0.tar.gz 16 | mkdir boost 17 | tar xf boost_1_87_0.tar.gz -C boost --strip-components 1 18 | export BOOST_ROOT=${PWD}/boost 19 | popd 20 | export METALL_TEST_DIR=${GITHUB_JOB} 21 | export CC=gcc-12 22 | export CXX=g++-12 23 | cd $GITHUB_WORKSPACE 24 | bash ./scripts/CI/build_and_test.sh 25 | 26 | bst1-80-0: 27 | runs-on: ubuntu-latest 28 | env: 29 | METALL_LIMIT_MAKE_PARALLELS: 8 30 | steps: 31 | - uses: actions/checkout@v3 32 | - name: Test 33 | run: | 34 | pushd /dev/shm 35 | wget -q https://archives.boost.io/release/1.80.0/source/boost_1_80_0.tar.gz 36 | mkdir boost 37 | tar xf boost_1_80_0.tar.gz -C boost --strip-components 1 38 | export BOOST_ROOT=${PWD}/boost 39 | popd 40 | export METALL_TEST_DIR=${GITHUB_JOB} 41 | export CC=gcc-12 42 | export CXX=g++-12 43 | cd $GITHUB_WORKSPACE 44 | bash ./scripts/CI/build_and_test.sh 45 | -------------------------------------------------------------------------------- /.github/workflows/deploy-api-doc.yml: -------------------------------------------------------------------------------- 1 | name: Deploy API Doc 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | 8 | pull_request: 9 | branches: 10 | - master 11 | 12 | jobs: 13 | main: 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/checkout@v2 17 | 18 | - name: Generate 19 | run: | 20 | sudo apt-get install doxygen 21 | mkdir build_doc 22 | cd build_doc 23 | doxygen ../docs/Doxyfile.in 24 | mkdir -p ../public/api 25 | cp -r ./html/* ../public/api/ 26 | cd ../ 27 | 28 | - name: Deploy 29 | uses: peaceiris/actions-gh-pages@v3 30 | with: 31 | github_token: ${{ secrets.GITHUB_TOKEN }} 32 | publish_dir: ./public -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | CMakeCache.txt 2 | CMakeFiles 3 | cmake_install.cmake 4 | .swp 5 | build*/ 6 | .cobaltlog 7 | .error 8 | .autosave 9 | .DS_Store 10 | cmake-build-*/ 11 | .idea 12 | *~ 13 | ~* 14 | .vscode/ -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | build: 4 | os: ubuntu-22.04 5 | tools: 6 | python: "3.12" 7 | 8 | mkdocs: 9 | configuration: mkdocs.yml 10 | -------------------------------------------------------------------------------- /CITATION.cff: -------------------------------------------------------------------------------- 1 | # This CITATION.cff file was generated with cffinit. 2 | # Visit https://bit.ly/cffinit to generate yours today! 3 | 4 | cff-version: 1.2.0 5 | title: >- 6 | Metall: A Persistent Memory Allocator for 7 | Data-Centric Analytics 8 | message: >- 9 | If you use this software, please cite it using the 10 | metadata from this file. 11 | type: software 12 | authors: 13 | - given-names: Keita 14 | family-names: Iwabuchi 15 | email: kiwabuchi@llnl.gov 16 | affiliation: ' Lawrence Livermore National Laboratory' 17 | orcid: 'https://orcid.org/0000-0002-9395-0843' 18 | - given-names: 'Roger ' 19 | family-names: Pearce 20 | affiliation: Lawrence Livermore National Laboratory 21 | - given-names: Maya 22 | family-names: Gokhale 23 | affiliation: Lawrence Livermore National Laboratory 24 | identifiers: 25 | - type: url 26 | value: 'https://doi.org/10.11578/dc.20190410.1' 27 | repository-code: 'https://github.com/LLNL/metall' 28 | abstract: >- 29 | Metall is a persistent memory allocator built on 30 | top of the memory-mapped file mechanism. 31 | 32 | Metall enables applications to transparently 33 | allocate custom C++ data structures into various 34 | types of persistent memories. 35 | -------------------------------------------------------------------------------- /COPYRIGHT: -------------------------------------------------------------------------------- 1 | Intellectual Property Notice 2 | ------------------------------ 3 | 4 | Metall is the Apache License, Version 2.0 (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or 5 | the MIT license, (LICENSE-MIT or http://opensource.org/licenses/MIT), at your option. 6 | 7 | Copyrights and patents in the Metall project are retained by contributors. 8 | No copyright assignment is required to contribute to Metall. 9 | 10 | 11 | SPDX usage 12 | ------------ 13 | 14 | Individual files contain SPDX tags instead of the full license text. 15 | This enables machine processing of license information based on the SPDX 16 | License Identifiers that are available here: https://spdx.org/licenses/ 17 | 18 | Files that are dual-licensed as Apache-2.0 OR MIT contain the following 19 | text in the license header: 20 | 21 | SPDX-License-Identifier: (Apache-2.0 OR MIT) 22 | -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright 2019 Lawrence Livermore National Security, LLC and other 4 | Metall Project Developers. See the top-level COPYRIGHT file for details. 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. -------------------------------------------------------------------------------- /MetallConfig.h.in: -------------------------------------------------------------------------------- 1 | #define METALL_VERSION_MAJOR @Metall_VERSION_MAJOR@ 2 | #define METALL_VERSION_MINOR @Metall_VERSION_MINOR@ -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | This work was produced under the auspices of the U.S. Department of 2 | Energy by Lawrence Livermore National Laboratory under Contract 3 | DE-AC52-07NA27344. 4 | 5 | This work was prepared as an account of work sponsored by an agency of 6 | the United States Government. Neither the United States Government nor 7 | Lawrence Livermore National Security, LLC, nor any of their employees 8 | makes any warranty, expressed or implied, or assumes any legal liability 9 | or responsibility for the accuracy, completeness, or usefulness of any 10 | information, apparatus, product, or process disclosed, or represents that 11 | its use would not infringe privately owned rights. 12 | 13 | Reference herein to any specific commercial product, process, or service 14 | by trade name, trademark, manufacturer, or otherwise does not necessarily 15 | constitute or imply its endorsement, recommendation, or favoring by the 16 | United States Government or Lawrence Livermore National Security, LLC. 17 | 18 | The views and opinions of authors expressed herein do not necessarily 19 | state or reflect those of the United States Government or Lawrence 20 | Livermore National Security, LLC, and shall not be used for advertising 21 | or product endorsement purposes. -------------------------------------------------------------------------------- /bench/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_subdirectory(simple_alloc) 2 | add_subdirectory(adjacency_list) 3 | add_subdirectory(bfs) 4 | add_subdirectory(rand_engine) 5 | add_subdirectory(mapping) 6 | add_subdirectory(container) 7 | add_subdirectory(offset_ptr) -------------------------------------------------------------------------------- /bench/adjacency_list/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | include(setup_omp) 2 | 3 | add_metall_executable(run_adj_list_bench_stl run_adj_list_bench_stl.cpp) 4 | setup_omp_target(run_adj_list_bench_stl) 5 | 6 | add_metall_executable(run_adj_list_bench_bip run_adj_list_bench_bip.cpp) 7 | setup_omp_target(run_adj_list_bench_bip) 8 | 9 | add_metall_executable(run_adj_list_bench_bip_extend run_adj_list_bench_bip_extend.cpp) 10 | setup_omp_target(run_adj_list_bench_bip_extend) 11 | 12 | add_metall_executable(run_adj_list_bench_metall run_adj_list_bench_metall.cpp) 13 | setup_omp_target(run_adj_list_bench_metall) 14 | 15 | add_metall_executable(run_adj_list_bench_reflink_snapshot run_adj_list_bench_reflink_snapshot.cpp) 16 | setup_omp_target(run_adj_list_bench_reflink_snapshot) 17 | 18 | if (MEMKIND_ROOT) 19 | add_metall_executable(run_adj_list_bench_pmem run_adj_list_bench_pmem.cpp) 20 | target_include_directories(run_adj_list_bench_pmem PRIVATE ${MEMKIND_ROOT}/include) 21 | if (UNIX AND NOT APPLE) 22 | find_library(LIBMEMKIND NAMES memkind PATHS ${MEMKIND_ROOT}/lib) 23 | if (LIBMEMKIND) 24 | target_link_libraries(run_adj_list_bench_pmem PRIVATE ${LIBMEMKIND}) 25 | else () 26 | message(FATAL_ERROR "Cannot find memkind library") 27 | endif () 28 | endif () 29 | setup_omp_target(run_adj_list_bench_pmem) 30 | endif () 31 | 32 | configure_file(run_bench.sh run_bench.sh COPYONLY) 33 | 34 | add_subdirectory(edge_generator) 35 | add_subdirectory(test) -------------------------------------------------------------------------------- /bench/adjacency_list/edge_generator/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_metall_executable(generate_rmat_edge_list generate_rmat_edge_list.cpp) 2 | setup_omp_target(generate_rmat_edge_list) -------------------------------------------------------------------------------- /bench/adjacency_list/run_adj_list_bench_stl.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #include 7 | #include 8 | 9 | #include "../data_structure/multithread_adjacency_list.hpp" 10 | #include "bench_driver.hpp" 11 | 12 | using namespace adjacency_list_bench; 13 | 14 | using key_type = uint64_t; 15 | using value_type = uint64_t; 16 | 17 | using allocator_type = std::allocator; 18 | using adjacency_list_type = 19 | data_structure::multithread_adjacency_list; 21 | 22 | int main(int argc, char *argv[]) { 23 | bench_options option; 24 | if (!parse_options(argc, argv, &option)) { 25 | std::abort(); 26 | } 27 | 28 | { 29 | allocator_type allocator; 30 | adjacency_list_type adj_list(allocator); 31 | run_bench(option, &adj_list); 32 | } 33 | 34 | return 0; 35 | } -------------------------------------------------------------------------------- /bench/adjacency_list/test/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_metall_executable(open_metall open_metall.cpp) 2 | add_metall_executable(extend_metall extend_metall.cpp) 3 | add_metall_executable(destroy_metall destroy_metall.cpp) 4 | add_metall_executable(compare_key_value_lists compare_key_value_lists.cpp) 5 | 6 | configure_file(test.sh test.sh COPYONLY) 7 | configure_file(test_large.sh test_large.sh COPYONLY) 8 | 9 | add_subdirectory(data) -------------------------------------------------------------------------------- /bench/adjacency_list/test/compare_key_value_lists.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | /// Compare two key value files 7 | /// Example 8 | /// ./compare_key_value_lists kv_file1 kv_file2 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | using key_type = uint64_t; 18 | using value_type = uint64_t; 19 | using item_type = std::pair; 20 | using table_type = 21 | std::unordered_map>; 22 | 23 | void ingest_item(const std::string& file_name, table_type* table) { 24 | std::ifstream ifs(file_name); 25 | if (!ifs.is_open()) { 26 | std::cerr << "Cannot open " << file_name << std::endl; 27 | std::abort(); 28 | } 29 | 30 | key_type key; 31 | value_type value; 32 | while (ifs >> key >> value) { 33 | const item_type item(key, value); 34 | if (table->count(item) == 0) { 35 | table->emplace(item, 0); 36 | } 37 | ++(table->at(item)); 38 | } 39 | } 40 | 41 | int main(int argc, char* argv[]) { 42 | if (argc != 3) { 43 | std::cerr << "Wrong number of arguments" << std::endl; 44 | std::abort(); 45 | } 46 | 47 | const std::string item_list_file_name1 = argv[1]; 48 | const std::string item_list_file_name2 = argv[2]; 49 | 50 | table_type table1; 51 | ingest_item(item_list_file_name1, &table1); 52 | 53 | table_type table2; 54 | ingest_item(item_list_file_name2, &table2); 55 | 56 | if (table1 != table2) { 57 | std::cerr << "Failed – the two lists are not the same" << std::endl; 58 | std::abort(); 59 | } 60 | 61 | std::cout << "Succeeded!" << std::endl; 62 | 63 | return 0; 64 | } -------------------------------------------------------------------------------- /bench/adjacency_list/test/data/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | configure_file(edge_list_rmat_s10_0_of_4 edge_list_rmat_s10_0_of_4 COPYONLY) 2 | configure_file(edge_list_rmat_s10_1_of_4 edge_list_rmat_s10_1_of_4 COPYONLY) 3 | configure_file(edge_list_rmat_s10_2_of_4 edge_list_rmat_s10_2_of_4 COPYONLY) 4 | configure_file(edge_list_rmat_s10_3_of_4 edge_list_rmat_s10_3_of_4 COPYONLY) -------------------------------------------------------------------------------- /bench/adjacency_list/test/destroy_metall.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | #include 11 | #include "../bench_driver.hpp" 12 | #include "../../data_structure/multithread_adjacency_list.hpp" 13 | 14 | using namespace adjacency_list_bench; 15 | 16 | using key_type = uint64_t; 17 | using value_type = uint64_t; 18 | using adjacency_list_type = data_structure::multithread_adjacency_list< 19 | key_type, value_type, typename metall::manager::allocator_type>; 20 | 21 | int main(int argc, char *argv[]) { 22 | bench_options option; 23 | if (!parse_options(argc, argv, &option)) { 24 | std::abort(); 25 | } 26 | 27 | if (option.datastore_path_list.empty()) { 28 | std::cerr << "Out file name is required" << std::endl; 29 | std::abort(); 30 | } 31 | 32 | { 33 | metall::manager manager(metall::open_only, 34 | option.datastore_path_list[0].c_str()); 35 | manager.destroy(option.adj_list_key_name.c_str()); 36 | 37 | if (!manager.all_memory_deallocated()) { 38 | std::cerr << "Some memory have not been freed." << std::endl; 39 | std::abort(); 40 | } 41 | } 42 | 43 | return 0; 44 | } -------------------------------------------------------------------------------- /bench/adjacency_list/test/extend_metall.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | #include 11 | #include "../bench_driver.hpp" 12 | #include "../../data_structure/multithread_adjacency_list.hpp" 13 | 14 | using namespace adjacency_list_bench; 15 | 16 | using key_type = uint64_t; 17 | using value_type = uint64_t; 18 | using adjacency_list_type = data_structure::multithread_adjacency_list< 19 | key_type, value_type, typename metall::manager::allocator_type>; 20 | 21 | int main(int argc, char *argv[]) { 22 | bench_options option; 23 | if (!parse_options(argc, argv, &option)) { 24 | std::abort(); 25 | } 26 | 27 | if (option.datastore_path_list.empty()) { 28 | std::cerr << "Out file name is required" << std::endl; 29 | std::abort(); 30 | } 31 | 32 | { 33 | metall::manager manager(metall::open_only, 34 | option.datastore_path_list[0].c_str()); 35 | auto ret = 36 | manager.find(option.adj_list_key_name.c_str()); 37 | if (!ret.first) { 38 | std::cerr << "Cannot find an object " << option.adj_list_key_name 39 | << std::endl; 40 | std::abort(); 41 | } 42 | if (ret.second != 1) { 43 | std::cerr << "Its length is not correct" << std::endl; 44 | std::abort(); 45 | } 46 | 47 | adjacency_list_type *const adj_list = ret.first; 48 | 49 | run_bench(option, adj_list); 50 | } 51 | 52 | return 0; 53 | } -------------------------------------------------------------------------------- /bench/adjacency_list/test/open_metall.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | #include 11 | #include "../bench_driver.hpp" 12 | #include "../../data_structure/multithread_adjacency_list.hpp" 13 | 14 | using namespace adjacency_list_bench; 15 | 16 | using key_type = uint64_t; 17 | using value_type = uint64_t; 18 | using adjacency_list_type = data_structure::multithread_adjacency_list< 19 | key_type, value_type, typename metall::manager::allocator_type>; 20 | 21 | int main(int argc, char *argv[]) { 22 | bench_options option; 23 | if (!parse_options(argc, argv, &option)) { 24 | std::abort(); 25 | } 26 | 27 | if (option.datastore_path_list.empty()) { 28 | std::cerr << "Out file name is required" << std::endl; 29 | std::abort(); 30 | } 31 | 32 | { 33 | metall::manager manager(metall::open_read_only, 34 | option.datastore_path_list[0].c_str()); 35 | const auto ret = 36 | manager.find(option.adj_list_key_name.c_str()); 37 | if (!ret.first) { 38 | std::cerr << "Cannot find an object " << option.adj_list_key_name 39 | << std::endl; 40 | std::abort(); 41 | } 42 | if (ret.second != 1) { 43 | std::cerr << "Its length is not correct" << std::endl; 44 | std::abort(); 45 | } 46 | 47 | const adjacency_list_type *const adj_list = ret.first; 48 | if (!option.adj_list_dump_file_name.empty()) { 49 | dump_adj_list(*adj_list, option.adj_list_dump_file_name); 50 | } 51 | } 52 | 53 | return 0; 54 | } -------------------------------------------------------------------------------- /bench/bfs/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | include(setup_omp) 2 | 3 | add_metall_executable(run_bfs_bench_metall run_bfs_bench_metall.cpp) 4 | setup_omp_target(run_bfs_bench_metall) 5 | 6 | add_metall_executable(run_bfs_bench_metall_multiple run_bfs_bench_metall_multiple.cpp) 7 | setup_omp_target(run_bfs_bench_metall_multiple) 8 | 9 | add_metall_executable(run_bfs_bench_bip run_bfs_bench_bip.cpp) 10 | setup_omp_target(run_bfs_bench_bip) 11 | 12 | configure_file(run_bench.sh run_bench.sh COPYONLY) -------------------------------------------------------------------------------- /bench/bfs/run_bfs_bench_bip.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #include 7 | #include 8 | 9 | #include 10 | #include 11 | 12 | #include "../data_structure/multithread_adjacency_list.hpp" 13 | #include "bench_driver.hpp" 14 | 15 | using namespace bfs_bench; 16 | 17 | using vertex_id_type = uint64_t; 18 | 19 | namespace bip = boost::interprocess; 20 | using allocator_type = 21 | bip::allocator; 22 | using adjacency_list_type = 23 | data_structure::multithread_adjacency_list; 25 | 26 | int main(int argc, char *argv[]) { 27 | bench_options option; 28 | if (!parse_options(argc, argv, &option)) { 29 | std::abort(); 30 | } 31 | 32 | { 33 | bip::managed_mapped_file mfile(bip::open_only, 34 | option.graph_file_name_list[0].c_str()); 35 | auto adj_list = 36 | mfile.find(option.graph_key_name.c_str()).first; 37 | 38 | run_bench(*adj_list, option); 39 | } 40 | 41 | return 0; 42 | } -------------------------------------------------------------------------------- /bench/bfs/run_bfs_bench_metall.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | #include 11 | #include "../data_structure/multithread_adjacency_list.hpp" 12 | #include "bench_driver.hpp" 13 | 14 | using namespace bfs_bench; 15 | 16 | using vertex_id_type = uint64_t; 17 | 18 | using adjacency_list_type = data_structure::multithread_adjacency_list< 19 | vertex_id_type, vertex_id_type, 20 | typename metall::manager::allocator_type>; 21 | 22 | int main(int argc, char *argv[]) { 23 | bench_options option; 24 | if (!parse_options(argc, argv, &option)) { 25 | std::abort(); 26 | } 27 | 28 | { 29 | // metall::logger::set_log_level(metall::logger::level::verbose); 30 | 31 | metall::manager manager(metall::open_read_only, 32 | option.graph_file_name_list[0]); 33 | auto adj_list = 34 | manager.find(option.graph_key_name.c_str()).first; 35 | 36 | run_bench(*adj_list, option); 37 | 38 | // std::cout << "Writing profile" << std::endl; 39 | // manager.profile(&(std::cout)); 40 | } 41 | 42 | return 0; 43 | } -------------------------------------------------------------------------------- /bench/bfs/run_bfs_bench_metall_multiple.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | #include 11 | #include "../data_structure/multithread_adjacency_list.hpp" 12 | #include "../data_structure/partitioned_multithread_adjacency_list.hpp" 13 | #include "bench_driver.hpp" 14 | 15 | using namespace bfs_bench; 16 | using namespace data_structure; 17 | 18 | using vertex_id_type = uint64_t; 19 | 20 | using local_adjacency_list_type = multithread_adjacency_list< 21 | vertex_id_type, vertex_id_type, 22 | typename metall::manager::allocator_type>; 23 | using adjacency_list_type = 24 | partitioned_multithread_adjacency_list; 25 | 26 | int main(int argc, char *argv[]) { 27 | bench_options option; 28 | if (!parse_options(argc, argv, &option)) { 29 | std::abort(); 30 | } 31 | 32 | { 33 | std::vector managers; 34 | for (const auto &file_name : option.graph_file_name_list) { 35 | managers.emplace_back( 36 | new metall::manager(metall::open_read_only, file_name)); 37 | } 38 | 39 | auto adj_list = adjacency_list_type(option.graph_key_name, managers.begin(), 40 | managers.end()); 41 | 42 | run_bench(adj_list, option); 43 | } 44 | 45 | return 0; 46 | } -------------------------------------------------------------------------------- /bench/container/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_metall_executable(run_vector_bench run_vector_bench.cpp) 2 | add_metall_executable(run_map_bench run_map_bench.cpp) 3 | add_metall_executable(run_unordered_map_bench run_unordered_map_bench.cpp) 4 | -------------------------------------------------------------------------------- /bench/container/run_map_bench.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | /// \brief Benchmarks the STL map container using different allocators. 7 | /// Usage: 8 | /// ./run_map_bench 9 | /// # modify the values in the main(), if needed. 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | #include "bench_common.hpp" 19 | 20 | int main() { 21 | std::size_t scale = 17; 22 | std::size_t num_inputs = (1ULL << scale) * 16; 23 | std::vector> inputs; 24 | 25 | // gen_edges(scale, num_inputs, inputs); 26 | gen_random_values(num_inputs, inputs); 27 | std::cout << "Generated inputs\t" << inputs.size() << std::endl; 28 | 29 | { 30 | std::map map; 31 | 32 | const auto start = mdtl::elapsed_time_sec(); 33 | for (const auto &kv : inputs) { 34 | map[kv.first]; 35 | map[kv.second]; 36 | } 37 | const auto elapsed_time = mdtl::elapsed_time_sec(start); 38 | std::cout << "map took (s)\t" << elapsed_time << std::endl; 39 | } 40 | 41 | { 42 | boost::container::map map; 43 | 44 | const auto start = mdtl::elapsed_time_sec(); 45 | for (const auto &kv : inputs) { 46 | map[kv.first]; 47 | map[kv.second]; 48 | } 49 | const auto elapsed_time = mdtl::elapsed_time_sec(start); 50 | std::cout << "Boost map took (s)\t" << elapsed_time << std::endl; 51 | } 52 | 53 | { 54 | metall::manager mngr(metall::create_only, "/tmp/metall"); 55 | metall::container::map map(mngr.get_allocator()); 56 | 57 | const auto start = mdtl::elapsed_time_sec(); 58 | for (const auto &kv : inputs) { 59 | map[kv.first]; 60 | map[kv.second]; 61 | } 62 | const auto elapsed_time = mdtl::elapsed_time_sec(start); 63 | std::cout << "Boost map with Metall took (s)\t" << elapsed_time 64 | << std::endl; 65 | } 66 | 67 | return 0; 68 | } -------------------------------------------------------------------------------- /bench/container/run_unordered_map_bench.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | /// \brief Benchmarks the STL map container using different allocators. 7 | /// Usage: 8 | /// ./run_map_bench 9 | /// # modify the values in the main(), if needed. 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | #include "bench_common.hpp" 19 | 20 | int main() { 21 | std::size_t scale = 17; 22 | std::size_t num_inputs = (1ULL << scale) * 16; 23 | std::vector> inputs; 24 | 25 | // gen_edges(scale, num_inputs, inputs); 26 | gen_random_values(num_inputs, inputs); 27 | std::cout << "Generated inputs\t" << inputs.size() << std::endl; 28 | 29 | { 30 | std::unordered_map map; 31 | 32 | const auto start = mdtl::elapsed_time_sec(); 33 | for (const auto &kv : inputs) { 34 | map[kv.first]; 35 | map[kv.second]; 36 | } 37 | const auto elapsed_time = mdtl::elapsed_time_sec(start); 38 | std::cout << "unordered_map took (s)\t" << elapsed_time << std::endl; 39 | } 40 | 41 | { 42 | boost::unordered_map map; 43 | 44 | const auto start = mdtl::elapsed_time_sec(); 45 | for (const auto &kv : inputs) { 46 | map[kv.first]; 47 | map[kv.second]; 48 | } 49 | const auto elapsed_time = mdtl::elapsed_time_sec(start); 50 | std::cout << "Boost unordered_map took (s)\t" << elapsed_time << std::endl; 51 | } 52 | 53 | { 54 | metall::manager mngr(metall::create_only, "/tmp/metall"); 55 | metall::container::unordered_map map( 56 | mngr.get_allocator()); 57 | 58 | const auto start = mdtl::elapsed_time_sec(); 59 | for (const auto &kv : inputs) { 60 | map[kv.first]; 61 | map[kv.second]; 62 | } 63 | const auto elapsed_time = mdtl::elapsed_time_sec(start); 64 | std::cout << "Boost unordered_map with Metall took (s)\t" << elapsed_time 65 | << std::endl; 66 | } 67 | 68 | return 0; 69 | } -------------------------------------------------------------------------------- /bench/mapping/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_metall_executable(run_mapping_bench run_mapping_bench.cpp) -------------------------------------------------------------------------------- /bench/offset_ptr/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_metall_executable(run_offset_ptr_bench run_offset_ptr_bench.cpp) -------------------------------------------------------------------------------- /bench/offset_ptr/run_offset_ptr_bench.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | /// \brief Benchmarks the offset_ptr. 7 | /// Usage: 8 | /// ./run_offset_ptr_bench 9 | 10 | #include 11 | #include 12 | #include 13 | 14 | namespace mdtl = metall::mtlldetail; 15 | 16 | int main() { 17 | const std::size_t length = 1ULL << 22; 18 | auto *array = new uint64_t[length]; 19 | 20 | for (std::size_t i = 0; i < length; ++i) { 21 | array[i] = i; 22 | } 23 | std::cout << "Initialized array, length = " << length << std::endl; 24 | 25 | { 26 | auto *ptr = array; 27 | const auto start = mdtl::elapsed_time_sec(); 28 | for (std::size_t i = 0; i < length; ++i) { 29 | [[maybe_unused]] volatile const uint64_t x = *ptr; 30 | ++ptr; 31 | } 32 | const auto elapsed_time = mdtl::elapsed_time_sec(start); 33 | std::cout << "Raw pointer took (s)\t" << elapsed_time << std::endl; 34 | } 35 | 36 | { 37 | auto ofset_ptr = metall::offset_ptr(array); 38 | const auto start = mdtl::elapsed_time_sec(); 39 | for (std::size_t i = 0; i < length; ++i) { 40 | [[maybe_unused]] volatile const uint64_t x = *ofset_ptr; 41 | ++ofset_ptr; 42 | } 43 | const auto elapsed_time = mdtl::elapsed_time_sec(start); 44 | std::cout << "Offset pointer took (s)\t" << elapsed_time << std::endl; 45 | } 46 | 47 | return 0; 48 | } -------------------------------------------------------------------------------- /bench/rand_engine/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_metall_executable(run_rand_engine_bench run_rand_engine_bench.cpp) -------------------------------------------------------------------------------- /bench/rand_engine/run_rand_engine_bench.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | template 13 | auto run_bench(const uint64_t num_generate) { 14 | rand_engine_type rand_engine(123); 15 | const auto s = metall::mtlldetail::elapsed_time_sec(); 16 | for (uint64_t i = 0; i < num_generate; ++i) { 17 | [[maybe_unused]] volatile const uint64_t x = rand_engine(); 18 | } 19 | const auto t = metall::mtlldetail::elapsed_time_sec(s); 20 | return t; 21 | } 22 | 23 | int main() { 24 | const uint64_t num_generate = (1ULL << 20ULL); 25 | std::cout << "Generate " << num_generate << " values" << std::endl; 26 | 27 | std::cout << "std::mt19937_64 \t" << run_bench(num_generate) 28 | << std::endl; 29 | std::cout << "boost::mt19937_64\t" 30 | << run_bench(num_generate) << std::endl; 31 | std::cout << "xoshiro512++ \t" 32 | << run_bench(num_generate) << std::endl; 33 | std::cout << "xoshiro1024++ \t" 34 | << run_bench(num_generate) << std::endl; 35 | 36 | return 0; 37 | } 38 | -------------------------------------------------------------------------------- /bench/simple_alloc/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_metall_executable(run_simple_allocation_bench_stl run_simple_allocation_bench_stl.cpp) 2 | add_metall_executable(run_simple_allocation_bench_metall run_simple_allocation_bench_metall.cpp) 3 | add_metall_executable(run_simple_allocation_bench_bip run_simple_allocation_bench_bip.cpp) 4 | configure_file(run_bench.sh run_bench.sh COPYONLY) -------------------------------------------------------------------------------- /bench/simple_alloc/run_bench.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | NUM_ALLOCS=1000000 4 | FILE="/tmp/segment" 5 | LOG_FILE_PREFIX="out_simple_allocation_bench_" 6 | 7 | rm -rf ${FILE}* 8 | ./run_simple_allocation_bench_stl -n ${NUM_ALLOCS} | tee ${LOG_FILE_PREFIX}"stl.log" 9 | 10 | rm -rf ${FILE}* 11 | ./run_simple_allocation_bench_bip -n ${NUM_ALLOCS} -o ${FILE} | tee ${LOG_FILE_PREFIX}"bip.log" 12 | 13 | rm -rf ${FILE}* 14 | ./run_simple_allocation_bench_metall -n ${NUM_ALLOCS} -o ${FILE} | tee ${LOG_FILE_PREFIX}"metall.log" -------------------------------------------------------------------------------- /bench/simple_alloc/run_simple_allocation_bench_bip.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #include 7 | #include 8 | #include "kernel.hpp" 9 | 10 | namespace bip = boost::interprocess; 11 | 12 | int main(int argc, char *argv[]) { 13 | const auto option = simple_alloc_bench::parse_option(argc, argv); 14 | 15 | bip::file_mapping::remove(option.datastore_path.c_str()); 16 | 17 | const std::size_t max_alloc_size = 18 | *(std::max_element(option.size_list.begin(), option.size_list.end())) * 19 | option.num_allocations; 20 | bip::managed_mapped_file mfile( 21 | bip::create_only, option.datastore_path.c_str(), max_alloc_size * 2); 22 | 23 | simple_alloc_bench::run_bench(simple_alloc_bench::option_type{}, 24 | mfile.get_allocator()); 25 | 26 | bip::file_mapping::remove(option.datastore_path.c_str()); 27 | 28 | return 0; 29 | } -------------------------------------------------------------------------------- /bench/simple_alloc/run_simple_allocation_bench_metall.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | #include 11 | #include "kernel.hpp" 12 | 13 | int main(int argc, char *argv[]) { 14 | const auto option = simple_alloc_bench::parse_option(argc, argv); 15 | { 16 | metall::manager manager(metall::create_only, option.datastore_path); 17 | simple_alloc_bench::run_bench(option, manager.get_allocator()); 18 | } 19 | metall::manager::remove(option.datastore_path.c_str()); 20 | 21 | return 0; 22 | } -------------------------------------------------------------------------------- /bench/simple_alloc/run_simple_allocation_bench_stl.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #include 7 | #include 8 | 9 | #include "kernel.hpp" 10 | 11 | int main(int argc, char *argv[]) { 12 | const auto option = simple_alloc_bench::parse_option(argc, argv); 13 | 14 | simple_alloc_bench::run_bench(option, std::allocator()); 15 | 16 | return 0; 17 | } -------------------------------------------------------------------------------- /cmake/MetallConfig.cmake.in: -------------------------------------------------------------------------------- 1 | @PACKAGE_INIT@ 2 | 3 | include("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@Targets.cmake") 4 | check_required_components("@PROJECT_NAME@") -------------------------------------------------------------------------------- /cmake/build_doc.cmake: -------------------------------------------------------------------------------- 1 | function(build_doc) 2 | find_package(Doxygen) 3 | if (DOXYGEN_FOUND) 4 | set(DOXYGEN_IN ${CMAKE_CURRENT_SOURCE_DIR}/docs/Doxyfile.in) 5 | set(DOXYGEN_OUT ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile) 6 | 7 | configure_file(${DOXYGEN_IN} ${DOXYGEN_OUT} @ONLY) 8 | 9 | add_custom_target(build_doc ALL 10 | COMMAND ${DOXYGEN_EXECUTABLE} ${DOXYGEN_OUT} 11 | WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} 12 | COMMENT "Generating API documentation with Doxygen" 13 | VERBATIM) 14 | else () 15 | message(FATAL_ERROR "Can not find Doxygen") 16 | endif () 17 | endfunction() -------------------------------------------------------------------------------- /cmake/check_cxx_filesystem_library.cmake: -------------------------------------------------------------------------------- 1 | # Checks if the C++17 library is available. 2 | function(check_cxx_filesystem_library) 3 | 4 | set(FOUND_CXX17_FILESYSTEM_LIB FALSE PARENT_SCOPE) 5 | set(REQUIRE_LIB_STDCXX_FS FALSE PARENT_SCOPE) 6 | 7 | if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") # Clang or AppleClang 8 | if (${CMAKE_HOST_SYSTEM_NAME} MATCHES "Darwin") # macOS 9 | include(get_macos_version) 10 | get_macos_version() # Get macOS version 11 | message(VERBOSE "Detected macOS version ${MACOS_VERSION}") 12 | if (MACOS_VERSION VERSION_LESS 10.15) # macOS < 10.15 13 | message(FATAL_ERROR "macOS >= 10.15 is required to use the C++17 library.") 14 | endif () 15 | endif () 16 | endif () 17 | endfunction() -------------------------------------------------------------------------------- /cmake/setup_mpi.cmake: -------------------------------------------------------------------------------- 1 | if (NOT MPI_CXX_FOUND) 2 | find_package(MPI) 3 | endif () 4 | 5 | function(setup_mpi_target target) 6 | if (MPI_CXX_FOUND) 7 | target_link_libraries(${target} PRIVATE MPI::MPI_CXX) 8 | if (${CMAKE_HOST_SYSTEM_NAME} MATCHES "Linux") 9 | target_link_libraries(${target} PRIVATE rt) 10 | endif () 11 | else () 12 | message(SEND_ERROR "MPI not found, ${target} will not be built") 13 | endif () 14 | endfunction() -------------------------------------------------------------------------------- /cmake/setup_omp.cmake: -------------------------------------------------------------------------------- 1 | if (NOT OpenMP_CXX_FOUND) 2 | find_package(OpenMP) 3 | endif () 4 | 5 | function(setup_omp_target target) 6 | if (OpenMP_CXX_FOUND) 7 | target_link_libraries(${target} PUBLIC OpenMP::OpenMP_CXX) 8 | else () 9 | message(WARNING "OpenMP not found, ${target} will not be built with OpenMP support") 10 | endif () 11 | endfunction() -------------------------------------------------------------------------------- /docs/publications/metall_101.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/metall/dbb6e981c1f84d6979b2c28cf3b47779ab9ce779/docs/publications/metall_101.pdf -------------------------------------------------------------------------------- /docs/readthedocs/about/license_notice.md: -------------------------------------------------------------------------------- 1 | ## License 2 | 3 | Metall is distributed under the terms of both the MIT license and the Apache License (Version 2.0). 4 | Users may choose either license, at their option. 5 | 6 | All new contributions must be made under both the MIT and Apache-2.0 licenses. 7 | 8 | See [LICENSE-MIT](https://github.com/LLNL/metall/blob/develop/LICENSE-MIT), 9 | [LICENSE-APACHE](https://github.com/LLNL/metall/blob/develop/LICENSE-APACHE), 10 | [NOTICE](https://github.com/LLNL/metall/blob/develop/NOTICE), 11 | and [COPYRIGHT](https://github.com/LLNL/metall/blob/develop/COPYRIGHT) for details. 12 | 13 | SPDX-License-Identifier: (Apache-2.0 OR MIT) 14 | 15 | 16 | ## Release 17 | 18 | LLNL-CODE-768617 -------------------------------------------------------------------------------- /docs/readthedocs/about/publication.md: -------------------------------------------------------------------------------- 1 | # Publication 2 | 3 | ## Metall: A persistent memory allocator for data-centric analytics 4 | 5 | ``` 6 | Keita Iwabuchi, Karim Youssef, Kaushik Velusamy, Maya Gokhale, Roger Pearce, 7 | Metall: A persistent memory allocator for data-centric analytics, 8 | Parallel Computing, 2022, 102905, ISSN 0167-8191, https://doi.org/10.1016/j.parco.2022.102905. 9 | ``` 10 | 11 | - [Parallel Computing](https://www.sciencedirect.com/science/article/abs/pii/S0167819122000114) (journal) 12 | 13 | - [arXiv](https://arxiv.org/abs/2108.07223) (preprint) 14 | 15 | ## Metall: A Persistent Memory Allocator Enabling Graph Processing 16 | 17 | ```text 18 | K. Iwabuchi, L. Lebanoff, M. Gokhale and R. Pearce, 19 | "Metall: A Persistent Memory Allocator Enabling Graph Processing," 20 | 2019 IEEE/ACM 9th Workshop on Irregular Applications: Architectures and Algorithms (IA3), 2019, 21 | pp. 39-44, doi: 10.1109/IA349570.2019.00012. 22 | ``` 23 | 24 | - [Available PDF](https://www.osti.gov/servlets/purl/1576900) 25 | 26 | - A page in IEEE Xplore is [here](https://ieeexplore.ieee.org/document/8945094) 27 | -------------------------------------------------------------------------------- /docs/readthedocs/advanced_build/test.md: -------------------------------------------------------------------------------- 1 | # Test Metall for Development 2 | 3 | There are two types of tests programs in Metall. 4 | 5 | 6 | ## Google Test 7 | 8 | 9 | ### Build 10 | 11 | To run tests using Google Test, 12 | please see [this page](./cmake.md) about building the test programs. 13 | 14 | 15 | ### Run 16 | 17 | ```bash 18 | make test 19 | ``` 20 | 21 | The test programs make data stores into `/tmp` by default. 22 | 23 | To change the location, use an environmental value 'METALL_TEST_DIR'. 24 | 25 | ```bash 26 | env METALL_TEST_DIR="/mnt/ssd/" make test 27 | ``` 28 | 29 | 30 | ## Manual Test 31 | There is another test program in Metall's repository. 32 | The test program uses [Adjacency List Benchmark](https://github.com/LLNL/metall/tree/master/bench/adjacency_list/). 33 | One program creates a graph data and another program opens it. 34 | This test is useful to make sure that Metall can store data persistently. 35 | 36 | 37 | Here is how to run the test with small data. 38 | ```bash 39 | cd metall/build/bench/adjacency_list/ 40 | sh ../../../bench/adjacency_list/test/test.sh -d/path/to/store/data/store 41 | ``` 42 | 43 | 44 | Here is how to run the test with large data. 45 | ```bash 46 | cd metall/build/bench/adjacency_list/ 47 | sh ../../../bench/adjacency_list/test/test_large.sh -d/path/to/store/data/store -v17 48 | ``` 49 | 50 | In test_large.sh, the input data is generated on the fly using an R-MAT graph generator. 51 | `-vN` option controls the size of the graph to generate, where N is a int number called SCALE. 52 | The number of vertices in a generated R-MAT graph is `2^N` and that of edges (undirected) is `16 x 2^N`. -------------------------------------------------------------------------------- /docs/readthedocs/advanced_build/umap.md: -------------------------------------------------------------------------------- 1 | ## Use Umap in Metall 2 | 3 | Metall supports [Umap](https://github.com/LLNL/umap) as a replacement for the system mmap. 4 | 5 | Link Umap library and include its header files and define ```METALL_USE_UMAP``` at compile. 6 | An actual build command would be something like: 7 | 8 | ```bash 9 | g++ -std=c++17 -lstdc++fs your_program.cpp \ 10 | -I/path/to/metall/include -I/path/to/boost/include \ 11 | -I/path/to/umap/include -L/path/to/umap/lib -lumap -DMETALL_USE_UMAP 12 | ``` 13 | 14 | In addition, 15 | our CMake file has an option `UMAP_ROOT=/umap/install/root/path` to use Umap instead of the system mmap. 16 | 17 | ### UMap Store Handler Configuration 18 | UMap uses a sparse multi-file backing store handler that partitions the memory-mapped persistent region into multiple files with a configurable file granularity. 19 | The file granularity can be configured from Metall's side by setting the ```SPARSE_STORE_FILE_GRANULARITY``` environmemnt variable as follows: 20 | ```bash 21 | export SPARSE_STORE_FILE_GRANULARITY= 22 | ``` 23 | If the file granularity is not specified, Metall uses a default granularity of 1GB. 24 | 25 | ### Restriction with Umap 26 | 27 | Metall cannot free DRAM space (page buffer managed by Umap) and backing file space since Umap does not have such capabilities. 28 | -------------------------------------------------------------------------------- /docs/readthedocs/basics/compile_time_options.md: -------------------------------------------------------------------------------- 1 | # Compile-Time Options 2 | 3 | There are some compile-time options (C/C++ macro) to configure the behavior of Metall. 4 | Those macros are defined in `metall/include/metall/defs.hpp`. 5 | -------------------------------------------------------------------------------- /docs/readthedocs/css/custom.css: -------------------------------------------------------------------------------- 1 | body { 2 | font-family: "Hiragino Sans"; 3 | } 4 | 5 | img[src$="metall_overview.png"] { 6 | display: block; 7 | margin-left: auto; 8 | margin-right: auto; 9 | max-width: 70%; 10 | height: auto; 11 | } 12 | -------------------------------------------------------------------------------- /docs/readthedocs/detail/example.md: -------------------------------------------------------------------------------- 1 | ## Example 2 | 3 | Example programs are located in [example](https://github.com/LLNL/metall/tree/master/example) 4 | 5 | To build the examples see [build source files in Metall](../advanced_build/cmake.md) -------------------------------------------------------------------------------- /docs/readthedocs/detail/introduction.md: -------------------------------------------------------------------------------- 1 | ## Background 2 | 3 | Data-intensive applications play an essential role across many real-world data science domains. 4 | Often, these applications require storing data beyond a single process lifetime. 5 | Data often requires transformation into analytic-specific data structures to perform the analytic with reasonable execution time. 6 | The task of ingesting data, indexing and partitioning data in preparation of running an analytic, is often more expensive than the analytic itself. 7 | 8 | The promise of persistent memory is that, once constructed, data structures can be re-analyzed and updated beyond the lifetime of a single execution, and new forms of persistent memory are increasing the viability of processing complex data analytics. 9 | 10 | 11 | ## Metall 12 | 13 | * Enables applications to allocate heap-based objects into both block-storage and byte-addressable persistent memories, just like main-memory 14 | * e.g., NVMe SSD and Intel Optane DC Persistent Memory 15 | 16 | * Leverages a memory-mapped file mechanism (mmap) 17 | * Applications can access mapped area as if it were regular memory 18 | * mmap can map files bigger than the DRAM capacity 19 | 20 | * Incorporates the state-of-the-art allocation algorithms 21 | * Some key ideas from [SuperMalloc](https://dl.acm.org/doi/10.1145/2887746.2754178) 22 | 23 | * Provides the API developed by Boost.Interprocess 24 | * Boost.Interprocess is an interprocess communication library 25 | * Useful for allocating C++ custom data structures in persistent memory 26 | 27 | * Employs a coarse-grained consistency model, allowing the application to determine when it is appropriate to create durable snapshots of the persistent heap -------------------------------------------------------------------------------- /docs/readthedocs/detail/ls.md: -------------------------------------------------------------------------------- 1 | ## Metall datastore 'ls' 2 | 3 | `datastore_ls` is a utility program that lists object attributes in a Metall data store. 4 | 5 | `mpi_datastore_ls` is available for Metall MPI data store, 6 | which is created by [Metall MPI Adaptor](https://github.com/LLNL/metall/blob/master/include/metall/utility/metall_mpi_adaptor.hpp). 7 | `mpi_datastore_ls` is not a MPI program. 8 | 9 | ### Synopsis 10 | ```c++ 11 | datastore_ls [/path/to/datastore] 12 | mpi_datastore_ls [/path/to/datastore] [MPI rank number] 13 | ``` 14 | 15 | ### Example 16 | ```bash 17 | $ cmake [option]... 18 | $ make datastore_ls 19 | $ make install 20 | $/install/path/bin/datastore_ls /path/to/metall/datastore 21 | [Named Object] 22 | | Name | Length | Offset | Type-ID | Description | 23 | ---------------------------------------------------------------------------- 24 | | obj | 1 | 0 | 6253375586064260614 | description example | 25 | 26 | [Unique Object] 27 | | Name: typeid(T).name() | Length | Offset | Type-ID | Description | 28 | -------------------------------------------------------------------------------------- 29 | | c | 1 | 8 | 10959529184379665549 | | 30 | | St6vectorIiSaIiEE | 1 | 4194304 | 11508737342576383696 | | 31 | 32 | [Anonymous Object] 33 | | Length | Offset | Type-ID | Description | 34 | ----------------------------------------------------------- 35 | | 1 | 16 | 6253375586064260614 | | 36 | | 100 | 6291456 | 6253375586064260614 | | 37 | ``` -------------------------------------------------------------------------------- /docs/readthedocs/detail/persistence_policy.md: -------------------------------------------------------------------------------- 1 | ## Coarse-grained Persistence Policy vs Fine-grained Persistence Policy 2 | 3 | ### Coarse-grained Persistence Policy 4 | 5 | Metall employs snapshot consistency, an explicit coarse-grained persistence policy in which persistence is guaranteed only when the heap is saved in a *snapshot* to the backing store (backing files). 6 | 7 | The snapshot is created when the destructor or a snapshot method in Metall is invoked. 8 | Those methods flush the application data and the internal management data in Metall to the backing store. 9 | 10 | If an application crashes before Metall's destructor finishes successfully, 11 | there is a possibility of inconsistency between the memory mapping and the backing files. 12 | 13 | To protect application data from this hazard, 14 | the application must duplicate the backing files 15 | before reattaching the data either through the snapshot method or else using a copy command in the system. 16 | 17 | 18 | 19 | ### Fine-grained Persistence Policy 20 | 21 | In contrast, [libpmemobj](https://pmem.io/pmdk/libpmemobj/) in the [Persistent Memory Development Kit (PMDK)](https://pmem.io/pmdk/) builds on [Direct Access (DAX)](https://www.kernel.org/doc/Documentation/filesystems/dax.txt) and is designed to provide fine-grained persistence. 22 | 23 | Fine-grained persistence is highly useful (or almost necessary) to implement transactional object stores, leveraging new byte-addressable persistent memory fully, e.g., Intel Optane DC Persistent Memory. 24 | 25 | However, fine-grained persistence requires fine-grained cache-line flushes to the persistent media, which can incur an unnecessary overhead for applications that do not require such fine-grained consistency. 26 | It is also not possible to efficiently support such fine-grained consistency on more traditional NVMe devices. 27 | -------------------------------------------------------------------------------- /docs/readthedocs/detail/pointer.md: -------------------------------------------------------------------------------- 1 | ## Offset Pointer 2 | 3 | Applications have to take care of some restrictions regarding pointers to store objects in persistent memory. 4 | Applications cannot use raw pointers for data members in data structures stored in persistent memory 5 | because there is no guarantee that backing-files are mapped to the same virtual memory addresses every time. 6 | 7 | To fix the problem, the **offset pointer** has to be used instead of the raw pointer. 8 | An offset pointer holds an offset between the address pointing at and itself so that it can always point to the same location regardless of the VM address it is mapped. 9 | Additionally, references, virtual functions, and virtual base classes have to be removed since those mechanisms also use raw pointers internally. 10 | 11 | The [offset pointer in Metall](https://github.com/KIwabuchi/metall/blob/develop/include/metall/offset_ptr.hpp) is just an alias of [offset pointer in Boost.Interprocess](https://www.boost.org/doc/libs/release/doc/html/interprocess/offset_ptr.html). 12 | 13 | 14 | ## STL Container 15 | 16 | Unfortunately, some implementations of the STL container do not work with Boost.Interprocess and Metall due to offset pointer and some other reasons ([see detail](https://www.boost.org/doc/libs/release/doc/html/interprocess/allocators_containers.html#interprocess.allocators_containers.containers_explained.stl_container_requirements)). 17 | We recommend applications use containers in [Boost.Container](https://www.boost.org/doc/libs/release/doc/html/container.html). 18 | 19 | -------------------------------------------------------------------------------- /docs/readthedocs/detail/snapshot.md: -------------------------------------------------------------------------------- 1 | ## Efficient Snapshot 2 | 3 | In addition to the allocation APIs, Metall provides a snapshot feature that stores only the difference from the previous snapshot point instead of duplicating the entire persistent heap by leveraging [reflink](http://man7.org/linux/man-pages/man2/ioctl_ficlonerange.2.html). 4 | 5 | With reflink, a copied file shares the same data blocks with the existing file; 6 | data blocks are copied only when they are modified (*copy-on-write*). 7 | 8 | As reflink is relatively new, not all filesystems support it. 9 | Those that do include XFS, ZFS, Btrfs, and Apple File System (APFS) — we expect that more filesystems will implement this feature in the future. 10 | 11 | In case reflink is not supported by the underlying filesystem, 12 | Metall automatically falls back to a regular copy. -------------------------------------------------------------------------------- /docs/readthedocs/external/list.md: -------------------------------------------------------------------------------- 1 | # Open Source Projects Using Metall 2 | 3 | ## Collaboration Work with the ECP ExaGraph Project 4 | ### miniVite 5 | 6 | miniVite is a proxy app that implements a single phase of Louvain method in distributed memory for graph community detection. 7 | 8 | miniVite has a mode that uses Metall to store a graph in persistent memory to reuse the data and reduce the overall analytics workload. 9 | 10 | For building and running miniVite with Metall, 11 | see the details located [here](https://github.com/ECP-ExaGraph/miniVite/tree/metallds2#minivite--metall-and-umap). 12 | 13 | ### Ripples 14 | 15 | Ripples is a software framework to study the Influence Maximization problem developed at Pacific Northwest National Laboratory. 16 | 17 | To build Ripples with Metall, see the details located [here](./ripples.md). 18 | 19 | ## HavoqGT 20 | 21 | [HavoqGT](https://github.com/LLNL/havoqgt) (Highly Asynchronous Visitor Queue Graph Toolkit) is a framework for expressing asynchronous vertex-centric graph algorithms. 22 | 23 | Same as MiniVite, HavoqGT uses Metall to store a graph in persistent memory to reuse the data and reduce the overall analytics workload. 24 | 25 | ## saltatlas (DNND) 26 | 27 | [saltatlas](https://github.com/LLNL/saltatlas) is a distributed approximate k-nearest neighbors framework. 28 | saltatlas contains a distributed NNDescent algorithm implementation (DNND). 29 | DNND is designed to work with Metall to store its main data structure, which requires a heavy construction time, in persistent memory to avoid redundant data ingestion tasks. 30 | 31 | To use saltatlas DNND with Metall, see its [README](https://github.com/LLNL/saltatlas). 32 | -------------------------------------------------------------------------------- /docs/readthedocs/img/metall_architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/metall/dbb6e981c1f84d6979b2c28cf3b47779ab9ce779/docs/readthedocs/img/metall_architecture.png -------------------------------------------------------------------------------- /docs/readthedocs/img/metall_overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/metall/dbb6e981c1f84d6979b2c28cf3b47779ab9ce779/docs/readthedocs/img/metall_overview.png -------------------------------------------------------------------------------- /docs/readthedocs/index.md: -------------------------------------------------------------------------------- 1 | # Metall: Persistent Memory Allocator for Data-Centric Analytics 2 | 3 | This Read the Docs page describes Metall (open-source library available [here]((https://github.com/LLNL/metall))). 4 | 5 | ## Overview 6 | 7 | Metall is a persistent memory allocator designed to provide developers with an API to allocate custom C++ data structures in both block-storage and 8 | byte-addressable persistent memories (e.g., NVMe SSD and Intel Optane DC Persistent Memory). 9 | 10 | Metall relies on a memory-mapped file mechanism ([mmap](http://man7.org/linux/man-pages/man2/mmap.2.html)), 11 | to map a file in a filesystem into the virtual memory of an application, 12 | allowing the application to access the mapped region as if it were regular memory. 13 | 14 | Such mapped area can be larger than the physical main-memory of the system, allowing applications 15 | to address datasets larger than physical memory (often referred to as out-of-core or external memory). 16 | 17 | Metall incorporates state-of-the-art allocation algorithms in [SuperMalloc](https://dl.acm.org/doi/10.1145/2887746.2754178) with 18 | the rich C++ interface developed by [Boost.Interprocess](https://www.boost.org/doc/libs/release/doc/html/interprocess.html), 19 | and provides persistent memory snapshotting (versioning) capabilities. 20 | 21 | Example programs that use Metall are listed [here](detail/example.md). 22 | 23 | ![Metall Overview](./img/metall_overview.png) 24 | -------------------------------------------------------------------------------- /example/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_metall_executable(simple simple.cpp) 2 | 3 | add_metall_executable(vector_of_vectors vector_of_vectors.cpp) 4 | 5 | add_metall_executable(string string.cpp) 6 | 7 | add_metall_executable(string_map string_map.cpp) 8 | 9 | add_metall_executable(complex_map complex_map.cpp) 10 | 11 | add_metall_executable(offset_pointer offset_pointer.cpp) 12 | 13 | add_metall_executable(snapshot snapshot.cpp) 14 | 15 | add_metall_executable(csr_graph csr_graph.cpp) 16 | 17 | add_metall_executable(fallback_allocator fallback_allocator.cpp) 18 | 19 | add_metall_executable(datastore_description datastore_description.cpp) 20 | 21 | add_metall_executable(object_attribute object_attribute.cpp) 22 | 23 | add_metall_executable(object_attribute_api_list object_attribute_api_list.cpp) 24 | 25 | add_metall_executable(multilevel_containers multilevel_containers.cpp) 26 | 27 | add_metall_executable(adjacency_list_graph adjacency_list_graph.cpp) 28 | 29 | add_metall_executable(static_mutex static_mutex.cpp) 30 | 31 | add_metall_executable(concurrent_map concurrent_map.cpp) 32 | 33 | add_metall_executable(metall_containers metall_containers.cpp) 34 | 35 | add_metall_executable(allocator_aware_type allocator_aware_type.cpp) 36 | 37 | add_metall_executable(logger logger.cpp) 38 | 39 | add_metall_executable(concurrent concurrent.cpp) 40 | 41 | add_metall_executable(custom_logger custom_logger.cpp) 42 | target_compile_definitions(custom_logger PRIVATE METALL_LOGGER_EXTERN_C=1) 43 | 44 | if (BUILD_C) 45 | add_c_executable(c_api c_api.c) 46 | target_link_libraries(c_api PRIVATE metall_c) 47 | endif() 48 | 49 | include(setup_mpi) 50 | if (MPI_CXX_FOUND) 51 | add_metall_executable(mpi_create mpi_create.cpp) 52 | setup_mpi_target(mpi_create) 53 | 54 | add_metall_executable(mpi_open mpi_open.cpp) 55 | setup_mpi_target(mpi_open) 56 | else() 57 | message(STATUS "Will skip building the MPI examples") 58 | endif() 59 | 60 | add_subdirectory(json) 61 | add_subdirectory(container) -------------------------------------------------------------------------------- /example/adjacency_list_graph.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #include 7 | 8 | #include 9 | #include "graph_data_structure/adjacency_list.hpp" 10 | 11 | using vid_t = uint64_t; 12 | using adj_list_graph_t = 13 | adjacency_list>; 14 | 15 | int main() { 16 | { 17 | // Create a new Metall datastore in "/tmp/dir" 18 | metall::manager manager(metall::create_only, "/tmp/dir"); 19 | 20 | // Allocate and construct an object in the persistent memory with a name 21 | // "adj_list_graph" 22 | adj_list_graph_t *adj_list_graph = 23 | manager.construct("adj_list_graph")( 24 | manager.get_allocator()); // Arguments to the constructor 25 | 26 | // Add an edge (1, 2) 27 | adj_list_graph->add_edge(1, 2); 28 | } 29 | 30 | { 31 | // Open the existing Metall datastore in "/tmp/dir" 32 | metall::manager manager(metall::open_only, "/tmp/dir"); 33 | 34 | adj_list_graph_t *adj_list_graph = 35 | manager.find("adj_list_graph").first; 36 | 37 | // Add another edge 38 | adj_list_graph->add_edge(1, 3); 39 | 40 | // Print the edges of vertex 1 41 | for (auto edge = adj_list_graph->edges_begin(1); 42 | edge != adj_list_graph->edges_end(1); ++edge) { 43 | std::cout << "1 " << *edge << std::endl; 44 | } 45 | } 46 | 47 | return 0; 48 | } -------------------------------------------------------------------------------- /example/c_api.c: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | int main(void) { 11 | // Basic allocation 12 | { 13 | metall_manager* manager = metall_create("/tmp/metall1"); 14 | 15 | uint64_t* x = metall_malloc(manager, sizeof(uint64_t)); 16 | x[0] = 1; 17 | 18 | metall_free(manager, x); 19 | metall_close(manager); 20 | metall_remove("/tmp/metall1"); 21 | } 22 | 23 | // Allocate named object 24 | { 25 | metall_manager* manager = metall_create("/tmp/metall2"); 26 | 27 | uint64_t* array = metall_named_malloc(manager, "array", 28 | sizeof(uint64_t) * 10); 29 | 30 | array[0] = 0; 31 | 32 | metall_flush(manager); 33 | 34 | array[1] = 1; 35 | 36 | metall_snapshot(manager, "/tmp/metall2-snap"); 37 | metall_close(manager); 38 | } 39 | 40 | // Retrieve named object 41 | { 42 | metall_manager* manager = metall_open("/tmp/metall2"); 43 | 44 | uint64_t* array = metall_find(manager, "array"); 45 | 46 | assert(array[0] == 0); 47 | assert(array[1] == 1); 48 | 49 | metall_named_free(manager, "array"); 50 | metall_close(manager); 51 | metall_remove("/tmp/metall2"); 52 | } 53 | 54 | // Retrieve object snapshot 55 | { 56 | metall_manager* manager = metall_open("/tmp/metall2-snap"); 57 | 58 | uint64_t* array = metall_find(manager, "array"); 59 | 60 | assert(array[0] == 0); 61 | assert(array[1] == 1); 62 | 63 | metall_named_free(manager, "array"); 64 | metall_close(manager); 65 | metall_remove("/tmp/metall2-snap"); 66 | } 67 | 68 | return 0; 69 | } -------------------------------------------------------------------------------- /example/cmake/FetchContent/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.14) 2 | include(FetchContent) 3 | 4 | project(myproject LANGUAGES C CXX) 5 | 6 | # Metall requires C++ 17 7 | set(CMAKE_CXX_STANDARD 17) 8 | 9 | # Boost header files are required for using Metall. 10 | find_package(Boost 1.80) 11 | # Download Boost if it does not exit. 12 | # NOTE: this process may take a long time as Boost is a big repository. 13 | if (NOT Boost_FOUND) 14 | message(STATUS "Download or search previously downloaded Boost using FetchContent") 15 | FetchContent_Declare(Boost 16 | URL https://boostorg.jfrog.io/artifactory/main/release/1.80.0/source/boost_1_80_0.tar.bz2) 17 | FetchContent_MakeAvailable(Boost) 18 | find_package(Boost 1.80 REQUIRED) 19 | endif () 20 | 21 | # Download and setup Metall. 22 | if (NOT BUILD_C) 23 | # Just install Metall header files and avoid building executables in the repository. 24 | set(JUST_INSTALL_METALL_HEADER TRUE) 25 | endif () 26 | FetchContent_Declare( 27 | Metall 28 | GIT_REPOSITORY https://github.com/LLNL/metall.git 29 | GIT_TAG master 30 | ) 31 | FetchContent_MakeAvailable(Metall) 32 | 33 | # ---------------------------------------- # 34 | # For using Metall CXX API 35 | # ---------------------------------------- # 36 | find_package(Threads) 37 | 38 | add_executable(cpp_example ../src/cpp_example.cpp) 39 | 40 | # Need Boost header files 41 | target_include_directories(cpp_example PRIVATE ${Boost_INCLUDE_DIRS}) 42 | 43 | # This is required if one uses GCC. 44 | target_link_libraries(cpp_example PRIVATE stdc++fs) 45 | 46 | target_link_libraries(cpp_example PRIVATE Threads::Threads) 47 | 48 | # Link Metall 49 | # Although target_link_libraries() is used, no library file (e.g., *.a file) is linked. 50 | # Only include path will be set here. 51 | target_link_libraries(cpp_example PRIVATE Metall::Metall) 52 | 53 | # ---------------------------------------- # 54 | # For using Metall C API 55 | # ---------------------------------------- # 56 | if (BUILD_C) 57 | add_executable(c_example ../src/c_example.c) 58 | 59 | # Link Metall C library (libmetall_c) 60 | target_link_libraries(c_example PRIVATE Metall::metall_c) 61 | endif () -------------------------------------------------------------------------------- /example/cmake/FetchContent/README.md: -------------------------------------------------------------------------------- 1 | 2 | On this page, we describe how to download, install, and link Metall from a CMake project. 3 | 4 | To just find/link an already installed Metall package from a CMake project, see another example [here](../find_package). 5 | 6 | Here is an example CMake file that downloads, installs, and links Metall package [CMakeLists.txt](CMakeLists.txt). 7 | 8 | ## Build 9 | 10 | Here is how to use the CMake file in this directory. 11 | 12 | ```bash 13 | mkdir build 14 | cd build 15 | 16 | # BOOST_ROOT: Path to Boost C++ Libraries (option) 17 | # If Boost is not found in the local environment, the CMake file will download it automatically. 18 | # BUILD_C: Required to use Metall C (not 'C++') API (option) 19 | cmake ../ -DBOOST_ROOT=/path/to/boost -DBUILD_C=ON 20 | make 21 | ``` 22 | -------------------------------------------------------------------------------- /example/cmake/README.md: -------------------------------------------------------------------------------- 1 | 2 | To just **find/link an already installed** Metall package from a CMake project, see [find_package](./find_package). 3 | 4 | To **have CMake download, install, and link** Metall, see [FetchContent](./FetchContent). -------------------------------------------------------------------------------- /example/cmake/find_package/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | project(myproject LANGUAGES C CXX) 2 | 3 | # Metall requires C++ 17 4 | set(CMAKE_CXX_STANDARD 17) 5 | 6 | # Find Metall package 7 | # To configure the search path, see CMake documents. 8 | find_package(Metall REQUIRED) 9 | message(STATUS "Metall_FOUND = ${Metall_FOUND}") 10 | 11 | 12 | # ---------------------------------------- # 13 | # For using Metall CXX API 14 | # ---------------------------------------- # 15 | 16 | # Boost header files are required to use Metall C++ API 17 | find_package(Boost 1.80) 18 | find_package(Threads) 19 | 20 | if (Boost_FOUND AND Threads_FOUND) 21 | add_executable(cpp_example ../src/cpp_example.cpp) 22 | 23 | # Need Boost header files 24 | target_include_directories(cpp_example PRIVATE ${Boost_INCLUDE_DIRS}) 25 | 26 | # Link Metall 27 | # Although target_link_libraries() is used, no library file (e.g., *.a file) is linked. 28 | # Only include path will be set here. 29 | target_link_libraries(cpp_example PRIVATE Metall::Metall) 30 | 31 | # This is required if one uses GCC 32 | target_link_libraries(cpp_example PRIVATE stdc++fs) 33 | 34 | target_link_libraries(cpp_example PRIVATE Threads::Threads) 35 | endif () 36 | 37 | 38 | # ---------------------------------------- # 39 | # For using Metall C API 40 | # ---------------------------------------- # 41 | 42 | # Not required to do this. 43 | # Just checking if metall_c exists. 44 | find_library(LIB_METALL_C NAMES metall_c) 45 | 46 | if (LIB_METALL_C) 47 | add_executable(c_example ../src/c_example.c) 48 | 49 | # Link Metall C library (libmetall_c) 50 | target_link_libraries(c_example PRIVATE Metall::metall_c) 51 | endif() -------------------------------------------------------------------------------- /example/cmake/find_package/README.md: -------------------------------------------------------------------------------- 1 | # Find Metall from CMake Project 2 | 3 | On this page, we describe how to **find/link an already installed** Metall package from a CMake project. 4 | 5 | To have CMake download, install, and link Metall, see another example [here](../FetchContent). 6 | 7 | Here is an example CMake file that finds an already installed Metall package [CMakeLists.txt](CMakeLists.txt). 8 | 9 | ## 0. (pre-step) Install Metall 10 | 11 | Here is how to build example programs. 12 | 13 | ### 0-1. Install Metall Manually 14 | 15 | To install Metall at `"/path/to/install"`, for example: 16 | ```bash 17 | cd metall 18 | mkdir build 19 | cd build 20 | 21 | cmake ../ -DCMAKE_INSTALL_PREFIX="/path/to/install" 22 | # (option) add the following options to build the Metall C API library 23 | -DBOOST_ROOT=/path/to/boost -DBUILD_C=ON 24 | 25 | make && make install 26 | ``` 27 | 28 | 29 | ### 0-2. Install Metall using Spack 30 | 31 | Alternatively, one can install Metall using Spack. 32 | 33 | Please note that Metall C API is not supported with Spack. 34 | 35 | ```bash 36 | # Install Metall using Spack 37 | # Boost is also install 38 | spack install metall 39 | ``` 40 | 41 | 42 | ## 1. Build 43 | 44 | Here is how to use the CMake file in this directory. 45 | 46 | ```bash 47 | mkdir build 48 | cd build 49 | 50 | export CMAKE_PREFIX_PATH="/path/to/install" 51 | # Or (if one uses Spack) 52 | spack load metall # Spack exports CMAKE_PREFIX_PATH (and also BOOST_ROOT). 53 | 54 | cmake ../ \ 55 | -DBOOST_ROOT=/path/to/boost # Required if one wants to build programs that uses Metall C++ API. 56 | # Or (if one uses Spack) 57 | cmake ../ 58 | 59 | make 60 | ``` 61 | -------------------------------------------------------------------------------- /example/cmake/src/c_example.c: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #include 7 | 8 | int main(void) { 9 | metall_open(METALL_CREATE_ONLY, "/tmp/dir"); 10 | metall_close(); 11 | 12 | return 0; 13 | } -------------------------------------------------------------------------------- /example/cmake/src/cpp_example.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #include 7 | 8 | int main() { 9 | metall::manager manager(metall::create_only, "/tmp/dir"); 10 | return 0; 11 | } -------------------------------------------------------------------------------- /example/concurrent.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | /// \file concurrent.cpp 7 | /// \brief This example demonstrates Metall's concurrency support. 8 | /// Metall can be used in a multi-threaded environment. 9 | /// Please see the API documentation of the manager class to find out which 10 | /// functions are thread-safe. 11 | 12 | #include 13 | #include 14 | 15 | #include 16 | 17 | void metall_alloc(metall::manager& manager, const int tid) { 18 | for (int i = 0; i < 10; ++i) { 19 | if (tid % 2 == 0) { 20 | manager.deallocate(manager.allocate(10)); 21 | } else { 22 | manager.destroy_ptr(manager.construct(metall::anonymous_instance)()); 23 | } 24 | } 25 | } 26 | 27 | int main() { 28 | { 29 | metall::manager manager(metall::create_only, "/tmp/datastore"); 30 | 31 | { 32 | std::thread t1(metall_alloc, std::ref(manager), 1); 33 | std::thread t2(metall_alloc, std::ref(manager), 2); 34 | std::thread t3(metall_alloc, std::ref(manager), 3); 35 | std::thread t4(metall_alloc, std::ref(manager), 4); 36 | 37 | t1.join(); 38 | t2.join(); 39 | t3.join(); 40 | t4.join(); 41 | } 42 | assert(manager.check_sanity()); 43 | assert(manager.all_memory_deallocated()); 44 | } 45 | 46 | return 0; 47 | } -------------------------------------------------------------------------------- /example/container/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | if (Boost_VERSION_STRING VERSION_GREATER_EQUAL "1.75") 2 | add_metall_executable(string_key_store string_key_store.cpp) 3 | endif() -------------------------------------------------------------------------------- /example/csr_graph.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #include 7 | 8 | #include 9 | #include "graph_data_structure/csr.hpp" 10 | #include "graph_data_structure/csr_using_vector.hpp" 11 | 12 | using index_t = uint64_t; 13 | using vid_t = uint64_t; 14 | 15 | // We have two CSR graph data structures that have the same interfaces 16 | #if 0 17 | using csr_graph_t = csr>; 18 | #else 19 | using csr_graph_t = 20 | csr_using_vector>; 21 | #endif 22 | 23 | int main() { 24 | { 25 | // Create a new Metall datastore in "/tmp/dir" 26 | // If 'dir' does not exist, Metall creates automatically 27 | metall::manager manager(metall::create_only, "/tmp/dir"); 28 | 29 | std::size_t num_vertices = 16; 30 | std::size_t num_edges = 256; 31 | 32 | // Allocate and construct an object in the persistent memory with a name 33 | // "csr_graph" 34 | csr_graph_t *csr_graph = manager.construct("csr_graph")( 35 | num_vertices, num_edges, 36 | manager.get_allocator()); // Arguments to the constructor 37 | 38 | // You can use the arrays normally 39 | index_t *indices_array = csr_graph->indices(); 40 | vid_t *edges_array = csr_graph->edges(); 41 | edges_array[indices_array[1]++] = 10; 42 | } 43 | 44 | { 45 | // Open the existing Metall datastore in "/tmp/dir" 46 | metall::manager manager(metall::open_read_only, "/tmp/dir"); 47 | 48 | csr_graph_t *csr_graph = manager.find("csr_graph").first; 49 | if (!csr_graph) { 50 | std::cerr << "Object csr_graph does not exist" << std::endl; 51 | std::abort(); 52 | } 53 | 54 | // You can use the arrays normally 55 | index_t *indices_array = csr_graph->indices(); 56 | vid_t *edges_array = csr_graph->edges(); 57 | std::cout << edges_array[indices_array[0]] << std::endl; 58 | } 59 | 60 | return 0; 61 | } -------------------------------------------------------------------------------- /example/custom_logger.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #include 7 | 8 | /// \brief converts the given level into string literal 9 | const char *log_lvl_to_string(metall_log_level lvl) { 10 | switch (lvl) { 11 | case metall_critical: return "CRIT"; 12 | case metall_error: return "ERROR"; 13 | case metall_warning: return "WARN"; 14 | case metall_info: return "INFO"; 15 | case metall_debug: return "DEBUG"; 16 | case metall_verbose: return "VERBOSE"; 17 | default: return "UNKNOWN"; 18 | } 19 | } 20 | 21 | /// \brief custom logger implementation 22 | /// Output looks different than default. 23 | extern "C" void metall_log(metall_log_level lvl, const char *file, size_t line_no, const char *message) { 24 | std::cerr << log_lvl_to_string(lvl) << " metall{file=" << file 25 | << ", line=" << line_no << "}: " << message << std::endl; 26 | } 27 | 28 | int main() { 29 | // Do Metall operations 30 | metall::manager manager(metall::create_only, "/tmp/dir"); 31 | 32 | return 0; 33 | } 34 | -------------------------------------------------------------------------------- /example/datastore_description.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | #include 11 | 12 | int main() { 13 | // ----- Non-static methods ----- // 14 | { 15 | metall::manager manager(metall::create_only, "/tmp/dir"); 16 | 17 | std::string buf; 18 | [[maybe_unused]] const auto ret = manager.get_description(&buf); 19 | assert(!ret && buf.empty()); // 'ret' is false and 'buf' is empty since 20 | // description has not been created yet. 21 | 22 | manager.set_description("description example"); 23 | manager.get_description(&buf); 24 | std::cout << buf << std::endl; // Shows "description example" 25 | } 26 | 27 | // ----- Static methods ----- // 28 | { 29 | metall::manager::set_description("/tmp/dir", "description example 2"); 30 | 31 | std::string buf; 32 | metall::manager::get_description("/tmp/dir", &buf); 33 | std::cout << buf << std::endl; // Shows "description example 2" 34 | } 35 | 36 | return 0; 37 | } -------------------------------------------------------------------------------- /example/fallback_allocator.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #include 7 | 8 | #include 9 | #include 10 | 11 | using vector_t = 12 | metall::container::vector>; 13 | 14 | int main() { 15 | // Allocation using Metall 16 | { 17 | metall::manager manager(metall::create_only, "/tmp/dir"); 18 | auto pvec = manager.construct("vec")(manager.get_allocator()); 19 | pvec->push_back(1); 20 | std::cout << (*pvec)[0] << std::endl; 21 | } 22 | 23 | // Allocation not using Metall, i.e., uses a heap allocator (malloc()). 24 | // This code would cause a build error if fallback_allocator were not used. 25 | { 26 | vector_t vec; // As no metall allocator argument is passed, the fallback 27 | // allocator uses malloc() internally. 28 | vec.push_back(2); 29 | std::cout << vec[0] << std::endl; 30 | } 31 | 32 | return 0; 33 | } -------------------------------------------------------------------------------- /example/json/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | if (Boost_VERSION_STRING VERSION_GREATER_EQUAL "1.75") 2 | add_metall_executable(json_create json_create.cpp) 3 | add_metall_executable(json_open json_open.cpp) 4 | add_metall_executable(jgraph jgraph.cpp) 5 | add_metall_executable(vector_of_json vector_of_json.cpp) 6 | endif() -------------------------------------------------------------------------------- /example/json/json_create.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | int main() { 11 | std::string json_string = R"( 12 | { 13 | "pi": 3.141, 14 | "happy": true, 15 | "name": "Niels", 16 | "nothing": null, 17 | "answer": { 18 | "everything": 42 19 | }, 20 | "list": [1, 0, 2], 21 | "object": { 22 | "currency": "USD", 23 | "value": 42.99 24 | } 25 | } 26 | )"; 27 | 28 | std::cout << "Create" << std::endl; 29 | 30 | using metall_value_type = 31 | metall::json::value>; 32 | metall::manager manager(metall::create_only, "./test"); 33 | 34 | auto *value = manager.construct(metall::unique_instance)( 35 | metall::json::parse(json_string, manager.get_allocator())); 36 | metall::json::pretty_print(std::cout, *value); 37 | 38 | value->as_object()["name"].as_string() = "Alice"; // Change a string value 39 | 40 | value->as_object()["temperature"] = 25.2; // Insert a double value 41 | value->as_object()["unit"] = "celsius"; // Insert a string value 42 | 43 | value->as_object().erase("pi"); // Erase a value 44 | 45 | auto pos = value->as_object().find("happy"); 46 | std::cout << "Happy? : " << pos->value() << std::endl; 47 | 48 | metall::json::pretty_print(std::cout, *value); 49 | 50 | const auto clone(*value); 51 | std::cout << (clone == *value) << std::endl; 52 | 53 | return 0; 54 | } 55 | -------------------------------------------------------------------------------- /example/json/json_open.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | using metall_value_type = 6 | metall::json::value>; 7 | 8 | int main() { 9 | std::cout << "Open" << std::endl; 10 | { 11 | metall::manager manager(metall::open_read_only, "./test"); 12 | auto *value = 13 | manager.find(metall::unique_instance).first; 14 | metall::json::pretty_print(std::cout, *value); 15 | } 16 | 17 | { 18 | metall::manager manager(metall::open_only, "./test"); 19 | manager.destroy(metall::unique_instance); 20 | } 21 | 22 | return 0; 23 | } 24 | -------------------------------------------------------------------------------- /example/json/vector_of_json.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | namespace mc = metall::container; 12 | namespace mj = metall::json; 13 | 14 | int main() { 15 | // Metall JSON container works like the other containers w.r.t. allocation, 16 | // i.e., it takes an allocator type in its template parameter and an allocator 17 | // object in its constructors. 18 | using json_value_type = mj::value>; 19 | // Need to use scoped_allocator as this is a multi-layer container. 20 | using vector_json_type = 21 | mc::vector>; 23 | 24 | // An example input json strings. 25 | std::vector json_string_list{ 26 | R"({"name": "Alice", "list": [0, 1]})", 27 | R"({"name": "Brad", "list": [2, 3]})"}; 28 | 29 | // Create a vector-of-json object 30 | { 31 | metall::manager manager(metall::create_only, "./test"); 32 | auto *vec = manager.construct(metall::unique_instance)( 33 | manager.get_allocator()); 34 | for (const auto &json_string : json_string_list) { 35 | vec->emplace_back(mj::parse(json_string, manager.get_allocator())); 36 | } 37 | } 38 | 39 | // Reattach the vector-of-json object created above. 40 | { 41 | metall::manager manager(metall::open_read_only, "./test"); 42 | auto *vec = manager.find(metall::unique_instance).first; 43 | for (const auto &json : *vec) { 44 | mj::pretty_print(std::cout, json); // Show contents. 45 | } 46 | } 47 | 48 | return 0; 49 | } 50 | -------------------------------------------------------------------------------- /example/logger.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #include 7 | 8 | int main() { 9 | // Set the log level to , for example, verbose. 10 | // The log level can be changed at any time. 11 | metall::logger::set_log_level(metall::logger::level_filter::verbose); 12 | 13 | // Enable the program to abort when a critical log message is displayed. 14 | metall::logger::abort_on_critical_error(true); 15 | 16 | // Do Metall operations 17 | metall::manager manager(metall::create_only, "/tmp/dir"); 18 | 19 | return 0; 20 | } -------------------------------------------------------------------------------- /example/mpi_create.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #include 7 | #include 8 | 9 | int main(int argc, char **argv) { 10 | ::MPI_Init(&argc, &argv); 11 | { 12 | // This over-write mode fails if the existing file/directory is not Metall 13 | // datastore or the existing datastore was created by a different number of 14 | // MPI ranks. 15 | // To forcibly　remove the existing datastore, one can use the following 16 | // code. 17 | // metall::utility::filesystem::remove("/tmp/metall_mpi"); 18 | // ::MPI_Barrier(MPI_COMM_WORLD); 19 | bool overwrite = true; 20 | 21 | metall::utility::metall_mpi_adaptor mpi_adaptor( 22 | metall::create_only, "/tmp/metall_mpi", MPI_COMM_WORLD, overwrite); 23 | auto &metall_manager = mpi_adaptor.get_local_manager(); 24 | 25 | auto rank = metall_manager.construct("my-rank")(); 26 | ::MPI_Comm_rank(MPI_COMM_WORLD, rank); // Stores my rank 27 | } 28 | ::MPI_Finalize(); 29 | 30 | return 0; 31 | } -------------------------------------------------------------------------------- /example/mpi_open.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #include 7 | 8 | int main(int argc, char **argv) { 9 | ::MPI_Init(&argc, &argv); 10 | { 11 | metall::utility::metall_mpi_adaptor mpi_adaptor(metall::open_only, 12 | "/tmp/metall_mpi"); 13 | auto &metall_manager = mpi_adaptor.get_local_manager(); 14 | 15 | auto stored_rank = metall_manager.find("my-rank").first; 16 | 17 | int rank; 18 | ::MPI_Comm_rank(MPI_COMM_WORLD, &rank); 19 | 20 | // The line below should print out: 21 | // "Rank x opened value x" 22 | // , where x is a number 23 | std::cout << "Rank " << rank << " opened value " << *stored_rank 24 | << std::endl; 25 | } 26 | ::MPI_Finalize(); 27 | 28 | return 0; 29 | } -------------------------------------------------------------------------------- /example/object_attribute.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #include 7 | #include 8 | 9 | #include 10 | 11 | int main() { 12 | // Set and get object attributes via metall::manager object 13 | { 14 | metall::manager manager(metall::create_only, "/tmp/dir"); 15 | auto *obj = manager.construct("obj")(); 16 | 17 | std::string description; 18 | manager.get_instance_description(obj, &description); 19 | std::cout << "Name: " << manager.get_instance_name(obj) 20 | << ", Length: " << manager.get_instance_length(obj) 21 | << ", Description: " << description << std::endl; 22 | 23 | manager.set_instance_description(obj, "description example 1"); 24 | } 25 | 26 | // Set and get object attributes via object attribute accessor object 27 | // Using object attribute accessor, one can access object attribute without 28 | // allocating a metall::manager object (i.e., w/o memory mapping files). 29 | { 30 | auto accessor = metall::manager::access_named_object_attribute("/tmp/dir"); 31 | for (const auto &object : accessor) { 32 | std::cout << "Name: " << object.name() << ", Length: " << object.length() 33 | << ", Description: " << object.description() << std::endl; 34 | } 35 | 36 | auto itr = accessor.find("obj"); 37 | accessor.set_description(itr, "description example 2"); 38 | std::cout << "Name: " << itr->name() << ", Length: " << itr->length() 39 | << ", Description: " << itr->description() << std::endl; 40 | } 41 | 42 | return 0; 43 | } -------------------------------------------------------------------------------- /example/offset_pointer.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | // This is an example shows how to store pointers in persistent memory using the 7 | // offset pointer 8 | 9 | #include 10 | #include 11 | 12 | // metall::offset_ptr is just an alias of offset_ptr in Boost.Interprocess 13 | // https://www.boost.org/doc/libs/release/doc/html/interprocess/offset_ptr.html 14 | using int_offset_prt = metall::offset_ptr; 15 | 16 | int main() { 17 | { 18 | metall::manager manager(metall::create_only, "/tmp/datastore"); 19 | 20 | // Allocate a simple array in persistent memory 21 | int *array = static_cast(manager.allocate(10 * sizeof(int))); 22 | array[0] = 1; 23 | array[1] = 2; 24 | 25 | // Allocate an offset pointer with key 'ptr' and initialize its value with 26 | // the address of 'array' 27 | [[maybe_unused]] int_offset_prt *ptr = 28 | manager.construct("ptr")(array); 29 | } 30 | 31 | { 32 | metall::manager manager(metall::open_only, "/tmp/datastore"); 33 | 34 | int_offset_prt *ptr = manager.find("ptr").first; 35 | 36 | // metall::to_raw_pointer extracts a raw pointer from metall::offset_ptr 37 | // If a raw pointer is given, it just returns the address the given pointer 38 | // points to 39 | int *array = metall::to_raw_pointer(*ptr); 40 | 41 | std::cout << array[0] << std::endl; // Print 1 42 | std::cout << array[1] << std::endl; // Print 2 43 | 44 | // Deallocate the array 45 | manager.deallocate(metall::to_raw_pointer(*ptr)); 46 | *ptr = nullptr; 47 | 48 | // Destroy the offset pointer object 49 | manager.destroy("ptr"); 50 | } 51 | 52 | return 0; 53 | } -------------------------------------------------------------------------------- /example/simple.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #include 7 | 8 | #include 9 | 10 | #include // Only one header file is required to be included 11 | 12 | // Type alias 13 | // This is the standard way to give a custom allocator to a container 14 | using vector_t = 15 | boost::container::vector>; 16 | 17 | int main() { 18 | { 19 | // Construct a manager instance 20 | // A process can allocate multiple manager instances 21 | metall::manager manager( 22 | metall::create_only, // Create a new one 23 | "/tmp/dir"); // The directory to store backing datastore 24 | 25 | // Allocate and construct a vector instance of vector_t. 26 | // The name "vec" is saved inside Metall datastore 27 | // and used to find the instance later. 28 | // The arguments in the last () are passed to the constructor of vector_t. 29 | auto pvec = manager.construct("vec")(manager.get_allocator()); 30 | 31 | // From now on, the vector instance can be used normally. 32 | pvec->push_back(5); 33 | } // Implicitly sync with backing files, i.e., sync() is called in 34 | // metall::manager's destructor 35 | 36 | // ---------- Assume exit and restart the program at this point ---------- // 37 | 38 | /// consistent() returns true if a Metall data store exits at the path and 39 | /// was closed property. 40 | if (metall::manager::consistent("/tmp/dir")) { 41 | // Reattach the manager instance 42 | metall::manager manager(metall::open_only, "/tmp/dir"); 43 | 44 | // Find the previously constructed instance 45 | // Please do not forget to use ".first" at the end 46 | auto pvec = manager.find("vec").first; 47 | 48 | pvec->push_back(10); // Can restart to use containers normally 49 | 50 | std::cout << (*pvec)[0] << std::endl; // Will print "5" 51 | std::cout << (*pvec)[1] << std::endl; // Will print "10" 52 | 53 | manager.destroy("vec"); // Destroy the instance 54 | } else { 55 | std::cerr << "Cannot open a Metall data store" << std::endl; 56 | } 57 | 58 | return 0; 59 | } -------------------------------------------------------------------------------- /example/snapshot.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #include 7 | 8 | #include 9 | 10 | int main() { 11 | const std::string master_path("/tmp/dir_path"); 12 | const std::string snapshot_dir_prefix("/tmp/snapshot-"); 13 | const std::string snapshot_name0 = snapshot_dir_prefix + "ver0"; 14 | const std::string snapshot_name1 = snapshot_dir_prefix + "ver1"; 15 | 16 | { 17 | // Create the master data 18 | metall::manager manager(metall::create_only, master_path.c_str()); 19 | int *a = manager.construct(metall::unique_instance)(0); 20 | 21 | // Take a snapshot before updating to '1' 22 | manager.snapshot(snapshot_name0.c_str()); 23 | *a = 1; 24 | 25 | // Take a snapshot before updating to '2' 26 | manager.snapshot(snapshot_name1.c_str()); 27 | *a = 2; 28 | } 29 | 30 | // Open snapshot 0 if it is consistent, i.e., it was closed properly. 31 | if (metall::manager::consistent(snapshot_name0.c_str())) { 32 | metall::manager manager(metall::open_read_only, snapshot_name0.c_str()); 33 | int *a = manager.find(metall::unique_instance).first; 34 | std::cout << *a << std::endl; // Print 0 35 | } else { 36 | std::cerr << snapshot_name0 << " is inconsistent" << std::endl; 37 | } 38 | 39 | // Open snapshot 1 if it is consistent, i.e., it was closed properly. 40 | if (metall::manager::consistent(snapshot_name1.c_str())) { 41 | metall::manager manager(metall::open_read_only, snapshot_name1.c_str()); 42 | int *a = manager.find(metall::unique_instance).first; 43 | std::cout << *a << std::endl; // Print 1 44 | } else { 45 | std::cerr << snapshot_name1 << " is inconsistent" << std::endl; 46 | } 47 | 48 | // Open the master data if it is consistent, i.e., it was closed properly. 49 | if (metall::manager::consistent(master_path.c_str())) { 50 | metall::manager manager(metall::open_read_only, master_path.c_str()); 51 | int *a = manager.find(metall::unique_instance).first; 52 | std::cout << *a << std::endl; // Print 2 53 | } else { 54 | std::cerr << master_path << " is inconsistent" << std::endl; 55 | } 56 | 57 | return 0; 58 | } -------------------------------------------------------------------------------- /example/static_mutex.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #include 7 | #include 8 | 9 | #include 10 | 11 | static constexpr std::size_t k_num_mutexes = 2; 12 | 13 | void mutex_work(const int key, const int value, int* array) { 14 | { // A mutex block 15 | const int index = key % k_num_mutexes; 16 | auto guard = metall::utility::mutex::mutex_lock(index); 17 | array[index] = array[index] + value; // do some mutex work 18 | } // The mutex is released here 19 | } 20 | 21 | int main() { 22 | int array[k_num_mutexes] = {0, 0}; 23 | 24 | // Launch multiple concurrent jobs 25 | std::thread t1(mutex_work, 0, 1, array); // Add 1 to array[0] 26 | std::thread t2(mutex_work, 1, 2, array); // Add 2 to array[1] 27 | std::thread t3(mutex_work, 2, 3, array); // Add 3 to array[0] 28 | std::thread t4(mutex_work, 3, 4, array); // Add 4 to array[1] 29 | 30 | // Wait for the threads to finish 31 | t1.join(); 32 | t2.join(); 33 | t3.join(); 34 | t4.join(); 35 | 36 | std::cout << array[0] << std::endl; // Will show 4 37 | std::cout << array[1] << std::endl; // Will show 6 38 | 39 | return 0; 40 | } -------------------------------------------------------------------------------- /example/string.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | // String with Metall 12 | using persistent_string = 13 | boost::container::basic_string, 14 | metall::manager::allocator_type>; 15 | 16 | int main() { 17 | { 18 | metall::manager manager(metall::create_only, "/tmp/datastore"); 19 | auto pstr = manager.construct("mystring")( 20 | "Hello, World!", manager.get_allocator<>()); 21 | std::cout << *pstr << std::endl; 22 | } 23 | 24 | { 25 | metall::manager manager(metall::open_only, "/tmp/datastore"); 26 | auto pstr = manager.find("mystring").first; 27 | std::cout << *pstr << std::endl; 28 | } 29 | 30 | return 0; 31 | } -------------------------------------------------------------------------------- /include/metall/container/deque.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_CONTAINER_DEQUE_HPP 7 | #define METALL_CONTAINER_DEQUE_HPP 8 | 9 | #include 10 | 11 | #include 12 | 13 | namespace metall::container { 14 | 15 | /// \brief A deque container that uses Metall as its default allocator. 16 | template > 17 | using deque = boost::container::deque; 18 | 19 | } // namespace metall::container 20 | 21 | #endif // METALL_CONTAINER_DEQUE_HPP -------------------------------------------------------------------------------- /include/metall/container/experimental/json/json.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_CONTAINER_EXPERIMENT_JSON_JSON_HPP 7 | #define METALL_CONTAINER_EXPERIMENT_JSON_JSON_HPP 8 | 9 | #warning \ 10 | "metall/container/experimental/json/json.hpp is deprecated. Please use include/metall/json/json.hpp instead." 11 | 12 | #include 13 | 14 | #warning \ 15 | "Namespace metall::container::experimental::json is deprecated. Please use metall::json instead." 16 | namespace metall::container::experimental::json { 17 | using namespace metall::json; 18 | } 19 | 20 | #endif // METALL_CONTAINER_EXPERIMENT_JSON_JSON_HPP 21 | -------------------------------------------------------------------------------- /include/metall/container/list.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_CONTAINER_LIST_HPP 7 | #define METALL_CONTAINER_LIST_HPP 8 | 9 | #include 10 | 11 | #include 12 | 13 | namespace metall::container { 14 | 15 | /// \brief A list container that uses Metall as its default allocator. 16 | template > 17 | using list = boost::container::list; 18 | 19 | } // namespace metall::container 20 | 21 | #endif // METALL_CONTAINER_LIST_HPP -------------------------------------------------------------------------------- /include/metall/container/map.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_CONTAINER_MAP_HPP 7 | #define METALL_CONTAINER_MAP_HPP 8 | 9 | #include 10 | 11 | #include 12 | 13 | #include 14 | 15 | namespace metall::container { 16 | 17 | /// \brief A map container that uses Metall as its default allocator. 18 | template , 19 | class Allocator = manager::allocator_type>> 20 | using map = boost::container::map; 21 | 22 | /// \brief A multimap container that uses Metall as its default allocator. 23 | template , 24 | class Allocator = manager::allocator_type>> 25 | using multimap = boost::container::multimap; 26 | 27 | } // namespace metall::container 28 | 29 | #endif // METALL_CONTAINER_MAP_HPP 30 | -------------------------------------------------------------------------------- /include/metall/container/priority_queue.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_CONTAINER_PRIORITY_QUEUE_HPP 7 | #define METALL_CONTAINER_PRIORITY_QUEUE_HPP 8 | 9 | #include 10 | #include 11 | 12 | #include 13 | 14 | namespace metall::container { 15 | 16 | /// \brief A priority_queue container that uses Metall as its default allocator. 17 | template , 18 | typename Compare = std::less> 19 | using priority_queue = std::priority_queue; 20 | 21 | } // namespace metall::container 22 | 23 | #endif // METALL_CONTAINER_PRIORITY_QUEUE_HPP 24 | -------------------------------------------------------------------------------- /include/metall/container/queue.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_CONTAINER_QUEUE_HPP 7 | #define METALL_CONTAINER_QUEUE_HPP 8 | 9 | #include 10 | 11 | #include 12 | 13 | namespace metall::container { 14 | 15 | /// \brief A queue container that uses Metall as its default allocator. 16 | template > 17 | using queue = std::queue; 18 | 19 | } // namespace metall::container 20 | 21 | #endif // METALL_CONTAINER_QUEUE_HPP 22 | -------------------------------------------------------------------------------- /include/metall/container/scoped_allocator.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_CONTAINER_SCOPED_ALLOCATOR_HPP 7 | #define METALL_CONTAINER_SCOPED_ALLOCATOR_HPP 8 | 9 | #include 10 | 11 | namespace metall::container { 12 | 13 | /// \brief An allocator which can be used with multilevel containers. 14 | template 15 | using scoped_allocator_adaptor = 16 | boost::container::scoped_allocator_adaptor; 17 | 18 | } // namespace metall::container 19 | 20 | #endif // METALL_CONTAINER_SCOPED_ALLOCATOR_HPP 21 | -------------------------------------------------------------------------------- /include/metall/container/set.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_CONTAINER_SET_HPP 7 | #define METALL_CONTAINER_SET_HPP 8 | 9 | #include 10 | 11 | #include 12 | 13 | #include 14 | 15 | namespace metall::container { 16 | 17 | /// \brief A set container that uses Metall as its default allocator. 18 | template , 19 | class Allocator = manager::allocator_type> 20 | using set = boost::container::set; 21 | 22 | /// \brief A multiset container that uses Metall as its default allocator. 23 | template , 24 | class Allocator = manager::allocator_type> 25 | using multiset = boost::container::multiset; 26 | 27 | } // namespace metall::container 28 | 29 | #endif // METALL_CONTAINER_SET_HPP 30 | -------------------------------------------------------------------------------- /include/metall/container/stack.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_CONTAINER_STACK_HPP 7 | #define METALL_CONTAINER_STACK_HPP 8 | 9 | #include 10 | 11 | #include 12 | 13 | namespace metall::container { 14 | 15 | /// \brief A stack container that uses Metall as its default allocator. 16 | template > 17 | using stack = std::stack; 18 | 19 | } // namespace metall::container 20 | 21 | #endif // METALL_CONTAINER_STACK_HPP 22 | -------------------------------------------------------------------------------- /include/metall/container/string.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_CONTAINER_STRING_HPP 7 | #define METALL_CONTAINER_STRING_HPP 8 | 9 | #include 10 | #include 11 | 12 | #include 13 | 14 | namespace metall::container { 15 | 16 | /// \brief A string container that uses Metall as its default allocator. 17 | template , 18 | class Allocator = metall::manager::allocator_type> 19 | using basic_string = boost::container::basic_string; 20 | 21 | /// \brief A string container that uses char as its character type and Metall as 22 | /// its default allocator. 23 | using string = basic_string; 24 | 25 | /// \brief A string container that uses wchar_t as its character type and Metall 26 | /// as its default allocator. 27 | using wstring = basic_string; 28 | 29 | } // namespace metall::container 30 | 31 | #endif // METALL_CONTAINER_STRING_HPP 32 | -------------------------------------------------------------------------------- /include/metall/container/string_key_store_locator.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_CONTAINER_CONCURRENT__STRING_KEY_STORE_LOCATOR_HPP_ 7 | #define METALL_CONTAINER_CONCURRENT__STRING_KEY_STORE_LOCATOR_HPP_ 8 | 9 | namespace metall::container { 10 | 11 | template 12 | class string_key_store_locator { 13 | private: 14 | template 15 | friend class string_key_store; 16 | 17 | public: 18 | string_key_store_locator(iterator_type iterator) : m_iterator(iterator) {} 19 | 20 | string_key_store_locator &operator++() { 21 | ++m_iterator; 22 | return *this; 23 | } 24 | 25 | string_key_store_locator operator++(int) { 26 | auto tmp(*this); 27 | ++m_iterator; 28 | return tmp; 29 | } 30 | 31 | bool operator==(const string_key_store_locator &other) const { 32 | return m_iterator == other.m_iterator; 33 | } 34 | 35 | bool operator!=(const string_key_store_locator &other) const { 36 | return m_iterator != other.m_iterator; 37 | } 38 | 39 | private: 40 | iterator_type m_iterator; 41 | }; 42 | 43 | } // namespace metall::container 44 | 45 | #endif // METALL_CONTAINER_CONCURRENT__STRING_KEY_STORE_LOCATOR_HPP_ 46 | -------------------------------------------------------------------------------- /include/metall/container/unordered_flat_map.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_CONTAINER_UNORDERED_FLAT_MAP_HPP 7 | #define METALL_CONTAINER_UNORDERED_FLAT_MAP_HPP 8 | 9 | #include 10 | 11 | static_assert(BOOST_VERSION >= 108100, "Unsupported Boost version"); 12 | #include 13 | 14 | #include 15 | 16 | namespace metall::container { 17 | 18 | /// \brief An unordered_flat_map container that uses Metall as its default 19 | /// allocator. 20 | template , 21 | class KeyEqual = std::equal_to, 22 | class Allocator = manager::allocator_type>> 23 | using unordered_flat_map = 24 | boost::unordered_flat_map; 25 | 26 | } // namespace metall::container 27 | 28 | #endif // METALL_CONTAINER_UNORDERED_FLAT_MAP_HPP 29 | -------------------------------------------------------------------------------- /include/metall/container/unordered_flat_set.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_CONTAINER_UNORDERED_FLAT_SET_HPP 7 | #define METALL_CONTAINER_UNORDERED_FLAT_SET_HPP 8 | 9 | #include 10 | 11 | static_assert(BOOST_VERSION >= 108100, "Unsupported Boost version"); 12 | #include 13 | 14 | #include 15 | 16 | namespace metall::container { 17 | 18 | /// \brief An unordered_flat_set container that uses Metall as its default 19 | /// allocator. 20 | template , 21 | class KeyEqual = std::equal_to, 22 | class Allocator = manager::allocator_type> 23 | using unordered_flat_set = 24 | boost::unordered_flat_set; 25 | 26 | } // namespace metall::container 27 | 28 | #endif // METALL_CONTAINER_UNORDERED_FLAT_SET_HPP 29 | -------------------------------------------------------------------------------- /include/metall/container/unordered_map.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_CONTAINER_UNORDERED_MAP_HPP 7 | #define METALL_CONTAINER_UNORDERED_MAP_HPP 8 | 9 | #include 10 | 11 | #include 12 | 13 | #include 14 | 15 | namespace metall::container { 16 | 17 | /// \brief An unordered_map container that uses Metall as its default allocator. 18 | template , 19 | class KeyEqual = std::equal_to, 20 | class Allocator = manager::allocator_type>> 21 | using unordered_map = boost::unordered_map; 22 | 23 | /// \brief An unordered_multimap container that uses Metall as its default 24 | /// allocator. 25 | template , 26 | class KeyEqual = std::equal_to, 27 | class Allocator = manager::allocator_type>> 28 | using unordered_multimap = 29 | boost::unordered_multimap; 30 | 31 | } // namespace metall::container 32 | 33 | #endif // METALL_CONTAINER_UNORDERED_MAP_HPP 34 | -------------------------------------------------------------------------------- /include/metall/container/unordered_node_map.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_CONTAINER_UNORDERED_NODE_MAP_HPP 7 | #define METALL_CONTAINER_UNORDERED_NODE_MAP_HPP 8 | 9 | #include 10 | 11 | static_assert(BOOST_VERSION >= 108200, "Unsupported Boost version"); 12 | #include 13 | 14 | #include 15 | 16 | namespace metall::container { 17 | 18 | /// \brief An unordered_node_map container that uses Metall as its default 19 | /// allocator. 20 | template , 21 | class KeyEqual = std::equal_to, 22 | class Allocator = manager::allocator_type>> 23 | using unordered_node_map = 24 | boost::unordered_node_map; 25 | 26 | } // namespace metall::container 27 | 28 | #endif // METALL_CONTAINER_UNORDERED_NODE_MAP_HPP 29 | -------------------------------------------------------------------------------- /include/metall/container/unordered_node_set.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_CONTAINER_UNORDERED_NODE_SET_HPP 7 | #define METALL_CONTAINER_UNORDERED_NODE_SET_HPP 8 | 9 | #include 10 | 11 | static_assert(BOOST_VERSION >= 108200, "Unsupported Boost version"); 12 | #include 13 | 14 | #include 15 | 16 | namespace metall::container { 17 | 18 | /// \brief An unordered_node_set container that uses Metall as its default 19 | /// allocator. 20 | template , 21 | class KeyEqual = std::equal_to, 22 | class Allocator = manager::allocator_type> 23 | using unordered_node_set = 24 | boost::unordered_node_set; 25 | 26 | } // namespace metall::container 27 | 28 | #endif // METALL_CONTAINER_UNORDERED_NODE_SET_HPP 29 | -------------------------------------------------------------------------------- /include/metall/container/unordered_set.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_CONTAINER_UNORDERED_SET_HPP 7 | #define METALL_CONTAINER_UNORDERED_SET_HPP 8 | 9 | #include 10 | 11 | #include 12 | 13 | #include 14 | 15 | namespace metall::container { 16 | 17 | /// \brief An unordered_set container that uses Metall as its default allocator. 18 | template , 19 | class KeyEqual = std::equal_to, 20 | class Allocator = manager::allocator_type> 21 | using unordered_set = boost::unordered_set; 22 | 23 | /// \brief An unordered_multiset container that uses Metall as its default 24 | /// allocator. 25 | template , 26 | class KeyEqual = std::equal_to, 27 | class Allocator = manager::allocator_type> 28 | using unordered_multiset = 29 | boost::unordered_multiset; 30 | 31 | } // namespace metall::container 32 | 33 | #endif // METALL_CONTAINER_UNORDERED_SET_HPP 34 | -------------------------------------------------------------------------------- /include/metall/container/vector.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_CONTAINER_VECTOR_HPP 7 | #define METALL_CONTAINER_VECTOR_HPP 8 | 9 | #include 10 | 11 | #include 12 | 13 | namespace metall::container { 14 | 15 | /// \brief A vector container that uses Metall as its default allocator. 16 | template > 17 | using vector = boost::container::vector; 18 | 19 | } // namespace metall::container 20 | 21 | #endif // METALL_CONTAINER_VECTOR_HPP 22 | -------------------------------------------------------------------------------- /include/metall/detail/array_construct.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_DETAIL_UTILITY_ARRAY_CONSTRUCT_HPP 7 | #define METALL_DETAIL_UTILITY_ARRAY_CONSTRUCT_HPP 8 | 9 | #include 10 | 11 | namespace metall { 12 | namespace mtlldetail { 13 | 14 | using boost::interprocess::ipcdetail::array_construct; 15 | 16 | } // namespace mtlldetail 17 | } // namespace metall 18 | 19 | #endif // METALL_DETAIL_UTILITY_ARRAY_CONSTRUCT_HPP 20 | -------------------------------------------------------------------------------- /include/metall/detail/builtin_functions.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_DETAIL_UTILITY_BUILTIN_FUNCTIONS_HPP 7 | #define METALL_DETAIL_UTILITY_BUILTIN_FUNCTIONS_HPP 8 | 9 | namespace metall::mtlldetail { 10 | 11 | /// \brief Count Leading Zeros. 12 | inline int clzll(const unsigned long long x) noexcept { 13 | #if defined(__GNUG__) || defined(__clang__) 14 | return __builtin_clzll(x); 15 | #else 16 | #error "GCC or Clang must be used to use __builtin_clzll" << std::endl; 17 | #endif 18 | } 19 | 20 | /// \brief Count Trailing Zeros. 21 | inline int ctzll(const unsigned long long x) noexcept { 22 | #if defined(__GNUG__) || defined(__clang__) 23 | return __builtin_ctzll(x); 24 | #else 25 | #error "GCC or Clang must be used to use __builtin_ctzll" << std::endl; 26 | #endif 27 | } 28 | 29 | } // namespace metall::mtlldetail 30 | 31 | #endif // METALL_DETAIL_UTILITY_BUILTIN_FUNCTIONS_HPP 32 | -------------------------------------------------------------------------------- /include/metall/detail/char_ptr_holder.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_DETAIL_UTILITY_CHAR_OTR_HOLDER_HPP 7 | #define METALL_DETAIL_UTILITY_CHAR_OTR_HOLDER_HPP 8 | 9 | #include 10 | 11 | namespace metall::mtlldetail { 12 | 13 | template 14 | using char_ptr_holder = 15 | boost::interprocess::ipcdetail::char_ptr_holder; 16 | 17 | enum instance_kind { anonymous_kind, named_kind, unique_kind }; 18 | 19 | using anonymous_instance_t = 20 | boost::interprocess::ipcdetail::anonymous_instance_t; 21 | using unique_instance_t = boost::interprocess::ipcdetail::unique_instance_t; 22 | 23 | } // namespace metall::mtlldetail 24 | #endif // METALL_DETAIL_UTILITY_CHAR_OTR_HOLDER_HPP 25 | -------------------------------------------------------------------------------- /include/metall/detail/mutex.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_DETAIL_UTILITY_MUTEX_HPP 7 | #define METALL_DETAIL_UTILITY_MUTEX_HPP 8 | 9 | #include 10 | 11 | namespace metall::mtlldetail { 12 | 13 | using mutex = std::mutex; 14 | using mutex_lock_guard = std::lock_guard; 15 | 16 | } // namespace metall::mtlldetail 17 | #endif // METALL_DETAIL_UTILITY_MUTEX_HPP 18 | -------------------------------------------------------------------------------- /include/metall/detail/parameter_pack.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_DETAIL_UTILITY_PARAMETER_PACK_HPP 7 | #define METALL_DETAIL_UTILITY_PARAMETER_PACK_HPP 8 | 9 | namespace metall::mtlldetail { 10 | 11 | // ---------- get index ---------- // 12 | template 13 | struct get_index; 14 | 15 | // Found 16 | template 17 | struct get_index { 18 | static constexpr std::size_t value = 0; 19 | }; 20 | 21 | // Searching 22 | template 23 | struct get_index { 24 | static constexpr std::size_t value = get_index::value + 1; 25 | }; 26 | 27 | } // namespace metall::mtlldetail 28 | #endif // METALL_DETAIL_UTILITY_PARAMETER_PACK_HPP 29 | -------------------------------------------------------------------------------- /include/metall/detail/proc.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_DETAIL_UTILITY_PROC_HPP 7 | #define METALL_DETAIL_UTILITY_PROC_HPP 8 | 9 | #include 10 | 11 | #ifdef _GNU_SOURCE 12 | #include 13 | #define SUPPORT_GET_CPU_NO true 14 | #else 15 | #define SUPPORT_GET_CPU_NO false 16 | #endif 17 | 18 | #include 19 | 20 | namespace metall::mtlldetail { 21 | 22 | /// \brief Returns the number of the logical CPU core on which the calling 23 | /// thread is currently executing. 24 | inline unsigned int get_cpu_no() { 25 | #if SUPPORT_GET_CPU_NO 26 | 27 | const int cpu = ::sched_getcpu(); 28 | if (cpu == -1) { 29 | return 0; 30 | } 31 | return cpu; 32 | 33 | #else 34 | 35 | #ifdef METALL_VERBOSE_SYSTEM_SUPPORT_WARNING 36 | #warning "CPU number is always 0" 37 | #endif 38 | return 0; 39 | 40 | #endif 41 | } 42 | 43 | /// \brief Returns the number of the logical CPU cores on the system. 44 | inline unsigned int get_num_cpus() { 45 | #if SUPPORT_GET_CPU_NO 46 | return ::get_nprocs_conf(); 47 | #else 48 | return std::thread::hardware_concurrency(); 49 | #endif 50 | } 51 | 52 | } // namespace metall::mtlldetail 53 | #endif // METALL_DETAIL_UTILITY_PROC_HPP 54 | -------------------------------------------------------------------------------- /include/metall/detail/soft_dirty_page.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_DETAIL_UTILITY_SOFT_DIRTY_PAGE_HPP 7 | #define METALL_DETAIL_UTILITY_SOFT_DIRTY_PAGE_HPP 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | namespace metall::mtlldetail { 16 | 17 | inline bool reset_soft_dirty_bit() { 18 | std::ofstream ofs("/proc/self/clear_refs"); 19 | if (!ofs.is_open()) { 20 | logger::out(logger::level::error, __FILE__, __LINE__, 21 | "Cannot open file clear_refs"); 22 | return false; 23 | } 24 | 25 | ofs << "4"; 26 | ofs.close(); 27 | 28 | if (!ofs) { 29 | logger::out(logger::level::error, __FILE__, __LINE__, 30 | "Cannot write to /proc/self/clear_refs"); 31 | return false; 32 | } 33 | 34 | return true; 35 | } 36 | 37 | inline constexpr bool check_soft_dirty_page(const uint64_t pagemap_value) { 38 | return (pagemap_value >> 55ULL) & 1ULL; 39 | } 40 | 41 | inline constexpr bool check_swapped_page(const uint64_t pagemap_value) { 42 | return (pagemap_value >> 62ULL) & 1ULL; 43 | } 44 | 45 | inline constexpr bool check_present_page(const uint64_t pagemap_value) { 46 | return (pagemap_value >> 63ULL) & 1ULL; 47 | } 48 | 49 | } // namespace metall::mtlldetail 50 | 51 | #endif // METALL_DETAIL_UTILITY_SOFT_DIRTY_PAGE_HPP 52 | -------------------------------------------------------------------------------- /include/metall/detail/time.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_DETAIL_UTILITY_TIME_HPP 7 | #define METALL_DETAIL_UTILITY_TIME_HPP 8 | 9 | #include 10 | 11 | namespace metall::mtlldetail { 12 | 13 | inline std::chrono::high_resolution_clock::time_point elapsed_time_sec() { 14 | return std::chrono::high_resolution_clock::now(); 15 | } 16 | 17 | inline double elapsed_time_sec( 18 | const std::chrono::high_resolution_clock::time_point &tic) { 19 | auto duration_time = std::chrono::high_resolution_clock::now() - tic; 20 | return static_cast( 21 | std::chrono::duration_cast( 22 | duration_time) 23 | .count()) / 24 | 1e6; 25 | } 26 | 27 | } // namespace metall::mtlldetail 28 | #endif // METALL_DETAIL_UTILITY_TIME_HPP 29 | -------------------------------------------------------------------------------- /include/metall/detail/uuid.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_DETAIL_UTILITY_UUID_HPP 7 | #define METALL_DETAIL_UTILITY_UUID_HPP 8 | 9 | #include 10 | #include 11 | #include 12 | 13 | namespace metall::mtlldetail { 14 | 15 | using uuid = boost::uuids::uuid; 16 | using uuid_random_generator = boost::uuids::random_generator; 17 | 18 | } // namespace metall::mtlldetail 19 | 20 | #endif // METALL_DETAIL_UTILITY_UUID_HPP -------------------------------------------------------------------------------- /include/metall/json/json.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_JSON_JSON_HPP 7 | #define METALL_JSON_JSON_HPP 8 | 9 | #include 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | /// \example json_create.cpp 24 | /// This is an example of how to create a JSON object with Metall. 25 | 26 | /// \example json_open.cpp 27 | /// This is an example of how to open an already created JSON object. 28 | 29 | #endif // METALL_JSON_JSON_HPP 30 | -------------------------------------------------------------------------------- /include/metall/json/object.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_OBJECT_HPP 7 | #define METALL_OBJECT_HPP 8 | 9 | #include 10 | #include 11 | 12 | namespace metall::json { 13 | 14 | /// \brief JSON object. 15 | /// An object is a table key and value pairs. 16 | /// The order of key-value pairs depends on the implementation. 17 | #ifdef DOXYGEN_SKIP 18 | template > 19 | #else 20 | template 21 | #endif 22 | class object : public jsndtl::compact_object { 23 | using jsndtl::compact_object::compact_object; 24 | }; 25 | 26 | /// \brief Swap value instances. 27 | template 28 | inline void swap(object &lhd, 29 | object &rhd) noexcept { 30 | lhd.swap(rhd); 31 | } 32 | 33 | namespace jsndtl { 34 | 35 | /// \brief Provides 'equal' calculation for other object types that have the 36 | /// same interface as the object class. 37 | template 38 | inline bool general_object_equal( 39 | const object &object, 40 | const other_object_type &other_object) noexcept { 41 | return general_compact_object_equal(object, other_object); 42 | } 43 | 44 | } // namespace jsndtl 45 | 46 | } // namespace metall::json 47 | 48 | #endif // METALL_OBJECT_HPP 49 | -------------------------------------------------------------------------------- /include/metall/json/parse.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_JSON_PARSE_HPP 7 | #define METALL_JSON_PARSE_HPP 8 | 9 | #include 10 | #include 11 | #include 12 | 13 | #include 14 | 15 | namespace metall::json { 16 | 17 | namespace { 18 | namespace bj = boost::json; 19 | } 20 | 21 | /// \brief Parses a JSON represented as a string. 22 | /// \tparam allocator_type An allocator type. 23 | /// \param input_json_string An input JSON string. 24 | /// \param allocator An allocator object. 25 | /// \return Returns a constructed value. 26 | #ifdef DOXYGEN_SKIP 27 | template > 28 | inline value parse( 29 | std::string_view input_json_string, 30 | const allocator_type &allocator = allocator_type()) 31 | #else 32 | template 33 | inline value parse(std::string_view input_json_string, 34 | const allocator_type &allocator) 35 | #endif 36 | { 37 | bj::error_code ec; 38 | auto bj_value = bj::parse(input_json_string.data(), ec); 39 | if (ec) { 40 | std::cerr << "Failed to parse: " << ec.message() << std::endl; 41 | return value{allocator}; 42 | } 43 | 44 | return metall::json::value_from(std::move(bj_value), allocator); 45 | } 46 | 47 | } // namespace metall::json 48 | 49 | #endif // METALL_JSON_PARSE_HPP 50 | -------------------------------------------------------------------------------- /include/metall/json/serialize.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_JSON_SERIALIZE_HPP 7 | #define METALL_JSON_SERIALIZE_HPP 8 | 9 | #include 10 | 11 | #include 12 | #include 13 | 14 | namespace metall::json { 15 | 16 | namespace { 17 | namespace mc = metall::container; 18 | namespace bj = boost::json; 19 | } // namespace 20 | 21 | template 22 | std::string serialize(const value &input) { 23 | return bj::serialize(value_to(input)); 24 | } 25 | 26 | template 27 | std::string serialize(const object &input) { 28 | bj::object object; 29 | for (const auto &elem : input) { 30 | object[elem.key().data()] = value_to(elem.value()); 31 | } 32 | return bj::serialize(object); 33 | } 34 | 35 | template 36 | std::string serialize(const array &input) { 37 | bj::array array; 38 | for (const auto &elem : input) { 39 | array.emplace_back(value_to(elem)); 40 | } 41 | return bj::serialize(array); 42 | } 43 | 44 | template 45 | std::string serialize( 46 | const basic_string &input) { 47 | return input.data(); 48 | } 49 | 50 | template 51 | std::ostream &operator<<(std::ostream &os, const value &val) { 52 | os << serialize(val); 53 | return os; 54 | } 55 | 56 | template 57 | std::ostream &operator<<(std::ostream &os, const object &obj) { 58 | os << serialize(obj); 59 | return os; 60 | } 61 | 62 | template 63 | std::ostream &operator<<(std::ostream &os, const array &arr) { 64 | os << serialize(arr); 65 | return os; 66 | } 67 | 68 | } // namespace metall::json 69 | 70 | #endif // METALL_JSON_SERIALIZE_HPP 71 | -------------------------------------------------------------------------------- /include/metall/json/string.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_JSON_STRING_HPP 7 | #define METALL_JSON_STRING_HPP 8 | 9 | #include 10 | 11 | #include 12 | 13 | namespace metall::json::jsndtl { 14 | 15 | template 17 | inline bool general_string_equal( 18 | const basic_string &string, 19 | const other_string_type &other_string) noexcept { 20 | return std::strcmp(string.c_str(), other_string.c_str()) == 0; 21 | } 22 | 23 | } // namespace metall::json::jsndtl 24 | 25 | #endif // METALL_JSON_STRING_HPP 26 | -------------------------------------------------------------------------------- /include/metall/kernel/bin_number_manager.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_DETAIL_KERNEL_BIN_NUMBER_MANAGER_HPP 7 | #define METALL_DETAIL_KERNEL_BIN_NUMBER_MANAGER_HPP 8 | 9 | #include 10 | #include 11 | 12 | #include 13 | #include 14 | 15 | namespace metall { 16 | namespace kernel { 17 | 18 | /// \brief Bin number manager 19 | template 20 | class bin_number_manager { 21 | public: 22 | using size_type = std::size_t; 23 | 24 | private: 25 | using object_size_mngr = object_size_manager; 26 | static constexpr size_type k_num_small_bins = 27 | object_size_mngr::num_small_sizes(); 28 | static constexpr size_type k_num_large_bins = 29 | object_size_mngr::num_large_sizes(); 30 | static constexpr size_type k_num_bins = k_num_small_bins + k_num_large_bins; 31 | 32 | public: 33 | using bin_no_type = typename mdtl::unsigned_variable_type::type; 34 | 35 | bin_number_manager() = delete; 36 | ~bin_number_manager() = delete; 37 | bin_number_manager(const bin_number_manager &) = delete; 38 | bin_number_manager(bin_number_manager &&) noexcept = delete; 39 | bin_number_manager &operator=(const bin_number_manager &) = delete; 40 | bin_number_manager &operator=(bin_number_manager &&) noexcept = delete; 41 | 42 | static constexpr size_type num_bins() noexcept { return k_num_bins; } 43 | 44 | static constexpr size_type num_small_bins() noexcept { 45 | return k_num_small_bins; 46 | } 47 | 48 | static constexpr size_type num_large_bins() noexcept { 49 | return k_num_large_bins; 50 | } 51 | 52 | static constexpr bin_no_type to_bin_no(const size_type size) noexcept { 53 | return object_size_mngr::index(size); 54 | } 55 | 56 | static constexpr size_type to_object_size(const bin_no_type bin_no) noexcept { 57 | return object_size_mngr::at(bin_no); 58 | } 59 | }; 60 | 61 | } // namespace kernel 62 | } // namespace metall 63 | #endif // METALL_DETAIL_KERNEL_BIN_NUMBER_MANAGER_HPP 64 | -------------------------------------------------------------------------------- /include/metall/kernel/manager_kernel_fwd.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_KERNEL_MANAGER_KERNEL_FWD_HPP 7 | #define METALL_KERNEL_MANAGER_KERNEL_FWD_HPP 8 | 9 | namespace metall { 10 | namespace kernel { 11 | 12 | /// \brief Manager kernel class version 0 13 | /// \tparam storage Storage manager. 14 | /// \tparam segment_storage Segment storage manager. 15 | /// \tparam chunk_no_type Type of chunk number 16 | /// \tparam chunk_size Size of single chunk in byte 17 | template 19 | class manager_kernel; 20 | 21 | } // namespace kernel 22 | } // namespace metall 23 | 24 | #endif // METALL_KERNEL_MANAGER_KERNEL_FWD_HPP 25 | -------------------------------------------------------------------------------- /include/metall/kernel/manager_kernel_profile_impl.ipp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_DETAIL_KERNEL_MANAGER_KERNEL_PROFILE_IMPL_IPP 7 | #define METALL_DETAIL_KERNEL_MANAGER_KERNEL_PROFILE_IMPL_IPP 8 | 9 | #include 10 | 11 | namespace metall { 12 | namespace kernel { 13 | 14 | template 15 | template 16 | void manager_kernel::profile(out_stream_type *log_out) { 17 | m_segment_memory_allocator.profile(log_out); 18 | } 19 | 20 | } // namespace kernel 21 | } // namespace metall 22 | 23 | #endif // METALL_DETAIL_KERNEL_MANAGER_KERNEL_PROFILE_IMPL_IPP 24 | -------------------------------------------------------------------------------- /include/metall/kernel/segment_header.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_KERNEL_SEGMENT_HEADER_HPP 7 | #define METALL_KERNEL_SEGMENT_HEADER_HPP 8 | 9 | namespace metall::kernel { 10 | 11 | struct segment_header { 12 | void *manager_kernel_address{nullptr}; 13 | 14 | segment_header() = default; 15 | ~segment_header() noexcept { manager_kernel_address = nullptr; } 16 | }; 17 | 18 | } // namespace metall::kernel 19 | 20 | #endif // METALL_KERNEL_SEGMENT_HEADER_HPP 21 | -------------------------------------------------------------------------------- /include/metall/logger_interface.h: -------------------------------------------------------------------------------- 1 | #ifndef METALL_LOGGER_INTERFACE_H 2 | #define METALL_LOGGER_INTERFACE_H 3 | 4 | #ifdef __cplusplus 5 | extern "C" { 6 | #endif 7 | 8 | /// \brief Log message level 9 | typedef enum metall_log_level { 10 | /// \brief Critical logger message 11 | metall_critical = 5, 12 | /// \brief Error logger message 13 | metall_error = 4, 14 | /// \brief Warning logger message 15 | metall_warning = 3, 16 | /// \brief Info logger message 17 | metall_info = 2, 18 | /// \brief Debug logger message 19 | metall_debug = 1, 20 | /// \brief Verbose (lowest priority) logger message 21 | metall_verbose = 0, 22 | } metall_log_level; 23 | 24 | 25 | /// \brief Implementation of logging behaviour. 26 | /// 27 | /// If METALL_LOGGER_EXTERN_C is defined this function does not have an implementation 28 | /// and is intended to be implemented by consuming applications. 29 | /// 30 | /// If it is not defined the default implementation is in metall/logger.hpp. 31 | void metall_log(metall_log_level lvl, const char *file_name, size_t line_no, const char *message); 32 | 33 | #ifdef __cplusplus 34 | } 35 | #endif 36 | 37 | #endif // METALL_LOGGER_INTERFACE_H 38 | -------------------------------------------------------------------------------- /include/metall/offset_ptr.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_OFFSET_PTR_HPP 7 | #define METALL_OFFSET_PTR_HPP 8 | 9 | #include 10 | #include 11 | 12 | namespace metall { 13 | 14 | /// \tparam T A type. 15 | /// \brief Holds an offset between the address pointing at and itself. 16 | template 17 | using offset_ptr = boost::interprocess::offset_ptr; 18 | 19 | /// \brief Convert a offset pointer to the corresponding raw pointer 20 | using boost::interprocess::ipcdetail::to_raw_pointer; 21 | } // namespace metall 22 | 23 | /// \example offset_pointer.cpp 24 | /// This is an example of how to use offset_pointer. 25 | 26 | #endif // METALL_OFFSET_PTR_HPP 27 | -------------------------------------------------------------------------------- /include/metall/tags.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_TAGS_HPP 7 | #define METALL_TAGS_HPP 8 | 9 | #include 10 | 11 | namespace metall { 12 | 13 | /// \brief Tag type to create the segment always. 14 | /// The existing segment with the same name is over written. 15 | struct create_only_t {}; 16 | 17 | /// \brief Tag to create the segment always. 18 | /// The existing segment with the same name is over written. 19 | [[maybe_unused]] static const create_only_t create_only{}; 20 | 21 | /// \brief Tag type to open an already created segment. 22 | struct open_only_t {}; 23 | 24 | /// \brief Tag to open an already created segment. 25 | [[maybe_unused]] static const open_only_t open_only{}; 26 | 27 | /// \brief Tag type to open an already created segment as read only. 28 | struct open_read_only_t {}; 29 | 30 | /// \brief Tag to open an already created segment as read only. 31 | [[maybe_unused]] static const open_read_only_t open_read_only{}; 32 | 33 | /// \brief Tag to construct anonymous instances. 34 | [[maybe_unused]] static const mtlldetail::anonymous_instance_t 35 | *anonymous_instance = nullptr; 36 | 37 | /// \brief Tag to construct a unique instance of a type 38 | [[maybe_unused]] static const mtlldetail::unique_instance_t *unique_instance = 39 | nullptr; 40 | } // namespace metall 41 | 42 | #endif // METALL_TAGS_HPP 43 | -------------------------------------------------------------------------------- /include/metall/utility/fallback_allocator_adaptor.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_UTILITY_FALLBACK_ALLOCATOR_ADAPTOR_HPP 7 | #define METALL_UTILITY_FALLBACK_ALLOCATOR_ADAPTOR_HPP 8 | 9 | #include "metall/container/fallback_allocator.hpp" 10 | 11 | #warning \ 12 | "This file is deprecated. Use container/fallback_allocator.hpp instead." 13 | 14 | namespace metall::utility { 15 | template 16 | using fallback_allocator_adaptor = 17 | metall::container::fallback_allocator_adaptor; 18 | } 19 | 20 | #endif -------------------------------------------------------------------------------- /include/metall/utility/filesystem.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_UTILITY_FILESYSTEM_HPP 7 | #define METALL_UTILITY_FILESYSTEM_HPP 8 | 9 | #include 10 | 11 | namespace metall::utility::filesystem { 12 | 13 | /// \brief Remove a file or directory 14 | /// \return Upon successful completion, returns true; otherwise, false is 15 | /// returned. If the file or directory does not exist, true is returned. 16 | inline bool remove(std::string_view path) { 17 | return metall::mtlldetail::remove_file(path.data()); 18 | } 19 | 20 | } // namespace metall::utility::filesystem 21 | 22 | #endif // METALL_UTILITY_FILESYSTEM_HPP 23 | -------------------------------------------------------------------------------- /include/metall/utility/hash.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_UTILITY_HASH_HPP 7 | #define METALL_UTILITY_HASH_HPP 8 | 9 | #include 10 | 11 | namespace metall::utility { 12 | 13 | /// \brief Hash a value of type T. 14 | /// \tparam T Data type to hash. 15 | /// If void is specified, the hash data type is determined by () operator. 16 | /// \tparam seed A seed value used for hashing. 17 | template 18 | using hash = metall::mtlldetail::hash; 19 | 20 | /// \brief Hash function for std::string-compatible string container. 21 | /// \tparam seed A seed value used for hashing. 22 | template 23 | using str_hash = metall::mtlldetail::str_hash; 24 | 25 | } // namespace metall::utility 26 | 27 | #endif // METALL_UTILITY_HASH_HPP 28 | -------------------------------------------------------------------------------- /include/metall/utility/metall_mpi_datastore.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_UTILITY_METALL_MPI_DATASTORE_HPP 7 | #define METALL_UTILITY_METALL_MPI_DATASTORE_HPP 8 | 9 | #include 10 | 11 | /// \namespace metall::utility::mpi_datastore 12 | /// \brief Namespace for MPI datastore 13 | namespace metall::utility::mpi_datastore { 14 | 15 | /// \brief Makes a path of a root directory. 16 | /// The MPI ranks that share the same storage space create their data stores 17 | /// under the root directory. \param root_dir_prefix A prefix of a root 18 | /// directory path. \return A path of a root directory. 19 | inline std::string make_root_dir_path(const std::string &root_dir_prefix) { 20 | return root_dir_prefix + "/"; 21 | } 22 | 23 | /// \brief Makes the data store path of a MPI rank. 24 | /// \param root_dir_prefix A prefix of a root directory path. 25 | /// \param rank A MPI rank. 26 | /// \return A path to a local data store directory. 27 | /// The path can be passed to, for example, metall::manager to perform 28 | /// operations. 29 | inline std::string make_local_dir_path(const std::string &root_dir_prefix, 30 | const int rank) { 31 | return make_root_dir_path(root_dir_prefix) + "/subdir-" + 32 | std::to_string(rank); 33 | } 34 | 35 | } // namespace metall::utility::mpi_datastore 36 | 37 | #endif // METALL_UTILITY_METALL_MPI_DATASTORE_HPP 38 | -------------------------------------------------------------------------------- /include/metall/utility/mutex.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_UTILITY_MUTEX_HPP 7 | #define METALL_UTILITY_MUTEX_HPP 8 | 9 | #include 10 | #include 11 | 12 | namespace metall::utility { 13 | 14 | /// \namespace metall::utility::mutex 15 | /// \brief Namespace for mutex 16 | namespace mutex { 17 | 18 | /// \brief A utility function that returns a mutex lock allocated as a static 19 | /// object. This is an experimental implementation. Example: { // Mutex region 20 | /// const int bank_index = hash(key) % num_banks; 21 | /// auto guard = metall::utility::mutex::mutex_lock(bank_index); 22 | /// // do some mutex work 23 | /// } 24 | template 25 | inline std::unique_lock mutex_lock(const std::size_t index) { 26 | static std::mutex mutexes[num_banks]; 27 | assert(index < num_banks); 28 | return std::unique_lock(mutexes[index]); 29 | } 30 | 31 | } // namespace mutex 32 | } // namespace metall::utility 33 | 34 | /// \example static_mutex.cpp 35 | /// This is an example of how to use the mutex_lock function. 36 | 37 | #endif // METALL_UTILITY_MUTEX_HPP 38 | -------------------------------------------------------------------------------- /include/metall/version.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_VERSION_HPP 7 | #define METALL_VERSION_HPP 8 | 9 | /// \brief Metall version. 10 | /// Metall follows the Semantic Versioning 2.0.0. 11 | /// \details 12 | /// \code 13 | /// METALL_VERSION / 100000 // the major version. 14 | /// METALL_VERSION / 100 % 1000 // the minor version. 15 | /// METALL_VERSION % 100 // the patch level. 16 | /// \endcode 17 | #define METALL_VERSION 3000 18 | 19 | namespace metall { 20 | /// \brief Variable type to handle a version data. 21 | using version_type = uint32_t; 22 | static_assert(std::numeric_limits::max() >= METALL_VERSION, 23 | "version_type cannot handle the current version"); 24 | 25 | #ifndef DOXYGEN_SKIP 26 | /// \namespace metall::ver_detail 27 | /// \brief Namespace for the details of Metall version 28 | namespace ver_detail { 29 | static constexpr version_type k_error_version = 0; 30 | static_assert(k_error_version != METALL_VERSION, 31 | "The current version is equal to a error number"); 32 | } // namespace ver_detail 33 | #endif // DOXYGEN_SKIP 34 | 35 | /// \brief Converts a raw version number to a std::string with format 36 | /// 'MAJOR.MINOR.PATCH'. \details \code 37 | /// // Example usage 38 | /// std::string ver_string = to_version_string(METALL_VERSION); 39 | /// \endcode 40 | /// \param version A version number. 41 | /// \return A version string. 42 | inline std::string to_version_string(const version_type version) { 43 | return std::to_string(version / 100000) + "." + 44 | std::to_string(version / 100 % 1000) + "." + 45 | std::to_string(version % 100); 46 | } 47 | } // namespace metall 48 | 49 | #endif // METALL_VERSION_HPP 50 | -------------------------------------------------------------------------------- /mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: Metall 2 | site_author: 'Keita Iwabuchi (LLNL), Roger A Pearce (LLNL), and Maya B Gokhale (LLNL)' 3 | copyright: Copyright 2023 Lawrence Livermore National Security, LLC 4 | 5 | docs_dir: 'docs/readthedocs' 6 | theme: 'readthedocs' 7 | extra_css: 8 | - ./css/custom.css 9 | 10 | repo_url: https://github.com/LLNL/metall 11 | edit_uri: 'tree/develop/docs/readthedocs' 12 | 13 | nav: 14 | - 'HOME': 'index.md' 15 | - 'BASICS': 16 | - 'Getting Started': 'basics/getting_started.md' 17 | - 'Compile-time Options': 'basics/compile_time_options.md' 18 | - 'ADVANCED BUILD': 19 | - 'Build Executables': 'advanced_build/cmake.md' 20 | - 'Use Umap': 'advanced_build/umap.md' 21 | - 'Test Metall': 'advanced_build/test.md' 22 | - 'Metall 101': 23 | - 'Introduction': 'detail/introduction.md' 24 | - 'API': 'detail/api.md' 25 | - 'Data Store ls': 'detail/ls.md' 26 | - 'Pointers in Persistent Memory': 'detail/pointer.md' 27 | - 'Persistence Policy': 'detail/persistence_policy.md' 28 | - 'Snapshot': 'detail/snapshot.md' 29 | - 'Example': 'detail/example.md' 30 | - 'Internal Architecture': 'detail/internal_architecture.md' 31 | - 'External': 32 | - 'Open Source Projects Using Metall': 'external/list.md' 33 | - 'Use Ripples with Metall': 'external/ripples.md' 34 | - 'ABOUT': 35 | - 'Publication': 'about/publication.md' 36 | - 'License and Notice': 'about/license_notice.md' 37 | -------------------------------------------------------------------------------- /scripts/release_test/run_tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Runs all test cases with all GCC and boost combinations 4 | # Usage: 5 | # Run this script from the root directory of Metall 6 | # sh ./scripts/release_test/run_tests.sh 7 | 8 | source ./scripts/test_utility.sh 9 | 10 | GCC_VERSIONS=('14.2.0' '14.1.0' '13.3.0' '13.2.0' '13.1.0' '12.4.0' '12.3.0' '12.2.0' '12.1.0' '11.5.0' '11.4.0' '11.3.0' '11.2.0' '11.1.0' '10.5.0' '10.4.0' '10.3.0' '10.2.0' '10.1.0' '9.5.0' '9.4.0' '9.3.0' '9.2.0' '9.1.0' '8.5.0' '8.4.0' '8.3.0' '8.2.0' '8.1.0') 11 | BOOST_VERSiONS=('1.86.0' '1.85.0' '1.84.0' '1.83.0' '1.82.0' '1.81.0' '1.80.0') 12 | 13 | for GCC_VER in "${GCC_VERSIONS[@]}"; do 14 | for BOOST_VER in "${BOOST_VERSiONS[@]}"; do 15 | export METALL_TEST_DIR="/dev/shm/metall_test_gcc${GCC_VER}_boost${BOOST_VER}" 16 | export METALL_BUILD_DIR="/dev/shm/metall_test_build_gcc${GCC_VER}_boost${BOOST_VER}" 17 | 18 | spack load gcc@${GCC_VER} 19 | spack load boost@${BOOST_VER} 20 | 21 | or_die sh ./scripts/release_test/full_build_and_test.sh 22 | done 23 | done 24 | 25 | echo "Passed All Tests!!" 26 | -------------------------------------------------------------------------------- /scripts/sanitizer/run_sanitizers.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This script builds and runs example programs with various sanitizers. 4 | # Usage example: 5 | # cd metall # Metall root directory 6 | # sh scripts/sanitizer/run_sanitizers.sh 7 | 8 | # Change the following variables to your own settings 9 | CC=clang 10 | CXX=clang++ 11 | BOOST_ROOT="${HOME}/local/boost" 12 | METALL_ROOT=".." 13 | 14 | LOG_FILE=out-sanitizer.log 15 | 16 | exec_cmd () { 17 | echo "" >> ${LOG_FILE} 18 | echo "------------------------------------------------------------" >> ${LOG_FILE} 19 | echo "Command: " "$@" | tee -a ${LOG_FILE} 20 | "$@" >> ${LOG_FILE} 2>&1 21 | echo "------------------------------------------------------------" >> ${LOG_FILE} 22 | echo "" >> ${LOG_FILE} 23 | } 24 | 25 | build () { 26 | local target=$1 27 | exec_cmd ${CXX} -I ${BOOST_ROOT} -I ${METALL_ROOT}/include -std=c++17 ${SANITIZER_BUILD_ARGS} ${target} -DMETALL_DEFAULT_CAPACITY=$((1024*1024*500)) 28 | } 29 | 30 | run_sanitizer () { 31 | SANITIZER_BUILD_ARGS=$1 32 | 33 | # build "${METALL_ROOT}/example/simple.cpp" 34 | # ./a.out 2>&1 | tee -a ${LOG_FILE} 35 | # rm -f a.out 36 | 37 | build "${METALL_ROOT}/example/concurrent.cpp" 38 | echo "Running concurrent program" 39 | ./a.out 2>&1 | tee -a ${LOG_FILE} 40 | rm -f a.out 41 | } 42 | 43 | main() { 44 | mkdir build 45 | cd build 46 | rm -rf ${LOG_FILE} 47 | 48 | echo "" 49 | echo "=====================================================================" 50 | echo "Build with address sanitizer and leak sanitizer" 51 | run_sanitizer "-fsanitize=address -fno-omit-frame-pointer -O1 -g" 52 | 53 | echo "" 54 | echo "=====================================================================" 55 | echo "Build with thread sanitizer" 56 | run_sanitizer "-fsanitize=thread -fPIE -pie -O1 -g" 57 | 58 | echo "" 59 | echo "=====================================================================" 60 | echo "Build with memory sanitizer" 61 | run_sanitizer "-fsanitize=memory -fsanitize-memory-track-origins -fPIE -pie -fno-omit-frame-pointer -g -O2" 62 | 63 | echo "" 64 | echo "=====================================================================" 65 | echo "Build with undefined behavior sanitizer" 66 | run_sanitizer "-fsanitize=undefined -O1 -g" 67 | } 68 | 69 | main "$@" -------------------------------------------------------------------------------- /scripts/test_utility.sh: -------------------------------------------------------------------------------- 1 | ####################################### 2 | # Executes a command. 3 | # Dies (exit) the shell if the command has been failed. 4 | # Arguments: 5 | # A command with its arguments to execute. 6 | # Outputs: STDOUT and STDERR 7 | ####################################### 8 | or_die () { 9 | echo "" 10 | echo "Command: " "$@" 11 | echo ">>>>>>>>>>" 12 | "$@" 13 | local status=$? 14 | if [[ $status != 0 ]] ; then 15 | echo ERROR $status command: "$@" 16 | exit $status 17 | fi 18 | echo "<<<<<<<<<<" 19 | } 20 | 21 | ####################################### 22 | # Executes a command. 23 | # Arguments: 24 | # A command with its arguments to execute. 25 | # Outputs: STDOUT and STDERR 26 | ####################################### 27 | exec_cmd () { 28 | echo "" 29 | echo "Command: " "$@" 30 | echo ">>>>>>>>>>" 31 | "$@" 32 | echo "<<<<<<<<<<" 33 | } 34 | 35 | ####################################### 36 | # As multiple CI jobs could run on the same machine 37 | # generate an unique test dir for each CI job 38 | # Globals: 39 | # METALL_TEST_DIR (option, defined if not given) 40 | # Outputs: STDOUT 41 | # 42 | # MEMO: setup a filesystem in local 43 | # truncate --size 512m XFSfile 44 | # mkfs.xfs -m crc=1 -m reflink=1 XFSfile 45 | # mkdir XFSmountpoint 46 | # mount -o loop XFSfile XFSmountpoint 47 | # xfs_info XFSmountpoint 48 | # cd XFSmountpoint 49 | ####################################### 50 | setup_test_dir() { 51 | if [[ -z "${METALL_TEST_DIR}" ]]; then 52 | METALL_TEST_DIR="/tmp/${RANDOM}" 53 | fi 54 | 55 | # mkdir -p ${METALL_TEST_DIR} # Metall creates automatically if the directory does not exist 56 | echo "Store test data to ${METALL_TEST_DIR}" 57 | } 58 | 59 | ####################################### 60 | # Show some system information 61 | # Outputs: STDOUT and STDERR 62 | ####################################### 63 | show_system_info() { 64 | exec_cmd df -h 65 | exec_cmd df -ih 66 | exec_cmd free -g 67 | exec_cmd uname -r 68 | } -------------------------------------------------------------------------------- /src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | if (BUILD_UTILITY) 2 | add_metall_executable(datastore_ls datastore_ls.cpp) 3 | install(TARGETS datastore_ls RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) 4 | 5 | add_metall_executable(mpi_datastore_ls mpi_datastore_ls.cpp) 6 | install(TARGETS mpi_datastore_ls RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) 7 | endif () 8 | 9 | if (BUILD_C) 10 | if (NOT Boost_FOUND) 11 | return() 12 | endif () 13 | 14 | if (BUILD_SHARED_LIBS) 15 | message(STATUS "Building metall_c as a shared library") 16 | else () 17 | message(STATUS "Building metall_c as a static library") 18 | endif () 19 | # 'BUILD_SHARED_LIBS' variable is used to determine 20 | # whether to build a shared library or a static library. 21 | add_library(metall_c metall_c.cpp) 22 | get_target_property(PIC_metall_c metall_c POSITION_INDEPENDENT_CODE) 23 | if (PIC_metall_c) 24 | message(STATUS "POSITION_INDEPENDENT_CODE for metall_c : ON") 25 | else () 26 | message(STATUS "POSITION_INDEPENDENT_CODE for metall_c : OFF") 27 | endif () 28 | 29 | add_library(${PROJECT_NAME}::metall_c ALIAS metall_c) 30 | 31 | target_include_directories(metall_c PUBLIC 32 | $ 33 | $) 34 | 35 | common_setup_for_metall_executable(metall_c) 36 | 37 | install(TARGETS metall_c 38 | EXPORT ${PROJECT_NAME}_Targets 39 | ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} 40 | LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} 41 | RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) 42 | endif () -------------------------------------------------------------------------------- /src/datastore_ls.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #include 7 | #include 8 | 9 | #include 10 | 11 | int main(int argc, char *argv[]) { 12 | if (argc == 1) { 13 | std::cerr << "Empty datastore path" << std::endl; 14 | std::abort(); 15 | } 16 | 17 | const std::filesystem::path datastore_path = argv[1]; 18 | 19 | metall::utility::ls_named_object(datastore_path); 20 | std::cout << std::endl; 21 | 22 | metall::utility::ls_unique_object(datastore_path); 23 | std::cout << std::endl; 24 | 25 | metall::utility::ls_anonymous_object(datastore_path); 26 | std::cout << std::endl; 27 | 28 | return 0; 29 | } -------------------------------------------------------------------------------- /src/mpi_datastore_ls.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | #include 11 | #include 12 | 13 | int main(int argc, char *argv[]) { 14 | if (argc <= 1) { 15 | std::cerr << "Empty datastore path" << std::endl; 16 | std::abort(); 17 | } 18 | 19 | const std::filesystem::path datastore_path = argv[1]; 20 | const int mpi_rank = (argc < 3) ? 0 : std::stoi(argv[2]); 21 | 22 | const auto local_datastore_path = 23 | metall::utility::mpi_datastore::make_local_dir_path(datastore_path, 24 | mpi_rank); 25 | 26 | if (!metall::manager::consistent(local_datastore_path)) { 27 | std::cerr << "Inconsistent datastore or invalid datastore path" 28 | << std::endl; 29 | std::abort(); 30 | } 31 | 32 | metall::utility::ls_named_object(local_datastore_path); 33 | std::cout << std::endl; 34 | 35 | metall::utility::ls_unique_object(local_datastore_path); 36 | std::cout << std::endl; 37 | 38 | metall::utility::ls_anonymous_object(local_datastore_path); 39 | std::cout << std::endl; 40 | 41 | return 0; 42 | } -------------------------------------------------------------------------------- /test/CMakeListsGTest.txt.in: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.8.2) 2 | 3 | project(googletest-download NONE) 4 | 5 | include(ExternalProject) 6 | ExternalProject_Add(googletest 7 | GIT_REPOSITORY https://github.com/google/googletest.git 8 | GIT_TAG main 9 | SOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}/googletest-src" 10 | BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/googletest-build" 11 | CONFIGURE_COMMAND "" 12 | BUILD_COMMAND "" 13 | INSTALL_COMMAND "" 14 | TEST_COMMAND "" 15 | ) -------------------------------------------------------------------------------- /test/container/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_metall_test_executable(concurrent_map_test concurrent_map_test.cpp) 2 | 3 | add_metall_test_executable(stl_allocator_test stl_allocator_test.cpp) 4 | 5 | add_metall_test_executable(fallback_allocator_test fallback_allocator_test.cpp) 6 | 7 | add_metall_test_executable(string_key_store_test string_key_store_test.cpp) 8 | 9 | add_subdirectory(json) -------------------------------------------------------------------------------- /test/container/json/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | if (Boost_VERSION_STRING VERSION_GREATER_EQUAL "1.75") 2 | add_metall_test_executable(json_value json_value.cpp) 3 | add_metall_test_executable(json_object json_object.cpp) 4 | add_metall_test_executable(json_array json_array.cpp) 5 | endif () -------------------------------------------------------------------------------- /test/gtest_main.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #include 7 | #include "gtest/gtest.h" 8 | 9 | GTEST_API_ int main(int argc, char **argv) { 10 | printf("Running main() from %s\n", __FILE__); 11 | testing::InitGoogleTest(&argc, argv); 12 | return RUN_ALL_TESTS(); 13 | } -------------------------------------------------------------------------------- /test/kernel/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_metall_test_executable(attributed_object_directory_test attributed_object_directory_test.cpp) 2 | 3 | add_metall_test_executable(bin_manager_test bin_manager_test.cpp) 4 | 5 | add_metall_test_executable(bin_directory_test bin_directory_test.cpp) 6 | 7 | add_metall_test_executable(multilayer_bitset_test multilayer_bitset_test.cpp) 8 | 9 | add_metall_test_executable(chunk_directory_test chunk_directory_test.cpp) 10 | 11 | add_metall_test_executable(object_cache_test object_cache_test.cpp) 12 | 13 | add_metall_test_executable(manager_test manager_test.cpp) 14 | 15 | add_metall_test_executable(manager_test_single_thread manager_test.cpp) 16 | target_compile_definitions(manager_test_single_thread PRIVATE "METALL_DISABLE_CONCURRENCY") 17 | 18 | add_metall_test_executable(snapshot_test snapshot_test.cpp) 19 | 20 | add_metall_test_executable(copy_datastore_test copy_datastore_test.cpp) 21 | 22 | include(setup_omp) 23 | if (OpenMP_CXX_FOUND) 24 | add_metall_test_executable(manager_multithread_test manager_multithread_test.cpp) 25 | setup_omp_target(manager_multithread_test) 26 | else() 27 | MESSAGE(STATUS "OpenMP is not found. Will not run multi-thread test.") 28 | endif() 29 | 30 | add_metall_test_executable(multimanager_test multimanager_test.cpp) 31 | 32 | add_metall_test_executable(segment_storage_test segment_storage_test.cpp) 33 | 34 | add_metall_test_executable(object_attribute_accessor_test object_attribute_accessor_test.cpp) -------------------------------------------------------------------------------- /test/test_utility.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_TEST_UTILITY_HPP 7 | #define METALL_TEST_UTILITY_HPP 8 | 9 | #include "gtest/gtest.h" 10 | 11 | #include 12 | #include 13 | #include_next 14 | #include 15 | 16 | #include 17 | 18 | namespace test_utility { 19 | 20 | namespace { 21 | namespace fs = std::filesystem; 22 | } 23 | 24 | const char *k_test_dir_env_name = "METALL_TEST_DIR"; 25 | const char *k_default_test_dir = "/tmp/metall_test_dir"; 26 | 27 | namespace detail { 28 | inline fs::path get_test_dir() { 29 | if (const char *env_p = std::getenv(k_test_dir_env_name)) { 30 | return fs::path(env_p); 31 | } 32 | return fs::path(k_default_test_dir); 33 | } 34 | } // namespace detail 35 | 36 | inline bool create_test_dir() { 37 | if (!metall::mtlldetail::directory_exist(detail::get_test_dir())) 38 | return metall::mtlldetail::create_directory(detail::get_test_dir()); 39 | return true; 40 | } 41 | 42 | inline fs::path make_test_path(const fs::path &name = fs::path()) { 43 | std::stringstream file_name; 44 | file_name << "metalltest-" 45 | << ::testing::UnitTest::GetInstance()->current_test_case()->name() 46 | << "-" 47 | << ::testing::UnitTest::GetInstance()->current_test_info()->name() 48 | << "-" << name.string(); 49 | return detail::get_test_dir() / file_name.str(); 50 | } 51 | 52 | } // namespace test_utility 53 | #endif // METALL_TEST_UTILITY_HPP 54 | -------------------------------------------------------------------------------- /test/utility/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_metall_test_executable(bitset_test bitset_test.cpp) -------------------------------------------------------------------------------- /tool/waste_memory.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | usage () { 4 | echo "Usage:" 5 | echo "$0 Size — Size is the number of GB to waste." 6 | exit 1 7 | } 8 | 9 | or_die () { 10 | echo "====================" 11 | echo "Command: " "$@" 12 | echo "----------" 13 | 14 | "$@" 15 | local status=$? 16 | if [[ $status != 0 ]] ; then 17 | echo ERROR $status command: "$@" 18 | exit $status 19 | fi 20 | 21 | echo "====================" 22 | } 23 | 24 | show_mem_usage () { 25 | or_die free -h 26 | or_die numactl --hardware 27 | } 28 | 29 | waste_memory () { 30 | echo "Wasting $waste_size_gb GB of memory" 31 | 32 | or_die numactl --all --interleave=all dd if=/dev/zero of=/dev/shm/${waste_size_gb}GB bs=4096 count=$((${waste_size_gb}*256*1024)) 33 | } 34 | 35 | main() { 36 | if [ $# -ne 1 ]; then 37 | echo "Bad argument count: $#" 38 | usage 39 | fi 40 | 41 | waste_size_gb=$1 42 | 43 | waste_memory 44 | show_mem_usage 45 | } 46 | 47 | main "$@" -------------------------------------------------------------------------------- /tutorial/hands_on/open_terminal.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/metall/dbb6e981c1f84d6979b2c28cf3b47779ab9ce779/tutorial/hands_on/open_terminal.png -------------------------------------------------------------------------------- /tutorial/hands_on/t0.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | // This program shows how to allocate memory using Metall. 7 | 8 | #include 9 | #include 10 | 11 | int main() { 12 | metall::manager manager(metall::create_only, "/tmp/dir"); 13 | 14 | // Allocate 'sizeof(int)' bytes, like malloc(sizeof(int)) 15 | // The object is allocated into persistent memory; 16 | // however, one cannot reattached this object without storing its address. 17 | int *n = (int *)manager.allocate(sizeof(int)); 18 | 19 | *n = 10; 20 | std::cout << *n << std::endl; 21 | 22 | manager.deallocate(n); // Deallocate memory 23 | 24 | return 0; 25 | } -------------------------------------------------------------------------------- /tutorial/hands_on/t1-1.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | // This program allocates an simple (int) object and reattaches it using Metall. 7 | 8 | #include 9 | #include 10 | 11 | int main() { 12 | // Creating data into persistent memory 13 | { 14 | metall::manager manager(metall::create_only, "/tmp/dir"); 15 | 16 | int *n = manager.construct // Allocates an 'int' object 17 | ("name") // Stores the allocated memory address with key "name" 18 | (10); // Call a constructor of the object 19 | } 20 | 21 | // ---------------------------------------------------------------------------------------------------- 22 | // Assume that this program exits here and the following code block is 23 | // executed as another run 24 | // ---------------------------------------------------------------------------------------------------- 25 | 26 | // Reattaching the data 27 | { 28 | metall::manager manager(metall::open_only, "/tmp/dir"); 29 | 30 | int *n = manager.find("name").first; 31 | std::cout << *n << std::endl; 32 | 33 | manager.destroy_ptr(n); // Deallocate memory 34 | } 35 | 36 | return 0; 37 | } -------------------------------------------------------------------------------- /tutorial/hands_on/t1-2.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #include 7 | #include 8 | 9 | // This program allocates a user-defined struct/class object and reattaches it 10 | // using Metall. One can learn how non-primitive data type object is constructed 11 | // and destructed with Metall. 12 | 13 | struct my_data { 14 | my_data(int _n, double _f) : n(_n), f(_f) { 15 | std::cout << "Constructor is called" << std::endl; 16 | } 17 | 18 | ~my_data() { std::cout << "Destructor is called" << std::endl; } 19 | 20 | int n; 21 | double f; 22 | }; 23 | 24 | int main() { 25 | // Creating data into persistent memory 26 | { 27 | metall::manager manager(metall::create_only, "/tmp/dir"); 28 | 29 | std::cout << "Allocate and construct an object" << std::endl; 30 | manager.construct // Allocates an 'my_data' object 31 | ("data") // Stores the allocated memory address with key "name" 32 | (10, 20.0); // Call the constructor of the class 33 | } 34 | 35 | // ---------------------------------------------------------------------------------------------------- 36 | // Assume that this program exits here and the following code block is 37 | // executed as another run 38 | // ---------------------------------------------------------------------------------------------------- 39 | 40 | // Reattaching the data 41 | { 42 | metall::manager manager(metall::open_read_only, "/tmp/dir"); 43 | 44 | auto *data = manager.find("data").first; 45 | std::cout << data->n << " " << data->f << std::endl; 46 | 47 | manager.destroy_ptr(data); // Destruct the object and deallocate memory 48 | } 49 | 50 | return 0; 51 | } -------------------------------------------------------------------------------- /tutorial/hands_on/t2-1.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | // This program shows how to use STL containers with Metall 7 | 8 | #include 9 | #include 10 | 11 | // Metall contains basic STL containers that use metall as their default 12 | // allocators. 13 | #include 14 | 15 | int main() { 16 | { 17 | metall::manager manager(metall::create_only, "/tmp/dir"); 18 | 19 | // Allocate a vector object, passing an allocator object 20 | auto* vec = manager.construct>("vec")( 21 | manager.get_allocator()); 22 | vec->resize(1); 23 | (*vec)[0] = 10; 24 | vec->push_back(20); 25 | } 26 | 27 | { 28 | metall::manager manager(metall::open_only, "/tmp/dir"); 29 | 30 | auto* vec = manager.find>("vec").first; 31 | std::cout << "Size = " << vec->size() << std::endl; 32 | std::cout << "Value at 0 = " << (*vec)[0] << std::endl; 33 | std::cout << "Value at 1 = " << (*vec)[1] << std::endl; 34 | 35 | vec->push_back(30); 36 | } 37 | 38 | { 39 | metall::manager manager(metall::open_read_only, "/tmp/dir"); 40 | 41 | auto* vec = manager.find>("vec").first; 42 | std::cout << "Value at 2 = " << (*vec)[2] << std::endl; 43 | } 44 | 45 | return 0; 46 | } -------------------------------------------------------------------------------- /tutorial/hands_on/t2-2.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | // This program shows how to use multi-layer STL containers with Metall 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | // Vector of int 13 | using inner_vector_t = metall::container::vector; 14 | 15 | // Vector of vectors with scoped allocator adaptor 16 | // In multi-level containers, one has to use scoped_allocator_adaptor in the 17 | // most outer container so that the inner containers obtain their allocator 18 | // arguments from the outer container's scoped_allocator_adaptor (see details: 19 | // https://en.cppreference.com/w/cpp/memory/scoped_allocator_adaptor) 20 | // metall::manager has an allocator that is already wrapped with scoped 21 | // allocator adaptor so that applications can use simple statements. 22 | using outer_vector_type = metall::container::vector< 23 | inner_vector_t, metall::manager::scoped_allocator_type>; 24 | 25 | int main() { 26 | { 27 | metall::manager manager(metall::create_only, "/tmp/datastore"); 28 | auto* vec = manager.construct("vec-of-vecs")( 29 | manager.get_allocator()); 30 | 31 | vec->resize(2); 32 | (*vec)[0].push_back(0); 33 | (*vec)[1].push_back(1); 34 | } 35 | 36 | { 37 | metall::manager manager(metall::open_only, "/tmp/datastore"); 38 | auto* vec = manager.find("vec-of-vecs").first; 39 | 40 | std::cout << (*vec)[0][0] << std::endl; // Will print out 0 41 | std::cout << (*vec)[1][0] << std::endl; // Will print out 1 42 | } 43 | 44 | return 0; 45 | } -------------------------------------------------------------------------------- /tutorial/hands_on/t3.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | // This program shows how to store pointers in persistent memory using the 7 | // offset pointer 8 | 9 | #include 10 | #include 11 | 12 | // metall::offset_ptr is just an alias of offset_ptr in Boost.Interprocess 13 | // https://www.boost.org/doc/libs/release/doc/html/interprocess/offset_ptr.html 14 | 15 | struct my_data { 16 | int len{0}; 17 | metall::offset_ptr array{nullptr}; 18 | }; 19 | 20 | int main() { 21 | { 22 | metall::manager manager(metall::create_only, "/tmp/datastore"); 23 | 24 | auto* data = manager.construct("data")(); 25 | data->len = 10; 26 | data->array = static_cast(manager.allocate(data->len * sizeof(int))); 27 | for (int i = 0; i < data->len; ++i) data->array[i] = i; 28 | } 29 | 30 | { 31 | metall::manager manager(metall::open_only, "/tmp/datastore"); 32 | 33 | auto* data = manager.find("data").first; 34 | for (int i = 0; i < data->len; ++i) 35 | std::cout << data->array[i] << std::endl; 36 | 37 | manager.deallocate(data->array.get()); 38 | manager.destroy_ptr(data); 39 | } 40 | 41 | return 0; 42 | } -------------------------------------------------------------------------------- /tutorial/hands_on/t4-1.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | // This program shows how to make and use an allocator-aware data structure. 7 | 8 | #include 9 | #include 10 | #include "t4-1.hpp" 11 | 12 | using persit_array = dynamic_array>; 13 | 14 | int main() { 15 | // Creating data into persistent memory 16 | { 17 | metall::manager manager(metall::create_only, "/tmp/dir"); 18 | auto* array = 19 | manager.construct("array")(manager.get_allocator()); 20 | init(*array); 21 | } 22 | 23 | // Reattaching the data 24 | { 25 | metall::manager manager(metall::open_only, "/tmp/dir"); 26 | auto* array = manager.find("array").first; 27 | print(*array); 28 | manager.destroy_ptr(array); 29 | } 30 | 31 | // This is how to use dynamic_array using the normal allocator 32 | // As you can see, the same data structure works with Metall and the normal 33 | // allocator. 34 | { 35 | auto* array = new dynamic_array(); 36 | init(*array); 37 | print(*array); 38 | delete array; 39 | } 40 | 41 | return 0; 42 | } -------------------------------------------------------------------------------- /tutorial/hands_on/t4-1.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_TUTORIAL_NVMW21_T4_1_HPP 7 | #define METALL_TUTORIAL_NVMW21_T4_1_HPP 8 | 9 | #include 10 | #include 11 | #include 12 | 13 | // An example of an allocator-aware data structure (dynamic array data 14 | // structure) It is to be emphasized that this data structure does not contain 15 | // any code that is only dedicated for Metall The following template parameters 16 | // are the same as std::vector 17 | template > // Allocator type; use std::allocator as the 20 | // default one 21 | class dynamic_array { 22 | public: 23 | explicit dynamic_array(Alloc alloc = Alloc()) : m_array(alloc) {} 24 | 25 | // Provides '[ ]' operator 26 | // ex.) array[0] = 10; 27 | T& operator[](int pos) { return m_array[pos]; } 28 | 29 | // Change the capacity 30 | void resize(int n) { m_array.resize(n); } 31 | 32 | private: 33 | boost::container::vector m_array; 34 | }; 35 | 36 | // -------------------- 37 | // Helper functions 38 | // -------------------- 39 | 40 | // Initialize an array 41 | template 42 | void init(array_t& array) { 43 | array.resize(5); 44 | for (int i = 0; i < 5; ++i) array[i] = i; 45 | } 46 | 47 | // Print out values 48 | template 49 | void print(array_t& array) { 50 | for (int i = 0; i < 5; ++i) std::cout << array[i] << std::endl; 51 | } 52 | 53 | #endif // METALL_TUTORIAL_NVMW21_T4_1_HPP 54 | -------------------------------------------------------------------------------- /tutorial/hands_on/t4-2.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | // This program shows how to make and use an allocator-aware data structure. 7 | 8 | #include 9 | #include 10 | #include "t4-2.hpp" 11 | 12 | using persit_matrix = matrix>; 13 | 14 | int main() { 15 | // Creating data into persistent memory 16 | { 17 | metall::manager manager(metall::create_only, "/tmp/dir"); 18 | auto* mx = manager.construct("mx")(manager.get_allocator()); 19 | init_matrix(*mx); 20 | } 21 | 22 | // Reattaching the data 23 | { 24 | metall::manager manager(metall::open_only, "/tmp/dir"); 25 | auto* mx = manager.find("mx").first; 26 | print_matrix(*mx); 27 | manager.destroy_ptr(mx); 28 | } 29 | 30 | // This is how to use matrix using the normal allocator 31 | // As you can see, the same data structure works with Metall and the normal 32 | // allocator. 33 | { 34 | auto* mx = new matrix(); 35 | init_matrix(*mx); 36 | print_matrix(*mx); 37 | delete mx; 38 | } 39 | 40 | return 0; 41 | } 42 | -------------------------------------------------------------------------------- /tutorial/hands_on/t4-2.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_TUTORIAL_NVMW21_T4_2_HPP 7 | #define METALL_TUTORIAL_NVMW21_T4_2_HPP 8 | 9 | #include 10 | #include 11 | #include 12 | 13 | namespace { 14 | namespace mc = boost::container; 15 | } 16 | 17 | // An example of an allocator-aware data structure (2D array) 18 | // It is to be emphasized that this data structure does not contain any code 19 | // that is only dedicated for Metall 20 | template > 21 | class matrix { 22 | public: 23 | explicit matrix(Alloc alloc = Alloc()) : m_matrix(alloc) {} 24 | 25 | // Change capacity 26 | void resize(int num_rows, int num_cols) { 27 | m_matrix.resize(num_rows); 28 | for (auto& vec : m_matrix) { 29 | vec.resize(num_cols); 30 | } 31 | } 32 | 33 | // Set a value 34 | void set(int row, int col, T value) { m_matrix[row][col] = value; } 35 | 36 | // Returns a value 37 | T get(int row, int col) { return m_matrix[row][col]; } 38 | 39 | private: 40 | using inner_alloc = typename Alloc::template rebind::other; 41 | using inner_vector = mc::vector; 42 | 43 | using outer_alloc = typename Alloc::template rebind::other; 44 | using outer_vector = 45 | mc::vector>; 46 | 47 | outer_vector m_matrix; 48 | }; 49 | 50 | // -------------------- 51 | // Helper functions 52 | // -------------------- 53 | template 54 | void init_matrix(matrix_t& mx) { 55 | mx.resize(2, 2); 56 | float value = 0; 57 | for (int r = 0; r < 2; ++r) 58 | for (int c = 0; c < 2; ++c) mx.set(r, c, value++); 59 | } 60 | 61 | template 62 | void print_matrix(matrix_t& mx) { 63 | for (int r = 0; r < 2; ++r) { 64 | for (int c = 0; c < 2; ++c) { 65 | std::cout << mx.get(r, c) << " "; 66 | } 67 | std::cout << std::endl; 68 | } 69 | } 70 | 71 | #endif // METALL_TUTORIAL_NVMW21_T4_2_HPP 72 | -------------------------------------------------------------------------------- /tutorial/hands_on/t5-1.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | // This program shows how to use the snapshot feature in Metall 7 | 8 | #include 9 | #include 10 | 11 | int main() { 12 | { 13 | metall::manager manager(metall::create_only, "/tmp/dir"); 14 | auto *n = manager.construct("n")(); 15 | *n = 10; 16 | 17 | // Create a snapshot 18 | manager.snapshot("/tmp/snapshot"); 19 | std::cout << "Created a snapshot" << std::endl; 20 | 21 | *n = 20; 22 | } 23 | 24 | // Reattach the original data 25 | { 26 | metall::manager manager(metall::open_only, "/tmp/dir"); 27 | std::cout << "Opened the original data" << std::endl; 28 | auto *n = manager.find("n").first; 29 | std::cout << *n << std::endl; // Show '20' 30 | } 31 | 32 | // Reattach the snapshot 33 | // A snapshot can be used as a normal Metall data store 34 | { 35 | metall::manager manager(metall::open_only, "/tmp/snapshot"); 36 | std::cout << "Opened the snapshot" << std::endl; 37 | auto *n = manager.find("n").first; 38 | std::cout << *n << std::endl; // Show '10' since this snapshot was created 39 | // before assigning 20. 40 | } 41 | 42 | return 0; 43 | } -------------------------------------------------------------------------------- /tutorial/hands_on/t5-2_create.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | // This program shows the crash consistency of Metall snapshot 7 | 8 | #include 9 | #include 10 | 11 | int main() { 12 | metall::manager manager(metall::create_only, "/tmp/dir"); 13 | auto *n = manager.construct("n")(); 14 | *n = 10; 15 | 16 | manager.snapshot("/tmp/snapshot"); 17 | std::cout << "Created a snapshot" << std::endl; 18 | 19 | *n = 20; 20 | std::cout << "Going to abort (simulating a fatal error)" << std::endl; 21 | std::abort(); // Assumes that a fatal error happens here, i.e., Metall 22 | // "/tmp/dir" is not closed properly. 23 | 24 | return 0; 25 | } -------------------------------------------------------------------------------- /tutorial/hands_on/t5-2_open.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | // This program shows how to open existing Metall data store, checking whether 7 | // it is consistent Please execute t5-2_create.cpp beforehand 8 | 9 | #include 10 | #include 11 | 12 | int main() { 13 | if (metall::manager::consistent("/tmp/dir")) { 14 | metall::manager manager(metall::open_read_only, "/tmp/dir"); 15 | std::cout << "Opened /tmp/dir" << std::endl; 16 | auto *n = manager.find("n").first; 17 | std::cout << *n << std::endl; 18 | } else { 19 | // This block will be executed since "/tmp/dir" was not closed properly. 20 | std::cerr 21 | << "Inconsistent Metall data store --- /tmp/dir was not closed properly" 22 | << std::endl; 23 | } 24 | 25 | if (metall::manager::consistent("/tmp/snapshot")) { 26 | metall::manager manager(metall::open_read_only, "/tmp/snapshot"); 27 | std::cout << "Opened /tmp/snapshot" << std::endl; 28 | auto *n = manager.find("n").first; 29 | std::cout << *n << std::endl; 30 | } else { 31 | std::cerr << "Inconsistent Metall data store --- /tmp/snapshot was not " 32 | "closed properly" 33 | << std::endl; 34 | } 35 | 36 | return 0; 37 | } -------------------------------------------------------------------------------- /tutorial/hands_on/tutorial_flow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/metall/dbb6e981c1f84d6979b2c28cf3b47779ab9ce779/tutorial/hands_on/tutorial_flow.png -------------------------------------------------------------------------------- /tutorial/hands_on/use_googleclould.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/metall/dbb6e981c1f84d6979b2c28cf3b47779ab9ce779/tutorial/hands_on/use_googleclould.pdf -------------------------------------------------------------------------------- /tutorial/ics21/README.md: -------------------------------------------------------------------------------- 1 | # Welcome to Metall Tutorial ICS'21! 2 | 3 | In this tutorial, we introduce Metall, a persistent memory allocator designed to provide developers with an API to allocate custom C++ data structures in both block-storage and byte-addressable persistent memories (e.g., NVMe SSD and Intel Optane DC Persistent Memory). 4 | 5 | An often overlooked but common theme among the variety of data analytics platforms is the need to persist data beyond a single process lifecycle. 6 | For example, data analytics applications usually perform data ingestion tasks, which index and partition data with analytics-specific data structures before performing the analysis. 7 | 8 | However, the data ingestion task is often more expensive than the analytic itself, and the same or derived data is re-ingested frequently (e.g., running multiple analytics to the same data, developing/debugging analytics programs). 9 | The promise of persistent memory is that, once constructed, data structures can be re-analyzed and updated beyond the lifetime of a single execution. 10 | 11 | Thanks to the recent notable performance improvements and cost reductions in non-volatile memory (NVRAM) technology, we believe that leveraging persistent memory in this way brings significant benefits to data analytics applications. 12 | 13 | ## Agenda 14 | 15 | ### Section 1: Lecture (1.5 hour) 16 | 17 | #### Background 18 | - Persistent memory 19 | - Leveraging persistent memory in data analytics 20 | 21 | #### Metall 101 & Hands-on 22 | 23 | - API 24 | - Internal architecture 25 | - Hands-on 26 | - Performance evaluation 27 | 28 | 29 | ### Section 2: Application Case Studies (1 hour) 30 | 31 | - Exploratory data analytics using Metall 32 | - GraphBLAS Template Library + Metall 33 | 34 | 35 | ### Section 3: Q&A and Extra Hands-on Session (0.5—1 hour) 36 | 37 | - LLNL staff stay online to answer questions and work with anyone who wishes to experiment with provided example code 38 | 39 | 40 | ## Tutorial Materials 41 | 42 | * [Lecture slides](metall_101.pdf) 43 | * [Hands-on Examples](../hands_on) -------------------------------------------------------------------------------- /tutorial/ics21/metall_101.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/metall/dbb6e981c1f84d6979b2c28cf3b47779ab9ce779/tutorial/ics21/metall_101.pdf -------------------------------------------------------------------------------- /tutorial/nvmw21/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.10) 2 | project(Metall-Tutorial-NVMW21) 3 | 4 | # ---------- Compiler ---------- # 5 | set(CMAKE_CXX_STANDARD 17) 6 | set(CMAKE_BUILD_TYPE Debug) 7 | 8 | set(CMAKE_CXX_FLAGS "-Wall -pthread") 9 | 10 | # Debug 11 | set(CMAKE_CXX_FLAGS_DEBUG "-O0") 12 | if (CMAKE_SYSTEM_NAME STREQUAL "Linux") 13 | set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -pg") 14 | endif () 15 | 16 | 17 | # Requirements for GCC 18 | if (("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU") OR ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")) 19 | if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8.1) 20 | message(FATAL_ERROR "GCC version must be at least 8.1") 21 | endif () 22 | endif () 23 | 24 | 25 | # ---------- Boost ---------- # 26 | find_package(Boost 1.80 REQUIRED) 27 | 28 | # ---------- MPI ---------- # 29 | find_package(MPI) 30 | 31 | # ---------------------------------------- # 32 | # Targets 33 | # ---------------------------------------- # 34 | 35 | function(add_metall_exe name) 36 | add_executable(${name} ${name}.cpp) 37 | target_include_directories(${name} PRIVATE ../../include ${Boost_INCLUDE_DIRS}) 38 | target_link_libraries(${name} PRIVATE stdc++fs) 39 | endfunction() 40 | 41 | add_metall_exe(t0) 42 | add_metall_exe(t1-1) 43 | add_metall_exe(t1-2) 44 | add_metall_exe(t2-1) 45 | add_metall_exe(t2-2) 46 | add_metall_exe(t3) 47 | add_metall_exe(t4-1) 48 | add_metall_exe(t4-2) 49 | add_metall_exe(t5-1) 50 | add_metall_exe(t5-2_create) 51 | add_metall_exe(t5-2_open) -------------------------------------------------------------------------------- /tutorial/nvmw21/README.md: -------------------------------------------------------------------------------- 1 | # 0. Setting Up Environment for the Hands-on Session 2 | 3 | We recommend staring setting up one's environment before 4 | the hands-on session because some steps may take a long time. 5 | 6 | ### Machine 7 | 8 | Metall is designed to work on Linux machines. 9 | It also works on macOS (some internal performance optimizations are disabled). 10 | 11 | 12 | ***Google Cloud*** 13 | 14 | One also could use a public cloud system, such as [Google Cloud](https://cloud.google.com/). 15 | Here is an example of creating an account on Google Cloud: [use_googlecloud.pdf](use_googleclould.pdf). 16 | 17 | 18 | ### GCC 19 | 20 | Please install GCC 8.1 or more. One could also use Clang or Apple clang; however GCC is recommended. 21 | 22 | ### Boost C++ Libraries 23 | 24 | Only the header files are used by Metall. One does not have to build it. 25 | 26 | ```bash 27 | # Download Boost C++ Libraries (1.80 or more is required) 28 | wget https://dl.bintray.com/boostorg/release/1.80.0/source/boost_1_80_0.tar.gz 29 | tar xvf boost_1_80_0.tar.gz 30 | ``` 31 | 32 | # 1. Lecture Materials 33 | 34 | * [Metall 101 slides](metall_101.pdf) 35 | 36 | # 2. Hands-on 37 | 38 | ## 2-1. Recommended Flow 39 | 40 | ![](tutorial_flow.png) 41 | 42 | 43 | ## 2-2. Build and Run 44 | 45 | **Required** 46 | 47 | - GCC 8.1 or more 48 | 49 | **Build example** 50 | ```bash 51 | # Download Boost C++ Libraries (1.80 or more is required) 52 | # One can skip this step if Boost is already available. 53 | wget https://dl.bintray.com/boostorg/release/1.80.0/source/boost_1_80_0.tar.gz 54 | tar xvf boost_1_80_0.tar.gz 55 | export BOOST_ROOT=$PWD/boost_1_80_0 56 | 57 | git clone https://github.com/LLNL/metall 58 | cd metall/tutorial/nvmw21 59 | g++ -std=c++17 [tutorial_program.cpp] -lstdc++fs -I../../include -I${BOOST_ROOT} 60 | 61 | # All tutorial programs does not take command-line options 62 | ./a.out 63 | ``` 64 | 65 | 66 | ## 2-3. (optional) Build using Clang or Apple clang 67 | 68 | Clang (or Apple clang) could be used instead of GCC to build Metall. 69 | However, please note that we haven't tested it intensively. 70 | To run on macOS, Metall requires macOS >= 10.15. 71 | 72 | ```bash 73 | # Remove "-lstdc++fs" option 74 | clang++ -std=c++17 [tutorial_program.cpp] -I../../include -I${BOOST_ROOT} 75 | ``` -------------------------------------------------------------------------------- /tutorial/nvmw21/metall_101.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/metall/dbb6e981c1f84d6979b2c28cf3b47779ab9ce779/tutorial/nvmw21/metall_101.pdf -------------------------------------------------------------------------------- /tutorial/nvmw21/t0.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | // This program shows how to allocate memory using Metall. 7 | 8 | #include 9 | #include 10 | 11 | int main() { 12 | metall::manager manager(metall::create_only, "/tmp/dir"); 13 | 14 | // Allocate 'sizeof(int)' bytes, like malloc(sizeof(int)) 15 | // The object is allocated into persistent memory; 16 | // however, one cannot reattached this object without storing its address. 17 | int *n = (int *)manager.allocate(sizeof(int)); 18 | 19 | *n = 10; 20 | std::cout << *n << std::endl; 21 | 22 | manager.deallocate(n); // Deallocate memory 23 | 24 | return 0; 25 | } -------------------------------------------------------------------------------- /tutorial/nvmw21/t1-1.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | // This program allocates an simple (int) object and reattaches it using Metall. 7 | 8 | #include 9 | #include 10 | 11 | int main() { 12 | // Creating data into persistent memory 13 | { 14 | metall::manager manager(metall::create_only, "/tmp/dir"); 15 | 16 | int *n = manager.construct // Allocates an 'int' object 17 | ("name") // Stores the allocated memory address with key "name" 18 | (10); // Call a constructor of the object 19 | } 20 | 21 | // ---------------------------------------------------------------------------------------------------- 22 | // Assume that this program exits here and the following code block is 23 | // executed as another run 24 | // ---------------------------------------------------------------------------------------------------- 25 | 26 | // Reattaching the data 27 | { 28 | metall::manager manager(metall::open_only, "/tmp/dir"); 29 | 30 | int *n = manager.find("name").first; 31 | std::cout << *n << std::endl; 32 | 33 | manager.destroy_ptr(n); // Deallocate memory 34 | } 35 | 36 | return 0; 37 | } -------------------------------------------------------------------------------- /tutorial/nvmw21/t1-2.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #include 7 | #include 8 | 9 | // This program allocates a user-defined struct/class object and reattaches it 10 | // using Metall. One can learn how non-primitive data type object is constructed 11 | // and destructed with Metall. 12 | 13 | struct my_data { 14 | my_data(int _n, double _f) : n(_n), f(_f) { 15 | std::cout << "Constructor is called" << std::endl; 16 | } 17 | 18 | ~my_data() { std::cout << "Destructor is called" << std::endl; } 19 | 20 | int n; 21 | double f; 22 | }; 23 | 24 | int main() { 25 | // Creating data into persistent memory 26 | { 27 | metall::manager manager(metall::create_only, "/tmp/dir"); 28 | 29 | std::cout << "Allocate and construct an object" << std::endl; 30 | manager.construct // Allocates an 'my_data' object 31 | ("data") // Stores the allocated memory address with key "name" 32 | (10, 20.0); // Call the constructor of the class 33 | } 34 | 35 | // ---------------------------------------------------------------------------------------------------- 36 | // Assume that this program exits here and the following code block is 37 | // executed as another run 38 | // ---------------------------------------------------------------------------------------------------- 39 | 40 | // Reattaching the data 41 | { 42 | metall::manager manager(metall::open_read_only, "/tmp/dir"); 43 | 44 | auto *data = manager.find("data").first; 45 | std::cout << data->n << " " << data->f << std::endl; 46 | 47 | manager.destroy_ptr(data); // Destruct the object and deallocate memory 48 | } 49 | 50 | return 0; 51 | } -------------------------------------------------------------------------------- /tutorial/nvmw21/t2-1.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | // This program shows how to use STL containers with Metall 7 | 8 | #include 9 | #include 10 | 11 | // Metall contains basic STL containers that use metall as their default 12 | // allocators. 13 | #include 14 | 15 | int main() { 16 | { 17 | metall::manager manager(metall::create_only, "/tmp/dir"); 18 | 19 | // Allocate a vector object, passing an allocator object 20 | auto* vec = manager.construct>("vec")( 21 | manager.get_allocator()); 22 | vec->resize(1); 23 | (*vec)[0] = 10; 24 | vec->push_back(20); 25 | } 26 | 27 | { 28 | metall::manager manager(metall::open_only, "/tmp/dir"); 29 | 30 | auto* vec = manager.find>("vec").first; 31 | std::cout << "Size = " << vec->size() << std::endl; 32 | std::cout << "Value at 0 = " << (*vec)[0] << std::endl; 33 | std::cout << "Value at 1 = " << (*vec)[1] << std::endl; 34 | 35 | vec->push_back(30); 36 | } 37 | 38 | { 39 | metall::manager manager(metall::open_read_only, "/tmp/dir"); 40 | 41 | auto* vec = manager.find>("vec").first; 42 | std::cout << "Value at 2 = " << (*vec)[2] << std::endl; 43 | } 44 | 45 | return 0; 46 | } -------------------------------------------------------------------------------- /tutorial/nvmw21/t2-2.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | // This program shows how to use multi-layer STL containers with Metall 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | // Vector of int 13 | using inner_vector_t = metall::container::vector; 14 | 15 | // Vector of vectors with scoped allocator adaptor 16 | // In multi-level containers, one has to use scoped_allocator_adaptor in the 17 | // most outer container so that the inner containers obtain their allocator 18 | // arguments from the outer container's scoped_allocator_adaptor (see details: 19 | // https://en.cppreference.com/w/cpp/memory/scoped_allocator_adaptor) 20 | // metall::manager has an allocator that is already wrapped with scoped 21 | // allocator adaptor so that applications can use simple statements. 22 | using outer_vector_type = metall::container::vector< 23 | inner_vector_t, metall::manager::scoped_allocator_type>; 24 | 25 | int main() { 26 | { 27 | metall::manager manager(metall::create_only, "/tmp/datastore"); 28 | auto* vec = manager.construct("vec-of-vecs")( 29 | manager.get_allocator()); 30 | 31 | vec->resize(2); 32 | (*vec)[0].push_back(0); 33 | (*vec)[1].push_back(1); 34 | } 35 | 36 | { 37 | metall::manager manager(metall::open_only, "/tmp/datastore"); 38 | auto* vec = manager.find("vec-of-vecs").first; 39 | 40 | std::cout << (*vec)[0][0] << std::endl; // Will print out 0 41 | std::cout << (*vec)[1][0] << std::endl; // Will print out 1 42 | } 43 | 44 | return 0; 45 | } -------------------------------------------------------------------------------- /tutorial/nvmw21/t3.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | // This program shows how to store pointers in persistent memory using the 7 | // offset pointer 8 | 9 | #include 10 | #include 11 | 12 | // metall::offset_ptr is just an alias of offset_ptr in Boost.Interprocess 13 | // https://www.boost.org/doc/libs/release/doc/html/interprocess/offset_ptr.html 14 | 15 | struct my_data { 16 | int len{0}; 17 | metall::offset_ptr array{nullptr}; 18 | }; 19 | 20 | int main() { 21 | { 22 | metall::manager manager(metall::create_only, "/tmp/datastore"); 23 | 24 | auto* data = manager.construct("data")(); 25 | data->len = 10; 26 | data->array = static_cast(manager.allocate(data->len * sizeof(int))); 27 | for (int i = 0; i < data->len; ++i) data->array[i] = i; 28 | } 29 | 30 | { 31 | metall::manager manager(metall::open_only, "/tmp/datastore"); 32 | 33 | auto* data = manager.find("data").first; 34 | for (int i = 0; i < data->len; ++i) 35 | std::cout << data->array[i] << std::endl; 36 | 37 | manager.deallocate(data->array.get()); 38 | manager.destroy_ptr(data); 39 | } 40 | 41 | return 0; 42 | } -------------------------------------------------------------------------------- /tutorial/nvmw21/t4-1.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | // This program shows how to make and use an allocator-aware data structure. 7 | 8 | #include 9 | #include 10 | #include "t4-1.hpp" 11 | 12 | using persit_array = dynamic_array>; 13 | 14 | int main() { 15 | // Creating data into persistent memory 16 | { 17 | metall::manager manager(metall::create_only, "/tmp/dir"); 18 | auto* array = 19 | manager.construct("array")(manager.get_allocator()); 20 | init(*array); 21 | } 22 | 23 | // Reattaching the data 24 | { 25 | metall::manager manager(metall::open_only, "/tmp/dir"); 26 | auto* array = manager.find("array").first; 27 | print(*array); 28 | manager.destroy_ptr(array); 29 | } 30 | 31 | // This is how to use dynamic_array using the normal allocator 32 | // As you can see, the same data structure works with Metall and the normal 33 | // allocator. 34 | { 35 | auto* array = new dynamic_array(); 36 | init(*array); 37 | print(*array); 38 | delete array; 39 | } 40 | 41 | return 0; 42 | } -------------------------------------------------------------------------------- /tutorial/nvmw21/t4-1.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_TUTORIAL_NVMW21_T4_1_HPP 7 | #define METALL_TUTORIAL_NVMW21_T4_1_HPP 8 | 9 | #include 10 | #include 11 | #include 12 | 13 | // An example of an allocator-aware data structure (dynamic array data 14 | // structure) It is to be emphasized that this data structure does not contain 15 | // any code that is only dedicated for Metall The following template parameters 16 | // are the same as std::vector 17 | template > // Allocator type; use std::allocator as the 20 | // default one 21 | class dynamic_array { 22 | public: 23 | explicit dynamic_array(Alloc alloc = Alloc()) : m_array(alloc) {} 24 | 25 | // Provides '[ ]' operator 26 | // ex.) array[0] = 10; 27 | T& operator[](int pos) { return m_array[pos]; } 28 | 29 | // Change the capacity 30 | void resize(int n) { m_array.resize(n); } 31 | 32 | private: 33 | boost::container::vector m_array; 34 | }; 35 | 36 | // -------------------- 37 | // Helper functions 38 | // -------------------- 39 | 40 | // Initialize an array 41 | template 42 | void init(array_t& array) { 43 | array.resize(5); 44 | for (int i = 0; i < 5; ++i) array[i] = i; 45 | } 46 | 47 | // Print out values 48 | template 49 | void print(array_t& array) { 50 | for (int i = 0; i < 5; ++i) std::cout << array[i] << std::endl; 51 | } 52 | 53 | #endif // METALL_TUTORIAL_NVMW21_T4_1_HPP 54 | -------------------------------------------------------------------------------- /tutorial/nvmw21/t4-2.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | // This program shows how to make and use an allocator-aware data structure. 7 | 8 | #include 9 | #include 10 | #include "t4-2.hpp" 11 | 12 | using persit_matrix = matrix>; 13 | 14 | int main() { 15 | // Creating data into persistent memory 16 | { 17 | metall::manager manager(metall::create_only, "/tmp/dir"); 18 | auto* mx = manager.construct("mx")(manager.get_allocator()); 19 | init_matrix(*mx); 20 | } 21 | 22 | // Reattaching the data 23 | { 24 | metall::manager manager(metall::open_only, "/tmp/dir"); 25 | auto* mx = manager.find("mx").first; 26 | print_matrix(*mx); 27 | manager.destroy_ptr(mx); 28 | } 29 | 30 | // This is how to use matrix using the normal allocator 31 | // As you can see, the same data structure works with Metall and the normal 32 | // allocator. 33 | { 34 | auto* mx = new matrix(); 35 | init_matrix(*mx); 36 | print_matrix(*mx); 37 | delete mx; 38 | } 39 | 40 | return 0; 41 | } 42 | -------------------------------------------------------------------------------- /tutorial/nvmw21/t4-2.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #ifndef METALL_TUTORIAL_NVMW21_T4_2_HPP 7 | #define METALL_TUTORIAL_NVMW21_T4_2_HPP 8 | 9 | #include 10 | #include 11 | #include 12 | 13 | namespace { 14 | namespace mc = boost::container; 15 | } 16 | 17 | // An example of an allocator-aware data structure (2D array) 18 | // It is to be emphasized that this data structure does not contain any code 19 | // that is only dedicated for Metall 20 | template > 21 | class matrix { 22 | public: 23 | explicit matrix(Alloc alloc = Alloc()) : m_matrix(alloc) {} 24 | 25 | // Change capacity 26 | void resize(int num_rows, int num_cols) { 27 | m_matrix.resize(num_rows); 28 | for (auto& vec : m_matrix) { 29 | vec.resize(num_cols); 30 | } 31 | } 32 | 33 | // Set a value 34 | void set(int row, int col, T value) { m_matrix[row][col] = value; } 35 | 36 | // Returns a value 37 | T get(int row, int col) { return m_matrix[row][col]; } 38 | 39 | private: 40 | using inner_alloc = typename Alloc::template rebind::other; 41 | using inner_vector = mc::vector; 42 | 43 | using outer_alloc = typename Alloc::template rebind::other; 44 | using outer_vector = 45 | mc::vector>; 46 | 47 | outer_vector m_matrix; 48 | }; 49 | 50 | // -------------------- 51 | // Helper functions 52 | // -------------------- 53 | template 54 | void init_matrix(matrix_t& mx) { 55 | mx.resize(2, 2); 56 | float value = 0; 57 | for (int r = 0; r < 2; ++r) 58 | for (int c = 0; c < 2; ++c) mx.set(r, c, value++); 59 | } 60 | 61 | template 62 | void print_matrix(matrix_t& mx) { 63 | for (int r = 0; r < 2; ++r) { 64 | for (int c = 0; c < 2; ++c) { 65 | std::cout << mx.get(r, c) << " "; 66 | } 67 | std::cout << std::endl; 68 | } 69 | } 70 | 71 | #endif // METALL_TUTORIAL_NVMW21_T4_2_HPP 72 | -------------------------------------------------------------------------------- /tutorial/nvmw21/t5-1.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | // This program shows how to use the snapshot feature in Metall 7 | 8 | #include 9 | #include 10 | 11 | int main() { 12 | { 13 | metall::manager manager(metall::create_only, "/tmp/dir"); 14 | auto *n = manager.construct("n")(); 15 | *n = 10; 16 | 17 | // Create a snapshot 18 | manager.snapshot("/tmp/snapshot"); 19 | std::cout << "Created a snapshot" << std::endl; 20 | 21 | *n = 20; 22 | } 23 | 24 | // Reattach the original data 25 | { 26 | metall::manager manager(metall::open_only, "/tmp/dir"); 27 | std::cout << "Opened the original data" << std::endl; 28 | auto *n = manager.find("n").first; 29 | std::cout << *n << std::endl; // Show '20' 30 | } 31 | 32 | // Reattach the snapshot 33 | // A snapshot can be used as a normal Metall data store 34 | { 35 | metall::manager manager(metall::open_only, "/tmp/snapshot"); 36 | std::cout << "Opened the snapshot" << std::endl; 37 | auto *n = manager.find("n").first; 38 | std::cout << *n << std::endl; // Show '10' since this snapshot was created 39 | // before assigning 20. 40 | } 41 | 42 | return 0; 43 | } -------------------------------------------------------------------------------- /tutorial/nvmw21/t5-2_create.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | // This program shows the crash consistency of Metall snapshot 7 | 8 | #include 9 | #include 10 | 11 | int main() { 12 | metall::manager manager(metall::create_only, "/tmp/dir"); 13 | auto *n = manager.construct("n")(); 14 | *n = 10; 15 | 16 | manager.snapshot("/tmp/snapshot"); 17 | std::cout << "Created a snapshot" << std::endl; 18 | 19 | *n = 20; 20 | std::cout << "Going to abort (simulating a fatal error)" << std::endl; 21 | std::abort(); // Assumes that a fatal error happens here, i.e., Metall 22 | // "/tmp/dir" is not closed properly. 23 | 24 | return 0; 25 | } -------------------------------------------------------------------------------- /tutorial/nvmw21/t5-2_open.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | // This program shows how to open existing Metall data store, checking whether 7 | // it is consistent Please execute t5-2_create.cpp beforehand 8 | 9 | #include 10 | #include 11 | 12 | int main() { 13 | if (metall::manager::consistent("/tmp/dir")) { 14 | metall::manager manager(metall::open_read_only, "/tmp/dir"); 15 | std::cout << "Opened /tmp/dir" << std::endl; 16 | auto *n = manager.find("n").first; 17 | std::cout << *n << std::endl; 18 | } else { 19 | // This block will be executed since "/tmp/dir" was not closed properly. 20 | std::cerr 21 | << "Inconsistent Metall data store --- /tmp/dir was not closed properly" 22 | << std::endl; 23 | } 24 | 25 | if (metall::manager::consistent("/tmp/snapshot")) { 26 | metall::manager manager(metall::open_read_only, "/tmp/snapshot"); 27 | std::cout << "Opened /tmp/snapshot" << std::endl; 28 | auto *n = manager.find("n").first; 29 | std::cout << *n << std::endl; 30 | } else { 31 | std::cerr << "Inconsistent Metall data store --- /tmp/snapshot was not " 32 | "closed properly" 33 | << std::endl; 34 | } 35 | 36 | return 0; 37 | } -------------------------------------------------------------------------------- /tutorial/nvmw21/tutorial_flow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/metall/dbb6e981c1f84d6979b2c28cf3b47779ab9ce779/tutorial/nvmw21/tutorial_flow.png -------------------------------------------------------------------------------- /tutorial/nvmw21/use_googleclould.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/metall/dbb6e981c1f84d6979b2c28cf3b47779ab9ce779/tutorial/nvmw21/use_googleclould.pdf -------------------------------------------------------------------------------- /verification/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_subdirectory(file_clone) 2 | add_subdirectory(free_mmap_region) 3 | add_subdirectory(mmap) 4 | add_subdirectory(object_size) 5 | add_subdirectory(logger) 6 | add_subdirectory(soft_dirty) 7 | add_subdirectory(file_io) 8 | add_subdirectory(sparse_copy) -------------------------------------------------------------------------------- /verification/file_clone/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_metall_executable(verify_file_clone verify_file_clone.cpp) 2 | -------------------------------------------------------------------------------- /verification/file_io/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_metall_executable(verify_64bits_file_io verify_64bits_file_io.cpp) -------------------------------------------------------------------------------- /verification/file_io/verify_64bits_file_io.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | int main(int argc, char *argv[]) { 13 | #ifdef _FILE_OFFSET_BITS 14 | std::cout << "_FILE_OFFSET_BITS = " << _FILE_OFFSET_BITS << std::endl; 15 | #else 16 | std::cerr << "_FILE_OFFSET_BITS is not defined" << std::endl; 17 | #endif 18 | 19 | std::cout << "sizeof(off_t) = " << sizeof(off_t) << std::endl; 20 | 21 | if (argc != 3) { 22 | std::cerr << "Wrong arguments" << std::endl; 23 | std::cerr << "./verify_64bits_file_io [file_name] [size]" << std::endl; 24 | std::abort(); 25 | } 26 | std::string file_name = argv[1]; 27 | ssize_t size = std::stoull(argv[2]); 28 | 29 | const int fd = 30 | ::open(file_name.c_str(), O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); 31 | if (fd == -1) { 32 | metall::logger::perror(metall::logger::level::critical, __FILE__, __LINE__, 33 | "open"); 34 | std::abort(); 35 | } 36 | 37 | auto buf = new char[size]; 38 | for (ssize_t i = 0; i < size; ++i) { 39 | buf[i] = static_cast(i); 40 | } 41 | 42 | const auto written_size = ::pwrite(fd, buf, size, 0); 43 | if (written_size == size) { 44 | std::cout << "Write succeeded!" << std::endl; 45 | } else { 46 | std::cerr << "Requested write size " << size << std::endl; 47 | std::cerr << "Actually written size " << written_size << std::endl; 48 | metall::logger::perror(metall::logger::level::critical, __FILE__, __LINE__, 49 | "write"); 50 | std::abort(); 51 | } 52 | 53 | if (::close(fd) == -1) { 54 | metall::logger::perror(metall::logger::level::critical, __FILE__, __LINE__, 55 | "close"); 56 | std::abort(); 57 | } 58 | 59 | delete[] buf; 60 | 61 | return 0; 62 | } -------------------------------------------------------------------------------- /verification/free_mmap_region/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_metall_executable(run_free_file_space_bench run_free_file_space_bench.cpp) 2 | add_metall_executable(verify_mmap_region_free verify_mmap_region_free.cpp) -------------------------------------------------------------------------------- /verification/logger/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_metall_executable(verify_logger verify_logger.cpp) -------------------------------------------------------------------------------- /verification/mmap/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_metall_executable(run_many_files_mmap_bench run_many_files_mmap_bench.cpp) 2 | add_metall_executable(verify_mmap verify_mmap.cpp) -------------------------------------------------------------------------------- /verification/object_size/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_metall_executable(show_object_sizes show_object_sizes.cpp) -------------------------------------------------------------------------------- /verification/object_size/show_object_sizes.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Lawrence Livermore National Security, LLC and other Metall 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | static constexpr std::size_t k_max_segment_size = 1ULL << 48ULL; 11 | using object_size_manager = 12 | metall::kernel::object_size_manager; 14 | 15 | int main() { 16 | std::cout << "Bin number,\tSize,\tMax Internal Fragmentation Size" 17 | << std::endl; 18 | std::size_t pre_size = 8; 19 | for (std::size_t i = 0; i < object_size_manager::num_sizes(); ++i) { 20 | const auto size = object_size_manager::at(i); 21 | if (i == 0 || i >= object_size_manager::num_small_sizes()) 22 | std::cout << i << "\t" << size << "\tN/A" << std::endl; 23 | else 24 | std::cout << i << "\t" << size << "\t" 25 | << (double)(size - pre_size - 1) / (pre_size + 1) << std::endl; 26 | pre_size = size; 27 | } 28 | 29 | return 0; 30 | } -------------------------------------------------------------------------------- /verification/soft_dirty/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_metall_executable(verify_soft_dirty verify_soft_dirty.cpp) -------------------------------------------------------------------------------- /verification/sparse_copy/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_metall_executable(verify_sparse_copy verify_sparse_copy.cpp) 2 | 3 | if (CMAKE_SYSTEM_NAME STREQUAL "Linux") 4 | add_metall_executable(verify_sparse_copy_syscalls verify_sparse_copy_syscalls.cpp) 5 | endif () 6 | --------------------------------------------------------------------------------