├── test ├── data │ ├── parquet_files │ │ ├── .hidden.parquet │ │ ├── non-parquet.parquet │ │ ├── file_1.parquet │ │ ├── file_2.parquet │ │ ├── file_3.parquet │ │ └── parquet.wrong-extension │ ├── short.txt │ ├── README.md │ ├── csv_headers.csv │ ├── 3.ndjson │ ├── 100.csv │ ├── bad.ndjson │ ├── test_parquet_generator.py │ └── loremipsum │ │ ├── loremipsum_3.txt │ │ ├── loremipsum_2.txt │ │ └── loremipsum_1.txt ├── test_comm_2.cpp ├── test_boost_string.cpp ├── test_csv_parser.cpp ├── test_reduce.cpp ├── test_interrupt_mask.cpp ├── test_lambda_capture.cpp ├── test_cereal_boost_container.cpp ├── test_barrier.cpp ├── test_recursion_progress.cpp ├── test_ndjson_parser.cpp ├── test_cereal_archive.cpp ├── test_wout_werr.cpp ├── test_large_messages.cpp ├── test_csv_headers.cpp ├── test_progress_indicator.cpp ├── test_gather_topk.cpp ├── test_cereal_boost_json.cpp ├── test_local_visit.cpp ├── test_reduce_by_key.cpp ├── test_random.cpp ├── test_recursion_large_messages.cpp ├── test_tagged_bag.cpp ├── test_collective.cpp ├── test_reducing_adapter.cpp ├── test_byte_vector.cpp ├── test_transform.cpp ├── CMakeLists.txt ├── test_traits.cpp ├── test_concepts.cpp ├── test_logger.cpp ├── test_line_parser.cpp ├── test_container_traits.cpp └── test_layout.cpp ├── docs ├── ygm.pdf ├── sphinx │ ├── requirements.txt │ ├── ygm │ │ ├── comm │ │ │ └── comm.rst │ │ ├── container │ │ │ ├── bag.rst │ │ │ ├── map.rst │ │ │ ├── set.rst │ │ │ ├── array.rst │ │ │ ├── counting_set.rst │ │ │ └── disjoint_set.rst │ │ ├── io │ │ │ ├── csv_parser.rst │ │ │ ├── line_parser.rst │ │ │ ├── daily_output.rst │ │ │ ├── multi_output.rst │ │ │ ├── ndjson_parser.rst │ │ │ └── parquet_parser.rst │ │ ├── utility │ │ │ ├── timer.rst │ │ │ └── progress_indicator.rst │ │ ├── comm.rst │ │ └── utility.rst │ ├── index.rst │ ├── dev.rst │ ├── CMakeLists.txt │ └── conf.py └── CMakeLists.txt ├── .gitignore ├── examples └── Readme.md ├── cmake ├── ygmConfig.cmake.in ├── FindSphinx.cmake ├── FindPython3Module.cmake └── PythonUtilities.cmake ├── include └── ygm │ ├── version.hpp │ ├── random.hpp │ ├── detail │ ├── std_traits.hpp │ ├── interrupt_mask.hpp │ ├── lambda_compliance.hpp │ ├── ygm_ptr.hpp │ ├── lambda_map.hpp │ ├── random.hpp │ ├── logger.hpp │ ├── meta │ │ └── functional.hpp │ ├── ygm_traits.hpp │ ├── mpi.hpp │ └── comm_router.hpp │ ├── utility │ ├── timer.hpp │ ├── boost_static_string.hpp │ ├── assert.hpp │ └── world.hpp │ ├── container │ ├── detail │ │ ├── round_robin_partitioner.hpp │ │ ├── base_count.hpp │ │ ├── type_traits.hpp │ │ ├── hash.hpp │ │ ├── hash_partitioner.hpp │ │ ├── base_misc.hpp │ │ ├── base_async_contains.hpp │ │ ├── base_async_insert_or_assign.hpp │ │ ├── base_async_reduce.hpp │ │ ├── base_async_insert_contains.hpp │ │ ├── filter_proxy.hpp │ │ ├── base_async_erase.hpp │ │ ├── flatten_proxy.hpp │ │ ├── base_iterators.hpp │ │ ├── base_concepts.hpp │ │ └── base_async_insert.hpp │ ├── disjoint_set.hpp │ ├── container_traits.hpp │ └── experimental │ │ └── detail │ │ ├── algorithms │ │ └── spmv.hpp │ │ └── row_view_impl.hpp │ └── io │ ├── daily_output.hpp │ ├── csv_parser.hpp │ └── ndjson_parser.hpp ├── .readthedocs.yaml ├── .devcontainer ├── Dockerfile └── devcontainer.json ├── .github └── workflows │ ├── test_docs_build.yaml │ ├── build-pdf-docs.yaml │ ├── ci-test-macos.yaml │ └── ci-test.yaml ├── COPYRIGHT ├── NOTICE ├── LICENSE-MIT ├── .gitlab-ci.yml ├── .cmake-format.py └── .clang-format /test/data/parquet_files/.hidden.parquet: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/data/short.txt: -------------------------------------------------------------------------------- 1 | This file 2 | has two lines. -------------------------------------------------------------------------------- /test/data/parquet_files/non-parquet.parquet: -------------------------------------------------------------------------------- 1 | test 2 | -------------------------------------------------------------------------------- /docs/ygm.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/ygm/HEAD/docs/ygm.pdf -------------------------------------------------------------------------------- /test/data/README.md: -------------------------------------------------------------------------------- 1 | Generated loremipsum from: https://loremipsum.io/ -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | *#* 3 | build* 4 | .vscode* 5 | .idea* 6 | .cache/ 7 | -------------------------------------------------------------------------------- /docs/sphinx/requirements.txt: -------------------------------------------------------------------------------- 1 | cmake 2 | sphinx 3 | breathe 4 | sphinx-rtd-theme -------------------------------------------------------------------------------- /test/data/csv_headers.csv: -------------------------------------------------------------------------------- 1 | zero, four, two, six 2 | 0, 4, 2, 6 3 | 0, 4, 2, 6 4 | 0, 4, 2, 6 5 | 0, 4, 2, 6 6 | -------------------------------------------------------------------------------- /test/data/parquet_files/file_1.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/ygm/HEAD/test/data/parquet_files/file_1.parquet -------------------------------------------------------------------------------- /test/data/parquet_files/file_2.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/ygm/HEAD/test/data/parquet_files/file_2.parquet -------------------------------------------------------------------------------- /test/data/parquet_files/file_3.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/ygm/HEAD/test/data/parquet_files/file_3.parquet -------------------------------------------------------------------------------- /examples/Readme.md: -------------------------------------------------------------------------------- 1 | ## Examples 2 | 3 | For examples, please visit the [ygm-tutorial](https://github.com/llnl/ygm-tutorial) repo. 4 | -------------------------------------------------------------------------------- /test/data/parquet_files/parquet.wrong-extension: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/ygm/HEAD/test/data/parquet_files/parquet.wrong-extension -------------------------------------------------------------------------------- /test/data/3.ndjson: -------------------------------------------------------------------------------- 1 | {"id":0, "name":"first"} 2 | {"id":1, "name":"second", "foo":"barr", "special":1} 3 | {"id":2, "name":"third", "extra":"value"} -------------------------------------------------------------------------------- /docs/sphinx/ygm/comm/comm.rst: -------------------------------------------------------------------------------- 1 | .. _ygm-comm 2 | 3 | ygm::comm 4 | ========= 5 | 6 | .. doxygenclass:: ygm::comm 7 | :members: 8 | :undoc-members: 9 | -------------------------------------------------------------------------------- /docs/sphinx/ygm/container/bag.rst: -------------------------------------------------------------------------------- 1 | .. _ygm-container-bag: 2 | 3 | bag 4 | =========================== 5 | 6 | .. doxygenclass:: ygm::container::bag 7 | :members: 8 | :undoc-members: -------------------------------------------------------------------------------- /docs/sphinx/ygm/container/map.rst: -------------------------------------------------------------------------------- 1 | .. _ygm-container-map: 2 | 3 | map 4 | =========================== 5 | 6 | .. doxygenclass:: ygm::container::map 7 | :members: 8 | :undoc-members: -------------------------------------------------------------------------------- /docs/sphinx/ygm/container/set.rst: -------------------------------------------------------------------------------- 1 | .. _ygm-container-set: 2 | 3 | set 4 | =========================== 5 | 6 | .. doxygenclass:: ygm::container::set 7 | :members: 8 | :undoc-members: -------------------------------------------------------------------------------- /docs/sphinx/ygm/container/array.rst: -------------------------------------------------------------------------------- 1 | .. _ygm-container-array: 2 | 3 | array 4 | =========================== 5 | 6 | .. doxygenclass:: ygm::container::array 7 | :members: 8 | :undoc-members: -------------------------------------------------------------------------------- /docs/sphinx/ygm/io/csv_parser.rst: -------------------------------------------------------------------------------- 1 | .. _ygm-io-csv-parser: 2 | 3 | ygm::io::csv_parser 4 | ==================== 5 | 6 | .. doxygenclass:: ygm::io::csv_parser 7 | :members: 8 | :undoc-members: 9 | -------------------------------------------------------------------------------- /docs/sphinx/ygm/io/line_parser.rst: -------------------------------------------------------------------------------- 1 | .. _ygm-io-line-parser: 2 | 3 | ygm::io::line_parser 4 | ==================== 5 | 6 | .. doxygenclass:: ygm::io::line_parser 7 | :members: 8 | :undoc-members: 9 | -------------------------------------------------------------------------------- /docs/sphinx/ygm/utility/timer.rst: -------------------------------------------------------------------------------- 1 | .. _ygm-utility-timer: 2 | 3 | ygm::utility::timer 4 | =================== 5 | 6 | .. doxygenclass:: ygm::utility::timer 7 | :members: 8 | :undoc-members: 9 | -------------------------------------------------------------------------------- /docs/sphinx/ygm/io/daily_output.rst: -------------------------------------------------------------------------------- 1 | .. _ygm-io-daily_output: 2 | 3 | ygm::io::daily_output 4 | ===================== 5 | 6 | .. doxygenclass:: ygm::io::daily_output 7 | :members: 8 | :undoc-members: 9 | -------------------------------------------------------------------------------- /docs/sphinx/ygm/io/multi_output.rst: -------------------------------------------------------------------------------- 1 | .. _ygm-io-multi_output: 2 | 3 | ygm::io::multi_output 4 | ===================== 5 | 6 | .. doxygenclass:: ygm::io::multi_output 7 | :members: 8 | :undoc-members: 9 | -------------------------------------------------------------------------------- /docs/sphinx/ygm/io/ndjson_parser.rst: -------------------------------------------------------------------------------- 1 | .. _ygm-io-ndjson-parser: 2 | 3 | ygm::io::ndjson_parser 4 | ====================== 5 | 6 | .. doxygenclass:: ygm::io::ndjson_parser 7 | :members: 8 | :undoc-members: 9 | -------------------------------------------------------------------------------- /docs/sphinx/ygm/io/parquet_parser.rst: -------------------------------------------------------------------------------- 1 | .. _ygm-io-parquet-parser: 2 | 3 | ygm::io::parquet_parser 4 | ======================= 5 | 6 | .. doxygenclass:: ygm::io::parquet_parser 7 | :members: 8 | :undoc-members: 9 | -------------------------------------------------------------------------------- /docs/sphinx/ygm/container/counting_set.rst: -------------------------------------------------------------------------------- 1 | .. _ygm-container-counting_set: 2 | 3 | counting_set 4 | =========================== 5 | 6 | .. doxygenclass:: ygm::container::counting_set 7 | :members: 8 | :undoc-members: -------------------------------------------------------------------------------- /docs/sphinx/ygm/container/disjoint_set.rst: -------------------------------------------------------------------------------- 1 | .. _ygm-container-disjoint_set: 2 | 3 | disjoint_set 4 | =========================== 5 | 6 | .. doxygenclass:: ygm::container::disjoint_set 7 | :members: 8 | :undoc-members: -------------------------------------------------------------------------------- /test/data/100.csv: -------------------------------------------------------------------------------- 1 | 1,1,1,1,1,1,1,1,1,1 2 | 1,1,1,1,1,1,1,1,1,1 3 | 1,1,1,1,1,1,1,1,1,1 4 | 1,1,1,1,1,1,1,1,1,1 5 | 1,1,1,1,1,1,1,1,1,1 6 | 1,1,1,1,1,1,1,1,1,1 7 | 1,1,1,1,1,1,1,1,1,1 8 | 1,1,1,1,1,1,1,1,1,1 9 | 1,1,1,1,1,1,1,1,1,1 10 | 1,1,1,1,1,1,1,1,1,1 -------------------------------------------------------------------------------- /docs/sphinx/ygm/utility/progress_indicator.rst: -------------------------------------------------------------------------------- 1 | .. _ygm-utility-progress_indicator: 2 | 3 | ygm::utility::progress_indicator 4 | ================================ 5 | 6 | .. doxygenclass:: ygm::utility::progress_indicator 7 | :members: 8 | :undoc-members: 9 | -------------------------------------------------------------------------------- /test/data/bad.ndjson: -------------------------------------------------------------------------------- 1 | {"id":0, "name":"first"} 2 | {"id":0, "name":"first" 3 | {"id":1, "name":"second", "foo":"barr", "special":1} 4 | {"id":1 "name":"second", "foo":"barr", "special":1} 5 | {"id":2, "name":"third", "extra":"value"} 6 | "id":2, "name":"third", "extra":"value"} 7 | -------------------------------------------------------------------------------- /cmake/ygmConfig.cmake.in: -------------------------------------------------------------------------------- 1 | @PACKAGE_INIT@ 2 | 3 | find_package(Threads REQUIRED) 4 | find_package(MPI REQUIRED) 5 | # If cereal not found, hopefully included via FetchContent 6 | find_package(cereal CONFIG QUIET) 7 | 8 | include("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@Targets.cmake") 9 | check_required_components("@PROJECT_NAME@") -------------------------------------------------------------------------------- /include/ygm/version.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2025 Lawrence Livermore National Security, LLC and other YGM 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: MIT 5 | 6 | #include 7 | 8 | namespace ygm { 9 | static const std::string ygm_version("v0.8"); 10 | } 11 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | # Set OS and Python versions 4 | build: 5 | os: ubuntu-24.04 6 | tools: 7 | python: "3.12" 8 | 9 | # Change the location of the requirements file 10 | python: 11 | install: 12 | - requirements: docs/sphinx/requirements.txt 13 | 14 | # Change the location of the configuration file 15 | sphinx: 16 | configuration: docs/sphinx/conf.py 17 | -------------------------------------------------------------------------------- /cmake/FindSphinx.cmake: -------------------------------------------------------------------------------- 1 | #Look for an executable called sphinx-build 2 | find_program(SPHINX_EXECUTABLE 3 | NAMES sphinx-build 4 | DOC "Path to sphinx-build executable") 5 | 6 | include(FindPackageHandleStandardArgs) 7 | 8 | #Handle standard arguments to find_package like REQUIRED and QUIET 9 | find_package_handle_standard_args(Sphinx 10 | "Failed to find sphinx-build executable" 11 | SPHINX_EXECUTABLE) -------------------------------------------------------------------------------- /.devcontainer/Dockerfile: -------------------------------------------------------------------------------- 1 | # FROM mcr.microsoft.com/devcontainers/cpp:1-ubuntu-24.04 2 | FROM mcr.microsoft.com/devcontainers/cpp:dev-ubuntu-24.04 3 | 4 | RUN apt update && export DEBIAN_FRONTEND=noninteractive && apt -y install clangd-19 clang-tidy-19 openmpi-bin openmpi-doc libopenmpi-dev python3-sphinx python3-breathe doxygen python3-sphinx-rtd-theme texlive-full 5 | RUN update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-19 100 6 | RUN update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-19 100 -------------------------------------------------------------------------------- /include/ygm/random.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2025 Lawrence Livermore National Security, LLC and other YGM 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: MIT 5 | 6 | #pragma once 7 | 8 | #include 9 | 10 | namespace ygm { 11 | 12 | /// @brief A simple offset rng alias 13 | /// @tparam RandomEngine The underlying random engine, e.g. std::mt19937 14 | template 15 | using default_random_engine = 16 | ygm::detail::random_engine; 17 | 18 | } // namespace ygm -------------------------------------------------------------------------------- /test/test_comm_2.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2025 Lawrence Livermore National Security, LLC and other YGM 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: MIT 5 | 6 | #undef NDEBUG 7 | #include 8 | #include 9 | 10 | int main(int argc, char** argv) { 11 | int provided; 12 | YGM_ASSERT_MPI(MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided)); 13 | YGM_ASSERT_RELEASE(MPI_THREAD_MULTIPLE == provided); 14 | 15 | for (size_t i = 0; i < 1000; ++i) { ygm::comm world(MPI_COMM_WORLD); } 16 | 17 | YGM_ASSERT_MPI(MPI_Finalize()); 18 | return 0; 19 | } -------------------------------------------------------------------------------- /include/ygm/detail/std_traits.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2025 Lawrence Livermore National Security, LLC and other YGM 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: MIT 5 | 6 | #pragma once 7 | #include 8 | #include 9 | 10 | namespace ygm::detail { 11 | 12 | // is_std_pair 13 | template 14 | struct is_std_pair_impl : std::false_type {}; 15 | 16 | template 17 | struct is_std_pair_impl> : std::true_type {}; 18 | 19 | template 20 | constexpr bool is_std_pair = is_std_pair_impl::value; 21 | 22 | } // namespace ygm::detail 23 | -------------------------------------------------------------------------------- /.github/workflows/test_docs_build.yaml: -------------------------------------------------------------------------------- 1 | name: Test Documentation Generation 2 | 3 | on: 4 | pull_request: 5 | branches: [ master, '**-dev' ] 6 | push: 7 | branches: [ 'feature/**', 'hotfix/**'] 8 | 9 | jobs: 10 | build_docs: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v4 14 | - name: Update apt 15 | run: | 16 | sudo apt-get update 17 | - name: Install dependencies 18 | run: | 19 | sudo apt-get install doxygen sphinx 20 | pip install -r docs/sphinx/requirements.txt 21 | - name: make 22 | run: | 23 | mkdir build 24 | cd build 25 | cmake ../ -DYGM_DOCS_ONLY=On 26 | make sphinx 27 | -------------------------------------------------------------------------------- /COPYRIGHT: -------------------------------------------------------------------------------- 1 | Intellectual Property Notice 2 | ------------------------------ 3 | 4 | YGM uses the MIT license, (LICENSE-MIT or http://opensource.org/licenses/MIT). 5 | 6 | Copyrights and patents in the YGM project are retained by contributors. 7 | No copyright assignment is required to contribute to YGM. 8 | 9 | 10 | SPDX usage 11 | ------------ 12 | 13 | Individual files contain SPDX tags instead of the full license text. 14 | This enables machine processing of license information based on the SPDX 15 | License Identifiers that are available here: https://spdx.org/licenses/ 16 | 17 | Files that are licensed under MIT contain the following text in the license header: 18 | 19 | SPDX-License-Identifier: MIT 20 | -------------------------------------------------------------------------------- /docs/sphinx/index.rst: -------------------------------------------------------------------------------- 1 | .. YGM documentation master file, created by 2 | sphinx-quickstart on Mon Apr 3 18:10:58 2023. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | YGM library documentation 7 | ===================================== 8 | 9 | .. toctree:: 10 | :maxdepth: 2 11 | :caption: Contents: 12 | 13 | getting_started 14 | ygm/comm 15 | ygm/container 16 | ygm/io 17 | ygm/utility 18 | 19 | Documents for YGM developers 20 | --------- 21 | 22 | .. toctree:: 23 | :maxdepth: 1 24 | 25 | dev 26 | 27 | Indices and tables 28 | ================== 29 | 30 | * :ref:`genindex` 31 | * :ref:`modindex` 32 | * :ref:`search` 33 | -------------------------------------------------------------------------------- /test/test_boost_string.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2025 Lawrence Livermore National Security, LLC and other YGM 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: MIT 5 | 6 | #undef NDEBUG 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | int main(int argc, char **argv) { 14 | ygm::comm world(&argc, &argv); 15 | 16 | { 17 | ygm::container::set> sset(world); 18 | sset.async_insert("dog"); 19 | sset.async_insert("apple"); 20 | sset.async_insert("red"); 21 | world.barrier(); 22 | for (auto s : sset) { 23 | world.cout(s); 24 | } 25 | } 26 | } -------------------------------------------------------------------------------- /include/ygm/utility/timer.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2025 Lawrence Livermore National Security, LLC and other YGM 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: MIT 5 | 6 | #pragma once 7 | 8 | #include 9 | 10 | namespace ygm::utility { 11 | 12 | /** 13 | * @brief Simple timer class using `MPI_Wtime()` 14 | */ 15 | class timer { 16 | public: 17 | timer() { reset(); } 18 | 19 | /** 20 | * @brief Get time since timer creation or last `reset()` 21 | * 22 | * @return Elapsed time 23 | */ 24 | double elapsed() { return MPI_Wtime() - m_start; } 25 | 26 | /** 27 | * @brief Restart timer 28 | */ 29 | void reset() { m_start = MPI_Wtime(); } 30 | 31 | private: 32 | double m_start; 33 | }; 34 | } // namespace ygm::utility 35 | -------------------------------------------------------------------------------- /include/ygm/detail/interrupt_mask.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2025 Lawrence Livermore National Security, LLC and other YGM 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: MIT 5 | 6 | #pragma once 7 | 8 | #include 9 | 10 | namespace ygm { 11 | 12 | namespace detail { 13 | 14 | class interrupt_mask { 15 | public: 16 | interrupt_mask(ygm::comm &c) : m_comm(c) { 17 | m_comm.m_enable_interrupts = false; 18 | } 19 | 20 | ~interrupt_mask() { 21 | m_comm.m_enable_interrupts = true; 22 | m_comm.check_if_production_halt_required(); 23 | // m_comm.process_receive_queue(); //causes recursion into 24 | // process_receive_queue 25 | } 26 | 27 | private: 28 | ygm::comm &m_comm; 29 | }; 30 | 31 | } // namespace detail 32 | } // namespace ygm 33 | -------------------------------------------------------------------------------- /cmake/FindPython3Module.cmake: -------------------------------------------------------------------------------- 1 | # Find a Python3 module using CMake's FindPython3 module. 2 | # Input: module name to find 3 | # Python3_ROOT_DIR can be used as a hint to find Python3 4 | # 5 | # Output: PYTHON3_MODULE_PATH is set to the path of the module if found 6 | function(find_python3_module module_name) 7 | find_package(Python3 COMPONENTS Interpreter REQUIRED) 8 | 9 | execute_process( 10 | COMMAND ${Python3_EXECUTABLE} -c "import importlib; import sys; module_name = '${module_name}'; spec = importlib.util.find_spec(module_name); print(spec.origin if spec else ''); sys.exit(0 if spec else 1)" 11 | OUTPUT_VARIABLE MODULE_PATH 12 | OUTPUT_STRIP_TRAILING_WHITESPACE 13 | ) 14 | 15 | if (Python3_FOUND AND MODULE_PATH) 16 | set(PYTHON3_MODULE_PATH ${MODULE_PATH} PARENT_SCOPE) 17 | endif () 18 | endfunction() -------------------------------------------------------------------------------- /test/test_csv_parser.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2025 Lawrence Livermore National Security, LLC and other YGM 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: MIT 5 | 6 | #undef NDEBUG 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | int main(int argc, char** argv) { 13 | ygm::comm world(&argc, &argv); 14 | 15 | size_t local_count{0}; 16 | ygm::io::csv_parser csvp(world, std::vector{"data/100.csv"}); 17 | csvp.for_all([&world, &local_count](const auto& vfields) { 18 | for (auto f : vfields) { 19 | YGM_ASSERT_RELEASE(f.is_integer()); 20 | local_count += f.as_integer(); 21 | } 22 | }); 23 | 24 | world.barrier(); 25 | YGM_ASSERT_RELEASE(ygm::sum(local_count, world) == 100); 26 | 27 | return 0; 28 | } 29 | -------------------------------------------------------------------------------- /test/test_reduce.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2025 Lawrence Livermore National Security, LLC and other YGM 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: MIT 5 | 6 | #undef NDEBUG 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | int main(int argc, char** argv) { 17 | ygm::comm world(&argc, &argv); 18 | 19 | { 20 | ygm::container::bag ibag(world, {42, 1, 8, 16, 32, 3, 4, 5, 6, 7}); 21 | 22 | int sum = ibag.reduce(std::plus()); 23 | YGM_ASSERT_RELEASE(sum = 124); 24 | 25 | 26 | int even_sum = ibag.filter([](int i){return i%2==0;}).reduce(std::plus()); 27 | YGM_ASSERT_RELEASE(even_sum = 108); 28 | } 29 | 30 | } -------------------------------------------------------------------------------- /include/ygm/container/detail/round_robin_partitioner.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2025 Lawrence Livermore National Security, LLC and other YGM 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: MIT 5 | 6 | #pragma once 7 | 8 | namespace ygm::container::detail { 9 | 10 | /** 11 | * @brief Assigns items to ranks according to a cyclic distribution 12 | * 13 | */ 14 | struct round_robin_partitioner { 15 | round_robin_partitioner(const ygm::comm &comm) 16 | : m_next(comm.rank()), m_comm_size(comm.size()) {} 17 | 18 | /** 19 | * @brief Assigns item to next rank for storage 20 | * 21 | * @return Next rank in cycle 22 | */ 23 | template 24 | int owner(const Item &) { 25 | if (++m_next >= m_comm_size) { 26 | m_next = 0; 27 | } 28 | return m_next; 29 | } 30 | int m_next; 31 | int m_comm_size; 32 | }; 33 | 34 | } // namespace ygm::container::detail 35 | -------------------------------------------------------------------------------- /test/test_interrupt_mask.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2025 Lawrence Livermore National Security, LLC and other YGM 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: MIT 5 | 6 | #undef NDEBUG 7 | #include 8 | #include 9 | 10 | int main(int argc, char** argv) { 11 | ::setenv("YGM_COMM_BUFFER_SIZE_KB", "1", 1); 12 | ygm::comm world(&argc, &argv); 13 | 14 | int count{0}; 15 | auto count_ptr = world.make_ygm_ptr(count); 16 | int num_sends{100}; 17 | { 18 | ygm::detail::interrupt_mask mask(world); 19 | 20 | for (int i = 0; i < num_sends; ++i) { 21 | world.async( 22 | 0, [](auto count_ptr) { ++(*count_ptr); }, count_ptr); 23 | } 24 | 25 | world.cf_barrier(); 26 | 27 | YGM_ASSERT_RELEASE(count == 0); 28 | } 29 | 30 | world.barrier(); 31 | 32 | if (world.rank0()) { 33 | YGM_ASSERT_RELEASE(count == num_sends * world.size()); 34 | } 35 | 36 | return 0; 37 | } 38 | -------------------------------------------------------------------------------- /test/test_lambda_capture.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | #include 4 | 5 | #include 6 | 7 | int main(int argc, char **argv) { 8 | ygm::comm world(&argc, &argv); 9 | 10 | // Capture in comm::async 11 | { 12 | int a = 12; 13 | world.async(0, [a]() { YGM_ASSERT_RELEASE(a == 12); }); 14 | } 15 | 16 | // Capture in comm::async_bcast 17 | { 18 | int a = 12; 19 | if (world.rank0()) { 20 | world.async_bcast([a]() { YGM_ASSERT_RELEASE(a == 12); }); 21 | } 22 | } 23 | 24 | // Capture in container::map::async_visit 25 | { 26 | ygm::container::map my_map(world); 27 | 28 | int a = world.rank(); 29 | my_map.async_visit("key", 30 | [a](const std::string &key, int &val) { val += a; }); 31 | 32 | world.barrier(); 33 | my_map.for_all([&world](const std::string &key, const int &val) { 34 | YGM_ASSERT_RELEASE(val == world.size() * (world.size() - 1) / 2); 35 | }); 36 | } 37 | 38 | return 0; 39 | } 40 | -------------------------------------------------------------------------------- /.devcontainer/devcontainer.json: -------------------------------------------------------------------------------- 1 | // For format details, see https://aka.ms/devcontainer.json. For config options, see the 2 | // README at: https://github.com/devcontainers/templates/tree/main/src/ubuntu 3 | { 4 | "name": "YGM", 5 | "build": { 6 | "dockerfile": "Dockerfile" 7 | }, 8 | // Features to add to the dev container. More info: https://containers.dev/features. 9 | // "features": {}, 10 | // Use 'forwardPorts' to make a list of ports inside the container available locally. 11 | // "forwardPorts": [], 12 | // Use 'postCreateCommand' to run commands after the container is created. 13 | // "postCreateCommand": "uname -a", 14 | "extensions": [ 15 | "llvm-vs-code-extensions.vscode-clangd", 16 | // add other extensions as needed 17 | ] 18 | "mounts": [ 19 | "source=${localEnv:HOME}${localEnv:USERPROFILE},target=/host-home,type=bind,consistency=cached", 20 | ] 21 | // Configure tool-specific properties. 22 | // "customizations": {}, 23 | // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root. 24 | // "remoteUser": "root" 25 | } -------------------------------------------------------------------------------- /test/test_cereal_boost_container.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2025 Lawrence Livermore National Security, LLC and other YGM 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: MIT 5 | 6 | #undef NDEBUG 7 | 8 | #include 9 | 10 | #include 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | int main() { 18 | // Currently, we only support boost::container::vector 19 | 20 | boost::container::vector original_value = {1, 2, 3, 4, 5}; 21 | ygm::detail::byte_vector cereal_buffer; 22 | { 23 | cereal::YGMOutputArchive archive(cereal_buffer); 24 | archive(original_value); 25 | } 26 | 27 | { 28 | cereal::YGMInputArchive archive(cereal_buffer.data(), cereal_buffer.size()); 29 | boost::container::vector load_value; 30 | archive(load_value); 31 | 32 | YGM_ASSERT_RELEASE(original_value == load_value); 33 | } 34 | 35 | return 0; 36 | } -------------------------------------------------------------------------------- /include/ygm/detail/lambda_compliance.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2025 Lawrence Livermore National Security, LLC and other YGM 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: MIT 5 | 6 | #pragma once 7 | 8 | #include 9 | #include 10 | 11 | // YGM_SKIP_ASYNC_LAMBDA_COMPLIANCE should not be used under normal 12 | // circumstances but can be useful for testing, such as allowing functors with 13 | // non-default constructors that print when called 14 | #if YGM_SKIP_ASYNC_LAMBDA_COMPLIANCE 15 | #define YGM_CHECK_ASYNC_LAMBDA_COMPLIANCE(func, location) 16 | #else 17 | #define YGM_CHECK_ASYNC_LAMBDA_COMPLIANCE(func, location) \ 18 | static_assert( \ 19 | std::is_trivially_copyable>::value && \ 20 | std::is_standard_layout>::value, \ 21 | location \ 22 | " function object must be is_trivially_copyable & is_standard_layout.") 23 | #endif 24 | -------------------------------------------------------------------------------- /.github/workflows/build-pdf-docs.yaml: -------------------------------------------------------------------------------- 1 | name: Build PDF Documentation 2 | 3 | on: 4 | pull_request: 5 | types: 6 | - closed 7 | 8 | jobs: 9 | build_docs: 10 | if: github.event.pull_request.merged == true 11 | runs-on: ubuntu-latest 12 | permissions: 13 | contents: write 14 | steps: 15 | - uses: actions/checkout@v4 16 | - name: Update apt 17 | run: | 18 | sudo apt-get update 19 | - name: Install dependencies 20 | run: | 21 | sudo apt-get install doxygen sphinx 22 | pip install -r docs/sphinx/requirements.txt 23 | sudo apt-get install texlive-latex-base texlive-latex-extra texlive-fonts-recommended texlive-fonts-extra latexmk 24 | - name: make 25 | run: | 26 | mkdir build 27 | cd build 28 | cmake ../ -DYGM_DOCS_ONLY=On 29 | make sphinx-pdf 30 | - name: Move PDF 31 | run: | 32 | cp build/docs/sphinx/latex/ygm.pdf ./docs 33 | - uses: EndBug/add-and-commit@v9 34 | with: 35 | author_name: YGM-docs bot 36 | message: 'Generating and committing docs' 37 | env: 38 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 39 | -------------------------------------------------------------------------------- /include/ygm/container/detail/base_count.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2025 Lawrence Livermore National Security, LLC and other YGM 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: MIT 5 | 6 | #pragma once 7 | 8 | #include 9 | #include 10 | 11 | namespace ygm::container::detail { 12 | 13 | /** 14 | * @brief Curiously-recurring template parameter struct that provides 15 | * count operation 16 | */ 17 | template 18 | struct base_count { 19 | /** 20 | * @brief Counts all occurrences of a value within a container. 21 | * 22 | * @param value Value to search for within container (key in the case of 23 | * containers with keys) 24 | * @return Count of times `value` is seen in container 25 | */ 26 | size_t count( 27 | const typename std::tuple_element<0, for_all_args>::type& value) const { 28 | const derived_type* derived_this = static_cast(this); 29 | derived_this->comm().barrier(); 30 | return ::ygm::sum(derived_this->local_count(value), derived_this->comm()); 31 | } 32 | }; 33 | 34 | } // namespace ygm::container::detail 35 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | This work was produced under the auspices of the U.S. Department of 2 | Energy by Lawrence Livermore National Laboratory under Contract 3 | DE-AC52-07NA27344. 4 | 5 | This work was prepared as an account of work sponsored by an agency of 6 | the United States Government. Neither the United States Government nor 7 | Lawrence Livermore National Security, LLC, nor any of their employees 8 | makes any warranty, expressed or implied, or assumes any legal liability 9 | or responsibility for the accuracy, completeness, or usefulness of any 10 | information, apparatus, product, or process disclosed, or represents that 11 | its use would not infringe privately owned rights. 12 | 13 | Reference herein to any specific commercial product, process, or service 14 | by trade name, trademark, manufacturer, or otherwise does not necessarily 15 | constitute or imply its endorsement, recommendation, or favoring by the 16 | United States Government or Lawrence Livermore National Security, LLC. 17 | 18 | The views and opinions of authors expressed herein do not necessarily 19 | state or reflect those of the United States Government or Lawrence 20 | Livermore National Security, LLC, and shall not be used for advertising 21 | or product endorsement purposes. -------------------------------------------------------------------------------- /test/test_barrier.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2025 Lawrence Livermore National Security, LLC and other YGM 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: MIT 5 | 6 | #undef NDEBUG 7 | 8 | #include 9 | 10 | int main(int argc, char **argv) { 11 | ygm::comm world(&argc, &argv); 12 | 13 | // Test barriers for early exit 14 | { 15 | int num_rounds = 100; 16 | static int round = 0; 17 | for (int i = 0; i < num_rounds; ++i) { 18 | world.async_bcast( 19 | [](int curr_round) { YGM_ASSERT_RELEASE(curr_round == round); }, 20 | round); 21 | 22 | world.barrier(); 23 | 24 | ++round; 25 | } 26 | } 27 | 28 | // Tests async_barriers 29 | int num_rounds = 100; 30 | static int round = 0; 31 | for (int i = 0; i < num_rounds; ++i) { 32 | world.async_bcast( 33 | [](int curr_round) { YGM_ASSERT_RELEASE(curr_round == round); }, round); 34 | 35 | for (int j = 0; j < world.rank(); 36 | ++j) { // each rank calls 'rank' number of async_barriers 37 | world.async_barrier(); 38 | } 39 | world.barrier(); 40 | 41 | ++round; 42 | } 43 | 44 | return 0; 45 | } 46 | -------------------------------------------------------------------------------- /test/test_recursion_progress.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2025 Lawrence Livermore National Security, LLC and other YGM 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: MIT 5 | 6 | #undef NDEBUG 7 | #include 8 | 9 | static size_t counter = 0; 10 | 11 | // Recursive message spawns one child to next rank. 12 | struct recursive_functor { 13 | template 14 | void operator()(Comm *pcomm, size_t hops) { 15 | counter++; 16 | 17 | hops--; 18 | 19 | if (hops > 0) { 20 | pcomm->async((pcomm->rank() + 1) % pcomm->size(), recursive_functor(), 21 | hops); 22 | pcomm->local_progress(); 23 | } 24 | } 25 | }; 26 | 27 | int main(int argc, char **argv) { 28 | ygm::comm world(&argc, &argv); 29 | 30 | // Test recursion with local_progress() in 'around the world' format 31 | { 32 | size_t trips = 100; 33 | 34 | size_t desired_hops = world.size() * trips + 1; 35 | 36 | if (world.rank0()) { 37 | world.async(0, recursive_functor(), desired_hops); 38 | } 39 | 40 | world.barrier(); 41 | YGM_ASSERT_RELEASE(ygm::sum(counter, world) == (world.size() * trips + 1)); 42 | } 43 | 44 | return 0; 45 | } 46 | -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright 2019-2025 Lawrence Livermore National Security, LLC and other 4 | YGM Project Developers. See the top-level COPYRIGHT file for details. 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | -------------------------------------------------------------------------------- /.gitlab-ci.yml: -------------------------------------------------------------------------------- 1 | # Ruby-GCC9-openmpi-release: 2 | # tags: 3 | # - ruby 4 | # - shell 5 | # variables: 6 | # OMP_PROC_BIND: "false" 7 | # script: 8 | # - . /usr/workspace/llamag/spack/share/spack/setup-env.sh 9 | # - spack load gcc@9 10 | # - module load openmpi 11 | # - spack load nlohmann-json 12 | # - mkdir build 13 | # - cd build 14 | # - cmake ../ -DTEST_WITH_SLURM=ON -DCMAKE_BUILD_TYPE=Release 15 | # - make VERBOSE=1 16 | # - ctest -VV -R SEQ 17 | # - salloc -N2 --ntasks-per-node=4 -ppdebug --mpibind=on ctest -VV -E SEQ 18 | # - salloc -N4 --ntasks-per-node=16 -ppdebug --mpibind=on ctest -VV -E SEQ 19 | 20 | 21 | Ruby-GCC-mvapich2-release: 22 | tags: 23 | - ruby 24 | - shell 25 | variables: 26 | OMP_PROC_BIND: "false" 27 | script: 28 | - . /usr/workspace/llamag/spack_2021.01.26/share/spack/setup-env.sh 29 | - spack load gcc@9 30 | - module load mvapich2 31 | - spack load nlohmann-json 32 | - mkdir build 33 | - cd build 34 | - cmake ../ -DTEST_WITH_SLURM=ON -DCMAKE_BUILD_TYPE=Release 35 | - make VERBOSE=1 36 | - ctest -VV -R SEQ 37 | - salloc -N2 --ntasks-per-node=4 -ppdebug --mpibind=on ctest -VV -E SEQ 38 | - salloc -N4 --ntasks-per-node=16 -ppdebug --mpibind=on ctest -VV -E SEQ 39 | 40 | -------------------------------------------------------------------------------- /test/test_ndjson_parser.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2025 Lawrence Livermore National Security, LLC and other YGM 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: MIT 5 | 6 | #undef NDEBUG 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | int main(int argc, char** argv) { 13 | ygm::comm world(&argc, &argv); 14 | 15 | { 16 | size_t local_count{0}; 17 | ygm::io::ndjson_parser jsonp(world, 18 | std::vector{"data/3.ndjson"}); 19 | jsonp.for_all([&world, &local_count](const auto& json) { ++local_count; }); 20 | 21 | world.barrier(); 22 | YGM_ASSERT_RELEASE(ygm::sum(local_count, world) == 3); 23 | } 24 | 25 | // Test json with bad lines 26 | { 27 | size_t local_count{0}; 28 | ygm::io::ndjson_parser jsonp(world, 29 | std::vector{"data/bad.ndjson"}); 30 | jsonp.for_all([&world, &local_count](const auto& json) { ++local_count; }); 31 | 32 | world.barrier(); 33 | YGM_ASSERT_RELEASE(ygm::sum(local_count, world) == 3); 34 | YGM_ASSERT_RELEASE(jsonp.num_invalid_records() == 3); 35 | } 36 | 37 | return 0; 38 | } 39 | -------------------------------------------------------------------------------- /test/test_cereal_archive.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2025 Lawrence Livermore National Security, LLC and other YGM 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: MIT 5 | 6 | #undef NDEBUG 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | int main() { 15 | std::vector vec_sentences = { 16 | "Four score and seven years ago", 17 | "our fathers brought forth on this continent", 18 | "a new nation conceived in liberty"}; 19 | 20 | ygm::detail::byte_vector buffer; 21 | { 22 | cereal::YGMOutputArchive archive(buffer); 23 | for (const auto& s : vec_sentences) { archive(s); } 24 | } 25 | 26 | { 27 | std::ifstream is("out.cereal", std::ios::binary); 28 | cereal::YGMInputArchive archive(buffer.data(), buffer.size()); 29 | 30 | std::vector out_sentences; 31 | while (!archive.empty()) { 32 | std::string tmp; 33 | archive(tmp); 34 | // std::cout << tmp << std::endl; 35 | out_sentences.push_back(tmp); 36 | } 37 | YGM_ASSERT_RELEASE(vec_sentences == out_sentences); 38 | } 39 | 40 | return 0; 41 | } 42 | -------------------------------------------------------------------------------- /test/test_wout_werr.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2025 Lawrence Livermore National Security, LLC and other YGM 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: MIT 5 | 6 | #undef NDEBUG 7 | 8 | #include 9 | 10 | int main(int argc, char** argv) { 11 | ygm::comm world(&argc, &argv); 12 | 13 | { // testing comm's ostream helpers 14 | world.cout() << "cout all ranks." << std::endl; 15 | world.cerr() << "cerr all ranks." << std::endl; 16 | world.cout0() << "cout only rank 0." << std::endl; 17 | world.cerr0() << "cerr only rank 0." << std::endl; 18 | 19 | world.cout("variadic cout all ranks."); 20 | world.cerr("variadic cerr all ranks."); 21 | world.cout0("variadic cout only rank 0."); 22 | world.cerr0("variadic cerr only rank 0."); 23 | } 24 | 25 | { // testing ygm::'s ostream helpers 26 | ygm::wcout() << "cout all ranks." << std::endl; 27 | ygm::wcerr() << "cerr all ranks." << std::endl; 28 | ygm::wcout0() << "cout only rank 0." << std::endl; 29 | ygm::wcerr0() << "cerr only rank 0." << std::endl; 30 | 31 | ygm::wcout("variadic cout all ranks."); 32 | ygm::wcerr("variadic cerr all ranks."); 33 | ygm::wcout0("variadic cout only rank 0."); 34 | ygm::wcerr0("variadic cerr only rank 0."); 35 | } 36 | } -------------------------------------------------------------------------------- /test/test_large_messages.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2025 Lawrence Livermore National Security, LLC and other YGM 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: MIT 5 | 6 | #undef NDEBUG 7 | #include 8 | #include 9 | #include 10 | 11 | int main(int argc, char** argv) { 12 | // Create comm for very small messages 13 | ::setenv("YGM_COMM_BUFFER_SIZE_KB", "1", 1); 14 | ygm::comm world(&argc, &argv); 15 | 16 | // Test Rank 0 large message to all ranks 17 | { 18 | size_t large_msg_size = 1024 * 1024; 19 | size_t counter{}; 20 | auto pcounter = world.make_ygm_ptr(counter); 21 | if (world.rank() == 0) { 22 | std::vector large_msg(large_msg_size); 23 | for (int dest = 0; dest < world.size(); ++dest) { 24 | // Count elements in large message's vector 25 | world.async( 26 | dest, 27 | [](auto pcomm, auto pcounter, const std::vector& vec) { 28 | for (size_t i = 0; i < vec.size(); ++i) { 29 | (*pcounter)++; 30 | } 31 | }, 32 | pcounter, large_msg); 33 | } 34 | } 35 | 36 | world.barrier(); 37 | YGM_ASSERT_RELEASE(counter == large_msg_size); 38 | } 39 | 40 | return 0; 41 | } 42 | -------------------------------------------------------------------------------- /test/test_csv_headers.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2025 Lawrence Livermore National Security, LLC and other YGM 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: MIT 5 | 6 | #undef NDEBUG 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | int main(int argc, char** argv) { 13 | ygm::comm world(&argc, &argv); 14 | 15 | ygm::io::csv_parser csvp(world, 16 | std::vector{"data/csv_headers.csv"}); 17 | csvp.read_headers(); 18 | csvp.for_all([&world](const auto& vfields) { 19 | // Test lookups by header names 20 | YGM_ASSERT_RELEASE(vfields["zero"].as_integer() == 0); 21 | YGM_ASSERT_RELEASE(vfields["two"].as_integer() == 2); 22 | YGM_ASSERT_RELEASE(vfields["four"].as_integer() == 4); 23 | YGM_ASSERT_RELEASE(vfields["six"].as_integer() == 6); 24 | 25 | // Test lookup by column names agrees with positional lookups 26 | YGM_ASSERT_RELEASE(vfields["zero"].as_integer() == vfields[0].as_integer()); 27 | YGM_ASSERT_RELEASE(vfields["two"].as_integer() == vfields[2].as_integer()); 28 | YGM_ASSERT_RELEASE(vfields["four"].as_integer() == vfields[1].as_integer()); 29 | YGM_ASSERT_RELEASE(vfields["six"].as_integer() == vfields[3].as_integer()); 30 | }); 31 | 32 | world.barrier(); 33 | 34 | return 0; 35 | } 36 | -------------------------------------------------------------------------------- /test/test_progress_indicator.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2025 Lawrence Livermore National Security, LLC and other YGM 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: MIT 5 | 6 | #include 7 | #include 8 | #undef NDEBUG 9 | 10 | #include 11 | #include 12 | 13 | int main(int argc, char **argv) { 14 | ygm::comm world(&argc, &argv); 15 | 16 | { 17 | ygm::utility::progress_indicator prog(world, {.message = "Test 1"}); 18 | for (size_t i = 0; i < 1000; ++i) { 19 | prog.async_inc(); 20 | std::this_thread::sleep_for(std::chrono::milliseconds(1)); 21 | } 22 | prog.complete(); 23 | world.barrier(); 24 | } 25 | 26 | { 27 | ygm::utility::progress_indicator prog(world, {.message = "Test 2"}); 28 | for (size_t i = 0; i < 1000; ++i) { 29 | prog.async_inc(); 30 | world.async_barrier(); 31 | } 32 | prog.complete(); 33 | world.barrier(); 34 | } 35 | 36 | // Testing Barrier before Complete. THIS IS NOT IDEAL USAGE 37 | { 38 | ygm::utility::progress_indicator prog(world, {.message = "Test 3"}); 39 | for (size_t i = 0; i < 1000 * (world.rank() + 100); ++i) { 40 | prog.async_inc(); 41 | world.async_barrier(); 42 | } 43 | world.barrier(); 44 | prog.complete(); 45 | } 46 | 47 | return 0; 48 | } 49 | -------------------------------------------------------------------------------- /test/test_gather_topk.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2025 Lawrence Livermore National Security, LLC and other YGM 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: MIT 5 | 6 | #undef NDEBUG 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | int main(int argc, char** argv) { 17 | ygm::comm world(&argc, &argv); 18 | 19 | { 20 | ygm::container::bag ibag(world, {42, 1, 8, 16, 32, 3, 4, 5, 6, 7}); 21 | 22 | auto top2 = ibag.gather_topk(2); 23 | 24 | YGM_ASSERT_RELEASE(top2[0] == 42); 25 | YGM_ASSERT_RELEASE(top2[1] == 32); 26 | YGM_ASSERT_RELEASE(top2.size() == 2); 27 | } 28 | 29 | { 30 | ygm::container::counting_set cs(world); 31 | cs.async_insert("one"); 32 | cs.async_insert("fish"); 33 | cs.async_insert("two"); 34 | cs.async_insert("fish"); 35 | cs.async_insert("red"); 36 | cs.async_insert("fish"); 37 | cs.async_insert("blue"); 38 | cs.async_insert("fish"); 39 | 40 | std::vector< std::pair > top1 = cs.gather_topk( 41 | 1, [](auto p1, auto p2) { return p1.second > p2.second; }); 42 | 43 | YGM_ASSERT_RELEASE(top1[0].first == "fish"); 44 | YGM_ASSERT_RELEASE(top1[0].second == 4 * world.size()); 45 | } 46 | } -------------------------------------------------------------------------------- /.cmake-format.py: -------------------------------------------------------------------------------- 1 | # ----------------------------- 2 | # Options effecting formatting. 3 | # 4 | # Requires cmake-format (pip install cmake_format). See 5 | # https://cmake-format.readthedocs.io/en/latest/configuration.html for more 6 | # examples. 7 | # ----------------------------- 8 | with section("format"): 9 | 10 | # How wide to allow formatted cmake files 11 | line_width = 80 12 | 13 | # How many spaces to tab for indent 14 | tab_size = 4 15 | 16 | use_tabchars = False 17 | 18 | # If true, separate flow control names from their parentheses with a space 19 | separate_ctrl_name_with_space = True 20 | 21 | # If true, separate function names from parentheses with a space 22 | separate_fn_name_with_space = False 23 | 24 | # If a statement is wrapped to more than one line, than dangle the closing 25 | # parenthesis on its own line. 26 | dangle_parens = True 27 | 28 | # If the trailing parenthesis must be 'dangled' on its on line, then align it 29 | # to this reference: `prefix`: the start of the statement, `prefix-indent`: 30 | # the start of the statement, plus one indentation level, `child`: align to 31 | # the column of the arguments 32 | dangle_align = "prefix" 33 | 34 | # Format command names consistently as 'lower' or 'upper' case 35 | command_case = "canonical" 36 | 37 | # Format keywords consistently as 'lower' or 'upper' case 38 | keyword_case = "unchanged" 39 | -------------------------------------------------------------------------------- /include/ygm/utility/boost_static_string.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2025 Lawrence Livermore National Security, LLC and other YGM 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: MIT 5 | 6 | #pragma once 7 | 8 | /** 9 | * @file 10 | * @brief Support for boost::static 11 | */ 12 | 13 | #include 14 | #include 15 | 16 | namespace cereal { 17 | /// \brief Save function for boost::json::string, 18 | /// which is different std::string or boost::string. 19 | template 20 | void CEREAL_SAVE_FUNCTION_NAME(Archive &archive, 21 | const boost::static_string &str) { 22 | // Length (#of chars in the string) 23 | archive(cereal::make_size_tag(static_cast(str.size()))); 24 | 25 | // String data 26 | archive(cereal::binary_data(str.data(), str.size() * sizeof(char))); 27 | } 28 | 29 | /// \brief Load function for boost::json::string, 30 | /// which is different std::string or boost::string. 31 | template 32 | void CEREAL_LOAD_FUNCTION_NAME(Archive &archive, boost::static_string &str) { 33 | std::size_t size; 34 | archive(cereal::make_size_tag(size)); 35 | 36 | str.resize(size); 37 | archive( 38 | cereal::binary_data(const_cast(str.data()), size * sizeof(char))); 39 | } 40 | 41 | } // namespace cereal -------------------------------------------------------------------------------- /test/test_cereal_boost_json.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2025 Lawrence Livermore National Security, LLC and other YGM 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: MIT 5 | 6 | #undef NDEBUG 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | namespace bj = boost::json; 17 | 18 | std::string json_string = R"( 19 | { 20 | "pi": 3.141, 21 | "happy": true, 22 | "name": "Alice", 23 | "nothing": null, 24 | "list": [1, 0, 2], 25 | "object": { 26 | "currency": "USD", 27 | "value": -10 28 | } 29 | } 30 | )"; 31 | 32 | int main() { 33 | ygm::detail::byte_vector buffer; 34 | { 35 | const bj::value value = bj::parse(json_string); 36 | cereal::YGMOutputArchive archive(buffer); 37 | archive(value); 38 | } 39 | 40 | { 41 | cereal::YGMInputArchive archive(buffer.data(), buffer.size()); 42 | bj::value load_value; 43 | archive(load_value); 44 | 45 | const bj::value original_value = bj::parse(json_string); 46 | // std::cout << original_value << std::endl; 47 | // std::cout << load_value << std::endl; 48 | YGM_ASSERT_RELEASE(original_value == load_value); 49 | } 50 | 51 | return 0; 52 | } -------------------------------------------------------------------------------- /include/ygm/container/detail/type_traits.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2025 Lawrence Livermore National Security, LLC and other YGM 2 | // Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: MIT 5 | 6 | #pragma once 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | namespace ygm::container::detail::type_traits { 13 | template