├── ci ├── __init__.py ├── hermes │ ├── __init__.py │ ├── packages │ │ ├── hermes │ │ │ └── __init__.py │ │ └── hermes_shm │ │ │ └── __init__.py │ └── repo.yaml ├── module_load.sh ├── lint.sh ├── docs.sh ├── install_deps.sh ├── coverage.sh ├── install_docs.sh ├── resource_graph.yaml └── build_hermes.sh ├── test ├── unit │ ├── external │ │ ├── CPPLINT.cfg │ │ ├── CMakeLists.txt │ │ └── external.cc │ ├── pipelines │ │ ├── mpiio │ │ │ ├── test_mpiio_basic.yaml │ │ │ ├── test_hermes_mpiio_basic_async.yaml │ │ │ └── test_hermes_mpiio_basic_sync.yaml │ │ ├── posix │ │ │ ├── test_posix_basic.yaml │ │ │ ├── test_posix_basic_mpi.yaml │ │ │ ├── test_posix_simple_io_omp.yaml │ │ │ ├── test_hermes_posix_simple_io_omp.yaml │ │ │ ├── test_hermes_posix_basic_small.yaml │ │ │ ├── test_hermes_posix_basic_large.yaml │ │ │ ├── test_hermes_posix_basic_mpi_large.yaml │ │ │ └── test_hermes_posix_basic_mpi_small.yaml │ │ ├── stdio │ │ │ ├── test_stdio_basic.yaml │ │ │ ├── test_stdio_basic_mpi.yaml │ │ │ ├── test_hermes_stdio_low_buf.yaml │ │ │ ├── test_hermes_stdio_mapper.yaml │ │ │ ├── test_hermes_stdio_basic_small.yaml │ │ │ ├── test_hermes_stdio_basic_large.yaml │ │ │ ├── test_hermes_stdio_basic_mpi_small.yaml │ │ │ ├── test_hermes_stdio_basic_mpi_large.yaml │ │ │ ├── test_hermes_stdio_adapter_bypass.yaml │ │ │ ├── test_hermes_stdio_adapter_scratch.yaml │ │ │ └── test_hermes_stdio_adapter_default.yaml │ │ ├── hermes │ │ │ └── test_hermes.yaml │ │ ├── vfd │ │ │ ├── test_hermes_vfd_scratch.yaml │ │ │ └── test_hermes_vfd_basic.yaml │ │ ├── test_borg.yaml │ │ └── test_ior.yaml │ ├── asan.supp │ ├── CMakeLists.txt │ ├── hermes_adapters │ │ ├── CMakeLists.txt │ │ ├── posix │ │ │ ├── hdf5_write_read.py │ │ │ └── posix_adapter_test.cc │ │ ├── vfd │ │ │ ├── hermes_vfd_test.cc │ │ │ └── CMakeLists.txt │ │ ├── mpiio │ │ │ ├── mpiio_adapter_test.cc │ │ │ ├── parallel.cc │ │ │ └── CMakeLists.txt │ │ ├── stdio │ │ │ ├── stdio_adapter_test.cc │ │ │ └── stdio_adapter_shared_test.cc │ │ └── adapter_test_utils.h │ ├── boost │ │ ├── test_init.cc │ │ ├── test_init.h │ │ └── CMakeLists.txt │ ├── config │ │ ├── test_init.cc │ │ ├── test_init.h │ │ ├── CMakeLists.txt │ │ └── test_config.cc │ ├── ipc │ │ ├── test_init.h │ │ ├── test_finalize.cc │ │ ├── test_init.cc │ │ └── CMakeLists.txt │ ├── hermes │ │ ├── test_init.h │ │ ├── test_init.cc │ │ └── CMakeLists.txt │ ├── main.cc │ ├── main_mpi.cc │ └── basic_test.h └── CMakeLists.txt ├── wrapper ├── python │ ├── requirements.txt │ ├── CMakeLists.txt │ ├── test │ │ └── unit │ │ │ └── test_hermes.py │ ├── setup.py │ └── cpp │ │ └── CMakeLists.txt └── CMakeLists.txt ├── hrun ├── config │ ├── hrun_client_default.yaml │ └── hrun_server_default.yaml ├── tasks_required │ ├── TASK_NAME │ │ ├── include │ │ │ └── TASK_NAME │ │ │ │ ├── TASK_NAME_methods.yaml │ │ │ │ └── TASK_NAME_methods.h │ │ ├── CMakeLists.txt │ │ └── src │ │ │ ├── TASK_NAME.cc │ │ │ └── CMakeLists.txt │ ├── proc_queue │ │ ├── include │ │ │ └── proc_queue │ │ │ │ ├── proc_queue_methods.yaml │ │ │ │ └── proc_queue_methods.h │ │ ├── CMakeLists.txt │ │ └── src │ │ │ └── CMakeLists.txt │ ├── remote_queue │ │ ├── include │ │ │ └── remote_queue │ │ │ │ ├── remote_queue_methods.yaml │ │ │ │ └── remote_queue_methods.h │ │ ├── CMakeLists.txt │ │ └── src │ │ │ └── CMakeLists.txt │ ├── small_message │ │ ├── include │ │ │ └── small_message │ │ │ │ ├── small_message_methods.yaml │ │ │ │ └── small_message_methods.h │ │ ├── CMakeLists.txt │ │ └── src │ │ │ └── CMakeLists.txt │ ├── worch_proc_round_robin │ │ ├── include │ │ │ └── worch_proc_round_robin │ │ │ │ ├── worch_proc_round_robin_methods.yaml │ │ │ │ ├── worch_proc_round_robin_methods.h │ │ │ │ ├── worch_proc_round_robin.h │ │ │ │ └── worch_proc_round_robin_tasks.h │ │ ├── CMakeLists.txt │ │ └── src │ │ │ ├── worch_proc_round_robin.cc │ │ │ └── CMakeLists.txt │ ├── worch_queue_round_robin │ │ ├── include │ │ │ └── worch_queue_round_robin │ │ │ │ ├── worch_queue_round_robin_methods.yaml │ │ │ │ ├── worch_queue_round_robin_methods.h │ │ │ │ ├── worch_queue_round_robin.h │ │ │ │ └── worch_queue_round_robin_tasks.h │ │ ├── CMakeLists.txt │ │ └── src │ │ │ └── CMakeLists.txt │ ├── CMakeLists.txt │ └── hrun_admin │ │ ├── include │ │ └── hrun_admin │ │ │ ├── hrun_admin_methods.yaml │ │ │ └── hrun_admin_methods.h │ │ ├── CMakeLists.txt │ │ └── src │ │ └── CMakeLists.txt ├── include │ └── hrun │ │ ├── config │ │ ├── config_client_default.h │ │ └── config_client.h │ │ ├── api │ │ ├── hrun_runtime.h │ │ └── template │ │ │ ├── hrun_task_cc.template │ │ │ ├── hrun_task_node_admin_root.template │ │ │ └── hrun_task_node_push_root.template │ │ ├── queue_manager │ │ ├── queue_factory.h │ │ └── queue_manager_client.h │ │ ├── hrun_constants.h │ │ ├── hrun_namespace.h │ │ └── work_orchestrator │ │ └── scheduler.h └── src │ ├── hrun_stop_runtime.cc │ ├── hrun_client.cc │ ├── hrun_start_runtime.cc │ └── config_client.cc ├── tasks ├── hermes_mdm │ ├── include │ │ └── hermes_mdm │ │ │ ├── hermes_mdm_methods.yaml │ │ │ ├── hermes_mdm_methods.h │ │ │ └── hermes_mdm.h │ ├── CMakeLists.txt │ └── src │ │ ├── hermes_mdm.cc │ │ └── CMakeLists.txt ├── hermes_data_op │ ├── include │ │ └── hermes_data_op │ │ │ ├── hermes_data_op_methods.yaml │ │ │ └── hermes_data_op_methods.h │ ├── CMakeLists.txt │ └── src │ │ └── CMakeLists.txt ├── bdev │ ├── include │ │ └── bdev │ │ │ ├── bdev_methods.yaml │ │ │ ├── bdev_methods.h │ │ │ └── bdev_namespace.h │ └── CMakeLists.txt ├── data_stager │ ├── include │ │ └── data_stager │ │ │ ├── data_stager_methods.yaml │ │ │ ├── data_stager_methods.h │ │ │ └── factory │ │ │ ├── abstract_stager.h │ │ │ └── stager_factory.h │ ├── CMakeLists.txt │ └── src │ │ └── CMakeLists.txt ├── CMakeLists.txt ├── hermes_bucket_mdm │ ├── include │ │ └── hermes_bucket_mdm │ │ │ ├── hermes_bucket_mdm_methods.yaml │ │ │ └── hermes_bucket_mdm_methods.h │ ├── CMakeLists.txt │ └── src │ │ └── CMakeLists.txt ├── hermes_blob_mdm │ ├── include │ │ └── hermes_blob_mdm │ │ │ ├── hermes_blob_mdm_methods.yaml │ │ │ └── hermes_blob_mdm_methods.h │ ├── CMakeLists.txt │ └── src │ │ └── CMakeLists.txt ├── ram_bdev │ ├── CMakeLists.txt │ ├── include │ │ └── ram_bdev │ │ │ └── ram_bdev.h │ └── src │ │ └── CMakeLists.txt └── posix_bdev │ ├── CMakeLists.txt │ ├── include │ └── posix_bdev │ │ └── posix_bdev.h │ └── src │ └── CMakeLists.txt ├── include └── hermes │ ├── CPPLINT.cfg │ ├── config_client_default.h │ ├── statuses.h │ ├── status.h │ └── dpe │ ├── dpe.h │ └── dpe_factory.h ├── .gitmodules ├── CPPLINT.cfg ├── codegen ├── codegen │ ├── util │ │ ├── conv.py │ │ ├── paths.py │ │ └── naming.py │ └── hrun_config │ │ └── generator.py ├── make_macro ├── hrun_config ├── hermes_config ├── update_task └── make_task ├── CTestConfig.cmake ├── config └── hermes_client_default.yaml ├── src ├── hermes_config_manager.cc └── CMakeLists.txt ├── hermes_adapters ├── CMakeLists.txt ├── filesystem │ ├── filesystem_mdm_singleton.cc │ └── CMakeLists.txt ├── adapter_constants.h ├── mapper │ ├── mapper_factory.h │ └── balanced_mapper.h ├── mpiio │ └── CMakeLists.txt ├── posix │ └── CMakeLists.txt └── stdio │ └── CMakeLists.txt ├── .gitignore ├── benchmark ├── test_init.cc ├── test_init.h ├── test_zmq.cc └── CMakeLists.txt ├── README.md ├── .github └── workflows │ ├── docs.yml │ └── build_and_push_docker_images.yml └── docker ├── user.Dockerfile └── deps.Dockerfile /ci/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ci/hermes/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ci/hermes/packages/hermes/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ci/hermes/packages/hermes_shm/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/unit/external/CPPLINT.cfg: -------------------------------------------------------------------------------- 1 | exclude_files=.*build -------------------------------------------------------------------------------- /wrapper/python/requirements.txt: -------------------------------------------------------------------------------- 1 | pybind11 2 | pytest -------------------------------------------------------------------------------- /ci/hermes/repo.yaml: -------------------------------------------------------------------------------- 1 | repo: 2 | namespace: 'hermes' 3 | -------------------------------------------------------------------------------- /hrun/config/hrun_client_default.yaml: -------------------------------------------------------------------------------- 1 | thread_model: kStd -------------------------------------------------------------------------------- /wrapper/python/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | add_subdirectory(cpp) -------------------------------------------------------------------------------- /tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_methods.yaml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /hrun/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_methods.yaml: -------------------------------------------------------------------------------- 1 | kCustom: 0 -------------------------------------------------------------------------------- /hrun/tasks_required/proc_queue/include/proc_queue/proc_queue_methods.yaml: -------------------------------------------------------------------------------- 1 | kPush: 0 -------------------------------------------------------------------------------- /hrun/tasks_required/remote_queue/include/remote_queue/remote_queue_methods.yaml: -------------------------------------------------------------------------------- 1 | kPush: 0 2 | kDup: 1 -------------------------------------------------------------------------------- /include/hermes/CPPLINT.cfg: -------------------------------------------------------------------------------- 1 | exclude_files=.*config_client_default 2 | exclude_files=.*config_server_default -------------------------------------------------------------------------------- /test/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.10) 2 | project(hermes) 3 | 4 | add_subdirectory(unit) -------------------------------------------------------------------------------- /hrun/tasks_required/small_message/include/small_message/small_message_methods.yaml: -------------------------------------------------------------------------------- 1 | kMd: 0 2 | kIo: 1 3 | kMdPush: 2 -------------------------------------------------------------------------------- /hrun/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_methods.yaml: -------------------------------------------------------------------------------- 1 | kSchedule: 0 -------------------------------------------------------------------------------- /tasks/hermes_data_op/include/hermes_data_op/hermes_data_op_methods.yaml: -------------------------------------------------------------------------------- 1 | kRegisterOp: 0 2 | kRegisterData: 1 3 | kRunOp: 2 -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "external/pybind11"] 2 | path = external/pybind11 3 | url = https://github.com/pybind/pybind11.git 4 | -------------------------------------------------------------------------------- /hrun/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_methods.yaml: -------------------------------------------------------------------------------- 1 | kSchedule: 0 -------------------------------------------------------------------------------- /tasks/bdev/include/bdev/bdev_methods.yaml: -------------------------------------------------------------------------------- 1 | kWrite: 0 2 | kRead: 1 3 | kAllocate: 2 4 | kFree: 3 5 | kStatBdev: 4 6 | kUpdateScore: 5 7 | kLast: 6 -------------------------------------------------------------------------------- /tasks/data_stager/include/data_stager/data_stager_methods.yaml: -------------------------------------------------------------------------------- 1 | kRegisterStager: 0 2 | kUnregisterStager: 1 3 | kStageIn: 2 4 | kStageOut: 3 5 | kUpdateSize: 4 -------------------------------------------------------------------------------- /CPPLINT.cfg: -------------------------------------------------------------------------------- 1 | set noparent 2 | filter=-build/header_guard,-build/include_subdir,-runtime/references,-readability/casting,-runtime/int,-build/include,-build/c++11,-build/c++14 -------------------------------------------------------------------------------- /ci/module_load.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ -d /etc/profile.d ]; then 3 | for i in /etc/profile.d/*.sh; do 4 | if [ -r $i ]; then 5 | . $i 6 | fi 7 | done 8 | fi -------------------------------------------------------------------------------- /wrapper/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.10) 2 | project(hermes) 3 | 4 | if (HERMES_ENABLE_PYTHON) 5 | message("Python bindings enabled") 6 | add_subdirectory(python) 7 | endif() 8 | -------------------------------------------------------------------------------- /ci/lint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | HRUN_ROOT=$1 4 | 5 | echo "RUNNING CPPLINT" 6 | cpplint --recursive \ 7 | "${HRUN_ROOT}/src" "${HRUN_ROOT}/include" "${HRUN_ROOT}/test" \ 8 | --exclude="${HRUN_ROOT}/test/unit/external" -------------------------------------------------------------------------------- /test/unit/pipelines/mpiio/test_mpiio_basic.yaml: -------------------------------------------------------------------------------- 1 | name: hermes_unit_mpiio_basic 2 | env: hermes 3 | pkgs: 4 | - pkg_type: hermes_mpiio_tests 5 | pkg_name: hermes_mpiio_tests 6 | test_file: mpiio_basic 7 | hermes: false -------------------------------------------------------------------------------- /test/unit/pipelines/posix/test_posix_basic.yaml: -------------------------------------------------------------------------------- 1 | name: hermes_unit_posix_basic 2 | env: hermes 3 | pkgs: 4 | - pkg_type: hermes_posix_tests 5 | pkg_name: hermes_posix_tests 6 | test_file: posix_basic 7 | hermes: false -------------------------------------------------------------------------------- /test/unit/pipelines/stdio/test_stdio_basic.yaml: -------------------------------------------------------------------------------- 1 | name: hermes_unit_stdio_basic 2 | env: hermes 3 | pkgs: 4 | - pkg_type: hermes_stdio_tests 5 | pkg_name: hermes_stdio_tests 6 | test_file: stdio_basic 7 | hermes: false -------------------------------------------------------------------------------- /test/unit/pipelines/posix/test_posix_basic_mpi.yaml: -------------------------------------------------------------------------------- 1 | name: hermes_unit_posix_basic_mpi 2 | env: hermes 3 | pkgs: 4 | - pkg_type: hermes_posix_tests 5 | pkg_name: hermes_posix_tests 6 | test_file: posix_basic_mpi 7 | hermes: false -------------------------------------------------------------------------------- /test/unit/pipelines/stdio/test_stdio_basic_mpi.yaml: -------------------------------------------------------------------------------- 1 | name: hermes_unit_stdio_basic_mpi 2 | env: hermes 3 | pkgs: 4 | - pkg_type: hermes_stdio_tests 5 | pkg_name: hermes_stdio_tests 6 | test_file: stdio_basic_mpi 7 | hermes: false -------------------------------------------------------------------------------- /ci/docs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | make dox >& out.txt 4 | # cat out.txt | grep warning | grep -v "ignoring unsupported tag" 5 | rec="$( grep warning build/out.txt | grep -v "ignoring unsupported tag" | wc -l )" 6 | echo "$rec" 7 | exit "$rec" 8 | 9 | -------------------------------------------------------------------------------- /test/unit/asan.supp: -------------------------------------------------------------------------------- 1 | # Ignore leaks from external libraries 2 | leak:libfabric.so 3 | leak:libabt.so 4 | leak:libmargo.so 5 | leak:libmpi.so 6 | leak:librdmacm.so 7 | leak:libhwloc.so 8 | leak:libmpich.so 9 | leak:aiori-HDF5.c 10 | leak:ior.c -------------------------------------------------------------------------------- /test/unit/pipelines/posix/test_posix_simple_io_omp.yaml: -------------------------------------------------------------------------------- 1 | name: hermes_unit_posix_simple_io_omp 2 | env: hermes 3 | pkgs: 4 | - pkg_type: hermes_posix_tests 5 | pkg_name: hermes_posix_tests 6 | test_file: posix_simple_io_omp 7 | hermes: false -------------------------------------------------------------------------------- /codegen/codegen/util/conv.py: -------------------------------------------------------------------------------- 1 | 2 | def str_to_bool(text): 3 | text = str(text).lower().strip() 4 | if text == 'false': 5 | return False 6 | elif text == 'true': 7 | return True 8 | raise Exception("Neither true or false (str_too_bool)") -------------------------------------------------------------------------------- /CTestConfig.cmake: -------------------------------------------------------------------------------- 1 | set(CTEST_PROJECT_NAME "HERMES") 2 | set(CTEST_NIGHTLY_START_TIME "00:00:00 EST") 3 | 4 | set(CTEST_DROP_METHOD "https") 5 | set(CTEST_DROP_SITE "my.cdash.org") 6 | set(CTEST_DROP_LOCATION "/submit.php?project=HERMES") 7 | set(CTEST_DROP_SITE_CDASH TRUE) 8 | -------------------------------------------------------------------------------- /tasks/hermes_mdm/include/hermes_mdm/hermes_mdm_methods.h: -------------------------------------------------------------------------------- 1 | #ifndef HRUN_HERMES_MDM_METHODS_H_ 2 | #define HRUN_HERMES_MDM_METHODS_H_ 3 | 4 | /** The set of methods in the admin task */ 5 | struct Method : public TaskMethod { 6 | }; 7 | 8 | #endif // HRUN_HERMES_MDM_METHODS_H_ -------------------------------------------------------------------------------- /hrun/include/hrun/config/config_client_default.h: -------------------------------------------------------------------------------- 1 | #ifndef HRUN_SRC_CONFIG_HRUN_CLIENT_DEFAULT_H_ 2 | #define HRUN_SRC_CONFIG_HRUN_CLIENT_DEFAULT_H_ 3 | const inline char* kHrunClientDefaultConfigStr = 4 | "thread_model: kStd\n"; 5 | #endif // HRUN_SRC_CONFIG_HRUN_CLIENT_DEFAULT_H_ -------------------------------------------------------------------------------- /tasks/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_subdirectory(bdev) 2 | add_subdirectory(ram_bdev) 3 | add_subdirectory(posix_bdev) 4 | add_subdirectory(hermes_mdm) 5 | add_subdirectory(hermes_blob_mdm) 6 | add_subdirectory(hermes_bucket_mdm) 7 | add_subdirectory(hermes_data_op) 8 | add_subdirectory(data_stager) -------------------------------------------------------------------------------- /hrun/tasks_required/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_subdirectory(TASK_NAME) 2 | add_subdirectory(small_message) 3 | add_subdirectory(hrun_admin) 4 | add_subdirectory(remote_queue) 5 | add_subdirectory(worch_proc_round_robin) 6 | add_subdirectory(worch_queue_round_robin) 7 | add_subdirectory(proc_queue) -------------------------------------------------------------------------------- /config/hermes_client_default.yaml: -------------------------------------------------------------------------------- 1 | stop_daemon: false 2 | path_inclusions: ["/tmp/test_hermes/*"] 3 | path_exclusions: ["/*"] 4 | file_page_size: 1024KB 5 | base_adapter_mode: kDefault 6 | flushing_mode: kAsync 7 | file_adapter_configs: 8 | - path: "/*" 9 | page_size: 1MB 10 | mode: kDefault -------------------------------------------------------------------------------- /test/unit/pipelines/hermes/test_hermes.yaml: -------------------------------------------------------------------------------- 1 | name: hermes_unit_hermes_mpiio_basic_large 2 | env: hermes 3 | pkgs: 4 | - pkg_type: hermes_run 5 | pkg_name: hermes_run 6 | ram: 16m 7 | sleep: 5 8 | - pkg_type: hermes_unit_tests 9 | pkg_name: hermes_unit_tests 10 | TEST_CASE: hermes 11 | -------------------------------------------------------------------------------- /hrun/tasks_required/hrun_admin/include/hrun_admin/hrun_admin_methods.yaml: -------------------------------------------------------------------------------- 1 | kCreateTaskState: 0 2 | kDestroyTaskState: 1 3 | kRegisterTaskLib: 2 4 | kDestroyTaskLib: 3 5 | kGetOrCreateTaskStateId: 4 6 | kGetTaskStateId: 5 7 | kStopRuntime: 6 8 | kSetWorkOrchQueuePolicy: 7 9 | kSetWorkOrchProcPolicy: 8 10 | kFlush: 9 -------------------------------------------------------------------------------- /codegen/make_macro: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """ 4 | USAGE: ./make_macro [PATH] 5 | """ 6 | 7 | import os 8 | import sys 9 | from codegen.hrun_config.generator import print_macro 10 | 11 | PATH = sys.argv[1] 12 | MACRO_NAME = os.path.basename(PATH).upper().split('.')[0] 13 | print_macro(PATH, MACRO_NAME) 14 | -------------------------------------------------------------------------------- /hrun/tasks_required/TASK_NAME/include/TASK_NAME/TASK_NAME_methods.h: -------------------------------------------------------------------------------- 1 | #ifndef HRUN_TASK_NAME_METHODS_H_ 2 | #define HRUN_TASK_NAME_METHODS_H_ 3 | 4 | /** The set of methods in the admin task */ 5 | struct Method : public TaskMethod { 6 | TASK_METHOD_T kCustom = kLast + 0; 7 | }; 8 | 9 | #endif // HRUN_TASK_NAME_METHODS_H_ -------------------------------------------------------------------------------- /hrun/tasks_required/proc_queue/include/proc_queue/proc_queue_methods.h: -------------------------------------------------------------------------------- 1 | #ifndef HRUN_PROC_QUEUE_METHODS_H_ 2 | #define HRUN_PROC_QUEUE_METHODS_H_ 3 | 4 | /** The set of methods in the admin task */ 5 | struct Method : public TaskMethod { 6 | TASK_METHOD_T kPush = kLast + 0; 7 | }; 8 | 9 | #endif // HRUN_PROC_QUEUE_METHODS_H_ -------------------------------------------------------------------------------- /test/unit/pipelines/stdio/test_hermes_stdio_low_buf.yaml: -------------------------------------------------------------------------------- 1 | name: hermes_unit_hermes_stdio_basic_small 2 | env: hermes 3 | pkgs: 4 | - pkg_type: hermes_run 5 | pkg_name: hermes_run 6 | sleep: 2 7 | - pkg_type: hermes_stdio_tests 8 | pkg_name: hermes_stdio_tests 9 | test_file: stdio_low_buf 10 | hermes: true -------------------------------------------------------------------------------- /test/unit/pipelines/stdio/test_hermes_stdio_mapper.yaml: -------------------------------------------------------------------------------- 1 | name: hermes_unit_hermes_stdio_basic_small 2 | env: hermes 3 | pkgs: 4 | - pkg_type: hermes_run 5 | pkg_name: hermes_run 6 | sleep: 2 7 | - pkg_type: hermes_stdio_tests 8 | pkg_name: hermes_stdio_tests 9 | test_file: stdio_mapper 10 | hermes: true -------------------------------------------------------------------------------- /test/unit/pipelines/vfd/test_hermes_vfd_scratch.yaml: -------------------------------------------------------------------------------- 1 | name: hermes_unit_hermes_vfd_basic 2 | env: hermes 3 | pkgs: 4 | - pkg_type: hermes_run 5 | pkg_name: hermes_run 6 | sleep: 5 7 | - pkg_type: hermes_vfd_tests 8 | pkg_name: hermes_vfd_tests 9 | test_file: vfd_basic 10 | hermes: true 11 | mode: scratch 12 | -------------------------------------------------------------------------------- /test/unit/pipelines/mpiio/test_hermes_mpiio_basic_async.yaml: -------------------------------------------------------------------------------- 1 | name: hermes_unit_hermes_mpiio_basic_large 2 | env: hermes 3 | pkgs: 4 | - pkg_type: hermes_run 5 | pkg_name: hermes_run 6 | sleep: 5 7 | - pkg_type: hermes_mpiio_tests 8 | pkg_name: hermes_mpiio_tests 9 | test_file: mpiio_basic 10 | hermes: true 11 | 12 | -------------------------------------------------------------------------------- /test/unit/pipelines/posix/test_hermes_posix_simple_io_omp.yaml: -------------------------------------------------------------------------------- 1 | name: hermes_unit_hermes_posix_simple_io_omp 2 | env: hermes 3 | pkgs: 4 | - pkg_type: hermes_run 5 | pkg_name: hermes_run 6 | sleep: 2 7 | - pkg_type: hermes_posix_tests 8 | pkg_name: hermes_posix_tests 9 | test_file: posix_simple_io_omp 10 | hermes: true -------------------------------------------------------------------------------- /test/unit/pipelines/mpiio/test_hermes_mpiio_basic_sync.yaml: -------------------------------------------------------------------------------- 1 | name: hermes_unit_hermes_mpiio_basic_async 2 | env: hermes 3 | pkgs: 4 | - pkg_type: hermes_run 5 | pkg_name: hermes_run 6 | sleep: 2 7 | - pkg_type: hermes_mpiio_tests 8 | pkg_name: hermes_mpiio_tests 9 | test_file: mpiio_basic 10 | hermes: true 11 | sync: async -------------------------------------------------------------------------------- /test/unit/pipelines/posix/test_hermes_posix_basic_small.yaml: -------------------------------------------------------------------------------- 1 | name: hermes_unit_hermes_posix_basic_small 2 | env: hermes 3 | pkgs: 4 | - pkg_type: hermes_run 5 | pkg_name: hermes_run 6 | sleep: 2 7 | - pkg_type: hermes_posix_tests 8 | pkg_name: hermes_posix_tests 9 | test_file: posix_basic 10 | hermes: true 11 | size: small -------------------------------------------------------------------------------- /test/unit/pipelines/stdio/test_hermes_stdio_basic_small.yaml: -------------------------------------------------------------------------------- 1 | name: hermes_unit_hermes_stdio_basic_small 2 | env: hermes 3 | pkgs: 4 | - pkg_type: hermes_run 5 | pkg_name: hermes_run 6 | sleep: 2 7 | - pkg_type: hermes_stdio_tests 8 | pkg_name: hermes_stdio_tests 9 | test_file: stdio_basic 10 | hermes: true 11 | size: small -------------------------------------------------------------------------------- /test/unit/pipelines/posix/test_hermes_posix_basic_large.yaml: -------------------------------------------------------------------------------- 1 | name: hermes_unit_hermes_posix_basic_large 2 | env: hermes 3 | pkgs: 4 | - pkg_type: hermes_run 5 | pkg_name: hermes_run 6 | sleep: 5 7 | - pkg_type: hermes_posix_tests 8 | pkg_name: hermes_posix_tests 9 | test_file: posix_basic 10 | hermes: true 11 | size: large 12 | -------------------------------------------------------------------------------- /test/unit/pipelines/stdio/test_hermes_stdio_basic_large.yaml: -------------------------------------------------------------------------------- 1 | name: hermes_unit_hermes_stdio_basic_large 2 | env: hermes 3 | pkgs: 4 | - pkg_type: hermes_run 5 | pkg_name: hermes_run 6 | sleep: 2 7 | - pkg_type: hermes_stdio_tests 8 | pkg_name: hermes_stdio_tests 9 | test_file: stdio_basic 10 | hermes: true 11 | size: large 12 | -------------------------------------------------------------------------------- /test/unit/pipelines/posix/test_hermes_posix_basic_mpi_large.yaml: -------------------------------------------------------------------------------- 1 | name: hermes_unit_hermes_posix_basic_mpi_large 2 | env: hermes 3 | pkgs: 4 | - pkg_type: hermes_run 5 | pkg_name: hermes_run 6 | sleep: 2 7 | - pkg_type: hermes_posix_tests 8 | pkg_name: hermes_posix_tests 9 | test_file: posix_basic_mpi 10 | hermes: true 11 | size: large -------------------------------------------------------------------------------- /test/unit/pipelines/posix/test_hermes_posix_basic_mpi_small.yaml: -------------------------------------------------------------------------------- 1 | name: hermes_unit_hermes_posix_basic_mpi_small 2 | env: hermes 3 | pkgs: 4 | - pkg_type: hermes_run 5 | pkg_name: hermes_run 6 | sleep: 2 7 | - pkg_type: hermes_posix_tests 8 | pkg_name: hermes_posix_tests 9 | test_file: posix_basic_mpi 10 | hermes: true 11 | size: small -------------------------------------------------------------------------------- /test/unit/pipelines/stdio/test_hermes_stdio_basic_mpi_small.yaml: -------------------------------------------------------------------------------- 1 | name: hermes_unit_hermes_stdio_basic_mpi_small 2 | env: hermes 3 | pkgs: 4 | - pkg_type: hermes_run 5 | pkg_name: hermes_run 6 | sleep: 2 7 | - pkg_type: hermes_stdio_tests 8 | pkg_name: hermes_stdio_tests 9 | test_file: stdio_basic_mpi 10 | hermes: true 11 | size: small -------------------------------------------------------------------------------- /hrun/include/hrun/api/hrun_runtime.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by lukemartinlogan on 10/24/23. 3 | // 4 | 5 | #ifndef HERMES_HRUN_INCLUDE_HRUN_API_HRUN_RUNTIME_H_ 6 | #define HERMES_HRUN_INCLUDE_HRUN_API_HRUN_RUNTIME_H_ 7 | 8 | #include "hrun_runtime_.h" 9 | #include "hrun/work_orchestrator/worker.h" 10 | 11 | #endif // HERMES_HRUN_INCLUDE_HRUN_API_HRUN_RUNTIME_H_ 12 | -------------------------------------------------------------------------------- /test/unit/pipelines/stdio/test_hermes_stdio_basic_mpi_large.yaml: -------------------------------------------------------------------------------- 1 | name: hermes_unit_hermes_stdio_basic_mpi_large 2 | env: hermes 3 | pkgs: 4 | - pkg_type: hermes_run 5 | pkg_name: hermes_run 6 | sleep: 2 7 | - pkg_type: hermes_stdio_tests 8 | pkg_name: hermes_stdio_tests 9 | test_file: stdio_basic_mpi 10 | hermes: true 11 | size: large 12 | -------------------------------------------------------------------------------- /hrun/tasks_required/remote_queue/include/remote_queue/remote_queue_methods.h: -------------------------------------------------------------------------------- 1 | #ifndef HRUN_REMOTE_QUEUE_METHODS_H_ 2 | #define HRUN_REMOTE_QUEUE_METHODS_H_ 3 | 4 | /** The set of methods in the admin task */ 5 | struct Method : public TaskMethod { 6 | TASK_METHOD_T kPush = kLast + 0; 7 | TASK_METHOD_T kDup = kLast + 1; 8 | }; 9 | 10 | #endif // HRUN_REMOTE_QUEUE_METHODS_H_ -------------------------------------------------------------------------------- /hrun/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_methods.h: -------------------------------------------------------------------------------- 1 | #ifndef HRUN_WORCH_PROC_ROUND_ROBIN_METHODS_H_ 2 | #define HRUN_WORCH_PROC_ROUND_ROBIN_METHODS_H_ 3 | 4 | /** The set of methods in the admin task */ 5 | struct Method : public TaskMethod { 6 | TASK_METHOD_T kSchedule = kLast + 0; 7 | }; 8 | 9 | #endif // HRUN_WORCH_PROC_ROUND_ROBIN_METHODS_H_ -------------------------------------------------------------------------------- /hrun/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_methods.h: -------------------------------------------------------------------------------- 1 | #ifndef HRUN_WORCH_QUEUE_ROUND_ROBIN_METHODS_H_ 2 | #define HRUN_WORCH_QUEUE_ROUND_ROBIN_METHODS_H_ 3 | 4 | /** The set of methods in the admin task */ 5 | struct Method : public TaskMethod { 6 | TASK_METHOD_T kSchedule = kLast + 0; 7 | }; 8 | 9 | #endif // HRUN_WORCH_QUEUE_ROUND_ROBIN_METHODS_H_ -------------------------------------------------------------------------------- /test/unit/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.10) 2 | project(hermes) 3 | 4 | set(CMAKE_CXX_STANDARD 17) 5 | 6 | include_directories(${CMAKE_CURRENT_SOURCE_DIR}) 7 | include_directories(${CMAKE_SOURCE_DIR}/tasks/hrun_admin/include) 8 | add_subdirectory(ipc) 9 | add_subdirectory(config) 10 | add_subdirectory(hermes) 11 | add_subdirectory(hermes_adapters) 12 | add_subdirectory(boost) 13 | -------------------------------------------------------------------------------- /test/unit/pipelines/test_borg.yaml: -------------------------------------------------------------------------------- 1 | name: hermes_unit_ior 2 | env: hermes 3 | pkgs: 4 | - pkg_type: hermes_run 5 | pkg_name: hermes_run 6 | recency_max: 1 7 | ram: 10m 8 | sleep: 5 9 | do_dbg: true 10 | dbg_port: 4000 11 | - pkg_type: hermes_unit_tests 12 | pkg_name: hermes_unit_tests 13 | TEST_CASE: TestHermesPutGet 14 | do_dbg: true 15 | dbg_port: 4001 16 | -------------------------------------------------------------------------------- /test/unit/pipelines/stdio/test_hermes_stdio_adapter_bypass.yaml: -------------------------------------------------------------------------------- 1 | name: hermes_unit_hermes_stdio_basic_small 2 | env: hermes 3 | pkgs: 4 | - pkg_type: hermes_run 5 | pkg_name: hermes_run 6 | sleep: 2 7 | - pkg_type: hermes_stdio_tests 8 | pkg_name: hermes_stdio_tests 9 | test_file: stdio_adapter_mode 10 | test_case: BatchedWriteSequentialBypass 11 | hermes: true 12 | size: small -------------------------------------------------------------------------------- /test/unit/pipelines/stdio/test_hermes_stdio_adapter_scratch.yaml: -------------------------------------------------------------------------------- 1 | name: hermes_unit_hermes_stdio_basic_small 2 | env: hermes 3 | pkgs: 4 | - pkg_type: hermes_run 5 | pkg_name: hermes_run 6 | sleep: 2 7 | - pkg_type: hermes_stdio_tests 8 | pkg_name: hermes_stdio_tests 9 | test_file: stdio_adapter_mode 10 | test_case: BatchedWriteSequentialScratch 11 | hermes: true 12 | size: small -------------------------------------------------------------------------------- /test/unit/pipelines/vfd/test_hermes_vfd_basic.yaml: -------------------------------------------------------------------------------- 1 | name: hermes_unit_hermes_vfd_basic 2 | env: hermes 3 | pkgs: 4 | - pkg_type: hermes_run 5 | pkg_name: hermes_run 6 | sleep: 5 7 | - pkg_type: hermes_api 8 | pkg_name: hermes_api 9 | vfd: true 10 | - pkg_type: hermes_vfd_tests 11 | pkg_name: hermes_vfd_tests 12 | test_file: vfd_basic 13 | hermes: true 14 | mode: default 15 | -------------------------------------------------------------------------------- /wrapper/python/test/unit/test_hermes.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from py_hermes import Hermes, TRANSPARENT_HERMES 3 | import pathlib 4 | import os 5 | 6 | class TestHermes(TestCase): 7 | def test_metadata_query(self): 8 | TRANSPARENT_HERMES() 9 | hermes = Hermes() 10 | mdm = hermes.CollectMetadataSnapshot() 11 | print(mdm.blob_info) 12 | print("Done") 13 | -------------------------------------------------------------------------------- /test/unit/pipelines/stdio/test_hermes_stdio_adapter_default.yaml: -------------------------------------------------------------------------------- 1 | name: hermes_unit_hermes_stdio_basic_small 2 | env: hermes 3 | pkgs: 4 | - pkg_type: hermes_run 5 | pkg_name: hermes_run 6 | sleep: 2 7 | - pkg_type: hermes_stdio_tests 8 | pkg_name: hermes_stdio_tests 9 | test_file: stdio_adapter_mode 10 | test_case: BatchedWriteSequentialPersistent 11 | hermes: true 12 | size: small -------------------------------------------------------------------------------- /ci/install_deps.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | set -e 5 | set -o pipefail 6 | 7 | # Pull the Hermes dependencies image 8 | docker pull hdfgroup/hermes-deps:latest 9 | docker run -d \ 10 | --mount src=${PWD},target=/hermes,type=bind \ 11 | --name hermes_deps_c \ 12 | --network host \ 13 | --memory=8G \ 14 | --shm-size=8G \ 15 | -p 4000:4000 \ 16 | -p 4001:4001 \ 17 | hdfgroup/hermes-deps \ 18 | tail -f /dev/null 19 | -------------------------------------------------------------------------------- /hrun/tasks_required/small_message/include/small_message/small_message_methods.h: -------------------------------------------------------------------------------- 1 | #ifndef HRUN_SMALL_MESSAGE_METHODS_H_ 2 | #define HRUN_SMALL_MESSAGE_METHODS_H_ 3 | 4 | /** The set of methods in the admin task */ 5 | struct Method : public TaskMethod { 6 | TASK_METHOD_T kMd = kLast + 0; 7 | TASK_METHOD_T kIo = kLast + 1; 8 | TASK_METHOD_T kMdPush = kLast + 2; 9 | }; 10 | 11 | #endif // HRUN_SMALL_MESSAGE_METHODS_H_ -------------------------------------------------------------------------------- /tasks/hermes_data_op/include/hermes_data_op/hermes_data_op_methods.h: -------------------------------------------------------------------------------- 1 | #ifndef HRUN_HERMES_DATA_OP_METHODS_H_ 2 | #define HRUN_HERMES_DATA_OP_METHODS_H_ 3 | 4 | /** The set of methods in the admin task */ 5 | struct Method : public TaskMethod { 6 | TASK_METHOD_T kRegisterOp = kLast + 0; 7 | TASK_METHOD_T kRegisterData = kLast + 1; 8 | TASK_METHOD_T kRunOp = kLast + 2; 9 | }; 10 | 11 | #endif // HRUN_HERMES_DATA_OP_METHODS_H_ -------------------------------------------------------------------------------- /test/unit/pipelines/test_ior.yaml: -------------------------------------------------------------------------------- 1 | name: hermes_unit_ior 2 | env: hermes 3 | pkgs: 4 | - pkg_type: hermes_run 5 | pkg_name: hermes_run 6 | include: /tmp/test_hermes 7 | sleep: 5 8 | ram: 1g 9 | - pkg_type: hermes_api 10 | pkg_name: hermes_api 11 | mpi: true 12 | - pkg_type: ior 13 | pkg_name: ior 14 | api: mpiio 15 | out: /tmp/test_hermes/ior.bin 16 | xfer: 1m 17 | block: 32g 18 | nprocs: 4 -------------------------------------------------------------------------------- /codegen/codegen/util/paths.py: -------------------------------------------------------------------------------- 1 | import os,sys 2 | import pathlib 3 | 4 | def GetHermesRoot(): 5 | util_path = pathlib.Path(__file__).parent.resolve() 6 | code_generators_path = os.path.dirname(util_path) 7 | code_generators_path = os.path.dirname(code_generators_path) 8 | hermes_path = os.path.dirname(code_generators_path) 9 | return hermes_path 10 | 11 | HERMES_ROOT = GetHermesRoot() 12 | HRUN_ROOT = f'{HERMES_ROOT}/hrun' 13 | -------------------------------------------------------------------------------- /tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_methods.yaml: -------------------------------------------------------------------------------- 1 | kGetOrCreateTag: 0 2 | kGetTagId: 1 3 | kGetTagName: 2 4 | kRenameTag: 3 5 | kDestroyTag: 4 6 | kTagAddBlob: 5 7 | kTagRemoveBlob: 6 8 | # kTagGroupBy: 7 9 | # kTagAddTrait: 8 10 | # kTagRemoveTrait: 9 11 | kTagClearBlobs: 10 12 | kUpdateSize: 11 13 | kAppendBlobSchema: 12 14 | kAppendBlob: 13 15 | kGetSize: 14 16 | kSetBlobMdm: 15 17 | kGetContainedBlobIds: 16 18 | kPollTagMetadata: 17 -------------------------------------------------------------------------------- /tasks/bdev/include/bdev/bdev_methods.h: -------------------------------------------------------------------------------- 1 | #ifndef HRUN_BDEV_METHODS_H_ 2 | #define HRUN_BDEV_METHODS_H_ 3 | 4 | /** The set of methods in the admin task */ 5 | struct Method : public TaskMethod { 6 | TASK_METHOD_T kWrite = kLast + 0; 7 | TASK_METHOD_T kRead = kLast + 1; 8 | TASK_METHOD_T kAllocate = kLast + 2; 9 | TASK_METHOD_T kFree = kLast + 3; 10 | TASK_METHOD_T kStatBdev = kLast + 4; 11 | TASK_METHOD_T kUpdateScore = kLast + 5; 12 | }; 13 | 14 | #endif // HRUN_BDEV_METHODS_H_ -------------------------------------------------------------------------------- /tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_methods.yaml: -------------------------------------------------------------------------------- 1 | kPutBlob: 0 2 | kGetBlob: 1 3 | kTruncateBlob: 2 4 | kDestroyBlob: 3 5 | kTagBlob: 4 6 | #kUntagBlob: 5 7 | kBlobHasTag: 6 8 | #kGetBlobTags: 7 9 | kGetBlobId: 8 10 | kGetOrCreateBlobId: 9 11 | kGetBlobName: 10 12 | kGetBlobSize: 11 13 | kGetBlobScore: 12 14 | kGetBlobBuffers: 13 15 | kRenameBlob: 14 16 | kReorganizeBlob: 15 17 | kSetBucketMdm: 16 18 | kFlushData: 17 19 | kPollBlobMetadata: 18 20 | kPollTargetMetadata: 19 -------------------------------------------------------------------------------- /test/unit/external/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.10) 2 | project(hermes) 3 | 4 | set(CMAKE_CXX_STANDARD 17) 5 | find_package(Hermes REQUIRED) 6 | find_package(MPI REQUIRED COMPONENTS C CXX) 7 | include_directories(${Hermes_INCLUDE_DIRS}) 8 | add_executable(test_hermes_external_compile 9 | external.cc 10 | ) 11 | target_link_libraries(test_hermes_external_compile 12 | ${Hermes_LIBRARIES} Catch2::Catch2 MPI::MPI_CXX) 13 | message("${Hermes_LIBRARIES}") 14 | 15 | -------------------------------------------------------------------------------- /tasks/data_stager/include/data_stager/data_stager_methods.h: -------------------------------------------------------------------------------- 1 | #ifndef HRUN_DATA_STAGER_METHODS_H_ 2 | #define HRUN_DATA_STAGER_METHODS_H_ 3 | 4 | /** The set of methods in the admin task */ 5 | struct Method : public TaskMethod { 6 | TASK_METHOD_T kRegisterStager = kLast + 0; 7 | TASK_METHOD_T kUnregisterStager = kLast + 1; 8 | TASK_METHOD_T kStageIn = kLast + 2; 9 | TASK_METHOD_T kStageOut = kLast + 3; 10 | TASK_METHOD_T kUpdateSize = kLast + 4; 11 | }; 12 | 13 | #endif // HRUN_DATA_STAGER_METHODS_H_ -------------------------------------------------------------------------------- /tasks/bdev/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | #------------------------------------------------------------------------------ 2 | # Build Hrun Admin Task Library 3 | #------------------------------------------------------------------------------ 4 | include_directories(include) 5 | 6 | #----------------------------------------------------------------------------- 7 | # Install HRUN Admin Task Library Headers 8 | #----------------------------------------------------------------------------- 9 | install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) 10 | -------------------------------------------------------------------------------- /codegen/codegen/util/naming.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | def to_camel_case(string): 4 | if string is None: 5 | return 6 | words = re.sub(r"(_|-)+", " ", string).split() 7 | words = [word.capitalize() for word in words] 8 | return "".join(words) 9 | 10 | def to_snake_case(string): 11 | if string is None: 12 | return 13 | string = re.sub('(\.|-)+', '_', string) 14 | words = re.split('([A-Z][^A-Z]*)', string) 15 | words = [word for word in words if len(word)] 16 | string = "_".join(words) 17 | return string.lower() -------------------------------------------------------------------------------- /include/hermes/config_client_default.h: -------------------------------------------------------------------------------- 1 | #ifndef HRUN_SRC_CONFIG_HERMES_CLIENT_DEFAULT_H_ 2 | #define HRUN_SRC_CONFIG_HERMES_CLIENT_DEFAULT_H_ 3 | const inline char* kHermesClientDefaultConfigStr = 4 | "stop_daemon: false\n" 5 | "path_inclusions: [\"/tmp/test_hermes/*\"]\n" 6 | "path_exclusions: [\"/*\"]\n" 7 | "file_page_size: 1024KB\n" 8 | "base_adapter_mode: kDefault\n" 9 | "flushing_mode: kAsync\n" 10 | "file_adapter_configs:\n" 11 | " - path: \"/*\"\n" 12 | " page_size: 1MB\n" 13 | " mode: kDefault\n"; 14 | #endif // HRUN_SRC_CONFIG_HERMES_CLIENT_DEFAULT_H_ -------------------------------------------------------------------------------- /tasks/ram_bdev/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | #------------------------------------------------------------------------------ 2 | # Build Hrun Admin Task Library 3 | #------------------------------------------------------------------------------ 4 | include_directories(include) 5 | add_subdirectory(src) 6 | 7 | #----------------------------------------------------------------------------- 8 | # Install HRUN Admin Task Library Headers 9 | #----------------------------------------------------------------------------- 10 | install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) 11 | -------------------------------------------------------------------------------- /tasks/data_stager/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | #------------------------------------------------------------------------------ 2 | # Build Hrun Admin Task Library 3 | #------------------------------------------------------------------------------ 4 | include_directories(include) 5 | add_subdirectory(src) 6 | 7 | #----------------------------------------------------------------------------- 8 | # Install HRUN Admin Task Library Headers 9 | #----------------------------------------------------------------------------- 10 | install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) 11 | -------------------------------------------------------------------------------- /tasks/hermes_data_op/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | #------------------------------------------------------------------------------ 2 | # Build Hrun Admin Task Library 3 | #------------------------------------------------------------------------------ 4 | include_directories(include) 5 | add_subdirectory(src) 6 | 7 | #----------------------------------------------------------------------------- 8 | # Install HRUN Admin Task Library Headers 9 | #----------------------------------------------------------------------------- 10 | install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) 11 | -------------------------------------------------------------------------------- /tasks/hermes_mdm/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | #------------------------------------------------------------------------------ 2 | # Build Hrun Admin Task Library 3 | #------------------------------------------------------------------------------ 4 | include_directories(include) 5 | add_subdirectory(src) 6 | 7 | #----------------------------------------------------------------------------- 8 | # Install HRUN Admin Task Library Headers 9 | #----------------------------------------------------------------------------- 10 | install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) 11 | -------------------------------------------------------------------------------- /tasks/posix_bdev/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | #------------------------------------------------------------------------------ 2 | # Build Hrun Admin Task Library 3 | #------------------------------------------------------------------------------ 4 | include_directories(include) 5 | add_subdirectory(src) 6 | 7 | #----------------------------------------------------------------------------- 8 | # Install HRUN Admin Task Library Headers 9 | #----------------------------------------------------------------------------- 10 | install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) 11 | -------------------------------------------------------------------------------- /hrun/tasks_required/TASK_NAME/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | #------------------------------------------------------------------------------ 2 | # Build Hrun Admin Task Library 3 | #------------------------------------------------------------------------------ 4 | include_directories(include) 5 | add_subdirectory(src) 6 | 7 | #----------------------------------------------------------------------------- 8 | # Install HRUN Admin Task Library Headers 9 | #----------------------------------------------------------------------------- 10 | install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) 11 | -------------------------------------------------------------------------------- /hrun/tasks_required/hrun_admin/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | #------------------------------------------------------------------------------ 2 | # Build Hrun Admin Task Library 3 | #------------------------------------------------------------------------------ 4 | include_directories(include) 5 | add_subdirectory(src) 6 | 7 | #----------------------------------------------------------------------------- 8 | # Install HRUN Admin Task Library Headers 9 | #----------------------------------------------------------------------------- 10 | install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) 11 | -------------------------------------------------------------------------------- /hrun/tasks_required/proc_queue/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | #------------------------------------------------------------------------------ 2 | # Build Hrun Admin Task Library 3 | #------------------------------------------------------------------------------ 4 | include_directories(include) 5 | add_subdirectory(src) 6 | 7 | #----------------------------------------------------------------------------- 8 | # Install HRUN Admin Task Library Headers 9 | #----------------------------------------------------------------------------- 10 | install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) 11 | -------------------------------------------------------------------------------- /hrun/tasks_required/small_message/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | #------------------------------------------------------------------------------ 2 | # Build Hrun Admin Task Library 3 | #------------------------------------------------------------------------------ 4 | include_directories(include) 5 | add_subdirectory(src) 6 | 7 | #----------------------------------------------------------------------------- 8 | # Install HRUN Admin Task Library Headers 9 | #----------------------------------------------------------------------------- 10 | install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) 11 | -------------------------------------------------------------------------------- /hrun/tasks_required/worch_proc_round_robin/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | #------------------------------------------------------------------------------ 2 | # Build Hrun Admin Task Library 3 | #------------------------------------------------------------------------------ 4 | include_directories(include) 5 | add_subdirectory(src) 6 | 7 | #----------------------------------------------------------------------------- 8 | # Install HRUN Admin Task Library Headers 9 | #----------------------------------------------------------------------------- 10 | install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) 11 | -------------------------------------------------------------------------------- /hrun/tasks_required/worch_queue_round_robin/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | #------------------------------------------------------------------------------ 2 | # Build Hrun Admin Task Library 3 | #------------------------------------------------------------------------------ 4 | include_directories(include) 5 | add_subdirectory(src) 6 | 7 | #----------------------------------------------------------------------------- 8 | # Install HRUN Admin Task Library Headers 9 | #----------------------------------------------------------------------------- 10 | install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) 11 | -------------------------------------------------------------------------------- /tasks/ram_bdev/include/ram_bdev/ram_bdev.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by lukemartinlogan on 6/29/23. 3 | // 4 | 5 | #ifndef HRUN_ram_bdev_H_ 6 | #define HRUN_ram_bdev_H_ 7 | 8 | #include "hrun/api/hrun_client.h" 9 | #include "hrun/task_registry/task_lib.h" 10 | #include "hrun_admin/hrun_admin.h" 11 | #include "hrun/queue_manager/queue_manager_client.h" 12 | #include "hermes/hermes_types.h" 13 | #include "bdev/bdev.h" 14 | #include "hrun/hrun_namespace.h" 15 | 16 | namespace hermes::ram_bdev { 17 | #include "bdev/bdev_namespace.h" 18 | } // namespace hrun 19 | 20 | 21 | #endif // HRUN_ram_bdev_H_ 22 | -------------------------------------------------------------------------------- /tasks/posix_bdev/include/posix_bdev/posix_bdev.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by lukemartinlogan on 6/29/23. 3 | // 4 | 5 | #ifndef HRUN_posix_bdev_H_ 6 | #define HRUN_posix_bdev_H_ 7 | 8 | #include "hrun/api/hrun_client.h" 9 | #include "hrun/task_registry/task_lib.h" 10 | #include "hrun_admin/hrun_admin.h" 11 | #include "hrun/queue_manager/queue_manager_client.h" 12 | #include "hermes/hermes_types.h" 13 | #include "bdev/bdev.h" 14 | #include "hrun/hrun_namespace.h" 15 | 16 | namespace hermes::posix_bdev { 17 | #include "bdev/bdev_namespace.h" 18 | } // namespace hrun 19 | 20 | #endif // HRUN_posix_bdev_H_ 21 | -------------------------------------------------------------------------------- /tasks/hermes_blob_mdm/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | #------------------------------------------------------------------------------ 2 | # Build Hermes Blob MDM Task Library 3 | #------------------------------------------------------------------------------ 4 | include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include) 5 | add_subdirectory(src) 6 | 7 | #----------------------------------------------------------------------------- 8 | # Install HRUN Admin Task Library Headers 9 | #----------------------------------------------------------------------------- 10 | install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) 11 | -------------------------------------------------------------------------------- /tasks/hermes_bucket_mdm/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | #------------------------------------------------------------------------------ 2 | # Build Hermes Bucket MDM Task Library 3 | #------------------------------------------------------------------------------ 4 | include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include) 5 | add_subdirectory(src) 6 | 7 | #----------------------------------------------------------------------------- 8 | # Install HRUN Admin Task Library Headers 9 | #----------------------------------------------------------------------------- 10 | install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) 11 | -------------------------------------------------------------------------------- /ci/coverage.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | COVERAGE_DIR="$1" 3 | BUILD_DIR="$2" 4 | mkdir -p "${COVERAGE_DIR}" 5 | cd "${BUILD_DIR}" 6 | echo $BUILD_DIR 7 | lcov -c -d . -o "${COVERAGE_DIR}/tmp.info" 8 | lcov --remove "${COVERAGE_DIR}/tmp.info" \ 9 | "/usr/*" \ 10 | "*/spack/*" \ 11 | -o "${COVERAGE_DIR}/lcov.info" 12 | genhtml "${COVERAGE_DIR}/lcov.info" --output-directory coverage_report 13 | 14 | echo "Placed coverage info in ${COVERAGE_DIR}/lcov.info" 15 | cd ${COVERAGE_DIR} 16 | `pwd` 17 | ls 18 | 19 | #lcov -c -d . -o "tmp.info" 20 | #lcov --remove "tmp.info" \ 21 | # "/usr/*" \ 22 | # "*/spack/*" \ 23 | # -o "lcov.info" 24 | #genhtml "lcov.info" --output-directory coverage_report 25 | -------------------------------------------------------------------------------- /hrun/include/hrun/api/template/hrun_task_cc.template: -------------------------------------------------------------------------------- 1 | extern "C" { 2 | void* alloc_state(hrun::Admin::CreateTaskStateTask *task, const char *state_name) { 3 | hrun::TaskState *exec = reinterpret_cast( 4 | new TYPE_UNWRAP(TRAIT_CLASS)()); 5 | exec->Init(task->id_, state_name); 6 | return exec; 7 | } 8 | void* create_state(hrun::Admin::CreateTaskStateTask *task, const char *state_name) { 9 | hrun::TaskState *exec = reinterpret_cast( 10 | new TYPE_UNWRAP(TRAIT_CLASS)()); 11 | exec->Init(task->id_, state_name); 12 | RunContext rctx(0); 13 | exec->Run(hrun::TaskMethod::kConstruct, task, rctx); 14 | return exec; 15 | } 16 | const char* get_task_lib_name(void) { return TASK_NAME; } 17 | bool is_hrun_task_ = true; 18 | } -------------------------------------------------------------------------------- /hrun/tasks_required/hrun_admin/include/hrun_admin/hrun_admin_methods.h: -------------------------------------------------------------------------------- 1 | #ifndef HRUN_HRUN_ADMIN_METHODS_H_ 2 | #define HRUN_HRUN_ADMIN_METHODS_H_ 3 | 4 | /** The set of methods in the admin task */ 5 | struct Method : public TaskMethod { 6 | TASK_METHOD_T kCreateTaskState = kLast + 0; 7 | TASK_METHOD_T kDestroyTaskState = kLast + 1; 8 | TASK_METHOD_T kRegisterTaskLib = kLast + 2; 9 | TASK_METHOD_T kDestroyTaskLib = kLast + 3; 10 | TASK_METHOD_T kGetOrCreateTaskStateId = kLast + 4; 11 | TASK_METHOD_T kGetTaskStateId = kLast + 5; 12 | TASK_METHOD_T kStopRuntime = kLast + 6; 13 | TASK_METHOD_T kSetWorkOrchQueuePolicy = kLast + 7; 14 | TASK_METHOD_T kSetWorkOrchProcPolicy = kLast + 8; 15 | TASK_METHOD_T kFlush = kLast + 9; 16 | }; 17 | 18 | #endif // HRUN_HRUN_ADMIN_METHODS_H_ -------------------------------------------------------------------------------- /hrun/tasks_required/remote_queue/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # find_package(PkgConfig REQUIRED) 2 | # pkg_search_module(libfabric REQUIRED libfabric) 3 | # message("LibFabric found at: ${libfabric_INCLUDE_DIRS} with libs ${libfabric_LIBRARIES} in dir ${libfabric_LIBRARY_DIRS}") 4 | 5 | #------------------------------------------------------------------------------ 6 | # Build Hrun Admin Task Library 7 | #------------------------------------------------------------------------------ 8 | include_directories(include) 9 | add_subdirectory(src) 10 | 11 | #----------------------------------------------------------------------------- 12 | # Install HRUN Admin Task Library Headers 13 | #----------------------------------------------------------------------------- 14 | install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX}) 15 | -------------------------------------------------------------------------------- /wrapper/python/setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | setuptools.setup( 4 | name="hermes", 5 | packages=setuptools.find_packages(), 6 | version="0.0.1", 7 | author="Luke Logan", 8 | author_email="lukemartinlogan@gmail.com", 9 | description="An I/O buffering library", 10 | url="https://github.com/HDFGroup/hermes.git", 11 | classifiers = [ 12 | "Programming Language :: Python", 13 | "Programming Language :: Python :: 3", 14 | "Development Status :: 1.1", 15 | "Environment :: Other Environment", 16 | "Intended Audience :: Developers", 17 | "License :: None", 18 | "Operating System :: Linux", 19 | "Topic :: Software Development :: Libraries :: Python Modules", 20 | "Topic :: I/O", 21 | ], 22 | long_description="" 23 | ) -------------------------------------------------------------------------------- /tasks/bdev/include/bdev/bdev_namespace.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by lukemartinlogan on 8/14/23. 3 | // 4 | 5 | #ifndef HRUN_TASKS_BDEV_INCLUDE_BDEV_BDEV_NAMESPACE_H_ 6 | #define HRUN_TASKS_BDEV_INCLUDE_BDEV_BDEV_NAMESPACE_H_ 7 | 8 | #include "bdev_tasks.h" 9 | #include "bdev.h" 10 | #include "hrun/hrun_namespace.h" 11 | 12 | /** The set of methods in the admin task */ 13 | using ::hermes::bdev::Method; 14 | using ::hermes::bdev::ConstructTask; 15 | using ::hermes::bdev::DestructTask; 16 | using ::hermes::bdev::AllocateTask; 17 | using ::hermes::bdev::FreeTask; 18 | using ::hermes::bdev::ReadTask; 19 | using ::hermes::bdev::WriteTask; 20 | using ::hermes::bdev::StatBdevTask; 21 | using ::hermes::bdev::UpdateScoreTask; 22 | 23 | /** Create admin requests */ 24 | using ::hermes::bdev::Client; 25 | 26 | #endif // HRUN_TASKS_BDEV_INCLUDE_BDEV_BDEV_NAMESPACE_H_ 27 | -------------------------------------------------------------------------------- /codegen/hrun_config: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """ 4 | USAGE: 5 | cd codegen/bin 6 | python3 hrun_config.py 7 | 8 | OUTPUT: 9 | ${HRUN}/src/config_client_default.h (if client) 10 | ${HRUN}/src/config_server_default.h (if server) 11 | """ 12 | 13 | from codegen.hrun_config.generator import create_config 14 | from codegen.util.paths import HRUN_ROOT 15 | 16 | create_config( 17 | path=f"{HRUN_ROOT}/config/hrun_client_default.yaml", 18 | var_name="kHrunClientDefaultConfigStr", 19 | config_path=f"{HRUN_ROOT}/include/hrun/config/config_client_default.h", 20 | macro_name="HRUN_CLIENT" 21 | ) 22 | 23 | create_config( 24 | path=f"{HRUN_ROOT}/config/hrun_server_default.yaml", 25 | var_name="kHrunServerDefaultConfigStr", 26 | config_path=f"{HRUN_ROOT}/include/hrun/config/config_server_default.h", 27 | macro_name="HRUN_SERVER" 28 | ) -------------------------------------------------------------------------------- /codegen/hermes_config: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """ 4 | USAGE: 5 | cd code_generators/bin 6 | python3 hermes_config.py 7 | 8 | OUTPUT: 9 | ${HERMES}/src/config_client_default.h (if client) 10 | ${HERMES}/src/config_server_default.h (if server) 11 | """ 12 | 13 | from codegen.hrun_config.generator import create_config 14 | from codegen.util.paths import HERMES_ROOT 15 | 16 | create_config( 17 | path=f"{HERMES_ROOT}/config/hermes_client_default.yaml", 18 | var_name="kHermesClientDefaultConfigStr", 19 | config_path=f"{HERMES_ROOT}/include/hermes/config_client_default.h", 20 | macro_name="HERMES_CLIENT" 21 | ) 22 | 23 | create_config( 24 | path=f"{HERMES_ROOT}/config/hermes_server_default.yaml", 25 | var_name="kHermesServerDefaultConfigStr", 26 | config_path=f"{HERMES_ROOT}/include/hermes/config_server_default.h", 27 | macro_name="HERMES_SERVER" 28 | ) 29 | -------------------------------------------------------------------------------- /test/unit/hermes_adapters/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DHERMES_PRELOAD -DHERMES_RPC_THALLIUM") 2 | set(HERMES_ADAPTER_TEST_DIR ${HERMES_ADAPTER_DIR}/test) 3 | 4 | include_directories(${CMAKE_CURRENT_SOURCE_DIR}) 5 | 6 | find_package(Catch2 REQUIRED) 7 | find_program(BASH_PROGRAM bash) 8 | find_package(OpenMP REQUIRED COMPONENTS C CXX) 9 | 10 | enable_testing() 11 | 12 | if(HERMES_ENABLE_POSIX_ADAPTER) 13 | add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/posix) 14 | endif() 15 | 16 | if(HERMES_ENABLE_STDIO_ADAPTER) 17 | add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/stdio) 18 | endif() 19 | 20 | if(HERMES_ENABLE_MPIIO_ADAPTER) 21 | add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/mpiio) 22 | endif() 23 | 24 | if(HERMES_ENABLE_PUBSUB_ADAPTER) 25 | # add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/pubsub) 26 | endif() 27 | 28 | if(HERMES_ENABLE_VFD) 29 | add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/vfd) 30 | endif() -------------------------------------------------------------------------------- /tasks/hermes_bucket_mdm/include/hermes_bucket_mdm/hermes_bucket_mdm_methods.h: -------------------------------------------------------------------------------- 1 | #ifndef HRUN_HERMES_BUCKET_MDM_METHODS_H_ 2 | #define HRUN_HERMES_BUCKET_MDM_METHODS_H_ 3 | 4 | /** The set of methods in the admin task */ 5 | struct Method : public TaskMethod { 6 | TASK_METHOD_T kGetOrCreateTag = kLast + 0; 7 | TASK_METHOD_T kGetTagId = kLast + 1; 8 | TASK_METHOD_T kGetTagName = kLast + 2; 9 | TASK_METHOD_T kRenameTag = kLast + 3; 10 | TASK_METHOD_T kDestroyTag = kLast + 4; 11 | TASK_METHOD_T kTagAddBlob = kLast + 5; 12 | TASK_METHOD_T kTagRemoveBlob = kLast + 6; 13 | TASK_METHOD_T kTagClearBlobs = kLast + 10; 14 | TASK_METHOD_T kUpdateSize = kLast + 11; 15 | TASK_METHOD_T kAppendBlobSchema = kLast + 12; 16 | TASK_METHOD_T kAppendBlob = kLast + 13; 17 | TASK_METHOD_T kGetSize = kLast + 14; 18 | TASK_METHOD_T kSetBlobMdm = kLast + 15; 19 | TASK_METHOD_T kGetContainedBlobIds = kLast + 16; 20 | TASK_METHOD_T kPollTagMetadata = kLast + 17; 21 | }; 22 | 23 | #endif // HRUN_HERMES_BUCKET_MDM_METHODS_H_ -------------------------------------------------------------------------------- /test/unit/hermes_adapters/posix/hdf5_write_read.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import h5py 3 | 4 | def write_hdf5(fname, dset, dset_name, mode): 5 | with h5py.File(fname, mode, swmr=False) as h5_file: #swmr=False has async issue 6 | h5_file.create_dataset( 7 | dset_name, 8 | data=dset) 9 | 10 | print(f"{dset_name} {dset.shape} write complete ...") 11 | 12 | def read_hdf5(fname): 13 | mode='r' 14 | with h5py.File(fname, mode, swmr=False) as h5_file: 15 | dsets = h5_file.keys() 16 | for ds in dsets: 17 | tmp_d = h5_file.get(ds) 18 | print(f"{ds} {tmp_d.shape} read complete ...") 19 | 20 | 21 | if __name__ == "__main__": 22 | d1 = np.random.random(size = (1000,20)) 23 | d2 = np.random.random(size = (200,200)) 24 | # print(d1.shape, d2.shape) 25 | 26 | fname = '/tmp/test_hermes/test_data.h5' 27 | 28 | write_hdf5(fname,d1,'dataset_1','w') 29 | # (fname,d2,'dataset_2','a') 30 | 31 | read_hdf5(fname) 32 | -------------------------------------------------------------------------------- /src/hermes_config_manager.cc: -------------------------------------------------------------------------------- 1 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 | * Distributed under BSD 3-Clause license. * 3 | * Copyright by The HDF Group. * 4 | * Copyright by the Illinois Institute of Technology. * 5 | * All rights reserved. * 6 | * * 7 | * This file is part of Hermes. The full Hermes copyright notice, including * 8 | * terms governing use, modification, and redistribution, is contained in * 9 | * the COPYING file, which can be found at the top directory. If you do not * 10 | * have access to the file, you may request a copy from help@hdfgroup.org. * 11 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 12 | 13 | #include "hermes/config_manager.h" 14 | 15 | DEFINE_SINGLETON_CC(hermes::ConfigurationManager) 16 | -------------------------------------------------------------------------------- /tasks/data_stager/include/data_stager/factory/abstract_stager.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by lukemartinlogan on 9/30/23. 3 | // 4 | 5 | #ifndef HERMES_TASKS_DATA_STAGER_SRC_ABSTRACT_STAGER_H_ 6 | #define HERMES_TASKS_DATA_STAGER_SRC_ABSTRACT_STAGER_H_ 7 | 8 | #include "../data_stager.h" 9 | #include "hermes_bucket_mdm/hermes_bucket_mdm.h" 10 | 11 | namespace hermes::data_stager { 12 | 13 | class AbstractStager { 14 | public: 15 | std::string path_; 16 | std::string params_; 17 | 18 | AbstractStager() = default; 19 | ~AbstractStager() = default; 20 | 21 | virtual void RegisterStager(RegisterStagerTask *task, RunContext &rctx) = 0; 22 | virtual void StageIn(blob_mdm::Client &blob_mdm, StageInTask *task, RunContext &rctx) = 0; 23 | virtual void StageOut(blob_mdm::Client &blob_mdm, StageOutTask *task, RunContext &rctx) = 0; 24 | virtual void UpdateSize(bucket_mdm::Client &bkt_mdm, UpdateSizeTask *task, RunContext &rctx) = 0; 25 | }; 26 | 27 | } // namespace hermes 28 | 29 | #endif // HERMES_TASKS_DATA_STAGER_SRC_ABSTRACT_STAGER_H_ 30 | -------------------------------------------------------------------------------- /hermes_adapters/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | #------------------------------------------------------------------------------ 2 | # Build Hrun Admin Task Library 3 | #------------------------------------------------------------------------------ 4 | add_subdirectory(filesystem) 5 | add_subdirectory(posix) 6 | if (HERMES_ENABLE_STDIO_ADAPTER) 7 | add_subdirectory(stdio) 8 | endif() 9 | if (HERMES_ENABLE_MPIIO_ADAPTER) 10 | add_subdirectory(mpiio) 11 | endif() 12 | if (HERMES_ENABLE_VFD) 13 | add_subdirectory(vfd) 14 | endif() 15 | 16 | #----------------------------------------------------------------------------- 17 | # Install HRUN Admin Task Library Headers 18 | #----------------------------------------------------------------------------- 19 | file(GLOB_RECURSE HERMES_HEADERS "*.h") 20 | install( 21 | FILES 22 | ${HERMES_HEADERS} 23 | DESTINATION 24 | ${CMAKE_INSTALL_PREFIX}/include/hermes_adapters 25 | COMPONENT 26 | headers 27 | ) 28 | install(DIRECTORY mapper DESTINATION ${CMAKE_INSTALL_PREFIX}/include/hermes_adapters) -------------------------------------------------------------------------------- /hrun/src/hrun_stop_runtime.cc: -------------------------------------------------------------------------------- 1 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 | * Distributed under BSD 3-Clause license. * 3 | * Copyright by The HDF Group. * 4 | * Copyright by the Illinois Institute of Technology. * 5 | * All rights reserved. * 6 | * * 7 | * This file is part of Hermes. The full Hermes copyright notice, including * 8 | * terms governing use, modification, and redistribution, is contained in * 9 | * the COPYING file, which can be found at the top directory. If you do not * 10 | * have access to the file, you may request a copy from help@hdfgroup.org. * 11 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 12 | 13 | #include "hrun_admin/hrun_admin.h" 14 | 15 | int main() { 16 | TRANSPARENT_HRUN(); 17 | HRUN_ADMIN->StopRuntimeRoot(); 18 | } -------------------------------------------------------------------------------- /ci/install_docs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # CD into git workspace 4 | cd ${GITHUB_WORKSPACE} 5 | 6 | # Make build directory 7 | mkdir build 8 | cd build 9 | 10 | INSTALL_PREFIX="${HOME}" 11 | 12 | export CXXFLAGS="${CXXFLAGS} -std=c++17 -Werror -Wall -Wextra" 13 | cmake \ 14 | -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \ 15 | -DCMAKE_PREFIX_PATH=${INSTALL_PREFIX} \ 16 | -DCMAKE_BUILD_RPATH=${INSTALL_PREFIX}/lib \ 17 | -DCMAKE_INSTALL_RPATH=${INSTALL_PREFIX}/lib \ 18 | -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ 19 | -DBUILD_SHARED_LIBS=ON \ 20 | -DHERMES_ENABLE_DOXYGEN=ON \ 21 | -DHERMES_ENABLE_COVERAGE=ON \ 22 | .. 23 | make dox >& out.txt 24 | # cat out.txt | grep warning | grep -v "ignoring unsupported tag" 25 | rec="$( grep warning build/out.txt | grep -v "ignoring unsupported tag" | wc -l )" 26 | echo "$rec" 27 | exit "$rec" 28 | -------------------------------------------------------------------------------- /test/unit/hermes_adapters/vfd/hermes_vfd_test.cc: -------------------------------------------------------------------------------- 1 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 | * Distributed under BSD 3-Clause license. * 3 | * Copyright by The HDF Group. * 4 | * Copyright by the Illinois Institute of Technology. * 5 | * All rights reserved. * 6 | * * 7 | * This file is part of Hermes. The full Hermes copyright notice, including * 8 | * terms governing use, modification, and redistribution, is contained in * 9 | * the COPYING file, which can be found at the top directory. If you do not * 10 | * have access to the file, you may request a copy from help@hdfgroup.org. * 11 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 12 | 13 | #include "hermes_vfd_test.h" 14 | 15 | int main(int argc, char **argv) { 16 | TESTER->Init(argc, argv); 17 | } 18 | -------------------------------------------------------------------------------- /hrun/src/hrun_client.cc: -------------------------------------------------------------------------------- 1 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 | * Distributed under BSD 3-Clause license. * 3 | * Copyright by The HDF Group. * 4 | * Copyright by the Illinois Institute of Technology. * 5 | * All rights reserved. * 6 | * * 7 | * This file is part of Hermes. The full Hermes copyright notice, including * 8 | * terms governing use, modification, and redistribution, is contained in * 9 | * the COPYING file, which can be found at the top directory. If you do not * 10 | * have access to the file, you may request a copy from help@hdfgroup.org. * 11 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 12 | 13 | #include "hermes_shm/util/singleton.h" 14 | #include "hrun/api/hrun_client.h" 15 | 16 | /** Runtime singleton */ 17 | DEFINE_SINGLETON_CC(hrun::Client) -------------------------------------------------------------------------------- /test/unit/hermes_adapters/mpiio/mpiio_adapter_test.cc: -------------------------------------------------------------------------------- 1 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 | * Distributed under BSD 3-Clause license. * 3 | * Copyright by The HDF Group. * 4 | * Copyright by the Illinois Institute of Technology. * 5 | * All rights reserved. * 6 | * * 7 | * This file is part of Hermes. The full Hermes copyright notice, including * 8 | * terms governing use, modification, and redistribution, is contained in * 9 | * the COPYING file, which can be found at the top directory. If you do not * 10 | * have access to the file, you may request a copy from help@hdfgroup.org. * 11 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 12 | 13 | #include "mpiio_adapter_test.h" 14 | 15 | int main(int argc, char **argv) { 16 | TESTER->Init(argc, argv); 17 | } 18 | -------------------------------------------------------------------------------- /test/unit/hermes_adapters/posix/posix_adapter_test.cc: -------------------------------------------------------------------------------- 1 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 | * Distributed under BSD 3-Clause license. * 3 | * Copyright by The HDF Group. * 4 | * Copyright by the Illinois Institute of Technology. * 5 | * All rights reserved. * 6 | * * 7 | * This file is part of Hermes. The full Hermes copyright notice, including * 8 | * terms governing use, modification, and redistribution, is contained in * 9 | * the COPYING file, which can be found at the top directory. If you do not * 10 | * have access to the file, you may request a copy from help@hdfgroup.org. * 11 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 12 | 13 | #include "posix_adapter_test.h" 14 | 15 | int main(int argc, char **argv) { 16 | TESTER->Init(argc, argv); 17 | } 18 | -------------------------------------------------------------------------------- /test/unit/hermes_adapters/stdio/stdio_adapter_test.cc: -------------------------------------------------------------------------------- 1 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 | * Distributed under BSD 3-Clause license. * 3 | * Copyright by The HDF Group. * 4 | * Copyright by the Illinois Institute of Technology. * 5 | * All rights reserved. * 6 | * * 7 | * This file is part of Hermes. The full Hermes copyright notice, including * 8 | * terms governing use, modification, and redistribution, is contained in * 9 | * the COPYING file, which can be found at the top directory. If you do not * 10 | * have access to the file, you may request a copy from help@hdfgroup.org. * 11 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 12 | 13 | #include "stdio_adapter_test.h" 14 | 15 | int main(int argc, char **argv) { 16 | TESTER->Init(argc, argv); 17 | } 18 | 19 | -------------------------------------------------------------------------------- /test/unit/hermes_adapters/vfd/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(HERMES_VFD_DIR ${HERMES_ADAPTER_DIR}/vfd) 2 | 3 | set(hermes_vfd_tests 4 | hermes_vfd_adapter_test 5 | ) 6 | 7 | add_executable(hermes_vfd_adapter_test 8 | ${CMAKE_CURRENT_SOURCE_DIR}/hermes_vfd_test.cc 9 | ${CMAKE_CURRENT_SOURCE_DIR}/hermes_vfd_basic_test.cc) 10 | target_include_directories(hermes_vfd_adapter_test PRIVATE ${HERMES_VFD_DIR}) 11 | target_include_directories(hermes_vfd_adapter_test PRIVATE ${HERMES_ADAPTER_TEST_DIR}) 12 | target_include_directories(hermes_vfd_adapter_test 13 | SYSTEM PRIVATE ${HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES} 14 | ) 15 | target_compile_definitions(hermes_vfd_adapter_test PUBLIC 16 | HERMES_INTERCEPT=1 HERMES_MPI_TESTS=true) 17 | add_dependencies(hermes_vfd_adapter_test hermes) 18 | target_link_libraries(hermes_vfd_adapter_test 19 | hermes 20 | Catch2::Catch2 21 | MPI::MPI_CXX 22 | stdc++fs 23 | ${HDF5_HERMES_VFD_EXT_LIB_DEPENDENCIES}) 24 | 25 | jarvis_test(vfd test_hermes_vfd_basic) 26 | jarvis_test(vfd test_hermes_vfd_scratch) -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | cmake* 2 | .idea 3 | *.cmd 4 | *.symvers 5 | *.cmd 6 | *.mod 7 | *.mod.c 8 | *.ko 9 | *.order 10 | *.o 11 | *.o.cmd 12 | *.o.d 13 | build 14 | 15 | build/ 16 | benchmarks/HermesVFD 17 | 18 | hermes_shm/.git* 19 | 20 | GPATH 21 | GRTAGS 22 | GTAGS 23 | 24 | /compiler_data 25 | 26 | .gdb_history 27 | 28 | cmake-build-* 29 | .idea 30 | .clang-format 31 | __pycache__/ 32 | /hrun/src/hermes_adapters/posix/cmake-build-debug-system/CMakeFiles/clion-log.txt 33 | /hermes_adapters/test/posix/Testing/Temporary/LastTest.log 34 | .cache 35 | 36 | wrapper/java/out 37 | 38 | Testing 39 | 40 | .gradle 41 | **/build/ 42 | !hrun/src/**/build/ 43 | 44 | # Ignore Gradle GUI config 45 | gradle-app.setting 46 | 47 | # Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored) 48 | !gradle-wrapper.jar 49 | 50 | # Avoid ignore Gradle wrappper properties 51 | !gradle-wrapper.properties 52 | 53 | # Cache of project 54 | .gradletasknamecache 55 | 56 | # Eclipse Gradle plugin generated files 57 | # Eclipse Core 58 | .project 59 | # JDT-specific (Eclipse Java Development Tools) 60 | .classpath -------------------------------------------------------------------------------- /hermes_adapters/filesystem/filesystem_mdm_singleton.cc: -------------------------------------------------------------------------------- 1 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 | * Distributed under BSD 3-Clause license. * 3 | * Copyright by The HDF Group. * 4 | * Copyright by the Illinois Institute of Technology. * 5 | * All rights reserved. * 6 | * * 7 | * This file is part of Hermes. The full Hermes copyright notice, including * 8 | * terms governing use, modification, and redistribution, is contained in * 9 | * the COPYING file, which can be found at the top directory. If you do not * 10 | * have access to the file, you may request a copy from help@hdfgroup.org. * 11 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 12 | 13 | #include "hermes_shm/util/singleton.h" 14 | 15 | #include "filesystem_mdm.h" 16 | DEFINE_SINGLETON_CC(hermes::adapter::MetadataManager) 17 | -------------------------------------------------------------------------------- /benchmark/test_init.cc: -------------------------------------------------------------------------------- 1 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 | * Distributed under BSD 3-Clause license. * 3 | * Copyright by The HDF Group. * 4 | * Copyright by the Illinois Institute of Technology. * 5 | * All rights reserved. * 6 | * * 7 | * This file is part of Hermes. The full Hermes copyright notice, including * 8 | * terms governing use, modification, and redistribution, is contained in * 9 | * the COPYING file, which can be found at the top directory. If you do not * 10 | * have access to the file, you may request a copy from help@hdfgroup.org. * 11 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 12 | 13 | 14 | #include "hrun/api/hrun_client.h" 15 | #include "basic_test.h" 16 | #include "test_init.h" 17 | 18 | void MainPretest() { 19 | } 20 | 21 | void MainPosttest() { 22 | } 23 | -------------------------------------------------------------------------------- /benchmark/test_init.h: -------------------------------------------------------------------------------- 1 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 | * Distributed under BSD 3-Clause license. * 3 | * Copyright by The HDF Group. * 4 | * Copyright by the Illinois Institute of Technology. * 5 | * All rights reserved. * 6 | * * 7 | * This file is part of Hermes. The full Hermes copyright notice, including * 8 | * terms governing use, modification, and redistribution, is contained in * 9 | * the COPYING file, which can be found at the top directory. If you do not * 10 | * have access to the file, you may request a copy from help@hdfgroup.org. * 11 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 12 | 13 | 14 | #ifndef HRUN_TEST_UNIT_IPC_TEST_INIT_H_ 15 | #define HRUN_TEST_UNIT_IPC_TEST_INIT_H_ 16 | 17 | #include "hrun/hrun_types.h" 18 | 19 | #endif // HRUN_TEST_UNIT_IPC_TEST_INIT_H_ 20 | -------------------------------------------------------------------------------- /test/unit/boost/test_init.cc: -------------------------------------------------------------------------------- 1 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 | * Distributed under BSD 3-Clause license. * 3 | * Copyright by The HDF Group. * 4 | * Copyright by the Illinois Institute of Technology. * 5 | * All rights reserved. * 6 | * * 7 | * This file is part of Hermes. The full Hermes copyright notice, including * 8 | * terms governing use, modification, and redistribution, is contained in * 9 | * the COPYING file, which can be found at the top directory. If you do not * 10 | * have access to the file, you may request a copy from help@hdfgroup.org. * 11 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 12 | 13 | 14 | #include "hrun/api/hrun_client.h" 15 | #include "basic_test.h" 16 | #include "test_init.h" 17 | 18 | void MainPretest() { 19 | } 20 | 21 | void MainPosttest() { 22 | } 23 | -------------------------------------------------------------------------------- /test/unit/config/test_init.cc: -------------------------------------------------------------------------------- 1 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 | * Distributed under BSD 3-Clause license. * 3 | * Copyright by The HDF Group. * 4 | * Copyright by the Illinois Institute of Technology. * 5 | * All rights reserved. * 6 | * * 7 | * This file is part of Hermes. The full Hermes copyright notice, including * 8 | * terms governing use, modification, and redistribution, is contained in * 9 | * the COPYING file, which can be found at the top directory. If you do not * 10 | * have access to the file, you may request a copy from help@hdfgroup.org. * 11 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 12 | 13 | 14 | #include "hrun/api/hrun_client.h" 15 | #include "basic_test.h" 16 | #include "test_init.h" 17 | 18 | void MainPretest() { 19 | } 20 | 21 | void MainPosttest() { 22 | } 23 | -------------------------------------------------------------------------------- /wrapper/python/cpp/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | #------------------------------------------------------------------------------ 2 | # Find Packages 3 | #------------------------------------------------------------------------------ 4 | # find_package(pybind11 CONFIG REQUIRED) 5 | # message(STATUS "Found pybind11 v${pybind11_VERSION}: ${pybind11_INCLUDE_DIRS}") 6 | 7 | #------------------------------------------------------------------------------ 8 | # Compile Python Hermes 9 | #------------------------------------------------------------------------------ 10 | pybind11_add_module(py_hermes MODULE py_hermes.cpp) 11 | add_dependencies(py_hermes ${Hermes_CLIENT_DEPS} hermes) 12 | target_link_libraries(py_hermes PUBLIC 13 | ${Hermes_CLIENT_LIBRARIES} hermes pybind11::module) 14 | 15 | #------------------------------------------------------------------------------ 16 | # Install Targets 17 | #------------------------------------------------------------------------------ 18 | install(TARGETS 19 | py_hermes 20 | LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} 21 | ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) -------------------------------------------------------------------------------- /codegen/update_task: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """ 4 | USAGE: ./make_task [TASK_ROOT] 5 | """ 6 | 7 | import os 8 | import sys 9 | from codegen.util.paths import HRUN_ROOT 10 | 11 | def copy_replace(TASK_ROOT, TASK_TEMPL_ROOT, rel_path, TASK_NAME): 12 | with open(f'{TASK_TEMPL_ROOT}/{rel_path}') as fp: 13 | text = fp.read() 14 | text = text.replace('TASK_NAME', TASK_NAME) 15 | rel_path = rel_path.replace('TASK_NAME', TASK_NAME) 16 | with open(f'{TASK_ROOT}/{rel_path}', 'w') as fp: 17 | fp.write(text) 18 | 19 | TASK_ROOT = sys.argv[1] 20 | TASK_NAME = os.path.basename(TASK_ROOT) 21 | TASK_TEMPL_ROOT = f'{HRUN_ROOT}/tasks_required/TASK_NAME 22 | 23 | os.makedirs(f'{TASK_ROOT}/src', exist_ok=True) 24 | os.makedirs(f'{TASK_ROOT}/include/{TASK_NAME}', exist_ok=True) 25 | copy_replace(TASK_ROOT, TASK_TEMPL_ROOT, 'CMakeLists.txt', TASK_NAME) 26 | copy_replace(TASK_ROOT, TASK_TEMPL_ROOT, 'src/CMakeLists.txt', TASK_NAME) 27 | copy_replace(TASK_ROOT, TASK_TEMPL_ROOT, 'src/TASK_NAME.cc', TASK_NAME) 28 | copy_replace(TASK_ROOT, TASK_TEMPL_ROOT, 'include/TASK_NAME/TASK_NAME.h', TASK_NAME) 29 | -------------------------------------------------------------------------------- /test/unit/ipc/test_init.h: -------------------------------------------------------------------------------- 1 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 | * Distributed under BSD 3-Clause license. * 3 | * Copyright by The HDF Group. * 4 | * Copyright by the Illinois Institute of Technology. * 5 | * All rights reserved. * 6 | * * 7 | * This file is part of Hermes. The full Hermes copyright notice, including * 8 | * terms governing use, modification, and redistribution, is contained in * 9 | * the COPYING file, which can be found at the top directory. If you do not * 10 | * have access to the file, you may request a copy from help@hdfgroup.org. * 11 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 12 | 13 | 14 | #ifndef HRUN_TEST_UNIT_IPC_TEST_INIT_H_ 15 | #define HRUN_TEST_UNIT_IPC_TEST_INIT_H_ 16 | 17 | #include "hrun/hrun_types.h" 18 | 19 | #endif // HRUN_TEST_UNIT_IPC_TEST_INIT_H_ 20 | -------------------------------------------------------------------------------- /test/unit/boost/test_init.h: -------------------------------------------------------------------------------- 1 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 | * Distributed under BSD 3-Clause license. * 3 | * Copyright by The HDF Group. * 4 | * Copyright by the Illinois Institute of Technology. * 5 | * All rights reserved. * 6 | * * 7 | * This file is part of Hermes. The full Hermes copyright notice, including * 8 | * terms governing use, modification, and redistribution, is contained in * 9 | * the COPYING file, which can be found at the top directory. If you do not * 10 | * have access to the file, you may request a copy from help@hdfgroup.org. * 11 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 12 | 13 | 14 | #ifndef HRUN_TEST_UNIT_IPC_TEST_INIT_H_ 15 | #define HRUN_TEST_UNIT_IPC_TEST_INIT_H_ 16 | 17 | #include "hrun/hrun_types.h" 18 | 19 | #endif // HRUN_TEST_UNIT_IPC_TEST_INIT_H_ 20 | -------------------------------------------------------------------------------- /test/unit/config/test_init.h: -------------------------------------------------------------------------------- 1 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 | * Distributed under BSD 3-Clause license. * 3 | * Copyright by The HDF Group. * 4 | * Copyright by the Illinois Institute of Technology. * 5 | * All rights reserved. * 6 | * * 7 | * This file is part of Hermes. The full Hermes copyright notice, including * 8 | * terms governing use, modification, and redistribution, is contained in * 9 | * the COPYING file, which can be found at the top directory. If you do not * 10 | * have access to the file, you may request a copy from help@hdfgroup.org. * 11 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 12 | 13 | 14 | #ifndef HRUN_TEST_UNIT_IPC_TEST_INIT_H_ 15 | #define HRUN_TEST_UNIT_IPC_TEST_INIT_H_ 16 | 17 | #include "hrun/hrun_types.h" 18 | 19 | #endif // HRUN_TEST_UNIT_IPC_TEST_INIT_H_ 20 | -------------------------------------------------------------------------------- /test/unit/hermes/test_init.h: -------------------------------------------------------------------------------- 1 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 | * Distributed under BSD 3-Clause license. * 3 | * Copyright by The HDF Group. * 4 | * Copyright by the Illinois Institute of Technology. * 5 | * All rights reserved. * 6 | * * 7 | * This file is part of Hermes. The full Hermes copyright notice, including * 8 | * terms governing use, modification, and redistribution, is contained in * 9 | * the COPYING file, which can be found at the top directory. If you do not * 10 | * have access to the file, you may request a copy from help@hdfgroup.org. * 11 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 12 | 13 | 14 | #ifndef HRUN_TEST_UNIT_IPC_TEST_INIT_H_ 15 | #define HRUN_TEST_UNIT_IPC_TEST_INIT_H_ 16 | 17 | #include "hrun/hrun_types.h" 18 | 19 | #endif // HRUN_TEST_UNIT_IPC_TEST_INIT_H_ 20 | -------------------------------------------------------------------------------- /test/unit/ipc/test_finalize.cc: -------------------------------------------------------------------------------- 1 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 | * Distributed under BSD 3-Clause license. * 3 | * Copyright by The HDF Group. * 4 | * Copyright by the Illinois Institute of Technology. * 5 | * All rights reserved. * 6 | * * 7 | * This file is part of Hermes. The full Hermes copyright notice, including * 8 | * terms governing use, modification, and redistribution, is contained in * 9 | * the COPYING file, which can be found at the top directory. If you do not * 10 | * have access to the file, you may request a copy from help@hdfgroup.org. * 11 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 12 | 13 | #include "basic_test.h" 14 | #include "hrun/api/hrun_client.h" 15 | #include "hrun_admin/hrun_admin.h" 16 | 17 | TEST_CASE("TestFinalize") { 18 | HRUN_ADMIN->StopRuntimeRoot(); 19 | } 20 | -------------------------------------------------------------------------------- /test/unit/ipc/test_init.cc: -------------------------------------------------------------------------------- 1 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 | * Distributed under BSD 3-Clause license. * 3 | * Copyright by The HDF Group. * 4 | * Copyright by the Illinois Institute of Technology. * 5 | * All rights reserved. * 6 | * * 7 | * This file is part of Hermes. The full Hermes copyright notice, including * 8 | * terms governing use, modification, and redistribution, is contained in * 9 | * the COPYING file, which can be found at the top directory. If you do not * 10 | * have access to the file, you may request a copy from help@hdfgroup.org. * 11 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 12 | 13 | 14 | #include "hrun/api/hrun_client.h" 15 | #include "basic_test.h" 16 | #include "test_init.h" 17 | 18 | void MainPretest() { 19 | TRANSPARENT_HRUN(); 20 | } 21 | 22 | void MainPosttest() { 23 | } 24 | -------------------------------------------------------------------------------- /tasks/hermes_mdm/include/hermes_mdm/hermes_mdm.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by lukemartinlogan on 6/29/23. 3 | // 4 | 5 | #ifndef HRUN_hermes_mdm_H_ 6 | #define HRUN_hermes_mdm_H_ 7 | 8 | #include "hermes_mdm_tasks.h" 9 | 10 | namespace hermes::mdm { 11 | 12 | /** Create requests */ 13 | class Client : public TaskLibClient { 14 | public: 15 | /** Default constructor */ 16 | Client() = default; 17 | 18 | /** Destructor */ 19 | ~Client() = default; 20 | 21 | /** Create a hermes_mdm */ 22 | HSHM_ALWAYS_INLINE 23 | void CreateRoot(const DomainId &domain_id, 24 | const std::string &state_name) { 25 | id_ = TaskStateId::GetNull(); 26 | std::vector queue_info; 27 | id_ = HRUN_ADMIN->CreateTaskStateRoot( 28 | domain_id, state_name, id_, queue_info); 29 | Init(id_, HRUN_ADMIN->queue_id_); 30 | } 31 | 32 | /** Destroy task state + queue */ 33 | HSHM_ALWAYS_INLINE 34 | void DestroyRoot(const DomainId &domain_id) { 35 | HRUN_ADMIN->DestroyTaskStateRoot(domain_id, id_); 36 | } 37 | }; 38 | 39 | } // namespace hrun 40 | 41 | #endif // HRUN_hermes_mdm_H_ 42 | -------------------------------------------------------------------------------- /test/unit/hermes/test_init.cc: -------------------------------------------------------------------------------- 1 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 | * Distributed under BSD 3-Clause license. * 3 | * Copyright by The HDF Group. * 4 | * Copyright by the Illinois Institute of Technology. * 5 | * All rights reserved. * 6 | * * 7 | * This file is part of Hermes. The full Hermes copyright notice, including * 8 | * terms governing use, modification, and redistribution, is contained in * 9 | * the COPYING file, which can be found at the top directory. If you do not * 10 | * have access to the file, you may request a copy from help@hdfgroup.org. * 11 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 12 | 13 | 14 | #include "hrun/api/hrun_client.h" 15 | #include "basic_test.h" 16 | #include "test_init.h" 17 | 18 | void MainPretest() { 19 | TRANSPARENT_HRUN(); 20 | } 21 | 22 | void MainPosttest() { 23 | } 24 | -------------------------------------------------------------------------------- /tasks/hermes_blob_mdm/include/hermes_blob_mdm/hermes_blob_mdm_methods.h: -------------------------------------------------------------------------------- 1 | #ifndef HRUN_HERMES_BLOB_MDM_METHODS_H_ 2 | #define HRUN_HERMES_BLOB_MDM_METHODS_H_ 3 | 4 | /** The set of methods in the admin task */ 5 | struct Method : public TaskMethod { 6 | TASK_METHOD_T kPutBlob = kLast + 0; 7 | TASK_METHOD_T kGetBlob = kLast + 1; 8 | TASK_METHOD_T kTruncateBlob = kLast + 2; 9 | TASK_METHOD_T kDestroyBlob = kLast + 3; 10 | TASK_METHOD_T kTagBlob = kLast + 4; 11 | TASK_METHOD_T kBlobHasTag = kLast + 6; 12 | TASK_METHOD_T kGetBlobId = kLast + 8; 13 | TASK_METHOD_T kGetOrCreateBlobId = kLast + 9; 14 | TASK_METHOD_T kGetBlobName = kLast + 10; 15 | TASK_METHOD_T kGetBlobSize = kLast + 11; 16 | TASK_METHOD_T kGetBlobScore = kLast + 12; 17 | TASK_METHOD_T kGetBlobBuffers = kLast + 13; 18 | TASK_METHOD_T kRenameBlob = kLast + 14; 19 | TASK_METHOD_T kReorganizeBlob = kLast + 15; 20 | TASK_METHOD_T kSetBucketMdm = kLast + 16; 21 | TASK_METHOD_T kFlushData = kLast + 17; 22 | TASK_METHOD_T kPollBlobMetadata = kLast + 18; 23 | TASK_METHOD_T kPollTargetMetadata = kLast + 19; 24 | }; 25 | 26 | #endif // HRUN_HERMES_BLOB_MDM_METHODS_H_ -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Hermes 2 | 3 | Hermes is a heterogeneous-aware, multi-tiered, dynamic, and distributed I/O 4 | buffering system that aims to significantly accelerate I/O performance. 5 | See the [official site](http://www.cs.iit.edu/~scs/assets/projects/Hermes/Hermes.html) for more information. For design documents, 6 | architecture description, performance data, and individual component design, 7 | see the [wiki](https://grc.iit.edu/docs/category/hermes). 8 | 9 | ![Build](https://github.com/HDFGroup/hermes/workflows/GitHub%20Actions/badge.svg) 10 | 11 | [![Coverage Status](https://coveralls.io/repos/github/HDFGroup/hermes/badge.svg?branch=master)](https://coveralls.io/github/HDFGroup/hermes?branch=master) 12 | 13 | ## Building 14 | 15 | Read the guide on [Building Hermes](https://grc.iit.edu/docs/hermes/building-hermes). 16 | 17 | ## Contributing 18 | 19 | We follow the [Google C++ Style Guide](https://google.github.io/styleguide/cppguide.html). You can run `make lint` to ensure that your code conforms to the style. This requires the `cpplint` Python module (`pip install cpplint`). Alternatively, you can let the CI build inform you of required style changes. 20 | -------------------------------------------------------------------------------- /hrun/src/hrun_start_runtime.cc: -------------------------------------------------------------------------------- 1 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 | * Distributed under BSD 3-Clause license. * 3 | * Copyright by The HDF Group. * 4 | * Copyright by the Illinois Institute of Technology. * 5 | * All rights reserved. * 6 | * * 7 | * This file is part of Hermes. The full Hermes copyright notice, including * 8 | * terms governing use, modification, and redistribution, is contained in * 9 | * the COPYING file, which can be found at the top directory. If you do not * 10 | * have access to the file, you may request a copy from help@hdfgroup.org. * 11 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 12 | 13 | #include "hermes_shm/util/singleton.h" 14 | #include "hrun/api/hrun_runtime.h" 15 | 16 | int main(int argc, char **argv) { 17 | HRUN_RUNTIME->Create(); 18 | HRUN_RUNTIME->RunDaemon(); 19 | HRUN_RUNTIME->Finalize(); 20 | return 0; 21 | } 22 | -------------------------------------------------------------------------------- /tasks/data_stager/include/data_stager/factory/stager_factory.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by lukemartinlogan on 9/30/23. 3 | // 4 | 5 | #ifndef HERMES_TASKS_DATA_STAGER_SRC_STAGER_FACTORY_H_ 6 | #define HERMES_TASKS_DATA_STAGER_SRC_STAGER_FACTORY_H_ 7 | 8 | #include "../data_stager.h" 9 | #include "abstract_stager.h" 10 | #include "binary_stager.h" 11 | 12 | namespace hermes::data_stager { 13 | 14 | class StagerFactory { 15 | public: 16 | static std::unique_ptr Get(const std::string &path, 17 | const std::string ¶ms) { 18 | std::string protocol; 19 | hrun::LocalDeserialize srl(params); 20 | srl >> protocol; 21 | 22 | std::unique_ptr stager; 23 | if (protocol == "file") { 24 | stager = std::make_unique(); 25 | } else if (protocol == "parquet") { 26 | } else if (protocol == "hdf5") { 27 | } else { 28 | throw std::runtime_error("Unknown stager type"); 29 | } 30 | stager->path_ = path; 31 | stager->params_ = params; 32 | return stager; 33 | } 34 | }; 35 | 36 | } // namespace hermes 37 | 38 | #endif // HERMES_TASKS_DATA_STAGER_SRC_STAGER_FACTORY_H_ 39 | -------------------------------------------------------------------------------- /hrun/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by lukemartinlogan on 6/29/23. 3 | // 4 | 5 | #ifndef HRUN_worch_proc_round_robin_H_ 6 | #define HRUN_worch_proc_round_robin_H_ 7 | 8 | #include "worch_proc_round_robin_tasks.h" 9 | 10 | namespace hrun::worch_proc_round_robin { 11 | 12 | /** Create admin requests */ 13 | class Client : public TaskLibClient { 14 | 15 | public: 16 | /** Default constructor */ 17 | Client() = default; 18 | 19 | /** Destructor */ 20 | ~Client() = default; 21 | 22 | /** Create a worch_proc_round_robin */ 23 | HSHM_ALWAYS_INLINE 24 | void CreateRoot(const DomainId &domain_id, 25 | const std::string &state_name) { 26 | id_ = TaskStateId::GetNull(); 27 | std::vector queue_info; 28 | id_ = HRUN_ADMIN->CreateTaskStateRoot( 29 | domain_id, state_name, id_, queue_info); 30 | } 31 | 32 | /** Destroy state */ 33 | HSHM_ALWAYS_INLINE 34 | void DestroyRoot(const DomainId &domain_id) { 35 | HRUN_ADMIN->DestroyTaskStateRoot(domain_id, id_); 36 | } 37 | }; 38 | 39 | } // namespace hrun 40 | 41 | #endif // HRUN_worch_proc_round_robin_H_ 42 | -------------------------------------------------------------------------------- /hermes_adapters/adapter_constants.h: -------------------------------------------------------------------------------- 1 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 | * Distributed under BSD 3-Clause license. * 3 | * Copyright by The HDF Group. * 4 | * Copyright by the Illinois Institute of Technology. * 5 | * All rights reserved. * 6 | * * 7 | * This file is part of Hermes. The full Hermes copyright notice, including * 8 | * terms governing use, modification, and redistribution, is contained in * 9 | * the COPYING file, which can be found at the top directory. If you do not * 10 | * have access to the file, you may request a copy from help@hdfgroup.org. * 11 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 12 | 13 | #ifndef HERMES_ADAPTER_ADAPTER_CONSTANTS_H_ 14 | #define HERMES_ADAPTER_ADAPTER_CONSTANTS_H_ 15 | 16 | #include "mapper/abstract_mapper.h" 17 | 18 | namespace hermes::adapter { 19 | 20 | static inline const MapperType kMapperType = MapperType::kBalancedMapper; 21 | 22 | } 23 | 24 | #endif // HERMES_ADAPTER_ADAPTER_CONSTANTS_H_ 25 | -------------------------------------------------------------------------------- /hrun/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by lukemartinlogan on 6/29/23. 3 | // 4 | 5 | #ifndef HRUN_worch_queue_round_robin_H_ 6 | #define HRUN_worch_queue_round_robin_H_ 7 | 8 | #include "worch_queue_round_robin_tasks.h" 9 | 10 | namespace hrun::worch_queue_round_robin { 11 | 12 | /** Create admin requests */ 13 | class Client : public TaskLibClient { 14 | 15 | public: 16 | /** Default constructor */ 17 | Client() = default; 18 | 19 | /** Destructor */ 20 | ~Client() = default; 21 | 22 | /** Create a worch_queue_round_robin */ 23 | HSHM_ALWAYS_INLINE 24 | void CreateRoot(const DomainId &domain_id, 25 | const std::string &state_name) { 26 | id_ = TaskStateId::GetNull(); 27 | std::vector queue_info; 28 | id_ = HRUN_ADMIN->CreateTaskStateRoot( 29 | domain_id, state_name, id_, queue_info); 30 | Init(id_, HRUN_ADMIN->queue_id_); 31 | } 32 | 33 | /** Destroy task state */ 34 | HSHM_ALWAYS_INLINE 35 | void DestroyRoot(const DomainId &domain_id) { 36 | HRUN_ADMIN->DestroyTaskStateRoot(domain_id, id_); 37 | } 38 | }; 39 | 40 | } // namespace hrun 41 | 42 | #endif // HRUN_worch_queue_round_robin_H_ 43 | -------------------------------------------------------------------------------- /test/unit/hermes_adapters/adapter_test_utils.h: -------------------------------------------------------------------------------- 1 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 | * Distributed under BSD 3-Clause license. * 3 | * Copyright by The HDF Group. * 4 | * Copyright by the Illinois Institute of Technology. * 5 | * All rights reserved. * 6 | * * 7 | * This file is part of Hermes. The full Hermes copyright notice, including * 8 | * terms governing use, modification, and redistribution, is contained in * 9 | * the COPYING file, which can be found at the top directory. If you do not * 10 | * have access to the file, you may request a copy from help@hdfgroup.org. * 11 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 12 | 13 | #ifndef HERMES_ADAPTER_TEST_UTILS_H 14 | #define HERMES_ADAPTER_TEST_UTILS_H 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | #define CATCH_CONFIG_RUNNER 21 | #include 22 | namespace cl = Catch::Clara; 23 | 24 | #endif // HERMES_ADAPTER_TEST_UTILS_H 25 | -------------------------------------------------------------------------------- /test/unit/main.cc: -------------------------------------------------------------------------------- 1 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 | * Distributed under BSD 3-Clause license. * 3 | * Copyright by The HDF Group. * 4 | * Copyright by the Illinois Institute of Technology. * 5 | * All rights reserved. * 6 | * * 7 | * This file is part of Hermes. The full Hermes copyright notice, including * 8 | * terms governing use, modification, and redistribution, is contained in * 9 | * the COPYING file, which can be found at the top directory. If you do not * 10 | * have access to the file, you may request a copy from help@hdfgroup.org. * 11 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 12 | 13 | 14 | #include "basic_test.h" 15 | 16 | int main(int argc, char **argv) { 17 | int rc; 18 | Catch::Session session; 19 | auto cli = session.cli(); 20 | session.cli(cli); 21 | rc = session.applyCommandLine(argc, argv); 22 | if (rc != 0) return rc; 23 | MainPretest(); 24 | rc = session.run(); 25 | MainPosttest(); 26 | if (rc != 0) return rc; 27 | return rc; 28 | } 29 | -------------------------------------------------------------------------------- /codegen/codegen/hrun_config/generator.py: -------------------------------------------------------------------------------- 1 | import sys, os 2 | 3 | def print_macro(path, macro_name): 4 | with open(path) as fp: 5 | lines = fp.read().splitlines() 6 | macro_def = f'#define {macro_name}\\\n' 7 | macro_body = '\\\n'.join(lines) 8 | print(f'{macro_def}{macro_body}') 9 | 10 | def create_config(path, var_name, config_path, macro_name): 11 | with open(path) as fp: 12 | yaml_config_lines = fp.read().splitlines() 13 | 14 | # Create the hermes config string 15 | string_lines = [] 16 | string_lines.append(f"const inline char* {var_name} = ") 17 | for line in yaml_config_lines: 18 | line = line.replace('\"', '\\\"') 19 | line = line.replace('\'', '\\\'') 20 | string_lines.append(f"\"{line}\\n\"") 21 | string_lines[-1] = string_lines[-1] + ';' 22 | 23 | # Create the configuration 24 | config_lines = [] 25 | config_lines.append(f"#ifndef HRUN_SRC_CONFIG_{macro_name}_DEFAULT_H_") 26 | config_lines.append(f"#define HRUN_SRC_CONFIG_{macro_name}_DEFAULT_H_") 27 | config_lines += string_lines 28 | config_lines.append(f"#endif // HRUN_SRC_CONFIG_{macro_name}_DEFAULT_H_") 29 | 30 | # Persist 31 | config = "\n".join(config_lines) 32 | with open(config_path, 'w') as fp: 33 | fp.write(config) 34 | 35 | -------------------------------------------------------------------------------- /hrun/include/hrun/queue_manager/queue_factory.h: -------------------------------------------------------------------------------- 1 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 | * Distributed under BSD 3-Clause license. * 3 | * Copyright by The HDF Group. * 4 | * Copyright by the Illinois Institute of Technology. * 5 | * All rights reserved. * 6 | * * 7 | * This file is part of Hermes. The full Hermes copyright notice, including * 8 | * terms governing use, modification, and redistribution, is contained in * 9 | * the COPYING file, which can be found at the top directory. If you do not * 10 | * have access to the file, you may request a copy from help@hdfgroup.org. * 11 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 12 | 13 | #ifndef HRUN_INCLUDE_HRUN_QUEUE_MANAGER_QUEUE_FACTORY_H_ 14 | #define HRUN_INCLUDE_HRUN_QUEUE_MANAGER_QUEUE_FACTORY_H_ 15 | 16 | #include "queues/hshm_queue.h" 17 | 18 | namespace hrun { 19 | 20 | #ifdef QUEUE_TYPE 21 | using MultiQueue = MultiQueueT; 22 | #else 23 | using MultiQueue = MultiQueueT; 24 | #endif 25 | } // namespace hrun 26 | 27 | #endif // HRUN_INCLUDE_HRUN_QUEUE_MANAGER_QUEUE_FACTORY_H_ 28 | -------------------------------------------------------------------------------- /test/unit/main_mpi.cc: -------------------------------------------------------------------------------- 1 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 | * Distributed under BSD 3-Clause license. * 3 | * Copyright by The HDF Group. * 4 | * Copyright by the Illinois Institute of Technology. * 5 | * All rights reserved. * 6 | * * 7 | * This file is part of Hermes. The full Hermes copyright notice, including * 8 | * terms governing use, modification, and redistribution, is contained in * 9 | * the COPYING file, which can be found at the top directory. If you do not * 10 | * have access to the file, you may request a copy from help@hdfgroup.org. * 11 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 12 | 13 | #include "basic_test.h" 14 | #include 15 | 16 | int main(int argc, char **argv) { 17 | int rc; 18 | MPI_Init(&argc, &argv); 19 | Catch::Session session; 20 | auto cli = session.cli(); 21 | session.cli(cli); 22 | rc = session.applyCommandLine(argc, argv); 23 | if (rc != 0) return rc; 24 | MainPretest(); 25 | rc = session.run(); 26 | MainPosttest(); 27 | if (rc != 0) return rc; 28 | MPI_Finalize(); 29 | return rc; 30 | } 31 | -------------------------------------------------------------------------------- /codegen/make_task: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """ 4 | USAGE: ./make_task [TASK_ROOT] 5 | """ 6 | 7 | import os 8 | import sys 9 | from codegen.util.paths import HRUN_ROOT 10 | 11 | def copy_replace(TASK_ROOT, TASK_TEMPL_ROOT, rel_path, TASK_NAME): 12 | with open(f'{TASK_TEMPL_ROOT}/{rel_path}') as fp: 13 | text = fp.read() 14 | text = text.replace('TASK_NAME', TASK_NAME) 15 | rel_path = rel_path.replace('TASK_NAME', TASK_NAME) 16 | with open(f'{TASK_ROOT}/{rel_path}', 'w') as fp: 17 | fp.write(text) 18 | 19 | TASK_ROOT = sys.argv[1] 20 | TASK_NAME = os.path.basename(TASK_ROOT) 21 | TASK_TEMPL_ROOT = f'{HRUN_ROOT}/tasks_required/TASK_NAME' 22 | 23 | os.makedirs(f'{TASK_ROOT}/src', exist_ok=True) 24 | os.makedirs(f'{TASK_ROOT}/include/{TASK_NAME}', exist_ok=True) 25 | copy_replace(TASK_ROOT, TASK_TEMPL_ROOT, 'CMakeLists.txt', TASK_NAME) 26 | copy_replace(TASK_ROOT, TASK_TEMPL_ROOT, 'src/CMakeLists.txt', TASK_NAME) 27 | copy_replace(TASK_ROOT, TASK_TEMPL_ROOT, 'src/TASK_NAME.cc', TASK_NAME) 28 | copy_replace(TASK_ROOT, TASK_TEMPL_ROOT, 'include/TASK_NAME/TASK_NAME.h', TASK_NAME) 29 | copy_replace(TASK_ROOT, TASK_TEMPL_ROOT, 'include/TASK_NAME/TASK_NAME_lib_exec.h', TASK_NAME) 30 | copy_replace(TASK_ROOT, TASK_TEMPL_ROOT, 'include/TASK_NAME/TASK_NAME_tasks.h', TASK_NAME) 31 | copy_replace(TASK_ROOT, TASK_TEMPL_ROOT, 'include/TASK_NAME/TASK_NAME_methods.h', TASK_NAME) 32 | copy_replace(TASK_ROOT, TASK_TEMPL_ROOT, 'include/TASK_NAME/TASK_NAME_methods.yaml', TASK_NAME) 33 | -------------------------------------------------------------------------------- /include/hermes/statuses.h: -------------------------------------------------------------------------------- 1 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 | * Distributed under BSD 3-Clause license. * 3 | * Copyright by The HDF Group. * 4 | * Copyright by the Illinois Institute of Technology. * 5 | * All rights reserved. * 6 | * * 7 | * This file is part of Hermes. The full Hermes copyright notice, including * 8 | * terms governing use, modification, and redistribution, is contained in * 9 | * the COPYING file, which can be found at the top directory. If you do not * 10 | * have access to the file, you may request a copy from help@hdfgroup.org. * 11 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 12 | 13 | #ifndef HRUN_TASKS_HERMES_INCLUDE_STATUSES_H_ 14 | #define HRUN_TASKS_HERMES_INCLUDE_STATUSES_H_ 15 | 16 | #include "status.h" 17 | 18 | #define STATUS_T static inline const Status 19 | 20 | namespace hermes { 21 | 22 | STATUS_T NOT_IMPLEMENTED(0, "Function was not implemented"); 23 | STATUS_T DPE_PLACEMENT_SCHEMA_EMPTY(1, "DPE placement schema is empty"); 24 | STATUS_T DPE_NO_SPACE(1, "Placement failed. Non-fatal."); 25 | STATUS_T DPE_MIN_IO_TIME_NO_SOLUTION( 26 | 1, "DPE could not find solution for the minimize I/O time DPE"); 27 | 28 | } // namespace hermes 29 | 30 | #endif // HRUN_TASKS_HERMES_INCLUDE_STATUSES_H_ 31 | -------------------------------------------------------------------------------- /hrun/src/config_client.cc: -------------------------------------------------------------------------------- 1 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 | * Distributed under BSD 3-Clause license. * 3 | * Copyright by The HDF Group. * 4 | * Copyright by the Illinois Institute of Technology. * 5 | * All rights reserved. * 6 | * * 7 | * This file is part of Hermes. The full Hermes copyright notice, including * 8 | * terms governing use, modification, and redistribution, is contained in * 9 | * the COPYING file, which can be found at the top directory. If you do not * 10 | * have access to the file, you may request a copy from help@hdfgroup.org. * 11 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 12 | 13 | #include "hrun/config/config_client.h" 14 | #include "hrun/config/config_client_default.h" 15 | #include "hermes_shm/util/config_parse.h" 16 | #include 17 | 18 | namespace stdfs = std::filesystem; 19 | 20 | namespace hrun::config { 21 | 22 | /** parse the YAML node */ 23 | void ClientConfig::ParseYAML(YAML::Node &yaml_conf) { 24 | if (yaml_conf["thread_model"]) { 25 | thread_model_ = yaml_conf["thread_model"].as(); 26 | } 27 | } 28 | 29 | /** Load the default configuration */ 30 | void ClientConfig::LoadDefault() { 31 | LoadText(kHrunClientDefaultConfigStr, false); 32 | } 33 | 34 | } // namespace hrun::config 35 | -------------------------------------------------------------------------------- /tasks/hermes_mdm/src/hermes_mdm.cc: -------------------------------------------------------------------------------- 1 | // 2 | // Created by lukemartinlogan on 6/29/23. 3 | // 4 | 5 | #include "hrun_admin/hrun_admin.h" 6 | #include "hrun/api/hrun_runtime.h" 7 | #include "hermes/config_server.h" 8 | #include "hermes_mdm/hermes_mdm.h" 9 | #include "hermes_blob_mdm/hermes_blob_mdm.h" 10 | #include "hermes_bucket_mdm/hermes_bucket_mdm.h" 11 | #include "hermes/dpe/dpe_factory.h" 12 | #include "bdev/bdev.h" 13 | 14 | namespace hermes::mdm { 15 | 16 | class Server : public TaskLib { 17 | public: 18 | /**==================================== 19 | * Configuration 20 | * ===================================*/ 21 | u32 node_id_; 22 | blob_mdm::Client blob_mdm_; 23 | bucket_mdm::Client bkt_mdm_; 24 | 25 | public: 26 | Server() = default; 27 | 28 | /** Construct hermes MDM */ 29 | void Construct(ConstructTask *task, RunContext &rctx) { 30 | HILOG(kDebug, "ConstructTaskPhase::kLoadConfig") 31 | std::string config_path = task->server_config_path_->str(); 32 | HERMES_CONF->LoadServerConfig(config_path); 33 | node_id_ = HRUN_CLIENT->node_id_; 34 | task->SetModuleComplete(); 35 | } 36 | void MonitorConstruct(u32 mode, ConstructTask *task, RunContext &rctx) { 37 | } 38 | 39 | /** Destory hermes MDM */ 40 | void Destruct(DestructTask *task, RunContext &rctx) { 41 | task->SetModuleComplete(); 42 | } 43 | void MonitorDestruct(u32 mode, DestructTask *task, RunContext &rctx) { 44 | } 45 | 46 | public: 47 | #include "hermes_mdm/hermes_mdm_lib_exec.h" 48 | }; 49 | 50 | } // namespace hrun 51 | 52 | HRUN_TASK_CC(hermes::mdm::Server, "hermes_mdm"); 53 | -------------------------------------------------------------------------------- /hrun/include/hrun/api/template/hrun_task_node_admin_root.template: -------------------------------------------------------------------------------- 1 | template 2 | hipc::LPointer Async##CUSTOM##Alloc(const TaskNode &task_node, 3 | Args&& ...args) { 4 | hipc::LPointer task = HRUN_CLIENT->AllocateTask(); 5 | Async##CUSTOM##Construct(task.ptr_, task_node, std::forward(args)...); 6 | return task; 7 | } 8 | template 9 | hipc::LPointer Async##CUSTOM(const TaskNode &task_node, 10 | Args&& ...args) { 11 | hipc::LPointer task = Async##CUSTOM##Alloc(task_node, std::forward(args)...); 12 | MultiQueue *queue = HRUN_CLIENT->GetQueue(queue_id_); 13 | queue->Emplace(task.ptr_->prio_, task.ptr_->lane_hash_, task.shm_); 14 | return task; 15 | } 16 | template 17 | hipc::LPointer Async##CUSTOM##Emplace(MultiQueue *queue, 18 | const TaskNode &task_node, 19 | Args&& ...args) { 20 | hipc::LPointer task = Async##CUSTOM##Alloc(task_node, std::forward(args)...); 21 | queue->Emplace(task.ptr_->prio_, task.ptr_->lane_hash_, task.shm_); 22 | return task; 23 | } 24 | template 25 | hipc::LPointer Async##CUSTOM##Root(Args&& ...args) { 26 | TaskNode task_node = HRUN_CLIENT->MakeTaskNodeId(); 27 | hipc::LPointer task = Async##CUSTOM(task_node + 1, std::forward(args)...); 28 | return task; 29 | } -------------------------------------------------------------------------------- /hrun/include/hrun/hrun_constants.h: -------------------------------------------------------------------------------- 1 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 | * Distributed under BSD 3-Clause license. * 3 | * Copyright by The HDF Group. * 4 | * Copyright by the Illinois Institute of Technology. * 5 | * All rights reserved. * 6 | * * 7 | * This file is part of Hermes. The full Hermes copyright notice, including * 8 | * terms governing use, modification, and redistribution, is contained in * 9 | * the COPYING file, which can be found at the top directory. If you do not * 10 | * have access to the file, you may request a copy from help@hdfgroup.org. * 11 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 12 | 13 | #ifndef HRUN_INCLUDE_HRUN_HRUN_CONSTANTS_H_ 14 | #define HRUN_INCLUDE_HRUN_HRUN_CONSTANTS_H_ 15 | 16 | namespace hrun { 17 | 18 | #include 19 | 20 | class Constants { 21 | public: 22 | inline static const std::string kClientConfEnv = "HERMES_CLIENT_CONF"; 23 | inline static const std::string kServerConfEnv = "HERMES_CONF"; 24 | 25 | static std::string GetEnvSafe(const std::string &env_name) { 26 | char *data = getenv(env_name.c_str()); 27 | if (data == nullptr) { 28 | return ""; 29 | } else { 30 | return data; 31 | } 32 | } 33 | }; 34 | 35 | } // namespace hrun 36 | 37 | #endif // HRUN_INCLUDE_HRUN_HRUN_CONSTANTS_H_ 38 | -------------------------------------------------------------------------------- /hrun/include/hrun/config/config_client.h: -------------------------------------------------------------------------------- 1 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 | * Distributed under BSD 3-Clause license. * 3 | * Copyright by The HDF Group. * 4 | * Copyright by the Illinois Institute of Technology. * 5 | * All rights reserved. * 6 | * * 7 | * This file is part of Hermes. The full Hermes copyright notice, including * 8 | * terms governing use, modification, and redistribution, is contained in * 9 | * the COPYING file, which can be found at the top directory. If you do not * 10 | * have access to the file, you may request a copy from help@hdfgroup.org. * 11 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 12 | 13 | #ifndef HRUN_SRC_CONFIG_CLIENT_H_ 14 | #define HRUN_SRC_CONFIG_CLIENT_H_ 15 | 16 | #include 17 | #include "config.h" 18 | 19 | namespace stdfs = std::filesystem; 20 | 21 | namespace hrun::config { 22 | 23 | /** 24 | * Configuration used to intialize client 25 | * */ 26 | class ClientConfig : public BaseConfig { 27 | public: 28 | /** The thread model of the application */ 29 | std::string thread_model_; 30 | 31 | private: 32 | void ParseYAML(YAML::Node &yaml_conf) override; 33 | void LoadDefault() override; 34 | }; 35 | 36 | } // namespace hrun::config 37 | 38 | namespace hrun { 39 | using ClientConfig = config::ClientConfig; 40 | } // namespace hrun 41 | 42 | #endif // HRUN_SRC_CONFIG_CLIENT_H_ 43 | -------------------------------------------------------------------------------- /include/hermes/status.h: -------------------------------------------------------------------------------- 1 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 | * Distributed under BSD 3-Clause license. * 3 | * Copyright by The HDF Group. * 4 | * Copyright by the Illinois Institute of Technology. * 5 | * All rights reserved. * 6 | * * 7 | * This file is part of Hermes. The full Hermes copyright notice, including * 8 | * terms governing use, modification, and redistribution, is contained in * 9 | * the COPYING file, which can be found at the top directory. If you do not * 10 | * have access to the file, you may request a copy from help@hdfgroup.org. * 11 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 12 | 13 | #ifndef HERMES_STATUS_H_ 14 | #define HERMES_STATUS_H_ 15 | 16 | #include 17 | 18 | /** \file hermes_status.h */ 19 | 20 | namespace hermes { 21 | 22 | class Status { 23 | public: 24 | int code_; 25 | const char *msg_; 26 | 27 | public: 28 | Status() : code_(0) {} 29 | 30 | explicit Status(int code, const char *msg) : code_(code), msg_(msg) {} 31 | 32 | Status(const Status &other) { 33 | code_ = other.code_; 34 | msg_ = other.msg_; 35 | } 36 | 37 | const char* Msg() const { 38 | return msg_; 39 | } 40 | 41 | bool Success() { 42 | return code_ == 0; 43 | } 44 | 45 | bool Fail() { 46 | return code_ != 0; 47 | } 48 | }; 49 | 50 | } // namespace hermes 51 | #endif // HERMES_STATUS_H_ 52 | -------------------------------------------------------------------------------- /test/unit/boost/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.10) 2 | project(hermes) 3 | 4 | set(CMAKE_CXX_STANDARD 17) 5 | 6 | #------------------------------------------------------------------------------ 7 | # Build Tests 8 | #------------------------------------------------------------------------------ 9 | 10 | add_executable(test_boost_exec 11 | ${TEST_MAIN}/main.cc 12 | test_init.cc 13 | test_boost.cc 14 | ) 15 | add_dependencies(test_boost_exec 16 | ${Hermes_RUNTIME_DEPS} hermes) 17 | target_link_libraries(test_boost_exec 18 | ${Hermes_RUNTIME_LIBRARIES} Catch2::Catch2 MPI::MPI_CXX) 19 | 20 | #------------------------------------------------------------------------------ 21 | # Test Cases 22 | #------------------------------------------------------------------------------ 23 | 24 | #add_test(NAME test_boost COMMAND 25 | # ${CMAKE_BINARY_DIR}/bin/test_messages "TestBoost") 26 | 27 | #------------------------------------------------------------------------------ 28 | # Install Targets 29 | #------------------------------------------------------------------------------ 30 | install(TARGETS 31 | test_boost_exec 32 | EXPORT 33 | ${HERMES_EXPORTED_TARGETS} 34 | LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} 35 | ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} 36 | RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR}) 37 | 38 | #----------------------------------------------------------------------------- 39 | # Coverage 40 | #----------------------------------------------------------------------------- 41 | if(HERMES_ENABLE_COVERAGE) 42 | set_coverage_flags(test_boost_exec) 43 | endif() 44 | -------------------------------------------------------------------------------- /ci/resource_graph.yaml: -------------------------------------------------------------------------------- 1 | fs: 2 | - avail: 17179869184 3 | dev_type: hdd 4 | device: /dev/hdd1 5 | fs_mount: /hdd 6 | fs_size: 16G 7 | fs_type: ext4 8 | host: localhost 9 | label: null 10 | model: CT500P1SSD8 11 | mount: /hdd 12 | parent: /dev/hdd 13 | partlabel: null 14 | partuuid: d4ff55be-ac9f-4bd3-9602-48e8b673fa37 15 | rota: false 16 | shared: false 17 | size: 17179869184 18 | tran: nvme 19 | use%: 0% 20 | used: 0G 21 | uuid: 8debe05d-9177-40b5-8bb0-82fc969ce919 22 | - avail: 17179869184 23 | dev_type: ssd 24 | device: /dev/ssd1 25 | fs_mount: /ssd 26 | fs_size: 16G 27 | fs_type: ext4 28 | host: localhost 29 | label: null 30 | model: CT500P1SSD8 31 | mount: /ssd 32 | parent: /dev/ssd 33 | partlabel: null 34 | partuuid: d4ff55be-ac9f-4bd3-9602-48e8b673fa37 35 | rota: false 36 | shared: false 37 | size: 17179869184 38 | tran: nvme 39 | use%: 0% 40 | used: 0G 41 | uuid: 8debe05d-9177-40b5-8bb0-82fc969ce919 42 | - avail: 17179869184 43 | dev_type: nvme 44 | device: /dev/nvme1n1p3 45 | fs_mount: /nvme 46 | fs_size: 16G 47 | fs_type: ext4 48 | host: localhost 49 | label: null 50 | model: CT500P1SSD8 51 | mount: /nvme 52 | parent: /dev/nvme1n1 53 | partlabel: null 54 | partuuid: d4ff55be-ac9f-4bd3-9602-48e8b673fa37 55 | rota: false 56 | shared: false 57 | size: 17179869184 58 | tran: nvme 59 | use%: 0% 60 | used: 0G 61 | uuid: 8debe05d-9177-40b5-8bb0-82fc969ce919 62 | hosts: 63 | - localhost 64 | net: 65 | - domain: lo 66 | fabric: 127.0.0.0/8 67 | host: localhost 68 | protocol: FI_PROTO_SOCK_TCP 69 | provider: sockets 70 | shared: false 71 | speed: 1073741824 72 | type: FI_EP_RDM 73 | version: '2.0' 74 | -------------------------------------------------------------------------------- /test/unit/config/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.10) 2 | project(hermes) 3 | 4 | set(CMAKE_CXX_STANDARD 17) 5 | 6 | #------------------------------------------------------------------------------ 7 | # Build Tests 8 | #------------------------------------------------------------------------------ 9 | 10 | add_executable(test_config_exec 11 | ${TEST_MAIN}/main_mpi.cc 12 | test_init.cc 13 | test_config.cc 14 | ) 15 | add_dependencies(test_config_exec 16 | ${Hermes_CLIENT_DEPS} hermes) 17 | target_link_libraries(test_config_exec 18 | ${Hermes_CLIENT_LIBRARIES} hermes Catch2::Catch2 MPI::MPI_CXX) 19 | jarvis_test(hermes test_hermes) 20 | 21 | #------------------------------------------------------------------------------ 22 | # Test Cases 23 | #------------------------------------------------------------------------------ 24 | 25 | # STRING TESTS 26 | #add_test(NAME test_ipc COMMAND 27 | # ${CMAKE_BINARY_DIR}/bin/test_messages "TestIpc") 28 | 29 | #------------------------------------------------------------------------------ 30 | # Install Targets 31 | #------------------------------------------------------------------------------ 32 | install(TARGETS 33 | test_config_exec 34 | EXPORT 35 | ${HERMES_EXPORTED_TARGETS} 36 | LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} 37 | ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} 38 | RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR}) 39 | 40 | #----------------------------------------------------------------------------- 41 | # Coverage 42 | #----------------------------------------------------------------------------- 43 | if(HERMES_ENABLE_COVERAGE) 44 | set_coverage_flags(test_config_exec) 45 | endif() 46 | -------------------------------------------------------------------------------- /test/unit/hermes/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.10) 2 | project(hermes) 3 | 4 | set(CMAKE_CXX_STANDARD 17) 5 | 6 | #------------------------------------------------------------------------------ 7 | # Build Tests 8 | #------------------------------------------------------------------------------ 9 | 10 | add_executable(test_hermes_exec 11 | ${TEST_MAIN}/main_mpi.cc 12 | test_init.cc 13 | test_bucket.cc 14 | ) 15 | add_dependencies(test_hermes_exec 16 | ${Hermes_CLIENT_DEPS} hermes) 17 | target_link_libraries(test_hermes_exec 18 | ${Hermes_CLIENT_LIBRARIES} hermes Catch2::Catch2 MPI::MPI_CXX) 19 | jarvis_test(hermes test_hermes) 20 | 21 | #------------------------------------------------------------------------------ 22 | # Test Cases 23 | #------------------------------------------------------------------------------ 24 | 25 | # STRING TESTS 26 | #add_test(NAME test_ipc COMMAND 27 | # ${CMAKE_BINARY_DIR}/bin/test_messages "TestIpc") 28 | 29 | #------------------------------------------------------------------------------ 30 | # Install Targets 31 | #------------------------------------------------------------------------------ 32 | install(TARGETS 33 | test_hermes_exec 34 | EXPORT 35 | ${HERMES_EXPORTED_TARGETS} 36 | LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} 37 | ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} 38 | RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR}) 39 | 40 | #----------------------------------------------------------------------------- 41 | # Coverage 42 | #----------------------------------------------------------------------------- 43 | if(HERMES_ENABLE_COVERAGE) 44 | set_coverage_flags(test_hermes_exec) 45 | endif() 46 | -------------------------------------------------------------------------------- /test/unit/ipc/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.10) 2 | project(hermes) 3 | 4 | set(CMAKE_CXX_STANDARD 17) 5 | 6 | #------------------------------------------------------------------------------ 7 | # Build Tests 8 | #------------------------------------------------------------------------------ 9 | 10 | add_executable(test_ipc_exec 11 | ${TEST_MAIN}/main_mpi.cc 12 | test_init.cc 13 | test_finalize.cc 14 | test_ipc.cc 15 | test_serialize.cc 16 | ) 17 | 18 | add_dependencies(test_ipc_exec ${Hermes_CLIENT_DEPS}) 19 | target_link_libraries(test_ipc_exec 20 | ${Hermes_CLIENT_LIBRARIES} Catch2::Catch2 MPI::MPI_CXX OpenMP::OpenMP_CXX) 21 | 22 | #------------------------------------------------------------------------------ 23 | # Test Cases 24 | #------------------------------------------------------------------------------ 25 | 26 | # STRING TESTS 27 | #add_test(NAME test_ipc COMMAND 28 | # ${CMAKE_BINARY_DIR}/bin/test_messages "TestIpc") 29 | 30 | #------------------------------------------------------------------------------ 31 | # Install Targets 32 | #------------------------------------------------------------------------------ 33 | install(TARGETS 34 | test_ipc_exec 35 | EXPORT 36 | ${HERMES_EXPORTED_TARGETS} 37 | LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} 38 | ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} 39 | RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR}) 40 | 41 | #----------------------------------------------------------------------------- 42 | # Coverage 43 | #----------------------------------------------------------------------------- 44 | if(HERMES_ENABLE_COVERAGE) 45 | set_coverage_flags(test_ipc_exec) 46 | endif() 47 | -------------------------------------------------------------------------------- /test/unit/external/external.cc: -------------------------------------------------------------------------------- 1 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 | * Distributed under BSD 3-Clause license. * 3 | * Copyright by The HDF Group. * 4 | * Copyright by the Illinois Institute of Technology. * 5 | * All rights reserved. * 6 | * * 7 | * This file is part of Hermes. The full Hermes copyright notice, including * 8 | * terms governing use, modification, and redistribution, is contained in * 9 | * the COPYING file, which can be found at the top directory. If you do not * 10 | * have access to the file, you may request a copy from help@hdfgroup.org. * 11 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 12 | 13 | #define CATCH_CONFIG_RUNNER 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include "hermes/hermes.h" 19 | 20 | namespace cl = Catch::Clara; 21 | cl::Parser define_options(); 22 | 23 | int main(int argc, char **argv) { 24 | int rc; 25 | MPI_Init(&argc, &argv); 26 | Catch::Session session; 27 | auto cli = session.cli(); 28 | session.cli(cli); 29 | rc = session.applyCommandLine(argc, argv); 30 | if (rc != 0) return rc; 31 | rc = session.run(); 32 | if (rc != 0) return rc; 33 | MPI_Finalize(); 34 | return rc; 35 | } 36 | 37 | TEST_CASE("TestHermesConnect") { 38 | int rank, nprocs; 39 | MPI_Barrier(MPI_COMM_WORLD); 40 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 41 | MPI_Comm_size(MPI_COMM_WORLD, &nprocs); 42 | HERMES->ClientInit(); 43 | MPI_Barrier(MPI_COMM_WORLD); 44 | } 45 | -------------------------------------------------------------------------------- /include/hermes/dpe/dpe.h: -------------------------------------------------------------------------------- 1 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 | * Distributed under BSD 3-Clause license. * 3 | * Copyright by The HDF Group. * 4 | * Copyright by the Illinois Institute of Technology. * 5 | * All rights reserved. * 6 | * * 7 | * This file is part of Hermes. The full Hermes copyright notice, including * 8 | * terms governing use, modification, and redistribution, is contained in * 9 | * the COPYING file, which can be found at the top directory. If you do not * 10 | * have access to the file, you may request a copy from help@hdfgroup.org. * 11 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 12 | 13 | #ifndef HERMES_DATA_PLACEMENT_ENGINE_H_ 14 | #define HERMES_DATA_PLACEMENT_ENGINE_H_ 15 | 16 | #include 17 | 18 | #include "hermes/hermes_types.h" 19 | #include "hermes_mdm/hermes_mdm.h" 20 | #include "bdev/bdev.h" 21 | 22 | namespace hermes { 23 | 24 | /** 25 | A class to represent data placement engine 26 | */ 27 | class Dpe { 28 | public: 29 | /** Constructor. */ 30 | Dpe() = default; 31 | 32 | /** Destructor. */ 33 | virtual ~Dpe() = default; 34 | 35 | /** 36 | * Calculate the placement of a set of blobs using a particular 37 | * algorithm given a context. 38 | * */ 39 | virtual Status Placement(const std::vector &blob_sizes, 40 | std::vector &targets, 41 | Context &ctx, 42 | std::vector &output) = 0; 43 | }; 44 | 45 | } // namespace hermes 46 | #endif // HERMES_DATA_PLACEMENT_ENGINE_H_ 47 | -------------------------------------------------------------------------------- /hrun/include/hrun/queue_manager/queue_manager_client.h: -------------------------------------------------------------------------------- 1 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 | * Distributed under BSD 3-Clause license. * 3 | * Copyright by The HDF Group. * 4 | * Copyright by the Illinois Institute of Technology. * 5 | * All rights reserved. * 6 | * * 7 | * This file is part of Hermes. The full Hermes copyright notice, including * 8 | * terms governing use, modification, and redistribution, is contained in * 9 | * the COPYING file, which can be found at the top directory. If you do not * 10 | * have access to the file, you may request a copy from help@hdfgroup.org. * 11 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 12 | 13 | #ifndef HRUN_INCLUDE_HRUN_QUEUE_MANAGER_QUEUE_MANAGER_CLIENT_H_ 14 | #define HRUN_INCLUDE_HRUN_QUEUE_MANAGER_QUEUE_MANAGER_CLIENT_H_ 15 | 16 | #include "queue_manager.h" 17 | 18 | namespace hrun { 19 | 20 | #define HRUN_QM_CLIENT \ 21 | (&HRUN_CLIENT->queue_manager_) 22 | 23 | /** Enable client programs to access queues */ 24 | class QueueManagerClient : public QueueManager { 25 | public: 26 | hipc::Allocator *alloc_; 27 | 28 | public: 29 | /** Default constructor */ 30 | QueueManagerClient() = default; 31 | 32 | /** Destructor*/ 33 | ~QueueManagerClient() = default; 34 | 35 | /** Initialize client */ 36 | void ClientInit(hipc::Allocator *alloc, QueueManagerShm &shm, u32 node_id) { 37 | alloc_ = alloc; 38 | queue_map_ = shm.queue_map_.get(); 39 | Init(node_id); 40 | } 41 | }; 42 | 43 | } // namespace hrun 44 | 45 | #endif // HRUN_INCLUDE_HRUN_QUEUE_MANAGER_QUEUE_MANAGER_CLIENT_H_ 46 | -------------------------------------------------------------------------------- /test/unit/hermes_adapters/stdio/stdio_adapter_shared_test.cc: -------------------------------------------------------------------------------- 1 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 | * Distributed under BSD 3-Clause license. * 3 | * Copyright by The HDF Group. * 4 | * Copyright by the Illinois Institute of Technology. * 5 | * All rights reserved. * 6 | * * 7 | * This file is part of Hermes. The full Hermes copyright notice, including * 8 | * terms governing use, modification, and redistribution, is contained in * 9 | * the COPYING file, which can be found at the top directory. If you do not * 10 | * have access to the file, you may request a copy from help@hdfgroup.org. * 11 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 12 | 13 | #include "stdio_adapter_test.h" 14 | 15 | TEST_CASE("SharedSTDIORead", "[process=" + std::to_string(TESTER->comm_size_) + 16 | "]" 17 | "[operation=batched_read]" 18 | "[request_size=type-fixed][repetition=" + 19 | std::to_string(TESTER->num_iterations_) + 20 | "]" 21 | "[mode=shared]" 22 | "[pattern=sequential][file=1]") { 23 | TESTER->Pretest(); 24 | 25 | SECTION("read from existing file") { 26 | TESTER->test_fopen(TESTER->shared_existing_file_, "r+"); 27 | REQUIRE(TESTER->fh_orig_ != nullptr); 28 | std::string data(TESTER->request_size_, '1'); 29 | for (size_t i = 0; i < TESTER->num_iterations_; ++i) { 30 | TESTER->test_fread(data.data(), TESTER->request_size_); 31 | REQUIRE(TESTER->size_read_orig_ == TESTER->request_size_); 32 | } 33 | TESTER->test_fclose(); 34 | REQUIRE(TESTER->status_orig_ == 0); 35 | } 36 | TESTER->Posttest(); 37 | } 38 | -------------------------------------------------------------------------------- /hrun/include/hrun/api/template/hrun_task_node_push_root.template: -------------------------------------------------------------------------------- 1 | template 2 | hipc::LPointer Async##CUSTOM##Alloc(const TaskNode &task_node, 3 | Args&& ...args) { 4 | hipc::LPointer task = HRUN_CLIENT->AllocateTask(); 5 | Async##CUSTOM##Construct(task.ptr_, task_node, std::forward(args)...); 6 | return task; 7 | } 8 | template 9 | hipc::LPointer Async##CUSTOM(const TaskNode &task_node, 10 | Args&& ...args) { 11 | hipc::LPointer task = Async##CUSTOM##Alloc(task_node, std::forward(args)...); 12 | MultiQueue *queue = HRUN_CLIENT->GetQueue(queue_id_); 13 | queue->Emplace(task.ptr_->prio_, task.ptr_->lane_hash_, task.shm_); 14 | return task; 15 | } 16 | template 17 | hipc::LPointer Async##CUSTOM##Emplace(MultiQueue *queue, 18 | const TaskNode &task_node, 19 | Args&& ...args) { 20 | hipc::LPointer task = Async##CUSTOM##Alloc(task_node, std::forward(args)...); 21 | queue->Emplace(task.ptr_->prio_, task.ptr_->lane_hash_, task.shm_); 22 | return task; 23 | } 24 | template 25 | hipc::LPointer> Async##CUSTOM##Root(Args&& ...args) { 26 | TaskNode task_node = HRUN_CLIENT->MakeTaskNodeId(); 27 | hipc::LPointer task = Async##CUSTOM##Alloc(task_node + 1, std::forward(args)...); 28 | hipc::LPointer> push_task = 29 | HRUN_PROCESS_QUEUE->AsyncPush(task_node, 30 | DomainId::GetLocal(), 31 | task); 32 | return push_task; 33 | } -------------------------------------------------------------------------------- /.github/workflows/docs.yml: -------------------------------------------------------------------------------- 1 | name: doxygen 2 | on: 3 | push: 4 | branches: [ master ] 5 | workflow_dispatch: 6 | 7 | env: 8 | # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.) 9 | BUILD_TYPE: Debug 10 | LOCAL: local 11 | 12 | jobs: 13 | # This workflow contains a single job called "build" 14 | build: 15 | # The type of runner that the job will run on 16 | runs-on: ubuntu-latest 17 | steps: 18 | - name: Get Sources 19 | uses: actions/checkout@v2 20 | 21 | - name: Setup python 22 | uses: actions/setup-python@v4 23 | 24 | - name: Cache Spack packages 25 | uses: actions/cache@v2 26 | id: spack-cache 27 | with: 28 | path: ~/${{ env.LOCAL }} 29 | key: ${{ runner.os }}-${{ hashFiles('ci/**') }} 30 | 31 | - name: Install APT Dependencies 32 | run: | 33 | sudo apt update -y 34 | sudo apt install -y libunwind-dev 35 | sudo apt-get install -y autoconf 36 | sudo apt-get install -y automake 37 | sudo apt-get install -y libtool 38 | sudo apt-get install -y libtool-bin 39 | sudo apt-get install -y mpich 40 | sudo apt-get install -y lcov 41 | sudo apt-get install -y zlib1g-dev 42 | sudo apt-get install -y libsdl2-dev 43 | sudo apt-get install -y graphviz 44 | sudo apt-get install -y --no-install-recommends doxygen 45 | 46 | - name: Build And Install Dependencies 47 | if: steps.spack-cache.outputs.cache-hit != 'true' 48 | run: ci/install_deps.sh 49 | 50 | - name: Build and Test 51 | run: ci/install_docs.sh 52 | 53 | - name: Deploy 54 | uses: peaceiris/actions-gh-pages@v3 55 | with: 56 | publish_dir: ./build/_build/html 57 | destination_dir: html 58 | github_token: ${{ secrets.GITHUB_TOKEN }} 59 | -------------------------------------------------------------------------------- /hermes_adapters/mapper/mapper_factory.h: -------------------------------------------------------------------------------- 1 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 | * Distributed under BSD 3-Clause license. * 3 | * Copyright by The HDF Group. * 4 | * Copyright by the Illinois Institute of Technology. * 5 | * All rights reserved. * 6 | * * 7 | * This file is part of Hermes. The full Hermes copyright notice, including * 8 | * terms governing use, modification, and redistribution, is contained in * 9 | * the COPYING file, which can be found at the top directory. If you do not * 10 | * have access to the file, you may request a copy from help@hdfgroup.org. * 11 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 12 | 13 | #ifndef HERMES_ADAPTER_FACTORY_H 14 | #define HERMES_ADAPTER_FACTORY_H 15 | 16 | #include "abstract_mapper.h" 17 | #include "balanced_mapper.h" 18 | #include "hermes_shm/util/singleton.h" 19 | 20 | namespace hermes::adapter { 21 | /** 22 | A class to represent mapper factory pattern 23 | */ 24 | class MapperFactory { 25 | public: 26 | /** 27 | * Return the instance of mapper given a type. Uses factory pattern. 28 | * 29 | * @param[in] type type of mapper to be used by the POSIX adapter. 30 | * @return Instance of mapper given a type. 31 | */ 32 | static AbstractMapper* Get(const MapperType& type) { 33 | switch (type) { 34 | case MapperType::kBalancedMapper: { 35 | return hshm::EasySingleton::GetInstance(); 36 | } 37 | default: { 38 | // TODO(llogan): @error_handling Mapper not implemented 39 | } 40 | } 41 | return NULL; 42 | } 43 | }; 44 | } // namespace hermes::adapter 45 | #endif // HERMES_ADAPTER_FACTORY_H 46 | -------------------------------------------------------------------------------- /hrun/include/hrun/hrun_namespace.h: -------------------------------------------------------------------------------- 1 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 | * Distributed under BSD 3-Clause license. * 3 | * Copyright by The HDF Group. * 4 | * Copyright by the Illinois Institute of Technology. * 5 | * All rights reserved. * 6 | * * 7 | * This file is part of Hermes. The full Hermes copyright notice, including * 8 | * terms governing use, modification, and redistribution, is contained in * 9 | * the COPYING file, which can be found at the top directory. If you do not * 10 | * have access to the file, you may request a copy from help@hdfgroup.org. * 11 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 12 | 13 | #ifndef HRUN_INCLUDE_HRUN_HRUN_NAMESPACE_H_ 14 | #define HRUN_INCLUDE_HRUN_HRUN_NAMESPACE_H_ 15 | 16 | #include "hrun/api/hrun_client.h" 17 | #include "hrun/task_registry/task_lib.h" 18 | 19 | using hrun::TaskMethod; 20 | using hrun::BinaryOutputArchive; 21 | using hrun::BinaryInputArchive; 22 | using hrun::Task; 23 | using hrun::TaskPointer; 24 | using hrun::MultiQueue; 25 | using hrun::PriorityInfo; 26 | using hrun::TaskNode; 27 | using hrun::DomainId; 28 | using hrun::TaskStateId; 29 | using hrun::QueueId; 30 | using hrun::TaskFlags; 31 | using hrun::DataTransfer; 32 | using hrun::TaskLib; 33 | using hrun::TaskLibClient; 34 | using hrun::config::QueueManagerInfo; 35 | using hrun::TaskPrio; 36 | using hrun::RunContext; 37 | 38 | using hshm::RwLock; 39 | using hshm::Mutex; 40 | using hshm::bitfield; 41 | using hshm::bitfield32_t; 42 | typedef hshm::bitfield bitfield64_t; 43 | using hshm::ScopedRwReadLock; 44 | using hshm::ScopedRwWriteLock; 45 | using hipc::LPointer; 46 | 47 | #endif // HRUN_INCLUDE_HRUN_HRUN_NAMESPACE_H_ 48 | -------------------------------------------------------------------------------- /test/unit/hermes_adapters/mpiio/parallel.cc: -------------------------------------------------------------------------------- 1 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 | * Distributed under BSD 3-Clause license. * 3 | * Copyright by The HDF Group. * 4 | * Copyright by the Illinois Institute of Technology. * 5 | * All rights reserved. * 6 | * * 7 | * This file is part of Hermes. The full Hermes copyright notice, including * 8 | * terms governing use, modification, and redistribution, is contained in * 9 | * the COPYING file, which can be found at the top directory. If you do not * 10 | * have access to the file, you may request a copy from help@hdfgroup.org. * 11 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 12 | 13 | #include "mpi.h" 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | namespace stdfs = std::filesystem; 20 | 21 | int main(int argc, char **argv) { 22 | MPI_File f; 23 | MPI_Status status; 24 | int count = 1024 * 1024 / 8; 25 | int rank, nprocs; 26 | MPI_Init(&argc, &argv); 27 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 28 | MPI_Comm_size(MPI_COMM_WORLD, &nprocs); 29 | std::string path = argv[1]; 30 | std::vector buf(count, rank); 31 | if (rank == 0) { 32 | FILE *fp = fopen(path.c_str(), "w"); 33 | std::vector init(count*nprocs, -1); 34 | fwrite(init.data(), 1, count*nprocs, fp); 35 | fclose(fp); 36 | } 37 | MPI_Barrier(MPI_COMM_WORLD); 38 | 39 | MPI_File_open(MPI_COMM_WORLD, path.c_str(), MPI_MODE_WRONLY | MPI_MODE_CREATE, 40 | MPI_INFO_NULL, &f); 41 | MPI_File_write_at(f, rank*count, buf.data(), count, 42 | MPI_CHAR, &status); 43 | MPI_File_sync(f); 44 | MPI_File_close(&f); 45 | MPI_Finalize(); 46 | } 47 | -------------------------------------------------------------------------------- /test/unit/basic_test.h: -------------------------------------------------------------------------------- 1 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 | * Distributed under BSD 3-Clause license. * 3 | * Copyright by The HDF Group. * 4 | * Copyright by the Illinois Institute of Technology. * 5 | * All rights reserved. * 6 | * * 7 | * This file is part of Hermes. The full Hermes copyright notice, including * 8 | * terms governing use, modification, and redistribution, is contained in * 9 | * the COPYING file, which can be found at the top directory. If you do not * 10 | * have access to the file, you may request a copy from help@hdfgroup.org. * 11 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 12 | 13 | #ifndef HRUN_TEST_UNIT_BASIC_TEST_H_ 14 | #define HRUN_TEST_UNIT_BASIC_TEST_H_ 15 | 16 | #define CATCH_CONFIG_RUNNER 17 | #include 18 | 19 | namespace cl = Catch::Clara; 20 | cl::Parser define_options(); 21 | 22 | #include 23 | #include 24 | 25 | static inline bool VerifyBuffer(char *ptr, size_t size, char nonce) { 26 | for (size_t i = 0; i < size; ++i) { 27 | if (ptr[i] != nonce) { 28 | std::cout << (int)ptr[i] << std::endl; 29 | return false; 30 | } 31 | } 32 | return true; 33 | } 34 | 35 | static inline bool CompareBuffers(char *p1, size_t s1, char *p2, size_t s2, 36 | size_t off) { 37 | if (s1 != s2) { 38 | return false; 39 | } 40 | for (size_t i = off; i < s1; ++i) { 41 | if (p1[i] != p2[i]) { 42 | std::cout << "Mismatch at: " << (int)i << std::endl; 43 | return false; 44 | } 45 | } 46 | return true; 47 | } 48 | 49 | void MainPretest(); 50 | void MainPosttest(); 51 | 52 | #define PAGE_DIVIDE(TEXT) 53 | 54 | #endif // HRUN_TEST_UNIT_BASIC_TEST_H_ 55 | -------------------------------------------------------------------------------- /hermes_adapters/mpiio/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | include_directories( 2 | ${CMAKE_SOURCE_DIR} 3 | ${HERMES_SRC_DIR} 4 | ${HERMES_ADAPTER_DIR} 5 | ${HERMES_IO_CLIENT_DIR} 6 | .) 7 | 8 | # Create the MPIIO interceptor 9 | if (HERMES_MPICH) 10 | message(STATUS "Using HERMES_MPICH") 11 | add_definitions(-DHERMES_MPICH) 12 | elseif(HERMES_OPENMPI) 13 | message(STATUS "Using HERMES_OPENMPI") 14 | add_definitions(-DHERMES_OPENMPI) 15 | endif() 16 | set(INTERCEPTOR_DEPS 17 | hermes hermes_fs_base) 18 | add_library(hermes_mpiio SHARED 19 | ${CMAKE_CURRENT_SOURCE_DIR}/mpiio_api.cc) 20 | add_dependencies(hermes_mpiio ${INTERCEPTOR_DEPS}) 21 | target_link_libraries(hermes_mpiio 22 | MPI::MPI_CXX stdc++fs ${libelf_LIBRARIES} dl ${INTERCEPTOR_DEPS}) 23 | 24 | #----------------------------------------------------------------------------- 25 | # Add Target(s) to CMake Install 26 | #----------------------------------------------------------------------------- 27 | install( 28 | TARGETS 29 | hermes_mpiio 30 | EXPORT 31 | ${HERMES_EXPORTED_TARGETS} 32 | LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} 33 | ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} 34 | RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} 35 | ) 36 | 37 | #----------------------------------------------------------------------------- 38 | # Install headers 39 | #----------------------------------------------------------------------------- 40 | file(GLOB_RECURSE HERMES_HEADERS "*.h") 41 | install( 42 | FILES 43 | ${HERMES_HEADERS} 44 | DESTINATION 45 | ${CMAKE_INSTALL_PREFIX}/include/hermes_adapters/stdio 46 | COMPONENT 47 | headers 48 | ) 49 | 50 | #----------------------------------------------------------------------------- 51 | # Add Target(s) to Coverage 52 | #----------------------------------------------------------------------------- 53 | if(HERMES_ENABLE_COVERAGE) 54 | set_coverage_flags(hermes_mpiio) 55 | endif() -------------------------------------------------------------------------------- /hrun/tasks_required/TASK_NAME/src/TASK_NAME.cc: -------------------------------------------------------------------------------- 1 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 | * Distributed under BSD 3-Clause license. * 3 | * Copyright by The HDF Group. * 4 | * Copyright by the Illinois Institute of Technology. * 5 | * All rights reserved. * 6 | * * 7 | * This file is part of Hermes. The full Hermes copyright notice, including * 8 | * terms governing use, modification, and redistribution, is contained in * 9 | * the COPYING file, which can be found at the top directory. If you do not * 10 | * have access to the file, you may request a copy from help@hdfgroup.org. * 11 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 12 | 13 | #include "hrun_admin/hrun_admin.h" 14 | #include "hrun/api/hrun_runtime.h" 15 | #include "TASK_NAME/TASK_NAME.h" 16 | 17 | namespace hrun::TASK_NAME { 18 | 19 | class Server : public TaskLib { 20 | public: 21 | Server() = default; 22 | 23 | /** Construct TASK_NAME */ 24 | void Construct(ConstructTask *task, RunContext &rctx) { 25 | task->SetModuleComplete(); 26 | } 27 | void MonitorConstruct(u32 mode, ConstructTask *task, RunContext &rctx) { 28 | } 29 | 30 | /** Destroy TASK_NAME */ 31 | void Destruct(DestructTask *task, RunContext &rctx) { 32 | task->SetModuleComplete(); 33 | } 34 | void MonitorDestruct(u32 mode, DestructTask *task, RunContext &rctx) { 35 | } 36 | 37 | /** A custom method */ 38 | void Custom(CustomTask *task, RunContext &rctx) { 39 | task->SetModuleComplete(); 40 | } 41 | void MonitorCustom(u32 mode, CustomTask *task, RunContext &rctx) { 42 | } 43 | public: 44 | #include "TASK_NAME/TASK_NAME_lib_exec.h" 45 | }; 46 | 47 | } // namespace hrun::TASK_NAME 48 | 49 | HRUN_TASK_CC(hrun::TASK_NAME::Server, "TASK_NAME"); 50 | -------------------------------------------------------------------------------- /ci/build_hermes.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # ARGS: 4 | # SPACK_DIR: the path to spack 5 | 6 | # Ensure environment modules are loaded 7 | . /hermes/ci/module_load.sh 8 | 9 | # THIS SCRIPT IS EXECUTED BY CONTAINER!!! 10 | set -x 11 | set -e 12 | set -o pipefail 13 | 14 | # Update jarvis-cd 15 | pushd jarvis-cd 16 | git pull 17 | pip install -e . -r requirements.txt 18 | popd 19 | 20 | # Update scspkg 21 | pushd scspkg 22 | git pull 23 | pip install -e . -r requirements.txt 24 | popd 25 | 26 | # Load scspkg environment 27 | module use "$(scspkg module dir)" 28 | 29 | # Load hermes_shm 30 | . "${SPACK_DIR}/share/spack/setup-env.sh" 31 | spack module tcl refresh --delete-tree -y 32 | spack load hermes_shm 33 | # module use "${SPACK_DIR}/share/spack/modules/linux-ubuntu22.04-zen2" 34 | 35 | # Create Hermes module 36 | scspkg create hermes 37 | scspkg env prepend hermes PATH /hermes/build/bin 38 | scspkg env prepend hermes LIBRARY_PATH /hermes/build/bin 39 | scspkg env prepend hermes LD_LIBRARY_PATH /hermes/build/bin 40 | module load hermes 41 | 42 | # Initialize the Jarvis testing Hermes environment 43 | jarvis init \ 44 | "${HOME}/jarvis-config" \ 45 | "${HOME}/jarvis-priv" \ 46 | "${HOME}/jarvis-shared" 47 | cp /hermes/ci/resource_graph.yaml /jarvis-cd/config/resource_graph.yaml 48 | jarvis env build hermes 49 | 50 | # CD into Hermes directory in container 51 | cd /hermes 52 | git config --global --add safe.directory '*' 53 | git submodule update --init 54 | 55 | # Build Hermes 56 | mkdir -p build 57 | cd build 58 | cmake ../ \ 59 | -DCMAKE_BUILD_TYPE=Debug \ 60 | -DCMAKE_INSTALL_PREFIX="$(scspkg pkg root hermes)" \ 61 | -DHERMES_ENABLE_MPIIO_ADAPTER=ON \ 62 | -DHERMES_MPICH=ON \ 63 | -DHERMES_ENABLE_STDIO_ADAPTER=ON \ 64 | -DHERMES_ENABLE_POSIX_ADAPTER=ON \ 65 | -DHERMES_ENABLE_COVERAGE=ON 66 | make -j4 67 | make install 68 | 69 | # Test Hermes 70 | export CXXFLAGS=-Wall 71 | ctest -D Experimental 72 | 73 | # Run make install unit test 74 | cd /hermes/test/unit/external 75 | mkdir -p build 76 | cd build 77 | cmake ../ 78 | make -j4 79 | -------------------------------------------------------------------------------- /test/unit/config/test_config.cc: -------------------------------------------------------------------------------- 1 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 | * Distributed under BSD 3-Clause license. * 3 | * Copyright by The HDF Group. * 4 | * Copyright by the Illinois Institute of Technology. * 5 | * All rights reserved. * 6 | * * 7 | * This file is part of Hermes. The full Hermes copyright notice, including * 8 | * terms governing use, modification, and redistribution, is contained in * 9 | * the COPYING file, which can be found at the top directory. If you do not * 10 | * have access to the file, you may request a copy from help@hdfgroup.org. * 11 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 12 | 13 | #include "basic_test.h" 14 | #include "hrun/api/hrun_client.h" 15 | #include "hrun_admin/hrun_admin.h" 16 | #include "hermes/hermes.h" 17 | #include "hermes/bucket.h" 18 | #include "data_stager/factory/binary_stager.h" 19 | #include 20 | 21 | TEST_CASE("TestHermesPaths") { 22 | PAGE_DIVIDE("Directory path") { 23 | hermes::config::UserPathInfo info("/home/hello", true, true); 24 | REQUIRE(info.Match("/home") == false); 25 | REQUIRE(info.Match("/home/hello") == true); 26 | REQUIRE(info.Match("/home/hello/hi.txt") == true); 27 | } 28 | 29 | PAGE_DIVIDE("Simple path") { 30 | hermes::config::UserPathInfo info("/home/hello.txt", true, false); 31 | REQUIRE(info.Match("/home/hello") == false); 32 | REQUIRE(info.Match("/home/hello.txt") == true); 33 | } 34 | 35 | PAGE_DIVIDE("Wildcard path") { 36 | hermes::config::UserPathInfo info("/home/hello/*.json", true, false); 37 | REQUIRE(info.Match("/home/hello") == false); 38 | REQUIRE(info.Match("/home/hello/hi.json") == true); 39 | REQUIRE(info.Match("/home/hello/.json") == true); 40 | } 41 | } 42 | 43 | -------------------------------------------------------------------------------- /src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | #------------------------------------------------------------------------------ 2 | # Build Hermes 3 | #------------------------------------------------------------------------------ 4 | add_library(hermes SHARED 5 | hermes_config_manager.cc) 6 | add_dependencies(hermes ${Hermes_RUNTIME_DEPS}) 7 | target_link_libraries(hermes ${Hermes_RUNTIME_LIBRARIES}) 8 | 9 | #------------------------------------------------------------------------------ 10 | # Install Hermes Library 11 | #------------------------------------------------------------------------------ 12 | install( 13 | TARGETS 14 | hermes 15 | EXPORT 16 | ${HERMES_EXPORTED_TARGETS} 17 | LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} 18 | ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} 19 | RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} 20 | ) 21 | 22 | #----------------------------------------------------------------------------- 23 | # Add Target(s) to CMake Install for import into other projects 24 | #----------------------------------------------------------------------------- 25 | install( 26 | EXPORT 27 | ${HERMES_EXPORTED_TARGETS} 28 | DESTINATION 29 | ${HERMES_INSTALL_DATA_DIR}/cmake/hermes 30 | FILE 31 | ${HERMES_EXPORTED_TARGETS}.cmake 32 | ) 33 | 34 | #----------------------------------------------------------------------------- 35 | # Export all exported targets to the build tree for use by parent project 36 | #----------------------------------------------------------------------------- 37 | set(HERMES_EXPORTED_LIBS 38 | hermes 39 | ${HERMES_EXPORTED_LIBS}) 40 | if(NOT HERMES_EXTERNALLY_CONFIGURED) 41 | EXPORT ( 42 | TARGETS 43 | ${HERMES_EXPORTED_LIBS} 44 | FILE 45 | ${HERMES_EXPORTED_TARGETS}.cmake 46 | ) 47 | endif() 48 | 49 | #------------------------------------------------------------------------------ 50 | # Coverage 51 | #------------------------------------------------------------------------------ 52 | if(HERMES_ENABLE_COVERAGE) 53 | set_coverage_flags(hermes) 54 | endif() 55 | -------------------------------------------------------------------------------- /hermes_adapters/mapper/balanced_mapper.h: -------------------------------------------------------------------------------- 1 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 | * Distributed under BSD 3-Clause license. * 3 | * Copyright by The HDF Group. * 4 | * Copyright by the Illinois Institute of Technology. * 5 | * All rights reserved. * 6 | * * 7 | * This file is part of Hermes. The full Hermes copyright notice, including * 8 | * terms governing use, modification, and redistribution, is contained in * 9 | * the COPYING file, which can be found at the top directory. If you do not * 10 | * have access to the file, you may request a copy from help@hdfgroup.org. * 11 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 12 | 13 | #ifndef HERMES_BALANCED_MAPPER_H 14 | #define HERMES_BALANCED_MAPPER_H 15 | 16 | #include 17 | 18 | #include "abstract_mapper.h" 19 | 20 | namespace hermes::adapter { 21 | /** 22 | * Implement balanced mapping 23 | */ 24 | class BalancedMapper : public AbstractMapper { 25 | public: 26 | /** Virtual destructor */ 27 | virtual ~BalancedMapper() = default; 28 | 29 | /** Divides an I/O size evenly by into units of page_size */ 30 | void map(size_t off, size_t size, size_t page_size, 31 | BlobPlacements &ps) override { 32 | size_t kPageSize = page_size; 33 | size_t size_mapped = 0; 34 | while (size > size_mapped) { 35 | BlobPlacement p; 36 | p.bucket_off_ = off + size_mapped; 37 | p.page_ = p.bucket_off_ / kPageSize; 38 | p.blob_off_ = p.bucket_off_ % kPageSize; 39 | auto left_size_page = kPageSize - p.blob_off_; 40 | p.blob_size_ = left_size_page < size - size_mapped ? left_size_page 41 | : size - size_mapped; 42 | ps.emplace_back(p); 43 | size_mapped += p.blob_size_; 44 | } 45 | } 46 | }; 47 | } // namespace hermes::adapter 48 | 49 | #endif // HERMES_BALANCED_MAPPER_H 50 | -------------------------------------------------------------------------------- /hrun/tasks_required/worch_proc_round_robin/src/worch_proc_round_robin.cc: -------------------------------------------------------------------------------- 1 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 | * Distributed under BSD 3-Clause license. * 3 | * Copyright by The HDF Group. * 4 | * Copyright by the Illinois Institute of Technology. * 5 | * All rights reserved. * 6 | * * 7 | * This file is part of Hermes. The full Hermes copyright notice, including * 8 | * terms governing use, modification, and redistribution, is contained in * 9 | * the COPYING file, which can be found at the top directory. If you do not * 10 | * have access to the file, you may request a copy from help@hdfgroup.org. * 11 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 12 | 13 | #include "hrun_admin/hrun_admin.h" 14 | #include "hrun/api/hrun_runtime.h" 15 | #include "worch_proc_round_robin/worch_proc_round_robin.h" 16 | 17 | namespace hrun::worch_proc_round_robin { 18 | 19 | class Server : public TaskLib { 20 | public: 21 | /** Construct the work orchestrator process scheduler */ 22 | void Construct(ConstructTask *task, RunContext &rctx) { 23 | task->SetModuleComplete(); 24 | } 25 | void MonitorConstruct(u32 mode, ConstructTask *task, RunContext &rctx) { 26 | } 27 | 28 | /** Destroy the work orchestrator process queue */ 29 | void Destruct(DestructTask *task, RunContext &rctx) { 30 | task->SetModuleComplete(); 31 | } 32 | void MonitorDestruct(u32 mode, DestructTask *task, RunContext &rctx) { 33 | } 34 | 35 | /** Schedule running processes */ 36 | void Schedule(ScheduleTask *task, RunContext &rctx) { 37 | HRUN_WORK_ORCHESTRATOR->DedicateCores(); 38 | } 39 | void MonitorSchedule(u32 mode, ScheduleTask *task, RunContext &rctx) { 40 | } 41 | 42 | #include "worch_proc_round_robin/worch_proc_round_robin_lib_exec.h" 43 | }; 44 | 45 | } // namespace hrun 46 | 47 | HRUN_TASK_CC(hrun::worch_proc_round_robin::Server, "worch_proc_round_robin"); 48 | -------------------------------------------------------------------------------- /tasks/ram_bdev/src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | #------------------------------------------------------------------------------ 2 | # Build Small Message Task Library 3 | #------------------------------------------------------------------------------ 4 | add_library(ram_bdev SHARED 5 | ram_bdev.cc) 6 | add_dependencies(ram_bdev ${Hermes_RUNTIME_DEPS}) 7 | target_link_libraries(ram_bdev ${Hermes_RUNTIME_LIBRARIES}) 8 | 9 | #------------------------------------------------------------------------------ 10 | # Install Small Message Task Library 11 | #------------------------------------------------------------------------------ 12 | install( 13 | TARGETS 14 | ram_bdev 15 | EXPORT 16 | ${HERMES_EXPORTED_TARGETS} 17 | LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} 18 | ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} 19 | RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} 20 | ) 21 | 22 | #----------------------------------------------------------------------------- 23 | # Add Target(s) to CMake Install for import into other projects 24 | #----------------------------------------------------------------------------- 25 | install( 26 | EXPORT 27 | ${HERMES_EXPORTED_TARGETS} 28 | DESTINATION 29 | ${HERMES_INSTALL_DATA_DIR}/cmake/hermes 30 | FILE 31 | ${HERMES_EXPORTED_TARGETS}.cmake 32 | ) 33 | 34 | #----------------------------------------------------------------------------- 35 | # Export all exported targets to the build tree for use by parent project 36 | #----------------------------------------------------------------------------- 37 | set(HERMES_EXPORTED_LIBS 38 | ram_bdev 39 | ${HERMES_EXPORTED_LIBS}) 40 | if(NOT HERMES_EXTERNALLY_CONFIGURED) 41 | EXPORT ( 42 | TARGETS 43 | ${HERMES_EXPORTED_LIBS} 44 | FILE 45 | ${HERMES_EXPORTED_TARGETS}.cmake 46 | ) 47 | endif() 48 | 49 | #------------------------------------------------------------------------------ 50 | # Coverage 51 | #------------------------------------------------------------------------------ 52 | if(HERMES_ENABLE_COVERAGE) 53 | set_coverage_flags(ram_bdev) 54 | endif() 55 | -------------------------------------------------------------------------------- /tasks/posix_bdev/src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | #------------------------------------------------------------------------------ 2 | # Build Small Message Task Library 3 | #------------------------------------------------------------------------------ 4 | add_library(posix_bdev SHARED 5 | posix_bdev.cc) 6 | add_dependencies(posix_bdev ${Hermes_RUNTIME_DEPS}) 7 | target_link_libraries(posix_bdev ${Hermes_RUNTIME_LIBRARIES}) 8 | 9 | #------------------------------------------------------------------------------ 10 | # Install Small Message Task Library 11 | #------------------------------------------------------------------------------ 12 | install( 13 | TARGETS 14 | posix_bdev 15 | EXPORT 16 | ${HERMES_EXPORTED_TARGETS} 17 | LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} 18 | ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} 19 | RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} 20 | ) 21 | 22 | #----------------------------------------------------------------------------- 23 | # Add Target(s) to CMake Install for import into other projects 24 | #----------------------------------------------------------------------------- 25 | install( 26 | EXPORT 27 | ${HERMES_EXPORTED_TARGETS} 28 | DESTINATION 29 | ${HERMES_INSTALL_DATA_DIR}/cmake/hermes 30 | FILE 31 | ${HERMES_EXPORTED_TARGETS}.cmake 32 | ) 33 | 34 | #----------------------------------------------------------------------------- 35 | # Export all exported targets to the build tree for use by parent project 36 | #----------------------------------------------------------------------------- 37 | set(HERMES_EXPORTED_LIBS 38 | posix_bdev 39 | ${HERMES_EXPORTED_LIBS}) 40 | if(NOT HERMES_EXTERNALLY_CONFIGURED) 41 | EXPORT ( 42 | TARGETS 43 | ${HERMES_EXPORTED_LIBS} 44 | FILE 45 | ${HERMES_EXPORTED_TARGETS}.cmake 46 | ) 47 | endif() 48 | 49 | #------------------------------------------------------------------------------ 50 | # Coverage 51 | #------------------------------------------------------------------------------ 52 | if(HERMES_ENABLE_COVERAGE) 53 | set_coverage_flags(posix_bdev) 54 | endif() 55 | -------------------------------------------------------------------------------- /hrun/tasks_required/TASK_NAME/src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | #------------------------------------------------------------------------------ 2 | # Build Small Message Task Library 3 | #------------------------------------------------------------------------------ 4 | add_library(TASK_NAME SHARED 5 | TASK_NAME.cc) 6 | add_dependencies(TASK_NAME ${Hermes_RUNTIME_DEPS}) 7 | target_link_libraries(TASK_NAME ${Hermes_RUNTIME_LIBRARIES}) 8 | 9 | #------------------------------------------------------------------------------ 10 | # Install Small Message Task Library 11 | #------------------------------------------------------------------------------ 12 | install( 13 | TARGETS 14 | TASK_NAME 15 | EXPORT 16 | ${HERMES_EXPORTED_TARGETS} 17 | LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} 18 | ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} 19 | RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} 20 | ) 21 | 22 | #----------------------------------------------------------------------------- 23 | # Add Target(s) to CMake Install for import into other projects 24 | #----------------------------------------------------------------------------- 25 | install( 26 | EXPORT 27 | ${HERMES_EXPORTED_TARGETS} 28 | DESTINATION 29 | ${HERMES_INSTALL_DATA_DIR}/cmake/hermes 30 | FILE 31 | ${HERMES_EXPORTED_TARGETS}.cmake 32 | ) 33 | 34 | #----------------------------------------------------------------------------- 35 | # Export all exported targets to the build tree for use by parent project 36 | #----------------------------------------------------------------------------- 37 | set(HERMES_EXPORTED_LIBS 38 | TASK_NAME 39 | ${HERMES_EXPORTED_LIBS}) 40 | if(NOT HERMES_EXTERNALLY_CONFIGURED) 41 | EXPORT ( 42 | TARGETS 43 | ${HERMES_EXPORTED_LIBS} 44 | FILE 45 | ${HERMES_EXPORTED_TARGETS}.cmake 46 | ) 47 | endif() 48 | 49 | #------------------------------------------------------------------------------ 50 | # Coverage 51 | #------------------------------------------------------------------------------ 52 | if(HERMES_ENABLE_COVERAGE) 53 | set_coverage_flags(TASK_NAME) 54 | endif() 55 | -------------------------------------------------------------------------------- /tasks/data_stager/src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | #------------------------------------------------------------------------------ 2 | # Build Small Message Task Library 3 | #------------------------------------------------------------------------------ 4 | add_library(data_stager SHARED 5 | data_stager.cc) 6 | add_dependencies(data_stager ${Hermes_RUNTIME_DEPS}) 7 | target_link_libraries(data_stager ${Hermes_RUNTIME_LIBRARIES}) 8 | 9 | #------------------------------------------------------------------------------ 10 | # Install Small Message Task Library 11 | #------------------------------------------------------------------------------ 12 | install( 13 | TARGETS 14 | data_stager 15 | EXPORT 16 | ${HERMES_EXPORTED_TARGETS} 17 | LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} 18 | ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} 19 | RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} 20 | ) 21 | 22 | #----------------------------------------------------------------------------- 23 | # Add Target(s) to CMake Install for import into other projects 24 | #----------------------------------------------------------------------------- 25 | install( 26 | EXPORT 27 | ${HERMES_EXPORTED_TARGETS} 28 | DESTINATION 29 | ${HERMES_INSTALL_DATA_DIR}/cmake/hermes 30 | FILE 31 | ${HERMES_EXPORTED_TARGETS}.cmake 32 | ) 33 | 34 | #----------------------------------------------------------------------------- 35 | # Export all exported targets to the build tree for use by parent project 36 | #----------------------------------------------------------------------------- 37 | set(HERMES_EXPORTED_LIBS 38 | data_stager 39 | ${HERMES_EXPORTED_LIBS}) 40 | if(NOT HERMES_EXTERNALLY_CONFIGURED) 41 | EXPORT ( 42 | TARGETS 43 | ${HERMES_EXPORTED_LIBS} 44 | FILE 45 | ${HERMES_EXPORTED_TARGETS}.cmake 46 | ) 47 | endif() 48 | 49 | #------------------------------------------------------------------------------ 50 | # Coverage 51 | #------------------------------------------------------------------------------ 52 | if(HERMES_ENABLE_COVERAGE) 53 | set_coverage_flags(data_stager) 54 | endif() 55 | -------------------------------------------------------------------------------- /hrun/tasks_required/proc_queue/src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | #------------------------------------------------------------------------------ 2 | # Build Small Message Task Library 3 | #------------------------------------------------------------------------------ 4 | add_library(proc_queue SHARED 5 | proc_queue.cc) 6 | add_dependencies(proc_queue ${Hermes_RUNTIME_DEPS}) 7 | target_link_libraries(proc_queue ${Hermes_RUNTIME_LIBRARIES}) 8 | 9 | #------------------------------------------------------------------------------ 10 | # Install Small Message Task Library 11 | #------------------------------------------------------------------------------ 12 | install( 13 | TARGETS 14 | proc_queue 15 | EXPORT 16 | ${HERMES_EXPORTED_TARGETS} 17 | LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} 18 | ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} 19 | RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} 20 | ) 21 | 22 | #----------------------------------------------------------------------------- 23 | # Add Target(s) to CMake Install for import into other projects 24 | #----------------------------------------------------------------------------- 25 | install( 26 | EXPORT 27 | ${HERMES_EXPORTED_TARGETS} 28 | DESTINATION 29 | ${HERMES_INSTALL_DATA_DIR}/cmake/hermes 30 | FILE 31 | ${HERMES_EXPORTED_TARGETS}.cmake 32 | ) 33 | 34 | #----------------------------------------------------------------------------- 35 | # Export all exported targets to the build tree for use by parent project 36 | #----------------------------------------------------------------------------- 37 | set(HERMES_EXPORTED_LIBS 38 | proc_queue 39 | ${HERMES_EXPORTED_LIBS}) 40 | if(NOT HERMES_EXTERNALLY_CONFIGURED) 41 | EXPORT ( 42 | TARGETS 43 | ${HERMES_EXPORTED_LIBS} 44 | FILE 45 | ${HERMES_EXPORTED_TARGETS}.cmake 46 | ) 47 | endif() 48 | 49 | #------------------------------------------------------------------------------ 50 | # Coverage 51 | #------------------------------------------------------------------------------ 52 | if(HERMES_ENABLE_COVERAGE) 53 | set_coverage_flags(proc_queue) 54 | endif() 55 | -------------------------------------------------------------------------------- /tasks/hermes_mdm/src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | #------------------------------------------------------------------------------ 2 | # Build Small Message Task Library 3 | #------------------------------------------------------------------------------ 4 | add_library(hermes_mdm SHARED 5 | hermes_mdm.cc) 6 | add_dependencies(hermes_mdm ${Hermes_RUNTIME_DEPS} hermes) 7 | target_link_libraries(hermes_mdm ${Hermes_RUNTIME_LIBRARIES} hermes) 8 | 9 | #------------------------------------------------------------------------------ 10 | # Install Small Message Task Library 11 | #------------------------------------------------------------------------------ 12 | install( 13 | TARGETS 14 | hermes_mdm 15 | EXPORT 16 | ${HERMES_EXPORTED_TARGETS} 17 | LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} 18 | ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} 19 | RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} 20 | ) 21 | 22 | #----------------------------------------------------------------------------- 23 | # Add Target(s) to CMake Install for import into other projects 24 | #----------------------------------------------------------------------------- 25 | install( 26 | EXPORT 27 | ${HERMES_EXPORTED_TARGETS} 28 | DESTINATION 29 | ${HERMES_INSTALL_DATA_DIR}/cmake/hermes 30 | FILE 31 | ${HERMES_EXPORTED_TARGETS}.cmake 32 | ) 33 | 34 | #----------------------------------------------------------------------------- 35 | # Export all exported targets to the build tree for use by parent project 36 | #----------------------------------------------------------------------------- 37 | set(HERMES_EXPORTED_LIBS 38 | hermes_mdm 39 | ${HERMES_EXPORTED_LIBS}) 40 | if(NOT HERMES_EXTERNALLY_CONFIGURED) 41 | EXPORT ( 42 | TARGETS 43 | ${HERMES_EXPORTED_LIBS} 44 | FILE 45 | ${HERMES_EXPORTED_TARGETS}.cmake 46 | ) 47 | endif() 48 | 49 | #------------------------------------------------------------------------------ 50 | # Coverage 51 | #------------------------------------------------------------------------------ 52 | if(HERMES_ENABLE_COVERAGE) 53 | set_coverage_flags(hermes_mdm) 54 | endif() 55 | -------------------------------------------------------------------------------- /tasks/hermes_data_op/src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | #------------------------------------------------------------------------------ 2 | # Build Small Message Task Library 3 | #------------------------------------------------------------------------------ 4 | add_library(hermes_data_op SHARED 5 | hermes_data_op.cc) 6 | add_dependencies(hermes_data_op ${Hermes_RUNTIME_DEPS}) 7 | target_link_libraries(hermes_data_op ${Hermes_RUNTIME_LIBRARIES}) 8 | 9 | #------------------------------------------------------------------------------ 10 | # Install Small Message Task Library 11 | #------------------------------------------------------------------------------ 12 | install( 13 | TARGETS 14 | hermes_data_op 15 | EXPORT 16 | ${HERMES_EXPORTED_TARGETS} 17 | LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} 18 | ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} 19 | RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} 20 | ) 21 | 22 | #----------------------------------------------------------------------------- 23 | # Add Target(s) to CMake Install for import into other projects 24 | #----------------------------------------------------------------------------- 25 | install( 26 | EXPORT 27 | ${HERMES_EXPORTED_TARGETS} 28 | DESTINATION 29 | ${HERMES_INSTALL_DATA_DIR}/cmake/hermes 30 | FILE 31 | ${HERMES_EXPORTED_TARGETS}.cmake 32 | ) 33 | 34 | #----------------------------------------------------------------------------- 35 | # Export all exported targets to the build tree for use by parent project 36 | #----------------------------------------------------------------------------- 37 | set(HERMES_EXPORTED_LIBS 38 | hermes_data_op 39 | ${HERMES_EXPORTED_LIBS}) 40 | if(NOT HERMES_EXTERNALLY_CONFIGURED) 41 | EXPORT ( 42 | TARGETS 43 | ${HERMES_EXPORTED_LIBS} 44 | FILE 45 | ${HERMES_EXPORTED_TARGETS}.cmake 46 | ) 47 | endif() 48 | 49 | #------------------------------------------------------------------------------ 50 | # Coverage 51 | #------------------------------------------------------------------------------ 52 | if(HERMES_ENABLE_COVERAGE) 53 | set_coverage_flags(hermes_data_op) 54 | endif() 55 | -------------------------------------------------------------------------------- /test/unit/hermes_adapters/mpiio/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # MPI adapter tests without hermes 2 | add_executable(mpiio_adapter_test 3 | mpiio_adapter_test.cc 4 | mpiio_adapter_basic_test.cc) 5 | target_link_libraries(mpiio_adapter_test 6 | hermes) 7 | add_dependencies(mpiio_adapter_test 8 | hermes) 9 | target_compile_definitions(mpiio_adapter_test PUBLIC 10 | HERMES_MPI_TESTS=true) 11 | jarvis_test(mpiio test_mpiio_basic) 12 | 13 | # MPI adapter tests with hermes 14 | add_executable(hermes_mpiio_adapter_test 15 | mpiio_adapter_test.cc 16 | mpiio_adapter_basic_test.cc) 17 | target_link_libraries(hermes_mpiio_adapter_test 18 | hermes_mpiio) 19 | add_dependencies(hermes_mpiio_adapter_test 20 | hermes_mpiio) 21 | target_compile_definitions(hermes_mpiio_adapter_test PUBLIC 22 | HERMES_INTERCEPT=1 HERMES_MPI_TESTS=true) 23 | jarvis_test(mpiio test_hermes_mpiio_basic_sync) 24 | jarvis_test(mpiio test_hermes_mpiio_basic_async) 25 | 26 | set(MPIIO_TESTS 27 | mpiio_adapter_test 28 | hermes_mpiio_adapter_test 29 | ) 30 | 31 | foreach(program ${MPIIO_TESTS}) 32 | target_include_directories(${program} PRIVATE ${HERMES_ADAPTER_DIR}) 33 | target_include_directories(${program} PRIVATE ${HERMES_ADAPTER_TEST_DIR}) 34 | target_link_libraries(${program} Catch2::Catch2 -lstdc++fs -lc MPI::MPI_CXX) 35 | endforeach() 36 | 37 | if(HERMES_INSTALL_TESTS) 38 | foreach(program ${MPIIO_TESTS}) 39 | install( 40 | TARGETS 41 | ${program} 42 | LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} 43 | ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} 44 | RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} 45 | ) 46 | endforeach() 47 | endif() 48 | 49 | add_executable(mpi_parallel parallel.cc) 50 | add_dependencies(mpi_parallel hermes_mpiio) 51 | target_link_libraries(mpi_parallel hermes_mpiio Catch2::Catch2 -lstdc++fs -lc MPI::MPI_CXX) 52 | set_target_properties(mpi_parallel PROPERTIES COMPILE_FLAGS "-DHERMES_INTERCEPT=1") 53 | 54 | install( 55 | TARGETS 56 | mpi_parallel 57 | LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} 58 | ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} 59 | RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} 60 | ) -------------------------------------------------------------------------------- /hrun/tasks_required/hrun_admin/src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | #------------------------------------------------------------------------------ 2 | # Build Hrun Admin Task Library 3 | #------------------------------------------------------------------------------ 4 | add_library(hrun_admin SHARED 5 | ${CMAKE_CURRENT_SOURCE_DIR}/hrun_admin.cc) 6 | add_dependencies(hrun_admin ${Hermes_RUNTIME_DEPS}) 7 | target_link_libraries(hrun_admin ${Hermes_RUNTIME_LIBRARIES}) 8 | 9 | #------------------------------------------------------------------------------ 10 | # Install Hrun Admin Task Library 11 | #------------------------------------------------------------------------------ 12 | install( 13 | TARGETS 14 | hrun_admin 15 | EXPORT 16 | ${HERMES_EXPORTED_TARGETS} 17 | LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} 18 | ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} 19 | RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} 20 | ) 21 | 22 | #----------------------------------------------------------------------------- 23 | # Add Target(s) to CMake Install for import into other projects 24 | #----------------------------------------------------------------------------- 25 | install( 26 | EXPORT 27 | ${HERMES_EXPORTED_TARGETS} 28 | DESTINATION 29 | ${HERMES_INSTALL_DATA_DIR}/cmake/hermes 30 | FILE 31 | ${HERMES_EXPORTED_TARGETS}.cmake 32 | ) 33 | 34 | #----------------------------------------------------------------------------- 35 | # Export all exported targets to the build tree for use by parent project 36 | #----------------------------------------------------------------------------- 37 | set(HERMES_EXPORTED_LIBS 38 | hrun_admin 39 | ${HERMES_EXPORTED_LIBS}) 40 | if(NOT HERMES_EXTERNALLY_CONFIGURED) 41 | EXPORT ( 42 | TARGETS 43 | ${HERMES_EXPORTED_LIBS} 44 | FILE 45 | ${HERMES_EXPORTED_TARGETS}.cmake 46 | ) 47 | endif() 48 | 49 | #------------------------------------------------------------------------------ 50 | # Coverage 51 | #------------------------------------------------------------------------------ 52 | if(HERMES_ENABLE_COVERAGE) 53 | set_coverage_flags(hrun_admin) 54 | endif() 55 | -------------------------------------------------------------------------------- /hrun/tasks_required/small_message/src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | #------------------------------------------------------------------------------ 2 | # Build Small Message Task Library 3 | #------------------------------------------------------------------------------ 4 | add_library(small_message SHARED 5 | small_message.cc) 6 | add_dependencies(small_message ${Hermes_RUNTIME_DEPS}) 7 | target_link_libraries(small_message ${Hermes_RUNTIME_LIBRARIES}) 8 | 9 | #------------------------------------------------------------------------------ 10 | # Install Small Message Task Library 11 | #------------------------------------------------------------------------------ 12 | install( 13 | TARGETS 14 | small_message 15 | EXPORT 16 | ${HERMES_EXPORTED_TARGETS} 17 | LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} 18 | ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} 19 | RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} 20 | ) 21 | 22 | #----------------------------------------------------------------------------- 23 | # Add Target(s) to CMake Install for import into other projects 24 | #----------------------------------------------------------------------------- 25 | install( 26 | EXPORT 27 | ${HERMES_EXPORTED_TARGETS} 28 | DESTINATION 29 | ${HERMES_INSTALL_DATA_DIR}/cmake/hermes 30 | FILE 31 | ${HERMES_EXPORTED_TARGETS}.cmake 32 | ) 33 | 34 | #----------------------------------------------------------------------------- 35 | # Export all exported targets to the build tree for use by parent project 36 | #----------------------------------------------------------------------------- 37 | set(HERMES_EXPORTED_LIBS 38 | small_message 39 | ${HERMES_EXPORTED_LIBS}) 40 | if(NOT HERMES_EXTERNALLY_CONFIGURED) 41 | EXPORT ( 42 | TARGETS 43 | ${HERMES_EXPORTED_LIBS} 44 | FILE 45 | ${HERMES_EXPORTED_TARGETS}.cmake 46 | ) 47 | endif() 48 | 49 | #------------------------------------------------------------------------------ 50 | # Coverage 51 | #------------------------------------------------------------------------------ 52 | if(HERMES_ENABLE_COVERAGE) 53 | set_coverage_flags(small_message) 54 | endif() 55 | -------------------------------------------------------------------------------- /tasks/hermes_blob_mdm/src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | #------------------------------------------------------------------------------ 2 | # Build Small Message Task Library 3 | #------------------------------------------------------------------------------ 4 | add_library(hermes_blob_mdm SHARED 5 | hermes_blob_mdm.cc) 6 | add_dependencies(hermes_blob_mdm ${Hermes_RUNTIME_DEPS}) 7 | target_link_libraries(hermes_blob_mdm ${Hermes_RUNTIME_LIBRARIES}) 8 | 9 | #------------------------------------------------------------------------------ 10 | # Install Small Message Task Library 11 | #------------------------------------------------------------------------------ 12 | install( 13 | TARGETS 14 | hermes_blob_mdm 15 | EXPORT 16 | ${HERMES_EXPORTED_TARGETS} 17 | LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} 18 | ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} 19 | RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} 20 | ) 21 | 22 | #----------------------------------------------------------------------------- 23 | # Add Target(s) to CMake Install for import into other projects 24 | #----------------------------------------------------------------------------- 25 | install( 26 | EXPORT 27 | ${HERMES_EXPORTED_TARGETS} 28 | DESTINATION 29 | ${HERMES_INSTALL_DATA_DIR}/cmake/hermes 30 | FILE 31 | ${HERMES_EXPORTED_TARGETS}.cmake 32 | ) 33 | 34 | #----------------------------------------------------------------------------- 35 | # Export all exported targets to the build tree for use by parent project 36 | #----------------------------------------------------------------------------- 37 | set(HERMES_EXPORTED_LIBS 38 | hermes_blob_mdm 39 | ${HERMES_EXPORTED_LIBS}) 40 | if(NOT HERMES_EXTERNALLY_CONFIGURED) 41 | EXPORT ( 42 | TARGETS 43 | ${HERMES_EXPORTED_LIBS} 44 | FILE 45 | ${HERMES_EXPORTED_TARGETS}.cmake 46 | ) 47 | endif() 48 | 49 | #------------------------------------------------------------------------------ 50 | # Coverage 51 | #------------------------------------------------------------------------------ 52 | if(HERMES_ENABLE_COVERAGE) 53 | set_coverage_flags(hermes_blob_mdm) 54 | endif() 55 | -------------------------------------------------------------------------------- /docker/user.Dockerfile: -------------------------------------------------------------------------------- 1 | # Install ubuntu 22.04 2 | FROM ubuntu:latest 3 | LABEL maintainer="llogan@hawk.iit.edu" 4 | LABEL version="0.0" 5 | LABEL description="Hermes Docker image with CI" 6 | 7 | # Disable Prompt During Packages Installation 8 | ARG DEBIAN_FRONTEND=noninteractive 9 | 10 | # Update ubuntu 11 | SHELL ["/bin/bash", "-c"] 12 | RUN apt update && apt install 13 | 14 | # Install some basic packages 15 | RUN apt install -y \ 16 | openssh-server \ 17 | sudo \ 18 | git \ 19 | gcc g++ gfortran make binutils gpg \ 20 | tar zip xz-utils bzip2 \ 21 | perl m4 libncurses5-dev libxml2-dev diffutils \ 22 | pkg-config cmake pkg-config \ 23 | python3 python3-pip doxygen \ 24 | lcov zlib1g-dev hdf5-tools \ 25 | build-essential ca-certificates \ 26 | coreutils curl environment-modules \ 27 | gfortran git gpg lsb-release python3 python3-distutils \ 28 | python3-venv unzip zip \ 29 | bash jq python gdbserver gdb 30 | 31 | # Setup basic environment 32 | ENV USER="root" 33 | ENV HOME="/root" 34 | ENV SPACK_DIR="${HOME}/spack" 35 | ENV SPACK_VERSION="v0.20.2" 36 | ENV HERMES_DEPS_DIR="${HOME}/hermes_deps" 37 | ENV HERMES_DIR="${HOME}/hermes" 38 | COPY ci/module_load.sh /module_load.sh 39 | 40 | # Install Spack 41 | RUN . /module_load.sh && \ 42 | git clone -b ${SPACK_VERSION} https://github.com/spack/spack ${SPACK_DIR} && \ 43 | . "${SPACK_DIR}/share/spack/setup-env.sh" && \ 44 | git clone -b dev https://github.com/lukemartinlogan/hermes.git ${HERMES_DEPS_DIR} && \ 45 | # git clone -b dev https://github.com/HDFGroup/hermes.git ${HERMES_DEPS_DIR} && \ 46 | spack repo add ${HERMES_DEPS_DIR}/ci/hermes && \ 47 | mkdir -p ${HERMES_DIR} && \ 48 | spack external find 49 | 50 | # Install hermes 51 | RUN . /module_load.sh && \ 52 | . "${SPACK_DIR}/share/spack/setup-env.sh" && \ 53 | spack external find && \ 54 | spack install hermes@master+vfd+mpiio^mpich@3.3.2 55 | 56 | # Install jarvis-cd 57 | RUN git clone https://github.com/grc-iit/jarvis-cd.git && \ 58 | cd jarvis-cd && \ 59 | pip install -e . -r requirements.txt 60 | 61 | # Install scspkg 62 | RUN git clone https://github.com/grc-iit/scspkg.git && \ 63 | cd scspkg && \ 64 | pip install -e . -r requirements.txt 65 | -------------------------------------------------------------------------------- /tasks/hermes_bucket_mdm/src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | #------------------------------------------------------------------------------ 2 | # Build Small Message Task Library 3 | #------------------------------------------------------------------------------ 4 | add_library(hermes_bucket_mdm SHARED 5 | hermes_bucket_mdm.cc) 6 | add_dependencies(hermes_bucket_mdm ${Hermes_RUNTIME_DEPS}) 7 | target_link_libraries(hermes_bucket_mdm ${Hermes_RUNTIME_LIBRARIES}) 8 | 9 | #------------------------------------------------------------------------------ 10 | # Install Small Message Task Library 11 | #------------------------------------------------------------------------------ 12 | install( 13 | TARGETS 14 | hermes_bucket_mdm 15 | EXPORT 16 | ${HERMES_EXPORTED_TARGETS} 17 | LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} 18 | ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} 19 | RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} 20 | ) 21 | 22 | #----------------------------------------------------------------------------- 23 | # Add Target(s) to CMake Install for import into other projects 24 | #----------------------------------------------------------------------------- 25 | install( 26 | EXPORT 27 | ${HERMES_EXPORTED_TARGETS} 28 | DESTINATION 29 | ${HERMES_INSTALL_DATA_DIR}/cmake/hermes 30 | FILE 31 | ${HERMES_EXPORTED_TARGETS}.cmake 32 | ) 33 | 34 | #----------------------------------------------------------------------------- 35 | # Export all exported targets to the build tree for use by parent project 36 | #----------------------------------------------------------------------------- 37 | set(HERMES_EXPORTED_LIBS 38 | hermes_bucket_mdm 39 | ${HERMES_EXPORTED_LIBS}) 40 | if(NOT HERMES_EXTERNALLY_CONFIGURED) 41 | EXPORT ( 42 | TARGETS 43 | ${HERMES_EXPORTED_LIBS} 44 | FILE 45 | ${HERMES_EXPORTED_TARGETS}.cmake 46 | ) 47 | endif() 48 | 49 | #------------------------------------------------------------------------------ 50 | # Coverage 51 | #------------------------------------------------------------------------------ 52 | if(HERMES_ENABLE_COVERAGE) 53 | set_coverage_flags(hermes_bucket_mdm) 54 | endif() 55 | -------------------------------------------------------------------------------- /benchmark/test_zmq.cc: -------------------------------------------------------------------------------- 1 | // 2 | // Created by lukemartinlogan on 8/25/23. 3 | // 4 | 5 | #include 6 | #include "basic_test.h" 7 | #include "hrun/api/hrun_client.h" 8 | #include "hrun_admin/hrun_admin.h" 9 | #include "small_message/small_message.h" 10 | #include "hermes_shm/util/timer.h" 11 | #include "hrun/work_orchestrator/affinity.h" 12 | 13 | /** ZeroMQ allocate + free request */ 14 | TEST_CASE("TestZeromqAllocateFree") { 15 | zmq::context_t context(1); 16 | 17 | // Create a PUSH socket to push requests 18 | zmq::socket_t pushSocket(context, ZMQ_PUSH); 19 | 20 | // Bind the PUSH socket to a specific address 21 | pushSocket.bind("ipc:///tmp/shared_memory"); 22 | 23 | // Create a PULL socket to receive requests 24 | zmq::socket_t pullSocket(context, ZMQ_PULL); 25 | 26 | // Connect the PULL socket to the same address 27 | pullSocket.connect("ipc:///tmp/shared_memory"); 28 | 29 | hshm::Timer t; 30 | t.Resume(); 31 | size_t ops = (1 << 20); 32 | for (size_t i = 0; i < ops; ++i) { 33 | zmq::message_t message(sizeof(hrun::Task)); 34 | } 35 | t.Pause(); 36 | 37 | HILOG(kInfo, "Latency: {} MOps", ops / t.GetUsec()); 38 | } 39 | 40 | /** Single-thread performance of zeromq */ 41 | TEST_CASE("TestZeromqAllocateEmplacePop") { 42 | zmq::context_t context(1); 43 | 44 | // Create a PUSH socket to push requests 45 | zmq::socket_t pushSocket(context, ZMQ_PUSH); 46 | 47 | // Bind the PUSH socket to a specific address 48 | pushSocket.bind("ipc:///tmp/shared_memory"); 49 | 50 | // Create a PULL socket to receive requests 51 | zmq::socket_t pullSocket(context, ZMQ_PULL); 52 | 53 | // Connect the PULL socket to the same address 54 | pullSocket.connect("ipc:///tmp/shared_memory"); 55 | 56 | hshm::Timer t; 57 | t.Resume(); 58 | size_t ops = (1 << 20); 59 | for (size_t i = 0; i < ops; ++i) { 60 | // Send a request 61 | zmq::message_t message(sizeof(hrun::Task)); 62 | pushSocket.send(message, zmq::send_flags::none); 63 | 64 | // Receive the request 65 | zmq::message_t receivedMessage; 66 | zmq::recv_result_t result = pullSocket.recv(receivedMessage); 67 | REQUIRE(receivedMessage.size() == sizeof(hrun::Task)); 68 | } 69 | t.Pause(); 70 | 71 | HILOG(kInfo, "Latency: {} MOps", ops / t.GetUsec()); 72 | } -------------------------------------------------------------------------------- /.github/workflows/build_and_push_docker_images.yml: -------------------------------------------------------------------------------- 1 | name: Build and Push Docker Images 2 | on: 3 | push: 4 | branches: 5 | - master 6 | - main 7 | - dev 8 | workflow_dispatch: 9 | branches: 10 | - master 11 | - main 12 | - dev 13 | inputs: 14 | # build_docker_images: 15 | # description: 'If ''true'', force a build of the docker images and push them to dockerhub' 16 | # required: false 17 | # default: false 18 | jobs: 19 | build-docker-images: 20 | if: ${{ github.ref_type == 'tag' || github.event_name == 'workflow_dispatch' }} 21 | runs-on: ubuntu-20.04 22 | steps: 23 | - name: Cache Hermes deps Docker 24 | uses: actions/cache@v3 25 | id: docker-deps-cache 26 | with: 27 | path: ci/deps/deps.Dockerfile 28 | key: docker-${{ hashFiles('docker/deps.Dockerfile') }} 29 | - name: Checkout 30 | uses: actions/checkout@v3 31 | - name: Login to Docker Hub 32 | uses: docker/login-action@v2 33 | with: 34 | username: ${{ secrets.DOCKER_HUB_USERNAME }} 35 | password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} 36 | - name: Set up Docker Buildx 37 | uses: docker/setup-buildx-action@v2 38 | - name: Build and push deps.Dockerfile 39 | uses: docker/build-push-action@v4 40 | if: steps.docker-deps-cache.outputs.cache-hit != 'true' 41 | with: 42 | context: ./ 43 | file: ./docker/deps.Dockerfile 44 | builder: ${{ steps.buildx.outputs.name }} 45 | push: true 46 | tags: hdfgroup/hermes-deps:latest 47 | # - name: Build and push dev.Dockerfile 48 | # uses: docker/build-push-action@v4 49 | # with: 50 | # context: ./ 51 | # file: ./docker/dev.Dockerfile 52 | # builder: ${{ steps.buildx.outputs.name }} 53 | # push: true 54 | # tags: hdfgroup/hermes-dev:latest 55 | # - name: Build and push user.Dockerfile 56 | # uses: docker/build-push-action@v4 57 | # with: 58 | # context: ./ 59 | # file: ./docker/user.Dockerfile 60 | # builder: ${{ steps.buildx.outputs.name }} 61 | # push: true 62 | # tags: hdfgroup/hermes:latest 63 | -------------------------------------------------------------------------------- /hermes_adapters/posix/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | include_directories( 2 | ${CMAKE_SOURCE_DIR} 3 | ${HERMES_SRC_DIR} 4 | ${HERMES_ADAPTER_DIR} 5 | ${HERMES_IO_CLIENT_DIR} 6 | .) 7 | 8 | # Create the POSIX interceptor 9 | set(INTERCEPTOR_DEPS 10 | hermes hermes_fs_base) 11 | add_library(hermes_posix SHARED 12 | ${CMAKE_CURRENT_SOURCE_DIR}/posix_api.cc) 13 | add_dependencies(hermes_posix ${INTERCEPTOR_DEPS}) 14 | target_link_libraries(hermes_posix MPI::MPI_CXX stdc++fs ${libelf_LIBRARIES} dl ${INTERCEPTOR_DEPS}) 15 | 16 | #----------------------------------------------------------------------------- 17 | # Add Target(s) to CMake Install 18 | #----------------------------------------------------------------------------- 19 | install( 20 | TARGETS 21 | hermes_posix 22 | EXPORT 23 | ${HERMES_EXPORTED_TARGETS} 24 | LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} 25 | ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} 26 | RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} 27 | ) 28 | 29 | #----------------------------------------------------------------------------- 30 | # Export all exported targets to the build tree for use by parent project 31 | #----------------------------------------------------------------------------- 32 | set(HERMES_EXPORTED_LIBS 33 | hermes_posix 34 | ${HERMES_EXPORTED_LIBS}) 35 | if(NOT HERMES_EXTERNALLY_CONFIGURED) 36 | EXPORT ( 37 | TARGETS 38 | ${HERMES_EXPORTED_LIBS} 39 | FILE 40 | ${HERMES_EXPORTED_TARGETS}.cmake 41 | ) 42 | endif() 43 | 44 | #----------------------------------------------------------------------------- 45 | # Install headers 46 | #----------------------------------------------------------------------------- 47 | file(GLOB_RECURSE HERMES_HEADERS "*.h") 48 | install( 49 | FILES 50 | ${HERMES_HEADERS} 51 | DESTINATION 52 | ${CMAKE_INSTALL_PREFIX}/include/hermes_adapters/posix 53 | COMPONENT 54 | headers 55 | ) 56 | 57 | #----------------------------------------------------------------------------- 58 | # Add Target(s) to Coverage 59 | #----------------------------------------------------------------------------- 60 | if(HERMES_ENABLE_COVERAGE) 61 | set_coverage_flags(hermes_posix) 62 | endif() 63 | -------------------------------------------------------------------------------- /hrun/tasks_required/worch_proc_round_robin/src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | #------------------------------------------------------------------------------ 2 | # Build Small Message Task Library 3 | #------------------------------------------------------------------------------ 4 | add_library(worch_proc_round_robin SHARED 5 | worch_proc_round_robin.cc) 6 | add_dependencies(worch_proc_round_robin ${Hermes_RUNTIME_DEPS}) 7 | target_link_libraries(worch_proc_round_robin ${Hermes_RUNTIME_LIBRARIES}) 8 | 9 | #------------------------------------------------------------------------------ 10 | # Install Small Message Task Library 11 | #------------------------------------------------------------------------------ 12 | install( 13 | TARGETS 14 | worch_proc_round_robin 15 | EXPORT 16 | ${HERMES_EXPORTED_TARGETS} 17 | LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} 18 | ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} 19 | RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} 20 | ) 21 | 22 | #----------------------------------------------------------------------------- 23 | # Add Target(s) to CMake Install for import into other projects 24 | #----------------------------------------------------------------------------- 25 | install( 26 | EXPORT 27 | ${HERMES_EXPORTED_TARGETS} 28 | DESTINATION 29 | ${HERMES_INSTALL_DATA_DIR}/cmake/hermes 30 | FILE 31 | ${HERMES_EXPORTED_TARGETS}.cmake 32 | ) 33 | 34 | #----------------------------------------------------------------------------- 35 | # Export all exported targets to the build tree for use by parent project 36 | #----------------------------------------------------------------------------- 37 | set(HERMES_EXPORTED_LIBS 38 | worch_proc_round_robin 39 | ${HERMES_EXPORTED_LIBS}) 40 | if(NOT HERMES_EXTERNALLY_CONFIGURED) 41 | EXPORT ( 42 | TARGETS 43 | ${HERMES_EXPORTED_LIBS} 44 | FILE 45 | ${HERMES_EXPORTED_TARGETS}.cmake 46 | ) 47 | endif() 48 | 49 | #------------------------------------------------------------------------------ 50 | # Coverage 51 | #------------------------------------------------------------------------------ 52 | if(HERMES_ENABLE_COVERAGE) 53 | set_coverage_flags(worch_proc_round_robin) 54 | endif() 55 | -------------------------------------------------------------------------------- /hrun/tasks_required/worch_queue_round_robin/src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | #------------------------------------------------------------------------------ 2 | # Build Small Message Task Library 3 | #------------------------------------------------------------------------------ 4 | add_library(worch_queue_round_robin SHARED 5 | worch_queue_round_robin.cc) 6 | add_dependencies(worch_queue_round_robin ${Hermes_RUNTIME_DEPS}) 7 | target_link_libraries(worch_queue_round_robin ${Hermes_RUNTIME_LIBRARIES}) 8 | 9 | #------------------------------------------------------------------------------ 10 | # Install Small Message Task Library 11 | #------------------------------------------------------------------------------ 12 | install( 13 | TARGETS 14 | worch_queue_round_robin 15 | EXPORT 16 | ${HERMES_EXPORTED_TARGETS} 17 | LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} 18 | ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} 19 | RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} 20 | ) 21 | 22 | #----------------------------------------------------------------------------- 23 | # Add Target(s) to CMake Install for import into other projects 24 | #----------------------------------------------------------------------------- 25 | install( 26 | EXPORT 27 | ${HERMES_EXPORTED_TARGETS} 28 | DESTINATION 29 | ${HERMES_INSTALL_DATA_DIR}/cmake/hermes 30 | FILE 31 | ${HERMES_EXPORTED_TARGETS}.cmake 32 | ) 33 | 34 | #----------------------------------------------------------------------------- 35 | # Export all exported targets to the build tree for use by parent project 36 | #----------------------------------------------------------------------------- 37 | set(HERMES_EXPORTED_LIBS 38 | worch_queue_round_robin 39 | ${HERMES_EXPORTED_LIBS}) 40 | if(NOT HERMES_EXTERNALLY_CONFIGURED) 41 | EXPORT ( 42 | TARGETS 43 | ${HERMES_EXPORTED_LIBS} 44 | FILE 45 | ${HERMES_EXPORTED_TARGETS}.cmake 46 | ) 47 | endif() 48 | 49 | #------------------------------------------------------------------------------ 50 | # Coverage 51 | #------------------------------------------------------------------------------ 52 | if(HERMES_ENABLE_COVERAGE) 53 | set_coverage_flags(worch_queue_round_robin) 54 | endif() 55 | -------------------------------------------------------------------------------- /hrun/config/hrun_server_default.yaml: -------------------------------------------------------------------------------- 1 | ### Runtime orchestration settings 2 | work_orchestrator: 3 | # The max number of dedicated worker threads 4 | max_dworkers: 4 5 | # The max number of overlapping threads 6 | max_oworkers: 32 7 | # The max number of total dedicated cores 8 | owork_per_core: 32 9 | 10 | ### Queue Manager settings 11 | queue_manager: 12 | # The default depth of process queue 13 | proc_queue_depth: 8192 14 | # The default depth of allocated queues 15 | queue_depth: 100000 16 | # The maximum number of lanes per queue 17 | max_lanes: 16 18 | # The maximum number of queues 19 | max_queues: 1024 20 | # The shared memory allocator to use 21 | shm_allocator: kScalablePageAllocator 22 | # The name of the shared memory region to create 23 | shm_name: "hrun_shm" 24 | # The size of the shared memory region to allocate for general data structures 25 | shm_size: 0g 26 | # The size of the shared memory to allocate for data buffers 27 | data_shm_size: 4g 28 | # The size of the shared memory to allocate for runtime data buffers 29 | rdata_shm_size: 4g 30 | 31 | ### Define properties of RPCs 32 | rpc: 33 | # A path to a file containing a list of server names, 1 per line. If your 34 | # servers are named according to a pattern (e.g., server-1, server-2, etc.), 35 | # prefer the `rpc_server_base_name` and `rpc_host_number_range` options. If this 36 | # option is not empty, it will override anything in `rpc_server_base_name`. 37 | host_file: "" 38 | 39 | # Host names can be defined using the following syntax: 40 | # ares-comp-[0-9]-40g will convert to ares-comp-0-40g, ares-comp-1-40g, ... 41 | # ares-comp[00-09] will convert to ares-comp-00, ares-comp-01, ... 42 | host_names: ["localhost"] 43 | 44 | # The RPC protocol. This must come from the documentation of the specific RPC 45 | # library in use. 46 | protocol: "ofi+sockets" 47 | 48 | # RPC domain name for verbs transport. Blank for tcp. 49 | domain: "" 50 | 51 | # Desired RPC port number. 52 | port: 8080 53 | 54 | # The number of handler threads for each RPC server. 55 | num_threads: 32 56 | 57 | ### Task Registry 58 | task_registry: [ 59 | 'hermes_mdm', 60 | 'hermes_blob_mdm', 61 | 'hermes_bucket_mdm', 62 | 'hermes_data_op', 63 | 'data_stager', 64 | 'posix_bdev', 65 | 'ram_bdev' 66 | ] -------------------------------------------------------------------------------- /hermes_adapters/stdio/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | include_directories( 2 | ${CMAKE_SOURCE_DIR} 3 | ${HERMES_SRC_DIR} 4 | ${HERMES_ADAPTER_DIR} 5 | ${HERMES_IO_CLIENT_DIR} 6 | .) 7 | 8 | # Create the STDIO interceptor 9 | set(INTERCEPTOR_DEPS 10 | hermes hermes_fs_base) 11 | add_library(hermes_stdio SHARED 12 | ${CMAKE_CURRENT_SOURCE_DIR}/stdio_api.cc) 13 | add_dependencies(hermes_stdio ${INTERCEPTOR_DEPS}) 14 | target_link_libraries(hermes_stdio MPI::MPI_CXX stdc++fs ${libelf_LIBRARIES} dl ${INTERCEPTOR_DEPS}) 15 | 16 | #----------------------------------------------------------------------------- 17 | # Add Target(s) to CMake Install 18 | #----------------------------------------------------------------------------- 19 | install( 20 | TARGETS 21 | hermes_stdio 22 | EXPORT 23 | ${HERMES_EXPORTED_TARGETS} 24 | LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} 25 | ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} 26 | RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} 27 | ) 28 | 29 | #----------------------------------------------------------------------------- 30 | # Export all exported targets to the build tree for use by parent project 31 | #----------------------------------------------------------------------------- 32 | set(HERMES_EXPORTED_LIBS 33 | hermes_stdio 34 | ${HERMES_EXPORTED_LIBS}) 35 | if(NOT HERMES_EXTERNALLY_CONFIGURED) 36 | EXPORT ( 37 | TARGETS 38 | ${HERMES_EXPORTED_LIBS} 39 | FILE 40 | ${HERMES_EXPORTED_TARGETS}.cmake 41 | ) 42 | endif() 43 | 44 | #----------------------------------------------------------------------------- 45 | # Install headers 46 | #----------------------------------------------------------------------------- 47 | file(GLOB_RECURSE HERMES_HEADERS "*.h") 48 | install( 49 | FILES 50 | ${HERMES_HEADERS} 51 | DESTINATION 52 | ${CMAKE_INSTALL_PREFIX}/include/hermes_adapters/stdio 53 | COMPONENT 54 | headers 55 | ) 56 | 57 | #----------------------------------------------------------------------------- 58 | # Add Target(s) to Coverage 59 | #----------------------------------------------------------------------------- 60 | if(HERMES_ENABLE_COVERAGE) 61 | set_coverage_flags(hermes_stdio) 62 | endif() 63 | -------------------------------------------------------------------------------- /hrun/tasks_required/remote_queue/src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | #------------------------------------------------------------------------------ 2 | # Build Small Message Task Library 3 | #------------------------------------------------------------------------------ 4 | include_directories(myapp PRIVATE ${libfabric_INCLUDE_DIRS}) 5 | link_directories(${libfabric_LIBRARY_DIRS}) 6 | add_library(remote_queue SHARED 7 | remote_queue.cc) 8 | add_dependencies(remote_queue ${Hermes_RUNTIME_DEPS}) 9 | target_link_libraries(remote_queue 10 | hrun_runtime ${libfabric_LIBRARIES}) 11 | 12 | #------------------------------------------------------------------------------ 13 | # Install Small Message Task Library 14 | #------------------------------------------------------------------------------ 15 | install( 16 | TARGETS 17 | remote_queue 18 | EXPORT 19 | ${HERMES_EXPORTED_TARGETS} 20 | LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} 21 | ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} 22 | RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} 23 | ) 24 | 25 | #----------------------------------------------------------------------------- 26 | # Add Target(s) to CMake Install for import into other projects 27 | #----------------------------------------------------------------------------- 28 | install( 29 | EXPORT 30 | ${HERMES_EXPORTED_TARGETS} 31 | DESTINATION 32 | ${HERMES_INSTALL_DATA_DIR}/cmake/hermes 33 | FILE 34 | ${HERMES_EXPORTED_TARGETS}.cmake 35 | ) 36 | 37 | #----------------------------------------------------------------------------- 38 | # Export all exported targets to the build tree for use by parent project 39 | #----------------------------------------------------------------------------- 40 | set(HERMES_EXPORTED_LIBS 41 | remote_queue 42 | ${HERMES_EXPORTED_LIBS}) 43 | if(NOT HERMES_EXTERNALLY_CONFIGURED) 44 | EXPORT ( 45 | TARGETS 46 | ${HERMES_EXPORTED_LIBS} 47 | FILE 48 | ${HERMES_EXPORTED_TARGETS}.cmake 49 | ) 50 | endif() 51 | 52 | #------------------------------------------------------------------------------ 53 | # Coverage 54 | #------------------------------------------------------------------------------ 55 | if(HERMES_ENABLE_COVERAGE) 56 | set_coverage_flags(remote_queue) 57 | endif() 58 | -------------------------------------------------------------------------------- /docker/deps.Dockerfile: -------------------------------------------------------------------------------- 1 | # NOTE(llogan): This dockerfile assumes that 2 | # hermes github is the current working directory 3 | 4 | # Install ubuntu 22.04 5 | FROM ubuntu:22.04 6 | LABEL maintainer="llogan@hawk.iit.edu" 7 | LABEL version="0.0" 8 | LABEL description="Hermes Docker image with CI" 9 | 10 | # Disable Prompt During Packages Installation 11 | ARG DEBIAN_FRONTEND=noninteractive 12 | 13 | # Update ubuntu 14 | SHELL ["/bin/bash", "-c"] 15 | RUN apt update && apt install 16 | 17 | # Install some basic packages 18 | RUN apt install -y \ 19 | openssh-server \ 20 | sudo \ 21 | git \ 22 | gcc g++ gfortran make binutils gpg \ 23 | tar zip xz-utils bzip2 \ 24 | perl m4 libncurses5-dev libxml2-dev diffutils \ 25 | pkg-config cmake pkg-config \ 26 | python3 python3-pip python3 python3-distutils python3-venv\ 27 | doxygen lcov zlib1g-dev hdf5-tools \ 28 | build-essential ca-certificates \ 29 | coreutils curl environment-modules \ 30 | gfortran git gpg lsb-release \ 31 | unzip zip libelf-dev \ 32 | bash jq gdbserver gdb 33 | 34 | # Setup basic environment 35 | ENV USER="root" 36 | ENV HOME="/root" 37 | ENV SPACK_DIR="${HOME}/spack" 38 | ENV SPACK_VERSION="v0.20.2" 39 | ENV HERMES_DEPS_DIR="${HOME}/hermes_deps" 40 | ENV HERMES_DIR="${HOME}/hermes" 41 | COPY ci/module_load.sh module_load.sh 42 | 43 | # Install Spack 44 | RUN . /module_load.sh && \ 45 | git clone -b ${SPACK_VERSION} https://github.com/spack/spack ${SPACK_DIR} && \ 46 | . "${SPACK_DIR}/share/spack/setup-env.sh" && \ 47 | git clone -b dev https://github.com/lukemartinlogan/hermes.git ${HERMES_DEPS_DIR} && \ 48 | # git clone -b dev https://github.com/HDFGroup/hermes.git ${HERMES_DEPS_DIR} && \ 49 | spack repo add ${HERMES_DEPS_DIR}/ci/hermes && \ 50 | mkdir -p ${HERMES_DIR} && \ 51 | spack external find 52 | 53 | # Install hermes_shm 54 | RUN . /module_load.sh && \ 55 | . "${SPACK_DIR}/share/spack/setup-env.sh" && \ 56 | spack external find && \ 57 | spack install hermes_shm@master+vfd+mpiio^mpich@3.3.2 58 | 59 | # Install jarvis-cd 60 | RUN git clone https://github.com/grc-iit/jarvis-cd.git && \ 61 | cd jarvis-cd && \ 62 | pip install -e . -r requirements.txt 63 | 64 | # Install scspkg 65 | RUN git clone https://github.com/grc-iit/scspkg.git && \ 66 | cd scspkg && \ 67 | pip install -e . -r requirements.txt 68 | -------------------------------------------------------------------------------- /hrun/tasks_required/worch_proc_round_robin/include/worch_proc_round_robin/worch_proc_round_robin_tasks.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by lukemartinlogan on 8/14/23. 3 | // 4 | 5 | #ifndef HRUN_WORCH_PROC_ROUND_ROBIN_TASKS_H__ 6 | #define HRUN_WORCH_PROC_ROUND_ROBIN_TASKS_H__ 7 | 8 | #include "hrun/api/hrun_client.h" 9 | #include "hrun/task_registry/task_lib.h" 10 | #include "hrun/work_orchestrator/scheduler.h" 11 | #include "hrun_admin/hrun_admin.h" 12 | #include "hrun/queue_manager/queue_manager_client.h" 13 | #include "proc_queue/proc_queue.h" 14 | 15 | namespace hrun::worch_proc_round_robin { 16 | 17 | #include "hrun/hrun_namespace.h" 18 | 19 | /** The set of methods in the worch task */ 20 | typedef SchedulerMethod Method; 21 | 22 | /** 23 | * A task to create worch_proc_round_robin 24 | * */ 25 | using hrun::Admin::CreateTaskStateTask; 26 | struct ConstructTask : public CreateTaskStateTask { 27 | /** SHM default constructor */ 28 | HSHM_ALWAYS_INLINE explicit 29 | ConstructTask(hipc::Allocator *alloc) : CreateTaskStateTask(alloc) {} 30 | 31 | /** Emplace constructor */ 32 | HSHM_ALWAYS_INLINE 33 | ConstructTask(hipc::Allocator *alloc, 34 | const TaskNode &task_node, 35 | const DomainId &domain_id, 36 | const std::string &state_name, 37 | const TaskStateId &id, 38 | const std::vector &queue_info) 39 | : CreateTaskStateTask(alloc, task_node, domain_id, state_name, 40 | "worch_proc_round_robin", id, queue_info) { 41 | } 42 | }; 43 | 44 | /** A task to destroy worch_proc_round_robin */ 45 | using hrun::Admin::DestroyTaskStateTask; 46 | struct DestructTask : public DestroyTaskStateTask { 47 | /** SHM default constructor */ 48 | HSHM_ALWAYS_INLINE explicit 49 | DestructTask(hipc::Allocator *alloc) : DestroyTaskStateTask(alloc) {} 50 | 51 | /** Emplace constructor */ 52 | HSHM_ALWAYS_INLINE 53 | DestructTask(hipc::Allocator *alloc, 54 | const TaskNode &task_node, 55 | const DomainId &domain_id, 56 | TaskStateId &state_id) 57 | : DestroyTaskStateTask(alloc, task_node, domain_id, state_id) {} 58 | 59 | /** Create group */ 60 | HSHM_ALWAYS_INLINE 61 | u32 GetGroup(hshm::charbuf &group) { 62 | return TASK_UNORDERED; 63 | } 64 | }; 65 | 66 | } // namespace hrun::worch_proc_round_robin 67 | 68 | #endif // HRUN_WORCH_PROC_ROUND_ROBIN_TASKS_H__ 69 | -------------------------------------------------------------------------------- /hermes_adapters/filesystem/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | include_directories( 2 | ${CMAKE_SOURCE_DIR} 3 | ${HERMES_SRC_DIR} 4 | ${HERMES_ADAPTER_DIR} 5 | ${HERMES_IO_CLIENT_DIR} 6 | .) 7 | 8 | # Create the metadata manager singleton + FS base class 9 | add_library(hermes_fs_base SHARED 10 | ${CMAKE_CURRENT_SOURCE_DIR}/filesystem_mdm_singleton.cc) 11 | add_dependencies(hermes_fs_base 12 | hermes) 13 | target_link_libraries(hermes_fs_base 14 | MPI::MPI_CXX 15 | hermes) 16 | 17 | #----------------------------------------------------------------------------- 18 | # Add Target(s) to CMake Install 19 | #----------------------------------------------------------------------------- 20 | install( 21 | TARGETS 22 | hermes_fs_base 23 | EXPORT 24 | ${HERMES_EXPORTED_TARGETS} 25 | LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} 26 | ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} 27 | RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} 28 | ) 29 | install( 30 | FILES 31 | filesystem_io_client.h 32 | DESTINATION 33 | ${HERMES_INSTALL_INCLUDE_DIR} 34 | COMPONENT 35 | headers 36 | ) 37 | 38 | #----------------------------------------------------------------------------- 39 | # Export all exported targets to the build tree for use by parent project 40 | #----------------------------------------------------------------------------- 41 | set(HERMES_EXPORTED_LIBS 42 | hermes_fs_base 43 | ${HERMES_EXPORTED_LIBS}) 44 | if(NOT HERMES_EXTERNALLY_CONFIGURED) 45 | EXPORT ( 46 | TARGETS 47 | ${HERMES_EXPORTED_LIBS} 48 | FILE 49 | ${HERMES_EXPORTED_TARGETS}.cmake 50 | ) 51 | endif() 52 | 53 | #----------------------------------------------------------------------------- 54 | # Install headers 55 | #----------------------------------------------------------------------------- 56 | file(GLOB_RECURSE HERMES_HEADERS "*.h") 57 | install( 58 | FILES 59 | ${HERMES_HEADERS} 60 | DESTINATION 61 | ${CMAKE_INSTALL_PREFIX}/include/hermes_adapters/filesystem 62 | COMPONENT 63 | headers 64 | ) 65 | 66 | #----------------------------------------------------------------------------- 67 | # Add Target(s) to Coverage 68 | #----------------------------------------------------------------------------- 69 | if(HERMES_ENABLE_COVERAGE) 70 | set_coverage_flags(hermes_fs_base) 71 | endif() 72 | -------------------------------------------------------------------------------- /hrun/include/hrun/work_orchestrator/scheduler.h: -------------------------------------------------------------------------------- 1 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 | * Distributed under BSD 3-Clause license. * 3 | * Copyright by The HDF Group. * 4 | * Copyright by the Illinois Institute of Technology. * 5 | * All rights reserved. * 6 | * * 7 | * This file is part of Hermes. The full Hermes copyright notice, including * 8 | * terms governing use, modification, and redistribution, is contained in * 9 | * the COPYING file, which can be found at the top directory. If you do not * 10 | * have access to the file, you may request a copy from help@hdfgroup.org. * 11 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 12 | 13 | #ifndef HRUN_INCLUDE_HRUN_WORK_ORCHESTRATOR_SCHEDULER_H_ 14 | #define HRUN_INCLUDE_HRUN_WORK_ORCHESTRATOR_SCHEDULER_H_ 15 | 16 | #include "hrun/task_registry/task.h" 17 | 18 | namespace hrun { 19 | 20 | /** The set of methods in the admin task */ 21 | struct SchedulerMethod : public TaskMethod { 22 | TASK_METHOD_T kSchedule = TaskMethod::kLast; 23 | }; 24 | 25 | /** The task type used for scheduling */ 26 | struct ScheduleTask : public Task, TaskFlags { 27 | OUT hipc::pod_array ret_; 28 | 29 | /** SHM default constructor */ 30 | HSHM_ALWAYS_INLINE explicit 31 | ScheduleTask(hipc::Allocator *alloc) : Task(alloc) {} 32 | 33 | /** Emplace constructor */ 34 | HSHM_ALWAYS_INLINE explicit 35 | ScheduleTask(hipc::Allocator *alloc, 36 | const TaskNode &task_node, 37 | const DomainId &domain_id, 38 | TaskStateId &state_id) : Task(alloc) { 39 | // Initialize task 40 | task_node_ = task_node; 41 | lane_hash_ = 0; 42 | prio_ = TaskPrio::kLongRunning; 43 | task_state_ = state_id; 44 | method_ = SchedulerMethod::kSchedule; 45 | task_flags_.SetBits(TASK_LONG_RUNNING | TASK_REMOTE_DEBUG_MARK); 46 | SetPeriodMs(250); 47 | domain_id_ = domain_id; 48 | 49 | // Custom params 50 | ret_.construct(alloc, 1); 51 | } 52 | 53 | /** Create group */ 54 | HSHM_ALWAYS_INLINE 55 | u32 GetGroup(hshm::charbuf &group) { 56 | return TASK_UNORDERED; 57 | } 58 | }; 59 | 60 | } // namespace hrun 61 | 62 | #endif // HRUN_INCLUDE_HRUN_WORK_ORCHESTRATOR_SCHEDULER_H_ 63 | -------------------------------------------------------------------------------- /benchmark/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.10) 2 | project(hermes) 3 | 4 | set(CMAKE_CXX_STANDARD 17) 5 | 6 | #------------------------------------------------------------------------------ 7 | # Build Tests 8 | #------------------------------------------------------------------------------ 9 | 10 | add_executable(test_performance_exec 11 | ${TEST_MAIN}/main.cc 12 | test_init.cc 13 | test_latency.cc) 14 | add_dependencies(test_performance_exec 15 | ${Hermes_RUNTIME_DEPS} hermes) 16 | target_link_libraries(test_performance_exec 17 | ${Hermes_RUNTIME_LIBRARIES} hermes Catch2::Catch2 18 | MPI::MPI_CXX) 19 | 20 | #add_executable(test_performance_exec 21 | # ${TEST_MAIN}/main.cc 22 | # test_init.cc 23 | # test_latency.cc 24 | # test_zmq.cc 25 | #) 26 | #add_dependencies(test_performance_exec 27 | # ${Hermes_CLIENT_DEPS} hermes) 28 | #target_link_libraries(test_performance_exec 29 | # ${Hermes_CLIENT_LIBRARIES} hermes Catch2::Catch2 30 | # MPI::MPI_CXX ${ZMQ_LIBRARIES}) 31 | 32 | add_executable(hermes_api_bench 33 | hermes_api_bench.cc) 34 | add_dependencies(hermes_api_bench 35 | ${Hermes_CLIENT_DEPS} hermes) 36 | target_link_libraries(hermes_api_bench 37 | ${Hermes_CLIENT_LIBRARIES} hermes 38 | Catch2::Catch2 MPI::MPI_CXX) 39 | 40 | #------------------------------------------------------------------------------ 41 | # Test Cases 42 | #------------------------------------------------------------------------------ 43 | 44 | # STRING TESTS 45 | #add_test(NAME test_performance COMMAND 46 | # ${CMAKE_BINARY_DIR}/bin/test_performance_exec "TestIpc") 47 | 48 | #------------------------------------------------------------------------------ 49 | # Install Targets 50 | #------------------------------------------------------------------------------ 51 | install(TARGETS 52 | test_performance_exec 53 | hermes_api_bench 54 | EXPORT 55 | ${HERMES_EXPORTED_TARGETS} 56 | LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} 57 | ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} 58 | RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR}) 59 | 60 | #----------------------------------------------------------------------------- 61 | # Coverage 62 | #----------------------------------------------------------------------------- 63 | if(HERMES_ENABLE_COVERAGE) 64 | set_coverage_flags(test_performance_exec) 65 | set_coverage_flags(hermes_api_bench) 66 | endif() 67 | -------------------------------------------------------------------------------- /include/hermes/dpe/dpe_factory.h: -------------------------------------------------------------------------------- 1 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 | * Distributed under BSD 3-Clause license. * 3 | * Copyright by The HDF Group. * 4 | * Copyright by the Illinois Institute of Technology. * 5 | * All rights reserved. * 6 | * * 7 | * This file is part of Hermes. The full Hermes copyright notice, including * 8 | * terms governing use, modification, and redistribution, is contained in * 9 | * the COPYING file, which can be found at the top directory. If you do not * 10 | * have access to the file, you may request a copy from help@hdfgroup.org. * 11 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 12 | 13 | #ifndef HERMES_SRC_DPE_DATA_PLACEMENT_ENGINE_FACTORY_H_ 14 | #define HERMES_SRC_DPE_DATA_PLACEMENT_ENGINE_FACTORY_H_ 15 | 16 | #include "minimize_io_time.h" 17 | #include "random.h" 18 | #include "round_robin.h" 19 | #include "dpe.h" 20 | #include "hermes/hermes.h" 21 | 22 | namespace hermes { 23 | 24 | /** 25 | A class to represent Data Placement Engine Factory 26 | */ 27 | class DpeFactory { 28 | public: 29 | /** 30 | * return a pointer to data placement engine given a policy type. 31 | * This uses factory pattern. 32 | * 33 | * @param[in] type a placement policy type to be used by the 34 | * data placement engine factory. 35 | * @return pointer to DataPlacementEngine given \a type PlacementPolicy. 36 | */ 37 | static Dpe* Get(PlacementPolicy type) { 38 | if (type == PlacementPolicy::kNone) { 39 | type = HERMES_SERVER_CONF.dpe_.default_policy_; 40 | } 41 | switch (type) { 42 | case PlacementPolicy::kRandom: { 43 | return hshm::EasySingleton::GetInstance(); 44 | } 45 | case PlacementPolicy::kRoundRobin: { 46 | return hshm::EasySingleton::GetInstance(); 47 | } 48 | case PlacementPolicy::kMinimizeIoTime: { 49 | return hshm::EasySingleton::GetInstance(); 50 | } 51 | case PlacementPolicy::kNone: 52 | default: { 53 | HELOG(kFatal, "PlacementPolicy not implemented") 54 | return NULL; 55 | } 56 | } 57 | } 58 | }; 59 | 60 | } // namespace hermes 61 | 62 | #endif // HERMES_SRC_DPE_DATA_PLACEMENT_ENGINE_FACTORY_H_ 63 | -------------------------------------------------------------------------------- /hrun/tasks_required/worch_queue_round_robin/include/worch_queue_round_robin/worch_queue_round_robin_tasks.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by lukemartinlogan on 8/14/23. 3 | // 4 | 5 | #ifndef HRUN_WORCH_QUEUE_ROUND_ROBIN_TASKS_H_ 6 | #define HRUN_WORCH_QUEUE_ROUND_ROBIN_TASKS_H_ 7 | 8 | #include "hrun/api/hrun_client.h" 9 | #include "hrun/task_registry/task_lib.h" 10 | #include "hrun_admin/hrun_admin.h" 11 | #include "hrun/work_orchestrator/scheduler.h" 12 | #include "hrun/queue_manager/queue_manager_client.h" 13 | #include "proc_queue/proc_queue.h" 14 | 15 | namespace hrun::worch_queue_round_robin { 16 | 17 | #include "hrun/hrun_namespace.h" 18 | 19 | /** The set of methods in the worch task */ 20 | typedef SchedulerMethod Method; 21 | 22 | /** 23 | * A task to create worch_queue_round_robin 24 | * */ 25 | using hrun::Admin::CreateTaskStateTask; 26 | struct ConstructTask : public CreateTaskStateTask { 27 | /** SHM default constructor */ 28 | HSHM_ALWAYS_INLINE explicit 29 | ConstructTask(hipc::Allocator *alloc) : CreateTaskStateTask(alloc) {} 30 | 31 | /** Emplace constructor */ 32 | HSHM_ALWAYS_INLINE 33 | ConstructTask(hipc::Allocator *alloc, 34 | const TaskNode &task_node, 35 | const DomainId &domain_id, 36 | const std::string &state_name, 37 | const TaskStateId &id, 38 | const std::vector &queue_info) 39 | : CreateTaskStateTask(alloc, task_node, domain_id, state_name, 40 | "worch_queue_round_robin", id, queue_info) { 41 | } 42 | 43 | /** Destructor */ 44 | HSHM_ALWAYS_INLINE 45 | ~ConstructTask() {} 46 | }; 47 | 48 | /** A task to destroy worch_queue_round_robin */ 49 | using hrun::Admin::DestroyTaskStateTask; 50 | struct DestructTask : public DestroyTaskStateTask { 51 | /** SHM default constructor */ 52 | HSHM_ALWAYS_INLINE explicit 53 | DestructTask(hipc::Allocator *alloc) : DestroyTaskStateTask(alloc) {} 54 | 55 | /** Emplace constructor */ 56 | HSHM_ALWAYS_INLINE 57 | DestructTask(hipc::Allocator *alloc, 58 | const TaskNode &task_node, 59 | TaskStateId &state_id, 60 | const DomainId &domain_id) 61 | : DestroyTaskStateTask(alloc, task_node, domain_id, state_id) {} 62 | 63 | /** Create group */ 64 | HSHM_ALWAYS_INLINE 65 | u32 GetGroup(hshm::charbuf &group) { 66 | return TASK_UNORDERED; 67 | } 68 | }; 69 | 70 | } // namespace hrun::worch_queue_round_robin 71 | 72 | #endif // HRUN_WORCH_QUEUE_ROUND_ROBIN_TASKS_H_ 73 | --------------------------------------------------------------------------------