├── .clang-format ├── .devcontainer ├── devcontainer.Dockerfile ├── devcontainer.json ├── post-attach.sh ├── post-create.sh └── post-start.sh ├── .docker ├── base.Dockerfile ├── dev.Dockerfile ├── dev.Dockerfile.dockerignore ├── dev_base.Dockerfile ├── local.Dockerfile ├── local.Dockerfile.dockerignore ├── publish_dev_base.sh └── run_dev_base.sh ├── .github ├── CODEOWNERS ├── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── feature_request.md │ ├── install_problem.md │ └── question.md ├── pull_request_template.md └── workflows │ ├── clang-format-check.yml │ ├── clang-format-fix.yml │ ├── dependencies-linux.sh │ ├── dependencies-macos.sh │ ├── macos.yml │ ├── nersc.yml │ ├── store-metrics.py │ ├── ubuntu-cache.yml │ └── ubuntu-no-cache.yaml ├── .gitignore ├── .gitlab-ci.yml ├── .readthedocs.yaml ├── Authors ├── CMake ├── FindDRC.cmake ├── FindJULIA.cmake ├── FindMERCURY.cmake ├── FindUUID.cmake ├── pdc-config-version.cmake.in ├── pdc-config.cmake.build.in └── pdc-config.cmake.install.in ├── CMakeLists.txt ├── COPYING ├── CTestConfig.cmake ├── Legal.txt ├── README.md ├── benchmark └── idioms │ ├── CMakeLists.txt │ ├── cjson │ ├── cJSON.c │ └── cJSON.h │ └── llsm_metadata_bench.c ├── docs ├── Makefile ├── _static │ ├── .DS_Store │ ├── css │ │ └── pdc.css │ ├── custom.css │ ├── image │ │ └── pdc.png │ └── pdc_logo.png ├── container_hashtable_checkpoint.png ├── data_hashtable_checkpoint.png ├── data_server_checkpoint.png ├── make.bat ├── pdc_metadata_flow.png ├── pdc_plots.pptx ├── pdc_region_transfer_request_flow.png ├── readme.md ├── requirements.txt └── source │ ├── Doxyfile.in │ ├── api.rst │ ├── assumptions.rst │ ├── conf.py │ ├── definitions.rst │ ├── developer-notes.rst │ ├── documentation │ ├── analysis.rst │ ├── objects.rst │ ├── query.rst │ ├── regions.rst │ └── transformation.rst │ ├── examples.rst │ ├── futurework.rst │ ├── getting_started.rst │ ├── hdf5vol.rst │ ├── hellopdcexample.rst │ ├── images │ ├── .DS_Store │ ├── pdc.png │ └── pdclogo.png │ ├── index.rst │ ├── inflightanalysis.rst │ ├── introduction.rst │ ├── performance.rst │ └── tools.rst ├── examples ├── C_plus_plus_example │ ├── H5Timing.cc │ ├── H5Timing.h │ ├── Makefile │ ├── multidataset_plugin.cc │ ├── multidataset_plugin.h │ └── region_transfer_1D_append.cc ├── Makefile ├── README.md ├── bdcats.c ├── bdcats_batch.c ├── bdcats_old.c ├── haccio.c ├── haccio_v2.c ├── llsm │ ├── .gitignore │ ├── CMakeLists.txt │ ├── LLSM_IMPORTER.md │ ├── llsm_aux │ │ ├── csvReader.c │ │ ├── csvReader.h │ │ ├── parallelReadTiff.c │ │ ├── parallelReadTiff.h │ │ ├── pdc_list.c │ │ └── pdc_list.h │ └── llsm_importer.c ├── mpi_test.sh ├── multiple_mpi_test.sh ├── obj_get_data.c ├── pdc_init.c ├── read_write_col_perf.c ├── read_write_perf.c ├── region_transfer_2D.c ├── region_transfer_3D.c ├── run_checkpoint_restart_test.sh ├── run_multiple_test.sh ├── run_test.sh ├── script_cori_shared.sh ├── tileio.c ├── tileio_v2.c ├── vpicio.c ├── vpicio_batch.c ├── vpicio_batch_results.png ├── vpicio_dynamic_partition_4.txt ├── vpicio_local_partition_4.txt ├── vpicio_object_partition_4.txt ├── vpicio_old.c └── vpicio_static_partition_4.txt ├── license.txt ├── parser_pdc_log.py ├── pdc_config.h.in ├── pdc_config_sys.h.cmake ├── pdc_naming_policy.txt ├── pdc_start ├── scripts ├── dart_attr_dist_test │ ├── clean.sh │ ├── gen_script.sh │ ├── submit.sh │ └── template.sh ├── kvtag_add_get_benchmark │ └── perlmutter │ │ ├── clean.sh │ │ ├── gen_scripts.sh │ │ ├── submit.sh │ │ └── template.sh ├── kvtag_add_get_scale │ └── perlmutter │ │ ├── clean.sh │ │ ├── gen_scripts.sh │ │ ├── submit.sh │ │ └── template.sh ├── kvtag_affix_query_scale │ ├── clean.sh │ ├── gen_script.sh │ ├── submit.sh │ └── template.sh ├── kvtag_query_scale │ ├── clean.sh │ ├── gen_script.sh │ ├── submit.sh │ └── template.sh ├── kvtag_range_query_scale │ ├── clean.sh │ ├── gen_script.sh │ ├── submit.sh │ └── template.sh ├── llsm_idioms_bench │ ├── clean.sh │ ├── gen_script.sh │ ├── submit.sh │ └── template.sh └── llsm_importer │ ├── clean.sh │ ├── gen_script.sh │ ├── submit.sh │ └── template.sh └── src ├── api ├── CMakeLists.txt ├── close_server.c ├── include │ ├── pdc.h │ └── pdc_client_connect.h ├── pdc.c ├── pdc_analysis │ ├── include │ │ ├── pdc_analysis.h │ │ ├── pdc_analysis_and_transforms_common.h │ │ ├── pdc_analysis_pkg.h │ │ └── pdc_hist_pkg.h │ ├── pdc_analysis.c │ ├── pdc_analysis_and_transforms_connect.c │ ├── pdc_analysis_common.c │ └── pdc_hist_pkg.c ├── pdc_client_connect.c ├── pdc_meta │ ├── include │ │ └── pdc_metadata_client.h │ └── pdc_metadata_client.c ├── pdc_obj │ ├── include │ │ ├── pdc_cont.h │ │ ├── pdc_cont_pkg.h │ │ ├── pdc_dt_conv.h │ │ ├── pdc_mpi.h │ │ ├── pdc_obj.h │ │ ├── pdc_obj_pkg.h │ │ ├── pdc_prop.h │ │ └── pdc_prop_pkg.h │ ├── pdc_cont.c │ ├── pdc_dt_conv.c │ ├── pdc_mpi.c │ ├── pdc_obj.c │ └── pdc_prop.c ├── pdc_query │ ├── include │ │ └── pdc_query.h │ └── pdc_query.c ├── pdc_region │ ├── include │ │ ├── pdc_region.h │ │ └── pdc_region_pkg.h │ ├── pdc_region.c │ └── pdc_region_transfer.c └── pdc_transform │ ├── include │ ├── pdc_transform.h │ ├── pdc_transforms_common.h │ └── pdc_transforms_pkg.h │ ├── pdc_transform.c │ └── pdc_transforms_common.c ├── commons ├── CMakeLists.txt ├── Readme.md ├── collections │ ├── Readme.md │ ├── art.c │ ├── include │ │ ├── art.h │ │ ├── pdc_compare.h │ │ ├── pdc_deque.h │ │ ├── pdc_hash.h │ │ ├── pdc_hash_table.h │ │ ├── pdc_set.h │ │ ├── pdc_stack.h │ │ └── pdc_vector.h │ ├── libhl │ │ ├── include │ │ │ ├── atomic_defs.h │ │ │ ├── comparators.h │ │ │ ├── linklist.h │ │ │ └── rbtree.h │ │ ├── linklist.c │ │ └── rbtree.c │ ├── pdc_compare.c │ ├── pdc_deque.c │ ├── pdc_hash.c │ ├── pdc_hash_table.c │ ├── pdc_set.c │ ├── pdc_set_test.c │ ├── pdc_stack.c │ └── pdc_vector.c ├── file │ ├── Readme.md │ ├── bin_file_ops.c │ ├── common_io.c │ └── include │ │ ├── bin_file_ops.h │ │ └── common_io.h ├── generic │ ├── include │ │ └── pdc_generic.h │ └── pdc_generic.c ├── include │ └── pdc_public.h ├── index │ ├── Readme.md │ └── dart │ │ ├── dart_algo.c │ │ ├── dart_core.c │ │ ├── dart_core_test.c │ │ ├── dart_math.c │ │ ├── dart_utils.c │ │ ├── include │ │ ├── dart_algo.h │ │ ├── dart_core.h │ │ ├── dart_math.h │ │ └── dart_utils.h │ │ └── index │ │ └── idioms │ │ ├── idioms_local_index.c │ │ ├── idioms_local_index_test.c │ │ ├── idioms_persistence.c │ │ └── include │ │ ├── idioms_local_index.h │ │ └── idioms_persistence.h ├── logging │ ├── include │ │ └── pdc_logger.h │ └── pdc_logger.c ├── profiling │ ├── include │ │ ├── pdc_hashtab.h │ │ └── pdc_stack_ops.h │ ├── pdc_hashtab.c │ └── pdc_stack_ops.c ├── query │ ├── include │ │ └── qc_parser.h │ ├── qc_parser.c │ └── qc_parser_test.c ├── serde │ └── bulki │ │ ├── bulki.c │ │ ├── bulki_endianness_util.c │ │ ├── bulki_serde.c │ │ ├── bulki_serde_test.c │ │ ├── bulki_vle_util.c │ │ └── include │ │ ├── bulki.h │ │ ├── bulki_endianness_util.h │ │ ├── bulki_serde.h │ │ └── bulki_vle_util.h └── utils │ ├── Readme.md │ ├── include │ ├── pdc_id_pkg.h │ ├── pdc_linkedlist.h │ ├── pdc_malloc.h │ ├── pdc_private.h │ ├── pdc_timing.h │ ├── query_utils.h │ ├── string_utils.h │ ├── thpool.h │ └── timer_utils.h │ ├── pdc_malloc.c │ ├── pdc_timing.c │ ├── query_utils.c │ ├── query_utils_test.c │ ├── string_utils.c │ ├── thpool.c │ └── timer_utils.c ├── server ├── CMakeLists.txt ├── dablooms │ ├── LICENSE │ ├── README.md │ ├── pdc_dablooms.c │ ├── pdc_dablooms.h │ ├── pdc_murmur.c │ └── pdc_murmur.h ├── include │ ├── pdc_client_server_common.h │ ├── pdc_server.h │ ├── pdc_server_common.h │ ├── pdc_server_metadata.h │ ├── pdc_server_metadata_index.h │ └── pdc_utlist.h ├── pdc_client_server_common.c ├── pdc_server.c ├── pdc_server_analysis │ ├── include │ │ └── pdc_server_analysis.h │ └── pdc_server_analysis.c ├── pdc_server_main.c ├── pdc_server_metadata.c ├── pdc_server_metadata_index.c ├── pdc_server_metadata_index_test.c └── pdc_server_region │ ├── include │ ├── pdc_server_data.h │ ├── pdc_server_region_cache.h │ ├── pdc_server_region_transfer.h │ └── pdc_server_region_transfer_metadata_query.h │ ├── pdc_server_data.c │ ├── pdc_server_region_cache.c │ ├── pdc_server_region_request_handler.h │ ├── pdc_server_region_transfer.c │ └── pdc_server_region_transfer_metadata_query.c ├── tests ├── CMakeLists.txt ├── bdcats.c ├── bdcats_old.c ├── bdcats_v2.c ├── blocked_lock.c ├── buf_map_mpi_v2.c ├── buf_obj_map.c ├── buf_obj_map_mpi.c ├── client_server.c ├── consistency_semantics.c ├── cont_add_del.c ├── cont_del.c ├── cont_getid.c ├── cont_info.c ├── cont_iter.c ├── cont_iter_mt.c ├── cont_life.c ├── cont_life_mt.c ├── cont_tags.c ├── create_cont.c ├── create_cont_coll.c ├── create_obj.c ├── create_obj_coll.c ├── create_obj_scale.c ├── create_prop.c ├── create_region.c ├── dart_algo_sim.c ├── dart_attr_dist_test.c ├── dart_func_test.c ├── dart_test.c ├── data_rw_test.c ├── data_server_meta.c ├── data_server_meta_test.c ├── data_server_read.c ├── data_server_read_multi.c ├── data_server_read_vpic.c ├── data_server_read_vpic_multits.c ├── data_server_read_vpic_spatial.c ├── data_server_read_vpic_spatial_multits.c ├── data_server_write.c ├── data_server_write_multi.c ├── data_server_write_vpic.c ├── data_server_write_vpic_multits.c ├── delete_obj.c ├── delete_obj_scale.c ├── dt_conv.c ├── dup_prop.c ├── gen_fastbit_idx.c ├── hash_table.c ├── helper │ ├── JuliaHelper.jl │ ├── include │ │ └── julia_helper_loader.h │ └── julia_helper_loader.c ├── idioms_ci_test.c ├── import_vpic.c ├── init_only.c ├── kvtag_add_get.c ├── kvtag_add_get_benchmark.c ├── kvtag_add_get_scale.c ├── kvtag_affix_query_scale.c ├── kvtag_get.c ├── kvtag_query.c ├── kvtag_range_query_scale.c ├── list_all.c ├── llsm_idioms_bench.c ├── mpi_test.sh ├── obj_buf.c ├── obj_del.c ├── obj_dim.c ├── obj_get_data.c ├── obj_info.c ├── obj_iter.c ├── obj_life.c ├── obj_lock.c ├── obj_map.c ├── obj_put_data.c ├── obj_round_robin_io.c ├── obj_round_robin_io_all.c ├── obj_tags.c ├── obj_transformation.c ├── open_cont.c ├── open_existing_obj.c ├── open_obj.c ├── open_obj_round_robin.c ├── pdc_analysis_lib.c ├── pdc_init.c ├── pdc_transforms_lib.c ├── producer_waitall.c ├── query_data.c ├── query_metadata_agg.c ├── query_vpic.c ├── query_vpic_bin_sds1_nopreload.c ├── query_vpic_bin_sds1_preload.c ├── query_vpic_bin_sds_nopreload.c ├── query_vpic_bin_sds_preload.c ├── query_vpic_create_data.c ├── query_vpic_exyz_nopreload.c ├── query_vpic_exyz_preload.c ├── query_vpic_multi.c ├── query_vpic_multi_nopreload.c ├── query_vpic_multi_nopreload1.c ├── query_vpic_multi_preload.c ├── read_obj.c ├── read_obj_shared.c ├── read_write_col_perf.c ├── read_write_perf.c ├── region_obj_map.c ├── region_obj_map_2D.c ├── region_obj_map_2D_partial.c ├── region_obj_map_3D.c ├── region_obj_map_3D_partial.c ├── region_obj_map_merge.c ├── region_obj_map_overlap.c ├── region_obj_map_overlap_2D.c ├── region_obj_map_overlap_3D.c ├── region_obj_map_partial.c ├── region_transfer.c ├── region_transfer_2D.c ├── region_transfer_2D_partial.c ├── region_transfer_2D_skewed.c ├── region_transfer_3D.c ├── region_transfer_3D_partial.c ├── region_transfer_3D_skewed.c ├── region_transfer_all.c ├── region_transfer_all_2D.c ├── region_transfer_all_3D.c ├── region_transfer_all_append.c ├── region_transfer_all_append_2D.c ├── region_transfer_all_append_3D.c ├── region_transfer_all_fold.c ├── region_transfer_all_split_wait.c ├── region_transfer_all_test.c ├── region_transfer_merge.c ├── region_transfer_overlap.c ├── region_transfer_overlap_2D.c ├── region_transfer_overlap_3D.c ├── region_transfer_partial.c ├── region_transfer_query.c ├── region_transfer_read_only.c ├── region_transfer_set_dims.c ├── region_transfer_set_dims_2D.c ├── region_transfer_set_dims_3D.c ├── region_transfer_skewed.c ├── region_transfer_status.c ├── region_transfer_write_only.c ├── region_transfer_write_read.c ├── run_checkpoint_restart_test.sh ├── run_multiple_mpi_test.sh ├── run_multiple_test.sh ├── run_test.sh ├── search_obj.c ├── search_obj_scale.c ├── set_prop.c ├── stat_obj.c ├── update_obj.c ├── vpicio.c ├── vpicio_mts.c ├── vpicio_old.c ├── vpicio_v2.c └── write_obj_shared.c ├── tools ├── CMakeLists.txt ├── cjson │ ├── cJSON.c │ └── cJSON.h ├── fs │ ├── fs_ops.c │ └── fs_ops.h ├── hdf5_access_eqsim.c ├── meta_json │ ├── Readme.md │ ├── llsm_converter │ │ ├── __init__.py │ │ ├── llsm_csv_metadata_extractor.py │ │ ├── requirement.txt │ │ └── validator │ │ │ ├── metadata_json_schema.json │ │ │ └── metadata_schema_validator.py │ ├── metadata_json_importer.c │ ├── metadata_json_importer.h │ ├── metadata_json_printer.c │ ├── metadata_json_printer.h │ ├── metadata_json_processor.h │ └── test.json ├── metadata_json_loader.c ├── pdc_access_eqsim.c ├── pdc_export.c ├── pdc_import.c ├── pdc_import_eqsim.c ├── pdc_ls.c └── pdc_query_eqsim.c └── utils ├── include └── pdc_interface.h ├── pdc_interface.c └── pdc_region_utils.c /.clang-format: -------------------------------------------------------------------------------- 1 | --- 2 | Language: Cpp 3 | BasedOnStyle: LLVM 4 | AlignConsecutiveAssignments: true 5 | #llvm11: AlignConsecutiveBitFields: false 6 | AlignConsecutiveDeclarations: true 7 | AlignConsecutiveMacros: true 8 | #llvm10-11: AlignOperands: true - Align 9 | #llvm11: AllowShortEnumsOnASingleLine: true 10 | AllowShortFunctionsOnASingleLine: None 11 | AlwaysBreakAfterReturnType: AllDefinitions 12 | BraceWrapping: 13 | AfterFunction: true 14 | #llvm10-11: AfterControlStatement: false - Never 15 | BeforeCatch: true 16 | BeforeElse: true 17 | #llvm11: BeforeLambdaBody: false 18 | #llvm11: BeforeWhile: false 19 | BreakBeforeBraces: Stroustrup 20 | BreakAfterJavaFieldAnnotations: true 21 | BreakStringLiterals: true 22 | ColumnLimit: 110 # Update $max_trace_macro_line_len in bin/trace also 23 | IncludeCategories: 24 | - Regex: '^"(llvm|llvm-c|clang|clang-c)/' 25 | Priority: 3 26 | SortPriority: 0 27 | - Regex: '^(<|"(gtest|gmock|isl|json)/)' 28 | Priority: 4 29 | SortPriority: 0 30 | - Regex: '.*' 31 | Priority: 0 32 | SortPriority: 0 33 | - Regex: '^PDC*.*' 34 | Priority: 1 35 | SortPriority: 0 36 | - Regex: 'private.*' 37 | Priority: 2 38 | SortPriority: 0 39 | IncludeIsMainRegex: '(public)?$' 40 | IndentCaseLabels: true 41 | #llvm11: IndentCaseBlocks: false 42 | IndentGotoLabels: false 43 | #llvm11: IndentExternBlock: AfterExternBlock 44 | IndentWidth: 4 45 | #llvm11: InsertTrailingCommas: None 46 | # MacroBlockBegin: "^BEGIN_FUNC" 47 | # MacroBlockEnd: "^END_FUNC" 48 | ObjCBlockIndentWidth: 4 49 | #llvm11: ObjCBreakBeforeNestedBlockParam: true 50 | ReflowComments: true 51 | SortIncludes: false 52 | StatementMacros: 53 | - FUNC_ENTER 54 | - FUNC_LEAVE 55 | - PGOTO_DONE 56 | - PGOTO_ERROR 57 | #llvm10: TypenameMacros: 58 | #llvm10: - STACK_OF 59 | #llvm10: - LIST 60 | #llvm11: WhitespaceSensitiveMacros: 61 | #llvm11: - STRINGIZE 62 | #llvm11: - PP_STRINGIZE 63 | ... 64 | 65 | -------------------------------------------------------------------------------- /.devcontainer/devcontainer.Dockerfile: -------------------------------------------------------------------------------- 1 | # Note: Run `docker build -f .devcontainer/Dockerfile -t pdc:latest .` from the root directory of the repository to build the docker image. 2 | 3 | # Use Ubuntu Jammy (latest LTS) as the base image 4 | # FROM ubuntu:jammy 5 | FROM hpcio/pdc-dev-base:latest 6 | 7 | RUN rm -rf $PDC_SRC_DIR && \ 8 | rm -rf $PDC_DIR 9 | -------------------------------------------------------------------------------- /.devcontainer/post-attach.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | 4 | WORK_SPACE_INITIALIZED_FILE=/workspaces/.workspace_initialized 5 | 6 | if ! [ -f $WORK_SPACE_INITIALIZED_FILE ]; then 7 | /bin/bash /workspaces/pdc/.devcontainer/post-start.sh 8 | watch -t -n 5 'echo "Press Ctrl+C when there is no building processes."; echo "Number of initial PDC building processes:"; ps -ef | grep make | grep -v -c grep' 9 | else 10 | echo "Welcome Back!" 11 | fi 12 | 13 | /bin/bash -------------------------------------------------------------------------------- /.devcontainer/post-create.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | 4 | WORK_SPACE_INITIALIZED_FILE=/workspaces/.workspace_initialized 5 | 6 | if ! [ -f $WORK_SPACE_INITIALIZED_FILE ]; then 7 | touch $WORK_SPACE_INITIALIZED_FILE 8 | echo "First time to create workspace, start to install PDC" 9 | else 10 | echo "Workspace already initialized, skip the installation" 11 | exit 0 12 | fi 13 | 14 | rm -rf $PDC_SRC_DIR 15 | rm -rf $PDC_DIR 16 | 17 | 18 | ln -s /workspaces/pdc $(dirname $PDC_SRC_DIR) 19 | 20 | mkdir -p /workspaces/source 21 | ln -s $PDC_SRC_DIR /workspaces/source/pdc 22 | 23 | mkdir -p /workspaces/install/pdc 24 | ln -s /workspaces/install/pdc $(dirname $PDC_SRC_DIR) 25 | 26 | # Build and install PDC 27 | export PDC_CMAKE_FLAGS="-DBUILD_MPI_TESTING=ON -DBUILD_SHARED_LIBS=ON -DBUILD_TOOLS=OFF -DBUILD_TESTING=ON -DCMAKE_INSTALL_PREFIX=$PDC_DIR -DPDC_ENABLE_MPI=ON -DMERCURY_DIR=$MERCURY_DIR -DCMAKE_C_COMPILER=mpicc -DMPI_RUN_CMD=mpiexec " 28 | 29 | cd $PDC_SRC_DIR 30 | rm -rf build && mkdir -p build 31 | 32 | 33 | cd ${PDC_SRC_DIR}/build 34 | cmake $PDC_CMAKE_FLAGS ../ 2>&1 > ./cmake_config.log || echo "ignoring cmake config error and proceed" 35 | make -j && make install 36 | 37 | # Set the environment variables 38 | export LD_LIBRARY_PATH="$PDC_DIR/lib:$LD_LIBRARY_PATH" 39 | export PATH="$PDC_DIR/include:$PDC_DIR/lib:$PATH" 40 | echo 'export LD_LIBRARY_PATH=$PDC_DIR/lib:$LD_LIBRARY_PATH' >> $WORK_SPACE/pdc_env.sh 41 | echo 'export PATH=$PDC_DIR/include:$PDC_DIR/lib:$PATH' >> $WORK_SPACE/pdc_env.sh 42 | -------------------------------------------------------------------------------- /.devcontainer/post-start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | nohup /bin/bash /workspaces/pdc/.devcontainer/post-create.sh 2>&1 > /workspaces/pdc_install.out & 4 | 5 | echo "Wait for 10 seconds for the building processes to start." 6 | sleep 10s 7 | -------------------------------------------------------------------------------- /.docker/dev.Dockerfile: -------------------------------------------------------------------------------- 1 | # Note: Run `docker build -f .docker/Dockerfile -t pdc:latest .` from the root directory of the repository to build the docker image. 2 | 3 | # Use Ubuntu Jammy (latest LTS) as the base image 4 | FROM zhangwei217245/pdc_dev_base:latest 5 | 6 | # Build and install PDC 7 | ENV PDC_CMAKE_FLAGS="-DBUILD_MPI_TESTING=ON -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=ON -DCMAKE_INSTALL_PREFIX=$PDC_DIR -DPDC_ENABLE_MPI=ON -DMERCURY_DIR=$MERCURY_DIR -DCMAKE_C_COMPILER=mpicc -DMPI_RUN_CMD=mpiexec " 8 | 9 | 10 | WORKDIR $PDC_SRC_DIR 11 | RUN rm -rf build && \ 12 | mkdir -p build 13 | 14 | # COPY ../ ${PDC_SRC_DIR} 15 | # RUN ls -l $PDC_SRC_DIR 16 | 17 | WORKDIR ${PDC_SRC_DIR}/build 18 | RUN cmake $PDC_CMAKE_FLAGS ../ 2>&1 > ./cmake_config.log || echo "ignoring cmake config error and proceed" && \ 19 | make -j && make install 20 | 21 | # Set the environment variables 22 | ENV LD_LIBRARY_PATH="$PDC_DIR/lib:$LD_LIBRARY_PATH" 23 | ENV PATH="$PDC_DIR/include:$PDC_DIR/lib:$PATH" 24 | RUN echo 'export LD_LIBRARY_PATH=$PDC_DIR/lib:$LD_LIBRARY_PATH' >> $WORK_SPACE/pdc_env.sh && \ 25 | echo 'export PATH=$PDC_DIR/include:$PDC_DIR/lib:$PATH' >> $WORK_SPACE/pdc_env.sh 26 | 27 | 28 | # WORKDIR $PDC_SRC_DIR/build 29 | # RUN ctest -------------------------------------------------------------------------------- /.docker/dev.Dockerfile.dockerignore: -------------------------------------------------------------------------------- 1 | # Exclude files and directories from the Docker build context 2 | !/.git/ 3 | -------------------------------------------------------------------------------- /.docker/local.Dockerfile: -------------------------------------------------------------------------------- 1 | # Note: Run `docker build -f .docker/Dockerfile -t pdc:latest .` from the root directory of the repository to build the docker image. 2 | 3 | # Use Ubuntu Jammy (latest LTS) as the base image 4 | FROM zhangwei217245/pdc_dev_base:latest 5 | 6 | # Build and install PDC 7 | ENV PDC_CMAKE_FLAGS="-DBUILD_MPI_TESTING=ON -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=ON -DCMAKE_INSTALL_PREFIX=$PDC_DIR -DPDC_ENABLE_MPI=ON -DMERCURY_DIR=$MERCURY_DIR -DCMAKE_C_COMPILER=mpicc -DMPI_RUN_CMD=mpiexec " 8 | 9 | 10 | WORKDIR $PDC_SRC_DIR 11 | RUN rm -rf build && \ 12 | mkdir -p build 13 | 14 | # COPY ../ ${PDC_SRC_DIR} 15 | # RUN ls -l $PDC_SRC_DIR 16 | 17 | WORKDIR ${PDC_SRC_DIR}/build 18 | RUN cmake $PDC_CMAKE_FLAGS ../ 2>&1 > ./cmake_config.log || echo "ignoring cmake config error and proceed" && \ 19 | make -j && make install 20 | 21 | # Set the environment variables 22 | ENV LD_LIBRARY_PATH="$PDC_DIR/lib:$LD_LIBRARY_PATH" 23 | ENV PATH="$PDC_DIR/include:$PDC_DIR/lib:$PATH" 24 | RUN echo 'export LD_LIBRARY_PATH=$PDC_DIR/lib:$LD_LIBRARY_PATH' >> $WORK_SPACE/pdc_env.sh && \ 25 | echo 'export PATH=$PDC_DIR/include:$PDC_DIR/lib:$PATH' >> $WORK_SPACE/pdc_env.sh 26 | 27 | 28 | # WORKDIR $PDC_SRC_DIR/build 29 | # RUN ctest -------------------------------------------------------------------------------- /.docker/local.Dockerfile.dockerignore: -------------------------------------------------------------------------------- 1 | # Exclude files and directories from the Docker build context 2 | !/.git/ 3 | -------------------------------------------------------------------------------- /.docker/publish_dev_base.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Check if $1 is unset or empty 4 | if [ "$#" -lt 2 ]; then 5 | echo "./publish_dev_base.sh " 6 | echo "For building and publishing image, just ignore the second argument. For processing manifest, set the second argument to 1." 7 | echo "For best performance, you need to run this script on the same architecture as the target image. But you may specify the second argument to 1 on the machine where fastest network is available to process the manifest." 8 | exit 1 9 | fi 10 | 11 | IMG_NS=$1 12 | VERSION=$2 13 | arch=$(uname -m) 14 | ARCH_CODE="" 15 | 16 | case $arch in 17 | x86_64) 18 | ARCH_CODE="amd64" 19 | echo "You are running on x86_64 (AMD64) architecture." 20 | ;; 21 | arm64 | aarch64) 22 | ARCH_CODE="arm64v8" 23 | echo "You are running on ARM64 (AArch64) architecture." 24 | ;; 25 | i386 | i686) 26 | ARCH_CODE="i386" 27 | echo "You are running on x86 (32-bit) architecture." 28 | ;; 29 | arm*) 30 | ARCH_CODE="arm32v7" 31 | echo "You are running on ARM (32-bit) architecture." 32 | ;; 33 | ppc64le) 34 | ARCH_CODE="ppc64le" 35 | echo "You are running on PowerPC (64-bit little-endian) architecture." 36 | ;; 37 | s390x) 38 | ARCH_CODE="s390x" 39 | echo "You are running on IBM Z (s390x) architecture." 40 | ;; 41 | *) 42 | echo "Unknown or unsupported architecture: $arch" 43 | exit 1 44 | ;; 45 | esac 46 | 47 | if [ -z "$3" ] || [ "$3" -eq 0 ]; then 48 | docker build -t ${IMG_NS}/pdc-dev-base:${VERSION}-${ARCH_CODE} -f .docker/dev_base.Dockerfile --build-arg ARCH=${ARCH_CODE}/ --build-arg ARCH_CODE=${ARCH_CODE} . 49 | docker push ${IMG_NS}/pdc-dev-base:${VERSION}-${ARCH_CODE} 50 | exit 0 51 | else 52 | echo "Processing manifest..." 53 | # Process manifest 54 | # arch_strings=("amd64" "arm64v8" "i386" "arm32v7" "ppc64le" "s390x") 55 | arch_strings=("amd64" "arm64v8") 56 | manifest_args=() 57 | for arch in "${arch_strings[@]}"; do 58 | echo "Processing architecture: $arch" 59 | manifest_args+=("--amend" "${IMG_NS}/pdc-dev-base:${VERSION}-${arch}") 60 | if [[ "$arch" == "$ARCH_CODE" ]]; then 61 | echo "Skipping pulling current architecture: $arch" 62 | continue 63 | fi 64 | docker pull ${IMG_NS}/pdc-dev-base:${VERSION}-${arch} 65 | done 66 | 67 | docker manifest create ${IMG_NS}/pdc-dev-base:${VERSION} ${manifest_args[@]} 68 | docker manifest push ${IMG_NS}/pdc-dev-base:${VERSION} 69 | docker manifest rm ${IMG_NS}/pdc-dev-base:latest 70 | docker manifest create ${IMG_NS}/pdc-dev-base:latest ${manifest_args[@]} 71 | docker manifest push ${IMG_NS}/pdc-dev-base:latest 72 | 73 | fi 74 | 75 | 76 | -------------------------------------------------------------------------------- /.docker/run_dev_base.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DEFAULT_WORKSPACE=/workspaces/pdc 4 | 5 | LAST_PDC_DEV_CONTAINER=$(docker ps -a | grep pdc_dev_base | head -1 | awk '{print $NF}') 6 | 7 | # if this is empty, then we need to create a new container 8 | if [ -z "$LAST_PDC_DEV_CONTAINER" ]; then 9 | echo "No existing pdc_dev_base container found, creating a new one" 10 | docker image rm -f hpcio/pdc-dev-base:latest 11 | docker create -it -v $(pwd):${DEFAULT_WORKSPACE} -w ${DEFAULT_WORKSPACE} --entrypoint /bin/bash hpcio/pdc-dev-base:latest 12 | sleep 1 13 | LAST_PDC_DEV_CONTAINER=$(docker ps -a | grep pdc_dev_base | head -1 | awk '{print $NF}') 14 | echo "Created pdc_dev_base container: $LAST_PDC_DEV_CONTAINER. To stop it, run 'docker stop $LAST_PDC_DEV_CONTAINER'" 15 | docker start $LAST_PDC_DEV_CONTAINER 16 | echo "Wait for 5 seconds for the container to start." 17 | sleep 5 18 | docker exec -it $LAST_PDC_DEV_CONTAINER /bin/sh -c "/bin/bash ${DEFAULT_WORKSPACE}/.devcontainer/post-attach.sh" 19 | else 20 | echo "Found existing pdc_dev_base container $LAST_PDC_DEV_CONTAINER, start it. To stop it, run 'docker stop $LAST_PDC_DEV_CONTAINER'" 21 | docker start $LAST_PDC_DEV_CONTAINER 22 | docker exec -it $LAST_PDC_DEV_CONTAINER /bin/bash -c "/bin/bash ${DEFAULT_WORKSPACE}/.devcontainer/post-attach.sh" 23 | fi -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # This is a comment. 2 | # Each line is a file pattern followed by one or more owners. 3 | 4 | # For more info about this file, please see 5 | # https://docs.github.com/en/enterprise/2.18/user/github/creating-cloning-and-archiving-repositories/about-code-owners 6 | 7 | # These owners will be the default owners for everything in 8 | # the repo. Unless a later match takes precedence, 9 | 10 | * @hpc-io/pdc-developers 11 | 12 | # This directory controls the permissions/review requirements 13 | .github/ @hpc-io/pdc -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug Report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: 'type: bug' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Bug Report** 11 | 12 | A clear and concise description of what the bug is, including module and feature. 13 | 14 | **To Reproduce** 15 | 16 | How are you building/running PDC? 17 | 18 | - version of PDC: [e.g., 0.3, branch, or hash] 19 | - installed PDC using: [spack, from source] 20 | - operating system: [name and version] 21 | - machine: [Are you running on a supercomputer or public cluster?] 22 | - version of Mercury: [e.g., 1.12.0] 23 | - name and version of MPI: [e.g., OpenMPI 4.1.1] 24 | 25 | What did you use to build PDC (cmake command)? 26 | 27 | ```bash 28 | ... 29 | ``` 30 | 31 | What is the running setup you use? 32 | 33 | ```bash 34 | ... 35 | ``` 36 | 37 | **Expected Behavior** 38 | 39 | A clear and concise description of what you expected to happen. 40 | 41 | **Additional Information** 42 | 43 | If built from source, include the cmake options you used. 44 | Add any other information about the problem here. 45 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature Request 3 | about: Request a new feature in PDC 4 | title: '' 5 | labels: 'type: new feature' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **What does this feature solve or improve?** 11 | 12 | A clear and concise description of what the problem is. 13 | Example: I always do the following workflow [...] 14 | 15 | **Describe the solution you'd like** 16 | 17 | A clear and concise description of what you want to happen. 18 | Example: It would be fantastic if one could [...] 19 | 20 | **Describe alternatives you've considered** 21 | 22 | A clear and concise description of any alternative solutions or features you've considered. 23 | 24 | **Additional Information** 25 | 26 | Add any other information about the feature request here. 27 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/install_problem.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Installation Problem 3 | about: Report an installation issue 4 | title: '' 5 | labels: 'type: bug' 6 | assignees: '' 7 | 8 | --- 9 | 10 | 11 | **Issue Description** 12 | 13 | This is what I have tried so far: 14 | 15 | 16 | ```commandline 17 | ... 18 | ``` 19 | 20 | This is the error I am facing during installation: 21 | 22 | ``` 23 | ... 24 | ``` 25 | 26 | **Software Environment** 27 | 28 | - version of PDC: [e.g. 0.3] 29 | - installed PDC using: [spack, from source] 30 | - operating system: [name and version] 31 | - machine: [Are you running on a supercomputer or public cluster?] 32 | - version of Mercury: [e.g. 1.12.0] 33 | - name and version of MPI: [e.g. OpenMPI 4.1.1] 34 | 35 | **Additional Information** 36 | 37 | Add any other information about the problem here. 38 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/question.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Question 3 | about: Ask a question about PDC usage 4 | title: '' 5 | labels: 'type: question' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Usage Case** 11 | 12 | A clear and concise description of what your usage case is. 13 | 14 | **Additional Information** 15 | 16 | Add any other information about your intended usage, limitation, or questions here. 17 | 18 | **Software Environment** 19 | 20 | - version of PDC: [e.g. 0.3] 21 | - installed PDC using: [spack, from source] 22 | - operating system: [name and version] 23 | - machine: [Are you running on a supercomputer or public cluster?] 24 | - version of Mercury: [e.g. 1.12.0] 25 | - name and version of MPI: [e.g. OpenMPI 4.1.1] 26 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | # Related Issues / Pull Requests 2 | 3 | List all related issues and/or pull requests if there are any. 4 | 5 | # Description 6 | 7 | Include a brief summary of the proposed changes. 8 | 9 | # What changes are proposed in this pull request? 10 | 11 | - [ ] Bug fix (non-breaking change which fixes an issue) 12 | - [ ] New feature (non-breaking change which adds functionality) 13 | - [ ] Breaking change (fix or feature that would cause existing functionality not to work as expected; for instance, examples in this repository must be updated too) 14 | - [ ] This change requires a documentation update 15 | 16 | # Checklist: 17 | 18 | - [ ] My code modifies existing public API, or introduces new public API, and I updated or wrote docstrings 19 | - [ ] I have commented my code 20 | - [ ] My code requires documentation updates, and I have made corresponding changes to the documentation 21 | - [ ] I have added tests that prove my fix is effective or that my feature works 22 | - [ ] New and existing unit tests pass locally with my changes 23 | -------------------------------------------------------------------------------- /.github/workflows/clang-format-check.yml: -------------------------------------------------------------------------------- 1 | name: clang-format Check 2 | on: 3 | pull_request: 4 | jobs: 5 | formatting-check: 6 | name: Formatting Check 7 | runs-on: ubuntu-latest 8 | if: "!contains(github.event.head_commit.message, 'skip-ci')" 9 | steps: 10 | - uses: actions/checkout@v2 11 | - name: Run clang-format style check for C programs. 12 | uses: DoozyX/clang-format-lint-action@v0.18.1 13 | with: 14 | source: '.' 15 | extensions: 'c,h,cpp,hpp' 16 | clangFormatVersion: 10 17 | style: file 18 | # exclude: './config' 19 | -------------------------------------------------------------------------------- /.github/workflows/clang-format-fix.yml: -------------------------------------------------------------------------------- 1 | name: clang-format Commit Chanages 2 | on: 3 | workflow_dispatch: 4 | push: 5 | jobs: 6 | formatting-check: 7 | name: Commit Format Changes 8 | runs-on: ubuntu-latest 9 | if: "!contains(github.event.head_commit.message, 'skip-ci')" 10 | permissions: 11 | contents: write # In order to allow EndBug/add-and-commit to commit changes 12 | steps: 13 | - uses: actions/checkout@v3 14 | - name: Fix C formatting issues detected by clang-format 15 | uses: DoozyX/clang-format-lint-action@v0.13 16 | with: 17 | source: '.' 18 | extensions: 'c,h,cpp,hpp' 19 | clangFormatVersion: 10 20 | inplace: True 21 | style: file 22 | # exclude: './config ' 23 | - uses: EndBug/add-and-commit@v9 24 | with: 25 | author_name: github-actions 26 | author_email: github-actions[bot]@users.noreply.github.com 27 | message: 'Committing clang-format changes' 28 | -------------------------------------------------------------------------------- /.github/workflows/dependencies-linux.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -eu -o pipefail 4 | 5 | sudo apt-get update 6 | sudo apt-get install libopenmpi-dev libhdf5-dev uuid-dev cmake 7 | 8 | # libfabric 9 | wget https://github.com/ofiwg/libfabric/archive/refs/tags/v1.12.1.tar.gz 10 | tar xf v1.12.1.tar.gz 11 | cd libfabric-1.12.1 12 | ./autogen.sh 13 | ./configure --disable-usnic --disable-mrail --disable-rstream --disable-perf --disable-efa --disable-psm2 --disable-psm --disable-verbs --disable-shm --disable-static --disable-silent-rules 14 | make -j2 && sudo make install 15 | make check 16 | cd .. 17 | 18 | # Mercury 19 | git clone --recursive https://github.com/mercury-hpc/mercury.git 20 | cd mercury 21 | git checkout v2.2.0 22 | mkdir build && cd build 23 | cmake ../ -DCMAKE_C_COMPILER=gcc -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=ON -DNA_USE_OFI=ON -DNA_USE_SM=OFF -DMERCURY_USE_CHECKSUMS=OFF -DCMAKE_POLICY_VERSION_MINIMUM=3.5 24 | make -j2 && sudo make install 25 | ctest 26 | -------------------------------------------------------------------------------- /.github/workflows/dependencies-macos.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -eu -o pipefail 4 | 5 | export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=FALSE 6 | 7 | brew install open-mpi automake 8 | 9 | # libfabric 10 | wget https://github.com/ofiwg/libfabric/archive/refs/tags/v1.15.2.tar.gz 11 | tar xf v1.15.2.tar.gz 12 | cd libfabric-1.15.2 13 | ./autogen.sh 14 | ./configure --disable-usnic --disable-mrail --disable-rstream --disable-perf --disable-efa --disable-psm2 --disable-psm --disable-verbs --disable-shm --disable-static --disable-silent-rules 15 | make -j2 && sudo make install 16 | make check 17 | cd .. 18 | 19 | # Mercury 20 | git clone --recursive https://github.com/mercury-hpc/mercury.git 21 | cd mercury 22 | git checkout v2.2.0 23 | mkdir build && cd build 24 | cmake ../ -DCMAKE_C_COMPILER=gcc -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=ON -DNA_USE_OFI=ON -DNA_USE_SM=OFF -DMERCURY_USE_CHECKSUMS=OFF -DNA_OFI_TESTING_PROTOCOL=sockets 25 | make -j2 && sudo make install 26 | ctest 27 | -------------------------------------------------------------------------------- /.github/workflows/macos.yml: -------------------------------------------------------------------------------- 1 | name: MacOS 2 | 3 | on: 4 | pull_request: 5 | branches: [ stable, develop ] 6 | 7 | push: 8 | branches: [ stable, develop ] 9 | 10 | # Allows to run this workflow manually from the Actions tab 11 | workflow_dispatch: 12 | 13 | jobs: 14 | PDC: 15 | runs-on: macos-13 16 | timeout-minutes: 60 17 | 18 | steps: 19 | - uses: actions/checkout@v3 20 | 21 | - name: Dependencies 22 | run: .github/workflows/dependencies-macos.sh 23 | 24 | - name: Build PDC 25 | run: | 26 | mkdir build && cd build 27 | cmake ../ -DBUILD_MPI_TESTING=ON -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=ON -DPDC_ENABLE_MPI=ON -DCMAKE_C_COMPILER=mpicc 28 | make -j 2 29 | 30 | - name: Test PDC 31 | working-directory: build 32 | run: | 33 | sudo sh -c 'echo "`ipconfig getifaddr en0` PDC" >> /etc/hosts' 34 | sudo scutil --set HostName PDC 35 | export HG_TRANSPORT="sockets" 36 | ctest -L serial 37 | -------------------------------------------------------------------------------- /.github/workflows/nersc.yml: -------------------------------------------------------------------------------- 1 | name: NERSC 2 | 3 | on: 4 | pull_request_target 5 | 6 | env: 7 | PR_NUMBER: ${{ github.event.number }} 8 | 9 | jobs: 10 | authorize: 11 | environment: 12 | ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name != github.repository && 'external' || 'internal' }} 13 | runs-on: ubuntu-latest 14 | steps: 15 | - run: true 16 | 17 | sync-with-nersc: 18 | needs: authorize 19 | runs-on: ubuntu-latest 20 | timeout-minutes: 2 21 | 22 | steps: 23 | - uses: actions/checkout@v3 24 | with: 25 | fetch-depth: 0 26 | 27 | - name: Setup git 28 | run: | 29 | git config --global user.name "PDC BOT" 30 | git config --global user.email "pdc@example.com" 31 | 32 | - name: Create PR branch 33 | run: | 34 | PR_SHA=$(git rev-parse --short "$GITHUB_SHA") 35 | git fetch origin pull/${PR_NUMBER}/head:PR-${PR_SHA} 36 | git remote add gitlab https://${{ secrets.GITLAB_TOKEN_NAME }}:${{ secrets.GITLAB_TOKEN }}@${{ secrets.GITLAB_URL }} 37 | git checkout PR-${PR_SHA} 38 | git push -f gitlab -u PR-${PR_SHA} 39 | -------------------------------------------------------------------------------- /.github/workflows/ubuntu-cache.yml: -------------------------------------------------------------------------------- 1 | name: Ubuntu (cache) 2 | 3 | on: 4 | pull_request: 5 | branches: [ stable, develop ] 6 | 7 | push: 8 | branches: [ stable, develop ] 9 | 10 | workflow_dispatch: 11 | 12 | jobs: 13 | PDC: 14 | runs-on: ubuntu-latest 15 | timeout-minutes: 60 16 | 17 | steps: 18 | - uses: actions/checkout@v3 19 | 20 | - name: Dependencies 21 | run: .github/workflows/dependencies-linux.sh 22 | 23 | - name: Build PDC 24 | run: | 25 | mkdir build && cd build 26 | cmake ../ -DBUILD_MPI_TESTING=ON -DBUILD_SHARED_LIBS=ON -DPDC_SERVER_CACHE=ON -DBUILD_TESTING=ON -DPDC_ENABLE_MPI=ON -DCMAKE_C_COMPILER=mpicc -DCMAKE_POLICY_VERSION_MINIMUM=3.5 27 | make -j2 28 | 29 | - name: Test PDC 30 | working-directory: build 31 | run: ctest -L serial 32 | -------------------------------------------------------------------------------- /.github/workflows/ubuntu-no-cache.yaml: -------------------------------------------------------------------------------- 1 | name: Ubuntu (no-cache) 2 | 3 | on: 4 | pull_request: 5 | branches: [ stable, develop ] 6 | 7 | push: 8 | branches: [ stable, develop ] 9 | 10 | workflow_dispatch: 11 | 12 | jobs: 13 | PDC: 14 | runs-on: ubuntu-latest 15 | timeout-minutes: 60 16 | 17 | steps: 18 | - uses: actions/checkout@v3 19 | 20 | - name: Dependencies 21 | run: .github/workflows/dependencies-linux.sh 22 | 23 | - name: Build PDC 24 | run: | 25 | mkdir build && cd build 26 | cmake ../ -DBUILD_MPI_TESTING=ON -DBUILD_SHARED_LIBS=ON -DPDC_SERVER_CACHE=OFF -DBUILD_TESTING=ON -DPDC_ENABLE_MPI=ON -DCMAKE_C_COMPILER=mpicc -DCMAKE_POLICY_VERSION_MINIMUM=3.5 27 | make -j2 28 | 29 | - name: Test PDC 30 | working-directory: build 31 | run: ctest -L serial 32 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # ignore swp files 2 | *.swp 3 | 4 | # exes 5 | *.exe 6 | 7 | *.pdf 8 | 9 | src/install 10 | 11 | # ignore vscode files 12 | .vscode 13 | 14 | # ignore macos files 15 | .DS_Store 16 | 17 | # ignore build dir 18 | build 19 | 20 | # ignore docs build dir 21 | docs/build 22 | 23 | # ignore venv 24 | **/*venv 25 | 26 | # ignore .zed 27 | .zed -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # .readthedocs.yaml 2 | # Read the Docs configuration file 3 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 4 | 5 | # Required 6 | version: 2 7 | 8 | # Set the version of Python and other tools you might need 9 | build: 10 | os: ubuntu-22.04 11 | tools: 12 | python: "3.11" 13 | 14 | # Build documentation in the docs/ directory with Sphinx 15 | sphinx: 16 | configuration: docs/source/conf.py 17 | 18 | # We recommend specifying your dependencies to enable reproducible builds: 19 | # https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html 20 | python: 21 | install: 22 | - requirements: docs/requirements.txt -------------------------------------------------------------------------------- /Authors: -------------------------------------------------------------------------------- 1 | Authors of the Proactive Data Containers (PDC) software library and utilities: 2 | ------------------------------------------------------------------------------ 3 | 4 | Contact: Suren Byna (SByna@lbl.gov) 5 | 6 | Lawrence Berkeley National Laboratory (LBNL) 7 | -------------------------------------------- 8 | Houjun Tang 9 | Qiao Kang 10 | Bin Dong 11 | Quincey Koziol 12 | Suren Byna 13 | 14 | The HDF Group (THG) 15 | ------------------- 16 | Kimmy Mu 17 | Richard Warren 18 | Jerome Soumagne 19 | 20 | Argonne National Laboratory (ANL) 21 | --------------------------------- 22 | François Tessier 23 | Venkat Vishwanath 24 | 25 | -------------------------------------------------------------------------------- /CMake/FindDRC.cmake: -------------------------------------------------------------------------------- 1 | # - Try to find DRC 2 | # Once done this will define 3 | # DRC_FOUND - System has DRC 4 | # DRC_INCLUDE_DIRS - The DRC include directories 5 | # DRC_LIBRARIES - The libraries needed to use DRC 6 | 7 | find_package(PkgConfig) 8 | pkg_check_modules(PC_DRC cray-drc) 9 | 10 | find_path(DRC_INCLUDE_DIR rdmacred.h 11 | HINTS ${PC_DRC_INCLUDEDIR} ${PC_DRC_INCLUDE_DIRS}) 12 | 13 | find_library(DRC_LIBRARY NAMES drc 14 | HINTS ${PC_DRC_LIBDIR} ${PC_DRC_LIBRARY_DIRS}) 15 | 16 | set(DRC_INCLUDE_DIRS ${DRC_INCLUDE_DIR}) 17 | set(DRC_LIBRARIES ${DRC_LIBRARY}) 18 | 19 | include(FindPackageHandleStandardArgs) 20 | # handle the QUIETLY and REQUIRED arguments and set DRC_FOUND to TRUE 21 | # if all listed variables are TRUE 22 | find_package_handle_standard_args(DRC DEFAULT_MSG 23 | DRC_INCLUDE_DIR DRC_LIBRARY) 24 | 25 | mark_as_advanced(DRC_INCLUDE_DIR DRC_LIBRARY) 26 | -------------------------------------------------------------------------------- /CMake/FindJULIA.cmake: -------------------------------------------------------------------------------- 1 | # FindJULIA.cmake 2 | 3 | # Check if JULIA_HOME is set 4 | if(NOT DEFINED ENV{JULIA_HOME}) 5 | message(FATAL_ERROR "JULIA_HOME environment variable is not set. Please set it to the JULIA installation directory.") 6 | endif() 7 | 8 | # Set JULIA installation directory 9 | set(JULIA_INSTALL_DIR $ENV{JULIA_HOME}) 10 | 11 | # Find JULIA library 12 | find_library(JULIA_LIBRARY 13 | NAMES julia 14 | PATHS ${JULIA_INSTALL_DIR}/lib 15 | NO_DEFAULT_PATH 16 | NO_CMAKE_FIND_ROOT_PATH 17 | ) 18 | 19 | # Find JULIA include directory 20 | find_path(JULIA_INCLUDE_DIR 21 | NAMES julia.h 22 | PATHS ${JULIA_INSTALL_DIR}/include/julia 23 | NO_DEFAULT_PATH 24 | NO_CMAKE_FIND_ROOT_PATH 25 | ) 26 | 27 | # Check if JULIA library and include directory are found 28 | if(NOT JULIA_LIBRARY) 29 | message(FATAL_ERROR "JULIA library not found. Please set the JULIA_HOME environment variable to the correct JULIA installation directory.") 30 | endif() 31 | 32 | if(NOT JULIA_INCLUDE_DIR) 33 | message(FATAL_ERROR "JULIA include directory not found. Please set the JULIA_HOME environment variable to the correct JULIA installation directory.") 34 | endif() 35 | 36 | # Set JULIA_FOUND variable 37 | set(JULIA_FOUND TRUE CACHE BOOL "JULIA found") 38 | 39 | # Set JULIA_INCLUDE_DIRS and JULIA_LIBRARIES variables 40 | set(JULIA_INCLUDE_DIRS ${JULIA_INCLUDE_DIR}) 41 | set(JULIA_LIBRARIES ${JULIA_LIBRARY}) 42 | 43 | # Report JULIA library and include directory 44 | message(STATUS "JULIA library found: ${JULIA_LIBRARY}") 45 | message(STATUS "JULIA include directory found: ${JULIA_INCLUDE_DIR}") -------------------------------------------------------------------------------- /CMake/FindMERCURY.cmake: -------------------------------------------------------------------------------- 1 | # FindMERCURY.cmake 2 | 3 | # Find the system's MERCURY library 4 | # This will define: 5 | # 6 | # MERCURY_FOUND - System has MERCURY 7 | # MERCURY_INCLUDE_DIRS - The MERCURY include directory 8 | # MERCURY_LIBRARIES - The libraries needed to use MERCURY 9 | 10 | find_package(MERCURY QUIET HINTS $ENV{MERCURY_HOME} $ENV{MERCURY_DIR} $ENV{MERCURY_ROOT} $ENV{MERCURYPATH} $ENV{MERCURY_PATH}) 11 | if(MERCURY_FOUND) 12 | message(STATUS "mercury dir = ${MERCURY_DIR}") 13 | # NOTE: enable the following if you need ${MERCURY_INCLUDE_DIR} in the future 14 | # NOTE: remember to add MERCURY_HOME to PATH or CMAKE_PREFIX_PATH if you enable the following. 15 | find_path(MERCURY_INCLUDE_DIR mercury.h HINTS ${MERCURY_DIR}) 16 | find_library(MERCURY_LIBRARY 17 | NAMES 18 | mercury 19 | mercury_debug 20 | HINTS ${MERCURY_DIR} 21 | ) 22 | 23 | find_library(MERCURY_NA_LIBRARY 24 | NAMES 25 | na 26 | na_debug 27 | HINTS ${MERCURY_DIR} 28 | ) 29 | 30 | find_library(MERCURY_UTIL_LIBRARY 31 | NAMES 32 | mercury_util 33 | HINTS ${MERCURY_DIR} 34 | ) 35 | 36 | set(MERCURY_LIBRARIES ${MERCURY_LIBRARY} ${MERCURY_NA_LIBRARY} ${MERCURY_UTIL_LIBRARY}) 37 | # set(MERCURY_LIBRARIES ${MERCURY_LIBRARY} ${MERCURY_NA_LIBRARY}) 38 | set(MERCURY_INCLUDE_DIRS ${MERCURY_INCLUDE_DIR}) 39 | message(STATUS "mercury include dir = ${MERCURY_INCLUDE_DIRS}") 40 | message(STATUS "mercury lib = ${MERCURY_LIBRARIES}") 41 | include(FindPackageHandleStandardArgs) 42 | find_package_handle_standard_args(MERCURY DEFAULT_MSG MERCURY_LIBRARY MERCURY_INCLUDE_DIR) 43 | else(MERCURY_FOUND) 44 | set(MERCURY_LIBRARIES "") 45 | endif() 46 | 47 | mark_as_advanced(MERCURY_INCLUDE_DIR MERCURY_LIBRARY) 48 | -------------------------------------------------------------------------------- /CMake/FindUUID.cmake: -------------------------------------------------------------------------------- 1 | # On Mac OS X the uuid functions are in the System library 2 | 3 | if(APPLE) 4 | set(UUID_LIBRARY_VAR System) 5 | else() 6 | set(UUID_LIBRARY_VAR uuid) 7 | endif() 8 | 9 | find_library(UUID_LIBRARY 10 | NAMES ${UUID_LIBRARY_VAR} 11 | PATHS /usr/local/lib64 /usr/local/lib /usr/lib64 /usr/lib 12 | ) 13 | 14 | find_path(UUID_INCLUDE_DIR uuid/uuid.h 15 | HINTS ${PC_UUID_INCLUDEDIR} ${PC_UUID_INCLUDE_DIRS} 16 | PATHS /usr/local/include /usr/include 17 | ) 18 | 19 | if (UUID_LIBRARY AND UUID_INCLUDE_DIR) 20 | set(UUID_LIBRARIES ${UUID_LIBRARY}) 21 | set(UUID_FOUND "TRUE") 22 | else () 23 | set(UUID_FOUND "FALSE") 24 | endif () 25 | 26 | if (UUID_FOUND) 27 | if (NOT UUID_FIND_QUIETLY) 28 | message(STATUS "Found UUID: ${UUID_LIBRARIES}") 29 | endif () 30 | else () 31 | if (UUID_FIND_REQUIRED) 32 | message( "library: ${UUID_LIBRARY}" ) 33 | message( "include: ${UUID_INCLUDE_DIR}" ) 34 | message(FATAL_ERROR "Could not find UUID library") 35 | endif () 36 | endif () 37 | 38 | mark_as_advanced( 39 | UUID_LIBRARY 40 | UUID_INCLUDE_DIR 41 | ) 42 | -------------------------------------------------------------------------------- /CMake/pdc-config-version.cmake.in: -------------------------------------------------------------------------------- 1 | #----------------------------------------------------------------------------- 2 | # Version file for install directory 3 | #----------------------------------------------------------------------------- 4 | set(PACKAGE_VERSION @PDC_PACKAGE_VERSION@) 5 | 6 | if("${PACKAGE_FIND_VERSION_MAJOR}" EQUAL @PDC_VERSION_MAJOR@) 7 | if("${PACKAGE_FIND_VERSION_MINOR}" EQUAL @PDC_VERSION_MINOR@) 8 | set(PACKAGE_VERSION_COMPATIBLE 1) 9 | if("${PACKAGE_FIND_VERSION_PATCH}" EQUAL @PDC_VERSION_PATCH@) 10 | set(PACKAGE_VERSION_EXACT 1) 11 | endif() 12 | endif() 13 | endif() 14 | -------------------------------------------------------------------------------- /CMake/pdc-config.cmake.build.in: -------------------------------------------------------------------------------- 1 | #----------------------------------------------------------------------------- 2 | # Config file for compiling against the build directory 3 | #----------------------------------------------------------------------------- 4 | get_filename_component(SELF_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH) 5 | 6 | #----------------------------------------------------------------------------- 7 | # Directories 8 | #----------------------------------------------------------------------------- 9 | set(PDC_INCLUDE_DIR "@PDC_INCLUDES_BUILD_TIME@") 10 | 11 | #----------------------------------------------------------------------------- 12 | # Version Strings 13 | #----------------------------------------------------------------------------- 14 | set(PDC_VERSION_STRING @PDC_PACKAGE_VERSION@) 15 | set(PDC_VERSION_MAJOR @PDC_VERSION_MAJOR@) 16 | set(PDC_VERSION_MINOR @PDC_VERSION_MINOR@) 17 | set(PDC_VERSION_PATCH @PDC_VERSION_PATCH@) 18 | 19 | #----------------------------------------------------------------------------- 20 | # Don't include targets if this file is being picked up by another 21 | # project which has already built PDC as a subproject 22 | #----------------------------------------------------------------------------- 23 | if(NOT TARGET "pdc" AND NOT PDC_INSTALL_SKIP_TARGETS) 24 | if(NOT TARGET "mercury") 25 | include(@MERCURY_DIR@/mercury-config.cmake) 26 | endif() 27 | include(@PDC_INSTALL_SHARE_DIR@/pdc-targets.cmake) 28 | endif() 29 | -------------------------------------------------------------------------------- /CMake/pdc-config.cmake.install.in: -------------------------------------------------------------------------------- 1 | #----------------------------------------------------------------------------- 2 | # Config file for compiling against the build directory 3 | #----------------------------------------------------------------------------- 4 | get_filename_component(SELF_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH) 5 | 6 | #----------------------------------------------------------------------------- 7 | # Directories 8 | #----------------------------------------------------------------------------- 9 | set(PDC_INCLUDE_DIR "@PDC_INCLUDES_INSTALL_TIME@") 10 | 11 | #----------------------------------------------------------------------------- 12 | # Version Strings 13 | #----------------------------------------------------------------------------- 14 | set(PDC_VERSION_STRING @PDC_PACKAGE_VERSION@) 15 | set(PDC_VERSION_MAJOR @PDC_VERSION_MAJOR@) 16 | set(PDC_VERSION_MINOR @PDC_VERSION_MINOR@) 17 | set(PDC_VERSION_PATCH @PDC_VERSION_PATCH@) 18 | 19 | #----------------------------------------------------------------------------- 20 | # Don't include targets if this file is being picked up by another 21 | # project which has already built PDC as a subproject 22 | #----------------------------------------------------------------------------- 23 | if(NOT TARGET "pdc" AND NOT PDC_INSTALL_SKIP_TARGETS) 24 | if(NOT TARGET "mercury") 25 | include(@MERCURY_DIR@/mercury-config.cmake) 26 | endif() 27 | include(@PDC_INSTALL_SHARE_DIR@/pdc-targets.cmake) 28 | endif() 29 | -------------------------------------------------------------------------------- /CTestConfig.cmake: -------------------------------------------------------------------------------- 1 | ## This file should be placed in the root directory of your project. 2 | ## Then modify the CMakeLists.txt file in the root directory of your 3 | ## project to incorporate the testing dashboard. 4 | ## # The following are required to uses Dart and the Cdash dashboard 5 | ## ENABLE_TESTING() 6 | ## INCLUDE(Dart) 7 | 8 | set(CTEST_PROJECT_NAME "PDC") 9 | set(CTEST_NIGHTLY_START_TIME "00:00:00 CST") 10 | 11 | set(CTEST_DROP_METHOD "http") 12 | set(CTEST_DROP_SITE "cdash.hdfgroup.org") 13 | set(CTEST_DROP_LOCATION "/submit.php?project=PDC") 14 | set(CTEST_DROP_SITE_CDASH TRUE) 15 | 16 | -------------------------------------------------------------------------------- /Legal.txt: -------------------------------------------------------------------------------- 1 | Copyright Notice for 2 | Proactive Data Containers (PDC) Software Library and Utilities 3 | ----------------------------------------------------------------------------- 4 | 5 | *** Copyright Notice *** 6 | 7 | Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the 8 | University of California, through Lawrence Berkeley National Laboratory, 9 | UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF 10 | Group (subject to receipt of any required approvals from the U.S. Dept. of 11 | Energy). All rights reserved. 12 | 13 | If you have questions about your rights to use or distribute this software, 14 | please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. 15 | 16 | NOTICE. This Software was developed under funding from the U.S. Department of 17 | Energy and the U.S. Government consequently retains certain rights. As such, the 18 | U.S. Government has been granted for itself and others acting on its behalf a 19 | paid-up, nonexclusive, irrevocable, worldwide license in the Software to 20 | reproduce, distribute copies to the public, prepare derivative works, and 21 | perform publicly and display publicly, and to permit other to do so. 22 | 23 | -------------------------------------------------------------------------------- /benchmark/idioms/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required (VERSION 2.8.12) 2 | 3 | # Setup cmake policies. 4 | foreach(p 5 | CMP0012 6 | CMP0013 7 | CMP0014 8 | CMP0022 # CMake 2.8.12 9 | CMP0025 # CMake 3.0 10 | CMP0053 # CMake 3.1 11 | CMP0054 # CMake 3.1 12 | CMP0074 # CMake 3.12 13 | CMP0075 # CMake 3.12 14 | CMP0083 # CMake 3.14 15 | CMP0093 # CMake 3.15 16 | ) 17 | if(POLICY ${p}) 18 | cmake_policy(SET ${p} NEW) 19 | endif() 20 | endforeach() 21 | 22 | project(METADATA_JSON_LOADER C) 23 | 24 | set(JMD_LDR_EXT_INCLUDE_DIRS "") 25 | set(JMD_LDR_EXT_LIBRARIES "") 26 | 27 | set(CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING "Choose the type of build." FORCE) 28 | # set(JMD_DEBUG ON CACHE BOOL "Enable debug mode.") 29 | option(JMD_DEBUG_MODE "Enable debug mode." ON) 30 | if(JMD_DEBUG_MODE) 31 | add_definitions(-DJMD_DEBUG=1) 32 | endif() 33 | 34 | option(USE_SYSTEM_MPI "Use system-installed MPI." ON) 35 | if(USE_SYSTEM_MPI) 36 | find_package(MPI) 37 | if(MPI_FOUND) 38 | add_definitions(-DJMD_LDR_ENABLE_MPI=1) 39 | SET(CMAKE_C_COMPILER ${MPI_C_COMPILER}) 40 | SET(CMAKE_CXX_COMPILER ${MPI_CXX_COMPILER}) 41 | set(JMD_LDR_EXT_INCLUDE_DIRS ${MPI_C_INCLUDE_PATH} 42 | ${JMD_LDR_EXT_INCLUDE_DIRS} 43 | ) 44 | set(JMD_LDR_EXT_LIBRARIES ${MPI_C_LIBRARIES} ${JMD_LDR_EXT_LIBRARIES}) 45 | endif() 46 | endif() 47 | 48 | find_package(PDC REQUIRED) 49 | if(PDC_FOUND) 50 | #message(STATUS "PDC include directory: ${PDC_INCLUDE_DIR}") 51 | set(JMD_LDR_EXT_INCLUDE_DIRS ${PDC_INCLUDE_DIR} 52 | ${JMD_LDR_EXT_INCLUDE_DIRS} 53 | ) 54 | set(JMD_LDR_EXT_LIBRARIES pdc ${JMD_LDR_EXT_LIBRARIES}) 55 | endif() 56 | 57 | 58 | add_library(cjson cjson/cJSON.c) 59 | 60 | add_executable(metadata_json_loader 61 | metadata_json_loader.c 62 | metadata_json_processor.h 63 | metadata_json_printer.c 64 | metadata_json_printer.h 65 | metadata_json_importer.c 66 | metadata_json_importer.h) 67 | target_link_libraries(metadata_json_loader ${PDC_EXT_LIB_DEPENDENCIES} pdc cjson ${JMD_LDR_EXT_LIBRARIES}) 68 | target_include_directories(metadata_json_loader PUBLIC ${PDC_EXT_INCLUDE_DEPENDENCIES} ${JMD_LDR_EXT_INCLUDE_DIRS}) -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = source 9 | BUILDDIR = build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/_static/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hpc-io/pdc/89268e80d232f33758516cc26ddcb81f5a493700/docs/_static/.DS_Store -------------------------------------------------------------------------------- /docs/_static/css/pdc.css: -------------------------------------------------------------------------------- 1 | div.wy-nav-content { 2 | max-width: 1024px; 3 | } 4 | 5 | nav.wy-nav-side, div.wy-side-nav-search, 6 | div.ethical-dark-theme .ethical-sidebar, 7 | div.rst-versions .rst-current-version { 8 | background: #efefef; 9 | } 10 | 11 | div.wy-side-nav-search > a img.logo { 12 | margin: 40px auto; 13 | } 14 | 15 | div.wy-menu-vertical p.caption { 16 | color: #503788; 17 | } 18 | 19 | div.wy-menu-vertical a { 20 | color: #404040; 21 | } 22 | 23 | div.wy-menu-vertical a:hover { 24 | background-color: #ffffff; 25 | } 26 | 27 | .rst-content div[class^=highlight], .rst-content pre.literal-block { 28 | border-radius: 5px; 29 | background: #efefef; 30 | } 31 | 32 | .admonition note { 33 | border-radius: 5px; 34 | } 35 | 36 | .admonition-title { 37 | border-radius: 5px 5px 0 0; 38 | } 39 | 40 | div.rst-versions span.rst-current-version { 41 | color: #344f90; 42 | } 43 | 44 | div.rst-versions span.fa { 45 | color: #404040!important; 46 | } 47 | 48 | div.ethical-dark-theme a, div.ethical-dark-theme a:visited { 49 | color: #404040!important; 50 | } 51 | 52 | div.rst-versions { 53 | color: #404040; 54 | } 55 | 56 | div.rst-content dl dt { 57 | border-radius: 3px; 58 | display: block; 59 | } 60 | 61 | .cpp .function > dt.cpp { 62 | background: #efe7fa; 63 | color: #65419d; 64 | border-left: 3px solid #65419d; 65 | } -------------------------------------------------------------------------------- /docs/_static/custom.css: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /docs/_static/image/pdc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hpc-io/pdc/89268e80d232f33758516cc26ddcb81f5a493700/docs/_static/image/pdc.png -------------------------------------------------------------------------------- /docs/_static/pdc_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hpc-io/pdc/89268e80d232f33758516cc26ddcb81f5a493700/docs/_static/pdc_logo.png -------------------------------------------------------------------------------- /docs/container_hashtable_checkpoint.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hpc-io/pdc/89268e80d232f33758516cc26ddcb81f5a493700/docs/container_hashtable_checkpoint.png -------------------------------------------------------------------------------- /docs/data_hashtable_checkpoint.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hpc-io/pdc/89268e80d232f33758516cc26ddcb81f5a493700/docs/data_hashtable_checkpoint.png -------------------------------------------------------------------------------- /docs/data_server_checkpoint.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hpc-io/pdc/89268e80d232f33758516cc26ddcb81f5a493700/docs/data_server_checkpoint.png -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=source 11 | set BUILDDIR=build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.http://sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /docs/pdc_metadata_flow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hpc-io/pdc/89268e80d232f33758516cc26ddcb81f5a493700/docs/pdc_metadata_flow.png -------------------------------------------------------------------------------- /docs/pdc_plots.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hpc-io/pdc/89268e80d232f33758516cc26ddcb81f5a493700/docs/pdc_plots.pptx -------------------------------------------------------------------------------- /docs/pdc_region_transfer_request_flow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hpc-io/pdc/89268e80d232f33758516cc26ddcb81f5a493700/docs/pdc_region_transfer_request_flow.png -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | sphinx==6.2.1 2 | sphinx-rtd-theme==1.2.2 3 | sphinxemoji 4 | breathe 5 | -------------------------------------------------------------------------------- /docs/source/assumptions.rst: -------------------------------------------------------------------------------- 1 | ================================ 2 | Assumptions 3 | ================================ -------------------------------------------------------------------------------- /docs/source/definitions.rst: -------------------------------------------------------------------------------- 1 | ================================ 2 | Definitions 3 | ================================ -------------------------------------------------------------------------------- /docs/source/documentation/analysis.rst: -------------------------------------------------------------------------------- 1 | ================================ 2 | Analysis 3 | ================================ 4 | 5 | .. doxygenfile:: pdc_analysis.c 6 | .. doxygenfile:: pdc_analysis_common.c 7 | .. doxygenfile:: pdc_analysis_and_transforms_connect.c 8 | .. doxygenfile:: pdc_hist_pkg.c 9 | -------------------------------------------------------------------------------- /docs/source/documentation/objects.rst: -------------------------------------------------------------------------------- 1 | ================================ 2 | Containers 3 | ================================ 4 | 5 | .. doxygenfile:: pdc_cont.c 6 | 7 | ================================ 8 | Objects 9 | ================================ 10 | 11 | .. doxygenfile:: pdc_obj.c 12 | .. doxygenfile:: pdc_mpi.c 13 | 14 | ================================ 15 | Preperties 16 | ================================ 17 | 18 | .. doxygenfile:: pdc_prop.c -------------------------------------------------------------------------------- /docs/source/documentation/query.rst: -------------------------------------------------------------------------------- 1 | ================================ 2 | Query 3 | ================================ 4 | 5 | .. doxygenfile:: pdc_query.c -------------------------------------------------------------------------------- /docs/source/documentation/regions.rst: -------------------------------------------------------------------------------- 1 | ================================ 2 | Regions 3 | ================================ 4 | 5 | .. doxygenfile:: pdc_region.c 6 | .. doxygenfile:: pdc_region_transfer.c 7 | -------------------------------------------------------------------------------- /docs/source/documentation/transformation.rst: -------------------------------------------------------------------------------- 1 | ================================ 2 | Transformation 3 | ================================ 4 | 5 | .. doxygenfile:: pdc_transform.c 6 | -------------------------------------------------------------------------------- /docs/source/futurework.rst: -------------------------------------------------------------------------------- 1 | ================================ 2 | Future Work 3 | ================================ -------------------------------------------------------------------------------- /docs/source/hdf5vol.rst: -------------------------------------------------------------------------------- 1 | ================================ 2 | HDF5 VOL for PDC 3 | ================================ 4 | 5 | ++++++++++++++++++++++++++++++++++ 6 | Installing HDF5 VOL for PDC from source 7 | ++++++++++++++++++++++++++++++++++ 8 | 9 | The following instructions are for installing PDC on Linux and Cray machines. These instructions assume that PDC and its dependencies have all already been installed from source (libfabric and Mercury). 10 | 11 | Install HDF5 12 | --------------------------- 13 | 14 | If a local version of HDF5 is to be used, then the following can be used to install HDF5. 15 | 16 | .. code-block:: Bash 17 | 18 | $ wget "https://www.hdfgroup.org/package/hdf5-1-12-1-tar-gz/?wpdmdl=15727&refresh=612559667d6521629837670" 19 | $ mv index.html?wpdmdl=15727&refresh=612559667d6521629837670 hdf5-1.12.1.tar.gz 20 | $ tar zxf hdf5-1.12.1.tar.gz 21 | $ cd hdf5-1.12.1 22 | $ ./configure --prefix=$HDF5_DIR 23 | $ make 24 | $ make check 25 | $ make install 26 | $ make check-install 27 | 28 | Building VOL-PDC 29 | --------------------------- 30 | 31 | .. code-block:: Bash 32 | 33 | $ git clone https://github.com/hpc-io/vol-pdc.git 34 | $ cd vol-pdc 35 | $ mkdir build 36 | $ cd build 37 | $ cmake ../ -DHDF5_INCLUDE_DIR=$HDF5_INCLUDE_DIR -DHDF5_LIBRARY=$HDF5_LIBRARY -DBUILD_SHARED_LIBS=ON -DHDF5_DIR=$HDF5_DIR 38 | $ make 39 | $ make install 40 | 41 | To compile and run examples: 42 | 43 | .. code-block:: Bash 44 | 45 | $ cd vol-pdc/examples 46 | $ cmake . 47 | $ make 48 | $ mpirun -N 1 -n 1 -c 1 /bin/pdc_server.exe & 49 | $ mpirun -N 1 -n 1 -c 1 ./h5pdc_vpicio test 50 | $ mpirun -N 1 -n 1 -c 1 /bin/close_server 51 | 52 | 53 | ++++++++++++++++++++++++++++++++++ 54 | VOL Function Notes 55 | ++++++++++++++++++++++++++++++++++ 56 | 57 | The following functions have been properly defined: 58 | 59 | * H5VL_pdc_info_copy 60 | * H5VL_pdc_info_cmp 61 | * H5VL_pdc_info_free 62 | * H5VL_pdc_info_to_str 63 | * H5VL_pdc_str_to_info 64 | 65 | * H5VL_pdc_file_create 66 | * H5VL_pdc_file_open 67 | * H5VL_pdc_file_close 68 | * H5VL_pdc_file_specific 69 | 70 | * H5VL_pdc_dataset_create 71 | * H5VL_pdc_dataset_open 72 | * H5VL_pdc_dataset_write 73 | * H5VL_pdc_dataset_read 74 | * H5VL_pdc_dataset_get 75 | * H5VL_pdc_dataset_close 76 | 77 | * H5VL_pdc_introspect_get_conn_cls 78 | * H5VL_pdc_introspect_get_cap_flags 79 | 80 | * H5VL_pdc_group_create 81 | * H5VL_pdc_group_open 82 | 83 | * H5VL_pdc_attr_create 84 | * H5VL_pdc_attr_open 85 | * H5VL_pdc_attr_read 86 | * H5VL_pdc_attr_write 87 | * H5VL_pdc_attr_get 88 | 89 | * H5VL_pdc_object_open 90 | * H5VL_pdc_object_get 91 | * H5VL_pdc_object_specific 92 | 93 | Any function not listed above is either not currently used by the VOL and therefore does nothing or is called by the VOL, but doesn't need to do anything relevant to the VOL and therefore does nothing. 94 | -------------------------------------------------------------------------------- /docs/source/hellopdcexample.rst: -------------------------------------------------------------------------------- 1 | ================================ 2 | Hello PDC Example 3 | ================================ 4 | 5 | --------------------------- 6 | PDC Hello World 7 | --------------------------- 8 | 9 | * pdc_init.c 10 | * A PDC program starts with PDCinit and finishes with PDCclose. 11 | * To a simple hello world program for PDC, use the following command. 12 | 13 | .. code-block:: Bash 14 | 15 | make pdc_init 16 | ./run_test.sh ./pdc_init 17 | 18 | * The script "run_test.sh" starts a server first. Then program "obj_get_data" is executed. Finally, the PDC servers are closed. 19 | * Alternatively, the following command can be used for multile MPI processes. 20 | 21 | 22 | .. code-block:: Bash 23 | 24 | make pdc_init 25 | ./mpi_test.sh ./pdc_init mpiexec 2 4 26 | 27 | * The above command will start a server with 2 processes. Then it will start the application program with 4 processes. Finally, all servers are closed. 28 | * On supercomputers, "mpiexec" can be replaced with "srun", "jsrun" or "aprun". -------------------------------------------------------------------------------- /docs/source/images/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hpc-io/pdc/89268e80d232f33758516cc26ddcb81f5a493700/docs/source/images/.DS_Store -------------------------------------------------------------------------------- /docs/source/images/pdc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hpc-io/pdc/89268e80d232f33758516cc26ddcb81f5a493700/docs/source/images/pdc.png -------------------------------------------------------------------------------- /docs/source/images/pdclogo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hpc-io/pdc/89268e80d232f33758516cc26ddcb81f5a493700/docs/source/images/pdclogo.png -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | .. PDC documentation master file, created by 2 | sphinx-quickstart on Thu Apr 15 14:28:56 2021. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Proactive Data Containers (PDC) 7 | =============================== 8 | 9 | Proactive Data Containers (PDC) software provides an object-focused data management API, a runtime system with a set of scalable data object management services, and tools for managing data objects stored in the PDC system. The PDC API allows efficient and transparent data movement in complex memory and storage hierarchy. The PDC runtime system performs data movement asynchronously and provides scalable metadata operations to find and manipulate data objects. PDC revolutionizes how data is managed and accessed by using object-centric abstractions to represent data that moves in the high-performance computing (HPC) memory and storage subsystems. PDC manages extensive metadata to describe data objects to find desired data efficiently as well as to store information in the data objects. 10 | 11 | More information and publications of PDC is available at https://sdm.lbl.gov/pdc 12 | 13 | If you use PDC in your research, please use the following citation: 14 | 15 | Byna, Suren, Dong, Bin, Tang, Houjun, Koziol, Quincey, Mu, Jingqing, Soumagne, Jerome, Vishwanath, Venkat, Warren, Richard, and Tessier, François. Proactive Data Containers (PDC) v0.1. Computer Software. https://github.com/hpc-io/pdc. USDOE. 11 May. 2017. Web. doi:10.11578/dc.20210325.1. 16 | 17 | .. toctree:: 18 | :maxdepth: 2 19 | :caption: Getting Started 20 | 21 | getting_started 22 | .. definitions 23 | .. assumptions 24 | 25 | .. toctree:: 26 | :maxdepth: 2 27 | :caption: Overview 28 | 29 | introduction 30 | hdf5vol 31 | .. performance 32 | 33 | .. toctree:: 34 | :maxdepth: 2 35 | :caption: Resources 36 | 37 | hellopdcexample 38 | api 39 | tools 40 | .. inflightanalysis 41 | .. futurework 42 | 43 | 44 | .. toctree:: 45 | :maxdepth: 2 46 | :caption: Developer Documentation 47 | 48 | developer-notes 49 | documentation/objects 50 | documentation/regions 51 | documentation/query 52 | documentation/analysis 53 | documentation/transformation 54 | 55 | 56 | Indices and tables 57 | ================== 58 | 59 | * :ref:`genindex` 60 | * :ref:`modindex` 61 | * :ref:`search` 62 | -------------------------------------------------------------------------------- /docs/source/inflightanalysis.rst: -------------------------------------------------------------------------------- 1 | ================================ 2 | In-flight Analysis 3 | ================================ -------------------------------------------------------------------------------- /docs/source/performance.rst: -------------------------------------------------------------------------------- 1 | ================================ 2 | Performance 3 | ================================ -------------------------------------------------------------------------------- /examples/C_plus_plus_example/H5Timing.h: -------------------------------------------------------------------------------- 1 | #ifndef H5_TIMING_H 2 | 3 | #define H5_TIMING_H 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #define PDC_PATCH 14 | 15 | #ifdef PDC_PATCH 16 | extern "C" { 17 | #include "pdc.h" 18 | } 19 | #endif 20 | 21 | #define H5_TIMING_ENABLE 22 | 23 | #ifdef H5_TIMING_ENABLE 24 | typedef struct H5Timer { 25 | double start; 26 | double end; 27 | size_t data_size; 28 | char * name; 29 | } H5Timer; 30 | 31 | typedef struct H5TimerClass { 32 | std::vector *dataset_timers; 33 | std::vector *dataset_sz_timers; 34 | std::vector *dataset_read_timers; 35 | std::vector *dataset_sz_read_timers; 36 | #ifdef PDC_PATCH 37 | int PDCwrite_count; 38 | int PDCread_count; 39 | int PDCstart_time; 40 | int PDCwait_time; 41 | #else 42 | int H5Dwrite_count; 43 | int H5Dread_count; 44 | double H5Dclose_time; 45 | double wrap_requests_time; 46 | double H5Dwrite_time; 47 | double H5Dread_time; 48 | #endif 49 | double merge_requests_time; 50 | double total_start_time; 51 | double total_end_time; 52 | } H5TimerClass; 53 | 54 | int init_timers(); 55 | int register_timer_start(double *start_time); 56 | #ifdef PDC_PATCH 57 | int register_PDCstart_timer_end(double start_time); 58 | int register_PDCwait_timer_end(double start_time); 59 | int increment_PDCwrite(); 60 | int increment_PDCread(); 61 | #else 62 | int register_wrap_requests_timer_end(double start_time); 63 | int register_H5Dclose_timer_end(double start_time); 64 | int register_H5Dwrite_timer_end(double start_time); 65 | int increment_H5Dwrite(); 66 | int increment_H5Dread(); 67 | #endif 68 | int register_merge_requests_timer_end(double start_time); 69 | int register_dataset_timer_start(const char *name); 70 | int register_dataset_timer_end(size_t data_size); 71 | int register_dataset_sz_timer_start(const char *name); 72 | int register_dataset_sz_timer_end(size_t data_size); 73 | int register_dataset_read_timer_start(const char *name); 74 | int register_dataset_read_timer_end(size_t data_size); 75 | int register_dataset_sz_read_timer_start(const char *name); 76 | int register_dataset_sz_read_timer_end(size_t data_size); 77 | int finalize_timers(); 78 | #endif 79 | 80 | #endif 81 | -------------------------------------------------------------------------------- /examples/C_plus_plus_example/Makefile: -------------------------------------------------------------------------------- 1 | CC = mpicxx 2 | CFLAGS = -O2 -Wall -Wextra 3 | 4 | INCLUDES = -I. -I$(PDC_DIR)/include -I$(HDF5_DIR)/include 5 | LDFLAGS = -Wl,-rpath,$(PDC_DIR)/lib -L$(PDC_DIR)/lib -L$(HDF5_DIR)/lib 6 | 7 | LIBS = -lpdc -lhdf5 8 | 9 | .cc.o: 10 | $(CC) $(CFLAGS) $(INCLUDES) -c $< 11 | 12 | all: region_transfer_1D_append 13 | 14 | 15 | region_transfer_1D_append: multidataset_plugin.o H5Timing.o region_transfer_1D_append.o 16 | $(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS) $(LIBS) 17 | 18 | 19 | 20 | clean: 21 | rm -f *.o region_transfer_1D_append 22 | -------------------------------------------------------------------------------- /examples/C_plus_plus_example/multidataset_plugin.h: -------------------------------------------------------------------------------- 1 | #ifndef MULTIDATASET_PLUGIN_H 2 | #define MULTIDATASET_PLUGIN_H 3 | 4 | //#include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include "H5Timing.h" 10 | #define ENABLE_MULTIDATASET 0 11 | #define MULTIDATASET_DEFINE 1 12 | 13 | #if MULTIDATASET_DEFINE == 1 14 | typedef struct H5D_rw_multi_t { 15 | hid_t dset_id; /* dataset ID */ 16 | hid_t dset_space_id; /* dataset selection dataspace ID */ 17 | hid_t mem_type_id; /* memory datatype ID */ 18 | hid_t mem_space_id; /* memory selection dataspace ID */ 19 | union { 20 | void * rbuf; /* pointer to read buffer */ 21 | const void *wbuf; /* pointer to write buffer */ 22 | } u; 23 | } H5D_rw_multi_t; 24 | #endif 25 | 26 | typedef struct multidataset_array { 27 | std::vector * temp_mem; 28 | std::vector *start; 29 | std::vector *end; 30 | hsize_t last_end; 31 | #ifdef PDC_PATCH 32 | pdcid_t did; 33 | #else 34 | hid_t did; 35 | #endif 36 | hid_t mtype; /* memory datatype ID */ 37 | } multidataset_array; 38 | 39 | int set_hdf5_method(int hdf5_method); 40 | int get_hdf5_method(); 41 | int init_multidataset(); 42 | int finalize_multidataset(); 43 | int register_multidataset_request_append(const char *name, hid_t gid, void *buf, hsize_t data_size, 44 | hid_t mtype); 45 | int flush_multidatasets(); 46 | // int check_write_status(); 47 | #endif 48 | -------------------------------------------------------------------------------- /examples/Makefile: -------------------------------------------------------------------------------- 1 | CC = mpicc 2 | CFLAGS = -O2 -Wall -Wextra 3 | 4 | INCLUDES = -I. -I$(PDC_DIR)/include 5 | LDFLAGS = -Wl,-rpath,$(PDC_DIR)/lib -L$(PDC_DIR)/lib 6 | 7 | LIBS = -lpdc -lpdc_commons 8 | 9 | .c.o: 10 | $(CC) $(CFLAGS) $(INCLUDES) -c $< 11 | 12 | all: region_transfer_2D region_transfer_3D pdc_init obj_get_data bdcats vpicio vpicio_batch bdcats_batch bdcats_old vpicio_old read_write_perf read_write_col_perf haccio haccio_v2 tileio tileio_v2 13 | 14 | 15 | region_transfer_2D: region_transfer_2D.o 16 | $(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS) $(LIBS) 17 | region_transfer_3D: region_transfer_3D.o 18 | $(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS) $(LIBS) 19 | region_obj_map_2D: region_obj_map_2D.o 20 | $(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS) $(LIBS) 21 | region_obj_map_3D: region_obj_map_3D.o 22 | $(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS) $(LIBS) 23 | obj_get_data: obj_get_data.o 24 | $(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS) $(LIBS) 25 | pdc_init: pdc_init.o 26 | $(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS) $(LIBS) 27 | bdcats: bdcats.o 28 | $(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS) $(LIBS) 29 | vpicio: vpicio.o 30 | $(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS) $(LIBS) 31 | vpicio_batch: vpicio_batch.o 32 | $(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS) $(LIBS) 33 | bdcats_batch: bdcats_batch.o 34 | $(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS) $(LIBS) 35 | bdcats_old: bdcats_old.o 36 | $(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS) $(LIBS) 37 | vpicio_old: vpicio_old.o 38 | $(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS) $(LIBS) 39 | haccio: haccio.o 40 | $(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS) $(LIBS) 41 | haccio_v2: haccio_v2.o 42 | $(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS) $(LIBS) 43 | tileio: tileio.o 44 | $(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS) $(LIBS) 45 | tileio_v2: tileio_v2.o 46 | $(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS) $(LIBS) 47 | read_write_perf: read_write_perf.o 48 | $(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS) $(LIBS) 49 | read_write_col_perf: read_write_col_perf.o 50 | $(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS) $(LIBS) 51 | 52 | 53 | clean: 54 | rm -f *.o region_transfer_2D region_transfer_3D pdc_init obj_get_data vpicio vpicio_batch bdcats vpicio_old bdcats_old read_write_perf read_write_col_perf haccio tileio haccio_v2 tileio_v2 55 | -------------------------------------------------------------------------------- /examples/llsm/.gitignore: -------------------------------------------------------------------------------- 1 | build -------------------------------------------------------------------------------- /examples/llsm/llsm_aux/parallelReadTiff.h: -------------------------------------------------------------------------------- 1 | #ifndef PARALLELREADTIFF_H 2 | #define PARALLELREADTIFF_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | typedef struct { 14 | uint64_t *range; 15 | size_t length; 16 | } parallel_tiff_range_t; 17 | 18 | typedef struct { 19 | uint64_t x; 20 | uint64_t y; 21 | uint64_t z; 22 | uint64_t bits; 23 | uint64_t startSlice; 24 | uint64_t stripeSize; 25 | uint64_t is_imageJ; 26 | uint64_t imageJ_Z; 27 | void * tiff_ptr; 28 | size_t tiff_size; 29 | } image_info_t; 30 | 31 | void parallel_TIFF_load(char *fileName, uint8_t flipXY, parallel_tiff_range_t *strip_range, 32 | image_info_t **image_info_t); 33 | 34 | #endif // PARALLELREADTIFF_H -------------------------------------------------------------------------------- /examples/mpi_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # This version of the test runner doesn't attempt to run any parallel tests. 3 | # We assume too, that if the library build has enabled MPI, that LD_LIBRARY_PATH is 4 | # defined and points to the MPI libraries used by the linker (e.g. -L 26 | #include 27 | #include 28 | #include "pdc.h" 29 | 30 | int 31 | main(int argc, char **argv) 32 | { 33 | pdcid_t pdc; 34 | int rank = 0, size = 1; 35 | int ret_value = 0; 36 | 37 | // create a pdc 38 | #ifdef ENABLE_MPI 39 | MPI_Init(&argc, &argv); 40 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 41 | MPI_Comm_size(MPI_COMM_WORLD, &size); 42 | #endif 43 | 44 | pdc = PDCinit("pdc"); 45 | LOGI("generated new pdc\n"); 46 | 47 | // close pdc 48 | if (PDCclose(pdc) < 0) { 49 | LOGI("Failed to close PDC\n"); 50 | ret_value = 1; 51 | } 52 | #ifdef ENABLE_MPI 53 | MPI_Finalize(); 54 | #endif 55 | return ret_value; 56 | } 57 | -------------------------------------------------------------------------------- /examples/run_checkpoint_restart_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # This version of the test runner doesn't attempt to run any parallel tests. 3 | # We assume too, that if the library build has enabled MPI, that LD_LIBRARY_PATH is 4 | # defined and points to the MPI libraries used by the linker (e.g. -L= 4) 20 | #define PDC_ABI_IMPORT __attribute__((visibility("default"))) 21 | #define PDC_ABI_EXPORT __attribute__((visibility("default"))) 22 | #define PDC_ABI_HIDDEN __attribute__((visibility("hidden"))) 23 | #else 24 | #define PDC_ABI_IMPORT 25 | #define PDC_ABI_EXPORT 26 | #define PDC_ABI_HIDDEN 27 | #endif 28 | 29 | #cmakedefine PDC_BUILD_SHARED_LIBS 30 | 31 | #ifdef PDC_BUILD_SHARED_LIBS 32 | #ifdef pdc_EXPORTS 33 | #define PDC_EXPORT PDC_ABI_EXPORT 34 | #else 35 | #define PDC_EXPORT PDC_ABI_IMPORT 36 | #endif 37 | #else 38 | #define PDC_EXPORT 39 | #endif 40 | 41 | /* TODO To be fixed 42 | #ifdef _WIN32 43 | typedef signed __int64 pdc_int64_t; 44 | typedef signed __int32 pdc_int32_t; 45 | typedef signed __int16 pdc_int16_t; 46 | typedef signed __int8 pdc_int8_t; 47 | typedef unsigned __int64 pdc_uint64_t; 48 | typedef unsigned __int32 pdc_uint32_t; 49 | typedef unsigned __int16 pdc_uint16_t; 50 | typedef unsigned __int8 pdc_uint8_t; 51 | #else 52 | #include 53 | #include 54 | typedef int64_t pdc_int64_t; 55 | typedef int32_t pdc_int32_t; 56 | typedef int16_t pdc_int16_t; 57 | typedef int8_t pdc_int8_t; 58 | typedef uint64_t pdc_uint64_t; 59 | typedef uint32_t pdc_uint32_t; 60 | typedef uint16_t pdc_uint16_t; 61 | typedef uint8_t pdc_uint8_t; 62 | #endif 63 | typedef pdc_uint64_t pdc_ptr_t; 64 | typedef pdc_uint8_t pdc_bool_t; 65 | #define PDC_TRUE 1 66 | #define PDC_FALSE 0 67 | 68 | #ifdef _WIN32 69 | #define PDC_INLINE __inline 70 | #else 71 | #define PDC_INLINE __inline__ 72 | #endif 73 | */ 74 | 75 | #endif /* _pdc_config_H */ 76 | -------------------------------------------------------------------------------- /pdc_config_sys.h.cmake: -------------------------------------------------------------------------------- 1 | /* config.h Generated by cmake with CMakelists.txt. from config.h.cmake */ 2 | 3 | /* Define if you want to enable MPI */ 4 | #cmakedefine ENABLE_MPI 5 | 6 | /* Define if you want to use lustre */ 7 | #cmakedefine ENABLE_LUSTRE 8 | 9 | /* Define if you want to enable timing */ 10 | #cmakedefine PDC_TIMING 11 | 12 | /* Define if you want to enable server cache */ 13 | #cmakedefine PDC_SERVER_CACHE 14 | 15 | /* Define the max region cache size in GB */ 16 | #cmakedefine PDC_SERVER_CACHE_MAX_GB @PDC_SERVER_CACHE_MAX_GB@ 17 | 18 | /* Define region cache flush time interval */ 19 | #cmakedefine PDC_SERVER_CACHE_FLUSH_TIME @PDC_SERVER_CACHE_FLUSH_TIME@ 20 | 21 | /* Define if you want to enable checkpoint */ 22 | #cmakedefine PDC_ENABLE_CHECKPOINT 23 | 24 | /* Define if you want to enable profiling */ 25 | #cmakedefine ENABLE_PROFILING 26 | 27 | /* Define if you want to enable DART_SUFFIX_TREE mode */ 28 | #cmakedefine PDC_DART_SFX_TREE 29 | 30 | /* Define if you want to enable Julia */ 31 | #cmakedefine PDC_ENABLE_JULIA 32 | 33 | /* Define if you want to enable multithread */ 34 | #cmakedefine ENABLE_MULTITHREAD 35 | 36 | /* Define if compiler supports attributes */ 37 | #cmakedefine HAVE_ATTRIBUTE 38 | 39 | /* Define if has Cray DRC */ 40 | #cmakedefine PDC_HAS_CRAY_DRC 41 | 42 | /* Define if use shared server */ 43 | #cmakedefine PDC_HAS_SHARED_SERVER 44 | 45 | /* Define if closing server by application */ 46 | #cmakedefine ENABLE_APP_CLOSE_SERVER 47 | 48 | /* Define if wait for data to be fanalized when closing dataset */ 49 | #cmakedefine ENABLE_WAIT_DATA 50 | 51 | /* Define size of float type */ 52 | #cmakedefine VAR_SIZE_FLOAT @VAR_SIZE_FLOAT@ 53 | 54 | /* Define size of double type */ 55 | #cmakedefine VAR_SIZE_DOUBLE @VAR_SIZE_DOUBLE@ 56 | 57 | /* Define size of char type */ 58 | #cmakedefine VAR_SIZE_CHAR @VAR_SIZE_CHAR@ 59 | 60 | /* Define size of short int type */ 61 | #cmakedefine VAR_SIZE_SINT @VAR_SIZE_SINT@ 62 | 63 | /* Define size of int type */ 64 | #cmakedefine VAR_SIZE_INT @VAR_SIZE_INT@ 65 | 66 | /* Define size of long int type */ 67 | #cmakedefine VAR_SIZE_LINT @VAR_SIZE_LINT@ 68 | 69 | /* Define size of long long int type */ 70 | #cmakedefine VAR_SIZE_LLINT @VAR_SIZE_LLINT@ 71 | 72 | /* Define size of unsigned int type */ 73 | #cmakedefine VAR_SIZE_UINT @VAR_SIZE_UINT@ 74 | 75 | /* Define size of unsigned long int type */ 76 | #cmakedefine VAR_SIZE_ULINT @VAR_SIZE_ULINT@ 77 | 78 | /* Define size of unsigned long long int type */ 79 | #cmakedefine VAR_SIZE_ULLINT @VAR_SIZE_ULLINT@ 80 | 81 | /* Define size of int64_t int type */ 82 | #cmakedefine VAR_SIZE_64INT @VAR_SIZE_64INT@ 83 | 84 | /* Define size of uint64_t int type */ 85 | #cmakedefine VAR_SIZE_U64INT @VAR_SIZE_U64INT@ 86 | -------------------------------------------------------------------------------- /pdc_naming_policy.txt: -------------------------------------------------------------------------------- 1 | {\rtf1\ansi\ansicpg1252\cocoartf1561\cocoasubrtf600 2 | {\fonttbl\f0\fswiss\fcharset0 Helvetica;} 3 | {\colortbl;\red255\green255\blue255;\red38\green38\blue38;} 4 | {\*\expandedcolortbl;;\csgenericrgb\c14902\c14902\c14902;} 5 | \margl1440\margr1440\vieww12600\viewh10200\viewkind0 6 | \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0 7 | 8 | \f0\fs28 \cf0 PDC funtion, struct, typedef naming policy\ 9 | \ 10 | \pard\pardeftab720\sl400\partightenfactor0 11 | \cf2 \expnd0\expndtw0\kerning0 12 | Public function: starting with capital PDC and no underscore "_" after, example\ 13 | PDCinit()\ 14 | PDCprop_create()\ 15 | \ 16 | Private function: starting with capital PDC and underscore "_" after, example\ 17 | PDC_prop_init()\ 18 | \ 19 | Public struct: starting with lower case pdc and underscore "_" after, example\ 20 | pdc_request\ 21 | \ 22 | Private struct: starting with an underscore "_" followed by lower case pdc and underscore "_" after, example\ 23 | _pdc_server_info\ 24 | \ 25 | Public typedef: starting with lower case pdc and underscore "_" after, and ending with "_t", example\ 26 | pdc_var_type_t\ 27 | \ 28 | Private typedef: starting with an underscore "_" followed by lower case pdc and underscore "_" after, and ending with "_t", example\ 29 | _pdc_loci_t\ 30 | } -------------------------------------------------------------------------------- /pdc_start: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | SRCDIR=`dirname ${BASH_SOURCE[0]}` 4 | 5 | export PDC_ENABLE_MPI=OFF 6 | export ENABLE_LUSTRE=OFF 7 | export PDC_TIMING=OFF 8 | export PDC_ENABLE_CHECKPOINT=ON 9 | export PDC_SERVER_CACHE=ON 10 | export ENABLE_MULTITHREAD=OFF 11 | export BUILD_TESTING=ON 12 | export BUILD_MPI_TESTING=OFF 13 | export BUILD_DOCUMENTATION=OFF 14 | export MPI_TEST_CMD=mpiexec 15 | export CMAKE_PREFIX_PATH=/Users/kmu/Research/PDC/tools/mercury/share/cmake/mercury 16 | export PDC_DATA_LOC=/Users/kmu/Research/PDC/pdc/src/build/gen_data 17 | export HG_TRANSPORT=bmi+tcp 18 | 19 | cmake $SRCDIR 20 | -------------------------------------------------------------------------------- /scripts/dart_attr_dist_test/clean.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | MAX_NODE=512 3 | 4 | for (( i = 1; i <= $MAX_NODE; i*=2 )); do 5 | 6 | rm -rf $i/* 7 | 8 | done -------------------------------------------------------------------------------- /scripts/dart_attr_dist_test/gen_script.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | PROJECT_NAME=$1 4 | 5 | # Per node configuration of your HPC system. 6 | MAX_PYSICAL_CORE=128 7 | MAX_HYPERTHREADING=2 8 | 9 | # Designated number of threads per process on each node 10 | # (this should be associated with -c option in srun) 11 | NUM_THREAD_PER_SERVER_PROC=2 12 | NUM_THREAD_PER_CLIENT_PROC=2 13 | 14 | 15 | # Designated number of processes for server anc client on each node 16 | # (this should be associated with -n option in srun) 17 | NUM_SERVER_PROC_PER_NODE=1 18 | NUM_CLIENT_PROC_PER_NODE=64 19 | 20 | 21 | MAX_NODE=512 22 | MAX_ATTR=1024 23 | MAX_ATTRLEN=1000 24 | 25 | PROG_BASENAME=dart_attr_dist_test 26 | 27 | for (( i = 1; i <= $MAX_NODE; i*=2 )); do 28 | mkdir -p $i 29 | JOBNAME=${PROG_BASENAME}_${i} 30 | TARGET=./$i/$JOBNAME.sbatch 31 | cp template.sh $TARGET 32 | sed -i "s/JOBNAME/${JOBNAME}/g" $TARGET 33 | sed -i "s/NODENUM/${i}/g" $TARGET 34 | sed -i "s/MPHYSICALCORE/${MAX_PYSICAL_CORE}/g" $TARGET 35 | sed -i "s/MHYPERTHREADING/${MAX_HYPERTHREADING}/g" $TARGET 36 | sed -i "s/N_SERVER_PROC/${NUM_SERVER_PROC_PER_NODE}/g" $TARGET 37 | sed -i "s/N_CLIENT_PROC/${NUM_CLIENT_PROC_PER_NODE}/g" $TARGET 38 | sed -i "s/NTHREAD_PER_SPROC/${NUM_THREAD_PER_SERVER_PROC}/g" $TARGET 39 | sed -i "s/NTHREAD_PER_CPROC/${NUM_THREAD_PER_CLIENT_PROC}/g" $TARGET 40 | sed -i "s/PROJNAME/${PROJECT_NAME}/g" $TARGET 41 | if [[ "$i" -gt "4" ]]; then 42 | sed -i "s/REG//g" $TARGET 43 | else 44 | sed -i "s/DBG//g" $TARGET 45 | fi 46 | done 47 | -------------------------------------------------------------------------------- /scripts/dart_attr_dist_test/submit.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # MIN_PROC=4 4 | # MAX_PROC=128 5 | MIN_PROC=1 6 | MAX_PROC=512 7 | MAX_ATTR=1024 8 | MAX_ATTRLEN=1000 9 | 10 | PROG_BASENAME=llsm_importer 11 | 12 | curdir=$(pwd) 13 | 14 | first_submit=1 15 | 16 | 17 | for (( i = 1; i <= $MAX_PROC; i*=2 )); do 18 | mkdir -p $i 19 | JOBNAME=${PROG_BASENAME}_${i} 20 | TARGET=./$i/JOBNAME.sh 21 | 22 | njob=`squeue -u $USER | grep ${PROG_BASENAME} | wc -l` 23 | echo $njob 24 | while [ $njob -ge 4 ] 25 | do 26 | sleeptime=$[ ( $RANDOM % 1000 ) ] 27 | sleep $sleeptime 28 | njob=`squeue -u $USER | grep ${PROG_BASENAME} | wc -l` 29 | echo $njob 30 | done 31 | 32 | if [[ $first_submit == 1 ]]; then 33 | # Submit first job w/o dependency 34 | echo "Submitting $TARGET" 35 | job=`sbatch $TARGET` 36 | first_submit=0 37 | else 38 | echo "Submitting $TARGET after ${job: -8}" 39 | job=`sbatch -d afterany:${job: -8} $TARGET` 40 | fi 41 | 42 | sleeptime=$[ ( $RANDOM % 5 ) ] 43 | sleep $sleeptime 44 | done 45 | 46 | 47 | # for (( j = $MIN_PROC; j <= $MAX_PROC ; j*=2 )); do 48 | 49 | # njob=`squeue -u $USER | grep vpic | wc -l` 50 | # echo $njob 51 | # while [ $njob -ge 4 ] 52 | # do 53 | # sleeptime=$[ ( $RANDOM % 1000 ) ] 54 | # sleep $sleeptime 55 | # njob=`squeue -u $USER | grep vpic | wc -l` 56 | # echo $njob 57 | # done 58 | 59 | 60 | # cd $curdir/$j 61 | # for filename in ./*.sh ; do 62 | 63 | # if [[ $first_submit == 1 ]]; then 64 | # # Submit first job w/o dependency 65 | # echo "Submitting $filename" 66 | # job=`sbatch $filename` 67 | # first_submit=0 68 | # else 69 | # echo "Submitting $filename after ${job: -8}" 70 | # job=`sbatch -d afterany:${job: -8} $filename` 71 | # fi 72 | 73 | # sleeptime=$[ ( $RANDOM % 5 ) ] 74 | # sleep $sleeptime 75 | 76 | # done 77 | # done 78 | -------------------------------------------------------------------------------- /scripts/kvtag_add_get_benchmark/perlmutter/clean.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | MAX_NODE=512 3 | 4 | for (( i = 1; i <= $MAX_NODE; i*=2 )); do 5 | 6 | rm -rf $i/* 7 | 8 | done -------------------------------------------------------------------------------- /scripts/kvtag_add_get_benchmark/perlmutter/gen_scripts.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | N_THREAD=NO 3 | MAX_NODE=512 4 | MAX_ATTR=1024 5 | MAX_ATTRLEN=1000000 6 | 7 | PROJECT_NAME=$1 8 | 9 | for (( i = 1; i <= $MAX_NODE; i*=2 )); do 10 | mkdir -p $i 11 | for (( j = 1; j <= $MAX_ATTR; j*=4 )); do 12 | for (( k = 100; k <= $MAX_ATTRLEN; k*=10 )); do 13 | JOBNAME=kvtag_bench_${i}_${j}_${k} 14 | TARGET=./$i/$JOBNAME.sbatch 15 | cp template.sh $TARGET 16 | sed -i "s/JOBNAME/${JOBNAME}/g" $TARGET 17 | sed -i "s/NODENUM/${i}/g" $TARGET 18 | sed -i "s/ATTRNUM/${j}/g" $TARGET 19 | sed -i "s/ATTRLEN/${k}/g" $TARGET 20 | sed -i "s/PROJNAME/${PROJECT_NAME}/g" $TARGET 21 | if [[ "$i" -gt "16" ]]; then 22 | sed -i "s/REG//g" $TARGET 23 | else 24 | sed -i "s/DBG//g" $TARGET 25 | fi 26 | done 27 | done 28 | done 29 | -------------------------------------------------------------------------------- /scripts/kvtag_add_get_benchmark/perlmutter/submit.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # MIN_PROC=4 4 | # MAX_PROC=128 5 | MIN_PROC=1 6 | MAX_PROC=512 7 | MAX_ATTR=1024 8 | MAX_ATTRLEN=1000 9 | 10 | curdir=$(pwd) 11 | 12 | first_submit=1 13 | 14 | for (( i = 1; i <= $MAX_PROC; i*=2 )); do 15 | mkdir -p $i 16 | for (( j = 1; j <= $MAX_ATTR; j*=4 )); do 17 | for (( k = 100; k <= $MAX_ATTRLEN; k*=10 )); do 18 | JOBNAME=kvtag_bench_${i}_${j}_${k} 19 | TARGET=./$i/JOBNAME.sh 20 | 21 | njob=`squeue -u $USER | grep kvtag_bench | wc -l` 22 | echo $njob 23 | while [ $njob -ge 4 ] 24 | do 25 | sleeptime=$[ ( $RANDOM % 1000 ) ] 26 | sleep $sleeptime 27 | njob=`squeue -u $USER | grep kvtag_bench | wc -l` 28 | echo $njob 29 | done 30 | 31 | if [[ $first_submit == 1 ]]; then 32 | # Submit first job w/o dependency 33 | echo "Submitting $TARGET" 34 | job=`sbatch $TARGET` 35 | first_submit=0 36 | else 37 | echo "Submitting $TARGET after ${job: -8}" 38 | job=`sbatch -d afterany:${job: -8} $TARGET` 39 | fi 40 | 41 | sleeptime=$[ ( $RANDOM % 5 ) ] 42 | sleep $sleeptime 43 | done 44 | done 45 | done 46 | 47 | 48 | # for (( j = $MIN_PROC; j <= $MAX_PROC ; j*=2 )); do 49 | 50 | # njob=`squeue -u $USER | grep vpic | wc -l` 51 | # echo $njob 52 | # while [ $njob -ge 4 ] 53 | # do 54 | # sleeptime=$[ ( $RANDOM % 1000 ) ] 55 | # sleep $sleeptime 56 | # njob=`squeue -u $USER | grep vpic | wc -l` 57 | # echo $njob 58 | # done 59 | 60 | 61 | # cd $curdir/$j 62 | # for filename in ./*.sh ; do 63 | 64 | # if [[ $first_submit == 1 ]]; then 65 | # # Submit first job w/o dependency 66 | # echo "Submitting $filename" 67 | # job=`sbatch $filename` 68 | # first_submit=0 69 | # else 70 | # echo "Submitting $filename after ${job: -8}" 71 | # job=`sbatch -d afterany:${job: -8} $filename` 72 | # fi 73 | 74 | # sleeptime=$[ ( $RANDOM % 5 ) ] 75 | # sleep $sleeptime 76 | 77 | # done 78 | # done 79 | -------------------------------------------------------------------------------- /scripts/kvtag_add_get_benchmark/perlmutter/template.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -l 2 | 3 | #REGSBATCH -q regular 4 | #DBGSBATCH -q debug 5 | #SBATCH -N NODENUM 6 | #REGSBATCH -t 1:00:00 7 | #DBGSBATCH -t 0:30:00 8 | #SBATCH -C cpu 9 | #SBATCH -J JOBNAME 10 | #SBATCH -A PROJNAME 11 | #SBATCH -o o%j.JOBNAME.out 12 | #SBATCH -e o%j.JOBNAME.out 13 | 14 | 15 | # export PDC_DEBUG=0 16 | 17 | export PDC_TMPDIR=$SCRATCH/data/pdc/conf 18 | 19 | rm -rf $PDC_TMPDIR/* 20 | 21 | REPEAT=1 22 | 23 | N_NODE=NODENUM 24 | # NCLIENT=127 25 | NCLIENT=31 26 | 27 | export PDC_TMPDIR=${PDC_TMPDIR}/$N_NODE 28 | mkdir -p $PDC_TMPDIR 29 | 30 | let TOTALPROC=$NCLIENT*$N_NODE 31 | 32 | EXECPATH=/global/cfs/cdirs/m2621/wzhang5/perlmutter/install/pdc/share/test/bin 33 | SERVER=$EXECPATH/pdc_server.exe 34 | CLIENT=$EXECPATH/kvtag_add_get_benchmark 35 | CLOSE=$EXECPATH/close_server 36 | 37 | chmod +x $EXECPATH/* 38 | 39 | MAX_OBJ_COUNT=$((1024*1024)) 40 | OBJ_INCR=$((MAX_OBJ_COUNT/1024)) 41 | ATTR_COUNT=ATTRNUM 42 | ATTR_LENGTH=ATTRLEN 43 | QUERY_COUNT=$((OBJ_INCR)) 44 | 45 | date 46 | 47 | 48 | echo "" 49 | echo "=============" 50 | echo "$i Init server" 51 | echo "=============" 52 | stdbuf -i0 -o0 -e0 srun -N $N_NODE -n $((N_NODE*1)) -c 2 --cpu_bind=cores $SERVER & 53 | sleep 5 54 | 55 | 56 | echo "============================================" 57 | echo "KVTAGS with $N_NODE nodes" 58 | echo "============================================" 59 | stdbuf -i0 -o0 -e0 srun -N $N_NODE -n $TOTALPROC -c 2 --cpu_bind=cores $CLIENT $MAX_OBJ_COUNT $OBJ_INCR $ATTR_COUNT $ATTR_LENGTH $QUERY_COUNT $N_NODE 60 | 61 | echo "" 62 | echo "=================" 63 | echo "$i Closing server" 64 | echo "=================" 65 | stdbuf -i0 -o0 -e0 srun -N 1 -n 1 -c 2 --mem=25600 --cpu_bind=cores $CLOSE 66 | 67 | date 68 | -------------------------------------------------------------------------------- /scripts/kvtag_add_get_scale/perlmutter/clean.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | MAX_NODE=512 3 | 4 | for (( i = 1; i <= $MAX_NODE; i*=2 )); do 5 | 6 | rm -rf $i/* 7 | 8 | done -------------------------------------------------------------------------------- /scripts/kvtag_add_get_scale/perlmutter/gen_scripts.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | N_THREAD=NO 3 | MAX_NODE=512 4 | MAX_ATTR=1024 5 | MAX_ATTRLEN=1000 6 | 7 | PROJECT_NAME=$1 8 | 9 | for (( i = 1; i <= $MAX_NODE; i*=2 )); do 10 | mkdir -p $i 11 | JOBNAME=kvtag_scale_${i} 12 | TARGET=./$i/$JOBNAME.sbatch 13 | cp template.sh $TARGET 14 | sed -i "s/JOBNAME/${JOBNAME}/g" $TARGET 15 | sed -i "s/NODENUM/${i}/g" $TARGET 16 | sed -i "s/PROJNAME/${PROJECT_NAME}/g" $TARGET 17 | if [[ "$i" -gt "16" ]]; then 18 | sed -i "s/REG//g" $TARGET 19 | else 20 | sed -i "s/DBG//g" $TARGET 21 | fi 22 | done 23 | -------------------------------------------------------------------------------- /scripts/kvtag_add_get_scale/perlmutter/submit.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # MIN_PROC=4 4 | # MAX_PROC=128 5 | MIN_PROC=1 6 | MAX_PROC=512 7 | MAX_ATTR=1024 8 | MAX_ATTRLEN=1000 9 | 10 | curdir=$(pwd) 11 | 12 | first_submit=1 13 | 14 | for (( i = 1; i <= $MAX_PROC; i*=2 )); do 15 | mkdir -p $i 16 | JOBNAME=kvtag_scale_${i} 17 | TARGET=./$i/JOBNAME.sh 18 | 19 | njob=`squeue -u $USER | grep kvtag_scale | wc -l` 20 | echo $njob 21 | while [ $njob -ge 4 ] 22 | do 23 | sleeptime=$[ ( $RANDOM % 1000 ) ] 24 | sleep $sleeptime 25 | njob=`squeue -u $USER | grep kvtag_scale | wc -l` 26 | echo $njob 27 | done 28 | 29 | if [[ $first_submit == 1 ]]; then 30 | # Submit first job w/o dependency 31 | echo "Submitting $TARGET" 32 | job=`sbatch $TARGET` 33 | first_submit=0 34 | else 35 | echo "Submitting $TARGET after ${job: -8}" 36 | job=`sbatch -d afterany:${job: -8} $TARGET` 37 | fi 38 | 39 | sleeptime=$[ ( $RANDOM % 5 ) ] 40 | sleep $sleeptime 41 | done 42 | 43 | 44 | # for (( j = $MIN_PROC; j <= $MAX_PROC ; j*=2 )); do 45 | 46 | # njob=`squeue -u $USER | grep vpic | wc -l` 47 | # echo $njob 48 | # while [ $njob -ge 4 ] 49 | # do 50 | # sleeptime=$[ ( $RANDOM % 1000 ) ] 51 | # sleep $sleeptime 52 | # njob=`squeue -u $USER | grep vpic | wc -l` 53 | # echo $njob 54 | # done 55 | 56 | 57 | # cd $curdir/$j 58 | # for filename in ./*.sh ; do 59 | 60 | # if [[ $first_submit == 1 ]]; then 61 | # # Submit first job w/o dependency 62 | # echo "Submitting $filename" 63 | # job=`sbatch $filename` 64 | # first_submit=0 65 | # else 66 | # echo "Submitting $filename after ${job: -8}" 67 | # job=`sbatch -d afterany:${job: -8} $filename` 68 | # fi 69 | 70 | # sleeptime=$[ ( $RANDOM % 5 ) ] 71 | # sleep $sleeptime 72 | 73 | # done 74 | # done 75 | -------------------------------------------------------------------------------- /scripts/kvtag_add_get_scale/perlmutter/template.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -l 2 | 3 | #REGSBATCH -q regular 4 | #DBGSBATCH -q debug 5 | #SBATCH -N NODENUM 6 | #REGSBATCH -t 1:00:00 7 | #DBGSBATCH -t 0:30:00 8 | #SBATCH -C cpu 9 | #SBATCH -J JOBNAME 10 | #SBATCH -A PROJNAME 11 | #SBATCH -o o%j.JOBNAME.out 12 | #SBATCH -e o%j.JOBNAME.out 13 | 14 | # export PDC_DEBUG=0 15 | 16 | export PDC_TMPDIR=$SCRATCH/data/pdc/conf 17 | 18 | rm -rf $PDC_TMPDIR/* 19 | 20 | REPEAT=1 21 | 22 | N_NODE=NODENUM 23 | NCLIENT=31 24 | # NCLIENT=126 25 | 26 | export PDC_TMPDIR=${PDC_TMPDIR}/$N_NODE 27 | mkdir -p $PDC_TMPDIR 28 | 29 | let TOTALPROC=$NCLIENT*$N_NODE 30 | 31 | EXECPATH=/global/cfs/cdirs/m2621/wzhang5/perlmutter/install/pdc/share/test/bin 32 | SERVER=$EXECPATH/pdc_server.exe 33 | CLIENT=$EXECPATH/kvtag_add_get_scale 34 | CLOSE=$EXECPATH/close_server 35 | 36 | chmod +x $EXECPATH/* 37 | 38 | NUM_OBJ=$((1024*1024*1024)) 39 | NUM_TAGS=$NUM_OBJ 40 | NUM_QUERY=$((NUM_OBJ)) 41 | 42 | date 43 | 44 | 45 | echo "" 46 | echo "=============" 47 | echo "$i Init server" 48 | echo "=============" 49 | stdbuf -i0 -o0 -e0 srun -N $N_NODE -n $((N_NODE*1)) -c 2 --cpu_bind=cores $SERVER & 50 | sleep 5 51 | 52 | 53 | echo "============================================" 54 | echo "KVTAGS with $N_NODE nodes" 55 | echo "============================================" 56 | stdbuf -i0 -o0 -e0 srun -N $N_NODE -n $TOTALPROC -c 2 --cpu_bind=cores $CLIENT $NUM_OBJ $NUM_TAGS $NUM_QUERY 57 | 58 | echo "" 59 | echo "=================" 60 | echo "$i Closing server" 61 | echo "=================" 62 | stdbuf -i0 -o0 -e0 srun -N 1 -n 1 -c 2 --mem=25600 --cpu_bind=cores $CLOSE 63 | 64 | date 65 | -------------------------------------------------------------------------------- /scripts/kvtag_affix_query_scale/clean.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [[ "$#" -ne 1 ]]; then 4 | echo "Usage: $0 " 5 | exit 1 6 | fi 7 | 8 | # test if $1 is a single digit number between 0 and 1 (inclusive) 9 | if [[ "$1" =~ ^[0-1]$ ]]; then 10 | echo "Error: clean_dir should be a single digit number between 0 and 1 (inclusive). 1 means clean the directories named with numbers, 0 means clean the sbatch script only." 11 | exit 1 12 | fi 13 | 14 | CLEAN_DIR=$1 15 | MAX_NODE=512 16 | 17 | # if CLEAN_DIR is set to '1', then clean all the directories named with numbers, otherwise, clean the sbatch script only 18 | find ./ -name "*.sbatch*" -delete 19 | 20 | if [[ "$CLEAN_DIR" -eq "1" ]]; then 21 | for (( i = 1; i <= $MAX_NODE; i*=2 )); do 22 | 23 | rm -rf $i/* 24 | 25 | done 26 | fi -------------------------------------------------------------------------------- /scripts/kvtag_affix_query_scale/submit.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | MIN_PROC=16 4 | MAX_PROC=128 5 | 6 | PROG_BASENAME=kvafxqry 7 | 8 | curdir=$(pwd) 9 | 10 | first_submit=1 11 | 12 | if [[ "$#" -ne 3 ]]; then 13 | echo "Usage: $0 " 14 | exit 1 15 | fi 16 | 17 | # test if $1 is a single digit number between 0 and 1 (inclusive) 18 | re='^[0-1]$' 19 | if ! [[ "$1" =~ $re ]]; then 20 | echo "Error: i_type should be a single digit number between 0 and 1 (inclusive), 0 means not using index, 1 means using index" 21 | exit 1 22 | fi 23 | 24 | # test if $2 is a single digit number between 0 and 3 (inclusive) 25 | re='^[0-3]$' 26 | if ! [[ "$2" =~ $re ]]; then 27 | echo "Error: q_type should be a single digit number between 0 and 3 (inclusive), 0: exact query, 1: prefix query, 2: suffix query, 3: infix query" 28 | exit 1 29 | fi 30 | 31 | # test if $3 is a single digit number between 0 and 1 (inclusive) 32 | re='^[0-1]$' 33 | if ! [[ "$3" =~ $re ]]; then 34 | echo "Error: c_type should be a single digit number between 0 and 1 (inclusive), 0 means using non-collective mode, 1 means using collective mode" 35 | exit 1 36 | fi 37 | 38 | 39 | i_type=$1 40 | q_type=$2 41 | c_type=$3 42 | 43 | for (( i = $MIN_PROC; i <= $MAX_PROC; i*=2 )); do 44 | 45 | cd $curdir/$i 46 | 47 | JOBNAME=${PROG_BASENAME}_${i}_${i_type}_${q_type}_${c_type} 48 | TARGET=$JOBNAME.sbatch 49 | 50 | njob=`squeue -u $USER | grep ${PROG_BASENAME} | wc -l` 51 | echo $njob 52 | while [ $njob -ge 16 ] 53 | do 54 | sleeptime=$[ ( $RANDOM % 5 ) ] 55 | sleep $sleeptime 56 | njob=`squeue -u $USER | grep ${PROG_BASENAME} | wc -l` 57 | echo $njob 58 | done 59 | 60 | if [[ $first_submit == 1 ]]; then 61 | # Submit first job w/o dependency 62 | echo "Submitting $TARGET" 63 | job=`sbatch $TARGET` 64 | first_submit=0 65 | else 66 | echo "Submitting $TARGET after ${job: -8}" 67 | job=`sbatch -d afterany:${job: -8} $TARGET` 68 | fi 69 | 70 | sleeptime=$[ ( $RANDOM % 5 ) ] 71 | sleep $sleeptime 72 | 73 | done 74 | -------------------------------------------------------------------------------- /scripts/kvtag_query_scale/clean.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | MAX_NODE=512 3 | 4 | for (( i = 1; i <= $MAX_NODE; i*=2 )); do 5 | 6 | rm -rf $i/* 7 | 8 | done -------------------------------------------------------------------------------- /scripts/kvtag_query_scale/gen_script.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | PROJECT_NAME=$1 4 | 5 | # Per node configuration of your HPC system. 6 | MAX_PYSICAL_CORE=128 7 | MAX_HYPERTHREADING=2 8 | 9 | # Designated number of threads per process on each node 10 | # (this should be associated with -c option in srun) 11 | NUM_THREAD_PER_SERVER_PROC=2 12 | NUM_THREAD_PER_CLIENT_PROC=2 13 | 14 | 15 | # Designated number of processes for server anc client on each node 16 | # (this should be associated with -n option in srun) 17 | NUM_SERVER_PROC_PER_NODE=1 18 | NUM_CLIENT_PROC_PER_NODE=1 19 | 20 | 21 | MAX_NODE=512 22 | MAX_ATTR=1024 23 | MAX_ATTRLEN=1000 24 | 25 | PROG_BASENAME=kvtag_query_scale 26 | 27 | for (( i = 1; i <= $MAX_NODE; i*=2 )); do 28 | for (( j = 0; j <= 1; j+=1 )); do 29 | mkdir -p $i 30 | JOBNAME=${PROG_BASENAME}_${i} 31 | TARGET=./$i/$JOBNAME.sbatch.${j} 32 | cp template.sh $TARGET 33 | sed -i "s/JOBNAME/${JOBNAME}/g" $TARGET 34 | sed -i "s/NODENUM/${i}/g" $TARGET 35 | sed -i "s/MPHYSICALCORE/${MAX_PYSICAL_CORE}/g" $TARGET 36 | sed -i "s/MHYPERTHREADING/${MAX_HYPERTHREADING}/g" $TARGET 37 | sed -i "s/N_SERVER_PROC/${NUM_SERVER_PROC_PER_NODE}/g" $TARGET 38 | sed -i "s/N_CLIENT_PROC/${NUM_CLIENT_PROC_PER_NODE}/g" $TARGET 39 | sed -i "s/NTHREAD_PER_SPROC/${NUM_THREAD_PER_SERVER_PROC}/g" $TARGET 40 | sed -i "s/NTHREAD_PER_CPROC/${NUM_THREAD_PER_CLIENT_PROC}/g" $TARGET 41 | sed -i "s/PROJNAME/${PROJECT_NAME}/g" $TARGET 42 | sed -i "s/USING_DART/${j}/g" $TARGET 43 | if [[ "$i" -gt "4" ]]; then 44 | sed -i "s/REG//g" $TARGET 45 | else 46 | sed -i "s/DBG//g" $TARGET 47 | fi 48 | done 49 | done 50 | -------------------------------------------------------------------------------- /scripts/kvtag_query_scale/submit.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # MIN_PROC=4 4 | # MAX_PROC=128 5 | MIN_PROC=1 6 | MAX_PROC=512 7 | MAX_ATTR=1024 8 | MAX_ATTRLEN=1000 9 | 10 | PROG_BASENAME=llsm_importer 11 | 12 | curdir=$(pwd) 13 | 14 | first_submit=1 15 | 16 | for (( i = 1; i <= $MAX_PROC; i*=2 )); do 17 | mkdir -p $i 18 | JOBNAME=${PROG_BASENAME}_${i} 19 | TARGET=./$i/JOBNAME.sh 20 | 21 | njob=`squeue -u $USER | grep ${PROG_BASENAME} | wc -l` 22 | echo $njob 23 | while [ $njob -ge 4 ] 24 | do 25 | sleeptime=$[ ( $RANDOM % 1000 ) ] 26 | sleep $sleeptime 27 | njob=`squeue -u $USER | grep ${PROG_BASENAME} | wc -l` 28 | echo $njob 29 | done 30 | 31 | if [[ $first_submit == 1 ]]; then 32 | # Submit first job w/o dependency 33 | echo "Submitting $TARGET" 34 | job=`sbatch $TARGET` 35 | first_submit=0 36 | else 37 | echo "Submitting $TARGET after ${job: -8}" 38 | job=`sbatch -d afterany:${job: -8} $TARGET` 39 | fi 40 | 41 | sleeptime=$[ ( $RANDOM % 5 ) ] 42 | sleep $sleeptime 43 | done 44 | 45 | 46 | # for (( j = $MIN_PROC; j <= $MAX_PROC ; j*=2 )); do 47 | 48 | # njob=`squeue -u $USER | grep vpic | wc -l` 49 | # echo $njob 50 | # while [ $njob -ge 4 ] 51 | # do 52 | # sleeptime=$[ ( $RANDOM % 1000 ) ] 53 | # sleep $sleeptime 54 | # njob=`squeue -u $USER | grep vpic | wc -l` 55 | # echo $njob 56 | # done 57 | 58 | 59 | # cd $curdir/$j 60 | # for filename in ./*.sh ; do 61 | 62 | # if [[ $first_submit == 1 ]]; then 63 | # # Submit first job w/o dependency 64 | # echo "Submitting $filename" 65 | # job=`sbatch $filename` 66 | # first_submit=0 67 | # else 68 | # echo "Submitting $filename after ${job: -8}" 69 | # job=`sbatch -d afterany:${job: -8} $filename` 70 | # fi 71 | 72 | # sleeptime=$[ ( $RANDOM % 5 ) ] 73 | # sleep $sleeptime 74 | 75 | # done 76 | # done 77 | -------------------------------------------------------------------------------- /scripts/kvtag_range_query_scale/clean.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [[ "$#" -ne 1 ]]; then 4 | echo "Usage: $0 " 5 | exit 1 6 | fi 7 | 8 | # test if $1 is a single digit number between 0 and 1 (inclusive) 9 | if [[ "$1" =~ ^[0-1]$ ]]; then 10 | echo "Error: clean_dir should be a single digit number between 0 and 1 (inclusive). 1 means clean the directories named with numbers, 0 means clean the sbatch script only." 11 | exit 1 12 | fi 13 | 14 | CLEAN_DIR=$1 15 | MAX_NODE=512 16 | 17 | # if CLEAN_DIR is set to '1', then clean all the directories named with numbers, otherwise, clean the sbatch script only 18 | find ./ -name "*.sbatch*" -delete 19 | 20 | if [[ "$CLEAN_DIR" -eq "1" ]]; then 21 | for (( i = 1; i <= $MAX_NODE; i*=2 )); do 22 | 23 | rm -rf $i/* 24 | 25 | done 26 | fi -------------------------------------------------------------------------------- /scripts/kvtag_range_query_scale/submit.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | MIN_PROC=16 4 | MAX_PROC=128 5 | 6 | PROG_BASENAME=kvrngqry 7 | 8 | curdir=$(pwd) 9 | 10 | first_submit=1 11 | 12 | if [[ "$#" -ne 3 ]]; then 13 | echo "Usage: $0 " 14 | exit 1 15 | fi 16 | 17 | # test if $1 is a single digit number between 0 and 1 (inclusive) 18 | re='^[0-1]$' 19 | if ! [[ "$1" =~ $re ]]; then 20 | echo "Error: i_type should be a single digit number between 0 and 1 (inclusive), 0 means not using index, 1 means using index" 21 | exit 1 22 | fi 23 | 24 | # test if $2 is a single digit number between 0 and 3 (inclusive) 25 | re='^[0-3]$' 26 | if ! [[ "$2" =~ $re ]]; then 27 | echo "Error: q_type should be a single digit number between 0 and 3 (inclusive), 0: exact query, 1: prefix query, 2: suffix query, 3: infix query" 28 | exit 1 29 | fi 30 | 31 | # test if $3 is a single digit number between 0 and 1 (inclusive) 32 | re='^[0-1]$' 33 | if ! [[ "$3" =~ $re ]]; then 34 | echo "Error: c_type should be a single digit number between 0 and 1 (inclusive), 0 means using non-collective mode, 1 means using collective mode" 35 | exit 1 36 | fi 37 | 38 | 39 | i_type=$1 40 | q_type=$2 41 | c_type=$3 42 | 43 | for (( i = $MIN_PROC; i <= $MAX_PROC; i*=2 )); do 44 | 45 | cd $curdir/$i 46 | 47 | JOBNAME=${PROG_BASENAME}_${i}_${i_type}_${q_type}_${c_type} 48 | TARGET=$JOBNAME.sbatch 49 | 50 | njob=`squeue -u $USER | grep ${PROG_BASENAME} | wc -l` 51 | echo $njob 52 | while [ $njob -ge 16 ] 53 | do 54 | sleeptime=$[ ( $RANDOM % 5 ) ] 55 | sleep $sleeptime 56 | njob=`squeue -u $USER | grep ${PROG_BASENAME} | wc -l` 57 | echo $njob 58 | done 59 | 60 | if [[ $first_submit == 1 ]]; then 61 | # Submit first job w/o dependency 62 | echo "Submitting $TARGET" 63 | job=`sbatch $TARGET` 64 | first_submit=0 65 | else 66 | echo "Submitting $TARGET after ${job: -8}" 67 | job=`sbatch -d afterany:${job: -8} $TARGET` 68 | fi 69 | 70 | sleeptime=$[ ( $RANDOM % 5 ) ] 71 | sleep $sleeptime 72 | 73 | done 74 | -------------------------------------------------------------------------------- /scripts/llsm_idioms_bench/clean.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [[ "$#" -ne 1 ]]; then 4 | echo "Usage: $0 " 5 | exit 1 6 | fi 7 | 8 | # test if $1 is a single digit number between 0 and 1 (inclusive) 9 | if [[ "$1" =~ ^[0-1]$ ]]; then 10 | echo "Error: clean_dir should be a single digit number between 0 and 1 (inclusive). 1 means clean the directories named with numbers, 0 means clean the sbatch script only." 11 | exit 1 12 | fi 13 | 14 | CLEAN_DIR=$1 15 | MAX_NODE=512 16 | 17 | # if CLEAN_DIR is set to '1', then clean all the directories named with numbers, otherwise, clean the sbatch script only 18 | find ./ -name "*.sbatch*" -delete 19 | 20 | if [[ "$CLEAN_DIR" -eq "1" ]]; then 21 | for (( i = 1; i <= $MAX_NODE; i*=2 )); do 22 | 23 | rm -rf $i/* 24 | 25 | done 26 | fi -------------------------------------------------------------------------------- /scripts/llsm_idioms_bench/submit.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | MIN_PROC=16 4 | MAX_PROC=128 5 | 6 | PROG_BASENAME=idioms 7 | 8 | curdir=$(pwd) 9 | 10 | first_submit=1 11 | 12 | if [[ "$#" -ne 3 ]]; then 13 | echo "Usage: $0 " 14 | exit 1 15 | fi 16 | 17 | # test if $1 is a single digit number between 0 and 1 (inclusive) 18 | re='^[0-1]$' 19 | if ! [[ "$1" =~ $re ]]; then 20 | echo "Error: i_type should be a single digit number between 0 and 1 (inclusive), 0 means not using index, 1 means using index" 21 | exit 1 22 | fi 23 | 24 | # test if $2 is a single digit number between 0 and 3 (inclusive) 25 | re='^[0-3]$' 26 | if ! [[ "$2" =~ $re ]]; then 27 | echo "Error: q_type should be a single digit number between 0 and 3 (inclusive), 0: exact query, 1: prefix query, 2: suffix query, 3: infix query" 28 | exit 1 29 | fi 30 | 31 | # test if $3 is a single digit number between 0 and 1 (inclusive) 32 | re='^[0-1]$' 33 | if ! [[ "$3" =~ $re ]]; then 34 | echo "Error: c_type should be a single digit number between 0 and 1 (inclusive), 0 means using non-collective mode, 1 means using collective mode" 35 | exit 1 36 | fi 37 | 38 | 39 | i_type=$1 40 | q_type=$2 41 | c_type=$3 42 | 43 | for (( i = $MIN_PROC; i <= $MAX_PROC; i*=2 )); do 44 | 45 | cd $curdir/$i 46 | 47 | JOBNAME=${PROG_BASENAME}_${i}_${i_type}_${q_type}_${c_type} 48 | TARGET=$JOBNAME.sbatch 49 | 50 | njob=`squeue -u $USER | grep ${PROG_BASENAME} | wc -l` 51 | echo $njob 52 | while [ $njob -ge 16 ] 53 | do 54 | sleeptime=$[ ( $RANDOM % 5 ) ] 55 | sleep $sleeptime 56 | njob=`squeue -u $USER | grep ${PROG_BASENAME} | wc -l` 57 | echo $njob 58 | done 59 | 60 | if [[ $first_submit == 1 ]]; then 61 | # Submit first job w/o dependency 62 | echo "Submitting $TARGET" 63 | job=`sbatch $TARGET` 64 | first_submit=0 65 | else 66 | echo "Submitting $TARGET after ${job: -8}" 67 | job=`sbatch -d afterany:${job: -8} $TARGET` 68 | fi 69 | 70 | sleeptime=$[ ( $RANDOM % 5 ) ] 71 | sleep $sleeptime 72 | 73 | done 74 | -------------------------------------------------------------------------------- /scripts/llsm_importer/clean.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | MAX_NODE=512 3 | 4 | for (( i = 1; i <= $MAX_NODE; i*=2 )); do 5 | 6 | rm -rf $i/* 7 | 8 | done -------------------------------------------------------------------------------- /scripts/llsm_importer/gen_script.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | PROJECT_NAME=$1 4 | 5 | # Per node configuration of your HPC system. 6 | MAX_PYSICAL_CORE=128 7 | MAX_HYPERTHREADING=2 8 | 9 | # Designated number of threads per process on each node 10 | # (this should be associated with -c option in srun) 11 | NUM_THREAD_PER_SERVER_PROC=64 12 | NUM_THREAD_PER_CLIENT_PROC=64 13 | 14 | 15 | # Designated number of processes for server anc client on each node 16 | # (this should be associated with -n option in srun) 17 | NUM_SERVER_PROC_PER_NODE=1 18 | NUM_CLIENT_PROC_PER_NODE=1 19 | 20 | 21 | MAX_NODE=512 22 | MAX_ATTR=1024 23 | MAX_ATTRLEN=1000 24 | 25 | PROG_BASENAME=llsm_importer 26 | 27 | for (( i = 1; i <= $MAX_NODE; i*=2 )); do 28 | mkdir -p $i 29 | JOBNAME=${PROG_BASENAME}_${i} 30 | TARGET=./$i/$JOBNAME.sbatch 31 | cp template.sh $TARGET 32 | sed -i "s/JOBNAME/${JOBNAME}/g" $TARGET 33 | sed -i "s/NODENUM/${i}/g" $TARGET 34 | sed -i "s/MPHYSICALCORE/${MAX_PYSICAL_CORE}/g" $TARGET 35 | sed -i "s/MHYPERTHREADING/${MAX_HYPERTHREADING}/g" $TARGET 36 | sed -i "s/N_SERVER_PROC/${NUM_SERVER_PROC_PER_NODE}/g" $TARGET 37 | sed -i "s/N_CLIENT_PROC/${NUM_CLIENT_PROC_PER_NODE}/g" $TARGET 38 | sed -i "s/NTHREAD_PER_SPROC/${NUM_THREAD_PER_SERVER_PROC}/g" $TARGET 39 | sed -i "s/NTHREAD_PER_CPROC/${NUM_THREAD_PER_CLIENT_PROC}/g" $TARGET 40 | sed -i "s/PROJNAME/${PROJECT_NAME}/g" $TARGET 41 | if [[ "$i" -gt "4" ]]; then 42 | sed -i "s/REG//g" $TARGET 43 | else 44 | sed -i "s/DBG//g" $TARGET 45 | fi 46 | done 47 | -------------------------------------------------------------------------------- /scripts/llsm_importer/submit.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # MIN_PROC=4 4 | # MAX_PROC=128 5 | MIN_PROC=1 6 | MAX_PROC=512 7 | MAX_ATTR=1024 8 | MAX_ATTRLEN=1000 9 | 10 | PROG_BASENAME=llsm_importer 11 | 12 | curdir=$(pwd) 13 | 14 | first_submit=1 15 | 16 | for (( i = 1; i <= $MAX_PROC; i*=2 )); do 17 | mkdir -p $i 18 | JOBNAME=${PROG_BASENAME}_${i} 19 | TARGET=./$i/JOBNAME.sh 20 | 21 | njob=`squeue -u $USER | grep ${PROG_BASENAME} | wc -l` 22 | echo $njob 23 | while [ $njob -ge 4 ] 24 | do 25 | sleeptime=$[ ( $RANDOM % 1000 ) ] 26 | sleep $sleeptime 27 | njob=`squeue -u $USER | grep ${PROG_BASENAME} | wc -l` 28 | echo $njob 29 | done 30 | 31 | if [[ $first_submit == 1 ]]; then 32 | # Submit first job w/o dependency 33 | echo "Submitting $TARGET" 34 | job=`sbatch $TARGET` 35 | first_submit=0 36 | else 37 | echo "Submitting $TARGET after ${job: -8}" 38 | job=`sbatch -d afterany:${job: -8} $TARGET` 39 | fi 40 | 41 | sleeptime=$[ ( $RANDOM % 5 ) ] 42 | sleep $sleeptime 43 | done 44 | 45 | 46 | # for (( j = $MIN_PROC; j <= $MAX_PROC ; j*=2 )); do 47 | 48 | # njob=`squeue -u $USER | grep vpic | wc -l` 49 | # echo $njob 50 | # while [ $njob -ge 4 ] 51 | # do 52 | # sleeptime=$[ ( $RANDOM % 1000 ) ] 53 | # sleep $sleeptime 54 | # njob=`squeue -u $USER | grep vpic | wc -l` 55 | # echo $njob 56 | # done 57 | 58 | 59 | # cd $curdir/$j 60 | # for filename in ./*.sh ; do 61 | 62 | # if [[ $first_submit == 1 ]]; then 63 | # # Submit first job w/o dependency 64 | # echo "Submitting $filename" 65 | # job=`sbatch $filename` 66 | # first_submit=0 67 | # else 68 | # echo "Submitting $filename after ${job: -8}" 69 | # job=`sbatch -d afterany:${job: -8} $filename` 70 | # fi 71 | 72 | # sleeptime=$[ ( $RANDOM % 5 ) ] 73 | # sleep $sleeptime 74 | 75 | # done 76 | # done 77 | -------------------------------------------------------------------------------- /src/api/close_server.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Notice for 3 | * Proactive Data Containers (PDC) Software Library and Utilities 4 | * ----------------------------------------------------------------------------- 5 | 6 | *** Copyright Notice *** 7 | 8 | * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the 9 | * University of California, through Lawrence Berkeley National Laboratory, 10 | * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF 11 | * Group (subject to receipt of any required approvals from the U.S. Dept. of 12 | * Energy). All rights reserved. 13 | 14 | * If you have questions about your rights to use or distribute this software, 15 | * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. 16 | 17 | * NOTICE. This Software was developed under funding from the U.S. Department of 18 | * Energy and the U.S. Government consequently retains certain rights. As such, the 19 | * U.S. Government has been granted for itself and others acting on its behalf a 20 | * paid-up, nonexclusive, irrevocable, worldwide license in the Software to 21 | * reproduce, distribute copies to the public, prepare derivative works, and 22 | * perform publicly and display publicly, and to permit other to do so. 23 | */ 24 | 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | 31 | #include "pdc.h" 32 | #include "pdc_logger.h" 33 | #include "pdc_client_connect.h" 34 | 35 | int 36 | main(int argc, char *argv[]) 37 | { 38 | pdcid_t pdc; 39 | #ifdef ENABLE_MPI 40 | int rank; 41 | double start; 42 | MPI_Init(&argc, &argv); 43 | 44 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 45 | start = MPI_Wtime(); 46 | #endif 47 | pdc = PDCinit("pdc"); 48 | 49 | PDC_Client_close_all_server(); 50 | 51 | if (PDCclose(pdc) < 0) 52 | LOG_ERROR("Failed to close PDC\n"); 53 | 54 | #ifdef ENABLE_MPI 55 | if (!rank) { 56 | LOG_INFO("total close time = %lf\n", MPI_Wtime() - start); 57 | } 58 | MPI_Finalize(); 59 | #endif 60 | return 0; 61 | } 62 | -------------------------------------------------------------------------------- /src/api/include/pdc.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Notice for 3 | * Proactive Data Containers (PDC) Software Library and Utilities 4 | * ----------------------------------------------------------------------------- 5 | 6 | *** Copyright Notice *** 7 | 8 | * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the 9 | * University of California, through Lawrence Berkeley National Laboratory, 10 | * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF 11 | * Group (subject to receipt of any required approvals from the U.S. Dept. of 12 | * Energy). All rights reserved. 13 | 14 | * If you have questions about your rights to use or distribute this software, 15 | * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. 16 | 17 | * NOTICE. This Software was developed under funding from the U.S. Department of 18 | * Energy and the U.S. Government consequently retains certain rights. As such, the 19 | * U.S. Government has been granted for itself and others acting on its behalf a 20 | * paid-up, nonexclusive, irrevocable, worldwide license in the Software to 21 | * reproduce, distribute copies to the public, prepare derivative works, and 22 | * perform publicly and display publicly, and to permit other to do so. 23 | */ 24 | 25 | #ifndef PDC_H 26 | #define PDC_H 27 | 28 | #include "pdc_config.h" 29 | #include "pdc_public.h" 30 | #include "pdc_prop.h" 31 | #include "pdc_cont.h" 32 | #include "pdc_logger.h" 33 | 34 | #ifdef ENABLE_MPI 35 | #include "pdc_mpi.h" 36 | #endif 37 | 38 | #include "pdc_obj.h" 39 | #include "pdc_region.h" 40 | #include "pdc_query.h" 41 | #include "pdc_analysis.h" 42 | #include "pdc_transform.h" 43 | 44 | int PDC_timing_report(const char *prefix); 45 | 46 | /*********************/ 47 | /* Public Prototypes */ 48 | /*********************/ 49 | /** 50 | * Initialize the PDC layer 51 | * 52 | * \param pdc_name [IN] Name of the PDC 53 | * 54 | * \return PDC id on success / 0 on failure 55 | */ 56 | pdcid_t PDCinit(const char *pdc_name); 57 | 58 | /** 59 | * Close the PDC layer 60 | * 61 | * \param pdc_id [IN] ID of the PDC 62 | * 63 | * \return Non-negative on success/Negative on failure 64 | */ 65 | perr_t PDCclose(pdcid_t pdcid); 66 | 67 | #endif 68 | -------------------------------------------------------------------------------- /src/api/pdc_analysis/include/pdc_hist_pkg.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Notice for 3 | * Proactive Data Containers (PDC) Software Library and Utilities 4 | * ----------------------------------------------------------------------------- 5 | 6 | *** Copyright Notice *** 7 | 8 | * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the 9 | * University of California, through Lawrence Berkeley National Laboratory, 10 | * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF 11 | * Group (subject to receipt of any required approvals from the U.S. Dept. of 12 | * Energy). All rights reserved. 13 | 14 | * If you have questions about your rights to use or distribute this software, 15 | * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. 16 | 17 | * NOTICE. This Software was developed under funding from the U.S. Department of 18 | * Energy and the U.S. Government consequently retains certain rights. As such, the 19 | * U.S. Government has been granted for itself and others acting on its behalf a 20 | * paid-up, nonexclusive, irrevocable, worldwide license in the Software to 21 | * reproduce, distribute copies to the public, prepare derivative works, and 22 | * perform publicly and display publicly, and to permit other to do so. 23 | */ 24 | 25 | #ifndef PDC_HIST_H 26 | #define PDC_HIST_H 27 | 28 | #include "pdc_private.h" 29 | #include "pdc_public.h" 30 | #include "math.h" 31 | #include 32 | #include 33 | 34 | /** 35 | * ******** 36 | * 37 | * \param obj_name [IN] Data type 38 | * \param n [IN] ******* 39 | * \param data [IN] ******* 40 | * 41 | * \return ******** 42 | */ 43 | pdc_histogram_t *PDC_gen_hist(pdc_var_type_t dtype, uint64_t n, void *data); 44 | 45 | /** 46 | * ******** 47 | * 48 | * \param hist [IN] ******** 49 | * 50 | * \return ******** 51 | */ 52 | pdc_histogram_t *PDC_dup_hist(pdc_histogram_t *hist); 53 | 54 | /** 55 | * ******** 56 | * 57 | * \param n [IN] ******** 58 | * \param hist [IN] ******** 59 | * 60 | * \return ******** 61 | */ 62 | pdc_histogram_t *PDC_merge_hist(int n, pdc_histogram_t **hists); 63 | 64 | /** 65 | * ******** 66 | * 67 | * \param hist [IN] ******** 68 | */ 69 | void PDC_free_hist(pdc_histogram_t *hist); 70 | 71 | /** 72 | * ******** 73 | * 74 | * \param hist [IN] ******** 75 | */ 76 | void PDC_print_hist(pdc_histogram_t *hist); 77 | 78 | /** 79 | * ******** 80 | * 81 | * \param to [IN] ******** 82 | * \param hist [IN] ******** 83 | */ 84 | void PDC_copy_hist(pdc_histogram_t *to, pdc_histogram_t *from); 85 | 86 | #endif /* PDC_HIST_H */ 87 | -------------------------------------------------------------------------------- /src/api/pdc_meta/include/pdc_metadata_client.h: -------------------------------------------------------------------------------- 1 | #ifndef PDC_METADATA_CLIENT_H 2 | #define PDC_METADATA_CLIENT_H 3 | 4 | #include "pdc_client_connect.h" 5 | #include "pdc_client_server_common.h" 6 | 7 | typedef struct { 8 | int n_res; 9 | uint64_t *out; 10 | char * condition; 11 | } separate_query_result_t; 12 | 13 | size_t PDC_metadata_multi_condition_query(char *queryString, uint64_t **object_id_list, uint64_t *count); 14 | 15 | #endif // PDC_METADATA_CLIENT_H -------------------------------------------------------------------------------- /src/api/pdc_obj/include/pdc_cont_pkg.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Notice for 3 | * Proactive Data Containers (PDC) Software Library and Utilities 4 | * ----------------------------------------------------------------------------- 5 | 6 | *** Copyright Notice *** 7 | 8 | * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the 9 | * University of California, through Lawrence Berkeley National Laboratory, 10 | * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF 11 | * Group (subject to receipt of any required approvals from the U.S. Dept. of 12 | * Energy). All rights reserved. 13 | 14 | * If you have questions about your rights to use or distribute this software, 15 | * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. 16 | 17 | * NOTICE. This Software was developed under funding from the U.S. Department of 18 | * Energy and the U.S. Government consequently retains certain rights. As such, the 19 | * U.S. Government has been granted for itself and others acting on its behalf a 20 | * paid-up, nonexclusive, irrevocable, worldwide license in the Software to 21 | * reproduce, distribute copies to the public, prepare derivative works, and 22 | * perform publicly and display publicly, and to permit other to do so. 23 | */ 24 | 25 | #ifndef PDC_CONT_PKG_H 26 | #define PDC_CONT_PKG_H 27 | 28 | #include "pdc_public.h" 29 | 30 | /**************************/ 31 | /* Library Private Struct */ 32 | /**************************/ 33 | struct _pdc_cont_info { 34 | struct pdc_cont_info * cont_info_pub; 35 | struct _pdc_cont_prop *cont_pt; 36 | }; 37 | 38 | /***************************************/ 39 | /* Library-private Function Prototypes */ 40 | /***************************************/ 41 | /** 42 | * container initialization 43 | * 44 | * \return Non-negative on success/Negative on failure 45 | */ 46 | perr_t PDC_cont_init(); 47 | 48 | /** 49 | * Create a container locally 50 | * 51 | * \param pdc [IN] PDC ID 52 | * \param cont_name [IN] Name of the container 53 | * \param cont_meta_id [out] Metadata id of container 54 | * 55 | * \return Container id on success/Zero on failure 56 | */ 57 | pdcid_t PDC_cont_create_local(pdcid_t pdc, const char *cont_name, uint64_t cont_meta_id); 58 | 59 | /** 60 | * Check if container list is empty 61 | * 62 | * \param pdc_id [IN] ID of the PDC 63 | * 64 | * \return SUCCEED if empty/FAIL if not empty 65 | */ 66 | perr_t PDC_cont_list_null(); 67 | 68 | /** 69 | * Return a container property 70 | * 71 | * \param cont_name [IN] Name of the container 72 | * 73 | * \return Container struct on success/NULL on failure 74 | */ 75 | struct _pdc_cont_info *PDC_cont_get_info(pdcid_t cont_id); 76 | 77 | /** 78 | * PDC container finalize 79 | * 80 | * \return Non-negative on success/Negative on failure 81 | */ 82 | perr_t PDC_cont_end(); 83 | 84 | #endif /* PDC_CONT_PKG_H */ 85 | -------------------------------------------------------------------------------- /src/api/pdc_obj/include/pdc_mpi.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Notice for 3 | * Proactive Data Containers (PDC) Software Library and Utilities 4 | * ----------------------------------------------------------------------------- 5 | 6 | *** Copyright Notice *** 7 | 8 | * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the 9 | * University of California, through Lawrence Berkeley National Laboratory, 10 | * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF 11 | * Group (subject to receipt of any required approvals from the U.S. Dept. of 12 | * Energy). All rights reserved. 13 | 14 | * If you have questions about your rights to use or distribute this software, 15 | * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. 16 | 17 | * NOTICE. This Software was developed under funding from the U.S. Department of 18 | * Energy and the U.S. Government consequently retains certain rights. As such, the 19 | * U.S. Government has been granted for itself and others acting on its behalf a 20 | * paid-up, nonexclusive, irrevocable, worldwide license in the Software to 21 | * reproduce, distribute copies to the public, prepare derivative works, and 22 | * perform publicly and display publicly, and to permit other to do so. 23 | */ 24 | 25 | #ifndef PDC_MPI_H 26 | #define PDC_MPI_H 27 | 28 | #include "mpi.h" 29 | 30 | /*********************/ 31 | /* Public Prototypes */ 32 | /*********************/ 33 | /** 34 | * Create an object 35 | * 36 | * \param cont_id [IN] ID of the container 37 | * \param obj_name [IN] Name of the object 38 | * \param obj_create_prop [IN] ID of object property, 39 | * returned by PDCprop_create(PDC_OBJ_CREATE) 40 | * \param rank_id [IN] MPI process rank 41 | * 42 | * \return Object ID on success/Negative on failure 43 | */ 44 | pdcid_t PDCobj_create_mpi(pdcid_t cont_id, const char *obj_name, pdcid_t obj_create_prop, int rank_id, 45 | MPI_Comm comm); 46 | 47 | #endif /* PDC_MPI_H */ 48 | -------------------------------------------------------------------------------- /src/api/pdc_region/include/pdc_region_pkg.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Notice for 3 | * Proactive Data Containers (PDC) Software Library and Utilities 4 | * ----------------------------------------------------------------------------- 5 | 6 | *** Copyright Notice *** 7 | 8 | * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the 9 | * University of California, through Lawrence Berkeley National Laboratory, 10 | * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF 11 | * Group (subject to receipt of any required approvals from the U.S. Dept. of 12 | * Energy). All rights reserved. 13 | 14 | * If you have questions about your rights to use or distribute this software, 15 | * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. 16 | 17 | * NOTICE. This Software was developed under funding from the U.S. Department of 18 | * Energy and the U.S. Government consequently retains certain rights. As such, the 19 | * U.S. Government has been granted for itself and others acting on its behalf a 20 | * paid-up, nonexclusive, irrevocable, worldwide license in the Software to 21 | * reproduce, distribute copies to the public, prepare derivative works, and 22 | * perform publicly and display publicly, and to permit other to do so. 23 | */ 24 | 25 | #ifndef PDC_REGION_PKG_H 26 | #define PDC_REGION_PKG_H 27 | 28 | #include "pdc_private.h" 29 | 30 | /**************************/ 31 | /* Library Private Struct */ 32 | /**************************/ 33 | struct region_map_list { 34 | pdcid_t orig_reg_id; 35 | pdcid_t des_obj_id; 36 | pdcid_t des_reg_id; 37 | struct region_map_list *prev; 38 | struct region_map_list *next; 39 | }; 40 | 41 | /***************************************/ 42 | /* Library-private Function Prototypes */ 43 | /***************************************/ 44 | /** 45 | * PDC region initialization 46 | * 47 | * \return Non-negative on success/Negative on failure 48 | */ 49 | perr_t PDC_region_init(); 50 | /** 51 | * PDC region transfer instance initialization 52 | * 53 | * \return Non-negative on success/Negative on failure 54 | */ 55 | perr_t PDC_transfer_request_init(); 56 | 57 | /** 58 | * PDC region finalize 59 | * 60 | * \return Non-negative on success/Negative on failure 61 | */ 62 | perr_t PDC_region_end(); 63 | 64 | /** 65 | * Check if region list is empty 66 | * 67 | * \return SUCCEED if empty/FAIL if not empty 68 | */ 69 | perr_t PDC_region_list_null(); 70 | 71 | #endif /* PDC_REGION_PKG_H */ 72 | -------------------------------------------------------------------------------- /src/api/pdc_transform/include/pdc_transforms_pkg.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Notice for 3 | * Proactive Data Containers (PDC) Software Library and Utilities 4 | * ----------------------------------------------------------------------------- 5 | 6 | *** Copyright Notice *** 7 | 8 | * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the 9 | * University of California, through Lawrence Berkeley National Laboratory, 10 | * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF 11 | * Group (subject to receipt of any required approvals from the U.S. Dept. of 12 | * Energy). All rights reserved. 13 | 14 | * If you have questions about your rights to use or distribute this software, 15 | * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. 16 | 17 | * NOTICE. This Software was developed under funding from the U.S. Department of 18 | * Energy and the U.S. Government consequently retains certain rights. As such, the 19 | * U.S. Government has been granted for itself and others acting on its behalf a 20 | * paid-up, nonexclusive, irrevocable, worldwide license in the Software to 21 | * reproduce, distribute copies to the public, prepare derivative works, and 22 | * perform publicly and display publicly, and to permit other to do so. 23 | */ 24 | #ifndef PDC_TRANSFORMS_H 25 | #define PDC_TRANSFORMS_H 26 | 27 | #include "pdc_private.h" 28 | #include "pdc_transform.h" 29 | 30 | #define DATA_ANY 7 31 | 32 | /***************************/ 33 | /* Library Private Structs */ 34 | /***************************/ 35 | typedef enum { C_lang = 0, FORTRAN_lang, PYTHON_lang, JULIA_lang, N_LANGUAGES } _pdc_analysis_language_t; 36 | 37 | struct _pdc_region_transform_ftn_info { 38 | pdcid_t object_id; 39 | pdcid_t region_id; 40 | int local_regIndex; 41 | int meta_index; 42 | struct pdc_region_info *src_region; 43 | struct pdc_region_info *dest_region; 44 | size_t (*ftnPtr)(); 45 | int ftn_lastResult; 46 | int readyState; 47 | int nextState; 48 | int client_id; 49 | size_t type_extent; 50 | size_t dest_extent; 51 | pdc_var_type_t type; 52 | pdc_var_type_t dest_type; 53 | pdc_obj_transform_t op_type; 54 | pdc_data_movement_t when; 55 | _pdc_analysis_language_t lang; 56 | void * data; 57 | void * result; 58 | }; 59 | 60 | /***************************************/ 61 | /* Library-private Function Prototypes */ 62 | /***************************************/ 63 | /** 64 | * To end PDC transform 65 | * 66 | * \return Non-negative on success/Negative on failure 67 | */ 68 | perr_t PDC_transform_end(); 69 | 70 | #endif /* PDC_TRANSFORMS_H */ 71 | -------------------------------------------------------------------------------- /src/commons/Readme.md: -------------------------------------------------------------------------------- 1 | Commons directory should contain a collection of code that will be shared by both Server and Client-side APIs 2 | 3 | -------------------------------------------------------------------------------- /src/commons/collections/Readme.md: -------------------------------------------------------------------------------- 1 | Collections directory should contain a bunch of generic and basic data structures for PDC to use. 2 | 3 | At least, this directory should contain: 4 | 5 | 1. Vector/List Implementation 6 | 2. Hash Table Implementation 7 | 3. Hash Set Implementation 8 | 4. Art Implementation 9 | 5. BST Implementation -------------------------------------------------------------------------------- /src/commons/collections/include/pdc_deque.h: -------------------------------------------------------------------------------- 1 | #ifndef PDC_DEQUE_H 2 | #define PDC_DEQUE_H 3 | 4 | #include 5 | 6 | // Struct for the deque data structure 7 | typedef struct PDC_deque_t { 8 | void **data; // Pointer to the data array 9 | size_t size; // Current size of the deque 10 | size_t capacity; // Current capacity of the deque 11 | size_t head; // Index of the head of the deque 12 | size_t tail; // Index of the tail of the deque 13 | } PDC_deque_t; 14 | 15 | // Initialize the deque 16 | void PDC_deque_init(PDC_deque_t *deque); 17 | 18 | // Push an element onto the front of the deque 19 | void PDC_deque_push_front(PDC_deque_t *deque, void *value); 20 | 21 | // Push an element onto the back of the deque 22 | void PDC_deque_push_back(PDC_deque_t *deque, void *value); 23 | 24 | // Pop an element from the front of the deque 25 | void *PDC_deque_pop_front(PDC_deque_t *deque); 26 | 27 | // Pop an element from the back of the deque 28 | void *PDC_deque_pop_back(PDC_deque_t *deque); 29 | 30 | // Free memory allocated for the deque 31 | void PDC_deque_free(PDC_deque_t *deque); 32 | 33 | #endif /* PDC_DEQUE_H */ -------------------------------------------------------------------------------- /src/commons/collections/include/pdc_hash.h: -------------------------------------------------------------------------------- 1 | /** 2 | * @file pdc_hash.h 3 | * 4 | * Hash function for integer, pointer, and string. See @ref xxx_hash. 5 | */ 6 | 7 | #ifndef PDC_HASH_H 8 | #define PDC_HASH_H 9 | 10 | #ifdef __cplusplus 11 | extern "C" { 12 | #endif 13 | 14 | /** 15 | * Generate a hash key for a pointer to an integer. The value pointed 16 | * at is used to generate the key. 17 | * 18 | * @param location The pointer. 19 | * @return A hash key for the value at the location. 20 | */ 21 | 22 | unsigned int int_hash(void *location); 23 | 24 | /** 25 | * Generate a hash key for a pointer to an uint64_t integer. The value pointed 26 | * at is used to generate the key. 27 | * 28 | * @param location The pointer. 29 | * @return A hash key for the value at the location. 30 | */ 31 | unsigned int ui64_hash(void *location); 32 | 33 | /** 34 | * Generate a hash key for a pointer. The value pointed at by the pointer 35 | * is not used, only the pointer itself. 36 | * 37 | * @param location The pointer 38 | * @return A hash key for the pointer. 39 | */ 40 | 41 | unsigned int pointer_hash(void *location); 42 | 43 | /** 44 | * Generate a hash key from a string. 45 | * 46 | * @param string The string. 47 | * @return A hash key for the string. 48 | */ 49 | 50 | unsigned int string_hash(void *string); 51 | 52 | /** 53 | * Generate a hash key from a string, ignoring the case of letters. 54 | * 55 | * @param string The string. 56 | * @return A hash key for the string. 57 | */ 58 | 59 | unsigned int string_nocase_hash(void *string); 60 | 61 | #ifdef __cplusplus 62 | } 63 | #endif 64 | 65 | #endif /* #ifndef PDC_HASH_H */ -------------------------------------------------------------------------------- /src/commons/collections/include/pdc_stack.h: -------------------------------------------------------------------------------- 1 | #ifndef PDC_STACK_H 2 | #define PDC_STACK_H 3 | 4 | #include 5 | 6 | // Struct for the stack data structure 7 | typedef struct PDC_stack_t { 8 | void **data; // Pointer to the data array 9 | size_t size; // Current size of the stack 10 | size_t capacity; // Current capacity of the stack 11 | } PDC_stack_t; 12 | 13 | // Initialize the stack 14 | void PDC_stack_init(PDC_stack_t *stack); 15 | 16 | // Push an element onto the stack 17 | void PDC_stack_push(PDC_stack_t *stack, void *value); 18 | 19 | // Pop an element from the stack 20 | void *PDC_stack_pop(PDC_stack_t *stack); 21 | 22 | // Free memory allocated for the stack 23 | void PDC_stack_free(PDC_stack_t *stack); 24 | 25 | #endif /* PDC_STACK_H */ -------------------------------------------------------------------------------- /src/commons/collections/pdc_deque.c: -------------------------------------------------------------------------------- 1 | #include "pdc_deque.h" 2 | #include "pdc_logger.h" 3 | #include 4 | 5 | #define DEFAULT_CAPACITY 16 6 | 7 | void 8 | PDC_deque_init(PDC_deque_t *deque) 9 | { 10 | deque->data = malloc(sizeof(void *) * DEFAULT_CAPACITY); 11 | deque->size = 0; 12 | deque->capacity = DEFAULT_CAPACITY; 13 | deque->head = 0; 14 | deque->tail = 0; 15 | } 16 | 17 | static void 18 | resize_deque(PDC_deque_t *deque, size_t new_capacity) 19 | { 20 | void **new_data = malloc(sizeof(void *) * new_capacity); 21 | if (new_data == NULL) { 22 | LOG_ERROR("Failed to allocate memory for deque!\n"); 23 | exit(1); 24 | } 25 | 26 | size_t i = 0; 27 | while (deque->head != deque->tail) { 28 | new_data[i++] = deque->data[deque->head]; 29 | deque->head = (deque->head + 1) % deque->capacity; 30 | } 31 | 32 | free(deque->data); 33 | deque->data = new_data; 34 | deque->capacity = new_capacity; 35 | deque->head = 0; 36 | deque->tail = i; 37 | } 38 | 39 | void 40 | PDC_deque_push_front(PDC_deque_t *deque, void *value) 41 | { 42 | if (deque->size == deque->capacity) { 43 | resize_deque(deque, deque->capacity * 2); 44 | } 45 | deque->head = (deque->head - 1 + deque->capacity) % deque->capacity; 46 | deque->data[deque->head] = value; 47 | deque->size++; 48 | } 49 | 50 | void 51 | PDC_deque_push_back(PDC_deque_t *deque, void *value) 52 | { 53 | if (deque->size == deque->capacity) { 54 | resize_deque(deque, deque->capacity * 2); 55 | } 56 | deque->data[deque->tail] = value; 57 | deque->tail = (deque->tail + 1) % deque->capacity; 58 | deque->size++; 59 | } 60 | 61 | void * 62 | PDC_deque_pop_front(PDC_deque_t *deque) 63 | { 64 | if (deque->size == 0) { 65 | return NULL; 66 | } 67 | void *value = deque->data[deque->head]; 68 | deque->head = (deque->head + 1) % deque->capacity; 69 | deque->size--; 70 | if (deque->size <= deque->capacity / 4) { 71 | resize_deque(deque, deque->capacity / 2); 72 | } 73 | return value; 74 | } 75 | 76 | void * 77 | PDC_deque_pop_back(PDC_deque_t *deque) 78 | { 79 | if (deque->size == 0) { 80 | return NULL; 81 | } 82 | deque->tail = (deque->tail - 1 + deque->capacity) % deque->capacity; 83 | void *value = deque->data[deque->tail]; 84 | deque->size--; 85 | if (deque->size <= deque->capacity / 4) { 86 | resize_deque(deque, deque->capacity / 2); 87 | } 88 | return value; 89 | } 90 | 91 | void 92 | PDC_deque_free(PDC_deque_t *deque) 93 | { 94 | free(deque->data); 95 | } -------------------------------------------------------------------------------- /src/commons/collections/pdc_hash.c: -------------------------------------------------------------------------------- 1 | #include "pdc_hash.h" 2 | #include 3 | #include 4 | #include 5 | 6 | /* Hash function for a pointer to an integer */ 7 | 8 | unsigned int 9 | int_hash(void *vlocation) 10 | { 11 | int *location; 12 | 13 | location = (int *)vlocation; 14 | 15 | return (unsigned int)*location; 16 | } 17 | 18 | /* Hash function for a pointer to a uint64_t integer */ 19 | 20 | unsigned int 21 | ui64_hash(void *vlocation) 22 | { 23 | uint64_t *location; 24 | 25 | location = (uint64_t *)vlocation; 26 | 27 | return (unsigned int)*location; 28 | } 29 | 30 | /* Hash function for a generic pointer */ 31 | 32 | unsigned int 33 | pointer_hash(void *location) 34 | { 35 | return (unsigned int)(unsigned long)location; 36 | } 37 | 38 | /* String hash function */ 39 | 40 | unsigned int 41 | string_hash(void *string) 42 | { 43 | /* This is the djb2 string hash function */ 44 | 45 | unsigned int result = 5381; 46 | unsigned char *p; 47 | 48 | p = (unsigned char *)string; 49 | 50 | while (*p != '\0') { 51 | result = (result << 5) + result + *p; 52 | ++p; 53 | } 54 | 55 | return result; 56 | } 57 | 58 | /* The same function, with a tolower on every character so that 59 | * case is ignored. This code is duplicated for performance. */ 60 | 61 | unsigned int 62 | string_nocase_hash(void *string) 63 | { 64 | unsigned int result = 5381; 65 | unsigned char *p; 66 | 67 | p = (unsigned char *)string; 68 | 69 | while (*p != '\0') { 70 | result = (result << 5) + result + (unsigned int)tolower(*p); 71 | ++p; 72 | } 73 | 74 | return result; 75 | } -------------------------------------------------------------------------------- /src/commons/collections/pdc_set_test.c: -------------------------------------------------------------------------------- 1 | #include "pdc_set.h" 2 | #include "pdc_compare.h" 3 | #include "pdc_hash.h" 4 | #include "pdc_logger.h" 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | void 12 | set_value_free(SetValue value) 13 | { 14 | free((void *)value); 15 | } 16 | 17 | int 18 | main(int argc, char **argv) 19 | { 20 | // read the max id from the command line 21 | char * endptr; 22 | uint64_t max_id = strtoull(argv[1], &endptr, 10); 23 | if (*endptr != '\0') { 24 | LOG_ERROR("Invalid number: %s\n", argv[1]); 25 | return 1; 26 | } 27 | 28 | Set *set = set_new(ui64_hash, ui64_equal); 29 | set_register_free_function(set, set_value_free); 30 | 31 | for (uint64_t i = 0; i < max_id; i++) { 32 | uint64_t *value = malloc(sizeof(uint64_t)); 33 | *value = i; 34 | set_insert(set, value); 35 | } 36 | 37 | // test if the size of set is correct 38 | 39 | if (set_num_entries(set) != (unsigned int)max_id) { 40 | LOG_ERROR("Error: set size is not correct\n"); 41 | return 1; 42 | } 43 | 44 | // retrieve all values from the set 45 | for (uint64_t i = 0; i < max_id; i++) { 46 | uint64_t *value = malloc(sizeof(uint64_t)); 47 | *value = i; 48 | if (!set_query(set, value)) { 49 | LOG_ERROR("Error: value %" PRIu64 " not found in the set\n", i); 50 | return 1; 51 | } 52 | } 53 | 54 | // iterate through all values 55 | SetIterator *it; 56 | set_iterate(set, it); 57 | while (set_iter_has_more(it)) { 58 | uint64_t *value = set_iter_next(it); 59 | if (!set_query(set, value)) { 60 | LOG_ERROR("Error: value %" PRIu64 " not found in the set\n", *value); 61 | return 1; 62 | } 63 | } 64 | 65 | set_free(set); 66 | return 0; 67 | } -------------------------------------------------------------------------------- /src/commons/collections/pdc_stack.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "pdc_logger.h" 5 | #include "pdc_stack.h" 6 | 7 | #define DEFAULT_CAPACITY 16 8 | 9 | void 10 | stack_init(PDC_stack_t *stack) 11 | { 12 | stack->data = malloc(sizeof(void *) * DEFAULT_CAPACITY); 13 | stack->size = 0; 14 | stack->capacity = DEFAULT_CAPACITY; 15 | } 16 | 17 | void 18 | stack_push(PDC_stack_t *stack, void *value) 19 | { 20 | if (stack->size == stack->capacity) { 21 | size_t new_capacity = stack->capacity * 2; 22 | void **new_data = realloc(stack->data, sizeof(void *) * new_capacity); 23 | if (new_data == NULL) { 24 | LOG_ERROR("Failed to reallocate memory for stack!\n"); 25 | exit(1); 26 | } 27 | stack->data = new_data; 28 | stack->capacity = new_capacity; 29 | } 30 | stack->data[stack->size++] = value; 31 | } 32 | 33 | void * 34 | stack_pop(PDC_stack_t *stack) 35 | { 36 | if (stack->size == 0) { 37 | LOG_ERROR("Stack underflow!\n"); 38 | exit(1); 39 | } 40 | return stack->data[--stack->size]; 41 | } 42 | 43 | void 44 | stack_free(PDC_stack_t *stack) 45 | { 46 | free(stack->data); 47 | } -------------------------------------------------------------------------------- /src/commons/file/Readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hpc-io/pdc/89268e80d232f33758516cc26ddcb81f5a493700/src/commons/file/Readme.md -------------------------------------------------------------------------------- /src/commons/file/include/bin_file_ops.h: -------------------------------------------------------------------------------- 1 | 2 | /* File foo. */ 3 | #ifndef BIN_FILE_OPS_H 4 | #define BIN_FILE_OPS_H 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | void bin_append_int(int data, FILE *stream); 12 | 13 | void bin_append_double(double data, FILE *stream); 14 | 15 | void bin_append_string(char *data, FILE *stream); 16 | 17 | void bin_append_string_with_len(char *data, size_t len, FILE *stream); 18 | 19 | void bin_append_uint64(uint64_t data, FILE *stream); 20 | 21 | void bin_append_size_t(size_t data, FILE *stream); 22 | 23 | void bin_append_type(int type, FILE *stream); 24 | 25 | int *bin_read_int(FILE *file); 26 | 27 | double *bin_read_double(FILE *file); 28 | 29 | void *bin_read_index_numeric_value(int *is_float, FILE *file); 30 | 31 | char *bin_read_string(FILE *file); 32 | 33 | uint64_t *bin_read_uint64(FILE *file); 34 | 35 | size_t *bin_read_size_t(FILE *file); 36 | 37 | size_t bin_skip_field(FILE *stream); 38 | 39 | #endif /* !BIN_FILE_OPS_H */ 40 | -------------------------------------------------------------------------------- /src/commons/generic/pdc_generic.c: -------------------------------------------------------------------------------- 1 | #include "pdc_generic.h" 2 | 3 | size_t 4 | get_number_from_string(char *str, pdc_c_var_type_t type, void **val_ptr) 5 | { 6 | if (val_ptr == NULL) { 7 | return 0; 8 | } 9 | 10 | void * k = NULL; 11 | size_t key_len = get_size_by_dtype(type); 12 | 13 | k = malloc(key_len); 14 | 15 | switch (type) { 16 | case PDC_SHORT: 17 | *((short *)k) = (short)strtol(str, NULL, 10); 18 | break; 19 | case PDC_INT: 20 | case PDC_INT32: 21 | *((int *)k) = (int)strtol(str, NULL, 10); 22 | break; 23 | case PDC_UINT: 24 | case PDC_UINT32: 25 | *((unsigned int *)k) = (unsigned int)strtoul(str, NULL, 10); 26 | break; 27 | case PDC_LONG: 28 | *((long *)k) = strtol(str, NULL, 10); 29 | break; 30 | case PDC_INT8: 31 | *((int8_t *)k) = (int8_t)strtol(str, NULL, 10); 32 | break; 33 | case PDC_UINT8: 34 | *((uint8_t *)k) = (uint8_t)strtoul(str, NULL, 10); 35 | break; 36 | case PDC_INT16: 37 | *((int16_t *)k) = (int16_t)strtol(str, NULL, 10); 38 | break; 39 | case PDC_UINT16: 40 | *((uint16_t *)k) = (uint16_t)strtoul(str, NULL, 10); 41 | break; 42 | case PDC_INT64: 43 | *((int64_t *)k) = strtoll(str, NULL, 10); 44 | break; 45 | case PDC_UINT64: 46 | *((uint64_t *)k) = strtoull(str, NULL, 10); 47 | break; 48 | case PDC_FLOAT: 49 | *((float *)k) = strtof(str, NULL); 50 | break; 51 | case PDC_DOUBLE: 52 | *((double *)k) = strtod(str, NULL); 53 | break; 54 | default: 55 | free(k); 56 | return 0; 57 | } 58 | 59 | *val_ptr = k; 60 | return key_len; 61 | } -------------------------------------------------------------------------------- /src/commons/include/pdc_public.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Notice for 3 | * Proactive Data Containers (PDC) Software Library and Utilities 4 | * ----------------------------------------------------------------------------- 5 | 6 | *** Copyright Notice *** 7 | 8 | * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the 9 | * University of California, through Lawrence Berkeley National Laboratory, 10 | * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF 11 | * Group (subject to receipt of any required approvals from the U.S. Dept. of 12 | * Energy). All rights reserved. 13 | 14 | * If you have questions about your rights to use or distribute this software, 15 | * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. 16 | 17 | * NOTICE. This Software was developed under funding from the U.S. Department of 18 | * Energy and the U.S. Government consequently retains certain rights. As such, the 19 | * U.S. Government has been granted for itself and others acting on its behalf a 20 | * paid-up, nonexclusive, irrevocable, worldwide license in the Software to 21 | * reproduce, distribute copies to the public, prepare derivative works, and 22 | * perform publicly and display publicly, and to permit other to do so. 23 | */ 24 | 25 | #ifndef PDC_PUBLIC_H 26 | #define PDC_PUBLIC_H 27 | 28 | #include 29 | #include 30 | #include 31 | #include "pdc_generic.h" 32 | 33 | /*******************/ 34 | /* Public Typedefs */ 35 | /*******************/ 36 | typedef int perr_t; 37 | typedef uint64_t pdcid_t; 38 | typedef unsigned long long psize_t; 39 | typedef bool pbool_t; 40 | 41 | typedef int PDC_int_t; 42 | typedef float PDC_float_t; 43 | typedef double PDC_double_t; 44 | 45 | typedef pdc_c_var_type_t pdc_var_type_t; 46 | 47 | // FIXME: common data structure should be defined in a group of common header files. 48 | typedef struct pdc_kvtag_t { 49 | char * name; 50 | uint32_t size; 51 | int8_t type; 52 | void * value; 53 | } pdc_kvtag_t; 54 | 55 | typedef enum { PDC_PERSIST, PDC_TRANSIENT } pdc_lifetime_t; 56 | 57 | typedef enum { PDC_SERVER_DEFAULT = 0, PDC_SERVER_PER_CLIENT = 1 } pdc_server_selection_t; 58 | 59 | typedef struct pdc_histogram_t { 60 | pdc_var_type_t dtype; 61 | int nbin; 62 | double incr; 63 | double * range; 64 | uint64_t * bin; 65 | } pdc_histogram_t; 66 | 67 | #define SUCCEED 0 68 | #define FAIL (-1) 69 | 70 | #define PDC_SIZE_UNLIMITED UINT64_MAX 71 | 72 | #endif /* PDC_PUBLIC_H */ 73 | -------------------------------------------------------------------------------- /src/commons/index/Readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hpc-io/pdc/89268e80d232f33758516cc26ddcb81f5a493700/src/commons/index/Readme.md -------------------------------------------------------------------------------- /src/commons/index/dart/dart_math.c: -------------------------------------------------------------------------------- 1 | #include "dart_math.h" 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | double 9 | log_with_base(double base, double x) 10 | { 11 | return log(x) / log(base); 12 | } 13 | 14 | uint32_t 15 | uint32_pow(uint32_t base, uint32_t pow) 16 | { 17 | uint32_t p = 0; 18 | uint32_t rst = 1; 19 | for (p = 0; p < pow; p++) { 20 | rst = base * rst; 21 | } 22 | return rst; 23 | } 24 | 25 | uint64_t 26 | uint64_pow(uint64_t base, uint64_t pow) 27 | { 28 | uint64_t p = 0; 29 | uint64_t rst = 1; 30 | for (p = 0; p < pow; p++) { 31 | rst = base * rst; 32 | } 33 | return rst; 34 | } 35 | 36 | int 37 | int_abs(int a) 38 | { 39 | if (a < 0) { 40 | return 0 - a; 41 | } 42 | return a; 43 | } -------------------------------------------------------------------------------- /src/commons/index/dart/dart_utils.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include "dart_utils.h" 7 | #ifndef M_PI 8 | #define M_PI 3.14159265358979323846 9 | #endif 10 | 11 | void 12 | random_seed(long int seed) 13 | { 14 | if (seed > 0) 15 | srand(seed); 16 | else if (seed == 0) 17 | srand(time(NULL)); 18 | } 19 | 20 | double 21 | random_range(double min, double max) 22 | { 23 | // return numbers between a given range 24 | double range = (max - min); 25 | double div = RAND_MAX / range; 26 | return min + (rand() / div); 27 | } 28 | 29 | double 30 | random_uniform() 31 | { 32 | // return numbers between 0 and 1 33 | return (double)rand() / (RAND_MAX + 1.0); 34 | } 35 | 36 | double 37 | random_normal(double mean, double dev) 38 | { 39 | // returns numbers based on normal distribution given mean and std dev 40 | 41 | // normal distribution centered on 0 with std dev of 1 42 | // Box-Muller transform 43 | double randomNum_normal = 44 | sqrt(-2 * log((rand() + 1.0) / (RAND_MAX + 1.0))) * cos(2 * M_PI * (rand() + 1.0) / (RAND_MAX + 1.0)); 45 | double y = mean + dev * randomNum_normal; 46 | 47 | return y; 48 | } 49 | 50 | double 51 | random_exponential(double mean) 52 | { 53 | // returns numbers based on exp distr given mean 54 | double x = 1.0 / mean; // inverse lambda 55 | 56 | double u; // this will be my uniform random variable 57 | double exp_value; 58 | 59 | // Pull a uniform random number (0 < z < 1) 60 | do { 61 | u = random_uniform(); 62 | } while ((u == 0) || (u == 1)); 63 | exp_value = -x * log(u); 64 | 65 | return (exp_value); 66 | } 67 | 68 | int64_t 69 | get_timestamp_ns() 70 | { 71 | struct timespec current; 72 | int64_t rst = -1; 73 | 74 | if (clock_gettime(CLOCK_REALTIME, ¤t) == -1) { 75 | return rst; 76 | } 77 | 78 | return current.tv_sec * (int64_t)1e9 + current.tv_nsec; 79 | } 80 | 81 | int64_t 82 | get_timestamp_us() 83 | { 84 | return get_timestamp_ns() / 1000; 85 | } 86 | 87 | int64_t 88 | get_timestamp_ms() 89 | { 90 | return get_timestamp_us() / 1000; 91 | } 92 | 93 | int64_t 94 | get_timestamp_s() 95 | { 96 | return get_timestamp_ms() / 1000; 97 | } -------------------------------------------------------------------------------- /src/commons/index/dart/include/dart_algo.h: -------------------------------------------------------------------------------- 1 | #ifndef DART_ALGO_H 2 | #define DART_ALGO_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include "dart_math.h" 11 | 12 | void to_bytes(uint32_t val, uint8_t *bytes); 13 | 14 | uint32_t to_int32(const uint8_t *bytes); 15 | 16 | void md5(const uint8_t *initial_msg, size_t initial_len, uint8_t *digest); 17 | 18 | uint32_t murmur3_32(const uint8_t *key, size_t len, uint32_t seed); 19 | 20 | uint32_t md5_hash(int prefix_len, char *word); 21 | 22 | uint32_t djb2_hash(char *str, int len); 23 | 24 | #endif // DART_ALGO_H -------------------------------------------------------------------------------- /src/commons/index/dart/include/dart_math.h: -------------------------------------------------------------------------------- 1 | #ifndef DART_MATH_H 2 | #define DART_MATH_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include "dart_utils.h" 10 | 11 | double log_with_base(double base, double x); 12 | uint32_t uint32_pow(uint32_t base, uint32_t pow); 13 | uint64_t uint64_pow(uint64_t base, uint64_t pow); 14 | int int_abs(int a); 15 | 16 | #endif // END DART_MATH_H -------------------------------------------------------------------------------- /src/commons/index/dart/include/dart_utils.h: -------------------------------------------------------------------------------- 1 | #ifndef DART_UTILS_H 2 | #define DART_UTILS_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | void random_seed(long int seed); 12 | double random_uniform(); 13 | double random_range(double min, double max); 14 | double random_normal(double mean, double dev); 15 | double random_exponential(double mean); 16 | 17 | int64_t get_timestamp_ns(); 18 | 19 | int64_t get_timestamp_us(); 20 | 21 | int64_t get_timestamp_ms(); 22 | 23 | int64_t get_timestamp_s(); 24 | 25 | #endif // DART_UTILS_H -------------------------------------------------------------------------------- /src/commons/index/dart/index/idioms/include/idioms_persistence.h: -------------------------------------------------------------------------------- 1 | #ifndef IDIOMS_PERSISTENCE_H 2 | #define IDIOMS_PERSISTENCE_H 3 | 4 | #include "idioms_local_index.h" 5 | 6 | /** 7 | * @brief Dumping the index to a file. 8 | * @param dir_path The directory path to store the index file. 9 | * @param serverID The server ID. 10 | * @return perr_t SUCCESS on success, FAIL on failure 11 | */ 12 | perr_t idioms_metadata_index_dump(IDIOMS_t *idioms, char *dir_path, uint32_t serverID); 13 | 14 | /** 15 | * @brief Recovering the index from a file. Please initialize idioms before calling this function. 16 | * @param dir_path The directory path to store the index file. 17 | * @param num_server The number of servers. 18 | * @param serverID The server ID. 19 | * @return perr_t SUCCESS on success, FAIL on failure 20 | */ 21 | perr_t idioms_metadata_index_recover(IDIOMS_t *idioms, char *dir_path, int num_server, uint32_t serverID); 22 | 23 | // /** 24 | // * @brief Initialize the DART space via idioms. 25 | // * @param dart The DART space to be initialized. 26 | // * @param num_client The number of clients. 27 | // * @param num_server The number of servers. 28 | // * @param max_server_num_to_adapt The maximum number of servers to adapt. 29 | // */ 30 | // void init_dart_space_via_idioms(DART *dart, int num_client, int num_server, int max_server_num_to_adapt); 31 | 32 | #endif // IDIOMS_PERSISTENCE_H -------------------------------------------------------------------------------- /src/commons/logging/include/pdc_logger.h: -------------------------------------------------------------------------------- 1 | #ifndef PDC_LOGGER_H 2 | #define PDC_LOGGER_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #define MAX_LOG_MSG_LENGTH 1024 11 | #define MAX_LOG_FILE_SIZE (10 * 1024 * 1024) // 10 MB 12 | #define MAX_LOG_FILE_NAME_LENGTH 256 13 | 14 | typedef enum { LOG_LEVEL_ERROR, LOG_LEVEL_WARNING, LOG_LEVEL_INFO, LOG_LEVEL_DEBUG } PDC_LogLevel; 15 | 16 | static FILE * logFiles[4] = {NULL}; // Log files for each log level 17 | static char logFilenames[4][MAX_LOG_FILE_NAME_LENGTH]; 18 | static PDC_LogLevel logLevel = LOG_LEVEL_INFO; 19 | 20 | void setLogFile(PDC_LogLevel level, const char *fileName); 21 | 22 | void setLogLevel(PDC_LogLevel level); 23 | 24 | /** 25 | * just_print is equivalent to calling printf("%s", args) meaning no extra information 26 | * such as the file and line number will be printed 27 | */ 28 | void log_message(bool just_print, PDC_LogLevel level, const char *file, const char *func, int line_number, 29 | const char *format, ...); 30 | 31 | #define LOG_ERROR(format, ...) \ 32 | log_message(false, LOG_LEVEL_ERROR, __FILE__, __func__, __LINE__, format, ##__VA_ARGS__) 33 | #define LOG_WARNING(format, ...) \ 34 | log_message(false, LOG_LEVEL_WARNING, __FILE__, __func__, __LINE__, format, ##__VA_ARGS__) 35 | #define LOG_INFO(format, ...) \ 36 | log_message(false, LOG_LEVEL_INFO, __FILE__, __func__, __LINE__, format, ##__VA_ARGS__) 37 | #define LOG_DEBUG(format, ...) \ 38 | log_message(false, LOG_LEVEL_DEBUG, __FILE__, __func__, __LINE__, format, ##__VA_ARGS__) 39 | #define LOG_JUST_PRINT(format, ...) \ 40 | log_message(true, LOG_LEVEL_INFO, __FILE__, __func__, __LINE__, format, ##__VA_ARGS__) 41 | 42 | #endif // PDC_LOGGER_H -------------------------------------------------------------------------------- /src/commons/query/include/qc_parser.h: -------------------------------------------------------------------------------- 1 | #ifndef QC_PARSER_H 2 | #define QC_PARSER_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include "string_utils.h" 8 | 9 | typedef enum { NONE = 0, OR = 1, AND = 2, NOT = 3 } logical_op_t; 10 | 11 | typedef enum { FACTOR = 0, TERM = 1, EXPRESSION = 2 } condition_type_t; 12 | 13 | static const char *logicalOpStr[] = {NULL, "OR", "AND", "NOT"}; 14 | 15 | typedef struct subClause { 16 | int start; 17 | int end; 18 | int level; 19 | struct subClause *subClauseArray; 20 | int numSubClauses; 21 | } subClause; 22 | 23 | /** 24 | * 1. Factor -> left and right are NULL && op is NONE. if the corresponding string contains 'AND'/'OR' then it 25 | * is a string of expression. 26 | * 2. Term -> either left or right must be factor, the other can be a factor or a term 27 | * 3. Expression -> either left or right must be a term, the other can be a term or an expression 28 | */ 29 | typedef struct Condition { 30 | int start; 31 | int end; 32 | int level; 33 | condition_type_t type; 34 | struct Condition *left; 35 | struct Condition *right; 36 | logical_op_t op; 37 | } Condition; 38 | 39 | int isFactor(char *expression, Condition *condition); 40 | int isTerm(char *expression, Condition *condition); 41 | int isExpression(char *expression, Condition *condition); 42 | int isExpressionStringFactor(char *expression, Condition *condition); 43 | 44 | int extractExpression(char *expression, Condition *condition); 45 | 46 | int extractTerm(char *expression, Condition *condition); 47 | 48 | int extractFactor(char *expression, Condition *condition); 49 | 50 | int splitCondition(char *expression, Condition *condition); 51 | 52 | int subClauseExtractor(char *expression, int start, int end, int level, subClause **subPtr); 53 | 54 | void printSubClauses(char *expression, subClause *subPtr, int numSubClauses); 55 | 56 | #endif // QC_PARSER_H -------------------------------------------------------------------------------- /src/commons/query/qc_parser_test.c: -------------------------------------------------------------------------------- 1 | #include "qc_parser.h" 2 | 3 | int 4 | main() 5 | { 6 | char *expression = 7 | "z=\"tsa\" AND t=234234 OR NOT (e=234 AND (a=123 OR b=\"abc\") AND ((NOT c=987) OR d=258))"; 8 | Condition *root = (Condition *)malloc(sizeof(Condition)); 9 | root->start = 0; 10 | root->end = strlen(expression); 11 | root->level = 0; 12 | splitCondition(expression, root); 13 | } -------------------------------------------------------------------------------- /src/commons/serde/bulki/bulki_vle_util.c: -------------------------------------------------------------------------------- 1 | #include "bulki_vle_util.h" 2 | 3 | // Encode an unsigned integer using Variable-Length Encoding 4 | size_t 5 | BULKI_vle_encode_uint(uint64_t value, uint8_t *buffer) 6 | { 7 | size_t bytes_written = 0; 8 | do { 9 | uint8_t byte = value & 0x7F; 10 | value >>= 7; 11 | if (value != 0) { 12 | byte |= 0x80; 13 | } 14 | buffer[bytes_written++] = byte; 15 | } while (value != 0); 16 | return bytes_written; 17 | } 18 | 19 | // quickly calculate the encoded size of an unsigned integer 20 | size_t 21 | BULKI_vle_encoded_uint_size(uint64_t value) 22 | { 23 | size_t bytes_written = 1; 24 | // 0x80 is 128 in decimal, meaning we have more than 7 bits to encode 25 | while (value >= 0x80) { // this test will help us to save some computation. 26 | value >>= 7; 27 | bytes_written++; 28 | } 29 | return bytes_written; 30 | } 31 | 32 | // Decode an unsigned integer using Variable-Length Encoding 33 | uint64_t 34 | BULKI_vle_decode_uint(const uint8_t *buffer, size_t *bytes_read) 35 | { 36 | uint64_t result = 0; 37 | size_t shift = 0; 38 | size_t byte_count = 0; 39 | 40 | while (1) { 41 | uint8_t byte = buffer[byte_count++]; 42 | result |= (uint64_t)(byte & 0x7F) << shift; 43 | if ((byte & 0x80) == 0) { 44 | break; 45 | } 46 | shift += 7; 47 | } 48 | 49 | if (bytes_read != NULL) { 50 | *bytes_read = byte_count; 51 | } 52 | 53 | return result; 54 | } 55 | 56 | size_t 57 | BULKI_vle_encode_int(int64_t value, uint8_t *buffer) 58 | { 59 | // ZigZag encode the signed integer 60 | uint64_t zigzag_encoded = (value << 1) ^ (value >> 63); 61 | return BULKI_vle_encode_uint(zigzag_encoded, buffer); 62 | } 63 | 64 | int64_t 65 | BULKI_vle_decode_int(const uint8_t *buffer, size_t *bytes_read) 66 | { 67 | uint64_t zigzag_encoded = BULKI_vle_decode_uint(buffer, bytes_read); 68 | return (zigzag_encoded >> 1) ^ -(zigzag_encoded & 1); 69 | } 70 | 71 | size_t 72 | BULKI_vle_encoded_int_size(int64_t value) 73 | { 74 | // ZigZag encode the signed integer 75 | uint64_t zigzag_encoded = (value << 1) ^ (value >> 63); 76 | return BULKI_vle_encoded_uint_size(zigzag_encoded); 77 | } -------------------------------------------------------------------------------- /src/commons/serde/bulki/include/bulki_endianness_util.h: -------------------------------------------------------------------------------- 1 | #ifndef BULKI_ENDIANNESS_UTIL_H 2 | #define BULKI_ENDIANNESS_UTIL_H 3 | 4 | #include 5 | 6 | // Function to check if the system is little-endian 7 | int BULKI_is_system_le(); 8 | 9 | // Function to check if the system is big-endian 10 | int BULKI_is_system_be(); 11 | 12 | // Functions to convert from big-endian to little-endian 13 | uint8_t BULKI_to_little_endian_u8(uint8_t value); 14 | uint16_t BULKI_to_little_endian_u16(uint16_t value); 15 | uint32_t BULKI_to_little_endian_u32(uint32_t value); 16 | uint64_t BULKI_to_little_endian_u64(uint64_t value); 17 | int8_t BULKI_to_little_endian_i8(int8_t value); 18 | int16_t BULKI_to_little_endian_i16(int16_t value); 19 | int32_t BULKI_to_little_endian_i32(int32_t value); 20 | int64_t BULKI_to_little_endian_i64(int64_t value); 21 | float BULKI_to_little_endian_f32(float value); 22 | double BULKI_to_little_endian_f64(double value); 23 | int BULKI_to_little_endian_int(int value); 24 | long BULKI_to_little_endian_long(long value); 25 | short BULKI_to_little_endian_short(short value); 26 | char BULKI_to_little_endian_char(char value); 27 | 28 | #endif -------------------------------------------------------------------------------- /src/commons/serde/bulki/include/bulki_vle_util.h: -------------------------------------------------------------------------------- 1 | #ifndef BULKI_VLE_UTIL_H 2 | #define BULKI_VLE_UTIL_H 3 | 4 | // Include necessary headers here if needed 5 | 6 | // Function declarations for VLE (Variable-Length Encoding) utilities can be added here 7 | #include 8 | #include 9 | 10 | // Encode an unsigned integer using Variable-Length Encoding 11 | size_t BULKI_vle_encode_uint(uint64_t value, uint8_t *buffer); 12 | 13 | // Decode an unsigned integer using Variable-Length Encoding 14 | uint64_t BULKI_vle_decode_uint(const uint8_t *buffer, size_t *bytes_read); 15 | 16 | // quickly calculate the encoded size of an unsigned integer 17 | size_t BULKI_vle_encoded_uint_size(uint64_t value); 18 | 19 | // Encode a signed integer using Variable-Length Encoding 20 | size_t BULKI_vle_encode_int(int64_t value, uint8_t *buffer); 21 | 22 | // Decode a signed integer using Variable-Length Encoding 23 | int64_t BULKI_vle_decode_int(const uint8_t *buffer, size_t *bytes_read); 24 | 25 | // quickly calculate the encoded size of a signed integer 26 | size_t BULKI_vle_encoded_int_size(int64_t value); 27 | 28 | #endif // BULKI_VLE_UTIL_H 29 | -------------------------------------------------------------------------------- /src/commons/utils/Readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hpc-io/pdc/89268e80d232f33758516cc26ddcb81f5a493700/src/commons/utils/Readme.md -------------------------------------------------------------------------------- /src/commons/utils/include/pdc_id_pkg.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Notice for 3 | * Proactive Data Containers (PDC) Software Library and Utilities 4 | * ----------------------------------------------------------------------------- 5 | 6 | *** Copyright Notice *** 7 | 8 | * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the 9 | * University of California, through Lawrence Berkeley National Laboratory, 10 | * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF 11 | * Group (subject to receipt of any required approvals from the U.S. Dept. of 12 | * Energy). All rights reserved. 13 | 14 | * If you have questions about your rights to use or distribute this software, 15 | * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. 16 | 17 | * NOTICE. This Software was developed under funding from the U.S. Department of 18 | * Energy and the U.S. Government consequently retains certain rights. As such, the 19 | * U.S. Government has been granted for itself and others acting on its behalf a 20 | * paid-up, nonexclusive, irrevocable, worldwide license in the Software to 21 | * reproduce, distribute copies to the public, prepare derivative works, and 22 | * perform publicly and display publicly, and to permit other to do so. 23 | */ 24 | 25 | #ifndef PDC_ID_PKG_H 26 | #define PDC_ID_PKG_H 27 | 28 | #include "pdc_public.h" 29 | #include "pdc_linkedlist.h" 30 | #include "mercury_atomic.h" 31 | /* 32 | * Number of bits to use for ID Type in each atom. Increase if more types 33 | * are needed (though this will decrease the number of available IDs per 34 | * type). This is the only number that must be changed since all other bit 35 | * field sizes and masks are calculated from TYPE_BITS. 36 | */ 37 | #define TYPE_BITS 8 38 | #define TYPE_MASK (((pdcid_t)1 << TYPE_BITS) - 1) 39 | #define PDC_MAX_NUM_TYPES TYPE_MASK 40 | /* 41 | * Number of bits to use for the Atom index in each atom (assumes 8-bit 42 | * bytes). We don't use the sign bit. 43 | */ 44 | #define ID_BITS ((sizeof(pdcid_t) * 8) - (TYPE_BITS + 1)) 45 | #define ID_MASK (((pdcid_t)1 << ID_BITS) - 1) 46 | 47 | /* Map an atom to an ID type number */ 48 | #define PDC_TYPE(a) ((PDC_type_t)(((pdcid_t)(a) >> ID_BITS) & TYPE_MASK)) 49 | 50 | struct _pdc_class { 51 | char * name; 52 | pdcid_t local_id; 53 | }; 54 | 55 | struct _pdc_id_info { 56 | pdcid_t id; /* ID for this info */ 57 | hg_atomic_int32_t count; /* ref. count for this atom */ 58 | void * obj_ptr; /* pointer associated with the atom */ 59 | PDC_LIST_ENTRY(_pdc_id_info) entry; 60 | }; 61 | 62 | #endif /* PDC_ID_PKG_H */ 63 | -------------------------------------------------------------------------------- /src/commons/utils/include/timer_utils.h: -------------------------------------------------------------------------------- 1 | /** timer.h 2 | * Microsecond timer in C, using `gettimeofday()`. 3 | * 4 | * Note: `gettimeofday()` is Unix, GNU/Linux and 5 | * Mac OS X system-specific. 6 | * 7 | * The only portable function is time.h's `clock()`, 8 | * but it isn't very precise. 9 | * 10 | * See: http://www.songho.ca/misc/timer/timer.html 11 | * 12 | * Copyright (c) 2011,2013 Alexandre Dantas 13 | */ 14 | 15 | #ifndef TIMER_UTILS_H_DEFINED 16 | #define TIMER_UTILS_H_DEFINED 17 | 18 | #include 19 | #include /* gettimeofday() */ 20 | #include /* bool */ 21 | #include /* int64_t */ 22 | 23 | typedef struct stopwatch_t { 24 | int64_t start_mark; /* Timer start point */ 25 | int64_t pause_mark; /* In case we pause the timer */ 26 | bool running; /* Is it running? */ 27 | bool paused; /* Is it paused? */ 28 | 29 | } stopwatch_t; 30 | 31 | int64_t timer_us_timestamp(); 32 | 33 | int64_t timer_ms_timestamp(); 34 | 35 | int64_t timer_s_timestamp(); 36 | 37 | /** Starts the timer. 38 | * 39 | * @note If called multiple times, restarts the timer. 40 | */ 41 | void timer_start(stopwatch_t *t); 42 | 43 | /** Pauses the timer. 44 | * 45 | * @note If called multiple times, it does nothing. 46 | */ 47 | void timer_pause(stopwatch_t *t); 48 | 49 | /** Unpauses the timer. 50 | * 51 | * @note If the timer's not paused or this is called 52 | * multiple times, it does nothing. 53 | */ 54 | void timer_unpause(stopwatch_t *t); 55 | 56 | /** Returns the time difference in microseconds 57 | * 58 | * @note (1/1000000 seconds) 59 | */ 60 | double timer_delta_us(stopwatch_t *t); 61 | 62 | /** Returns the time difference in miliseconds. 63 | * 64 | * @note (1/1000 seconds) 65 | */ 66 | double timer_delta_ms(stopwatch_t *t); 67 | 68 | /** Returns the time difference in seconds. */ 69 | double timer_delta_s(stopwatch_t *t); 70 | 71 | /** Returns the time difference in minutes (60 seconds). */ 72 | double timer_delta_m(stopwatch_t *t); 73 | 74 | /** Returns the time difference in hours (3600 seconds). */ 75 | double timer_delta_h(stopwatch_t *t); 76 | 77 | #endif /* TIMER_UTILS_H_DEFINED */ 78 | -------------------------------------------------------------------------------- /src/commons/utils/query_utils_test.c: -------------------------------------------------------------------------------- 1 | #include "query_utils.h" 2 | 3 | void 4 | print_query_output(query_gen_output_t *output) 5 | { 6 | println("key query: %s, len: %lu", output->key_query, output->key_query_len); 7 | println("value query: %s, len: %lu", output->value_query, output->value_query_len); 8 | char *final_query_str = gen_query_str(output); 9 | println("final query: %s, len: %lu", final_query_str, strlen(final_query_str)); 10 | free(final_query_str); 11 | } 12 | 13 | int 14 | main(int argc, char *argv[]) 15 | { 16 | int affix_length = atoi(argv[1]); 17 | pdc_kvtag_t *base_string_tag; 18 | base_string_tag = (pdc_kvtag_t *)calloc(1, sizeof(pdc_kvtag_t)); 19 | base_string_tag->name = "testname"; 20 | base_string_tag->type = PDC_STRING; 21 | base_string_tag->value = "testvalue"; 22 | 23 | for (int i = 0; i < 4; i++) { 24 | int key_query_type = i; 25 | int value_query_type = i; 26 | query_gen_input_t input; 27 | query_gen_output_t output; 28 | input.base_tag = base_string_tag; 29 | input.key_query_type = key_query_type; 30 | input.value_query_type = value_query_type; 31 | input.affix_len = affix_length; 32 | gen_query_key_value(&input, &output); 33 | print_query_output(&output); 34 | free_query_output(&output); 35 | } 36 | 37 | pdc_kvtag_t *base_int_tag; 38 | int int_value = 234; 39 | base_int_tag = (pdc_kvtag_t *)calloc(1, sizeof(pdc_kvtag_t)); 40 | base_int_tag->name = "testname"; 41 | base_int_tag->type = PDC_INT; 42 | base_int_tag->value = (void *)calloc(1, sizeof(int)); 43 | ((int *)base_int_tag->value)[0] = int_value; 44 | 45 | for (int i = 4; i < 6; i++) { 46 | int key_query_type = 0; 47 | int value_query_type = i; 48 | int range_offset = 10; 49 | int range_lo = int_value - range_offset; 50 | int range_hi = int_value + range_offset; 51 | query_gen_input_t input; 52 | query_gen_output_t output; 53 | input.base_tag = base_int_tag; 54 | input.key_query_type = key_query_type; 55 | input.value_query_type = value_query_type; 56 | input.range_lo = range_lo; 57 | input.range_hi = range_hi; 58 | gen_query_key_value(&input, &output); 59 | print_query_output(&output); 60 | free_query_output(&output); 61 | } 62 | 63 | return 0; 64 | } -------------------------------------------------------------------------------- /src/commons/utils/timer_utils.c: -------------------------------------------------------------------------------- 1 | #include "timer_utils.h" 2 | 3 | #define MICROSECONDS_IN_SECONDS 1000000 4 | 5 | /** 6 | * Function that returns the current timestamp in microseconds 7 | * since the Epoch (1970-01-01 00:00:00:000:000 UTC). 8 | */ 9 | int64_t 10 | timer_us_timestamp() 11 | { 12 | struct timeval tmp; 13 | gettimeofday(&(tmp), NULL); 14 | return ((int64_t)tmp.tv_usec) + (int64_t)(tmp.tv_sec * MICROSECONDS_IN_SECONDS); 15 | } 16 | 17 | /** 18 | * Function that returns the current timestamp in miliseconds 19 | * since the Epoch (1970-01-01 00:00:00:000 UTC). 20 | */ 21 | int64_t 22 | timer_ms_timestamp() 23 | { 24 | return (timer_us_timestamp() / 1000); 25 | } 26 | 27 | /** 28 | * Function that returns the current timestamp in seconds 29 | * since the Epoch (1970-01-01 00:00:00 UTC). 30 | */ 31 | int64_t 32 | timer_s_timestamp() 33 | { 34 | return (timer_ms_timestamp() / 1000); 35 | } 36 | 37 | void 38 | timer_start(stopwatch_t *t) 39 | { 40 | t->start_mark = timer_us_timestamp(); 41 | t->pause_mark = 0; 42 | t->running = true; 43 | t->paused = false; 44 | } 45 | 46 | void 47 | timer_pause(stopwatch_t *t) 48 | { 49 | if (!(t->running) || (t->paused)) 50 | return; 51 | 52 | t->pause_mark = timer_us_timestamp() - (t->start_mark); 53 | t->running = false; 54 | t->paused = true; 55 | } 56 | 57 | void 58 | timer_unpause(stopwatch_t *t) 59 | { 60 | if (t->running || !(t->paused)) 61 | return; 62 | 63 | t->start_mark = timer_us_timestamp() - (t->pause_mark); 64 | t->running = true; 65 | t->paused = false; 66 | } 67 | 68 | double 69 | timer_delta_us(stopwatch_t *t) 70 | { 71 | if (t->running) 72 | return timer_us_timestamp() - (t->start_mark); 73 | 74 | if (t->paused) 75 | return t->pause_mark; 76 | 77 | // Will never actually get here 78 | return (double)((t->pause_mark) - (t->start_mark)); 79 | } 80 | 81 | double 82 | timer_delta_ms(stopwatch_t *t) 83 | { 84 | return (timer_delta_us(t) / 1000.0); 85 | } 86 | 87 | double 88 | timer_delta_s(stopwatch_t *t) 89 | { 90 | return (timer_delta_ms(t) / 1000.0); 91 | } 92 | 93 | double 94 | timer_delta_m(stopwatch_t *t) 95 | { 96 | return (timer_delta_s(t) / 60.0); 97 | } 98 | 99 | double 100 | timer_delta_h(stopwatch_t *t) 101 | { 102 | return (timer_delta_m(t) / 60.0); 103 | } 104 | -------------------------------------------------------------------------------- /src/server/dablooms/LICENSE: -------------------------------------------------------------------------------- 1 | Permission is hereby granted, free of charge, to any person obtaining a copy 2 | of this software and associated documentation files (the "Software"), to deal 3 | in the Software without restriction, including without limitation the rights 4 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 5 | copies of the Software, and to permit persons to whom the Software is 6 | furnished to do so, subject to the following conditions: 7 | 8 | The above copyright notice and this permission notice shall be included in 9 | all copies or substantial portions of the Software. 10 | 11 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 12 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 13 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 14 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 15 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 16 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 17 | THE SOFTWARE. 18 | -------------------------------------------------------------------------------- /src/server/dablooms/pdc_murmur.h: -------------------------------------------------------------------------------- 1 | //----------------------------------------------------------------------------- 2 | // MurmurHash3 was written by Austin Appleby, and is placed in the public 3 | // domain. The author hereby disclaims copyright to this source code. 4 | 5 | #ifndef _MURMURHASH3_H_ 6 | #define _MURMURHASH3_H_ 7 | 8 | #include 9 | 10 | void MurmurHash3_x64_128(const void *key, int len, uint32_t seed, void *out); 11 | 12 | #endif // _MURMURHASH3_H_ 13 | -------------------------------------------------------------------------------- /src/server/include/pdc_server_common.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Notice for 3 | * Proactive Data Containers (PDC) Software Library and Utilities 4 | * ----------------------------------------------------------------------------- 5 | 6 | *** Copyright Notice *** 7 | 8 | * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the 9 | * University of California, through Lawrence Berkeley National Laboratory, 10 | * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF 11 | * Group (subject to receipt of any required approvals from the U.S. Dept. of 12 | * Energy). All rights reserved. 13 | 14 | * If you have questions about your rights to use or distribute this software, 15 | * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. 16 | 17 | * NOTICE. This Software was developed under funding from the U.S. Department of 18 | * Energy and the U.S. Government consequently retains certain rights. As such, the 19 | * U.S. Government has been granted for itself and others acting on its behalf a 20 | * paid-up, nonexclusive, irrevocable, worldwide license in the Software to 21 | * reproduce, distribute copies to the public, prepare derivative works, and 22 | * perform publicly and display publicly, and to permit other to do so. 23 | */ 24 | 25 | #ifndef PDC_SERVER_COMMON_H 26 | #define PDC_SERVER_COMMON_H 27 | 28 | #define PDC_ALLOC_BASE_NUM 64 29 | #define PDC_SERVER_TASK_INIT_VALUE 1000 30 | 31 | /* extern hg_class_t *hg_class_g; */ 32 | /* extern hg_thread_mutex_t pdc_client_connect_mutex_g; */ 33 | typedef struct server_lookup_args_t { 34 | int server_id; 35 | uint32_t client_id; 36 | int ret_int; 37 | char * ret_string; 38 | void * void_buf; 39 | char * server_addr; 40 | pdc_metadata_t *meta; 41 | region_list_t **region_lists; 42 | uint32_t n_loc; 43 | hg_handle_t rpc_handle; 44 | } server_lookup_args_t; 45 | 46 | struct transfer_buf_map { 47 | hg_handle_t handle; 48 | buf_map_in_t in; 49 | }; 50 | 51 | typedef struct pdc_remote_server_info_t { 52 | char * addr_string; 53 | int addr_valid; 54 | hg_addr_t addr; 55 | } pdc_remote_server_info_t; 56 | 57 | typedef struct pdc_client_info_t { 58 | char addr_string[ADDR_MAX]; 59 | int addr_valid; 60 | hg_addr_t addr; 61 | } pdc_client_info_t; 62 | 63 | #endif /* PDC_SERVER_COMMON_H */ 64 | -------------------------------------------------------------------------------- /src/server/include/pdc_server_metadata_index.h: -------------------------------------------------------------------------------- 1 | #ifndef PDC_SERVER_METADATA_INDEX_H 2 | #define PDC_SERVER_METADATA_INDEX_H 3 | 4 | #include "pdc_client_server_common.h" 5 | 6 | // #include "hashtab.h" 7 | #include "query_utils.h" 8 | #include "timer_utils.h" 9 | #include "art.h" 10 | #include "pdc_set.h" 11 | #include "pdc_hash.h" 12 | #include "pdc_compare.h" 13 | #include "dart_core.h" 14 | #include "pdc_hash_table.h" 15 | #include "bin_file_ops.h" 16 | 17 | /** 18 | * @brief Initialize local index 19 | * @param num_server The number of servers 20 | * @param server_id The server ID 21 | */ 22 | void PDC_Server_metadata_index_init(uint32_t num_server, uint32_t server_id); 23 | 24 | /** 25 | * @brief Get the server information for the metadata index 26 | * @param in [IN] Input parameters for the server info 27 | * @param out [OUT] Output parameters for the server info 28 | * @return perr_t SUCCESS on success, FAIL on failure 29 | */ 30 | perr_t PDC_Server_dart_get_server_info(dart_get_server_info_in_t *in, dart_get_server_info_out_t *out); 31 | 32 | /** 33 | * @brief Perform various of DART operations on one single server. 34 | * @param in [IN] Input parameters for the DART operation 35 | * @param out [OUT] Output parameters for the DART operation 36 | * @return perr_t SUCCESS on success, FAIL on failure 37 | */ 38 | perr_t PDC_Server_dart_perform_one_server(dart_perform_one_server_in_t * in, 39 | dart_perform_one_server_out_t *out, uint64_t *n_obj_ids_ptr, 40 | uint64_t **buf_ptrs); 41 | /** 42 | * @brief Dumping the index to a file. 43 | * @param checkpoint_dir The directory path to store the index file. 44 | * @param serverID The server ID. 45 | * @return perr_t SUCCESS on success, FAIL on failure 46 | */ 47 | perr_t metadata_index_dump(char *checkpoint_dir, uint32_t serverID); 48 | 49 | /** 50 | * @brief Recovering the index from a file. Please initialize idioms before calling this function. 51 | * @param checkpiont_dir The directory path to store the index file. 52 | * @param num_server The number of servers. 53 | * @param serverID The server ID. 54 | */ 55 | perr_t metadata_index_recover(char *checkpiont_dir, int num_server, uint32_t serverID); 56 | 57 | #endif /* PDC_SERVER_METADATA_INDEX_H */ -------------------------------------------------------------------------------- /src/server/pdc_server_analysis/include/pdc_server_analysis.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Notice for 3 | * Proactive Data Containers (PDC) Software Library and Utilities 4 | * ----------------------------------------------------------------------------- 5 | 6 | *** Copyright Notice *** 7 | 8 | * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the 9 | * University of California, through Lawrence Berkeley National Laboratory, 10 | * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF 11 | * Group (subject to receipt of any required approvals from the U.S. Dept. of 12 | * Energy). All rights reserved. 13 | 14 | * If you have questions about your rights to use or distribute this software, 15 | * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. 16 | 17 | * NOTICE. This Software was developed under funding from the U.S. Department of 18 | * Energy and the U.S. Government consequently retains certain rights. As such, the 19 | * U.S. Government has been granted for itself and others acting on its behalf a 20 | * paid-up, nonexclusive, irrevocable, worldwide license in the Software to 21 | * reproduce, distribute copies to the public, prepare derivative works, and 22 | * perform publicly and display publicly, and to permit other to do so. 23 | */ 24 | 25 | #ifndef PDC_ANALYSIS_SERVER_H 26 | #define PDC_ANALYSIS_SERVER_H 27 | 28 | #include "pdc_public.h" 29 | #include "pdc_analysis_and_transforms_common.h" 30 | 31 | perr_t PDC_Server_instantiate_data_iterator(obj_data_iterator_in_t *in, obj_data_iterator_out_t *out); 32 | 33 | int PDC_get_analysis_registry(struct _pdc_region_analysis_ftn_info ***registry); 34 | 35 | #endif 36 | -------------------------------------------------------------------------------- /src/server/pdc_server_main.c: -------------------------------------------------------------------------------- 1 | #include "pdc_server.h" 2 | 3 | int 4 | main(int argc, char *argv[]) 5 | { 6 | return server_run(argc, argv); 7 | } -------------------------------------------------------------------------------- /src/server/pdc_server_region/include/pdc_server_region_cache.h: -------------------------------------------------------------------------------- 1 | #ifndef PDC_REGION_CACHE_H 2 | #define PDC_REGION_CACHE_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include "pdc_region.h" 9 | #include "pdc_client_server_common.h" 10 | #include "pdc_timing.h" 11 | 12 | #ifdef PDC_SERVER_CACHE 13 | 14 | #define PDC_REGION_CONTAINED 0 15 | #define PDC_REGION_CONTAINED_BY 1 16 | #define PDC_REGION_PARTIAL_OVERLAP 2 17 | #define PDC_REGION_NO_OVERLAP 3 18 | #define PDC_MERGE_FAILED 4 19 | #define PDC_MERGE_SUCCESS 5 20 | 21 | pthread_mutex_t pdc_obj_cache_list_mutex; 22 | 23 | int PDC_region_server_cache_init(); 24 | int PDC_region_server_cache_finalize(); 25 | int PDC_region_cache_flush_all(); 26 | int PDC_region_cache_flush(uint64_t obj_id); 27 | int PDC_region_fetch(uint64_t obj_id, int obj_ndim, const uint64_t *obj_dims, 28 | struct pdc_region_info *region_info, void *buf, size_t unit); 29 | int PDC_region_cache_register(uint64_t obj_id, int obj_ndim, const uint64_t *obj_dims, const char *buf, 30 | size_t buf_size, const uint64_t *offset, const uint64_t *size, int ndim, 31 | size_t unit); 32 | void *PDC_region_cache_clock_cycle(void *ptr); 33 | 34 | perr_t PDC_transfer_request_data_read_from(uint64_t obj_id, int obj_ndim, const uint64_t *obj_dims, 35 | struct pdc_region_info *region_info, void *buf, size_t unit); 36 | perr_t PDC_transfer_request_data_write_out(uint64_t obj_id, int obj_ndim, const uint64_t *obj_dims, 37 | struct pdc_region_info *region_info, void *buf, size_t unit); 38 | 39 | #endif 40 | 41 | #endif 42 | -------------------------------------------------------------------------------- /src/server/pdc_server_region/include/pdc_server_region_transfer_metadata_query.h: -------------------------------------------------------------------------------- 1 | perr_t transfer_request_metadata_query_init(int pdc_server_size_input, char *checkpoint); 2 | perr_t transfer_request_metadata_query_finalize(); 3 | perr_t transfer_request_metadata_query_checkpoint(char **checkpoint, uint64_t *checkpoint_size); 4 | perr_t transfer_request_metadata_query_lookup_query_buf(uint64_t query_id, char **buf_ptr); 5 | uint64_t transfer_request_metadata_query_parse(int32_t n_objs, char *buf, uint8_t partition_type, 6 | uint64_t *total_buf_size_ptr); 7 | -------------------------------------------------------------------------------- /src/tests/create_region.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Notice for 3 | * Proactive Data Containers (PDC) Software Library and Utilities 4 | * ----------------------------------------------------------------------------- 5 | 6 | *** Copyright Notice *** 7 | 8 | * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the 9 | * University of California, through Lawrence Berkeley National Laboratory, 10 | * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF 11 | * Group (subject to receipt of any required approvals from the U.S. Dept. of 12 | * Energy). All rights reserved. 13 | 14 | * If you have questions about your rights to use or distribute this software, 15 | * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. 16 | 17 | * NOTICE. This Software was developed under funding from the U.S. Department of 18 | * Energy and the U.S. Government consequently retains certain rights. As such, the 19 | * U.S. Government has been granted for itself and others acting on its behalf a 20 | * paid-up, nonexclusive, irrevocable, worldwide license in the Software to 21 | * reproduce, distribute copies to the public, prepare derivative works, and 22 | * perform publicly and display publicly, and to permit other to do so. 23 | */ 24 | 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | 34 | #include "pdc.h" 35 | 36 | int 37 | main(int argc, char **argv) 38 | { 39 | pdcid_t pdc; 40 | int ret_value = 0; 41 | uint64_t offset[3], offset_length[3]; 42 | int rank = 0, size = 1; 43 | offset[0] = 0; 44 | offset[1] = 2; 45 | offset[2] = 5; 46 | offset_length[0] = 2; 47 | offset_length[1] = 3; 48 | offset_length[2] = 5; 49 | 50 | #ifdef ENABLE_MPI 51 | MPI_Init(&argc, &argv); 52 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 53 | MPI_Comm_size(MPI_COMM_WORLD, &size); 54 | #endif 55 | 56 | // create a pdc 57 | 58 | pdc = PDCinit("pdc"); 59 | LOG_INFO("create a new pdc\n"); 60 | 61 | PDCregion_create(3, offset, offset_length); 62 | 63 | PDCregion_create(2, offset, offset_length); 64 | 65 | PDCregion_create(1, offset, offset_length); 66 | 67 | // close pdc 68 | if (PDCclose(pdc) < 0) { 69 | LOG_ERROR("Failed to close PDC\n"); 70 | ret_value = 1; 71 | } 72 | 73 | #ifdef ENABLE_MPI 74 | MPI_Finalize(); 75 | #endif 76 | return ret_value; 77 | } 78 | -------------------------------------------------------------------------------- /src/tests/dt_conv.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Notice for 3 | * Proactive Data Containers (PDC) Software Library and Utilities 4 | * ----------------------------------------------------------------------------- 5 | 6 | *** Copyright Notice *** 7 | 8 | * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the 9 | * University of California, through Lawrence Berkeley National Laboratory, 10 | * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF 11 | * Group (subject to receipt of any required approvals from the U.S. Dept. of 12 | * Energy). All rights reserved. 13 | 14 | * If you have questions about your rights to use or distribute this software, 15 | * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. 16 | 17 | * NOTICE. This Software was developed under funding from the U.S. Department of 18 | * Energy and the U.S. Government consequently retains certain rights. As such, the 19 | * U.S. Government has been granted for itself and others acting on its behalf a 20 | * paid-up, nonexclusive, irrevocable, worldwide license in the Software to 21 | * reproduce, distribute copies to the public, prepare derivative works, and 22 | * perform publicly and display publicly, and to permit other to do so. 23 | */ 24 | 25 | #include 26 | #include 27 | #include 28 | #include "pdc.h" 29 | 30 | int 31 | main(int argc, char **argv) 32 | { 33 | int rank = 0, size = 1; 34 | 35 | #ifdef ENABLE_MPI 36 | MPI_Init(&argc, &argv); 37 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 38 | MPI_Comm_size(MPI_COMM_WORLD, &size); 39 | #endif 40 | 41 | // create a pdc 42 | pdcid_t pdc = PDC_init("pdc"); 43 | LOG_INFO("generated new pdc, id is %lld\n", pdc); 44 | 45 | float a[10] = {1.1, 2.1, 3.1, 4.1, 5.1, 6.1, 7.1, 8.1, 9.1, 10.1}; 46 | int b[5]; 47 | pdc_type_conv(PDC_FLOAT, PDC_INT, a, b, 5, 2); 48 | 49 | int i; 50 | for (i = 0; i < 5; i++) 51 | LOG_INFO("b[%d] is %d\n", i, b[i]); 52 | // close pdc 53 | if (PDC_close(pdc) < 0) 54 | LOG_ERROR("Failed to close PDC\n"); 55 | else 56 | LOG_INFO("PDC is closed\n"); 57 | 58 | #ifdef ENABLE_MPI 59 | MPI_Finalize(); 60 | #endif 61 | } 62 | -------------------------------------------------------------------------------- /src/tests/gen_fastbit_idx.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include "pdc.h" 10 | #include "pdc_client_connect.h" 11 | 12 | int 13 | main(int argc, char **argv) 14 | { 15 | uint64_t nhits; 16 | char * var_name; 17 | pdc_query_t * qpreload_x; 18 | pdc_metadata_t *meta; 19 | pdcid_t pdc, id; 20 | float preload_value = -2000000000.0; 21 | 22 | if (argc < 2) { 23 | LOG_ERROR("Please enter var name as input!\n"); 24 | fflush(stdout); 25 | } 26 | var_name = argv[1]; 27 | 28 | pdc = PDCinit("pdc"); 29 | 30 | // Query the created object 31 | PDC_Client_query_metadata_name_timestep(var_name, 0, &meta); 32 | if (meta == NULL || meta->obj_id == 0) { 33 | LOG_ERROR("Error with [%s] metadata!\n", var_name); 34 | goto done; 35 | } 36 | id = meta->obj_id; 37 | 38 | qpreload_x = PDCquery_create(id, PDC_GT, PDC_FLOAT, &preload_value); 39 | 40 | PDCquery_get_nhits(qpreload_x, &nhits); 41 | LOG_INFO("Generated Fastbit index for [%s], total %" PRIu64 " elements\n", var_name, nhits); 42 | PDCquery_free_all(qpreload_x); 43 | 44 | done: 45 | if (PDCclose(pdc) < 0) 46 | LOG_ERROR("Failed to close PDC\n"); 47 | 48 | return 0; 49 | } 50 | -------------------------------------------------------------------------------- /src/tests/init_only.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include "pdc.h" 7 | 8 | void 9 | print_usage() 10 | { 11 | LOG_JUST_PRINT("Usage: srun -n ./data_server_read obj_name size_MB\n"); 12 | } 13 | 14 | int 15 | main(int argc, char **argv) 16 | { 17 | int rank = 0, size = 1; 18 | pdcid_t pdc, cont_prop, cont; 19 | 20 | #ifdef ENABLE_MPI 21 | MPI_Init(&argc, &argv); 22 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 23 | MPI_Comm_size(MPI_COMM_WORLD, &size); 24 | #endif 25 | 26 | // create a pdc 27 | pdc = PDCinit("pdc"); 28 | 29 | // create a container property 30 | cont_prop = PDCprop_create(PDC_CONT_CREATE, pdc); 31 | if (cont_prop <= 0) 32 | LOG_ERROR("Failed to create container property"); 33 | 34 | // create a container 35 | cont = PDCcont_create("c1", cont_prop); 36 | if (cont <= 0) 37 | LOG_ERROR("Failed to create container"); 38 | 39 | LOG_INFO("PROC[%d] FINISHED!\n", rank); 40 | 41 | // close a container 42 | if (PDCcont_close(cont) < 0) 43 | LOG_ERROR("Failed to close container\n"); 44 | 45 | // close a container property 46 | if (PDCprop_close(cont_prop) < 0) 47 | LOG_ERROR("Failed to close property"); 48 | 49 | if (PDCclose(pdc) < 0) 50 | LOG_ERROR("Failed to close PDC\n"); 51 | 52 | #ifdef ENABLE_MPI 53 | MPI_Finalize(); 54 | #endif 55 | 56 | return 0; 57 | } 58 | -------------------------------------------------------------------------------- /src/tests/mpi_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # This version of the test runner doesn't attempt to run any parallel tests. 3 | # We assume too, that if the library build has enabled MPI, that LD_LIBRARY_PATH is 4 | # defined and points to the MPI libraries used by the linker (e.g. -L 26 | #include 27 | #include 28 | #include "pdc.h" 29 | 30 | int 31 | main(int argc, char **argv) 32 | { 33 | pdcid_t pdc; 34 | int rank = 0, size = 1; 35 | int ret_value = 0; 36 | 37 | // create a pdc 38 | #ifdef ENABLE_MPI 39 | MPI_Init(&argc, &argv); 40 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 41 | MPI_Comm_size(MPI_COMM_WORLD, &size); 42 | #endif 43 | 44 | pdc = PDCinit("pdc"); 45 | LOG_INFO("generated new pdc\n"); 46 | 47 | // close pdc 48 | if (PDCclose(pdc) < 0) { 49 | LOG_ERROR("Failed to close PDC\n"); 50 | ret_value = 1; 51 | } 52 | #ifdef ENABLE_MPI 53 | MPI_Finalize(); 54 | #endif 55 | return ret_value; 56 | } 57 | -------------------------------------------------------------------------------- /src/tests/query_vpic.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include "pdc.h" 10 | #include "pdc_client_connect.h" 11 | 12 | int 13 | main(void) 14 | { 15 | uint64_t i; 16 | pdc_metadata_t *energy_meta; 17 | pdcid_t pdc, energy_id; 18 | 19 | // Construct query constraints 20 | float energy_lo0 = 3.75, energy_hi0 = 4.0; 21 | pdc_query_t * q0l, *q0h, *q; 22 | pdc_selection_t sel; 23 | 24 | struct timeval pdc_timer_start; 25 | struct timeval pdc_timer_end; 26 | 27 | pdc = PDCinit("pdc"); 28 | 29 | PDC_Client_query_metadata_name_timestep("Energy", 0, &energy_meta); 30 | if (energy_meta == NULL || energy_meta->obj_id == 0) { 31 | LOG_ERROR("Error with energy metadata!\n"); 32 | goto done; 33 | } 34 | energy_id = energy_meta->obj_id; 35 | 36 | q0l = PDCquery_create(energy_id, PDC_GTE, PDC_FLOAT, &energy_lo0); 37 | q0h = PDCquery_create(energy_id, PDC_LTE, PDC_FLOAT, &energy_hi0); 38 | q = PDCquery_and(q0l, q0h); 39 | 40 | // Get selection 41 | gettimeofday(&pdc_timer_start, 0); 42 | 43 | PDCquery_get_selection(q, &sel); 44 | 45 | gettimeofday(&pdc_timer_end, 0); 46 | double get_sel_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); 47 | LOG_INFO("Get selection time: %.5e\n", get_sel_time); 48 | 49 | LOG_INFO(" Query results:\n"); 50 | if (sel.nhits < 500) 51 | PDCselection_print_all(&sel); 52 | else 53 | PDCselection_print(&sel); 54 | 55 | float *energy_data; 56 | if (sel.nhits > 0) 57 | energy_data = (float *)calloc(sel.nhits, sizeof(float)); 58 | 59 | // Get data 60 | gettimeofday(&pdc_timer_start, 0); 61 | 62 | PDCquery_get_data(energy_id, &sel, energy_data); 63 | 64 | gettimeofday(&pdc_timer_end, 0); 65 | double get_data_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); 66 | LOG_INFO("Get data time: %.5e\n", get_data_time); 67 | 68 | LOG_INFO("Query result energy data (%" PRIu64 " hits):\n", sel.nhits); 69 | for (i = 0; i < sel.nhits; i++) { 70 | if (energy_data[i] > energy_hi0 || energy_data[i] < energy_lo0) { 71 | LOG_ERROR("Error with result %" PRIu64 ": %.5e\n", i, energy_data[i]); 72 | } 73 | } 74 | LOG_INFO("Verified: all correct!\n"); 75 | 76 | PDCquery_free_all(q); 77 | PDCselection_free(&sel); 78 | 79 | done: 80 | if (PDCclose(pdc) < 0) 81 | LOG_ERROR("Failed to close PDC\n"); 82 | 83 | return 0; 84 | } 85 | -------------------------------------------------------------------------------- /src/tests/run_checkpoint_restart_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # This version of the test runner doesn't attempt to run any parallel tests. 3 | # We assume too, that if the library build has enabled MPI, that LD_LIBRARY_PATH is 4 | # defined and points to the MPI libraries used by the linker (e.g. -L 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | typedef enum { ASC, DESC } sorting_direction_t; 18 | 19 | void collect_dir(const char *dir_path, int (*selector)(const struct dirent *), 20 | int (*cmp)(const struct dirent **, const struct dirent **), sorting_direction_t sd, int topk, 21 | int (*on_file)(struct dirent *f_entry, const char *parent_path, void *args), 22 | int (*on_dir)(struct dirent *d_entry, const char *parent_path, void *args), void *coll_args, 23 | int (*pre_op)(void *coll_args), int (*post_op)(void *coll_args)); 24 | 25 | int is_regular_file(const char *path); 26 | 27 | size_t get_file_size(const char *filename); 28 | 29 | int dir_exists(char *dirname); 30 | 31 | /* Function with behaviour like `mkdir -p' */ 32 | int mkpath(const char *s, mode_t mode); 33 | 34 | #endif /* !PDC_TOOLS_FS_OPS */ -------------------------------------------------------------------------------- /src/tools/meta_json/Readme.md: -------------------------------------------------------------------------------- 1 | # This is a directory for the meta_json tool. 2 | 3 | The meta_json tool is a command line tool that can be used to generate a JSON file that contains metadata about the source code in a directory. The metadata includes information about the files in the directory, the functions in the files, and the classes in the files. -------------------------------------------------------------------------------- /src/tools/meta_json/llsm_converter/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hpc-io/pdc/89268e80d232f33758516cc26ddcb81f5a493700/src/tools/meta_json/llsm_converter/__init__.py -------------------------------------------------------------------------------- /src/tools/meta_json/llsm_converter/requirement.txt: -------------------------------------------------------------------------------- 1 | pandas 2 | pyarrow 3 | numpy 4 | jsonschema 5 | argparse -------------------------------------------------------------------------------- /src/tools/meta_json/llsm_converter/validator/metadata_schema_validator.py: -------------------------------------------------------------------------------- 1 | import json 2 | import argparse 3 | from jsonschema import validate 4 | from jsonschema.exceptions import ValidationError 5 | 6 | def load_json(file_path): 7 | """Load JSON data from a file.""" 8 | with open(file_path, 'r') as file: 9 | return json.load(file) 10 | 11 | def validate_json(json_data, json_schema): 12 | """Validate JSON data against a schema.""" 13 | try: 14 | validate(instance=json_data, schema=json_schema) 15 | return "valid" 16 | except ValidationError as e: 17 | print(e) 18 | return "invalid" 19 | 20 | def main(schema_path, data_path): 21 | """Main function to load and validate JSON data against a schema.""" 22 | schema = load_json(schema_path) 23 | print("Schema {} loaded".format(schema_path)) 24 | data = load_json(data_path) 25 | print("JSON {} loaded".format(data_path)) 26 | result = validate_json(data, schema) 27 | print("{} is {}. Object count: {}".format(data_path, result, len(data["objects"]))) 28 | 29 | if __name__ == "__main__": 30 | parser = argparse.ArgumentParser(description="Validate JSON against a schema.") 31 | parser.add_argument("-s", "--schema", dest="schema_path", required=True, help="Path to the JSON schema file") 32 | parser.add_argument("-d", "--data", dest="data_path", required=True, help="Path to the JSON data file") 33 | args = parser.parse_args() 34 | main(args.schema_path, args.data_path) 35 | -------------------------------------------------------------------------------- /src/tools/meta_json/metadata_json_importer.h: -------------------------------------------------------------------------------- 1 | #ifndef METADATA_JSON_IMPORTER_H 2 | #define METADATA_JSON_IMPORTER_H 3 | #include 4 | #include 5 | #include 6 | #include "cjson/cJSON.h" 7 | #include "metadata_json_processor.h" 8 | 9 | // Your code here 10 | 11 | /** 12 | * @brief init_importer 13 | * @param md_json_args 14 | * @return 0 if success, -1 if error 15 | */ 16 | int init_importer(MD_JSON_ARGS *md_json_args); 17 | 18 | /** 19 | * @brief import_json_header 20 | * @param dataset_name 21 | * @param dataset_description 22 | * @param source_URL 23 | * @param collector 24 | * @param md_json_args -> here we should have the object ID stored in md_json_args 25 | * @return 0 if success, -1 if error 26 | */ 27 | int import_json_header(cJSON *dataset_name, cJSON *dataset_description, cJSON *source_URL, cJSON *collector, 28 | MD_JSON_ARGS *md_json_args); 29 | 30 | /** 31 | * @brief import_object_base 32 | * @param name 33 | * @param type 34 | * @param full_path 35 | * @param md_json_args -> here we should have the object ID in md_json_args already 36 | * @return 0 if success, -1 if error 37 | */ 38 | int import_object_base(cJSON *name, cJSON *type, cJSON *full_path, MD_JSON_ARGS *md_json_args); 39 | 40 | /** 41 | * @brief import_object_property 42 | * @param name 43 | * @param type 44 | * @param cls 45 | * @param value 46 | * @param md_json_args -> here we should have the object ID in md_json_args already 47 | * @return 0 if success, -1 if error 48 | * 49 | */ 50 | int import_object_property(cJSON *name, cJSON *type, cJSON *cls, cJSON *value, MD_JSON_ARGS *md_json_args); 51 | 52 | /** 53 | * @brief finish_import_one_json 54 | * @param md_json_args 55 | * @return 0 if success, -1 if error 56 | */ 57 | int finish_import_one_json(MD_JSON_ARGS *md_json_args); 58 | 59 | /** 60 | * @brief finalize_importer 61 | * @param md_json_args 62 | * @return 0 if success, -1 if error 63 | */ 64 | int finalize_importer(MD_JSON_ARGS *md_json_args); 65 | 66 | /** 67 | * @brief create_md_json_importer 68 | * @return 69 | */ 70 | MD_JSON_PROCESSOR *create_md_json_importer(); 71 | 72 | #endif // METADATA_JSON_IMPORTER_H 73 | -------------------------------------------------------------------------------- /src/tools/meta_json/metadata_json_printer.c: -------------------------------------------------------------------------------- 1 | #include "metadata_json_printer.h" 2 | 3 | int 4 | init_printer(MD_JSON_ARGS *md_json_args) 5 | { 6 | return EXIT_SUCCESS; 7 | } 8 | 9 | int 10 | print_json_header(cJSON *dataset_name, cJSON *dataset_description, cJSON *source_URL, cJSON *collector, 11 | MD_JSON_ARGS *md_json_args) 12 | { 13 | LOG_JUST_PRINT("Dataset Name: %s\n", dataset_name->valuestring); 14 | LOG_JUST_PRINT("Dataset Description: %s\n", dataset_description->valuestring); 15 | LOG_JUST_PRINT("Source URL: %s\n", source_URL->valuestring); 16 | LOG_JUST_PRINT("Collector: %s\n\n", collector->valuestring); 17 | return 0; 18 | } 19 | 20 | int 21 | print_object_base(cJSON *name, cJSON *type, cJSON *full_path, MD_JSON_ARGS *md_json_args) 22 | { 23 | LOG_JUST_PRINT(" Object Name: %s\n", name->valuestring); 24 | LOG_JUST_PRINT(" Object Type: %s\n", type->valuestring); 25 | LOG_JUST_PRINT(" Object Full Path: %s\n\n", full_path->valuestring); 26 | return 0; 27 | } 28 | 29 | int 30 | print_object_property(cJSON *name, cJSON *type, cJSON *cls, cJSON *value, MD_JSON_ARGS *md_json_args) 31 | { 32 | LOG_JUST_PRINT(" Property Name: %s\n", name->valuestring); 33 | if (cJSON_IsString(value)) { 34 | LOG_JUST_PRINT(" Property Value: %s\n", value->valuestring); 35 | } 36 | else if (cJSON_IsNumber(value)) { 37 | LOG_JUST_PRINT(" Property Value: %f\n", value->valuedouble); 38 | } 39 | LOG_JUST_PRINT(" Property Class: %s\n", cls->valuestring); 40 | LOG_JUST_PRINT(" Property Type: %s\n\n", type->valuestring); 41 | return 0; 42 | } 43 | 44 | int 45 | done_printing_one_json(MD_JSON_ARGS *md_json_args) 46 | { 47 | return 0; 48 | } 49 | 50 | int 51 | finalize_printer(MD_JSON_ARGS *md_json_args) 52 | { 53 | return 0; 54 | } 55 | 56 | MD_JSON_PROCESSOR * 57 | create_md_json_printer() 58 | { 59 | MD_JSON_PROCESSOR *md_json_processor = (MD_JSON_PROCESSOR *)malloc(sizeof(MD_JSON_PROCESSOR)); 60 | md_json_processor->init_processor = init_printer; 61 | md_json_processor->process_json_header = print_json_header; 62 | md_json_processor->process_object_base = print_object_base; 63 | md_json_processor->process_object_property = print_object_property; 64 | md_json_processor->complete_one_json_file = done_printing_one_json; 65 | md_json_processor->finalize_processor = finalize_printer; 66 | return md_json_processor; 67 | } -------------------------------------------------------------------------------- /src/tools/meta_json/metadata_json_printer.h: -------------------------------------------------------------------------------- 1 | #ifndef METADATA_JSON_PRINTER_H 2 | #define METADATA_JSON_PRINTER_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include "cjson/cJSON.h" 8 | #include "metadata_json_processor.h" 9 | 10 | /** 11 | * @brief init_printer 12 | * @param md_json_args 13 | * @return 0 if success, -1 if error 14 | */ 15 | int init_printer(MD_JSON_ARGS *md_json_args); 16 | 17 | /** 18 | * @brief print_json_header 19 | * @param dataset_name 20 | * @param dataset_description 21 | * @param source_URL 22 | * @param collector 23 | * @param md_json_args 24 | * @return 0 if success, -1 if error 25 | */ 26 | int print_json_header(cJSON *dataset_name, cJSON *dataset_description, cJSON *source_URL, cJSON *collector, 27 | MD_JSON_ARGS *md_json_args); 28 | 29 | /** 30 | * @brief print_object_base 31 | * @param name 32 | * @param type 33 | * @param full_path 34 | * @return 0 if success, -1 if error 35 | */ 36 | int print_object_base(cJSON *name, cJSON *type, cJSON *full_path, MD_JSON_ARGS *md_json_args); 37 | 38 | /** 39 | * @brief print_object_property 40 | * @param name 41 | * @param type 42 | * @param cls 43 | * @param value 44 | * @param md_json_args 45 | * @return 0 if success, -1 if error 46 | * 47 | */ 48 | int print_object_property(cJSON *name, cJSON *type, cJSON *cls, cJSON *value, MD_JSON_ARGS *md_json_args); 49 | 50 | /** 51 | * @brief done_printing_one_json 52 | * @param md_json_args 53 | * @return 0 if success, -1 if error 54 | */ 55 | int done_printing_one_json(MD_JSON_ARGS *md_json_args); 56 | 57 | /** 58 | * @brief finalize_printer 59 | * @param md_json_args 60 | * @return 0 if success, -1 if error 61 | */ 62 | int finalize_printer(MD_JSON_ARGS *md_json_args); 63 | 64 | /** 65 | * @brief create_md_json_printer 66 | * @return 67 | */ 68 | MD_JSON_PROCESSOR *create_md_json_printer(); 69 | 70 | #endif // METADATA_JSON_PRINTER_H 71 | -------------------------------------------------------------------------------- /src/tools/meta_json/metadata_json_processor.h: -------------------------------------------------------------------------------- 1 | #ifndef METADATA_JSON_PROCESSOR_H 2 | #define METADATA_JSON_PROCESSOR_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include "cjson/cJSON.h" 9 | 10 | typedef struct { 11 | void * processor_args; // pdc_importer_args_t 12 | void * arg1; // unused 13 | void * arg2; // unused 14 | int current_file_count; 15 | int processed_file_count; 16 | int mpi_size; 17 | int mpi_rank; 18 | uint64_t total_obj_count; 19 | uint64_t total_prop_count; 20 | } MD_JSON_ARGS; 21 | 22 | typedef struct { 23 | 24 | int (*init_processor)(MD_JSON_ARGS *); 25 | 26 | int (*process_json_header)(cJSON *, cJSON *, cJSON *, cJSON *, MD_JSON_ARGS *); 27 | 28 | int (*process_object_base)(cJSON *, cJSON *, cJSON *, MD_JSON_ARGS *); 29 | 30 | int (*process_object_property)(cJSON *, cJSON *, cJSON *, cJSON *, MD_JSON_ARGS *); 31 | 32 | int (*complete_one_json_file)(MD_JSON_ARGS *); 33 | 34 | int (*finalize_processor)(MD_JSON_ARGS *); 35 | 36 | } MD_JSON_PROCESSOR; 37 | 38 | #endif // METADATA_JSON_PROCESSOR_H --------------------------------------------------------------------------------