├── VERSION ├── bin ├── mana_start_coordinator ├── mana_status ├── mana_coordinator └── mpicc_mana ├── mpi-proxy-split ├── lower-half │ ├── .gitignore │ ├── VERSION │ ├── patch-trampoline.h │ ├── lh-func-ptr.c │ ├── README │ ├── mem-wrapper.h │ ├── Makefile │ └── logging.h ├── .gitignore ├── doc-old │ ├── figures │ │ ├── mpiDrainAlgo1.pdf │ │ ├── mpiDrainAlgo2.pdf │ │ ├── mpiDrainAlgo3.pdf │ │ ├── mpiDrainAlgo4.pdf │ │ └── mpiDominoEffect.pdf │ ├── README │ ├── 2pcmodelv4 │ │ ├── 2pcmodelv4mc.tla │ │ ├── 2pcmodelv4mc.cfg │ │ └── Makefile │ ├── design.bib │ └── Makefile ├── mana_header.h ├── TODO.md ├── test │ ├── Comm_compare_test.c │ ├── Abort_test.c │ ├── Comm_free_test.c │ ├── f_ibarrier.f90 │ ├── Get_processor_name_test.c │ ├── Initialized_test.c │ ├── unsync_Finalize.c │ ├── Scan_test.c │ ├── Barrier_test.c │ ├── Alloc_mem.c │ ├── file_test.c │ ├── MPI_Double_Int_test.c │ ├── Gather_test.c │ ├── Scatter_test.c │ ├── MANA_MPI_Initialized_test.c │ ├── hello_mpi_init_thread.c │ ├── Cart_map_test.c │ ├── send_recv.c │ ├── Ibarrier_test.c │ ├── File_size_test.c │ ├── Cartdim_get_test.c │ ├── send_recv_many.c │ ├── Gatherv_test.c │ ├── Bcast_test.c │ ├── Type_commit_contiguous.c │ ├── Comm_dup_test.c │ ├── Group_size_rank.c │ ├── sendrecv_replace_test.c │ ├── File_mixed_fds_test.c │ ├── multi_send_recv.c │ ├── keyval_test.c │ ├── Comm_split_test.c │ ├── Allreduce_test.c │ ├── Sendrecv_test.c │ ├── Reduce_test.c │ ├── Type_vector_test.c │ ├── Waitall_test.c │ ├── Type_create_resized_test.c │ ├── Type_create_hindexed_block_test.c │ ├── integrated_dmtcp_test.c │ ├── Type_hvector_test.c │ ├── Waitany_test.c │ ├── File_read_write_test.c │ ├── Type_dup_test.c │ ├── File_read_write_all_test.c │ ├── mpi_hello_world.c │ ├── Scatterv_test.c │ ├── Isend_test.c │ ├── ping_pong.c │ ├── large_async_p2p.c │ ├── Allgather_test.c │ ├── Type_create_hindexed_test.c │ ├── File_characteristics_test.c │ ├── Comm_get_attr_test.c │ ├── Irecv_test.c │ ├── two-phase-commit-3.c │ ├── Alltoall_test.c │ └── send_recv_loop.c ├── util │ └── mpi-logger │ │ ├── Makefile │ │ ├── README │ │ └── fortran_constants.f90 ├── mpi_files.h ├── seq_num.h ├── cartesian.h ├── mpi-wrappers │ ├── mpi_win_wrappers.cpp │ ├── p2p-deterministic.py │ ├── fortran_constants.f90 │ └── mpi_error_wrappers.cpp ├── unit-test │ ├── record-replay-group-test.cpp │ └── Makefile ├── Makefile_config.in ├── test-ckpt-restart │ ├── Makefile │ ├── mpi_hello_world.c │ └── ping_pong.c ├── uh_wrappers.h ├── autotest.py ├── virtual_id.h ├── mpi_plugin.h └── p2p_drain_send_recv.h ├── util ├── memory_access_tracker │ ├── .gitignore │ ├── Makefile │ ├── memory_access_tracker.h │ ├── README.md │ └── test_memory_tracker.c ├── git-apply-pr ├── hostid.c ├── name_change.sh ├── clear_remoteps.sh ├── hosts ├── proc_maps_size.sh ├── list_ckpt_files.sh ├── clear_logs.sh ├── remoteps.sh ├── sanSize.sh ├── calctotalsize.sh ├── start_mpds.sh ├── readdmtcp.sh ├── fsgsbase-enabled.c ├── README ├── hooks │ ├── pre-commit │ └── post-rewrite ├── cpplint.py.patch ├── git-bisect.sh ├── git-bisect-m32.sh └── attach_all.py ├── .gitmodules ├── manpages ├── README └── Makefile.in ├── autogen.sh ├── ci └── unit-test.sh ├── .gitignore ├── README.md ├── m4 └── ax_check_compile_flag.m4 ├── configure-mana └── .clang-format /VERSION: -------------------------------------------------------------------------------- 1 | 1.2.0 2 | -------------------------------------------------------------------------------- /bin/mana_start_coordinator: -------------------------------------------------------------------------------- 1 | mana_coordinator -------------------------------------------------------------------------------- /mpi-proxy-split/lower-half/.gitignore: -------------------------------------------------------------------------------- 1 | lh_proxy 2 | lh_proxy_da 3 | -------------------------------------------------------------------------------- /mpi-proxy-split/lower-half/VERSION: -------------------------------------------------------------------------------- 1 | Kernel loader version: 12a 2 | -------------------------------------------------------------------------------- /util/memory_access_tracker/.gitignore: -------------------------------------------------------------------------------- 1 | *.o 2 | *.so 3 | *.a 4 | core.* 5 | test_memory_tracker 6 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "dmtcp"] 2 | branch = master 3 | path = dmtcp 4 | url = https://github.com/dmtcp/dmtcp 5 | -------------------------------------------------------------------------------- /util/git-apply-pr: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | curl https://patch-diff.githubusercontent.com/raw/dmtcp/dmtcp/pull/${1}.patch | git am 4 | -------------------------------------------------------------------------------- /util/hostid.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | void 4 | main() 5 | { 6 | printf("%d\n", gethostid()); 7 | } 8 | -------------------------------------------------------------------------------- /util/name_change.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | for i in ./ckpt_rank_* 3 | do 4 | cd $i; 5 | mv *.dmtcp ckpt.dmtcp; 6 | cd ..; 7 | done 8 | -------------------------------------------------------------------------------- /mpi-proxy-split/.gitignore: -------------------------------------------------------------------------------- 1 | test/*.exe 2 | unit-test/*.exe 3 | proxy 4 | mpi-wrappers/*.c* 5 | mpi-wrappers/*.so* 6 | Makefile_config 7 | -------------------------------------------------------------------------------- /mpi-proxy-split/doc-old/figures/mpiDrainAlgo1.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mpickpt/mana/HEAD/mpi-proxy-split/doc-old/figures/mpiDrainAlgo1.pdf -------------------------------------------------------------------------------- /mpi-proxy-split/doc-old/figures/mpiDrainAlgo2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mpickpt/mana/HEAD/mpi-proxy-split/doc-old/figures/mpiDrainAlgo2.pdf -------------------------------------------------------------------------------- /mpi-proxy-split/doc-old/figures/mpiDrainAlgo3.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mpickpt/mana/HEAD/mpi-proxy-split/doc-old/figures/mpiDrainAlgo3.pdf -------------------------------------------------------------------------------- /mpi-proxy-split/doc-old/figures/mpiDrainAlgo4.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mpickpt/mana/HEAD/mpi-proxy-split/doc-old/figures/mpiDrainAlgo4.pdf -------------------------------------------------------------------------------- /mpi-proxy-split/doc-old/README: -------------------------------------------------------------------------------- 1 | This 'doc-old' directory is temporarily retained in case it may be useful for the future. Soon, it will be deleted. 2 | -------------------------------------------------------------------------------- /mpi-proxy-split/doc-old/figures/mpiDominoEffect.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mpickpt/mana/HEAD/mpi-proxy-split/doc-old/figures/mpiDominoEffect.pdf -------------------------------------------------------------------------------- /manpages/README: -------------------------------------------------------------------------------- 1 | Markdown file(s) are the source files that should be converted into manpages 2 | using pandoc. Please check the commented-out lines Makefile for recipe. 3 | -------------------------------------------------------------------------------- /mpi-proxy-split/lower-half/patch-trampoline.h: -------------------------------------------------------------------------------- 1 | #ifndef PATCH_TRAMPOLINE_H 2 | #define PATCH_TRAMPOLINE_H 3 | 4 | void patch_trampoline(void *from_addr, void *to_addr); 5 | #endif 6 | -------------------------------------------------------------------------------- /mpi-proxy-split/mana_header.h: -------------------------------------------------------------------------------- 1 | #ifndef MANA_HEADER_H 2 | #define MANA_HEADER_H 3 | 4 | #define MPI_INIT_NO_THREAD -1 5 | 6 | typedef struct ManaHeader { 7 | int init_flag; 8 | } ManaHeader; 9 | 10 | #endif // MANA_HEADER 11 | -------------------------------------------------------------------------------- /util/clear_remoteps.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | #this script can be used to kill all user processes running on the remote nodes. 3 | #it takes no args and expects a file 'hosts' to contain a list of nodes 4 | 5 | for X in `cat < hosts` 6 | do 7 | echo "Clearing $X..." 8 | ssh $X kill -9 -1 & 9 | done 10 | 11 | wait 12 | 13 | 14 | -------------------------------------------------------------------------------- /util/hosts: -------------------------------------------------------------------------------- 1 | c1-1 2 | c1-2 3 | c1-3 4 | c1-4 5 | c1-5 6 | c1-6 7 | c1-7 8 | c1-8 9 | c1-9 10 | c1-10 11 | c2-1 12 | c2-2 13 | c2-3 14 | c2-4 15 | c2-5 16 | c2-6 17 | c2-7 18 | c2-8 19 | c2-9 20 | c2-10 21 | c2-11 22 | c2-12 23 | c2-13 24 | c2-14 25 | c2-15 26 | c2-16 27 | c2-17 28 | c2-18 29 | c2-19 30 | c2-20 31 | c2-21 32 | c2-22 33 | -------------------------------------------------------------------------------- /util/proc_maps_size.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ -z "$1" ]; then 4 | echo " Not enough params. Wand process PID" 5 | fi 6 | 7 | cat /proc/$1/maps | awk '{ if( $6 == "" ) print "anonym "$1; else print $6" "$1 }' | awk '{ split($2,array,"-"); print $1" 0x"array[2]" 0x"array[1] }' \ 8 | | awk --non-decimal-data '{ print $1": "($2 - $3) }' -------------------------------------------------------------------------------- /util/list_ckpt_files.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | #this script was used to free up space after testing on teracluster 3 | #it also wipes out debugging logs 4 | #it takes no args and expects a file 'hosts' to contain a list of nodes 5 | 6 | for X in localhost `cat < hosts` 7 | do 8 | # echo "Clearing $X..." 9 | ssh $X ls -l /tmp |grep ckpt_ | grep $USER 10 | done 11 | 12 | wait 13 | 14 | 15 | -------------------------------------------------------------------------------- /mpi-proxy-split/lower-half/lh-func-ptr.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "lower-half-api.h" 4 | 5 | static void* MPI_Fnc_Ptrs[] = { 6 | NULL, 7 | FOREACH_FNC(GENERATE_FNC_PTR) 8 | NULL, 9 | }; 10 | 11 | void* lh_dlsym(enum MPI_Fncs fnc) { 12 | if (fnc < MPI_Fnc_NULL || fnc > MPI_Fnc_Invalid) { 13 | return NULL; 14 | } 15 | return MPI_Fnc_Ptrs[fnc]; 16 | } 17 | -------------------------------------------------------------------------------- /autogen.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Does autoconf/automake as needed for DMTCP: 3 | # See for overview: http://www.openismus.com/documents/linux/automake/automake 4 | # INPUT: dmtcp/src/Makefile.am, configure.ac 5 | # RUNS: autoconf, autoheader, automake, etc., in the right order 6 | # OUTPUT: config.h.in, Makefile.in, configure, etc. 7 | 8 | autoreconf --force --install --verbose 9 | # aclocal 10 | # autoconf 11 | -------------------------------------------------------------------------------- /util/clear_logs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | #this script was used to free up space after testing on teracluster 3 | #it also wipes out debugging logs 4 | #it takes no args and expects a file 'hosts' to contain a list of nodes 5 | 6 | for X in localhost `cat < hosts` 7 | do 8 | echo "Clearing $X..." 9 | ssh $X rm -f /tmp/{jassertlog,dmtcpConTable,mpd2}.\* {/dev/shm,~/san,/tmp}/ckpt\*mtcp /tmp/pts\* 10 | done 11 | wait 12 | -------------------------------------------------------------------------------- /mpi-proxy-split/doc-old/2pcmodelv4/2pcmodelv4mc.tla: -------------------------------------------------------------------------------- 1 | ---- MODULE 2pcmodelv4mc ---- 2 | EXTENDS 2pcmodelv4, TLC 3 | 4 | UT_OFFSET_TMP == 1000 5 | BarrierList_TMP == {1, 2} 6 | Procs_TMP == {1, 2} 7 | CkptThreads_TMP == {1 + UT_OFFSET, 2 + UT_OFFSET} 8 | COORD_PROC_TMP == 3 9 | 10 | ------------------------------------------------------------------------------- 11 | =============================================================================== 12 | -------------------------------------------------------------------------------- /util/remoteps.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | #this script list the processes owned by user on all nodes of teracluster 3 | #it takes no args and expects a file 'hosts' to contain a list of nodes 4 | 5 | for X in localhost `cat < hosts` 6 | do 7 | echo "Listing Processes from $X..." 8 | ssh $X "ps aux |grep vsrid| grep -v YACCO | (while read Y; do echo $X \$Y; done)" & 9 | #ssh $X "ps aux |grep $USER| grep -v YACCO | (while read Y; do echo $X \$Y; done)" & 10 | done 11 | 12 | wait 13 | 14 | 15 | -------------------------------------------------------------------------------- /mpi-proxy-split/TODO.md: -------------------------------------------------------------------------------- 1 | ## Here's a list of remaining things 2 | 3 | - [x] Add checkpoint logic -- i.e., skip lower half memory regions 4 | - [x] Add restart logic -- i.e., replicate some of mini-DMTCP functionality 5 | - [x] Test checkpoint-restart with hello world 6 | - [x] Test checkpoint-restart with ping-pong 7 | - [x] Test runtime with Gromacs 8 | - [x] Add record-replay for comm, groups, carts, etc. 9 | - [x] Add checkpoint-restart for barriers 10 | - [x] Test checkpoint-restart for Gromacs 11 | -------------------------------------------------------------------------------- /util/sanSize.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | #this script tells the total size of all the ckpt files on teracluster 3 | #it takes no args and expects a file 'hosts' to contain a list of nodes 4 | 5 | a=0 6 | for X in 0 1 2 3 4 5 6 7 7 | do 8 | for Y in `ls -l /san/global_$X/$USER | tr -s ' ' | cut -f 5 -d ' '` 9 | do 10 | a=`expr $a + $Y` 11 | done 12 | done 13 | a=`echo "scale=2; $a/$[1024*1024]" | bc` 14 | #a=`expr $a / $[ 1024 * 1024 ]` 15 | echo "Total Size : $a MB" 16 | #echo "$@, $a" >> ckpt_sizes 17 | wait 18 | 19 | 20 | -------------------------------------------------------------------------------- /util/calctotalsize.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | #this script tells the total size of all the ckpt files on teracluster 3 | #it takes no args and expects a file 'hosts' to contain a list of nodes 4 | 5 | a=0 6 | for X in localhost `cat < hosts` 7 | do 8 | for Y in `ssh $X ls -l /tmp | grep ckpt_| grep $USER | tr -s ' ' | cut -f 5 -d ' '` 9 | do 10 | a=`expr $a + $Y` 11 | done 12 | done 13 | a=`echo "scale=2; $a/$[1024*1024]" | bc` 14 | #a=`expr $a / $[ 1024 * 1024 ]` 15 | echo "Total Size: $a MB" 16 | #echo "$@, $a" >> ckpt_sizes 17 | wait 18 | 19 | 20 | -------------------------------------------------------------------------------- /ci/unit-test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -x 3 | 4 | SCRIPT_DIR="$(dirname $(readlink -f "$0"))" 5 | 6 | cd $SCRIPT_DIR/.. 7 | 8 | OS_VER=$(grep '^VERSION_ID' /etc/os-release | sed 's/"//g'| cut -d "=" -f2) 9 | #GCC 8 or higher 10 | if [[ $OS_VER == 7 ]]; then 11 | source scl_source enable devtoolset-8 12 | fi 13 | 14 | git submodule update --init 15 | 16 | ./configure 17 | make -j 8 mana 18 | if [ \! -f "bin/dmtcp_launch" ];then echo ERROR: make failed;exit 1;fi 19 | 20 | cd $SCRIPT_DIR/../mpi-proxy-split/unit-test 21 | make || exit 1 22 | make clean 23 | make check || exit 1 24 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/Comm_compare_test.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | int 5 | main() 6 | { 7 | int rank, size, result; 8 | MPI_Comm comm; 9 | 10 | MPI_Init(NULL, NULL); 11 | MPI_Comm_dup(MPI_COMM_WORLD, &comm); 12 | MPI_Comm_rank(comm, &rank); 13 | MPI_Comm_size(comm, &size); 14 | MPI_Comm_compare(MPI_COMM_WORLD, comm, &result); 15 | if (!result) { 16 | printf("[Rank = %d] Comm compare fail! comm = %d, MPI_COMM_WORLD = %d\n", 17 | rank, comm, MPI_COMM_WORLD); 18 | assert(0); 19 | } 20 | MPI_Finalize(); 21 | return 0; 22 | } 23 | -------------------------------------------------------------------------------- /mpi-proxy-split/util/mpi-logger/Makefile: -------------------------------------------------------------------------------- 1 | MPICC = CC 2 | MPIFORTRAN = ftn 3 | default: build 4 | build: fortran_constants.o get_fortran_constants.o 5 | ${MPICC} -I../../include -fPIC -c -g3 -o mpi_logger.o mpi_logger.cpp -lmpi -O0 6 | ${MPICC} -shared -fPIC -o liblogger.so mpi_logger.o get_fortran_constants.o fortran_constants.o 7 | clean: 8 | rm -f -v *.o *.so 9 | 10 | fortran_constants.o: fortran_constants.f90 11 | ${MPIFORTRAN} -g3 -O0 -fPIC -c -o $@ $< 12 | 13 | get_fortran_constants.o: 14 | cc -g -O2 -std=gnu11 -fPIC -I../../dmtcp/include -I.. -I../lower-half -c -o get_fortran_constants.o get_fortran_constants.c 15 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/Abort_test.c: -------------------------------------------------------------------------------- 1 | /* 2 | Source: http://mpi.deino.net/mpi_functions/MPI_Abort.html 3 | */ 4 | #include 5 | #include 6 | #include 7 | int 8 | main(int argc, char *argv[]) 9 | { 10 | MPI_Init(NULL, NULL); 11 | /* 12 | MPI_Abort does not return so any return value is erroneous, 13 | even MPI_SUCCESS. 14 | */ 15 | int ret = 1234; // random number 16 | ret = MPI_Abort(MPI_COMM_WORLD, 911); 17 | /* No further code will execute */ 18 | printf("Return value =%d", ret); 19 | printf("MPI Abort implementation Failed\n"); 20 | fflush(stdout); 21 | MPI_Finalize(); 22 | return 0; 23 | } 24 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/Comm_free_test.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | int 5 | main() 6 | { 7 | int rank, size; 8 | MPI_Comm comm; 9 | 10 | MPI_Init(NULL, NULL); 11 | MPI_Comm_dup(MPI_COMM_WORLD, &comm); 12 | MPI_Comm_rank(comm, &rank); 13 | MPI_Comm_size(comm, &size); 14 | MPI_Comm_free(&comm); 15 | if (comm != MPI_COMM_NULL) { 16 | printf("[Rank = %d] Freed comm was not set to COMM_NULL\n", rank); 17 | fflush(stdout); 18 | assert(0); 19 | } 20 | if (rank == 0) { 21 | printf("Test Passed!\n"); 22 | fflush(stdout); 23 | } 24 | MPI_Finalize(); 25 | return 0; 26 | } 27 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/f_ibarrier.f90: -------------------------------------------------------------------------------- 1 | PROGRAM hello_world_mpi 2 | include 'mpif.h' 3 | 4 | integer process_Rank, size_Of_Cluster, ierror, tag, flag 5 | integer request, status(MPI_STATUS_SIZE) 6 | logical dummy 7 | 8 | call MPI_INIT(ierror) 9 | call MPI_COMM_SIZE(MPI_COMM_WORLD, size_Of_Cluster, ierror) 10 | call MPI_COMM_RANK(MPI_COMM_WORLD, process_Rank, ierror) 11 | call MPI_IBARRIER(MPI_COMM_WORLD, request, ierror) 12 | dummy = .true. 13 | do while (dummy) 14 | end do 15 | call MPI_Wait(request, status, ierror) 16 | 17 | print *, 'Hello World from process: ', process_Rank, 'of ', size_Of_Cluster 18 | 19 | call MPI_FINALIZE(ierror) 20 | END PROGRAM 21 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/Get_processor_name_test.c: -------------------------------------------------------------------------------- 1 | /* 2 | Source: http://mpi.deino.net/mpi_functions/MPI_Get_processor_name.html 3 | */ 4 | #include 5 | #include 6 | #include 7 | 8 | int 9 | main(int argc, char *argv[]) 10 | { 11 | int rank, nprocs, len; 12 | char name[MPI_MAX_PROCESSOR_NAME]; 13 | 14 | MPI_Init(&argc, &argv); 15 | MPI_Comm_size(MPI_COMM_WORLD, &nprocs); 16 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 17 | MPI_Get_processor_name(name, &len); 18 | printf("Hello, world. I am %d of %d on %s with length %d\n", rank, nprocs, 19 | name, len); 20 | fflush(stdout); 21 | MPI_Finalize(); 22 | return 0; 23 | } 24 | -------------------------------------------------------------------------------- /manpages/Makefile.in: -------------------------------------------------------------------------------- 1 | mandir=@mandir@ 2 | PANDOC=@PANDOC@ 3 | INSTALL_DATA = @INSTALL_DATA@ 4 | 5 | MANPAGES = mana.1.gz 6 | 7 | default: ${MANPAGES} 8 | 9 | %.1: %.1.md 10 | @- if test "$(PANDOC)" = "no" ; then \ 11 | echo "Skipping manpage generation; using pregenerated copy."; \ 12 | else \ 13 | $(PANDOC) -s -o $@ $<; \ 14 | fi 15 | 16 | %.1.gz: %.1 17 | gzip --force --keep $< 18 | 19 | install: ${MANPAGES} 20 | $(INSTALL) -d $(DESTDIR)$(mandir)/man1 21 | ${INSTALL_DATA} ${MANPAGES} $(DESTDIR)$(mandir)/man1 22 | 23 | uninstall: 24 | cd $(DESTDIR)$(mandir)/man1 && rm -f ${MANPAGES} 25 | 26 | clean: 27 | rm -f *.1.gz 28 | 29 | distclean: clean 30 | rm -f Makefile 31 | -------------------------------------------------------------------------------- /util/start_mpds.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | #this script was used to initialize a ring of MPDs on teracluster 3 | #the mpd started scripts were too annoying 4 | #it expects a file 'hosts' to list nodes in the cluster 5 | #it takes one arg, the number of remote mpd's to start 6 | 7 | 8 | set -m 9 | 10 | MPD="dmtcp_launch $HOME/mpich2/bin/mpd " 11 | HOST="teracluster" 12 | PORT="7778" 13 | 14 | if test -z "$1" 15 | then 16 | echo "usage $0 N" 17 | exit 1 18 | fi 19 | 20 | #start local 21 | $MPD --listenport=7778 & 22 | sleep 1 23 | 24 | for X in `head -n $1 < hosts` 25 | do 26 | echo "Starting $X..." 27 | ssh $X "$MPD --host=$HOST --port=$PORT /dev/null 2>/dev/null"& 28 | done 29 | 30 | wait 31 | 32 | 33 | -------------------------------------------------------------------------------- /mpi-proxy-split/doc-old/2pcmodelv4/2pcmodelv4mc.cfg: -------------------------------------------------------------------------------- 1 | SPECIFICATION Spec 2 | CONSTANT defaultInitValue = defaultInitValue 3 | \* Add statements after this line. 4 | CONSTANT IS_READY = IS_READY 5 | CONSTANT PHASE_1 = PHASE_1 6 | CONSTANT IN_CS = IN_CS 7 | CONSTANT PHASE_2 = PHASE_2 8 | CONSTANT CKPTED = CKPTED 9 | CONSTANT IS_DONE = IS_DONE 10 | CONSTANT READY_FOR_CKPT = READY_FOR_CKPT 11 | CONSTANT NONE = NONE 12 | CONSTANT GET_STATUS = GET_STATUS 13 | CONSTANT FREE_PASS = FREE_PASS 14 | CONSTANT INTENT = INTENT 15 | CONSTANT CKPT = CKPT 16 | CONSTANT NO_RESPONSE = NO_RESPONSE 17 | CONSTANT UT_OFFSET <- UT_OFFSET_TMP 18 | CONSTANT BarrierList <- BarrierList_TMP 19 | CONSTANT Procs <- Procs_TMP 20 | CONSTANT CkptThreads <- CkptThreads_TMP 21 | CONSTANT COORD_PROC <- COORD_PROC_TMP 22 | -------------------------------------------------------------------------------- /mpi-proxy-split/mpi_files.h: -------------------------------------------------------------------------------- 1 | #ifndef MANA_FILES_H 2 | #define MANA_FILES_H 3 | 4 | #include 5 | #include 6 | 7 | typedef struct OpenFileParameters { 8 | // Initial file creation parameters 9 | MPI_Comm _comm; 10 | int _mode; 11 | MPI_Info _info; 12 | char _filepath[PATH_MAX]; 13 | 14 | // File characteristics set after creation 15 | int _atomicity; 16 | MPI_Offset _size; 17 | MPI_Offset _viewDisp; 18 | MPI_Datatype _viewEType; 19 | MPI_Datatype _viewFType; 20 | MPI_Info _viewInfo; 21 | int _viewSet; 22 | char _datarep[16]; // One of internal, native, or external32 23 | 24 | // File characteristics not necessarily explicitly set, but must be restored 25 | MPI_Offset _offset; 26 | 27 | } OpenFileParameters; 28 | 29 | #endif // MANA_FILES 30 | -------------------------------------------------------------------------------- /util/memory_access_tracker/Makefile: -------------------------------------------------------------------------------- 1 | 2 | all: lib test 3 | 4 | DEFINES := -D_GNU_SOURCE 5 | CFLAGS := -Wall 6 | 7 | DEBUG ?= 1 8 | ifeq ($(DEBUG), 1) 9 | DEFINES += -DDEBUG 10 | CFLAGS += -O0 -ggdb3 11 | else 12 | DEFINES += -DNDEBUG 13 | CFLAGS += -O2 -g 14 | endif 15 | 16 | memory_access_tracker.o: memory_access_tracker.c memory_access_tracker.h 17 | gcc $(DEFINES) $(CFLAGS) -fPIC -c -o $@ $< 18 | 19 | libmemtracker.so: memory_access_tracker.o 20 | gcc -o $@ -shared $^ 21 | 22 | libmemtracker.a: memory_access_tracker.o 23 | ar rcs $@ $^ 24 | 25 | lib: libmemtracker.so libmemtracker.a 26 | 27 | test_memory_tracker: test_memory_tracker.c libmemtracker.a 28 | gcc $(DEFINES) $(CFLAGS) -o $@ $^ 29 | 30 | test: test_memory_tracker 31 | 32 | clean: 33 | rm *.o libmemtracker.so libmemtracker.a test_memory_tracker 34 | -------------------------------------------------------------------------------- /mpi-proxy-split/doc-old/2pcmodelv4/Makefile: -------------------------------------------------------------------------------- 1 | # File with TLA+ model 2 | FILE=2pcmodelv4 3 | # File with model checking configuration 4 | MC=${FILE}mc 5 | # Path to TLA install dir 6 | TLA_PATH=/Users/rgarg/software/tla/tla/ 7 | # Path to Java 8 | JAVA=/usr/bin/java 9 | # Num. of worker threads for model checking 10 | TLC_OPTIONS=-workers 8 11 | 12 | default: build 13 | 14 | build: ${FILE}.tla 15 | CLASSPATH=${TLA_PATH} ${JAVA} pcal.trans $< 16 | 17 | vi vim: ${FILE}.tla 18 | vim $< 19 | 20 | run: build 21 | CLASSPATH=${TLA_PATH} ${JAVA} tlc2.TLC ${TLC_OPTIONS} ${MC} > error.trace 22 | 23 | clean: 24 | rm -rf states *.old 25 | 26 | tidy: 27 | rm -f error.trace 28 | 29 | dist: clean 30 | (dir=`basename $$PWD` && cd .. && tar zcvf $$dir.tgz $$dir) 31 | (dir=`basename $$PWD` && ls -l ../$$dir.tgz) 32 | 33 | .PHONY: default build run clean vi vim tidy dist 34 | -------------------------------------------------------------------------------- /util/readdmtcp.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | if test "$1" = ""; then 4 | echo 'Usage: readdmtcp.sh ' 5 | echo 'Example: util/readdmtcp.sh ckpt_dmtcp1_*.dmtcp' 6 | exit 0 7 | fi 8 | 9 | trap 'rm -f ckpt_tmp.dmtcp' INT QUIT EXIT 10 | 11 | if file $1 | grep gzip > /dev/null; then 12 | echo '***' $1 is a gzipped file. Will uncompress it into ckpt_tmp.dmcp first. 13 | gzip -dc $1 > ckpt_tmp.dmtcp 14 | set ckpt_tmp.dmtcp 15 | elif echo $1 | grep '.dmtcp$'; then 16 | true # Pass 17 | else 18 | echo '***' $1 is neither a gzipped file or a .dmtcp file. Exiting.; 19 | exit 1; 20 | fi 21 | 22 | dir=`dirname $0` 23 | 24 | if which mtcp_restart > /dev/null 2> /dev/null; then 25 | mtcp_restart --simulate $1 2>&1 26 | exit 0 27 | fi 28 | 29 | # This next one assumes that this script resides in DMTCP_ROOT/util/ 30 | if test -x $dir/../bin/mtcp_restart; then 31 | $dir/../bin/mtcp_restart --simulate $1 2>&1 32 | exit 0 33 | fi 34 | -------------------------------------------------------------------------------- /mpi-proxy-split/util/mpi-logger/README: -------------------------------------------------------------------------------- 1 | # MPI-LOGGER 2 | The purpose of mpi-logger is to intercept MPI calls from a native program 3 | without involving running MANA.
4 | In situations where the MPI application is close to deterministic 5 | (e.g., one rank, or multiple ranks communicating in a deterministic pattern), 6 | this tool helps find where the calls to MPI functions in MANA don't reflect 7 | the correct semantics.
8 | 9 | ## To build MPI-Logger 10 | To build, go to ./mpi-proxy-split/util/mpi-logger and run `make` 11 | 12 | ## To use MPI-Logger 13 | To run, use LD_PRELOAD to preload the generated .so file before you run. 14 | ``` 15 | Loaded with environment variables. 16 | 17 | srun env LD_PRELOAD=$PATH_TO_FILE/liblogger.so 18 | ``` 19 | 20 | ## To edit MPI-Logger 21 | To edit which MPI calls areintercepted or what content (e.g., which arguments) 22 | gets printed, edit ./mpi-proxy-split/util/mpi-logger/mpi_logger.cpp. -------------------------------------------------------------------------------- /mpi-proxy-split/doc-old/design.bib: -------------------------------------------------------------------------------- 1 | @phdthesis{caophdthesis, 2 | author = {Jiajun Cao}, 3 | title = {{Transparent Checkpointing over RDMA-based Networks}}, 4 | school = {Northeastern University}, 5 | year = 2017, 6 | doi = {10.17760/D20290419}, 7 | month = Dec, 8 | } 9 | 10 | @inproceedings{garg2018crum, 11 | title={{CRUM: Checkpoint-Restart Support for CUDA's Unified Memory}}, 12 | author={Garg, Rohan and Mohan, Apoorve and Sullivan, Michael and Cooperman, Gene}, 13 | booktitle={Proceedings of the 20th International Conference on Cluster Computing (CLUSTER)}, 14 | year={2018}, 15 | organization={IEEE} 16 | } 17 | 18 | @article{lamport1999tla, 19 | title={{Specifying Concurrent Systems with TLA\^{}+}}, 20 | author={Lamport, Leslie}, 21 | journal={NATO ASI SERIES F COMPUTER AND SYSTEMS SCIENCES}, 22 | volume={173}, 23 | pages={183--250}, 24 | year={1999}, 25 | publisher={SPRINGER VERLAG} 26 | } 27 | -------------------------------------------------------------------------------- /mpi-proxy-split/seq_num.h: -------------------------------------------------------------------------------- 1 | #ifndef SEQ_NUM_H 2 | #define SEQ_NUM_H 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | typedef enum _phase_t { 9 | IN_CS, 10 | IS_READY 11 | } phase_t; 12 | 13 | // Global communicator for MANA internal use 14 | extern MPI_Comm g_world_comm; 15 | 16 | // The main functions of the sequence number algorithm for MPI collectives 17 | void commit_begin(MPI_Comm comm); 18 | void commit_finish(MPI_Comm comm); 19 | 20 | void drain_mpi_collective(); 21 | void share_seq_nums(); 22 | int check_seq_nums(); 23 | void print_seq_nums(); 24 | void seq_num_init(); 25 | void seq_num_reset(); 26 | void seq_num_destroy(); 27 | 28 | extern std::unordered_map seq_num; 29 | extern std::unordered_map target; 30 | extern std::unordered_map ggid_table; 31 | typedef std::pair comm_seq_pair_t; 32 | 33 | #endif // ifndef SEQ_NUM_H 34 | -------------------------------------------------------------------------------- /util/fsgsbase-enabled.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | /* The user-space FSGSBASE patch was added to Linux kernel version 5.9. */ 6 | 7 | /* Will be eventually in asm/hwcap.h */ 8 | #ifndef HWCAP2_FSGSBASE 9 | # define HWCAP2_FSGSBASE (1 << 1) 10 | #endif 11 | 12 | int main() { 13 | unsigned val = getauxval(AT_HWCAP2); 14 | if (val & HWCAP2_FSGSBASE) { 15 | printf("FSGSBASE enabled (both in CPU and Linux kernel)\n"); 16 | return 0; 17 | } else { 18 | printf("FSGSBASE _NOT_ enabled\n"); 19 | return 1; 20 | } 21 | } 22 | 23 | /* If FSGSBASE is enabled, then the following user-space versions: 24 | * unsigned long int fsbase; 25 | * asm volatile("rex.W\n rdfsbase %0" : "=r" (fsbase) :: "memory"); 26 | * asm volatile("rex.W\n wrfsbase %0" :: "r" (fsbase) : "memory"); 27 | * can replace: 28 | * unsigned long int fsbase; 29 | * syscall(SYS_arch_prctl, ARCH_GET_FS, fsbase); 30 | * syscall(SYS_arch_prctl, ARCH_SET_FS, fsbase); 31 | */ 32 | -------------------------------------------------------------------------------- /mpi-proxy-split/cartesian.h: -------------------------------------------------------------------------------- 1 | #ifndef CARTESIAN_H 2 | #define CARTESIAN_H 3 | 4 | #define MAX_PROCESSES 100 5 | #define MAX_CART_PROP_SIZE 10 6 | 7 | // This struct is used to store mapping of , and 8 | // . 9 | typedef struct CartesianInfo { 10 | int comm_old_rank, comm_cart_rank; 11 | int coordinates[MAX_CART_PROP_SIZE]; 12 | } CartesianInfo; 13 | 14 | // This struct is used to store the cartesian communicator properties, i.e., the 15 | // arguments passed to the MPI_Cart_Create() API. 16 | // For reference, 17 | // int MPI_Cart_create(MPI_Comm comm_old, int ndims, const int dims[], 18 | // const int periods[], int reorder, MPI_Comm * comm_cart) 19 | typedef struct CartesianProperties { 20 | int comm_old_size; 21 | int comm_cart_size; 22 | int comm_old_rank; 23 | int comm_cart_rank; 24 | int coordinates[MAX_CART_PROP_SIZE]; 25 | int ndims; 26 | int dimensions[MAX_CART_PROP_SIZE]; 27 | int periods[MAX_CART_PROP_SIZE]; 28 | int reorder; 29 | } CartesianProperties; 30 | 31 | #endif // ifndef CARTESIAN_H 32 | -------------------------------------------------------------------------------- /mpi-proxy-split/mpi-wrappers/mpi_win_wrappers.cpp: -------------------------------------------------------------------------------- 1 | #include "mpi_plugin.h" 2 | #include "config.h" 3 | #include "dmtcp.h" 4 | #include "util.h" 5 | #include "jassert.h" 6 | #include "jfilesystem.h" 7 | #include "protectedfds.h" 8 | #include "mpi_nextfunc.h" 9 | 10 | extern "C" { 11 | 12 | #pragma weak MPI_Alloc_mem = PMPI_Alloc_mem 13 | int PMPI_Alloc_mem(MPI_Aint size, MPI_Info info, void *baseptr) 14 | { 15 | // Since memory allocated by the lower half will be discarded during 16 | // checkpoint, we need to translate MPI_Alloc_mem and MPI_Free_mem 17 | // to malloc and free. This may slow down the program. 18 | *(void**)baseptr = malloc(size * sizeof(MPI_Aint)); 19 | return MPI_SUCCESS; 20 | } 21 | 22 | #pragma weak MPI_Free_mem = PMPI_Free_mem 23 | int PMPI_Free_mem(void *baseptr) 24 | { 25 | // Since memory allocated by the lower half will be discarded during 26 | // checkpoint, we need to translate MPI_Alloc_mem and MPI_Free_mem 27 | // to malloc and free. This may slow down the program. 28 | free(baseptr); 29 | return MPI_SUCCESS; 30 | } 31 | 32 | } // end of: extern "C" 33 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/Initialized_test.c: -------------------------------------------------------------------------------- 1 | /* 2 | Test for the MPI_Initialized method 3 | 4 | Run with >1 ranks for non-trivial results 5 | Defaults to 10000 iterations 6 | Intended to be run with mana_test.py 7 | 8 | */ 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | int 16 | main(int argc, char *argv[]) 17 | { 18 | // Parse runtime argument 19 | int max_iterations = 10000; // default 20 | if (argc != 1) { 21 | max_iterations = atoi(argv[1]); 22 | } 23 | 24 | int flag, error; 25 | 26 | flag = 0; 27 | MPI_Initialized(&flag); 28 | if (flag) { 29 | printf("MPI_Initialized returned true before MPI_Init.\n"); 30 | assert(0); 31 | } 32 | 33 | MPI_Init(&argc, &argv); 34 | 35 | for (int iterations = 0; iterations < max_iterations; iterations++) { 36 | flag = 0; 37 | MPI_Initialized(&flag); 38 | if (!flag) { 39 | printf("MPI_Initialized returned false after MPI_Init.\n"); 40 | fflush(stdout); 41 | MPI_Abort(MPI_COMM_WORLD, error); 42 | assert(0); 43 | } 44 | } 45 | 46 | MPI_Finalize(); 47 | return 0; 48 | } 49 | -------------------------------------------------------------------------------- /mpi-proxy-split/unit-test/record-replay-group-test.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | 5 | #include "record-replay.h" 6 | #include "virtual-ids.h" 7 | 8 | #undef DMTCP_PLUGIN_ENABLE_CKPT 9 | #undef DMTCP_PLUGIN_DISABLE_CKPT 10 | #undef JUMP_TO_LOWER_HALF 11 | #undef RETURN_TO_UPPER_HALF 12 | 13 | using namespace dmtcp_mpi; 14 | 15 | class GroupTest : public ::testing::Test 16 | { 17 | protected: 18 | MPI_Comm _comm; 19 | 20 | void SetUp() override 21 | { 22 | int flag = 0; 23 | this->_comm = MPI_COMM_WORLD; 24 | if (MPI_Initialized(&flag) == MPI_SUCCESS && !flag) { 25 | MPI_Init(NULL, NULL); 26 | } 27 | } 28 | 29 | void TearDown() override 30 | { 31 | // MPI_Finalize(); 32 | CLEAR_LOG(); 33 | } 34 | }; 35 | 36 | TEST_F(GroupTest, testGroupAPI) 37 | { 38 | MPI_Group group = MPI_GROUP_NULL; 39 | EXPECT_EQ(MPI_Comm_group(_comm, &group), MPI_SUCCESS); 40 | EXPECT_NE(group, MPI_GROUP_NULL); 41 | int size = -1; 42 | EXPECT_EQ(MPI_Group_size(group, &size), MPI_SUCCESS); 43 | EXPECT_EQ(size, 1); 44 | } 45 | 46 | int 47 | main(int argc, char **argv) 48 | { 49 | initializeJalib(); 50 | ::testing::InitGoogleTest(&argc, argv); 51 | return RUN_ALL_TESTS(); 52 | } 53 | -------------------------------------------------------------------------------- /util/README: -------------------------------------------------------------------------------- 1 | This directory contains some useful scripts to work with DMTCP on clusters. 2 | This directory is becoming somewhat crowded, but some particularly useful 3 | utilities to highlight are: 4 | 5 | * readdmtcp.sh - read the memory map of a DMTCP checkpoint image (ckpt_*.dmtcp) 6 | * git-bisect.sh - a template for doing 'git bisect run ./git-bisect.sh' 7 | tests on the DMTCP revision history in git 8 | * gdb-add-symbol-file - After DMTCP restart, the debug symbol information 9 | for a library in memory may no longer be available, 10 | since the in-memory copy was restored rather than 11 | loaded from the library on disk. This allows you 12 | to recover debugging symbol information on that library. 13 | * attach_all.py - attach GDB to all ranks at the same time and run a GDB command 14 | file on every rank. One rank can also be skipped to be debugged 15 | manually. This is useful to debug issues that only arise on large 16 | numbers of ranks. 17 | [ Contributors: please add to this list, above. ] 18 | 19 | OLD TEXT: 20 | It also contains gdb-add-symbol-file. Execute it with no arguments 21 | for instructions on its usage. It loads library debugging symbols 22 | into gdb that might not otherwise be there, such as libc-2.7.so. 23 | 24 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/unsync_Finalize.c: -------------------------------------------------------------------------------- 1 | /* 2 | * FILE: unsync_Finalize.c 3 | * Description: This test triggers an *unsynchronized call* 4 | * to `MPI_Finalize` by intentionally delaying 5 | * the finalize call in rank 0. 6 | * If correct synchronization mechanism is not employed 7 | * in MPI_Finalize wrapper, it causes premature termination. 8 | */ 9 | 10 | #include 11 | #include 12 | #include 13 | 14 | int main(int argc, char* argv[]){ 15 | int rank, size; 16 | MPI_Init(&argc, &argv); 17 | 18 | // get rank and comm size 19 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 20 | MPI_Comm_size(MPI_COMM_WORLD, &size); 21 | 22 | // requires atleast 2 processes 23 | if (size < 2) { 24 | printf("ERROR: requires at least 2 processes!\n"); 25 | return 1; 26 | } 27 | 28 | printf("[%d/%d]: Says Hello!\n", rank, size); 29 | 30 | // wait for all the processes to say hello! 31 | MPI_Barrier(MPI_COMM_WORLD); 32 | 33 | // Rank 0: sleep for 4 sec before saying goodbye 34 | if(rank == 0){ 35 | printf("Rank 0 sleeping for 4 sec\n"); 36 | sleep(4); 37 | printf("Rank %d says: Goodbye!\n", rank); 38 | } 39 | else{ 40 | printf("Rank %d says: Goodbye!\n", rank); 41 | } 42 | 43 | MPI_Finalize(); 44 | } 45 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/Scan_test.c: -------------------------------------------------------------------------------- 1 | /* 2 | Test for the MPI_Scan method 3 | 4 | Run with >1 ranks for non-trivial results 5 | Defaults to 10000 iterations 6 | Intended to be run with mana_test.py 7 | */ 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | int 15 | main(int argc, char **argv) 16 | { 17 | // Parse runtime argument 18 | int max_iterations = 10000; // default 19 | if (argc != 1) { 20 | max_iterations = atoi(argv[1]); 21 | } 22 | 23 | int rank, size, recv; 24 | 25 | MPI_Init(&argc, &argv); 26 | MPI_Comm_size(MPI_COMM_WORLD, &size); 27 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 28 | 29 | if (rank == 0) { 30 | printf("Running test for %d iterations\n", max_iterations); 31 | } 32 | 33 | for (int iterations = 0; iterations < max_iterations; iterations++) { 34 | int ret = MPI_Scan(&rank, &recv, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); 35 | assert(ret == MPI_SUCCESS); 36 | int expected_sum = 0; 37 | for (int i = 0; i <= rank; i++) { 38 | expected_sum += i; 39 | } 40 | printf("[Rank %d] => recveived sum = %d, expected sum = %d\n", rank, recv, 41 | expected_sum); 42 | fflush(stdout); 43 | assert(recv == expected_sum); 44 | recv = 0; 45 | } 46 | 47 | MPI_Finalize(); 48 | } 49 | -------------------------------------------------------------------------------- /mpi-proxy-split/mpi-wrappers/p2p-deterministic.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import sys 3 | import os 4 | 5 | def macroize_line(line): 6 | if "//" in line: 7 | line = line.replace("//", "/*", 1) + " */" 8 | n = max(0, 60 - len(line)) 9 | return line + n*" " + " \\" 10 | 11 | def macroize(lines): 12 | in_macro = False 13 | out = ["// *** THIS FILE IS AUTO-GENERATED! DO 'make' TO UPDATE. ***", ""] 14 | lines = [line.rstrip() for line in lines] 15 | for line in lines: 16 | if not in_macro and line.startswith("#define ") and "(" in line: 17 | in_macro = True 18 | out.append(macroize_line(line)) 19 | elif not in_macro: 20 | out.append(line) 21 | elif in_macro and line.strip(): # if non-empty line of macro 22 | out.append(macroize_line(line)) 23 | elif in_macro and not line.strip(): # if empty line of macro: end macro 24 | out[-1] = out[-1][:-2].rstrip() # Strip final " \\" 25 | out.append("") 26 | in_macro = False; 27 | return "\n".join(out) 28 | 29 | if len(sys.argv) == 2 and os.path.isfile(sys.argv[1]) and \ 30 | os.access(sys.argv[1], os.R_OK): 31 | print(macroize(open(sys.argv[1]).readlines())) 32 | else: 33 | sys.stderr.write("macroize: Can't macroize; no such file or not readable\n" + 34 | "USAGE: macroize.py FILE\n") 35 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/Barrier_test.c: -------------------------------------------------------------------------------- 1 | /* 2 | Test for the MPI_Barrier method 3 | 4 | Run with >1 ranks for non-trivial results 5 | Defaults to 5 iterations 6 | Intended to be run with mana_test.py 7 | */ 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #define BUFFER_SIZE 100 17 | #define SLEEP_PER_ITERATION 1 18 | 19 | int 20 | main(int argc, char *argv[]) 21 | { 22 | // Parse runtime argument 23 | int max_iterations = 5; // default 24 | if (argc != 1) { 25 | max_iterations = atoi(argv[1]); 26 | } 27 | 28 | int rank, nprocs; 29 | MPI_Init(&argc, &argv); 30 | MPI_Comm_size(MPI_COMM_WORLD, &nprocs); 31 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 32 | 33 | if (rank == 0) { 34 | printf("Running test for %d iterations\n", max_iterations); 35 | } 36 | 37 | for (int iterations = 0; iterations < max_iterations; iterations++) { 38 | printf("Entering %d of %d\n", rank, nprocs); 39 | fflush(stdout); 40 | sleep(rank * SLEEP_PER_ITERATION); 41 | int ret = MPI_Barrier(MPI_COMM_WORLD); 42 | assert(ret == MPI_SUCCESS); 43 | printf("Everyone should be entered by now. If not then Test Failed!\n"); 44 | fflush(stdout); 45 | } 46 | 47 | MPI_Finalize(); 48 | return 0; 49 | } 50 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/Alloc_mem.c: -------------------------------------------------------------------------------- 1 | /* 2 | Test for the MPI_Alloc_mem method 3 | 4 | Defaults to 15 iterations 5 | Intended to be run with mana_test.py 6 | */ 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | #define CHAR_COUNT 100 16 | #define SLEEP_PER_ITERATION 2 17 | 18 | int 19 | main(int argc, char *argv[]) 20 | { 21 | // Parse runtime argument 22 | int max_iterations = 15; // default 23 | if (argc != 1) { 24 | max_iterations = atoi(argv[1]); 25 | } 26 | 27 | char *mem; 28 | int rank, ret; 29 | 30 | MPI_Init(&argc, &argv); 31 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 32 | 33 | if (rank == 0) { 34 | printf("Running test for %d iterations\n", max_iterations); 35 | } 36 | 37 | for (int iterations = 0; iterations < max_iterations; iterations++) { 38 | ret = MPI_Alloc_mem(CHAR_COUNT * sizeof(char), MPI_INFO_NULL, &mem); 39 | assert(ret == MPI_SUCCESS); 40 | printf("Rank %d allocated %d chars at %p\n", rank, CHAR_COUNT, &mem); 41 | fflush(stdout); 42 | 43 | sleep(SLEEP_PER_ITERATION); 44 | 45 | ret = MPI_Free_mem(mem); 46 | assert(ret == MPI_SUCCESS); 47 | printf("Rank %d freed %p\n", rank, &mem); 48 | fflush(stdout); 49 | } 50 | MPI_Finalize(); 51 | return 0; 52 | } 53 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # git ls-files --others --exclude-from=.git/info/exclude 2 | # Lines that start with '#' are comments. 3 | # For a project mostly in C, the following would be a good set of 4 | # exclude patterns (uncomment them if you want to use them): 5 | *.[oat] 6 | *~ 7 | *.swp 8 | *.swo 9 | *.log 10 | *.status 11 | *.so 12 | *.pyc 13 | *.dirstamp 14 | Makefile 15 | core 16 | cscope.* 17 | dmtcpaware* 18 | *aclocal.m4* 19 | 20 | autom4te.cache 21 | 22 | bin/ 23 | lib/ 24 | local/ 25 | include/config.h 26 | include/dmtcp.h 27 | include/stamp-h1 28 | dmtcp/src/dmtcp_checkpoint 29 | dmtcp/src/dmtcp_launch 30 | dmtcp/src/dmtcp_command 31 | dmtcp/src/dmtcp_coordinator 32 | dmtcp/src/dmtcp_nocheckpoint 33 | dmtcp/src/dmtcp_restart 34 | dmtcp/src/libdmtcpaware.so* 35 | plugin/ipc/dmtcp_ssh* 36 | dmtcp_discover_rm 37 | dmtcp_srun_helper 38 | 39 | mtcp/libmtcp.so* 40 | mtcp/mtcp.lis 41 | mtcp/mtcp.map 42 | mtcp/mtcp_restart 43 | mtcp/readmtcp 44 | mtcp/mtcp.t.orig 45 | mtcp/testmtcp6 46 | mtcp/util-arch 47 | 48 | test/* 49 | !*.{c,h,cpp,java,py,in} 50 | test/testconfig.py 51 | 52 | *.deps* 53 | 54 | ckpt_*.dmtcp 55 | dmtcp_restart_script* 56 | 57 | GPATH 58 | GRTAGS 59 | GTAGS 60 | 61 | .idea/ 62 | build/ 63 | 64 | mpi-proxy-split/mpi-wrappers/p2p-deterministic.h 65 | mpi-proxy-split/mpi-wrappers/mana_p2p_update_logs 66 | restart_plugin/lower_half_api.h 67 | manpages/mana.1.gz 68 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/file_test.c: -------------------------------------------------------------------------------- 1 | /* 2 | Test for the MPI_File_open and MPI_File_close methods 3 | 4 | Defaults to 10000 iterations 5 | Intended to be run with mana_test.py 6 | 7 | Source: https://www.rookiehpc.com/mpi/docs/mpi_file_open.php 8 | */ 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | int 18 | main(int argc, char *argv[]) 19 | { 20 | // Parse runtime argument 21 | int max_iterations = 10000; // default 22 | if (argc != 1) { 23 | max_iterations = atoi(argv[1]); 24 | } 25 | 26 | int access_mode; 27 | MPI_Comm comm; 28 | 29 | MPI_Init(&argc, &argv); 30 | comm = MPI_COMM_WORLD; 31 | 32 | MPI_File handle; 33 | access_mode = MPI_MODE_CREATE | MPI_MODE_EXCL | MPI_MODE_RDWR | 34 | MPI_MODE_UNIQUE_OPEN | MPI_MODE_DELETE_ON_CLOSE; 35 | 36 | for (int iterations = 0; iterations < max_iterations; iterations++) { 37 | int ret; 38 | ret = MPI_File_open(comm, "file.tmp", access_mode, MPI_INFO_NULL, &handle); 39 | assert(ret == MPI_SUCCESS); 40 | printf("File opened successfully.\n"); 41 | fflush(stdout); 42 | 43 | ret = MPI_File_close(&handle); 44 | assert(ret == MPI_SUCCESS); 45 | printf("File closed successfully.\n"); 46 | } 47 | 48 | MPI_Finalize(); 49 | return 0; 50 | } 51 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/MPI_Double_Int_test.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | /* 5 | * MPICH treats MPI_DOUBLE_INT as a negative 32-bit integer, 6 | * signifying its 'struct' nature. 7 | * MANA has defined look-up table for pre-defined MPI_Datatypes as 8 | * map to accomodate for OpenMPI library. 9 | * 10 | * But, When handeling negative 64-bit value in MPICh with int32_t MPI_Datatypes, 11 | * conversion back to 64-bit results in loss of upper-bits, violating integrity of argument passed. 12 | * 13 | * This tests for a possible regression. 14 | */ 15 | 16 | int main(int argc, char** argv) { 17 | MPI_Init(&argc, &argv); 18 | 19 | int rank, size; 20 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 21 | MPI_Comm_size(MPI_COMM_WORLD, &size); 22 | 23 | struct {double value; int rank;} local_data, global_data; 24 | 25 | // assign a value (eg: some computational result) 26 | local_data.value = rank * 1.5; // different value for each rank 27 | local_data.rank = rank; 28 | /* 29 | volatile int dummy = 1; 30 | printf("Dummy Hit!\n"); 31 | while(dummy); 32 | */ 33 | MPI_Allreduce(&local_data, &global_data, 1, MPI_DOUBLE_INT, MPI_MAXLOC, MPI_COMM_WORLD); 34 | 35 | // print result 36 | printf("[Rank %d]: Max value = %f from rank %d\n", rank, global_data.value, global_data.rank); 37 | 38 | MPI_Finalize(); 39 | return 0; 40 | 41 | } 42 | 43 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/Gather_test.c: -------------------------------------------------------------------------------- 1 | /* 2 | Test for the MPI_Gather method 3 | 4 | Must run with 3 ranks 5 | Defaults to 10000 iterations 6 | Intended to be run with mana_test.py 7 | */ 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | int 15 | main(int argc, char **argv) 16 | { 17 | // Parse runtime argument 18 | int max_iterations = 10000; // default 19 | if (argc != 1) { 20 | max_iterations = atoi(argv[1]); 21 | } 22 | 23 | int send, recv[3]; 24 | int rank, size; 25 | 26 | MPI_Init(&argc, &argv); 27 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 28 | MPI_Comm_size(MPI_COMM_WORLD, &size); 29 | if (size != 3 && rank == 0) { 30 | printf("please run with 3 processes!\n"); 31 | exit(1); 32 | } 33 | if (rank == 0) { 34 | printf("Running test for %d iterations\n", max_iterations); 35 | } 36 | 37 | send = rank + 1; 38 | for (int iterations = 0; iterations < max_iterations; iterations++) { 39 | int ret = 40 | MPI_Gather(&send, 1, MPI_INT, &recv, 1, MPI_INT, 0, MPI_COMM_WORLD); 41 | assert(ret == MPI_SUCCESS); 42 | 43 | if (rank == 0) { 44 | printf("recv = %d %d %d\n", recv[0], recv[1], recv[2]); 45 | for (int i = 0; i < 3; i++) { 46 | assert(recv[i] == i + 1); 47 | } 48 | } 49 | fflush(stdout); 50 | for (int i = 0; i < 3; i++) { 51 | recv[i] = 0; 52 | } 53 | } 54 | 55 | MPI_Finalize(); 56 | return 0; 57 | } 58 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/Scatter_test.c: -------------------------------------------------------------------------------- 1 | /* 2 | Test for the MPI_Scatter method 3 | 4 | Must run with 3 ranks 5 | Defaults to 10000 iterations 6 | Intended to be run with mana_test.py 7 | */ 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | int 15 | main(int argc, char **argv) 16 | { 17 | // Parse runtime argument 18 | int max_iterations = 10000; // default 19 | if (argc != 1) { 20 | max_iterations = atoi(argv[1]); 21 | } 22 | 23 | int send[3], recv; 24 | int rank, size, i; 25 | 26 | MPI_Init(&argc, &argv); 27 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 28 | MPI_Comm_size(MPI_COMM_WORLD, &size); 29 | if (size != 3) { 30 | printf("Run with 3 processes\n"); 31 | exit(1); 32 | } 33 | if (rank == 0) { 34 | printf("Running test for %d iterations\n", max_iterations); 35 | } 36 | 37 | if (rank == 0) { 38 | for (i = 0; i < size; i++) { 39 | send[i] = i + 1; 40 | } 41 | } 42 | 43 | for (int iterations = 0; iterations < max_iterations; iterations++) { 44 | int ret = 45 | MPI_Scatter(&send, 1, MPI_INT, &recv, 1, MPI_INT, 0, MPI_COMM_WORLD); 46 | assert(ret == MPI_SUCCESS); 47 | 48 | printf("rank = %d \t recv = %d\n", rank, recv); 49 | fflush(stdout); 50 | assert(recv == rank + 1); 51 | MPI_Finalize(); 52 | 53 | if (rank == 0) { 54 | for (i = 0; i < size; i++) { 55 | send[i] = i + 1; 56 | } 57 | } 58 | recv = 0; 59 | } 60 | return 0; 61 | } 62 | -------------------------------------------------------------------------------- /util/hooks/pre-commit: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # A hook script to verify what is about to be committed. 4 | # Called by "git commit" with no arguments. The hook exits with 5 | # non-zero status if the files being committed do not conform to 6 | # the DMTCP style. 7 | # 8 | # To enable this hook, do this from the root of the repo: 9 | # 10 | # $ ln -s ../../util/hooks/pre-commit .git/hooks/pre-commit 11 | 12 | if git rev-parse --verify HEAD >/dev/null 2>&1 13 | then 14 | against=HEAD 15 | else 16 | # Initial commit: diff against an empty tree object 17 | against=f04e29e5f295620e25db4a8fc26870733f039081 18 | fi 19 | 20 | # Redirect output to stderr. 21 | exec 1>&2 22 | 23 | # If there are whitespace errors, print the offending file names and fail. 24 | git diff-index --check --cached $against -- || exit 1 25 | 26 | # Check DMTCP style. 27 | ADDED_OR_MODIFIED=$(git diff --cached --name-only --diff-filter=AM) 28 | if [ -n "$ADDED_OR_MODIFIED" ]; then 29 | # NOTE: We choose to implement this as a conditional rather than a call to 30 | # `xargs` on purpose. Some implementations of `xargs` will call your script 31 | # even if the arguments you pass in are empty. In our case, this would 32 | # cause us to erroneously lint every file in the repository. Additionally, 33 | # many implementations do not support the `-r` flag, (which instructs 34 | # `xargs` to not run the script if the arguments are empty), so we also 35 | # cannot use that. 36 | ./util/dmtcp-style.py $ADDED_OR_MODIFIED || exit 1 37 | fi 38 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/MANA_MPI_Initialized_test.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | /* 6 | * MANA Initializes MPI in the lower half, even before 7 | * user applications calls for MPI_Init in its code. 8 | * Therefore, a call to MPI_Initialized in user-applciation 9 | * before calling MPI_Init will return 'TRUE' 10 | * (which is incorrect behavior since we want MANA to be 11 | * an invisible layer between MPI-library and user-applciation). 12 | * 13 | * Therefore, we have added a patch to mitigate this and 14 | * this test is created to detect future regression. 15 | * 16 | */ 17 | int main(int argc, char *argv[]) { 18 | int flag; 19 | int rank, size; 20 | 21 | // check if MPI is initialized before MPI_Init 22 | MPI_Initialized(&flag); 23 | if (flag) { 24 | fprintf(stderr, "ERROR: MPI should not be initialized before MPI_Init!\n"); 25 | exit(EXIT_FAILURE); 26 | } 27 | 28 | // Initialize MPI env 29 | MPI_Init(NULL, NULL); 30 | 31 | // check if MPI is initialized after MPI_Init 32 | MPI_Initialized(&flag); 33 | if (!flag) { 34 | fprintf(stderr, "ERROR: MPI should be initialized after MPI_Init!\n"); 35 | exit(EXIT_FAILURE); 36 | } 37 | 38 | // making MPI-function call to conduct operational sanity test 39 | MPI_Comm_size(MPI_COMM_WORLD, &size); 40 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 41 | printf("[%d/%d]: Hello World!\n", rank, size); 42 | fflush(stdout); 43 | 44 | // Finalize MPI env 45 | MPI_Finalize(); 46 | 47 | return 0; 48 | 49 | } 50 | 51 | -------------------------------------------------------------------------------- /mpi-proxy-split/lower-half/README: -------------------------------------------------------------------------------- 1 | This lower-half program has the capability of: 2 | 1. Linking to an MPI library and initializing it 3 | 2. Executing a "kernel-laoder" that will load the upp-half program (the user 4 | MPI program containing the MPI application code). 5 | 3. Restoring the upper-half program at the time of restart. 6 | 7 | During launch, it (1) calls MPI_Init to initialize the MPI library in 8 | the lower-half. Then (2) writes the lh_info address to an environment 9 | variable MANA_LH_INFO_ADDR so that the upper-half program (including DMTCP 10 | and MANA's wrappers) knows information of the lower-half program. After 11 | setting up, the lower-half program (3) execute a "kernel-loader" to load 12 | the upper-half program. 13 | 14 | When an MPI function is called in the upper-half, the MANA MPI wrapper 15 | functions in the upper-half will redirect the call to the real MPI 16 | function linked with the lower-half program. 17 | 18 | During restart, it (1) calls MPI_Init to get its global MPI rank number 19 | in order to find the corresponding checkpoint image file. Then (2) 20 | connects to the DMTCP coordinator, and (3) uses DMTCP APIs to restore 21 | the upper-half program. During restart, the lh_info address is shared 22 | using an external file because environment variables inthe upper-half 23 | are inheritated from the checkpoint time, instead of from the lower-half 24 | program. These temp files will be saved in the /tmp/ directory, and 25 | cleaned after the upper-half program is properly restored. The filename 26 | format is "mana_tmp_lh_info_[hostname]_[pid]". 27 | -------------------------------------------------------------------------------- /mpi-proxy-split/Makefile_config.in: -------------------------------------------------------------------------------- 1 | CFLAGS = @CFLAGS@ 2 | CXXFLAGS = @CXXFLAGS@ 3 | FFLAGS = ${CXXFLAGS} @FFLAGS@ 4 | 5 | PLATFORM=${shell echo $$HOST} 6 | 7 | # Mark the platform as Perlmutter for a compute node. 8 | ifeq ($(findstring nid0,$(PLATFORM)),nid0) 9 | IS_PERLMUTTER = 1 10 | endif 11 | # Perlmutter's login node starts with login keyword 12 | ifeq ($(findstring login,$(PLATFORM)),login) 13 | IS_PERLMUTTER = 1 14 | endif 15 | 16 | ifeq (${IS_PERLMUTTER}, 1) 17 | MPICC = cc 18 | MPICXX = CC -std=c++14 19 | MPIFORTRAN = ftn 20 | MPI_LD_FLAG = -lmpich 21 | MPIRUN = srun 22 | MPI_CFLAGS?= @CFLAGS@ -g3 -fPIC 23 | MPI_CXXFLAGS?= @CXXFLAGS@ -g3 -fPIC 24 | MPI_LDFLAGS?= -dynamic 25 | # This is needed, or else mana_coordinator.o becomes the default target. 26 | default2: default 27 | #mana_coordinator.o: mana_coordinator.cpp 28 | # ${CXX} ${CXXFLAGS} `CC --cray-print-opts=cflags` -c -o $@ $< 29 | ## On Perlmutter, the generic settings of 'else' seem to work. 30 | else 31 | MPICC = mpicc 32 | MPICXX = mpic++ -std=c++14 33 | MPIFORTRAN = mpifort 34 | MPIRUN = mpirun 35 | # FIXME: The following paths are hardwired for non-NERSC platforms on which 36 | # MPICH and other extra libraries are installed from source. 37 | # Eventually, the LD_FLAGS for MPI should be configured in the 38 | # `../configure-mana` script. 39 | MPI_LD_FLAG = -L$$HOME/mpich-static/usr/lib64 -lmpi -L$$HOME/local_install/lib -llzma -lz -lm -lxml2 40 | MPI_CFLAGS?= @CFLAGS@ -g3 -fPIC 41 | MPI_CXXFLAGS?= @CXXFLAGS@ -g3 -fPIC 42 | MPI_FFLAGS = @FFLAGS@ -g3 43 | MPI_LDFLAGS?= 44 | endif 45 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/hello_mpi_init_thread.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | int 7 | main(int argc, char *argv[]) 8 | { 9 | int errs = 0; 10 | int provided, flag, claimed; 11 | 12 | int ret = MPI_Init_thread(0, 0, MPI_THREAD_MULTIPLE, &provided); 13 | if (ret != MPI_SUCCESS) { 14 | printf("MPI_Init_thread failed\n"); 15 | exit(1); 16 | } 17 | 18 | // FIXME: MPI_Is_thread_main is not implemented in wrapper 19 | /* MPI_Is_thread_main(&flag); 20 | * if (!flag) { 21 | * errs++; 22 | * printf("This thread called init_thread but Is_thread_main gave false\n"); 23 | * fflush(stdout); 24 | * } 25 | * MPI_Query_thread(&claimed); 26 | * if (claimed != provided) { 27 | * errs++; 28 | * printf("Query thread gave thread level %d but Init_thread gave %d\n", 29 | * claimed, provided ); 30 | * fflush(stdout); 31 | * } 32 | */ 33 | 34 | int rank, nprocs; 35 | MPI_Comm_size(MPI_COMM_WORLD, &nprocs); 36 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 37 | printf("Hello, world. I am %d of %d\n", rank, nprocs); 38 | fflush(stdout); 39 | printf("Will now sleep for 500 seconds ...\n"); 40 | fflush(stdout); 41 | unsigned int remaining = 300; 42 | while (remaining > 0) { 43 | remaining = sleep(remaining); 44 | if (remaining > 0) { 45 | printf("Signal received; continuing sleep for %d seconds.\n", remaining); 46 | fflush(stdout); 47 | } 48 | } 49 | printf("**** %s is now exiting.\n", argv[0]); 50 | 51 | MPI_Finalize(); 52 | return errs; 53 | } 54 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/Cart_map_test.c: -------------------------------------------------------------------------------- 1 | /* 2 | Test for the MPI_Cart_map method 3 | 4 | Run with >1 ranks for non-trivial results 5 | Defaults to 10000 iterations 6 | Intended to be run with mana_test.py 7 | */ 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | int 15 | main(int argc, char *argv[]) 16 | { 17 | // Parse runtime argument 18 | int max_iterations = 10000; // default 19 | if (argc != 1) { 20 | max_iterations = atoi(argv[1]); 21 | } 22 | 23 | int errs = 0; 24 | int dims[2]; 25 | int periods[2]; 26 | int size, rank, newrank; 27 | 28 | MPI_Init(&argc, &argv); 29 | MPI_Comm_size(MPI_COMM_WORLD, &size); 30 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 31 | 32 | if (rank == 0) { 33 | printf("Running test for %d iterations\n", max_iterations); 34 | } 35 | 36 | /* This defines a cartision grid with a single point */ 37 | periods[0] = 1; 38 | dims[0] = 1; 39 | for (int iterations = 0; iterations < max_iterations; iterations++) { 40 | int ret = MPI_Cart_map(MPI_COMM_WORLD, 1, dims, periods, &newrank); 41 | assert(ret == MPI_SUCCESS); 42 | if (rank > 0) { 43 | if (newrank != MPI_UNDEFINED) { 44 | errs++; 45 | printf("rank outside of input communicator not UNDEFINED\n"); 46 | fflush(stdout); 47 | } 48 | } else { 49 | if (rank != newrank) { 50 | errs++; 51 | printf("Newrank not defined and should be 0\n"); 52 | fflush(stdout); 53 | } 54 | } 55 | } 56 | MPI_Finalize(); 57 | assert(errs == 0); 58 | return 0; 59 | } 60 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/send_recv.c: -------------------------------------------------------------------------------- 1 | // Author: Wes Kendall 2 | // Copyright 2011 www.mpitutorial.com 3 | // This code is provided freely with the tutorials on mpitutorial.com. Feel 4 | // free to modify it for your own use. Any distribution of the code must 5 | // either provide a link to www.mpitutorial.com or keep this header in tact. 6 | // 7 | // MPI_Send, MPI_Recv example. Communicates the number -1 from process 0 8 | // to processe 1. 9 | // 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | int 17 | main(int argc, char **argv) 18 | { 19 | // Initialize the MPI environment 20 | MPI_Init(NULL, NULL); 21 | // Find out rank, size 22 | int world_rank; 23 | MPI_Comm_rank(MPI_COMM_WORLD, &world_rank); 24 | int world_size; 25 | MPI_Comm_size(MPI_COMM_WORLD, &world_size); 26 | 27 | // We are assuming at least 2 processes for this task 28 | if (world_size < 2) { 29 | fprintf(stderr, "World size must be greater than 1 for %s\n", argv[0]); 30 | MPI_Abort(MPI_COMM_WORLD, 1); 31 | } 32 | 33 | int number; 34 | if (world_rank == 0) { 35 | // If we are rank 0, set the number to -1 and send it to process 1 36 | number = 3; 37 | sleep(10); 38 | MPI_Send(&number, 1, MPI_INT, 1, 0, MPI_COMM_WORLD); 39 | } else if (world_rank == 1) { 40 | MPI_Recv(&number, 1, MPI_INT, 0, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); 41 | printf("Process 1 received number %d from process 0; Expected number = 3\n", 42 | number); 43 | fflush(stdout); 44 | assert(number == 3); 45 | } 46 | MPI_Finalize(); 47 | } 48 | -------------------------------------------------------------------------------- /util/hooks/post-rewrite: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # A hook script to verify what is about to be committed. 4 | # Called by "git commit --amend" or "git rebase". The hook exits with 5 | # non-zero status and warning messages if the files being rewritten do 6 | # not conform to the DMTCP style. 7 | # 8 | # To enable this hook, do this from the root of the repo: 9 | # 10 | # $ ln -s ../../util/hooks/post_rewrite .git/hooks/post_rewrite 11 | 12 | # Redirect output to stderr. 13 | exec 1>&2 14 | 15 | # If there are whitespace errors, print the offending file names. 16 | ## In git, '@~' represent previous commit. We check the whitespace between 17 | ## current head and previous commit after a commit is rewritten. 18 | git diff-index --check @~ -- || exit 1 19 | 20 | # Check DMTCP style. 21 | ## In git, '@' represent current head, '@~' represent previous commit. We check 22 | ## the style of changes between current head and previous commit after a commit 23 | ## is rewritten. 24 | ADDED_OR_MODIFIED=$(git diff --name-only --diff-filter=AM @~..@) 25 | if [ "$ADDED_OR_MODIFIED" ]; then 26 | # NOTE: We choose to implement this as a conditional rather than a call to 27 | # `xargs` on purpose. Some implementations of `xargs` will call your script 28 | # even if the arguments you pass in are empty. In our case, this would 29 | # cause us to erroneously lint every file in the repository. Additionally, 30 | # many implementations do not support the `-r` flag, (which instructs 31 | # `xargs` to not run the script if the arguments are empty), so we also 32 | # cannot use that. 33 | ./util/dmtcp-style.py $ADDED_OR_MODIFIED || exit 1 34 | fi 35 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/Ibarrier_test.c: -------------------------------------------------------------------------------- 1 | /* 2 | Test for the Ibarrier_test method 3 | 4 | Run with >2 ranks for non-trivial results 5 | Defaults to 5 iterations 6 | Intended to be run with mana_test.py 7 | 8 | */ 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #define SLEEP_PER_ITERATION 5 18 | 19 | #define MPI_TEST 20 | #ifndef MPI_TEST 21 | #define MPI_WAIT 22 | #endif 23 | 24 | int 25 | main(int argc, char *argv[]) 26 | { 27 | // Parse runtime argument 28 | int max_iterations = 5; // default 29 | if (argc != 1) { 30 | max_iterations = atoi(argv[1]); 31 | } 32 | 33 | int i, iter, myid, numprocs, flag; 34 | int iterations; 35 | MPI_Status status; 36 | MPI_Request request = MPI_REQUEST_NULL; 37 | 38 | MPI_Init(&argc, &argv); 39 | MPI_Comm_size(MPI_COMM_WORLD, &numprocs); 40 | MPI_Comm_rank(MPI_COMM_WORLD, &myid); 41 | 42 | if (myid == 0) { 43 | printf("Running test for %d iterations\n", max_iterations); 44 | } 45 | 46 | for (int iterations = 0; iterations < max_iterations; iterations++) { 47 | int ret = MPI_Ibarrier(MPI_COMM_WORLD, &request); 48 | assert(ret == MPI_SUCCESS); 49 | ret = MPI_Test(&request, &flag, &status); 50 | assert(ret == MPI_SUCCESS); 51 | #ifdef MPI_TEST 52 | while (1) { 53 | int flag = 0; 54 | MPI_Test(&request, &flag, &status); 55 | if (flag) { 56 | break; 57 | } 58 | } 59 | #endif 60 | #ifdef MPI_WAIT 61 | MPI_Wait(&request, &status); 62 | #endif 63 | 64 | sleep(SLEEP_PER_ITERATION); 65 | } 66 | MPI_Finalize(); 67 | } 68 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/File_size_test.c: -------------------------------------------------------------------------------- 1 | /* 2 | Test for the MPI_File_set_size and MPI_File_get_size methods 3 | 4 | Defaults to 10 iterations 5 | Intended to be run with mana_test.py 6 | 7 | */ 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #define BUF_SIZE 4096 17 | 18 | int 19 | main(int argc, char *argv[]) 20 | { 21 | // Parse runtime argument 22 | int max_iterations = 10; // default 23 | if (argc != 1) { 24 | max_iterations = atoi(argv[1]); 25 | } 26 | 27 | int access_mode, ret; 28 | MPI_Comm comm; 29 | 30 | MPI_Init(&argc, &argv); 31 | comm = MPI_COMM_WORLD; 32 | 33 | // Open file 34 | MPI_File handle; 35 | access_mode = MPI_MODE_CREATE | MPI_MODE_RDWR | MPI_MODE_DELETE_ON_CLOSE; 36 | ret = MPI_File_open(comm, "file.tmp", access_mode, MPI_INFO_NULL, &handle); 37 | assert(ret == MPI_SUCCESS); 38 | 39 | // Initialize buffers and other local variables 40 | MPI_Offset size = 0; 41 | 42 | for (int iterations = 0; iterations < max_iterations; iterations++) { 43 | // Set size either larger or smaller than current size 44 | ret = MPI_File_set_size(handle, BUF_SIZE / ((iterations & 1) + 1)); 45 | assert(ret == MPI_SUCCESS); 46 | 47 | sleep(5); 48 | 49 | // Read back current size 50 | MPI_File_get_size(handle, &size); 51 | assert(size == BUF_SIZE / ((iterations & 1) + 1)); 52 | 53 | fprintf(stderr, "Iteration %d complete\n", iterations); 54 | fflush(stderr); 55 | } 56 | ret = MPI_File_close(&handle); 57 | assert(ret == MPI_SUCCESS); 58 | 59 | MPI_Finalize(); 60 | return 0; 61 | } 62 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## MANA (MPI-Agnostic, Network-Agnostic MPI) 2 | 3 | MANA is an implementation of transparent checkpointing for MPI. It is 4 | built as a plugin on top of [DMTCP](https://github.com/dmtcp/dmtcp). 5 | 6 | For details of installing and using MANA, please see: 7 | - [MANA documentation (https://mana-doc.readthedocs.io/en/latest/) ](https://mana-doc.readthedocs.io/en/latest/) 8 | - [the MANA 'man' page](manpages/mana.1.md) (or `man ./mana.1` on a local copy) 9 | 10 | As seen in the MANA documentation, for install and a quick start, do: 11 | 12 | git clone https://github.com/mpickpt/mana 13 | cd mana 14 | git submodule init 15 | git submodule update 16 | ./configure 17 | make -j8 18 | # Testing on a single host (but see MANA docs for running on a cluster): 19 | PATH_TO_MANA/bin/mana_coordinator 20 | PATH_TO_MANA/bin/mana_launch [mana options] [user_program and args] 21 | 22 | --- 23 | 24 | To cite this project in a publicatoin, please cite: 25 | 26 | * "MANA for MPI: MPI-Agnostic Network-Agnostic Transparent Checkpointing", 27 | Rohan Garg, Gregory Price, and Gene Cooperman, HPDC'19. 28 | (original academic prototype) 29 | 30 | For other MANA publications with updates to this project, see: 31 | * "Enabling Practical Transparent Checkpointing for MPI: A Topological 32 | Sort Approach", 33 | Yao Xu and Gene Cooperman, 34 | IEEE International Conference on Cluster Computing (Cluster'24) 35 | 36 | * "Implementation-Oblivious Transparent Checkpoint-Restart for MPI", 37 | Yao Xu, Leonid Belyaev, Twinkle Jain, 38 | Derek Schafer, Anthony Skjellum, Gene Cooperman, SuperCheck-SC23 Workshop 39 | at SC'23. 40 | (near production version of MANA) 41 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/Cartdim_get_test.c: -------------------------------------------------------------------------------- 1 | /* 2 | Test for the MPI_Cart_dim method 3 | 4 | Must run with 12 ranks 5 | Defaults to 10000 iterations 6 | Intended to be run with mana_test.py 7 | 8 | Source: http://mpi.deino.net/mpi_functions/MPI_Cart_coords.html 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | /* A two-dimensional torus of 12 processes in a 4x3 grid */ 16 | int 17 | main(int argc, char *argv[]) 18 | { 19 | // Parse runtime argument 20 | int max_iterations = 10000; // default 21 | if (argc != 1) { 22 | max_iterations = atoi(argv[1]); 23 | } 24 | 25 | int rank, size; 26 | MPI_Comm comm; 27 | int dim[2], period[2], reorder; 28 | int coord[2], id; 29 | 30 | MPI_Init(&argc, &argv); 31 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 32 | MPI_Comm_size(MPI_COMM_WORLD, &size); 33 | if (size != 12) { 34 | printf("Please run with 12 processes.\n"); 35 | fflush(stdout); 36 | MPI_Abort(MPI_COMM_WORLD, 1); 37 | } 38 | 39 | if (rank == 0) { 40 | printf("Running test for %d iterations\n", max_iterations); 41 | } 42 | 43 | dim[0] = 4; 44 | dim[1] = 3; 45 | period[0] = 1; 46 | period[1] = 0; 47 | reorder = 1; 48 | int ret = MPI_Cart_create(MPI_COMM_WORLD, 2, dim, period, reorder, &comm); 49 | assert(ret == MPI_SUCCESS); 50 | for (int iterations = 0; iterations < max_iterations; iterations++) { 51 | int ndims = 0; 52 | ret = MPI_Cartdim_get(comm, &ndims); 53 | assert(ret == MPI_SUCCESS); 54 | if (rank == 0) { 55 | printf("number of dimensions = %d\n", ndims); 56 | fflush(stdout); 57 | } 58 | assert(ndims == 2); 59 | } 60 | 61 | MPI_Finalize(); 62 | return 0; 63 | } 64 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/send_recv_many.c: -------------------------------------------------------------------------------- 1 | // Author: Wes Kendall 2 | // Copyright 2011 www.mpitutorial.com 3 | // This code is provided freely with the tutorials on mpitutorial.com. Feel 4 | // free to modify it for your own use. Any distribution of the code must 5 | // either provide a link to www.mpitutorial.com or keep this header in tact. 6 | // 7 | // MPI_Send, MPI_Recv example. Communicates the number -1 from process 0 8 | // to processe 1. 9 | // 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | int 17 | main(int argc, char **argv) 18 | { 19 | // Initialize the MPI environment 20 | MPI_Init(NULL, NULL); 21 | // Find out rank, size 22 | int world_rank; 23 | MPI_Comm_rank(MPI_COMM_WORLD, &world_rank); 24 | int world_size; 25 | MPI_Comm_size(MPI_COMM_WORLD, &world_size); 26 | 27 | // We are assuming at least 2 processes for this task 28 | if (world_size < 2) { 29 | fprintf(stderr, "World size must be greater than 1 for %s\n", argv[0]); 30 | MPI_Abort(MPI_COMM_WORLD, 1); 31 | } 32 | 33 | int rank = 0; 34 | int number = 11223344; 35 | for (rank = 0; rank < world_size; rank++) { 36 | if (rank == world_rank) 37 | continue; 38 | MPI_Send(&number, 1, MPI_INT, rank, 0, MPI_COMM_WORLD); 39 | } 40 | printf("%d sleeping\n", world_rank); 41 | fflush(stdout); 42 | sleep(5); 43 | for (rank = 0; rank < world_size; rank++) { 44 | if (rank == world_rank) 45 | continue; 46 | MPI_Recv(&number, 1, MPI_INT, rank, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); 47 | assert(number == 11223344); 48 | printf("%d received %d from %d\n", world_rank, number, rank); 49 | fflush(stdout); 50 | } 51 | MPI_Finalize(); 52 | } 53 | -------------------------------------------------------------------------------- /mpi-proxy-split/test-ckpt-restart/Makefile: -------------------------------------------------------------------------------- 1 | include ../Makefile_config 2 | 3 | ifndef DMTCP_ROOT 4 | DMTCP_ROOT=../../dmtcp/ 5 | endif 6 | 7 | DMTCP_INCLUDE=${DMTCP_ROOT}/include 8 | LIBNAME=libdmtcp_mpi-proxy 9 | DEMO_PORT=7787 10 | 11 | # FILES=mpi_hello_world Abort_test Allreduce_test Alltoall_test \ 12 | # Alltoallv_test Barrier_test Bcast_test Comm_split_test Reduce_test \ 13 | # send_recv send_recv_many ping_pong Comm_dup_test Sendrecv_test \ 14 | # Waitall_test Allgather_test Group_size_rank Type_commit_contiguous 15 | FILES=ping_pong 16 | 17 | OBJS=$(addsuffix .o, ${FILES}) 18 | 19 | TESTS_DUMMY=$(addsuffix .dummy.exe, ${FILES}) 20 | TESTS=$(addsuffix .exe, ${FILES}) 21 | LDFLAGS_DUMMY=-L../mpi-wrappers -lmpistub 22 | 23 | default: ${TESTS} ${TESTS_DUMMY} 24 | 25 | integrated_dmtcp_test: integrated_dmtcp_test.c 26 | ${MPICC} -I${DMTCP_INCLUDE} -I${JALIB_INCLUDE} -DDMTCP -fPIC -g3 -O0 -o $@ $< 27 | 28 | check: tidy integrated_dmtcp_test 29 | ${DMTCP_ROOT}/bin/dmtcp_launch --port ${DEMO_PORT}\ 30 | --with-plugin $$PWD/../${LIBNAME}.so ./integrated_dmtcp_test && \ 31 | echo "Restarting..." && \ 32 | ${DMTCP_ROOT}/bin/dmtcp_restart --port ${DEMO_PORT} \ 33 | ckpt*.dmtcp 34 | 35 | %.o: %.c 36 | $(MPICC) -fPIC -g3 -O0 -c -o $@ $< $(CFLAGS) 37 | 38 | %.exe: %.o 39 | $(MPICC) -o $@ $< ${LDFLAGS} 40 | 41 | %.dummy.exe: %.o 42 | $(MPICC) -o $@ $< ${LDFLAGS} ${LDFLAGS_DUMMY} 43 | 44 | %.exe: %.cpp 45 | $(MPICXX) -fPIC -g3 -O0 -o $@ $< $(CXXFLAGS) 46 | 47 | tidy: 48 | rm -f ckpt_*.dmtcp dmtcp_restart_script* \ 49 | dmtcp-shared-memory.* dmtcp-test-typescript.tmp core* 50 | rm -rf ckpt_* 51 | rm -f dmtcp_coordinator_db-*.json 52 | 53 | clean: tidy 54 | rm -f $(TESTS) ${OBJS} ${TESTS_DUMMY} *.pyc *.so 55 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/Gatherv_test.c: -------------------------------------------------------------------------------- 1 | /* 2 | Test for the MPI_Gatherv method 3 | 4 | Must run with 4 ranks 5 | Defaults to 10000 iterations 6 | Intended to be run with mana_test.py 7 | */ 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | int 15 | main(int argc, char *argv[]) 16 | { 17 | // Parse runtime argument 18 | int max_iterations = 10000; // default 19 | if (argc != 1) { 20 | max_iterations = atoi(argv[1]); 21 | } 22 | 23 | int buffer[6]; 24 | int rank, size, i; 25 | int receive_counts[4] = { 0, 1, 2, 3 }; 26 | int receive_displacements[4] = { 0, 0, 1, 3 }; 27 | 28 | MPI_Init(&argc, &argv); 29 | MPI_Comm_size(MPI_COMM_WORLD, &size); 30 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 31 | if (size != 4) { 32 | if (rank == 0) { 33 | printf("Please run with 4 processes\n"); 34 | fflush(stdout); 35 | } 36 | MPI_Finalize(); 37 | return 0; 38 | } 39 | 40 | if (rank == 0) { 41 | printf("Running test for %d iterations\n", max_iterations); 42 | } 43 | 44 | for (int iterations = 0; iterations < max_iterations; iterations++) { 45 | for (i = 0; i < rank; i++) { 46 | buffer[i] = rank; 47 | } 48 | int expected_buf[6] = { 1, 2, 2, 3, 3, 3 }; 49 | int ret = MPI_Gatherv(buffer, rank, MPI_INT, buffer, receive_counts, 50 | receive_displacements, MPI_INT, 0, MPI_COMM_WORLD); 51 | assert(ret == MPI_SUCCESS); 52 | if (rank == 0) { 53 | for (i = 0; i < 6; i++) { 54 | printf("buffer[%d] = %d, expected = %d\n", i, buffer[i], 55 | expected_buf[i]); 56 | assert(buffer[i] == expected_buf[i]); 57 | } 58 | printf("\n"); 59 | fflush(stdout); 60 | } 61 | } 62 | MPI_Finalize(); 63 | return 0; 64 | } 65 | -------------------------------------------------------------------------------- /util/cpplint.py.patch: -------------------------------------------------------------------------------- 1 | diff --git a/util/cpplint.py b/util/cpplint.py 2 | index 3366b39..e36cd43 100644 3 | --- a/util/cpplint.py 4 | +++ b/util/cpplint.py 5 | @@ -2592,10 +2592,10 @@ class NestingState(object): 6 | if access_match: 7 | classinfo.access = access_match.group(2) 8 | 9 | - # Check that access keywords are indented +1 space. Skip this 10 | + # Check that access keywords are indented +2 space. Skip this 11 | # check if the keywords are not preceded by whitespaces. 12 | indent = access_match.group(1) 13 | - if (len(indent) != classinfo.class_indent + 1 and 14 | + if (len(indent) != classinfo.class_indent + 2 and 15 | Match(r'^\s*$', indent)): 16 | if classinfo.is_struct: 17 | parent = 'struct ' + classinfo.name 18 | @@ -3032,13 +3032,10 @@ def CheckComment(line, filename, linenum, next_line_start, error): 19 | # Check if the // may be in quotes. If so, ignore it 20 | if re.sub(r'\\.', '', line[0:commentpos]).count('"') % 2 == 0: 21 | # Allow one space for new scopes, two spaces otherwise: 22 | - if (not (Match(r'^.*{ *//', line) and next_line_start == commentpos) and 23 | - ((commentpos >= 1 and 24 | - line[commentpos-1] not in string.whitespace) or 25 | - (commentpos >= 2 and 26 | - line[commentpos-2] not in string.whitespace))): 27 | + if (commentpos >= 1 and 28 | + line[commentpos-1] not in string.whitespace): 29 | error(filename, linenum, 'whitespace/comments', 2, 30 | - 'At least two spaces is best between code and comments') 31 | + 'At least a single space is required between code and comments') 32 | 33 | # Checks for common mistakes in TODO comments. 34 | comment = line[commentpos:] 35 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/Bcast_test.c: -------------------------------------------------------------------------------- 1 | /* 2 | Test for the MPI_Allgather method 3 | 4 | Run with 4 ranks 5 | Defaults to 5 iterations 6 | Intended to be run with mana_test.py 7 | 8 | Source: http://hpc.mines.edu/examples/examples/mpi/c_ex04.c 9 | */ 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | #define NUM_RANKS 4 20 | 21 | int 22 | main(int argc, char *argv[]) 23 | { 24 | // Parse runtime argument 25 | int max_iterations = 5; // default 26 | if (argc != 1) { 27 | max_iterations = atoi(argv[1]); 28 | } 29 | 30 | int i, myid, numprocs; 31 | int source; 32 | int buffer[NUM_RANKS]; 33 | int expected_output[NUM_RANKS]; 34 | MPI_Status status; 35 | MPI_Request request; 36 | 37 | MPI_Init(&argc, &argv); 38 | MPI_Comm_size(MPI_COMM_WORLD, &numprocs); 39 | MPI_Comm_rank(MPI_COMM_WORLD, &myid); 40 | 41 | if (myid == 0) { 42 | printf("Running test for %d iterations\n", max_iterations); 43 | } 44 | 45 | assert(numprocs == NUM_RANKS); 46 | 47 | for (int iterations = 0; iterations < max_iterations; iterations++) { 48 | source = iterations % NUM_RANKS; 49 | if (myid == source) { 50 | for (i = 0; i < NUM_RANKS; i++) 51 | buffer[i] = i + iterations; 52 | } 53 | for (i = 0; i < NUM_RANKS; i++) { 54 | expected_output[i] = i + iterations; 55 | } 56 | int ret = MPI_Bcast(buffer, NUM_RANKS, MPI_INT, source, MPI_COMM_WORLD); 57 | assert(ret == MPI_SUCCESS); 58 | printf("[Rank = %d]", myid); 59 | for (i = 0; i < NUM_RANKS; i++) { 60 | assert(buffer[i] == expected_output[i]); 61 | printf(" %d", buffer[i]); 62 | } 63 | printf("\n"); 64 | fflush(stdout); 65 | } 66 | MPI_Finalize(); 67 | } 68 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/Type_commit_contiguous.c: -------------------------------------------------------------------------------- 1 | /* 2 | Test for the MPI_Type_commit method 3 | 4 | Run with >1 ranks for non-trivial results 5 | Defaults to 10000 iterations 6 | Intended to be run with mana_test.py 7 | 8 | Source: http://mpi.deino.net/mpi_functions/MPI_Type_commit.html 9 | */ 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | #define BUFFER_SIZE 100 19 | #define NUM_RANKS 4 20 | 21 | int 22 | main(int argc, char *argv[]) 23 | { 24 | // Parse runtime argument 25 | int max_iterations = 10000; // default 26 | if (argc != 1) { 27 | max_iterations = atoi(argv[1]); 28 | } 29 | 30 | int myrank; 31 | MPI_Status status; 32 | MPI_Datatype type; 33 | int buffer[BUFFER_SIZE]; 34 | int buf[BUFFER_SIZE]; 35 | 36 | MPI_Init(&argc, &argv); 37 | 38 | MPI_Type_contiguous(BUFFER_SIZE, MPI_INT, &type); 39 | MPI_Type_commit(&type); 40 | MPI_Comm_rank(MPI_COMM_WORLD, &myrank); 41 | 42 | if (myrank == 0) { 43 | printf("Running test for %d iterations\n", max_iterations); 44 | } 45 | 46 | for (int iterations = 0; iterations < max_iterations; iterations++) { 47 | for (int i = 0; i < BUFFER_SIZE; i++) { 48 | buffer[i] = i + iterations; 49 | } 50 | if (myrank == 0) { 51 | int ret = MPI_Send(buffer, 1, type, 1, 123 + iterations, MPI_COMM_WORLD); 52 | assert(ret == MPI_SUCCESS); 53 | } else if (myrank == 1) { 54 | int ret = 55 | MPI_Recv(buf, 1, type, 0, 123 + iterations, MPI_COMM_WORLD, &status); 56 | assert(ret == MPI_SUCCESS); 57 | for (int i = 0; i < BUFFER_SIZE; i++) { 58 | assert(buffer[i] == buf[i]); 59 | } 60 | } 61 | printf("Iteration %d completed\n", iterations); 62 | fflush(stdout); 63 | } 64 | MPI_Finalize(); 65 | return 0; 66 | } 67 | -------------------------------------------------------------------------------- /mpi-proxy-split/doc-old/Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # Makefile for acmart package 3 | # 4 | # This file is in public domain 5 | # 6 | # $Id: Makefile,v 1.10 2016/04/14 21:55:57 boris Exp $ 7 | # 8 | 9 | PAPER=design 10 | 11 | PDF = $(PAPER).pdf 12 | 13 | all: ${PDF} 14 | 15 | $(PAPER).pdf: $(PAPER).tex 16 | bibtex $(PAPER); \ 17 | latexmk -pdf -pdflatex="pdflatex -interaction=nonstopmode" -use-make $(PAPER).tex ;\ 18 | latexmk -pdf -pdflatex="pdflatex -interaction=nonstopmode" -use-make $(PAPER).tex 19 | 20 | open: $(PAPER).pdf # Open on Mac 21 | open $< 22 | 23 | vi: $(PAPER).tex 24 | vim $< 25 | 26 | gvi: $(PAPER).tex 27 | gvim $< 28 | 29 | mvi: $(PAPER).tex 30 | mvim $< 31 | 32 | pdf: $(PAPER).pdf 33 | 34 | renoir: $(PAPER).pdf # For printing to renoir 35 | a2ps -Prenoir -1 -s2 $< 36 | 37 | touch: $(PAPER).tex 38 | touch $< 39 | 40 | evince: $(PAPER).pdf 41 | evince $< 42 | 43 | okular: $(PAPER).pdf 44 | okular $< 45 | 46 | spell: $(PAPER).tex 47 | sed -f detex.sed $(PAPER).tex | spell | sort | uniq | less 48 | 49 | %.cls: %.ins %.dtx 50 | pdflatex $< 51 | 52 | deploy: $(PAPER).pdf $(PAPER).html 53 | scp $^ login:~/.www/thesis-proposal/ 54 | 55 | clean: 56 | $(RM) *.log $(PAPER).pdf \ 57 | *.cfg *.glo *.idx *.toc \ 58 | *.ilg *.ind *.out *.lof \ 59 | *.lot *.bbl *.blg *.gls *.cut *.hd \ 60 | *.dvi *.ps *.thm *.tgz *.zip *.rpi 61 | 62 | distclean: clean 63 | $(RM) $(PDF) *-converted-to.pdf 64 | 65 | # 66 | # Archive for the distribution. Includes typeset documentation 67 | # 68 | archive: all clean 69 | tar -C .. -czvf ../$(PACKAGE).tgz --exclude '*~' --exclude '*.tgz' --exclude '*.zip' --exclude CVS --exclude '.git*' $(PACKAGE); mv ../$(PACKAGE).tgz . 70 | 71 | zip: all clean 72 | zip -r $(PACKAGE).zip * -x '*~' -x '*.tgz' -x '*.zip' -x CVS -x 'CVS/*' 73 | 74 | .PHONY: pdf renoir okular evince open vi gvi mvi all clean spell archive zip distclean touch deploy 75 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/Comm_dup_test.c: -------------------------------------------------------------------------------- 1 | /* 2 | Test for the MPI_Comm_dup method 3 | Also tests MPI_Comm_create, MPI_Comm group methods 4 | 5 | Run with >2 ranks for non-trivial results 6 | Defaults to 5 iterations 7 | Intended to be run with mana_test.py 8 | 9 | Source: http://mpi.deino.net/mpi_functions/MPI_Comm_dup.html 10 | */ 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | #define SLEEP_PER_ITERATION 5 20 | 21 | int 22 | main(int argc, char *argv[]) 23 | { 24 | // Parse runtime argument 25 | int max_iterations = 5; // default 26 | if (argc != 1) { 27 | max_iterations = atoi(argv[1]); 28 | } 29 | 30 | MPI_Comm dup_comm_world, world_comm; 31 | MPI_Group world_group; 32 | int world_rank, world_size, rank, size; 33 | 34 | MPI_Init(&argc, &argv); 35 | MPI_Comm_rank(MPI_COMM_WORLD, &world_rank); 36 | MPI_Comm_size(MPI_COMM_WORLD, &world_size); 37 | 38 | if (world_rank == 0) { 39 | printf("Running test for %d iterations\n", max_iterations); 40 | } 41 | 42 | for (int iterations = 0; iterations < max_iterations; iterations++) { 43 | int ret = MPI_Comm_dup(MPI_COMM_WORLD, &dup_comm_world); 44 | assert(ret == MPI_SUCCESS); 45 | /* Exercise Comm_create by creating an equivalent 46 | to dup_comm_world (sans attributes) */ 47 | MPI_Comm_group(dup_comm_world, &world_group); 48 | MPI_Comm_create(dup_comm_world, world_group, &world_comm); 49 | 50 | sleep(SLEEP_PER_ITERATION); 51 | 52 | MPI_Comm_rank(world_comm, &rank); 53 | if (rank != world_rank) { 54 | printf("incorrect rank in world comm: %d\n", rank); 55 | fflush(stdout); 56 | MPI_Abort(MPI_COMM_WORLD, 3001); 57 | } 58 | printf("[Rank %d] \n", rank); 59 | fflush(stdout); 60 | } 61 | MPI_Finalize(); 62 | return 0; 63 | } 64 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/Group_size_rank.c: -------------------------------------------------------------------------------- 1 | /* 2 | Test for the MPI_Group methods 3 | 4 | Run with >2 ranks for non-trivial results 5 | Defaults to 10000 iterations 6 | Intended to be run with mana_test.py 7 | 8 | Source: http://mpi.deino.net/mpi_functions/MPI_Group_size.html 9 | */ 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | int 19 | main(int argc, char **argv) 20 | { 21 | // Parse runtime argument 22 | int max_iterations = 10000; // default 23 | if (argc != 1) { 24 | max_iterations = atoi(argv[1]); 25 | } 26 | 27 | int errs = 0, toterr; 28 | MPI_Group basegroup; 29 | MPI_Comm comm; 30 | int grp_rank, rank, grp_size, size; 31 | int worldrank; 32 | 33 | MPI_Init(NULL, NULL); 34 | MPI_Comm_rank(MPI_COMM_WORLD, &worldrank); 35 | comm = MPI_COMM_WORLD; 36 | MPI_Comm_group(comm, &basegroup); 37 | MPI_Comm_rank(comm, &rank); 38 | MPI_Comm_size(comm, &size); 39 | 40 | if (rank == 0) { 41 | printf("Running test for %d iterations\n", max_iterations); 42 | } 43 | 44 | for (int iterations = 0; iterations < max_iterations; iterations++) { 45 | /* Get the basic information on this group */ 46 | int ret = MPI_Group_rank(basegroup, &grp_rank); 47 | assert(ret == MPI_SUCCESS); 48 | if (grp_rank != rank) { 49 | errs++; 50 | fprintf(stdout, "group rank %d != comm rank %d\n", grp_rank, rank); 51 | fflush(stdout); 52 | } 53 | ret = MPI_Group_size(basegroup, &grp_size); 54 | assert(ret == MPI_SUCCESS); 55 | if (grp_size != size) { 56 | errs++; 57 | fprintf(stdout, "group size %d != comm size %d\n", grp_size, size); 58 | fflush(stdout); 59 | } 60 | assert(errs == 0); 61 | if (rank == 0) { 62 | printf("Test passed!\n"); 63 | fflush(stdout); 64 | } 65 | } 66 | return 0; 67 | } 68 | -------------------------------------------------------------------------------- /mpi-proxy-split/uh_wrappers.h: -------------------------------------------------------------------------------- 1 | /**************************************************************************** 2 | * Copyright (C) 2019-2020 by Twinkle Jain, Rohan garg, and Gene Cooperman * 3 | * jain.t@husky.neu.edu, rohgarg@ccs.neu.edu, gene@ccs.neu.edu * 4 | * * 5 | * This file is part of DMTCP. * 6 | * * 7 | * DMTCP is free software: you can redistribute it and/or * 8 | * modify it under the terms of the GNU Lesser General Public License as * 9 | * published by the Free Software Foundation, either version 3 of the * 10 | * License, or (at your option) any later version. * 11 | * * 12 | * DMTCP is distributed in the hope that it will be useful, * 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of * 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * 15 | * GNU Lesser General Public License for more details. * 16 | * * 17 | * You should have received a copy of the GNU Lesser General Public * 18 | * License along with DMTCP:dmtcp/src. If not, see * 19 | * . * 20 | ****************************************************************************/ 21 | 22 | #ifndef UPPER_HALF_WRAPPERS_H 23 | #define UPPER_HALF_WRAPPERS_H 24 | 25 | extern int initialized; 26 | 27 | extern void initialize_wrappers(); 28 | extern void reset_wrappers(); 29 | extern LowerHalfInfo_t *lh_info; 30 | 31 | #endif // ifndef UPPER_HALF_WRAPPERS_H 32 | -------------------------------------------------------------------------------- /mpi-proxy-split/test-ckpt-restart/mpi_hello_world.c: -------------------------------------------------------------------------------- 1 | // Author: Wes Kendall 2 | // Copyright 2011 www.mpitutorial.com 3 | // This code is provided freely with the tutorials on mpitutorial.com. Feel 4 | // free to modify it for your own use. Any distribution of the code must 5 | // either provide a link to www.mpitutorial.com or keep this header intact. 6 | // 7 | // An intro MPI hello world program that uses MPI_Init, MPI_Comm_size, 8 | // MPI_Comm_rank, MPI_Finalize, and MPI_Get_processor_name. 9 | // 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | int main(int argc, char** argv) 16 | { 17 | // Initialize the MPI environment. The two arguments to MPI Init are not 18 | // currently used by MPI implementations, but are there in case future 19 | // implementations might need the arguments. 20 | MPI_Init(NULL, NULL); 21 | 22 | // Get the number of processes 23 | int world_size; 24 | MPI_Comm_size(MPI_COMM_WORLD, &world_size); 25 | 26 | // Get the rank of the process 27 | int world_rank; 28 | MPI_Comm_rank(MPI_COMM_WORLD, &world_rank); 29 | int original_rank = world_rank, original_size = world_size; 30 | // Get the name of the processor 31 | char processor_name[MPI_MAX_PROCESSOR_NAME] = {"test"}; 32 | //int name_len; 33 | //MPI_Get_processor_name(processor_name, &name_len); 34 | 35 | // Print off a hello world message 36 | printf("Hello world from processor %s, rank %d out of %d processors\n", 37 | processor_name, world_rank, world_size); 38 | 39 | MPI_Comm_rank(MPI_COMM_WORLD, &world_rank); 40 | MPI_Comm_size(MPI_COMM_WORLD, &world_size); 41 | assert(world_rank == original_rank); 42 | assert(world_size == original_size); 43 | printf("Hello world from processor %s, rank %d out of %d processors\n", 44 | processor_name, world_rank, world_size); 45 | 46 | // Finalize the MPI environment. No more MPI calls can be made after this 47 | MPI_Finalize(); 48 | } 49 | -------------------------------------------------------------------------------- /bin/mana_status: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # FIXME: USER FORGOT TO DO 'salloc' (detect and report error?): 4 | # gdc0@cori11:~/mana-rohgarg-orig> srun -N 2 bin/mana_launch --verbose contrib/mpi-proxy-split/test/ping_pong.mana.exe 5 | # srun: error: No architecture specified, cannot estimate job costs. 6 | # srun: error: Unable to allocate resources: Unspecified error 7 | 8 | 9 | # if [ -z "$1" ]; then 10 | # echo "USAGE: $0 [--verbose] [DMTCP_OPTIONS ...]" 11 | # echo " For DMTCP options, do: $0 --verbose --help" 12 | # exit 1 13 | # fi 14 | 15 | if [ ! -z "$SLURM_JOB_ID" ]; then 16 | MANA_RC=$HOME/.mana-slurm-$SLURM_JOB_ID.rc 17 | else 18 | MANA_RC=$HOME/.mana.rc 19 | fi 20 | 21 | dir=`dirname $0` 22 | host=`hostname` 23 | submissionHost=`grep Host: $MANA_RC | sed -e 's%Host: %%'|sed -e 's% .*$%%'` 24 | submissionPort=`grep Port: $MANA_RC | sed -e 's%Port: %%'|sed -e 's% .*$%%'` 25 | 26 | options="" 27 | verbose=0 28 | help=0 29 | while [ -n "$1" ]; do 30 | if [ "$1" = --verbose ]; then 31 | verbose=1 32 | elif [ "$1" = --help ]; then 33 | help=1 34 | else 35 | options="$options $1" 36 | fi 37 | shift 38 | done 39 | 40 | if [ "$help" -eq 1 ]; then 41 | $dir/dmtcp_command --help $options 42 | exit 0 43 | fi 44 | 45 | coordinator_found=0 46 | $dir/dmtcp_command -s -h $submissionHost -p $submissionPort 1>/dev/null \ 47 | && coordinator_found=1 48 | if [ "$coordinator_found" = 0 ]; then 49 | echo "*** Checking for coordinator:" 50 | set -x 51 | $dir/dmtcp_command --status \ 52 | --coord-host $submissionHost --coord-port $submissionPort 53 | set +x 54 | echo " No coordinator detected. Try:" 55 | echo " $dir/mana_coordinator" 56 | echo " Or:" 57 | echo " $dir/dmtcp_coordinator --exit-on-last -q --daemon" 58 | echo " For help, do: $dir/mana_command --help" 59 | exit 2 60 | fi 61 | 62 | if [ "$verbose" = 1 ]; then set -x; fi 63 | $dir/dmtcp_command -h $submissionHost -p $submissionPort --list $options 64 | set +x 65 | -------------------------------------------------------------------------------- /mpi-proxy-split/autotest.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from shlex import split 4 | from sys import stdout 5 | from subprocess import call, PIPE 6 | from os import path, getcwd 7 | from time import sleep 8 | from sys import argv 9 | 10 | TESTS=["mpi_hello_world", "Abort_test", "Allreduce_test", "Alltoall_test", 11 | "Alltoallv_test", "Barrier_test", "Bcast_test", "Comm_split_test", 12 | "Reduce_test", "send_recv", "send_recv_many", "ping_pong", 13 | "Comm_dup_test", "Sendrecv_test", "Waitall_test", "Allgather_test", 14 | "Group_size_rank", "Type_commit_contiguous"] 15 | 16 | NAME=path.basename(getcwd()) 17 | LIBNAME="libdmtcp_%s" % (NAME) 18 | DMTCP_ROOT="../dmtcp 19 | # srun is for cori; change it to mpirun for local usage 20 | MPIRUN="srun" 21 | MPIRUN_FLAGS="-n 4" 22 | DMTCP_COORD="%s/bin/dmtcp_coordinator" % (DMTCP_ROOT) 23 | DMTCP_LAUNCH="%s/bin/dmtcp_launch" % (DMTCP_ROOT) 24 | DMTCP_LAUNCH_FLAGS="-j --with-plugin ./%s.so" % (LIBNAME) 25 | 26 | def checkTestSuccess(test=None, retcode=0): 27 | if "Abort_test" in test and retcode == 134: 28 | return "Pass" 29 | elif retcode == 0: 30 | return "Pass" 31 | else: 32 | return "Fail" 33 | 34 | def startCoordinator(): 35 | cmd = "%s -q -q --exit-on-last --daemon" % (DMTCP_COORD) 36 | return call(split(cmd)) == 0 37 | 38 | def main(): 39 | for idx, t in enumerate(sorted(TESTS)): 40 | TEST = "test/%s.exe" % (t) 41 | if (len(argv) > 1) and (TEST != argv[1]): 42 | continue 43 | if not startCoordinator(): 44 | continue 45 | cmd = "%s %s %s %s %s" % \ 46 | (MPIRUN, MPIRUN_FLAGS, DMTCP_LAUNCH, DMTCP_LAUNCH_FLAGS, TEST) 47 | output = "(%d/%d) Testing: %s ... " % (idx + 1, len(TESTS), t) 48 | stdout.write("%s" % (output)) 49 | stdout.flush() 50 | rc = call(split(cmd), stdout=PIPE, stderr=PIPE) 51 | stdout.write("%s\n" % (checkTestSuccess(test=t, retcode=rc))) 52 | stdout.flush() 53 | 54 | if __name__ == "__main__": 55 | # Execute only if run as a script 56 | main() 57 | -------------------------------------------------------------------------------- /mpi-proxy-split/lower-half/mem-wrapper.h: -------------------------------------------------------------------------------- 1 | /**************************************************************************** 2 | * Copyright (C) 2019-2020 by Twinkle Jain, Rohan garg, and Gene Cooperman * 3 | * jain.t@husky.neu.edu, rohgarg@ccs.neu.edu, gene@ccs.neu.edu * 4 | * * 5 | * This file is part of DMTCP. * 6 | * * 7 | * DMTCP is free software: you can redistribute it and/or * 8 | * modify it under the terms of the GNU Lesser General Public License as * 9 | * published by the Free Software Foundation, either version 3 of the * 10 | * License, or (at your option) any later version. * 11 | * * 12 | * DMTCP is distributed in the hope that it will be useful, * 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of * 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * 15 | * GNU Lesser General Public License for more details. * 16 | * * 17 | * You should have received a copy of the GNU Lesser General Public * 18 | * License along with DMTCP:dmtcp/src. If not, see * 19 | * . * 20 | ****************************************************************************/ 21 | 22 | #ifndef MMAP_WRAPPER_H 23 | #define MMAP_WRAPPER_H 24 | #include 25 | #include "lower-half-api.h" 26 | void* mmap_wrapper(void *, size_t , int , int , int , off_t ); 27 | void* restore_mmap(void *, size_t , int , int , int , off_t ); 28 | int munmap_wrapper(void *, size_t); 29 | std::vector &get_mmapped_list(int *num); 30 | #endif // MMAP_WRAPPER_H 31 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/sendrecv_replace_test.c: -------------------------------------------------------------------------------- 1 | /* 2 | Test for the MPI_Sendrecv_replace method 3 | 4 | Must run with 3 ranks 5 | Defaults to 100000 iterations 6 | Intended to be run with mana_test.py 7 | 8 | */ 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #define BUFFER_SIZE 3 17 | 18 | int main(int argc, char** argv) { 19 | // Parse runtime argument 20 | int max_iterations = 100000; // default 21 | if (argc != 1) { 22 | max_iterations = atoi(argv[1]); 23 | } 24 | 25 | int buf[BUFFER_SIZE]; 26 | int exp[BUFFER_SIZE]; 27 | int retval; 28 | 29 | // Initialize the MPI environment 30 | MPI_Init(NULL, NULL); 31 | // Find out rank and size 32 | int rank; 33 | retval = MPI_Comm_rank(MPI_COMM_WORLD, &rank); 34 | assert(retval == MPI_SUCCESS); 35 | int world_size; 36 | retval = MPI_Comm_size(MPI_COMM_WORLD, &world_size); 37 | assert(retval == MPI_SUCCESS); 38 | 39 | if (rank == 0) { 40 | printf("Running test for %d iterations\n", max_iterations); 41 | } 42 | 43 | // We are assuming 3 processes for this task 44 | if (world_size != 3) { 45 | fprintf(stderr, "World size must be equal to 3 for %s\n", argv[0]); 46 | MPI_Abort(MPI_COMM_WORLD, 1); 47 | } 48 | MPI_Status status; 49 | 50 | for (int iterations = 0; iterations < max_iterations; iterations++) { 51 | for (int i = 0; i < BUFFER_SIZE; i++) { 52 | buf[i] = iterations + rank + i; 53 | exp[i] = iterations + ((rank+1)%3) + i; 54 | } 55 | int dst = (rank+1) % 3; 56 | int src = (rank+2) % 3; 57 | int tag = 123; 58 | retval = MPI_Sendrecv_replace(buf, BUFFER_SIZE, MPI_INT, src, 59 | iterations+tag, dst, iterations+tag, 60 | MPI_COMM_WORLD, &status); 61 | assert(retval == MPI_SUCCESS); 62 | for (int i = 0; i < BUFFER_SIZE; i++) { 63 | assert(exp[i] == buf[i]); 64 | } 65 | 66 | } 67 | MPI_Finalize(); 68 | return 0; 69 | } 70 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/File_mixed_fds_test.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | // This program tests a mixture of normal and MPI file descriptors 11 | int main() 12 | { 13 | MPI_Init(NULL, NULL); 14 | MPI_File f1; 15 | MPI_File f3; 16 | MPI_Status stat; 17 | 18 | // Open files 19 | MPI_File_open(MPI_COMM_WORLD, "test1.txt", 20 | MPI_MODE_CREATE|MPI_MODE_RDWR, MPI_INFO_NULL, &f1); 21 | int f2 = open("test2.txt", O_RDWR|O_CREAT); 22 | fchmod(f2, S_IRUSR|S_IWUSR|S_IXUSR|S_IRGRP|S_IWGRP 23 | |S_IXGRP|S_IROTH|S_IWOTH|S_IXOTH); 24 | MPI_File_open(MPI_COMM_WORLD, "test3.txt", 25 | MPI_MODE_CREATE|MPI_MODE_RDWR, MPI_INFO_NULL, &f3); 26 | 27 | // Write content to each file 28 | char buf[1024]; 29 | strcpy(buf, "abcd"); 30 | fprintf(stderr, "%s\n", buf); fflush(stderr); 31 | MPI_File_write_at(f1, 0, buf, strlen(buf), MPI_CHAR, &stat); 32 | strcpy(buf, "efgh"); 33 | fprintf(stderr, "%s\n", buf); fflush(stderr); 34 | write(f2, buf, strlen(buf)); 35 | strcpy(buf, "ijkl"); 36 | fprintf(stderr, "%s\n", buf); fflush(stderr); 37 | MPI_File_write_at(f3, 0, buf, strlen(buf), MPI_CHAR, &stat); 38 | strcpy(buf, "mnop"); 39 | fprintf(stderr, "%s\n", buf); fflush(stderr); 40 | 41 | // Sleep 42 | fprintf(stderr, "sleeping\n"); fflush(stderr); 43 | sleep(10); 44 | 45 | // Read contents from files 46 | MPI_File_read_at_all(f1, 0, buf, 5, MPI_CHAR, &stat); 47 | fprintf(stderr, "%s\n", buf); fflush(stderr); 48 | lseek(f2, 0, SEEK_SET); 49 | int r = read(f2, buf, 5); 50 | fprintf(stderr, "%s\n", buf); fflush(stderr); 51 | MPI_File_read_at_all(f3, 0, buf, 5, MPI_CHAR, &stat); 52 | fprintf(stderr, "%s\n", buf); fflush(stderr); 53 | 54 | // Close file descriptors 55 | MPI_File_close(&f1); 56 | MPI_File_close(&f3); 57 | close(f2); 58 | 59 | MPI_Finalize(); 60 | } -------------------------------------------------------------------------------- /mpi-proxy-split/test/multi_send_recv.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | int 8 | main(int argc, char **argv) 9 | { 10 | int data = 0; 11 | int recv_buf = 0; 12 | // Initialize the MPI environment 13 | MPI_Init(NULL, NULL); 14 | // Find out rank, size 15 | int world_rank; 16 | MPI_Comm_rank(MPI_COMM_WORLD, &world_rank); 17 | int world_size; 18 | MPI_Comm_size(MPI_COMM_WORLD, &world_size); 19 | 20 | // We are assuming 2 processes for this task 21 | if (world_size != 2) { 22 | fprintf(stderr, "World size must be 2 for %s\n", argv[0]); 23 | MPI_Abort(MPI_COMM_WORLD, 1); 24 | } 25 | 26 | while (1) { 27 | MPI_Barrier(MPI_COMM_WORLD); 28 | // For rank 0, send data 3 time. After each send, increment the data by 1. 29 | if (world_rank == 0) { 30 | printf("Rank %d sending integer %d to %d\n", world_rank, data, data + 2); 31 | fflush(stdout); 32 | MPI_Send(&data, 1, MPI_INT, 1, 0, MPI_COMM_WORLD); 33 | data++; 34 | MPI_Send(&data, 1, MPI_INT, 1, 0, MPI_COMM_WORLD); 35 | data++; 36 | MPI_Send(&data, 1, MPI_INT, 1, 0, MPI_COMM_WORLD); 37 | data++; 38 | } 39 | 40 | printf("Rank %d sleeping\n", world_rank); 41 | fflush(stdout); 42 | sleep(5); 43 | 44 | // For rank 1, recv data 3 time. After each recv, increment the data by 1. 45 | // Also chech the data received. 46 | if (world_rank == 1) { 47 | printf("Rank %d recving integer %d to %d\n", world_rank, data, data + 2); 48 | fflush(stdout); 49 | MPI_Recv(&recv_buf, 1, MPI_INT, 0, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); 50 | assert(recv_buf == data); 51 | data++; 52 | MPI_Recv(&recv_buf, 1, MPI_INT, 0, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); 53 | assert(recv_buf == data); 54 | data++; 55 | MPI_Recv(&recv_buf, 1, MPI_INT, 0, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); 56 | assert(recv_buf == data); 57 | data++; 58 | } 59 | MPI_Barrier(MPI_COMM_WORLD); 60 | } 61 | MPI_Finalize(); 62 | } 63 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/keyval_test.c: -------------------------------------------------------------------------------- 1 | /* 2 | Test for the MPI_Comm keyval methods 3 | 4 | Must run with >2 ranks for non-trivial results 5 | Defaults to 5 iterations 6 | Intended to be run with mana_test.py 7 | */ 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | 17 | #define BUFFER_SIZE 3 18 | #define SLEEP_PER_ITERATION 5 19 | 20 | int 21 | main(int argc, char *argv[]) 22 | { 23 | // Parse runtime argument 24 | int max_iterations = 5; // default 25 | if (argc != 1) { 26 | max_iterations = atoi(argv[1]); 27 | } 28 | 29 | int key[BUFFER_SIZE], attrval[BUFFER_SIZE]; 30 | int i; 31 | int flag; 32 | MPI_Comm comm; 33 | 34 | MPI_Init(&argc, &argv); 35 | comm = MPI_COMM_WORLD; 36 | /* Create key values */ 37 | for (i = 0; i < BUFFER_SIZE; i++) { 38 | MPI_Comm_create_keyval(MPI_NULL_COPY_FN, MPI_NULL_DELETE_FN, &key[i], 39 | (void *)0); 40 | } 41 | 42 | for (int iterations = 0; iterations < max_iterations; iterations++) { 43 | int *val; 44 | for (int i = 0; i < BUFFER_SIZE; i++) { 45 | attrval[i] = 100 + i + iterations; 46 | } 47 | for (i = 0; i < BUFFER_SIZE; i++) { 48 | int ret = MPI_Comm_set_attr(comm, key[i], &attrval[i]); 49 | assert(ret == MPI_SUCCESS); 50 | ret = MPI_Comm_get_attr(comm, key[i], &val, &flag); 51 | assert(ret == MPI_SUCCESS); 52 | printf("keyval: %d, attrval: %d\n", key[i], *val); 53 | fflush(stdout); 54 | } 55 | 56 | printf("Will now sleep for %d seconds ...\n", SLEEP_PER_ITERATION); 57 | fflush(stdout); 58 | sleep(SLEEP_PER_ITERATION); 59 | 60 | for (i = 0; i < BUFFER_SIZE; i++) { 61 | int ret = MPI_Comm_get_attr(comm, key[i], &val, &flag); 62 | assert(ret == MPI_SUCCESS); 63 | printf("keyval: %d, attrval: %d\n", key[i], *val); 64 | fflush(stdout); 65 | } 66 | } 67 | 68 | for (i = 0; i < BUFFER_SIZE; i++) { 69 | MPI_Comm_free_keyval(&key[i]); 70 | } 71 | 72 | MPI_Finalize(); 73 | return 0; 74 | } 75 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/Comm_split_test.c: -------------------------------------------------------------------------------- 1 | /* 2 | Test for the MPI_Comm_split method 3 | 4 | Must run with an even multiple of NUM_RANKS (default 4) ranks 5 | Defaults to 60 iterations 6 | Intended to be run with mana_test.py 7 | 8 | Source: www.mpitutorial.com 9 | */ 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | #define NUM_RANKS 4 19 | #define SLEEP_PER_ITERATION 0.5 20 | 21 | int 22 | main(int argc, char **argv) 23 | { 24 | // Parse runtime argument 25 | int max_iterations = 60; // default 26 | if (argc != 1) { 27 | max_iterations = atoi(argv[1]); 28 | } 29 | 30 | MPI_Init(NULL, NULL); 31 | 32 | // Get the rank and size in the original communicator 33 | int world_rank, world_size; 34 | MPI_Comm_rank(MPI_COMM_WORLD, &world_rank); 35 | MPI_Comm_size(MPI_COMM_WORLD, &world_size); 36 | 37 | if (world_rank == 0) { 38 | printf("Running test for %d iterations\n", max_iterations); 39 | } 40 | 41 | if (world_size % NUM_RANKS != 0) { 42 | fprintf(stderr, "World size should be multiple \ 43 | of %d for %s\n", 44 | NUM_RANKS, argv[0]); 45 | MPI_Abort(MPI_COMM_WORLD, 1); 46 | } 47 | int color = world_rank / NUM_RANKS; // Determine color based on row 48 | 49 | // Split the communicator based on the color 50 | // and use the original rank for ordering 51 | MPI_Comm row_comm; 52 | for (int iterations = 0; iterations < max_iterations; iterations++) { 53 | int ret = MPI_Comm_split(MPI_COMM_WORLD, color, world_rank, &row_comm); 54 | assert(ret == MPI_SUCCESS); 55 | 56 | sleep(SLEEP_PER_ITERATION); 57 | 58 | int row_rank, row_size; 59 | MPI_Comm_rank(row_comm, &row_rank); 60 | MPI_Comm_size(row_comm, &row_size); 61 | assert(row_size == NUM_RANKS); 62 | assert(row_rank == world_rank % NUM_RANKS); 63 | printf("WORLD RANK/SIZE: %d/%d --- ROW RANK/SIZE: %d/%d\n", world_rank, 64 | world_size, row_rank, row_size); 65 | fflush(stdout); 66 | } 67 | 68 | MPI_Comm_free(&row_comm); 69 | MPI_Finalize(); 70 | } 71 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/Allreduce_test.c: -------------------------------------------------------------------------------- 1 | /* 2 | Test for the MPI_Allreduce method 3 | 4 | Run with >2 ranks for non-trivial results 5 | Defaults to 10000 iterations 6 | Intended to be run with mana_test.py 7 | 8 | Source: http://mpi.deino.net/mpi_functions/MPI_Allreduce.html 9 | */ 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #define BUFFER_SIZE 1000 18 | 19 | int 20 | main(int argc, char *argv[]) 21 | { 22 | // Parse runtime argument 23 | int max_iterations = 10000; // default 24 | if (argc != 1) { 25 | max_iterations = atoi(argv[1]); 26 | } 27 | 28 | int *in, *out, *sol; 29 | int i, fnderr = 0; 30 | int rank, size; 31 | 32 | MPI_Init(&argc, &argv); 33 | MPI_Comm_size(MPI_COMM_WORLD, &size); 34 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 35 | in = (int *)malloc(BUFFER_SIZE * sizeof(int)); 36 | out = (int *)malloc(BUFFER_SIZE * sizeof(int)); 37 | sol = (int *)malloc(BUFFER_SIZE * sizeof(int)); 38 | 39 | if (rank == 0) { 40 | printf("Running test for %d iterations\n", max_iterations); 41 | fflush(stdout); 42 | } 43 | 44 | for (int iterations = 0; iterations < max_iterations; iterations++) { 45 | for (i = 0; i < BUFFER_SIZE; i++) { 46 | *(in + i) = i; 47 | *(sol + i) = i * size; 48 | *(out + i) = 0; 49 | } 50 | int ret = 51 | MPI_Allreduce(in, out, BUFFER_SIZE, MPI_INT, MPI_SUM, MPI_COMM_WORLD); 52 | assert(ret == MPI_SUCCESS); 53 | for (i = 0; i < BUFFER_SIZE; i++) { 54 | printf("[Rank = %d] at index = %d: In = %d, Out = %d, Expected Out = %d\ 55 | \n", 56 | rank, i, *(in + i), *(out + i), *(sol + i)); 57 | fflush(stdout); 58 | 59 | assert((*(out + i) == *(sol + i))); 60 | if (*(out + i) != *(sol + i)) { 61 | fnderr++; 62 | } 63 | } 64 | if (fnderr) { 65 | fprintf(stderr, "(%d) Error for type MPI_INT and op MPI_SUM\n", rank); 66 | fflush(stderr); 67 | } 68 | } 69 | free(in); 70 | free(out); 71 | free(sol); 72 | MPI_Finalize(); 73 | return EXIT_SUCCESS; 74 | } 75 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/Sendrecv_test.c: -------------------------------------------------------------------------------- 1 | /* 2 | Test for the MPI_Sendrecv_replace method 3 | 4 | Must run with 3 ranks 5 | Defaults to 100000 iterations 6 | Intended to be run with mana_test.py 7 | 8 | */ 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | #define BUFFER_SIZE 3 20 | 21 | int 22 | main(int argc, char **argv) 23 | { 24 | // Parse runtime argument 25 | int max_iterations; 26 | max_iterations = 100000; 27 | if (argc != 1) { 28 | max_iterations = atoi(argv[1]); 29 | } 30 | 31 | int sendbuf[BUFFER_SIZE]; 32 | int recvbuf[BUFFER_SIZE]; 33 | int exp[BUFFER_SIZE]; 34 | int retval; 35 | 36 | // Initialize the MPI environment 37 | MPI_Init(NULL, NULL); 38 | // Find out rank and size 39 | int rank; 40 | retval = MPI_Comm_rank(MPI_COMM_WORLD, &rank); 41 | assert(retval == MPI_SUCCESS); 42 | int world_size; 43 | retval = MPI_Comm_size(MPI_COMM_WORLD, &world_size); 44 | assert(retval == MPI_SUCCESS); 45 | 46 | if (rank == 0) { 47 | printf("Running test for %d iterations\n", max_iterations); 48 | } 49 | 50 | // We are assuming 3 processes for this task 51 | if (world_size != 3) { 52 | fprintf(stderr, "World size must be equal to 3 for %s\n", argv[0]); 53 | MPI_Abort(MPI_COMM_WORLD, 1); 54 | } 55 | MPI_Status status; 56 | 57 | for (int iterations = 0; iterations < max_iterations; iterations++) { 58 | for (int i = 0; i < BUFFER_SIZE; i++) { 59 | sendbuf[i] = iterations + rank + i; 60 | exp[i] = iterations + ((rank + 1) % 3) + i; 61 | } 62 | int src = (rank + 1) % 3; 63 | int dst = (rank + 2) % 3; 64 | int tag = 123; 65 | retval = MPI_Sendrecv(sendbuf, BUFFER_SIZE, MPI_INT, dst, tag + iterations, 66 | recvbuf, BUFFER_SIZE, MPI_INT, src, tag + iterations, 67 | MPI_COMM_WORLD, &status); 68 | assert(retval == MPI_SUCCESS); 69 | for (int i = 0; i < BUFFER_SIZE; i++) { 70 | assert(exp[i] == recvbuf[i]); 71 | } 72 | } 73 | MPI_Finalize(); 74 | return 0; 75 | } 76 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/Reduce_test.c: -------------------------------------------------------------------------------- 1 | /* 2 | Test for the MPI_Reduce method 3 | 4 | Run with >1 ranks for non-trivial results 5 | Defaults to 10000 iterations 6 | Intended to be run with mana_test.py 7 | 8 | Source: http://mpi.deino.net/mpi_functions/MPI_Reduce.html 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #define BUFFER_SIZE 100 18 | 19 | /* A simple test of Reduce with all choices of root process */ 20 | int 21 | main(int argc, char *argv[]) 22 | { 23 | // Parse runtime argument 24 | int max_iterations = 10000; // default 25 | if (argc != 1) { 26 | max_iterations = atoi(argv[1]); 27 | } 28 | 29 | int errs = 0; 30 | int rank, size, root; 31 | int *sendbuf, *recvbuf, i; 32 | int minsize = 2, count; 33 | MPI_Comm comm; 34 | MPI_Init(&argc, &argv); 35 | 36 | comm = MPI_COMM_WORLD; 37 | /* Determine the sender and receiver */ 38 | MPI_Comm_rank(comm, &rank); 39 | MPI_Comm_size(comm, &size); 40 | 41 | if (rank == 0) { 42 | printf("Running test for %d iterations\n", max_iterations); 43 | } 44 | 45 | for (int iterations = 0; iterations < max_iterations; iterations++) { 46 | for (count = 1; count < 10; count = count * 2) { 47 | sendbuf = (int *)malloc(count * sizeof(int)); 48 | recvbuf = (int *)malloc(count * sizeof(int)); 49 | for (root = 0; root < size; root++) { 50 | for (i = 0; i < count; i++) 51 | sendbuf[i] = i + iterations; 52 | for (i = 0; i < count; i++) 53 | recvbuf[i] = -1; 54 | int ret = 55 | MPI_Reduce(sendbuf, recvbuf, count, MPI_INT, MPI_SUM, root, comm); 56 | assert(ret == MPI_SUCCESS); 57 | if (rank == root) { 58 | for (i = 0; i < count; i++) { 59 | printf("[Rank = %d]: recvd = %d, expected = %d\n", rank, recvbuf[i], 60 | (i + iterations) * size); 61 | fflush(stdout); 62 | assert(recvbuf[i] == (i + iterations) * size); 63 | } 64 | } 65 | } 66 | free(sendbuf); 67 | free(recvbuf); 68 | } 69 | } 70 | MPI_Finalize(); 71 | return EXIT_SUCCESS; 72 | } 73 | -------------------------------------------------------------------------------- /mpi-proxy-split/lower-half/Makefile: -------------------------------------------------------------------------------- 1 | # MPI compiler according to the platform 2 | UNAME_RELEASE=${shell uname -r} 3 | include ../Makefile_config 4 | 5 | # Modify if your MANA_ROOT is located elsewhere. 6 | ifndef MANA_ROOT 7 | MANA_ROOT=../.. 8 | endif 9 | # Modify if your DMTCP_ROOT is located elsewhere. 10 | ifndef DMTCP_ROOT 11 | DMTCP_ROOT=${MANA_ROOT}/dmtcp 12 | endif 13 | 14 | ifndef PLUGIN_ROOT 15 | PLUGIN_ROOT=.. 16 | endif 17 | 18 | DMTCP_INCLUDE=${DMTCP_ROOT}/include 19 | JALIB_INCLUDE=${DMTCP_ROOT}/jalib 20 | MANA_INCLUDE=${MANA_ROOT}/mpi-proxy-split 21 | PLUGIN_INCLUDE=${PLUGIN_ROOT} 22 | 23 | DMTCP_DEPS = ${DMTCP_ROOT}/src/libdmtcprestart.a \ 24 | ${DMTCP_ROOT}/src/libdmtcpinternal.a \ 25 | ${DMTCP_ROOT}/src/libjalib.a \ 26 | ${DMTCP_ROOT}/src/libnohijack.a 27 | 28 | override CFLAGS += -fPIC -I${DMTCP_INCLUDE} -I${MANA_INCLUDE} -g3 -O0 \ 29 | -I${DMTCP_ROOT}/src 30 | override CXXFLAGS += -fPIC -I${DMTCP_INCLUDE} -I${JALIB_INCLUDE} -I${MANA_INCLUDE} -I${DMTCP_ROOT}/src/mtcp \ 31 | -I${DMTCP_ROOT}/src 32 | 33 | TEXTSEG_ADDR_FLAG=-Wl,-Ttext-segment=0x10000000 34 | 35 | default: lower-half 36 | 37 | .cpp.o: 38 | ${MPICXX} ${CXXFLAGS} -c -o $@ $< 39 | 40 | .c.o: 41 | ${MPICXX} ${CFLAGS} -c -o $@ $< 42 | 43 | # Normally, gcc (or clang) will automatically link with libatomic.so. 44 | # However, for come MPICXX it's hard to know if they cover it. 45 | # So this is good as a safety measure. 46 | lower-half: get-symbol-offset.o copy-stack.o patch-trampoline.o lh-func-ptr.o mem-wrapper.o switch-context.o lower-half.o mmap-fixed-noreplace.o ${DMTCP_DEPS} 47 | ${MPICXX} -g3 ${TEXTSEG_ADDR_FLAG} ${CXXFLAGS} -o $@ $^ -lpthread -ldl -latomic 48 | 49 | install: lower-half 50 | cp -f $^ ${MANA_ROOT}/bin/ 51 | 52 | tidy: 53 | rm -f *~ .*.swp dmtcp_restart_script*.sh ckpt_*.dmtcp 54 | rm -rf ckpt_rank_* 55 | rm -rf dmtcp_coordinator_db-*.json 56 | 57 | clean: tidy 58 | rm -f lower-half 59 | rm -f ${MANA_ROOT}/bin/lower-half 60 | rm -f *.o 61 | 62 | distclean: clean 63 | 64 | dist: distclean 65 | dir=`basename $$PWD` && cd .. && tar czvf $$dir.tgz ./$$dir 66 | dir=`basename $$PWD` && ls -l ../$$dir.tgz 67 | 68 | .PHONY: default clean dist distclean install 69 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/Type_vector_test.c: -------------------------------------------------------------------------------- 1 | /* 2 | Test for the MPI_Type_vector method 3 | 4 | Run with 2 ranks 5 | Defaults to 10000 iterations 6 | Intended to be run with mana_test.py 7 | 8 | Inspired by: 9 | https://rookiehpc.github.io/mpi/docs/mpi_type_create_hvector/index.html 10 | */ 11 | 12 | #define _POSIX_C_SOURCE 199309L 13 | 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #define BUFFER_SIZE 100 22 | 23 | int 24 | main(int argc, char **argv) 25 | { 26 | // Parse runtime argument 27 | int max_iterations = 10000; // default 28 | if (argc != 1) { 29 | max_iterations = atoi(argv[1]); 30 | } 31 | 32 | int rank, comm_size; 33 | 34 | MPI_Init(&argc, &argv); 35 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 36 | MPI_Comm_size(MPI_COMM_WORLD, &comm_size); 37 | if (rank == 0) { 38 | printf("Comm. size = %d\n", comm_size); 39 | printf("Running test for %d iterations\n", max_iterations); 40 | } 41 | fflush(stdout); 42 | assert(comm_size == 2); 43 | 44 | MPI_Datatype column_type; 45 | MPI_Type_vector(BUFFER_SIZE, 1, BUFFER_SIZE, MPI_INT, &column_type); 46 | MPI_Type_commit(&column_type); 47 | int buffer[BUFFER_SIZE][BUFFER_SIZE]; 48 | int recvbuf[BUFFER_SIZE]; 49 | int expected_output[BUFFER_SIZE][BUFFER_SIZE]; 50 | for (int i = 0; i < max_iterations; i++) { 51 | if (i % 100 == 99) { 52 | printf("Completed %d iterations\n", i + 1); 53 | fflush(stdout); 54 | } 55 | for (int j = 0; j < BUFFER_SIZE; j++) { 56 | for (int k = 0; k < BUFFER_SIZE; k++) { 57 | buffer[j][k] = j + k + i; 58 | expected_output[j][k] = j + k + i; 59 | } 60 | } 61 | 62 | if (rank == 0) { 63 | int ret = MPI_Send(&buffer[0][1], 1, column_type, 1, 0, MPI_COMM_WORLD); 64 | assert(ret == MPI_SUCCESS); 65 | } else { 66 | int ret = MPI_Recv(&recvbuf, BUFFER_SIZE, MPI_INT, 0, 0, MPI_COMM_WORLD, 67 | MPI_STATUS_IGNORE); 68 | assert(ret == MPI_SUCCESS); 69 | for (int i = 0; i < BUFFER_SIZE; i++) { 70 | assert(recvbuf[i] == buffer[i][1]); 71 | } 72 | } 73 | } 74 | MPI_Finalize(); 75 | return EXIT_SUCCESS; 76 | } -------------------------------------------------------------------------------- /mpi-proxy-split/test/Waitall_test.c: -------------------------------------------------------------------------------- 1 | /* 2 | Test for the MPI_Waitall method 3 | 4 | Must run with NUM_RANKS (4) processes 5 | Defaults to 5 iterations 6 | Intended to be run with mana_test.py 7 | 8 | Source: http://mpi.deino.net/mpi_functions/MPI_Waitall.html 9 | */ 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | #define BUFFER_SIZE 100 19 | #define SLEEP_PER_ITERATION 5 20 | #define NUM_RANKS 4 21 | 22 | int 23 | main(int argc, char *argv[]) 24 | { 25 | // Parse runtime argument 26 | int max_iterations = 5; // default 27 | if (argc != 1) { 28 | max_iterations = atoi(argv[1]); 29 | } 30 | 31 | int rank, size; 32 | int i; 33 | int buffer[NUM_RANKS * BUFFER_SIZE]; 34 | MPI_Request request[NUM_RANKS]; 35 | MPI_Status status[NUM_RANKS]; 36 | 37 | MPI_Init(&argc, &argv); 38 | MPI_Comm_size(MPI_COMM_WORLD, &size); 39 | if (size != NUM_RANKS) { 40 | printf("Please run with %d processes.\n", NUM_RANKS); 41 | fflush(stdout); 42 | MPI_Finalize(); 43 | return 1; 44 | } 45 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 46 | 47 | if (rank == 0) { 48 | printf("Running test for %d iterations\n", max_iterations); 49 | } 50 | 51 | for (int iterations = 0; iterations < max_iterations; iterations++) { 52 | if (rank == 0) { 53 | for (i = 0; i < size * BUFFER_SIZE; i++) 54 | buffer[i] = i / BUFFER_SIZE + iterations; 55 | for (i = 0; i < size - 1; i++) { 56 | int ret = 57 | MPI_Isend(&buffer[i * BUFFER_SIZE], BUFFER_SIZE, MPI_INT, i + 1, 58 | 123 + iterations, MPI_COMM_WORLD, &request[i]); 59 | assert(ret == MPI_SUCCESS); 60 | } 61 | MPI_Waitall(size - 1, request, status); 62 | sleep(SLEEP_PER_ITERATION); 63 | } else { 64 | sleep(SLEEP_PER_ITERATION); 65 | int ret = MPI_Recv(buffer, BUFFER_SIZE, MPI_INT, 0, 123 + iterations, 66 | MPI_COMM_WORLD, &status[0]); 67 | assert(ret == MPI_SUCCESS); 68 | printf("%d: buffer[0] = %d\n", rank, buffer[0]); 69 | fflush(stdout); 70 | assert(buffer[0] == rank - 1 + iterations); 71 | } 72 | } 73 | 74 | MPI_Finalize(); 75 | return 0; 76 | } 77 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/Type_create_resized_test.c: -------------------------------------------------------------------------------- 1 | /* 2 | Test for the MPI_Type_create_resized method 3 | 4 | Run with 2 ranks 5 | Defaults to 10000 iterations 6 | Intended to be run with mana_test.py 7 | 8 | Inspired by: 9 | http://mpi.deino.net/mpi_functions/MPI_Type_create_resized.html 10 | */ 11 | 12 | #define _POSIX_C_SOURCE 199309L 13 | 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #define BUFFER_SIZE 100 22 | 23 | int 24 | main(int argc, char **argv) 25 | { 26 | // Parse runtime argument 27 | int max_iterations = 10000; // default 28 | if (argc != 1) { 29 | max_iterations = atoi(argv[1]); 30 | } 31 | 32 | int rank, comm_size, ret; 33 | 34 | MPI_Init(&argc, &argv); 35 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 36 | MPI_Comm_size(MPI_COMM_WORLD, &comm_size); 37 | if (rank == 0) { 38 | printf("Comm. size = %d\n", comm_size); 39 | printf("Running test for %d iterations\n", max_iterations); 40 | } 41 | fflush(stdout); 42 | assert(comm_size == 2); 43 | 44 | MPI_Datatype newtype; 45 | ret = MPI_Type_create_resized(MPI_INT, 0, 3 * sizeof(int), &newtype); 46 | assert(ret == MPI_SUCCESS); 47 | ret = MPI_Type_commit(&newtype); 48 | assert(ret == MPI_SUCCESS); 49 | int buf[BUFFER_SIZE*3]; 50 | memset(buf, 0, BUFFER_SIZE*3*sizeof(int)); 51 | 52 | for (int i = 0; i < max_iterations; i++) { 53 | if (i % 100 == 99) { 54 | printf("Completed %d iterations\n", i + 1); 55 | fflush(stdout); 56 | } 57 | if (rank == 0) { 58 | for (int k = 0; k < BUFFER_SIZE; k++) { 59 | buf[3*k] = i+k; 60 | } 61 | ret = MPI_Send(buf, BUFFER_SIZE, newtype, 1, 123+i, MPI_COMM_WORLD); 62 | assert(ret == MPI_SUCCESS); 63 | } else { 64 | memset(buf, -1, sizeof(int)*3*BUFFER_SIZE); 65 | int ret = MPI_Recv(buf, BUFFER_SIZE, newtype, 0, 123+i, MPI_COMM_WORLD, 66 | MPI_STATUS_IGNORE); 67 | assert(ret == MPI_SUCCESS); 68 | for (int k = 0; k < BUFFER_SIZE; k++) { 69 | assert(buf[3*k] == i+k); 70 | } 71 | } 72 | MPI_Barrier(MPI_COMM_WORLD); 73 | } 74 | MPI_Type_free(&newtype); 75 | MPI_Finalize(); 76 | return EXIT_SUCCESS; 77 | } 78 | -------------------------------------------------------------------------------- /util/git-bisect.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copy this script to the DMTCP root director. 4 | # Then modify this script first for your purposes. 5 | # In particular, search below for the two strings 6 | # ("Test if good", "Test if bad"), and modify them for your test. 7 | # Then execute something like: 8 | # git bisect start 9 | # git bisect bad 10 | # git bisect good 2.0 # tag 2.0, or pick any older commit that was good 11 | # git bisect run ./git-bisect.sh 12 | 13 | # While this is executing, in another window, you can see the progress with: 14 | # git bisect log 15 | # To modify and redo: 16 | # git bisect log > git-bisect.log 17 | # tail -f git-bisect.out 18 | 19 | # The bad commit could be a false alarm witin a range of good commit. 20 | # To analyze a given range, try something like: 21 | # git log --oneline 1a7d8dbb0dc48..master | wc 22 | # git checkout master~ # where is the number of lines, from wc 23 | # and try testing a few more recent branches 24 | # Then EDIT git-bisect.log 25 | # git bisect replay git-bisect.log 26 | # git bisect ./git-bisect.sh 27 | 28 | 29 | # Test if good 30 | git log -1 > git-bisect.out 31 | echo 32 | echo "************ NEW RUN **************" 33 | git log -1 | cat 34 | good_match_1='ckpt:[\t ]*PASSED[\t ]*rstr:[\t ]*PASSED[\t ]*;' 35 | good_match_2='[\t ]*ckpt:[\t ]*PASSED[\t ]*rstr:[\t ]*PASSED' 36 | good_match="$good_match_1$good_match_2" 37 | # older DMTCP had Makefile that tried to call automake 38 | # older DMTCP required '.' in PATH for 'cd mtcp && make build' 39 | ./configure AUTOMAKE=echo && make -j clean && rm -rf bin lib && \ 40 | (PATH=$PATH:. make -j) && (cd src && make -j || true) && \ 41 | (mkdir -p bin && cd bin && ln -sf ../lib/dmtcp/32/bin/* ./ || true) && \ 42 | (cd lib/dmtcp && ln -sf 32/lib/dmtcp/* ./ || true) && \ 43 | (cd lib/dmtcp/32/bin && ln -s mtcp_restart-32 mtcp_restart || true) && \ 44 | make -j check-dmtcp1 >> git-bisect.out 2>&1 && \ 45 | grep "$good_match" git-bisect.out \ 46 | > /dev/null && \ 47 | exit 0 48 | 49 | # Test if bad 50 | # Consider a run "bad" only if it compiled and started autotest.py and failed. 51 | # Anything else should be skipped as a problem for that single revision. 52 | grep 'ckpt:' git-bisect.out > /dev/null && exit 1 53 | 54 | # Skip it (unsure) 55 | exit 125 56 | -------------------------------------------------------------------------------- /util/memory_access_tracker/memory_access_tracker.h: -------------------------------------------------------------------------------- 1 | #/***************************************************************************** 2 | # * Copyright (C) 2021 Jun Gan * 3 | # * * 4 | # * DMTCP is free software: you can redistribute it and/or * 5 | # * modify it under the terms of the GNU Lesser General Public License as * 6 | # * published by the Free Software Foundation, either version 3 of the * 7 | # * License, or (at your option) any later version. * 8 | # * * 9 | # * DMTCP is distributed in the hope that it will be useful, * 10 | # * but WITHOUT ANY WARRANTY; without even the implied warranty of * 11 | # * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * 12 | # * GNU Lesser General Public License for more details. * 13 | # * * 14 | # * You should have received a copy of the GNU Lesser General Public * 15 | # * License along with DMTCP. If not, see . * 16 | # *****************************************************************************/ 17 | 18 | #ifndef MEMORY_ACCESS_TRACKER 19 | #define MEMORY_ACCESS_TRACKER 20 | 21 | #include 22 | #include 23 | 24 | #ifdef __cplusplus 25 | extern "C" { 26 | #endif 27 | 28 | typedef struct MemTracker { 29 | /* TODO: support multiple trackers */ 30 | // struct list_head node; 31 | void *addr; 32 | size_t len; 33 | int fd; // for debug 34 | uint32_t max_num; // max number of access will be tracked 35 | /* If there're too many accesses to the same address, 36 | * May change to use map instead here */ 37 | uint32_t num_access; // number of access happened 38 | void** addrs; // addresses have been accessed 39 | } MemTracker_t; 40 | 41 | struct MemTracker* StartTrackMemory(void *addr, size_t len, uint32_t max_num, int fd); 42 | void EndTrackMemory(struct MemTracker *tracker); 43 | void FreeMemTracker(struct MemTracker *tracker); 44 | 45 | #ifdef __cplusplus 46 | } 47 | #endif 48 | #endif 49 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/Type_create_hindexed_block_test.c: -------------------------------------------------------------------------------- 1 | /* 2 | Test for the MPI_Type_create_hindexed_block method 3 | 4 | Run with 2 ranks 5 | Defaults to 10000 iterations 6 | Intended to be run with mana_test.py 7 | 8 | Inspired by: 9 | https://rookiehpc.github.io/mpi/docs/ 10 | mpi_type_create_hindexed_block/index.html 11 | */ 12 | 13 | #define _POSIX_C_SOURCE 199309L 14 | 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | 22 | int 23 | main(int argc, char **argv) 24 | { 25 | // Parse runtime argument 26 | int max_iterations = 10000; // default 27 | if (argc != 1) { 28 | max_iterations = atoi(argv[1]); 29 | } 30 | 31 | int rank, comm_size, ret; 32 | 33 | MPI_Init(&argc, &argv); 34 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 35 | MPI_Comm_size(MPI_COMM_WORLD, &comm_size); 36 | if (rank == 0) { 37 | printf("Comm. size = %d\n", comm_size); 38 | printf("Running test for %d iterations\n", max_iterations); 39 | } 40 | fflush(stdout); 41 | assert(comm_size == 2); 42 | 43 | MPI_Datatype corner_type; 44 | MPI_Aint displacements[4] = { 0, 2 * sizeof(int), 6 * sizeof(int), 8 * sizeof(int)}; 45 | MPI_Type_create_hindexed_block(4, 1, displacements, MPI_INT, &corner_type); 46 | MPI_Type_commit(&corner_type); 47 | int buf[9]; 48 | int recvbuf[4]; 49 | 50 | for (int i = 0; i < max_iterations; i++) { 51 | if (i % 100 == 99) { 52 | printf("Completed %d iterations\n", i + 1); 53 | fflush(stdout); 54 | } 55 | if (rank == 0) { 56 | for (int k = 0; k < 9; k++) { 57 | buf[k] = i+k; 58 | } 59 | ret = MPI_Send(buf, 1, corner_type, 1, 123+i, MPI_COMM_WORLD); 60 | assert(ret == MPI_SUCCESS); 61 | } else { 62 | memset(recvbuf, 0, 4*sizeof(int)); 63 | int ret = MPI_Recv(recvbuf, 4, MPI_INT, 0, 123+i, MPI_COMM_WORLD, 64 | MPI_STATUS_IGNORE); 65 | assert(ret == MPI_SUCCESS); 66 | assert(recvbuf[0] == i); 67 | assert(recvbuf[1] == i+2); 68 | assert(recvbuf[2] == i+6); 69 | assert(recvbuf[3] == i+8); 70 | } 71 | 72 | MPI_Barrier(MPI_COMM_WORLD); 73 | } 74 | MPI_Type_free(&corner_type); 75 | MPI_Finalize(); 76 | return EXIT_SUCCESS; 77 | } 78 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/integrated_dmtcp_test.c: -------------------------------------------------------------------------------- 1 | // Author: Wes Kendall 2 | // Copyright 2011 www.mpitutorial.com 3 | // This code is provided freely with the tutorials on mpitutorial.com. Feel 4 | // free to modify it for your own use. Any distribution of the code must 5 | // either provide a link to www.mpitutorial.com or keep this header intact. 6 | // 7 | // An intro MPI hello world program that uses MPI_Init, MPI_Comm_size, 8 | // MPI_Comm_rank, MPI_Finalize, and MPI_Get_processor_name. 9 | // 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #include "dmtcp.h" 18 | 19 | int 20 | main(int argc, char **argv) 21 | { 22 | // Initialize the MPI environment. The two arguments to MPI Init are not 23 | // currently used by MPI implementations, but are there in case future 24 | // implementations might need the arguments. 25 | MPI_Init(NULL, NULL); 26 | 27 | // Get the number of processes 28 | int world_size; 29 | MPI_Comm_size(MPI_COMM_WORLD, &world_size); 30 | 31 | // Get the rank of the process 32 | int world_rank; 33 | MPI_Comm_rank(MPI_COMM_WORLD, &world_rank); 34 | int original_rank = world_rank, original_size = world_size; 35 | // Get the name of the processor 36 | char processor_name[MPI_MAX_PROCESSOR_NAME] = { "test" }; 37 | // int name_len; 38 | // MPI_Get_processor_name(processor_name, &name_len); 39 | // Print off a hello world message 40 | printf("Hello world from processor %s, rank %d out of %d processors\n", 41 | processor_name, world_rank, world_size); 42 | printf("Checkpointing...\n"); 43 | fflush(stdout); 44 | int retval = dmtcp_checkpoint(); 45 | if (retval == DMTCP_AFTER_CHECKPOINT) { 46 | printf("Exiting...\n"); 47 | exit(0); 48 | } 49 | sleep(1); 50 | printf("Restarting...\n"); 51 | fflush(stdout); 52 | MPI_Comm_rank(MPI_COMM_WORLD, &world_rank); 53 | MPI_Comm_size(MPI_COMM_WORLD, &world_size); 54 | assert(world_rank == original_rank); 55 | assert(world_size == original_size); 56 | printf("Hello world from processor %s, rank %d out of %d processors\n", 57 | processor_name, world_rank, world_size); 58 | 59 | // Finalize the MPI environment. No more MPI calls can be made after this 60 | MPI_Finalize(); 61 | } 62 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/Type_hvector_test.c: -------------------------------------------------------------------------------- 1 | /* 2 | Test for the MPI_Type_create_hvector method 3 | 4 | Run with 2 ranks 5 | Defaults to 10000 iterations 6 | Intended to be run with mana_test.py 7 | 8 | Inspired by: 9 | https://rookiehpc.github.io/mpi/docs/mpi_type_create_hvector/index.html 10 | */ 11 | 12 | #define _POSIX_C_SOURCE 199309L 13 | 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #define BUFFER_SIZE 100 22 | 23 | int 24 | main(int argc, char **argv) 25 | { 26 | // Parse runtime argument 27 | int max_iterations = 10000; // default 28 | if (argc != 1) { 29 | max_iterations = atoi(argv[1]); 30 | } 31 | 32 | int rank, comm_size; 33 | 34 | MPI_Init(&argc, &argv); 35 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 36 | MPI_Comm_size(MPI_COMM_WORLD, &comm_size); 37 | if (rank == 0) { 38 | printf("Comm. size = %d\n", comm_size); 39 | printf("Running test for %d iterations\n", max_iterations); 40 | } 41 | fflush(stdout); 42 | assert(comm_size == 2); 43 | 44 | MPI_Datatype column_type; 45 | MPI_Type_create_hvector(BUFFER_SIZE, 1, BUFFER_SIZE * sizeof(int), MPI_INT, 46 | &column_type); 47 | MPI_Type_commit(&column_type); 48 | int buffer[BUFFER_SIZE][BUFFER_SIZE]; 49 | int recvbuf[BUFFER_SIZE]; 50 | int expected_output[BUFFER_SIZE][BUFFER_SIZE]; 51 | for (int i = 0; i < max_iterations; i++) { 52 | if (i % 100 == 99) { 53 | printf("Completed %d iterations\n", i + 1); 54 | fflush(stdout); 55 | } 56 | for (int j = 0; j < BUFFER_SIZE; j++) { 57 | for (int k = 0; k < BUFFER_SIZE; k++) { 58 | buffer[j][k] = j + k + i; 59 | expected_output[j][k] = j + k + i; 60 | } 61 | } 62 | 63 | if (rank == 0) { 64 | int ret = MPI_Send(&buffer[0][1], 1, column_type, 1, 0, MPI_COMM_WORLD); 65 | assert(ret == MPI_SUCCESS); 66 | } else { 67 | int ret = MPI_Recv(&recvbuf, BUFFER_SIZE, MPI_INT, 0, 0, MPI_COMM_WORLD, 68 | MPI_STATUS_IGNORE); 69 | assert(ret == MPI_SUCCESS); 70 | for (int i = 0; i < BUFFER_SIZE; i++) { 71 | assert(recvbuf[i] == buffer[i][1]); 72 | } 73 | } 74 | } 75 | MPI_Finalize(); 76 | return EXIT_SUCCESS; 77 | } 78 | -------------------------------------------------------------------------------- /m4/ax_check_compile_flag.m4: -------------------------------------------------------------------------------- 1 | # =========================================================================== 2 | # https://www.gnu.org/software/autoconf-archive/ax_check_compile_flag.html 3 | # =========================================================================== 4 | # 5 | # SYNOPSIS 6 | # 7 | # AX_CHECK_COMPILE_FLAG(FLAG, [ACTION-SUCCESS], [ACTION-FAILURE], [EXTRA-FLAGS], [INPUT]) 8 | # 9 | # DESCRIPTION 10 | # 11 | # Check whether the given FLAG works with the current language's compiler 12 | # or gives an error. (Warnings, however, are ignored) 13 | # 14 | # ACTION-SUCCESS/ACTION-FAILURE are shell commands to execute on 15 | # success/failure. 16 | # 17 | # If EXTRA-FLAGS is defined, it is added to the current language's default 18 | # flags (e.g. CFLAGS) when the check is done. The check is thus made with 19 | # the flags: "CFLAGS EXTRA-FLAGS FLAG". This can for example be used to 20 | # force the compiler to issue an error when a bad flag is given. 21 | # 22 | # INPUT gives an alternative input source to AC_COMPILE_IFELSE. 23 | # 24 | # NOTE: Implementation based on AX_CFLAGS_GCC_OPTION. Please keep this 25 | # macro in sync with AX_CHECK_{PREPROC,LINK}_FLAG. 26 | # 27 | # LICENSE 28 | # 29 | # Copyright (c) 2008 Guido U. Draheim 30 | # Copyright (c) 2011 Maarten Bosmans 31 | # 32 | # Copying and distribution of this file, with or without modification, are 33 | # permitted in any medium without royalty provided the copyright notice 34 | # and this notice are preserved. This file is offered as-is, without any 35 | # warranty. 36 | 37 | #serial 6 38 | 39 | AC_DEFUN([AX_CHECK_COMPILE_FLAG], 40 | [AC_PREREQ(2.64)dnl for _AC_LANG_PREFIX and AS_VAR_IF 41 | AS_VAR_PUSHDEF([CACHEVAR],[ax_cv_check_[]_AC_LANG_ABBREV[]flags_$4_$1])dnl 42 | AC_CACHE_CHECK([whether _AC_LANG compiler accepts $1], CACHEVAR, [ 43 | ax_check_save_flags=$[]_AC_LANG_PREFIX[]FLAGS 44 | _AC_LANG_PREFIX[]FLAGS="$[]_AC_LANG_PREFIX[]FLAGS $4 $1" 45 | AC_COMPILE_IFELSE([m4_default([$5],[AC_LANG_PROGRAM()])], 46 | [AS_VAR_SET(CACHEVAR,[yes])], 47 | [AS_VAR_SET(CACHEVAR,[no])]) 48 | _AC_LANG_PREFIX[]FLAGS=$ax_check_save_flags]) 49 | AS_VAR_IF(CACHEVAR,yes, 50 | [m4_default([$2], :)], 51 | [m4_default([$3], :)]) 52 | AS_VAR_POPDEF([CACHEVAR])dnl 53 | ])dnl AX_CHECK_COMPILE_FLAGS 54 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/Waitany_test.c: -------------------------------------------------------------------------------- 1 | /* 2 | Test for the MPI_Waitany method 3 | 4 | Must run with NUM_RANKS (4) processes 5 | Defaults to 5 iterations 6 | Intended to be run with mana_test.py 7 | 8 | Source: http://mpi.deino.net/mpi_functions/MPI_Waitany.html 9 | */ 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | #define BUFFER_SIZE 100 19 | #define SLEEP_PER_ITERATION 5 20 | #define NUM_RANKS 4 21 | 22 | int 23 | main(int argc, char *argv[]) 24 | { 25 | // Parse runtime argument 26 | int max_iterations = 5; // default 27 | if (argc != 1) { 28 | max_iterations = atoi(argv[1]); 29 | } 30 | 31 | int rank, size; 32 | int i, index; 33 | int buffer[NUM_RANKS * BUFFER_SIZE]; 34 | MPI_Request request[NUM_RANKS]; 35 | MPI_Status status; 36 | 37 | MPI_Init(&argc, &argv); 38 | MPI_Comm_size(MPI_COMM_WORLD, &size); 39 | if (size != NUM_RANKS) { 40 | printf("Please run with %d processes.\n", NUM_RANKS); 41 | fflush(stdout); 42 | MPI_Finalize(); 43 | return 1; 44 | } 45 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 46 | 47 | if (rank == 0) { 48 | printf("Running test for %d iterations\n", max_iterations); 49 | } 50 | 51 | for (int iterations = 0; iterations < max_iterations; iterations++) { 52 | if (rank == 0) { 53 | for (i = 0; i < size * BUFFER_SIZE; i++) 54 | buffer[i] = i / BUFFER_SIZE + iterations; 55 | for (i = 0; i < size - 1; i++) { 56 | int ret = 57 | MPI_Isend(&buffer[i * BUFFER_SIZE], BUFFER_SIZE, MPI_INT, i + 1, 58 | 123 + iterations, MPI_COMM_WORLD, &request[i]); 59 | assert(ret == MPI_SUCCESS); 60 | } 61 | MPI_Waitany(size - 1, request, &index, &status); 62 | sleep(SLEEP_PER_ITERATION); 63 | } else { 64 | sleep(SLEEP_PER_ITERATION); 65 | int ret = MPI_Recv(buffer, BUFFER_SIZE, MPI_INT, 0, 123 + iterations, 66 | MPI_COMM_WORLD, &status); 67 | assert(ret == MPI_SUCCESS); 68 | printf("%d: buffer[0] = %d\n", rank, buffer[0]); 69 | fflush(stdout); 70 | assert(buffer[0] == rank - 1 + iterations); 71 | } 72 | memset(buffer, 0, BUFFER_SIZE * sizeof(int)); 73 | } 74 | 75 | MPI_Finalize(); 76 | return 0; 77 | } 78 | -------------------------------------------------------------------------------- /util/memory_access_tracker/README.md: -------------------------------------------------------------------------------- 1 | # Memory Access Tracker 2 | 3 | Linux only provides the way to track memory access in page level, like soft dirty bits and userfault. This tool can track memory access in a more granuale level. 4 | 5 | # APIs 6 | 7 | `StartTrackMemory` starts track memory start from `addr`(has to be page aligned) with size `len`. User also needs to specify the `max_num` which is the maxinum number of access it trackes and `fd` which it can write some logs. 8 | 9 | **Warning** The performance will be dramatically downgraded. 10 | 11 | `EndTrackMemory` Stops tracking the memory access, after this, user can get the number of accesses(`num_access`) and the accessed memory addresses(`addrs`) from the tracker after this. 12 | 13 | `FreeMemTracker` frees the memory used by the data structure. 14 | 15 | # Sample useage 16 | 17 | You can check the program example in `test_memory_tracker.c` 18 | 19 | ``` 20 | $ make 21 | gcc -D_GNU_SOURCE -DDEBUG -Wall -O0 -ggdb3 -fPIC -c -o memory_access_tracker.o memory_access_tracker.c 22 | gcc -o libmemtracker.so -shared memory_access_tracker.o 23 | ar rcs libmemtracker.a memory_access_tracker.o 24 | gcc -D_GNU_SOURCE -DDEBUG -Wall -O0 -ggdb3 -o test_memory_tracker test_memory_tracker.c libmemtracker.a 25 | $ ./test_memory_tracker 26 | main 35 Test 0x7f32a1e06000 27 | main 40 Read 0x7f32a1e06000: 0 28 | main 46 Read 0x7f32a1e06000: 1 29 | main 49 Read 0x7f32a1e06400: 0 30 | main 53 access 39 times 31 | 0x7f32a1e06000 32 | 0x7f32a1e06000 33 | 0x7f32a1e06000 34 | 0x7f32a1e06064 35 | 0x7f32a1e06800 36 | 0x7f32a1e06bf0 37 | 0x7f32a1e06be0 38 | 0x7f32a1e06bd0 39 | 0x7f32a1e06bc0 40 | 0x7f32a1e06840 41 | 0x7f32a1e06860 42 | 0x7f32a1e06880 43 | 0x7f32a1e068a0 44 | 0x7f32a1e068c0 45 | 0x7f32a1e068e0 46 | 0x7f32a1e06900 47 | 0x7f32a1e06920 48 | 0x7f32a1e06940 49 | 0x7f32a1e06960 50 | 0x7f32a1e06980 51 | 0x7f32a1e069a0 52 | 0x7f32a1e069c0 53 | 0x7f32a1e069e0 54 | 0x7f32a1e06a00 55 | 0x7f32a1e06a20 56 | 0x7f32a1e06a40 57 | 0x7f32a1e06a60 58 | 0x7f32a1e06a80 59 | 0x7f32a1e06aa0 60 | 0x7f32a1e06ac0 61 | 0x7f32a1e06ae0 62 | 0x7f32a1e06b00 63 | 0x7f32a1e06b20 64 | 0x7f32a1e06b40 65 | 0x7f32a1e06b60 66 | 0x7f32a1e06b80 67 | 0x7f32a1e06ba0 68 | 0x7f32a1e06bc0 69 | 0x7f32a1e06be0 70 | ``` 71 | 72 | # Reference 73 | 74 | https://programmer.group/x86_-principle-and-example-of-single-step-debugging-for-64-platform.html 75 | -------------------------------------------------------------------------------- /bin/mana_coordinator: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # if [ -n "$1" ]; then 4 | # echo "USAGE: $0 [--verbose] [DMTCP_OPTIONS ...]" 5 | # echo " For other DMTCP coordinator options, do:" 6 | # echo " $0 --help" 7 | # exit 1 8 | # fi 9 | 10 | mana_coord=$0 11 | dir=`dirname $0` 12 | 13 | options="" 14 | verbose=0 15 | help=0 16 | while [ -n "$1" ]; do 17 | if [ "$1" = --verbose ]; then 18 | verbose=1 19 | elif [ "$1" = --help ]; then 20 | help=1 21 | else 22 | options="$options $1" 23 | fi 24 | shift 25 | done 26 | 27 | if [ "$help" -eq 1 ]; then 28 | $dir/dmtcp_coordinator --help $options 29 | exit 0 30 | fi 31 | 32 | SITE=unknown 33 | if [ "$NERSC_HOST" = "cori" ]; then 34 | SITE=nersc 35 | elif [ "$NERSC_HOST" = "gerty" ]; then 36 | SITE=nersc 37 | fi 38 | 39 | if [ "$SITE" = "nersc" ]; then 40 | if [ -z "$SLURM_JOB_ID" ]; then 41 | echo "SLURM_JOB_ID env variable not set; No salloc/sbatch jobs running?" 42 | echo "For help, do: $mana_coord --help" 43 | exit 2 44 | fi 45 | fi 46 | 47 | if [ "$verbose" = 0 ]; then 48 | options="$options -q -q" 49 | fi 50 | 51 | if [ "$verbose" = 1 ]; then 52 | set -x 53 | fi 54 | 55 | coordinator_found=0 56 | 57 | if [ ! -z "$SLURM_JOB_ID" ]; then 58 | MANA_RC=$HOME/.mana-slurm-$SLURM_JOB_ID.rc 59 | else 60 | MANA_RC=$HOME/.mana.rc 61 | fi 62 | 63 | $dir/dmtcp_coordinator $options --exit-on-last -q --daemon --status-file $MANA_RC && coordinator_found=1 64 | set +x 65 | if [ $coordinator_found -eq 0 ]; then 66 | exit 3 67 | fi 68 | 69 | if [ ! -z "$SLURM_JOB_ID" ]; then 70 | if [ -e "$MANA_RC" ]; then 71 | echo "SLURM_JOB_ID: $SLURM_JOB_ID" >> $MANA_RC 72 | else 73 | echo '*** MANA problem: Running in Slurm job ' \ 74 | "$SLURM_JOB_ID; $MANA_RC not found" 75 | fi 76 | fi 77 | echo '' >> $MANA_RC 78 | echo '# This is a temporary file for communication between' >> $MANA_RC 79 | echo '# mana_coordinator and mana_launch/mana_restart.' >> $MANA_RC 80 | echo '*** '"Coordinator/job information written to $MANA_RC" 81 | 82 | # srun -n1 -c1 --cpu-bind=cores bin/dmtcp_launch -i10 -h `hostname` --no-gzip --join --disable-dl-plugin --with-plugin $PWD/lib/dmtcp/libmana.so contrib/mpi-proxy-split/test/mpi_hello_world.mana.exe 83 | 84 | # srun -n1 -c1 --cpu-bind=cores bin/mana_launch -i10 contrib/mpi-proxy-split/test/mpi_hello_world.mana.exe 85 | -------------------------------------------------------------------------------- /mpi-proxy-split/test-ckpt-restart/ping_pong.c: -------------------------------------------------------------------------------- 1 | // Author: Wes Kendall 2 | // Copyright 2011 www.mpitutorial.com 3 | // This code is provided freely with the tutorials on mpitutorial.com. Feel 4 | // free to modify it for your own use. Any distribution of the code must 5 | // either provide a link to www.mpitutorial.com or keep this header in tact. 6 | // 7 | // Ping pong example with MPI_Send and MPI_Recv. Two processes ping pong a 8 | // number back and forth, incrementing it until it reaches a given value. 9 | // 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | int main(int argc, char** argv) { 17 | const int PING_PONG_LIMIT = 100; 18 | 19 | // Initialize the MPI environment 20 | MPI_Init(NULL, NULL); 21 | // Find out rank, size 22 | int world_rank; 23 | MPI_Comm_rank(MPI_COMM_WORLD, &world_rank); 24 | int world_size; 25 | MPI_Comm_size(MPI_COMM_WORLD, &world_size); 26 | 27 | // We are assuming at least 2 processes for this task 28 | if (world_size % 2 != 0) { 29 | fprintf(stderr, "World size must be a multiple of two for %s\n", argv[0]); 30 | MPI_Abort(MPI_COMM_WORLD, 1); 31 | } 32 | 33 | int ping_pong_count = 0; 34 | int partner_rank = world_rank % 2 == 0 ? 35 | (world_rank + 1) : (world_rank - 1); 36 | while (ping_pong_count < PING_PONG_LIMIT) { 37 | if ((world_rank + ping_pong_count) % 2) { 38 | // Increment the ping pong count before you send it 39 | ping_pong_count++; 40 | printf("[%d] Sending %d to %d\n", 41 | world_rank, ping_pong_count, partner_rank); fflush(stdout); 42 | MPI_Send(&ping_pong_count, 1, MPI_INT, partner_rank, 0, MPI_COMM_WORLD); 43 | printf("[%d] Sent ping_pong_count %d to %d\n", 44 | world_rank, ping_pong_count, partner_rank); fflush(stdout); 45 | } else { 46 | printf("[%d] Receiving from %d\n", 47 | world_rank, partner_rank); fflush(stdout); 48 | MPI_Recv(&ping_pong_count, 1, MPI_INT, partner_rank, 0, MPI_COMM_WORLD, 49 | MPI_STATUS_IGNORE); 50 | assert((world_rank + ping_pong_count) % 2 != 0); 51 | printf("[%d] Received ping_pong_count %d from %d\n", 52 | world_rank, ping_pong_count, partner_rank); fflush(stdout); 53 | } 54 | sleep(5); 55 | } 56 | MPI_Finalize(); 57 | } 58 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/File_read_write_test.c: -------------------------------------------------------------------------------- 1 | /* 2 | Test for the MPI_File_read and MPI_File_write methods 3 | Also tests MPI_Seek method 4 | 5 | Defaults to 10 iterations 6 | Intended to be run with mana_test.py 7 | 8 | */ 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #define BUF_SIZE 8 18 | 19 | int 20 | main(int argc, char *argv[]) 21 | { 22 | // Parse runtime argument 23 | int max_iterations = 10; // default 24 | if (argc != 1) { 25 | max_iterations = atoi(argv[1]); 26 | } 27 | 28 | int access_mode, ret; 29 | MPI_Comm comm; 30 | 31 | MPI_Init(&argc, &argv); 32 | comm = MPI_COMM_WORLD; 33 | 34 | // Open file 35 | MPI_File handle; 36 | access_mode = MPI_MODE_CREATE | MPI_MODE_RDWR | MPI_MODE_DELETE_ON_CLOSE; 37 | ret = MPI_File_open(comm, "file.tmp", access_mode, MPI_INFO_NULL, &handle); 38 | assert(ret == MPI_SUCCESS); 39 | 40 | // Initialize input/output buffers 41 | int writebuf[BUF_SIZE]; 42 | int readbuf[BUF_SIZE]; 43 | for (int i = 0; i < BUF_SIZE; i++) { 44 | writebuf[i] = i; 45 | } 46 | 47 | for (int iterations = 0; iterations < max_iterations; iterations++) { 48 | // Write to file 49 | ret = MPI_File_write(handle, writebuf, BUF_SIZE, MPI_INT, 50 | MPI_STATUS_IGNORE); 51 | assert(ret == MPI_SUCCESS); 52 | // Reset file location before checkpoint to test that the offset 53 | // is correctly being restored 54 | ret = MPI_File_seek(handle, sizeof(int)*BUF_SIZE*iterations, 55 | MPI_SEEK_SET); 56 | assert(ret == MPI_SUCCESS); 57 | 58 | sleep(5); 59 | 60 | // Read from file and check results 61 | ret = MPI_File_read(handle, readbuf, BUF_SIZE, MPI_INT, 62 | MPI_STATUS_IGNORE); 63 | assert(ret == MPI_SUCCESS); 64 | 65 | // Update buffer for next iteration 66 | for (int i = 0; i < BUF_SIZE; i++) { 67 | assert(readbuf[i] == writebuf[i]); 68 | } 69 | 70 | for (int i = 0; i < BUF_SIZE; i++) { 71 | writebuf[i]++; 72 | } 73 | fprintf(stderr, "Iteration %d complete\n", iterations); 74 | fflush(stderr); 75 | } 76 | ret = MPI_File_close(&handle); 77 | assert(ret == MPI_SUCCESS); 78 | 79 | MPI_Finalize(); 80 | return 0; 81 | } 82 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/Type_dup_test.c: -------------------------------------------------------------------------------- 1 | /* 2 | Test for the MPI_Type_dup method 3 | 4 | Run with 2 ranks 5 | Defaults to 10000 iterations 6 | Intended to be run with mana_test.py 7 | 8 | Inspired by: 9 | https://rookiehpc.github.io/mpi/docs/mpi_type_create_hvector/index.html 10 | */ 11 | 12 | #define _POSIX_C_SOURCE 199309L 13 | 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #define BUFFER_SIZE 100 22 | 23 | int 24 | main(int argc, char **argv) 25 | { 26 | // Parse runtime argument 27 | int max_iterations = 10000; // default 28 | if (argc != 1) { 29 | max_iterations = atoi(argv[1]); 30 | } 31 | 32 | int rank, comm_size; 33 | 34 | MPI_Init(&argc, &argv); 35 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 36 | MPI_Comm_size(MPI_COMM_WORLD, &comm_size); 37 | if (rank == 0) { 38 | printf("Comm. size = %d\n", comm_size); 39 | printf("Running test for %d iterations\n", max_iterations); 40 | } 41 | fflush(stdout); 42 | assert(comm_size == 2); 43 | 44 | MPI_Datatype column_type[2]; 45 | MPI_Type_vector(BUFFER_SIZE, 1, BUFFER_SIZE, MPI_INT, &column_type[0]); 46 | MPI_Type_commit(&column_type[0]); 47 | int buffer[BUFFER_SIZE][BUFFER_SIZE]; 48 | int recvbuf[BUFFER_SIZE]; 49 | int expected_output[BUFFER_SIZE][BUFFER_SIZE]; 50 | for (int i = 0; i < max_iterations; i++) { 51 | MPI_Type_dup(column_type[i & 1], &column_type[(i+1) & 1]); 52 | MPI_Type_free(&column_type[i & 1]); 53 | if (i % 100 == 99) { 54 | printf("Completed %d iterations\n", i + 1); 55 | fflush(stdout); 56 | } 57 | for (int j = 0; j < BUFFER_SIZE; j++) { 58 | for (int k = 0; k < BUFFER_SIZE; k++) { 59 | buffer[j][k] = j + k + i; 60 | expected_output[j][k] = j + k + i; 61 | } 62 | } 63 | 64 | if (rank == 0) { 65 | int ret = MPI_Send(&buffer[0][1], 1, column_type[(i+1) & 1], 1, 0, 66 | MPI_COMM_WORLD); 67 | assert(ret == MPI_SUCCESS); 68 | } else { 69 | int ret = MPI_Recv(&recvbuf, BUFFER_SIZE, MPI_INT, 0, 0, MPI_COMM_WORLD, 70 | MPI_STATUS_IGNORE); 71 | assert(ret == MPI_SUCCESS); 72 | for (int i = 0; i < BUFFER_SIZE; i++) { 73 | assert(recvbuf[i] == buffer[i][1]); 74 | } 75 | } 76 | } 77 | MPI_Finalize(); 78 | return EXIT_SUCCESS; 79 | } -------------------------------------------------------------------------------- /mpi-proxy-split/test/File_read_write_all_test.c: -------------------------------------------------------------------------------- 1 | /* 2 | Test for the MPI_File_read_at_all and MPI_File_write_at_all methods 3 | 4 | Defaults to 1000000 iterations 5 | Intended to be run with mana_test.py 6 | 7 | Run with >2 ranks for non-trivial results 8 | 9 | */ 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | #define BUF_SIZE 8 19 | 20 | int 21 | main(int argc, char *argv[]) 22 | { 23 | // Parse runtime argument 24 | int max_iterations = 1000000; // default 25 | if (argc != 1) { 26 | max_iterations = atoi(argv[1]); 27 | } 28 | 29 | int access_mode, ret, rank, size; 30 | MPI_Comm comm; 31 | 32 | MPI_Init(&argc, &argv); 33 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 34 | MPI_Comm_size(MPI_COMM_WORLD, &size); 35 | comm = MPI_COMM_WORLD; 36 | 37 | // Open file 38 | MPI_File handle; 39 | access_mode = MPI_MODE_CREATE | MPI_MODE_RDWR | MPI_MODE_DELETE_ON_CLOSE; 40 | ret = MPI_File_open(comm, "file.tmp", access_mode, MPI_INFO_NULL, &handle); 41 | assert(ret == MPI_SUCCESS); 42 | 43 | // Initialize input/output buffers 44 | int writebuf[BUF_SIZE]; 45 | int readbuf[BUF_SIZE]; 46 | 47 | ret = MPI_File_set_size(handle, BUF_SIZE*size); 48 | 49 | for (int iterations = 0; iterations < max_iterations; iterations++) { 50 | for (int i = 0; i < BUF_SIZE; i++) { 51 | writebuf[i] = i+rank; 52 | } 53 | // Write to file 54 | ret = MPI_File_write_at_all(handle, (MPI_Offset)rank*sizeof(int)*BUF_SIZE, 55 | writebuf, BUF_SIZE, MPI_INT, MPI_STATUS_IGNORE); 56 | assert(ret == MPI_SUCCESS); 57 | 58 | // Read from file and check results 59 | ret = MPI_File_read_at_all(handle, (MPI_Offset)rank*sizeof(int)*BUF_SIZE, 60 | readbuf, BUF_SIZE, MPI_INT, MPI_STATUS_IGNORE); 61 | assert(ret == MPI_SUCCESS); 62 | 63 | // Update buffer for next iteration 64 | for (int i = 0; i < BUF_SIZE; i++) { 65 | assert(readbuf[i] == writebuf[i]); 66 | } 67 | 68 | for (int i = 0; i < BUF_SIZE; i++) { 69 | writebuf[i]++; 70 | } 71 | fprintf(stderr, "Iteration %d complete\n", iterations); 72 | fflush(stderr); 73 | } 74 | ret = MPI_File_close(&handle); 75 | assert(ret == MPI_SUCCESS); 76 | 77 | MPI_Finalize(); 78 | return 0; 79 | } 80 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/mpi_hello_world.c: -------------------------------------------------------------------------------- 1 | // Author: Wes Kendall 2 | // Copyright 2011 www.mpitutorial.com 3 | // This code is provided freely with the tutorials on mpitutorial.com. Feel 4 | // free to modify it for your own use. Any distribution of the code must 5 | // either provide a link to www.mpitutorial.com or keep this header intact. 6 | // 7 | // An intro MPI hello world program that uses MPI_Init, MPI_Comm_size, 8 | // MPI_Comm_rank, MPI_Finalize, and MPI_Get_processor_name. 9 | // 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | int 18 | main(int argc, char **argv) 19 | { 20 | void *ret = mmap(NULL, 0x1000, PROT_READ|PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); 21 | printf("ret: %p\n", ret); 22 | // Initialize the MPI environment. The two arguments to MPI Init are not 23 | // currently used by MPI implementations, but are there in case future 24 | // implementations might need the arguments. 25 | MPI_Init(NULL, NULL); 26 | 27 | // Get the number of processes 28 | int world_size; 29 | MPI_Comm_size(MPI_COMM_WORLD, &world_size); 30 | 31 | // Get the rank of the process 32 | int world_rank; 33 | MPI_Comm_rank(MPI_COMM_WORLD, &world_rank); 34 | int original_rank = world_rank, original_size = world_size; 35 | // Get the name of the processor 36 | char processor_name[MPI_MAX_PROCESSOR_NAME] = { "test" }; 37 | // int name_len; 38 | // MPI_Get_processor_name(processor_name, &name_len); 39 | 40 | MPI_Comm_rank(MPI_COMM_WORLD, &world_rank); 41 | MPI_Comm_size(MPI_COMM_WORLD, &world_size); 42 | assert(world_rank == original_rank); 43 | assert(world_size == original_size); 44 | // Print off a hello world message 45 | printf("Hello world from processor %s, rank %d out of %d processors\n", 46 | processor_name, world_rank, world_size); 47 | fflush(stdout); 48 | 49 | printf("Will now sleep for 500 seconds ...\n"); 50 | fflush(stdout); 51 | unsigned int remaining = 300; 52 | while (remaining > 0) { 53 | remaining = sleep(remaining); 54 | if (remaining > 0) { 55 | printf("Signal received; continuing sleep for %d seconds.\n", remaining); 56 | fflush(stdout); 57 | } 58 | } 59 | printf("**** %s is now exiting.\n", argv[0]); 60 | 61 | // Finalize the MPI environment. No more MPI calls can be made after this 62 | MPI_Finalize(); 63 | } 64 | -------------------------------------------------------------------------------- /util/git-bisect-m32.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copy this script to the DMTCP root director. 4 | # Then modify this script first for your purposes. 5 | # In particular, look for the two grep strings (test if good, test if bad), and 6 | # modify them for your test. 7 | # Then execute something like: 8 | # git bisect start 9 | # git bisect bad 10 | # git bisect good 2.0 # tag 2.0, or pick any older commit that was good 11 | # git bisect run ./git-bisect.sh 12 | 13 | # While this is executing, in another window, you can see the progress with: 14 | # git bisect log 15 | # To modify and redo: 16 | # git bisect log > git-bisect.log 17 | # tail -f git-bisect.out 18 | 19 | # The bad commit could be a false alarm witin a range of good commit. 20 | # To analyze a given range, try something like: 21 | # git log --oneline 1a7d8dbb0dc48..master | wc 22 | # git checkout master~ # where is the number of lines, from wc 23 | # and try testing a few more recent branches 24 | # Then EDIT git-bisect.log 25 | # git bisect replay git-bisect.log 26 | # git bisect ./git-bisect.sh 27 | 28 | 29 | # Test if good 30 | git log -1 > git-bisect.out 31 | echo 32 | echo "************ NEW RUN **************" 33 | git log -1 | cat 34 | good_match_1='ckpt:[\t ]*PASSED[\t ]*rstr:[\t ]*PASSED[\t ]*;' 35 | good_match_2='[\t ]*ckpt:[\t ]*PASSED[\t ]*rstr:[\t ]*PASSED' 36 | good_match="$good_match_1$good_match_2" 37 | # older DMTCP had Makefile that tried to call automake 38 | # older DMTCP required '.' in PATH for 'cd mtcp && make build' 39 | # older DMTCP had dmtcp/src directory 40 | # older DMTCP needed mtcp_restart instead of mtcp_restart-32 41 | ./configure --enable-m32 AUTOMAKE=echo && make -j clean && rm -rf bin lib && \ 42 | (PATH=$PATH:. make -j) && (cd src && make -j || true) && \ 43 | (cd dmtcp/src && make -j || true) && \ 44 | (mkdir -p bin && cd bin && ln -sf ../lib/dmtcp/32/bin/* ./ || true) && \ 45 | (cd lib/dmtcp && ln -sf 32/lib/dmtcp/* ./ || true) && \ 46 | (cd lib/dmtcp/32/bin && ln -s mtcp_restart-32 mtcp_restart || true) && \ 47 | make -j check-dmtcp1 >> git-bisect.out 2>&1 && \ 48 | grep "$good_match" git-bisect.out \ 49 | > /dev/null && \ 50 | exit 0 51 | 52 | # Test if bad 53 | # We consider a run "bad" only if it compiled and started autotext.py. 54 | # Anything else should be skipped as a problem for that single revision. 55 | grep 'ckpt:' git-bisect.out > /dev/null && exit 1 56 | 57 | # Skip it (unsure) 58 | exit 125 59 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/Scatterv_test.c: -------------------------------------------------------------------------------- 1 | /* 2 | Test for the MPI_Scatterv method 3 | 4 | Run with >2 ranks for non-trivial results 5 | Defaults to 10000 iterations 6 | Intended to be run with mana_test.py 7 | */ 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #define MAX_PROCESSES 10 15 | 16 | int 17 | main(int argc, char **argv) 18 | { 19 | // Parse runtime argument 20 | int max_iterations = 10000; // default 21 | if (argc != 1) { 22 | max_iterations = atoi(argv[1]); 23 | } 24 | 25 | int rank, size, i, j; 26 | int table[MAX_PROCESSES][MAX_PROCESSES]; 27 | int row[MAX_PROCESSES]; 28 | int errors = 0; 29 | int participants; 30 | int displs[MAX_PROCESSES]; 31 | int send_counts[MAX_PROCESSES]; 32 | 33 | MPI_Init(&argc, &argv); 34 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 35 | MPI_Comm_size(MPI_COMM_WORLD, &size); 36 | 37 | if (rank == 0) { 38 | printf("Running test for %d iterations\n", max_iterations); 39 | } 40 | 41 | /* A maximum of MAX_PROCESSES processes can participate */ 42 | if (size > MAX_PROCESSES) 43 | participants = MAX_PROCESSES; 44 | else 45 | participants = size; 46 | if (rank < participants) { 47 | for (int iterations = 0; iterations < max_iterations; iterations++) { 48 | int recv_count = MAX_PROCESSES; 49 | 50 | /* If I'm the root (process 0), then fill out the big 51 | table and setup send_counts and displs arrays */ 52 | if (rank == 0) 53 | for (i = 0; i < participants; i++) { 54 | send_counts[i] = recv_count; 55 | displs[i] = i * MAX_PROCESSES; 56 | for (j = 0; j < MAX_PROCESSES; j++) 57 | table[i][j] = i + j; 58 | } 59 | 60 | /* Scatter the big table to everybody's little table */ 61 | int ret = MPI_Scatterv(&table[0][0], send_counts, displs, MPI_INT, 62 | &row[0], recv_count, MPI_INT, 0, MPI_COMM_WORLD); 63 | assert(ret == MPI_SUCCESS); 64 | 65 | /* Now see if our row looks right */ 66 | for (i = 0; i < MAX_PROCESSES; i++) { 67 | printf("[Rank =%d] recved = %d \t expected = %d\n", rank, row[i], 68 | i + rank); 69 | fflush(stdout); 70 | assert(row[i] == i + rank); 71 | } 72 | 73 | for (i = 0; i < MAX_PROCESSES; i++) { 74 | row[i] = 0; 75 | } 76 | } 77 | } 78 | 79 | MPI_Finalize(); 80 | return 0; 81 | } 82 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/Isend_test.c: -------------------------------------------------------------------------------- 1 | /* 2 | Test for the MPI_Isend method 3 | 4 | Must run with >1 ranks 5 | Defaults to 5 iterations 6 | Intended to be run with mana_test.py 7 | 8 | */ 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #define BUFFER_SIZE 3 18 | #define SLEEP_PER_ITERATION 5 19 | 20 | int 21 | main(int argc, char **argv) 22 | { 23 | // Parse runtime argument 24 | int max_iterations = 5; // default 25 | if (argc != 1) { 26 | max_iterations = atoi(argv[1]); 27 | } 28 | 29 | // Initialize the MPI environment 30 | MPI_Init(NULL, NULL); 31 | // Find out rank and size 32 | int world_rank; 33 | MPI_Comm_rank(MPI_COMM_WORLD, &world_rank); 34 | int world_size; 35 | MPI_Comm_size(MPI_COMM_WORLD, &world_size); 36 | 37 | if (world_rank == 0) { 38 | printf("Running test for %d iterations\n", max_iterations); 39 | } 40 | 41 | // We are assuming at least 2 processes for this task 42 | if (world_size < 2) { 43 | fprintf(stderr, "World size must be greater than 1 for %s\n", argv[0]); 44 | MPI_Abort(MPI_COMM_WORLD, 1); 45 | } 46 | 47 | int rank = 0; 48 | int number = 11223344; 49 | MPI_Request reqs[world_size]; 50 | 51 | for (int iterations = 0; iterations < max_iterations; iterations++) { 52 | for (rank = 0; rank < world_size; rank++) { 53 | if (rank == world_rank) 54 | continue; 55 | int ret = 56 | MPI_Isend(&number, 1, MPI_INT, rank, 0, MPI_COMM_WORLD, &reqs[rank]); 57 | assert(ret == MPI_SUCCESS); 58 | } 59 | printf("%d sleeping\n", world_rank); 60 | fflush(stdout); 61 | sleep(SLEEP_PER_ITERATION); 62 | for (rank = 0; rank < world_size; rank++) { 63 | if (rank == world_rank) 64 | continue; 65 | int flag = 0; 66 | while (!flag) { 67 | MPI_Test(&reqs[rank], &flag, MPI_STATUS_IGNORE); 68 | } 69 | } 70 | 71 | for (rank = 0; rank < world_size; rank++) { 72 | if (rank == world_rank) 73 | continue; 74 | int ret = MPI_Recv(&number, 1, MPI_INT, rank, 0, MPI_COMM_WORLD, 75 | MPI_STATUS_IGNORE); 76 | assert(ret == MPI_SUCCESS); 77 | assert(number == 11223344 + iterations); 78 | printf("%d received %d from %d\n", world_rank, number, rank); 79 | fflush(stdout); 80 | } 81 | number++; 82 | } 83 | MPI_Finalize(); 84 | } 85 | -------------------------------------------------------------------------------- /mpi-proxy-split/mpi-wrappers/fortran_constants.f90: -------------------------------------------------------------------------------- 1 | ! Tested on Cori with: ftn -c THIS_FILE.f90 2 | ! C binding: https://gcc.gnu.org/onlinedocs/gfortran/Interoperable-Subroutines-and-Functions.html 3 | ! MPICH: integer(c_int),bind(C, name=" MPIR_F08_MPI_IN_PLACE ") & :: MPI_IN_PLACE 4 | ! FIXME: Make this cleaner, either use bind() command, or use an out parameter. 5 | 6 | ! NOTE: In MPI, the FORTRAN constants can have different values from 7 | ! one process to the next. So, we need to discover the 8 | ! FORTRAN constants in each new MPI process, to recognize 9 | ! them at C level. 10 | ! For example, note that the MPI_Allreduce wrapper contains: 11 | ! if (sendbuf == FORTRAN_MPI_IN_PLACE) { 12 | ! sendbuf = MPI_IN_PLACE; 13 | ! } 14 | ! MPI 3.1 standard: 15 | ! The constants that cannot be used in initialization expressions or 16 | ! assignments in Fortran are as follows: 17 | ! MPI_BOTTOM 18 | ! MPI_STATUS_IGNORE 19 | ! MPI_STATUSES_IGNORE 20 | ! MPI_ERRCODES_IGNORE 21 | ! MPI_IN_PLACE 22 | ! MPI_ARGV_NULL 23 | ! MPI_ARGVS_NULL 24 | ! MPI_UNWEIGHTED 25 | ! MPI_WEIGHTS_EMPTY 26 | 27 | 28 | subroutine get_fortran_constants() 29 | implicit none 30 | include 'mpif.h' 31 | 32 | ! explicit interfaces 33 | interface 34 | subroutine get_fortran_constants_helper(t) 35 | implicit none 36 | integer,intent(in) :: t 37 | end subroutine get_fortran_constants_helper 38 | subroutine get_fortran_arrays_ignore(t) 39 | implicit none 40 | integer :: t(*) 41 | end subroutine get_fortran_arrays_ignore 42 | end interface 43 | ! These must match the list in get_fortran_constants.c 44 | call get_fortran_constants_helper(MPI_BOTTOM) 45 | ! MPI_STATUS_IGNORE is a struct, similar to a length-1 array 46 | call get_fortran_arrays_helper(MPI_STATUS_IGNORE) 47 | call get_fortran_arrays_helper(MPI_STATUSES_IGNORE) 48 | call get_fortran_arrays_helper(MPI_ERRCODES_IGNORE) 49 | call get_fortran_constants_helper(MPI_IN_PLACE) 50 | ! FIXME: MPI_ARGV_NULL is a CHARACTER(1), not supported in MANA 51 | ! call get_fortran_constants_helper(MPI_ARGV_NULL) 52 | ! FIXME: MPI_ARGV_NULL is a CHARACTER(1) in mpich-gnu, not supported 53 | ! in MANA 54 | ! call get_fortran_arrays_helper(MPI_ARGVS_NULL) 55 | end subroutine get_fortran_constants 56 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/ping_pong.c: -------------------------------------------------------------------------------- 1 | // Author: Wes Kendall 2 | // Copyright 2011 www.mpitutorial.com 3 | // This code is provided freely with the tutorials on mpitutorial.com. Feel 4 | // free to modify it for your own use. Any distribution of the code must 5 | // either provide a link to www.mpitutorial.com or keep this header in tact. 6 | // 7 | // Ping pong example with MPI_Send and MPI_Recv. Two processes ping pong a 8 | // number back and forth, incrementing it until it reaches a given value. 9 | // 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | int 17 | main(int argc, char **argv) 18 | { 19 | const int PING_PONG_LIMIT = 10; 20 | 21 | // Initialize the MPI environment 22 | MPI_Init(NULL, NULL); 23 | // Find out rank, size 24 | int world_rank; 25 | MPI_Comm_rank(MPI_COMM_WORLD, &world_rank); 26 | int world_size; 27 | MPI_Comm_size(MPI_COMM_WORLD, &world_size); 28 | 29 | // We are assuming at least 2 processes for this task 30 | if (world_size % 2 != 0) { 31 | fprintf(stderr, "World size must be a multiple of two for %s\n", argv[0]); 32 | MPI_Abort(MPI_COMM_WORLD, 1); 33 | } 34 | 35 | int ping_pong_count = 0; 36 | int partner_rank = world_rank % 2 == 0 ? (world_rank + 1) : (world_rank - 1); 37 | while (ping_pong_count < PING_PONG_LIMIT) { 38 | if ((world_rank + ping_pong_count) % 2) { 39 | // Increment the ping pong count before you send it 40 | ping_pong_count++; 41 | printf("[%d] Sending %d to %d\n", world_rank, ping_pong_count, 42 | partner_rank); 43 | fflush(stdout); 44 | fflush(stdout); 45 | MPI_Send(&ping_pong_count, 1, MPI_INT, partner_rank, 0, MPI_COMM_WORLD); 46 | printf("[%d] Sent ping_pong_count %d to %d\n", world_rank, 47 | ping_pong_count, partner_rank); 48 | fflush(stdout); 49 | fflush(stdout); 50 | } else { 51 | printf("[%d] Receiving from %d\n", world_rank, partner_rank); 52 | fflush(stdout); 53 | fflush(stdout); 54 | MPI_Recv(&ping_pong_count, 1, MPI_INT, partner_rank, 0, MPI_COMM_WORLD, 55 | MPI_STATUS_IGNORE); 56 | fflush(stdout); 57 | assert((world_rank + ping_pong_count) % 2 != 0); 58 | printf("[%d] Received ping_pong_count %d from %d\n", world_rank, 59 | ping_pong_count, partner_rank); 60 | fflush(stdout); 61 | fflush(stdout); 62 | } 63 | sleep(5); 64 | } 65 | MPI_Finalize(); 66 | } 67 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/large_async_p2p.c: -------------------------------------------------------------------------------- 1 | /* 2 | Test for the async P2P methods with large payload 3 | 4 | Must run with 2 ranks 5 | Defaults to 5 iterations 6 | Intended to be run with mana_test.py 7 | */ 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #define MSG_SIZE 256 * 1024 // large message 17 | #define SLEEP_PER_ITERATION 5 18 | 19 | int 20 | main(int argc, char **argv) 21 | { 22 | // Parse runtime argument 23 | int max_iterations = 5; // default 24 | if (argc != 1) { 25 | max_iterations = atoi(argv[1]); 26 | } 27 | 28 | char *data = malloc(MSG_SIZE); 29 | int counter = 0; 30 | char *recv_buf = malloc(MSG_SIZE); 31 | // Initialize the MPI environment 32 | MPI_Init(NULL, NULL); 33 | // Find out rank, size 34 | int world_size, world_rank; 35 | MPI_Comm_rank(MPI_COMM_WORLD, &world_rank); 36 | MPI_Comm_size(MPI_COMM_WORLD, &world_size); 37 | MPI_Request req; 38 | 39 | if (world_rank == 0) { 40 | printf("Running test for %d iterations\n", max_iterations); 41 | } 42 | 43 | // We are assuming 2 processes for this task 44 | if (world_size != 2) { 45 | fprintf(stderr, "World size must be 2 for %s\n", argv[0]); 46 | MPI_Abort(MPI_COMM_WORLD, 1); 47 | } 48 | 49 | for (int iterations = 0; iterations < max_iterations; iterations++) { 50 | for (int i = 0; i < MSG_SIZE; i++) { 51 | data[i] = i + iterations; 52 | } 53 | if (world_rank == 0) { 54 | printf("Rank 0 sleeping\n"); 55 | fflush(stdout); 56 | sleep(SLEEP_PER_ITERATION); 57 | printf("Rank 0 draining from rank 1\n"); 58 | fflush(stdout); 59 | int ret = 60 | MPI_Irecv(recv_buf, MSG_SIZE, MPI_BYTE, 1, 0, MPI_COMM_WORLD, &req); 61 | assert(ret == MPI_SUCCESS); 62 | ret = MPI_Wait(&req, MPI_STATUSES_IGNORE); 63 | assert(ret == MPI_SUCCESS); 64 | printf("Rank 0 drained the message\n"); 65 | assert(memcmp(data, recv_buf, MSG_SIZE) == 0); 66 | fflush(stdout); 67 | } 68 | 69 | if (world_rank == 1) { 70 | printf("Rank 1 sending to rank 0\n"); 71 | fflush(stdout); 72 | int ret = MPI_Isend(data, MSG_SIZE, MPI_BYTE, 0, 0, MPI_COMM_WORLD, &req); 73 | assert(ret == MPI_SUCCESS); 74 | ret = MPI_Wait(&req, MPI_STATUSES_IGNORE); 75 | assert(ret == MPI_SUCCESS); 76 | } 77 | } 78 | 79 | free(data); 80 | free(recv_buf); 81 | MPI_Finalize(); 82 | } 83 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/Allgather_test.c: -------------------------------------------------------------------------------- 1 | /* 2 | Test for the MPI_Allgather method 3 | 4 | Run with >2 ranks for non-trivial results 5 | Defaults to 10000 iterations 6 | Intended to be run with mana_test.py 7 | */ 8 | 9 | #define _POSIX_C_SOURCE 199309L 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | #define BUFFER_SIZE 100 19 | 20 | int 21 | main(int argc, char **argv) 22 | { 23 | // Parse runtime argument 24 | int max_iterations; 25 | max_iterations = 10000; 26 | if (argc != 1) { 27 | max_iterations = atoi(argv[1]); 28 | } 29 | int rank, comm_size; 30 | 31 | MPI_Init(&argc, &argv); 32 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 33 | MPI_Comm_size(MPI_COMM_WORLD, &comm_size); 34 | if (rank == 0) { 35 | printf("Comm. size = %d\n", comm_size); 36 | printf("Running test for %d iterations\n", max_iterations); 37 | } 38 | fflush(stdout); 39 | assert(comm_size > 0); 40 | 41 | int send_buf[BUFFER_SIZE] = { 0 }; 42 | int *recv_buf = (int *)malloc(comm_size * BUFFER_SIZE * sizeof(int)); 43 | for (int iterations = 0; iterations < max_iterations; iterations++) { 44 | for (int i = 0; i < BUFFER_SIZE; i++) { 45 | send_buf[i] = (rank + 1) * 100 + i + iterations; 46 | } 47 | 48 | int ret = MPI_Allgather(send_buf, BUFFER_SIZE, MPI_INT, recv_buf, 49 | BUFFER_SIZE, MPI_INT, MPI_COMM_WORLD); 50 | assert(ret == MPI_SUCCESS); 51 | 52 | int total_rank = 0; 53 | for (int i = 0; i < comm_size; i++) { 54 | // Infer rank of this section of buffer from first value 55 | int buf_portion_rank = (recv_buf[i * BUFFER_SIZE] - iterations) / 100 - 1; 56 | total_rank += buf_portion_rank; 57 | for (int j = 0; j < BUFFER_SIZE; j++) { 58 | assert(recv_buf[i * BUFFER_SIZE + j] == 59 | (buf_portion_rank + 1) * 100 + j + iterations); 60 | } 61 | } 62 | 63 | // Check that each rank is present exactly once 64 | assert(total_rank == (comm_size * (comm_size - 1)) / 2); 65 | 66 | printf("[Rank = %d]: received correctly!\n", rank); 67 | fflush(stdout); 68 | for (int i = 0; i < comm_size * BUFFER_SIZE; i++) { 69 | #ifdef DEBUG 70 | printf("[Rank = %d]: receive buffer[%d] = %d\n", rank, i, recv_buf[i]); 71 | fflush(stdout); 72 | #endif 73 | 74 | // clear the buffer 75 | recv_buf[i] = 0; 76 | } 77 | } 78 | 79 | free(recv_buf); 80 | MPI_Finalize(); 81 | return EXIT_SUCCESS; 82 | } 83 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/Type_create_hindexed_test.c: -------------------------------------------------------------------------------- 1 | /* 2 | Test for the MPI_Type_create_hindexed method 3 | 4 | Due to MANA implementatino, also tests MPI_Type_hindexed 5 | 6 | Run with 2 ranks 7 | Defaults to 10000 iterations 8 | Intended to be run with mana_test.py 9 | 10 | Inspired by: 11 | http://mpi.deino.net/mpi_functions/MPI_Type_create_hindexed.html 12 | */ 13 | 14 | #define _POSIX_C_SOURCE 199309L 15 | 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | int 24 | main(int argc, char **argv) 25 | { 26 | // Parse runtime argument 27 | int max_iterations = 10000; // default 28 | if (argc != 1) { 29 | max_iterations = atoi(argv[1]); 30 | } 31 | 32 | int rank, comm_size; 33 | 34 | MPI_Init(&argc, &argv); 35 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 36 | MPI_Comm_size(MPI_COMM_WORLD, &comm_size); 37 | if (rank == 0) { 38 | printf("Comm. size = %d\n", comm_size); 39 | printf("Running test for %d iterations\n", max_iterations); 40 | } 41 | fflush(stdout); 42 | assert(comm_size == 2); 43 | 44 | MPI_Datatype type, type2; 45 | int blocklen[3] = { 2, 3, 1 }; 46 | MPI_Aint displacement[3] = { 0, 7*sizeof(int), 18*sizeof(int) }; 47 | 48 | MPI_Type_contiguous(3, MPI_INT, &type2); 49 | MPI_Type_commit(&type2); 50 | MPI_Type_create_hindexed(3, blocklen, displacement, type2, &type); 51 | MPI_Type_commit(&type); 52 | 53 | int buffer[21]; 54 | 55 | for (int i = 0; i < max_iterations; i++) { 56 | if (i % 100 == 0) { 57 | fprintf(stderr, "Iteration: %d complete\n", i); 58 | fflush(stderr); 59 | } 60 | for (int j = 0; j < 21; j++) { 61 | buffer[j] = i + j; 62 | } 63 | 64 | if (rank == 0) { 65 | int ret = MPI_Send(buffer, 1, type, 1, 123 + i, MPI_COMM_WORLD); 66 | assert(ret == MPI_SUCCESS); 67 | } else { 68 | memset(buffer, 0, 21*sizeof(int)); 69 | int ret = MPI_Recv(buffer, 1, type, 0, 123 + i, MPI_COMM_WORLD, 70 | MPI_STATUS_IGNORE); 71 | assert(ret == MPI_SUCCESS); 72 | fflush(stderr); 73 | for (int j = 0; j < 21; j++) { 74 | fflush(stderr); 75 | if (j != 6 && j != 16 && j != 17) { 76 | assert(buffer[j] == i+j); 77 | } 78 | else { 79 | assert(buffer[j] == 0); 80 | } 81 | } 82 | } 83 | MPI_Barrier(MPI_COMM_WORLD); 84 | } 85 | MPI_Type_free(&type); 86 | MPI_Type_free(&type2); 87 | MPI_Finalize(); 88 | return EXIT_SUCCESS; 89 | } 90 | -------------------------------------------------------------------------------- /bin/mpicc_mana: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This is a custom mpicc wrapper to link against libmpistub.so 4 | # This is mostly useful when the upper-half (user MPI application) 5 | # is compiled with MPI libraries using constructors, which conflict with MANA's 6 | # kernel loader approach used in lower-half. 7 | 8 | # set custom MPI paths 9 | prefix=$(dirname "$(dirname "$(realpath "$0")")") 10 | libdir=$prefix/lib/dmtcp 11 | CC="gcc -std=gnu99" 12 | 13 | # linker flag to pass through compiler 14 | wl="-Wl," 15 | 16 | # static library suffix 17 | libext="a" 18 | 19 | # shared library suffix 20 | shlibext="so" 21 | 22 | # naming covention for library names 23 | libname_spec="lib\$name" 24 | 25 | # Flag to hardcode $libdir into a binary during linking. 26 | # This must work even if $libdir does not exist. 27 | hardcode_libdir_flag_spec="\${wl}-rpath \${wl}\$libdir" 28 | 29 | # ensure command line arguments are provided 30 | if [ $# -lt 1 ]; then 31 | echo "Error: Command line argument is needed!" 32 | "$0" -help 33 | exit 1 34 | fi 35 | 36 | # processing command line argumnets 37 | linking=yes 38 | allargs=("$@") 39 | argno=0 40 | interlib_deps=yes 41 | static_mpi=no 42 | showinfo="" 43 | 44 | 45 | for arg in "$@" ; do 46 | addarg=yes 47 | case "$arg" in 48 | -c|-S|-E|-M|-MM) 49 | linking=no 50 | ;; 51 | -static) 52 | interlib_deps=no 53 | static_mpi=yes 54 | addarg=no 55 | echo "Does not support Static Library" 56 | exit 0 57 | ;; 58 | -show) 59 | addarg=no 60 | Show=echo 61 | ;; 62 | -show-link-info) 63 | addarg=no 64 | Show=echo 65 | show_info=link 66 | ;; 67 | -show-comile-info) 68 | addarg=no 69 | Show=echo 70 | show_info=compile 71 | ;; 72 | -cc=*) 73 | CC=`echo A$arg | sed -e 's/A-cc=//g'` 74 | addarg=no 75 | ;; 76 | -v) 77 | echo "Custom mpicc for MANA" 78 | if [ "$#" -eq "1" ] ; then 79 | linking=no 80 | fi 81 | ;; 82 | -help) 83 | echo "Usage: mpicc_mana [Options] " 84 | exit 0 85 | ;; 86 | esac 87 | done 88 | 89 | # prevent recursive execution 90 | if [ -n "$MANA_MPICC_RECURSION_CHECK" ] ; then 91 | echo "This script ($0) is being called resursively." 92 | exit 1 93 | fi 94 | 95 | MANA_MPICC_RECURSION_CHECK=1 96 | export MANA_MPICC_RECURSION_CHECK 97 | 98 | # Constructing Compilation and Linking Commands 99 | $Show $CC "${allargs[@]}" -L$libdir -lmpistub 100 | 101 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/File_characteristics_test.c: -------------------------------------------------------------------------------- 1 | /* 2 | Test for the MPI_File_set_view, MPI_File_get_view, MPI_File_get_atomicity, 3 | and MPI_File_set_atomicity 4 | 5 | Defaults to 10 iterations 6 | Intended to be run with mana_test.py 7 | 8 | */ 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #define BUF_SIZE 4096 18 | 19 | int 20 | main(int argc, char *argv[]) 21 | { 22 | // Parse runtime argument 23 | int max_iterations = 10; // default 24 | if (argc != 1) { 25 | max_iterations = atoi(argv[1]); 26 | } 27 | 28 | int access_mode, ret; 29 | MPI_Comm comm; 30 | 31 | MPI_Init(&argc, &argv); 32 | comm = MPI_COMM_WORLD; 33 | 34 | // Open file 35 | MPI_File handle; 36 | access_mode = MPI_MODE_CREATE | MPI_MODE_RDWR | MPI_MODE_DELETE_ON_CLOSE; 37 | ret = MPI_File_open(comm, "file.tmp", access_mode, MPI_INFO_NULL, &handle); 38 | assert(ret == MPI_SUCCESS); 39 | 40 | // Initialize buffers and other local variables 41 | char writebuf[BUF_SIZE]; 42 | char recv_rep[16]; 43 | const char* datareps[3] = {"native", "internal", "external32"}; 44 | for (int i = 0; i < BUF_SIZE; i++) { 45 | writebuf[i] = i; 46 | } 47 | MPI_Datatype etype = MPI_CHAR; 48 | MPI_Datatype ftype = MPI_CHAR; 49 | MPI_Offset disp = 0; 50 | MPI_File_write(handle, writebuf, BUF_SIZE, MPI_INT, MPI_STATUS_IGNORE); 51 | int flag = 1; 52 | 53 | for (int iterations = 0; iterations < max_iterations; iterations++) { 54 | // Set view and atomicity 55 | ret = MPI_File_set_view(handle, (MPI_Offset)iterations, MPI_INT, MPI_INT, 56 | datareps[iterations % 3], MPI_INFO_NULL); 57 | assert(ret == MPI_SUCCESS); 58 | ret = MPI_File_set_atomicity(handle, (iterations & 1)); 59 | assert(ret == MPI_SUCCESS); 60 | 61 | sleep(5); 62 | 63 | // Check view and atomicity 64 | ret = MPI_File_get_view(handle, &disp, &etype, &ftype, recv_rep); 65 | assert(ret == MPI_SUCCESS); 66 | 67 | ret = MPI_File_get_atomicity(handle, &flag); 68 | assert(ret == MPI_SUCCESS); 69 | 70 | assert(disp == iterations); 71 | assert(etype == MPI_INT); 72 | assert(ftype == MPI_INT); 73 | 74 | assert(flag == (iterations & 1)); 75 | 76 | // Update variables for next iteration 77 | etype = MPI_CHAR; 78 | ftype = MPI_CHAR; 79 | disp = 0; 80 | 81 | fprintf(stderr, "Iteration %d complete\n", iterations); 82 | fflush(stderr); 83 | } 84 | ret = MPI_File_close(&handle); 85 | assert(ret == MPI_SUCCESS); 86 | 87 | MPI_Finalize(); 88 | return 0; 89 | } 90 | -------------------------------------------------------------------------------- /mpi-proxy-split/util/mpi-logger/fortran_constants.f90: -------------------------------------------------------------------------------- 1 | ! Tested on Cori with: ftn -c THIS_FILE.f90 2 | ! C binding: https://gcc.gnu.org/onlinedocs/gfortran/Interoperable-Subroutines-and-Functions.html 3 | ! MPICH: integer(c_int),bind(C, name=" MPIR_F08_MPI_IN_PLACE ") & :: MPI_IN_PLACE 4 | ! FIXME: Make this cleaner, either use bind() command, or use an out parameter. 5 | 6 | ! NOTE: In MPI, the FORTRAN constants can have different values from 7 | ! one process to the next. So, we need to discover the 8 | ! FORTRAN constants in each new MPI process, to recognize 9 | ! them at C level. 10 | ! For example, note that the MPI_Allreduce wrapper contains: 11 | ! if (sendbuf == FORTRAN_MPI_IN_PLACE) { 12 | ! retval = NEXT_FUNC(Allreduce)(MPI_IN_PLACE, recvbuf, count, 13 | ! realType, realOp, realComm); 14 | ! MPI 3.1 standard: 15 | ! The constants that cannot be used in initialization expressions or 16 | ! assignments in Fortran are as follows: 17 | ! MPI_BOTTOM 18 | ! MPI_STATUS_IGNORE 19 | ! MPI_STATUSES_IGNORE 20 | ! MPI_ERRCODES_IGNORE 21 | ! MPI_IN_PLACE 22 | ! MPI_ARGV_NULL 23 | ! MPI_ARGVS_NULL 24 | ! MPI_UNWEIGHTED 25 | ! MPI_WEIGHTS_EMPTY 26 | 27 | 28 | subroutine get_fortran_constants() 29 | implicit none 30 | include 'mpif.h' 31 | 32 | ! explicit interfaces 33 | interface 34 | subroutine get_fortran_constants_helper(t) 35 | implicit none 36 | integer,intent(in) :: t 37 | end subroutine get_fortran_constants_helper 38 | subroutine get_fortran_arrays_ignore(t) 39 | implicit none 40 | integer :: t(*) 41 | end subroutine get_fortran_arrays_ignore 42 | end interface 43 | ! These must match the list in get_fortran_constants.c 44 | call get_fortran_constants_helper(MPI_BOTTOM) 45 | ! MPI_STATUS_IGNORE is a struct, similar to a length-1 array 46 | call get_fortran_arrays_helper(MPI_STATUS_IGNORE) 47 | call get_fortran_arrays_helper(MPI_STATUSES_IGNORE) 48 | call get_fortran_arrays_helper(MPI_ERRCODES_IGNORE) 49 | call get_fortran_constants_helper(MPI_IN_PLACE) 50 | ! FIXME: MPI_ARGV_NULL is a CHARACTER(1), not supported in MANA 51 | ! call get_fortran_constants_helper(MPI_ARGV_NULL) 52 | ! FIXME: MPI_ARGV_NULL is a CHARACTER(1) in mpich-gnu, not supported 53 | ! in MANA 54 | ! call get_fortran_arrays_helper(MPI_ARGVS_NULL) 55 | call get_fortran_constants_helper(MPI_UNWEIGHTED) 56 | call get_fortran_constants_helper(MPI_WEIGHTS_EMPTY) 57 | end subroutine get_fortran_constants 58 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/Comm_get_attr_test.c: -------------------------------------------------------------------------------- 1 | /* 2 | Test for the MPI_Comm_get_attr method 3 | 4 | Defaults to 30 iterations 5 | Intended to be run with mana_test.py 6 | 7 | Source: http://mpi.deino.net/mpi_functions/MPI_Comm_dup.html 8 | */ 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #define SLEEP_PER_ITERATION 1 18 | 19 | int 20 | main(int argc, char **argv) 21 | { 22 | // Parse runtime argument 23 | int max_iterations = 30; // default 24 | if (argc != 1) { 25 | max_iterations = atoi(argv[1]); 26 | } 27 | 28 | void *v; 29 | int flag; 30 | int vval; 31 | int rank, size; 32 | 33 | MPI_Init(&argc, &argv); 34 | MPI_Comm_size(MPI_COMM_WORLD, &size); 35 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 36 | 37 | if (rank == 0) { 38 | printf("Running test for %d iterations\n", max_iterations); 39 | } 40 | 41 | for (int iterations = 0; iterations < max_iterations; iterations++) { 42 | MPI_Comm_get_attr(MPI_COMM_WORLD, MPI_TAG_UB, &v, &flag); 43 | assert(flag); 44 | vval = *(int *)v; 45 | assert(vval >= 32767); 46 | printf("Got MPI_TAG_UB\n"); 47 | fflush(stdout); 48 | 49 | MPI_Comm_get_attr(MPI_COMM_WORLD, MPI_HOST, &v, &flag); 50 | assert(flag); 51 | vval = *(int *)v; 52 | assert(!((vval < 0 || vval >= size) && vval != MPI_PROC_NULL)); 53 | printf("Got HOST\n"); 54 | fflush(stdout); 55 | 56 | MPI_Comm_get_attr(MPI_COMM_WORLD, MPI_IO, &v, &flag); 57 | assert(flag); 58 | vval = *(int *)v; 59 | assert(!((vval < 0 || vval >= size) && vval != MPI_ANY_SOURCE && 60 | vval != MPI_PROC_NULL)); 61 | printf("Got MPI_IO\n"); 62 | fflush(stdout); 63 | 64 | MPI_Comm_get_attr(MPI_COMM_WORLD, MPI_WTIME_IS_GLOBAL, &v, &flag); 65 | vval = *(int *)v; 66 | assert(!flag || !(vval < 0 || vval > 1)); 67 | printf("Got MPI_WTIME_IS_GLOBAL\n"); 68 | fflush(stdout); 69 | 70 | MPI_Comm_get_attr(MPI_COMM_WORLD, MPI_APPNUM, &v, &flag); 71 | vval = *(int *)v; 72 | assert(!flag || vval >= 0); 73 | printf("Got MPI_APPNUM\n"); 74 | fflush(stdout); 75 | 76 | MPI_Comm_get_attr(MPI_COMM_WORLD, MPI_UNIVERSE_SIZE, &v, &flag); 77 | vval = *(int *)v; 78 | assert(!flag || vval >= size); 79 | printf("Got MPI_UNIVERSE_SIZE\n"); 80 | fflush(stdout); 81 | 82 | MPI_Comm_get_attr(MPI_COMM_WORLD, MPI_LASTUSEDCODE, &v, &flag); 83 | vval = *(int *)v; 84 | assert(!flag || vval >= MPI_ERR_LASTCODE); 85 | printf("Got MPI_LASTUSEDCODE\n"); 86 | fflush(stdout); 87 | 88 | sleep(SLEEP_PER_ITERATION); 89 | } 90 | MPI_Finalize(); 91 | return 0; 92 | } 93 | -------------------------------------------------------------------------------- /mpi-proxy-split/unit-test/Makefile: -------------------------------------------------------------------------------- 1 | # MPI compiler 2 | include ../Makefile_config 3 | 4 | ifeq (${PERLMUTTER}, 1) 5 | MPIRUN = srun 6 | MPICC = cc 7 | MPICXX = CC 8 | else 9 | MPIRUN = mpirun 10 | MPICC = mpicc 11 | MPICXX = mpic++ 12 | endif 13 | 14 | TESTS = record-replay-comm-test \ 15 | record-replay-group-test \ 16 | record-replay-types-test 17 | 18 | TEST_OBJS=$(addsuffix .o, ${TESTS}) 19 | TEST_BINS=$(addsuffix .exe, ${TESTS}) 20 | 21 | # Modify if your DMTCP_ROOT is located elsewhere. 22 | ifndef DMTCP_ROOT 23 | DMTCP_ROOT=../../dmtcp 24 | endif 25 | DMTCP_INCLUDE=${DMTCP_ROOT}/include 26 | JALIB_INCLUDE=${DMTCP_ROOT}/jalib 27 | LOWER_HALF_INCLUDE=../lower-half 28 | MPI_WRAPPERS_INCLUDE=../mpi-wrappers 29 | 30 | override CFLAGS += -g3 -O0 -fPIC -I${DMTCP_INCLUDE} -I.. 31 | override CXXFLAGS += -g3 -O0 -fPIC -I${DMTCP_INCLUDE} \ 32 | -I.. -I${JALIB_INCLUDE} \ 33 | -I${LOWER_HALF_INCLUDE} \ 34 | -I${MPI_WRAPPERS_INCLUDE} \ 35 | -I${DMTCP_ROOT}/src \ 36 | -D'NEXT_FUNC(fnc)=MPI_\#\#fnc' 37 | 38 | TEST_LD_FLAGS=${DMTCP_ROOT}/src/libdmtcpinternal.a \ 39 | ${DMTCP_ROOT}/src/libjalib.a \ 40 | ${DMTCP_ROOT}/src/libnohijack.a \ 41 | -lgtest -lpthread -ldl 42 | 43 | DRAIN_TEST_OBJS = drain-send-recv-test.o ../p2p_drain_send_recv.cpp \ 44 | ../p2p_log_replay.cpp ${DMTCP_ROOT}/src/lookup_service.o 45 | 46 | default: ${TEST_BINS} 47 | 48 | ../record-replay.o ../drain_send_recv_packets.o: 49 | @make -C .. 50 | 51 | # NOTE: The objects files split_process.o and procmapsutils.o 52 | # below are required only when MANA is configured with SINGLE_CART_REORDER 53 | # C/C++ flag. One should remove these two obj files when we decide to remove 54 | # the SINGLE_CART_REORDER macro implementation entirely from MANA. 55 | %.exe: %.o ../record-replay.o ../split_process.o ../lower-half/procmapsutils.o 56 | ${MPICXX} -fPIC -g3 -O0 -o $@ $^ ${TEST_LD_FLAGS} 57 | 58 | drain-send-recv-test.exe: ${DRAIN_TEST_OBJS} 59 | ${MPICXX} -fPIC -g3 -O0 ${CXXFLAGS} -o $@ $^ ${TEST_LD_FLAGS} 60 | 61 | .c.o: 62 | ${MPICC} ${CFLAGS} -g3 -O0 -c -o $@ $< 63 | 64 | .cpp.o: 65 | ${MPICXX} ${CXXFLAGS} -g3 -O0 -c -o $@ $< 66 | 67 | gdb-%: ${TEST_BINS} 68 | gdb --args ./record-replay-$*-test.exe 69 | 70 | check: ${TEST_BINS} 71 | @for x in $^; do ${MPIRUN} -n 1 ./$$x; done 72 | 73 | clean: tidy 74 | rm -f ${TEST_BINS} ${TEST_OBJS} 75 | 76 | distclean: clean 77 | 78 | dist: distclean 79 | dir=`basename $$PWD` && cd .. && tar czvf $$dir.tgz ./$$dir 80 | dir=`basename $$PWD` && ls -l ../$$dir.tgz 81 | 82 | .PHONY: default clean dist distclean vi vim touch gdb tidy check 83 | -------------------------------------------------------------------------------- /mpi-proxy-split/mpi-wrappers/mpi_error_wrappers.cpp: -------------------------------------------------------------------------------- 1 | /**************************************************************************** 2 | * Copyright (C) 2019-2022 by Illio Suardi * 3 | * illio@u.nus.edu * 4 | * * 5 | * This file is part of DMTCP. * 6 | * * 7 | * DMTCP is free software: you can redistribute it and/or * 8 | * modify it under the terms of the GNU Lesser General Public License as * 9 | * published by the Free Software Foundation, either version 3 of the * 10 | * License, or (at your option) any later version. * 11 | * * 12 | * DMTCP is distributed in the hope that it will be useful, * 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of * 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * 15 | * GNU Lesser General Public License for more details. * 16 | * * 17 | * You should have received a copy of the GNU Lesser General Public * 18 | * License in the files COPYING and COPYING.LESSER. If not, see * 19 | * . * 20 | ****************************************************************************/ 21 | 22 | #include "mpi_plugin.h" 23 | #include "config.h" 24 | #include "dmtcp.h" 25 | #include "util.h" 26 | #include "jassert.h" 27 | #include "jfilesystem.h" 28 | #include "protectedfds.h" 29 | 30 | #include "mpi_nextfunc.h" 31 | 32 | extern "C" { 33 | 34 | #pragma weak MPI_Error_class = PMPI_Error_class 35 | int PMPI_Error_class(int errorcode, int *errorclass) 36 | { 37 | int retval = 0; 38 | DMTCP_PLUGIN_DISABLE_CKPT(); 39 | JUMP_TO_LOWER_HALF(lh_info->fsaddr); 40 | retval = NEXT_FUNC(Error_class)(errorcode, errorclass); 41 | RETURN_TO_UPPER_HALF(); 42 | DMTCP_PLUGIN_ENABLE_CKPT(); 43 | return retval; 44 | } 45 | 46 | #pragma weak MPI_Error_string = PMPI_Error_string 47 | int PMPI_Error_string(int errorcode, char *string, int *resultlen) 48 | { 49 | int retval = 0; 50 | DMTCP_PLUGIN_DISABLE_CKPT(); 51 | JUMP_TO_LOWER_HALF(lh_info->fsaddr); 52 | retval = NEXT_FUNC(Error_string)(errorcode, string, resultlen); 53 | RETURN_TO_UPPER_HALF(); 54 | DMTCP_PLUGIN_ENABLE_CKPT(); 55 | return retval; 56 | } 57 | 58 | } // end of: extern "C" 59 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/Irecv_test.c: -------------------------------------------------------------------------------- 1 | /* 2 | Test for the MPI_Irecv method 3 | 4 | Must run with >1 ranks 5 | Defaults to 5 6 | Intended to be run with mana_test.py 7 | 8 | */ 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #define BUFFER_SIZE 100 18 | #define SLEEP_PER_ITERATION 5 19 | 20 | int 21 | main(int argc, char **argv) 22 | { 23 | // Parse runtime argument 24 | int max_iterations = 5; // default 25 | if (argc != 1) { 26 | max_iterations = atoi(argv[1]); 27 | } 28 | 29 | // Initialize the MPI environment 30 | MPI_Init(NULL, NULL); 31 | // Find out rank, size 32 | int myrank; 33 | MPI_Comm_rank(MPI_COMM_WORLD, &myrank); 34 | int world_size; 35 | MPI_Comm_size(MPI_COMM_WORLD, &world_size); 36 | 37 | if (myrank == 0) { 38 | printf("Running test for %d iterations\n", max_iterations); 39 | } 40 | 41 | // We are assuming at least 2 processes for this task 42 | if (world_size < 2) { 43 | fprintf(stderr, "World size must be greater than 1 for %s\n", argv[0]); 44 | MPI_Abort(MPI_COMM_WORLD, 1); 45 | } 46 | 47 | int rank = 0; 48 | int number = 11223344; 49 | int i; 50 | 51 | for (int iterations = 0; iterations < max_iterations; iterations++) { 52 | for (rank = 0; rank < world_size; rank++) { 53 | if (rank == myrank) 54 | continue; 55 | int ret = MPI_Send(&number, 1, MPI_INT, rank, 0, MPI_COMM_WORLD); 56 | assert(number == 11223344 + iterations); 57 | assert(ret == MPI_SUCCESS); 58 | printf("%d sent %d to %d\n", myrank, number, rank); 59 | fflush(stdout); 60 | } 61 | 62 | MPI_Request reqs[world_size]; 63 | for (rank = 0; rank < world_size; rank++) { 64 | if (rank == myrank) 65 | continue; 66 | int ret = 67 | MPI_Irecv(&number, 1, MPI_INT, rank, 0, MPI_COMM_WORLD, &reqs[rank]); 68 | assert(ret == MPI_SUCCESS); 69 | } 70 | printf("%d sleeping\n", myrank); 71 | fflush(stdout); 72 | sleep(SLEEP_PER_ITERATION); 73 | for (rank = 0; rank < world_size; rank++) { 74 | if (rank == myrank) 75 | continue; 76 | int flag = 0; 77 | int first_test = 1; 78 | while (!flag) { 79 | if (first_test) { 80 | printf("%d testing request from %d\n", myrank, rank); 81 | fflush(stdout); 82 | first_test = 0; 83 | } 84 | MPI_Test(&reqs[rank], &flag, MPI_STATUS_IGNORE); 85 | if (flag) { 86 | printf("%d completed request from %d\n", myrank, rank); 87 | fflush(stdout); 88 | } 89 | } 90 | } 91 | number++; 92 | } 93 | 94 | MPI_Finalize(); 95 | } 96 | -------------------------------------------------------------------------------- /configure-mana: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # These optional autoconf variables will be configured 4 | # via contrib/mpi-proxy-split/Makefile_config.in 5 | 6 | # Do not add spaces to the next two lines. 7 | # If MPI will be used on a different computer than where you configure, 8 | # then maybe replace this with the result of 'ip addr' on the compute nodes. 9 | MPI_ETHERNET_INTERFACE=\ 10 | `PATH=$PATH:/usr/sbin ip addr |grep -B1 link/ether | head -1 |sed -e 's%[^ ]*: \([^ ]*\): .*%\1%'` 11 | 12 | # Note: modify this path according to your environment. 13 | if [[ -z "${MPI_INSTALL_DIR}" ]]; then 14 | MPI_INSTALL_DIR=/usr/local 15 | fi 16 | 17 | # This works on our local test computer (CentOS 7). 18 | # But also read the note right after this command. 19 | ./configure --enable-debug \ 20 | CFLAGS=-fno-stack-protector \ 21 | CXXFLAGS=-fno-stack-protector \ 22 | MPI_BIN=$MPI_INSTALL_DIR/bin \ 23 | MPI_INCLUDE=$MPI_INSTALL_DIR/include \ 24 | MPI_LIB=$MPI_INSTALL_DIR/lib \ 25 | MPICC='${MPI_BIN}/mpicc' \ 26 | MPICXX='${MPI_BIN}/mpic++' \ 27 | MPIFORTRAN='${MPI_BIN}/mpifort' \ 28 | MPIRUN='${MPI_BIN}/mpirun -iface '${MPI_ETHERNET_INTERFACE} \ 29 | MPI_LD_FLAG=-lmpich \ 30 | MPI_CFLAGS= \ 31 | MPI_CXXFLAGS= \ 32 | MPI_LDFLAGS= 33 | 34 | # NOTE: To make it work, two changes were needed: 35 | # =============================================================== 36 | # 1. MANA requires 'mpicc -static' for linking under MPICH. We saw multiple 37 | # problems in this. Hence, it is recommended to build your own MPICH. 38 | # We found that 'mpicc -static' fails to link with MPICH, unless you 39 | # configure MPICH with: ./configure --disable-xml2 --disable-libxml2 ... 40 | # See the contrib/mpi-proxy-split/INSTALL file for details. 41 | 42 | # =============================================================== 43 | # 2. Recent versions of gcc ran into problems with the stack protector. 44 | # This happens in the DMTCP call to arch_prctl(ARCH_SET_FS, ...). 45 | # (Actually, this is called through syscall.) So, stack protection 46 | # must be turned off in the files interposing on syscall(). For convenience, 47 | # we have turned it off everywhere, by setting CFLAGS and CXXFLAGS 48 | # in the 'configure' command, above. 49 | # =============================================================== 50 | 51 | # THEN IT COMPILES TO BUILD lh_proxy. 52 | # Note: also, compiling coordinator_mana.cpp in the plugin requires 53 | # '#include ' in order to compile the data structure rank_state_t . 54 | # Because of this, we need to use 'mpicxx' to build the coordinator code. 55 | # That works. 56 | # (BUT IS THERE SOMETHING CLEANER? Perhaps modifying the result of 57 | # 'mpicc -show' can help.) 58 | -------------------------------------------------------------------------------- /.clang-format: -------------------------------------------------------------------------------- 1 | --- 2 | BasedOnStyle: Google 3 | 4 | # Good: 5 | # 6 | # class Foo 7 | # { 8 | # public: 9 | # 10 | # Foo() = default; 11 | # 12 | # }; 13 | # 14 | # Bad: 15 | # 16 | # class Foo 17 | # { 18 | # public: 19 | # 20 | # Foo() = default; 21 | # 22 | # }; 23 | # 24 | AccessModifierOffset: 0 25 | 26 | # Good: 27 | # 28 | # clang = Clang(); // Comment 1 29 | # format = Format(); // Comment 2 30 | # 31 | # Bad: 32 | # clang = Clang(); // Comment 1 33 | # format = Format(); // Comment 2 34 | # 35 | AlignTrailingComments: false 36 | 37 | # Good: 38 | # 39 | # class Foo 40 | # { 41 | # public: 42 | # 43 | # Foo() 44 | # : first(1), second(2) {} 45 | # 46 | # }; 47 | # 48 | # Bad: 49 | # bool foo() { return true; } 50 | # 51 | AllowShortFunctionsOnASingleLine: Inline 52 | 53 | AllowShortIfStatementsOnASingleLine: false 54 | AllowShortLoopsOnASingleLine: false 55 | 56 | AlwaysBreakAfterDefinitionReturnType: None 57 | 58 | AlwaysBreakAfterDefinitionReturnType: TopLevel 59 | 60 | AlwaysBreakBeforeMultilineStrings: false 61 | 62 | Cpp11BracedListStyle: false 63 | 64 | PointerAlignment: Right 65 | # BreakBeforeTernaryOperators: true 66 | # KeepEmptyLinesAtTheStartOfBlocks: false 67 | 68 | # Good: 69 | # 70 | # void F(int first, 71 | # int second, 72 | # int third, 73 | # int fourth, 74 | # int fifth, 75 | # int sixth, 76 | # int seventh, 77 | # int eighth) {} 78 | # 79 | # Bad: 80 | # 81 | # void F(int first, int second, int third, int fourth, int fifth, int sixth, 82 | # int seventh, int eighth) {} 83 | # 84 | BinPackParameters: false 85 | 86 | # Like 'Attach', but break before braces on function, namespaces, class, struct 87 | # and union definitions. 88 | # 89 | BreakBeforeBraces: Linux 90 | 91 | # Good: 92 | # 93 | # class Foo 94 | # { 95 | # public: 96 | # 97 | # Foo() 98 | # : first(1), second(2) {} 99 | # 100 | # }; 101 | # 102 | # Bad: 103 | # 104 | # class Foo 105 | # { 106 | # public: 107 | # 108 | # Foo() 109 | # : first(1), second(2) {} 110 | # 111 | # }; 112 | # 113 | ConstructorInitializerIndentWidth: 2 114 | 115 | # Continuation indents such as assignment statements are indented by 2 spaces. 116 | ContinuationIndentWidth: 2 117 | 118 | # Don't try to guess the pointer alignment 119 | DerivePointerAlignment: false 120 | 121 | # List of foreach macros due to lack of range-based for loops. 122 | ForEachMacros: [ foreach, foreachkey, foreachvalue, foreachpair ] 123 | 124 | # Maximum number of empty lines to keep. 125 | MaxEmptyLinesToKeep: 2 126 | 127 | # Good: 128 | # 129 | # x = 42; // Comment 130 | # 131 | # Bad: 132 | # 133 | # x = 42; // Comment 134 | # 135 | SpacesBeforeTrailingComments: 1 136 | ... 137 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/two-phase-commit-3.c: -------------------------------------------------------------------------------- 1 | #include "mpi.h" 2 | #include 3 | #include 4 | #include 5 | 6 | int iterations = 100; 7 | 8 | int main( int argc, char *argv[] ) 9 | { 10 | int provided, flag, claimed; 11 | int comm1_counter = 0; 12 | int comm2_counter = 0; 13 | 14 | int ret = MPI_Init( 0, 0 ); 15 | if (ret != MPI_SUCCESS) { 16 | printf("MPI_Init failed\n"); 17 | exit(1); 18 | } 19 | 20 | int rank, nprocs; 21 | MPI_Comm_size(MPI_COMM_WORLD,&nprocs); 22 | MPI_Comm_rank(MPI_COMM_WORLD,&rank); 23 | printf("Hello, world. I am %d of %d\n", rank, nprocs);fflush(stdout); 24 | if (nprocs < 3) { 25 | printf("This test needs at least 3 ranks.\n"); 26 | MPI_Finalize(); 27 | return 1; 28 | } 29 | int group1_ranks[] = {0, 1}; 30 | int group2_ranks[10000]; 31 | int i,j; 32 | for (i = 1; i < nprocs; i++) { 33 | group2_ranks[i-1] = i; 34 | } 35 | 36 | MPI_Group world_group; 37 | MPI_Group group1; 38 | MPI_Group group2; 39 | MPI_Comm_group(MPI_COMM_WORLD, &world_group); 40 | MPI_Group_incl(world_group, 2, group1_ranks, &group1); 41 | MPI_Group_incl(world_group, nprocs-1, group2_ranks, &group2); 42 | 43 | MPI_Comm comm1; // Set to MPI_COMM_NULL by default. 44 | MPI_Comm comm2; // Set to MPI_COMM_NULL by default. 45 | MPI_Comm_create(MPI_COMM_WORLD, group1, &comm1); 46 | MPI_Comm_create(MPI_COMM_WORLD, group2, &comm2); 47 | printf("rank: %d, group1: %x, comm1: %x\n", rank, group1, comm1); 48 | printf("rank: %d, group2: %x, comm2: %x\n", rank, group2, comm2); 49 | 50 | //=============================================================== 51 | // Coll. 1: | === 52 | // | | | 53 | // Coll. 2: | === 54 | // | | | 55 | // Coll. 3: === | 56 | // | | | 57 | // Coll. 4: | === 58 | //=============================================================== 59 | 60 | for (i = 0; i < iterations; i++) { 61 | if (comm2 != MPI_COMM_NULL) { 62 | comm2_counter++; 63 | printf("Rank %d entering comm2, iteration %d\n", rank, comm2_counter); 64 | fflush(stdout); 65 | MPI_Barrier(comm2); 66 | printf("Rank %d leaving comm2, iteration %d\n", rank, comm2_counter); 67 | fflush(stdout); 68 | sleep(1); 69 | comm2_counter++; 70 | printf("Rank %d entering comm2, iteration %d\n", rank, comm2_counter); 71 | fflush(stdout); 72 | MPI_Barrier(comm2); 73 | printf("Rank %d leaving comm2, iteration %d\n", rank, comm2_counter); 74 | fflush(stdout); 75 | sleep(1); 76 | } 77 | if (comm1 != MPI_COMM_NULL) { 78 | comm1_counter++; 79 | printf("Rank %d entering comm1, iteration %d\n", rank, comm1_counter); 80 | fflush(stdout); 81 | MPI_Barrier(comm1); 82 | printf("Rank %d leaving comm1, iteration %d\n", rank, comm1_counter); 83 | fflush(stdout); 84 | sleep(1); 85 | } 86 | } 87 | 88 | MPI_Finalize(); 89 | return 0; 90 | } 91 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/Alltoall_test.c: -------------------------------------------------------------------------------- 1 | /* 2 | Test for the MPI_Alltoall method 3 | 4 | Run with >2 ranks for non-trivial results 5 | Defaults to 10000 iterations 6 | Intended to be run with mana_test.py 7 | 8 | Source: http://mpi.deino.net/mpi_functions/MPI_Alltoall.html 9 | */ 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | int 19 | main(int argc, char *argv[]) 20 | { 21 | // Parse runtime argument 22 | int max_iterations; 23 | max_iterations = 10000; 24 | if (argc != 1) { 25 | max_iterations = atoi(argv[1]); 26 | } 27 | 28 | int rank, size; 29 | int chunk = 100; 30 | int i; 31 | int *sb; 32 | int *rb; 33 | int status, gstatus, ret; 34 | 35 | MPI_Init(&argc, &argv); 36 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 37 | MPI_Comm_size(MPI_COMM_WORLD, &size); 38 | for (i = 1; i < argc; ++i) { 39 | if (argv[i][0] != '-') 40 | continue; 41 | switch (argv[i][1]) { 42 | case 'm': 43 | chunk = atoi(argv[++i]); 44 | break; 45 | default: 46 | fprintf(stderr, "Unrecognized argument %s\n", argv[i]); 47 | fflush(stderr); 48 | MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); 49 | } 50 | } 51 | sb = (int *)malloc(size * chunk * sizeof(int)); 52 | if (!sb) { 53 | perror("can't allocate send buffer"); 54 | fflush(stderr); 55 | MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); 56 | } 57 | rb = (int *)malloc(size * chunk * sizeof(int)); 58 | if (!rb) { 59 | perror("can't allocate recv buffer"); 60 | fflush(stderr); 61 | free(sb); 62 | MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); 63 | } 64 | for (i = 0; i < size * chunk; ++i) { 65 | sb[i] = rank + 1; 66 | rb[i] = 0; 67 | } 68 | for (int j = 0; j < max_iterations; j++) { 69 | status = 70 | MPI_Alltoall(sb, chunk, MPI_INT, rb, chunk, MPI_INT, MPI_COMM_WORLD); 71 | #ifdef DEBUG 72 | printf("[Rank = %d] Status = %d, size = %d, chunk = %d\n", rank, status, 73 | size, chunk); 74 | fflush(stdout); 75 | #endif 76 | ret = MPI_Allreduce(&status, &gstatus, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); 77 | assert(ret == MPI_SUCCESS); 78 | for (i = 0; i < size * chunk; ++i) { 79 | assert(rb[i] == (int)(i / chunk) + 1); 80 | } 81 | if (rank == 0) { 82 | if (gstatus != 0) { 83 | printf("all_to_all returned %d\n", gstatus); 84 | fflush(stdout); 85 | assert(gstatus == MPI_SUCCESS); 86 | } 87 | } 88 | for (i = 0; i < size * chunk; ++i) { 89 | assert(rb[i] == (int)(i / chunk) + 1); 90 | rb[i] = 0; // clear the recv buffer 91 | } 92 | #ifdef DEBUG 93 | printf("[Rank %d]: Test Passed!\n", rank); 94 | fflush(stdout); 95 | #endif 96 | } 97 | 98 | free(sb); 99 | free(rb); 100 | MPI_Finalize(); 101 | return (EXIT_SUCCESS); 102 | } 103 | -------------------------------------------------------------------------------- /mpi-proxy-split/virtual_id.h: -------------------------------------------------------------------------------- 1 | #ifndef MANA_VIRTUAL_ID_H 2 | #define MANA_VIRTUAL_ID_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #define MANA_COMM_KIND 1 11 | #define MANA_GROUP_KIND 2 12 | #define MANA_DATATYPE_KIND 3 13 | #define MANA_OP_KIND 4 14 | #define MANA_REQUEST_KIND 5 15 | #define MANA_FILE_KIND 6 16 | #define MANA_VIRT_ID_KIND_SHIFT 28 17 | 18 | extern MPI_Group g_world_group; 19 | extern MPI_Comm g_world_comm; 20 | 21 | typedef union { 22 | int _handle; 23 | int64_t _handle64; 24 | MPI_Comm comm; 25 | MPI_Group group; 26 | MPI_Request request; 27 | MPI_Op op; 28 | MPI_Datatype datatype; 29 | MPI_File file; 30 | } mana_mpi_handle; 31 | 32 | typedef struct { 33 | int size; 34 | int rank; 35 | int *global_ranks; 36 | } mana_group_desc; 37 | 38 | typedef struct { 39 | int size; 40 | int rank; 41 | int *global_ranks; 42 | } mana_comm_desc; 43 | 44 | typedef struct { 45 | // For now, there's no need for additional informations. 46 | // NULL will be used in virt_id_entry of request. 47 | } mana_request_desc; 48 | 49 | typedef struct { 50 | MPI_User_function *user_fn; 51 | int commute; 52 | } mana_op_desc; 53 | 54 | typedef struct { 55 | // It's hard to decode and reconstruct "double derived datatypes", 56 | // which means datatypes that are created using derived datatypes. 57 | // So we decided to use the old record-and-replay approach to 58 | // reconstruct datatypes at restart. Therefore, there's no data 59 | // needs to be saved in the descriptor. We keep this structure 60 | // definition for future use. 61 | } mana_datatype_desc; 62 | 63 | typedef struct { 64 | // Use the g_param for restoring files for now. 65 | // Migarate codes to here later. 66 | } mana_file_desc; 67 | 68 | typedef struct { 69 | mana_mpi_handle real_id; 70 | void *desc; 71 | } virt_id_entry; 72 | 73 | extern std::map virt_ids; 74 | typedef std::map::iterator virt_id_iterator; 75 | 76 | extern int g_world_rank; 77 | 78 | void init_predefined_virt_ids(); 79 | MPI_Comm new_virt_comm(MPI_Comm real_comm); 80 | MPI_Group new_virt_group(MPI_Group real_group); 81 | MPI_Op new_virt_op(MPI_Op real_op); 82 | MPI_Datatype new_virt_datatype(MPI_Datatype real_datatype); 83 | MPI_Request new_virt_request(MPI_Request real_request); 84 | MPI_File new_virt_file(MPI_File real_request); 85 | 86 | int is_predefined_id(mana_mpi_handle id); 87 | mana_mpi_handle add_virt_id(mana_mpi_handle real_id, void *desc, int kind); 88 | virt_id_entry* get_virt_id_entry(mana_mpi_handle virt_id); 89 | mana_mpi_handle get_real_id(mana_mpi_handle virt_id); 90 | void* get_virt_id_desc(mana_mpi_handle virt_id); 91 | void free_desc(void *desc, int kind); 92 | void free_virt_id(mana_mpi_handle virt_id); 93 | void update_virt_id(mana_mpi_handle virt_id, mana_mpi_handle real_id); 94 | 95 | void reconstruct_descriptors(); 96 | void init_predefined_virt_ids(); 97 | #endif // MANA_VIRTUAL_ID_H 98 | -------------------------------------------------------------------------------- /mpi-proxy-split/mpi_plugin.h: -------------------------------------------------------------------------------- 1 | /**************************************************************************** 2 | * Copyright (C) 2019-2021 by Gene Cooperman, Rohan Garg, Yao Xu * 3 | * gene@ccs.neu.edu, rohgarg@ccs.neu.edu, xu.yao1@northeastern.edu * 4 | * * 5 | * This file is part of DMTCP. * 6 | * * 7 | * DMTCP is free software: you can redistribute it and/or * 8 | * modify it under the terms of the GNU Lesser General Public License as * 9 | * published by the Free Software Foundation, either version 3 of the * 10 | * License, or (at your option) any later version. * 11 | * * 12 | * DMTCP is distributed in the hope that it will be useful, * 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of * 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * 15 | * GNU Lesser General Public License for more details. * 16 | * * 17 | * You should have received a copy of the GNU Lesser General Public * 18 | * License in the files COPYING and COPYING.LESSER. If not, see * 19 | * . * 20 | ****************************************************************************/ 21 | 22 | #ifndef _MPI_PLUGIN_H 23 | #define _MPI_PLUGIN_H 24 | 25 | #include 26 | #include 27 | #include 28 | 29 | #include "lower-half-api.h" 30 | #include "dmtcp_dlsym.h" 31 | 32 | #define _real_fork NEXT_FNC_DEFAULT(fork) 33 | 34 | #define NOT_IMPLEMENTED(op) \ 35 | { \ 36 | if (op > MPIProxy_ERROR && op < MPIProxy_Cmd_Shutdown_Proxy) \ 37 | fprintf(stdout, "[%d:%s] NOT IMPLEMENTED\n", op, __FUNCTION__); \ 38 | else \ 39 | fprintf(stdout, "[%s] NOT IMPLEMENTED\n", __FUNCTION__); \ 40 | exit(EXIT_FAILURE); \ 41 | } 42 | 43 | #define NOT_IMPL(func) \ 44 | EXTERNC func \ 45 | { \ 46 | NOT_IMPLEMENTED(MPIProxy_Cmd_Shutdown_Proxy); \ 47 | } 48 | 49 | bool isUsingCollectiveToP2p(); 50 | void recordPreMpiInitMaps(); 51 | void recordPostMpiInitMaps(); 52 | 53 | enum mana_state_t { 54 | UNKNOWN_STATE, 55 | RUNNING, 56 | CKPT_COLLECTIVE, 57 | CKPT_P2P, 58 | RESTART_RETORE, 59 | RESTART_REPLAY 60 | }; 61 | 62 | extern mana_state_t mana_state; 63 | extern bool g_libmana_is_initialized; 64 | 65 | #endif // ifndef _MPI_PLUGIN_H 66 | -------------------------------------------------------------------------------- /util/attach_all.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | import re 3 | import sys 4 | import threading 5 | import socket 6 | import argparse 7 | 8 | from subprocess import PIPE 9 | 10 | outlock = threading.Lock() 11 | 12 | 13 | def parse_ranks(mana_status_path): 14 | ranks = {} 15 | stat = subprocess.run([f'{mana_status_path}'], stdout=PIPE) 16 | stat.check_returncode() 17 | rank_arr = stat.stdout.decode('utf-8').split('\n') 18 | rank_arr = rank_arr[5:len(rank_arr)-1] 19 | for rank in rank_arr: 20 | r_ind = rank[0:rank.index(',')] 21 | ranks[r_ind] = {} 22 | rank = rank[rank.index(',')+1:] 23 | ranks[r_ind]['host'] = rank[rank.index('@')+1:rank.index(',')] 24 | ranks[r_ind]['pid'] = rank[rank.index(':')+1:rank.index(']')] 25 | return ranks 26 | 27 | 28 | def attach_to_rank(rank, pid, host, infile, outfile): 29 | proc = subprocess.Popen(['ssh', f'{host}'], stdout=PIPE, stdin=PIPE, 30 | stderr=PIPE) 31 | out, errs = proc.communicate(f'gdb attach {pid} -batch -x={infile}'. 32 | encode()) 33 | outlock.acquire() 34 | file = open(outfile, 'a') 35 | file.write(f'RANK {rank}:\n') 36 | file.write(out.decode('utf-8')) 37 | file.write('=================================================\n\n') 38 | file.close() 39 | outlock.release() 40 | 41 | class CustomParser(argparse.ArgumentParser): 42 | def error(self, message): 43 | sys.stderr.write('error: %s\n' % message) 44 | self.print_help() 45 | sys.exit(2) 46 | 47 | def main(): 48 | parser = CustomParser(description='Attach GDB to all ranks') 49 | parser.add_argument('infile',metavar='I', help='GDB command file') 50 | parser.add_argument('outfile', metavar='O', help='Output log file') 51 | parser.add_argument('-m','--mana_status', metavar='M', help='Absolute path\ 52 | to mana_status executable', default='mana_status', 53 | required=False) 54 | parser.add_argument('-s','--skip', help='Skip one rank to debug manually', 55 | action='store_true') 56 | 57 | args = parser.parse_args() 58 | infile = args.infile 59 | outfile = args.outfile 60 | skipped = not args.skip 61 | mana_status_path = args.mana_status 62 | 63 | file = open(outfile, "w+") 64 | file.truncate(0) 65 | file.close() 66 | 67 | ranks = parse_ranks(mana_status_path) 68 | threads = [] 69 | for rank in ranks: 70 | pid = ranks[rank]['pid'] 71 | host = ranks[rank]['host'] 72 | if not skipped and socket.gethostname() == host: 73 | print(f"Skipping {pid} on {host} for manual debugging") 74 | skipped = True 75 | continue 76 | 77 | th = threading.Thread(target=attach_to_rank, args=(rank, pid, host, 78 | infile, outfile,)) 79 | th.start() 80 | 81 | for thread in threads: 82 | thread.join() 83 | 84 | if __name__ == '__main__': 85 | main() 86 | -------------------------------------------------------------------------------- /mpi-proxy-split/p2p_drain_send_recv.h: -------------------------------------------------------------------------------- 1 | /**************************************************************************** 2 | * Copyright (C) 2019-2021 by Gene Cooperman, Rohan Garg, Yao Xu * 3 | * gene@ccs.neu.edu, rohgarg@ccs.neu.edu, xu.yao1@northeastern.edu * 4 | * * 5 | * This file is part of DMTCP. * 6 | * * 7 | * DMTCP is free software: you can redistribute it and/or * 8 | * modify it under the terms of the GNU Lesser General Public License as * 9 | * published by the Free Software Foundation, either version 3 of the * 10 | * License, or (at your option) any later version. * 11 | * * 12 | * DMTCP is distributed in the hope that it will be useful, * 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of * 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * 15 | * GNU Lesser General Public License for more details. * 16 | * * 17 | * You should have received a copy of the GNU Lesser General Public * 18 | * License in the files COPYING and COPYING.LESSER. If not, see * 19 | * . * 20 | ****************************************************************************/ 21 | 22 | #ifndef _P2P_SEND_RECV_H 23 | #define _P2P_SEND_RECV_H 24 | 25 | #include 26 | #include "dmtcp.h" 27 | #include "dmtcpalloc.h" 28 | #include "p2p_log_replay.h" 29 | 30 | #ifdef DEBUG_P2P 31 | extern int *g_sendBytesByRank; // Number of bytes sent to other ranks 32 | extern int *g_rsendBytesByRank; // Number of bytes sent to other ranks by MPI_Rsend 33 | extern int *g_bytesSentToUsByRank; // Number of bytes other ranks sent to us 34 | extern int *g_recvBytesByRank; // Number of bytes received from other ranks 35 | #endif 36 | extern int64_t global_sent_messages, global_recv_messages; 37 | extern int64_t local_sent_messages, local_recv_messages; 38 | extern std::unordered_set active_comms; 39 | extern dmtcp::vector g_message_queue; 40 | 41 | void initialize_drain_send_recv(); 42 | void registerLocalSendsAndRecvs(); 43 | void drainSendRecv(); 44 | int drainRemainingP2pMsgs(int source); 45 | int recvMsgIntoInternalBuffer(MPI_Status status); 46 | bool existsMatchingMsgBuffer(int source, int tag, MPI_Comm comm, int *flag, 47 | MPI_Status *status); 48 | int consumeMatchingMsgBuffer(void *buf, int count, MPI_Datatype datatype, 49 | int source, int tag, MPI_Comm comm, 50 | MPI_Status *mpi_status, int size); 51 | void removePendingSendRequests(); 52 | void resetDrainCounters(); 53 | int localRankToGlobalRank(int localRank, MPI_Comm localComm); 54 | #endif 55 | -------------------------------------------------------------------------------- /mpi-proxy-split/lower-half/logging.h: -------------------------------------------------------------------------------- 1 | /**************************************************************************** 2 | * Copyright (C) 2019-2020 by Twinkle Jain, Rohan garg, and Gene Cooperman * 3 | * jain.t@husky.neu.edu, rohgarg@ccs.neu.edu, gene@ccs.neu.edu * 4 | * * 5 | * This file is part of DMTCP. * 6 | * * 7 | * DMTCP is free software: you can redistribute it and/or * 8 | * modify it under the terms of the GNU Lesser General Public License as * 9 | * published by the Free Software Foundation, either version 3 of the * 10 | * License, or (at your option) any later version. * 11 | * * 12 | * DMTCP is distributed in the hope that it will be useful, * 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of * 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * 15 | * GNU Lesser General Public License for more details. * 16 | * * 17 | * You should have received a copy of the GNU Lesser General Public * 18 | * License along with DMTCP:dmtcp/src. If not, see * 19 | * . * 20 | ****************************************************************************/ 21 | 22 | #ifndef LOGGING_H 23 | #define LOGGING_H 24 | 25 | #include 26 | #include 27 | 28 | #include 29 | 30 | // Logging levels 31 | #define NOISE 3 // Verbose 32 | #define INFO 2 // Informational logs 33 | #define ERROR 1 // Error logs 34 | 35 | #define KNRM "\x1B[0m" 36 | #define KRED "\x1B[31m" 37 | #define KGRN "\x1B[32m" 38 | #define KYEL "\x1B[33m" 39 | #define KBLU "\x1B[34m" 40 | #define KMAG "\x1B[35m" 41 | #define KCYN "\x1B[36m" 42 | #define KWHT "\x1B[37m" 43 | 44 | static const char *colors[] = {KNRM, KRED, KGRN, KYEL}; 45 | 46 | #ifndef DEBUG_LEVEL 47 | // Let's announce errors out loud 48 | # define DEBUG_LEVEL 0 49 | #endif // ifndef DEBUG_LEVEL 50 | 51 | #define CLOG(LOG_LEVEL, fmt, ...) \ 52 | fprintf(stderr, "%s[%s +%d]: " fmt KNRM, colors[LOG_LEVEL], __FILE__, \ 53 | __LINE__, ##__VA_ARGS__); \ 54 | 55 | #define DLOG(LOG_LEVEL, fmt, ...) \ 56 | do { \ 57 | if (DEBUG_LEVEL) { \ 58 | if (LOG_LEVEL <= DEBUG_LEVEL) \ 59 | CLOG(LOG_LEVEL, fmt, ##__VA_ARGS__) \ 60 | } \ 61 | } while(0) 62 | 63 | #endif // ifndef LOGGING_H 64 | -------------------------------------------------------------------------------- /mpi-proxy-split/test/send_recv_loop.c: -------------------------------------------------------------------------------- 1 | /* 2 | Test for the MPI_Send and MPI_Recv methods 3 | 4 | Must run with >2 ranks 5 | Defaults to 5 iterations 6 | Intended to be run with mana_test.py 7 | 8 | Source: www.mpitutorial.com 9 | */ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #define SLEEP_PER_ITERATION 5 18 | #define MESSAGES_PER_ITERATION 10 19 | 20 | int 21 | main(int argc, char **argv) 22 | { 23 | // Parse runtime argument 24 | int max_iterations = 5; // default 25 | if (argc != 1) { 26 | max_iterations = atoi(argv[1]); 27 | } 28 | 29 | // Initialize the MPI environment 30 | MPI_Init(NULL, NULL); 31 | // Find out rank and size 32 | int world_rank, world_size; 33 | MPI_Comm_rank(MPI_COMM_WORLD, &world_rank); 34 | MPI_Comm_size(MPI_COMM_WORLD, &world_size); 35 | 36 | if (world_rank == 0) { 37 | printf("Running test for %d iterations\n", max_iterations); 38 | } 39 | 40 | // We are assuming at least 3 processes for this task 41 | if (world_size < 3) { 42 | fprintf(stderr, "World size must be greater than or equal to 3 for %s\n", 43 | argv[0]); 44 | MPI_Abort(MPI_COMM_WORLD, 1); 45 | } 46 | 47 | for (int iterations = 0; iterations < max_iterations; iterations++) { 48 | // rank 0 wait a while then send messages to rank 1 49 | int number = 0; 50 | if (world_rank == 0) { 51 | sleep(SLEEP_PER_ITERATION); 52 | for (int i = 0; i < MESSAGES_PER_ITERATION; i++) { 53 | int ret = MPI_Send(&number, 1, MPI_INT, 1, 0, MPI_COMM_WORLD); 54 | assert(ret == MPI_SUCCESS); 55 | number++; 56 | ret = MPI_Barrier(MPI_COMM_WORLD); 57 | assert(ret == MPI_SUCCESS); 58 | printf("Rank 0 have successfully sent %d messages to rank 1\n", i + 1); 59 | fflush(stdout); 60 | } 61 | } else if (world_rank == 1) { 62 | // rank 1 receive message from 0 first and then 2 63 | int number = 0; 64 | for (int i = 0; i < MESSAGES_PER_ITERATION; i++) { 65 | int recv_number = -1; 66 | MPI_Recv(&recv_number, 1, MPI_INT, 0, 0, MPI_COMM_WORLD, 67 | MPI_STATUS_IGNORE); 68 | assert(number == recv_number); 69 | MPI_Recv(&recv_number, 1, MPI_INT, 2, 0, MPI_COMM_WORLD, 70 | MPI_STATUS_IGNORE); 71 | assert(number == recv_number); 72 | number++; 73 | MPI_Barrier(MPI_COMM_WORLD); 74 | printf("Rank 1 have successfully received %d messages\n", (i + 1) * 2); 75 | fflush(stdout); 76 | } 77 | } else { 78 | // rank 2 send messages to rank 1 right away 79 | int number = 0; 80 | for (int i = 0; i < MESSAGES_PER_ITERATION; i++) { 81 | int ret = MPI_Send(&number, 1, MPI_INT, 1, 0, MPI_COMM_WORLD); 82 | assert(ret == MPI_SUCCESS); 83 | number++; 84 | ret = MPI_Barrier(MPI_COMM_WORLD); 85 | assert(ret == MPI_SUCCESS); 86 | printf("Rank 2 have successfully sent %d messages to rank 1\n", i + 1); 87 | fflush(stdout); 88 | } 89 | } 90 | } 91 | MPI_Finalize(); 92 | return 0; 93 | } 94 | -------------------------------------------------------------------------------- /util/memory_access_tracker/test_memory_tracker.c: -------------------------------------------------------------------------------- 1 | #/***************************************************************************** 2 | # * Copyright (C) 2021 Jun Gan * 3 | # * * 4 | # * DMTCP is free software: you can redistribute it and/or * 5 | # * modify it under the terms of the GNU Lesser General Public License as * 6 | # * published by the Free Software Foundation, either version 3 of the * 7 | # * License, or (at your option) any later version. * 8 | # * * 9 | # * DMTCP is distributed in the hope that it will be useful, * 10 | # * but WITHOUT ANY WARRANTY; without even the implied warranty of * 11 | # * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * 12 | # * GNU Lesser General Public License for more details. * 13 | # * * 14 | # * You should have received a copy of the GNU Lesser General Public * 15 | # * License along with DMTCP. If not, see . * 16 | # *****************************************************************************/ 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | 26 | #include "memory_access_tracker.h" 27 | 28 | int main () 29 | { 30 | int i; 31 | int log_fd = open("test.log", O_CREAT | O_WRONLY | O_APPEND | O_CLOEXEC | O_TRUNC, 0777); 32 | if (log_fd < 0) { 33 | perror("Failed to open test log"); 34 | exit(EXIT_FAILURE); 35 | } 36 | 37 | // allocate 1 page memory 38 | void *test_addr = mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); 39 | if (test_addr == MAP_FAILED) { 40 | close(log_fd); 41 | perror("Map failed"); 42 | exit(EXIT_FAILURE); 43 | } 44 | 45 | memset(test_addr, 0, 1024); 46 | memset(test_addr + 1024, 1, 1024); 47 | memset(test_addr + 2048, 2, 1024); 48 | memset(test_addr + 3072, 3, 1024); 49 | 50 | struct MemTracker *tracker = StartTrackMemory(test_addr, 4096, 100, log_fd); 51 | 52 | printf("%s %d Test %p\n", __func__, __LINE__, test_addr); 53 | 54 | // should trigger segv 55 | uint8_t c = *(uint8_t *)test_addr; 56 | 57 | printf("%s %d Read %p: %u\n", __func__, __LINE__, test_addr, c); 58 | 59 | *(uint8_t *)test_addr = 1; 60 | 61 | c = *(uint8_t *)test_addr; 62 | 63 | printf("%s %d Read %p: %u\n", __func__, __LINE__, test_addr, c); 64 | 65 | c = *(uint8_t *)(test_addr + 100); 66 | printf("%s %d Read %p: %u\n", __func__, __LINE__, test_addr + 1024, c); 67 | 68 | memset(test_addr + 2048, 3, 1024); 69 | EndTrackMemory(tracker); 70 | printf("%s %d access %u times\n", __func__, __LINE__, tracker->num_access); 71 | for (i = 0; i < tracker->num_access; i++) { 72 | printf("%p\n", tracker->addrs[i]); 73 | } 74 | 75 | FreeMemTracker(tracker); 76 | 77 | close(log_fd); 78 | return 0; 79 | } 80 | --------------------------------------------------------------------------------