├── .github └── workflows │ └── ci.yml ├── .gitignore ├── API.md ├── LICENSE ├── Lookup.md ├── Makefile ├── NOTES.md ├── README.md ├── generate-datatype-table.sh ├── generate-op-table.sh ├── header.c ├── impl-alltoallw.h ├── impl-commgroup-functions.c ├── impl-constant-conversions.c ├── impl-constant-conversions.h ├── impl-file-functions.c ├── impl-fpointers.h ├── impl-functions.c ├── impl-grequest-functions.c ├── impl-handle-conversions.h ├── impl-keyval-map-commattr.h ├── impl-keyval-map-commerrh.h ├── impl-keyval-map-fileerrh.h ├── impl-keyval-map-fileerrh2.h ├── impl-keyval-map-ireqa2aw.h ├── impl-keyval-map-opuserfn.h ├── impl-keyval-map-preqa2aw.h ├── impl-keyval-map-typeattr.h ├── impl-keyval-map-winattr.h ├── impl-keyval-map-winerrh.h ├── impl-keyval-map.cc ├── impl-keyval-map.h ├── impl-keyval.c ├── impl-load-functions.c ├── impl-predefined-handle.h ├── impl-predefined.c ├── impl-reduce-functions.c ├── impl-rma-functions.c ├── impl-scalar-types.h ├── impl-session-functions.c ├── impl-status.h ├── impl-type-functions.c ├── impl-wait-functions.c ├── libinit.c ├── mpi-constants.h ├── mpi-fortran.h ├── mpi-handle-typedefs.h ├── mpi-predefined.h ├── mpi-prototypes.h ├── mpi-typedefs.h ├── mpi.h ├── muk-dl.h ├── muk-function-pointers.h ├── muk-predefined.h ├── muk.h ├── test.sh ├── testbottom.c ├── testcart.c ├── testcoll.c ├── testcoll2.c ├── testcomm.c ├── testconstants.c ├── testerrh.c ├── testgroup.c ├── testinit.c ├── testmalloc.c ├── testops.c ├── testreqs.c ├── testtypes.c ├── testtypes2.c ├── testwin.c └── wrap-handle-typedefs.h /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: ci 2 | 3 | on: 4 | #push: 5 | # branches: 6 | # - main 7 | pull_request: 8 | branches: 9 | - main 10 | 11 | concurrency: 12 | group: ${{ github.workflow }}-${{ github.ref }} 13 | cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} 14 | 15 | permissions: 16 | contents: read 17 | 18 | jobs: 19 | test: 20 | runs-on: ${{ matrix.os }} 21 | timeout-minutes: 30 22 | strategy: 23 | fail-fast: false 24 | matrix: 25 | os: 26 | - ubuntu-latest 27 | - macos-latest 28 | 29 | steps: 30 | 31 | - name: Checkout 32 | uses: actions/checkout@v4 33 | 34 | - name: Setup MPICH 35 | uses: mpi4py/setup-mpi@v1 36 | with: 37 | mpi: mpich 38 | 39 | - name: Setup Open MPI 40 | uses: mpi4py/setup-mpi@v1 41 | with: 42 | mpi: openmpi 43 | 44 | - name: Build 45 | run: make 46 | 47 | - name: Test 48 | if: false 49 | run: make check 50 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Prerequisites 2 | *.d 3 | 4 | # Object files 5 | *.i 6 | *.s 7 | *.o 8 | *.ko 9 | *.obj 10 | *.elf 11 | 12 | # Linker output 13 | *.ilk 14 | *.map 15 | *.exp 16 | 17 | # Precompiled Headers 18 | *.gch 19 | *.pch 20 | 21 | # Libraries 22 | *.lib 23 | *.a 24 | *.la 25 | *.lo 26 | 27 | # Shared objects (inc. Windows DLLs) 28 | *.dll 29 | *.so 30 | *.so.* 31 | *.dylib 32 | 33 | # Executables 34 | *.x 35 | *.exe 36 | *.out 37 | *.app 38 | *.i*86 39 | *.x86_64 40 | *.hex 41 | 42 | # Debug files 43 | *.dSYM/ 44 | *.su 45 | *.idb 46 | *.pdb 47 | 48 | # Kernel Module Compile Results 49 | *.mod* 50 | *.cmd 51 | .tmp_versions/ 52 | modules.order 53 | Module.symvers 54 | Mkfile.old 55 | dkms.conf 56 | 57 | # VIM 58 | *.swp 59 | Makefile.local 60 | -------------------------------------------------------------------------------- /API.md: -------------------------------------------------------------------------------- 1 | These are all of the functions that do not depend on MPI types. 2 | 3 | ### Library control and query 4 | ```c 5 | MPI_Init(int *argc, char ***argv); 6 | MPI_Init_thread(int *argc, char ***argv, int required, int *provided); 7 | MPI_Finalize(); 8 | MPI_Finalized(int *flag); 9 | MPI_Initialized(int *flag); 10 | MPI_Is_thread_main(int *flag); 11 | MPI_Query_thread(int *provided); 12 | MPI_Get_library_version(char *version, int *resultlen); 13 | MPI_Get_processor_name(char *name, int *resultlen); 14 | MPI_Get_version(int *version, int *subversion); 15 | ``` 16 | 17 | ### Errors 18 | ```c 19 | MPI_Add_error_class(int *errorclass); 20 | MPI_Add_error_code(int errorclass, int *errorcode); 21 | MPI_Add_error_string(int errorcode, const char *string); 22 | MPI_Error_class(int errorcode, int *errorclass); 23 | MPI_Error_string(int errorcode, char *string, int *resultlen); 24 | ``` 25 | 26 | ### Random 27 | ```c 28 | MPI_Pcontrol(const int level, ...); 29 | MPI_Close_port(const char *port_name); 30 | MPI_Dims_create(int nnodes, int ndims, int dims[]); 31 | ``` 32 | 33 | ### Keyvals 34 | ```c 35 | MPI_Comm_free_keyval(int *comm_keyval); 36 | MPI_Keyval_free(int *keyval); 37 | MPI_Type_free_keyval(int *type_keyval); 38 | MPI_Win_free_keyval(int *win_keyval); 39 | ``` 40 | 41 | ### Memory 42 | 43 | Because `MPI_Alloc_mem` isn't on this list, we cannot do anything interesting here. 44 | ```c 45 | MPI_Buffer_attach(void *buffer, int size); 46 | MPI_Buffer_detach(void *buffer_addr, int *size); 47 | MPI_Free_mem(void *base); 48 | ``` 49 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Jeff Hammond 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Lookup.md: -------------------------------------------------------------------------------- 1 | # Mukautuva lookup requirements 2 | 3 | Goal: 4 | 5 | A container C that satisfies the following: 6 | 7 | ```c++ 8 | // input = K and V 9 | // return (0/1) = whether the K was new and added or existing and the value overwritten 10 | C.add_or_replace(Keytype K, Valuetype V); 11 | ``` 12 | 13 | ```c++ 14 | // input = remove, which indicates whether or not the KV should be removed or not. 15 | // input = K 16 | // output = V 17 | // return = found (boolean), which indicates whether or not the value was found. 18 | found = C.fetch_value(remove, Keytype K, Valuetype & V); 19 | ``` 20 | 21 | ```c++ 22 | // input = K 23 | C.remove(Keytype K); 24 | ``` 25 | 26 | where 27 | 28 | - Keytype is either an `int` or a `intptr_t`. Keys are unique. 29 | - Valuetype is either a pointer or a pair of pointers; memory blocks of 8 or 16 bytes are sufficient. 30 | 31 | `C.add_or_replace` can be as expensive as necessary. 32 | `C.fetch_value` must be as fast as possible when `K` is _not_ in `C`; the speed on hits matters less. 33 | 34 | The size of C is likely to be small (less than 20 members) but is permitted to be arbitrarily large in pathological cases. 35 | 36 | `C.fetch_value` may be called hundreds or thousands of times in a loop where most or all of the `K` are not in `C`. 37 | It would be ideal to optimize for this scenario, with an interface that takes a vector of `Keytype` if necessary. 38 | 39 | The following a C-compatible API prototype for such a function: 40 | ```c++ 41 | // input = remove, which indicates whether or not the KVs should be removed or not. 42 | // input = key_count, the size of K[] 43 | // input = K[], a vector of keys 44 | // input = value_count, the length of V[], i.e. the maximum number of keys that should be found before the method returns 45 | // output = V[], a vector of values the length of K[] 46 | // return = found_count, indicates how many values were found and can be read from V[] 47 | found_count = C.fetch_value_many(remove, key_count, Keytype K[], value_count, Valuetype & V[]); 48 | ``` 49 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | UNAME_S := $(shell uname) 2 | 3 | ifeq ($(UNAME_S),Darwin) 4 | brew_prefix := $(shell brew --prefix) 5 | OMPICC=$(wildcard $(brew_prefix)/Cellar/open-mpi/*/bin/mpicc) 6 | OMPICXX=$(wildcard $(brew_prefix)/Cellar/open-mpi/*/bin/mpicxx) 7 | MPICHCC=$(wildcard $(brew_prefix)/Cellar/mpich/*/bin/mpicc) 8 | MPICHCXX=$(wildcard $(brew_prefix)/Cellar/mpich/*/bin/mpicxx) 9 | CC=clang 10 | CFLAGS=-ferror-limit=1 # Clang 11 | #CFLAGS+=-Wno-c2x-extensions 12 | #CFLAGS+=-Wno-unused-function 13 | #CFLAGS+=-Wno-incompatible-function-pointer-types 14 | else 15 | OSID := $(shell grep '^ID=' /etc/os-release | cut -d= -f2) 16 | ifeq ($(OSID),ubuntu) 17 | OMPICC=/usr/bin/mpicc.openmpi 18 | OMPICXX=/usr/bin/mpicxx.openmpi 19 | MPICHCC=/usr/bin/mpicc.mpich 20 | MPICHCXX=/usr/bin/mpicxx.mpich 21 | CC=gcc 22 | endif 23 | ifeq ($(OSID),fedora) 24 | OMPICC=/usr/lib64/openmpi/bin/mpicc 25 | OMPICXX=/usr/lib64/openmpi/bin/mpicxx 26 | MPICHCC=/usr/lib64/mpich/bin/mpicc 27 | MPICHCXX=/usr/lib64/mpich/bin/mpicxx 28 | CFLAGS=-fmax-errors=1 # GCC 29 | #CFLAGS+= -fsanitize=address 30 | # these suppress true errors with callbacks 31 | #CFLAGS+=-Wno-incompatible-pointer-types 32 | #CFLAGS+=-Wno-cast-function-type 33 | #CFLAGS+=-Wno-unused-parameter -Wno-unused-variable -Wno-unused-function 34 | endif 35 | endif 36 | -include Makefile.local 37 | 38 | CFLAGS += -g -O2 -Wall -Wextra #-Werror # -Wpedantic 39 | CFLAGS += -fPIC 40 | CXXFLAGS = -x c++ -std=c++17 41 | 42 | SO = $(if $(findstring Darwin,$(UNAME_S)),dylib,so) 43 | RPATH = $(if $(findstring Darwin,$(UNAME_S)),'@rpath','$$ORIGIN') 44 | SOFLAGS = -shared 45 | SOLIBS = 46 | 47 | AR = ar 48 | ARFLAGS = -r 49 | 50 | 51 | all: libs tests 52 | 53 | RUNTESTS = testcoll.x testcoll2.x testcomm.x testinit.x testreqs.x \ 54 | testwin.x testgroup.x testtypes.x testtypes2.x testops.x \ 55 | testbottom.x testcart.x testerrh.x 56 | 57 | tests: header.o testconstants.x $(RUNTESTS) 58 | 59 | other: testmalloc.x 60 | 61 | testmalloc.x: testmalloc.c 62 | $(OMPICC) $(CFLAGS) $< -o $@ 63 | 64 | libs: libmuk.a libmuk.$(SO) mpich-wrap.$(SO) ompi-wrap.$(SO) 65 | 66 | %.x: %.c libmuk.$(SO) mpi.h 67 | $(CC) $(CFLAGS) $< -L. -Wl,-rpath,$(RPATH) -lmuk -o $@ 68 | 69 | MPI_H = mpi.h mpi-constants.h mpi-handle-typedefs.h mpi-typedefs.h mpi-predefined.h mpi-prototypes.h mpi-fortran.h muk-predefined.h 70 | 71 | IMPL_H = impl-alltoallw.h impl-constant-conversions.h \ 72 | impl-fpointers.h impl-handle-conversions.h \ 73 | impl-predefined-handle.h impl-scalar-types.h \ 74 | impl-status.h impl-keyval-map.h 75 | 76 | IMPL_CXX_H = impl-keyval-map-commattr.h \ 77 | impl-keyval-map-typeattr.h \ 78 | impl-keyval-map-winattr.h \ 79 | impl-keyval-map-commerrh.h \ 80 | impl-keyval-map-fileerrh.h \ 81 | impl-keyval-map-fileerrh2.h \ 82 | impl-keyval-map-winerrh.h \ 83 | impl-keyval-map-opuserfn.h \ 84 | impl-keyval-map-preqa2aw.h 85 | 86 | IMPL_FUNCTION_C := impl-functions.c impl-load-functions.c impl-keyval.c \ 87 | impl-constant-conversions.c impl-predefined.c \ 88 | impl-commgroup-functions.c impl-rma-functions.c \ 89 | impl-wait-functions.c impl-session-functions.c \ 90 | impl-file-functions.c impl-reduce-functions.c \ 91 | impl-type-functions.c impl-grequest-functions.c 92 | 93 | IMPL_FUNCTION_O := $(patsubst %.c,%.o,$(IMPL_FUNCTION_C)) 94 | MPICH_FUNCTION_O := $(subst impl,mpich,$(IMPL_FUNCTION_O)) mpich-keyval-map.o 95 | OMPI_FUNCTION_O := $(subst impl,ompi,$(IMPL_FUNCTION_O)) ompi-keyval-map.o 96 | 97 | # this just tests if mpi.h can be compiled without errors 98 | header.o: header.c $(MPI_H) 99 | $(CC) $(CFLAGS) -c $< -o $@ 100 | 101 | libmuk.a: libinit.o 102 | $(AR) $(ARFLAGS) $@ $< 103 | 104 | 105 | ifeq ($(UNAME_S),Darwin) 106 | libmuk.$(SO): WRAPLIBS+=-Wl,-rpath,$(RPATH) 107 | libmuk.$(SO): WRAPLIBS+=-Wl,mpich-wrap.dylib 108 | libmuk.$(SO): WRAPLIBS+=-Wl,ompi-wrap.dylib 109 | endif 110 | libmuk.$(SO): SOLIBS+=-ldl 111 | libmuk.$(SO): libinit.o | mpich-wrap.$(SO) ompi-wrap.$(SO) 112 | $(CC) $< $(SOFLAGS) $(SOLIBS) $(WRAPLIBS) -o $@ 113 | 114 | libinit.o: libinit.c muk.h muk-dl.h $(MPI_H) 115 | $(CC) $(CFLAGS) -c $< -o $@ 116 | 117 | libinit.i: libinit.c muk.h muk-dl.h $(MPI_H) 118 | $(CC) $(CFLAGS) -E $< -o $@ 119 | 120 | mpich-wrap.$(SO): $(MPICH_FUNCTION_O) 121 | $(MPICHCXX) $(SOFLAGS) $(SOLIBS) $^ -o $@ 122 | 123 | ompi-wrap.$(SO): $(OMPI_FUNCTION_O) 124 | $(OMPICXX) $(SOFLAGS) $(SOLIBS) $^ -o $@ 125 | 126 | mpich-predefined.o: impl-predefined.c muk-predefined.h 127 | $(MPICHCC) $(CFLAGS) -c $< -o $@ 128 | 129 | ompi-predefined.o: impl-predefined.c muk-predefined.h 130 | $(OMPICC) $(CFLAGS) -c $< -o $@ 131 | 132 | mpich-functions.o: impl-functions.c $(IMPL_H) 133 | $(MPICHCC) $(CFLAGS) -c $< -o $@ 134 | 135 | ompi-functions.o: impl-functions.c $(IMPL_H) 136 | $(OMPICC) $(CFLAGS) -c $< -o $@ 137 | 138 | mpich-commgroup-functions.o: impl-commgroup-functions.c $(IMPL_H) 139 | $(MPICHCC) $(CFLAGS) -c $< -o $@ 140 | 141 | ompi-commgroup-functions.o: impl-commgroup-functions.c $(IMPL_H) 142 | $(OMPICC) $(CFLAGS) -c $< -o $@ 143 | 144 | mpich-reduce-functions.o: impl-reduce-functions.c $(IMPL_H) 145 | $(MPICHCC) $(CFLAGS) -c $< -o $@ 146 | 147 | ompi-reduce-functions.o: impl-reduce-functions.c $(IMPL_H) 148 | $(OMPICC) $(CFLAGS) -c $< -o $@ 149 | 150 | mpich-rma-functions.o: impl-rma-functions.c $(IMPL_H) 151 | $(MPICHCC) $(CFLAGS) -c $< -o $@ 152 | 153 | ompi-rma-functions.o: impl-rma-functions.c $(IMPL_H) 154 | $(OMPICC) $(CFLAGS) -c $< -o $@ 155 | 156 | mpich-grequest-functions.o: impl-grequest-functions.c $(IMPL_H) 157 | $(MPICHCC) $(CFLAGS) -c $< -o $@ 158 | 159 | ompi-grequest-functions.o: impl-grequest-functions.c $(IMPL_H) 160 | $(OMPICC) $(CFLAGS) -c $< -o $@ 161 | 162 | mpich-type-functions.o: impl-type-functions.c $(IMPL_H) 163 | $(MPICHCC) $(CFLAGS) -c $< -o $@ 164 | 165 | ompi-type-functions.o: impl-type-functions.c $(IMPL_H) 166 | $(OMPICC) $(CFLAGS) -c $< -o $@ 167 | 168 | mpich-wait-functions.o: impl-wait-functions.c $(IMPL_H) 169 | $(MPICHCC) $(CFLAGS) -c $< -o $@ 170 | 171 | ompi-wait-functions.o: impl-wait-functions.c $(IMPL_H) 172 | $(OMPICC) $(CFLAGS) -c $< -o $@ 173 | 174 | mpich-file-functions.o: impl-file-functions.c $(IMPL_H) 175 | $(MPICHCC) $(CFLAGS) -c $< -o $@ 176 | 177 | ompi-file-functions.o: impl-file-functions.c $(IMPL_H) 178 | $(OMPICC) $(CFLAGS) -c $< -o $@ 179 | 180 | mpich-session-functions.o: impl-session-functions.c $(IMPL_H) 181 | $(MPICHCC) $(CFLAGS) -c $< -o $@ 182 | 183 | ompi-session-functions.o: impl-session-functions.c $(IMPL_H) 184 | $(OMPICC) $(CFLAGS) -c $< -o $@ 185 | 186 | mpich-load-functions.o: impl-load-functions.c $(IMPL_H) 187 | $(MPICHCC) $(CFLAGS) -c $< -o $@ 188 | 189 | ompi-load-functions.o: impl-load-functions.c $(IMPL_H) 190 | $(OMPICC) $(CFLAGS) -c $< -o $@ 191 | 192 | mpich-constant-conversions.o: impl-constant-conversions.c $(IMPL_H) 193 | $(MPICHCC) $(CFLAGS) -c $< -o $@ 194 | 195 | ompi-constant-conversions.o: impl-constant-conversions.c $(IMPL_H) 196 | $(OMPICC) $(CFLAGS) -c $< -o $@ 197 | 198 | mpich-keyval.o: impl-keyval.c $(IMPL_H) 199 | $(MPICHCC) $(CFLAGS) -c $< -o $@ 200 | 201 | ompi-keyval.o: impl-keyval.c $(IMPL_H) 202 | $(OMPICC) $(CFLAGS) -c $< -o $@ 203 | 204 | mpich-keyval-map.o: impl-keyval-map.cc $(IMPL_H) $(IMPL_CXX_H) 205 | $(MPICHCC) $(CXXFLAGS) $(CFLAGS) -c $< -o $@ 206 | 207 | ompi-keyval-map.o: impl-keyval-map.cc $(IMPL_H) $(IMPL_CXX_H) 208 | $(OMPICC) $(CXXFLAGS) $(CFLAGS) -c $< -o $@ 209 | 210 | check: $(RUNTESTS) 211 | ./test.sh ./testcoll.x 212 | ./test.sh ./testcoll2.x 213 | ./test.sh ./testcomm.x 214 | ./test.sh ./testcart.x 215 | ./test.sh ./testgroup.x 216 | ./test.sh ./testinit.x 217 | ./test.sh ./testops.x 218 | ./test.sh ./testreqs.x 219 | ./test.sh ./testtypes.x 220 | ./test.sh ./testtypes2.x 221 | ./test.sh ./testwin.x 222 | ./test.sh ./testbottom.x 223 | ./test.sh ./testerrh.x 224 | 225 | clean: 226 | -rm -f *.o *.x *.s *.a *.i *.$(SO) 227 | -rm -rf *.dSYM 228 | -rm -rf *.btr # backtrace 229 | -------------------------------------------------------------------------------- /NOTES.md: -------------------------------------------------------------------------------- 1 | # General 2 | 3 | ``` 4 | export MUK_PATH=${HOME}/mukautuva 5 | export MUK_PATH=${HOME}/Work/MPI/mukautuva 6 | export LD_LIBRARY_PATH=${MUK_PATH} 7 | export DYLD_LIBRARY_PATH=${MUK_PATH} 8 | ``` 9 | 10 | # Intel MPI 11 | ``` 12 | . /opt/intel/oneapi/setvars.sh --force 13 | ``` 14 | 15 | # My Tests 16 | 17 | ``` 18 | for t in testcoll.x testcomm.x testinit.x testreqs.x testwin.x testgroup.x testtypes.x testops.x ; do ./test.sh ./$t ; done 19 | ``` 20 | 21 | # OSU MPI Tests 22 | 23 | ``` 24 | wget https://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-7.0.1.tar.gz 25 | tar -xzf osu*gz 26 | cd osu* 27 | ./configure CC=gcc CXX=g++ CFLAGS=-g3 --enable-g CPPFLAGS=-I${MUK_PATH} LDFLAGS=-L${MUK_PATH} LIBS=-lmuk && make -j8 28 | ``` 29 | 30 | Run tests: 31 | ``` 32 | cd c/mpi 33 | # Linux 34 | for t in `find . -type f -executable -print` ; do LD_LIBRARY_PATH=${MUK_PATH} mpirun -n 2 ./$t ; done 35 | # Mac 36 | for t in `find . -type f -perm 755 -print` ; do LD_LIBRARY_PATH=${MUK_PATH} mpirun -n 2 ./$t ; done 37 | ``` 38 | 39 | At least one RMA test with Open-MPI requires `--mca osc ucx` (vader bug). https://github.com/jeffhammond/mukautuva/issues/2 40 | 41 | MPICH with OFI requires `RDMAV_FORK_SAFE=1`. https://github.com/jeffhammond/mukautuva/issues/3 42 | 43 | # ARMCI-MPI 44 | 45 | Clone as a subdirectory of `./mukautuva`: 46 | ``` 47 | git clone https://github.com/pmodels/armci-mpi.git 48 | ``` 49 | 50 | Configure like this, e.g.: 51 | ``` 52 | ./configure CC=gcc CFLAGS=-g3 --enable-g CPPFLAGS=-I${MUK_PATH} LDFLAGS="-L${MUK_PATH} -Wl,-rpath=${MUK_PATH}" LIBS="-lmuk -ldl" 53 | ``` 54 | 55 | Debug a single test failure like this: 56 | ``` 57 | make checkprogs -j4 && LD_LIBRARY_PATH=.. MUK_MPI_LIB=/usr/lib/x86_64-linux-gnu/libmpich.so mpirun.mpich -n 4 gdb -ex "set width 1000" -ex "thread apply all bt" -ex run -ex bt -ex "set confirm off" -ex quit --args ./tests/test_groups 58 | make checkprogs -j4 && LD_LIBRARY_PATH=.. MUK_MPI_LIB=/usr/lib/x86_64-linux-gnu/libmpi.so mpirun.openmpi -n 4 gdb -ex "set width 1000" -ex "thread apply all bt" -ex run -ex bt -ex "set confirm off" -ex quit --args ./tests/test_groups 59 | ``` 60 | 61 | # MPICH Test Suite 62 | 63 | Required on Linux with Open-MPI (see https://github.com/open-mpi/ompi/issues/7701). 64 | ```sh 65 | export HWLOC_COMPONENTS=-gl 66 | ``` 67 | 68 | Required because of https://github.com/pmodels/mpich/issues/6423: 69 | ``` 70 | export MPIR_CVAR_IALLGATHER_INTRA_ALGORITHM=sched_ring 71 | ``` 72 | 73 | ```sh 74 | export LD_LIBRARY_PATH=${MUK_PATH} 75 | mkdir -p ${MUK_PATH}/bin 76 | ln -sf `which gcc` ${MUK_PATH}/bin/mpicc 77 | ``` 78 | 79 | # MacOS 80 | ```sh 81 | ln -s ${MUK_PATH}/bin/mpicc /usr/bin/clang 82 | ln -s ${MUK_PATH}/bin/mpicxx /usr/bin/clang++ 83 | ln -s ${MUK_PATH}/bin/mpifort /opt/homebrew/bin/gfortran 84 | # in mpich-testsuite... 85 | ln -s ../ompi-wrap.so . 86 | ln -s ../mpich-wrap.so . 87 | ln -s ../libmuk.so libmuk.so 88 | ``` 89 | 90 | ``` 91 | wget https://www.mpich.org/static/downloads/4.1/mpich-testsuite-4.1.tar.gz 92 | tar -xaf mpich-testsuite-4.1.tar.gz 93 | cd mpich-testsuite-4.1/ 94 | # Linux 95 | ./configure CC=gcc CXX=g++ FC=false CPPFLAGS="-I${MUK_PATH}" LDFLAGS="-L${MUK_PATH}" LIBS="-lmuk" --enable-strictmpi --with-mpi=${MUK_PATH} 96 | # MacOS 97 | ./configure CPPFLAGS="-I${MUK_PATH}" LDFLAGS="-L${MUK_PATH}" LIBS="-lmuk" --enable-strictmpi --with-mpi=${MUK_PATH} --disable-cxx CC=clang CXX=clang++ 98 | ``` 99 | 100 | Useful for running multiple tests manually: 101 | ```sh 102 | find . -type f -executable -print 103 | for t in `find . -type f -executable -print` ; do echo Starting $t && mpirun -n 4 $t ; done 104 | ``` 105 | 106 | Run tests the right way: 107 | ``` 108 | MPIEXEC="mpirun.openmpi --oversubscribe --quiet" \ 109 | ./runtests -tests=testlist -maxnp=8 -verbose -showprogress -debug 110 | ``` 111 | 112 | # Intel MPI Benchmarks 113 | 114 | ```sh 115 | git clone https://github.com/jeffhammond/mpi-benchmarks.git IMB 116 | ``` 117 | 118 | ```sh 119 | cd src_c && make -j8 120 | # DYLD_LIBRARY_PATH=${HOME}/Work/MPI/mukautuva 121 | for p in IMB-EXT IMB-MPI1 IMB-NBC IMB-RMA IMB-IO ; do mpirun -n 4 ./$p ; done 122 | ``` 123 | 124 | ```sh 125 | cd src_cpp && make -j8 126 | for t in example EXT HALO IO MPI1 MT NBC RMA ; do make TARGET=$t ; done 127 | for t in example EXT HALO MPI1 MT NBC RMA ; do mpirun -n 4 ./IMB-$t ; done 128 | ``` 129 | 130 | # Open-MPI Debug Build 131 | 132 | ``` 133 | 134 | 135 | ``` 136 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Mukautuva 2 | 3 | Mukautuva means "adaptable" in Finnish. The purpose of Mukautuva (MUK for short) is to allow one to compile 4 | an MPI application once against the MUK ABI, which is a prototype of the MPI-5 ABI we are developing, 5 | and run using any MPI implementation. 6 | 7 | Currently, we support Open-MPI, MPICH and Intel MPI. The latter two are already ABI-compatible, while 8 | MPICH and Open-MPI are quite different. Internally, MUK supports MPICH and Open-MPI, so if you use an 9 | MPI ABI that is not based on either of these, some work may be required. However, as long as an MPI 10 | implementation handle is compatible with an `int`, a `void*`, or an `intptr_t`, then the only work to 11 | support it is detection. 12 | 13 | To use MUK, you compile using the implementation-agnostic parts with the system C compiler. 14 | You must tell the build system which `mpicc` to use for MPICH and Open-MPI. 15 | At runtime, you must use the appropriate `mpirun` for your implementation, which is specified 16 | by `MUK_MPI_LIB=`. MUK knows the defaults for some systems so you can use 17 | `mpirun` and not set `MUK_MPI_LIB` and it will probably work. 18 | 19 | You configure the build system by editing the top of `Makefile`. 20 | More complex build systems are a pointless distraction and will not be supported. 21 | 22 | # Current status 23 | 24 | The overwhelming majority of features are working, including all the common ones. 25 | 26 | ## Successes 27 | 28 | The following are known to work: 29 | - The [ARMCI-MPI](https://github.com/pmodels/armci-mpi) test suite, which exercises passive target RMA and some interesting group and communicator features. 30 | - The OSU MPI Benchmarks ([OMB](https://mvapich.cse.ohio-state.edu/benchmarks/)), which test MPI performance. 31 | - The Intel MPI Benchmarks ([IMB](https://github.com/intel/mpi-benchmarks)), which test MPI performance, including I/O. 32 | - Most of the MPICH test suite, with exceptions noted below. 33 | 34 | Specifically, the following MPICH test groups are passing: 35 | - Attributes: all OK. 36 | - Basic: all OK. 37 | - Collectives: all OK. 38 | - Comm: all OK. 39 | - Datatypes: all OK. 40 | - Group: all OK. 41 | - Initialization: see below. 42 | - IO: all OK. 43 | - P2P: mostly OK. Failures related to Fortran handles (maybe). 44 | - RMA: all OK. 45 | - Topology: all OK. 46 | 47 | ## Known Issues 48 | 49 | - Spawn is not supported. Simple cases will work, but the shared-library lookup situation is more complicated with Spawn on multiple nodes. 50 | - Most of the things that are supposed to work before MPI_Init don’t, because I need to reorganize all the `dlsym` stuff to make that work. 51 | - Sessions are not supported because they are a more complicated version of the "before init" problem. 52 | - Fortran interoperability features (handle conversion) are ignored. These will be implemented in conjunction with [VAPAA](https://github.com/jeffhammond/vapaa) at some point. 53 | 54 | There are things that do not work because the underlying implementation does not support them yet. For example, MPICH has a bug with `MPI_BOTTOM` throwing an error when it shouldn't, and Open-MPI does not support much of MPI-4 right now. 55 | 56 | # Design 57 | 58 | Most things are straightforward. Some times are not. 59 | 60 | ## Layering 61 | 62 | We need to create an abstraction layer between the MPI namespace that Mukautuva exposes to MPI applications 63 | and the MPI implementation namespace. This is done using `dlopen`. There are two shared libraries. 64 | The first is `libmuk.so`, which implements the MPI API in terms of an internal namespace, MUK. 65 | The MUK API is almost identical to the MPI. There is one pair of exceptions that are not interesting. 66 | When possible, we implement deprecated or deleted APIs in terms of their replacements. 67 | 68 | The second shared library is implementation dependent, so it is named `ompi-wrap.so` or `mpich-wrap.so`. 69 | It implements the MUK API in the WRAP namespace. The MPI implementation is loaded with symbols that 70 | have the IMPL namespace prefix. 71 | 72 | The `libmuk.so` library is implemented in `libinit.c`. Almost all of it is boilerplate to `dlsym` 73 | MUK functions to the WRAP ones in the back-end. However, it first has to figure out which implementation 74 | is used. It does this by looking at `libmpi.so` (or whatever the user provides via the `MUK_MPI_LIB` 75 | environment variable). Because the necessary features for this are ABI-agnostic (i.e. they use 76 | only built-in C types), we can load these functions directly from the implementation shared object (SO). 77 | We call `MPI_Get_library_version` to see if we are using Open-MPI or an MPICH-derivative 78 | (only Intel MPI is recognized right now, but MVAPICH2 is trivial to add). 79 | Once we know if we are working with the Open-MPI or MPICH ABI, we `dlopen` the appropriate `-wrap.so` 80 | and `dlsym` all of the WRAP symbols. 81 | 82 | Because all of the `dlsym` mappings are setup during MPI initialization, most of the MPI functionality 83 | that is required to work before initialization does not work correctly. 84 | We implement the ABI-agnostic operations to some extent, since they are easy. 85 | For similar reasons, we do not implement MPI sessions, which are not widely available in implementations 86 | right now anyways. 87 | 88 | We are definitely not using `dlopen` optimally right now. This project is the first time 89 | I've used it and I am only doing what is absolutely necessary to make Mukautuva work on Linux 90 | and MacOS. Some things are brittle due to lack of careful namespacing, which we triggered 91 | during development by using `MPI_Comm_rank` in the wrong place, which caused the MPI 92 | implementation symbols to be used instead of Mukautuva MPI symbols, which causes 93 | some really nasty, hard-to-debug crashes. 94 | Erik Schnetter appears to be doing sensible things in 95 | [MPItrampoline](https://github.com/eschnett/MPItrampoline), so we will try 96 | to learn from that at some point. 97 | 98 | ## Translation of predefined constants and handles 99 | 100 | All of the WRAP functions map the MUK ABI to the IMPL ABI. 101 | For example, we have special integer constants like `MPI_ANY_SOURCE` and `MPI_PROC_NULL` that 102 | are not standardized values, so we have a function to convert all rank-like arguments. 103 | Similar recognition and translation occurs for buffer sentinels like `MPI_STATUS_IGNORE`. 104 | Wherever possible, i.e. when MPICH and Open-MPI agree on values, e.g. `MPI_BOTTOM=NULL`, 105 | we just use the same value to avoid translation overhead. 106 | MPICH made an unfortunate design choice with `MPI_UNWEIGHTED`, we must handle this one 107 | in a special way, which is the one exception to the 1:1 mapping between MPI and MUK APIs. 108 | 109 | MUK handles are exposed to the user as incomplete `struct` pointers for type-safety. 110 | Internally, they are a `union` of `{ int, void*, intptr_t }`. 111 | We use the appropriate field for the MPICH and Open-MPI ABI, respectively, 112 | and `intptr_t` to `printf` handles in diagnostic messages (one must ignore 113 | the `0xFFFF..` junk bits to recognize MPICH handles). 114 | We should implement a proper abstraction for printing handles but since 115 | these should only be read by developers, it is not a high priority. 116 | 117 | ## Translation of callbacks 118 | 119 | Callbacks are the most difficult part of Mukautuva. 120 | Recall that the application will be compiled against the MUK ABI. 121 | Callbacks are invoked by the MPI implementation with the IMPL ABI. 122 | Furthermore, even when handles are passed by pointer, they are checked 123 | for validity, so we can't just pass a `MUK_Datatype*` when a `IMPL_Datatype*` 124 | is expected, because this will cause an error. 125 | 126 | The way we support callbacks is to replace the callback argument with a 127 | trampoline function that matches the function pointer signature of the 128 | IMPL ABI. The trampoline must look up the WRAP ABI function pointer 129 | and call it with the appropriate translated handles, as well as do 130 | back-translation of error codes, where appropriate. 131 | 132 | MPI is far from consistent in the callback signatures, so we have to use 133 | different strategies. 134 | 135 | ## Reduction callbacks 136 | 137 | In `MPI_Op_create`, we create a map (a C++ `std::map`) between the 138 | `MPI_User_function(_c)` and the `MPI_Op` handle. 139 | When the `MPI_Op` is used inside of e.g. `MPI_Allreduce`, 140 | we lookup the function pointer via the `MPI_Op` and attach 141 | it to a duplicate of the `MPI_Datatype` argument to that function. 142 | Why the `MPI_Datatype` and why a duplicate? 143 | We use the `MPI_Datatype` because that is the only thing we 144 | can uniquely identify inside of the trampoline. 145 | It is a duplicate because we can't be sure that the user won't 146 | call two different reductions with the same datatype before 147 | the callback is done. 148 | Inside of the callback, we get the datatype attribute 149 | that contains the function pointer of the original callback, 150 | and we invoke it with the translated datatype, then free 151 | it if it was the copy of derived datatype. 152 | 153 | The biggest problem here is that MPI implementations are expected 154 | to ref-count everything, so it is legal to call `Op_create`, 155 | `Iallreduce` and `Op_free` before calling `Wait`, which means 156 | that the op-callback mapping will be destroyed while the reduction 157 | is in-flight. 158 | Our first few designs were bad. We used a heap object for the attribute, 159 | which is freed during `Op_free`. We can fix this and just 160 | attach the function pointer directly to the datatype 161 | so no heap allocation is necessary. However, we store the mapping 162 | between the `MPI_Op` and `MPI_User_function(_c)` forever, which 163 | has trivial space and time overheads in all but the most pathological 164 | use cases. (If you call `MPI_Op_create` millions of times in your program, 165 | I don't know what to tell you.) 166 | 167 | ## Errhandler callbacks 168 | 169 | It is legal - and practiced by the MPICH test suite - to free 170 | an errhandler and still use it. 171 | We solved this by using the function pointer as the attribute directly. 172 | Obviously, the function pointer itself cannot go out of scope, 173 | so this eliminates the use-after-free issue above. 174 | 175 | Unfortunately, `MPI_File` objects don't support attributes, so 176 | we need a second mapping between those handles and the errhandler 177 | callbacks, but the overheads of anything related to file errhandlers 178 | is irrelevant compared to actual file I/O. 179 | 180 | ## Attribute callbacks 181 | 182 | We use the C++ `std::map` container, which provides exactly what we need, 183 | at the small cost of requiring awareness of C++ in the project. 184 | The C++ code itself is encapsulated in a single file, which 185 | implements a C API that can be used in the rest of the library 186 | without requiring any C++. 187 | The build system links the shared libraries using a C++ compiler 188 | to ensure that the C++ runtime library is linked automatically. 189 | 190 | Ironically, using C++ defeats the purpose of having a clean ABI 191 | abstraction layer, because we are now dependent on the C++ ABI, 192 | but this is not expected to be a major issue, since almost everyone 193 | will use the system default C++ library. However, since 194 | we are only using C++ in the IMPL back-end, we can, in theory, 195 | `dlopen` the correct C++ library there without interfering with 196 | the user's use of a different C++ library, or aversion to C++ 197 | altogether, but we are not going to implement this without 198 | significant incentives. 199 | 200 | ## Nonblocking alltoallw 201 | 202 | All of the nonblocking variants of nonblocking alltoallw 203 | require us to track the state of the temporary arrays of 204 | converted datatypes, which cannot be freed until the operation 205 | finishes. We implement this using a mapping between the 206 | nonblocking request and array pointers. This means that 207 | all request completion functions need to look to see 208 | if there are any completed requests associated with 209 | alltoallw state. This introduces some overhead if we do 210 | something silly like initiate a nonblocking alltoall, 211 | then run an MPI message rate benchmark that calls 212 | `MPI_Waitall` with lots of requests. However, if there 213 | are no outstanding nonblocking alltoalw operations, the 214 | overhead is a single branch (per request, although we could 215 | theoretically optimize it to one branch per request-completion 216 | call). 217 | 218 | ## Persistent collectives 219 | 220 | Persistent reductions are working correctly. 221 | 222 | ## Generalized requests 223 | 224 | Because generalized requests use `extra_state`, we can stick everything to support 225 | the trampoline in there. 226 | 227 | # Acknowledgements 228 | 229 | Hui Zhou and Ken Raffineti of the MPICH team were extremely helpful when debugging a number of issues with Mukautuva. 230 | 231 | My team at NVIDIA allowed me to focus on this project for the month or so it took to create. 232 | -------------------------------------------------------------------------------- /generate-datatype-table.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | let c=0 4 | for t in \ 5 | MPI_DATATYPE_NULL \ 6 | MPI_CHAR \ 7 | MPI_SHORT \ 8 | MPI_INT \ 9 | MPI_LONG \ 10 | MPI_LONG_LONG_INT \ 11 | MPI_LONG_LONG \ 12 | MPI_SIGNED_CHAR \ 13 | MPI_UNSIGNED_CHAR \ 14 | MPI_UNSIGNED_SHORT \ 15 | MPI_UNSIGNED \ 16 | MPI_UNSIGNED_LONG \ 17 | MPI_UNSIGNED_LONG_LONG \ 18 | MPI_FLOAT \ 19 | MPI_DOUBLE \ 20 | MPI_LONG_DOUBLE \ 21 | MPI_WCHAR \ 22 | MPI_C_BOOL \ 23 | MPI_INT8_T \ 24 | MPI_INT16_T \ 25 | MPI_INT32_T \ 26 | MPI_INT64_T \ 27 | MPI_UINT8_T \ 28 | MPI_UINT16_T \ 29 | MPI_UINT32_T \ 30 | MPI_UINT64_T \ 31 | MPI_AINT \ 32 | MPI_COUNT \ 33 | MPI_OFFSET \ 34 | MPI_C_COMPLEX \ 35 | MPI_C_FLOAT_COMPLEX \ 36 | MPI_C_DOUBLE_COMPLEX \ 37 | MPI_C_LONG_DOUBLE_COMPLEX \ 38 | MPI_BYTE \ 39 | MPI_PACKED \ 40 | MPI_CXX_BOOL \ 41 | MPI_CXX_FLOAT_COMPLEX \ 42 | MPI_CXX_DOUBLE_COMPLEX \ 43 | MPI_CXX_LONG_DOUBLE_COMPLEX \ 44 | MPI_INTEGER \ 45 | MPI_REAL \ 46 | MPI_DOUBLE_PRECISION \ 47 | MPI_COMPLEX \ 48 | MPI_LOGICAL \ 49 | MPI_CHARACTER \ 50 | MPI_DOUBLE_COMPLEX \ 51 | MPI_INTEGER1 \ 52 | MPI_INTEGER2 \ 53 | MPI_INTEGER4 \ 54 | MPI_INTEGER8 \ 55 | MPI_INTEGER16 \ 56 | MPI_REAL2 \ 57 | MPI_REAL4 \ 58 | MPI_REAL8 \ 59 | MPI_REAL16 \ 60 | MPI_COMPLEX4 \ 61 | MPI_COMPLEX8 \ 62 | MPI_COMPLEX16 \ 63 | MPI_COMPLEX32 \ 64 | MPI_FLOAT_INT \ 65 | MPI_DOUBLE_INT \ 66 | MPI_LONG_INT \ 67 | MPI_2INT \ 68 | MPI_SHORT_INT \ 69 | MPI_LONG_DOUBLE_INT \ 70 | MPI_2REAL \ 71 | MPI_2DOUBLE_PRECISION \ 72 | MPI_2INTEGER \ 73 | MPI_LB \ 74 | MPI_UB \ 75 | ; do 76 | #echo "#define ${t} ((MPI_Datatype)${c})" 77 | u=$(echo "$t" | sed "s/MPI/MUK/") 78 | #echo "${t} ${u}" 79 | # FORWARD 80 | # echo "#ifdef ${t} 81 | # else if (datatype.ip == (intptr_t)${u}) { 82 | # return ${t}; 83 | # } 84 | ##endif" 85 | # BACKWARD 86 | echo "#ifdef ${t} 87 | else if (datatype == ${t}) { 88 | wrap.ip = (intptr_t)${u}; 89 | } 90 | #endif" 91 | let c++ 92 | done 93 | -------------------------------------------------------------------------------- /generate-op-table.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | let c=0 4 | for t in \ 5 | MPI_SUM \ 6 | MPI_NO_OP \ 7 | MPI_REPLACE \ 8 | MPI_MAX \ 9 | MPI_MIN \ 10 | MPI_LAND \ 11 | MPI_BAND \ 12 | MPI_LOR \ 13 | MPI_BOR \ 14 | MPI_LXOR \ 15 | MPI_BXOR \ 16 | MPI_MAXLOC \ 17 | MPI_MINLOC \ 18 | MPI_OP_NULL \ 19 | MPI_PROD \ 20 | ; do 21 | #echo "#define ${t} ((MPI_Op)${c})" 22 | u=$(echo "$t" | sed "s/MPI/MUK/") 23 | #echo "${t} ${u}" 24 | # FORWARD 25 | echo " else if (op.ip == (intptr_t)${u}) { 26 | return ${t}; 27 | }" 28 | # BACKWARD 29 | # echo " else if (op == ${t}) { 30 | # wrap.ip = (intptr_t)${u}; 31 | # }" 32 | let c++ 33 | done 34 | -------------------------------------------------------------------------------- /header.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | #include "mpi.h" 4 | 5 | int main(int argc, char* argv[]) 6 | { 7 | MPI_Init(&argc,&argv); 8 | MPI_Finalize(); 9 | return 0; 10 | } 11 | -------------------------------------------------------------------------------- /impl-alltoallw.h: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | #ifndef IMPL_ALLTOALLW_H 4 | #define IMPL_ALLTOALLW_H 5 | 6 | static int ALLTOALLW_SETUP(bool in_place, const MPI_Comm comm, const WRAP_Datatype sendtypes[], const WRAP_Datatype recvtypes[], 7 | MPI_Datatype ** impl_sendtypes, MPI_Datatype ** impl_recvtypes) 8 | { 9 | int rc; 10 | int sendnum, recvnum; 11 | 12 | // ignore this for now - we can always allocate too much memory for types in neigherbor_alltoallw 13 | #if 0 14 | int topo; 15 | rc = MPI_Topo_test(comm, &topo); 16 | if (rc != MPI_SUCCESS) { 17 | return rc; 18 | } 19 | #endif 20 | 21 | int is_intercomm; 22 | rc = IMPL_Comm_test_inter(comm, &is_intercomm); 23 | if (rc != MPI_SUCCESS) { 24 | return rc; 25 | } 26 | 27 | if (is_intercomm) { 28 | int remote_size; 29 | rc = IMPL_Comm_remote_size(comm,&remote_size); 30 | if (rc != MPI_SUCCESS) { 31 | return rc; 32 | } 33 | sendnum = remote_size; 34 | recvnum = remote_size; 35 | } 36 | #if 0 37 | else if (topo == MPI_GRAPH || topo == MPI_CART || topo == MPI_DIST_GRAPH) { 38 | } 39 | #endif 40 | else { 41 | int size; 42 | rc = IMPL_Comm_size(comm,&size); 43 | if (rc != MPI_SUCCESS) { 44 | return rc; 45 | } 46 | sendnum = size; 47 | recvnum = size; 48 | } 49 | //printf("sendnum=%d recvnum=%d\n", sendnum, recvnum); fflush(0); 50 | 51 | if (!in_place) { 52 | *impl_sendtypes = calloc(sendnum,sizeof(MPI_Datatype)); 53 | if (*impl_sendtypes == NULL) return MPI_ERR_OTHER; 54 | for (int i=0; i 7 | #endif 8 | 9 | #include 10 | 11 | #include "muk-predefined.h" 12 | 13 | // declaration for impl-functions.c symbol 14 | // impl-fpointers.h could be used instead but we need only two 15 | extern int (*IMPL_Comm_rank)(MPI_Comm comm, int * rank); 16 | extern int (*IMPL_Error_class)(int errorcode, int *errorclass); 17 | extern int (*IMPL_Error_string)(int errorcode, char *string, int *resultlen); 18 | 19 | int ERROR_CODE_IMPL_TO_MUK(int error_c) 20 | { 21 | int error; 22 | // "To make it possible for an application to interpret an error code, 23 | // the routine MPI_ERROR_CLASS converts any error code into one of a 24 | // small set of standard error codes, called error classes." 25 | // MPI_Error_class is one of the functions that can be called after MPI_Finalize. 26 | IMPL_Error_class(error_c, &error); 27 | 28 | // If, for some reason, you need to known the actual error returned from 29 | // the MPI library, and not just the class, enable the following code. 30 | #if 0 31 | int me; 32 | IMPL_Comm_rank(MPI_COMM_WORLD,&me); 33 | int len; 34 | char name[MPI_MAX_ERROR_STRING] = {0}; 35 | #if 0 36 | IMPL_Error_string(error, name, &len); 37 | printf("%d: Real error code returned from the C library: %d=%x, name=%s\n", 38 | me, error, error, name); 39 | #else 40 | IMPL_Error_string(error_c, name, &len); 41 | printf("%d: Real error code returned from the C library: %d=%x, name=%s\n", 42 | me, error_c, error_c, name); 43 | #endif 44 | #endif 45 | 46 | if (error == MPI_SUCCESS ) { return MUK_SUCCESS; } 47 | else if (error == MPI_ERR_BUFFER ) { return MUK_ERR_BUFFER; } 48 | else if (error == MPI_ERR_COUNT ) { return MUK_ERR_COUNT; } 49 | else if (error == MPI_ERR_TYPE ) { return MUK_ERR_TYPE; } 50 | else if (error == MPI_ERR_TAG ) { return MUK_ERR_TAG; } 51 | else if (error == MPI_ERR_COMM ) { return MUK_ERR_COMM; } 52 | else if (error == MPI_ERR_RANK ) { return MUK_ERR_RANK; } 53 | else if (error == MPI_ERR_REQUEST ) { return MUK_ERR_REQUEST; } 54 | else if (error == MPI_ERR_ROOT ) { return MUK_ERR_ROOT; } 55 | else if (error == MPI_ERR_GROUP ) { return MUK_ERR_GROUP; } 56 | else if (error == MPI_ERR_OP ) { return MUK_ERR_OP; } 57 | else if (error == MPI_ERR_TOPOLOGY ) { return MUK_ERR_TOPOLOGY; } 58 | else if (error == MPI_ERR_DIMS ) { return MUK_ERR_DIMS; } 59 | else if (error == MPI_ERR_ARG ) { return MUK_ERR_ARG; } 60 | else if (error == MPI_ERR_UNKNOWN ) { return MUK_ERR_UNKNOWN; } 61 | else if (error == MPI_ERR_TRUNCATE ) { return MUK_ERR_TRUNCATE; } 62 | else if (error == MPI_ERR_OTHER ) { return MUK_ERR_OTHER; } 63 | else if (error == MPI_ERR_INTERN ) { return MUK_ERR_INTERN; } 64 | else if (error == MPI_ERR_PENDING ) { return MUK_ERR_PENDING; } 65 | else if (error == MPI_ERR_IN_STATUS ) { return MUK_ERR_IN_STATUS; } 66 | else if (error == MPI_ERR_ACCESS ) { return MUK_ERR_ACCESS; } 67 | else if (error == MPI_ERR_AMODE ) { return MUK_ERR_AMODE; } 68 | else if (error == MPI_ERR_ASSERT ) { return MUK_ERR_ASSERT; } 69 | else if (error == MPI_ERR_BAD_FILE ) { return MUK_ERR_BAD_FILE; } 70 | else if (error == MPI_ERR_BASE ) { return MUK_ERR_BASE; } 71 | else if (error == MPI_ERR_CONVERSION ) { return MUK_ERR_CONVERSION; } 72 | else if (error == MPI_ERR_DISP ) { return MUK_ERR_DISP; } 73 | else if (error == MPI_ERR_DUP_DATAREP ) { return MUK_ERR_DUP_DATAREP; } 74 | else if (error == MPI_ERR_FILE_EXISTS ) { return MUK_ERR_FILE_EXISTS; } 75 | else if (error == MPI_ERR_FILE_IN_USE ) { return MUK_ERR_FILE_IN_USE; } 76 | else if (error == MPI_ERR_FILE ) { return MUK_ERR_FILE; } 77 | else if (error == MPI_ERR_INFO_KEY ) { return MUK_ERR_INFO_KEY; } 78 | else if (error == MPI_ERR_INFO_NOKEY ) { return MUK_ERR_INFO_NOKEY; } 79 | else if (error == MPI_ERR_INFO_VALUE ) { return MUK_ERR_INFO_VALUE; } 80 | else if (error == MPI_ERR_INFO ) { return MUK_ERR_INFO; } 81 | else if (error == MPI_ERR_IO ) { return MUK_ERR_IO; } 82 | else if (error == MPI_ERR_KEYVAL ) { return MUK_ERR_KEYVAL; } 83 | else if (error == MPI_ERR_LOCKTYPE ) { return MUK_ERR_LOCKTYPE; } 84 | else if (error == MPI_ERR_NAME ) { return MUK_ERR_NAME; } 85 | else if (error == MPI_ERR_NO_MEM ) { return MUK_ERR_NO_MEM; } 86 | else if (error == MPI_ERR_NOT_SAME ) { return MUK_ERR_NOT_SAME; } 87 | else if (error == MPI_ERR_NO_SPACE ) { return MUK_ERR_NO_SPACE; } 88 | else if (error == MPI_ERR_NO_SUCH_FILE ) { return MUK_ERR_NO_SUCH_FILE; } 89 | else if (error == MPI_ERR_PORT ) { return MUK_ERR_PORT; } 90 | #if (MPI_VERSION >= 4) 91 | else if (error == MPI_ERR_PROC_ABORTED ) { return MUK_ERR_PROC_ABORTED; } 92 | #endif 93 | else if (error == MPI_ERR_QUOTA ) { return MUK_ERR_QUOTA; } 94 | else if (error == MPI_ERR_READ_ONLY ) { return MUK_ERR_READ_ONLY; } 95 | else if (error == MPI_ERR_RMA_ATTACH ) { return MUK_ERR_RMA_ATTACH; } 96 | else if (error == MPI_ERR_RMA_CONFLICT ) { return MUK_ERR_RMA_CONFLICT; } 97 | else if (error == MPI_ERR_RMA_RANGE ) { return MUK_ERR_RMA_RANGE; } 98 | else if (error == MPI_ERR_RMA_SHARED ) { return MUK_ERR_RMA_SHARED; } 99 | else if (error == MPI_ERR_RMA_SYNC ) { return MUK_ERR_RMA_SYNC; } 100 | else if (error == MPI_ERR_RMA_FLAVOR ) { return MUK_ERR_RMA_FLAVOR; } 101 | else if (error == MPI_ERR_SERVICE ) { return MUK_ERR_SERVICE; } 102 | #if (MPI_VERSION >= 4) 103 | else if (error == MPI_ERR_SESSION ) { return MUK_ERR_SESSION; } 104 | #endif 105 | else if (error == MPI_ERR_SIZE ) { return MUK_ERR_SIZE; } 106 | else if (error == MPI_ERR_SPAWN ) { return MUK_ERR_SPAWN; } 107 | else if (error == MPI_ERR_UNSUPPORTED_DATAREP ) { return MUK_ERR_UNSUPPORTED_DATAREP; } 108 | else if (error == MPI_ERR_UNSUPPORTED_OPERATION ) { return MUK_ERR_UNSUPPORTED_OPERATION; } 109 | #if (MPI_VERSION >= 4) 110 | else if (error == MPI_ERR_VALUE_TOO_LARGE ) { return MUK_ERR_VALUE_TOO_LARGE; } 111 | #endif 112 | else if (error == MPI_ERR_WIN ) { return MUK_ERR_WIN; } 113 | else if (error == MPI_T_ERR_CANNOT_INIT ) { return MUK_T_ERR_CANNOT_INIT; } 114 | //else if (error == MPI_T_ERR_NOT_ACCESSIBLE ) { return MUK_T_ERR_NOT_ACCESSIBLE; } 115 | else if (error == MPI_T_ERR_NOT_INITIALIZED ) { return MUK_T_ERR_NOT_INITIALIZED; } 116 | #if (MPI_VERSION >= 4) 117 | else if (error == MPI_T_ERR_NOT_SUPPORTED ) { return MUK_T_ERR_NOT_SUPPORTED; } 118 | #endif 119 | else if (error == MPI_T_ERR_MEMORY ) { return MUK_T_ERR_MEMORY; } 120 | else if (error == MPI_T_ERR_INVALID ) { return MUK_T_ERR_INVALID; } 121 | else if (error == MPI_T_ERR_INVALID_INDEX ) { return MUK_T_ERR_INVALID_INDEX; } 122 | else if (error == MPI_T_ERR_INVALID_ITEM ) { return MUK_T_ERR_INVALID_ITEM; } 123 | else if (error == MPI_T_ERR_INVALID_SESSION ) { return MUK_T_ERR_INVALID_SESSION; } 124 | else if (error == MPI_T_ERR_INVALID_HANDLE ) { return MUK_T_ERR_INVALID_HANDLE; } 125 | else if (error == MPI_T_ERR_INVALID_NAME ) { return MUK_T_ERR_INVALID_NAME; } 126 | else if (error == MPI_T_ERR_OUT_OF_HANDLES ) { return MUK_T_ERR_OUT_OF_HANDLES; } 127 | else if (error == MPI_T_ERR_OUT_OF_SESSIONS ) { return MUK_T_ERR_OUT_OF_SESSIONS; } 128 | else if (error == MPI_T_ERR_CVAR_SET_NOT_NOW ) { return MUK_T_ERR_CVAR_SET_NOT_NOW; } 129 | else if (error == MPI_T_ERR_CVAR_SET_NEVER ) { return MUK_T_ERR_CVAR_SET_NEVER; } 130 | else if (error == MPI_T_ERR_PVAR_NO_WRITE ) { return MUK_T_ERR_PVAR_NO_WRITE; } 131 | else if (error == MPI_T_ERR_PVAR_NO_STARTSTOP ) { return MUK_T_ERR_PVAR_NO_STARTSTOP; } 132 | else if (error == MPI_T_ERR_PVAR_NO_ATOMIC ) { return MUK_T_ERR_PVAR_NO_ATOMIC; } 133 | else if (error == MPI_ERR_LASTCODE ) { return MUK_ERR_LASTCODE; } 134 | else { 135 | #if 1 136 | int me; 137 | IMPL_Comm_rank(MPI_COMM_WORLD,&me); 138 | int len; 139 | char name[MPI_MAX_ERROR_STRING] = {0}; 140 | // using error_c here causes MPICH to include more information in the error string 141 | IMPL_Error_string(error_c, name, &len); 142 | printf("%d: Unknown error returned from the C library: code=%d=%x, class=%d=%x, name=%s\n", 143 | me, error_c, error_c, error, error, name); 144 | #endif 145 | return MUK_ERR_UNKNOWN; 146 | } 147 | } 148 | 149 | int ERROR_CODE_MUK_TO_IMPL(int error_muk) 150 | { 151 | if (error_muk == MUK_SUCCESS ) { return MPI_SUCCESS; } 152 | else if (error_muk == MUK_ERR_BUFFER ) { return MPI_ERR_BUFFER; } 153 | else if (error_muk == MUK_ERR_COUNT ) { return MPI_ERR_COUNT; } 154 | else if (error_muk == MUK_ERR_TYPE ) { return MPI_ERR_TYPE; } 155 | else if (error_muk == MUK_ERR_TAG ) { return MPI_ERR_TAG; } 156 | else if (error_muk == MUK_ERR_COMM ) { return MPI_ERR_COMM; } 157 | else if (error_muk == MUK_ERR_RANK ) { return MPI_ERR_RANK; } 158 | else if (error_muk == MUK_ERR_REQUEST ) { return MPI_ERR_REQUEST; } 159 | else if (error_muk == MUK_ERR_ROOT ) { return MPI_ERR_ROOT; } 160 | else if (error_muk == MUK_ERR_GROUP ) { return MPI_ERR_GROUP; } 161 | else if (error_muk == MUK_ERR_OP ) { return MPI_ERR_OP; } 162 | else if (error_muk == MUK_ERR_TOPOLOGY ) { return MPI_ERR_TOPOLOGY; } 163 | else if (error_muk == MUK_ERR_DIMS ) { return MPI_ERR_DIMS; } 164 | else if (error_muk == MUK_ERR_ARG ) { return MPI_ERR_ARG; } 165 | else if (error_muk == MUK_ERR_UNKNOWN ) { return MPI_ERR_UNKNOWN; } 166 | else if (error_muk == MUK_ERR_TRUNCATE ) { return MPI_ERR_TRUNCATE; } 167 | else if (error_muk == MUK_ERR_OTHER ) { return MPI_ERR_OTHER; } 168 | else if (error_muk == MUK_ERR_INTERN ) { return MPI_ERR_INTERN; } 169 | else if (error_muk == MUK_ERR_PENDING ) { return MPI_ERR_PENDING; } 170 | else if (error_muk == MUK_ERR_IN_STATUS ) { return MPI_ERR_IN_STATUS; } 171 | else if (error_muk == MUK_ERR_ACCESS ) { return MPI_ERR_ACCESS; } 172 | else if (error_muk == MUK_ERR_AMODE ) { return MPI_ERR_AMODE; } 173 | else if (error_muk == MUK_ERR_ASSERT ) { return MPI_ERR_ASSERT; } 174 | else if (error_muk == MUK_ERR_BAD_FILE ) { return MPI_ERR_BAD_FILE; } 175 | else if (error_muk == MUK_ERR_BASE ) { return MPI_ERR_BASE; } 176 | else if (error_muk == MUK_ERR_CONVERSION ) { return MPI_ERR_CONVERSION; } 177 | else if (error_muk == MUK_ERR_DISP ) { return MPI_ERR_DISP; } 178 | else if (error_muk == MUK_ERR_DUP_DATAREP ) { return MPI_ERR_DUP_DATAREP; } 179 | else if (error_muk == MUK_ERR_FILE_EXISTS ) { return MPI_ERR_FILE_EXISTS; } 180 | else if (error_muk == MUK_ERR_FILE_IN_USE ) { return MPI_ERR_FILE_IN_USE; } 181 | else if (error_muk == MUK_ERR_FILE ) { return MPI_ERR_FILE; } 182 | else if (error_muk == MUK_ERR_INFO_KEY ) { return MPI_ERR_INFO_KEY; } 183 | else if (error_muk == MUK_ERR_INFO_NOKEY ) { return MPI_ERR_INFO_NOKEY; } 184 | else if (error_muk == MUK_ERR_INFO_VALUE ) { return MPI_ERR_INFO_VALUE; } 185 | else if (error_muk == MUK_ERR_INFO ) { return MPI_ERR_INFO; } 186 | else if (error_muk == MUK_ERR_IO ) { return MPI_ERR_IO; } 187 | else if (error_muk == MUK_ERR_KEYVAL ) { return MPI_ERR_KEYVAL; } 188 | else if (error_muk == MUK_ERR_LOCKTYPE ) { return MPI_ERR_LOCKTYPE; } 189 | else if (error_muk == MUK_ERR_NAME ) { return MPI_ERR_NAME; } 190 | else if (error_muk == MUK_ERR_NO_MEM ) { return MPI_ERR_NO_MEM; } 191 | else if (error_muk == MUK_ERR_NOT_SAME ) { return MPI_ERR_NOT_SAME; } 192 | else if (error_muk == MUK_ERR_NO_SPACE ) { return MPI_ERR_NO_SPACE; } 193 | else if (error_muk == MUK_ERR_NO_SUCH_FILE ) { return MPI_ERR_NO_SUCH_FILE; } 194 | else if (error_muk == MUK_ERR_PORT ) { return MPI_ERR_PORT; } 195 | #if (MPI_VERSION >= 4) 196 | else if (error_muk == MUK_ERR_PROC_ABORTED ) { return MPI_ERR_PROC_ABORTED; } 197 | #endif 198 | else if (error_muk == MUK_ERR_QUOTA ) { return MPI_ERR_QUOTA; } 199 | else if (error_muk == MUK_ERR_READ_ONLY ) { return MPI_ERR_READ_ONLY; } 200 | else if (error_muk == MUK_ERR_RMA_ATTACH ) { return MPI_ERR_RMA_ATTACH; } 201 | else if (error_muk == MUK_ERR_RMA_CONFLICT ) { return MPI_ERR_RMA_CONFLICT; } 202 | else if (error_muk == MUK_ERR_RMA_RANGE ) { return MPI_ERR_RMA_RANGE; } 203 | else if (error_muk == MUK_ERR_RMA_SHARED ) { return MPI_ERR_RMA_SHARED; } 204 | else if (error_muk == MUK_ERR_RMA_SYNC ) { return MPI_ERR_RMA_SYNC; } 205 | else if (error_muk == MUK_ERR_RMA_FLAVOR ) { return MPI_ERR_RMA_FLAVOR; } 206 | else if (error_muk == MUK_ERR_SERVICE ) { return MPI_ERR_SERVICE; } 207 | #if (MPI_VERSION >= 4) 208 | else if (error_muk == MUK_ERR_SESSION ) { return MPI_ERR_SESSION; } 209 | #endif 210 | else if (error_muk == MUK_ERR_SIZE ) { return MPI_ERR_SIZE; } 211 | else if (error_muk == MUK_ERR_SPAWN ) { return MPI_ERR_SPAWN; } 212 | else if (error_muk == MUK_ERR_UNSUPPORTED_DATAREP ) { return MPI_ERR_UNSUPPORTED_DATAREP; } 213 | else if (error_muk == MUK_ERR_UNSUPPORTED_OPERATION ) { return MPI_ERR_UNSUPPORTED_OPERATION; } 214 | #if (MPI_VERSION >= 4) 215 | else if (error_muk == MUK_ERR_VALUE_TOO_LARGE ) { return MPI_ERR_VALUE_TOO_LARGE; } 216 | #endif 217 | else if (error_muk == MUK_ERR_WIN ) { return MPI_ERR_WIN; } 218 | else if (error_muk == MUK_T_ERR_CANNOT_INIT ) { return MPI_T_ERR_CANNOT_INIT; } 219 | //else if (error_muk == MUK_T_ERR_NOT_ACCESSIBLE ) { return MPI_T_ERR_NOT_ACCESSIBLE; } 220 | else if (error_muk == MUK_T_ERR_NOT_INITIALIZED ) { return MPI_T_ERR_NOT_INITIALIZED; } 221 | #if (MPI_VERSION >= 4) 222 | else if (error_muk == MUK_T_ERR_NOT_SUPPORTED ) { return MPI_T_ERR_NOT_SUPPORTED; } 223 | #endif 224 | else if (error_muk == MUK_T_ERR_MEMORY ) { return MPI_T_ERR_MEMORY; } 225 | else if (error_muk == MUK_T_ERR_INVALID ) { return MPI_T_ERR_INVALID; } 226 | else if (error_muk == MUK_T_ERR_INVALID_INDEX ) { return MPI_T_ERR_INVALID_INDEX; } 227 | else if (error_muk == MUK_T_ERR_INVALID_ITEM ) { return MPI_T_ERR_INVALID_ITEM; } 228 | else if (error_muk == MUK_T_ERR_INVALID_SESSION ) { return MPI_T_ERR_INVALID_SESSION; } 229 | else if (error_muk == MUK_T_ERR_INVALID_HANDLE ) { return MPI_T_ERR_INVALID_HANDLE; } 230 | else if (error_muk == MUK_T_ERR_INVALID_NAME ) { return MPI_T_ERR_INVALID_NAME; } 231 | else if (error_muk == MUK_T_ERR_OUT_OF_HANDLES ) { return MPI_T_ERR_OUT_OF_HANDLES; } 232 | else if (error_muk == MUK_T_ERR_OUT_OF_SESSIONS ) { return MPI_T_ERR_OUT_OF_SESSIONS; } 233 | else if (error_muk == MUK_T_ERR_CVAR_SET_NOT_NOW ) { return MPI_T_ERR_CVAR_SET_NOT_NOW; } 234 | else if (error_muk == MUK_T_ERR_CVAR_SET_NEVER ) { return MPI_T_ERR_CVAR_SET_NEVER; } 235 | else if (error_muk == MUK_T_ERR_PVAR_NO_WRITE ) { return MPI_T_ERR_PVAR_NO_WRITE; } 236 | else if (error_muk == MUK_T_ERR_PVAR_NO_STARTSTOP ) { return MPI_T_ERR_PVAR_NO_STARTSTOP; } 237 | else if (error_muk == MUK_T_ERR_PVAR_NO_ATOMIC ) { return MPI_T_ERR_PVAR_NO_ATOMIC; } 238 | else if (error_muk == MUK_ERR_LASTCODE ) { return MPI_ERR_LASTCODE; } 239 | else { 240 | //fprintf(stderr, "Unknown error code returned from the F library: %d\n", error_muk); 241 | return MPI_ERR_UNKNOWN; 242 | } 243 | } 244 | -------------------------------------------------------------------------------- /impl-constant-conversions.h: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | #ifndef MUK_IMPL_CONSTANT_CONVERSIONS_H 4 | #define MUK_IMPL_CONSTANT_CONVERSIONS_H 5 | 6 | #define MAYBE_UNUSED __attribute__((unused)) 7 | 8 | // success is the common case, so static inline it. 9 | // jump to the error conversion only when necessary. 10 | // the ERROR_CODE functions need to be symbols because 11 | // they are used in the outer layer for the functions 12 | // that are used directly (because they do not have 13 | // handle arguments). 14 | int ERROR_CODE_IMPL_TO_MUK(int error_c); 15 | static inline int RETURN_CODE_IMPL_TO_MUK(int error_c) 16 | { 17 | if (error_c == 0) return 0; 18 | return ERROR_CODE_IMPL_TO_MUK(error_c); 19 | } 20 | 21 | int ERROR_CODE_MUK_TO_IMPL(int error_muk); 22 | static inline int RETURN_CODE_MUK_TO_IMPL(int error_muk) 23 | { 24 | if (error_muk == 0) return 0; 25 | return ERROR_CODE_MUK_TO_IMPL(error_muk); 26 | } 27 | 28 | MAYBE_UNUSED 29 | static int COMBINER_CODE_IMPL_TO_MUK(int in) 30 | { 31 | if (in == MPI_COMBINER_CONTIGUOUS) { return MUK_COMBINER_CONTIGUOUS; } 32 | else if (in == MPI_COMBINER_DARRAY) { return MUK_COMBINER_DARRAY; } 33 | else if (in == MPI_COMBINER_DUP) { return MUK_COMBINER_DUP; } 34 | else if (in == MPI_COMBINER_F90_COMPLEX) { return MUK_COMBINER_F90_COMPLEX; } 35 | else if (in == MPI_COMBINER_F90_INTEGER) { return MUK_COMBINER_F90_INTEGER; } 36 | else if (in == MPI_COMBINER_F90_REAL) { return MUK_COMBINER_F90_REAL; } 37 | else if (in == MPI_COMBINER_HINDEXED) { return MUK_COMBINER_HINDEXED; } 38 | else if (in == MPI_COMBINER_HVECTOR) { return MUK_COMBINER_HVECTOR; } 39 | else if (in == MPI_COMBINER_INDEXED_BLOCK) { return MUK_COMBINER_INDEXED_BLOCK; } 40 | else if (in == MPI_COMBINER_HINDEXED_BLOCK) { return MUK_COMBINER_HINDEXED_BLOCK; } 41 | else if (in == MPI_COMBINER_INDEXED) { return MUK_COMBINER_INDEXED; } 42 | else if (in == MPI_COMBINER_NAMED) { return MUK_COMBINER_NAMED; } 43 | else if (in == MPI_COMBINER_RESIZED) { return MUK_COMBINER_RESIZED; } 44 | else if (in == MPI_COMBINER_STRUCT) { return MUK_COMBINER_STRUCT; } 45 | else if (in == MPI_COMBINER_SUBARRAY) { return MUK_COMBINER_SUBARRAY; } 46 | else if (in == MPI_COMBINER_VECTOR) { return MUK_COMBINER_VECTOR; } 47 | else { return MUK_UNDEFINED; } 48 | } 49 | 50 | // MUK, MPICH and OMPI agree that ANY_TAG = -1 so no conversion is required. 51 | #define TAG_MUK_TO_IMPL(tag) (tag) 52 | #define TAG_IMPL_TO_MUK(tag) (tag) 53 | 54 | MAYBE_UNUSED 55 | static inline int RANK_MUK_TO_IMPL(int rank_muk) 56 | { 57 | // predefined constants are always negative 58 | if (rank_muk >= 0) { 59 | return rank_muk; 60 | } 61 | else if (rank_muk == MUK_ANY_SOURCE) { 62 | return MPI_ANY_SOURCE; 63 | } 64 | else if (rank_muk == MUK_PROC_NULL) { 65 | return MPI_PROC_NULL; 66 | } 67 | // This gets used by MPICH at least, as an initialization value. 68 | else if (rank_muk == MUK_UNDEFINED) { 69 | return MPI_UNDEFINED; 70 | } 71 | #ifdef MPICH 72 | // https://github.com/pmodels/mpich/issues/6421 73 | else if (rank_muk == -269488145) { 74 | return MPI_UNDEFINED; 75 | } 76 | #endif 77 | // this one only applies to intercomms 78 | else if (rank_muk == MUK_ROOT) { 79 | return MPI_ROOT; 80 | } 81 | else { 82 | // This will generate false-positive warnings with MPICH 83 | // of the following form, so we handle this case above. 84 | // RANK_MUK_TO_IMPL rank=-269488145=efefefef 85 | #if 1 86 | printf("RANK_MUK_TO_IMPL rank=%d=%x\n", rank_muk, rank_muk); 87 | #endif 88 | return rank_muk; 89 | } 90 | } 91 | 92 | MAYBE_UNUSED 93 | static inline int RANK_IMPL_TO_MUK(int rank_impl) 94 | { 95 | // predefined constants are always negative 96 | if (rank_impl >= 0) { 97 | return rank_impl; 98 | } 99 | else if (rank_impl == MPI_ANY_SOURCE) { 100 | return MUK_ANY_SOURCE; 101 | } 102 | else if (rank_impl == MPI_PROC_NULL) { 103 | return MUK_PROC_NULL; 104 | } 105 | // This gets used by MPICH at least, as an initialization value. 106 | else if (rank_impl == MPI_UNDEFINED) { 107 | return MUK_UNDEFINED; 108 | } 109 | #ifdef MPICH 110 | // https://github.com/pmodels/mpich/issues/6421 111 | else if (rank_impl == -269488145) { 112 | return MPI_UNDEFINED; 113 | } 114 | #endif 115 | #if 0 116 | // this one only applies to intercomms 117 | else if (rank_impl == MPI_ROOT) { 118 | return MUK_ROOT; 119 | } 120 | #endif 121 | else { 122 | #if DEBUG 123 | printf("RANK_IMPL_TO_MUK rank=%d=%x\n", rank_impl, rank_impl); 124 | #endif 125 | return rank_impl; 126 | } 127 | } 128 | 129 | // mode constant conversion - this needs to handle multiple modes OR-ed together 130 | MAYBE_UNUSED 131 | static int IO_MODE_MUK_TO_IMPL(int mode_muk) 132 | { 133 | int mode_impl = 0; 134 | if (mode_muk & MUK_MODE_APPEND) { 135 | mode_impl |= MPI_MODE_APPEND; 136 | //printf("mode_impl |= MPI_MODE_APPEND\n"); 137 | } 138 | if (mode_muk & MUK_MODE_CREATE) { 139 | mode_impl |= MPI_MODE_CREATE; 140 | //printf("mode_impl |= MPI_MODE_CREATE\n"); 141 | } 142 | if (mode_muk & MUK_MODE_DELETE_ON_CLOSE) { 143 | mode_impl |= MPI_MODE_DELETE_ON_CLOSE; 144 | //printf("mode_impl |= MPI_MODE_DELETE_ON_CLOSE\n"); 145 | } 146 | if (mode_muk & MUK_MODE_EXCL) { 147 | mode_impl |= MPI_MODE_EXCL; 148 | //printf("mode_impl |= MPI_MODE_EXCL\n"); 149 | } 150 | if (mode_muk & MUK_MODE_RDONLY) { 151 | mode_impl |= MPI_MODE_RDONLY; 152 | //printf("mode_impl |= MPI_MODE_RDONLY\n"); 153 | } 154 | if (mode_muk & MUK_MODE_RDWR) { 155 | mode_impl |= MPI_MODE_RDWR; 156 | //printf("mode_impl |= MPI_MODE_RDWR\n"); 157 | } 158 | if (mode_muk & MUK_MODE_SEQUENTIAL) { 159 | mode_impl |= MPI_MODE_SEQUENTIAL; 160 | //printf("mode_impl |= MPI_MODE_SEQUENTIAL\n"); 161 | } 162 | if (mode_muk & MUK_MODE_UNIQUE_OPEN) { 163 | mode_impl |= MPI_MODE_UNIQUE_OPEN; 164 | //printf("mode_impl |= MPI_MODE_UNIQUE_OPEN\n"); 165 | } 166 | if (mode_muk & MUK_MODE_WRONLY) { 167 | mode_impl |= MPI_MODE_WRONLY; 168 | //printf("mode_impl |= MPI_MODE_WRONLY\n"); 169 | } 170 | return mode_impl; 171 | } 172 | 173 | MAYBE_UNUSED 174 | static int IO_MODE_IMPL_TO_MUK(int mode_impl) 175 | { 176 | int mode_muk = 0; 177 | if (mode_impl & MUK_MODE_APPEND) { mode_muk |= MPI_MODE_APPEND; } 178 | if (mode_impl & MUK_MODE_CREATE) { mode_muk |= MPI_MODE_CREATE; } 179 | if (mode_impl & MUK_MODE_DELETE_ON_CLOSE) { mode_muk |= MPI_MODE_DELETE_ON_CLOSE; } 180 | if (mode_impl & MUK_MODE_EXCL) { mode_muk |= MPI_MODE_EXCL; } 181 | if (mode_impl & MUK_MODE_RDONLY) { mode_muk |= MPI_MODE_RDONLY; } 182 | if (mode_impl & MUK_MODE_RDWR) { mode_muk |= MPI_MODE_RDWR; } 183 | if (mode_impl & MUK_MODE_SEQUENTIAL) { mode_muk |= MPI_MODE_SEQUENTIAL; } 184 | if (mode_impl & MUK_MODE_UNIQUE_OPEN) { mode_muk |= MPI_MODE_UNIQUE_OPEN; } 185 | if (mode_impl & MUK_MODE_WRONLY) { mode_muk |= MPI_MODE_WRONLY; } 186 | return mode_muk; 187 | } 188 | 189 | // mode constant conversion - this needs to handle multiple modes OR-ed together 190 | MAYBE_UNUSED 191 | static int RMA_MODE_MUK_TO_IMPL(int mode_muk) 192 | { 193 | int mode_impl = 0; 194 | if (mode_muk & MUK_MODE_NOCHECK) { mode_impl |= MPI_MODE_NOCHECK; } 195 | if (mode_muk & MUK_MODE_NOPRECEDE) { mode_impl |= MPI_MODE_NOPRECEDE; } 196 | if (mode_muk & MUK_MODE_NOPUT) { mode_impl |= MPI_MODE_NOPUT; } 197 | if (mode_muk & MUK_MODE_NOSTORE) { mode_impl |= MPI_MODE_NOSTORE; } 198 | if (mode_muk & MUK_MODE_NOSUCCEED) { mode_impl |= MPI_MODE_NOSUCCEED; } 199 | return mode_impl; 200 | } 201 | 202 | // predefined attribute conversion 203 | MAYBE_UNUSED 204 | static int KEY_MUK_TO_IMPL(int key_muk) 205 | { 206 | if (key_muk == MUK_TAG_UB) { return MPI_TAG_UB; } 207 | else if (key_muk == MUK_IO) { return MPI_IO; } 208 | else if (key_muk == MUK_HOST) { return MPI_HOST; } 209 | else if (key_muk == MUK_WTIME_IS_GLOBAL) { return MPI_WTIME_IS_GLOBAL; } 210 | else if (key_muk == MUK_APPNUM) { return MPI_APPNUM; } 211 | else if (key_muk == MUK_LASTUSEDCODE) { return MPI_LASTUSEDCODE; } 212 | else if (key_muk == MUK_UNIVERSE_SIZE) { return MPI_UNIVERSE_SIZE; } 213 | else if (key_muk == MUK_WIN_BASE) { return MPI_WIN_BASE; } 214 | else if (key_muk == MUK_WIN_DISP_UNIT) { return MPI_WIN_DISP_UNIT; } 215 | else if (key_muk == MUK_WIN_SIZE) { return MPI_WIN_SIZE; } 216 | else if (key_muk == MUK_WIN_CREATE_FLAVOR) { return MPI_WIN_CREATE_FLAVOR; } 217 | else if (key_muk == MUK_WIN_MODEL) { return MPI_WIN_MODEL; } 218 | else { return key_muk; } 219 | } 220 | 221 | MAYBE_UNUSED 222 | static const int * WEIGHTS_MUK_TO_IMPL(const int * weights_muk, const int * mpich_unweighted, const int * mpich_weights_empty) 223 | { 224 | if ((intptr_t)weights_muk == (intptr_t)MUK_UNWEIGHTED) { 225 | #if OPEN_MPI 226 | return MPI_UNWEIGHTED; 227 | (void)mpich_unweighted; 228 | #else 229 | return mpich_unweighted; 230 | #endif 231 | } 232 | else if ((intptr_t)weights_muk == (intptr_t)MUK_WEIGHTS_EMPTY) { 233 | #if OPEN_MPI 234 | return MPI_WEIGHTS_EMPTY; 235 | (void)mpich_weights_empty; 236 | #else 237 | return mpich_weights_empty; 238 | #endif 239 | } 240 | else { 241 | return weights_muk; 242 | } 243 | } 244 | 245 | MAYBE_UNUSED 246 | static const int * WEIGHTS_IMPL_TO_MUK(const int * weights_impl) 247 | { 248 | if ((intptr_t)weights_impl == (intptr_t)MPI_UNWEIGHTED) { 249 | return MUK_UNWEIGHTED; 250 | } 251 | else if ((intptr_t)weights_impl == (intptr_t)MPI_WEIGHTS_EMPTY) { 252 | return MUK_WEIGHTS_EMPTY; 253 | } 254 | else { 255 | return weights_impl; 256 | } 257 | } 258 | #endif 259 | -------------------------------------------------------------------------------- /impl-grequest-functions.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include 12 | 13 | #include "muk-predefined.h" 14 | 15 | #define MUK_EXTERN extern 16 | #include "impl-fpointers.h" 17 | //#include "impl-constant-conversions.h" 18 | #include "impl-handle-conversions.h" 19 | //#include "impl-keyval-map.h" 20 | #include "impl-status.h" 21 | 22 | typedef struct 23 | { 24 | void * user_state; 25 | WRAP_Grequest_query_function * query_fn; 26 | WRAP_Grequest_free_function * free_fn; 27 | WRAP_Grequest_cancel_function * cancel_fn; 28 | } 29 | grequest_wrapped_state; 30 | 31 | int query_trampoline(void * extra_state, MPI_Status * impl_status) 32 | { 33 | grequest_wrapped_state * wrapped_state = (grequest_wrapped_state*)extra_state; 34 | void * user_state = wrapped_state->user_state; 35 | WRAP_Grequest_query_function* query_fn = wrapped_state->query_fn; 36 | WRAP_Status wrap_status; 37 | MPI_Status_to_WRAP_Status(impl_status, &wrap_status); 38 | int rc = (*query_fn)(user_state, &wrap_status); 39 | WRAP_Status_to_MPI_Status(&wrap_status, impl_status); 40 | return RETURN_CODE_MUK_TO_IMPL(rc); 41 | } 42 | 43 | int free_trampoline(void *extra_state) 44 | { 45 | grequest_wrapped_state * wrapped_state = (grequest_wrapped_state*)extra_state; 46 | void * user_state = wrapped_state->user_state; 47 | WRAP_Grequest_free_function* free_fn = wrapped_state->free_fn; 48 | int rc = (*free_fn)(user_state); 49 | return RETURN_CODE_MUK_TO_IMPL(rc); 50 | } 51 | 52 | int cancel_trampoline(void *extra_state, int complete) 53 | { 54 | grequest_wrapped_state * wrapped_state = (grequest_wrapped_state*)extra_state; 55 | void * user_state = wrapped_state->user_state; 56 | WRAP_Grequest_cancel_function* cancel_fn = wrapped_state->cancel_fn; 57 | int rc = (*cancel_fn)(user_state, complete); 58 | return RETURN_CODE_MUK_TO_IMPL(rc); 59 | } 60 | 61 | 62 | // WRAP->IMPL functions 63 | 64 | int WRAP_Grequest_complete(WRAP_Request request) 65 | { 66 | MPI_Request impl_request = CONVERT_MPI_Request(request); 67 | int rc = IMPL_Grequest_complete(impl_request); 68 | return RETURN_CODE_IMPL_TO_MUK(rc); 69 | } 70 | 71 | int WRAP_Grequest_start(WRAP_Grequest_query_function *query_fn, WRAP_Grequest_free_function *free_fn, WRAP_Grequest_cancel_function *cancel_fn, void *user_state, WRAP_Request *request) 72 | { 73 | int rc; 74 | MPI_Request impl_request = CONVERT_MPI_Request(*request); 75 | 76 | grequest_wrapped_state * wrapped_state = calloc(1,sizeof(grequest_wrapped_state)); 77 | if (wrapped_state == NULL) { 78 | rc = MPI_ERR_INTERN; 79 | goto end; 80 | } 81 | 82 | wrapped_state->user_state = user_state; 83 | wrapped_state->query_fn = query_fn; 84 | wrapped_state->free_fn = free_fn; 85 | wrapped_state->cancel_fn = cancel_fn; 86 | 87 | MPI_Grequest_query_function * impl_query_fn = query_trampoline; 88 | MPI_Grequest_free_function * impl_free_fn = free_trampoline; 89 | MPI_Grequest_cancel_function * impl_cancel_fn = cancel_trampoline; 90 | 91 | rc = IMPL_Grequest_start(impl_query_fn, impl_free_fn, impl_cancel_fn, wrapped_state, &impl_request); 92 | *request = OUTPUT_MPI_Request(impl_request); 93 | end: 94 | return RETURN_CODE_IMPL_TO_MUK(rc); 95 | } 96 | -------------------------------------------------------------------------------- /impl-keyval-map-commattr.h: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | int add_comm_keyval_callbacks(int keyval, 4 | WRAP_Comm_copy_attr_function * comm_copy_attr_fn, 5 | WRAP_Comm_delete_attr_function * comm_delete_attr_fn) 6 | { 7 | const std::lock_guard lock(keyval_comm_attr_cb_mutex); 8 | #if DEBUG 9 | printf("%s: insert_or_assign(keyval=%d, comm_copy_attr_fn=%p, comm_delete_attr_fn=%p)\n", 10 | __func__, keyval, comm_copy_attr_fn, comm_delete_attr_fn); 11 | #endif 12 | // insert_or_assign (C++17) inserts an element or assigns to the current element if the key already exists 13 | auto [it,rc] = keyval_comm_attr_cb_map.insert_or_assign(keyval, 14 | std::make_pair(comm_copy_attr_fn, 15 | comm_delete_attr_fn)); 16 | return 1; // SUCCESS int{rc}; 17 | (void)it; 18 | (void)rc; 19 | } 20 | 21 | int find_comm_keyval_callbacks(int keyval, 22 | WRAP_Comm_copy_attr_function ** comm_copy_attr_fn, 23 | WRAP_Comm_delete_attr_function ** comm_delete_attr_fn) 24 | { 25 | const std::lock_guard lock(keyval_comm_attr_cb_mutex); 26 | try { 27 | auto [copy_fn,delete_fn] = keyval_comm_attr_cb_map.at(keyval); 28 | #if DEBUG 29 | printf("%s: lookup(keyval=%d) -> [comm_copy_attr_fn=%p, comm_delete_attr_fn=%p]\n", 30 | __func__, keyval, copy_fn, delete_fn); 31 | #endif 32 | if (comm_copy_attr_fn != NULL) { 33 | *comm_copy_attr_fn = copy_fn; 34 | } 35 | if (comm_delete_attr_fn != NULL) { 36 | *comm_delete_attr_fn = delete_fn; 37 | } 38 | return 1; 39 | } 40 | catch (...) { 41 | printf("%s: lookup(keyval=%d) failed\n", __func__, keyval); 42 | return 0; 43 | } 44 | } 45 | 46 | int remove_comm_keyval_callbacks(int keyval) 47 | { 48 | const std::lock_guard lock(keyval_comm_attr_cb_mutex); 49 | // returns the number of elements removed, so 0=failure and 1=success 50 | return keyval_comm_attr_cb_map.erase(keyval); 51 | } 52 | -------------------------------------------------------------------------------- /impl-keyval-map-commerrh.h: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | int add_comm_errhandler_callback(MPI_Errhandler errhandler, WRAP_Comm_errhandler_function * user_fn) 4 | { 5 | const std::lock_guard lock(errhandler_comm_cb_mutex); 6 | #if DEBUG 7 | printf("%s: insert_or_assign(errhandler=%lx, user_fn=%p)\n", 8 | __func__, (intptr_t)errhandler, user_fn); 9 | #endif 10 | // insert_or_assign (C++17) inserts an element or assigns to the current element if the key already exists 11 | auto [it,rc] = errhandler_comm_cb_map.insert_or_assign(errhandler, user_fn); 12 | return 1; // SUCCESS int{rc}; 13 | (void)it; 14 | (void)rc; 15 | } 16 | 17 | int find_comm_errhandler_callback(MPI_Errhandler errhandler, WRAP_Comm_errhandler_function ** user_fn) 18 | { 19 | const std::lock_guard lock(errhandler_comm_cb_mutex); 20 | try { 21 | auto fn = errhandler_comm_cb_map.at(errhandler); 22 | #if DEBUG 23 | printf("%s: lookup(errhandler=%lx) -> user_fn=%p\n", 24 | __func__, (intptr_t)errhandler, user_fn); 25 | #endif 26 | if (fn != NULL) { 27 | *user_fn = fn; 28 | } 29 | return 1; 30 | } 31 | catch (...) { 32 | printf("%s: lookup(errhandler=%lx) failed\n", __func__, (intptr_t)errhandler); 33 | return 0; 34 | } 35 | } 36 | 37 | int remove_comm_errhandler_callback(MPI_Errhandler errhandler) 38 | { 39 | const std::lock_guard lock(errhandler_comm_cb_mutex); 40 | // returns the number of elements removed, so 0=failure and 1=success 41 | return errhandler_comm_cb_map.erase(errhandler); 42 | } 43 | -------------------------------------------------------------------------------- /impl-keyval-map-fileerrh.h: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | int add_file_errhandler_callback(MPI_Errhandler errhandler, WRAP_File_errhandler_function * user_fn) 4 | { 5 | const std::lock_guard lock(errhandler_file_cb_mutex); 6 | #if DEBUG 7 | printf("%s: insert_or_assign(errhandler=%lx, user_fn=%p)\n", 8 | __func__, (intptr_t)errhandler, user_fn); 9 | #endif 10 | // insert_or_assign (C++17) inserts an element or assigns to the current element if the key already exists 11 | auto [it,rc] = errhandler_file_cb_map.insert_or_assign(errhandler, user_fn); 12 | return 1; // SUCCESS int{rc}; 13 | (void)it; 14 | (void)rc; 15 | } 16 | 17 | int find_file_errhandler_callback(MPI_Errhandler errhandler, WRAP_File_errhandler_function ** user_fn) 18 | { 19 | const std::lock_guard lock(errhandler_file_cb_mutex); 20 | try { 21 | auto fn = errhandler_file_cb_map.at(errhandler); 22 | #if DEBUG 23 | printf("%s: lookup(errhandler=%lx) -> user_fn=%p\n", 24 | __func__, (intptr_t)errhandler, user_fn); 25 | #endif 26 | if (fn != NULL) { 27 | *user_fn = fn; 28 | } 29 | return 1; 30 | } 31 | catch (...) { 32 | printf("%s: lookup(errhandler=%lx) failed\n", __func__, (intptr_t)errhandler); 33 | return 0; 34 | } 35 | } 36 | 37 | int remove_file_errhandler_callback(MPI_Errhandler errhandler) 38 | { 39 | const std::lock_guard lock(errhandler_file_cb_mutex); 40 | // returns the number of elements removed, so 0=failure and 1=success 41 | return errhandler_file_cb_map.erase(errhandler); 42 | } 43 | -------------------------------------------------------------------------------- /impl-keyval-map-fileerrh2.h: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | int add_file_errhandler_callback_2(MPI_File file, WRAP_File_errhandler_function * user_fn) 4 | { 5 | const std::lock_guard lock(file_errhandler_cb_mutex); 6 | #if DEBUG 7 | printf("%s: insert_or_assign(file=%lx, user_fn=%p)\n", 8 | __func__, (intptr_t)file, user_fn); 9 | #endif 10 | // insert_or_assign (C++17) inserts an element or assigns to the current element if the key already exists 11 | auto [it,rc] = file_errhandler_cb_map.insert_or_assign(file, user_fn); 12 | return 1; // SUCCESS int{rc}; 13 | (void)it; 14 | (void)rc; 15 | } 16 | 17 | int find_file_errhandler_callback_2(MPI_File file, WRAP_File_errhandler_function ** user_fn) 18 | { 19 | const std::lock_guard lock(file_errhandler_cb_mutex); 20 | try { 21 | auto fn = file_errhandler_cb_map.at(file); 22 | #if DEBUG 23 | printf("%s: lookup(file=%lx) -> user_fn=%p\n", 24 | __func__, (intptr_t)file, user_fn); 25 | #endif 26 | if (fn != NULL) { 27 | *user_fn = fn; 28 | } 29 | return 1; 30 | } 31 | catch (...) { 32 | printf("%s: lookup(file=%lx) failed\n", __func__, (intptr_t)file); 33 | return 0; 34 | } 35 | } 36 | 37 | int remove_file_errhandler_callback_2(MPI_File file) 38 | { 39 | const std::lock_guard lock(file_errhandler_cb_mutex); 40 | // returns the number of elements removed, so 0=failure and 1=success 41 | return file_errhandler_cb_map.erase(file); 42 | } 43 | -------------------------------------------------------------------------------- /impl-keyval-map-ireqa2aw.h: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | int add_nonblocking_request_alltoallw_buffers(MPI_Request request, MPI_Datatype * sendtypes, MPI_Datatype * recvtypes) 4 | { 5 | const std::lock_guard lock(request_nonblocking_alltoallw_mutex); 6 | #if DEBUG 7 | printf("%s: insert_or_assign(request=%lx, sendtypes=%p, recvtypes=%p)\n", 8 | __func__, (intptr_t)request, sendtypes, recvtypes); 9 | #endif 10 | // insert_or_assign (C++17) inserts an element or assigns to the current element if the key already exists 11 | auto [it,rc] = request_nonblocking_alltoallw_map.insert_or_assign(request, std::make_pair(sendtypes, recvtypes)); 12 | #if DEBUG 13 | printf("%s: insert_or_assign(request=%lx, sendtypes=%p, recvtypes=%p) returned %d\n", 14 | __func__, (intptr_t)request, sendtypes, recvtypes, rc); 15 | #endif 16 | return 1; // SUCCESS int{rc}; 17 | (void)it; 18 | (void)rc; 19 | } 20 | 21 | int find_nonblocking_request_alltoallw_buffers(MPI_Request request, MPI_Datatype ** sendtypes, MPI_Datatype ** recvtypes) 22 | { 23 | const std::lock_guard lock(request_nonblocking_alltoallw_mutex); 24 | if (request_nonblocking_alltoallw_map.empty()) return 0; 25 | 26 | try { 27 | auto [stypes,rtypes] = request_nonblocking_alltoallw_map.at(request); 28 | #if DEBUG 29 | printf("%s: lookup(request=%lx) -> [sendtypes=%p, sendtypes=%p]\n", 30 | __func__, (intptr_t)request, sendtypes, recvtypes); 31 | #endif 32 | if (stypes != NULL) { 33 | *sendtypes = stypes; 34 | } 35 | else if (rtypes != NULL) { 36 | *recvtypes = rtypes; 37 | } 38 | return 1; 39 | } 40 | catch (...) { 41 | #if DEBUG 42 | printf("%s: lookup(request=%lx) failed\n", __func__, (intptr_t)request); 43 | #endif 44 | return 0; 45 | } 46 | } 47 | 48 | int remove_nonblocking_request_alltoallw_buffers(MPI_Request request) 49 | { 50 | const std::lock_guard lock(request_nonblocking_alltoallw_mutex); 51 | // returns the number of elements removed, so 0=failure and 1=success 52 | return request_nonblocking_alltoallw_map.erase(request); 53 | } 54 | -------------------------------------------------------------------------------- /impl-keyval-map-opuserfn.h: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | int add_comm_op_callback(MPI_Op op, 4 | WRAP_User_function * user_fn, 5 | WRAP_User_function_c * user_fn_c) 6 | { 7 | const std::lock_guard lock(op_user_function_mutex); 8 | #if DEBUG 9 | printf("%s: insert_or_assign(op=%lx, user_fn=%p, user_fn_c=%p)\n", 10 | __func__, (intptr_t)op, user_fn, user_fn_c); 11 | #endif 12 | // insert_or_assign (C++17) inserts an element or assigns to the current element if the key already exists 13 | auto [it,rc] = op_user_function_map.insert_or_assign(op, std::make_pair(user_fn, user_fn_c)); 14 | #if DEBUG 15 | printf("%s: insert_or_assign(op=%lx, user_fn=%p, user_fn_c=%p) returned %d\n", 16 | __func__, (intptr_t)op, user_fn, user_fn_c, rc); 17 | #endif 18 | return 1; // SUCCESS int{rc}; 19 | (void)it; 20 | (void)rc; 21 | } 22 | 23 | int find_comm_op_callback(MPI_Op op, 24 | WRAP_User_function ** user_fn, 25 | WRAP_User_function_c ** user_fn_c) 26 | { 27 | const std::lock_guard lock(op_user_function_mutex); 28 | try { 29 | auto [fn,fn_c] = op_user_function_map.at(op); 30 | #if DEBUG 31 | printf("%s: lookup(op=%lx) -> [user_fn=%p, user_fn=%p]\n", 32 | __func__, (intptr_t)op, user_fn, user_fn_c); 33 | #endif 34 | if (fn != NULL) { 35 | *user_fn = fn; 36 | } 37 | else if (fn_c != NULL) { 38 | *user_fn_c = fn_c; 39 | } 40 | return 1; 41 | } 42 | catch (...) { 43 | printf("%s: lookup(op=%lx) failed\n", __func__, (intptr_t)op); 44 | return 0; 45 | } 46 | } 47 | 48 | int remove_comm_op_callback(MPI_Op op) 49 | { 50 | const std::lock_guard lock(op_user_function_mutex); 51 | // returns the number of elements removed, so 0=failure and 1=success 52 | return op_user_function_map.erase(op); 53 | } 54 | -------------------------------------------------------------------------------- /impl-keyval-map-preqa2aw.h: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | int add_persistent_request_alltoallw_buffers(MPI_Request request, MPI_Datatype * sendtypes, MPI_Datatype * recvtypes) 4 | { 5 | const std::lock_guard lock(request_persistent_alltoallw_mutex); 6 | #if DEBUG 7 | printf("%s: insert_or_assign(request=%lx, sendtypes=%p, recvtypes=%p)\n", 8 | __func__, (intptr_t)request, sendtypes, recvtypes); 9 | #endif 10 | // insert_or_assign (C++17) inserts an element or assigns to the current element if the key already exists 11 | auto [it,rc] = request_persistent_alltoallw_map.insert_or_assign(request, std::make_pair(sendtypes, recvtypes)); 12 | #if DEBUG 13 | printf("%s: insert_or_assign(request=%lx, sendtypes=%p, recvtypes=%p) returned %d\n", 14 | __func__, (intptr_t)request, sendtypes, recvtypes, rc); 15 | #endif 16 | return 1; // SUCCESS int{rc}; 17 | (void)it; 18 | (void)rc; 19 | } 20 | 21 | int find_persistent_request_alltoallw_buffers(MPI_Request request, MPI_Datatype ** sendtypes, MPI_Datatype ** recvtypes) 22 | { 23 | const std::lock_guard lock(request_persistent_alltoallw_mutex); 24 | if (request_persistent_alltoallw_map.empty()) return 0; 25 | 26 | try { 27 | auto [stypes,rtypes] = request_persistent_alltoallw_map.at(request); 28 | #if DEBUG 29 | printf("%s: lookup(request=%lx) -> [sendtypes=%p, sendtypes=%p]\n", 30 | __func__, (intptr_t)request, sendtypes, recvtypes); 31 | #endif 32 | if (stypes != NULL) { 33 | *sendtypes = stypes; 34 | } 35 | else if (rtypes != NULL) { 36 | *recvtypes = rtypes; 37 | } 38 | return 1; 39 | } 40 | catch (...) { 41 | #if DEBUG 42 | printf("%s: lookup(request=%lx) failed\n", __func__, (intptr_t)request); 43 | #endif 44 | return 0; 45 | } 46 | } 47 | 48 | int remove_persistent_request_alltoallw_buffers(MPI_Request request) 49 | { 50 | const std::lock_guard lock(request_persistent_alltoallw_mutex); 51 | // returns the number of elements removed, so 0=failure and 1=success 52 | return request_persistent_alltoallw_map.erase(request); 53 | } 54 | -------------------------------------------------------------------------------- /impl-keyval-map-typeattr.h: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | int add_type_keyval_callbacks(int keyval, 4 | WRAP_Type_copy_attr_function * type_copy_attr_fn, 5 | WRAP_Type_delete_attr_function * type_delete_attr_fn) 6 | { 7 | const std::lock_guard lock(keyval_type_attr_cb_mutex); 8 | #if DEBUG 9 | printf("%s: insert_or_assign(keyval=%d, type_copy_attr_fn=%p, type_delete_attr_fn=%p)\n", 10 | __func__, keyval, type_copy_attr_fn, type_delete_attr_fn); 11 | #endif 12 | // insert_or_assign (C++17) inserts an element or assigns to the current element if the key already exists 13 | auto [it,rc] = keyval_type_attr_cb_map.insert_or_assign(keyval, 14 | std::make_pair(type_copy_attr_fn, 15 | type_delete_attr_fn)); 16 | return 1; // SUCCESS int{rc}; 17 | (void)it; 18 | (void)rc; 19 | } 20 | 21 | int find_type_keyval_callbacks(int keyval, 22 | WRAP_Type_copy_attr_function ** type_copy_attr_fn, 23 | WRAP_Type_delete_attr_function ** type_delete_attr_fn) 24 | { 25 | const std::lock_guard lock(keyval_type_attr_cb_mutex); 26 | try { 27 | auto [copy_fn,delete_fn] = keyval_type_attr_cb_map.at(keyval); 28 | #if DEBUG 29 | printf("%s: lookup(keyval=%d) -> [type_copy_attr_fn=%p, type_delete_attr_fn=%p]\n", 30 | __func__, keyval, copy_fn, delete_fn); 31 | #endif 32 | if (type_copy_attr_fn != NULL) { 33 | *type_copy_attr_fn = copy_fn; 34 | } 35 | if (type_delete_attr_fn != NULL) { 36 | *type_delete_attr_fn = delete_fn; 37 | } 38 | return 1; 39 | } 40 | catch (...) { 41 | printf("%s: lookup(keyval=%d) failed\n", __func__, keyval); 42 | return 0; 43 | } 44 | } 45 | 46 | int remove_type_keyval_callbacks(int keyval) 47 | { 48 | const std::lock_guard lock(keyval_type_attr_cb_mutex); 49 | // returns the number of elements removed, so 0=failure and 1=success 50 | return keyval_type_attr_cb_map.erase(keyval); 51 | } 52 | -------------------------------------------------------------------------------- /impl-keyval-map-winattr.h: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | int add_win_keyval_callbacks(int keyval, 4 | WRAP_Win_copy_attr_function * win_copy_attr_fn, 5 | WRAP_Win_delete_attr_function * win_delete_attr_fn) 6 | { 7 | const std::lock_guard lock(keyval_win_attr_cb_mutex); 8 | #if DEBUG 9 | printf("%s: insert_or_assign(keyval=%d, win_copy_attr_fn=%p, win_delete_attr_fn=%p)\n", 10 | __func__, keyval, win_copy_attr_fn, win_delete_attr_fn); 11 | #endif 12 | // insert_or_assign (C++17) inserts an element or assigns to the current element if the key already exists 13 | auto [it,rc] = keyval_win_attr_cb_map.insert_or_assign(keyval, 14 | std::make_pair(win_copy_attr_fn, 15 | win_delete_attr_fn)); 16 | return 1; // SUCCESS int{rc}; 17 | (void)it; 18 | (void)rc; 19 | } 20 | 21 | int find_win_keyval_callbacks(int keyval, 22 | WRAP_Win_copy_attr_function ** win_copy_attr_fn, 23 | WRAP_Win_delete_attr_function ** win_delete_attr_fn) 24 | { 25 | const std::lock_guard lock(keyval_win_attr_cb_mutex); 26 | try { 27 | auto [copy_fn,delete_fn] = keyval_win_attr_cb_map.at(keyval); 28 | #if DEBUG 29 | printf("%s: lookup(keyval=%d) -> [win_copy_attr_fn=%p, win_delete_attr_fn=%p]\n", 30 | __func__, keyval, copy_fn, delete_fn); 31 | #endif 32 | if (win_copy_attr_fn != NULL) { 33 | *win_copy_attr_fn = copy_fn; 34 | } 35 | if (win_delete_attr_fn != NULL) { 36 | *win_delete_attr_fn = delete_fn; 37 | } 38 | return 1; 39 | } 40 | catch (...) { 41 | printf("%s: lookup(keyval=%d) failed\n", __func__, keyval); 42 | return 0; 43 | } 44 | } 45 | 46 | int remove_win_keyval_callbacks(int keyval) 47 | { 48 | const std::lock_guard lock(keyval_win_attr_cb_mutex); 49 | // returns the number of elements removed, so 0=failure and 1=success 50 | return keyval_win_attr_cb_map.erase(keyval); 51 | } 52 | -------------------------------------------------------------------------------- /impl-keyval-map-winerrh.h: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | int add_win_errhandler_callback(MPI_Errhandler errhandler, WRAP_Win_errhandler_function * user_fn) 4 | { 5 | const std::lock_guard lock(errhandler_win_cb_mutex); 6 | #if DEBUG 7 | printf("%s: insert_or_assign(errhandler=%lx, user_fn=%p)\n", 8 | __func__, (intptr_t)errhandler, user_fn); 9 | #endif 10 | // insert_or_assign (C++17) inserts an element or assigns to the current element if the key already exists 11 | auto [it,rc] = errhandler_win_cb_map.insert_or_assign(errhandler, user_fn); 12 | return 1; // SUCCESS int{rc}; 13 | (void)it; 14 | (void)rc; 15 | } 16 | 17 | int find_win_errhandler_callback(MPI_Errhandler errhandler, WRAP_Win_errhandler_function ** user_fn) 18 | { 19 | const std::lock_guard lock(errhandler_win_cb_mutex); 20 | try { 21 | auto fn = errhandler_win_cb_map.at(errhandler); 22 | #if DEBUG 23 | printf("%s: lookup(errhandler=%lx) -> user_fn=%p\n", 24 | __func__, (intptr_t)errhandler, user_fn); 25 | #endif 26 | if (fn != NULL) { 27 | *user_fn = fn; 28 | } 29 | return 1; 30 | } 31 | catch (...) { 32 | printf("%s: lookup(errhandler=%lx) failed\n", __func__, (intptr_t)errhandler); 33 | return 0; 34 | } 35 | } 36 | 37 | int remove_win_errhandler_callback(MPI_Errhandler errhandler) 38 | { 39 | const std::lock_guard lock(errhandler_win_cb_mutex); 40 | // returns the number of elements removed, so 0=failure and 1=success 41 | return errhandler_win_cb_map.erase(errhandler); 42 | } 43 | -------------------------------------------------------------------------------- /impl-keyval-map.cc: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | #ifndef __cplusplus 4 | #error Sorry, but you have to use a C++ compiler. 5 | #endif 6 | #if defined(__cplusplus) && (__cplusplus < 201703L) 7 | #error Sorry, but you have to use a C++17 compiler. 8 | #endif 9 | 10 | #include 11 | #include 12 | #include 13 | 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | extern "C" { 20 | 21 | // we do not want anything MPI C++ related here. 22 | // this header is only used to determine the MPI ABI we are compiling against, 23 | // and then only the scalar types (impl-scalar-types.h) it seems. 24 | #define OMPI_SKIP_MPICXX 25 | #define MPICH_SKIP_MPICXX 26 | #include 27 | 28 | #include "muk-predefined.h" 29 | #include "wrap-handle-typedefs.h" 30 | 31 | } 32 | 33 | // we will not try to purge the callback mappings when handle keys are no longer used 34 | // because that will require reference counting. instead, we will just keep 35 | // them around forever, which has the potential to waste a trivial amount of memory - 36 | // it is key and two pointers worth of storage (20 or 24 bytes) per entry. 37 | // 38 | // the lookup cost of a std::map is logarithmic in the size of the container, which 39 | // is a price worth paying to avoid dealing with the use-after-free situation. 40 | // 41 | // TODO: 42 | // to please memory tools like valgrind, we will purge everything using clear() 43 | // during finalization. 44 | 45 | // for user-defined reductions 46 | std::map> op_user_function_map{}; 47 | 48 | // for attributes 49 | std::map> keyval_comm_attr_cb_map{}; 50 | std::map> keyval_type_attr_cb_map{}; 51 | std::map> keyval_win_attr_cb_map{}; 52 | 53 | // for errhandlers 54 | std::map errhandler_comm_cb_map{}; 55 | std::map errhandler_win_cb_map{}; 56 | 57 | // because we do not have file attributes, we need a second map based on the 58 | // file handle itself, to do the lookup. 59 | // File_create_errhandler adds to the first. 60 | // File_set_errhandler looks in the first and adds to the second. 61 | // file_errhandler_trampoline looks in the second. 62 | // File_close removes from the second. 63 | // nothing removes from the first (see above). 64 | std::map errhandler_file_cb_map{}; 65 | std::map file_errhandler_cb_map{}; 66 | 67 | // for nonblocking alltoallw 68 | // 69 | // persistent ialltoallw must be concluded with MPI_Request_free so we should have a dedicated 70 | // list for these, to ensure that we only pay the lookup cost in MPI_Request_free. 71 | // 72 | // for neighborhood ialltoallw, we can associate the allocations with the communicator 73 | // and clean up when it is freed, although more than one outstanding ialltoallw per communicator 74 | // is permitted, so we need to use a std::multimap here. 75 | // 76 | // for ialltoallw, we should just assume the user will not free the associated communicator 77 | // in a timely fashion (it will probably be a predefined one) and we have to use the request 78 | // as the key, which means putting the lookup on the critical path of request completion. 79 | // 80 | std::map> request_nonblocking_alltoallw_map{}; 81 | std::map> request_persistent_alltoallw_map{}; 82 | 83 | std::mutex keyval_comm_attr_cb_mutex; 84 | std::mutex keyval_type_attr_cb_mutex; 85 | std::mutex keyval_win_attr_cb_mutex; 86 | std::mutex op_user_function_mutex; 87 | std::mutex errhandler_comm_cb_mutex; 88 | std::mutex errhandler_win_cb_mutex; 89 | std::mutex errhandler_file_cb_mutex; 90 | std::mutex file_errhandler_cb_mutex; 91 | std::mutex request_nonblocking_alltoallw_mutex; 92 | std::mutex request_persistent_alltoallw_mutex; 93 | 94 | extern "C" { 95 | 96 | // in all these APIS, we use int as the boolean return code 97 | // to avoid theoretical incompatibilities between C and C++ bool 98 | 99 | #include "impl-keyval-map-commattr.h" // keyval_comm_attr_cb_map 100 | #include "impl-keyval-map-typeattr.h" // keyval_type_attr_cb_map 101 | #include "impl-keyval-map-winattr.h" // keyval_win_attr_cb_map 102 | #include "impl-keyval-map-opuserfn.h" // op_user_function_map 103 | #include "impl-keyval-map-commerrh.h" // errhandler_comm_cb_map 104 | #include "impl-keyval-map-winerrh.h" // errhandler_win_cb_map 105 | #include "impl-keyval-map-fileerrh.h" // errhandler_file_cb_map 106 | #include "impl-keyval-map-fileerrh2.h" // file_errhandler_cb_map 107 | #include "impl-keyval-map-ireqa2aw.h" // request_nonblocking_alltoallw_map 108 | #include "impl-keyval-map-preqa2aw.h" // request_persistent_alltoallw_map 109 | 110 | int cleanup_mapped_request(MPI_Request request) 111 | { 112 | // look for nonblocking alltoallw first 113 | if (!request_nonblocking_alltoallw_map.empty()) 114 | { 115 | MPI_Datatype * sendtypes = NULL; 116 | MPI_Datatype * recvtypes = NULL; 117 | int found = find_nonblocking_request_alltoallw_buffers(request, &sendtypes, &recvtypes); 118 | if (found) { 119 | if (sendtypes != NULL) { 120 | free(sendtypes); 121 | sendtypes = NULL; 122 | } 123 | if (recvtypes != NULL) { 124 | free(recvtypes); 125 | recvtypes = NULL; 126 | } 127 | int rc = remove_nonblocking_request_alltoallw_buffers(request); 128 | if (!rc) { 129 | printf("%s: found request=%lx but could not remove it\n",__func__,(intptr_t)request); 130 | } 131 | return 1; 132 | } 133 | } 134 | 135 | // look for persistent alltoallw next 136 | if (!request_persistent_alltoallw_map.empty()) 137 | { 138 | MPI_Datatype * sendtypes = NULL; 139 | MPI_Datatype * recvtypes = NULL; 140 | int found = find_persistent_request_alltoallw_buffers(request, &sendtypes, &recvtypes); 141 | if (found) { 142 | if (sendtypes != NULL) { 143 | free(sendtypes); 144 | sendtypes = NULL; 145 | } 146 | if (recvtypes != NULL) { 147 | free(recvtypes); 148 | recvtypes = NULL; 149 | } 150 | int rc =remove_persistent_request_alltoallw_buffers(request); 151 | if (!rc) { 152 | printf("%s: found request=%lx but could not remove it\n",__func__,(intptr_t)request); 153 | } 154 | return 1; 155 | } 156 | } 157 | return 0; 158 | } 159 | 160 | void WRAP_Clear_maps(void) 161 | { 162 | // user reduce ops 163 | op_user_function_map.clear(); 164 | 165 | // attr 166 | keyval_comm_attr_cb_map.clear(); 167 | keyval_type_attr_cb_map.clear(); 168 | keyval_win_attr_cb_map.clear(); 169 | 170 | // errhandlers 171 | errhandler_comm_cb_map.clear(); 172 | errhandler_win_cb_map.clear(); 173 | errhandler_file_cb_map.clear(); 174 | file_errhandler_cb_map.clear(); 175 | 176 | // alltoallw 177 | request_nonblocking_alltoallw_map.clear(); 178 | request_persistent_alltoallw_map.clear(); 179 | } 180 | 181 | } // extern "C" 182 | -------------------------------------------------------------------------------- /impl-keyval-map.h: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | #ifndef IMPL_KEYVAL_MAP_H 4 | #define IMPL_KEYVAL_MAP_H 5 | 6 | // C declarations of the map stuff written in C++ in impl-keyval-map.cc 7 | 8 | // this header is only used to determine the MPI ABI we are compiling against, 9 | // and then only the scalar types (impl-scalar-types.h) it seems. 10 | #include 11 | 12 | #include "muk-predefined.h" 13 | #include "wrap-handle-typedefs.h" 14 | 15 | // return values are 0=failure 1=success but are int not bool because C and C++ bool 16 | // are not guaranteed to be the same. 17 | 18 | // OP USERFN 19 | 20 | int add_comm_op_callback(MPI_Op op, 21 | WRAP_User_function * user_fn, 22 | WRAP_User_function_c * user_fn_c); 23 | 24 | int find_comm_op_callback(MPI_Op op, 25 | WRAP_User_function ** user_fn, 26 | WRAP_User_function_c ** user_fn_c); 27 | 28 | int remove_comm_op_callback(MPI_Op op); 29 | 30 | // COMM KEYVAL 31 | 32 | int add_comm_keyval_callbacks(int keyval, 33 | WRAP_Comm_copy_attr_function * comm_copy_attr_fn, 34 | WRAP_Comm_delete_attr_function * comm_delete_attr_fn); 35 | 36 | int find_comm_keyval_callbacks(int keyval, 37 | WRAP_Comm_copy_attr_function ** comm_copy_attr_fn, 38 | WRAP_Comm_delete_attr_function ** comm_delete_attr_fn); 39 | 40 | int remove_comm_keyval_callbacks(int keyval); 41 | 42 | // TYPE KEYVAL 43 | 44 | int add_type_keyval_callbacks(int keyval, 45 | WRAP_Type_copy_attr_function * type_copy_attr_fn, 46 | WRAP_Type_delete_attr_function * type_delete_attr_fn); 47 | 48 | int find_type_keyval_callbacks(int keyval, 49 | WRAP_Type_copy_attr_function ** type_copy_attr_fn, 50 | WRAP_Type_delete_attr_function ** type_delete_attr_fn); 51 | 52 | int remove_type_keyval_callbacks(int keyval); 53 | 54 | // WIN KEYVAL 55 | 56 | int add_win_keyval_callbacks(int keyval, 57 | WRAP_Win_copy_attr_function * win_copy_attr_fn, 58 | WRAP_Win_delete_attr_function * win_delete_attr_fn); 59 | 60 | int find_win_keyval_callbacks(int keyval, 61 | WRAP_Win_copy_attr_function ** win_copy_attr_fn, 62 | WRAP_Win_delete_attr_function ** win_delete_attr_fn); 63 | 64 | int remove_win_keyval_callbacks(int keyval); 65 | 66 | // COMM ERRH 67 | 68 | int add_comm_errhandler_callback(MPI_Errhandler errhandler, WRAP_Comm_errhandler_function * user_fn); 69 | int find_comm_errhandler_callback(MPI_Errhandler errhandler, WRAP_Comm_errhandler_function ** user_fn); 70 | int remove_comm_errhandler_callback(MPI_Errhandler errhandler); 71 | 72 | // WIN ERRH 73 | 74 | int add_win_errhandler_callback(MPI_Errhandler errhandler, WRAP_Win_errhandler_function * user_fn); 75 | int find_win_errhandler_callback(MPI_Errhandler errhandler, WRAP_Win_errhandler_function ** user_fn); 76 | int remove_win_errhandler_callback(MPI_Errhandler errhandler); 77 | 78 | // FILE ERRH 79 | 80 | int add_file_errhandler_callback(MPI_Errhandler errhandler, WRAP_File_errhandler_function * user_fn); 81 | int find_file_errhandler_callback(MPI_Errhandler errhandler, WRAP_File_errhandler_function ** user_fn); 82 | int remove_file_errhandler_callback(MPI_Errhandler errhandler); 83 | 84 | // FILE ERRH 2 85 | 86 | int add_file_errhandler_callback_2(MPI_File file, WRAP_File_errhandler_function * user_fn); 87 | int find_file_errhandler_callback_2(MPI_File file, WRAP_File_errhandler_function ** user_fn); 88 | int remove_file_errhandler_callback_2(MPI_File file); 89 | 90 | // ALLTOALLW - NONBLOCKING 91 | 92 | int add_nonblocking_request_alltoallw_buffers(MPI_Request request, MPI_Datatype * sendtypes, MPI_Datatype * recvtypes); 93 | int find_nonblocking_request_alltoallw_buffers(MPI_Request request, MPI_Datatype ** sendtypes, MPI_Datatype ** recvtypes); 94 | int remove_nonblocking_request_alltoallw_buffers(MPI_Request request); 95 | 96 | // ALLTOALLW - PERSISTENT 97 | 98 | int add_persistent_request_alltoallw_buffers(MPI_Request request, MPI_Datatype * sendtypes, MPI_Datatype * recvtypes); 99 | int find_persistent_request_alltoallw_buffers(MPI_Request request, MPI_Datatype ** sendtypes, MPI_Datatype ** recvtypes); 100 | int remove_persistent_request_alltoallw_buffers(MPI_Request request); 101 | 102 | // AGGREGATE REQUEST CLEANUP 103 | 104 | int cleanup_mapped_request(MPI_Request request); 105 | 106 | #endif // IMPL_KEYVAL_MAP_H 107 | -------------------------------------------------------------------------------- /impl-keyval.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | #include 8 | 9 | #define MUK_EXTERN extern 10 | #include "impl-fpointers.h" 11 | 12 | // for user-defined reductions 13 | int TYPE_ATTR_FOR_USER_OP_FN = MPI_KEYVAL_INVALID; 14 | int COMM_EH_HANDLE_KEY = MPI_KEYVAL_INVALID; 15 | int WIN_EH_HANDLE_KEY = MPI_KEYVAL_INVALID; 16 | 17 | void WRAP_Init_handle_key(void) 18 | { 19 | int rc; 20 | rc = IMPL_Type_create_keyval(MPI_TYPE_NULL_COPY_FN, MPI_TYPE_NULL_DELETE_FN, &TYPE_ATTR_FOR_USER_OP_FN, NULL); 21 | if (rc != MPI_SUCCESS) { 22 | printf("IMPL_Type_create_keyval(TYPE_ATTR_FOR_USER_OP_FN) failed: %d\n", rc); 23 | } 24 | // use the DUP copy function so that the attribute containing the errh callback fptr is copied 25 | // to the new communicator. we need to remember to delete the cookie when the comm/win objects are freed. 26 | //rc = IMPL_Comm_create_keyval(MPI_COMM_DUP_FN, comm_delete_cookie, &COMM_EH_HANDLE_KEY, NULL); 27 | rc = IMPL_Comm_create_keyval(MPI_COMM_DUP_FN, MPI_COMM_NULL_DELETE_FN, &COMM_EH_HANDLE_KEY, NULL); 28 | if (rc != MPI_SUCCESS) { 29 | printf("IMPL_Comm_create_keyval(COMM_EH_HANDLE_KEY) failed: %d\n", rc); 30 | } 31 | //printf("%s: COMM_EH_HANDLE_KEY=%x\n", __func__, COMM_EH_HANDLE_KEY); 32 | //rc = IMPL_Win_create_keyval(MPI_WIN_DUP_FN, win_delete_cookie, &WIN_EH_HANDLE_KEY, NULL); 33 | rc = IMPL_Win_create_keyval(MPI_WIN_DUP_FN, MPI_WIN_NULL_DELETE_FN, &WIN_EH_HANDLE_KEY, NULL); 34 | if (rc != MPI_SUCCESS) { 35 | printf("IMPL_Win_create_keyval(WIN_EH_HANDLE_KEY) failed: %d\n", rc); 36 | } 37 | //printf("%s: WIN_EH_HANDLE_KEY=%x\n", __func__, WIN_EH_HANDLE_KEY); 38 | } 39 | 40 | void WRAP_Finalize_handle_key(void) 41 | { 42 | int rc = MPI_SUCCESS; 43 | if (TYPE_ATTR_FOR_USER_OP_FN != MPI_KEYVAL_INVALID) { 44 | rc = IMPL_Type_free_keyval(&TYPE_ATTR_FOR_USER_OP_FN); 45 | if (rc != MPI_SUCCESS) { 46 | printf("IMPL_Type_free_keyval(TYPE_ATTR_FOR_USER_OP_FN) failed: %d\n", rc); 47 | } 48 | } 49 | if (COMM_EH_HANDLE_KEY != MPI_KEYVAL_INVALID) { 50 | rc = IMPL_Comm_free_keyval(&COMM_EH_HANDLE_KEY); 51 | if (rc != MPI_SUCCESS) { 52 | printf("IMPL_Comm_free_keyval(COMM_EH_HANDLE_KEY) failed: %d\n", rc); 53 | } 54 | } 55 | if (WIN_EH_HANDLE_KEY != MPI_KEYVAL_INVALID) { 56 | rc = IMPL_Win_free_keyval(&WIN_EH_HANDLE_KEY); 57 | if (rc != MPI_SUCCESS) { 58 | printf("IMPL_Win_free_keyval(WIN_EH_HANDLE_KEY) failed: %d\n", rc); 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /impl-predefined-handle.h: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | #ifndef IMPL_PREDEFINED_OP_H 4 | #define IMPL_PREDEFINED_OP_H 5 | 6 | #define MAYBE_UNUSED __attribute__((unused)) 7 | 8 | MAYBE_UNUSED 9 | static bool IS_PREDEFINED_OP(MPI_Op op) 10 | { 11 | // it is ideal to optimize the branching order here by ordering the tests 12 | // in order of the frequency that the predefine ops are used in real code. 13 | return (op == MPI_SUM || 14 | op == MPI_MAX || 15 | op == MPI_MIN || 16 | op == MPI_LAND || 17 | op == MPI_LOR || 18 | op == MPI_BAND || 19 | op == MPI_BOR || 20 | op == MPI_LXOR || 21 | op == MPI_BXOR || 22 | op == MPI_MAXLOC || 23 | op == MPI_MINLOC || 24 | // no one uses this, so it is last among the valid ones. 25 | op == MPI_PROD || 26 | // these two are invalid for reductions but they are predefined 27 | // so we will include them and let the implementation throw an 28 | // error in the reduction. 29 | op == MPI_REPLACE || 30 | op == MPI_NO_OP); 31 | } 32 | 33 | MAYBE_UNUSED 34 | static bool IS_PREDEFINED_ERRHANDLER(MPI_Errhandler errhandler) 35 | { 36 | return ( (errhandler == MPI_ERRORS_ARE_FATAL) || 37 | (errhandler == MPI_ERRORS_RETURN) || 38 | #if MPI_VERSION >= 4 39 | (errhandler == MPI_ERRORS_ABORT) || 40 | #endif 41 | (errhandler == MPI_ERRHANDLER_NULL) 42 | ); 43 | } 44 | 45 | MAYBE_UNUSED 46 | static bool IS_DERIVED_DATATYPE(MPI_Datatype type) 47 | { 48 | int ni, na, nd, combiner; 49 | int rc = IMPL_Type_get_envelope(type, &ni, &na, &nd, &combiner); 50 | return ((combiner != MPI_COMBINER_NAMED) && (combiner != MPI_COMBINER_F90_INTEGER) && 51 | (combiner != MPI_COMBINER_F90_REAL) && (combiner != MPI_COMBINER_F90_COMPLEX)); 52 | (void)rc; 53 | } 54 | 55 | #endif 56 | -------------------------------------------------------------------------------- /impl-predefined.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | #include 4 | #include 5 | #include "mpi.h" 6 | #include "muk-predefined.h" 7 | 8 | // Buffer Address Constants 9 | #if ( __STDC_VERSION__ >= 201112L) && !defined(__clang__) 10 | // Clang refuses to do this 11 | _Static_assert(MPI_BOTTOM == NULL , "MPI_BOTTOM == NULL" ); 12 | #endif 13 | void * IMPL_BOTTOM = MPI_BOTTOM; 14 | void * IMPL_IN_PLACE = MPI_IN_PLACE; 15 | 16 | // we avoid converting thread levels because every known implementation uses 17 | // the same values for them, but we explicitly verify the assumption here. 18 | #if ( __STDC_VERSION__ >= 201112L) 19 | _Static_assert( (int)MUK_UNDEFINED == (int)MPI_UNDEFINED , "MUK_UNDEFINED == MPI_UNDEFINED" ); 20 | #endif 21 | 22 | // we avoid converting thread levels because every known implementation uses 23 | // the same values for them, but we explicitly verify the assumption here. 24 | #if ( __STDC_VERSION__ >= 201112L) 25 | _Static_assert( (int)MUK_IDENT == (int)MPI_IDENT , "MUK_IDENT == MPI_IDENT" ); 26 | _Static_assert( (int)MUK_CONGRUENT == (int)MPI_CONGRUENT , "MUK_CONGRUENT == MPI_CONGRUENT" ); 27 | _Static_assert( (int)MUK_SIMILAR == (int)MPI_SIMILAR , "MUK_SIMILAR == MPI_SIMILAR" ); 28 | _Static_assert( (int)MUK_UNEQUAL == (int)MPI_UNEQUAL , "MUK_UNEQUAL == MPI_UNEQUAL" ); 29 | #endif 30 | 31 | // Results of communicator and group comparisons 32 | int IMPL_IDENT = MPI_IDENT; 33 | int IMPL_CONGRUENT = MPI_CONGRUENT; 34 | int IMPL_SIMILAR = MPI_SIMILAR; 35 | int IMPL_UNEQUAL = MPI_UNEQUAL; 36 | 37 | // Environmental inquiry keys 38 | int IMPL_TAG_UB = MPI_TAG_UB; 39 | int IMPL_IO = MPI_IO; 40 | int IMPL_HOST = MPI_HOST; 41 | int IMPL_WTIME_IS_GLOBAL = MPI_WTIME_IS_GLOBAL; 42 | 43 | // Topologies 44 | int IMPL_GRAPH = MPI_GRAPH; 45 | int IMPL_CART = MPI_CART; 46 | int IMPL_DIST_GRAPH = MPI_DIST_GRAPH; 47 | 48 | // Predefined functions 49 | // TODO 50 | // Deprecated predefined functions 51 | // TODO 52 | 53 | // Predefined Attribute Keys 54 | int IMPL_APPNUM = MPI_APPNUM; 55 | int IMPL_LASTUSEDCODE = MPI_LASTUSEDCODE; 56 | int IMPL_UNIVERSE_SIZE = MPI_UNIVERSE_SIZE; 57 | int IMPL_WIN_BASE = MPI_WIN_BASE; 58 | int IMPL_WIN_DISP_UNIT = MPI_WIN_DISP_UNIT; 59 | int IMPL_WIN_SIZE = MPI_WIN_SIZE; 60 | int IMPL_WIN_CREATE_FLAVOR = MPI_WIN_CREATE_FLAVOR; 61 | int IMPL_WIN_MODEL = MPI_WIN_MODEL; 62 | 63 | // MPI Window Create Flavors 64 | int IMPL_WIN_FLAVOR_CREATE = MPI_WIN_FLAVOR_CREATE; 65 | int IMPL_WIN_FLAVOR_ALLOCATE = MPI_WIN_FLAVOR_ALLOCATE; 66 | int IMPL_WIN_FLAVOR_DYNAMIC = MPI_WIN_FLAVOR_DYNAMIC; 67 | int IMPL_WIN_FLAVOR_SHARED = MPI_WIN_FLAVOR_SHARED; 68 | 69 | // MPI Window Models 70 | int IMPL_WIN_SEPARATE = MPI_WIN_SEPARATE; 71 | int IMPL_WIN_UNIFIED = MPI_WIN_UNIFIED; 72 | 73 | // Mode Constants 74 | int IMPL_MODE_APPEND = MPI_MODE_APPEND; 75 | int IMPL_MODE_CREATE = MPI_MODE_CREATE; 76 | int IMPL_MODE_DELETE_ON_CLOSE = MPI_MODE_DELETE_ON_CLOSE; 77 | int IMPL_MODE_EXCL = MPI_MODE_EXCL; 78 | int IMPL_MODE_NOCHECK = MPI_MODE_NOCHECK; 79 | int IMPL_MODE_NOPRECEDE = MPI_MODE_NOPRECEDE; 80 | int IMPL_MODE_NOPUT = MPI_MODE_NOPUT; 81 | int IMPL_MODE_NOSTORE = MPI_MODE_NOSTORE; 82 | int IMPL_MODE_NOSUCCEED = MPI_MODE_NOSUCCEED; 83 | int IMPL_MODE_RDONLY = MPI_MODE_RDONLY; 84 | int IMPL_MODE_RDWR = MPI_MODE_RDWR; 85 | int IMPL_MODE_SEQUENTIAL = MPI_MODE_SEQUENTIAL; 86 | int IMPL_MODE_UNIQUE_OPEN = MPI_MODE_UNIQUE_OPEN; 87 | int IMPL_MODE_WRONLY = MPI_MODE_WRONLY; 88 | 89 | // Datatype Decoding Constants 90 | int IMPL_COMBINER_CONTIGUOUS = MPI_COMBINER_CONTIGUOUS; 91 | int IMPL_COMBINER_DARRAY = MPI_COMBINER_DARRAY; 92 | int IMPL_COMBINER_DUP = MPI_COMBINER_DUP; 93 | int IMPL_COMBINER_F90_COMPLEX = MPI_COMBINER_F90_COMPLEX; 94 | int IMPL_COMBINER_F90_INTEGER = MPI_COMBINER_F90_INTEGER; 95 | int IMPL_COMBINER_F90_REAL = MPI_COMBINER_F90_REAL; 96 | int IMPL_COMBINER_HINDEXED = MPI_COMBINER_HINDEXED; 97 | int IMPL_COMBINER_HVECTOR = MPI_COMBINER_HVECTOR; 98 | int IMPL_COMBINER_INDEXED_BLOCK = MPI_COMBINER_INDEXED_BLOCK; 99 | int IMPL_COMBINER_HINDEXED_BLOCK = MPI_COMBINER_HINDEXED_BLOCK; 100 | int IMPL_COMBINER_INDEXED = MPI_COMBINER_INDEXED; 101 | int IMPL_COMBINER_NAMED = MPI_COMBINER_NAMED; 102 | int IMPL_COMBINER_RESIZED = MPI_COMBINER_RESIZED; 103 | int IMPL_COMBINER_STRUCT = MPI_COMBINER_STRUCT; 104 | int IMPL_COMBINER_SUBARRAY = MPI_COMBINER_SUBARRAY; 105 | int IMPL_COMBINER_VECTOR = MPI_COMBINER_VECTOR; 106 | 107 | 108 | // we avoid converting thread levels because every known implementation uses 109 | // the same values for them, but we explicitly verify the assumption here. 110 | #if ( __STDC_VERSION__ >= 201112L) 111 | _Static_assert( (int)MUK_THREAD_FUNNELED == (int)MPI_THREAD_FUNNELED , "MUK_THREAD_FUNNELED == MPI_THREAD_FUNNELED " ); 112 | _Static_assert( (int)MUK_THREAD_MULTIPLE == (int)MPI_THREAD_MULTIPLE , "MUK_THREAD_MULTIPLE == MPI_THREAD_MULTIPLE " ); 113 | _Static_assert( (int)MUK_THREAD_SERIALIZED == (int)MPI_THREAD_SERIALIZED , "MUK_THREAD_SERIALIZED == MPI_THREAD_SERIALIZED" ); 114 | _Static_assert( (int)MUK_THREAD_SINGLE == (int)MPI_THREAD_SINGLE , "MUK_THREAD_SINGLE == MPI_THREAD_SINGLE " ); 115 | #endif 116 | 117 | // Threads Constants 118 | int IMPL_THREAD_FUNNELED = MPI_THREAD_FUNNELED; 119 | int IMPL_THREAD_MULTIPLE = MPI_THREAD_MULTIPLE; 120 | int IMPL_THREAD_SERIALIZED = MPI_THREAD_SERIALIZED; 121 | int IMPL_THREAD_SINGLE = MPI_THREAD_SINGLE; 122 | 123 | // File Operation Constants, Part 1 124 | int IMPL_DISPLACEMENT_CURRENT = MPI_DISPLACEMENT_CURRENT; 125 | 126 | // File Operation Constants, Part 2 127 | int IMPL_DISTRIBUTE_BLOCK = MPI_DISTRIBUTE_BLOCK; 128 | int IMPL_DISTRIBUTE_CYCLIC = MPI_DISTRIBUTE_CYCLIC; 129 | int IMPL_DISTRIBUTE_DFLT_DARG = MPI_DISTRIBUTE_DFLT_DARG; 130 | int IMPL_DISTRIBUTE_NONE = MPI_DISTRIBUTE_NONE; 131 | int IMPL_ORDER_C = MPI_ORDER_C; 132 | int IMPL_ORDER_FORTRAN = MPI_ORDER_FORTRAN; 133 | int IMPL_SEEK_CUR = MPI_SEEK_CUR; 134 | int IMPL_SEEK_END = MPI_SEEK_END; 135 | int IMPL_SEEK_SET = MPI_SEEK_SET; 136 | 137 | // F90 Datatype Matching Constants 138 | int IMPL_TYPECLASS_COMPLEX = MPI_TYPECLASS_COMPLEX; 139 | int IMPL_TYPECLASS_INTEGER = MPI_TYPECLASS_INTEGER; 140 | int IMPL_TYPECLASS_REAL = MPI_TYPECLASS_REAL; 141 | -------------------------------------------------------------------------------- /impl-scalar-types.h: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | #ifndef IMPL_SCALAR_TYPES_H 4 | #define IMPL_SCALAR_TYPES_H 5 | 6 | // This is terrible but compiles complain even though they are the same size 7 | #if defined(OPEN_MPI) 8 | typedef long int WRAP_Aint; 9 | typedef long long int WRAP_Count; 10 | typedef long long int WRAP_Offset; 11 | #elif defined(MPICH) 12 | typedef long int WRAP_Aint; 13 | #ifdef __MACH__ 14 | // Mac 15 | typedef long long int WRAP_Count; 16 | typedef long long int WRAP_Offset; 17 | #elif defined(I_MPI_VERSION) 18 | typedef long long int WRAP_Count; 19 | typedef long long int WRAP_Offset; 20 | #else 21 | // Linux 22 | typedef long long int WRAP_Count; 23 | typedef long long int WRAP_Offset; 24 | #endif 25 | #else 26 | #error NO ABI 27 | #endif 28 | 29 | #endif 30 | -------------------------------------------------------------------------------- /impl-session-functions.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include 12 | 13 | //#include "muk-dl.h" 14 | #include "muk-predefined.h" 15 | 16 | //#include "impl-scalar-types.h" 17 | //#include "impl-status.h" 18 | //#include "wrap-handle-typedefs.h" 19 | 20 | #define MUK_EXTERN extern 21 | #include "impl-fpointers.h" 22 | #include "impl-constant-conversions.h" 23 | #include "impl-handle-conversions.h" 24 | 25 | // WRAP->IMPL functions 26 | 27 | #if 0 28 | #if MPI_VERSION >= 4 29 | int WRAP_Session_call_errhandler(WRAP_Session session, int errorcode) 30 | { 31 | MPI_Session impl_session = CONVERT_MPI_Session(session); 32 | int rc = IMPL_Session_call_errhandler(impl_session, errorcode); 33 | return RETURN_CODE_IMPL_TO_MUK(rc); 34 | } 35 | 36 | #if MPI_VERSION >= 4 37 | int WRAP_Session_create_errhandler(WRAP_Session_errhandler_function *session_errhandler_fn, WRAP_Errhandler *errhandler) 38 | { 39 | MPI_Errhandler impl_errhandler; 40 | int rc = IMPL_Session_create_errhandler(session_errhandler_fn, &impl_errhandler); 41 | *errhandler = OUTPUT_MPI_Errhandler(impl_errhandler); 42 | return RETURN_CODE_IMPL_TO_MUK(rc); 43 | } 44 | #endif 45 | #endif 46 | 47 | int WRAP_Session_finalize(WRAP_Session *session) 48 | { 49 | MPI_Session impl_session = CONVERT_MPI_Session(*session); 50 | int rc = IMPL_Session_finalize(&impl_session); 51 | *session = OUTPUT_MPI_Session(impl_session); 52 | return RETURN_CODE_IMPL_TO_MUK(rc); 53 | } 54 | 55 | int WRAP_Session_get_errhandler(WRAP_Session session, WRAP_Errhandler *errhandler) 56 | { 57 | MPI_Session impl_session = CONVERT_MPI_Session(session); 58 | MPI_Errhandler impl_errhandler; 59 | int rc = IMPL_Session_get_errhandler(impl_session, &impl_errhandler); 60 | *errhandler = OUTPUT_MPI_Errhandler(impl_errhandler); 61 | return RETURN_CODE_IMPL_TO_MUK(rc); 62 | } 63 | 64 | int WRAP_Session_get_info(WRAP_Session session, WRAP_Info *info_used) 65 | { 66 | MPI_Session impl_session = CONVERT_MPI_Session(session); 67 | MPI_Info impl_info_used; 68 | int rc = IMPL_Session_get_info(impl_session, &impl_info_used); 69 | *info_used = OUTPUT_MPI_Info(impl_info_used); 70 | return RETURN_CODE_IMPL_TO_MUK(rc); 71 | } 72 | 73 | int WRAP_Session_get_nth_pset(WRAP_Session session, WRAP_Info info, int n, int *pset_len, char *pset_name) 74 | { 75 | MPI_Info impl_info = CONVERT_MPI_Info(info); 76 | MPI_Session impl_session = CONVERT_MPI_Session(session); 77 | int rc = IMPL_Session_get_nth_pset(impl_session, impl_info, n, pset_len, pset_name); 78 | return RETURN_CODE_IMPL_TO_MUK(rc); 79 | } 80 | 81 | int WRAP_Session_get_num_psets(WRAP_Session session, WRAP_Info info, int *npset_names) 82 | { 83 | MPI_Info impl_info = CONVERT_MPI_Info(info); 84 | MPI_Session impl_session = CONVERT_MPI_Session(session); 85 | int rc = IMPL_Session_get_num_psets(impl_session, impl_info, npset_names); 86 | return RETURN_CODE_IMPL_TO_MUK(rc); 87 | } 88 | 89 | int WRAP_Session_get_pset_info(WRAP_Session session, const char *pset_name, WRAP_Info *info) 90 | { 91 | MPI_Session impl_session = CONVERT_MPI_Session(session); 92 | MPI_Info impl_info; 93 | int rc = IMPL_Session_get_pset_info(impl_session, pset_name, &impl_info); 94 | *info = OUTPUT_MPI_Info(impl_info); 95 | return RETURN_CODE_IMPL_TO_MUK(rc); 96 | } 97 | 98 | int WRAP_Session_init(WRAP_Info info, WRAP_Errhandler errhandler, WRAP_Session *session) 99 | { 100 | MPI_Info impl_info = CONVERT_MPI_Info(info); 101 | MPI_Errhandler impl_errhandler = CONVERT_MPI_Errhandler(errhandler); 102 | MPI_Session impl_session = CONVERT_MPI_Session(*session); 103 | int rc = IMPL_Session_init(impl_info, impl_errhandler, &impl_session); 104 | *session = OUTPUT_MPI_Session(impl_session); 105 | return RETURN_CODE_IMPL_TO_MUK(rc); 106 | } 107 | 108 | int WRAP_Session_set_errhandler(WRAP_Session session, WRAP_Errhandler errhandler) 109 | { 110 | MPI_Session impl_session = CONVERT_MPI_Session(session); 111 | MPI_Errhandler impl_errhandler = CONVERT_MPI_Errhandler(errhandler); 112 | int rc = IMPL_Session_set_errhandler(impl_session, impl_errhandler); 113 | return RETURN_CODE_IMPL_TO_MUK(rc); 114 | } 115 | #endif 116 | 117 | -------------------------------------------------------------------------------- /impl-status.h: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | #ifndef IMPL_STATUS_H 4 | #define IMPL_STATUS_H 5 | 6 | #include 7 | 8 | #include "impl-constant-conversions.h" 9 | 10 | // status typedef 11 | typedef struct 12 | { 13 | int MPI_SOURCE; 14 | int MPI_TAG; 15 | int MPI_ERROR; 16 | int __kielletty__[5]; 17 | } 18 | WRAP_Status; 19 | 20 | // SUPPORT_F08 enables F08 stuff I am ignoring for now 21 | 22 | #ifdef SUPPORT_F08 23 | // status typedef 24 | typedef struct 25 | { 26 | MPI_Fint MPI_SOURCE; 27 | MPI_Fint MPI_TAG; 28 | MPI_Fint MPI_ERROR; 29 | MPI_Fint __kielletty__[5]; 30 | } 31 | WRAP_F08_status; 32 | #endif 33 | 34 | static inline bool IS_STATUS_IGNORE(const WRAP_Status * status) 35 | { 36 | return ((intptr_t)status == (intptr_t)MUK_STATUS_IGNORE); 37 | } 38 | 39 | static inline bool IS_STATUSES_IGNORE(const WRAP_Status * status) 40 | { 41 | return ((intptr_t)status == (intptr_t)MUK_STATUSES_IGNORE); 42 | } 43 | 44 | static inline void WRAP_Status_empty(const bool ignore, WRAP_Status * w) 45 | { 46 | if (ignore) return; 47 | 48 | // An empty status is a status which is set to return 49 | // tag = MPI_ANY_TAG 50 | // source = MPI_ANY_SOURCE 51 | // error = MPI_SUCCESS 52 | // and is also internally configured so that calls to 53 | // MPI_GET_COUNT, MPI_GET_ELEMENTS, and MPI_GET_ELEMENTS_X 54 | // return count = 0 and MPI_TEST_CANCELLED returns false. 55 | 56 | w->MPI_SOURCE = MUK_ANY_SOURCE; 57 | w->MPI_TAG = MUK_ANY_TAG; 58 | w->MPI_ERROR = MUK_SUCCESS; 59 | 60 | #if defined(MPICH) 61 | memset(w->__kielletty__, 0, 2*sizeof(int)); 62 | #elif defined(OPEN_MPI) 63 | memset(w->__kielletty__, 0, 3*sizeof(int)); 64 | #else 65 | #error Need MPI_Status ABI support 66 | #endif 67 | } 68 | 69 | static inline void WRAP_Status_to_MPI_Status(const WRAP_Status * w, MPI_Status * m) 70 | { 71 | if (IS_STATUSES_IGNORE(w)) { 72 | printf("WRAP_Status_to_MPI_Status passed STATUS_IGNORE\n"); 73 | return; 74 | } 75 | if (w == NULL || m == NULL) { 76 | printf("WRAP_Status_to_MPI_Status passed NULL (w=%p m=%p)\n",w,m); 77 | return; 78 | } 79 | 80 | memset(m,0,sizeof(MPI_Status)); 81 | 82 | //printf("WRAP_Status = { .SOURCE=%d .TAG=%d .ERROR=%d } \n", w->MPI_SOURCE, w->MPI_TAG, w->MPI_ERROR); 83 | 84 | m->MPI_SOURCE = RANK_MUK_TO_IMPL(w->MPI_SOURCE); 85 | m->MPI_TAG = TAG_MUK_TO_IMPL(w->MPI_TAG); 86 | m->MPI_ERROR = (w->MPI_ERROR) ? ERROR_CODE_MUK_TO_IMPL(w->MPI_ERROR) : 0; 87 | 88 | #if defined(MPICH) 89 | memcpy(&(m->count_lo), w->__kielletty__, 2*sizeof(int)); 90 | #elif defined(OPEN_MPI) 91 | memcpy(&(m->_cancelled), w->__kielletty__, 3*sizeof(int)); 92 | #else 93 | #error Need MPI_Status ABI support 94 | #endif 95 | } 96 | 97 | static inline void MPI_Status_to_WRAP_Status(const MPI_Status * m, WRAP_Status * w) 98 | { 99 | if (IS_STATUSES_IGNORE(w)) { 100 | printf("MPI_Status_to_WRAP_Status passed STATUS_IGNORE\n"); 101 | return; 102 | } 103 | if (w == NULL || m == NULL) { 104 | printf("MPI_Status_to_WRAP_Status passed NULL (m=%p w=%p)\n",m,w); 105 | return; 106 | } 107 | 108 | memset(w,0,sizeof(WRAP_Status)); 109 | 110 | w->MPI_SOURCE = RANK_IMPL_TO_MUK(m->MPI_SOURCE); 111 | w->MPI_TAG = TAG_IMPL_TO_MUK(m->MPI_TAG); 112 | w->MPI_ERROR = (m->MPI_ERROR) ? ERROR_CODE_IMPL_TO_MUK(m->MPI_ERROR) : 0; 113 | 114 | #if defined(MPICH) 115 | memcpy(w->__kielletty__, &(m->count_lo), 2*sizeof(int)); 116 | #elif defined(OPEN_MPI) 117 | memcpy(w->__kielletty__, &(m->_cancelled), 3*sizeof(int)); 118 | #else 119 | #error Need MPI_Status ABI support 120 | #endif 121 | } 122 | 123 | #endif 124 | -------------------------------------------------------------------------------- /mpi-constants.h: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | #ifndef MUK_MPI_CONSTANTS_H 4 | #define MUK_MPI_CONSTANTS_H 5 | 6 | #define MPI_VERSION 4 7 | #define MPI_SUBVERSION 0 8 | 9 | #define MPI_MAX_PROCESSOR_NAME 256 10 | #define MPI_MAX_LIBRARY_VERSION_STRING 8192 11 | #define MPI_MAX_ERROR_STRING 512 12 | #define MPI_MAX_DATAREP_STRING 128 13 | #define MPI_MAX_PORT_NAME 1024 14 | #define MPI_MAX_OBJECT_NAME 128 15 | #define MPI_MAX_STRINGTAG_LEN 256 16 | #define MPI_MAX_PSET_NAME_LEN 256 17 | #define MPI_MAX_INFO_KEY 255 18 | #define MPI_MAX_INFO_VAL 1024 19 | 20 | #endif 21 | -------------------------------------------------------------------------------- /mpi-fortran.h: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | #ifndef MUK_MPI_FORTRAN_H 4 | #define MUK_MPI_FORTRAN_H 5 | 6 | #define MPI_Status_c2f(c_status, f_status) 0 7 | #define MPI_Status_f2c(f_status, c_status) 0 8 | 9 | #define MPI_Comm_c2f(arg) 0 10 | #define MPI_Comm_f2c(arg) NULL 11 | #define MPI_Errhandler_c2f(arg) 0 12 | #define MPI_Errhandler_f2c(arg) NULL 13 | #define MPI_Type_c2f(arg) 0 14 | #define MPI_Type_f2c(arg) NULL 15 | #define MPI_File_c2f(arg) 0 16 | #define MPI_File_f2c(arg) NULL 17 | #define MPI_Group_c2f(arg) 0 18 | #define MPI_Group_f2c(arg) NULL 19 | #define MPI_Info_c2f(arg) 0 20 | #define MPI_Info_f2c(arg) NULL 21 | #define MPI_Message_c2f(arg) 0 22 | #define MPI_Message_f2c(arg) NULL 23 | #define MPI_Op_c2f(arg) 0 24 | #define MPI_Op_f2c(arg) NULL 25 | #define MPI_Request_c2f(arg) 0 26 | #define MPI_Request_f2c(arg) NULL 27 | #define MPI_Session_c2f(arg) 0 28 | #define MPI_Session_f2c(arg) NULL 29 | #define MPI_Win_c2f(arg) 0 30 | #define MPI_Win_f2c(arg) NULL 31 | 32 | #endif 33 | -------------------------------------------------------------------------------- /mpi-handle-typedefs.h: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | #ifndef MUK_MPI_HANDLE_TYPEDEFS_H 4 | #define MUK_MPI_HANDLE_TYPEDEFS_H 5 | 6 | #ifdef MUK_INTERNAL 7 | #include 8 | typedef union 9 | { 10 | void * p; 11 | int i; 12 | intptr_t ip; 13 | } 14 | MUK_Handle; 15 | typedef MUK_Handle MPI_Comm; 16 | typedef MUK_Handle MPI_Datatype; 17 | typedef MUK_Handle MPI_Errhandler; 18 | typedef MUK_Handle MPI_File; 19 | typedef MUK_Handle MPI_Group; 20 | typedef MUK_Handle MPI_Info; 21 | typedef MUK_Handle MPI_Message; 22 | typedef MUK_Handle MPI_Op; 23 | typedef MUK_Handle MPI_Request; 24 | typedef MUK_Handle MPI_Session; 25 | typedef MUK_Handle MPI_Win; 26 | #else 27 | typedef struct MPI_ABI_Comm * MPI_Comm; 28 | typedef struct MPI_ABI_Datatype * MPI_Datatype; 29 | typedef struct MPI_ABI_Errhandler * MPI_Errhandler; 30 | typedef struct MPI_ABI_File * MPI_File; 31 | typedef struct MPI_ABI_Group * MPI_Group; 32 | typedef struct MPI_ABI_Info * MPI_Info; 33 | typedef struct MPI_ABI_Message * MPI_Message; 34 | typedef struct MPI_ABI_Op * MPI_Op; 35 | typedef struct MPI_ABI_Request * MPI_Request; 36 | typedef struct MPI_ABI_Session * MPI_Session; 37 | typedef struct MPI_ABI_Win * MPI_Win; 38 | #endif 39 | 40 | #endif 41 | -------------------------------------------------------------------------------- /mpi-typedefs.h: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | #ifndef MUK_MPI_TYPEDEFS_H 4 | #define MUK_MPI_TYPEDEFS_H 5 | 6 | #include 7 | 8 | // basic typedefs 9 | typedef ptrdiff_t MPI_Aint; 10 | typedef ptrdiff_t MPI_Count; 11 | typedef ptrdiff_t MPI_Offset; 12 | 13 | // deal with this later during VAPAA integration 14 | typedef int MPI_Fint; 15 | 16 | // status typedef 17 | typedef struct 18 | { 19 | int MPI_SOURCE; 20 | int MPI_TAG; 21 | int MPI_ERROR; 22 | int __kielletty__[5]; 23 | } 24 | MPI_Status; 25 | 26 | // callback typedefs 27 | 28 | // reductions 29 | typedef void MPI_User_function(void *invec, void *inoutvec, int *len, MPI_Datatype *datatype); 30 | typedef void MPI_User_function_c(void *invec, void *inoutvec, MPI_Count *len, MPI_Datatype *datatype); 31 | 32 | // comm attr 33 | typedef int MPI_Comm_copy_attr_function(MPI_Comm oldcomm, int comm_keyval, void *extra_state, void *attribute_val_in, void *attribute_val_out, int *flag); 34 | typedef int MPI_Comm_delete_attr_function(MPI_Comm comm, int comm_keyval, void *attribute_val, void *extra_state); 35 | // these are deprecated or deleted 36 | typedef int MPI_Copy_function(MPI_Comm oldcomm, int keyval, void *extra_state, void *attribute_val_in, void *attribute_val_out, int *flag); 37 | typedef int MPI_Delete_function(MPI_Comm comm, int keyval, void *attribute_val, void *extra_state); 38 | typedef void MPI_Handler_function(MPI_Comm *comm, int *error_code,...); 39 | 40 | // win attr 41 | typedef int MPI_Win_copy_attr_function(MPI_Win oldwin, int win_keyval, void *extra_state, void *attribute_val_in, void *attribute_val_out, int *flag); 42 | typedef int MPI_Win_delete_attr_function(MPI_Win win, int win_keyval, void *attribute_val, void *extra_state); 43 | 44 | // type attr 45 | typedef int MPI_Type_copy_attr_function(MPI_Datatype oldtype, int type_keyval, void *extra_state, void *attribute_val_in, void *attribute_val_out, int *flag); 46 | typedef int MPI_Type_delete_attr_function(MPI_Datatype datatype, int type_keyval, void *attribute_val, void *extra_state); 47 | 48 | // errhandler 49 | typedef void MPI_Comm_errhandler_function(MPI_Comm *comm, int *error_code, ...); 50 | typedef void MPI_Win_errhandler_function(MPI_Win *win, int *error_code, ...); 51 | typedef void MPI_File_errhandler_function(MPI_File *file, int *error_code, ...); 52 | typedef void MPI_Session_errhandler_function(MPI_Session *session, int *error_code, ...); 53 | 54 | // grequest 55 | typedef int MPI_Grequest_query_function(void *extra_state, MPI_Status *status); 56 | typedef int MPI_Grequest_free_function(void *extra_state); 57 | typedef int MPI_Grequest_cancel_function(void *extra_state, int complete); 58 | 59 | // datarep 60 | typedef int MPI_Datarep_extent_function(MPI_Datatype datatype, MPI_Aint *extent, void *extra_state); 61 | typedef int MPI_Datarep_conversion_function(void *userbuf, MPI_Datatype datatype, int count, void *filebuf, MPI_Offset position, void *extra_state); 62 | typedef int MPI_Datarep_conversion_function_c(void *userbuf, MPI_Datatype datatype, MPI_Count count, void *filebuf, MPI_Offset position, void *extra_state); 63 | 64 | #endif 65 | -------------------------------------------------------------------------------- /mpi.h: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | #ifndef MUK_MPI_H 4 | #define MUK_MPI_H 5 | 6 | #if defined(__cplusplus) 7 | extern "C" { 8 | #endif 9 | 10 | #include "mpi-constants.h" 11 | #include "mpi-handle-typedefs.h" 12 | #include "mpi-typedefs.h" 13 | #include "mpi-predefined.h" 14 | #include "mpi-prototypes.h" 15 | #include "mpi-fortran.h" 16 | 17 | #if defined(__cplusplus) 18 | } 19 | #endif 20 | 21 | 22 | #endif 23 | -------------------------------------------------------------------------------- /muk-dl.h: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | #ifndef MUK_DL_H 4 | #define MUK_DL_H 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #include 14 | 15 | // this is for function symbols, or other symbols that are not optional. 16 | static inline void * MUK_DLSYM(void * restrict handle, const char * restrict symbol) 17 | { 18 | void * fp = dlsym(handle, symbol); 19 | if (fp == NULL) { 20 | //fprintf(stderr, "MUK_DLSYM: failed to find %s - %s\n", symbol, dlerror() ); 21 | //fflush(0); 22 | } 23 | return fp; 24 | } 25 | 26 | // this is for cases where symbols are optional, i.e. predefined handles, 27 | // where we provide the value to use where the symbol is not found. 28 | static inline void * MUK_DLSYM_OPT(void * restrict handle, const char * restrict symbol, void * fallback) 29 | { 30 | void * p = dlsym(handle, symbol); 31 | if (p == NULL) { 32 | p = fallback; 33 | } 34 | return p; 35 | } 36 | 37 | #endif 38 | -------------------------------------------------------------------------------- /muk-predefined.h: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | #ifndef MUK_PREDEFINED_H 4 | #define MUK_PREDEFINED_H 5 | 6 | // handle this better 7 | typedef struct MUK_ABI_Comm * MUK_Comm; 8 | typedef struct MUK_ABI_Datatype * MUK_Datatype; 9 | typedef struct MUK_ABI_Errhandler * MUK_Errhandler; 10 | typedef struct MUK_ABI_File * MUK_File; 11 | typedef struct MUK_ABI_Group * MUK_Group; 12 | typedef struct MUK_ABI_Info * MUK_Info; 13 | typedef struct MUK_ABI_Message * MUK_Message; 14 | typedef struct MUK_ABI_Op * MUK_Op; 15 | typedef struct MUK_ABI_Request * MUK_Request; 16 | typedef struct MUK_ABI_Session * MUK_Session; 17 | typedef struct MUK_ABI_Win * MUK_Win; 18 | 19 | // error codes 20 | enum { 21 | MUK_SUCCESS = 0, 22 | MUK_ERR_BUFFER = 1, 23 | MUK_ERR_COUNT = 2, 24 | MUK_ERR_TYPE = 3, 25 | MUK_ERR_TAG = 4, 26 | MUK_ERR_COMM = 5, 27 | MUK_ERR_RANK = 6, 28 | MUK_ERR_REQUEST = 7, 29 | MUK_ERR_ROOT = 8, 30 | MUK_ERR_GROUP = 9, 31 | MUK_ERR_OP = 10, 32 | MUK_ERR_TOPOLOGY = 11, 33 | MUK_ERR_DIMS = 12, 34 | MUK_ERR_ARG = 13, 35 | MUK_ERR_UNKNOWN = 14, 36 | MUK_ERR_TRUNCATE = 15, 37 | MUK_ERR_OTHER = 16, 38 | MUK_ERR_INTERN = 17, 39 | MUK_ERR_PENDING = 18, 40 | MUK_ERR_IN_STATUS = 19, 41 | MUK_ERR_ACCESS = 20, 42 | MUK_ERR_AMODE = 21, 43 | MUK_ERR_ASSERT = 22, 44 | MUK_ERR_BAD_FILE = 23, 45 | MUK_ERR_BASE = 24, 46 | MUK_ERR_CONVERSION = 25, 47 | MUK_ERR_DISP = 26, 48 | MUK_ERR_DUP_DATAREP = 27, 49 | MUK_ERR_FILE_EXISTS = 28, 50 | MUK_ERR_FILE_IN_USE = 29, 51 | MUK_ERR_FILE = 30, 52 | MUK_ERR_INFO_KEY = 31, 53 | MUK_ERR_INFO_NOKEY = 32, 54 | MUK_ERR_INFO_VALUE = 33, 55 | MUK_ERR_INFO = 34, 56 | MUK_ERR_IO = 35, 57 | MUK_ERR_KEYVAL = 36, 58 | MUK_ERR_LOCKTYPE = 37, 59 | MUK_ERR_NAME = 38, 60 | MUK_ERR_NO_MEM = 39, 61 | MUK_ERR_NOT_SAME = 40, 62 | MUK_ERR_NO_SPACE = 41, 63 | MUK_ERR_NO_SUCH_FILE = 42, 64 | MUK_ERR_PORT = 43, 65 | MUK_ERR_PROC_ABORTED = 44, 66 | MUK_ERR_QUOTA = 45, 67 | MUK_ERR_READ_ONLY = 46, 68 | MUK_ERR_RMA_ATTACH = 47, 69 | MUK_ERR_RMA_CONFLICT = 48, 70 | MUK_ERR_RMA_RANGE = 49, 71 | MUK_ERR_RMA_SHARED = 50, 72 | MUK_ERR_RMA_SYNC = 51, 73 | MUK_ERR_RMA_FLAVOR = 52, 74 | MUK_ERR_SERVICE = 53, 75 | MUK_ERR_SESSION = 54, 76 | MUK_ERR_SIZE = 55, 77 | MUK_ERR_SPAWN = 56, 78 | MUK_ERR_UNSUPPORTED_DATAREP = 57, 79 | MUK_ERR_UNSUPPORTED_OPERATION = 58, 80 | MUK_ERR_VALUE_TOO_LARGE = 59, 81 | MUK_ERR_WIN = 60, 82 | MUK_T_ERR_CANNOT_INIT = 61, 83 | MUK_T_ERR_NOT_ACCESSIBLE = 62, 84 | MUK_T_ERR_NOT_INITIALIZED = 63, 85 | MUK_T_ERR_NOT_SUPPORTED = 64, 86 | MUK_T_ERR_MEMORY = 65, 87 | MUK_T_ERR_INVALID = 66, 88 | MUK_T_ERR_INVALID_INDEX = 67, 89 | MUK_T_ERR_INVALID_ITEM = 68, 90 | MUK_T_ERR_INVALID_SESSION = 69, 91 | MUK_T_ERR_INVALID_HANDLE = 70, 92 | MUK_T_ERR_INVALID_NAME = 71, 93 | MUK_T_ERR_OUT_OF_HANDLES = 72, 94 | MUK_T_ERR_OUT_OF_SESSIONS = 73, 95 | MUK_T_ERR_CVAR_SET_NOT_NOW = 74, 96 | MUK_T_ERR_CVAR_SET_NEVER = 75, 97 | MUK_T_ERR_PVAR_NO_WRITE = 76, 98 | MUK_T_ERR_PVAR_NO_STARTSTOP = 77, 99 | MUK_T_ERR_PVAR_NO_ATOMIC = 78, 100 | MUK_ERR_LASTCODE = 79 101 | }; 102 | 103 | // Buffer Address Constants 104 | 105 | // Assorted Constants 106 | enum { 107 | // these should be different and never valid ranks 108 | // MUK and OMPI are the same. MPICH in parentheses. 109 | MUK_ANY_SOURCE = -1, // (-2) 110 | MUK_PROC_NULL = -2, // (-1) 111 | MUK_ROOT = -4 // (-3) 112 | }; 113 | enum { 114 | // MUK, MPICH and OMPI agree on this 115 | MUK_ANY_TAG = -1 116 | }; 117 | enum { 118 | // MUK, MPICH and OMPI agree on this 119 | MUK_UNDEFINED = -32766 120 | }; 121 | enum { 122 | MUK_BSEND_OVERHEAD = 128 123 | }; 124 | enum { 125 | MUK_KEYVAL_INVALID = -7 126 | }; 127 | enum { 128 | MUK_LOCK_EXCLUSIVE = -8, 129 | MUK_LOCK_SHARED = -9 130 | }; 131 | 132 | #if FORTRAN 133 | //Fortran status array size and reserved index values (C only) 134 | enum { 135 | // 256-bit Status is proposed 136 | MUK_F_STATUS_SIZE = 8, 137 | // the public fields should come first 138 | MUK_F_SOURCE = 0, 139 | MUK_F_TAG = 1, 140 | MUK_F_ERROR = 2, 141 | // ignore Fortran for now 142 | MUK_ADDRESS_KIND = -1, 143 | MUK_COUNT_KIND = -1, 144 | MUK_INTEGER_KIND = -1, 145 | MUK_OFFSET_KIND = -1 146 | }; 147 | #endif 148 | 149 | // Error-handling specifiers 150 | #define MUK_ERRHANDLER_NULL (MUK_Errhandler)0 151 | #define MUK_ERRORS_ARE_FATAL (MUK_Errhandler)1 152 | #define MUK_ERRORS_RETURN (MUK_Errhandler) 2 153 | #define MUK_ERRORS_ABORT (MUK_Errhandler) 3 154 | 155 | // Named Predefined Datatypes 156 | #define MUK_DATATYPE_NULL ((MUK_Datatype)0) 157 | #define MUK_CHAR ((MUK_Datatype)1) 158 | #define MUK_SHORT ((MUK_Datatype)2) 159 | #define MUK_INT ((MUK_Datatype)3) 160 | #define MUK_LONG ((MUK_Datatype)4) 161 | #define MUK_LONG_LONG_INT ((MUK_Datatype)5) 162 | #define MUK_LONG_LONG ((MUK_Datatype)6) 163 | #define MUK_SIGNED_CHAR ((MUK_Datatype)7) 164 | #define MUK_UNSIGNED_CHAR ((MUK_Datatype)8) 165 | #define MUK_UNSIGNED_SHORT ((MUK_Datatype)9) 166 | #define MUK_UNSIGNED ((MUK_Datatype)10) 167 | #define MUK_UNSIGNED_LONG ((MUK_Datatype)11) 168 | #define MUK_UNSIGNED_LONG_LONG ((MUK_Datatype)12) 169 | #define MUK_FLOAT ((MUK_Datatype)13) 170 | #define MUK_DOUBLE ((MUK_Datatype)14) 171 | #define MUK_LONG_DOUBLE ((MUK_Datatype)15) 172 | #define MUK_WCHAR ((MUK_Datatype)16) 173 | #define MUK_C_BOOL ((MUK_Datatype)17) 174 | #define MUK_INT8_T ((MUK_Datatype)18) 175 | #define MUK_INT16_T ((MUK_Datatype)19) 176 | #define MUK_INT32_T ((MUK_Datatype)20) 177 | #define MUK_INT64_T ((MUK_Datatype)21) 178 | #define MUK_UINT8_T ((MUK_Datatype)22) 179 | #define MUK_UINT16_T ((MUK_Datatype)23) 180 | #define MUK_UINT32_T ((MUK_Datatype)24) 181 | #define MUK_UINT64_T ((MUK_Datatype)25) 182 | #define MUK_AINT ((MUK_Datatype)26) 183 | #define MUK_COUNT ((MUK_Datatype)27) 184 | #define MUK_OFFSET ((MUK_Datatype)28) 185 | #define MUK_C_COMPLEX ((MUK_Datatype)29) 186 | #define MUK_C_FLOAT_COMPLEX ((MUK_Datatype)30) 187 | #define MUK_C_DOUBLE_COMPLEX ((MUK_Datatype)31) 188 | #define MUK_C_LONG_DOUBLE_COMPLEX ((MUK_Datatype)32) 189 | #define MUK_BYTE ((MUK_Datatype)33) 190 | #define MUK_PACKED ((MUK_Datatype)34) 191 | #define MUK_CXX_BOOL ((MUK_Datatype)35) 192 | #define MUK_CXX_FLOAT_COMPLEX ((MUK_Datatype)36) 193 | #define MUK_CXX_DOUBLE_COMPLEX ((MUK_Datatype)37) 194 | #define MUK_CXX_LONG_DOUBLE_COMPLEX ((MUK_Datatype)38) 195 | #define MUK_INTEGER ((MUK_Datatype)39) 196 | #define MUK_REAL ((MUK_Datatype)40) 197 | #define MUK_DOUBLE_PRECISION ((MUK_Datatype)41) 198 | #define MUK_COMPLEX ((MUK_Datatype)42) 199 | #define MUK_LOGICAL ((MUK_Datatype)43) 200 | #define MUK_CHARACTER ((MUK_Datatype)44) 201 | #define MUK_DOUBLE_COMPLEX ((MUK_Datatype)45) 202 | #define MUK_INTEGER1 ((MUK_Datatype)46) 203 | #define MUK_INTEGER2 ((MUK_Datatype)47) 204 | #define MUK_INTEGER4 ((MUK_Datatype)48) 205 | #define MUK_INTEGER8 ((MUK_Datatype)49) 206 | #define MUK_INTEGER16 ((MUK_Datatype)50) 207 | #define MUK_REAL2 ((MUK_Datatype)51) 208 | #define MUK_REAL4 ((MUK_Datatype)52) 209 | #define MUK_REAL8 ((MUK_Datatype)53) 210 | #define MUK_REAL16 ((MUK_Datatype)54) 211 | #define MUK_COMPLEX4 ((MUK_Datatype)55) 212 | #define MUK_COMPLEX8 ((MUK_Datatype)56) 213 | #define MUK_COMPLEX16 ((MUK_Datatype)57) 214 | #define MUK_COMPLEX32 ((MUK_Datatype)58) 215 | #define MUK_FLOAT_INT ((MUK_Datatype)59) 216 | #define MUK_DOUBLE_INT ((MUK_Datatype)60) 217 | #define MUK_LONG_INT ((MUK_Datatype)61) 218 | #define MUK_2INT ((MUK_Datatype)62) 219 | #define MUK_SHORT_INT ((MUK_Datatype)63) 220 | #define MUK_LONG_DOUBLE_INT ((MUK_Datatype)64) 221 | #define MUK_2REAL ((MUK_Datatype)65) 222 | #define MUK_2DOUBLE_PRECISION ((MUK_Datatype)66) 223 | #define MUK_2INTEGER ((MUK_Datatype)67) 224 | #define MUK_LB ((MUK_Datatype)68) 225 | #define MUK_UB ((MUK_Datatype)69) 226 | 227 | // Reserved communicators 228 | #define MUK_COMM_NULL ((MUK_Comm)0) 229 | #define MUK_COMM_SELF ((MUK_Comm)1) 230 | #define MUK_COMM_WORLD ((MUK_Comm)2) 231 | 232 | // Reserved groups 233 | #define MUK_GROUP_NULL ((MUK_Group)0) 234 | #define MUK_GROUP_EMPTY ((MUK_Group)1) 235 | 236 | // Communicator split type constants 237 | enum { 238 | MUK_COMM_TYPE_SHARED = -100, 239 | MUK_COMM_TYPE_HW_UNGUIDED = -101, 240 | MUK_COMM_TYPE_HW_GUIDED = -102 241 | }; 242 | 243 | // Results of communicator and group comparisons 244 | enum { 245 | // MPICH and OMPI agree on this 246 | MUK_IDENT = 0, 247 | MUK_CONGRUENT = 1, 248 | MUK_SIMILAR = 2, 249 | MUK_UNEQUAL = 3 250 | }; 251 | 252 | // Environmental inquiry info key 253 | #define MUK_INFO_NULL (MUK_Info)0 254 | #define MUK_INFO_ENV (MUK_Info)1 255 | 256 | // Collective Operations 257 | #define MUK_OP_NULL ((MUK_Op)0) 258 | #define MUK_MAX ((MUK_Op)1) 259 | #define MUK_MIN ((MUK_Op)2) 260 | #define MUK_SUM ((MUK_Op)3) 261 | #define MUK_PROD ((MUK_Op)4) 262 | #define MUK_LAND ((MUK_Op)5) 263 | #define MUK_BAND ((MUK_Op)6) 264 | #define MUK_LOR ((MUK_Op)7) 265 | #define MUK_BOR ((MUK_Op)8) 266 | #define MUK_LXOR ((MUK_Op)9) 267 | #define MUK_BXOR ((MUK_Op)10) 268 | #define MUK_MAXLOC ((MUK_Op)11) 269 | #define MUK_MINLOC ((MUK_Op)12) 270 | #define MUK_REPLACE ((MUK_Op)13) 271 | #define MUK_NO_OP ((MUK_Op)14) 272 | 273 | // Predefined message handles 274 | #define MUK_MESSAGE_NULL (MUK_Message)0 275 | #define MUK_MESSAGE_NO_PROC ((MUK_Message)1) 276 | 277 | // Null Handles 278 | #define MUK_REQUEST_NULL (MUK_Request)0 279 | #define MUK_FILE_NULL (MUK_File)0 280 | #define MUK_WIN_NULL (MUK_Win)0 281 | #if 1 //MUK_VERSION >= 4 282 | #define MUK_SESSION_NULL (MUK_Session)0 283 | #endif 284 | 285 | // Topologies 286 | enum { 287 | MUK_GRAPH = -200, 288 | MUK_CART = -201, 289 | MUK_DIST_GRAPH = -202 290 | }; 291 | 292 | enum { 293 | // Environmental inquiry keys 294 | MUK_TAG_UB, 295 | MUK_IO, 296 | MUK_HOST, 297 | MUK_WTIME_IS_GLOBAL, 298 | // Predefined Attribute Keys 299 | MUK_APPNUM, 300 | MUK_LASTUSEDCODE, 301 | MUK_UNIVERSE_SIZE, 302 | MUK_WIN_BASE, 303 | MUK_WIN_DISP_UNIT, 304 | MUK_WIN_SIZE, 305 | MUK_WIN_CREATE_FLAVOR, 306 | MUK_WIN_MODEL 307 | }; 308 | 309 | // MPI Window Create Flavors 310 | enum { 311 | // MPICH and OMPI agree on this 312 | MUK_WIN_FLAVOR_CREATE = 1, 313 | MUK_WIN_FLAVOR_ALLOCATE = 2, 314 | MUK_WIN_FLAVOR_DYNAMIC = 3, 315 | MUK_WIN_FLAVOR_SHARED = 4 316 | }; 317 | 318 | // MPI Window Models 319 | enum { 320 | MUK_WIN_SEPARATE = -300, 321 | MUK_WIN_UNIFIED = -301 322 | }; 323 | 324 | // Mode Constants 325 | // The values of these constants must be defined such that the bitwise OR 326 | // and the sum of any distinct set of these constants is equivalent. 327 | enum { 328 | // none of these should be 0 329 | // I/O constants 330 | MUK_MODE_APPEND = 1<<1, 331 | MUK_MODE_CREATE = 1<<2, 332 | MUK_MODE_DELETE_ON_CLOSE = 1<<3, 333 | MUK_MODE_EXCL = 1<<4, 334 | MUK_MODE_RDONLY = 1<<5, 335 | MUK_MODE_RDWR = 1<<6, 336 | MUK_MODE_SEQUENTIAL = 1<<7, 337 | MUK_MODE_UNIQUE_OPEN = 1<<8, 338 | MUK_MODE_WRONLY = 1<<9, 339 | // RMA constants 340 | MUK_MODE_NOCHECK = 1<<15, 341 | MUK_MODE_NOPRECEDE = 1<<16, 342 | MUK_MODE_NOPUT = 1<<17, 343 | MUK_MODE_NOSTORE = 1<<18, 344 | MUK_MODE_NOSUCCEED = 1<<19 345 | }; 346 | 347 | // Datatype Decoding Constants 348 | enum { 349 | MUK_COMBINER_CONTIGUOUS, 350 | MUK_COMBINER_DARRAY, 351 | MUK_COMBINER_DUP, 352 | MUK_COMBINER_F90_COMPLEX, 353 | MUK_COMBINER_F90_INTEGER, 354 | MUK_COMBINER_F90_REAL, 355 | MUK_COMBINER_HINDEXED, 356 | MUK_COMBINER_HVECTOR, 357 | MUK_COMBINER_INDEXED_BLOCK, 358 | MUK_COMBINER_HINDEXED_BLOCK, 359 | MUK_COMBINER_INDEXED, 360 | MUK_COMBINER_NAMED, 361 | MUK_COMBINER_RESIZED, 362 | MUK_COMBINER_STRUCT, 363 | MUK_COMBINER_SUBARRAY, 364 | MUK_COMBINER_VECTOR 365 | }; 366 | 367 | // Threads Constants 368 | enum { 369 | // MPICH and OMPI agree on this 370 | MUK_THREAD_SINGLE = 0, 371 | MUK_THREAD_FUNNELED = 1, 372 | MUK_THREAD_SERIALIZED = 2, 373 | MUK_THREAD_MULTIPLE = 3 374 | }; 375 | 376 | enum { 377 | // File Operation Constants, Part 1 378 | MUK_DISPLACEMENT_CURRENT = -2, 379 | // File Operation Constants, Part 2 380 | // use MPICH values 381 | MUK_DISTRIBUTE_BLOCK = 121, 382 | MUK_DISTRIBUTE_CYCLIC = 122, 383 | MUK_DISTRIBUTE_NONE = 123, 384 | // This one has to be negative 385 | MUK_DISTRIBUTE_DFLT_DARG = -1, 386 | // Values are descriptive 387 | MUK_ORDER_C = 4321, 388 | MUK_ORDER_FORTRAN = 1234, 389 | // use OMPI values 390 | MUK_SEEK_CUR = 600, 391 | MUK_SEEK_END = 602, 392 | MUK_SEEK_SET = 604 393 | }; 394 | 395 | // F90 Datatype Matching Constants 396 | enum { 397 | MUK_TYPECLASS_COMPLEX, 398 | MUK_TYPECLASS_INTEGER, 399 | MUK_TYPECLASS_REAL 400 | }; 401 | 402 | // Buffer Address Constants 403 | // MPICH, OMPI and MUK agree 404 | #define MUK_BOTTOM ((void *)0) 405 | // OMPI and MUK agree. MPICH is -1 406 | #define MUK_IN_PLACE ((void *)1) 407 | 408 | // Constants Specifying Empty or Ignored Input 409 | // These match MPICH and OMPI 410 | #define MUK_ARGV_NULL ((char**)0) 411 | #define MUK_ARGVS_NULL ((char***)0) 412 | #define MUK_ERRCODES_IGNORE ((int*)0) 413 | 414 | // These match OMPI (MPICH sets to 1) 415 | #define MUK_STATUSES_IGNORE ((void*)0) 416 | #define MUK_STATUS_IGNORE ((void*)0) 417 | 418 | // These match OMPI (MPICH sets to extern-something) 419 | #define MUK_UNWEIGHTED ((int*)2) 420 | #define MUK_WEIGHTS_EMPTY ((int*)3) 421 | 422 | #include 423 | 424 | // basic typedefs 425 | typedef ptrdiff_t MUK_Aint; 426 | typedef ptrdiff_t MUK_Count; 427 | typedef ptrdiff_t MUK_Offset; 428 | 429 | // status typedef 430 | typedef struct 431 | { 432 | int MPI_SOURCE; 433 | int MPI_TAG; 434 | int MPI_ERROR; 435 | int __kielletty__[5]; 436 | } 437 | MUK_Status; 438 | 439 | typedef void MUK_User_function(void *invec, void *inoutvec, int *len, MUK_Datatype *datatype); 440 | typedef void MUK_User_function_c(void *invec, void *inoutvec, MUK_Count *len, MUK_Datatype *datatype); 441 | typedef int MUK_Comm_copy_attr_function(MUK_Comm oldcomm, int comm_keyval, void *extra_state, void *attribute_val_in, void *attribute_val_out, int *flag); 442 | typedef int MUK_Comm_delete_attr_function(MUK_Comm comm, int comm_keyval, void *attribute_val, void *extra_state); 443 | typedef int MUK_Win_copy_attr_function(MUK_Win oldwin, int win_keyval, void *extra_state, void *attribute_val_in, void *attribute_val_out, int *flag); 444 | typedef int MUK_Win_delete_attr_function(MUK_Win win, int win_keyval, void *attribute_val, void *extra_state); 445 | typedef int MUK_Type_copy_attr_function(MUK_Datatype oldtype, int type_keyval, void *extra_state, void *attribute_val_in, void *attribute_val_out, int *flag); 446 | typedef int MUK_Type_delete_attr_function(MUK_Datatype datatype, int type_keyval, void *attribute_val, void *extra_state); 447 | typedef void MUK_Comm_errhandler_function(MUK_Comm *comm, int *error_code, ...); 448 | typedef void MUK_Win_errhandler_function(MUK_Win *win, int *error_code, ...); 449 | typedef void MUK_File_errhandler_function(MUK_File *file, int *error_code, ...); 450 | typedef void MUK_Session_errhandler_function(MUK_Session *session, int *error_code, ...); 451 | typedef int MUK_Grequest_query_function(void *extra_state, MUK_Status *status); 452 | typedef int MUK_Grequest_free_function(void *extra_state); 453 | typedef int MUK_Grequest_cancel_function(void *extra_state, int complete); 454 | typedef int MUK_Datarep_extent_function(MUK_Datatype datatype, MUK_Aint *extent, void *extra_state); 455 | typedef int MUK_Datarep_conversion_function(void *userbuf, MUK_Datatype datatype, int count, void *filebuf, MUK_Offset position, void *extra_state); 456 | typedef int MUK_Datarep_conversion_function_c(void *userbuf, MUK_Datatype datatype, MUK_Count count, void *filebuf, MUK_Offset position, void *extra_state); 457 | 458 | // these are deprecated 459 | typedef int MUK_Copy_function(MUK_Comm oldcomm, int keyval, void *extra_state, void *attribute_val_in, void *attribute_val_out, int *flag); 460 | typedef int MUK_Delete_function(MUK_Comm comm, int keyval, void *attribute_val, void *extra_state); 461 | 462 | #define MUK_NULL_COPY_FN ((MUK_Copy_function*)NULL) 463 | #define MUK_DUP_FN ((MUK_Copy_function*)0x1) 464 | #define MUK_NULL_DELETE_FN ((MUK_Delete_function*)NULL) 465 | #define MUK_COMM_NULL_COPY_FN ((MUK_Comm_copy_attr_function*)NULL) 466 | #define MUK_COMM_DUP_FN ((MUK_Comm_copy_attr_function*)0x1) 467 | #define MUK_COMM_NULL_DELETE_FN ((MUK_Comm_delete_attr_function*)NULL) 468 | #define MUK_TYPE_NULL_COPY_FN ((MUK_Type_copy_attr_function*)NULL) 469 | #define MUK_TYPE_DUP_FN ((MUK_Type_copy_attr_function*)0x1) 470 | #define MUK_TYPE_NULL_DELETE_FN ((MUK_Type_delete_attr_function*)NULL) 471 | #define MUK_WIN_NULL_COPY_FN ((MUK_Win_copy_attr_function*)NULL) 472 | #define MUK_WIN_DUP_FN ((MUK_Win_copy_attr_function*)0x1) 473 | #define MUK_WIN_NULL_DELETE_FN ((MUK_Win_delete_attr_function*)NULL) 474 | 475 | #endif 476 | -------------------------------------------------------------------------------- /muk.h: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | #ifndef MUK_H 4 | #define MUK_H 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include 12 | 13 | typedef enum { 14 | MPICH = 1, 15 | OMPI = 2, 16 | INTEL = 3, 17 | UNKNOWN = 4 18 | } Which_MPI_e; 19 | 20 | extern Which_MPI_e whose_mpi; 21 | 22 | int (*MUK_Load_functions)(void * restrict h, int major, int minor); 23 | int (*MUK_Load_predefined)(void * restrict h); 24 | 25 | #endif 26 | -------------------------------------------------------------------------------- /test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | 5 | if [ $(uname -s) == Darwin ]; then 6 | brew_prefix=$(brew --prefix) 7 | OMPIRUN=$(ls $brew_prefix/Cellar/open-mpi/*/bin/mpirun) 8 | OMPILIB=$(ls $brew_prefix/Cellar/open-mpi/*/lib/libmpi.dylib) 9 | MPICHRUN=$(ls $brew_prefix/Cellar/mpich/*/bin/mpirun) 10 | MPICHLIB=$(ls $brew_prefix/Cellar/mpich/*/lib/libmpi.dylib) 11 | IMPIRUN=true 12 | IMPILIB= 13 | DBG=lldb 14 | DBGARGS="--one-line 'run' --one-line-on-crash 'bt' --one-line 'quit' --" 15 | else 16 | OSID=$(grep '^ID=' /etc/os-release | cut -d= -f2) 17 | export HWLOC_COMPONENTS=-gl 18 | if [ "$OSID" == "ubuntu" ]; then 19 | OMPIRUN="/usr/bin/mpirun.openmpi --tag-output" 20 | OMPILIB=/usr/lib/x86_64-linux-gnu/libmpi.so 21 | MPICHRUN="/usr/bin/mpirun.mpich -l" 22 | MPICHLIB=/usr/lib/x86_64-linux-gnu/libmpich.so 23 | fi 24 | if [ "$OSID" == "fedora" ]; then 25 | OMPIRUN="/usr/lib64/openmpi/bin/mpirun --tag-output" 26 | OMPILIB=/usr/lib64/openmpi/lib/libmpi.so 27 | MPICHRUN="/usr/lib64/mpich/bin/mpirun -l" 28 | MPICHLIB=/usr/lib64/mpich/lib/libmpi.so 29 | fi 30 | if [ -e /opt/intel/oneapi/mpi/latest ]; then 31 | IMPIRUN="/opt/intel/oneapi/mpi/latest/bin/mpirun -l" 32 | IMPILIB=/opt/intel/oneapi/mpi/latest/lib/debug/libmpi.so 33 | #IMPILIB=/opt/intel/oneapi/mpi/latest/lib/release/libmpi.so 34 | fi 35 | #OPTS="--mca osc ucx" 36 | #OMPIRUN="/usr/bin/mpirun.openmpi --tag-output" 37 | #OMPILIB=/usr/lib/x86_64-linux-gnu/libmpi.so 38 | #OMPIRUN=/opt/ompi/gcc-debug/bin/mpirun 39 | #OMPILIB=/opt/ompi/gcc-debug/lib/libmpi.so 40 | #MPICHRUN="/usr/bin/mpirun.mpich -l" 41 | #MPICHLIB=/usr/lib/x86_64-linux-gnu/libmpich.so 42 | #MPICHRUN="/opt/mpich/gcc/debug-ch4ucx/bin/mpirun -l" 43 | #MPICHLIB=/opt/mpich/gcc/debug-ch4ucx/lib/libmpi.so 44 | #IMPIRUN="/opt/intel/oneapi/mpi/2021.8.0/bin/mpirun -l" 45 | #IMPILIB=/opt/intel/oneapi/mpi/2021.8.0/lib/debug/libmpi.so 46 | #IMPILIB=/opt/intel/oneapi/mpi/2021.8.0/lib/release/libmpi.so 47 | DBG=gdb 48 | DBGARGS='-ex "set width 1000" -ex "thread apply all bt" -ex run -ex bt -ex "set confirm off" -ex quit --args' 49 | fi 50 | 51 | if [ "$GITHUB_ACTIONS" == "true" ]; then 52 | DBG=false 53 | DBGARGS= 54 | fi 55 | 56 | NP=2 57 | #OPTS="${OPTS} -quiet" 58 | 59 | make -j $1 60 | 61 | if [ -n "${OMPILIB}" ]; then 62 | MUK_MPI_LIB=${OMPILIB} ${OMPIRUN} ${OPTS} -n ${NP} $1 || \ 63 | MUK_MPI_LIB=${OMPILIB} ${OMPIRUN} ${OPTS} -n ${NP} ${DBG} ${DBGARGS} $1 ; \ 64 | fi 65 | 66 | if [ -n "${MPICHLIB}" ]; then 67 | MUK_MPI_LIB=${MPICHLIB} ${MPICHRUN} -n ${NP} $1 || \ 68 | MUK_MPI_LIB=${MPICHLIB} ${MPICHRUN} -n ${NP} ${DBG} ${DBGARGS} $1 69 | fi 70 | 71 | if [ -n "${IMPILIB}" ]; then 72 | MUK_MPI_LIB=${IMPILIB} ${IMPIRUN} -n ${NP} $1 || \ 73 | MUK_MPI_LIB=${IMPILIB} ${IMPIRUN} -n ${NP} ${DBG} ${DBGARGS} $1 74 | fi 75 | -------------------------------------------------------------------------------- /testbottom.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | 10 | #ifdef NOWRAP 11 | #include 12 | #else 13 | #include "mpi.h" 14 | #endif 15 | 16 | int main(int argc, char* argv[]) 17 | { 18 | int rc; 19 | rc = MPI_Init(&argc,&argv); 20 | 21 | int me, np; 22 | MPI_Comm_rank(MPI_COMM_WORLD,&me); 23 | MPI_Comm_size(MPI_COMM_WORLD,&np); 24 | printf("I am %d of %d\n", me, np); 25 | fflush(0); 26 | MPI_Barrier(MPI_COMM_WORLD); 27 | usleep(1); 28 | 29 | if (np < 2) { 30 | printf("this test requires np > 1\n"); 31 | MPI_Abort(MPI_COMM_WORLD,np); 32 | } 33 | 34 | // Example 5.22 An elaborate example. 35 | { 36 | int position, i = 200; 37 | float a[200]; 38 | char buff[1000]; /* larger than or equal to the size returned from MPI_PACK_SIZE for 1,newtype */ 39 | if (me == 0) 40 | { 41 | /* SENDER CODE */ 42 | int len[2]; 43 | MPI_Aint disp[2]; 44 | MPI_Datatype type[2], newtype; 45 | 46 | /* build datatype for i followed by a[0]...a[i-1] */ 47 | len[0] = 1; 48 | len[1] = i; 49 | MPI_Get_address(&i, disp); 50 | MPI_Get_address(a, disp+1); 51 | 52 | type[0] = MPI_INT; 53 | type[1] = MPI_FLOAT; 54 | MPI_Type_create_struct(2, len, disp, type, &newtype); 55 | MPI_Type_commit(&newtype); 56 | 57 | /* Pack i followed by a[0]...a[i-1]*/ 58 | position = 0; 59 | MPI_Pack(MPI_BOTTOM, 1, newtype, buff, 1000, &position, MPI_COMM_WORLD); 60 | 61 | /* Send */ 62 | MPI_Send(buff, position, MPI_PACKED, 1, 0, MPI_COMM_WORLD); 63 | // One can replace the last three lines with 64 | // MPI_Send(MPI_BOTTOM, 1, newtype, 1, 0, MPI_COMM_WORLD); 65 | 66 | MPI_Type_free(&newtype); 67 | } 68 | else if (me == 1) 69 | { 70 | /* RECEIVER CODE */ 71 | MPI_Status status; 72 | /* Receive */ 73 | MPI_Recv(buff, 1000, MPI_PACKED, 0, 0, MPI_COMM_WORLD, &status); 74 | /* Unpack i */ 75 | position = 0; 76 | MPI_Unpack(buff, 1000, &position, &i, 1, MPI_INT, MPI_COMM_WORLD); 77 | /* Unpack a[0]...a[i-1] */ 78 | MPI_Unpack(buff, 1000, &position, a, i, MPI_FLOAT, MPI_COMM_WORLD); 79 | } 80 | } 81 | 82 | fflush(0); 83 | usleep(1); 84 | MPI_Barrier(MPI_COMM_WORLD); 85 | if (me==0) printf("all done\n"); 86 | 87 | rc = MPI_Finalize(); 88 | 89 | return rc; 90 | } 91 | -------------------------------------------------------------------------------- /testcart.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | 10 | #ifdef NOWRAP 11 | #include 12 | #else 13 | #include "mpi.h" 14 | #endif 15 | 16 | int main(int argc, char* argv[]) 17 | { 18 | int rc; 19 | rc = MPI_Init(&argc,&argv); 20 | 21 | 22 | int me, np; 23 | MPI_Comm_rank(MPI_COMM_WORLD,&me); 24 | MPI_Comm_size(MPI_COMM_WORLD,&np); 25 | printf("I am %d of %d\n", me, np); 26 | 27 | { 28 | MPI_Comm cart_world; 29 | int dims[1] = { np }; 30 | int periodic[1] = { 0 }; 31 | int reorder = 0; 32 | 33 | MPI_Cart_create(MPI_COMM_WORLD, 1, dims, periodic, reorder, &cart_world); 34 | 35 | int result; 36 | MPI_Comm_compare(MPI_COMM_WORLD,cart_world,&result); 37 | if (result != MPI_CONGRUENT) { 38 | printf("cartesian world is not congruent: %d\n", result); 39 | MPI_Abort(MPI_COMM_WORLD,result); 40 | } 41 | 42 | MPI_Comm_free(&cart_world); 43 | if (cart_world != MPI_COMM_NULL) { 44 | printf("freed window is not null\n"); 45 | MPI_Abort(MPI_COMM_WORLD,1); 46 | } 47 | } 48 | 49 | fflush(0); 50 | usleep(1); 51 | MPI_Barrier(MPI_COMM_WORLD); 52 | if (me==0) printf("all done\n"); 53 | 54 | rc = MPI_Finalize(); 55 | 56 | return rc; 57 | } 58 | -------------------------------------------------------------------------------- /testcoll.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | 10 | #ifdef NOWRAP 11 | #include 12 | #else 13 | #include "mpi.h" 14 | #endif 15 | 16 | #define N 10 17 | 18 | int main(int argc, char* argv[]) 19 | { 20 | int rc; 21 | rc = MPI_Init(&argc,&argv); 22 | 23 | int me, np; 24 | MPI_Comm_rank(MPI_COMM_WORLD,&me); 25 | MPI_Comm_size(MPI_COMM_WORLD,&np); 26 | printf("I am %d of %d\n", me, np); 27 | 28 | { 29 | int x[N]; 30 | int y[N]; 31 | for (size_t i=0; i 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | 10 | #ifdef NOWRAP 11 | #include 12 | #else 13 | #include "mpi.h" 14 | #endif 15 | 16 | int main(int argc, char* argv[]) 17 | { 18 | int rc; 19 | rc = MPI_Init(&argc,&argv); 20 | 21 | int n = (argc > 1) ? atoi(argv[1]) : 1000; 22 | 23 | int me, np; 24 | MPI_Comm_rank(MPI_COMM_WORLD,&me); 25 | MPI_Comm_size(MPI_COMM_WORLD,&np); 26 | //printf("I am %d of %d\n", me, np); 27 | if (me==0) printf("nonblocking collectives: %d iterations\n", n); 28 | 29 | { 30 | if (me==0) printf("MPI_Ibcast\n"); 31 | 32 | MPI_Request * r = calloc(n,sizeof(MPI_Request)); 33 | int * x = calloc(n,sizeof(int)); 34 | for (int i=0; i 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | 10 | #ifdef NOWRAP 11 | #include 12 | #else 13 | #include "mpi.h" 14 | #endif 15 | 16 | int main(int argc, char* argv[]) 17 | { 18 | int rc; 19 | rc = MPI_Init(&argc,&argv); 20 | 21 | printf("main: MPI_COMM_WORLD=%lx\n", (intptr_t)MPI_COMM_WORLD); 22 | 23 | int me, np; 24 | MPI_Comm_rank(MPI_COMM_WORLD,&me); 25 | MPI_Comm_size(MPI_COMM_WORLD,&np); 26 | printf("I am %d of %d\n", me, np); 27 | 28 | { 29 | int result; 30 | MPI_Comm world = MPI_COMM_WORLD; 31 | MPI_Comm_compare(MPI_COMM_WORLD,world,&result); 32 | if (result != MPI_IDENT) MPI_Abort(MPI_COMM_WORLD,result); 33 | 34 | MPI_Comm dup; 35 | MPI_Comm_dup(MPI_COMM_WORLD,&dup); 36 | MPI_Barrier(dup); 37 | 38 | MPI_Comm_compare(MPI_COMM_WORLD,dup,&result); 39 | if (result != MPI_CONGRUENT) MPI_Abort(MPI_COMM_WORLD,result); 40 | 41 | MPI_Comm split; 42 | MPI_Comm_split(MPI_COMM_WORLD,0,-me,&split); 43 | MPI_Barrier(split); 44 | 45 | MPI_Comm_compare(MPI_COMM_WORLD,split,&result); 46 | if (np > 1 && result != MPI_SIMILAR) { 47 | printf("result=%d, MPI_CONGRUENT=%d MPI_SIMILAR=%d\n", result, MPI_CONGRUENT, MPI_SIMILAR); 48 | MPI_Abort(MPI_COMM_WORLD,result); 49 | } 50 | else if (np == 1 && result != MPI_CONGRUENT) { 51 | printf("result=%d, MPI_CONGRUENT=%d MPI_SIMILAR=%d\n", result, MPI_CONGRUENT, MPI_SIMILAR); 52 | MPI_Abort(MPI_COMM_WORLD,result); 53 | } 54 | 55 | MPI_Comm oddeven; 56 | MPI_Comm_split(MPI_COMM_WORLD,me%2,me,&oddeven); 57 | MPI_Barrier(oddeven); 58 | 59 | MPI_Comm_compare(MPI_COMM_WORLD,oddeven,&result); 60 | if ((np > 1) && (result != MPI_UNEQUAL)) MPI_Abort(MPI_COMM_WORLD,result); 61 | 62 | MPI_Comm shared; 63 | MPI_Comm_split_type(MPI_COMM_WORLD,MPI_COMM_TYPE_SHARED,0,MPI_INFO_NULL,&shared); 64 | MPI_Barrier(shared); 65 | 66 | MPI_Comm_free(&split); 67 | MPI_Comm_free(&oddeven); 68 | 69 | MPI_Comm_free(&shared); 70 | if (shared != MPI_COMM_NULL) { 71 | printf("freed window is not null\n"); 72 | MPI_Abort(MPI_COMM_WORLD,1); 73 | } 74 | 75 | // disconnect is the same as free except it synchronizes 76 | MPI_Comm_disconnect(&dup); 77 | if (dup != MPI_COMM_NULL) { 78 | printf("freed window is not null\n"); 79 | MPI_Abort(MPI_COMM_WORLD,1); 80 | } 81 | } 82 | 83 | { 84 | MPI_Comm split; 85 | MPI_Comm_split(MPI_COMM_WORLD,MPI_UNDEFINED,0,&split); 86 | if (split != MPI_COMM_NULL) { 87 | printf("color=MPI_UNDEFINED but split=%p != MPI_COMM_NULL=%p\n", split, MPI_COMM_NULL); 88 | MPI_Abort(MPI_COMM_WORLD,1); 89 | } 90 | } 91 | 92 | fflush(0); 93 | usleep(1); 94 | MPI_Barrier(MPI_COMM_WORLD); 95 | if (me==0) printf("all done\n"); 96 | 97 | rc = MPI_Finalize(); 98 | 99 | return rc; 100 | } 101 | -------------------------------------------------------------------------------- /testconstants.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | #ifdef NOWRAP 4 | #include 5 | #else 6 | #include "mpi.h" 7 | #endif 8 | 9 | // clearly, not ALL of them yet... 10 | static MPI_Comm all_comm_constants [] = { 11 | MPI_COMM_WORLD, 12 | MPI_COMM_SELF 13 | }; 14 | 15 | // clearly, not ALL of them yet... 16 | static MPI_Datatype all_datatype_constants [] = { 17 | MPI_CHAR, MPI_SHORT, MPI_INT, MPI_LONG, MPI_LONG_LONG, MPI_FLOAT, MPI_DOUBLE 18 | }; 19 | 20 | static int all_integer_constants [] = { 21 | MPI_SUCCESS, 22 | MPI_ERR_BUFFER, 23 | MPI_ERR_COUNT, 24 | MPI_ERR_TYPE, 25 | MPI_ERR_TAG, 26 | MPI_ERR_COMM, 27 | MPI_ERR_RANK, 28 | MPI_ERR_REQUEST, 29 | MPI_ERR_ROOT, 30 | MPI_ERR_GROUP, 31 | MPI_ERR_OP, 32 | MPI_ERR_TOPOLOGY, 33 | MPI_ERR_DIMS, 34 | MPI_ERR_ARG, 35 | MPI_ERR_UNKNOWN, 36 | MPI_ERR_TRUNCATE, 37 | MPI_ERR_OTHER, 38 | MPI_ERR_INTERN, 39 | MPI_ERR_PENDING, 40 | MPI_ERR_IN_STATUS, 41 | MPI_ERR_ACCESS, 42 | MPI_ERR_AMODE, 43 | MPI_ERR_ASSERT, 44 | MPI_ERR_BAD_FILE, 45 | MPI_ERR_BASE, 46 | MPI_ERR_CONVERSION, 47 | MPI_ERR_DISP, 48 | MPI_ERR_DUP_DATAREP, 49 | MPI_ERR_FILE_EXISTS, 50 | MPI_ERR_FILE_IN_USE, 51 | MPI_ERR_FILE, 52 | MPI_ERR_INFO_KEY, 53 | MPI_ERR_INFO_NOKEY, 54 | MPI_ERR_INFO_VALUE, 55 | MPI_ERR_INFO, 56 | MPI_ERR_IO, 57 | MPI_ERR_KEYVAL, 58 | MPI_ERR_LOCKTYPE, 59 | MPI_ERR_NAME, 60 | MPI_ERR_NO_MEM, 61 | MPI_ERR_NOT_SAME, 62 | MPI_ERR_NO_SPACE, 63 | MPI_ERR_NO_SUCH_FILE, 64 | MPI_ERR_PORT, 65 | #if MPI_VERSION >= 4 66 | MPI_ERR_PROC_ABORTED, 67 | #endif 68 | MPI_ERR_QUOTA, 69 | MPI_ERR_READ_ONLY, 70 | MPI_ERR_RMA_ATTACH, 71 | MPI_ERR_RMA_CONFLICT, 72 | MPI_ERR_RMA_RANGE, 73 | MPI_ERR_RMA_SHARED, 74 | MPI_ERR_RMA_SYNC, 75 | MPI_ERR_RMA_FLAVOR, 76 | MPI_ERR_SERVICE, 77 | #if MPI_VERSION >= 4 78 | MPI_ERR_SESSION, 79 | #endif 80 | MPI_ERR_SIZE, 81 | MPI_ERR_SPAWN, 82 | MPI_ERR_UNSUPPORTED_DATAREP, 83 | MPI_ERR_UNSUPPORTED_OPERATION, 84 | #if MPI_VERSION >= 4 85 | MPI_ERR_VALUE_TOO_LARGE, 86 | #endif 87 | MPI_ERR_WIN, 88 | MPI_T_ERR_CANNOT_INIT, 89 | #if MPICH_BUG_IS_FIXED 90 | MPI_T_ERR_NOT_ACCESSIBLE, 91 | #endif 92 | MPI_T_ERR_NOT_INITIALIZED, 93 | #if MPI_VERSION >= 4 94 | MPI_T_ERR_NOT_SUPPORTED, 95 | #endif 96 | MPI_T_ERR_MEMORY, 97 | MPI_T_ERR_INVALID, 98 | MPI_T_ERR_INVALID_INDEX, 99 | MPI_T_ERR_INVALID_ITEM, 100 | MPI_T_ERR_INVALID_SESSION, 101 | MPI_T_ERR_INVALID_HANDLE, 102 | MPI_T_ERR_INVALID_NAME, 103 | MPI_T_ERR_OUT_OF_HANDLES, 104 | MPI_T_ERR_OUT_OF_SESSIONS, 105 | MPI_T_ERR_CVAR_SET_NOT_NOW, 106 | MPI_T_ERR_CVAR_SET_NEVER, 107 | MPI_T_ERR_PVAR_NO_WRITE, 108 | MPI_T_ERR_PVAR_NO_STARTSTOP, 109 | MPI_T_ERR_PVAR_NO_ATOMIC, 110 | MPI_ERR_LASTCODE, 111 | MPI_PROC_NULL, 112 | MPI_ANY_SOURCE, 113 | MPI_ROOT, 114 | MPI_ANY_TAG, 115 | MPI_UNDEFINED, 116 | MPI_BSEND_OVERHEAD, 117 | MPI_KEYVAL_INVALID, 118 | MPI_LOCK_EXCLUSIVE, 119 | MPI_LOCK_SHARED, 120 | MPI_COMM_TYPE_SHARED, 121 | #if MPI_VERSION >= 4 122 | MPI_COMM_TYPE_HW_UNGUIDED, 123 | MPI_COMM_TYPE_HW_GUIDED, 124 | #endif 125 | MPI_IDENT, 126 | MPI_CONGRUENT, 127 | MPI_SIMILAR, 128 | MPI_UNEQUAL, 129 | MPI_TAG_UB, 130 | MPI_IO, 131 | MPI_HOST, 132 | MPI_WTIME_IS_GLOBAL, 133 | MPI_GRAPH, 134 | MPI_CART, 135 | MPI_DIST_GRAPH, 136 | MPI_APPNUM, 137 | MPI_LASTUSEDCODE, 138 | MPI_UNIVERSE_SIZE, 139 | MPI_WIN_BASE, 140 | MPI_WIN_DISP_UNIT, 141 | MPI_WIN_SIZE, 142 | MPI_WIN_CREATE_FLAVOR, 143 | MPI_WIN_MODEL, 144 | MPI_WIN_FLAVOR_CREATE, 145 | MPI_WIN_FLAVOR_ALLOCATE, 146 | MPI_WIN_FLAVOR_DYNAMIC, 147 | MPI_WIN_FLAVOR_SHARED, 148 | MPI_WIN_SEPARATE, 149 | MPI_WIN_UNIFIED, 150 | MPI_MODE_APPEND, 151 | MPI_MODE_CREATE, 152 | MPI_MODE_DELETE_ON_CLOSE, 153 | MPI_MODE_EXCL, 154 | MPI_MODE_NOCHECK, 155 | MPI_MODE_NOPRECEDE, 156 | MPI_MODE_NOPUT, 157 | MPI_MODE_NOSTORE, 158 | MPI_MODE_NOSUCCEED, 159 | MPI_MODE_RDONLY, 160 | MPI_MODE_RDWR, 161 | MPI_MODE_SEQUENTIAL, 162 | MPI_MODE_UNIQUE_OPEN, 163 | MPI_MODE_WRONLY, 164 | MPI_COMBINER_CONTIGUOUS, 165 | MPI_COMBINER_DARRAY, 166 | MPI_COMBINER_DUP, 167 | MPI_COMBINER_HINDEXED, 168 | MPI_COMBINER_HVECTOR, 169 | MPI_COMBINER_INDEXED_BLOCK, 170 | MPI_COMBINER_HINDEXED_BLOCK, 171 | MPI_COMBINER_INDEXED, 172 | MPI_COMBINER_NAMED, 173 | MPI_COMBINER_RESIZED, 174 | MPI_COMBINER_STRUCT, 175 | MPI_COMBINER_SUBARRAY, 176 | MPI_COMBINER_VECTOR, 177 | MPI_THREAD_FUNNELED, 178 | MPI_THREAD_MULTIPLE, 179 | MPI_THREAD_SERIALIZED, 180 | MPI_THREAD_SINGLE, 181 | MPI_DISPLACEMENT_CURRENT, 182 | MPI_DISTRIBUTE_BLOCK, 183 | MPI_DISTRIBUTE_CYCLIC, 184 | MPI_DISTRIBUTE_DFLT_DARG, 185 | MPI_DISTRIBUTE_NONE, 186 | MPI_ORDER_C, 187 | MPI_ORDER_FORTRAN, 188 | MPI_SEEK_CUR, 189 | MPI_SEEK_END, 190 | MPI_SEEK_SET, 191 | MPI_TYPECLASS_COMPLEX, 192 | MPI_TYPECLASS_INTEGER, 193 | MPI_TYPECLASS_REAL 194 | }; 195 | 196 | int main(void) 197 | { 198 | (void)all_integer_constants; 199 | (void)all_comm_constants; 200 | (void)all_datatype_constants; 201 | return MPI_SUCCESS; 202 | } 203 | -------------------------------------------------------------------------------- /testerrh.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | #ifdef NOWRAP 8 | #include 9 | #else 10 | #include "mpi.h" 11 | #endif 12 | 13 | void callback(MPI_Comm * comm, int *err, ...) 14 | { 15 | (void)comm; 16 | (void)err; 17 | } 18 | 19 | int main(int argc, char *argv[]) 20 | { 21 | MPI_Init(&argc, &argv); 22 | 23 | MPI_Errhandler errh; 24 | MPI_Comm_create_errhandler(callback, &errh); 25 | 26 | MPI_Comm_set_errhandler(MPI_COMM_WORLD, errh); 27 | 28 | MPI_Comm_call_errhandler(MPI_COMM_WORLD,99); 29 | 30 | MPI_Comm dup; 31 | MPI_Comm_dup(MPI_COMM_WORLD,&dup); 32 | 33 | MPI_Errhandler temp0; 34 | MPI_Comm_get_errhandler(MPI_COMM_WORLD,&temp0); 35 | 36 | MPI_Errhandler temp1; 37 | MPI_Comm_get_errhandler(dup,&temp1); 38 | 39 | MPI_Comm_free(&dup); 40 | 41 | MPI_Errhandler_free(&temp0); 42 | 43 | MPI_Errhandler_free(&temp1); 44 | 45 | MPI_Errhandler_free(&errh); 46 | 47 | #if 0 48 | MPI_Errhandler_free(&errh); 49 | #endif 50 | MPI_Finalize(); 51 | return 0; 52 | } 53 | -------------------------------------------------------------------------------- /testgroup.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | 10 | #ifdef NOWRAP 11 | #include 12 | #else 13 | #include "mpi.h" 14 | #endif 15 | 16 | int main(int argc, char* argv[]) 17 | { 18 | int rc; 19 | rc = MPI_Init(&argc,&argv); 20 | 21 | int me, np; 22 | MPI_Comm_rank(MPI_COMM_WORLD,&me); 23 | MPI_Comm_size(MPI_COMM_WORLD,&np); 24 | printf("I am %d of %d\n", me, np); 25 | fflush(0); 26 | usleep(1); 27 | MPI_Barrier(MPI_COMM_WORLD); 28 | 29 | // create the world group 30 | MPI_Group group_world = MPI_GROUP_NULL; 31 | MPI_Comm_group(MPI_COMM_WORLD, &group_world); 32 | 33 | // query the world group 34 | int gme, gnp; 35 | MPI_Group_rank(group_world,&gme); 36 | MPI_Group_size(group_world,&gnp); 37 | if (me != gme || np != gnp) { 38 | printf("%d : %d :: %d : %d\n", me, gme, np, gnp); 39 | MPI_Abort(MPI_COMM_WORLD,1); 40 | } 41 | 42 | // query the empty group 43 | MPI_Group_rank(MPI_GROUP_EMPTY,&gme); 44 | MPI_Group_size(MPI_GROUP_EMPTY,&gnp); 45 | if (MPI_UNDEFINED != gme || 0 != gnp) { 46 | printf("%d : %d :: %d : %d\n", MPI_UNDEFINED, gme, 0, gnp); 47 | MPI_Abort(MPI_COMM_WORLD,1); 48 | } 49 | 50 | // create a new world group 51 | MPI_Group dup1; 52 | MPI_Comm_group(MPI_COMM_WORLD, &dup1); 53 | 54 | // compare two world groups 55 | int result; 56 | MPI_Group_compare(group_world,dup1,&result); 57 | if (result != MPI_IDENT) { 58 | printf("%d: result = %d MPI_IDENT = %d MPI_SIMILAR = %d MPI_UNEQUAL = %d\n", 59 | me, result, MPI_IDENT, MPI_SIMILAR, MPI_UNEQUAL); 60 | fflush(0); 61 | usleep(1); 62 | MPI_Abort(MPI_COMM_WORLD,1); 63 | } 64 | 65 | // test excl using all but rank 0 66 | MPI_Group excl1; 67 | int excluded[1] = { 0 }; 68 | MPI_Group_excl(group_world,1,excluded,&excl1); 69 | MPI_Group_size(excl1,&gnp); 70 | if (gnp != (np-1)) { 71 | printf("%d: excl size = %d\n", me, gnp); 72 | fflush(0); 73 | usleep(1); 74 | MPI_Abort(MPI_COMM_WORLD,1); 75 | } 76 | MPI_Barrier(MPI_COMM_WORLD); 77 | 78 | // test incl using all but rank 0 79 | MPI_Group incl1; 80 | int * included = calloc((np-1),sizeof(int)); 81 | for (int i=1; i 0) { 142 | printf("%d: errors = %d ranks2 = ", me, errors); 143 | for (int i=0; i 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | 10 | #ifdef NOWRAP 11 | #include 12 | #else 13 | #include "mpi.h" 14 | #endif 15 | 16 | int main(int argc, char* argv[]) 17 | { 18 | int rc, flag; 19 | rc = MPI_Initialized(&flag); 20 | printf("is init? %d\n", flag); 21 | 22 | int major, minor; 23 | rc = MPI_Get_version(&major,&minor); 24 | printf("BEFORE major=%d, minor=%d\n", major, minor); 25 | 26 | int len; 27 | char version[MPI_MAX_LIBRARY_VERSION_STRING]; 28 | rc = MPI_Get_library_version(version,&len); 29 | printf("BEFORE version=%s\n", version); 30 | 31 | rc = MPI_Init(&argc,&argv); 32 | 33 | rc = MPI_Get_version(&major,&minor); 34 | printf("AFTER major=%d, minor=%d\n", major, minor); 35 | 36 | rc = MPI_Get_library_version(version,&len); 37 | printf("AFTER version=%s\n", version); 38 | 39 | double t0 = MPI_Wtime(); 40 | 41 | rc = MPI_Initialized(&flag); 42 | printf("is init? %d\n", flag); 43 | rc = MPI_Finalized(&flag); 44 | printf("is final? %d\n", flag); 45 | rc = MPI_Is_thread_main(&flag); 46 | printf("is thread main? %d\n", flag); 47 | 48 | double t1 = MPI_Wtime(); 49 | 50 | printf("dt=%lf\n",t1-t0); 51 | 52 | rc = MPI_Finalize(); 53 | 54 | rc = MPI_Finalized(&flag); 55 | printf("is final? %d\n", flag); 56 | 57 | printf("all done\n"); 58 | 59 | return rc; 60 | } 61 | -------------------------------------------------------------------------------- /testmalloc.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | int main(int argc, char* argv[]) 8 | { 9 | MPI_Init(&argc,&argv); 10 | 11 | const int nr = (argc > 1) ? atoi(argv[1]) : 1000; 12 | const int nm = (argc > 2) ? atoi(argv[2]) : 1000; 13 | 14 | printf("iterations = %d, mallocs = %d\n", nr, nm); 15 | 16 | void ** pp = calloc(nm,sizeof(void*)); 17 | 18 | double t0,t1,t2,t3,tt01=0,tt12=0,tt23=0; 19 | 20 | for (int r=0; r<=nr; r++) { 21 | 22 | t0 = MPI_Wtime(); 23 | 24 | for (int m=0; m 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | 10 | #ifdef NOWRAP 11 | #include 12 | #else 13 | #include "mpi.h" 14 | #endif 15 | 16 | static MPI_Op ops[] = { MPI_MIN, MPI_MAX, MPI_SUM, 17 | MPI_BAND, MPI_BOR, MPI_BXOR }; 18 | 19 | void my_reduce_op_1(void *invec, void *inoutvec, int *len, MPI_Datatype * datatype) 20 | { 21 | for (int i=0; i<*len; i++) { 22 | if (*datatype == MPI_INT) { 23 | ((int*)inoutvec)[i] += ((int*)invec)[i]; 24 | } 25 | else { 26 | printf("my_reduce_op_1: unsupported datatype\n"); 27 | } 28 | } 29 | fflush(0); 30 | } 31 | 32 | void my_reduce_op_2(void *invec, void *inoutvec, int *len, MPI_Datatype * datatype) 33 | { 34 | for (int i=0; i<*len; i++) { 35 | if (*datatype == MPI_INT) { 36 | ((int*)inoutvec)[i] += ((int*)invec)[i]; 37 | } 38 | else { 39 | printf("my_reduce_op_2: unsupported datatype\n"); 40 | } 41 | } 42 | fflush(0); 43 | } 44 | void my_reduce_op_3(void *invec, void *inoutvec, int *len, MPI_Datatype * datatype) 45 | { 46 | for (int i=0; i<*len; i++) { 47 | if (*datatype == MPI_INT) { 48 | ((int*)inoutvec)[i] += ((int*)invec)[i]; 49 | } 50 | else { 51 | printf("my_reduce_op_3: unsupported datatype\n"); 52 | } 53 | } 54 | fflush(0); 55 | } 56 | 57 | void my_reduce_op(void *invec, void *inoutvec, int *len, MPI_Datatype * datatype) 58 | { 59 | for (int i=0; i<*len; i++) { 60 | if (*datatype == MPI_INT) { 61 | ((int*)inoutvec)[i] += ((int*)invec)[i]; 62 | } 63 | else { 64 | printf("my_reduce_op: unsupported datatype\n"); 65 | } 66 | } 67 | fflush(0); 68 | } 69 | 70 | int main(int argc, char* argv[]) 71 | { 72 | int rc; 73 | rc = MPI_Init(&argc,&argv); 74 | 75 | int me, np; 76 | MPI_Comm_rank(MPI_COMM_WORLD,&me); 77 | MPI_Comm_size(MPI_COMM_WORLD,&np); 78 | //printf("I am %d of %d\n", me, np); 79 | fflush(0); 80 | MPI_Barrier(MPI_COMM_WORLD); 81 | usleep(1); 82 | 83 | if (me==0) printf("built-in ops\n"); 84 | for (int o=0; o<(int)(sizeof(ops)/sizeof(ops[0])); o++) 85 | { 86 | MPI_Op op = ops[o]; 87 | 88 | int in, out = -1; 89 | 90 | if (op == MPI_MIN || op == MPI_MAX || op == MPI_SUM) { 91 | in = 100 + me; 92 | } 93 | else if (op == MPI_BAND || op == MPI_BOR || op == MPI_BXOR) { 94 | in = 1; 95 | } 96 | 97 | rc = MPI_Allreduce(&in, &out, 1, MPI_INT, op, MPI_COMM_WORLD); 98 | 99 | int ref = 0; 100 | if (op == MPI_MIN) { 101 | ref = 100; 102 | } 103 | else if (op == MPI_MAX) { 104 | ref = 100 + (np-1); 105 | } 106 | else if (op == MPI_SUM) { 107 | ref = 100 * np + (np-1) * np / 2; 108 | } 109 | else if (op == MPI_BAND) { 110 | ref = 1; 111 | for (int i=0; i 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | 10 | #ifdef NOWRAP 11 | #include 12 | #else 13 | #include "mpi.h" 14 | #endif 15 | 16 | int main(int argc, char* argv[]) 17 | { 18 | int rc; 19 | rc = MPI_Init(&argc,&argv); 20 | 21 | int me, np; 22 | MPI_Comm_rank(MPI_COMM_WORLD,&me); 23 | MPI_Comm_size(MPI_COMM_WORLD,&np); 24 | printf("I am %d of %d\n", me, np); 25 | fflush(0); 26 | MPI_Barrier(MPI_COMM_WORLD); 27 | usleep(1); 28 | 29 | if (1) 30 | { 31 | MPI_Barrier(MPI_COMM_WORLD); 32 | fflush(0); 33 | if (me==0) printf("Ibarrier+...\n"); 34 | fflush(0); 35 | 36 | MPI_Request r; 37 | int flag; 38 | 39 | MPI_Ibarrier(MPI_COMM_WORLD,&r); 40 | MPI_Wait(&r,MPI_STATUS_IGNORE); 41 | if (r != MPI_REQUEST_NULL) { 42 | printf("freed request is not null\n"); 43 | printf("%d: request=0x%lx MPI_REQUEST_NULL=0x%lx\n", me, (intptr_t)r, (intptr_t)MPI_REQUEST_NULL); 44 | MPI_Abort(MPI_COMM_WORLD,1); 45 | } 46 | 47 | MPI_Ibarrier(MPI_COMM_WORLD,&r); 48 | MPI_Waitall(1,&r,MPI_STATUSES_IGNORE); 49 | if (r != MPI_REQUEST_NULL) { 50 | printf("freed request is not null\n"); 51 | printf("%d: request=0x%lx MPI_REQUEST_NULL=0x%lx\n", me, (intptr_t)r, (intptr_t)MPI_REQUEST_NULL); 52 | MPI_Abort(MPI_COMM_WORLD,1); 53 | } 54 | 55 | MPI_Ibarrier(MPI_COMM_WORLD,&r); 56 | flag = 0; 57 | while (!flag) { 58 | MPI_Test(&r,&flag,MPI_STATUS_IGNORE); 59 | } 60 | if (r != MPI_REQUEST_NULL) { 61 | printf("freed request is not null\n"); 62 | printf("%d: request=0x%lx MPI_REQUEST_NULL=0x%lx\n", me, (intptr_t)r, (intptr_t)MPI_REQUEST_NULL); 63 | MPI_Abort(MPI_COMM_WORLD,1); 64 | } 65 | 66 | MPI_Ibarrier(MPI_COMM_WORLD,&r); 67 | flag = 0; 68 | while (!flag) { 69 | MPI_Testall(1,&r,&flag,MPI_STATUSES_IGNORE); 70 | } 71 | if (r != MPI_REQUEST_NULL) { 72 | printf("freed request is not null\n"); 73 | printf("%d: request=0x%lx MPI_REQUEST_NULL=0x%lx\n", me, (intptr_t)r, (intptr_t)MPI_REQUEST_NULL); 74 | MPI_Abort(MPI_COMM_WORLD,1); 75 | } 76 | } 77 | 78 | if (1) 79 | { 80 | MPI_Barrier(MPI_COMM_WORLD); 81 | fflush(0); 82 | if (me==0) printf("Isend+Irecv+Waitall\n"); 83 | fflush(0); 84 | 85 | int buffer[2] = { me }; 86 | MPI_Request r[2]; 87 | MPI_Isend(&buffer[0], 1, MPI_INT, me, 99, MPI_COMM_WORLD, &r[0]); 88 | MPI_Irecv(&buffer[1], 1, MPI_INT, me, 99, MPI_COMM_WORLD, &r[1]); 89 | MPI_Waitall(2,r,MPI_STATUSES_IGNORE); 90 | } 91 | 92 | if (1) 93 | { 94 | MPI_Barrier(MPI_COMM_WORLD); 95 | fflush(0); 96 | if (me==0) printf("Isend+Recv(s)+Wait\n"); 97 | fflush(0); 98 | 99 | int buffer[2] = { me }; 100 | MPI_Request r = MPI_REQUEST_NULL; 101 | MPI_Status s; 102 | memset(&s,255,sizeof(MPI_Status)); 103 | MPI_Isend(&buffer[0], 1, MPI_INT, me, 99, MPI_COMM_WORLD, &r); 104 | MPI_Recv(&buffer[1], 1, MPI_INT, me, 99, MPI_COMM_WORLD, &s); 105 | MPI_Wait(&r,MPI_STATUS_IGNORE); 106 | 107 | int rcount = -3; 108 | MPI_Get_count(&s, MPI_INT, &rcount); 109 | if ((s.MPI_SOURCE != me) || (s.MPI_TAG != 99) || (rcount != 1)) { 110 | printf("[%d]: SOURCE=%d TAG=%d count=%d\n", 111 | me, s.MPI_SOURCE, s.MPI_TAG, rcount); 112 | MPI_Abort(MPI_COMM_WORLD,1); 113 | } 114 | } 115 | 116 | if (1) 117 | { 118 | MPI_Barrier(MPI_COMM_WORLD); 119 | fflush(0); 120 | if (me==0) printf("Isend+Irecv+Wait+Wait(s)\n"); 121 | fflush(0); 122 | 123 | int buffer[2] = { me }; 124 | MPI_Request r[2] = { MPI_REQUEST_NULL }; 125 | MPI_Status s; 126 | memset(&s,255,sizeof(MPI_Status)); 127 | MPI_Isend(&buffer[0], 1, MPI_INT, me, 99, MPI_COMM_WORLD, &r[0]); 128 | MPI_Irecv(&buffer[1], 1, MPI_INT, me, 99, MPI_COMM_WORLD, &r[1]); 129 | MPI_Wait(&r[1],&s); 130 | MPI_Wait(&r[0],MPI_STATUS_IGNORE); 131 | 132 | int rcount = -3; 133 | MPI_Get_count(&s, MPI_INT, &rcount); 134 | if ((s.MPI_SOURCE != me) || (s.MPI_TAG != 99) || (rcount != 1)) { 135 | printf("[%d]: SOURCE=%d TAG=%d count=%d\n", 136 | me, s.MPI_SOURCE, s.MPI_TAG, rcount); 137 | MPI_Abort(MPI_COMM_WORLD,1); 138 | } 139 | } 140 | 141 | if (1) 142 | { 143 | MPI_Barrier(MPI_COMM_WORLD); 144 | fflush(0); 145 | if (me==0) printf("Isend+Irecv+Waitall(s)\n"); 146 | fflush(0); 147 | 148 | int buffer[2] = { me }; 149 | MPI_Request r[2] = { MPI_REQUEST_NULL }; 150 | MPI_Status s[2]; 151 | memset(&s,255,2*sizeof(MPI_Status)); 152 | MPI_Isend(&buffer[0], 1, MPI_INT, me, 99, MPI_COMM_WORLD, &r[0]); 153 | MPI_Irecv(&buffer[1], 1, MPI_INT, me, 99, MPI_COMM_WORLD, &r[1]); 154 | MPI_Waitall(2,r,s); 155 | 156 | int rcount = -3; 157 | MPI_Get_count(&s[1], MPI_INT, &rcount); 158 | if ((s[1].MPI_SOURCE != me) || (s[1].MPI_TAG != 99) || (rcount != 1)) { 159 | printf("[%d]: SOURCE=%d TAG=%d count=%d\n", 160 | me, s[1].MPI_SOURCE, s[1].MPI_TAG, rcount); 161 | MPI_Abort(MPI_COMM_WORLD,1); 162 | } 163 | } 164 | 165 | if (1) 166 | { 167 | MPI_Barrier(MPI_COMM_WORLD); 168 | fflush(0); 169 | if (me==0) printf("Isend+Irecv+Testall(s)\n"); 170 | fflush(0); 171 | 172 | int buffer[2] = { me }; 173 | MPI_Request r[2] = { MPI_REQUEST_NULL }; 174 | MPI_Status s[2]; 175 | memset(&s,255,2*sizeof(MPI_Status)); 176 | MPI_Isend(&buffer[0], 1, MPI_INT, me, 99, MPI_COMM_WORLD, &r[0]); 177 | MPI_Irecv(&buffer[1], 1, MPI_INT, me, 99, MPI_COMM_WORLD, &r[1]); 178 | 179 | int flag = 0; 180 | while (!flag) { 181 | MPI_Testall(2,r,&flag,s); 182 | } 183 | 184 | int rcount = -3; 185 | MPI_Get_count(&s[1], MPI_INT, &rcount); 186 | if ((s[1].MPI_SOURCE != me) || (s[1].MPI_TAG != 99) || (rcount != 1)) { 187 | printf("[%d]: SOURCE=%d TAG=%d count=%d\n", 188 | me, s[1].MPI_SOURCE, s[1].MPI_TAG, rcount); 189 | MPI_Abort(MPI_COMM_WORLD,1); 190 | } 191 | } 192 | 193 | if (1) 194 | { 195 | MPI_Barrier(MPI_COMM_WORLD); 196 | fflush(0); 197 | if (me==0) printf("Isend+Irecv+Waitall\n"); 198 | fflush(0); 199 | 200 | // MPICH fails around 131072 (https://github.com/pmodels/mpich/issues/6389) 201 | const int n = 100000; 202 | 203 | int * buffer = calloc(2*n,sizeof(int)); 204 | for (int i=0; i 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | 10 | #ifdef NOWRAP 11 | #include 12 | #else 13 | #include "mpi.h" 14 | #endif 15 | 16 | static MPI_Datatype types[] = { MPI_CHAR, MPI_SHORT, MPI_INT, MPI_LONG, MPI_LONG_LONG, MPI_FLOAT, MPI_DOUBLE }; 17 | static size_t type_sizes[] = { sizeof(char), sizeof(short), sizeof(int), sizeof(long), 18 | sizeof(long long), sizeof(float), sizeof(double) }; 19 | static char type_names[][20] = { "MPI_CHAR", "MPI_SHORT", "MPI_INT", "MPI_LONG", 20 | "MPI_LONG_LONG_INT", "MPI_FLOAT", "MPI_DOUBLE" }; 21 | 22 | int main(int argc, char* argv[]) 23 | { 24 | int rc; 25 | rc = MPI_Init(&argc,&argv); 26 | 27 | int me, np; 28 | MPI_Comm_rank(MPI_COMM_WORLD,&me); 29 | MPI_Comm_size(MPI_COMM_WORLD,&np); 30 | printf("I am %d of %d\n", me, np); 31 | fflush(0); 32 | MPI_Barrier(MPI_COMM_WORLD); 33 | usleep(1); 34 | 35 | for (int t=0; t<(int)(sizeof(types)/sizeof(types[0])); t++) 36 | { 37 | MPI_Datatype type = types[t]; 38 | 39 | int size; 40 | MPI_Type_size(type, &size); 41 | if ((size_t)size != type_sizes[t]) { 42 | printf("t=%d type=%p\n", t, type); 43 | MPI_Abort(MPI_COMM_WORLD,size); 44 | } 45 | 46 | int len; 47 | char name[MPI_MAX_OBJECT_NAME] = {0}; 48 | MPI_Type_get_name(type, name, &len); 49 | int cmp = strcmp(name, type_names[t]); 50 | if (cmp != 0) { 51 | printf("name=%s\n", name); 52 | MPI_Abort(MPI_COMM_WORLD,cmp); 53 | } 54 | 55 | fflush(0); 56 | usleep(1); 57 | MPI_Barrier(MPI_COMM_WORLD); 58 | 59 | void * buffer = malloc(2*size); 60 | memset(buffer,0xF,2*size); 61 | MPI_Request r[2]; 62 | MPI_Isend(buffer, 1, type, me, t, MPI_COMM_WORLD, &r[0]); 63 | MPI_Irecv(buffer+size, 1, type, me, t, MPI_COMM_WORLD, &r[1]); 64 | MPI_Waitall(2,r,MPI_STATUSES_IGNORE); 65 | } 66 | 67 | { 68 | MPI_Datatype contig = MPI_DATATYPE_NULL; 69 | MPI_Type_contiguous(100, MPI_BYTE, &contig); 70 | MPI_Type_commit(&contig); 71 | 72 | int size; 73 | MPI_Type_size(contig, &size); 74 | if (size != 100) { 75 | printf("wrong: size=%d\n", size); 76 | MPI_Abort(MPI_COMM_WORLD,size); 77 | } 78 | 79 | int psize; 80 | MPI_Pack_size(1, contig, MPI_COMM_SELF, &psize); 81 | if (psize != 100) { 82 | printf("wrong: pack size=%d\n", psize); 83 | MPI_Abort(MPI_COMM_WORLD,psize); 84 | } 85 | 86 | MPI_Aint lb, extent; 87 | MPI_Type_get_extent(contig, &lb, &extent); 88 | if (lb !=0 || extent != 100) { 89 | printf("\n"); 90 | printf("wrong: lb=%zu extent=%ld\n", lb, extent); 91 | MPI_Abort(MPI_COMM_WORLD,extent); 92 | } 93 | 94 | int ni, na, nd, combiner; 95 | MPI_Type_get_envelope(contig, &ni, &na, &nd, &combiner); 96 | if (ni != 1 || na != 0 || nd != 1 || combiner != MPI_COMBINER_CONTIGUOUS) { 97 | printf("get_envelope: ni=%d na=%d nd=%d combiner=%d\n", ni, na, nd, combiner); 98 | printf("MPI_COMBINER_CONTIGUOUS = %d\n", MPI_COMBINER_CONTIGUOUS); 99 | MPI_Abort(MPI_COMM_WORLD,combiner); 100 | } 101 | 102 | MPI_Type_free(&contig); 103 | if (contig != MPI_DATATYPE_NULL) { 104 | printf("freed handle is not null\n"); 105 | MPI_Abort(MPI_COMM_WORLD,1); 106 | } 107 | } 108 | 109 | { 110 | MPI_Datatype vector = MPI_DATATYPE_NULL; 111 | MPI_Type_vector(10, 10, 20, MPI_BYTE, &vector); 112 | MPI_Type_commit(&vector); 113 | 114 | MPI_Type_set_name(vector,"vector: 10, 10, 20, MPI_BYTE"); 115 | int len; 116 | char name[MPI_MAX_OBJECT_NAME] = {0}; 117 | MPI_Type_get_name(vector, name, &len); 118 | int cmp = strcmp(name, "vector: 10, 10, 20, MPI_BYTE"); 119 | if (cmp != 0) { 120 | printf("name=%s\n", name); 121 | MPI_Abort(MPI_COMM_WORLD,cmp); 122 | } 123 | 124 | int size; 125 | MPI_Type_size(vector, &size); 126 | if (size != 100) { 127 | printf("wrong: size=%d\n", size); 128 | MPI_Abort(MPI_COMM_WORLD,size); 129 | } 130 | 131 | int psize; 132 | MPI_Pack_size(1, vector, MPI_COMM_SELF, &psize); 133 | if (psize != 100) { 134 | printf("wrong: pack size=%d\n", psize); 135 | MPI_Abort(MPI_COMM_WORLD,psize); 136 | } 137 | 138 | MPI_Aint lb, extent; 139 | MPI_Type_get_extent(vector, &lb, &extent); 140 | if (lb !=0 || extent != 190) { 141 | printf("\n"); 142 | printf("wrong: lb=%zu extent=%ld\n", lb, extent); 143 | MPI_Abort(MPI_COMM_WORLD,extent); 144 | } 145 | 146 | int ni, na, nd, combiner; 147 | MPI_Type_get_envelope(vector, &ni, &na, &nd, &combiner); 148 | if (ni != 3 || na != 0 || nd != 1 || combiner != MPI_COMBINER_VECTOR) { 149 | printf("get_envelope: ni=%d na=%d nd=%d combiner=%d\n", ni, na, nd, combiner); 150 | printf("MPI_COMBINER_VECTOR = %d\n", MPI_COMBINER_VECTOR); 151 | MPI_Abort(MPI_COMM_WORLD,combiner); 152 | } 153 | 154 | MPI_Type_free(&vector); 155 | if (vector != MPI_DATATYPE_NULL) { 156 | printf("freed handle is not null\n"); 157 | MPI_Abort(MPI_COMM_WORLD,1); 158 | } 159 | } 160 | 161 | { 162 | MPI_Datatype subarray = MPI_DATATYPE_NULL; 163 | int sizes[2] = {20,20}; 164 | int subsizes[2] = {10,10}; 165 | int starts[2] = {0,0}; 166 | MPI_Type_create_subarray(2, sizes, subsizes, starts, MPI_ORDER_FORTRAN, MPI_BYTE, &subarray); 167 | MPI_Type_commit(&subarray); 168 | 169 | int size; 170 | MPI_Type_size(subarray, &size); 171 | if (size != 100) { 172 | printf("wrong: size=%d\n", size); 173 | MPI_Abort(MPI_COMM_WORLD,size); 174 | } 175 | 176 | int psize; 177 | MPI_Pack_size(1, subarray, MPI_COMM_SELF, &psize); 178 | if (psize != 100) { 179 | printf("wrong: pack size=%d\n", psize); 180 | MPI_Abort(MPI_COMM_WORLD,psize); 181 | } 182 | 183 | MPI_Aint lb, extent; 184 | MPI_Type_get_extent(subarray, &lb, &extent); 185 | if (lb !=0 || extent != 400) { 186 | printf("\n"); 187 | printf("wrong: lb=%zu extent=%ld\n", lb, extent); 188 | MPI_Abort(MPI_COMM_WORLD,extent); 189 | } 190 | 191 | int ni, na, nd, combiner; 192 | MPI_Type_get_envelope(subarray, &ni, &na, &nd, &combiner); 193 | if (ni != 8 || na != 0 || nd != 1 || combiner != MPI_COMBINER_SUBARRAY) { 194 | printf("get_envelope: ni=%d na=%d nd=%d combiner=%d\n", ni, na, nd, combiner); 195 | printf("MPI_COMBINER_SUBARRAY = %d\n", MPI_COMBINER_SUBARRAY); 196 | MPI_Abort(MPI_COMM_WORLD,combiner); 197 | } 198 | 199 | MPI_Type_free(&subarray); 200 | if (subarray != MPI_DATATYPE_NULL) { 201 | printf("freed handle is not null\n"); 202 | MPI_Abort(MPI_COMM_WORLD,1); 203 | } 204 | } 205 | 206 | { 207 | MPI_Datatype subarray = MPI_DATATYPE_NULL; 208 | int sizes[2] = {20,20}; 209 | int subsizes[2] = {10,10}; 210 | int starts[2] = {0,0}; 211 | MPI_Type_create_subarray(2, sizes, subsizes, starts, MPI_ORDER_C, MPI_BYTE, &subarray); 212 | MPI_Type_commit(&subarray); 213 | 214 | int size; 215 | MPI_Type_size(subarray, &size); 216 | if (size != 100) { 217 | printf("wrong: size=%d\n", size); 218 | MPI_Abort(MPI_COMM_WORLD,size); 219 | } 220 | 221 | int psize; 222 | MPI_Pack_size(1, subarray, MPI_COMM_SELF, &psize); 223 | if (psize != 100) { 224 | printf("wrong: pack size=%d\n", psize); 225 | MPI_Abort(MPI_COMM_WORLD,psize); 226 | } 227 | 228 | MPI_Aint lb, extent; 229 | MPI_Type_get_extent(subarray, &lb, &extent); 230 | if (lb !=0 || extent != 400) { 231 | printf("\n"); 232 | printf("wrong: lb=%zu extent=%ld\n", lb, extent); 233 | MPI_Abort(MPI_COMM_WORLD,extent); 234 | } 235 | 236 | int ni, na, nd, combiner; 237 | MPI_Type_get_envelope(subarray, &ni, &na, &nd, &combiner); 238 | if (ni != 8 || na != 0 || nd != 1 || combiner != MPI_COMBINER_SUBARRAY) { 239 | printf("get_envelope: ni=%d na=%d nd=%d combiner=%d\n", ni, na, nd, combiner); 240 | printf("MPI_COMBINER_SUBARRAY = %d\n", MPI_COMBINER_SUBARRAY); 241 | MPI_Abort(MPI_COMM_WORLD,combiner); 242 | } 243 | 244 | MPI_Type_free(&subarray); 245 | if (subarray != MPI_DATATYPE_NULL) { 246 | printf("freed handle is not null\n"); 247 | MPI_Abort(MPI_COMM_WORLD,1); 248 | } 249 | } 250 | 251 | fflush(0); 252 | usleep(1); 253 | MPI_Barrier(MPI_COMM_WORLD); 254 | if (me==0) printf("all done\n"); 255 | 256 | rc = MPI_Finalize(); 257 | 258 | return rc; 259 | } 260 | -------------------------------------------------------------------------------- /testtypes2.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | 10 | #ifdef NOWRAP 11 | #include 12 | #ifdef MPICH 13 | #define TYPE_FORMAT "%x" 14 | #define PRINT_REAL_HANDLE(h) printf("real handle=%x\n",h) 15 | #else 16 | #define TYPE_FORMAT "%p" 17 | #define PRINT_REAL_HANDLE(h) printf("real handle=%p\n",h) 18 | #endif 19 | #else 20 | #include "mpi.h" 21 | #define TYPE_FORMAT "%p" 22 | #define PRINT_REAL_HANDLE(h) printf("real handle=%x=%p\n",*(int*)h,*(void**)h) 23 | #endif 24 | 25 | int main(int argc, char* argv[]) 26 | { 27 | int rc; 28 | rc = MPI_Init(&argc,&argv); 29 | 30 | int me, np; 31 | MPI_Comm_rank(MPI_COMM_WORLD,&me); 32 | MPI_Comm_size(MPI_COMM_WORLD,&np); 33 | printf("I am %d of %d\n", me, np); 34 | fflush(0); 35 | MPI_Barrier(MPI_COMM_WORLD); 36 | usleep(1); 37 | 38 | { 39 | MPI_Datatype c1 = MPI_DATATYPE_NULL; 40 | MPI_Type_contiguous(100, MPI_BYTE, &c1); 41 | if (c1 == MPI_DATATYPE_NULL) { 42 | printf("fail 1\n"); 43 | MPI_Abort(MPI_COMM_WORLD,1); 44 | } 45 | 46 | MPI_Datatype c2 = MPI_DATATYPE_NULL; 47 | MPI_Type_contiguous(100, c1, &c2); 48 | if (c2 == MPI_DATATYPE_NULL) { 49 | printf("fail 2\n"); 50 | MPI_Abort(MPI_COMM_WORLD,2); 51 | } 52 | 53 | int ni, na, nd, combiner; 54 | MPI_Type_get_envelope(c1, &ni, &na, &nd, &combiner); 55 | if (ni != 1 || na != 0 || nd != 1 || combiner != MPI_COMBINER_CONTIGUOUS) { 56 | printf("c1 get_envelope: ni=%d na=%d nd=%d combiner=%d\n", ni, na, nd, combiner); 57 | printf("MPI_COMBINER_CONTIGUOUS = %d\n", MPI_COMBINER_CONTIGUOUS); 58 | MPI_Abort(MPI_COMM_WORLD,3); 59 | } 60 | 61 | int * ai = malloc(sizeof(int)); 62 | MPI_Datatype * ad = malloc(sizeof(MPI_Datatype)); 63 | MPI_Type_get_contents(c1, 1, 0, 1, ai, NULL, ad); 64 | if (ai[0] != 100 || ad[0] != MPI_BYTE) { 65 | printf("c1 get_contents: ai[]=%d ad[]=" TYPE_FORMAT "\n", ai[0], ad[0]); 66 | MPI_Abort(MPI_COMM_WORLD,101); 67 | } 68 | 69 | MPI_Type_get_envelope(c2, &ni, &na, &nd, &combiner); 70 | if (ni != 1 || na != 0 || nd != 1 || combiner != MPI_COMBINER_CONTIGUOUS) { 71 | printf("c2 get_envelope: ni=%d na=%d nd=%d combiner=%d\n", ni, na, nd, combiner); 72 | printf("MPI_COMBINER_CONTIGUOUS = %d\n", MPI_COMBINER_CONTIGUOUS); 73 | MPI_Abort(MPI_COMM_WORLD,4); 74 | } 75 | 76 | MPI_Datatype c3; 77 | MPI_Type_get_contents(c2, 1, 0, 1, ai, NULL, &c3); 78 | if (ai[0] != 100) { 79 | printf("c2 get_contents: ai[]=%d ad[]=" TYPE_FORMAT "\n", ai[0], c3); 80 | MPI_Abort(MPI_COMM_WORLD,101); 81 | } 82 | 83 | // now see if c3=ad[0] is the same as c1... 84 | MPI_Type_get_envelope(c3, &ni, &na, &nd, &combiner); 85 | if (ni != 1 || na != 0 || nd != 1 || combiner != MPI_COMBINER_CONTIGUOUS) { 86 | printf("ad[0] get_envelope: ni=%d na=%d nd=%d combiner=%d\n", ni, na, nd, combiner); 87 | printf("MPI_COMBINER_CONTIGUOUS = %d\n", MPI_COMBINER_CONTIGUOUS); 88 | MPI_Abort(MPI_COMM_WORLD,5); 89 | } 90 | 91 | MPI_Type_get_contents(c3, 1, 0, 1, ai, NULL, ad); 92 | if (ai[0] != 100 || ad[0] != MPI_BYTE) { 93 | printf("ad[0] get_contents: ai[]=%d ad[]=" TYPE_FORMAT "\n", ai[0], ad[0]); 94 | MPI_Abort(MPI_COMM_WORLD,101); 95 | } 96 | 97 | free(ai); 98 | free(ad); 99 | 100 | MPI_Type_free(&c3); 101 | if (c3 != MPI_DATATYPE_NULL) { 102 | printf("c3 freed handle is not null\n"); 103 | MPI_Abort(MPI_COMM_WORLD,13); 104 | } 105 | 106 | MPI_Type_free(&c2); 107 | if (c2 != MPI_DATATYPE_NULL) { 108 | printf("c2 freed handle is not null\n"); 109 | MPI_Abort(MPI_COMM_WORLD,12); 110 | } 111 | 112 | MPI_Type_free(&c1); 113 | if (c1 != MPI_DATATYPE_NULL) { 114 | printf("c1 freed handle is not null\n"); 115 | MPI_Abort(MPI_COMM_WORLD,11); 116 | } 117 | } 118 | 119 | fflush(0); 120 | usleep(1); 121 | MPI_Barrier(MPI_COMM_WORLD); 122 | if (me==0) printf("all done\n"); 123 | 124 | rc = MPI_Finalize(); 125 | 126 | return rc; 127 | } 128 | -------------------------------------------------------------------------------- /testwin.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | 10 | #ifdef NOWRAP 11 | #include 12 | #else 13 | #include "mpi.h" 14 | #endif 15 | 16 | int main(int argc, char* argv[]) 17 | { 18 | int rc; 19 | MPI_Init(&argc,&argv); 20 | 21 | int me, np; 22 | MPI_Comm_rank(MPI_COMM_WORLD,&me); 23 | MPI_Comm_size(MPI_COMM_WORLD,&np); 24 | printf("I am %d of %d\n", me, np); 25 | 26 | MPI_Win a, as, c, cd; 27 | void *ba, *bas; 28 | char bc[1024], b[1024]; 29 | 30 | rc = MPI_Win_allocate(1024,1,MPI_INFO_NULL,MPI_COMM_WORLD,&ba,&a); 31 | if (rc) { 32 | printf("rc = %d line = %d\n", rc, __LINE__); 33 | MPI_Abort(MPI_COMM_WORLD,rc); 34 | } 35 | MPI_Win_fence(0,a); 36 | MPI_Win_free(&a); 37 | if (a != MPI_WIN_NULL) { 38 | printf("win null? %d\n",a==MPI_WIN_NULL); 39 | MPI_Abort(MPI_COMM_WORLD,1); 40 | } 41 | 42 | rc = MPI_Win_allocate_shared(1024,1,MPI_INFO_NULL,MPI_COMM_WORLD,&bas,&as); 43 | if (rc) { 44 | printf("rc = %d line = %d\n", rc, __LINE__); 45 | MPI_Abort(MPI_COMM_WORLD,rc); 46 | } 47 | MPI_Win_fence(0,as); 48 | MPI_Win_free(&as); 49 | if (as != MPI_WIN_NULL) { 50 | printf("win null? %d\n",as==MPI_WIN_NULL); 51 | MPI_Abort(MPI_COMM_WORLD,2); 52 | } 53 | 54 | // this is broken with OMPI and np=1 55 | rc = MPI_Win_create(&bc,1024,1,MPI_INFO_NULL,MPI_COMM_WORLD,&c); 56 | if (rc) { 57 | printf("rc = %d line = %d\n", rc, __LINE__); 58 | MPI_Abort(MPI_COMM_WORLD,rc); 59 | } 60 | MPI_Win_fence(0,c); 61 | MPI_Win_free(&c); 62 | if (c != MPI_WIN_NULL) { 63 | printf("win null? %d\n",c==MPI_WIN_NULL); 64 | MPI_Abort(MPI_COMM_WORLD,3); 65 | } 66 | 67 | rc = MPI_Win_create_dynamic(MPI_INFO_NULL,MPI_COMM_WORLD,&cd); 68 | if (rc) { 69 | printf("rc = %d line = %d\n", rc, __LINE__); 70 | MPI_Abort(MPI_COMM_WORLD,rc); 71 | } 72 | MPI_Win_attach(cd,&b,1024); 73 | MPI_Win_fence(0,cd); 74 | MPI_Win_detach(cd,&b); 75 | MPI_Win_free(&cd); 76 | if (cd != MPI_WIN_NULL) { 77 | printf("win null? %d\n",cd==MPI_WIN_NULL); 78 | MPI_Abort(MPI_COMM_WORLD,4); 79 | } 80 | 81 | fflush(0); 82 | usleep(1); 83 | MPI_Barrier(MPI_COMM_WORLD); 84 | if (me==0) printf("all done\n"); 85 | 86 | MPI_Finalize(); 87 | 88 | return 0; 89 | } 90 | -------------------------------------------------------------------------------- /wrap-handle-typedefs.h: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | #ifndef WRAP_HANDLE_TYPEDEFS_H 4 | #define WRAP_HANDLE_TYPEDEFS_H 5 | 6 | #include "impl-scalar-types.h" 7 | #include "impl-status.h" 8 | 9 | typedef union 10 | { 11 | void * p; 12 | int i; 13 | intptr_t ip; 14 | } 15 | MUK_Handle; 16 | 17 | typedef MUK_Handle WRAP_Comm; 18 | typedef MUK_Handle WRAP_Datatype; 19 | typedef MUK_Handle WRAP_Errhandler; 20 | typedef MUK_Handle WRAP_File; 21 | typedef MUK_Handle WRAP_Group; 22 | typedef MUK_Handle WRAP_Info; 23 | typedef MUK_Handle WRAP_Message; 24 | typedef MUK_Handle WRAP_Op; 25 | typedef MUK_Handle WRAP_Request; 26 | typedef MUK_Handle WRAP_Session; 27 | typedef MUK_Handle WRAP_Win; 28 | 29 | // callback typedefs 30 | 31 | // reductions 32 | typedef void WRAP_User_function(void *invec, void *inoutvec, int *len, WRAP_Datatype *datatype); 33 | typedef void WRAP_User_function_c(void *invec, void *inoutvec, WRAP_Count *len, WRAP_Datatype *datatype); 34 | 35 | // comm attr 36 | typedef int WRAP_Comm_copy_attr_function(WRAP_Comm oldcomm, int comm_keyval, void *extra_state, void *attribute_val_in, void *attribute_val_out, int *flag); 37 | typedef int WRAP_Comm_delete_attr_function(WRAP_Comm comm, int comm_keyval, void *attribute_val, void *extra_state); 38 | // these are deprecated/deleted 39 | typedef int WRAP_Copy_function(WRAP_Comm oldcomm, int keyval, void *extra_state, void *attribute_val_in, void *attribute_val_out, int *flag); 40 | typedef int WRAP_Delete_function(WRAP_Comm comm, int keyval, void *attribute_val, void *extra_state); 41 | 42 | // win attr 43 | typedef int WRAP_Win_copy_attr_function(WRAP_Win oldwin, int win_keyval, void *extra_state, void *attribute_val_in, void *attribute_val_out, int *flag); 44 | typedef int WRAP_Win_delete_attr_function(WRAP_Win win, int win_keyval, void *attribute_val, void *extra_state); 45 | 46 | // type attr 47 | typedef int WRAP_Type_copy_attr_function(WRAP_Datatype oldtype, int type_keyval, void *extra_state, void *attribute_val_in, void *attribute_val_out, int *flag); 48 | typedef int WRAP_Type_delete_attr_function(WRAP_Datatype datatype, int type_keyval, void *attribute_val, void *extra_state); 49 | 50 | // errhandler 51 | typedef void WRAP_Comm_errhandler_function(WRAP_Comm *comm, int *error_code, ...); 52 | typedef void WRAP_Win_errhandler_function(WRAP_Win *win, int *error_code, ...); 53 | typedef void WRAP_File_errhandler_function(WRAP_File *file, int *error_code, ...); 54 | typedef void WRAP_Session_errhandler_function(WRAP_Session *session, int *error_code, ...); 55 | 56 | // grequest 57 | typedef int WRAP_Grequest_query_function(void *extra_state, WRAP_Status *status); 58 | typedef int WRAP_Grequest_free_function(void *extra_state); 59 | typedef int WRAP_Grequest_cancel_function(void *extra_state, int complete); 60 | 61 | // datarep 62 | typedef int WRAP_Datarep_extent_function(WRAP_Datatype datatype, WRAP_Aint *extent, void *extra_state); 63 | typedef int WRAP_Datarep_conversion_function(void *userbuf, WRAP_Datatype datatype, int count, void *filebuf, WRAP_Offset position, void *extra_state); 64 | typedef int WRAP_Datarep_conversion_function_c(void *userbuf, WRAP_Datatype datatype, WRAP_Count count, void *filebuf, WRAP_Offset position, void *extra_state); 65 | 66 | #endif 67 | --------------------------------------------------------------------------------