├── example ├── simple │ └── main.go ├── scalarwave │ ├── loops.go │ ├── main_test.go │ ├── output.go │ ├── storage.go │ ├── main.go │ ├── utility.go │ ├── evolve.go │ └── grid.go ├── alltoall │ └── main.go └── nonblocking │ └── main.go ├── mpi_utils.go ├── LICENSE ├── README.md └── mpi.go /example/simple/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/marcusthierfelder/mpi" 6 | ) 7 | 8 | func main() { 9 | 10 | mpi.Init() 11 | 12 | fmt.Println(mpi.Comm_size(mpi.COMM_WORLD), mpi.Comm_rank(mpi.COMM_WORLD)) 13 | 14 | mpi.Finalize() 15 | 16 | } 17 | -------------------------------------------------------------------------------- /example/scalarwave/loops.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | _ "fmt" 5 | _ "math" 6 | ) 7 | 8 | func (box *Box) innerpoint(i, j, k int) bool { 9 | n := 1 10 | if i > n && j > n && k > n && i < box.nxyz[0]-n && j < box.nxyz[1]-n && k < box.nxyz[2]-n { 11 | return true 12 | } 13 | 14 | return false 15 | } 16 | -------------------------------------------------------------------------------- /example/scalarwave/main_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | func TestInterpolate(t *testing.T) { 9 | 10 | u := [][][]float64{{{0, 0}, {0, 0}}, {{1, 1}, {1, 1}}} 11 | x := [3]float64{1, 1, 1} 12 | dx := [3]float64{2, 2, 2} 13 | x0 := [3]float64{0, 0, 0} 14 | 15 | fmt.Println(u) 16 | fmt.Println(x, interpolate_TriN(x, x0, dx, u)) 17 | 18 | x = [3]float64{0, 0, 0} 19 | fmt.Println(x, interpolate_TriN(x, x0, dx, u)) 20 | x = [3]float64{0, 1, 1.5} 21 | fmt.Println(x, interpolate_TriN(x, x0, dx, u)) 22 | 23 | t.Errorf("---") 24 | 25 | } 26 | -------------------------------------------------------------------------------- /example/alltoall/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/marcusthierfelder/mpi" 6 | ) 7 | 8 | func main() { 9 | 10 | mpi.Init() 11 | 12 | chunk := 4 * mpi.Comm_size(mpi.COMM_WORLD) 13 | rank := mpi.Comm_rank(mpi.COMM_WORLD) 14 | //size := mpi.Comm_size(mpi.COMM_WORLD) 15 | 16 | sb := make([]int, chunk) 17 | rb := make([]int, chunk) 18 | 19 | for i := 0; i < chunk; i++ { 20 | sb[i] = rank + 1 21 | rb[i] = 0 22 | } 23 | 24 | //mpi.Alltoall_int(sb, rb, mpi.COMM_WORLD) 25 | mpi.Alltoall(sb, rb, mpi.COMM_WORLD) 26 | 27 | fmt.Println(sb, rb) 28 | 29 | mpi.Finalize() 30 | 31 | } 32 | -------------------------------------------------------------------------------- /mpi_utils.go: -------------------------------------------------------------------------------- 1 | package mpi 2 | 3 | /* 4 | #include 5 | #include 6 | #include "mpi.h" 7 | #cgo LDFLAGS: -lmpi 8 | #cgo CFLAGS: -std=gnu99 -Wall 9 | 10 | void call_freopen(char *file) { 11 | freopen(file, "w", stdout); 12 | freopen(file, "w", stderr); 13 | } 14 | 15 | */ 16 | import "C" 17 | 18 | import ( 19 | "fmt" 20 | ) 21 | 22 | 23 | // if you want to redirect each stdout per process in a file 24 | // in order to avoid a parallel stdout mess (no standard mpi function) 25 | func Redirect_STDOUT(comm C.MPI_Comm) { 26 | rank := Comm_rank(comm) 27 | file := fmt.Sprintf("stdout.%04d", rank) 28 | 29 | if rank != 0 { 30 | C.call_freopen(C.CString(file)) 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Marcus Thierfelder 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /example/nonblocking/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/marcusthierfelder/mpi" 6 | ) 7 | 8 | func main() { 9 | 10 | mpi.Init() 11 | 12 | chunk := 10 13 | rank := mpi.Comm_rank(mpi.COMM_WORLD) 14 | size := mpi.Comm_size(mpi.COMM_WORLD) 15 | 16 | sb := make([]int, chunk) 17 | rb := make([]int, chunk) 18 | sbf := make([]float64, chunk) 19 | rbf := make([]float64, chunk) 20 | 21 | for i := 0; i < chunk; i++ { 22 | sb[i] = rank + 1 23 | rb[i] = 0 24 | sbf[i] = float64(rank + 1) 25 | rbf[i] = 0. 26 | } 27 | 28 | var request, request2 mpi.Request 29 | var status mpi.Status 30 | 31 | right := (rank + 1) % size 32 | left := rank - 1 33 | if left < 0 { 34 | left = size - 1 35 | } 36 | mpi.Irecv_int(rb, left, 123, mpi.COMM_WORLD, &request) 37 | mpi.Isend_int(sb, right, 123, mpi.COMM_WORLD, &request2) 38 | mpi.Wait(&request, &status) 39 | mpi.Wait(&request2, &status) 40 | 41 | fmt.Println(sb, rb) 42 | 43 | 44 | mpi.Irecv_float64(rbf, left, 1234, mpi.COMM_WORLD, &request) 45 | mpi.Isend_float64(sbf, right, 1234, mpi.COMM_WORLD, &request2) 46 | mpi.Wait(&request, &status) 47 | mpi.Wait(&request2, &status) 48 | 49 | fmt.Println(sbf, rbf) 50 | 51 | 52 | mpi.Finalize() 53 | 54 | } 55 | -------------------------------------------------------------------------------- /example/scalarwave/output.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/marcusthierfelder/mpi" 6 | "io" 7 | "log" 8 | "os" 9 | _ "reflect" 10 | ) 11 | 12 | func (grid *Grid) output() { 13 | defer un(trace("output")) 14 | 15 | grid.output_1d("f", "f.x", 0) 16 | grid.output_1d("g", "g.x", 0) 17 | } 18 | 19 | /* this a a very slow and lazy implementation for x-direction */ 20 | func (grid *Grid) output_1d(data string, file string, d int) { 21 | 22 | /* init buffer */ 23 | buffer := make([]float64, 2*grid.nxyz[d]) 24 | x := make([]float64, grid.nxyz[d]) 25 | ptr := grid.GetVar(data) 26 | pos := [3]float64{0., 0., 0.} 27 | 28 | for i := 0; i < grid.nxyz[d]; i++ { 29 | x[i] = grid.xyz0[d] + float64(i)*grid.dxyz[d] 30 | pos[d] = x[i] 31 | v, b := grid.box.interpolate(pos, ptr) 32 | buffer[i], buffer[grid.nxyz[d]+i] = v, float64(btoi(b)) 33 | } 34 | //fmt.Println(rank, x,buffer[:grid.nxyz[d]]) 35 | 36 | /* find data using interpolation */ 37 | recvbuf := make([]float64, 2*grid.nxyz[d]) 38 | mpi.Allreduce_float64(buffer, recvbuf, mpi.SUM, mpi.COMM_WORLD) 39 | for i := 0; i < grid.nxyz[d]; i++ { 40 | buffer[i] = recvbuf[i] / recvbuf[i+grid.nxyz[d]] 41 | } 42 | //fmt.Println(recvbuf) 43 | 44 | /* create or append? */ 45 | if proc0 { 46 | var flag int 47 | if grid.time == 0. { 48 | flag = os.O_CREATE | os.O_TRUNC | os.O_RDWR 49 | } else { 50 | flag = os.O_APPEND | os.O_RDWR 51 | } 52 | 53 | f, err := os.OpenFile(file, flag, 0666) 54 | if err != nil { 55 | log.Fatal(err) 56 | return 57 | } 58 | 59 | fmt.Println("write " + file) 60 | io.WriteString(f, fmt.Sprintf("#Time = %e\n", grid.time)) 61 | for i := 0; i < grid.nxyz[d]; i++ { 62 | io.WriteString(f, fmt.Sprintf("%e %e\n", x[i], buffer[i])) 63 | } 64 | io.WriteString(f, "\n") 65 | 66 | f.Close() 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /example/scalarwave/storage.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | _ "reflect" 7 | ) 8 | 9 | /* variable storage stuff */ 10 | func (grid *Grid) AddVar(name string, sync bool) { 11 | fmt.Println("AddVar: ", name) 12 | 13 | if false { 14 | fmt.Println(grid.field, len(grid.field)) 15 | } 16 | 17 | l := len(grid.field) 18 | tmp := make([]Field, l+1) 19 | copy(tmp, grid.field) 20 | grid.field = tmp 21 | 22 | f := Field{name: name, sync: sync, 23 | data: make([]float64, grid.box.nxyz[0]*grid.box.nxyz[1]*grid.box.nxyz[2])} 24 | 25 | grid.field[l] = f 26 | } 27 | 28 | func (grid *Grid) GetVar(name string) []float64 { 29 | ptr := grid.GetField(name) 30 | return ptr.data 31 | } 32 | 33 | func (grid *Grid) GetField(name string) *Field { 34 | var ptr *Field 35 | 36 | ptr = nil 37 | for i, f := range grid.field { 38 | if f.name == name { 39 | ptr = &(grid.field[i]) 40 | // this DOES NOT WORK, why? 41 | // ptr =&f 42 | } 43 | } 44 | 45 | if ptr == nil { 46 | log.Fatal("var \"" + name + "\" does not exist") 47 | } 48 | return ptr 49 | } 50 | 51 | func (grid *Grid) PrintVars() { 52 | fmt.Println("Variables:") 53 | for i, v := range grid.field { 54 | fmt.Println(" ", i, v.name) 55 | } 56 | } 57 | 58 | /* varlist stuff */ 59 | func (grid *Grid) vlalloc() VarList { 60 | var vl VarList 61 | vl.grid = grid 62 | 63 | return vl 64 | } 65 | 66 | func (vl *VarList) AddVar(name string) { 67 | l := len(vl.field) 68 | tmp := make([]*Field, l+1) 69 | copy(tmp, vl.field) 70 | vl.field = tmp 71 | 72 | vl.field[l] = vl.grid.GetField(name) 73 | } 74 | 75 | func (vl *VarList) GetVar(i int) []float64 { 76 | if i >= len(vl.field) { 77 | log.Fatal("GetVar, number out of list") 78 | } 79 | return vl.field[i].data 80 | } 81 | 82 | func (vl *VarList) PrintVars() { 83 | fmt.Println("VarList:") 84 | for i, v := range vl.field { 85 | fmt.Println(" ", i, v.name) 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /example/scalarwave/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/marcusthierfelder/mpi" 6 | ) 7 | 8 | var ( 9 | testMPI bool = false 10 | rank, size int = 0, 1 11 | proc0 bool = false 12 | stdout string 13 | ) 14 | 15 | type Grid struct { 16 | xyz0, dxyz [3]float64 // there are no ghost points included 17 | nxyz [3]int // ... as in here 18 | gh int // number of ghosts 19 | time float64 // time ... 20 | 21 | box Box // local informations 22 | field []Field // data storage 23 | } 24 | type Box struct { 25 | xyz0, xyz1 [3]float64 26 | dxyz [3]float64 27 | nxyz [3]int 28 | noff [3]int 29 | 30 | di, dj, dk int 31 | oodx, oodx2 [3]float64 32 | 33 | comm Comm 34 | grid *Grid 35 | } 36 | type Field struct { 37 | name string 38 | sync bool 39 | data []float64 40 | } 41 | type Comm struct { 42 | neighbour [6]int // number of touching processor 43 | npts [6]int // number of points which have to be syncd 44 | send, recv [6][]int // stack of position(ijk) to sync efficiently 45 | } 46 | 47 | type VarList struct { 48 | field []*Field 49 | grid *Grid 50 | } 51 | 52 | func main() { 53 | 54 | if testMPI == false { 55 | mpi.Init() 56 | size = mpi.Comm_size(mpi.COMM_WORLD) 57 | rank = mpi.Comm_rank(mpi.COMM_WORLD) 58 | 59 | mpi.Redirect_STDOUT(mpi.COMM_WORLD) 60 | } 61 | proc0 = rank == 0 62 | fmt.Println(rank, proc0) 63 | 64 | var grid Grid 65 | grid.nxyz = [3]int{11, 11, 11} 66 | grid.dxyz = [3]float64{1, 1, 1} 67 | grid.xyz0 = [3]float64{-5., -5., -5.} 68 | grid.gh = 1 69 | dt := 0.1 70 | 71 | grid.create() 72 | grid.init() 73 | 74 | vl := grid.initialdata() 75 | grid.rk4_init(vl) 76 | 77 | grid.output() 78 | 79 | for ti := 0; ti < 1; ti++ { 80 | if proc0 { 81 | fmt.Printf(" %4d %2.4f\n", ti, grid.time) 82 | } 83 | grid.rk4(vl, dt) 84 | grid.time += dt 85 | //grid.sync_all() 86 | 87 | //grid.output() 88 | } 89 | 90 | if testMPI == false { 91 | mpi.Finalize() 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /example/scalarwave/utility.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | _ "os" 6 | ) 7 | 8 | var pr = true 9 | 10 | /* some helpers to see which function is called */ 11 | func trace(s string) string { 12 | if pr == true { 13 | fmt.Println("entering:", s) 14 | } else { 15 | fmt.Print("") 16 | } 17 | return s 18 | } 19 | 20 | func un(s string) { 21 | if pr == true { 22 | fmt.Println("leaving: ", s) 23 | } else { 24 | fmt.Print("") 25 | } 26 | } 27 | 28 | /* lagrange interpolation functions */ 29 | func coefficients_lagrange_N(x, xmin, h float64, c []float64) { 30 | n := len(c) 31 | 32 | for i := 0; i < n; i++ { 33 | d := 1. 34 | for j := 0; j < n; j++ { 35 | if j != i { 36 | d *= (x - xmin - float64(j)*h) / (float64(i-j) * h) 37 | } 38 | } 39 | c[i] = d 40 | } 41 | } 42 | 43 | func interpolate_lagrange_N(x, xmin, h float64, c, u []float64) float64 { 44 | n := len(c) 45 | sum := 0. 46 | 47 | for i := 0; i < n; i++ { 48 | sum += c[i] * u[i] 49 | } 50 | 51 | return sum 52 | } 53 | 54 | func interpolate_TriN(x, xmin, dx [3]float64, u [][][]float64) float64 { 55 | 56 | //fmt.Println("-----") 57 | //fmt.Println(x, xmin, dx) 58 | 59 | n := len(u) 60 | sum := 3.141 61 | c := make([]float64, n) 62 | v := make([][]float64, n) 63 | for i := 0; i < n; i++ { 64 | v[i] = make([]float64, n) 65 | } 66 | w := make([]float64, n) 67 | 68 | //fmt.Println(u) 69 | 70 | coefficients_lagrange_N(x[2], xmin[2], dx[2], c) 71 | for i := 0; i < n; i++ { 72 | for j := 0; j < n; j++ { 73 | v[i][j] = interpolate_lagrange_N(x[2], xmin[2], dx[2], c, u[i][j]) 74 | } 75 | } 76 | //fmt.Println(c,v) 77 | 78 | coefficients_lagrange_N(x[1], xmin[1], dx[1], c) 79 | for i := 0; i < n; i++ { 80 | w[i] = interpolate_lagrange_N(x[1], xmin[1], dx[1], c, v[i]) 81 | } 82 | //fmt.Println(c,w) 83 | 84 | coefficients_lagrange_N(x[0], xmin[0], dx[0], c) 85 | sum = interpolate_lagrange_N(x[0], xmin[0], dx[0], c, w) 86 | 87 | //fmt.Println(c,sum) 88 | //os.Exit(1) 89 | 90 | return sum 91 | } 92 | 93 | /* some other stuff */ 94 | func btoi(b bool) int { 95 | if b { 96 | return 1 97 | } 98 | return 0 99 | } 100 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | mpi 2 | === 3 | mpi-binding package for golang. It is created and tested for openmpi and 4 | should work with all other mpi libraries. Only a small fraction of methods 5 | are already implemented... more are comming if needed. The names are quite 6 | similar to the mpi-library and they should do the same. 7 | 8 | It is tested for golang go1.0.2,go1.0.3 and Open MPI v1.6.2 9 | see 10 | 11 | http://www.open-mpi.org/doc/v1.6/ 12 | 13 | 14 | 15 | Quick Usage 16 | =========== 17 | 18 | go get github.com/marcusthierfelder/mpi 19 | 20 | 21 | 22 | Detailed Usage (linux) 23 | ====================== 24 | In order to generate the binding library you need to install mpi on you system. 25 | On ubuntu/debain use: 26 | 27 | sudo apt-get install openmpi-dev openmpi-common 28 | 29 | To install this library, cgo needs the location of mpi-header (mpi.h) and the 30 | mpi-library (mpi.a). Sometimes the system already "knows" these locations. 31 | If not, you have to find them and export the path. On my system I needed: 32 | 33 | export C_INCLUDE_PATH=/usr/include/openmpi 34 | export LD_LIBRARY_PATH=/usr/lib/openmpi/lib 35 | 36 | On some machines the compiler does not use LD_LIBRARY_PATH, then try: 37 | 38 | export LIBRARY_PATH=/usr/lib/openmpi/lib 39 | 40 | To start a parallel job on 4 cores do: 41 | mpirun -np 4 my_prog 42 | 43 | 44 | 45 | Detailed Usage (mac osx) 46 | ======================== 47 | You need gcc in order to compile mpi and bind it to golang. Easiest way to get gcc is by 48 | installing xcode and and the command line tools (xcode -> preferences -> downloads). 49 | Afterwards gcc should work. 50 | 51 | To install mpi use this page: 52 | 53 | https://sites.google.com/site/dwhipp/tutorials/installing-open-mpi-on-mac-os-x 54 | 55 | or in short, download the newest version of openmpi (currently it is 1.6.5) and 56 | untar it somewhere. Use the terminal and go into the folder 57 | 58 | ./configure 59 | make 60 | sudo make install 61 | 62 | 63 | 64 | Examples 65 | ======== 66 | 67 | ### simple: 68 | 69 | simple example which writes the rank of each processor and the 70 | total number of mpi-jobs 71 | 72 | ### alltoall: 73 | 74 | simple example which communicates a small number of integer between all 75 | processors 76 | 77 | ### nonblocking: 78 | 79 | simple send recv example 80 | 81 | ### scalarwave: 82 | 83 | fancy scalarwave example in 3d with goroutines and mpi decomposition. 84 | you can choose between several integrators and orders of finite differencing. 85 | there are only simple boundary and output options. 86 | does not work properly yet. 87 | 88 | note: this is not optimised, but can be used to test clusters for scaling etc. 89 | 90 | 91 | 92 | Errors/Problems 93 | =============== 94 | 95 | If something is wrong or not working or missing, feel free to contact me 96 | or post an issue on github.com. 97 | 98 | 99 | 100 | 101 | 102 | -------------------------------------------------------------------------------- /example/scalarwave/evolve.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | _ "fmt" 5 | "math" 6 | ) 7 | 8 | var ( 9 | tmp_vl []VarList 10 | ) 11 | 12 | /* intial data */ 13 | func (grid *Grid) initialdata() VarList { 14 | defer un(trace("initaldata")) 15 | grid.AddVar("f", true) 16 | grid.AddVar("g", true) 17 | 18 | vl := grid.vlalloc() 19 | vl.AddVar("f") 20 | vl.AddVar("g") 21 | 22 | box := grid.box 23 | f := grid.GetVar("f") 24 | g := grid.GetVar("g") 25 | x := grid.GetVar("x") 26 | y := grid.GetVar("y") 27 | z := grid.GetVar("z") 28 | 29 | ijk := 0 30 | for k := 0; k < box.nxyz[2]; k++ { 31 | for j := 0; j < box.nxyz[1]; j++ { 32 | for i := 0; i < box.nxyz[0]; i++ { 33 | r := math.Sqrt(x[ijk]*x[ijk] + y[ijk]*y[ijk] + z[ijk]*z[ijk]) 34 | 35 | f[ijk] = math.Exp(-r * r) 36 | g[ijk] = 0. 37 | 38 | ijk++ 39 | } 40 | } 41 | } 42 | 43 | return vl 44 | } 45 | 46 | /* rhs computation */ 47 | func (grid *Grid) rhs(r VarList, uc VarList) { 48 | 49 | box := grid.box 50 | f := uc.GetVar(0) 51 | g := uc.GetVar(1) 52 | rf := r.GetVar(0) 53 | rg := r.GetVar(1) 54 | 55 | di := box.di 56 | dj := box.dj 57 | dk := box.dk 58 | 59 | ijk := 0 60 | for k := 0; k < box.nxyz[2]; k++ { 61 | for j := 0; j < box.nxyz[1]; j++ { 62 | for i := 0; i < box.nxyz[0]; i++ { 63 | 64 | if box.innerpoint(i, j, k) { 65 | // second order laplacian 66 | laplace := box.oodx2[0] * (-6.*f[ijk] + 67 | f[ijk-di] + f[ijk+di] + f[ijk-dj] + f[ijk+dj] + f[ijk-dk] + f[ijk+dk]) 68 | 69 | rf[ijk] = g[ijk] 70 | rg[ijk] = laplace 71 | 72 | } else { 73 | // simple boundary condition 74 | rf[ijk] = 0. 75 | rg[ijk] = 0. 76 | 77 | } 78 | ijk++ 79 | } 80 | } 81 | } 82 | 83 | grid.sync_vl(r) 84 | } 85 | 86 | /* helper functions for the time integrator */ 87 | func (grid *Grid) cpy(vl1 VarList, vl2 VarList) { 88 | ntot := grid.box.nxyz[0] * grid.box.nxyz[1] * grid.box.nxyz[2] 89 | 90 | for i, v1 := range vl1.field { 91 | v2 := vl2.field[i] 92 | 93 | for ijk := 0; ijk < ntot; ijk++ { 94 | v1.data[ijk] = v2.data[ijk] 95 | } 96 | } 97 | } 98 | 99 | func (grid *Grid) addto(vl1 VarList, c float64, vl2 VarList) { 100 | ntot := grid.box.nxyz[0] * grid.box.nxyz[1] * grid.box.nxyz[2] 101 | 102 | for i, v1 := range vl1.field { 103 | v2 := vl2.field[i] 104 | 105 | for ijk := 0; ijk < ntot; ijk++ { 106 | v1.data[ijk] += c * v2.data[ijk] 107 | } 108 | } 109 | } 110 | 111 | func (grid *Grid) add(vl1 VarList, c2 float64, vl2 VarList, c3 float64, vl3 VarList) { 112 | ntot := grid.box.nxyz[0] * grid.box.nxyz[1] * grid.box.nxyz[2] 113 | 114 | for i, v1 := range vl1.field { 115 | v2 := vl2.field[i] 116 | v3 := vl3.field[i] 117 | 118 | for ijk := 0; ijk < ntot; ijk++ { 119 | v1.data[ijk] = c2*v2.data[ijk] + c3*v3.data[ijk] 120 | 121 | } 122 | } 123 | } 124 | 125 | func (grid *Grid) rk4_init(uc VarList) { 126 | defer un(trace("rk4_init")) 127 | 128 | tmp_vl = []VarList{grid.vlalloc(), grid.vlalloc(), grid.vlalloc()} 129 | 130 | for _, v := range uc.field { 131 | grid.AddVar(v.name+"_p", v.sync) 132 | tmp_vl[0].AddVar(v.name + "_p") 133 | grid.AddVar(v.name+"_r", v.sync) 134 | tmp_vl[1].AddVar(v.name + "_r") 135 | grid.AddVar(v.name+"_v", v.sync) 136 | tmp_vl[2].AddVar(v.name + "_v") 137 | } 138 | } 139 | 140 | func (grid *Grid) rk4(uc VarList, dt float64) { 141 | 142 | p := tmp_vl[0] 143 | r := tmp_vl[1] 144 | v := tmp_vl[2] 145 | 146 | grid.cpy(p, uc) 147 | 148 | grid.rhs(r, uc) 149 | grid.addto(uc, dt/6., r) 150 | 151 | grid.add(v, 1.0, p, dt/2., r) 152 | grid.rhs(r, v) 153 | grid.addto(uc, dt/3., r) 154 | 155 | grid.add(v, 1.0, p, dt/2., r) 156 | grid.rhs(r, v) 157 | grid.addto(uc, dt/3., r) 158 | 159 | grid.add(v, 1.0, p, dt, r) 160 | grid.rhs(r, v) 161 | grid.addto(uc, dt/6., r) 162 | 163 | } 164 | -------------------------------------------------------------------------------- /example/scalarwave/grid.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/marcusthierfelder/mpi" 6 | "log" 7 | "math" 8 | 9 | _ "os" 10 | ) 11 | 12 | var ( 13 | eps float64 = 1e-10 14 | ) 15 | 16 | func compGridSize(nijk [3]int, nxyz [3]int) ([][]int, float64) { 17 | 18 | // compute splitting in each direction 19 | nx := make([]int, nijk[0]) 20 | sum := 0 21 | for i := 0; i < nijk[0]-1; i++ { 22 | nx[i] = nxyz[0] / nijk[0] 23 | sum += nx[i] 24 | } 25 | nx[nijk[0]-1] = nxyz[0] - sum 26 | 27 | ny := make([]int, nijk[1]) 28 | sum = 0 29 | for i := 0; i < nijk[1]-1; i++ { 30 | ny[i] = nxyz[1] / nijk[1] 31 | sum += ny[i] 32 | } 33 | ny[nijk[1]-1] = nxyz[1] - sum 34 | 35 | nz := make([]int, nijk[2]) 36 | sum = 0 37 | for i := 0; i < nijk[2]-1; i++ { 38 | nz[i] = nxyz[2] / nijk[2] 39 | sum += nz[i] 40 | } 41 | nz[nijk[2]-1] = nxyz[2] - sum 42 | 43 | //fmt.Println(nijk,nxyz) 44 | //fmt.Println(nx,ny,nz) 45 | 46 | // now greate size of each processor 47 | nprocs := nijk[0] * nijk[1] * nijk[2] 48 | size := make([][]int, nprocs) 49 | //fmt.Println(nprocs,size) 50 | for i := range size { 51 | size[i] = make([]int, 3) 52 | 53 | size[i][0] = nx[i%nijk[0]] 54 | size[i][1] = ny[(i/nijk[0])%nijk[1]] 55 | size[i][2] = nz[(i/(nijk[0]*nijk[1]))%nijk[2]] 56 | //fmt.Println(i, size) 57 | } 58 | 59 | // compute volume and surface to choose best ratio 60 | // -> equil to most efficient parallelization 61 | V := float64(nxyz[0] * nxyz[1] * nxyz[2]) 62 | A := float64(nxyz[0]*nxyz[1]*(nijk[2]+1) + 63 | nxyz[0]*nxyz[2]*(nijk[1]+1) + 64 | nxyz[1]*nxyz[2]*(nijk[0]+1)) 65 | 66 | return size, V / A 67 | } 68 | 69 | func splitGrid(nxyz [3]int, nproc int) ([][]int, [3]int) { 70 | var procs [10000][3]int 71 | 72 | l := 0 73 | lmax := 0 74 | rmax := 0. 75 | // test all parallelization versions and test which 76 | // splitting is most efficient 77 | for i := 1; i <= nproc; i++ { 78 | if nproc%i == 0 { 79 | for j := 1; j <= nproc; j++ { 80 | if nproc%(i*j) == 0 || i*j <= nproc { 81 | for k := 1; k <= nproc; k++ { 82 | if i*j*k == nproc { 83 | procs[l][0] = i 84 | procs[l][1] = j 85 | procs[l][2] = k 86 | 87 | nijk := [3]int{i, j, k} 88 | _, r := compGridSize(nijk, nxyz) 89 | //fmt.Println(g,r) 90 | if r > rmax { 91 | rmax = r 92 | lmax = l 93 | } 94 | 95 | l++ 96 | if l > 10000 { 97 | log.Fatal("splitGrid: you use too many processors") 98 | } 99 | } 100 | } 101 | } 102 | } 103 | } 104 | } 105 | 106 | //fmt.Println(lmax) 107 | // take the best choice 108 | nijk := [3]int{procs[lmax][0], procs[lmax][1], procs[lmax][2]} 109 | size, _ := compGridSize(nijk, nxyz) 110 | //fmt.Println("") 111 | //fmt.Println(size,r) 112 | 113 | return size, nijk 114 | } 115 | 116 | func (grid *Grid) create() { 117 | defer un(trace("createGrid")) 118 | //fmt.Println(rank, size, proc0) 119 | 120 | g, nijk := splitGrid(grid.nxyz, size) 121 | //fmt.Println(g, nijk) 122 | 123 | // set box sizes for each processor 124 | grid.box.nxyz[0] = g[rank][0] + 2*grid.gh 125 | grid.box.dxyz[0] = grid.dxyz[0] 126 | grid.box.xyz0[0] = grid.xyz0[0] 127 | for i := 0; i < rank%nijk[0]; i++ { 128 | grid.box.xyz0[0] += grid.dxyz[0] * float64(g[i][0]) 129 | } 130 | grid.box.xyz0[0] -= grid.dxyz[0] * float64(grid.gh) 131 | grid.box.xyz1[0] = grid.box.xyz0[0] + float64(grid.box.nxyz[0]-1)*grid.dxyz[0] 132 | 133 | grid.box.nxyz[1] = g[rank][1] + 2*grid.gh 134 | grid.box.dxyz[1] = grid.dxyz[1] 135 | grid.box.xyz0[1] = grid.xyz0[1] 136 | for i := 0; i < (rank/nijk[0])%nijk[1]; i++ { 137 | grid.box.xyz0[1] += grid.dxyz[1] * float64(g[i][1]) 138 | } 139 | grid.box.xyz0[1] -= grid.dxyz[1] * float64(grid.gh) 140 | grid.box.xyz1[1] = grid.box.xyz0[1] + float64(grid.box.nxyz[1]-1)*grid.dxyz[1] 141 | 142 | grid.box.nxyz[2] = g[rank][2] + 2*grid.gh 143 | grid.box.dxyz[2] = grid.dxyz[2] 144 | grid.box.xyz0[2] = grid.xyz0[2] 145 | for i := 0; i < (rank/(nijk[0]*nijk[1]))%nijk[2]; i++ { 146 | grid.box.xyz0[2] += grid.dxyz[2] * float64(g[i][2]) 147 | } 148 | grid.box.xyz0[2] -= grid.dxyz[2] * float64(grid.gh) 149 | grid.box.xyz1[2] = grid.box.xyz0[2] + float64(grid.box.nxyz[2]-1)*grid.dxyz[2] 150 | 151 | //fmt.Println(grid.box) 152 | 153 | // helpers 154 | for i := 0; i < 3; i++ { 155 | grid.box.oodx[i] = 1. / grid.box.dxyz[i] 156 | grid.box.oodx2[i] = 1. / (grid.box.dxyz[i] * grid.box.dxyz[i]) 157 | } 158 | grid.box.di = 1 159 | grid.box.dj = grid.box.nxyz[0] 160 | grid.box.dk = grid.box.nxyz[0] * grid.box.nxyz[1] 161 | 162 | // find neighbours 163 | i := rank % nijk[0] 164 | j := (rank / nijk[0]) % nijk[1] 165 | k := (rank / (nijk[0] * nijk[1])) % nijk[2] 166 | if i > 0 { 167 | grid.box.comm.neighbour[0] = (i - 1) + j*nijk[0] + k*nijk[0]*nijk[1] 168 | } else { 169 | grid.box.comm.neighbour[0] = -1 170 | } 171 | if i < nijk[0]-1 { 172 | grid.box.comm.neighbour[1] = (i + 1) + j*nijk[0] + k*nijk[0]*nijk[1] 173 | } else { 174 | grid.box.comm.neighbour[1] = -1 175 | } 176 | if j > 0 { 177 | grid.box.comm.neighbour[2] = i + (j-1)*nijk[0] + k*nijk[0]*nijk[1] 178 | } else { 179 | grid.box.comm.neighbour[2] = -1 180 | } 181 | if j < nijk[1]-1 { 182 | grid.box.comm.neighbour[3] = i + (j+1)*nijk[0] + k*nijk[0]*nijk[1] 183 | } else { 184 | grid.box.comm.neighbour[3] = -1 185 | } 186 | if k > 0 { 187 | grid.box.comm.neighbour[4] = i + j*nijk[0] + (k-1)*nijk[0]*nijk[1] 188 | } else { 189 | grid.box.comm.neighbour[4] = -1 190 | } 191 | if k < nijk[2]-1 { 192 | grid.box.comm.neighbour[5] = i + j*nijk[0] + (k+1)*nijk[0]*nijk[1] 193 | } else { 194 | grid.box.comm.neighbour[5] = -1 195 | } 196 | 197 | // set communication buffers 198 | // for each of the 6 sides, go through gh points and store ijk in a 199 | // buffer and send it to the neighbour processor - if there are one 200 | grid.box.comm.npts[0] = grid.gh * grid.box.nxyz[1] * grid.box.nxyz[2] 201 | grid.box.comm.npts[1] = grid.gh * grid.box.nxyz[1] * grid.box.nxyz[2] 202 | grid.box.comm.npts[2] = grid.gh * grid.box.nxyz[0] * grid.box.nxyz[2] 203 | grid.box.comm.npts[3] = grid.gh * grid.box.nxyz[0] * grid.box.nxyz[2] 204 | grid.box.comm.npts[4] = grid.gh * grid.box.nxyz[0] * grid.box.nxyz[1] 205 | grid.box.comm.npts[5] = grid.gh * grid.box.nxyz[0] * grid.box.nxyz[1] 206 | for i := 0; i < 6; i++ { 207 | grid.box.comm.send[i] = make([]int, grid.box.comm.npts[i]) 208 | grid.box.comm.recv[i] = make([]int, grid.box.comm.npts[i]) 209 | } 210 | 211 | ijk0 := [12]int{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} 212 | ijk := 0 213 | for k := 0; k < grid.box.nxyz[2]; k++ { 214 | for j := 0; j < grid.box.nxyz[1]; j++ { 215 | for i := 0; i < grid.box.nxyz[0]; i++ { 216 | 217 | // x dir 218 | if i < grid.gh { 219 | grid.box.comm.send[0][ijk0[0]] = ijk 220 | ijk0[0]++ 221 | } 222 | if i >= grid.gh && i < 2*grid.gh { 223 | grid.box.comm.recv[0][ijk0[6]] = ijk 224 | ijk0[6]++ 225 | } 226 | if i >= grid.box.nxyz[0]-grid.gh { 227 | grid.box.comm.send[1][ijk0[1]] = ijk 228 | ijk0[1]++ 229 | } 230 | if i < grid.box.nxyz[0]-grid.gh && i >= grid.box.nxyz[0]-2*grid.gh { 231 | grid.box.comm.recv[1][ijk0[7]] = ijk 232 | ijk0[7]++ 233 | } 234 | 235 | // y dir 236 | if j < grid.gh { 237 | grid.box.comm.send[2][ijk0[2]] = ijk 238 | ijk0[2]++ 239 | } 240 | if j >= grid.gh && j < 2*grid.gh { 241 | grid.box.comm.recv[2][ijk0[8]] = ijk 242 | ijk0[8]++ 243 | } 244 | if j >= grid.box.nxyz[1]-grid.gh { 245 | grid.box.comm.send[3][ijk0[3]] = ijk 246 | ijk0[3]++ 247 | } 248 | if j < grid.box.nxyz[1]-grid.gh && j >= grid.box.nxyz[1]-2*grid.gh { 249 | grid.box.comm.recv[3][ijk0[9]] = ijk 250 | ijk0[9]++ 251 | } 252 | 253 | // z dir 254 | if k < grid.gh { 255 | grid.box.comm.send[4][ijk0[4]] = ijk 256 | ijk0[4]++ 257 | } 258 | if k >= grid.gh && k < 2*grid.gh { 259 | grid.box.comm.recv[4][ijk0[10]] = ijk 260 | ijk0[10]++ 261 | } 262 | if k >= grid.box.nxyz[2]-grid.gh { 263 | grid.box.comm.send[5][ijk0[5]] = ijk 264 | ijk0[5]++ 265 | } 266 | if k < grid.box.nxyz[2]-grid.gh && k >= grid.box.nxyz[2]-2*grid.gh { 267 | grid.box.comm.recv[5][ijk0[11]] = ijk 268 | ijk0[11]++ 269 | } 270 | 271 | ijk++ 272 | } 273 | } 274 | } 275 | 276 | if false { 277 | fmt.Println(grid.box) 278 | } 279 | } 280 | 281 | func (grid *Grid) init() { 282 | defer un(trace("initGrid")) 283 | 284 | grid.AddVar("x", false) 285 | grid.AddVar("y", false) 286 | grid.AddVar("z", false) 287 | 288 | xp := grid.GetVar("x") 289 | yp := grid.GetVar("y") 290 | zp := grid.GetVar("z") 291 | 292 | ijk := 0 293 | for k := 0; k < grid.box.nxyz[2]; k++ { 294 | for j := 0; j < grid.box.nxyz[1]; j++ { 295 | for i := 0; i < grid.box.nxyz[0]; i++ { 296 | xp[ijk] = grid.box.xyz0[0] + float64(i)*grid.box.dxyz[0] 297 | yp[ijk] = grid.box.xyz0[1] + float64(j)*grid.box.dxyz[1] 298 | zp[ijk] = grid.box.xyz0[2] + float64(k)*grid.box.dxyz[2] 299 | 300 | ijk++ 301 | } 302 | } 303 | } 304 | } 305 | 306 | /* pure mpi synchronization */ 307 | func (grid *Grid) sync_all() { 308 | grid.sync_one(grid.GetVar("f")) 309 | } 310 | 311 | func (grid *Grid) sync_vl(vl VarList) { 312 | for _, v := range vl.field { 313 | grid.sync_one(v.data) 314 | } 315 | } 316 | 317 | func (grid *Grid) sync_one(data []float64) { 318 | c := grid.box.comm 319 | mpi.Barrier(mpi.COMM_WORLD) 320 | 321 | /* go through all 6 sides seperatly, first x-dir both sides, 322 | then the other sides */ 323 | for d := 0; d < 6; d++ { 324 | e := (d/2)*2 + 1 - d%2 325 | 326 | // fist one direction 327 | sendbuf := make([]float64, c.npts[d]) 328 | recvbuf := make([]float64, c.npts[e]) 329 | if c.neighbour[d] != -1 { 330 | for i := 0; i < c.npts[d]; i++ { 331 | sendbuf[i] = data[c.send[d][i]] 332 | } 333 | } 334 | 335 | if c.neighbour[e] != -1 { 336 | mpi.Recv_float64(recvbuf, c.neighbour[e], 123, mpi.COMM_WORLD) 337 | //mpi.Wait(&request1, &status) 338 | fmt.Println("recv---", rank, c.neighbour[e], recvbuf) 339 | } 340 | if c.neighbour[d] != -1 { 341 | fmt.Println("send---", rank, c.neighbour[d], sendbuf) 342 | mpi.Send_float64(sendbuf, c.neighbour[d], 123, mpi.COMM_WORLD) 343 | //mpi.Wait(&request2, &status) 344 | } 345 | 346 | if c.neighbour[e] != -1 { 347 | for i := 0; i < c.npts[e]; i++ { 348 | data[c.recv[e][i]] = recvbuf[i] 349 | } 350 | } 351 | 352 | mpi.Barrier(mpi.COMM_WORLD) 353 | 354 | } 355 | } 356 | 357 | /* box stuff */ 358 | func (box *Box) inside(pos [3]float64) bool { 359 | if pos[0] < box.xyz0[0]-eps || 360 | pos[1] < box.xyz0[1]-eps || 361 | pos[2] < box.xyz0[2]-eps || 362 | pos[0] > box.xyz0[0]+float64(box.nxyz[0]-1)*box.dxyz[0]+eps || 363 | pos[1] > box.xyz0[1]+float64(box.nxyz[1]-1)*box.dxyz[1]+eps || 364 | pos[2] > box.xyz0[2]+float64(box.nxyz[2]-1)*box.dxyz[2]+eps { 365 | return false 366 | } 367 | return true 368 | } 369 | 370 | func (box *Box) interpolate(pos [3]float64, data []float64) (float64, bool) { 371 | 372 | fmt.Println(box.xyz0, box.xyz1, pos) 373 | 374 | // test if point is inside box 375 | if box.inside(pos) == false { 376 | return 0., false 377 | } 378 | 379 | // find interpolation offset for 2nd order interpolation 380 | var n [3]int 381 | var x0 [3]float64 382 | for i := 0; i < 3; i++ { 383 | n[i] = int(math.Trunc((pos[i] - box.xyz0[i]) / box.dxyz[i])) 384 | if n[i] == box.nxyz[i] { 385 | n[i]-- 386 | //return 0., false 387 | } 388 | x0[i] = box.xyz0[i] + float64(n[i])*box.dxyz[i] 389 | } 390 | //fmt.Println("1 ",x0) 391 | 392 | // find values of the cube around the point 393 | v := [][][]float64{{{0, 0}, {0, 0}}, {{0, 0}, {0, 0}}} 394 | for k := 0; k < 2; k++ { 395 | for j := 0; j < 2; j++ { 396 | for i := 0; i < 2; i++ { 397 | ijk := (n[0] + i) + (n[1]+j)*box.nxyz[0] + (n[2]+k)*box.nxyz[0]*box.nxyz[1] 398 | v[i][j][k] = data[ijk] 399 | } 400 | } 401 | } 402 | //fmt.Println(v) 403 | 404 | // now interpolte tri-polynomial 2nd order 405 | interp := interpolate_TriN(pos, x0, box.dxyz, v) 406 | 407 | return interp, true 408 | } 409 | -------------------------------------------------------------------------------- /mpi.go: -------------------------------------------------------------------------------- 1 | package mpi 2 | 3 | /* 4 | #include "mpi.h" 5 | #cgo LDFLAGS: -lmpi 6 | #cgo CFLAGS: -std=gnu99 -Wall 7 | 8 | MPI_Comm get_MPI_COMM_WORLD() { 9 | return (MPI_Comm)(MPI_COMM_WORLD); 10 | } 11 | 12 | MPI_Datatype get_MPI_Datatype(int i) { 13 | if (i==0) return (MPI_Datatype)(MPI_INT); 14 | else if (i==1) return (MPI_Datatype)(MPI_LONG); 15 | else if (i==2) return (MPI_Datatype)(MPI_FLOAT); 16 | else if (i==3) return (MPI_Datatype)(MPI_DOUBLE); 17 | else return NULL; 18 | } 19 | 20 | MPI_Op get_MPI_Op(int i) { 21 | if (i==0) return (MPI_Op)(MPI_MAX); 22 | else if (i==1) return (MPI_Op)(MPI_MIN); 23 | else if (i==2) return (MPI_Op)(MPI_SUM); 24 | else if (i==3) return (MPI_Op)(MPI_PROD); 25 | else return NULL; 26 | } 27 | 28 | */ 29 | import "C" 30 | 31 | import ( 32 | _ "fmt" 33 | "log" 34 | 35 | _"reflect" 36 | "unsafe" 37 | ) 38 | 39 | /* 40 | implementational details: 41 | all #define values within mpi.h cannot be accessed directly, so 42 | go needs c-wrappers. below there is a small subsection of all 43 | values, which have to be extended, if needed. 44 | (mth: I implemented currently only things I need) 45 | */ 46 | var ( 47 | 48 | //communication structures 49 | COMM_WORLD C.MPI_Comm = C.get_MPI_COMM_WORLD() 50 | 51 | //datatypes 52 | INT C.MPI_Datatype = C.get_MPI_Datatype(0) 53 | INT32 C.MPI_Datatype = C.get_MPI_Datatype(0) 54 | INT64 C.MPI_Datatype = C.get_MPI_Datatype(1) 55 | FLOAT32 C.MPI_Datatype = C.get_MPI_Datatype(2) 56 | FLOAT64 C.MPI_Datatype = C.get_MPI_Datatype(3) 57 | 58 | //operations 59 | MAX C.MPI_Op = C.get_MPI_Op(0) 60 | MIN C.MPI_Op = C.get_MPI_Op(1) 61 | SUM C.MPI_Op = C.get_MPI_Op(2) 62 | PROD C.MPI_Op = C.get_MPI_Op(3) 63 | ) 64 | 65 | // now mpi has also some types, which we directly map 66 | type Request C.MPI_Request 67 | type Status C.MPI_Status 68 | 69 | func Abort(comm C.MPI_Comm, errorcode int) { 70 | err := C.MPI_Abort(comm, C.int(errorcode)) 71 | 72 | if err != 0 { 73 | log.Fatal(err) 74 | } 75 | } 76 | 77 | // wrapper for all types (should be used, but this is slower) 78 | func Allreduce(sendbuf, recvbuf interface{}, op C.MPI_Op, comm C.MPI_Comm) { 79 | 80 | if sb, ok := sendbuf.([]int); ok { 81 | if rv, ok := recvbuf.([]int); ok { 82 | Allreduce_int(sb, rv, op, comm) 83 | } else { 84 | log.Fatal("type of sendbuf and recvbuf do not match") 85 | } 86 | } else if sb, ok := sendbuf.([]int32); ok { 87 | if rv, ok := recvbuf.([]int32); ok { 88 | Allreduce_int32(sb, rv, op, comm) 89 | } else { 90 | log.Fatal("type of sendbuf and recvbuf do not match") 91 | } 92 | } else if sb, ok := sendbuf.([]int64); ok { 93 | if rv, ok := recvbuf.([]int64); ok { 94 | Allreduce_int64(sb, rv, op, comm) 95 | } else { 96 | log.Fatal("type of sendbuf and recvbuf do not match") 97 | } 98 | } else if sb, ok := sendbuf.([]float32); ok { 99 | if rv, ok := recvbuf.([]float32); ok { 100 | Allreduce_float32(sb, rv, op, comm) 101 | } else { 102 | log.Fatal("type of sendbuf and recvbuf do not match") 103 | } 104 | } else if sb, ok := sendbuf.([]float64); ok { 105 | if rv, ok := recvbuf.([]float64); ok { 106 | Allreduce_float64(sb, rv, op, comm) 107 | } else { 108 | log.Fatal("type of sendbuf and recvbuf do not match") 109 | } 110 | } else { 111 | log.Fatal("type not yet supported") 112 | } 113 | } 114 | 115 | func Allreduce_int(sendbuf, recvbuf []int, op C.MPI_Op, comm C.MPI_Comm) { 116 | 117 | // mpi communication call 118 | err := C.MPI_Allreduce( 119 | unsafe.Pointer(&sendbuf[0]), unsafe.Pointer(&recvbuf[0]), 120 | C.int(len(sendbuf)), INT, op, comm) 121 | 122 | if err != 0 { 123 | log.Fatal(err) 124 | } 125 | } 126 | 127 | func Allreduce_int32(sendbuf, recvbuf []int32, op C.MPI_Op, comm C.MPI_Comm) { 128 | 129 | // mpi communication call 130 | err := C.MPI_Allreduce( 131 | unsafe.Pointer(&sendbuf[0]), unsafe.Pointer(&recvbuf[0]), 132 | C.int(len(sendbuf)), INT32, op, comm) 133 | 134 | if err != 0 { 135 | log.Fatal(err) 136 | } 137 | } 138 | 139 | func Allreduce_int64(sendbuf, recvbuf []int64, op C.MPI_Op, comm C.MPI_Comm) { 140 | 141 | // mpi communication call 142 | err := C.MPI_Allreduce( 143 | unsafe.Pointer(&sendbuf[0]), unsafe.Pointer(&recvbuf[0]), 144 | C.int(len(sendbuf)), INT64, op, comm) 145 | 146 | if err != 0 { 147 | log.Fatal(err) 148 | } 149 | } 150 | 151 | func Allreduce_float32(sendbuf, recvbuf []float32, op C.MPI_Op, comm C.MPI_Comm) { 152 | 153 | // mpi communication call 154 | err := C.MPI_Allreduce( 155 | unsafe.Pointer(&sendbuf[0]), unsafe.Pointer(&recvbuf[0]), 156 | C.int(len(sendbuf)), FLOAT32, op, comm) 157 | 158 | if err != 0 { 159 | log.Fatal(err) 160 | } 161 | } 162 | 163 | func Allreduce_float64(sendbuf, recvbuf []float64, op C.MPI_Op, comm C.MPI_Comm) { 164 | 165 | // mpi communication call 166 | err := C.MPI_Allreduce( 167 | unsafe.Pointer(&sendbuf[0]), unsafe.Pointer(&recvbuf[0]), 168 | C.int(len(sendbuf)), FLOAT64, op, comm) 169 | 170 | if err != 0 { 171 | log.Fatal(err) 172 | } 173 | } 174 | 175 | // wrapper for all types (should be used, but this is slower) 176 | func Alltoall(sendbuf, recvbuf interface{}, comm C.MPI_Comm) { 177 | 178 | if sb, ok := sendbuf.([]int); ok { 179 | if rv, ok := recvbuf.([]int); ok { 180 | Alltoall_int(sb, rv, comm) 181 | } else { 182 | log.Fatal("type of sendbuf and recvbuf do not match") 183 | } 184 | } else if sb, ok := sendbuf.([]int32); ok { 185 | if rv, ok := recvbuf.([]int32); ok { 186 | Alltoall_int32(sb, rv, comm) 187 | } else { 188 | log.Fatal("type of sendbuf and recvbuf do not match") 189 | } 190 | } else if sb, ok := sendbuf.([]int64); ok { 191 | if rv, ok := recvbuf.([]int64); ok { 192 | Alltoall_int64(sb, rv, comm) 193 | } else { 194 | log.Fatal("type of sendbuf and recvbuf do not match") 195 | } 196 | } else if sb, ok := sendbuf.([]float32); ok { 197 | if rv, ok := recvbuf.([]float32); ok { 198 | Alltoall_float32(sb, rv, comm) 199 | } else { 200 | log.Fatal("type of sendbuf and recvbuf do not match") 201 | } 202 | } else if sb, ok := sendbuf.([]float64); ok { 203 | if rv, ok := recvbuf.([]float64); ok { 204 | Alltoall_float64(sb, rv, comm) 205 | } else { 206 | log.Fatal("type of sendbuf and recvbuf do not match") 207 | } 208 | } else { 209 | log.Fatal("type not yet supported") 210 | } 211 | } 212 | 213 | func Alltoall_int(sendbuf, recvbuf []int, comm C.MPI_Comm) { 214 | lsend := len(sendbuf) 215 | lrecv := len(recvbuf) 216 | size := Comm_size(comm) 217 | if lsend%size != 0 || lrecv%size != 0 { 218 | log.Fatal("Alltoall: the bufferlength is not consistent with the number of processors") 219 | } 220 | 221 | // mpi communication call 222 | err := C.MPI_Alltoall( 223 | unsafe.Pointer(&sendbuf[0]), C.int(lsend/size), INT, 224 | unsafe.Pointer(&recvbuf[0]), C.int(lrecv/size), INT, 225 | comm) 226 | 227 | if err != 0 { 228 | log.Fatal(err) 229 | } 230 | } 231 | 232 | func Alltoall_int32(sendbuf, recvbuf []int32, comm C.MPI_Comm) { 233 | lsend := len(sendbuf) 234 | lrecv := len(recvbuf) 235 | size := Comm_size(comm) 236 | if lsend%size != 0 || lrecv%size != 0 { 237 | log.Fatal("Alltoall: the bufferlength is not consistent with the number of processors") 238 | } 239 | 240 | // mpi communication call 241 | err := C.MPI_Alltoall( 242 | unsafe.Pointer(&sendbuf[0]), C.int(lsend/size), INT32, 243 | unsafe.Pointer(&recvbuf[0]), C.int(lrecv/size), INT32, 244 | comm) 245 | 246 | if err != 0 { 247 | log.Fatal(err) 248 | } 249 | } 250 | 251 | func Alltoall_int64(sendbuf, recvbuf []int64, comm C.MPI_Comm) { 252 | lsend := len(sendbuf) 253 | lrecv := len(recvbuf) 254 | size := Comm_size(comm) 255 | if lsend%size != 0 || lrecv%size != 0 { 256 | log.Fatal("Alltoall: the bufferlength is not consistent with the number of processors") 257 | } 258 | 259 | // mpi communication call 260 | err := C.MPI_Alltoall( 261 | unsafe.Pointer(&sendbuf[0]), C.int(lsend/size), INT64, 262 | unsafe.Pointer(&recvbuf[0]), C.int(lrecv/size), INT64, 263 | comm) 264 | 265 | if err != 0 { 266 | log.Fatal(err) 267 | } 268 | } 269 | 270 | func Alltoall_float32(sendbuf, recvbuf []float32, comm C.MPI_Comm) { 271 | lsend := len(sendbuf) 272 | lrecv := len(recvbuf) 273 | size := Comm_size(comm) 274 | if lsend%size != 0 || lrecv%size != 0 { 275 | log.Fatal("Alltoall: the bufferlength is not consistent with the number of processors") 276 | } 277 | 278 | // mpi communication call 279 | err := C.MPI_Alltoall( 280 | unsafe.Pointer(&sendbuf[0]), C.int(lsend/size), FLOAT32, 281 | unsafe.Pointer(&recvbuf[0]), C.int(lrecv/size), FLOAT32, 282 | comm) 283 | 284 | if err != 0 { 285 | log.Fatal(err) 286 | } 287 | } 288 | 289 | func Alltoall_float64(sendbuf, recvbuf []float64, comm C.MPI_Comm) { 290 | lsend := len(sendbuf) 291 | lrecv := len(recvbuf) 292 | size := Comm_size(comm) 293 | if lsend%size != 0 || lrecv%size != 0 { 294 | log.Fatal("Alltoall: the bufferlength is not consistent with the number of processors") 295 | } 296 | 297 | // mpi communication call 298 | err := C.MPI_Alltoall( 299 | unsafe.Pointer(&sendbuf[0]), C.int(lsend/size), FLOAT64, 300 | unsafe.Pointer(&recvbuf[0]), C.int(lrecv/size), FLOAT64, 301 | comm) 302 | 303 | if err != 0 { 304 | log.Fatal(err) 305 | } 306 | } 307 | 308 | func Barrier(comm C.MPI_Comm) { 309 | // 310 | C.MPI_Barrier(comm) 311 | } 312 | 313 | func Comm_size(comm C.MPI_Comm) int { 314 | n := C.int(-1) 315 | C.MPI_Comm_size(comm, &n) 316 | return int(n) 317 | } 318 | 319 | func Comm_rank(comm C.MPI_Comm) int { 320 | n := C.int(-1) 321 | C.MPI_Comm_rank(comm, &n) 322 | return int(n) 323 | } 324 | 325 | func Init() { 326 | // initialize the actual mpi stuff, but with two nil 327 | err := C.MPI_Init(nil, nil) 328 | 329 | if err != 0 { 330 | log.Fatal(err) 331 | } 332 | } 333 | 334 | func Finalize() { 335 | err := C.MPI_Finalize() 336 | 337 | if err != 0 { 338 | log.Fatal(err) 339 | } 340 | } 341 | 342 | // wrapper for all types (should be used, but this is slower) 343 | func Irecv(recvbuf interface{}, source, tag int, comm C.MPI_Comm, request *Request) { 344 | 345 | if rb, ok := recvbuf.([]int); ok { 346 | Irecv_int(rb, source, tag, comm, request) 347 | } else if rb, ok := recvbuf.([]int32); ok { 348 | Irecv_int32(rb, source, tag, comm, request) 349 | } else if rb, ok := recvbuf.([]int64); ok { 350 | Irecv_int64(rb, source, tag, comm, request) 351 | } else if rb, ok := recvbuf.([]float32); ok { 352 | Irecv_float32(rb, source, tag, comm, request) 353 | } else if rb, ok := recvbuf.([]float64); ok { 354 | Irecv_float64(rb, source, tag, comm, request) 355 | } else { 356 | log.Fatal("type not yet supported") 357 | } 358 | } 359 | 360 | func Irecv_int(recvbuf []int, source, tag int, comm C.MPI_Comm, request *Request) { 361 | 362 | err := C.MPI_Irecv( 363 | unsafe.Pointer(&recvbuf[0]), C.int(len(recvbuf)), INT, 364 | C.int(source), C.int(tag), comm, (*C.MPI_Request)(request)) 365 | 366 | if err != 0 { 367 | log.Fatal(err) 368 | } 369 | } 370 | 371 | func Irecv_int32(recvbuf []int32, source, tag int, comm C.MPI_Comm, request *Request) { 372 | 373 | err := C.MPI_Irecv( 374 | unsafe.Pointer(&recvbuf[0]), C.int(len(recvbuf)), INT32, 375 | C.int(source), C.int(tag), comm, (*C.MPI_Request)(request)) 376 | 377 | if err != 0 { 378 | log.Fatal(err) 379 | } 380 | } 381 | 382 | func Irecv_int64(recvbuf []int64, source, tag int, comm C.MPI_Comm, request *Request) { 383 | 384 | err := C.MPI_Irecv( 385 | unsafe.Pointer(&recvbuf[0]), C.int(len(recvbuf)), INT64, 386 | C.int(source), C.int(tag), comm, (*C.MPI_Request)(request)) 387 | 388 | if err != 0 { 389 | log.Fatal(err) 390 | } 391 | } 392 | 393 | func Irecv_float32(recvbuf []float32, source, tag int, comm C.MPI_Comm, request *Request) { 394 | 395 | err := C.MPI_Irecv( 396 | unsafe.Pointer(&recvbuf[0]), C.int(len(recvbuf)), FLOAT32, 397 | C.int(source), C.int(tag), comm, (*C.MPI_Request)(request)) 398 | 399 | if err != 0 { 400 | log.Fatal(err) 401 | } 402 | } 403 | 404 | func Irecv_float64(recvbuf []float64, source, tag int, comm C.MPI_Comm, request *Request) { 405 | 406 | err := C.MPI_Irecv( 407 | unsafe.Pointer(&recvbuf[0]), C.int(len(recvbuf)), FLOAT64, 408 | C.int(source), C.int(tag), comm, (*C.MPI_Request)(request)) 409 | 410 | if err != 0 { 411 | log.Fatal(err) 412 | } 413 | } 414 | 415 | // wrapper for all types (should be used, but this is slower) 416 | func Isend(sendbuf interface{}, dest, tag int, comm C.MPI_Comm, request *Request) { 417 | 418 | if sb, ok := sendbuf.([]int); ok { 419 | Isend_int(sb, dest, tag, comm, request) 420 | } else if sb, ok := sendbuf.([]int32); ok { 421 | Isend_int32(sb, dest, tag, comm, request) 422 | } else if sb, ok := sendbuf.([]int64); ok { 423 | Isend_int64(sb, dest, tag, comm, request) 424 | } else if sb, ok := sendbuf.([]float32); ok { 425 | Isend_float32(sb, dest, tag, comm, request) 426 | } else if sb, ok := sendbuf.([]float64); ok { 427 | Isend_float64(sb, dest, tag, comm, request) 428 | } else { 429 | log.Fatal("type not yet supported") 430 | } 431 | } 432 | 433 | func Isend_int(sendbuf []int, dest, tag int, comm C.MPI_Comm, request *Request) { 434 | 435 | err := C.MPI_Isend( 436 | unsafe.Pointer(&sendbuf[0]), C.int(len(sendbuf)), INT, 437 | C.int(dest), C.int(tag), comm, (*C.MPI_Request)(request)) 438 | 439 | if err != 0 { 440 | log.Fatal(err) 441 | } 442 | } 443 | 444 | func Isend_int32(sendbuf []int32, dest, tag int, comm C.MPI_Comm, request *Request) { 445 | 446 | err := C.MPI_Isend( 447 | unsafe.Pointer(&sendbuf[0]), C.int(len(sendbuf)), INT32, 448 | C.int(dest), C.int(tag), comm, (*C.MPI_Request)(request)) 449 | 450 | if err != 0 { 451 | log.Fatal(err) 452 | } 453 | } 454 | 455 | func Isend_int64(sendbuf []int64, dest, tag int, comm C.MPI_Comm, request *Request) { 456 | 457 | err := C.MPI_Isend( 458 | unsafe.Pointer(&sendbuf[0]), C.int(len(sendbuf)), INT64, 459 | C.int(dest), C.int(tag), comm, (*C.MPI_Request)(request)) 460 | 461 | if err != 0 { 462 | log.Fatal(err) 463 | } 464 | } 465 | 466 | func Isend_float32(sendbuf []float32, dest, tag int, comm C.MPI_Comm, request *Request) { 467 | 468 | err := C.MPI_Isend( 469 | unsafe.Pointer(&sendbuf[0]), C.int(len(sendbuf)), FLOAT32, 470 | C.int(dest), C.int(tag), comm, (*C.MPI_Request)(request)) 471 | 472 | if err != 0 { 473 | log.Fatal(err) 474 | } 475 | } 476 | 477 | func Isend_float64(sendbuf []float64, dest, tag int, comm C.MPI_Comm, request *Request) { 478 | 479 | err := C.MPI_Isend( 480 | unsafe.Pointer(&sendbuf[0]), C.int(len(sendbuf)), FLOAT64, 481 | C.int(dest), C.int(tag), comm, (*C.MPI_Request)(request)) 482 | 483 | if err != 0 { 484 | log.Fatal(err) 485 | } 486 | } 487 | 488 | // wrapper for all types (should be used, but this is slower) 489 | func Recv(recvbuf interface{}, source, tag int, comm C.MPI_Comm) { 490 | 491 | if rb, ok := recvbuf.([]int); ok { 492 | Recv_int(rb, source, tag, comm) 493 | } else if rb, ok := recvbuf.([]int32); ok { 494 | Recv_int32(rb, source, tag, comm) 495 | } else if rb, ok := recvbuf.([]int64); ok { 496 | Recv_int64(rb, source, tag, comm) 497 | } else if rb, ok := recvbuf.([]float32); ok { 498 | Recv_float32(rb, source, tag, comm) 499 | } else if rb, ok := recvbuf.([]float64); ok { 500 | Recv_float64(rb, source, tag, comm) 501 | } else { 502 | log.Fatal("type not yet supported") 503 | } 504 | } 505 | 506 | func Recv_int(recvbuf []int, source, tag int, comm C.MPI_Comm) { 507 | 508 | err := C.MPI_Recv( 509 | unsafe.Pointer(&recvbuf[0]), C.int(len(recvbuf)), INT, 510 | C.int(source), C.int(tag), comm, nil) 511 | 512 | if err != 0 { 513 | log.Fatal(err) 514 | } 515 | } 516 | 517 | func Recv_int32(recvbuf []int32, source, tag int, comm C.MPI_Comm) { 518 | 519 | err := C.MPI_Recv( 520 | unsafe.Pointer(&recvbuf[0]), C.int(len(recvbuf)), INT32, 521 | C.int(source), C.int(tag), comm, nil) 522 | 523 | if err != 0 { 524 | log.Fatal(err) 525 | } 526 | } 527 | 528 | func Recv_int64(recvbuf []int64, source, tag int, comm C.MPI_Comm) { 529 | 530 | err := C.MPI_Recv( 531 | unsafe.Pointer(&recvbuf[0]), C.int(len(recvbuf)), INT64, 532 | C.int(source), C.int(tag), comm, nil) 533 | 534 | if err != 0 { 535 | log.Fatal(err) 536 | } 537 | } 538 | 539 | func Recv_float32(recvbuf []float32, source, tag int, comm C.MPI_Comm) { 540 | 541 | err := C.MPI_Recv( 542 | unsafe.Pointer(&recvbuf[0]), C.int(len(recvbuf)), FLOAT32, 543 | C.int(source), C.int(tag), comm, nil) 544 | 545 | if err != 0 { 546 | log.Fatal(err) 547 | } 548 | } 549 | 550 | func Recv_float64(recvbuf []float64, source, tag int, comm C.MPI_Comm) { 551 | 552 | err := C.MPI_Recv( 553 | unsafe.Pointer(&recvbuf[0]), C.int(len(recvbuf)), FLOAT64, 554 | C.int(source), C.int(tag), comm, nil) 555 | 556 | if err != 0 { 557 | log.Fatal(err) 558 | } 559 | } 560 | 561 | // wrapper for all types (should be used, but this is slower) 562 | func Send(sendbuf interface{}, dest, tag int, comm C.MPI_Comm) { 563 | 564 | if sb, ok := sendbuf.([]int); ok { 565 | Send_int(sb, dest, tag, comm) 566 | } else if sb, ok := sendbuf.([]int32); ok { 567 | Send_int32(sb, dest, tag, comm) 568 | } else if sb, ok := sendbuf.([]int64); ok { 569 | Send_int64(sb, dest, tag, comm) 570 | } else if sb, ok := sendbuf.([]float32); ok { 571 | Send_float32(sb, dest, tag, comm) 572 | } else if sb, ok := sendbuf.([]float64); ok { 573 | Send_float64(sb, dest, tag, comm) 574 | } else { 575 | log.Fatal("type not yet supported") 576 | } 577 | } 578 | 579 | func Send_int(sendbuf []int, dest, tag int, comm C.MPI_Comm) { 580 | 581 | err := C.MPI_Send( 582 | unsafe.Pointer(&sendbuf[0]), C.int(len(sendbuf)), INT, 583 | C.int(dest), C.int(tag), comm) 584 | 585 | if err != 0 { 586 | log.Fatal(err) 587 | } 588 | } 589 | 590 | func Send_int32(sendbuf []int32, dest, tag int, comm C.MPI_Comm) { 591 | 592 | err := C.MPI_Send( 593 | unsafe.Pointer(&sendbuf[0]), C.int(len(sendbuf)), INT32, 594 | C.int(dest), C.int(tag), comm) 595 | 596 | if err != 0 { 597 | log.Fatal(err) 598 | } 599 | } 600 | 601 | func Send_int64(sendbuf []int64, dest, tag int, comm C.MPI_Comm) { 602 | 603 | err := C.MPI_Send( 604 | unsafe.Pointer(&sendbuf[0]), C.int(len(sendbuf)), INT64, 605 | C.int(dest), C.int(tag), comm) 606 | 607 | if err != 0 { 608 | log.Fatal(err) 609 | } 610 | } 611 | 612 | func Send_float32(sendbuf []float32, dest, tag int, comm C.MPI_Comm) { 613 | 614 | err := C.MPI_Send( 615 | unsafe.Pointer(&sendbuf[0]), C.int(len(sendbuf)), FLOAT32, 616 | C.int(dest), C.int(tag), comm) 617 | 618 | if err != 0 { 619 | log.Fatal(err) 620 | } 621 | } 622 | 623 | func Send_float64(sendbuf []float64, dest, tag int, comm C.MPI_Comm) { 624 | 625 | err := C.MPI_Send( 626 | unsafe.Pointer(&sendbuf[0]), C.int(len(sendbuf)), FLOAT64, 627 | C.int(dest), C.int(tag), comm) 628 | 629 | if err != 0 { 630 | log.Fatal(err) 631 | } 632 | } 633 | 634 | func Wait(request *Request, status *Status) { 635 | err := C.MPI_Wait((*C.MPI_Request)(request), (*C.MPI_Status)(status)) 636 | 637 | if err != 0 { 638 | log.Fatal(err) 639 | } 640 | } 641 | 642 | func Waitall() { 643 | 644 | /* 645 | int MPI_Waitall(int count, MPI_Request *array_of_requests, 646 | MPI_Status *array_of_statuses) 647 | */ 648 | } 649 | --------------------------------------------------------------------------------