├── .github └── workflows │ └── main.yml ├── .gitignore ├── LICENSE ├── Makefile.inc ├── README.md ├── amr ├── Makefile ├── README ├── adiosamr.c ├── adiosamr.h ├── amr.c ├── cubes.c ├── cubes.h ├── hdf5amr.c ├── hdf5amr.h ├── ncamr.c ├── ncamr.h ├── processlog.py ├── tests.sh ├── vtkout.c └── vtkout.h ├── args.h ├── autoinit.sh ├── cartiso ├── Makefile ├── README ├── adiosfull.c ├── adiosfull.h ├── adiosiso.c ├── adiosiso.h ├── cartiso.c ├── hdf5cartiso.h ├── hdf5i.c ├── hdf5p.c ├── iso.c ├── iso.h ├── nccartiso.h ├── nci.c ├── ncp.c ├── processlog.py ├── pvti.c ├── pvti.h ├── pvtp.c ├── pvtp.h ├── sfc.c ├── sfc.h └── test.sh ├── osn ├── Makefile ├── open-simplex-noise-test.c ├── open-simplex-noise.c └── open-simplex-noise.h ├── pdirs.h ├── struct ├── Makefile ├── README ├── adiosstruct.c ├── adiosstruct.h ├── hdf5struct.c ├── hdf5struct.h ├── netcdfstruct.c ├── netcdfstruct.h ├── processlog.py ├── run_hdf.sh ├── run_nc.sh ├── splitspace.c ├── splitspace.h ├── struct.c ├── test.sh └── test1-64.sh ├── timer.h └── unstruct ├── Makefile ├── adiosunstruct.c ├── adiosunstruct.h ├── hdf5unstruct.c ├── hdf5unstruct.h ├── ncunstruct.c ├── ncunstruct.h ├── processlog.py ├── przm.c ├── przm.h ├── read.src.py ├── test.sh └── unstruct.c /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: miniIO 2 | 3 | # Controls when the action will run. 4 | #Triggers the workflow on push or pull requests. 5 | on: 6 | push: 7 | branches: [ master ] 8 | pull_request: 9 | branches: [ master ] 10 | 11 | # A workflow run is made up of one or more jobs that 12 | # can run sequentially or in parallel 13 | jobs: 14 | # This workflow contains a single job called "build" 15 | build: 16 | strategy: 17 | matrix: 18 | name: ["Ubuntu Latest GCC"] 19 | include: 20 | - name: "Ubuntu Latest GCC" 21 | artifact: "Linux.tar.xz" 22 | os: ubuntu-latest 23 | 24 | name: ${{ matrix.name }} 25 | # The type of runner that the job will run on. 26 | runs-on: ${{ matrix.os }} 27 | if: "!contains(github.event.head_commit.message, 'skip-ci')" 28 | 29 | # Steps represent a sequence of tasks that will be executed 30 | # as part of the job. 31 | steps: 32 | - name: Install Dependencies (Linux) 33 | if: matrix.os == 'ubuntu-latest' 34 | run: | 35 | sudo apt-get update 36 | sudo apt-get install mpich libhdf5-mpich-dev 37 | sudo apt-get install -f 38 | # Set env vars 39 | echo "CC=/usr/bin/h5pcc.mpich" >> $GITHUB_ENV 40 | echo "CPPFLAGS=-I/usr/include/hdf5/mpich" >> $GITHUB_ENV 41 | echo "LDFLAGS=-L/usr/lib/x86_64-linux-gnu/hdf5/mpich" >> $GITHUB_ENV 42 | echo "LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu" >> $GITHUB_ENV 43 | # echo "LIBS=-L/usr/lib/x86_64-linux-gnu -lsz" >> $GITHUB_ENV 44 | # echo "LIBS=/usr/lib/x86_64-linux-gnu/libsz.a" >> $GITHUB_ENV 45 | # echo "LIBS= -ldl -lm" >> $GITHUB_ENV 46 | # cat /usr/bin/h5pcc.mpich 47 | echo "HDF5_DIR=/usr/lib/x86_64-linux-gnu/hdf5/mpich" >> $GITHUB_ENV 48 | 49 | # Checks-out the repository under $GITHUB_WORKSPACE so the job can access it. 50 | - name: Get Sources 51 | uses: actions/checkout@v2 52 | 53 | ################################## 54 | # BUILD and TEST miniIO 55 | ################################## 56 | 57 | - name: test minIO 58 | run: | 59 | cd cartiso 60 | make 61 | ./test.sh hdf5 test 62 | cd ../struct 63 | make 64 | ./test.sh 65 | cd ../unstruct 66 | make 67 | ./test.sh 68 | 69 | shell: bash 70 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Vagrant stuff 2 | .vagrant 3 | html 4 | 5 | # Mac & windows stuff 6 | .DS_Store 7 | .DS_Store? 8 | Thumbs.db 9 | # Python 10 | *.py[cod] 11 | # Compiler 12 | *.o 13 | # Vi/vim 14 | *.swp 15 | # Executables 16 | amr/amr 17 | cartiso/cartiso 18 | unstruct/unstruct 19 | struct/struct 20 | 21 | # Editor generated files 22 | *~ 23 | *# 24 | **/*~ 25 | **/*.bak 26 | **/*.sw[abcdefghijklmnop] 27 | 28 | # Program ouput files 29 | **/*.h5 30 | **/*.xmf 31 | **/*.nc 32 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016, Department of Defense (DoD) High Performance Computing Modernization Program (HPCMP) Productivity Enhancement, Technology Transfer and Training (PETTT) 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | * Redistributions of source code must retain the above copyright 7 | notice, this list of conditions and the following disclaimer. 8 | * Redistributions in binary form must reproduce the above copyright 9 | notice, this list of conditions and the following disclaimer in the 10 | documentation and/or other materials provided with the distribution. 11 | * Neither the name of the DoD, HPCMP, PETTT, nor the 12 | names of its contributors may be used to endorse or promote products 13 | derived from this software without specific prior written permission. 14 | 15 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 16 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 17 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 18 | DISCLAIMED. IN NO EVENT SHALL DOD, HPCMP, OR PETTT BE LIABLE FOR ANY 19 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 20 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 21 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 22 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 24 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 | 26 | -------------------------------------------------------------------------------- /Makefile.inc: -------------------------------------------------------------------------------- 1 | 2 | # MPI Compiler wrappers preferred; just compilers otherwise 3 | ifeq ($(LIBS),) 4 | LIBS = 5 | endif 6 | 7 | CC ?= mpicc 8 | 9 | # Optimization and debugging flags, (un)comment as desired 10 | OPT ?= -O2 11 | DBG ?= -g 12 | 13 | # Add HDF5 library, set to install directory 14 | # ** set via the HDF5_DIR environment variable ** 15 | 16 | # Flags, set as needed; add MPI flags if wrappers weren't available 17 | ifeq ($(CFLAGS),) 18 | CFLAGS = 19 | endif 20 | ifeq ($(LDFLAGS),) 21 | LDFLAGS = 22 | endif 23 | LIBS += -lm 24 | 25 | # You should probably leave the rest below alone 26 | 27 | CFLAGS += $(OPT) $(DBG) -I.. 28 | LDFLAGS += $(OPT) $(DBG) -I.. 29 | 30 | # Common rules for those needing osn 31 | # just add $(OSNOBJ) to list of dependencies (and objects if not using $^) 32 | # and add $(OSNINC) for includes 33 | 34 | OSNINC = -I../osn 35 | OSNOBJ = ../osn/open-simplex-noise.o 36 | 37 | #.PHONY: $(OSNOBJ) 38 | 39 | $(OSNOBJ): ../osn/open-simplex-noise.h ../osn/open-simplex-noise.c 40 | $(MAKE) -C ../osn open-simplex-noise.o 41 | 42 | # Don't let OBNOBJ be the default goal 43 | # (a gnu make-ism, so be careful with weird makes) 44 | .DEFAULT_GOAL := 45 | 46 | ### if ADIOS_HOME is set then enable ADIOS ### 47 | ifdef ADIOS_HOME 48 | CFLAGS += -DHAS_ADIOS $(shell $(ADIOS_HOME)/bin/adios_config -c) 49 | LIBS += $(shell $(ADIOS_HOME)/bin/adios_config -l) 50 | endif 51 | 52 | ### if NC_DIR is set then enable NetCDF-C ### 53 | ifdef NC_DIR 54 | LIBS += $(NC_DIR)/lib/libnetcdf.a $(NC_DIR)/lib/libhdf5_hl.a $(NC_DIR)/lib/libhdf5.a -lz -ldl 55 | INCLUDE += -I$(NC_DIR)/include 56 | CFLAGS += -DHAS_NC 57 | endif 58 | 59 | ### if HDF5_DIR is set then enable HDF5 ### 60 | ifdef HDF5_DIR 61 | 62 | H5PCC = $(HDF5_DIR)/bin/h5pcc 63 | 64 | TESTING = $(shell if [ -f /usr/bin/h5pcc.mpich ]; then echo "yes"; else echo "no"; fi) 65 | 66 | ifeq ($(TESTING),yes) 67 | H5PCC = /usr/bin/h5pcc.mpich 68 | endif 69 | 70 | ## PROCESS h5pcc to get linking options ## 71 | 72 | LINK_LIBS = $(shell $(H5PCC) -showconfig | grep "AM_LDFLAGS:" | sed -n -e 's/^.*AM_LDFLAGS: //p') 73 | LINK_LIBS += $(shell $(H5PCC) -showconfig | grep "Extra libraries:" | sed -n -e 's/^.*Extra libraries: //p') 74 | 75 | LIBDIR = $(shell if [ -d $$HDF5_DIR/lib ]; then echo "$$HDF5_DIR/lib"; else echo "$$HDF5_DIR/lib64"; fi) 76 | 77 | LIBS += $(LIBDIR)/libhdf5.a $(LINK_LIBS) 78 | INCLUDE += -I$(HDF5_DIR)/include 79 | CFLAGS += -DHAS_HDF5 80 | endif 81 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # miniIO 2 | 3 | miniIO is a series of mini-apps that provide the data structures necessary 4 | to reproduce the I/O behavior of representative types of HPC simulation 5 | applications. The computational code for each mini-app is only enough to fill 6 | the data structures with reasonable values and scales trivially in MPI. This 7 | simulates the expected growth in scaling of codes without having to wait for 8 | such codes to be optimized or refactored, allowing one to perform I/O 9 | scaling runs to maximum system or queue sizes. 10 | 11 | The mini-apps include stubbed interface methods and make files to add I/O 12 | libraries or routines in modular fashion. Several I/O libraries/methods are 13 | already supported, ADIOS, HDF5, netCDF, and MPI-IO (not in all mini-apps). This allows users of the mini-apps to: 14 | 15 | - add new I/O types to the mini-apps easily, 16 | - switch between I/O types at will for benchmarks, 17 | - disable each I/O library component if it is not available for a given system. 18 | 19 | The mini-apps also share a simplectic noise method to generate 20 | pseudo-realistic synthetic data to fill data structure arrays so that one can 21 | produce reliable performance results with I/O compression algorithms. 22 | 23 | There are currently 4 mini-apps: 24 | 25 | ### 1. struct 26 | 27 | The struct mini-app produces Cartesian, rectilinear, and curvilinear grids. 28 | Structured grids have regular, implicit connectivity, resulting in a constant 29 | number of points along each dimension with two neighboring points on each 30 | dimension (except edges). However, these structured grids can include a 31 | _missing_ data mask that marks grid points as somehow unavailable for 32 | computation. A real-world example would be land points in an ocean model. Masked points are important because they create load imbalance in a simulation. Simulation codes often load balance their domain decomposition for better scalability. Unfortunately this unbalances the I/O load since masked data points still get written out. This mini-app helps characterize the effects of this I/O load imbalance on I/O performance. 33 | 34 | ### 2. unstruct 35 | 36 | The unstruct mini-app produces unstructured grids. Unstructured grids do not 37 | have implicit connectivity. Any given data point could be connected to any 38 | nearby point and a special data structure of connections must be provided (or 39 | inferred from base geometry in the case of strand grids). The grids themselves 40 | can be steady or time-variant. The grid generation is implemented using a 41 | superquadric equation whose _roundness_ parameters can be varied over time to 42 | emulate a time-variant grid. The grid is also constructed as a series of 43 | volume-inclusive shells so that both traditional unstructured grids and newer 44 | strand grids can be represented. 45 | 46 | ### 3. amr 47 | 48 | Adaptive Mesh Refinement (AMR) grids adapt their resolution to fit a solution 49 | space as appropriate. AMR often starts with a coarse Cartesian base grid. Then, 50 | when some parameter of the model at a given location indicates that higher 51 | resolution is required, that region or single cell of the grid is subdivided 52 | into smaller Cartesian sub-grids. 53 | 54 | ### 4. cartiso 55 | 56 | Derived output occurs when analysts that use a simulation code wish to pull out 57 | subsets of the simulation data for post-hoc analysis. Since it is difficult to 58 | predict where these derivations lie within the model data space, often the 59 | derivations are very load imbalanced. For highly scalable models, the load 60 | imbalance across tens of thousands of compute cores reduces performance 61 | significantly. The cartiso mini-app is named as such because it generates a Cartesian volumetric grid as the basis for computing an isosurface. As such, it is two mini-apps in one, as one can write out either the Cartesian arrays or the isosurface geometry or both. 62 | 63 | ## Contents Overview 64 | 65 | - LICENSE - BSD 3-clause license text 66 | - Makefile.inc - common included makefile 67 | - pdirs.h, timer.h - common definitions, functions, etc. 68 | - osn - OpenSimplexNoise library; used by all mini-apps 69 | - struct, unstruct, amr, cartiso - the mini-apps! 70 | - README - all mini-apps provide their own README 71 | - Makefile - all mini-apps use a similar Make approach 72 | - {miniapp}.c - mini-app main is in .c file of its own name 73 | - adios*.c|.h - ADIOS I/O methods, as supported 74 | - hdf5*.c|.h - HDF5 I/O methods, as supported 75 | - netcdf*.c|.h, nc*.c|.h - netCDF I/O methods, as supported 76 | - Other I/O methods may or may not be the same across mini-apps. 77 | 78 | ## Building 79 | 80 | For now, there is no single Makefile for all mini-apps. One enters the 81 | directory of each desired mini-app to build. In general, just follow the README 82 | in each mini-app directory. The general approach is: 83 | 84 | 1. Edit Makefile.inc and set the correct compilers and flags for the system. 85 | 2. Enter desired mini-app directory. 86 | 3. Edit Makefile to enable/disable I/O modules 87 | 4. ``make`` 88 | 89 | Repeat as desired for each mini-app. The mini-apps are aware of their shared 90 | dependencies (e.g., osn), so they will build it if necessary. 91 | 92 | ### Adding HDF5 support 93 | 94 | In order to build the mini-apps with HDF5 support, simply set the environment variable 95 | *HDF5_DIR* to the directory where HDF5 is installed. Additionally, the HDF5 output 96 | can be generated by setting the command line option --hdf5 for most of the 97 | mini-apps (the exception being the _cartiso_ app). 98 | 99 | ### Adding netCDF support 100 | 101 | In order to build the mini-apps with netCDF support, simply set the environment variable *NC_DIR* to the directory where netCDF is installed. Additionally, the netCDF output can be generated by setting the command line option `--nc` for most of the mini-apps (the exception being the *cartiso* app, which uses the `--ncp` and `--nci` option). 102 | 103 | ### Adding ADIOS support 104 | 105 | In order to build the mini-apps with ADIOS support, simply set the environment variable 106 | *ADIOS_HOME* to the directory where ADIOS is installed. 107 | 108 | 109 | ## Running 110 | 111 | The mini-apps use a common command-line argument format where possible. In 112 | general, a run will look something like this: 113 | ```sh 114 | mpiexec -np N ./miniapp --required_arg option [--optional_arg option] 115 | ``` 116 | where: 117 | - mpiexec = actual executable depends on your MPI implementation 118 | - N = number of MPI tasks 119 | - miniapp = name of actual mini-app 120 | - required_arg = some required command-line argument; varies per mini-app 121 | - option = some arguments require options after them 122 | - optional_arg = optional command-line argument 123 | 124 | See the README and USAGE statement for each mini-app for more details. 125 | 126 | ## Adding an I/O Module to a Mini-app 127 | 128 | Follow the README for each mini-app for specifics. Each mini-app carefully and 129 | clearly comments where new I/O methods get added. The general overview of 130 | adding an I/O module for each mini-app is as follows: 131 | 132 | 1. Add to the I/O Modules sections in the Makefile 133 | 2. Add includes to main .c file 134 | 3. Recommended: add usage string to main .c file 135 | 4. Recommended: Add state variables; you'll need some for command line options 136 | 5. Recommended: add your command line options 137 | 6. Optional: Add initialization, if your code will need it 138 | 7. Add output routine or function call in marked section of time step loop 139 | 8. Optional: Add cleanup/finalization code 140 | 9. Update Make dependencies (``make depend``) if new files were added 141 | 142 | If any of the above steps, especially initialization (#6), time step output 143 | (#7), or cleanup (#8), are longer than 2-3 lines, please put them in a function 144 | in a separate .h and .c file (and update dependencies). This keeps the main 145 | mini-app code much cleaner. We won't accept pull requests or submitted code 146 | that doesn't follow this standard. 147 | 148 | The "recommended" steps aren't necessary to get working code. They are 149 | recommended to provide proper options and documentation to users of your code. 150 | We won't accept pull requests or submitted code that don't do these. 151 | -------------------------------------------------------------------------------- /amr/Makefile: -------------------------------------------------------------------------------- 1 | 2 | include ../Makefile.inc 3 | 4 | OBJS = amr.o cubes.o 5 | SRCS = amr.c cubes.c 6 | 7 | ### Add Output Modules Here ### 8 | 9 | # VTK Output Module 10 | ENABLE_VTKOUT = 1 11 | ifeq ($(ENABLE_VTKOUT),1) 12 | OBJS += vtkout.o 13 | SRCS += vtkout.c 14 | CFLAGS += -DHAS_VTKOUT 15 | endif 16 | 17 | ifdef ADIOS_HOME 18 | OBJS += adiosamr.o 19 | SRCS += adiosamr.c 20 | endif 21 | 22 | ifdef NC_DIR 23 | OBJS += ncamr.o 24 | SRCS += ncamr.c 25 | endif 26 | 27 | ifdef HDF5_DIR 28 | OBJS += hdf5amr.o 29 | SRCS += hdf5amr.c 30 | endif 31 | 32 | ### End of Output Modules ### 33 | 34 | CFLAGS += $(OSNINC) 35 | 36 | .PHONY: clean depend 37 | 38 | amr: $(OSNOBJ) $(OBJS) 39 | $(CC) $(LDFLAGS) -o $@ $^ $(LIBS) 40 | 41 | clean: 42 | rm -f *.o amr *.h5 43 | 44 | depend: 45 | makedepend -- $(CFLAGS) -- -Y $(SRCS) # ignore warnings about system headers 46 | 47 | .c.o: 48 | $(CC) $(CFLAGS) -c $< $(INCLUDE) 49 | 50 | # DO NOT DELETE 51 | -------------------------------------------------------------------------------- /amr/README: -------------------------------------------------------------------------------- 1 | TEST examples 2 | 3 | # Smallest test (test1): 4 | mpirun -np 1 ./amr --tasks 1 1 1 --size 2 2 2 --levels 0 --tsteps 1 5 | 6 | # Smallest test with debugging statements (test1_debug): 7 | mpirun -np 1 ./amr --tasks 1 1 1 --size 2 2 2 --levels 0 --tsteps 1 --debug 8 | 9 | # Smallest test with default values passed same as test1: 10 | mpirun -np 1 ./amr --tasks 1 1 1 --size 2 2 2 --threshold 0 --levels 0 --tsteps 1 --noisespacefreq 10 --noisetimefreq .25 11 | 12 | # Generate VTK output 13 | mpirun -np 8 ./amr --tasks 2 2 2 --size 3 3 3 --levels 3 --tsteps 1 --vtkout 14 | 15 | # larger grid with VTK output 16 | mpirun -np 8 ./amr --tasks 2 2 2 --size 9 9 9 --levels 3 --tsteps 1 --vtkout 17 | 18 | # larger grid with ADIOS output 19 | mpirun -np 8 ./amr --tasks 2 2 2 --size 9 9 9 --levels 3 --tsteps 1 --adios MPI -------------------------------------------------------------------------------- /amr/adiosamr.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) DoD HPCMP PETTT. All rights reserved. 3 | * See LICENSE file for details. 4 | */ 5 | 6 | #include 7 | #include 8 | #include "adiosamr.h" 9 | 10 | static const int fnstrmax = 4095; 11 | 12 | void adiosamr_init(struct adiosamrinfo *nfo, char *method, char *name, 13 | MPI_Comm comm, int rank, int nprocs, int tsteps) { 14 | /* Set up struct, haven't decided if using all of these yet */ 15 | nfo->name = name; 16 | nfo->comm = comm; 17 | nfo->rank = rank; 18 | nfo->nprocs = nprocs; 19 | nfo->tsteps = tsteps; 20 | 21 | nfo->numxvars = 0; 22 | nfo->maxxvars = 1000; 23 | nfo->xvarnames = (char **) malloc(nfo->maxxvars * sizeof(char *)); 24 | nfo->bufallocsize = 0; 25 | 26 | /* Set up ADIOS */ 27 | adios_init_noxml(comm); 28 | adios_declare_group(&nfo->gid, nfo->name, "", adios_flag_no); 29 | adios_select_method(nfo->gid, method, "", ""); 30 | 31 | /* Define output variables */ 32 | adios_define_var(nfo->gid, "rank", "", adios_integer, "", "", ""); 33 | adios_define_var(nfo->gid, "tstep", "", adios_integer, "", "", ""); 34 | adios_define_var(nfo->gid, "cnpoints", "", adios_unsigned_long, "", "", ""); 35 | adios_define_var(nfo->gid, "npoints", "", adios_unsigned_long, "", "", ""); 36 | adios_define_var(nfo->gid, "cstart", "", adios_unsigned_long, "", "", ""); 37 | } 38 | 39 | void adiosamr_addxvar(struct adiosamrinfo *nfo, char *varname) { 40 | if(nfo->numxvars > nfo->maxxvars) 41 | return; /* Just ignore too many variables, for now */ 42 | nfo->xvarnames[nfo->numxvars] = varname; 43 | nfo->numxvars++; 44 | adios_define_var(nfo->gid, varname, "", adios_real, "cnpoints", "npoints", "cstart"); 45 | } 46 | 47 | void adiosamr_write(struct adiosamrinfo *nfo, int tstep, uint64_t cnpoints, float *points, float **xvals) { 48 | char fname[fnstrmax+1]; 49 | int timedigits = 4; 50 | uint64_t groupsize, totalsize; 51 | int64_t handle; 52 | int ret; 53 | int bufneeded; 54 | uint64_t i, totalpoints, cstart, *npts_all; 55 | 56 | 57 | /* numCoords = npoints * 8; */ 58 | groupsize = sizeof(int) * 2 /*rank-numtask*/ + 59 | sizeof(uint64_t)*3 /*cnpoints-cstart*/ + 60 | sizeof(float)*cnpoints*nfo->numxvars; /*xvars*/ 61 | 62 | /* Allocate buffer large enough for all data to write, if not done already */ 63 | bufneeded = (int)(groupsize/(1024*1024)); 64 | 65 | # if ADIOS_VERSION_GE(1,10,0) 66 | bufneeded += bufneeded + 5; /* Double and add an extra 5MB to be sure */ 67 | # else 68 | bufneeded += (bufneeded * .3) + 5; /* Add an extra 30% & 5MB to be sure */ 69 | # endif 70 | 71 | if(nfo->bufallocsize < bufneeded) { 72 | # if ADIOS_VERSION_GE(1,10,0) 73 | adios_set_max_buffer_size(bufneeded); 74 | # else 75 | adios_allocate_buffer(ADIOS_BUFFER_ALLOC_NOW, bufneeded); 76 | # endif 77 | nfo->bufallocsize = bufneeded; 78 | } 79 | 80 | /* Determine global sizes */ 81 | npts_all = (uint64_t *) malloc(nfo->nprocs * sizeof(uint64_t)); 82 | MPI_Allgather(&cnpoints, 1, MPI_UNSIGNED_LONG_LONG, 83 | npts_all, 1, MPI_UNSIGNED_LONG_LONG, nfo->comm); 84 | for(totalpoints = 0, i = 0; i < nfo->nprocs; ++i) 85 | totalpoints += npts_all[i]; 86 | for(cstart = 0, i = 0; i < nfo->rank; ++i) 87 | cstart += npts_all[i]; 88 | free(npts_all); 89 | 90 | /* Set filename */ 91 | snprintf(fname, fnstrmax, "%s.out.%0*d.bp", nfo->name, timedigits, tstep); 92 | 93 | /* Open & Write */ 94 | ret = adios_open(&handle, nfo->name, fname, "w", nfo->comm); 95 | if(ret) { 96 | fprintf(stderr, "Error opening ADIOS file: %s\n", fname); 97 | return; 98 | } 99 | # if ADIOS_VERSION_LE(1,9,0) 100 | adios_group_size(handle, groupsize, &totalsize); 101 | # endif 102 | 103 | adios_write(handle, "rank", &nfo->rank); 104 | adios_write(handle, "tstep", &tstep); 105 | adios_write(handle, "cnpoints", &cnpoints); 106 | adios_write(handle, "npoints", &totalpoints); 107 | adios_write(handle, "cstart", &cstart); 108 | for(i = 0; i < nfo->numxvars; i++){ 109 | adios_write(handle, nfo->xvarnames[i], xvals[i]); 110 | } 111 | adios_close(handle); 112 | 113 | } 114 | 115 | void adiosamr_finalize(struct adiosamrinfo *nfo) { 116 | free(nfo->xvarnames); 117 | adios_finalize(nfo->rank); 118 | } 119 | -------------------------------------------------------------------------------- /amr/adiosamr.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) DoD HPCMP PETTT. All rights reserved. 3 | * See LICENSE file for details. 4 | */ 5 | 6 | #include 7 | #include 8 | 9 | struct adiosamrinfo { 10 | char *name; 11 | MPI_Comm comm; 12 | int rank; 13 | int nprocs; 14 | int tsteps; 15 | 16 | int numxvars; 17 | int maxxvars; 18 | char **xvarnames; 19 | 20 | int bufallocsize; 21 | 22 | int64_t gid; 23 | }; 24 | 25 | void adiosamr_init(struct adiosamrinfo *nfo, char *method, char *name, MPI_Comm comm, int rank, int nprocs, int tsteps); 26 | 27 | void adiosamr_addxvar(struct adiosamrinfo *nfo, char *varname); 28 | 29 | void adiosamr_write(struct adiosamrinfo *nfo, int tstep, uint64_t cnpoints, float *points, float **xvals); 30 | 31 | void adiosamr_finalize(struct adiosamrinfo *nfo); 32 | -------------------------------------------------------------------------------- /amr/amr.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) DoD HPCMP PETTT. All rights reserved. 3 | * See LICENSE file for details. 4 | */ 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include "open-simplex-noise.h" 11 | #include "cubes.h" 12 | #include "timer.h" 13 | 14 | #ifdef HAS_VTKOUT 15 | #include "vtkout.h" 16 | #endif 17 | 18 | #ifdef HAS_ADIOS 19 | # include "adiosamr.h" 20 | #endif 21 | 22 | #ifdef HAS_NC 23 | # include "ncamr.h" 24 | #endif 25 | 26 | #ifdef HAS_HDF5 27 | # include "hdf5amr.h" 28 | #endif 29 | 30 | 31 | void print_usage(int rank, const char *errstr); 32 | 33 | 34 | int main(int argc, char **argv) { 35 | int debug=0; 36 | int i, j, k, a, block_id, t; /* loop indices */ 37 | int tt; /* Actual time step from tstart */ 38 | float x, y, z; 39 | double noisespacefreq = 10.0; /* Spatial frequency of noise */ 40 | double noisetimefreq = 0.25; /* Temporal frequency of noise */ 41 | int tstart = 0; 42 | int nt = 50; /* Number of time steps */ 43 | float deltax, deltay, deltaz; 44 | int maxLevel = 4; 45 | float threshold=0.0; 46 | int inp = 0; /* Number of tasks in i */ 47 | int jnp = 0; /* Number of tasks in j */ 48 | int knp = 0; /* Number of tasks in k */ 49 | int ni = 0; /* Global grid size */ 50 | int nj = 0; 51 | int nk = 0; 52 | cubeInfo cubedata; 53 | struct osn_context *simpnoise; /* Open simplex noise context */ 54 | double computetime, outtime; /* Timers */ 55 | 56 | /* MPI vars */ 57 | MPI_Comm comm = MPI_COMM_WORLD; 58 | int cprocs[3], cpers[3], crnk[3]; /* MPI Cartesian info */ 59 | int rank, nprocs; 60 | int cni, cnj, cnk; /* Points in this task */ 61 | int is, js, ks; /* Global index starting points */ 62 | float xs, ys, zs; /* Global coordinate starting points */ 63 | 64 | #ifdef HAS_VTKOUT 65 | int vtkout = 0; 66 | #endif 67 | 68 | #ifdef HAS_ADIOS 69 | char *adios_groupname="amr"; 70 | char *adios_method=NULL; /* POSIX|MPI|MPI_LUSTRE|MPI_AGGREGATE|PHDF5 */ 71 | struct adiosamrinfo adiosamr_nfo; 72 | #endif 73 | 74 | #ifdef HAS_NC 75 | char *nc_groupname="amr"; 76 | struct ncamrinfo ncamr_nfo; 77 | int ncout = 0; 78 | #endif 79 | 80 | #ifdef HAS_HDF5 81 | char *hdf5_groupname="amr"; 82 | struct hdf5amrinfo hdf5amr_nfo; 83 | int hdf5out = 0; 84 | #endif 85 | 86 | MPI_Init(&argc, &argv); 87 | MPI_Comm_rank(comm, &rank); 88 | MPI_Comm_size(comm, &nprocs); 89 | 90 | /* Parse command line */ 91 | for(a = 1; a < argc; a++) { 92 | if(!strcasecmp(argv[a], "--tasks")) { 93 | inp = atoi(argv[++a]); 94 | jnp = atoi(argv[++a]); 95 | knp = atoi(argv[++a]); 96 | } else if(!strcasecmp(argv[a], "--size")) { 97 | ni = atoi(argv[++a]); 98 | nj = atoi(argv[++a]); 99 | nk = atoi(argv[++a]); 100 | }else if(!strcasecmp(argv[a], "--threshold")) { 101 | threshold = strtof(argv[++a], NULL); 102 | } else if(!strcasecmp(argv[a], "--levels")) { 103 | maxLevel = atoi(argv[++a]); 104 | } else if(!strcasecmp(argv[a], "--noisespacefreq")) { 105 | noisespacefreq = strtod(argv[++a], NULL); 106 | } else if(!strcasecmp(argv[a], "--noisetimefreq")) { 107 | noisetimefreq = strtod(argv[++a], NULL); 108 | } else if(!strcasecmp(argv[a], "--tsteps")) { 109 | nt = atoi(argv[++a]); 110 | } else if(!strcasecmp(argv[a], "--tstart")) { 111 | tstart = atoi(argv[++a]); 112 | }else if(!strcasecmp(argv[a], "--debug")) { 113 | debug = 1; 114 | } 115 | 116 | #ifdef HAS_ADIOS 117 | else if(!strcasecmp(argv[a], "--adios")) { 118 | adios_method = argv[++a]; 119 | } 120 | #endif 121 | 122 | else if(!strcasecmp(argv[a], "--hdf5")) { 123 | #ifdef HAS_HDF5 124 | hdf5out = 1; 125 | #else 126 | if(rank == 0) fprintf(stderr, "HDF5 option not available: %s\n\n", argv[a]); 127 | print_usage(rank, NULL); 128 | MPI_Abort(comm, 1); 129 | #endif 130 | } 131 | 132 | #ifdef HAS_NC 133 | else if(!strcasecmp(argv[a], "--nc")) { 134 | ncout = 1; 135 | } 136 | #endif 137 | 138 | #ifdef HAS_VTKOUT 139 | else if(!strcasecmp(argv[a], "--vtkout")) { 140 | vtkout = 1; 141 | } 142 | #endif 143 | 144 | 145 | else { 146 | if(rank == 0) fprintf(stderr, "Option not recognized: %s\n\n", argv[a]); 147 | print_usage(rank, NULL); 148 | MPI_Abort(comm, 1); 149 | } 150 | } 151 | 152 | /* Check arguments & proc counts */ 153 | if(inp < 0 || jnp < 0 || knp < 0) { 154 | print_usage(rank, "Error: tasks not specified or incorrect"); 155 | MPI_Abort(MPI_COMM_WORLD, 1); 156 | } 157 | 158 | if(ni < 2 || nj < 2 || nk < 2) { 159 | print_usage(rank, "Error: size not specified or incorrect"); 160 | MPI_Abort(MPI_COMM_WORLD, 1); 161 | } 162 | 163 | if(inp*jnp*knp != nprocs) { 164 | print_usage(rank, "Error: product of tasks does not equal total MPI tasks"); 165 | MPI_Abort(MPI_COMM_WORLD, 1); 166 | } 167 | 168 | if((ni-1) % inp || (nj-1) % jnp || (nk-1) % knp) { 169 | print_usage(rank, "Error: number of points-1 on an axis is not evenly divisible " 170 | "by axis tasks.\n This is required for proper load balancing."); 171 | MPI_Abort(MPI_COMM_WORLD, 1); 172 | } 173 | 174 | 175 | if(maxLevel < 0 ) { 176 | print_usage(rank, "Error: number of levels not specified or incorrect"); 177 | MPI_Abort(comm, 1); 178 | } 179 | 180 | if (nt < 1) { 181 | print_usage(rank, "Error: number of timesteps not specified or incorrect"); 182 | MPI_Abort(comm, 1); 183 | } 184 | 185 | /* Set up Cartesian communicator */ 186 | cprocs[0] = inp; cprocs[1] = jnp; cprocs[2] = knp; 187 | cpers[0] = 0; cpers[1] = 0; cpers[2] = 0; /* No periodicity */ 188 | MPI_Cart_create(MPI_COMM_WORLD, 3, cprocs, cpers, 1, &comm); 189 | MPI_Comm_rank(comm, &rank); 190 | MPI_Cart_coords(comm, rank, 3, crnk); 191 | 192 | deltax = 1.f/(ni-1); 193 | deltay = 1.f/(nj-1); 194 | deltaz = 1.f/(nk-1); 195 | cni = (ni-1) / inp; 196 | cnj = (nj-1) / jnp; 197 | cnk = (nk-1) / knp; 198 | is = crnk[0] * cni; 199 | js = crnk[1] * cnj; 200 | ks = crnk[2] * cnk; 201 | xs = is * deltax; 202 | ys = js * deltay; 203 | zs = ks * deltaz; 204 | 205 | 206 | 207 | /* Set up osn */ 208 | open_simplex_noise(12345, &simpnoise); /* Fixed seed, for now */ 209 | 210 | /* Allocate arrays */ 211 | cubesinit(&cubedata, cni*cnj*cnk, maxLevel, debug); 212 | 213 | /* init ADIOS */ 214 | #ifdef HAS_ADIOS 215 | if (adios_method) { 216 | adiosamr_init(&adiosamr_nfo, adios_method, adios_groupname, comm, rank, nprocs, nt); 217 | adiosamr_addxvar(&adiosamr_nfo, "data"); 218 | } 219 | #endif 220 | 221 | #ifdef HAS_HDF5 222 | if(hdf5out) { 223 | hdf5_init(&hdf5amr_nfo, hdf5_groupname, comm, rank, nprocs, nt); 224 | hdf5_addxvar(&hdf5amr_nfo, "data"); 225 | } 226 | #endif 227 | 228 | #ifdef HAS_NC 229 | if(ncout) { 230 | nc_init(&ncamr_nfo, nc_groupname, comm, rank, nprocs, nt); 231 | nc_addxvar(&ncamr_nfo,"data"); 232 | } 233 | #endif 234 | 235 | if (debug) { 236 | printf("(cni=%d, cnj=%d, cnk=%d) \n", cni, cnj, cnk); 237 | } 238 | 239 | for(t = 0, tt = tstart; t < nt; t++, tt++) { 240 | size_t ii; /* data index */ 241 | 242 | cubedata.npoints = 0; 243 | cubedata.ncubes = 0; 244 | 245 | 246 | if (debug) { 247 | printf("Hi: rank=%d: %d of %d: timestep=%d\n", rank, rank+1, nprocs, tt); 248 | } 249 | 250 | timer_tick(&computetime, comm, 1); 251 | 252 | z = zs; 253 | for(k = 0, ii = 0; k < cnk; k++) { 254 | y = ys; 255 | for(j = 0; j < cnj; j++) { 256 | x = xs; 257 | for(i = 0; i < cni; i++, ii++) { 258 | 259 | /* calculate block_id */ 260 | block_id = ii; 261 | 262 | if (debug) { 263 | printf("Start from main Block_id=%d\n", block_id+1); 264 | } 265 | refine(&cubedata, tt, (block_id+1), threshold, 0, x, y, z, deltax, deltay, deltaz, simpnoise, maxLevel, noisespacefreq, noisetimefreq); 266 | 267 | x += deltax; 268 | } 269 | y += deltay; 270 | } 271 | z += deltaz; 272 | } 273 | 274 | timer_tock(&computetime); 275 | 276 | /* print out data */ 277 | if (debug) { 278 | cubeprint(&cubedata); 279 | } 280 | 281 | 282 | #ifdef HAS_VTKOUT 283 | if(vtkout) { 284 | if(rank == 0) { 285 | printf(" Writing VTK ...\n"); fflush(stdout); 286 | } 287 | writevtk("amr.out", comm, rank, nprocs, tt, cubedata.npoints, cubedata.ncubes, 288 | cubedata.points, cubedata.data, "data", debug); 289 | } 290 | #endif 291 | 292 | timer_tick(&outtime, comm, 1); 293 | 294 | #ifdef HAS_ADIOS 295 | if (adios_method) { 296 | if(rank == 0) { 297 | printf(" Writing ADIOS ...\n"); fflush(stdout); 298 | } 299 | adiosamr_write(&adiosamr_nfo, tt, cubedata.npoints, cubedata.points, &cubedata.data); 300 | } 301 | #endif 302 | 303 | #ifdef HAS_HDF5 304 | if(hdf5out) { 305 | if(rank == 0) { 306 | printf(" Writing HDF5 ...\n"); fflush(stdout); 307 | } 308 | hdf5_write(&hdf5amr_nfo, tt, cubedata.npoints, cubedata.points, &cubedata.data); 309 | } 310 | #endif 311 | 312 | #ifdef HAS_NC 313 | if(ncout) { 314 | if(rank == 0) { 315 | printf(" Writing netCDF ...\n"); fflush(stdout); 316 | } 317 | nc_write(&ncamr_nfo, tt, cubedata.npoints, cubedata.points, &cubedata.data); 318 | } 319 | #endif 320 | 321 | timer_tock(&outtime); 322 | timer_collectprintstats(computetime, comm, 0, " Compute"); 323 | timer_collectprintstats(outtime, comm, 0, " Output"); 324 | } 325 | 326 | 327 | if (debug) 328 | printf("Finalizing:rank %d \n", rank); 329 | 330 | /* finalize ADIOS */ 331 | #ifdef HAS_ADIOS 332 | if (adios_method) adiosamr_finalize(&adiosamr_nfo); 333 | #endif 334 | 335 | #ifdef HAS_HDF5 336 | if(hdf5out) { 337 | hdf5_finalize(&hdf5amr_nfo); 338 | } 339 | #endif 340 | 341 | #ifdef HAS_NC 342 | if(ncout) { 343 | nc_final(&ncamr_nfo); 344 | } 345 | #endif 346 | 347 | open_simplex_noise_free(simpnoise); 348 | cubesfree(&cubedata); 349 | MPI_Finalize(); 350 | 351 | return 0; 352 | } 353 | 354 | 355 | 356 | void print_usage(int rank, const char *errstr) 357 | { 358 | if(rank != 0) return; 359 | if(errstr) 360 | fprintf(stderr, "%s\n\n", errstr); 361 | fprintf(stderr, 362 | "Usage: mpi_launcher [-n|-np NPROCS] ./arm --tasks INP JNP KNP --size NI NJ NK [options]\n" 363 | " NPROCS : # of tasks launched by MPI; may or may not be implied or required by system\n\n" 364 | " Required:\n" 365 | " --tasks INP JNP KNP : Specifies the parallel decomposition of tasks\n" 366 | " INP : # of tasks along the I (X) axis\n" 367 | " JNP : # of tasks along the J (Y) axis\n" 368 | " KNP : # of tasks along the K (Z) axis\n" 369 | " NOTE that INP * JNP * KNP == NPROCS is required!\n" 370 | " --size NI NJ NK : Specifies the size of the grid\n" 371 | " NI, NJ, NK : Number of grid points along the I,J,K axes respectively\n\n" 372 | " Optional:\n" 373 | " --debug: Turns on debugging statements \n" 374 | " --threshold T : Mask theshold; valid values are floats between -1.0 and 1.0 \n" 375 | " T : threshold value; Default: 0.0\n" 376 | " --levels L : Maximum levels of refinement; valid values are >= 0 \n" 377 | " L : max refinmentment levels value; Default: 8\n" 378 | " --noisespacefreq FNS : Spatial frequency of noise function\n" 379 | " FNS : space frequency value; Default: 10.0\n" 380 | " --noisetimefreq FNT : Temporal frequency of noise function\n" 381 | " FNT : time frequency value; Default: 0.25\n" 382 | " --tsteps NT : Number of time steps; valid values are > 0; Default: 50)\n" 383 | " --tstart TS : Starting time step; valid values are > 0; Default: 0\n" 384 | 385 | #ifdef HAS_NC 386 | "\n --nc: Enable netCDF output\n" 387 | #endif 388 | 389 | #ifdef HAS_ADIOS 390 | " --adios [POSIX|MPI|MPI_LUSTRE|MPI_AGGREGATE|PHDF5]: Enable ADIOS output\n" 391 | #endif 392 | ); 393 | 394 | #ifdef HAS_VTKOUT 395 | fprintf(stderr, " --vtkout : Enable VTK output.\n"); 396 | #endif 397 | 398 | /*## End of Output Module Usage Strings ##*/ 399 | } 400 | -------------------------------------------------------------------------------- /amr/cubes.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) DoD HPCMP PETTT. All rights reserved. 3 | * See LICENSE file for details. 4 | */ 5 | 6 | #include 7 | #include 8 | #include 9 | #include "open-simplex-noise.h" 10 | #include "cubes.h" 11 | 12 | void cubesinit(cubeInfo *nfo, int task, int levels, int debug) { 13 | int maxpoints=0; 14 | 15 | maxpoints = pow(8, levels+1); 16 | nfo->ncubes = 0; 17 | nfo->npoints = 0; 18 | nfo->debug = debug; 19 | 20 | if (nfo->debug) { 21 | printf("Init cubes maxpoints=%d task=%d +++++++++++++++\n", maxpoints, task); 22 | } 23 | 24 | /* Allocate list for all possible cubes and points */ 25 | nfo->points = (float *) malloc((size_t)task*maxpoints*3*sizeof(float) ); 26 | nfo->data = (float *) malloc((size_t)task*maxpoints*sizeof(float) ); 27 | 28 | } 29 | 30 | void cubesfree(cubeInfo *nfo) { 31 | nfo->debug =0; 32 | nfo->ncubes = 0; 33 | nfo->npoints = 0; 34 | free(nfo->points); 35 | free(nfo->data); 36 | nfo->points = NULL; 37 | nfo->data = NULL; 38 | } 39 | 40 | void refine(cubeInfo *nfo, int t, int rpId, float thres, int level_start, float x_start, float y_start, float z_start, float dx_start, 41 | float dy_start, float dz_start, struct osn_context *osn, int maxLevel, double noisespacefreq, double noisetimefreq) { 42 | /* double noisespacefreq = 10; /\* Spatial frequency of noise *\/ */ 43 | /* double noisetimefreq = 0.25; /\* Temporal frequency of noise *\/ */ 44 | float x_center, y_center, z_center, center_val; 45 | float xpts[8], ypts[8], zpts[8]; 46 | float curdata[8]; 47 | int i; 48 | int split; 49 | int refinePathID = 0; 50 | int new_refinePathID = 0; 51 | int inside; 52 | stack octStack; 53 | float x, y, z, dx, dy, dz; 54 | int level; 55 | int stacksize; 56 | 57 | 58 | if (nfo->debug) 59 | printf("Start start initial refinement on Block = %d\n", rpId); 60 | 61 | stacksize = 8 * (maxLevel+1); 62 | stack_new(&octStack, stacksize, nfo->debug); 63 | 64 | stack_push(&octStack, x_start, y_start, z_start, dx_start, dy_start, dz_start, level_start, rpId); 65 | 66 | while (!stack_isempty(&octStack)){ 67 | split = 0; 68 | inside = 0; 69 | 70 | /* get next cube to test */ 71 | stack_pop(&octStack, &x, &y, &z, &dx, &dy, &dz, &level, &refinePathID); 72 | 73 | /* refinePathID = rpId; */ 74 | 75 | if (nfo->debug) 76 | printf("Block refinePathID = %d\n", refinePathID); 77 | 78 | 79 | /* given a unit cube (8 points) */ 80 | /* - calulate poistion x,y,z for all points */ 81 | xpts[0] = x; 82 | ypts[0] = y; 83 | zpts[0] = z; 84 | xpts[1] = x + dx; 85 | ypts[1] = y; 86 | zpts[1] = z; 87 | xpts[2] = x; 88 | ypts[2] = y + dy; 89 | zpts[2] = z; 90 | xpts[3] = x + dx; 91 | ypts[3] = y + dy; 92 | zpts[3] = z; 93 | xpts[4] = x; 94 | ypts[4] = y; 95 | zpts[4] = z + dz; 96 | xpts[5] = x + dx; 97 | ypts[5] = y; 98 | zpts[5] = z + dz; 99 | xpts[6] = x; 100 | ypts[6] = y + dy; 101 | zpts[6] = z + dz; 102 | xpts[7] = x + dx; 103 | ypts[7] = y + dy; 104 | zpts[7] = z + dz; 105 | 106 | /* Get block center value */ 107 | x_center = x + dx/2; 108 | y_center = y + dy/2; 109 | z_center = z + dz/2; 110 | center_val = (float)open_simplex_noise4(osn, x_center*noisespacefreq, 111 | y_center*noisespacefreq, z_center*noisespacefreq, t*noisetimefreq); 112 | 113 | if (center_val > thres) 114 | inside = 1; 115 | 116 | if (inside && (level < maxLevel)) 117 | split = 1; 118 | 119 | if (nfo->debug) 120 | printf("Block_Center: (%f %f, %f)=%f thres=%f level=%d inside=%d split=%d\n", x_center, y_center, z_center, center_val, thres, level, inside, split); 121 | 122 | /* Refine if criteria satisfied */ 123 | if (split) { 124 | xpts[0] = x; 125 | ypts[0] = y; 126 | zpts[0] = z; 127 | xpts[1] = x + dx/2; 128 | ypts[1] = y; 129 | zpts[1] = z; 130 | xpts[2] = x; 131 | ypts[2] = y + dy/2; 132 | zpts[2] = z; 133 | xpts[3] = x + dx/2; 134 | ypts[3] = y + dy/2; 135 | zpts[3] = z; 136 | xpts[4] = x; 137 | ypts[4] = y; 138 | zpts[4] = z + dz/2; 139 | xpts[5] = x + dx/2; 140 | ypts[5] = y; 141 | zpts[5] = z + dz/2; 142 | xpts[6] = x; 143 | ypts[6] = y + dy/2; 144 | zpts[6] = z + dz/2; 145 | xpts[7] = x + dx/2; 146 | ypts[7] = y + dy/2; 147 | zpts[7] = z + dz/2; 148 | 149 | level++; 150 | for (i=0; i<8; i++) { 151 | new_refinePathID = (refinePathID *10)+ (i+1); 152 | 153 | if (nfo->debug) 154 | printf("Refine on cell %d: (%f %f, %f) ... refinePathID=%d\n", i+1, xpts[i], ypts[i], zpts[i], new_refinePathID); 155 | 156 | stack_push(&octStack, xpts[7-i], ypts[7-i], zpts[7-i], dx/2.0, dy/2.0, dz/2.0, level, new_refinePathID); 157 | } 158 | } 159 | else { 160 | nfo->ncubes++; 161 | 162 | /* - calulate value using open_simplex_noise4 */ 163 | for (i=0; i<8; i++) { 164 | uint64_t npoints3 = nfo->npoints * 3; 165 | 166 | nfo->data[nfo->npoints] = curdata[i] = (float)open_simplex_noise4(osn, xpts[i]*noisespacefreq, 167 | ypts[i]*noisespacefreq, zpts[i]*noisespacefreq, t*noisetimefreq); 168 | 169 | nfo->points[npoints3] = xpts[i]; 170 | nfo->points[npoints3+1] = ypts[i]; 171 | nfo->points[npoints3+2] = zpts[i]; 172 | 173 | if (nfo->debug) 174 | printf("Cube_Point_%d: (%f %f, %f) = %f \n", i,nfo->points[npoints3],nfo->points[npoints3+1] , nfo->points[npoints3+2] , nfo->data[nfo->npoints] ); 175 | 176 | nfo->npoints++; 177 | } 178 | 179 | if (nfo->debug) 180 | printf("CubeID %llu, refinePathID %d connot be Refined ... refinement level=%d npoints=%llu +++++++++++++++\n", nfo->ncubes, refinePathID, level, nfo->npoints); 181 | 182 | } 183 | } 184 | 185 | if (nfo->debug) 186 | printf("Ended refinement on initial Block = %d\n", rpId); 187 | 188 | stack_delete(&octStack); 189 | 190 | 191 | } 192 | 193 | void cubeprint(cubeInfo *nfo) { 194 | int i; 195 | 196 | printf("NumBlock=%llu NumPoints=%llu +++++++++++++++\n", nfo->ncubes, nfo->npoints); 197 | /* for (i=0; i< nfo->npoints; i++) { */ 198 | /* printf("%f \n", nfo->data[i]); */ 199 | /* } */ 200 | 201 | } 202 | 203 | /* create a new stack */ 204 | void stack_new(stack *s, int maxsize, int debug) { 205 | cubeItem *newcubes; 206 | 207 | newcubes = (cubeItem *) malloc((size_t)maxsize*sizeof(cubeItem) ); 208 | 209 | s->cubes = newcubes; 210 | s->top = -1; 211 | s->maxsize = maxsize; 212 | s->size = 0; 213 | s->debug = debug; 214 | 215 | if (s->debug) { 216 | printf("Created stack\n"); 217 | } 218 | } 219 | 220 | /* check if stack is empty */ 221 | int stack_isempty(stack *s) { 222 | int empty=0; 223 | 224 | if (s->top < 0) 225 | empty=1; 226 | 227 | return empty; 228 | } 229 | 230 | /* check if stack is full */ 231 | int stack_isfull(stack *s) { 232 | int full=0; 233 | 234 | if (s->top >= s->maxsize) 235 | full=1; 236 | 237 | return full; 238 | } 239 | 240 | /* delete a stack */ 241 | void stack_delete(stack *s) { 242 | 243 | if (s->debug) { 244 | printf("Start to delete stack\n"); 245 | } 246 | free(s->cubes); 247 | s->cubes = NULL; 248 | s->top = -1; 249 | s->maxsize = 0; 250 | s->size = 0; 251 | s->debug = 0; 252 | 253 | if (s->debug) { 254 | printf("Deleted stack\n"); 255 | } 256 | } 257 | 258 | /* pushes ne element onto stack */ 259 | void stack_push(stack *s, float xval, float yval, float zval, float deltax, float deltay, float deltaz, int level, int r_id) { 260 | 261 | cubeItem newCube; 262 | 263 | if (stack_isfull(s)) { 264 | printf("ERROR: Stack FULL .... Could not push new cube data onto stack ...\n"); 265 | exit(1); 266 | } 267 | 268 | newCube.r_id = r_id; 269 | newCube.level = level; 270 | newCube.x = xval; 271 | newCube.y = yval; 272 | newCube.z = zval; 273 | newCube.dx = deltax; 274 | newCube.dy = deltay; 275 | newCube.dz = deltaz; 276 | 277 | s->top++; 278 | s->size++; 279 | s->cubes[s->top] = newCube; 280 | 281 | if (s->debug) { 282 | printf("Pushed Cube LRcorner position (%f %f, %f) stack_size=%d\n", s->cubes[s->top].x, s->cubes[s->top].y, s->cubes[s->top].z, s->size); 283 | printf("Pushed Cube delta values (%f %f, %f)\n", s->cubes[s->top].dx, s->cubes[s->top].dy, s->cubes[s->top].dz); 284 | } 285 | } 286 | 287 | /* removes top element from stack, */ 288 | void stack_pop(stack *s, float *xval, float *yval, float *zval, float *deltax, float *deltay, float *deltaz, int *level, int *r_id) { 289 | 290 | if (stack_isempty(s)) { 291 | printf("ERROR: Stack EMPTY ... Could not pop cube data off stack ...\n"); 292 | exit(1); 293 | } 294 | 295 | 296 | /* update data */ 297 | *r_id = s->cubes[s->top].r_id; 298 | *level = s->cubes[s->top].level; 299 | *xval = s->cubes[s->top].x; 300 | *yval = s->cubes[s->top].y; 301 | *zval = s->cubes[s->top].z; 302 | *deltax = s->cubes[s->top].dx; 303 | *deltay = s->cubes[s->top].dy; 304 | *deltaz = s->cubes[s->top].dz; 305 | 306 | if (s->debug) { 307 | printf("Popped Cube LRcorner position (%f %f, %f) stack_size=%d\n", s->cubes[s->top].x, s->cubes[s->top].y, s->cubes[s->top].z, s->size); 308 | printf("Popped Cube delta values (%f %f, %f)\n", s->cubes[s->top].dx, s->cubes[s->top].dy, s->cubes[s->top].dz); 309 | } 310 | 311 | s->top--; 312 | s->size--; 313 | } 314 | -------------------------------------------------------------------------------- /amr/cubes.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) DoD HPCMP PETTT. All rights reserved. 3 | * See LICENSE file for details. 4 | */ 5 | 6 | typedef struct cubeInfo { 7 | int debug; 8 | uint64_t ncubes; /* Number of cubes */ 9 | uint64_t npoints; /* Number of cube points */ 10 | float *points; /* Points of cube */ 11 | float *data; /* Values on points */ 12 | }cubeInfo; 13 | 14 | 15 | /* holds an entry in the stack (in this case cube information) */ 16 | typedef struct cubeItem { 17 | int r_id; 18 | int level; 19 | float x; 20 | float y; 21 | float z; 22 | float dx; 23 | float dy; 24 | float dz; 25 | } cubeItem; 26 | 27 | /* A stack that holds and array of stack items (in this case an array of cube information) */ 28 | typedef struct { 29 | cubeItem *cubes; 30 | int top; 31 | int size; 32 | int maxsize; 33 | int debug; 34 | } stack; 35 | 36 | 37 | void cubesinit(cubeInfo *nfo, int task, int levels, int debug); 38 | 39 | void cubesfree(cubeInfo *nfo); 40 | 41 | void refine(cubeInfo *nfo, int t, int rpId, float thres, int level_start, float x_start, float y_start, 42 | float z_start, float dx_start, float dy_start, float dz_start, struct osn_context *osn, int maxLevel, 43 | double noisespacefreq, double noisetimefreq); 44 | 45 | void cubeprint(cubeInfo *nfo); 46 | 47 | void stack_new(stack *astack, int maxsize, int debug); 48 | 49 | int stack_isempty(stack *s); 50 | 51 | int stack_isfull(stack *s); 52 | 53 | void stack_delete(stack *s); 54 | 55 | void stack_push(stack *s, float xval, float yval, float zval, float deltax, float deltay, float deltaz, int level, int r_id); 56 | 57 | void stack_pop(stack *s, float *xval, float *yval, float *zval, float *deltax, float *deltay, float *deltaz, int *level, int * r_id); 58 | 59 | -------------------------------------------------------------------------------- /amr/hdf5amr.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) DoD HPCMP PETTT. All rights reserved. 3 | * See LICENSE file for details. 4 | */ 5 | 6 | #include 7 | #include 8 | #include "hdf5.h" 9 | #include "hdf5amr.h" 10 | 11 | static const int fnstrmax = 4095; 12 | 13 | void hdf5_init(struct hdf5amrinfo *nfo, char *name, 14 | MPI_Comm comm, int rank, int nprocs, int tsteps) { 15 | /* Set up struct, haven't decided if using all of these yet */ 16 | nfo->name = name; 17 | nfo->comm = comm; 18 | nfo->rank = rank; 19 | nfo->nprocs = nprocs; 20 | nfo->tsteps = tsteps; 21 | 22 | nfo->numxvars = 0; 23 | nfo->maxxvars = 1000; 24 | nfo->xvarnames = (char **) malloc(nfo->maxxvars * sizeof(char *)); 25 | 26 | } 27 | 28 | void hdf5_addxvar(struct hdf5amrinfo *nfo, char *varname) { 29 | if(nfo->numxvars > nfo->maxxvars) 30 | return; /* Just ignore too many variables, for now */ 31 | nfo->xvarnames[nfo->numxvars] = varname; 32 | nfo->numxvars++; 33 | } 34 | 35 | void hdf5_write(struct hdf5amrinfo *nfo, int tstep, uint64_t cnpoints, float *points, float **xvals) { 36 | char fname[fnstrmax+1]; 37 | int timedigits = 4; 38 | uint64_t i, totalpoints, *npts_all; 39 | 40 | hid_t file_id; 41 | hid_t plist_id; 42 | hid_t memspace; 43 | hid_t filespace; 44 | hid_t attr_id; 45 | hid_t did; 46 | hsize_t start[1], count[1]; 47 | hsize_t dims[1]; 48 | herr_t err; 49 | MPI_Info info = MPI_INFO_NULL; 50 | uint64_t attr_data[2]; 51 | 52 | npts_all = (uint64_t *) malloc(nfo->nprocs * sizeof(uint64_t)); 53 | 54 | MPI_Allgather(&cnpoints, 1, MPI_UNSIGNED_LONG_LONG, 55 | npts_all, 1, MPI_UNSIGNED_LONG_LONG, nfo->comm); 56 | 57 | /* Set filename */ 58 | snprintf(fname, fnstrmax, "%s.%0*d.h5", nfo->name, timedigits, tstep); 59 | 60 | if(nfo->rank == 0) { 61 | 62 | for(totalpoints = 0, i = 0; i < nfo->nprocs; ++i) 63 | totalpoints += npts_all[i]; 64 | 65 | if( (file_id = H5Fcreate(fname, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) { 66 | fprintf(stderr, "writehdf5 error: could not create %s \n", fname); 67 | MPI_Abort(nfo->comm, 1); 68 | } 69 | 70 | dims[0] = totalpoints; 71 | filespace = H5Screate_simple(1, dims, NULL); 72 | 73 | /* Create the dataset with default properties */ 74 | 75 | for(i = 0; i < nfo->numxvars; i++){ 76 | 77 | did = H5Dcreate(file_id, nfo->xvarnames[i], H5T_NATIVE_FLOAT, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); 78 | H5Dclose(did); 79 | 80 | } 81 | /* Close filespace. */ 82 | H5Sclose(filespace); 83 | 84 | dims[0] = nfo->nprocs; 85 | filespace = H5Screate_simple(1, dims, NULL); 86 | 87 | did = H5Dcreate(file_id, "cnpoints", H5T_NATIVE_ULLONG, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); 88 | 89 | err = H5Dwrite(did, H5T_NATIVE_ULLONG, H5S_ALL, H5S_ALL, H5P_DEFAULT, npts_all); 90 | 91 | H5Dclose(did); 92 | H5Sclose(filespace); 93 | 94 | /** WRITE ATTRIBUTE INFORMATION **/ 95 | 96 | attr_data[0] = tstep; 97 | attr_data[1] = totalpoints; 98 | 99 | /* Create the data space for the attribute. */ 100 | dims[0] = 2; 101 | filespace = H5Screate_simple(1, dims, NULL); 102 | 103 | /* Create a dataset attribute. */ 104 | attr_id = H5Acreate2 (file_id, "tstep, totalpoints", H5T_NATIVE_ULLONG, filespace, 105 | H5P_DEFAULT, H5P_DEFAULT); 106 | 107 | /* Write the attribute data. */ 108 | err = H5Awrite(attr_id, H5T_NATIVE_ULLONG, attr_data); 109 | 110 | /* Close the attribute. */ 111 | err = H5Aclose(attr_id); 112 | 113 | /* Close the dataspace. */ 114 | err = H5Sclose(filespace); 115 | 116 | /* Close the file */ 117 | H5Fclose(file_id); 118 | 119 | } 120 | 121 | /* Set up MPI info */ 122 | MPI_Info_create(&info); 123 | MPI_Info_set(info, "striping_factor", "1"); 124 | 125 | /* Set up file access property list with parallel I/O access */ 126 | if( (plist_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) { 127 | printf("hdf5 error: Could not create property list \n"); 128 | MPI_Abort(nfo->comm, 1); 129 | } 130 | 131 | H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST); 132 | 133 | if(H5Pset_fapl_mpio(plist_id, MPI_COMM_WORLD, info) < 0) { 134 | printf("hdf5 error: Could not create property list \n"); 135 | MPI_Abort(nfo->comm, 1); 136 | } 137 | 138 | H5Pset_all_coll_metadata_ops(plist_id, 1 ); 139 | H5Pset_coll_metadata_write(plist_id, 1); 140 | 141 | MPI_Barrier(nfo->comm); 142 | if( (file_id = H5Fopen(fname, H5F_ACC_RDWR, plist_id)) < 0) { 143 | fprintf(stderr, "writehdf5p error: could not open %s \n", fname); 144 | MPI_Abort(nfo->comm, 1); 145 | } 146 | 147 | if(H5Pclose(plist_id) < 0) { 148 | printf("hdf5 error: Could not close property list \n"); 149 | MPI_Abort(nfo->comm, 1); 150 | } 151 | for(start[0] = 0, i = 0; i < nfo->rank; ++i) { 152 | start[0] += (hsize_t)npts_all[i]; 153 | } 154 | 155 | count[0] = (hsize_t)(cnpoints); 156 | 157 | memspace = H5Screate_simple(1, count, NULL); 158 | 159 | /* Create property list for collective dataset write. */ 160 | plist_id = H5Pcreate(H5P_DATASET_XFER); 161 | H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE); 162 | 163 | for(i = 0; i < nfo->numxvars; i++){ 164 | 165 | did = H5Dopen(file_id, nfo->xvarnames[i], H5P_DEFAULT); 166 | 167 | /* 168 | * Each process defines dataset in memory and writes it to the hyperslab 169 | * in the file. 170 | */ 171 | 172 | /* Select hyperslab in the file.*/ 173 | filespace = H5Dget_space(did); 174 | H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, NULL, count, NULL ); 175 | 176 | err = H5Dwrite(did, H5T_NATIVE_FLOAT, memspace, filespace, plist_id, xvals[i]); 177 | 178 | if( err < 0) { 179 | fprintf(stderr, "hdf5 error: could not write datset %s \n", nfo->xvarnames[i]); 180 | MPI_Abort(nfo->comm, 1); 181 | } 182 | 183 | err = H5Dclose(did); 184 | } 185 | 186 | if(H5Sclose(memspace) != 0) 187 | printf("hdf5 error: Could not close memory space \n"); 188 | 189 | if(H5Pclose(plist_id) < 0) 190 | printf("hdf5 error: Could not close property list \n"); 191 | 192 | if(H5Fclose(file_id) != 0) 193 | printf("hdf5 error: Could not close HDF5 file \n"); 194 | 195 | free(npts_all); 196 | 197 | } 198 | 199 | void hdf5_finalize(struct hdf5amrinfo *nfo) { 200 | free(nfo->xvarnames); 201 | } 202 | -------------------------------------------------------------------------------- /amr/hdf5amr.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) DoD HPCMP PETTT. All rights reserved. 3 | * See LICENSE file for details. 4 | */ 5 | 6 | #include 7 | 8 | struct hdf5amrinfo { 9 | char *name; 10 | MPI_Comm comm; 11 | int rank; 12 | int nprocs; 13 | int tsteps; 14 | 15 | int numxvars; 16 | int maxxvars; 17 | char **xvarnames; 18 | 19 | }; 20 | 21 | void hdf5_addxvar(struct hdf5amrinfo *nfo, char *varname); 22 | 23 | void hdf5_write(struct hdf5amrinfo *nfo, int tstep, uint64_t cnpoints, float *points, float **xvals); 24 | 25 | void hdf5_finalize(struct hdf5amrinfo *nfo); 26 | 27 | void hdf5_init(struct hdf5amrinfo *nfo, char *name, 28 | MPI_Comm comm, int rank, int nprocs, int tsteps); 29 | -------------------------------------------------------------------------------- /amr/ncamr.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) DoD HPCMP PETTT. All rights reserved. 3 | * See LICENSE file for details. 4 | */ 5 | 6 | #include 7 | #include 8 | 9 | #include "ncamr.h" 10 | 11 | #include "netcdf.h" 12 | #include "netcdf_par.h" 13 | 14 | #define NCERR {if(err != NC_NOERR) {printf("(rank %d) Error at line %d: %s\n",rank,__LINE__,nc_strerror(err)); fflush(stdout); MPI_Abort(nfo->comm,1);}} 15 | 16 | static const int fnstrmax = 4095; 17 | 18 | void nc_init(struct ncamrinfo *nfo, char *name, 19 | MPI_Comm comm, int rank, int nprocs, int tsteps) { 20 | /* Set up struct, haven't decided if using all of these yet */ 21 | nfo->name = name; 22 | nfo->comm = comm; 23 | nfo->rank = rank; 24 | nfo->nprocs = nprocs; 25 | nfo->tsteps = tsteps; 26 | 27 | nfo->numxvars = 0; 28 | nfo->maxxvars = 1000; 29 | nfo->xvarnames = (char **) malloc(nfo->maxxvars * sizeof(char *)); 30 | 31 | } 32 | 33 | void nc_addxvar(struct ncamrinfo *nfo, char *varname) { 34 | if(nfo->numxvars > nfo->maxxvars) 35 | return; /* Just ignore too many variables, for now */ 36 | nfo->xvarnames[nfo->numxvars] = varname; 37 | nfo->numxvars++; 38 | } 39 | 40 | void nc_write(struct ncamrinfo *nfo, int tstep, uint64_t cnpoints, float *points, float **xvals) { 41 | 42 | char fname[fnstrmax+1]; 43 | char dimname[fnstrmax+1]; 44 | int timedigits = 4; 45 | uint64_t i, totalpoints, *npts_all; 46 | 47 | int ncid; 48 | size_t start[1] = {0}; 49 | size_t count[1] = {0}; 50 | size_t dims[1] = {0}; 51 | int err = 0; 52 | MPI_Info info = MPI_INFO_NULL; 53 | uint64_t attr_data[2] = {0,0}; 54 | int *dimids; 55 | int *varids; 56 | int varid; 57 | int dimid; 58 | int rank = nfo->rank; 59 | 60 | npts_all = (uint64_t *) malloc(nfo->nprocs * sizeof(uint64_t)); 61 | 62 | MPI_Allgather(&cnpoints, 1, MPI_UNSIGNED_LONG_LONG, 63 | npts_all, 1, MPI_UNSIGNED_LONG_LONG, nfo->comm); 64 | 65 | /* Set filename */ 66 | snprintf(fname, fnstrmax, "%s.%0*d.nc", nfo->name, timedigits, tstep); 67 | 68 | //if(nfo->rank == 0) { 69 | 70 | for(totalpoints = 0, i = 0; i < nfo->nprocs; ++i) 71 | totalpoints += npts_all[i]; 72 | 73 | err = nc_create_par(fname,NC_NETCDF4|NC_MPIIO,nfo->comm,info,&ncid); NCERR; 74 | 75 | dims[0] = totalpoints; 76 | count[0] = nfo->nprocs; 77 | 78 | err = nc_def_dim(ncid,"phony_dim_0",nfo->nprocs,&dimid); NCERR; 79 | err = nc_def_var(ncid,"cnpoints",NC_UINT64,1,&dimid,&varid); NCERR; 80 | 81 | /* A little bit of memory allocation. */ 82 | dimids = (int*)malloc(sizeof(int)*nfo->numxvars); 83 | varids = (int*)malloc(sizeof(int)*nfo->numxvars); 84 | 85 | /* Create the dataset with default properties, one dimension per variable.*/ 86 | for(i = 0; i < nfo->numxvars; i++){ 87 | 88 | /* Define phony dim. */ 89 | snprintf(dimname,fnstrmax,"phony_dim_%jd",i+1); 90 | err = nc_def_dim(ncid,dimname,totalpoints,&dimids[i]); NCERR; 91 | 92 | /* Define variable.*/ 93 | 94 | err = nc_def_var(ncid,nfo->xvarnames[i],NC_FLOAT,1,&dimids[i],&varids[i]); NCERR; 95 | err = nc_var_par_access(ncid,varids[i],NC_COLLECTIVE); NCERR; 96 | } 97 | 98 | err = nc_put_vara_long(ncid,varid,start,count,npts_all); NCERR; 99 | 100 | attr_data[0] = tstep; 101 | attr_data[1] = totalpoints; 102 | 103 | /* Create the data space for the attribute. */ 104 | dims[0] = 2; 105 | err = nc_put_att(ncid,NC_GLOBAL,"tstep, totalpoints",NC_UINT64,2,attr_data); 106 | 107 | err = nc_enddef(ncid); NCERR; 108 | 109 | MPI_Barrier(nfo->comm); 110 | 111 | MPI_Info_create(&info); 112 | MPI_Info_set(info, "striping_factor", "1"); 113 | 114 | MPI_Barrier(nfo->comm); 115 | 116 | for(start[0] = 0, i = 0; i < nfo->rank; ++i) { 117 | start[0] += (size_t)npts_all[i]; 118 | } 119 | 120 | count[0] = (size_t)(cnpoints); 121 | 122 | /* Create property list for collective dataset write. */ 123 | for(i = 0; i < nfo->numxvars; i++){ 124 | err = nc_put_vara_float(ncid,varids[i],start,count,xvals[i]); NCERR; 125 | } 126 | 127 | MPI_Barrier(nfo->comm); 128 | nc_close(ncid); NCERR; 129 | 130 | free(npts_all); 131 | free(dimids); 132 | free(varids); 133 | 134 | } 135 | 136 | void nc_final(struct ncamrinfo *nfo) { 137 | free(nfo->xvarnames); 138 | } 139 | -------------------------------------------------------------------------------- /amr/ncamr.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) DoD HPCMP PETTT. All rights reserved. 3 | * See LICENSE file for details. 4 | */ 5 | 6 | #ifndef NCAMR_H 7 | #define NCAMR_H 8 | 9 | #include 10 | #include 11 | 12 | struct ncamrinfo { 13 | char *name; 14 | MPI_Comm comm; 15 | int rank; 16 | int nprocs; 17 | int tsteps; 18 | 19 | int numxvars; 20 | int maxxvars; 21 | char **xvarnames; 22 | }; 23 | 24 | void nc_addxvar(struct ncamrinfo *nfo, char *varname); 25 | 26 | void nc_write(struct ncamrinfo *nfo, int tstep, uint64_t cnpoints, float *points, float **xvals); 27 | 28 | void nc_final(struct ncamrinfo *nfo); 29 | 30 | void nc_init(struct ncamrinfo *nfo, char *name, 31 | MPI_Comm comm, int rank, int nprocs, int tsteps); 32 | 33 | 34 | #endif 35 | -------------------------------------------------------------------------------- /amr/processlog.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Process an output log from unstruct and produce a results table 4 | # + Determines output method from log contents 5 | # + Can also check filename for method 6 | # - Filename must contain .przm., .posix., .mpi., etc (see fname2method) 7 | # - If filename & log contents don't agree, will abort with error 8 | # - If you didn't name file properly (or at all), just gives a warning 9 | # + Each column is a time step 10 | # + Table rows: method, cores, grid points (Million), full file size, time, rate (GB/s) 11 | # + Multiple methods/types can be pasted into Excel or similar for plotting/analysis 12 | 13 | import sys 14 | import re 15 | 16 | fname2method = { ".przm.": "MPI-Indiv", ".posix.": "ADIOS-POSIX", 17 | ".mpi.": "ADIOS-MPI", ".mpilus.": "ADIOS-MPI_Lustre", ".mpiagg.": "ADIOS-MPI_Aggr", 18 | ".phdf5.": "ADIOS-PHDF5", ".hdf5.": "HDF5", } 19 | lname2method = { "przm": "MPI-Indiv", "POSIX": "ADIOS-POSIX", 20 | "MPI ": "ADIOS-MPI", "MPI_LUSTRE": "ADIOS-MPI_Lustre", 21 | "MPI_AGGREGATE": "ADIOS-MPI_Aggr", "PHDF5": "ADIOS-PHDF5", 22 | "hdf5": "HDF5" } 23 | 24 | v = [] 25 | 26 | for fname in sys.argv[1:]: 27 | 28 | # Determine method from file name, if it's there 29 | method1key = next((x for x in fname2method.keys() if x in fname), False) 30 | if not method1key: print "WARNING: method not specified in filename" 31 | 32 | f = open(fname) 33 | for l in f: 34 | ls = l.split() 35 | 36 | if(len(ls) < 1): continue 37 | if(len(ls) < 2): ls.append("") # Ensure all outer if's can handle ls 38 | 39 | if ls[0][0:4] == "tsk=": 40 | if ls[0][4] == "\"": continue # Skip LSF copy of batch script 41 | 42 | cores = int(ls[0][4:]) * int(ls[1]) * int(ls[2]) 43 | size = ls[3][2:] + 'x' + ls[3][2:] + 'x' + ls[3][2:] 44 | sizeprod = float(ls[3][2:]) * float(ls[3][2:]) * float(ls[3][2:]) 45 | sizegb = sizeprod*4*8*8*8*.8/1024**3 46 | 47 | # Determine method from line; make sure matches one from file name 48 | method2key = next((x for x in lname2method.keys() if re.search(x, l)), False) 49 | if method2key: 50 | if method1key: 51 | if fname2method[method1key] != lname2method[method2key]: 52 | print "Error: method in filename doesn't match method in file" 53 | sys.exit(1) 54 | else: 55 | method = fname2method[method1key] 56 | else: 57 | method = lname2method[method2key] 58 | else: 59 | print "WARNING: method not specified in log file" 60 | if method1key: 61 | method = fname2method[method1key] 62 | else: 63 | print "ERROR: not method specified, cannot proceed" 64 | sys.exit(1) 65 | elif ls[0] == "Output": 66 | outtime = float(ls[11][:-1]) 67 | v.append([method, cores, size, sizegb, outtime, sizegb/outtime]) 68 | 69 | print "%-16s %5s %15s %5s %5s %5s" % ("method", "cores", "size", "f_GB", "ftime", "fGB/s") 70 | for i in v: 71 | print "%-16s %5d %8s %7.1f %6.2f %6.2f" % tuple(i) 72 | 73 | -------------------------------------------------------------------------------- /amr/tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "Running HDF5 tests." 4 | mpirun -np 8 ./amr --tasks 2 2 2 --size 9 9 9 --levels 3 --tsteps 1 --hdf5 5 | echo "" 6 | echo "Runing netCDF tests." 7 | mpirun -np 8 ./amr --tasks 2 2 2 --size 9 9 9 --levels 3 --tsteps 1 --nc 8 | echo "" 9 | echo "Finished." 10 | -------------------------------------------------------------------------------- /amr/vtkout.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) DoD HPCMP PETTT. All rights reserved. 3 | * See LICENSE file for details. 4 | */ 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #include "vtkout.h" 15 | 16 | static const int fnstrmax = 4095; 17 | #define cubeVertices 8 18 | #define vertexDims 3 19 | 20 | //static const int cubeVertices = 8; 21 | //static const int vertexDims= 3; 22 | static const int cubeVertexDims = cubeVertices * vertexDims; 23 | 24 | void writevtk(char *name, MPI_Comm comm, int rank, int nprocs, int tstep, 25 | uint64_t npoints, uint64_t ncubes, float *points, float *xvals, char *xname, int debug) { 26 | 27 | char fname[fnstrmax+1]; 28 | int timedigits = 4; 29 | FILE *f; 30 | int i; 31 | uint64_t allcubes; /* total cube count: sum over all ranks */ 32 | uint64_t *rncubes=NULL; /* cube count for currrent rank */ 33 | 34 | int nallpoints=0; 35 | int nallxvals=0; 36 | float *allpoints=NULL; 37 | float *allxvals=NULL; 38 | 39 | int cpointcnts; 40 | int cpointoffsets; 41 | int *pointcnts=NULL; 42 | int *pointoffsets=NULL; 43 | 44 | int cxvalcnts; 45 | int cxvaloffsets; 46 | int *xvalcnts=NULL; 47 | int *xvaloffsets=NULL; 48 | 49 | if (debug) { 50 | printf("Debug 0.0 (rank=%d): Entered writevtk routine\n", rank); 51 | } 52 | 53 | pointcnts = (int *) malloc(nprocs*sizeof(int)); 54 | pointoffsets = (int *) malloc(nprocs*sizeof(int)); 55 | xvalcnts = (int *) malloc(nprocs*sizeof(int)); 56 | xvaloffsets = (int *) malloc(nprocs*sizeof(int)); 57 | 58 | if(rank == 0) { 59 | rncubes = (uint64_t *) malloc(nprocs*sizeof(uint64_t)); 60 | } 61 | 62 | MPI_Gather(&ncubes, 1, MPI_UNSIGNED_LONG_LONG, rncubes, 1, MPI_LONG_LONG, 0, comm); 63 | 64 | cpointcnts = ncubes * cubeVertexDims; 65 | MPI_Allgather(&cpointcnts, 1, MPI_INT, pointcnts, 1, MPI_INT, comm); 66 | 67 | cxvalcnts = ncubes * cubeVertices; 68 | MPI_Allgather(&cxvalcnts, 1, MPI_INT, xvalcnts, 1, MPI_INT, comm); 69 | 70 | cpointoffsets = 0; 71 | cxvaloffsets = 0; 72 | for(i = 1; i < rank+1; i++) { 73 | cpointoffsets += pointcnts[i-1]; 74 | cxvaloffsets += xvalcnts[i-1]; 75 | } 76 | 77 | MPI_Allgather(&cpointoffsets, 1, MPI_INT, pointoffsets, 1, MPI_INT, comm); 78 | MPI_Allgather(&cxvaloffsets, 1, MPI_INT, xvaloffsets, 1, MPI_INT, comm); 79 | 80 | if(rank == 0) { 81 | allcubes=0; 82 | for(i = 0; i < nprocs; i++) { 83 | allcubes += rncubes[i]; 84 | } 85 | nallpoints = allcubes*cubeVertexDims; 86 | nallxvals = allcubes*cubeVertices; 87 | 88 | allpoints = (float *) malloc(nallpoints*sizeof(float)); 89 | allxvals = (float *) malloc(nallxvals*sizeof(float)); 90 | 91 | for(i = 0; i 9 | #include 10 | 11 | /* Convert a string with/out trailing 'x' to an integer and bool (with/out the x) 12 | * s: pointer to input string 13 | * i: pointer to an output integer 14 | * hasx: pointer to output boolean: true of string ends in x 15 | */ 16 | 17 | void atoix(const char *s, int *i, int *hasx) 18 | { 19 | char *ss = strdup(s); 20 | int len = strlen(ss); 21 | if(ss[len-1] == 'x' || ss[len-1] == 'X') { 22 | *hasx = 1; 23 | ss[len-1] = 0; 24 | len--; 25 | } else 26 | *hasx = 0; 27 | *i = atoi(ss); 28 | free(ss); 29 | } 30 | 31 | /* Convert a string with/out trailing 'x' to a double and bool (with/out the x) 32 | * s: pointer to input string 33 | * d: pointer to an output integer 34 | * hasx: pointer to output boolean: true of string ends in x 35 | */ 36 | 37 | void atodx(const char *s, double *d, int *hasx) 38 | { 39 | char *ss = strdup(s); 40 | int len = strlen(ss); 41 | if(ss[len-1] == 'x' || ss[len-1] == 'X') { 42 | *hasx = 1; 43 | ss[len-1] = 0; 44 | len--; 45 | } else 46 | *hasx = 0; 47 | *d = strtod(ss, NULL); 48 | free(ss); 49 | } 50 | 51 | -------------------------------------------------------------------------------- /autoinit.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # source me (sh/ksh/bash) 3 | # For known systems, to set environment variables before running make 4 | 5 | ### All DSRC systems 6 | if [ -n "$BC_HOST" ]; then 7 | 8 | export CC=mpicc 9 | export OPT="-O3 -xhost -ip" # Assumes intel compiler 10 | 11 | ## All Cray's using the CrayPE compiler environment 12 | if [ -n "$CRAYPE_DIR" ]; then 13 | 14 | echo "Using DSRC Cray with CrayPE configuration" 15 | prgenv=`echo $LOADEDMODULES | grep -Po 'PrgEnv-.*?/'` 16 | echo " Current environment: $prgenv" 17 | export CC=cc # CrayPE compiler wrapper 18 | if [ "$prgenv" == "PrgEnv-intel/" ]; then 19 | export OPT="-O3 -xhost -ip" 20 | elif [ "$prgenv" == "PrgEnv-gnu/" ]; then 21 | export OPT="-O3" 22 | else 23 | export OPT="-O2" 24 | fi 25 | echo " Set OPT=$OPT" 26 | 27 | ## Topaz/SGI 28 | elif [ "$BC_HOST" == "topaz" ]; then 29 | echo "Using DSRC Topaz configuration" 30 | export CC=icc # Assume intel compiler for now 31 | export LIBS="-lmpi" 32 | 33 | ## Other DSRC 34 | else 35 | echo "Using default DSRC configuration" 36 | fi 37 | 38 | ### All POD systems (all seem to define the following variable) 39 | elif [ -n "$POD_PBSSERVERS" ]; then 40 | 41 | echo "Using default POD configuration" 42 | export CC=mpicc 43 | export OPT="-O3 -xhost -ip" # Assumes intel compiler 44 | 45 | ### Nothing else to try 46 | else 47 | echo "No special configuration detected" 48 | if [ -n "`which mpicc`" ]; then 49 | export CC=mpicc 50 | echo " Found mpicc - setting CC=$CC" 51 | fi 52 | fi 53 | 54 | -------------------------------------------------------------------------------- /cartiso/Makefile: -------------------------------------------------------------------------------- 1 | 2 | include ../Makefile.inc 3 | 4 | OBJS = iso.o sfc.o cartiso.o 5 | SRCS = iso.c sfc.c cartiso.c 6 | 7 | ### Add Output Modules Here ### 8 | 9 | # PVTI Full Output Module 10 | ENABLE_PVTI = 1 11 | ifeq ($(ENABLE_PVTI),1) 12 | OBJS += pvti.o 13 | SRCS += pvti.c 14 | CFLAGS += -DHAS_PVTI 15 | endif 16 | 17 | # PVTP Isosurface Output Module 18 | ENABLE_PVTP = 1 19 | ifeq ($(ENABLE_PVTP),1) 20 | OBJS += pvtp.o 21 | SRCS += pvtp.c 22 | CFLAGS += -DHAS_PVTP 23 | endif 24 | 25 | ifdef ADIOS_HOME 26 | # ADIOS Full Output Module 27 | OBJS += adiosfull.o 28 | SRCS += adiosfull.c 29 | # ADIOS Isosurface Output Module 30 | OBJS += adiosiso.o 31 | SRCS += adiosiso.c 32 | endif 33 | 34 | ### if NETCDF_DIR set then enable netCDF 35 | ifdef NETCDF_DIR 36 | OBJS += nci.o ncp.o 37 | SRCS += nci.c ncp.c 38 | endif 39 | 40 | ### if HDF5_DIR set then enable HDF5 ### 41 | ifdef HDF5_DIR 42 | OBJS += hdf5i.o hdf5p.o 43 | SRCS += hdf5i.c hdf5p.c 44 | endif 45 | 46 | ### End of Output Modules ### 47 | 48 | CFLAGS += $(OSNINC) 49 | 50 | .PHONY: clean depend 51 | 52 | cartiso: $(OSNOBJ) $(OBJS) 53 | $(CC) -o $@ $^ $(LDFLAGS) $(LIBS) 54 | 55 | clean: 56 | rm -f *.o cartiso sfc 57 | 58 | depend: 59 | makedepend -- $(CFLAGS) -- -Y $(SRCS) # ignore warnings about system headers 60 | 61 | # Test version of sfc (for testing proper functioning of sfc code) 62 | sfc: sfc.c 63 | cc $(CFLAGS) -DSFC_SERPENTINE_TEST -o sfc sfc.c 64 | 65 | .c.o: 66 | $(CC) $(CFLAGS) -c $< $(INCLUDE) 67 | 68 | # DO NOT DELETE 69 | 70 | iso.o: iso.h 71 | sfc.o: sfc.h 72 | cartiso.o: sfc.h iso.h ../timer.h ../osn/open-simplex-noise.h pvti.h pvtp.h 73 | cartiso.o: adiosfull.h adiosiso.h 74 | pvti.o: ../pdirs.h pvti.h 75 | pvtp.o: ../pdirs.h pvtp.h 76 | adiosfull.o: adiosfull.h ../pdirs.h 77 | adiosiso.o: adiosiso.h 78 | -------------------------------------------------------------------------------- /cartiso/README: -------------------------------------------------------------------------------- 1 | 2 | Copyright (c) DoD HPCMP PETTT. All rights reserved. 3 | See LICENSE file for details. 4 | 5 | #. Building 6 | 7 | One may build from the parent miniIO directory or within this directory. 8 | Either way, first edit the parent directory's Makefile.inc and set the 9 | correct compilers and flags for the system. See the corresponding ../README 10 | file for more specifics. To compile and link in this directory, simply execute 11 | "make". 12 | 13 | #. Running 14 | 15 | cartiso is an MPI program and thus requires an MPI launcher (mpirun, mpiexec, 16 | aprun, etc.). It will look something like this; 17 | mpirun -np N ./cartiso --tasks # # # --size # # # [optional_options] 18 | 19 | The "tasks" and "size" arguments are required. The corresponding values for 20 | those arguments have certain constraints. The meaning of those arguments, the 21 | constraints, and information about optional arguments are described in 22 | cartiso.c and are printed out when no options or bad options are provided to 23 | the cartiso executable. 24 | 25 | #. Adding an Output Module 26 | 27 | cartiso allows for the addition of custom output modules which allow the 28 | benchmarking/testing/profiling of alternate output methods. 29 | 30 | There are two types of output modules in cartiso: (1) full output and (2) 31 | isosurface output. Full output writes out the entire Cartesian data arrays. 32 | Isosurface output writes out the triangles from the isosurface generation. 33 | The PVTI and PVTP Output Modules are examples of each, respectively. At a 34 | minimum, we assume the Output Module has (a) a header file with the function 35 | signature and (b) a source file with the function. 36 | The following is the process for adding an Output Module: 37 | 38 | #.1. Add to I/O Modules section in Makefile 39 | 40 | Modules with no external dependences probably only require adding to OBJS, SRCS, 41 | and CFLAGS. For example: 42 | OBJS += myoutputmodule.o # Add my object file 43 | # ^^ Note the "+="; Do NOT use = 44 | SRCS += myoutputmodule.c # Add my source file 45 | CFLAGS += -DHAS_MYOUTPUT # Add a define for conditional compilation 46 | 47 | If you have external dependences, such as I/O libraries like HDF5, NetCDF, 48 | ADIOS, etc., you can also update CFLAGS and LIBS to include those: 49 | CFLAGS += -I/path/to/library/include 50 | LIBS += -L/path/to/library/lib -lthelib 51 | 52 | #.2. Add Includes to cartiso.c 53 | 54 | The section near the top of cartiso.c is marked with comments. Add an #include 55 | directive for your include file, but wrapped in conditional compilation 56 | directives using the "-D..." flag that you created in the Makefile, e.g.: 57 | #ifdef HAS_MYOUPUT 58 | # include "myoutputmodule.h" 59 | #endif 60 | 61 | #.3. Recommended: Add Usage String to cartiso.c 62 | 63 | The section for usage strings for Output Modules is marked with comments. It 64 | is recommended to provide command line options for your users to enable your 65 | output method and perhaps choose file names or other options. Provide the 66 | usage documentation for those here. As before, please add these with 67 | conditional compilation (#ifdef ...). 68 | 69 | #.4. Recommended: Add Variables to cartiso.c 70 | 71 | The section for variables for Output Modules is marked with comments. You may 72 | or may not need any variables. However, it is recommended to provide command 73 | line options for your users to enable your output method and perhaps choose 74 | file names or other options. Variables needed to track this should be added 75 | here. As before, please add these with conditional compilation (#ifdef ...). 76 | 77 | #.5. Recommended: Add Command Line Options to cartiso.c 78 | 79 | The section in the command line parsing section for Output Modules is marked 80 | with comments. It isn't required, but it is recommended to provide command 81 | line options for your users to enable your output method and perhaps choose 82 | file names or other options. As before, please add these with conditional 83 | compilation (#ifdef ...). Follow the approach used to process other command 84 | line parameters; in particular, use a complete "else if" block, and increment 85 | the "a" variable if you process additional parameters after your initial one. 86 | Don't forget the conditional compilation directives. 87 | 88 | #.6. Optional: Add Initialization to cartiso.c 89 | 90 | The section for initialization is marked with comments. This may not be 91 | necessary for all Output modules. However some modules may require the 92 | initialization of an external library, for example. This is the most 93 | reasonable location to perform the initialization, because command line options 94 | have been parsed, MPI has been initialized, and the grid has been set up. If 95 | you need rare, special initialization, e.g., before MPI is initialized, please 96 | comment your addition adequately. Don't forget the conditional compilation 97 | directives, wherever you initialize. 98 | 99 | #.7. Add FULL OUTPUT Per Time Step Function Calls to cartiso.c 100 | 101 | This is for Full Output modules that write out the entire data array only. The 102 | section for Full Output "per time step" function calls is commented. This is 103 | where the real work happens. It is expected that after every time step your 104 | Output Module will write out the updated data. Don't forget the conditional 105 | compilation directives. 106 | 107 | 108 | #.8. Add ISOSURFACE OUTPUT Per Time Step Function Calls to cartiso.c 109 | 110 | This is for Isosurface Output modules that write out the isosurface only. The 111 | section for Isosurface Output "per time step" function calls is commented. 112 | This is where the real work happens. It is expected that after every time step 113 | your Output Module will write out the updated isosurface. Don't forget the 114 | conditional compilation directives. 115 | 116 | #.9. Optional: Add Cleanup to cartiso.c 117 | 118 | The section for cleanup is marked with comments. This may not be necessary for 119 | all Output Modules. However, some Modules may require the cleanup/close-out/ 120 | finalization of an external library, for example. This is the most reasonable 121 | location for the cleanup, as it occurs before the cleanup of the rest of code. 122 | Don't forget the conditional compilation directives. 123 | 124 | #.10. Update Make Dependencies 125 | 126 | Adding your source filenames to the Makefile's SRCS means that a rerun of "make 127 | depend" will update the Makefile with your new dependencies. 128 | 129 | -------------------------------------------------------------------------------- /cartiso/adiosfull.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) DoD HPCMP PETTT. All rights reserved. 3 | * See LICENSE file for details. 4 | */ 5 | 6 | #include 7 | #include 8 | 9 | #include "adiosfull.h" 10 | #include "pdirs.h" 11 | 12 | static const int fnstrmax = 4095; 13 | 14 | void adiosfull_init(struct adiosfullinfo *nfo, char *method, char *transform, 15 | char *name, MPI_Comm comm, int rank, int nprocs, 16 | int tsteps, int ni, int nj, int nk, int is, int cni, int js, int cnj, 17 | int ks, int cnk, float deltax, float deltay, float deltaz, char *adiosopts) 18 | { 19 | char dirname[fnstrmax+1]; 20 | static char emptystr[] = ""; 21 | 22 | /* Make directory for output collection, all timesteps */ 23 | /*snprintf(dirname, fnstrmax, "%s_adios.d/", name); 24 | mkdir1task(dirname, comm);*/ 25 | 26 | /* Set up struct */ 27 | nfo->name = name; 28 | nfo->transform = transform; 29 | nfo->comm = comm; 30 | nfo->rank = rank; 31 | nfo->nprocs = nprocs; 32 | nfo->tsteps = tsteps; 33 | nfo->ni = ni; 34 | nfo->nj = nj; 35 | nfo->nk = nk; 36 | nfo->is = is; 37 | nfo->cni = cni; 38 | nfo->js = js; 39 | nfo->cnj = cnj; 40 | nfo->ks = ks; 41 | nfo->cnk = cnk; 42 | nfo->deltax = deltax; 43 | nfo->deltay = deltay; 44 | nfo->deltaz = deltaz; 45 | nfo->nvars = 0; 46 | nfo->maxvars = 1000; 47 | nfo->varnames = (char **) malloc(nfo->maxvars * sizeof(char *)); 48 | nfo->datas = (float **) malloc(nfo->maxvars * sizeof(float *)); 49 | nfo->bufallocsize = 0; 50 | 51 | if(!adiosopts) adiosopts = emptystr; 52 | 53 | /* Set up ADIOS */ 54 | adios_init_noxml(comm); 55 | adios_declare_group(&nfo->gid, nfo->name, "", adios_flag_no); 56 | /*chkdir1task(dirname, comm);*/ 57 | adios_select_method(nfo->gid, method, adiosopts, /*dirname*/""); 58 | 59 | /* Define output variables */ 60 | adios_define_var(nfo->gid, "rank", "", adios_integer, "", "", ""); 61 | adios_define_var(nfo->gid, "tstep", "", adios_integer, "", "", ""); 62 | adios_define_var(nfo->gid, "ni", "", adios_integer, "", "", ""); 63 | adios_define_var(nfo->gid, "nj", "", adios_integer, "", "", ""); 64 | adios_define_var(nfo->gid, "nk", "", adios_integer, "", "", ""); 65 | adios_define_var(nfo->gid, "is", "", adios_integer, "", "", ""); 66 | adios_define_var(nfo->gid, "cni", "", adios_integer, "", "", ""); 67 | adios_define_var(nfo->gid, "js", "", adios_integer, "", "", ""); 68 | adios_define_var(nfo->gid, "cnj", "", adios_integer, "", "", ""); 69 | adios_define_var(nfo->gid, "ks", "", adios_integer, "", "", ""); 70 | adios_define_var(nfo->gid, "cnk", "", adios_integer, "", "", ""); 71 | adios_define_var(nfo->gid, "deltax", "", adios_real, "", "", ""); 72 | adios_define_var(nfo->gid, "deltay", "", adios_real, "", "", ""); 73 | adios_define_var(nfo->gid, "deltaz", "", adios_real, "", "", ""); 74 | } 75 | 76 | void adiosfull_addvar(struct adiosfullinfo *nfo, char *varname, float *data) 77 | { 78 | int64_t varid; 79 | if(nfo->nvars >= nfo->maxvars) 80 | return; /* Just ignore too many variables, for now */ 81 | nfo->varnames[nfo->nvars] = varname; 82 | nfo->datas[nfo->nvars] = data; 83 | nfo->nvars++; 84 | varid = adios_define_var(nfo->gid, varname, "", adios_real, "cni,cnj,cnk", 85 | "ni,nj,nk", "is,js,ks"); 86 | if(nfo->transform) 87 | adios_set_transform(varid, nfo->transform); 88 | } 89 | 90 | void adiosfull_write(struct adiosfullinfo *nfo, int tstep) 91 | { 92 | char fname[fnstrmax+1]; 93 | int timedigits = 4; 94 | uint64_t ijkelems, groupsize, totalsize; 95 | int64_t handle; 96 | int ret, i; 97 | int bufneeded; 98 | 99 | /* Set sizes */ 100 | ijkelems = (uint64_t)nfo->cni * nfo->cnj * nfo->cnk; 101 | groupsize = sizeof(int) /*rank*/ + sizeof(int) /*tstep*/ + 102 | sizeof(int)*3 /*ni,nj,nk*/ + sizeof(int)*6 /*is-cnk*/ + 103 | sizeof(float)*3 /*deltax,y,z*/ + 104 | sizeof(float) * ijkelems * nfo->nvars + 1024; 105 | 106 | /* Allocate buffer large enough for all data to write, if not done already */ 107 | bufneeded = (int)(groupsize/(1024*1024)); 108 | bufneeded += bufneeded/8 + 2; /* Add an extra 12.5% & 2MB to be sure */ 109 | if(nfo->bufallocsize < bufneeded) { 110 | # if ADIOS_VERSION_GE(1,13,0) 111 | /* Don't set any max buffer size - seems to work best without it */ 112 | # elif ADIOS_VERSION_GE(1,10,0) 113 | adios_set_max_buffer_size(bufneeded); 114 | # else 115 | adios_allocate_buffer(ADIOS_BUFFER_ALLOC_NOW, bufneeded); 116 | # endif 117 | nfo->bufallocsize = bufneeded; 118 | } 119 | 120 | /* Set filename */ 121 | snprintf(fname, fnstrmax, "%s_%0*d", nfo->name, timedigits, tstep); 122 | 123 | /* Open & Write */ 124 | ret = adios_open(&handle, nfo->name, fname, "w", nfo->comm); 125 | if(ret) { 126 | fprintf(stderr, "Error opening ADIOS file: %s\n", fname); 127 | return; 128 | } 129 | # if ADIOS_VERSION_LE(1,9,0) 130 | adios_group_size(handle, groupsize, &totalsize); 131 | # endif 132 | 133 | adios_write(handle, "rank", &nfo->rank); 134 | adios_write(handle, "tstep", &tstep); 135 | adios_write(handle, "ni", &nfo->ni); 136 | adios_write(handle, "nj", &nfo->nj); 137 | adios_write(handle, "nk", &nfo->nk); 138 | adios_write(handle, "is", &nfo->is); 139 | adios_write(handle, "cni", &nfo->cni); 140 | adios_write(handle, "js", &nfo->js); 141 | adios_write(handle, "cnj", &nfo->cnj); 142 | adios_write(handle, "ks", &nfo->ks); 143 | adios_write(handle, "cnk", &nfo->cnk); 144 | adios_write(handle, "deltax", &nfo->deltax); 145 | adios_write(handle, "deltay", &nfo->deltay); 146 | adios_write(handle, "deltaz", &nfo->deltaz); 147 | for(i = 0; i < nfo->nvars; i++) 148 | adios_write(handle, nfo->varnames[i], nfo->datas[i]); 149 | 150 | adios_close(handle); 151 | } 152 | 153 | void adiosfull_finalize(struct adiosfullinfo *nfo) 154 | { 155 | free(nfo->varnames); 156 | free(nfo->datas); 157 | adios_finalize(nfo->rank); 158 | } 159 | 160 | -------------------------------------------------------------------------------- /cartiso/adiosfull.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) DoD HPCMP PETTT. All rights reserved. 3 | * See LICENSE file for details. 4 | */ 5 | 6 | #include 7 | #include 8 | 9 | struct adiosfullinfo { 10 | char *name; 11 | char *transform; 12 | MPI_Comm comm; 13 | int rank; 14 | int nprocs; 15 | int tsteps; 16 | int ni, nj, nk; 17 | int is, cni, js, cnj, ks, cnk; 18 | float deltax, deltay, deltaz; 19 | 20 | int nvars; 21 | int maxvars; 22 | char **varnames; 23 | float **datas; 24 | 25 | int bufallocsize; 26 | 27 | int64_t gid; 28 | }; 29 | 30 | void adiosfull_init(struct adiosfullinfo *nfo, char *method, char *transform, 31 | char *name, MPI_Comm comm, int rank, int nprocs, 32 | int tsteps, int ni, int nj, int nk, int is, int cni, int js, int cnj, 33 | int ks, int cnk, float deltax, float deltay, float deltaz, char *adiosopts); 34 | 35 | void adiosfull_addvar(struct adiosfullinfo *nfo, char *varname, float *data); 36 | 37 | void adiosfull_write(struct adiosfullinfo *nfo, int tstep); 38 | 39 | void adiosfull_finalize(struct adiosfullinfo *nfo); 40 | 41 | -------------------------------------------------------------------------------- /cartiso/adiosiso.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) DoD HPCMP PETTT. All rights reserved. 3 | * See LICENSE file for details. 4 | */ 5 | 6 | #include 7 | #include 8 | 9 | #include "adiosiso.h" 10 | 11 | static const int fnstrmax = 4095; 12 | 13 | 14 | void adiosiso_init(struct adiosisoinfo *nfo, char *method, char *transform, char *name, 15 | MPI_Comm comm, int rank, int nprocs, int tsteps, int ni, int nj, int nk, 16 | int cni, int cnj, int cnk, char *adiosopts) 17 | { 18 | static char emptystr[] = ""; 19 | 20 | /* Set up struct, haven't decided if using all of these yet */ 21 | nfo->name = name; 22 | nfo->transform = transform; 23 | nfo->comm = comm; 24 | nfo->rank = rank; 25 | nfo->nprocs = nprocs; 26 | nfo->tsteps = tsteps; 27 | nfo->ni = ni; 28 | nfo->nj = nj; 29 | nfo->nk = nk; 30 | nfo->cni = cni; 31 | nfo->cnj = cnj; 32 | nfo->cnk = cnk; 33 | nfo->numxvars = 0; 34 | nfo->maxxvars = 1000; 35 | nfo->xvarnames = (char **) malloc(nfo->maxxvars * sizeof(char *)); 36 | nfo->bufallocsize = 0; 37 | 38 | if(!adiosopts) adiosopts = emptystr; 39 | 40 | /* Set up ADIOS */ 41 | adios_init_noxml(comm); /* Not sure if this would conflict with adiosfull */ 42 | adios_declare_group(&nfo->gid, nfo->name, "", adios_flag_no); 43 | adios_select_method(nfo->gid, method, adiosopts, ""); 44 | 45 | /* Define output variables */ 46 | adios_define_var(nfo->gid, "rank", "", adios_integer, "", "", ""); 47 | adios_define_var(nfo->gid, "tstep", "", adios_integer, "", "", ""); 48 | adios_define_var(nfo->gid, "npoints", "", adios_unsigned_long, "", "", ""); 49 | adios_define_var(nfo->gid, "cnpoints", "", adios_unsigned_long, "", "", ""); 50 | adios_define_var(nfo->gid, "cstart", "", adios_unsigned_long, "", "", ""); 51 | adios_define_var(nfo->gid, "npoints3", "", adios_unsigned_long, "", "", ""); 52 | adios_define_var(nfo->gid, "cnpoints3", "", adios_unsigned_long, "", "", ""); 53 | adios_define_var(nfo->gid, "cstart3", "", adios_unsigned_long, "", "", ""); 54 | adios_define_var(nfo->gid, "coords", "", adios_real, 55 | "cnpoints3", "npoints3", "cstart3"); 56 | adios_define_var(nfo->gid, "norms", "", adios_real, 57 | "cnpoints3", "npoints3", "cstart3"); 58 | adios_define_var(nfo->gid, "connections", "", adios_unsigned_long, 59 | "cnpoints", "npoints", "cstart"); 60 | } 61 | 62 | void adiosiso_addxvar(struct adiosisoinfo *nfo, char *varname) 63 | { 64 | int64_t varid; 65 | if(nfo->numxvars > nfo->maxxvars) 66 | return; /* Just ignore too many variables, for now */ 67 | nfo->xvarnames[nfo->numxvars] = varname; 68 | nfo->numxvars++; 69 | varid = adios_define_var(nfo->gid, varname, "", adios_real, 70 | "cnpoints", "npoints", "cstart"); 71 | if(nfo->transform) 72 | adios_set_transform(varid, nfo->transform); 73 | } 74 | 75 | void adiosiso_write(struct adiosisoinfo *nfo, int tstep, uint64_t ntris, float *points, 76 | float *norms, float **xvals) 77 | { 78 | char fname[fnstrmax+1]; 79 | int timedigits = 4; 80 | uint64_t groupsize, totalsize; 81 | int64_t handle; 82 | int ret; 83 | int bufneeded; 84 | uint64_t i, npoints, cnpoints, cstart, npoints3, cnpoints3, cstart3, *npts_all; 85 | uint64_t *conns; 86 | 87 | /* Set local sizes */ 88 | cnpoints = ntris * 3; 89 | cnpoints3 = cnpoints * 3; 90 | groupsize = sizeof(int) /*rank*/ + sizeof(int) /*tstep*/ + 91 | sizeof(uint64_t)*3 /*npoints-cstart*/ + 92 | sizeof(uint64_t)*3 /*npoints3-cstart3*/ + 93 | sizeof(float)*cnpoints3 /*coords*/ + 94 | sizeof(float)*cnpoints3 /*norms*/ + 95 | sizeof(uint64_t)*cnpoints /*connections*/ + 96 | sizeof(float)*cnpoints*nfo->numxvars; /*xvars*/ 97 | 98 | /* Allocate buffer large enough for all data to write, if not done already */ 99 | bufneeded = (int)(groupsize/(1024*1024)); 100 | bufneeded += bufneeded/10 + 5; /* Add an extra 10% & 5MB to be sure */ 101 | if(nfo->bufallocsize < bufneeded) { 102 | # if ADIOS_VERSION_GE(1,13,0) 103 | /* Don't set any max buffer size - seems to work best without it */ 104 | # elif ADIOS_VERSION_GE(1,10,0) 105 | adios_set_max_buffer_size(bufneeded); 106 | # else 107 | adios_allocate_buffer(ADIOS_BUFFER_ALLOC_NOW, bufneeded); 108 | # endif 109 | nfo->bufallocsize = bufneeded; 110 | } 111 | 112 | /* Determine global sizes */ 113 | npts_all = (uint64_t *) malloc(nfo->nprocs * sizeof(uint64_t)); 114 | MPI_Allgather(&cnpoints, 1, MPI_UNSIGNED_LONG_LONG, 115 | npts_all, 1, MPI_UNSIGNED_LONG_LONG, nfo->comm); 116 | for(npoints = 0, i = 0; i < nfo->nprocs; ++i) 117 | npoints += npts_all[i]; 118 | for(cstart = 0, i = 0; i < nfo->rank; ++i) 119 | cstart += npts_all[i]; 120 | npoints3 = npoints * 3; 121 | cstart3 = cstart *3; 122 | free(npts_all); 123 | 124 | /* Create connections */ 125 | conns = (uint64_t *) malloc(cnpoints * sizeof(uint64_t)); 126 | for(i = 0; i < cnpoints; i++) 127 | conns[i] = i; 128 | 129 | /* Set filename */ 130 | snprintf(fname, fnstrmax, "%s.%0*d.bp", nfo->name, timedigits, tstep); 131 | 132 | /* Open & Write */ 133 | ret = adios_open(&handle, nfo->name, fname, "w", nfo->comm); 134 | if(ret) { 135 | fprintf(stderr, "Error opening ADIOS file: %s\n", fname); 136 | return; 137 | } 138 | # if ADIOS_VERSION_LE(1,9,0) 139 | adios_group_size(handle, groupsize, &totalsize); 140 | # endif 141 | 142 | adios_write(handle, "rank", &nfo->rank); 143 | adios_write(handle, "tstep", &tstep); 144 | adios_write(handle, "npoints", &npoints); 145 | adios_write(handle, "cnpoints", &cnpoints); 146 | adios_write(handle, "cstart", &cstart); 147 | adios_write(handle, "npoints3", &npoints3); 148 | adios_write(handle, "cnpoints3", &cnpoints3); 149 | adios_write(handle, "cstart3", &cstart3); 150 | adios_write(handle, "coords", points); 151 | adios_write(handle, "norms", norms); 152 | adios_write(handle, "connections", conns); 153 | for(i = 0; i < nfo->numxvars; i++) 154 | adios_write(handle, nfo->xvarnames[i], xvals[i]); 155 | 156 | adios_close(handle); 157 | free(conns); 158 | } 159 | 160 | void adiosiso_finalize(struct adiosisoinfo *nfo) 161 | { 162 | free(nfo->xvarnames); 163 | adios_finalize(nfo->rank); 164 | } 165 | 166 | -------------------------------------------------------------------------------- /cartiso/adiosiso.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) DoD HPCMP PETTT. All rights reserved. 3 | * See LICENSE file for details. 4 | */ 5 | 6 | #include 7 | #include 8 | 9 | struct adiosisoinfo { 10 | char *name; 11 | char *transform; 12 | MPI_Comm comm; 13 | int rank; 14 | int nprocs; 15 | int tsteps; 16 | int ni, nj, nk; 17 | int cni, cnj, cnk; 18 | 19 | int numxvars; 20 | int maxxvars; 21 | char **xvarnames; 22 | 23 | int bufallocsize; 24 | 25 | int64_t gid; 26 | }; 27 | 28 | void adiosiso_init(struct adiosisoinfo *nfo, char *method, char *transform, char *name, 29 | MPI_Comm comm, int rank, int nprocs, int tsteps, int ni, int nj, int nk, 30 | int cni, int cnj, int cnk, char *adiosopts); 31 | 32 | void adiosiso_addxvar(struct adiosisoinfo *nfo, char *varname); 33 | 34 | void adiosiso_write(struct adiosisoinfo *nfo, int tstep, uint64_t ntris, float *points, 35 | float *norms, float **xvals); 36 | 37 | void adiosiso_finalize(struct adiosisoinfo *nfo); 38 | 39 | -------------------------------------------------------------------------------- /cartiso/hdf5cartiso.h: -------------------------------------------------------------------------------- 1 | #ifdef HAS_HDF5 2 | # include "hdf5.h" 3 | #endif 4 | 5 | void writehdf5p(char *name, char *varname, MPI_Comm comm, int rank, int nprocs, 6 | int tstep, uint64_t ntris, float *points, float *norms, 7 | float *xvals, char *xname, hsize_t *h5_chunk, 8 | char *hdf5_compress, size_t cd_nelmts, unsigned int *cd_values); 9 | 10 | void writehdf5i(char *name, char *varname, MPI_Comm comm, int rank, int nprocs, 11 | int tstep, int ni, int nj, int nk, int is, int ie, int js, int je, 12 | int ks, int ke, float deltax, float deltay, float deltaz, int nci, int ncj, int nck, float *data, 13 | hsize_t *h5_chunk, char* hdf5_compress, size_t cd_nelmts, unsigned int *cd_values); 14 | 15 | -------------------------------------------------------------------------------- /cartiso/hdf5i.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) DoD HPCMP PETTT. All rights reserved. 3 | * See LICENSE file for details. 4 | */ 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #include "hdf5.h" 15 | 16 | static const int fnstrmax = 4095; 17 | 18 | void 19 | write_xdmf_xml_value(char *fname, char *fname_xdmf, char *varname, float deltax, float deltay, float deltaz, int ni, int nj, int nk); 20 | 21 | void writehdf5i(char *name, char *varname, MPI_Comm comm, int rank, int nprocs, 22 | int tstep, int ni, int nj, int nk, int is, int ie, int js, int je, 23 | int ks, int ke, float deltax, float deltay, float deltaz, int nci, int ncj, int nck, float *data, 24 | hsize_t *h5_chunk, char *hdf5_compress, size_t cd_nelmts, unsigned int *cd_values) 25 | { 26 | char fname[fnstrmax+1]; 27 | char fname_xdmf[fnstrmax+1]; 28 | int timedigits = 4; 29 | MPI_Info info = MPI_INFO_NULL; 30 | 31 | hid_t file_id; 32 | hid_t plist_id; 33 | hid_t memspace; 34 | hid_t filespace; 35 | hid_t did; 36 | hsize_t start[3], count[3]; 37 | hsize_t dims[3]; 38 | herr_t err; 39 | hid_t chunk_pid; 40 | 41 | snprintf(fname, fnstrmax, "cart.%s_t%0*d.h5", varname, timedigits, tstep); 42 | snprintf(fname_xdmf, fnstrmax, "cart.%s_t%0*d.xmf", varname, timedigits, tstep); 43 | 44 | /* Create pvti file */ 45 | if(rank == 0) { 46 | 47 | if( (file_id = H5Fcreate(fname, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) { 48 | fprintf(stderr, "writehdf5i error: could not create %s \n", fname); 49 | MPI_Abort(comm, 1); 50 | } 51 | 52 | /* Create dataset */ 53 | dims[0] = (hsize_t)nk; 54 | dims[1] = (hsize_t)nj; 55 | dims[2] = (hsize_t)ni; 56 | filespace = H5Screate_simple(3, dims, NULL); 57 | 58 | chunk_pid = H5Pcreate(H5P_DATASET_CREATE); 59 | 60 | H5Pset_alloc_time(chunk_pid, H5D_ALLOC_TIME_EARLY); 61 | 62 | if(h5_chunk) { 63 | H5Pset_layout(chunk_pid, H5D_CHUNKED); 64 | if( H5Pset_fill_time(chunk_pid, H5D_FILL_TIME_NEVER) < 0 ) { 65 | printf("writehdf5i error: Could not set fill time\n"); 66 | MPI_Abort(comm, 1); 67 | } 68 | H5Pset_chunk(chunk_pid, 3, h5_chunk); 69 | 70 | if(strcmp(hdf5_compress,"gzip") == 0) { 71 | 72 | /* Set ZLIB / DEFLATE Compression. */ 73 | 74 | if( H5Pset_deflate (chunk_pid, cd_values[0]) < 0 ) { 75 | printf("writehdf5i error: Could not set compression\n"); 76 | MPI_Abort(comm, 1); 77 | } 78 | } else if(strcmp(hdf5_compress,"szip") == 0) { 79 | 80 | /* SZIP Compression. */ 81 | 82 | if( H5Pset_szip (chunk_pid, cd_values[0], cd_values[1]) < 0 ) { 83 | printf("writehdf5i error: Could not set compression\n"); 84 | MPI_Abort(comm, 1); 85 | } 86 | } 87 | #ifdef H5Z_ZFP_USE_PLUGIN 88 | else if(strcmp(hdf5_compress,"zfp") == 0) { 89 | H5Z_filter_t filter_id; 90 | filter_id = 32013; 91 | 92 | if( H5Pset_filter(chunk_pid, filter_id, H5Z_FLAG_MANDATORY, cd_nelmts, cd_values) < 0 ) { 93 | printf("writehdf5i error: Could not set compression\n"); 94 | MPI_Abort(comm, 1); 95 | } 96 | } 97 | #endif 98 | } 99 | 100 | /* Create the dataset with default properties and close filespace. */ 101 | did = H5Dcreate(file_id, varname, H5T_NATIVE_FLOAT, filespace, H5P_DEFAULT, chunk_pid, H5P_DEFAULT); 102 | H5Dclose(did); 103 | 104 | if(h5_chunk) 105 | H5Pclose(chunk_pid); 106 | 107 | H5Sclose(filespace); 108 | 109 | H5Fclose(file_id); 110 | 111 | } /* end rank==0 */ 112 | 113 | MPI_Barrier(comm); 114 | 115 | /* Set up MPI info */ 116 | MPI_Info_create(&info); 117 | MPI_Info_set(info, "striping_factor", "1"); 118 | 119 | /* Set up file access property list with parallel I/O access */ 120 | if( (plist_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) { 121 | printf("writehdf5i error: Could not create property list \n"); 122 | MPI_Abort(comm, 1); 123 | } 124 | H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST); 125 | if(H5Pset_fapl_mpio(plist_id, MPI_COMM_WORLD, info) < 0) { 126 | printf("writehdf5i error: Could not create property list \n"); 127 | MPI_Abort(comm, 1); 128 | } 129 | 130 | H5Pset_all_coll_metadata_ops(plist_id, 1 ); 131 | H5Pset_coll_metadata_write(plist_id, 1); 132 | if( (file_id = H5Fopen(fname, H5F_ACC_RDWR, plist_id)) < 0) { 133 | fprintf(stderr, "writehdf5i error: could not open %s \n", fname); 134 | MPI_Abort(comm, 1); 135 | } 136 | 137 | if(H5Pclose(plist_id) < 0) { 138 | printf("writehdf5i error: Could not close property list \n"); 139 | MPI_Abort(comm, 1); 140 | } 141 | 142 | start[0] = ks; 143 | start[1] = js; 144 | start[2] = is; 145 | 146 | count[0] = (hsize_t)nck; 147 | count[1] = (hsize_t)ncj; 148 | count[2] = (hsize_t)nci; 149 | 150 | if( (memspace = H5Screate_simple(3, count, NULL)) < 0) { 151 | printf("writehdf5i error: Could not create memory space \n"); 152 | MPI_Abort(comm, 1); 153 | }; 154 | 155 | if( (did = H5Dopen(file_id, varname, H5P_DEFAULT)) < 0) { 156 | printf("writehdf5i error: Could not open data space \n"); 157 | MPI_Abort(comm, 1); 158 | }; 159 | 160 | /* Select hyperslab in the file.*/ 161 | filespace = H5Dget_space(did); 162 | if( H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, NULL, count, NULL ) < 0) { 163 | printf("writehdf5i error: Could not select hyperslab \n"); 164 | MPI_Abort(comm, 1); 165 | }; 166 | 167 | /* Create property list for collective dataset write. */ 168 | plist_id = H5Pcreate(H5P_DATASET_XFER); 169 | H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE); 170 | 171 | err = H5Dwrite(did, H5T_NATIVE_FLOAT, memspace, filespace, plist_id, data); 172 | if( err < 0) { 173 | fprintf(stderr, "writehdf5i error: could not write dataset %s \n", varname); 174 | MPI_Abort(comm, 1); 175 | } 176 | 177 | 178 | err = H5Dclose(did); 179 | 180 | err = H5Sclose(filespace); 181 | err = H5Sclose(memspace); 182 | 183 | if(H5Pclose(plist_id) < 0) 184 | printf("writehdf5i error: Could not close property list \n"); 185 | 186 | if(H5Fclose(file_id) != 0) 187 | printf("writehdf5i error: Could not close HDF5 file \n"); 188 | 189 | /* Create xdmf file for timestep */ 190 | if(rank == 0) { 191 | write_xdmf_xml_value(fname, fname_xdmf, varname, deltax, deltay, deltaz, ni, nj, nk); 192 | } 193 | } 194 | 195 | 196 | void 197 | write_xdmf_xml_value(char *fname, char *fname_xdmf, char *varname, float deltax, float deltay, float deltaz, int ni, int nj, int nk) 198 | { 199 | FILE *xmf = 0; 200 | 201 | /* 202 | * Open the file and write the XML description of the mesh. 203 | */ 204 | 205 | xmf = fopen(fname_xdmf, "w"); 206 | fprintf(xmf, "\n"); 207 | fprintf(xmf, "\n"); 208 | fprintf(xmf, "\n"); 209 | fprintf(xmf, " \n"); 210 | fprintf(xmf, " \n"); 211 | fprintf(xmf, " \n", ni, nj, nk ); 212 | fprintf(xmf, " \n"); 213 | fprintf(xmf, " \n"); 214 | fprintf(xmf, " 0 0 0 \n"); 215 | fprintf(xmf, " \n"); 216 | fprintf(xmf, " \n"); 217 | fprintf(xmf, " %f %f %f \n",deltax, deltay, deltaz); 218 | fprintf(xmf, " \n"); 219 | fprintf(xmf, " \n"); 220 | fprintf(xmf, " \n", varname); 221 | fprintf(xmf, " \n", ni, nj, nk); 222 | fprintf(xmf, " %s:/%s\n", fname, varname); 223 | fprintf(xmf, " \n"); 224 | fprintf(xmf, " \n"); 225 | fprintf(xmf, " \n"); 226 | fprintf(xmf, " \n"); 227 | fprintf(xmf, "\n"); 228 | fclose(xmf); 229 | } 230 | 231 | -------------------------------------------------------------------------------- /cartiso/iso.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) DoD HPCMP PETTT. All rights reserved. 3 | * See LICENSE file for details. 4 | */ 5 | 6 | #include 7 | 8 | struct isoinfo { 9 | float x0, y0, z0; /* Starting coordinates */ 10 | float xd, yd, zd; /* Coordinate deltas */ 11 | int xdim, ydim, zdim; /* Dimensions of local data */ 12 | uint64_t ntris; /* Number of triangles */ 13 | float *points; /* Points of triangles */ 14 | float *norms; /* Normals of triangles */ 15 | float *xvals; /* Extra values on triangle points */ 16 | }; 17 | 18 | /* Initialize info for isosurface 19 | * Must be done before the first call to isosurf. 20 | * Does not need to be repeated for subsequent calls to isosurf, 21 | * unless the coordinate, deltas, or dimensions change, but in 22 | * that case, isofree must be called first */ 23 | 24 | void isoinit(struct isoinfo *nfo, float x0, float y0, float z0, 25 | float xd, float yd, float zd, int xdim, int ydim, int zdim, 26 | int numxarrays); 27 | 28 | /* Free up resources from isosurface */ 29 | 30 | void isofree(struct isoinfo *nfo); 31 | 32 | /* Generate an isosurface */ 33 | 34 | void isosurf(struct isoinfo *nfo, /* Isosurface info */ 35 | float thresh, /* Isosurface threshold */ 36 | const float *data, /* Pointer to data, i,j,k order */ 37 | const float *xdata); /* Extra data */ 38 | 39 | -------------------------------------------------------------------------------- /cartiso/nccartiso.h: -------------------------------------------------------------------------------- 1 | #ifndef NCCARTISO_H 2 | #define NCCARTISO_H 3 | 4 | 5 | #ifdef HAS_NC 6 | #include "netcdf.h" 7 | #endif 8 | 9 | void writencp(char *name, char *varname, MPI_Comm comm, int rank, int nprocs, 10 | int tstep, uint64_t ntris, float *points, float *norms, 11 | float *xvals, char *xname); 12 | 13 | void writenci(char *name, char *varname, MPI_Comm comm, int rank, int nprocs, 14 | int tstep, int ni, int nj, int nk, int is, int ie, int js, int je, 15 | int ks, int ke, float deltax, float deltay, float deltaz, int nci, int ncj, int nck, 16 | float *data); 17 | 18 | #define MAX(a,b) ((a) > (b) ? (a) : (b)) 19 | 20 | 21 | 22 | #endif 23 | -------------------------------------------------------------------------------- /cartiso/nci.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) DoD HPCMP PETTT. All rights reserved. 3 | * See LICENSE file for details. 4 | */ 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #include "nccartiso.h" 15 | #include "netcdf.h" 16 | #include "netcdf_par.h" 17 | 18 | static const int fnstrmax = 4095; 19 | 20 | #define NCERR {if(err != NC_NOERR) {printf("(rank %d) Error at line %d: %s\n",rank,__LINE__,nc_strerror(err)); fflush(stdout); MPI_Abort(comm,1);}} 21 | 22 | 23 | /*! Write nci 24 | 25 | */ 26 | void writenci(char *name, char *varname, MPI_Comm comm, int rank, int nprocs, 27 | int tstep, int ni, int nj, int nk, int is, int ie, int js, int je, 28 | int ks, int ke, float deltax, float deltay, float deltaz, int nci, 29 | int ncj, int nck, float *data) 30 | { 31 | char fname[fnstrmax+1]; 32 | int timedigits = 4; 33 | MPI_Info info = MPI_INFO_NULL; 34 | int dimid = 0; 35 | int ncid = 0; 36 | int varid = 0; 37 | int dimlen = 0; 38 | size_t start[3] = {0,0,0}; 39 | size_t count[3] = {0,0,0}; 40 | int dims[3] = {0,0,0}; 41 | int dimids[3] = {0,0,0}; 42 | int err = 0; 43 | int i; 44 | 45 | snprintf(fname, fnstrmax, "cart.%s_t%0*d.nc", varname, timedigits, tstep); 46 | dimlen = MAX(ni,MAX(nj,nk)); 47 | /* Set up MPI info */ 48 | MPI_Info_create(&info); 49 | 50 | err = nc_create_par(fname,NC_NETCDF4|NC_MPIIO,comm,info,&ncid); NCERR; 51 | 52 | /* Create dataset */ 53 | dims[0] = (size_t)nk; 54 | dims[1] = (size_t)nj; 55 | dims[2] = (size_t)ni; 56 | 57 | /* Create the dimension, variable. */ 58 | 59 | err = nc_def_dim(ncid,"phony_dim_0",dimlen,&dimid); NCERR; 60 | 61 | for(i = 0; i < 3; i++) { 62 | dimids[i] = dimid; 63 | } 64 | 65 | err = nc_def_var(ncid,varname,NC_FLOAT,3,&dimids[0],&varid); NCERR; 66 | 67 | err = nc_enddef(ncid); NCERR; 68 | MPI_Barrier(comm); 69 | 70 | /* Set up MPI info */ 71 | MPI_Info_create(&info); 72 | //MPI_Info_set(info, "striping_factor", "1"); 73 | 74 | /* Set up parallel access for the variable. */ 75 | err = nc_var_par_access(ncid,varid,NC_COLLECTIVE); NCERR; 76 | 77 | start[0] = ks; 78 | start[1] = js; 79 | start[2] = is; 80 | 81 | count[0] = (size_t)nck; 82 | count[1] = (size_t)ncj; 83 | count[2] = (size_t)nci; 84 | 85 | err = nc_put_vara_float(ncid,varid,start,count,data); NCERR; 86 | 87 | MPI_Barrier(comm); 88 | 89 | err = nc_close(ncid); NCERR; 90 | } 91 | -------------------------------------------------------------------------------- /cartiso/ncp.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) DoD HPCMP PETTT. All rights reserved. 3 | * See LICENSE file for details. 4 | */ 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | #include "nccartiso.h" 16 | #include "netcdf.h" 17 | #include "netcdf_par.h" 18 | static const int fnstrmax = 4095; 19 | 20 | #define NCERR {if(err != NC_NOERR) {printf("(rank %d) Error at line %d: %s\n",rank,__LINE__,nc_strerror(err)); fflush(stdout); MPI_Abort(comm,1);}} 21 | 22 | /*! Write ncp. 23 | 24 | */ 25 | void writencp(char *name, char *varname, MPI_Comm comm, int rank, int nprocs, 26 | int tstep, uint64_t ntris, float *points, float *norms, 27 | float *xvals, char *xname) 28 | { 29 | char fname[fnstrmax+1]; 30 | char fname_xdmf[fnstrmax+1]; 31 | int timedigits = 4; 32 | MPI_Info info = MPI_INFO_NULL; 33 | uint64_t *rntris; /* All triangle counts from each task */ 34 | uint64_t tot_tris; 35 | int grp_id_grid_pts; 36 | int var_id_xyz; 37 | int var_id_normals; 38 | int var_id_conn; 39 | int var_id_xname; 40 | size_t start[1]; 41 | size_t count[1]; 42 | 43 | int var_dim_phony_0; 44 | int var_dim_phony_1; 45 | 46 | int ncid = 0; 47 | 48 | int j; 49 | uint64_t *temparr; 50 | int err = 0; 51 | 52 | snprintf(fname, fnstrmax, "cartiso_t%0*d.nc", timedigits, tstep); 53 | 54 | /* Gather tri counts, in case some are zero, to leave them out */ 55 | rntris = (uint64_t *) malloc(nprocs*sizeof(uint64_t)); 56 | 57 | MPI_Allgather(&ntris, 1, MPI_UNSIGNED_LONG_LONG, rntris, 1, MPI_LONG_LONG, comm); NCERR; 58 | 59 | uint64_t i; 60 | 61 | tot_tris=0; 62 | for (i=0; i print average rate(s) after each file 20 | -n => don't print the column headings (good for auto-copy-paste to excel) 21 | """ 22 | 23 | import sys 24 | import re 25 | 26 | if len(sys.argv) <= 1: 27 | print __doc__ 28 | 29 | phases = ["sin2gauss", "gaussmove", "gaussresize", "gaussmove_r"] 30 | name2otype = { "pvti": "Full", "pvtp": "Iso", "adiosfull": "Full", "adiosiso": "Iso", 31 | "hdf5i": "Full", "hdf5p": "Iso", "nci": "Full", "ncp": "Iso" } 32 | fname2method = { ".pvti.": "MPI-Indiv", ".pvtp.": "MPI-Indiv", ".posix.": "ADIOS-POSIX", 33 | ".mpi.": "ADIOS-MPI", ".mpilus.": "ADIOS-MPI_Lustre", ".mpiag.": "ADIOS-MPI_Aggr", 34 | ".phdf5.": "ADIOS-PHDF5", ".hdf5i.": "HDF5", ".hdf5p.": "HDF5", ".nci.": "netCDF", 35 | ".ncp.": "netCDF" } 36 | lname2method = { "pvti": "MPI-Indiv", "pvtp": "MPI-Indiv", "POSIX": "ADIOS-POSIX", 37 | "MPI ": "ADIOS-MPI", "MPI$": "ADIOS-MPI", "MPI_LUSTRE": "ADIOS-MPI_Lustre", 38 | "MPI_AGGREGATE": "ADIOS-MPI_Aggr", "PHDF5": "ADIOS-PHDF5", 39 | "hdf5i": "HDF5", "hdf5p": "HDF5", "nci": "netCDF", "ncp": "netCDF" } 40 | 41 | totalrate = -1.0 42 | totalvrate = -1.0 43 | header = True 44 | a = 1 45 | for arg in sys.argv[1:3]: 46 | if arg == "-a": # produce average rate 47 | totalrate = 0.0 48 | totalvrate = 0.0 49 | a += 1 50 | if arg == "-n": # no header 51 | header = False 52 | a += 1 53 | 54 | # Iter over all files 55 | for fname in sys.argv[a:]: 56 | v = [] 57 | if totalrate > 0.0: totalrate = 0.0 58 | if totalvrate > 0.0: totalvrate = 0.0 59 | 60 | # Determine type and method from file name, if it's there 61 | otype1key = next((x for x in name2otype.keys() if x in fname), False) 62 | if not otype1key: print "WARNING: type not specified in filename" 63 | method1key = next((x for x in fname2method.keys() if x in fname), False) 64 | if not method1key: print "WARNING: method not specified in filename" 65 | 66 | f = open(fname) 67 | phase = -1 68 | for l in f: 69 | ls = l.split() 70 | if(len(ls) < 1): continue 71 | if(len(ls) < 2): ls.append("") # Ensure all outer if's can handle ls 72 | 73 | if ls[0][0:4] == "tsk=": 74 | if ls[0][4] == "\"": continue # Skip LSF copy of batch script 75 | 76 | cores = int(ls[0][4:]) * int(ls[1]) * int(ls[2]) 77 | if ls[3][:3] == "nc=": # Added nc= field to SGI output, and size not cubed 78 | size = ls[7][2:] + 'x' + ls[8] + 'x' + ls[9] 79 | sizeprod = float(ls[7][2:]) * float(ls[8]) * float(ls[9]) 80 | fullgb = sizeprod*4*2/1024**3 81 | else: 82 | size = ls[6][2:] 83 | sizec = int(ls[6][2:]) 84 | fullgb = float(sizec)**3*4*2/1024**3 85 | 86 | # Determine type & method from line; make sure matches one from file name 87 | otype2key = next((x for x in name2otype.keys() if x in l), False) 88 | if otype2key: 89 | if otype1key: 90 | if otype1key != otype2key: 91 | print "ERROR: type in filename doesn't match type in file" 92 | sys.exit(1) 93 | else: 94 | otype = name2otype[otype1key] 95 | else: 96 | otype = name2otype[otype2key] 97 | else: 98 | print "WARNING: type not specified in log file" 99 | if otype1key: 100 | otype = name2otype[otype1key] 101 | else: 102 | print "ERROR: no type specified, cannot proceed" 103 | sys.exit(1) 104 | method2key = next((x for x in lname2method.keys() if re.search(x, l)), False) 105 | if method2key: 106 | if method1key: 107 | if fname2method[method1key] != lname2method[method2key]: 108 | print "Error: method in filename doesn't match method in file" 109 | sys.exit(1) 110 | else: 111 | method = fname2method[method1key] 112 | else: 113 | method = lname2method[method2key] 114 | else: 115 | print "WARNING: method not specified in log file" 116 | if method1key: 117 | method = fname2method[method1key] 118 | else: 119 | print "ERROR: not method specified, cannot proceed" 120 | sys.exit(1) 121 | 122 | elif ls[0] == "isothresh": 123 | phase += 1 124 | elif ls[0] == "Total" and ls[1] == "tris": 125 | tris = float(ls[3][:-1]) 126 | # Size of output depends upon method 127 | if method == "MPI-Indiv": 128 | # ... (coords + var + norms + conns + offsets) ... 129 | isogb = tris*(3*3*4 + 3*4 + 3*3*4 + 3*8 + 8) / 1024**3 130 | else: 131 | # ... (coords + var + norms + conns) ... 132 | isogb = tris*(3*3*4 + 3*4 + 3*3*3 + 3*8) / 1024**3 133 | limb = float(ls[13]) 134 | elif ls[0] == "FullOutput": 135 | fulltime = float(ls[11][:-1]) 136 | elif (ls[0],ls[1]) == ("Isosurface","timer"): 137 | isoctime = float(ls[11][:-1]) 138 | elif ls[0] == "IsoOutput": 139 | isotime = float(ls[11][:-1]) 140 | if otype == "Full": 141 | rate = fullgb/fulltime 142 | if totalrate >= 0.0: totalrate += rate 143 | v.append([method, cores, size, fullgb, fulltime, rate]) 144 | elif otype == "Iso": 145 | vtime = isoctime+isotime #equivlent time if iso+isoout replaces full output 146 | vrate = fullgb/vtime 147 | rate = isogb/isotime 148 | if totalrate >= 0.0: totalrate += rate 149 | if totalvrate >= 0.0: totalvrate += vrate 150 | v.append([method, cores, size, fullgb, vtime, vrate, phases[phase], 151 | limb, limb/cores, tris, isogb, isotime, rate]) 152 | 153 | if otype == "Full": 154 | if header: print "%-16s %5s %15s %5s %5s %5s" % ("method", "cores", "size", "f_GB", "ftime", "fGB/s") 155 | for i in v: 156 | print "%-16s %5d %15s %5.0f %5.2f %5.2f" % tuple(i) 157 | if totalrate >= 0.0: print "Avg rate =", totalrate/len(v) 158 | elif otype == "Iso": 159 | if header: print "%-16s %5s %15s %5s %5s %6s %11s %8s %5s %11s %7s %4s %6s" % ("method", "cores", 160 | "size", "f_GB", "vtime", "vGB/s", "phase", "loadimb", "limb%", "tris", "i_GB", "itim", "i_GB/s") 161 | for i in v: 162 | print "%-16s %5d %15s %5.0f %5.2f %6.2f %11s %8.2f %5.3f %11.0f %7.3f %4.2f %6.3f" % tuple(i) 163 | if totalrate >= 0.0: print "Avg vrate =", totalvrate/len(v), ", rate =", totalrate/len(v) 164 | 165 | -------------------------------------------------------------------------------- /cartiso/pvti.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) DoD HPCMP PETTT. All rights reserved. 3 | * See LICENSE file for details. 4 | */ 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #include "pdirs.h" 15 | #include "pvti.h" 16 | 17 | static const int fnstrmax = 4095; 18 | 19 | void writepvti(char *name, char *varname, MPI_Comm comm, int rank, int nprocs, 20 | int tstep, int ni, int nj, int nk, int is, int ie, int js, int je, 21 | int ks, int ke, float deltax, float deltay, float deltaz, float *data) 22 | { 23 | char dirname[fnstrmax+1]; 24 | char fname[fnstrmax+1]; 25 | char line[fnstrmax+1]; 26 | int rankdigits = nprocs > 1 ? (int)(log10(nprocs-1)+1.5) : 1; 27 | int timedigits = 4; 28 | char typestr[] = "Float32"; 29 | char endianstr[] = "LittleEndian"; 30 | FILE *f; 31 | MPI_File mf; 32 | MPI_Status mstat; 33 | MPI_Info info = MPI_INFO_NULL; 34 | int r; 35 | int subset[6] = { is, ie, js, je, ks, ke }; /* Local point indices */ 36 | int *rsubsets = NULL; /* All point indices from each task */ 37 | int ret; 38 | uint64_t ijkelems; 39 | int32_t ijkbytes32; 40 | 41 | /* Make directory for timestep */ 42 | snprintf(dirname, fnstrmax, "%s.%s.%0*d.d", name, varname, timedigits, tstep); 43 | mkdir1task(dirname, comm); 44 | 45 | /* Gather subset sizes from all processes of communicator to rank 0 */ 46 | if(rank == 0) { 47 | rsubsets = (int *) malloc(nprocs*6*sizeof(int)); 48 | } 49 | MPI_Gather(subset, 6, MPI_INT, rsubsets, 6, MPI_INT, 0, comm); 50 | 51 | /* Create pvti file */ 52 | if(rank == 0) { 53 | snprintf(fname, fnstrmax, "%s.%s.%0*d.pvti", name, varname, timedigits, tstep); 54 | if( ! (f = fopen(fname, "w")) ) { 55 | fprintf(stderr, "writepvti error: Could not create .pvti file.\n"); 56 | MPI_Abort(comm, 1); 57 | } 58 | fprintf(f, "\n" 59 | "\n", 60 | endianstr); 61 | fprintf(f, " \n", ni-1, nj-1, nk-1, 63 | deltax, deltay, deltaz); 64 | fprintf(f, " \n" 65 | " \n" 66 | " \n", varname, typestr, varname); 67 | /* Write info for each block */ 68 | for(r = 0; r < nprocs; ++r) { 69 | int *rsub = rsubsets + r*6; /* Remote subset indices */ 70 | snprintf(fname, fnstrmax, "%s.%s.%0*d.d/%0*d.vti", name, varname, timedigits, 71 | tstep, rankdigits, r); 72 | fprintf(f, " \n", 73 | rsub[0], rsub[1], rsub[2], rsub[3], rsub[4], rsub[5], fname); 74 | } 75 | /* Finish the .pvti file */ 76 | fprintf(f, " \n\n"); 77 | fclose(f); 78 | free(rsubsets); 79 | } /* end rank==0 */ 80 | 81 | chkdir1task(dirname, comm); 82 | 83 | /* Set up MPI info */ 84 | MPI_Info_create(&info); 85 | MPI_Info_set(info, "striping_factor", "1"); 86 | 87 | /* Write .vti files */ 88 | snprintf(fname, fnstrmax, "%s.%s.%0*d.d/%0*d.vti", name, varname, timedigits, 89 | tstep, rankdigits, rank); 90 | ret = MPI_File_open(MPI_COMM_SELF, fname, MPI_MODE_WRONLY | MPI_MODE_CREATE, 91 | info, &mf); 92 | if(ret) { 93 | fprintf(stderr, "writepvti error: could not open %s\n", fname); 94 | MPI_Abort(comm, 1); 95 | } 96 | snprintf(line, fnstrmax, "\n" 97 | "\n", 98 | endianstr); 99 | MPI_File_write(mf, line, strlen(line), MPI_CHAR, &mstat); 100 | snprintf(line, fnstrmax, " \n " 102 | "\n", is, ie, js, je, ks, ke, 103 | deltax, deltay, deltaz, is, ie, js, je, ks, ke); 104 | MPI_File_write(mf, line, strlen(line), MPI_CHAR, &mstat); 105 | snprintf(line, fnstrmax, " \n", varname); 106 | MPI_File_write(mf, line, strlen(line), MPI_CHAR, &mstat); 107 | snprintf(line, fnstrmax, " \n", typestr, varname); 109 | MPI_File_write(mf, line, strlen(line), MPI_CHAR, &mstat); 110 | snprintf(line, fnstrmax, " \n" 111 | " \n" 112 | " \n" 113 | " \n" 114 | " \n" 115 | " _"); 116 | MPI_File_write(mf, line, strlen(line), MPI_CHAR, &mstat); 117 | ijkelems = (ie-is+1) * (je-js+1) * (ke-ks+1); 118 | ijkbytes32 = (int32_t)(ijkelems * sizeof(float)); 119 | MPI_File_write(mf, &ijkbytes32, 1, MPI_INT, &mstat); 120 | MPI_File_write(mf, data, ijkelems, MPI_FLOAT, &mstat); 121 | snprintf(line, fnstrmax, "\n \n\n"); 122 | MPI_File_write(mf, line, strlen(line), MPI_CHAR, &mstat); 123 | MPI_File_close(&mf); 124 | } 125 | 126 | 127 | -------------------------------------------------------------------------------- /cartiso/pvti.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) DoD HPCMP PETTT. All rights reserved. 3 | * See LICENSE file for details. 4 | */ 5 | 6 | #define __STDC_FORMAT_MACROS 7 | #include 8 | 9 | void writepvti(char *name, char *varname, MPI_Comm comm, int rank, int nprocs, 10 | int tstep, int ni, int nj, int nk, int is, int ie, int js, int je, 11 | int ks, int ke, float deltax, float deltay, float deltaz, float *data); 12 | -------------------------------------------------------------------------------- /cartiso/pvtp.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) DoD HPCMP PETTT. All rights reserved. 3 | * See LICENSE file for details. 4 | */ 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #include "pdirs.h" 15 | #include "pvtp.h" 16 | 17 | static const int fnstrmax = 4095; 18 | 19 | void writepvtp(char *name, char *varname, MPI_Comm comm, int rank, int nprocs, 20 | int tstep, uint64_t ntris, float *points, float *norms, 21 | float *xvals, char *xname) 22 | { 23 | char dirname[fnstrmax+1]; 24 | char fname[fnstrmax+1]; 25 | char line[fnstrmax+1]; 26 | int rankdigits = nprocs > 1 ? (int)(log10(nprocs-1)+1.5) : 1; 27 | int timedigits = 4; 28 | char typestr[] = "Float32"; 29 | char endianstr[] = "LittleEndian"; 30 | FILE *f; 31 | MPI_File mf; 32 | MPI_Status mstat; 33 | MPI_Info info = MPI_INFO_NULL; 34 | int r; 35 | uint64_t *rntris=NULL; /* All triangle counts from each task */ 36 | int ret; 37 | 38 | /* Make directory for timestep */ 39 | snprintf(dirname, fnstrmax, "%s.%s.%0*d.d", name, varname, timedigits, tstep); 40 | mkdir1task(dirname, comm); 41 | 42 | /* Gather tri counts, in case some are zero, to leave them out */ 43 | if(rank == 0) 44 | rntris = (uint64_t *) malloc(nprocs*sizeof(uint64_t)); 45 | MPI_Gather(&ntris, 1, MPI_UNSIGNED_LONG_LONG, rntris, 1, MPI_LONG_LONG, 0, comm); 46 | 47 | /* Create pvtp file */ 48 | if(rank == 0) { 49 | snprintf(fname, fnstrmax, "%s.%s.%0*d.pvtp", name, varname, timedigits, tstep); 50 | if( ! (f = fopen(fname, "w")) ) { 51 | fprintf(stderr, "writepvtp error: Could not create .pvtp file.\n"); 52 | MPI_Abort(comm, 1); 53 | } 54 | fprintf(f, "\n" 55 | "\n", 57 | endianstr); 58 | fprintf(f, " \n"); 59 | fprintf(f, " \n" 60 | " \n", 61 | typestr); 62 | if(xvals) { 63 | fprintf(f, " \n", 64 | typestr, xname); 65 | } 66 | fprintf(f, " \n"); 67 | fprintf(f, " \n" 68 | " \n" 69 | " \n", typestr); 70 | /* Write info for each block */ 71 | for(r = 0; r < nprocs; ++r) { 72 | if(rntris[r] > 0) { /* But only for blocks that have triangles */ 73 | snprintf(fname, fnstrmax, "%s.%s.%0*d.d/%0*d.vtp", name, varname, timedigits, 74 | tstep, rankdigits, r); 75 | fprintf(f, " \n", fname); 76 | } 77 | } 78 | /* Finish the .pvtp file */ 79 | fprintf(f, " \n\n"); 80 | fclose(f); 81 | free(rntris); 82 | } /* end rank==0 */ 83 | 84 | chkdir1task(dirname, comm); 85 | 86 | /* Set up MPI info */ 87 | MPI_Info_create(&info); 88 | MPI_Info_set(info, "striping_factor", "1"); 89 | 90 | /* Write .vtp files, if we have polys */ 91 | if(ntris > 0) { 92 | uint64_t i, offsets = 0; /* Offsets into the binary portion for each field */ 93 | uint64_t *temparr; 94 | snprintf(fname, fnstrmax, "%s.%s.%0*d.d/%0*d.vtp", name, varname, timedigits, 95 | tstep, rankdigits, rank); 96 | ret = MPI_File_open(MPI_COMM_SELF, fname, MPI_MODE_WRONLY | MPI_MODE_CREATE, 97 | info, &mf); 98 | if(ret) { 99 | fprintf(stderr, "writepvtp error: could not open %s\n", fname); 100 | MPI_Abort(comm, 1); 101 | } 102 | snprintf(line, fnstrmax, "\n" 103 | "\n" 105 | " \n", endianstr); 106 | MPI_File_write(mf, line, strlen(line), MPI_CHAR, &mstat); 107 | snprintf(line, fnstrmax, " \n", 109 | ntris*3, ntris); 110 | MPI_File_write(mf, line, strlen(line), MPI_CHAR, &mstat); 111 | snprintf(line, fnstrmax, " \n" 112 | " \n", typestr); 114 | MPI_File_write(mf, line, strlen(line), MPI_CHAR, &mstat); 115 | offsets += ntris*9*sizeof(float)+sizeof(uint64_t); /* Add size of normals for points */ 116 | if(xvals) { 117 | snprintf(line, fnstrmax, " \n", typestr, xname, offsets); 120 | MPI_File_write(mf, line, strlen(line), MPI_CHAR, &mstat); 121 | offsets += ntris*3*sizeof(float)+sizeof(uint64_t); /* Add size of xdata array */ 122 | } 123 | snprintf(line, fnstrmax, " \n" 124 | " \n" 125 | " \n" 127 | " \n", typestr, offsets); 128 | MPI_File_write(mf, line, strlen(line), MPI_CHAR, &mstat); 129 | offsets += ntris*9*sizeof(float)+sizeof(uint64_t); /* Add size of points */ 130 | snprintf(line, fnstrmax, " \n" 131 | " \n", offsets); 133 | MPI_File_write(mf, line, strlen(line), MPI_CHAR, &mstat); 134 | offsets += ntris*3*sizeof(uint64_t)+sizeof(uint64_t); /* Add size of connections */ 135 | snprintf(line, fnstrmax, " \n" 137 | " \n \n \n" 138 | " \n" 139 | " _", offsets); 140 | MPI_File_write(mf, line, strlen(line), MPI_CHAR, &mstat); 141 | 142 | /* Normals header & array */ 143 | offsets = ntris*9*sizeof(float); 144 | MPI_File_write(mf, &offsets, 1, MPI_UNSIGNED_LONG_LONG, &mstat); 145 | MPI_File_write(mf, norms, ntris*9, MPI_FLOAT, &mstat); 146 | /* xvals header & array */ 147 | if(xvals) { 148 | offsets = ntris*3*sizeof(float); 149 | MPI_File_write(mf, &offsets, 1, MPI_UNSIGNED_LONG_LONG, &mstat); 150 | MPI_File_write(mf, xvals, ntris*3, MPI_FLOAT, &mstat); 151 | } 152 | /* Points header & array */ 153 | offsets = ntris*9*sizeof(float); 154 | MPI_File_write(mf, &offsets, 1, MPI_UNSIGNED_LONG_LONG, &mstat); 155 | MPI_File_write(mf, points, ntris*9, MPI_FLOAT, &mstat); 156 | /* Create connections & write */ 157 | temparr = (uint64_t *) malloc(ntris*3*sizeof(uint64_t)); 158 | for(i = 0; i < ntris*3; i++) 159 | temparr[i] = i; 160 | offsets = ntris*3*sizeof(uint64_t); 161 | MPI_File_write(mf, &offsets, 1, MPI_UNSIGNED_LONG_LONG, &mstat); 162 | MPI_File_write(mf, temparr, ntris*3, MPI_UNSIGNED_LONG_LONG, &mstat); 163 | /* Create offsets & write */ 164 | for(i = 1; i <= ntris; i++) 165 | temparr[i] = i*3; 166 | offsets = ntris*sizeof(uint64_t); 167 | MPI_File_write(mf, &offsets, 1, MPI_UNSIGNED_LONG_LONG, &mstat); 168 | MPI_File_write(mf, temparr, ntris, MPI_UNSIGNED_LONG_LONG, &mstat); 169 | /* Finish and close */ 170 | free(temparr); 171 | snprintf(line, fnstrmax, "\n \n\n"); 172 | MPI_File_write(mf, line, strlen(line), MPI_CHAR, &mstat); 173 | MPI_File_close(&mf); 174 | } 175 | 176 | } 177 | 178 | -------------------------------------------------------------------------------- /cartiso/pvtp.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) DoD HPCMP PETTT. All rights reserved. 3 | * See LICENSE file for details. 4 | */ 5 | 6 | #define __STDC_FORMAT_MACROS 7 | #include 8 | 9 | void writepvtp(char *name, char *varname, MPI_Comm comm, int rank, int nprocs, 10 | int tstep, uint64_t ntris, float *points, float *norms, 11 | float *xvals, char *xname); 12 | -------------------------------------------------------------------------------- /cartiso/sfc.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) DoD HPCMP PETTT. All rights reserved. 3 | * See LICENSE file for details. 4 | */ 5 | 6 | #include "sfc.h" 7 | 8 | void sfc3_serpentine_init(struct sfc3_ctx *ctx, int ni, int nj, int nk, int reverse) 9 | { 10 | ctx->ni = ni; 11 | ctx->nj = nj; 12 | ctx->nk = nk; 13 | ctx->reverse = reverse; 14 | ctx->ndx = 0; 15 | ctx->valid = 1; 16 | ctx->i = 0; 17 | ctx->j = 0; 18 | ctx->k = 0; 19 | ctx->idir = 1; 20 | ctx->jdir = 1; 21 | ctx->kdir = 1; 22 | if(reverse) { 23 | ctx->k = nk - 1; 24 | ctx->kdir = -1; 25 | if(nk % 2) { /* nk odd => j ends opposite */ 26 | ctx->j = nj - 1; 27 | ctx->jdir = -1; 28 | if(nj % 2) { /* nj & nk odd => i ends opposite */ 29 | ctx->i = ni - 1; 30 | ctx->idir = -1; 31 | } 32 | } 33 | } 34 | } 35 | 36 | void sfc3_serpentine_next(struct sfc3_ctx *ctx) 37 | { 38 | ctx->i += ctx->idir; 39 | if(ctx->i == ctx->ni) { 40 | ctx->i -= 1; 41 | ctx->j += ctx->jdir; 42 | ctx->idir = -1; 43 | } else if(ctx->i == -1) { 44 | ctx->i += 1; 45 | ctx->j += ctx->jdir; 46 | ctx->idir = 1; 47 | } 48 | if(ctx->j == ctx->nj) { 49 | ctx->j -= 1; 50 | ctx->k += ctx->kdir; 51 | ctx->jdir = -1; 52 | } else if(ctx->j == -1) { 53 | ctx->j += 1; 54 | ctx->k += ctx->kdir; 55 | ctx->jdir = 1; 56 | } 57 | ctx->ndx += 1; 58 | if(ctx->k >= ctx->nk || ctx->k < 0) /* In either case, we're done */ 59 | ctx->valid = 0; 60 | } 61 | 62 | #ifdef SFC_SERPENTINE_TEST 63 | 64 | #include 65 | #include 66 | 67 | int main(int argc, char **argv) 68 | { 69 | struct sfc3_ctx sfc; 70 | int ni, nj, nk, reverse; 71 | 72 | if(argc != 5) { 73 | printf("Invalid arguments.\nUsage: %s ni nj nk reverse\n", argv[0]); 74 | printf(" reverse = 0 or 1 1 runs the curve in reverse from the end"); 75 | return 1; 76 | } 77 | 78 | ni = atoi(argv[1]); 79 | nj = atoi(argv[2]); 80 | nk = atoi(argv[3]); 81 | reverse = atoi(argv[4]); 82 | 83 | if(ni < 1 || nj < 1 || nk < 1) { 84 | printf("Invalid number or ni, nj, or nk.\n"); 85 | return 1; 86 | } 87 | 88 | sfc3_serpentine_init(&sfc, ni, nj, nk, reverse); 89 | 90 | do { 91 | printf("%d %d %d\n", sfc.i, sfc.j, sfc.k); 92 | sfc3_serpentine_next(&sfc); 93 | } while(sfc.valid); 94 | 95 | return 0; 96 | } 97 | 98 | #endif 99 | -------------------------------------------------------------------------------- /cartiso/sfc.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) DoD HPCMP PETTT. All rights reserved. 3 | * See LICENSE file for details. 4 | */ 5 | 6 | struct sfc3_ctx { 7 | int ni, nj, nk; /* Total size of space */ 8 | int reverse; /* Whether the curve advances in reverse from the end */ 9 | int ndx; /* Current index */ 10 | int valid; /* Are the current indices valid */ 11 | int i, j, k; /* Current coordinates in space */ 12 | int idir; /* Current direction along i, +1 or -1 */ 13 | int jdir; /* Current direction along j, +1 or -1 */ 14 | int kdir; /* Current direction along k, +1 or -1 */ 15 | }; 16 | 17 | void sfc3_serpentine_init(struct sfc3_ctx *ctx, int ni, int nj, int nk, int reverse); 18 | void sfc3_serpentine_next(struct sfc3_ctx *ctx); 19 | 20 | -------------------------------------------------------------------------------- /cartiso/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Script to demonstrate a range of cartiso features & cycle through all 4 | # four time step modes 5 | 6 | # First script parameter determines output type 7 | # Both full and isosurface output types (don't do that for benchmarks) 8 | if [ "$1" == vtk ]; then # VTK XML Output options 9 | output="--pvti --pvtp" 10 | elif [ "$1" == adios ]; then # ADIOS options 11 | output="--adiosfull MPI --adiosiso MPI" 12 | elif [ "$1" == nc ]; then # NetCDF 13 | output="--nci --ncp" 14 | elif [ "$1" == hdf5 ];then # HDF5 with chunking parameters too 15 | output="--hdf5i --hdf5p --hdf5i_chunk 256 256 256 --hdf5p_chunk 100" 16 | else 17 | echo "usage: test.sh " 18 | fi 19 | 20 | scale=1 21 | if [ "$#" == 2 ]; then 22 | if [ "$2" == test ]; then 23 | scale=10 24 | else 25 | echo "error: invalid 2nd parameter, $2" 26 | echo "usage: test.sh " 27 | exit 1 28 | fi 29 | fi 30 | 31 | tsk="2 2 2" # 8 ranks with 2x2x2 decomposition 32 | fq="2 2 2" # 2x2x2 sinusoid frequencies 33 | s=256 # Grid will be 256^3 34 | sg=0.29 # Initial Gaussian sigma 35 | sge=0.55 # Ending Gaussian sigma 36 | 37 | # Phase 1: 20 time steps of sin2gauss mode 38 | mpirun -np 8 ./cartiso --tasks $tsk --size $s $s $s --sigma $sg $sg $sg \ 39 | --centertask --freq $fq --tsteps $((20/$scale)) --sin2gauss $output 40 | 41 | # Phase 2: 80 time steps of gaussmove mode 42 | mpirun -np 8 ./cartiso --tasks $tsk --size $s $s $s --sigma $sg $sg $sg \ 43 | --centertask --freq $fq --tsteps $((80/$scale)) --tstart $((20/$scale)) --gaussmove $output 44 | 45 | # Phase 3: 20 time steps of gaussresize mode 46 | mpirun -np 8 ./cartiso --tasks $tsk --size $s $s $s --sigma $sg $sg $sg \ 47 | --centertask --freq $fq --tsteps $((20/$scale)) --tstart $((100/$scale)) --gaussresize \ 48 | --backward --sigmaend $sge $sge $sge $output 49 | 50 | # Phase 4: 80 time steps of backward gaussmove mode 51 | mpirun -np 8 ./cartiso --tasks $tsk --size $s $s $s --sigma $sge $sge $sge \ 52 | --centertask --freq $fq --tsteps $((80/$scale)) --tstart $((120/$scale)) --gaussmove --backward $output 53 | 54 | -------------------------------------------------------------------------------- /osn/Makefile: -------------------------------------------------------------------------------- 1 | 2 | include ../Makefile.inc 3 | 4 | CFLAGS += -W -Wall -Wextra 5 | 6 | # Flags specific to building the test app (requires serial compiler) 7 | SERIAL_CC = gcc 8 | PNGINC = 9 | PNGLIBPATH = 10 | 11 | .PHONY: clean 12 | 13 | open-simplex-noise.o: open-simplex-noise.h open-simplex-noise.c Makefile 14 | $(CC) $(CFLAGS) -c open-simplex-noise.c 15 | 16 | open-simplex-noise-test: open-simplex-noise-test.c open-simplex-noise.o 17 | $(SERIAL_CC) $(CFLAGS) $(PNGINC) -o open-simplex-noise-test open-simplex-noise.o open-simplex-noise-test.c $(PNGLIBPATH) -lpng 18 | 19 | clean: 20 | rm -f open-simplex-noise.o open-simplex-noise-test test2d.png test3d.png test4d.png 21 | 22 | -------------------------------------------------------------------------------- /osn/open-simplex-noise-test.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) DoD HPCMP PETTT. All rights reserved. 3 | * See LICENSE file for details. 4 | * New copyright, applies to all modifications since the original copyright. 5 | * 6 | * Original Copyright: 7 | * This is free and unencumbered software released into the public domain. 8 | */ 9 | 10 | #include 11 | #if ((__STDC_VERSION__ >= 199901L) || (_MSC_VER)) 12 | #include 13 | #endif 14 | #include 15 | #include 16 | 17 | #include 18 | 19 | #include "open-simplex-noise.h" 20 | 21 | #define WIDTH 512 22 | #define HEIGHT 512 23 | #define FEATURE_SIZE 24 24 | 25 | static int write_png_image(const char *filename, unsigned char *pixels, int w, int h, int has_alpha) 26 | { 27 | png_structp png_ptr; 28 | png_infop info_ptr; 29 | png_byte **row; 30 | int x, y, rc, colordepth = 8; 31 | int bytes_per_pixel = has_alpha ? 4 : 3; 32 | FILE *f; 33 | 34 | f = fopen(filename, "w"); 35 | if (!f) { 36 | fprintf(stderr, "fopen: %s:%s\n", filename, strerror(errno)); 37 | return -1; 38 | } 39 | png_ptr = png_create_write_struct(PNG_LIBPNG_VER_STRING, NULL, NULL, NULL); 40 | if (!png_ptr) 41 | goto cleanup1; 42 | info_ptr = png_create_info_struct(png_ptr); 43 | if (!info_ptr) 44 | goto cleanup2; 45 | if (setjmp(png_jmpbuf(png_ptr))) /* oh libpng, you're old as dirt, aren't you. */ 46 | goto cleanup2; 47 | 48 | png_set_IHDR(png_ptr, info_ptr, (size_t) w, (size_t) h, colordepth, 49 | has_alpha ? PNG_COLOR_TYPE_RGBA : PNG_COLOR_TYPE_RGB, 50 | PNG_INTERLACE_NONE, PNG_COMPRESSION_TYPE_DEFAULT, 51 | PNG_FILTER_TYPE_DEFAULT); 52 | 53 | row = png_malloc(png_ptr, h * sizeof(*row)); 54 | for (y = 0; y < h; y++) { 55 | row[y] = png_malloc(png_ptr, w * bytes_per_pixel); 56 | for (x = 0; x < w; x++) { 57 | unsigned char *r = (unsigned char *) row[y]; 58 | unsigned char *src = (unsigned char *) 59 | &pixels[y * w * bytes_per_pixel + x * bytes_per_pixel]; 60 | unsigned char *dest = &r[x * bytes_per_pixel]; 61 | memcpy(dest, src, bytes_per_pixel); 62 | } 63 | } 64 | 65 | png_init_io(png_ptr, f); 66 | png_set_rows(png_ptr, info_ptr, row); 67 | png_write_png(png_ptr, info_ptr, PNG_TRANSFORM_PACKING, NULL); 68 | 69 | for (y = 0; y < h; y++) 70 | png_free(png_ptr, row[y]); 71 | png_free(png_ptr, row); 72 | rc = 0; 73 | cleanup2: 74 | png_destroy_write_struct(&png_ptr, &info_ptr); 75 | cleanup1: 76 | fclose(f); 77 | return rc; 78 | } 79 | 80 | int main(__attribute__((unused)) int argc, __attribute__((unused)) char *argv[]) 81 | { 82 | int x, y; 83 | double value; 84 | double v0, v1, v2; /* values from different octaves. */ 85 | uint32_t rgb; 86 | uint32_t image2d[HEIGHT][WIDTH]; 87 | uint32_t image3d[HEIGHT][WIDTH]; 88 | uint32_t image4d[HEIGHT][WIDTH]; 89 | struct osn_context *ctx; 90 | 91 | open_simplex_noise(77374, &ctx); 92 | 93 | for (y = 0; y < HEIGHT; y++) { 94 | for (x = 0; x < WIDTH; x++) { 95 | #if defined(SINGLE_OCTAVE) 96 | value = open_simplex_noise4(ctx, (double) x / FEATURE_SIZE, (double) y / FEATURE_SIZE, 0.0, 0.0); 97 | #else 98 | /* Use three octaves: frequency N, N/2 and N/4 with relative amplitudes 4:2:1. */ 99 | v0 = open_simplex_noise4(ctx, (double) x / FEATURE_SIZE / 4, 100 | (double) y / FEATURE_SIZE / 4, 0.0, 0.0); 101 | v1 = open_simplex_noise4(ctx, (double) x / FEATURE_SIZE / 2, 102 | (double) y / FEATURE_SIZE / 2, 0.0, 0.0); 103 | v2 = open_simplex_noise4(ctx, (double) x / FEATURE_SIZE / 1, 104 | (double) y / FEATURE_SIZE / 1, 0.0, 0.0); 105 | value = v0 * 4 / 7.0 + v1 * 2 / 7.0 + v2 * 1 / 7.0; 106 | #endif 107 | rgb = 0x010101 * (uint32_t) ((value + 1) * 127.5); 108 | image2d[y][x] = (0x0ff << 24) | (rgb); 109 | 110 | value = open_simplex_noise2(ctx, (double) x / FEATURE_SIZE, (double) y / FEATURE_SIZE); 111 | rgb = 0x010101 * (uint32_t) ((value + 1) * 127.5); 112 | image3d[y][x] = (0x0ff << 24) | (rgb); 113 | 114 | value = open_simplex_noise3(ctx, (double) x / FEATURE_SIZE, (double) y / FEATURE_SIZE, 0.0); 115 | rgb = 0x010101 * (uint32_t) ((value + 1) * 127.5); 116 | image4d[y][x] = (0x0ff << 24) | (rgb); 117 | } 118 | } 119 | write_png_image("test2d.png", (unsigned char *) image2d, WIDTH, HEIGHT, 1); 120 | write_png_image("test3d.png", (unsigned char *) image3d, WIDTH, HEIGHT, 1); 121 | write_png_image("test4d.png", (unsigned char *) image4d, WIDTH, HEIGHT, 1); 122 | open_simplex_noise_free(ctx); 123 | return 0; 124 | } 125 | 126 | -------------------------------------------------------------------------------- /osn/open-simplex-noise.h: -------------------------------------------------------------------------------- 1 | #ifndef OPEN_SIMPLEX_NOISE_H__ 2 | #define OPEN_SIMPLEX_NOISE_H__ 3 | 4 | /* 5 | * OpenSimplex (Simplectic) Noise in C. 6 | * Ported to C from Kurt Spencer's java implementation by Stephen M. Cameron 7 | * 8 | * v1.1 (October 6, 2014) 9 | * - Ported to C 10 | * 11 | * v1.1 (October 5, 2014) 12 | * - Added 2D and 4D implementations. 13 | * - Proper gradient sets for all dimensions, from a 14 | * dimensionally-generalizable scheme with an actual 15 | * rhyme and reason behind it. 16 | * - Removed default permutation array in favor of 17 | * default seed. 18 | * - Changed seed-based constructor to be independent 19 | * of any particular randomization library, so results 20 | * will be the same when ported to other languages. 21 | * 22 | * Copyright (c) DoD HPCMP PETTT. All rights reserved. 23 | * See LICENSE file for details. 24 | * New copyright, applies to all modifications since the original copyright. 25 | * 26 | * Original Copyright: 27 | * This is free and unencumbered software released into the public domain. 28 | */ 29 | 30 | #if ((__GNUC_STDC_INLINE__) || (__STDC_VERSION__ >= 199901L)) 31 | #include 32 | #define INLINE inline 33 | #elif (defined (_MSC_VER) || defined (__GNUC_GNU_INLINE__)) 34 | #include 35 | #define INLINE __inline 36 | #else 37 | /* ANSI C doesn't have inline or stdint.h. */ 38 | #define INLINE 39 | #endif 40 | 41 | #ifdef __cplusplus 42 | extern "C" { 43 | #endif 44 | 45 | struct osn_context; 46 | 47 | int open_simplex_noise(int64_t seed, struct osn_context **ctx); 48 | void open_simplex_noise_free(struct osn_context *ctx); 49 | int open_simplex_noise_init_perm(struct osn_context *ctx, int16_t p[], int nelements); 50 | double open_simplex_noise2(struct osn_context *ctx, double x, double y); 51 | double open_simplex_noise3(struct osn_context *ctx, double x, double y, double z); 52 | double open_simplex_noise4(struct osn_context *ctx, double x, double y, double z, double w); 53 | 54 | #ifdef __cplusplus 55 | } 56 | #endif 57 | 58 | #endif 59 | -------------------------------------------------------------------------------- /pdirs.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Functions to create and check directories from an MPI code 3 | * on a parallel file system 4 | * 5 | * Copyright (c) DoD HPCMP PETTT. All rights reserved. 6 | * See LICENSE file for details. 7 | */ 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | /* Create a directory from one rank (rank=0) 15 | * dirname: name of directory 16 | * comm: MPI communicator 17 | * note: this can be called from any/all ranks as long as rank 0 calls it 18 | * note: aborts for any error except when the directory already exists */ 19 | 20 | static void mkdir1task(char *dirname, MPI_Comm comm) 21 | { 22 | int rank, ret; 23 | 24 | MPI_Comm_rank(comm, &rank); 25 | if(rank == 0) { 26 | ret = mkdir(dirname, 0777); 27 | if( ret && errno != EEXIST ) { /* An error for anything but pre-existing */ 28 | fprintf(stderr, "pvtk error: Could not create directory.\n"); 29 | MPI_Abort(comm, 1); 30 | } 31 | } 32 | } 33 | 34 | /* Make sure a directory exists from one rank (rank=size-1) 35 | * dirname: name of directory 36 | * comm: MPI communicator 37 | * note: intended to be used in conjunction with mkdir1task to insure that 38 | * a created directory shows up on other ranks 39 | * note: this can be called from any/all ranks as long as rank size-1 calls it 40 | * note: tries several times and aborts if it doesn't exist 41 | */ 42 | 43 | static void chkdir1task(char *dirname, MPI_Comm comm) 44 | { 45 | int rank, nprocs, ret; 46 | struct stat fsbuf; 47 | 48 | MPI_Comm_rank(comm, &rank); 49 | MPI_Comm_size(comm, &nprocs); 50 | 51 | /* Have the last rank ensure that the directory is available */ 52 | if(rank == nprocs-1) { 53 | do { 54 | int retry = 0; 55 | usleep(200000); 56 | ret = stat(dirname, &fsbuf); 57 | if(ret && retry > 50) { /* 50 retries * 200000 us = 10 s timeout */ 58 | fprintf(stderr, "pvtk error: timeout waiting for directory %s\n", 59 | dirname); 60 | MPI_Abort(comm, 1); 61 | } 62 | } while(ret); 63 | } 64 | MPI_Barrier(comm); /* Everyone is safe to use directory after barrier */ 65 | } 66 | 67 | -------------------------------------------------------------------------------- /struct/Makefile: -------------------------------------------------------------------------------- 1 | 2 | include ../Makefile.inc 3 | 4 | OBJS = struct.o splitspace.o 5 | SRCS = struct.c splitspace.c 6 | 7 | ifdef ADIOS_HOME 8 | OBJS += adiosstruct.o 9 | SRCS += adiosstruct.c 10 | endif 11 | 12 | ifdef HDF5_DIR 13 | OBJS += hdf5struct.o 14 | SRCS += hdf5struct.c 15 | endif 16 | 17 | ifdef NETCDF_DIR 18 | OBJS += netcdfstruct.o 19 | SRCS += netcdfstruct.c 20 | endif 21 | 22 | ### End of Output Modules ### 23 | 24 | CFLAGS += $(OSNINC) 25 | 26 | .PHONY: clean depend 27 | 28 | struct: $(OSNOBJ) $(OBJS) 29 | $(CC) $(LDFLAGS) -o $@ $^ $(LIBS) 30 | 31 | clean: 32 | rm -f *.o struct 33 | 34 | depend: 35 | makedepend -- $(CFLAGS) -- -Y $(SRCS) # ignore warnings about system headers 36 | 37 | .c.o: 38 | $(CC) $(CFLAGS) -c $< $(INCLUDE) 39 | 40 | # DO NOT DELETE 41 | -------------------------------------------------------------------------------- /struct/README: -------------------------------------------------------------------------------- 1 | TEST examples 2 | mpirun --np 4 ./struct --tasks 2 2 --size 16 12 2 --tsteps 1 --hdf5 --hdf5_chunk 2 2 3 | # Smallest test (test1): 4 | mpirun --np 4 ./struct --tasks 2 2 --size 2 2 2 --tsteps 1 5 | 6 | # Smallest test with debugging statements (test1_debug): 7 | mpirun --np 4 ./struct --tasks 2 2 --size 2 2 2 --tsteps 1 --debug 8 | 9 | # Smallest test that generates ADIOS output for debugging (test2): 10 | mpirun --np 4 ./struct --tasks 2 2 --size 2 2 2 --tsteps 1 --adios MPI 11 | 12 | # Smallest test that generates ADIOS output for debugging (test2_iodebug: generates mask and height data): 13 | mpirun --np 4 ./struct --tasks 2 2 --size 2 2 2 --tsteps 1 --adios MPI --debugIO 14 | 15 | # Smallest test with default values passed same as test1: 16 | mpirun --np 4 ./struct --tasks 2 2 --size 2 2 2 --tsteps 1 --maskthreshold 0 --noisespacefreq 10 --noisetimefreq .25 17 | 18 | # larger grid that generates ADIOS output 19 | mpirun --np 4 ./struct --tasks 2 2 --size 100 100 60 --tsteps 1 --maskthreshold .2 --noisespacefreq 7 --noisetimefreq .25 --adios MPI 20 | 21 | 22 | # convert bp to nectcdf 23 | ~/ace4/adios/bin/bp2ncd struct.0000.bp 24 | 25 | # view bp data 26 | ~/ace4/adios/bin/bpls -d struct.0000.bp 27 | 28 | 29 | -------------------------------------------------------------------------------- /struct/adiosstruct.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) DoD HPCMP PETTT. All rights reserved. 3 | * See LICENSE file for details. 4 | */ 5 | 6 | #include 7 | #include 8 | 9 | #include "adiosstruct.h" 10 | #include "pdirs.h" 11 | 12 | static const int fnstrmax = 4095; 13 | 14 | void adiosstruct_init(struct adiosstructinfo *nfo, char *method, char *transform, 15 | char *name, char *adiosopts, MPI_Comm comm, int rank, int nprocs, 16 | int tsteps, int ni, int nj, int nk, int is, int cni, int js, int cnj, 17 | int ks, int cnk, float deltax, float deltay, float deltaz, float fillvalue) 18 | { 19 | static char emptystr[] = ""; 20 | 21 | nfo->name = name; 22 | nfo->transform = transform; 23 | nfo->comm = comm; 24 | nfo->rank = rank; 25 | nfo->nprocs = nprocs; 26 | nfo->tsteps = tsteps; 27 | nfo->ni = ni; 28 | nfo->nj = nj; 29 | nfo->nk = nk; 30 | nfo->is = is; 31 | nfo->cni = cni; 32 | nfo->js = js; 33 | nfo->cnj = cnj; 34 | nfo->ks = ks; 35 | nfo->cnk = cnk; 36 | nfo->deltax = deltax; 37 | nfo->deltay = deltay; 38 | nfo->deltaz = deltaz; 39 | nfo->fillvalue = fillvalue; 40 | 41 | nfo->maxvars = 1000; 42 | nfo->nrealvars = 0; 43 | nfo->realvarnames = (char **) malloc(nfo->maxvars * sizeof(char *)); 44 | nfo->realdatas = (float **) malloc(nfo->maxvars * sizeof(float *)); 45 | nfo->nintvars = 0; 46 | nfo->intvarnames = (char **) malloc(nfo->maxvars * sizeof(char *)); 47 | nfo->intdatas = (int **) malloc(nfo->maxvars * sizeof(int *)); 48 | 49 | nfo->bufallocsize = 0; 50 | 51 | if(!adiosopts) adiosopts = emptystr; 52 | 53 | /* Set up ADIOS */ 54 | adios_init_noxml(comm); 55 | adios_declare_group(&nfo->gid, nfo->name, "", adios_flag_no); 56 | adios_select_method(nfo->gid, method, adiosopts, ""); 57 | 58 | /* Define output variables */ 59 | adios_define_var(nfo->gid, "rank", "", adios_integer, "", "", ""); 60 | adios_define_var(nfo->gid, "tstep", "", adios_integer, "", "", ""); 61 | adios_define_var(nfo->gid, "ni", "", adios_integer, "", "", ""); 62 | adios_define_var(nfo->gid, "nj", "", adios_integer, "", "", ""); 63 | adios_define_var(nfo->gid, "nk", "", adios_integer, "", "", ""); 64 | adios_define_var(nfo->gid, "is", "", adios_integer, "", "", ""); 65 | adios_define_var(nfo->gid, "cni", "", adios_integer, "", "", ""); 66 | adios_define_var(nfo->gid, "js", "", adios_integer, "", "", ""); 67 | adios_define_var(nfo->gid, "cnj", "", adios_integer, "", "", ""); 68 | adios_define_var(nfo->gid, "ks", "", adios_integer, "", "", ""); 69 | adios_define_var(nfo->gid, "cnk", "", adios_integer, "", "", ""); 70 | adios_define_var(nfo->gid, "deltax", "", adios_real, "", "", ""); 71 | adios_define_var(nfo->gid, "deltay", "", adios_real, "", "", ""); 72 | adios_define_var(nfo->gid, "deltaz", "", adios_real, "", "", ""); 73 | } 74 | 75 | void adiosstruct_addrealxvar(struct adiosstructinfo *nfo, char *varname, float *data) { 76 | 77 | int64_t varid; 78 | 79 | if((nfo->nrealvars + nfo->nintvars) >= nfo->maxvars) 80 | return; /* Just ignore too many variables, for now */ 81 | 82 | nfo->realvarnames[nfo->nrealvars] = varname; 83 | nfo->realdatas[nfo->nrealvars] = data; 84 | nfo->nrealvars++; 85 | 86 | /* add vars fill value attribute */ 87 | adios_define_attribute_byvalue(nfo->gid, "_FillValue", varname , adios_real , 1, &nfo->fillvalue); 88 | 89 | /* add var */ 90 | varid = adios_define_var(nfo->gid, varname, "", adios_real, "cnk,cnj,cni", 91 | "nk,nj,ni", "ks,js,is"); 92 | if(nfo->transform) 93 | adios_set_transform(varid, nfo->transform); 94 | } 95 | 96 | void adiosstruct_addintxvar(struct adiosstructinfo *nfo, char *varname, int *data) { 97 | 98 | int64_t varid; 99 | 100 | if((nfo->nrealvars + nfo->nintvars) >= nfo->maxvars) 101 | return; /* Just ignore too many variables, for now */ 102 | nfo->intvarnames[nfo->nintvars] = varname; 103 | nfo->intdatas[nfo->nintvars] = data; 104 | nfo->nintvars++; 105 | 106 | /* add var */ 107 | varid = adios_define_var(nfo->gid, varname, "", adios_integer, "cnk,cnj,cni", 108 | "nk,nj,ni", "ks,js,is"); 109 | if(nfo->transform) 110 | adios_set_transform(varid, nfo->transform); 111 | } 112 | 113 | void adiosstruct_write(struct adiosstructinfo *nfo, int tstep) { 114 | char fname[fnstrmax+1]; 115 | int timedigits = 4; 116 | uint64_t ijkelems, groupsize, totalsize; 117 | int64_t handle; 118 | int ret, i; 119 | int bufneeded; 120 | 121 | ijkelems = (uint64_t) nfo->cni * nfo->cnj * nfo->cnk; 122 | groupsize = sizeof(int) /*rank*/ + sizeof(int) /*tstep*/ + 123 | sizeof(int)*3 /*ni,nj,nk*/ + sizeof(int)*6 /*is-cnk*/ + 124 | sizeof(float)*4 /*deltax,y,z - FVal*/ + 125 | sizeof(int) * ijkelems * nfo->nintvars /* mask */ + 126 | sizeof(float) * ijkelems * nfo->nrealvars; 127 | 128 | /* Allocate buffer large enough for all data to write, if not done already */ 129 | bufneeded = (int)(groupsize/(1024*1024)); 130 | bufneeded += bufneeded/4 + 20; /* Add an extra 25% & 20MB to be sure */ 131 | if(nfo->bufallocsize < bufneeded) { 132 | # if ADIOS_VERSION_GE(1,13,0) 133 | /* Don't set any max buffer size - seems to work best without it */ 134 | # elif ADIOS_VERSION_GE(1,10,0) 135 | adios_set_max_buffer_size(bufneeded); /* Set kludgy max buffer size */ 136 | # else 137 | adios_allocate_buffer(ADIOS_BUFFER_ALLOC_NOW, bufneeded); /* Old size */ 138 | # endif 139 | nfo->bufallocsize = bufneeded; 140 | } 141 | 142 | /* Set filename */ 143 | snprintf(fname, fnstrmax, "%s.out.%0*d.bp", nfo->name, timedigits, tstep); 144 | 145 | /* Open & Write */ 146 | ret = adios_open(&handle, nfo->name, fname, "w", nfo->comm); 147 | if(ret) { 148 | fprintf(stderr, "Error opening ADIOS file: %s\n", fname); 149 | return; 150 | } 151 | # if ADIOS_VERSION_LE(1,9,0) 152 | adios_group_size(handle, groupsize, &totalsize); 153 | # endif 154 | 155 | adios_write(handle, "rank", &nfo->rank); 156 | adios_write(handle, "tstep", &tstep); 157 | adios_write(handle, "ni", &nfo->ni); 158 | adios_write(handle, "nj", &nfo->nj); 159 | adios_write(handle, "nk", &nfo->nk); 160 | adios_write(handle, "is", &nfo->is); 161 | adios_write(handle, "cni", &nfo->cni); 162 | adios_write(handle, "js", &nfo->js); 163 | adios_write(handle, "cnj", &nfo->cnj); 164 | adios_write(handle, "ks", &nfo->ks); 165 | adios_write(handle, "cnk", &nfo->cnk); 166 | adios_write(handle, "deltax", &nfo->deltax); 167 | adios_write(handle, "deltay", &nfo->deltay); 168 | adios_write(handle, "deltaz", &nfo->deltaz); 169 | for(i = 0; i < nfo->nrealvars; i++) { 170 | adios_write(handle, nfo->realvarnames[i], nfo->realdatas[i]); 171 | } 172 | for(i = 0; i < nfo->nintvars; i++) { 173 | adios_write(handle, nfo->intvarnames[i], nfo->intdatas[i]); 174 | } 175 | adios_close(handle); 176 | } 177 | 178 | void adiosstruct_finalize(struct adiosstructinfo *nfo) { 179 | free(nfo->realvarnames); 180 | free(nfo->realdatas); 181 | free(nfo->intvarnames); 182 | free(nfo->intdatas); 183 | adios_finalize(nfo->rank); 184 | } 185 | 186 | -------------------------------------------------------------------------------- /struct/adiosstruct.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) DoD HPCMP PETTT. All rights reserved. 3 | * See LICENSE file for details. 4 | */ 5 | 6 | 7 | #include 8 | #include 9 | 10 | struct adiosstructinfo { 11 | char *transform; 12 | char *name; 13 | MPI_Comm comm; 14 | int rank; 15 | int nprocs; 16 | int tsteps; 17 | int ni, nj, nk; 18 | int is, cni, js, cnj, ks, cnk; 19 | float deltax, deltay, deltaz, fillvalue; 20 | 21 | int maxvars; 22 | 23 | int nrealvars; 24 | char **realvarnames; 25 | float **realdatas; 26 | 27 | int nintvars; 28 | char **intvarnames; 29 | int **intdatas; 30 | 31 | int bufallocsize; 32 | 33 | int64_t gid; 34 | }; 35 | 36 | void adiosstruct_init(struct adiosstructinfo *nfo, char *method, char *transform, 37 | char *name, char *adiosopts, MPI_Comm comm, int rank, int nprocs, 38 | int tsteps, int ni, int nj, int nk, int is, int cni, int js, int cnj, 39 | int ks, int cnk, float deltax, float deltay, float deltaz, float fillvalue); 40 | 41 | void adiosstruct_addrealxvar(struct adiosstructinfo *nfo, char *varname, float *data); 42 | 43 | void adiosstruct_addintxvar(struct adiosstructinfo *nfo, char *varname, int *data); 44 | 45 | void adiosstruct_write(struct adiosstructinfo *nfo, int tstep); 46 | 47 | void adiosstruct_finalize(struct adiosstructinfo *nfo); 48 | 49 | -------------------------------------------------------------------------------- /struct/hdf5struct.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) DoD HPCMP PETTT. All rights reserved. 3 | * See LICENSE file for details. 4 | */ 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #include "hdf5.h" 14 | #include "hdf5struct.h" 15 | 16 | #define TIMEIO 17 | 18 | #ifdef TIMEIO 19 | extern void timer_tock(double *timer); 20 | extern void timer_tick(double *timer, MPI_Comm comm, int barrier); 21 | extern void timer_collectprintstats(double timer, MPI_Comm comm, int destrank, char *prefix); 22 | #endif 23 | 24 | static const int fnstrmax = 4095; 25 | 26 | void 27 | write_xdmf_xml(char *fname, char *fname_xdmf, int num_xname, char **xname, 28 | int ni, int nj, int nk, 29 | float deltax, float deltay, float deltaz); 30 | 31 | void writehdf5(const int num_varnames, char **varnames, MPI_Comm comm, int rank, int nprocs, int tstep, 32 | int is, int js, int ks, 33 | int ni, int nj, int nk, int cni, int cnj, int cnk, 34 | float deltax, float deltay, float deltaz, 35 | float *data, hsize_t *h5_chunk, char *hdf5_compress, unsigned int *compress_par) 36 | { 37 | char fname[fnstrmax+1]; 38 | char fname_xdmf[fnstrmax+1]; 39 | int timedigits = 4; 40 | MPI_Info info = MPI_INFO_NULL; 41 | 42 | hid_t file_id; 43 | hid_t plist_id; 44 | hid_t memspace; 45 | hid_t filespace; 46 | hid_t did; 47 | hsize_t start[3], count[3]; 48 | hsize_t *block=NULL; 49 | hsize_t dimsf[3]; 50 | hsize_t dimsm[3]; 51 | int j; 52 | herr_t err; 53 | hid_t chunk_pid; 54 | hsize_t chunk[3]; 55 | 56 | snprintf(fname, fnstrmax, "struct_t%0*d.h5", timedigits, tstep); 57 | snprintf(fname_xdmf, fnstrmax, "struct_t%0*d.xmf", timedigits, tstep); 58 | 59 | dimsf[0] = nk; 60 | dimsf[1] = nj; 61 | dimsf[2] = ni; 62 | dimsm[0] = cnk; 63 | dimsm[1] = cnj; 64 | dimsm[2] = cni; 65 | 66 | #ifdef TIMEIO 67 | double createfile, prewrite, write, postwrite; /* Timers */ 68 | timer_tick(&createfile, comm, 0); 69 | #endif 70 | 71 | if(rank == 0) { 72 | 73 | if( (file_id = H5Fcreate(fname, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) { 74 | fprintf(stderr, "writehdf5 error: could not create %s \n", fname); 75 | MPI_Abort(comm, 1); 76 | } 77 | 78 | filespace = H5Screate_simple(3, dimsf, NULL); 79 | 80 | chunk_pid = H5Pcreate(H5P_DATASET_CREATE); 81 | 82 | H5Pset_alloc_time(chunk_pid, H5D_ALLOC_TIME_EARLY); 83 | 84 | if(h5_chunk) { 85 | H5Pset_layout(chunk_pid, H5D_CHUNKED); 86 | if( H5Pset_fill_time(chunk_pid, H5D_FILL_TIME_NEVER) < 0 ) { 87 | printf("writehdf5 error: Could not set fill time\n"); 88 | MPI_Abort(comm, 1); 89 | } 90 | chunk[0]=dimsm[0]/h5_chunk[1]; 91 | chunk[1]=dimsm[1]/h5_chunk[0]; 92 | chunk[2]=dimsm[2]; 93 | 94 | H5Pset_chunk(chunk_pid, 3, chunk); 95 | 96 | if(strcasecmp(hdf5_compress, "\0") != 0) { 97 | 98 | if(!strcasecmp(hdf5_compress, "gzip")) { 99 | /* Set ZLIB / DEFLATE Compression. */ 100 | if( H5Pset_deflate (chunk_pid, compress_par[0]) < 0 ) { 101 | printf("writehdf5 error: Could not set compression\n"); 102 | MPI_Abort(comm, 1); 103 | } 104 | } else if(!strcasecmp(hdf5_compress, "shuffle+gzip")) { 105 | /* Set Shuffle before Deflate */ 106 | if( H5Pset_shuffle (chunk_pid) < 0 ) { 107 | printf("writehdf5 error: H5Pset_shuffle failed\n"); 108 | MPI_Abort(comm, 1); 109 | } 110 | /* Set ZLIB / DEFLATE Compression. */ 111 | if( H5Pset_deflate (chunk_pid, compress_par[0]) < 0 ) { 112 | printf("writehdf5 error: Could not set compression\n"); 113 | MPI_Abort(comm, 1); 114 | } 115 | } else if(!strcasecmp(hdf5_compress, "szip")) { 116 | /* Set SZIP Compression. */ 117 | if( H5Pset_szip (chunk_pid, compress_par[0], compress_par[1]) < 0 ) { 118 | printf("writehdf5 error: Could not set compression\n"); 119 | MPI_Abort(comm, 1); 120 | } 121 | } else 122 | fprintf(stderr, "WARNING: Compression option not recognized: %s\n", hdf5_compress); 123 | } 124 | } 125 | 126 | /* Create the dataset with default properties */ 127 | 128 | for (j=0; j\n"); 294 | fprintf(xmf, "\n"); 295 | fprintf(xmf, "\n"); 296 | fprintf(xmf, " \n\n"); 297 | fprintf(xmf, " \n"); 298 | fprintf(xmf, " \n", nk, nj, ni); 299 | fprintf(xmf, " \n\n"); 300 | fprintf(xmf, " \n"); 301 | fprintf(xmf, " \n"); 302 | fprintf(xmf, " \n"); 303 | fprintf(xmf, " 0.0 0.0 0.0 \n"); 304 | fprintf(xmf, " \n"); 305 | fprintf(xmf, " \n"); 306 | fprintf(xmf, " \n"); 307 | fprintf(xmf, " %.6f %.6f %.6f \n", deltax, deltay, deltaz); 308 | fprintf(xmf, " \n"); 309 | fprintf(xmf, " \n"); 310 | for (j=0; j\n", varnames[j]); 312 | if(strcmp(varnames[j],"data") == 0 || strcmp(varnames[j],"height") == 0) { 313 | fprintf(xmf, " \n", nk*nj*ni); 314 | } else { 315 | fprintf(xmf, " \n", nk*nj*ni); 316 | } 317 | fprintf(xmf, " %s:/%s\n", fname, varnames[j]); 318 | fprintf(xmf, " \n"); 319 | fprintf(xmf, " \n"); 320 | } 321 | fprintf(xmf, " \n"); 322 | fprintf(xmf, " \n"); 323 | fprintf(xmf, "\n"); 324 | fclose(xmf); 325 | } 326 | -------------------------------------------------------------------------------- /struct/hdf5struct.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) DoD HPCMP PETTT. All rights reserved. 3 | * See LICENSE file for details. 4 | */ 5 | #ifdef HAS_HDF5 6 | # include "hdf5.h" 7 | #endif 8 | void writehdf5(const int num_varnames, char **varnames, MPI_Comm comm, int rank, int nprocs, int tstep, 9 | int is, int js, int ks, 10 | int ni, int nj, int nk, int cni, int cnj, int cnk, 11 | float deltax, float deltay, float deltaz, 12 | float *data, hsize_t *h5_chunk, char *hdf5_compress, unsigned int *compress_par); 13 | -------------------------------------------------------------------------------- /struct/netcdfstruct.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) DoD HPCMP PETTT. All rights reserved. 3 | * See LICENSE file for details. 4 | */ 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #include "netcdf.h" 14 | #include "netcdf_par.h" 15 | #include "netcdfstruct.h" 16 | #include "hdf5.h" 17 | 18 | #define NC_MAX_NAME_LENGTH 4095 19 | #define NDIMS 3 20 | 21 | #define ERR {if(err != NC_NOERR) {printf("(rank %d) Error at line %d: %s\n",rank,__LINE__,nc_strerror(err)); fflush(stdout); MPI_Abort(comm,1);}} 22 | 23 | /*! Write structured grid via netCDF4 API. 24 | * 25 | * Writing a structured grid for the struct mini app. 26 | * 27 | * @param[in] num_varnames Number of variable names. 28 | * @param[in] varnames Variable names. 29 | * @param[in] comm MPI_Comm variable. 30 | * @param[in] rank MPI_Comm_rank variable. 31 | * @param[in] nprocs MPI_Comm_size variable. 32 | * @param[in] tstep 33 | * @param[in] is global index starting point. 34 | * @param[in] js global index starting point. 35 | * @param[in] ks global index starting point. 36 | * @param[in] ni global grid size. 37 | * @param[in] nj global grid size. 38 | * @param[in] nk global grid size. 39 | * @param[in] cni points in this task. 40 | * @param[in] cnj points in this task. 41 | * @param[in] cnk points in this task. 42 | * @param[in] data data to be written. 43 | * @param[in] height data to be writen. 44 | * @param[in] ola_mask data to be written. 45 | * @param[in] ol_mask data to be written. 46 | */ 47 | void writenc(const int num_varnames, char **varnames, MPI_Comm comm, int rank, 48 | int nprocs, int tstep, 49 | int is, int js, int ks, 50 | int ni, int nj, int nk, int cni, int cnj, int cnk, 51 | float *data) { 52 | 53 | 54 | char fname[NC_MAX_NAME_LENGTH +1]; 55 | int i = 0; 56 | int loc_numvars = 0; 57 | int timedigits = 4; 58 | MPI_Info info = MPI_INFO_NULL; 59 | int err=0; 60 | size_t start[3] = {0,0,0}; 61 | size_t count[3] = {0,0,0}; 62 | size_t dimsf[3]; 63 | size_t dimsm[3]; 64 | int j; 65 | int ncid; 66 | int dimids[NDIMS]; 67 | int varids[4]; 68 | snprintf(fname, NC_MAX_NAME_LENGTH, "struct_t%0*d.nc", timedigits, tstep); 69 | 70 | dimsf[0] = (hsize_t)ni; 71 | dimsf[1] = (hsize_t)nj; 72 | dimsf[2] = (hsize_t)nk; 73 | dimsm[0] = (hsize_t)cni; 74 | dimsm[1] = (hsize_t)cnj; 75 | dimsm[2] = (hsize_t)cnk; 76 | 77 | 78 | err = nc_create_par(fname,NC_NETCDF4|NC_MPIIO,comm,info,&ncid); 79 | /*create dimensions. */ 80 | err = nc_def_dim(ncid,"k", nk, &dimids[0]); ERR; 81 | err = nc_def_dim(ncid,"j", nj, &dimids[1]); ERR; 82 | err = nc_def_dim(ncid,"i", ni, &dimids[2]); ERR; 83 | 84 | for(i = 0; i < num_varnames; i++) { 85 | if(strcmp(varnames[i],"data") == 0 || strcmp(varnames[i],"height") == 0) { 86 | nc_def_var(ncid,varnames[i], NC_FLOAT, 3, &dimids[0], &varids[i]); 87 | } else { 88 | nc_def_var(ncid,varnames[i], NC_INT, 3, &dimids[0], &varids[i]); 89 | } 90 | } 91 | /* End definition mode & close file.*/ 92 | err = nc_enddef(ncid); ERR; 93 | 94 | /* Wait for everybody to get to this point, 95 | * reopen file and do that thing. */ 96 | MPI_Barrier(comm); 97 | 98 | MPI_Info_create(&info); 99 | 100 | err = nc_inq_varids(ncid,&loc_numvars,&varids[0]); ERR; 101 | 102 | for(i = 0; i < loc_numvars; i++) { 103 | err = nc_var_par_access(ncid,varids[i],NC_COLLECTIVE); ERR; 104 | } 105 | 106 | start[0] = (hsize_t)ks; 107 | start[1] = (hsize_t)js; 108 | start[2] = (hsize_t)is; 109 | count[0] = (hsize_t)cnk; 110 | count[1] = (hsize_t)cnj; 111 | count[2] = (hsize_t)cni; 112 | /* Write out the data for each variable. */ 113 | for(i = 0; i < num_varnames; i++) { 114 | if(strcmp(varnames[i],"data") == 0) { 115 | err = nc_put_vara_float(ncid,varids[i],start,count,data); ERR; 116 | /*} else if(strcmp(varnames[i],"height") == 0) { 117 | err = nc_put_vara_float(ncid,varids[i],start,count,height); ERR; 118 | } else if(strcmp(varnames[i],"ola_mask") == 0) { 119 | err = nc_put_vara_int(ncid,varids[i],start,count,ola_mask); ERR; 120 | } else if (strcmp(varnames[i],"ol_mask") == 0) { 121 | err = nc_put_vara_int(ncid,varids[i],start,count,ol_mask); ERR; */ 122 | } else { 123 | printf("writenc error: Unknown variable %s\n",varnames[i]); 124 | MPI_Abort(comm,1); 125 | } 126 | MPI_Barrier(comm); 127 | } /* End loop to write variables. */ 128 | 129 | MPI_Barrier(comm); 130 | /* Close out, lets go home. */ 131 | err = nc_close(ncid); ERR; 132 | 133 | } 134 | -------------------------------------------------------------------------------- /struct/netcdfstruct.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) DoD HPCMP PETTT. All rights reserved. 3 | * See LICENSE file for details. 4 | */ 5 | 6 | #ifdef HAS_NC 7 | #include "netcdf.h" 8 | #endif 9 | 10 | void writenc(const int num_varnames, char **varnames, MPI_Comm comm, int rank, int nprocs, int tstep, 11 | int is, int js, int ks, 12 | int ni, int nj, int nk, int cni, int cnj, int cnk, 13 | float *data); 14 | -------------------------------------------------------------------------------- /struct/processlog.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Process an output log from unstruct and produce a results table 4 | # + Determines output method from log contents 5 | # + Can also check filename for method 6 | # - Filename must contain .przm., .posix., .mpi., etc (see fname2method) 7 | # - If filename & log contents don't agree, will abort with error 8 | # - If you didn't name file properly (or at all), just gives a warning 9 | # + Each column is a time step 10 | # + Table rows: method, cores, grid points (Million), full file size, time, rate (GB/s) 11 | # + Multiple methods/types can be pasted into Excel or similar for plotting/analysis 12 | 13 | import sys 14 | import re 15 | 16 | fname2method = { ".przm.": "MPI-Indiv", ".posix.": "ADIOS-POSIX", 17 | ".mpi.": "ADIOS-MPI", ".mpilus.": "ADIOS-MPI_Lustre", ".mpiag.": "ADIOS-MPI_Aggr", 18 | ".phdf5.": "ADIOS-PHDF5", ".hdf5.": "HDF5", } 19 | lname2method = { "przm": "MPI-Indiv", "POSIX": "ADIOS-POSIX", 20 | "MPI ": "ADIOS-MPI", "MPI_LUSTRE": "ADIOS-MPI_Lustre", 21 | "MPI_AGGREGATE": "ADIOS-MPI_Aggr", "PHDF5": "ADIOS-PHDF5", 22 | "hdf5": "HDF5" } 23 | 24 | totalrate = -1.0 25 | a = 1 26 | printFName = False 27 | 28 | for arg in sys.argv[1:2]: 29 | if arg == "-a": # produce average rate 30 | totalrate = 0.0 31 | a += 1 32 | 33 | if len(sys.argv[a:]) > 1: printFName = True 34 | 35 | for fname in sys.argv[a:]: 36 | 37 | v = [] 38 | if totalrate > 0.0: totalrate = 0.0 # reset for new file 39 | bal = False 40 | 41 | # Determine method from file name, if it's there 42 | method1key = next((x for x in fname2method.keys() if x in fname), False) 43 | if not method1key: print "WARNING: method not specified in filename" 44 | 45 | f = open(fname) 46 | for l in f: 47 | ls = l.split() 48 | 49 | if(len(ls) < 1): continue 50 | if(len(ls) < 2): ls.append("") # Ensure all outer if's can handle ls 51 | 52 | if ls[0][0:4] == "tsk=": 53 | if ls[0][4] == "\"": continue # Skip LSF copy of batch script 54 | 55 | print ls 56 | cores = int(ls[0][4:]) * int(ls[1]) 57 | #size = ls[2][3:] + 'x' +ls[3][3:] + 'x' + ls[4][3:] 58 | #sizeprod = float(ls[2][3:]) * float(ls[3][3:]) * float(ls[4][3:]) 59 | #sizegb = sizeprod*4/1024**3 60 | 61 | # Determine method from line; make sure matches one from file name 62 | method2key = next((x for x in lname2method.keys() if re.search(x, l)), False) 63 | if(re.search("balance", l)): 64 | bal= True 65 | 66 | if method2key: 67 | if method1key: 68 | if fname2method[method1key] != lname2method[method2key]: 69 | print "Error: method in filename doesn't match method in file" 70 | sys.exit(1) 71 | else: 72 | method = fname2method[method1key] 73 | else: 74 | method = lname2method[method2key] 75 | else: 76 | print "WARNING: method not specified in log file" 77 | if method1key: 78 | method = fname2method[method1key] 79 | else: 80 | print "ERROR: not method specified, cannot proceed" 81 | sys.exit(1) 82 | 83 | #if bal: 84 | # method = method + "_Bal" 85 | # bal = False 86 | elif ls[0] == "size:": 87 | size = ls[1] + "x" + ls[2] + "x" + ls[3] 88 | 89 | elif ls[0] == "datasize:": 90 | sizegb = float(ls[1]) / 1024**3 91 | 92 | elif ls[0] == "Output": 93 | outtime = float(ls[11][:-1]) 94 | rate = sizegb/outtime 95 | balstr = "Yes" if bal else "No" 96 | if totalrate >= 0.0: totalrate += rate 97 | v.append([method, balstr, cores, size, sizegb, outtime, rate]) 98 | 99 | if printFName: print fname 100 | print "%-16s %3s %5s %-16s %7s %6s %6s" % ("method", "bal", "cores", "size", "f_GB", "ftime", "fGB/s") 101 | for i in v: 102 | print "%-16s %3s %5d %-16s %7.1f %6.2f %6.2f" % tuple(i) 103 | if totalrate >= 0.0: print "Avg rate =", totalrate/len(v) 104 | 105 | -------------------------------------------------------------------------------- /struct/run_hdf.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Utility script to easily run a netCDF file. 3 | 4 | set -e 5 | 6 | mpirun --np 4 ./struct --tasks 2 2 --size 2 2 2 --tsteps 10 --hdf5 7 | #mpirun --np 4 ./struct --tasks 2 2 --size 16 12 2 --tsteps 1 --hdf5 8 | -------------------------------------------------------------------------------- /struct/run_nc.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Utility script to easily run a netCDF file. 3 | 4 | set -e 5 | 6 | #mpirun --np 4 ./struct --tasks 2 2 --size 2 2 2 --tsteps 1 --nc 7 | mpirun --np 4 ./struct --tasks 2 2 --size 2 2 2 --tsteps 10 --nc 8 | -------------------------------------------------------------------------------- /struct/splitspace.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) DoD HPCMP PETTT. All rights reserved. 3 | * See LICENSE file for details. 4 | */ 5 | 6 | struct factorstruct { 7 | int *fac; 8 | int size; 9 | int n; 10 | }; 11 | 12 | typedef struct subArrayItem { 13 | 14 | int start_index; 15 | int size; 16 | 17 | } subArrayItem; 18 | 19 | /* holds an entry in the stack (in this case cube information) */ 20 | typedef struct recItem { 21 | 22 | int x0; 23 | int y0; 24 | int x1; 25 | int y1; 26 | } recItem; 27 | 28 | /* A stack that holds and array of stack items (in this case an array of cube information) */ 29 | typedef struct { 30 | recItem *recs; 31 | int top; 32 | int size; 33 | int max_size; 34 | } recList; 35 | 36 | void init_primeFactors(struct factorstruct *factors, int n); 37 | void free_primeFactors(struct factorstruct *factors); 38 | void get_primeFactors(struct factorstruct *factors); 39 | void print_primeFactors(struct factorstruct *factors); 40 | void init_rects(recList *l, struct factorstruct *factors); 41 | void add_rect(recList *l, int x0, int y0, int x1, int y1); 42 | void free_rects(recList *l); 43 | void empty_rects(recList *l); 44 | void print_rects(recList *l); 45 | void count_rects(recList *l); 46 | void copy_rectList(recList *l, recList *l2); 47 | void append_rectList(recList *l, recList *l2); 48 | void subdivideArray(int *cvals, subArrayItem *subarray, int size, int n); 49 | int subarray_sum(int *cvals, subArrayItem subarray); 50 | float subarrays_var(int *cvals, subArrayItem *subarray, int n); 51 | void subdivide_rects(recList *l, recItem rec, int *data, int n) ; 52 | void init_splitspace(struct factorstruct *factors, recList *rects_list, int nprocs); 53 | void splitspace(recList *l, int x_dims, int y_dims, struct factorstruct *factors, int *data); 54 | void free_splitspace(struct factorstruct *factors, recList *rects_list); 55 | -------------------------------------------------------------------------------- /struct/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Script for testing struct 4 | 5 | INP=2 6 | JNP=4 7 | 8 | NI=64 9 | NJ=128 10 | NK=256 11 | 12 | NT=2 13 | 14 | CHUNK_Y=$(($NJ/$JNP)) 15 | CHUNK_Z=32 16 | 17 | output="--hdf5" 18 | mpirun -np 8 ./struct --tasks $INP $JNP --size $NI $NJ $NK --tsteps $NT $output 19 | 20 | output="--hdf5 --hdf5_chunk $CHUNK_Y $CHUNK_Z --hdf5_compress gzip,9" 21 | mpirun -np 8 ./struct --tasks $INP $JNP --size $NI $NJ $NK --tsteps $NT $output 22 | 23 | 24 | -------------------------------------------------------------------------------- /struct/test1-64.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ## Struct test script for 1 - 64 ranks 4 | 5 | # Set up work directory 6 | w=/mnt/ssd/teststruct1-64.$BASHPID 7 | mkdir -p $w 8 | cp ./struct ./processlog.py $w 9 | cd $w 10 | 11 | # Parameter spaces to cover 12 | ps=("1,1" "2,1" "2,2" "4,2" "4,4" "8,4" "11,4" "8,8") # "i,j" ranks 13 | outputs=( "--adios POSIX" "--adios MPI_LUSTRE" ) # output types 14 | comps=( "" "--adios_transform zlib" "--adios_transform szip" "--adios_transform zfp:accuracy=0.0001" "--adios_transform sz:absolute=0.0001" ) 15 | balances=( "" "--balance" ) # to balance or not to balance 16 | cleanup=1 # Remove data directories after each run: 0 or 1 17 | progress=1 # Print progress: 0 or 1 18 | mpi=mpirun # MPI launcher 19 | ni=128x # i points (with x: per i rank) 20 | nj=128x # j points (with x: per j rank) 21 | nk=1024 # k levels stays constant 22 | noiseij=10x # frequency of spatial noise (with x: per i,j rank) 23 | noisek=10 # frequency of spatial noise k 24 | ts=11 # time steps 25 | 26 | for p in "${ps[@]}"; do 27 | IFS="," read -a pp <<< "${p}" # Convert i,j ranks to array 28 | pi=${pp[0]} # i ranks 29 | pj=${pp[1]} # j ranks 30 | nc=$(($pi * $pj)) # total ranks 31 | tsk="$pi $pj" # to pass to command line 32 | i=0 33 | 34 | for output in "${outputs[@]}"; do 35 | for comp in "${comps[@]}"; do 36 | for balance in "${balances[@]}"; do 37 | name=$(printf "test.%02d.%02d" $nc $i) 38 | log=../${name}.log 39 | dir=${name}.d 40 | mkdir $dir 41 | cd $dir 42 | echo "tsk=$tsk ni=$ni nj=$nj nk=$nk ts=$ts output=$output $comp $balance hints=$MPICH_MPIIO_HINTS" > $log 43 | echo "$mpi -n $nc ../struct --tasks $tsk --size $ni $nj $nk --tsteps $ts --noisespacefreq $noiseij $noiseij $noisek $output $comp $balance" >> $log 44 | if [ $progress -ne 0 ]; then echo "$tsk $output $comp $balance ..."; fi 45 | 46 | # Real work happens here 47 | $mpi -n $nc ../struct --tasks $tsk --size $ni $nj $nk --tsteps $ts --noisespacefreq $noiseij $noiseij $noisek $output $comp $balance 2>&1 >> $log 48 | 49 | ls -l >> $log 50 | if [ $(ls -1d *.dir 2>/dev/null | wc -l) -gt 0 ]; then # list ADIOS .dirs if exist 51 | ls -l *.dir >> $log 52 | fi 53 | 54 | if [ $progress -ne 0 ]; then ../processlog.py -a $log; fi 55 | 56 | cd .. 57 | if [ $cleanup -ne 0 ]; then 58 | rm -r $dir 59 | fi 60 | ((i++)) 61 | done 62 | done 63 | done 64 | done 65 | 66 | -------------------------------------------------------------------------------- /timer.h: -------------------------------------------------------------------------------- 1 | /* 2 | * A set of parallel timer convenience functions 3 | * 4 | * Copyright (c) DoD HPCMP PETTT. All rights reserved. 5 | * See LICENSE file for details. 6 | */ 7 | 8 | #include 9 | #include 10 | 11 | /* Initialize a timer 12 | * timer: pointer to timer to initialize 13 | * comm: communicator for barrier 14 | * barrier: flag, 0=no barrier, 1=perform barrier */ 15 | 16 | void timer_tick(double *timer, MPI_Comm comm, int barrier) 17 | { 18 | if(barrier) 19 | MPI_Barrier(comm); 20 | *timer = MPI_Wtime(); 21 | } 22 | 23 | /* Statistics structure: contains stats that aggregate timer 24 | * values from all ranks */ 25 | 26 | struct timer_statinfo { 27 | double min; 28 | double max; 29 | double mean; 30 | double std; 31 | }; 32 | 33 | /* Stop a timer and record elapsed time 34 | * timer: pointer to timer; will contain elapsed time of rank when done */ 35 | 36 | void timer_tock(double *timer) 37 | { 38 | double t2 = MPI_Wtime(); 39 | *timer = t2 - *timer; 40 | } 41 | 42 | /* Collect statistics of timers on all ranks 43 | * timer: elapsed time for rank 44 | * comm: communicator for collecting stats 45 | * destrank: the rank to which to collect stats 46 | * stats: pointer to timerstats */ 47 | 48 | void timer_collectstats(double timer, MPI_Comm comm, int destrank, struct timer_statinfo *stats) 49 | { 50 | int rank, nprocs, i; 51 | double *rtimers=NULL; /* All timers from ranks */ 52 | 53 | MPI_Comm_rank(comm, &rank); 54 | MPI_Comm_size(comm, &nprocs); 55 | if(rank == destrank) { 56 | rtimers = (double *) malloc(nprocs*sizeof(double)); 57 | stats->mean = 0.; 58 | stats->min = timer; 59 | stats->max = timer; 60 | stats->std = 0.f; 61 | } 62 | MPI_Gather(&timer, 1, MPI_DOUBLE, rtimers, 1, MPI_DOUBLE, destrank, comm); 63 | if(rank == destrank) { 64 | for(i = 0; i < nprocs; i++) { 65 | if(rtimers[i] > stats->max) stats->max = rtimers[i]; 66 | if(rtimers[i] < stats->min) stats->min = rtimers[i]; 67 | stats->mean += rtimers[i]; 68 | } 69 | stats->mean /= nprocs; 70 | for(i = 0; i < nprocs; i++) 71 | stats->std += (rtimers[i]-stats->mean)*(rtimers[i]-stats->mean); 72 | stats->std = sqrt(stats->std / nprocs); 73 | free(rtimers); 74 | } 75 | } 76 | 77 | /* Print time statistics 78 | * prefix: string to print in front of stats, use "" for no string 79 | * stats: pointer to timerstats */ 80 | 81 | void timer_printstats(char *prefix, struct timer_statinfo *stats) 82 | { 83 | printf("%s timer seconds mean = %.2f, min = %.2f, max = %.2f, std = %.3f\n", 84 | prefix, stats->mean, stats->min, stats->max, stats->std); 85 | } 86 | 87 | /* Collect statistics of timers on all ranks to one rank and print from that rank 88 | * timer: elapsed time for rank 89 | * comm: communicator for collecting stats 90 | * destrank: the rank to which to collect stats 91 | * prefix: string to print in front of stats, use "" for no string */ 92 | 93 | void timer_collectprintstats(double timer, MPI_Comm comm, int destrank, char *prefix) 94 | { 95 | int rank; 96 | struct timer_statinfo stats; 97 | 98 | MPI_Comm_rank(comm, &rank); 99 | timer_collectstats(timer, comm, destrank, &stats); 100 | if(rank == destrank) 101 | timer_printstats(prefix, &stats); 102 | } 103 | 104 | -------------------------------------------------------------------------------- /unstruct/Makefile: -------------------------------------------------------------------------------- 1 | 2 | include ../Makefile.inc 3 | 4 | OBJS = unstruct.o 5 | SRCS = unstruct.c 6 | 7 | ### Add Output Modules Here ### 8 | 9 | # PRZM Output Module 10 | OBJS += przm.o 11 | SRCS += przm.c 12 | CFLAGS += -DHAS_PRZM 13 | 14 | # ADIOS Output Module 15 | ifdef ADIOS_HOME 16 | OBJS += adiosunstruct.o 17 | SRCS += adiosunstruct.c 18 | endif 19 | 20 | ifdef HDF5_DIR 21 | OBJS += hdf5unstruct.o 22 | SRCS += hdf5unstruct.c 23 | endif 24 | 25 | ### if NC_DIR is set then also enable netCDF Output Module ### 26 | ifdef NC_DIR 27 | OBJS += ncunstruct.o 28 | SRCS += ncunstruct.c 29 | endif 30 | 31 | ### End of Output Modules ### 32 | 33 | CFLAGS += $(OSNINC) 34 | 35 | .PHONY: clean depend 36 | 37 | unstruct: $(OSNOBJ) $(OBJS) 38 | $(CC) $(LDFLAGS) -o $@ $^ $(LIBS) 39 | 40 | clean: 41 | rm -f *.o unstruct 42 | 43 | depend: 44 | makedepend -- $(CFLAGS) -- -Y $(SRCS) # ignore warnings about system headers 45 | 46 | .c.o: 47 | $(CC) $(CFLAGS) -c $< $(INCLUDE) 48 | 49 | # DO NOT DELETE 50 | 51 | unstruct.o: ../osn/open-simplex-noise.h przm.h adiosunstruct.h 52 | przm.o: ../pdirs.h przm.h 53 | adiosunstruct.o: adiosunstruct.h 54 | netcdfunstruct.o: ../pdirs.h netcdf.h 55 | hdf5unstruct.o: ../pdirs.h 56 | -------------------------------------------------------------------------------- /unstruct/adiosunstruct.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) DoD HPCMP PETTT. All rights reserved. 3 | * See LICENSE file for details. 4 | */ 5 | 6 | #include 7 | #include 8 | 9 | #include "adiosunstruct.h" 10 | 11 | static const int fnstrmax = 4095; 12 | 13 | void adiosunstruct_init(struct adiosinfo *nfo, char *method, char *name, MPI_Comm comm, 14 | int rank, int nprocs, int tsteps, uint64_t nptstask, 15 | uint64_t nelems3, uint64_t nelems2) 16 | { 17 | /* Set up struct */ 18 | nfo->name = name; 19 | nfo->comm = comm; 20 | nfo->rank = rank; 21 | nfo->nprocs = nprocs; 22 | nfo->tsteps = tsteps; 23 | nfo->cnpoints = nptstask; 24 | nfo->cnelems3 = nelems3; 25 | nfo->cnelems2 = nelems2; 26 | nfo->nvars = 0; 27 | nfo->maxvars = 1000; 28 | nfo->varnames = (char **) malloc(nfo->maxvars * sizeof(char *)); 29 | nfo->bufallocsize = 0; 30 | 31 | /* Set up ADIOS */ 32 | adios_init_noxml(comm); 33 | adios_declare_group(&nfo->gid, nfo->name, "", adios_flag_no); 34 | /*chkdir1task(dirname, comm);*/ 35 | adios_select_method(nfo->gid, method, "", /*dirname*/""); 36 | 37 | /* Define output variables */ 38 | adios_define_var(nfo->gid, "rank", "", adios_integer, "", "", ""); 39 | adios_define_var(nfo->gid, "tstep", "", adios_integer, "", "", ""); 40 | adios_define_var(nfo->gid, "npoints", "", adios_unsigned_long, "", "", ""); 41 | adios_define_var(nfo->gid, "cnpoints", "", adios_unsigned_long, "", "", ""); 42 | adios_define_var(nfo->gid, "cspoints", "", adios_unsigned_long, "", "", ""); 43 | adios_define_var(nfo->gid, "nconns3", "", adios_unsigned_long, "", "", ""); 44 | adios_define_var(nfo->gid, "cnconns3", "", adios_unsigned_long, "", "", ""); 45 | adios_define_var(nfo->gid, "csconns3", "", adios_unsigned_long, "", "", ""); 46 | adios_define_var(nfo->gid, "nconns2", "", adios_unsigned_long, "", "", ""); 47 | adios_define_var(nfo->gid, "cnconns2", "", adios_unsigned_long, "", "", ""); 48 | adios_define_var(nfo->gid, "csconns2", "", adios_unsigned_long, "", "", ""); 49 | adios_define_var(nfo->gid, "xpts", "", adios_real, 50 | "cnpoints", "npoints", "cspoints"); 51 | adios_define_var(nfo->gid, "ypts", "", adios_real, 52 | "cnpoints", "npoints", "cspoints"); 53 | adios_define_var(nfo->gid, "zpts", "", adios_real, 54 | "cnpoints", "npoints", "cspoints"); 55 | adios_define_var(nfo->gid, "conns3", "", adios_unsigned_long, 56 | "cnconns3", "nconns3", "csconns3"); 57 | adios_define_var(nfo->gid, "conns2", "", adios_unsigned_long, 58 | "cnconns2", "nconns2", "csconns2"); 59 | } 60 | 61 | void adiosunstruct_addvar(struct adiosinfo *nfo, char *varname) 62 | { 63 | if(nfo->nvars > nfo->maxvars) 64 | return; /* Just ignore too many variables, for now */ 65 | nfo->varnames[nfo->nvars] = varname; 66 | nfo->nvars++; 67 | adios_define_var(nfo->gid, varname, "", adios_real, 68 | "cnpoints", "npoints", "cspoints"); 69 | } 70 | 71 | void adiosunstruct_write(struct adiosinfo *nfo, int tstep, float *xpts, float *ypts, 72 | float *zpts, uint64_t *conns3, uint64_t *conns2, float **vars) 73 | { 74 | char fname[fnstrmax+1]; 75 | int timedigits = 4; 76 | uint64_t groupsize, totalsize; 77 | int64_t handle; 78 | int ret; 79 | int bufneeded; 80 | uint64_t i; 81 | uint64_t npoints, cspoints; 82 | uint64_t cnconns3, nconns3, csconns3; 83 | uint64_t cnconns2, nconns2, csconns2; 84 | 85 | /* Set global sizes: we assume all tasks have the same size unstructured data!!! */ 86 | npoints = nfo->cnpoints * nfo->nprocs; /* global # of points */ 87 | cspoints = nfo->cnpoints * nfo->rank; /* local starting points */ 88 | cnconns3 = nfo->cnelems3 * 6; /* 6 connections per 3D prism element */ 89 | nconns3 = cnconns3 * nfo->nprocs; /* global # of 3D connections */ 90 | csconns3 = cnconns3 * nfo->rank; /* local starting 3D connections */ 91 | cnconns2 = nfo->cnelems2 * 3; /* 3 connections per 2D triangle element */ 92 | nconns2 = cnconns2 * nfo->nprocs; /* global # of 2D connections */ 93 | csconns2 = cnconns2 * nfo->rank; /* local starting 2D connections */ 94 | 95 | /* ADIOS group size */ 96 | groupsize = sizeof(int) /*rank*/ + sizeof(int) /*tstep*/ + 97 | sizeof(uint64_t)*9 /*npoints-cselems2*/ + 98 | sizeof(float)*3*nfo->cnpoints /*xpts-zpts*/ + 99 | sizeof(uint64_t)*cnconns3 /*conns3*/ + 100 | sizeof(uint64_t)*cnconns2 /*conns2*/ + 101 | sizeof(float)*nfo->cnpoints*nfo->nvars; /*vars*/ 102 | 103 | /* Allocate buffer large enough for all data to write, if not done already */ 104 | bufneeded = (int)(groupsize/(1024*1024)); 105 | bufneeded += bufneeded/8 + 5; /* Add an extra 12.5% & 5MB to be sure */ 106 | if(nfo->bufallocsize < bufneeded) { 107 | # if ADIOS_VERSION_GE(1,10,0) 108 | adios_set_max_buffer_size(bufneeded); 109 | # else 110 | adios_allocate_buffer(ADIOS_BUFFER_ALLOC_NOW, bufneeded); 111 | # endif 112 | nfo->bufallocsize = bufneeded; 113 | } 114 | 115 | /* Set filename */ 116 | snprintf(fname, fnstrmax, "%s.%0*d.bp", nfo->name, timedigits, tstep); 117 | 118 | /* Open & Write */ 119 | ret = adios_open(&handle, nfo->name, fname, "w", nfo->comm); 120 | if(ret) { 121 | fprintf(stderr, "Error opening ADIOS file: %s\n", fname); 122 | return; 123 | } 124 | # if ADIOS_VERSION_LE(1,9,0) 125 | adios_group_size(handle, groupsize, &totalsize); 126 | # endif 127 | 128 | adios_write(handle, "rank", &nfo->rank); 129 | adios_write(handle, "tstep", &tstep); 130 | adios_write(handle, "npoints", &npoints); 131 | adios_write(handle, "cnpoints", &nfo->cnpoints); 132 | adios_write(handle, "cspoints", &cspoints); 133 | adios_write(handle, "nconns3", &nconns3); 134 | adios_write(handle, "cnconns3", &cnconns3); 135 | adios_write(handle, "csconns3", &csconns3); 136 | adios_write(handle, "nconns2", &nconns2); 137 | adios_write(handle, "cnconns2", &cnconns2); 138 | adios_write(handle, "csconns2", &csconns2); 139 | adios_write(handle, "xpts", xpts); 140 | adios_write(handle, "ypts", ypts); 141 | adios_write(handle, "zpts", zpts); 142 | adios_write(handle, "conns3", conns3); 143 | adios_write(handle, "conns2", conns2); 144 | for(i = 0; i < nfo->nvars; i++) 145 | adios_write(handle, nfo->varnames[i], vars[i]); 146 | 147 | adios_close(handle); 148 | } 149 | 150 | void adiosunstruct_finalize(struct adiosinfo *nfo) 151 | { 152 | free(nfo->varnames); 153 | adios_finalize(nfo->rank); 154 | } 155 | 156 | -------------------------------------------------------------------------------- /unstruct/adiosunstruct.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) DoD HPCMP PETTT. All rights reserved. 3 | * See LICENSE file for details. 4 | */ 5 | 6 | #include 7 | #include 8 | 9 | struct adiosinfo { 10 | char *name; 11 | MPI_Comm comm; 12 | int rank; 13 | int nprocs; 14 | int tsteps; 15 | uint64_t cnpoints; /* number of points in local task */ 16 | uint64_t cnelems3; /* local number of triangular elements in 2D base grid */ 17 | uint64_t cnelems2; /* local number of prism elements in 3D grid */ 18 | 19 | int nvars; 20 | int maxvars; 21 | char **varnames; 22 | 23 | int bufallocsize; 24 | 25 | int64_t gid; 26 | }; 27 | 28 | void adiosunstruct_init(struct adiosinfo *nfo, char *method, char *name, MPI_Comm comm, 29 | int rank, int nprocs, int tsteps, uint64_t nptstask, 30 | uint64_t nelems3, uint64_t nelems2); 31 | 32 | void adiosunstruct_addvar(struct adiosinfo *nfo, char *varname); 33 | 34 | void adiosunstruct_write(struct adiosinfo *nfo, int tstep, float *xpts, float *ypts, 35 | float *zpts, uint64_t *conns3, uint64_t *conns2, float **vars); 36 | 37 | void adiosunstruct_finalize(struct adiosinfo *nfo); 38 | 39 | -------------------------------------------------------------------------------- /unstruct/hdf5unstruct.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) DoD HPCMP PETTT. All rights reserved. 3 | * See LICENSE file for details. 4 | */ 5 | 6 | #include 7 | void writehdf5(char *name, MPI_Comm comm, int tstep, uint64_t npoints, uint64_t nptstask, 8 | float *xpts, float *ypts, float *zpts, uint64_t nelems3, uint64_t *conns3, 9 | uint64_t nelems2, uint64_t *conns2, char *varname, float *data, hsize_t *h5_chunk, int filter_id, size_t cd_nelmts, unsigned int *cd_values); 10 | -------------------------------------------------------------------------------- /unstruct/ncunstruct.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) DoD HPCMP PETTT. All rights reserved. 3 | * See LICENSE file for details. 4 | */ 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | #include 11 | 12 | #include "netcdf.h" 13 | #include "netcdf_par.h" 14 | #include "ncunstruct.h" 15 | 16 | //#define TIMEIO 17 | 18 | static const int fnstrmax = 4095; 19 | 20 | #define NCERR {if(err != NC_NOERR) {printf("(rank %d) Error at line %d: %s\n",rank,__LINE__,nc_strerror(err)); fflush(stdout); MPI_Abort(comm,1);}} 21 | 22 | uint64_t nelems_in[2]; 23 | uint64_t nelems_out[2]; 24 | 25 | void writenc(char *name, MPI_Comm comm, int tstep, uint64_t npoints, 26 | uint64_t nptstask, float *xpts, float *ypts, float *zpts, 27 | uint64_t nelems3, uint64_t *conns3, uint64_t nelems2, 28 | uint64_t *conns2, char *varname, float *data) { 29 | 30 | char dirname[fnstrmax+1]; 31 | char fname[fnstrmax+1]; 32 | char rel_fname[fnstrmax+1]; 33 | char fname_xdmf[fnstrmax+1]; 34 | int rank, nprocs; 35 | int timedigits = 4; 36 | MPI_Info info = MPI_INFO_NULL; 37 | 38 | int conns3id; 39 | int conns2id; 40 | int ncid; 41 | int grid_id; 42 | int varsid; 43 | 44 | int phony_dim_0_id; 45 | int phony_dim_1_id; 46 | int phony_dim_2_id; 47 | int phony_dim_3_id; 48 | 49 | int xyz_ids[3] = {0,0,0}; 50 | 51 | int err = 0; 52 | 53 | size_t start[1] = {0}; 54 | size_t count[1] = {0}; 55 | 56 | size_t dims[1] = {0}; 57 | 58 | 59 | int chunk_pid; 60 | size_t chunk; 61 | 62 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 63 | MPI_Comm_size(MPI_COMM_WORLD, &nprocs); 64 | 65 | /* Make dir for all output and subdir for timestep */ 66 | snprintf(dirname, fnstrmax, "%s.nc.d", name); 67 | mkdir1task(dirname, comm); 68 | snprintf(dirname, fnstrmax, "%s.nc.d/t%0*d.d", name, timedigits, tstep); 69 | mkdir1task(dirname, comm); 70 | 71 | /* Set up MPI info */ 72 | MPI_Info_create(&info); 73 | /* MPI_Info_set(info, "striping_factor", "1"); */ 74 | 75 | chkdir1task(dirname, comm); 76 | 77 | snprintf(fname, fnstrmax, "unstruct.nc.d/t%0*d.d/r.nc", timedigits, tstep); 78 | snprintf(rel_fname, fnstrmax, "t%0*d.d/r.nc", timedigits, tstep); 79 | snprintf(fname_xdmf, fnstrmax, "unstruct.nc.d/t%0*d.d.xmf", timedigits, tstep); 80 | 81 | nelems_in[0] = nelems3 ; 82 | nelems_in[1] = nelems2 ; 83 | 84 | MPI_Reduce( nelems_in, nelems_out, 2, MPI_UNSIGNED_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD ); 85 | 86 | #ifdef TIMEIO 87 | double createfile, prewrite, write, postwrite; /* Timers */ 88 | timer_tick(&createfile, comm, 0); 89 | #endif 90 | 91 | err = nc_create_par(fname,NC_NETCDF4|NC_MPIIO,comm,info,&ncid); NCERR; 92 | 93 | /* Optional grid points */ 94 | if(xpts && ypts && zpts) { 95 | /* Create the dataspace for the dataset. */ 96 | dims[0] = (size_t)npoints; 97 | count[0] =(size_t)nptstask; 98 | //filespace = H5Screate_simple(1, dims, NULL); 99 | 100 | /* Create Grid Group */ 101 | err = nc_def_grp(ncid,"grid points",&grid_id); NCERR; 102 | err = nc_def_dim(grid_id,"phony_dim_0",dims[0],&phony_dim_0_id); NCERR; 103 | err = nc_def_var(grid_id,"x",NC_FLOAT,1,&phony_dim_0_id,&xyz_ids[0]); NCERR; 104 | err = nc_def_var(grid_id,"y",NC_FLOAT,1,&phony_dim_0_id,&xyz_ids[1]); NCERR; 105 | err = nc_def_var(grid_id,"z",NC_FLOAT,1,&phony_dim_0_id,&xyz_ids[2]); NCERR; 106 | 107 | } /* if x,y,z */ 108 | 109 | 110 | /* MSB is it possible that some processors have 0? */ 111 | 112 | /* Optional grid connections, writes a 64-bit 0 if no connections */ 113 | if(conns3 && nelems3) { 114 | 115 | /* Create the dataspace for the dataset. */ 116 | dims[0] = (size_t)nelems_out[0]*6; 117 | err = nc_def_dim(ncid,"phony_dim_3",dims[0],&phony_dim_3_id); NCERR; 118 | 119 | err = nc_def_var(ncid,"conns3",NC_UINT64,1,&phony_dim_3_id,&conns3id); NCERR; 120 | 121 | } /* if conns & nelems3 */ 122 | 123 | /* Optional 2D surface triangle connections, writes a 64-bit 0 if none */ 124 | if(conns2 && nelems2) { 125 | 126 | /* Create the dataspace for the dataset. */ 127 | dims[0] = (size_t)nelems_out[1]*3; 128 | 129 | err = nc_def_dim(ncid,"phony_dim_1",dims[0],&phony_dim_1_id); NCERR; 130 | err = nc_def_var(ncid,"conns2",NC_UINT64,1,&phony_dim_1_id,&conns2id); NCERR; 131 | 132 | } /* if conns2 & nelems2 */ 133 | 134 | /* Optional variable data, starting with number of variables */ 135 | if(data && varname) { 136 | /* Create the dataspace for the dataset. */ 137 | dims[0] = (size_t)npoints; 138 | err = nc_def_dim(ncid,"phony_dim_2",dims[0],&phony_dim_2_id); NCERR; 139 | err = nc_def_var(ncid,"vars",NC_FLOAT,1,&phony_dim_2_id,&varsid); 140 | 141 | } /* if data & varname */ 142 | 143 | MPI_Barrier(comm); 144 | 145 | err = nc_enddef(ncid); NCERR; 146 | 147 | if(xpts && ypts && zpts) { 148 | 149 | err = nc_var_par_access(grid_id,xyz_ids[0],NC_COLLECTIVE); NCERR; 150 | err = nc_var_par_access(grid_id,xyz_ids[1],NC_COLLECTIVE); NCERR; 151 | err = nc_var_par_access(grid_id,xyz_ids[2],NC_COLLECTIVE); NCERR; 152 | 153 | } 154 | 155 | if(conns3 && nelems3) { 156 | err = nc_var_par_access(ncid,conns3id,NC_COLLECTIVE); NCERR; 157 | } 158 | 159 | if(conns2 && nelems2) { 160 | err = nc_var_par_access(ncid,conns2id,NC_COLLECTIVE); NCERR; 161 | } 162 | 163 | if(data && varname) { 164 | err = nc_var_par_access(ncid,varsid,NC_COLLECTIVE); NCERR; 165 | } 166 | 167 | MPI_Barrier(comm); 168 | // } /* Rank == 0 */ 169 | 170 | 171 | 172 | 173 | 174 | #ifdef TIMEIO 175 | timer_tock(&createfile); 176 | timer_collectprintstats(createfile, comm, 0, "CreateFile"); 177 | timer_tick(&prewrite, comm, 0); 178 | #endif 179 | 180 | #ifdef TIMEIO 181 | timer_tock(&prewrite); 182 | timer_collectprintstats(prewrite, comm, 0, "PreWrite"); 183 | 184 | #endif 185 | #ifdef TIMEIO 186 | timer_tick(&write, comm, 0); 187 | #endif 188 | /* Optional grid points */ 189 | if(xpts && ypts && zpts) { 190 | 191 | /* 192 | * Each process defines dataset in memory and writes it to the hyperslab 193 | * in the file. 194 | */ 195 | start[0] =(size_t)(nptstask*rank); 196 | count[0] =(size_t)nptstask; 197 | 198 | err = nc_put_vara_float(grid_id,xyz_ids[0],start,count,xpts); NCERR; 199 | err = nc_put_vara_float(grid_id,xyz_ids[1],start,count,ypts); NCERR; 200 | err = nc_put_vara_float(grid_id,xyz_ids[2],start,count,zpts); NCERR; 201 | 202 | } /* if x,y,z pts */ 203 | 204 | 205 | /* Optional grid connections, writes a 64-bit 0 if no connections */ 206 | if(conns3 && nelems3) { 207 | 208 | /* 209 | * Each process defines dataset in memory and writes it to the hyperslab 210 | * in the file. 211 | */ 212 | start[0] =(size_t)(nelems3*6*rank); 213 | count[0] =(size_t)nelems3*6; 214 | 215 | err = nc_put_vara(ncid,conns3id,start,count,conns3); NCERR; 216 | 217 | } 218 | 219 | /* Optional 2D surface triangle connections, writes a 64-bit 0 if none */ 220 | if(conns2 && nelems2) { 221 | /* 222 | * Each process defines dataset in memory and writes it to the hyperslab 223 | * in the file. 224 | */ 225 | start[0] =(size_t)(nelems2*3*rank); 226 | count[0] =(size_t)nelems2*3; 227 | 228 | err = nc_put_vara(ncid,conns2id,start,count,conns3); NCERR; 229 | } 230 | 231 | /* Optional variable data, starting with number of variables */ 232 | if(data && varname) { 233 | 234 | /* 235 | * Each process defines dataset in memory and writes it to the hyperslab 236 | * in the file. 237 | */ 238 | start[0] =(size_t)(nptstask*rank); 239 | count[0] =(size_t)nptstask; 240 | 241 | err = nc_put_vara_float(ncid,varsid,start,count,data); NCERR; 242 | } 243 | 244 | #ifdef TIMEIO 245 | timer_tock(&write); 246 | timer_collectprintstats(write, comm, 0, "write"); 247 | #endif 248 | 249 | #ifdef TIMEIO 250 | timer_tick(&postwrite, comm, 0); 251 | #endif 252 | 253 | #ifdef TIMEIO 254 | timer_tock(&postwrite); 255 | timer_collectprintstats(postwrite, comm, 0, "PostWrite"); 256 | #endif 257 | 258 | //MPI_Barrier(MPI_COMM_WORLD); 259 | err = nc_close(ncid); 260 | 261 | } 262 | -------------------------------------------------------------------------------- /unstruct/ncunstruct.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) DoD HPCMP PETTT. All rights reserved. 3 | * See LICENSE file for details. 4 | */ 5 | 6 | //void timer_tick(double *, MPI_Comm, int); 7 | //void timer_tock(double *); 8 | //void timer_collectstats(double,MPI_Comm,int,struct timer_statinfo*); 9 | 10 | void writenc(char *name, MPI_Comm comm, int tstep, uint64_t npoints, uint64_t nptstask, 11 | float *xpts, float *ypts, float *zpts, uint64_t nelems3, uint64_t *conns3, 12 | uint64_t nelems2, uint64_t *conns2, char *varname, float *data); 13 | -------------------------------------------------------------------------------- /unstruct/processlog.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Process an output log from unstruct and produce a results table 4 | # + Determines output method from log contents 5 | # + Can also check filename for method 6 | # - Filename must contain .przm., .posix., .mpi., etc (see fname2method) 7 | # - If filename & log contents don't agree, will abort with error 8 | # - If you didn't name file properly (or at all), just gives a warning 9 | # + Each column is a time step 10 | # + Table rows: method, cores, grid points (Million), full file size, time, rate (GB/s) 11 | # + Multiple methods/types can be pasted into Excel or similar for plotting/analysis 12 | 13 | import sys 14 | import re 15 | 16 | fname2method = { ".przm.": "MPI-Indiv", ".posix.": "ADIOS-POSIX", 17 | ".mpi.": "ADIOS-MPI", ".mpilus.": "ADIOS-MPI_Lustre", ".mpiag.": "ADIOS-MPI_Aggr", 18 | ".phdf5.": "ADIOS-PHDF5", ".hdf5.": "HDF5", } 19 | lname2method = { "przm": "MPI-Indiv", "POSIX": "ADIOS-POSIX", 20 | "MPI ": "ADIOS-MPI", "MPI_LUSTRE": "ADIOS-MPI_Lustre", 21 | "MPI_AGGREGATE": "ADIOS-MPI_Aggr", "PHDF5": "ADIOS-PHDF5", 22 | "hdf5": "HDF5" } 23 | 24 | v = [] 25 | totalrate = -1.0 26 | a = 1 27 | if sys.argv[a] == "-a": # produce average rate 28 | totalrate = 0.0 29 | a += 1 30 | 31 | for fname in sys.argv[a:]: 32 | if totalrate > 0.0: totalrate = 0.0 33 | 34 | # Determine method from file name, if it's there 35 | method1key = next((x for x in fname2method.keys() if x in fname), False) 36 | if not method1key: print "WARNING: method not specified in filename" 37 | 38 | f = open(fname) 39 | for l in f: 40 | ls = l.split() 41 | if(len(ls) < 1): continue 42 | if(len(ls) < 2): ls.append("") # Ensure all outer if's can handle ls 43 | 44 | if ls[0][0:4] == "tsk=": 45 | if ls[0][4] == "\"": continue # Skip LSF copy of batch script 46 | 47 | cores = int(ls[0][4:]) 48 | 49 | # Determine method from line; make sure matches one from file name 50 | method2key = next((x for x in lname2method.keys() if re.search(x, l)), False) 51 | if method2key: 52 | if method1key: 53 | if fname2method[method1key] != lname2method[method2key]: 54 | print "Error: method in filename doesn't match method in file" 55 | sys.exit(1) 56 | else: 57 | method = fname2method[method1key] 58 | else: 59 | method = lname2method[method2key] 60 | else: 61 | print "WARNING: method not specified in log file" 62 | if method1key: 63 | method = fname2method[method1key] 64 | else: 65 | print "ERROR: not method specified, cannot proceed" 66 | sys.exit(1) 67 | 68 | elif ls[0] == "Actual" and ls[1] == "points:": 69 | pointsM = long(ls[2]) / 1000000 70 | elif ls[0] == "data" and ls[1] == "size": 71 | sizegb = float(ls[10]) / 1024**3 72 | elif ls[0] == "Output": 73 | outtime = float(ls[11][:-1]) 74 | rate = sizegb/outtime 75 | if totalrate >= 0.0: totalrate += rate 76 | v.append([method, cores, pointsM, sizegb, outtime, rate]) 77 | 78 | print "%-16s %5s %8s %7s %6s %6s" % ("method", "cores", "points*M", "f_GB", "ftime", "fGB/s") 79 | for i in v: 80 | print "%-16s %5d %8s %7.1f %6.2f %6.2f" % tuple(i) 81 | if totalrate >= 0.0: print "Avg rate =", totalrate/len(v) 82 | 83 | -------------------------------------------------------------------------------- /unstruct/przm.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) DoD HPCMP PETTT. All rights reserved. 3 | * See LICENSE file for details. 4 | */ 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | #include 11 | #include "przm.h" 12 | 13 | static const int fnstrmax = 4095; 14 | 15 | void writeprzm(char *name, MPI_Comm comm, int tstep, uint64_t npoints, 16 | float *xpts, float *ypts, float *zpts, uint64_t nelems3, uint64_t *conns3, 17 | uint64_t nelems2, uint64_t *conns2, char *varname, float *data) 18 | { 19 | char dirname[fnstrmax+1]; 20 | char fname[fnstrmax+1]; 21 | char line[fnstrmax+1]; 22 | int rank, nprocs; 23 | int rankdigits; 24 | int timedigits = 4; 25 | FILE *f; 26 | MPI_File mf; 27 | MPI_Status mstat; 28 | MPI_Info info = MPI_INFO_NULL; 29 | int r; 30 | uint64_t *rntris; /* All triangle counts from each task */ 31 | int ret; 32 | 33 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 34 | MPI_Comm_size(MPI_COMM_WORLD, &nprocs); 35 | rankdigits = nprocs > 1 ? (int)(log10(nprocs-1)+1.5) : 1; 36 | 37 | /* Make dir for all output and subdir for timestep */ 38 | snprintf(dirname, fnstrmax, "%s.przm", name); 39 | mkdir1task(dirname, comm); 40 | snprintf(dirname, fnstrmax, "%s.przm/t%0*d.d", name, timedigits, tstep); 41 | mkdir1task(dirname, comm); 42 | 43 | /* Placeholder to create a metadata file, if we decide we need one for reading */ 44 | 45 | /* Set up MPI info */ 46 | MPI_Info_create(&info); 47 | MPI_Info_set(info, "striping_factor", "1"); 48 | 49 | chkdir1task(dirname, comm); 50 | 51 | /* Write files per rank */ 52 | 53 | snprintf(fname, fnstrmax, "%s/r%0*d.dat", dirname, rankdigits, rank); 54 | ret = MPI_File_open(MPI_COMM_SELF, fname, MPI_MODE_WRONLY | MPI_MODE_CREATE, 55 | info, &mf); 56 | if(ret) { 57 | fprintf(stderr, "writeprzm error: could not open %s\n", fname); 58 | MPI_Abort(comm, 1); 59 | } 60 | 61 | MPI_File_write(mf, &npoints, 1, MPI_UNSIGNED_LONG_LONG, &mstat); 62 | 63 | /* Optional grid points */ 64 | if(xpts && ypts && zpts) { 65 | uint32_t hasgrid = 1; 66 | MPI_File_write(mf, &hasgrid, 1, MPI_UNSIGNED, &mstat); 67 | MPI_File_write(mf, xpts, npoints, MPI_FLOAT, &mstat); 68 | MPI_File_write(mf, ypts, npoints, MPI_FLOAT, &mstat); 69 | MPI_File_write(mf, zpts, npoints, MPI_FLOAT, &mstat); 70 | } else { 71 | uint32_t hasgrid = 0; 72 | MPI_File_write(mf, &hasgrid, 1, MPI_UNSIGNED, &mstat); 73 | } 74 | 75 | /* Optional grid connections, writes a 64-bit 0 if no connections */ 76 | if(conns3 && nelems3) { 77 | MPI_File_write(mf, &nelems3, 1, MPI_UNSIGNED_LONG_LONG, &mstat); 78 | MPI_File_write(mf, conns3, nelems3*6, MPI_UNSIGNED_LONG_LONG, &mstat); 79 | } else { 80 | uint64_t hasconn = 0; 81 | MPI_File_write(mf, &hasconn, 1, MPI_UNSIGNED_LONG_LONG, &mstat); 82 | } 83 | 84 | /* Optional 2D surface triangle connections, writes a 64-bit 0 if none */ 85 | if(conns2 && nelems2) { 86 | MPI_File_write(mf, &nelems2, 1, MPI_UNSIGNED_LONG_LONG, &mstat); 87 | MPI_File_write(mf, conns2, nelems2*3, MPI_UNSIGNED_LONG_LONG, &mstat); 88 | } else { 89 | uint64_t hasconn2 = 0; 90 | MPI_File_write(mf, &hasconn2, 1, MPI_UNSIGNED_LONG_LONG, &mstat); 91 | } 92 | 93 | /* Optional variable data, starting with number of variables */ 94 | if(data && varname) { 95 | uint32_t nvars = 1; 96 | MPI_File_write(mf, &nvars, 1, MPI_UNSIGNED, &mstat); 97 | MPI_File_write(mf, data, npoints, MPI_FLOAT, &mstat); 98 | } else { 99 | uint32_t nvars = 0; 100 | MPI_File_write(mf, &nvars, 1, MPI_UNSIGNED, &mstat); 101 | } 102 | 103 | MPI_File_close(&mf); 104 | } 105 | 106 | -------------------------------------------------------------------------------- /unstruct/przm.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) DoD HPCMP PETTT. All rights reserved. 3 | * See LICENSE file for details. 4 | */ 5 | 6 | void writeprzm(char *name, MPI_Comm comm, int tstep, uint64_t npoints, 7 | float *xpts, float *ypts, float *zpts, uint64_t nelems3, uint64_t *conns3, 8 | uint64_t nelems2, uint64_t *conns2, char *varname, float *data); 9 | 10 | -------------------------------------------------------------------------------- /unstruct/read.src.py: -------------------------------------------------------------------------------- 1 | # ParaView Programmable Source: vtkUnstructuredGrid 2 | # Read a przm output collection 3 | # For now, reads parallel output serially 4 | # 5 | # Copyright (c) DoD HPCMP PETTT. All rights reserved. 6 | # See LICENSE file for details. 7 | 8 | from paraview import vtk 9 | import numpy as np 10 | from glob import glob 11 | przmname = "/Users/sean/sync/wrk/ace4/miniIO/unstruct/unstruct.przm" 12 | tstep = 0 13 | 14 | tfiles = glob(przmname+"/t*.d") 15 | rfiles = glob(tfiles[tstep]+"/r*.dat") 16 | nr = len(rfiles) 17 | 18 | f = [None]*nr; # One file handle per rank file 19 | hasGrid = False 20 | pts = vtk.vtkPoints() 21 | 22 | for r,rfile in enumerate(rfiles): 23 | f[r] = open(rfile) 24 | 25 | numPoints = np.fromfile(f[r], np.uint64, 1)[0] 26 | hasGrid = np.fromfile(f[r], np.uint32, 1)[0] 27 | if hasGrid: 28 | xpts = np.fromfile(f[r], np.float32, numPoints) 29 | ypts = np.fromfile(f[r], np.float32, numPoints) 30 | zpts = np.fromfile(f[r], np.float32, numPoints) 31 | for i in range(len(xpts)): 32 | pts.InsertNextPoint(xpts[i], ypts[i], zpts[i]) 33 | if hasGrid: output.SetPoints(pts) 34 | 35 | needalloc = True 36 | 37 | for r in range(nr): 38 | numCells = np.fromfile(f[r], np.uint64, 1)[0] 39 | numConns = numCells * np.uint64(6) 40 | if numCells: 41 | if needalloc: 42 | output.Allocate(numCells * np.uint64(nr), 1000) 43 | needalloc = False 44 | conns = np.fromfile(f[r], np.uint64, numConns) 45 | for i in range(numCells): 46 | pointIds = vtk.vtkIdList() 47 | for j in range(6): 48 | pointIds.InsertId(j, conns[i*6+j]) 49 | output.InsertNextCell(13, pointIds) 50 | 51 | tris = [] 52 | 53 | for r in range(nr): 54 | numTriPts = int(np.fromfile(f[r], np.uint64, 1)[0]) * 3 55 | if numTriPts: 56 | tris = np.concatenate((tris, np.fromfile(f[r], np.uint64, numTriPts))) 57 | #print tris.shape 58 | if type(tris) is not list: 59 | sdata = np.zeros(numPoints*np.uint64(nr), dtype=np.uint8) 60 | sdata[np.uint32(tris)] = 1 61 | output.PointData.append(sdata, "surfflag") 62 | 63 | data = [] 64 | 65 | for r in range(nr): 66 | hasData = np.fromfile(f[r], np.uint32, 1)[0] 67 | if hasData: 68 | data = np.concatenate((data, np.fromfile(f[r], np.float32, numPoints))) 69 | f[r].close() 70 | if type(data) is not list: 71 | output.PointData.append(data, "noise") 72 | 73 | -------------------------------------------------------------------------------- /unstruct/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Script for testing unstruct 4 | 5 | PTS=512 6 | 7 | 8 | NT=2 9 | 10 | output="--hdf5" 11 | mpirun -np 8 ./unstruct --pointspertask $PTS --tsteps $NT $output 12 | 13 | output="--hdf5 --hdf5_chunk 18 --hdf5_compress gzip,9" 14 | mpirun -np 8 ./unstruct --pointspertask $PTS --tsteps $NT $output 15 | 16 | 17 | --------------------------------------------------------------------------------