├── src ├── problem │ ├── advection_test │ │ ├── problemdata.h │ │ ├── config.h.in │ │ └── CMakeLists.txt │ ├── anisotropic_conduction │ │ ├── problemdata.h │ │ ├── config.h.in │ │ └── CMakeLists.txt │ ├── magnetized_explosion │ │ ├── problemdata.h │ │ ├── config.h.in │ │ └── CMakeLists.txt │ ├── magnetized_field_loop_advection │ │ ├── problemdata.h │ │ ├── config.h.in │ │ └── CMakeLists.txt │ ├── CMakeLists.txt │ ├── problem.hpp │ ├── bondi_viscous │ │ ├── problemdata.h │ │ ├── runtimeoptions │ │ ├── bondi_viscous.h │ │ ├── config.h.in │ │ └── CMakeLists.txt │ ├── atmosphere │ │ ├── problemdata.h │ │ ├── config.h.in │ │ └── CMakeLists.txt │ ├── torus │ │ ├── torus.hpp │ │ └── params.cpp │ ├── orzag_tang │ │ ├── problem.cpp │ │ ├── config.h.in │ │ └── params.cpp │ ├── shock_tests │ │ └── params.cpp │ ├── performance_testing │ │ ├── params.cpp │ │ └── problem.cpp │ ├── linear_modes │ │ └── params.cpp │ ├── buoyancy_instabilities │ │ └── params.cpp │ └── bondi_inflow │ │ └── problem.cpp ├── cmake │ └── modules │ │ ├── ReplicatePythonSourceTree.cmake │ │ ├── README │ │ ├── CorrectWindowsPaths.cmake │ │ ├── FindFFTW.cmake │ │ ├── FindCubature.cmake │ │ ├── FindCuba.cmake │ │ ├── FindNumPy.cmake │ │ ├── FindLibXML2.cmake │ │ ├── LICENSE │ │ ├── FindCython.cmake │ │ ├── FindOpenCL.cmake │ │ ├── FindNetCDF.cmake │ │ ├── FindYAML.cmake │ │ ├── ResolveCompilerPaths.cmake │ │ ├── FindGSL.cmake │ │ ├── FindPackageMultipass.cmake │ │ └── FindLAPACKE.cmake ├── grid │ ├── CMakeLists.txt │ ├── gridPy.pxd │ ├── conftest.py │ ├── gridHeaders.pxd │ └── grid.hpp ├── geometry │ ├── CMakeLists.txt │ ├── geometryPy.pxd │ ├── geometryHeaders.pxd │ ├── conftest.py │ ├── geometry.hpp │ ├── geometryPy.pyx │ └── CoordinateChangeFunctionsArray.hpp ├── physics │ ├── CMakeLists.txt │ ├── physicsPy.pxd │ ├── physicsHeaders.pxd │ ├── conftest.py │ ├── physicsPy.pyx │ ├── test_physics.py │ └── physics.hpp ├── grim.hpp ├── boundary │ ├── CMakeLists.txt │ ├── boundaryHeaders.pxd │ ├── boundary.hpp │ ├── conftest.py │ └── boundaryPy.pyx ├── reconstruction │ ├── CMakeLists.txt │ ├── reconstructionHeaders.pxd │ ├── reconstructionPy.pyx │ ├── reconstruction.cpp │ ├── reconstruction.hpp │ ├── minmod.cpp │ └── ppm.cpp ├── timestepper │ ├── CMakeLists.txt │ ├── timeStepperPy.pxd │ ├── conftest.py │ ├── timestepperHeaders.pxd │ ├── constrainedtransport.cpp │ ├── timestepper.hpp │ └── test_CT.py ├── timer.hpp ├── grim.cpp └── params.hpp ├── make_summit.sh ├── README.md ├── .gitignore └── run_summit.bsub /src/problem/advection_test/problemdata.h: -------------------------------------------------------------------------------- 1 | 2 | /* Problem specific data structure. */ 3 | 4 | struct problemData 5 | { 6 | }; 7 | 8 | -------------------------------------------------------------------------------- /src/problem/anisotropic_conduction/problemdata.h: -------------------------------------------------------------------------------- 1 | 2 | /* Problem specific data structure. */ 3 | 4 | struct problemData 5 | { 6 | }; 7 | 8 | -------------------------------------------------------------------------------- /src/problem/magnetized_explosion/problemdata.h: -------------------------------------------------------------------------------- 1 | 2 | /* Problem specific data structure. */ 3 | 4 | struct problemData 5 | { 6 | }; 7 | 8 | -------------------------------------------------------------------------------- /src/problem/magnetized_field_loop_advection/problemdata.h: -------------------------------------------------------------------------------- 1 | 2 | /* Problem specific data structure. */ 3 | 4 | struct problemData 5 | { 6 | }; 7 | 8 | -------------------------------------------------------------------------------- /src/problem/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_library(problem ${PROBLEM_DIR}/problem.cpp) 2 | add_library(params ${PROBLEM_DIR}/params.cpp) 3 | target_link_libraries(problem params geometry) 4 | -------------------------------------------------------------------------------- /src/problem/problem.hpp: -------------------------------------------------------------------------------- 1 | #ifndef GRIM_PROBLEM_H_ 2 | #define GRIM_PROBLEM_H_ 3 | 4 | #include "../timestepper/timestepper.hpp" 5 | #include "../physics/physics.hpp" 6 | 7 | #endif /* GRIM_PROBLEM_H_ */ 8 | -------------------------------------------------------------------------------- /src/problem/bondi_viscous/problemdata.h: -------------------------------------------------------------------------------- 1 | 2 | /* Problem specific data structure. */ 3 | 4 | struct problemData 5 | { 6 | REAL primVarsLeftEdge[NG][DOF]; 7 | REAL primVarsRightEdge[NG][DOF]; 8 | }; 9 | 10 | -------------------------------------------------------------------------------- /src/problem/bondi_viscous/runtimeoptions: -------------------------------------------------------------------------------- 1 | -snes_monitor 2 | -snes_converged_reason 3 | -snes_max_it 3 4 | -snes_lag_jacobian 10 5 | -pc_type lu 6 | -ksp_type preonly 7 | -pc_factor_shift_type NONZERO 8 | -pc_factor_shift_amount 1. 9 | -------------------------------------------------------------------------------- /src/problem/bondi_viscous/bondi_viscous.h: -------------------------------------------------------------------------------- 1 | #ifndef GRIM_TORUS_2D_H_ 2 | #define GRIM_TORUS_2D_H_ 3 | 4 | void inflowCheck(const struct gridZone zone[ARRAY_ARGS 1], 5 | REAL primTile[ARRAY_ARGS TILE_SIZE]); 6 | #endif 7 | -------------------------------------------------------------------------------- /src/cmake/modules/ReplicatePythonSourceTree.cmake: -------------------------------------------------------------------------------- 1 | # Note: when executed in the build dir, then CMAKE_CURRENT_SOURCE_DIR is the 2 | # build dir. 3 | file( COPY setup.py src test bin DESTINATION "${CMAKE_ARGV3}" 4 | FILES_MATCHING PATTERN "*.py" ) 5 | -------------------------------------------------------------------------------- /src/grid/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_library(grid grid.cpp grid.hpp) 2 | 3 | set_source_files_properties(gridPy.pyx PROPERTIES CYTHON_IS_CXX TRUE) 4 | 5 | cython_add_module(gridPy gridPy.pyx) 6 | target_link_libraries(gridPy grid ${PETSC_LIBRARIES} ${ArrayFire_LIBRARIES}) 7 | -------------------------------------------------------------------------------- /src/problem/atmosphere/problemdata.h: -------------------------------------------------------------------------------- 1 | #include "../../inputs.h" 2 | #include "../../physics/physics.h" 3 | 4 | /* Problem specific data structure. */ 5 | 6 | struct problemData 7 | { 8 | REAL primVarsLeftEdge[NG][DOF]; 9 | REAL primVarsRightEdge[NG][DOF]; 10 | }; 11 | 12 | -------------------------------------------------------------------------------- /src/geometry/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_library(geometry geometry.cpp geometry.hpp) 2 | target_link_libraries(geometry grid) 3 | 4 | set_source_files_properties(geometryPy.pyx PROPERTIES CYTHON_IS_CXX TRUE) 5 | 6 | cython_add_module(geometryPy geometryPy.pyx) 7 | target_link_libraries(geometryPy geometry params ${ArrayFire_LIBRARIES}) 8 | -------------------------------------------------------------------------------- /src/physics/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_library(physics physics.cpp physics.hpp riemannsolver.cpp tetrads.cpp) 2 | target_link_libraries(physics reconstruction grid geometry problem) 3 | 4 | set_source_files_properties(physicsPy.pyx PROPERTIES CYTHON_IS_CXX TRUE) 5 | cython_add_module(physicsPy physicsPy.pyx) 6 | target_link_libraries(physicsPy physics grid geometry ${ArrayFire_LIBRARIES}) 7 | -------------------------------------------------------------------------------- /src/grim.hpp: -------------------------------------------------------------------------------- 1 | #ifndef GRIM_H_ 2 | #define GRIM_H_ 3 | 4 | //#include 5 | #include "params.hpp" 6 | #include "grid/grid.hpp" 7 | #include "geometry/geometry.hpp" 8 | #include "physics/physics.hpp" 9 | #include "timestepper/timestepper.hpp" 10 | 11 | static const char help[] = 12 | "GRIM -- General Relativistic Implicit Magnetohydrodynamics"; 13 | 14 | #endif /* GRIM_H_ */ 15 | -------------------------------------------------------------------------------- /src/boundary/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_library(boundary boundary.cpp boundary.hpp) 2 | target_link_libraries(boundary grid) 3 | 4 | set_source_files_properties(boundaryPy.pyx 5 | PROPERTIES CYTHON_IS_CXX TRUE) 6 | 7 | cython_add_module(boundaryPy boundaryPy.pyx) 8 | 9 | target_link_libraries(boundaryPy boundary 10 | ${ArrayFire_LIBRARIES} 11 | ) 12 | -------------------------------------------------------------------------------- /src/physics/physicsPy.pxd: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | cimport numpy as np 3 | from physicsHeaders cimport fluidElement 4 | from gridPy cimport gridPy, coordinatesGridPy 5 | 6 | cdef class fluidElementPy(object): 7 | cdef fluidElement *elemPtr 8 | cdef setElemPtr(self, fluidElement *elemPtr) 9 | cdef fluidElement* getElemPtr(self, fluidElement *elemPtr) 10 | cdef int usingExternalPtr 11 | 12 | @staticmethod 13 | cdef createFluidElementPyFromElemPtr(fluidElement *elemPtr) 14 | 15 | -------------------------------------------------------------------------------- /src/cmake/modules/README: -------------------------------------------------------------------------------- 1 | This folder is directly taken from https://github.com/jedbrown/cmake-modules 2 | Blame Jed Brown if something doesn't work. 3 | 4 | Original README follows: 5 | 6 | A collection of CMake modules, these can mostly be used independently. 7 | 8 | The utilities for writing robust Find* modules might be useful until 9 | CMake takes static libraries and multiple active configurations 10 | seriously. 11 | 12 | http://www.cmake.org/Wiki/CMake:Improving_Find*_Modules 13 | -------------------------------------------------------------------------------- /src/reconstruction/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_library(reconstruction reconstruction.hpp minmod.cpp weno5.cpp ppm.cpp 2 | reconstruction.cpp) 3 | target_link_libraries(reconstruction ${ArrayFire_LIBRARIES}) 4 | 5 | set_source_files_properties(reconstructionPy.pyx 6 | PROPERTIES CYTHON_IS_CXX TRUE) 7 | 8 | cython_add_module(reconstructionPy 9 | reconstructionPy.pyx 10 | ) 11 | 12 | target_link_libraries(reconstructionPy reconstruction params) 13 | -------------------------------------------------------------------------------- /src/cmake/modules/CorrectWindowsPaths.cmake: -------------------------------------------------------------------------------- 1 | # CorrectWindowsPaths - this module defines one macro 2 | # 3 | # CONVERT_CYGWIN_PATH( PATH ) 4 | # This uses the command cygpath (provided by cygwin) to convert 5 | # unix-style paths into paths useable by cmake on windows 6 | 7 | macro (CONVERT_CYGWIN_PATH _path) 8 | if (WIN32) 9 | EXECUTE_PROCESS(COMMAND cygpath.exe -m ${${_path}} 10 | OUTPUT_VARIABLE ${_path}) 11 | string (STRIP ${${_path}} ${_path}) 12 | endif (WIN32) 13 | endmacro (CONVERT_CYGWIN_PATH) 14 | 15 | -------------------------------------------------------------------------------- /src/timestepper/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_library(timestepper timestepper.cpp timestepper.hpp timestep.cpp 2 | fvmfluxes.cpp residual.cpp solve.cpp constrainedtransport.cpp) 3 | target_link_libraries(timestepper geometry grid physics) 4 | 5 | set_source_files_properties(timeStepperPy.pyx PROPERTIES CYTHON_IS_CXX TRUE) 6 | cython_add_module(timeStepperPy timeStepperPy.pyx) 7 | target_link_libraries(timeStepperPy timestepper problem 8 | reconstruction boundary ${PETSC_LIBRARIES} 9 | ) 10 | -------------------------------------------------------------------------------- /src/physics/physicsHeaders.pxd: -------------------------------------------------------------------------------- 1 | from gridHeaders cimport grid, coordinatesGrid 2 | from geometryHeaders cimport geometry 3 | 4 | cdef extern from "physics.hpp": 5 | cdef cppclass fluidElement: 6 | fluidElement(const grid &prim, 7 | const geometry &geom, 8 | int &numReads, 9 | int &numWrites 10 | ) 11 | void computeFluxes(const int direction, 12 | grid &flux, 13 | int &numReads, 14 | int &numWrites 15 | ) 16 | -------------------------------------------------------------------------------- /src/geometry/geometryPy.pxd: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | cimport numpy as np 3 | from geometryHeaders cimport geometry 4 | 5 | cdef class geometryPy(object): 6 | cdef geometry *geometryPtr 7 | cdef int usingExternalPtr 8 | cdef np.ndarray gCov 9 | cdef np.ndarray gCon 10 | cdef np.ndarray g 11 | cdef np.ndarray alpha 12 | cdef np.ndarray gammaUpDownDown 13 | cdef np.ndarray xCoords 14 | cdef geometry* getGeometryPtr(self) 15 | cdef setGeometryPtr(self, geometry *geometryPtr) 16 | 17 | @staticmethod 18 | cdef createGeometryPyFromGeometryPtr(geometry *geometryPtr) 19 | -------------------------------------------------------------------------------- /src/boundary/boundaryHeaders.pxd: -------------------------------------------------------------------------------- 1 | from gridHeaders cimport grid 2 | 3 | cdef extern from "boundary.hpp": 4 | cdef enum: 5 | BOUNDARIES_PERIODIC "boundaries::PERIODIC" 6 | BOUNDARIES_MIRROR "boundaries::MIRROR" 7 | BOUNDARIES_OUTFLOW "boundaries::OUTFLOW" 8 | BOUNDARIES_DIRICHLET "boundaries::DIRICHLET" 9 | 10 | cdef extern from "boundary.hpp" namespace "boundaries": 11 | void applyBoundaryConditions(const int boundaryLeft, const int boundaryRight, 12 | const int boundaryTop, const int boundaryBottom, 13 | const int boundaryFront, const int boundaryBack, 14 | grid &prim 15 | ) 16 | -------------------------------------------------------------------------------- /src/cmake/modules/FindFFTW.cmake: -------------------------------------------------------------------------------- 1 | # - Find FFTW 2 | # Find the native FFTW includes and library 3 | # 4 | # FFTW_INCLUDES - where to find fftw3.h 5 | # FFTW_LIBRARIES - List of libraries when using FFTW. 6 | # FFTW_FOUND - True if FFTW found. 7 | 8 | if (FFTW_INCLUDES) 9 | # Already in cache, be silent 10 | set (FFTW_FIND_QUIETLY TRUE) 11 | endif (FFTW_INCLUDES) 12 | 13 | find_path (FFTW_INCLUDES fftw3.h) 14 | 15 | find_library (FFTW_LIBRARIES NAMES fftw3) 16 | 17 | # handle the QUIETLY and REQUIRED arguments and set FFTW_FOUND to TRUE if 18 | # all listed variables are TRUE 19 | include (FindPackageHandleStandardArgs) 20 | find_package_handle_standard_args (FFTW DEFAULT_MSG FFTW_LIBRARIES FFTW_INCLUDES) 21 | 22 | mark_as_advanced (FFTW_LIBRARIES FFTW_INCLUDES) 23 | -------------------------------------------------------------------------------- /src/boundary/boundary.hpp: -------------------------------------------------------------------------------- 1 | #ifndef GRIM_BOUNDARY_H_ 2 | #define GRIM_BOUNDARY_H_ 3 | 4 | #include "../params.hpp" 5 | #include "../grid/grid.hpp" 6 | 7 | namespace boundaries 8 | { 9 | void applyBoundaryConditions(const int boundaryLeft, const int boundaryRight, 10 | const int boundaryTop, const int boundaryBottom, 11 | const int boundaryFront, const int boundaryBack, 12 | grid &prim 13 | ); 14 | } 15 | /* Periodic boundary conditions are handled by Petsc since data needs to be 16 | * copied if running on more than 1 MPI node. Petsc does all that and puts the 17 | * appropriate data in the local array. Simply copy that */ 18 | 19 | #endif /* GRIM_BOUNDARY_H_ */ 20 | -------------------------------------------------------------------------------- /make_summit.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Compile grim, assuming we're on Summit and the dependencies are available 4 | # USE: 5 | # ./make.sh [clean] 6 | 7 | NPROC= 8 | if [[ $(hostname) == "login"* ]]; then 9 | NPROC=16 10 | fi 11 | 12 | if [ ! -d build ]; then 13 | echo "Run \"./make_summit.sh clean\" first!" 14 | exit 15 | fi 16 | 17 | module load gcc cuda/11.4 petsc boost python 18 | 19 | module list 20 | 21 | if [[ "$*" == *"clean"* ]]; then 22 | rm -rf build 23 | mkdir build 24 | cd build 25 | cmake -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx \ 26 | -DPETSC_EXECUTABLE_RUNS=1 \ 27 | -DArrayFire_ROOT_DIR=/gpfs/alpine/proj-shared/ast171/libs/arrayfire-cuda114 \ 28 | ../src 29 | cd .. 30 | fi 31 | 32 | cd build 33 | make -j$NPROC 34 | cp grim .. 35 | -------------------------------------------------------------------------------- /src/timestepper/timeStepperPy.pxd: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | cimport numpy as np 3 | from timestepperHeaders cimport timeStepper 4 | from gridPy cimport gridPy, coordinatesGridPy 5 | from geometryPy cimport geometryPy 6 | from physicsPy cimport fluidElementPy 7 | 8 | cdef class timeStepperPy(object): 9 | cdef timeStepper *timeStepperPtr 10 | cdef coordinatesGridPy XCoords 11 | cdef geometryPy geomCenter 12 | cdef geometryPy geomLeft, geomRight 13 | cdef geometryPy geomTop, geomBottom 14 | cdef geometryPy geomFront, geomBack 15 | cdef gridPy prim, primOld, primHalfStep 16 | cdef gridPy fluxesX1, fluxesX2, fluxesX3 17 | cdef gridPy divFluxes 18 | cdef gridPy emfX1, emfX2, emfX3 19 | cdef gridPy sourcesExplicit 20 | cdef gridPy divB 21 | cdef fluidElementPy elem, elemOld, elemHalfStep 22 | -------------------------------------------------------------------------------- /src/grid/gridPy.pxd: -------------------------------------------------------------------------------- 1 | from gridHeaders cimport grid, coordinatesGrid 2 | 3 | # Only the declarations here. Definitions in gridPy.pyx. This split is needed so 4 | # that we can use share the gridPy module across different Cython modules using 5 | # "from gridPy cimport gridPy". 6 | # See "Sharing Extension Types" in 7 | # http://docs.cython.org/src/userguide/sharing_declarations.html 8 | cdef class gridPy(object): 9 | cdef grid *gridPtr 10 | cdef grid* getGridPtr(self) 11 | cdef setGridPtr(self, grid *gridPtr) 12 | cdef int usingExternalPtr 13 | 14 | @staticmethod 15 | cdef createGridPyFromGridPtr(grid *gridPtr) 16 | 17 | cdef class coordinatesGridPy(object): 18 | cdef coordinatesGrid *coordGridPtr 19 | cdef coordinatesGrid* getGridPtr(self) 20 | cdef setGridPtr(self, coordinatesGrid *coordGridPtr) 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | grim 2 | ==== 3 | 4 | General Relativistic Implicit Magnetohydrodynamics 5 | 6 | Instructions to compile PETSc on UIUC campus cluster: 7 | 8 | 1) Get L1 cacheline size: 9 | 10 | $ getconf LEVEL1_DCACHE_LINESIZE 11 | 12 | 2) Get L1 cache size 13 | 14 | either 15 | 16 | $ grep . /sys/devices/system/cpu/cpu0/cache/index*/* 17 | 18 | or 19 | 20 | $ lstopo-no-graphics 21 | 22 | 3) To get L1 cache associativity: 23 | 24 | $ grep . /sys/devices/system/cpu/cpu0/cache/index*/* 25 | 26 | 4) Compiling petsc 27 | ./configure --prefix=/home/manic/petsc_optimized/ --with-debugging=0 COPTFLAGS='-O3 -qopt-report=5 -qopt-report-phase=vec -xhost' CXXOPTFLAGS='-O3 -qopt-report=5 -qopt-report-phase=vec -xhost' --with-hdf5=1 --with-clean=1 --with-mpi-dir=/usr/local/mpi/openmpi-1.8.4-intel-15.0/ --with-memalign=64 --known-level1-dcache-size=32768 --known-level1-dcache-linesize=64 --known-level1-dcache-assoc=8 28 | -------------------------------------------------------------------------------- /src/reconstruction/reconstructionHeaders.pxd: -------------------------------------------------------------------------------- 1 | from gridHeaders cimport grid 2 | 3 | cdef extern from "reconstruction.hpp" namespace "reconstruction": 4 | void reconstruct(const grid &prim, 5 | const int dir, 6 | grid &primLeft, 7 | grid &primRight, 8 | int &numReads, 9 | int &numWrites 10 | ) 11 | void reconstructMM(const grid &prim, 12 | const int dir, 13 | grid &primLeft, 14 | grid &primRight, 15 | int &numReads, 16 | int &numWrites 17 | ) 18 | void reconstructWENO5(const grid &prim, 19 | const int dir, 20 | grid &primLeft, 21 | grid &primRight, 22 | int &numReads, 23 | int &numWrites 24 | ) 25 | -------------------------------------------------------------------------------- /src/grid/conftest.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import pytest 3 | 4 | def pytest_addoption(parser): 5 | parser.addoption("--build_path", action="store", default=None, 6 | help='set build directory path' 7 | ) 8 | parser.addoption("--N1", action="store", default=64, 9 | help='grid zones in X1' 10 | ) 11 | parser.addoption("--N2", action="store", default=64, 12 | help='grid zones in X2' 13 | ) 14 | parser.addoption("--N3", action="store", default=64, 15 | help='grid zones in X3' 16 | ) 17 | parser.addoption("--dim", action="store", default=64, 18 | help='grid dimension' 19 | ) 20 | 21 | 22 | def pytest_configure(config): 23 | buildPath = config.getvalue('build_path') 24 | gridPath = buildPath + '/grid/' 25 | sys.path.append(gridPath) 26 | -------------------------------------------------------------------------------- /src/problem/torus/torus.hpp: -------------------------------------------------------------------------------- 1 | #ifndef GRIM_TORUS_HPP 2 | #define GRIM_TORUS_HPP 3 | 4 | #include "../problem.hpp" 5 | 6 | /* Internal functions */ 7 | double lFishboneMoncrief(double a, double r, double theta); 8 | 9 | double lnOfhTerm1(double a, 10 | double r, double theta, 11 | double l); 12 | 13 | double lnOfhTerm2(double a, 14 | double r, double theta, 15 | double l); 16 | 17 | double lnOfhTerm3(double a, 18 | double r, double theta, 19 | double l); 20 | 21 | double computeLnOfh(double a, double r, double theta); 22 | 23 | double computeDelta(double a, double r, double theta); 24 | 25 | double computeSigma(double a, double r, double theta); 26 | 27 | double computeA(double a, double r, double theta); 28 | 29 | void applyFloor(grid* prim, fluidElement* elem, geometry* geom, int &numReads,int &numWrites); 30 | 31 | #endif 32 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled source # 2 | ################### 3 | *.com 4 | *.class 5 | *.dll 6 | *.dylib 7 | *.exe 8 | *.o 9 | *.ko 10 | *.so 11 | *.so.* 12 | *.obj 13 | *.elf 14 | *.pyo 15 | *.pyc 16 | grim 17 | 18 | # Libraries # 19 | ############# 20 | *.lib 21 | *.a 22 | *.la 23 | *.lo 24 | 25 | # Packages # 26 | ############ 27 | # it's better to unpack these files and commit the raw source 28 | # git has its own built in compression methods 29 | *.7z 30 | *.dmg 31 | *.gz 32 | *.iso 33 | *.jar 34 | *.rar 35 | *.tar 36 | *.zip 37 | 38 | # Logs and databases # 39 | ###################### 40 | *.log 41 | *.sql 42 | *.sqlite 43 | 44 | # OS generated files # 45 | ###################### 46 | .DS_Store 47 | .DS_Store? 48 | ._* 49 | .Spotlight-V100 50 | .Trashes 51 | ehthumbs.db 52 | Thumbs.db 53 | 54 | # Editor files # 55 | ################ 56 | *.bak 57 | *.swp 58 | *~ 59 | 60 | # build directory # 61 | ################### 62 | build/ 63 | -------------------------------------------------------------------------------- /src/geometry/geometryHeaders.pxd: -------------------------------------------------------------------------------- 1 | from gridHeaders cimport grid, coordinatesGrid 2 | 3 | cdef extern from "geometry.hpp": 4 | cdef enum: 5 | METRICS_MINKOWSKI "metrics::MINKOWSKI" 6 | METRICS_MODIFIED_KERR_SCHILD "metrics::MODIFIED_KERR_SCHILD" 7 | 8 | cdef cppclass geometry: 9 | geometry(const int metric, 10 | const double blackHoleSpin, 11 | const double hSlope, 12 | const coordinatesGrid &XCoordsGrid 13 | ) 14 | int N1, N2, N3, dim, numGhost 15 | int metric 16 | double blackHoleSpin 17 | double hSlope 18 | 19 | void computeConnectionCoeffs() 20 | 21 | grid *gCovGrid 22 | grid *gConGrid 23 | grid *gGrid 24 | grid *alphaGrid 25 | grid *gammaUpDownDownGrid 26 | grid *xCoordsGrid 27 | 28 | void setgConGrid() 29 | void setgCovGrid() 30 | void setgGrid() 31 | void setalphaGrid() 32 | void setgammaUpDownDownGrid() 33 | void setxCoordsGrid() 34 | -------------------------------------------------------------------------------- /src/boundary/conftest.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import pytest 3 | 4 | def pytest_addoption(parser): 5 | parser.addoption("--build_path", action="store", default=None, 6 | help='set build directory path' 7 | ) 8 | parser.addoption("--N1", action="store", default=64, 9 | help='grid zones in X1' 10 | ) 11 | parser.addoption("--N2", action="store", default=64, 12 | help='grid zones in X2' 13 | ) 14 | parser.addoption("--N3", action="store", default=64, 15 | help='grid zones in X3' 16 | ) 17 | parser.addoption("--dim", action="store", default=64, 18 | help='grid dimension' 19 | ) 20 | 21 | 22 | def pytest_configure(config): 23 | buildPath = config.getvalue('build_path') 24 | gridPath = buildPath + '/grid/' 25 | boundaryPath = buildPath + '/boundary/' 26 | sys.path.append(gridPath) 27 | sys.path.append(boundaryPath) 28 | -------------------------------------------------------------------------------- /src/cmake/modules/FindCubature.cmake: -------------------------------------------------------------------------------- 1 | # - Try to find Cubature source code and headers. 2 | # 3 | # Usage of this module as follows: 4 | # 5 | # find_package(Cubature) 6 | # 7 | # Variables used by this module, they can change the default behaviour and need 8 | # to be set before calling find_package: 9 | # 10 | # CUBATURE_DIR Set this variable to the root installation of 11 | # CUBATURE if the module has problems finding 12 | # the proper installation path. 13 | # 14 | # Variables defined by this module: 15 | # 16 | # CUBATURE_FOUND System has Cubature headers 17 | # CUBATURE_INCLUDE_DIR The location of Cubature headers 18 | 19 | find_path(CUBATURE_DIR 20 | NAMES cubature.h 21 | ) 22 | 23 | find_path(CUBATURE_INCLUDE_DIR 24 | NAMES cubature.h 25 | HINTS ${CUBATURE_DIR} 26 | ) 27 | 28 | include(FindPackageHandleStandardArgs) 29 | find_package_handle_standard_args(CUBATURE DEFAULT_MSG 30 | CUBATURE_INCLUDE_DIR 31 | ) 32 | 33 | mark_as_advanced( 34 | CUBATURE_DIR 35 | CUBATURE_INCLUDE_DIR 36 | ) 37 | -------------------------------------------------------------------------------- /src/boundary/boundaryPy.pyx: -------------------------------------------------------------------------------- 1 | """ 2 | boundaryPy 3 | ====== 4 | 5 | Python interface to the boundary conditions 6 | """ 7 | from gridPy cimport gridPy 8 | 9 | from boundaryHeaders cimport BOUNDARIES_OUTFLOW 10 | from boundaryHeaders cimport BOUNDARIES_MIRROR 11 | from boundaryHeaders cimport BOUNDARIES_DIRICHLET 12 | from boundaryHeaders cimport BOUNDARIES_PERIODIC 13 | 14 | from boundaryHeaders cimport applyBoundaryConditions 15 | 16 | # boundary macros 17 | OUTFLOW = BOUNDARIES_OUTFLOW 18 | MIRROR = BOUNDARIES_MIRROR 19 | DIRICHLET = BOUNDARIES_DIRICHLET 20 | PERIODIC = BOUNDARIES_PERIODIC 21 | 22 | def applyBoundaryConditionsPy(int boundaryLeft, int boundaryRight, 23 | int boundaryTop, int boundaryBottom, 24 | int boundaryFront, int boundaryBack, 25 | gridPy prim 26 | ): 27 | applyBoundaryConditions(boundaryLeft, boundaryRight, 28 | boundaryTop, boundaryBottom, 29 | boundaryFront, boundaryBack, 30 | prim.getGridPtr()[0] 31 | ) 32 | -------------------------------------------------------------------------------- /src/cmake/modules/FindCuba.cmake: -------------------------------------------------------------------------------- 1 | # - Try to find Cuba headers and libraries. 2 | # 3 | # Usage of this module as follows: 4 | # 5 | # find_package(Cuba) 6 | # 7 | # Variables used by this module, they can change the default behaviour and need 8 | # to be set before calling find_package: 9 | # 10 | # CUBA_DIR Set this variable to the root installation of 11 | # CUBA if the module has problems finding 12 | # the proper installation path. 13 | # 14 | # Variables defined by this module: 15 | # 16 | # CUBA_FOUND System has Cuba libs/headers 17 | # CUBA_LIBRARIES The Cuba libraries 18 | # CUBA_INCLUDE_DIR The location of Cuba headers 19 | 20 | find_path(CUBA_DIR 21 | NAMES cuba.h 22 | ) 23 | 24 | find_library(CUBA_LIBRARIES 25 | NAMES cuba 26 | HINTS ${CUBA_DIR} 27 | ) 28 | 29 | find_path(CUBA_INCLUDE_DIR 30 | NAMES cuba.h 31 | HINTS ${CUBA_DIR} 32 | ) 33 | 34 | include(FindPackageHandleStandardArgs) 35 | find_package_handle_standard_args(CUBA DEFAULT_MSG 36 | CUBA_LIBRARIES 37 | CUBA_INCLUDE_DIR 38 | ) 39 | 40 | mark_as_advanced( 41 | CUBA_DIR 42 | CUBA_LIBRARIES 43 | CUBA_INCLUDE_DIR 44 | ) 45 | -------------------------------------------------------------------------------- /src/cmake/modules/FindNumPy.cmake: -------------------------------------------------------------------------------- 1 | # Find the Python NumPy package 2 | # PYTHON_NUMPY_INCLUDE_DIR 3 | # NUMPY_FOUND 4 | # will be set by this script 5 | 6 | cmake_minimum_required(VERSION 2.6) 7 | 8 | if(NOT PYTHON_EXECUTABLE) 9 | if(NumPy_FIND_QUIETLY) 10 | find_package(PythonInterp QUIET) 11 | else() 12 | find_package(PythonInterp) 13 | set(_numpy_out 1) 14 | endif() 15 | endif() 16 | 17 | if (PYTHON_EXECUTABLE) 18 | # write a python script that finds the numpy path 19 | file(WRITE ${PROJECT_BINARY_DIR}/FindNumpyPath.py 20 | "try: import numpy; print(numpy.get_include())\nexcept:pass\n") 21 | 22 | # execute the find script 23 | exec_program("${PYTHON_EXECUTABLE}" ${PROJECT_BINARY_DIR} 24 | ARGS "FindNumpyPath.py" 25 | OUTPUT_VARIABLE NUMPY_PATH) 26 | elseif(_numpy_out) 27 | message(STATUS "Python executable not found.") 28 | endif(PYTHON_EXECUTABLE) 29 | 30 | find_path(PYTHON_NUMPY_INCLUDE_DIR numpy/arrayobject.h 31 | HINTS "${NUMPY_PATH}" "${PYTHON_INCLUDE_PATH}") 32 | 33 | if(PYTHON_NUMPY_INCLUDE_DIR) 34 | set(PYTHON_NUMPY_FOUND 1 CACHE INTERNAL "Python numpy found") 35 | endif(PYTHON_NUMPY_INCLUDE_DIR) 36 | 37 | include(FindPackageHandleStandardArgs) 38 | find_package_handle_standard_args(NumPy DEFAULT_MSG PYTHON_NUMPY_INCLUDE_DIR) 39 | -------------------------------------------------------------------------------- /src/geometry/conftest.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import pytest 3 | 4 | def pytest_addoption(parser): 5 | parser.addoption("--build_path", action="store", default=None, 6 | help='set build directory path' 7 | ) 8 | parser.addoption("--N1", action="store", default=64, 9 | help='grid zones in X1' 10 | ) 11 | parser.addoption("--N2", action="store", default=64, 12 | help='grid zones in X2' 13 | ) 14 | parser.addoption("--N3", action="store", default=64, 15 | help='grid zones in X3' 16 | ) 17 | parser.addoption("--dim", action="store", default=3, 18 | help='grid dimension' 19 | ) 20 | parser.addoption("--blackHoleSpin", action="store", default=0.5, 21 | help='black hole spin' 22 | ) 23 | parser.addoption("--hSlope", action="store", default=0.7, 24 | help='Refinement factor in theta: 1 - no refinement' 25 | ) 26 | 27 | 28 | def pytest_configure(config): 29 | buildPath = config.getvalue('build_path') 30 | gridPath = buildPath + '/grid/' 31 | geometryPath = buildPath + '/geometry/' 32 | sys.path.append(gridPath) 33 | sys.path.append(geometryPath) 34 | -------------------------------------------------------------------------------- /src/cmake/modules/FindLibXML2.cmake: -------------------------------------------------------------------------------- 1 | # Taken from https://github.com/bro/cmake/blob/master/FindLibXML2.cmake 2 | # 3 | # - Try to find LibXML2 headers and libraries. 4 | # 5 | # Usage of this module as follows: 6 | # 7 | # find_package(LibXML2) 8 | # 9 | # Variables used by this module, they can change the default behaviour and need 10 | # to be set before calling find_package: 11 | # 12 | # LibXML2_DIR Set this variable to the root installation of 13 | # LibXML2 if the module has problems finding 14 | # the proper installation path. 15 | # 16 | # Variables defined by this module: 17 | # 18 | # LIBXML2_FOUND System has LibXML2 libs/headers 19 | # LibXML2_LIBRARIES The LibXML2 libraries 20 | # LibXML2_INCLUDE_DIR The location of LibXML2 headers 21 | 22 | find_path(LibXML2_DIR 23 | NAMES include/libxml2/libxml/tree.h 24 | ) 25 | 26 | find_library(LibXML2_LIBRARIES 27 | NAMES xml2 28 | HINTS ${LibXML2_DIR}/lib 29 | ) 30 | 31 | find_path(LibXML2_INCLUDE_DIR 32 | NAMES libxml/tree.h 33 | HINTS ${LibXML2_DIR}/include/libxml2 34 | ) 35 | 36 | include(FindPackageHandleStandardArgs) 37 | find_package_handle_standard_args(LibXML2 DEFAULT_MSG 38 | LibXML2_LIBRARIES 39 | LibXML2_INCLUDE_DIR 40 | ) 41 | 42 | mark_as_advanced( 43 | LibXML2_DIR 44 | LibXML2_LIBRARIES 45 | LibXML2_INCLUDE_DIR 46 | ) 47 | -------------------------------------------------------------------------------- /src/cmake/modules/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright $(git shortlog -s) 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, 5 | are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, this 11 | list of conditions and the following disclaimer in the documentation and/or 12 | other materials provided with the distribution. 13 | 14 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 15 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 16 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 17 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR 18 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 19 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 20 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 21 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 23 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | -------------------------------------------------------------------------------- /src/physics/conftest.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import pytest 3 | 4 | def pytest_addoption(parser): 5 | parser.addoption("--build_path", action="store", default=None, 6 | help='set build directory path' 7 | ) 8 | parser.addoption("--N1", action="store", default=64, 9 | help='grid zones in X1' 10 | ) 11 | parser.addoption("--N2", action="store", default=64, 12 | help='grid zones in X2' 13 | ) 14 | parser.addoption("--N3", action="store", default=64, 15 | help='grid zones in X3' 16 | ) 17 | parser.addoption("--dim", action="store", default=3, 18 | help='grid dimension' 19 | ) 20 | parser.addoption("--blackHoleSpin", action="store", default=0.5, 21 | help='black hole spin' 22 | ) 23 | parser.addoption("--hSlope", action="store", default=0.7, 24 | help='Refinement factor in theta: 1 - no refinement' 25 | ) 26 | 27 | 28 | def pytest_configure(config): 29 | buildPath = config.getvalue('build_path') 30 | gridPath = buildPath + '/grid/' 31 | geometryPath = buildPath + '/geometry/' 32 | physicsPath = buildPath + '/physics/' 33 | sys.path.append(gridPath) 34 | sys.path.append(geometryPath) 35 | sys.path.append(physicsPath) 36 | -------------------------------------------------------------------------------- /src/timer.hpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | // This file is mostly a copy from SpEC UtilsForTiming.*pp 6 | // We define utilities allowing us to determine for how long the 7 | // code has been running (useful for automated restarts) 8 | 9 | class TimeStamp { 10 | friend double operator-(const TimeStamp &end,const TimeStamp &start); 11 | public: 12 | TimeStamp(); // Initialized with current time 13 | // Initialized with number of seconds since 00:00 Jan 1 1970 GMT. 14 | double SecondsSinceEpoch() const; 15 | private: 16 | long mSeconds,mMicroSeconds; // Since the epoch 17 | }; 18 | double operator-(const TimeStamp &end,const TimeStamp &start); 19 | const TimeStamp& TimeWhenExecutableStarted(); 20 | 21 | TimeStamp::TimeStamp() { 22 | timeval tv; 23 | gettimeofday(&tv,nullptr); // Second argument is a timezone ptr, unused/deprecated. 24 | mSeconds = tv.tv_sec; 25 | mMicroSeconds = tv.tv_usec; 26 | } 27 | 28 | double operator-(const TimeStamp &end,const TimeStamp &start) { 29 | return static_cast(end.mMicroSeconds-start.mMicroSeconds)/1000000. 30 | + static_cast(end.mSeconds-start.mSeconds); 31 | } 32 | 33 | double TimeStamp::SecondsSinceEpoch() const { 34 | return static_cast(mMicroSeconds)/1000000. 35 | + static_cast(mSeconds); 36 | } 37 | 38 | namespace { 39 | const TimeStamp TstartOfExecutable; 40 | } 41 | 42 | const TimeStamp& TimeWhenExecutableStarted() { 43 | return TstartOfExecutable; 44 | } 45 | -------------------------------------------------------------------------------- /src/timestepper/conftest.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import pytest 3 | 4 | def pytest_addoption(parser): 5 | parser.addoption("--build_path", action="store", default=None, 6 | help='set build directory path' 7 | ) 8 | parser.addoption("--N1", action="store", default=64, 9 | help='grid zones in X1' 10 | ) 11 | parser.addoption("--N2", action="store", default=64, 12 | help='grid zones in X2' 13 | ) 14 | parser.addoption("--N3", action="store", default=64, 15 | help='grid zones in X3' 16 | ) 17 | parser.addoption("--dim", action="store", default=3, 18 | help='grid dimension' 19 | ) 20 | parser.addoption("--blackHoleSpin", action="store", default=0.5, 21 | help='black hole spin' 22 | ) 23 | parser.addoption("--hSlope", action="store", default=0.7, 24 | help='Refinement factor in theta: 1 - no refinement' 25 | ) 26 | 27 | 28 | def pytest_configure(config): 29 | buildPath = config.getvalue('build_path') 30 | gridPath = buildPath + '/grid/' 31 | geometryPath = buildPath + '/geometry/' 32 | boundaryPath = buildPath + '/boundary/' 33 | physicsPath = buildPath + '/physics/' 34 | timestepperPath = buildPath + '/timestepper/' 35 | sys.path.append(gridPath) 36 | sys.path.append(geometryPath) 37 | sys.path.append(boundaryPath) 38 | sys.path.append(physicsPath) 39 | sys.path.append(timestepperPath) 40 | -------------------------------------------------------------------------------- /src/cmake/modules/FindCython.cmake: -------------------------------------------------------------------------------- 1 | # Find the Cython compiler. 2 | # 3 | # This code sets the following variables: 4 | # 5 | # CYTHON_EXECUTABLE 6 | # 7 | # See also UseCython.cmake 8 | 9 | #============================================================================= 10 | # Copyright 2011 Kitware, Inc. 11 | # 12 | # Licensed under the Apache License, Version 2.0 (the "License"); 13 | # you may not use this file except in compliance with the License. 14 | # You may obtain a copy of the License at 15 | # 16 | # http://www.apache.org/licenses/LICENSE-2.0 17 | # 18 | # Unless required by applicable law or agreed to in writing, software 19 | # distributed under the License is distributed on an "AS IS" BASIS, 20 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 21 | # See the License for the specific language governing permissions and 22 | # limitations under the License. 23 | #============================================================================= 24 | 25 | # Use the Cython executable that lives next to the Python executable 26 | # if it is a local installation. 27 | find_package( PythonInterp ) 28 | if( PYTHONINTERP_FOUND ) 29 | get_filename_component( _python_path ${PYTHON_EXECUTABLE} PATH ) 30 | find_program( CYTHON_EXECUTABLE 31 | NAMES cython cython.bat cython3 32 | HINTS ${_python_path} 33 | ) 34 | else() 35 | find_program( CYTHON_EXECUTABLE 36 | NAMES cython cython.bat cython3 37 | ) 38 | endif() 39 | 40 | 41 | include( FindPackageHandleStandardArgs ) 42 | FIND_PACKAGE_HANDLE_STANDARD_ARGS( Cython REQUIRED_VARS CYTHON_EXECUTABLE ) 43 | 44 | mark_as_advanced( CYTHON_EXECUTABLE ) 45 | -------------------------------------------------------------------------------- /src/reconstruction/reconstructionPy.pyx: -------------------------------------------------------------------------------- 1 | from gridPy cimport gridPy # cimport stands for importing C types. Needed here 2 | # to set the definitions for the arguments of 3 | # reconstructPy 4 | # Import the C++ functions 5 | from reconstructionHeaders cimport reconstruct 6 | from reconstructionHeaders cimport reconstructMM 7 | from reconstructionHeaders cimport reconstructWENO5 8 | 9 | def reconstructPy(gridPy prim, 10 | int dir, 11 | gridPy primLeft, 12 | gridPy primRight 13 | ): 14 | cdef int numReads = 0 15 | cdef int numWrites = 0 16 | reconstruct(prim.getGridPtr()[0], 17 | dir, 18 | primLeft.getGridPtr()[0], 19 | primRight.getGridPtr()[0], 20 | numReads, numWrites 21 | ) 22 | return numReads, numWrites 23 | 24 | def reconstructMinModPy(gridPy prim, 25 | int dir, 26 | gridPy primLeft, 27 | gridPy primRight 28 | ): 29 | cdef int numReads = 0 30 | cdef int numWrites = 0 31 | reconstructMM(prim.getGridPtr()[0], 32 | dir, 33 | primLeft.getGridPtr()[0], 34 | primRight.getGridPtr()[0], 35 | numReads, numWrites 36 | ) 37 | return numReads, numWrites 38 | 39 | def reconstructWENO5Py(gridPy prim, dir, gridPy primLeft, gridPy primRight): 40 | cdef int numReads = 0 41 | cdef int numWrites = 0 42 | reconstructWENO5(prim.getGridPtr()[0], 43 | dir, 44 | primLeft.getGridPtr()[0], 45 | primRight.getGridPtr()[0], 46 | numReads, numWrites 47 | ) 48 | return numReads, numWrites 49 | -------------------------------------------------------------------------------- /run_summit.bsub: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #BSUB -P AST171 3 | #BSUB -W 00:10 4 | #BSUB -J GRIM 5 | # debug or batch 6 | #BSUB -q debug 7 | 8 | #BSUB -nnodes 1 9 | NNODES=1 10 | 11 | # Make bsub behave like sbatch, moving to CWD 12 | # But don't cd anywhere under $HOME, that would be bad 13 | if [[ "$(realpath $LS_SUBCWD)" != *"home"* ]]; then 14 | cd $LS_SUBCWD 15 | fi 16 | 17 | # Stuff for posterity 18 | date 19 | echo "Job run on nodes:" 20 | jsrun -n $NNODES -r 1 hostname 21 | 22 | module load gcc cuda/11.4 petsc boost nsight-systems 23 | 24 | # Arrayfire configuration: 25 | # Limit kernel lengths for performance 26 | export AF_CUDA_MAX_JIT_LEN=30 27 | # Cache to a directory the compute nodes can actually *see* 28 | export AF_JIT_KERNEL_CACHE_DIRECTORY=$(realpath $PWD)/.arrayfire 29 | # Debugging output 30 | #export AF_TRACE=all 31 | 32 | # Tell OpenMP explicitly what it's dealing with 33 | # This also ensures that if we set e.g. NTHREADS=6, 34 | # they will be spread 1/core 35 | # Remember to change this if using 7 cores! 36 | export OMP_PROC_BIND=spread 37 | export OMP_PLACES=threads 38 | export OMP_NUM_THREADS=24 39 | 40 | # In order to run anything on the compute nodes, we must specify 41 | # how we wish to slice them up. This is done with arguments to 42 | # 'jsrun', which takes the place of 'mpirun' on Summit 43 | # -n # resource sets (== MPI tasks for us) 44 | # -r # rs per node 45 | # -a # MPI task per rs 46 | # -g # GPU per rs 47 | # -c # CPU cores (physical) per rs (total of 42=(22-1)*2, but 7 is such an awkward number) 48 | # -b binding strat *within* a resource set 49 | # -l latency optimization in choosing cores (CPU-CPU, GPU-CPU, CPU-MEM) 50 | 51 | # The "smpiargs" argument is used to tell Spectrum MPI we're GPU-aware 52 | jsrun --smpiargs="-gpu" -n $(( $NNODES * 6 )) -r 6 -a 1 -g 1 -d packed -c 6 -b packed:6 -l GPU-CPU \ 53 | nsys profile --capture-range=cudaProfilerApi -o grim_%q{OMPI_COMM_WORLD_RANK} ~/grim/grim -------------------------------------------------------------------------------- /src/problem/orzag_tang/problem.cpp: -------------------------------------------------------------------------------- 1 | #include "../problem.hpp" 2 | 3 | void timeStepper::initialConditions(int &numReads, int &numWrites) 4 | { 5 | double X1Center = (params::X1Start + params::X1End)/2.; 6 | double X2Center = (params::X2Start + params::X2End)/2.; 7 | 8 | array r = sqrt( pow((XCoords->vars[directions::X1] - X1Center), 2.) + 9 | pow((XCoords->vars[directions::X2] - X2Center), 2.) 10 | ); 11 | 12 | primOld->vars[vars::RHO] = 25./(36.*M_PI); 13 | primOld->vars[vars::U] = 5./(12.*M_PI*(params::adiabaticIndex - 1.)); 14 | //primOld->vars[vars::RHO] = 1. + af::exp(-r*r/0.01); 15 | //primOld->vars[vars::U] = 1./(params::adiabaticIndex - 1.); 16 | 17 | array v1 = -0.5*af::sin(2*M_PI*XCoords->vars[directions::X2]); 18 | array v2 = 0.5*af::sin(2*M_PI*XCoords->vars[directions::X1]); 19 | //array v1 = 0.7 * elem->one; 20 | //array v2 = 0.7 * elem->one; 21 | array v3 = 0.*primOld->vars[vars::U3]; 22 | array gamma = 1./af::sqrt(1. - v1*v1 - v2*v2 - v3*v3); 23 | 24 | primOld->vars[vars::U1] = gamma*v1; 25 | primOld->vars[vars::U2] = gamma*v2; 26 | primOld->vars[vars::U3] = gamma*v3; 27 | 28 | primOld->vars[vars::B1] = 29 | -1./sqrt(4*M_PI) * af::sin(2*M_PI*XCoords->vars[directions::X2]); 30 | primOld->vars[vars::B2] = 31 | 1./sqrt(4*M_PI) * af::sin(4*M_PI*XCoords->vars[directions::X1]); 32 | primOld->vars[vars::B3] = 0.; 33 | 34 | } 35 | 36 | void fluidElement::setFluidElementParameters() 37 | { 38 | tau = one; 39 | chi_emhd = 0.01 * one; 40 | nu_emhd = 0.01 * one; 41 | } 42 | 43 | void timeStepper::halfStepDiagnostics(int &numReads,int &numWrites) 44 | { 45 | 46 | } 47 | 48 | void timeStepper::fullStepDiagnostics(int &numReads,int &numWrites) 49 | { 50 | 51 | } 52 | 53 | void timeStepper::setProblemSpecificBCs(int &numReads,int &numWrites) 54 | { 55 | 56 | } 57 | 58 | void timeStepper::applyProblemSpecificFluxFilter(int &numReads,int &numWrites) 59 | { 60 | 61 | } 62 | -------------------------------------------------------------------------------- /src/physics/physicsPy.pyx: -------------------------------------------------------------------------------- 1 | """ 2 | physicsPy 3 | ====== 4 | 5 | Python interface to the fluidElement and riemannSolver class. 6 | """ 7 | 8 | import numpy as np 9 | from gridPy cimport gridPy, coordinatesGridPy 10 | from geometryPy cimport geometryPy 11 | from physicsHeaders cimport fluidElement 12 | 13 | cdef class fluidElementPy(object): 14 | 15 | def __cinit__(self, gridPy prim = gridPy(), 16 | geometryPy geom = geometryPy() 17 | ): 18 | cdef int numReads = 0 19 | cdef int numWrites = 0 20 | if (prim.usingExternalPtr): 21 | self.usingExternalPtr = 1 22 | self.elemPtr = NULL 23 | return 24 | 25 | self.usingExternalPtr = 0 26 | self.elemPtr = new fluidElement(prim.getGridPtr()[0], 27 | geom.getGeometryPtr()[0], 28 | numReads, numWrites 29 | ) 30 | 31 | def computeFluxes(self, geometryPy geom, 32 | const int direction, 33 | gridPy flux 34 | ): 35 | cdef int numReads = 0 36 | cdef int numWrites = 0 37 | self.elemPtr.computeFluxes(direction, 38 | flux.getGridPtr()[0], 39 | numReads, numWrites 40 | ) 41 | return numReads, numWrites 42 | 43 | def __dealloc__(self): 44 | if (self.usingExternalPtr): 45 | return 46 | del self.elemPtr 47 | 48 | cdef setElemPtr(self, fluidElement *elemPtr): 49 | self.elemPtr = elemPtr 50 | 51 | cdef fluidElement* getElemPtr(self, fluidElement *elemPtr): 52 | return self.elemPtr 53 | 54 | @staticmethod 55 | cdef createFluidElementPyFromElemPtr(fluidElement *elemPtr): 56 | cdef fluidElementPy fluidElementPyObject = fluidElementPy() 57 | fluidElementPyObject.usingExternalPtr = 1 58 | fluidElementPyObject.setElemPtr(elemPtr) 59 | return fluidElementPyObject 60 | -------------------------------------------------------------------------------- /src/reconstruction/reconstruction.cpp: -------------------------------------------------------------------------------- 1 | #include "reconstruction.hpp" 2 | 3 | void reconstruction::reconstruct(const grid &prim, 4 | const int dir, 5 | grid &primLeft, 6 | grid &primRight, 7 | int &numReads, 8 | int &numWrites 9 | ) 10 | { 11 | switch (params::reconstruction) 12 | { 13 | case reconstructionOptions::MINMOD: 14 | 15 | reconstruction::reconstructMM(prim, dir, primLeft, primRight, 16 | numReads, numWrites 17 | ); 18 | 19 | break; 20 | 21 | case reconstructionOptions::WENO5: 22 | 23 | reconstruction::reconstructWENO5(prim, dir, primLeft, primRight, 24 | numReads, numWrites 25 | ); 26 | 27 | break; 28 | 29 | case reconstructionOptions::PPM: 30 | 31 | reconstruction::reconstructPPM(prim, dir, primLeft, primRight, 32 | numReads, numWrites 33 | ); 34 | 35 | break; 36 | } 37 | 38 | } 39 | 40 | array reconstruction::slope(const int dir, const double dX, 41 | const array& in, 42 | int &numReads, 43 | int &numWrites 44 | ) 45 | { 46 | switch (params::reconstruction) 47 | { 48 | case reconstructionOptions::MINMOD: 49 | 50 | return reconstruction::slopeMM(dir,dX,in, numReads, numWrites); 51 | 52 | case reconstructionOptions::WENO5: 53 | 54 | return reconstruction::slopeWENO5(dir,dX,in, numReads, numWrites); 55 | 56 | case reconstructionOptions::PPM: 57 | 58 | return reconstruction::slopePPM(dir,dX,in, numReads, numWrites); 59 | 60 | default: 61 | 62 | return reconstruction::slopeMM(dir,dX,in, numReads, numWrites); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/grim.cpp: -------------------------------------------------------------------------------- 1 | #include "grim.hpp" 2 | #include "params.hpp" 3 | 4 | #include "cuda_profiler_api.h" 5 | 6 | int main(int argc, char **argv) 7 | { 8 | PetscInitialize(&argc, &argv, NULL, help); 9 | 10 | int world_rank; 11 | MPI_Comm_rank(PETSC_COMM_WORLD, &world_rank); 12 | af::setDevice(world_rank%params::numDevices); 13 | 14 | /* Local scope so that destructors of all classes are called before 15 | * PetscFinalize() */ 16 | { 17 | timeStepper ts(params::N1, params::N2, params::N3, 18 | params::dim, vars::dof, params::numGhost, 19 | params::Time, params::InitialDt, 20 | params::boundaryLeft, params::boundaryRight, 21 | params::boundaryTop, params::boundaryBottom, 22 | params::boundaryFront, params::boundaryBack, 23 | params::metric, params::blackHoleSpin, params::hSlope, 24 | params::X1Start, params::X1End, 25 | params::X2Start, params::X2End, 26 | params::X3Start, params::X3End 27 | ); 28 | 29 | PetscPrintf(PETSC_COMM_WORLD, " Generating compute kernels...\n\n"); 30 | int numReads, numWrites; 31 | ts.timeStep(numReads, numWrites); 32 | 33 | af::sync(); 34 | 35 | PetscPrintf(PETSC_COMM_WORLD, "\n Kernel compilation complete\n"); 36 | 37 | cudaProfilerStart(); 38 | 39 | int n=0; 40 | int StopRunning = 0; 41 | while(ts.time 3 | 4 | namespace params 5 | { 6 | int numDevices = 1; 7 | 8 | int N1 = 512; 9 | int N2 = 1; 10 | int N3 = 1; 11 | 12 | int dim = 1; 13 | int numGhost = 3; 14 | 15 | int timeStepper = timeStepping::EXPLICIT; 16 | double InitialDt = 1e-5; 17 | double Time = 0.; 18 | double CourantFactor = 0.25; 19 | double maxDtIncrement = 1.3; 20 | double finalTime = 5.; 21 | double WriteDataEveryDt = 0.01; 22 | int metric = metrics::MINKOWSKI; 23 | int restart = 0; 24 | std::string restartFile = "restartFile.h5"; 25 | std::string restartFileName = "restartFileName.txt"; 26 | std::string restartFileTime = "restartFileTime.txt"; 27 | 28 | double X1Start = -.5, X1End = 1.5; 29 | double X2Start = 0., X2End = 1.; 30 | double X3Start = 0., X3End = 1.; 31 | 32 | int boundaryLeft = boundaries::OUTFLOW; 33 | int boundaryRight = boundaries::OUTFLOW; 34 | 35 | int boundaryTop = boundaries::PERIODIC; 36 | int boundaryBottom = boundaries::PERIODIC; 37 | 38 | int boundaryFront = boundaries::PERIODIC; 39 | int boundaryBack = boundaries::PERIODIC; 40 | 41 | double rhoFloorInFluidElement = 1e-20; 42 | double uFloorInFluidElement = 1e-20; 43 | double bSqrFloorInFluidElement = 1e-20; 44 | double temperatureFloorInFluidElement = 1e-20; 45 | 46 | int conduction = 1; 47 | int viscosity = 1; 48 | int highOrderTermsConduction = 1; 49 | int highOrderTermsViscosity = 1; 50 | double ConductionAlpha = 5.; 51 | double ViscosityAlpha = 3.; 52 | 53 | double adiabaticIndex = 4./3; 54 | 55 | std::string shockTest = "stationary_shock_BVP_input"; 56 | 57 | double slopeLimTheta = 2; 58 | int reconstruction = reconstructionOptions::MINMOD; 59 | int riemannSolver = riemannSolvers::LOCAL_LAX_FRIEDRICH; 60 | 61 | int maxNonLinearIter = 3; 62 | int maxLineSearchIters = 3; 63 | 64 | //Parameters controlling accuracy of nonlinear solver 65 | double nonlinearsolve_atol = 1.e-20; 66 | double JacobianAssembleEpsilon = 4.e-8; 67 | double linesearchfloor = 1.e-24; 68 | 69 | // Linear solver options 70 | int linearSolver = linearSolvers::GPU_BATCH_SOLVER; 71 | 72 | //Unused params - do we need to define them? 73 | double hSlope = 0.3; 74 | double blackHoleSpin = 0.9375; 75 | int DerefineThetaHorizon = 1; 76 | int DoCylindrify = 1; 77 | double X1cyl = 0.; 78 | double X2cyl = 1./N2; 79 | }; 80 | 81 | namespace vars 82 | { 83 | int Q = 5; 84 | int DP = 5 + params::conduction; 85 | int numFluidVars = 5 + params::conduction + params::viscosity; 86 | 87 | int B1 = 5 + params::conduction + params::viscosity; 88 | int B2 = 6 + params::conduction + params::viscosity; 89 | int B3 = 7 + params::conduction + params::viscosity; 90 | int dof = 8 + params::conduction + params::viscosity; 91 | }; 92 | -------------------------------------------------------------------------------- /src/grid/grid.hpp: -------------------------------------------------------------------------------- 1 | #ifndef GRIM_GRID_H_ 2 | #define GRIM_GRID_H_ 3 | 4 | #include "../params.hpp" 5 | #include 6 | #include 7 | #include 8 | 9 | using af::array; 10 | using af::span; 11 | using af::shift; 12 | 13 | class grid 14 | { 15 | void copyLocalVecToVars(); 16 | 17 | public: 18 | DM dm, coordDM; 19 | Vec globalVec, localVec, coordVec; 20 | 21 | int numGhost, numVars, dim; 22 | 23 | int boundaryLeft, boundaryRight; 24 | int boundaryTop, boundaryBottom; 25 | int boundaryFront, boundaryBack; 26 | int periodicBoundariesX1; 27 | int periodicBoundariesX2; 28 | int periodicBoundariesX3; 29 | 30 | DMBoundaryType DMBoundaryLeft, DMBoundaryRight; 31 | DMBoundaryType DMBoundaryTop, DMBoundaryBottom; 32 | DMBoundaryType DMBoundaryFront, DMBoundaryBack; 33 | 34 | int iLocalStart, jLocalStart, kLocalStart; 35 | int iLocalEnd, jLocalEnd, kLocalEnd; 36 | int N1, N2, N3; 37 | int N1Local, N2Local, N3Local; 38 | int N1Total, N2Total, N3Total; 39 | int numGhostX1, numGhostX2, numGhostX3; 40 | 41 | af::seq *domainX1, *domainX2, *domainX3; 42 | 43 | array *vars, varsSoA; 44 | array indices[3]; 45 | double *hostPtr; 46 | bool hasHostPtrBeenAllocated; 47 | bool havexCoordsBeenSet; 48 | 49 | grid(const int N1, 50 | const int N2, 51 | const int N3, 52 | const int dim, 53 | const int numVars, 54 | const int numGhost, 55 | const int periodicBoundariesX1, 56 | const int periodicBoundariesX2, 57 | const int periodicBoundariesX3 58 | ); 59 | ~grid(); 60 | 61 | void communicate(); 62 | void copyVarsToHostPtr(); 63 | void copyVarsToGlobalVec(); 64 | void copyHostPtrToVars(const double *hostPtr); 65 | void dump(const std::string varsName, const std::string filename); 66 | void dumpVTS(const grid &xCoords, 67 | const std::string *varNames, 68 | const std::string filename 69 | ); 70 | void load(const std::string varsName, const std::string filename); 71 | }; 72 | 73 | class coordinatesGrid : public grid 74 | { 75 | public: 76 | double X1Start, X1End; 77 | double X2Start, X2End; 78 | double X3Start, X3End; 79 | 80 | double dX1, dX2, dX3; 81 | 82 | coordinatesGrid(const int N1, 83 | const int N2, 84 | const int N3, 85 | const int dim, 86 | const int numGhost, 87 | const double X1Start, const double X1End, 88 | const double X2Start, const double X2End, 89 | const double X3Start, const double X3End 90 | ); 91 | ~coordinatesGrid(); 92 | 93 | void setXCoords(const int location); 94 | }; 95 | 96 | #endif /* GRIM_GRID_H_ */ 97 | -------------------------------------------------------------------------------- /src/problem/anisotropic_conduction/config.h.in: -------------------------------------------------------------------------------- 1 | #ifndef GRIM_INPUT_H_ 2 | #define GRIM_INPUT_H_ 3 | 4 | /* Immutable constants. Should not be changed */ 5 | #define NDIM (4) 6 | #define M_PI (3.141592653589793238462643383279) 7 | /* End of immutable constants */ 8 | 9 | #cmakedefine OUTPUT_DIR ${OUTPUT_DIR} 10 | #cmakedefine01 RESTART 11 | #cmakedefine RESTART_FILE ${RESTART_FILE} 12 | #cmakedefine DUMP_FILE_PREFIX ${DUMP_FILE_PREFIX} 13 | #cmakedefine TIME_STEPPING ${TIME_STEPPING} 14 | #cmakedefine DT (${DT}) 15 | #cmakedefine DT_DUMP (${DT_DUMP}) 16 | #cmakedefine START_TIME (${START_TIME}) 17 | #cmakedefine FINAL_TIME (${FINAL_TIME}) 18 | #cmakedefine COURANT (${COURANT}) 19 | #cmakedefine MAX_DT_INCREMENT (${MAX_DT_INCREMENT}) 20 | #cmakedefine REAL ${REAL} 21 | #cmakedefine ARRAY_ARGS ${ARRAY_ARGS} 22 | #cmakedefine PROBLEM_DATA ${PROBLEM_DATA} 23 | 24 | /* Domain inputs */ 25 | #cmakedefine COMPUTE_DIM (${COMPUTE_DIM}) 26 | #cmakedefine N1 (${N1}) 27 | #cmakedefine N2 (${N2}) 28 | #cmakedefine TILE_SIZE_X1 (${TILE_SIZE_X1}) 29 | #cmakedefine TILE_SIZE_X2 (${TILE_SIZE_X2}) 30 | #cmakedefine01 USE_OPENMP 31 | 32 | 33 | #cmakedefine ADIABATIC_INDEX (${ADIABATIC_INDEX}) 34 | #cmakedefine01 CONDUCTION 35 | 36 | 37 | /* Initial condition parameters */ 38 | #cmakedefine AMPLITUDE (${AMPLITUDE}) 39 | #cmakedefine RADIUS (${RADIUS}) 40 | #cmakedefine MEAN_MAGNETIC_FIELD (${MEAN_MAGNETIC_FIELD}) 41 | #cmakedefine WAVE_NUMBER (${WAVE_NUMBER}) 42 | 43 | /* Geometry */ 44 | #cmakedefine METRIC (${METRIC}) 45 | #cmakedefine EPS (${EPS}) 46 | 47 | /* Domain */ 48 | #cmakedefine X1_A (${X1_A}) 49 | #cmakedefine X1_B (${X1_B}) 50 | #cmakedefine X2_A (${X2_A}) 51 | #cmakedefine X2_B (${X2_B}) 52 | 53 | /* Boundary conditions */ 54 | #cmakedefine PHYSICAL_BOUNDARY_LEFT_EDGE (${PHYSICAL_BOUNDARY_LEFT_EDGE}) 55 | #cmakedefine PHYSICAL_BOUNDARY_RIGHT_EDGE (${PHYSICAL_BOUNDARY_RIGHT_EDGE}) 56 | #cmakedefine PHYSICAL_BOUNDARY_TOP_EDGE (${PHYSICAL_BOUNDARY_TOP_EDGE}) 57 | #cmakedefine PHYSICAL_BOUNDARY_BOTTOM_EDGE (${PHYSICAL_BOUNDARY_BOTTOM_EDGE}) 58 | 59 | /* Reconstruction */ 60 | #cmakedefine RECONSTRUCTION (${RECONSTRUCTION}) 61 | 62 | /* Floor values */ 63 | #cmakedefine RHO_FLOOR (${RHO_FLOOR}) 64 | #cmakedefine UU_FLOOR (${UU_FLOOR}) 65 | #cmakedefine RHO_FLOOR_MIN (${RHO_FLOOR_MIN}) 66 | #cmakedefine UU_FLOOR_MIN (${UU_FLOOR_MIN}) 67 | 68 | /* Number of ghost zones */ 69 | #if (RECONSTRUCTION==MONOTONIZED_CENTRAL || RECONSTRUCTION==MIN_MOD) 70 | #define NG (3) 71 | #elif (RECONSTRUCTION==MP5) 72 | #define NG (4) 73 | #endif 74 | 75 | #endif /* GRIM_INPUT_H_ */ 76 | -------------------------------------------------------------------------------- /src/cmake/modules/FindOpenCL.cmake: -------------------------------------------------------------------------------- 1 | # Taken from ViennaCL dev version 2 | # - Find the OpenCL headers and library 3 | # 4 | # Defines the following if found: 5 | # OPENCL_FOUND : TRUE if found, FALSE otherwise 6 | # OPENCL_INCLUDE_DIRS : Include directories for OpenCL 7 | # OPENCL_LIBRARIES : The libraries to link against 8 | # 9 | # The user can set the OPENCLROOT environment variable to help finding OpenCL 10 | # if it is installed in a non-standard place. 11 | 12 | set(ENV_ATISTREAMSDKROOT "$ENV{ATISTREAMSDKROOT}") 13 | if(ENV_ATISTREAMSDKROOT) 14 | set(ENV_OPENCLROOT "$ENV{ATISTREAMSDKROOT}") 15 | endif(ENV_ATISTREAMSDKROOT) 16 | 17 | set(ENV_AMDAPPSDKROOT "$ENV{AMDAPPSDKROOT}") 18 | if(ENV_AMDAPPSDKROOT) 19 | set(ENV_OPENCLROOT "$ENV{AMDAPPSDKROOT}") 20 | endif(ENV_AMDAPPSDKROOT) 21 | 22 | set(ENV_INTELOCLSDKROOT "$ENV{INTELOCLSDKROOT}") 23 | if(ENV_INTELOCLSDKROOT) 24 | set(ENV_OPENCLROOT "$ENV{INTELOCLSDKROOT}") 25 | endif(ENV_INTELOCLSDKROOT) 26 | 27 | set(ENV_OPENCLROOT2 "$ENV{OPENCLROOT}") 28 | if(ENV_OPENCLROOT2) 29 | set(ENV_OPENCLROOT "$ENV{OPENCLROOT}") 30 | endif(ENV_OPENCLROOT2) 31 | 32 | if(ENV_OPENCLROOT) 33 | find_path( 34 | OPENCL_INCLUDE_DIR 35 | NAMES CL/cl.h OpenCL/cl.h 36 | PATHS "${ENV_OPENCLROOT}/include" 37 | #NO_DEFAULT_PATH #uncomment this is you wish to surpress the use of default paths for OpenCL 38 | ) 39 | 40 | if (("${CMAKE_SYSTEM_NAME}" MATCHES "Linux") OR ("${CMAKE_SYSTEM_NAME}" MATCHES "Windows")) 41 | if(CMAKE_SIZEOF_VOID_P EQUAL 4) 42 | set(OPENCL_LIB_SEARCH_PATH 43 | "${OPENCL_LIB_SEARCH_PATH}" 44 | "${ENV_OPENCLROOT}/lib/x86") 45 | else(CMAKE_SIZEOF_VOID_P EQUAL 4) 46 | set(OPENCL_LIB_SEARCH_PATH 47 | "${OPENCL_LIB_SEARCH_PATH}" 48 | "${ENV_OPENCLROOT}/lib/x86_64") 49 | endif(CMAKE_SIZEOF_VOID_P EQUAL 4) 50 | endif(("${CMAKE_SYSTEM_NAME}" MATCHES "Linux") OR ("${CMAKE_SYSTEM_NAME}" MATCHES "Windows")) 51 | find_library( 52 | OPENCL_LIBRARY 53 | NAMES OpenCL 54 | PATHS "${OPENCL_LIB_SEARCH_PATH}" 55 | #NO_DEFAULT_PATH #uncomment this is you wish to surpress the use of default paths for OpenCL 56 | ) 57 | else(ENV_OPENCLROOT) 58 | find_path( 59 | OPENCL_INCLUDE_DIR 60 | NAMES CL/cl.h OpenCL/cl.h 61 | PATHS "${PROJECT_SOURCE_DIR}" #use the CL/ include folder provided with ViennaCL 62 | ) 63 | 64 | find_library( 65 | OPENCL_LIBRARY 66 | NAMES OpenCL 67 | ) 68 | endif(ENV_OPENCLROOT) 69 | 70 | include(FindPackageHandleStandardArgs) 71 | find_package_handle_standard_args( 72 | OPENCL 73 | DEFAULT_MSG 74 | OPENCL_LIBRARY OPENCL_INCLUDE_DIR 75 | ) 76 | 77 | if(OPENCL_FOUND) 78 | set(OPENCL_INCLUDE_DIRS "${OPENCL_INCLUDE_DIR}") 79 | set(OPENCL_LIBRARIES "${OPENCL_LIBRARY}") 80 | else(OPENCL_FOUND) 81 | set(OPENCL_INCLUDE_DIRS) 82 | set(OPENCL_LIBRARIES) 83 | endif(OPENCL_FOUND) 84 | 85 | mark_as_advanced( 86 | OPENCL_INCLUDE_DIR 87 | OPENCL_LIBRARY 88 | ) 89 | 90 | -------------------------------------------------------------------------------- /src/problem/magnetized_explosion/config.h.in: -------------------------------------------------------------------------------- 1 | #ifndef GRIM_INPUT_H_ 2 | #define GRIM_INPUT_H_ 3 | 4 | /* Immutable constants. Should not be changed */ 5 | #define NDIM (4) 6 | #define M_PI (3.141592653589793238462643383279) 7 | /* End of immutable constants */ 8 | 9 | #cmakedefine OUTPUT_DIR ${OUTPUT_DIR} 10 | #cmakedefine01 RESTART 11 | #cmakedefine RESTART_FILE ${RESTART_FILE} 12 | #cmakedefine DUMP_FILE_PREFIX ${DUMP_FILE_PREFIX} 13 | #cmakedefine TIME_STEPPING ${TIME_STEPPING} 14 | #cmakedefine DT (${DT}) 15 | #cmakedefine DT_DUMP (${DT_DUMP}) 16 | #cmakedefine START_TIME (${START_TIME}) 17 | #cmakedefine FINAL_TIME (${FINAL_TIME}) 18 | #cmakedefine COURANT (${COURANT}) 19 | #cmakedefine MAX_DT_INCREMENT (${MAX_DT_INCREMENT}) 20 | #cmakedefine REAL ${REAL} 21 | #cmakedefine ARRAY_ARGS ${ARRAY_ARGS} 22 | #cmakedefine PROBLEM_DATA ${PROBLEM_DATA} 23 | 24 | /* Domain inputs */ 25 | #cmakedefine COMPUTE_DIM (${COMPUTE_DIM}) 26 | #cmakedefine N1 (${N1}) 27 | #cmakedefine N2 (${N2}) 28 | #cmakedefine TILE_SIZE_X1 (${TILE_SIZE_X1}) 29 | #cmakedefine TILE_SIZE_X2 (${TILE_SIZE_X2}) 30 | #cmakedefine01 USE_OPENMP 31 | 32 | 33 | #cmakedefine ADIABATIC_INDEX (${ADIABATIC_INDEX}) 34 | #cmakedefine01 CONDUCTION 35 | 36 | 37 | /* Initial condition parameters */ 38 | #cmakedefine INITIAL_RADIUS (${INITIAL_RADIUS}) 39 | #cmakedefine MEAN_MAGNETIC_FIELD (${MEAN_MAGNETIC_FIELD}) 40 | #cmakedefine RHO_INSIDE (${RHO_INSIDE}) 41 | #cmakedefine RHO_OUTSIDE (${RHO_OUTSIDE}) 42 | #cmakedefine PRESSURE_INSIDE (${PRESSURE_INSIDE}) 43 | #cmakedefine PRESSURE_OUTSIDE (${PRESSURE_OUTSIDE}) 44 | #cmakedefine SMOOTHNESS (${SMOOTHNESS}) 45 | 46 | /* Geometry */ 47 | #cmakedefine METRIC (${METRIC}) 48 | #cmakedefine EPS (${EPS}) 49 | 50 | /* Domain */ 51 | #cmakedefine X1_A (${X1_A}) 52 | #cmakedefine X1_B (${X1_B}) 53 | #cmakedefine X2_A (${X2_A}) 54 | #cmakedefine X2_B (${X2_B}) 55 | 56 | /* Boundary conditions */ 57 | #cmakedefine PHYSICAL_BOUNDARY_LEFT_EDGE (${PHYSICAL_BOUNDARY_LEFT_EDGE}) 58 | #cmakedefine PHYSICAL_BOUNDARY_RIGHT_EDGE (${PHYSICAL_BOUNDARY_RIGHT_EDGE}) 59 | #cmakedefine PHYSICAL_BOUNDARY_TOP_EDGE (${PHYSICAL_BOUNDARY_TOP_EDGE}) 60 | #cmakedefine PHYSICAL_BOUNDARY_BOTTOM_EDGE (${PHYSICAL_BOUNDARY_BOTTOM_EDGE}) 61 | 62 | /* Reconstruction */ 63 | #cmakedefine RECONSTRUCTION (${RECONSTRUCTION}) 64 | 65 | /* Floor values */ 66 | #cmakedefine RHO_FLOOR (${RHO_FLOOR}) 67 | #cmakedefine UU_FLOOR (${UU_FLOOR}) 68 | #cmakedefine RHO_FLOOR_MIN (${RHO_FLOOR_MIN}) 69 | #cmakedefine UU_FLOOR_MIN (${UU_FLOOR_MIN}) 70 | 71 | /* Number of ghost zones */ 72 | #if (RECONSTRUCTION==MONOTONIZED_CENTRAL || RECONSTRUCTION==MIN_MOD) 73 | #define NG (3) 74 | #elif (RECONSTRUCTION==MP5) 75 | #define NG (4) 76 | #endif 77 | 78 | #endif /* GRIM_INPUT_H_ */ 79 | -------------------------------------------------------------------------------- /src/reconstruction/minmod.cpp: -------------------------------------------------------------------------------- 1 | #include "reconstruction.hpp" 2 | 3 | array reconstruction::minmod(array &x, array &y, array &z, 4 | int &numReads, int &numWrites 5 | ) 6 | { 7 | array minOfAll = af::min(af::min(af::abs(x), af::abs(y)), 8 | af::abs(z) 9 | ); 10 | 11 | //Stupid convention in ArrayFire: sign(x)=1 for x<0 and sign(x)=0 for x>0 12 | array signx = 1.-2.*sign(x); 13 | array signy = 1.-2.*sign(y); 14 | array signz = 1.-2.*sign(z); 15 | 16 | array result = 0.25 * af::abs(signx + signy ) * (signx + signz ) * minOfAll; 17 | 18 | return result; 19 | } 20 | 21 | array reconstruction::slopeMM(const int dir,const double dX, const array& in, 22 | int &numReads, 23 | int &numWrites 24 | ) 25 | { 26 | int X1_plus, X2_plus, X3_plus; 27 | int X1_minus, X2_minus, X3_minus; 28 | switch (dir) 29 | { 30 | case directions::X1: 31 | 32 | X1_plus = -1; X2_plus = 0; X3_plus = 0; 33 | X1_minus = 1; X2_minus = 0; X3_minus = 0; 34 | 35 | break; 36 | 37 | case directions::X2: 38 | 39 | X1_plus = 0; X2_plus = -1; X3_plus = 0; 40 | X1_minus = 0; X2_minus = 1; X3_minus = 0; 41 | 42 | break; 43 | 44 | case directions::X3: 45 | 46 | X1_plus = 0; X2_plus = 0; X3_plus = -1; 47 | X1_minus = 0; X2_minus = 0; X3_minus = 1; 48 | 49 | break; 50 | } 51 | 52 | array in_plus = af::shift(in, X1_plus, X2_plus, X3_plus ); 53 | array in_minus = af::shift(in, X1_minus, X2_minus, X3_minus); 54 | 55 | array forwardDiff = (in_plus - in)/dX; 56 | array backwardDiff = (in - in_minus)/dX; 57 | array centralDiff = backwardDiff + forwardDiff; 58 | 59 | /* TODO: add an argument to slopeLimTheta.*/ 60 | double slopeLimTheta = params::slopeLimTheta; 61 | array left = slopeLimTheta * backwardDiff; 62 | array center = 0.5 * centralDiff; 63 | array right = slopeLimTheta * forwardDiff; 64 | 65 | int numReadsMinMod, numWritesMinMod; 66 | array result = minmod(left, center, right, 67 | numReadsMinMod, numWritesMinMod 68 | ); 69 | 70 | return result; 71 | } 72 | 73 | void reconstruction::reconstructMM(const grid &prim, 74 | const int dir, 75 | grid &primLeft, 76 | grid &primRight, 77 | int &numReads, 78 | int &numWrites 79 | ) 80 | { 81 | int numReadsMinMod, numWritesMinMod; 82 | for(int var=0; vardim >= 2) 8 | { 9 | int numReadsEMF, numWritesEMF; 10 | computeEMF(numReadsEMF, numWritesEMF); 11 | 12 | fluxesX1->vars[vars::B1] = 0.; 13 | 14 | fluxesX1->vars[vars::B2] = 15 | 0.5*( emfX3->vars[0] 16 | + shift(emfX3->vars[0], 0, -1, 0) 17 | ); 18 | 19 | fluxesX2->vars[vars::B1] = 20 | -0.5*( emfX3->vars[0] 21 | + shift(emfX3->vars[0], -1, 0, 0) 22 | ); 23 | 24 | fluxesX2->vars[vars::B2] = 0.; 25 | 26 | if (fluxesX1->dim == 3) 27 | { 28 | fluxesX1->vars[vars::B3] = 29 | -0.5*( emfX2->vars[0] 30 | + shift(emfX2->vars[0], 0, 0, -1) 31 | ); 32 | 33 | fluxesX2->vars[vars::B3] = 34 | 0.5*( emfX1->vars[0] 35 | + shift(emfX1->vars[0], 0, 0, -1) 36 | ); 37 | 38 | fluxesX3->vars[vars::B1] = 39 | 0.5*( emfX2->vars[0] 40 | + shift(emfX2->vars[0], -1, 0, 0) 41 | ); 42 | 43 | fluxesX3->vars[vars::B2] = 44 | -0.5*( emfX1->vars[0] 45 | + shift(emfX1->vars[0], 0, -1, 0) 46 | ); 47 | 48 | fluxesX3->vars[vars::B3] = 0.; 49 | } 50 | } 51 | } 52 | 53 | void timeStepper::computeEMF(int &numReadsEMF, 54 | int &numWritesEMF 55 | ) 56 | { 57 | if (fluxesX1->dim >= 2) 58 | { 59 | emfX3->vars[0] = 60 | 0.25*( fluxesX1->vars[vars::B2] 61 | + shift(fluxesX1->vars[vars::B2], 0, 1, 0) 62 | - fluxesX2->vars[vars::B1] 63 | - shift(fluxesX2->vars[vars::B1], 1, 0, 0) 64 | ); 65 | 66 | if (fluxesX1->dim == 3) 67 | { 68 | emfX1->vars[0] = 69 | 0.25*( fluxesX2->vars[vars::B3] 70 | + shift(fluxesX2->vars[vars::B3], 0, 0, 1) 71 | - fluxesX3->vars[vars::B2] 72 | - shift(fluxesX3->vars[vars::B2], 0, 1, 0) 73 | ); 74 | 75 | emfX2->vars[0] = 76 | 0.25*( fluxesX3->vars[vars::B1] 77 | + shift(fluxesX3->vars[vars::B1], 1, 0, 0) 78 | - fluxesX1->vars[vars::B3] 79 | - shift(fluxesX1->vars[vars::B3], 0, 0, 1) 80 | ); 81 | } 82 | } 83 | } 84 | 85 | void timeStepper::computeDivB(const grid &prim, 86 | int &numReads, 87 | int &numWrites 88 | ) 89 | { 90 | 91 | array B1 = prim.vars[vars::B1]; 92 | array B2 = prim.vars[vars::B2]; 93 | array B3 = prim.vars[vars::B3]; 94 | array g = geomCenter->g; 95 | 96 | if (prim.dim == 2) 97 | { 98 | divB->vars[0] = 99 | ( g*B1 + shift(g*B1, 0, 1, 0) 100 | - shift(g*B1, 1, 0, 0) - shift(g*B1, 1, 1, 0) 101 | )/(2.*XCoords->dX1) 102 | + 103 | ( g*B2 + shift(g*B2, 1, 0, 0) 104 | - shift(g*B2, 0, 1, 0) - shift(g*B2, 1, 1, 0) 105 | )/(2.*XCoords->dX2); 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /src/problem/bondi_viscous/config.h.in: -------------------------------------------------------------------------------- 1 | #ifndef GRIM_INPUT_H_ 2 | #define GRIM_INPUT_H_ 3 | 4 | /* Immutable constants. Should not be changed */ 5 | #define NDIM (4) 6 | #define M_PI (3.141592653589793238462643383279) 7 | /* End of immutable constants */ 8 | 9 | #cmakedefine OUTPUT_DIR ${OUTPUT_DIR} 10 | #cmakedefine01 RESTART 11 | #cmakedefine RESTART_FILE ${RESTART_FILE} 12 | #cmakedefine DUMP_FILE_PREFIX ${DUMP_FILE_PREFIX} 13 | #cmakedefine TIME_STEPPING ${TIME_STEPPING} 14 | #cmakedefine DT (${DT}) 15 | #cmakedefine DT_DUMP (${DT_DUMP}) 16 | #cmakedefine START_TIME (${START_TIME}) 17 | #cmakedefine FINAL_TIME (${FINAL_TIME}) 18 | #cmakedefine COURANT (${COURANT}) 19 | #cmakedefine MAX_DT_INCREMENT (${MAX_DT_INCREMENT}) 20 | #cmakedefine REAL ${REAL} 21 | #cmakedefine ARRAY_ARGS ${ARRAY_ARGS} 22 | #cmakedefine PROBLEM_DATA ${PROBLEM_DATA} 23 | 24 | /* Domain inputs */ 25 | #cmakedefine COMPUTE_DIM (${COMPUTE_DIM}) 26 | #cmakedefine N1 (${N1}) 27 | #cmakedefine N2 (${N2}) 28 | #cmakedefine TILE_SIZE_X1 (${TILE_SIZE_X1}) 29 | #cmakedefine TILE_SIZE_X2 (${TILE_SIZE_X2}) 30 | #cmakedefine01 USE_OPENMP 31 | 32 | 33 | #cmakedefine ADIABATIC_INDEX (${ADIABATIC_INDEX}) 34 | #cmakedefine01 CONDUCTION 35 | #cmakedefine01 VISCOSITY 36 | 37 | /* Initial condition parameters */ 38 | #cmakedefine SONICRADIUS (${SONICRADIUS}) 39 | #cmakedefine MDOT (${MDOT}) 40 | #cmakedefine BMAG (${BMAG}) 41 | 42 | /* Geometry */ 43 | #cmakedefine METRIC (${METRIC}) 44 | #cmakedefine EPS (${EPS}) 45 | #cmakedefine M (${M}) 46 | #cmakedefine BH_SPIN (${BH_SPIN}) 47 | #cmakedefine H_SLOPE (${H_SLOPE}) 48 | 49 | /* Domain */ 50 | #cmakedefine R_A (${R_A}) 51 | #cmakedefine R_B (${R_B}) 52 | 53 | #cmakedefine X1_A (${X1_A}) 54 | #cmakedefine X1_B (${X1_B}) 55 | #cmakedefine X2_A (${X2_A}) 56 | #cmakedefine X2_B (${X2_B}) 57 | 58 | /* Boundary conditions */ 59 | #cmakedefine PHYSICAL_BOUNDARY_LEFT_EDGE (${PHYSICAL_BOUNDARY_LEFT_EDGE}) 60 | #cmakedefine PHYSICAL_BOUNDARY_RIGHT_EDGE (${PHYSICAL_BOUNDARY_RIGHT_EDGE}) 61 | #cmakedefine PHYSICAL_BOUNDARY_TOP_EDGE (${PHYSICAL_BOUNDARY_TOP_EDGE}) 62 | #cmakedefine PHYSICAL_BOUNDARY_BOTTOM_EDGE (${PHYSICAL_BOUNDARY_BOTTOM_EDGE}) 63 | 64 | /* Reconstruction */ 65 | #cmakedefine RECONSTRUCTION (${RECONSTRUCTION}) 66 | 67 | /* Floor values */ 68 | #cmakedefine RHO_FLOOR (${RHO_FLOOR}) 69 | #cmakedefine UU_FLOOR (${UU_FLOOR}) 70 | #cmakedefine RHO_FLOOR_FALLOFF (${RHO_FLOOR_FALLOFF}) 71 | #cmakedefine UU_FLOOR_FALLOFF (${UU_FLOOR_FALLOFF}) 72 | #cmakedefine RHO_FLOOR_MIN (${RHO_FLOOR_MIN}) 73 | #cmakedefine UU_FLOOR_MIN (${UU_FLOOR_MIN}) 74 | #cmakedefine GAMMA_MAX (${GAMMA_MAX}) 75 | 76 | /* Number of ghost zones */ 77 | #if (RECONSTRUCTION==MONOTONIZED_CENTRAL || RECONSTRUCTION==MIN_MOD) 78 | #define NG (3) 79 | #elif (RECONSTRUCTION==MP5) 80 | #define NG (4) 81 | #endif 82 | 83 | #endif /* GRIM_INPUT_H_ */ 84 | -------------------------------------------------------------------------------- /src/cmake/modules/FindNetCDF.cmake: -------------------------------------------------------------------------------- 1 | # - Find NetCDF 2 | # Find the native NetCDF includes and library 3 | # 4 | # NETCDF_INCLUDES - where to find netcdf.h, etc 5 | # NETCDF_LIBRARIES - Link these libraries when using NetCDF 6 | # NETCDF_FOUND - True if NetCDF found including required interfaces (see below) 7 | # 8 | # Your package can require certain interfaces to be FOUND by setting these 9 | # 10 | # NETCDF_CXX - require the C++ interface and link the C++ library 11 | # NETCDF_F77 - require the F77 interface and link the fortran library 12 | # NETCDF_F90 - require the F90 interface and link the fortran library 13 | # 14 | # The following are not for general use and are included in 15 | # NETCDF_LIBRARIES if the corresponding option above is set. 16 | # 17 | # NETCDF_LIBRARIES_C - Just the C interface 18 | # NETCDF_LIBRARIES_CXX - C++ interface, if available 19 | # NETCDF_LIBRARIES_F77 - Fortran 77 interface, if available 20 | # NETCDF_LIBRARIES_F90 - Fortran 90 interface, if available 21 | # 22 | # Normal usage would be: 23 | # set (NETCDF_F90 "YES") 24 | # find_package (NetCDF REQUIRED) 25 | # target_link_libraries (uses_f90_interface ${NETCDF_LIBRARIES}) 26 | # target_link_libraries (only_uses_c_interface ${NETCDF_LIBRARIES_C}) 27 | 28 | if (NETCDF_INCLUDES AND NETCDF_LIBRARIES) 29 | # Already in cache, be silent 30 | set (NETCDF_FIND_QUIETLY TRUE) 31 | endif (NETCDF_INCLUDES AND NETCDF_LIBRARIES) 32 | 33 | find_path (NETCDF_INCLUDES netcdf.h 34 | HINTS NETCDF_DIR ENV NETCDF_DIR) 35 | 36 | find_library (NETCDF_LIBRARIES_C NAMES netcdf) 37 | mark_as_advanced(NETCDF_LIBRARIES_C) 38 | 39 | set (NetCDF_has_interfaces "YES") # will be set to NO if we're missing any interfaces 40 | set (NetCDF_libs "${NETCDF_LIBRARIES_C}") 41 | 42 | get_filename_component (NetCDF_lib_dirs "${NETCDF_LIBRARIES_C}" PATH) 43 | 44 | macro (NetCDF_check_interface lang header libs) 45 | if (NETCDF_${lang}) 46 | find_path (NETCDF_INCLUDES_${lang} NAMES ${header} 47 | HINTS "${NETCDF_INCLUDES}" NO_DEFAULT_PATH) 48 | find_library (NETCDF_LIBRARIES_${lang} NAMES ${libs} 49 | HINTS "${NetCDF_lib_dirs}" NO_DEFAULT_PATH) 50 | mark_as_advanced (NETCDF_INCLUDES_${lang} NETCDF_LIBRARIES_${lang}) 51 | if (NETCDF_INCLUDES_${lang} AND NETCDF_LIBRARIES_${lang}) 52 | list (INSERT NetCDF_libs 0 ${NETCDF_LIBRARIES_${lang}}) # prepend so that -lnetcdf is last 53 | else (NETCDF_INCLUDES_${lang} AND NETCDF_LIBRARIES_${lang}) 54 | set (NetCDF_has_interfaces "NO") 55 | message (STATUS "Failed to find NetCDF interface for ${lang}") 56 | endif (NETCDF_INCLUDES_${lang} AND NETCDF_LIBRARIES_${lang}) 57 | endif (NETCDF_${lang}) 58 | endmacro (NetCDF_check_interface) 59 | 60 | NetCDF_check_interface (CXX netcdfcpp.h netcdf_c++) 61 | NetCDF_check_interface (F77 netcdf.inc netcdff) 62 | NetCDF_check_interface (F90 netcdf.mod netcdff) 63 | 64 | set (NETCDF_LIBRARIES "${NetCDF_libs}" CACHE STRING "All NetCDF libraries required for interface level") 65 | 66 | # handle the QUIETLY and REQUIRED arguments and set NETCDF_FOUND to TRUE if 67 | # all listed variables are TRUE 68 | include (FindPackageHandleStandardArgs) 69 | find_package_handle_standard_args (NetCDF DEFAULT_MSG NETCDF_LIBRARIES NETCDF_INCLUDES NetCDF_has_interfaces) 70 | 71 | mark_as_advanced (NETCDF_LIBRARIES NETCDF_INCLUDES) 72 | -------------------------------------------------------------------------------- /src/problem/performance_testing/params.cpp: -------------------------------------------------------------------------------- 1 | #include "../../params.hpp" 2 | #include 3 | 4 | namespace params 5 | { 6 | int numDevices = 1; 7 | 8 | int N1 = 96; 9 | int N2 = 96; 10 | int N3 = 64; 11 | 12 | int dim = 3; 13 | int numGhost = 3; 14 | 15 | int timeStepper = timeStepping::EXPLICIT; 16 | double InitialDt = .5/N1; 17 | double maxDtIncrement = 1.3; 18 | double Time = 0.; 19 | double CourantFactor = 0.9; 20 | double finalTime = 2.; 21 | int metric = metrics::MINKOWSKI; 22 | int restart = 0; 23 | std::string restartFile = "restartFile.h5"; 24 | std::string restartFileName = "restartFileName.txt"; 25 | std::string restartFileTime = "restartFileTime.txt"; 26 | 27 | double X1Start = 0., X1End = 1.; 28 | double X2Start = 0., X2End = 1.; 29 | double X3Start = 0., X3End = 1.; 30 | 31 | int boundaryLeft = boundaries::PERIODIC; 32 | int boundaryRight = boundaries::PERIODIC; 33 | 34 | int boundaryTop = boundaries::PERIODIC; 35 | int boundaryBottom = boundaries::PERIODIC; 36 | 37 | int boundaryFront = boundaries::PERIODIC; 38 | int boundaryBack = boundaries::PERIODIC; 39 | 40 | double rhoFloorInFluidElement = 1e-20; 41 | double uFloorInFluidElement = 1e-20; 42 | double bSqrFloorInFluidElement = 1e-20; 43 | double temperatureFloorInFluidElement = 1e-20; 44 | 45 | int conduction = 1; 46 | int viscosity = 1; 47 | int highOrderTermsConduction = 1; 48 | int highOrderTermsViscosity = 1; 49 | double ConductionAlpha = 1.; 50 | double ViscosityAlpha = 1.; 51 | 52 | double adiabaticIndex = 4./3; 53 | double Aw = 1.e-8; 54 | double k1 = 2.*M_PI; 55 | double k2 = 4.*M_PI; 56 | double Gamma = - 0.5533585207638141; 57 | double Omega = - 3.6262571286888425; 58 | 59 | double slopeLimTheta = 2; 60 | int reconstruction = reconstructionOptions::MINMOD; 61 | int riemannSolver = riemannSolvers::LOCAL_LAX_FRIEDRICH; 62 | 63 | int maxNonLinearIter = 3; 64 | int maxLineSearchIters = 3; 65 | 66 | // Perf testing parameters 67 | int nIters = 10; 68 | 69 | //Parameters controlling accuracy of nonlinear solver 70 | double nonlinearsolve_atol = 1.e-20; 71 | double JacobianAssembleEpsilon = 4.e-8; 72 | double linesearchfloor = 1.e-24; 73 | 74 | //Unused params - do we need to define them? 75 | double hSlope = 0.3; 76 | double blackHoleSpin = 0.9375; 77 | int DerefineThetaHorizon = 1; 78 | int DoCylindrify = 1; 79 | double X1cyl = 0.; 80 | double X2cyl = 1./N2; 81 | }; 82 | 83 | namespace vars 84 | { 85 | int Q = 5; 86 | int DP = 5 + params::conduction; 87 | int numFluidVars = 5 + params::conduction + params::viscosity; 88 | 89 | int B1 = 5 + params::conduction + params::viscosity; 90 | int B2 = 6 + params::conduction + params::viscosity; 91 | int B3 = 7 + params::conduction + params::viscosity; 92 | int dof = 8 + params::conduction + params::viscosity; 93 | }; 94 | 95 | namespace dumpVars 96 | { 97 | int Q = 5; 98 | int DP = 5 + params::conduction; 99 | int numFluidVars = 5 + params::conduction + params::viscosity; 100 | 101 | int B1 = 5 + params::conduction + params::viscosity; 102 | int B2 = 6 + params::conduction + params::viscosity; 103 | int B3 = 7 + params::conduction + params::viscosity; 104 | int dof = 8 + params::conduction + params::viscosity; 105 | }; 106 | -------------------------------------------------------------------------------- /src/problem/linear_modes/params.cpp: -------------------------------------------------------------------------------- 1 | #include "../../params.hpp" 2 | #include 3 | 4 | namespace params 5 | { 6 | int numDevices = 1; 7 | 8 | int N1 = 288; 9 | int N2 = 128; 10 | int N3 = 128; 11 | 12 | int dim = 3; 13 | int numGhost = 3; 14 | 15 | int timeStepper = timeStepping::EXPLICIT; 16 | double InitialDt = .5/N1; 17 | double maxDtIncrement = 1.3; 18 | double Time = 0.; 19 | double CourantFactor = 0.9; 20 | double finalTime = 2.; 21 | int metric = metrics::MINKOWSKI; 22 | int restart = 0; 23 | std::string restartFile = "restartFile.h5"; 24 | std::string restartFileName = "restartFileName.txt"; 25 | std::string restartFileTime = "restartFileTime.txt"; 26 | 27 | double X1Start = 0., X1End = 1.; 28 | double X2Start = 0., X2End = 1.; 29 | double X3Start = 0., X3End = 1.; 30 | 31 | int boundaryLeft = boundaries::PERIODIC; 32 | int boundaryRight = boundaries::PERIODIC; 33 | 34 | int boundaryTop = boundaries::PERIODIC; 35 | int boundaryBottom = boundaries::PERIODIC; 36 | 37 | int boundaryFront = boundaries::PERIODIC; 38 | int boundaryBack = boundaries::PERIODIC; 39 | 40 | double rhoFloorInFluidElement = 1e-20; 41 | double uFloorInFluidElement = 1e-20; 42 | double bSqrFloorInFluidElement = 1e-20; 43 | double temperatureFloorInFluidElement = 1e-20; 44 | 45 | int conduction = 1; 46 | int viscosity = 1; 47 | int highOrderTermsConduction = 1; 48 | int highOrderTermsViscosity = 1; 49 | double ConductionAlpha = 1.; 50 | double ViscosityAlpha = 1.; 51 | 52 | double adiabaticIndex = 4./3; 53 | double Aw = 1.e-8; 54 | double k1 = 2.*M_PI; 55 | double k2 = 4.*M_PI; 56 | double Gamma = - 0.5533585207638141; 57 | double Omega = - 3.6262571286888425; 58 | 59 | double slopeLimTheta = 2; 60 | int reconstruction = reconstructionOptions::MINMOD; 61 | int riemannSolver = riemannSolvers::HLL; 62 | 63 | int maxNonLinearIter = 3; 64 | int maxLineSearchIters = 3; 65 | 66 | //Parameters controlling accuracy of nonlinear solver 67 | double nonlinearsolve_atol = 1.e-20; 68 | double JacobianAssembleEpsilon = 4.e-8; 69 | double linesearchfloor = 1.e-24; 70 | 71 | // Linear solver options 72 | int linearSolver = linearSolvers::GPU_BATCH_SOLVER; 73 | 74 | //Unused params - do we need to define them? 75 | double hSlope = 0.3; 76 | double blackHoleSpin = 0.9375; 77 | int DerefineThetaHorizon = 1; 78 | int DoCylindrify = 1; 79 | double X1cyl = 0.; 80 | double X2cyl = 1./N2; 81 | }; 82 | 83 | namespace vars 84 | { 85 | int Q = 5; 86 | int DP = 5 + params::conduction; 87 | int numFluidVars = 5 + params::conduction + params::viscosity; 88 | 89 | int B1 = 5 + params::conduction + params::viscosity; 90 | int B2 = 6 + params::conduction + params::viscosity; 91 | int B3 = 7 + params::conduction + params::viscosity; 92 | int dof = 8 + params::conduction + params::viscosity; 93 | }; 94 | 95 | namespace dumpVars 96 | { 97 | int Q = 5; 98 | int DP = 5 + params::conduction; 99 | int numFluidVars = 5 + params::conduction + params::viscosity; 100 | 101 | int B1 = 5 + params::conduction + params::viscosity; 102 | int B2 = 6 + params::conduction + params::viscosity; 103 | int B3 = 7 + params::conduction + params::viscosity; 104 | int dof = 8 + params::conduction + params::viscosity; 105 | }; 106 | -------------------------------------------------------------------------------- /src/problem/orzag_tang/params.cpp: -------------------------------------------------------------------------------- 1 | #include "../../params.hpp" 2 | #include 3 | 4 | namespace params 5 | { 6 | int numDevices = 1; 7 | 8 | int N1 = 256; 9 | int N2 = 256; 10 | int N3 = 1; 11 | int dim = 2; 12 | int numGhost = 3; 13 | 14 | int timeStepper = timeStepping::EXPLICIT; 15 | double InitialDt = .002; 16 | double maxDtIncrement = 1.3; 17 | double CourantFactor = 0.9; 18 | double Time = 0.; 19 | double finalTime = 50.; 20 | int metric = metrics::MINKOWSKI; 21 | int restart = 0; 22 | std::string restartFile = "restartFile.h5"; 23 | double hSlope = 0.3; 24 | 25 | int ObserveEveryNSteps = 10; 26 | int StepNumber = 0; 27 | 28 | double adiabaticIndex = 5./3; 29 | double blackHoleSpin = 0.9375; 30 | double InnerEdgeRadius = 6.; 31 | double PressureMaxRadius = 12.; 32 | double MinPlasmaBeta = 15.; 33 | double MagneticLoops = 1; 34 | double Adiabat = 0.001; 35 | 36 | double X1Start = 0., X1End = 1.; 37 | double X2Start = 0., X2End = 1.; 38 | double X3Start = 0., X3End = 1.; 39 | 40 | int boundaryLeft = boundaries::PERIODIC; 41 | int boundaryRight = boundaries::PERIODIC; 42 | 43 | int boundaryTop = boundaries::PERIODIC; 44 | int boundaryBottom = boundaries::PERIODIC; 45 | 46 | int boundaryFront = boundaries::PERIODIC; 47 | int boundaryBack = boundaries::PERIODIC; 48 | 49 | double rhoFloorInFluidElement = 1e-20; 50 | double uFloorInFluidElement = 1e-20; 51 | double bSqrFloorInFluidElement = 1e-20; 52 | double temperatureFloorInFluidElement = 1e-20; 53 | 54 | //Atmosphere parameters 55 | // Floors are Ampl*pow(radius,power) 56 | double RhoFloorAmpl = 1.e-3; 57 | double UFloorAmpl = 1.e-5; 58 | double RhoFloorSlope = -1.5; 59 | double UFloorSlope = -2.5; 60 | // Floors for magnetically dominated regions 61 | double BsqrOverRhoMax = 10.; 62 | double BsqrOverUMax = 500.; 63 | 64 | int conduction = 0; 65 | int viscosity = 0; 66 | int highOrderTermsConduction = 1.; 67 | int highOrderTermsViscosity = 1.; 68 | double ConductionAlpha = 1.; 69 | double ViscosityAlpha = 1.; 70 | double ConductionClosureFactor = 1.; 71 | double ViscosityClosureFactor = 1.; 72 | 73 | double slopeLimTheta = 2.; 74 | int reconstruction = reconstructionOptions::MINMOD; 75 | int riemannSolver = riemannSolvers::LOCAL_LAX_FRIEDRICH; 76 | 77 | int maxNonLinearIter = 5; 78 | int maxLineSearchIters = 3; 79 | 80 | //Parameters controlling accuracy of nonlinear solver 81 | double nonlinearsolve_atol = 1.e-12; 82 | double JacobianAssembleEpsilon = 4.e-8; 83 | double linesearchfloor = 1.e-24; 84 | 85 | double InitialPerturbationAmplitude = 4e-2; 86 | double ObserveEveryDt = 1.; 87 | double WriteDataEveryDt = 100.; 88 | 89 | //Unused params - do we need to define them? 90 | int DerefineThetaHorizon = 1; 91 | int DoCylindrify = 1; 92 | double X1cyl = 0.; 93 | double X2cyl = 1./N2; 94 | }; 95 | 96 | namespace vars 97 | { 98 | int Q = 5; 99 | int DP = 5 + params::conduction; 100 | int numFluidVars = 5 + params::conduction + params::viscosity; 101 | 102 | int B1 = 5 + params::conduction + params::viscosity; 103 | int B2 = 6 + params::conduction + params::viscosity; 104 | int B3 = 7 + params::conduction + params::viscosity; 105 | int dof = 8 + params::conduction + params::viscosity; 106 | }; 107 | -------------------------------------------------------------------------------- /src/problem/buoyancy_instabilities/params.cpp: -------------------------------------------------------------------------------- 1 | #include "../../params.hpp" 2 | #include 3 | 4 | namespace params 5 | { 6 | int numDevices = 4; 7 | 8 | int N1 = 64; 9 | int N2 = 64; 10 | int N3 = 128; 11 | int dim = 2; 12 | int numGhost = 3; 13 | 14 | int timeStepper = timeStepping::EXPLICIT; 15 | double InitialDt = 0.01; 16 | double CourantFactor = 0.5; 17 | double maxDtIncrement = 1.3; 18 | double Time = 0; 19 | double finalTime = 10.; 20 | 21 | int restart = 0; 22 | std::string restartFile = "restartFile.h5"; 23 | 24 | int ObserveEveryNSteps = 100; 25 | int StepNumber = 0; 26 | 27 | double adiabaticIndex = 5./3; 28 | double blackHoleSpin = 0.; 29 | double InnerEdgeRadius = 6.; 30 | double PressureMaxRadius = 12.; 31 | double MinPlasmaBeta = 15.; 32 | double MagneticLoops = 1; 33 | double Adiabat = 0.001; 34 | 35 | double Rin = 100*(1.+sqrt(1.-blackHoleSpin*blackHoleSpin)); 36 | double Rout = 300.; 37 | 38 | // Grid parameters 39 | int metric = metrics::MODIFIED_KERR_SCHILD; 40 | double hSlope = 1.; 41 | int DerefineThetaHorizon = 1; 42 | int DoCylindrify = 1; 43 | double X1cyl = log(4.*Rin); 44 | double X2cyl = 3./N2; 45 | 46 | 47 | double X1Start = log(Rin), X1End = log(Rout); 48 | double X2Start = 0.+1.e-8, X2End = 1.-1.e-8; 49 | double X3Start = 0., X3End = M_PI*1.; 50 | 51 | int boundaryLeft = boundaries::OUTFLOW; 52 | int boundaryRight = boundaries::OUTFLOW; 53 | 54 | int boundaryTop = boundaries::MIRROR; 55 | int boundaryBottom = boundaries::MIRROR; 56 | 57 | int boundaryFront = boundaries::PERIODIC; 58 | int boundaryBack = boundaries::PERIODIC; 59 | 60 | double rhoFloorInFluidElement = 1e-20; 61 | double uFloorInFluidElement = 1e-20; 62 | double bSqrFloorInFluidElement = 1e-20; 63 | double temperatureFloorInFluidElement = 1e-20; 64 | 65 | //Atmosphere parameters 66 | // Floors are Ampl*pow(radius,power) 67 | double RhoFloorAmpl = 1.e-3; 68 | double UFloorAmpl = 1.e-5; 69 | double RhoFloorSlope = -1.5; 70 | double UFloorSlope = -2.5; 71 | // Floors for magnetically dominated regions 72 | double BsqrOverRhoMax = 10.; 73 | double BsqrOverUMax = 500.; 74 | 75 | int conduction = 1; 76 | int viscosity = 0; 77 | int highOrderTermsConduction = 1.; 78 | int highOrderTermsViscosity = 1.; 79 | double ConductionAlpha = 1.; 80 | double ViscosityAlpha = 1.; 81 | double ConductionClosureFactor = 1.; 82 | double ViscosityClosureFactor = 1.; 83 | 84 | double slopeLimTheta = 2; 85 | int reconstruction = reconstructionOptions::MINMOD; 86 | int riemannSolver = riemannSolvers::LOCAL_LAX_FRIEDRICH; 87 | 88 | int maxNonLinearIter = 3; 89 | int maxLineSearchIters = 3; 90 | 91 | //Parameters controlling accuracy of nonlinear solver 92 | double nonlinearsolve_atol = 1.e-6; 93 | double JacobianAssembleEpsilon = 4.e-8; 94 | double linesearchfloor = 1.e-24; 95 | 96 | double InitialPerturbationAmplitude = 4e-2; 97 | double ObserveEveryDt = 1.; 98 | double WriteDataEveryDt = 20.; 99 | }; 100 | 101 | namespace vars 102 | { 103 | int Q = 5; 104 | int DP = 5 + params::conduction; 105 | int numFluidVars = 5 + params::conduction + params::viscosity; 106 | 107 | int B1 = 5 + params::conduction + params::viscosity; 108 | int B2 = 6 + params::conduction + params::viscosity; 109 | int B3 = 7 + params::conduction + params::viscosity; 110 | int dof = 8 + params::conduction + params::viscosity; 111 | 112 | }; 113 | -------------------------------------------------------------------------------- /src/problem/atmosphere/config.h.in: -------------------------------------------------------------------------------- 1 | #ifndef GRIM_INPUT_H_ 2 | #define GRIM_INPUT_H_ 3 | 4 | /* Immutable constants. Should not be changed */ 5 | #define NDIM (4) 6 | #define M_PI (3.141592653589793238462643383279) 7 | 8 | /* Magnetic field configurations */ 9 | #define VERTICAL (0) 10 | #define THETA (1) 11 | #define RADIAL (2) 12 | /* End of immutable constants */ 13 | 14 | #cmakedefine OUTPUT_DIR ${OUTPUT_DIR} 15 | #cmakedefine01 RESTART 16 | #cmakedefine RESTART_FILE ${RESTART_FILE} 17 | #cmakedefine DUMP_FILE_PREFIX ${DUMP_FILE_PREFIX} 18 | #cmakedefine TIME_STEPPING ${TIME_STEPPING} 19 | #cmakedefine DT (${DT}) 20 | #cmakedefine DT_DUMP (${DT_DUMP}) 21 | #cmakedefine START_TIME (${START_TIME}) 22 | #cmakedefine FINAL_TIME (${FINAL_TIME}) 23 | #cmakedefine COURANT (${COURANT}) 24 | #cmakedefine MAX_DT_INCREMENT (${MAX_DT_INCREMENT}) 25 | #cmakedefine REAL ${REAL} 26 | #cmakedefine ARRAY_ARGS ${ARRAY_ARGS} 27 | #cmakedefine PROBLEM_DATA ${PROBLEM_DATA} 28 | 29 | /* Domain inputs */ 30 | #cmakedefine COMPUTE_DIM (${COMPUTE_DIM}) 31 | #cmakedefine N1 (${N1}) 32 | #cmakedefine N2 (${N2}) 33 | #cmakedefine TILE_SIZE_X1 (${TILE_SIZE_X1}) 34 | #cmakedefine TILE_SIZE_X2 (${TILE_SIZE_X2}) 35 | #cmakedefine01 USE_OPENMP 36 | 37 | 38 | #cmakedefine ADIABATIC_INDEX (${ADIABATIC_INDEX}) 39 | #cmakedefine01 CONDUCTION 40 | 41 | 42 | /* Initial condition parameters */ 43 | #cmakedefine PLASMA_BETA (${PLASMA_BETA}) 44 | #cmakedefine PERTURBATIONS_AMPLITUDE (${PERTURBATIONS_AMPLITUDE}) 45 | #cmakedefine MAGNETIC_FIELD_CONFIGURATION (${MAGNETIC_FIELD_CONFIGURATION}) 46 | #cmakedefine RHO_INPUT_FILE ${RHO_INPUT_FILE} 47 | #cmakedefine UU_INPUT_FILE ${UU_INPUT_FILE} 48 | #cmakedefine UR_INPUT_FILE ${UR_INPUT_FILE} 49 | #cmakedefine PHI_INPUT_FILE ${PHI_INPUT_FILE} 50 | #cmakedefine RCOORDS_INPUT_FILE ${RCOORDS_INPUT_FILE} 51 | 52 | /* Geometry */ 53 | #cmakedefine METRIC (${METRIC}) 54 | #cmakedefine EPS (${EPS}) 55 | #cmakedefine M (${M}) 56 | #cmakedefine BH_SPIN (${BH_SPIN}) 57 | #cmakedefine H_SLOPE (${H_SLOPE}) 58 | 59 | /* Domain */ 60 | #cmakedefine R_A (${R_A}) 61 | #cmakedefine R_B (${R_B}) 62 | 63 | #cmakedefine X1_A (${X1_A}) 64 | #cmakedefine X1_B (${X1_B}) 65 | #cmakedefine X2_A (${X2_A}) 66 | #cmakedefine X2_B (${X2_B}) 67 | 68 | /* Boundary conditions */ 69 | #cmakedefine PHYSICAL_BOUNDARY_LEFT_EDGE (${PHYSICAL_BOUNDARY_LEFT_EDGE}) 70 | #cmakedefine PHYSICAL_BOUNDARY_RIGHT_EDGE (${PHYSICAL_BOUNDARY_RIGHT_EDGE}) 71 | #cmakedefine PHYSICAL_BOUNDARY_TOP_EDGE (${PHYSICAL_BOUNDARY_TOP_EDGE}) 72 | #cmakedefine PHYSICAL_BOUNDARY_BOTTOM_EDGE (${PHYSICAL_BOUNDARY_BOTTOM_EDGE}) 73 | 74 | /* Reconstruction */ 75 | #cmakedefine RECONSTRUCTION (${RECONSTRUCTION}) 76 | 77 | /* Floor values */ 78 | #cmakedefine RHO_FLOOR (${RHO_FLOOR}) 79 | #cmakedefine UU_FLOOR (${UU_FLOOR}) 80 | #cmakedefine RHO_FLOOR_FALLOFF (${RHO_FLOOR_FALLOFF}) 81 | #cmakedefine UU_FLOOR_FALLOFF (${UU_FLOOR_FALLOFF}) 82 | #cmakedefine RHO_FLOOR_MIN (${RHO_FLOOR_MIN}) 83 | #cmakedefine UU_FLOOR_MIN (${UU_FLOOR_MIN}) 84 | #cmakedefine GAMMA_MAX (${GAMMA_MAX}) 85 | 86 | /* Number of ghost zones */ 87 | #if (RECONSTRUCTION==MONOTONIZED_CENTRAL || RECONSTRUCTION==MIN_MOD) 88 | #define NG (3) 89 | #elif (RECONSTRUCTION==MP5) 90 | #define NG (4) 91 | #endif 92 | 93 | #endif /* GRIM_INPUT_H_ */ 94 | -------------------------------------------------------------------------------- /src/cmake/modules/FindYAML.cmake: -------------------------------------------------------------------------------- 1 | 2 | #-------------------------------------------------------------------------------- 3 | # Copyright (c) 2012-2013, Lars Baehren 4 | # All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without modification, 7 | # are permitted provided that the following conditions are met: 8 | # 9 | # * Redistributions of source code must retain the above copyright notice, this 10 | # list of conditions and the following disclaimer. 11 | # * Redistributions in binary form must reproduce the above copyright notice, 12 | # this list of conditions and the following disclaimer in the documentation 13 | # and/or other materials provided with the distribution. 14 | # 15 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 16 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 18 | # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 19 | # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 22 | # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 23 | # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 24 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 | #-------------------------------------------------------------------------------- 26 | 27 | # - Check for the presence of YAML 28 | # 29 | # The following variables are set when YAML is found: 30 | # YAML_FOUND = Set to true, if all components of YAML have been found. 31 | # YAML_INCLUDES = Include path for the header files of YAML 32 | # YAML_LIBRARIES = Link these to use YAML 33 | # YAML_LFLAGS = Linker flags (optional) 34 | 35 | if (NOT YAML_FOUND) 36 | 37 | if (NOT YAML_ROOT_DIR) 38 | set (YAML_ROOT_DIR ${CMAKE_INSTALL_PREFIX}) 39 | endif (NOT YAML_ROOT_DIR) 40 | 41 | ##_____________________________________________________________________________ 42 | ## Check for the header files 43 | 44 | find_path (YAML_INCLUDES yaml-cpp/yaml.h yaml-cpp/node.h 45 | HINTS ${YAML_ROOT_DIR} ${CMAKE_INSTALL_PREFIX} 46 | PATH_SUFFIXES include 47 | ) 48 | 49 | ##_____________________________________________________________________________ 50 | ## Check for the library 51 | 52 | find_library (YAML_LIBRARIES yaml-cpp 53 | HINTS ${YAML_ROOT_DIR} ${CMAKE_INSTALL_PREFIX} 54 | PATH_SUFFIXES lib 55 | ) 56 | 57 | ##_____________________________________________________________________________ 58 | ## Actions taken when all components have been found 59 | 60 | find_package_handle_standard_args (YAML DEFAULT_MSG YAML_LIBRARIES YAML_INCLUDES) 61 | 62 | if (YAML_INCLUDES AND YAML_LIBRARIES) 63 | set (YAML_FOUND TRUE) 64 | else (YAML_INCLUDES AND YAML_LIBRARIES) 65 | set (YAML_FOUND FALSE) 66 | if (NOT YAML_FIND_QUIETLY) 67 | if (NOT YAML_INCLUDES) 68 | message (STATUS "Unable to find YAML header files!") 69 | endif (NOT YAML_INCLUDES) 70 | if (NOT YAML_LIBRARIES) 71 | message (STATUS "Unable to find YAML library files!") 72 | endif (NOT YAML_LIBRARIES) 73 | endif (NOT YAML_FIND_QUIETLY) 74 | endif (YAML_INCLUDES AND YAML_LIBRARIES) 75 | 76 | if (YAML_FOUND) 77 | if (NOT YAML_FIND_QUIETLY) 78 | message (STATUS "Found components for YAML") 79 | message (STATUS "YAML_ROOT_DIR = ${YAML_ROOT_DIR}") 80 | message (STATUS "YAML_INCLUDES = ${YAML_INCLUDES}") 81 | message (STATUS "YAML_LIBRARIES = ${YAML_LIBRARIES}") 82 | endif (NOT YAML_FIND_QUIETLY) 83 | else (YAML_FOUND) 84 | if (YAML_FIND_REQUIRED) 85 | message (FATAL_ERROR "Could not find YAML!") 86 | endif (YAML_FIND_REQUIRED) 87 | endif (YAML_FOUND) 88 | 89 | ## Compatibility setting 90 | set (YAML_CPP_FOUND ${YAML_FOUND}) 91 | 92 | ##_____________________________________________________________________________ 93 | ## Mark advanced variables 94 | 95 | mark_as_advanced ( 96 | YAML_ROOT_DIR 97 | YAML_INCLUDES 98 | YAML_LIBRARIES 99 | ) 100 | 101 | endif (NOT YAML_FOUND) 102 | -------------------------------------------------------------------------------- /src/timestepper/timestepper.hpp: -------------------------------------------------------------------------------- 1 | #ifndef GRIM_TIMESTEPPER_H_ 2 | #define GRIM_TIMESTEPPER_H_ 3 | 4 | #include 5 | #include "../params.hpp" 6 | #include "../grid/grid.hpp" 7 | #include "../physics/physics.hpp" 8 | #include "../geometry/geometry.hpp" 9 | #include "../boundary/boundary.hpp" 10 | 11 | namespace timeStepperSwitches 12 | { 13 | enum 14 | { 15 | HALF_STEP, FULL_STEP 16 | }; 17 | }; 18 | 19 | class timeStepper 20 | { 21 | int world_rank, world_size; 22 | 23 | grid *primGuessLineSearchTrial; 24 | grid *primGuessPlusEps; 25 | grid *primIC; 26 | grid *residual; 27 | grid *residualPlusEps; 28 | 29 | array residualSoA; 30 | array jacobianSoA; 31 | array deltaPrimAoS; 32 | array stepLength; 33 | 34 | double *AHostPtr, *bHostPtr; 35 | 36 | void solve(grid &primGuess); 37 | void computeResidual(const grid &prim, grid &residual, 38 | int &numReads, 39 | int &numWrites 40 | ); 41 | void batchLinearSolve(const array &A, const array &b, array &x); 42 | double linearSolverTime; 43 | double lineSearchTime; 44 | double jacobianAssemblyTime; 45 | 46 | af::seq domainX1, domainX2, domainX3; 47 | array residualMask; 48 | 49 | double memoryBandwidth(const double numReads, 50 | const double numWrites, 51 | const double numEvals, 52 | const double timeElapsed 53 | ); 54 | 55 | double bandwidthTest(const int numEvals); 56 | 57 | public: 58 | double dt, time; 59 | int N1, N2, N3, numGhost, dim; 60 | int numVars; 61 | 62 | int boundaryLeft, boundaryRight; 63 | int boundaryTop, boundaryBottom; 64 | int boundaryFront, boundaryBack; 65 | 66 | coordinatesGrid *XCoords; 67 | grid *prim, *primHalfStep, *primOld; 68 | grid *cons, *consOld; 69 | grid *sourcesExplicit; 70 | grid *sourcesImplicit; 71 | grid *sourcesImplicitOld; 72 | grid *sourcesTimeDer; 73 | grid *primLeft, *primRight; 74 | grid *fluxesX1, *fluxesX2, *fluxesX3; 75 | grid *emfX1, *emfX2, *emfX3; 76 | grid *divFluxes; 77 | grid *divB; 78 | grid *dump; 79 | 80 | geometry *geomLeft, *geomRight; 81 | geometry *geomBottom, *geomTop; 82 | geometry *geomCenter; 83 | 84 | fluidElement *elem, *elemOld, *elemHalfStep; 85 | 86 | riemannSolver *riemann; 87 | 88 | void computeDivOfFluxes(const grid &prim, 89 | int &numReads, int &numWrites 90 | ); 91 | 92 | int currentStep; 93 | 94 | timeStepper(const int N1, 95 | const int N2, 96 | const int N3, 97 | const int dim, 98 | const int numVars, 99 | const int numGhost, 100 | const double time, 101 | const double dt, 102 | const int boundaryLeft, const int boundaryRight, 103 | const int boundaryTop, const int boundaryBottom, 104 | const int boundaryFront, const int boundaryBack, 105 | const int metric, 106 | const double blackHoleSpin, 107 | const double hSlope, 108 | const double X1Start, const double X1End, 109 | const double X2Start, const double X2End, 110 | const double X3Start, const double X3End 111 | ); 112 | ~timeStepper(); 113 | 114 | void timeStep(int &numReads, int &numWrites); 115 | 116 | void fluxCT(int &numReads, int &numWrites); 117 | void computeEMF(int &numReadsEMF, int &numWritesEMF); 118 | void computeDivB(const grid &prim, 119 | int &numReads, 120 | int &numWrites 121 | ); 122 | 123 | void computeDt(int &numReads, int &numWrites); 124 | 125 | /* Function definitions in the problem folder */ 126 | void initialConditions(int &numReads, int &numWrites); 127 | void halfStepDiagnostics(int &numReads, int &numWrites); 128 | void fullStepDiagnostics(int &numReads, int &numWrites); 129 | void setProblemSpecificBCs(int &numReads, int &numWrites); 130 | void applyProblemSpecificFluxFilter(int &numReads, int &numWrites); 131 | int CheckWallClockTermination(); 132 | }; 133 | 134 | #endif /* GRIM_TIMESTEPPER_H_ */ 135 | -------------------------------------------------------------------------------- /src/physics/physics.hpp: -------------------------------------------------------------------------------- 1 | #ifndef GRIM_PHYSICS_H_ 2 | #define GRIM_PHYSICS_H_ 3 | 4 | #include "../params.hpp" 5 | #include "../grid/grid.hpp" 6 | #include "../geometry/geometry.hpp" 7 | #include "../reconstruction/reconstruction.hpp" 8 | 9 | inline int DELTA(int const &mu, int const &nu) 10 | { 11 | return (mu==nu ? 1 : 0); 12 | } 13 | 14 | class fluidElement 15 | { 16 | void computeEMHDGradients(const double dX[3], 17 | int &numReads, 18 | int &numWrites 19 | ); 20 | public: 21 | array zero, one; 22 | 23 | /* fluidElement parameters */ 24 | array tau, chi_emhd, nu_emhd; 25 | 26 | array rho, u, u1, u2, u3, B1, B2, B3; 27 | array pressure, temperature; 28 | array qTilde, deltaPTilde; 29 | array q, deltaP; 30 | 31 | array gammaLorentzFactor, uCon[NDIM], uCov[NDIM]; 32 | array bSqr, bCon[NDIM], bCov[NDIM]; 33 | array soundSpeed; 34 | 35 | array NUp[NDIM]; 36 | array TUpDown[NDIM][NDIM]; 37 | 38 | array gradT[NDIM]; 39 | array dtuCov[NDIM]; 40 | array graduCov[NDIM][NDIM]; 41 | array divuCov; 42 | array deltaP0; 43 | array q0; 44 | array bNorm; 45 | 46 | fluidElement(const grid &prim, 47 | geometry &geom, 48 | int &numReads, 49 | int &numWrites 50 | ); 51 | ~fluidElement(); 52 | 53 | void set(const grid &prim, 54 | geometry &geom, 55 | int &numReads, 56 | int &numWrites 57 | ); 58 | void setFluidElementParameters(); 59 | void computeFluxes(const int direction, 60 | grid &flux, 61 | int &numReads, 62 | int &numWrites 63 | ); 64 | 65 | void computeMinMaxCharSpeeds(const int dir, 66 | array &MinSpeed, 67 | array &MaxSpeed, 68 | int &numReads, 69 | int &numWrites 70 | ); 71 | 72 | void computeTimeDerivSources(const fluidElement &elemOld, 73 | const fluidElement &elemNew, 74 | const double dt, 75 | grid &sources, 76 | int &numReads, 77 | int &numWrites 78 | ); 79 | 80 | void computeImplicitSources(grid &sources, 81 | array &tauDamp, 82 | int &numReads, 83 | int &numWrites 84 | ); 85 | 86 | void computeExplicitSources(const double dX[3], 87 | grid &sources, 88 | int &numReads, 89 | int &numWrites 90 | ); 91 | 92 | void constructTetrads(); 93 | void tetradConToCoordCon(const array vTetrad[NDIM], array vCoord[NDIM]); 94 | void coordConToTetradCon(const array vCoord[NDIM], array vTetrad[NDIM]); 95 | 96 | private: 97 | array eCon[NDIM][NDIM]; 98 | array eCov[NDIM][NDIM]; 99 | geometry *geom; 100 | 101 | void normalize(array vCon[NDIM] 102 | ); 103 | void projectOut(const array vConB[NDIM], 104 | array vConA[NDIM] 105 | ); 106 | void normalizeNull(array vCon[NDIM] 107 | ); 108 | }; 109 | 110 | class riemannSolver 111 | { 112 | public: 113 | fluidElement *elemFace; 114 | 115 | grid *fluxLeft, *fluxRight; 116 | grid *consLeft, *consRight; 117 | 118 | array minSpeedLeft, maxSpeedLeft; 119 | array minSpeedRight, maxSpeedRight; 120 | 121 | riemannSolver(const grid &prim, geometry &geom); 122 | ~riemannSolver(); 123 | 124 | void solve(const grid &primLeft, 125 | const grid &primRight, 126 | geometry &geomLeft, 127 | geometry &geomRight, 128 | const int dir, 129 | grid &flux, 130 | int &numReads, 131 | int &numWrites 132 | ); 133 | }; 134 | 135 | #endif /* GRIM_PHYSICS_H_ */ 136 | -------------------------------------------------------------------------------- /src/timestepper/test_CT.py: -------------------------------------------------------------------------------- 1 | import mpi4py, petsc4py 2 | from petsc4py import PETSc 3 | import numpy as np 4 | import pylab as pl 5 | import pytest 6 | import gridPy 7 | import geometryPy 8 | import boundaryPy 9 | import timeStepperPy 10 | 11 | petsc4py.init() 12 | petscComm = petsc4py.PETSc.COMM_WORLD 13 | comm = petscComm.tompi4py() 14 | rank = comm.Get_rank() 15 | numProcs = comm.Get_size() 16 | PETSc.Sys.Print("Using %d procs" % numProcs) 17 | 18 | N1 = int(pytest.config.getoption('N1')) 19 | N2 = int(pytest.config.getoption('N2')) 20 | N3 = int(pytest.config.getoption('N3')) 21 | dim = int(pytest.config.getoption('dim')) 22 | 23 | # Geometry parameters 24 | blackHoleSpin = float(pytest.config.getoption('blackHoleSpin')) 25 | hSlope = float(pytest.config.getoption('hSlope')) 26 | numGhost = 3 27 | 28 | Rin = 0.98*(1.+np.sqrt(1.-blackHoleSpin*blackHoleSpin)); 29 | Rout = 40. 30 | 31 | #X1Start = np.log(Rin); X1End = np.log(Rout) 32 | #X2Start = 1e-8; X2End = 1.-1e-8 33 | #X3Start = 0.; X3End = 2.*np.pi 34 | #boundaryLeft = boundaryPy.OUTFLOW 35 | #boundaryRight = boundaryPy.OUTFLOW 36 | #boundaryTop = boundaryPy.OUTFLOW 37 | #boundaryBottom = boundaryPy.OUTFLOW 38 | #boundaryFront = boundaryPy.PERIODIC 39 | #boundaryBack = boundaryPy.PERIODIC 40 | 41 | X1Start = 0.; X1End = 1. 42 | X2Start = 0.; X2End = 1. 43 | X3Start = 0.; X3End = 1. 44 | 45 | boundaryLeft = boundaryPy.PERIODIC 46 | boundaryRight = boundaryPy.PERIODIC 47 | boundaryTop = boundaryPy.PERIODIC 48 | boundaryBottom = boundaryPy.PERIODIC 49 | boundaryFront = boundaryPy.PERIODIC 50 | boundaryBack = boundaryPy.PERIODIC 51 | 52 | 53 | time = 0. 54 | dt = 0.002 55 | numVars = 8 56 | #metric = geometryPy.MODIFIED_KERR_SCHILD 57 | metric = geometryPy.MINKOWSKI 58 | ts = timeStepperPy.timeStepperPy(N1, N2, N3, 59 | dim, numVars, numGhost, 60 | time, dt, 61 | boundaryLeft, boundaryRight, 62 | boundaryTop, boundaryBottom, 63 | boundaryFront, boundaryBack, 64 | metric, blackHoleSpin, hSlope, 65 | X1Start, X1End, 66 | X2Start, X2End, 67 | X3Start, X3End 68 | ) 69 | #ts.computeDivB(ts.primOld) 70 | #print "divB.shape = ", ts.divB.shape 71 | #print "Div B = ", np.max(np.abs(ts.divB.getVars()[0, 0, numGhost:N2+numGhost, 72 | # numGhost:N1+numGhost])) 73 | #pl.contourf(np.log10(np.abs(ts.divB.getVars()[0, 0,numGhost:N2+numGhost, 74 | # numGhost:N1+numGhost])), 100) 75 | #pl.colorbar() 76 | #pl.savefig("divB_initial_conditions.png") 77 | #pl.clf() 78 | #ts.timeStep() 79 | #ts.computeDivB(ts.primOld) 80 | #pl.contourf(np.log10(np.abs(ts.divB.getVars()[0, 0,numGhost:N2+numGhost, 81 | # numGhost:N1+numGhost])), 100) 82 | #pl.colorbar() 83 | #pl.savefig("divB_timestepped.png") 84 | 85 | #for n in xrange(10): 86 | # ts.timeStep() 87 | #ts.computeDivOfFluxes(ts.primHalfStep) 88 | #pl.contourf(ts.emfX3.getVars()[0, 0,numGhost:N2+numGhost, 89 | # numGhost:N1+numGhost], 100) 90 | #pl.colorbar() 91 | #pl.savefig("EMF_initial_conditions.png") 92 | #np.savetxt("emf.txt", ts.divFluxes.getVars()[5, 0, numGhost:N2+numGhost, 93 | # numGhost:N1+numGhost 94 | # ] 95 | # ) 96 | 97 | 98 | for n in xrange(1000): 99 | print "Time step = ", n 100 | ts.timeStep() 101 | ts.computeDivB(ts.primHalfStep) 102 | print "Div B primHalfStep = ", \ 103 | np.max(np.abs(ts.divB.getVars()[0, 0, numGhost:N2+numGhost, 104 | numGhost:N1+numGhost] 105 | ) 106 | ) 107 | ts.computeDivB(ts.primOld) 108 | print "Div B primOld = ", \ 109 | np.max(np.abs(ts.divB.getVars()[0, 0, numGhost:N2+numGhost, numGhost:N1+numGhost])) 110 | 111 | pl.contourf(np.log10(np.abs(ts.divB.getVars()[0, 0,numGhost:N2+numGhost, 112 | numGhost:N1+numGhost])), 100) 113 | pl.colorbar() 114 | pl.savefig("divB_" + str(n) + ".png") 115 | pl.clf() 116 | 117 | pl.contourf(ts.primOld.getVars()[0, 0, :, :], 100) 118 | pl.savefig("rho_" + str(n) + ".png") 119 | pl.clf() 120 | -------------------------------------------------------------------------------- /src/cmake/modules/ResolveCompilerPaths.cmake: -------------------------------------------------------------------------------- 1 | # ResolveCompilerPaths - this module defines two macros 2 | # 3 | # RESOLVE_LIBRARIES (XXX_LIBRARIES LINK_LINE) 4 | # This macro is intended to be used by FindXXX.cmake modules. 5 | # It parses a compiler link line and resolves all libraries 6 | # (-lfoo) using the library path contexts (-L/path) in scope. 7 | # The result in XXX_LIBRARIES is the list of fully resolved libs. 8 | # Example: 9 | # 10 | # RESOLVE_LIBRARIES (FOO_LIBRARIES "-L/A -la -L/B -lb -lc -ld") 11 | # 12 | # will be resolved to 13 | # 14 | # FOO_LIBRARIES:STRING="/A/liba.so;/B/libb.so;/A/libc.so;/usr/lib/libd.so" 15 | # 16 | # if the filesystem looks like 17 | # 18 | # /A: liba.so libc.so 19 | # /B: liba.so libb.so 20 | # /usr/lib: liba.so libb.so libc.so libd.so 21 | # 22 | # and /usr/lib is a system directory. 23 | # 24 | # Note: If RESOLVE_LIBRARIES() resolves a link line differently from 25 | # the native linker, there is a bug in this macro (please report it). 26 | # 27 | # RESOLVE_INCLUDES (XXX_INCLUDES INCLUDE_LINE) 28 | # This macro is intended to be used by FindXXX.cmake modules. 29 | # It parses a compile line and resolves all includes 30 | # (-I/path/to/include) to a list of directories. Other flags are ignored. 31 | # Example: 32 | # 33 | # RESOLVE_INCLUDES (FOO_INCLUDES "-I/A -DBAR='\"irrelevant -I/string here\"' -I/B") 34 | # 35 | # will be resolved to 36 | # 37 | # FOO_INCLUDES:STRING="/A;/B" 38 | # 39 | # assuming both directories exist. 40 | # Note: as currently implemented, the -I/string will be picked up mistakenly (cry, cry) 41 | include (CorrectWindowsPaths) 42 | 43 | macro (RESOLVE_LIBRARIES LIBS LINK_LINE) 44 | string (REGEX MATCHALL "((-L|-l|-Wl)([^\" ]+|\"[^\"]+\")|[^\" ]+\\.(a|so|dll|lib))" _all_tokens "${LINK_LINE}") 45 | set (_libs_found) 46 | set (_directory_list) 47 | foreach (token ${_all_tokens}) 48 | if (token MATCHES "-L([^\" ]+|\"[^\"]+\")") 49 | # If it's a library path, add it to the list 50 | string (REGEX REPLACE "^-L" "" token ${token}) 51 | string (REGEX REPLACE "//" "/" token ${token}) 52 | convert_cygwin_path(token) 53 | list (APPEND _directory_list ${token}) 54 | elseif (token MATCHES "^(-l([^\" ]+|\"[^\"]+\")|[^\" ]+\\.(a|so|dll|lib))") 55 | # It's a library, resolve the path by looking in the list and then (by default) in system directories 56 | if (WIN32) #windows expects "libfoo", linux expects "foo" 57 | string (REGEX REPLACE "^-l" "lib" token ${token}) 58 | else (WIN32) 59 | string (REGEX REPLACE "^-l" "" token ${token}) 60 | endif (WIN32) 61 | set (_root) 62 | if (token MATCHES "^/") # We have an absolute path 63 | #separate into a path and a library name: 64 | string (REGEX MATCH "[^/]*\\.(a|so|dll|lib)$" libname ${token}) 65 | string (REGEX MATCH ".*[^${libname}$]" libpath ${token}) 66 | convert_cygwin_path(libpath) 67 | set (_directory_list ${_directory_list} ${libpath}) 68 | set (token ${libname}) 69 | endif (token MATCHES "^/") 70 | set (_lib "NOTFOUND" CACHE FILEPATH "Cleared" FORCE) 71 | find_library (_lib ${token} HINTS ${_directory_list} ${_root}) 72 | if (_lib) 73 | string (REPLACE "//" "/" _lib ${_lib}) 74 | list (APPEND _libs_found ${_lib}) 75 | else (_lib) 76 | message (STATUS "Unable to find library ${token}") 77 | endif (_lib) 78 | endif (token MATCHES "-L([^\" ]+|\"[^\"]+\")") 79 | endforeach (token) 80 | set (_lib "NOTFOUND" CACHE INTERNAL "Scratch variable" FORCE) 81 | # only the LAST occurence of each library is required since there should be no circular dependencies 82 | if (_libs_found) 83 | list (REVERSE _libs_found) 84 | list (REMOVE_DUPLICATES _libs_found) 85 | list (REVERSE _libs_found) 86 | endif (_libs_found) 87 | set (${LIBS} "${_libs_found}") 88 | endmacro (RESOLVE_LIBRARIES) 89 | 90 | macro (RESOLVE_INCLUDES INCS COMPILE_LINE) 91 | string (REGEX MATCHALL "-I([^\" ]+|\"[^\"]+\")" _all_tokens "${COMPILE_LINE}") 92 | set (_incs_found) 93 | foreach (token ${_all_tokens}) 94 | string (REGEX REPLACE "^-I" "" token ${token}) 95 | string (REGEX REPLACE "//" "/" token ${token}) 96 | convert_cygwin_path(token) 97 | if (EXISTS ${token}) 98 | list (APPEND _incs_found ${token}) 99 | else (EXISTS ${token}) 100 | message (STATUS "Include directory ${token} does not exist") 101 | endif (EXISTS ${token}) 102 | endforeach (token) 103 | list (REMOVE_DUPLICATES _incs_found) 104 | set (${INCS} "${_incs_found}") 105 | endmacro (RESOLVE_INCLUDES) 106 | -------------------------------------------------------------------------------- /src/problem/advection_test/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Dump folder 2 | set(OUTPUT_DIR "\"/home/mc/data/mc/PhD/grim/build/\"") 3 | set(RESTART "OFF") 4 | set(RESTART_FILE "\"restartfile.h5\"") 5 | set(DUMP_FILE_PREFIX "\"data\"") 6 | set(PROBLEM_DATA "\"${PROBLEM}/problemdata.h\"") 7 | 8 | # Time stepping options: EXPLICIT, IMEX or IMPLICIT 9 | set(TIME_STEPPING "EXPLICIT") 10 | set(DT "0.002") 11 | set(DT_DUMP ".1") 12 | set(START_TIME "0.") 13 | set(FINAL_TIME "10./7.") 14 | set(COURANT "0.5") 15 | set(MAX_DT_INCREMENT "1.3") 16 | 17 | # Domain size. If the problem is 1D then N2 is ignored. 18 | set(COMPUTE_DIM "2") 19 | set(N1 "70") 20 | set(N2 "70") 21 | 22 | # The entire global domain is divided into tiles that are small enough to fit 23 | # into the cache of the compute node or an accelerator. This technique 24 | # optimizes cache usage and makes explicit use of the deep memory hierarchies 25 | # which are prevalent in all kinds of processors and accelerators. 26 | # 27 | # Caution: 1) The tile sizes need to divide the global domain size in each 28 | # direction exactly! 29 | # 2) We want a tile size that is as large as the cache. Too large a 30 | # size and the code can fail silently because local memory in the 31 | # OpenCL specification is a different amount for different 32 | # machines. 33 | # NEED TO PUT A CHECK FOR THIS. 34 | set(TILE_SIZE_X1 "10") 35 | set(TILE_SIZE_X2 "10") 36 | 37 | set(USE_OPENMP "YES") 38 | 39 | # Physics variables 40 | set(ADIABATIC_INDEX "4./3") 41 | set(CONDUCTION "OFF") 42 | 43 | # Geometry 44 | set(METRIC "MINKOWSKI") 45 | set(EPS "1e-5") 46 | 47 | # Domain 48 | set(X1_A "0.") 49 | set(X1_B "1.") 50 | set(X2_A "0.") 51 | set(X2_B "1.") 52 | 53 | # Boundary conditions 54 | set(PHYSICAL_BOUNDARY_LEFT_EDGE "PERIODIC") 55 | set(PHYSICAL_BOUNDARY_RIGHT_EDGE "PERIODIC") 56 | set(PHYSICAL_BOUNDARY_TOP_EDGE "PERIODIC") 57 | set(PHYSICAL_BOUNDARY_BOTTOM_EDGE "PERIODIC") 58 | 59 | # Reconstrution options 60 | # MP5, MONOTONIZED_CENTRAL or MIN_MOD 61 | set(RECONSTRUCTION "MONOTONIZED_CENTRAL") 62 | 63 | # Initial condition parameters 64 | # V1, V2, V3 -- 3 velocities 65 | # Parameters taken from the HARM paper, section 4.3 66 | set(V1 "0.7") 67 | set(V2 "0.7") 68 | set(V3 "0.") 69 | 70 | # Floor values 71 | set(RHO_FLOOR "1e-5") 72 | SET(UU_FLOOR "1e-7") 73 | 74 | message("") 75 | message("#########################") 76 | message("# Configuration options #") 77 | message("#########################") 78 | message("") 79 | message("Problem : " ${PROBLEM}) 80 | message("Output dir : " ${OUTPUT_DIR}) 81 | message("Time stepping : " ${TIME_STEPPING}) 82 | message("Dimensions : " ${COMPUTE_DIM}) 83 | message("Resolution : " ${N1} " x " ${N2}) 84 | message("Tile size : " ${TILE_SIZE_X1} " x " ${TILE_SIZE_X2}) 85 | message("Metric : " ${METRIC}) 86 | 87 | message("") 88 | message("##################################") 89 | message("# Domain and boundary conditions #") 90 | message("##################################") 91 | message("") 92 | message(" " ${PHYSICAL_BOUNDARY_TOP_EDGE}) 93 | message(" (" ${X1_A},${X2_B} ")+-----------+" "(" ${X1_B},${X2_B} ")" ) 94 | message( " | |" ) 95 | message( " | |" ) 96 | message(" " ${PHYSICAL_BOUNDARY_LEFT_EDGE} " | | " ${PHYSICAL_BOUNDARY_RIGHT_EDGE}) 97 | message( " | |" ) 98 | message( " | |" ) 99 | message(" (" ${X1_A},${X2_A} ")+-----------+" "(" ${X1_B},${X2_A} ")" ) 100 | message(" " ${PHYSICAL_BOUNDARY_BOTTOM_EDGE}) 101 | 102 | message("") 103 | message("###################") 104 | message("# Physics options #") 105 | message("###################") 106 | message("") 107 | message("Adiabatic index : " ${ADIABATIC_INDEX}) 108 | message("Conduction : " ${CONDUCTION}) 109 | 110 | message("") 111 | message("##################") 112 | message("# Reconstruction #") 113 | message("##################") 114 | message("") 115 | message("Reconstruction : " ${RECONSTRUCTION}) 116 | 117 | message("") 118 | message("######################") 119 | message("# Initial conditions #") 120 | message("######################") 121 | message("") 122 | message("V1 : " ${V1}) 123 | message("V2 : " ${V2}) 124 | message("V3 : " ${V3}) 125 | 126 | message("") 127 | message("##########") 128 | message("# Floors #") 129 | message("##########") 130 | message("") 131 | message("Density floor : " ${RHO_FLOOR}) 132 | message("Internal energy floor : " ${UU_FLOOR}) 133 | -------------------------------------------------------------------------------- /src/problem/magnetized_field_loop_advection/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Dump folder 2 | set(OUTPUT_DIR "\"/home/mc/data/mc/PhD/grim/build/\"") 3 | set(RESTART "OFF") 4 | set(RESTART_FILE "\"restartfile.h5\"") 5 | set(DUMP_FILE_PREFIX "\"data\"") 6 | set(PROBLEM_DATA "\"${PROBLEM}/problemdata.h\"") 7 | 8 | # Time stepping options: EXPLICIT, IMEX or IMPLICIT 9 | set(TIME_STEPPING "EXPLICIT") 10 | set(DT "0.002") 11 | set(DT_DUMP ".1") 12 | set(START_TIME "0.") 13 | set(FINAL_TIME "10.") 14 | set(COURANT "0.5") 15 | set(MAX_DT_INCREMENT "1.3") 16 | 17 | # Domain size. If the problem is 1D then N2 is ignored. 18 | set(COMPUTE_DIM "2") 19 | set(N1 "280") 20 | set(N2 "280") 21 | 22 | # The entire global domain is divided into tiles that are small enough to fit 23 | # into the cache of the compute node or an accelerator. This technique 24 | # optimizes cache usage and makes explicit use of the deep memory hierarchies 25 | # which are prevalent in all kinds of processors and accelerators. 26 | # 27 | # Caution: 1) The tile sizes need to divide the global domain size in each 28 | # direction exactly! 29 | # 2) We want a tile size that is as large as the cache. Too large a 30 | # size and the code can fail silently because local memory in the 31 | # OpenCL specification is a different amount for different 32 | # machines. 33 | # NEED TO PUT A CHECK FOR THIS. 34 | set(TILE_SIZE_X1 "10") 35 | set(TILE_SIZE_X2 "10") 36 | set(USE_OPENMP "NO") 37 | 38 | # Physics variables 39 | set(ADIABATIC_INDEX "4./3") 40 | set(CONDUCTION "OFF") 41 | 42 | # Geometry 43 | set(METRIC "MINKOWSKI") 44 | set(EPS "1e-5") 45 | 46 | # Domain 47 | set(X1_A "0.") 48 | set(X1_B "1.") 49 | set(X2_A "0.") 50 | set(X2_B "1.") 51 | 52 | # Boundary conditions 53 | set(PHYSICAL_BOUNDARY_LEFT_EDGE "PERIODIC") 54 | set(PHYSICAL_BOUNDARY_RIGHT_EDGE "PERIODIC") 55 | set(PHYSICAL_BOUNDARY_TOP_EDGE "PERIODIC") 56 | set(PHYSICAL_BOUNDARY_BOTTOM_EDGE "PERIODIC") 57 | 58 | # Reconstrution options 59 | # MP5, MONOTONIZED_CENTRAL or MIN_MOD 60 | set(RECONSTRUCTION "MP5") 61 | 62 | # Initial condition parameters 63 | # V1, V2, V3 -- 3 velocities 64 | # A0, R -- magnitude and width of the field loop 65 | # Parameters taken from K. Beckwith and J. Stone, 2011 66 | set(V1 "0.2/sqrt(6)") 67 | set(V2 "0.1/sqrt(6)") 68 | set(V3 "0.1/sqrt(6)") 69 | set(A0 "1e-3") 70 | set(R "0.3") 71 | 72 | # Floor values 73 | set(RHO_FLOOR "1e-5") 74 | SET(UU_FLOOR "1e-7") 75 | 76 | message("") 77 | message("#########################") 78 | message("# Configuration options #") 79 | message("#########################") 80 | message("") 81 | message("Problem : " ${PROBLEM}) 82 | message("Output dir : " ${OUTPUT_DIR}) 83 | message("Time stepping : " ${TIME_STEPPING}) 84 | message("Dimensions : " ${COMPUTE_DIM}) 85 | message("Resolution : " ${N1} " x " ${N2}) 86 | message("Tile size : " ${TILE_SIZE_X1} " x " ${TILE_SIZE_X2}) 87 | message("Metric : " ${METRIC}) 88 | 89 | message("") 90 | message("##################################") 91 | message("# Domain and boundary conditions #") 92 | message("##################################") 93 | message("") 94 | message(" " ${PHYSICAL_BOUNDARY_TOP_EDGE}) 95 | message(" (" ${X1_A},${X2_B} ")+-----------+" "(" ${X1_B},${X2_B} ")" ) 96 | message( " | |" ) 97 | message( " | |" ) 98 | message(" " ${PHYSICAL_BOUNDARY_LEFT_EDGE} " | | " ${PHYSICAL_BOUNDARY_RIGHT_EDGE}) 99 | message( " | |" ) 100 | message( " | |" ) 101 | message(" (" ${X1_A},${X2_A} ")+-----------+" "(" ${X1_B},${X2_A} ")" ) 102 | message(" " ${PHYSICAL_BOUNDARY_BOTTOM_EDGE}) 103 | 104 | message("") 105 | message("###################") 106 | message("# Physics options #") 107 | message("###################") 108 | message("") 109 | message("Adiabatic index : " ${ADIABATIC_INDEX}) 110 | message("Conduction : " ${CONDUCTION}) 111 | 112 | message("") 113 | message("##################") 114 | message("# Reconstruction #") 115 | message("##################") 116 | message("") 117 | message("Reconstruction : " ${RECONSTRUCTION}) 118 | 119 | message("") 120 | message("######################") 121 | message("# Initial conditions #") 122 | message("######################") 123 | message("") 124 | message("V1 : " ${V1}) 125 | message("V2 : " ${V2}) 126 | message("V3 : " ${V3}) 127 | message("A0 : " ${A0}) 128 | message("R : " ${R}) 129 | 130 | message("") 131 | message("##########") 132 | message("# Floors #") 133 | message("##########") 134 | message("") 135 | message("Density floor : " ${RHO_FLOOR}) 136 | message("Internal energy floor : " ${UU_FLOOR}) 137 | -------------------------------------------------------------------------------- /src/params.hpp: -------------------------------------------------------------------------------- 1 | #ifndef GRIM_PARAMS_H_ 2 | #define GRIM_PARAMS_H_ 3 | #include 4 | 5 | const int NDIM = 4; 6 | const int LOCATIONS = 7; 7 | 8 | namespace vars 9 | { 10 | enum 11 | { 12 | RHO, U, U1, U2, U3 13 | }; 14 | extern int Q, DP; 15 | extern int B1, B2, B3; 16 | extern int dof; 17 | extern int numFluidVars; 18 | }; 19 | 20 | 21 | namespace dumpVars 22 | { 23 | enum 24 | { 25 | RHO, U, U1, U2, U3 26 | }; 27 | extern int Q, DP; 28 | extern int B1, B2, B3, BSQR, GAMMA; 29 | extern int dof; 30 | }; 31 | 32 | namespace locations 33 | { 34 | enum 35 | { 36 | CENTER, LEFT, RIGHT, TOP, BOTTOM, FRONT, BACK 37 | }; 38 | }; 39 | 40 | namespace directions 41 | { 42 | enum 43 | { 44 | X1, X2, X3 45 | }; 46 | }; 47 | 48 | namespace boundaries 49 | { 50 | enum 51 | { 52 | PERIODIC, OUTFLOW, MIRROR, DIRICHLET 53 | }; 54 | }; 55 | 56 | namespace metrics 57 | { 58 | enum 59 | { 60 | MINKOWSKI, MODIFIED_KERR_SCHILD 61 | }; 62 | }; 63 | 64 | namespace timeStepping 65 | { 66 | enum 67 | { 68 | EXPLICIT, IMEX, IMPLICIT 69 | }; 70 | }; 71 | 72 | namespace reconstructionOptions 73 | { 74 | enum 75 | { 76 | MINMOD, WENO5, PPM 77 | }; 78 | }; 79 | 80 | namespace riemannSolvers 81 | { 82 | enum 83 | { 84 | HLL, LOCAL_LAX_FRIEDRICH 85 | }; 86 | }; 87 | 88 | namespace linearSolvers 89 | { 90 | enum 91 | { 92 | GPU_BATCH_SOLVER, CPU_BATCH_SOLVER 93 | }; 94 | }; 95 | 96 | namespace params 97 | { 98 | extern int numDevices; 99 | 100 | extern int N1; 101 | extern int N2; 102 | extern int N3; 103 | extern int dim; 104 | extern int numGhost; 105 | 106 | extern int timeStepper; 107 | extern double InitialDt; 108 | extern double CourantFactor; 109 | extern double maxDtIncrement; 110 | extern double Time; 111 | extern double finalTime; 112 | extern int metric; 113 | extern double hSlope; 114 | extern double blackHoleSpin; 115 | 116 | extern int restart; 117 | extern std::string restartFile; 118 | extern std::string restartFileName; 119 | extern std::string restartFileTime; 120 | extern double MaxWallTime; 121 | extern int numDumpVars; 122 | 123 | extern double X1Start, X1End; 124 | extern double X2Start, X2End; 125 | extern double X3Start, X3End; 126 | 127 | extern int DerefineThetaHorizon; 128 | extern int DoCylindrify; 129 | extern double X1cyl, X2cyl; 130 | 131 | extern int boundaryLeft; 132 | extern int boundaryRight; 133 | 134 | extern int boundaryTop; 135 | extern int boundaryBottom; 136 | 137 | extern int boundaryFront; 138 | extern int boundaryBack; 139 | 140 | extern double rhoFloorInFluidElement; 141 | extern double uFloorInFluidElement; 142 | extern double bSqrFloorInFluidElement; 143 | extern double temperatureFloorInFluidElement; 144 | 145 | extern int conduction; 146 | extern int viscosity; 147 | extern int highOrderTermsConduction; 148 | extern int highOrderTermsViscosity; 149 | extern double adiabaticIndex; 150 | extern double ConductionAlpha; 151 | extern double ViscosityAlpha; 152 | 153 | extern double slopeLimTheta; 154 | extern int reconstruction; 155 | extern int riemannSolver; 156 | 157 | extern int maxNonLinearIter; 158 | extern int maxLineSearchIters; 159 | 160 | extern double nonlinearsolve_atol; 161 | extern double JacobianAssembleEpsilon; 162 | extern double linesearchfloor; 163 | extern int linearSolver; 164 | 165 | //Atmosphere parameters 166 | extern double MaxLorentzFactor; 167 | // Floors are Ampl*pow(radius,power) 168 | extern double RhoFloorAmpl; 169 | extern double UFloorAmpl; 170 | extern double RhoFloorSlope; 171 | extern double UFloorSlope; 172 | // Floors for magnetically dominated regions 173 | extern double BsqrOverRhoMax; 174 | extern double BsqrOverUMax; 175 | 176 | extern double ConductionClosureFactor; 177 | extern double ViscosityClosureFactor; 178 | 179 | extern int ObserveEveryNSteps; 180 | extern int StepNumber; 181 | 182 | /* Torus parameters */ 183 | extern double InnerEdgeRadius ; 184 | extern double PressureMaxRadius; 185 | extern double MinPlasmaBeta; 186 | extern double MagneticLoops; 187 | extern double Adiabat; 188 | extern double InitialPerturbationAmplitude; 189 | extern bool UseMADdisk; 190 | extern double ObserveEveryDt; 191 | extern double WriteDataEveryDt; 192 | 193 | /* Linear modes parameters */ 194 | extern double Aw; 195 | extern double k1; 196 | extern double k2; 197 | extern double Gamma; 198 | extern double Omega; 199 | 200 | /* For bondi_inflow */ 201 | extern double sonicRadius; 202 | extern double mDot; 203 | extern double bMag; 204 | 205 | /* shock tests */ 206 | extern std::string shockTest; 207 | 208 | /* Perf testing */ 209 | extern int nIters; 210 | }; 211 | 212 | #endif /* GRIM_PARAMS_H_ */ 213 | -------------------------------------------------------------------------------- /src/problem/torus/params.cpp: -------------------------------------------------------------------------------- 1 | #include "torus.hpp" 2 | 3 | namespace params 4 | { 5 | // 4 GPUs on SAVIO 6 | int numDevices = 1; 7 | 8 | // Grid size options 9 | int N1 = 128; 10 | int N2 = 128; 11 | int N3 = 64; 12 | int N3Full = 64; 13 | int dim = 3; 14 | int numGhost = 3; 15 | 16 | // (Re)start options 17 | double Time = 0.0; 18 | double InitialDt = 0.005; 19 | double maxDtIncrement = 1.3; 20 | int restart = 0; 21 | std::string restartFile = "restartFile.h5"; 22 | std::string restartFileName = "restartFileName.txt"; 23 | std::string restartFileTime = "restartFileTime.txt"; 24 | // Maximum run time, in seconds 25 | double MaxWallTime = 3600*23.5; 26 | 27 | // Observation / checkpointing intervals 28 | double ObserveEveryDt = .1; 29 | double WriteDataEveryDt = 2.; 30 | 31 | // Timestepper opts 32 | int timeStepper = timeStepping::EXPLICIT; 33 | double CourantFactor = 0.9; 34 | double finalTime = 20000.; 35 | int metric = metrics::MODIFIED_KERR_SCHILD; 36 | 37 | // Initial conditions 38 | bool UseMADdisk = true; 39 | double adiabaticIndex = 5./3; 40 | double blackHoleSpin = 0.5; 41 | double InnerEdgeRadius = 15.; 42 | double PressureMaxRadius = 32.; 43 | /* MinPlasmaBeta is min(Pgas/Pmag) for SANE 44 | disks, but max(Pgas)/max(Pmag) for MAD! */ 45 | double MinPlasmaBeta = 100.; 46 | double MagneticLoops = 1; 47 | double Adiabat = 0.001; 48 | double InitialPerturbationAmplitude = 4.e-2; 49 | 50 | // Grid size options 51 | double Rin = 0.85*(1.+sqrt(1.-blackHoleSpin*blackHoleSpin)); 52 | double Rout = 3000.; 53 | 54 | // EMHD model 55 | int conduction = 1; 56 | int viscosity = 1; 57 | int highOrderTermsConduction = 1.; 58 | int highOrderTermsViscosity = 1.; 59 | // Phi and Psi from Chandra et al. 2015 60 | // Note that when using Phi=Psi, approximate 61 | // limits for the characteristic speeds to remain 62 | // subluminal are 0.29 (Gamma=5/3) and 1.3 (Gamma=4/3). 63 | double ConductionAlpha = 1.; 64 | double ViscosityAlpha = 1.; 65 | double ConductionClosureFactor = 1.; 66 | double ViscosityClosureFactor = 1.; 67 | 68 | //Atmosphere parameters 69 | double MaxLorentzFactor = 10.; 70 | // Floors are Ampl*pow(radius,power) 71 | double RhoFloorAmpl = 1.e-3; 72 | double UFloorAmpl = 1.e-5; 73 | double RhoFloorSlope = -2.5; 74 | double UFloorSlope = -2.5; 75 | // Floors for magnetically dominated regions 76 | double BsqrOverRhoMax = 10.; 77 | double BsqrOverUMax = 500.; 78 | 79 | // Automatic grid boundaries - do not change 80 | double X1Start = log(Rin), X1End = log(Rout); 81 | double X2Start = 0.+1.e-8, X2End = 1.-1.e-8; 82 | double X3Start = 0., X3End = 2.*M_PI*N3/N3Full; 83 | 84 | // Grid parameters 85 | double hSlope = 0.3; 86 | int DerefineThetaHorizon = 1; 87 | int DoCylindrify = 1; 88 | double X1cyl = log(4.*Rin); 89 | double X2cyl = 1./N2; 90 | 91 | // Boundary Conditions 92 | // Radial 93 | int boundaryLeft = boundaries::OUTFLOW; 94 | int boundaryRight = boundaries::OUTFLOW; 95 | // Theta 96 | int boundaryTop = boundaries::MIRROR; 97 | int boundaryBottom = boundaries::MIRROR; 98 | // Phi 99 | int boundaryFront = boundaries::PERIODIC; 100 | int boundaryBack = boundaries::PERIODIC; 101 | 102 | // Basic thresholds in fluid element 103 | double rhoFloorInFluidElement = 1e-20; 104 | double uFloorInFluidElement = 1e-20; 105 | double bSqrFloorInFluidElement = 1e-20; 106 | double temperatureFloorInFluidElement = 1e-20; 107 | 108 | // Reconstruction options 109 | double slopeLimTheta = 2.; 110 | int reconstruction = reconstructionOptions::PPM; 111 | int riemannSolver = riemannSolvers::LOCAL_LAX_FRIEDRICH; 112 | 113 | 114 | //Parameters controlling accuracy of nonlinear solver 115 | int maxNonLinearIter = 3; 116 | int maxLineSearchIters = 3; 117 | double nonlinearsolve_atol = 1.e-3; 118 | double JacobianAssembleEpsilon = 4.e-8; 119 | double linesearchfloor = 1.e-24; 120 | 121 | // Linear solver options 122 | int linearSolver = linearSolvers::GPU_BATCH_SOLVER; 123 | }; 124 | 125 | namespace vars 126 | { 127 | int Q = 5; 128 | int DP = 5 + params::conduction; 129 | int numFluidVars = 5 + params::conduction + params::viscosity; 130 | 131 | int B1 = 5 + params::conduction + params::viscosity; 132 | int B2 = 6 + params::conduction + params::viscosity; 133 | int B3 = 7 + params::conduction + params::viscosity; 134 | int dof = 8 + params::conduction + params::viscosity; 135 | }; 136 | 137 | namespace dumpVars 138 | { 139 | int Q = 5; 140 | int DP = 5 + params::conduction; 141 | 142 | int B1 = 5 + params::conduction + params::viscosity; 143 | int B2 = 6 + params::conduction + params::viscosity; 144 | int B3 = 7 + params::conduction + params::viscosity; 145 | int BSQR = 8 + params::conduction + params::viscosity; 146 | int GAMMA = 9 + params::conduction + params::viscosity; 147 | int dof = 10 + params::conduction + params::viscosity; 148 | }; 149 | -------------------------------------------------------------------------------- /src/cmake/modules/FindGSL.cmake: -------------------------------------------------------------------------------- 1 | # Taken from the GNU Radio project 2 | # Try to find gnu scientific library GSL 3 | # See 4 | # http://www.gnu.org/software/gsl/ and 5 | # http://gnuwin32.sourceforge.net/packages/gsl.htm 6 | # 7 | # Based on a script of Felix Woelk and Jan Woetzel 8 | # (www.mip.informatik.uni-kiel.de) 9 | # 10 | # It defines the following variables: 11 | # GSL_FOUND - system has GSL lib 12 | # GSL_INCLUDE_DIRS - where to find headers 13 | # GSL_LIBRARIES - full path to the libraries 14 | # GSL_LIBRARY_DIRS, the directory where the PLplot library is found. 15 | 16 | # CMAKE_GSL_CXX_FLAGS = Unix compiler flags for GSL, essentially "`gsl-config --cxxflags`" 17 | # GSL_LINK_DIRECTORIES = link directories, useful for rpath on Unix 18 | # GSL_EXE_LINKER_FLAGS = rpath on Unix 19 | 20 | INCLUDE(FindPkgConfig) 21 | PKG_CHECK_MODULES(GSL "gsl >= 1.10") 22 | IF(NOT GSL_FOUND) 23 | 24 | set( GSL_FOUND OFF ) 25 | set( GSL_CBLAS_FOUND OFF ) 26 | 27 | # Windows, but not for Cygwin and MSys where gsl-config is available 28 | if( WIN32 AND NOT CYGWIN AND NOT MSYS ) 29 | # look for headers 30 | find_path( GSL_INCLUDE_DIR 31 | NAMES gsl/gsl_cdf.h gsl/gsl_randist.h 32 | ) 33 | if( GSL_INCLUDE_DIR ) 34 | # look for gsl library 35 | find_library( GSL_LIBRARY 36 | NAMES gsl 37 | ) 38 | if( GSL_LIBRARY ) 39 | set( GSL_INCLUDE_DIRS ${GSL_INCLUDE_DIR} ) 40 | get_filename_component( GSL_LIBRARY_DIRS ${GSL_LIBRARY} PATH ) 41 | set( GSL_FOUND ON ) 42 | endif( GSL_LIBRARY ) 43 | 44 | # look for gsl cblas library 45 | find_library( GSL_CBLAS_LIBRARY 46 | NAMES gslcblas 47 | ) 48 | if( GSL_CBLAS_LIBRARY ) 49 | set( GSL_CBLAS_FOUND ON ) 50 | endif( GSL_CBLAS_LIBRARY ) 51 | 52 | set( GSL_LIBRARIES ${GSL_LIBRARY} ${GSL_CBLAS_LIBRARY} ) 53 | endif( GSL_INCLUDE_DIR ) 54 | 55 | #mark_as_advanced( 56 | # GSL_INCLUDE_DIR 57 | # GSL_LIBRARY 58 | # GSL_CBLAS_LIBRARY 59 | #) 60 | else( WIN32 AND NOT CYGWIN AND NOT MSYS ) 61 | if( UNIX OR MSYS ) 62 | find_program( GSL_CONFIG_EXECUTABLE gsl-config 63 | /usr/bin/ 64 | /usr/local/bin 65 | ) 66 | 67 | if( GSL_CONFIG_EXECUTABLE ) 68 | set( GSL_FOUND ON ) 69 | 70 | # run the gsl-config program to get cxxflags 71 | execute_process( 72 | COMMAND sh "${GSL_CONFIG_EXECUTABLE}" --cflags 73 | OUTPUT_VARIABLE GSL_CFLAGS 74 | RESULT_VARIABLE RET 75 | ERROR_QUIET 76 | ) 77 | if( RET EQUAL 0 ) 78 | string( STRIP "${GSL_CFLAGS}" GSL_CFLAGS ) 79 | separate_arguments( GSL_CFLAGS ) 80 | 81 | # parse definitions from cflags; drop -D* from CFLAGS 82 | string( REGEX MATCHALL "-D[^;]+" 83 | GSL_DEFINITIONS "${GSL_CFLAGS}" ) 84 | string( REGEX REPLACE "-D[^;]+;" "" 85 | GSL_CFLAGS "${GSL_CFLAGS}" ) 86 | 87 | # parse include dirs from cflags; drop -I prefix 88 | string( REGEX MATCHALL "-I[^;]+" 89 | GSL_INCLUDE_DIRS "${GSL_CFLAGS}" ) 90 | string( REPLACE "-I" "" 91 | GSL_INCLUDE_DIRS "${GSL_INCLUDE_DIRS}") 92 | string( REGEX REPLACE "-I[^;]+;" "" 93 | GSL_CFLAGS "${GSL_CFLAGS}") 94 | 95 | message("GSL_DEFINITIONS=${GSL_DEFINITIONS}") 96 | message("GSL_INCLUDE_DIRS=${GSL_INCLUDE_DIRS}") 97 | message("GSL_CFLAGS=${GSL_CFLAGS}") 98 | else( RET EQUAL 0 ) 99 | set( GSL_FOUND FALSE ) 100 | endif( RET EQUAL 0 ) 101 | 102 | # run the gsl-config program to get the libs 103 | execute_process( 104 | COMMAND sh "${GSL_CONFIG_EXECUTABLE}" --libs 105 | OUTPUT_VARIABLE GSL_LIBRARIES 106 | RESULT_VARIABLE RET 107 | ERROR_QUIET 108 | ) 109 | if( RET EQUAL 0 ) 110 | string(STRIP "${GSL_LIBRARIES}" GSL_LIBRARIES ) 111 | separate_arguments( GSL_LIBRARIES ) 112 | 113 | # extract linkdirs (-L) for rpath (i.e., LINK_DIRECTORIES) 114 | string( REGEX MATCHALL "-L[^;]+" 115 | GSL_LIBRARY_DIRS "${GSL_LIBRARIES}" ) 116 | string( REPLACE "-L" "" 117 | GSL_LIBRARY_DIRS "${GSL_LIBRARY_DIRS}" ) 118 | else( RET EQUAL 0 ) 119 | set( GSL_FOUND FALSE ) 120 | endif( RET EQUAL 0 ) 121 | 122 | MARK_AS_ADVANCED( 123 | GSL_CFLAGS 124 | ) 125 | message( STATUS "Using GSL from ${GSL_PREFIX}" ) 126 | else( GSL_CONFIG_EXECUTABLE ) 127 | message( STATUS "FindGSL: gsl-config not found.") 128 | endif( GSL_CONFIG_EXECUTABLE ) 129 | endif( UNIX OR MSYS ) 130 | endif( WIN32 AND NOT CYGWIN AND NOT MSYS ) 131 | 132 | if( GSL_FOUND ) 133 | if( NOT GSL_FIND_QUIETLY ) 134 | message( STATUS "FindGSL: Found both GSL headers and library" ) 135 | endif( NOT GSL_FIND_QUIETLY ) 136 | else( GSL_FOUND ) 137 | if( GSL_FIND_REQUIRED ) 138 | message( FATAL_ERROR "FindGSL: Could not find GSL headers or library" ) 139 | endif( GSL_FIND_REQUIRED ) 140 | endif( GSL_FOUND ) 141 | 142 | INCLUDE(FindPackageHandleStandardArgs) 143 | FIND_PACKAGE_HANDLE_STANDARD_ARGS(GSL DEFAULT_MSG GSL_LIBRARIES GSL_INCLUDE_DIRS) 144 | ENDIF(NOT GSL_FOUND) 145 | -------------------------------------------------------------------------------- /src/problem/anisotropic_conduction/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Dump folder 2 | set(OUTPUT_DIR "\"/home/mc/data/mc/PhD/grim/build/\"") 3 | set(RESTART "OFF") 4 | set(RESTART_FILE "\"restartfile.h5\"") 5 | set(DUMP_FILE_PREFIX "\"data\"") 6 | set(PROBLEM_DATA "\"${PROBLEM}/problemdata.h\"") 7 | 8 | # Time stepping options: EXPLICIT, IMEX or IMPLICIT 9 | set(TIME_STEPPING "EXPLICIT") 10 | set(DT "1e-3") 11 | set(DT_DUMP ".1") 12 | set(START_TIME "0.") 13 | set(FINAL_TIME "5.") 14 | set(COURANT "0.9") 15 | set(MAX_DT_INCREMENT "1.3") 16 | 17 | # Domain size. If the problem is 1D then N2 is ignored. 18 | set(COMPUTE_DIM "2") 19 | set(N1 "256") 20 | set(N2 "256") 21 | 22 | # The entire global domain is divided into tiles that are small enough to fit 23 | # into the cache of the compute node or an accelerator. This technique 24 | # optimizes cache usage and makes explicit use of the deep memory hierarchies 25 | # which are prevalent in all kinds of processors and accelerators. 26 | # 27 | # Caution: 1) The tile sizes need to divide the global domain size in each 28 | # direction exactly! 29 | # 2) We want a tile size that is as large as the cache. Too large a 30 | # size and the code can fail silently because local memory in the 31 | # OpenCL specification is a different amount for different 32 | # machines. 33 | # NEED TO PUT A CHECK FOR THIS. 34 | set(TILE_SIZE_X1 "8") 35 | set(TILE_SIZE_X2 "8") 36 | set(USE_OPENMP "YES") 37 | 38 | # Physics variables 39 | set(ADIABATIC_INDEX "5./3") 40 | set(CONDUCTION "ON") 41 | 42 | # Geometry 43 | set(METRIC "MINKOWSKI") 44 | set(EPS "1e-5") 45 | 46 | # Domain 47 | set(X1_A "0.") 48 | set(X1_B "1.") 49 | set(X2_A "0.") 50 | set(X2_B "1.") 51 | 52 | # Boundary conditions 53 | set(PHYSICAL_BOUNDARY_LEFT_EDGE "PERIODIC") 54 | set(PHYSICAL_BOUNDARY_RIGHT_EDGE "PERIODIC") 55 | set(PHYSICAL_BOUNDARY_TOP_EDGE "PERIODIC") 56 | set(PHYSICAL_BOUNDARY_BOTTOM_EDGE "PERIODIC") 57 | 58 | # Reconstrution options 59 | # MP5, MONOTONIZED_CENTRAL or MIN_MOD 60 | set(RECONSTRUCTION "MONOTONIZED_CENTRAL") 61 | 62 | # Initial condition parameters 63 | # AMPLITUDE -- Temperature perturbation amplitude 64 | # RADIUS -- Radius of the perturbation 65 | # MEAN_MAGNETIC_FIELD -- Magnetic field in the x direction 66 | # WAVE_NUMBER -- Choose how wavy you want the field lines to be 67 | set(AMPLITUDE ".2") 68 | set(RADIUS "sqrt(0.005)") 69 | set(MEAN_MAGNETIC_FIELD "0.0001") 70 | set(WAVE_NUMBER "4.") 71 | 72 | # Floor values 73 | set(RHO_FLOOR "1e-15") 74 | SET(UU_FLOOR "1e-15") 75 | set(RHO_FLOOR_MIN "1e-15") 76 | set(UU_FLOOR_MIN "1e-15") 77 | 78 | message("") 79 | message("#########################") 80 | message("# Configuration options #") 81 | message("#########################") 82 | message("") 83 | message("Problem : " ${PROBLEM}) 84 | message("Output dir : " ${OUTPUT_DIR}) 85 | message("Time stepping : " ${TIME_STEPPING}) 86 | message("Dimensions : " ${COMPUTE_DIM}) 87 | message("Resolution : " ${N1} " x " ${N2}) 88 | message("Tile size : " ${TILE_SIZE_X1} " x " ${TILE_SIZE_X2}) 89 | message("Metric : " ${METRIC}) 90 | 91 | message("") 92 | message("##################################") 93 | message("# Domain and boundary conditions #") 94 | message("##################################") 95 | message("") 96 | message(" " ${PHYSICAL_BOUNDARY_TOP_EDGE}) 97 | message(" (" ${X1_A},${X2_B} ")+-----------+" "(" ${X1_B},${X2_B} ")" ) 98 | message( " | |" ) 99 | message( " | |" ) 100 | message(" " ${PHYSICAL_BOUNDARY_LEFT_EDGE} " | | " ${PHYSICAL_BOUNDARY_RIGHT_EDGE}) 101 | message( " | |" ) 102 | message( " | |" ) 103 | message(" (" ${X1_A},${X2_A} ")+-----------+" "(" ${X1_B},${X2_A} ")" ) 104 | message(" " ${PHYSICAL_BOUNDARY_BOTTOM_EDGE}) 105 | 106 | message("") 107 | message("###################") 108 | message("# Physics options #") 109 | message("###################") 110 | message("") 111 | message("Adiabatic index : " ${ADIABATIC_INDEX}) 112 | message("Conduction : " ${CONDUCTION}) 113 | 114 | message("") 115 | message("##################") 116 | message("# Reconstruction #") 117 | message("##################") 118 | message("") 119 | message("Reconstruction : " ${RECONSTRUCTION}) 120 | 121 | message("") 122 | message("######################") 123 | message("# Initial conditions #") 124 | message("######################") 125 | message("") 126 | message("Temperature perturbation amplitude : " ${AMPLITUDE}) 127 | message("Wave number of perturbation : " ${WAVE_NUMBER}) 128 | message("Radius : " ${RADIUS}) 129 | message("Mean magnetic field : " ${MEAN_MAGNETIC_FIELD}) 130 | 131 | message("") 132 | message("##########") 133 | message("# Floors #") 134 | message("##########") 135 | message("") 136 | message("Density floor : " ${RHO_FLOOR}) 137 | message("Internal energy floor : " ${UU_FLOOR}) 138 | -------------------------------------------------------------------------------- /src/cmake/modules/FindPackageMultipass.cmake: -------------------------------------------------------------------------------- 1 | # PackageMultipass - this module defines two macros 2 | # 3 | # FIND_PACKAGE_MULTIPASS (Name CURRENT 4 | # STATES VAR0 VAR1 ... 5 | # DEPENDENTS DEP0 DEP1 ...) 6 | # 7 | # This function creates a cache entry _CURRENT which 8 | # the user can set to "NO" to trigger a reconfiguration of the package. 9 | # The first time this function is called, the values of 10 | # _VAR0, ... are saved. If _CURRENT 11 | # is false or if any STATE has changed since the last time 12 | # FIND_PACKAGE_MULTIPASS() was called, then CURRENT will be set to "NO", 13 | # otherwise CURRENT will be "YES". IF not CURRENT, then 14 | # _DEP0, ... will be FORCED to NOTFOUND. 15 | # Example: 16 | # find_path (FOO_DIR include/foo.h) 17 | # FIND_PACKAGE_MULTIPASS (Foo foo_current 18 | # STATES DIR 19 | # DEPENDENTS INCLUDES LIBRARIES) 20 | # if (NOT foo_current) 21 | # # Make temporary files, run programs, etc, to determine FOO_INCLUDES and FOO_LIBRARIES 22 | # endif (NOT foo_current) 23 | # 24 | # MULTIPASS_SOURCE_RUNS (Name INCLUDES LIBRARIES SOURCE RUNS LANGUAGE) 25 | # Always runs the given test, use this when you need to re-run tests 26 | # because parent variables have made old cache entries stale. The LANGUAGE 27 | # variable is either C or CXX indicating which compiler the test should 28 | # use. 29 | # MULTIPASS_C_SOURCE_RUNS (Name INCLUDES LIBRARIES SOURCE RUNS) 30 | # DEPRECATED! This is only included for backwards compatability. Use 31 | # the more general MULTIPASS_SOURCE_RUNS instead. 32 | # Always runs the given test, use this when you need to re-run tests 33 | # because parent variables have made old cache entries stale. 34 | 35 | macro (FIND_PACKAGE_MULTIPASS _name _current) 36 | string (TOUPPER ${_name} _NAME) 37 | set (_args ${ARGV}) 38 | list (REMOVE_AT _args 0 1) 39 | 40 | set (_states_current "YES") 41 | list (GET _args 0 _cmd) 42 | if (_cmd STREQUAL "STATES") 43 | list (REMOVE_AT _args 0) 44 | list (GET _args 0 _state) 45 | while (_state AND NOT _state STREQUAL "DEPENDENTS") 46 | # The name of the stored value for the given state 47 | set (_stored_var PACKAGE_MULTIPASS_${_NAME}_${_state}) 48 | if (NOT "${${_stored_var}}" STREQUAL "${${_NAME}_${_state}}") 49 | set (_states_current "NO") 50 | endif (NOT "${${_stored_var}}" STREQUAL "${${_NAME}_${_state}}") 51 | set (${_stored_var} "${${_NAME}_${_state}}" CACHE INTERNAL "Stored state for ${_name}." FORCE) 52 | list (REMOVE_AT _args 0) 53 | list (GET _args 0 _state) 54 | endwhile (_state AND NOT _state STREQUAL "DEPENDENTS") 55 | endif (_cmd STREQUAL "STATES") 56 | 57 | set (_stored ${_NAME}_CURRENT) 58 | if (NOT ${_stored}) 59 | set (${_stored} "YES" CACHE BOOL "Is the configuration for ${_name} current? Set to \"NO\" to reconfigure." FORCE) 60 | set (_states_current "NO") 61 | endif (NOT ${_stored}) 62 | 63 | set (${_current} ${_states_current}) 64 | if (NOT ${_current} AND PACKAGE_MULTIPASS_${_name}_CALLED) 65 | message (STATUS "Clearing ${_name} dependent variables") 66 | # Clear all the dependent variables so that the module can reset them 67 | list (GET _args 0 _cmd) 68 | if (_cmd STREQUAL "DEPENDENTS") 69 | list (REMOVE_AT _args 0) 70 | foreach (dep ${_args}) 71 | set (${_NAME}_${dep} "NOTFOUND" CACHE INTERNAL "Cleared" FORCE) 72 | endforeach (dep) 73 | endif (_cmd STREQUAL "DEPENDENTS") 74 | set (${_NAME}_FOUND "NOTFOUND" CACHE INTERNAL "Cleared" FORCE) 75 | endif () 76 | set (PACKAGE_MULTIPASS_${name}_CALLED YES CACHE INTERNAL "Private" FORCE) 77 | endmacro (FIND_PACKAGE_MULTIPASS) 78 | 79 | 80 | macro (MULTIPASS_SOURCE_RUNS includes libraries source runs language) 81 | include (Check${language}SourceRuns) 82 | # This is a ridiculous hack. CHECK_${language}_SOURCE_* thinks that if the 83 | # *name* of the return variable doesn't change, then the test does 84 | # not need to be re-run. We keep an internal count which we 85 | # increment to guarantee that every test name is unique. If we've 86 | # gotten here, then the configuration has changed enough that the 87 | # test *needs* to be rerun. 88 | if (NOT MULTIPASS_TEST_COUNT) 89 | set (MULTIPASS_TEST_COUNT 00) 90 | endif (NOT MULTIPASS_TEST_COUNT) 91 | math (EXPR _tmp "${MULTIPASS_TEST_COUNT} + 1") # Why can't I add to a cache variable? 92 | set (MULTIPASS_TEST_COUNT ${_tmp} CACHE INTERNAL "Unique test ID") 93 | set (testname MULTIPASS_TEST_${MULTIPASS_TEST_COUNT}_${runs}) 94 | set (CMAKE_REQUIRED_INCLUDES ${includes}) 95 | set (CMAKE_REQUIRED_LIBRARIES ${libraries}) 96 | if(${language} STREQUAL "C") 97 | check_c_source_runs ("${source}" ${testname}) 98 | elseif(${language} STREQUAL "CXX") 99 | check_cxx_source_runs ("${source}" ${testname}) 100 | endif() 101 | set (${runs} "${${testname}}") 102 | endmacro (MULTIPASS_SOURCE_RUNS) 103 | 104 | macro (MULTIPASS_C_SOURCE_RUNS includes libraries source runs) 105 | multipass_source_runs("${includes}" "${libraries}" "${source}" ${runs} "C") 106 | endmacro (MULTIPASS_C_SOURCE_RUNS) 107 | -------------------------------------------------------------------------------- /src/cmake/modules/FindLAPACKE.cmake: -------------------------------------------------------------------------------- 1 | # - Find the LAPACKE library 2 | # 3 | # Usage: 4 | # FIND_PACKAGE(LAPACKE [REQUIRED] [QUIET] ) 5 | # 6 | # It sets the following variables: 7 | # LAPACK_FOUND ... true if LAPACKE is found on the system 8 | # LAPACK_LIBRARIES ... full path to LAPACKE library 9 | # LAPACK_INCLUDES ... LAPACKE include directory 10 | # 11 | 12 | SET(LAPACKE_ROOT_DIR CACHE STRING 13 | "Root directory for custom LAPACK implementation") 14 | 15 | IF (NOT INTEL_MKL_ROOT_DIR) 16 | SET(INTEL_MKL_ROOT_DIR $ENV{INTEL_MKL_ROOT}) 17 | ENDIF() 18 | 19 | IF(NOT LAPACKE_ROOT_DIR) 20 | 21 | IF (ENV{LAPACKEDIR}) 22 | SET(LAPACKE_ROOT_DIR $ENV{LAPACKEDIR}) 23 | ENDIF() 24 | 25 | IF (ENV{LAPACKE_ROOT_DIR_DIR}) 26 | SET(LAPACKE_ROOT_DIR $ENV{LAPACKE_ROOT_DIR}) 27 | ENDIF() 28 | 29 | IF (INTEL_MKL_ROOT_DIR) 30 | SET(LAPACKE_ROOT_DIR ${INTEL_MKL_ROOT_DIR}) 31 | ENDIF() 32 | ENDIF() 33 | 34 | # Check if we can use PkgConfig 35 | FIND_PACKAGE(PkgConfig) 36 | 37 | #Determine from PKG 38 | IF(PKG_CONFIG_FOUND AND NOT LAPACKE_ROOT_DIR) 39 | PKG_CHECK_MODULES( PC_LAPACKE QUIET "lapacke") 40 | ENDIF() 41 | 42 | IF(PC_LAPACKE_FOUND) 43 | FOREACH(PC_LIB ${PC_LAPACKE_LIBRARIES}) 44 | FIND_LIBRARY(${PC_LIB}_LIBRARY NAMES ${PC_LIB} HINTS ${PC_LAPACKE_LIBRARY_DIRS} ) 45 | IF (NOT ${PC_LIB}_LIBRARY) 46 | MESSAGE(FATAL_ERROR "Something is wrong in your pkg-config file - lib ${PC_LIB} not found in ${PC_LAPACKE_LIBRARY_DIRS}") 47 | ENDIF (NOT ${PC_LIB}_LIBRARY) 48 | LIST(APPEND LAPACKE_LIB ${${PC_LIB}_LIBRARY}) 49 | ENDFOREACH(PC_LIB) 50 | 51 | FIND_PATH( 52 | LAPACKE_INCLUDES 53 | NAMES "lapacke.h" 54 | PATHS 55 | ${PC_LAPACKE_INCLUDE_DIRS} 56 | ${INCLUDE_INSTALL_DIR} 57 | /usr/include 58 | /usr/local/include 59 | /sw/include 60 | /opt/local/include 61 | DOC "LAPACKE Include Directory" 62 | ) 63 | 64 | FIND_PACKAGE_HANDLE_STANDARD_ARGS(LAPACKE DEFAULT_MSG LAPACKE_LIB) 65 | MARK_AS_ADVANCED(LAPACKE_INCLUDES LAPACKE_LIB) 66 | 67 | ELSE(PC_LAPACKE_FOUND) 68 | 69 | IF(LAPACKE_ROOT_DIR) 70 | #find libs 71 | FIND_LIBRARY( 72 | LAPACKE_LIB 73 | NAMES "lapacke" "LAPACKE" "liblapacke" "mkl_rt" 74 | PATHS ${LAPACKE_ROOT_DIR} 75 | PATH_SUFFIXES "lib" "lib64" "lib/ia32" "lib/intel64" 76 | DOC "LAPACKE Library" 77 | NO_DEFAULT_PATH 78 | ) 79 | FIND_LIBRARY( 80 | LAPACK_LIB 81 | NAMES "lapack" "LAPACK" "liblapack" "mkl_rt" 82 | PATHS ${LAPACKE_ROOT_DIR} 83 | PATH_SUFFIXES "lib" "lib64" "lib/ia32" "lib/intel64" 84 | DOC "LAPACK Library" 85 | NO_DEFAULT_PATH 86 | ) 87 | FIND_PATH( 88 | LAPACKE_INCLUDES 89 | NAMES "lapacke.h" "mkl_lapacke.h" 90 | PATHS ${LAPACKE_ROOT_DIR} 91 | PATH_SUFFIXES "include" 92 | DOC "LAPACKE Include Directory" 93 | NO_DEFAULT_PATH 94 | ) 95 | ELSE() 96 | FIND_LIBRARY( 97 | LAPACKE_LIB 98 | NAMES "lapacke" "liblapacke" "openblas" "mkl_rt" 99 | PATHS 100 | ${PC_LAPACKE_LIBRARY_DIRS} 101 | ${LIB_INSTALL_DIR} 102 | /opt/intel/mkl/lib/ia32 103 | /opt/intel/mkl/lib/intel64 104 | /usr/lib64 105 | /usr/lib 106 | /usr/local/lib64 107 | /usr/local/lib 108 | /sw/lib 109 | /opt/local/lib 110 | DOC "LAPACKE Library" 111 | ) 112 | FIND_LIBRARY( 113 | LAPACK_LIB 114 | NAMES "lapack" "liblapack" "openblas" "mkl_rt" 115 | PATHS 116 | ${PC_LAPACKE_LIBRARY_DIRS} 117 | ${LIB_INSTALL_DIR} 118 | /opt/intel/mkl/lib/ia32 119 | /opt/intel/mkl/lib/intel64 120 | /usr/lib64 121 | /usr/lib 122 | /usr/local/lib64 123 | /usr/local/lib 124 | /sw/lib 125 | /opt/local/lib 126 | DOC "LAPACK Library" 127 | ) 128 | FIND_PATH( 129 | LAPACKE_INCLUDES 130 | NAMES "lapacke.h" "mkl_lapacke.h" 131 | PATHS 132 | ${PC_LAPACKE_INCLUDE_DIRS} 133 | ${INCLUDE_INSTALL_DIR} 134 | /opt/intel/mkl/include 135 | /usr/include 136 | /usr/local/include 137 | /sw/include 138 | /opt/local/include 139 | DOC "LAPACKE Include Directory" 140 | PATH_SUFFIXES 141 | lapacke 142 | ) 143 | ENDIF(LAPACKE_ROOT_DIR) 144 | ENDIF(PC_LAPACKE_FOUND) 145 | 146 | IF(LAPACKE_LIB AND LAPACK_LIB) 147 | SET(LAPACK_LIBRARIES ${LAPACKE_LIB} ${LAPACK_LIB}) 148 | ENDIF() 149 | IF(LAPACKE_INCLUDES) 150 | SET(LAPACK_INCLUDE_DIR ${LAPACKE_INCLUDES}) 151 | ENDIF() 152 | 153 | INCLUDE(FindPackageHandleStandardArgs) 154 | FIND_PACKAGE_HANDLE_STANDARD_ARGS(LAPACK DEFAULT_MSG 155 | LAPACK_INCLUDE_DIR LAPACK_LIBRARIES) 156 | 157 | MARK_AS_ADVANCED(LAPACK_INCLUDES LAPACK_LIBRARIES) 158 | -------------------------------------------------------------------------------- /src/problem/magnetized_explosion/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Dump folder 2 | set(OUTPUT_DIR "\"/home/mc/data/mc/PhD/grim/build/\"") 3 | set(RESTART "OFF") 4 | set(RESTART_FILE "\"restartfile.h5\"") 5 | set(DUMP_FILE_PREFIX "\"data\"") 6 | set(PROBLEM_DATA "\"${PROBLEM}/problemdata.h\"") 7 | 8 | # Time stepping options: EXPLICIT, IMEX or IMPLICIT 9 | set(TIME_STEPPING "EXPLICIT") 10 | set(DT "1e-5") 11 | set(DT_DUMP ".1") 12 | set(START_TIME "0.") 13 | set(FINAL_TIME "4.") 14 | set(COURANT "0.9") 15 | set(MAX_DT_INCREMENT "1.3") 16 | 17 | # Domain size. If the problem is 1D then N2 is ignored. 18 | set(COMPUTE_DIM "2") 19 | set(N1 "128") 20 | set(N2 "128") 21 | 22 | # The entire global domain is divided into tiles that are small enough to fit 23 | # into the cache of the compute node or an accelerator. This technique 24 | # optimizes cache usage and makes explicit use of the deep memory hierarchies 25 | # which are prevalent in all kinds of processors and accelerators. 26 | # 27 | # Caution: 1) The tile sizes need to divide the global domain size in each 28 | # direction exactly! 29 | # 2) We want a tile size that is as large as the cache. Too large a 30 | # size and the code can fail silently because local memory in the 31 | # OpenCL specification is a different amount for different 32 | # machines. 33 | # NEED TO PUT A CHECK FOR THIS. 34 | set(TILE_SIZE_X1 "8") 35 | set(TILE_SIZE_X2 "8") 36 | set(USE_OPENMP "YES") 37 | 38 | # Physics variables 39 | set(ADIABATIC_INDEX "5./3") 40 | set(CONDUCTION "OFF") 41 | 42 | # Geometry 43 | set(METRIC "MINKOWSKI") 44 | set(EPS "1e-5") 45 | 46 | # Domain 47 | set(X1_A "-6.") 48 | set(X1_B "6.") 49 | set(X2_A "-6.") 50 | set(X2_B "6.") 51 | 52 | # Boundary conditions 53 | set(PHYSICAL_BOUNDARY_LEFT_EDGE "MIRROR") 54 | set(PHYSICAL_BOUNDARY_RIGHT_EDGE "MIRROR") 55 | set(PHYSICAL_BOUNDARY_TOP_EDGE "MIRROR") 56 | set(PHYSICAL_BOUNDARY_BOTTOM_EDGE "MIRROR") 57 | 58 | # Reconstrution options 59 | # MP5, MONOTONIZED_CENTRAL or MIN_MOD 60 | set(RECONSTRUCTION "MONOTONIZED_CENTRAL") 61 | 62 | # Initial condition parameters 63 | # MEAN_MAGNETIC_FIELD -- Magnetic field in the x direction 64 | # SMOOTHNESS -- Parameter for the shell edge 65 | # Parameters taken from Komissarov, 1999 66 | set(INITIAL_RADIUS "1.") 67 | set(MEAN_MAGNETIC_FIELD ".1") 68 | set(RHO_INSIDE "1e-2") 69 | set(RHO_OUTSIDE "1e-4") 70 | set(PRESSURE_INSIDE "1.") 71 | set(PRESSURE_OUTSIDE "3e-5") 72 | set(SMOOTHNESS "20.") 73 | 74 | # Floor values 75 | set(RHO_FLOOR "1e-15") 76 | SET(UU_FLOOR "1e-15") 77 | set(RHO_FLOOR_MIN "1e-15") 78 | set(UU_FLOOR_MIN "1e-15") 79 | 80 | message("") 81 | message("#########################") 82 | message("# Configuration options #") 83 | message("#########################") 84 | message("") 85 | message("Problem : " ${PROBLEM}) 86 | message("Output dir : " ${OUTPUT_DIR}) 87 | message("Time stepping : " ${TIME_STEPPING}) 88 | message("Dimensions : " ${COMPUTE_DIM}) 89 | message("Resolution : " ${N1} " x " ${N2}) 90 | message("Tile size : " ${TILE_SIZE_X1} " x " ${TILE_SIZE_X2}) 91 | message("Metric : " ${METRIC}) 92 | 93 | message("") 94 | message("##################################") 95 | message("# Domain and boundary conditions #") 96 | message("##################################") 97 | message("") 98 | message(" " ${PHYSICAL_BOUNDARY_TOP_EDGE}) 99 | message(" (" ${X1_A},${X2_B} ")+-----------+" "(" ${X1_B},${X2_B} ")" ) 100 | message( " | |" ) 101 | message( " | |" ) 102 | message(" " ${PHYSICAL_BOUNDARY_LEFT_EDGE} " | | " ${PHYSICAL_BOUNDARY_RIGHT_EDGE}) 103 | message( " | |" ) 104 | message( " | |" ) 105 | message(" (" ${X1_A},${X2_A} ")+-----------+" "(" ${X1_B},${X2_A} ")" ) 106 | message(" " ${PHYSICAL_BOUNDARY_BOTTOM_EDGE}) 107 | 108 | message("") 109 | message("###################") 110 | message("# Physics options #") 111 | message("###################") 112 | message("") 113 | message("Adiabatic index : " ${ADIABATIC_INDEX}) 114 | message("Conduction : " ${CONDUCTION}) 115 | 116 | message("") 117 | message("##################") 118 | message("# Reconstruction #") 119 | message("##################") 120 | message("") 121 | message("Reconstruction : " ${RECONSTRUCTION}) 122 | 123 | message("") 124 | message("######################") 125 | message("# Initial conditions #") 126 | message("######################") 127 | message("") 128 | message("Initial radius : " ${INITIAL_RADIUS}) 129 | message("Mean magnetic field : " ${MEAN_MAGNETIC_FIELD}) 130 | message("Density inside : " ${RHO_INSIDE}) 131 | message("Density outside : " ${RHO_OUTSIDE}) 132 | message("Pressure inside : " ${PRESSURE_INSIDE}) 133 | message("Pressure outside : " ${PRESSURE_OUTSIDE}) 134 | message("Smoothness factor : " ${SMOOTHNESS}) 135 | 136 | message("") 137 | message("##########") 138 | message("# Floors #") 139 | message("##########") 140 | message("") 141 | message("Density floor : " ${RHO_FLOOR}) 142 | message("Internal energy floor : " ${UU_FLOOR}) 143 | -------------------------------------------------------------------------------- /src/problem/bondi_viscous/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Dump folder 2 | set(OUTPUT_DIR "\"/Users/francoisfoucart/Work/grim/Runs/bondi_viscous\"") 3 | set(RESTART "OFF") 4 | set(RESTART_FILE "\"restartfile.h5\"") 5 | set(DUMP_FILE_PREFIX "\"data\"") 6 | set(PROBLEM_DATA "\"${PROBLEM}/problemdata.h\"") 7 | 8 | # Time stepping options: EXPLICIT, IMEX or IMPLICIT 9 | set(TIME_STEPPING "EXPLICIT") 10 | set(DT "1e-5") 11 | set(DT_DUMP "100.") 12 | set(START_TIME "0.") 13 | set(FINAL_TIME "1000.") 14 | set(COURANT "0.5") 15 | set(MAX_DT_INCREMENT "1.3") 16 | 17 | # Domain size. If the problem is 1D then N2 is ignored. 18 | set(COMPUTE_DIM "2") 19 | set(N1 "128") 20 | set(N2 "4") 21 | 22 | # The entire global domain is divided into tiles that are small enough to fit 23 | # into the cache of the compute node or an accelerator. This technique 24 | # optimizes cache usage and makes explicit use of the deep memory hierarchies 25 | # which are prevalent in all kinds of processors and accelerators. 26 | # 27 | # Caution: 1) The tile sizes need to divide the global domain size in each 28 | # direction exactly! 29 | # 2) We want a tile size that is as large as the cache. Too large a 30 | # size and the code can fail silently because local memory in the 31 | # OpenCL specification is a different amount for different 32 | # machines. 33 | # NEED TO PUT A CHECK FOR THIS. 34 | set(TILE_SIZE_X1 "8") 35 | set(TILE_SIZE_X2 "4") 36 | 37 | set(USE_OPENMP "NO") 38 | 39 | # Physics variables 40 | set(ADIABATIC_INDEX "4./3") 41 | set(CONDUCTION "OFF") 42 | set(VISCOSITY "ON") 43 | 44 | # Geometry 45 | set(METRIC "KERRSCHILD") 46 | set(EPS "1e-5") 47 | set(M "1.") 48 | set(BH_SPIN "0.0") 49 | set(H_SLOPE "1.") # Refinement in the theta direction to better resolve the 50 | # disk. H_SLOPE = 1 is no refinement and H_SLOPE -> 0 pushes 51 | # all points towards the midplane. 52 | 53 | # Domain 54 | set(R_A "3.1") 55 | set(R_B "35.") 56 | 57 | set(X1_A "log(R_A)") 58 | set(X1_B "log(R_B)") 59 | set(X2_A "1e-20") 60 | set(X2_B "1. - 1e-20") 61 | 62 | # Boundary conditions 63 | set(PHYSICAL_BOUNDARY_LEFT_EDGE "DIRICHLET") 64 | set(PHYSICAL_BOUNDARY_RIGHT_EDGE "DIRICHLET") 65 | set(PHYSICAL_BOUNDARY_TOP_EDGE "PERIODIC") 66 | set(PHYSICAL_BOUNDARY_BOTTOM_EDGE "PERIODIC") 67 | 68 | # Reconstrution options 69 | # MP5, MONOTONIZED_CENTRAL or MIN_MOD 70 | set(RECONSTRUCTION "MONOTONIZED_CENTRAL") 71 | 72 | # Initial condition parameters 73 | # Everything in units of M 74 | set(SONICRADIUS "8.") 75 | set(MDOT "2.") 76 | set(BMAG "1.") 77 | 78 | # Floor values 79 | set(RHO_FLOOR "1e-4") 80 | set(UU_FLOOR "1e-6") 81 | set(RHO_FLOOR_FALLOFF "-1.5") 82 | set(UU_FLOOR_FALLOFF "-2.5") 83 | set(RHO_FLOOR_MIN "1e-15") 84 | set(UU_FLOOR_MIN "1e-15") 85 | set(GAMMA_MAX "10.") 86 | 87 | message("") 88 | message("#########################") 89 | message("# Configuration options #") 90 | message("#########################") 91 | message("") 92 | message("Problem : " ${PROBLEM}) 93 | message("Output dir : " ${OUTPUT_DIR}) 94 | message("Time stepping : " ${TIME_STEPPING}) 95 | message("Dimensions : " ${COMPUTE_DIM}) 96 | message("Resolution : " ${N1} " x " ${N2}) 97 | message("Tile size : " ${TILE_SIZE_X1} " x " ${TILE_SIZE_X2}) 98 | message("Metric : " ${METRIC}) 99 | 100 | message("") 101 | message("#########################") 102 | message("# Geometry options #") 103 | message("#########################") 104 | message("") 105 | message("Metric : " ${METRIC}) 106 | message("Black hole mass M : " ${M}) 107 | message("Black hole spin BH_SPIN : " ${BH_SPIN}) 108 | 109 | message("") 110 | message("##################################") 111 | message("# Domain and boundary conditions #") 112 | message("##################################") 113 | message("") 114 | message("Inner radius : " ${R_A}) 115 | message("Outer radius : " ${R_B}) 116 | message("Inner radial boundary : " ${PHYSICAL_BOUNDARY_LEFT_EDGE}) 117 | message("Outer radial boundary : " ${PHYSICAL_BOUNDARY_RIGHT_EDGE}) 118 | message("Bottom theta boundary : " ${PHYSICAL_BOUNDARY_BOTTOM_EDGE}) 119 | message("Top theta boundary : " ${PHYSICAL_BOUNDARY_TOP_EDGE}) 120 | message("Grid refinement factor H_SLOPE : " ${H_SLOPE}) 121 | 122 | message("") 123 | message("###################") 124 | message("# Physics options #") 125 | message("###################") 126 | message("") 127 | message("Adiabatic index : " ${ADIABATIC_INDEX}) 128 | message("Conduction : " ${CONDUCTION}) 129 | 130 | message("") 131 | message("##################") 132 | message("# Reconstruction #") 133 | message("##################") 134 | message("") 135 | message("Reconstruction : " ${RECONSTRUCTION}) 136 | 137 | message("") 138 | message("######################") 139 | message("# Initial conditions #") 140 | message("######################") 141 | message("") 142 | message("SONICRADIUS : " ${SONICRADIUS}) 143 | message("MDOT : " ${MDOT}) 144 | message("BMAG : " ${BMAG}) 145 | 146 | message("") 147 | message("##########") 148 | message("# Floors #") 149 | message("##########") 150 | message("") 151 | message("Density floor : ${RHO_FLOOR} x r^${RHO_FLOOR_FALLOFF}") 152 | message("Internal energy floor : ${UU_FLOOR} x r^${UU_FLOOR_FALLOFF}") 153 | message("Maximum Lorentz factor : " ${GAMMA_MAX}) 154 | -------------------------------------------------------------------------------- /src/geometry/geometryPy.pyx: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | cimport numpy as np 3 | from gridPy cimport gridPy, coordinatesGridPy 4 | from geometryHeaders cimport geometry 5 | from geometryHeaders cimport METRICS_MINKOWSKI 6 | from geometryHeaders cimport METRICS_MODIFIED_KERR_SCHILD 7 | 8 | np.import_array() 9 | 10 | # Metric macros 11 | MINKOWSKI = METRICS_MINKOWSKI 12 | MODIFIED_KERR_SCHILD = METRICS_MODIFIED_KERR_SCHILD 13 | 14 | cdef class geometryPy(object): 15 | 16 | def setGeometryPy(self): 17 | gCovGridPy = gridPy.createGridPyFromGridPtr(self.geometryPtr.gCovGrid) 18 | gConGridPy = gridPy.createGridPyFromGridPtr(self.geometryPtr.gConGrid) 19 | gGridPy = gridPy.createGridPyFromGridPtr(self.geometryPtr.gGrid) 20 | alphaGridPy = gridPy.createGridPyFromGridPtr(self.geometryPtr.alphaGrid) 21 | 22 | xCoordsGridPy = gridPy.createGridPyFromGridPtr(self.geometryPtr.xCoordsGrid) 23 | 24 | self.gCov = gCovGridPy.getVars() 25 | self.gCov = self.gCov.reshape([4, 4, 26 | self.gCov.shape[1], 27 | self.gCov.shape[2], 28 | self.gCov.shape[3] 29 | ] 30 | ) 31 | 32 | self.gCon = gConGridPy.getVars() 33 | self.gCon = self.gCon.reshape([4, 4, 34 | self.gCon.shape[1], 35 | self.gCon.shape[2], 36 | self.gCon.shape[3] 37 | ] 38 | ) 39 | 40 | self.g = gGridPy.getVars() 41 | self.g = self.g.reshape([self.g.shape[1], 42 | self.g.shape[2], 43 | self.g.shape[3] 44 | ] 45 | ) 46 | self.alpha = alphaGridPy.getVars() 47 | self.alpha = self.alpha.reshape([self.alpha.shape[1], 48 | self.alpha.shape[2], 49 | self.alpha.shape[3] 50 | ] 51 | ) 52 | self.xCoords = xCoordsGridPy.getVars() 53 | 54 | def __cinit__(self, int metric = 9999, 55 | double blackHoleSpin = 9999, 56 | double hSlope = 9999, 57 | coordinatesGridPy XCoordsGrid = coordinatesGridPy() 58 | ): 59 | if (metric == 9999): 60 | self.usingExternalPtr = 1 61 | self.geometryPtr = NULL 62 | return 63 | 64 | self.usingExternalPtr = 0 65 | self.geometryPtr = new geometry(metric, 66 | blackHoleSpin, 67 | hSlope, 68 | XCoordsGrid.getGridPtr()[0] 69 | ) 70 | self.setGeometryPy() 71 | 72 | def __dealloc__(self): 73 | if (self.usingExternalPtr): 74 | return 75 | del self.geometryPtr 76 | 77 | @staticmethod 78 | cdef createGeometryPyFromGeometryPtr(geometry *geometryPtr): 79 | cdef geometryPy geometryPyObject = geometryPy() 80 | geometryPyObject.usingExternalPtr = 1 81 | geometryPyObject.setGeometryPtr(geometryPtr) 82 | geometryPyObject.setGeometryPy() 83 | return geometryPyObject 84 | 85 | def computeConnectionCoeffs(self): 86 | self.geometryPtr.computeConnectionCoeffs() 87 | self.geometryPtr.setgammaUpDownDownGrid() 88 | gammaUpDownDownGridPy = gridPy() 89 | gammaUpDownDownGridPy.setGridPtr(self.geometryPtr.gammaUpDownDownGrid) 90 | self.gammaUpDownDown = gammaUpDownDownGridPy.getVars() 91 | self.gammaUpDownDown = \ 92 | self.gammaUpDownDown.reshape([4, 4, 4, 93 | self.gammaUpDownDown.shape[1], 94 | self.gammaUpDownDown.shape[2], 95 | self.gammaUpDownDown.shape[3] 96 | ] 97 | ) 98 | 99 | cdef geometry* getGeometryPtr(self): 100 | return self.geometryPtr 101 | 102 | cdef setGeometryPtr(self, geometry *geometryPtr): 103 | self.geometryPtr = geometryPtr 104 | 105 | property N1: 106 | def __get__(self): 107 | return self.geometryPtr.N1 108 | 109 | property N2: 110 | def __get__(self): 111 | return self.geometryPtr.N2 112 | 113 | property N3: 114 | def __get__(self): 115 | return self.geometryPtr.N3 116 | 117 | property dim: 118 | def __get__(self): 119 | return self.geometryPtr.dim 120 | 121 | property numGhost: 122 | def __get__(self): 123 | return self.geometryPtr.numGhost 124 | 125 | property gCon: 126 | def __get__(self): 127 | return self.gCon 128 | 129 | property gCov: 130 | def __get__(self): 131 | return self.gCov 132 | 133 | property g: 134 | def __get__(self): 135 | return self.g 136 | 137 | property alpha: 138 | def __get__(self): 139 | return self.alpha 140 | 141 | property gammaUpDownDown: 142 | def __get__(self): 143 | return self.gammaUpDownDown 144 | 145 | property xCoords: 146 | def __get__(self): 147 | return self.xCoords 148 | 149 | property metric: 150 | def __get__(self): 151 | return self.geometryPtr.metric 152 | 153 | property blackHoleSpin: 154 | def __get__(self): 155 | return self.geometryPtr.blackHoleSpin 156 | 157 | property hSlope: 158 | def __get__(self): 159 | return self.geometryPtr.hSlope 160 | -------------------------------------------------------------------------------- /src/problem/bondi_inflow/problem.cpp: -------------------------------------------------------------------------------- 1 | #include "../problem.hpp" 2 | 3 | void fluidElement::setFluidElementParameters() 4 | { 5 | tau = one; 6 | chi_emhd = soundSpeed*soundSpeed*tau; 7 | nu_emhd = soundSpeed*soundSpeed*tau; 8 | } 9 | 10 | void timeStepper::applyProblemSpecificFluxFilter(int &numReads,int &numWrites) 11 | { 12 | } 13 | 14 | double FuncT(const double T, const double R, const double C1, const double C2) 15 | { 16 | double nPoly = 1./(params::adiabaticIndex-1.); 17 | return pow(1.+(1.+nPoly)*T,2.)*(1.-2./R+pow(C1/R/R/pow(T,nPoly),2.))-C2; 18 | } 19 | 20 | double SolveT(const double R, const double C1, const double C2) 21 | { 22 | double nPoly = 1./(params::adiabaticIndex-1.); 23 | double rtol = 1.e-12; 24 | double ftol = 1.e-14; 25 | double Tmin = 0.6*(sqrt(C2)-1.)/(nPoly+1.); 26 | double Tmax = pow(C1*sqrt(2./R/R/R),1./nPoly); 27 | double f0,f1,fh; 28 | double T0,T1,Th; 29 | T0=0.6*Tmin; 30 | f0=FuncT(T0,R,C1,C2); 31 | T1=Tmax; 32 | f1=FuncT(T1,R,C1,C2); 33 | if(f0*f1>0.) 34 | { 35 | printf("Failed solving for T at R = %f; C1 = %f; C2 = %f \n",R,C1,C2); 36 | PetscPrintf(PETSC_COMM_WORLD, "Failed determination of T \n"); 37 | exit(1); 38 | } 39 | Th = (f1*T0-f0*T1)/(f1-f0); 40 | fh = FuncT(Th,R,C1,C2); 41 | double EpsT = rtol*(Tmin+Tmax); 42 | while(fabs(Th-T0)>EpsT && fabs(Th-T1)>EpsT && fabs(fh)>ftol) 43 | { 44 | if(fh*f0<0.) 45 | { 46 | T0=Th; 47 | f0=fh; 48 | } 49 | else 50 | { 51 | T1=Th; 52 | f1=fh; 53 | } 54 | Th = (f1*T0-f0*T1)/(f1-f0); 55 | fh = FuncT(Th,R,C1,C2); 56 | } 57 | return Th; 58 | } 59 | 60 | void timeStepper::initialConditions(int &numReads,int &numWrites) 61 | { 62 | const double nPoly = 1./(params::adiabaticIndex-1.); 63 | const double Rc = params::sonicRadius; 64 | const double uc = sqrt(0.5/Rc); 65 | const double vc2 = nPoly*uc*uc/(1.-3.*uc*uc); 66 | const double Tc = vc2/(1.-vc2)/(nPoly+1.); 67 | const double C1 = pow(Tc,nPoly)*uc*Rc*Rc; 68 | const double C2 = pow(1.+(nPoly+1.)*Tc,2.)*(1.-2./Rc+uc*uc); 69 | const double Kp = pow(params::mDot/4./M_PI/C1,-1./nPoly); 70 | 71 | array xCoords[3]; 72 | for(int d=0;d<3;d++) 73 | xCoords[d]=XCoords->vars[d]; 74 | geomCenter->XCoordsToxCoords(XCoords->vars,xCoords); 75 | 76 | const array& R = xCoords[0]; 77 | 78 | //Ok, let's ignore ArrayFire here - it's only the initial 79 | //conditions of a test... 80 | const int N1g = params::N1+2*params::numGhost; 81 | const int N2g = params::dim>1 ? params::N2+2*params::numGhost : 1; 82 | const int N3g = params::dim>2 ? params::N3+2*params::numGhost : 1; 83 | 84 | printf("Using grid of %i x %i x %i\n",N1g,N2g,N3g); 85 | 86 | double* host_R = R.host(); 87 | double T[N1g]; 88 | double Rho0[N1g]; 89 | double ur[N1g]; 90 | double ut[N1g]; 91 | 92 | for(int i=0;i2.+1.e-8) 97 | { 98 | Tl = SolveT(Rl,C1,C2); 99 | rhol = pow(Tl/Kp,nPoly); 100 | url = -C1/Rl/Rl/pow(Tl,nPoly); 101 | utl = (2.*url/Rl+sqrt(url*url+1.-2./Rl))/(1.-2./Rl); 102 | } 103 | else 104 | { 105 | Tl = 1.; rhol=params::rhoFloorInFluidElement; 106 | url=0.; utl=1.; 107 | } 108 | T[i]=Tl; Rho0[i]=rhol; ur[i]=url; ut[i]=utl; 109 | //printf("r=%e; rho=%e; T=%e;\n",Rl,rhol,Tl); 110 | } 111 | 112 | //Set initial conditions 113 | primOld->vars[vars::RHO] = 1.; 114 | primOld->vars[vars::U] = 0.; 115 | primOld->vars[vars::U1] = 0.; 116 | primOld->vars[vars::B1] = 0.; 117 | for(int i=0;ivars[vars::RHO](i,j,k) = Rho0[i]; 124 | primOld->vars[vars::U](i,j,k) = nPoly*Rho0[i]*T[i]; 125 | primOld->vars[vars::U1](i,j,k) = (ur[i] + ut[i]*2./(Rl+2.))/Rl; 126 | primOld->vars[vars::B1](i,j,k) = params::bMag/Rl/Rl/Rl; 127 | } 128 | } 129 | primOld->vars[vars::U2] = 0.; 130 | primOld->vars[vars::U3] = 0.; 131 | primOld->vars[vars::B2] = 0.; 132 | primOld->vars[vars::B3] = 0.; 133 | 134 | for (int var=0; varvars[var].eval(); 137 | } 138 | 139 | for (int var=0; varvars[var]=1.*primOld->vars[var]; 142 | primIC->vars[var].eval(); 143 | } 144 | 145 | af::sync(); 146 | } 147 | 148 | void timeStepper::halfStepDiagnostics(int &numReads,int &numWrites) 149 | { 150 | printf("halfStepDiagnostics\n"); 151 | } 152 | 153 | void timeStepper::fullStepDiagnostics(int &numReads,int &numWrites) 154 | { 155 | printf("fullStepDiagnostics: Time = %e\n",params::Time); 156 | //af_print(primOld->varsOld->vars[vars::RHO],8.); 157 | } 158 | 159 | void timeStepper::setProblemSpecificBCs(int &numReads,int &numWrites) 160 | { 161 | for (int var=0; varvars[var](i,span,span)=primIC->vars[var](i,span,span); 165 | for(int i=params::N1+params::numGhost;ivars[var](i,span,span)=primIC->vars[var](i,span,span); 167 | if(params::dim==1) 168 | continue; 169 | for(int i=0;ivars[var](span,i,span)=primIC->vars[var](span,i,span); 171 | for(int i=params::N2+params::numGhost;ivars[var](span,i,span)=primIC->vars[var](span,i,span); 173 | if(params::dim==2) 174 | continue; 175 | for(int i=0;ivars[var](span,span,i)=primIC->vars[var](span,span,i); 177 | for(int i=params::N3+params::numGhost;ivars[var](span,span,i)=primIC->vars[var](span,span,i); 179 | } 180 | for (int var=0; varvars[var].eval(); 182 | af::sync(); 183 | }; 184 | -------------------------------------------------------------------------------- /src/geometry/CoordinateChangeFunctionsArray.hpp: -------------------------------------------------------------------------------- 1 | // Original HARM coordinate map R(X1), Theta(X2) 2 | // from Charles Gammie 3 | 4 | array GammieRadius(const array X1) 5 | { 6 | array res = af::exp(X1); 7 | res.eval(); 8 | return res; 9 | } 10 | 11 | array GammieTheta(const array X2) 12 | { 13 | array res = M_PI*X2+ 0.5*(1 - params::hSlope)*af::sin(2.*M_PI*X2); 14 | res.eval(); 15 | return res; 16 | } 17 | 18 | 19 | // Modify Theta without cylindrification, but 20 | // setting a constant spacing on the horizon 21 | array Ftr( const array x ); 22 | array ThetaNoCyl(const array X1, const array X2) 23 | { 24 | if(params::DerefineThetaHorizon) 25 | { 26 | // This leads to Theta(X2) as in HARM at large distances 27 | // and Theta ~ X2 for small radii: 28 | // X2=cst lines are straight lines in the r-z plane 29 | // The lines are equally spaced on the excision surface 30 | // but use increase resolution in the equatorial plane 31 | // at large radii, as in HARM 32 | array cTh0 = af::cos(M_PI*X2); 33 | array cTh1 = af::cos(GammieTheta(X2)); 34 | array cTh = cTh1 + exp(params::X1Start)/af::exp(X1)*(cTh0-cTh1); 35 | array res = af::acos(cTh); 36 | res.eval(); 37 | return res; 38 | } 39 | else 40 | { 41 | // Use standard HARM choice 42 | array res = GammieTheta(X2); 43 | res.eval(); 44 | return res; 45 | } 46 | } 47 | 48 | 49 | /************************************** 50 | // The following code is largely copied 51 | // from Sasha Tchekhovskoy's version of HARM, 52 | // adapted to arrayFire (and without the generic mapping) 53 | ***************************************/ 54 | 55 | //smooth step function: 56 | // Ftr = 0 if x < 0, Ftr = 1 if x > 1 and smoothly interps. in btw. 57 | array Ftr( const array x ) 58 | { 59 | array condition1 = (x<=0); 60 | array condition2 = (x>=1); 61 | array res = condition2; 62 | res = res + (condition1-1)*(condition2-1)* 63 | (af::cos(x*5*M_PI) + 70*af::sin(((2*x-1)*M_PI)/2.) + 5*af::sin(((2*x-1)*3*M_PI)/2.) + 64.)/128.; 64 | res.eval(); 65 | return res; 66 | } 67 | 68 | array Ftrgenlin( const array x, const array xa, const array xb, const array ya, const array yb ) 69 | { 70 | array res = (x*ya)/xa + (-((x*ya)/xa) + ((x - xb)*(1 - yb))/(1 - xb) + yb)*Ftr((x - xa)/(xb - xa)); 71 | res.eval(); 72 | return res; 73 | } 74 | 75 | //goes from ya to yb as x goes from xa to xb 76 | array Ftrgen(const array x, const array xa, const array xb, const array ya, const array yb ) 77 | { 78 | array res = ya + (yb-ya)*Ftr( (x-xa)/(xb-xa) ); 79 | res.eval(); 80 | return res; 81 | } 82 | 83 | array Fangle(const array x ) 84 | { 85 | array condition1 = (x<=-1); 86 | array condition2 = (x>=1); 87 | array res = condition2*x + (condition1-1)*(condition2-1)* 88 | (x + 1. + (-140*af::sin(((x+1)*M_PI)/2.) + (10*af::sin(((x+1)*3*M_PI)/2.))/3. + (2*af::sin(((x+1)*5*M_PI)/2.))/5.)/(64.*M_PI))/2.; 89 | res.eval(); 90 | return res; 91 | } 92 | 93 | array limlin( const array x, const array x0, const array dx, const array y0 ) 94 | { 95 | array res = y0 - dx * Fangle((x-x0)*(-1.)/dx); 96 | res.eval(); 97 | return res; 98 | } 99 | 100 | array minlin(const array x,const array x0,const array dx,const array y0 ) 101 | { 102 | array res = y0 + dx * Fangle((x-x0)/dx); 103 | res.eval(); 104 | return res; 105 | } 106 | 107 | array mins(const array f1,const array f2,const array df ) 108 | { 109 | return limlin(f1, f2, df, f2); 110 | } 111 | 112 | array maxs(const array f1,const array f2,const array df ) 113 | { 114 | return (mins(-f1, -f2, df)*(-1.) ); 115 | } 116 | 117 | //=mins if dir < 0 118 | //=maxs if dir >= 0 119 | array minmaxs(const array f1,const array f2,const array df,const array dir ) 120 | { 121 | array condition = (dir>=0); 122 | array res = condition*maxs(f1, f2, df) 123 | +(1-condition)*mins(f1, f2, df); 124 | res.eval(); 125 | return res; 126 | } 127 | 128 | 129 | array sinth0( array X0[3], array X[3]) 130 | { 131 | array Xc0[3]; 132 | for(int j=0;j<3;j++) 133 | Xc0[j]=X[j]; 134 | Xc0[directions::X2] = X0[directions::X2]; 135 | array res = GammieRadius(X0[directions::X1]) * af::sin(ThetaNoCyl(X0[directions::X1],X0[directions::X2])) / GammieRadius(Xc0[directions::X1]); 136 | res.eval(); 137 | return res; 138 | } 139 | 140 | array sinth1in( array X0[3], array X[3]) 141 | { 142 | array X0c[3]; 143 | for(int j=0;j<3;j++) 144 | X0c[j]=X0[j]; 145 | X0c[directions::X2] = X[directions::X2]; 146 | array res = GammieRadius(X0[directions::X1]) * af::sin(ThetaNoCyl(X0c[directions::X1],X0c[directions::X2])) / GammieRadius(X[directions::X1]); 147 | res.eval(); 148 | return res; 149 | } 150 | 151 | 152 | array th2in( array X0[3], array X[3]) 153 | { 154 | array Xc0[3]; 155 | array Xcmid[3]; 156 | for(int j=0;j<3;j++) 157 | Xc0[j]=X[j]; 158 | Xc0[directions::X2] = X0[directions::X2]; 159 | for(int j=0;j<3;j++) 160 | Xcmid[j]=X[j]; 161 | Xcmid[directions::X2] = 0.5; 162 | 163 | array th0 = af::asin( sinth0(X0, X) ); 164 | th0.eval(); 165 | array res = (ThetaNoCyl(X[directions::X1],X[directions::X2]) - ThetaNoCyl(Xc0[directions::X1],Xc0[directions::X2]))/ 166 | (ThetaNoCyl(Xcmid[directions::X1],Xcmid[directions::X2]) - ThetaNoCyl(Xc0[directions::X1],Xc0[directions::X2])) * 167 | (ThetaNoCyl(Xcmid[directions::X1],Xcmid[directions::X2])-th0) + th0; 168 | res.eval(); 169 | return res; 170 | } 171 | 172 | array func1( array X0[3], array X[3]) 173 | { 174 | array res = af::sin(ThetaNoCyl(X[directions::X1],X[directions::X2])); 175 | res.eval(); 176 | return res; 177 | } 178 | 179 | array func2( array X0[3], array X[3]) 180 | { 181 | array Xca[3]; 182 | for(int j=0;j<3;j++) 183 | Xca[j]=X[j]; 184 | Xca[directions::X2] = 0.; 185 | 186 | array sth1in = sinth1in( X0, X); 187 | array sth2in = af::sin( th2in(X0, X)); 188 | 189 | array sth1inaxis = sinth1in( X0, Xca); 190 | array sth2inaxis = af::sin( th2in(X0, Xca) ); 191 | 192 | array func2 = minmaxs( sth1in, sth2in, af::abs(sth2inaxis-sth1inaxis)+1.e-20, X[directions::X1] - X0[directions::X1] ); 193 | func2.eval(); 194 | return func2; 195 | } 196 | -------------------------------------------------------------------------------- /src/reconstruction/ppm.cpp: -------------------------------------------------------------------------------- 1 | #include "reconstruction.hpp" 2 | 3 | array reconstruction::slopePPM(const int dir,const double dX, const array& in, 4 | int &numReads, int &numWrites 5 | ) 6 | { 7 | // Construct stencil 8 | int x0Shift, y0Shift, z0Shift; 9 | int x1Shift, y1Shift, z1Shift; 10 | int x3Shift, y3Shift, z3Shift; 11 | int x4Shift, y4Shift, z4Shift; 12 | switch (dir) 13 | { 14 | case directions::X1: 15 | x0Shift = 2; y0Shift = 0; z0Shift = 0; 16 | x1Shift = 1; y1Shift = 0; z1Shift = 0; 17 | x3Shift = -1; y3Shift = 0; z3Shift = 0; 18 | x4Shift = -2; y4Shift = 0; z4Shift = 0; 19 | break; 20 | 21 | case directions::X2: 22 | x0Shift = 0; y0Shift = 2; z0Shift = 0; 23 | x1Shift = 0; y1Shift = 1; z1Shift = 0; 24 | x3Shift = 0; y3Shift = -1; z3Shift = 0; 25 | x4Shift = 0; y4Shift = -2; z4Shift = 0; 26 | break; 27 | 28 | case directions::X3: 29 | x0Shift = 0; y0Shift = 0; z0Shift = 2; 30 | x1Shift = 0; y1Shift = 0; z1Shift = 1; 31 | x3Shift = 0; y3Shift = 0; z3Shift = -1; 32 | x4Shift = 0; y4Shift = 0; z4Shift = -2; 33 | break; 34 | } 35 | 36 | array y0 = af::shift(in, x0Shift, y0Shift, z0Shift); 37 | array y1 = af::shift(in, x1Shift, y1Shift, z1Shift); 38 | array y2 = in; 39 | array y3 = af::shift(in, x3Shift, y3Shift, z3Shift); 40 | array y4 = af::shift(in, x4Shift, y4Shift, z4Shift); 41 | 42 | // Approximants for slopes 43 | array d0 = 2.*(y1-y0); 44 | array d1 = 2.*(y2-y1); 45 | array d2 = 2.*(y3-y2); 46 | array d3 = 2.*(y4-y3); 47 | array D1 = 0.5*(y2-y0); 48 | array D2 = 0.5*(y3-y1); 49 | array D3 = 0.5*(y4-y2); 50 | 51 | array condZeroSlope1 = (d1*d0<=0.); 52 | array sign1 = (D1>0.)*2.-1.; 53 | array DQ1 = (1.-condZeroSlope1)*sign1*af::min(af::abs(D1),af::min(af::abs(d0),af::abs(d1))); 54 | array condZeroSlope2 = (d2*d1<=0.); 55 | array sign2 = (D2>0.)*2.-1.; 56 | array DQ2 = (1.-condZeroSlope2)*sign2*af::min(af::abs(D2),af::min(af::abs(d1),af::abs(d2))); 57 | array condZeroSlope3 = (d3*d2<=0.); 58 | array sign3 = (D3>0.)*2.-1.; 59 | array DQ3 = (1.-condZeroSlope3)*sign3*af::min(af::abs(D3),af::min(af::abs(d2),af::abs(d3))); 60 | 61 | // Base high-order PPM reconstruction 62 | array leftV = 0.5*(y2+y1)-1./6.*(DQ2-DQ1); 63 | array rightV = 0.5*(y3+y2)-1./6.*(DQ3-DQ2); 64 | 65 | // Corrections 66 | array corr1 = ((rightV-y2)*(y2-leftV)<=0.); 67 | array qd = rightV-leftV; 68 | array qe = 6.*(y2-0.5*(rightV+leftV)); 69 | array corr2 = (qd*(qd-qe)<0.); 70 | array corr3 = (qd*(qd+qe)<0.); 71 | leftV = leftV*(1.-corr1)+corr1*y2; 72 | rightV = rightV*(1.-corr1)+corr1*y2; 73 | 74 | leftV = leftV*(1.-corr2)+corr2*(3.*y2-2.*rightV); 75 | rightV = rightV*corr2+(1.-corr2)*rightV*(1.-corr3)+(1.-corr2)*corr3*(3.*y2-2.*leftV); 76 | return (rightV-leftV)/dX; 77 | } 78 | 79 | // PPM algorithm, adapted from HARM code (by X. Guan) 80 | // ref. Colella && Woodward's PPM paper 81 | void reconstruction::reconstructPPM(const grid &prim, 82 | const int dir, 83 | grid &primLeft, 84 | grid &primRight, 85 | int &numReads, 86 | int &numWrites 87 | ) 88 | { 89 | // Construct stencil 90 | int x0Shift, y0Shift, z0Shift; 91 | int x1Shift, y1Shift, z1Shift; 92 | int x3Shift, y3Shift, z3Shift; 93 | int x4Shift, y4Shift, z4Shift; 94 | switch (dir) 95 | { 96 | case directions::X1: 97 | x0Shift = 2; y0Shift = 0; z0Shift = 0; 98 | x1Shift = 1; y1Shift = 0; z1Shift = 0; 99 | x3Shift = -1; y3Shift = 0; z3Shift = 0; 100 | x4Shift = -2; y4Shift = 0; z4Shift = 0; 101 | break; 102 | 103 | case directions::X2: 104 | x0Shift = 0; y0Shift = 2; z0Shift = 0; 105 | x1Shift = 0; y1Shift = 1; z1Shift = 0; 106 | x3Shift = 0; y3Shift = -1; z3Shift = 0; 107 | x4Shift = 0; y4Shift = -2; z4Shift = 0; 108 | break; 109 | 110 | case directions::X3: 111 | x0Shift = 0; y0Shift = 0; z0Shift = 2; 112 | x1Shift = 0; y1Shift = 0; z1Shift = 1; 113 | x3Shift = 0; y3Shift = 0; z3Shift = -1; 114 | x4Shift = 0; y4Shift = 0; z4Shift = -2; 115 | break; 116 | } 117 | 118 | for(int var=0; var0.)*2.-1.; 137 | array DQ1 = (1.-condZeroSlope1)*sign1*af::min(af::abs(D1),af::min(af::abs(d0),af::abs(d1))); 138 | array condZeroSlope2 = (d2*d1<=0.); 139 | array sign2 = (D2>0.)*2.-1.; 140 | array DQ2 = (1.-condZeroSlope2)*sign2*af::min(af::abs(D2),af::min(af::abs(d1),af::abs(d2))); 141 | array condZeroSlope3 = (d3*d2<=0.); 142 | array sign3 = (D3>0.)*2.-1.; 143 | array DQ3 = (1.-condZeroSlope3)*sign3*af::min(af::abs(D3),af::min(af::abs(d2),af::abs(d3))); 144 | 145 | // Base high-order PPM reconstruction 146 | array leftV = 0.5*(y2+y1)-1./6.*(DQ2-DQ1); 147 | array rightV = 0.5*(y3+y2)-1./6.*(DQ3-DQ2); 148 | 149 | // Corrections 150 | array corr1 = ((rightV-y2)*(y2-leftV)<=0.); 151 | array qd = rightV-leftV; 152 | array qe = 6.*(y2-0.5*(rightV+leftV)); 153 | array corr2 = (qd*(qd-qe)<0.); 154 | array corr3 = (qd*(qd+qe)<0.); 155 | leftV = leftV*(1.-corr1)+corr1*y2; 156 | rightV = rightV*(1.-corr1)+corr1*y2; 157 | 158 | leftV = leftV*(1.-corr2)+corr2*(3.*y2-2.*rightV); 159 | rightV = rightV*corr2+(1.-corr2)*rightV*(1.-corr3)+(1.-corr2)*corr3*(3.*y2-2.*leftV); 160 | 161 | primLeft.vars[var] = leftV; 162 | primRight.vars[var] = rightV; 163 | } 164 | } 165 | -------------------------------------------------------------------------------- /src/problem/atmosphere/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Dump folder 2 | set(OUTPUT_DIR "\"/home/manic/grim_refactored/grim/build/\"") 3 | set(RESTART "OFF") 4 | set(RESTART_FILE "\"restartfile.h5\"") 5 | set(DUMP_FILE_PREFIX "\"data\"") 6 | set(PROBLEM_DATA "\"${PROBLEM}/problemdata.h\"") 7 | 8 | # Time stepping options: EXPLICIT, IMEX or IMPLICIT 9 | set(TIME_STEPPING "EXPLICIT") 10 | set(DT "0.0001") 11 | set(DT_DUMP "1.") 12 | set(START_TIME "0.") 13 | set(FINAL_TIME "100.") 14 | set(COURANT "0.5") 15 | set(MAX_DT_INCREMENT "1.3") 16 | 17 | # Domain size. If the problem is 1D then N2 is ignored. 18 | set(COMPUTE_DIM "2") 19 | set(N1 "128") 20 | set(N2 "128") 21 | 22 | # The entire global domain is divided into tiles that are small enough to fit 23 | # into the cache of the compute node or an accelerator. This technique 24 | # optimizes cache usage and makes explicit use of the deep memory hierarchies 25 | # which are prevalent in all kinds of processors and accelerators. 26 | # 27 | # Caution: 1) The tile sizes need to divide the global domain size in each 28 | # direction exactly! 29 | # 2) We want a tile size that is as large as the cache. Too large a 30 | # size and the code can fail silently because local memory in the 31 | # OpenCL specification is a different amount for different 32 | # machines. 33 | # NEED TO PUT A CHECK FOR THIS. 34 | set(TILE_SIZE_X1 "16") 35 | set(TILE_SIZE_X2 "16") 36 | 37 | set(USE_OPENMP "NO") 38 | 39 | # Physics variables 40 | set(ADIABATIC_INDEX "4./3") 41 | set(CONDUCTION "ON") 42 | 43 | # Geometry 44 | set(METRIC "KERRSCHILD") 45 | set(EPS "1e-5") 46 | set(M "1.") 47 | set(BH_SPIN "0.") 48 | set(H_SLOPE "1.") # Refinement in the theta direction to better resolve the 49 | # disk. H_SLOPE = 1 is no refinement and H_SLOPE -> 0 pushes 50 | # all points towards the midplane. 51 | 52 | # Domain 53 | set(R_A "10.*(1. + sqrt(1. - ${BH_SPIN}*${BH_SPIN}))") 54 | set(R_B "300.") 55 | 56 | set(X1_A "log(R_A)") 57 | set(X1_B "log(R_B)") 58 | set(X2_A "1e-20") 59 | set(X2_B "1. - 1e-20") 60 | 61 | # Boundary conditions 62 | set(PHYSICAL_BOUNDARY_LEFT_EDGE "DIRICHLET") 63 | set(PHYSICAL_BOUNDARY_RIGHT_EDGE "DIRICHLET") 64 | set(PHYSICAL_BOUNDARY_TOP_EDGE "MIRROR") 65 | set(PHYSICAL_BOUNDARY_BOTTOM_EDGE "MIRROR") 66 | 67 | # Reconstrution options 68 | # MP5, MONOTONIZED_CENTRAL or MIN_MOD 69 | set(RECONSTRUCTION "MONOTONIZED_CENTRAL") 70 | 71 | # Initial condition parameters 72 | # Everything in units of M 73 | set(PLASMA_BETA "1e5") 74 | set(PERTURBATIONS_AMPLITUDE "4e-2") 75 | set(MAGNETIC_FIELD_CONFIGURATION "RADIAL") 76 | set(RHO_INPUT_FILE "\"/home/mc/data/mc/PhD/grim/build/atmosphere_soln_rho.txt\"") 77 | set(UU_INPUT_FILE "\"/home/mc/data/mc/PhD/grim/build/atmosphere_soln_u.txt\"") 78 | set(PHI_INPUT_FILE "\"/home/mc/data/mc/PhD/grim/build/atmosphere_soln_phi.txt\"") 79 | set(RCOORDS_INPUT_FILE "\"/home/mc/data/mc/PhD/grim/build/atmosphere_soln_rCoords.txt\"") 80 | 81 | # Floor values 82 | set(RHO_FLOOR "1e-4") 83 | set(UU_FLOOR "1e-6") 84 | set(RHO_FLOOR_FALLOFF "-1.5") 85 | set(UU_FLOOR_FALLOFF "-2.5") 86 | set(RHO_FLOOR_MIN "1e-15") 87 | set(UU_FLOOR_MIN "1e-15") 88 | 89 | message("") 90 | message("#########################") 91 | message("# Configuration options #") 92 | message("#########################") 93 | message("") 94 | message("Problem : " ${PROBLEM}) 95 | message("Output dir : " ${OUTPUT_DIR}) 96 | message("Time stepping : " ${TIME_STEPPING}) 97 | message("Dimensions : " ${COMPUTE_DIM}) 98 | message("Resolution : " ${N1} " x " ${N2}) 99 | message("Tile size : " ${TILE_SIZE_X1} " x " ${TILE_SIZE_X2}) 100 | message("Metric : " ${METRIC}) 101 | 102 | message("") 103 | message("#########################") 104 | message("# Geometry options #") 105 | message("#########################") 106 | message("") 107 | message("Metric : " ${METRIC}) 108 | message("Black hole mass M : " ${M}) 109 | message("Black hole spin BH_SPIN : " ${BH_SPIN}) 110 | 111 | message("") 112 | message("##################################") 113 | message("# Domain and boundary conditions #") 114 | message("##################################") 115 | message("") 116 | message("Inner radius : " ${R_A}) 117 | message("Outer radius : " ${R_B}) 118 | message("Inner radial boundary : " ${PHYSICAL_BOUNDARY_LEFT_EDGE}) 119 | message("Outer radial boundary : " ${PHYSICAL_BOUNDARY_RIGHT_EDGE}) 120 | message("Bottom theta boundary : " ${PHYSICAL_BOUNDARY_BOTTOM_EDGE}) 121 | message("Top theta boundary : " ${PHYSICAL_BOUNDARY_TOP_EDGE}) 122 | message("Grid refinement factor H_SLOPE : " ${H_SLOPE}) 123 | 124 | message("") 125 | message("###################") 126 | message("# Physics options #") 127 | message("###################") 128 | message("") 129 | message("Adiabatic index : " ${ADIABATIC_INDEX}) 130 | message("Conduction : " ${CONDUCTION}) 131 | 132 | message("") 133 | message("##################") 134 | message("# Reconstruction #") 135 | message("##################") 136 | message("") 137 | message("Reconstruction : " ${RECONSTRUCTION}) 138 | 139 | message("") 140 | message("######################") 141 | message("# Initial conditions #") 142 | message("######################") 143 | message("") 144 | message("Magnetic field configuration : " ${MAGNETIC_FIELD_CONFIGURATION}) 145 | message("Plasma beta : " ${PLASMA_BETA}) 146 | message("Pressure perturbation amplitude : " ${PERTURBATIONS_AMPLITUDE}) 147 | message("Density input file : " ${RHO_INPUT_FILE}) 148 | message("Internal energy input file : " ${UU_INPUT_FILE}) 149 | message("Radial velocity input file : " ${UR_INPUT_FILE}) 150 | message("Heat flux input file : " ${PHI_INPUT_FILE}) 151 | message("Radial coordinates input file : " ${RCOORDS_INPUT_FILE}) 152 | 153 | message("") 154 | message("##########") 155 | message("# Floors #") 156 | message("##########") 157 | message("") 158 | message("Density floor : ${RHO_FLOOR} x r^${RHO_FLOOR_FALLOFF}") 159 | message("Internal energy floor : ${UU_FLOOR} x r^${UU_FLOOR_FALLOFF}") 160 | -------------------------------------------------------------------------------- /src/problem/performance_testing/problem.cpp: -------------------------------------------------------------------------------- 1 | #include "../problem.hpp" 2 | 3 | void fluidElement::setFluidElementParameters() 4 | { 5 | tau = one; 6 | chi_emhd = soundSpeed*soundSpeed*tau; 7 | nu_emhd = soundSpeed*soundSpeed*tau; 8 | } 9 | 10 | void timeStepper::initialConditions(int &numReads,int &numWrites) 11 | { 12 | array &rho = primOld->vars[vars::RHO]; 13 | array &u = primOld->vars[vars::U]; 14 | array &u1 = primOld->vars[vars::U1]; 15 | array &u2 = primOld->vars[vars::U2]; 16 | array &u3 = primOld->vars[vars::U3]; 17 | array &q = primOld->vars[vars::Q]; 18 | array &dP = primOld->vars[vars::DP]; 19 | 20 | PetscRandom randNumGen; 21 | PetscRandomCreate(PETSC_COMM_SELF, &randNumGen); 22 | PetscRandomSetType(randNumGen, PETSCRAND48); 23 | int numGhostX1 = primOld->numGhostX1; 24 | int numGhostX2 = primOld->numGhostX2; 25 | int numGhostX3 = primOld->numGhostX3; 26 | int N1Local = primOld->N1Local; 27 | int N2Local = primOld->N2Local; 28 | int N3Local = primOld->N3Local; 29 | 30 | for (int k=numGhostX3; k < N3Local+numGhostX3; k++) 31 | { 32 | for (int j=numGhostX2; j < N2Local+numGhostX2; j++) 33 | { 34 | for (int i=numGhostX1; i < N1Local+numGhostX1; i++) 35 | { 36 | double randNum = 0.5; 37 | 38 | PetscRandomGetValue(randNumGen, &randNum); 39 | rho(i,j,k) = 1 + 0.001*(randNum-0.5); // Needs to be positive 40 | 41 | PetscRandomGetValue(randNumGen, &randNum); 42 | u(i,j,k) = 1 + 0.001*(randNum-0.5); // same as above 43 | 44 | PetscRandomGetValue(randNumGen, &randNum); 45 | u1(i,j,k) = 0.001*(randNum-0.5); 46 | 47 | PetscRandomGetValue(randNumGen, &randNum); 48 | u2(i,j,k) = 0.001*(randNum-0.5); 49 | 50 | PetscRandomGetValue(randNumGen, &randNum); 51 | u3(i,j,k) = 0.001*(randNum-0.5); 52 | 53 | PetscRandomGetValue(randNumGen, &randNum); 54 | q(i,j,k) = 0.001*(randNum-0.5); 55 | 56 | PetscRandomGetValue(randNumGen, &randNum); 57 | dP(i,j,k) = 0.001*(randNum-0.5); 58 | } 59 | } 60 | } 61 | PetscRandomDestroy(&randNumGen); 62 | } 63 | 64 | void timeStepper::halfStepDiagnostics(int &numReadsTmp,int &numWritesTmp) 65 | { 66 | //solve(*primOld); 67 | 68 | array jacobianAoS = af::reorder(jacobianSoA, 3, 0, 1, 2); 69 | 70 | /* RHS of Ax = b in Array of Structs format */ 71 | array bAoS = -af::reorder(residualSoA, 3, 0, 1, 2); 72 | 73 | /* Now solve Ax = b using direct inversion, where 74 | * A = Jacobian 75 | * x = deltaPrim 76 | * b = -residual */ 77 | batchLinearSolve(jacobianAoS, bAoS, deltaPrimAoS); 78 | 79 | int numReads, numWrites; 80 | computeDivOfFluxes(*primHalfStep, numReads, numWrites); 81 | 82 | elemOld->set(*primOld, *geomCenter, 83 | numReads, numWrites 84 | ); 85 | elemOld->computeFluxes(0, *consOld, 86 | numReads, numWrites 87 | ); 88 | af::sync(); 89 | 90 | /* ======= End of Warm up ======== */ 91 | 92 | af::timer linearSolverTimer = af::timer::start(); 93 | for (int i=0; i < params::nIters; i++) 94 | { 95 | array jacobianAoS = af::reorder(jacobianSoA, 3, 0, 1, 2); 96 | array bAoS = -af::reorder(residualSoA, 3, 0, 1, 2); 97 | batchLinearSolve(jacobianAoS, bAoS, deltaPrimAoS); 98 | af::sync(); 99 | } 100 | double linearSolverTime = af::timer::stop(linearSolverTimer); 101 | 102 | double zoneCyclesPerSec = params::nIters 103 | * (params::N1*params::N2*params::N3) 104 | / (linearSolverTime * 2. * params::maxNonLinearIter); // factor of 2 to account for half step and full step 105 | PetscPrintf(PETSC_COMM_WORLD, "\nTime taken for linear solver = %g secs, Zone cycles/sec = %g\n", 106 | linearSolverTime, zoneCyclesPerSec 107 | ); 108 | 109 | af::sync(); 110 | af::timer divFluxTimer = af::timer::start(); 111 | for (int i=0; i < params::nIters; i++) 112 | { 113 | computeDivOfFluxes(*primHalfStep, numReads, numWrites); 114 | af::sync(); 115 | } 116 | double divFluxTime = af::timer::stop(divFluxTimer); 117 | 118 | zoneCyclesPerSec = params::nIters 119 | * (params::N1*params::N2*params::N3) 120 | / (divFluxTime * 2.); // factor of 2 to account for half step and full step 121 | PetscPrintf(PETSC_COMM_WORLD, "\nTime taken for divFlux = %g secs, Zone cycles/sec = %g\n", 122 | divFluxTime, zoneCyclesPerSec 123 | ); 124 | 125 | af::sync(); 126 | af::timer elemSetTimer = af::timer::start(); 127 | for (int i=0; i < params::nIters; i++) 128 | { 129 | elemOld->set(*primOld, *geomCenter, 130 | numReads, numWrites 131 | ); 132 | elemOld->computeFluxes(0, *consOld, 133 | numReads, numWrites 134 | ); 135 | af::eval(consOld->vars[vars::RHO], 136 | consOld->vars[vars::U], 137 | consOld->vars[vars::U1], 138 | consOld->vars[vars::U2], 139 | consOld->vars[vars::U3], 140 | consOld->vars[vars::B1], 141 | consOld->vars[vars::B2], 142 | consOld->vars[vars::B3], 143 | consOld->vars[vars::Q], 144 | consOld->vars[vars::DP] 145 | ); 146 | af::sync(); 147 | } 148 | double elemSetTime = af::timer::stop(elemSetTimer); 149 | 150 | int numCallsInRiemannSolver = 2; 151 | int numCallsInTimeStep = 2; 152 | int numCallsinComputeResidual = 1; 153 | int numCalls = 2 * (numCallsInTimeStep + (3*numCallsInRiemannSolver)); // 3 : one for each direction 154 | zoneCyclesPerSec = params::nIters 155 | * (params::N1*params::N2*params::N3) 156 | / (elemSetTime * numCalls); 157 | PetscPrintf(PETSC_COMM_WORLD, "\nTime taken for elem->Set(), elem->ComputeFluxes(0) [conserved vars] = %g secs, Zone cycles/sec = %g\n", 158 | elemSetTime, zoneCyclesPerSec 159 | ); 160 | 161 | /* TODO: Check computeResidual() */ 162 | 163 | PetscEnd(); 164 | } 165 | 166 | void timeStepper::fullStepDiagnostics(int &numReads,int &numWrites) 167 | { 168 | 169 | } 170 | 171 | void timeStepper::setProblemSpecificBCs(int &numReads,int &numWrites) 172 | { 173 | 174 | } 175 | 176 | void timeStepper::applyProblemSpecificFluxFilter(int &numReads,int &numWrites) 177 | { 178 | 179 | } 180 | 181 | 182 | int timeStepper::CheckWallClockTermination() 183 | { 184 | return 0; 185 | } 186 | --------------------------------------------------------------------------------