├── .gitignore ├── AMReX_Amr101 ├── Exec │ ├── GNUmakefile │ ├── GNUmakefile_movie │ ├── Make.Adv │ ├── Make.package │ ├── Prob.H │ ├── inputs │ ├── inputs_for_scaling │ └── paraview_amr101.py ├── README └── Source │ ├── AdvancePhiAllLevels.cpp │ ├── AdvancePhiAtLevel.cpp │ ├── AmrCoreAdv.H │ ├── AmrCoreAdv.cpp │ ├── DefineVelocity.cpp │ ├── Kernels.H │ ├── Make.package │ ├── Src_K │ ├── Adv_K.H │ ├── Make.package │ ├── compute_flux_2D_K.H │ ├── compute_flux_3D_K.H │ └── slope_K.H │ ├── Tagging.H │ ├── bc_fill.H │ ├── face_velocity.H │ └── main.cpp ├── AMReX_Amr102 ├── Exec │ ├── GNUmakefile │ ├── GNUmakefile_movie │ ├── Make.Amr │ ├── inputs │ └── paraview_amr102.py └── Source │ ├── DefineVelocity.cpp │ ├── EB_Cylinder.cpp │ ├── FluidParticleContainer.H │ ├── FluidParticleContainer.cpp │ ├── Indexing.H │ ├── Make.package │ ├── face_velocity.H │ ├── mac_project_velocity.cpp │ └── main.cpp ├── AMReX_EB_MacProj ├── GNUmakefile ├── GNUmakefile_from_source ├── Make.package ├── MyParticleContainer.H ├── MyParticleContainer.cpp ├── README ├── initial_particles_3d ├── inputs_3d ├── macproj.gif ├── main.cpp ├── makeparticlefile_3d.py └── paraview_vis_script.py ├── AMReX_EB_Pachinko ├── GNUmakefile ├── GNUmakefile_movie ├── Make.package ├── MyParticleContainer.H ├── MyParticleContainer.cpp ├── README ├── initial_particles_3d ├── inputs_3d ├── main.cpp ├── pachinko.gif └── paraview_pachinko.py ├── SUNDIALS+AMReX ├── CMakeLists.txt ├── HandsOn.hpp ├── HandsOn1.cpp ├── HandsOn2.cpp ├── HandsOn3.cpp ├── HandsOn_main.cpp ├── README.md ├── diag_tools.py ├── inputs ├── inputs-1 ├── inputs-2 ├── inputs-3 ├── inputs-ref ├── process_ARKStep_diags.py ├── reference_solution │ ├── Header │ └── Level_0 │ │ ├── Cell_D_00000 │ │ ├── Cell_D_00001 │ │ ├── Cell_D_00002 │ │ ├── Cell_D_00003 │ │ └── Cell_H └── source_cooley_plotfile_tools.sh ├── TAO-of-AMReX ├── AMReX_BCUtil.cpp ├── MyTest.H ├── MyTest.cpp ├── MyTestPlotfile.cpp ├── README.md ├── boundary_control.cpp ├── initProb.cpp ├── initProb_K.H ├── inputs └── makefile ├── amrex-plotfile-example ├── GNUmakefile ├── README.md └── main.cpp ├── source_cooley_plotfile_tools.sh └── vis ├── ffmpeg_make_mp4 ├── mkmovie.sh └── slice.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Prerequisites 2 | *.d 3 | 4 | # Compiled Object files 5 | *.slo 6 | *.lo 7 | *.o 8 | *.obj 9 | 10 | # Precompiled Headers 11 | *.gch 12 | *.pch 13 | 14 | # Compiled Dynamic libraries 15 | *.so 16 | *.dylib 17 | *.dll 18 | 19 | # Fortran module files 20 | *.mod 21 | *.smod 22 | 23 | # Compiled Static libraries 24 | *.lai 25 | *.la 26 | *.a 27 | *.lib 28 | 29 | # Executables 30 | *.exe 31 | *.out 32 | *.app 33 | *.ex 34 | 35 | # Merge files 36 | *.orig 37 | 38 | # Backup Files 39 | *~ 40 | 41 | # Build directory 42 | build/ 43 | 44 | # Output directories and files 45 | plt* 46 | movie.visit 47 | ARKStep_diagnostics.txt 48 | 49 | # Debugging 50 | Backtrace.* -------------------------------------------------------------------------------- /AMReX_Amr101/Exec/GNUmakefile: -------------------------------------------------------------------------------- 1 | AMREX_HOME ?= ../../../amrex 2 | 3 | PRECISION = DOUBLE 4 | PROFILE = FALSE 5 | 6 | DEBUG = TRUE 7 | DEBUG = FALSE 8 | 9 | #DIM = 2 10 | DIM = 3 11 | 12 | COMP = gnu 13 | 14 | USE_MPI = TRUE 15 | USE_OMP = FALSE 16 | USE_CUDA = FALSE 17 | 18 | BL_NO_FORT = TRUE 19 | 20 | Bpack := ./Make.package 21 | Blocs := . 22 | 23 | include Make.Adv 24 | -------------------------------------------------------------------------------- /AMReX_Amr101/Exec/GNUmakefile_movie: -------------------------------------------------------------------------------- 1 | PARAVIEW_PATH ?= /path/containing/paraview/executable 2 | 3 | movie2D: 4 | @echo "Making a movie from 2D simulation, this will probably take <30 seconds ..." 5 | @$(PARAVIEW_PATH)/pvpython paraview_amr101.py -d 2 > /dev/null 2>&1 6 | @echo "Done! Generated amr101_2D.avi and amr101_2D.gif" 7 | 8 | movie3D: 9 | @echo "Making a movie from 3D simulation, this will probably take <30 seconds ..." 10 | @$(PARAVIEW_PATH)/pvpython paraview_amr101.py > /dev/null 2>&1 11 | @echo "Done! Generated amr101_3D.avi and amr101_3D.gif" 12 | -------------------------------------------------------------------------------- /AMReX_Amr101/Exec/Make.Adv: -------------------------------------------------------------------------------- 1 | AMREX_HOME ?= ../../../.. 2 | ADV_DIR ?= ../ 3 | 4 | TOP := $(ADV_DIR) 5 | 6 | EBASE := main 7 | 8 | BL_NO_FORT = TRUE 9 | 10 | include $(AMREX_HOME)/Tools/GNUMake/Make.defs 11 | 12 | Bdirs := Source Source/Src_K 13 | Bpack += $(foreach dir, $(Bdirs), $(TOP)/$(dir)/Make.package) 14 | Blocs += $(foreach dir, $(Bdirs), $(TOP)/$(dir)) 15 | 16 | include $(Bpack) 17 | 18 | INCLUDE_LOCATIONS += $(Blocs) 19 | VPATH_LOCATIONS += $(Blocs) 20 | 21 | Pdirs := Base Boundary AmrCore 22 | Ppack += $(foreach dir, $(Pdirs), $(AMREX_HOME)/Src/$(dir)/Make.package) 23 | 24 | include $(Ppack) 25 | 26 | all: $(executable) 27 | @echo SUCCESS 28 | 29 | include $(AMREX_HOME)/Tools/GNUMake/Make.rules 30 | 31 | -------------------------------------------------------------------------------- /AMReX_Amr101/Exec/Make.package: -------------------------------------------------------------------------------- 1 | CEXE_headers += Prob.H 2 | -------------------------------------------------------------------------------- /AMReX_Amr101/Exec/Prob.H: -------------------------------------------------------------------------------- 1 | #ifndef PROB_H_ 2 | #define PROB_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | using namespace amrex; 9 | 10 | AMREX_GPU_DEVICE 11 | AMREX_FORCE_INLINE 12 | void 13 | initdata(Box const& bx, Array4 const& phi, GeometryData const& geomdata) 14 | { 15 | const auto lo = lbound(bx); 16 | const auto hi = ubound(bx); 17 | 18 | const Real* AMREX_RESTRICT prob_lo = geomdata.ProbLo(); 19 | const Real* AMREX_RESTRICT dx = geomdata.CellSize(); 20 | 21 | #ifdef _OPENMP 22 | #pragma omp parallel for collapse(2) if (GPU::notInLaunchRegion()) 23 | #endif 24 | for (int k = lo.z; k <= hi.z; ++k) { 25 | for (int j = lo.y; j <= hi.y; ++j) { 26 | Real z = prob_lo[2] + (0.5+k) * dx[2]; 27 | Real y = prob_lo[1] + (0.5+j) * dx[1]; 28 | AMREX_PRAGMA_SIMD 29 | for (int i = lo.x; i <= hi.x; ++i) { 30 | Real x = prob_lo[0] + (0.5+i) * dx[0]; 31 | Real r2 = (pow(x-0.5, 2) + pow((y-0.75),2)) / 0.01; 32 | phi(i,j,k) = 1.0 + std::exp(-r2); 33 | } 34 | } 35 | } 36 | } 37 | 38 | #endif 39 | -------------------------------------------------------------------------------- /AMReX_Amr101/Exec/inputs: -------------------------------------------------------------------------------- 1 | # ***************************************************************** 2 | # Run until nsteps == max_step or time == stop_time, 3 | # whichever comes first 4 | # ***************************************************************** 5 | max_step = 1000 6 | stop_time = 2.0 7 | 8 | # ***************************************************************** 9 | # Are we restarting from an existing checkpoint file? 10 | # ***************************************************************** 11 | #amr.restart = chk00060 # restart from this checkpoint file 12 | 13 | # ***************************************************************** 14 | # Problem size and geometry 15 | # ***************************************************************** 16 | geometry.prob_lo = 0.0 0.0 0.0 17 | geometry.prob_hi = 1.0 1.0 0.125 18 | geometry.is_periodic = 1 1 1 19 | 20 | # ***************************************************************** 21 | # VERBOSITY 22 | # ***************************************************************** 23 | amr.v = 1 # verbosity in Amr 24 | 25 | # ***************************************************************** 26 | # Resolution and refinement 27 | # ***************************************************************** 28 | amr.n_cell = 64 64 8 29 | amr.max_level = 3 # maximum level number allowed -- 30 | # number of levels = max_level + 1 31 | 32 | amr.ref_ratio = 2 2 2 2 # refinement ratio between levels 33 | 34 | # ***************************************************************** 35 | # Control of grid creation 36 | # ***************************************************************** 37 | # Blocking factor for grid creation in each dimension -- 38 | # this ensures that every grid is coarsenable by a factor of 8 -- 39 | # this is mostly relevant for multigrid performance 40 | amr.blocking_factor_x = 8 41 | amr.blocking_factor_y = 8 42 | amr.blocking_factor_z = 8 43 | 44 | amr.max_grid_size = 64 45 | 46 | amr.regrid_int = 2 # how often to regrid 47 | 48 | # ***************************************************************** 49 | # Time step control 50 | # ***************************************************************** 51 | adv.cfl = 0.7 # CFL constraint for explicit advection 52 | 53 | adv.do_subcycle = 1 # Do we subcycle in time? 54 | 55 | # ***************************************************************** 56 | # Should we reflux at coarse-fine boundaries? 57 | # ***************************************************************** 58 | adv.do_reflux = 1 59 | 60 | # ***************************************************************** 61 | # Tagging - if phi > 1.01 at level 0, then refine 62 | # if phi > 1.1 at level 1, then refine 63 | # if phi > 1.5 at level 2, then refine 64 | # ***************************************************************** 65 | adv.phierr = 1.01 1.1 1.5 66 | 67 | # ***************************************************************** 68 | # Plotfile name and frequency 69 | # ***************************************************************** 70 | amr.plot_file = plt # root name of plot file 71 | amr.plot_int = 5 # number of timesteps between plot files 72 | # if negative then no plot files will be written 73 | 74 | # ***************************************************************** 75 | # Checkpoint name and frequency 76 | # ***************************************************************** 77 | amr.chk_file = chk # root name of checkpoint file 78 | amr.chk_int = -1 # number of timesteps between checkpoint files 79 | # if negative then no checkpoint files will be written 80 | -------------------------------------------------------------------------------- /AMReX_Amr101/Exec/inputs_for_scaling: -------------------------------------------------------------------------------- 1 | # ***************************************************************** 2 | # Run until nsteps == max_step or time == stop_time, 3 | # whichever comes first 4 | # ***************************************************************** 5 | max_step = 1000 6 | max_step = 10 7 | stop_time = 2.0 8 | 9 | # ***************************************************************** 10 | # Are we restarting from an existing checkpoint file? 11 | # ***************************************************************** 12 | #amr.restart = chk00060 # restart from this checkpoint file 13 | 14 | # ***************************************************************** 15 | # Problem size and geometry 16 | # ***************************************************************** 17 | geometry.prob_lo = 0.0 0.0 0.0 18 | geometry.prob_hi = 1.0 1.0 0.25 19 | geometry.is_periodic = 1 1 1 20 | 21 | # ***************************************************************** 22 | # VERBOSITY 23 | # ***************************************************************** 24 | amr.v = 1 # verbosity in Amr 25 | 26 | # ***************************************************************** 27 | # Resolution and refinement 28 | # ***************************************************************** 29 | amr.n_cell = 128 128 32 30 | amr.max_level = 3 # maximum level number allowed -- 31 | # number of levels = max_level + 1 32 | 33 | amr.ref_ratio = 2 2 2 2 # refinement ratio between levels 34 | 35 | # ***************************************************************** 36 | # Control of grid creation 37 | # ***************************************************************** 38 | # Blocking factor for grid creation in each dimension -- 39 | # this ensures that every grid is coarsenable by a factor of 8 -- 40 | # this is mostly relevant for multigrid performance 41 | amr.blocking_factor_x = 8 42 | amr.blocking_factor_y = 8 43 | amr.blocking_factor_z = 8 44 | 45 | amr.max_grid_size = 64 46 | 47 | amr.regrid_int = 2 # how often to regrid 48 | 49 | # ***************************************************************** 50 | # Time step control 51 | # ***************************************************************** 52 | adv.cfl = 0.9 # CFL constraint for explicit advection 53 | 54 | adv.do_subcycle = 1 # Do we subcycle in time? 55 | 56 | # ***************************************************************** 57 | # Should we reflux at coarse-fine boundaries? 58 | # ***************************************************************** 59 | adv.do_reflux = 1 60 | 61 | # ***************************************************************** 62 | # Tagging - if phi > 1.01 at level 0, then refine 63 | # if phi > 1.1 at level 1, then refine 64 | # if phi > 1.5 at level 2, then refine 65 | # ***************************************************************** 66 | adv.phierr = 1.01 1.1 1.5 67 | 68 | # ***************************************************************** 69 | # Plotfile name and frequency 70 | # ***************************************************************** 71 | amr.plot_file = plt # root name of plot file 72 | amr.plot_int = -1 # number of timesteps between plot files 73 | # if negative then no plot files will be written 74 | 75 | # ***************************************************************** 76 | # Checkpoint name and frequency 77 | # ***************************************************************** 78 | amr.chk_file = chk # root name of checkpoint file 79 | amr.chk_int = -1 # number of timesteps between checkpoint files 80 | # if negative then no checkpoint files will be written 81 | -------------------------------------------------------------------------------- /AMReX_Amr101/Exec/paraview_amr101.py: -------------------------------------------------------------------------------- 1 | # based on traces generated using paraview version 5.8.0 2 | # 3 | # To ensure correct image size when batch processing, please search 4 | # for and uncomment the line `# renderView*.ViewSize = [*,*]` 5 | 6 | #### import the simple module from the paraview 7 | from paraview.simple import * 8 | 9 | import subprocess 10 | import glob 11 | import argparse 12 | 13 | parser = argparse.ArgumentParser() 14 | parser.add_argument('-f', '--frame_rate', type=int, default=15, help="Frame rate for generating movies, i.e. number of plots per second in the movie.") 15 | parser.add_argument('-r', '--resolution', type=int, default=1024, help="(Square) resolution of output movie.") 16 | parser.add_argument('-d', '--spacedim', type=int, default=3, help="Dimensionality of the problem: 2 or 3") 17 | args = parser.parse_args() 18 | 19 | def generate_movie_3D(AllPlotFiles): 20 | #### disable automatic camera reset on 'Show' 21 | paraview.simple._DisableFirstRenderCameraReset() 22 | 23 | # create a new 'AMReX/BoxLib Grid Reader' 24 | plt00 = AMReXBoxLibGridReader(FileNames=AllPlotFiles) 25 | plt00.CellArrayStatus = [] 26 | 27 | # get animation scene 28 | animationScene1 = GetAnimationScene() 29 | 30 | # get the time-keeper 31 | timeKeeper1 = GetTimeKeeper() 32 | 33 | # update animation scene based on data timesteps 34 | animationScene1.UpdateAnimationUsingDataTimeSteps() 35 | 36 | # Properties modified on plt00 37 | plt00.CellArrayStatus = ['phi'] 38 | 39 | # get active view 40 | renderView1 = GetActiveViewOrCreate('RenderView') 41 | renderView1.ViewSize = [1200, 1200] 42 | 43 | # get layout 44 | layout1 = GetLayout() 45 | 46 | # show data in view 47 | plt00Display = Show(plt00, renderView1, 'AMRRepresentation') 48 | 49 | # trace defaults for the display properties. 50 | plt00Display.Representation = 'Outline' 51 | plt00Display.ColorArrayName = [None, ''] 52 | plt00Display.OSPRayScaleFunction = 'PiecewiseFunction' 53 | plt00Display.SelectOrientationVectors = 'None' 54 | plt00Display.ScaleFactor = 0.1 55 | plt00Display.SelectScaleArray = 'None' 56 | plt00Display.GlyphType = 'Arrow' 57 | plt00Display.GlyphTableIndexArray = 'None' 58 | plt00Display.GaussianRadius = 0.005 59 | plt00Display.SetScaleArray = [None, ''] 60 | plt00Display.ScaleTransferFunction = 'PiecewiseFunction' 61 | plt00Display.OpacityArray = [None, ''] 62 | plt00Display.OpacityTransferFunction = 'PiecewiseFunction' 63 | plt00Display.DataAxesGrid = 'GridAxesRepresentation' 64 | plt00Display.PolarAxes = 'PolarAxesRepresentation' 65 | plt00Display.ScalarOpacityUnitDistance = 0.030761993184097912 66 | 67 | # reset view to fit data 68 | renderView1.ResetCamera() 69 | 70 | # get the material library 71 | materialLibrary1 = GetMaterialLibrary() 72 | 73 | # update the view to ensure updated data information 74 | renderView1.Update() 75 | 76 | # change solid color 77 | plt00Display.AmbientColor = [0.0, 1.0, 0.0] 78 | plt00Display.DiffuseColor = [0.0, 1.0, 0.0] 79 | 80 | # create a new 'Slice' 81 | slice1 = Slice(Input=plt00) 82 | slice1.SliceType = 'Plane' 83 | slice1.HyperTreeGridSlicer = 'Plane' 84 | slice1.SliceOffsetValues = [0.0] 85 | 86 | # init the 'Plane' selected for 'SliceType' 87 | slice1.SliceType.Origin = [0.5, 0.5, 0.0625] 88 | 89 | # init the 'Plane' selected for 'HyperTreeGridSlicer' 90 | slice1.HyperTreeGridSlicer.Origin = [0.5, 0.5, 0.0625] 91 | 92 | # toggle 3D widget visibility (only when running from the GUI) 93 | #Hide3DWidgets(proxy=slice1.SliceType) 94 | 95 | # Properties modified on slice1.SliceType 96 | slice1.SliceType.Normal = [0.0, 0.0, 1.0] 97 | 98 | # show data in view 99 | slice1Display = Show(slice1, renderView1, 'GeometryRepresentation') 100 | 101 | # trace defaults for the display properties. 102 | slice1Display.Representation = 'Surface' 103 | slice1Display.ColorArrayName = [None, ''] 104 | slice1Display.OSPRayScaleFunction = 'PiecewiseFunction' 105 | slice1Display.SelectOrientationVectors = 'None' 106 | slice1Display.ScaleFactor = 0.1 107 | slice1Display.SelectScaleArray = 'None' 108 | slice1Display.GlyphType = 'Arrow' 109 | slice1Display.GlyphTableIndexArray = 'None' 110 | slice1Display.GaussianRadius = 0.005 111 | slice1Display.SetScaleArray = [None, ''] 112 | slice1Display.ScaleTransferFunction = 'PiecewiseFunction' 113 | slice1Display.OpacityArray = [None, ''] 114 | slice1Display.OpacityTransferFunction = 'PiecewiseFunction' 115 | slice1Display.DataAxesGrid = 'GridAxesRepresentation' 116 | slice1Display.PolarAxes = 'PolarAxesRepresentation' 117 | 118 | # update the view to ensure updated data information 119 | renderView1.Update() 120 | 121 | # set scalar coloring 122 | ColorBy(slice1Display, ('FIELD', 'vtkBlockColors')) 123 | 124 | # get color transfer function/color map for 'vtkBlockColors' 125 | vtkBlockColorsLUT = GetColorTransferFunction('vtkBlockColors') 126 | 127 | # get opacity transfer function/opacity map for 'vtkBlockColors' 128 | vtkBlockColorsPWF = GetOpacityTransferFunction('vtkBlockColors') 129 | 130 | # set scalar coloring 131 | ColorBy(slice1Display, ('CELLS', 'phi')) 132 | 133 | # rescale color and/or opacity maps used to include current data range 134 | slice1Display.RescaleTransferFunctionToDataRange(True, False) 135 | 136 | # get color transfer function/color map for 'phi' 137 | phiLUT = GetColorTransferFunction('phi') 138 | 139 | # get opacity transfer function/opacity map for 'phi' 140 | phiPWF = GetOpacityTransferFunction('phi') 141 | 142 | # show color bar/color legend 143 | slice1Display.SetScalarBarVisibility(renderView1, True) 144 | 145 | # get color legend/bar for phiLUT in view renderView1 146 | phiLUTColorBar = GetScalarBar(phiLUT, renderView1) 147 | 148 | # change scalar bar placement 149 | phiLUTColorBar.WindowLocation = 'Any Location' 150 | phiLUTColorBar.Position = [0, 0.75] 151 | phiLUTColorBar.ScalarBarLength = 0.2 152 | 153 | # current camera placement for renderView1 154 | renderView1.CameraPosition = [0.5, 0.5, 2.3291959654285184] 155 | renderView1.CameraFocalPoint = [0.5, 0.5, 0.0625] 156 | renderView1.CameraParallelScale = 0.7098635432250342 157 | 158 | # save animation 159 | output_movie_base = "amr101_3D" 160 | output_movie = output_movie_base + ".avi" 161 | SaveAnimation(output_movie, 162 | renderView1, 163 | ImageResolution=[1200, 1200], 164 | FrameRate=args.frame_rate, 165 | FrameWindow=[0, len(AllPlotFiles)-1]) 166 | 167 | return output_movie_base, output_movie 168 | 169 | def generate_movie_2D(AllPlotFiles): 170 | #### disable automatic camera reset on 'Show' 171 | paraview.simple._DisableFirstRenderCameraReset() 172 | 173 | # create a new 'AMReX/BoxLib Grid Reader' 174 | plt00 = AMReXBoxLibGridReader(FileNames=AllPlotFiles) 175 | plt00.CellArrayStatus = [] 176 | 177 | # get animation scene 178 | animationScene1 = GetAnimationScene() 179 | 180 | # get the time-keeper 181 | timeKeeper1 = GetTimeKeeper() 182 | 183 | # update animation scene based on data timesteps 184 | animationScene1.UpdateAnimationUsingDataTimeSteps() 185 | 186 | # Properties modified on plt00 187 | plt00.CellArrayStatus = ['phi'] 188 | 189 | # get active view 190 | renderView1 = GetActiveViewOrCreate('RenderView') 191 | # uncomment following to set a specific view size 192 | # renderView1.ViewSize = [1309, 923] 193 | renderView1.ViewSize = [1200, 1200] 194 | 195 | # get layout 196 | layout1 = GetLayout() 197 | 198 | # show data in view 199 | plt00Display = Show(plt00, renderView1, 'AMRRepresentation') 200 | 201 | # trace defaults for the display properties. 202 | plt00Display.Representation = 'Outline' 203 | plt00Display.ColorArrayName = [None, ''] 204 | plt00Display.OSPRayScaleFunction = 'PiecewiseFunction' 205 | plt00Display.SelectOrientationVectors = 'None' 206 | plt00Display.ScaleFactor = 0.1 207 | plt00Display.SelectScaleArray = 'None' 208 | plt00Display.GlyphType = 'Arrow' 209 | plt00Display.GlyphTableIndexArray = 'None' 210 | plt00Display.GaussianRadius = 0.005 211 | plt00Display.SetScaleArray = [None, ''] 212 | plt00Display.ScaleTransferFunction = 'PiecewiseFunction' 213 | plt00Display.OpacityArray = [None, ''] 214 | plt00Display.OpacityTransferFunction = 'PiecewiseFunction' 215 | plt00Display.DataAxesGrid = 'GridAxesRepresentation' 216 | plt00Display.PolarAxes = 'PolarAxesRepresentation' 217 | plt00Display.ScalarOpacityUnitDistance = 0.0701538780193358 218 | 219 | # reset view to fit data 220 | renderView1.ResetCamera() 221 | 222 | #changing interaction mode based on data extents 223 | renderView1.InteractionMode = '2D' 224 | renderView1.CameraPosition = [0.5, 0.5, 10000.0] 225 | renderView1.CameraFocalPoint = [0.5, 0.5, 0.0] 226 | 227 | # get the material library 228 | materialLibrary1 = GetMaterialLibrary() 229 | 230 | # update the view to ensure updated data information 231 | renderView1.Update() 232 | 233 | # change representation type 234 | plt00Display.SetRepresentationType('Surface') 235 | 236 | # set scalar coloring 237 | ColorBy(plt00Display, ('CELLS', 'phi')) 238 | 239 | # rescale color and/or opacity maps used to include current data range 240 | plt00Display.RescaleTransferFunctionToDataRange(True, False) 241 | 242 | # get color transfer function/color map for 'phi' 243 | phiLUT = GetColorTransferFunction('phi') 244 | 245 | # get opacity transfer function/opacity map for 'phi' 246 | phiPWF = GetOpacityTransferFunction('phi') 247 | 248 | # show color bar/color legend 249 | plt00Display.SetScalarBarVisibility(renderView1, True) 250 | 251 | # create a new 'AMReX/BoxLib Grid Reader' 252 | plt00_1 = AMReXBoxLibGridReader(FileNames=AllPlotFiles) 253 | plt00_1.CellArrayStatus = [] 254 | 255 | # show data in view 256 | plt00_1Display = Show(plt00_1, renderView1, 'AMRRepresentation') 257 | 258 | # trace defaults for the display properties. 259 | plt00_1Display.Representation = 'Outline' 260 | plt00_1Display.ColorArrayName = [None, ''] 261 | plt00_1Display.OSPRayScaleFunction = 'PiecewiseFunction' 262 | plt00_1Display.SelectOrientationVectors = 'None' 263 | plt00_1Display.ScaleFactor = 0.1 264 | plt00_1Display.SelectScaleArray = 'None' 265 | plt00_1Display.GlyphType = 'Arrow' 266 | plt00_1Display.GlyphTableIndexArray = 'None' 267 | plt00_1Display.GaussianRadius = 0.005 268 | plt00_1Display.SetScaleArray = [None, ''] 269 | plt00_1Display.ScaleTransferFunction = 'PiecewiseFunction' 270 | plt00_1Display.OpacityArray = [None, ''] 271 | plt00_1Display.OpacityTransferFunction = 'PiecewiseFunction' 272 | plt00_1Display.DataAxesGrid = 'GridAxesRepresentation' 273 | plt00_1Display.PolarAxes = 'PolarAxesRepresentation' 274 | plt00_1Display.ScalarOpacityUnitDistance = 0.0701538780193358 275 | 276 | # update the view to ensure updated data information 277 | renderView1.Update() 278 | 279 | # change solid color 280 | plt00_1Display.AmbientColor = [0.0, 1.0, 0.0] 281 | plt00_1Display.DiffuseColor = [0.0, 1.0, 0.0] 282 | 283 | # get color legend/bar for phiLUT in view renderView1 284 | phiLUTColorBar = GetScalarBar(phiLUT, renderView1) 285 | 286 | # change scalar bar placement 287 | phiLUTColorBar.WindowLocation = 'AnyLocation' 288 | phiLUTColorBar.Position = [0, 0.75] 289 | phiLUTColorBar.ScalarBarLength = 0.2 290 | 291 | # current camera placement for renderView1 292 | renderView1.InteractionMode = '2D' 293 | renderView1.CameraPosition = [0.5, 0.5, 10000.0] 294 | renderView1.CameraFocalPoint = [0.5, 0.5, 0.0] 295 | renderView1.CameraParallelScale = 0.5843857695756589 296 | 297 | # save animation 298 | output_movie_base = "amr101_2D" 299 | output_movie = output_movie_base + ".avi" 300 | SaveAnimation(output_movie, 301 | renderView1, 302 | ImageResolution=[1200, 1200], 303 | FrameRate=args.frame_rate, 304 | FrameWindow=[0, len(AllPlotFiles)-1]) 305 | 306 | return output_movie_base, output_movie 307 | 308 | def convert_avi_to_gif(output_movie_base, output_movie): 309 | # use ffmpeg to convert the avi movie into an animated gif 310 | ffmpeg_convert_to_gif = 'ffmpeg -y -i {} -vf "fps=35,scale={}:-1:flags=lanczos,split[s0][s1];[s0]palettegen[p];[s1][p]paletteuse" -loop 0 {}.gif'.format(output_movie, args.resolution, output_movie_base) 311 | subprocess.run(ffmpeg_convert_to_gif, shell=True) 312 | 313 | if __name__ == "__main__": 314 | if not (args.spacedim == 2 or args.spacedim == 3): 315 | print("Please specify --spacedim D (with D=2 or D=3)") 316 | exit() 317 | 318 | if args.frame_rate <= 0: 319 | print("Please specify --frame_rate F (with F > 0)") 320 | exit() 321 | 322 | if args.resolution <= 0: 323 | print("Please specify --resolution R (with R > 0)") 324 | exit() 325 | 326 | # get all the plotfiles 327 | PlotFiles = sorted(glob.glob("plt" + "[0-9]"*5)) 328 | 329 | # call the 2D or 3D vis function 330 | output_movie_base = None 331 | output_movie = None 332 | 333 | if args.spacedim == 3: 334 | output_movie_base, output_movie = generate_movie_3D(PlotFiles) 335 | elif args.spacedim == 2: 336 | output_movie_base, output_movie = generate_movie_2D(PlotFiles) 337 | 338 | # convert the avi movie into an animated gif 339 | convert_avi_to_gif(output_movie_base, output_movie) 340 | -------------------------------------------------------------------------------- /AMReX_Amr101/README: -------------------------------------------------------------------------------- 1 | Advection_AmrCore: This tutorial contains an AMR advection code that advects a single 2 | scalar field with a velocity field that is specified on faces. 3 | 4 | It also has an option to use the MAC projection to enforce that the specified 5 | velocity field is discretely divergence-free. 6 | 7 | It is an AMReX based code designed to run in parallel using MPI/OMP. 8 | 9 | This example uses source code from the amrex/Src/Base, Boundary, and AmrCore directories. 10 | To see a similar example that uses amrex/Src/Amr rather than AmrCore, see 11 | the tutorial in amrex/Tutorials/Amr/Advection_AmrLevel. 12 | 13 | Each directorie -- Exec/SingleVortex and Exec/SingleVortexProjected -- 14 | includes a makefile and a sample inputs file. 15 | Plotfiles are generated that can be viewed with amrvis2d / amrvis3d 16 | (CCSE's native vis / spreadsheet tool, downloadable separately from 17 | https://github.com/amrex-codes/amrvis) or with VisIt, yt or Paraview. 18 | -------------------------------------------------------------------------------- /AMReX_Amr101/Source/AmrCoreAdv.H: -------------------------------------------------------------------------------- 1 | #ifndef AmrCoreAdv_H_ 2 | #define AmrCoreAdv_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #ifdef _OPENMP 9 | #include 10 | #endif 11 | 12 | #include 13 | #include 14 | #include 15 | 16 | using namespace amrex; 17 | 18 | class AmrCoreAdv 19 | : public amrex::AmrCore 20 | { 21 | public: 22 | 23 | //////////////// 24 | // public member functions 25 | 26 | // constructor - reads in parameters from inputs file 27 | // - sizes multilevel arrays and data structures 28 | AmrCoreAdv (); 29 | virtual ~AmrCoreAdv(); 30 | 31 | // advance solution to final time 32 | void Evolve (); 33 | 34 | // initializes multilevel data 35 | void InitData (); 36 | 37 | // Make a new level using provided BoxArray and DistributionMapping and 38 | // fill with interpolated coarse level data. 39 | // overrides the pure virtual function in AmrCore 40 | virtual void MakeNewLevelFromCoarse (int lev, amrex::Real time, const amrex::BoxArray& ba, 41 | const amrex::DistributionMapping& dm) override; 42 | 43 | // Remake an existing level using provided BoxArray and DistributionMapping and 44 | // fill with existing fine and coarse data. 45 | // overrides the pure virtual function in AmrCore 46 | virtual void RemakeLevel (int lev, amrex::Real time, const amrex::BoxArray& ba, 47 | const amrex::DistributionMapping& dm) override; 48 | 49 | // Delete level data 50 | // overrides the pure virtual function in AmrCore 51 | virtual void ClearLevel (int lev) override; 52 | 53 | // Make a new level from scratch using provided BoxArray and DistributionMapping. 54 | // Only used during initialization. 55 | // overrides the pure virtual function in AmrCore 56 | virtual void MakeNewLevelFromScratch (int lev, amrex::Real time, const amrex::BoxArray& ba, 57 | const amrex::DistributionMapping& dm) override; 58 | 59 | // tag all cells for refinement 60 | // overrides the pure virtual function in AmrCore 61 | virtual void ErrorEst (int lev, amrex::TagBoxArray& tags, amrex::Real time, int ngrow) override; 62 | 63 | // Advance phi at a single level for a single time step, update flux registers 64 | void AdvancePhiAtLevel (int lev, amrex::Real time, amrex::Real dt_lev, int iteration, int ncycle); 65 | 66 | // Advance phi at all levels for a single time step 67 | void AdvancePhiAllLevels (amrex::Real time, amrex::Real dt_lev, int iteration); 68 | 69 | // Define the advection velocity as the curl of a scalar field 70 | void DefineVelocityAtLevel (int lev, amrex::Real time); 71 | 72 | void DefineVelocityAllLevels (amrex::Real time); 73 | 74 | // compute dt from CFL considerations 75 | Real EstTimeStep (int lev, amrex::Real time, bool local=false); 76 | 77 | private: 78 | 79 | //////////////// 80 | // private member functions 81 | 82 | // read in some parameters from inputs file 83 | void ReadParameters(); 84 | 85 | // set covered coarse cells to be the average of overlying fine cells 86 | void AverageDown (); 87 | 88 | // more flexible version of AverageDown() that lets you average down across multiple levels 89 | void AverageDownTo (int crse_lev); 90 | 91 | // compute a new multifab by coping in phi from valid region and filling ghost cells 92 | // works for single level and 2-level cases (fill fine grid ghost by interpolating from coarse) 93 | void FillPatch (int lev, amrex::Real time, amrex::MultiFab& mf, int icomp, int ncomp); 94 | 95 | // fill an entire multifab by interpolating from the coarser level 96 | // this comes into play when a new level of refinement appears 97 | void FillCoarsePatch (int lev, amrex::Real time, amrex::MultiFab& mf, int icomp, int ncomp); 98 | 99 | // utility to copy in data from phi_old and/or phi_new into another multifab 100 | void GetData (int lev, amrex::Real time, amrex::Vector& data, 101 | amrex::Vector& datatime); 102 | 103 | // Advance a level by dt - includes a recursive call for finer levels 104 | void timeStepWithSubcycling (int lev, amrex::Real time, int iteration); 105 | 106 | // Advance all levels by the same dt 107 | void timeStepNoSubcycling (amrex::Real time, int iteration); 108 | 109 | // a wrapper for EstTimeStep(0 110 | void ComputeDt (); 111 | 112 | // get plotfile name 113 | std::string PlotFileName (int lev) const; 114 | 115 | // put together an array of multifabs for writing 116 | amrex::Vector PlotFileMF () const; 117 | 118 | // set plotfile variables names 119 | amrex::Vector PlotFileVarNames () const; 120 | 121 | // write plotfile to disk 122 | void WritePlotFile () const; 123 | 124 | // write checkpoint file to disk 125 | void WriteCheckpointFile () const; 126 | 127 | // read checkpoint file from disk 128 | void ReadCheckpointFile (); 129 | 130 | // utility to skip to next line in Header 131 | static void GotoNextLine (std::istream& is); 132 | 133 | //////////////// 134 | // private data members 135 | 136 | amrex::Vector istep; // which step? 137 | amrex::Vector nsubsteps; // how many substeps on each level? 138 | 139 | // keep track of old time, new time, and time step at each level 140 | amrex::Vector t_new; 141 | amrex::Vector t_old; 142 | amrex::Vector dt; 143 | 144 | // array of multifabs to store the solution at each level of refinement 145 | // after advancing a level we use "swap". 146 | amrex::Vector phi_new; 147 | amrex::Vector phi_old; 148 | 149 | // this is essentially a 2*DIM integer array storing the physical boundary 150 | // condition types at the lo/hi walls in each direction 151 | amrex::Vector bcs; // 1-component 152 | 153 | // stores fluxes at coarse-fine interface for synchronization 154 | // this will be sized "nlevs_max+1" 155 | // NOTE: the flux register associated with flux_reg[lev] is associated 156 | // with the lev/lev-1 interface (and has grid spacing associated with lev-1) 157 | // therefore flux_reg[0] and flux_reg[nlevs_max] are never actually 158 | // used in the reflux operation 159 | amrex::Vector > flux_reg; 160 | 161 | // Velocity on all faces at all levels 162 | amrex::Vector< Array > facevel; 163 | 164 | //////////////// 165 | // runtime parameters 166 | 167 | // maximum number of steps and stop time 168 | int max_step = std::numeric_limits::max(); 169 | amrex::Real stop_time = std::numeric_limits::max(); 170 | 171 | // if >= 0 we restart from a checkpoint 172 | std::string restart_chkfile = ""; 173 | 174 | // advective cfl number - dt = cfl*dx/umax 175 | amrex::Real cfl = 0.7; 176 | 177 | // how often each level regrids the higher levels of refinement 178 | // (after a level advances that many time steps) 179 | int regrid_int = 2; 180 | 181 | // hyperbolic refluxing as part of multilevel synchronization 182 | int do_reflux = 1; 183 | 184 | // do we subcycle in time? 185 | int do_subcycle = 1; 186 | 187 | // plotfile prefix and frequency 188 | std::string plot_file {"plt"}; 189 | int plot_int = -1; 190 | 191 | // checkpoint prefix and frequency 192 | std::string chk_file {"chk"}; 193 | int chk_int = -1; 194 | }; 195 | 196 | #endif 197 | -------------------------------------------------------------------------------- /AMReX_Amr101/Source/DefineVelocity.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | 6 | using namespace amrex; 7 | 8 | void 9 | AmrCoreAdv::DefineVelocityAllLevels (Real time) 10 | { 11 | for (int lev = 0; lev <= finest_level; ++lev) 12 | DefineVelocityAtLevel(lev,time); 13 | 14 | // ======================================================= 15 | // Average down face velocities before using them 16 | // ======================================================= 17 | for (int lev = finest_level; lev > 0; lev--) 18 | { 19 | average_down_faces(amrex::GetArrOfConstPtrs(facevel[lev ]), 20 | amrex::GetArrOfPtrs (facevel[lev-1]), 21 | MaxRefRatio(lev-1), 0); 22 | } 23 | } 24 | 25 | void 26 | AmrCoreAdv::DefineVelocityAtLevel (int lev, Real time) 27 | { 28 | const auto dx = geom[lev].CellSizeArray(); 29 | const Real* prob_lo = geom[lev].ProbLo(); 30 | 31 | #ifdef _OPENMP 32 | #pragma omp parallel if (Gpu::notInLaunchRegion()) 33 | #endif 34 | { 35 | for (MFIter mfi(phi_new[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi) 36 | { 37 | 38 | // ======== GET FACE VELOCITY ========= 39 | GpuArray nbx; 40 | AMREX_D_TERM(nbx[0] = mfi.nodaltilebox(0);, 41 | nbx[1] = mfi.nodaltilebox(1);, 42 | nbx[2] = mfi.nodaltilebox(2);); 43 | 44 | AMREX_D_TERM(const Box& ngbxx = amrex::grow(mfi.nodaltilebox(0),1);, 45 | const Box& ngbxy = amrex::grow(mfi.nodaltilebox(1),1);, 46 | const Box& ngbxz = amrex::grow(mfi.nodaltilebox(2),1);); 47 | 48 | GpuArray, AMREX_SPACEDIM> vel{ AMREX_D_DECL( facevel[lev][0].array(mfi), 49 | facevel[lev][1].array(mfi), 50 | facevel[lev][2].array(mfi)) }; 51 | 52 | const Box& psibox = Box(IntVect(AMREX_D_DECL(std::min(ngbxx.smallEnd(0)-1, ngbxy.smallEnd(0)-1), 53 | std::min(ngbxx.smallEnd(1)-1, ngbxy.smallEnd(0)-1), 54 | 0)), 55 | IntVect(AMREX_D_DECL(std::max(ngbxx.bigEnd(0), ngbxy.bigEnd(0)+1), 56 | std::max(ngbxx.bigEnd(1)+1, ngbxy.bigEnd(1)), 57 | 0))); 58 | 59 | FArrayBox psifab(psibox, 1); 60 | Elixir psieli = psifab.elixir(); 61 | Array4 psi = psifab.array(); 62 | GeometryData geomdata = geom[lev].data(); 63 | auto prob_lo = geom[lev].ProbLoArray(); 64 | auto dx = geom[lev].CellSizeArray(); 65 | 66 | amrex::launch(psibox, 67 | [=] AMREX_GPU_DEVICE (const Box& tbx) 68 | { 69 | get_face_velocity_psi(tbx, time, psi, geomdata); 70 | }); 71 | 72 | AMREX_D_TERM( 73 | amrex::ParallelFor(ngbxx, 74 | [=] AMREX_GPU_DEVICE (int i, int j, int k) 75 | { 76 | get_face_velocity_x(i, j, k, vel[0], psi, prob_lo, dx); 77 | });, 78 | 79 | amrex::ParallelFor(ngbxy, 80 | [=] AMREX_GPU_DEVICE (int i, int j, int k) 81 | { 82 | get_face_velocity_y(i, j, k, vel[1], psi, prob_lo, dx); 83 | });, 84 | 85 | amrex::ParallelFor(ngbxz, 86 | [=] AMREX_GPU_DEVICE (int i, int j, int k) 87 | { 88 | get_face_velocity_z(i, j, k, vel[2], psi, prob_lo, dx); 89 | }); 90 | ); 91 | } 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /AMReX_Amr101/Source/Kernels.H: -------------------------------------------------------------------------------- 1 | #ifndef Kernels_H_ 2 | #define Kernels_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #if (AMREX_SPACEDIM == 2) 12 | #include 13 | #else 14 | #include 15 | #endif 16 | 17 | #endif 18 | -------------------------------------------------------------------------------- /AMReX_Amr101/Source/Make.package: -------------------------------------------------------------------------------- 1 | CEXE_sources += AdvancePhiAtLevel.cpp 2 | CEXE_sources += AdvancePhiAllLevels.cpp 3 | CEXE_sources += AmrCoreAdv.cpp 4 | CEXE_sources += DefineVelocity.cpp 5 | CEXE_sources += main.cpp 6 | 7 | CEXE_headers += AmrCoreAdv.H 8 | CEXE_headers += bc_fill.H 9 | CEXE_headers += face_velocity.H 10 | CEXE_headers += Kernels.H 11 | CEXE_headers += Tagging.H 12 | -------------------------------------------------------------------------------- /AMReX_Amr101/Source/Src_K/Adv_K.H: -------------------------------------------------------------------------------- 1 | #ifndef _Adv_K_H_ 2 | #define _Adv_K_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | using namespace amrex; 9 | 10 | AMREX_GPU_DEVICE 11 | AMREX_FORCE_INLINE 12 | void conservative(int i, int j, int k, 13 | Array4 const& phi_in, 14 | Array4 const& phi_out, 15 | AMREX_D_DECL(Array4 const& flxx, 16 | Array4 const& flxy, 17 | Array4 const& flxz), 18 | const GpuArray& dtdx) 19 | { 20 | phi_out(i,j,k) = phi_in(i,j,k) + 21 | ( AMREX_D_TERM( (flxx(i,j,k) - flxx(i+1,j,k)) * dtdx[0], 22 | + (flxy(i,j,k) - flxy(i,j+1,k)) * dtdx[1], 23 | + (flxz(i,j,k) - flxz(i,j,k+1)) * dtdx[2] ) ); 24 | } 25 | 26 | #if (AMREX_SPACEDIM > 2) 27 | 28 | AMREX_GPU_DEVICE 29 | AMREX_FORCE_INLINE 30 | void flux_scale_x(int i, int j, int k, 31 | Array4 const& flxx, 32 | Real dt, 33 | const GpuArray& dx) 34 | { 35 | flxx(i,j,k) = flxx(i,j,k) * (dt * dx[1]*dx[2]); 36 | } 37 | 38 | AMREX_GPU_DEVICE 39 | AMREX_FORCE_INLINE 40 | void flux_scale_y(int i, int j, int k, 41 | Array4 const& flxy, 42 | Real dt, 43 | const GpuArray& dx) 44 | { 45 | flxy(i,j,k) = flxy(i,j,k) * (dt * dx[0]*dx[2]); 46 | } 47 | 48 | AMREX_GPU_DEVICE 49 | AMREX_FORCE_INLINE 50 | void flux_scale_z(int i, int j ,int k, 51 | Array4 const& flxz, 52 | Real dt, 53 | const GpuArray& dx) 54 | { 55 | flxz(i,j,k) = flxz(i,j,k) * (dt * dx[0]*dx[1]); 56 | } 57 | 58 | #else 59 | 60 | AMREX_GPU_DEVICE 61 | AMREX_FORCE_INLINE 62 | void flux_scale_x(int i, int j, int k, 63 | Array4 const& flxx, 64 | Real dt, 65 | const GpuArray& dx) 66 | { 67 | flxx(i,j,k) = flxx(i,j,k) * (dt * dx[1]); 68 | } 69 | 70 | AMREX_GPU_DEVICE 71 | AMREX_FORCE_INLINE 72 | void flux_scale_y(int i, int j, int k, 73 | Array4 const& flxy, 74 | Real dt, 75 | const GpuArray& dx) 76 | { 77 | flxy(i,j,k) = flxy(i,j,k) * (dt * dx[0]); 78 | } 79 | 80 | #endif 81 | 82 | #endif 83 | -------------------------------------------------------------------------------- /AMReX_Amr101/Source/Src_K/Make.package: -------------------------------------------------------------------------------- 1 | CEXE_headers += Adv_K.H 2 | CEXE_headers += compute_flux_K_$(DIM).H 3 | CEXE_headers += slope_K.H 4 | -------------------------------------------------------------------------------- /AMReX_Amr101/Source/Src_K/compute_flux_2D_K.H: -------------------------------------------------------------------------------- 1 | #ifndef _compute_flux_2d_H_ 2 | #define _compute_flux_2d_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | using namespace amrex; 9 | 10 | AMREX_GPU_DEVICE 11 | AMREX_FORCE_INLINE 12 | void flux_x(int i, int j, int k, 13 | Array4 const& phi, 14 | Array4 const& vx, 15 | Array4 const& px, 16 | Array4 const& slope, 17 | const GpuArray& dtdx) 18 | { 19 | px(i,j,k) = ( (vx(i,j,k) < 0) ? 20 | phi(i ,j,k) - slope(i ,j,k)*(0.5 + 0.5*dtdx[0]*vx(i,j,k)) : 21 | phi(i-1,j,k) + slope(i-1,j,k)*(0.5 - 0.5*dtdx[0]*vx(i,j,k)) ); 22 | } 23 | 24 | AMREX_GPU_DEVICE 25 | AMREX_FORCE_INLINE 26 | void flux_y(int i, int j, int k, 27 | Array4 const& phi, 28 | Array4 const& vy, 29 | Array4 const& py, 30 | Array4 const& slope, 31 | const GpuArray& dtdx) 32 | { 33 | py(i,j,k) = ( (vy(i,j,k) < 0) ? 34 | phi(i,j ,k) - slope(i,j ,k)*(0.5 + 0.5*dtdx[0]*vy(i,j,k)) : 35 | phi(i,j-1,k) + slope(i,j-1,k)*(0.5 - 0.5*dtdx[0]*vy(i,j,k)) ); 36 | } 37 | 38 | AMREX_GPU_DEVICE 39 | AMREX_FORCE_INLINE 40 | void create_flux_x(int i, int j, int k, 41 | AMREX_D_DECL(Array4 const& vx, 42 | Array4 const& vy, 43 | Array4 const& vz), 44 | Array4 const& px, 45 | Array4 const& py, 46 | Array4 const& fx, 47 | const GpuArray& dtdx) 48 | { 49 | fx(i,j,k) = ( (vx(i,j,k) < 0) ? 50 | (px(i,j,k) - 0.5*dtdx[1] * ( 0.5*(vy(i ,j+1,k ) + vy(i ,j,k)) * (py(i ,j+1,k )-py(i ,j,k))))*vx(i,j,k) : 51 | (px(i,j,k) - 0.5*dtdx[1] * ( 0.5*(vy(i-1,j+1,k ) + vy(i-1,j,k)) * (py(i-1,j+1,k )-py(i-1,j,k))))*vx(i,j,k) ); 52 | } 53 | 54 | AMREX_GPU_DEVICE 55 | AMREX_FORCE_INLINE 56 | void create_flux_y(int i, int j, int k, 57 | AMREX_D_DECL(Array4 const& vx, 58 | Array4 const& vy, 59 | Array4 const& vz), 60 | Array4 const& py, 61 | Array4 const& px, 62 | Array4 const& fy, 63 | const GpuArray& dtdx) 64 | { 65 | fy(i,j,k) = ( (vy(i,j,k) < 0) ? 66 | (py(i,j,k) - 0.5*dtdx[0] * ( 0.5*(vx(i+1,j ,k ) + vx(i,j ,k)) * (px(i+1,j ,k )-px(i,j ,k))))*vy(i,j,k) : 67 | (py(i,j,k) - 0.5*dtdx[0] * ( 0.5*(vx(i+1,j-1,k ) + vx(i,j-1,k)) * (px(i+1,j-1,k )-px(i,j-1,k))))*vy(i,j,k) ); 68 | } 69 | 70 | #endif 71 | -------------------------------------------------------------------------------- /AMReX_Amr101/Source/Src_K/compute_flux_3D_K.H: -------------------------------------------------------------------------------- 1 | #ifndef _compute_flux_3d_H_ 2 | #define _compute_flux_3d_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | using namespace amrex; 9 | 10 | AMREX_GPU_DEVICE 11 | AMREX_FORCE_INLINE 12 | void flux_x(int i, int j, int k, 13 | Array4 const& phi, 14 | Array4 const& vx, 15 | Array4 const& px, 16 | Array4 const& slope, 17 | const GpuArray& dtdx) 18 | { 19 | px(i,j,k) = ( (vx(i,j,k) < 0) ? 20 | phi(i ,j,k) - slope(i ,j,k)*(0.5 + 0.5*dtdx[0]*vx(i,j,k)) : 21 | phi(i-1,j,k) + slope(i-1,j,k)*(0.5 - 0.5*dtdx[0]*vx(i,j,k)) ); 22 | } 23 | 24 | AMREX_GPU_DEVICE 25 | AMREX_FORCE_INLINE 26 | void flux_y(int i, int j, int k, 27 | Array4 const& phi, 28 | Array4 const& vy, 29 | Array4 const& py, 30 | Array4 const& slope, 31 | const GpuArray& dtdx) 32 | { 33 | py(i,j,k) = ( (vy(i,j,k) < 0) ? 34 | phi(i,j ,k) - slope(i,j ,k)*(0.5 + 0.5*dtdx[0]*vy(i,j,k)) : 35 | phi(i,j-1,k) + slope(i,j-1,k)*(0.5 - 0.5*dtdx[0]*vy(i,j,k)) ); 36 | } 37 | 38 | AMREX_GPU_DEVICE 39 | AMREX_FORCE_INLINE 40 | void flux_z(int i, int j, int k, 41 | Array4 const& phi, 42 | Array4 const& vz, 43 | Array4 const& pz, 44 | Array4 const& slope, 45 | const GpuArray& dtdx) 46 | { 47 | pz(i,j,k) = ( (vz(i,j,k) < 0) ? 48 | phi(i,j,k ) - slope(i,j,k )*(0.5 + 0.5*dtdx[0]*vz(i,j,k)) : 49 | phi(i,j,k-1) + slope(i,j,k-1)*(0.5 - 0.5*dtdx[0]*vz(i,j,k)) ); 50 | } 51 | 52 | AMREX_GPU_DEVICE 53 | AMREX_FORCE_INLINE 54 | void flux_xy(int i, int j, int k, 55 | AMREX_D_DECL(Array4 const& vx, 56 | Array4 const& vy, 57 | Array4 const& vz), 58 | AMREX_D_DECL(Array4 const& px, 59 | Array4 const& py, 60 | Array4 const& pz), 61 | Array4 const& pxy, 62 | const GpuArray& dtdx) 63 | { 64 | pxy(i,j,k) = ( (vx(i,j,k) < 0) ? 65 | px(i,j,k) - dtdx[1]/3.0 * ( 0.5*(vy(i, j+1,k) + vy(i ,j,k)) * (py(i ,j+1,k) - py(i ,j,k))) : 66 | px(i,j,k) - dtdx[1]/3.0 * ( 0.5*(vy(i-1,j+1,k) + vy(i-1,j,k)) * (py(i-1,j+1,k) - py(i-1,j,k))) ); 67 | } 68 | 69 | AMREX_GPU_DEVICE 70 | AMREX_FORCE_INLINE 71 | void flux_xz(int i, int j, int k, 72 | AMREX_D_DECL(Array4 const& vx, 73 | Array4 const& vy, 74 | Array4 const& vz), 75 | AMREX_D_DECL(Array4 const& px, 76 | Array4 const& py, 77 | Array4 const& pz), 78 | Array4 const& pxz, 79 | const GpuArray& dtdx) 80 | { 81 | pxz(i,j,k) = ( (vx(i,j,k) < 0) ? 82 | px(i,j,k) - dtdx[2]/3.0 * ( 0.5*(vz(i, j,k+1) + vz(i ,j,k)) * (pz(i ,j,k+1) - pz(i ,j,k))) : 83 | px(i,j,k) - dtdx[2]/3.0 * ( 0.5*(vz(i-1,j,k+1) + vz(i-1,j,k)) * (pz(i-1,j,k+1) - pz(i-1,j,k))) ); 84 | } 85 | 86 | AMREX_GPU_DEVICE 87 | AMREX_FORCE_INLINE 88 | void flux_yx(int i, int j, int k, 89 | AMREX_D_DECL(Array4 const& vx, 90 | Array4 const& vy, 91 | Array4 const& vz), 92 | AMREX_D_DECL(Array4 const& px, 93 | Array4 const& py, 94 | Array4 const& pz), 95 | Array4 const& pyx, 96 | const GpuArray& dtdx) 97 | { 98 | pyx(i,j,k) = ( (vy(i,j,k) < 0) ? 99 | py(i,j,k) - dtdx[0]/3.0 * ( 0.5*(vx(i+1,j ,k) + vx(i,j ,k)) * (px(i+1,j ,k) - px(i,j ,k))) : 100 | py(i,j,k) - dtdx[0]/3.0 * ( 0.5*(vx(i+1,j-1,k) + vx(i,j-1,k)) * (px(i+1,j-1,k) - px(i,j-1,k))) ); 101 | } 102 | 103 | AMREX_GPU_DEVICE 104 | AMREX_FORCE_INLINE 105 | void flux_yz(int i, int j, int k, 106 | AMREX_D_DECL(Array4 const& vx, 107 | Array4 const& vy, 108 | Array4 const& vz), 109 | AMREX_D_DECL(Array4 const& px, 110 | Array4 const& py, 111 | Array4 const& pz), 112 | Array4 const& pyz, 113 | const GpuArray& dtdx) 114 | { 115 | pyz(i,j,k) = ( (vy(i,j,k) < 0) ? 116 | py(i,j,k) - dtdx[2]/3.0 * ( 0.5*(vz(i, j,k+1) + vz(i,j ,k)) * (pz(i,j ,k+1) - pz(i,j ,k))) : 117 | py(i,j,k) - dtdx[2]/3.0 * ( 0.5*(vz(i,j-1,k+1) + vz(i,j-1,k)) * (pz(i,j-1,k+1) - pz(i,j-1,k))) ); 118 | } 119 | 120 | AMREX_GPU_DEVICE 121 | AMREX_FORCE_INLINE 122 | void flux_zx(int i, int j, int k, 123 | AMREX_D_DECL(Array4 const& vx, 124 | Array4 const& vy, 125 | Array4 const& vz), 126 | AMREX_D_DECL(Array4 const& px, 127 | Array4 const& py, 128 | Array4 const& pz), 129 | Array4 const& pzx, 130 | const GpuArray& dtdx) 131 | { 132 | pzx(i,j,k) = ( (vz(i,j,k) < 0) ? 133 | pz(i,j,k) - dtdx[0]/3.0 * ( 0.5*(vx(i+1,j,k ) + vx(i,j,k )) * (px(i+1,j,k ) - px(i,j,k ))) : 134 | pz(i,j,k) - dtdx[0]/3.0 * ( 0.5*(vx(i+1,j,k-1) + vx(i,j,k-1)) * (px(i+1,j,k-1) - px(i,j,k-1))) ); 135 | } 136 | 137 | AMREX_GPU_DEVICE 138 | AMREX_FORCE_INLINE 139 | void flux_zy(int i, int j, int k, 140 | AMREX_D_DECL(Array4 const& vx, 141 | Array4 const& vy, 142 | Array4 const& vz), 143 | AMREX_D_DECL(Array4 const& px, 144 | Array4 const& py, 145 | Array4 const& pz), 146 | Array4 const& pzy, 147 | const GpuArray& dtdx) 148 | { 149 | pzy(i,j,k) = ( (vz(i,j,k) < 0) ? 150 | pz(i,j,k) - dtdx[1]/3.0 * ( 0.5*(vy(i,j+1,k ) + vy(i,j,k )) * (py(i,j+1,k ) - py(i,j,k ))) : 151 | pz(i,j,k) - dtdx[1]/3.0 * ( 0.5*(vy(i,j+1,k-1) + vy(i,j,k-1)) * (py(i,j+1,k-1) - py(i,j,k-1))) ); 152 | } 153 | 154 | AMREX_GPU_DEVICE 155 | AMREX_FORCE_INLINE 156 | void create_flux_x(int i, int j, int k, 157 | AMREX_D_DECL(Array4 const& vx, 158 | Array4 const& vy, 159 | Array4 const& vz), 160 | Array4 const& px, 161 | Array4 const& pyz, 162 | Array4 const& pzy, 163 | Array4 const& fx, 164 | const GpuArray& dtdx) 165 | { 166 | px(i,j,k) = ( (vx(i,j,k) < 0) ? 167 | px(i,j,k) - 0.5*dtdx[1] * ( 0.5*(vy(i ,j+1,k ) + vy(i ,j,k)) * (pyz(i ,j+1,k )-pyz(i ,j,k))) 168 | - 0.5*dtdx[2] * ( 0.5*(vz(i ,j ,k+1) + vz(i ,j,k)) * (pzy(i ,j ,k+1)-pzy(i ,j,k))) : 169 | px(i,j,k) - 0.5*dtdx[1] * ( 0.5*(vy(i-1,j+1,k ) + vy(i-1,j,k)) * (pyz(i-1,j+1,k )-pyz(i-1,j,k))) 170 | - 0.5*dtdx[2] * ( 0.5*(vz(i-1,j ,k+1) + vz(i-1,j,k)) * (pzy(i-1,j ,k+1)-pzy(i-1,j,k))) ); 171 | 172 | fx(i,j,k) = vx(i,j,k)*px(i,j,k); 173 | } 174 | 175 | AMREX_GPU_DEVICE 176 | AMREX_FORCE_INLINE 177 | void create_flux_y(int i, int j, int k, 178 | AMREX_D_DECL(Array4 const& vx, 179 | Array4 const& vy, 180 | Array4 const& vz), 181 | Array4 const& py, 182 | Array4 const& pxz, 183 | Array4 const& pzx, 184 | Array4 const& fy, 185 | const GpuArray& dtdx) 186 | { 187 | py(i,j,k) = ( (vy(i,j,k) < 0) ? 188 | py(i,j,k) - 0.5*dtdx[0] * ( 0.5*(vx(i+1,j ,k ) + vx(i,j ,k)) * (pxz(i+1,j ,k )-pxz(i,j ,k))) 189 | - 0.5*dtdx[2] * ( 0.5*(vz(i, j ,k+1) + vz(i,j ,k)) * (pzx(i, j ,k+1)-pzx(i,j ,k))) : 190 | py(i,j,k) - 0.5*dtdx[0] * ( 0.5*(vx(i+1,j-1,k ) + vx(i,j-1,k)) * (pxz(i+1,j-1,k )-pxz(i,j-1,k))) 191 | - 0.5*dtdx[2] * ( 0.5*(vz(i ,j-1,k+1) + vz(i,j-1,k)) * (pzx(i ,j-1,k+1)-pzx(i,j-1,k))) ); 192 | 193 | fy(i,j,k) = vy(i,j,k)*py(i,j,k); 194 | } 195 | 196 | AMREX_GPU_DEVICE 197 | AMREX_FORCE_INLINE 198 | void create_flux_z(int i, int j, int k, 199 | AMREX_D_DECL(Array4 const& vx, 200 | Array4 const& vy, 201 | Array4 const& vz), 202 | Array4 const& pz, 203 | Array4 const& pxy, 204 | Array4 const& pyx, 205 | Array4 const& fz, 206 | const GpuArray& dtdx) 207 | { 208 | pz(i,j,k) = ( (vz(i,j,k) < 0) ? 209 | pz(i,j,k) - 0.5*dtdx[0] * ( 0.5*(vx(i+1,j ,k ) + vx(i,j,k )) * (pxy(i+1,j ,k )-pxy(i,j,k ))) 210 | - 0.5*dtdx[1] * ( 0.5*(vy(i, j+1,k ) + vy(i,j,k )) * (pyx(i, j+1,k )-pyx(i,j,k ))) : 211 | pz(i,j,k) - 0.5*dtdx[0] * ( 0.5*(vx(i+1,j ,k-1) + vx(i,j,k-1)) * (pxy(i+1,j ,k-1)-pxy(i,j,k-1))) 212 | - 0.5*dtdx[1] * ( 0.5*(vy(i ,j+1,k-1) + vy(i,j,k-1)) * (pyx(i ,j+1,k-1)-pyx(i,j,k-1))) ); 213 | 214 | fz(i,j,k) = vz(i,j,k)*pz(i,j,k); 215 | } 216 | 217 | #endif 218 | -------------------------------------------------------------------------------- /AMReX_Amr101/Source/Src_K/slope_K.H: -------------------------------------------------------------------------------- 1 | #ifndef slope_K_H_ 2 | #define slope_K_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | AMREX_GPU_DEVICE 9 | AMREX_FORCE_INLINE 10 | void slopex2(amrex::Box const& bx, 11 | amrex::Array4 const& q, 12 | amrex::Array4 const& dq) 13 | { 14 | const auto lo = amrex::lbound(bx); 15 | const auto hi = amrex::ubound(bx); 16 | 17 | for (int k = lo.z; k <= hi.z; ++k) { 18 | for (int j = lo.y; j <= hi.y; ++j) { 19 | for (int i = lo.x; i <= hi.x; ++i) { 20 | Real dlft = q(i,j,k) - q(i-1,j,k); 21 | Real drgt = q(i+1,j,k) - q(i,j,k); 22 | Real dcen = 0.5*(dlft+drgt); 23 | Real dsgn = amrex::Math::copysign(1.0, dcen); 24 | Real dslop = 2.0 * ((amrex::Math::abs(dlft) < amrex::Math::abs(drgt)) ? 25 | amrex::Math::abs(dlft) : amrex::Math::abs(drgt)); 26 | Real dlim = (dlft*drgt >= 0.0) ? dslop : 0.0; 27 | dq(i,j,k) = dsgn*amrex::min(dlim, amrex::Math::abs(dcen)); 28 | } 29 | } 30 | } 31 | } 32 | 33 | AMREX_GPU_DEVICE 34 | AMREX_FORCE_INLINE 35 | void slopex4(amrex::Box const& bx, 36 | amrex::Array4 const& q, 37 | amrex::Array4 const& dq, 38 | amrex::Array4 const& dq4) 39 | { 40 | const auto lo = amrex::lbound(bx); 41 | const auto hi = amrex::ubound(bx); 42 | 43 | for (int k = lo.z; k <= hi.z; ++k) { 44 | for (int j = lo.y; j <= hi.y; ++j) { 45 | for (int i = lo.x; i <= hi.x; ++i) { 46 | Real dlft = q(i,j,k) - q(i-1,j,k); 47 | Real drgt = q(i+1,j,k) - q(i,j,k); 48 | Real dcen = 0.5*(dlft+drgt); 49 | Real dsgn = amrex::Math::copysign(1.0, dcen); 50 | Real dslop = 2.0 * ((amrex::Math::abs(dlft) < amrex::Math::abs(drgt)) ? 51 | amrex::Math::abs(dlft) : amrex::Math::abs(drgt)); 52 | Real dlim = (dlft*drgt >= 0.0) ? dslop : 0.0; 53 | Real dq1 = 4.0/3.0*dcen - (1.0/6.0)*(dq(i+1,j,k) + dq(i-1,j,k)); 54 | dq4(i,j,k) = dsgn*amrex::min(dlim, amrex::Math::abs(dq1)); 55 | } 56 | } 57 | } 58 | } 59 | 60 | // *********************************************************** 61 | 62 | AMREX_GPU_DEVICE 63 | AMREX_FORCE_INLINE 64 | void slopey2(amrex::Box const& bx, 65 | amrex::Array4 const& q, 66 | amrex::Array4 const& dq) 67 | { 68 | const auto lo = amrex::lbound(bx); 69 | const auto hi = amrex::ubound(bx); 70 | 71 | for (int k = lo.z; k <= hi.z; ++k) { 72 | for (int j = lo.y; j <= hi.y; ++j) { 73 | for (int i = lo.x; i <= hi.x; ++i) { 74 | Real dlft = q(i,j,k) - q(i,j-1,k); 75 | Real drgt = q(i,j+1,k) - q(i,j,k); 76 | Real dcen = 0.5*(dlft+drgt); 77 | Real dsgn = amrex::Math::copysign(1.0, dcen); 78 | Real dslop = 2.0 * ((amrex::Math::abs(dlft) < amrex::Math::abs(drgt)) ? 79 | amrex::Math::abs(dlft) : amrex::Math::abs(drgt)); 80 | Real dlim = (dlft*drgt >= 0.0) ? dslop : 0.0; 81 | dq(i,j,k) = dsgn*amrex::min(dlim, amrex::Math::abs(dcen)); 82 | } 83 | } 84 | } 85 | } 86 | 87 | AMREX_GPU_DEVICE 88 | AMREX_FORCE_INLINE 89 | void slopey4(amrex::Box const& bx, 90 | amrex::Array4 const& q, 91 | amrex::Array4 const& dq, 92 | amrex::Array4 const& dq4) 93 | { 94 | const auto lo = amrex::lbound(bx); 95 | const auto hi = amrex::ubound(bx); 96 | 97 | for (int k = lo.z; k <= hi.z; ++k) { 98 | for (int j = lo.y; j <= hi.y; ++j) { 99 | for (int i = lo.x; i <= hi.x; ++i) { 100 | Real dlft = q(i,j,k) - q(i,j-1,k); 101 | Real drgt = q(i,j+1,k) - q(i,j,k); 102 | Real dcen = 0.5*(dlft+drgt); 103 | Real dsgn = amrex::Math::copysign(1.0, dcen); 104 | Real dslop = 2.0 * ((amrex::Math::abs(dlft) < amrex::Math::abs(drgt)) ? 105 | amrex::Math::abs(dlft) : amrex::Math::abs(drgt)); 106 | Real dlim = (dlft*drgt >= 0.0) ? dslop : 0.0; 107 | Real dq1 = 4.0/3.0*dcen - (1.0/6.0)*(dq(i,j+1,k) + dq(i,j-1,k)); 108 | dq4(i,j,k) = dsgn*amrex::min(dlim, amrex::Math::abs(dq1)); 109 | } 110 | } 111 | } 112 | } 113 | 114 | // *********************************************************** 115 | 116 | AMREX_GPU_DEVICE 117 | AMREX_FORCE_INLINE 118 | void slopez2(amrex::Box const& bx, 119 | amrex::Array4 const& q, 120 | amrex::Array4 const& dq) 121 | { 122 | const auto lo = amrex::lbound(bx); 123 | const auto hi = amrex::ubound(bx); 124 | 125 | for (int k = lo.z; k <= hi.z; ++k) { 126 | for (int j = lo.y; j <= hi.y; ++j) { 127 | for (int i = lo.x; i <= hi.x; ++i) { 128 | Real dlft = q(i,j,k) - q(i,j,k-1); 129 | Real drgt = q(i,j,k+1) - q(i,j,k); 130 | Real dcen = 0.5*(dlft+drgt); 131 | Real dsgn = amrex::Math::copysign(1.0, dcen); 132 | Real dslop = 2.0 * ((amrex::Math::abs(dlft) < amrex::Math::abs(drgt)) ? 133 | amrex::Math::abs(dlft) : amrex::Math::abs(drgt)); 134 | Real dlim = (dlft*drgt >= 0.0) ? dslop : 0.0; 135 | dq(i,j,k) = dsgn*amrex::min(dlim, amrex::Math::abs(dcen)); 136 | } 137 | } 138 | } 139 | } 140 | 141 | AMREX_GPU_DEVICE 142 | AMREX_FORCE_INLINE 143 | void slopez4(amrex::Box const& bx, 144 | amrex::Array4 const& q, 145 | amrex::Array4 const& dq, 146 | amrex::Array4 const& dq4) 147 | { 148 | const auto lo = amrex::lbound(bx); 149 | const auto hi = amrex::ubound(bx); 150 | 151 | for (int k = lo.z; k <= hi.z; ++k) { 152 | for (int j = lo.y; j <= hi.y; ++j) { 153 | for (int i = lo.x; i <= hi.x; ++i) { 154 | Real dlft = q(i,j,k) - q(i,j,k-1); 155 | Real drgt = q(i,j,k+1) - q(i,j,k); 156 | Real dcen = 0.5*(dlft+drgt); 157 | Real dsgn = amrex::Math::copysign(1.0, dcen); 158 | Real dslop = 2.0 * ((amrex::Math::abs(dlft) < amrex::Math::abs(drgt)) ? 159 | amrex::Math::abs(dlft) : amrex::Math::abs(drgt)); 160 | Real dlim = (dlft*drgt >= 0.0) ? dslop : 0.0; 161 | Real dq1 = 4.0/3.0*dcen - (1.0/6.0)*(dq(i,j,k+1) + dq(i,j,k-1)); 162 | dq4(i,j,k) = dsgn*amrex::min(dlim, amrex::Math::abs(dq1)); 163 | } 164 | } 165 | } 166 | } 167 | 168 | #endif 169 | -------------------------------------------------------------------------------- /AMReX_Amr101/Source/Tagging.H: -------------------------------------------------------------------------------- 1 | #ifndef TAGGING_H 2 | #define TAGGING_H 3 | 4 | #include 5 | 6 | AMREX_GPU_HOST_DEVICE 7 | AMREX_FORCE_INLINE 8 | void 9 | state_error (int i, int j, int k, 10 | amrex::Array4 const& tag, 11 | amrex::Array4 const& state, 12 | amrex::Real phierr, char tagval) 13 | { 14 | if (state(i,j,k) > phierr) 15 | tag(i,j,k) = tagval; 16 | } 17 | 18 | #endif 19 | -------------------------------------------------------------------------------- /AMReX_Amr101/Source/bc_fill.H: -------------------------------------------------------------------------------- 1 | #ifndef BCFILL_H 2 | #define BCFILL_H 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | using namespace amrex; 9 | 10 | struct AmrCoreFill 11 | { 12 | AMREX_GPU_DEVICE 13 | void operator() (const IntVect& iv, Array4 const& data, 14 | const int dcomp, const int numcomp, 15 | GeometryData const& geom, const Real time, 16 | const BCRec* bcr, const int bcomp, 17 | const int orig_comp) const 18 | { 19 | // do something for external Dirichlet (BCType::ext_dir) 20 | } 21 | }; 22 | 23 | #endif 24 | -------------------------------------------------------------------------------- /AMReX_Amr101/Source/face_velocity.H: -------------------------------------------------------------------------------- 1 | #ifndef FACE_VELOCITY_H_ 2 | #define FACE_VELOCITY_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | using namespace amrex; 9 | 10 | AMREX_GPU_DEVICE 11 | AMREX_FORCE_INLINE 12 | void get_face_velocity_psi(Box const& bx, 13 | const Real time, 14 | Array4 const& psi, 15 | GeometryData const& geomdata) 16 | { 17 | const auto lo = lbound(bx); 18 | const auto hi = ubound(bx); 19 | 20 | const Real* AMREX_RESTRICT prob_lo = geomdata.ProbLo(); 21 | const Real* AMREX_RESTRICT dx = geomdata.CellSize(); 22 | 23 | for (int j = lo.y; j <= hi.y; ++j) { 24 | Real y = dx[1]*(0.5+j) + prob_lo[1]; 25 | AMREX_PRAGMA_SIMD 26 | for (int i = lo.x; i <= hi.x; ++i) { 27 | Real x = dx[0]*(0.5+i) + prob_lo[0]; 28 | psi(i,j,0) = pow(sin(M_PI*x), 2) * pow(sin(M_PI*y), 2) 29 | * cos(M_PI*time/2.0) * 1.0/M_PI; 30 | } 31 | } 32 | } 33 | 34 | AMREX_GPU_DEVICE 35 | AMREX_FORCE_INLINE 36 | void get_face_velocity_x(int i, int j, int k, 37 | Array4 const& vx, 38 | Array4 const& psi, 39 | GpuArray prob_lo, 40 | GpuArray dx) 41 | { 42 | vx(i,j,k) = -( (psi(i,j+1,0)+psi(i-1,j+1,0)) - (psi(i,j-1,0)+psi(i-1,j-1,0)) ) * (0.25/dx[1]); 43 | } 44 | 45 | AMREX_GPU_DEVICE 46 | AMREX_FORCE_INLINE 47 | void get_face_velocity_y(int i, int j, int k, 48 | Array4 const& vy, 49 | Array4 const& psi, 50 | GpuArray prob_lo, 51 | GpuArray dx) 52 | { 53 | vy(i,j,k) = ( (psi(i+1,j,0)+psi(i+1,j-1,0)) - (psi(i-1,j,0)+psi(i-1,j-1,0)) ) * (0.25/dx[0]); 54 | } 55 | 56 | AMREX_GPU_DEVICE 57 | AMREX_FORCE_INLINE 58 | void get_face_velocity_z(int i, int j, int k, 59 | Array4 const& vz, 60 | Array4 const& psi, 61 | GpuArray prob_lo, 62 | GpuArray dx) 63 | { 64 | vz(i,j,k) = 0.0; 65 | } 66 | 67 | #endif 68 | -------------------------------------------------------------------------------- /AMReX_Amr101/Source/main.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | 10 | using namespace amrex; 11 | 12 | int main(int argc, char* argv[]) 13 | { 14 | amrex::Initialize(argc,argv); 15 | 16 | // timer for profiling 17 | BL_PROFILE_VAR("main()", pmain); 18 | 19 | // wallclock time 20 | const Real strt_total = amrex::second(); 21 | 22 | { 23 | // constructor - reads in parameters from inputs file 24 | // - sizes multilevel arrays and data structures 25 | AmrCoreAdv amr_core_adv; 26 | 27 | // initialize AMR data 28 | amr_core_adv.InitData(); 29 | 30 | // advance solution to final time 31 | amr_core_adv.Evolve(); 32 | 33 | // wallclock time 34 | Real end_total = amrex::second() - strt_total; 35 | 36 | // print wallclock time 37 | ParallelDescriptor::ReduceRealMax(end_total ,ParallelDescriptor::IOProcessorNumber()); 38 | if (amr_core_adv.Verbose()) { 39 | amrex::Print() << "\nTotal Time: " << end_total << '\n'; 40 | } 41 | } 42 | 43 | // destroy timer for profiling 44 | BL_PROFILE_VAR_STOP(pmain); 45 | 46 | amrex::Finalize(); 47 | } 48 | -------------------------------------------------------------------------------- /AMReX_Amr102/Exec/GNUmakefile: -------------------------------------------------------------------------------- 1 | AMREX_HOME ?= ../../../amrex 2 | AMREX_HYDRO_HOME ?= ../../../AMReX-Hydro 3 | 4 | USE_MPI = TRUE 5 | USE_OMP = FALSE 6 | 7 | COMP = gnu 8 | 9 | DIM = 2 10 | DIM = 3 11 | 12 | DEBUG = FALSE 13 | 14 | USE_EB = TRUE 15 | USE_PARTICLES = TRUE 16 | 17 | USE_HYPRE = TRUE 18 | USE_HYPRE = FALSE 19 | 20 | include ./Make.Amr 21 | 22 | movie: 23 | ffmpeg -framerate 10 -pattern_type glob -i "p*.png" -r 5 -vf scale=1024:-1 amr102.gif 24 | -------------------------------------------------------------------------------- /AMReX_Amr102/Exec/GNUmakefile_movie: -------------------------------------------------------------------------------- 1 | PARAVIEW_PATH ?= /path/containing/paraview/executable 2 | 3 | movie2D: 4 | @echo "Making a movie from 2D simulation, this will probably take <30 seconds ..." 5 | @$(PARAVIEW_PATH)/pvpython paraview_amr102.py -d 2 > /dev/null 2>&1 6 | @echo "Done! Generated amr102_2D.avi and amr102_2D.gif" 7 | 8 | movie3D: 9 | @echo "Making a movie from 3D simulation, this will probably take <30 seconds ..." 10 | @$(PARAVIEW_PATH)/pvpython paraview_amr102.py > /dev/null 2>&1 11 | @echo "Done! Generated amr102_3D.avi and amr102_3D.gif" 12 | -------------------------------------------------------------------------------- /AMReX_Amr102/Exec/Make.Amr: -------------------------------------------------------------------------------- 1 | AMREX_HOME ?= ../../../.. 2 | TOP = ../ 3 | 4 | EBASE := main 5 | 6 | BL_NO_FORT = FALSE 7 | 8 | include $(AMREX_HOME)/Tools/GNUMake/Make.defs 9 | 10 | include $(TOP)/Source/Make.package 11 | 12 | INCLUDE_LOCATIONS += $(TOP)/Source 13 | VPATH_LOCATIONS += $(TOP)/Source 14 | 15 | Pdirs := Base Boundary AmrCore EB LinearSolvers/MLMG Particle 16 | Ppack += $(foreach dir, $(Pdirs), $(AMREX_HOME)/Src/$(dir)/Make.package) 17 | 18 | include $(Ppack) 19 | 20 | Hdirs := Projections 21 | Hpack += $(foreach dir, $(Hdirs), $(AMREX_HYDRO_HOME)/$(dir)/Make.package) 22 | Hlocs += $(foreach dir, $(Hdirs), $(AMREX_HYDRO_HOME)/$(dir)) 23 | 24 | include $(Hpack) 25 | 26 | VPATH_LOCATIONS += $(Hlocs) 27 | INCLUDE_LOCATIONS += $(Hlocs) 28 | 29 | all: $(executable) 30 | @echo SUCCESS 31 | 32 | include $(AMREX_HOME)/Tools/GNUMake/Make.rules 33 | 34 | -------------------------------------------------------------------------------- /AMReX_Amr102/Exec/inputs: -------------------------------------------------------------------------------- 1 | max_step = 200 2 | stop_time = 2.0 3 | 4 | n_cell = 64 # number of cells in x- and y-directions; z-dir has 1/8 n_cell 5 | 6 | max_grid_size = 32 # the maximum number of cells in any direction in a single grid 7 | 8 | plot_int = 5 # frequency of writing plotfiles 9 | 10 | ##################################################################### 11 | # Control the number of particles and the particle/mesh interpolation 12 | ##################################################################### 13 | 14 | n_ppc = 100 # number of particles per cell for representing the fluid 15 | 16 | pic_interpolation = 1 # Particle In Cell interpolation scheme: 17 | # 0 = nearest grid point 18 | # 1 = cloud in cell 19 | 20 | write_initial_phi = 0 21 | 22 | ################################################### 23 | # Control the verbosity and tolerance of the solver 24 | ################################################### 25 | 26 | mac_proj.verbose = 0 27 | mac_proj.bottom_verbose = 0 28 | 29 | use_hypre = 0 # use hypre instead of native GMG to solve the problem 30 | 31 | ############################################### 32 | # Specify the location and size of the cylinder 33 | ############################################### 34 | 35 | cylinder.direction = 2 # cylinder axis aligns with z-axis 36 | cylinder.radius = 0.1 # cylinder axis aligns with z-axis 37 | cylinder.center = 0.7 0.5 0.5 38 | 39 | cylinder.internal_flow = false # we are computing flow around the cylinder, not inside it 40 | 41 | write_eb_geom = 1 42 | 43 | ###### 44 | # Misc 45 | ###### 46 | amrex.fpe_trap_invalid = 1 # generate a backtrace if any floating point errors are encountered 47 | -------------------------------------------------------------------------------- /AMReX_Amr102/Source/DefineVelocity.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | using namespace amrex; 5 | 6 | void 7 | define_velocity (const Real time, const Geometry& geom, Array& vel_out, const MultiFab& phi) 8 | { 9 | const auto dx = geom.CellSizeArray(); 10 | const auto prob_lo = geom.ProbLoArray(); 11 | const Box& domain = geom.Domain(); 12 | 13 | const auto domlo = amrex::lbound(domain); 14 | const auto domhi = amrex::ubound(domain); 15 | 16 | #ifdef _OPENMP 17 | #pragma omp parallel if (Gpu::notInLaunchRegion()) 18 | #endif 19 | { 20 | for (MFIter mfi(phi,TilingIfNotGPU()); mfi.isValid(); ++mfi) 21 | { 22 | GpuArray nbx; 23 | AMREX_D_TERM(nbx[0] = mfi.tilebox(IntVect{AMREX_D_DECL(1,0,0)});, // x-face-based tilebox 24 | nbx[1] = mfi.tilebox(IntVect{AMREX_D_DECL(0,1,0)});, // y-face-based tilebox 25 | nbx[2] = mfi.tilebox(IntVect{AMREX_D_DECL(0,0,1)});); // z-face-based tilebox 26 | 27 | AMREX_D_TERM(const Box& ngbxx = amrex::grow(mfi.nodaltilebox(0),1);, 28 | const Box& ngbxy = amrex::grow(mfi.nodaltilebox(1),1);, 29 | const Box& ngbxz = amrex::grow(mfi.nodaltilebox(2),1);); 30 | 31 | GpuArray, AMREX_SPACEDIM> vel{ AMREX_D_DECL( vel_out[0].array(mfi), 32 | vel_out[1].array(mfi), 33 | vel_out[2].array(mfi)) }; 34 | 35 | const Box& psibox = Box(IntVect(AMREX_D_DECL(std::min(ngbxx.smallEnd(0)-1, ngbxy.smallEnd(0)-1), 36 | std::min(ngbxx.smallEnd(1)-1, ngbxy.smallEnd(0)-1), 37 | 0)), 38 | IntVect(AMREX_D_DECL(std::max(ngbxx.bigEnd(0), ngbxy.bigEnd(0)+1), 39 | std::max(ngbxx.bigEnd(1)+1, ngbxy.bigEnd(1)), 40 | 0))); 41 | 42 | FArrayBox psifab(psibox, 1); 43 | Elixir psieli = psifab.elixir(); 44 | Array4 psi = psifab.array(); 45 | GeometryData geomdata = geom.data(); 46 | 47 | amrex::launch(psibox, 48 | [=] AMREX_GPU_DEVICE (const Box& tbx) 49 | { 50 | get_face_velocity_psi(tbx, time, psi, geomdata); 51 | }); 52 | 53 | AMREX_D_TERM( 54 | amrex::ParallelFor(ngbxx, 55 | [=] AMREX_GPU_DEVICE (int i, int j, int k) 56 | { 57 | get_face_velocity_x(i, j, k, vel[0], psi, prob_lo, dx); 58 | if (i == domlo.x or i == domhi.x+1) vel[0](i,j,k) = 0.; 59 | });, 60 | 61 | amrex::ParallelFor(ngbxy, 62 | [=] AMREX_GPU_DEVICE (int i, int j, int k) 63 | { 64 | get_face_velocity_y(i, j, k, vel[1], psi, prob_lo, dx); 65 | if (j == domlo.y or j == domhi.y+1) vel[1](i,j,k) = 0.; 66 | });, 67 | 68 | amrex::ParallelFor(ngbxz, 69 | [=] AMREX_GPU_DEVICE (int i, int j, int k) 70 | { 71 | get_face_velocity_z(i, j, k, vel[2], psi, prob_lo, dx); 72 | if (k == domlo.z or k == domhi.z+1) vel[2](i,j,k) = 0.; 73 | }); 74 | ); 75 | } 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /AMReX_Amr102/Source/EB_Cylinder.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include 6 | 7 | using namespace amrex; 8 | 9 | /******************************************************************************** 10 | * * 11 | * Function to create a simple cylinder EB. * 12 | * * 13 | ********************************************************************************/ 14 | void 15 | make_eb_cylinder(const Geometry& geom) 16 | { 17 | // Initialise cylinder parameters 18 | bool inside = true; 19 | Real radius = 0.0002; 20 | int direction = 0; 21 | Vector centervec(3); 22 | 23 | // Get cylinder information from inputs file. * 24 | ParmParse pp("cylinder"); 25 | 26 | pp.query("internal_flow", inside); 27 | pp.query("radius", radius); 28 | pp.query("direction", direction); 29 | pp.getarr("center", centervec, 0, 3); 30 | Array center = {AMREX_D_DECL(centervec[0], centervec[1], centervec[2])}; 31 | 32 | // Print info about cylinder 33 | amrex::Print() << " " << std::endl; 34 | amrex::Print() << " Internal Flow: " << inside << std::endl; 35 | amrex::Print() << " Radius: " << radius << std::endl; 36 | amrex::Print() << " Direction: " << direction << std::endl; 37 | amrex::Print() << " Center: " << center[0] << ", " << center[1] 38 | #if (AMREX_SPACEDIM == 3) 39 | << ", " << center[2] 40 | #endif 41 | << std::endl; 42 | 43 | // Build the Cylinder implicit function representing the curved walls 44 | #if (AMREX_SPACEDIM == 2) 45 | // In 2D the sphere becomes a circle 46 | EB2::SphereIF my_cyl(radius, center, inside); 47 | #else 48 | EB2::CylinderIF my_cyl(radius, direction, center, inside); 49 | #endif 50 | 51 | // Generate GeometryShop 52 | auto gshop = EB2::makeShop(my_cyl); 53 | 54 | // Build index space 55 | int max_level_here = 0; 56 | int max_coarsening_level = 100; 57 | EB2::Build(gshop, geom, max_level_here, max_level_here + max_coarsening_level); 58 | } 59 | -------------------------------------------------------------------------------- /AMReX_Amr102/Source/FluidParticleContainer.H: -------------------------------------------------------------------------------- 1 | #ifndef BL_FLUIDPARTICLES_H_ 2 | #define BL_FLUIDPARTICLES_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | namespace Interpolation { 10 | enum {NGP=0, CIC}; 11 | } 12 | 13 | namespace PIdx { 14 | enum {AMREX_D_DECL(BufferX=0, BufferY, BufferZ), Weight, NStructReal}; 15 | } 16 | 17 | namespace amrex { 18 | 19 | class FluidParticleContainer 20 | : public ParticleContainer 21 | { 22 | private: 23 | int m_number_particles_per_cell; 24 | 25 | public: 26 | 27 | FluidParticleContainer (ParGDBBase* gdb) 28 | : ParticleContainer(gdb), 29 | m_number_particles_per_cell(0) 30 | {} 31 | 32 | FluidParticleContainer (const Geometry & geom, 33 | const DistributionMapping & dmap, 34 | const BoxArray & ba) 35 | : ParticleContainer(geom,dmap,ba), 36 | m_number_particles_per_cell(0) 37 | {} 38 | 39 | ~FluidParticleContainer () {} 40 | 41 | void InitParticles(const MultiFab& phi, const MultiFab& ebvol, Real density_cutoff, int nppc, int interpolation=Interpolation::CIC); 42 | 43 | int NumParticlesPerCell() { return m_number_particles_per_cell; } 44 | 45 | Real SumPhi(); 46 | 47 | void AdvectWithUmac (MultiFab* umac, int lev, Real dt); 48 | 49 | void DepositToMesh (MultiFab& phi, int interpolation=Interpolation::CIC); 50 | 51 | void InterpolateFromMesh (const MultiFab& phi, int interpolation=Interpolation::CIC); 52 | 53 | void RemoveCoveredParticles (const MultiFab& ebvol, Real density_cutoff); 54 | }; 55 | 56 | } 57 | 58 | #endif 59 | 60 | -------------------------------------------------------------------------------- /AMReX_Amr102/Source/Indexing.H: -------------------------------------------------------------------------------- 1 | #ifndef _INDEXING_H_ 2 | #define _INDEXING_H_ 3 | 4 | // Indexes into the grid data: velocity, processor id, and density phi 5 | namespace Idx { 6 | enum GridIndexes {AMREX_D_DECL(xvel=0, yvel, zvel), proc, phi}; 7 | } 8 | 9 | #endif 10 | -------------------------------------------------------------------------------- /AMReX_Amr102/Source/Make.package: -------------------------------------------------------------------------------- 1 | CEXE_sources += main.cpp 2 | CEXE_sources += DefineVelocity.cpp 3 | CEXE_sources += EB_Cylinder.cpp 4 | CEXE_sources += FluidParticleContainer.cpp 5 | CEXE_sources += mac_project_velocity.cpp 6 | 7 | CEXE_headers += face_velocity.H 8 | CEXE_headers += FluidParticleContainer.H 9 | -------------------------------------------------------------------------------- /AMReX_Amr102/Source/face_velocity.H: -------------------------------------------------------------------------------- 1 | #ifndef FACE_VELOCITY_H_ 2 | #define FACE_VELOCITY_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | using namespace amrex; 9 | 10 | AMREX_GPU_DEVICE 11 | AMREX_FORCE_INLINE 12 | void get_face_velocity_psi(Box const& bx, 13 | const Real time, 14 | Array4 const& psi, 15 | GeometryData const& geomdata) 16 | { 17 | const auto lo = lbound(bx); 18 | const auto hi = ubound(bx); 19 | 20 | const Real* AMREX_RESTRICT prob_lo = geomdata.ProbLo(); 21 | const Real* AMREX_RESTRICT dx = geomdata.CellSize(); 22 | 23 | for (int j = lo.y; j <= hi.y; ++j) { 24 | Real y = dx[1]*(0.5+j) + prob_lo[1]; 25 | AMREX_PRAGMA_SIMD 26 | for (int i = lo.x; i <= hi.x; ++i) { 27 | Real x = dx[0]*(0.5+i) + prob_lo[0]; 28 | psi(i,j,0) = pow(sin(M_PI*x), 2) * pow(sin(M_PI*y), 2) 29 | * cos(M_PI*time/2.0) * 1.0/M_PI; 30 | } 31 | } 32 | } 33 | 34 | AMREX_GPU_DEVICE 35 | AMREX_FORCE_INLINE 36 | void get_face_velocity_x(int i, int j, int k, 37 | Array4 const& vx, 38 | Array4 const& psi, 39 | GpuArray prob_lo, 40 | GpuArray dx) 41 | { 42 | vx(i,j,k) = -( (psi(i,j+1,0)+psi(i-1,j+1,0)) - (psi(i,j-1,0)+psi(i-1,j-1,0)) ) * (0.25/dx[1]); 43 | } 44 | 45 | AMREX_GPU_DEVICE 46 | AMREX_FORCE_INLINE 47 | void get_face_velocity_y(int i, int j, int k, 48 | Array4 const& vy, 49 | Array4 const& psi, 50 | GpuArray prob_lo, 51 | GpuArray dx) 52 | { 53 | vy(i,j,k) = ( (psi(i+1,j,0)+psi(i+1,j-1,0)) - (psi(i-1,j,0)+psi(i-1,j-1,0)) ) * (0.25/dx[0]); 54 | } 55 | 56 | AMREX_GPU_DEVICE 57 | AMREX_FORCE_INLINE 58 | void get_face_velocity_z(int i, int j, int k, 59 | Array4 const& vz, 60 | Array4 const& psi, 61 | GpuArray prob_lo, 62 | GpuArray dx) 63 | { 64 | vz(i,j,k) = 0.0; 65 | } 66 | 67 | #endif 68 | -------------------------------------------------------------------------------- /AMReX_Amr102/Source/mac_project_velocity.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | void mac_project_velocity(amrex::Array& vel, const amrex::Geometry& geom, int use_hypre) 4 | { 5 | using namespace amrex; 6 | using namespace Hydro; 7 | 8 | LPInfo lp_info; 9 | 10 | // If we want to use hypre to solve the full problem we need to not coarsen inside AMReX 11 | if (use_hypre) 12 | lp_info.setMaxCoarseningLevel(0); 13 | 14 | Array beta; 15 | 16 | for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { 17 | beta[idim].define(vel[idim].boxArray(), vel[idim].DistributionMap(), 1, 0, MFInfo(), vel[idim].Factory()); 18 | beta[idim].setVal(1.0); 19 | } 20 | 21 | MacProjector macproj({amrex::GetArrOfPtrs(vel)}, // mac velocity 22 | MLMG::Location::FaceCenter, // velocity located on face centers 23 | {amrex::GetArrOfConstPtrs(beta)}, // beta 24 | MLMG::Location::FaceCenter, // beta located on face centers 25 | MLMG::Location::CellCenter, // location of mac_phi 26 | {geom}, 27 | lp_info); // structure for passing info to the operator 28 | 29 | // Set bottom-solver to use hypre instead of native BiCGStab 30 | if (use_hypre) 31 | macproj.getMLMG().setBottomSolver(MLMG::BottomSolver::hypre); 32 | 33 | macproj.setDomainBC({AMREX_D_DECL(LinOpBCType::Neumann, 34 | LinOpBCType::Neumann, 35 | LinOpBCType::Periodic)}, 36 | {AMREX_D_DECL(LinOpBCType::Neumann, 37 | LinOpBCType::Neumann, 38 | LinOpBCType::Periodic)}); 39 | 40 | Real reltol = 1.e-8; 41 | Real abstol = 1.e-12; 42 | 43 | macproj.project(reltol, abstol); 44 | } 45 | -------------------------------------------------------------------------------- /AMReX_EB_MacProj/GNUmakefile: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------------------------ 2 | # Makefile to build examples 3 | # ------------------------------------------------------------------------------ 4 | # Note the following environment variables need to be set to in order to build: 5 | # AMREX_INSTALL_DIR = path to AMReX installation 6 | # MPICXX = mpicxx wrapper 7 | # 8 | # If any are unset, they assume the default values for compilation on Cooley 9 | # ------------------------------------------------------------------------------ 10 | 11 | DIM = 3 12 | 13 | # set default values for any unset environment variables 14 | ifeq ($(AMREX_INSTALL_DIR),) 15 | # AMREX_INSTALL_DIR = /projects/ATPESC2019/FASTMath/spack/opt/spack/linux-rhel7-x86_64/gcc-4.8.5/amrex-develop-fvkknjb363aqxpt6zu2tt6oojixnlnex 16 | AMREX_INSTALL_DIR = /projects/ATPESC2019/MathPackagesTraining/amrex/libamrex3D 17 | endif 18 | 19 | # These will only apply if they are not set as environment variables 20 | # These are the paths to the AMReX and HYPRE libraries on Cooley for ATPESC 21 | AMREX_INSTALL_DIR ?= /projects/ATPESC2019/MathPackagesTraining/amrex/libamrex3D 22 | HYPRE_DIR ?= /projects/ATPESC2019/MathPackagesTraining/spack/opt/spack/linux-rhel7-x86_64/gcc-4.8.5/hypre-develop-bjrvkgdwbwewvxhrv6wu4g6tgejcfnm4/ 23 | 24 | # Path to ParaView 5.6.1 MESA on Cooley 25 | PARAVIEW_DIR ?= /projects/ATPESC2019/MathPackagesTraining/ParaView-5.6.1-osmesa-MPI-Linux-64bit 26 | 27 | ifeq ($(MPICXX),) 28 | MPICXX = mpicxx 29 | endif 30 | 31 | FC = gfortran 32 | 33 | CPPFLAGS = -Ishared -I$(AMREX_INSTALL_DIR)/include -I. -I$(HYPRE_DIR)/include 34 | CPPFLAGS = -I$(AMREX_INSTALL_DIR)/include -I. -I$(HYPRE_DIR)/include 35 | CXXFLAGS = -O2 -std=c++14 36 | FFLAGS = -O2 37 | LDFLAGS = -L$(AMREX_INSTALL_DIR)/lib -L$(HYPRE_DIR)/lib -Wl,-rpath,$(AMREX_INSTALL_DIR)/lib -Wl,-rpath,$(HYPRE_DIR)/lib 38 | 39 | LIBRARIES = -lamrex -lHYPRE 40 | LIBRARIES += -lgfortran 41 | LIBRARIES += -lpthread 42 | 43 | 44 | default: main$(DIM)d.exe 45 | 46 | main$(DIM)d.exe: main.o MyParticleContainer.o 47 | $(MPICXX) -o $@ $(CXXFLAGS) $^ $(LDFLAGS) $(LIBRARIES) 48 | 49 | main.o: main.cpp MyParticleContainer.H 50 | $(MPICXX) -o $@ -c $(CXXFLAGS) $(CPPFLAGS) $< 51 | 52 | MyParticleContainer.o: MyParticleContainer.cpp MyParticleContainer.H 53 | $(MPICXX) -o $@ -c $(CXXFLAGS) $(CPPFLAGS) $< 54 | 55 | .PHONY: clean realclean pltclean 56 | 57 | clean: 58 | $(RM) *.o 59 | 60 | realclean: clean 61 | $(RM) *~ *.exe 62 | 63 | pltclean: 64 | $(RM) -rf plt*/ *.png eb*vtp 65 | 66 | movie: 67 | $(PARAVIEW_DIR)/bin/pvpython paraview_vis_script.py 68 | ffmpeg -r 20 -i off_to_the_races.%04d.jpeg -s 1400x800 -vcodec libx264 -crf 30 -pix_fmt yuv420p off_to_the_races.mp4 69 | -------------------------------------------------------------------------------- /AMReX_EB_MacProj/GNUmakefile_from_source: -------------------------------------------------------------------------------- 1 | USE_MPI = TRUE 2 | USE_OMP = FALSE 3 | 4 | COMP = gnu 5 | 6 | DIM = 3 7 | 8 | DEBUG = FALSE 9 | 10 | AMREX_HOME ?= ../../.. 11 | 12 | USE_EB = TRUE 13 | 14 | USE_HYPRE = TRUE 15 | USE_HYPRE = FALSE 16 | 17 | include $(AMREX_HOME)/Tools/GNUMake/Make.defs 18 | 19 | include ./Make.package 20 | 21 | Pdirs := AmrCore 22 | Pdirs += Base 23 | Pdirs += Boundary 24 | Pdirs += EB 25 | Pdirs += LinearSolvers/Projections 26 | Pdirs += LinearSolvers/MLMG 27 | Pdirs += Particle 28 | 29 | Ppack += $(foreach dir, $(Pdirs), $(AMREX_HOME)/Src/$(dir)/Make.package) 30 | 31 | include $(Ppack) 32 | 33 | include $(AMREX_HOME)/Tools/GNUMake/Make.rules 34 | 35 | movie: 36 | ffmpeg -framerate 10 -pattern_type glob -i "p*.png" -r 5 -vf scale=1024:-1 pachinko.gif 37 | -------------------------------------------------------------------------------- /AMReX_EB_MacProj/Make.package: -------------------------------------------------------------------------------- 1 | CEXE_sources += main.cpp 2 | CEXE_sources += MyParticleContainer.cpp 3 | CEXE_headers += MyParticleContainer.H 4 | -------------------------------------------------------------------------------- /AMReX_EB_MacProj/MyParticleContainer.H: -------------------------------------------------------------------------------- 1 | #ifndef BL_MYPARTICLES_H_ 2 | #define BL_MYPARTICLES_H_ 3 | 4 | #include 5 | 6 | namespace amrex { 7 | 8 | class MyParticleContainer 9 | : public TracerParticleContainer 10 | { 11 | public: 12 | 13 | MyParticleContainer (ParGDBBase* gdb) 14 | : TracerParticleContainer(gdb) 15 | {} 16 | 17 | MyParticleContainer (const Geometry & geom, 18 | const DistributionMapping & dmap, 19 | const BoxArray & ba) 20 | : TracerParticleContainer(geom,dmap,ba) 21 | {} 22 | 23 | ~MyParticleContainer () {} 24 | 25 | Real FindWinner (int n); 26 | 27 | void InitParticles (std::string initial_particle_file, Real zlen); 28 | 29 | }; 30 | 31 | } 32 | 33 | #endif 34 | 35 | -------------------------------------------------------------------------------- /AMReX_EB_MacProj/MyParticleContainer.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | 4 | namespace amrex { 5 | 6 | void 7 | MyParticleContainer::InitParticles (std::string initial_particle_file, Real zlen) 8 | { 9 | BL_PROFILE("MyParticleContainer::InitParticles()"); 10 | 11 | InitFromAsciiFile(initial_particle_file,0); 12 | 13 | int lev = 0; 14 | auto& pmap = GetParticles(lev); 15 | for (auto& kv : pmap) { 16 | int grid = kv.first.first; 17 | auto& pbox = kv.second.GetArrayOfStructs(); 18 | const int n = pbox.size(); 19 | 20 | for (int i = 0; i < n; i++) 21 | { 22 | ParticleType& p = pbox[i]; 23 | 24 | // We over-write the z-locations to make sure they're in the domain 25 | p.pos(2) = 0.5 * zlen; 26 | } 27 | } 28 | } 29 | 30 | Real 31 | MyParticleContainer::FindWinner (int n) 32 | { 33 | BL_PROFILE("MyParticleContainer::FindWinner()"); 34 | 35 | using ParticleType = MyParticleContainer::ParticleType; 36 | int nghost = 0; 37 | Real x = amrex::ReduceMax(*this, nghost, 38 | [=] AMREX_GPU_HOST_DEVICE (const ParticleType& p) noexcept -> Real 39 | { return p.pos(n); }); 40 | 41 | ParallelDescriptor::ReduceRealMax(x); 42 | return x; 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /AMReX_EB_MacProj/README: -------------------------------------------------------------------------------- 1 | "Off to the Races" 2 | 3 | This tutorial demonstrates the solution of a Poisson equation 4 | to compute a potential flow field obstacles, then the use of this 5 | flow field to advect particles from left to right around the obstacles. 6 | 7 | **************************************************************************************************** 8 | 9 | To run it in serial, 10 | 11 | ./main3d.ex inputs 12 | 13 | To run it in parallel, for example on 4 ranks: 14 | 15 | mpirun -n 4 ./main3d.ex inputs 16 | 17 | The following parameters can be set at run-time -- these are currently set in the inputs 18 | file but you can also set them on the command line. 19 | 20 | 21 | n_cell = 128 # number of cells in x-direction; we double this in the y-direction 22 | max_grid_size = 64 # the maximum number of cells in any direction in a single grid 23 | 24 | plot_int = 10 # frequency of writing plotfiles 25 | 26 | particle_file = initial_particles_3d # name of file where we specify the input positions of the particles 27 | 28 | time_step = 0.001 # we advance the particles with a fixed time step of this size 29 | 30 | max_time = 10.0 # the final time (if max_time < max_steps * time_step) 31 | 32 | max_steps = 10000 # the maximum number of steps (if max_steps * time_step < max_time)) 33 | 34 | obstacles = 0 1 2 3 4 5 6 7 8 # this is how we choose which obstacles to include 35 | 36 | **************************************************************************************************** 37 | 38 | The output from your run should look something like this: 39 | 40 | 41 | ******************************************************************** 42 | You specified 9 objects in the domain: 0 1 2 3 4 5 6 7 8 43 | ******************************************************************** 44 | 45 | ******************************************************************** 46 | First let's project the initial velocity to find 47 | the flow field around the obstacles ... 48 | ******************************************************************** 49 | 50 | 51 | ******************************************************************** 52 | Done! Now let's advect the particles ... 53 | ******************************************************************** 54 | 55 | Timestep 0, Time = 0.001 and leading particle now at 0.101179325 56 | Timestep 100, Time = 0.101 and leading particle now at 0.2444506795 57 | Timestep 200, Time = 0.201 and leading particle now at 0.4330191808 58 | Timestep 300, Time = 0.301 and leading particle now at 0.5611955983 59 | Timestep 400, Time = 0.401 and leading particle now at 0.7422046938 60 | Timestep 500, Time = 0.501 and leading particle now at 0.8955689091 61 | Timestep 600, Time = 0.601 and leading particle now at 1.044585496 62 | Timestep 700, Time = 0.701 and leading particle now at 1.225885881 63 | Timestep 800, Time = 0.801 and leading particle now at 1.34851225 64 | Timestep 900, Time = 0.901 and leading particle now at 1.45538891 65 | Timestep 1000, Time = 1.001 and leading particle now at 1.558181566 66 | Timestep 1100, Time = 1.101 and leading particle now at 1.659474158 67 | Timestep 1200, Time = 1.201 and leading particle now at 1.760129699 68 | Timestep 1300, Time = 1.301 and leading particle now at 1.860489498 69 | Timestep 1400, Time = 1.401 and leading particle now at 1.960718531 70 | 71 | ******************************************************************** 72 | We have a winner...and the winning time is 1.431 73 | ******************************************************************** 74 | -------------------------------------------------------------------------------- /AMReX_EB_MacProj/initial_particles_3d: -------------------------------------------------------------------------------- 1 | 50 2 | 0.100000 0.023090 0.500000 3 | 0.100000 0.043090 0.500000 4 | 0.100000 0.063090 0.500000 5 | 0.100000 0.083090 0.500000 6 | 0.100000 0.103090 0.500000 7 | 0.100000 0.123090 0.500000 8 | 0.100000 0.143090 0.500000 9 | 0.100000 0.163090 0.500000 10 | 0.100000 0.183090 0.500000 11 | 0.100000 0.203090 0.500000 12 | 0.100000 0.223090 0.500000 13 | 0.100000 0.243090 0.500000 14 | 0.100000 0.263090 0.500000 15 | 0.100000 0.283090 0.500000 16 | 0.100000 0.303090 0.500000 17 | 0.100000 0.323090 0.500000 18 | 0.100000 0.343090 0.500000 19 | 0.100000 0.363090 0.500000 20 | 0.100000 0.383090 0.500000 21 | 0.100000 0.403090 0.500000 22 | 0.100000 0.423090 0.500000 23 | 0.100000 0.443090 0.500000 24 | 0.100000 0.463090 0.500000 25 | 0.100000 0.483090 0.500000 26 | 0.100000 0.503090 0.500000 27 | 0.100000 0.523090 0.500000 28 | 0.100000 0.543090 0.500000 29 | 0.100000 0.563090 0.500000 30 | 0.100000 0.583090 0.500000 31 | 0.100000 0.603090 0.500000 32 | 0.100000 0.623090 0.500000 33 | 0.100000 0.643090 0.500000 34 | 0.100000 0.663090 0.500000 35 | 0.100000 0.683090 0.500000 36 | 0.100000 0.703090 0.500000 37 | 0.100000 0.723090 0.500000 38 | 0.100000 0.743090 0.500000 39 | 0.100000 0.763090 0.500000 40 | 0.100000 0.783090 0.500000 41 | 0.100000 0.803090 0.500000 42 | 0.100000 0.823090 0.500000 43 | 0.100000 0.843090 0.500000 44 | 0.100000 0.863090 0.500000 45 | 0.100000 0.883090 0.500000 46 | 0.100000 0.903090 0.500000 47 | 0.100000 0.923090 0.500000 48 | 0.100000 0.943090 0.500000 49 | 0.100000 0.963090 0.500000 50 | 0.100000 0.983090 0.500000 51 | 0.100000 1.003090 0.500000 52 | -------------------------------------------------------------------------------- /AMReX_EB_MacProj/inputs_3d: -------------------------------------------------------------------------------- 1 | n_cell = 128 # number of cells in x- and z-directions; we double this in the y-direction 2 | max_grid_size = 64 # the maximum number of cells in any direction in a single grid 3 | 4 | plot_int = 50 # frequency of writing plotfiles 5 | 6 | particle_file = initial_particles_3d # name of file where we specify the initial positions of the particles 7 | 8 | time_step = 0.001 # we advance the particles with a fixed time step of this size 9 | 10 | max_time = 10.0 # the final time (if max_time < max_steps * time_step) 11 | 12 | max_steps = 10000 # the maximum number of steps (if max_steps * time_step < max_time)) 13 | 14 | obstacles = 0 1 2 3 4 5 6 7 8 # this is how we choose which obstacles to include 15 | 16 | mg_verbose = 2 17 | cg_verbose = 2 18 | 19 | use_hypre = 1 # use hypre instead of native GMG to solve the problem 20 | -------------------------------------------------------------------------------- /AMReX_EB_MacProj/macproj.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AMReX-Codes/ATPESC-codes/2f5da61d159822df3c74f75035367feadfa399e3/AMReX_EB_MacProj/macproj.gif -------------------------------------------------------------------------------- /AMReX_EB_MacProj/makeparticlefile_3d.py: -------------------------------------------------------------------------------- 1 | #This will create a particle input file based on the number 2 | #of particles in each direction (NX,NY,NZ) and the physical domain 3 | #boundaries (dimx,dimy,dimz). Amrex can then read this file in with 4 | #the function TracerParticleContainer::InitFromAsciiFile() 5 | 6 | import numpy as np 7 | 8 | NX = 1 9 | NY = 50 10 | 11 | dimx = [0,0.1] #They'll start evenly close to the edge/source and flow through 12 | dimy = [0,1] 13 | 14 | dx = (dimx[1] - dimx[0]) / NX 15 | dy = (dimy[1] - dimy[0]) / NY 16 | 17 | f = open("initial_particles_3d", "w") 18 | 19 | z_loc = 0.5 20 | 21 | for i in range(NX): 22 | for j in range(NY): 23 | if (i==0 and j==0): 24 | f.write("%d \r\n" % (NX*NY)) 25 | f.write("%f %f %f \r\n" % ((i+1)*dx, (j+1)*dy+0.00309, z_loc)) 26 | else: 27 | f.write("%f %f %f \r\n" % ((i+1)*dx, (j+1)*dy+0.00309, z_loc)) 28 | f.close() 29 | 30 | -------------------------------------------------------------------------------- /AMReX_EB_Pachinko/GNUmakefile: -------------------------------------------------------------------------------- 1 | 2 | USE_MPI = TRUE 3 | USE_OMP = FALSE 4 | 5 | COMP = gnu 6 | 7 | DIM = 3 8 | 9 | DEBUG = FALSE 10 | 11 | AMREX_HOME ?= ../../amrex 12 | 13 | USE_EB = TRUE 14 | USE_PARTICLES=TRUE 15 | 16 | include $(AMREX_HOME)/Tools/GNUMake/Make.defs 17 | 18 | include ./Make.package 19 | 20 | Pdirs := AmrCore 21 | Pdirs += Base 22 | Pdirs += Boundary 23 | Pdirs += EB 24 | Pdirs += Particle 25 | 26 | Ppack += $(foreach dir, $(Pdirs), $(AMREX_HOME)/Src/$(dir)/Make.package) 27 | 28 | include $(Ppack) 29 | 30 | include $(AMREX_HOME)/Tools/GNUMake/Make.rules 31 | -------------------------------------------------------------------------------- /AMReX_EB_Pachinko/GNUmakefile_movie: -------------------------------------------------------------------------------- 1 | PARAVIEW_PATH ?= /path/containing/paraview/executable 2 | 3 | movie: 4 | @echo "Making a movie from 3D simulation, this will probably take <30 seconds ..." 5 | @$(PARAVIEW_PATH)/pvpython paraview_pachinko.py > /dev/null 2>&1 6 | @echo "Done! Generated pachinko.avi and pachinko.gif" 7 | -------------------------------------------------------------------------------- /AMReX_EB_Pachinko/Make.package: -------------------------------------------------------------------------------- 1 | CEXE_sources += main.cpp 2 | CEXE_sources += MyParticleContainer.cpp 3 | CEXE_headers += MyParticleContainer.H 4 | -------------------------------------------------------------------------------- /AMReX_EB_Pachinko/MyParticleContainer.H: -------------------------------------------------------------------------------- 1 | #ifndef BL_MYPARTICLES_H_ 2 | #define BL_MYPARTICLES_H_ 3 | 4 | #include 5 | 6 | namespace amrex { 7 | 8 | struct PIdx 9 | { 10 | enum { 11 | vx = 0, vy, 12 | #if (AMREX_SPACEDIM == 3) 13 | vz, 14 | #endif 15 | ncomps 16 | }; 17 | }; 18 | 19 | class MyParticleContainer 20 | : public amrex::ParticleContainer 21 | { 22 | 23 | public: 24 | 25 | MyParticleContainer (const amrex::Geometry & a_geom, 26 | const amrex::DistributionMapping & a_dmap, 27 | const amrex::BoxArray & a_ba) 28 | : ParticleContainer(a_geom, a_dmap, a_ba) 29 | {} 30 | 31 | ~MyParticleContainer () {}; 32 | 33 | void InitPachinko (std::string initial_particle_file, Real zlen); 34 | 35 | void AdvectPachinko (Real dt, amrex::Gpu::DeviceVector& obstacle_center, 36 | Real obstacle_radius, Real particle_radius); 37 | }; 38 | 39 | } 40 | 41 | #endif 42 | -------------------------------------------------------------------------------- /AMReX_EB_Pachinko/MyParticleContainer.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | 4 | namespace amrex { 5 | 6 | void 7 | MyParticleContainer::InitPachinko (std::string initial_particle_file, Real zlen) 8 | { 9 | BL_PROFILE("MyParticleContainer::InitPachinko()"); 10 | 11 | InitFromAsciiFile(initial_particle_file,0); 12 | 13 | int lev = 0; 14 | auto& pmap = GetParticles(lev); 15 | for (auto& kv : pmap) { 16 | auto& pbox = kv.second.GetArrayOfStructs(); 17 | auto pstruct = &pbox[0]; 18 | const int n = pbox.size(); 19 | 20 | amrex::ParallelFor( n, 21 | [=] AMREX_GPU_DEVICE (int i) noexcept 22 | { 23 | ParticleType& p = pstruct[i]; 24 | 25 | p.rdata(PIdx::vx) = 0.0; 26 | p.rdata(PIdx::vy) = -1.0; 27 | #if (AMREX_SPACEDIM == 3) 28 | p.rdata(PIdx::vz) = 0.0; 29 | #endif 30 | 31 | // We over-write the z-locations to make sure they're in the domain 32 | p.pos(2) = 0.5 * zlen; 33 | }); 34 | } 35 | } 36 | 37 | void 38 | MyParticleContainer::AdvectPachinko (Real dt, amrex::Gpu::DeviceVector& obstacle_center, 39 | Real obstacle_radius, Real particle_radius) 40 | { 41 | BL_PROFILE("MyParticleContainer::AdvectPachinko()"); 42 | 43 | int lev = 0; 44 | 45 | // Particle radius 46 | Real prad = particle_radius; 47 | 48 | // Obstracle radiu 49 | Real crad = obstacle_radius; 50 | 51 | Real rad_squared = (prad+crad)*(prad+crad); 52 | 53 | Real grav = -40.; 54 | 55 | Real restitution_coeff = 0.9; 56 | 57 | const auto prob_lo = Geom(0).ProbLoArray(); 58 | const auto prob_hi = Geom(0).ProbHiArray(); 59 | 60 | int num_obstacles = obstacle_center.size(); 61 | auto obstacle_center_data = obstacle_center.data(); 62 | 63 | auto& pmap = GetParticles(lev); 64 | for (auto& kv : pmap) { 65 | auto& pbox = kv.second.GetArrayOfStructs(); 66 | auto pstruct = &pbox[0]; 67 | const int n = pbox.size(); 68 | 69 | // std::cout << "Number of particles: " << n << std::endl; 70 | 71 | // NOTE: we assume that all particle motion occurs in a plane! 72 | // Even if we run in 3-d we assume no motion in the z-direction 73 | 74 | amrex::ParallelFor( n, 75 | [=] AMREX_GPU_DEVICE (int i) noexcept 76 | { 77 | 78 | ParticleType& p = pstruct[i]; 79 | 80 | // If particle in hitting the lower wall 81 | if (p.pos(1) < prob_lo[1] + particle_radius) 82 | { 83 | // Reflect position 84 | p.pos(0) += dt * p.rdata(PIdx::vx); 85 | p.pos(1) = 2.0 * (prob_lo[1] + particle_radius) - p.pos(1); 86 | 87 | // Reverse y velocity and apply restitution 88 | p.rdata(PIdx::vy) = -p.rdata(PIdx::vy) * restitution_coeff; 89 | 90 | // Apply restitution to x velocity as well 91 | p.rdata(PIdx::vx) *= restitution_coeff; 92 | } 93 | 94 | else 95 | { 96 | p.pos(0) += 0.5 * dt * p.rdata(PIdx::vx); 97 | p.pos(1) += 0.5 * dt * p.rdata(PIdx::vy); 98 | 99 | // Accleration under graivty 100 | p.rdata(PIdx::vy) += dt * grav; 101 | 102 | p.pos(0) += 0.5 * dt * p.rdata(PIdx::vx); 103 | p.pos(1) += 0.5 * dt * p.rdata(PIdx::vy); 104 | 105 | p.pos(0) += dt * p.rdata(PIdx::vx); 106 | p.pos(1) += dt * p.rdata(PIdx::vy); 107 | 108 | for (int ind = 0; ind < num_obstacles; ind++) 109 | { 110 | Real x_diff = p.pos(0) - obstacle_center_data[ind][0]; 111 | Real y_diff = p.pos(1) - obstacle_center_data[ind][1]; 112 | Real diff_sq = x_diff * x_diff + y_diff * y_diff; 113 | 114 | if ( diff_sq < rad_squared ) 115 | { 116 | Real diff = std::sqrt(diff_sq); 117 | Real overshoot = (prad+crad) - diff; 118 | 119 | Real norm_x = x_diff / diff; 120 | Real norm_y = y_diff / diff; 121 | 122 | Real tang_x = -norm_y; 123 | Real tang_y = norm_x; 124 | 125 | // Incoming velocity dot normal = (norm_x, norm_y) 126 | Real vel_norm = p.rdata(PIdx::vx) * norm_x + 127 | p.rdata(PIdx::vy) * norm_y; 128 | 129 | // Incoming velocity dot tangent = (x_tang, y_tang) = (-y_norm, x_norm) 130 | Real vel_tang = p.rdata(PIdx::vx) * norm_y + 131 | p.rdata(PIdx::vy) * norm_x; 132 | 133 | // Original velocity was (vel_norm) * (norm_x, norm_y) 134 | // + (vel_tang) * (tang_x, tang_y) 135 | 136 | // New velocity is MINUS (vel_norm) * (norm_x, norm_y) 137 | // + (vel_tang) * (tang_x, tang_y) 138 | 139 | p.rdata(PIdx::vx) = -vel_norm * norm_x + vel_tang * tang_x; 140 | p.rdata(PIdx::vy) = -vel_norm * norm_y + vel_tang * tang_y; 141 | 142 | p.rdata(PIdx::vx) *= restitution_coeff; 143 | p.rdata(PIdx::vy) *= restitution_coeff; 144 | 145 | // Reflect particle position as well 146 | Real ref_pos_x = obstacle_center_data[ind][0] + (prad+crad)*norm_x; 147 | Real ref_pos_y = obstacle_center_data[ind][1] + (prad+crad)*norm_y; 148 | 149 | p.pos(0) = ref_pos_x + overshoot * norm_x; 150 | p.pos(1) = ref_pos_y + overshoot * norm_y; 151 | } 152 | } 153 | 154 | // Bounce off left wall 155 | if (p.pos(0) < (prob_lo[0]+prad)) 156 | { 157 | p.pos(0) = 2.0*(prob_lo[0]+prad) - p.pos(0); 158 | p.rdata(PIdx::vx) = -p.rdata(PIdx::vx); 159 | p.rdata(PIdx::vx) *= restitution_coeff; 160 | p.rdata(PIdx::vy) *= restitution_coeff; 161 | } 162 | 163 | // Bounce off right wall 164 | if (p.pos(0) > (prob_hi[0]-prad)) 165 | { 166 | p.pos(0) = 2.0*(prob_hi[0]-prad) - p.pos(0); 167 | p.rdata(PIdx::vx) = -0.9*p.rdata(PIdx::vx); 168 | p.rdata(PIdx::vx) *= restitution_coeff; 169 | p.rdata(PIdx::vy) *= restitution_coeff; 170 | } 171 | } 172 | }); 173 | } 174 | } 175 | } 176 | -------------------------------------------------------------------------------- /AMReX_EB_Pachinko/README: -------------------------------------------------------------------------------- 1 | "AMReX-Pachinko" 2 | 3 | This tutorial demonstrates particles accelerating downward with 4 | gravity and bouncing off the side walls and off the solid obstacles. 5 | 6 | We can build the executable in 2-d or 3-d. In this example the problem 7 | is the same in both cases since we assume the particles never move in the z-direction. 8 | 9 | **************************************************************************************************** 10 | 11 | In this example we freeze the obstacles but can change the initial particle locations. 12 | 13 | To run it in serial, 14 | 15 | ./main3d.ex inputs_3d 16 | 17 | To run it in parallel, for example on 4 ranks: 18 | 19 | mpirun -n 4 ./main3d.ex inputs_3d 20 | 21 | The following parameters can be set at run-time -- these are currently set in the inputs_3d 22 | file but you can also set them on the command line. In this specific example we use only 4 23 | cells in the z-direction (if in 3-d) regardless of n_cell. 24 | 25 | ``` 26 | n_cell = 125 # number of cells in x-direction; we double this in the y-direction 27 | max_grid_size = 25 # the maximum number of cells in any direction in a single grid 28 | 29 | plot_int = 10 # frequency of writing plotfiles 30 | 31 | particle_file = initial_particles_3d # name of file where we specify the input positions of the particles 32 | 33 | time_step = 0.001 # we take a fixed time step of this size 34 | 35 | max_time = 3.0 # the final time (if max_time < max_steps * time_step) 36 | max_steps = 100000 # the maximum number of steps (if max_steps * time_step < max_time)) 37 | ``` 38 | 39 | For example, 40 | ``` 41 | mpirun -n 4 ./main3d.ex inputs_3d particle_file=my_file 42 | ``` 43 | 44 | will read the particles from a file called "my_file" 45 | 46 | **************************************************************************************************** 47 | 48 | The output from your run should look something like this: 49 | 50 | ``` 51 | ******************************************************************** 52 | Let's advect the particles ... 53 | We'll print a dot every 10 time steps. 54 | ******************************************************************** 55 | 56 | ............................................................................................................................................................................................................................................................................................................. 57 | 58 | ******************************************************************** 59 | We've finished moving the particles to time 3 60 | That took 1.145916707 seconds. 61 | ******************************************************************** 62 | -------------------------------------------------------------------------------- /AMReX_EB_Pachinko/initial_particles_3d: -------------------------------------------------------------------------------- 1 | 12 2 | 0.055 1.9 0.5 3 | 0.155 1.9 0.5 4 | 0.255 1.9 0.5 5 | 0.355 1.9 0.5 6 | 0.455 1.9 0.5 7 | 0.555 1.9 0.5 8 | 0.655 1.9 0.5 9 | 0.755 1.9 0.5 10 | 0.855 1.9 0.5 11 | 0.955 1.9 0.5 12 | 1.055 1.9 0.5 13 | 1.155 1.9 0.5 14 | -------------------------------------------------------------------------------- /AMReX_EB_Pachinko/inputs_3d: -------------------------------------------------------------------------------- 1 | n_cell = 125 # number of cells in x-direction; we double this in the y-direction 2 | max_grid_size = 25 # the maximum number of cells in any direction in a single grid 3 | 4 | plot_int = 15 # frequency of writing plotfiles 5 | 6 | particle_file = initial_particles_3d # name of file where we specify the initial positions of the particles 7 | 8 | time_step = 0.001 # we take a fixed time step of this size 9 | 10 | max_time = 1.75 # the final time (if max_time < max_steps * time_step) 11 | max_steps = 100000 # the maximum number of steps (if max_steps * time_step < max_time)) 12 | -------------------------------------------------------------------------------- /AMReX_EB_Pachinko/main.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #include 14 | 15 | using namespace amrex; 16 | 17 | void write_plotfile(int step_counter, const Geometry& geom, const MultiFab& plotmf, MyParticleContainer& pc, 18 | const int ascii_particle_output) 19 | { 20 | std::stringstream sstream; 21 | sstream << "plt" << std::setw(5) << std::setfill('0') << step_counter; 22 | std::string plotfile_name = sstream.str(); 23 | 24 | EB_WriteSingleLevelPlotfile(plotfile_name, plotmf, 25 | { "proc" }, geom, step_counter, 0); 26 | 27 | if (ascii_particle_output) 28 | { 29 | sstream << "_particles"; 30 | const std::string particlefile_name = sstream.str(); 31 | pc.WriteAsciiFile(particlefile_name); 32 | } 33 | 34 | pc.Checkpoint(plotfile_name, "particles", true); // Write particles to plotfile 35 | } 36 | 37 | int main (int argc, char* argv[]) 38 | { 39 | amrex::SetVerbose(0); 40 | 41 | amrex::Initialize(argc, argv); 42 | 43 | // Turn off amrex-related output 44 | 45 | { 46 | int verbose = 0; 47 | int n_cell = 128; 48 | int max_grid_size = 32; 49 | std::string particle_file = ""; 50 | Real max_time = 1.0; 51 | int max_steps = 100; 52 | int plot_int = 1; 53 | Real time_step = 0.01; 54 | int ascii_particle_output = 0; 55 | 56 | Real obstacle_radius = 0.10; 57 | Real particle_radius = 0.02; 58 | 59 | // read parameters 60 | { 61 | ParmParse pp; 62 | pp.query("verbose", verbose); 63 | pp.query("n_cell", n_cell); 64 | pp.query("max_grid_size", max_grid_size); 65 | pp.query("particle_file", particle_file); 66 | pp.query("max_time", max_time); 67 | pp.query("max_steps", max_steps); 68 | pp.query("plot_int", plot_int); 69 | pp.query("time_step", time_step); 70 | pp.query("ascii_particle_output", ascii_particle_output); 71 | 72 | pp.query("obstacle_radius", obstacle_radius); 73 | pp.query("particle_radius", particle_radius); 74 | } 75 | 76 | int n_cell_x = n_cell; 77 | int n_cell_y = n_cell * 8/5; 78 | int n_cell_z = 4; 79 | 80 | Real zlen = 0.5; 81 | 82 | Geometry geom; 83 | BoxArray grids; 84 | DistributionMapping dmap; 85 | { 86 | RealBox rb({AMREX_D_DECL(0.,0.,0.)}, {AMREX_D_DECL(1.25,2.,zlen)}); 87 | Array isp{AMREX_D_DECL(0,1,1)}; 88 | Geometry::Setup(&rb, 0, isp.data()); 89 | Box domain(IntVect{AMREX_D_DECL(0,0,0)}, 90 | IntVect{AMREX_D_DECL(n_cell_x-1,n_cell_y-1,n_cell_z-1)}); 91 | geom.define(domain); 92 | 93 | grids.define(domain); 94 | grids.maxSize(max_grid_size); 95 | 96 | dmap.define(grids); 97 | } 98 | 99 | MultiFab plotfile_mf; 100 | 101 | amrex::Vector obstacle_center = { 102 | {AMREX_D_DECL(0.30,0.3,0.5*zlen)}, 103 | {AMREX_D_DECL(0.60,0.3,0.5*zlen)}, 104 | {AMREX_D_DECL(0.90,0.3,0.5*zlen)}, 105 | {AMREX_D_DECL(0.15,0.7,0.5*zlen)}, 106 | {AMREX_D_DECL(0.45,0.7,0.5*zlen)}, 107 | {AMREX_D_DECL(0.75,0.7,0.5*zlen)}, 108 | {AMREX_D_DECL(1.05,0.7,0.5*zlen)}, 109 | {AMREX_D_DECL(0.30,1.1,0.5*zlen)}, 110 | {AMREX_D_DECL(0.60,1.1,0.5*zlen)}, 111 | {AMREX_D_DECL(0.90,1.1,0.5*zlen)}, 112 | {AMREX_D_DECL(0.15,1.5,0.5*zlen)}, 113 | {AMREX_D_DECL(0.45,1.5,0.5*zlen)}, 114 | {AMREX_D_DECL(0.75,1.5,0.5*zlen)}, 115 | {AMREX_D_DECL(1.05,1.5,0.5*zlen)}}; 116 | 117 | // The "false" below is the boolean that determines if the fluid is inside ("true") or 118 | // outside ("false") the object(s) 119 | int direction = 2; 120 | Real height = -1.0; // Putting a negative number for height means it extends beyond the domain 121 | Array obstacles{ 122 | EB2::CylinderIF(obstacle_radius, height, direction, obstacle_center[ 0], false), 123 | EB2::CylinderIF(obstacle_radius, height, direction, obstacle_center[ 1], false), 124 | EB2::CylinderIF(obstacle_radius, height, direction, obstacle_center[ 2], false), 125 | EB2::CylinderIF(obstacle_radius, height, direction, obstacle_center[ 3], false), 126 | EB2::CylinderIF(obstacle_radius, height, direction, obstacle_center[ 4], false), 127 | EB2::CylinderIF(obstacle_radius, height, direction, obstacle_center[ 5], false), 128 | EB2::CylinderIF(obstacle_radius, height, direction, obstacle_center[ 6], false), 129 | EB2::CylinderIF(obstacle_radius, height, direction, obstacle_center[ 7], false), 130 | EB2::CylinderIF(obstacle_radius, height, direction, obstacle_center[ 8], false), 131 | EB2::CylinderIF(obstacle_radius, height, direction, obstacle_center[ 9], false), 132 | EB2::CylinderIF(obstacle_radius, height, direction, obstacle_center[10], false), 133 | EB2::CylinderIF(obstacle_radius, height, direction, obstacle_center[11], false), 134 | EB2::CylinderIF(obstacle_radius, height, direction, obstacle_center[12], false), 135 | EB2::CylinderIF(obstacle_radius, height, direction, obstacle_center[13], false)}; 136 | 137 | amrex::Gpu::DeviceVector obstacle_center_d; 138 | obstacle_center_d.resize(0); 139 | obstacle_center_d.resize(obstacle_center.size()); 140 | Gpu::copyAsync(Gpu::hostToDevice, obstacle_center.begin(), obstacle_center.end(), obstacle_center_d.begin()); 141 | 142 | auto group_1 = EB2::makeUnion(obstacles[0],obstacles[1],obstacles[2]); 143 | auto group_2 = EB2::makeUnion(obstacles[3],obstacles[4],obstacles[5]); 144 | auto group_3 = EB2::makeUnion(obstacles[6],obstacles[7],obstacles[8]); 145 | auto group_4 = EB2::makeUnion(obstacles[9],obstacles[10],obstacles[11]); 146 | auto group_5 = EB2::makeUnion(obstacles[12],obstacles[13]); 147 | auto all = EB2::makeUnion(group_1,group_2,group_3,group_4,group_5); 148 | auto gshop9 = EB2::makeShop(all); 149 | 150 | int required_coarsening_level = 0; // typically the same as the max AMR level index 151 | int max_coarsening_level = 0; // typically a huge number so MG coarsens as much as possible 152 | EB2::Build(gshop9, geom, required_coarsening_level, max_coarsening_level); 153 | 154 | const EB2::IndexSpace& eb_is = EB2::IndexSpace::top(); 155 | const EB2::Level& eb_level = eb_is.getLevel(geom); 156 | 157 | // options are basic, volume, or full 158 | EBSupport ebs = EBSupport::full; 159 | 160 | // number of ghost cells for each of the 3 EBSupport types 161 | Vector ng_ebs = {2,2,2}; 162 | 163 | // This object provides access to the EB database in the format of basic AMReX objects 164 | // such as BaseFab, FArrayBox, FabArray, and MultiFab 165 | EBFArrayBoxFactory factory(eb_level, geom, grids, dmap, ng_ebs, ebs); 166 | 167 | // Initialize Particles 168 | MyParticleContainer MyPC(geom, dmap, grids); 169 | MyPC.InitPachinko(particle_file,zlen); 170 | 171 | // Store processor id in the plotfile 172 | plotfile_mf.define(grids, dmap, 1, 0, MFInfo(), factory); 173 | 174 | amrex::Print() << " \n********************************************************************" << std::endl; 175 | amrex::Print() << " Let's advect the particles ... " << std::endl; 176 | amrex::Print() << " We'll print a dot every 10 time steps." << std::endl; 177 | amrex::Print() << "******************************************************************** \n" << std::endl; 178 | 179 | // copy processor id into plotfile_mf 180 | int lev = 0; 181 | for (MFIter mfi = MyPC.MakeMFIter(lev); mfi.isValid(); ++mfi) 182 | plotfile_mf[mfi].setVal(ParallelDescriptor::MyProc()); 183 | 184 | // wallclock time 185 | Real strt_total = amrex::second(); 186 | 187 | #ifdef _OPENMP 188 | #pragma omp parallel if (Gpu::notInLaunchRegion()) 189 | #endif 190 | 191 | Real time = 0.0; 192 | for (int i = 0; i < max_steps; i++) 193 | { 194 | if (time < max_time) { 195 | time_step = std::min(time_step, max_time - time); 196 | 197 | // Step Particles 198 | MyPC.AdvectPachinko(time_step,obstacle_center_d,obstacle_radius,particle_radius); 199 | 200 | MyPC.Redistribute(); 201 | 202 | // Write to a plotfile 203 | if (i%plot_int == 0) 204 | write_plotfile(i, geom, plotfile_mf, MyPC, ascii_particle_output); 205 | 206 | if (i%10 == 0) 207 | amrex::Print() << "."; 208 | 209 | // Increment time 210 | time += time_step; 211 | 212 | } else 213 | break; 214 | } 215 | 216 | amrex::Print() << "\n" << std::endl; 217 | amrex::Print() << "********************************************************************" << std::endl; 218 | amrex::Print() << "We've finished moving the particles to time " << time << std::endl; 219 | 220 | // wallclock time 221 | Real end_total = amrex::second() - strt_total; 222 | 223 | amrex::Print() << "That took " << end_total << " seconds." << std::endl; 224 | amrex::Print() << "******************************************************************** \n" << std::endl; 225 | 226 | Print() << "Writing EB surface" << std::endl; 227 | WriteEBSurface (grids, dmap, geom, &factory); 228 | } 229 | 230 | amrex::Finalize(); 231 | } 232 | -------------------------------------------------------------------------------- /AMReX_EB_Pachinko/pachinko.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AMReX-Codes/ATPESC-codes/2f5da61d159822df3c74f75035367feadfa399e3/AMReX_EB_Pachinko/pachinko.gif -------------------------------------------------------------------------------- /AMReX_EB_Pachinko/paraview_pachinko.py: -------------------------------------------------------------------------------- 1 | # trace generated using paraview version 5.8.0 2 | # 3 | # To ensure correct image size when batch processing, please search 4 | # for and uncomment the line `# renderView*.ViewSize = [*,*]` 5 | 6 | #### import the simple module from the paraview 7 | from paraview.simple import * 8 | 9 | import subprocess 10 | import glob 11 | import argparse 12 | 13 | parser = argparse.ArgumentParser() 14 | parser.add_argument('-f', '--frame_rate', type=int, default=15, help="Frame rate for generating movies, i.e. number of plots per second in the movie.") 15 | parser.add_argument('-r', '--resolution', type=int, default=1024, help="(Square) resolution of output movie.") 16 | args = parser.parse_args() 17 | 18 | def generate_movie_3D(AllPlotFiles): 19 | #### disable automatic camera reset on 'Show' 20 | paraview.simple._DisableFirstRenderCameraReset() 21 | 22 | # create a new 'XML Partitioned Polydata Reader' 23 | ebpvtp = XMLPartitionedPolydataReader(FileName=['eb.pvtp']) 24 | 25 | # get active view 26 | renderView1 = GetActiveViewOrCreate('RenderView') 27 | # uncomment following to set a specific view size 28 | renderView1.ViewSize = [1200, 1200] 29 | 30 | # get layout 31 | layout1 = GetLayout() 32 | 33 | # show data in view 34 | ebpvtpDisplay = Show(ebpvtp, renderView1, 'GeometryRepresentation') 35 | 36 | # trace defaults for the display properties. 37 | ebpvtpDisplay.Representation = 'Surface' 38 | ebpvtpDisplay.ColorArrayName = [None, ''] 39 | ebpvtpDisplay.OSPRayScaleFunction = 'PiecewiseFunction' 40 | ebpvtpDisplay.SelectOrientationVectors = 'None' 41 | ebpvtpDisplay.ScaleFactor = 0.14000000208616256 42 | ebpvtpDisplay.SelectScaleArray = 'None' 43 | ebpvtpDisplay.GlyphType = 'Arrow' 44 | ebpvtpDisplay.GlyphTableIndexArray = 'None' 45 | ebpvtpDisplay.GaussianRadius = 0.007000000104308128 46 | ebpvtpDisplay.SetScaleArray = [None, ''] 47 | ebpvtpDisplay.ScaleTransferFunction = 'PiecewiseFunction' 48 | ebpvtpDisplay.OpacityArray = [None, ''] 49 | ebpvtpDisplay.OpacityTransferFunction = 'PiecewiseFunction' 50 | ebpvtpDisplay.DataAxesGrid = 'GridAxesRepresentation' 51 | ebpvtpDisplay.PolarAxes = 'PolarAxesRepresentation' 52 | 53 | # reset view to fit data 54 | renderView1.ResetCamera() 55 | 56 | # get the material library 57 | materialLibrary1 = GetMaterialLibrary() 58 | 59 | # update the view to ensure updated data information 60 | renderView1.Update() 61 | 62 | # create a new 'AMReX/BoxLib Grid Reader' 63 | plt00000 = AMReXBoxLibGridReader(FileNames=['plt00000']) 64 | plt00000.CellArrayStatus = [] 65 | 66 | # Properties modified on plt00000 67 | plt00000.CellArrayStatus = ['proc', 'vfrac'] 68 | 69 | # show data in view 70 | plt00000Display = Show(plt00000, renderView1, 'AMRRepresentation') 71 | 72 | # trace defaults for the display properties. 73 | plt00000Display.Representation = 'Outline' 74 | plt00000Display.ColorArrayName = [None, ''] 75 | plt00000Display.OSPRayScaleFunction = 'PiecewiseFunction' 76 | plt00000Display.SelectOrientationVectors = 'None' 77 | plt00000Display.ScaleFactor = 0.2 78 | plt00000Display.SelectScaleArray = 'None' 79 | plt00000Display.GlyphType = 'Arrow' 80 | plt00000Display.GlyphTableIndexArray = 'None' 81 | plt00000Display.GaussianRadius = 0.01 82 | plt00000Display.SetScaleArray = [None, ''] 83 | plt00000Display.ScaleTransferFunction = 'PiecewiseFunction' 84 | plt00000Display.OpacityArray = [None, ''] 85 | plt00000Display.OpacityTransferFunction = 'PiecewiseFunction' 86 | plt00000Display.DataAxesGrid = 'GridAxesRepresentation' 87 | plt00000Display.PolarAxes = 'PolarAxesRepresentation' 88 | plt00000Display.ScalarOpacityUnitDistance = 0.051941539345088994 89 | 90 | # update the view to ensure updated data information 91 | renderView1.Update() 92 | 93 | # create a new 'Slice' 94 | slice1 = Slice(Input=plt00000) 95 | slice1.SliceType = 'Plane' 96 | slice1.HyperTreeGridSlicer = 'Plane' 97 | slice1.SliceOffsetValues = [0.0] 98 | 99 | # init the 'Plane' selected for 'SliceType' 100 | slice1.SliceType.Origin = [0.625, 1.0, 0.25] 101 | 102 | # init the 'Plane' selected for 'HyperTreeGridSlicer' 103 | slice1.HyperTreeGridSlicer.Origin = [0.625, 1.0, 0.25] 104 | 105 | # toggle 3D widget visibility (only when running from the GUI) 106 | #Hide3DWidgets(proxy=slice1.SliceType) 107 | 108 | # Properties modified on slice1.SliceType 109 | slice1.SliceType.Normal = [0.0, 0.0, 1.0] 110 | 111 | # show data in view 112 | slice1Display = Show(slice1, renderView1, 'GeometryRepresentation') 113 | 114 | # trace defaults for the display properties. 115 | slice1Display.Representation = 'Surface' 116 | slice1Display.ColorArrayName = [None, ''] 117 | slice1Display.OSPRayScaleFunction = 'PiecewiseFunction' 118 | slice1Display.SelectOrientationVectors = 'None' 119 | slice1Display.ScaleFactor = 0.2 120 | slice1Display.SelectScaleArray = 'None' 121 | slice1Display.GlyphType = 'Arrow' 122 | slice1Display.GlyphTableIndexArray = 'None' 123 | slice1Display.GaussianRadius = 0.01 124 | slice1Display.SetScaleArray = [None, ''] 125 | slice1Display.ScaleTransferFunction = 'PiecewiseFunction' 126 | slice1Display.OpacityArray = [None, ''] 127 | slice1Display.OpacityTransferFunction = 'PiecewiseFunction' 128 | slice1Display.DataAxesGrid = 'GridAxesRepresentation' 129 | slice1Display.PolarAxes = 'PolarAxesRepresentation' 130 | 131 | # update the view to ensure updated data information 132 | renderView1.Update() 133 | 134 | # set scalar coloring 135 | ColorBy(slice1Display, ('FIELD', 'vtkBlockColors')) 136 | 137 | # get color transfer function/color map for 'vtkBlockColors' 138 | vtkBlockColorsLUT = GetColorTransferFunction('vtkBlockColors') 139 | 140 | # get opacity transfer function/opacity map for 'vtkBlockColors' 141 | vtkBlockColorsPWF = GetOpacityTransferFunction('vtkBlockColors') 142 | 143 | # set scalar coloring 144 | ColorBy(slice1Display, ('CELLS', 'proc')) 145 | 146 | # rescale color and/or opacity maps used to include current data range 147 | slice1Display.RescaleTransferFunctionToDataRange(True, False) 148 | 149 | # get color transfer function/color map for 'proc' 150 | procLUT = GetColorTransferFunction('proc') 151 | 152 | # get opacity transfer function/opacity map for 'proc' 153 | procPWF = GetOpacityTransferFunction('proc') 154 | 155 | # create a new 'AMReX/BoxLib Particles Reader' 156 | plt0 = AMReXBoxLibParticlesReader(FileNames=AllPlotFiles) 157 | plt0.PointArrayStatus = ['id', 'cpu', 'int_comp0', 'real_comp0', 'real_comp1', 'real_comp2'] 158 | 159 | # get animation scene 160 | animationScene1 = GetAnimationScene() 161 | 162 | # get the time-keeper 163 | timeKeeper1 = GetTimeKeeper() 164 | 165 | # update animation scene based on data timesteps 166 | animationScene1.UpdateAnimationUsingDataTimeSteps() 167 | 168 | # show data in view 169 | plt0Display = Show(plt0, renderView1, 'GeometryRepresentation') 170 | 171 | # trace defaults for the display properties. 172 | plt0Display.Representation = 'Surface' 173 | plt0Display.ColorArrayName = [None, ''] 174 | plt0Display.OSPRayScaleArray = 'cpu' 175 | plt0Display.OSPRayScaleFunction = 'PiecewiseFunction' 176 | plt0Display.SelectOrientationVectors = 'None' 177 | plt0Display.ScaleFactor = 0.11000000000000001 178 | plt0Display.SelectScaleArray = 'None' 179 | plt0Display.GlyphType = 'Arrow' 180 | plt0Display.GlyphTableIndexArray = 'None' 181 | plt0Display.GaussianRadius = 0.0055000000000000005 182 | plt0Display.SetScaleArray = ['POINTS', 'cpu'] 183 | plt0Display.ScaleTransferFunction = 'PiecewiseFunction' 184 | plt0Display.OpacityArray = ['POINTS', 'cpu'] 185 | plt0Display.OpacityTransferFunction = 'PiecewiseFunction' 186 | plt0Display.DataAxesGrid = 'GridAxesRepresentation' 187 | plt0Display.PolarAxes = 'PolarAxesRepresentation' 188 | 189 | # init the 'PiecewiseFunction' selected for 'ScaleTransferFunction' 190 | plt0Display.ScaleTransferFunction.Points = [0.0, 0.0, 0.5, 0.0, 3.0, 1.0, 0.5, 0.0] 191 | 192 | # init the 'PiecewiseFunction' selected for 'OpacityTransferFunction' 193 | plt0Display.OpacityTransferFunction.Points = [0.0, 0.0, 0.5, 0.0, 3.0, 1.0, 0.5, 0.0] 194 | 195 | # update the view to ensure updated data information 196 | renderView1.Update() 197 | 198 | # set scalar coloring 199 | ColorBy(plt0Display, ('FIELD', 'vtkBlockColors')) 200 | 201 | # create a new 'Glyph' 202 | glyph1 = Glyph(Input=plt0, 203 | GlyphType='Arrow') 204 | glyph1.OrientationArray = ['POINTS', 'No orientation array'] 205 | glyph1.ScaleArray = ['POINTS', 'No scale array'] 206 | glyph1.ScaleFactor = 0.11000000000000001 207 | glyph1.GlyphTransform = 'Transform2' 208 | 209 | # Properties modified on glyph1 210 | glyph1.GlyphType = 'Sphere' 211 | glyph1.ScaleFactor = 0.05 212 | glyph1.GlyphMode = 'All Points' 213 | 214 | # show data in view 215 | glyph1Display = Show(glyph1, renderView1, 'GeometryRepresentation') 216 | 217 | # trace defaults for the display properties. 218 | glyph1Display.Representation = 'Surface' 219 | glyph1Display.ColorArrayName = [None, ''] 220 | glyph1Display.OSPRayScaleArray = 'Normals' 221 | glyph1Display.OSPRayScaleFunction = 'PiecewiseFunction' 222 | glyph1Display.SelectOrientationVectors = 'None' 223 | glyph1Display.ScaleFactor = 0.11487463433295489 224 | glyph1Display.SelectScaleArray = 'None' 225 | glyph1Display.GlyphType = 'Arrow' 226 | glyph1Display.GlyphTableIndexArray = 'None' 227 | glyph1Display.GaussianRadius = 0.005743731716647744 228 | glyph1Display.SetScaleArray = ['POINTS', 'Normals'] 229 | glyph1Display.ScaleTransferFunction = 'PiecewiseFunction' 230 | glyph1Display.OpacityArray = ['POINTS', 'Normals'] 231 | glyph1Display.OpacityTransferFunction = 'PiecewiseFunction' 232 | glyph1Display.DataAxesGrid = 'GridAxesRepresentation' 233 | glyph1Display.PolarAxes = 'PolarAxesRepresentation' 234 | 235 | # init the 'PiecewiseFunction' selected for 'ScaleTransferFunction' 236 | glyph1Display.ScaleTransferFunction.Points = [-0.9749279022216797, 0.0, 0.5, 0.0, 0.9749279022216797, 1.0, 0.5, 0.0] 237 | 238 | # init the 'PiecewiseFunction' selected for 'OpacityTransferFunction' 239 | glyph1Display.OpacityTransferFunction.Points = [-0.9749279022216797, 0.0, 0.5, 0.0, 0.9749279022216797, 1.0, 0.5, 0.0] 240 | 241 | # update the view to ensure updated data information 242 | renderView1.Update() 243 | 244 | # set scalar coloring 245 | ColorBy(glyph1Display, ('FIELD', 'vtkBlockColors')) 246 | 247 | # current camera placement for renderView1 248 | renderView1.CameraPosition = [0.5999999884516001, 0.9000000134110451, 5.480672965279949] 249 | renderView1.CameraFocalPoint = [0.5999999884516001, 0.9000000134110451, 0.25] 250 | renderView1.CameraParallelScale = 0.9246621010295244 251 | 252 | # save animation 253 | output_movie_base = "pachinko" 254 | output_movie = output_movie_base + ".avi" 255 | SaveAnimation(output_movie, 256 | renderView1, 257 | ImageResolution=[1200, 1200], 258 | FrameRate=args.frame_rate, 259 | FrameWindow=[0, len(AllPlotFiles)-1]) 260 | 261 | return output_movie_base, output_movie 262 | 263 | def convert_avi_to_gif(output_movie_base, output_movie): 264 | # use ffmpeg to convert the avi movie into an animated gif 265 | ffmpeg_convert_to_gif = 'ffmpeg -y -i {} -vf "fps=35,scale={}:-1:flags=lanczos,split[s0][s1];[s0]palettegen[p];[s1][p]paletteuse" -loop 0 {}.gif'.format(output_movie, args.resolution, output_movie_base) 266 | subprocess.run(ffmpeg_convert_to_gif, shell=True) 267 | 268 | if __name__ == "__main__": 269 | if args.frame_rate <= 0: 270 | print("Please specify --frame_rate F (with F > 0)") 271 | exit() 272 | 273 | if args.resolution <= 0: 274 | print("Please specify --resolution R (with R > 0)") 275 | exit() 276 | 277 | # get all the plotfiles 278 | PlotFiles = sorted(glob.glob("plt" + "[0-9]"*5)) 279 | 280 | # generate the avi movie file 281 | output_movie_base, output_movie = generate_movie_3D(PlotFiles) 282 | 283 | # convert the avi movie into an animated gif 284 | convert_avi_to_gif(output_movie_base, output_movie) 285 | -------------------------------------------------------------------------------- /SUNDIALS+AMReX/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.17) 2 | cmake_policy(SET CMP0109 NEW) 3 | 4 | # Prevent in-source builds 5 | if(CMAKE_BINARY_DIR STREQUAL CMAKE_SOURCE_DIR) 6 | message(FATAL_ERROR "\nIn-source builds are prohbited!\n") 7 | endif () 8 | 9 | # ATPESC example project 10 | project(atpesc-sundials-amrex-example 11 | DESCRIPTION "ATPESC SUNDIALS+AMReX Example" 12 | LANGUAGES C CXX) 13 | 14 | # Build options 15 | option(ENABLE_CUDA "Enable CUDA" OFF) 16 | option(ENABLE_HIP "Enable HIP" OFF) 17 | 18 | set(AMREX_ROOT "$ENV{AMREX_ROOT}" 19 | CACHE PATH "Path to AMReX installation directory") 20 | 21 | set(SUNDIALS_ROOT "$ENV{SUNDIALS_DIR}" 22 | CACHE PATH "Path to SUNDIALS installation directory") 23 | 24 | # Check for MPI 25 | find_package(MPI REQUIRED) 26 | 27 | # Compiler options 28 | if(NOT DEFINED CMAKE_CXX_STANDARD) 29 | set(CMAKE_CXX_STANDARD 14) 30 | set(CMAKE_CXX_STANDARD_REQUIRED TRUE) 31 | endif() 32 | 33 | # Setup CUDA 34 | if(ENABLE_CUDA) 35 | enable_language(CUDA) 36 | set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER}) 37 | find_package(CUDAToolkit REQUIRED) 38 | endif() 39 | 40 | if(ENABLE_HIP) 41 | if(NOT DEFINED ROCM_PATH) 42 | if(NOT DEFINED ENV{ROCM_PATH}) 43 | set(ROCM_PATH "/opt/rocm/" CACHE PATH "Path to which ROCm has been installed") 44 | else() 45 | set(ROCM_PATH "$ENV{ROCM_PATH}" CACHE PATH "Path to which ROCm has been installed") 46 | endif() 47 | endif() 48 | 49 | if(NOT DEFINED HIP_PATH) 50 | if(NOT DEFINED ENV{HIP_PATH}) 51 | set(HIP_PATH "/opt/rocm/hip" CACHE PATH "Path to which HIP has been installed") 52 | else() 53 | set(HIP_PATH "$ENV{HIP_PATH}" CACHE PATH "Path to which HIP has been installed") 54 | endif() 55 | endif() 56 | 57 | if(NOT DEFINED HIP_PLATFORM) 58 | if(NOT DEFINED ENV{HIP_PLATFORM}) 59 | set(HIP_PLATFORM "hcc" CACHE STRING "HIP platform (hcc, nvcc)") 60 | else() 61 | set(HIP_PLATFORM "$ENV{HIP_PLATFORM}" CACHE STRING "HIP platform (hcc, nvcc)") 62 | endif() 63 | endif() 64 | 65 | # Set CMAKE_PREFIX_PATH as the hip-config.cmake has some find_package calls 66 | # which don't have the proper path set (not sure if this is a bug or 67 | # intentional), so without this they will fail even if we provide the PATH 68 | # option to find_package(HIP). 69 | set(CMAKE_PREFIX_PATH "${ROCM_PATH};${HIP_PATH}") 70 | find_package(HIP REQUIRED CONFIG) 71 | find_package(rocrand REQUIRED CONFIG) 72 | find_package(rocprim REQUIRED CONFIG) 73 | find_package(hiprand REQUIRED CONFIG) 74 | 75 | if("${HIP_COMPILER}" STREQUAL "hcc") 76 | print_error("Deprecated HCC compiler is not supported" "Please update ROCm") 77 | endif() 78 | 79 | message(STATUS "HIP version: ${HIP_VERSION}") 80 | message(STATUS "HIP platform: ${HIP_PLATFORM}") 81 | message(STATUS "HIP compiler: ${HIP_COMPILER}") 82 | message(STATUS "HIP linker: ${CMAKE_CXX_LINK_EXECUTABLE}") 83 | message(STATUS "AMD targets: ${AMDGPU_TARGETS}") 84 | endif() 85 | 86 | # Check for AMReX 87 | 88 | # Determine AMReX components needed 89 | if(NOT AMREX_FIND_COMPONENTS) 90 | set(AMREX_FIND_COMPONENTS "SUNDIALS") 91 | if(ENABLE_CUDA) 92 | list(APPEND AMREX_FIND_COMPONENTS "CUDA") 93 | endif() 94 | if(ENABLE_HIP) 95 | list(APPEND AMREX_FIND_COMPONENTS "HIP") 96 | endif() 97 | endif() 98 | 99 | find_package(AMReX REQUIRED COMPONENTS ${AMREX_FIND_COMPONENTS} 100 | HINTS ${AMREX_DIR} ${AMREX_ROOT} ${CMAKE_PREFIX_PATH} 101 | NO_DEFAULT_PATH) 102 | 103 | # Check for SUNDIALS 104 | 105 | # Determine SUNDIALS components needed 106 | if(NOT SUNDIALS_FIND_COMPONENTS) 107 | set(SUNDIALS_FIND_COMPONENTS 108 | "arkode" 109 | "cvode") 110 | if(ENABLE_CUDA) 111 | list(APPEND SUNDIALS_FIND_COMPONENTS "nveccuda") 112 | endif() 113 | if(ENABLE_HIP) 114 | list(APPEND SUNDIALS_FIND_COMPONENTS "nvechip") 115 | endif() 116 | endif() 117 | 118 | find_package(SUNDIALS REQUIRED COMPONENTS ${SUNDIALS_FIND_COMPONENTS} 119 | HINTS ${SUNDIALS_DIR} ${SUNDIALS_ROOT} ${CMAKE_PREFIX_PATH} 120 | NO_DEFAULT_PATH) 121 | 122 | # Check for math library 123 | find_library(MATH_LIBRARY NAMES m) 124 | 125 | # Targets to build 126 | set(targets 127 | "HandsOn1" 128 | "HandsOn2" 129 | "HandsOn3") 130 | 131 | # Add the build and install targets for each example 132 | foreach(target ${targets}) 133 | 134 | if(ENABLE_CUDA) 135 | set(target_name "${target}.CUDA.exe") 136 | elseif(ENABLE_HIP) 137 | set(target_name "${target}.HIP.exe") 138 | else() 139 | set(target_name "${target}.exe") 140 | endif() 141 | 142 | add_executable(${target_name} ${target}.cpp HandsOn_main.cpp) 143 | 144 | if(ENABLE_CUDA) 145 | set_source_files_properties(${target}.cpp HandsOn_main.cpp 146 | PROPERTIES LANGUAGE CUDA) 147 | # Add -dc flag 148 | set_target_properties(${target_name} 149 | PROPERTIES CUDA_SEPARABLE_COMPILATION ON) 150 | endif() 151 | 152 | # Link to required libraries 153 | target_link_libraries(${target_name} PRIVATE 154 | AMReX::amrex_2d 155 | SUNDIALS::arkode 156 | SUNDIALS::cvode 157 | MPI::MPI_CXX 158 | m) 159 | 160 | # Link to HYPRE using target defined by AMReX 161 | if (AMReX_HYPRE) 162 | target_link_libraries(${target_name} PRIVATE HYPRE) 163 | endif() 164 | 165 | # Link to libraries needed for CUDA support 166 | if(ENABLE_CUDA) 167 | target_link_libraries(${target_name} PRIVATE 168 | SUNDIALS::nveccuda 169 | CUDA::cusparse 170 | CUDA::curand 171 | CUDA::cublas 172 | CUDA::nvToolsExt) 173 | endif() 174 | 175 | # Install execuatable 176 | install(TARGETS ${target_name} DESTINATION ${CMAKE_INSTALL_PREFIX}) 177 | 178 | endforeach() 179 | 180 | # Install input files and postprocessing scripts 181 | install(FILES 182 | diag_tools.py 183 | inputs 184 | inputs-1 185 | inputs-2 186 | inputs-3 187 | inputs-ref 188 | DESTINATION ${CMAKE_INSTALL_PREFIX}) 189 | 190 | install(FILES 191 | process_ARKStep_diags.py 192 | PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE 193 | DESTINATION ${CMAKE_INSTALL_PREFIX}) 194 | 195 | # Install reference solution 196 | install(DIRECTORY 197 | reference_solution 198 | DESTINATION ${CMAKE_INSTALL_PREFIX}) 199 | 200 | # find_program(AMREX_FCOMPARE fcompare ${AMREX_ROOT}/bin) 201 | # if(AMREX_FCOMPARE) 202 | # message(STATUS "Found fcompare: ${AMREX_FCOMPARE}") 203 | # install(FILES 204 | # ${AMREX_FCOMPARE} 205 | # PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE 206 | # DESTINATION ${CMAKE_INSTALL_PREFIX}) 207 | # else() 208 | # message(STATUS "Not found fcompare!") 209 | # endif() 210 | 211 | # Add target to clean up plots when running in the build directory 212 | add_custom_target(pltclean 213 | COMMAND ${CMAKE_COMMAND} -E rm -rf plt*/ *.png *_diagnostics.txt __pycache__) 214 | 215 | # Add target with list of plot headers for making a movie in Visit 216 | if(UNIX) 217 | add_custom_target(movie 218 | COMMAND ls -1 plt*/Header | tee movie.visit) 219 | endif() 220 | 221 | # Install source files 222 | if(INSTALL_SOURCE) 223 | install(FILES 224 | CMakeLists.txt 225 | HandsOn.hpp 226 | HandsOn1.cpp 227 | HandsOn2.cpp 228 | HandsOn3.cpp 229 | HandsOn_main.cpp 230 | README.md 231 | diag_tools.py 232 | inputs 233 | inputs-1 234 | inputs-2 235 | inputs-3 236 | inputs-ref 237 | DESTINATION SUNDIALS+AMReX) 238 | endif() 239 | -------------------------------------------------------------------------------- /SUNDIALS+AMReX/HandsOn.hpp: -------------------------------------------------------------------------------- 1 | /* ----------------------------------------------------------------------------- 2 | * Time Integration and Nonlinear Solvers Hands-on Lessons with SUNDIALS + AMReX 3 | * 2019-2022 Argonne Training Program in Extreme-Scale Computing 4 | * 5 | * Authors (alphabetical): 6 | * David Gardner (gardner48@llnl.gov) 7 | * John Loffeld (loffeld1@llnl.gov) 8 | * Daniel Reynolds (reynolds@smu.edu) 9 | * Donald Willcox (dewillcox@lbl.gov) 10 | * ----------------------------------------------------------------------------- 11 | * Hader file for SUNDIALS + AMReX 2D advection-diffusion example. 12 | * ---------------------------------------------------------------------------*/ 13 | 14 | #ifndef SUNDIALS_AMREX_EXAMPLE_H 15 | #define SUNDIALS_AMREX_EXAMPLE_H 16 | 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | 29 | // ------------------------ 30 | // User defined structures 31 | // ------------------------ 32 | 33 | // Structure for problem options 34 | struct ProblemOpt 35 | { 36 | int help = 0; // Print help message and exit (1) 37 | int plot_int = -1; // Enable (>0) or disable (<0) output files 38 | int arkode_order = 4; // ARKODE method order 39 | amrex::Real rtol = 1.0e-4; // Relative tolerance 40 | amrex::Real atol = 1.0e-9; // Absolute tolerance 41 | amrex::Real fixed_dt = -1.0; // Fixed time step size (-1 = adaptive) 42 | amrex::Real tfinal = 1.0e4; // Final integration time 43 | amrex::Real dtout = 1.0e4; // Output frequency 44 | int max_steps = 10000; // Max steps between outputs 45 | int write_diag = 1; // Enable (>0) diagnostics output 46 | int nls_max_iter = 3; // Max number of nonlinear iterations 47 | int ls_max_iter = 5; // Max number of linear iterations 48 | int rhs_adv = 1; // Implicit (0) or Explicit (1) advection 49 | int use_preconditioner = 0; // Enable (>0) preconditioner 50 | }; 51 | 52 | 53 | // User-defined structure passed through SUNDIALS to RHS functions 54 | struct ProblemData 55 | { 56 | // Grid options 57 | int n_cell = 128; // Number of cells on each side of a square domain. 58 | int max_grid_size = 64; // The domain split into boxes of size max_grid_size 59 | 60 | // AMReX grid data structures 61 | amrex::Geometry* geom = nullptr; 62 | amrex::BoxArray* grid = nullptr; 63 | amrex::DistributionMapping* dmap = nullptr; 64 | 65 | // AMReX MLMG preconditioner data and parameters 66 | amrex::MultiFab* acoef = nullptr; 67 | amrex::MultiFab* bcoef = nullptr; 68 | 69 | // MLMG Preconditioner options 70 | int mg_agglomeration = 1; 71 | int mg_consolidation = 1; 72 | int mg_max_coarsening_level = 1000; 73 | int mg_linop_maxorder = 2; 74 | int mg_max_iter = 1000; 75 | int mg_max_fmg_iter = 1000; 76 | int mg_fixed_iter = 0; 77 | int mg_verbose = 0; 78 | int mg_bottom_verbose = 0; 79 | int mg_use_hypre = 1; 80 | int mg_hypre_interface = 3; 81 | int mg_use_petsc = 0; 82 | amrex::Real mg_tol_rel = 1.0e-6; 83 | amrex::Real mg_tol_abs = 1.0e-6; 84 | 85 | // Advection coefficients 86 | amrex::Real advCoeffx = 5.0e-4; 87 | amrex::Real advCoeffy = 2.5e-4; 88 | 89 | // Diffusion coefficients 90 | amrex::Real diffCoeffx = 1.0e-6; 91 | amrex::Real diffCoeffy = 1.0e-6; 92 | 93 | // Scratch space for flux computation 94 | amrex::Array* flux = nullptr; 95 | }; 96 | 97 | // ----------------------- 98 | // User defined functions 99 | // ----------------------- 100 | 101 | // Read the problem inputs, print help info, and problem setup 102 | int ParseInputs(ProblemOpt& prob_opt, 103 | ProblemData& prob_data); 104 | 105 | // Print help information message and exit 106 | void PrintHelp(); 107 | 108 | // Print the problem setup 109 | void PrintSetup(ProblemOpt& prob_opt, 110 | ProblemData& prob_data); 111 | 112 | // Decompose the problem in space 113 | void SetUpGeometry(amrex::BoxArray& ba, 114 | amrex::Geometry& geom, 115 | ProblemData& prob_data); 116 | 117 | // Set the problem initial state 118 | void FillInitConds2D(amrex::MultiFab& sol, 119 | const amrex::Geometry& geom); 120 | 121 | // Setup the time integrator and advance the solution with SUNDIALS 122 | void ComputeSolution(N_Vector nv_sol, 123 | ProblemOpt* prob_opt, 124 | ProblemData* prob_data); 125 | 126 | // --------------------------------------------- 127 | // User-supplied ODE RHS functions for SUNDIALS 128 | // --------------------------------------------- 129 | 130 | // Advection RHS function 131 | int ComputeRhsAdv(amrex::Real t, 132 | N_Vector nv_sol, 133 | N_Vector nv_rhs, 134 | void* data); 135 | 136 | // Diffusion RHS function 137 | int ComputeRhsDiff(amrex::Real t, 138 | N_Vector nv_sol, 139 | N_Vector nv_rhs, 140 | void* data); 141 | 142 | // Advection-Diffusion RHS function 143 | int ComputeRhsAdvDiff(amrex::Real t, 144 | N_Vector nv_sol, 145 | N_Vector nv_rhs, 146 | void* data); 147 | 148 | // ----------------------------------------------- 149 | // Utility functions to compute ODE RHS functions 150 | // ----------------------------------------------- 151 | 152 | // Advective portion of ODE RHS 153 | void ComputeAdvectionUpwind(amrex::MultiFab& sol, 154 | amrex::MultiFab& advection, 155 | amrex::Geometry& geom, 156 | amrex::Real advCoeffx, 157 | amrex::Real advCoeffy); 158 | 159 | // Diffusive portion of ODE RHS 160 | void ComputeDiffusion(amrex::MultiFab& sol, 161 | amrex::MultiFab& diff_mf, 162 | amrex::MultiFab& fx_mf, 163 | amrex::MultiFab& fy_mf, 164 | amrex::Geometry& geom, 165 | amrex::Real diffCoeffx, 166 | amrex::Real diffCoeffy); 167 | 168 | // Utility functions for computing diffusion 169 | void ComputeDiffFlux(amrex::MultiFab& sol, 170 | amrex::MultiFab& fx, 171 | amrex::MultiFab& fy, 172 | amrex::Geometry& geom, 173 | amrex::Real diffCoeffx, 174 | amrex::Real diffCoeffy); 175 | 176 | void ComputeDivergence(amrex::MultiFab& div, 177 | amrex::MultiFab& fx, 178 | amrex::MultiFab& fy, 179 | amrex::Geometry& geom); 180 | 181 | // --------------------------------------------------- 182 | // User-supplied preconditioner function for SUNDIALS 183 | // --------------------------------------------------- 184 | 185 | int precondition_solve(amrex::Real tn, 186 | N_Vector u, 187 | N_Vector fu, 188 | N_Vector r, 189 | N_Vector z, 190 | amrex::Real gamma, 191 | amrex::Real delta, 192 | int lr, 193 | void *user_data); 194 | 195 | #endif 196 | -------------------------------------------------------------------------------- /SUNDIALS+AMReX/HandsOn1.cpp: -------------------------------------------------------------------------------- 1 | /*-------------------------------------------------------------------- 2 | Time Integration and Nonlinear Solvers 3 | Hands-on Lessons with SUNDIALS + AMReX 4 | 2019 Argonne Training Program in Extreme-Scale Computing 5 | 6 | Authors (alphabetical): 7 | David Gardner (gardner48@llnl.gov) 8 | John Loffeld (loffeld1@llnl.gov) 9 | Daniel Reynolds (reynolds@smu.edu) 10 | Donald Willcox (dewillcox@lbl.gov) 11 | 12 | -------------------------------------------------------------------- 13 | Implementation file for 'simplest' ARKode integration of 2D 14 | Advection-Diffusion example problem. This allows for either 15 | fixed-step or temporal adaptivity, but requires explicit time 16 | integration. 17 | --------------------------------------------------------------------*/ 18 | 19 | #include "HandsOn.hpp" 20 | #include 21 | 22 | using namespace amrex; 23 | 24 | // ----------------------------------------------------------------------------- 25 | // Evolve the problem with ARKODE using an explicit method 26 | // ----------------------------------------------------------------------------- 27 | 28 | void ComputeSolution(N_Vector nv_sol, ProblemOpt* prob_opt, 29 | ProblemData* prob_data) 30 | { 31 | BL_PROFILE("ComputeSolution()"); 32 | 33 | // Extract problem data and options 34 | Geometry* geom = prob_data->geom; 35 | int plot_int = prob_opt->plot_int; 36 | int arkode_order = prob_opt->arkode_order; 37 | Real rtol = prob_opt->rtol; 38 | Real atol = prob_opt->atol; 39 | Real fixed_dt = prob_opt->fixed_dt; 40 | Real tfinal = prob_opt->tfinal; 41 | Real dtout = prob_opt->dtout; 42 | int max_steps = prob_opt->max_steps; 43 | int write_diag = prob_opt->write_diag; 44 | 45 | // initial time, number of outputs, and error flag 46 | Real time = 0.0; 47 | int nout = ceil(tfinal/dtout); 48 | int ier = 0; 49 | 50 | // Write a plotfile of the initial data 51 | if (plot_int > 0) 52 | { 53 | const std::string& pltfile = amrex::Concatenate("plt", 0, 5); 54 | MultiFab* sol = amrex::sundials::N_VGetVectorPointer_MultiFab(nv_sol); 55 | WriteSingleLevelPlotfile(pltfile, *sol, {"u"}, *geom, time, 0); 56 | } 57 | 58 | // Create the ARK stepper 59 | void* arkode_mem = ARKStepCreate(ComputeRhsAdvDiff, nullptr, time, nv_sol, 60 | *amrex::sundials::The_Sundials_Context()); 61 | 62 | // Attach the user data structure to ARKStep 63 | ARKStepSetUserData(arkode_mem, prob_data); 64 | 65 | // Set the method order 66 | ARKStepSetOrder(arkode_mem, arkode_order); 67 | 68 | // Set the time step size or integration tolerances 69 | if (fixed_dt > 0.0) 70 | ARKStepSetFixedStep(arkode_mem, fixed_dt); 71 | else 72 | ARKStepSStolerances(arkode_mem, atol, rtol); 73 | 74 | // Set the max number of steps between outputs 75 | ARKStepSetMaxNumSteps(arkode_mem, max_steps); 76 | 77 | // Set file for writing ARKStep diagnostics 78 | FILE* diagfp = nullptr; 79 | if (write_diag) 80 | { 81 | diagfp = fopen("HandsOn1_diagnostics.txt", "w"); 82 | ARKStepSetDiagnostics(arkode_mem, diagfp); 83 | } 84 | 85 | // Advance the solution in time 86 | Real tout = time + dtout; // first output time 87 | Real tret; // return time 88 | for (int iout=0; iout < nout; iout++) 89 | { 90 | BL_PROFILE_VAR("ARKStepEvolve()", pevolve); 91 | ier = ARKStepEvolve(arkode_mem, tout, nv_sol, &tret, ARK_NORMAL); 92 | BL_PROFILE_VAR_STOP(pevolve); 93 | if (ier < 0) 94 | { 95 | amrex::Print() << "Error in ARKStepEvolve" << std::endl; 96 | return; 97 | } 98 | 99 | // Get integration stats 100 | long nfe_evals, nfi_evals; 101 | ARKStepGetNumRhsEvals(arkode_mem, &nfe_evals, &nfi_evals); 102 | amrex::Print() << "t = " << std::setw(5) << tret 103 | << " RHS evals = " << std::setw(7) << nfe_evals 104 | << std::endl; 105 | 106 | // Write output 107 | if (plot_int > 0) 108 | { 109 | const std::string& pltfile = amrex::Concatenate("plt", iout+1, 5); 110 | MultiFab* sol = amrex::sundials::N_VGetVectorPointer_MultiFab(nv_sol); 111 | WriteSingleLevelPlotfile(pltfile, *sol, {"u"}, *geom, tret, iout+1); 112 | } 113 | 114 | // Update output time 115 | tout += dtout; 116 | if (tout > tfinal) tout = tfinal; 117 | } 118 | 119 | // Output final solution statistics 120 | long int nst, nst_a, nfe, nfi, netf; 121 | nst = nst_a = nfe = nfi = netf = 0; 122 | ARKStepGetNumSteps(arkode_mem, &nst); 123 | ARKStepGetNumStepAttempts(arkode_mem, &nst_a); 124 | ARKStepGetNumRhsEvals(arkode_mem, &nfe, &nfi); 125 | ARKStepGetNumErrTestFails(arkode_mem, &netf); 126 | amrex::Print() << "\nFinal Solver Statistics:\n" 127 | << " Internal solver steps = " << nst << " (attempted = " << nst_a << ")\n" 128 | << " Total RHS evals = " << nfe << "\n" 129 | << " Total number of error test failures = " << netf << "\n"; 130 | 131 | // Close diagnostics file 132 | if (write_diag) fclose(diagfp); 133 | 134 | return; 135 | } 136 | 137 | 138 | // ----------------------------------------------------------------------------- 139 | // Print help message with relevant options for Hands-on 1 140 | // ----------------------------------------------------------------------------- 141 | 142 | 143 | void PrintHelp() 144 | { 145 | amrex::Print() 146 | << std:: endl 147 | << "Usage: HandsOn1.exe [fname] [options]" << std::endl 148 | << "Options:" << std::endl 149 | << " help=1" << std::endl 150 | << " Print this help message and exit." << std::endl 151 | << " plot_int=" << std::endl 152 | << " enable (1) or disable (0) plots [default=0]." << std::endl 153 | << " arkode_order=" << std::endl 154 | << " ARKStep method order [default=4]." << std::endl 155 | << " fixed_dt=" << std::endl 156 | << " use a fixed time step size (if value > 0.0) [default=-1.0]." << std::endl 157 | << " rtol=" << std::endl 158 | << " relative tolerance for time step adaptivity [default=1e-4]." << std::endl 159 | << " atol=" << std::endl 160 | << " absolute tolerance for time step adaptivity [default=1e-9]." << std::endl 161 | << " tfinal=" << std::endl 162 | << " final integration time [default=1e4]." << std::endl 163 | << " dtout=" << std::endl 164 | << " time between outputs [default=tfinal]." << std::endl 165 | << " max_steps=" << std::endl 166 | << " maximum number of internal steps between outputs [default=10000]." << std::endl 167 | << " write_diag=" << std::endl 168 | << " output ARKStep time step adaptivity diagnostics to a file [default=1]." << std::endl 169 | << " n_cell=" << std::endl 170 | << " number of cells on each side of the square domain [default=128]." << std::endl 171 | << " max_grid_size=" << std::endl 172 | << " max size of boxes in box array [default=64]." << std::endl 173 | << " advCoeffx=" << std::endl 174 | << " advection speed in the x-direction [default=5e-4]." << std::endl 175 | << " advCoeffy=" << std::endl 176 | << " advection speed in the y-direction [default=2.5e-4]." << std::endl 177 | << " diffCoeffx=" << std::endl 178 | << " diffusion coefficient in the x-direction [default=1e-6]." << std::endl 179 | << " diffCoeffy=" << std::endl 180 | << " diffusion coefficient in the y-direction [default=1e-6]." << std::endl << std::endl 181 | << "If a file name 'fname' is provided, it will be parsed for each of the above" << std::endl 182 | << "options. If an option is specified in both the input file and on the" << std::endl 183 | << "command line, then the command line option takes precedence." << std::endl << std::endl; 184 | return; 185 | } 186 | 187 | 188 | // ----------------------------------------------------------------------------- 189 | // Print relevant problem setup options for Hands-on 1 190 | // ----------------------------------------------------------------------------- 191 | 192 | 193 | void PrintSetup(ProblemOpt& prob_opt, ProblemData& prob_data) 194 | { 195 | // Ouput problem options and parameters 196 | amrex::Print() 197 | << "n_cell = " << prob_data.n_cell << std::endl 198 | << "max_grid_size = " << prob_data.max_grid_size << std::endl 199 | << "plot_int = " << prob_opt.plot_int << std::endl 200 | << "arkode_order = " << prob_opt.arkode_order << std::endl; 201 | if (prob_opt.fixed_dt > 0.0) 202 | amrex::Print() 203 | << "fixed_dt = " << prob_opt.fixed_dt << std::endl; 204 | else 205 | amrex::Print() 206 | << "rtol = " << prob_opt.rtol << std::endl 207 | << "atol = " << prob_opt.atol << std::endl; 208 | amrex::Print() 209 | << "tfinal = " << prob_opt.tfinal << std::endl 210 | << "dtout = " << prob_opt.dtout << std::endl 211 | << "write_diag = " << prob_opt.write_diag << std::endl 212 | << "advCoeffx = " << prob_data.advCoeffx << std::endl 213 | << "advCoeffy = " << prob_data.advCoeffy << std::endl 214 | << "diffCoeffx = " << prob_data.diffCoeffx << std::endl 215 | << "diffCoeffy = " << prob_data.diffCoeffy << std::endl; 216 | return; 217 | } 218 | -------------------------------------------------------------------------------- /SUNDIALS+AMReX/HandsOn2.cpp: -------------------------------------------------------------------------------- 1 | /* ----------------------------------------------------------------------------- 2 | * Time Integration and Nonlinear Solvers Hands-on Lessons with SUNDIALS + AMReX 3 | * 2019-2022 Argonne Training Program in Extreme-Scale Computing 4 | * 5 | * Authors (alphabetical): 6 | * David Gardner (gardner48@llnl.gov) 7 | * John Loffeld (loffeld1@llnl.gov) 8 | * Daniel Reynolds (reynolds@smu.edu) 9 | * Donald Willcox (dewillcox@lbl.gov) 10 | * ----------------------------------------------------------------------------- 11 | * Implementation file for 'intermediate' integration of 2D Advection-Diffusion 12 | * example problem using ARKODE. This treats the diffusion term of the problem 13 | * implicitly, but allows advection to be either implicit (via DIRK methods) or 14 | * explicit (via ARK-IMEX methods). The implicit portion is solved using a 15 | * Newton-Krylov method with unpreconditioned GMRES as the linear solver. This 16 | * program allows for either fixed time-step sizes or temporal adaptivity. 17 | * ---------------------------------------------------------------------------*/ 18 | 19 | #include "HandsOn.hpp" 20 | #include 21 | #include 22 | 23 | using namespace amrex; 24 | 25 | // ----------------------------------------------------------------------------- 26 | // Evolve the problem with ARKODE using an implicit or IMEX method 27 | // ----------------------------------------------------------------------------- 28 | 29 | void ComputeSolution(N_Vector nv_sol, ProblemOpt* prob_opt, 30 | ProblemData* prob_data) 31 | { 32 | BL_PROFILE("ComputeSolution()"); 33 | 34 | // Extract problem data and options 35 | Geometry* geom = prob_data->geom; 36 | int plot_int = prob_opt->plot_int; 37 | int arkode_order = prob_opt->arkode_order; 38 | int nls_max_iter = prob_opt->nls_max_iter; 39 | int ls_max_iter = prob_opt->ls_max_iter; 40 | int rhs_adv = prob_opt->rhs_adv; 41 | Real rtol = prob_opt->rtol; 42 | Real atol = prob_opt->atol; 43 | Real fixed_dt = prob_opt->fixed_dt; 44 | Real tfinal = prob_opt->tfinal; 45 | Real dtout = prob_opt->dtout; 46 | int max_steps = prob_opt->max_steps; 47 | int write_diag = prob_opt->write_diag; 48 | 49 | // initial time, number of outputs, and error flag 50 | Real time = 0.0; 51 | int nout = ceil(tfinal/dtout); 52 | int ier = 0; 53 | 54 | // Write a plotfile of the initial data 55 | if (plot_int > 0) 56 | { 57 | const std::string& pltfile = amrex::Concatenate("plt", 0, 5); 58 | MultiFab* sol = amrex::sundials::N_VGetVectorPointer_MultiFab(nv_sol); 59 | WriteSingleLevelPlotfile(pltfile, *sol, {"u"}, *geom, time, 0); 60 | } 61 | 62 | // Create the ARK stepper ***** UPDATED FROM HandsOn1 ***** 63 | void* arkode_mem = nullptr; 64 | if (rhs_adv) 65 | { 66 | // explicit advection and implicit diffusion 67 | arkode_mem = ARKStepCreate(ComputeRhsAdv, ComputeRhsDiff, time, nv_sol, 68 | *amrex::sundials::The_Sundials_Context()); 69 | } 70 | else 71 | { 72 | // implicit advection and diffusion 73 | arkode_mem = ARKStepCreate(nullptr, ComputeRhsAdvDiff, time, nv_sol, 74 | *amrex::sundials::The_Sundials_Context()); 75 | } 76 | 77 | // Attach the user data structure to ARKStep 78 | ARKStepSetUserData(arkode_mem, prob_data); 79 | 80 | // Set the method order 81 | ARKStepSetOrder(arkode_mem, arkode_order); 82 | 83 | // Set the time step size or integration tolerances 84 | if (fixed_dt > 0.0) 85 | ARKStepSetFixedStep(arkode_mem, fixed_dt); 86 | else 87 | ARKStepSStolerances(arkode_mem, atol, rtol); 88 | 89 | // Set the max number of steps between outputs 90 | ARKStepSetMaxNumSteps(arkode_mem, max_steps); 91 | 92 | // Set file for writing ARKStep diagnostics 93 | FILE* diagfp = nullptr; 94 | if (write_diag) 95 | { 96 | diagfp = fopen("HandsOn2_diagnostics.txt", "w"); 97 | ARKStepSetDiagnostics(arkode_mem, diagfp); 98 | } 99 | 100 | // Create and attach GMRES linear solver ***** UPDATED FROM HandsOn1 ***** 101 | SUNLinearSolver LS = SUNLinSol_SPGMR(nv_sol, PREC_NONE, ls_max_iter, 102 | *amrex::sundials::The_Sundials_Context()); 103 | ier = ARKStepSetLinearSolver(arkode_mem, LS, nullptr); 104 | if (ier != ARKLS_SUCCESS) 105 | { 106 | amrex::Print() << "Creating linear solver failed" << std::endl; 107 | return; 108 | } 109 | 110 | // Set max number of nonlinear iterations ***** UPDATED FROM HandsOn1 ***** 111 | ier = ARKStepSetMaxNonlinIters(arkode_mem, nls_max_iter); 112 | if (ier != ARK_SUCCESS) 113 | { 114 | amrex::Print() << "Error setting max number of nonlinear iterations" << std::endl; 115 | return; 116 | } 117 | 118 | // Advance the solution in time 119 | Real tout = time + dtout; // first output time 120 | Real tret; // return time 121 | for (int iout=0; iout < nout; iout++) 122 | { 123 | BL_PROFILE_VAR("ARKStepEvolve()", pevolve); 124 | ier = ARKStepEvolve(arkode_mem, tout, nv_sol, &tret, ARK_NORMAL); 125 | BL_PROFILE_VAR_STOP(pevolve); 126 | if (ier < 0) 127 | { 128 | amrex::Print() << "Error in ARKStepEvolve" << std::endl; 129 | return; 130 | } 131 | 132 | // Get integration stats ***** UPDATED FROM HandsOn1 ***** 133 | long nfe_evals, nfi_evals; 134 | ARKStepGetNumRhsEvals(arkode_mem, &nfe_evals, &nfi_evals); 135 | if (nfe_evals > 0) 136 | amrex::Print() << "t = " << std::setw(5) << tret 137 | << " explicit evals = " << std::setw(7) << nfe_evals 138 | << " implicit evals = " << std::setw(7) << nfi_evals 139 | << std::endl; 140 | else 141 | amrex::Print() << "t = " << std::setw(5) << tret 142 | << " RHS evals = " << std::setw(7) << nfi_evals 143 | << std::endl; 144 | 145 | // Write output 146 | if (plot_int > 0) 147 | { 148 | const std::string& pltfile = amrex::Concatenate("plt", iout+1, 5); 149 | MultiFab* sol = amrex::sundials::N_VGetVectorPointer_MultiFab(nv_sol); 150 | WriteSingleLevelPlotfile(pltfile, *sol, {"u"}, *geom, tret, iout+1); 151 | } 152 | 153 | // Update output time 154 | tout += dtout; 155 | if (tout > tfinal) tout = tfinal; 156 | } 157 | 158 | // Output final solution statistics ***** UPDATED FROM HandsOn1 ***** 159 | long int nst, nst_a, nfe, nfi, nsetups, nli, nJv, nlcf, nni, ncfn, netf; 160 | nst = nst_a = nfe = nfi = nsetups = nli = nJv = nlcf = nni = ncfn = netf = 0; 161 | ARKStepGetNumSteps(arkode_mem, &nst); 162 | ARKStepGetNumStepAttempts(arkode_mem, &nst_a); 163 | ARKStepGetNumRhsEvals(arkode_mem, &nfe, &nfi); 164 | ARKStepGetNumErrTestFails(arkode_mem, &netf); 165 | ARKStepGetNumNonlinSolvIters(arkode_mem, &nni); 166 | ARKStepGetNumNonlinSolvConvFails(arkode_mem, &ncfn); 167 | ARKStepGetNumLinSolvSetups(arkode_mem, &nsetups); 168 | ARKStepGetNumLinIters(arkode_mem, &nli); 169 | ARKStepGetNumJtimesEvals(arkode_mem, &nJv); 170 | ARKStepGetNumLinConvFails(arkode_mem, &nlcf); 171 | 172 | amrex::Print() << "\nFinal Solver Statistics:\n" 173 | << " Internal solver steps = " << nst << " (attempted = " << nst_a << ")\n"; 174 | if (nfe > 0) 175 | amrex::Print() << " Total RHS evals: Fe = " << nfe << ", Fi = " << nfi << "\n"; 176 | else 177 | amrex::Print() << " Total RHS evals = " << nfi << "\n"; 178 | amrex::Print() << " Total number of nonlinear iterations = " << nni << "\n" 179 | << " Total number of nonlinear solver convergence failures = " << ncfn << "\n" 180 | << " Total number of error test failures = " << netf << "\n"; 181 | amrex::Print() << " Total linear solver setups = " << nsetups << "\n" 182 | << " Total linear iterations = " << nli << "\n" 183 | << " Total number of Jacobian-vector products = " << nJv << "\n" 184 | << " Total number of linear solver convergence failures = " << nlcf << "\n"; 185 | 186 | // Close diagnostics file 187 | if (write_diag) fclose(diagfp); 188 | 189 | return; 190 | } 191 | 192 | 193 | // ----------------------------------------------------------------------------- 194 | // Print help message with relevant options for Hands-on 2 195 | // ----------------------------------------------------------------------------- 196 | 197 | 198 | void PrintHelp() 199 | { 200 | amrex::Print() 201 | << std:: endl 202 | << "Usage: HandsOn2.exe [fname] [options]" << std::endl 203 | << "Options:" << std::endl 204 | << " help=1" << std::endl 205 | << " Print this help message and exit." << std::endl 206 | << " plot_int=" << std::endl 207 | << " enable (1) or disable (0) plots [default=0]." << std::endl 208 | << " arkode_order=" << std::endl 209 | << " ARKStep method order [default=4]." << std::endl 210 | << " nls_max_iter=" << std::endl 211 | << " maximum number of nonlinear iterations [default=3]." << std::endl 212 | << " ls_max_iter=" << std::endl 213 | << " maximum number of linear iterations [default=5]." << std::endl 214 | << " rhs_adv=" << std::endl 215 | << " treat advection implicitly (0) or explicitly (1) [default=1]." << std::endl 216 | << " fixed_dt=" << std::endl 217 | << " use a fixed time step size (if value > 0.0) [default=-1.0]." << std::endl 218 | << " rtol=" << std::endl 219 | << " relative tolerance for time step adaptivity [default=1e-4]." << std::endl 220 | << " atol=" << std::endl 221 | << " absolute tolerance for time step adaptivity [default=1e-9]." << std::endl 222 | << " tfinal=" << std::endl 223 | << " final integration time [default=1e4]." << std::endl 224 | << " dtout=" << std::endl 225 | << " time between outputs [default=tfinal]." << std::endl 226 | << " max_steps=" << std::endl 227 | << " maximum number of internal steps between outputs [default=10000]." << std::endl 228 | << " write_diag=" << std::endl 229 | << " output ARKStep time step adaptivity diagnostics to a file [default=1]." << std::endl 230 | << " n_cell=" << std::endl 231 | << " number of cells on each side of the square domain [default=128]." << std::endl 232 | << " max_grid_size=" << std::endl 233 | << " max size of boxes in box array [default=64]." << std::endl 234 | << " advCoeffx=" << std::endl 235 | << " advection speed in the x-direction [default=5e-4]." << std::endl 236 | << " advCoeffy=" << std::endl 237 | << " advection speed in the y-direction [default=2.5e-4]." << std::endl 238 | << " diffCoeffx=" << std::endl 239 | << " diffusion coefficient in the x-direction [default=1e-6]." << std::endl 240 | << " diffCoeffy=" << std::endl 241 | << " diffusion coefficient in the y-direction [default=1e-6]." << std::endl << std::endl 242 | << "If a file name 'fname' is provided, it will be parsed for each of the above" << std::endl 243 | << "options. If an option is specified in both the input file and on the" << std::endl 244 | << "command line, then the command line option takes precedence." << std::endl << std::endl; 245 | return; 246 | } 247 | 248 | 249 | // ----------------------------------------------------------------------------- 250 | // Print relevant problem setup options for Hands-on 1 251 | // ----------------------------------------------------------------------------- 252 | 253 | 254 | void PrintSetup(ProblemOpt& prob_opt, ProblemData& prob_data) 255 | { 256 | // Ouput problem options and parameters 257 | amrex::Print() 258 | << "n_cell = " << prob_data.n_cell << std::endl 259 | << "max_grid_size = " << prob_data.max_grid_size << std::endl 260 | << "plot_int = " << prob_opt.plot_int << std::endl 261 | << "arkode_order = " << prob_opt.arkode_order << std::endl; 262 | if (prob_opt.rhs_adv) 263 | amrex::Print() 264 | << "ImEx treatment (implicit diffusion, explicit advection)" << std::endl; 265 | else 266 | amrex::Print() 267 | << "fully implicit treatment" << std::endl; 268 | if (prob_opt.fixed_dt > 0.0) 269 | amrex::Print() 270 | << "fixed_dt = " << prob_opt.fixed_dt << std::endl; 271 | else 272 | amrex::Print() 273 | << "rtol = " << prob_opt.rtol << std::endl 274 | << "atol = " << prob_opt.atol << std::endl; 275 | amrex::Print() 276 | << "tfinal = " << prob_opt.tfinal << std::endl 277 | << "dtout = " << prob_opt.dtout << std::endl 278 | << "write_diag = " << prob_opt.write_diag << std::endl 279 | << "advCoeffx = " << prob_data.advCoeffx << std::endl 280 | << "advCoeffy = " << prob_data.advCoeffy << std::endl 281 | << "diffCoeffx = " << prob_data.diffCoeffx << std::endl 282 | << "diffCoeffy = " << prob_data.diffCoeffy << std::endl; 283 | amrex::Print() 284 | << "Newton nonlinear solver:" << std::endl 285 | << " max_iter = " << prob_opt.nls_max_iter << std::endl 286 | << " ls_max_iter = " << prob_opt.ls_max_iter << std::endl; 287 | return; 288 | } 289 | -------------------------------------------------------------------------------- /SUNDIALS+AMReX/README.md: -------------------------------------------------------------------------------- 1 | # SUNDIALS+AMReX Example Code for ATPESC 2 | 3 | ## Advection-Diffusion Example 4 | 5 | In this problem, we consider a scalar-valued advection-diffusion problem for 6 | chemical transport. The governing equation is 7 | 8 | $$ \frac{\partial u}{\partial t} + \vec{a} \cdot \nabla u - \nabla \cdot ( D \nabla u ) = 0 $$ 9 | 10 | where $u = u(t,x,y)$ is the chemical concentration, $\vec{a}$ is the advection 11 | vector, and $D$ is a diagonal matrix of diffusion coefficients. The problem is 12 | solved on the square domain $[-1, 1]^2$ with periodic boundary conditions and 13 | the initial condition is 14 | 15 | $$ u(0,x,y) = u_0(x,y) = \frac{10}{\sqrt{2\pi}} e^{-50 (x^2 + y^2)} $$ 16 | 17 | ## Building 18 | 19 | Building the programs in this repository requires at least CMake version 3.17, 20 | SUNDIALS version 6.0.0, and AMReX version 22.04. To build the hands-on lessions, 21 | you can use the standard CMake procedure to build the ATPESC hands-on lesson 22 | executables, `HandsOn1.exe`, `HandsOn2.exe`, and `HandsOn3.exe`. 23 | ``` 24 | mkdir build 25 | cd build 26 | cmake .. \ 27 | -DAMREX_ROOT= \ 28 | -DSUNDIALS_ROOT= 29 | make 30 | ``` 31 | If SUNDIALS and AMReX were installed with CUDA support enabled, the option 32 | `-DENABLE_CUDA=ON` must be provided to enable GPU offloading. If AMReX was built 33 | with hypre support enabled, the option `HYPRE_ROOT` should be set to the path to 34 | the hypre installation directory. 35 | 36 | ## Problem Options 37 | 38 | The full set of problem options and the default values are shown below. Note 39 | each hands-on lesson will only utilize an appropriate subset of these options 40 | for that lesson. Use the input `help=1` to list the options relevant to a 41 | particular setup. 42 | 43 | | Option | Type | Description | Default | 44 | |----------------------|--------|----------------------------------------------------|----------| 45 | | `help` | `int` | print input options and exit (1) | 0 | 46 | | `n_cell` | `int` | number of cells on each side of the square domain | 128 | 47 | | `max_grid_size` | `int` | max size of boxes in box array | 64 | 48 | | `plot_int` | `int` | enable (1) or disable (0) plots | 0 | 49 | | `arkode_order` | `int` | ARKStep method order | 4 | 50 | | `nls_max_iter` | `int` | maximum number of nonlinear iterations | 3 | 51 | | `ls_max_iter` | `int` | maximum number of linear iterations | 5 | 52 | | `rhs_adv` | `int` | advection: implicit (0) or explicit (1) | 1 | 53 | | `rtol` | `Real` | relative tolerance | 1e-4 | 54 | | `atol` | `Real` | absolute tolerance | 1e-9 | 55 | | `fixed_dt` | `Real` | use a fixed time step size (if `fixed_dt` > 0.0) | -1.0 | 56 | | `tfinal` | `Real` | final integration time | 1e4 | 57 | | `dtout` | `Real` | output frequency | `tfinal` | 58 | | `max_steps` | `int` | maximum number of steps between outputs | 10000 | 59 | | `write_diag` | `int` | output ARKStep diagnostics to a file | 1 | 60 | | `advCoeffx` | `Real` | advection speed in the x-direction | 5.0e-4 | 61 | | `advCoeffy` | `Real` | advection speed in the y-direction | 2.5e-4 | 62 | | `diffCoeffx` | `Real` | diffusion coefficient in the x-direction | 1.0e-6 | 63 | | `diffCoeffy` | `Real` | diffusion coefficient in the y-direction | 1.0e-6 | 64 | | `use_preconditioner` | `int` | use preconditioning (1) or not (0) | 0 | 65 | 66 | If preconditioning is enabled, then additional options may be set (see AMReX 67 | documentation of the `MLMG` solver for descriptions): 68 | 69 | | Option | Type | Default | 70 | |-----------------------------|--------|---------| 71 | | `mlmg.agglomeration` | `int` | 1 | 72 | | `mlmg.consolidation` | `int` | 1 | 73 | | `mlmg.max_coarsening_level` | `int` | 1000 | 74 | | `mlmg.linop_maxorder` | `int` | 2 | 75 | | `mlmg.max_iter` | `int` | 1000 | 76 | | `mlmg.max_fmg_iter` | `int` | 1000 | 77 | | `mlmg.mg_fixed_iter` | `int` | 0 | 78 | | `mlmg.verbose` | `int` | 0 | 79 | | `mlmg.bottom_verbose` | `int` | 0 | 80 | | `mlmg.use_hypre` | `int` | 1 | 81 | | `mlmg.hypre_interface` | `int` | 3 | 82 | | `mlmg.use_petsc` | `int` | 0 | 83 | | `mlmg.tol_rel` | `Real` | 1.0e-6 | 84 | | `mlmg.tol_abs` | `Real` | 1.0e-6 | 85 | -------------------------------------------------------------------------------- /SUNDIALS+AMReX/inputs: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------------------------ 2 | # Input parameters see the README for parameter descriptions and defaults 3 | # ------------------------------------------------------------------------------ 4 | n_cell = 128 5 | max_grid_size = 64 6 | plot_int = 1 7 | arkode_order = 4 8 | rhs_adv = 0 9 | rtol = 1.0e-4 10 | atol = 1.0e-9 11 | tfinal = 1.0e4 12 | dtout = 300.0 13 | advCoeffx = 5.0e-4 14 | advCoeffy = 2.5e-4 15 | diffCoeffx = 1.0e-6 16 | diffCoeffy = 1.0e-6 17 | use_preconditioner = 1 18 | 19 | 20 | # ------------------------------------------------------------------------------ 21 | # Input parameters for AMReX MLMG interface used as a preconditioner 22 | # ------------------------------------------------------------------------------ 23 | 24 | mlmg.agglomeration = 1 25 | mlmg.consolidation = 1 26 | mlmg.max_coarsening_level = 1000 27 | mlmg.linop_maxorder = 2 28 | mlmg.max_iter = 1000 29 | mlmg.max_fmg_iter = 1000 30 | mlmg.verbose = 0 31 | mlmg.bottom_verbose = 0 32 | mlmg.use_hypre = 1 33 | 34 | # 1 = Structed, 2 = Semi-Structed, 3 = IJ 35 | mlmg.hypre_interface = 3 36 | 37 | mlmg.use_petsc = 0 38 | mlmg.tol_rel = 1.0e-6 39 | -------------------------------------------------------------------------------- /SUNDIALS+AMReX/inputs-1: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------------------------ 2 | # Input parameters for HandsOn1.exe -- see README.md for parameter descriptions 3 | # and defaults 4 | # ------------------------------------------------------------------------------ 5 | plot_int = 1 6 | arkode_order = 4 7 | fixed_dt = 5.0 8 | tfinal = 1.0e4 9 | dtout = 1.0e4 10 | max_steps = 10000 11 | n_cell = 128 12 | max_grid_size = 64 13 | advCoeffx = 5.0e-4 14 | advCoeffy = 2.5e-4 15 | diffCoeffx = 1.0e-6 16 | diffCoeffy = 1.0e-6 17 | -------------------------------------------------------------------------------- /SUNDIALS+AMReX/inputs-2: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------------------------ 2 | # Input parameters for HandsOn2.exe -- see README.md for parameter descriptions 3 | # and defaults 4 | # ------------------------------------------------------------------------------ 5 | plot_int = 1 6 | arkode_order = 4 7 | nls_max_iter = 3 8 | ls_max_iter = 100 9 | rhs_adv = 0 10 | fixed_dt = 5.0 11 | tfinal = 1.0e4 12 | dtout = 1.0e4 13 | max_steps = 10000 14 | n_cell = 128 15 | max_grid_size = 64 16 | advCoeffx = 5.0e-4 17 | advCoeffy = 2.5e-4 18 | diffCoeffx = 1.0e-6 19 | diffCoeffy = 1.0e-6 20 | -------------------------------------------------------------------------------- /SUNDIALS+AMReX/inputs-3: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------------------------ 2 | # Input parameters for HandsOn3.exe -- see README.md for parameter descriptions 3 | # and defaults 4 | # ------------------------------------------------------------------------------ 5 | plot_int = 1 6 | arkode_order = 4 7 | nls_max_iter = 3 8 | ls_max_iter = 100 9 | rhs_adv = 1 10 | rtol = 1.0e-4 11 | atol = 1.0e-9 12 | tfinal = 1.0e4 13 | dtout = 1.0e4 14 | max_steps = 10000 15 | write_diag = 1 16 | use_preconditioner = 1 17 | n_cell = 128 18 | max_grid_size = 64 19 | advCoeffx = 5.0e-4 20 | advCoeffy = 2.5e-4 21 | diffCoeffx = 1.0e-6 22 | diffCoeffy = 1.0e-6 23 | 24 | mlmg.agglomeration = 1 25 | mlmg.consolidation = 1 26 | mlmg.max_coarsening_level = 30 27 | mlmg.linop_maxorder = 2 28 | mlmg.max_iter = 100 29 | mlmg.max_fmg_iter = 0 30 | mlmg.verbose = 0 31 | mlmg.bottom_verbose = 0 32 | mlmg.use_hypre = 0 33 | mlmg.hypre_interface = 3 34 | mlmg.use_petsc = 0 35 | mlmg.tol_rel = 1.0e-4 36 | mlmg.fixed_iter = 1 37 | -------------------------------------------------------------------------------- /SUNDIALS+AMReX/inputs-ref: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------------------------ 2 | # Input parameters for reference solution (HandsOn2.exe or HandsOn3.exe) -- see 3 | # README.md for parameter descriptions and defaults e.g., on Polaris we use 4 | # mpiexec -n 4 ./HandsOn2.CUDA.exe inputs-ref 5 | # ------------------------------------------------------------------------------ 6 | plot_int = 1 7 | arkode_order = 5 8 | rtol = 1.0e-10 9 | atol = 1.0e-14 10 | tfinal = 1.0e4 11 | dtout = 1.0e4 12 | max_steps = 1000000 13 | n_cell = 128 14 | max_grid_size = 64 15 | advCoeffx = 5.0e-4 16 | advCoeffy = 2.5e-4 17 | diffCoeffx = 1.0e-6 18 | diffCoeffy = 1.0e-6 19 | nls_max_iter = 10 20 | ls_max_iter = 100 21 | rhs_adv = 0 22 | -------------------------------------------------------------------------------- /SUNDIALS+AMReX/process_ARKStep_diags.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | #------------------------------------------------------------ 3 | # Programmer(s): Daniel R. Reynolds @ SMU 4 | #------------------------------------------------------------ 5 | # Copyright (c) 2019, Southern Methodist University. 6 | # All rights reserved. 7 | # For details, see the LICENSE file. 8 | #------------------------------------------------------------ 9 | # analysis script for ARKODE solver statistics 10 | 11 | # imports 12 | import sys 13 | import diag_tools as diags 14 | 15 | # output error if no diagnostics output file is provided 16 | if (len(sys.argv) == 1): 17 | sys.exit("\nCalling syntax error: filename containing diagnostics must be provided, e.g.\n $ process_ARKStep_diags.py fname.txt\n") 18 | 19 | # load the diagnostics output file (passed on the command line) 20 | fname = sys.argv[len(sys.argv)-1] 21 | 22 | # load the time step data 23 | Tdiags = diags.load_diags(fname) 24 | 25 | # generate plots 26 | print('\nGenerating plots h_vs_t.png, h_vs_iter.png, and oversolve_vs_t.png') 27 | diags.plot_h_vs_t(Tdiags,'h_vs_t.png') 28 | diags.plot_h_vs_tstep(Tdiags,'h_vs_iter.png') 29 | diags.plot_oversolve_vs_t(Tdiags,'oversolve_vs_t.png') 30 | 31 | # print solve statistics to screen 32 | diags.etest_stats(Tdiags, sys.stdout) 33 | 34 | ##### end of script ##### 35 | -------------------------------------------------------------------------------- /SUNDIALS+AMReX/reference_solution/Header: -------------------------------------------------------------------------------- 1 | HyperCLaw-V1.1 2 | 1 3 | u 4 | 2 5 | 10000 6 | 0 7 | -1 -1 8 | 1 1 9 | 10 | ((0,0) (127,127) (0,0)) 11 | 1 12 | 0.015625 0.015625 13 | 0 14 | 0 15 | 0 4 10000 16 | 1 17 | -1 0 18 | -1 0 19 | 0 1 20 | -1 0 21 | -1 0 22 | 0 1 23 | 0 1 24 | 0 1 25 | Level_0/Cell 26 | -------------------------------------------------------------------------------- /SUNDIALS+AMReX/reference_solution/Level_0/Cell_D_00000: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AMReX-Codes/ATPESC-codes/2f5da61d159822df3c74f75035367feadfa399e3/SUNDIALS+AMReX/reference_solution/Level_0/Cell_D_00000 -------------------------------------------------------------------------------- /SUNDIALS+AMReX/reference_solution/Level_0/Cell_D_00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AMReX-Codes/ATPESC-codes/2f5da61d159822df3c74f75035367feadfa399e3/SUNDIALS+AMReX/reference_solution/Level_0/Cell_D_00001 -------------------------------------------------------------------------------- /SUNDIALS+AMReX/reference_solution/Level_0/Cell_D_00002: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AMReX-Codes/ATPESC-codes/2f5da61d159822df3c74f75035367feadfa399e3/SUNDIALS+AMReX/reference_solution/Level_0/Cell_D_00002 -------------------------------------------------------------------------------- /SUNDIALS+AMReX/reference_solution/Level_0/Cell_D_00003: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AMReX-Codes/ATPESC-codes/2f5da61d159822df3c74f75035367feadfa399e3/SUNDIALS+AMReX/reference_solution/Level_0/Cell_D_00003 -------------------------------------------------------------------------------- /SUNDIALS+AMReX/reference_solution/Level_0/Cell_H: -------------------------------------------------------------------------------- 1 | 1 2 | 1 3 | 1 4 | 0 5 | (4 0 6 | ((0,0) (63,63) (0,0)) 7 | ((64,0) (127,63) (0,0)) 8 | ((0,64) (63,127) (0,0)) 9 | ((64,64) (127,127) (0,0)) 10 | ) 11 | 4 12 | FabOnDisk: Cell_D_00000 0 13 | FabOnDisk: Cell_D_00001 0 14 | FabOnDisk: Cell_D_00002 0 15 | FabOnDisk: Cell_D_00003 0 16 | 17 | 4,1 18 | 1.2827437405102236e-05, 19 | 1.2501582345868270e-05, 20 | 1.4128284652038571e-03, 21 | 1.3769385745040692e-03, 22 | 23 | 4,1 24 | 7.6686132392254136e-02, 25 | 7.6626187732608347e-02, 26 | 4.5484401538439850e-01, 27 | 4.5448847144086912e-01, 28 | 29 | -------------------------------------------------------------------------------- /SUNDIALS+AMReX/source_cooley_plotfile_tools.sh: -------------------------------------------------------------------------------- 1 | ../source_cooley_plotfile_tools.sh -------------------------------------------------------------------------------- /TAO-of-AMReX/AMReX_BCUtil.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "MyTest.H" 4 | 5 | namespace amrex 6 | { 7 | 8 | namespace 9 | { 10 | 11 | void fill_extdir(Box const &bx, Array4 const &dest, 12 | const int dcomp, const int numcomp, 13 | GeometryData const &geom, const Real time, 14 | const BCRec *bcr, const int bcomp, 15 | const int orig_comp) 16 | { 17 | const auto &domain_box = geom.Domain(); 18 | const auto &domain_lo = amrex::lbound(domain_box); 19 | const auto &domain_hi = amrex::ubound(domain_box); 20 | 21 | const auto box_lo = amrex::lbound(bx); 22 | const auto box_hi = amrex::ubound(bx); 23 | 24 | // fill left edge 25 | const int k = 0; 26 | 27 | auto key = make_box_corner_tuple(bx); 28 | auto &vvr = ExtTaoBC::ext_dir_bcs[key]; 29 | 30 | 31 | bool aligned_lower = false; 32 | bool aligned_left = false; 33 | bool aligned_upper = false; 34 | 35 | int ivec_lower = 0; 36 | int ivec_left = 0; 37 | int ivec_upper = 0; 38 | 39 | auto& vbc_lower = vvr(ExtTaoBC::lower_boundary); 40 | auto& vbc_left = vvr(ExtTaoBC::left_boundary); 41 | auto& vbc_upper = vvr(ExtTaoBC::upper_boundary); 42 | 43 | /* 44 | Print() << "domain lo = " << domain_lo.x << " " << domain_lo.y << " " << domain_lo.z << "\n"; 45 | Print() << "domain hi = " << domain_hi.x << " " << domain_hi.y << " " << domain_hi.z << "\n"; 46 | Print() << "box lo = " << box_lo.x << " " << box_lo.y << " " << box_lo.z << "\n"; 47 | Print() << "box hi = " << box_hi.x << " " << box_hi.y << " " << box_hi.z << "\n"; 48 | Print() << "lower size = " << vbc_lower.size() << "\n"; 49 | Print() << "left size = " << vbc_left.size() << "\n"; 50 | Print() << "upper size = " << vbc_upper.size() << "\n"; 51 | */ 52 | 53 | // Lower boundary 54 | if (box_lo.y < domain_lo.y) 55 | { 56 | // Print() << "aligned lower\n"; 57 | aligned_lower = true; 58 | const int j = box_lo.y; 59 | for (int i = box_lo.x + 1; i < box_hi.x; ++i) 60 | { 61 | // Print() << "i = " << i << "\n"; 62 | // Print() << "ivec_lower = " << ivec_lower << "\n"; 63 | dest(i, j, k, dcomp) = vbc_lower(ivec_lower); ivec_lower++; 64 | } 65 | } 66 | 67 | // Left boundary 68 | if (box_lo.x < domain_lo.x) 69 | { 70 | // Print() << "aligned left\n"; 71 | aligned_left = true; 72 | const int i = box_lo.x; 73 | for (int j = box_lo.y + 1; j < box_hi.y; ++j) 74 | { 75 | // Print() << "ivec_left = " << ivec_left << "\n"; 76 | dest(i, j, k, dcomp) = vbc_left(ivec_left); ivec_left++; 77 | } 78 | } 79 | 80 | // Upper boundary 81 | if (box_hi.y > domain_hi.y) 82 | { 83 | // Print() << "aligned upper\n"; 84 | aligned_upper = true; 85 | const int j = box_hi.y; 86 | for (int i = box_lo.x + 1; i < box_hi.x; ++i) 87 | { 88 | // Print() << "ivec_upper = " << ivec_upper << "\n"; 89 | dest(i, j, k, dcomp) = vbc_upper(ivec_upper); ivec_upper++; 90 | } 91 | } 92 | 93 | // Left Lower corner 94 | if (aligned_left && aligned_lower) { 95 | dest(box_lo.x, box_lo.y, k) = vbc_left(ivec_left); ivec_left++; 96 | } 97 | 98 | // Left Upper corner 99 | if (aligned_left && aligned_upper) { 100 | dest(box_lo.x, box_hi.y, k) = vbc_left(ivec_left); ivec_left++; 101 | } 102 | } 103 | } // namespace 104 | 105 | void FillDomainBoundary(MultiFab &phi, const Geometry &geom, const Vector &bc) 106 | { 107 | if (geom.isAllPeriodic()) 108 | return; 109 | if (phi.nGrow() == 0) 110 | return; 111 | 112 | AMREX_ALWAYS_ASSERT(phi.ixType().cellCentered()); 113 | 114 | CpuBndryFuncFab cpu_bndry_func(fill_extdir); 115 | PhysBCFunct physbcf(geom, bc, cpu_bndry_func); 116 | physbcf.FillBoundary(phi, 0, phi.nComp(), 0.0, 0); 117 | } 118 | 119 | } // namespace amrex 120 | -------------------------------------------------------------------------------- /TAO-of-AMReX/MyTest.H: -------------------------------------------------------------------------------- 1 | #ifndef MY_TEST_H_ 2 | #define MY_TEST_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #ifdef AMREX_USE_HYPRE 9 | #include 10 | #endif 11 | 12 | struct EdgeBoundaryValues 13 | { 14 | amrex::Vector x; 15 | 16 | EdgeBoundaryValues() { x.resize(0); } 17 | 18 | amrex::Real& operator()(int i) { return x[static_cast(i)]; } 19 | 20 | void resize(int n) { x.resize(n); } 21 | 22 | int size() { return x.size(); } 23 | 24 | void push_back(amrex::Real xi) { x.push_back(xi); } 25 | }; 26 | 27 | struct BoxBoundaryValues 28 | { 29 | amrex::Vector x; 30 | 31 | BoxBoundaryValues() { x.resize(0); } 32 | 33 | EdgeBoundaryValues& operator()(int i) { return x[i]; } 34 | 35 | amrex::Real& operator()(int i, int j) { return x[i](j); } 36 | 37 | void resize(int n) { x.resize(n); } 38 | 39 | int size() { return x.size(); } 40 | 41 | void push_back(EdgeBoundaryValues xi) { x.push_back(xi); } 42 | }; 43 | 44 | typedef std::tuple BoxCornerTuple; 45 | 46 | BoxCornerTuple make_box_corner_tuple(const amrex::Box& bx); 47 | 48 | namespace ExtTaoBC 49 | { 50 | extern std::map ext_dir_bcs; 51 | 52 | const int lower_boundary = 0; 53 | const int left_boundary = 1; 54 | const int upper_boundary = 2; 55 | } 56 | 57 | class MyTest 58 | { 59 | public: 60 | 61 | MyTest (); 62 | void initData (); 63 | void solve (); 64 | void writePlotfile (); 65 | void writeMinimalPlotfile (); 66 | int n_cell = 128; 67 | int nb, nl, nt; 68 | int Nb, Nl, Nt; 69 | bool fd_grad = false; 70 | bool plot = false; 71 | 72 | public: // make these public for cuda 73 | void initProbPoisson (); 74 | void initProbABecLaplacian (); 75 | void get_number_global_bcs(int& num_lower, int& num_left, int& num_upper); 76 | void get_number_local_bcs(int& num_lower, int& num_left, int& num_upper); 77 | void set_target_solution(amrex::Real (*ftarget)(amrex::Real* coords)); 78 | void update_boundary_values(int nb, const amrex::Real *xb, int nl, const amrex::Real *xl, int nt, const amrex::Real *xt); 79 | 80 | amrex::Real calculate_obj_val(); 81 | void calculate_opt_gradient(int nlower, amrex::Real* glower, 82 | int nleft, amrex::Real* gleft, 83 | int nupper, amrex::Real* gupper); 84 | void setup_adjoint_system(); 85 | void solve_adjoint_system(); 86 | void write_plotfile(bool minimal=false); 87 | void update_counter(); 88 | 89 | private: 90 | 91 | void readParameters (); 92 | void solvePoisson(amrex::MultiFab &solution, amrex::MultiFab &rhs); 93 | void solveABecLaplacian (); 94 | std::string get_iteration_filename(std::string filename); 95 | void update_target_solution(); 96 | 97 | int max_level = 1; 98 | int ref_ratio = 2; 99 | int max_grid_size = 64; 100 | const int ngrow = 1; // number of ghost cells for solution 101 | 102 | bool composite_solve = true; 103 | 104 | int prob_type = 1; // 1. Poisson, 2. ABecLaplacian 105 | 106 | int iteration_counter = 0; 107 | 108 | // For MLMG solver 109 | int verbose = 2; 110 | int bottom_verbose = 0; 111 | int max_iter = 100; 112 | int max_fmg_iter = 0; 113 | int linop_maxorder = 2; 114 | bool agglomeration = true; 115 | bool consolidation = true; 116 | int max_coarsening_level = 30; 117 | bool use_hypre = false; 118 | bool use_petsc = false; 119 | amrex::Real mlmg_tol_rel = 1.0e-12; 120 | amrex::Real mlmg_tol_abs = 1.0e-6; 121 | 122 | #ifdef AMREX_USE_HYPRE 123 | int hypre_interface_i = 1; // 1. structed, 2. semi-structed, 3. ij 124 | amrex::Hypre::Interface hypre_interface = amrex::Hypre::Interface::structed; 125 | #endif 126 | 127 | amrex::BCRec bcs; // BCs for 1 component 128 | 129 | amrex::Geometry geom; 130 | amrex::BoxArray grids; 131 | amrex::DistributionMapping dmap; 132 | 133 | amrex::MultiFab solution; 134 | amrex::MultiFab rhs; 135 | amrex::MultiFab adjoint; 136 | amrex::MultiFab adjoint_rhs; 137 | amrex::MultiFab exact_solution; 138 | amrex::MultiFab acoef; 139 | amrex::MultiFab bcoef; 140 | 141 | amrex::Real ascalar = 1.e-3; 142 | amrex::Real bscalar = 1.0; 143 | 144 | amrex::Real (*target_function)(amrex::Real* physical_coordinates); 145 | }; 146 | 147 | #endif 148 | -------------------------------------------------------------------------------- /TAO-of-AMReX/MyTestPlotfile.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include "MyTest.H" 3 | #include 4 | 5 | using namespace amrex; 6 | 7 | void 8 | MyTest::writePlotfile () 9 | { 10 | const int ncomp = 6; 11 | Vector varname = {"solution", "rhs", "exact_solution", "error", "adjoint", "adjoint_rhs"}; 12 | 13 | const int nlevels = max_level+1; 14 | 15 | MultiFab plotmf; 16 | for (int ilev = 0; ilev < nlevels; ++ilev) 17 | { 18 | plotmf.define(grids, dmap, ncomp, 0); 19 | MultiFab::Copy(plotmf, solution , 0, 0, 1, 0); 20 | MultiFab::Copy(plotmf, rhs , 0, 1, 1, 0); 21 | MultiFab::Copy(plotmf, exact_solution, 0, 2, 1, 0); 22 | MultiFab::Copy(plotmf, solution , 0, 3, 1, 0); 23 | MultiFab::Copy(plotmf, adjoint , 0, 4, 1, 0); 24 | MultiFab::Copy(plotmf, adjoint_rhs , 0, 5, 1, 0); 25 | MultiFab::Subtract(plotmf, plotmf, 2, 3, 1, 0); // error = soln - exact 26 | } 27 | 28 | std::string filename = get_iteration_filename("plt"); 29 | WriteSingleLevelPlotfile(filename, plotmf, varname, geom, 0.0, 1); 30 | } 31 | 32 | void 33 | MyTest::writeMinimalPlotfile () 34 | { 35 | const int ncomp = 2; 36 | Vector varname = {"solution", "rhs"}; 37 | 38 | const int nlevels = max_level+1; 39 | 40 | MultiFab plotmf; 41 | for (int ilev = 0; ilev < nlevels; ++ilev) 42 | { 43 | plotmf.define(grids, dmap, ncomp, 0); 44 | MultiFab::Copy(plotmf, solution , 0, 0, 1, 0); 45 | MultiFab::Copy(plotmf, rhs , 0, 1, 1, 0); 46 | } 47 | 48 | std::string filename = get_iteration_filename("plt"); 49 | WriteSingleLevelPlotfile(filename, plotmf, varname, geom, 0.0, 1); 50 | } 51 | 52 | -------------------------------------------------------------------------------- /TAO-of-AMReX/README.md: -------------------------------------------------------------------------------- 1 | # Boundary Control with 2-D Laplace Equation 2 | 3 | ## Introduction 4 | 5 | ATPESC 2019 Numerical Software Track 6 | 7 | [__Hands-on Lesson Page__](https://xsdk-project.github.io/MathPackagesTraining/lessons/boundary_control_tao/) 8 | 9 | __Developed by:__ 10 | 11 | * Alp Dener, Postdoctoral Researcher, Argonne National Laboratory 12 | 13 | * Donald E. Willcox, Postdoctoral Researcher. Lawrence-Livermore National Laboratory 14 | 15 | ## Dependencies 16 | 17 | 1. [PETSc/TAO](https://www.mcs.anl.gov/petsc/) 18 | - Must be configured with MPI 19 | - `PETSC_DIR` and `PETSC_ARCH` environment variables must be set 20 | 21 | 2. [AMReX](https://amrex-codes.github.io/amrex/) 22 | - Must be compiled as a library with `DIM=2` (2-D configuration) 23 | - Modify problem makefile to provide the correct `ATPESC_HOME` directory 24 | 25 | 3. [MPICH](https://www.mpich.org) or [OpenMPI](https://www.open-mpi.org) 26 | - PETSc can provide an MPICH installation if configured with `--download-mpich` 27 | 28 | -------------------------------------------------------------------------------- /TAO-of-AMReX/boundary_control.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | using namespace std::chrono; 3 | 4 | #include 5 | #include "MyTest.H" 6 | 7 | #include 8 | 9 | amrex::Real TargetSolution(amrex::Real* coords); 10 | PetscErrorCode FormFunction(Tao tao, Vec P, PetscReal *f, void *ptr); 11 | PetscErrorCode FormFunctionGradient(Tao tao, Vec P, PetscReal *f, Vec G, void *ptr); 12 | 13 | int main (int argc, char* argv[]) 14 | { 15 | PetscErrorCode ierr; 16 | PetscReal one = 1.0; 17 | int nb, nl, nt, n; 18 | int Nb, Nl, Nt, N; 19 | int no_args = 2; 20 | Vec P; 21 | Tao tao; 22 | PetscBool flg, fd=PETSC_FALSE, plot=PETSC_FALSE; 23 | PetscMPIInt rank; 24 | 25 | amrex::Initialize(no_args, argv); 26 | ierr = PetscInitialize(&argc, &argv, (char*)0, (char*)0); if (ierr) return ierr; 27 | ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank);CHKERRQ(ierr); 28 | 29 | { 30 | BL_PROFILE("main"); 31 | MyTest mytest; 32 | 33 | // Get domain sizing and gradient type from command line options 34 | ierr = PetscOptionsGetInt(NULL,NULL,"-nx",&mytest.n_cell,&flg);CHKERRQ(ierr); 35 | ierr = PetscOptionsGetBool(NULL,NULL,"-fd",&fd,&flg);CHKERRQ(ierr); 36 | if (fd) mytest.fd_grad = true; 37 | ierr = PetscOptionsGetBool(NULL,NULL,"-plot",&plot,&flg);CHKERRQ(ierr); 38 | if (plot) mytest.plot = true; 39 | 40 | // Prep the solver and set the target solution we want to recover on the right edge of the domain 41 | // u_target = 4.0*(y-0.5)^2; 42 | mytest.initData(); 43 | mytest.set_target_solution(TargetSolution); 44 | 45 | // Create PETSc vector for the bottom, left and top Dirichlet boundaries 46 | mytest.get_number_local_bcs(mytest.nb, mytest.nl, mytest.nt); 47 | n = mytest.nb + mytest.nl + mytest.nt; // total local size 48 | mytest.get_number_global_bcs(mytest.Nb, mytest.Nl, mytest.Nt); 49 | N = mytest.Nb + mytest.Nl + mytest.Nt; // total global size 50 | ierr = VecCreateMPI(PETSC_COMM_WORLD, n, N, &P);CHKERRQ(ierr); 51 | ierr = VecSet(P, one); CHKERRQ(ierr); 52 | 53 | // Create and setup the TAO optimization algorithm 54 | ierr = TaoCreate(PETSC_COMM_WORLD, &tao); CHKERRQ(ierr); 55 | ierr = TaoSetType(tao, TAOBQNLS); CHKERRQ(ierr); // TAOBQNLS is a bound-constrained quasi-Newton linesearch alg, 56 | ierr = TaoSetInitialVector(tao, P); CHKERRQ(ierr); 57 | if (!mytest.fd_grad) { 58 | // Sets the callback for the gradient computed with the adjoint method 59 | ierr = TaoSetObjectiveAndGradientRoutine(tao, FormFunctionGradient, &mytest); CHKERRQ(ierr); 60 | } else { 61 | // TAO provides a default gradient function for the central finite-difference formula 62 | ierr = TaoSetObjectiveRoutine(tao, FormFunction, &mytest);CHKERRQ(ierr); 63 | ierr = TaoSetGradientRoutine(tao, TaoDefaultComputeGradient, &mytest); CHKERRQ(ierr); 64 | } 65 | 66 | // This lets TAO read command line options 67 | ierr = TaoSetFromOptions(tao); CHKERRQ(ierr); 68 | 69 | // Start the optimization solution 70 | auto start = high_resolution_clock::now(); 71 | ierr = TaoSolve(tao); CHKERRQ(ierr); 72 | auto stop = high_resolution_clock::now(); 73 | 74 | auto duration = duration_cast(stop - start); 75 | if (rank == 0) std::cout << "TaoSolve() duration: " << duration.count() << " microseconds" << std::endl; 76 | } 77 | 78 | // Cleanup and exit 79 | ierr = TaoDestroy(&tao);CHKERRQ(ierr); 80 | ierr = VecDestroy(&P);CHKERRQ(ierr); 81 | ierr = PetscFinalize(); 82 | amrex::Finalize(); 83 | } 84 | 85 | /* -------------------------------------------------------------------- */ 86 | 87 | amrex::Real TargetSolution(amrex::Real* coords) 88 | { 89 | amrex::Real y = coords[1]; 90 | amrex::Real utarg = 4.*(y-0.5)*(y-0.5) - 0.5; 91 | return utarg; 92 | } 93 | 94 | /* -------------------------------------------------------------------- */ 95 | 96 | // This function computes only the objective value 97 | PetscErrorCode FormFunction(Tao tao, Vec P, PetscReal *f, void *ptr) 98 | { 99 | MyTest *mytest = (MyTest *) ptr; 100 | PetscErrorCode ierr; 101 | PetscReal ff=0; 102 | const PetscScalar *pp; 103 | amrex::Real *pb, *pl, *pt; 104 | 105 | PetscFunctionBeginUser; 106 | 107 | // Allocate arrays for bottom, left and top Dirichlet boundaries 108 | ierr = PetscMalloc1(mytest->nb * sizeof(amrex::Real), &pb);CHKERRQ(ierr); 109 | ierr = PetscMalloc1(mytest->nl * sizeof(amrex::Real), &pl);CHKERRQ(ierr); 110 | ierr = PetscMalloc1(mytest->nt * sizeof(amrex::Real), &pt);CHKERRQ(ierr); 111 | 112 | // Split the PETSc vector of optimization variables into its components 113 | ierr = VecGetArrayRead(P, &pp);CHKERRQ(ierr); 114 | for (int i=0; inb; i++) pb[i] = pp[i]; 115 | for (int i=0; inl; i++) pl[i] = pp[mytest->nb + i]; 116 | for (int i=0; int; i++) pt[i] = pp[mytest->nb + mytest->nl + i]; 117 | ierr = VecRestoreArrayRead(P, &pp);CHKERRQ(ierr); 118 | 119 | // Set the boundary values from TAO into the AMReX solver 120 | mytest->update_boundary_values(mytest->nb, pb, mytest->nl, pl, mytest->nt, pt); 121 | 122 | // Clean-up 123 | ierr = PetscFree(pb);CHKERRQ(ierr); 124 | ierr = PetscFree(pl);CHKERRQ(ierr); 125 | ierr = PetscFree(pt);CHKERRQ(ierr); 126 | 127 | /// Solve the Laplace equations with the prescribed boundary values 128 | mytest->solve(); 129 | 130 | // Compute the objective function using the AMReX solution 131 | // f = 0.5 * \int_0^1 (u(1, y) - u_targ)^2 dy 132 | ff = mytest->calculate_obj_val(); 133 | *f = ff; 134 | 135 | if (mytest->fd_grad && mytest->plot) { 136 | // Perform other misc operations like visualization and I/O 137 | mytest->write_plotfile(); 138 | mytest->update_counter(); 139 | } 140 | 141 | PetscFunctionReturn(0); 142 | } 143 | 144 | /* -------------------------------------------------------------------- */ 145 | 146 | PetscErrorCode FormFunctionGradient(Tao tao, Vec P, PetscReal *f, Vec G, void *ptr) 147 | { 148 | MyTest *mytest = (MyTest *) ptr; 149 | PetscErrorCode ierr; 150 | amrex::Real *gb, *gl, *gt; 151 | PetscScalar *gg; 152 | 153 | PetscFunctionBeginUser; 154 | 155 | // Evaluate the objective function 156 | ierr = FormFunction(tao, P, f, ptr);CHKERRQ(ierr); 157 | 158 | // Allocate arrays for bottom, left and top Dirichlet boundaries 159 | ierr = PetscMalloc1(mytest->nb * sizeof(amrex::Real), &gb);CHKERRQ(ierr); 160 | ierr = PetscMalloc1(mytest->nl * sizeof(amrex::Real), &gl);CHKERRQ(ierr); 161 | ierr = PetscMalloc1(mytest->nt * sizeof(amrex::Real), >);CHKERRQ(ierr); 162 | 163 | // Solve the adjoint problem and assemble the gradient for each boundary 164 | mytest->setup_adjoint_system(); 165 | mytest->solve_adjoint_system(); 166 | mytest->calculate_opt_gradient(mytest->nb, gb, mytest->nl, gl, mytest->nt, gt); 167 | 168 | // Write the AMReX gradient data into the PETSc vector 169 | ierr = VecGetArray(G, &gg);CHKERRQ(ierr); 170 | for (int i=0; inb; i++) gg[i] = gb[i]; 171 | for (int i=0; inl; i++) gg[mytest->nb + i] = gl[i]; 172 | for (int i=0; int; i++) gg[mytest->nb + mytest->nl + i] = gt[i]; 173 | ierr = VecRestoreArray(G, &gg);CHKERRQ(ierr); 174 | ierr = VecScale(G, mytest->n_cell*mytest->n_cell);CHKERRQ(ierr); 175 | 176 | // Clean-up 177 | ierr = PetscFree(gb);CHKERRQ(ierr); 178 | ierr = PetscFree(gl);CHKERRQ(ierr); 179 | ierr = PetscFree(gt);CHKERRQ(ierr); 180 | 181 | // Perform other misc operations like visualization and I/O 182 | if (mytest->plot) { 183 | mytest->write_plotfile(); 184 | mytest->update_counter(); 185 | } 186 | 187 | PetscFunctionReturn(0); 188 | } 189 | -------------------------------------------------------------------------------- /TAO-of-AMReX/initProb.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include "MyTest.H" 3 | #include "initProb_K.H" 4 | 5 | using namespace amrex; 6 | 7 | void 8 | MyTest::initProbPoisson () 9 | { 10 | for (int ilev = 0; ilev <= max_level; ++ilev) 11 | { 12 | const auto prob_lo = geom.ProbLoArray(); 13 | const auto dx = geom.CellSizeArray(); 14 | #ifdef _OPENMP 15 | #pragma omp parallel if (Gpu::notInLaunchRegion()) 16 | #endif 17 | for (MFIter mfi(rhs, TilingIfNotGPU()); mfi.isValid(); ++mfi) 18 | { 19 | const Box& bx = mfi.tilebox(); 20 | auto rhsfab = rhs.array(mfi); 21 | auto exactfab = exact_solution.array(mfi); 22 | AMREX_PARALLEL_FOR_3D ( bx, i, j, k, 23 | { 24 | actual_init_poisson(i,j,k,rhsfab,exactfab,prob_lo,dx); 25 | }); 26 | } 27 | 28 | solution.setVal(0.0); 29 | } 30 | } 31 | 32 | -------------------------------------------------------------------------------- /TAO-of-AMReX/initProb_K.H: -------------------------------------------------------------------------------- 1 | #ifndef INIT_PROB_K_H_ 2 | #define INIT_PROB_K_H_ 3 | 4 | #include 5 | 6 | AMREX_GPU_DEVICE AMREX_INLINE 7 | void actual_init_poisson (int i, int j, int k, 8 | amrex::Array4 const& rhs, 9 | amrex::Array4 const& exact, 10 | amrex::GpuArray const& prob_lo, 11 | amrex::GpuArray const& dx) 12 | { 13 | constexpr amrex::Real tpi = 2.*3.1415926535897932; 14 | constexpr amrex::Real fpi = 4.*3.1415926535897932; 15 | constexpr amrex::Real fac = tpi*tpi*3.; 16 | amrex::Real x = prob_lo[0] + dx[0] * (i + 0.5); 17 | amrex::Real y = prob_lo[1] + dx[1] * (j + 0.5); 18 | amrex::Real z = prob_lo[2] + dx[2] * (k + 0.5); 19 | 20 | exact(i,j,k) = (std::sin(tpi*x) * std::sin(tpi*y) * std::sin(tpi*z)) 21 | + .25 * (std::sin(fpi*x) * std::sin(fpi*y) * std::sin(fpi*z)); 22 | 23 | rhs(i,j,k) = -fac * (std::sin(tpi*x) * std::sin(tpi*y) * std::sin(tpi*z)) 24 | -fac * (std::sin(fpi*x) * std::sin(fpi*y) * std::sin(fpi*z)); 25 | } 26 | 27 | #endif 28 | -------------------------------------------------------------------------------- /TAO-of-AMReX/inputs: -------------------------------------------------------------------------------- 1 | 2 | max_level = 1 3 | ref_ratio = 2 4 | n_cell = 128 5 | max_grid_size = 64 6 | composite_solve = 1 7 | 8 | # For MLMG 9 | verbose = 0 10 | bottom_verbose = 0 11 | max_iter = 100 12 | max_fmg_iter = 0 # # of F-cycles before switching to V. To do pure V-cycle, set to 0 13 | linop_maxorder = 2 14 | agglomeration = 1 # Do agglomeration on AMR Level 0? 15 | consolidation = 1 # Do consolidation? 16 | mlmg_tol_rel = 1.0e-12 17 | mlmg_tol_abs = 1.0e-10 18 | -------------------------------------------------------------------------------- /TAO-of-AMReX/makefile: -------------------------------------------------------------------------------- 1 | ALL: boundary_control 2 | 3 | SPACK_PKGS = /home/adener/FASTMath/spack/opt/spack/linux-rhel7-x86_64/gcc-4.8.5 4 | AMREX_HOME = ${SPACK_PKGS}/amrex-develop-fvkknjb363aqxpt6zu2tt6oojixnlnex 5 | 6 | CFLAGS = 7 | FFLAGS = 8 | CPPFLAGS = -I${AMREX_HOME}/include 9 | FPPFLAGS = 10 | CLEANFILES = boundary_control 11 | PETSC_DIR = ${SPACK_PKGS}/petsc-develop-muikvcmmgnpc2r7socr77wu55mcmhmxi 12 | 13 | include ${PETSC_DIR}/lib/petsc/conf/variables 14 | include ${PETSC_DIR}/lib/petsc/conf/rules 15 | include ${PETSC_DIR}/lib/petsc/conf/test 16 | 17 | LDFLAGS += -L${AMREX_HOME}/lib 18 | LDLIBS += -lamrex 19 | 20 | boundary_control: boundary_control.o MyTest.o MyTestPlotfile.o initProb.o AMReX_BCUtil.o -------------------------------------------------------------------------------- /amrex-plotfile-example/GNUmakefile: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------------------------ 2 | # Makefile to build examples 3 | # ------------------------------------------------------------------------------ 4 | # Note the following environment variables need to be set to in order to build: 5 | # AMREX_INSTALL_DIR = path to AMReX installation 6 | # MPICXX = mpicxx wrapper 7 | # 8 | # If any are unset, they assume the default values for compilation on Cooley 9 | # ------------------------------------------------------------------------------ 10 | 11 | # set default values for any unset environment variables 12 | ifeq ($(AMREX_INSTALL_DIR),) 13 | AMREX_INSTALL_DIR = /projects/ATPESC2019/FASTMath/spack/opt/spack/linux-rhel7-x86_64/gcc-4.8.5/amrex-develop-fvkknjb363aqxpt6zu2tt6oojixnlnex 14 | endif 15 | ifeq ($(MPICXX),) 16 | MPICXX = mpicxx 17 | endif 18 | 19 | CPPFLAGS = -Ishared -I$(AMREX_INSTALL_DIR)/include 20 | CXXFLAGS = -O2 -std=c++11 21 | FFLAGS = -O2 22 | LDFLAGS = -L$(AMREX_INSTALL_DIR)/lib -Wl,-rpath,$(AMREX_INSTALL_DIR)/lib 23 | 24 | LIBRARIES = -lamrex 25 | 26 | LIBRARIES += -lgfortran 27 | 28 | main.exe: main.cpp 29 | $(MPICXX) -o $@ $(CXXFLAGS) $(CPPFLAGS) $^ $(LDFLAGS) $(LIBRARIES) 30 | 31 | .PHONY: movie clean realclean pltclean 32 | 33 | movie: 34 | ls -1 plt*/Header | tee movie.visit 35 | 36 | clean: 37 | $(RM) *.o 38 | 39 | realclean: clean 40 | $(RM) *~ *.exe 41 | 42 | pltclean: 43 | $(RM) -rf plt*/ *.png *_diagnostics.txt __pycache__ 44 | -------------------------------------------------------------------------------- /amrex-plotfile-example/README.md: -------------------------------------------------------------------------------- 1 | # AMReX Plotfile Example 2 | 3 | This sample code initializes a MultiFab with one component "phi" using 4 | the `(i,j,k)` indices of each cell as: 5 | 6 | ``` 7 | phi(i,j,k) = i + 100.0*j + 10000.0*k 8 | ``` 9 | 10 | The MultiFab is then written to a plotfile. 11 | 12 | ## Compiling 13 | 14 | First, compile AMReX as a library using the directions here: 15 | https://amrex-codes.github.io/amrex/docs_html/BuildingAMReX.html#building-libamrex 16 | 17 | This was tested with g++ 5.4.0 with the default options supplied to `./configure` as: 18 | 19 | ``` 20 | ./configure --prefix=[AMReX library prefix] 21 | ``` 22 | 23 | Then compile this example as: 24 | 25 | ``` 26 | make AMREX_LIBRARY_HOME=[AMReX library prefix] 27 | ``` 28 | 29 | ## Running 30 | 31 | If compilation is successful you should have a `main.exe` executable. 32 | 33 | This can be run as-is with defaults corresponding to a 32^3 domain and 34 | max grid size of 16. This will create a domain of size 32^3 cells in 35 | 3D and divide it up into 8 boxes each of size 16^3. 36 | 37 | It will also write a plotfile named `plt_32_32_32_16`. The naming 38 | convention for the plotfiles is: 39 | 40 | ``` 41 | plt_[# cells x]_[# cells y]_[# cellsz]_[max grid size] 42 | ``` 43 | 44 | To change these defaults, you can use command line options. For 45 | example, this will set up a 64^3 domain and divide it up into 8 boxes 46 | of size 32^3: 47 | 48 | ``` 49 | ./main.exe n_cells=64 64 64 max_grid_size=32 50 | ``` 51 | 52 | ## Setting C++ flags and library paths manually (optional) 53 | 54 | The makefile should automatically find and use the correct C++ and 55 | library flags corresponding to the ones AMReX was built with. 56 | 57 | If you need for some reason to change this, or if the flags cannot be 58 | automatically found, then set `CFLAGS` and `LFLAGS` appropriately in 59 | the GNUmakefile. 60 | 61 | These flags are automatically extracted from the following file: 62 | 63 | ``` 64 | [AMReX library prefix]/lib/pkgconfig/amrex.pc 65 | ``` 66 | 67 | This file lists entries for `Cflags: ...` and `Libs: ...` something 68 | like the following, but for your system: 69 | 70 | ``` 71 | ... 72 | Cflags: -I${includedir} -Werror=return-type -g -O3 -std=c++14 73 | Libs: -L${libdir} -lamrex -L/usr/lib/gcc/x86_64-linux-gnu/5/ -Wl,-Bsymbolic-functions -Wl,-z,relro -I/usr/include/mpich -I/usr/include/mpich -L/usr/lib/x86_64-linux-gnu -lmpichfort -lmpich -lgfortran -lquadmath 74 | ... 75 | ``` 76 | -------------------------------------------------------------------------------- /amrex-plotfile-example/main.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | using namespace amrex; 10 | 11 | void initialize_domain(const Box& bx, Array4 fabarray) 12 | { 13 | auto lo = bx.loVect3d(); 14 | auto hi = bx.hiVect3d(); 15 | for (int k = lo[2]; k <= hi[2]; k++) { 16 | for (int j = lo[1]; j <= hi[1]; j++) { 17 | for (int i = lo[0]; i <= hi[0]; i++) { 18 | fabarray(i, j, k) = static_cast(i + 100.0*j + 10000.0*k); 19 | } 20 | } 21 | } 22 | } 23 | 24 | int main(int argc, 25 | char* argv[]) 26 | { 27 | Initialize(argc, argv); 28 | 29 | { 30 | int number_cells = 32; 31 | std::vector ncells {AMREX_D_DECL(number_cells, number_cells, number_cells)}; 32 | int max_grid_size = 16; 33 | 34 | // Read input parameters 35 | { 36 | ParmParse pp; 37 | if (!pp.queryarr("n_cells", ncells, 0, AMREX_SPACEDIM)) 38 | Print() << "n_cells not specified, so using 32 cells in each dimension.\n"; 39 | 40 | if (!pp.query("max_grid_size", max_grid_size)) 41 | Print() << "max_grid_size not specified, so using 16.\n"; 42 | } 43 | 44 | BoxArray ba; 45 | Geometry geo; 46 | 47 | // Define BoxArray and Geometry for our domain 48 | { 49 | // Define index space 50 | Box bx(IntVect(AMREX_D_DECL(0, 0, 0)), 51 | IntVect(AMREX_D_DECL(ncells[0]-1, ncells[1]-1, ncells[2]-1)), 52 | IntVect(AMREX_D_DECL(0, 0, 0))); 53 | ba.define(bx); 54 | ba.maxSize(max_grid_size); 55 | 56 | // Define physical space 57 | RealBox rbox(AMREX_D_DECL(0.0, 0.0, 0.0), 58 | AMREX_D_DECL(1.0, 1.0, 1.0)); 59 | 60 | // Cartesian coordinate system 61 | int coordinates = 0; 62 | 63 | // Fully periodic domain 64 | std::array is_periodic {AMREX_D_DECL(1,1,1)}; 65 | 66 | // Define Geometry 67 | geo.define(bx, &rbox, coordinates, is_periodic.data()); 68 | } 69 | 70 | // Construct DistributionMapping 71 | DistributionMapping dm {ba}; 72 | 73 | // 1 component, no ghost cells 74 | int num_components = 1; 75 | int num_ghost_cell = 0; 76 | 77 | // Build MultiFab 78 | MultiFab state(ba, dm, num_components, num_ghost_cell); 79 | 80 | // Initialize MultiFab 81 | for (MFIter mfi(state); mfi.isValid(); ++mfi) { 82 | const Box &bx = mfi.validbox(); 83 | Array4 fabarray = state.array(mfi); 84 | initialize_domain(bx, fabarray); 85 | } 86 | 87 | // Write MultiFab to plotfile 88 | std::string pfname = "plt_" + std::to_string(ncells[0]); 89 | #if (AMREX_SPACEDIM >= 2) 90 | pfname += "_" + std::to_string(ncells[1]); 91 | #endif 92 | #if (AMREX_SPACEDIM == 3) 93 | pfname += "_" + std::to_string(ncells[2]); 94 | #endif 95 | pfname += "_" + std::to_string(max_grid_size); 96 | 97 | const Vector varnames {"phi"}; 98 | 99 | WriteSingleLevelPlotfile(pfname, state, varnames, geo, 0.0, 0); 100 | 101 | } 102 | 103 | amrex::Finalize(); 104 | } 105 | -------------------------------------------------------------------------------- /source_cooley_plotfile_tools.sh: -------------------------------------------------------------------------------- 1 | # `source` this file to add the directory for AMReX plotfile tools to your PATH on Cooley: 2 | # e.g. do `source source_cooley_plotfile_tools.sh` 3 | export PATH="/lus/grand/projects/ATPESC2021/usr/MathPackages/spack/opt/spack/linux-rhel7-haswell/gcc-8.2.0/amrex-21.07-iwaygptbziyyoogyo55ktavrf7vrfoqa/bin:$PATH" 4 | -------------------------------------------------------------------------------- /vis/ffmpeg_make_mp4: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Use ffpmeg to transform a set of png files to an mp4 video. 4 | 5 | Donald E. Willcox 6 | """ 7 | import os 8 | import argparse 9 | from itertools import zip_longest 10 | 11 | parser = argparse.ArgumentParser() 12 | parser.add_argument('infiles', type=str, nargs='+', 13 | help='List of input files for encoding into a movie.') 14 | parser.add_argument('-s', '--sort', action='store_true', 15 | help='Sort input files sequentially by interpreting as an integer the first contiguous subset of the filename strings where the filenames differ. Does not strip leading paths.') 16 | parser.add_argument('-name', '--moviename', type=str, 17 | default='movie', 18 | help='Name of output (not extension). Default is "movie.mp4".') 19 | parser.add_argument('-ifps', '--input_fps', type=int, default=15, 20 | help='Input rate at which to read images, i.e. number of pngs per second. Default is 15.') 21 | parser.add_argument('-ofps', '--output_fps', type=int, default=25, 22 | help='Output rate of movie frames (not the rate of plot images). Default is 25.') 23 | args = parser.parse_args() 24 | 25 | def compare_mask(s1, s2): 26 | """Compare string s1 to string s2, return elementwise equality as boolean mask.""" 27 | 28 | return [a==b for a, b in zip_longest(str(s1), str(s2))] 29 | 30 | def first_diff_substring(s, sref): 31 | """Return the first substring of string s which differs from the reference string sref. 32 | Return the empty string if the two strings are identical.""" 33 | 34 | mask = compare_mask(s, sref) 35 | set_ifirst = False 36 | ifirst = 0 37 | ilast = len(s)-1 38 | for i, x in enumerate(mask): 39 | if not x and not set_ifirst: 40 | ifirst = i 41 | set_ifirst = True 42 | elif x and set_ifirst: 43 | ilast = i-1 44 | break 45 | 46 | if set_ifirst and ifirst <= len(s)-1: 47 | return s[ifirst:ilast+1] 48 | else: 49 | return '' 50 | 51 | def longest_diff_substring_list(s, sref_list): 52 | """Return the longest substring of string s which differs from all strings in sref_list. 53 | Return the empty string if no diffs can be found.""" 54 | 55 | maxlen = 0 56 | maxdff = '' 57 | for sref in sref_list: 58 | sdiff = first_diff_substring(s, sref) 59 | if sdiff: 60 | if len(sdiff) > maxlen: 61 | maxdff = sdiff 62 | maxlen = len(sdiff) 63 | return maxdff 64 | 65 | def get_sort_key(s, sref_list): 66 | """Return an integer sorting key for string s given the list of strings sref_list.""" 67 | 68 | sdiff = longest_diff_substring_list(s, sref_list) 69 | if sdiff: 70 | return int(sdiff) 71 | else: 72 | return 0 73 | 74 | def sort_input_filenames(filenames): 75 | """If sorting is desired, sort the input file list. 76 | Determine the substring to interpret as an integer by comparing file names.""" 77 | 78 | if len(filenames) > 1: 79 | filenames = sorted(filenames, key=lambda x: get_sort_key(x, filenames)) 80 | return filenames 81 | 82 | def symlink_inputs(base_link, filenames): 83 | """Create temporary symbolic links for the filenames.""" 84 | 85 | symnames = [] 86 | for i, f in enumerate(filenames): 87 | symn = '{}_{:010d}.png'.format(base_link, i) 88 | os.symlink(f, symn) 89 | symnames.append(symn) 90 | return symnames 91 | 92 | def remove_symlinks(symnames): 93 | """Delete the temporary symbolic links.""" 94 | 95 | for sf in symnames: 96 | os.remove(sf) 97 | 98 | if __name__=='__main__': 99 | # Sort input files if needed 100 | if args.sort: 101 | input_files = sort_input_filenames(args.infiles) 102 | else: 103 | input_files = args.infiles 104 | 105 | # Symbolically link input files to a series of temporary files 106 | base_link = '__fftemp' 107 | symnames = symlink_inputs(base_link, input_files) 108 | 109 | # Run ffmpeg on the files specified 110 | cmd = 'ffmpeg -y -r {} -i __fftemp_%010d.png -s 1440x1080 -vcodec libx264 -crf {} -pix_fmt yuv420p {}.mp4'.format(args.input_fps, 111 | args.output_fps, 112 | args.moviename) 113 | os.system(cmd) 114 | 115 | # Remove temporary files 116 | remove_symlinks(symnames) 117 | -------------------------------------------------------------------------------- /vis/mkmovie.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | 3 | for f in $(ls -d plt?????); do 4 | python3 slice.py $f -f u -ax z -min 0.0 -max 3.5 & 5 | done 6 | 7 | wait 8 | 9 | python3 ffmpeg_make_mp4 plt*.png -s -ifps 5 > /dev/null 2>&1 10 | -------------------------------------------------------------------------------- /vis/slice.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import yt 4 | import argparse 5 | 6 | parser = argparse.ArgumentParser() 7 | parser.add_argument("infile", type=str, help="Path to input file to slice.") 8 | parser.add_argument("-f", "--field", type=str, default="u", help="Name of field to plot. Default is u.") 9 | parser.add_argument("-ax", "--axis", type=str, default="z", help="Axis to slice across (x, y, or z). Default is z.") 10 | parser.add_argument("-min", "--field_min", type=float, default=1.0, help="Minimum value of field to plot.") 11 | parser.add_argument("-max", "--field_max", type=float, default=-1.0, help="Maximum value of field to plot.") 12 | 13 | args = parser.parse_args() 14 | 15 | if __name__ == "__main__": 16 | ds = yt.load(args.infile) 17 | 18 | field_to_plot = args.field 19 | for cat, name in ds.field_list: 20 | if name == args.field: 21 | field_to_plot = (cat, name) 22 | 23 | slice = yt.SlicePlot(ds, args.axis, field_to_plot) 24 | 25 | slice.set_log(field_to_plot, False) 26 | if args.field_min < args.field_max: 27 | slice.set_zlim(args.field, args.field_min, args.field_max) 28 | 29 | slice.save("{}.png".format(args.infile)) 30 | --------------------------------------------------------------------------------