├── .gitignore ├── .gitpod.dependencies.sh ├── .gitpod.dockerfile ├── .gitpod.yml ├── .jupyter └── jupyter_notebook_config.py ├── 01_HelloWorld ├── CMakeLists.txt └── main.cpp ├── 02_ParmParse ├── CMakeLists.txt ├── inputs └── main.cpp ├── 03_HeatEquation ├── CMakeLists.txt ├── Visualization.ipynb ├── inputs ├── main.cpp ├── myfunc.H ├── myfunc.cpp └── mykernel.H ├── 04_ParticleMesh ├── CMakeLists.txt ├── Visualization.ipynb ├── inputs └── main.cpp ├── 05_ParticleMesh_MultiLevel ├── CMakeLists.txt ├── Visualization.ipynb ├── inputs └── main.cpp ├── 06_Advection_Amr ├── Adv_K.H ├── AdvancePhiAllLevels.cpp ├── AdvancePhiAtLevel.cpp ├── AmrCoreAdv.H ├── AmrCoreAdv.cpp ├── CMakeLists.txt ├── DefineVelocity.cpp ├── Kernels.H ├── Prob.H ├── Tagging.H ├── Visualization.ipynb ├── bc_fill.H ├── compute_flux_2D_K.H ├── compute_flux_3D_K.H ├── face_velocity.H ├── inputs ├── main.cpp └── slope_K.H ├── 07_Advection_EB ├── CMakeLists.txt ├── DefineVelocity.cpp ├── EB_Cylinder.cpp ├── FluidParticleContainer.H ├── FluidParticleContainer.cpp ├── Indexing.H ├── Make.package ├── Visualization.ipynb ├── face_velocity.H ├── inputs ├── mac_project_velocity.cpp └── main.cpp ├── 08_Pachinko ├── CMakeLists.txt ├── MyParticleContainer.H ├── MyParticleContainer.cpp ├── Visualization.ipynb ├── initial_particles_3d ├── inputs_3d ├── main.cpp └── paraview_pachinko.py ├── CMakeLists.txt ├── LICENSE ├── README.md └── cmake └── SetupTutorials.cmake /.gitignore: -------------------------------------------------------------------------------- 1 | nohup.out 2 | *.[oa] 3 | *.ex 4 | *.ex.dSYM 5 | *.exe 6 | *.exe.dSYM 7 | *~ 8 | build/ 9 | tmp_build_dir/ 10 | d/ 11 | f/ 12 | o/ 13 | t/ 14 | TAGS 15 | 16 | # latex 17 | *.aux 18 | *.log 19 | *.bbl 20 | *.toc 21 | *.dvi 22 | *-eps-converted-to.pdf 23 | 24 | # python 25 | *.pyc 26 | 27 | .nfs* 28 | 29 | # vi 30 | *.sw[a-z] 31 | 32 | # files generated by tutorials 33 | *.vth 34 | *.vti 35 | *.vtr 36 | *.vtp 37 | *.pvtp 38 | *.pvd 39 | *.visit 40 | *.pvsm 41 | 42 | -------------------------------------------------------------------------------- /.gitpod.dependencies.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -eu -o pipefail 4 | 5 | #sudo update-alternatives --install /usr/bin/python python /usr/bin/python3 2 6 | #sudo update-alternatives --set python /usr/bin/python3 7 | python -m pip install --upgrade pip 8 | python -m pip install --upgrade wheel 9 | python -m pip install --upgrade cmake matplotlib==3.2.2 numpy yt 10 | -------------------------------------------------------------------------------- /.gitpod.dockerfile: -------------------------------------------------------------------------------- 1 | FROM gitpod/workspace-full-vnc 2 | 3 | RUN sudo apt-get update && sudo apt-get install -y --no-install-recommends build-essential g++ libopenmpi-dev openmpi-bin paraview python3-paraview ffmpeg python3 python3-pip python3-setuptools && sudo rm -rf /var/lib/apt/lists/* -------------------------------------------------------------------------------- /.gitpod.yml: -------------------------------------------------------------------------------- 1 | # The Docker image to run your workspace in. Defaults to gitpod/workspace-full 2 | image: 3 | file: .gitpod.dockerfile 4 | tasks: 5 | - init: /workspace/ecp-tutorials/.gitpod.dependencies.sh 6 | command: cp -r /workspace/ecp-tutorials/.jupyter /home/gitpod/ 7 | # Ports to expose on workspace startup (optional) 8 | ports: 9 | - port: 8000 10 | onOpen: open-preview 11 | - port: 8888 12 | onOpen: open-browser 13 | github: 14 | prebuilds: 15 | # enable for the master/default branch (defaults to true) 16 | master: true 17 | # enable for all branches in this repo (defaults to false) 18 | branches: false 19 | # enable for pull requests coming from this repo (defaults to true) 20 | pullRequests: false 21 | # enable for pull requests coming from forks (defaults to false) 22 | pullRequestsFromForks: false 23 | # add a check to pull requests (defaults to true) 24 | addCheck: false 25 | # add a "Review in Gitpod" button as a comment to pull requests (defaults to false) 26 | addComment: false 27 | # add a "Review in Gitpod" button to the pull request's description (defaults to false) 28 | addBadge: false 29 | # add a label once the prebuild is ready to pull requests (defaults to false) 30 | addLabel: false 31 | -------------------------------------------------------------------------------- /.jupyter/jupyter_notebook_config.py: -------------------------------------------------------------------------------- 1 | c.NotebookApp.ip = '0.0.0.0' 2 | c.NotebookApp.allow_origin = '*' 3 | c.NotebookApp.open_browser = False 4 | -------------------------------------------------------------------------------- /01_HelloWorld/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # List of source files 2 | set(_sources main.cpp) 3 | 4 | # List of input files 5 | set(_input_files) 6 | 7 | setup_tutorial(_sources _input_files) 8 | 9 | unset( _sources ) 10 | unset( _input_files ) 11 | -------------------------------------------------------------------------------- /01_HelloWorld/main.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | #include 4 | 5 | int main(int argc, char* argv[]) 6 | { 7 | amrex::Initialize(argc,argv); 8 | { 9 | amrex::Print() << "Hello world from AMReX version " << amrex::Version() << "\n"; 10 | } 11 | amrex::Finalize(); 12 | } 13 | 14 | -------------------------------------------------------------------------------- /02_ParmParse/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | if (NOT AMReX_MPI) 2 | return() 3 | endif () 4 | 5 | set(_sources main.cpp) 6 | set(_input_files inputs) 7 | 8 | setup_tutorial(_sources _input_files) 9 | 10 | unset(_sources) 11 | unset(_input_files) 12 | -------------------------------------------------------------------------------- /02_ParmParse/inputs: -------------------------------------------------------------------------------- 1 | an_int_scalar = 2 2 | a_bool_scalar = true 3 | a_real_array = 1. 2. 3. 4. 4 | 5 | a_prefix.a_real_scalar = 99.0 6 | a_prefix.a_string = "option" 7 | a_prefix.an_int_array = 4 5 6 -------------------------------------------------------------------------------- /02_ParmParse/main.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | void test_parameters (); 6 | 7 | int main(int argc, char* argv[]) 8 | { 9 | amrex::Initialize(argc, argv); 10 | 11 | test_parameters(); 12 | 13 | amrex::Finalize(); 14 | } 15 | 16 | void test_parameters () 17 | { 18 | { 19 | amrex::ParmParse pp; 20 | int i; 21 | bool b; 22 | std::vector ra; 23 | pp.get("an_int_scalar", i); 24 | pp.get("a_bool_scalar",b); 25 | pp.getarr("a_real_array", ra); 26 | amrex::Print() << "an_int_scalar = " << i << "\n" 27 | << "a_bool_scalar = " << b << "\n"; 28 | amrex::Print() << "a_real_array = "; 29 | for (auto x : ra) { 30 | amrex::Print() << x << " "; 31 | } 32 | amrex::Print() << "\n"; 33 | } 34 | 35 | { 36 | amrex::ParmParse pp("a_prefix"); 37 | std::vector ia; 38 | amrex::Real r; 39 | std::string s; 40 | pp.getarr("an_int_array", ia); 41 | pp.get("a_real_scalar", r); 42 | pp.get("a_string", s); 43 | amrex::Print() << "an_int_array = "; 44 | for (auto x : ia) { 45 | amrex::Print() << x << " "; 46 | } 47 | amrex::Print() << "\n"; 48 | amrex::Print() << "a_prefix.a_real_scalar = " << r << "\n" 49 | << "a_prefix.a_string = " << s << "\n"; 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /03_HeatEquation/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | if (AMReX_SPACEDIM EQUAL 1) 2 | return() 3 | endif () 4 | 5 | # List of source files 6 | set(_sources main.cpp myfunc.cpp myfunc.H mykernel.H) 7 | set(_input_files inputs Visualization.ipynb) 8 | 9 | setup_tutorial(_sources _input_files) 10 | 11 | unset( _sources ) 12 | unset( _input_files ) 13 | -------------------------------------------------------------------------------- /03_HeatEquation/Visualization.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "839a3961", 6 | "metadata": {}, 7 | "source": [ 8 | "## Example: 3D Heat Equation\n", 9 | "\n", 10 | "### What Features Are We Using\n", 11 | "\n", 12 | "* Mesh data \n", 13 | "* Domain Decomposition\n", 14 | "\n", 15 | "We now look at a more complicated example at and show how simulation results\n", 16 | "can be visualized. This example solves the heat equation,\n", 17 | "\n", 18 | "$$\\frac{\\partial\\phi}{\\partial t} = \\nabla^2\\phi$$\n", 19 | "\n", 20 | "using forward Euler temporal integration on a periodic domain. We could use a\n", 21 | "5-point (in 2D) or 7-point (in 3D) stencil, but for demonstration purposes we\n", 22 | "spatially discretize the PDE by first constructing (negative) fluxes on cell faces, e.g.,\n", 23 | "\n", 24 | "$$F_{i+^1\\!/_2,\\,j} = \\frac{\\phi_{i+1,j}-\\phi_{i,j}}{\\Delta x}$$\n", 25 | "\n", 26 | "and then taking the divergence to update the cells:\n", 27 | "\n", 28 | " $$\\phi_{i,\\,j}^{n+1} = \\phi_{i,\\,j}^n\n", 29 | " + \\frac{\\Delta t}{\\Delta x}\\left(F_{i+^1\\!/_2,\\,j}-F_{i-^1\\!/_2,\\,j}\\right)\n", 30 | " + \\frac{\\Delta t}{\\Delta y}\\left(F_{i,\\,j+^1\\!/_2}-F_{i,\\,j-^1\\!/_2}\\right)$$" 31 | ] 32 | }, 33 | { 34 | "cell_type": "markdown", 35 | "id": "08b4ec5f", 36 | "metadata": {}, 37 | "source": [ 38 | "The code to generate the initial condition is in `mykernel.H` and looks like: \n", 39 | "```C++\n", 40 | "{\n", 41 | " Real x = prob_lo[0] + (i+Real(0.5)) * dx[0];\n", 42 | " Real y = prob_lo[1] + (j+Real(0.5)) * dx[1];\n", 43 | " Real z = prob_lo[2] + (k+Real(0.5)) * dx[2];\n", 44 | " Real r2 = ((x-Real(0.25))*(x-Real(0.25))+(y-Real(0.25))*(y-Real(0.25))+(z-Real(0.25))*(z-Real(0.25)))/Real(0.01);\n", 45 | " phi(i,j,k) = Real(1.) + std::exp(-r2);\n", 46 | "}\n", 47 | "```" 48 | ] 49 | }, 50 | { 51 | "cell_type": "markdown", 52 | "id": "33bd71f2", 53 | "metadata": {}, 54 | "source": [ 55 | "## Running the code\n", 56 | "\n", 57 | "The simulation can be ran as `./03_HeatEquation inputs`. \n", 58 | "\n", 59 | "The following inputs parameters could be tweaked:\n", 60 | "\n", 61 | "```\n", 62 | "nsteps = 1000 # number of time steps to take\n", 63 | "plot_int = 100 # write plots every n steps\n", 64 | "n_cell = 128 # number of cells in the domain\n", 65 | "max_grid_size = 64 # max grid size used for domain decomposition\n", 66 | "\n", 67 | "```\n", 68 | "\n", 69 | "Although we are running this example in serial, we decompose the domain into multiple boxes, anticipating more complicated problems where we have mesh refinement:\n", 70 | "\n", 71 | "```C++\n", 72 | " IntVect dom_lo(AMREX_D_DECL( 0, 0, 0));\n", 73 | " IntVect dom_hi(AMREX_D_DECL(n_cell-1, n_cell-1, n_cell-1));\n", 74 | " Box domain(dom_lo, dom_hi);\n", 75 | "\n", 76 | " // Initialize the boxarray \"ba\" from the single box \"bx\"\n", 77 | " ba.define(domain);\n", 78 | " // Break up boxarray \"ba\" into chunks no larger than \"max_grid_size\" along a direction\n", 79 | " ba.maxSize(max_grid_size);\n", 80 | "\n", 81 | "```" 82 | ] 83 | }, 84 | { 85 | "cell_type": "markdown", 86 | "id": "b772b897", 87 | "metadata": {}, 88 | "source": [ 89 | "## Visualizating the results\n", 90 | "\n", 91 | "Below we give some python code to visualizate the solution using yt:" 92 | ] 93 | }, 94 | { 95 | "cell_type": "code", 96 | "execution_count": null, 97 | "id": "29609c78", 98 | "metadata": {}, 99 | "outputs": [], 100 | "source": [ 101 | "import yt\n", 102 | "from yt.frontends.boxlib.data_structures import AMReXDataset" 103 | ] 104 | }, 105 | { 106 | "cell_type": "code", 107 | "execution_count": null, 108 | "id": "18ec883c", 109 | "metadata": {}, 110 | "outputs": [], 111 | "source": [ 112 | "ls" 113 | ] 114 | }, 115 | { 116 | "cell_type": "code", 117 | "execution_count": null, 118 | "id": "cb533aa1", 119 | "metadata": {}, 120 | "outputs": [], 121 | "source": [ 122 | "ds = AMReXDataset(\"plt00000\")" 123 | ] 124 | }, 125 | { 126 | "cell_type": "code", 127 | "execution_count": null, 128 | "id": "2d834eaf", 129 | "metadata": {}, 130 | "outputs": [], 131 | "source": [ 132 | "ds.field_list" 133 | ] 134 | }, 135 | { 136 | "cell_type": "code", 137 | "execution_count": null, 138 | "id": "899767fc", 139 | "metadata": {}, 140 | "outputs": [], 141 | "source": [ 142 | "sl = yt.SlicePlot(ds, 2, ('boxlib', 'phi'))" 143 | ] 144 | }, 145 | { 146 | "cell_type": "code", 147 | "execution_count": null, 148 | "id": "679eaca7", 149 | "metadata": {}, 150 | "outputs": [], 151 | "source": [ 152 | "sl" 153 | ] 154 | }, 155 | { 156 | "cell_type": "code", 157 | "execution_count": null, 158 | "id": "72af42b5", 159 | "metadata": {}, 160 | "outputs": [], 161 | "source": [ 162 | "ds = AMReXDataset(\"plt01000\")\n", 163 | "sl = yt.SlicePlot(ds, 2, ('boxlib', 'phi'))\n", 164 | "sl" 165 | ] 166 | }, 167 | { 168 | "cell_type": "code", 169 | "execution_count": null, 170 | "id": "bd7332dd", 171 | "metadata": {}, 172 | "outputs": [], 173 | "source": [ 174 | "sl.annotate_grids()" 175 | ] 176 | } 177 | ], 178 | "metadata": { 179 | "kernelspec": { 180 | "display_name": "Python 3", 181 | "language": "python", 182 | "name": "python3" 183 | }, 184 | "language_info": { 185 | "codemirror_mode": { 186 | "name": "ipython", 187 | "version": 3 188 | }, 189 | "file_extension": ".py", 190 | "mimetype": "text/x-python", 191 | "name": "python", 192 | "nbconvert_exporter": "python", 193 | "pygments_lexer": "ipython3", 194 | "version": "3.8.8" 195 | } 196 | }, 197 | "nbformat": 4, 198 | "nbformat_minor": 5 199 | } 200 | -------------------------------------------------------------------------------- /03_HeatEquation/inputs: -------------------------------------------------------------------------------- 1 | nsteps = 1000 2 | plot_int = 100 3 | n_cell = 128 4 | max_grid_size = 64 5 | 6 | 7 | -------------------------------------------------------------------------------- /03_HeatEquation/main.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | #include "myfunc.H" 8 | 9 | using namespace amrex; 10 | 11 | int main (int argc, char* argv[]) 12 | { 13 | amrex::Initialize(argc,argv); 14 | 15 | main_main(); 16 | 17 | amrex::Finalize(); 18 | return 0; 19 | } 20 | 21 | void main_main () 22 | { 23 | // What time is it now? We'll use this to compute total run time. 24 | auto strt_time = ParallelDescriptor::second(); 25 | 26 | // AMREX_SPACEDIM: number of dimensions 27 | int n_cell, max_grid_size, nsteps, plot_int; 28 | 29 | // inputs parameters 30 | { 31 | // ParmParse is way of reading inputs from the inputs file 32 | ParmParse pp; 33 | 34 | // We need to get n_cell from the inputs file - this is the number of cells on each side of 35 | // a square (or cubic) domain. 36 | pp.get("n_cell",n_cell); 37 | 38 | // The domain is broken into boxes of size max_grid_size 39 | pp.get("max_grid_size",max_grid_size); 40 | 41 | // Default plot_int to -1, allow us to set it to something else in the inputs file 42 | // If plot_int < 0 then no plot files will be written 43 | plot_int = -1; 44 | pp.query("plot_int",plot_int); 45 | 46 | // Default nsteps to 10, allow us to set it to something else in the inputs file 47 | nsteps = 10; 48 | pp.query("nsteps",nsteps); 49 | } 50 | 51 | // make BoxArray and Geometry 52 | BoxArray ba; 53 | Geometry geom; 54 | { 55 | IntVect dom_lo(AMREX_D_DECL( 0, 0, 0)); 56 | IntVect dom_hi(AMREX_D_DECL(n_cell-1, n_cell-1, n_cell-1)); 57 | Box domain(dom_lo, dom_hi); 58 | 59 | // Initialize the boxarray "ba" from the single box "bx" 60 | ba.define(domain); 61 | // Break up boxarray "ba" into chunks no larger than "max_grid_size" along a direction 62 | ba.maxSize(max_grid_size); 63 | 64 | // This defines the physical box, [-1,1] in each direction. 65 | RealBox real_box({AMREX_D_DECL(-Real(1.0),-Real(1.0),-Real(1.0))}, 66 | {AMREX_D_DECL( Real(1.0), Real(1.0), Real(1.0))}); 67 | 68 | // periodic in all direction 69 | Array is_periodic{AMREX_D_DECL(1,1,1)}; 70 | 71 | // This defines a Geometry object 72 | geom.define(domain,real_box,CoordSys::cartesian,is_periodic); 73 | } 74 | 75 | // Nghost = number of ghost cells for each array 76 | int Nghost = 1; 77 | 78 | // Ncomp = number of components for each array 79 | int Ncomp = 1; 80 | 81 | // How Boxes are distrubuted among MPI processes 82 | DistributionMapping dm(ba); 83 | 84 | // we allocate two phi multifabs; one will store the old state, the other the new. 85 | MultiFab phi_old(ba, dm, Ncomp, Nghost); 86 | MultiFab phi_new(ba, dm, Ncomp, Nghost); 87 | 88 | GpuArray dx = geom.CellSizeArray(); 89 | 90 | init_phi(phi_new, geom); 91 | // ======================================== 92 | 93 | Real cfl = 0.9; 94 | Real coeff = AMREX_D_TERM( 1./(dx[0]*dx[0]), 95 | + 1./(dx[1]*dx[1]), 96 | + 1./(dx[2]*dx[2]) ); 97 | Real dt = cfl/(2.0*coeff); 98 | 99 | // time = starting time in the simulation 100 | Real time = 0.0; 101 | 102 | // Write a plotfile of the initial data if plot_int > 0 (plot_int was defined in the inputs file) 103 | if (plot_int > 0) 104 | { 105 | int n = 0; 106 | const std::string& pltfile = amrex::Concatenate("plt",n,5); 107 | WriteSingleLevelPlotfile(pltfile, phi_new, {"phi"}, geom, time, 0); 108 | } 109 | 110 | // build the flux multifabs 111 | Array flux; 112 | for (int dir = 0; dir < AMREX_SPACEDIM; dir++) 113 | { 114 | // flux(dir) has one component, zero ghost cells, and is nodal in direction dir 115 | BoxArray edge_ba = ba; 116 | edge_ba.surroundingNodes(dir); 117 | flux[dir].define(edge_ba, dm, 1, 0); 118 | } 119 | 120 | for (int n = 1; n <= nsteps; ++n) 121 | { 122 | MultiFab::Copy(phi_old, phi_new, 0, 0, 1, 0); 123 | 124 | // new_phi = old_phi + dt * (something) 125 | advance(phi_old, phi_new, flux, dt, geom); 126 | time = time + dt; 127 | 128 | // Tell the I/O Processor to write out which step we're doing 129 | amrex::Print() << "Advanced step " << n << "\n"; 130 | 131 | // Write a plotfile of the current data (plot_int was defined in the inputs file) 132 | if (plot_int > 0 && n%plot_int == 0) 133 | { 134 | const std::string& pltfile = amrex::Concatenate("plt",n,5); 135 | WriteSingleLevelPlotfile(pltfile, phi_new, {"phi"}, geom, time, n); 136 | } 137 | } 138 | 139 | // Call the timer again and compute the maximum difference between the start time and stop time 140 | // over all processors 141 | auto stop_time = ParallelDescriptor::second() - strt_time; 142 | const int IOProc = ParallelDescriptor::IOProcessorNumber(); 143 | ParallelDescriptor::ReduceRealMax(stop_time,IOProc); 144 | 145 | // Tell the I/O Processor to write out the "run time" 146 | amrex::Print() << "Run time = " << stop_time << std::endl; 147 | } 148 | -------------------------------------------------------------------------------- /03_HeatEquation/myfunc.H: -------------------------------------------------------------------------------- 1 | #ifndef MYFUNC_H_ 2 | #define MYFUNC_H_ 3 | 4 | #include 5 | #include 6 | 7 | void main_main (); 8 | 9 | void advance (amrex::MultiFab& phi_old, 10 | amrex::MultiFab& phi_new, 11 | amrex::Array& flux, 12 | amrex::Real dt, 13 | amrex::Geometry const& geom); 14 | 15 | void init_phi (amrex::MultiFab& phi_new, amrex::Geometry const& geom); 16 | #endif 17 | -------------------------------------------------------------------------------- /03_HeatEquation/myfunc.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include "myfunc.H" 3 | #include "mykernel.H" 4 | 5 | using namespace amrex; 6 | 7 | void advance (MultiFab& phi_old, 8 | MultiFab& phi_new, 9 | Array& flux, 10 | Real dt, 11 | Geometry const& geom) 12 | { 13 | 14 | // Fill the ghost cells of each grid from the other grids 15 | // includes periodic domain boundaries. 16 | // There are no physical domain boundaries to fill in this example. 17 | phi_old.FillBoundary(geom.periodicity()); 18 | 19 | // 20 | // Note that this simple example is not optimized. 21 | // The following two MFIter loops could be merged 22 | // and we do not have to use flux MultiFab. 23 | // 24 | // ======================================================= 25 | 26 | // This example supports both 2D and 3D. Otherwise, 27 | // we would not need to use AMREX_D_TERM. 28 | AMREX_D_TERM(const Real dxinv = geom.InvCellSize(0);, 29 | const Real dyinv = geom.InvCellSize(1);, 30 | const Real dzinv = geom.InvCellSize(2);); 31 | 32 | // Compute fluxes one grid at a time 33 | for ( MFIter mfi(phi_old); mfi.isValid(); ++mfi ) 34 | { 35 | const Box& xbx = mfi.nodaltilebox(0); 36 | const Box& ybx = mfi.nodaltilebox(1); 37 | auto const& fluxx = flux[0].array(mfi); 38 | auto const& fluxy = flux[1].array(mfi); 39 | #if (AMREX_SPACEDIM > 2) 40 | const Box& zbx = mfi.nodaltilebox(2); 41 | auto const& fluxz = flux[2].array(mfi); 42 | #endif 43 | auto const& phi = phi_old.const_array(mfi); 44 | 45 | amrex::ParallelFor(xbx, 46 | [=] AMREX_GPU_DEVICE (int i, int j, int k) 47 | { 48 | compute_flux_x(i,j,k,fluxx,phi,dxinv); 49 | }); 50 | 51 | amrex::ParallelFor(ybx, 52 | [=] AMREX_GPU_DEVICE (int i, int j, int k) 53 | { 54 | compute_flux_y(i,j,k,fluxy,phi,dyinv); 55 | }); 56 | 57 | #if (AMREX_SPACEDIM > 2) 58 | amrex::ParallelFor(zbx, 59 | [=] AMREX_GPU_DEVICE (int i, int j, int k) 60 | { 61 | compute_flux_z(i,j,k,fluxz,phi,dzinv); 62 | }); 63 | #endif 64 | } 65 | 66 | // Advance the solution one grid at a time 67 | for ( MFIter mfi(phi_old); mfi.isValid(); ++mfi ) 68 | { 69 | const Box& vbx = mfi.validbox(); 70 | auto const& fluxx = flux[0].const_array(mfi); 71 | auto const& fluxy = flux[1].const_array(mfi); 72 | #if (AMREX_SPACEDIM > 2) 73 | auto const& fluxz = flux[2].const_array(mfi); 74 | #endif 75 | auto const& phiOld = phi_old.const_array(mfi); 76 | auto const& phiNew = phi_new.array(mfi); 77 | 78 | amrex::ParallelFor(vbx, 79 | [=] AMREX_GPU_DEVICE (int i, int j, int k) 80 | { 81 | update_phi(i,j,k,phiOld,phiNew, 82 | AMREX_D_DECL(fluxx,fluxy,fluxz), 83 | dt, 84 | AMREX_D_DECL(dxinv,dyinv,dzinv)); 85 | }); 86 | } 87 | } 88 | 89 | void init_phi(amrex::MultiFab& phi_new, amrex::Geometry const& geom){ 90 | 91 | GpuArray dx = geom.CellSizeArray(); 92 | GpuArray prob_lo = geom.ProbLoArray(); 93 | // ======================================= 94 | // Initialize phi_new by calling a Fortran routine. 95 | // MFIter = MultiFab Iterator 96 | for (MFIter mfi(phi_new); mfi.isValid(); ++mfi) 97 | { 98 | const Box& vbx = mfi.validbox(); 99 | auto const& phiNew = phi_new.array(mfi); 100 | amrex::ParallelFor(vbx, 101 | [=] AMREX_GPU_DEVICE(int i, int j, int k) 102 | { 103 | init_phi(i,j,k,phiNew,dx,prob_lo); 104 | }); 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /03_HeatEquation/mykernel.H: -------------------------------------------------------------------------------- 1 | #ifndef MY_KERNEL_H_ 2 | #define MY_KERNEL_H_ 3 | 4 | #include 5 | 6 | AMREX_GPU_DEVICE AMREX_FORCE_INLINE 7 | void init_phi (int i, int j, int k, 8 | amrex::Array4 const& phi, 9 | amrex::GpuArray const& dx, 10 | amrex::GpuArray const& prob_lo) 11 | { 12 | using amrex::Real;; 13 | 14 | Real x = prob_lo[0] + (i+Real(0.5)) * dx[0]; 15 | Real y = prob_lo[1] + (j+Real(0.5)) * dx[1]; 16 | #if (AMREX_SPACEDIM > 2) 17 | Real z = prob_lo[2] + (k+Real(0.5)) * dx[2]; 18 | Real r2 = ((x-Real(0.25))*(x-Real(0.25))+(y-Real(0.25))*(y-Real(0.25))+(z-Real(0.25))*(z-Real(0.25)))/Real(0.01); 19 | #else 20 | Real z = Real(0.); 21 | Real r2 = ((x-Real(0.25))*(x-Real(0.25))+(y-Real(0.25))*(y-Real(0.25)))/Real(0.01); 22 | #endif 23 | phi(i,j,k) = Real(1.) + std::exp(-r2); 24 | } 25 | 26 | 27 | AMREX_GPU_DEVICE AMREX_FORCE_INLINE 28 | void compute_flux_x (int i, int j, int k, 29 | amrex::Array4 const& fluxx, 30 | amrex::Array4 const& phi, amrex::Real dxinv) 31 | { 32 | fluxx(i,j,k) = (phi(i,j,k)-phi(i-1,j,k)) * dxinv; 33 | } 34 | 35 | 36 | AMREX_GPU_DEVICE AMREX_FORCE_INLINE 37 | void compute_flux_y (int i, int j, int k, 38 | amrex::Array4 const& fluxy, 39 | amrex::Array4 const& phi, amrex::Real dyinv) 40 | { 41 | fluxy(i,j,k) = (phi(i,j,k)-phi(i,j-1,k)) * dyinv; 42 | } 43 | 44 | 45 | #if (AMREX_SPACEDIM > 2) 46 | AMREX_GPU_DEVICE AMREX_FORCE_INLINE 47 | void compute_flux_z (int i, int j, int k, 48 | amrex::Array4 const& fluxz, 49 | amrex::Array4 const& phi, amrex::Real dzinv) 50 | { 51 | fluxz(i,j,k) = (phi(i,j,k)-phi(i,j,k-1)) * dzinv; 52 | } 53 | #endif 54 | 55 | AMREX_GPU_DEVICE AMREX_FORCE_INLINE 56 | void update_phi (int i, int j, int k, 57 | amrex::Array4 const& phiold, 58 | amrex::Array4 const& phinew, 59 | AMREX_D_DECL(amrex::Array4 const& fluxx, 60 | amrex::Array4 const& fluxy, 61 | amrex::Array4 const& fluxz), 62 | amrex::Real dt, 63 | AMREX_D_DECL(amrex::Real dxinv, 64 | amrex::Real dyinv, 65 | amrex::Real dzinv)) 66 | { 67 | phinew(i,j,k) = phiold(i,j,k) 68 | + dt * dxinv * (fluxx(i+1,j ,k ) - fluxx(i,j,k)) 69 | + dt * dyinv * (fluxy(i ,j+1,k ) - fluxy(i,j,k)) 70 | #if (AMREX_SPACEDIM > 2) 71 | + dt * dzinv * (fluxz(i ,j ,k+1) - fluxz(i,j,k)); 72 | #else 73 | ; 74 | #endif 75 | } 76 | 77 | #endif 78 | -------------------------------------------------------------------------------- /04_ParticleMesh/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(_sources main.cpp) 2 | set(_input_files inputs Visualization.ipynb) 3 | 4 | setup_tutorial(_sources _input_files) 5 | 6 | unset(_sources) 7 | unset(_input_files) 8 | -------------------------------------------------------------------------------- /04_ParticleMesh/Visualization.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "1540d072", 6 | "metadata": {}, 7 | "source": [ 8 | "## Example: Particle Mesh Interpolation\n", 9 | "\n", 10 | "### What Features Are We Using\n", 11 | "\n", 12 | "* Mesh data\n", 13 | "* Particle data\n", 14 | "* Domain Decomposition\n", 15 | "\n", 16 | "Thus far we have only encountered mesh data. This example shows how to use particle data \n", 17 | "and how to interpolate between the particles and the mesh.\n", 18 | "\n", 19 | "To tell amrex how many particle components we want, pass in template parameters to `amrex::ParticleContainer`.\n", 20 | " \n", 21 | "```C++\n", 22 | " typedef ParticleContainer<1 + 2*AMREX_SPACEDIM> MyParticleContainer;\n", 23 | " MyParticleContainer myPC(geom, dmap, ba);\n", 24 | "```\n", 25 | "\n", 26 | "Here we have asked for `1 + 2*AMREX_SPACEDIM` real components per particle, stored in Array-of-Struct fashion.\n", 27 | "These will represent the mass, three velocity components, and three acceleration components of each particle.\n", 28 | "\n", 29 | "We initialize the particle positions randomly throughout the domain, based on a target average number per cell.\n", 30 | "\n", 31 | "Particle-mesh operations can be done with the `amrex::ParticleToMesh` and `amrex::MeshToParticle` routines. The type of interpolation to perform is passed in as a lambda function:\n", 32 | "\n", 33 | "```C++\n", 34 | " amrex::ParticleToMesh(myPC, partMF, 0,\n", 35 | " [=] AMREX_GPU_DEVICE (const MyParticleContainer::ParticleType& p,\n", 36 | " amrex::Array4 const& rho)\n", 37 | " {\n", 38 | " amrex::Real lx = (p.pos(0) - plo[0]) * dxi[0] + 0.5;\n", 39 | " amrex::Real ly = (p.pos(1) - plo[1]) * dxi[1] + 0.5;\n", 40 | " amrex::Real lz = (p.pos(2) - plo[2]) * dxi[2] + 0.5;\n", 41 | "\n", 42 | " int i = amrex::Math::floor(lx);\n", 43 | " int j = amrex::Math::floor(ly);\n", 44 | " int k = amrex::Math::floor(lz);\n", 45 | "\n", 46 | " amrex::Real xint = lx - i;\n", 47 | " amrex::Real yint = ly - j;\n", 48 | " amrex::Real zint = lz - k;\n", 49 | "\n", 50 | " amrex::Real sx[] = {1.-xint, xint};\n", 51 | " amrex::Real sy[] = {1.-yint, yint};\n", 52 | " amrex::Real sz[] = {1.-zint, zint};\n", 53 | "\n", 54 | " for (int kk = 0; kk <= 1; ++kk) {\n", 55 | " for (int jj = 0; jj <= 1; ++jj) {\n", 56 | " for (int ii = 0; ii <= 1; ++ii) {\n", 57 | " amrex::Gpu::Atomic::AddNoRet(&rho(i+ii-1, j+jj-1, k+kk-1, 0),\n", 58 | " sx[ii]*sy[jj]*sz[kk]*p.rdata(0));\n", 59 | " }\n", 60 | " }\n", 61 | " }\n", 62 | " }\n", 63 | "```\n", 64 | "\n", 65 | "The above code is for \"Cloud in cell\" interpolation. Under the hood, the deposition algorithm will be changed based on the whether the parallel backend is targetting GPUs or CPUs." 66 | ] 67 | }, 68 | { 69 | "cell_type": "markdown", 70 | "id": "ec315f02", 71 | "metadata": {}, 72 | "source": [ 73 | "## Running the code\n", 74 | "\n", 75 | "The simulation can be ran as `./04_ParticleMesh inputs`. \n", 76 | "\n", 77 | "The following inputs parameters could be tweaked:\n", 78 | "\n", 79 | "```\n", 80 | "nx = 128 # number of grid points along the x axis\n", 81 | "ny = 128 # number of grid points along the y axis\n", 82 | "nz = 128 # number of grid points along the z axis\n", 83 | "max_grid_size = 32 # grid size used for domain decomposition\n", 84 | "nppc = 10 # average number of particles per cell \n", 85 | "\n", 86 | "```" 87 | ] 88 | }, 89 | { 90 | "cell_type": "code", 91 | "execution_count": null, 92 | "id": "83170d52", 93 | "metadata": {}, 94 | "outputs": [], 95 | "source": [ 96 | "import yt" 97 | ] 98 | }, 99 | { 100 | "cell_type": "code", 101 | "execution_count": null, 102 | "id": "b1a070d3", 103 | "metadata": {}, 104 | "outputs": [], 105 | "source": [ 106 | "ds = yt.load(\"plot\")" 107 | ] 108 | }, 109 | { 110 | "cell_type": "code", 111 | "execution_count": null, 112 | "id": "cac891d9", 113 | "metadata": {}, 114 | "outputs": [], 115 | "source": [ 116 | "sl = yt.SlicePlot(ds, 2, 'density')" 117 | ] 118 | }, 119 | { 120 | "cell_type": "code", 121 | "execution_count": null, 122 | "id": "cdee59ea", 123 | "metadata": {}, 124 | "outputs": [], 125 | "source": [ 126 | "sl.annotate_grids()" 127 | ] 128 | }, 129 | { 130 | "cell_type": "code", 131 | "execution_count": null, 132 | "id": "e695489d", 133 | "metadata": {}, 134 | "outputs": [], 135 | "source": [ 136 | "ad = ds.all_data()\n", 137 | "ad.quantities.weighted_average_quantity('density', 'cell_volume')" 138 | ] 139 | } 140 | ], 141 | "metadata": { 142 | "kernelspec": { 143 | "display_name": "Python 3", 144 | "language": "python", 145 | "name": "python3" 146 | }, 147 | "language_info": { 148 | "codemirror_mode": { 149 | "name": "ipython", 150 | "version": 3 151 | }, 152 | "file_extension": ".py", 153 | "mimetype": "text/x-python", 154 | "name": "python", 155 | "nbconvert_exporter": "python", 156 | "pygments_lexer": "ipython3", 157 | "version": "3.8.8" 158 | } 159 | }, 160 | "nbformat": 4, 161 | "nbformat_minor": 5 162 | } 163 | -------------------------------------------------------------------------------- /04_ParticleMesh/inputs: -------------------------------------------------------------------------------- 1 | 2 | # Domain size 3 | 4 | #nx = 32 # number of grid points along the x axis 5 | #ny = 32 # number of grid points along the y axis 6 | #nz = 32 # number of grid points along the z axis 7 | 8 | #nx = 64 # number of grid points along the x axis 9 | #ny = 64 # number of grid points along the y axis 10 | #nz = 64 # number of grid points along the z axis 11 | 12 | nx = 128 # number of grid points along the x axis 13 | ny = 128 # number of grid points along the y axis 14 | nz = 128 # number of grid points along the z axis 15 | 16 | # Maximum allowable size of each subdomain in the problem domain; 17 | # this is used to decompose the domain for parallel calculations. 18 | max_grid_size = 32 19 | 20 | # Number of particles per cell 21 | nppc = 10 22 | 23 | # Verbosity 24 | verbose = true # set to true to get more verbosity 25 | -------------------------------------------------------------------------------- /04_ParticleMesh/main.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | using namespace amrex; 11 | 12 | struct TestParams { 13 | int nx; 14 | int ny; 15 | int nz; 16 | int max_grid_size; 17 | int nppc; 18 | bool verbose; 19 | }; 20 | 21 | void testParticleMesh(TestParams& parms) 22 | { 23 | 24 | RealBox real_box; 25 | for (int n = 0; n < BL_SPACEDIM; n++) { 26 | real_box.setLo(n, 0.0); 27 | real_box.setHi(n, 1.0); 28 | } 29 | 30 | IntVect domain_lo(AMREX_D_DECL(0, 0, 0)); 31 | IntVect domain_hi(AMREX_D_DECL(parms.nx - 1, parms.ny - 1, parms.nz-1)); 32 | const Box domain(domain_lo, domain_hi); 33 | 34 | // This sets the boundary conditions to be doubly or triply periodic 35 | int is_per[BL_SPACEDIM]; 36 | for (int i = 0; i < BL_SPACEDIM; i++) 37 | is_per[i] = 1; 38 | Geometry geom(domain, &real_box, CoordSys::cartesian, is_per); 39 | 40 | BoxArray ba(domain); 41 | ba.maxSize(parms.max_grid_size); 42 | if (parms.verbose && ParallelDescriptor::IOProcessor()) { 43 | std::cout << "Number of boxes : " << ba[0].size() << '\n' << '\n'; 44 | } 45 | 46 | DistributionMapping dmap(ba); 47 | 48 | MultiFab partMF(ba, dmap, 1 + BL_SPACEDIM, 1); 49 | partMF.setVal(0.0); 50 | 51 | typedef ParticleContainer<1 + 2*BL_SPACEDIM> MyParticleContainer; 52 | MyParticleContainer myPC(geom, dmap, ba); 53 | myPC.SetVerbose(false); 54 | 55 | int num_particles = parms.nppc * parms.nx * parms.ny * parms.nz; 56 | if (ParallelDescriptor::IOProcessor()) 57 | std::cout << "Total number of particles : " << num_particles << '\n' << '\n'; 58 | 59 | bool serialize = true; 60 | int iseed = 451; 61 | Real mass = 10.0; 62 | 63 | MyParticleContainer::ParticleInitData pdata = {mass, AMREX_D_DECL(1.0, 2.0, 3.0), AMREX_D_DECL(0.0, 0.0, 0.0)}; 64 | myPC.InitRandom(num_particles, iseed, pdata, serialize); 65 | 66 | int nc = 1 + BL_SPACEDIM; 67 | const auto plo = geom.ProbLoArray(); 68 | const auto dxi = geom.InvCellSizeArray(); 69 | amrex::ParticleToMesh(myPC, partMF, 0, 70 | [=] AMREX_GPU_DEVICE (const MyParticleContainer::ParticleType& p, 71 | amrex::Array4 const& rho) 72 | { 73 | amrex::Real lx = (p.pos(0) - plo[0]) * dxi[0] + 0.5; 74 | amrex::Real ly = (p.pos(1) - plo[1]) * dxi[1] + 0.5; 75 | amrex::Real lz = (p.pos(2) - plo[2]) * dxi[2] + 0.5; 76 | 77 | int i = amrex::Math::floor(lx); 78 | int j = amrex::Math::floor(ly); 79 | int k = amrex::Math::floor(lz); 80 | 81 | amrex::Real xint = lx - i; 82 | amrex::Real yint = ly - j; 83 | amrex::Real zint = lz - k; 84 | 85 | amrex::Real sx[] = {1.-xint, xint}; 86 | amrex::Real sy[] = {1.-yint, yint}; 87 | amrex::Real sz[] = {1.-zint, zint}; 88 | 89 | for (int kk = 0; kk <= 1; ++kk) { 90 | for (int jj = 0; jj <= 1; ++jj) { 91 | for (int ii = 0; ii <= 1; ++ii) { 92 | amrex::Gpu::Atomic::AddNoRet(&rho(i+ii-1, j+jj-1, k+kk-1, 0), 93 | sx[ii]*sy[jj]*sz[kk]*p.rdata(0)); 94 | } 95 | } 96 | } 97 | 98 | for (int comp=1; comp < nc; ++comp) { 99 | for (int kk = 0; kk <= 1; ++kk) { 100 | for (int jj = 0; jj <= 1; ++jj) { 101 | for (int ii = 0; ii <= 1; ++ii) { 102 | amrex::Gpu::Atomic::AddNoRet(&rho(i+ii-1, j+jj-1, k+kk-1, comp), 103 | sx[ii]*sy[jj]*sz[kk]*p.rdata(0)*p.rdata(comp)); 104 | } 105 | } 106 | } 107 | } 108 | }); 109 | 110 | MultiFab acceleration(ba, dmap, BL_SPACEDIM, 1); 111 | acceleration.setVal(5.0); 112 | 113 | nc = BL_SPACEDIM; 114 | amrex::MeshToParticle(myPC, acceleration, 0, 115 | [=] AMREX_GPU_DEVICE (MyParticleContainer::ParticleType& p, 116 | amrex::Array4 const& acc) 117 | { 118 | amrex::Real lx = (p.pos(0) - plo[0]) * dxi[0] + 0.5; 119 | amrex::Real ly = (p.pos(1) - plo[1]) * dxi[1] + 0.5; 120 | amrex::Real lz = (p.pos(2) - plo[2]) * dxi[2] + 0.5; 121 | 122 | int i = amrex::Math::floor(lx); 123 | int j = amrex::Math::floor(ly); 124 | int k = amrex::Math::floor(lz); 125 | 126 | amrex::Real xint = lx - i; 127 | amrex::Real yint = ly - j; 128 | amrex::Real zint = lz - k; 129 | 130 | amrex::Real sx[] = {1.-xint, xint}; 131 | amrex::Real sy[] = {1.-yint, yint}; 132 | amrex::Real sz[] = {1.-zint, zint}; 133 | 134 | for (int comp=0; comp < nc; ++comp) { 135 | for (int kk = 0; kk <= 1; ++kk) { 136 | for (int jj = 0; jj <= 1; ++jj) { 137 | for (int ii = 0; ii <= 1; ++ii) { 138 | p.rdata(4+comp) += sx[ii]*sy[jj]*sz[kk]*acc(i+ii-1,j+jj-1,k+kk-1,comp); 139 | } 140 | } 141 | } 142 | } 143 | }); 144 | 145 | WriteSingleLevelPlotfile("plot", partMF, 146 | {"density", "vx", "vy", "vz"}, 147 | geom, 0.0, 0); 148 | 149 | myPC.Checkpoint("plot", "particle0"); 150 | } 151 | 152 | int main(int argc, char* argv[]) 153 | { 154 | amrex::Initialize(argc,argv); 155 | 156 | ParmParse pp; 157 | 158 | TestParams parms; 159 | 160 | pp.get("nx", parms.nx); 161 | pp.get("ny", parms.ny); 162 | pp.get("nz", parms.nz); 163 | pp.get("max_grid_size", parms.max_grid_size); 164 | pp.get("nppc", parms.nppc); 165 | if (parms.nppc < 1 && ParallelDescriptor::IOProcessor()) 166 | amrex::Abort("Must specify at least one particle per cell"); 167 | 168 | parms.verbose = false; 169 | pp.query("verbose", parms.verbose); 170 | 171 | if (parms.verbose && ParallelDescriptor::IOProcessor()) { 172 | std::cout << std::endl; 173 | std::cout << "Number of particles per cell : "; 174 | std::cout << parms.nppc << std::endl; 175 | std::cout << "Size of domain : "; 176 | std::cout << parms.nx << " " << parms.ny << " " << parms.nz << std::endl; 177 | } 178 | 179 | testParticleMesh(parms); 180 | 181 | amrex::Finalize(); 182 | } 183 | -------------------------------------------------------------------------------- /05_ParticleMesh_MultiLevel/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(_sources main.cpp) 2 | set(_input_files inputs Visualization.ipynb) 3 | 4 | setup_tutorial(_sources _input_files) 5 | 6 | unset(_sources) 7 | unset(_input_files) 8 | -------------------------------------------------------------------------------- /05_ParticleMesh_MultiLevel/Visualization.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "ee0b42ec", 6 | "metadata": {}, 7 | "source": [ 8 | "## Example: Particle Mesh Interpolation with multiple levels\n", 9 | "\n", 10 | "### What Features Are We Using\n", 11 | "\n", 12 | "* Mesh data\n", 13 | "* Particle data\n", 14 | "* Static mesh refinement\n", 15 | "\n", 16 | "The examples so far used domain decomposition but no mesh refinement. This time, we alter the previous example\n", 17 | "to work on hierarchy of meshes. We now have a `Vector`, `Vector`, `Vector`,\n", 18 | "and a `Vector` telling us the refinement ratios between levels. We set up the grids to refine the central region of the domain with each level:\n", 19 | "\n", 20 | "```C++\n", 21 | " Box domain = base_domain;\n", 22 | " IntVect size = IntVect(AMREX_D_DECL(parms.nx, parms.ny, parms.nz));\n", 23 | " for (int lev = 0; lev < parms.nlevs; ++lev)\n", 24 | " {\n", 25 | " ba[lev].define(domain);\n", 26 | " ba[lev].maxSize(parms.max_grid_size);\n", 27 | " dm[lev].define(ba[lev]);\n", 28 | " domain.grow(-size/4); // fine level cover the middle of the coarse domain\n", 29 | " domain.refine(2);\n", 30 | " }\n", 31 | "```\n", 32 | "\n", 33 | "We again use \"Cloud in cell\" interpolation with randomly positioned particles. To handle particles near coarse-fine interfaces, we use the following algorithm. Particles with positions on the coarse level are deposited with the coarse cell spacing, and any mass that lands under a fine level is interpolated using piece-wise constant interpolation. Particles on the fine level are deposited with the fine cell spacing, and mass that \"hangs off\" the fine level is added to the coarse cell below it. \n", 34 | "\n", 35 | "The inputs parameters are the same as before, except you can now specify the number of levels to use as `nlevs`." 36 | ] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "execution_count": null, 41 | "id": "83170d52", 42 | "metadata": {}, 43 | "outputs": [], 44 | "source": [ 45 | "import yt" 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": null, 51 | "id": "b1a070d3", 52 | "metadata": {}, 53 | "outputs": [], 54 | "source": [ 55 | "ds = yt.load(\"plt00000\")" 56 | ] 57 | }, 58 | { 59 | "cell_type": "code", 60 | "execution_count": null, 61 | "id": "cac891d9", 62 | "metadata": {}, 63 | "outputs": [], 64 | "source": [ 65 | "sl = yt.SlicePlot(ds, 2, 'density')" 66 | ] 67 | }, 68 | { 69 | "cell_type": "code", 70 | "execution_count": null, 71 | "id": "cdee59ea", 72 | "metadata": {}, 73 | "outputs": [], 74 | "source": [ 75 | "sl.annotate_grids()" 76 | ] 77 | }, 78 | { 79 | "cell_type": "code", 80 | "execution_count": null, 81 | "id": "81f8fd11", 82 | "metadata": {}, 83 | "outputs": [], 84 | "source": [ 85 | "ad = ds.all_data()\n", 86 | "ad.quantities.weighted_average_quantity('density', 'cell_volume')" 87 | ] 88 | } 89 | ], 90 | "metadata": { 91 | "kernelspec": { 92 | "display_name": "Python 3", 93 | "language": "python", 94 | "name": "python3" 95 | }, 96 | "language_info": { 97 | "codemirror_mode": { 98 | "name": "ipython", 99 | "version": 3 100 | }, 101 | "file_extension": ".py", 102 | "mimetype": "text/x-python", 103 | "name": "python", 104 | "nbconvert_exporter": "python", 105 | "pygments_lexer": "ipython3", 106 | "version": "3.8.8" 107 | } 108 | }, 109 | "nbformat": 4, 110 | "nbformat_minor": 5 111 | } 112 | -------------------------------------------------------------------------------- /05_ParticleMesh_MultiLevel/inputs: -------------------------------------------------------------------------------- 1 | 2 | # Domain size 3 | 4 | #nx = 32 # number of grid points along the x axis 5 | #ny = 32 # number of grid points along the y axis 6 | #nz = 32 # number of grid points along the z axis 7 | 8 | #nx = 64 # number of grid points along the x axis 9 | #ny = 64 # number of grid points along the y axis 10 | #nz = 64 # number of grid points along the z axis 11 | 12 | nx = 128 # number of grid points along the x axis 13 | ny = 128 # number of grid points along the y axis 14 | nz = 128 # number of grid points along the z axis 15 | 16 | # Maximum allowable size of each subdomain in the problem domain; 17 | # this is used to decompose the domain for parallel calculations. 18 | max_grid_size = 32 19 | 20 | # Number of particles per cell 21 | nppc = 10 22 | 23 | # Verbosity 24 | verbose = true # set to true to get more verbosity 25 | 26 | # number of levels 27 | nlevs = 2 -------------------------------------------------------------------------------- /05_ParticleMesh_MultiLevel/main.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | #include 5 | #include 6 | #include "AMReX_Particles.H" 7 | #include "AMReX_PlotFileUtil.H" 8 | #include 9 | 10 | using namespace amrex; 11 | 12 | struct TestParams { 13 | int nx; 14 | int ny; 15 | int nz; 16 | int nlevs; 17 | int max_grid_size; 18 | int nppc; 19 | bool verbose; 20 | }; 21 | 22 | void testParticleMesh (TestParams& parms) 23 | { 24 | Vector rr(parms.nlevs-1); 25 | for (int lev = 1; lev < parms.nlevs; lev++) 26 | rr[lev-1] = IntVect(AMREX_D_DECL(2,2,2)); 27 | 28 | RealBox real_box; 29 | for (int n = 0; n < BL_SPACEDIM; n++) { 30 | real_box.setLo(n, 0.0); 31 | real_box.setHi(n, 1.0); 32 | } 33 | 34 | IntVect domain_lo(AMREX_D_DECL(0, 0, 0)); 35 | IntVect domain_hi(AMREX_D_DECL(parms.nx - 1, parms.ny - 1, parms.nz-1)); 36 | const Box base_domain(domain_lo, domain_hi); 37 | 38 | // This sets the boundary conditions to be doubly or triply periodic 39 | int is_per[BL_SPACEDIM]; 40 | for (int i = 0; i < BL_SPACEDIM; i++) 41 | is_per[i] = 1; 42 | 43 | Vector geom(parms.nlevs); 44 | geom[0].define(base_domain, &real_box, CoordSys::cartesian, is_per); 45 | for (int lev = 1; lev < parms.nlevs; lev++) { 46 | geom[lev].define(amrex::refine(geom[lev-1].Domain(), rr[lev-1]), 47 | &real_box, CoordSys::cartesian, is_per); 48 | } 49 | 50 | Vector ba(parms.nlevs); 51 | Vector dm(parms.nlevs); 52 | 53 | Box domain = base_domain; 54 | IntVect size = IntVect(AMREX_D_DECL(parms.nx, parms.ny, parms.nz)); 55 | for (int lev = 0; lev < parms.nlevs; ++lev) 56 | { 57 | ba[lev].define(domain); 58 | ba[lev].maxSize(parms.max_grid_size); 59 | dm[lev].define(ba[lev]); 60 | domain.grow(-size/4); // fine level cover the middle of the coarse domain 61 | domain.refine(2); 62 | } 63 | 64 | Vector density(parms.nlevs); 65 | for (int lev = 0; lev < parms.nlevs; lev++) { 66 | density[lev].define(ba[lev], dm[lev], 1, 1); 67 | density[lev].setVal(0.0); 68 | } 69 | 70 | typedef ParticleContainer<1> MyParticleContainer; 71 | MyParticleContainer myPC(geom, dm, ba, rr); 72 | myPC.SetVerbose(false); 73 | 74 | int num_particles = parms.nppc * parms.nx * parms.ny * parms.nz; 75 | amrex::Print() << "Total number of particles : " << num_particles << '\n' << '\n'; 76 | 77 | bool serialize = true; 78 | int iseed = 451; 79 | Real mass = 10.0; 80 | 81 | MyParticleContainer::ParticleInitData pdata = {{mass}, {}, {}, {}}; 82 | myPC.InitRandom(num_particles, iseed, pdata, serialize); 83 | 84 | amrex::ParticleToMesh(myPC, GetVecOfPtrs(density), 0, parms.nlevs-1, 85 | [=] AMREX_GPU_DEVICE (const MyParticleContainer::ParticleType& p, 86 | amrex::Array4 const& rho, 87 | amrex::GpuArray const& plo, 88 | amrex::GpuArray const& dxi) noexcept 89 | { 90 | amrex::Real lx = (p.pos(0) - plo[0]) * dxi[0] + 0.5; 91 | amrex::Real ly = (p.pos(1) - plo[1]) * dxi[1] + 0.5; 92 | amrex::Real lz = (p.pos(2) - plo[2]) * dxi[2] + 0.5; 93 | 94 | int i = static_cast(amrex::Math::floor(lx)); 95 | int j = static_cast(amrex::Math::floor(ly)); 96 | int k = static_cast(amrex::Math::floor(lz)); 97 | 98 | amrex::Real xint = lx - i; 99 | amrex::Real yint = ly - j; 100 | amrex::Real zint = lz - k; 101 | 102 | amrex::Real sx[] = {1.-xint, xint}; 103 | amrex::Real sy[] = {1.-yint, yint}; 104 | amrex::Real sz[] = {1.-zint, zint}; 105 | 106 | for (int kk = 0; kk <= 1; ++kk) { 107 | for (int jj = 0; jj <= 1; ++jj) { 108 | for (int ii = 0; ii <= 1; ++ii) { 109 | amrex::Gpu::Atomic::AddNoRet(&rho(i+ii-1, j+jj-1, k+kk-1, 0), 110 | sx[ii]*sy[jj]*sz[kk]*p.rdata(0)); 111 | } 112 | } 113 | } 114 | }); 115 | 116 | Vector varnames; 117 | varnames.push_back("density"); 118 | 119 | Vector particle_varnames; 120 | particle_varnames.push_back("mass"); 121 | 122 | Vector level_steps; 123 | level_steps.push_back(0); 124 | level_steps.push_back(0); 125 | 126 | int output_levs = parms.nlevs; 127 | 128 | Vector outputMF(output_levs); 129 | Vector outputRR(output_levs); 130 | for (int lev = 0; lev < output_levs; ++lev) { 131 | outputMF[lev] = &density[lev]; 132 | outputRR[lev] = IntVect(AMREX_D_DECL(2, 2, 2)); 133 | } 134 | 135 | WriteMultiLevelPlotfile("plt00000", output_levs, outputMF, 136 | varnames, geom, 0.0, level_steps, outputRR); 137 | myPC.Checkpoint("plt00000", "particle0", true, particle_varnames); 138 | } 139 | 140 | int main(int argc, char* argv[]) 141 | { 142 | amrex::Initialize(argc,argv); 143 | 144 | ParmParse pp; 145 | 146 | TestParams parms; 147 | 148 | pp.get("nx", parms.nx); 149 | pp.get("ny", parms.ny); 150 | pp.get("nz", parms.nz); 151 | pp.get("max_grid_size", parms.max_grid_size); 152 | pp.get("nppc", parms.nppc); 153 | pp.get("nlevs", parms.nlevs); 154 | if (parms.nppc < 1 && ParallelDescriptor::IOProcessor()) 155 | amrex::Abort("Must specify at least one particle per cell"); 156 | 157 | parms.verbose = false; 158 | pp.query("verbose", parms.verbose); 159 | 160 | if (parms.verbose) { 161 | amrex::Print() << std::endl; 162 | amrex::Print() << "Number of particles per cell : "; 163 | amrex::Print() << parms.nppc << std::endl; 164 | amrex::Print() << "Size of domain : "; 165 | amrex::Print() << parms.nx << " " << parms.ny << " " << parms.nz << std::endl; 166 | } 167 | 168 | testParticleMesh(parms); 169 | 170 | amrex::Finalize(); 171 | } 172 | -------------------------------------------------------------------------------- /06_Advection_Amr/Adv_K.H: -------------------------------------------------------------------------------- 1 | #ifndef _Adv_K_H_ 2 | #define _Adv_K_H_ 3 | 4 | #include 5 | #include 6 | 7 | AMREX_GPU_DEVICE 8 | AMREX_FORCE_INLINE 9 | void conservative(int i, int j, int k, 10 | amrex::Array4 const& phi_out, 11 | amrex::Array4 const& phi_in, 12 | AMREX_D_DECL(amrex::Array4 const& flxx, 13 | amrex::Array4 const& flxy, 14 | amrex::Array4 const& flxz), 15 | AMREX_D_DECL(amrex::Real dtdx, amrex::Real dtdy, amrex::Real dtdz)) 16 | { 17 | phi_out(i,j,k) = phi_in(i,j,k) + 18 | ( AMREX_D_TERM( (flxx(i,j,k) - flxx(i+1,j,k)) * dtdx, 19 | + (flxy(i,j,k) - flxy(i,j+1,k)) * dtdy, 20 | + (flxz(i,j,k) - flxz(i,j,k+1)) * dtdz ) ); 21 | } 22 | 23 | #endif 24 | -------------------------------------------------------------------------------- /06_Advection_Amr/AdvancePhiAllLevels.cpp: -------------------------------------------------------------------------------- 1 | #include "AmrCoreAdv.H" 2 | #include "Kernels.H" 3 | 4 | #include 5 | 6 | using namespace amrex; 7 | 8 | // advance all levels for a single time step 9 | void 10 | AmrCoreAdv::AdvancePhiAllLevels (Real time, Real dt_lev, int /*iteration*/) 11 | { 12 | constexpr int num_grow = 3; 13 | 14 | Vector< Array > fluxes(finest_level+1); 15 | for (int lev = 0; lev <= finest_level; lev++) 16 | { 17 | for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) 18 | { 19 | BoxArray ba = grids[lev]; 20 | ba.surroundingNodes(idim); 21 | fluxes[lev][idim] = MultiFab(ba, dmap[lev], 1, 0); 22 | } 23 | } 24 | 25 | for (int lev = 0; lev <= finest_level; lev++) 26 | { 27 | std::swap(phi_old[lev], phi_new[lev]); 28 | t_old[lev] = t_new[lev]; 29 | t_new[lev] += dt_lev; 30 | 31 | const auto dx = geom[lev].CellSizeArray(); 32 | AMREX_D_TERM(Real dtdx = dt_lev/dx[0];, 33 | Real dtdy = dt_lev/dx[1];, 34 | Real dtdz = dt_lev/dx[2]); 35 | 36 | // State with ghost cells 37 | MultiFab Sborder(grids[lev], dmap[lev], phi_new[lev].nComp(), num_grow); 38 | FillPatch(lev, time, Sborder, 0, Sborder.nComp()); 39 | 40 | #ifdef AMREX_USE_OMP 41 | #pragma omp parallel if (Gpu::notInLaunchRegion()) 42 | #endif 43 | { 44 | FArrayBox tmpfab; 45 | for (MFIter mfi(phi_new[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi) 46 | { 47 | AMREX_D_TERM(Array4 velx = facevel[lev][0].const_array(mfi);, 48 | Array4 vely = facevel[lev][1].const_array(mfi);, 49 | Array4 velz = facevel[lev][2].const_array(mfi)); 50 | 51 | const Box& bx = mfi.tilebox(); 52 | const Box& gbx = amrex::grow(bx, 1); 53 | 54 | Array4 statein = Sborder.const_array(mfi); 55 | 56 | AMREX_D_TERM(Array4 fluxx = fluxes[lev][0].array(mfi);, 57 | Array4 fluxy = fluxes[lev][1].array(mfi);, 58 | Array4 fluxz = fluxes[lev][2].array(mfi)); 59 | 60 | int ntmpcomps = (AMREX_SPACEDIM == 2) ? 4 : 11; 61 | tmpfab.resize(amrex::grow(bx,2),ntmpcomps); 62 | Elixir tmpeli = tmpfab.elixir(); 63 | int itmp = 0; 64 | 65 | Array4 slope2 = tmpfab.array(itmp); 66 | Array4 slope2_c = slope2; 67 | itmp += 1; 68 | Array4 slope4 = tmpfab.array(itmp); 69 | Array4 slope4_c = slope4; 70 | itmp += 1; 71 | 72 | // compute longitudinal fluxes 73 | // =========================== 74 | 75 | // x ------------------------- 76 | Array4 phix = tmpfab.array(itmp); 77 | Array4 phix_c = phix; 78 | itmp += 1; 79 | 80 | amrex::launch(amrex::grow(gbx,Direction::x,1), 81 | [=] AMREX_GPU_DEVICE (const Box& tbx) 82 | { 83 | slopex2(tbx, slope2, statein); 84 | }); 85 | 86 | amrex::launch(gbx, 87 | [=] AMREX_GPU_DEVICE (const Box& tbx) 88 | { 89 | slopex4(tbx, slope4, statein, slope2_c); 90 | }); 91 | 92 | Box b = gbx; 93 | amrex::ParallelFor(b.grow(Direction::x,-1).surroundingNodes(Direction::x), 94 | [=] AMREX_GPU_DEVICE (int i, int j, int k) 95 | { 96 | flux_x(i, j, k, phix, statein, velx, slope4_c, dtdx); 97 | }); 98 | 99 | // y ------------------------- 100 | Array4 phiy = tmpfab.array(itmp); 101 | Array4 phiy_c = phiy; 102 | itmp += 1; 103 | 104 | amrex::launch(amrex::grow(gbx,Direction::y,1), 105 | [=] AMREX_GPU_DEVICE (const Box& tbx) 106 | { 107 | slopey2(tbx, slope2, statein); 108 | }); 109 | 110 | amrex::launch(gbx, 111 | [=] AMREX_GPU_DEVICE (const Box& tbx) 112 | { 113 | slopey4(tbx, slope4, statein, slope2_c); 114 | }); 115 | 116 | b = gbx; 117 | amrex::ParallelFor(b.grow(Direction::y,-1).surroundingNodes(Direction::y), 118 | [=] AMREX_GPU_DEVICE (int i, int j, int k) 119 | { 120 | flux_y(i, j, k, phiy, statein, vely, slope4_c, dtdy); 121 | }); 122 | 123 | #if (AMREX_SPACEDIM > 2) 124 | // z ------------------------- 125 | Array4 phiz = tmpfab.array(itmp); 126 | Array4 phiz_c = phiz; 127 | itmp += 1; 128 | 129 | amrex::launch(amrex::grow(gbx,Direction::z,1), 130 | [=] AMREX_GPU_DEVICE (const Box& tbx) 131 | { 132 | slopez2(tbx, slope2, statein); 133 | }); 134 | 135 | amrex::launch(gbx, 136 | [=] AMREX_GPU_DEVICE (const Box& tbx) 137 | { 138 | slopez4(tbx, slope4, statein, slope2_c); 139 | }); 140 | 141 | b = gbx; 142 | amrex::ParallelFor(b.grow(Direction::z,-1).surroundingNodes(Direction::z), 143 | [=] AMREX_GPU_DEVICE (int i, int j, int k) 144 | { 145 | flux_z(i, j, k, phiz, statein, velz, slope4_c, dtdz); 146 | }); 147 | 148 | // compute transverse fluxes (3D only) 149 | // =================================== 150 | 151 | // xy -------------------- 152 | Array4 phix_y = tmpfab.array(itmp); 153 | Array4 phix_y_c = phix_y; 154 | itmp += 1; 155 | 156 | b = bx; 157 | amrex::ParallelFor(b.grow(Direction::z,1).surroundingNodes(Direction::x), 158 | [=] AMREX_GPU_DEVICE (int i, int j, int k) 159 | { 160 | flux_xy(i, j, k, phix_y, 161 | velx, vely, 162 | phix_c, phiy_c, 163 | dtdy); 164 | }); 165 | 166 | // xz -------------------- 167 | Array4 phix_z = tmpfab.array(itmp); 168 | Array4 phix_z_c = phix_z; 169 | itmp += 1; 170 | 171 | b = bx; 172 | amrex::ParallelFor(b.grow(Direction::y,1).surroundingNodes(Direction::x), 173 | [=] AMREX_GPU_DEVICE (int i, int j, int k) 174 | { 175 | flux_xz(i, j, k, phix_z, 176 | velx, velz, 177 | phix, phiz, 178 | dtdz); 179 | }); 180 | 181 | // yx -------------------- 182 | Array4 phiy_x = tmpfab.array(itmp); 183 | Array4 phiy_x_c = phiy_x; 184 | itmp += 1; 185 | 186 | b = bx; 187 | amrex::ParallelFor(b.grow(Direction::z,1).surroundingNodes(Direction::y), 188 | [=] AMREX_GPU_DEVICE (int i, int j, int k) 189 | { 190 | flux_yx(i, j, k, phiy_x, 191 | velx, vely, 192 | phix, phiy, 193 | dtdx); 194 | }); 195 | 196 | // yz -------------------- 197 | Array4 phiy_z = tmpfab.array(itmp); 198 | Array4 phiy_z_c = phiy_z; 199 | itmp += 1; 200 | 201 | b = bx; 202 | amrex::ParallelFor(b.grow(Direction::x,1).surroundingNodes(Direction::y), 203 | [=] AMREX_GPU_DEVICE (int i, int j, int k) 204 | { 205 | flux_yz(i, j, k, phiy_z, 206 | vely, velz, 207 | phiy, phiz, 208 | dtdz); 209 | }); 210 | 211 | // zx -------------------- 212 | Array4 phiz_x = tmpfab.array(itmp); 213 | Array4 phiz_x_c = phiz_x; 214 | itmp += 1; 215 | 216 | b = bx; 217 | amrex::ParallelFor(b.grow(Direction::y,1).surroundingNodes(Direction::z), 218 | [=] AMREX_GPU_DEVICE (int i, int j, int k) 219 | { 220 | flux_zx(i, j, k, phiz_x, 221 | velx, velz, 222 | phix, phiz, 223 | dtdx); 224 | }); 225 | 226 | // zy -------------------- 227 | Array4 phiz_y = tmpfab.array(itmp); 228 | Array4 phiz_y_c = phiz_y; 229 | itmp += 1; 230 | 231 | b = bx; 232 | amrex::ParallelFor(b.grow(Direction::x,1).surroundingNodes(Direction::z), 233 | [=] AMREX_GPU_DEVICE (int i, int j, int k) 234 | { 235 | flux_zy(i, j, k, phiz_y, 236 | vely, velz, 237 | phiy, phiz, 238 | dtdy); 239 | }); 240 | #endif 241 | 242 | // final edge states 243 | // =========================== 244 | amrex::ParallelFor(mfi.nodaltilebox(0), 245 | [=] AMREX_GPU_DEVICE (int i, int j, int k) 246 | { 247 | create_flux_x(i, j, k, fluxx, 248 | AMREX_D_DECL(velx,vely,velz), 249 | #if (AMREX_SPACEDIM == 3) 250 | phix_c, phiy_z_c, phiz_y_c, 251 | dtdy, dtdz); 252 | #else 253 | phix_c, phiy_c, 254 | dtdy); 255 | #endif 256 | }); 257 | 258 | amrex::ParallelFor(mfi.nodaltilebox(1), 259 | [=] AMREX_GPU_DEVICE (int i, int j, int k) 260 | { 261 | create_flux_y(i, j, k, fluxy, 262 | AMREX_D_DECL(velx,vely,velz), 263 | #if (AMREX_SPACEDIM == 3) 264 | phiy_c, phix_z_c, phiz_x_c, 265 | dtdx, dtdz); 266 | #else 267 | phiy_c, phix_c, 268 | dtdx); 269 | #endif 270 | }); 271 | 272 | #if (AMREX_SPACEDIM == 3) 273 | amrex::ParallelFor(mfi.nodaltilebox(2), 274 | [=] AMREX_GPU_DEVICE (int i, int j, int k) 275 | { 276 | create_flux_z(i, j, k, fluxz, 277 | velx, vely, velz, 278 | phiz_c, phix_y_c, phiy_x_c, 279 | dtdx, dtdy); 280 | }); 281 | #endif 282 | AMREX_ASSERT(itmp == ntmpcomps); 283 | } // end mfi 284 | } // end omp 285 | } // end lev 286 | 287 | // ======================================================= 288 | // Average down the fluxes before using them to update phi 289 | // ======================================================= 290 | for (int lev = finest_level; lev > 0; lev--) 291 | { 292 | average_down_faces(amrex::GetArrOfConstPtrs(fluxes[lev ]), 293 | amrex::GetArrOfPtrs (fluxes[lev-1]), 294 | refRatio(lev-1), Geom(lev-1)); 295 | } 296 | 297 | for (int lev = 0; lev <= finest_level; lev++) 298 | { 299 | 300 | #ifdef AMREX_USE_OMP 301 | #pragma omp parallel if (Gpu::notInLaunchRegion()) 302 | #endif 303 | { 304 | const auto dx = geom[lev].CellSizeArray(); 305 | AMREX_D_TERM(Real dtdx = dt_lev/dx[0];, 306 | Real dtdy = dt_lev/dx[1];, 307 | Real dtdz = dt_lev/dx[2]); 308 | 309 | // =========================================== 310 | // Compute phi_new using a conservative update 311 | // =========================================== 312 | for (MFIter mfi(phi_new[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi) 313 | { 314 | Array4 statein = phi_old[lev].const_array(mfi); 315 | Array4 stateout = phi_new[lev].array(mfi); 316 | 317 | AMREX_D_TERM(Array4 fluxx = fluxes[lev][0].const_array(mfi);, 318 | Array4 fluxy = fluxes[lev][1].const_array(mfi);, 319 | Array4 fluxz = fluxes[lev][2].const_array(mfi)); 320 | 321 | const Box& bx = mfi.tilebox(); 322 | 323 | amrex::ParallelFor(bx, 324 | [=] AMREX_GPU_DEVICE (int i, int j, int k) 325 | { 326 | conservative(i, j, k, 327 | stateout, statein, 328 | AMREX_D_DECL(fluxx,fluxy,fluxz), 329 | AMREX_D_DECL(dtdx,dtdy,dtdz)); 330 | }); 331 | } // end mfi 332 | } // end omp 333 | } // end lev 334 | } 335 | -------------------------------------------------------------------------------- /06_Advection_Amr/AdvancePhiAtLevel.cpp: -------------------------------------------------------------------------------- 1 | #include "AmrCoreAdv.H" 2 | #include "Kernels.H" 3 | 4 | using namespace amrex; 5 | 6 | // Advance a single level for a single time step, updates flux registers 7 | void 8 | AmrCoreAdv::AdvancePhiAtLevel (int lev, Real time, Real dt_lev, int /*iteration*/, int /*ncycle*/) 9 | { 10 | constexpr int num_grow = 3; 11 | 12 | std::swap(phi_old[lev], phi_new[lev]); 13 | 14 | MultiFab& S_new = phi_new[lev]; 15 | 16 | const Real dx = geom[lev].CellSize(0); 17 | const Real dy = geom[lev].CellSize(1); 18 | const Real dz = (AMREX_SPACEDIM == 2) ? Real(1.0) : geom[lev].CellSize(2); 19 | AMREX_D_TERM(Real dtdx = dt_lev/dx;, 20 | Real dtdy = dt_lev/dy;, 21 | Real dtdz = dt_lev/dz); 22 | 23 | MultiFab fluxes[AMREX_SPACEDIM]; 24 | if (do_reflux) 25 | { 26 | for (int i = 0; i < AMREX_SPACEDIM; ++i) 27 | { 28 | BoxArray ba = grids[lev]; 29 | ba.surroundingNodes(i); 30 | fluxes[i].define(ba, dmap[lev], S_new.nComp(), 0); 31 | } 32 | } 33 | 34 | // State with ghost cells 35 | MultiFab Sborder(grids[lev], dmap[lev], S_new.nComp(), num_grow); 36 | FillPatch(lev, time, Sborder, 0, Sborder.nComp()); 37 | 38 | #ifdef AMREX_USE_OMP 39 | #pragma omp parallel if (Gpu::notInLaunchRegion()) 40 | #endif 41 | { 42 | FArrayBox tmpfab; 43 | for (MFIter mfi(S_new,TilingIfNotGPU()); mfi.isValid(); ++mfi) 44 | { 45 | AMREX_ASSERT(S_new.nComp() == 1); 46 | 47 | // ======== GET FACE VELOCITY ========= 48 | 49 | AMREX_D_TERM(Array4 velx = facevel[lev][0].const_array(mfi);, 50 | Array4 vely = facevel[lev][1].const_array(mfi);, 51 | Array4 velz = facevel[lev][2].const_array(mfi)); 52 | 53 | // ======== FLUX CALC AND UPDATE ========= 54 | 55 | const Box& bx = mfi.tilebox(); 56 | const Box& gbx = amrex::grow(bx, 1); 57 | 58 | Array4 statein = Sborder.const_array(mfi); 59 | Array4 stateout = S_new.array(mfi); 60 | 61 | int ntmpcomps = (AMREX_SPACEDIM == 2) ? 6 : 14; 62 | tmpfab.resize(amrex::grow(bx,2),ntmpcomps); 63 | Elixir tmpeli = tmpfab.elixir(); 64 | int itmp = 0; 65 | 66 | AMREX_D_TERM(Array4 tfluxx = tmpfab.array(itmp++);, 67 | Array4 tfluxy = tmpfab.array(itmp++);, 68 | Array4 tfluxz = tmpfab.array(itmp++)); 69 | 70 | Array4 slope2 = tmpfab.array(itmp++); 71 | Array4 slope2_c = slope2; 72 | Array4 slope4 = tmpfab.array(itmp++); 73 | Array4 slope4_c = slope4; 74 | 75 | // compute longitudinal fluxes 76 | // =========================== 77 | 78 | // x ------------------------- 79 | Array4 phix = tmpfab.array(itmp++); 80 | Array4 phix_c = phix; 81 | 82 | amrex::launch(amrex::grow(gbx,Direction::x,1), 83 | [=] AMREX_GPU_DEVICE (const Box& tbx) 84 | { 85 | slopex2(tbx, slope2, statein); 86 | }); 87 | 88 | amrex::launch(gbx, 89 | [=] AMREX_GPU_DEVICE (const Box& tbx) 90 | { 91 | slopex4(tbx, slope4, statein, slope2_c); 92 | }); 93 | 94 | Box b = gbx; 95 | amrex::ParallelFor(b.grow(Direction::x,-1).surroundingNodes(Direction::x), 96 | [=] AMREX_GPU_DEVICE (int i, int j, int k) 97 | { 98 | flux_x(i, j, k, phix, statein, velx, slope4_c, dtdx); 99 | }); 100 | 101 | // y ------------------------- 102 | Array4 phiy = tmpfab.array(itmp++); 103 | Array4 phiy_c = phiy; 104 | 105 | amrex::launch(amrex::grow(gbx,Direction::y,1), 106 | [=] AMREX_GPU_DEVICE (const Box& tbx) 107 | { 108 | slopey2(tbx, slope2, statein); 109 | }); 110 | 111 | amrex::launch(gbx, 112 | [=] AMREX_GPU_DEVICE (const Box& tbx) 113 | { 114 | slopey4(tbx, slope4, statein, slope2_c); 115 | }); 116 | 117 | b = gbx; 118 | amrex::ParallelFor(b.grow(Direction::y,-1).surroundingNodes(Direction::y), 119 | [=] AMREX_GPU_DEVICE (int i, int j, int k) 120 | { 121 | flux_y(i, j, k, phiy, statein, vely, slope4_c, dtdy); 122 | }); 123 | 124 | #if (AMREX_SPACEDIM > 2) 125 | // z ------------------------- 126 | Array4 phiz = tmpfab.array(itmp++); 127 | Array4 phiz_c = phiz; 128 | 129 | amrex::launch(amrex::grow(gbx,Direction::z,1), 130 | [=] AMREX_GPU_DEVICE (const Box& tbx) 131 | { 132 | slopez2(tbx, slope2, statein); 133 | }); 134 | 135 | amrex::launch(gbx, 136 | [=] AMREX_GPU_DEVICE (const Box& tbx) 137 | { 138 | slopez4(tbx, slope4, statein, slope2_c); 139 | }); 140 | 141 | b = gbx; 142 | amrex::ParallelFor(b.grow(Direction::z,-1).surroundingNodes(Direction::z), 143 | [=] AMREX_GPU_DEVICE (int i, int j, int k) 144 | { 145 | flux_z(i, j, k, phiz, statein, velz, slope4_c, dtdz); 146 | }); 147 | 148 | // compute transverse fluxes (3D only) 149 | // =================================== 150 | 151 | // xy -------------------- 152 | Array4 phix_y = tmpfab.array(itmp++); 153 | Array4 phix_y_c = phix_y; 154 | 155 | b = bx; 156 | amrex::ParallelFor(b.grow(Direction::z,1).surroundingNodes(Direction::x), 157 | [=] AMREX_GPU_DEVICE (int i, int j, int k) 158 | { 159 | flux_xy(i, j, k, phix_y, 160 | velx, vely, 161 | phix_c, phiy_c, 162 | dtdy); 163 | }); 164 | 165 | // xz -------------------- 166 | Array4 phix_z = tmpfab.array(itmp++); 167 | Array4 phix_z_c = phix_z; 168 | 169 | b = bx; 170 | amrex::ParallelFor(b.grow(Direction::y,1).surroundingNodes(Direction::x), 171 | [=] AMREX_GPU_DEVICE (int i, int j, int k) 172 | { 173 | flux_xz(i, j, k, phix_z, 174 | velx, velz, 175 | phix, phiz, 176 | dtdz); 177 | }); 178 | 179 | // yx -------------------- 180 | Array4 phiy_x = tmpfab.array(itmp++); 181 | Array4 phiy_x_c = phiy_x; 182 | 183 | b = bx; 184 | amrex::ParallelFor(b.grow(Direction::z,1).surroundingNodes(Direction::y), 185 | [=] AMREX_GPU_DEVICE (int i, int j, int k) 186 | { 187 | flux_yx(i, j, k, phiy_x, 188 | velx, vely, 189 | phix, phiy, 190 | dtdx); 191 | }); 192 | 193 | // yz -------------------- 194 | Array4 phiy_z = tmpfab.array(itmp++); 195 | Array4 phiy_z_c = phiy_z; 196 | 197 | b = bx; 198 | amrex::ParallelFor(b.grow(Direction::x,1).surroundingNodes(Direction::y), 199 | [=] AMREX_GPU_DEVICE (int i, int j, int k) 200 | { 201 | flux_yz(i, j, k, phiy_z, 202 | vely, velz, 203 | phiy, phiz, 204 | dtdz); 205 | }); 206 | 207 | // zx -------------------- 208 | Array4 phiz_x = tmpfab.array(itmp++); 209 | Array4 phiz_x_c = phiz_x; 210 | 211 | b = bx; 212 | amrex::ParallelFor(b.grow(Direction::y,1).surroundingNodes(Direction::z), 213 | [=] AMREX_GPU_DEVICE (int i, int j, int k) 214 | { 215 | flux_zx(i, j, k, phiz_x, 216 | velx, velz, 217 | phix, phiz, 218 | dtdx); 219 | }); 220 | 221 | // zy -------------------- 222 | Array4 phiz_y = tmpfab.array(itmp++); 223 | Array4 phiz_y_c = phiz_y; 224 | 225 | b = bx; 226 | amrex::ParallelFor(b.grow(Direction::x,1).surroundingNodes(Direction::z), 227 | [=] AMREX_GPU_DEVICE (int i, int j, int k) 228 | { 229 | flux_zy(i, j, k, phiz_y, 230 | vely, velz, 231 | phiy, phiz, 232 | dtdy); 233 | }); 234 | #endif 235 | 236 | // final edge states 237 | // =========================== 238 | amrex::ParallelFor(amrex::surroundingNodes(bx,Direction::x), 239 | [=] AMREX_GPU_DEVICE (int i, int j, int k) 240 | { 241 | create_flux_x(i, j, k, tfluxx, 242 | AMREX_D_DECL(velx,vely,velz), 243 | #if (AMREX_SPACEDIM == 3) 244 | phix_c, phiy_z_c, phiz_y_c, 245 | dtdy, dtdz); 246 | #else 247 | phix_c, phiy_c, 248 | dtdy); 249 | #endif 250 | }); 251 | 252 | amrex::ParallelFor(amrex::surroundingNodes(bx,Direction::y), 253 | [=] AMREX_GPU_DEVICE (int i, int j, int k) 254 | { 255 | create_flux_y(i, j, k, tfluxy, 256 | AMREX_D_DECL(velx,vely,velz), 257 | #if (AMREX_SPACEDIM == 3) 258 | phiy_c, phix_z_c, phiz_x_c, 259 | dtdx, dtdz); 260 | #else 261 | phiy_c, phix_c, 262 | dtdx); 263 | #endif 264 | }); 265 | 266 | #if (AMREX_SPACEDIM == 3) 267 | amrex::ParallelFor(amrex::surroundingNodes(bx,Direction::z), 268 | [=] AMREX_GPU_DEVICE (int i, int j, int k) 269 | { 270 | create_flux_z(i, j, k, tfluxz, 271 | velx, vely, velz, 272 | phiz_c, phix_y_c, phiy_x_c, 273 | dtdx, dtdy); 274 | }); 275 | #endif 276 | AMREX_ASSERT(itmp == ntmpcomps); 277 | 278 | // compute new state (stateout) and scale fluxes based on face area. 279 | // =========================== 280 | 281 | AMREX_D_TERM(Array4 tfluxx_c = tfluxx;, 282 | Array4 tfluxy_c = tfluxy;, 283 | Array4 tfluxz_c = tfluxz); 284 | 285 | // Do a conservative update 286 | amrex::ParallelFor(bx, 287 | [=] AMREX_GPU_DEVICE (int i, int j, int k) 288 | { 289 | conservative(i, j, k, 290 | stateout, statein, 291 | AMREX_D_DECL(tfluxx_c,tfluxy_c,tfluxz_c), 292 | AMREX_D_DECL(dtdx,dtdy,dtdz)); 293 | }); 294 | 295 | if (do_reflux) 296 | { 297 | // Scale by face area in order to correctly reflux 298 | amrex::ParallelFor( 299 | AMREX_D_DECL(amrex::surroundingNodes(bx,Direction::x), 300 | amrex::surroundingNodes(bx,Direction::y), 301 | amrex::surroundingNodes(bx,Direction::z)), 302 | AMREX_D_DECL([=] AMREX_GPU_DEVICE (int i, int j, int k) 303 | { 304 | tfluxx(i,j,k) *= dt_lev*dy*dz; 305 | }, 306 | [=] AMREX_GPU_DEVICE (int i, int j, int k) 307 | { 308 | tfluxy(i,j,k) *= dt_lev*dx*dz; 309 | }, 310 | [=] AMREX_GPU_DEVICE (int i, int j, int k) 311 | { 312 | tfluxz(i,j,k) *= dt_lev*dx*dy; 313 | })); 314 | 315 | // Copy into Flux MultiFab 316 | AMREX_D_TERM(Array4 fluxx = fluxes[0].array(mfi);, 317 | Array4 fluxy = fluxes[1].array(mfi);, 318 | Array4 fluxz = fluxes[2].array(mfi)); 319 | amrex::ParallelFor( 320 | AMREX_D_DECL(mfi.nodaltilebox(0), 321 | mfi.nodaltilebox(1), 322 | mfi.nodaltilebox(2)), 323 | AMREX_D_DECL([=] AMREX_GPU_DEVICE (int i, int j, int k) 324 | { 325 | fluxx(i,j,k) = tfluxx_c(i,j,k); 326 | }, 327 | [=] AMREX_GPU_DEVICE (int i, int j, int k) 328 | { 329 | fluxy(i,j,k) = tfluxy_c(i,j,k); 330 | }, 331 | [=] AMREX_GPU_DEVICE (int i, int j, int k) 332 | { 333 | fluxz(i,j,k) = tfluxz_c(i,j,k); 334 | })); 335 | } 336 | } 337 | } 338 | 339 | // ======== CFL CHECK, MOVED OUTSIDE MFITER LOOP ========= 340 | 341 | AMREX_D_TERM(Real umax = facevel[lev][0].norminf(0,0,true);, 342 | Real vmax = facevel[lev][1].norminf(0,0,true);, 343 | Real wmax = facevel[lev][2].norminf(0,0,true)); 344 | 345 | if (AMREX_D_TERM(umax*dt_lev > dx, || 346 | vmax*dt_lev > dy, || 347 | wmax*dt_lev > dz)) 348 | { 349 | #if (AMREX_SPACEDIM > 2) 350 | amrex::AllPrint() << "umax = " << umax << ", vmax = " << vmax << ", wmax = " << wmax 351 | << ", dt = " << dt_lev << " dx = " << dx << " " << dy << " " << dz << std::endl; 352 | #else 353 | amrex::AllPrint() << "umax = " << umax << ", vmax = " << vmax 354 | << ", dt = " << dt_lev << " dx = " << dx << " " << dy << " " << dz << std::endl; 355 | #endif 356 | amrex::Abort("CFL violation. use smaller adv.cfl."); 357 | } 358 | 359 | // increment or decrement the flux registers by area and time-weighted fluxes 360 | // Note that the fluxes have already been scaled by dt and area 361 | // In this example we are solving phi_t = -div(+F) 362 | // The fluxes contain, e.g., F_{i+1/2,j} = (phi*u)_{i+1/2,j} 363 | // Keep this in mind when considering the different sign convention for updating 364 | // the flux registers from the coarse or fine grid perspective 365 | // NOTE: the flux register associated with flux_reg[lev] is associated 366 | // with the lev/lev-1 interface (and has grid spacing associated with lev-1) 367 | if (do_reflux) { 368 | if (flux_reg[lev+1]) { 369 | for (int i = 0; i < AMREX_SPACEDIM; ++i) { 370 | // update the lev+1/lev flux register (index lev+1) 371 | flux_reg[lev+1]->CrseInit(fluxes[i],i,0,0,fluxes[i].nComp(), -1.0); 372 | } 373 | } 374 | if (flux_reg[lev]) { 375 | for (int i = 0; i < AMREX_SPACEDIM; ++i) { 376 | // update the lev/lev-1 flux register (index lev) 377 | flux_reg[lev]->FineAdd(fluxes[i],i,0,0,fluxes[i].nComp(), 1.0); 378 | } 379 | } 380 | } 381 | } 382 | -------------------------------------------------------------------------------- /06_Advection_Amr/AmrCoreAdv.H: -------------------------------------------------------------------------------- 1 | #ifndef AmrCoreAdv_H_ 2 | #define AmrCoreAdv_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #ifdef AMREX_USE_OMP 9 | #include 10 | #endif 11 | 12 | #include 13 | #include 14 | #include 15 | 16 | class AmrCoreAdv 17 | : public amrex::AmrCore 18 | { 19 | public: 20 | 21 | //////////////// 22 | // public member functions 23 | 24 | // constructor - reads in parameters from inputs file 25 | // - sizes multilevel arrays and data structures 26 | AmrCoreAdv (); 27 | virtual ~AmrCoreAdv(); 28 | 29 | // advance solution to final time 30 | void Evolve (); 31 | 32 | // initializes multilevel data 33 | void InitData (); 34 | 35 | // Make a new level using provided BoxArray and DistributionMapping and 36 | // fill with interpolated coarse level data. 37 | // overrides the pure virtual function in AmrCore 38 | virtual void MakeNewLevelFromCoarse (int lev, amrex::Real time, const amrex::BoxArray& ba, 39 | const amrex::DistributionMapping& dm) override; 40 | 41 | // Remake an existing level using provided BoxArray and DistributionMapping and 42 | // fill with existing fine and coarse data. 43 | // overrides the pure virtual function in AmrCore 44 | virtual void RemakeLevel (int lev, amrex::Real time, const amrex::BoxArray& ba, 45 | const amrex::DistributionMapping& dm) override; 46 | 47 | // Delete level data 48 | // overrides the pure virtual function in AmrCore 49 | virtual void ClearLevel (int lev) override; 50 | 51 | // Make a new level from scratch using provided BoxArray and DistributionMapping. 52 | // Only used during initialization. 53 | // overrides the pure virtual function in AmrCore 54 | virtual void MakeNewLevelFromScratch (int lev, amrex::Real time, const amrex::BoxArray& ba, 55 | const amrex::DistributionMapping& dm) override; 56 | 57 | // tag all cells for refinement 58 | // overrides the pure virtual function in AmrCore 59 | virtual void ErrorEst (int lev, amrex::TagBoxArray& tags, amrex::Real time, int ngrow) override; 60 | 61 | // Advance phi at a single level for a single time step, update flux registers 62 | void AdvancePhiAtLevel (int lev, amrex::Real time, amrex::Real dt_lev, int iteration, int ncycle); 63 | 64 | // Advance phi at all levels for a single time step 65 | void AdvancePhiAllLevels (amrex::Real time, amrex::Real dt_lev, int iteration); 66 | 67 | // Define the advection velocity as the curl of a scalar field 68 | void DefineVelocityAtLevel (int lev, amrex::Real time); 69 | 70 | void DefineVelocityAllLevels (amrex::Real time); 71 | 72 | // compute dt from CFL considerations 73 | amrex::Real EstTimeStep (int lev, amrex::Real time); 74 | 75 | private: 76 | 77 | //////////////// 78 | // private member functions 79 | 80 | // read in some parameters from inputs file 81 | void ReadParameters(); 82 | 83 | // set covered coarse cells to be the average of overlying fine cells 84 | void AverageDown (); 85 | 86 | // more flexible version of AverageDown() that lets you average down across multiple levels 87 | void AverageDownTo (int crse_lev); 88 | 89 | // compute a new multifab by coping in phi from valid region and filling ghost cells 90 | // works for single level and 2-level cases (fill fine grid ghost by interpolating from coarse) 91 | void FillPatch (int lev, amrex::Real time, amrex::MultiFab& mf, int icomp, int ncomp); 92 | 93 | // fill an entire multifab by interpolating from the coarser level 94 | // this comes into play when a new level of refinement appears 95 | void FillCoarsePatch (int lev, amrex::Real time, amrex::MultiFab& mf, int icomp, int ncomp); 96 | 97 | // utility to copy in data from phi_old and/or phi_new into another multifab 98 | void GetData (int lev, amrex::Real time, amrex::Vector& data, 99 | amrex::Vector& datatime); 100 | 101 | // Advance a level by dt - includes a recursive call for finer levels 102 | void timeStepWithSubcycling (int lev, amrex::Real time, int iteration); 103 | 104 | // Advance all levels by the same dt 105 | void timeStepNoSubcycling (amrex::Real time, int iteration); 106 | 107 | // a wrapper for EstTimeStep 108 | void ComputeDt (); 109 | 110 | // get plotfile name 111 | std::string PlotFileName (int lev) const; 112 | 113 | // put together an array of multifabs for writing 114 | amrex::Vector PlotFileMF () const; 115 | 116 | // set plotfile variables names 117 | amrex::Vector PlotFileVarNames () const; 118 | 119 | // write plotfile to disk 120 | void WritePlotFile () const; 121 | 122 | // write checkpoint file to disk 123 | void WriteCheckpointFile () const; 124 | 125 | // read checkpoint file from disk 126 | void ReadCheckpointFile (); 127 | 128 | //////////////// 129 | // private data members 130 | 131 | amrex::Vector istep; // which step? 132 | amrex::Vector nsubsteps; // how many substeps on each level? 133 | 134 | // keep track of old time, new time, and time step at each level 135 | amrex::Vector t_new; 136 | amrex::Vector t_old; 137 | amrex::Vector dt; 138 | 139 | // array of multifabs to store the solution at each level of refinement 140 | // after advancing a level we use "swap". 141 | amrex::Vector phi_new; 142 | amrex::Vector phi_old; 143 | 144 | // this is essentially a 2*DIM integer array storing the physical boundary 145 | // condition types at the lo/hi walls in each direction 146 | amrex::Vector bcs; // 1-component 147 | 148 | // stores fluxes at coarse-fine interface for synchronization 149 | // this will be sized "nlevs_max+1" 150 | // NOTE: the flux register associated with flux_reg[lev] is associated 151 | // with the lev/lev-1 interface (and has grid spacing associated with lev-1) 152 | // therefore flux_reg[0] and flux_reg[nlevs_max] are never actually 153 | // used in the reflux operation 154 | amrex::Vector > flux_reg; 155 | 156 | // Velocity on all faces at all levels 157 | amrex::Vector< amrex::Array > facevel; 158 | 159 | //////////////// 160 | // runtime parameters 161 | 162 | // maximum number of steps and stop time 163 | int max_step = std::numeric_limits::max(); 164 | amrex::Real stop_time = std::numeric_limits::max(); 165 | 166 | // if >= 0 we restart from a checkpoint 167 | std::string restart_chkfile = ""; 168 | 169 | // advective cfl number - dt = cfl*dx/umax 170 | amrex::Real cfl = 0.7; 171 | 172 | // how often each level regrids the higher levels of refinement 173 | // (after a level advances that many time steps) 174 | int regrid_int = 2; 175 | 176 | // hyperbolic refluxing as part of multilevel synchronization 177 | int do_reflux = 1; 178 | 179 | // do we subcycle in time? 180 | int do_subcycle = 1; 181 | 182 | // plotfile prefix and frequency 183 | std::string plot_file {"plt"}; 184 | int plot_int = -1; 185 | 186 | // checkpoint prefix and frequency 187 | std::string chk_file {"chk"}; 188 | int chk_int = -1; 189 | }; 190 | 191 | #endif 192 | -------------------------------------------------------------------------------- /06_Advection_Amr/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | if (AMReX_SPACEDIM EQUAL 1) 2 | return() 3 | endif () 4 | 5 | # List of source files 6 | set(_sources AdvancePhiAllLevels.cpp AdvancePhiAtLevel.cpp AmrCoreAdv.cpp AmrCoreAdv.H bc_fill.H) 7 | list(APPEND _sources DefineVelocity.cpp face_velocity.H Kernels.H main.cpp Tagging.H) 8 | list(APPEND _sources Adv_K.H compute_flux_${AMReX_SPACEDIM}D_K.H slope_K.H) 9 | list(APPEND _sources Prob.H) 10 | 11 | # List of input files 12 | set(_input_files inputs Visualization.ipynb) 13 | 14 | setup_tutorial(_sources _input_files) 15 | 16 | unset( _sources ) 17 | unset( _input_files ) 18 | -------------------------------------------------------------------------------- /06_Advection_Amr/DefineVelocity.cpp: -------------------------------------------------------------------------------- 1 | #include "AmrCoreAdv.H" 2 | #include "Kernels.H" 3 | 4 | #include 5 | 6 | using namespace amrex; 7 | 8 | void 9 | AmrCoreAdv::DefineVelocityAllLevels (Real time) 10 | { 11 | for (int lev = 0; lev <= finest_level; ++lev) 12 | DefineVelocityAtLevel(lev,time); 13 | } 14 | 15 | void 16 | AmrCoreAdv::DefineVelocityAtLevel (int lev, Real time) 17 | { 18 | const auto dx = geom[lev].CellSizeArray(); 19 | 20 | #ifdef AMREX_USE_OMP 21 | #pragma omp parallel if (Gpu::notInLaunchRegion()) 22 | #endif 23 | { 24 | for (MFIter mfi(phi_new[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi) 25 | { 26 | 27 | // ======== GET FACE VELOCITY ========= 28 | GpuArray nbx; 29 | AMREX_D_TERM(nbx[0] = mfi.nodaltilebox(0);, 30 | nbx[1] = mfi.nodaltilebox(1);, 31 | nbx[2] = mfi.nodaltilebox(2);); 32 | 33 | AMREX_D_TERM(const Box& ngbxx = amrex::grow(mfi.nodaltilebox(0),1);, 34 | const Box& ngbxy = amrex::grow(mfi.nodaltilebox(1),1);, 35 | const Box& ngbxz = amrex::grow(mfi.nodaltilebox(2),1);); 36 | 37 | GpuArray, AMREX_SPACEDIM> vel{ AMREX_D_DECL( facevel[lev][0].array(mfi), 38 | facevel[lev][1].array(mfi), 39 | facevel[lev][2].array(mfi)) }; 40 | 41 | const Box& psibox = Box(IntVect(AMREX_D_DECL(std::min(ngbxx.smallEnd(0)-1, ngbxy.smallEnd(0)-1), 42 | std::min(ngbxx.smallEnd(1)-1, ngbxy.smallEnd(1)-1), 43 | 0)), 44 | IntVect(AMREX_D_DECL(std::max(ngbxx.bigEnd(0), ngbxy.bigEnd(0)+1), 45 | std::max(ngbxx.bigEnd(1)+1, ngbxy.bigEnd(1)), 46 | 0))); 47 | 48 | FArrayBox psifab(psibox, 1); 49 | Elixir psieli = psifab.elixir(); 50 | Array4 psi = psifab.array(); 51 | GeometryData geomdata = geom[lev].data(); 52 | 53 | amrex::launch(psibox, 54 | [=] AMREX_GPU_DEVICE (const Box& tbx) 55 | { 56 | get_face_velocity_psi(tbx, time, psi, geomdata); 57 | }); 58 | 59 | amrex::ParallelFor 60 | (AMREX_D_DECL(ngbxx,ngbxy,ngbxz), 61 | AMREX_D_DECL( 62 | [=] AMREX_GPU_DEVICE (int i, int j, int k) 63 | { 64 | get_face_velocity_x(i, j, k, vel[0], psi, dx[1]); 65 | }, 66 | [=] AMREX_GPU_DEVICE (int i, int j, int k) 67 | { 68 | get_face_velocity_y(i, j, k, vel[1], psi, dx[0]); 69 | }, 70 | [=] AMREX_GPU_DEVICE (int i, int j, int k) 71 | { 72 | get_face_velocity_z(i, j, k, vel[2]); 73 | })); 74 | } 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /06_Advection_Amr/Kernels.H: -------------------------------------------------------------------------------- 1 | #ifndef Kernels_H_ 2 | #define Kernels_H_ 3 | 4 | #include "face_velocity.H" 5 | #include "Prob.H" 6 | #include "Adv_K.H" 7 | #include "slope_K.H" 8 | #include "Tagging.H" 9 | #include "bc_fill.H" 10 | 11 | #if (AMREX_SPACEDIM == 2) 12 | #include "compute_flux_2D_K.H" 13 | #else 14 | #include "compute_flux_3D_K.H" 15 | #endif 16 | 17 | #endif 18 | -------------------------------------------------------------------------------- /06_Advection_Amr/Prob.H: -------------------------------------------------------------------------------- 1 | #ifndef PROB_H_ 2 | #define PROB_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | AMREX_GPU_DEVICE 9 | AMREX_FORCE_INLINE 10 | void 11 | initdata(amrex::Box const& bx, amrex::Array4 const& phi, 12 | amrex::GpuArray const& prob_lo, 13 | amrex::GpuArray const& dx) 14 | { 15 | using namespace amrex; 16 | 17 | const auto lo = lbound(bx); 18 | const auto hi = ubound(bx); 19 | 20 | for (int k = lo.z; k <= hi.z; ++k) { 21 | for (int j = lo.y; j <= hi.y; ++j) { 22 | Real y = prob_lo[1] + (0.5+j) * dx[1]; 23 | for (int i = lo.x; i <= hi.x; ++i) { 24 | Real x = prob_lo[0] + (0.5+i) * dx[0]; 25 | Real r2 = (std::pow(x-0.5, 2) + std::pow((y-0.75),2)) / 0.01; 26 | phi(i,j,k) = 1.0 + std::exp(-r2); 27 | } 28 | } 29 | } 30 | } 31 | 32 | #endif 33 | -------------------------------------------------------------------------------- /06_Advection_Amr/Tagging.H: -------------------------------------------------------------------------------- 1 | #ifndef TAGGING_H 2 | #define TAGGING_H 3 | 4 | #include 5 | 6 | AMREX_GPU_HOST_DEVICE 7 | AMREX_FORCE_INLINE 8 | void 9 | state_error (int i, int j, int k, 10 | amrex::Array4 const& tag, 11 | amrex::Array4 const& state, 12 | amrex::Real phierr, char tagval) 13 | { 14 | if (state(i,j,k) > phierr) 15 | tag(i,j,k) = tagval; 16 | } 17 | 18 | #endif 19 | -------------------------------------------------------------------------------- /06_Advection_Amr/Visualization.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "1dcbcf62", 6 | "metadata": {}, 7 | "source": [ 8 | "## Example: AMR101: Multi-Level Scalar Advection\n", 9 | "\n", 10 | "### What Features Are We Using\n", 11 | "\n", 12 | "* Mesh data \n", 13 | "* Dynamic AMR with and without subcycling\n", 14 | "\n", 15 | "### The Problem\n", 16 | "\n", 17 | "Consider a drop of dye (we'll define $\\phi$ to be the concentration of dye) in a thin incompressible fluid that is spinning clock-wise then counter-clockwise with a prescribed motion. We consider the dye to be a passive tracer that is advected by the fluid velocity. The fluid is thin enough that we can model this as two-dimensional motion; here we have the option of solving in a 2D or 3D computational domain.\n", 18 | "\n", 19 | "In other words, we want to solve for $$\\phi(x,y,t)$$ by evolving\n", 20 | "\n", 21 | "$$\\frac{\\partial \\phi}{\\partial t} + \\nabla \\cdot (\\bf{u^{spec}} \\phi) = 0$$\n", 22 | "\n", 23 | "in time ($t$), where the velocity $${\\bf{u^{spec}}} = (u,v)$$ is a divergence-free field computed by defining\n", 24 | "\n", 25 | "$$\\psi(i,j) = \\sin^2(\\pi x) \\sin^2(\\pi y) \\cos (\\pi t / 2) / \\pi $$\n", 26 | "\n", 27 | "and defining\n", 28 | "\n", 29 | "$$u = -\\frac{\\partial \\psi}{\\partial y}, v = \\frac{\\partial \\psi}{\\partial x}.$$\n", 30 | "\n", 31 | "Note that because ${\\bf{u^{spec}}}$ is defined as the curl of a scalar field, it is analytically divergence-free\n", 32 | "\n", 33 | "In this example we'll be using AMR to resolve the scalar field since the location of the dye is what we care most about.\n", 34 | "\n", 35 | "### The AMR Algorithm\n", 36 | "\n", 37 | "To update the solution in a patch at a given level, we compute fluxes (${\\bf u^{spec}} \\phi$)\n", 38 | "on each face, and difference the fluxes to create the update to phi. The update routine\n", 39 | "in the code looks like\n", 40 | "\n", 41 | "```cplusplus\n", 42 | " // Do a conservative update\n", 43 | " {\n", 44 | " phi_out(i,j,k) = phi_in(i,j,k) +\n", 45 | " ( AMREX_D_TERM( (flxx(i,j,k) - flxx(i+1,j,k)) * dtdx[0],\n", 46 | " + (flxy(i,j,k) - flxy(i,j+1,k)) * dtdx[1],\n", 47 | " + (flxz(i,j,k) - flxz(i,j,k+1)) * dtdx[2] ) );\n", 48 | " }\n", 49 | "```\n", 50 | "\n", 51 | "In this routine we use the macro AMREX_D_TERM so that we can write dimension-independent code; \n", 52 | "in 3D this returns the flux differences in all three directions, but in 2D it does not include\n", 53 | "the z-fluxes.\n", 54 | "\n", 55 | "Knowing how to synchronize the solution at coarse/fine boundaries is essential in an AMR algorithm;\n", 56 | "here having the algorithm written in flux form allows us to either make the fluxes consistent between\n", 57 | "coarse and fine levels in a no-subcycling algorithm, or \"reflux\" after the update in a subcycling algorithm.\n", 58 | "\n", 59 | "The subcycling algorithm can be written as follows\n", 60 | "```C++\n", 61 | "void\n", 62 | "AmrCoreAdv::timeStepWithSubcycling (int lev, Real time, int iteration)\n", 63 | "{\n", 64 | "\n", 65 | " // Advance a single level for a single time step, and update flux registers\n", 66 | " Real t_nph = 0.5 * (t_old[lev] + t_new[lev]);\n", 67 | " DefineVelocityAtLevel(lev, t_nph);\n", 68 | " AdvancePhiAtLevel(lev, time, dt[lev], iteration, nsubsteps[lev]);\n", 69 | "\n", 70 | " ++istep[lev];\n", 71 | "\n", 72 | " if (lev < finest_level)\n", 73 | " {\n", 74 | " // recursive call for next-finer level\n", 75 | " for (int i = 1; i <= nsubsteps[lev+1]; ++i)\n", 76 | " {\n", 77 | " timeStepWithSubcycling(lev+1, time+(i-1)*dt[lev+1], i);\n", 78 | " }\n", 79 | "\n", 80 | " if (do_reflux)\n", 81 | " {\n", 82 | " // update lev based on coarse-fine flux mismatch\n", 83 | " flux_reg[lev+1]->Reflux(phi_new[lev], 1.0, 0, 0, phi_new[lev].nComp(), geom[lev]);\n", 84 | " }\n", 85 | "\n", 86 | " AverageDownTo(lev); // average lev+1 down to lev\n", 87 | " }\n", 88 | "\n", 89 | "}\n", 90 | "```\n", 91 | "\n", 92 | "while the no-subcycling algorithm looks like\n", 93 | "```C++\n", 94 | "void\n", 95 | "AmrCoreAdv::timeStepNoSubcycling (Real time, int iteration)\n", 96 | "{\n", 97 | " DefineVelocityAllLevels(time);\n", 98 | " AdvancePhiAllLevels (time, dt[0], iteration);\n", 99 | "\n", 100 | " // Make sure the coarser levels are consistent with the finer levels\n", 101 | " AverageDown ();\n", 102 | "\n", 103 | " for (int lev = 0; lev <= finest_level; lev++)\n", 104 | " ++istep[lev];\n", 105 | "}\n", 106 | "```" 107 | ] 108 | }, 109 | { 110 | "cell_type": "markdown", 111 | "id": "78f7656c", 112 | "metadata": {}, 113 | "source": [ 114 | "## Running the code\n", 115 | "\n", 116 | "To run in serial, \n", 117 | "\n", 118 | "```\n", 119 | "./06_Advection_Amr inputs\n", 120 | "```\n", 121 | "\n", 122 | "To run in parallel, for example on 4 ranks:\n", 123 | "\n", 124 | "```\n", 125 | "mpirun -n 4 ./06_Advection_Amr inputs\n", 126 | "```\n", 127 | "\n", 128 | "The following parameters can be set at run-time -- these are currently set in the inputs\n", 129 | "file but you can also set them on the command line. \n", 130 | "\n", 131 | "```\n", 132 | "stop_time = 2.0 # the final time (if we have not exceeded number of steps)\n", 133 | "max_step = 1000000 # the maximum number of steps (if we have not exceeded stop_time)\n", 134 | "\n", 135 | "amr.n_cell = 64 64 8 # number of cells at the coarsest AMR level in each coordinate direction\n", 136 | "\n", 137 | "amr.max_grid_size = 16 # the maximum number of cells in any direction in a single grid\n", 138 | "\n", 139 | "amr.plot_int = 10 # frequency of writing plotfiles\n", 140 | "\n", 141 | "adv.cfl = 0.9 # cfl number to be used for computing the time step\n", 142 | "\n", 143 | "adv.phierr = 1.01 1.1 1.5 # regridding criteria at each level\n", 144 | "\n", 145 | "```\n", 146 | "\n", 147 | "It may also be interesting to experiment with `adv.do_subcycle` and `adv.do_reflux`.\n", 148 | "\n", 149 | "The base grid here is a square of 64 x 64 x 8 cells, made up of 16 subgrids each of size 16x16x8 cells. \n", 150 | "The problem is periodic in all directions.\n", 151 | "\n", 152 | "We have hard-wired the code here to refine based on the magnitude of $\\phi$. Here we set the \n", 153 | "threshold level by level. If $\\phi > 1.01$ then we want to refine at least once; if $\\phi > 1.1$ we\n", 154 | "want to resolve $\\phi$ with two levels of refinement, and if $\\phi > 1.5$ we want even more refinement.\n", 155 | "\n", 156 | "Note that you can see the total runtime by looking at the line at the end of your run that says\n", 157 | "\n", 158 | "```\n", 159 | "Total Time: \n", 160 | "```\n", 161 | "\n", 162 | "and you can check conservation of $\\phi$ by checking the line that prints, e.g. \n", 163 | "\n", 164 | "```\n", 165 | "Coarse STEP 8 ends. TIME = 0.007031485953 DT = 0.0008789650903 Sum(Phi) = 540755.0014\n", 166 | "```\n", 167 | "\n", 168 | "Here Sum(Phi) is the sum of $\\phi$ over all the cells at the coarsest level." 169 | ] 170 | }, 171 | { 172 | "cell_type": "markdown", 173 | "id": "c826b957", 174 | "metadata": {}, 175 | "source": [ 176 | "### Visualizing the Results\n", 177 | "\n", 178 | "Below is some python code for visualizing the results with yt:" 179 | ] 180 | }, 181 | { 182 | "cell_type": "code", 183 | "execution_count": null, 184 | "id": "29609c78", 185 | "metadata": {}, 186 | "outputs": [], 187 | "source": [ 188 | "import yt\n", 189 | "from yt.frontends.boxlib.data_structures import AMReXDataset" 190 | ] 191 | }, 192 | { 193 | "cell_type": "code", 194 | "execution_count": null, 195 | "id": "cb533aa1", 196 | "metadata": {}, 197 | "outputs": [], 198 | "source": [ 199 | "ds = AMReXDataset(\"plt00060\")" 200 | ] 201 | }, 202 | { 203 | "cell_type": "code", 204 | "execution_count": null, 205 | "id": "ae107c08", 206 | "metadata": {}, 207 | "outputs": [], 208 | "source": [ 209 | "ds.field_list" 210 | ] 211 | }, 212 | { 213 | "cell_type": "code", 214 | "execution_count": null, 215 | "id": "899767fc", 216 | "metadata": {}, 217 | "outputs": [], 218 | "source": [ 219 | "sl = yt.SlicePlot(ds, 2, ('boxlib', 'phi'))\n", 220 | "sl.annotate_grids()" 221 | ] 222 | } 223 | ], 224 | "metadata": { 225 | "kernelspec": { 226 | "display_name": "Python 3", 227 | "language": "python", 228 | "name": "python3" 229 | }, 230 | "language_info": { 231 | "codemirror_mode": { 232 | "name": "ipython", 233 | "version": 3 234 | }, 235 | "file_extension": ".py", 236 | "mimetype": "text/x-python", 237 | "name": "python", 238 | "nbconvert_exporter": "python", 239 | "pygments_lexer": "ipython3", 240 | "version": "3.8.8" 241 | } 242 | }, 243 | "nbformat": 4, 244 | "nbformat_minor": 5 245 | } 246 | -------------------------------------------------------------------------------- /06_Advection_Amr/bc_fill.H: -------------------------------------------------------------------------------- 1 | #ifndef BCFILL_H 2 | #define BCFILL_H 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | struct AmrCoreFill 9 | { 10 | AMREX_GPU_DEVICE 11 | void operator() (const amrex::IntVect& /*iv*/, amrex::Array4 const& /*data*/, 12 | const int /*dcomp*/, const int /*numcomp*/, 13 | amrex::GeometryData const& /*geom*/, const amrex::Real /*time*/, 14 | const amrex::BCRec* /*bcr*/, const int /*bcomp*/, 15 | const int /*orig_comp*/) const 16 | { 17 | // do something for external Dirichlet (BCType::ext_dir) 18 | } 19 | }; 20 | 21 | #endif 22 | -------------------------------------------------------------------------------- /06_Advection_Amr/compute_flux_2D_K.H: -------------------------------------------------------------------------------- 1 | #ifndef _compute_flux_2d_H_ 2 | #define _compute_flux_2d_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | AMREX_GPU_DEVICE 9 | AMREX_FORCE_INLINE 10 | void flux_x(int i, int j, int k, 11 | amrex::Array4 const& px, 12 | amrex::Array4 const& phi, 13 | amrex::Array4 const& vx, 14 | amrex::Array4 const& slope, 15 | amrex::Real dtdx) 16 | { 17 | px(i,j,k) = ( (vx(i,j,k) < 0) ? 18 | phi(i ,j,k) - slope(i ,j,k)*(0.5 + 0.5*dtdx*vx(i,j,k)) : 19 | phi(i-1,j,k) + slope(i-1,j,k)*(0.5 - 0.5*dtdx*vx(i,j,k)) ); 20 | } 21 | 22 | AMREX_GPU_DEVICE 23 | AMREX_FORCE_INLINE 24 | void flux_y(int i, int j, int k, 25 | amrex::Array4 const& py, 26 | amrex::Array4 const& phi, 27 | amrex::Array4 const& vy, 28 | amrex::Array4 const& slope, 29 | amrex::Real dtdy) 30 | { 31 | py(i,j,k) = ( (vy(i,j,k) < 0) ? 32 | phi(i,j ,k) - slope(i,j ,k)*(0.5 + 0.5*dtdy*vy(i,j,k)) : 33 | phi(i,j-1,k) + slope(i,j-1,k)*(0.5 - 0.5*dtdy*vy(i,j,k)) ); 34 | } 35 | 36 | AMREX_GPU_DEVICE 37 | AMREX_FORCE_INLINE 38 | void create_flux_x(int i, int j, int k, 39 | amrex::Array4 const& fx, 40 | amrex::Array4 const& vx, 41 | amrex::Array4 const& vy, 42 | amrex::Array4 const& px, 43 | amrex::Array4 const& py, 44 | amrex::Real dtdy) 45 | { 46 | fx(i,j,k) = ( (vx(i,j,k) < 0) ? 47 | (px(i,j,k) - 0.5*dtdy * ( 0.5*(vy(i ,j+1,k ) + vy(i ,j,k)) * (py(i ,j+1,k )-py(i ,j,k))))*vx(i,j,k) : 48 | (px(i,j,k) - 0.5*dtdy * ( 0.5*(vy(i-1,j+1,k ) + vy(i-1,j,k)) * (py(i-1,j+1,k )-py(i-1,j,k))))*vx(i,j,k) ); 49 | } 50 | 51 | AMREX_GPU_DEVICE 52 | AMREX_FORCE_INLINE 53 | void create_flux_y(int i, int j, int k, 54 | amrex::Array4 const& fy, 55 | amrex::Array4 const& vx, 56 | amrex::Array4 const& vy, 57 | amrex::Array4 const& py, 58 | amrex::Array4 const& px, 59 | amrex::Real dtdx) 60 | { 61 | fy(i,j,k) = ( (vy(i,j,k) < 0) ? 62 | (py(i,j,k) - 0.5*dtdx * ( 0.5*(vx(i+1,j ,k ) + vx(i,j ,k)) * (px(i+1,j ,k )-px(i,j ,k))))*vy(i,j,k) : 63 | (py(i,j,k) - 0.5*dtdx * ( 0.5*(vx(i+1,j-1,k ) + vx(i,j-1,k)) * (px(i+1,j-1,k )-px(i,j-1,k))))*vy(i,j,k) ); 64 | } 65 | 66 | #endif 67 | -------------------------------------------------------------------------------- /06_Advection_Amr/compute_flux_3D_K.H: -------------------------------------------------------------------------------- 1 | #ifndef _compute_flux_3d_H_ 2 | #define _compute_flux_3d_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | using namespace amrex; 9 | 10 | AMREX_GPU_DEVICE 11 | AMREX_FORCE_INLINE 12 | void flux_x(int i, int j, int k, 13 | Array4 const& px, 14 | Array4 const& phi, 15 | Array4 const& vx, 16 | Array4 const& slope, 17 | Real dtdx) 18 | { 19 | px(i,j,k) = ( (vx(i,j,k) < 0) ? 20 | phi(i ,j,k) - slope(i ,j,k)*(0.5 + 0.5*dtdx*vx(i,j,k)) : 21 | phi(i-1,j,k) + slope(i-1,j,k)*(0.5 - 0.5*dtdx*vx(i,j,k)) ); 22 | } 23 | 24 | AMREX_GPU_DEVICE 25 | AMREX_FORCE_INLINE 26 | void flux_y(int i, int j, int k, 27 | Array4 const& py, 28 | Array4 const& phi, 29 | Array4 const& vy, 30 | Array4 const& slope, 31 | Real dtdy) 32 | { 33 | py(i,j,k) = ( (vy(i,j,k) < 0) ? 34 | phi(i,j ,k) - slope(i,j ,k)*(0.5 + 0.5*dtdy*vy(i,j,k)) : 35 | phi(i,j-1,k) + slope(i,j-1,k)*(0.5 - 0.5*dtdy*vy(i,j,k)) ); 36 | } 37 | 38 | AMREX_GPU_DEVICE 39 | AMREX_FORCE_INLINE 40 | void flux_z(int i, int j, int k, 41 | Array4 const& pz, 42 | Array4 const& phi, 43 | Array4 const& vz, 44 | Array4 const& slope, 45 | Real dtdz) 46 | { 47 | pz(i,j,k) = ( (vz(i,j,k) < 0) ? 48 | phi(i,j,k ) - slope(i,j,k )*(0.5 + 0.5*dtdz*vz(i,j,k)) : 49 | phi(i,j,k-1) + slope(i,j,k-1)*(0.5 - 0.5*dtdz*vz(i,j,k)) ); 50 | } 51 | 52 | AMREX_GPU_DEVICE 53 | AMREX_FORCE_INLINE 54 | void flux_xy(int i, int j, int k, 55 | Array4 const& pxy, 56 | Array4 const& vx, 57 | Array4 const& vy, 58 | Array4 const& px, 59 | Array4 const& py, 60 | Real dtdy) 61 | { 62 | pxy(i,j,k) = ( (vx(i,j,k) < 0) ? 63 | px(i,j,k) - dtdy/3.0 * ( 0.5*(vy(i, j+1,k) + vy(i ,j,k)) * (py(i ,j+1,k) - py(i ,j,k))) : 64 | px(i,j,k) - dtdy/3.0 * ( 0.5*(vy(i-1,j+1,k) + vy(i-1,j,k)) * (py(i-1,j+1,k) - py(i-1,j,k))) ); 65 | } 66 | 67 | AMREX_GPU_DEVICE 68 | AMREX_FORCE_INLINE 69 | void flux_xz(int i, int j, int k, 70 | Array4 const& pxz, 71 | Array4 const& vx, 72 | Array4 const& vz, 73 | Array4 const& px, 74 | Array4 const& pz, 75 | Real dtdz) 76 | { 77 | pxz(i,j,k) = ( (vx(i,j,k) < 0) ? 78 | px(i,j,k) - dtdz/3.0 * ( 0.5*(vz(i, j,k+1) + vz(i ,j,k)) * (pz(i ,j,k+1) - pz(i ,j,k))) : 79 | px(i,j,k) - dtdz/3.0 * ( 0.5*(vz(i-1,j,k+1) + vz(i-1,j,k)) * (pz(i-1,j,k+1) - pz(i-1,j,k))) ); 80 | } 81 | 82 | AMREX_GPU_DEVICE 83 | AMREX_FORCE_INLINE 84 | void flux_yx(int i, int j, int k, 85 | Array4 const& pyx, 86 | Array4 const& vx, 87 | Array4 const& vy, 88 | Array4 const& px, 89 | Array4 const& py, 90 | Real dtdx) 91 | { 92 | pyx(i,j,k) = ( (vy(i,j,k) < 0) ? 93 | py(i,j,k) - dtdx/3.0 * ( 0.5*(vx(i+1,j ,k) + vx(i,j ,k)) * (px(i+1,j ,k) - px(i,j ,k))) : 94 | py(i,j,k) - dtdx/3.0 * ( 0.5*(vx(i+1,j-1,k) + vx(i,j-1,k)) * (px(i+1,j-1,k) - px(i,j-1,k))) ); 95 | } 96 | 97 | AMREX_GPU_DEVICE 98 | AMREX_FORCE_INLINE 99 | void flux_yz(int i, int j, int k, 100 | Array4 const& pyz, 101 | Array4 const& vy, 102 | Array4 const& vz, 103 | Array4 const& py, 104 | Array4 const& pz, 105 | Real dtdz) 106 | { 107 | pyz(i,j,k) = ( (vy(i,j,k) < 0) ? 108 | py(i,j,k) - dtdz/3.0 * ( 0.5*(vz(i, j,k+1) + vz(i,j ,k)) * (pz(i,j ,k+1) - pz(i,j ,k))) : 109 | py(i,j,k) - dtdz/3.0 * ( 0.5*(vz(i,j-1,k+1) + vz(i,j-1,k)) * (pz(i,j-1,k+1) - pz(i,j-1,k))) ); 110 | } 111 | 112 | AMREX_GPU_DEVICE 113 | AMREX_FORCE_INLINE 114 | void flux_zx(int i, int j, int k, 115 | Array4 const& pzx, 116 | Array4 const& vx, 117 | Array4 const& vz, 118 | Array4 const& px, 119 | Array4 const& pz, 120 | Real dtdx) 121 | { 122 | pzx(i,j,k) = ( (vz(i,j,k) < 0) ? 123 | pz(i,j,k) - dtdx/3.0 * ( 0.5*(vx(i+1,j,k ) + vx(i,j,k )) * (px(i+1,j,k ) - px(i,j,k ))) : 124 | pz(i,j,k) - dtdx/3.0 * ( 0.5*(vx(i+1,j,k-1) + vx(i,j,k-1)) * (px(i+1,j,k-1) - px(i,j,k-1))) ); 125 | } 126 | 127 | AMREX_GPU_DEVICE 128 | AMREX_FORCE_INLINE 129 | void flux_zy(int i, int j, int k, 130 | Array4 const& pzy, 131 | Array4 const& vy, 132 | Array4 const& vz, 133 | Array4 const& py, 134 | Array4 const& pz, 135 | Real dtdy) 136 | { 137 | pzy(i,j,k) = ( (vz(i,j,k) < 0) ? 138 | pz(i,j,k) - dtdy/3.0 * ( 0.5*(vy(i,j+1,k ) + vy(i,j,k )) * (py(i,j+1,k ) - py(i,j,k ))) : 139 | pz(i,j,k) - dtdy/3.0 * ( 0.5*(vy(i,j+1,k-1) + vy(i,j,k-1)) * (py(i,j+1,k-1) - py(i,j,k-1))) ); 140 | } 141 | 142 | AMREX_GPU_DEVICE 143 | AMREX_FORCE_INLINE 144 | void create_flux_x(int i, int j, int k, 145 | Array4 const& fx, 146 | Array4 const& vx, 147 | Array4 const& vy, 148 | Array4 const& vz, 149 | Array4 const& px, 150 | Array4 const& pyz, 151 | Array4 const& pzy, 152 | Real dtdy, Real dtdz) 153 | { 154 | Real f = ( (vx(i,j,k) < 0) ? 155 | px(i,j,k) - 0.5*dtdy * ( 0.5*(vy(i ,j+1,k ) + vy(i ,j,k)) * (pyz(i ,j+1,k )-pyz(i ,j,k))) 156 | - 0.5*dtdz * ( 0.5*(vz(i ,j ,k+1) + vz(i ,j,k)) * (pzy(i ,j ,k+1)-pzy(i ,j,k))) : 157 | px(i,j,k) - 0.5*dtdy * ( 0.5*(vy(i-1,j+1,k ) + vy(i-1,j,k)) * (pyz(i-1,j+1,k )-pyz(i-1,j,k))) 158 | - 0.5*dtdz * ( 0.5*(vz(i-1,j ,k+1) + vz(i-1,j,k)) * (pzy(i-1,j ,k+1)-pzy(i-1,j,k))) ); 159 | 160 | fx(i,j,k) = vx(i,j,k)*f; 161 | } 162 | 163 | AMREX_GPU_DEVICE 164 | AMREX_FORCE_INLINE 165 | void create_flux_y(int i, int j, int k, 166 | Array4 const& fy, 167 | Array4 const& vx, 168 | Array4 const& vy, 169 | Array4 const& vz, 170 | Array4 const& py, 171 | Array4 const& pxz, 172 | Array4 const& pzx, 173 | Real dtdx, Real dtdz) 174 | { 175 | Real f = ( (vy(i,j,k) < 0) ? 176 | py(i,j,k) - 0.5*dtdx * ( 0.5*(vx(i+1,j ,k ) + vx(i,j ,k)) * (pxz(i+1,j ,k )-pxz(i,j ,k))) 177 | - 0.5*dtdz * ( 0.5*(vz(i, j ,k+1) + vz(i,j ,k)) * (pzx(i, j ,k+1)-pzx(i,j ,k))) : 178 | py(i,j,k) - 0.5*dtdx * ( 0.5*(vx(i+1,j-1,k ) + vx(i,j-1,k)) * (pxz(i+1,j-1,k )-pxz(i,j-1,k))) 179 | - 0.5*dtdz * ( 0.5*(vz(i ,j-1,k+1) + vz(i,j-1,k)) * (pzx(i ,j-1,k+1)-pzx(i,j-1,k))) ); 180 | 181 | fy(i,j,k) = vy(i,j,k)*f; 182 | } 183 | 184 | AMREX_GPU_DEVICE 185 | AMREX_FORCE_INLINE 186 | void create_flux_z(int i, int j, int k, 187 | Array4 const& fz, 188 | Array4 const& vx, 189 | Array4 const& vy, 190 | Array4 const& vz, 191 | Array4 const& pz, 192 | Array4 const& pxy, 193 | Array4 const& pyx, 194 | Real dtdx, Real dtdy) 195 | { 196 | Real f = ( (vz(i,j,k) < 0) ? 197 | pz(i,j,k) - 0.5*dtdx * ( 0.5*(vx(i+1,j ,k ) + vx(i,j,k )) * (pxy(i+1,j ,k )-pxy(i,j,k ))) 198 | - 0.5*dtdy * ( 0.5*(vy(i, j+1,k ) + vy(i,j,k )) * (pyx(i, j+1,k )-pyx(i,j,k ))) : 199 | pz(i,j,k) - 0.5*dtdx * ( 0.5*(vx(i+1,j ,k-1) + vx(i,j,k-1)) * (pxy(i+1,j ,k-1)-pxy(i,j,k-1))) 200 | - 0.5*dtdy * ( 0.5*(vy(i ,j+1,k-1) + vy(i,j,k-1)) * (pyx(i ,j+1,k-1)-pyx(i,j,k-1))) ); 201 | 202 | fz(i,j,k) = vz(i,j,k)*f; 203 | } 204 | 205 | #endif 206 | -------------------------------------------------------------------------------- /06_Advection_Amr/face_velocity.H: -------------------------------------------------------------------------------- 1 | #ifndef FACE_VELOCITY_H_ 2 | #define FACE_VELOCITY_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | AMREX_GPU_DEVICE 9 | AMREX_FORCE_INLINE 10 | void get_face_velocity_psi(amrex::Box const& bx, 11 | const amrex::Real time, 12 | amrex::Array4 const& psi, 13 | amrex::GeometryData const& geomdata) 14 | { 15 | using namespace amrex; 16 | constexpr Real PI = 3.1415926535897932384626; 17 | 18 | const auto lo = lbound(bx); 19 | const auto hi = ubound(bx); 20 | 21 | const Real* AMREX_RESTRICT prob_lo = geomdata.ProbLo(); 22 | const Real* AMREX_RESTRICT dx = geomdata.CellSize(); 23 | 24 | for (int j = lo.y; j <= hi.y; ++j) { 25 | Real y = dx[1]*(0.5+j) + prob_lo[1]; 26 | AMREX_PRAGMA_SIMD 27 | for (int i = lo.x; i <= hi.x; ++i) { 28 | Real x = dx[0]*(0.5+i) + prob_lo[0]; 29 | psi(i,j,0) = std::pow(std::sin(PI*x), 2) * std::pow(std::sin(PI*y), 2) 30 | * std::cos(PI*time/2.0) * 1.0/PI; 31 | } 32 | } 33 | } 34 | 35 | AMREX_GPU_DEVICE 36 | AMREX_FORCE_INLINE 37 | void get_face_velocity_x(int i, int j, int k, 38 | amrex::Array4 const& vx, 39 | amrex::Array4 const& psi, 40 | amrex::Real dy) 41 | { 42 | vx(i,j,k) = -( (psi(i,j+1,0)+psi(i-1,j+1,0)) - (psi(i,j-1,0)+psi(i-1,j-1,0)) ) * (0.25/dy); 43 | } 44 | 45 | AMREX_GPU_DEVICE 46 | AMREX_FORCE_INLINE 47 | void get_face_velocity_y(int i, int j, int k, 48 | amrex::Array4 const& vy, 49 | amrex::Array4 const& psi, 50 | amrex::Real dx) 51 | { 52 | vy(i,j,k) = ( (psi(i+1,j,0)+psi(i+1,j-1,0)) - (psi(i-1,j,0)+psi(i-1,j-1,0)) ) * (0.25/dx); 53 | } 54 | 55 | AMREX_GPU_DEVICE 56 | AMREX_FORCE_INLINE 57 | void get_face_velocity_z(int i, int j, int k, 58 | amrex::Array4 const& vz) 59 | { 60 | vz(i,j,k) = 0.0; 61 | } 62 | 63 | #endif 64 | -------------------------------------------------------------------------------- /06_Advection_Amr/inputs: -------------------------------------------------------------------------------- 1 | # ***************************************************************** 2 | # Run until nsteps == max_step or time == stop_time, 3 | # whichever comes first 4 | # ***************************************************************** 5 | max_step = 1000 6 | stop_time = 2.0 7 | 8 | # ***************************************************************** 9 | # Are we restarting from an existing checkpoint file? 10 | # ***************************************************************** 11 | #amr.restart = chk00060 # restart from this checkpoint file 12 | 13 | # ***************************************************************** 14 | # Problem size and geometry 15 | # ***************************************************************** 16 | geometry.prob_lo = 0.0 0.0 0.0 17 | geometry.prob_hi = 1.0 1.0 0.125 18 | geometry.is_periodic = 1 1 1 19 | 20 | # ***************************************************************** 21 | # VERBOSITY 22 | # ***************************************************************** 23 | amr.v = 1 # verbosity in Amr 24 | 25 | # ***************************************************************** 26 | # Resolution and refinement 27 | # ***************************************************************** 28 | amr.n_cell = 64 64 8 29 | amr.max_level = 2 # maximum level number allowed -- 30 | # number of levels = max_level + 1 31 | 32 | amr.ref_ratio = 2 2 2 2 # refinement ratio between levels 33 | 34 | # ***************************************************************** 35 | # Control of grid creation 36 | # ***************************************************************** 37 | # Blocking factor for grid creation in each dimension -- 38 | # this ensures that every grid is coarsenable by a factor of 8 -- 39 | # this is mostly relevant for multigrid performance 40 | amr.blocking_factor_x = 8 41 | amr.blocking_factor_y = 8 42 | amr.blocking_factor_z = 8 43 | 44 | amr.max_grid_size = 16 45 | 46 | amr.regrid_int = 2 # how often to regrid 47 | 48 | # ***************************************************************** 49 | # Time step control 50 | # ***************************************************************** 51 | adv.cfl = 0.7 # CFL constraint for explicit advection 52 | 53 | adv.do_subcycle = 1 # Do we subcycle in time? 54 | 55 | # ***************************************************************** 56 | # Should we reflux at coarse-fine boundaries? 57 | # ***************************************************************** 58 | adv.do_reflux = 1 59 | 60 | # ***************************************************************** 61 | # Tagging - if phi > 1.01 at level 0, then refine 62 | # if phi > 1.1 at level 1, then refine 63 | # if phi > 1.5 at level 2, then refine 64 | # ***************************************************************** 65 | adv.phierr = 1.01 1.1 1.5 66 | 67 | # ***************************************************************** 68 | # Plotfile name and frequency 69 | # ***************************************************************** 70 | amr.plot_file = plt # root name of plot file 71 | amr.plot_int = 10 # number of timesteps between plot files 72 | # if negative then no plot files will be written 73 | 74 | # ***************************************************************** 75 | # Checkpoint name and frequency 76 | # ***************************************************************** 77 | amr.chk_file = chk # root name of checkpoint file 78 | amr.chk_int = -1 # number of timesteps between checkpoint files 79 | # if negative then no checkpoint files will be written 80 | -------------------------------------------------------------------------------- /06_Advection_Amr/main.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #include "AmrCoreAdv.H" 9 | 10 | using namespace amrex; 11 | 12 | int main(int argc, char* argv[]) 13 | { 14 | amrex::Initialize(argc,argv); 15 | 16 | { 17 | // timer for profiling 18 | BL_PROFILE("main()"); 19 | 20 | // wallclock time 21 | const auto strt_total = amrex::second(); 22 | 23 | // constructor - reads in parameters from inputs file 24 | // - sizes multilevel arrays and data structures 25 | AmrCoreAdv amr_core_adv; 26 | 27 | // initialize AMR data 28 | amr_core_adv.InitData(); 29 | 30 | // advance solution to final time 31 | amr_core_adv.Evolve(); 32 | 33 | // wallclock time 34 | auto end_total = amrex::second() - strt_total; 35 | 36 | if (amr_core_adv.Verbose()) { 37 | // print wallclock time 38 | ParallelDescriptor::ReduceRealMax(end_total ,ParallelDescriptor::IOProcessorNumber()); 39 | amrex::Print() << "\nTotal Time: " << end_total << '\n'; 40 | } 41 | } 42 | 43 | amrex::Finalize(); 44 | } 45 | -------------------------------------------------------------------------------- /06_Advection_Amr/slope_K.H: -------------------------------------------------------------------------------- 1 | #ifndef slope_K_H_ 2 | #define slope_K_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | AMREX_GPU_DEVICE 9 | AMREX_FORCE_INLINE 10 | void slopex2(amrex::Box const& bx, 11 | amrex::Array4 const& dq, 12 | amrex::Array4 const& q) 13 | { 14 | using namespace amrex; 15 | 16 | const auto lo = amrex::lbound(bx); 17 | const auto hi = amrex::ubound(bx); 18 | 19 | for (int k = lo.z; k <= hi.z; ++k) { 20 | for (int j = lo.y; j <= hi.y; ++j) { 21 | for (int i = lo.x; i <= hi.x; ++i) { 22 | Real dlft = q(i,j,k) - q(i-1,j,k); 23 | Real drgt = q(i+1,j,k) - q(i,j,k); 24 | Real dcen = 0.5*(dlft+drgt); 25 | Real dsgn = amrex::Math::copysign(1.0, dcen); 26 | Real dslop = 2.0 * ((amrex::Math::abs(dlft) < amrex::Math::abs(drgt)) ? 27 | amrex::Math::abs(dlft) : amrex::Math::abs(drgt)); 28 | Real dlim = (dlft*drgt >= 0.0) ? dslop : 0.0; 29 | dq(i,j,k) = dsgn*amrex::min(dlim, amrex::Math::abs(dcen)); 30 | } 31 | } 32 | } 33 | } 34 | 35 | AMREX_GPU_DEVICE 36 | AMREX_FORCE_INLINE 37 | void slopex4(amrex::Box const& bx, 38 | amrex::Array4 const& dq4, 39 | amrex::Array4 const& q, 40 | amrex::Array4 const& dq) 41 | { 42 | using namespace amrex; 43 | 44 | const auto lo = amrex::lbound(bx); 45 | const auto hi = amrex::ubound(bx); 46 | 47 | for (int k = lo.z; k <= hi.z; ++k) { 48 | for (int j = lo.y; j <= hi.y; ++j) { 49 | for (int i = lo.x; i <= hi.x; ++i) { 50 | Real dlft = q(i,j,k) - q(i-1,j,k); 51 | Real drgt = q(i+1,j,k) - q(i,j,k); 52 | Real dcen = 0.5*(dlft+drgt); 53 | Real dsgn = amrex::Math::copysign(1.0, dcen); 54 | Real dslop = 2.0 * ((amrex::Math::abs(dlft) < amrex::Math::abs(drgt)) ? 55 | amrex::Math::abs(dlft) : amrex::Math::abs(drgt)); 56 | Real dlim = (dlft*drgt >= 0.0) ? dslop : 0.0; 57 | Real dq1 = 4.0/3.0*dcen - (1.0/6.0)*(dq(i+1,j,k) + dq(i-1,j,k)); 58 | dq4(i,j,k) = dsgn*amrex::min(dlim, amrex::Math::abs(dq1)); 59 | } 60 | } 61 | } 62 | } 63 | 64 | // *********************************************************** 65 | 66 | AMREX_GPU_DEVICE 67 | AMREX_FORCE_INLINE 68 | void slopey2(amrex::Box const& bx, 69 | amrex::Array4 const& dq, 70 | amrex::Array4 const& q) 71 | { 72 | using namespace amrex; 73 | 74 | const auto lo = amrex::lbound(bx); 75 | const auto hi = amrex::ubound(bx); 76 | 77 | for (int k = lo.z; k <= hi.z; ++k) { 78 | for (int j = lo.y; j <= hi.y; ++j) { 79 | for (int i = lo.x; i <= hi.x; ++i) { 80 | Real dlft = q(i,j,k) - q(i,j-1,k); 81 | Real drgt = q(i,j+1,k) - q(i,j,k); 82 | Real dcen = 0.5*(dlft+drgt); 83 | Real dsgn = amrex::Math::copysign(1.0, dcen); 84 | Real dslop = 2.0 * ((amrex::Math::abs(dlft) < amrex::Math::abs(drgt)) ? 85 | amrex::Math::abs(dlft) : amrex::Math::abs(drgt)); 86 | Real dlim = (dlft*drgt >= 0.0) ? dslop : 0.0; 87 | dq(i,j,k) = dsgn*amrex::min(dlim, amrex::Math::abs(dcen)); 88 | } 89 | } 90 | } 91 | } 92 | 93 | AMREX_GPU_DEVICE 94 | AMREX_FORCE_INLINE 95 | void slopey4(amrex::Box const& bx, 96 | amrex::Array4 const& dq4, 97 | amrex::Array4 const& q, 98 | amrex::Array4 const& dq) 99 | { 100 | using namespace amrex; 101 | 102 | const auto lo = amrex::lbound(bx); 103 | const auto hi = amrex::ubound(bx); 104 | 105 | for (int k = lo.z; k <= hi.z; ++k) { 106 | for (int j = lo.y; j <= hi.y; ++j) { 107 | for (int i = lo.x; i <= hi.x; ++i) { 108 | Real dlft = q(i,j,k) - q(i,j-1,k); 109 | Real drgt = q(i,j+1,k) - q(i,j,k); 110 | Real dcen = 0.5*(dlft+drgt); 111 | Real dsgn = amrex::Math::copysign(1.0, dcen); 112 | Real dslop = 2.0 * ((amrex::Math::abs(dlft) < amrex::Math::abs(drgt)) ? 113 | amrex::Math::abs(dlft) : amrex::Math::abs(drgt)); 114 | Real dlim = (dlft*drgt >= 0.0) ? dslop : 0.0; 115 | Real dq1 = 4.0/3.0*dcen - (1.0/6.0)*(dq(i,j+1,k) + dq(i,j-1,k)); 116 | dq4(i,j,k) = dsgn*amrex::min(dlim, amrex::Math::abs(dq1)); 117 | } 118 | } 119 | } 120 | } 121 | 122 | // *********************************************************** 123 | 124 | AMREX_GPU_DEVICE 125 | AMREX_FORCE_INLINE 126 | void slopez2(amrex::Box const& bx, 127 | amrex::Array4 const& dq, 128 | amrex::Array4 const& q) 129 | { 130 | using namespace amrex; 131 | 132 | const auto lo = amrex::lbound(bx); 133 | const auto hi = amrex::ubound(bx); 134 | 135 | for (int k = lo.z; k <= hi.z; ++k) { 136 | for (int j = lo.y; j <= hi.y; ++j) { 137 | for (int i = lo.x; i <= hi.x; ++i) { 138 | Real dlft = q(i,j,k) - q(i,j,k-1); 139 | Real drgt = q(i,j,k+1) - q(i,j,k); 140 | Real dcen = 0.5*(dlft+drgt); 141 | Real dsgn = amrex::Math::copysign(1.0, dcen); 142 | Real dslop = 2.0 * ((amrex::Math::abs(dlft) < amrex::Math::abs(drgt)) ? 143 | amrex::Math::abs(dlft) : amrex::Math::abs(drgt)); 144 | Real dlim = (dlft*drgt >= 0.0) ? dslop : 0.0; 145 | dq(i,j,k) = dsgn*amrex::min(dlim, amrex::Math::abs(dcen)); 146 | } 147 | } 148 | } 149 | } 150 | 151 | AMREX_GPU_DEVICE 152 | AMREX_FORCE_INLINE 153 | void slopez4(amrex::Box const& bx, 154 | amrex::Array4 const& dq4, 155 | amrex::Array4 const& q, 156 | amrex::Array4 const& dq) 157 | { 158 | using namespace amrex; 159 | 160 | const auto lo = amrex::lbound(bx); 161 | const auto hi = amrex::ubound(bx); 162 | 163 | for (int k = lo.z; k <= hi.z; ++k) { 164 | for (int j = lo.y; j <= hi.y; ++j) { 165 | for (int i = lo.x; i <= hi.x; ++i) { 166 | Real dlft = q(i,j,k) - q(i,j,k-1); 167 | Real drgt = q(i,j,k+1) - q(i,j,k); 168 | Real dcen = 0.5*(dlft+drgt); 169 | Real dsgn = amrex::Math::copysign(1.0, dcen); 170 | Real dslop = 2.0 * ((amrex::Math::abs(dlft) < amrex::Math::abs(drgt)) ? 171 | amrex::Math::abs(dlft) : amrex::Math::abs(drgt)); 172 | Real dlim = (dlft*drgt >= 0.0) ? dslop : 0.0; 173 | Real dq1 = 4.0/3.0*dcen - (1.0/6.0)*(dq(i,j,k+1) + dq(i,j,k-1)); 174 | dq4(i,j,k) = dsgn*amrex::min(dlim, amrex::Math::abs(dq1)); 175 | } 176 | } 177 | } 178 | } 179 | 180 | #endif 181 | -------------------------------------------------------------------------------- /07_Advection_EB/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | if (AMReX_SPACEDIM EQUAL 1) 2 | return() 3 | endif () 4 | 5 | # List of source files 6 | set(_sources EB_Cylinder.cpp FluidParticleContainer.cpp FluidParticleContainer.H) 7 | list(APPEND _sources DefineVelocity.cpp face_velocity.H main.cpp Indexing.H) 8 | list(APPEND _sources mac_project_velocity.cpp) 9 | 10 | # List of input files 11 | set(_input_files inputs Visualization.ipynb) 12 | 13 | setup_tutorial(_sources _input_files) 14 | 15 | unset( _sources ) 16 | unset( _input_files ) 17 | -------------------------------------------------------------------------------- /07_Advection_EB/DefineVelocity.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "face_velocity.H" 4 | 5 | using namespace amrex; 6 | 7 | void 8 | define_velocity (const Real time, const Geometry& geom, Array& vel_out, const MultiFab& phi) 9 | { 10 | const auto dx = geom.CellSizeArray(); 11 | const auto prob_lo = geom.ProbLoArray(); 12 | const Box& domain = geom.Domain(); 13 | 14 | const auto domlo = amrex::lbound(domain); 15 | const auto domhi = amrex::ubound(domain); 16 | 17 | #ifdef _OPENMP 18 | #pragma omp parallel if (Gpu::notInLaunchRegion()) 19 | #endif 20 | { 21 | for (MFIter mfi(phi,TilingIfNotGPU()); mfi.isValid(); ++mfi) 22 | { 23 | GpuArray nbx; 24 | AMREX_D_TERM(nbx[0] = mfi.tilebox(IntVect{AMREX_D_DECL(1,0,0)});, // x-face-based tilebox 25 | nbx[1] = mfi.tilebox(IntVect{AMREX_D_DECL(0,1,0)});, // y-face-based tilebox 26 | nbx[2] = mfi.tilebox(IntVect{AMREX_D_DECL(0,0,1)});); // z-face-based tilebox 27 | 28 | AMREX_D_TERM(const Box& ngbxx = amrex::grow(mfi.nodaltilebox(0),1);, 29 | const Box& ngbxy = amrex::grow(mfi.nodaltilebox(1),1);, 30 | const Box& ngbxz = amrex::grow(mfi.nodaltilebox(2),1);); 31 | 32 | GpuArray, AMREX_SPACEDIM> vel{ AMREX_D_DECL( vel_out[0].array(mfi), 33 | vel_out[1].array(mfi), 34 | vel_out[2].array(mfi)) }; 35 | 36 | const Box& psibox = Box(IntVect(AMREX_D_DECL(std::min(ngbxx.smallEnd(0)-1, ngbxy.smallEnd(0)-1), 37 | std::min(ngbxx.smallEnd(1)-1, ngbxy.smallEnd(0)-1), 38 | 0)), 39 | IntVect(AMREX_D_DECL(std::max(ngbxx.bigEnd(0), ngbxy.bigEnd(0)+1), 40 | std::max(ngbxx.bigEnd(1)+1, ngbxy.bigEnd(1)), 41 | 0))); 42 | 43 | FArrayBox psifab(psibox, 1); 44 | Elixir psieli = psifab.elixir(); 45 | Array4 psi = psifab.array(); 46 | GeometryData geomdata = geom.data(); 47 | 48 | amrex::launch(psibox, 49 | [=] AMREX_GPU_DEVICE (const Box& tbx) 50 | { 51 | get_face_velocity_psi(tbx, time, psi, geomdata); 52 | }); 53 | 54 | AMREX_D_TERM( 55 | amrex::ParallelFor(ngbxx, 56 | [=] AMREX_GPU_DEVICE (int i, int j, int k) 57 | { 58 | get_face_velocity_x(i, j, k, vel[0], psi, prob_lo, dx); 59 | if (i == domlo.x or i == domhi.x+1) vel[0](i,j,k) = 0.; 60 | });, 61 | 62 | amrex::ParallelFor(ngbxy, 63 | [=] AMREX_GPU_DEVICE (int i, int j, int k) 64 | { 65 | get_face_velocity_y(i, j, k, vel[1], psi, prob_lo, dx); 66 | if (j == domlo.y or j == domhi.y+1) vel[1](i,j,k) = 0.; 67 | });, 68 | 69 | amrex::ParallelFor(ngbxz, 70 | [=] AMREX_GPU_DEVICE (int i, int j, int k) 71 | { 72 | get_face_velocity_z(i, j, k, vel[2], psi, prob_lo, dx); 73 | if (k == domlo.z or k == domhi.z+1) vel[2](i,j,k) = 0.; 74 | }); 75 | ); 76 | } 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /07_Advection_EB/EB_Cylinder.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include 6 | 7 | using namespace amrex; 8 | 9 | /******************************************************************************** 10 | * * 11 | * Function to create a simple cylinder EB. * 12 | * * 13 | ********************************************************************************/ 14 | void 15 | make_eb_cylinder(const Geometry& geom) 16 | { 17 | // Initialise cylinder parameters 18 | bool inside = true; 19 | Real radius = 0.0002; 20 | int direction = 0; 21 | Vector centervec(3); 22 | 23 | // Get cylinder information from inputs file. * 24 | ParmParse pp("cylinder"); 25 | 26 | pp.query("internal_flow", inside); 27 | pp.query("radius", radius); 28 | pp.query("direction", direction); 29 | pp.getarr("center", centervec, 0, 3); 30 | Array center = {AMREX_D_DECL(centervec[0], centervec[1], centervec[2])}; 31 | 32 | // Print info about cylinder 33 | amrex::Print() << " " << std::endl; 34 | amrex::Print() << " Internal Flow: " << inside << std::endl; 35 | amrex::Print() << " Radius: " << radius << std::endl; 36 | amrex::Print() << " Direction: " << direction << std::endl; 37 | amrex::Print() << " Center: " << center[0] << ", " << center[1] 38 | #if (AMREX_SPACEDIM == 3) 39 | << ", " << center[2] 40 | #endif 41 | << std::endl; 42 | 43 | // Build the Cylinder implicit function representing the curved walls 44 | #if (AMREX_SPACEDIM == 2) 45 | // In 2D the sphere becomes a circle 46 | EB2::SphereIF my_cyl(radius, center, inside); 47 | #else 48 | EB2::CylinderIF my_cyl(radius, direction, center, inside); 49 | #endif 50 | 51 | // Generate GeometryShop 52 | auto gshop = EB2::makeShop(my_cyl); 53 | 54 | // Build index space 55 | int max_level_here = 0; 56 | int max_coarsening_level = 100; 57 | EB2::Build(gshop, geom, max_level_here, max_level_here + max_coarsening_level); 58 | } 59 | -------------------------------------------------------------------------------- /07_Advection_EB/FluidParticleContainer.H: -------------------------------------------------------------------------------- 1 | #ifndef BL_FLUIDPARTICLES_H_ 2 | #define BL_FLUIDPARTICLES_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #include "Indexing.H" 9 | 10 | namespace Interpolation { 11 | enum {NGP=0, CIC}; 12 | } 13 | 14 | namespace PIdx { 15 | enum {AMREX_D_DECL(BufferX=0, BufferY, BufferZ), Weight, NStructReal}; 16 | } 17 | 18 | namespace amrex { 19 | 20 | class FluidParticleContainer 21 | : public ParticleContainer 22 | { 23 | private: 24 | int m_number_particles_per_cell; 25 | 26 | public: 27 | 28 | FluidParticleContainer (ParGDBBase* gdb) 29 | : ParticleContainer(gdb), 30 | m_number_particles_per_cell(0) 31 | {} 32 | 33 | FluidParticleContainer (const Geometry & geom, 34 | const DistributionMapping & dmap, 35 | const BoxArray & ba) 36 | : ParticleContainer(geom,dmap,ba), 37 | m_number_particles_per_cell(0) 38 | {} 39 | 40 | ~FluidParticleContainer () {} 41 | 42 | void InitParticles(const MultiFab& phi, const MultiFab& ebvol, Real density_cutoff, int nppc, int interpolation=Interpolation::CIC); 43 | 44 | int NumParticlesPerCell() { return m_number_particles_per_cell; } 45 | 46 | Real SumPhi(); 47 | 48 | void AdvectWithUmac (MultiFab* umac, int lev, Real dt); 49 | 50 | void DepositToMesh (MultiFab& phi, int interpolation=Interpolation::CIC); 51 | 52 | void InterpolateFromMesh (const MultiFab& phi, int interpolation=Interpolation::CIC); 53 | 54 | void RemoveCoveredParticles (const MultiFab& ebvol, Real density_cutoff); 55 | }; 56 | 57 | } 58 | 59 | #endif 60 | 61 | -------------------------------------------------------------------------------- /07_Advection_EB/FluidParticleContainer.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include "FluidParticleContainer.H" 3 | 4 | namespace amrex { 5 | 6 | // 7 | // Initialize a random number of particles per cell determined by nppc 8 | // 9 | void 10 | FluidParticleContainer::InitParticles(const MultiFab& phi, const MultiFab& ebvol, Real density_cutoff, int nppc, int interpolation) 11 | { 12 | // Save the number of particles per cell we are using for the particle-mesh operations 13 | m_number_particles_per_cell = nppc; 14 | 15 | // Construct a ParticleInitData containing only zeros for the particle buffers and weights 16 | amrex::ParticleInitType pdata {}; 17 | 18 | if (nppc > 1) { 19 | // Create nppc random particles per cell, initialized with zero real struct data 20 | InitNRandomPerCell(nppc, pdata); 21 | } else if (nppc == 1) { 22 | // Or ... create just one particle per cell centered in the cell 23 | InitOnePerCell(0.5, 0.5, 0.5, pdata); 24 | } else { 25 | Print() << "No particles initialized.\n"; 26 | } 27 | 28 | // Interpolate from density field phi to set particle weights 29 | InterpolateFromMesh(phi, interpolation); 30 | 31 | // Set invalid particle IDs for particles from cells covered by the embedded geometry 32 | // or where density < density_cutoff 33 | RemoveCoveredParticles(ebvol, density_cutoff); 34 | 35 | // Redistribute to remove the EB-covered particles based on the invalid IDs 36 | Redistribute(); 37 | } 38 | 39 | // 40 | // Sum up particle phi across the domain from the weights 41 | // 42 | Real 43 | FluidParticleContainer::SumPhi() 44 | { 45 | const auto geom = Geom(0); 46 | const auto plo = geom.ProbLoArray(); 47 | const auto dxi = geom.InvCellSizeArray(); 48 | const auto dx = geom.CellSizeArray(); 49 | const Real cell_volume = AMREX_D_TERM(dx[0], *dx[1], *dx[2]); 50 | const Real volume_per_particle = cell_volume / NumParticlesPerCell(); 51 | 52 | using PType = typename FluidParticleContainer::SuperParticleType; 53 | Real sum_phi = amrex::ReduceSum(*this, [=] AMREX_GPU_HOST_DEVICE (const PType& p) -> Real { return p.rdata(PIdx::Weight) / volume_per_particle; }); 54 | 55 | ParallelDescriptor::ReduceRealSum(sum_phi); 56 | return sum_phi; 57 | } 58 | 59 | // 60 | // Uses midpoint method to advance particles using umac. 61 | // 62 | void 63 | FluidParticleContainer::AdvectWithUmac (MultiFab* umac, int lev, Real dt) 64 | { 65 | BL_PROFILE("FluidParticleContainer::AdvectWithUmac()"); 66 | AMREX_ASSERT(OK(lev, lev, umac[0].nGrow()-1)); 67 | AMREX_ASSERT(lev >= 0 && lev < GetParticles().size()); 68 | 69 | AMREX_D_TERM(AMREX_ASSERT(umac[0].nGrow() >= 1);, 70 | AMREX_ASSERT(umac[1].nGrow() >= 1);, 71 | AMREX_ASSERT(umac[2].nGrow() >= 1);); 72 | 73 | AMREX_D_TERM(AMREX_ASSERT(!umac[0].contains_nan());, 74 | AMREX_ASSERT(!umac[1].contains_nan());, 75 | AMREX_ASSERT(!umac[2].contains_nan());); 76 | 77 | const Real strttime = amrex::second(); 78 | const Geometry& geom = m_gdb->Geom(lev); 79 | const auto plo = geom.ProbLoArray(); 80 | const auto dxi = geom.InvCellSizeArray(); 81 | 82 | Vector > raii_umac(AMREX_SPACEDIM); 83 | Vector umac_pointer(AMREX_SPACEDIM); 84 | if (OnSameGrids(lev, umac[0])) 85 | { 86 | for (int i = 0; i < AMREX_SPACEDIM; i++) { 87 | umac_pointer[i] = &umac[i]; 88 | } 89 | } 90 | else 91 | { 92 | for (int i = 0; i < AMREX_SPACEDIM; i++) 93 | { 94 | int ng = umac[i].nGrow(); 95 | raii_umac[i].reset(new MultiFab(amrex::convert(m_gdb->ParticleBoxArray(lev), 96 | IntVect::TheDimensionVector(i)), 97 | 98 | m_gdb->ParticleDistributionMap(lev), 99 | umac[i].nComp(), ng)); 100 | 101 | 102 | umac_pointer[i] = raii_umac[i].get(); 103 | umac_pointer[i]->ParallelCopy(umac[i],0,0,umac[i].nComp(),ng,ng); 104 | } 105 | } 106 | 107 | for (int ipass = 0; ipass < 2; ipass++) 108 | { 109 | #ifdef _OPENMP 110 | #pragma omp parallel if (Gpu::notInLaunchRegion()) 111 | #endif 112 | for (ParIterType pti(*this, lev); pti.isValid(); ++pti) 113 | { 114 | int grid = pti.index(); 115 | auto& ptile = ParticlesAt(lev, pti); 116 | auto& aos = ptile.GetArrayOfStructs(); 117 | const int n = aos.numParticles(); 118 | auto p_pbox = aos().data(); 119 | const FArrayBox* fab[AMREX_SPACEDIM] = { AMREX_D_DECL(&((*umac_pointer[0])[grid]), 120 | &((*umac_pointer[1])[grid]), 121 | &((*umac_pointer[2])[grid])) }; 122 | 123 | //array of these pointers to pass to the GPU 124 | amrex::GpuArray, AMREX_SPACEDIM> 125 | const umacarr {{AMREX_D_DECL((*fab[0]).array(), 126 | (*fab[1]).array(), 127 | (*fab[2]).array() )}}; 128 | 129 | amrex::ParallelFor(n, 130 | [=] AMREX_GPU_DEVICE (int i) 131 | { 132 | ParticleType& p = p_pbox[i]; 133 | if (p.id() <= 0) return; 134 | Real v[AMREX_SPACEDIM]; 135 | mac_interpolate(p, plo, dxi, umacarr, v); 136 | if (ipass == 0) 137 | { 138 | for (int dim=0; dim < AMREX_SPACEDIM; dim++) 139 | { 140 | p.rdata(dim) = p.pos(dim); 141 | p.pos(dim) += 0.5*dt*v[dim]; 142 | } 143 | } 144 | else 145 | { 146 | for (int dim=0; dim < AMREX_SPACEDIM; dim++) 147 | { 148 | p.pos(dim) = p.rdata(dim) + dt*v[dim]; 149 | p.rdata(dim) = v[dim]; 150 | } 151 | } 152 | }); 153 | } 154 | } 155 | 156 | if (m_verbose > 1) 157 | { 158 | Real stoptime = amrex::second() - strttime; 159 | 160 | #ifdef AMREX_LAZY 161 | Lazy::QueueReduction( [=] () mutable { 162 | #endif 163 | ParallelReduce::Max(stoptime, ParallelContext::IOProcessorNumberSub(), 164 | ParallelContext::CommunicatorSub()); 165 | 166 | amrex::Print() << "FluidParticleContainer::AdvectWithUmac() time: " << stoptime << '\n'; 167 | #ifdef AMREX_LAZY 168 | }); 169 | #endif 170 | } 171 | } 172 | 173 | // 174 | // Deposit the particle weights to the mesh to set the number density phi 175 | // 176 | void 177 | FluidParticleContainer::DepositToMesh (MultiFab& phi, int interpolation) 178 | { 179 | const auto geom = Geom(0); 180 | const auto plo = geom.ProbLoArray(); 181 | const auto dxi = geom.InvCellSizeArray(); 182 | const Real inv_cell_volume = AMREX_D_TERM(dxi[0], *dxi[1], *dxi[2]); 183 | amrex::ParticleToMesh(*this, phi, 0, 184 | [=] AMREX_GPU_DEVICE (const FluidParticleContainer::ParticleType& p, 185 | amrex::Array4 const& phi_arr) 186 | { 187 | AMREX_D_TERM( 188 | amrex::Real lx = (p.pos(0) - plo[0]) * dxi[0] + 0.5;, 189 | amrex::Real ly = (p.pos(1) - plo[1]) * dxi[1] + 0.5;, 190 | amrex::Real lz = (p.pos(2) - plo[2]) * dxi[2] + 0.5; 191 | ); 192 | 193 | int i = 0; 194 | int j = 0; 195 | int k = 0; 196 | 197 | AMREX_D_TERM( 198 | i = amrex::Math::floor(lx);, 199 | j = amrex::Math::floor(ly);, 200 | k = amrex::Math::floor(lz); 201 | ); 202 | 203 | AMREX_D_TERM( 204 | amrex::Real xint = lx - i;, 205 | amrex::Real yint = ly - j;, 206 | amrex::Real zint = lz - k; 207 | ); 208 | 209 | // Nearest Grid Point Interpolation (NGP) 210 | amrex::Real sx[] = {0.0, 1.0}; 211 | amrex::Real sy[] = {0.0, 1.0}; 212 | amrex::Real sz[] = {0.0, 1.0}; 213 | 214 | // Cloud In Cell interpolation (CIC) 215 | if (interpolation == Interpolation::CIC) { 216 | AMREX_D_EXPR(sx[0] = 1.-xint, sy[0] = 1.-yint, sz[0] = 1.-zint); 217 | AMREX_D_EXPR(sx[1] = xint, sy[1] = yint, sz[1] = zint); 218 | } 219 | 220 | // Add up the number of physical particles represented by our particle weights to the grid 221 | // and divide by the cell volume to get the number density on the grid. 222 | for (int kk = AMREX_D_PICK(1, 1, 0); kk <= 1; ++kk) { 223 | for (int jj = AMREX_D_PICK(1, 0, 0); jj <= 1; ++jj) { 224 | for (int ii = 0; ii <= 1; ++ii) { 225 | amrex::Gpu::Atomic::Add(&phi_arr(i+ii-1, j+jj-1, k+kk-1), sx[ii]*sy[jj]*sz[kk] * p.rdata(PIdx::Weight) * inv_cell_volume); 226 | }}} 227 | }); 228 | } 229 | 230 | // 231 | // Interpolate number density phi to the particles to set their weights 232 | // 233 | void 234 | FluidParticleContainer::InterpolateFromMesh (const MultiFab& phi, int interpolation) 235 | { 236 | const auto geom = Geom(0); 237 | const auto plo = geom.ProbLoArray(); 238 | const auto dxi = geom.InvCellSizeArray(); 239 | const auto dx = geom.CellSizeArray(); 240 | const Real cell_volume = AMREX_D_TERM(dx[0], *dx[1], *dx[2]); 241 | const Real volume_per_particle = cell_volume / NumParticlesPerCell(); 242 | 243 | amrex::MeshToParticle(*this, phi, 0, 244 | [=] AMREX_GPU_DEVICE (FluidParticleContainer::ParticleType& p, 245 | amrex::Array4 const& phi_arr) 246 | { 247 | AMREX_D_TERM( 248 | amrex::Real lx = (p.pos(0) - plo[0]) * dxi[0] + 0.5;, 249 | amrex::Real ly = (p.pos(1) - plo[1]) * dxi[1] + 0.5;, 250 | amrex::Real lz = (p.pos(2) - plo[2]) * dxi[2] + 0.5; 251 | ); 252 | 253 | int i = 0; 254 | int j = 0; 255 | int k = 0; 256 | 257 | AMREX_D_TERM( 258 | i = amrex::Math::floor(lx);, 259 | j = amrex::Math::floor(ly);, 260 | k = amrex::Math::floor(lz); 261 | ); 262 | 263 | AMREX_D_TERM( 264 | amrex::Real xint = lx - i;, 265 | amrex::Real yint = ly - j;, 266 | amrex::Real zint = lz - k; 267 | ); 268 | 269 | // Nearest Grid Point Interpolation (NGP) 270 | amrex::Real sx[] = {0.0, 1.0}; 271 | amrex::Real sy[] = {0.0, 1.0}; 272 | amrex::Real sz[] = {0.0, 1.0}; 273 | 274 | // Cloud In Cell interpolation (CIC) 275 | if (interpolation == Interpolation::CIC) { 276 | AMREX_D_EXPR(sx[0] = 1.-xint, sy[0] = 1.-yint, sz[0] = 1.-zint); 277 | AMREX_D_EXPR(sx[1] = xint, sy[1] = yint, sz[1] = zint); 278 | } 279 | 280 | // The particle weight is the number of physical particles it represents. 281 | // This is set from the number density in the grid phi in 2 steps: 282 | 283 | // Step 1: interpolate number density phi using the particle shape factor for CIC or NGP 284 | amrex::Real interpolated_phi = 0.0; 285 | 286 | for (int kk = AMREX_D_PICK(1, 1, 0); kk <= 1; ++kk) { 287 | for (int jj = AMREX_D_PICK(1, 0, 0); jj <= 1; ++jj) { 288 | for (int ii = 0; ii <= 1; ++ii) { 289 | interpolated_phi += sx[ii]*sy[jj]*sz[kk] * phi_arr(i+ii-1,j+jj-1,k+kk-1); 290 | }}} 291 | 292 | // Step 2: scale interpolated number density by the volume per particle and set particle weight 293 | p.rdata(PIdx::Weight) = interpolated_phi * volume_per_particle; 294 | }); 295 | } 296 | 297 | // 298 | // Remove particles covered by the embedded geometry 299 | // 300 | void 301 | FluidParticleContainer::RemoveCoveredParticles (const MultiFab& ebvol, const Real density_cutoff) 302 | { 303 | const auto geom = Geom(0); 304 | const auto plo = geom.ProbLoArray(); 305 | const auto dxi = geom.InvCellSizeArray(); 306 | const auto dx = geom.CellSizeArray(); 307 | const Real cell_volume = AMREX_D_TERM(dx[0], *dx[1], *dx[2]); 308 | const Real volume_per_particle = cell_volume / NumParticlesPerCell(); 309 | 310 | // Get the particle weight corresponding to the density cutoff on the grid 311 | const amrex::Real weight_cutoff = density_cutoff * volume_per_particle; 312 | 313 | amrex::MeshToParticle(*this, ebvol, 0, 314 | [=] AMREX_GPU_DEVICE (FluidParticleContainer::ParticleType& p, 315 | amrex::Array4 const& vol_arr) 316 | { 317 | AMREX_D_TERM( 318 | amrex::Real lx = (p.pos(0) - plo[0]) * dxi[0] + 0.5;, 319 | amrex::Real ly = (p.pos(1) - plo[1]) * dxi[1] + 0.5;, 320 | amrex::Real lz = (p.pos(2) - plo[2]) * dxi[2] + 0.5; 321 | ); 322 | 323 | int i = 0; 324 | int j = 0; 325 | int k = 0; 326 | 327 | AMREX_D_TERM( 328 | i = amrex::Math::floor(lx);, 329 | j = amrex::Math::floor(ly);, 330 | k = amrex::Math::floor(lz); 331 | ); 332 | 333 | // Get the EB volume fraction of the cell this particle is in 334 | const amrex::Real cell_vol = vol_arr(i,j,k); 335 | 336 | // If the cell volume = 0, then the particle is covered by the EB 337 | // so we want to delete the particle in the next call to Redistribute(). 338 | // We also delete the particle if the weight is zero or less than the cutoff weight. 339 | if (cell_vol == 0.0 || p.rdata(PIdx::Weight) == 0.0 || p.rdata(PIdx::Weight) < weight_cutoff) { 340 | p.id() = -1; 341 | } 342 | }); 343 | } 344 | 345 | } 346 | -------------------------------------------------------------------------------- /07_Advection_EB/Indexing.H: -------------------------------------------------------------------------------- 1 | #ifndef _INDEXING_H_ 2 | #define _INDEXING_H_ 3 | 4 | // Indexes into the grid data: velocity, processor id, and density phi 5 | namespace Idx { 6 | enum GridIndexes {AMREX_D_DECL(xvel=0, yvel, zvel), proc, phi}; 7 | } 8 | 9 | #endif 10 | -------------------------------------------------------------------------------- /07_Advection_EB/Make.package: -------------------------------------------------------------------------------- 1 | CEXE_sources += main.cpp 2 | CEXE_sources += DefineVelocity.cpp 3 | CEXE_sources += EB_Cylinder.cpp 4 | CEXE_sources += FluidParticleContainer.cpp 5 | CEXE_sources += mac_project_velocity.cpp 6 | 7 | CEXE_headers += face_velocity.H 8 | CEXE_headers += FluidParticleContainer.H 9 | -------------------------------------------------------------------------------- /07_Advection_EB/Visualization.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "3643c386", 6 | "metadata": {}, 7 | "source": [ 8 | "## Example: \"AMR102: Advection of Particles Around Obstacles\"\n", 9 | "\n", 10 | "### What Features Are We Using\n", 11 | "\n", 12 | "* Mesh data with EB \n", 13 | "* Linear solvers (multigrid)\n", 14 | "* Particle-Mesh interpolation\n", 15 | "\n", 16 | "### The Problem\n", 17 | "\n", 18 | "Recall our previous problem of the drop of dye in a thin incompressible fluid that is spinning \n", 19 | "clock-wise then counter-clockwise with a prescribed motion. \n", 20 | "\n", 21 | "Now instead of advecting the dye as a scalar quantity defined on the mesh (the\n", 22 | "continuum representation), we define the dye as a collection of particles that\n", 23 | "are advected by the fluid velocity.\n", 24 | "\n", 25 | "Again the fluid is thin enough that we can model this as two-dimensional\n", 26 | "motion; again we have the option of solving in a 2D or 3D computational domain.\n", 27 | "\n", 28 | "To make things even more interesting, there is now an object in the flow, in this case a cylinder.\n", 29 | "It would be very difficult to analytically specify the flow field around the object, so instead \n", 30 | "we project the velocity field so that the resulting field represents incompressible flow around the object.\n", 31 | "\n", 32 | "### Projecting the Velocity for Incompressible Flow around the Cylinder\n", 33 | "\n", 34 | "Mathematically, projecting the specified velocity field means solving \n", 35 | "\n", 36 | "$$\\nabla \\cdot (\\beta \\nabla \\xi) = \\nabla \\cdot \\bf{u^{spec}}$$\n", 37 | "\n", 38 | "and setting \n", 39 | "\n", 40 | "$$\\bf{u} = \\bf{u^{spec}} - \\beta \\nabla \\xi$$\n", 41 | "\n", 42 | "To solve this variable coefficient Poisson equation, we use the native AMReX geometric multigrid solver.\n", 43 | "In our case $\\beta = 1 / \\rho = 1$ since we are assuming constant density $\\rho.$\n", 44 | "\n", 45 | "Note that for this example we are solving everything at a single level for convenience,\n", 46 | "but linear solvers, EB and particles all have full multi-level functionality.\n", 47 | "\n", 48 | "In each timestep we compute the projected velocity field, advect the particles with this velocity, then interpolate the particles onto the mesh to determine $\\phi(x,y,z)$.\n", 49 | "\n", 50 | "### Particle-In-Cell Algorithm for Advecting $\\phi$\n", 51 | "\n", 52 | "We achieve conservation by interpreting the scalar $\\phi$ as the number\n", 53 | "density of physical dye particles in the fluid, and we represent these physical\n", 54 | "particles by \"computational\" particles $p$. Each particle $p$ stores a\n", 55 | "weight $w_p$ equal to the number of physical dye particles it represents.\n", 56 | "\n", 57 | "This allows us to define particle-mesh interpolation that converts\n", 58 | "$\\phi(x,y,z)$ to a set of particles with weights $w_p$. First, we\n", 59 | "interpolate $\\phi(x,y,z)$ to $\\phi_p$ at each particle location by\n", 60 | "calculating:\n", 61 | "\n", 62 | "$$\\phi_p = \\sum_i \\sum_j \\sum_k S_x \\cdot S_y \\cdot S_z \\cdot \\phi(i,j,k)$$\n", 63 | "\n", 64 | "where in the above, $S_{x,y,z}$ are called shape factors, determined by the\n", 65 | "particle-in-cell interpolation scheme we wish to use.\n", 66 | "\n", 67 | "The simplest interpolation is nearest grid point (NGP), where $S_{x,y,z} = 1$\n", 68 | "if the particle $p$ is within cell $(i,j,k)$ and $S_{x,y,z} = 0$\n", 69 | "otherwise. An alternative is linear interpolation called cloud-in-cell (CIC),\n", 70 | "where the shape factors are determined by volume-weighting the particle's\n", 71 | "contribution to/from the nearest 8 cells (in 3D).\n", 72 | "\n", 73 | "Once we have interpolated $\\phi(x,y,z)$ to the particle to get $\\phi_p$, we\n", 74 | "scale it by the volume per particle to get the number of physical dye particles\n", 75 | "that our computational particle represents. Here, $n_{ppc}$ is the number of\n", 76 | "computational particles per cell.\n", 77 | "\n", 78 | "$$w_p = \\phi_p \\cdot \\dfrac{dx dy dz}{n_{ppc}}$$\n", 79 | "\n", 80 | "To go back from the particle representation to the grid representation, we\n", 81 | "reverse this procedure by summing up the number of physical dye particles in\n", 82 | "each grid cell and dividing by the grid cell volume. This recovers the number\n", 83 | "density $\\phi(x,y,z)$.\n", 84 | "\n", 85 | "$$\\phi(i,j,k) = \\dfrac{1}{dx dy dz} \\cdot \\sum_p S_x \\cdot S_y \\cdot S_z \\cdot w_p$$\n", 86 | "\n", 87 | "This approach is the basis for Particle-In-Cell (PIC) methods in a variety of\n", 88 | "fields, and in this tutorial you can experiment with the number of particles\n", 89 | "per cell and interpolation scheme to see how well you can resolve the dye\n", 90 | "advection." 91 | ] 92 | }, 93 | { 94 | "cell_type": "markdown", 95 | "id": "39ef1d22", 96 | "metadata": {}, 97 | "source": [ 98 | "## Running the code\n", 99 | "\n", 100 | "Similar to the last example, the following parameters can be set at run-time -- these are currently set in the inputs file.\n", 101 | "\n", 102 | "```\n", 103 | "stop_time = 2.0 # the final time (if we have not exceeded number of steps)\n", 104 | "max_step = 200 # the maximum number of steps (if we have not exceeded stop_time)\n", 105 | "\n", 106 | "n_cell = 64 # number of cells in x- and y-directions; z-dir has 1/8 n_cell (if 3D)\n", 107 | "\n", 108 | "max_grid_size = 32 # the maximum number of cells in any direction in a single grid\n", 109 | "\n", 110 | "plot_int = 10 # frequency of writing plotfiles\n", 111 | "\n", 112 | "```\n", 113 | "\n", 114 | "The size, orientation and location of the cylinder are specified in the inputs file as well:\n", 115 | "\n", 116 | "```\n", 117 | "cylinder.direction = 2 # cylinder axis aligns with z-axis\n", 118 | "cylinder.radius = 0.1 # cylinder radius\n", 119 | "cylinder.center = 0.7 0.5 0.5 # location of cylinder center (in domain that is unit box in xy plane)\n", 120 | "\n", 121 | "cylinder.internal_flow = false # we are computing flow around the cylinder, not inside it\n", 122 | "```\n", 123 | "\n", 124 | "Here you can play around with changing the size and location of the cylinder.\n", 125 | "\n", 126 | "The number of particles per cell and particle-mesh interpolation type are also specified in the inputs file:\n", 127 | "\n", 128 | "```\n", 129 | "n_ppc = 100 # number of particles per cell for representing the fluid\n", 130 | "\n", 131 | "pic_interpolation = 1 # Particle In Cell interpolation scheme:\n", 132 | " # 0 = Nearest Grid Point\n", 133 | " # 1 = Cloud In Cell\n", 134 | "```\n", 135 | "\n", 136 | "You can vary the number of particles per cell and interpolation to see how they influence the smoothness of the phi field." 137 | ] 138 | }, 139 | { 140 | "cell_type": "markdown", 141 | "id": "a6cf0d94", 142 | "metadata": {}, 143 | "source": [ 144 | "## Visualization" 145 | ] 146 | }, 147 | { 148 | "cell_type": "code", 149 | "execution_count": null, 150 | "id": "29609c78", 151 | "metadata": {}, 152 | "outputs": [], 153 | "source": [ 154 | "import yt\n", 155 | "from yt.frontends.boxlib.data_structures import AMReXDataset" 156 | ] 157 | }, 158 | { 159 | "cell_type": "code", 160 | "execution_count": null, 161 | "id": "cb533aa1", 162 | "metadata": {}, 163 | "outputs": [], 164 | "source": [ 165 | "ds = AMReXDataset(\"plt00120\")" 166 | ] 167 | }, 168 | { 169 | "cell_type": "code", 170 | "execution_count": null, 171 | "id": "ae107c08", 172 | "metadata": {}, 173 | "outputs": [], 174 | "source": [ 175 | "ds.field_list" 176 | ] 177 | }, 178 | { 179 | "cell_type": "code", 180 | "execution_count": null, 181 | "id": "899767fc", 182 | "metadata": {}, 183 | "outputs": [], 184 | "source": [ 185 | "sl = yt.SlicePlot(ds, 2, ('boxlib', 'phi'))\n", 186 | "sl.annotate_particles(ds.domain_width[2], stride=10, marker='.')" 187 | ] 188 | } 189 | ], 190 | "metadata": { 191 | "kernelspec": { 192 | "display_name": "Python 3", 193 | "language": "python", 194 | "name": "python3" 195 | }, 196 | "language_info": { 197 | "codemirror_mode": { 198 | "name": "ipython", 199 | "version": 3 200 | }, 201 | "file_extension": ".py", 202 | "mimetype": "text/x-python", 203 | "name": "python", 204 | "nbconvert_exporter": "python", 205 | "pygments_lexer": "ipython3", 206 | "version": "3.8.8" 207 | } 208 | }, 209 | "nbformat": 4, 210 | "nbformat_minor": 5 211 | } 212 | -------------------------------------------------------------------------------- /07_Advection_EB/face_velocity.H: -------------------------------------------------------------------------------- 1 | #ifndef FACE_VELOCITY_H_ 2 | #define FACE_VELOCITY_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | using namespace amrex; 9 | 10 | AMREX_GPU_DEVICE 11 | AMREX_FORCE_INLINE 12 | void get_face_velocity_psi(Box const& bx, 13 | const Real time, 14 | Array4 const& psi, 15 | GeometryData const& geomdata) 16 | { 17 | const auto lo = lbound(bx); 18 | const auto hi = ubound(bx); 19 | 20 | const Real* AMREX_RESTRICT prob_lo = geomdata.ProbLo(); 21 | const Real* AMREX_RESTRICT dx = geomdata.CellSize(); 22 | 23 | for (int j = lo.y; j <= hi.y; ++j) { 24 | Real y = dx[1]*(0.5+j) + prob_lo[1]; 25 | AMREX_PRAGMA_SIMD 26 | for (int i = lo.x; i <= hi.x; ++i) { 27 | Real x = dx[0]*(0.5+i) + prob_lo[0]; 28 | psi(i,j,0) = pow(sin(M_PI*x), 2) * pow(sin(M_PI*y), 2) 29 | * cos(M_PI*time/2.0) * 1.0/M_PI; 30 | } 31 | } 32 | } 33 | 34 | AMREX_GPU_DEVICE 35 | AMREX_FORCE_INLINE 36 | void get_face_velocity_x(int i, int j, int k, 37 | Array4 const& vx, 38 | Array4 const& psi, 39 | GpuArray prob_lo, 40 | GpuArray dx) 41 | { 42 | vx(i,j,k) = -( (psi(i,j+1,0)+psi(i-1,j+1,0)) - (psi(i,j-1,0)+psi(i-1,j-1,0)) ) * (0.25/dx[1]); 43 | } 44 | 45 | AMREX_GPU_DEVICE 46 | AMREX_FORCE_INLINE 47 | void get_face_velocity_y(int i, int j, int k, 48 | Array4 const& vy, 49 | Array4 const& psi, 50 | GpuArray prob_lo, 51 | GpuArray dx) 52 | { 53 | vy(i,j,k) = ( (psi(i+1,j,0)+psi(i+1,j-1,0)) - (psi(i-1,j,0)+psi(i-1,j-1,0)) ) * (0.25/dx[0]); 54 | } 55 | 56 | AMREX_GPU_DEVICE 57 | AMREX_FORCE_INLINE 58 | void get_face_velocity_z(int i, int j, int k, 59 | Array4 const& vz, 60 | Array4 const& psi, 61 | GpuArray prob_lo, 62 | GpuArray dx) 63 | { 64 | vz(i,j,k) = 0.0; 65 | } 66 | 67 | #endif 68 | -------------------------------------------------------------------------------- /07_Advection_EB/inputs: -------------------------------------------------------------------------------- 1 | max_step = 200 2 | stop_time = 2.0 3 | 4 | n_cell = 64 # number of cells in x- and y-directions; z-dir has 1/8 n_cell 5 | 6 | max_grid_size = 32 # the maximum number of cells in any direction in a single grid 7 | 8 | plot_int = 5 # frequency of writing plotfiles 9 | 10 | ##################################################################### 11 | # Control the number of particles and the particle/mesh interpolation 12 | ##################################################################### 13 | 14 | n_ppc = 100 # number of particles per cell for representing the fluid 15 | 16 | pic_interpolation = 1 # Particle In Cell interpolation scheme: 17 | # 0 = nearest grid point 18 | # 1 = cloud in cell 19 | 20 | write_initial_phi = 0 21 | 22 | ################################################### 23 | # Control the verbosity and tolerance of the solver 24 | ################################################### 25 | 26 | mac_proj.verbose = 0 27 | mac_proj.bottom_verbose = 0 28 | 29 | use_hypre = 0 # use hypre instead of native GMG to solve the problem 30 | 31 | ############################################### 32 | # Specify the location and size of the cylinder 33 | ############################################### 34 | 35 | cylinder.direction = 2 # cylinder axis aligns with z-axis 36 | cylinder.radius = 0.1 # cylinder axis aligns with z-axis 37 | cylinder.center = 0.7 0.5 0.5 38 | 39 | cylinder.internal_flow = false # we are computing flow around the cylinder, not inside it 40 | 41 | write_eb_geom = 1 42 | 43 | ###### 44 | # Misc 45 | ###### 46 | amrex.fpe_trap_invalid = 1 # generate a backtrace if any floating point errors are encountered 47 | -------------------------------------------------------------------------------- /07_Advection_EB/mac_project_velocity.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | void mac_project_velocity(amrex::Array& vel, const amrex::Geometry& geom, int use_hypre) 4 | { 5 | using namespace amrex; 6 | 7 | LPInfo lp_info; 8 | 9 | // If we want to use hypre to solve the full problem we need to not coarsen inside AMReX 10 | if (use_hypre) 11 | lp_info.setMaxCoarseningLevel(0); 12 | 13 | Array beta; 14 | 15 | for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { 16 | beta[idim].define(vel[idim].boxArray(), vel[idim].DistributionMap(), 1, 0, MFInfo(), vel[idim].Factory()); 17 | beta[idim].setVal(1.0); 18 | } 19 | 20 | MacProjector macproj({amrex::GetArrOfPtrs(vel)}, // mac velocity 21 | MLMG::Location::FaceCenter, // velocity located on face centers 22 | {amrex::GetArrOfConstPtrs(beta)}, // beta 23 | MLMG::Location::FaceCenter, // beta located on face centers 24 | MLMG::Location::CellCenter, // location of mac_phi 25 | {geom}, 26 | lp_info); // structure for passing info to the operator 27 | 28 | // Set bottom-solver to use hypre instead of native BiCGStab 29 | if (use_hypre) 30 | macproj.getMLMG().setBottomSolver(MLMG::BottomSolver::hypre); 31 | 32 | macproj.setDomainBC({AMREX_D_DECL(LinOpBCType::Neumann, 33 | LinOpBCType::Neumann, 34 | LinOpBCType::Periodic)}, 35 | {AMREX_D_DECL(LinOpBCType::Neumann, 36 | LinOpBCType::Neumann, 37 | LinOpBCType::Periodic)}); 38 | 39 | Real reltol = 1.e-8; 40 | Real abstol = 1.e-12; 41 | 42 | macproj.project(reltol, abstol); 43 | } 44 | -------------------------------------------------------------------------------- /07_Advection_EB/main.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #include "Indexing.H" 18 | #include "FluidParticleContainer.H" 19 | 20 | using namespace amrex; 21 | 22 | extern void make_eb_cylinder(const Geometry& geom); 23 | extern void define_velocity(const Real time, const Geometry& geo, Array& vel_out, const MultiFab& phi); 24 | extern void mac_project_velocity(Array& vel_out, const Geometry& geom, int use_hypre); 25 | 26 | Real est_time_step(const Real current_dt, const Geometry& geom, Array& vel, const Real cfl) 27 | { 28 | const Real change_max = 1.1; 29 | 30 | Real dt_est = std::numeric_limits::max(); 31 | 32 | const Real* dx = geom.CellSize(); 33 | 34 | const Vector coord_dir {AMREX_D_DECL("x", "y", "z")}; 35 | 36 | for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) 37 | { 38 | Real est = vel[idim].norm0(0,0,false); 39 | // amrex::Print() << "Max vel in " << coord_dir[idim] << "-direction is " << est << std::endl; 40 | dt_est = amrex::min(dt_est, dx[idim]/est); 41 | } 42 | 43 | ParallelDescriptor::ReduceRealMin(dt_est); 44 | 45 | // Apply our CFL 46 | dt_est *= cfl; 47 | 48 | // Do not grow the timestep by more than a ratio of change_max 49 | dt_est = std::min(dt_est, current_dt * change_max); 50 | 51 | return dt_est; 52 | } 53 | 54 | void write_plotfile(int step, Real time, const Geometry& geom, MultiFab& plotmf, 55 | FluidParticleContainer& pc, int write_ascii) 56 | { 57 | // Copy processor id into the component of plotfile_mf immediately following velocities 58 | int proc_comp = AMREX_SPACEDIM; 59 | for (MFIter mfi(plotmf); mfi.isValid(); ++mfi) 60 | plotmf[mfi].setVal(ParallelDescriptor::MyProc(),mfi.validbox(),proc_comp,1); 61 | 62 | std::stringstream sstream; 63 | sstream << "plt" << std::setw(5) << std::setfill('0') << step; 64 | std::string plotfile_name = sstream.str(); 65 | 66 | #if (AMREX_SPACEDIM == 2) 67 | EB_WriteSingleLevelPlotfile(plotfile_name, plotmf, 68 | { "xvel", "yvel", "proc", "phi" }, 69 | geom, time, 0); 70 | #elif (AMREX_SPACEDIM == 3) 71 | EB_WriteSingleLevelPlotfile(plotfile_name, plotmf, 72 | { "xvel", "yvel", "zvel", "proc", "phi" }, 73 | geom, time, 0); 74 | #endif 75 | 76 | pc.Checkpoint(plotfile_name, "particles", true); // Write particles to plotfile 77 | 78 | std::stringstream pstream; 79 | pstream << "part" << std::setw(5) << std::setfill('0') << step; 80 | const std::string ascii_filename = pstream.str(); 81 | 82 | if (write_ascii) { 83 | pc.WriteAsciiFile(ascii_filename); 84 | } 85 | 86 | } 87 | 88 | int main (int argc, char* argv[]) 89 | { 90 | // Turn off amrex-related output 91 | amrex::SetVerbose(0); 92 | 93 | amrex::Initialize(argc, argv); 94 | 95 | Real strt_time = amrex::second(); 96 | Real eb_strt_time; 97 | Real eb_stop_time; 98 | 99 | { 100 | int n_cell = 128; 101 | int max_grid_size = 32; 102 | int n_ppc = 4; 103 | int pic_interpolation = Interpolation::CIC; 104 | Real stop_time = 1000.0; 105 | int max_step = 100; 106 | int plot_int = 1; 107 | int write_ascii = 0; 108 | int write_initial_phi = 0; 109 | int write_eb_geom = 1; 110 | int use_hypre = 0; 111 | Real phi_cutoff = 0.1; 112 | Real cfl = 0.7; 113 | 114 | Real dt = std::numeric_limits::max(); 115 | 116 | // read parameters 117 | { 118 | ParmParse pp; 119 | pp.query("n_cell", n_cell); 120 | pp.query("max_grid_size", max_grid_size); 121 | pp.query("n_ppc", n_ppc); 122 | pp.query("pic_interpolation", pic_interpolation); 123 | pp.query("stop_time", stop_time); 124 | pp.query("max_step", max_step); 125 | pp.query("plot_int", plot_int); 126 | pp.query("use_hypre", use_hypre); 127 | pp.query("write_ascii", write_ascii); 128 | pp.query("write_initial_phi", write_initial_phi); 129 | pp.query("write_eb_geom", write_eb_geom); 130 | pp.query("phi_cutoff", phi_cutoff); 131 | pp.query("cfl", cfl); 132 | } 133 | 134 | #ifndef AMREX_USE_HYPRE 135 | if (use_hypre == 1) 136 | amrex::Abort("Cant use hypre if we dont build with USE_HYPRE=TRUE"); 137 | #endif 138 | 139 | if (n_cell%8 != 0) 140 | amrex::Abort("n_cell must be a multiple of 8"); 141 | 142 | int n_cell_x = n_cell; 143 | int n_cell_y = n_cell; 144 | #if (AMREX_SPACEDIM == 3) 145 | int n_cell_z = n_cell/8; 146 | #endif 147 | 148 | Geometry geom; 149 | BoxArray grids; 150 | DistributionMapping dmap; 151 | { 152 | RealBox rb({AMREX_D_DECL(0.,0.,0.)}, {AMREX_D_DECL(1.0,1.0,0.125)}); 153 | 154 | Array is_periodic{AMREX_D_DECL(0,0,1)}; 155 | Geometry::Setup(&rb, 0, is_periodic.data()); 156 | Box domain(IntVect{AMREX_D_DECL(0,0,0)}, 157 | IntVect{AMREX_D_DECL(n_cell_x-1,n_cell_y-1,n_cell_z-1)}); 158 | geom.define(domain); 159 | 160 | grids.define(domain); 161 | grids.maxSize(max_grid_size); 162 | 163 | dmap.define(grids); 164 | } 165 | 166 | const int ncomp_phi = 1; 167 | Vector phi_bc(ncomp_phi); 168 | for (int n = 0; n < ncomp_phi; ++n) 169 | { 170 | for (int i = 0; i < AMREX_SPACEDIM; ++i) 171 | { 172 | // is_periodic overrides inputs in domain_(lo/hi)_bc_type 173 | if (geom.isPeriodic(i)) 174 | { 175 | // Set the BCs to interior Dirichlet if we are periodic 176 | // in this dimension: 177 | phi_bc[n].setLo(i, BCType::int_dir); 178 | phi_bc[n].setHi(i, BCType::int_dir); 179 | } 180 | else 181 | { 182 | // Use first order extrapolation to enforce Neumann BCs with 0 gradient 183 | // at the boundaries if we are not periodic in this dimension: 184 | phi_bc[n].setLo(i, BCType::foextrap); 185 | phi_bc[n].setHi(i, BCType::foextrap); 186 | } 187 | } 188 | } 189 | 190 | Array vel; 191 | MultiFab plotfile_mf; 192 | MultiFab phi_mf; 193 | 194 | int required_coarsening_level = 0; // typically the same as the max AMR level index 195 | int max_coarsening_level = 100; // typically a huge number so MG coarsens as much as possible 196 | 197 | eb_strt_time = amrex::second(); 198 | make_eb_cylinder(geom); 199 | eb_stop_time = amrex::second() - eb_strt_time; 200 | 201 | std::unique_ptr > factory = 202 | makeEBFabFactory(geom, grids, dmap, {4, 4, 2}, EBSupport::full); 203 | const EBFArrayBoxFactory* ebfact = &(static_cast(*factory)); 204 | 205 | // Velocities are face-centered 206 | for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { 207 | vel[idim].define (amrex::convert(grids,IntVect::TheDimensionVector(idim)), dmap, 1, 1, MFInfo(), *factory); 208 | } 209 | 210 | // store plotfile variables; velocity, processor id, and phi (the EB writer appends volfrac) 211 | plotfile_mf.define(grids, dmap, AMREX_SPACEDIM+2, 0, MFInfo(), *factory); 212 | 213 | // make a separate phi MultiFab for the particle-mesh operations because we need a ghost cell 214 | phi_mf.define(grids, dmap, 1, 1, MFInfo(), *factory); 215 | 216 | // Get volume fraction for the embedded geometry 217 | // volume fraction = 0 for cells covered by the embedded geometry 218 | // volume fraction = 1 for cells not covered by the embedded geometry 219 | // 0 < volume fraction < 1 for cells cut by the embedded geometry 220 | const MultiFab& vol_mf = ebfact->getVolFrac(); 221 | 222 | // Initialize a gaussian density profile in the domain for cells 223 | // not covered by the embedded geometry by multiplying phi 224 | // by the volume fraction. 225 | for (MFIter mfi(phi_mf); mfi.isValid(); ++mfi) 226 | { 227 | const Box& bx = mfi.validbox(); 228 | Array4 phi = phi_mf[mfi].array(); 229 | Array4 vol = vol_mf[mfi].array(); 230 | const auto plo = geom.ProbLoArray(); 231 | const auto dx = geom.CellSizeArray(); 232 | 233 | amrex::ParallelFor(bx, 234 | [=] AMREX_GPU_DEVICE (int i, int j, int k) 235 | { 236 | Real x = plo[0] + (0.5+i) * dx[0]; 237 | Real y = plo[1] + (0.5+j) * dx[1]; 238 | 239 | Real r2 = (pow(x-0.5, 2) + pow(y-0.75,2)) / 0.01; 240 | 241 | Real gauss = std::exp(-r2); 242 | gauss = gauss >= phi_cutoff ? gauss : 0.0; 243 | 244 | phi(i,j,k) = gauss * vol(i,j,k); 245 | }); 246 | } 247 | 248 | // Fill periodic BCs and interior ghost cells 249 | phi_mf.FillBoundary(geom.periodicity()); 250 | 251 | // Fill non-periodic domain BCs (e.g. Neumann in x & y dimensions) 252 | FillDomainBoundary(phi_mf, geom, phi_bc); 253 | 254 | // Set cells covered by the embedded boundary to phi = 0 so 255 | // when we interpolate to particles, the covered cells do not contribute. 256 | EB_set_covered(phi_mf,0.0); 257 | 258 | if (write_initial_phi) { 259 | const std::string pfname = "initial_phi"; 260 | WriteSingleLevelPlotfile(pfname, phi_mf, {"phi"}, geom, 0.0, 0); 261 | } 262 | 263 | // Initialize Particles 264 | FluidParticleContainer FPC(geom, dmap, grids); 265 | 266 | // Initialize n_ppc randomly located particles per cell. 267 | // Particles are weighted by interpolated density field phi. 268 | // Only creates particles in regions not covered by the embedded geometry. 269 | FPC.InitParticles(phi_mf, vol_mf, phi_cutoff, n_ppc, pic_interpolation); 270 | 271 | FPC.DepositToMesh(phi_mf, pic_interpolation); 272 | EB_set_covered(phi_mf,-1.0); 273 | 274 | if (write_initial_phi) { 275 | const std::string pfname = "initial_phi_after_deposit"; 276 | WriteSingleLevelPlotfile(pfname, phi_mf, {"phi"}, geom, 0.0, 0); 277 | } 278 | 279 | // set initial velocity to u=(1,0,0) 280 | AMREX_D_TERM(vel[0].setVal(1.0);, 281 | vel[1].setVal(0.0);, 282 | vel[2].setVal(0.0);); 283 | 284 | #if (AMREX_SPACEDIM == 3) 285 | if (write_eb_geom) 286 | { 287 | amrex::Print() << "Writing EB surface" << std::endl; 288 | WriteEBSurface (grids, dmap, geom, ebfact); 289 | } 290 | #endif 291 | 292 | Real time = 0.0; 293 | 294 | // Write out the initial data 295 | { 296 | amrex::Print() << "Creating the initial velocity field " << std::endl; 297 | define_velocity(time,geom,vel,phi_mf); 298 | mac_project_velocity(vel,geom,use_hypre); 299 | EB_average_face_to_cellcenter(plotfile_mf,0,amrex::GetArrOfConstPtrs(vel)); 300 | 301 | // copy initial deposited phi into the plotfile 302 | MultiFab::Copy(plotfile_mf, phi_mf, 0, Idx::phi, 1, 0); 303 | 304 | amrex::Print() << "Writing the initial data into plt00000\n" << std::endl; 305 | write_plotfile(0, time, geom, plotfile_mf, FPC, write_ascii); 306 | } 307 | 308 | // This computes the first dt 309 | dt = est_time_step(dt, geom, vel, cfl); 310 | 311 | int nstep = 0; 312 | 313 | // Sum phi to check conservation 314 | // Real sum_phi = phi_mf.sum(); 315 | Real sum_phi = FPC.SumPhi(); 316 | 317 | amrex::Print() << "Initial sum of phi is " << sum_phi << std::endl; 318 | 319 | for (int i = 0; i < max_step; i++) 320 | { 321 | if (time < stop_time) 322 | { 323 | amrex::Print() << "STEP " << i+1 << " starts at TIME = " << time 324 | << " DT = " << dt << std::endl; 325 | 326 | dt = amrex::min(dt, stop_time - time); 327 | 328 | Real t_nph = time + 0.5 * dt; 329 | 330 | define_velocity(t_nph,geom,vel,phi_mf); 331 | mac_project_velocity(vel,geom,use_hypre); 332 | 333 | for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { 334 | vel[idim].FillBoundary(geom.periodicity()); 335 | } 336 | 337 | // Step Particles 338 | FPC.AdvectWithUmac(vel.data(), 0, dt); 339 | 340 | // Redistribute Particles across MPI ranks with their new positions 341 | FPC.Redistribute(); 342 | 343 | // Deposit Particles to the grid to update phi 344 | FPC.DepositToMesh(phi_mf, pic_interpolation); 345 | EB_set_covered(phi_mf,-1.0); 346 | 347 | // Increment time 348 | time += dt; 349 | nstep++; 350 | 351 | // Write to a plotfile 352 | if (plot_int > 0 && (i+1)%plot_int == 0) 353 | { 354 | average_face_to_cellcenter(plotfile_mf,0,amrex::GetArrOfConstPtrs(vel)); 355 | 356 | // copy phi into the plotfile 357 | MultiFab::Copy(plotfile_mf, phi_mf, 0, Idx::phi, 1, 0); 358 | 359 | write_plotfile(i+1, time, geom, plotfile_mf, FPC, write_ascii); 360 | } 361 | 362 | // Sum phi to check conservation 363 | // sum_phi = phi_mf.sum(); 364 | sum_phi = FPC.SumPhi(); 365 | 366 | amrex::Print() << "STEP " << i+1 << " ends at TIME = " << time 367 | << " DT = " << dt << " Sum(Phi) = " << sum_phi << "\n" << std::endl; 368 | 369 | // Compute lagged dt for next time step based on this half-time velocity 370 | dt = est_time_step(dt, geom, vel, cfl); 371 | 372 | } else { 373 | 374 | // Copy velocity into plotfile 375 | average_face_to_cellcenter(plotfile_mf,0,amrex::GetArrOfConstPtrs(vel)); 376 | 377 | // Write to a plotfile 378 | write_plotfile(i+1, time, geom, plotfile_mf, FPC, write_ascii); 379 | break; 380 | } 381 | } 382 | } 383 | 384 | Real stop_time = amrex::second() - strt_time; 385 | amrex::Print() << "\nTime to create EB geometry " << eb_stop_time << std::endl; 386 | amrex::Print() << "Total run time " << stop_time << std::endl; 387 | 388 | amrex::Finalize(); 389 | } 390 | -------------------------------------------------------------------------------- /08_Pachinko/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | if (AMReX_SPACEDIM EQUAL 1) 2 | return() 3 | endif () 4 | 5 | # List of source files 6 | set(_sources main.cpp MyParticleContainer.cpp MyParticleContainer.H) 7 | 8 | # List of input files 9 | set(_input_files inputs_3d initial_particles_3d paraview_pachinko.py Visualization.ipynb) 10 | 11 | setup_tutorial(_sources _input_files) 12 | 13 | unset( _sources ) 14 | unset( _input_files ) 15 | -------------------------------------------------------------------------------- /08_Pachinko/MyParticleContainer.H: -------------------------------------------------------------------------------- 1 | #ifndef BL_MYPARTICLES_H_ 2 | #define BL_MYPARTICLES_H_ 3 | 4 | #include 5 | 6 | namespace amrex { 7 | 8 | struct PIdx 9 | { 10 | enum { 11 | vx = 0, vy, 12 | #if (AMREX_SPACEDIM == 3) 13 | vz, 14 | #endif 15 | ncomps 16 | }; 17 | }; 18 | 19 | class MyParticleContainer 20 | : public amrex::ParticleContainer 21 | { 22 | 23 | public: 24 | 25 | MyParticleContainer (const amrex::Geometry & a_geom, 26 | const amrex::DistributionMapping & a_dmap, 27 | const amrex::BoxArray & a_ba) 28 | : ParticleContainer(a_geom, a_dmap, a_ba) 29 | {} 30 | 31 | ~MyParticleContainer () {}; 32 | 33 | void InitPachinko (std::string initial_particle_file, Real zlen); 34 | 35 | void AdvectPachinko (Real dt, amrex::Vector& obstacle_center, 36 | Real obstacle_radius, Real particle_radius); 37 | }; 38 | 39 | } 40 | 41 | #endif 42 | -------------------------------------------------------------------------------- /08_Pachinko/MyParticleContainer.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include "MyParticleContainer.H" 3 | 4 | namespace amrex { 5 | 6 | void 7 | MyParticleContainer::InitPachinko (std::string initial_particle_file, Real zlen) 8 | { 9 | BL_PROFILE("MyParticleContainer::InitPachinko()"); 10 | 11 | InitFromAsciiFile(initial_particle_file,0); 12 | 13 | int lev = 0; 14 | auto& pmap = GetParticles(lev); 15 | for (auto& kv : pmap) { 16 | int grid = kv.first.first; 17 | auto& pbox = kv.second.GetArrayOfStructs(); 18 | const int n = pbox.size(); 19 | 20 | for (int i = 0; i < n; i++) 21 | { 22 | ParticleType& p = pbox[i]; 23 | 24 | p.rdata(PIdx::vx) = 0.0; 25 | p.rdata(PIdx::vy) = -1.0; 26 | #if (AMREX_SPACEDIM == 3) 27 | p.rdata(PIdx::vz) = 0.0; 28 | #endif 29 | 30 | // We over-write the z-locations to make sure they're in the domain 31 | p.pos(2) = 0.5 * zlen; 32 | } 33 | } 34 | } 35 | 36 | void 37 | MyParticleContainer::AdvectPachinko (Real dt, amrex::Vector& obstacle_center, 38 | Real obstacle_radius, Real particle_radius) 39 | { 40 | BL_PROFILE("MyParticleContainer::AdvectPachinko()"); 41 | 42 | int lev = 0; 43 | 44 | // Particle radius 45 | Real prad = particle_radius; 46 | 47 | // Obstracle radiu 48 | Real crad = obstacle_radius; 49 | 50 | Real rad_squared = (prad+crad)*(prad+crad); 51 | 52 | Real grav = -40.; 53 | 54 | Real restitution_coeff = 0.9; 55 | 56 | const auto prob_lo = Geom(0).ProbLoArray(); 57 | const auto prob_hi = Geom(0).ProbHiArray(); 58 | 59 | int num_obstacles = obstacle_center.size(); 60 | 61 | auto& pmap = GetParticles(lev); 62 | for (auto& kv : pmap) { 63 | int grid = kv.first.first; 64 | auto& pbox = kv.second.GetArrayOfStructs(); 65 | const int n = pbox.size(); 66 | 67 | // std::cout << "Number of particles: " << n << std::endl; 68 | 69 | // NOTE: we assume that all particle motion occurs in a plane! 70 | // Even if we run in 3-d we assume no motion in the z-direction 71 | 72 | for (int i = 0; i < n; i++) 73 | { 74 | ParticleType& p = pbox[i]; 75 | 76 | // Let particles stop at the bottom 77 | if (p.pos(1) >= prob_lo[1] + 0.05) 78 | { 79 | p.pos(0) += 0.5 * dt * p.rdata(PIdx::vx); 80 | p.pos(1) += 0.5 * dt * p.rdata(PIdx::vy); 81 | 82 | // Accleration under graivty 83 | p.rdata(PIdx::vy) += dt * grav; 84 | 85 | p.pos(0) += 0.5 * dt * p.rdata(PIdx::vx); 86 | p.pos(1) += 0.5 * dt * p.rdata(PIdx::vy); 87 | 88 | p.pos(0) += dt * p.rdata(PIdx::vx); 89 | p.pos(1) += dt * p.rdata(PIdx::vy); 90 | 91 | for (int ind = 0; ind < num_obstacles; ind++) 92 | { 93 | Real x_diff = p.pos(0) - obstacle_center[ind][0]; 94 | Real y_diff = p.pos(1) - obstacle_center[ind][1]; 95 | Real diff_sq = x_diff * x_diff + y_diff * y_diff; 96 | 97 | if ( diff_sq < rad_squared ) 98 | { 99 | Real diff = std::sqrt(diff_sq); 100 | Real overshoot = (prad+crad) - diff; 101 | 102 | Real norm_x = x_diff / diff; 103 | Real norm_y = y_diff / diff; 104 | 105 | Real tang_x = -norm_y; 106 | Real tang_y = norm_x; 107 | 108 | // Incoming velocity dot normal = (norm_x, norm_y) 109 | Real vel_norm = p.rdata(PIdx::vx) * norm_x + 110 | p.rdata(PIdx::vy) * norm_y; 111 | 112 | // Incoming velocity dot tangent = (x_tang, y_tang) = (-y_norm, x_norm) 113 | Real vel_tang = p.rdata(PIdx::vx) * norm_y + 114 | p.rdata(PIdx::vy) * norm_x; 115 | 116 | // Original velocity was (vel_norm) * (norm_x, norm_y) 117 | // + (vel_tang) * (tang_x, tang_y) 118 | 119 | // New velocity is MINUS (vel_norm) * (norm_x, norm_y) 120 | // + (vel_tang) * (tang_x, tang_y) 121 | 122 | p.rdata(PIdx::vx) = -vel_norm * norm_x + vel_tang * tang_x; 123 | p.rdata(PIdx::vy) = -vel_norm * norm_y + vel_tang * tang_y; 124 | 125 | p.rdata(PIdx::vx) *= restitution_coeff; 126 | p.rdata(PIdx::vy) *= restitution_coeff; 127 | 128 | // Reflect particle position as well 129 | Real ref_pos_x = obstacle_center[ind][0] + (prad+crad)*norm_x; 130 | Real ref_pos_y = obstacle_center[ind][1] + (prad+crad)*norm_y; 131 | 132 | p.pos(0) = ref_pos_x + overshoot * norm_x; 133 | p.pos(1) = ref_pos_y + overshoot * norm_y; 134 | } 135 | } 136 | 137 | // Bounce off left wall 138 | if (p.pos(0) < (prob_lo[0]+prad)) 139 | { 140 | p.pos(0) = 2.0*(prob_lo[0]+prad) - p.pos(0); 141 | p.rdata(PIdx::vx) = -p.rdata(PIdx::vx); 142 | p.rdata(PIdx::vx) *= restitution_coeff; 143 | p.rdata(PIdx::vy) *= restitution_coeff; 144 | } 145 | 146 | // Bounce off right wall 147 | if (p.pos(0) > (prob_hi[0]-prad)) 148 | { 149 | p.pos(0) = 2.0*(prob_hi[0]-prad) - p.pos(0); 150 | p.rdata(PIdx::vx) = -0.9*p.rdata(PIdx::vx); 151 | p.rdata(PIdx::vx) *= restitution_coeff; 152 | p.rdata(PIdx::vy) *= restitution_coeff; 153 | } 154 | } 155 | } 156 | } 157 | } 158 | } 159 | -------------------------------------------------------------------------------- /08_Pachinko/Visualization.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "600be7ab", 6 | "metadata": {}, 7 | "source": [ 8 | "## Example: AMReX-Pachinko\n", 9 | "\n", 10 | "### What Features Are We Using\n", 11 | "\n", 12 | "* EB for obstacles \n", 13 | "* Particle-obstacle and particle-wall collisions \n", 14 | "\n", 15 | "### The Problem\n", 16 | "\n", 17 | "Have you ever played pachinko? \n", 18 | "\n", 19 | "A pachinko machine is like a vertical pinball machine. \n", 20 | "\n", 21 | "Balls are released at the top of the \"playing field\", and bounce off obstacles as they fall.\n", 22 | "\n", 23 | "The object of the game is to \"capture\" as many balls as possible.\n", 24 | "\n", 25 | "In the AMReX-Pachinko game you can release as many particles as you like at the top of the domain,\n", 26 | "and the balls will freeze when they hit the bottom so you can see where they landed.\n", 27 | "\n", 28 | "Your goal here is to see if you can cover the floor of the pachinko machine.\n", 29 | "\n", 30 | "(Note that this is not completely realistic -- the balls here don't feel each other so they can overlap.)\n", 31 | "\n", 32 | "In this example there is no fluid (or other variable) stored on the mesh\n", 33 | "but we still sort the particles according to our spatial decomposition of the domain.\n", 34 | "If we run in parallel with 4 processors, we see the domain decomposition below -- this results\n", 35 | "from using a z-order space-filling curve with the number of cells per grid as the cost function.\n", 36 | "\n", 37 | "For now we freeze the obstacles (although if you look in the code it's not hard to figure out\n", 38 | "how to change them!) but we can change the initial particle locations at run-time by editing the\n", 39 | "initial_particles_3d file." 40 | ] 41 | } 42 | ], 43 | "metadata": { 44 | "kernelspec": { 45 | "display_name": "Python 3", 46 | "language": "python", 47 | "name": "python3" 48 | }, 49 | "language_info": { 50 | "codemirror_mode": { 51 | "name": "ipython", 52 | "version": 3 53 | }, 54 | "file_extension": ".py", 55 | "mimetype": "text/x-python", 56 | "name": "python", 57 | "nbconvert_exporter": "python", 58 | "pygments_lexer": "ipython3", 59 | "version": "3.8.8" 60 | } 61 | }, 62 | "nbformat": 4, 63 | "nbformat_minor": 5 64 | } 65 | -------------------------------------------------------------------------------- /08_Pachinko/initial_particles_3d: -------------------------------------------------------------------------------- 1 | 12 2 | 0.055 1.9 0.5 3 | 0.155 1.9 0.5 4 | 0.255 1.9 0.5 5 | 0.355 1.9 0.5 6 | 0.455 1.9 0.5 7 | 0.555 1.9 0.5 8 | 0.655 1.9 0.5 9 | 0.755 1.9 0.5 10 | 0.855 1.9 0.5 11 | 0.955 1.9 0.5 12 | 1.055 1.9 0.5 13 | 1.155 1.9 0.5 14 | -------------------------------------------------------------------------------- /08_Pachinko/inputs_3d: -------------------------------------------------------------------------------- 1 | n_cell = 125 # number of cells in x-direction; we double this in the y-direction 2 | max_grid_size = 25 # the maximum number of cells in any direction in a single grid 3 | 4 | plot_int = 15 # frequency of writing plotfiles 5 | 6 | particle_file = initial_particles_3d # name of file where we specify the initial positions of the particles 7 | 8 | time_step = 0.001 # we take a fixed time step of this size 9 | 10 | max_time = 1.75 # the final time (if max_time < max_steps * time_step) 11 | max_steps = 100000 # the maximum number of steps (if max_steps * time_step < max_time)) 12 | -------------------------------------------------------------------------------- /08_Pachinko/main.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #include "MyParticleContainer.H" 14 | 15 | using namespace amrex; 16 | 17 | void write_plotfile(int step_counter, const Geometry& geom, const MultiFab& plotmf, MyParticleContainer& pc, 18 | const int ascii_particle_output) 19 | { 20 | std::stringstream sstream; 21 | sstream << "plt" << std::setw(5) << std::setfill('0') << step_counter; 22 | std::string plotfile_name = sstream.str(); 23 | 24 | if (step_counter == 0) 25 | { 26 | EB_WriteSingleLevelPlotfile(plotfile_name, plotmf, 27 | { "proc" }, geom, 0.0, 0); 28 | } 29 | 30 | if (ascii_particle_output) 31 | { 32 | sstream << "_particles"; 33 | const std::string particlefile_name = sstream.str(); 34 | pc.WriteAsciiFile(particlefile_name); 35 | } 36 | 37 | pc.Checkpoint(plotfile_name, "particles", true); // Write particles to plotfile 38 | } 39 | 40 | int main (int argc, char* argv[]) 41 | { 42 | amrex::SetVerbose(0); 43 | 44 | amrex::Initialize(argc, argv); 45 | 46 | // Turn off amrex-related output 47 | 48 | { 49 | int verbose = 0; 50 | int n_cell = 128; 51 | int max_grid_size = 32; 52 | std::string particle_file = ""; 53 | Real max_time = 1.0; 54 | int max_steps = 100; 55 | int plot_int = 1; 56 | Real time_step = 0.01; 57 | int ascii_particle_output = 0; 58 | 59 | Real obstacle_radius = 0.10; 60 | Real particle_radius = 0.02; 61 | 62 | // read parameters 63 | { 64 | ParmParse pp; 65 | pp.query("verbose", verbose); 66 | pp.query("n_cell", n_cell); 67 | pp.query("max_grid_size", max_grid_size); 68 | pp.query("particle_file", particle_file); 69 | pp.query("max_time", max_time); 70 | pp.query("max_steps", max_steps); 71 | pp.query("plot_int", plot_int); 72 | pp.query("time_step", time_step); 73 | pp.query("ascii_particle_output", ascii_particle_output); 74 | 75 | pp.query("obstacle_radius", obstacle_radius); 76 | pp.query("particle_radius", particle_radius); 77 | } 78 | 79 | int n_cell_x = n_cell; 80 | int n_cell_y = n_cell * 8/5; 81 | int n_cell_z = 4; 82 | 83 | Real zlen = 0.5; 84 | 85 | Geometry geom; 86 | BoxArray grids; 87 | DistributionMapping dmap; 88 | { 89 | RealBox rb({AMREX_D_DECL(0.,0.,0.)}, {AMREX_D_DECL(1.25,2.,zlen)}); 90 | Array isp{AMREX_D_DECL(0,1,1)}; 91 | Geometry::Setup(&rb, 0, isp.data()); 92 | Box domain(IntVect{AMREX_D_DECL(0,0,0)}, 93 | IntVect{AMREX_D_DECL(n_cell_x-1,n_cell_y-1,n_cell_z-1)}); 94 | geom.define(domain); 95 | 96 | grids.define(domain); 97 | grids.maxSize(max_grid_size); 98 | 99 | dmap.define(grids); 100 | } 101 | 102 | MultiFab plotfile_mf; 103 | 104 | amrex::Vector obstacle_center = { 105 | {AMREX_D_DECL(0.30,0.3,0.5*zlen)}, 106 | {AMREX_D_DECL(0.60,0.3,0.5*zlen)}, 107 | {AMREX_D_DECL(0.90,0.3,0.5*zlen)}, 108 | {AMREX_D_DECL(0.15,0.7,0.5*zlen)}, 109 | {AMREX_D_DECL(0.45,0.7,0.5*zlen)}, 110 | {AMREX_D_DECL(0.75,0.7,0.5*zlen)}, 111 | {AMREX_D_DECL(1.05,0.7,0.5*zlen)}, 112 | {AMREX_D_DECL(0.30,1.1,0.5*zlen)}, 113 | {AMREX_D_DECL(0.60,1.1,0.5*zlen)}, 114 | {AMREX_D_DECL(0.90,1.1,0.5*zlen)}, 115 | {AMREX_D_DECL(0.15,1.5,0.5*zlen)}, 116 | {AMREX_D_DECL(0.45,1.5,0.5*zlen)}, 117 | {AMREX_D_DECL(0.75,1.5,0.5*zlen)}, 118 | {AMREX_D_DECL(1.05,1.5,0.5*zlen)}}; 119 | 120 | // The "false" below is the boolean that determines if the fluid is inside ("true") or 121 | // outside ("false") the object(s) 122 | int direction = 2; 123 | Real height = -1.0; // Putting a negative number for height means it extends beyond the domain 124 | Array obstacles{ 125 | EB2::CylinderIF(obstacle_radius, height, direction, obstacle_center[ 0], false), 126 | EB2::CylinderIF(obstacle_radius, height, direction, obstacle_center[ 1], false), 127 | EB2::CylinderIF(obstacle_radius, height, direction, obstacle_center[ 2], false), 128 | EB2::CylinderIF(obstacle_radius, height, direction, obstacle_center[ 3], false), 129 | EB2::CylinderIF(obstacle_radius, height, direction, obstacle_center[ 4], false), 130 | EB2::CylinderIF(obstacle_radius, height, direction, obstacle_center[ 5], false), 131 | EB2::CylinderIF(obstacle_radius, height, direction, obstacle_center[ 6], false), 132 | EB2::CylinderIF(obstacle_radius, height, direction, obstacle_center[ 7], false), 133 | EB2::CylinderIF(obstacle_radius, height, direction, obstacle_center[ 8], false), 134 | EB2::CylinderIF(obstacle_radius, height, direction, obstacle_center[ 9], false), 135 | EB2::CylinderIF(obstacle_radius, height, direction, obstacle_center[10], false), 136 | EB2::CylinderIF(obstacle_radius, height, direction, obstacle_center[11], false), 137 | EB2::CylinderIF(obstacle_radius, height, direction, obstacle_center[12], false), 138 | EB2::CylinderIF(obstacle_radius, height, direction, obstacle_center[13], false)}; 139 | 140 | auto group_1 = EB2::makeUnion(obstacles[0],obstacles[1],obstacles[2]); 141 | auto group_2 = EB2::makeUnion(obstacles[3],obstacles[4],obstacles[5]); 142 | auto group_3 = EB2::makeUnion(obstacles[6],obstacles[7],obstacles[8]); 143 | auto group_4 = EB2::makeUnion(obstacles[9],obstacles[10],obstacles[11]); 144 | auto group_5 = EB2::makeUnion(obstacles[12],obstacles[13]); 145 | auto all = EB2::makeUnion(group_1,group_2,group_3,group_4,group_5); 146 | auto gshop9 = EB2::makeShop(all); 147 | 148 | int required_coarsening_level = 0; // typically the same as the max AMR level index 149 | int max_coarsening_level = 0; // typically a huge number so MG coarsens as much as possible 150 | EB2::Build(gshop9, geom, required_coarsening_level, max_coarsening_level); 151 | 152 | const EB2::IndexSpace& eb_is = EB2::IndexSpace::top(); 153 | const EB2::Level& eb_level = eb_is.getLevel(geom); 154 | 155 | // options are basic, volume, or full 156 | EBSupport ebs = EBSupport::full; 157 | 158 | // number of ghost cells for each of the 3 EBSupport types 159 | Vector ng_ebs = {2,2,2}; 160 | 161 | // This object provides access to the EB database in the format of basic AMReX objects 162 | // such as BaseFab, FArrayBox, FabArray, and MultiFab 163 | EBFArrayBoxFactory factory(eb_level, geom, grids, dmap, ng_ebs, ebs); 164 | 165 | // Initialize Particles 166 | MyParticleContainer MyPC(geom, dmap, grids); 167 | MyPC.InitPachinko(particle_file,zlen); 168 | 169 | // Store processor id in the plotfile 170 | plotfile_mf.define(grids, dmap, 1, 0, MFInfo(), factory); 171 | 172 | amrex::Print() << " \n********************************************************************" << std::endl; 173 | amrex::Print() << " Let's advect the particles ... " << std::endl; 174 | amrex::Print() << " We'll print a dot every 10 time steps." << std::endl; 175 | amrex::Print() << "******************************************************************** \n" << std::endl; 176 | 177 | // copy processor id into plotfile_mf 178 | int lev = 0; 179 | for (MFIter mfi = MyPC.MakeMFIter(lev); mfi.isValid(); ++mfi) 180 | plotfile_mf[mfi].setVal(ParallelDescriptor::MyProc()); 181 | 182 | // wallclock time 183 | Real strt_total = amrex::second(); 184 | 185 | #ifdef _OPENMP 186 | #pragma omp parallel if (Gpu::notInLaunchRegion()) 187 | #endif 188 | 189 | Real time = 0.0; 190 | for (int i = 0; i < max_steps; i++) 191 | { 192 | if (time < max_time) { 193 | time_step = std::min(time_step, max_time - time); 194 | 195 | // Step Particles 196 | MyPC.AdvectPachinko(time_step,obstacle_center,obstacle_radius,particle_radius); 197 | 198 | MyPC.Redistribute(); 199 | 200 | // Write to a plotfile 201 | if (i%plot_int == 0) 202 | write_plotfile(i, geom, plotfile_mf, MyPC, ascii_particle_output); 203 | 204 | if (i%10 == 0) 205 | amrex::Print() << "."; 206 | 207 | // Increment time 208 | time += time_step; 209 | 210 | } else 211 | break; 212 | } 213 | 214 | amrex::Print() << "\n" << std::endl; 215 | amrex::Print() << "********************************************************************" << std::endl; 216 | amrex::Print() << "We've finished moving the particles to time " << time << std::endl; 217 | 218 | // wallclock time 219 | Real end_total = amrex::second() - strt_total; 220 | 221 | amrex::Print() << "That took " << end_total << " seconds." << std::endl; 222 | amrex::Print() << "******************************************************************** \n" << std::endl; 223 | 224 | Print() << "Writing EB surface" << std::endl; 225 | WriteEBSurface (grids, dmap, geom, &factory); 226 | } 227 | 228 | amrex::Finalize(); 229 | } 230 | -------------------------------------------------------------------------------- /08_Pachinko/paraview_pachinko.py: -------------------------------------------------------------------------------- 1 | # trace generated using paraview version 5.8.0 2 | # 3 | # To ensure correct image size when batch processing, please search 4 | # for and uncomment the line `# renderView*.ViewSize = [*,*]` 5 | 6 | #### import the simple module from the paraview 7 | from paraview.simple import * 8 | 9 | import subprocess 10 | import glob 11 | import argparse 12 | 13 | parser = argparse.ArgumentParser() 14 | parser.add_argument('-f', '--frame_rate', type=int, default=15, help="Frame rate for generating movies, i.e. number of plots per second in the movie.") 15 | parser.add_argument('-r', '--resolution', type=int, default=1024, help="(Square) resolution of output movie.") 16 | args = parser.parse_args() 17 | 18 | def generate_movie_3D(AllPlotFiles): 19 | #### disable automatic camera reset on 'Show' 20 | paraview.simple._DisableFirstRenderCameraReset() 21 | 22 | # create a new 'XML Partitioned Polydata Reader' 23 | ebpvtp = XMLPartitionedPolydataReader(FileName=['eb.pvtp']) 24 | 25 | # get active view 26 | renderView1 = GetActiveViewOrCreate('RenderView') 27 | # uncomment following to set a specific view size 28 | renderView1.ViewSize = [1200, 1200] 29 | 30 | # get layout 31 | layout1 = GetLayout() 32 | 33 | # show data in view 34 | ebpvtpDisplay = Show(ebpvtp, renderView1) 35 | 36 | # trace defaults for the display properties. 37 | ebpvtpDisplay.Representation = 'Surface' 38 | ebpvtpDisplay.ColorArrayName = [None, ''] 39 | ebpvtpDisplay.OSPRayScaleFunction = 'PiecewiseFunction' 40 | ebpvtpDisplay.SelectOrientationVectors = 'None' 41 | ebpvtpDisplay.ScaleFactor = 0.14000000208616256 42 | ebpvtpDisplay.SelectScaleArray = 'None' 43 | ebpvtpDisplay.GlyphType = 'Arrow' 44 | ebpvtpDisplay.GlyphTableIndexArray = 'None' 45 | ebpvtpDisplay.GaussianRadius = 0.007000000104308128 46 | ebpvtpDisplay.SetScaleArray = [None, ''] 47 | ebpvtpDisplay.ScaleTransferFunction = 'PiecewiseFunction' 48 | ebpvtpDisplay.OpacityArray = [None, ''] 49 | ebpvtpDisplay.OpacityTransferFunction = 'PiecewiseFunction' 50 | ebpvtpDisplay.DataAxesGrid = 'GridAxesRepresentation' 51 | ebpvtpDisplay.PolarAxes = 'PolarAxesRepresentation' 52 | 53 | # reset view to fit data 54 | renderView1.ResetCamera() 55 | 56 | # get the material library 57 | materialLibrary1 = GetMaterialLibrary() 58 | 59 | # update the view to ensure updated data information 60 | renderView1.Update() 61 | 62 | # create a new 'AMReX/BoxLib Grid Reader' 63 | plt00000 = AMReXBoxLibGridReader(FileNames=['plt00000']) 64 | plt00000.CellArrayStatus = [] 65 | 66 | # Properties modified on plt00000 67 | plt00000.CellArrayStatus = ['proc', 'vfrac'] 68 | 69 | # show data in view 70 | plt00000Display = Show(plt00000, renderView1) 71 | 72 | # trace defaults for the display properties. 73 | plt00000Display.Representation = 'Outline' 74 | plt00000Display.ColorArrayName = [None, ''] 75 | plt00000Display.OSPRayScaleFunction = 'PiecewiseFunction' 76 | plt00000Display.SelectOrientationVectors = 'None' 77 | plt00000Display.ScaleFactor = 0.2 78 | plt00000Display.SelectScaleArray = 'None' 79 | plt00000Display.GlyphType = 'Arrow' 80 | plt00000Display.GlyphTableIndexArray = 'None' 81 | plt00000Display.GaussianRadius = 0.01 82 | plt00000Display.SetScaleArray = [None, ''] 83 | plt00000Display.ScaleTransferFunction = 'PiecewiseFunction' 84 | plt00000Display.OpacityArray = [None, ''] 85 | plt00000Display.OpacityTransferFunction = 'PiecewiseFunction' 86 | plt00000Display.DataAxesGrid = 'GridAxesRepresentation' 87 | plt00000Display.PolarAxes = 'PolarAxesRepresentation' 88 | plt00000Display.ScalarOpacityUnitDistance = 0.051941539345088994 89 | 90 | # update the view to ensure updated data information 91 | renderView1.Update() 92 | 93 | # create a new 'Slice' 94 | slice1 = Slice(Input=plt00000) 95 | slice1.SliceType = 'Plane' 96 | #slice1.HyperTreeGridSlicer = 'Plane' 97 | slice1.SliceOffsetValues = [0.0] 98 | 99 | # init the 'Plane' selected for 'SliceType' 100 | slice1.SliceType.Origin = [0.625, 1.0, 0.25] 101 | 102 | # init the 'Plane' selected for 'HyperTreeGridSlicer' 103 | # slice1.HyperTreeGridSlicer.Origin = [0.625, 1.0, 0.25] 104 | 105 | # toggle 3D widget visibility (only when running from the GUI) 106 | Hide3DWidgets(proxy=slice1.SliceType) 107 | 108 | # Properties modified on slice1.SliceType 109 | slice1.SliceType.Normal = [0.0, 0.0, 1.0] 110 | 111 | # show data in view 112 | slice1Display = Show(slice1, renderView1) 113 | 114 | # trace defaults for the display properties. 115 | slice1Display.Representation = 'Surface' 116 | slice1Display.ColorArrayName = [None, ''] 117 | slice1Display.OSPRayScaleFunction = 'PiecewiseFunction' 118 | slice1Display.SelectOrientationVectors = 'None' 119 | slice1Display.ScaleFactor = 0.2 120 | slice1Display.SelectScaleArray = 'None' 121 | slice1Display.GlyphType = 'Arrow' 122 | slice1Display.GlyphTableIndexArray = 'None' 123 | slice1Display.GaussianRadius = 0.01 124 | slice1Display.SetScaleArray = [None, ''] 125 | slice1Display.ScaleTransferFunction = 'PiecewiseFunction' 126 | slice1Display.OpacityArray = [None, ''] 127 | slice1Display.OpacityTransferFunction = 'PiecewiseFunction' 128 | slice1Display.DataAxesGrid = 'GridAxesRepresentation' 129 | slice1Display.PolarAxes = 'PolarAxesRepresentation' 130 | 131 | # update the view to ensure updated data information 132 | renderView1.Update() 133 | 134 | # set scalar coloring 135 | ColorBy(slice1Display, ('FIELD', 'vtkBlockColors')) 136 | 137 | # get color transfer function/color map for 'vtkBlockColors' 138 | vtkBlockColorsLUT = GetColorTransferFunction('vtkBlockColors') 139 | 140 | # get opacity transfer function/opacity map for 'vtkBlockColors' 141 | vtkBlockColorsPWF = GetOpacityTransferFunction('vtkBlockColors') 142 | 143 | # set scalar coloring 144 | ColorBy(slice1Display, ('CELLS', 'proc')) 145 | 146 | # rescale color and/or opacity maps used to include current data range 147 | slice1Display.RescaleTransferFunctionToDataRange(True, False) 148 | 149 | # get color transfer function/color map for 'proc' 150 | procLUT = GetColorTransferFunction('proc') 151 | 152 | # get opacity transfer function/opacity map for 'proc' 153 | procPWF = GetOpacityTransferFunction('proc') 154 | 155 | # create a new 'AMReX/BoxLib Particles Reader' 156 | plt0 = AMReXBoxLibParticlesReader(FileNames=AllPlotFiles) 157 | plt0.PointArrayStatus = ['id', 'cpu', 'int_comp0', 'real_comp0', 'real_comp1', 'real_comp2'] 158 | 159 | # get animation scene 160 | animationScene1 = GetAnimationScene() 161 | 162 | # get the time-keeper 163 | timeKeeper1 = GetTimeKeeper() 164 | 165 | # update animation scene based on data timesteps 166 | animationScene1.UpdateAnimationUsingDataTimeSteps() 167 | 168 | # show data in view 169 | plt0Display = Show(plt0, renderView1) 170 | 171 | # trace defaults for the display properties. 172 | plt0Display.Representation = 'Surface' 173 | plt0Display.ColorArrayName = [None, ''] 174 | plt0Display.OSPRayScaleArray = 'cpu' 175 | plt0Display.OSPRayScaleFunction = 'PiecewiseFunction' 176 | plt0Display.SelectOrientationVectors = 'None' 177 | plt0Display.ScaleFactor = 0.11000000000000001 178 | plt0Display.SelectScaleArray = 'None' 179 | plt0Display.GlyphType = 'Arrow' 180 | plt0Display.GlyphTableIndexArray = 'None' 181 | plt0Display.GaussianRadius = 0.0055000000000000005 182 | plt0Display.SetScaleArray = ['POINTS', 'cpu'] 183 | plt0Display.ScaleTransferFunction = 'PiecewiseFunction' 184 | plt0Display.OpacityArray = ['POINTS', 'cpu'] 185 | plt0Display.OpacityTransferFunction = 'PiecewiseFunction' 186 | plt0Display.DataAxesGrid = 'GridAxesRepresentation' 187 | plt0Display.PolarAxes = 'PolarAxesRepresentation' 188 | 189 | # init the 'PiecewiseFunction' selected for 'ScaleTransferFunction' 190 | plt0Display.ScaleTransferFunction.Points = [0.0, 0.0, 0.5, 0.0, 3.0, 1.0, 0.5, 0.0] 191 | 192 | # init the 'PiecewiseFunction' selected for 'OpacityTransferFunction' 193 | plt0Display.OpacityTransferFunction.Points = [0.0, 0.0, 0.5, 0.0, 3.0, 1.0, 0.5, 0.0] 194 | 195 | # update the view to ensure updated data information 196 | renderView1.Update() 197 | 198 | # set scalar coloring 199 | ColorBy(plt0Display, ('FIELD', 'vtkBlockColors')) 200 | 201 | # create a new 'Glyph' 202 | glyph1 = Glyph(Input=plt0, 203 | GlyphType='Arrow') 204 | glyph1.OrientationArray = ['POINTS', 'No orientation array'] 205 | glyph1.ScaleArray = ['POINTS', 'No scale array'] 206 | glyph1.ScaleFactor = 0.11000000000000001 207 | glyph1.GlyphTransform = 'Transform2' 208 | 209 | # Properties modified on glyph1 210 | glyph1.GlyphType = 'Sphere' 211 | glyph1.ScaleFactor = 0.05 212 | glyph1.GlyphMode = 'All Points' 213 | 214 | # show data in view 215 | glyph1Display = Show(glyph1, renderView1) 216 | 217 | # trace defaults for the display properties. 218 | glyph1Display.Representation = 'Surface' 219 | glyph1Display.ColorArrayName = [None, ''] 220 | glyph1Display.OSPRayScaleArray = 'Normals' 221 | glyph1Display.OSPRayScaleFunction = 'PiecewiseFunction' 222 | glyph1Display.SelectOrientationVectors = 'None' 223 | glyph1Display.ScaleFactor = 0.11487463433295489 224 | glyph1Display.SelectScaleArray = 'None' 225 | glyph1Display.GlyphType = 'Arrow' 226 | glyph1Display.GlyphTableIndexArray = 'None' 227 | glyph1Display.GaussianRadius = 0.005743731716647744 228 | glyph1Display.SetScaleArray = ['POINTS', 'Normals'] 229 | glyph1Display.ScaleTransferFunction = 'PiecewiseFunction' 230 | glyph1Display.OpacityArray = ['POINTS', 'Normals'] 231 | glyph1Display.OpacityTransferFunction = 'PiecewiseFunction' 232 | glyph1Display.DataAxesGrid = 'GridAxesRepresentation' 233 | glyph1Display.PolarAxes = 'PolarAxesRepresentation' 234 | 235 | # init the 'PiecewiseFunction' selected for 'ScaleTransferFunction' 236 | glyph1Display.ScaleTransferFunction.Points = [-0.9749279022216797, 0.0, 0.5, 0.0, 0.9749279022216797, 1.0, 0.5, 0.0] 237 | 238 | # init the 'PiecewiseFunction' selected for 'OpacityTransferFunction' 239 | glyph1Display.OpacityTransferFunction.Points = [-0.9749279022216797, 0.0, 0.5, 0.0, 0.9749279022216797, 1.0, 0.5, 0.0] 240 | 241 | # update the view to ensure updated data information 242 | renderView1.Update() 243 | 244 | # set scalar coloring 245 | ColorBy(glyph1Display, ('FIELD', 'vtkBlockColors')) 246 | 247 | # current camera placement for renderView1 248 | renderView1.CameraPosition = [0.5999999884516001, 0.9000000134110451, 5.480672965279949] 249 | renderView1.CameraFocalPoint = [0.5999999884516001, 0.9000000134110451, 0.25] 250 | renderView1.CameraParallelScale = 0.9246621010295244 251 | 252 | # save animation 253 | output_movie_base = "pachinko" 254 | output_movie = output_movie_base + ".avi" 255 | SaveAnimation(output_movie, 256 | renderView1, 257 | ImageResolution=[1200, 1200], 258 | FrameRate=args.frame_rate, 259 | FrameWindow=[0, len(AllPlotFiles)-1]) 260 | 261 | return output_movie_base, output_movie 262 | 263 | def convert_avi_to_gif(output_movie_base, output_movie): 264 | # use ffmpeg to convert the avi movie into an animated gif 265 | ffmpeg_convert_to_gif = 'ffmpeg -y -i {} -vf "fps=35,scale={}:-1:flags=lanczos,split[s0][s1];[s0]palettegen[p];[s1][p]paletteuse" -loop 0 {}.gif'.format(output_movie, args.resolution, output_movie_base) 266 | subprocess.run(ffmpeg_convert_to_gif, shell=True) 267 | 268 | if __name__ == "__main__": 269 | if args.frame_rate <= 0: 270 | print("Please specify --frame_rate F (with F > 0)") 271 | exit() 272 | 273 | if args.resolution <= 0: 274 | print("Please specify --resolution R (with R > 0)") 275 | exit() 276 | 277 | # get all the plotfiles 278 | PlotFiles = sorted(glob.glob("plt" + "[0-9]"*5)) 279 | 280 | # generate the avi movie file 281 | output_movie_base, output_movie = generate_movie_3D(PlotFiles) 282 | 283 | # convert the avi movie into an animated gif 284 | convert_avi_to_gif(output_movie_base, output_movie) 285 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.14) 2 | 3 | project( ECP-Tutorials 4 | DESCRIPTION "Tutorials for the AMReX adaptive mesh refinement framework" 5 | #VERSION ${AMREX_PKG_VERSION} 6 | # Check that the line below points to the correct repo 7 | HOMEPAGE_URL "https://amrex-codes.github.io/amrex/tutorials_html/index.html" 8 | ) 9 | 10 | set( CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/cmake ) 11 | 12 | set(AMReX_PARTICLES ON) 13 | set(AMReX_EB ON) 14 | set(AMReX_FORTRAN OFF) 15 | 16 | # 17 | # Fetch amrex repo 18 | # 19 | set(AMReX_GIT_BRANCH "development" CACHE STRING "The AMReX branch to checkout") 20 | set(AMReX_INSTALL "NO" CACHE INTERNAL "Disable install target for amrex") 21 | 22 | include(FetchContent) 23 | set(FETCHCONTENT_QUIET OFF) # Verbose ON 24 | 25 | FetchContent_Declare( 26 | amrex 27 | GIT_REPOSITORY https://github.com/AMReX-Codes/amrex.git/ 28 | GIT_TAG ${AMReX_GIT_BRANCH} 29 | ) 30 | 31 | if(NOT ${amrex}_POPULATED) 32 | FetchContent_Populate(amrex) 33 | 34 | list(APPEND CMAKE_MODULE_PATH ${amrex_SOURCE_DIR}/Tools/CMake) 35 | 36 | # Load amrex options here so that they are 37 | # available to the entire project 38 | include(AMReXOptions) 39 | 40 | if (AMReX_FORTRAN) 41 | enable_language(Fortran) 42 | endif () 43 | 44 | if (AMReX_GPU_BACKEND STREQUAL "CUDA") 45 | enable_language(CUDA) 46 | include(AMReX_SetupCUDA) 47 | endif () 48 | 49 | # Bring the populated content into the build 50 | add_subdirectory(${amrex_SOURCE_DIR} ${amrex_BINARY_DIR}) 51 | endif() 52 | 53 | # 54 | # List of subdirectories to search for CMakeLists. 55 | # 56 | set( AMREX_TUTORIALS_SUBDIRS 01_HelloWorld 02_ParmParse 57 | 03_HeatEquation 04_ParticleMesh 05_ParticleMesh_MultiLevel 58 | 06_Advection_Amr 08_Pachinko) 59 | 60 | list(TRANSFORM AMREX_TUTORIALS_SUBDIRS PREPEND "${CMAKE_CURRENT_LIST_DIR}/") 61 | 62 | # 63 | # Search for CMakelists.txt in the subdirectories defined in the list above 64 | # and include those that contain one into the build 65 | # 66 | include(SetupTutorials) 67 | foreach (_subdir IN LISTS AMREX_TUTORIALS_SUBDIRS) 68 | file( GLOB_RECURSE _tests "${_subdir}/*CMakeLists.txt" ) 69 | foreach ( _item IN LISTS _tests) 70 | get_filename_component(_dir ${_item} DIRECTORY ) 71 | add_subdirectory(${_dir}) 72 | endforeach () 73 | endforeach () 74 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2021, The Regents of the University of California, 2 | through Lawrence Berkeley National Laboratory and the Alliance for 3 | Sustainable Energy, LLC., through National Renewable Energy Laboratory 4 | (subject to receipt of any required approvals from the U.S. Dept. of 5 | Energy). All rights reserved. 6 | 7 | Redistribution and use in source and binary forms, with or without 8 | modification, are permitted provided that the following conditions are 9 | met: 10 | 11 | (1) Redistributions of source code must retain the above copyright 12 | notice, this list of conditions and the following disclaimer. 13 | 14 | (2) Redistributions in binary form must reproduce the above copyright 15 | notice, this list of conditions and the following disclaimer in the 16 | documentation and/or other materials provided with the distribution. 17 | 18 | (3) Neither the name of the University of California, Lawrence 19 | Berkeley National Laboratory, Alliance for Sustainable Energy, LLC., 20 | National Renewable Energy Laboratory, U.S. Dept. of Energy nor the 21 | names of its contributors may be used to endorse or promote products 22 | derived from this software without specific prior written permission. 23 | 24 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 25 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE 28 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 30 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 31 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 32 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 33 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 34 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Open in Gitpod](https://gitpod.io/button/open-in-gitpod.svg)](https://gitpod.io/#https://github.com/atmyers/ecp-tutorials) 2 | 3 | This repository contains tutorials built using the AMReX adaptive 4 | mesh refinement framework, available at: https://github.com/AMReX-Codes/amrex/ 5 | 6 | To work through this material online, click the "Open in Gitpod" button above. 7 | You will land in a pre-built environment where you can explore the tutorials 8 | without having to install any dependencies. 9 | 10 | NOTICE. This Software was developed under funding from the 11 | U.S. Department of Energy and the U.S. Government consequently retains 12 | certain rights. As such, the U.S. Government has been granted for 13 | itself and others acting on its behalf a paid-up, nonexclusive, 14 | irrevocable, worldwide license in the Software to reproduce, 15 | distribute copies to the public, prepare derivative works, and perform 16 | publicly and display publicly, and to permit other to do so. 17 | 18 | The license for these tutorials can be found at [LICENSE](LICENSE). 19 | -------------------------------------------------------------------------------- /cmake/SetupTutorials.cmake: -------------------------------------------------------------------------------- 1 | # 2 | # Function to setup the tutorials 3 | # 4 | function (setup_tutorial _srcs _inputs) 5 | 6 | cmake_parse_arguments( "" "HAS_FORTRAN_MODULES" 7 | "BASE_NAME;RUNTIME_SUBDIR;EXTRA_DEFINITIONS" "" ${ARGN} ) 8 | 9 | if (_BASE_NAME) 10 | set(_base_name ${_BASE_NAME}) 11 | else () 12 | string(REGEX REPLACE ".*ecp-tutorials/" "" _base_name ${CMAKE_CURRENT_LIST_DIR}) 13 | string(REPLACE "/" "_" _base_name ${_base_name}) 14 | endif () 15 | 16 | if (_RUNTIME_SUBDIR) 17 | set(_exe_dir ${CMAKE_CURRENT_BINARY_DIR}/${_RUNTIME_SUBDIR}) 18 | else () 19 | set(_exe_dir ${CMAKE_CURRENT_BINARY_DIR}) 20 | endif () 21 | 22 | set( _exe_name ${_base_name} ) 23 | 24 | add_executable( ${_exe_name} ) 25 | 26 | target_sources( ${_exe_name} PRIVATE ${${_srcs}} ) 27 | 28 | set_target_properties( ${_exe_name} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${_exe_dir} ) 29 | 30 | if (_EXTRA_DEFINITIONS) 31 | target_compile_definitions(${_exe_name} PRIVATE ${_EXTRA_DEFINITIONS}) 32 | endif () 33 | 34 | # Find out which include directory is needed 35 | set(_includes ${${_srcs}}) 36 | list(FILTER _includes INCLUDE REGEX "\\.H$") 37 | foreach(_item IN LISTS _includes) 38 | get_filename_component( _include_dir ${_item} DIRECTORY) 39 | target_include_directories( ${_exe_name} PRIVATE ${_include_dir} ) 40 | endforeach() 41 | 42 | if (_HAS_FORTRAN_MODULES) 43 | target_include_directories(${_exe_name} 44 | PRIVATE 45 | ${CMAKE_CURRENT_BINARY_DIR}/${EXENAME}_mod_files) 46 | set_target_properties( ${_exe_name} 47 | PROPERTIES 48 | Fortran_MODULE_DIRECTORY 49 | ${CMAKE_CURRENT_BINARY_DIR}/${EXENAME}_mod_files ) 50 | endif () 51 | 52 | target_link_libraries( ${_exe_name} amrex ) 53 | 54 | if (AMReX_CUDA) 55 | setup_target_for_cuda_compilation( ${_exe_name} ) 56 | endif () 57 | 58 | if (${_inputs}) 59 | file( COPY ${${_inputs}} DESTINATION ${_exe_dir} ) 60 | endif () 61 | 62 | endfunction () 63 | --------------------------------------------------------------------------------