├── .gitattributes ├── .github └── workflows │ └── main.yml ├── .gitignore ├── CODE_OF_CONDUCT.md ├── LICENSE ├── Make.projects ├── Make.sledgehamr ├── README.md ├── assets ├── axion.gif ├── minimal_example_box.png ├── minimal_example_slice_level_0.png ├── minimal_example_slice_level_1.png ├── next_to_minimal_example_projection.png ├── next_to_minimal_example_spectrum.png ├── next_to_minimal_example_vev.png ├── spatial_evolution.pdf ├── temporal_evolution.pdf └── truncation_error.pdf ├── data └── spectra_ks.hdf5 ├── docs └── doxygen │ ├── doxygen.conf │ └── main.dox ├── examples ├── MinimalExample │ ├── Makefile │ ├── README.md │ ├── inputs │ └── run.sh └── NextToMinimalExample │ ├── Makefile │ ├── README.md │ ├── inputs │ └── run.sh ├── notebooks ├── AddSpectrumBins.ipynb ├── MinimalExample.ipynb ├── NextToMinimalExample.ipynb └── Subcycling.ipynb ├── projects ├── AxionStrings │ ├── AxionStrings.h │ ├── cosmology.cpp │ ├── cosmology.h │ ├── kernels_energy_densities.h │ ├── kernels_rhs.h │ ├── kernels_tagging.h │ └── setup.h ├── AxionStringsPostevolution │ ├── AxionStringsPostevolution.cpp │ ├── AxionStringsPostevolution.h │ └── kernels_rhs.h ├── AxionStringsPreevolution │ ├── AxionStringsPreevolution.cpp │ ├── AxionStringsPreevolution.h │ └── kernels_rhs.h ├── FirstOrderPhaseTransition │ ├── FirstOrderPhaseTransition.cpp │ ├── FirstOrderPhaseTransition.h │ ├── bubbles.h │ ├── kernels_misc.h │ ├── kernels_rhs.h │ ├── kernels_tagging.h │ ├── setup.h │ └── spectrum_modifier.h ├── MinimalExample │ └── MinimalExample.h └── NextToMinimalExample │ ├── NextToMinimalExample.cpp │ └── NextToMinimalExample.h ├── pySledgehamr ├── AxionStrings.py ├── Output.py ├── Sledgehamr.py └── __init__.py └── source ├── Make.package ├── fill_level.cpp ├── fill_level.h ├── gravitational_waves.cpp ├── gravitational_waves.h ├── integrators ├── Make.package ├── amrex_integrators.cpp ├── amrex_integrators.h ├── integrator.cpp ├── integrator.h ├── lsssprk3.cpp ├── lsssprk3.h ├── rkn.cpp └── rkn.h ├── io_module.cpp ├── io_module.h ├── kernels.h ├── level_data.h ├── level_synchronizer.cpp ├── level_synchronizer.h ├── local_regrid ├── Make.package ├── local_regrid.cpp ├── local_regrid.h ├── location.cpp ├── location.h ├── unique_layout.cpp └── unique_layout.h ├── macros.h ├── main.cpp ├── output_types ├── Make.package ├── amrex_plotfile.cpp ├── amrex_plotfile.h ├── checkpoint.cpp ├── checkpoint.h ├── level_writer.cpp ├── level_writer.h ├── output_module.cpp ├── output_module.h ├── projection.cpp ├── projection.h ├── slices.cpp ├── slices.h ├── spectrum.cpp └── spectrum.h ├── performance_monitor.cpp ├── performance_monitor.h ├── regrid_scheduler.cpp ├── regrid_scheduler.h ├── scalars.h ├── sledgehamr.cpp ├── sledgehamr.h ├── sledgehamr_init.cpp ├── sledgehamr_init.h ├── time_stepper.cpp ├── time_stepper.h ├── timer.cpp ├── timer.h └── utils ├── Make.package ├── fft.h ├── hdf5_utils.h └── sledgehamr_utils.h /.gitattributes: -------------------------------------------------------------------------------- 1 | *ipynb linguist-vendored 2 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: Doxygen Action 2 | 3 | # Controls when the action will run. Triggers the workflow on push or pull request 4 | # events but only for the master branch 5 | on: [push] 6 | 7 | # A workflow run is made up of one or more jobs that can run sequentially or in parallel 8 | jobs: 9 | # This workflow contains a single job called "build" 10 | build: 11 | # The type of runner that the job will run on 12 | runs-on: ubuntu-latest 13 | 14 | # Steps represent a sequence of tasks that will be executed as part of the job 15 | steps: 16 | # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it 17 | - uses: actions/checkout@v2 18 | 19 | - name: Doxygen Action 20 | uses: mattnotmitt/doxygen-action@v1.3.1 21 | with: 22 | doxyfile-path: "doxygen.conf" 23 | working-directory: "docs/doxygen/" 24 | 25 | - name: Deploy 26 | uses: peaceiris/actions-gh-pages@v3.9.3 27 | with: 28 | github_token: ${{ secrets.GITHUB_TOKEN }} 29 | # Default Doxyfile build documentation to html directory. 30 | publish_dir: ./html 31 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Python 2 | __pycache__/ 3 | 4 | # vim: 5 | *~ 6 | *.swp 7 | *.swo 8 | 9 | # generated code 10 | source/projects.h 11 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | We as members, contributors, and leaders pledge to make participation in our 6 | community a harassment-free experience for everyone, regardless of age, body 7 | size, visible or invisible disability, ethnicity, sex characteristics, gender 8 | identity and expression, level of experience, education, socio-economic status, 9 | nationality, personal appearance, race, religion, or sexual identity 10 | and orientation. 11 | 12 | We pledge to act and interact in ways that contribute to an open, welcoming, 13 | diverse, inclusive, and healthy community. 14 | 15 | ## Our Standards 16 | 17 | Examples of behavior that contributes to a positive environment for our 18 | community include: 19 | 20 | * Demonstrating empathy and kindness toward other people 21 | * Being respectful of differing opinions, viewpoints, and experiences 22 | * Giving and gracefully accepting constructive feedback 23 | * Accepting responsibility and apologizing to those affected by our mistakes, 24 | and learning from the experience 25 | * Focusing on what is best not just for us as individuals, but for the 26 | overall community 27 | 28 | Examples of unacceptable behavior include: 29 | 30 | * The use of sexualized language or imagery, and sexual attention or 31 | advances of any kind 32 | * Trolling, insulting or derogatory comments, and personal or political attacks 33 | * Public or private harassment 34 | * Publishing others' private information, such as a physical or email 35 | address, without their explicit permission 36 | * Other conduct which could reasonably be considered inappropriate in a 37 | professional setting 38 | 39 | ## Enforcement Responsibilities 40 | 41 | Community leaders are responsible for clarifying and enforcing our standards of 42 | acceptable behavior and will take appropriate and fair corrective action in 43 | response to any behavior that they deem inappropriate, threatening, offensive, 44 | or harmful. 45 | 46 | Community leaders have the right and responsibility to remove, edit, or reject 47 | comments, commits, code, wiki edits, issues, and other contributions that are 48 | not aligned to this Code of Conduct, and will communicate reasons for moderation 49 | decisions when appropriate. 50 | 51 | ## Scope 52 | 53 | This Code of Conduct applies within all community spaces, and also applies when 54 | an individual is officially representing the community in public spaces. 55 | Examples of representing our community include using an official e-mail address, 56 | posting via an official social media account, or acting as an appointed 57 | representative at an online or offline event. 58 | 59 | ## Enforcement 60 | 61 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 62 | reported to the community leaders responsible for enforcement at 63 | m.s.a.buschmann@uva.nl. 64 | All complaints will be reviewed and investigated promptly and fairly. 65 | 66 | All community leaders are obligated to respect the privacy and security of the 67 | reporter of any incident. 68 | 69 | ## Enforcement Guidelines 70 | 71 | Community leaders will follow these Community Impact Guidelines in determining 72 | the consequences for any action they deem in violation of this Code of Conduct: 73 | 74 | ### 1. Correction 75 | 76 | **Community Impact**: Use of inappropriate language or other behavior deemed 77 | unprofessional or unwelcome in the community. 78 | 79 | **Consequence**: A private, written warning from community leaders, providing 80 | clarity around the nature of the violation and an explanation of why the 81 | behavior was inappropriate. A public apology may be requested. 82 | 83 | ### 2. Warning 84 | 85 | **Community Impact**: A violation through a single incident or series 86 | of actions. 87 | 88 | **Consequence**: A warning with consequences for continued behavior. No 89 | interaction with the people involved, including unsolicited interaction with 90 | those enforcing the Code of Conduct, for a specified period of time. This 91 | includes avoiding interactions in community spaces as well as external channels 92 | like social media. Violating these terms may lead to a temporary or 93 | permanent ban. 94 | 95 | ### 3. Temporary Ban 96 | 97 | **Community Impact**: A serious violation of community standards, including 98 | sustained inappropriate behavior. 99 | 100 | **Consequence**: A temporary ban from any sort of interaction or public 101 | communication with the community for a specified period of time. No public or 102 | private interaction with the people involved, including unsolicited interaction 103 | with those enforcing the Code of Conduct, is allowed during this period. 104 | Violating these terms may lead to a permanent ban. 105 | 106 | ### 4. Permanent Ban 107 | 108 | **Community Impact**: Demonstrating a pattern of violation of community 109 | standards, including sustained inappropriate behavior, harassment of an 110 | individual, or aggression toward or disparagement of classes of individuals. 111 | 112 | **Consequence**: A permanent ban from any sort of public interaction within 113 | the community. 114 | 115 | ## Attribution 116 | 117 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 118 | version 2.0, available at 119 | https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. 120 | 121 | Community Impact Guidelines were inspired by [Mozilla's code of conduct 122 | enforcement ladder](https://github.com/mozilla/diversity). 123 | 124 | [homepage]: https://www.contributor-covenant.org 125 | 126 | For answers to common questions about this code of conduct, see the FAQ at 127 | https://www.contributor-covenant.org/faq. Translations are available at 128 | https://www.contributor-covenant.org/translations. 129 | -------------------------------------------------------------------------------- /Make.projects: -------------------------------------------------------------------------------- 1 | genfile = $(SRC)/projects.h 2 | 3 | $(shell echo "/* This file is auto-generated. MODIFICATIONS ARE FUTILE! */" > $(genfile)) 4 | $(shell echo "" >> $(genfile)) 5 | 6 | $(shell echo "#ifndef SLEDGEHAMR_PROJECTS_H_" >> $(genfile)) 7 | $(shell echo "#define SLEDGEHAMR_PROJECTS_H_" >> $(genfile)) 8 | $(shell echo "" >> $(genfile)) 9 | 10 | $(foreach prj, $(PROJECTS), $(shell echo "#include \"$(prj).h\"" >> $(genfile))) 11 | $(shell echo "" >> $(genfile)) 12 | 13 | $(shell echo "#define SLEDGEHAMR_PROJECT(str) {\\" >> $(genfile)) 14 | $(foreach prj, $(PROJECTS), $(shell echo " if (str == \"$(prj)\") return new $(prj)::$(prj);\\" >> $(genfile))) 15 | $(shell echo "}" >> $(genfile)) 16 | $(shell echo "" >> $(genfile)) 17 | 18 | $(shell echo "#endif // SLEDGEHAMR_PROJECTS_H_" >> $(genfile)) 19 | -------------------------------------------------------------------------------- /Make.sledgehamr: -------------------------------------------------------------------------------- 1 | # sledgeHAMR 2 | SLEDGEHAMR_HOME_ABS = $(realpath $(SLEDGEHAMR_HOME)) 3 | SRC = $(SLEDGEHAMR_HOME_ABS)/source 4 | DEFINES += -DSLEDGEHAMR_DATA_PATH=\"$(SLEDGEHAMR_HOME_ABS)/data/\" 5 | 6 | VPATH_LOCATIONS += $(SRC) 7 | INCLUDE_LOCATIONS += $(SRC) 8 | include $(SRC)/Make.package 9 | 10 | VPATH_LOCATIONS += $(SRC)/integrators 11 | INCLUDE_LOCATIONS += $(SRC)/integrators 12 | include $(SRC)/integrators/Make.package 13 | 14 | VPATH_LOCATIONS += $(SRC)/local_regrid 15 | INCLUDE_LOCATIONS += $(SRC)/local_regrid 16 | include $(SRC)/local_regrid/Make.package 17 | 18 | VPATH_LOCATIONS += $(SRC)/output_types 19 | INCLUDE_LOCATIONS += $(SRC)/output_types 20 | include $(SRC)/output_types/Make.package 21 | 22 | VPATH_LOCATIONS += $(SRC)/utils 23 | INCLUDE_LOCATIONS += $(SRC)/utils 24 | include $(SRC)/utils/Make.package 25 | 26 | # Projects 27 | SLEDGEHAMR_PROJECT_PATH ?= $(SLEDGEHAMR_HOME_ABS)/projects 28 | PROJECTS = $(shell ls $(SLEDGEHAMR_PROJECT_PATH)/) 29 | VPATH_LOCATIONS += $(foreach dir, $(PROJECTS), $(SLEDGEHAMR_PROJECT_PATH)/$(dir)) 30 | INCLUDE_LOCATIONS += $(foreach dir, $(PROJECTS), $(SLEDGEHAMR_PROJECT_PATH)/$(dir)) 31 | CEXE_sources += $(shell ls $(SLEDGEHAMR_PROJECT_PATH)/*/*.cpp | xargs -n 1 basename) 32 | CEXE_headers += $(shell ls $(SLEDGEHAMR_PROJECT_PATH)/*/*.h | xargs -n 1 basename) 33 | 34 | include $(SLEDGEHAMR_HOME_ABS)/Make.projects 35 | 36 | # SWFFT 37 | include $(AMREX_HOME)/Src/Extern/SWFFT/Make.package 38 | INCLUDE_LOCATIONS += $(AMREX_HOME)/Src/Extern/SWFFT 39 | VPATH_LOCATIONS += $(AMREX_HOME)/Src/Extern/SWFFT 40 | LIBRARIES += -L$(FFTW_DIR) -lfftw3_mpi -lfftw3_omp -lfftw3 41 | 42 | # Boost 43 | #LIBRARIES += -lboost_system -lboost_filesystem 44 | 45 | # HDF5 46 | ENABLE_HDF5 = yes 47 | LIBRARIES += -lhdf5 48 | 49 | # AMReX 50 | USE_MPI = TRUE 51 | USE_OMP = TRUE 52 | DIM = 3 53 | 54 | include $(AMREX_HOME)/Tools/GNUMake/Make.defs 55 | include $(AMREX_HOME)/Src/Base/Make.package 56 | 57 | AMReXdirs := Base Boundary AmrCore 58 | AMReXpack += $(foreach dir, $(AMReXdirs), $(AMREX_HOME)/Src/$(dir)/Make.package) 59 | 60 | include $(AMReXpack) 61 | include $(AMREX_HOME)/Tools/GNUMake/Make.rules 62 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![arXiv](https://img.shields.io/badge/arXiv-2404.02950%20-green.svg)](https://arxiv.org/abs/2404.02950) 2 | 3 | # Sledgehamr: Simulating scalar fields with adaptive mesh refinement 4 | Sledgehamr (**S**ca**L**ar fi**E**ld **D**ynamics **G**etting solv**E**d wit**H** **A**daptive **M**esh **R**efinement) is an AMReX-based code to simulate the dynamics of coupled scalar fields on a 3-dimensional mesh. Adaptive mesh refinement (AMR) can boost performance if spatially localized regions of the scalar field require high resolution. Compatible with both GPU and CPU clusters, sledgehamr offers a flexible and customizable framework. This framework enables various applications, such as the generation of gravitational wave spectra. 5 | 6 |

7 | 8 |

9 | 10 | For a detailed description of the code please consult the accompanying paper: 11 | https://arxiv.org/abs/arXiv:2404.02950 12 | 13 | For questions, comments, or bug reports please use the GitHub issues feature, or contact me via email: 14 | m.s.a.buschmann@uva.nl 15 | 16 | ## Installation 17 | 18 | ### Prerequisites 19 | * AMReX (```git clone https://github.com/AMReX-Codes/amrex```) 20 | * FFTW3 21 | 22 | ### Running the minimal examples 23 | The minimal examples can be run by following the instructions in the Jupyter notebooks ```notebooks/MinimalExample.ipynb``` and ```notebooks/NextToMinimalExample.ipynb```. 24 | 25 | ### Create a project 26 | Sledgehamr comes with a few different physics scenarios already implemented such as axion strings and a first-order phase transition of a single scalar field + gravitational waves, as well as two example projects ```MinimalExample``` and ```NextToMinimalExample```. The accompanying paper describes in detail how other scenarios can be implemented. 27 | 28 | ## Code documentation 29 | * https://msabuschmann.github.io/sledgehamr/ 30 | * https://arxiv.org/abs/2404.02950 31 | 32 | ## How to cite 33 | If you use sledgehamr, please cite its accompanying paper: 34 | 35 | * Malte Buschmann, "Sledgehamr: Simulation Scalar Fields with Adaptive Mesh Refinement", 36 | Astrophys.J. 979 (2025) 2, 220, arXiv:2404.02950 37 | 38 | BibTex: 39 | ``` 40 | @article{Buschmann:2024bfj, 41 | author = "Buschmann, Malte", 42 | title = "{Sledgehamr: Simulating Scalar Fields with Adaptive Mesh Refinement}", 43 | eprint = "2404.02950", 44 | archivePrefix = "arXiv", 45 | primaryClass = "hep-ph", 46 | doi = "10.3847/1538-4357/ad9ea2", 47 | journal = "Astrophys. J.", 48 | volume = "979", 49 | number = "2", 50 | pages = "220", 51 | year = "2025" 52 | } 53 | ``` 54 | 55 | ## Publications using this code 56 | 57 | ### 2022 58 | * "Dark matter from axion strings with adaptive mesh refinement", M. Buschmann et al., Nature Commun. 13 (2022) 1, 1049, https://arxiv.org/abs/2108.05368 59 | 60 | ### 2024 61 | * "Signatures of primordial energy injection from axion strings", J. Benabou et al., Phys.Rev.D 109 (2024) 5, 055005, https://arxiv.org/abs/2308.01334 62 | * "Axion mass prediction from adaptive mesh refinement cosmological lattice simulations", J. Benabou et al., https://arxiv.org/abs/2412.08699 63 | 64 | ### 2025 65 | * "Thick and Thin Wall Collisions with Adaptive Mesh Refinement", M. Buschmann et al., to appear 66 | 67 | 68 | -------------------------------------------------------------------------------- /assets/axion.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MSABuschmann/sledgehamr/c92e964b2b6d4fb332eba55d6a05ae9de04ee050/assets/axion.gif -------------------------------------------------------------------------------- /assets/minimal_example_box.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MSABuschmann/sledgehamr/c92e964b2b6d4fb332eba55d6a05ae9de04ee050/assets/minimal_example_box.png -------------------------------------------------------------------------------- /assets/minimal_example_slice_level_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MSABuschmann/sledgehamr/c92e964b2b6d4fb332eba55d6a05ae9de04ee050/assets/minimal_example_slice_level_0.png -------------------------------------------------------------------------------- /assets/minimal_example_slice_level_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MSABuschmann/sledgehamr/c92e964b2b6d4fb332eba55d6a05ae9de04ee050/assets/minimal_example_slice_level_1.png -------------------------------------------------------------------------------- /assets/next_to_minimal_example_projection.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MSABuschmann/sledgehamr/c92e964b2b6d4fb332eba55d6a05ae9de04ee050/assets/next_to_minimal_example_projection.png -------------------------------------------------------------------------------- /assets/next_to_minimal_example_spectrum.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MSABuschmann/sledgehamr/c92e964b2b6d4fb332eba55d6a05ae9de04ee050/assets/next_to_minimal_example_spectrum.png -------------------------------------------------------------------------------- /assets/next_to_minimal_example_vev.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MSABuschmann/sledgehamr/c92e964b2b6d4fb332eba55d6a05ae9de04ee050/assets/next_to_minimal_example_vev.png -------------------------------------------------------------------------------- /assets/spatial_evolution.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MSABuschmann/sledgehamr/c92e964b2b6d4fb332eba55d6a05ae9de04ee050/assets/spatial_evolution.pdf -------------------------------------------------------------------------------- /assets/temporal_evolution.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MSABuschmann/sledgehamr/c92e964b2b6d4fb332eba55d6a05ae9de04ee050/assets/temporal_evolution.pdf -------------------------------------------------------------------------------- /assets/truncation_error.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MSABuschmann/sledgehamr/c92e964b2b6d4fb332eba55d6a05ae9de04ee050/assets/truncation_error.pdf -------------------------------------------------------------------------------- /data/spectra_ks.hdf5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MSABuschmann/sledgehamr/c92e964b2b6d4fb332eba55d6a05ae9de04ee050/data/spectra_ks.hdf5 -------------------------------------------------------------------------------- /docs/doxygen/main.dox: -------------------------------------------------------------------------------- 1 | /** 2 | \mainpage sledgehamr: Technical Documentation 3 | 4 | \tableofcontents 5 | 6 | \section welcome Welcome to the technical documentation of sledgehamr! 7 | 8 | This site provides the low-level information about sledgehamr. 9 | 10 | Sledgehamr (ScaLar fiEld Dynamics Getting solvEd witH Adaptive Mesh Refinement) is an AMReX-based code to simulate the dynamics of coupled scalar fields on a 3-dimensional mesh. Adaptive mesh refinement (AMR) can boost performance if spatially localized regions of the scalar field require high resolution. Compatible with both GPU and CPU clusters, sledgehamr offers a flexible and customizable framework. This framework enables various applications, such as the generation of gravitational wave spectra. 11 | 12 | */ 13 | -------------------------------------------------------------------------------- /examples/MinimalExample/Makefile: -------------------------------------------------------------------------------- 1 | # AMREX_HOME defines the directory in which we will find all the AMReX code. 2 | # If you set AMREX_HOME as an environment variable, this line will be ignored 3 | # AMREX_HOME ?= 4 | SLEDGEHAMR_HOME ?= ../../../sledgehamr 5 | #SLEDGEHAMR_PROJECT_PATH = 6 | 7 | # compiler 8 | COMP = gnu 9 | USE_CUDA = FALSE 10 | 11 | # Optional debugging options. 12 | DEBUG = FALSE 13 | USE_ASSERTION = FALSE 14 | FSANITIZER = FALSE 15 | MEM_PROFILE = TRUE 16 | 17 | include $(SLEDGEHAMR_HOME)/Make.sledgehamr 18 | -------------------------------------------------------------------------------- /examples/MinimalExample/README.md: -------------------------------------------------------------------------------- 1 | # Minimal Example 2 | 3 | This minimal example illustrates how to run sledgehamr. We use axion strings as an example, which are based on a complex scalar field $\Phi$ with dimensionless components $(\psi_1, \psi_2)^T$. 4 | 5 | The equations of motion are 6 | * $\psi_i' = \Pi_i$ 7 | 8 | and 9 | 10 | * $\Pi_i' = -\frac{2}{\eta}\Pi_i + \nabla^2\psi_i - \eta^2\psi_i(\psi_1^2+\psi_2^2-1) - a\psi_i$, 11 | 12 | where $\Pi_i$ is the conjugate momenta of $\psi_i$, $\eta$ is time, and $a$ some constant. 13 | The code implementation can be found under ```projects/MinimalExample/``` 14 | 15 | ## How to run 16 | 1. Ensure prerequisites are installed: Boost, FFTW3, and HDF5. 17 | 2. Clone the AMReX repository (https://github.com/AMReX-Codes/amrex) 18 | 3. Make sure the paths to the sledgehamr repository (```$SLEDGEHAMR_HOME```) and AMReX 19 | repository (```$AMREX_HOME```) are set in ```Makefile```. Adjust other compiler flags 20 | if needed. 21 | 4. Compile sledgehamr: ```make -j 6``` 22 | 5. Use the Jupyter Notebook ```notebooks/MinimalExample.ipynb``` to generate an 23 | initial state. 24 | 6. Run the executable. An example slurm submission script is provided: ```run.sh``` 25 | 5. Use the example notebook ```notebooks/MinimalExample.ipynb``` to plot the results 26 | 27 | ## Simulation output 28 | Below are illustrations of the simulation output for the two scalar degrees of freedom, the axion $a=\mathrm{arctan2}(\psi2, \psi1)$ and the radial mode $r=\sqrt{\psi_1^2+\psi_2^2}$. 29 | 30 |

31 | 32 |

33 | 34 |

35 | 36 |

37 | 38 |

39 | 40 |

41 | -------------------------------------------------------------------------------- /examples/MinimalExample/inputs: -------------------------------------------------------------------------------- 1 | # ----------------- Select project 2 | project.name = MinimalExample 3 | 4 | # ----------------- Simulation parameters 5 | sim.t_start = 0.1 6 | sim.t_end = 5.5 7 | sim.L = 15 8 | sim.cfl = 0.3 9 | 10 | # ----------------- Integrator 11 | integrator.type = 10 12 | 13 | # ----------------- AMR parameters 14 | amr.coarse_level_grid_size = 256 15 | amr.blocking_factor = 4 16 | amr.nghost = 2 17 | amr.max_refinement_levels = 2 18 | amr.n_error_buf = 3 19 | amr.regrid_dt = 0.2 20 | 21 | amr.te_crit = 1e-2 22 | 23 | # ----------------- Input parameters 24 | input.initial_state = initial_state_256.hdf5 25 | 26 | # ----------------- Output settings 27 | output.output_folder = output 28 | output.slices.interval = 0.5 29 | output.coarse_box.interval = 3 30 | -------------------------------------------------------------------------------- /examples/MinimalExample/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #SBATCH --constraint=cpu 3 | #SBATCH --nodes=1 4 | #SBATCH --tasks-per-node=8 5 | #SBATCH --cpus-per-task=16 6 | #SBATCH --qos=debug 7 | #SBATCH --time=00:15:00 8 | cd $SLURM_SUBMIT_DIR 9 | 10 | export SLURM_CPU_BIND="cores" 11 | export OMP_PLACES=threads 12 | export OMP_PROC_BIND=spread 13 | export OMP_NUM_THREADS=16 14 | 15 | srun main3d.gnu.x86-milan.MPROF.MPI.OMP.ex inputs 16 | -------------------------------------------------------------------------------- /examples/NextToMinimalExample/Makefile: -------------------------------------------------------------------------------- 1 | # AMREX_HOME defines the directory in which we will find all the AMReX code. 2 | # If you set AMREX_HOME as an environment variable, this line will be ignored 3 | # AMREX_HOME ?= 4 | SLEDGEHAMR_HOME ?= ../../../sledgehamr 5 | #SLEDGEHAMR_PROJECT_PATH = 6 | 7 | # compiler 8 | COMP = gnu 9 | USE_CUDA = FALSE 10 | 11 | # Optional debugging options. 12 | DEBUG = FALSE 13 | USE_ASSERTION = FALSE 14 | FSANITIZER = FALSE 15 | MEM_PROFILE = TRUE 16 | 17 | include $(SLEDGEHAMR_HOME)/Make.sledgehamr 18 | -------------------------------------------------------------------------------- /examples/NextToMinimalExample/README.md: -------------------------------------------------------------------------------- 1 | # Next-to-Minimal Example 2 | 3 | This example is based on the same scenario as the Minimal Example, however, uses some of the features sledgehamr offers: 4 | * We introduce an extra run-time parameter $\lambda$ which is loaded from the inputs file and used in the equations of motion. 5 | * We output a projection of the axion energy density $\dot{a}^2$. 6 | * We output the power-spectrum of $\dot{a}^2$. 7 | * We implement a custom output type where we save the time-evolution of the average value of the radial mode $r$. 8 | 9 | The code implementation can be found under ```projects/NextToMinimalExample/```. 10 | 11 | ## How to run 12 | 1. Ensure prerequisites are installed: Boost, FFTW3, and HDF5. 13 | 2. Clone the AMReX repository (https://github.com/AMReX-Codes/amrex) 14 | 3. Make sure the paths to the sledgehamr repository (```$SLEDGEHAMR_HOME```) and AMReX 15 | repository (```$AMREX_HOME```) are set in ```Makefile```. Adjust other compiler flags 16 | if needed. 17 | 4. Compile sledgehamr: ```make -j 6``` 18 | 5. Use the Jupyter Notebook ```notebooks/MinimalExample.ipynb``` to generate an 19 | initial state if it hasn't already been generated for the Minimal Example. 20 | 6. Run the executable. An example slurm submission script is provided: ```run.sh``` 21 | 5. Use the example notebook ```notebooks/NextToMinimalExample.ipynb``` to plot the results 22 | 23 | ## Simulation output 24 | Below are illustrations of the simulation output: 25 | 26 |

27 | 28 |

29 | 30 |

31 | 32 |

33 | 34 |

35 | 36 |

37 | -------------------------------------------------------------------------------- /examples/NextToMinimalExample/inputs: -------------------------------------------------------------------------------- 1 | # ----------------- Select project 2 | project.name = NextToMinimalExample 3 | project.lambda = 1 4 | 5 | # ----------------- Simulation parameters 6 | sim.t_start = 0.1 7 | sim.t_end = 5.5 8 | sim.L = 15 9 | sim.cfl = 0.3 10 | 11 | # ----------------- Integrator 12 | integrator.type = 10 13 | 14 | # ----------------- AMR parameters 15 | amr.coarse_level_grid_size = 256 16 | amr.blocking_factor = 4 17 | amr.nghost = 2 18 | amr.max_refinement_levels = 2 19 | amr.n_error_buf = 3 20 | amr.regrid_dt = 0.2 21 | 22 | amr.te_crit = 1e-2 23 | 24 | # ----------------- Input parameters 25 | input.initial_state = ../MinimalExample/initial_state_256.hdf5 26 | 27 | # ----------------- Output settings 28 | output.output_folder = output 29 | output.slices.interval = 0.5 30 | output.coarse_box.interval = 3 31 | output.projections.interval = 3 32 | output.spectra.interval = 3 33 | output.avg.interval = 0.2 34 | -------------------------------------------------------------------------------- /examples/NextToMinimalExample/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #SBATCH --constraint=cpu 3 | #SBATCH --nodes=1 4 | #SBATCH --tasks-per-node=8 5 | #SBATCH --cpus-per-task=16 6 | #SBATCH --qos=debug 7 | #SBATCH --time=00:15:00 8 | cd $SLURM_SUBMIT_DIR 9 | 10 | export SLURM_CPU_BIND="cores" 11 | export OMP_PLACES=threads 12 | export OMP_PROC_BIND=spread 13 | export OMP_NUM_THREADS=16 14 | 15 | srun main3d.gnu.x86-milan.MPROF.MPI.OMP.ex inputs 16 | -------------------------------------------------------------------------------- /notebooks/AddSpectrumBins.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "b19057e7-4490-4a9c-9e1d-7780c14c8cde", 6 | "metadata": { 7 | "tags": [] 8 | }, 9 | "source": [ 10 | "### Pre-compute all unique $k$ values needed for the computation of (gravitional wave) spectra.\n", 11 | "\n", 12 | "To compute (unbinned) spectra sledgehamr will need to know all unique $k$ values in advance before running the simulation. These values are stored in a file within the 'data' directory which by default already contains values for a few different grid sizes (the number of unique $k$ values depends on the number of coarse level grid cells). For larger grids, however, you will need to precompute them yourself using the script below. This is necessary as otherwise the file would be too large for a github repo (~Gb). This computation needs to be done only once for of a given number of coarse level grid cells." 13 | ] 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": 1, 18 | "id": "a9e9c257-6ebc-4276-adb7-e6e5c266b780", 19 | "metadata": { 20 | "tags": [] 21 | }, 22 | "outputs": [], 23 | "source": [ 24 | "import sys\n", 25 | "sledgehamr_dir = '../.'\n", 26 | "sys.path.append(sledgehamr_dir)\n", 27 | "\n", 28 | "import pySledgehamr as sledgehamr" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "execution_count": 2, 34 | "id": "3564bc52-3595-4941-8363-2f160ba49729", 35 | "metadata": { 36 | "tags": [] 37 | }, 38 | "outputs": [ 39 | { 40 | "name": "stdout", 41 | "output_type": "stream", 42 | "text": [ 43 | "Binning already exists in file.\n" 44 | ] 45 | } 46 | ], 47 | "source": [ 48 | "data_dir = sledgehamr_dir + '/data'\n", 49 | "coarse_level_cells = 256\n", 50 | "sledgehamr.CreateSpectrumBins(data_dir, coarse_level_cells)" 51 | ] 52 | } 53 | ], 54 | "metadata": { 55 | "kernelspec": { 56 | "display_name": "NERSC Python", 57 | "language": "python", 58 | "name": "python3" 59 | }, 60 | "language_info": { 61 | "codemirror_mode": { 62 | "name": "ipython", 63 | "version": 3 64 | }, 65 | "file_extension": ".py", 66 | "mimetype": "text/x-python", 67 | "name": "python", 68 | "nbconvert_exporter": "python", 69 | "pygments_lexer": "ipython3", 70 | "version": "3.11.6" 71 | } 72 | }, 73 | "nbformat": 4, 74 | "nbformat_minor": 5 75 | } 76 | -------------------------------------------------------------------------------- /projects/AxionStrings/AxionStrings.h: -------------------------------------------------------------------------------- 1 | #ifndef PROJECTS_AXION_STRINGS_H_ 2 | #define PROJECTS_AXION_STRINGS_H_ 3 | 4 | #include 5 | 6 | #include "cosmology.h" 7 | #include "kernels_rhs.h" 8 | 9 | namespace AxionStrings { 10 | 11 | SLEDGEHAMR_FINISH_SETUP 12 | 13 | /** @brief Class to simulate axion strings. 14 | */ 15 | class AxionStrings : public sledgehamr::Sledgehamr { 16 | public: 17 | SLEDGEHAMR_INITIALIZE_PROJECT(AxionStrings) 18 | 19 | /** @brief Override Init function and let cosmology module handle the setup. 20 | */ 21 | void Init() override { 22 | cosmo.Init(this); 23 | }; 24 | 25 | /** @brief We want to create a level only when string width is less than 26 | * threshold. 27 | * @param lev Level in question. 28 | * @param time Current time. 29 | * @return Whether we want to create level or not. 30 | */ 31 | bool CreateLevelIf(const int lev, const double time) override { 32 | return cosmo.CreateLevelIf(lev, time); 33 | }; 34 | 35 | private: 36 | /** @brief Axion cosmology module to set up the scenario. 37 | */ 38 | Cosmology cosmo; 39 | }; 40 | 41 | }; // namespace AxionStrings 42 | 43 | #endif // PROJECTS_AXION_STRINGS_H_ 44 | -------------------------------------------------------------------------------- /projects/AxionStrings/cosmology.cpp: -------------------------------------------------------------------------------- 1 | #include "cosmology.h" 2 | #include 3 | #include 4 | 5 | namespace AxionStrings { 6 | 7 | /* @brief Init function to parse variables and setup output types. 8 | */ 9 | void Cosmology::Init(sledgehamr::Sledgehamr* owner) { 10 | sim = owner; 11 | ParseVariables(); 12 | PrintRefinementTimes(); 13 | SetProjections(); 14 | SetSpectra(); 15 | SetXiMeasurement(); 16 | } 17 | 18 | /** @brief Parse external variables. 19 | */ 20 | void Cosmology::ParseVariables() { 21 | amrex::ParmParse pp_prj("project"); 22 | pp_prj.get("string_width_threshold", string_width_threshold); 23 | } 24 | 25 | /** @brief Prints out when a new refinement level will be introduced. 26 | */ 27 | void Cosmology::PrintRefinementTimes() { 28 | for (int lev = 1; lev <= sim->GetMaxLevel(); ++lev) { 29 | amrex::Print() << "Level " << lev << " (" 30 | << sledgehamr::utils::LevelName(lev) 31 | << ") will be introduced at eta = " 32 | << RefinementTime(lev-1) << std::endl; 33 | } 34 | } 35 | 36 | /** @brief Sets up projections. 37 | */ 38 | void Cosmology::SetProjections() { 39 | // Add projections. 40 | sim->io_module->projections.emplace_back(a_prime2, "a_prime2"); 41 | sim->io_module->projections.emplace_back(r_prime2, "r_prime2"); 42 | } 43 | 44 | /** @brief Sets up spectrum. 45 | */ 46 | void Cosmology::SetSpectra() { 47 | // Add spectra and change time interval to log(m_r/H). 48 | sim->io_module->spectra.emplace_back(a_prime_screened, "a_prime_screened"); 49 | sim->io_module->output[sim->io_module->idx_spectra].SetTimeFunction( 50 | TIME_FCT(Cosmology::Log)); 51 | } 52 | 53 | /** @brief Adds custom output type to calculate string length \xi on the fly. 54 | */ 55 | void Cosmology::SetXiMeasurement() { 56 | sim->io_module->output.emplace_back("xi", OUTPUT_FCT(Cosmology::WriteXi)); 57 | sim->io_module->output.back().SetTimeFunction(TIME_FCT(Cosmology::Log)); 58 | } 59 | 60 | /** @brief OUTPUT_FUNCTION to calculate and write string length \xi. 61 | * @param time Current time. 62 | * @param prefix Folder to which to write output. 63 | */ 64 | bool Cosmology::WriteXi(double time, std::string prefix) { 65 | int lev = sim->GetFinestLevel(); 66 | double xi = Xi(lev, time); 67 | double log = Log(time); 68 | constexpr int nsize = 4; 69 | double data[nsize] = {static_cast(lev), time, log, xi}; 70 | 71 | amrex::Print() << "Write xi: " << prefix << ", xi=" << xi << std::endl; 72 | 73 | if (amrex::ParallelDescriptor::IOProcessor()) { 74 | std::string filename = prefix + "/xi.h5"; 75 | hid_t file_id = H5Fcreate(filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, 76 | H5P_DEFAULT); 77 | sledgehamr::utils::hdf5::Write(file_id, "data", data, nsize); 78 | H5Fclose(file_id); 79 | } 80 | 81 | return true; 82 | } 83 | 84 | /** @brief Calculates string length \xi. 85 | * @param lev On which level to calculate \xi. 86 | * @param eta Current time \eta. 87 | * @return \xi. 88 | */ 89 | double Cosmology::Xi(const int lev, const double eta) { 90 | long string_tags = GetStringTags(lev); 91 | 92 | const double T1 = 5.4954174414835757e+17; 93 | const double mpl = 1.22e19; 94 | const double gStar = 106; 95 | 96 | double string_length_sim_units = static_cast(string_tags) 97 | * 2./3. * sim->GetDx(lev); 98 | double physical_string_length = BoxToPhysical(string_length_sim_units, eta, 99 | T1, mpl, gStar); 100 | double physical_box_size = BoxToPhysical(sim->GetL(), eta, T1, mpl, 101 | gStar); 102 | 103 | double T = XiTemp(eta, T1); 104 | double time = XiTime(T, mpl, gStar); 105 | double xi = physical_string_length * std::pow(time, 2) 106 | / std::pow(physical_box_size, 3); 107 | return xi; 108 | } 109 | 110 | /** @brief Returns the number of string-plaquette piercings. 111 | * @param lev On which level to identify piercings. 112 | * @return Number of tags. 113 | */ 114 | long Cosmology::GetStringTags(const int lev) { 115 | const sledgehamr::LevelData& state = sim->GetLevelData(lev); 116 | double dx = sim->GetDx(lev); 117 | double dt = sim->GetDt(lev); 118 | long ntags = 0; 119 | 120 | // Lets just do this on CPU even if GPUs available. This section is not 121 | // performace critical and it is a lot simpler this way. 122 | #pragma omp parallel reduction(+: ntags) 123 | for (amrex::MFIter mfi(state, true); mfi.isValid(); ++mfi) { 124 | const amrex::Array4& state_fab = state.array(mfi); 125 | const amrex::Box& tilebox = mfi.tilebox(); 126 | const amrex::Dim3 lo = amrex::lbound(tilebox); 127 | const amrex::Dim3 hi = amrex::ubound(tilebox); 128 | 129 | for (int k = lo.z; k <= hi.z; ++k) { 130 | for (int j = lo.y; j <= hi.y; ++j) { 131 | AMREX_PRAGMA_SIMD 132 | for (int i = lo.x; i <= hi.x; ++i) { 133 | ntags += TagCellForRefinement(state_fab, i, j, k, lev, 134 | state.t, dt, dx, NULL); 135 | } 136 | } 137 | } 138 | } 139 | 140 | amrex::ParallelDescriptor::ReduceLongSum(ntags); 141 | return ntags; 142 | } 143 | 144 | }; // namespace AxionStrings 145 | -------------------------------------------------------------------------------- /projects/AxionStrings/cosmology.h: -------------------------------------------------------------------------------- 1 | #ifndef PROJECTS_AXION_STRINGS_COSMOLOGY_H_ 2 | #define PROJECTS_AXION_STRINGS_COSMOLOGY_H_ 3 | 4 | #include 5 | 6 | #include "setup.h" 7 | #include "kernels_tagging.h" 8 | #include "kernels_energy_densities.h" 9 | 10 | namespace AxionStrings { 11 | 12 | /** @brief Class to simulate axion strings. This is not a sledgehamr class but 13 | * will set up the axion string scenario if added to the project class. 14 | */ 15 | class Cosmology { 16 | public: 17 | void Init(sledgehamr::Sledgehamr* owner); 18 | 19 | /** @brief We only want to create a new level if string width is below 20 | * threshold. 21 | * @param lev Current level. 22 | * @param time Current time. 23 | * @return Whether we want to create level lev. 24 | */ 25 | bool CreateLevelIf(const int lev, const double time) { 26 | return StringWidth(lev-1, time) <= string_width_threshold; 27 | } 28 | 29 | /** @brief Current radial mode mass. 30 | * @param eta Current time eta. 31 | * @return m_r(eta) 32 | */ 33 | double Mr(const double eta) { 34 | return std::sqrt(2. * lambda) * eta; 35 | } 36 | 37 | /** @brief Current hubble time. 38 | * @param eta Current time eta. 39 | * @return H(eta). 40 | */ 41 | double H(const double eta) { 42 | return 1./eta; 43 | } 44 | 45 | /** @brief Current string width in units of the grid spacing. 46 | * @param lev Current level. 47 | * @param eta Current time eta. 48 | * @return String width. 49 | */ 50 | double StringWidth(const int lev, const double eta) { 51 | return 1./(Mr(eta) * sim->GetDx(lev)); 52 | } 53 | 54 | /** @brief Returns time at which a level will be introduced. 55 | * @param lev Level. 56 | * @return Refinement time for level lev. 57 | */ 58 | double RefinementTime(const int lev) { 59 | return sim->GetDimN(lev) / (sqrt(2.*lambda) 60 | * string_width_threshold * sim->GetL()); 61 | } 62 | 63 | /** @brief Computes scale separation. 64 | * @param eta Current time eta. 65 | * @return log(m_r/H) 66 | */ 67 | double Log(const double eta) { 68 | if (eta <= 0) 69 | return -DBL_MAX; 70 | return std::log( Mr(eta) / H(eta) ); 71 | } 72 | 73 | /** @brief Computes the physical box length. 74 | * @param L Comoving box length. 75 | * @param eta Current time eta. 76 | * @param T1 T_1 temperature. 77 | * @param mpl Planck mass. 78 | * @param gStar Degrees of freedom g_*. 79 | * @return Physical box length. 80 | */ 81 | double BoxToPhysical(const double L, const double eta, const double T1, 82 | const double mpl, const double gStar) { 83 | return L * eta / Hubble(T1, mpl, gStar); 84 | } 85 | 86 | /** @brief Computes the physical Hubble parameter. 87 | * @param T Temperature. 88 | * @param mpl Planck mass. 89 | * @param gStar Degrees of freedom g_*. 90 | * @return Physical Hubble parameter. 91 | */ 92 | double Hubble(const double T, const double mpl, const double gStar) { 93 | return std::sqrt(4.*std::pow(M_PI, 3) / 45. * gStar * std::pow(T, 4) 94 | / std::pow(mpl, 2)); 95 | } 96 | 97 | double XiTime(double T, double mpl, double gStar) { 98 | return 0.3012 / std::sqrt(gStar) * mpl / std::pow(T, 2); 99 | } 100 | 101 | double XiTemp(double eta, double T1) { 102 | return T1 / eta; 103 | } 104 | 105 | double Xi(const int lev, const double eta); 106 | 107 | private: 108 | void ParseVariables(); 109 | void PrintRefinementTimes(); 110 | void SetProjections(); 111 | void SetSpectra(); 112 | void SetXiMeasurement(); 113 | 114 | long GetStringTags(const int lev); 115 | bool WriteXi(double time, std::string prefix); 116 | 117 | /** @brief Minimum allowed string width. 118 | */ 119 | double string_width_threshold; 120 | 121 | /** @brief Minimum log for which we compute spectra. 122 | */ 123 | double spectra_log_min = 5; 124 | 125 | /** @brief At what interval we want to write string length \xi. 126 | */ 127 | double interval_xi_log = 0; 128 | 129 | /** @brief lambda parameter in Lagrangian. 130 | */ 131 | const double lambda = 1; 132 | 133 | /** @brief Pointer to the simulation. 134 | */ 135 | sledgehamr::Sledgehamr* sim; 136 | }; 137 | 138 | }; // namespace AxionStrings 139 | 140 | #endif // PROJECTS_AXION_STRINGS_COSMOLOGY_H_ 141 | -------------------------------------------------------------------------------- /projects/AxionStrings/kernels_energy_densities.h: -------------------------------------------------------------------------------- 1 | #ifndef PROJECTS_AXION_STRINGS_KERNELS_ENERGY_DENSITIES_H_ 2 | #define PROJECTS_AXION_STRINGS_KERNELS_ENERGY_DENSITIES_H_ 3 | 4 | #include "setup.h" 5 | 6 | namespace AxionStrings { 7 | 8 | /** @brief Kernel function computing axion energy density a'^2. 9 | * @param state Data from which to calculate RHS (current state). 10 | * @param i i-th cell index. 11 | * @param j j-th cell index. 12 | * @param k k-th cell index. 13 | * @param lev Current level. 14 | * @param time Current time. 15 | * @param dt Time step size. 16 | * @param dx Grid spacing. 17 | * @param params Optional parameters. 18 | * @return a'^2. 19 | */ 20 | AMREX_FORCE_INLINE 21 | double a_prime2(amrex::Array4 const& state, const int i, 22 | const int j, const int k, const int lev, const double time, 23 | const double dt, const double dx, 24 | const std::vector& params) { 25 | double Psi1 = state(i, j, k, Scalar::Psi1); 26 | double Psi2 = state(i, j, k, Scalar::Psi2); 27 | double Pi1 = state(i, j, k, Scalar::Pi1); 28 | double Pi2 = state(i, j, k, Scalar::Pi2); 29 | double r2 = Psi1*Psi1 + Psi2*Psi2; 30 | double prime_a = (Psi1*Pi2 - Psi2*Pi1)/r2; 31 | return prime_a*prime_a; 32 | } 33 | 34 | /** @brief Kernel function computing the squareroot of the screened axion energy 35 | density a'_{screened} = a' * r^2 36 | * @param state Data from which to calculate RHS (current state). 37 | * @param i i-th cell index. 38 | * @param j j-th cell index. 39 | * @param k k-th cell index. 40 | * @param lev Current level. 41 | * @param time Current time. 42 | * @param dt Time step size. 43 | * @param dx Grid spacing. 44 | * @param params Optional parameters. 45 | * @return a'_{screened} = a' * r^2 46 | */ 47 | AMREX_FORCE_INLINE 48 | double a_prime_screened(amrex::Array4 const& state, 49 | const int i, const int j, const int k, const int lev, const double time, 50 | const double dt, const double dx, 51 | const std::vector& params) { 52 | double Psi1 = state(i, j, k, Scalar::Psi1); 53 | double Psi2 = state(i, j, k, Scalar::Psi2); 54 | double Pi1 = state(i, j, k, Scalar::Pi1); 55 | double Pi2 = state(i, j, k, Scalar::Pi2); 56 | double prime_a = (Psi1*Pi2 - Psi2*Pi1); 57 | return prime_a; 58 | } 59 | 60 | /** @brief Kernel function computing r'^2. 61 | * @param state Data from which to calculate RHS (current state). 62 | * @param i i-th cell index. 63 | * @param j j-th cell index. 64 | * @param k k-th cell index. 65 | * @param lev Current level. 66 | * @param time Current time. 67 | * @param dt Time step size. 68 | * @param dx Grid spacing. 69 | * @param params Optional parameters. 70 | * @return r'^2. 71 | */ 72 | AMREX_FORCE_INLINE 73 | double r_prime2(amrex::Array4 const& state, const int i, 74 | const int j, const int k, const int lev, const double time, 75 | const double dt, const double dx, 76 | const std::vector& params) { 77 | double Psi1 = state(i, j, k, Scalar::Psi1); 78 | double Psi2 = state(i, j, k, Scalar::Psi2); 79 | double Pi1 = state(i, j, k, Scalar::Pi1); 80 | double Pi2 = state(i, j, k, Scalar::Pi2); 81 | double r2 = Psi1*Psi1 + Psi2*Psi2; 82 | double prime_r = (Psi1*Pi1 + Psi2*Pi2); 83 | return prime_r*prime_r/r2; 84 | } 85 | 86 | }; // namespace AxionStrings 87 | 88 | #endif // PROJECTS_AXION_STRINGS_KERNELS_ENERGY_DENSITIES_H_ 89 | -------------------------------------------------------------------------------- /projects/AxionStrings/kernels_rhs.h: -------------------------------------------------------------------------------- 1 | #ifndef PROJECTS_AXION_STRINGS_KERNELS_RHS_H_ 2 | #define PROJECTS_AXION_STRINGS_KERNELS_RHS_H_ 3 | 4 | #include 5 | #include "setup.h" 6 | 7 | namespace AxionStrings { 8 | 9 | /** @brief Function that calculates the RHS of the EOM at a single cell. 10 | * @param rhs Container to be filled with RHS. 11 | * @param state Data from which to calculate RHS (current state). 12 | * @param i i-th cell index. 13 | * @param j j-th cell index. 14 | * @param k k-th cell index. 15 | * @param lev Current level. 16 | * @param time Current time. 17 | * @param dt Time step size. 18 | * @param dx Grid spacing. 19 | * @param params Optional parameters. 20 | */ 21 | AMREX_GPU_DEVICE AMREX_FORCE_INLINE 22 | void Rhs(const amrex::Array4& rhs, 23 | const amrex::Array4& state, 24 | const int i, const int j, const int k, const int lev, 25 | const double time, const double dt, const double dx, 26 | const double* params) { 27 | // Fetch field values. 28 | double Psi1 = state(i, j, k, Scalar::Psi1); 29 | double Psi2 = state(i, j, k, Scalar::Psi2); 30 | double Pi1 = state(i, j, k, Scalar::Pi1); 31 | double Pi2 = state(i, j, k, Scalar::Pi2); 32 | 33 | double eta = time; 34 | 35 | // Compute Laplacians. 36 | constexpr int order = 2; 37 | double laplacian_Psi1 = sledgehamr::utils::Laplacian( 38 | state, i, j, k, Scalar::Psi1, dx*dx); 39 | double laplacian_Psi2 = sledgehamr::utils::Laplacian( 40 | state, i, j, k, Scalar::Psi2, dx*dx); 41 | 42 | // Compute EOM. 43 | double potential = eta*eta*( Psi1*Psi1 + Psi2*Psi2 - 1. ) + 0.56233; 44 | 45 | rhs(i, j, k, Scalar::Psi1) = Pi1; 46 | rhs(i, j, k, Scalar::Psi2) = Pi2; 47 | rhs(i, j, k, Scalar::Pi1) = -Pi1*2./eta + laplacian_Psi1 - Psi1*potential; 48 | rhs(i, j, k, Scalar::Pi2) = -Pi2*2./eta + laplacian_Psi2 - Psi2*potential; 49 | } 50 | 51 | /** @brief Function that calculates the RHS of the GW EOM at a single cell. 52 | * @param rhs Container to be filled with RHS. 53 | * @param state Data from which to calculate RHS (current state). 54 | * @param i i-th cell index. 55 | * @param j j-th cell index. 56 | * @param k k-th cell index. 57 | * @param lev Current level. 58 | * @param time Current time. 59 | * @param dt Time step size. 60 | * @param dx Grid spacing. 61 | * @param params Optional parameters. 62 | */ 63 | template<> AMREX_GPU_DEVICE AMREX_FORCE_INLINE 64 | void GravitationalWavesRhs(const amrex::Array4& rhs, 65 | const amrex::Array4& state, const int i, const int j, 66 | const int k, const int lev, const double time, const double dt, 67 | const double dx, const double* params) { 68 | // Fetch field values. 69 | double du_xx = state(i, j, k, Gw::du_xx); 70 | double du_yy = state(i, j, k, Gw::du_yy); 71 | double du_zz = state(i, j, k, Gw::du_zz); 72 | double du_xy = state(i, j, k, Gw::du_xy); 73 | double du_xz = state(i, j, k, Gw::du_xz); 74 | double du_yz = state(i, j, k, Gw::du_yz); 75 | 76 | double eta = time; 77 | 78 | // Compute Laplacians. 79 | double dx2 = dx * dx; 80 | constexpr int order = 2; 81 | double laplacian_u_xx = sledgehamr::utils::Laplacian( 82 | state, i, j, k, Gw::u_xx, dx2); 83 | double laplacian_u_yy = sledgehamr::utils::Laplacian( 84 | state, i, j, k, Gw::u_yy, dx2); 85 | double laplacian_u_zz = sledgehamr::utils::Laplacian( 86 | state, i, j, k, Gw::u_zz, dx2); 87 | double laplacian_u_xy = sledgehamr::utils::Laplacian( 88 | state, i, j, k, Gw::u_xy, dx2); 89 | double laplacian_u_xz = sledgehamr::utils::Laplacian( 90 | state, i, j, k, Gw::u_xz, dx2); 91 | double laplacian_u_yz = sledgehamr::utils::Laplacian( 92 | state, i, j, k, Gw::u_yz, dx2); 93 | 94 | // Compute gradients. 95 | double grad_x_Psi1 = sledgehamr::utils::Gradient( 96 | state, i, j, k, Scalar::Psi1, dx, 'x'); 97 | double grad_y_Psi1 = sledgehamr::utils::Gradient( 98 | state, i, j, k, Scalar::Psi1, dx, 'y'); 99 | double grad_z_Psi1 = sledgehamr::utils::Gradient( 100 | state, i, j, k, Scalar::Psi1, dx, 'z'); 101 | 102 | double grad_x_Psi2 = sledgehamr::utils::Gradient( 103 | state, i, j, k, Scalar::Psi2, dx, 'x'); 104 | double grad_y_Psi2 = sledgehamr::utils::Gradient( 105 | state, i, j, k, Scalar::Psi2, dx, 'y'); 106 | double grad_z_Psi2 = sledgehamr::utils::Gradient( 107 | state, i, j, k, Scalar::Psi2, dx, 'z'); 108 | 109 | // Compute EOM. 110 | rhs(i, j, k, Gw::u_xx) = du_xx; 111 | rhs(i, j, k, Gw::u_yy) = du_yy; 112 | rhs(i, j, k, Gw::u_zz) = du_zz; 113 | rhs(i, j, k, Gw::u_xy) = du_xy; 114 | rhs(i, j, k, Gw::u_xz) = du_xz; 115 | rhs(i, j, k, Gw::u_yz) = du_yz; 116 | 117 | rhs(i, j, k, Gw::u_xx) = - du_xx*2./eta + laplacian_u_xx 118 | + grad_x_Psi1*grad_x_Psi1 119 | + grad_x_Psi2*grad_x_Psi2; 120 | rhs(i, j, k, Gw::u_yy) = - du_yy*2./eta + laplacian_u_yy 121 | + grad_y_Psi1*grad_y_Psi1 122 | + grad_y_Psi2*grad_y_Psi2; 123 | rhs(i, j, k, Gw::u_zz) = - du_zz*2./eta + laplacian_u_zz 124 | + grad_z_Psi1*grad_z_Psi1 125 | + grad_z_Psi2*grad_z_Psi2; 126 | rhs(i, j, k, Gw::u_xy) = - du_xy*2./eta + laplacian_u_xy 127 | + grad_x_Psi1*grad_y_Psi1 128 | + grad_x_Psi2*grad_y_Psi2; 129 | rhs(i, j, k, Gw::u_xz) = - du_xz*2./eta + laplacian_u_xz 130 | + grad_x_Psi1*grad_z_Psi1 131 | + grad_x_Psi2*grad_z_Psi2; 132 | rhs(i, j, k, Gw::u_yz) = - du_yz*2./eta + laplacian_u_yz 133 | + grad_y_Psi1*grad_z_Psi1 134 | + grad_y_Psi2*grad_z_Psi2; 135 | } 136 | 137 | }; // namespace AxionStrings 138 | 139 | #endif // PROJECTS_AXION_STRINGS_KERNELS_RHS_H_ 140 | -------------------------------------------------------------------------------- /projects/AxionStrings/kernels_tagging.h: -------------------------------------------------------------------------------- 1 | #ifndef PROJECTS_AXION_STRINGS_KERNELS_TAGGING_H_ 2 | #define PROJECTS_AXION_STRINGS_KERNELS_TAGGING_H_ 3 | 4 | #include "setup.h" 5 | 6 | namespace AxionStrings { 7 | 8 | /** @brief Checks for zero-crossings between two points in the complex scalar 9 | * field. 10 | * @param Psi1_1 \Psi_1 of first point. 11 | * @param Psi2_1 \Psi_2 of first point. 12 | * @param Psi1_2 \Psi_1 of second point. 13 | * @param Psi2_2 \Psi_2 of second point. 14 | * @return Sign of slope of zero-crossing. 0 if no crossing. 15 | */ 16 | AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE 17 | int ZeroXing(double Psi1_1, double Psi2_1, double Psi1_2, double Psi2_2) { 18 | if (Psi2_1 * Psi2_2 >= 0) return 0; 19 | if (Psi2_1 * Psi1_2 - Psi1_1 * Psi2_2 > 0) return 1; 20 | return -1; 21 | } 22 | 23 | /** @brief Computes the winding factor along a given axis. Will be non-zero if 24 | * plaquette is pierced by a string. 25 | * @param state Current state. 26 | * @param i i-th cell index. 27 | * @param j j-th cell index. 28 | * @param k k-th cell index. 29 | * @return Winding factor. 30 | */ 31 | AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE 32 | int WindingAxis1(const amrex::Array4& state, 33 | const int i, const int j, const int k) { 34 | return ZeroXing(state(i ,j ,k ,Scalar::Psi1), 35 | state(i ,j ,k ,Scalar::Psi2), 36 | state(i+1,j ,k ,Scalar::Psi1), 37 | state(i+1,j ,k ,Scalar::Psi2)) 38 | + ZeroXing(state(i+1,j ,k ,Scalar::Psi1), 39 | state(i+1,j ,k ,Scalar::Psi2), 40 | state(i+1,j+1,k ,Scalar::Psi1), 41 | state(i+1,j+1,k ,Scalar::Psi2)) 42 | + ZeroXing(state(i+1,j+1,k ,Scalar::Psi1), 43 | state(i+1,j+1,k ,Scalar::Psi2), 44 | state(i ,j+1,k ,Scalar::Psi1), 45 | state(i ,j+1,k ,Scalar::Psi2)) 46 | + ZeroXing(state(i ,j+1,k ,Scalar::Psi1), 47 | state(i ,j+1,k ,Scalar::Psi2), 48 | state(i ,j ,k ,Scalar::Psi1), 49 | state(i ,j ,k ,Scalar::Psi2)); 50 | } 51 | 52 | AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE 53 | int WindingAxis2(const amrex::Array4& state, 54 | const int i, const int j, const int k) { 55 | return ZeroXing(state(i ,j ,k ,Scalar::Psi1), 56 | state(i ,j ,k ,Scalar::Psi2), 57 | state(i+1,j ,k ,Scalar::Psi1), 58 | state(i+1,j ,k ,Scalar::Psi2)) 59 | + ZeroXing(state(i+1,j ,k ,Scalar::Psi1), 60 | state(i+1,j ,k ,Scalar::Psi2), 61 | state(i+1,j ,k+1,Scalar::Psi1), 62 | state(i+1,j ,k+1,Scalar::Psi2)) 63 | + ZeroXing(state(i+1,j ,k+1,Scalar::Psi1), 64 | state(i+1,j ,k+1,Scalar::Psi2), 65 | state(i ,j ,k+1,Scalar::Psi1), 66 | state(i ,j ,k+1,Scalar::Psi2)) 67 | + ZeroXing(state(i ,j ,k+1,Scalar::Psi1), 68 | state(i ,j ,k+1,Scalar::Psi2), 69 | state(i ,j ,k ,Scalar::Psi1), 70 | state(i ,j ,k ,Scalar::Psi2)); 71 | } 72 | 73 | AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE 74 | int WindingAxis3(const amrex::Array4& state, 75 | const int i, const int j, const int k) { 76 | return ZeroXing(state(i ,j ,k ,Scalar::Psi1), 77 | state(i ,j ,k ,Scalar::Psi2), 78 | state(i ,j+1,k ,Scalar::Psi1), 79 | state(i ,j+1,k ,Scalar::Psi2)) 80 | + ZeroXing(state(i ,j+1,k ,Scalar::Psi1), 81 | state(i ,j+1,k ,Scalar::Psi2), 82 | state(i ,j+1,k+1,Scalar::Psi1), 83 | state(i ,j+1,k+1,Scalar::Psi2)) 84 | + ZeroXing(state(i ,j+1,k+1,Scalar::Psi1), 85 | state(i ,j+1,k+1,Scalar::Psi2), 86 | state(i ,j ,k+1,Scalar::Psi1), 87 | state(i ,j ,k+1,Scalar::Psi2)) 88 | + ZeroXing(state(i ,j ,k+1,Scalar::Psi1), 89 | state(i ,j ,k+1,Scalar::Psi2), 90 | state(i ,j ,k ,Scalar::Psi1), 91 | state(i ,j ,k ,Scalar::Psi2)); 92 | } 93 | 94 | /** @brief Function that tags individual cells for refinement. 95 | * @return Boolean value as to whether cell should be refined or not. 96 | */ 97 | template<> AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE 98 | bool TagCellForRefinement(const amrex::Array4& state, 99 | const int i, const int j, const int k, const int lev, const double time, 100 | const double dt, const double dx, const double* params) { 101 | // Check all three plaquettes (in positive index direction) for string 102 | // piercings. 103 | if (WindingAxis1(state, i, j, k) != 0) return true; 104 | if (WindingAxis2(state, i, j, k) != 0) return true; 105 | if (WindingAxis3(state, i, j, k) != 0) return true; 106 | 107 | return false; 108 | } 109 | 110 | /** @brief Modifies the truncation error criteria for Pi1 and Pi2 from its 111 | * default \tau > \tau_{crit} to \tau * \Delta t_{\ell} > \tau_{crit}. 112 | * @param truncation_error \tau 113 | * @return f(\tau) for criteria f(\tau) > \tau_{crit}. 114 | */ 115 | #define AXION_STRING_TRUNCATION_MODIFIER(x) \ 116 | template<> AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE \ 117 | double TruncationModifier(const amrex::Array4& state, \ 118 | const int i, const int j, const int k, const int lev, \ 119 | const double time, const double dt, const double dx, \ 120 | const double truncation_error, const double* params) { \ 121 | return truncation_error * dt; \ 122 | } 123 | 124 | AXION_STRING_TRUNCATION_MODIFIER(Scalar::Pi1) 125 | AXION_STRING_TRUNCATION_MODIFIER(Scalar::Pi2) 126 | AXION_STRING_TRUNCATION_MODIFIER(Gw::du_xx) 127 | AXION_STRING_TRUNCATION_MODIFIER(Gw::du_yy) 128 | AXION_STRING_TRUNCATION_MODIFIER(Gw::du_zz) 129 | AXION_STRING_TRUNCATION_MODIFIER(Gw::du_xy) 130 | AXION_STRING_TRUNCATION_MODIFIER(Gw::du_xz) 131 | AXION_STRING_TRUNCATION_MODIFIER(Gw::du_yz) 132 | 133 | }; // namespace AxionStrings 134 | 135 | #endif // PROJECTS_AXION_STRINGS_KERNELS_TAGGING_H_ 136 | -------------------------------------------------------------------------------- /projects/AxionStrings/setup.h: -------------------------------------------------------------------------------- 1 | #ifndef PROJECTS_AXION_STRINGS_SETUP_H_ 2 | #define PROJECTS_AXION_STRINGS_SETUP_H_ 3 | 4 | namespace AxionStrings { 5 | 6 | SLEDGEHAMR_ADD_SCALARS(Psi1, Psi2) 7 | SLEDGEHAMR_ADD_CONJUGATE_MOMENTA(Pi1, Pi2) 8 | 9 | }; // namespace AxionStrings 10 | 11 | #endif // PROJECTS_AXION_STRINGS_SETUP_H_ 12 | -------------------------------------------------------------------------------- /projects/AxionStringsPostevolution/AxionStringsPostevolution.cpp: -------------------------------------------------------------------------------- 1 | #include "AxionStringsPostevolution.h" 2 | #include 3 | 4 | namespace AxionStringsPostevolution { 5 | 6 | void AxionStringsPostevolution::SetParamsRhs( 7 | std::vector& params, const double time, const int lev) { 8 | double eta = time; 9 | double eta_pre = eta_pre_0 + (eta - eta_0); 10 | double frac = 1./(1. + std::exp(-f_transition*(eta-eta_transition))); 11 | 12 | params.push_back(eta_0); 13 | params.push_back(eta_pre); 14 | params.push_back(frac); 15 | } 16 | 17 | void AxionStringsPostevolution::Init() { 18 | cosmo.Init(this); 19 | ParseConstants(); 20 | GetPreevolutionTime(); 21 | ReinterpretInitialState(); 22 | } 23 | 24 | void AxionStringsPostevolution::ParseConstants() { 25 | amrex::ParmParse pp("project"); 26 | pp.get("starting_log", log_0); 27 | pp.get("eta_transition", eta_transition); 28 | pp.get("f_transition", f_transition); 29 | 30 | eta_0 = std::sqrt(std::exp(log_0)/std::sqrt(2.)); 31 | } 32 | 33 | void AxionStringsPostevolution::ReinterpretInitialState() { 34 | // Set starting time if we are starting a new sim. 35 | if (!restart_sim) { 36 | grid_new[0].t = eta_0; 37 | } 38 | } 39 | 40 | void AxionStringsPostevolution::GetPreevolutionTime() { 41 | // Get end time of the preevolution from the initial checkpoint file. 42 | std::string folder; 43 | amrex::ParmParse pp("input"); 44 | pp.get("initial_state", folder); 45 | 46 | const int nparams = 8; 47 | double header[nparams]; 48 | std::string filename = folder + "/Meta.hdf5"; 49 | sledgehamr::utils::hdf5::Read(filename, {"Header"}, header); 50 | eta_pre_0 = header[0]; 51 | } 52 | 53 | } // namespace AxionStringsPostevolution 54 | -------------------------------------------------------------------------------- /projects/AxionStringsPostevolution/AxionStringsPostevolution.h: -------------------------------------------------------------------------------- 1 | #ifndef PROJECTS_AXION_STRINGS_POSTEVOLUTION_H_ 2 | #define PROJECTS_AXION_STRINGS_POSTEVOLUTION_H_ 3 | 4 | #include 5 | #include "../AxionStrings/cosmology.h" 6 | #include "kernels_rhs.h" 7 | 8 | namespace AxionStringsPostevolution { 9 | using AxionStrings::GravitationalWavesRhs; 10 | using AxionStrings::TruncationModifier; 11 | using AxionStrings::TagCellForRefinement; 12 | 13 | SLEDGEHAMR_FINISH_SETUP 14 | 15 | /** @brief Class to simulate axion strings. 16 | */ 17 | class AxionStringsPostevolution : public sledgehamr::Sledgehamr { 18 | public: 19 | SLEDGEHAMR_INITIALIZE_PROJECT(AxionStringsPostevolution) 20 | 21 | void SetParamsRhs(std::vector& params, const double time, 22 | const int lev) override; 23 | 24 | void Init() override; 25 | 26 | bool CreateLevelIf(const int lev, const double time) override { 27 | return cosmo.CreateLevelIf(lev, time); 28 | }; 29 | 30 | private: 31 | void ParseConstants(); 32 | void GetPreevolutionTime(); 33 | void ReinterpretInitialState(); 34 | 35 | double log_0 = 2; 36 | double eta_0 = 2.3; 37 | double eta_pre_0 = -1; 38 | double eta_transition = 2.8; 39 | double f_transition = 10; 40 | 41 | Cosmology cosmo; 42 | }; 43 | 44 | }; // namespace AxionStringsPostevolution 45 | 46 | #endif // PROJECTS_AXION_STRINGS_POSTEVOLUTION_H_ 47 | -------------------------------------------------------------------------------- /projects/AxionStringsPostevolution/kernels_rhs.h: -------------------------------------------------------------------------------- 1 | #ifndef PROJECTS_AXION_STRINGS_POSTEVOLUTION_KERNELS_RHS_H_ 2 | #define PROJECTS_AXION_STRINGS_POSTEVOLUTION_KERNELS_RHS_H_ 3 | 4 | #include 5 | #include "../AxionStrings/cosmology.h" 6 | 7 | namespace AxionStringsPostevolution { 8 | using namespace AxionStrings; 9 | 10 | /** @brief Function that calculates the RHS of the EOM at a single cell. 11 | * @param rhs Container to be filled with RHS. 12 | * @param state Data from which to calculate RHS (current state). 13 | * @param i i-th cell index. 14 | * @param j j-th cell index. 15 | * @param k k-th cell index. 16 | * @param lev Current level. 17 | * @param time Current time. 18 | * @param dt Time step size. 19 | * @param dx Grid spacing. 20 | */ 21 | AMREX_GPU_DEVICE AMREX_FORCE_INLINE 22 | void Rhs(const amrex::Array4& rhs, 23 | const amrex::Array4& state, 24 | const int i, const int j, const int k, const int lev, 25 | const double time, const double dt, const double dx, 26 | const double* params) { 27 | // Fetch field values. 28 | double Psi1 = state(i, j, k, Scalar::Psi1); 29 | double Psi2 = state(i, j, k, Scalar::Psi2); 30 | double Pi1 = state(i, j, k, Scalar::Pi1); 31 | double Pi2 = state(i, j, k, Scalar::Pi2); 32 | 33 | double eta = time; 34 | 35 | // Compute Laplacians. 36 | constexpr int order = 2; 37 | double laplacian_Psi1 = sledgehamr::utils::Laplacian( 38 | state, i, j, k, Scalar::Psi1, dx*dx); 39 | double laplacian_Psi2 = sledgehamr::utils::Laplacian( 40 | state, i, j, k, Scalar::Psi2, dx*dx); 41 | 42 | // Compute EOM. 43 | const double eta_0 = params[0]; 44 | const double eta_pre = params[1]; 45 | const double eta_sq = eta_pre*eta_pre; 46 | const double lambda = eta_0*eta_0; 47 | const double drag = eta_0; 48 | 49 | double potential_pre = lambda/eta_sq*( Psi1*Psi1 + Psi2*Psi2 - 1. ); 50 | double rhs_Psi1_pre = -Pi1*3./eta_pre + laplacian_Psi1/eta_sq/drag 51 | - Psi1*potential_pre; 52 | double rhs_Psi2_pre = -Pi2*3./eta_pre + laplacian_Psi2/eta_sq/drag 53 | - Psi2*potential_pre; 54 | 55 | double potential_post = eta*eta*( Psi1*Psi1 + Psi2*Psi2 - 1. ); 56 | double rhs_Psi1_post = -Pi1*2./eta + laplacian_Psi1 - Psi1*potential_post; 57 | double rhs_Psi2_post = -Pi2*2./eta + laplacian_Psi2 - Psi2*potential_post; 58 | 59 | double frac = params[2]; 60 | rhs(i, j, k, Scalar::Psi1) = Pi1; 61 | rhs(i, j, k, Scalar::Psi2) = Pi2; 62 | rhs(i, j, k, Scalar::Pi1) = (1.-frac)*rhs_Psi1_pre + frac*rhs_Psi1_post; 63 | rhs(i, j, k, Scalar::Pi2) = (1.-frac)*rhs_Psi2_pre + frac*rhs_Psi2_post; 64 | } 65 | 66 | }; // namespace AxionStringsPostevolution 67 | 68 | #endif // PROJECTS_AXION_STRINGS_POSTEVOLUTION_KERNELS_RHS_H_ 69 | -------------------------------------------------------------------------------- /projects/AxionStringsPreevolution/AxionStringsPreevolution.cpp: -------------------------------------------------------------------------------- 1 | #include "AxionStringsPreevolution.h" 2 | 3 | namespace AxionStringsPreevolution { 4 | 5 | void AxionStringsPreevolution::SetParamsRhs(std::vector ¶ms, 6 | const double time, const int lev) { 7 | params.push_back(eta_0); 8 | } 9 | 10 | void AxionStringsPreevolution::Init() { 11 | cosmo.Init(this); 12 | ParseConstants(); 13 | 14 | if (random_state > 0) 15 | SetRandomState(); 16 | } 17 | 18 | bool AxionStringsPreevolution::StopRunning(const double time) { 19 | double xi = cosmo.Xi(finest_level, eta_0); 20 | amrex::Print() << "String length: " << xi << ", target: " << xi_0 21 | << std::endl; 22 | return (xi <= xi_0 && time >= min_eta); 23 | } 24 | 25 | void AxionStringsPreevolution::SetRandomState() { 26 | sledgehamr::LevelData &state = GetLevelData(0); 27 | amrex::InitRandom(random_state + amrex::ParallelDescriptor::MyProc()); 28 | amrex::Print() << "Set Random Initial Conditions" << std::endl; 29 | 30 | // #pragma omp parallel 31 | for (amrex::MFIter mfi(state, true); mfi.isValid(); ++mfi) { 32 | const amrex::Array4 &state_arr = state.array(mfi); 33 | const amrex::Box &bx = mfi.tilebox(); 34 | 35 | const amrex::Dim3 lo = amrex::lbound(bx); 36 | const amrex::Dim3 hi = amrex::ubound(bx); 37 | 38 | for (int k = lo.z; k <= hi.z; ++k) { 39 | for (int j = lo.y; j <= hi.y; ++j) { 40 | AMREX_PRAGMA_SIMD 41 | for (int i = lo.x; i <= hi.x; ++i) { 42 | state_arr(i, j, k, Scalar::Psi1) = 43 | amrex::Random() * 2. - 1.; 44 | state_arr(i, j, k, Scalar::Psi2) = 45 | amrex::Random() * 2. - 1.; 46 | } 47 | } 48 | } 49 | } 50 | } 51 | 52 | void AxionStringsPreevolution::ParseConstants() { 53 | amrex::ParmParse pp("project"); 54 | pp.get("random", random_state); 55 | pp.get("starting_log", log_0); 56 | pp.get("starting_xi", xi_0); 57 | pp.get("min_eta", min_eta); 58 | eta_0 = std::sqrt(std::exp(log_0) / std::sqrt(2.)); 59 | } 60 | 61 | } // namespace AxionStringsPreevolution 62 | -------------------------------------------------------------------------------- /projects/AxionStringsPreevolution/AxionStringsPreevolution.h: -------------------------------------------------------------------------------- 1 | #ifndef PROJECTS_AXION_STRINGS_PREEVOLUTION_H_ 2 | #define PROJECTS_AXION_STRINGS_PREEVOLUTION_H_ 3 | 4 | #include "../AxionStrings/cosmology.h" 5 | #include "kernels_rhs.h" 6 | #include 7 | 8 | namespace AxionStringsPreevolution { 9 | 10 | SLEDGEHAMR_FINISH_SETUP 11 | 12 | /** @brief Class to simulate axion strings. 13 | */ 14 | class AxionStringsPreevolution : public sledgehamr::Sledgehamr { 15 | public: 16 | SLEDGEHAMR_INITIALIZE_PROJECT(AxionStringsPreevolution) 17 | 18 | void SetParamsRhs(std::vector ¶ms, const double time, 19 | const int lev) override; 20 | 21 | void Init() override; 22 | 23 | bool StopRunning(const double time) override; 24 | 25 | void SetRandomState(); 26 | 27 | private: 28 | void ParseConstants(); 29 | 30 | double log_0 = 2; 31 | double eta_0 = 2.3; 32 | double xi_0 = 0.18; 33 | double min_eta = 2; 34 | int random_state = -1; 35 | Cosmology cosmo; 36 | }; 37 | 38 | }; // namespace AxionStringsPreevolution 39 | 40 | #endif // PROJECTS_AXION_STRINGS_PREEVOLUTION_H_ 41 | -------------------------------------------------------------------------------- /projects/AxionStringsPreevolution/kernels_rhs.h: -------------------------------------------------------------------------------- 1 | #ifndef PROJECTS_AXION_STRINGS_PREEVOLUTION_KERNELS_RHS_H_ 2 | #define PROJECTS_AXION_STRINGS_PREEVOLUTION_KERNELS_RHS_H_ 3 | 4 | #include "../AxionStrings/cosmology.h" 5 | #include 6 | 7 | namespace AxionStringsPreevolution { 8 | using namespace AxionStrings; 9 | 10 | /** @brief Function that calculates the RHS of the EOM at a single cell. 11 | * @param rhs Container to be filled with RHS. 12 | * @param state Data from which to calculate RHS (current state). 13 | * @param i i-th cell index. 14 | * @param j j-th cell index. 15 | * @param k k-th cell index. 16 | * @param lev Current level. 17 | * @param time Current time. 18 | * @param dt Time step size. 19 | * @param dx Grid spacing. 20 | */ 21 | AMREX_GPU_DEVICE AMREX_FORCE_INLINE void 22 | Rhs(const amrex::Array4 &rhs, const amrex::Array4 &state, 23 | const int i, const int j, const int k, const int lev, const double time, 24 | const double dt, const double dx, const double *params) { 25 | // Fetch field values. 26 | double Psi1 = state(i, j, k, Scalar::Psi1); 27 | double Psi2 = state(i, j, k, Scalar::Psi2); 28 | double Pi1 = state(i, j, k, Scalar::Pi1); 29 | double Pi2 = state(i, j, k, Scalar::Pi2); 30 | 31 | double eta = time; 32 | 33 | // Compute Laplacians. 34 | constexpr int order = 2; 35 | double laplacian_Psi1 = sledgehamr::utils::Laplacian( 36 | state, i, j, k, Scalar::Psi1, dx * dx); 37 | double laplacian_Psi2 = sledgehamr::utils::Laplacian( 38 | state, i, j, k, Scalar::Psi2, dx * dx); 39 | 40 | // Compute EOM. 41 | const double eta_0 = params[0]; 42 | const double eta_sq = eta * eta; 43 | const double lambda = eta_0 * eta_0; 44 | const double drag = std::sqrt(eta_0); 45 | // const double drag = eta_0; 46 | 47 | double potential = lambda / eta_sq * (Psi1 * Psi1 + Psi2 * Psi2 - 1.); 48 | 49 | rhs(i, j, k, Scalar::Psi1) = Pi1; 50 | rhs(i, j, k, Scalar::Psi2) = Pi2; 51 | rhs(i, j, k, Scalar::Pi1) = 52 | -Pi1 * 3. / eta + laplacian_Psi1 / eta_sq / drag - Psi1 * potential; 53 | rhs(i, j, k, Scalar::Pi2) = 54 | -Pi2 * 3. / eta + laplacian_Psi2 / eta_sq / drag - Psi2 * potential; 55 | } 56 | 57 | }; // namespace AxionStringsPreevolution 58 | 59 | #endif // PROJECTS_AXION_STRINGS_PREEVOLUTION_KERNELS_RHS_H_ 60 | -------------------------------------------------------------------------------- /projects/FirstOrderPhaseTransition/FirstOrderPhaseTransition.h: -------------------------------------------------------------------------------- 1 | #ifndef PROJECTS_FIRST_ORDER_PHASE_TRANSITION_H_ 2 | #define PROJECTS_FIRST_ORDER_PHASE_TRANSITION_H_ 3 | 4 | #include 5 | #include 6 | 7 | #include "bubbles.h" 8 | #include "kernels_misc.h" 9 | #include "kernels_rhs.h" 10 | #include "kernels_tagging.h" 11 | #include "setup.h" 12 | 13 | namespace FirstOrderPhaseTransition { 14 | 15 | SLEDGEHAMR_FINISH_SETUP 16 | 17 | /** @brief Class to simulate a first order phase transition. 18 | */ 19 | class FirstOrderPhaseTransition : public sledgehamr::Sledgehamr { 20 | public: 21 | SLEDGEHAMR_INITIALIZE_PROJECT(FirstOrderPhaseTransition) 22 | 23 | void Init() override; 24 | void SetParamsRhs(std::vector ¶ms, const double time, 25 | const int lev) override; 26 | void SetParamsGravitationalWaveRhs(std::vector ¶ms, 27 | const double time, 28 | const int lev) override; 29 | void SetParamsTruncationModifier(std::vector ¶ms, 30 | const double time, const int lev) override; 31 | void BeforeTimestep(const double time) override; 32 | 33 | private: 34 | void ParseVariables(); 35 | void ParseBubbles(); 36 | void ComputeParameters(); 37 | void SetProjections(); 38 | void AddSpectrumModification(); 39 | 40 | std::vector FindBubbles(const double time); 41 | void InjectBubbles(const double time); 42 | void InjectBubbleLevels(std::vector ab); 43 | void FillBubbleLayout(const int lev, std::vector ab); 44 | void AddBubbleValues(std::vector ab); 45 | 46 | bool GwSpectrum_UtimesK(double time, std::string prefix); 47 | bool GwSpectrum_2BubblesFrom1(double time, std::string prefix); 48 | 49 | double lambda_bar; 50 | double quadratic, cubic, quartic; 51 | double tc = -1; 52 | double t0 = -1; 53 | 54 | std::vector bubbles; 55 | int next_bubble = 0; 56 | int idx_perfmon_add_bubbles; 57 | std::vector bubbles_to_inject; 58 | 59 | std::vector field_maxima; 60 | amrex::Vector comp_vector; 61 | double maxima_time = -DBL_MAX; 62 | }; 63 | 64 | }; // namespace FirstOrderPhaseTransition 65 | 66 | #endif // PROJECTS_FIRST_ORDER_PHASE_TRANSITION_H_ 67 | -------------------------------------------------------------------------------- /projects/FirstOrderPhaseTransition/bubbles.h: -------------------------------------------------------------------------------- 1 | #ifndef PROJECTS_FIRST_ORDER_PHASE_TRANSITION_BUBBLE_H_ 2 | #define PROJECTS_FIRST_ORDER_PHASE_TRANSITION_BUBBLE_H_ 3 | 4 | namespace FirstOrderPhaseTransition { 5 | 6 | class Bubble { 7 | public: 8 | double GetPos(const double D) const { 9 | return D >= p_bubble->L ? -1. : D * p_bubble->inv_dx; 10 | } 11 | 12 | int GetFinestLevel() const { 13 | return p_bubble->finest_level; 14 | } 15 | 16 | double GetVal(const int n, const int ind, const double frac) const { 17 | return p_bubble->profile[n][ind] * (1.-frac) 18 | + p_bubble->profile[n][ind+1] * frac; 19 | } 20 | 21 | double GetLevel(const int ind) const { 22 | return p_bubble->level[ind]; 23 | } 24 | 25 | int GetNBins() const { 26 | return p_bubble->profile.size(); 27 | } 28 | 29 | bool operator<(const Bubble& other) const { 30 | return t < other.t; 31 | } 32 | 33 | double x, y, z; // bubble location 34 | double t; // injection time 35 | std::vector> profile; // bubble profile 36 | std::vector level; // bubble level 37 | double inv_dx; // inverse dx of profile binning 38 | double L; // profile cut-off (assume profile=0 beyond that) 39 | int finest_level; // finest level at which the profile is to be injected 40 | Bubble* p_bubble; // pointer to bubble which has the correct profile (can be this) 41 | }; 42 | 43 | } 44 | 45 | #endif // PROJECTS_FIRST_ORDER_PHASE_TRANSITION_BUBBLE_H_ 46 | -------------------------------------------------------------------------------- /projects/FirstOrderPhaseTransition/kernels_misc.h: -------------------------------------------------------------------------------- 1 | #ifndef PROJECTS_FIRST_ORDER_PHASE_TRANSITION_KERNELS_MISC_H_ 2 | #define PROJECTS_FIRST_ORDER_PHASE_TRANSITION_KERNELS_MISC_H_ 3 | 4 | #include "setup.h" 5 | #include "bubbles.h" 6 | 7 | namespace FirstOrderPhaseTransition { 8 | 9 | /** @brief TODO 10 | */ 11 | AMREX_FORCE_INLINE 12 | double dPhi2(amrex::Array4 const& state, const int i, 13 | const int j, const int k, const int lev, const double time, 14 | const double dt, const double dx, 15 | const std::vector& params) { 16 | double dPhi = state(i, j, k, Scalar::dPhi); 17 | return dPhi * dPhi; 18 | } 19 | 20 | AMREX_FORCE_INLINE 21 | double Distance(const double a, const double b, const double L) { 22 | const double dist = fabs(a-b); 23 | return dist < L/2. ? dist : L - dist; 24 | } 25 | 26 | AMREX_FORCE_INLINE 27 | void AddBubble(const int i, const int j, const int k, const double dx, 28 | const double L, amrex::Array4 const& fab, 29 | const Bubble& bubble) { 30 | const double Dx = Distance(i*dx,bubble.x, L); 31 | const double Dy = Distance(j*dx,bubble.y, L); 32 | const double Dz = Distance(k*dx,bubble.z, L); 33 | 34 | double D = std::sqrt(Dx*Dx + Dy*Dy + Dz*Dz); 35 | double pos = bubble.GetPos(D); 36 | if( pos == -1 ) { 37 | return; 38 | } 39 | int ind = pos; 40 | double frac = pos - (double)ind; 41 | 42 | for (int n = 0; n < fab.nComp(); ++n) { 43 | int n0 = n; 44 | if (n == Scalar::Phi) n0 = 0; 45 | else if (n == Scalar::dPhi) n0 = 1; 46 | else continue; 47 | 48 | double val = bubble.GetVal(n0, ind, frac); 49 | fab(i, j, k, n) += val; 50 | } 51 | } 52 | 53 | }; // namespace FirstOrderPhaseTransition 54 | 55 | #endif // PROJECTS_FIRST_ORDER_PHASE_TRANSITION_KERNELS_MISC_H_ 56 | -------------------------------------------------------------------------------- /projects/FirstOrderPhaseTransition/kernels_rhs.h: -------------------------------------------------------------------------------- 1 | #ifndef PROJECTS_FIRST_ORDER_PHASE_TRANSITION_KERNELS_RHS_H_ 2 | #define PROJECTS_FIRST_ORDER_PHASE_TRANSITION_KERNELS_RHS_H_ 3 | 4 | #include "setup.h" 5 | #include 6 | 7 | namespace FirstOrderPhaseTransition { 8 | 9 | /** @brief Function that calculates the RHS of the EOM at a single cell. 10 | * @param rhs Container to be filled with RHS. 11 | * @param state Data from which to calculate RHS (current state). 12 | * @param i i-th cell index. 13 | * @param j j-th cell index. 14 | * @param k k-th cell index. 15 | * @param lev Current level. 16 | * @param time Current time. 17 | * @param dt Time step size. 18 | * @param dx Grid spacing. 19 | */ 20 | AMREX_GPU_DEVICE AMREX_FORCE_INLINE void 21 | Rhs(const amrex::Array4 &rhs, const amrex::Array4 &state, 22 | const int i, const int j, const int k, const int lev, const double time, 23 | const double dt, const double dx, const double *params) { 24 | double quadratic = params[0]; 25 | double cubic = params[1]; 26 | double quartic = params[2]; 27 | double Phi = state(i, j, k, Scalar::Phi); 28 | double potential = 29 | quadratic * Phi + cubic * Phi * Phi + quartic * Phi * Phi * Phi; 30 | 31 | constexpr int order = 2; 32 | double laplacian_Phi = sledgehamr::utils::Laplacian( 33 | state, i, j, k, Scalar::Phi, dx * dx); 34 | 35 | rhs(i, j, k, Scalar::Phi) = state(i, j, k, Scalar::dPhi); 36 | rhs(i, j, k, Scalar::dPhi) = laplacian_Phi + potential; 37 | } 38 | 39 | template <> 40 | AMREX_GPU_DEVICE AMREX_FORCE_INLINE void GravitationalWavesRhs( 41 | const amrex::Array4 &rhs, const amrex::Array4 &state, 42 | const int i, const int j, const int k, const int lev, const double time, 43 | const double dt, const double dx, const double *params) { 44 | double tc = params[0]; 45 | double t0 = params[1]; 46 | 47 | // Compute Laplacians. 48 | double dx2 = dx * dx; 49 | constexpr int order = 2; 50 | double laplacian_u_xx = 51 | sledgehamr::utils::Laplacian(state, i, j, k, Gw::u_xx, dx2); 52 | double laplacian_u_yy = 53 | sledgehamr::utils::Laplacian(state, i, j, k, Gw::u_yy, dx2); 54 | double laplacian_u_zz = 55 | sledgehamr::utils::Laplacian(state, i, j, k, Gw::u_zz, dx2); 56 | double laplacian_u_xy = 57 | sledgehamr::utils::Laplacian(state, i, j, k, Gw::u_xy, dx2); 58 | double laplacian_u_xz = 59 | sledgehamr::utils::Laplacian(state, i, j, k, Gw::u_xz, dx2); 60 | double laplacian_u_yz = 61 | sledgehamr::utils::Laplacian(state, i, j, k, Gw::u_yz, dx2); 62 | 63 | // Compute gradients. 64 | constexpr int order2 = 2; 65 | double grad_x_Phi = sledgehamr::utils::Gradient( 66 | state, i, j, k, Scalar::Phi, dx, 'x'); 67 | double grad_y_Phi = sledgehamr::utils::Gradient( 68 | state, i, j, k, Scalar::Phi, dx, 'y'); 69 | double grad_z_Phi = sledgehamr::utils::Gradient( 70 | state, i, j, k, Scalar::Phi, dx, 'z'); 71 | 72 | double C = 1; 73 | if (time > tc && tc >= 0) { 74 | C = std::exp(-(time - tc) * (time - tc) / (t0 * t0)); 75 | } 76 | 77 | // Compute EOM. 78 | rhs(i, j, k, Gw::u_xx) = state(i, j, k, Gw::du_xx); 79 | rhs(i, j, k, Gw::u_yy) = state(i, j, k, Gw::du_yy); 80 | rhs(i, j, k, Gw::u_zz) = state(i, j, k, Gw::du_zz); 81 | rhs(i, j, k, Gw::u_xy) = state(i, j, k, Gw::du_xy); 82 | rhs(i, j, k, Gw::u_xz) = state(i, j, k, Gw::du_xz); 83 | rhs(i, j, k, Gw::u_yz) = state(i, j, k, Gw::du_yz); 84 | rhs(i, j, k, Gw::du_xx) = (laplacian_u_xx + grad_x_Phi * grad_x_Phi) * C; 85 | rhs(i, j, k, Gw::du_yy) = (laplacian_u_yy + grad_y_Phi * grad_y_Phi) * C; 86 | rhs(i, j, k, Gw::du_zz) = (laplacian_u_zz + grad_z_Phi * grad_z_Phi) * C; 87 | rhs(i, j, k, Gw::du_xy) = (laplacian_u_xy + grad_x_Phi * grad_y_Phi) * C; 88 | rhs(i, j, k, Gw::du_xz) = (laplacian_u_xz + grad_x_Phi * grad_z_Phi) * C; 89 | rhs(i, j, k, Gw::du_yz) = (laplacian_u_yz + grad_y_Phi * grad_z_Phi) * C; 90 | } 91 | 92 | }; // namespace FirstOrderPhaseTransition 93 | 94 | #endif // PROJECTS_FIRST_ORDER_PHASE_TRANSITION_KERNELS_RHS_H_ 95 | -------------------------------------------------------------------------------- /projects/FirstOrderPhaseTransition/kernels_tagging.h: -------------------------------------------------------------------------------- 1 | #ifndef PROJECTS_FIRST_ORDER_PHASE_TRANSITION_KERNELS_TAGGING_H_ 2 | #define PROJECTS_FIRST_ORDER_PHASE_TRANSITION_KERNELS_TAGGING_H_ 3 | 4 | #include "setup.h" 5 | 6 | namespace FirstOrderPhaseTransition { 7 | 8 | /** @brief Modifies the truncation error criteria for Pi1 and Pi2 from its 9 | * default \tau > \tau_{crit} to \tau * \Delta t_{\ell} > \tau_{crit}. 10 | * @param truncation_error \tau 11 | * @return f(\tau) for criteria f(\tau) > \tau_{crit}. 12 | */ 13 | #define FIRST_ORDER_PHASE_TRANSITION_TRUNCATION_MODIFIER(x) \ 14 | template<> AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE \ 15 | double TruncationModifier(const amrex::Array4& state, \ 16 | const int i, const int j, const int k, const int lev, \ 17 | const double time, const double dt, const double dx, \ 18 | const double truncation_error, const double* params) { \ 19 | return truncation_error / params[x]; \ 20 | } 21 | 22 | #define FIRST_ORDER_PHASE_TRANSITION_TRUNCATION_MODIFIER_DT(x) \ 23 | template<> AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE \ 24 | double TruncationModifier(const amrex::Array4& state, \ 25 | const int i, const int j, const int k, const int lev, \ 26 | const double time, const double dt, const double dx, \ 27 | const double truncation_error, const double* params) { \ 28 | return truncation_error * dt / params[x]; \ 29 | } 30 | 31 | FIRST_ORDER_PHASE_TRANSITION_TRUNCATION_MODIFIER(Scalar::Phi) 32 | FIRST_ORDER_PHASE_TRANSITION_TRUNCATION_MODIFIER(Gw::u_xx) 33 | FIRST_ORDER_PHASE_TRANSITION_TRUNCATION_MODIFIER(Gw::u_yy) 34 | FIRST_ORDER_PHASE_TRANSITION_TRUNCATION_MODIFIER(Gw::u_zz) 35 | FIRST_ORDER_PHASE_TRANSITION_TRUNCATION_MODIFIER(Gw::u_xy) 36 | FIRST_ORDER_PHASE_TRANSITION_TRUNCATION_MODIFIER(Gw::u_xz) 37 | FIRST_ORDER_PHASE_TRANSITION_TRUNCATION_MODIFIER(Gw::u_yz) 38 | 39 | FIRST_ORDER_PHASE_TRANSITION_TRUNCATION_MODIFIER_DT(Scalar::dPhi) 40 | FIRST_ORDER_PHASE_TRANSITION_TRUNCATION_MODIFIER_DT(Gw::du_xx) 41 | FIRST_ORDER_PHASE_TRANSITION_TRUNCATION_MODIFIER_DT(Gw::du_yy) 42 | FIRST_ORDER_PHASE_TRANSITION_TRUNCATION_MODIFIER_DT(Gw::du_zz) 43 | FIRST_ORDER_PHASE_TRANSITION_TRUNCATION_MODIFIER_DT(Gw::du_xy) 44 | FIRST_ORDER_PHASE_TRANSITION_TRUNCATION_MODIFIER_DT(Gw::du_xz) 45 | FIRST_ORDER_PHASE_TRANSITION_TRUNCATION_MODIFIER_DT(Gw::du_yz) 46 | 47 | }; // namespace FirstOrderPhaseTransition 48 | 49 | #endif // PROJECTS_FIRST_ORDER_PHASE_TRANSITION_KERNELS_TAGGING_H_ 50 | -------------------------------------------------------------------------------- /projects/FirstOrderPhaseTransition/setup.h: -------------------------------------------------------------------------------- 1 | #ifndef PROJECTS_FIRST_ORDER_PHASE_TRANSITION_SETUP_H_ 2 | #define PROJECTS_FIRST_ORDER_PHASE_TRANSITION_SETUP_H_ 3 | 4 | #include 5 | 6 | namespace FirstOrderPhaseTransition { 7 | 8 | SLEDGEHAMR_ADD_SCALARS(Phi) 9 | SLEDGEHAMR_ADD_CONJUGATE_MOMENTA(dPhi) 10 | 11 | }; // namespace FirstOrderPhaseTransition 12 | 13 | #endif // PROJECTS_FIRST_ORDER_PHASE_TRANSITION_SETUP_H_ 14 | -------------------------------------------------------------------------------- /projects/FirstOrderPhaseTransition/spectrum_modifier.h: -------------------------------------------------------------------------------- 1 | #ifndef PROJECTS_FIRST_ORDER_PHASE_TRANSITION_SPECTRUM_MODIFIER_H_ 2 | #define PROJECTS_FIRST_ORDER_PHASE_TRANSITION_SPECTRUM_MODIFIER_H_ 3 | 4 | #include 5 | 6 | namespace FirstOrderPhaseTransition { 7 | 8 | struct SpectrumModifier_UtimesK 9 | : sledgehamr::GravitationalWavesSpectrumModifier { 10 | virtual void SelectComponents(int components[6]) { 11 | components[0] = sledgehamr::GravitationalWaves::Gw::u_xx; 12 | components[1] = sledgehamr::GravitationalWaves::Gw::u_xy; 13 | components[2] = sledgehamr::GravitationalWaves::Gw::u_xz; 14 | components[3] = sledgehamr::GravitationalWaves::Gw::u_yy; 15 | components[4] = sledgehamr::GravitationalWaves::Gw::u_yz; 16 | components[5] = sledgehamr::GravitationalWaves::Gw::u_zz; 17 | }; 18 | 19 | virtual void FourierSpaceModifications( 20 | amrex::MultiFab du_real[6], amrex::MultiFab du_imag[6], 21 | const double dk, const int dimN) { 22 | 23 | #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) 24 | for (amrex::MFIter mfi(du_real[0], true); mfi.isValid(); ++mfi) { 25 | const amrex::Box& bx = mfi.tilebox(); 26 | amrex::Array4 const& dr0 = du_real[0].array(mfi); 27 | amrex::Array4 const& dr1 = du_real[1].array(mfi); 28 | amrex::Array4 const& dr2 = du_real[2].array(mfi); 29 | amrex::Array4 const& dr3 = du_real[3].array(mfi); 30 | amrex::Array4 const& dr4 = du_real[4].array(mfi); 31 | amrex::Array4 const& dr5 = du_real[5].array(mfi); 32 | amrex::Array4 const& di0 = du_imag[0].array(mfi); 33 | amrex::Array4 const& di1 = du_imag[1].array(mfi); 34 | amrex::Array4 const& di2 = du_imag[2].array(mfi); 35 | amrex::Array4 const& di3 = du_imag[3].array(mfi); 36 | amrex::Array4 const& di4 = du_imag[4].array(mfi); 37 | amrex::Array4 const& di5 = du_imag[5].array(mfi); 38 | const amrex::Array4* du_real_arr[6] = {&dr0, &dr1, &dr2, 39 | &dr3, &dr4, &dr5}; 40 | const amrex::Array4* du_imag_arr[6] = {&di0, &di1, &di2, 41 | &di3, &di4, &di5}; 42 | 43 | const amrex::Dim3 lo = amrex::lbound(bx); 44 | const amrex::Dim3 hi = amrex::ubound(bx); 45 | 46 | for (int c = lo.z; c <= hi.z; ++c) { 47 | for (int b = lo.y; b <= hi.y; ++b) { 48 | AMREX_PRAGMA_SIMD 49 | for (int a = lo.x; a <= hi.x; ++a) { 50 | int kx = a >= dimN/2 ? a-dimN : a; 51 | int ky = b >= dimN/2 ? b-dimN : b; 52 | int kz = c >= dimN/2 ? c-dimN : c; 53 | double k = std::sqrt(kx*kx + ky*ky + kz*kz) * dk; 54 | 55 | for (int i = 0; i < 6; ++i) { 56 | (*du_real_arr)[i](a, b, c) *= k; 57 | (*du_imag_arr)[i](a, b, c) *= k; 58 | } 59 | } 60 | } 61 | } 62 | } 63 | }; 64 | }; 65 | 66 | struct SpectrumModifier_2BubblesFrom1 67 | : sledgehamr::GravitationalWavesSpectrumModifier { 68 | SpectrumModifier_2BubblesFrom1(const double distance[3]) { 69 | std::memcpy(d, distance, sizeof(d)); 70 | } 71 | 72 | virtual void FourierSpaceModifications( 73 | amrex::MultiFab du_real[6], amrex::MultiFab du_imag[6], 74 | const double dk, const int dimN) { 75 | 76 | #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) 77 | for (amrex::MFIter mfi(du_real[0], true); mfi.isValid(); ++mfi) { 78 | const amrex::Box& bx = mfi.tilebox(); 79 | amrex::Array4 const& dr0 = du_real[0].array(mfi); 80 | amrex::Array4 const& dr1 = du_real[1].array(mfi); 81 | amrex::Array4 const& dr2 = du_real[2].array(mfi); 82 | amrex::Array4 const& dr3 = du_real[3].array(mfi); 83 | amrex::Array4 const& dr4 = du_real[4].array(mfi); 84 | amrex::Array4 const& dr5 = du_real[5].array(mfi); 85 | amrex::Array4 const& di0 = du_imag[0].array(mfi); 86 | amrex::Array4 const& di1 = du_imag[1].array(mfi); 87 | amrex::Array4 const& di2 = du_imag[2].array(mfi); 88 | amrex::Array4 const& di3 = du_imag[3].array(mfi); 89 | amrex::Array4 const& di4 = du_imag[4].array(mfi); 90 | amrex::Array4 const& di5 = du_imag[5].array(mfi); 91 | const amrex::Array4* du_real_arr[6] = {&dr0, &dr1, &dr2, 92 | &dr3, &dr4, &dr5}; 93 | const amrex::Array4* du_imag_arr[6] = {&di0, &di1, &di2, 94 | &di3, &di4, &di5}; 95 | 96 | const amrex::Dim3 lo = amrex::lbound(bx); 97 | const amrex::Dim3 hi = amrex::ubound(bx); 98 | 99 | for (int c = lo.z; c <= hi.z; ++c) { 100 | for (int b = lo.y; b <= hi.y; ++b) { 101 | AMREX_PRAGMA_SIMD 102 | for (int a = lo.x; a <= hi.x; ++a) { 103 | int kx = a >= dimN/2 ? a-dimN : a; 104 | int ky = b >= dimN/2 ? b-dimN : b; 105 | int kz = c >= dimN/2 ? c-dimN : c; 106 | double kd = (kx*d[0] + ky*d[1] + kz*d[2]) * dk; 107 | double sx = std::sin(kd); 108 | double cx = std::cos(kd); 109 | 110 | for (int i = 0; i < 6; ++i) { 111 | double dur = (*du_real_arr)[i](a, b, c); 112 | double dui = (*du_imag_arr)[i](a, b, c); 113 | (*du_real_arr)[i](a,b,c) = dur + (cx*dur - sx*dui); 114 | (*du_imag_arr)[i](a,b,c) = dui + (sx*dur + cx*dui); 115 | } 116 | } 117 | } 118 | } 119 | } 120 | }; 121 | 122 | double d[3] = {0,0,0}; 123 | }; 124 | 125 | }; 126 | 127 | #endif // PROJECTS_FIRST_ORDER_PHASE_TRANSITION_SPECTRUM_MODIFIER_H_ 128 | -------------------------------------------------------------------------------- /projects/MinimalExample/MinimalExample.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | namespace MinimalExample { 6 | 7 | SLEDGEHAMR_ADD_SCALARS(Psi1, Psi2) 8 | SLEDGEHAMR_ADD_CONJUGATE_MOMENTA(Pi1, Pi2) 9 | 10 | AMREX_GPU_DEVICE AMREX_FORCE_INLINE 11 | void Rhs(const amrex::Array4& rhs, 12 | const amrex::Array4& state, 13 | const int i, const int j, const int k, const int lev, 14 | const double time, const double dt, const double dx, 15 | const double* params) { 16 | // Fetch field values. 17 | double Psi1 = state(i, j, k, Scalar::Psi1); 18 | double Psi2 = state(i, j, k, Scalar::Psi2); 19 | double Pi1 = state(i, j, k, Scalar::Pi1); 20 | double Pi2 = state(i, j, k, Scalar::Pi2); 21 | 22 | double eta = time; 23 | 24 | // Compute Laplacians. 25 | constexpr int order = 2; 26 | double laplacian_Psi1 = sledgehamr::utils::Laplacian( 27 | state, i, j, k, Scalar::Psi1, dx*dx); 28 | double laplacian_Psi2 = sledgehamr::utils::Laplacian( 29 | state, i, j, k, Scalar::Psi2, dx*dx); 30 | 31 | // Compute potential. 32 | double potential = eta*eta*( Psi1*Psi1 + Psi2*Psi2 - 1. ) + 0.56233; 33 | 34 | // Full EOM. 35 | rhs(i, j, k, Scalar::Psi1) = Pi1; 36 | rhs(i, j, k, Scalar::Psi2) = Pi2; 37 | rhs(i, j, k, Scalar::Pi1) = -Pi1*2./eta + laplacian_Psi1 - Psi1*potential; 38 | rhs(i, j, k, Scalar::Pi2) = -Pi2*2./eta + laplacian_Psi2 - Psi2*potential; 39 | } 40 | 41 | SLEDGEHAMR_FINISH_SETUP 42 | 43 | class MinimalExample : public sledgehamr::Sledgehamr { 44 | public: 45 | SLEDGEHAMR_INITIALIZE_PROJECT(MinimalExample) 46 | }; 47 | 48 | }; // namespace MinimalExample 49 | -------------------------------------------------------------------------------- /projects/NextToMinimalExample/NextToMinimalExample.cpp: -------------------------------------------------------------------------------- 1 | #include "NextToMinimalExample.h" 2 | #include 3 | 4 | namespace NextToMinimalExample { 5 | 6 | /** @brief This function computes and write the average value of the radial mode 7 | * r = sqrt(Psi1**2 + Psi2**2). As the field r is becoming dynamical it 8 | * will start to oscillate around its vev ~1. 9 | */ 10 | bool NextToMinimalExample::WriteAvg(double time, std::string prefix) { 11 | // Grab current coarse level. 12 | const int lev = 0; 13 | const sledgehamr::LevelData& state = GetLevelData(lev); 14 | 15 | // Create a new state r and compute r = sqrt(Psi1**2 + Psi2**2). 16 | sledgehamr::LevelData r(state.boxArray(), state.DistributionMap(), 1, 0, 0); 17 | 18 | // Loop over all boxes. 19 | #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) 20 | for (amrex::MFIter mfi(r, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) { 21 | const amrex::Box& bx = mfi.tilebox(); 22 | const auto& state_arr = state.array(mfi); 23 | const auto& r_arr = r.array(mfi); 24 | 25 | // Loop over all cells within box. 26 | amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) 27 | noexcept { 28 | double Psi1 = state_arr(i, j, k, Scalar::Psi1); 29 | double Psi2 = state_arr(i, j, k, Scalar::Psi2); 30 | r_arr(i, j, k, 0) = std::sqrt(Psi1*Psi1 + Psi2*Psi2); 31 | }); 32 | } 33 | 34 | // Compute average value: 35 | double avg = r.sum() / r.boxArray().d_numPts(); 36 | 37 | // Write information to file. 38 | constexpr int nsize = 4; 39 | double data[nsize] = {time, avg}; 40 | if (amrex::ParallelDescriptor::IOProcessor()) { 41 | std::string filename = prefix + "/vev.h5"; 42 | hid_t file_id = H5Fcreate(filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, 43 | H5P_DEFAULT); 44 | sledgehamr::utils::hdf5::Write(file_id, "data", data, nsize); 45 | H5Fclose(file_id); 46 | } 47 | 48 | return true; 49 | } 50 | 51 | }; // namespace NextToMinimalExample 52 | -------------------------------------------------------------------------------- /projects/NextToMinimalExample/NextToMinimalExample.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | namespace NextToMinimalExample { 6 | 7 | SLEDGEHAMR_ADD_SCALARS(Psi1, Psi2) 8 | SLEDGEHAMR_ADD_CONJUGATE_MOMENTA(Pi1, Pi2) 9 | 10 | AMREX_GPU_DEVICE AMREX_FORCE_INLINE 11 | void Rhs(const amrex::Array4& rhs, 12 | const amrex::Array4& state, 13 | const int i, const int j, const int k, const int lev, 14 | const double time, const double dt, const double dx, 15 | const double* params) { 16 | // Retrieve parameter. 17 | double lambda = params[0]; 18 | 19 | // Fetch field values. 20 | double Psi1 = state(i, j, k, Scalar::Psi1); 21 | double Psi2 = state(i, j, k, Scalar::Psi2); 22 | double Pi1 = state(i, j, k, Scalar::Pi1); 23 | double Pi2 = state(i, j, k, Scalar::Pi2); 24 | 25 | double eta = time; 26 | 27 | // Compute Laplacians. 28 | constexpr int order = 2; 29 | double laplacian_Psi1 = sledgehamr::utils::Laplacian( 30 | state, i, j, k, Scalar::Psi1, dx*dx); 31 | double laplacian_Psi2 = sledgehamr::utils::Laplacian( 32 | state, i, j, k, Scalar::Psi2, dx*dx); 33 | 34 | // Compute potential. 35 | double potential = eta*eta*( Psi1*Psi1 + Psi2*Psi2 - 1. ) + 0.56233; 36 | potential *= lambda; 37 | 38 | // Full EOM. 39 | rhs(i, j, k, Scalar::Psi1) = Pi1; 40 | rhs(i, j, k, Scalar::Psi2) = Pi2; 41 | rhs(i, j, k, Scalar::Pi1) = -Pi1*2./eta + laplacian_Psi1 - Psi1*potential; 42 | rhs(i, j, k, Scalar::Pi2) = -Pi2*2./eta + laplacian_Psi2 - Psi2*potential; 43 | } 44 | 45 | 46 | // Extra kernel function to compute \dot{a}^2 = (Psi1*Pi2 - Psi2-Pi1)^2 / r^4 47 | // with r^2 = Psi1*Psi2 + Psi2*Psi2 48 | AMREX_FORCE_INLINE 49 | double a_dot_sq( 50 | amrex::Array4 const& state, 51 | const int i, const int j, const int k, 52 | const int lev, const double time, const double dt, const double dx, 53 | const std::vector& params) { 54 | double Psi1 = state(i, j, k, Scalar::Psi1); 55 | double Psi2 = state(i, j, k, Scalar::Psi2); 56 | double Pi1 = state(i, j, k, Scalar::Pi1); 57 | double Pi2 = state(i, j, k, Scalar::Pi2); 58 | double r_sq = Psi1*Psi1 + Psi2*Psi2; 59 | double dot_a = (Psi1*Pi2 - Psi2*Pi1)/r_sq; 60 | return dot_a*dot_a; 61 | } 62 | 63 | SLEDGEHAMR_FINISH_SETUP 64 | 65 | class NextToMinimalExample : public sledgehamr::Sledgehamr { 66 | public: 67 | SLEDGEHAMR_INITIALIZE_PROJECT(NextToMinimalExample) 68 | 69 | void Init() override { 70 | // Read in parameter from inputs file 71 | amrex::ParmParse pp("project"); 72 | pp.get("lambda", lambda); 73 | 74 | // Let us output a projection of \dot{a}^2, the power spectrum of it, 75 | // and also compute the average of the radial mode sqrt(Psi1^2 + Psi2^2) 76 | io_module->projections.emplace_back(a_dot_sq, "a_dot_sq"); 77 | io_module->spectra.emplace_back(a_dot_sq, "a_dot_sq"); 78 | io_module->output.emplace_back( 79 | "avg", OUTPUT_FCT(NextToMinimalExample::WriteAvg)); 80 | } 81 | 82 | void SetParamsRhs(std::vector& params, const double time, 83 | const int lev) override { 84 | params.push_back(lambda); 85 | } 86 | 87 | bool WriteAvg(double time, std::string prefix); 88 | 89 | private: 90 | double lambda; 91 | }; 92 | 93 | }; // namespace NextToMinimalExample 94 | -------------------------------------------------------------------------------- /pySledgehamr/Sledgehamr.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import h5py 3 | 4 | ## Generates the unique k values (combinations of i^2 + j^2 + k^2). 5 | # @param dim Maximum value for i-1, j-1, k-1. 6 | # @return Unique values. 7 | def CreateUniqueVals(dim): 8 | kVals = np.array(np.fft.fftfreq(dim)*dim, dtype = 'int') 9 | unique_mags = np.unique(np.unique(np.unique(kVals**2)[:, None]\ 10 | + np.unique(kVals**2)[None, :])[:, None]\ 11 | + np.unique(kVals[None, :]**2)[None, :]) 12 | return unique_mags 13 | 14 | ## Compute the effective momentum k from an index for different projection types. 15 | # @param a Index. 16 | # @param N Maximum index. 17 | # @param projection_type Projection type. 18 | def IndexToK(a, N, projection_type): 19 | n_tilde = a 20 | n_tilde[np.where(a - N > -N / 2 - 1)] -= N 21 | 22 | two_pi_n_tilde = 2. * np.pi / N * n_tilde 23 | if projection_type == 1: 24 | return np.sin(two_pi_n_tilde) 25 | elif projection_type == 2: 26 | return (8. * np.sin(two_pi_n_tilde) - np.sin(2. * two_pi_n_tilde)) / 6. 27 | return 0. 28 | 29 | ## Add unique k values to file. 30 | # @param data_dir Sledgehamr data directory. 31 | # @param dim Dimension for which to generate k values. 32 | def CreateSpectrumBins(data_dir, dim): 33 | file = 'spectra_ks.hdf5' 34 | filename = data_dir +'/' + file 35 | 36 | archive = h5py.File(filename, 'a') 37 | if str(dim) not in archive.keys(): 38 | vals = CreateUniqueVals(dim) 39 | i = np.linspace(0, dim-1, dim) 40 | archive.create_dataset(str(dim)+'_bins', data = vals) 41 | archive.create_dataset(str(dim)+'_nks', data = np.array([len(vals)])) 42 | archive.create_dataset(str(dim)+'_k1', data = IndexToK(np.copy(i), dim, 1)) 43 | archive.create_dataset(str(dim)+'_k2', data = IndexToK(np.copy(i), dim, 2)) 44 | archive.close() 45 | print('Binning for a '+str(dim)+'^3 spectrum added!') 46 | else: 47 | print('Binning already exists in file.') 48 | -------------------------------------------------------------------------------- /pySledgehamr/__init__.py: -------------------------------------------------------------------------------- 1 | from pySledgehamr.Sledgehamr import * 2 | from pySledgehamr.Output import Output 3 | from pySledgehamr.AxionStrings import AxionStrings 4 | -------------------------------------------------------------------------------- /source/Make.package: -------------------------------------------------------------------------------- 1 | CEXE_sources += main.cpp 2 | 3 | CEXE_headers += sledgehamr.h 4 | CEXE_sources += sledgehamr.cpp 5 | 6 | CEXE_headers += sledgehamr_init.h 7 | CEXE_sources += sledgehamr_init.cpp 8 | 9 | CEXE_headers += level_synchronizer.h 10 | CEXE_sources += level_synchronizer.cpp 11 | 12 | CEXE_headers += local_regrid.h 13 | CEXE_sources += local_regrid.cpp 14 | 15 | CEXE_headers += unique_layout.h 16 | CEXE_sources += unique_layout.cpp 17 | 18 | CEXE_headers += location.h 19 | CEXE_sources += location.cpp 20 | 21 | CEXE_headers += io_module.h 22 | CEXE_sources += io_module.cpp 23 | 24 | CEXE_headers += fill_level.h 25 | CEXE_sources += fill_level.cpp 26 | 27 | CEXE_headers += performance_monitor.h 28 | CEXE_sources += performance_monitor.cpp 29 | 30 | CEXE_headers += timer.h 31 | CEXE_sources += timer.cpp 32 | 33 | CEXE_headers += time_stepper.h 34 | CEXE_sources += time_stepper.cpp 35 | 36 | CEXE_headers += regrid_scheduler.h 37 | CEXE_sources += regrid_scheduler.cpp 38 | 39 | CEXE_headers += gravitational_waves.h 40 | CEXE_sources += gravitational_waves.cpp 41 | 42 | CEXE_headers += level_data.h 43 | CEXE_headers += scalars.h 44 | CEXE_headers += projects.h 45 | CEXE_headers += macros.h 46 | CEXE_headers += kernels.h 47 | -------------------------------------------------------------------------------- /source/fill_level.h: -------------------------------------------------------------------------------- 1 | #ifndef SLEDGEHAMR_FILL_LEVEL_H_ 2 | #define SLEDGEHAMR_FILL_LEVEL_H_ 3 | 4 | #include "sledgehamr.h" 5 | 6 | namespace sledgehamr { 7 | 8 | /** @brief This class will fill an entire level with data from either a 9 | * checkpoint, an hdf5 file, an array or using a constant value, 10 | * depending on the chosen routine. Operates on a single level given in 11 | * the constructor unless we are trying to initialize a run via a 12 | * checkpoint. 13 | */ 14 | class FillLevel { 15 | public: 16 | /** @brief Constructor. Doesn't do any work except collect meta data. 17 | * @param owner Pointer to the simulation. 18 | * @param level Number of level to be filled with data (unless we 19 | * initialize from a checkpoint). 20 | */ 21 | FillLevel(Sledgehamr *owner, const int level) : sim(owner), lev(level){}; 22 | 23 | void FromInitialStateFile(); 24 | void FromCheckpoint(std::string folder); 25 | void FromHdf5File(std::string initial_state_file); 26 | void FromArray(const int comp, amrex::Gpu::AsyncArray &data, 27 | const long long dimN); 28 | void FromArrayChunks(const int comp, amrex::Gpu::AsyncArray &data); 29 | void FromConst(const int comp, const double c); 30 | 31 | private: 32 | /** @brief Pointer to the simulation. 33 | */ 34 | Sledgehamr *sim; 35 | 36 | /** @brief Number of level to be filled with data (unless we initialize from 37 | * a checkpoint). 38 | */ 39 | const int lev; 40 | }; 41 | 42 | }; // namespace sledgehamr 43 | 44 | #endif // SLEDGEHAMR_FILL_LEVEL_H_ 45 | -------------------------------------------------------------------------------- /source/gravitational_waves.h: -------------------------------------------------------------------------------- 1 | #ifndef SLEDGEHAMR_GRAVITATIONAL_WAVES_H_ 2 | #define SLEDGEHAMR_GRAVITATIONAL_WAVES_H_ 3 | 4 | #include 5 | 6 | #include "sledgehamr.h" 7 | 8 | namespace sledgehamr { 9 | 10 | struct GravitationalWavesSpectrumModifier; 11 | class Sledgehamr; 12 | 13 | /** @brief This will setup everything needed to simulate gravitational waves and 14 | * compute the gravitational wave spectrum. 15 | */ 16 | class GravitationalWaves { 17 | public: 18 | GravitationalWaves(Sledgehamr *owner); 19 | 20 | void 21 | ComputeSpectrum(hid_t file_id, 22 | GravitationalWavesSpectrumModifier *modifier = nullptr); 23 | 24 | int GetProjectionType() const { return projection_type; } 25 | 26 | /** @brief enum with the tensor components. 27 | */ 28 | enum Gw { 29 | u_xx = 0, 30 | u_yy, 31 | u_zz, 32 | u_xy, 33 | u_xz, 34 | u_yz, 35 | du_xx, 36 | du_yy, 37 | du_zz, 38 | du_xy, 39 | du_xz, 40 | du_yz, 41 | NGwScalars 42 | }; 43 | 44 | private: 45 | double IndexToK(int a, int N); 46 | double GetProjection(int i, int j, int abc[3], int N); 47 | double GetLambda(int i, int j, int l, int m, int abc[3], int N); 48 | 49 | /** @brief Pointer to the simulation. 50 | */ 51 | Sledgehamr *sim; 52 | 53 | /** @brief Offset corresponding the number of user-defined scalar fields. 54 | * We need this as all gravitional wave fields are appended to the 55 | * end of that list. 56 | */ 57 | int idx_offset; 58 | 59 | /** @brief Number of gravitational wave components 60 | */ 61 | static constexpr int NScalars = 12; 62 | 63 | /** @brief Projection type when computing the spectrum. This should be set 64 | * to the same order as the gradient computation in the EOM. 65 | * However, since this defined in a kernel function we need to keep 66 | * this is a free parameter here and rely on the user to select 67 | * the appropiate projection type through the input file. 68 | * Will be set by 'output.gw_spectra.projection_type'. 69 | */ 70 | int projection_type = 2; 71 | 72 | std::unique_ptr default_modifier; 73 | }; 74 | 75 | struct GravitationalWavesSpectrumModifier { 76 | virtual void SelectComponents(int components[6]) { 77 | components[0] = GravitationalWaves::Gw::du_xx; 78 | components[1] = GravitationalWaves::Gw::du_xy; 79 | components[2] = GravitationalWaves::Gw::du_xz; 80 | components[3] = GravitationalWaves::Gw::du_yy; 81 | components[4] = GravitationalWaves::Gw::du_yz; 82 | components[5] = GravitationalWaves::Gw::du_zz; 83 | }; 84 | 85 | virtual void FourierSpaceModifications(amrex::MultiFab du_real[6], 86 | amrex::MultiFab du_imag[6], 87 | const double dk, const int dimN){}; 88 | }; 89 | 90 | }; // namespace sledgehamr 91 | 92 | #endif // SLEDGEHAMR_GRAVITATIONAL_WAVES_H_ 93 | -------------------------------------------------------------------------------- /source/integrators/Make.package: -------------------------------------------------------------------------------- 1 | CEXE_headers += integrator.h 2 | CEXE_sources += integrator.cpp 3 | 4 | CEXE_headers += amrex_integrators.h 5 | CEXE_sources += amrex_integrators.cpp 6 | 7 | CEXE_headers += lsssprk3.h 8 | CEXE_sources += lsssprk3.cpp 9 | 10 | CEXE_headers += rkn.h 11 | CEXE_sources += rkn.cpp 12 | -------------------------------------------------------------------------------- /source/integrators/amrex_integrators.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "amrex_integrators.h" 4 | 5 | namespace sledgehamr { 6 | 7 | /** @brief Advances one level by one time step using the Runge-Kutta 8 | * integration scheme. 9 | * @param mf_old Current state. 10 | * @param mf_new New state after advancement. 11 | * @param lev Current level. 12 | * @param dt Time step size. 13 | * @param dx Grid spacing. 14 | */ 15 | void IntegratorAMReX::Integrate(LevelData& mf_old, LevelData& mf_new, 16 | const int lev, const double dt, const double dx) { 17 | // TODO Make 4th order time interpolation available. 18 | amrex::TimeIntegrator integrator(mf_old); 19 | 20 | auto source_fun = [&](amrex::MultiFab& rhs, const amrex::MultiFab& state, 21 | const double time) { 22 | sim->FillRhs(rhs, state, time, lev, dt, dx); 23 | }; 24 | 25 | auto post_update_fun = [&](amrex::MultiFab& S_data, const double time) { 26 | sim->level_synchronizer->FillIntermediatePatch(lev, time, S_data); 27 | }; 28 | 29 | integrator.set_rhs(source_fun); 30 | integrator.set_post_update(post_update_fun); 31 | integrator.advance(mf_old, mf_new, mf_old.t, dt); 32 | } 33 | 34 | }; // namespace sledgehamr 35 | -------------------------------------------------------------------------------- /source/integrators/amrex_integrators.h: -------------------------------------------------------------------------------- 1 | #ifndef SLEDGEHAMR_INTEGRATOR_AMREX_H_ 2 | #define SLEDGEHAMR_INTEGRATOR_AMREX_H_ 3 | 4 | #include "integrator.h" 5 | 6 | namespace sledgehamr { 7 | 8 | /** @brief Implements the Runge-Kutta integration scheme provided within AMReX. 9 | */ 10 | class IntegratorAMReX : public Integrator { 11 | using Integrator::Integrator; 12 | 13 | protected: 14 | virtual void Integrate(LevelData& mf_old, LevelData& mf_new, const int lev, 15 | const double dt, const double dx) override; 16 | }; 17 | 18 | }; // namespace sledgehamr 19 | 20 | #endif // SLEDGEHAMR_INTEGRATOR_AMREX_H_ 21 | -------------------------------------------------------------------------------- /source/integrators/integrator.cpp: -------------------------------------------------------------------------------- 1 | #include "integrator.h" 2 | #include "level_data.h" 3 | 4 | namespace sledgehamr{ 5 | 6 | /** @brief Wrapper function to advance a level by one time step. This function 7 | * selects and swaps data appropriately in particular regarding to the 8 | * shadow level. 9 | * @param lev Level to be advanced. 10 | */ 11 | void Integrator::Advance(const int lev) { 12 | if (lev >= 0) { 13 | sim->grid_old[lev].contains_truncation_errors = false; 14 | std::swap(sim->grid_old[lev], sim->grid_new[lev]); 15 | } 16 | 17 | const double dt = lev < 0 ? sim->dt[0]*2. : sim->dt[lev]; 18 | const double dx = lev < 0 ? sim->dx[0]*2. : sim->dx[lev]; 19 | LevelData& mf_old = lev < 0 ? sim->shadow_level_tmp : sim->grid_old[lev]; 20 | LevelData& mf_new = lev < 0 ? sim->shadow_level : sim->grid_new[lev]; 21 | 22 | sim->level_synchronizer->FillPatch(lev, mf_old.t, mf_old); 23 | Integrate(mf_old, mf_new, lev, dt, dx); 24 | 25 | mf_new.t = mf_old.t + dt; 26 | mf_new.istep = mf_old.istep + 1; 27 | 28 | if (lev < 0) { 29 | sim->shadow_level_tmp.clear(); 30 | } 31 | } 32 | 33 | /** @brief Creates a display name containing the selected integrator type. 34 | * @param type Integrator type. 35 | * @return Display name. 36 | */ 37 | std::string Integrator::Name(IntegratorType type) { 38 | switch (type) { 39 | case AmrexRkButcherTableau: 40 | return "User-defined RK Butcher Tableau"; 41 | case AmrexForwardEuler: 42 | return "Forward Euler"; 43 | case AmrexTrapezoid: 44 | return "Trapezoid Method"; 45 | case AmrexSsprk3: 46 | return "SSPRK3 (AMReX implementation)"; 47 | case AmrexRk4: 48 | return "RK4"; 49 | case Lsssprk3: 50 | return "SSPRK3 (Low-storage sledgehamr implementation)"; 51 | case RknButcherTableau: 52 | return "User-defined RKN Butcher Tableau"; 53 | case Rkn4: 54 | return "4th order RKN"; 55 | case Rkn5: 56 | return "5th order RKN"; 57 | default: 58 | return "Unkown!"; 59 | } 60 | } 61 | 62 | void Integrator::DebugMessage(amrex::MultiFab& mf, std::string msg) { } 63 | 64 | }; // namespace sledgehamr 65 | -------------------------------------------------------------------------------- /source/integrators/integrator.h: -------------------------------------------------------------------------------- 1 | #ifndef SLEDGEHAMR_INTEGRATOR_H_ 2 | #define SLEDGEHAMR_INTEGRATOR_H_ 3 | 4 | #include "sledgehamr.h" 5 | #include "level_data.h" 6 | 7 | namespace sledgehamr { 8 | 9 | /** @brief Enum containing all valid integration types. These are the values 10 | * that are being used in the inputs file under 'integrator.type'. 11 | */ 12 | enum IntegratorType { 13 | AmrexRkButcherTableau = 0, 14 | AmrexForwardEuler = 1, 15 | AmrexTrapezoid = 2, 16 | AmrexSsprk3 = 3, 17 | AmrexRk4 = 4, 18 | Lsssprk3 = 10, 19 | RknButcherTableau = 20, 20 | Rkn4 = 21, 21 | Rkn5 = 22 22 | }; 23 | 24 | class Sledgehamr; 25 | 26 | /** @brief Abstract base class that handles the time integration for a 27 | * single level. 28 | */ 29 | class Integrator { 30 | public: 31 | Integrator(Sledgehamr* owner) : sim(owner) {}; 32 | virtual void Advance(const int lev); 33 | static std::string Name(IntegratorType type); 34 | static void DebugMessage(amrex::MultiFab& mf, std::string msg); 35 | 36 | protected: 37 | /** @brief Purely virtual function that advances one level by one time step 38 | * using an integration scheme of your choice. 39 | * @param mf_old Current state. 40 | * @param mf_new New state after advancement. 41 | * @param lev Current level. 42 | * @param dt Time step size. 43 | * @param dx Grid spacing. 44 | */ 45 | virtual void Integrate(LevelData& mf_old, LevelData& mf_new, const int lev, 46 | const double dt, const double dx) = 0; 47 | 48 | /** @brief Pointer to the simulation. 49 | */ 50 | Sledgehamr* sim; 51 | }; 52 | 53 | }; // namespace sledgehamr 54 | 55 | #endif // SLEDGEHAMR_INTEGRATOR_H_ 56 | -------------------------------------------------------------------------------- /source/integrators/lsssprk3.cpp: -------------------------------------------------------------------------------- 1 | #include "lsssprk3.h" 2 | 3 | namespace sledgehamr { 4 | 5 | /** @brief Advances one level by one time step using the low-storage strong 6 | * stability preserving third order Runge-Kutte integration scheme 7 | * (LSSSPRK3). 8 | * @param mf_old Current state. 9 | * @param mf_new New state after advancement. 10 | * @param lev Current level. 11 | * @param dt Time step size. 12 | * @param dx Grid spacing. 13 | */ 14 | void IntegratorLsssprk3::Integrate(LevelData& mf_old, LevelData& mf_new, 15 | const int lev, const double dt, const double dx) { 16 | const int ncomp = mf_old.nComp(); 17 | const double t0 = mf_old.t; 18 | const double t1 = t0 + dt; 19 | amrex::MultiFab k1(mf_old.boxArray(), mf_old.DistributionMap(), ncomp, 20 | sim->nghost); 21 | 22 | sim->FillRhs(k1, mf_old, t0, lev, dt, dx); 23 | amrex::MultiFab::LinComb(mf_new, 1, mf_old, 0, dt, k1, 0, 0, ncomp, 0); 24 | sim->level_synchronizer->FillIntermediatePatch(lev, t1, mf_new); 25 | sim->FillAddRhs(k1, mf_new, t1, lev, dt, dx, 1.); 26 | amrex::MultiFab::LinComb(mf_new, 1, mf_old, 0, dt/4., k1, 0, 0, ncomp, 0); 27 | sim->level_synchronizer->FillIntermediatePatch(lev, t0 + dt/2., mf_new); 28 | sim->FillAddRhs(k1, mf_new, t0 + dt/2., lev, dt, dx, 0.25); 29 | amrex::MultiFab::LinComb(mf_new, 1, mf_old, 0, dt*2./3., k1, 0, 0, ncomp,0); 30 | sim->level_synchronizer->FillIntermediatePatch(lev, t1, mf_new); 31 | } 32 | 33 | }; // namespace sledgehamr 34 | -------------------------------------------------------------------------------- /source/integrators/lsssprk3.h: -------------------------------------------------------------------------------- 1 | #ifndef SLEDGEHAMR_INTEGRATOR_LSSSPRK3_H_ 2 | #define SLEDGEHAMR_INTEGRATOR_LSSSPRK3_H_ 3 | 4 | #include "integrator.h" 5 | 6 | namespace sledgehamr { 7 | 8 | /** @brief Implementation of the low-storage strong stability preserving third 9 | * order Runge-Kutta integration scheme (LSSSPRK3). 10 | */ 11 | class IntegratorLsssprk3 : public Integrator { 12 | using Integrator::Integrator; 13 | 14 | protected: 15 | virtual void Integrate(LevelData& mf_old, LevelData& mf_new, const int lev, 16 | const double dt, const double dx) override; 17 | }; 18 | 19 | }; // namespace sledgehamr 20 | 21 | #endif // SLEDGEHAMR_INTEGRATOR_LSSSPRK3_H_ 22 | -------------------------------------------------------------------------------- /source/integrators/rkn.cpp: -------------------------------------------------------------------------------- 1 | #include "rkn.h" 2 | #include "sledgehamr_utils.h" 3 | 4 | namespace sledgehamr { 5 | 6 | /** @brief Set up integrator in advance. 7 | * @param owner Pointer to simulation. 8 | * @param id Integrator type. Must be either RknButcherTableau, Rkn4, or 9 | * Rkn5. 10 | */ 11 | IntegratorRkn::IntegratorRkn(Sledgehamr* owner, const IntegratorType id) 12 | : Integrator{owner}, 13 | integrator_type(id) { 14 | SetButcherTableau(); 15 | } 16 | 17 | /** @brief Advances one level by one time step using the Runge-Kutta-Nystroem 18 | * integration scheme. 19 | * @param mf_old Current state. 20 | * @param mf_new New state after advancement. 21 | * @param lev Current level. 22 | * @param dt Time step size. 23 | * @param dx Grid spacing. 24 | */ 25 | void IntegratorRkn::Integrate(LevelData& mf_old, LevelData& mf_new, 26 | const int lev, const double dt, const double dx) { 27 | const double t = mf_old.t; 28 | const int nghost = sim->nghost; 29 | 30 | // Total number of fields, gravitational field components and 31 | // user-defined fields (includes conjugate momenta). 32 | const int N = mf_old.nComp(); 33 | const int Ngrav = sim->with_gravitational_waves ? 12 : 0; 34 | const int Nf = N - Ngrav; 35 | 36 | // Start and end points of field pairings for user fields and 37 | // gravitational wave components. 38 | const int uN = Nf/2; 39 | const int uN0 = 0; 40 | const int uN1 = uN0 + uN; 41 | 42 | const int gN = Ngrav/2; 43 | const int gN0 = Nf; 44 | const int gN1 = gN0 + gN; 45 | 46 | std::vector > F_nodes; 47 | for (int i = 0; i < number_nodes; ++i) { 48 | F_nodes.emplace_back( std::make_unique( 49 | mf_old.boxArray(), mf_old.DistributionMap(), N, nghost) ); 50 | } 51 | 52 | for (int i = 0; i < number_nodes; ++i) { 53 | // stage_time = x_0 + c_i*h 54 | double stage_time = t + dt * nodes[i]; 55 | 56 | // mf_new = (y_0, y'_0) 57 | amrex::MultiFab::Copy(mf_new, mf_old, 0, 0, N, nghost); 58 | 59 | // mf_new = (y_0 + c_i * h * y'_0, y'_0) 60 | amrex::MultiFab::Saxpy(mf_new, dt * nodes[i], mf_old, 61 | uN1, uN0, uN, nghost); 62 | if (Ngrav > 0) 63 | amrex::MultiFab::Saxpy(mf_new, dt * nodes[i], mf_old, 64 | gN1, gN0 , gN, nghost); 65 | 66 | if (i > 0) { 67 | // mf_new = (y_0 + c_i * h * y'_0 + sum_j( h^2 * a_ij * k'_j ), 68 | // y'_0) 69 | for (int j = 0; j < i; ++j) { 70 | amrex::MultiFab::Saxpy( 71 | mf_new, dt*dt * tableau[i][j], *F_nodes[j], 72 | uN1, uN0 , uN, nghost); 73 | 74 | if (Ngrav == 0) continue; 75 | 76 | amrex::MultiFab::Saxpy( 77 | mf_new, dt*dt * tableau[i][j], *F_nodes[j], 78 | gN1, gN0 , gN, nghost); 79 | } 80 | 81 | } 82 | 83 | sim->level_synchronizer->FillIntermediatePatch( 84 | lev, stage_time, mf_new); 85 | 86 | // F_nodes[i] = (null, f(stage_time, mf_new) 87 | // = (null, k'_i) 88 | sim->FillRhs(*F_nodes[i], mf_new, stage_time, lev, dt, dx); 89 | } 90 | 91 | // mf_new = (y_0, y'_0) 92 | amrex::MultiFab::Copy(mf_new, mf_old, 0, 0, N, nghost); 93 | 94 | // mf_new = (y_0 + h * y'_0, y'_0) 95 | amrex::MultiFab::Saxpy(mf_new, dt, mf_new, uN1, uN0, uN, nghost); 96 | if( Ngrav > 0 ) 97 | amrex::MultiFab::Saxpy(mf_new, dt, mf_new, gN1, gN0, gN, nghost); 98 | 99 | // mf_new = (y_0 + h * y'_0 + sum_i(h^2*\bar[b_i]*k'_i), 100 | // y'_0 + sum_i(b_i*k'_i)) 101 | for (int i = 0; i < number_nodes; ++i) { 102 | amrex::MultiFab::Saxpy(mf_new, dt * dt * weights_bar_b[i], 103 | *F_nodes[i], 104 | uN1, uN0, uN, nghost); 105 | amrex::MultiFab::Saxpy(mf_new, dt * weights_b[i], *F_nodes[i], 106 | uN1, uN1, uN, nghost); 107 | 108 | if (Ngrav == 0) continue; 109 | 110 | amrex::MultiFab::Saxpy(mf_new, dt * dt * weights_bar_b[i], 111 | *F_nodes[i], 112 | gN1, gN0, gN, nghost); 113 | amrex::MultiFab::Saxpy(mf_new, dt * weights_b[i], *F_nodes[i], 114 | gN1, gN1, gN, nghost); 115 | } 116 | 117 | // Call the post-update hook for S_new 118 | sim->level_synchronizer->FillIntermediatePatch( 119 | lev, t + dt, mf_new); 120 | } 121 | 122 | /** @brief Sets up the correct Butcher Tableau given an integration scheme. 123 | */ 124 | void IntegratorRkn::SetButcherTableau() { 125 | switch (integrator_type) { 126 | case RknButcherTableau: 127 | ReadUserDefinedButcherTableau(); 128 | break; 129 | 130 | case Rkn4: 131 | nodes = {0.0, 132 | 0.5, 133 | 1.0}; 134 | tableau = {{0.0}, 135 | {1./8., 0.0}, 136 | {0.0, 0.5, 0.0}}; 137 | weights_bar_b = {1./6., 1./3., 0.0}; 138 | weights_b = {1./6., 4./6., 1./6.}; 139 | break; 140 | 141 | case Rkn5: 142 | nodes = {0.0, 143 | 1./5., 144 | 2./3., 145 | 1.0}; 146 | tableau = {{0.0}, 147 | {1./50., 0.0}, 148 | {-1./27., 7./27., 0.0}, 149 | {3./10., -2./35., 9./35., 0.0}}; 150 | weights_bar_b = {14./336., 100./336., 54./336., 0.0}; 151 | weights_b = {14./336., 125./336., 162./336., 35./336.}; 152 | break; 153 | } 154 | 155 | number_nodes = weights_b.size(); 156 | } 157 | 158 | /** @brief Reads in a user-defined Butcher Tableau. 159 | */ 160 | void IntegratorRkn::ReadUserDefinedButcherTableau() { 161 | amrex::ParmParse pp("integrator.rkn"); 162 | 163 | pp.getarr("weights_bar_b", weights_bar_b); 164 | pp.getarr("weights_b", weights_b); 165 | pp.getarr("nodes", nodes); 166 | 167 | amrex::Vector btable; // flattened into row major format 168 | pp.getarr("tableau", btable); 169 | 170 | if (weights_bar_b.size() != nodes.size() || 171 | weights_b.size() != nodes.size()) { 172 | std::string msg = "integrator.weights should be the same length as "; 173 | msg += "integrator.nodes"; 174 | amrex::Error(msg); 175 | } else { 176 | number_nodes = weights_b.size(); 177 | const int nTableau = (number_nodes * (number_nodes + 1)) / 2; 178 | if (btable.size() != nTableau) { 179 | std::string msg = "integrator.tableau incorrect length - "; 180 | msg += "should include the Butcher Tableau diagonal."; 181 | amrex::Error(msg); 182 | } 183 | } 184 | 185 | // Fill tableau from the flattened entries. 186 | int k = 0; 187 | for (int i = 0; i < number_nodes; ++i) { 188 | std::vector stage_row; 189 | for (int j = 0; j <= i; ++j) { 190 | stage_row.push_back(btable[k]); 191 | ++k; 192 | } 193 | 194 | tableau.push_back(stage_row); 195 | } 196 | 197 | // Check that this is an explicit method. 198 | for (const auto& astage : tableau) { 199 | if (astage.back() != 0.0) { 200 | std::string msg = "RKN integrator currently only supports "; 201 | msg += "explicit Butcher tableaus."; 202 | amrex::Error(msg); 203 | } 204 | } 205 | } 206 | 207 | }; // namespace sledgehamr 208 | -------------------------------------------------------------------------------- /source/integrators/rkn.h: -------------------------------------------------------------------------------- 1 | #ifndef SLEDGEHAMR_RKN_H_ 2 | #define SLEDGEHAMR_RKN_H_ 3 | 4 | #include 5 | 6 | #include "integrator.h" 7 | 8 | namespace sledgehamr { 9 | 10 | /** @brief Implementation of the Runge-Kutta-Nystroem method of arbitrary order. 11 | */ 12 | class IntegratorRkn : public Integrator { 13 | public: 14 | IntegratorRkn(Sledgehamr* owner, const IntegratorType id); 15 | 16 | protected: 17 | virtual void Integrate(LevelData& mf_old, LevelData& mf_new, const int lev, 18 | const double dt, const double dx) override; 19 | 20 | private: 21 | void SetButcherTableau(); 22 | void ReadUserDefinedButcherTableau(); 23 | 24 | /** @brief Number of nodes involved. 25 | */ 26 | int number_nodes; 27 | 28 | /** @brief Butcher Tableau values. 29 | */ 30 | std::vector > tableau; 31 | 32 | /** @brief b-weights. 33 | */ 34 | std::vector weights_b; 35 | 36 | /** @brief \bar{b} weights. 37 | */ 38 | std::vector weights_bar_b; 39 | 40 | /** @brief Node values. 41 | */ 42 | std::vector nodes; 43 | 44 | /** @brief Selected integrator type. 45 | */ 46 | IntegratorType integrator_type; 47 | }; 48 | 49 | }; // namespace sledgehamr 50 | 51 | #endif // SLEDGEHAMR_RKN_H_ 52 | -------------------------------------------------------------------------------- /source/io_module.h: -------------------------------------------------------------------------------- 1 | #ifndef SLEDGEHAMR_IO_MODULE_H_ 2 | #define SLEDGEHAMR_IO_MODULE_H_ 3 | 4 | #include 5 | 6 | #include "sledgehamr.h" 7 | #include "output_module.h" 8 | #include "projection.h" 9 | 10 | namespace sledgehamr { 11 | 12 | class Sledgehamr; 13 | class Projection; 14 | class Spectrum; 15 | 16 | /** @brief Class that handles all I/O operations. 17 | */ 18 | class IOModule { 19 | public: 20 | IOModule (Sledgehamr* owner); 21 | 22 | void Write(bool force=false); 23 | void RestartSim(); 24 | void UpdateOutputModules(); 25 | void WriteBoxArray(amrex::BoxArray& ba); 26 | 27 | /** @brief Vectors containing instructions for projections. 28 | */ 29 | std::vector projections; 30 | 31 | /** @brief Vector containing instructions for spectrum computations. 32 | * Gravitational wave spectra are handled separately and no 33 | * instructions need to be provided. 34 | */ 35 | std::vector spectra; 36 | 37 | /** @brief Output module ID's of various output types. 38 | */ 39 | int idx_slices = -1; 40 | int idx_coarse_box = -1; 41 | int idx_full_box = -1; 42 | int idx_slices_truncation_error = -1; 43 | int idx_coarse_box_truncation_error = -1; 44 | int idx_full_box_truncation_error = -1; 45 | int idx_projections = -1; 46 | int idx_spectra = -1; 47 | int idx_gw_spectra = -1; 48 | int idx_performance_monitor = -1; 49 | int idx_amrex_plotfile = -1; 50 | int idx_checkpoints = -1; 51 | 52 | /** @brief Vector of output modules. 53 | */ 54 | std::vector output; 55 | 56 | /** @brief Path to output folder. 57 | */ 58 | std::string output_folder; 59 | 60 | /** @brief Path to alternative output folder if one is provided. 61 | */ 62 | std::string alternative_output_folder = ""; 63 | 64 | /** @brief Path to last checkpoint. Needed so we can delete it if 65 | * we need to. 66 | */ 67 | std::string old_checkpoint = ""; 68 | 69 | private: 70 | bool WriteSlices(double time, std::string prefix); 71 | bool WriteSlicesTruncationError(double time, std::string prefix); 72 | bool WriteCoarseBox(double time, std::string prefix); 73 | bool WriteCoarseBoxTruncationError(double time, std::string prefix); 74 | bool WriteFullBox(double time, std::string prefix); 75 | bool WriteFullBoxTruncationError(double time, std::string prefix); 76 | bool WriteProjections(double time, std::string prefix); 77 | bool WriteSpectra(double time, std::string prefix); 78 | bool WriteGravitationalWaveSpectrum(double time, std::string prefix); 79 | bool WritePerformanceMonitor(double time, std::string prefix); 80 | bool WriteAmrexPlotFile(double time, std::string prefix); 81 | bool WriteCheckpoint(double time, std::string prefix); 82 | 83 | int FindLatestCheckpoint(std::string folder); 84 | std::vector GetDirectories(const std::string prefix); 85 | void CheckIfOutputAlreadyExists(std::string folder); 86 | void CreateOutputFolder(std::string folder); 87 | 88 | void AddOutputModules(); 89 | void ParseParams(); 90 | 91 | /** @brief Path to initial checkpoint file if any. 92 | */ 93 | std::string initial_chk = ""; 94 | 95 | /** @brief If we are using rolling checkpoints, i.e. only ever keep the 96 | * latest one. 97 | */ 98 | bool rolling_checkpoints = false; 99 | 100 | /** @brief If we want to delete the checkpoint we restarted the sim from. 101 | * Will only be deleted once we created a newer one. 102 | */ 103 | bool delete_restart_checkpoint = false; 104 | 105 | /** @brief Pointer to the simulation. 106 | */ 107 | Sledgehamr* sim; 108 | }; 109 | 110 | }; // namespace sledgehamr 111 | 112 | #endif // SLEDGEHAMR_IO_MODULE_H_ 113 | -------------------------------------------------------------------------------- /source/kernels.h: -------------------------------------------------------------------------------- 1 | #ifndef SLEDGEHAMR_KERNELS_H_ 2 | #define SLEDGEHAMR_KERNELS_H_ 3 | 4 | namespace sledgehamr { 5 | namespace kernels { 6 | 7 | /** @brief Averages down a set of fine level cells onto a coarse level cell. 8 | * Computes truncation errors at the same time. 9 | * @param i i-th cell index. 10 | * @param j j-th cell index. 11 | * @param k k-th cell index. 12 | * @param ncomp Number of scalar fields. 13 | * @param crse Coarse level data. 14 | * @param fine Fine level data. 15 | * @param te Container to save truncation errors. Same dimensions as 16 | * fine. 17 | */ 18 | AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE 19 | void AverageDownWithTruncationError(int i, int j, int k, const int ncomp, 20 | amrex::Array4 const& crse, 21 | amrex::Array4 const& fine, 22 | amrex::Array4 const& te) { 23 | const int ratio = 2; 24 | const double volfrac = 1.0/(double)(ratio*ratio*ratio); 25 | const int ii = i*ratio; 26 | const int jj = j*ratio; 27 | const int kk = k*ratio; 28 | 29 | for (int n = 0; n < ncomp; ++n ) { 30 | double c = fine(ii, jj, kk, n); 31 | c += fine(ii+1, jj , kk , n); 32 | c += fine(ii , jj+1, kk , n); 33 | c += fine(ii , jj , kk+1, n); 34 | c += fine(ii+1, jj+1, kk , n); 35 | c += fine(ii+1, jj , kk+1, n); 36 | c += fine(ii , jj+1, kk+1, n); 37 | c += fine(ii+1, jj+1, kk+1, n); 38 | 39 | te(ii,jj,kk,n) = fabs(crse(i,j,k,n) - volfrac * c); 40 | crse(i,j,k,n) = volfrac * c; 41 | } 42 | } 43 | 44 | /** @brief Template declaration for Kreiss-Oliger dissipation compution at a 45 | * single cell. Template parameter corresponds to the order of 46 | * dissipation. 47 | * @param state Current field state. 48 | * @param i i-th cells index. 49 | * @param j j-th cell index. 50 | * @param k k-th cell index. 51 | * @param c Scalar component. 52 | * @param dx Grid spacing. 53 | * @param dissipation_strength Dissipation strength. 54 | * @return Dissipation value. 55 | */ 56 | template AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE 57 | double KreissOligerDissipation(const amrex::Array4& state, 58 | const int i, const int j, const int k, 59 | const int c, const double dx, 60 | const double dissipation_strength); 61 | 62 | /** @brief Template specialization for 2nd order Kreiss-Oliger dissipation. See 63 | * declartion of kernels::KreissOligerDissipation for more details. 64 | */ 65 | template<> AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE 66 | double KreissOligerDissipation<2>(const amrex::Array4& state, 67 | const int i, const int j, const int k, 68 | const int c, const double dx, 69 | const double dissipation_strength) { 70 | double dx4 = state(i+2,j,k,c) - 4.*state(i+1,j,k,c) 71 | + state(i-2,j,k,c) - 4.*state(i-1,j,k,c) 72 | + 6.*state(i,j,k,c); 73 | 74 | double dy4 = state(i,j+2,k,c) - 4.*state(i,j+1,k,c) 75 | + state(i,j-2,k,c) - 4.*state(i,j-1,k,c) 76 | + 6.*state(i,j,k,c); 77 | 78 | double dz4 = state(i,j,k+2,c) - 4.*state(i,j,k+1,c) 79 | + state(i,j,k-2,c) - 4.*state(i,j,k-1,c) 80 | + 6.*state(i,j,k,c); 81 | 82 | return -dissipation_strength * (dx4+dy4+dz4) / 16. / dx; 83 | } 84 | 85 | /** @brief Template specialization for 3rd order Kreiss-Oliger dissipation. See 86 | * declartion of kernels::KreissOligerDissipation for more details. 87 | */ 88 | template<> AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE 89 | double KreissOligerDissipation<3>(const amrex::Array4& state, 90 | const int i, const int j, const int k, 91 | const int c, const double dx, 92 | const double dissipation_strength) { 93 | double dx6 = state(i+3,j,k,c) - 6.*state(i+2,j,k,c) + 15.*state(i+1,j,k,c) 94 | + state(i-3,j,k,c) - 6.*state(i-2,j,k,c) + 15.*state(i-1,j,k,c) 95 | - 20.*state(i,j,k,c); 96 | 97 | double dy6 = state(i,j+3,k,c) - 6.*state(i,j+2,k,c) + 15.*state(i,j+1,k,c) 98 | + state(i,j-3,k,c) - 6.*state(i,j-2,k,c) + 15.*state(i,j-1,k,c) 99 | - 20.*state(i,j,k,c); 100 | 101 | double dz6 = state(i,j,k+3,c) - 6.*state(i,j,k+2,c) + 15.*state(i,j,k+1,c) 102 | + state(i,j,k-3,c) - 6.*state(i,j,k-2,c) + 15.*state(i,j,k-1,c) 103 | - 20.*state(i,j,k,c); 104 | 105 | return dissipation_strength * (dx6+dy6+dz6) / 64. / dx; 106 | } 107 | 108 | }; // namespace kernels 109 | }; // namespace sledgehamr 110 | 111 | #endif // SLEDGEHAMR_KERNELS_H_ 112 | -------------------------------------------------------------------------------- /source/level_data.h: -------------------------------------------------------------------------------- 1 | #ifndef SLEDGEHAMR_LEVEL_DATA_H 2 | #define SLEDGEHAMR_LEVEL_DATA_H 3 | 4 | #include 5 | 6 | namespace sledgehamr { 7 | 8 | /** @brief Class that holds the MultiFab data while also keeping track of time 9 | * and step numbers. 10 | */ 11 | class LevelData : public amrex::MultiFab { 12 | public: 13 | LevelData() : amrex::MultiFab{} {}; 14 | 15 | /** @brief Construct data using the given grid layout. 16 | * @param ba BoxArray. 17 | * @param dm DistributionMapping. 18 | * @param ncomp Total number of scalar fields components. 19 | * @param nghost Number of ghost cells. 20 | * @param time Time. 21 | */ 22 | LevelData(amrex::BoxArray ba, amrex::DistributionMapping dm, int ncomp, 23 | int nghost, double time=-DBL_MAX) 24 | : amrex::MultiFab{ba, dm, ncomp, nghost}, t{time} {}; 25 | 26 | using amrex::MultiFab::MultiFab; 27 | 28 | /** @brief Overload amrex::MultiFab::define function to include time. 29 | * @param ba BoxArray. 30 | * @param dm DistributionMapping. 31 | * @param ncomp Total number of scalar components. 32 | * @param nghost Number of ghost cells. 33 | * @param time Time. 34 | */ 35 | void define(amrex::BoxArray ba, amrex::DistributionMapping dm, int ncomp, 36 | int nghost, double time) { 37 | define(ba, dm, ncomp, nghost); 38 | t = time; 39 | } 40 | 41 | using amrex::MultiFab::define; 42 | 43 | /** @brief Static method that returns vector of times from a given LevelData 44 | * vector. 45 | * @param mfs Vector of pointers to MultiFab objects. MultiFab needs to be 46 | * castable to LevelData. 47 | * @return Vector of times. Needs to be amrex::Vector to be usable with 48 | * e.g. amrex::FillPatchTwoLevels. 49 | */ 50 | static amrex::Vector getTimes(std::vector& mfs) { 51 | amrex::Vector times; 52 | 53 | for (amrex::MultiFab* mf : mfs) { 54 | times.push_back( static_cast(mf)->t ); 55 | } 56 | 57 | return times; 58 | }; 59 | 60 | /** @brief Time corresponding to amrex::MultiFab data. 61 | */ 62 | double t = -DBL_MAX; 63 | 64 | /** @brief How often this level has been advanced. 65 | */ 66 | int istep = 0; 67 | 68 | /** @brief Flag as to whether the MutliFab has truncation errors encoded 69 | * into it. 70 | */ 71 | bool contains_truncation_errors = false; 72 | }; 73 | 74 | }; // namespace sledgehamr 75 | 76 | #endif // SLEDGEHAMR_LEVEL_DATA_H_ 77 | -------------------------------------------------------------------------------- /source/level_synchronizer.h: -------------------------------------------------------------------------------- 1 | #ifndef SLEDGEHAMR_LEVEL_SYNCHRONIZER_H_ 2 | #define SLEDGEHAMR_LEVEL_SYNCHRONIZER_H_ 3 | 4 | #include 5 | #include 6 | 7 | #include "sledgehamr.h" 8 | #include "level_data.h" 9 | 10 | namespace sledgehamr { 11 | 12 | /** @brief Struct with overloaded operator to handle boundary conditions. Empty 13 | * because we do not have boundary conditions beyond periodic currently. 14 | */ 15 | struct NullFill { 16 | AMREX_GPU_DEVICE 17 | void operator() (const amrex::IntVect& /*iv*/, 18 | amrex::Array4 const& /*dest*/, 19 | const int /*dcomp*/, const int /*numcomp*/, 20 | amrex::GeometryData const& /*geom*/, 21 | const amrex::Real /*time*/, const amrex::BCRec* /*bcr*/, 22 | const int /*bcomp*/, const int /*orig_comp*/) const {} 23 | }; 24 | 25 | class Sledgehamr; 26 | 27 | /** @brief This class handles all operations between two levels such as 28 | * averaging down, interpolation to fine, filling of ghost cells, etc. 29 | * Class is friend of sledgehamr. 30 | */ 31 | class LevelSynchronizer { 32 | public: 33 | LevelSynchronizer(Sledgehamr* owner); 34 | 35 | void FillCoarsePatch(const int lev, const double time, amrex::MultiFab& ld); 36 | void FillPatch(const int lev, const double time, amrex::MultiFab& mf, 37 | const int scomp = 0, const int dcomp = 0, int ncomp = -1); 38 | void FillIntermediatePatch(const int lev, const double time, 39 | amrex::MultiFab& mf, const int scomp = 0, 40 | const int dcomp = 0, const int ncomp = -1); 41 | 42 | void AverageDownTo(const int lev); 43 | void ComputeTruncationErrors(const int lev); 44 | 45 | void IncreaseCoarseLevelResolution(); 46 | void FromArrayChunksAndUpsample(const int lev, const int comp, double* data, 47 | int up); 48 | void ChangeNGhost(int new_nghost); 49 | void RegridCoarse(); 50 | 51 | /** @brief Integer array containing the type of boundary condition at each 52 | * boundary edge. Needs to be amrex::Vector not std::vector. 53 | */ 54 | amrex::Vector bcs; 55 | 56 | private: 57 | amrex::Vector GetLevelData(const int lev, 58 | const double time); 59 | 60 | /** @brief Pointer to AMReX interpolator to be used between levels. 61 | */ 62 | amrex::Interpolater* mapper = nullptr; 63 | 64 | /** @brief Pointer to the simulation. 65 | */ 66 | Sledgehamr* sim; 67 | 68 | /** @brief enum for the various interpolation types. 69 | */ 70 | enum InterpType { 71 | PCInterp = 0, 72 | CellConservativeLinear = 1, 73 | CellQuadratic = 2, 74 | CellConservativeQuartic = 4, 75 | }; 76 | }; 77 | 78 | }; // namespace sledgehamr 79 | 80 | #endif // SLEDGEHAMR_LEVEL_SYNCHRONIZER_H_ 81 | -------------------------------------------------------------------------------- /source/local_regrid/Make.package: -------------------------------------------------------------------------------- 1 | CEXE_headers += local_regrid.h 2 | CEXE_sources += local_regrid.cpp 3 | 4 | CEXE_headers += unique_layout.h 5 | CEXE_sources += unique_layout.cpp 6 | 7 | CEXE_headers += location.h 8 | CEXE_sources += location.cpp 9 | -------------------------------------------------------------------------------- /source/local_regrid/local_regrid.h: -------------------------------------------------------------------------------- 1 | #ifndef SLEDGEHAMR_LOCAL_REGRID_H_ 2 | #define SLEDGEHAMR_LOCAL_REGRID_H_ 3 | 4 | #include "boost/multi_array.hpp" 5 | 6 | #include "sledgehamr.h" 7 | #include "unique_layout.h" 8 | #include "location.h" 9 | 10 | namespace sledgehamr { 11 | 12 | class Sledgehamr; 13 | class UniqueLayout; 14 | class Location; 15 | 16 | /** @brief Class to perform a local regrid (if possible). 17 | */ 18 | class LocalRegrid { 19 | public: 20 | LocalRegrid(Sledgehamr* owner); 21 | 22 | bool AttemptRegrid(const int lev); 23 | void DidGlobalRegrid(const int lev); 24 | 25 | void InitializeLayout(const int max_lev); 26 | void ClearLayout(); 27 | void FinalizeLayout(const int lev); 28 | void WrapIndices(const int lev); 29 | 30 | void JoinBoxArrays(const int lev, amrex::BoxArray& ba); 31 | void AddBoxes(const int lev, amrex::BoxArray& ba); 32 | void FixNesting(const int lev); 33 | void AddToLayout(const int lev, const int thread, const int i, const int j, 34 | const int k); 35 | 36 | /** @brief Flag that will be checked the TimeStepper module to force a 37 | * global regrid. 38 | */ 39 | std::vector do_global_regrid; 40 | 41 | /** @brief MPI communication matrix. Look-up table used by the 42 | * UniqueLayout class. 43 | */ 44 | std::vector< std::vector > comm_matrix; 45 | 46 | private: 47 | /* @brief Enum with possible course of actions after a veto. 48 | */ 49 | enum VetoResult { 50 | DoGlobalRegrid = 0, 51 | DoNoRegrid = 1, 52 | DoLocalRegrid = 2, 53 | }; 54 | 55 | bool Prechecks(const int lev); 56 | void InitializeLocalRegrid(); 57 | void ParseInput(); 58 | void CreateCommMatrix(); 59 | 60 | bool DoAttemptRegrid(const int lev); 61 | void DetermineAllBoxArrays(const int lev); 62 | double DetermineNewBoxArray(const int lev); 63 | void FixAllNesting(); 64 | void JoinAllBoxArrays(std::vector& box_arrays); 65 | void AddAllBoxes(std::vector& box_arrays); 66 | amrex::BoxArray WrapBoxArray(amrex::BoxArray& ba, int N); 67 | 68 | inline int GetBoxCoarseFineBorders( 69 | const amrex::Box& tilebox, const amrex::IntVect& c0, 70 | const amrex::IntVect& c1, const int lev, 71 | boost::multi_array& border); 72 | 73 | inline void TagAndMeasure( 74 | const amrex::Dim3& lo, const amrex::Dim3& hi, int remaining, 75 | const amrex::Array4& tag_arr, const amrex::IntVect& c0, 76 | const amrex::IntVect& c1, const int lev, const int ibff, 77 | const double bff, boost::multi_array& border, 78 | std::vector& closest_locations, const double threshold, 79 | const int omp_thread_num); 80 | 81 | inline void CheckBorders( 82 | const amrex::IntVect& ci, const amrex::IntVect& c0, 83 | const amrex::IntVect& c1, const int ibff, const double bff, 84 | int remaining, const int lev, boost::multi_array& border, 85 | std::vector& closest_locations, const double threshold, 86 | const int omp_thread_num); 87 | 88 | bool CheckThresholds(const int lev, amrex::BoxArray& box_array); 89 | void ComputeLatestPossibleRegridTime(const int l, const int lev); 90 | bool CheckForVeto(const int lev, 91 | std::vector& box_arrays); 92 | VetoResult DealWithVeto(const int lev); 93 | 94 | /** @brief Strong threshold that decide whether we want to do a local or 95 | * global regrid. 96 | */ 97 | double volume_threshold_accumulated = 1.1; 98 | 99 | /** @brief Weak threshold that decide whether we want to do a local or 100 | * global regrid. 101 | */ 102 | double volume_threshold_single = 1.05; 103 | 104 | /** @brief Lowest level that would rather do a global regrid than a local. 105 | */ 106 | int veto_level = -1; 107 | 108 | /** @brief If flag set to true we will skip the local regrid once. 109 | */ 110 | bool force_global_regrid_at_restart = 0; 111 | 112 | /** @brief How often we are allowed to do a local regrid in a row. 113 | */ 114 | int max_local_regrids = 10; 115 | 116 | /** @brief Counts number of consecutive local regrids. 117 | */ 118 | int nregrids = 0; 119 | 120 | /** @brief Size of error buffer. 121 | */ 122 | int n_error_buf = 1; 123 | 124 | /** @brief Number of cells contained in each level after the last global 125 | * regrid. 126 | */ 127 | std::vector last_numPts; 128 | 129 | /** @brief If we want to skip the local regrid entirely at a given level. 130 | */ 131 | std::vector no_local_regrid; 132 | 133 | /** @brief Look-up table of wrapped indices for periodic boundary 134 | * conditions. 135 | */ 136 | std::vector< std::vector > wrapped_index; 137 | 138 | /** @brief Contains the time by which we should have done a regrid. 139 | */ 140 | std::vector latest_possible_regrid_time; 141 | 142 | /** @brief Minimum distance of a tagged cell to a coarse/fine boundary for 143 | * each OpenMP thread. 144 | */ 145 | std::vector min_distance; 146 | 147 | /** @brief Vector of UniqueLayouts for each level and each core. 148 | */ 149 | std::vector< std::vector< std::unique_ptr > > layouts; 150 | 151 | /** @brief Pointer to the simulation. 152 | */ 153 | Sledgehamr* sim; 154 | }; 155 | 156 | }; // namespace sledgehamr 157 | 158 | #endif // SLEDGEHAMR_LOCAL_REGRID_H_ 159 | -------------------------------------------------------------------------------- /source/local_regrid/location.cpp: -------------------------------------------------------------------------------- 1 | #include "location.h" 2 | 3 | namespace sledgehamr { 4 | 5 | /** @brief Overrides current location if provided new location is closer. 6 | * @param i_new New i-th location. 7 | * @param j_new New j-th location. 8 | * @param k_new New k-th location. 9 | * @param distance_sq_new New distance square. 10 | */ 11 | void Location::SelectClosest(const int i_new, const int j_new, const int k_new, 12 | const int distance_sq_new) { 13 | if (distance_sq_new < distance_sq) { 14 | i = i_new; 15 | j = j_new; 16 | k = k_new; 17 | distance_sq = distance_sq_new; 18 | } 19 | } 20 | 21 | /** @brief Overrides current location if provided new location is closer. 22 | * @param location New location. 23 | */ 24 | void Location::SelectClosest(Location location) { 25 | SelectClosest(location.i, location.j, location.k, location.distance_sq); 26 | } 27 | 28 | /** @brief Finds the location with the globally shortest distance after doing 29 | * OpenMP and MPI reduction. 30 | * @param locations Location at each OpenMP thread. 31 | * @return Closest location. 32 | */ 33 | Location Location::FindClosestGlobally( 34 | const std::vector& locations) { 35 | Location closest; 36 | 37 | // OpenMP reduction 38 | for (Location location : locations) 39 | closest.SelectClosest(location); 40 | 41 | // MPI reducation 42 | std::vector all_i = AllGather(closest.i); 43 | std::vector all_j = AllGather(closest.j); 44 | std::vector all_k = AllGather(closest.k); 45 | std::vector all_d = AllGather(closest.distance_sq); 46 | 47 | for (int l = 0; l < all_d.size(); ++l) { 48 | closest.SelectClosest(all_i[l], all_j[l], all_k[l], all_d[l]); 49 | } 50 | 51 | return closest; 52 | } 53 | 54 | /** @brief MPI Gather. 55 | * @param val Value to gather. 56 | * @return Gathered values. 57 | */ 58 | std::vector Location::Gather(const int val) { 59 | std::vector all(amrex::ParallelDescriptor::NProcs()); 60 | amrex::ParallelDescriptor::Gather(&val, 1, &all[0], 1, 61 | amrex::ParallelDescriptor::IOProcessorNumber()); 62 | return all; 63 | } 64 | 65 | /** @brief MPI AllGather. 66 | * @param val Value to gather. 67 | * @return Gathered values. 68 | */ 69 | std::vector Location::AllGather(const int val) { 70 | std::vector all(amrex::ParallelDescriptor::NProcs()); 71 | amrex::ParallelAllGather::AllGather(&val, 1, &all[0], MPI_COMM_WORLD); 72 | return all; 73 | } 74 | 75 | }; // namespace sledgehamr 76 | 77 | -------------------------------------------------------------------------------- /source/local_regrid/location.h: -------------------------------------------------------------------------------- 1 | #ifndef SLEDGEHAMR_LOCATION_H_ 2 | #define SLEDGEHAMR_LOCATION_H_ 3 | 4 | #include "sledgehamr.h" 5 | 6 | namespace sledgehamr { 7 | 8 | /** @brief Keeps track of a location and a distance and can do MPI and OpenMP 9 | * reductions to determine the one with the smallest distance. 10 | */ 11 | class Location { 12 | public: 13 | Location() {}; 14 | Location(const int i_new, const int j_new, const int k_new, 15 | const int distance_sq_new) 16 | : i(i_new), j(j_new), k(k_new), distance_sq(distance_sq_new) {}; 17 | 18 | void SelectClosest(const int i_new, const int j_new, const int k_new, 19 | const int distance_sq_new); 20 | void SelectClosest(const Location location); 21 | 22 | static Location FindClosestGlobally( 23 | const std::vector& locations); 24 | 25 | /** @brief Locations i-th index. 26 | */ 27 | int i = -1; 28 | 29 | /** @brief Locations j-th index. 30 | */ 31 | int j = -1; 32 | 33 | /** @brief Locations k-th index. 34 | */ 35 | int k = -1; 36 | 37 | /** @brief Distance square of location. 38 | */ 39 | int distance_sq = INT_MAX; 40 | 41 | private: 42 | static std::vector Gather(const int val); 43 | static std::vector AllGather(const int val); 44 | }; 45 | 46 | }; // namespace sledgehamr 47 | 48 | #endif // SLEDGEHAMR_LOCATION_H_ 49 | -------------------------------------------------------------------------------- /source/local_regrid/unique_layout.h: -------------------------------------------------------------------------------- 1 | #ifndef SLEDGEHAMR_UNIQUE_LAYOUT_H_ 2 | #define SLEDGEHAMR_UNIQUE_LAYOUT_H_ 3 | 4 | #include "local_regrid.h" 5 | 6 | namespace sledgehamr { 7 | 8 | class LocalRegrid; 9 | 10 | typedef unsigned short uit; 11 | typedef std::set row; 12 | typedef std::unordered_map plane; 13 | 14 | /** @brief This class ensures the uniqueness of a grid by reducing the problem 15 | * to sets of touples. Each core can work independently on its own 16 | * UniqueLayout which can then be efficiently merged across cores and 17 | * nodes to a global unique grid. Memory consumption scales with the 18 | * number of boxes added, not with the potential number of boxes. 19 | */ 20 | class UniqueLayout { 21 | public: 22 | UniqueLayout(LocalRegrid* local_regrid, const int N); 23 | 24 | void Add(const uit i, const uit j, const uit l) const; 25 | void Merge(std::vector >& uls); 26 | void Distribute(); 27 | 28 | void Clear(); 29 | bool Contains(const uit i, const uit j, const uit k) const; 30 | int Size(); 31 | int SizeAll(); 32 | 33 | amrex::BoxList BoxList(const int blocking_factor); 34 | amrex::BoxArray BoxArray(const int blocking_factor); 35 | 36 | private: 37 | void MergePlane(const uit cp, plane* pm); 38 | void IncorporatePlanes(); 39 | inline int Wrap(const int i); 40 | bool Owns(const uit cp); 41 | 42 | void SendDistribution(const int op); 43 | void RecvDistribution(const int op); 44 | inline void SendVector(const int op, const std::vector& v); 45 | inline std::vector RecvVector(const int op); 46 | 47 | /** Total number of planes. 48 | */ 49 | const uit Np; 50 | 51 | /** Number of planes owned by this node. 52 | */ 53 | uit Np_this; 54 | 55 | /** List of owners of each plane. 56 | */ 57 | std::vector< std::vector > owner_of; 58 | 59 | /** Vector of planes that require incorporation. 60 | */ 61 | std::vector nps; 62 | 63 | /** Array of Np planes. 64 | */ 65 | std::unique_ptr p; 66 | 67 | /** Number of nodes. 68 | */ 69 | const int mpi_n; 70 | 71 | /** Own MPI rank. 72 | */ 73 | const int mpi_mp; 74 | 75 | /** Pointer to LocalRegrid module. 76 | */ 77 | LocalRegrid* lr; 78 | }; 79 | 80 | }; // namespace sledgehamr 81 | 82 | #endif // SLEDGEHAMR_UNIQUE_LAYOUT_H_ 83 | -------------------------------------------------------------------------------- /source/main.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include "sledgehamr.h" 10 | #include "sledgehamr_init.h" 11 | 12 | int main(int argc, char *argv[]) { 13 | // Fix needed to accomodate change in amrex version 23.08. Will need long 14 | // term solution for this. 15 | amrex::ParmParse pp("amrex"); 16 | pp.add("the_arena_is_managed", 1); 17 | 18 | amrex::Initialize(argc, argv); 19 | 20 | // Timer for profiling. 21 | BL_PROFILE_VAR("main()", pmain); 22 | 23 | // wallclock time 24 | const double strt_total = amrex::ParallelDescriptor::second(); 25 | 26 | // Start sledgeHAMR. 27 | sledgehamr::SledgehamrInit init; 28 | sledgehamr::Sledgehamr *sledge = init.CreateInstance(); 29 | sledge->InitSledgehamr(); 30 | sledge->Evolve(); 31 | 32 | // Print runtime. 33 | double end_total = amrex::ParallelDescriptor::second() - strt_total; 34 | amrex::ParallelDescriptor::ReduceRealMax( 35 | end_total, amrex::ParallelDescriptor::IOProcessorNumber()); 36 | amrex::Print() << std::endl 37 | << "Total Run Time: " << end_total << "s" << std::endl; 38 | 39 | // Destroy timer for profiling. 40 | BL_PROFILE_VAR_STOP(pmain); 41 | 42 | amrex::Finalize(); 43 | } 44 | -------------------------------------------------------------------------------- /source/output_types/Make.package: -------------------------------------------------------------------------------- 1 | CEXE_headers += output_module.h 2 | CEXE_sources += output_module.cpp 3 | 4 | CEXE_headers += slices.h 5 | CEXE_sources += slices.cpp 6 | 7 | CEXE_headers += level_writer.h 8 | CEXE_sources += level_writer.cpp 9 | 10 | CEXE_headers += projection.h 11 | CEXE_sources += projection.cpp 12 | 13 | CEXE_headers += spectrum.h 14 | CEXE_sources += spectrum.cpp 15 | 16 | CEXE_headers += checkpoint.h 17 | CEXE_sources += checkpoint.cpp 18 | 19 | CEXE_headers += amrex_plotfile.h 20 | CEXE_sources += amrex_plotfile.cpp 21 | -------------------------------------------------------------------------------- /source/output_types/amrex_plotfile.cpp: -------------------------------------------------------------------------------- 1 | #include "amrex_plotfile.h" 2 | #include 3 | 4 | namespace sledgehamr { 5 | 6 | /** @brief Writes the AMReX output file. 7 | */ 8 | void AmrexPlotFile::Write() { 9 | std::string plotfilename = folder + "/output"; 10 | int nlevels = sim->GetFinestLevel() + 1; 11 | 12 | amrex::Vector r; 13 | amrex::Vector level_steps; 14 | amrex::Vector ref_ratio; 15 | double time; 16 | for (int lev = 0; lev < nlevels; ++lev) { 17 | LevelData* ld = &sim->GetLevelData(lev); 18 | r.push_back(ld); 19 | level_steps.push_back(ld->istep); 20 | ref_ratio.emplace_back(2,2,2); 21 | time = ld->t; 22 | } 23 | 24 | amrex::Vector varnames; 25 | for (int c = 0; c < r[0]->nComp(); ++c) { 26 | varnames.push_back(sim->GetScalarFieldName(c)); 27 | } 28 | 29 | amrex::WriteMultiLevelPlotfile(plotfilename, nlevels, r, varnames, 30 | sim->GetGeometry(), time, level_steps, 31 | ref_ratio); 32 | } 33 | 34 | }; // namespace sledgehamr 35 | -------------------------------------------------------------------------------- /source/output_types/amrex_plotfile.h: -------------------------------------------------------------------------------- 1 | #ifndef SLEDGEHAMR_OUTPUT_TYPES_AMREX_PLOTFILE_H_ 2 | #define SLEDGEHAMR_OUTPUT_TYPES_AMREX_PLOTFILE_H_ 3 | 4 | #include "sledgehamr.h" 5 | 6 | namespace sledgehamr { 7 | 8 | /** @brief Writes an AMReX plotfile for yt compatability. 9 | */ 10 | class AmrexPlotFile { 11 | public: 12 | /** @brief Constructior 13 | * @param owner Pointer to simulation. 14 | * @param prefix Local output folder. 15 | */ 16 | AmrexPlotFile(Sledgehamr* owner, std::string prefix) 17 | : sim(owner), folder(prefix) {}; 18 | 19 | void Write(); 20 | 21 | private: 22 | /** @brief Local output folder. 23 | */ 24 | std::string folder; 25 | 26 | /** @brief Pointer to simulation. 27 | */ 28 | Sledgehamr* sim; 29 | }; 30 | 31 | }; // namespace sledgehamr 32 | 33 | #endif // SLEDGEHAMR_OUTPUT_TYPES_AMREX_PLOTFILE_H_ 34 | -------------------------------------------------------------------------------- /source/output_types/checkpoint.h: -------------------------------------------------------------------------------- 1 | #ifndef SLEDGEHAMR_CHECKPOINT_H_ 2 | #define SLEDGEHAMR_CHECKPOINT_H_ 3 | 4 | #include "sledgehamr.h" 5 | 6 | namespace sledgehamr { 7 | 8 | /** @brief Writes or reads a checkpoint. 9 | */ 10 | class Checkpoint { 11 | public: 12 | /** @brief Constructor. 13 | * @param owner Pointer to simulation. 14 | * @param chk_folder Checkpoint directoy. 15 | */ 16 | Checkpoint(Sledgehamr* owner, std::string chk_folder) 17 | : sim(owner), folder(chk_folder) {}; 18 | 19 | void Write(); 20 | void Read(); 21 | bool ReadHeader(); 22 | void UpdateOutputModules(); 23 | void Delete(); 24 | 25 | /** @brief Returns the time of the checkpoint. 26 | */ 27 | double GetTime() const { 28 | return time; 29 | } 30 | 31 | private: 32 | static void GotoNextLine(std::istream& is); 33 | void UpdateLevels(); 34 | 35 | /** @brief Returns the full path to the meta data header file. 36 | */ 37 | std::string GetHeaderName() const { 38 | return folder + "/Meta.hdf5"; 39 | } 40 | 41 | /** @brief Returns the full path to the box array data file. 42 | */ 43 | std::string GetBoxArrayName() const { 44 | return folder + "/BoxArrays"; 45 | } 46 | 47 | /** @brief Returns the full path to the individual level data folder. 48 | * @param lev Level. 49 | */ 50 | std::string GetLevelDirName(const int lev) const { 51 | return folder + "/Level_" + std::to_string(lev); 52 | } 53 | 54 | /** @brief Pointer to Simulation. 55 | */ 56 | Sledgehamr* sim; 57 | 58 | /** @brief Checkpoint directory. 59 | */ 60 | std::string folder; 61 | 62 | /** @brief Time of checkpoint. 63 | */ 64 | double time; 65 | 66 | /** @brief Number of MPI ranks with which checkpoint was written. 67 | */ 68 | int MPIranks; 69 | 70 | /** @brief Finest level in checkpoint. 71 | */ 72 | int finest_level; 73 | 74 | /** @brief Coarse level grid cells along one axis in checkoint. 75 | */ 76 | int dim0; 77 | 78 | /** @brief Number of ghost cells in checkpoint. 79 | */ 80 | int nghost; 81 | 82 | /** @brief Number of scalar fields in checkpoint. 83 | */ 84 | int nscalars; 85 | 86 | /** @brief Total number of output types with meta data. 87 | */ 88 | int noutput; 89 | 90 | /** @brief Number of pre-defined output types with meta data. 91 | */ 92 | int npredefoutput; 93 | }; 94 | 95 | }; // namespace sledgehamr 96 | 97 | #endif // SLEDGEHAMR_CHECKPOINT_H_ 98 | -------------------------------------------------------------------------------- /source/output_types/level_writer.h: -------------------------------------------------------------------------------- 1 | #ifndef SLEDGEHAMR_OUTPUT_TYPES_LEVEL_WRITE_H_ 2 | #define SLEDGEHAMR_OUTPUT_TYPES_LEVEL_WRITE_H_ 3 | 4 | #include "sledgehamr.h" 5 | 6 | namespace sledgehamr { 7 | 8 | /** @brief Writes level data to disk. 9 | */ 10 | class LevelWriter { 11 | public: 12 | LevelWriter(Sledgehamr* owner, std::string prefix, int output_type); 13 | void Write(); 14 | 15 | private: 16 | void DetermineSetup(); 17 | void ParseParams(); 18 | void CheckDownsampleFactor(); 19 | void WriteSingleLevel(const LevelData* state, int lev, hid_t file_id, 20 | std::string ident, bool is_truncation_error); 21 | 22 | /** @brief Pointer to simulation. 23 | */ 24 | Sledgehamr* sim; 25 | 26 | /** @brief Local output folder. 27 | */ 28 | std::string folder; 29 | 30 | /** @brief Mininum level to write (inclusive). 31 | */ 32 | int level_min; 33 | 34 | /** @brief Maximum level to write (inclusive). 35 | */ 36 | int level_max; 37 | 38 | /** @brief Local unique output id. Must be either 39 | * - io_module->idx_coarse_box 40 | * - io_module->idx_coarse_box_truncation_error 41 | * - io_module->idx_full_box 42 | * - io_module->idx_full_box_truncation_error 43 | */ 44 | const int output_id; 45 | 46 | /** @brief Display name of output type. 47 | */ 48 | std::string name; 49 | 50 | /** @brief Information to print to screen for this output type. 51 | */ 52 | std::string info; 53 | 54 | /** @brief Whether we are including truncation errors estimates. 55 | */ 56 | bool with_truncation_errors; 57 | 58 | /** @brief Downsample data by this factor before writing. 59 | */ 60 | int downsample_factor = 1; 61 | }; 62 | 63 | }; // namespace sledgehamr 64 | 65 | #endif // SLEDGEHAMR_OUTPUT_TYPES_LEVEL_WRITE_H_ 66 | -------------------------------------------------------------------------------- /source/output_types/output_module.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | 5 | #include "output_module.h" 6 | 7 | namespace sledgehamr { 8 | 9 | /** @brief Constructor that provides the settings. 10 | * @param module_name Name of output (to appear in file path e.g.). 11 | * @param function Function pointer to function that writes the 12 | * actual output. 13 | * @param is_forcable If true, output will be written at the very end 14 | * of the simulation independent of the time 15 | * interval. 16 | */ 17 | OutputModule::OutputModule(std::string module_name, output_fct function, 18 | bool is_forceable) 19 | : fct(function), 20 | name(module_name), 21 | forceable(is_forceable) { 22 | ParseParams(); 23 | CreateParentFolder(prefix); 24 | if (alternate) 25 | CreateParentFolder(alt_prefix); 26 | } 27 | 28 | /** @brief Parses all parameters related to this output type. 29 | */ 30 | void OutputModule::ParseParams() { 31 | amrex::ParmParse pp_out("output"); 32 | pp_out.get("output_folder", prefix); 33 | pp_out.query("alternative_output_folder", alt_prefix); 34 | 35 | std::string pre = "output." + name; 36 | amrex::ParmParse pp(pre); 37 | pp.query("interval", interval); 38 | pp.query("alternate", alternate); 39 | pp.query("min_t", t_min); 40 | pp.query("max_t", t_max); 41 | 42 | if (alternate && alt_prefix == "") { 43 | std::string msg = "sledgehamr::OutputModule::ParseParams: " 44 | "Alternating output selected but no alternative " 45 | "output folder given"; 46 | amrex::Abort(msg); 47 | } 48 | } 49 | 50 | /** @brief Creates a parent folder for the output. 51 | * @param this_prefix parent folder. 52 | */ 53 | void OutputModule::CreateParentFolder(std::string this_prefix) { 54 | std::string output_folder = this_prefix + "/" + name; 55 | if (!amrex::UtilCreateDirectory(output_folder, 0755)) { 56 | std::string msg = "sledgehamr::OutputModule::CreateParentFolder: " 57 | "Could not create output folder " + output_folder; 58 | amrex::Abort(msg); 59 | } 60 | } 61 | 62 | /** @brief Does the actual writing if criteria are met. 63 | * @param time Current time. 64 | * @param force Output will be written independent of the current time 65 | * interval if forceable=true. 66 | */ 67 | void OutputModule::Write(double time, bool force) { 68 | if (interval < 0) return; 69 | 70 | // Check if it is time to write output. 71 | double t_now = time_modifier(time); 72 | double t_last = time_modifier(last_written); 73 | 74 | if (t_now > t_max || t_now < t_min) return; 75 | if (t_now - t_last < interval && (!force && forceable)) return; 76 | 77 | std::string this_prefix = (alternate && next_id%2 == 1) ? 78 | alt_prefix : prefix; 79 | 80 | // Create output folder. 81 | std::string folder = this_prefix + "/" + name + "/" 82 | + std::to_string(next_id); 83 | amrex::UtilCreateCleanDirectory(folder, true); 84 | folder += "/"; 85 | 86 | 87 | // Attempt to write. 88 | if (fct(time, folder)) { 89 | next_id++; 90 | last_written = time; 91 | amrex::Print() << "Wrote " << name << ": " << folder << std::endl; 92 | } else { 93 | std::filesystem::remove(folder); 94 | } 95 | } 96 | 97 | }; // namespace sledgehamr 98 | -------------------------------------------------------------------------------- /source/output_types/output_module.h: -------------------------------------------------------------------------------- 1 | #ifndef SLEDGEHAMR_OUTPUTMODULE_H_ 2 | #define SLEDGEHAMR_OUTPUTMODULE_H_ 3 | 4 | #include 5 | #include 6 | 7 | #include 8 | #include 9 | 10 | namespace sledgehamr { 11 | 12 | /** @brief Macro and typedef to make the handling of function pointers easier. 13 | * The function pointer will point to a function that does the actual 14 | * writing of output. 15 | */ 16 | #define OUTPUT_FCT(fct) std::bind(&fct, this, std::placeholders::_1,\ 17 | std::placeholders::_2) 18 | typedef std::function output_fct; 19 | 20 | /** @brief Macro and typedef to make it possible to modify the time interval. 21 | */ 22 | #define TIME_FCT(fct) std::bind(&fct, this, std::placeholders::_1) 23 | typedef std::function time_fct; 24 | 25 | /** @brief This class handles the writing of an individual output format 26 | * provided through a function pointer. It keeps track of timings to 27 | * check if this particular format should be written at the current time 28 | * or not. 29 | */ 30 | class OutputModule { 31 | public: 32 | OutputModule(std::string module_name, output_fct function, 33 | bool is_forceable=true); 34 | void Write(double time, bool force=false); 35 | 36 | /** @brief Change the time interval to something arbitrary. 37 | * @param mod Time modifier function. 38 | */ 39 | void SetTimeFunction(time_fct mod) { 40 | time_modifier = mod; 41 | }; 42 | 43 | /** @brief Default time interval is linear in time. 44 | * @param time Current time. 45 | */ 46 | double DefaultInterval(double time) { 47 | return time; 48 | }; 49 | 50 | /** @brief Change time interval width. 51 | */ 52 | void SetInterval(double new_interval) { 53 | interval = new_interval; 54 | }; 55 | 56 | /** @brief Get id of next output. 57 | */ 58 | int GetNextId() const { 59 | return next_id; 60 | }; 61 | 62 | /** @brief Sets the id of the next output. 63 | */ 64 | void SetNextId(int id) { 65 | next_id = id; 66 | }; 67 | 68 | /** @brief Sets the time we wrote output last. 69 | */ 70 | void SetLastTimeWritten(double time) { 71 | last_written = time; 72 | }; 73 | 74 | /** @brief Sets whether we want to alternate between output paths. 75 | */ 76 | void Alternate(bool do_alternate) { 77 | alternate = do_alternate; 78 | } 79 | 80 | /** @brief Sets and create alternative output folder. 81 | */ 82 | void SetAlternativePrefix(bool alternative_prefix) { 83 | alt_prefix = alternative_prefix; 84 | CreateParentFolder(alt_prefix); 85 | } 86 | 87 | /** @brief Returns the last time we wrote output. 88 | */ 89 | double GetLastTimeWritten() const { 90 | return last_written; 91 | }; 92 | 93 | /** @brief Returns the name of this output type. 94 | */ 95 | std::string GetName() const { 96 | return name; 97 | }; 98 | 99 | private: 100 | void ParseParams(); 101 | void CreateParentFolder(std::string this_prefix); 102 | 103 | /** @brief Function pointer to function that does the actual writing. 104 | */ 105 | output_fct fct; 106 | 107 | /** @brief Time interval modifier function. By default linear in time. 108 | */ 109 | time_fct time_modifier = TIME_FCT(OutputModule::DefaultInterval); 110 | 111 | /** @brief Next output index. Will be iterated +1 after each writing. 112 | */ 113 | int next_id = 0; 114 | 115 | /** @brief Time at which this output has been written last. 116 | */ 117 | double last_written = -DBL_MAX; 118 | 119 | /** @brief Interval at which this output shall be written. 120 | */ 121 | double interval = -1; 122 | 123 | /** @brief Prefix of the output folder. 124 | */ 125 | std::string prefix; 126 | 127 | /** @brief Prefix of alternative output folder. 128 | */ 129 | std::string alt_prefix = ""; 130 | 131 | /** @brief Whether we want to alternate between two output folders. 132 | */ 133 | bool alternate = false; 134 | 135 | /** @brief Minimum time before we start writing. 136 | */ 137 | double t_min = -DBL_MAX; 138 | 139 | /** @brief Maximum time before we start writing. 140 | */ 141 | double t_max = DBL_MAX; 142 | 143 | /** @brief Defines whether we can force a write, e.g. at the end of the 144 | * simulation. Default is forceable=true. 145 | */ 146 | bool forceable; 147 | 148 | /** @brief Output name. 149 | */ 150 | std::string name = "Unknown"; 151 | }; 152 | 153 | }; // namespace sledgehamr 154 | 155 | #endif // SLEDGEHAMR_OUTPUTMODULE_H_ 156 | -------------------------------------------------------------------------------- /source/output_types/projection.cpp: -------------------------------------------------------------------------------- 1 | #include "projection.h" 2 | #include "hdf5_utils.h" 3 | 4 | namespace sledgehamr { 5 | 6 | /** @brief Computes and saves a line-of-sight projection. 7 | * @param id Output counter id. 8 | * @param file_id HDF5 file id. 9 | * @param sim Pointer to the simulation. 10 | */ 11 | void Projection::Compute(const int id, const hid_t file_id, Sledgehamr* sim) { 12 | int mlevel = INT_MAX; 13 | amrex::ParmParse pp("output.projections"); 14 | pp.query("max_level", mlevel); 15 | mlevel = std::min(mlevel, sim->finest_level); 16 | 17 | const int dimN = sim->dimN[mlevel]; 18 | long long N = dimN*dimN; 19 | amrex::Vector d_projection(N); 20 | amrex::Vector n_projection(N); 21 | 22 | std::vector params; 23 | sim->SetParamsProjections(params, sim->grid_new[0].t); 24 | 25 | for (int lev = 0; lev <= mlevel; ++lev) { 26 | const int dimN_lev = sim->dimN[lev]; 27 | const int ratio = dimN / dimN_lev; 28 | const double dx = sim->dx[lev]; 29 | const double dt = sim->dt[lev]; 30 | const double time = sim->grid_new[lev].t; 31 | 32 | amrex::BoxArray ba; 33 | if (lev != mlevel) 34 | ba = sim->grid_new[lev+1].boxArray(); 35 | 36 | // No OpenMP for thread-safety. 37 | for (amrex::MFIter mfi(sim->grid_new[lev], false); mfi.isValid(); 38 | ++mfi) { 39 | const amrex::Box& bx = mfi.tilebox(); 40 | const auto& state_fab = sim->grid_new[lev].array(mfi); 41 | 42 | const amrex::Dim3 lo = amrex::lbound(bx); 43 | const amrex::Dim3 hi = amrex::ubound(bx); 44 | 45 | for (int k = lo.z; k <= hi.z; ++k) { 46 | for (int j = lo.y; j <= hi.y; ++j) { 47 | for (int i = lo.x; i <= hi.x; ++i) { 48 | bool contd = false; 49 | 50 | // Only include if cell is not refined. 51 | if (lev == mlevel ) { 52 | contd = true; 53 | } else { 54 | if (!ba.contains(amrex::IntVect(i*2, j*2, k*2))) 55 | contd = true; 56 | } 57 | 58 | if (contd) { 59 | double val = fct(state_fab, i, j, k, lev, time, dt, 60 | dx, params); 61 | Add(i, j, k, &d_projection[0], &n_projection[0], ratio, 62 | dimN, val); 63 | } 64 | } 65 | } 66 | } 67 | } 68 | } 69 | 70 | amrex::ParallelDescriptor::ReduceRealSum(d_projection.dataPtr(), N, 71 | amrex::ParallelDescriptor::IOProcessorNumber()); 72 | 73 | amrex::ParallelDescriptor::ReduceIntSum(n_projection.dataPtr(), N, 74 | amrex::ParallelDescriptor::IOProcessorNumber()); 75 | 76 | if (amrex::ParallelDescriptor::IOProcessor()) { 77 | if (id == 0) { 78 | const int nparams = 2; 79 | hsize_t dims[1] = {nparams}; 80 | double header_data[nparams] = {sim->grid_new[0].t, (double)dimN}; 81 | utils::hdf5::Write(file_id, "Header", header_data, nparams); 82 | } 83 | 84 | utils::hdf5::Write(file_id, ident + "_data", &d_projection[0], N); 85 | utils::hdf5::Write(file_id, ident + "_n", &n_projection[0], N); 86 | } 87 | } 88 | 89 | /** @brief Projection operator. 90 | * @param i i-th index of cell to project. 91 | * @param j j-th index of cell to project. 92 | * @param k k-th index of cell to project. 93 | * @param projection Existing projection. 94 | * @param n_projection Number of cells projected. 95 | * @param ratio Refinement ratio to coarse level. 96 | * @param dimN This levels number of theoretical cells along on 97 | * axis. 98 | * @param val Cells value. 99 | */ 100 | AMREX_FORCE_INLINE 101 | void Projection::Add(const int i, const int j, const int k, double* projection, 102 | int* n_projection, const int ratio, const int dimN, 103 | const double val) { 104 | // TODO different axis. 105 | int ii = i*ratio; 106 | int jj = j*ratio; 107 | for (int ia = 0; ia < ratio; ++ia) { 108 | for (int ja = 0; ja < ratio; ++ja) { 109 | long long ind = (ii + ia)*dimN + (jj+ja); 110 | if (mode == 0) { 111 | projection[ind] += ratio*val; 112 | } else if (mode == 1) { 113 | projection[ind] = std::max(projection[ind], val); 114 | } 115 | n_projection[ind] += 1; 116 | } 117 | } 118 | } 119 | 120 | }; // namespace sledgehamr 121 | -------------------------------------------------------------------------------- /source/output_types/projection.h: -------------------------------------------------------------------------------- 1 | #ifndef SLEDGEHAMR_PROJECTION_H_ 2 | #define SLEDGEHAMR_PROJECTION_H_ 3 | 4 | #include 5 | 6 | #include 7 | 8 | #include "sledgehamr.h" 9 | 10 | namespace sledgehamr { 11 | 12 | class Sledgehamr; 13 | 14 | /** @brief Function typdef that computes the quantity to project. 15 | */ 16 | typedef std::function const&, const int, 17 | const int, const int, const int, const double, const double, 18 | const double, const std::vector&)> projection_fct; 19 | 20 | /** @brief Computes a line-of-sight projection for an arbitrary quantity and 21 | * saves it to disk. Will up-sample the 2D projection to a finer level. 22 | */ 23 | class Projection { 24 | public: 25 | /** @brief Gather metadata. 26 | * @param function Computes quantity to project. 27 | * @param identification Unique name of projection. 28 | * @param projection_mode Projection mode. See mode for more details. 29 | */ 30 | Projection(projection_fct function, std::string identification, 31 | int projection_mode = 0) 32 | : fct{function}, ident{identification}, mode{projection_mode} {}; 33 | 34 | void Compute(const int id, const hid_t file_id, Sledgehamr* sim); 35 | 36 | /** @brief Function pointer to projection function. 37 | */ 38 | projection_fct fct; 39 | 40 | /** @brief Unique name of projection. 41 | */ 42 | std::string ident = "None"; 43 | 44 | /** @brief Projection mode: 45 | 0: Line-of-sight sum. 46 | 1: Maximum along the line-of-sight. 47 | */ 48 | int mode = 0; 49 | 50 | private: 51 | void Add(const int i, const int j, const int k, double* projection, 52 | int* n_projection, const int ratio, const int dimN, 53 | const double val); 54 | }; 55 | 56 | }; // namespace sledgehamr 57 | 58 | #endif // SLEDGEHAMR_PROJECTION_H_ 59 | -------------------------------------------------------------------------------- /source/output_types/slices.cpp: -------------------------------------------------------------------------------- 1 | #include "slices.h" 2 | #include "hdf5_utils.h" 3 | 4 | namespace sledgehamr { 5 | 6 | /** @brief Writes slices along all the directions through all scalar fields and 7 | * levels. 8 | */ 9 | void Slices::Write() { 10 | for (int lev = 0; lev <= sim->GetFinestLevel(); ++lev) { 11 | const LevelData* state = &sim->GetLevelData(lev); 12 | if (with_truncation_errors && !state->contains_truncation_errors ) 13 | continue; 14 | 15 | // Create folder and file. 16 | std::string subfolder = folder + "/Level_" + std::to_string(lev); 17 | amrex::UtilCreateDirectory(subfolder.c_str(), 0755); 18 | 19 | std::string filename = subfolder + "/" 20 | + std::to_string(amrex::ParallelDescriptor::MyProc()) 21 | + ".hdf5"; 22 | hid_t file_id = H5Fcreate(filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, 23 | H5P_DEFAULT); 24 | 25 | // Write field data. 26 | WriteSingleSlice(state, lev, file_id, "x", 0, 1, 2, false); 27 | WriteSingleSlice(state, lev, file_id, "y", 1, 0, 2, false); 28 | WriteSingleSlice(state, lev, file_id, "z", 2, 0, 1, false); 29 | 30 | if (with_truncation_errors) { 31 | // Write truncation errors. 32 | const LevelData* state_old = &sim->GetOldLevelData(lev); 33 | WriteSingleSlice(state_old, lev, file_id, "te_x", 0, 1, 2, true); 34 | WriteSingleSlice(state_old, lev, file_id, "te_y", 1, 0, 2, true); 35 | WriteSingleSlice(state_old, lev, file_id, "te_z", 2, 0, 1, true); 36 | } 37 | 38 | H5Fclose(file_id); 39 | } 40 | } 41 | 42 | /** @brief Writes a single slices. 43 | * @param state State data. 44 | * @param lev Current level. 45 | * @param file_id HDF5 file id. 46 | * @param ident Unique identifier string. 47 | * @param d1 Orientation 1. 48 | * @param d2 Orientation 2. 49 | * @param d3 Orientation 3. 50 | * @param is_truncation_error Whether state data contains truncation error 51 | * estimates. 52 | */ 53 | void Slices::WriteSingleSlice(const LevelData* state, int lev, hid_t file_id, 54 | std::string ident, int d1, int d2, int d3, 55 | bool is_truncation_error) { 56 | std::vector le1, he1, le2, he2; 57 | const int ndist = is_truncation_error ? 2 : 1; 58 | 59 | // Not performance critical. Do not use OpenMP because not thread-safe. 60 | for (amrex::MFIter mfi(*state, false); mfi.isValid(); ++mfi){ 61 | const amrex::Box& bx = mfi.tilebox(); 62 | 63 | // Find box that cuts through selected plane. 64 | if (bx.smallEnd(d1) == 0) { 65 | const auto& state_arr = state->array(mfi); 66 | 67 | // Get box dimensions 68 | int l1 = bx.smallEnd(d2); 69 | int l2 = bx.smallEnd(d3); 70 | int h1 = bx.bigEnd(d2)+1; 71 | int h2 = bx.bigEnd(d3)+1; 72 | le1.push_back(l1); 73 | le2.push_back(l2); 74 | he1.push_back(h1); 75 | he2.push_back(h2); 76 | 77 | int dim1 = (h1-l1)/ndist; 78 | int dim2 = (h2-l2)/ndist; 79 | 80 | // Copy data into flattened array for each scalar field. 81 | long len = dim1 * dim2; 82 | // TODO Adjust output type. 83 | // std::unique_ptr output_arr(new float[len]); 84 | // std::fill_n(output_arr.get(), len, 0.0f); 85 | std::vector output_arr(len); 86 | 87 | for (int f=0; fnComp(); ++f) { 88 | for (int j=l2; jGetScalarFieldName(f) 106 | + "_" + ident 107 | + "_" + std::to_string(le1.size()); 108 | utils::hdf5::Write(file_id, dset_name, &(output_arr[0]), len); 109 | //utils::hdf5::Write(file_id, dset_name, output_arr.get(), len); 110 | } 111 | // output_arr.reset(); 112 | } 113 | } 114 | 115 | // Write header information for this slice. 116 | const int nparams = 5; 117 | double header_data[nparams] = { 118 | state->t, 119 | (double)amrex::ParallelDescriptor::NProcs(), 120 | (double)(sim->GetFinestLevel()), 121 | (double)sim->GetDimN(lev), 122 | (double)le1.size()}; 123 | utils::hdf5::Write(file_id, "Header_"+ident, header_data, nparams); 124 | 125 | // Write box dimensions so we can reassemble slice. 126 | if (le1.size() == 0) 127 | return; 128 | 129 | utils::hdf5::Write(file_id, "le1_"+ident, (int*)&(le1[0]), le1.size()); 130 | utils::hdf5::Write(file_id, "le2_"+ident, (int*)&(le2[0]), le2.size()); 131 | utils::hdf5::Write(file_id, "he1_"+ident, (int*)&(he1[0]), he1.size()); 132 | utils::hdf5::Write(file_id, "he2_"+ident, (int*)&(he2[0]), he2.size()); 133 | } 134 | 135 | }; // namespace sledgehamr 136 | -------------------------------------------------------------------------------- /source/output_types/slices.h: -------------------------------------------------------------------------------- 1 | #ifndef SLEDGEHAMR_OUTPUT_TYPES_SLICES_H_ 2 | #define SLEDGEHAMR_OUTPUT_TYPES_SLICES_H_ 3 | 4 | #include "sledgehamr.h" 5 | 6 | namespace sledgehamr { 7 | 8 | /** @brief Writes slices through the field to disk. 9 | */ 10 | class Slices { 11 | public: 12 | /** @brief Collect metadata. 13 | * @param owner Pointer to simulation. 14 | * @param prefix Local output path. 15 | * @param write_with_truncation_errors Whether to include truncation 16 | error estimates. 17 | */ 18 | Slices(Sledgehamr* owner, std::string prefix, 19 | bool write_with_truncation_errors) 20 | : sim(owner), 21 | folder(prefix), 22 | with_truncation_errors(write_with_truncation_errors) {}; 23 | 24 | void Write(); 25 | 26 | private: 27 | void WriteSingleSlice(const LevelData* state, int lev, hid_t file_id, 28 | std::string ident, int d1, int d2, int d3, 29 | bool is_truncation_error); 30 | 31 | /** @brief Local output folder. 32 | */ 33 | std::string folder; 34 | 35 | /** @brief Whether to include truncation error estimates. 36 | */ 37 | bool with_truncation_errors; 38 | 39 | /** @brief Pointer to simulation. 40 | */ 41 | Sledgehamr* sim; 42 | }; 43 | 44 | }; // namespace sledgehamr 45 | 46 | #endif // SLEDGEHAMR_OUTPUT_TYPES_SLICES_H_ 47 | -------------------------------------------------------------------------------- /source/output_types/spectrum.cpp: -------------------------------------------------------------------------------- 1 | #include "spectrum.h" 2 | #include "fft.h" 3 | #include "hdf5_utils.h" 4 | 5 | namespace sledgehamr { 6 | 7 | /** @brief Computes a spectrum and saves it to disk. 8 | * @param id Output counter id. 9 | * @param file_id HDF5 file id. 10 | * @param sim Pointer to the simulation. 11 | */ 12 | void Spectrum::Compute(const int id, const hid_t file_id, Sledgehamr *sim) { 13 | const int lev = 0; 14 | const int dimN = sim->dimN[lev]; 15 | const double dx = sim->dx[lev]; 16 | const double dt = sim->dt[lev]; 17 | const double time = sim->grid_new[lev].t; 18 | const LevelData &state = sim->grid_new[lev]; 19 | const amrex::BoxArray &ba = state.boxArray(); 20 | 21 | amrex::MultiFab field, field_fft; 22 | field.define(ba, sim->dmap[lev], 1, 0); 23 | field_fft.define(ba, sim->dmap[lev], 1, 0); 24 | 25 | std::vector params; 26 | sim->SetParamsSpectra(params, time); 27 | 28 | #pragma omp parallel 29 | for (amrex::MFIter mfi(field, true); mfi.isValid(); ++mfi) { 30 | const amrex::Box &bx = mfi.tilebox(); 31 | const auto &field_arr = field.array(mfi); 32 | const auto &state_arr = state.array(mfi); 33 | 34 | const amrex::Dim3 lo = amrex::lbound(bx); 35 | const amrex::Dim3 hi = amrex::ubound(bx); 36 | 37 | for (int k = lo.z; k <= hi.z; ++k) { 38 | for (int j = lo.y; j <= hi.y; ++j) { 39 | for (int i = lo.x; i <= hi.x; ++i) { 40 | field_arr(i, j, k, 0) = 41 | fct(state_arr, i, j, k, lev, time, dt, dx, params); 42 | } 43 | } 44 | } 45 | } 46 | 47 | utils::Fft(field, 0, field_fft, field_fft, sim->geom[lev], true); 48 | 49 | double fac = pow(1. / dimN, 6); 50 | double dk = 2. * M_PI / sim->L; 51 | double pre = fac * state.t / dk; 52 | 53 | std::vector &ks = sim->spectrum_ks; 54 | const int kmax = ks.size(); 55 | constexpr int NTHREADS = 16; 56 | const unsigned long SpecLen = kmax * NTHREADS; 57 | std::unique_ptr spectrum(new double[SpecLen]); 58 | std::fill_n(spectrum.get(), SpecLen, 0.0); 59 | 60 | #pragma omp parallel num_threads(std::min(NTHREADS, omp_get_max_threads())) 61 | for (amrex::MFIter mfi(field_fft, true); mfi.isValid(); ++mfi) { 62 | const amrex::Box &bx = mfi.tilebox(); 63 | const amrex::Array4 &field_fft_arr = field_fft.array(mfi); 64 | 65 | const int il = bx.smallEnd()[0]; 66 | const int ih = bx.bigEnd()[0]; 67 | const int jl = bx.smallEnd()[1]; 68 | const int jh = bx.bigEnd()[1]; 69 | const int kl = bx.smallEnd()[2]; 70 | const int kh = bx.bigEnd()[2]; 71 | 72 | for (int i = il; i <= ih; ++i) { 73 | for (int j = jl; j <= jh; ++j) { 74 | AMREX_PRAGMA_SIMD 75 | for (int k = kl; k <= kh; ++k) { 76 | int li = i >= dimN / 2 ? i - dimN : i; 77 | int lj = j >= dimN / 2 ? j - dimN : j; 78 | int lk = k >= dimN / 2 ? k - dimN : k; 79 | unsigned int sq = li * li + lj * lj + lk * lk; 80 | unsigned long index = 81 | std::lower_bound(ks.begin(), ks.end(), sq) - 82 | ks.begin() + omp_get_thread_num() * kmax; 83 | spectrum[index] += pre * field_fft_arr(i, j, k, 0) * 84 | field_fft_arr(i, j, k, 0); 85 | } 86 | } 87 | } 88 | } 89 | 90 | for (int a = 1; a < NTHREADS; ++a) { 91 | for (int c = 0; c < kmax; ++c) { 92 | spectrum[c] += spectrum[a * kmax + c]; 93 | } 94 | } 95 | 96 | amrex::ParallelDescriptor::ReduceRealSum( 97 | spectrum.get(), kmax, amrex::ParallelDescriptor::IOProcessorNumber()); 98 | 99 | if (amrex::ParallelDescriptor::IOProcessor()) { 100 | if (id == 0) { 101 | const int nparams = 3; 102 | double header_data[nparams] = {sim->grid_new[lev].t, (double)dimN, 103 | (double)kmax}; 104 | utils::hdf5::Write(file_id, "Header", header_data, nparams); 105 | utils::hdf5::Write(file_id, "k_sq", &(ks[0]), kmax); 106 | } 107 | 108 | utils::hdf5::Write(file_id, ident, spectrum.get(), kmax); 109 | } 110 | } 111 | 112 | }; // namespace sledgehamr 113 | -------------------------------------------------------------------------------- /source/output_types/spectrum.h: -------------------------------------------------------------------------------- 1 | #ifndef SLEDGEHAMR_SPECTRUM_H_ 2 | #define SLEDGEHAMR_SPECTRUM_H_ 3 | 4 | #include 5 | 6 | #include "sledgehamr.h" 7 | 8 | namespace sledgehamr { 9 | 10 | class Sledgehamr; 11 | 12 | typedef std::function const&, const int, 13 | const int, const int, const int, const double, const double, 14 | const double, const std::vector&)> spectrum_fct; 15 | 16 | /** @brief Computes the spectrum given a quantity function and saves it to disk. 17 | */ 18 | class Spectrum { 19 | public: 20 | /** @brief Collects metadata. 21 | * @param function Integrand. 22 | * @param identification Unique identification string. 23 | */ 24 | Spectrum(spectrum_fct function, std::string identification) 25 | : fct{function}, ident{identification} { }; 26 | 27 | void Compute(const int id, const hid_t file_id, Sledgehamr* sim); 28 | 29 | static void Fft(const amrex::MultiFab& field, const int comp, 30 | amrex::MultiFab& field_fft_real_or_abs, 31 | amrex::MultiFab& field_fft_imag, 32 | const amrex::Geometry& geom, bool abs); 33 | 34 | /** @brief Function pointer to integrand. 35 | */ 36 | spectrum_fct fct; 37 | 38 | /** @brief Unique identification string. 39 | */ 40 | std::string ident = "None"; 41 | }; 42 | 43 | }; // namespace sledgehamr 44 | 45 | #endif // SLEDGEHAMR_SPECTRUM_H_ 46 | -------------------------------------------------------------------------------- /source/performance_monitor.cpp: -------------------------------------------------------------------------------- 1 | #include "performance_monitor.h" 2 | #include "sledgehamr_utils.h" 3 | 4 | namespace sledgehamr { 5 | 6 | /** @brief Initalizes timers for all tracked functions. 7 | * @param owner Pointer to simulation. 8 | */ 9 | PerformanceMonitor::PerformanceMonitor(Sledgehamr* owner) 10 | : sim{owner} { 11 | amrex::ParmParse pp("output.performance_monitor"); 12 | pp.query("interval", interval); 13 | active = (interval > 0); 14 | 15 | if (!active) 16 | return; 17 | 18 | idx_total = timer.size(); 19 | timer.emplace_back("Total time"); 20 | timer[idx_total].Start(); 21 | 22 | // + 1 for shadow level support. 23 | idx_rhs = timer.size() + 1; 24 | for(int lev = -1; lev <= sim->max_level; ++lev) { 25 | std::string post = utils::LevelName(lev); 26 | timer.emplace_back("::Rhs " + post); 27 | } 28 | 29 | idx_fill_patch = timer.size() + 1; 30 | for(int lev = -1; lev <= sim->max_level; ++lev) { 31 | std::string post = utils::LevelName(lev); 32 | timer.emplace_back("LevelSynchronizer::FillPatch " + post); 33 | } 34 | 35 | idx_fill_intermediate_patch = timer.size() + 1; 36 | for(int lev = -1; lev <= sim->max_level; ++lev) { 37 | std::string post = utils::LevelName(lev); 38 | timer.emplace_back("LevelSynchronizer::FillIntermediatePatch " + post); 39 | } 40 | 41 | idx_average_down = timer.size() + 1; 42 | for(int lev = -1; lev <= sim->max_level; ++lev) { 43 | std::string post = utils::LevelName(lev); 44 | timer.emplace_back("LevelSynchronizer::AverageDownTo " + post); 45 | } 46 | 47 | idx_truncation_error = timer.size() + 1; 48 | for(int lev = -1; lev <= sim->max_level; ++lev) { 49 | std::string post = utils::LevelName(lev); 50 | timer.emplace_back( 51 | "LevelSynchronizer::ComputeTruncationErrors " + post); 52 | } 53 | 54 | idx_tagging = timer.size() + 1; 55 | for(int lev = -1; lev <= sim->max_level; ++lev) { 56 | std::string post = utils::LevelName(lev); 57 | timer.emplace_back("Sledgehamr::ErrorEst " + post); 58 | } 59 | 60 | idx_local_regrid = timer.size() + 1; 61 | for(int lev = -1; lev <= sim->max_level; ++lev) { 62 | std::string post = utils::LevelName(lev); 63 | timer.emplace_back("LocalRegrid::AttemptRegrid " + post 64 | + " (and higher)"); 65 | } 66 | 67 | idx_global_regrid = timer.size() + 1; 68 | for(int lev = -1; lev <= sim->max_level; ++lev) { 69 | std::string post = utils::LevelName(lev); 70 | timer.emplace_back("AmrCore::regrid " + post + " (and higher)"); 71 | } 72 | 73 | idx_read_input = timer.size(); 74 | if (sim->restart_sim) 75 | timer.emplace_back("IOModule::RestartSim"); 76 | else 77 | timer.emplace_back("Sledgehamr::InitFromScratch"); 78 | 79 | idx_output = timer.size(); 80 | for(OutputModule& out : sim->io_module->output) { 81 | timer.emplace_back("OutputModule::Write " + out.GetName()); 82 | } 83 | } 84 | 85 | /** @brief Starts a timer. 86 | * @param id ID of timer to be started. 87 | * @param offset Offset to be added to the timer ID. 88 | */ 89 | void PerformanceMonitor::Start(int id, int offset) { 90 | if (active) 91 | timer[id + offset].Start(); 92 | } 93 | 94 | /** @brief Stops a timer. 95 | * @param id ID of timer to be stopped. 96 | * @param offset Offset to be added to the timer ID. 97 | * @return Time passed since start of timer in seconds. If performance monitor 98 | * has not been active then -DBL_MAX will be returned. 99 | */ 100 | double PerformanceMonitor::Stop(int id, int offset) { 101 | if (active) { 102 | timer[id + offset].Stop(); 103 | return timer[id + offset].GetLastDurationSeconds(); 104 | } else { 105 | return -DBL_MAX; 106 | } 107 | } 108 | 109 | /** @brief Sorts all timers by total time passed. 110 | * @param timers Vector of timers. 111 | * @return Vector of indices that would sort the timers. 112 | */ 113 | std::vector PerformanceMonitor::TimerArgsort(std::vector timers) { 114 | std::vector idx(timers.size()); 115 | std::iota(idx.begin(), idx.end(), 0); 116 | 117 | std::stable_sort(idx.begin(), idx.end(), 118 | [&timers](int i1, int i2) { 119 | return timers[i1].GetTotalTimeSeconds() 120 | > timers[i2].GetTotalTimeSeconds(); 121 | }); 122 | 123 | return idx; 124 | } 125 | 126 | /** @brief Prints total time passed of all timers. 127 | * @param file_id HDF5 file to log the times. Currently not implemented. 128 | */ 129 | void PerformanceMonitor::Log(hid_t file_id) { 130 | std::vector idx = TimerArgsort(timer); 131 | 132 | amrex::Print() << " ------------------------ PERFORMANCE" 133 | << " ------------------------------------\n"; 134 | for(int i : idx) { 135 | double d = timer[i].GetTotalTimeSeconds(); 136 | if (d != 0) { 137 | amrex::Print() << std::left << std::setw(60) << timer[i].GetName() 138 | << d << "s\n"; 139 | } 140 | } 141 | amrex::Print() << " ------------------------------------" 142 | << "-------------------------------------" << std::endl; 143 | } 144 | 145 | }; // namespace sledgehamr 146 | -------------------------------------------------------------------------------- /source/performance_monitor.h: -------------------------------------------------------------------------------- 1 | #ifndef SLEDGEHAMR_PERFORMANCE_MONITOR_H_ 2 | #define SLEDGEHAMR_PERFORMANCE_MONITOR_H_ 3 | 4 | #include "sledgehamr.h" 5 | #include "timer.h" 6 | 7 | namespace sledgehamr { 8 | 9 | class PerformanceMonitor { 10 | public: 11 | PerformanceMonitor(Sledgehamr* owner); 12 | 13 | void Start(int id, int offset = 0); 14 | double Stop(int id, int offset = 0); 15 | void Log(hid_t file_id); 16 | 17 | /** @brief Returns whether the performance monitor is active. 18 | */ 19 | bool IsActive() const { 20 | return active; 21 | }; 22 | 23 | /** @brief IDs of timers. 24 | */ 25 | int idx_total = -1; 26 | int idx_rhs = -1; 27 | int idx_fill_patch = -1; 28 | int idx_fill_intermediate_patch = -1; 29 | int idx_average_down = -1; 30 | int idx_truncation_error = -1; 31 | int idx_tagging = -1; 32 | int idx_local_regrid = -1; 33 | int idx_global_regrid = -1; 34 | int idx_read_input = -1; 35 | int idx_output = -1; 36 | 37 | /** @brief Vector of all timers. 38 | */ 39 | std::vector timer; 40 | 41 | private: 42 | std::vector TimerArgsort(std::vector timers); 43 | 44 | /** @brief Interval at which we want to print out times. 45 | */ 46 | double interval = -1; 47 | 48 | /** @brief Is the performance monitor active. 49 | */ 50 | bool active = false; 51 | 52 | /** @brief Pointer to the simulation. 53 | */ 54 | Sledgehamr* sim; 55 | }; 56 | 57 | }; // namespace sledgehamr 58 | 59 | #endif // SLEDGEHAMR_PERFORMANCE_MONITOR_H_ 60 | -------------------------------------------------------------------------------- /source/regrid_scheduler.cpp: -------------------------------------------------------------------------------- 1 | #include "regrid_scheduler.h" 2 | 3 | namespace sledgehamr { 4 | 5 | /** @brief Schedules a regrid. 6 | * @param lev Lowest level to be regridded. 7 | * @param t Regrid time. 8 | */ 9 | void RegridScheduler::Schedule(int lev, double t) { 10 | int id = FindSchedule(t); 11 | 12 | if (id < 0) { 13 | schedule.emplace_back(lev, t); 14 | } else { 15 | schedule[id].lowest_level = std::min(lev, schedule[id].lowest_level); 16 | } 17 | } 18 | 19 | /** @brief Whether we have scheduled a regrid for this level and time. 20 | * @param lev Level. 21 | * @param t Time. 22 | * @return Whether it has been scheduled or not. 23 | */ 24 | bool RegridScheduler::DoRegrid(int lev, double t) const { 25 | int id = FindSchedule(t); 26 | 27 | if (id < 0) 28 | return false; 29 | 30 | return (lev == schedule[id].lowest_level); 31 | } 32 | 33 | /** @brief Whether we have scheduled a regrid for this level and time and need 34 | * to compute truncation errors. 35 | * @param lev Level. 36 | * @param t Time. 37 | * @return Whether it has been scheduled or not. 38 | */ 39 | bool RegridScheduler::NeedTruncationError(int lev, double t) const { 40 | int id = FindSchedule(t); 41 | 42 | if (id < 0) 43 | return false; 44 | 45 | return (lev >= schedule[id].lowest_level); 46 | } 47 | 48 | /** @brief Removes all scheduled regrids at the given time. 49 | * @param t Time. 50 | */ 51 | void RegridScheduler::DidRegrid(double t) { 52 | int id = FindSchedule(t); 53 | 54 | if (id >= 0) { 55 | schedule.erase(schedule.begin() + id); 56 | } 57 | } 58 | 59 | /** @brief Finds the internal ID of a regrid scheduled at the given time. 60 | * @param t Time. 61 | * @return Internal ID of scheduled regrid. Returns -1 if none is found. 62 | */ 63 | int RegridScheduler::FindSchedule(double t) const { 64 | int id = -1; 65 | 66 | if (!schedule.empty()) 67 | id = std::distance(schedule.begin(), std::find(schedule.begin(), 68 | schedule.end(), t)); 69 | 70 | if (id == schedule.size()) 71 | id = -1; 72 | 73 | return id; 74 | } 75 | 76 | }; // namespace sledgehamr 77 | -------------------------------------------------------------------------------- /source/regrid_scheduler.h: -------------------------------------------------------------------------------- 1 | #ifndef SLEDGEHAMR_REGRID_SCHEDULER_H_ 2 | #define SLEDGEHAMR_REGRID_SCHEDULER_H_ 3 | 4 | #include "sledgehamr.h" 5 | 6 | namespace sledgehamr { 7 | 8 | class SledgeHAMR; 9 | 10 | /** @brief Simple class that holds a single scheduled regrid. This is used by 11 | * the RegridScheduler. 12 | */ 13 | class ScheduledRegrid { 14 | public: 15 | /** @brief Does nothing but fetch meta data. 16 | * @param lev Lowest level to be regridded. 17 | * @param t_regrid Time at which we want to regrid. 18 | */ 19 | ScheduledRegrid(int lev, double t_regrid) 20 | : lowest_level{lev}, t{t_regrid} {}; 21 | 22 | /** @brief Checks whether this regrid has been scheduled at a given time. 23 | * @param time Time to match. 24 | * @return Whether times match. 25 | */ 26 | bool operator==(const double time) const { 27 | double teps = t*1e-12; 28 | return (t > time - teps && t < time + teps); 29 | } 30 | 31 | /** @brief Lowest level to be regridded. 32 | */ 33 | int lowest_level; 34 | 35 | /** @brief Regrid time. 36 | */ 37 | double t; 38 | }; 39 | 40 | /** @brief A class that helps us keep track of when and on what levels we want 41 | * to regrid. That way we can adjust our workflow accordingly by e.g. 42 | * computing truncation error estimates ahead of time or creating 43 | * shadow level. 44 | */ 45 | class RegridScheduler { 46 | public: 47 | /** @brief Does nothing. 48 | */ 49 | RegridScheduler() {}; 50 | 51 | void Schedule(int lev, double t); 52 | bool DoRegrid(int lev, double t) const; 53 | bool NeedTruncationError(int lev, double t) const; 54 | void DidRegrid(double t); 55 | 56 | private: 57 | int FindSchedule(double t) const; 58 | 59 | /** @brief Vector containing all scheduled regrids. 60 | */ 61 | std::vector schedule; 62 | }; 63 | 64 | }; // namespace sledgehamr 65 | 66 | #endif // SLEDGEHAMR_REGRID_SCHEDULER_H_ 67 | -------------------------------------------------------------------------------- /source/scalars.h: -------------------------------------------------------------------------------- 1 | #ifndef SLEDGEHAMR_SCALARS_H_ 2 | #define SLEDGEHAMR_SCALARS_H_ 3 | 4 | #include "sledgehamr.h" 5 | 6 | namespace sledgehamr { 7 | 8 | /** @brief Class to keep track of scalar fields. 9 | */ 10 | class ScalarField{ 11 | public: 12 | /** @brief Upon construction this class will automatically add itself to a 13 | * scalar field vector sfv in order to be simulated by the code. 14 | * It will store its amrex::MultiFab component number as id. 15 | * @param str Name of scalar field. 16 | * @param sfv Vector of scalar fields to which this field will be 17 | * added. 18 | * @param is_momentum Whether this component is a conjugate field. 19 | */ 20 | ScalarField(std::string str, std::vector& sfv, 21 | bool is_momentum) 22 | : name(str), id(sfv.size()) { 23 | sfv.push_back(this); 24 | }; 25 | 26 | /** @brief Convenient operator to return internal id. 27 | */ 28 | operator int() const { 29 | return id; 30 | }; 31 | 32 | /** @brief Name of field by which it will be referred to in any input and 33 | * output. 34 | */ 35 | std::string name; 36 | 37 | /** @brief Internal id. Corresponds to component in amrex::Multifab. 38 | */ 39 | const int id; 40 | 41 | /** @brief Whether this component is a conjugate field. 42 | */ 43 | bool is_conjugate_momentum; 44 | }; 45 | 46 | }; // namespace sledgehamr 47 | 48 | #endif // SLEDGEHAMR_SCALARS_H_ 49 | -------------------------------------------------------------------------------- /source/sledgehamr_init.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | #include "sledgehamr_init.h" 8 | #include "projects.h" 9 | 10 | namespace sledgehamr { 11 | 12 | /** @brief General initialization needed before sledgehamr itself can be 13 | * initialized. 14 | */ 15 | SledgehamrInit::SledgehamrInit() { 16 | DetermineProjectName(); 17 | FinishAMReXSetup(); 18 | } 19 | 20 | /** @brief Will return a pointer to a Sledgehamr instance depending on which 21 | * project has been selected. This class will remain owner of instance 22 | * but given the singleton nature of everything we just ignore this. 23 | * We do an implicit cast here from derived to base. 24 | * return Sledgehamr instance. NULL if no corresponding project has been 25 | * found. 26 | */ 27 | Sledgehamr* SledgehamrInit::CreateInstance() { 28 | // Call macro generated by 'make'. Will create and return new instance of 29 | // derived project class corresponding to the requested project name. Will 30 | // do nothing if no matching project class found. Implicit cast to base upon 31 | // return. 32 | SLEDGEHAMR_PROJECT(project_name); 33 | amrex::Print() << "Project not found!" << std::endl; 34 | return NULL; 35 | } 36 | 37 | /** @brief Sets project name using the inputs file. 38 | */ 39 | void SledgehamrInit::DetermineProjectName() { 40 | amrex::ParmParse pp("project"); 41 | pp.get("name",project_name); 42 | } 43 | 44 | /** @brief Overrides and/or sets parameters to force correct behaviour of 45 | * amrex::AmrCore. 46 | */ 47 | void SledgehamrInit::FinishAMReXSetup() { 48 | // Determine requested grid size. 49 | amrex::ParmParse pp_amr("amr"); 50 | int grid_size = 0; 51 | pp_amr.get("coarse_level_grid_size", grid_size); 52 | std::vector grid_vect(3,grid_size); 53 | pp_amr.addarr("n_cell", grid_vect); 54 | pp_amr.add("max_grid_size", grid_size); 55 | 56 | // Determine maximum number of levels. 57 | int max_refinement_levels = 0; 58 | pp_amr.get("max_refinement_levels", max_refinement_levels); 59 | pp_amr.add("max_level", max_refinement_levels); 60 | 61 | // Set box dimensions. 62 | amrex::ParmParse pp_sim("sim"); 63 | double L; 64 | pp_sim.get("L", L); 65 | 66 | std::vector prob_lo(3,0); 67 | std::vector prob_hi(3,L); 68 | std::vector periodic(3,1); 69 | 70 | amrex::ParmParse pp_geo("geometry"); 71 | pp_geo.addarr("prob_lo", prob_lo); 72 | pp_geo.addarr("prob_hi", prob_hi); 73 | pp_geo.addarr("is_periodic", periodic); 74 | 75 | // Set Integrator type. 76 | amrex::ParmParse pp_inte("integrator"); 77 | int inte_type; 78 | pp_inte.get("type", inte_type); 79 | 80 | amrex::ParmParse pp_intn("integration"); 81 | if (inte_type >= 0 && inte_type <= 4) { 82 | pp_intn.add("type", 1); 83 | pp_intn.add("rk.type", inte_type); 84 | } 85 | } 86 | 87 | }; // namespace sledgehamr 88 | -------------------------------------------------------------------------------- /source/sledgehamr_init.h: -------------------------------------------------------------------------------- 1 | #ifndef SLEDGEHAMR_SLEDGEHAMRINIT_H_ 2 | #define SLEDGEHAMR_SLEDGEHAMRINIT_H_ 3 | 4 | #include "sledgehamr.h" 5 | 6 | namespace sledgehamr { 7 | 8 | /** @brief Determines with project has been requested and creates correct 9 | * sledgehamr instance. Also feeds extra derived information to 10 | * amrex::AmrCore to make sure it plays nicely. 11 | */ 12 | class SledgehamrInit { 13 | public: 14 | SledgehamrInit(); 15 | Sledgehamr* CreateInstance(); 16 | 17 | private: 18 | void DetermineProjectName(); 19 | void FinishAMReXSetup(); 20 | 21 | /** @brief Contains 'project.name' from inputs file. 22 | */ 23 | std::string project_name; 24 | }; 25 | 26 | }; // namespace sledgehamr 27 | 28 | #endif // SLEDGEHAMR_SLEDGEHAMRINIT_H_ 29 | -------------------------------------------------------------------------------- /source/time_stepper.h: -------------------------------------------------------------------------------- 1 | #ifndef SLEDGEHAMR_TIMESTEPPER_H_ 2 | #define SLEDGEHAMR_TIMESTEPPER_H_ 3 | 4 | #include "sledgehamr.h" 5 | #include "local_regrid/local_regrid.h" 6 | #include "regrid_scheduler.h" 7 | #include "integrators/integrator.h" 8 | 9 | namespace sledgehamr { 10 | 11 | class SledgeHAMR; 12 | class Integrator; 13 | class LocalRegrid; 14 | class RegridScheduler; 15 | 16 | /** @brief Class that takes care of the sub-cycling in time algorithm and 17 | * schedules regrids when necessary. 18 | */ 19 | class TimeStepper { 20 | public: 21 | TimeStepper(Sledgehamr* owner); 22 | void Advance(int lev); 23 | 24 | /** @brief Vector of regridding intervals at each level. 25 | */ 26 | std::vector regrid_dt; 27 | 28 | /** @brief Pointer to integration module. 29 | */ 30 | std::unique_ptr integrator; 31 | 32 | /** @brief Pointer to the local regrid module. 33 | */ 34 | std::unique_ptr local_regrid; 35 | 36 | /** @brief Pointer to the regrid scheduler. 37 | */ 38 | std::unique_ptr scheduler; 39 | 40 | /** @brief Vector of times at which a given level has been regridded last. 41 | */ 42 | std::vector last_regrid_time; 43 | 44 | private: 45 | void SynchronizeLevels(int lev); 46 | void SynchronizeTimes(); 47 | 48 | void PreAdvanceMessage(int lev); 49 | void PostAdvanceMessage(int lev, double duration); 50 | std::string LevelMessage(int lev, int istep); 51 | 52 | void ScheduleRegrid(int lev); 53 | void DoRegridIfScheduled(int lev); 54 | void NoShadowRegrid(int lev); 55 | void DoRegrid(int lev, double time); 56 | 57 | void ParseParams(); 58 | void SetIntegrator(); 59 | 60 | /** @brief Whether we want to force write output at the beginning of the 61 | * sim. 62 | */ 63 | bool output_of_initial_state = true; 64 | 65 | /** @brief Whether we are running a semi-static sim. 66 | */ 67 | bool semistatic_sim = false; 68 | 69 | /** @brief Pointer to the simulation. 70 | */ 71 | Sledgehamr* sim; 72 | }; 73 | 74 | }; // namespace sledgehamr 75 | 76 | #endif // SLEDGEHAMR_TIMESTEPPER_H_ 77 | -------------------------------------------------------------------------------- /source/timer.cpp: -------------------------------------------------------------------------------- 1 | #include "timer.h" 2 | 3 | namespace sledgehamr { 4 | 5 | /** @brief Starts the timer by setting the start timer point. 6 | */ 7 | void Timer::Start() { 8 | if (is_running) 9 | return; 10 | 11 | //amrex::ParallelDescriptor::Barrier(); 12 | start_time = std::chrono::steady_clock::now(); 13 | is_running = true; 14 | } 15 | 16 | /** @brief Stops the timer and checks time passed since it started running. 17 | */ 18 | void Timer::Stop() { 19 | if (!is_running) 20 | return; 21 | 22 | //amrex::ParallelDescriptor::Barrier(); 23 | CheckClock(); 24 | total_micro += static_cast(last_duration_micro.count()); 25 | is_running = false; 26 | } 27 | 28 | /** @brief Returns total time in seconds the timer was running across multiple 29 | * start/stop cycles. 30 | * return Time in seconds. 31 | */ 32 | double Timer::GetTotalTimeSeconds() { 33 | double extra_micro = 0; 34 | if (is_running) { 35 | CheckClock(); 36 | extra_micro = static_cast(last_duration_micro.count()); 37 | } 38 | 39 | return (total_micro + extra_micro)/1e6; 40 | }; 41 | 42 | /** @brief Same as GetTotalTimeSeconds() but only time since the timer was 43 | * last started. 44 | */ 45 | double Timer::GetLastDurationSeconds() { 46 | if (is_running) 47 | CheckClock(); 48 | 49 | return static_cast(last_duration_micro.count())/1e6; 50 | }; 51 | 52 | /** @brief Computes time passed between now and the last starting time point 53 | internally. 54 | */ 55 | void Timer::CheckClock() { 56 | stop_time = std::chrono::steady_clock::now(); 57 | last_duration_micro = 58 | std::chrono::duration_cast( 59 | stop_time - start_time); 60 | } 61 | 62 | }; // namespace sledgehamr 63 | -------------------------------------------------------------------------------- /source/timer.h: -------------------------------------------------------------------------------- 1 | #ifndef SLEDGEHAMR_TIMER_H_ 2 | #define SLEDGEHAMR_TIMER_H_ 3 | 4 | #include 5 | 6 | #include 7 | 8 | namespace sledgehamr { 9 | 10 | /** @brief This class keeps track of passed time just like stopwatch. 11 | */ 12 | class Timer { 13 | public: 14 | /** @brief Give the timer a display name during construction. 15 | * @param my_name Display name of timner. 16 | */ 17 | Timer(std::string my_name) : name{my_name} {}; 18 | 19 | void Start(); 20 | void Stop(); 21 | 22 | double GetTotalTimeSeconds(); 23 | double GetLastDurationSeconds(); 24 | 25 | /** @brief Returns display name of timer. 26 | */ 27 | std::string GetName() const { 28 | return name; 29 | }; 30 | 31 | /** @brief Returns whether the timer is currently running. 32 | */ 33 | bool IsRunning() const { 34 | return is_running; 35 | }; 36 | 37 | private: 38 | void CheckClock(); 39 | 40 | /** @brief Time point of when pointer has been started last. 41 | */ 42 | std::chrono::steady_clock::time_point start_time; 43 | 44 | /** @brief Time point of when pointer has been stopped last. 45 | */ 46 | std::chrono::steady_clock::time_point stop_time; 47 | 48 | /** @brief Time in milliseconds from the last starting time point to when 49 | * Check() has been called last. 50 | */ 51 | std::chrono::microseconds last_duration_micro; 52 | 53 | /** @brief Total time in milliseconds that the timer has been running 54 | * overall (across multiple start/stop cycles). 55 | */ 56 | double total_micro = 0; 57 | 58 | /** @brief Display name of timer. 59 | */ 60 | const std::string name = "Unknown Timer"; 61 | 62 | /** @brief Flag whether timer is currently running. 63 | */ 64 | bool is_running = false; 65 | }; 66 | 67 | }; // namespace sledgehamr 68 | 69 | #endif // SLEDGEHAMR_TIMER_H_ 70 | -------------------------------------------------------------------------------- /source/utils/Make.package: -------------------------------------------------------------------------------- 1 | CEXE_headers += sledgehamr_utils.h 2 | CEXE_headers += io_module_utils.h 3 | CEXE_headers += fft.h 4 | -------------------------------------------------------------------------------- /source/utils/fft.h: -------------------------------------------------------------------------------- 1 | #ifndef SLEDGEHAMR_FFT_H_ 2 | #define SLEDGEHAMR_FFT_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | namespace sledgehamr { 9 | namespace utils { 10 | 11 | /** @brief This function computes the FFT of some quantity. 12 | * @param field State to compute the FFT of. 13 | * @param field_fft_real_or_abs Contains the real or absolute part of the 14 | * FFT. 15 | * @param field_fft_imag Contains the imaginary part of the FFT, if 16 | * any. 17 | * @param geom Geometry of the data. 18 | * @param abs Wheter to keep the real and imaginary part 19 | * of the FFT or to compute the absolute part. 20 | */ 21 | static void Fft(const amrex::MultiFab& field, const int comp, 22 | amrex::MultiFab& field_fft_real_or_abs, 23 | amrex::MultiFab& field_fft_imag, const amrex::Geometry& geom, 24 | bool abs) { 25 | const amrex::BoxArray& ba = field.boxArray(); 26 | const amrex::DistributionMapping& dm = field.DistributionMap(); 27 | amrex::MultiFab field_tmp(ba, dm, 1, 0); 28 | 29 | #ifdef _OPENMP 30 | #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) 31 | #endif 32 | for ( amrex::MFIter mfi(field, amrex::TilingIfNotGPU()); 33 | mfi.isValid(); 34 | ++mfi ){ 35 | const amrex::Box& bx = mfi.tilebox(); 36 | const auto& state_arr = field.array(mfi); 37 | const auto& field_tmp_arr = field_tmp.array(mfi); 38 | 39 | amrex::ParallelFor(bx, 40 | [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept { 41 | field_tmp_arr(i, j, k, 0) = state_arr(i, j, k, comp); 42 | }); 43 | } 44 | 45 | int nx = ba[0].size()[0]; 46 | int ny = ba[0].size()[1]; 47 | int nz = ba[0].size()[2]; 48 | 49 | amrex::Box domain(geom.Domain()); 50 | int nbx = domain.length(0) / nx; 51 | int nby = domain.length(1) / ny; 52 | int nbz = domain.length(2) / nz; 53 | 54 | int nboxes = nbx * nby * nbz; 55 | 56 | amrex::Vector rank_mapping; 57 | rank_mapping.resize(nboxes); 58 | 59 | for (int ib = 0; ib < nboxes; ++ib) { 60 | int i = ba[ib].smallEnd(0) / nx; 61 | int j = ba[ib].smallEnd(1) / ny; 62 | int k = ba[ib].smallEnd(2) / nz; 63 | 64 | int local_index = k*nbx*nby + j*nbx + i; 65 | 66 | rank_mapping[local_index] = dm[ib]; 67 | } 68 | 69 | int Ndims[3] = { nbz, nby, nbx }; 70 | int n[3] = { domain.length(2), domain.length(1), domain.length(0)}; 71 | hacc::Distribution d(MPI_COMM_WORLD, n, Ndims, &rank_mapping[0]); 72 | hacc::Dfft dfft(d); 73 | 74 | for (amrex::MFIter mfi(field, false); mfi.isValid(); ++mfi) { 75 | int gid = mfi.index(); 76 | 77 | size_t local_size = dfft.local_size(); 78 | 79 | constexpr int ALIGN = 16; 80 | std::vector > a; 81 | std::vector > b; 82 | 83 | a.resize(nx*ny*nz); 84 | b.resize(nx*ny*nz); 85 | 86 | dfft.makePlans(&a[0], &b[0], &a[0], &b[0]); 87 | 88 | size_t local_indx = 0; 89 | for(size_t k=0; k<(size_t)nz; k++) { 90 | for(size_t j=0; j<(size_t)ny; j++) { 91 | for(size_t i=0; i<(size_t)nx; i++) { 92 | complex_t temp(field_tmp[mfi].dataPtr()[local_indx],0.); 93 | a[local_indx] = temp; 94 | local_indx++; 95 | } 96 | } 97 | } 98 | 99 | dfft.forward(&a[0]); 100 | d.redistribute_2_to_3(&a[0], &b[0], 2); 101 | size_t global_size = dfft.global_size(); 102 | 103 | local_indx = 0; 104 | 105 | if (abs) { 106 | for(size_t k=0; k<(size_t)nz; k++) { 107 | for(size_t j=0; j<(size_t)ny; j++) { 108 | for(size_t i=0; i<(size_t)nx; i++) { 109 | field_fft_real_or_abs[mfi].dataPtr()[local_indx] = 110 | std::abs(b[local_indx]); 111 | local_indx++; 112 | } 113 | } 114 | } 115 | } else { 116 | for(size_t k=0; k<(size_t)nz; k++) { 117 | for(size_t j=0; j<(size_t)ny; j++) { 118 | for(size_t i=0; i<(size_t)nx; i++) { 119 | field_fft_real_or_abs[mfi].dataPtr()[local_indx] = 120 | std::real(b[local_indx]); 121 | field_fft_imag[mfi].dataPtr()[local_indx] = 122 | std::imag(b[local_indx]); 123 | local_indx++; 124 | } 125 | } 126 | } 127 | } 128 | } 129 | } 130 | 131 | }; // namespace utils 132 | }; // namespace sledgehamr 133 | 134 | #endif // SLEDGEHAMR_FFT_H_ 135 | -------------------------------------------------------------------------------- /source/utils/hdf5_utils.h: -------------------------------------------------------------------------------- 1 | #ifndef SLEDGEHAMR_HDF5_UTILS_H_ 2 | #define SLEDGEHAMR_HDF5_UTILS_H_ 3 | 4 | #include 5 | 6 | #include 7 | 8 | namespace sledgehamr { 9 | namespace utils { 10 | namespace hdf5 { 11 | 12 | /** @brief Template function that reads dataset of type T from an HDF5 file. 13 | * @param filename HDF5 filename. 14 | * @param dnames List of data set names to try. 15 | * @param data Array pointer to data. 16 | * @return Whether the read was successfull. 17 | */ 18 | template 19 | static bool Read(std::string filename, std::vector dnames, 20 | T *data) { 21 | // Identify datatype. 22 | hid_t mem_type_id; 23 | if (std::is_same::value) { 24 | mem_type_id = H5T_NATIVE_FLOAT; 25 | } else if (std::is_same::value) { 26 | mem_type_id = H5T_NATIVE_DOUBLE; 27 | } else if (std::is_same::value) { 28 | mem_type_id = H5T_NATIVE_INT; 29 | } 30 | 31 | if (!amrex::FileExists(filename)) 32 | return false; 33 | 34 | // Try and open HDF5 file. 35 | hid_t file_id = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT); 36 | if (file_id == H5I_INVALID_HID) { 37 | amrex::Abort("#error: Could not open file: " + filename); 38 | } 39 | 40 | // Try and find dataset. Iterate over vector, use first to be found. 41 | const bool verbose = false; 42 | std::string dname_conc = " |"; 43 | std::string dname_found = ""; 44 | for (std::string dname : dnames) { 45 | htri_t exists = H5Lexists(file_id, dname.c_str(), H5P_DEFAULT); 46 | if (verbose) { 47 | amrex::Print() << "Attempt reading dataset " << dname << ":" 48 | << exists << std::endl; 49 | } 50 | 51 | if (exists > 0) { 52 | dname_found = dname; 53 | break; 54 | } 55 | 56 | dname_conc += " " + dname; 57 | } 58 | 59 | if (dname_found == "") { 60 | H5Fclose(file_id); 61 | 62 | return false; 63 | } 64 | 65 | // Read dataset. 66 | hid_t dataset_id = H5Dopen2(file_id, dname_found.c_str(), H5P_DEFAULT); 67 | if (H5Dread(dataset_id, mem_type_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, data) < 68 | 0) { 69 | return false; 70 | } 71 | 72 | H5Dclose(dataset_id); 73 | H5Fclose(file_id); 74 | 75 | return true; 76 | } 77 | 78 | /** @brief Template function to write a dataset of type T to an HDF5 file. 79 | * @param file_id HDF5 file id. 80 | * @param dset Dataset name. 81 | * @param data Array pointer to data. 82 | * @param size Length of data. 83 | */ 84 | template 85 | static void Write(hid_t file_id, std::string dset, T *data, 86 | unsigned long long size) { 87 | // Identify datatype. 88 | hid_t mem_type_id, dset_type_id; 89 | if (std::is_same::value) { 90 | mem_type_id = H5T_NATIVE_FLOAT; 91 | dset_type_id = H5T_IEEE_F32LE; 92 | } else if (std::is_same::value) { 93 | mem_type_id = H5T_NATIVE_DOUBLE; 94 | dset_type_id = H5T_IEEE_F64LE; 95 | } else if (std::is_same::value) { 96 | mem_type_id = H5T_NATIVE_INT; 97 | dset_type_id = H5T_IEEE_F64LE; 98 | } else { 99 | amrex::Abort("#error: Writing of dataset " + dset + 100 | " failed due to unknown datatype."); 101 | } 102 | 103 | // Create and write dataset. 104 | hsize_t dims[1] = {size}; 105 | hid_t space = H5Screate_simple(1, dims, NULL); 106 | hid_t dataset_id = H5Dcreate(file_id, dset.c_str(), dset_type_id, space, 107 | H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); 108 | H5Dwrite(dataset_id, mem_type_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); 109 | H5Dclose(dataset_id); 110 | } 111 | 112 | /** @brief Checks from a list of datasets whether one of them exists in a given 113 | * HDF5 file. 114 | * @param filename HDF5 filename. 115 | * @param dnames List of datasets. 116 | * @return The existing dataset or an empty string of none exist. 117 | */ 118 | static std::string FindDataset(std::string filename, 119 | std::vector dnames) { 120 | // Try and open HDF5 file. 121 | hid_t file_id = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT); 122 | if (file_id == H5I_INVALID_HID) { 123 | amrex::Abort("#error: Could not open file: " + filename); 124 | } 125 | 126 | // Try and find dataset. Iterate over vector, use first to be found. 127 | std::string dname_found = ""; 128 | for (std::string dname : dnames) { 129 | htri_t exists = H5Lexists(file_id, dname.c_str(), H5P_DEFAULT); 130 | 131 | if (exists > 0) 132 | return dname; 133 | } 134 | 135 | return ""; 136 | } 137 | 138 | }; // namespace hdf5 139 | }; // namespace utils 140 | }; // namespace sledgehamr 141 | 142 | #endif // SLEDGEHAMR_HDF5_UTILS_H_ 143 | --------------------------------------------------------------------------------