├── .gitignore ├── CHANGELOG.md ├── COPYING ├── COPYING.LESSER ├── MANIFEST.in ├── README.md ├── docs ├── Makefile ├── conf.py ├── decomposition.png ├── density.png ├── index.rst ├── make.bat ├── modules.rst ├── pyDive.adios.rst ├── pyDive.cloned_ndarray.rst ├── pyDive.gpu.rst ├── pyDive.h5.rst ├── pyDive.ndarray.rst ├── reference.rst ├── sample.h5 ├── spectrum.png ├── start.rst └── tutorial.rst ├── pyDive ├── IPParallelClient.py ├── __init__.py ├── algorithm.py ├── arrays │ ├── __init__.py │ ├── ad_ndarray.py │ ├── gpu_ndarray.py │ ├── h5_ndarray.py │ ├── local │ │ ├── __init__.py │ │ ├── ad_ndarray.py │ │ ├── gpu_ndarray.py │ │ └── h5_ndarray.py │ └── ndarray.py ├── cloned_ndarray │ ├── __init__.py │ ├── cloned_ndarray.py │ └── factories.py ├── distribution │ ├── __init__.py │ ├── helper.py │ ├── interengine.py │ ├── multiple_axes.py │ └── multiple_axes_funcs.py ├── fragment.py ├── mappings.py ├── picongpu.py ├── pyDive.py ├── structured.py └── test │ ├── __init__.py │ ├── conftest.py │ ├── m2p_CIC.npy │ ├── p2m_CIC.npy │ ├── sample.h5 │ ├── test_algorithm.py │ ├── test_cloned_ndarray.py │ ├── test_gpu.py │ ├── test_h5_ndarray.py │ ├── test_mappings.py │ ├── test_ndarray.py │ └── test_structured.py ├── requirements.txt └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod~] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | env/ 11 | bin/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | eggs/ 16 | lib/ 17 | lib64/ 18 | parts/ 19 | sdist/ 20 | var/ 21 | *.egg-info/ 22 | .installed.cfg 23 | *.egg 24 | 25 | # Installer logs 26 | pip-log.txt 27 | pip-delete-this-directory.txt 28 | 29 | # Unit test / coverage reports 30 | htmlcov/ 31 | .tox/ 32 | .coverage 33 | .cache 34 | nosetests.xml 35 | coverage.xml 36 | 37 | # Translations 38 | *.mo 39 | 40 | # Mr Developer 41 | .mr.developer.cfg 42 | .project 43 | .pydevproject 44 | 45 | # Rope 46 | .ropeproject 47 | 48 | # Django stuff: 49 | *.log 50 | *.pot 51 | 52 | # Sphinx documentation 53 | docs/_build/ 54 | 55 | *.nja* 56 | *.rst~* 57 | *.md~* 58 | pyDive/debug.py 59 | autorsync* 60 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | Changelog for pyDive 2 | ==================== 3 | 4 | 1.2.2 5 | ----- 6 | **Date:** 2015-07-10 7 | 8 | - add `setup_requires` to setup.py 9 | 10 | 1.2.1 11 | ----- 12 | **Date:** 2015-07-10 13 | 14 | - add `arrays.local` package to setup.py 15 | 16 | 1.2 17 | --- 18 | **Date:** 2015-06-22 19 | 20 | **New Features:** 21 | - arrays can be distributed along multiple axes 22 | - new distributed array: `pyDive.arrays.gpu_ndarray`. Distributed version of a pycuda array with extra support 23 | for non-contiguous memory (needed for working with sliced arrays). 24 | - `gather()` is called implicitly when the user wants to access an attribute, e.g. datapointer, which only 25 | the local array has. 26 | - implement indexing by bitmask 27 | 28 | **Bug Fixes:** 29 | - integer indices were not wrapped in `__getitem__()` 30 | - fix exception in picongpu.py if dataset has no attributes 31 | 32 | **Misc**- 33 | - `distaxes` parameter defaults to 'all' 34 | - all local array classes are located in pyDive/arrays/local 35 | - rename "arrayOfStructs" module to "structured" 36 | 37 | 1.1 38 | --- 39 | **Date:** 2015-03-29 40 | 41 | **New Features:** 42 | - adios support 43 | - implement `pyDive.fragment` 44 | - distributed array classes are auto-generated from local array class. 45 | 46 | **Misc:** 47 | - restructure project files 48 | - optimize `pyDive.ndarray` 49 | 50 | 1.0.2 51 | ----- 52 | **Date:** 2014-11-06 53 | 54 | - add MANIFEST.in 55 | 56 | 1.0.1 57 | ----- 58 | **Date:** 2014-11-06 59 | 60 | - fix setup.py 61 | 62 | 1.0 63 | --- 64 | **Date:** 2014-11-05 65 | 66 | Development status: **beta** 67 | -------------------------------------------------------------------------------- /COPYING.LESSER: -------------------------------------------------------------------------------- 1 | GNU LESSER GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | 9 | This version of the GNU Lesser General Public License incorporates 10 | the terms and conditions of version 3 of the GNU General Public 11 | License, supplemented by the additional permissions listed below. 12 | 13 | 0. Additional Definitions. 14 | 15 | As used herein, "this License" refers to version 3 of the GNU Lesser 16 | General Public License, and the "GNU GPL" refers to version 3 of the GNU 17 | General Public License. 18 | 19 | "The Library" refers to a covered work governed by this License, 20 | other than an Application or a Combined Work as defined below. 21 | 22 | An "Application" is any work that makes use of an interface provided 23 | by the Library, but which is not otherwise based on the Library. 24 | Defining a subclass of a class defined by the Library is deemed a mode 25 | of using an interface provided by the Library. 26 | 27 | A "Combined Work" is a work produced by combining or linking an 28 | Application with the Library. The particular version of the Library 29 | with which the Combined Work was made is also called the "Linked 30 | Version". 31 | 32 | The "Minimal Corresponding Source" for a Combined Work means the 33 | Corresponding Source for the Combined Work, excluding any source code 34 | for portions of the Combined Work that, considered in isolation, are 35 | based on the Application, and not on the Linked Version. 36 | 37 | The "Corresponding Application Code" for a Combined Work means the 38 | object code and/or source code for the Application, including any data 39 | and utility programs needed for reproducing the Combined Work from the 40 | Application, but excluding the System Libraries of the Combined Work. 41 | 42 | 1. Exception to Section 3 of the GNU GPL. 43 | 44 | You may convey a covered work under sections 3 and 4 of this License 45 | without being bound by section 3 of the GNU GPL. 46 | 47 | 2. Conveying Modified Versions. 48 | 49 | If you modify a copy of the Library, and, in your modifications, a 50 | facility refers to a function or data to be supplied by an Application 51 | that uses the facility (other than as an argument passed when the 52 | facility is invoked), then you may convey a copy of the modified 53 | version: 54 | 55 | a) under this License, provided that you make a good faith effort to 56 | ensure that, in the event an Application does not supply the 57 | function or data, the facility still operates, and performs 58 | whatever part of its purpose remains meaningful, or 59 | 60 | b) under the GNU GPL, with none of the additional permissions of 61 | this License applicable to that copy. 62 | 63 | 3. Object Code Incorporating Material from Library Header Files. 64 | 65 | The object code form of an Application may incorporate material from 66 | a header file that is part of the Library. You may convey such object 67 | code under terms of your choice, provided that, if the incorporated 68 | material is not limited to numerical parameters, data structure 69 | layouts and accessors, or small macros, inline functions and templates 70 | (ten or fewer lines in length), you do both of the following: 71 | 72 | a) Give prominent notice with each copy of the object code that the 73 | Library is used in it and that the Library and its use are 74 | covered by this License. 75 | 76 | b) Accompany the object code with a copy of the GNU GPL and this license 77 | document. 78 | 79 | 4. Combined Works. 80 | 81 | You may convey a Combined Work under terms of your choice that, 82 | taken together, effectively do not restrict modification of the 83 | portions of the Library contained in the Combined Work and reverse 84 | engineering for debugging such modifications, if you also do each of 85 | the following: 86 | 87 | a) Give prominent notice with each copy of the Combined Work that 88 | the Library is used in it and that the Library and its use are 89 | covered by this License. 90 | 91 | b) Accompany the Combined Work with a copy of the GNU GPL and this license 92 | document. 93 | 94 | c) For a Combined Work that displays copyright notices during 95 | execution, include the copyright notice for the Library among 96 | these notices, as well as a reference directing the user to the 97 | copies of the GNU GPL and this license document. 98 | 99 | d) Do one of the following: 100 | 101 | 0) Convey the Minimal Corresponding Source under the terms of this 102 | License, and the Corresponding Application Code in a form 103 | suitable for, and under terms that permit, the user to 104 | recombine or relink the Application with a modified version of 105 | the Linked Version to produce a modified Combined Work, in the 106 | manner specified by section 6 of the GNU GPL for conveying 107 | Corresponding Source. 108 | 109 | 1) Use a suitable shared library mechanism for linking with the 110 | Library. A suitable mechanism is one that (a) uses at run time 111 | a copy of the Library already present on the user's computer 112 | system, and (b) will operate properly with a modified version 113 | of the Library that is interface-compatible with the Linked 114 | Version. 115 | 116 | e) Provide Installation Information, but only if you would otherwise 117 | be required to provide such information under section 6 of the 118 | GNU GPL, and only to the extent that such information is 119 | necessary to install and execute a modified version of the 120 | Combined Work produced by recombining or relinking the 121 | Application with a modified version of the Linked Version. (If 122 | you use option 4d0, the Installation Information must accompany 123 | the Minimal Corresponding Source and Corresponding Application 124 | Code. If you use option 4d1, you must provide the Installation 125 | Information in the manner specified by section 6 of the GNU GPL 126 | for conveying Corresponding Source.) 127 | 128 | 5. Combined Libraries. 129 | 130 | You may place library facilities that are a work based on the 131 | Library side by side in a single library together with other library 132 | facilities that are not Applications and are not covered by this 133 | License, and convey such a combined library under terms of your 134 | choice, if you do both of the following: 135 | 136 | a) Accompany the combined library with a copy of the same work based 137 | on the Library, uncombined with any other library facilities, 138 | conveyed under the terms of this License. 139 | 140 | b) Give prominent notice with the combined library that part of it 141 | is a work based on the Library, and explaining where to find the 142 | accompanying uncombined form of the same work. 143 | 144 | 6. Revised Versions of the GNU Lesser General Public License. 145 | 146 | The Free Software Foundation may publish revised and/or new versions 147 | of the GNU Lesser General Public License from time to time. Such new 148 | versions will be similar in spirit to the present version, but may 149 | differ in detail to address new problems or concerns. 150 | 151 | Each version is given a distinguishing version number. If the 152 | Library as you received it specifies that a certain numbered version 153 | of the GNU Lesser General Public License "or any later version" 154 | applies to it, you have the option of following the terms and 155 | conditions either of that published version or of any later version 156 | published by the Free Software Foundation. If the Library as you 157 | received it does not specify a version number of the GNU Lesser 158 | General Public License, you may choose any version of the GNU Lesser 159 | General Public License ever published by the Free Software Foundation. 160 | 161 | If the Library as you received it specifies that a proxy can decide 162 | whether future versions of the GNU Lesser General Public License shall 163 | apply, that proxy's public statement of acceptance of any version is 164 | permanent authorization for you to choose that version for the 165 | Library. 166 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.md 2 | include *.txt 3 | include COPYING* 4 | include pyDive/test/*.h5 pyDive/test/*.npy 5 | include docs/* 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | pyDive 2 | ====== 3 | 4 | Distributed Interactive Visualization and Exploration of large datasets. 5 | 6 | ## What is pyDive? 7 | 8 | Use pyDive to work with homogeneous, n-dimensional arrays that are too big to fit into your local machine's memory. 9 | pyDive provides containers whose elements are distributed across a cluster or stored in 10 | a large hdf5/adios-file if the cluster is still too small. All computation and data-access is then done in parallel by the cluster nodes in the background. 11 | If you feel like working with [numpy](http://www.numpy.org) arrays pyDive has reached the goal! 12 | 13 | pyDive is developed and maintained by the **[Junior Group Computational Radiation Physics](http://www.hzdr.de/db/Cms?pNid=132&pOid=30354)** 14 | at the [Institute for Radiation Physics](http://www.hzdr.de/db/Cms?pNid=132) 15 | at [HZDR](http://www.hzdr.de/). 16 | 17 | **Features:** 18 | - Since all cluster management is given to [IPython.parallel](http://ipython.org/ipython-doc/dev/parallel/) you can take your 19 | existing profiles for pyDive. No further cluster configuration needed. 20 | - Save bandwidth by slicing an array in parallel on disk first before loading it into main memory! 21 | - GPU-cluster array available thanks to [pycuda](http://mathema.tician.de/software/pycuda/) with additional support for non-contiguous memory. 22 | - As all of pyDive's distributed array types are auto-generated from local arrays like numpy, hdf5, pycuda, etc... 23 | you can easily make your own local array classes distributed too. 24 | 25 | ## Dive in! 26 | 27 | ```python 28 | import pyDive 29 | pyDive.init(profile='mpi') 30 | 31 | h5field = pyDive.h5.open("myData.h5", "myDataset", distaxes=(0,1)) 32 | ones = pyDive.ones_like(h5field) 33 | 34 | # Distribute file i/o and computation across the cluster 35 | h5field[::10,:] = h5field[::10,:].load() + 5.0 * ones[::10,:] 36 | ``` 37 | 38 | ## Documentation 39 | 40 | In our [Online Documentation](http://ComputationalRadiationPhysics.github.io/pyDive/), [pdf](http://ComputationalRadiationPhysics.github.io/pyDive/pyDive.pdf) you can find 41 | detailed information on all interfaces as well as some [Tutorials](http://computationalradiationphysics.github.io/pyDive/tutorial.html) 42 | and a [Quickstart](http://computationalradiationphysics.github.io/pyDive/start.html). 43 | 44 | ## Software License 45 | 46 | pyDive is licensed under the **GPLv3+ and LGPLv3+** (it is *dual licensed*). 47 | Licences can be found in [GPL](COPYING) or [LGPL](COPYING.LESSER), respectively. 48 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 21 | 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 23 | 24 | help: 25 | @echo "Please use \`make ' where is one of" 26 | @echo " html to make standalone HTML files" 27 | @echo " dirhtml to make HTML files named index.html in directories" 28 | @echo " singlehtml to make a single large HTML file" 29 | @echo " pickle to make pickle files" 30 | @echo " json to make JSON files" 31 | @echo " htmlhelp to make HTML files and a HTML help project" 32 | @echo " qthelp to make HTML files and a qthelp project" 33 | @echo " devhelp to make HTML files and a Devhelp project" 34 | @echo " epub to make an epub" 35 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 36 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 37 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 38 | @echo " text to make text files" 39 | @echo " man to make manual pages" 40 | @echo " texinfo to make Texinfo files" 41 | @echo " info to make Texinfo files and run them through makeinfo" 42 | @echo " gettext to make PO message catalogs" 43 | @echo " changes to make an overview of all changed/added/deprecated items" 44 | @echo " xml to make Docutils-native XML files" 45 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 46 | @echo " linkcheck to check all external links for integrity" 47 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 48 | 49 | clean: 50 | rm -rf $(BUILDDIR)/* 51 | 52 | html: 53 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 54 | @echo 55 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 56 | 57 | dirhtml: 58 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 59 | @echo 60 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 61 | 62 | singlehtml: 63 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 64 | @echo 65 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 66 | 67 | pickle: 68 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 69 | @echo 70 | @echo "Build finished; now you can process the pickle files." 71 | 72 | json: 73 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 74 | @echo 75 | @echo "Build finished; now you can process the JSON files." 76 | 77 | htmlhelp: 78 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 79 | @echo 80 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 81 | ".hhp project file in $(BUILDDIR)/htmlhelp." 82 | 83 | qthelp: 84 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 85 | @echo 86 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 87 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 88 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/pyDive.qhcp" 89 | @echo "To view the help file:" 90 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/pyDive.qhc" 91 | 92 | devhelp: 93 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 94 | @echo 95 | @echo "Build finished." 96 | @echo "To view the help file:" 97 | @echo "# mkdir -p $$HOME/.local/share/devhelp/pyDive" 98 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/pyDive" 99 | @echo "# devhelp" 100 | 101 | epub: 102 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 103 | @echo 104 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 105 | 106 | latex: 107 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 108 | @echo 109 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 110 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 111 | "(use \`make latexpdf' here to do that automatically)." 112 | 113 | latexpdf: 114 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 115 | @echo "Running LaTeX files through pdflatex..." 116 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 117 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 118 | 119 | latexpdfja: 120 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 121 | @echo "Running LaTeX files through platex and dvipdfmx..." 122 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 123 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 124 | 125 | text: 126 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 127 | @echo 128 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 129 | 130 | man: 131 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 132 | @echo 133 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 134 | 135 | texinfo: 136 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 137 | @echo 138 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 139 | @echo "Run \`make' in that directory to run these through makeinfo" \ 140 | "(use \`make info' here to do that automatically)." 141 | 142 | info: 143 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 144 | @echo "Running Texinfo files through makeinfo..." 145 | make -C $(BUILDDIR)/texinfo info 146 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 147 | 148 | gettext: 149 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 150 | @echo 151 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 152 | 153 | changes: 154 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 155 | @echo 156 | @echo "The overview file is in $(BUILDDIR)/changes." 157 | 158 | linkcheck: 159 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 160 | @echo 161 | @echo "Link check complete; look for any errors in the above output " \ 162 | "or in $(BUILDDIR)/linkcheck/output.txt." 163 | 164 | doctest: 165 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 166 | @echo "Testing of doctests in the sources finished, look at the " \ 167 | "results in $(BUILDDIR)/doctest/output.txt." 168 | 169 | xml: 170 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 171 | @echo 172 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 173 | 174 | pseudoxml: 175 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 176 | @echo 177 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 178 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # pyDive documentation build configuration file, created by 4 | # sphinx-quickstart on Wed Jul 23 15:53:56 2014. 5 | # 6 | # This file is execfile()d with the current directory set to its 7 | # containing dir. 8 | # 9 | # Note that not all possible configuration values are present in this 10 | # autogenerated file. 11 | # 12 | # All configuration values have a default; values that are commented out 13 | # serve to show the default. 14 | 15 | import sys 16 | import os 17 | 18 | # If extensions (or modules to document with autodoc) are in another directory, 19 | # add these directories to sys.path here. If the directory is relative to the 20 | # documentation root, use os.path.abspath to make it absolute, like shown here. 21 | #sys.path.insert(0, os.path.abspath('.')) 22 | 23 | # -- General configuration ------------------------------------------------ 24 | 25 | # If your documentation needs a minimal Sphinx version, state it here. 26 | #needs_sphinx = '1.0' 27 | 28 | # Add any Sphinx extension module names here, as strings. They can be 29 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 30 | # ones. 31 | extensions = [ 32 | 'sphinx.ext.autodoc', 33 | 'sphinx.ext.viewcode', 34 | ] 35 | 36 | # Add any paths that contain templates here, relative to this directory. 37 | templates_path = ['_templates'] 38 | 39 | # The suffix of source filenames. 40 | source_suffix = '.rst' 41 | 42 | # The encoding of source files. 43 | #source_encoding = 'utf-8-sig' 44 | 45 | # The master toctree document. 46 | master_doc = 'index' 47 | 48 | # General information about the project. 49 | project = u'pyDive' 50 | copyright = u'2014,2015, Heiko Burau' 51 | 52 | # The version info for the project you're documenting, acts as replacement for 53 | # |version| and |release|, also used in various other places throughout the 54 | # built documents. 55 | # 56 | # The short X.Y version. 57 | version = '1.2.2' 58 | # The full version, including alpha/beta/rc tags. 59 | release = '1.2.2 beta' 60 | 61 | # The language for content autogenerated by Sphinx. Refer to documentation 62 | # for a list of supported languages. 63 | #language = None 64 | 65 | # There are two options for replacing |today|: either, you set today to some 66 | # non-false value, then it is used: 67 | #today = '' 68 | # Else, today_fmt is used as the format for a strftime call. 69 | #today_fmt = '%B %d, %Y' 70 | 71 | # List of patterns, relative to source directory, that match files and 72 | # directories to ignore when looking for source files. 73 | exclude_patterns = ['_build'] 74 | 75 | # The reST default role (used for this markup: `text`) to use for all 76 | # documents. 77 | #default_role = None 78 | 79 | # If true, '()' will be appended to :func: etc. cross-reference text. 80 | #add_function_parentheses = True 81 | 82 | # If true, the current module name will be prepended to all description 83 | # unit titles (such as .. function::). 84 | #add_module_names = True 85 | 86 | # If true, sectionauthor and moduleauthor directives will be shown in the 87 | # output. They are ignored by default. 88 | #show_authors = False 89 | 90 | # The name of the Pygments (syntax highlighting) style to use. 91 | pygments_style = 'sphinx' 92 | 93 | # A list of ignored prefixes for module index sorting. 94 | #modindex_common_prefix = [] 95 | 96 | # If true, keep warnings as "system message" paragraphs in the built documents. 97 | #keep_warnings = False 98 | 99 | 100 | # -- Options for HTML output ---------------------------------------------- 101 | 102 | # The theme to use for HTML and HTML Help pages. See the documentation for 103 | # a list of builtin themes. 104 | html_theme = 'sphinxdoc' 105 | 106 | # Theme options are theme-specific and customize the look and feel of a theme 107 | # further. For a list of options available for each theme, see the 108 | # documentation. 109 | #html_theme_options = {} 110 | 111 | # Add any paths that contain custom themes here, relative to this directory. 112 | #html_theme_path = [] 113 | 114 | # The name for this set of Sphinx documents. If None, it defaults to 115 | # " v documentation". 116 | #html_title = None 117 | 118 | # A shorter title for the navigation bar. Default is the same as html_title. 119 | #html_short_title = None 120 | 121 | # The name of an image file (relative to this directory) to place at the top 122 | # of the sidebar. 123 | #html_logo = None 124 | 125 | # The name of an image file (within the static path) to use as favicon of the 126 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 127 | # pixels large. 128 | #html_favicon = None 129 | 130 | # Add any paths that contain custom static files (such as style sheets) here, 131 | # relative to this directory. They are copied after the builtin static files, 132 | # so a file named "default.css" will overwrite the builtin "default.css". 133 | html_static_path = ['_static'] 134 | 135 | # Add any extra paths that contain custom files (such as robots.txt or 136 | # .htaccess) here, relative to this directory. These files are copied 137 | # directly to the root of the documentation. 138 | #html_extra_path = [] 139 | 140 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 141 | # using the given strftime format. 142 | #html_last_updated_fmt = '%b %d, %Y' 143 | 144 | # If true, SmartyPants will be used to convert quotes and dashes to 145 | # typographically correct entities. 146 | #html_use_smartypants = True 147 | 148 | # Custom sidebar templates, maps document names to template names. 149 | #html_sidebars = {} 150 | 151 | # Additional templates that should be rendered to pages, maps page names to 152 | # template names. 153 | #html_additional_pages = {} 154 | 155 | # If false, no module index is generated. 156 | #html_domain_indices = True 157 | 158 | # If false, no index is generated. 159 | #html_use_index = True 160 | 161 | # If true, the index is split into individual pages for each letter. 162 | #html_split_index = False 163 | 164 | # If true, links to the reST sources are added to the pages. 165 | #html_show_sourcelink = True 166 | 167 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 168 | #html_show_sphinx = True 169 | 170 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 171 | #html_show_copyright = True 172 | 173 | # If true, an OpenSearch description file will be output, and all pages will 174 | # contain a tag referring to it. The value of this option must be the 175 | # base URL from which the finished HTML is served. 176 | #html_use_opensearch = '' 177 | 178 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 179 | #html_file_suffix = None 180 | 181 | # Output file base name for HTML help builder. 182 | htmlhelp_basename = 'pyDivedoc' 183 | 184 | 185 | # -- Options for LaTeX output --------------------------------------------- 186 | 187 | latex_elements = { 188 | # The paper size ('letterpaper' or 'a4paper'). 189 | #'papersize': 'letterpaper', 190 | 191 | # The font size ('10pt', '11pt' or '12pt'). 192 | #'pointsize': '10pt', 193 | 194 | # Additional stuff for the LaTeX preamble. 195 | #'preamble': '', 196 | } 197 | 198 | # Grouping the document tree into LaTeX files. List of tuples 199 | # (source start file, target name, title, 200 | # author, documentclass [howto, manual, or own class]). 201 | latex_documents = [ 202 | ('index', 'pyDive.tex', u'pyDive Documentation', 203 | u'Heiko Burau', 'manual'), 204 | ] 205 | 206 | # The name of an image file (relative to this directory) to place at the top of 207 | # the title page. 208 | #latex_logo = None 209 | 210 | # For "manual" documents, if this is true, then toplevel headings are parts, 211 | # not chapters. 212 | #latex_use_parts = False 213 | 214 | # If true, show page references after internal links. 215 | #latex_show_pagerefs = False 216 | 217 | # If true, show URL addresses after external links. 218 | #latex_show_urls = False 219 | 220 | # Documents to append as an appendix to all manuals. 221 | #latex_appendices = [] 222 | 223 | # If false, no module index is generated. 224 | #latex_domain_indices = True 225 | 226 | 227 | # -- Options for manual page output --------------------------------------- 228 | 229 | # One entry per manual page. List of tuples 230 | # (source start file, name, description, authors, manual section). 231 | man_pages = [ 232 | ('index', 'pydive', u'pyDive Documentation', 233 | [u'Heiko Burau'], 1) 234 | ] 235 | 236 | # If true, show URL addresses after external links. 237 | #man_show_urls = False 238 | 239 | 240 | # -- Options for Texinfo output ------------------------------------------- 241 | 242 | # Grouping the document tree into Texinfo files. List of tuples 243 | # (source start file, target name, title, author, 244 | # dir menu entry, description, category) 245 | texinfo_documents = [ 246 | ('index', 'pyDive', u'pyDive Documentation', 247 | u'Heiko Burau', 'pyDive', 'One line description of project.', 248 | 'Miscellaneous'), 249 | ] 250 | 251 | # Documents to append as an appendix to all manuals. 252 | #texinfo_appendices = [] 253 | 254 | # If false, no module index is generated. 255 | #texinfo_domain_indices = True 256 | 257 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 258 | #texinfo_show_urls = 'footnote' 259 | 260 | # If true, do not generate a @detailmenu in the "Top" node's menu. 261 | #texinfo_no_detailmenu = False 262 | 263 | 264 | # -- Options for Epub output ---------------------------------------------- 265 | 266 | # Bibliographic Dublin Core info. 267 | epub_title = u'pyDive' 268 | epub_author = u'Author' 269 | epub_publisher = u'Author' 270 | epub_copyright = u'2014, Author' 271 | 272 | # The basename for the epub file. It defaults to the project name. 273 | #epub_basename = u'pyDive' 274 | 275 | # The HTML theme for the epub output. Since the default themes are not optimized 276 | # for small screen space, using the same theme for HTML and epub output is 277 | # usually not wise. This defaults to 'epub', a theme designed to save visual 278 | # space. 279 | #epub_theme = 'epub' 280 | 281 | # The language of the text. It defaults to the language option 282 | # or en if the language is not set. 283 | #epub_language = '' 284 | 285 | # The scheme of the identifier. Typical schemes are ISBN or URL. 286 | #epub_scheme = '' 287 | 288 | # The unique identifier of the text. This can be a ISBN number 289 | # or the project homepage. 290 | #epub_identifier = '' 291 | 292 | # A unique identification for the text. 293 | #epub_uid = '' 294 | 295 | # A tuple containing the cover image and cover page html template filenames. 296 | #epub_cover = () 297 | 298 | # A sequence of (type, uri, title) tuples for the guide element of content.opf. 299 | #epub_guide = () 300 | 301 | # HTML files that should be inserted before the pages created by sphinx. 302 | # The format is a list of tuples containing the path and title. 303 | #epub_pre_files = [] 304 | 305 | # HTML files shat should be inserted after the pages created by sphinx. 306 | # The format is a list of tuples containing the path and title. 307 | #epub_post_files = [] 308 | 309 | # A list of files that should not be packed into the epub file. 310 | epub_exclude_files = ['search.html'] 311 | 312 | # The depth of the table of contents in toc.ncx. 313 | #epub_tocdepth = 3 314 | 315 | # Allow duplicate toc entries. 316 | #epub_tocdup = True 317 | 318 | # Choose between 'default' and 'includehidden'. 319 | #epub_tocscope = 'default' 320 | 321 | # Fix unsupported image types using the PIL. 322 | #epub_fix_images = False 323 | 324 | # Scale large images. 325 | #epub_max_image_width = 0 326 | 327 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 328 | #epub_show_urls = 'inline' 329 | 330 | # If false, no index is generated. 331 | #epub_use_index = True 332 | -------------------------------------------------------------------------------- /docs/decomposition.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ComputationalRadiationPhysics/pyDive/67d36420c2f1d59bef8633b2dea94b4ea4ab8ceb/docs/decomposition.png -------------------------------------------------------------------------------- /docs/density.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ComputationalRadiationPhysics/pyDive/67d36420c2f1d59bef8633b2dea94b4ea4ab8ceb/docs/density.png -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. pyDive documentation master file, created by 2 | sphinx-quickstart on Wed Jul 23 15:53:56 2014. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to pyDive's documentation! 7 | ================================== 8 | 9 | Contents: 10 | 11 | .. toctree:: 12 | :maxdepth: 4 13 | 14 | start 15 | tutorial 16 | reference 17 | 18 | 19 | Indices and tables 20 | ================== 21 | 22 | * :ref:`genindex` 23 | * :ref:`modindex` 24 | * :ref:`search` 25 | 26 | .. glossary:: 27 | 28 | engine 29 | The cluster nodes of *IPython.parallel* are called *engines*. Sometimes they are also called *targets*. 30 | They are the workers of pyDive performing all the computation and file i/o and they hold the actual array-memory. 31 | From the user perspective you don't to deal with them directly. 32 | 33 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=_build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . 10 | set I18NSPHINXOPTS=%SPHINXOPTS% . 11 | if NOT "%PAPER%" == "" ( 12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 14 | ) 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "help" ( 19 | :help 20 | echo.Please use `make ^` where ^ is one of 21 | echo. html to make standalone HTML files 22 | echo. dirhtml to make HTML files named index.html in directories 23 | echo. singlehtml to make a single large HTML file 24 | echo. pickle to make pickle files 25 | echo. json to make JSON files 26 | echo. htmlhelp to make HTML files and a HTML help project 27 | echo. qthelp to make HTML files and a qthelp project 28 | echo. devhelp to make HTML files and a Devhelp project 29 | echo. epub to make an epub 30 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 31 | echo. text to make text files 32 | echo. man to make manual pages 33 | echo. texinfo to make Texinfo files 34 | echo. gettext to make PO message catalogs 35 | echo. changes to make an overview over all changed/added/deprecated items 36 | echo. xml to make Docutils-native XML files 37 | echo. pseudoxml to make pseudoxml-XML files for display purposes 38 | echo. linkcheck to check all external links for integrity 39 | echo. doctest to run all doctests embedded in the documentation if enabled 40 | goto end 41 | ) 42 | 43 | if "%1" == "clean" ( 44 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 45 | del /q /s %BUILDDIR%\* 46 | goto end 47 | ) 48 | 49 | 50 | %SPHINXBUILD% 2> nul 51 | if errorlevel 9009 ( 52 | echo. 53 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 54 | echo.installed, then set the SPHINXBUILD environment variable to point 55 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 56 | echo.may add the Sphinx directory to PATH. 57 | echo. 58 | echo.If you don't have Sphinx installed, grab it from 59 | echo.http://sphinx-doc.org/ 60 | exit /b 1 61 | ) 62 | 63 | if "%1" == "html" ( 64 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 65 | if errorlevel 1 exit /b 1 66 | echo. 67 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 68 | goto end 69 | ) 70 | 71 | if "%1" == "dirhtml" ( 72 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 73 | if errorlevel 1 exit /b 1 74 | echo. 75 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 76 | goto end 77 | ) 78 | 79 | if "%1" == "singlehtml" ( 80 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 81 | if errorlevel 1 exit /b 1 82 | echo. 83 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 84 | goto end 85 | ) 86 | 87 | if "%1" == "pickle" ( 88 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 89 | if errorlevel 1 exit /b 1 90 | echo. 91 | echo.Build finished; now you can process the pickle files. 92 | goto end 93 | ) 94 | 95 | if "%1" == "json" ( 96 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 97 | if errorlevel 1 exit /b 1 98 | echo. 99 | echo.Build finished; now you can process the JSON files. 100 | goto end 101 | ) 102 | 103 | if "%1" == "htmlhelp" ( 104 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 105 | if errorlevel 1 exit /b 1 106 | echo. 107 | echo.Build finished; now you can run HTML Help Workshop with the ^ 108 | .hhp project file in %BUILDDIR%/htmlhelp. 109 | goto end 110 | ) 111 | 112 | if "%1" == "qthelp" ( 113 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 114 | if errorlevel 1 exit /b 1 115 | echo. 116 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 117 | .qhcp project file in %BUILDDIR%/qthelp, like this: 118 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\pyDive.qhcp 119 | echo.To view the help file: 120 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\pyDive.ghc 121 | goto end 122 | ) 123 | 124 | if "%1" == "devhelp" ( 125 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 126 | if errorlevel 1 exit /b 1 127 | echo. 128 | echo.Build finished. 129 | goto end 130 | ) 131 | 132 | if "%1" == "epub" ( 133 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 134 | if errorlevel 1 exit /b 1 135 | echo. 136 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 137 | goto end 138 | ) 139 | 140 | if "%1" == "latex" ( 141 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 142 | if errorlevel 1 exit /b 1 143 | echo. 144 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 145 | goto end 146 | ) 147 | 148 | if "%1" == "latexpdf" ( 149 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 150 | cd %BUILDDIR%/latex 151 | make all-pdf 152 | cd %BUILDDIR%/.. 153 | echo. 154 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 155 | goto end 156 | ) 157 | 158 | if "%1" == "latexpdfja" ( 159 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 160 | cd %BUILDDIR%/latex 161 | make all-pdf-ja 162 | cd %BUILDDIR%/.. 163 | echo. 164 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 165 | goto end 166 | ) 167 | 168 | if "%1" == "text" ( 169 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 170 | if errorlevel 1 exit /b 1 171 | echo. 172 | echo.Build finished. The text files are in %BUILDDIR%/text. 173 | goto end 174 | ) 175 | 176 | if "%1" == "man" ( 177 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 178 | if errorlevel 1 exit /b 1 179 | echo. 180 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 181 | goto end 182 | ) 183 | 184 | if "%1" == "texinfo" ( 185 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 186 | if errorlevel 1 exit /b 1 187 | echo. 188 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 189 | goto end 190 | ) 191 | 192 | if "%1" == "gettext" ( 193 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 194 | if errorlevel 1 exit /b 1 195 | echo. 196 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 197 | goto end 198 | ) 199 | 200 | if "%1" == "changes" ( 201 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 202 | if errorlevel 1 exit /b 1 203 | echo. 204 | echo.The overview file is in %BUILDDIR%/changes. 205 | goto end 206 | ) 207 | 208 | if "%1" == "linkcheck" ( 209 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 210 | if errorlevel 1 exit /b 1 211 | echo. 212 | echo.Link check complete; look for any errors in the above output ^ 213 | or in %BUILDDIR%/linkcheck/output.txt. 214 | goto end 215 | ) 216 | 217 | if "%1" == "doctest" ( 218 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 219 | if errorlevel 1 exit /b 1 220 | echo. 221 | echo.Testing of doctests in the sources finished, look at the ^ 222 | results in %BUILDDIR%/doctest/output.txt. 223 | goto end 224 | ) 225 | 226 | if "%1" == "xml" ( 227 | %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml 228 | if errorlevel 1 exit /b 1 229 | echo. 230 | echo.Build finished. The XML files are in %BUILDDIR%/xml. 231 | goto end 232 | ) 233 | 234 | if "%1" == "pseudoxml" ( 235 | %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml 236 | if errorlevel 1 exit /b 1 237 | echo. 238 | echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. 239 | goto end 240 | ) 241 | 242 | :end 243 | -------------------------------------------------------------------------------- /docs/modules.rst: -------------------------------------------------------------------------------- 1 | .. note:: All functions of these modules are also directly accessable from the ``pyDive`` module. 2 | 3 | pyDive.arrayOfStructs module 4 | ---------------------------- 5 | 6 | .. automodule:: pyDive.arrayOfStructs 7 | :members: 8 | 9 | pyDive.algorithm module 10 | ----------------------- 11 | 12 | .. automodule:: pyDive.algorithm 13 | :members: 14 | 15 | pyDive.fragment module 16 | ---------------------- 17 | 18 | .. automodule:: pyDive.fragment 19 | :members: 20 | 21 | pyDive.mappings module 22 | ---------------------- 23 | 24 | .. automodule:: pyDive.mappings 25 | :members: 26 | 27 | pyDive.picongpu module 28 | ---------------------- 29 | 30 | .. automodule:: pyDive.picongpu 31 | :members: 32 | 33 | pyDive.pyDive module 34 | -------------------- 35 | 36 | .. automodule:: pyDive.pyDive 37 | :members: 38 | 39 | -------------------------------------------------------------------------------- /docs/pyDive.adios.rst: -------------------------------------------------------------------------------- 1 | 2 | pyDive.arrays.ad_ndarray module 3 | =============================== 4 | 5 | .. note:: This module has a shortcut: ``pyDive.adios``. 6 | 7 | .. autoclass:: pyDive.arrays.ad_ndarray.ad_ndarray 8 | :members: __init__, load 9 | 10 | .. automodule:: pyDive.arrays.ad_ndarray 11 | :members: 12 | -------------------------------------------------------------------------------- /docs/pyDive.cloned_ndarray.rst: -------------------------------------------------------------------------------- 1 | .. _pyDive.cloned_ndarray: 2 | 3 | pyDive.cloned_ndarray package 4 | ============================= 5 | 6 | Submodules 7 | ---------- 8 | 9 | pyDive.cloned_ndarray.cloned_ndarray module 10 | ------------------------------------------- 11 | 12 | .. autoclass:: pyDive.cloned_ndarray.cloned_ndarray.cloned_ndarray 13 | :members: __init__, merge, sum 14 | 15 | pyDive.cloned_ndarray.factories module 16 | -------------------------------------- 17 | 18 | .. automodule:: pyDive.cloned_ndarray.factories 19 | :members: 20 | 21 | -------------------------------------------------------------------------------- /docs/pyDive.gpu.rst: -------------------------------------------------------------------------------- 1 | .. _pyDive.h5_ndarray: 2 | 3 | pyDive.arrays.gpu_ndarray module 4 | =============================== 5 | 6 | .. note:: This module has a shortcut: ``pyDive.gpu``. 7 | 8 | .. autoclass:: pyDive.arrays.gpu_ndarray.gpu_ndarray 9 | :members: __init__, to_cpu 10 | 11 | .. automodule:: pyDive.arrays.gpu_ndarray 12 | :members: 13 | -------------------------------------------------------------------------------- /docs/pyDive.h5.rst: -------------------------------------------------------------------------------- 1 | .. _pyDive.h5_ndarray: 2 | 3 | pyDive.arrays.h5_ndarray module 4 | =============================== 5 | 6 | .. note:: This module has a shortcut: ``pyDive.h5``. 7 | 8 | .. autoclass:: pyDive.arrays.h5_ndarray.h5_ndarray 9 | :members: __init__, load 10 | 11 | .. automodule:: pyDive.arrays.h5_ndarray 12 | :members: 13 | -------------------------------------------------------------------------------- /docs/pyDive.ndarray.rst: -------------------------------------------------------------------------------- 1 | .. _pyDive.ndarray: 2 | 3 | pyDive.arrays.ndarray module 4 | ============================ 5 | 6 | .. note:: All of this module's functions and classes are also directly accessable from the :mod:`pyDive` module. 7 | 8 | pyDive.ndarray class 9 | -------------------- 10 | 11 | .. autoclass:: pyDive.ndarray 12 | :members: __init__, gather, copy, dist_like 13 | 14 | Factory functions 15 | ----------------- 16 | These are convenient functions to create a *pyDive.ndarray* instance. 17 | 18 | .. automodule:: pyDive.arrays.ndarray 19 | :members: array, empty, empty_like, hollow, hollow_like, zeros, zeros_like, ones, ones_like 20 | 21 | Universal functions 22 | ------------------- 23 | 24 | *numpy* knows the so called *ufuncs* (universal function). These are functions which can be applied 25 | elementwise on an array, like *sin*, *cos*, *exp*, *sqrt*, etc. All of these *ufuncs* from *numpy* are 26 | also available for *pyDive.ndarray* arrays, e.g. :: 27 | 28 | a = pyDive.ones([100]) 29 | a = pyDive.sin(a) 30 | 31 | -------------------------------------------------------------------------------- /docs/reference.rst: -------------------------------------------------------------------------------- 1 | Reference 2 | ========= 3 | 4 | Arrays 5 | -------- 6 | 7 | .. toctree:: 8 | 9 | pyDive.ndarray 10 | pyDive.h5 11 | pyDive.adios 12 | pyDive.gpu 13 | pyDive.cloned_ndarray 14 | 15 | Modules 16 | ------- 17 | 18 | .. toctree:: 19 | 20 | modules 21 | -------------------------------------------------------------------------------- /docs/sample.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ComputationalRadiationPhysics/pyDive/67d36420c2f1d59bef8633b2dea94b4ea4ab8ceb/docs/sample.h5 -------------------------------------------------------------------------------- /docs/spectrum.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ComputationalRadiationPhysics/pyDive/67d36420c2f1d59bef8633b2dea94b4ea4ab8ceb/docs/spectrum.png -------------------------------------------------------------------------------- /docs/start.rst: -------------------------------------------------------------------------------- 1 | Getting started 2 | =============== 3 | 4 | Quickstart 5 | ---------- 6 | 7 | pyDive is built on top of *IPython.parallel*, *numpy* and *mpi4py*. *h5py*, *adios* and *pycuda* are optional. Running ``python setup.py install`` will install 8 | pyDive with these and other required packages from `requirements.txt`. Alternatively you can install it via pip: ``pip install pyDive``. 9 | 10 | Basic code example: :: 11 | 12 | import pyDive 13 | pyDive.init(profile='mpi') 14 | 15 | arrayA = pyDive.ones((1000, 1000, 1000), distaxes='all') 16 | arrayB = pyDive.zeros_like(arrayA) 17 | 18 | # do some array operations, + - * / sin cos, ..., slicing, etc... 19 | arrayC = arrayA + arrayB 20 | 21 | # plot result 22 | import matplotlib.pyplot as plt 23 | plt.imshow(arrayC[500,::10,::10]) 24 | 25 | Before actually running this script there must have been an IPython.parallel cluster launched (see section below) otherwise `pyDive.init()` fails. 26 | 27 | pyDive distributes array-memory along one or multiple user-specified axes: 28 | 29 | .. image:: decomposition.png 30 | 31 | You can either specify the exact decomposition for each axis or leave the default which persuits to squared chunks. 32 | 33 | Although the array elements are stored on the cluster nodes you have full access through indexing. If you want to have a local array 34 | from a pyDive-array anyway you can call ``array.gather()`` but make sure that your pyDive-array is small enough to fit 35 | into your local machine's memory. If not you may want to slice it first. Note that an array is also gathered implicitly 36 | if you try to access an attribute which is only available for the local array. This is why there is no ``gather()`` in the example above 37 | when calling ``imshow()``. 38 | 39 | .. _cluster-config: 40 | 41 | Setup an IPython.parallel cluster configuration 42 | ----------------------------------------------- 43 | 44 | The first step is to create an IPython.parallel profile: http://ipython.org/ipython-doc/2/parallel/parallel_process.html. 45 | The name of this profile is the argument of :func:`pyDive.init`. It defaults to ``"mpi"``. 46 | Starting the cluster is then the second and final step:: 47 | 48 | $ ipcluster start -n 4 --profile=mpi 49 | 50 | Run tests 51 | --------- 52 | 53 | In order to test the pyDive installation run:: 54 | 55 | $ python setup.py test 56 | 57 | This will ask you for the IPython.parallel profile to be used and the number of engines to be started, e.g.: :: 58 | 59 | $ Name of your IPython-parallel profile you want to run the tests with: pbs 60 | $ Number of engines: 256 61 | 62 | Then the script starts the cluster, runs the tests and finally stops the cluster. If you have already a cluster running by your own 63 | you can also run the tests by launching ``py.test`` from the pyDive directory and setting the environment variable ``IPP_PROFILE_NAME`` 64 | to the profile's name. 65 | 66 | Overview 67 | -------- 68 | 69 | pyDive knows different kinds of distributed arrays, all corresponding to a local, non-distributed array: 70 | - numpy -> :obj:`pyDive.ndarray` -> Stores array elements in cluster nodes' memory. 71 | - hdf5 -> :obj:`pyDive.h5.h5_ndarray` -> Stores array elements in a hdf5-file. 72 | - adios -> :obj:`pyDive.ad.ad_ndarray` -> Stores array elements in a adios-file. 73 | - gpu -> :obj:`pyDive.gpu.gpu_ndarray` -> Stores array elements in clusters' gpus. 74 | - :obj:`pyDive.cloned_ndarray` -> Holds independent copies of one array on cluster nodes. 75 | 76 | Among these three packages there are a few modules: 77 | - :mod:`pyDive.structered` -> structured datatypes 78 | - :mod:`pyDive.algorithm` -> map, reduce, mapReduce 79 | - :mod:`pyDive.fragment` -> fragment file-disk array to fit into the cluster's main memory 80 | - :mod:`pyDive.mappings` -> particle-mesh mappings 81 | - :mod:`pyDive.picongpu` -> helper functions for picongpu-users 82 | - :mod:`pyDive.pyDive` -> shortcuts for most used functions 83 | 84 | -------------------------------------------------------------------------------- /docs/tutorial.rst: -------------------------------------------------------------------------------- 1 | Tutorials 2 | ========= 3 | 4 | In this section we are going through a few use cases for pyDive. If you want to test the code you can download 5 | the :download:`sample hdf5-file`. 6 | It has the following dataset structure:: 7 | 8 | $ h5ls -r sample.h5 9 | / Group 10 | /fields Group 11 | /fields/fieldB Group 12 | /fields/fieldB/z Dataset {256, 256} 13 | /fields/fieldE Group 14 | /fields/fieldE/x Dataset {256, 256} 15 | /fields/fieldE/y Dataset {256, 256} 16 | /particles Group 17 | /particles/cellidx Group 18 | /particles/cellidx/x Dataset {10000} 19 | /particles/cellidx/y Dataset {10000} 20 | /particles/pos Group 21 | /particles/pos/x Dataset {10000} 22 | /particles/pos/y Dataset {10000} 23 | /particles/vel Group 24 | /particles/vel/x Dataset {10000} 25 | /particles/vel/y Dataset {10000} 26 | /particles/vel/z Dataset {10000} 27 | 28 | After launching the cluster (:ref:`cluster-config`) the first step is to initialize pyDive: :: 29 | 30 | import pyDive 31 | pyDive.init() 32 | 33 | Load a single dataset: :: 34 | 35 | h5fieldB_z = pyDive.h5.open("sample.h5", "/fields/fieldB/z", distaxes='all') 36 | 37 | assert type(h5fieldB_z) is pyDive.h5.h5_ndarray 38 | 39 | *h5fieldB_z* just holds a dataset *handle*. To read out data into memory call ``load()``: :: 40 | 41 | fieldB_z = h5fieldB_z.load() 42 | 43 | assert type(fieldB_z) is pyDive.ndarray 44 | 45 | This loads the entire dataset into the main memory of all :term:`engines`. The array elements are distributed 46 | along all axes. 47 | 48 | We can also load a hdf5-group: :: 49 | 50 | h5fieldE = pyDive.h5.open("sample.h5", "/fields/fieldE", distaxes='all') 51 | fieldE = h5fieldE.load() 52 | 53 | *h5fieldE* and *fieldE* are some so called "virtual array-of-structures", see: :mod:`pyDive.structered`. :: 54 | 55 | >>> print h5fieldE 56 | VirtualArrayOfStructs, shape: [256, 256]>: 57 | y -> float32 58 | x -> float32 59 | 60 | >>> print fieldE 61 | VirtualArrayOfStructs, shape: [256, 256]>: 62 | y -> float32 63 | x -> float32 64 | 65 | Now, let's do some calculations! 66 | 67 | Example 1: Total field energy 68 | ----------------------------- 69 | 70 | Computing the total field energy of an electromagnetic field means squaring and summing or in pyDive's words: :: 71 | 72 | import pyDive 73 | import numpy as np 74 | pyDive.init() 75 | 76 | h5input = "sample.h5" 77 | 78 | h5fields = pyDive.h5.open(h5input, "/fields") # defaults to distaxes='all' 79 | fields = h5fields.load() # read out all fields into cluster's main memory in parallel 80 | 81 | energy_field = fields.fieldE.x**2 + fields.fieldE.y**2 + fields.fieldB.z**2 82 | 83 | total_energy = pyDive.reduce(energy_field, np.add) 84 | print total_energy 85 | 86 | Output: :: 87 | 88 | $ python example1.py 89 | 557502.0 90 | 91 | Well this was just a very small hdf5-sample of 1.3 MB however in real world we deal with a lot greater data volumes. 92 | So what happens if *h5fields* is too large to be stored in the main memory of the whole cluster? The line ``fields = h5fields.load()`` will crash. 93 | In this case we want to load the hdf5 data piece by piece. The function :obj:`pyDive.fragment` helps us doing so: :: 94 | 95 | import pyDive 96 | import numpy as np 97 | pyDive.init() 98 | 99 | h5input = "sample.h5" 100 | 101 | big_h5fields = pyDive.h5.open(h5input, "/fields") 102 | # big_h5fields.load() # would cause a crash 103 | 104 | total_energy = 0.0 105 | for h5fields in pyDive.fragment(big_h5fields): 106 | fields = h5fields.load() 107 | 108 | energy_field = fields.fieldE.x**2 + fields.fieldE.y**2 + fields.fieldB.z**2 109 | 110 | total_energy += pyDive.reduce(energy_field, np.add) 111 | 112 | print total_energy 113 | 114 | An equivalent way to get this result is a :obj:`pyDive.mapReduce`: :: 115 | 116 | ... 117 | def square_fields(h5fields): 118 | fields = h5fields.load() 119 | return fields.fieldE.x**2 + fields.fieldE.y**2 + fields.fieldB.z**2 120 | 121 | total_energy = pyDive.mapReduce(square_fields, np.add, h5fields) 122 | print total_energy 123 | 124 | *square_fields* is called on each :term:`engine` where *h5fields* is a structure (:mod:`pyDive.arrayOfStructs`) of :obj:`h5_ndarrays` representing a sub part of the big *h5fields*. 125 | :func:`pyDive.algorithm.mapReduce` can be called with an arbitrary number of arrays including 126 | :obj:`pyDive.ndarrays`, :obj:`pyDive.h5.h5_ndarrays`, :obj:`pyDive.adios.ad_ndarrays` and :obj:`pyDive.cloned_ndarrays`. If there are :obj:`pyDive.h5.h5_ndarrays` or :obj:`pyDive.adios.ad_ndarrays` it will 127 | check whether they fit into the combined main memory of all cluster nodes as a whole and loads them piece by piece if not. 128 | 129 | Now let's say our dataset is really big and we just want to get a first estimate of the total energy: :: 130 | 131 | ... 132 | total_energy = pyDive.mapReduce(square_fields, np.add, h5fields[::10, ::10]) * 10.0**2 133 | 134 | Slicing on pyDive-arrays is always allowed. 135 | 136 | If you use `picongpu `_ 137 | here is an example of how to get the total field energy for each timestep (see :mod:`pyDive.picongpu`): :: 138 | 139 | import pyDive 140 | import numpy as np 141 | pyDive.init() 142 | 143 | def square_field(h5field): 144 | field = h5field.load() 145 | return field.x**2 + field.x**2 + field.x**2 146 | 147 | for step, h5field in pyDive.picongpu.loadAllSteps("/.../simOutput", "fields/FieldE"): 148 | total_energy = pyDive.mapReduce(square_field, np.add, h5field) 149 | 150 | print step, total_energy 151 | 152 | Example 2: Particle density field 153 | --------------------------------- 154 | 155 | Given the list of particles in our ``sample.h5`` we want to create a 2D density field out of it. For this particle-to-mesh 156 | mapping we need to apply a certain particle shape like cloud-in-cell (CIC), triangular-shaped-cloud (TSC), and so on. A list of 157 | these together with the actual mapping functions can be found in the :mod:`pyDive.mappings` module. If you miss a shape you can 158 | easily create one by your own by defining a particle shape function. Note that if you have `numba `_ 159 | installed the shape function will be compiled resulting in a significant speed-up. 160 | 161 | We assume that the particle positions are distributed randomly. This means although each engine is loading a separate part of all particles it needs to 162 | write to the entire density field. Therefore the density field must have a whole representation on each participating engine. 163 | This is the job of :class:`pyDive.cloned_ndarray.cloned_ndarray.cloned_ndarray`. :: 164 | 165 | import pyDive 166 | import numpy as np 167 | pyDive.init() 168 | 169 | shape = [256, 256] 170 | density = pyDive.cloned.zeros(shape) 171 | 172 | h5input = "sample.h5" 173 | 174 | particles = pyDive.h5.open(h5input, "/particles") 175 | 176 | def particles2density(particles, density): 177 | particles = particles.load() 178 | total_pos = particles.cellidx.astype(np.float32) + particles.pos 179 | 180 | # convert total_pos to an (N, 2) shaped array 181 | total_pos = np.hstack((total_pos.x[:,np.newaxis], 182 | total_pos.y[:,np.newaxis])) 183 | 184 | par_weighting = np.ones(particles.shape) 185 | import pyDive.mappings 186 | pyDive.mappings.particles2mesh(density, par_weighting, total_pos, pyDive.mappings.CIC) 187 | 188 | pyDive.map(particles2density, particles, density) 189 | 190 | final_density = density.sum() # add up all local copies 191 | 192 | from matplotlib import pyplot as plt 193 | plt.imshow(final_density) 194 | plt.show() 195 | 196 | Output: 197 | 198 | .. image:: density.png 199 | 200 | Here, as in the first example, *particles2density* is a function executed on the :term:`engines ` by :func:`pyDive.algorithm.map`. 201 | All of its arguments are numpy-arrays or structures (:mod:`pyDive.arrayOfStructs`) of numpy-arrays. 202 | 203 | :func:`pyDive.algorithm.map` can also be used as a decorator: :: 204 | 205 | @pyDive.map 206 | def particles2density(particles, density): 207 | ... 208 | 209 | particles2density(particles, density) 210 | 211 | 212 | Example 3: Particle energy spectrum 213 | ----------------------------------- 214 | 215 | :: 216 | 217 | import pyDive 218 | import numpy as np 219 | pyDive.init() 220 | 221 | bins = 256 222 | spectrum = pyDive.cloned.zeros([bins]) 223 | 224 | h5input = "sample.h5" 225 | 226 | velocities = pyDive.h5.open(h5input, "/particles/vel") 227 | 228 | @pyDive.map 229 | def vel2spectrum(velocities, spectrum, bins): 230 | velocities = velocities.load() 231 | mass = 1.0 232 | energies = 0.5 * mass * (velocities.x**2 + velocities.y**2 + velocities.z**2) 233 | 234 | spectrum[:], bin_edges = np.histogram(energies, bins) 235 | 236 | vel2spectrum(velocities, spectrum, bins=bins) 237 | 238 | final_spectrum = spectrum.sum() # add up all local copies 239 | 240 | from matplotlib import pyplot as plt 241 | plt.plot(final_spectrum) 242 | plt.show() 243 | 244 | Output: 245 | 246 | .. image:: spectrum.png 247 | 248 | -------------------------------------------------------------------------------- /pyDive/IPParallelClient.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2014 Heiko Burau 3 | 4 | This file is part of pyDive. 5 | 6 | pyDive is free software: you can redistribute it and/or modify 7 | it under the terms of of either the GNU General Public License or 8 | the GNU Lesser General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | pyDive is distributed in the hope that it will be useful, 12 | but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | GNU General Public License and the GNU Lesser General Public License 15 | for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | and the GNU Lesser General Public License along with pyDive. 19 | If not, see . 20 | """ 21 | __doc__ = None 22 | 23 | from IPython.parallel import Client 24 | from IPython.parallel import interactive 25 | from collections import Counter 26 | import sys 27 | 28 | #: IPParallel direct view 29 | view = None 30 | #: number of processes per node 31 | ppn = None 32 | 33 | def init(profile='mpi'): 34 | """Initialize pyDive. 35 | 36 | :param str profile: The name of the cluster profile of *IPython.parallel*. Has to be an MPI-profile.\ 37 | Defaults to 'mpi'. 38 | """ 39 | # init direct view 40 | global view 41 | 42 | client = Client(profile=profile) 43 | client.clear() 44 | view = client[:] 45 | view.block = True 46 | view.execute('''\ 47 | import numpy as np 48 | from mpi4py import MPI 49 | import h5py as h5 50 | import os, sys 51 | import psutil 52 | import math 53 | os.environ["onTarget"] = 'True' 54 | from pyDive import structured 55 | from pyDive import algorithm 56 | from pyDive.distribution import interengine 57 | try: 58 | import pyDive.arrays.local.h5_ndarray 59 | except ImportError: 60 | pass 61 | try: 62 | import pyDive.arrays.local.ad_ndarray 63 | except ImportError: 64 | pass 65 | try: 66 | import pyDive.arrays.local.gpu_ndarray 67 | import pycuda.autoinit 68 | except ImportError: 69 | pass 70 | ''') 71 | 72 | # get number of processes per node (ppn) 73 | def hostname(): 74 | import socket 75 | return socket.gethostname() 76 | hostnames = view.apply(interactive(hostname)) 77 | global ppn 78 | ppn = max(Counter(hostnames).values()) 79 | 80 | # mpi ranks 81 | get_rank = interactive(lambda: MPI.COMM_WORLD.Get_rank()) 82 | all_ranks = view.apply(get_rank) 83 | view['target2rank'] = all_ranks 84 | 85 | def getView(): 86 | global view 87 | assert view is not None, "pyDive.init() has not been called yet." 88 | return view 89 | 90 | def getPPN(): 91 | global ppn 92 | assert ppn is not None, "pyDive.init() has not been called yet." 93 | return ppn 94 | -------------------------------------------------------------------------------- /pyDive/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2014 Heiko Burau 3 | 4 | This file is part of pyDive. 5 | 6 | pyDive is free software: you can redistribute it and/or modify 7 | it under the terms of of either the GNU General Public License or 8 | the GNU Lesser General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | pyDive is distributed in the hope that it will be useful, 12 | but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | GNU General Public License and the GNU Lesser General Public License 15 | for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | and the GNU Lesser General Public License along with pyDive. 19 | If not, see . 20 | """ 21 | __doc__ = None 22 | 23 | # -*- coding: utf-8 -*- 24 | __version__ = "1.2.2" 25 | 26 | import os 27 | onTarget = os.environ.get("onTarget", 'False') 28 | 29 | # import only if this code is not executed on engine 30 | if onTarget == 'False': 31 | from pyDive import * 32 | -------------------------------------------------------------------------------- /pyDive/algorithm.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2014 Heiko Burau 3 | 4 | This file is part of pyDive. 5 | 6 | pyDive is free software: you can redistribute it and/or modify 7 | it under the terms of of either the GNU General Public License or 8 | the GNU Lesser General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | pyDive is distributed in the hope that it will be useful, 12 | but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | GNU General Public License and the GNU Lesser General Public License 15 | for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | and the GNU Lesser General Public License along with pyDive. 19 | If not, see . 20 | """ 21 | __doc__ = None 22 | 23 | import os 24 | # check whether this code is executed on target or not 25 | onTarget = os.environ.get("onTarget", 'False') 26 | if onTarget == 'False': 27 | from arrays.ndarray import ndarray 28 | from arrays.h5_ndarray import h5_ndarray 29 | from cloned_ndarray.cloned_ndarray import cloned_ndarray 30 | import IPParallelClient as com 31 | from IPython.parallel import interactive 32 | from fragment import fragment, hdd_arraytypes 33 | from structured import VirtualArrayOfStructs 34 | import numpy as np 35 | 36 | def map(f, *arrays, **kwargs): 37 | """Applies *f* on :term:`engine` on local arrays related to *arrays*. 38 | Example: :: 39 | 40 | cluster_array = pyDive.ones(shape=[100], distaxes=0) 41 | 42 | cluster_array *= 2.0 43 | # equivalent to 44 | pyDive.map(lambda a: a *= 2.0, cluster_array) # a is the local numpy-array of *cluster_array* 45 | 46 | Or, as a decorator: :: 47 | 48 | @pyDive.map 49 | def twice(a): 50 | a *= 2.0 51 | 52 | twice(cluster_array) 53 | 54 | :param callable f: function to be called on :term:`engine`. Has to accept *numpy-arrays* and *kwargs* 55 | :param arrays: list of arrays including *pyDive.ndarrays*, *pyDive.h5_ndarrays* or *pyDive.cloned_ndarrays* 56 | :param kwargs: user-specified keyword arguments passed to *f* 57 | :raises AssertionError: if the *shapes* of *pyDive.ndarrays* and *pyDive.h5_ndarrays* do not match 58 | :raises AssertionError: if the *distaxes* attributes of *pyDive.ndarrays* and *pyDive.h5_ndarrays* do not match 59 | 60 | Notes: 61 | - If the hdf5 data exceeds the memory limit (currently 25% of the combined main memory of all cluster nodes)\ 62 | the data will be read block-wise so that a block fits into memory. 63 | - *map* chooses the list of *engines* from the **first** element of *arrays*. On these engines *f* is called.\ 64 | If the first array is a *pyDive.h5_ndarray* all engines will be used. 65 | - *map* is not writing data back to a *pyDive.h5_ndarray* yet. 66 | - *map* does not equalize the element distribution of *pyDive.ndarrays* before execution. 67 | """ 68 | if not arrays: 69 | # decorator mode 70 | def map_deco(*arrays, **kwargs): 71 | map(f, *arrays, **kwargs) 72 | return map_deco 73 | 74 | def map_wrapper(f, array_names, **kwargs): 75 | arrays = [globals()[array_name] for array_name in array_names] 76 | f(*arrays, **kwargs) 77 | 78 | view = com.getView() 79 | 80 | tmp_targets = view.targets # save current target list 81 | if type(arrays[0]) == VirtualArrayOfStructs: 82 | view.targets = arrays[0].firstArray.target_ranks 83 | else: 84 | view.targets = arrays[0].target_ranks 85 | 86 | hdd_arrays = [a for a in arrays if (hasattr(a, "arraytype") and a.arraytype in hdd_arraytypes) or type(a) in hdd_arraytypes] 87 | if hdd_arrays: 88 | cloned_arrays = [a for a in arrays if (hasattr(a, "arraytype") and a.arraytype is cloned_ndarray) or type(a) is cloned_ndarray] 89 | other_arrays = [a for a in arrays if not ((hasattr(a, "arraytype") and a.arraytype is cloned_ndarray) or type(a) is cloned_ndarray)] 90 | 91 | cloned_arrays_ids = [id(a) for a in cloned_arrays] 92 | other_arrays_ids = [id(a) for a in other_arrays] 93 | 94 | for fragments in fragment(*other_arrays): 95 | it_other_arrays = iter(other_arrays) 96 | it_cloned_arrays = iter(cloned_arrays) 97 | 98 | array_names = [] 99 | for a in arrays: 100 | if id(a) in cloned_arrays_ids: 101 | array_names.append(repr(it_cloned_arrays.next())) 102 | continue 103 | if id(a) in other_arrays_ids: 104 | array_names.append(repr(it_other_arrays.next())) 105 | continue 106 | 107 | view.apply(interactive(map_wrapper), interactive(f), array_names, **kwargs) 108 | else: 109 | array_names = [repr(a) for a in arrays] 110 | view.apply(interactive(map_wrapper), interactive(f), array_names, **kwargs) 111 | 112 | view.targets = tmp_targets # restore target list 113 | 114 | def reduce(array, op): 115 | """Perform a tree-like reduction over all axes of *array*. 116 | 117 | :param array: *pyDive.ndarray*, *pyDive.h5_ndarray* or *pyDive.cloned_ndarray* to be reduced 118 | :param numpy-ufunc op: reduce operation, e.g. *numpy.add*. 119 | 120 | If the hdf5 data exceeds the memory limit (currently 25% of the combined main memory of all cluster nodes)\ 121 | the data will be read block-wise so that a block fits into memory. 122 | """ 123 | def reduce_wrapper(array_name, op_name): 124 | array = globals()[array_name] 125 | op = eval("np." + op_name) 126 | return algorithm.__tree_reduce(array, axis=None, op=op) # reduction over all axes 127 | 128 | view = com.getView() 129 | 130 | tmp_targets = view.targets # save current target list 131 | if type(array) == VirtualArrayOfStructs: 132 | view.targets = array.firstArray.target_ranks 133 | else: 134 | view.targets = array.target_ranks 135 | 136 | result = None 137 | 138 | if (hasattr(array, "arraytype") and array.arraytype in hdd_arraytypes) or type(array) in hdd_arraytypes: 139 | for chunk in fragment(array): 140 | array_name = repr(chunk) 141 | 142 | targets_results = view.apply(interactive(reduce_wrapper), array_name, op.__name__) 143 | chunk_result = op.reduce(targets_results) # reduce over targets' results 144 | 145 | if result is None: 146 | result = chunk_result 147 | else: 148 | result = op(result, chunk_result) 149 | else: 150 | array_name = repr(array) 151 | 152 | targets_results = view.apply(interactive(reduce_wrapper), array_name, op.__name__) 153 | result = op.reduce(targets_results) # reduce over targets' results 154 | 155 | view.targets = tmp_targets # restore target list 156 | return result 157 | 158 | def mapReduce(map_func, reduce_op, *arrays, **kwargs): 159 | """Applies *map_func* on :term:`engine` on local arrays related to *arrays* 160 | and reduces its result in a tree-like fashion over all axes. 161 | Example: :: 162 | 163 | cluster_array = pyDive.ones(shape=[100], distaxes=0) 164 | 165 | s = pyDive.mapReduce(lambda a: a**2, np.add, cluster_array) # a is the local numpy-array of *cluster_array* 166 | assert s == 100 167 | 168 | :param callable f: function to be called on :term:`engine`. Has to accept *numpy-arrays* and *kwargs* 169 | :param numpy-ufunc reduce_op: reduce operation, e.g. *numpy.add*. 170 | :param arrays: list of arrays including *pyDive.ndarrays*, *pyDive.h5_ndarrays* or *pyDive.cloned_ndarrays* 171 | :param kwargs: user-specified keyword arguments passed to *f* 172 | :raises AssertionError: if the *shapes* of *pyDive.ndarrays* and *pyDive.h5_ndarrays* do not match 173 | :raises AssertionError: if the *distaxes* attributes of *pyDive.ndarrays* and *pyDive.h5_ndarrays* do not match 174 | 175 | Notes: 176 | - If the hdf5 data exceeds the memory limit (currently 25% of the combined main memory of all cluster nodes)\ 177 | the data will be read block-wise so that a block fits into memory. 178 | - *mapReduce* chooses the list of *engines* from the **first** element of *arrays*. On these engines the mapReduce will be executed.\ 179 | If the first array is a *pyDive.h5_ndarray* all engines will be used. 180 | - *mapReduce* is not writing data back to a *pyDive.h5_ndarray* yet. 181 | - *mapReduce* does not equalize the element distribution of *pyDive.ndarrays* before execution. 182 | """ 183 | def mapReduce_wrapper(map_func, reduce_op_name, array_names, **kwargs): 184 | arrays = [globals()[array_name] for array_name in array_names] 185 | reduce_op = eval("np." + reduce_op_name) 186 | return algorithm.__tree_reduce(map_func(*arrays, **kwargs), axis=None, op=reduce_op) 187 | 188 | view = com.getView() 189 | tmp_targets = view.targets # save current target list 190 | if type(arrays[0]) == VirtualArrayOfStructs: 191 | view.targets = arrays[0].firstArray.target_ranks 192 | else: 193 | view.targets = arrays[0].target_ranks 194 | 195 | result = None 196 | 197 | hdd_arrays = [a for a in arrays if (hasattr(a, "arraytype") and a.arraytype in hdd_arraytypes) or type(a) in hdd_arraytypes] 198 | if hdd_arrays: 199 | cloned_arrays = [a for a in arrays if (hasattr(a, "arraytype") and a.arraytype is cloned_ndarray) or type(a) is cloned_ndarray] 200 | other_arrays = [a for a in arrays if not ((hasattr(a, "arraytype") and a.arraytype is cloned_ndarray) or type(a) is cloned_ndarray)] 201 | 202 | cloned_arrays_ids = [id(a) for a in cloned_arrays] 203 | other_arrays_ids = [id(a) for a in other_arrays] 204 | 205 | for fragments in fragment(*other_arrays): 206 | it_other_arrays = iter(other_arrays) 207 | it_cloned_arrays = iter(cloned_arrays) 208 | 209 | array_names = [] 210 | for a in arrays: 211 | if id(a) in cloned_arrays_ids: 212 | array_names.append(repr(it_cloned_arrays.next())) 213 | continue 214 | if id(a) in other_arrays_ids: 215 | array_names.append(repr(it_other_arrays.next())) 216 | continue 217 | 218 | targets_results = view.apply(interactive(mapReduce_wrapper),\ 219 | interactive(map_func), reduce_op.__name__, array_names, **kwargs) 220 | 221 | fragment_result = reduce_op.reduce(targets_results) # reduce over targets' results 222 | if result is None: 223 | result = fragment_result 224 | else: 225 | result = reduce_op(result, fragment_result) 226 | else: 227 | array_names = [repr(a) for a in arrays] 228 | targets_results = view.apply(interactive(mapReduce_wrapper),\ 229 | interactive(map_func), reduce_op.__name__, array_names, **kwargs) 230 | 231 | result = reduce_op.reduce(targets_results) # reduce over targets' results 232 | 233 | view.targets = tmp_targets # restore target list 234 | 235 | return result 236 | 237 | def __tree_reduce(array, axis=None, op=np.add): 238 | # reduce all axes 239 | if axis is None: 240 | result = array 241 | for axis in range(len(array.shape))[::-1]: 242 | result = __tree_reduce(result, axis=axis, op=op) 243 | return result 244 | 245 | assert 0 <= axis and axis < len(array.shape) 246 | result = np.rollaxis(array, axis) 247 | 248 | while True: 249 | l = result.shape[0] 250 | 251 | # if axis is small enough to neglect rounding errors do a faster numpy-reduce 252 | if l < 10000: 253 | return op.reduce(result, axis=0) 254 | 255 | if l % 2 == 0: 256 | result = op(result[0:l/2], result[l/2:]) 257 | else: 258 | tmp = result[-1] 259 | result = op(result[0:l/2], result[l/2:-1]) 260 | result[0] = op(result[0], tmp) 261 | -------------------------------------------------------------------------------- /pyDive/arrays/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2015 Heiko Burau 3 | 4 | This file is part of pyDive. 5 | 6 | pyDive is free software: you can redistribute it and/or modify 7 | it under the terms of of either the GNU General Public License or 8 | the GNU Lesser General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | pyDive is distributed in the hope that it will be useful, 12 | but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | GNU General Public License and the GNU Lesser General Public License 15 | for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | and the GNU Lesser General Public License along with pyDive. 19 | If not, see . 20 | """ 21 | __doc__ = None 22 | 23 | # -*- coding: utf-8 -*- 24 | __all__ = ["h5_ndarray.py", "ndarray.py"] -------------------------------------------------------------------------------- /pyDive/arrays/ad_ndarray.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2015 Heiko Burau 3 | 4 | This file is part of pyDive. 5 | 6 | pyDive is free software: you can redistribute it and/or modify 7 | it under the terms of of either the GNU General Public License or 8 | the GNU Lesser General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | pyDive is distributed in the hope that it will be useful, 12 | but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | GNU General Public License and the GNU Lesser General Public License 15 | for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | and the GNU Lesser General Public License along with pyDive. 19 | If not, see . 20 | """ 21 | __doc__ = None 22 | 23 | import numpy as np 24 | import adios as ad 25 | from pyDive.arrays.ndarray import hollow_like 26 | import pyDive.distribution.multiple_axes as multiple_axes 27 | import pyDive.IPParallelClient as com 28 | from itertools import islice 29 | from .. import structured 30 | import pyDive.arrays.local.ad_ndarray 31 | 32 | ad_ndarray = multiple_axes.distribute(pyDive.arrays.local.ad_ndarray.ad_ndarray, "ad_ndarray", "ad_ndarray", may_allocate=False) 33 | 34 | def load(self): 35 | """Load array from file into main memory of all engines in parallel. 36 | 37 | :return: pyDive.ndarray instance 38 | """ 39 | result = hollow_like(self) 40 | view = com.getView() 41 | view.execute("{0} = {1}.load()".format(result.name, self.name), targets=result.target_ranks) 42 | return result 43 | ad_ndarray.load = load 44 | del load 45 | 46 | def open_variable(filename, variable_path, distaxis=0): 47 | """Create a pyDive.adios.ad_ndarray instance from file. 48 | 49 | :param filename: name of adios file. 50 | :param variable_path: path within adios file to a single variable. 51 | :param distaxis int: distributed axis 52 | :return: pyDive.adios.ad_ndarray instance 53 | """ 54 | fileHandle = ad.file(filename) 55 | variable = fileHandle.var[variable_path] 56 | dtype = variable.type 57 | shape = tuple(variable.dims) 58 | fileHandle.close() 59 | 60 | result = ad_ndarray(shape, dtype, distaxis, None, None, True) 61 | 62 | target_shapes = result.target_shapes() 63 | target_offset_vectors = result.target_offset_vectors() 64 | 65 | view = com.getView() 66 | view.scatter("shape", target_shapes, targets=result.target_ranks) 67 | view.scatter("offset", target_offset_vectors, targets=result.target_ranks) 68 | view.execute("{0} = pyDive.arrays.local.ad_ndarray.ad_ndarray('{1}','{2}',shape=shape[0],offset=offset[0])"\ 69 | .format(result.name, filename, variable_path), targets=result.target_ranks) 70 | 71 | return result 72 | 73 | def open(filename, datapath, distaxis=0): 74 | """Create a pyDive.adios.ad_ndarray instance respectively a structure of 75 | pyDive.adios.ad_ndarray instances from file. 76 | 77 | :param filename: name of adios file. 78 | :param datapath: path within adios file to a single variable or a group of variables. 79 | :param distaxis int: distributed axis 80 | :return: pyDive.adios.ad_ndarray instance 81 | """ 82 | fileHandle = ad.file(filename) 83 | variable_paths = fileHandle.var.keys() 84 | fileHandle.close() 85 | 86 | def update_tree(tree, variable_path, variable_path_iter, leaf): 87 | node = variable_path_iter.next() 88 | if node == leaf: 89 | tree[leaf] = open_variable(filename, variable_path, distaxis) 90 | return 91 | tree[node] = {} 92 | update_tree(tree[node], variable_path, variable_path_iter, leaf) 93 | 94 | n = len(datapath.split("/")) 95 | 96 | structOfArrays = {} 97 | for variable_path in variable_paths: 98 | if not variable_path.startswith(datapath): 99 | continue 100 | path_nodes = variable_path.split("/") 101 | path_nodes_it = iter(path_nodes) 102 | 103 | # advance 'path_nodes_it' n times 104 | next(islice(path_nodes_it, n, n), None) 105 | 106 | update_tree(structOfArrays, variable_path, path_nodes_it, path_nodes[-1]) 107 | 108 | return structured.structured(structOfArrays) 109 | -------------------------------------------------------------------------------- /pyDive/arrays/gpu_ndarray.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2015 Heiko Burau 3 | 4 | This file is part of pyDive. 5 | 6 | pyDive is free software: you can redistribute it and/or modify 7 | it under the terms of of either the GNU General Public License or 8 | the GNU Lesser General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | pyDive is distributed in the hope that it will be useful, 12 | but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | GNU General Public License and the GNU Lesser General Public License 15 | for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | and the GNU Lesser General Public License along with pyDive. 19 | If not, see . 20 | """ 21 | __doc__ = None 22 | 23 | import numpy as np 24 | import pyDive.distribution.multiple_axes as multiple_axes 25 | from pyDive.distribution.interengine import GPU_copier 26 | import pyDive.arrays.local.gpu_ndarray 27 | 28 | gpu_ndarray = multiple_axes.distribute(pyDive.arrays.local.gpu_ndarray.gpu_ndarray, "gpu_ndarray",\ 29 | "pyDive.arrays.local.gpu_ndarray", interengine_copier=GPU_copier) 30 | 31 | factories = multiple_axes.generate_factories(gpu_ndarray, ("empty", "zeros"), np.float) 32 | factories.update(multiple_axes.generate_factories_like(gpu_ndarray, ("empty_like", "zeros_like"))) 33 | globals().update(factories) 34 | 35 | def ones(shape, dtype=np.float, distaxes='all', **kwargs): 36 | result = zeros(shape, dtype, distaxes, **kwargs) 37 | result += 1 38 | return result 39 | 40 | def ones_like(other, **kwargs): 41 | result = zeros_like(other, **kwargs) 42 | result += 1 43 | return result 44 | 45 | import pyDive.IPParallelClient as com 46 | import pyDive.arrays.ndarray 47 | 48 | def to_cpu(self): 49 | """Copy array data to cpu main memory. 50 | 51 | :result pyDive.ndarray: distributed cpu array. 52 | """ 53 | result = pyDive.arrays.ndarray.hollow_like(self) 54 | view = com.getView() 55 | view.execute("{0} = {1}.to_cpu()".format(result.name, self.name), targets=result.target_ranks) 56 | return result 57 | gpu_ndarray.to_cpu = to_cpu 58 | del to_cpu 59 | 60 | def hollow(shape, dtype=np.float, distaxes='all'): 61 | """Create a pyDive.gpu_ndarray instance distributed across all engines without allocating a local 62 | gpu-array. 63 | 64 | :param ints shape: shape of array 65 | :param dtype: datatype of a single element 66 | :param ints distaxes: distributed axes. Defaults to 'all' meaning each axis is distributed. 67 | """ 68 | return gpu_ndarray(shape, dtype, distaxes, None, None, True) 69 | 70 | def hollow_like(other): 71 | """Create a pyDive.gpu_ndarray instance with the same 72 | shape, distribution and type as ``other`` without allocating a local gpu-array. 73 | """ 74 | return gpu_ndarray(other.shape, other.dtype, other.distaxes, other.target_offsets, other.target_ranks, True) 75 | 76 | def array(array_like, distaxes='all'): 77 | """Create a pyDive.gpu_ndarray instance from an array-like object. 78 | 79 | :param array_like: Any object exposing the array interface, e.g. numpy-array, python sequence, ... 80 | :param ints distaxis: distributed axes. Defaults to 'all' meaning each axis is distributed. 81 | """ 82 | result_cpu = pyDive.arrays.ndarray.array(array_like, distaxes) 83 | result = hollow_like(result_cpu) 84 | view = com.getView() 85 | view.execute("{0} = pyDive.arrays.local.gpu_ndarray.gpu_ndarray_cast(pycuda.gpuarray.to_gpu({1}))"\ 86 | .format(repr(result), repr(result_cpu)), targets=result.target_ranks) 87 | 88 | return result 89 | 90 | #ufunc_names = [key for key, value in np.__dict__.items() if isinstance(value, np.ufunc)] 91 | #ufuncs = multiple_axes.generate_ufuncs(ufunc_names, "np") 92 | 93 | #globals().update(ufuncs) -------------------------------------------------------------------------------- /pyDive/arrays/h5_ndarray.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2014 Heiko Burau 3 | 4 | This file is part of pyDive. 5 | 6 | pyDive is free software: you can redistribute it and/or modify 7 | it under the terms of of either the GNU General Public License or 8 | the GNU Lesser General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | pyDive is distributed in the hope that it will be useful, 12 | but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | GNU General Public License and the GNU Lesser General Public License 15 | for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | and the GNU Lesser General Public License along with pyDive. 19 | If not, see . 20 | """ 21 | __doc__ = None 22 | 23 | import numpy as np 24 | import h5py as h5 25 | from pyDive.arrays.ndarray import hollow_like 26 | import pyDive.distribution.multiple_axes as multiple_axes 27 | import pyDive.IPParallelClient as com 28 | from .. import structured 29 | import pyDive.arrays.local.h5_ndarray 30 | 31 | h5_ndarray = multiple_axes.distribute(pyDive.arrays.local.h5_ndarray.h5_ndarray, "h5_ndarray", "h5_ndarray", may_allocate=False) 32 | 33 | def load(self): 34 | """Load array from file into main memory of all engines in parallel. 35 | 36 | :return: pyDive.ndarray instance 37 | """ 38 | result = hollow_like(self) 39 | view = com.getView() 40 | view.execute("{0} = {1}.load()".format(result.name, self.name), targets=result.target_ranks) 41 | return result 42 | h5_ndarray.load = load 43 | del load 44 | 45 | def open_dset(filename, dataset_path, distaxes='all'): 46 | """Create a pyDive.h5.h5_ndarray instance from file. 47 | 48 | :param filename: name of hdf5 file. 49 | :param dataset_path: path within hdf5 file to a single dataset. 50 | :param distaxes ints: distributed axes. Defaults to 'all' meaning each axis is distributed. 51 | :return: pyDive.h5.h5_ndarray instance 52 | """ 53 | fileHandle = h5.File(filename, "r") 54 | dataset = fileHandle[dataset_path] 55 | dtype = dataset.dtype 56 | shape = dataset.shape 57 | fileHandle.close() 58 | 59 | result = h5_ndarray(shape, dtype, distaxes, None, None, True) 60 | 61 | target_shapes = result.target_shapes() 62 | target_offset_vectors = result.target_offset_vectors() 63 | 64 | view = com.getView() 65 | view.scatter("shape", target_shapes, targets=result.target_ranks) 66 | view.scatter("offset", target_offset_vectors, targets=result.target_ranks) 67 | view.execute("{0} = pyDive.arrays.local.h5_ndarray.h5_ndarray('{1}','{2}',shape=shape[0],offset=offset[0])"\ 68 | .format(result.name, filename, dataset_path), targets=result.target_ranks) 69 | 70 | return result 71 | 72 | def open(filename, datapath, distaxes='all'): 73 | """Create an pyDive.h5.h5_ndarray instance respectively a structure of 74 | pyDive.h5.h5_ndarray instances from file. 75 | 76 | :param filename: name of hdf5 file. 77 | :param dataset_path: path within hdf5 file to a single dataset or hdf5 group. 78 | :param distaxes ints: distributed axes. Defaults to 'all' meaning each axis is distributed. 79 | :return: pyDive.h5.h5_ndarray instance / structure of pyDive.h5.h5_ndarray instances 80 | """ 81 | hFile = h5.File(filename, 'r') 82 | datapath = datapath.rstrip("/") 83 | group_or_dataset = hFile[datapath] 84 | if type(group_or_dataset) is not h5._hl.group.Group: 85 | # dataset 86 | return open_dset(filename, datapath, distaxes) 87 | 88 | def create_tree(group, tree, dataset_path): 89 | for key, value in group.items(): 90 | # group 91 | if type(value) is h5._hl.group.Group: 92 | tree[key] = {} 93 | create_tree(value, tree[key], dataset_path + "/" + key) 94 | # dataset 95 | else: 96 | tree[key] = open_dset(filename, dataset_path + "/" + key, distaxes) 97 | 98 | group = group_or_dataset 99 | structOfArrays = {} 100 | create_tree(group, structOfArrays, datapath) 101 | return structured.structured(structOfArrays) 102 | -------------------------------------------------------------------------------- /pyDive/arrays/local/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Copyright 2015 Heiko Burau 4 | 5 | This file is part of pyDive. 6 | 7 | pyDive is free software: you can redistribute it and/or modify 8 | it under the terms of of either the GNU General Public License or 9 | the GNU Lesser General Public License as published by 10 | the Free Software Foundation, either version 3 of the License, or 11 | (at your option) any later version. 12 | pyDive is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | GNU General Public License and the GNU Lesser General Public License 16 | for more details. 17 | 18 | You should have received a copy of the GNU General Public License 19 | and the GNU Lesser General Public License along with pyDive. 20 | If not, see . 21 | """ 22 | __doc__ = None -------------------------------------------------------------------------------- /pyDive/arrays/local/ad_ndarray.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2015 Heiko Burau 3 | 4 | This file is part of pyDive. 5 | 6 | pyDive is free software: you can redistribute it and/or modify 7 | it under the terms of of either the GNU General Public License or 8 | the GNU Lesser General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | pyDive is distributed in the hope that it will be useful, 12 | but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | GNU General Public License and the GNU Lesser General Public License 15 | for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | and the GNU Lesser General Public License along with pyDive. 19 | If not, see . 20 | """ 21 | __doc__ = None 22 | 23 | 24 | import adios as ad 25 | import pyDive.distribution.helper as helper 26 | import numpy as np 27 | 28 | class ad_ndarray(object): 29 | 30 | def __init__(self, filename, variable_path, shape=None, window=None, offset=None): 31 | self.filename = filename 32 | self.variable_path = variable_path 33 | 34 | fileHandle = ad.file(filename) 35 | variable = fileHandle.var[variable_path] 36 | self.dtype = variable.type 37 | if shape is None: 38 | shape = variable.dims 39 | self.shape = tuple(shape) 40 | fileHandle.close() 41 | 42 | if window is None: 43 | window = [slice(0, s, 1) for s in shape] 44 | self.window = tuple(window) 45 | if offset is None: 46 | offset = (0,) * len(shape) 47 | self.offset = tuple(offset) 48 | 49 | #: total bytes consumed by the elements of the array. 50 | self.nbytes = np.dtype(self.dtype).itemsize * np.prod(self.shape) 51 | 52 | def load(self): 53 | begin = [] 54 | size = [] 55 | for o, w in zip(self.offset, self.window): 56 | if type(w) is int: 57 | begin.append(o+w) 58 | size.append(1) 59 | else: 60 | begin.append(o+w.start) 61 | size.append(w.stop-w.start) 62 | 63 | fileHandle = ad.file(self.filename) 64 | variable = fileHandle.var[self.variable_path] 65 | result = variable.read(tuple(begin), tuple(size)) 66 | fileHandle.close() 67 | 68 | # remove all single dimensional axes unless they are a result of slicing, i.e. a[n,n+1] 69 | single_dim_axes = [axis for axis in range(len(self.window)) if type(self.window[axis]) is int] 70 | if single_dim_axes: 71 | result = np.squeeze(result, axis=single_dim_axes) 72 | 73 | return result 74 | 75 | def __getitem__(self, args): 76 | if args == slice(None): 77 | args = (slice(None),) * len(self.shape) 78 | 79 | if not isinstance(args, list) and not isinstance(args, tuple): 80 | args = [args] 81 | 82 | assert len(args) == len(self.shape),\ 83 | "number of arguments (%d) does not correspond to the dimension (%d)"\ 84 | % (len(args), len(self.shape)) 85 | 86 | assert not all(type(arg) is int for arg in args),\ 87 | "single data access is not supported by adios" 88 | 89 | assert all(arg.step == 1 or arg.step == None for arg in args if type(arg) is slice),\ 90 | "strided access in not supported by adios" 91 | 92 | result_shape, clean_view = helper.view_of_shape(self.shape, args) 93 | 94 | # Applying 'clean_view' after 'self.window', results in 'result_window' 95 | result_window = helper.view_of_view(self.window, clean_view) 96 | 97 | return ad_ndarray(self.filename, self.variable_path, result_shape, result_window, self.offset) 98 | -------------------------------------------------------------------------------- /pyDive/arrays/local/gpu_ndarray.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2015 Heiko Burau 3 | 4 | This file is part of pyDive. 5 | 6 | pyDive is free software: you can redistribute it and/or modify 7 | it under the terms of of either the GNU General Public License or 8 | the GNU Lesser General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | pyDive is distributed in the hope that it will be useful, 12 | but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | GNU General Public License and the GNU Lesser General Public License 15 | for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | and the GNU Lesser General Public License along with pyDive. 19 | If not, see . 20 | """ 21 | __doc__ = None 22 | 23 | import pycuda.gpuarray 24 | import pycuda.driver as cuda 25 | import numpy as np 26 | 27 | def copy_non_contiguous(dst, src): 28 | """Copy ``src`` array to ``dst`` array. A gpu-array may have a non contiguous block of memory, 29 | i.e. it may have substancial pitches/strides. However a cpu-array must have a contiguous block of memory. 30 | All four directions are allowed. 31 | """ 32 | assert src.dtype == dst.dtype,\ 33 | "src ({}) and dst ({}) must have the same datatype.".format(str(src.dtype), str(dst.dtype)) 34 | assert dst.shape == src.shape,\ 35 | "Shapes do not match: " + str(dst.shape) + " <-> " + str(src.shape) 36 | 37 | itemsize = np.dtype(src.dtype).itemsize 38 | copy = cuda.Memcpy2D() 39 | src_on_gpu = isinstance(src, pycuda.gpuarray.GPUArray) 40 | dst_on_gpu = isinstance(dst, pycuda.gpuarray.GPUArray) 41 | if src_on_gpu: 42 | copy.set_src_device(src.gpudata) 43 | else: 44 | copy.set_src_host(src) 45 | if dst_on_gpu: 46 | copy.set_dst_device(dst.gpudata) 47 | else: 48 | copy.set_dst_host(dst) 49 | 50 | if len(src.shape) == 1: 51 | copy.src_pitch = src.strides[0] if src_on_gpu else itemsize 52 | copy.dst_pitch = dst.strides[0] if dst_on_gpu else itemsize 53 | copy.width_in_bytes = itemsize 54 | copy.height = src.shape[0] 55 | copy(aligned=False) 56 | 57 | elif len(src.shape) == 2: 58 | if (itemsize != src.strides[1] if src_on_gpu else False) or \ 59 | (itemsize != dst.strides[1] if dst_on_gpu else False): 60 | # arrays have to be copied column by column, because there a two substantial pitches/strides 61 | # which is not supported by cuda. 62 | copy.src_pitch = src.strides[0] if src_on_gpu else itemsize 63 | copy.dst_pitch = dst.strides[0] if dst_on_gpu else itemsize 64 | copy.width_in_bytes = itemsize 65 | copy.height = src.shape[0] 66 | 67 | for col in range(src.shape[1]): 68 | copy.src_x_in_bytes = col * src.strides[1] if src_on_gpu else col * itemsize 69 | copy.dst_x_in_bytes = col * dst.strides[1] if dst_on_gpu else col * itemsize 70 | copy(aligned=False) 71 | else: 72 | # both arrays have a contiguous block of memory for each row 73 | copy.src_pitch = src.strides[0] if src_on_gpu else itemsize * src.shape[1] 74 | copy.dst_pitch = dst.strides[0] if dst_on_gpu else itemsize * src.shape[1] 75 | copy.width_in_bytes = itemsize * src.shape[1] 76 | copy.height = src.shape[0] 77 | copy(aligned=False) 78 | 79 | elif len(src.shape) == 3: 80 | if (src.strides[0] != src.shape[1] * src.strides[1] if src_on_gpu else False) or \ 81 | (dst.strides[0] != dst.shape[1] * dst.strides[1] if dst_on_gpu else False): 82 | # arrays have to be copied plane by plane, because there a substantial pitche/stride 83 | # for the z-axis which is not supported by cuda. 84 | for plane in range(src.shape[0]): 85 | copy_non_contiguous(dst[plane,:,:], src[plane,:,:]) 86 | return 87 | 88 | copy = cuda.Memcpy3D() 89 | if src_on_gpu: 90 | copy.set_src_device(src.gpudata) 91 | else: 92 | copy.set_src_host(src) 93 | if dst_on_gpu: 94 | copy.set_dst_device(dst.gpudata) 95 | else: 96 | copy.set_dst_host(dst) 97 | 98 | copy.src_pitch = src.strides[1] if src_on_gpu else itemsize * src.shape[2] 99 | copy.dst_pitch = dst.strides[1] if dst_on_gpu else itemsize * src.shape[2] 100 | copy.width_in_bytes = itemsize * src.shape[2] 101 | copy.height = copy.src_height = copy.dst_height = src.shape[1] 102 | copy.depth = src.shape[0] 103 | 104 | copy() 105 | else: 106 | raise RuntimeError("dimension %d is not supported." % len(src.shape)) 107 | 108 | 109 | class gpu_ndarray(pycuda.gpuarray.GPUArray): 110 | 111 | def __init__(self, shape, dtype, allocator=cuda.mem_alloc,\ 112 | base=None, gpudata=None, strides=None, order="C"): 113 | if type(shape) not in (list, tuple): 114 | shape = (shape,) 115 | elif type(shape) is not tuple: 116 | shape = tuple(shape) 117 | super(gpu_ndarray, self).__init__(shape, np.dtype(dtype), allocator, base, gpudata, strides, order) 118 | 119 | def __setitem__(self, key, other): 120 | # if args is [:] then assign `other` to the entire ndarray 121 | if key == slice(None): 122 | if isinstance(other, pycuda.gpuarray.GPUArray): 123 | if self.flags.forc and other.flags.forc: 124 | # both arrays are a contiguous block of memory 125 | cuda.memcpy_dtod(self.gpudata, other.gpudata, self.nbytes) 126 | return 127 | else: 128 | if self.flags.forc: 129 | # both arrays are a contiguous block of memory 130 | cuda.memcpy_htod(self.gpudata, other) 131 | return 132 | 133 | copy_non_contiguous(self, other) 134 | return 135 | 136 | # assign `other` to sub-array of self 137 | sub_array = self[key] 138 | sub_array[:] = other 139 | 140 | def __cast_from_base(self, pycuda_array): 141 | return gpu_ndarray(pycuda_array.shape, pycuda_array.dtype, pycuda_array.allocator,\ 142 | pycuda_array.base, pycuda_array.gpudata, pycuda_array.strides) 143 | 144 | def __getitem__(self, args): 145 | pycuda_array = super(gpu_ndarray, self).__getitem__(tuple(args)) 146 | return self.__cast_from_base(pycuda_array) 147 | 148 | def copy(self): 149 | if self.flags.forc: 150 | return super(gpu_ndarray, self).copy() 151 | result = gpu_ndarray(shape=self.shape, dtype=self.dtype, allocator=self.allocator) 152 | result[:] = self 153 | return result 154 | 155 | def to_cpu(self): 156 | if self.flags.forc: 157 | return self.get(pagelocked=True) 158 | 159 | result = cuda.pagelocked_empty(self.shape, self.dtype) 160 | copy_non_contiguous(result, self) 161 | return result 162 | 163 | def __elementwise_op__(self, op, *args): 164 | # if arrays are not contiguous make a contiguous copy 165 | my_array = self.copy() if not self.flags.forc else self 166 | 167 | args = [arg.copy() if isinstance(arg, pycuda.gpuarray.GPUArray) and not arg.flags.forc else arg for arg in args] 168 | pycuda_array = getattr(super(gpu_ndarray, my_array), op)(*args) 169 | return self.__cast_from_base(pycuda_array) 170 | 171 | # add special operations like __add__, __mul__, etc. to `gpu_ndarray` 172 | 173 | binary_ops = ["add", "sub", "mul", "floordiv", "div", "mod", "pow", "lshift", "rshift", "and", "xor", "or"] 174 | 175 | binary_iops = ["__i" + op + "__" for op in binary_ops] 176 | binary_rops = ["__r" + op + "__" for op in binary_ops] 177 | binary_ops = ["__" + op + "__" for op in binary_ops] 178 | unary_ops = ["__neg__", "__pos__", "__abs__", "__invert__", "__complex__", "__int__", "__long__", "__float__", "__oct__", "__hex__"] 179 | comp_ops = ["__lt__", "__le__", "__eq__", "__ne__", "__ge__", "__gt__"] 180 | 181 | special_ops_avail = set(name for name in pycuda.gpuarray.GPUArray.__dict__.keys() if name.endswith("__")) 182 | 183 | make_special_op = lambda op: lambda self, *args: self.__elementwise_op__(op, *args) 184 | 185 | special_ops_dict = {op : make_special_op(op) for op in \ 186 | set(binary_ops + binary_rops + binary_iops + unary_ops + comp_ops) & special_ops_avail} 187 | 188 | from types import MethodType 189 | 190 | for name, func in special_ops_dict.items(): 191 | setattr(gpu_ndarray, name, MethodType(func, None, gpu_ndarray)) 192 | 193 | # -------------------- factories ----------------------------------- 194 | 195 | # get a `gpu_ndarray` instance out of a `pycuda.gpuarray.GPUArray` instance 196 | def gpu_ndarray_cast(pycuda_array): 197 | return gpu_ndarray(pycuda_array.shape, pycuda_array.dtype, pycuda_array.allocator,\ 198 | pycuda_array.base, pycuda_array.gpudata, pycuda_array.strides) 199 | 200 | def empty(shape, dtype): 201 | return gpu_ndarray(shape, dtype) 202 | 203 | pycuda_factories = ("zeros", "empty_like", "zeros_like") 204 | 205 | # wrap all factory functions from pycuda so that they return a `gpu_ndarray` instance 206 | from functools import wraps 207 | make_factory = lambda func : wraps(func)(lambda *args, **kwargs: gpu_ndarray_cast(func(*args, **kwargs))) 208 | factories = {func : make_factory(getattr(pycuda.gpuarray, func)) for func in pycuda_factories} 209 | 210 | globals().update(factories) 211 | -------------------------------------------------------------------------------- /pyDive/arrays/local/h5_ndarray.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2015 Heiko Burau 3 | 4 | This file is part of pyDive. 5 | 6 | pyDive is free software: you can redistribute it and/or modify 7 | it under the terms of of either the GNU General Public License or 8 | the GNU Lesser General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | pyDive is distributed in the hope that it will be useful, 12 | but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | GNU General Public License and the GNU Lesser General Public License 15 | for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | and the GNU Lesser General Public License along with pyDive. 19 | If not, see . 20 | """ 21 | __doc__ = None 22 | 23 | import h5py as h5 24 | import pyDive.distribution.helper as helper 25 | import numpy as np 26 | 27 | class h5_ndarray(object): 28 | 29 | def __init__(self, filename, dataset_path, shape=None, window=None, offset=None): 30 | self.filename = filename 31 | self.dataset_path = dataset_path 32 | 33 | fileHandle = h5.File(filename, "r") 34 | dataset = fileHandle[dataset_path] 35 | #self.attrs = dataset.attrs 36 | #: datatype of a single data value 37 | self.dtype = dataset.dtype 38 | if shape is None: 39 | shape = dataset.shape 40 | self.shape = tuple(shape) 41 | fileHandle.close() 42 | 43 | if window is None: 44 | window = [slice(0, s, 1) for s in shape] 45 | self.window = tuple(window) 46 | if offset is None: 47 | offset = (0,) * len(shape) 48 | self.offset = tuple(offset) 49 | 50 | #: total bytes consumed by the elements of the array. 51 | self.nbytes = self.dtype.itemsize * np.prod(self.shape) 52 | 53 | def load(self): 54 | window = list(self.window) 55 | for i in range(len(window)): 56 | if type(window[i]) is int: 57 | window[i] += self.offset[i] 58 | else: 59 | window[i] = slice(window[i].start + self.offset[i], window[i].stop + self.offset[i], window[i].step) 60 | 61 | fileHandle = h5.File(self.filename, "r") 62 | dataset = fileHandle[self.dataset_path] 63 | result = dataset[tuple(window)] 64 | fileHandle.close() 65 | return result 66 | 67 | def __getitem__(self, args): 68 | if args == slice(None): 69 | args = (slice(None),) * len(self.shape) 70 | 71 | if not isinstance(args, list) and not isinstance(args, tuple): 72 | args = [args] 73 | 74 | assert len(args) == len(self.shape),\ 75 | "number of arguments (%d) does not correspond to the dimension (%d)"\ 76 | % (len(args), len(self.shape)) 77 | 78 | assert not all(type(arg) is int for arg in args),\ 79 | "single data access is not supported" 80 | 81 | result_shape, clean_view = helper.view_of_shape(self.shape, args) 82 | 83 | # Applying 'clean_view' after 'self.window', results in 'result_window' 84 | result_window = helper.view_of_view(self.window, clean_view) 85 | 86 | return h5_ndarray(self.filename, self.dataset_path, result_shape, result_window, self.offset) 87 | -------------------------------------------------------------------------------- /pyDive/arrays/ndarray.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2014 Heiko Burau 3 | 4 | This file is part of pyDive. 5 | 6 | pyDive is free software: you can redistribute it and/or modify 7 | it under the terms of of either the GNU General Public License or 8 | the GNU Lesser General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | pyDive is distributed in the hope that it will be useful, 12 | but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | GNU General Public License and the GNU Lesser General Public License 15 | for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | and the GNU Lesser General Public License along with pyDive. 19 | If not, see . 20 | """ 21 | __doc__ = None 22 | 23 | import numpy as np 24 | import pyDive.distribution.multiple_axes as multiple_axes 25 | from pyDive.distribution.interengine import MPI_copier 26 | 27 | ndarray = multiple_axes.distribute(np.ndarray, "ndarray", "np", interengine_copier=MPI_copier) 28 | 29 | factories = multiple_axes.generate_factories(ndarray, ("empty", "zeros", "ones"), np.float) 30 | factories.update(multiple_axes.generate_factories_like(ndarray, ("empty_like", "zeros_like", "ones_like"))) 31 | 32 | globals().update(factories) 33 | 34 | def array(array_like, distaxes='all'): 35 | """Create a pyDive.ndarray instance from an array-like object. 36 | 37 | :param array_like: Any object exposing the array interface, e.g. numpy-array, python sequence, ... 38 | :param ints distaxes: distributed axes. Defaults to 'all' meaning each axis is distributed. 39 | """ 40 | np_array = np.array(array_like) 41 | result = empty(np_array.shape, np_array.dtype, distaxes) 42 | result[:] = np_array 43 | return result 44 | 45 | def hollow(shape, dtype=np.float, distaxes='all'): 46 | """Create a pyDive.ndarray instance distributed across all engines without allocating a local 47 | numpy-array. 48 | 49 | :param ints shape: shape of array 50 | :param dtype: datatype of a single element 51 | :param ints distaxes: distributed axes. Defaults to 'all' meaning each axis is distributed. 52 | """ 53 | return ndarray(shape, dtype, distaxes, None, None, True) 54 | 55 | def hollow_like(other): 56 | """Create a pyDive.ndarray instance with the same 57 | shape, distribution and type as ``other`` without allocating a local numpy-array. 58 | """ 59 | return ndarray(other.shape, other.dtype, other.distaxes, other.target_offsets, other.target_ranks, True) 60 | 61 | factories.update({"array" : array, "hollow" : hollow, "hollow_like" : hollow_like}) 62 | 63 | ufunc_names = [key for key, value in np.__dict__.items() if isinstance(value, np.ufunc)] 64 | ufuncs = multiple_axes.generate_ufuncs(ufunc_names, "np") 65 | 66 | globals().update(ufuncs) -------------------------------------------------------------------------------- /pyDive/cloned_ndarray/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2014 Heiko Burau 3 | 4 | This file is part of pyDive. 5 | 6 | pyDive is free software: you can redistribute it and/or modify 7 | it under the terms of of either the GNU General Public License or 8 | the GNU Lesser General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | pyDive is distributed in the hope that it will be useful, 12 | but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | GNU General Public License and the GNU Lesser General Public License 15 | for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | and the GNU Lesser General Public License along with pyDive. 19 | If not, see . 20 | """ 21 | __doc__ = None 22 | 23 | # -*- coding: utf-8 -*- 24 | __all__ = ["cloned_ndarray", "factories"] -------------------------------------------------------------------------------- /pyDive/cloned_ndarray/cloned_ndarray.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2014 Heiko Burau 3 | 4 | This file is part of pyDive. 5 | 6 | pyDive is free software: you can redistribute it and/or modify 7 | it under the terms of of either the GNU General Public License or 8 | the GNU Lesser General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | pyDive is distributed in the hope that it will be useful, 12 | but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | GNU General Public License and the GNU Lesser General Public License 15 | for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | and the GNU Lesser General Public License along with pyDive. 19 | If not, see . 20 | """ 21 | __doc__ = None 22 | 23 | from .. import IPParallelClient as com 24 | import numpy as np 25 | 26 | cloned_ndarray_id = 0 27 | 28 | class cloned_ndarray(object): 29 | """Represents a multidimensional, homogenous array of fixed-size elements which is cloned 30 | on the cluster nodes. *Cloned* means that every participating :term:`engine` holds an independent, local numpy-array 31 | of the user-defined shape. The user can then do e.g. some manual stuff on the local arrays or 32 | some computation with :mod:`pyDive.algorithm` on them. 33 | 34 | Note that there exists no 'original' array as the name might suggest but something like that can be 35 | generated by :meth:`merge`. 36 | """ 37 | def __init__(self, shape, dtype=np.float, target_ranks='all', no_allocation=False): 38 | """Creates an :class:`pyDive.cloned_ndarray.cloned_ndarray.cloned_ndarray` instance. 39 | This is a low-level method for instanciating a cloned_array. 40 | Cloned arrays should be constructed using 'empty', 'zeros' 41 | or 'empty_targets_like' (see :mod:`pyDive.cloned_ndarray.factories`). 42 | 43 | :param ints shape: size of the array on each axis 44 | :param numpy-dtype dtype: datatype of a single data value 45 | :param ints target_ranks: list of :term:`engine`-ids that share this array.\ 46 | Or 'all' for all engines. 47 | :param bool no_allocation: if ``True`` no actual memory, i.e. *numpy-array*, will be 48 | allocated on :term:`engine`. Useful when you want to assign an existing numpy array manually. 49 | """ 50 | self.shape = list(shape) 51 | self.dtype = dtype 52 | self.nbytes = np.dtype(dtype).itemsize * np.prod(self.shape) 53 | self.target_ranks = target_ranks 54 | self.view = com.getView() 55 | 56 | if self.target_ranks == 'all': 57 | self.target_ranks = list(self.view.targets) 58 | 59 | # generate a unique variable name used on target representing this instance 60 | global cloned_ndarray_id 61 | self.name = 'cloned_ndarray' + str(cloned_ndarray_id) 62 | cloned_ndarray_id += 1 63 | 64 | if no_allocation: 65 | self.view.push({self.name : None}, targets=self.target_ranks) 66 | else: 67 | self.view.push({'myshape' : self.shape, 'dtype' : self.dtype}, targets=self.target_ranks) 68 | self.view.execute('%s = np.empty(myshape, dtype=dtype)' % self.name, targets=self.target_ranks) 69 | 70 | def __del__(self): 71 | self.view.execute('del %s' % self.name, targets=self.target_ranks) 72 | 73 | def __repr__(self): 74 | return self.name 75 | 76 | def __setitem__(self, key, value): 77 | # if args is [:] then assign value to the entire ndarray 78 | if key == slice(None): 79 | assert isinstance(value, np.ndarray), "assignment available for numpy-arrays only" 80 | 81 | view = com.getView() 82 | view.push({'np_array' : value}, targets=self.target_ranks) 83 | view.execute("%s = np_array.copy()" % self.name, targets=self.target_ranks) 84 | 85 | return 86 | 87 | if not isinstance(key, list) and not isinstance(key, tuple): 88 | key = (key,) 89 | 90 | assert len(key) == len(self.shape) 91 | 92 | # assign value to sub-array of self 93 | sub_array = self[key] 94 | sub_array[:] = value 95 | 96 | def __getitem__(self, args): 97 | if not isinstance(args, list) and not isinstance(args, tuple): 98 | args = (args,) 99 | 100 | assert len(args) == len(self.shape),\ 101 | "number of arguments (%d) does not correspond to the dimension (%d)"\ 102 | % (len(args), len(self.shape)) 103 | 104 | # shape of the new sliced ndarray 105 | new_shape, clean_view = helper.view_of_shape(self.shape, args) 106 | 107 | result = pyDive.cloned.hollow_engines_like(new_shape, self.dtype, self) 108 | 109 | self.view.push({'args' : args}, targets=self.target_ranks) 110 | self.view.execute('%s = %s[args]' % (result.name, self.name), targets=self.target_ranks) 111 | 112 | return result 113 | 114 | def merge(self, op): 115 | """Merge all local arrays in a pair-wise operation into a single numpy-array. 116 | 117 | :param op: Merging operation. Expects two numpy-arrays and returns one. 118 | :return: merged numpy-array. 119 | """ 120 | result = self.view.pull(self.name, targets=self.target_ranks[0]) 121 | for target in self.target_ranks[1:]: 122 | result = op(result, self.view.pull(self.name, targets=target)) 123 | return result 124 | 125 | def sum(self): 126 | """Add up all local arrays. 127 | 128 | :return: numpy-array. 129 | """ 130 | return self.merge(lambda x, y: x+y) 131 | -------------------------------------------------------------------------------- /pyDive/cloned_ndarray/factories.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2014 Heiko Burau 3 | 4 | This file is part of pyDive. 5 | 6 | pyDive is free software: you can redistribute it and/or modify 7 | it under the terms of of either the GNU General Public License or 8 | the GNU Lesser General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | pyDive is distributed in the hope that it will be useful, 12 | but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | GNU General Public License and the GNU Lesser General Public License 15 | for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | and the GNU Lesser General Public License along with pyDive. 19 | If not, see . 20 | """ 21 | 22 | __doc__=\ 23 | """This module holds high-level functions for instanciating :ref:`pyDive.cloned_ndarrays `.""" 24 | 25 | import pyDive.cloned_ndarray.cloned_ndarray as cloned_ndarray 26 | from .. import IPParallelClient as com 27 | import numpy as np 28 | 29 | def hollow_engines_like(shape, dtype, a): 30 | """Return a new :obj:`pyDive.cloned_ndarray` utilizing the same engines *a* does 31 | without allocating a local *numpy-array*. 32 | 33 | :param ints shape: shape of the array 34 | :param numpy-dtype dtype: datatype of a single data value 35 | :param a: :ref:`pyDive.ndarray` 36 | """ 37 | return cloned_ndarray.cloned_ndarray(shape, dtype, a.target_ranks, True) 38 | 39 | def empty_engines_like(shape, dtype, a): 40 | """Return a new :obj:`pyDive.cloned_ndarray` utilizing the same engines *a* does 41 | without initializing elements. 42 | 43 | :param ints shape: shape of the array 44 | :param numpy-dtype dtype: datatype of a single data value 45 | :param a: :ref:`pyDive.ndarray` 46 | """ 47 | return cloned_ndarray.cloned_ndarray(shape, dtype, a.target_ranks) 48 | 49 | def zeros_engines_like(shape, dtype, a): 50 | """Return a new :ref:`pyDive.cloned_ndarray` utilizing the same engines *a* does 51 | filled with zeros. 52 | 53 | :param ints shape: shape of the array 54 | :param numpy-dtype dtype: datatype of a single data value 55 | :param a: :ref:`pyDive.ndarray` 56 | """ 57 | result = cloned_ndarray.cloned_ndarray(shape, dtype, a.target_ranks, True) 58 | view = com.getView() 59 | view.push({'myshape' : shape, 'dtype' : dtype}, targets=result.target_ranks) 60 | view.execute('%s = np.zeros(myshape, dtype)' % repr(result), targets=result.target_ranks) 61 | return result 62 | 63 | def hollow(shape, dtype=np.float): 64 | """Return a new :ref:`pyDive.cloned_ndarray` utilizing all engines without allocating a local 65 | *numpy-array*. 66 | 67 | :param ints shape: shape of the array 68 | :param numpy-dtype dtype: datatype of a single data value 69 | """ 70 | result = cloned_ndarray.cloned_ndarray(shape, dtype, target_ranks='all', no_allocation=True) 71 | return result 72 | 73 | def empty(shape, dtype=np.float): 74 | """Return a new :ref:`pyDive.cloned_ndarray` utilizing all engines without initializing elements. 75 | 76 | :param ints shape: shape of the array 77 | :param numpy-dtype dtype: datatype of a single data value 78 | """ 79 | result = cloned_ndarray.cloned_ndarray(shape, dtype, target_ranks='all') 80 | return result 81 | 82 | def zeros(shape, dtype=np.float): 83 | """Return a new :ref:`pyDive.cloned_ndarray` utilizing all engines filled with zeros. 84 | 85 | :param ints shape: shape of the array 86 | :param numpy-dtype dtype: datatype of a single data value 87 | """ 88 | result = cloned_ndarray.cloned_ndarray(shape, dtype, target_ranks='all', no_allocation=True) 89 | view = com.getView() 90 | view.push({'myshape' : shape, 'dtype' : dtype}, targets=result.target_ranks) 91 | view.execute('%s = np.zeros(myshape, dtype)' % repr(result), targets=result.target_ranks) 92 | return result 93 | 94 | def ones(shape, dtype=np.float): 95 | """Return a new :ref:`pyDive.cloned_ndarray` utilizing all engines filled with ones. 96 | 97 | :param ints shape: shape of the array 98 | :param numpy-dtype dtype: datatype of a single data value 99 | """ 100 | result = cloned_ndarray.cloned_ndarray(shape, dtype, target_ranks='all', no_allocation=True) 101 | view = com.getView() 102 | view.push({'myshape' : shape, 'dtype' : dtype}, targets=result.target_ranks) 103 | view.execute('%s = np.ones(myshape, dtype)' % repr(result), targets=result.target_ranks) 104 | return result -------------------------------------------------------------------------------- /pyDive/distribution/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | -------------------------------------------------------------------------------- /pyDive/distribution/helper.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2014 Heiko Burau 3 | 4 | This file is part of pyDive. 5 | 6 | pyDive is free software: you can redistribute it and/or modify 7 | it under the terms of of either the GNU General Public License or 8 | the GNU Lesser General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | pyDive is distributed in the hope that it will be useful, 12 | but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | GNU General Public License and the GNU Lesser General Public License 15 | for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | and the GNU Lesser General Public License along with pyDive. 19 | If not, see . 20 | """ 21 | __doc__ = None 22 | 23 | import numpy as np 24 | from collections import OrderedDict 25 | 26 | def getFirstSliceIdx(slice_obj, begin, end): 27 | if slice_obj.start > begin: 28 | if slice_obj.start >= end: return None 29 | return slice_obj.start 30 | i = (begin-1 - slice_obj.start) / slice_obj.step + 1 31 | idx = slice_obj.start + i * slice_obj.step 32 | if idx >= end or idx >= slice_obj.stop: return None 33 | return idx 34 | 35 | def view_of_shape(shape, window): 36 | new_shape = [] 37 | clean_view = [] 38 | for s, w in zip(shape, window): 39 | if type(w) is int: 40 | clean_view.append(w) 41 | continue 42 | # create a clean, wrapped slice object 43 | clean_slice = slice(*w.indices(s)) 44 | clean_view.append(clean_slice) 45 | # new size of axis i 46 | new_shape.append((clean_slice.stop-1 - clean_slice.start) / clean_slice.step + 1) 47 | return new_shape, clean_view 48 | 49 | def view_of_view(view, window): 50 | result_view = [] 51 | window = iter(window) 52 | for v in view: 53 | if type(v) is int: 54 | result_view.append(v) 55 | continue 56 | w = window.next() 57 | if type(w) is int: 58 | result_view.append(v.start + w * v.step) 59 | continue 60 | 61 | result_view.append(slice(v.start + w.start * v.step, v.start + w.stop * v.step, v.step * w.step)) 62 | 63 | return result_view 64 | 65 | # create local slice objects for each engine 66 | def createLocalSlices(slices, distaxis, target_offsets, shape): 67 | local_slices = [list(slices) for i in range(len(target_offsets))] 68 | distaxis_slice = slices[distaxis] 69 | for i in range(len(target_offsets)): 70 | begin = target_offsets[i] 71 | end = target_offsets[i+1] if i+1 < len(target_offsets) else shape[distaxis] 72 | 73 | local_slices[i][distaxis] = slice(distaxis_slice.start + distaxis_slice.step * begin,\ 74 | distaxis_slice.start + distaxis_slice.step * end,\ 75 | distaxis_slice.step) 76 | 77 | return local_slices 78 | 79 | def common_decomposition(axesA, offsetsA, axesB, offsetsB, shape): 80 | axes = list(OrderedDict.fromkeys(axesA + axesB)) # remove double axes while preserving order 81 | offsets = [] 82 | ids = [] 83 | for axis in axes: 84 | if not (axis in axesA and axis in axesB): 85 | offsets.append(offsetsA[axesA.index(axis)] if axis in axesA else offsetsB[axesB.index(axis)]) 86 | ids.append([]) 87 | continue 88 | 89 | offsetsA_sa = offsetsA[axesA.index(axis)] 90 | offsetsB_sa = offsetsB[axesB.index(axis)] 91 | 92 | offsets_sa = [] # sa = single axis 93 | ids_sa = [] 94 | A_idx = 0 # partition index 95 | B_idx = 0 96 | begin = 0 97 | # loop the common decomposition of A and B. 98 | # the current partition is given by [begin, end) 99 | while not begin == shape[axis]: 100 | end_A = offsetsA_sa[A_idx+1] if A_idx < len(offsetsA_sa)-1 else shape[axis] 101 | end_B = offsetsB_sa[B_idx+1] if B_idx < len(offsetsB_sa)-1 else shape[axis] 102 | end = min(end_A, end_B) 103 | 104 | offsets_sa.append(begin) 105 | ids_sa.append((A_idx, B_idx)) 106 | 107 | # go to next common partition 108 | if end == end_A: 109 | A_idx += 1 110 | if end == end_B: 111 | B_idx += 1 112 | 113 | begin = end 114 | 115 | offsets.append(np.array(offsets_sa)) 116 | ids.append(np.array(ids_sa)) 117 | return axes, offsets, ids -------------------------------------------------------------------------------- /pyDive/distribution/interengine.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2015 Heiko Burau 3 | 4 | This file is part of pyDive. 5 | 6 | pyDive is free software: you can redistribute it and/or modify 7 | it under the terms of of either the GNU General Public License or 8 | the GNU Lesser General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | pyDive is distributed in the hope that it will be useful, 12 | but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | GNU General Public License and the GNU Lesser General Public License 15 | for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | and the GNU Lesser General Public License along with pyDive. 19 | If not, see . 20 | """ 21 | __doc__ = None 22 | import numpy as np 23 | from mpi4py import MPI 24 | try: 25 | import pycuda.gpuarray 26 | except ImportError: 27 | pass 28 | 29 | #----------- cpu -> cpu ---------------- 30 | 31 | def scatterArrayMPI_async(in_array, commData, target2rank): 32 | tasks = [] 33 | for (dest_target, window, tag) in commData: 34 | tasks.append(MPI.COMM_WORLD.Isend(in_array[window].copy(), dest=target2rank[dest_target], tag=tag)) 35 | 36 | return tasks 37 | 38 | def gatherArraysMPI_async(out_array, commData, target2rank): 39 | recv_bufs = [] 40 | tasks = [] 41 | for (src_target, window, tag) in commData: 42 | recv_bufs.append(np.empty_like(out_array[window])) 43 | tasks.append(MPI.COMM_WORLD.Irecv(recv_bufs[-1], source=target2rank[src_target], tag=tag)) 44 | 45 | return tasks, recv_bufs 46 | 47 | def finish_MPIcommunication(out_array, commData, recv_bufs): 48 | for (src_target, window, tag), recv_buf in zip(commData, recv_bufs): 49 | out_array[window] = recv_buf 50 | 51 | #----------- gpu -> cpu -> cpu -> gpu ---------------- 52 | 53 | def scatterArrayGPU_async(in_array, commData, target2rank): 54 | tasks = [] 55 | for (dest_target, window, tag) in commData: 56 | tasks.append(MPI.COMM_WORLD.Isend(in_array[window].to_cpu(), dest=target2rank[dest_target], tag=tag)) 57 | 58 | return tasks 59 | 60 | def gatherArraysGPU_async(out_array, commData, target2rank): 61 | recv_bufs = [] 62 | tasks = [] 63 | for (src_target, window, tag) in commData: 64 | chunk = out_array[window] 65 | recv_bufs.append(np.empty(shape=chunk.shape, dtype=chunk.dtype)) 66 | tasks.append(MPI.COMM_WORLD.Irecv(recv_bufs[-1], source=target2rank[src_target], tag=tag)) 67 | 68 | return tasks, recv_bufs 69 | 70 | def finish_GPUcommunication(out_array, commData, recv_bufs): 71 | for (src_target, window, tag), recv_buf in zip(commData, recv_bufs): 72 | out_array[window] = pycuda.gpuarray.to_gpu(recv_buf) 73 | 74 | import os 75 | onTarget = os.environ.get("onTarget", 'False') 76 | 77 | # execute this code only if it is not executed on engine 78 | if onTarget == 'False': 79 | import pyDive.IPParallelClient as com 80 | 81 | def MPI_copier(source, dest): 82 | view = com.getView() 83 | 84 | # send 85 | view.execute('{0}_send_tasks = interengine.scatterArrayMPI_async({0}, src_commData[0], target2rank)'\ 86 | .format(source.name), targets=source.target_ranks) 87 | 88 | # receive 89 | view.execute("""\ 90 | {0}_recv_tasks, {0}_recv_bufs = interengine.gatherArraysMPI_async({1}, dest_commData[0], target2rank) 91 | """.format(source.name, dest.name),\ 92 | targets=dest.target_ranks) 93 | 94 | # finish communication 95 | view.execute('''\ 96 | if "{0}_send_tasks" in locals(): 97 | MPI.Request.Waitall({0}_send_tasks) 98 | del {0}_send_tasks 99 | if "{0}_recv_tasks" in locals(): 100 | MPI.Request.Waitall({0}_recv_tasks) 101 | interengine.finish_MPIcommunication({1}, dest_commData[0], {0}_recv_bufs) 102 | del {0}_recv_tasks, {0}_recv_bufs 103 | '''.format(source.name, dest.name), 104 | targets=tuple(set(source.target_ranks + dest.target_ranks))) 105 | 106 | def GPU_copier(source, dest): 107 | view = com.getView() 108 | 109 | # send 110 | view.execute('{0}_send_tasks = interengine.scatterArrayGPU_async({0}, src_commData[0], target2rank)'\ 111 | .format(source.name), targets=source.target_ranks) 112 | 113 | # receive 114 | view.execute("""\ 115 | {0}_recv_tasks, {0}_recv_bufs = interengine.gatherArraysGPU_async({1}, dest_commData[0], target2rank) 116 | """.format(source.name, dest.name),\ 117 | targets=dest.target_ranks) 118 | 119 | # finish communication 120 | view.execute('''\ 121 | if "{0}_send_tasks" in locals(): 122 | MPI.Request.Waitall({0}_send_tasks) 123 | del {0}_send_tasks 124 | if "{0}_recv_tasks" in locals(): 125 | MPI.Request.Waitall({0}_recv_tasks) 126 | interengine.finish_GPUcommunication({1}, dest_commData[0], {0}_recv_bufs) 127 | del {0}_recv_tasks, {0}_recv_bufs 128 | '''.format(source.name, dest.name), 129 | targets=tuple(set(source.target_ranks + dest.target_ranks))) 130 | -------------------------------------------------------------------------------- /pyDive/distribution/multiple_axes.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Copyright 2015 Heiko Burau 4 | 5 | This file is part of pyDive. 6 | 7 | pyDive is free software: you can redistribute it and/or modify 8 | it under the terms of of either the GNU General Public License or 9 | the GNU Lesser General Public License as published by 10 | the Free Software Foundation, either version 3 of the License, or 11 | (at your option) any later version. 12 | pyDive is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | GNU General Public License and the GNU Lesser General Public License 16 | for more details. 17 | 18 | You should have received a copy of the GNU General Public License 19 | and the GNU Lesser General Public License along with pyDive. 20 | If not, see . 21 | """ 22 | __doc__ = None 23 | 24 | import numpy as np 25 | import pyDive.IPParallelClient as com 26 | import helper 27 | from collections import defaultdict 28 | 29 | array_id = 0 30 | 31 | class DistributedGenericArray(object): 32 | """ 33 | Represents a cluster-wide, multidimensional, homogeneous array of fixed-size elements. 34 | *cluster-wide* means that its elements are distributed across IPython.parallel-engines. 35 | The distribution is done in one or multiply dimensions along user-specified axes. 36 | The user can optionally specify which engine maps to which index range or leave the default 37 | that persuits an uniform distribution across all engines. 38 | 39 | This **{arraytype_name}** - class is auto-generated out of its local counterpart: **{local_arraytype_name}**. 40 | 41 | The implementation is based on IPython.parallel and local {local_arraytype_name} - arrays. 42 | Every special operation {local_arraytype_name} implements ("__add__", "__le__", ...) is also 43 | available for {arraytype_name}. 44 | 45 | Note that array slicing is a cheap operation since no memory is copied. 46 | However this can easily lead to the situation where you end up with two arrays of the same size but of distinct element distribution. 47 | Therefore call dist_like() first before doing any manual stuff on their local arrays. 48 | However every cluster-wide array operation first equalizes the distribution of all involved arrays, 49 | so an explicit call to dist_like() is rather unlikely in most use cases. 50 | 51 | If you try to access an attribute that is only available for the local array, the request 52 | is forwarded to an internal local copy of the whole distributed array (see: :meth:`gather()`). 53 | This internal copy is only created when you want to access it and is held until ``__setitem__`` is called, 54 | i.e. the array's content is manipulated. 55 | """ 56 | local_arraytype = None 57 | target_modulename = None 58 | interengine_copier = None 59 | may_allocate = True 60 | 61 | def __init__(self, shape, dtype=np.float, distaxes='all', target_offsets=None, target_ranks=None, no_allocation=False, **kwargs): 62 | """Creates an instance of {arraytype_name}. This is a low-level method of instantiating an array, it should rather be 63 | constructed using factory functions ("empty", "zeros", "open", ...) 64 | 65 | :param ints shape: shape of array 66 | :param dtype: datatype of a single element 67 | :param ints distaxes: distributed axes. Accepts a single integer too. Defaults to 'all' meaning each axis is distributed. 68 | :param target_offsets: For each distributed axis there is a (inner) list in the outer list. 69 | The inner list contains the offsets of the local array. 70 | :type target_offsets: list of lists 71 | :param ints target_ranks: linear list of :term:`engine` ranks holding the local arrays. 72 | The last distributed axis is iterated over first. 73 | :param bool no_allocation: if ``True`` no instance of {local_arraytype_name} will be created on engine. Useful for 74 | manual instantiation of the local array. 75 | :param kwargs: additional keyword arguments are forwarded to the constructor of the local array. 76 | """ 77 | #: size of the array on each axis 78 | if type(shape) not in (list, tuple): 79 | shape = (shape,) 80 | elif type(shape) is not tuple: 81 | shape = tuple(shape) 82 | self.shape = shape 83 | ##: datatype of a single data value 84 | self.dtype = dtype 85 | if distaxes == 'all': 86 | distaxes = tuple(range(len(shape))) 87 | elif type(distaxes) not in (list, tuple): 88 | distaxes = (distaxes,) 89 | elif type(distaxes) is not tuple: 90 | distaxes = tuple(distaxes) 91 | #: axes on which memory is distributed across :term:`engines ` 92 | self.distaxes = distaxes 93 | #: total bytes consumed by elements of this array. 94 | self.nbytes = np.dtype(dtype).itemsize * np.prod(self.shape) 95 | self.view = com.getView() 96 | self.kwargs = kwargs 97 | self.local_copy_is_dirty = False 98 | 99 | assert len(distaxes) <= len(shape),\ 100 | "more distributed axes ({}) than dimensions ({})".format(len(distaxes), len(shape)) 101 | for distaxis in distaxes: 102 | assert distaxis >= 0 and distaxis < len(self.shape),\ 103 | "distributed axis ({}) has to be within [0,{}]".format(distaxis, len(self.shape)-1) 104 | 105 | if target_offsets is None and target_ranks is None: 106 | # create hypothetical patch with best surface-to-volume ratio 107 | patch_volume = np.prod([self.shape[distaxis] for distaxis in distaxes]) / float(len(self.view.targets)) 108 | patch_edge_length = pow(patch_volume, 1.0/len(distaxes)) 109 | 110 | def factorize(n): 111 | if n == 1: yield 1; return 112 | for f in range(2,n//2+1) + [n]: 113 | while n%f == 0: 114 | n //= f 115 | yield f 116 | prime_factors = list(factorize(len(self.view.targets)))[::-1] # get prime factors of number of engines in descending order 117 | sorted_distaxes = sorted(distaxes, key=lambda axis: self.shape[axis]) # sort distributed axes in ascending order 118 | # calculate number of available targets (engines) per distributed axis 119 | # This value should be close to array_edge_length / patch_edge_length 120 | num_targets_av = [1] * len(self.shape) 121 | 122 | for distaxis in sorted_distaxes[:-1]: 123 | num_patches = self.shape[distaxis] / patch_edge_length 124 | while float(num_targets_av[distaxis]) < num_patches and prime_factors: 125 | num_targets_av[distaxis] *= prime_factors.pop() 126 | # the largest axis gets the remaining (largest) prime_factors 127 | if prime_factors: 128 | num_targets_av[sorted_distaxes[-1]] *= np.prod(prime_factors) 129 | 130 | # calculate target_offsets 131 | localshape = np.array(self.shape) 132 | for distaxis in distaxes: 133 | localshape[distaxis] = (self.shape[distaxis] - 1) / num_targets_av[distaxis] + 1 134 | 135 | # number of occupied targets for each distributed axis by this ndarray instance 136 | num_targets = [(self.shape[distaxis] - 1) / localshape[distaxis] + 1 for distaxis in distaxes] 137 | 138 | # calculate target_offsets 139 | target_offsets = [np.arange(num_targets[i]) * localshape[distaxes[i]] for i in range(len(distaxes))] 140 | 141 | # generate target_ranks list 142 | target_ranks = tuple(range(np.prod(num_targets))) 143 | 144 | if target_offsets is None: 145 | localshape = np.array(self.shape) 146 | for distaxis in distaxes: 147 | localshape[distaxis] = (self.shape[distaxis] - 1) / len(target_ranks[distaxis]) + 1 148 | target_offsets = [np.arange(num_targets[i]) * localshape[distaxes[i]] for i in range(len(distaxes))] 149 | 150 | if target_ranks is None: 151 | num_targets = [len(target_offsets_axis) for target_offsets_axis in target_offsets] 152 | target_ranks = tuple(range(np.prod(num_targets))) 153 | elif type(target_ranks) is not tuple: 154 | target_ranks = tuple(target_ranks) 155 | 156 | self.target_offsets = target_offsets 157 | self.target_ranks = target_ranks 158 | 159 | # generate a unique variable name used on target representing this instance 160 | global array_id 161 | #: Unique variable name of the local *array* on *engine*. 162 | #: Unless you are doing manual stuff on the *engines* there is no need for dealing with this attribute. 163 | self.name = 'dist_array' + str(array_id) 164 | array_id += 1 165 | 166 | if no_allocation: 167 | self.view.push({self.name : None}, targets=self.target_ranks) 168 | else: 169 | target_shapes = self.target_shapes() 170 | 171 | self.view.scatter('target_shape', target_shapes, targets=self.target_ranks) 172 | self.view.push({'kwargs' : kwargs, 'dtype' : dtype}, targets=self.target_ranks) 173 | self.view.execute('%s = %s(shape=target_shape[0], dtype=dtype, **kwargs)' % \ 174 | (self.name, self.__class__.target_modulename + "." + self.__class__.local_arraytype.__name__), targets=self.target_ranks) 175 | 176 | def __del__(self): 177 | self.view.execute('del %s' % self.name, targets=self.target_ranks) 178 | 179 | def target_shapes(self): 180 | """generate a list of the local shape on each target in use""" 181 | targetshapes = [] 182 | num_targets = [len(target_offsets_axis) for target_offsets_axis in self.target_offsets] 183 | for rank_idx_vector in np.ndindex(*num_targets): # last dimension is iterated over first 184 | targetshape = list(self.shape) 185 | for distaxis_idx in range(len(self.distaxes)): 186 | distaxis = self.distaxes[distaxis_idx] 187 | i = rank_idx_vector[distaxis_idx] 188 | end = self.target_offsets[distaxis_idx][i+1] if i != num_targets[distaxis_idx] - 1 else self.shape[distaxis] 189 | targetshape[distaxis] = end - self.target_offsets[distaxis_idx][i] 190 | targetshapes.append(targetshape) 191 | 192 | return targetshapes 193 | 194 | def target_offset_vectors(self): 195 | """generate a list of the local offset vectors on each target in use""" 196 | target_offset_vectors = [] 197 | num_targets = [len(target_offsets_axis) for target_offsets_axis in self.target_offsets] 198 | for rank_idx_vector in np.ndindex(*num_targets): # last dimension is iterated over first 199 | target_offset_vector = [0] * len(self.shape) 200 | for distaxis_idx in range(len(self.distaxes)): 201 | distaxis = self.distaxes[distaxis_idx] 202 | i = rank_idx_vector[distaxis_idx] 203 | target_offset_vector[distaxis] = self.target_offsets[distaxis_idx][i] 204 | target_offset_vectors.append(target_offset_vector) 205 | 206 | return target_offset_vectors 207 | 208 | def __get_linear_rank_idx(self, rank_idx_vector): 209 | # convert rank_idx_vector to linear rank index 210 | # last dimension is iterated over first 211 | num_targets = [len(target_offsets_axis) for target_offsets_axis in self.target_offsets] 212 | pitch = [int(np.prod(num_targets[i+1:])) for i in range(len(self.distaxes))] 213 | return sum(rank_idx_component * pitch_component for rank_idx_component, pitch_component in zip(rank_idx_vector, pitch)) 214 | 215 | def __getitem__(self, args): 216 | # bitmask indexing 217 | if isinstance(args, self.__class__) and args.dtype == bool: 218 | bitmask = args 219 | assert bitmask.shape == self.shape,\ 220 | "shape of bitmask (%s) does not correspond to shape of array (%s)"\ 221 | % (str(bitmask.shape), str(self.shape)) 222 | 223 | bitmask = bitmask.dist_like(self) # equalize distribution if necessary 224 | self.view.execute("tmp = {0}[{1}]; tmp_size = tmp.shape[0]".format(repr(self), repr(bitmask)), targets=self.target_ranks) 225 | sizes = self.view.pull("tmp_size", targets=self.target_ranks) 226 | new_target_ranks = [rank for rank, size in zip(self.target_ranks, sizes) if size > 0] 227 | new_sizes = [size for size in sizes if size > 0] 228 | partial_sum = lambda a, b: a + [a[-1] + b] 229 | new_target_offsets = [ [0] + reduce(partial_sum, new_sizes[1:-1], new_sizes[0:1]) ] 230 | new_shape = [sum(new_sizes)] 231 | # create resulting ndarray 232 | result = self.__class__(new_shape, self.dtype, 0, new_target_offsets, new_target_ranks, no_allocation=True, **self.kwargs) 233 | self.view.execute("{0} = tmp; del tmp".format(result.name), targets=result.target_ranks) 234 | return result 235 | 236 | if args == slice(None): 237 | args = (slice(None),) * len(self.shape) 238 | 239 | if not isinstance(args, list) and not isinstance(args, tuple): 240 | args = (args,) 241 | 242 | assert len(args) == len(self.shape),\ 243 | "number of arguments (%d) does not correspond to the dimension (%d)"\ 244 | % (len(args), len(self.shape)) 245 | 246 | # wrap all integer indices 247 | args = [(arg + s) % s if type(arg) is int else arg for arg, s in zip(args, self.shape)] 248 | 249 | # shape of the new sliced ndarray 250 | new_shape, clean_view = helper.view_of_shape(self.shape, args) 251 | 252 | # if args is a list of indices then return a single data value 253 | if not new_shape: 254 | local_idx = list(args) 255 | rank_idx_vector = [] 256 | for distaxis, target_offsets in zip(self.distaxes, self.target_offsets): 257 | dist_idx = args[distaxis] 258 | rank_idx_component = np.searchsorted(target_offsets, dist_idx, side="right") - 1 259 | local_idx[distaxis] = dist_idx - target_offsets[rank_idx_component] 260 | rank_idx_vector.append(rank_idx_component) 261 | 262 | rank_idx = self.__get_linear_rank_idx(rank_idx_vector) 263 | return self.view.pull("%s%s" % (self.name, repr(local_idx)), targets=self.target_ranks[rank_idx]) 264 | 265 | if all(type(clean_view[distaxis]) is int for distaxis in self.distaxes): 266 | # return local array because all distributed axes have vanished 267 | rank_idx_vector = [] 268 | for distaxis, target_offsets in zip(self.distaxes, self.target_offsets): 269 | dist_idx = clean_view[distaxis] 270 | rank_idx_component = np.searchsorted(target_offsets, dist_idx, side="right") - 1 271 | clean_view[distaxis] = dist_idx - target_offsets[rank_idx_component] 272 | rank_idx_vector.append(rank_idx_component) 273 | 274 | rank_idx = self.__get_linear_rank_idx(rank_idx_vector) 275 | self.view.execute("sliced = %s%s" % (self.name, repr(clean_view)), targets=self.target_ranks[rank_idx]) 276 | return self.view.pull("sliced", targets=self.target_ranks[rank_idx]) 277 | 278 | # determine properties of the new, sliced ndarray 279 | # keep these in mind when reading the following for loop 280 | local_slices_aa = [] # aa = all (distributed) axes 281 | new_target_offsets = [] 282 | new_rank_ids_aa = [] 283 | new_distaxes = [] 284 | 285 | for distaxis, distaxis_idx, target_offsets \ 286 | in zip(self.distaxes, range(len(self.distaxes)), self.target_offsets): 287 | 288 | # slice object in the direction of the distributed axis 289 | distaxis_slice = clean_view[distaxis] 290 | 291 | # if it turns out that distaxis_slice is actually an int, this axis has to vanish. 292 | # That means the local slice object has to be an int too. 293 | if type(distaxis_slice) == int: 294 | dist_idx = distaxis_slice 295 | rank_idx_component = np.searchsorted(target_offsets, dist_idx, side="right") - 1 296 | local_idx = dist_idx - target_offsets[rank_idx_component] 297 | local_slices_aa.append((local_idx,)) 298 | new_rank_ids_aa.append((rank_idx_component,)) 299 | continue 300 | 301 | # determine properties of the new, sliced ndarray 302 | local_slices_sa = [] # sa = single axis 303 | new_target_offsets_sa = [] 304 | new_rank_ids_sa = [] 305 | total_ids = 0 306 | 307 | first_rank_idx = np.searchsorted(target_offsets, distaxis_slice.start, side="right") - 1 308 | last_rank_idx = np.searchsorted(target_offsets, distaxis_slice.stop, side="right") 309 | 310 | for i in range(first_rank_idx, last_rank_idx): 311 | 312 | # index range of current target 313 | begin = target_offsets[i] 314 | end = target_offsets[i+1] if i < len(target_offsets)-1 else self.shape[distaxis] 315 | # first slice index within [begin, end) 316 | firstSliceIdx = helper.getFirstSliceIdx(distaxis_slice, begin, end) 317 | if firstSliceIdx is None: continue 318 | # calculate last slice index of distaxis_slice 319 | tmp = (distaxis_slice.stop-1 - distaxis_slice.start) / distaxis_slice.step 320 | lastIdx = distaxis_slice.start + tmp * distaxis_slice.step 321 | # calculate last sub index within [begin,end) 322 | tmp = (end-1 - firstSliceIdx) / distaxis_slice.step 323 | lastSliceIdx = firstSliceIdx + tmp * distaxis_slice.step 324 | lastSliceIdx = min(lastSliceIdx, lastIdx) 325 | # slice object for current target 326 | local_slices_sa.append(slice(firstSliceIdx - begin, lastSliceIdx+1 - begin, distaxis_slice.step)) 327 | # number of indices remaining on the current target after slicing 328 | num_ids = (lastSliceIdx - firstSliceIdx) / distaxis_slice.step + 1 329 | # new offset for current target 330 | new_target_offsets_sa.append(total_ids) 331 | total_ids += num_ids 332 | # target rank index 333 | new_rank_ids_sa.append(i) 334 | 335 | new_rank_ids_sa = np.array(new_rank_ids_sa) 336 | 337 | local_slices_aa.append(local_slices_sa) 338 | new_target_offsets.append(new_target_offsets_sa) 339 | new_rank_ids_aa.append(new_rank_ids_sa) 340 | 341 | # shift distaxis by the number of vanished axes in the left of it 342 | new_distaxis = distaxis - sum(1 for arg in args[:distaxis] if type(arg) is int) 343 | new_distaxes.append(new_distaxis) 344 | 345 | # create list of targets which participate slicing, new_target_ranks 346 | num_ranks_aa = [len(new_rank_ids_sa) for new_rank_ids_sa in new_rank_ids_aa] 347 | new_target_ranks = [] 348 | for idx in np.ndindex(*num_ranks_aa): 349 | rank_idx_vector = [new_rank_ids_aa[i][idx[i]] for i in range(len(idx))] 350 | rank_idx = self.__get_linear_rank_idx(rank_idx_vector) 351 | new_target_ranks.append(self.target_ranks[rank_idx]) 352 | 353 | # create resulting ndarray 354 | result = self.__class__(new_shape, self.dtype, new_distaxes, new_target_offsets, new_target_ranks, no_allocation=True, **self.kwargs) 355 | 356 | # remote slicing 357 | local_args_list = [] 358 | for idx in np.ndindex(*num_ranks_aa): 359 | local_slices = [local_slices_aa[i][idx[i]] for i in range(len(idx))] 360 | local_args = list(args) 361 | for distaxis, distaxis_idx in zip(self.distaxes, range(len(self.distaxes))): 362 | local_args[distaxis] = local_slices[distaxis_idx] 363 | local_args_list.append(local_args) 364 | 365 | self.view.scatter('local_args', local_args_list, targets=result.target_ranks) 366 | self.view.execute('%s = %s[local_args[0]]' % (result.name, self.name), targets=result.target_ranks) 367 | 368 | return result 369 | 370 | def __setitem__(self, key, value): 371 | self.local_copy_is_dirty = True 372 | # bitmask indexing 373 | if isinstance(key, self.__class__) and key.dtype == bool: 374 | bitmask = key.dist_like(self) 375 | self.view.execute("%s[%s] = %s" % (repr(self), repr(bitmask), repr(value)), targets=self.target_ranks) 376 | return 377 | 378 | # if args is [:] then assign value to the entire ndarray 379 | if key == slice(None): 380 | # assign local array to self 381 | if isinstance(value, self.__class__.local_arraytype): 382 | subarrays = [] 383 | for target_offset_vector, target_shape in zip(self.target_offset_vectors(), self.target_shapes()): 384 | window = [slice(start, start+length) for start, length in zip(target_offset_vector, target_shape)] 385 | subarrays.append(value[window]) 386 | 387 | self.view.scatter("subarray", subarrays, targets=self.target_ranks) 388 | self.view.execute("%s[:] = subarray[0]" % self.name, targets=self.target_ranks) 389 | return 390 | 391 | # assign other array or value to self 392 | other = value.dist_like(self) if hasattr(value, "dist_like") else value 393 | self.view.execute("%s[:] = %s" % (repr(self), repr(other)), targets=self.target_ranks) 394 | return 395 | 396 | if not isinstance(key, list) and not isinstance(key, tuple): 397 | key = (key,) 398 | 399 | assert len(key) == len(self.shape),\ 400 | "number of arguments (%d) does not correspond to the dimension (%d)"\ 401 | % (len(key), len(self.shape)) 402 | 403 | # value assignment (key == list of indices) 404 | if all(type(k) is int for k in key): 405 | local_idx = list(key) 406 | local_idx = [(i + s) % s for i, s in zip(local_idx, self.shape)] 407 | rank_idx_vector = [] 408 | for distaxis, target_offsets in zip(self.distaxes, self.target_offsets): 409 | dist_idx = key[distaxis] 410 | rank_idx_component = np.searchsorted(target_offsets, dist_idx, side="right") - 1 411 | local_idx[distaxis] = dist_idx - target_offsets[rank_idx_component] 412 | rank_idx_vector.append(rank_idx_component) 413 | 414 | rank_idx = self.__get_linear_rank_idx(rank_idx_vector) 415 | self.view.push({'value' : value}, targets=self.target_ranks[rank_idx]) 416 | self.view.execute("%s%s = value" % (self.name, repr(local_idx)), targets=self.target_ranks[rank_idx]) 417 | return 418 | 419 | # assign value to sub-array of self 420 | sub_array = self[key] 421 | sub_array[:] = value 422 | 423 | def __str__(self): 424 | return self.gather().__str__() 425 | 426 | def __repr__(self): 427 | return self.name 428 | 429 | @property 430 | def local_copy(self): 431 | if not hasattr(self, "_local_copy") or self.local_copy_is_dirty: 432 | self._local_copy = self.gather() 433 | self.local_copy_is_dirty = False 434 | return self._local_copy 435 | 436 | def __getattr__(self, name): 437 | """If the requested attribute is an attribute of the local array and not of this array then 438 | gather() is called and the request is forwarded to the gathered array. This makes this 439 | distributed array more behaving like a local array.""" 440 | if not hasattr(self.__class__.local_arraytype, name): 441 | raise AttributeError(name) 442 | 443 | return getattr(self.local_copy, name) 444 | 445 | def gather(self): 446 | """Gathers local instances of {local_arraytype_name} from *engines*, concatenates them and returns 447 | the result. 448 | 449 | .. note:: You may not call this method explicitly because if you try to access an attribute 450 | of the local array ({local_arraytype_name}), ``gather()`` is called implicitly before the request is forwarded 451 | to that internal gathered array. Just access attributes like you do for the local array. 452 | The internal copy is held until ``__setitem__`` is called, e.g. ``a[1] = 3.0``, setting 453 | a dirty flag to the local copy. 454 | 455 | .. warning:: If another array overlapping this array is manipulating its data there is no chance to set 456 | the dirty flag so you have to keep in mind to call ``gather()`` explicitly in this case! 457 | 458 | :return: instance of {local_arraytype_name} 459 | """ 460 | local_arrays = self.view.pull(self.name, targets=self.target_ranks) 461 | 462 | result = self.__class__.local_arraytype(shape=self.shape, dtype=self.dtype, **self.kwargs) 463 | 464 | for target_offset_vector, target_shape, local_array \ 465 | in zip(self.target_offset_vectors(), self.target_shapes(), local_arrays): 466 | 467 | window = [slice(start, start+length) for start, length in zip(target_offset_vector, target_shape)] 468 | result[window] = local_array 469 | 470 | return result 471 | 472 | def copy(self): 473 | """Returns a hard copy of this array. 474 | """ 475 | assert self.__class__.may_allocate == True, "{0} is not allowed to allocate new memory.".format(self.__class__.__name__) 476 | 477 | result = self.__class__(self.shape, self.dtype, self.distaxes, self.target_offsets, self.target_ranks, no_allocation=True, **self.kwargs) 478 | self.view.execute("%s = %s.copy()" % (result.name, self.name), targets=self.target_ranks) 479 | return result 480 | 481 | def is_distributed_like(self, other): 482 | if self.distaxes == other.distaxes: 483 | if all(np.array_equal(my_target_offsets, other_target_offsets) \ 484 | for my_target_offsets, other_target_offsets in zip(self.target_offsets, other.target_offsets)): 485 | if self.target_ranks == other.target_ranks: 486 | return True 487 | return False 488 | 489 | def dist_like(self, other): 490 | """Redistributes a copy of this array (*self*) like *other* and returns the result. 491 | Checks whether redistribution is necessary and returns *self* if not. 492 | 493 | Redistribution involves inter-engine communication. 494 | 495 | :param other: target array 496 | :type other: distributed array 497 | :raises AssertionError: if the shapes of *self* and *other* don't match. 498 | :return: new array with the same content as *self* but distributed like *other*. 499 | If *self* is already distributed like *other* nothing is done and *self* is returned. 500 | """ 501 | assert self.shape == other.shape,\ 502 | "Shapes do not match: " + str(self.shape) + " <-> " + str(other.shape) 503 | 504 | # if self is already distributed like *other* do nothing 505 | if self.is_distributed_like(other): 506 | return self 507 | 508 | assert self.__class__.may_allocate, "{0} is not allowed to allocate new memory.".format(self.__class__.__name__) 509 | 510 | # todo: optimization -> helper.common_decomposition should be a generator 511 | common_axes, common_offsets, common_idx_pairs = \ 512 | helper.common_decomposition(self.distaxes, self.target_offsets, other.distaxes, other.target_offsets, self.shape) 513 | 514 | my_commData = defaultdict(list) 515 | other_commData = defaultdict(list) 516 | 517 | tag = 0 518 | num_offsets = [len(common_offsets_sa) for common_offsets_sa in common_offsets] 519 | 520 | for idx in np.ndindex(*num_offsets): 521 | my_rank_idx_vector = [0] * len(self.distaxes) 522 | other_rank_idx_vector = [0] * len(other.distaxes) 523 | my_window = [slice(None)] * len(self.shape) 524 | other_window = [slice(None)] * len(self.shape) 525 | 526 | for i in range(len(idx)): 527 | common_axis = common_axes[i] 528 | begin = common_offsets[i][idx[i]] 529 | end = common_offsets[i][idx[i]+1] if idx[i] < len(common_offsets[i]) -1 else self.shape[common_axes[i]] 530 | 531 | if common_axis in self.distaxes and common_axis in other.distaxes: 532 | my_rank_idx_comp = common_idx_pairs[i][idx[i],0] 533 | other_rank_idx_comp = common_idx_pairs[i][idx[i],1] 534 | my_rank_idx_vector[self.distaxes.index(common_axis)] = my_rank_idx_comp 535 | other_rank_idx_vector[other.distaxes.index(common_axis)] = other_rank_idx_comp 536 | 537 | my_offset = self.target_offsets[self.distaxes.index(common_axis)][my_rank_idx_comp] 538 | my_window[common_axis] = slice(begin - my_offset, end - my_offset) 539 | other_offset = other.target_offsets[other.distaxes.index(common_axis)][other_rank_idx_comp] 540 | other_window[common_axis] = slice(begin - other_offset, end - other_offset) 541 | 542 | continue 543 | 544 | if common_axis in self.distaxes: 545 | my_rank_idx_vector[self.distaxes.index(common_axis)] = idx[i] 546 | other_window[common_axis] = slice(begin, end) 547 | if common_axis in other.distaxes: 548 | other_rank_idx_vector[other.distaxes.index(common_axis)] = idx[i] 549 | my_window[common_axis] = slice(begin, end) 550 | 551 | my_rank = self.target_ranks[self.__get_linear_rank_idx(my_rank_idx_vector)] 552 | other_rank = other.target_ranks[other.__get_linear_rank_idx(other_rank_idx_vector)] 553 | 554 | my_commData[my_rank].append((other_rank, my_window, tag)) 555 | other_commData[other_rank].append((my_rank, other_window, tag)) 556 | 557 | tag += 1 558 | 559 | # push communication meta-data to engines 560 | self.view.scatter('src_commData', my_commData.values(), targets=my_commData.keys()) 561 | self.view.scatter('dest_commData', other_commData.values(), targets=other_commData.keys()) 562 | 563 | # result ndarray 564 | result = self.__class__(self.shape, self.dtype, other.distaxes, other.target_offsets, other.target_ranks, False, **self.kwargs) 565 | 566 | self.__class__.interengine_copier(self, result) 567 | 568 | return result 569 | 570 | def info(self, name): 571 | print name + " info:" 572 | print "{}.name".format(name), self.name 573 | print "{}.target_ranks".format(name), self.target_ranks 574 | print "{}.target_offsets".format(name), self.target_offsets 575 | print "{}.distaxes".format(name), self.distaxes 576 | self.view.execute("dt = str({}.dtype)".format(repr(self)), targets=self.target_ranks) 577 | print "{}.dtypes".format(name), self.view.pull("dt", targets=self.target_ranks) 578 | self.view.execute("t = str(type({}))".format(repr(self)), targets=self.target_ranks) 579 | print "{}.types".format(name), self.view.pull("t", targets=self.target_ranks) 580 | 581 | def __elementwise_op__(self, op, *args): 582 | args = [arg.dist_like(self) if hasattr(arg, "target_ranks") else arg for arg in args] 583 | arg_names = [repr(arg) for arg in args] 584 | arg_string = ",".join(arg_names) 585 | 586 | result = self.__class__(self.shape, self.dtype, self.distaxes, self.target_offsets, self.target_ranks, no_allocation=True, **self.kwargs) 587 | 588 | self.view.execute("{0} = {1}.{2}({3}); dtype={0}.dtype".format(repr(result), repr(self), op, arg_string), targets=self.target_ranks) 589 | result.dtype = self.view.pull("dtype", targets=result.target_ranks[0]) 590 | result.nbytes = np.dtype(result.dtype).itemsize * np.prod(result.shape) 591 | 592 | return result 593 | 594 | def __elementwise_iop__(self, op, *args): 595 | args = [arg.dist_like(self) if hasattr(arg, "target_ranks") else arg for arg in args] 596 | arg_names = [repr(arg) for arg in args] 597 | arg_string = ",".join(arg_names) 598 | self.view.execute("%s = %s.%s(%s)" % (repr(self), repr(self), op, arg_string), targets=self.target_ranks) 599 | return self 600 | 601 | #---------------------------------------------------------------- 602 | 603 | from multiple_axes_funcs import * -------------------------------------------------------------------------------- /pyDive/distribution/multiple_axes_funcs.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import types 4 | from multiple_axes import DistributedGenericArray 5 | import numpy as np 6 | import pyDive.IPParallelClient as com 7 | 8 | def distribute(local_arraytype, newclassname, target_modulename, interengine_copier=None, may_allocate = True): 9 | binary_ops = ["add", "sub", "mul", "floordiv", "div", "mod", "pow", "lshift", "rshift", "and", "xor", "or"] 10 | 11 | binary_iops = ["__i" + op + "__" for op in binary_ops] 12 | binary_rops = ["__r" + op + "__" for op in binary_ops] 13 | binary_ops = ["__" + op + "__" for op in binary_ops] 14 | unary_ops = ["__neg__", "__pos__", "__abs__", "__invert__", "__complex__", "__int__", "__long__", "__float__", "__oct__", "__hex__"] 15 | comp_ops = ["__lt__", "__le__", "__eq__", "__ne__", "__ge__", "__gt__"] 16 | 17 | special_ops_avail = set(name for name in local_arraytype.__dict__.keys() if name.endswith("__")) 18 | 19 | make_special_op = lambda op: lambda self, *args: self.__elementwise_op__(op, *args) 20 | make_special_iop = lambda op: lambda self, *args: self.__elementwise_iop__(op, *args) 21 | 22 | special_ops_dict = {op : make_special_op(op) for op in \ 23 | set(binary_ops + binary_rops + unary_ops + comp_ops) & special_ops_avail} 24 | special_iops_dict = {op : make_special_iop(op) for op in set(binary_iops) & special_ops_avail} 25 | 26 | formated_doc_funs = ("__init__", "gather") 27 | 28 | result_dict = dict(DistributedGenericArray.__dict__) 29 | 30 | # docs 31 | result_dict["__doc__"] = result_dict["__doc__"].format(\ 32 | local_arraytype_name=local_arraytype.__module__ + "." + local_arraytype.__name__, 33 | arraytype_name=newclassname) 34 | # copy methods which have formated docstrings because their docstrings are going to be modified 35 | copied_methods = {k : types.FunctionType(v.func_code, v.func_globals, name=v.func_name, argdefs=v.func_defaults)\ 36 | for k,v in result_dict.items() if k in formated_doc_funs} 37 | result_dict.update(copied_methods) 38 | 39 | result_dict.update(special_ops_dict) 40 | result_dict.update(special_iops_dict) 41 | 42 | result = type(newclassname, (), result_dict) 43 | result.local_arraytype = local_arraytype 44 | result.target_modulename = target_modulename 45 | result.interengine_copier = interengine_copier 46 | result.may_allocate = may_allocate 47 | 48 | # docs 49 | for method in (v for k,v in result.__dict__.items() if k in formated_doc_funs): 50 | method.__doc__ = method.__doc__.format(\ 51 | local_arraytype_name=local_arraytype.__module__ + "." + local_arraytype.__name__, 52 | arraytype_name=newclassname) 53 | 54 | return result 55 | 56 | def generate_factories(arraytype, factory_names, dtype_default): 57 | 58 | def factory_wrapper(factory_name, shape, dtype, distaxes, kwargs): 59 | result = arraytype(shape, dtype, distaxes, None, None, True, **kwargs) 60 | 61 | target_shapes = result.target_shapes() 62 | 63 | view = com.getView() 64 | view.scatter('target_shape', target_shapes, targets=result.target_ranks) 65 | view.push({'kwargs' : kwargs, 'dtype' : dtype}, targets=result.target_ranks) 66 | 67 | view.execute("{0} = {1}(shape=target_shape[0], dtype=dtype, **kwargs)".format(result.name, factory_name),\ 68 | targets=result.target_ranks) 69 | return result 70 | 71 | make_factory = lambda factory_name: lambda shape, dtype=dtype_default, distaxes='all', **kwargs:\ 72 | factory_wrapper(arraytype.target_modulename + "." + factory_name, shape, dtype, distaxes, kwargs) 73 | 74 | factories_dict = {factory_name : make_factory(factory_name) for factory_name in factory_names} 75 | 76 | # add docstrings 77 | for name, factory in factories_dict.items(): 78 | factory.__name__ = name 79 | factory.__doc__ = \ 80 | """Create a *{0}* instance. This function calls its local counterpart *{1}* on each :term:`engine`. 81 | 82 | :param ints shape: shape of array 83 | :param dtype: datatype of a single element 84 | :param ints distaxes: distributed axes 85 | :param kwargs: keyword arguments are passed to the local function *{1}* 86 | """.format(arraytype.__name__, str(arraytype.local_arraytype.__module__) + "." + name) 87 | 88 | return factories_dict 89 | 90 | def generate_factories_like(arraytype, factory_names): 91 | 92 | def factory_like_wrapper(factory_name, other, kwargs): 93 | result = arraytype(other.shape, other.dtype, other.distaxes, other.target_offsets, other.target_ranks, True, **kwargs) 94 | view = com.getView() 95 | view.push({'kwargs' : kwargs}, targets=result.target_ranks) 96 | view.execute("{0} = {1}({2}, **kwargs)".format(result.name, factory_name, other.name), targets=result.target_ranks) 97 | return result 98 | 99 | make_factory = lambda factory_name: lambda other, **kwargs: \ 100 | factory_like_wrapper(arraytype.target_modulename + "." + factory_name, other, kwargs) 101 | 102 | factories_dict = {factory_name : make_factory(factory_name) for factory_name in factory_names} 103 | 104 | # add docstrings 105 | for name, factory in factories_dict.items(): 106 | factory.__name__ = name 107 | factory.__doc__ = \ 108 | """Create a *{0}* instance with the same shape, dtype and distribution as ``other``. 109 | This function calls its local counterpart *{1}* on each :term:`engine`. 110 | 111 | :param other: other array 112 | :param kwargs: keyword arguments are passed to the local function *{1}* 113 | """.format(arraytype.__name__, str(arraytype.local_arraytype.__module__) + "." + name) 114 | 115 | return factories_dict 116 | 117 | def generate_ufuncs(ufunc_names, target_modulename): 118 | 119 | def ufunc_wrapper(ufunc_name, args, kwargs): 120 | arg0 = args[0] 121 | args = [arg.dist_like(arg0) if hasattr(arg, "target_ranks") else arg for arg in args] 122 | arg_names = [repr(arg) for arg in args] 123 | arg_string = ",".join(arg_names) 124 | 125 | view = com.getView() 126 | result = arg0.__class__(arg0.shape, arg0.dtype, arg0.distaxes, arg0.target_offsets, arg0.target_ranks, no_allocation=True, **arg0.kwargs) 127 | 128 | view.execute("{0} = {1}({2}); dtype={0}.dtype".format(repr(result), ufunc_name, arg_string), targets=arg0.target_ranks) 129 | result.dtype = view.pull("dtype", targets=result.target_ranks[0]) 130 | result.nbytes = np.dtype(result.dtype).itemsize * np.prod(result.shape) 131 | return result 132 | 133 | make_ufunc = lambda ufunc_name: lambda *args, **kwargs: ufunc_wrapper(target_modulename + "." + ufunc_name, args, kwargs) 134 | 135 | return {ufunc_name: make_ufunc(ufunc_name) for ufunc_name in ufunc_names} -------------------------------------------------------------------------------- /pyDive/fragment.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2014 Heiko Burau 3 | 4 | This file is part of pyDive. 5 | 6 | pyDive is free software: you can redistribute it and/or modify 7 | it under the terms of of either the GNU General Public License or 8 | the GNU Lesser General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | pyDive is distributed in the hope that it will be useful, 12 | but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | GNU General Public License and the GNU Lesser General Public License 15 | for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | and the GNU Lesser General Public License along with pyDive. 19 | If not, see . 20 | """ 21 | __doc__ = None 22 | 23 | import IPParallelClient as com 24 | from IPython.parallel import interactive 25 | try: 26 | from arrays.h5_ndarray import h5_ndarray 27 | except ImportError: 28 | h5_ndarray = None 29 | try: 30 | from arrays.ad_ndarray import ad_ndarray 31 | except ImportError: 32 | ad_ndarray = None 33 | import math 34 | 35 | #: list of array types that store their elements on hard disk 36 | hdd_arraytypes = (h5_ndarray, ad_ndarray) 37 | 38 | def __bestStepSize(arrays, axis, memory_limit): 39 | view = com.getView() 40 | 41 | # minimum amount of memory available and memory needed, both per engine 42 | get_mem_av_node = interactive(lambda: psutil.virtual_memory().available) 43 | tmp_targets = view.targets 44 | view.targets = 'all' 45 | mem_av = min(view.apply(get_mem_av_node)) / com.getPPN() 46 | mem_needed = sum(a.nbytes for a in arrays) / len(view) 47 | view.targets = tmp_targets 48 | 49 | # edge length of the whole array 50 | edge_length = arrays[0].shape[axis] 51 | # maximum edge length on one engine according to the available memory 52 | step_size = memory_limit * edge_length * mem_av / mem_needed 53 | 54 | if step_size >= edge_length: 55 | return edge_length 56 | 57 | # round 'step_size' down to nearest power of two 58 | return pow(2, int(math.log(step_size, 2))) 59 | 60 | def fragment(*arrays, **kwargs): 61 | """Create fragments of *arrays* so that each fragment will fit into the combined 62 | main memory of all engines when calling ``load()``. The fragmentation is done by array slicing 63 | along the longest axis of ``arrays[0]``. 64 | The edge size of the fragments is a power of two except for the last fragment. 65 | 66 | :param array: distributed arrays (e.g. pyDive.ndarray, pyDive.h5_ndarray, ...) 67 | :param kwargs: optional keyword arguments are: ``memory_limit`` and ``offset``. 68 | :param float memory_limit: fraction of the combined main memory of all engines reserved for fragmentation. 69 | Defaults to ``0.25``. 70 | :param bool offset: If ``True`` the returned tuple is extended by the fragments' offset (along the distributed axis). 71 | Defaults to ``False``. 72 | :raises AssertionError: If not all arrays have the same shape. 73 | :raises AssertionError: If not all arrays are distributed along the same axis. 74 | :return: generator object (list) of tuples. Each tuple consists of one fragment for each array in *arrays*. 75 | 76 | Note that *arrays* may contain an arbitrary number of distributed arrays of any type. 77 | While the fragments' size is solely calculated based on the memory consumption of 78 | arrays that store their elements on hard disk (see :obj:`hdd_arraytypes`), 79 | the fragmentation itself is applied on all arrays in the same way. 80 | 81 | Example: :: 82 | 83 | big_h5_array = pyDive.h5.open("monster.h5", "/") 84 | # big_h5_array.load() # crash 85 | 86 | for h5_array, offset in pyDive.fragment(big_h5_array, offset=True): 87 | a = h5_array.load() # no crash 88 | print "This fragment's offset is", offset, "on axis:", a.distaxis 89 | """ 90 | # default keyword arguments 91 | kwargs_defaults = {"memory_limit" : 0.25, "offset" : False} 92 | kwargs_defaults.update(kwargs) 93 | kwargs = kwargs_defaults 94 | 95 | memory_limit = kwargs["memory_limit"] 96 | offset = kwargs["offset"] 97 | 98 | if not arrays: return 99 | 100 | assert all(a.shape == arrays[0].shape for a in arrays), \ 101 | "all arrays must have the same shape" 102 | 103 | # calculate the best suitable step size (-> fragment's edge size) according to the amount 104 | # of available memory on the engines 105 | hdd_arrays = [a for a in arrays if (hasattr(a, "arraytype") and a.arraytype in hdd_arraytypes) or type(a) in hdd_arraytypes] 106 | longest_axis = max(range(len(arrays[0].shape)), key=lambda axis : arrays[0].shape[axis]) 107 | step = __bestStepSize(hdd_arrays, longest_axis, memory_limit) 108 | 109 | shape = arrays[0].shape 110 | # list of slices representing the fragment's shape 111 | fragment_window = [slice(None)] * len(shape) 112 | 113 | for begin in range(0, shape[longest_axis], step): 114 | end = min(begin + step, shape[longest_axis]) 115 | fragment_window[longest_axis] = slice(begin, end) 116 | 117 | result = [a[fragment_window] for a in arrays] 118 | if offset: 119 | offset_vector = [0] * len(shape) 120 | offset_vector[longest_axis] = begin 121 | result += [offset_vector] 122 | if len(result) == 1: 123 | yield result[0] 124 | else: 125 | yield result 126 | 127 | return 128 | -------------------------------------------------------------------------------- /pyDive/mappings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2014 Heiko Burau 3 | 4 | This file is part of pyDive. 5 | 6 | pyDive is free software: you can redistribute it and/or modify 7 | it under the terms of of either the GNU General Public License or 8 | the GNU Lesser General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | pyDive is distributed in the hope that it will be useful, 12 | but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | GNU General Public License and the GNU Lesser General Public License 15 | for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | and the GNU Lesser General Public License along with pyDive. 19 | If not, see . 20 | """ 21 | 22 | __doc__=\ 23 | """If `numba `_ is installed the particle shape functions will 24 | be compiled which gives an appreciable speedup. 25 | """ 26 | 27 | import numpy as np 28 | try: 29 | from numba import vectorize 30 | except ImportError: 31 | vectorize = None 32 | 33 | class NGP: 34 | """Nearest-Grid-Point 35 | """ 36 | support = 1.0 37 | 38 | @staticmethod 39 | def __call__(x): 40 | if abs(x) < 0.5: 41 | return 1.0 42 | return 0.0 43 | 44 | class CIC: 45 | """Cloud-in-Cell 46 | """ 47 | support = 2.0 48 | 49 | @staticmethod 50 | def __call__(x): 51 | if abs(x) < 1.0: 52 | return 1.0 - abs(x) 53 | return 0.0 54 | 55 | def __apply_MapOp(mesh, particles_pos, shape_function, mapOp): 56 | half_supp = 0.5 * shape_function.support 57 | num_coeffs_per_axis = int(np.ceil(shape_function.support)) 58 | dim = len(np.shape(mesh)) 59 | 60 | # ndarray which holds the non-zero particle assignment values of one particle 61 | coeffs = np.empty([num_coeffs_per_axis] * dim) 62 | 63 | if vectorize: 64 | # vectorize shape function to a compiled ufunc 65 | v_shape_function = vectorize(['f4(f4)', 'f8(f8)'])(shape_function.__call__) 66 | else: 67 | # numpy fallback 68 | v_shape_function = np.vectorize(shape_function.__call__) 69 | 70 | for idx, pos in enumerate(particles_pos): 71 | # lowest mesh indices that contributes to the mapping 72 | begin = np.ceil(pos - np.ones(dim) * half_supp).astype(int) 73 | 74 | # rearrange mapping area if it overlaps the border 75 | begin = np.maximum(begin, np.zeros(dim, dtype=int)) 76 | begin = np.minimum(begin, np.shape(mesh) - np.ones(dim) * half_supp) 77 | 78 | # compute coefficients (particle assignment values) 79 | for coeff_idx in np.ndindex(np.shape(coeffs)): 80 | rel_vec = begin + coeff_idx - pos 81 | coeffs[coeff_idx] = np.prod(v_shape_function(rel_vec)) 82 | 83 | # do the actual mapping 84 | window = [slice(begin[i], begin[i] + num_coeffs_per_axis) for i in range(dim)] 85 | mapOp(mesh[window], coeffs, idx) 86 | 87 | 88 | def mesh2particles(mesh, particles_pos, shape_function=CIC): 89 | """ 90 | Map mesh values to particles according to a particle shape function. 91 | 92 | :param array-like mesh: n-dimensional array. 93 | Dimension of *mesh* has to be greater or equal to the number of particle position components. 94 | :param particles_pos: 95 | 'd'-dim tuples for 'N' particle positions. The positions can be float32 or float64 and 96 | must be within the shape of *mesh*. 97 | :type particles_pos: (N, d) 98 | :param shape_function: 99 | Callable object returning the particle assignment value for a given param 'x'. 100 | Has to provide a 'support' float attribute which defines the width of the non-zero area. 101 | Defaults to cloud-in-cell. 102 | :type shape_function: callable, optional 103 | :return: Mapped mesh values for each particle. 104 | :type return: ndarray(N, dtype='f') 105 | 106 | Notes: 107 | - The particle shape function is not evaluated outside the mesh. 108 | """ 109 | # resulting ndarray 110 | particles = np.empty(len(particles_pos), dtype='f') 111 | 112 | def mapOp(sub_mesh, coeffs, particle_idx): 113 | particles[particle_idx] = np.add.reduce(sub_mesh * coeffs, axis=None) 114 | 115 | __apply_MapOp(mesh, particles_pos, shape_function, mapOp) 116 | 117 | return particles 118 | 119 | def particles2mesh(mesh, particles, particles_pos, shape_function=CIC): 120 | """ 121 | Map particle values to mesh according to a particle shape function. 122 | Particle values are added to the mesh. 123 | 124 | :param array-like mesh: n-dimensional array. 125 | Dimension of *mesh* has to be greater or equal to the number of particle position components. 126 | :param particles: particle data. 127 | len(*particles*) has to be the same as len(*particles_pos*) 128 | :type particles: array_like (1 dim) 129 | :param particles_pos: 130 | 'd'-dim tuples for 'N' particle positions. The positions can be float32 or float64 and 131 | must be within the shape of *mesh*. 132 | :type particles_pos: (N, d) 133 | :param shape_function: 134 | Callable object returning the particle assignment value for a given param 'x'. 135 | Has to provide a 'support' float attribute which defines the width of the non-zero area. 136 | Defaults to cloud-in-cell. 137 | :type shape_function: callable, optional 138 | :return: *mesh* 139 | 140 | Notes: 141 | - The particle shape function is not evaluated outside the mesh. 142 | """ 143 | 144 | def mapOp(sub_mesh, coeffs, particle_idx): 145 | sub_mesh += particles[particle_idx] * coeffs 146 | 147 | __apply_MapOp(mesh, particles_pos, shape_function, mapOp) 148 | 149 | return mesh 150 | -------------------------------------------------------------------------------- /pyDive/picongpu.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2014 Heiko Burau 3 | 4 | This file is part of pyDive. 5 | 6 | pyDive is free software: you can redistribute it and/or modify 7 | it under the terms of of either the GNU General Public License or 8 | the GNU Lesser General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | pyDive is distributed in the hope that it will be useful, 12 | but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | GNU General Public License and the GNU Lesser General Public License 15 | for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | and the GNU Lesser General Public License along with pyDive. 19 | If not, see . 20 | """ 21 | 22 | __doc__=\ 23 | """This module holds convenient functions for those who use pyDive together with `picongpu \ 24 | `_. 25 | """ 26 | 27 | import os 28 | import os.path 29 | import re 30 | import arrays.h5_ndarray as h5 31 | import structured 32 | 33 | def loadSteps(steps, folder_path, data_path, distaxis=0): 34 | """Python generator object looping all hdf5-data found in *folder_path* 35 | from timesteps appearing in *steps*. 36 | 37 | This generator doesn't read or write any data elements from hdf5 but returns dataset-handles 38 | covered by *pyDive.h5_ndarray* objects. 39 | 40 | All datasets within *data_path* must have the same shape. 41 | 42 | :param ints steps: list of timesteps to loop 43 | :param str folder_path: Path to the folder containing the hdf5-files 44 | :param str data_path: Relative path starting from "/data//" within hdf5-file to the dataset or group of datasets 45 | :param int distaxis: axis on which datasets are distributed over when once loaded into memory. 46 | :return: tuple of timestep and a :ref:`pyDive.h5_ndarray ` 47 | or a structure of pyDive.h5_ndarrays (:mod:`pyDive.structured`). Ordering is done by timestep. 48 | 49 | Notes: 50 | - If the dataset has a '**sim_unit**' attribute its value is stored in ``h5array.unit``. 51 | """ 52 | assert os.path.exists(folder_path), "folder '%s' does not exist" % folder_path 53 | 54 | timestep_and_filename = [] 55 | for filename in os.listdir(folder_path): 56 | if not filename.endswith('.h5'): continue 57 | 58 | timestep = int(re.findall("\d+", filename)[-2]) 59 | if not timestep in steps: continue 60 | timestep_and_filename.append((timestep, filename)) 61 | 62 | # sort by timestep 63 | timestep_and_filename.sort(key=lambda item: item[0]) 64 | 65 | for timestep, filename in timestep_and_filename: 66 | full_filename = os.path.join(folder_path, filename) 67 | full_datapath = os.path.join("/data", str(timestep), data_path) 68 | 69 | h5data = h5.open(full_filename, full_datapath, distaxis) 70 | 71 | # add 'sim_unit' as 'unit' attribute 72 | def add_sim_unit(array): 73 | if hasattr(array, "attrs") and 'sim_unit' in array.attrs: 74 | setattr(array, "unit", array.attrs["sim_unit"]) 75 | return array 76 | if type(h5data) is h5.h5_ndarray: 77 | h5data = add_sim_unit(h5data) 78 | else: 79 | h5data = structured.structured(\ 80 | structured.makeTree_fromTree(h5data.structOfArrays, add_sim_unit)) 81 | 82 | yield timestep, h5data 83 | 84 | def getSteps(folder_path): 85 | """Returns a list of all timesteps in *folder_path*. 86 | """ 87 | assert os.path.exists(folder_path), "folder '%s' does not exist" % folder_path 88 | 89 | result = [] 90 | for filename in os.listdir(folder_path): 91 | if not filename.endswith('.h5'): continue 92 | timestep = int(re.findall("\d+", filename)[-2]) 93 | result.append(timestep) 94 | return result 95 | 96 | def loadAllSteps(folder_path, data_path, distaxis=0): 97 | """Python generator object looping hdf5-data of all timesteps found in *folder_path*. 98 | 99 | This generator doesn't read or write any data elements from hdf5 but returns dataset-handles 100 | covered by *pyDive.h5_ndarray* objects. 101 | 102 | All datasets within *data_path* must have the same shape. 103 | 104 | :param str folder_path: Path to the folder containing the hdf5-files 105 | :param str data_path: Relative path starting from "/data//" within hdf5-file to the dataset or group of datasets 106 | :param int distaxis: axis on which datasets are distributed over when once loaded into memory. 107 | :return: tuple of timestep and a :ref:`pyDive.h5_ndarray ` 108 | or a structure of pyDive.h5_ndarrays (:mod:`pyDive.structured`). Ordering is done by timestep. 109 | 110 | Notes: 111 | - If the dataset has a '**sim_unit**' attribute its value is stored in ``h5array.unit``. 112 | """ 113 | steps = getSteps(folder_path) 114 | 115 | for timestep, data in loadSteps(steps, folder_path, data_path, distaxis): 116 | yield timestep, data 117 | 118 | def loadStep(step, folder_path, data_path, distaxis=0): 119 | """Load hdf5-data from a single timestep found in *folder_path*. 120 | 121 | All datasets within *data_path* must have the same shape. 122 | 123 | :param int step: timestep 124 | :param str folder_path: Path to the folder containing the hdf5-files 125 | :param str data_path: Relative path starting from "/data//" within hdf5-file to the dataset or group of datasets 126 | :param int distaxis: axis on which datasets are distributed over when once loaded into memory. 127 | :return: :ref:`pyDive.h5_ndarray ` 128 | or a structure of pyDive.h5_ndarrays (:mod:`pyDive.structured`). 129 | 130 | Notes: 131 | - If the dataset has a '**sim_unit**' attribute its value is stored in ``h5array.unit``. 132 | """ 133 | step, field = loadSteps([step], folder_path, data_path, distaxis).next() 134 | return field -------------------------------------------------------------------------------- /pyDive/pyDive.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2014 Heiko Burau 3 | 4 | This file is part of pyDive. 5 | 6 | pyDive is free software: you can redistribute it and/or modify 7 | it under the terms of of either the GNU General Public License or 8 | the GNU Lesser General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | pyDive is distributed in the hope that it will be useful, 12 | but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | GNU General Public License and the GNU Lesser General Public License 15 | for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | and the GNU Lesser General Public License along with pyDive. 19 | If not, see . 20 | """ 21 | 22 | __doc__=\ 23 | """Make most used functions and modules directly accessable from pyDive.""" 24 | 25 | # ndarray 26 | import arrays.ndarray 27 | globals().update(arrays.ndarray.factories) 28 | globals().update(arrays.ndarray.ufuncs) 29 | from arrays.ndarray import ndarray 30 | 31 | # hdf5 32 | try: 33 | import arrays.h5_ndarray as h5 34 | except ImportError: 35 | pass 36 | 37 | # adios 38 | try: 39 | import arrays.ad_ndarray as adios 40 | except ImportError: 41 | pass 42 | 43 | # gpu 44 | try: 45 | import arrays.gpu_ndarray as gpu 46 | except ImportError: 47 | pass 48 | 49 | # cloned_ndarray 50 | import cloned_ndarray.factories 51 | cloned = cloned_ndarray.factories 52 | 53 | # fragmentation 54 | from fragment import fragment 55 | 56 | ## algorithm 57 | import algorithm 58 | map = algorithm.map 59 | reduce = algorithm.reduce 60 | mapReduce = algorithm.mapReduce 61 | 62 | # particle-mesh mappings 63 | import mappings 64 | mesh2particles = mappings.mesh2particles 65 | particles2mesh = mappings.particles2mesh 66 | 67 | # structured 68 | import structured 69 | structured = structured.structured 70 | 71 | # picongpu 72 | import picongpu 73 | picongpu = picongpu 74 | 75 | # init 76 | import IPParallelClient 77 | init = IPParallelClient.init 78 | 79 | 80 | # module doc 81 | items = [item for item in globals().items() if not item[0].startswith("__")] 82 | items.sort(key=lambda item: item[0]) 83 | 84 | fun_names = [":obj:`" + item[0] + "<" + item[1].__module__ + "." + item[0] + ">`"\ 85 | for item in items if hasattr(item[1], "__module__")] 86 | 87 | import inspect 88 | module_names = [":mod:`" + item[0] + "<" + item[1].__name__ + ">`"\ 89 | for item in items if inspect.ismodule(item[1])] 90 | 91 | __doc__ += "\n\n**Functions**:\n\n" + "\n\n".join(fun_names)\ 92 | + "\n\n**Modules**:\n\n" + "\n\n".join(module_names) 93 | -------------------------------------------------------------------------------- /pyDive/structured.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2014 Heiko Burau 3 | 4 | This file is part of pyDive. 5 | 6 | pyDive is free software: you can redistribute it and/or modify 7 | it under the terms of of either the GNU General Public License or 8 | the GNU Lesser General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | pyDive is distributed in the hope that it will be useful, 12 | but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | GNU General Public License and the GNU Lesser General Public License 15 | for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | and the GNU Lesser General Public License along with pyDive. 19 | If not, see . 20 | """ 21 | 22 | __doc__ =\ 23 | """The *structured* module addresses the common problem when dealing with 24 | structured data: While the user likes an array-of-structures layout the machine prefers a structure-of-arrays. 25 | In pyDive the method of choice is a *virtual* *array-of-structures*-object. It holds array-like attributes 26 | such as shape and dtype and allows for slicing but is operating on a structure-of-arrays internally. 27 | 28 | Example: :: 29 | 30 | ... 31 | treeOfArrays = {"FieldE" : 32 | {"x" : fielde_x, 33 | "y" : fielde_y, 34 | "z" : fielde_z}, 35 | "FieldB" : 36 | {"x" : fieldb_x, 37 | "y" : fieldb_y, 38 | "z" : fieldb_z} 39 | } 40 | 41 | fields = pyDive.structured(treeOfArrays) 42 | 43 | half = fields[::2]["FieldE/x"] 44 | # equivalent to 45 | half = fields["FieldE/x"][::2] 46 | # equivalent to 47 | half = fields["FieldE"]["x"][::2] 48 | # equivalent to 49 | half = fields["FieldE"][::2]["x"] 50 | 51 | # equivalent to 52 | half = fields.FieldE.x[::2] 53 | 54 | The example shows that in fact *fields* can be treated as an array-of-structures 55 | **or** a structure-of-arrays depending on what is more appropriate. 56 | 57 | The goal is to make the virtual *array-of-structs*-object look like a real array. Therefore 58 | every method call or operation is forwarded to the individual arrays.:: 59 | 60 | new_field = fields.FieldE.astype(np.int) + fields.FieldB.astype(np.float) 61 | 62 | Here the forwarded method calls are ``astype`` and ``__add__``. 63 | """ 64 | 65 | import sys 66 | import os 67 | # check whether this code is executed on target or not 68 | onTarget = os.environ.get("onTarget", 'False') 69 | if onTarget == 'False': 70 | import IPParallelClient as com 71 | from IPython.parallel import interactive 72 | import numpy as np 73 | 74 | def makeTree_fromTree(tree, expression): 75 | def traverseTree(outTree, inTree): 76 | for key, value in inTree.items(): 77 | if type(value) is dict: 78 | outTree[key] = {} 79 | traverseTree(outTree[key], value) 80 | else: 81 | outTree[key] = expression(value) 82 | outTree = {} 83 | traverseTree(outTree, tree) 84 | return outTree 85 | 86 | def makeTree_fromTwoTrees(treeA, treeB, expression): 87 | def traverseTrees(outTree, inTreeA, inTreeB): 88 | for key, valueA in inTreeA.items(): 89 | valueB = inTreeB[key] 90 | if type(valueA) is dict: 91 | outTree[key] = {} 92 | traverseTrees(outTree[key], valueA, valueB) 93 | else: 94 | outTree[key] = expression(valueA, valueB) 95 | outTree = {} 96 | traverseTrees(outTree, treeA, treeB) 97 | return outTree 98 | 99 | def visitTwoTrees(treeA, treeB, visitor): 100 | # traverse trees 101 | for key, valueA in treeA.items(): 102 | valueB = treeB[key] 103 | if type(valueA) is dict: 104 | visitTwoTrees(valueA, valueB, visitor) 105 | else: 106 | visitor(treeA, treeB, key, valueA, valueB) 107 | return treeA, treeB 108 | 109 | # generator object which iterates the tree's leafs 110 | def treeItems(tree): 111 | items = list(tree.items()) 112 | while items: 113 | key, value = items.pop() 114 | if type(value) is dict: 115 | items += list(value.items()) 116 | else: 117 | yield key, value 118 | 119 | class ForeachLeafCall(object): 120 | def __init__(self, tree, op): 121 | self.tree = tree 122 | self.op = op 123 | 124 | def __call__(self, *args, **kwargs): 125 | def apply_unary(a): 126 | f = getattr(a, self.op) 127 | return f(*args, **kwargs) 128 | def apply_binary(a, b): 129 | f = getattr(a, self.op) 130 | return f(b, *args[1:], **kwargs) 131 | 132 | if args and type(args[0]).__name__ == "VirtualArrayOfStructs": 133 | structOfArrays = makeTree_fromTwoTrees(self.tree, args[0].structOfArrays, apply_binary) 134 | else: 135 | structOfArrays = makeTree_fromTree(self.tree, apply_unary) 136 | return structured(structOfArrays) 137 | 138 | 139 | arrayOfStructs_id = 0 140 | 141 | class VirtualArrayOfStructs(object): 142 | def __init__(self, structOfArrays): 143 | items = [item for item in treeItems(structOfArrays)] 144 | self.firstArray = items[0][1] 145 | self.arraytype = type(self.firstArray) 146 | assert all(type(a) == self.arraytype for name, a in items),\ 147 | "all arrays in 'structOfArrays' must be of the same type: " +\ 148 | str({name : type(a) for name, a in items}) 149 | assert all(a.shape == self.firstArray.shape for name, a in items),\ 150 | "all arrays in 'structOfArrays' must have the same shape: " +\ 151 | str({name : a.shape for name, a in items}) 152 | 153 | self.shape = self.firstArray.shape 154 | self.dtype = makeTree_fromTree(structOfArrays, lambda a: a.dtype) 155 | self.nbytes = sum(a.nbytes for name, a in items) 156 | self.structOfArrays = structOfArrays 157 | self.has_local_instance = False 158 | 159 | def __del__(self): 160 | if onTarget == 'False' and self.has_local_instance: 161 | # delete remote structured object 162 | self.view.execute('del %s' % self.name, targets=self.target_ranks) 163 | 164 | def __getattr__(self, name): 165 | if hasattr(self.firstArray, name): 166 | assert hasattr(getattr(self.firstArray, name), "__call__"),\ 167 | "Unlike method access, attribute access of individual arrays is not supported." 168 | return ForeachLeafCall(self.structOfArrays, name) 169 | 170 | return self[name] 171 | 172 | def __special_operation__(self, op, *args): 173 | return ForeachLeafCall(self.structOfArrays, op)(*args) 174 | 175 | def __repr__(self): 176 | # if arrays are distributed create a local representation of this object on engine 177 | if onTarget == 'False' and not self.has_local_instance and hasattr(self.firstArray, "target_ranks"): 178 | items = [item for item in treeItems(self.structOfArrays)] 179 | assert all(self.firstArray.is_distributed_like(a) for name, a in items),\ 180 | "Cannot create a local virtual array-of-structs because not all arrays are distributed equally." 181 | 182 | self.distaxes = self.firstArray.distaxes 183 | self.target_offsets = self.firstArray.target_offsets 184 | self.target_ranks = self.firstArray.target_ranks 185 | view = com.getView() 186 | self.view = view 187 | 188 | # generate a unique variable name used on target representing this instance 189 | global arrayOfStructs_id 190 | self.name = 'arrayOfStructsObj' + str(arrayOfStructs_id) 191 | arrayOfStructs_id += 1 192 | 193 | # create a VirtualArrayOfStructs object containing the local arrays on the targets in use 194 | names_tree = makeTree_fromTree(self.structOfArrays, lambda a: repr(a)) 195 | 196 | view.push({'names_tree' : names_tree}, targets=self.target_ranks) 197 | 198 | view.execute('''\ 199 | structOfArrays = structured.makeTree_fromTree(names_tree, lambda a_name: globals()[a_name]) 200 | %s = structured.structured(structOfArrays)''' % self.name,\ 201 | targets=self.target_ranks) 202 | 203 | self.has_local_instance = True 204 | 205 | return self.name 206 | 207 | def __str__(self): 208 | def printTree(tree, indent, result): 209 | for key, value in tree.items(): 210 | if type(value) is dict: 211 | result += indent + key + ":\n" 212 | result = printTree(tree[key], indent + " ", result) 213 | else: 214 | result += indent + key + " -> " + str(value.dtype) + "\n" 215 | return result 216 | 217 | result = "VirtualArrayOfStructs:\n" 219 | return printTree(self.structOfArrays, " ", result) 220 | 221 | def __getitem__(self, args): 222 | # component access 223 | # ---------------- 224 | if type(args) is str: 225 | node = self.structOfArrays # root node 226 | path = args.split('/') 227 | for node_name in path: 228 | node = node[node_name] 229 | if type(node) is dict: 230 | # node 231 | return structured(node) 232 | else: 233 | # leaf 234 | return node 235 | 236 | # slicing 237 | # ------- 238 | result = makeTree_fromTree(self.structOfArrays, lambda a: a[args]) 239 | 240 | # if args is a list of indices then return a single data value tree 241 | if type(args) not in (list, tuple): 242 | args = (args,) 243 | if all(type(arg) is int for arg in args): 244 | return result 245 | 246 | return structured(result) 247 | 248 | def __setitem__(self, args, other): 249 | # component access 250 | # ---------------- 251 | if type(args) is str: 252 | node = self.structOfArrays # root node 253 | path = args.split('/') 254 | for node_name in path[:-1]: 255 | node = node[node_name] 256 | last_node_name = path[-1] 257 | if type(node[last_node_name]) is dict: 258 | # node 259 | def doArrayAssignment(treeA, treeB, name, arrayA, arrayB): 260 | treeA[name] = arrayB 261 | 262 | visitTwoTrees(node[last_node_name], other.structOfArrays, doArrayAssignment) 263 | else: 264 | # leaf 265 | node[last_node_name] = other 266 | return 267 | 268 | # slicing 269 | # ------- 270 | def doArrayAssignmentWithSlice(treeA, treeB, name, arrayA, arrayB): 271 | arrayA[args] = arrayB 272 | 273 | visitTwoTrees(self.structOfArrays, other.structOfArrays, doArrayAssignmentWithSlice) 274 | 275 | # Add special methods like "__add__", "__sub__", ... that call __special_operation__ 276 | # forwarding them to the individual arrays. 277 | # All ordinary methods are forwarded by __getattr__ 278 | 279 | binary_ops = ["add", "sub", "mul", "floordiv", "div", "mod", "pow", "lshift", "rshift", "and", "xor", "or"] 280 | 281 | binary_iops = ["__i" + op + "__" for op in binary_ops] 282 | binary_rops = ["__r" + op + "__" for op in binary_ops] 283 | binary_ops = ["__" + op + "__" for op in binary_ops] 284 | unary_ops = ["__neg__", "__pos__", "__abs__", "__invert__", "__complex__", "__int__", "__long__", "__float__", "__oct__", "__hex__"] 285 | comp_ops = ["__lt__", "__le__", "__eq__", "__ne__", "__ge__", "__gt__"] 286 | 287 | make_special_op = lambda op: lambda self, *args: self.__special_operation__(op, *args) 288 | 289 | special_ops_dict = {op : make_special_op(op) for op in binary_ops + binary_rops + unary_ops + comp_ops} 290 | 291 | from types import MethodType 292 | 293 | for name, func in special_ops_dict.items(): 294 | setattr(VirtualArrayOfStructs, name, MethodType(func, None, VirtualArrayOfStructs)) 295 | 296 | 297 | def structured(structOfArrays): 298 | """Convert a *structure-of-arrays* into a virtual *array-of-structures*. 299 | 300 | :param structOfArrays: tree-like (dict-of-dicts) dictionary of arrays. 301 | :raises AssertionError: if the *arrays-types* do not match. Datatypes may differ. 302 | :raises AssertionError: if the shapes do not match. 303 | :return: Custom object representing a virtual array whose elements have the same tree-like structure 304 | as *structOfArrays*. 305 | """ 306 | 307 | return VirtualArrayOfStructs(structOfArrays) 308 | -------------------------------------------------------------------------------- /pyDive/test/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2014 Heiko Burau 3 | 4 | This file is part of pyDive. 5 | 6 | pyDive is free software: you can redistribute it and/or modify 7 | it under the terms of of either the GNU General Public License or 8 | the GNU Lesser General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | pyDive is distributed in the hope that it will be useful, 12 | but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | GNU General Public License and the GNU Lesser General Public License 15 | for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | and the GNU Lesser General Public License along with pyDive. 19 | If not, see . 20 | """ 21 | __doc__ = None 22 | -------------------------------------------------------------------------------- /pyDive/test/conftest.py: -------------------------------------------------------------------------------- 1 | import pyDive 2 | import pytest 3 | import os 4 | 5 | @pytest.fixture(scope="session") 6 | def init_pyDive(request): 7 | pyDive.init(os.environ["IPP_PROFILE_NAME"]) -------------------------------------------------------------------------------- /pyDive/test/m2p_CIC.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ComputationalRadiationPhysics/pyDive/67d36420c2f1d59bef8633b2dea94b4ea4ab8ceb/pyDive/test/m2p_CIC.npy -------------------------------------------------------------------------------- /pyDive/test/p2m_CIC.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ComputationalRadiationPhysics/pyDive/67d36420c2f1d59bef8633b2dea94b4ea4ab8ceb/pyDive/test/p2m_CIC.npy -------------------------------------------------------------------------------- /pyDive/test/sample.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ComputationalRadiationPhysics/pyDive/67d36420c2f1d59bef8633b2dea94b4ea4ab8ceb/pyDive/test/sample.h5 -------------------------------------------------------------------------------- /pyDive/test/test_algorithm.py: -------------------------------------------------------------------------------- 1 | import pyDive 2 | import numpy as np 3 | import os 4 | 5 | input_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), "sample.h5") 6 | 7 | def test_map(init_pyDive): 8 | input_array = pyDive.h5.open(input_file, "fields") 9 | 10 | ref_array = input_array["fieldE/x"].load().gather()**2 \ 11 | + input_array["fieldE/y"].load().gather()**2 \ 12 | + input_array["fieldB/z"].load().gather()**2 13 | 14 | test_array = pyDive.empty(input_array.shape, dtype=input_array.dtype["fieldE"]["x"]) 15 | 16 | def energy(out, h5fields): 17 | fields = h5fields.load() 18 | out[:] = fields["fieldE/x"]**2 + fields["fieldE/y"]**2 + fields["fieldB/z"]**2 19 | 20 | pyDive.map(energy, test_array, input_array) 21 | 22 | assert np.array_equal(ref_array, test_array.gather()) 23 | 24 | def test_reduce(init_pyDive): 25 | input_array = pyDive.h5.open(input_file, "fields").load() 26 | 27 | energy_array = pyDive.empty(input_array.shape, dtype=input_array.dtype["fieldE"]["x"]) 28 | 29 | def energy(out, fields): 30 | out[:] = fields["fieldE/x"]**2 + fields["fieldE/y"]**2 + fields["fieldB/z"]**2 31 | 32 | pyDive.map(energy, energy_array, input_array) 33 | 34 | test_total = pyDive.reduce(energy_array, np.add) 35 | ref_total = np.add.reduce(energy_array.gather(), axis=None) 36 | 37 | diff = abs(ref_total - test_total) 38 | assert diff / ref_total < 1.0e-5 39 | 40 | def test_mapReduce(init_pyDive): 41 | input_array = pyDive.h5.open(input_file, "fields") 42 | 43 | ref_array = input_array["fieldE/x"].load().gather()**2 \ 44 | + input_array["fieldE/y"].load().gather()**2 \ 45 | + input_array["fieldB/z"].load().gather()**2 46 | ref_total = np.add.reduce(ref_array, axis=None) 47 | 48 | test_total = pyDive.mapReduce(\ 49 | lambda fields: fields["fieldE/x"].load()**2 + fields["fieldE/y"].load()**2 + fields["fieldB/z"].load()**2, 50 | np.add, input_array) 51 | 52 | diff = abs(ref_total - test_total) 53 | assert diff / ref_total < 1.0e-5 -------------------------------------------------------------------------------- /pyDive/test/test_cloned_ndarray.py: -------------------------------------------------------------------------------- 1 | import pyDive 2 | import numpy as np 3 | import pytest 4 | import random 5 | from pyDive import IPParallelClient as com 6 | 7 | sizes = ((1,), (5,), 8 | (5, 29), (64, 1), (64, 64), 9 | (1, 1, 1), (12, 37, 50)) 10 | 11 | def test_cloned_ndarray(init_pyDive): 12 | view = com.getView() 13 | for size in sizes: 14 | ref_array = np.arange(np.prod(size)) 15 | 16 | test_array = pyDive.cloned.empty(size, dtype=np.int) 17 | test_array[:] = ref_array 18 | 19 | assert np.array_equal(ref_array * len(view), test_array.sum()) -------------------------------------------------------------------------------- /pyDive/test/test_gpu.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import pyDive 4 | import numpy as np 5 | import random 6 | import pytest 7 | 8 | gpu_enabled = pytest.mark.skipif(not hasattr(pyDive, "gpu"), reason="pycuda not installed") 9 | 10 | sizes = ((1,), (5,), (29,), (64,), 11 | (1, 1), (1, 5), (5, 29), (64, 1), (64, 64), 12 | (1, 1, 1), (8, 8, 8), (1, 2, 3), (12, 37, 50), (64,64,64)) 13 | dtypes = (np.float,) 14 | 15 | @gpu_enabled 16 | def test_basics(init_pyDive): 17 | for size in sizes: 18 | for dtype in dtypes: 19 | ref = (np.random.rand(*size) * 100.0).astype(dtype) 20 | test_array_cpu = pyDive.array(ref) 21 | test_array = pyDive.gpu.empty(size, dtype, distaxes=range(len(size))) 22 | 23 | test_array[:] = test_array_cpu 24 | 25 | if all(s > 3 for s in size): 26 | slices = [slice(1, -2, None) for i in range(len(size))] 27 | 28 | a = test_array[slices] 29 | a = a + 1 30 | a = a**2 31 | a = a + a 32 | test_array[slices] = a 33 | 34 | b = ref[slices] 35 | b = b + 1 36 | b = b**2 37 | b = b + b 38 | ref[slices] = b 39 | 40 | np.testing.assert_array_almost_equal(test_array.to_cpu().gather(), ref) 41 | 42 | @gpu_enabled 43 | def test_interengine(init_pyDive): 44 | for size in sizes: 45 | for dtype in dtypes: 46 | ref = (np.random.rand(*size) * 100.0).astype(dtype) 47 | test_array_cpu = pyDive.array(ref) 48 | test_array = pyDive.gpu.empty(size, dtype, distaxes=range(len(size))) 49 | 50 | test_array[:] = test_array_cpu 51 | 52 | for i in range(len(size)): 53 | if size[i] < 5: continue 54 | 55 | slicesA = [slice(None)] * len(size) 56 | slicesB = list(slicesA) 57 | slicesA[i] = slice(0, 5) 58 | slicesB[i] = slice(-5, None) 59 | 60 | test_array[slicesA] = test_array[slicesB] 61 | ref[slicesA] = ref[slicesB] 62 | 63 | assert np.array_equal(test_array.to_cpu().gather(), ref) 64 | 65 | slicesA = [s/2 for s in size] 66 | slicesB = list(slicesA) 67 | slicesA[i] = slice(0, 5) 68 | slicesB[i] = slice(-5, None) 69 | 70 | test_array[slicesA] = test_array[slicesB] 71 | ref[slicesA] = ref[slicesB] 72 | 73 | assert np.array_equal(test_array.to_cpu().gather(), ref) 74 | 75 | @gpu_enabled 76 | def test_misc(init_pyDive): 77 | sizes = ((10,20,30), (30,20,10), (16,16,16), (16,32,48), (13,29,37)) 78 | 79 | def do_funny_stuff(a, b): 80 | a[1:,1:,1:] = b[:-1,:-1,:-1] 81 | b[1:,1:,1:] = a[1:,:-1,:-1] 82 | a[1:,1:,1:] = b[1:,1:,:-1] 83 | b[1:,1:,1:] = a[1:,:-1,1:] 84 | a[:-1,:-3,:-4] = b[1:,3:,4:] 85 | b[0,:,0] += a[-2,:,-2] 86 | a[4:-3,2:-1,5:-2] = b[4:-3,2:-1,5:-2] 87 | b[1:3,1:4,1:5] *= a[-3:-1,-4:-1,-5:-1] 88 | #a[1,2,3] -= b[3,2,1] 89 | b[:,:,0] = a[:,:,1] 90 | 91 | for size in sizes: 92 | cpu_a = (np.random.rand(*size) * 100.0).astype(np.int) 93 | cpu_b = (np.random.rand(*size) * 100.0).astype(np.int) 94 | 95 | gpu_a = pyDive.gpu.array(cpu_a) 96 | gpu_b = pyDive.gpu.array(cpu_b) 97 | 98 | do_funny_stuff(cpu_a, cpu_b) 99 | do_funny_stuff(gpu_a, gpu_b) 100 | 101 | assert np.array_equal(gpu_a.to_cpu(), cpu_a) 102 | assert np.array_equal(gpu_b.to_cpu(), cpu_b) 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | -------------------------------------------------------------------------------- /pyDive/test/test_h5_ndarray.py: -------------------------------------------------------------------------------- 1 | import pyDive 2 | import numpy as np 3 | import random 4 | import h5py as h5 5 | import os 6 | 7 | input_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), "sample.h5") 8 | 9 | def test_h5_ndarray1(init_pyDive): 10 | dataset = "particles/pos/x" 11 | 12 | test_array = pyDive.h5.open(input_file, dataset) 13 | 14 | ref_array = h5.File(input_file, "r")[dataset][:] 15 | 16 | assert np.array_equal(ref_array, test_array.load().gather()) 17 | 18 | def test_h5_ndarray2(init_pyDive): 19 | window = np.s_[31:201:3, 2:233:5] 20 | dataset = "/fields/fieldE/x" 21 | 22 | test_array = pyDive.h5.open(input_file, dataset, distaxes=1) 23 | 24 | ref_array = h5.File(input_file, "r")[dataset][window] 25 | 26 | assert np.array_equal(ref_array, test_array[window].load().gather()) 27 | 28 | def test_h5(init_pyDive): 29 | test_array = pyDive.h5.open(input_file, "particles/pos") 30 | 31 | ref_array_x = h5.File(input_file, "r")["particles/pos/x"][:] 32 | ref_array_y = h5.File(input_file, "r")["particles/pos/y"][:] 33 | 34 | assert np.array_equal(ref_array_x, test_array["x"].load().gather()) 35 | assert np.array_equal(ref_array_y, test_array["y"].load().gather()) 36 | assert np.array_equal(ref_array_x, test_array.load()["x"].gather()) 37 | assert np.array_equal(ref_array_y, test_array.load()["y"].gather()) 38 | -------------------------------------------------------------------------------- /pyDive/test/test_mappings.py: -------------------------------------------------------------------------------- 1 | import pyDive 2 | import numpy as np 3 | import os 4 | 5 | dirname = os.path.dirname(os.path.abspath(__file__)) 6 | input_file = os.path.join(dirname, "sample.h5") 7 | 8 | def test_particles2mesh(init_pyDive): 9 | shape = [256, 256] 10 | density = pyDive.cloned.zeros(shape) 11 | 12 | particles = pyDive.h5.open(input_file, "/particles") 13 | 14 | def particles2density(particles, density): 15 | particles = particles.load() 16 | total_pos = particles["cellidx"].astype(np.float32) + particles["pos"] 17 | 18 | # convert total_pos to an (N, 2) shaped array 19 | total_pos = np.hstack((total_pos["x"][:,np.newaxis], 20 | total_pos["y"][:,np.newaxis])) 21 | 22 | par_weighting = np.ones(particles.shape) 23 | import pyDive.mappings 24 | pyDive.mappings.particles2mesh(density, par_weighting, total_pos, pyDive.mappings.CIC) 25 | 26 | pyDive.map(particles2density, particles, density) 27 | 28 | test_density = density.sum() # add up all local copies 29 | 30 | ref_density = np.load(os.path.join(dirname, "p2m_CIC.npy")) 31 | 32 | np.testing.assert_array_almost_equal(ref_density, test_density) 33 | 34 | def test_mesh2particles(init_pyDive): 35 | particles = pyDive.h5.open(input_file, "/particles").load() 36 | field = pyDive.h5.open(input_file, "/fields/fieldB/z").load().gather() 37 | 38 | field_strengths = pyDive.empty(particles.shape) 39 | 40 | @pyDive.map 41 | def mesh2particles(field_strengths, particles, field): 42 | total_pos = particles["cellidx"].astype(np.float32) + particles["pos"] 43 | 44 | # convert total_pos to an (N, 2) shaped array 45 | total_pos = np.hstack((total_pos["x"][:,np.newaxis], 46 | total_pos["y"][:,np.newaxis])) 47 | 48 | import pyDive.mappings 49 | field_strengths[:] = pyDive.mappings.mesh2particles(field, total_pos, pyDive.mappings.CIC) 50 | 51 | mesh2particles(field_strengths, particles, field=field) 52 | 53 | ref_field_strengths = np.load(os.path.join(dirname, "m2p_CIC.npy")) 54 | 55 | assert np.array_equal(ref_field_strengths, field_strengths.gather()) 56 | 57 | -------------------------------------------------------------------------------- /pyDive/test/test_ndarray.py: -------------------------------------------------------------------------------- 1 | import pyDive 2 | import numpy as np 3 | import random 4 | 5 | sizes = ((1,), (5,), (29,), (64,), 6 | (1, 1), (1, 5), (5, 29), (64, 1), (64, 64), 7 | (1, 1, 1), (8, 8, 8), (1, 2, 3), (12, 37, 50), (64,64,64)) 8 | dtypes = (np.int,) 9 | 10 | def test_slicing(init_pyDive): 11 | for size in sizes: 12 | for dtype in dtypes: 13 | ref = (np.random.rand(*size) * 100.0).astype(dtype) 14 | 15 | for distaxis in range(len(size)): 16 | test_array = pyDive.empty(size, dtype, distaxis) 17 | test_array[:] = ref 18 | 19 | slices = [] 20 | for i in range(len(size)): 21 | start = size[i] / 3 22 | stop = size[i] - size[i] / 5 23 | step = 2 24 | slices.append(slice(start, stop, step)) 25 | 26 | assert np.array_equal(ref[slices], test_array[slices].gather()) 27 | 28 | slices = [] 29 | for i in range(len(size)): 30 | slices.append(slice(-5, None, None)) 31 | 32 | assert np.array_equal(ref[slices], test_array[slices].gather()) 33 | 34 | slices = [] 35 | for i in range(len(size)): 36 | slices.append(slice(0, 5, None)) 37 | 38 | assert np.array_equal(ref[slices], test_array[slices].gather()) 39 | 40 | # bitmask indexing 41 | bitmask = pyDive.array(np.random.rand(*size) > 0.5, distaxes=test_array.distaxes) 42 | assert ref[bitmask.gather()].shape == test_array[bitmask].shape 43 | # ordering can be distinct, thus merely check if sets are equal 44 | assert set(ref[bitmask.gather()]) == set(test_array[bitmask].gather()) 45 | 46 | ref2 = ref.copy() 47 | test_array2 = test_array.copy() 48 | ref2[bitmask.gather()] = 1 49 | test_array2[bitmask] = 1 50 | assert np.array_equal(ref2, test_array2.gather()) 51 | 52 | def test_multiple_axes(init_pyDive): 53 | for size in sizes: 54 | for dtype in dtypes: 55 | ref = (np.random.rand(*size) * 100.0).astype(dtype) 56 | 57 | for distaxes in [range(i+1) for i in range(len(size))]: 58 | test_array = pyDive.empty(size, dtype, distaxes) 59 | test_array[:] = ref 60 | 61 | slices = [] 62 | for i in range(len(size)): 63 | start = size[i] / 3 64 | stop = size[i] - size[i] / 5 65 | step = 2 66 | slices.append(slice(start, stop, step)) 67 | 68 | assert np.array_equal(ref[slices], test_array[slices].gather()) 69 | 70 | slices = [] 71 | for i in range(len(size)): 72 | slices.append(slice(-5, None, None)) 73 | 74 | assert np.array_equal(ref[slices], test_array[slices].gather()) 75 | 76 | slices = [] 77 | for i in range(len(size)): 78 | slices.append(slice(0, 5, None)) 79 | 80 | assert np.array_equal(ref[slices], test_array[slices].gather()) 81 | 82 | # bitmask indexing 83 | bitmask = pyDive.array(np.random.rand(*size) > 0.5, distaxes=test_array.distaxes) 84 | assert ref[bitmask.gather()].shape == test_array[bitmask].shape 85 | # ordering can be distinct, thus merely check if sets are equal 86 | assert set(ref[bitmask.gather()]) == set(test_array[bitmask].gather()) 87 | 88 | ref2 = ref.copy() 89 | test_array2 = test_array.copy() 90 | ref2[bitmask.gather()] = 1 91 | test_array2[bitmask] = 1 92 | assert np.array_equal(ref2, test_array2.gather()) 93 | 94 | def test_interengine(init_pyDive): 95 | for size in sizes: 96 | for dtype in dtypes: 97 | ref = (np.random.rand(*size) * 100.0).astype(dtype) 98 | 99 | for distaxis in range(len(size)): 100 | if size[distaxis] < 5: continue 101 | 102 | test_array = pyDive.empty(size, dtype, distaxis) 103 | test_array[:] = ref 104 | 105 | slicesA = [slice(None)] * len(size) 106 | slicesB = list(slicesA) 107 | slicesA[distaxis] = slice(0, 5) 108 | slicesB[distaxis] = slice(-5, None) 109 | 110 | test_array[slicesA] = test_array[slicesB] 111 | ref[slicesA] = ref[slicesB] 112 | 113 | assert np.array_equal(test_array.gather(), ref) 114 | 115 | slicesA = [s/2 for s in size] 116 | slicesB = list(slicesA) 117 | slicesA[distaxis] = slice(0, 5) 118 | slicesB[distaxis] = slice(-5, None) 119 | 120 | test_array[slicesA] = test_array[slicesB] 121 | ref[slicesA] = ref[slicesB] 122 | 123 | assert np.array_equal(test_array.gather(), ref) 124 | 125 | def test_interengine_multiple_axes(init_pyDive): 126 | for size in sizes: 127 | for dtype in dtypes: 128 | ref = (np.random.rand(*size) * 100.0).astype(dtype) 129 | 130 | for distaxesA in [range(i+1) for i in range(len(size))]: 131 | for distaxesB in [range(i,len(size)) for i in range(len(size))]: 132 | test_arrayA = pyDive.empty(size, dtype, distaxesA) 133 | test_arrayB = pyDive.empty(size, dtype, distaxesB) 134 | 135 | test_arrayA[:] = ref 136 | test_arrayB[:] = ref 137 | 138 | for distaxis in range(len(size)): 139 | if size[distaxis] < 5: continue 140 | 141 | slicesA = [slice(None)] * len(size) 142 | slicesB = list(slicesA) 143 | slicesA[distaxis] = slice(0, 5) 144 | slicesB[distaxis] = slice(-5, None) 145 | 146 | ref_sum = ref[slicesA] + ref[slicesB] 147 | test_array_sum = test_arrayA[slicesA] + test_arrayB[slicesB] 148 | 149 | assert np.array_equal(ref_sum, test_array_sum.gather()) 150 | 151 | def test_misc(init_pyDive): 152 | sizes = ((10,20,30), (30,20,10), (16,16,16), (16,32,48), (13,29,37)) 153 | 154 | def do_funny_stuff(a, b): 155 | a[1:,1:,1:] = b[:-1,:-1,:-1] 156 | b[1:,1:,1:] = a[1:,:-1,:-1] 157 | a[1:,1:,1:] = b[1:,1:,:-1] 158 | b[1:,1:,1:] = a[1:,:-1,1:] 159 | a[:-1,:-3,:-4] = b[1:,3:,4:] 160 | b[0,:,0] += a[-2,:,-2] 161 | a[4:-3,2:-1,5:-2] = b[4:-3,2:-1,5:-2] 162 | b[1:3,1:4,1:5] *= a[-3:-1,-4:-1,-5:-1] 163 | a[1,2,3] -= b[3,2,1] 164 | b[:,:,0] = a[:,:,1] 165 | 166 | for size in sizes: 167 | print "size", size 168 | np_a = (np.random.rand(*size) * 100.0).astype(np.int) 169 | np_b = (np.random.rand(*size) * 100.0).astype(np.int) 170 | 171 | pd_a = pyDive.array(np_a, distaxes=(0,1,2)) 172 | pd_b = pyDive.array(np_b, distaxes=(0,1,2)) 173 | 174 | do_funny_stuff(np_a, np_b) 175 | do_funny_stuff(pd_a, pd_b) 176 | 177 | assert np.array_equal(pd_a, np_a) 178 | assert np.array_equal(pd_b, np_b) -------------------------------------------------------------------------------- /pyDive/test/test_structured.py: -------------------------------------------------------------------------------- 1 | import pyDive 2 | import numpy as np 3 | 4 | def test_arrayOfStructs(init_pyDive): 5 | fieldE_x = np.random.rand(100, 100) 6 | fieldE_y = np.random.rand(100, 100) 7 | fieldB_z = np.random.rand(100, 100) 8 | 9 | fields = {"fieldE": {"x" : fieldE_x, "y" : fieldE_y}, "fieldB" : {"z" : fieldB_z}} 10 | fields = pyDive.structured(fields) 11 | 12 | assert np.array_equal(fieldB_z, fields["fieldB/z"]) 13 | assert np.array_equal(fieldB_z, fields["fieldB"]["z"]) 14 | assert np.array_equal(fieldB_z, fields["fieldB"][:]["z"]) 15 | assert np.array_equal(fieldB_z, fields[:]["fieldB/z"]) 16 | assert np.array_equal(fieldB_z**2, (fields**2)["fieldB/z"]) 17 | assert np.array_equal(fieldB_z**2, (fields**2).fieldB.z) -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | ipython>=2.0.0 2 | numpy>=1.8.1 3 | mpi4py>=1.3.1 4 | h5py>=2.3.0 5 | psutil>=2.1.1 6 | pyzmq>=14.3.0 7 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2014 Heiko Burau 3 | 4 | This file is part of pyDive. 5 | 6 | pyDive is free software: you can redistribute it and/or modify 7 | it under the terms of of either the GNU General Public License or 8 | the GNU Lesser General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | pyDive is distributed in the hope that it will be useful, 12 | but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | GNU General Public License and the GNU Lesser General Public License 15 | for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | and the GNU Lesser General Public License along with pyDive. 19 | If not, see . 20 | """ 21 | 22 | from __future__ import print_function 23 | from setuptools import setup, find_packages 24 | from setuptools.command.test import test as TestCommand 25 | import io 26 | import codecs 27 | import os 28 | import sys 29 | import subprocess 30 | import time 31 | 32 | import pyDive 33 | 34 | here = os.path.abspath(os.path.dirname(__file__)) 35 | 36 | def read(*filenames, **kwargs): 37 | encoding = kwargs.get('encoding', 'utf-8') 38 | sep = kwargs.get('sep', '\n') 39 | buf = [] 40 | for filename in filenames: 41 | with io.open(filename, encoding=encoding) as f: 42 | buf.append(f.read()) 43 | return sep.join(buf) 44 | 45 | long_description = read('README.md')#, 'CHANGES.txt') 46 | requirements = [line.rstrip('\n') for line in open('requirements.txt')] 47 | 48 | class PyTest(TestCommand): 49 | def finalize_options(self): 50 | TestCommand.finalize_options(self) 51 | self.test_args = [] 52 | self.test_suite = True 53 | 54 | def run_tests(self): 55 | profile_name = raw_input("Name of your IPython-parallel profile you want to run the tests with: ") 56 | n_engines = raw_input("Number of engines: ") 57 | 58 | # start ipcluster 59 | print("Waiting for engines to start...") 60 | subprocess.Popen(("ipcluster", "start", "--n=%s" % n_engines,\ 61 | "--profile=%s" % profile_name)) 62 | time.sleep(35) 63 | 64 | import pytest 65 | # set profile name as environment variable 66 | os.environ["IPP_PROFILE_NAME"] = profile_name 67 | errcode = pytest.main(self.test_args) 68 | 69 | # stop ipcluster 70 | subprocess.Popen(("ipcluster", "stop", "--profile=%s" % profile_name)).wait() 71 | 72 | sys.exit(errcode) 73 | 74 | setup( 75 | name='pyDive', 76 | version=pyDive.__version__, 77 | url='http://github.com/ComputationalRadiationPhysics/pyDive', 78 | license='GNU Affero General Public License v3', 79 | author='Heiko Burau', 80 | tests_require=['pytest'], 81 | install_requires=requirements, 82 | setup_requires=requirements, 83 | cmdclass={'test': PyTest}, 84 | author_email='h.burau@hzdr.de', 85 | description='Distributed Interactive Visualization and Exploration of large datasets', 86 | long_description=long_description, 87 | packages=['pyDive', 'pyDive/distribution', 'pyDive/arrays', 'pyDive/arrays/local', 'pyDive/cloned_ndarray', 'pyDive/test'], 88 | include_package_data=True, 89 | platforms='any', 90 | #test_suite='sandman.test.test_sandman', 91 | classifiers = [ 92 | 'Programming Language :: Python', 93 | 'Development Status :: 4 - Beta', 94 | 'Natural Language :: English', 95 | 'Environment :: Console', 96 | 'Environment :: Web Environment', 97 | 'Intended Audience :: Science/Research', 98 | 'License :: OSI Approved :: GNU Affero General Public License v3', 99 | 'Operating System :: OS Independent', 100 | 'Topic :: Scientific/Engineering :: Information Analysis' 101 | ], 102 | extras_require={'testing': ['pytest'],} 103 | ) 104 | --------------------------------------------------------------------------------