├── .gitignore
├── CHANGELOG.md
├── COPYING
├── COPYING.LESSER
├── MANIFEST.in
├── README.md
├── docs
├── Makefile
├── conf.py
├── decomposition.png
├── density.png
├── index.rst
├── make.bat
├── modules.rst
├── pyDive.adios.rst
├── pyDive.cloned_ndarray.rst
├── pyDive.gpu.rst
├── pyDive.h5.rst
├── pyDive.ndarray.rst
├── reference.rst
├── sample.h5
├── spectrum.png
├── start.rst
└── tutorial.rst
├── pyDive
├── IPParallelClient.py
├── __init__.py
├── algorithm.py
├── arrays
│ ├── __init__.py
│ ├── ad_ndarray.py
│ ├── gpu_ndarray.py
│ ├── h5_ndarray.py
│ ├── local
│ │ ├── __init__.py
│ │ ├── ad_ndarray.py
│ │ ├── gpu_ndarray.py
│ │ └── h5_ndarray.py
│ └── ndarray.py
├── cloned_ndarray
│ ├── __init__.py
│ ├── cloned_ndarray.py
│ └── factories.py
├── distribution
│ ├── __init__.py
│ ├── helper.py
│ ├── interengine.py
│ ├── multiple_axes.py
│ └── multiple_axes_funcs.py
├── fragment.py
├── mappings.py
├── picongpu.py
├── pyDive.py
├── structured.py
└── test
│ ├── __init__.py
│ ├── conftest.py
│ ├── m2p_CIC.npy
│ ├── p2m_CIC.npy
│ ├── sample.h5
│ ├── test_algorithm.py
│ ├── test_cloned_ndarray.py
│ ├── test_gpu.py
│ ├── test_h5_ndarray.py
│ ├── test_mappings.py
│ ├── test_ndarray.py
│ └── test_structured.py
├── requirements.txt
└── setup.py
/.gitignore:
--------------------------------------------------------------------------------
1 | # Byte-compiled / optimized / DLL files
2 | __pycache__/
3 | *.py[cod~]
4 |
5 | # C extensions
6 | *.so
7 |
8 | # Distribution / packaging
9 | .Python
10 | env/
11 | bin/
12 | build/
13 | develop-eggs/
14 | dist/
15 | eggs/
16 | lib/
17 | lib64/
18 | parts/
19 | sdist/
20 | var/
21 | *.egg-info/
22 | .installed.cfg
23 | *.egg
24 |
25 | # Installer logs
26 | pip-log.txt
27 | pip-delete-this-directory.txt
28 |
29 | # Unit test / coverage reports
30 | htmlcov/
31 | .tox/
32 | .coverage
33 | .cache
34 | nosetests.xml
35 | coverage.xml
36 |
37 | # Translations
38 | *.mo
39 |
40 | # Mr Developer
41 | .mr.developer.cfg
42 | .project
43 | .pydevproject
44 |
45 | # Rope
46 | .ropeproject
47 |
48 | # Django stuff:
49 | *.log
50 | *.pot
51 |
52 | # Sphinx documentation
53 | docs/_build/
54 |
55 | *.nja*
56 | *.rst~*
57 | *.md~*
58 | pyDive/debug.py
59 | autorsync*
60 |
--------------------------------------------------------------------------------
/CHANGELOG.md:
--------------------------------------------------------------------------------
1 | Changelog for pyDive
2 | ====================
3 |
4 | 1.2.2
5 | -----
6 | **Date:** 2015-07-10
7 |
8 | - add `setup_requires` to setup.py
9 |
10 | 1.2.1
11 | -----
12 | **Date:** 2015-07-10
13 |
14 | - add `arrays.local` package to setup.py
15 |
16 | 1.2
17 | ---
18 | **Date:** 2015-06-22
19 |
20 | **New Features:**
21 | - arrays can be distributed along multiple axes
22 | - new distributed array: `pyDive.arrays.gpu_ndarray`. Distributed version of a pycuda array with extra support
23 | for non-contiguous memory (needed for working with sliced arrays).
24 | - `gather()` is called implicitly when the user wants to access an attribute, e.g. datapointer, which only
25 | the local array has.
26 | - implement indexing by bitmask
27 |
28 | **Bug Fixes:**
29 | - integer indices were not wrapped in `__getitem__()`
30 | - fix exception in picongpu.py if dataset has no attributes
31 |
32 | **Misc**-
33 | - `distaxes` parameter defaults to 'all'
34 | - all local array classes are located in pyDive/arrays/local
35 | - rename "arrayOfStructs" module to "structured"
36 |
37 | 1.1
38 | ---
39 | **Date:** 2015-03-29
40 |
41 | **New Features:**
42 | - adios support
43 | - implement `pyDive.fragment`
44 | - distributed array classes are auto-generated from local array class.
45 |
46 | **Misc:**
47 | - restructure project files
48 | - optimize `pyDive.ndarray`
49 |
50 | 1.0.2
51 | -----
52 | **Date:** 2014-11-06
53 |
54 | - add MANIFEST.in
55 |
56 | 1.0.1
57 | -----
58 | **Date:** 2014-11-06
59 |
60 | - fix setup.py
61 |
62 | 1.0
63 | ---
64 | **Date:** 2014-11-05
65 |
66 | Development status: **beta**
67 |
--------------------------------------------------------------------------------
/COPYING.LESSER:
--------------------------------------------------------------------------------
1 | GNU LESSER GENERAL PUBLIC LICENSE
2 | Version 3, 29 June 2007
3 |
4 | Copyright (C) 2007 Free Software Foundation, Inc.
5 | Everyone is permitted to copy and distribute verbatim copies
6 | of this license document, but changing it is not allowed.
7 |
8 |
9 | This version of the GNU Lesser General Public License incorporates
10 | the terms and conditions of version 3 of the GNU General Public
11 | License, supplemented by the additional permissions listed below.
12 |
13 | 0. Additional Definitions.
14 |
15 | As used herein, "this License" refers to version 3 of the GNU Lesser
16 | General Public License, and the "GNU GPL" refers to version 3 of the GNU
17 | General Public License.
18 |
19 | "The Library" refers to a covered work governed by this License,
20 | other than an Application or a Combined Work as defined below.
21 |
22 | An "Application" is any work that makes use of an interface provided
23 | by the Library, but which is not otherwise based on the Library.
24 | Defining a subclass of a class defined by the Library is deemed a mode
25 | of using an interface provided by the Library.
26 |
27 | A "Combined Work" is a work produced by combining or linking an
28 | Application with the Library. The particular version of the Library
29 | with which the Combined Work was made is also called the "Linked
30 | Version".
31 |
32 | The "Minimal Corresponding Source" for a Combined Work means the
33 | Corresponding Source for the Combined Work, excluding any source code
34 | for portions of the Combined Work that, considered in isolation, are
35 | based on the Application, and not on the Linked Version.
36 |
37 | The "Corresponding Application Code" for a Combined Work means the
38 | object code and/or source code for the Application, including any data
39 | and utility programs needed for reproducing the Combined Work from the
40 | Application, but excluding the System Libraries of the Combined Work.
41 |
42 | 1. Exception to Section 3 of the GNU GPL.
43 |
44 | You may convey a covered work under sections 3 and 4 of this License
45 | without being bound by section 3 of the GNU GPL.
46 |
47 | 2. Conveying Modified Versions.
48 |
49 | If you modify a copy of the Library, and, in your modifications, a
50 | facility refers to a function or data to be supplied by an Application
51 | that uses the facility (other than as an argument passed when the
52 | facility is invoked), then you may convey a copy of the modified
53 | version:
54 |
55 | a) under this License, provided that you make a good faith effort to
56 | ensure that, in the event an Application does not supply the
57 | function or data, the facility still operates, and performs
58 | whatever part of its purpose remains meaningful, or
59 |
60 | b) under the GNU GPL, with none of the additional permissions of
61 | this License applicable to that copy.
62 |
63 | 3. Object Code Incorporating Material from Library Header Files.
64 |
65 | The object code form of an Application may incorporate material from
66 | a header file that is part of the Library. You may convey such object
67 | code under terms of your choice, provided that, if the incorporated
68 | material is not limited to numerical parameters, data structure
69 | layouts and accessors, or small macros, inline functions and templates
70 | (ten or fewer lines in length), you do both of the following:
71 |
72 | a) Give prominent notice with each copy of the object code that the
73 | Library is used in it and that the Library and its use are
74 | covered by this License.
75 |
76 | b) Accompany the object code with a copy of the GNU GPL and this license
77 | document.
78 |
79 | 4. Combined Works.
80 |
81 | You may convey a Combined Work under terms of your choice that,
82 | taken together, effectively do not restrict modification of the
83 | portions of the Library contained in the Combined Work and reverse
84 | engineering for debugging such modifications, if you also do each of
85 | the following:
86 |
87 | a) Give prominent notice with each copy of the Combined Work that
88 | the Library is used in it and that the Library and its use are
89 | covered by this License.
90 |
91 | b) Accompany the Combined Work with a copy of the GNU GPL and this license
92 | document.
93 |
94 | c) For a Combined Work that displays copyright notices during
95 | execution, include the copyright notice for the Library among
96 | these notices, as well as a reference directing the user to the
97 | copies of the GNU GPL and this license document.
98 |
99 | d) Do one of the following:
100 |
101 | 0) Convey the Minimal Corresponding Source under the terms of this
102 | License, and the Corresponding Application Code in a form
103 | suitable for, and under terms that permit, the user to
104 | recombine or relink the Application with a modified version of
105 | the Linked Version to produce a modified Combined Work, in the
106 | manner specified by section 6 of the GNU GPL for conveying
107 | Corresponding Source.
108 |
109 | 1) Use a suitable shared library mechanism for linking with the
110 | Library. A suitable mechanism is one that (a) uses at run time
111 | a copy of the Library already present on the user's computer
112 | system, and (b) will operate properly with a modified version
113 | of the Library that is interface-compatible with the Linked
114 | Version.
115 |
116 | e) Provide Installation Information, but only if you would otherwise
117 | be required to provide such information under section 6 of the
118 | GNU GPL, and only to the extent that such information is
119 | necessary to install and execute a modified version of the
120 | Combined Work produced by recombining or relinking the
121 | Application with a modified version of the Linked Version. (If
122 | you use option 4d0, the Installation Information must accompany
123 | the Minimal Corresponding Source and Corresponding Application
124 | Code. If you use option 4d1, you must provide the Installation
125 | Information in the manner specified by section 6 of the GNU GPL
126 | for conveying Corresponding Source.)
127 |
128 | 5. Combined Libraries.
129 |
130 | You may place library facilities that are a work based on the
131 | Library side by side in a single library together with other library
132 | facilities that are not Applications and are not covered by this
133 | License, and convey such a combined library under terms of your
134 | choice, if you do both of the following:
135 |
136 | a) Accompany the combined library with a copy of the same work based
137 | on the Library, uncombined with any other library facilities,
138 | conveyed under the terms of this License.
139 |
140 | b) Give prominent notice with the combined library that part of it
141 | is a work based on the Library, and explaining where to find the
142 | accompanying uncombined form of the same work.
143 |
144 | 6. Revised Versions of the GNU Lesser General Public License.
145 |
146 | The Free Software Foundation may publish revised and/or new versions
147 | of the GNU Lesser General Public License from time to time. Such new
148 | versions will be similar in spirit to the present version, but may
149 | differ in detail to address new problems or concerns.
150 |
151 | Each version is given a distinguishing version number. If the
152 | Library as you received it specifies that a certain numbered version
153 | of the GNU Lesser General Public License "or any later version"
154 | applies to it, you have the option of following the terms and
155 | conditions either of that published version or of any later version
156 | published by the Free Software Foundation. If the Library as you
157 | received it does not specify a version number of the GNU Lesser
158 | General Public License, you may choose any version of the GNU Lesser
159 | General Public License ever published by the Free Software Foundation.
160 |
161 | If the Library as you received it specifies that a proxy can decide
162 | whether future versions of the GNU Lesser General Public License shall
163 | apply, that proxy's public statement of acceptance of any version is
164 | permanent authorization for you to choose that version for the
165 | Library.
166 |
--------------------------------------------------------------------------------
/MANIFEST.in:
--------------------------------------------------------------------------------
1 | include *.md
2 | include *.txt
3 | include COPYING*
4 | include pyDive/test/*.h5 pyDive/test/*.npy
5 | include docs/*
6 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | pyDive
2 | ======
3 |
4 | Distributed Interactive Visualization and Exploration of large datasets.
5 |
6 | ## What is pyDive?
7 |
8 | Use pyDive to work with homogeneous, n-dimensional arrays that are too big to fit into your local machine's memory.
9 | pyDive provides containers whose elements are distributed across a cluster or stored in
10 | a large hdf5/adios-file if the cluster is still too small. All computation and data-access is then done in parallel by the cluster nodes in the background.
11 | If you feel like working with [numpy](http://www.numpy.org) arrays pyDive has reached the goal!
12 |
13 | pyDive is developed and maintained by the **[Junior Group Computational Radiation Physics](http://www.hzdr.de/db/Cms?pNid=132&pOid=30354)**
14 | at the [Institute for Radiation Physics](http://www.hzdr.de/db/Cms?pNid=132)
15 | at [HZDR](http://www.hzdr.de/).
16 |
17 | **Features:**
18 | - Since all cluster management is given to [IPython.parallel](http://ipython.org/ipython-doc/dev/parallel/) you can take your
19 | existing profiles for pyDive. No further cluster configuration needed.
20 | - Save bandwidth by slicing an array in parallel on disk first before loading it into main memory!
21 | - GPU-cluster array available thanks to [pycuda](http://mathema.tician.de/software/pycuda/) with additional support for non-contiguous memory.
22 | - As all of pyDive's distributed array types are auto-generated from local arrays like numpy, hdf5, pycuda, etc...
23 | you can easily make your own local array classes distributed too.
24 |
25 | ## Dive in!
26 |
27 | ```python
28 | import pyDive
29 | pyDive.init(profile='mpi')
30 |
31 | h5field = pyDive.h5.open("myData.h5", "myDataset", distaxes=(0,1))
32 | ones = pyDive.ones_like(h5field)
33 |
34 | # Distribute file i/o and computation across the cluster
35 | h5field[::10,:] = h5field[::10,:].load() + 5.0 * ones[::10,:]
36 | ```
37 |
38 | ## Documentation
39 |
40 | In our [Online Documentation](http://ComputationalRadiationPhysics.github.io/pyDive/), [pdf](http://ComputationalRadiationPhysics.github.io/pyDive/pyDive.pdf) you can find
41 | detailed information on all interfaces as well as some [Tutorials](http://computationalradiationphysics.github.io/pyDive/tutorial.html)
42 | and a [Quickstart](http://computationalradiationphysics.github.io/pyDive/start.html).
43 |
44 | ## Software License
45 |
46 | pyDive is licensed under the **GPLv3+ and LGPLv3+** (it is *dual licensed*).
47 | Licences can be found in [GPL](COPYING) or [LGPL](COPYING.LESSER), respectively.
48 |
--------------------------------------------------------------------------------
/docs/Makefile:
--------------------------------------------------------------------------------
1 | # Makefile for Sphinx documentation
2 | #
3 |
4 | # You can set these variables from the command line.
5 | SPHINXOPTS =
6 | SPHINXBUILD = sphinx-build
7 | PAPER =
8 | BUILDDIR = _build
9 |
10 | # User-friendly check for sphinx-build
11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/)
13 | endif
14 |
15 | # Internal variables.
16 | PAPEROPT_a4 = -D latex_paper_size=a4
17 | PAPEROPT_letter = -D latex_paper_size=letter
18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
19 | # the i18n builder cannot share the environment and doctrees with the others
20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
21 |
22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext
23 |
24 | help:
25 | @echo "Please use \`make ' where is one of"
26 | @echo " html to make standalone HTML files"
27 | @echo " dirhtml to make HTML files named index.html in directories"
28 | @echo " singlehtml to make a single large HTML file"
29 | @echo " pickle to make pickle files"
30 | @echo " json to make JSON files"
31 | @echo " htmlhelp to make HTML files and a HTML help project"
32 | @echo " qthelp to make HTML files and a qthelp project"
33 | @echo " devhelp to make HTML files and a Devhelp project"
34 | @echo " epub to make an epub"
35 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
36 | @echo " latexpdf to make LaTeX files and run them through pdflatex"
37 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx"
38 | @echo " text to make text files"
39 | @echo " man to make manual pages"
40 | @echo " texinfo to make Texinfo files"
41 | @echo " info to make Texinfo files and run them through makeinfo"
42 | @echo " gettext to make PO message catalogs"
43 | @echo " changes to make an overview of all changed/added/deprecated items"
44 | @echo " xml to make Docutils-native XML files"
45 | @echo " pseudoxml to make pseudoxml-XML files for display purposes"
46 | @echo " linkcheck to check all external links for integrity"
47 | @echo " doctest to run all doctests embedded in the documentation (if enabled)"
48 |
49 | clean:
50 | rm -rf $(BUILDDIR)/*
51 |
52 | html:
53 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
54 | @echo
55 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
56 |
57 | dirhtml:
58 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
59 | @echo
60 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
61 |
62 | singlehtml:
63 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
64 | @echo
65 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
66 |
67 | pickle:
68 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
69 | @echo
70 | @echo "Build finished; now you can process the pickle files."
71 |
72 | json:
73 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
74 | @echo
75 | @echo "Build finished; now you can process the JSON files."
76 |
77 | htmlhelp:
78 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
79 | @echo
80 | @echo "Build finished; now you can run HTML Help Workshop with the" \
81 | ".hhp project file in $(BUILDDIR)/htmlhelp."
82 |
83 | qthelp:
84 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
85 | @echo
86 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \
87 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
88 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/pyDive.qhcp"
89 | @echo "To view the help file:"
90 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/pyDive.qhc"
91 |
92 | devhelp:
93 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
94 | @echo
95 | @echo "Build finished."
96 | @echo "To view the help file:"
97 | @echo "# mkdir -p $$HOME/.local/share/devhelp/pyDive"
98 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/pyDive"
99 | @echo "# devhelp"
100 |
101 | epub:
102 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
103 | @echo
104 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub."
105 |
106 | latex:
107 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
108 | @echo
109 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
110 | @echo "Run \`make' in that directory to run these through (pdf)latex" \
111 | "(use \`make latexpdf' here to do that automatically)."
112 |
113 | latexpdf:
114 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
115 | @echo "Running LaTeX files through pdflatex..."
116 | $(MAKE) -C $(BUILDDIR)/latex all-pdf
117 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
118 |
119 | latexpdfja:
120 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
121 | @echo "Running LaTeX files through platex and dvipdfmx..."
122 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja
123 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
124 |
125 | text:
126 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
127 | @echo
128 | @echo "Build finished. The text files are in $(BUILDDIR)/text."
129 |
130 | man:
131 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
132 | @echo
133 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man."
134 |
135 | texinfo:
136 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
137 | @echo
138 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
139 | @echo "Run \`make' in that directory to run these through makeinfo" \
140 | "(use \`make info' here to do that automatically)."
141 |
142 | info:
143 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
144 | @echo "Running Texinfo files through makeinfo..."
145 | make -C $(BUILDDIR)/texinfo info
146 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
147 |
148 | gettext:
149 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
150 | @echo
151 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
152 |
153 | changes:
154 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
155 | @echo
156 | @echo "The overview file is in $(BUILDDIR)/changes."
157 |
158 | linkcheck:
159 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
160 | @echo
161 | @echo "Link check complete; look for any errors in the above output " \
162 | "or in $(BUILDDIR)/linkcheck/output.txt."
163 |
164 | doctest:
165 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
166 | @echo "Testing of doctests in the sources finished, look at the " \
167 | "results in $(BUILDDIR)/doctest/output.txt."
168 |
169 | xml:
170 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml
171 | @echo
172 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml."
173 |
174 | pseudoxml:
175 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml
176 | @echo
177 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml."
178 |
--------------------------------------------------------------------------------
/docs/conf.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | #
3 | # pyDive documentation build configuration file, created by
4 | # sphinx-quickstart on Wed Jul 23 15:53:56 2014.
5 | #
6 | # This file is execfile()d with the current directory set to its
7 | # containing dir.
8 | #
9 | # Note that not all possible configuration values are present in this
10 | # autogenerated file.
11 | #
12 | # All configuration values have a default; values that are commented out
13 | # serve to show the default.
14 |
15 | import sys
16 | import os
17 |
18 | # If extensions (or modules to document with autodoc) are in another directory,
19 | # add these directories to sys.path here. If the directory is relative to the
20 | # documentation root, use os.path.abspath to make it absolute, like shown here.
21 | #sys.path.insert(0, os.path.abspath('.'))
22 |
23 | # -- General configuration ------------------------------------------------
24 |
25 | # If your documentation needs a minimal Sphinx version, state it here.
26 | #needs_sphinx = '1.0'
27 |
28 | # Add any Sphinx extension module names here, as strings. They can be
29 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
30 | # ones.
31 | extensions = [
32 | 'sphinx.ext.autodoc',
33 | 'sphinx.ext.viewcode',
34 | ]
35 |
36 | # Add any paths that contain templates here, relative to this directory.
37 | templates_path = ['_templates']
38 |
39 | # The suffix of source filenames.
40 | source_suffix = '.rst'
41 |
42 | # The encoding of source files.
43 | #source_encoding = 'utf-8-sig'
44 |
45 | # The master toctree document.
46 | master_doc = 'index'
47 |
48 | # General information about the project.
49 | project = u'pyDive'
50 | copyright = u'2014,2015, Heiko Burau'
51 |
52 | # The version info for the project you're documenting, acts as replacement for
53 | # |version| and |release|, also used in various other places throughout the
54 | # built documents.
55 | #
56 | # The short X.Y version.
57 | version = '1.2.2'
58 | # The full version, including alpha/beta/rc tags.
59 | release = '1.2.2 beta'
60 |
61 | # The language for content autogenerated by Sphinx. Refer to documentation
62 | # for a list of supported languages.
63 | #language = None
64 |
65 | # There are two options for replacing |today|: either, you set today to some
66 | # non-false value, then it is used:
67 | #today = ''
68 | # Else, today_fmt is used as the format for a strftime call.
69 | #today_fmt = '%B %d, %Y'
70 |
71 | # List of patterns, relative to source directory, that match files and
72 | # directories to ignore when looking for source files.
73 | exclude_patterns = ['_build']
74 |
75 | # The reST default role (used for this markup: `text`) to use for all
76 | # documents.
77 | #default_role = None
78 |
79 | # If true, '()' will be appended to :func: etc. cross-reference text.
80 | #add_function_parentheses = True
81 |
82 | # If true, the current module name will be prepended to all description
83 | # unit titles (such as .. function::).
84 | #add_module_names = True
85 |
86 | # If true, sectionauthor and moduleauthor directives will be shown in the
87 | # output. They are ignored by default.
88 | #show_authors = False
89 |
90 | # The name of the Pygments (syntax highlighting) style to use.
91 | pygments_style = 'sphinx'
92 |
93 | # A list of ignored prefixes for module index sorting.
94 | #modindex_common_prefix = []
95 |
96 | # If true, keep warnings as "system message" paragraphs in the built documents.
97 | #keep_warnings = False
98 |
99 |
100 | # -- Options for HTML output ----------------------------------------------
101 |
102 | # The theme to use for HTML and HTML Help pages. See the documentation for
103 | # a list of builtin themes.
104 | html_theme = 'sphinxdoc'
105 |
106 | # Theme options are theme-specific and customize the look and feel of a theme
107 | # further. For a list of options available for each theme, see the
108 | # documentation.
109 | #html_theme_options = {}
110 |
111 | # Add any paths that contain custom themes here, relative to this directory.
112 | #html_theme_path = []
113 |
114 | # The name for this set of Sphinx documents. If None, it defaults to
115 | # " v documentation".
116 | #html_title = None
117 |
118 | # A shorter title for the navigation bar. Default is the same as html_title.
119 | #html_short_title = None
120 |
121 | # The name of an image file (relative to this directory) to place at the top
122 | # of the sidebar.
123 | #html_logo = None
124 |
125 | # The name of an image file (within the static path) to use as favicon of the
126 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
127 | # pixels large.
128 | #html_favicon = None
129 |
130 | # Add any paths that contain custom static files (such as style sheets) here,
131 | # relative to this directory. They are copied after the builtin static files,
132 | # so a file named "default.css" will overwrite the builtin "default.css".
133 | html_static_path = ['_static']
134 |
135 | # Add any extra paths that contain custom files (such as robots.txt or
136 | # .htaccess) here, relative to this directory. These files are copied
137 | # directly to the root of the documentation.
138 | #html_extra_path = []
139 |
140 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
141 | # using the given strftime format.
142 | #html_last_updated_fmt = '%b %d, %Y'
143 |
144 | # If true, SmartyPants will be used to convert quotes and dashes to
145 | # typographically correct entities.
146 | #html_use_smartypants = True
147 |
148 | # Custom sidebar templates, maps document names to template names.
149 | #html_sidebars = {}
150 |
151 | # Additional templates that should be rendered to pages, maps page names to
152 | # template names.
153 | #html_additional_pages = {}
154 |
155 | # If false, no module index is generated.
156 | #html_domain_indices = True
157 |
158 | # If false, no index is generated.
159 | #html_use_index = True
160 |
161 | # If true, the index is split into individual pages for each letter.
162 | #html_split_index = False
163 |
164 | # If true, links to the reST sources are added to the pages.
165 | #html_show_sourcelink = True
166 |
167 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
168 | #html_show_sphinx = True
169 |
170 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
171 | #html_show_copyright = True
172 |
173 | # If true, an OpenSearch description file will be output, and all pages will
174 | # contain a tag referring to it. The value of this option must be the
175 | # base URL from which the finished HTML is served.
176 | #html_use_opensearch = ''
177 |
178 | # This is the file name suffix for HTML files (e.g. ".xhtml").
179 | #html_file_suffix = None
180 |
181 | # Output file base name for HTML help builder.
182 | htmlhelp_basename = 'pyDivedoc'
183 |
184 |
185 | # -- Options for LaTeX output ---------------------------------------------
186 |
187 | latex_elements = {
188 | # The paper size ('letterpaper' or 'a4paper').
189 | #'papersize': 'letterpaper',
190 |
191 | # The font size ('10pt', '11pt' or '12pt').
192 | #'pointsize': '10pt',
193 |
194 | # Additional stuff for the LaTeX preamble.
195 | #'preamble': '',
196 | }
197 |
198 | # Grouping the document tree into LaTeX files. List of tuples
199 | # (source start file, target name, title,
200 | # author, documentclass [howto, manual, or own class]).
201 | latex_documents = [
202 | ('index', 'pyDive.tex', u'pyDive Documentation',
203 | u'Heiko Burau', 'manual'),
204 | ]
205 |
206 | # The name of an image file (relative to this directory) to place at the top of
207 | # the title page.
208 | #latex_logo = None
209 |
210 | # For "manual" documents, if this is true, then toplevel headings are parts,
211 | # not chapters.
212 | #latex_use_parts = False
213 |
214 | # If true, show page references after internal links.
215 | #latex_show_pagerefs = False
216 |
217 | # If true, show URL addresses after external links.
218 | #latex_show_urls = False
219 |
220 | # Documents to append as an appendix to all manuals.
221 | #latex_appendices = []
222 |
223 | # If false, no module index is generated.
224 | #latex_domain_indices = True
225 |
226 |
227 | # -- Options for manual page output ---------------------------------------
228 |
229 | # One entry per manual page. List of tuples
230 | # (source start file, name, description, authors, manual section).
231 | man_pages = [
232 | ('index', 'pydive', u'pyDive Documentation',
233 | [u'Heiko Burau'], 1)
234 | ]
235 |
236 | # If true, show URL addresses after external links.
237 | #man_show_urls = False
238 |
239 |
240 | # -- Options for Texinfo output -------------------------------------------
241 |
242 | # Grouping the document tree into Texinfo files. List of tuples
243 | # (source start file, target name, title, author,
244 | # dir menu entry, description, category)
245 | texinfo_documents = [
246 | ('index', 'pyDive', u'pyDive Documentation',
247 | u'Heiko Burau', 'pyDive', 'One line description of project.',
248 | 'Miscellaneous'),
249 | ]
250 |
251 | # Documents to append as an appendix to all manuals.
252 | #texinfo_appendices = []
253 |
254 | # If false, no module index is generated.
255 | #texinfo_domain_indices = True
256 |
257 | # How to display URL addresses: 'footnote', 'no', or 'inline'.
258 | #texinfo_show_urls = 'footnote'
259 |
260 | # If true, do not generate a @detailmenu in the "Top" node's menu.
261 | #texinfo_no_detailmenu = False
262 |
263 |
264 | # -- Options for Epub output ----------------------------------------------
265 |
266 | # Bibliographic Dublin Core info.
267 | epub_title = u'pyDive'
268 | epub_author = u'Author'
269 | epub_publisher = u'Author'
270 | epub_copyright = u'2014, Author'
271 |
272 | # The basename for the epub file. It defaults to the project name.
273 | #epub_basename = u'pyDive'
274 |
275 | # The HTML theme for the epub output. Since the default themes are not optimized
276 | # for small screen space, using the same theme for HTML and epub output is
277 | # usually not wise. This defaults to 'epub', a theme designed to save visual
278 | # space.
279 | #epub_theme = 'epub'
280 |
281 | # The language of the text. It defaults to the language option
282 | # or en if the language is not set.
283 | #epub_language = ''
284 |
285 | # The scheme of the identifier. Typical schemes are ISBN or URL.
286 | #epub_scheme = ''
287 |
288 | # The unique identifier of the text. This can be a ISBN number
289 | # or the project homepage.
290 | #epub_identifier = ''
291 |
292 | # A unique identification for the text.
293 | #epub_uid = ''
294 |
295 | # A tuple containing the cover image and cover page html template filenames.
296 | #epub_cover = ()
297 |
298 | # A sequence of (type, uri, title) tuples for the guide element of content.opf.
299 | #epub_guide = ()
300 |
301 | # HTML files that should be inserted before the pages created by sphinx.
302 | # The format is a list of tuples containing the path and title.
303 | #epub_pre_files = []
304 |
305 | # HTML files shat should be inserted after the pages created by sphinx.
306 | # The format is a list of tuples containing the path and title.
307 | #epub_post_files = []
308 |
309 | # A list of files that should not be packed into the epub file.
310 | epub_exclude_files = ['search.html']
311 |
312 | # The depth of the table of contents in toc.ncx.
313 | #epub_tocdepth = 3
314 |
315 | # Allow duplicate toc entries.
316 | #epub_tocdup = True
317 |
318 | # Choose between 'default' and 'includehidden'.
319 | #epub_tocscope = 'default'
320 |
321 | # Fix unsupported image types using the PIL.
322 | #epub_fix_images = False
323 |
324 | # Scale large images.
325 | #epub_max_image_width = 0
326 |
327 | # How to display URL addresses: 'footnote', 'no', or 'inline'.
328 | #epub_show_urls = 'inline'
329 |
330 | # If false, no index is generated.
331 | #epub_use_index = True
332 |
--------------------------------------------------------------------------------
/docs/decomposition.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ComputationalRadiationPhysics/pyDive/67d36420c2f1d59bef8633b2dea94b4ea4ab8ceb/docs/decomposition.png
--------------------------------------------------------------------------------
/docs/density.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ComputationalRadiationPhysics/pyDive/67d36420c2f1d59bef8633b2dea94b4ea4ab8ceb/docs/density.png
--------------------------------------------------------------------------------
/docs/index.rst:
--------------------------------------------------------------------------------
1 | .. pyDive documentation master file, created by
2 | sphinx-quickstart on Wed Jul 23 15:53:56 2014.
3 | You can adapt this file completely to your liking, but it should at least
4 | contain the root `toctree` directive.
5 |
6 | Welcome to pyDive's documentation!
7 | ==================================
8 |
9 | Contents:
10 |
11 | .. toctree::
12 | :maxdepth: 4
13 |
14 | start
15 | tutorial
16 | reference
17 |
18 |
19 | Indices and tables
20 | ==================
21 |
22 | * :ref:`genindex`
23 | * :ref:`modindex`
24 | * :ref:`search`
25 |
26 | .. glossary::
27 |
28 | engine
29 | The cluster nodes of *IPython.parallel* are called *engines*. Sometimes they are also called *targets*.
30 | They are the workers of pyDive performing all the computation and file i/o and they hold the actual array-memory.
31 | From the user perspective you don't to deal with them directly.
32 |
33 |
--------------------------------------------------------------------------------
/docs/make.bat:
--------------------------------------------------------------------------------
1 | @ECHO OFF
2 |
3 | REM Command file for Sphinx documentation
4 |
5 | if "%SPHINXBUILD%" == "" (
6 | set SPHINXBUILD=sphinx-build
7 | )
8 | set BUILDDIR=_build
9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% .
10 | set I18NSPHINXOPTS=%SPHINXOPTS% .
11 | if NOT "%PAPER%" == "" (
12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS%
13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS%
14 | )
15 |
16 | if "%1" == "" goto help
17 |
18 | if "%1" == "help" (
19 | :help
20 | echo.Please use `make ^` where ^ is one of
21 | echo. html to make standalone HTML files
22 | echo. dirhtml to make HTML files named index.html in directories
23 | echo. singlehtml to make a single large HTML file
24 | echo. pickle to make pickle files
25 | echo. json to make JSON files
26 | echo. htmlhelp to make HTML files and a HTML help project
27 | echo. qthelp to make HTML files and a qthelp project
28 | echo. devhelp to make HTML files and a Devhelp project
29 | echo. epub to make an epub
30 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter
31 | echo. text to make text files
32 | echo. man to make manual pages
33 | echo. texinfo to make Texinfo files
34 | echo. gettext to make PO message catalogs
35 | echo. changes to make an overview over all changed/added/deprecated items
36 | echo. xml to make Docutils-native XML files
37 | echo. pseudoxml to make pseudoxml-XML files for display purposes
38 | echo. linkcheck to check all external links for integrity
39 | echo. doctest to run all doctests embedded in the documentation if enabled
40 | goto end
41 | )
42 |
43 | if "%1" == "clean" (
44 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i
45 | del /q /s %BUILDDIR%\*
46 | goto end
47 | )
48 |
49 |
50 | %SPHINXBUILD% 2> nul
51 | if errorlevel 9009 (
52 | echo.
53 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
54 | echo.installed, then set the SPHINXBUILD environment variable to point
55 | echo.to the full path of the 'sphinx-build' executable. Alternatively you
56 | echo.may add the Sphinx directory to PATH.
57 | echo.
58 | echo.If you don't have Sphinx installed, grab it from
59 | echo.http://sphinx-doc.org/
60 | exit /b 1
61 | )
62 |
63 | if "%1" == "html" (
64 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html
65 | if errorlevel 1 exit /b 1
66 | echo.
67 | echo.Build finished. The HTML pages are in %BUILDDIR%/html.
68 | goto end
69 | )
70 |
71 | if "%1" == "dirhtml" (
72 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml
73 | if errorlevel 1 exit /b 1
74 | echo.
75 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml.
76 | goto end
77 | )
78 |
79 | if "%1" == "singlehtml" (
80 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml
81 | if errorlevel 1 exit /b 1
82 | echo.
83 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml.
84 | goto end
85 | )
86 |
87 | if "%1" == "pickle" (
88 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle
89 | if errorlevel 1 exit /b 1
90 | echo.
91 | echo.Build finished; now you can process the pickle files.
92 | goto end
93 | )
94 |
95 | if "%1" == "json" (
96 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json
97 | if errorlevel 1 exit /b 1
98 | echo.
99 | echo.Build finished; now you can process the JSON files.
100 | goto end
101 | )
102 |
103 | if "%1" == "htmlhelp" (
104 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp
105 | if errorlevel 1 exit /b 1
106 | echo.
107 | echo.Build finished; now you can run HTML Help Workshop with the ^
108 | .hhp project file in %BUILDDIR%/htmlhelp.
109 | goto end
110 | )
111 |
112 | if "%1" == "qthelp" (
113 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp
114 | if errorlevel 1 exit /b 1
115 | echo.
116 | echo.Build finished; now you can run "qcollectiongenerator" with the ^
117 | .qhcp project file in %BUILDDIR%/qthelp, like this:
118 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\pyDive.qhcp
119 | echo.To view the help file:
120 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\pyDive.ghc
121 | goto end
122 | )
123 |
124 | if "%1" == "devhelp" (
125 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp
126 | if errorlevel 1 exit /b 1
127 | echo.
128 | echo.Build finished.
129 | goto end
130 | )
131 |
132 | if "%1" == "epub" (
133 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub
134 | if errorlevel 1 exit /b 1
135 | echo.
136 | echo.Build finished. The epub file is in %BUILDDIR%/epub.
137 | goto end
138 | )
139 |
140 | if "%1" == "latex" (
141 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
142 | if errorlevel 1 exit /b 1
143 | echo.
144 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex.
145 | goto end
146 | )
147 |
148 | if "%1" == "latexpdf" (
149 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
150 | cd %BUILDDIR%/latex
151 | make all-pdf
152 | cd %BUILDDIR%/..
153 | echo.
154 | echo.Build finished; the PDF files are in %BUILDDIR%/latex.
155 | goto end
156 | )
157 |
158 | if "%1" == "latexpdfja" (
159 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
160 | cd %BUILDDIR%/latex
161 | make all-pdf-ja
162 | cd %BUILDDIR%/..
163 | echo.
164 | echo.Build finished; the PDF files are in %BUILDDIR%/latex.
165 | goto end
166 | )
167 |
168 | if "%1" == "text" (
169 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text
170 | if errorlevel 1 exit /b 1
171 | echo.
172 | echo.Build finished. The text files are in %BUILDDIR%/text.
173 | goto end
174 | )
175 |
176 | if "%1" == "man" (
177 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man
178 | if errorlevel 1 exit /b 1
179 | echo.
180 | echo.Build finished. The manual pages are in %BUILDDIR%/man.
181 | goto end
182 | )
183 |
184 | if "%1" == "texinfo" (
185 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo
186 | if errorlevel 1 exit /b 1
187 | echo.
188 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo.
189 | goto end
190 | )
191 |
192 | if "%1" == "gettext" (
193 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale
194 | if errorlevel 1 exit /b 1
195 | echo.
196 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale.
197 | goto end
198 | )
199 |
200 | if "%1" == "changes" (
201 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes
202 | if errorlevel 1 exit /b 1
203 | echo.
204 | echo.The overview file is in %BUILDDIR%/changes.
205 | goto end
206 | )
207 |
208 | if "%1" == "linkcheck" (
209 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck
210 | if errorlevel 1 exit /b 1
211 | echo.
212 | echo.Link check complete; look for any errors in the above output ^
213 | or in %BUILDDIR%/linkcheck/output.txt.
214 | goto end
215 | )
216 |
217 | if "%1" == "doctest" (
218 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest
219 | if errorlevel 1 exit /b 1
220 | echo.
221 | echo.Testing of doctests in the sources finished, look at the ^
222 | results in %BUILDDIR%/doctest/output.txt.
223 | goto end
224 | )
225 |
226 | if "%1" == "xml" (
227 | %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml
228 | if errorlevel 1 exit /b 1
229 | echo.
230 | echo.Build finished. The XML files are in %BUILDDIR%/xml.
231 | goto end
232 | )
233 |
234 | if "%1" == "pseudoxml" (
235 | %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml
236 | if errorlevel 1 exit /b 1
237 | echo.
238 | echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml.
239 | goto end
240 | )
241 |
242 | :end
243 |
--------------------------------------------------------------------------------
/docs/modules.rst:
--------------------------------------------------------------------------------
1 | .. note:: All functions of these modules are also directly accessable from the ``pyDive`` module.
2 |
3 | pyDive.arrayOfStructs module
4 | ----------------------------
5 |
6 | .. automodule:: pyDive.arrayOfStructs
7 | :members:
8 |
9 | pyDive.algorithm module
10 | -----------------------
11 |
12 | .. automodule:: pyDive.algorithm
13 | :members:
14 |
15 | pyDive.fragment module
16 | ----------------------
17 |
18 | .. automodule:: pyDive.fragment
19 | :members:
20 |
21 | pyDive.mappings module
22 | ----------------------
23 |
24 | .. automodule:: pyDive.mappings
25 | :members:
26 |
27 | pyDive.picongpu module
28 | ----------------------
29 |
30 | .. automodule:: pyDive.picongpu
31 | :members:
32 |
33 | pyDive.pyDive module
34 | --------------------
35 |
36 | .. automodule:: pyDive.pyDive
37 | :members:
38 |
39 |
--------------------------------------------------------------------------------
/docs/pyDive.adios.rst:
--------------------------------------------------------------------------------
1 |
2 | pyDive.arrays.ad_ndarray module
3 | ===============================
4 |
5 | .. note:: This module has a shortcut: ``pyDive.adios``.
6 |
7 | .. autoclass:: pyDive.arrays.ad_ndarray.ad_ndarray
8 | :members: __init__, load
9 |
10 | .. automodule:: pyDive.arrays.ad_ndarray
11 | :members:
12 |
--------------------------------------------------------------------------------
/docs/pyDive.cloned_ndarray.rst:
--------------------------------------------------------------------------------
1 | .. _pyDive.cloned_ndarray:
2 |
3 | pyDive.cloned_ndarray package
4 | =============================
5 |
6 | Submodules
7 | ----------
8 |
9 | pyDive.cloned_ndarray.cloned_ndarray module
10 | -------------------------------------------
11 |
12 | .. autoclass:: pyDive.cloned_ndarray.cloned_ndarray.cloned_ndarray
13 | :members: __init__, merge, sum
14 |
15 | pyDive.cloned_ndarray.factories module
16 | --------------------------------------
17 |
18 | .. automodule:: pyDive.cloned_ndarray.factories
19 | :members:
20 |
21 |
--------------------------------------------------------------------------------
/docs/pyDive.gpu.rst:
--------------------------------------------------------------------------------
1 | .. _pyDive.h5_ndarray:
2 |
3 | pyDive.arrays.gpu_ndarray module
4 | ===============================
5 |
6 | .. note:: This module has a shortcut: ``pyDive.gpu``.
7 |
8 | .. autoclass:: pyDive.arrays.gpu_ndarray.gpu_ndarray
9 | :members: __init__, to_cpu
10 |
11 | .. automodule:: pyDive.arrays.gpu_ndarray
12 | :members:
13 |
--------------------------------------------------------------------------------
/docs/pyDive.h5.rst:
--------------------------------------------------------------------------------
1 | .. _pyDive.h5_ndarray:
2 |
3 | pyDive.arrays.h5_ndarray module
4 | ===============================
5 |
6 | .. note:: This module has a shortcut: ``pyDive.h5``.
7 |
8 | .. autoclass:: pyDive.arrays.h5_ndarray.h5_ndarray
9 | :members: __init__, load
10 |
11 | .. automodule:: pyDive.arrays.h5_ndarray
12 | :members:
13 |
--------------------------------------------------------------------------------
/docs/pyDive.ndarray.rst:
--------------------------------------------------------------------------------
1 | .. _pyDive.ndarray:
2 |
3 | pyDive.arrays.ndarray module
4 | ============================
5 |
6 | .. note:: All of this module's functions and classes are also directly accessable from the :mod:`pyDive` module.
7 |
8 | pyDive.ndarray class
9 | --------------------
10 |
11 | .. autoclass:: pyDive.ndarray
12 | :members: __init__, gather, copy, dist_like
13 |
14 | Factory functions
15 | -----------------
16 | These are convenient functions to create a *pyDive.ndarray* instance.
17 |
18 | .. automodule:: pyDive.arrays.ndarray
19 | :members: array, empty, empty_like, hollow, hollow_like, zeros, zeros_like, ones, ones_like
20 |
21 | Universal functions
22 | -------------------
23 |
24 | *numpy* knows the so called *ufuncs* (universal function). These are functions which can be applied
25 | elementwise on an array, like *sin*, *cos*, *exp*, *sqrt*, etc. All of these *ufuncs* from *numpy* are
26 | also available for *pyDive.ndarray* arrays, e.g. ::
27 |
28 | a = pyDive.ones([100])
29 | a = pyDive.sin(a)
30 |
31 |
--------------------------------------------------------------------------------
/docs/reference.rst:
--------------------------------------------------------------------------------
1 | Reference
2 | =========
3 |
4 | Arrays
5 | --------
6 |
7 | .. toctree::
8 |
9 | pyDive.ndarray
10 | pyDive.h5
11 | pyDive.adios
12 | pyDive.gpu
13 | pyDive.cloned_ndarray
14 |
15 | Modules
16 | -------
17 |
18 | .. toctree::
19 |
20 | modules
21 |
--------------------------------------------------------------------------------
/docs/sample.h5:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ComputationalRadiationPhysics/pyDive/67d36420c2f1d59bef8633b2dea94b4ea4ab8ceb/docs/sample.h5
--------------------------------------------------------------------------------
/docs/spectrum.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ComputationalRadiationPhysics/pyDive/67d36420c2f1d59bef8633b2dea94b4ea4ab8ceb/docs/spectrum.png
--------------------------------------------------------------------------------
/docs/start.rst:
--------------------------------------------------------------------------------
1 | Getting started
2 | ===============
3 |
4 | Quickstart
5 | ----------
6 |
7 | pyDive is built on top of *IPython.parallel*, *numpy* and *mpi4py*. *h5py*, *adios* and *pycuda* are optional. Running ``python setup.py install`` will install
8 | pyDive with these and other required packages from `requirements.txt`. Alternatively you can install it via pip: ``pip install pyDive``.
9 |
10 | Basic code example: ::
11 |
12 | import pyDive
13 | pyDive.init(profile='mpi')
14 |
15 | arrayA = pyDive.ones((1000, 1000, 1000), distaxes='all')
16 | arrayB = pyDive.zeros_like(arrayA)
17 |
18 | # do some array operations, + - * / sin cos, ..., slicing, etc...
19 | arrayC = arrayA + arrayB
20 |
21 | # plot result
22 | import matplotlib.pyplot as plt
23 | plt.imshow(arrayC[500,::10,::10])
24 |
25 | Before actually running this script there must have been an IPython.parallel cluster launched (see section below) otherwise `pyDive.init()` fails.
26 |
27 | pyDive distributes array-memory along one or multiple user-specified axes:
28 |
29 | .. image:: decomposition.png
30 |
31 | You can either specify the exact decomposition for each axis or leave the default which persuits to squared chunks.
32 |
33 | Although the array elements are stored on the cluster nodes you have full access through indexing. If you want to have a local array
34 | from a pyDive-array anyway you can call ``array.gather()`` but make sure that your pyDive-array is small enough to fit
35 | into your local machine's memory. If not you may want to slice it first. Note that an array is also gathered implicitly
36 | if you try to access an attribute which is only available for the local array. This is why there is no ``gather()`` in the example above
37 | when calling ``imshow()``.
38 |
39 | .. _cluster-config:
40 |
41 | Setup an IPython.parallel cluster configuration
42 | -----------------------------------------------
43 |
44 | The first step is to create an IPython.parallel profile: http://ipython.org/ipython-doc/2/parallel/parallel_process.html.
45 | The name of this profile is the argument of :func:`pyDive.init`. It defaults to ``"mpi"``.
46 | Starting the cluster is then the second and final step::
47 |
48 | $ ipcluster start -n 4 --profile=mpi
49 |
50 | Run tests
51 | ---------
52 |
53 | In order to test the pyDive installation run::
54 |
55 | $ python setup.py test
56 |
57 | This will ask you for the IPython.parallel profile to be used and the number of engines to be started, e.g.: ::
58 |
59 | $ Name of your IPython-parallel profile you want to run the tests with: pbs
60 | $ Number of engines: 256
61 |
62 | Then the script starts the cluster, runs the tests and finally stops the cluster. If you have already a cluster running by your own
63 | you can also run the tests by launching ``py.test`` from the pyDive directory and setting the environment variable ``IPP_PROFILE_NAME``
64 | to the profile's name.
65 |
66 | Overview
67 | --------
68 |
69 | pyDive knows different kinds of distributed arrays, all corresponding to a local, non-distributed array:
70 | - numpy -> :obj:`pyDive.ndarray` -> Stores array elements in cluster nodes' memory.
71 | - hdf5 -> :obj:`pyDive.h5.h5_ndarray` -> Stores array elements in a hdf5-file.
72 | - adios -> :obj:`pyDive.ad.ad_ndarray` -> Stores array elements in a adios-file.
73 | - gpu -> :obj:`pyDive.gpu.gpu_ndarray` -> Stores array elements in clusters' gpus.
74 | - :obj:`pyDive.cloned_ndarray` -> Holds independent copies of one array on cluster nodes.
75 |
76 | Among these three packages there are a few modules:
77 | - :mod:`pyDive.structered` -> structured datatypes
78 | - :mod:`pyDive.algorithm` -> map, reduce, mapReduce
79 | - :mod:`pyDive.fragment` -> fragment file-disk array to fit into the cluster's main memory
80 | - :mod:`pyDive.mappings` -> particle-mesh mappings
81 | - :mod:`pyDive.picongpu` -> helper functions for picongpu-users
82 | - :mod:`pyDive.pyDive` -> shortcuts for most used functions
83 |
84 |
--------------------------------------------------------------------------------
/docs/tutorial.rst:
--------------------------------------------------------------------------------
1 | Tutorials
2 | =========
3 |
4 | In this section we are going through a few use cases for pyDive. If you want to test the code you can download
5 | the :download:`sample hdf5-file`.
6 | It has the following dataset structure::
7 |
8 | $ h5ls -r sample.h5
9 | / Group
10 | /fields Group
11 | /fields/fieldB Group
12 | /fields/fieldB/z Dataset {256, 256}
13 | /fields/fieldE Group
14 | /fields/fieldE/x Dataset {256, 256}
15 | /fields/fieldE/y Dataset {256, 256}
16 | /particles Group
17 | /particles/cellidx Group
18 | /particles/cellidx/x Dataset {10000}
19 | /particles/cellidx/y Dataset {10000}
20 | /particles/pos Group
21 | /particles/pos/x Dataset {10000}
22 | /particles/pos/y Dataset {10000}
23 | /particles/vel Group
24 | /particles/vel/x Dataset {10000}
25 | /particles/vel/y Dataset {10000}
26 | /particles/vel/z Dataset {10000}
27 |
28 | After launching the cluster (:ref:`cluster-config`) the first step is to initialize pyDive: ::
29 |
30 | import pyDive
31 | pyDive.init()
32 |
33 | Load a single dataset: ::
34 |
35 | h5fieldB_z = pyDive.h5.open("sample.h5", "/fields/fieldB/z", distaxes='all')
36 |
37 | assert type(h5fieldB_z) is pyDive.h5.h5_ndarray
38 |
39 | *h5fieldB_z* just holds a dataset *handle*. To read out data into memory call ``load()``: ::
40 |
41 | fieldB_z = h5fieldB_z.load()
42 |
43 | assert type(fieldB_z) is pyDive.ndarray
44 |
45 | This loads the entire dataset into the main memory of all :term:`engines`. The array elements are distributed
46 | along all axes.
47 |
48 | We can also load a hdf5-group: ::
49 |
50 | h5fieldE = pyDive.h5.open("sample.h5", "/fields/fieldE", distaxes='all')
51 | fieldE = h5fieldE.load()
52 |
53 | *h5fieldE* and *fieldE* are some so called "virtual array-of-structures", see: :mod:`pyDive.structered`. ::
54 |
55 | >>> print h5fieldE
56 | VirtualArrayOfStructs, shape: [256, 256]>:
57 | y -> float32
58 | x -> float32
59 |
60 | >>> print fieldE
61 | VirtualArrayOfStructs, shape: [256, 256]>:
62 | y -> float32
63 | x -> float32
64 |
65 | Now, let's do some calculations!
66 |
67 | Example 1: Total field energy
68 | -----------------------------
69 |
70 | Computing the total field energy of an electromagnetic field means squaring and summing or in pyDive's words: ::
71 |
72 | import pyDive
73 | import numpy as np
74 | pyDive.init()
75 |
76 | h5input = "sample.h5"
77 |
78 | h5fields = pyDive.h5.open(h5input, "/fields") # defaults to distaxes='all'
79 | fields = h5fields.load() # read out all fields into cluster's main memory in parallel
80 |
81 | energy_field = fields.fieldE.x**2 + fields.fieldE.y**2 + fields.fieldB.z**2
82 |
83 | total_energy = pyDive.reduce(energy_field, np.add)
84 | print total_energy
85 |
86 | Output: ::
87 |
88 | $ python example1.py
89 | 557502.0
90 |
91 | Well this was just a very small hdf5-sample of 1.3 MB however in real world we deal with a lot greater data volumes.
92 | So what happens if *h5fields* is too large to be stored in the main memory of the whole cluster? The line ``fields = h5fields.load()`` will crash.
93 | In this case we want to load the hdf5 data piece by piece. The function :obj:`pyDive.fragment` helps us doing so: ::
94 |
95 | import pyDive
96 | import numpy as np
97 | pyDive.init()
98 |
99 | h5input = "sample.h5"
100 |
101 | big_h5fields = pyDive.h5.open(h5input, "/fields")
102 | # big_h5fields.load() # would cause a crash
103 |
104 | total_energy = 0.0
105 | for h5fields in pyDive.fragment(big_h5fields):
106 | fields = h5fields.load()
107 |
108 | energy_field = fields.fieldE.x**2 + fields.fieldE.y**2 + fields.fieldB.z**2
109 |
110 | total_energy += pyDive.reduce(energy_field, np.add)
111 |
112 | print total_energy
113 |
114 | An equivalent way to get this result is a :obj:`pyDive.mapReduce`: ::
115 |
116 | ...
117 | def square_fields(h5fields):
118 | fields = h5fields.load()
119 | return fields.fieldE.x**2 + fields.fieldE.y**2 + fields.fieldB.z**2
120 |
121 | total_energy = pyDive.mapReduce(square_fields, np.add, h5fields)
122 | print total_energy
123 |
124 | *square_fields* is called on each :term:`engine` where *h5fields* is a structure (:mod:`pyDive.arrayOfStructs`) of :obj:`h5_ndarrays` representing a sub part of the big *h5fields*.
125 | :func:`pyDive.algorithm.mapReduce` can be called with an arbitrary number of arrays including
126 | :obj:`pyDive.ndarrays`, :obj:`pyDive.h5.h5_ndarrays`, :obj:`pyDive.adios.ad_ndarrays` and :obj:`pyDive.cloned_ndarrays`. If there are :obj:`pyDive.h5.h5_ndarrays` or :obj:`pyDive.adios.ad_ndarrays` it will
127 | check whether they fit into the combined main memory of all cluster nodes as a whole and loads them piece by piece if not.
128 |
129 | Now let's say our dataset is really big and we just want to get a first estimate of the total energy: ::
130 |
131 | ...
132 | total_energy = pyDive.mapReduce(square_fields, np.add, h5fields[::10, ::10]) * 10.0**2
133 |
134 | Slicing on pyDive-arrays is always allowed.
135 |
136 | If you use `picongpu `_
137 | here is an example of how to get the total field energy for each timestep (see :mod:`pyDive.picongpu`): ::
138 |
139 | import pyDive
140 | import numpy as np
141 | pyDive.init()
142 |
143 | def square_field(h5field):
144 | field = h5field.load()
145 | return field.x**2 + field.x**2 + field.x**2
146 |
147 | for step, h5field in pyDive.picongpu.loadAllSteps("/.../simOutput", "fields/FieldE"):
148 | total_energy = pyDive.mapReduce(square_field, np.add, h5field)
149 |
150 | print step, total_energy
151 |
152 | Example 2: Particle density field
153 | ---------------------------------
154 |
155 | Given the list of particles in our ``sample.h5`` we want to create a 2D density field out of it. For this particle-to-mesh
156 | mapping we need to apply a certain particle shape like cloud-in-cell (CIC), triangular-shaped-cloud (TSC), and so on. A list of
157 | these together with the actual mapping functions can be found in the :mod:`pyDive.mappings` module. If you miss a shape you can
158 | easily create one by your own by defining a particle shape function. Note that if you have `numba `_
159 | installed the shape function will be compiled resulting in a significant speed-up.
160 |
161 | We assume that the particle positions are distributed randomly. This means although each engine is loading a separate part of all particles it needs to
162 | write to the entire density field. Therefore the density field must have a whole representation on each participating engine.
163 | This is the job of :class:`pyDive.cloned_ndarray.cloned_ndarray.cloned_ndarray`. ::
164 |
165 | import pyDive
166 | import numpy as np
167 | pyDive.init()
168 |
169 | shape = [256, 256]
170 | density = pyDive.cloned.zeros(shape)
171 |
172 | h5input = "sample.h5"
173 |
174 | particles = pyDive.h5.open(h5input, "/particles")
175 |
176 | def particles2density(particles, density):
177 | particles = particles.load()
178 | total_pos = particles.cellidx.astype(np.float32) + particles.pos
179 |
180 | # convert total_pos to an (N, 2) shaped array
181 | total_pos = np.hstack((total_pos.x[:,np.newaxis],
182 | total_pos.y[:,np.newaxis]))
183 |
184 | par_weighting = np.ones(particles.shape)
185 | import pyDive.mappings
186 | pyDive.mappings.particles2mesh(density, par_weighting, total_pos, pyDive.mappings.CIC)
187 |
188 | pyDive.map(particles2density, particles, density)
189 |
190 | final_density = density.sum() # add up all local copies
191 |
192 | from matplotlib import pyplot as plt
193 | plt.imshow(final_density)
194 | plt.show()
195 |
196 | Output:
197 |
198 | .. image:: density.png
199 |
200 | Here, as in the first example, *particles2density* is a function executed on the :term:`engines ` by :func:`pyDive.algorithm.map`.
201 | All of its arguments are numpy-arrays or structures (:mod:`pyDive.arrayOfStructs`) of numpy-arrays.
202 |
203 | :func:`pyDive.algorithm.map` can also be used as a decorator: ::
204 |
205 | @pyDive.map
206 | def particles2density(particles, density):
207 | ...
208 |
209 | particles2density(particles, density)
210 |
211 |
212 | Example 3: Particle energy spectrum
213 | -----------------------------------
214 |
215 | ::
216 |
217 | import pyDive
218 | import numpy as np
219 | pyDive.init()
220 |
221 | bins = 256
222 | spectrum = pyDive.cloned.zeros([bins])
223 |
224 | h5input = "sample.h5"
225 |
226 | velocities = pyDive.h5.open(h5input, "/particles/vel")
227 |
228 | @pyDive.map
229 | def vel2spectrum(velocities, spectrum, bins):
230 | velocities = velocities.load()
231 | mass = 1.0
232 | energies = 0.5 * mass * (velocities.x**2 + velocities.y**2 + velocities.z**2)
233 |
234 | spectrum[:], bin_edges = np.histogram(energies, bins)
235 |
236 | vel2spectrum(velocities, spectrum, bins=bins)
237 |
238 | final_spectrum = spectrum.sum() # add up all local copies
239 |
240 | from matplotlib import pyplot as plt
241 | plt.plot(final_spectrum)
242 | plt.show()
243 |
244 | Output:
245 |
246 | .. image:: spectrum.png
247 |
248 |
--------------------------------------------------------------------------------
/pyDive/IPParallelClient.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2014 Heiko Burau
3 |
4 | This file is part of pyDive.
5 |
6 | pyDive is free software: you can redistribute it and/or modify
7 | it under the terms of of either the GNU General Public License or
8 | the GNU Lesser General Public License as published by
9 | the Free Software Foundation, either version 3 of the License, or
10 | (at your option) any later version.
11 | pyDive is distributed in the hope that it will be useful,
12 | but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | GNU General Public License and the GNU Lesser General Public License
15 | for more details.
16 |
17 | You should have received a copy of the GNU General Public License
18 | and the GNU Lesser General Public License along with pyDive.
19 | If not, see .
20 | """
21 | __doc__ = None
22 |
23 | from IPython.parallel import Client
24 | from IPython.parallel import interactive
25 | from collections import Counter
26 | import sys
27 |
28 | #: IPParallel direct view
29 | view = None
30 | #: number of processes per node
31 | ppn = None
32 |
33 | def init(profile='mpi'):
34 | """Initialize pyDive.
35 |
36 | :param str profile: The name of the cluster profile of *IPython.parallel*. Has to be an MPI-profile.\
37 | Defaults to 'mpi'.
38 | """
39 | # init direct view
40 | global view
41 |
42 | client = Client(profile=profile)
43 | client.clear()
44 | view = client[:]
45 | view.block = True
46 | view.execute('''\
47 | import numpy as np
48 | from mpi4py import MPI
49 | import h5py as h5
50 | import os, sys
51 | import psutil
52 | import math
53 | os.environ["onTarget"] = 'True'
54 | from pyDive import structured
55 | from pyDive import algorithm
56 | from pyDive.distribution import interengine
57 | try:
58 | import pyDive.arrays.local.h5_ndarray
59 | except ImportError:
60 | pass
61 | try:
62 | import pyDive.arrays.local.ad_ndarray
63 | except ImportError:
64 | pass
65 | try:
66 | import pyDive.arrays.local.gpu_ndarray
67 | import pycuda.autoinit
68 | except ImportError:
69 | pass
70 | ''')
71 |
72 | # get number of processes per node (ppn)
73 | def hostname():
74 | import socket
75 | return socket.gethostname()
76 | hostnames = view.apply(interactive(hostname))
77 | global ppn
78 | ppn = max(Counter(hostnames).values())
79 |
80 | # mpi ranks
81 | get_rank = interactive(lambda: MPI.COMM_WORLD.Get_rank())
82 | all_ranks = view.apply(get_rank)
83 | view['target2rank'] = all_ranks
84 |
85 | def getView():
86 | global view
87 | assert view is not None, "pyDive.init() has not been called yet."
88 | return view
89 |
90 | def getPPN():
91 | global ppn
92 | assert ppn is not None, "pyDive.init() has not been called yet."
93 | return ppn
94 |
--------------------------------------------------------------------------------
/pyDive/__init__.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2014 Heiko Burau
3 |
4 | This file is part of pyDive.
5 |
6 | pyDive is free software: you can redistribute it and/or modify
7 | it under the terms of of either the GNU General Public License or
8 | the GNU Lesser General Public License as published by
9 | the Free Software Foundation, either version 3 of the License, or
10 | (at your option) any later version.
11 | pyDive is distributed in the hope that it will be useful,
12 | but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | GNU General Public License and the GNU Lesser General Public License
15 | for more details.
16 |
17 | You should have received a copy of the GNU General Public License
18 | and the GNU Lesser General Public License along with pyDive.
19 | If not, see .
20 | """
21 | __doc__ = None
22 |
23 | # -*- coding: utf-8 -*-
24 | __version__ = "1.2.2"
25 |
26 | import os
27 | onTarget = os.environ.get("onTarget", 'False')
28 |
29 | # import only if this code is not executed on engine
30 | if onTarget == 'False':
31 | from pyDive import *
32 |
--------------------------------------------------------------------------------
/pyDive/algorithm.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2014 Heiko Burau
3 |
4 | This file is part of pyDive.
5 |
6 | pyDive is free software: you can redistribute it and/or modify
7 | it under the terms of of either the GNU General Public License or
8 | the GNU Lesser General Public License as published by
9 | the Free Software Foundation, either version 3 of the License, or
10 | (at your option) any later version.
11 | pyDive is distributed in the hope that it will be useful,
12 | but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | GNU General Public License and the GNU Lesser General Public License
15 | for more details.
16 |
17 | You should have received a copy of the GNU General Public License
18 | and the GNU Lesser General Public License along with pyDive.
19 | If not, see .
20 | """
21 | __doc__ = None
22 |
23 | import os
24 | # check whether this code is executed on target or not
25 | onTarget = os.environ.get("onTarget", 'False')
26 | if onTarget == 'False':
27 | from arrays.ndarray import ndarray
28 | from arrays.h5_ndarray import h5_ndarray
29 | from cloned_ndarray.cloned_ndarray import cloned_ndarray
30 | import IPParallelClient as com
31 | from IPython.parallel import interactive
32 | from fragment import fragment, hdd_arraytypes
33 | from structured import VirtualArrayOfStructs
34 | import numpy as np
35 |
36 | def map(f, *arrays, **kwargs):
37 | """Applies *f* on :term:`engine` on local arrays related to *arrays*.
38 | Example: ::
39 |
40 | cluster_array = pyDive.ones(shape=[100], distaxes=0)
41 |
42 | cluster_array *= 2.0
43 | # equivalent to
44 | pyDive.map(lambda a: a *= 2.0, cluster_array) # a is the local numpy-array of *cluster_array*
45 |
46 | Or, as a decorator: ::
47 |
48 | @pyDive.map
49 | def twice(a):
50 | a *= 2.0
51 |
52 | twice(cluster_array)
53 |
54 | :param callable f: function to be called on :term:`engine`. Has to accept *numpy-arrays* and *kwargs*
55 | :param arrays: list of arrays including *pyDive.ndarrays*, *pyDive.h5_ndarrays* or *pyDive.cloned_ndarrays*
56 | :param kwargs: user-specified keyword arguments passed to *f*
57 | :raises AssertionError: if the *shapes* of *pyDive.ndarrays* and *pyDive.h5_ndarrays* do not match
58 | :raises AssertionError: if the *distaxes* attributes of *pyDive.ndarrays* and *pyDive.h5_ndarrays* do not match
59 |
60 | Notes:
61 | - If the hdf5 data exceeds the memory limit (currently 25% of the combined main memory of all cluster nodes)\
62 | the data will be read block-wise so that a block fits into memory.
63 | - *map* chooses the list of *engines* from the **first** element of *arrays*. On these engines *f* is called.\
64 | If the first array is a *pyDive.h5_ndarray* all engines will be used.
65 | - *map* is not writing data back to a *pyDive.h5_ndarray* yet.
66 | - *map* does not equalize the element distribution of *pyDive.ndarrays* before execution.
67 | """
68 | if not arrays:
69 | # decorator mode
70 | def map_deco(*arrays, **kwargs):
71 | map(f, *arrays, **kwargs)
72 | return map_deco
73 |
74 | def map_wrapper(f, array_names, **kwargs):
75 | arrays = [globals()[array_name] for array_name in array_names]
76 | f(*arrays, **kwargs)
77 |
78 | view = com.getView()
79 |
80 | tmp_targets = view.targets # save current target list
81 | if type(arrays[0]) == VirtualArrayOfStructs:
82 | view.targets = arrays[0].firstArray.target_ranks
83 | else:
84 | view.targets = arrays[0].target_ranks
85 |
86 | hdd_arrays = [a for a in arrays if (hasattr(a, "arraytype") and a.arraytype in hdd_arraytypes) or type(a) in hdd_arraytypes]
87 | if hdd_arrays:
88 | cloned_arrays = [a for a in arrays if (hasattr(a, "arraytype") and a.arraytype is cloned_ndarray) or type(a) is cloned_ndarray]
89 | other_arrays = [a for a in arrays if not ((hasattr(a, "arraytype") and a.arraytype is cloned_ndarray) or type(a) is cloned_ndarray)]
90 |
91 | cloned_arrays_ids = [id(a) for a in cloned_arrays]
92 | other_arrays_ids = [id(a) for a in other_arrays]
93 |
94 | for fragments in fragment(*other_arrays):
95 | it_other_arrays = iter(other_arrays)
96 | it_cloned_arrays = iter(cloned_arrays)
97 |
98 | array_names = []
99 | for a in arrays:
100 | if id(a) in cloned_arrays_ids:
101 | array_names.append(repr(it_cloned_arrays.next()))
102 | continue
103 | if id(a) in other_arrays_ids:
104 | array_names.append(repr(it_other_arrays.next()))
105 | continue
106 |
107 | view.apply(interactive(map_wrapper), interactive(f), array_names, **kwargs)
108 | else:
109 | array_names = [repr(a) for a in arrays]
110 | view.apply(interactive(map_wrapper), interactive(f), array_names, **kwargs)
111 |
112 | view.targets = tmp_targets # restore target list
113 |
114 | def reduce(array, op):
115 | """Perform a tree-like reduction over all axes of *array*.
116 |
117 | :param array: *pyDive.ndarray*, *pyDive.h5_ndarray* or *pyDive.cloned_ndarray* to be reduced
118 | :param numpy-ufunc op: reduce operation, e.g. *numpy.add*.
119 |
120 | If the hdf5 data exceeds the memory limit (currently 25% of the combined main memory of all cluster nodes)\
121 | the data will be read block-wise so that a block fits into memory.
122 | """
123 | def reduce_wrapper(array_name, op_name):
124 | array = globals()[array_name]
125 | op = eval("np." + op_name)
126 | return algorithm.__tree_reduce(array, axis=None, op=op) # reduction over all axes
127 |
128 | view = com.getView()
129 |
130 | tmp_targets = view.targets # save current target list
131 | if type(array) == VirtualArrayOfStructs:
132 | view.targets = array.firstArray.target_ranks
133 | else:
134 | view.targets = array.target_ranks
135 |
136 | result = None
137 |
138 | if (hasattr(array, "arraytype") and array.arraytype in hdd_arraytypes) or type(array) in hdd_arraytypes:
139 | for chunk in fragment(array):
140 | array_name = repr(chunk)
141 |
142 | targets_results = view.apply(interactive(reduce_wrapper), array_name, op.__name__)
143 | chunk_result = op.reduce(targets_results) # reduce over targets' results
144 |
145 | if result is None:
146 | result = chunk_result
147 | else:
148 | result = op(result, chunk_result)
149 | else:
150 | array_name = repr(array)
151 |
152 | targets_results = view.apply(interactive(reduce_wrapper), array_name, op.__name__)
153 | result = op.reduce(targets_results) # reduce over targets' results
154 |
155 | view.targets = tmp_targets # restore target list
156 | return result
157 |
158 | def mapReduce(map_func, reduce_op, *arrays, **kwargs):
159 | """Applies *map_func* on :term:`engine` on local arrays related to *arrays*
160 | and reduces its result in a tree-like fashion over all axes.
161 | Example: ::
162 |
163 | cluster_array = pyDive.ones(shape=[100], distaxes=0)
164 |
165 | s = pyDive.mapReduce(lambda a: a**2, np.add, cluster_array) # a is the local numpy-array of *cluster_array*
166 | assert s == 100
167 |
168 | :param callable f: function to be called on :term:`engine`. Has to accept *numpy-arrays* and *kwargs*
169 | :param numpy-ufunc reduce_op: reduce operation, e.g. *numpy.add*.
170 | :param arrays: list of arrays including *pyDive.ndarrays*, *pyDive.h5_ndarrays* or *pyDive.cloned_ndarrays*
171 | :param kwargs: user-specified keyword arguments passed to *f*
172 | :raises AssertionError: if the *shapes* of *pyDive.ndarrays* and *pyDive.h5_ndarrays* do not match
173 | :raises AssertionError: if the *distaxes* attributes of *pyDive.ndarrays* and *pyDive.h5_ndarrays* do not match
174 |
175 | Notes:
176 | - If the hdf5 data exceeds the memory limit (currently 25% of the combined main memory of all cluster nodes)\
177 | the data will be read block-wise so that a block fits into memory.
178 | - *mapReduce* chooses the list of *engines* from the **first** element of *arrays*. On these engines the mapReduce will be executed.\
179 | If the first array is a *pyDive.h5_ndarray* all engines will be used.
180 | - *mapReduce* is not writing data back to a *pyDive.h5_ndarray* yet.
181 | - *mapReduce* does not equalize the element distribution of *pyDive.ndarrays* before execution.
182 | """
183 | def mapReduce_wrapper(map_func, reduce_op_name, array_names, **kwargs):
184 | arrays = [globals()[array_name] for array_name in array_names]
185 | reduce_op = eval("np." + reduce_op_name)
186 | return algorithm.__tree_reduce(map_func(*arrays, **kwargs), axis=None, op=reduce_op)
187 |
188 | view = com.getView()
189 | tmp_targets = view.targets # save current target list
190 | if type(arrays[0]) == VirtualArrayOfStructs:
191 | view.targets = arrays[0].firstArray.target_ranks
192 | else:
193 | view.targets = arrays[0].target_ranks
194 |
195 | result = None
196 |
197 | hdd_arrays = [a for a in arrays if (hasattr(a, "arraytype") and a.arraytype in hdd_arraytypes) or type(a) in hdd_arraytypes]
198 | if hdd_arrays:
199 | cloned_arrays = [a for a in arrays if (hasattr(a, "arraytype") and a.arraytype is cloned_ndarray) or type(a) is cloned_ndarray]
200 | other_arrays = [a for a in arrays if not ((hasattr(a, "arraytype") and a.arraytype is cloned_ndarray) or type(a) is cloned_ndarray)]
201 |
202 | cloned_arrays_ids = [id(a) for a in cloned_arrays]
203 | other_arrays_ids = [id(a) for a in other_arrays]
204 |
205 | for fragments in fragment(*other_arrays):
206 | it_other_arrays = iter(other_arrays)
207 | it_cloned_arrays = iter(cloned_arrays)
208 |
209 | array_names = []
210 | for a in arrays:
211 | if id(a) in cloned_arrays_ids:
212 | array_names.append(repr(it_cloned_arrays.next()))
213 | continue
214 | if id(a) in other_arrays_ids:
215 | array_names.append(repr(it_other_arrays.next()))
216 | continue
217 |
218 | targets_results = view.apply(interactive(mapReduce_wrapper),\
219 | interactive(map_func), reduce_op.__name__, array_names, **kwargs)
220 |
221 | fragment_result = reduce_op.reduce(targets_results) # reduce over targets' results
222 | if result is None:
223 | result = fragment_result
224 | else:
225 | result = reduce_op(result, fragment_result)
226 | else:
227 | array_names = [repr(a) for a in arrays]
228 | targets_results = view.apply(interactive(mapReduce_wrapper),\
229 | interactive(map_func), reduce_op.__name__, array_names, **kwargs)
230 |
231 | result = reduce_op.reduce(targets_results) # reduce over targets' results
232 |
233 | view.targets = tmp_targets # restore target list
234 |
235 | return result
236 |
237 | def __tree_reduce(array, axis=None, op=np.add):
238 | # reduce all axes
239 | if axis is None:
240 | result = array
241 | for axis in range(len(array.shape))[::-1]:
242 | result = __tree_reduce(result, axis=axis, op=op)
243 | return result
244 |
245 | assert 0 <= axis and axis < len(array.shape)
246 | result = np.rollaxis(array, axis)
247 |
248 | while True:
249 | l = result.shape[0]
250 |
251 | # if axis is small enough to neglect rounding errors do a faster numpy-reduce
252 | if l < 10000:
253 | return op.reduce(result, axis=0)
254 |
255 | if l % 2 == 0:
256 | result = op(result[0:l/2], result[l/2:])
257 | else:
258 | tmp = result[-1]
259 | result = op(result[0:l/2], result[l/2:-1])
260 | result[0] = op(result[0], tmp)
261 |
--------------------------------------------------------------------------------
/pyDive/arrays/__init__.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2015 Heiko Burau
3 |
4 | This file is part of pyDive.
5 |
6 | pyDive is free software: you can redistribute it and/or modify
7 | it under the terms of of either the GNU General Public License or
8 | the GNU Lesser General Public License as published by
9 | the Free Software Foundation, either version 3 of the License, or
10 | (at your option) any later version.
11 | pyDive is distributed in the hope that it will be useful,
12 | but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | GNU General Public License and the GNU Lesser General Public License
15 | for more details.
16 |
17 | You should have received a copy of the GNU General Public License
18 | and the GNU Lesser General Public License along with pyDive.
19 | If not, see .
20 | """
21 | __doc__ = None
22 |
23 | # -*- coding: utf-8 -*-
24 | __all__ = ["h5_ndarray.py", "ndarray.py"]
--------------------------------------------------------------------------------
/pyDive/arrays/ad_ndarray.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2015 Heiko Burau
3 |
4 | This file is part of pyDive.
5 |
6 | pyDive is free software: you can redistribute it and/or modify
7 | it under the terms of of either the GNU General Public License or
8 | the GNU Lesser General Public License as published by
9 | the Free Software Foundation, either version 3 of the License, or
10 | (at your option) any later version.
11 | pyDive is distributed in the hope that it will be useful,
12 | but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | GNU General Public License and the GNU Lesser General Public License
15 | for more details.
16 |
17 | You should have received a copy of the GNU General Public License
18 | and the GNU Lesser General Public License along with pyDive.
19 | If not, see .
20 | """
21 | __doc__ = None
22 |
23 | import numpy as np
24 | import adios as ad
25 | from pyDive.arrays.ndarray import hollow_like
26 | import pyDive.distribution.multiple_axes as multiple_axes
27 | import pyDive.IPParallelClient as com
28 | from itertools import islice
29 | from .. import structured
30 | import pyDive.arrays.local.ad_ndarray
31 |
32 | ad_ndarray = multiple_axes.distribute(pyDive.arrays.local.ad_ndarray.ad_ndarray, "ad_ndarray", "ad_ndarray", may_allocate=False)
33 |
34 | def load(self):
35 | """Load array from file into main memory of all engines in parallel.
36 |
37 | :return: pyDive.ndarray instance
38 | """
39 | result = hollow_like(self)
40 | view = com.getView()
41 | view.execute("{0} = {1}.load()".format(result.name, self.name), targets=result.target_ranks)
42 | return result
43 | ad_ndarray.load = load
44 | del load
45 |
46 | def open_variable(filename, variable_path, distaxis=0):
47 | """Create a pyDive.adios.ad_ndarray instance from file.
48 |
49 | :param filename: name of adios file.
50 | :param variable_path: path within adios file to a single variable.
51 | :param distaxis int: distributed axis
52 | :return: pyDive.adios.ad_ndarray instance
53 | """
54 | fileHandle = ad.file(filename)
55 | variable = fileHandle.var[variable_path]
56 | dtype = variable.type
57 | shape = tuple(variable.dims)
58 | fileHandle.close()
59 |
60 | result = ad_ndarray(shape, dtype, distaxis, None, None, True)
61 |
62 | target_shapes = result.target_shapes()
63 | target_offset_vectors = result.target_offset_vectors()
64 |
65 | view = com.getView()
66 | view.scatter("shape", target_shapes, targets=result.target_ranks)
67 | view.scatter("offset", target_offset_vectors, targets=result.target_ranks)
68 | view.execute("{0} = pyDive.arrays.local.ad_ndarray.ad_ndarray('{1}','{2}',shape=shape[0],offset=offset[0])"\
69 | .format(result.name, filename, variable_path), targets=result.target_ranks)
70 |
71 | return result
72 |
73 | def open(filename, datapath, distaxis=0):
74 | """Create a pyDive.adios.ad_ndarray instance respectively a structure of
75 | pyDive.adios.ad_ndarray instances from file.
76 |
77 | :param filename: name of adios file.
78 | :param datapath: path within adios file to a single variable or a group of variables.
79 | :param distaxis int: distributed axis
80 | :return: pyDive.adios.ad_ndarray instance
81 | """
82 | fileHandle = ad.file(filename)
83 | variable_paths = fileHandle.var.keys()
84 | fileHandle.close()
85 |
86 | def update_tree(tree, variable_path, variable_path_iter, leaf):
87 | node = variable_path_iter.next()
88 | if node == leaf:
89 | tree[leaf] = open_variable(filename, variable_path, distaxis)
90 | return
91 | tree[node] = {}
92 | update_tree(tree[node], variable_path, variable_path_iter, leaf)
93 |
94 | n = len(datapath.split("/"))
95 |
96 | structOfArrays = {}
97 | for variable_path in variable_paths:
98 | if not variable_path.startswith(datapath):
99 | continue
100 | path_nodes = variable_path.split("/")
101 | path_nodes_it = iter(path_nodes)
102 |
103 | # advance 'path_nodes_it' n times
104 | next(islice(path_nodes_it, n, n), None)
105 |
106 | update_tree(structOfArrays, variable_path, path_nodes_it, path_nodes[-1])
107 |
108 | return structured.structured(structOfArrays)
109 |
--------------------------------------------------------------------------------
/pyDive/arrays/gpu_ndarray.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2015 Heiko Burau
3 |
4 | This file is part of pyDive.
5 |
6 | pyDive is free software: you can redistribute it and/or modify
7 | it under the terms of of either the GNU General Public License or
8 | the GNU Lesser General Public License as published by
9 | the Free Software Foundation, either version 3 of the License, or
10 | (at your option) any later version.
11 | pyDive is distributed in the hope that it will be useful,
12 | but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | GNU General Public License and the GNU Lesser General Public License
15 | for more details.
16 |
17 | You should have received a copy of the GNU General Public License
18 | and the GNU Lesser General Public License along with pyDive.
19 | If not, see .
20 | """
21 | __doc__ = None
22 |
23 | import numpy as np
24 | import pyDive.distribution.multiple_axes as multiple_axes
25 | from pyDive.distribution.interengine import GPU_copier
26 | import pyDive.arrays.local.gpu_ndarray
27 |
28 | gpu_ndarray = multiple_axes.distribute(pyDive.arrays.local.gpu_ndarray.gpu_ndarray, "gpu_ndarray",\
29 | "pyDive.arrays.local.gpu_ndarray", interengine_copier=GPU_copier)
30 |
31 | factories = multiple_axes.generate_factories(gpu_ndarray, ("empty", "zeros"), np.float)
32 | factories.update(multiple_axes.generate_factories_like(gpu_ndarray, ("empty_like", "zeros_like")))
33 | globals().update(factories)
34 |
35 | def ones(shape, dtype=np.float, distaxes='all', **kwargs):
36 | result = zeros(shape, dtype, distaxes, **kwargs)
37 | result += 1
38 | return result
39 |
40 | def ones_like(other, **kwargs):
41 | result = zeros_like(other, **kwargs)
42 | result += 1
43 | return result
44 |
45 | import pyDive.IPParallelClient as com
46 | import pyDive.arrays.ndarray
47 |
48 | def to_cpu(self):
49 | """Copy array data to cpu main memory.
50 |
51 | :result pyDive.ndarray: distributed cpu array.
52 | """
53 | result = pyDive.arrays.ndarray.hollow_like(self)
54 | view = com.getView()
55 | view.execute("{0} = {1}.to_cpu()".format(result.name, self.name), targets=result.target_ranks)
56 | return result
57 | gpu_ndarray.to_cpu = to_cpu
58 | del to_cpu
59 |
60 | def hollow(shape, dtype=np.float, distaxes='all'):
61 | """Create a pyDive.gpu_ndarray instance distributed across all engines without allocating a local
62 | gpu-array.
63 |
64 | :param ints shape: shape of array
65 | :param dtype: datatype of a single element
66 | :param ints distaxes: distributed axes. Defaults to 'all' meaning each axis is distributed.
67 | """
68 | return gpu_ndarray(shape, dtype, distaxes, None, None, True)
69 |
70 | def hollow_like(other):
71 | """Create a pyDive.gpu_ndarray instance with the same
72 | shape, distribution and type as ``other`` without allocating a local gpu-array.
73 | """
74 | return gpu_ndarray(other.shape, other.dtype, other.distaxes, other.target_offsets, other.target_ranks, True)
75 |
76 | def array(array_like, distaxes='all'):
77 | """Create a pyDive.gpu_ndarray instance from an array-like object.
78 |
79 | :param array_like: Any object exposing the array interface, e.g. numpy-array, python sequence, ...
80 | :param ints distaxis: distributed axes. Defaults to 'all' meaning each axis is distributed.
81 | """
82 | result_cpu = pyDive.arrays.ndarray.array(array_like, distaxes)
83 | result = hollow_like(result_cpu)
84 | view = com.getView()
85 | view.execute("{0} = pyDive.arrays.local.gpu_ndarray.gpu_ndarray_cast(pycuda.gpuarray.to_gpu({1}))"\
86 | .format(repr(result), repr(result_cpu)), targets=result.target_ranks)
87 |
88 | return result
89 |
90 | #ufunc_names = [key for key, value in np.__dict__.items() if isinstance(value, np.ufunc)]
91 | #ufuncs = multiple_axes.generate_ufuncs(ufunc_names, "np")
92 |
93 | #globals().update(ufuncs)
--------------------------------------------------------------------------------
/pyDive/arrays/h5_ndarray.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2014 Heiko Burau
3 |
4 | This file is part of pyDive.
5 |
6 | pyDive is free software: you can redistribute it and/or modify
7 | it under the terms of of either the GNU General Public License or
8 | the GNU Lesser General Public License as published by
9 | the Free Software Foundation, either version 3 of the License, or
10 | (at your option) any later version.
11 | pyDive is distributed in the hope that it will be useful,
12 | but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | GNU General Public License and the GNU Lesser General Public License
15 | for more details.
16 |
17 | You should have received a copy of the GNU General Public License
18 | and the GNU Lesser General Public License along with pyDive.
19 | If not, see .
20 | """
21 | __doc__ = None
22 |
23 | import numpy as np
24 | import h5py as h5
25 | from pyDive.arrays.ndarray import hollow_like
26 | import pyDive.distribution.multiple_axes as multiple_axes
27 | import pyDive.IPParallelClient as com
28 | from .. import structured
29 | import pyDive.arrays.local.h5_ndarray
30 |
31 | h5_ndarray = multiple_axes.distribute(pyDive.arrays.local.h5_ndarray.h5_ndarray, "h5_ndarray", "h5_ndarray", may_allocate=False)
32 |
33 | def load(self):
34 | """Load array from file into main memory of all engines in parallel.
35 |
36 | :return: pyDive.ndarray instance
37 | """
38 | result = hollow_like(self)
39 | view = com.getView()
40 | view.execute("{0} = {1}.load()".format(result.name, self.name), targets=result.target_ranks)
41 | return result
42 | h5_ndarray.load = load
43 | del load
44 |
45 | def open_dset(filename, dataset_path, distaxes='all'):
46 | """Create a pyDive.h5.h5_ndarray instance from file.
47 |
48 | :param filename: name of hdf5 file.
49 | :param dataset_path: path within hdf5 file to a single dataset.
50 | :param distaxes ints: distributed axes. Defaults to 'all' meaning each axis is distributed.
51 | :return: pyDive.h5.h5_ndarray instance
52 | """
53 | fileHandle = h5.File(filename, "r")
54 | dataset = fileHandle[dataset_path]
55 | dtype = dataset.dtype
56 | shape = dataset.shape
57 | fileHandle.close()
58 |
59 | result = h5_ndarray(shape, dtype, distaxes, None, None, True)
60 |
61 | target_shapes = result.target_shapes()
62 | target_offset_vectors = result.target_offset_vectors()
63 |
64 | view = com.getView()
65 | view.scatter("shape", target_shapes, targets=result.target_ranks)
66 | view.scatter("offset", target_offset_vectors, targets=result.target_ranks)
67 | view.execute("{0} = pyDive.arrays.local.h5_ndarray.h5_ndarray('{1}','{2}',shape=shape[0],offset=offset[0])"\
68 | .format(result.name, filename, dataset_path), targets=result.target_ranks)
69 |
70 | return result
71 |
72 | def open(filename, datapath, distaxes='all'):
73 | """Create an pyDive.h5.h5_ndarray instance respectively a structure of
74 | pyDive.h5.h5_ndarray instances from file.
75 |
76 | :param filename: name of hdf5 file.
77 | :param dataset_path: path within hdf5 file to a single dataset or hdf5 group.
78 | :param distaxes ints: distributed axes. Defaults to 'all' meaning each axis is distributed.
79 | :return: pyDive.h5.h5_ndarray instance / structure of pyDive.h5.h5_ndarray instances
80 | """
81 | hFile = h5.File(filename, 'r')
82 | datapath = datapath.rstrip("/")
83 | group_or_dataset = hFile[datapath]
84 | if type(group_or_dataset) is not h5._hl.group.Group:
85 | # dataset
86 | return open_dset(filename, datapath, distaxes)
87 |
88 | def create_tree(group, tree, dataset_path):
89 | for key, value in group.items():
90 | # group
91 | if type(value) is h5._hl.group.Group:
92 | tree[key] = {}
93 | create_tree(value, tree[key], dataset_path + "/" + key)
94 | # dataset
95 | else:
96 | tree[key] = open_dset(filename, dataset_path + "/" + key, distaxes)
97 |
98 | group = group_or_dataset
99 | structOfArrays = {}
100 | create_tree(group, structOfArrays, datapath)
101 | return structured.structured(structOfArrays)
102 |
--------------------------------------------------------------------------------
/pyDive/arrays/local/__init__.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | """
3 | Copyright 2015 Heiko Burau
4 |
5 | This file is part of pyDive.
6 |
7 | pyDive is free software: you can redistribute it and/or modify
8 | it under the terms of of either the GNU General Public License or
9 | the GNU Lesser General Public License as published by
10 | the Free Software Foundation, either version 3 of the License, or
11 | (at your option) any later version.
12 | pyDive is distributed in the hope that it will be useful,
13 | but WITHOUT ANY WARRANTY; without even the implied warranty of
14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 | GNU General Public License and the GNU Lesser General Public License
16 | for more details.
17 |
18 | You should have received a copy of the GNU General Public License
19 | and the GNU Lesser General Public License along with pyDive.
20 | If not, see .
21 | """
22 | __doc__ = None
--------------------------------------------------------------------------------
/pyDive/arrays/local/ad_ndarray.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2015 Heiko Burau
3 |
4 | This file is part of pyDive.
5 |
6 | pyDive is free software: you can redistribute it and/or modify
7 | it under the terms of of either the GNU General Public License or
8 | the GNU Lesser General Public License as published by
9 | the Free Software Foundation, either version 3 of the License, or
10 | (at your option) any later version.
11 | pyDive is distributed in the hope that it will be useful,
12 | but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | GNU General Public License and the GNU Lesser General Public License
15 | for more details.
16 |
17 | You should have received a copy of the GNU General Public License
18 | and the GNU Lesser General Public License along with pyDive.
19 | If not, see .
20 | """
21 | __doc__ = None
22 |
23 |
24 | import adios as ad
25 | import pyDive.distribution.helper as helper
26 | import numpy as np
27 |
28 | class ad_ndarray(object):
29 |
30 | def __init__(self, filename, variable_path, shape=None, window=None, offset=None):
31 | self.filename = filename
32 | self.variable_path = variable_path
33 |
34 | fileHandle = ad.file(filename)
35 | variable = fileHandle.var[variable_path]
36 | self.dtype = variable.type
37 | if shape is None:
38 | shape = variable.dims
39 | self.shape = tuple(shape)
40 | fileHandle.close()
41 |
42 | if window is None:
43 | window = [slice(0, s, 1) for s in shape]
44 | self.window = tuple(window)
45 | if offset is None:
46 | offset = (0,) * len(shape)
47 | self.offset = tuple(offset)
48 |
49 | #: total bytes consumed by the elements of the array.
50 | self.nbytes = np.dtype(self.dtype).itemsize * np.prod(self.shape)
51 |
52 | def load(self):
53 | begin = []
54 | size = []
55 | for o, w in zip(self.offset, self.window):
56 | if type(w) is int:
57 | begin.append(o+w)
58 | size.append(1)
59 | else:
60 | begin.append(o+w.start)
61 | size.append(w.stop-w.start)
62 |
63 | fileHandle = ad.file(self.filename)
64 | variable = fileHandle.var[self.variable_path]
65 | result = variable.read(tuple(begin), tuple(size))
66 | fileHandle.close()
67 |
68 | # remove all single dimensional axes unless they are a result of slicing, i.e. a[n,n+1]
69 | single_dim_axes = [axis for axis in range(len(self.window)) if type(self.window[axis]) is int]
70 | if single_dim_axes:
71 | result = np.squeeze(result, axis=single_dim_axes)
72 |
73 | return result
74 |
75 | def __getitem__(self, args):
76 | if args == slice(None):
77 | args = (slice(None),) * len(self.shape)
78 |
79 | if not isinstance(args, list) and not isinstance(args, tuple):
80 | args = [args]
81 |
82 | assert len(args) == len(self.shape),\
83 | "number of arguments (%d) does not correspond to the dimension (%d)"\
84 | % (len(args), len(self.shape))
85 |
86 | assert not all(type(arg) is int for arg in args),\
87 | "single data access is not supported by adios"
88 |
89 | assert all(arg.step == 1 or arg.step == None for arg in args if type(arg) is slice),\
90 | "strided access in not supported by adios"
91 |
92 | result_shape, clean_view = helper.view_of_shape(self.shape, args)
93 |
94 | # Applying 'clean_view' after 'self.window', results in 'result_window'
95 | result_window = helper.view_of_view(self.window, clean_view)
96 |
97 | return ad_ndarray(self.filename, self.variable_path, result_shape, result_window, self.offset)
98 |
--------------------------------------------------------------------------------
/pyDive/arrays/local/gpu_ndarray.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2015 Heiko Burau
3 |
4 | This file is part of pyDive.
5 |
6 | pyDive is free software: you can redistribute it and/or modify
7 | it under the terms of of either the GNU General Public License or
8 | the GNU Lesser General Public License as published by
9 | the Free Software Foundation, either version 3 of the License, or
10 | (at your option) any later version.
11 | pyDive is distributed in the hope that it will be useful,
12 | but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | GNU General Public License and the GNU Lesser General Public License
15 | for more details.
16 |
17 | You should have received a copy of the GNU General Public License
18 | and the GNU Lesser General Public License along with pyDive.
19 | If not, see .
20 | """
21 | __doc__ = None
22 |
23 | import pycuda.gpuarray
24 | import pycuda.driver as cuda
25 | import numpy as np
26 |
27 | def copy_non_contiguous(dst, src):
28 | """Copy ``src`` array to ``dst`` array. A gpu-array may have a non contiguous block of memory,
29 | i.e. it may have substancial pitches/strides. However a cpu-array must have a contiguous block of memory.
30 | All four directions are allowed.
31 | """
32 | assert src.dtype == dst.dtype,\
33 | "src ({}) and dst ({}) must have the same datatype.".format(str(src.dtype), str(dst.dtype))
34 | assert dst.shape == src.shape,\
35 | "Shapes do not match: " + str(dst.shape) + " <-> " + str(src.shape)
36 |
37 | itemsize = np.dtype(src.dtype).itemsize
38 | copy = cuda.Memcpy2D()
39 | src_on_gpu = isinstance(src, pycuda.gpuarray.GPUArray)
40 | dst_on_gpu = isinstance(dst, pycuda.gpuarray.GPUArray)
41 | if src_on_gpu:
42 | copy.set_src_device(src.gpudata)
43 | else:
44 | copy.set_src_host(src)
45 | if dst_on_gpu:
46 | copy.set_dst_device(dst.gpudata)
47 | else:
48 | copy.set_dst_host(dst)
49 |
50 | if len(src.shape) == 1:
51 | copy.src_pitch = src.strides[0] if src_on_gpu else itemsize
52 | copy.dst_pitch = dst.strides[0] if dst_on_gpu else itemsize
53 | copy.width_in_bytes = itemsize
54 | copy.height = src.shape[0]
55 | copy(aligned=False)
56 |
57 | elif len(src.shape) == 2:
58 | if (itemsize != src.strides[1] if src_on_gpu else False) or \
59 | (itemsize != dst.strides[1] if dst_on_gpu else False):
60 | # arrays have to be copied column by column, because there a two substantial pitches/strides
61 | # which is not supported by cuda.
62 | copy.src_pitch = src.strides[0] if src_on_gpu else itemsize
63 | copy.dst_pitch = dst.strides[0] if dst_on_gpu else itemsize
64 | copy.width_in_bytes = itemsize
65 | copy.height = src.shape[0]
66 |
67 | for col in range(src.shape[1]):
68 | copy.src_x_in_bytes = col * src.strides[1] if src_on_gpu else col * itemsize
69 | copy.dst_x_in_bytes = col * dst.strides[1] if dst_on_gpu else col * itemsize
70 | copy(aligned=False)
71 | else:
72 | # both arrays have a contiguous block of memory for each row
73 | copy.src_pitch = src.strides[0] if src_on_gpu else itemsize * src.shape[1]
74 | copy.dst_pitch = dst.strides[0] if dst_on_gpu else itemsize * src.shape[1]
75 | copy.width_in_bytes = itemsize * src.shape[1]
76 | copy.height = src.shape[0]
77 | copy(aligned=False)
78 |
79 | elif len(src.shape) == 3:
80 | if (src.strides[0] != src.shape[1] * src.strides[1] if src_on_gpu else False) or \
81 | (dst.strides[0] != dst.shape[1] * dst.strides[1] if dst_on_gpu else False):
82 | # arrays have to be copied plane by plane, because there a substantial pitche/stride
83 | # for the z-axis which is not supported by cuda.
84 | for plane in range(src.shape[0]):
85 | copy_non_contiguous(dst[plane,:,:], src[plane,:,:])
86 | return
87 |
88 | copy = cuda.Memcpy3D()
89 | if src_on_gpu:
90 | copy.set_src_device(src.gpudata)
91 | else:
92 | copy.set_src_host(src)
93 | if dst_on_gpu:
94 | copy.set_dst_device(dst.gpudata)
95 | else:
96 | copy.set_dst_host(dst)
97 |
98 | copy.src_pitch = src.strides[1] if src_on_gpu else itemsize * src.shape[2]
99 | copy.dst_pitch = dst.strides[1] if dst_on_gpu else itemsize * src.shape[2]
100 | copy.width_in_bytes = itemsize * src.shape[2]
101 | copy.height = copy.src_height = copy.dst_height = src.shape[1]
102 | copy.depth = src.shape[0]
103 |
104 | copy()
105 | else:
106 | raise RuntimeError("dimension %d is not supported." % len(src.shape))
107 |
108 |
109 | class gpu_ndarray(pycuda.gpuarray.GPUArray):
110 |
111 | def __init__(self, shape, dtype, allocator=cuda.mem_alloc,\
112 | base=None, gpudata=None, strides=None, order="C"):
113 | if type(shape) not in (list, tuple):
114 | shape = (shape,)
115 | elif type(shape) is not tuple:
116 | shape = tuple(shape)
117 | super(gpu_ndarray, self).__init__(shape, np.dtype(dtype), allocator, base, gpudata, strides, order)
118 |
119 | def __setitem__(self, key, other):
120 | # if args is [:] then assign `other` to the entire ndarray
121 | if key == slice(None):
122 | if isinstance(other, pycuda.gpuarray.GPUArray):
123 | if self.flags.forc and other.flags.forc:
124 | # both arrays are a contiguous block of memory
125 | cuda.memcpy_dtod(self.gpudata, other.gpudata, self.nbytes)
126 | return
127 | else:
128 | if self.flags.forc:
129 | # both arrays are a contiguous block of memory
130 | cuda.memcpy_htod(self.gpudata, other)
131 | return
132 |
133 | copy_non_contiguous(self, other)
134 | return
135 |
136 | # assign `other` to sub-array of self
137 | sub_array = self[key]
138 | sub_array[:] = other
139 |
140 | def __cast_from_base(self, pycuda_array):
141 | return gpu_ndarray(pycuda_array.shape, pycuda_array.dtype, pycuda_array.allocator,\
142 | pycuda_array.base, pycuda_array.gpudata, pycuda_array.strides)
143 |
144 | def __getitem__(self, args):
145 | pycuda_array = super(gpu_ndarray, self).__getitem__(tuple(args))
146 | return self.__cast_from_base(pycuda_array)
147 |
148 | def copy(self):
149 | if self.flags.forc:
150 | return super(gpu_ndarray, self).copy()
151 | result = gpu_ndarray(shape=self.shape, dtype=self.dtype, allocator=self.allocator)
152 | result[:] = self
153 | return result
154 |
155 | def to_cpu(self):
156 | if self.flags.forc:
157 | return self.get(pagelocked=True)
158 |
159 | result = cuda.pagelocked_empty(self.shape, self.dtype)
160 | copy_non_contiguous(result, self)
161 | return result
162 |
163 | def __elementwise_op__(self, op, *args):
164 | # if arrays are not contiguous make a contiguous copy
165 | my_array = self.copy() if not self.flags.forc else self
166 |
167 | args = [arg.copy() if isinstance(arg, pycuda.gpuarray.GPUArray) and not arg.flags.forc else arg for arg in args]
168 | pycuda_array = getattr(super(gpu_ndarray, my_array), op)(*args)
169 | return self.__cast_from_base(pycuda_array)
170 |
171 | # add special operations like __add__, __mul__, etc. to `gpu_ndarray`
172 |
173 | binary_ops = ["add", "sub", "mul", "floordiv", "div", "mod", "pow", "lshift", "rshift", "and", "xor", "or"]
174 |
175 | binary_iops = ["__i" + op + "__" for op in binary_ops]
176 | binary_rops = ["__r" + op + "__" for op in binary_ops]
177 | binary_ops = ["__" + op + "__" for op in binary_ops]
178 | unary_ops = ["__neg__", "__pos__", "__abs__", "__invert__", "__complex__", "__int__", "__long__", "__float__", "__oct__", "__hex__"]
179 | comp_ops = ["__lt__", "__le__", "__eq__", "__ne__", "__ge__", "__gt__"]
180 |
181 | special_ops_avail = set(name for name in pycuda.gpuarray.GPUArray.__dict__.keys() if name.endswith("__"))
182 |
183 | make_special_op = lambda op: lambda self, *args: self.__elementwise_op__(op, *args)
184 |
185 | special_ops_dict = {op : make_special_op(op) for op in \
186 | set(binary_ops + binary_rops + binary_iops + unary_ops + comp_ops) & special_ops_avail}
187 |
188 | from types import MethodType
189 |
190 | for name, func in special_ops_dict.items():
191 | setattr(gpu_ndarray, name, MethodType(func, None, gpu_ndarray))
192 |
193 | # -------------------- factories -----------------------------------
194 |
195 | # get a `gpu_ndarray` instance out of a `pycuda.gpuarray.GPUArray` instance
196 | def gpu_ndarray_cast(pycuda_array):
197 | return gpu_ndarray(pycuda_array.shape, pycuda_array.dtype, pycuda_array.allocator,\
198 | pycuda_array.base, pycuda_array.gpudata, pycuda_array.strides)
199 |
200 | def empty(shape, dtype):
201 | return gpu_ndarray(shape, dtype)
202 |
203 | pycuda_factories = ("zeros", "empty_like", "zeros_like")
204 |
205 | # wrap all factory functions from pycuda so that they return a `gpu_ndarray` instance
206 | from functools import wraps
207 | make_factory = lambda func : wraps(func)(lambda *args, **kwargs: gpu_ndarray_cast(func(*args, **kwargs)))
208 | factories = {func : make_factory(getattr(pycuda.gpuarray, func)) for func in pycuda_factories}
209 |
210 | globals().update(factories)
211 |
--------------------------------------------------------------------------------
/pyDive/arrays/local/h5_ndarray.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2015 Heiko Burau
3 |
4 | This file is part of pyDive.
5 |
6 | pyDive is free software: you can redistribute it and/or modify
7 | it under the terms of of either the GNU General Public License or
8 | the GNU Lesser General Public License as published by
9 | the Free Software Foundation, either version 3 of the License, or
10 | (at your option) any later version.
11 | pyDive is distributed in the hope that it will be useful,
12 | but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | GNU General Public License and the GNU Lesser General Public License
15 | for more details.
16 |
17 | You should have received a copy of the GNU General Public License
18 | and the GNU Lesser General Public License along with pyDive.
19 | If not, see .
20 | """
21 | __doc__ = None
22 |
23 | import h5py as h5
24 | import pyDive.distribution.helper as helper
25 | import numpy as np
26 |
27 | class h5_ndarray(object):
28 |
29 | def __init__(self, filename, dataset_path, shape=None, window=None, offset=None):
30 | self.filename = filename
31 | self.dataset_path = dataset_path
32 |
33 | fileHandle = h5.File(filename, "r")
34 | dataset = fileHandle[dataset_path]
35 | #self.attrs = dataset.attrs
36 | #: datatype of a single data value
37 | self.dtype = dataset.dtype
38 | if shape is None:
39 | shape = dataset.shape
40 | self.shape = tuple(shape)
41 | fileHandle.close()
42 |
43 | if window is None:
44 | window = [slice(0, s, 1) for s in shape]
45 | self.window = tuple(window)
46 | if offset is None:
47 | offset = (0,) * len(shape)
48 | self.offset = tuple(offset)
49 |
50 | #: total bytes consumed by the elements of the array.
51 | self.nbytes = self.dtype.itemsize * np.prod(self.shape)
52 |
53 | def load(self):
54 | window = list(self.window)
55 | for i in range(len(window)):
56 | if type(window[i]) is int:
57 | window[i] += self.offset[i]
58 | else:
59 | window[i] = slice(window[i].start + self.offset[i], window[i].stop + self.offset[i], window[i].step)
60 |
61 | fileHandle = h5.File(self.filename, "r")
62 | dataset = fileHandle[self.dataset_path]
63 | result = dataset[tuple(window)]
64 | fileHandle.close()
65 | return result
66 |
67 | def __getitem__(self, args):
68 | if args == slice(None):
69 | args = (slice(None),) * len(self.shape)
70 |
71 | if not isinstance(args, list) and not isinstance(args, tuple):
72 | args = [args]
73 |
74 | assert len(args) == len(self.shape),\
75 | "number of arguments (%d) does not correspond to the dimension (%d)"\
76 | % (len(args), len(self.shape))
77 |
78 | assert not all(type(arg) is int for arg in args),\
79 | "single data access is not supported"
80 |
81 | result_shape, clean_view = helper.view_of_shape(self.shape, args)
82 |
83 | # Applying 'clean_view' after 'self.window', results in 'result_window'
84 | result_window = helper.view_of_view(self.window, clean_view)
85 |
86 | return h5_ndarray(self.filename, self.dataset_path, result_shape, result_window, self.offset)
87 |
--------------------------------------------------------------------------------
/pyDive/arrays/ndarray.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2014 Heiko Burau
3 |
4 | This file is part of pyDive.
5 |
6 | pyDive is free software: you can redistribute it and/or modify
7 | it under the terms of of either the GNU General Public License or
8 | the GNU Lesser General Public License as published by
9 | the Free Software Foundation, either version 3 of the License, or
10 | (at your option) any later version.
11 | pyDive is distributed in the hope that it will be useful,
12 | but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | GNU General Public License and the GNU Lesser General Public License
15 | for more details.
16 |
17 | You should have received a copy of the GNU General Public License
18 | and the GNU Lesser General Public License along with pyDive.
19 | If not, see .
20 | """
21 | __doc__ = None
22 |
23 | import numpy as np
24 | import pyDive.distribution.multiple_axes as multiple_axes
25 | from pyDive.distribution.interengine import MPI_copier
26 |
27 | ndarray = multiple_axes.distribute(np.ndarray, "ndarray", "np", interengine_copier=MPI_copier)
28 |
29 | factories = multiple_axes.generate_factories(ndarray, ("empty", "zeros", "ones"), np.float)
30 | factories.update(multiple_axes.generate_factories_like(ndarray, ("empty_like", "zeros_like", "ones_like")))
31 |
32 | globals().update(factories)
33 |
34 | def array(array_like, distaxes='all'):
35 | """Create a pyDive.ndarray instance from an array-like object.
36 |
37 | :param array_like: Any object exposing the array interface, e.g. numpy-array, python sequence, ...
38 | :param ints distaxes: distributed axes. Defaults to 'all' meaning each axis is distributed.
39 | """
40 | np_array = np.array(array_like)
41 | result = empty(np_array.shape, np_array.dtype, distaxes)
42 | result[:] = np_array
43 | return result
44 |
45 | def hollow(shape, dtype=np.float, distaxes='all'):
46 | """Create a pyDive.ndarray instance distributed across all engines without allocating a local
47 | numpy-array.
48 |
49 | :param ints shape: shape of array
50 | :param dtype: datatype of a single element
51 | :param ints distaxes: distributed axes. Defaults to 'all' meaning each axis is distributed.
52 | """
53 | return ndarray(shape, dtype, distaxes, None, None, True)
54 |
55 | def hollow_like(other):
56 | """Create a pyDive.ndarray instance with the same
57 | shape, distribution and type as ``other`` without allocating a local numpy-array.
58 | """
59 | return ndarray(other.shape, other.dtype, other.distaxes, other.target_offsets, other.target_ranks, True)
60 |
61 | factories.update({"array" : array, "hollow" : hollow, "hollow_like" : hollow_like})
62 |
63 | ufunc_names = [key for key, value in np.__dict__.items() if isinstance(value, np.ufunc)]
64 | ufuncs = multiple_axes.generate_ufuncs(ufunc_names, "np")
65 |
66 | globals().update(ufuncs)
--------------------------------------------------------------------------------
/pyDive/cloned_ndarray/__init__.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2014 Heiko Burau
3 |
4 | This file is part of pyDive.
5 |
6 | pyDive is free software: you can redistribute it and/or modify
7 | it under the terms of of either the GNU General Public License or
8 | the GNU Lesser General Public License as published by
9 | the Free Software Foundation, either version 3 of the License, or
10 | (at your option) any later version.
11 | pyDive is distributed in the hope that it will be useful,
12 | but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | GNU General Public License and the GNU Lesser General Public License
15 | for more details.
16 |
17 | You should have received a copy of the GNU General Public License
18 | and the GNU Lesser General Public License along with pyDive.
19 | If not, see .
20 | """
21 | __doc__ = None
22 |
23 | # -*- coding: utf-8 -*-
24 | __all__ = ["cloned_ndarray", "factories"]
--------------------------------------------------------------------------------
/pyDive/cloned_ndarray/cloned_ndarray.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2014 Heiko Burau
3 |
4 | This file is part of pyDive.
5 |
6 | pyDive is free software: you can redistribute it and/or modify
7 | it under the terms of of either the GNU General Public License or
8 | the GNU Lesser General Public License as published by
9 | the Free Software Foundation, either version 3 of the License, or
10 | (at your option) any later version.
11 | pyDive is distributed in the hope that it will be useful,
12 | but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | GNU General Public License and the GNU Lesser General Public License
15 | for more details.
16 |
17 | You should have received a copy of the GNU General Public License
18 | and the GNU Lesser General Public License along with pyDive.
19 | If not, see .
20 | """
21 | __doc__ = None
22 |
23 | from .. import IPParallelClient as com
24 | import numpy as np
25 |
26 | cloned_ndarray_id = 0
27 |
28 | class cloned_ndarray(object):
29 | """Represents a multidimensional, homogenous array of fixed-size elements which is cloned
30 | on the cluster nodes. *Cloned* means that every participating :term:`engine` holds an independent, local numpy-array
31 | of the user-defined shape. The user can then do e.g. some manual stuff on the local arrays or
32 | some computation with :mod:`pyDive.algorithm` on them.
33 |
34 | Note that there exists no 'original' array as the name might suggest but something like that can be
35 | generated by :meth:`merge`.
36 | """
37 | def __init__(self, shape, dtype=np.float, target_ranks='all', no_allocation=False):
38 | """Creates an :class:`pyDive.cloned_ndarray.cloned_ndarray.cloned_ndarray` instance.
39 | This is a low-level method for instanciating a cloned_array.
40 | Cloned arrays should be constructed using 'empty', 'zeros'
41 | or 'empty_targets_like' (see :mod:`pyDive.cloned_ndarray.factories`).
42 |
43 | :param ints shape: size of the array on each axis
44 | :param numpy-dtype dtype: datatype of a single data value
45 | :param ints target_ranks: list of :term:`engine`-ids that share this array.\
46 | Or 'all' for all engines.
47 | :param bool no_allocation: if ``True`` no actual memory, i.e. *numpy-array*, will be
48 | allocated on :term:`engine`. Useful when you want to assign an existing numpy array manually.
49 | """
50 | self.shape = list(shape)
51 | self.dtype = dtype
52 | self.nbytes = np.dtype(dtype).itemsize * np.prod(self.shape)
53 | self.target_ranks = target_ranks
54 | self.view = com.getView()
55 |
56 | if self.target_ranks == 'all':
57 | self.target_ranks = list(self.view.targets)
58 |
59 | # generate a unique variable name used on target representing this instance
60 | global cloned_ndarray_id
61 | self.name = 'cloned_ndarray' + str(cloned_ndarray_id)
62 | cloned_ndarray_id += 1
63 |
64 | if no_allocation:
65 | self.view.push({self.name : None}, targets=self.target_ranks)
66 | else:
67 | self.view.push({'myshape' : self.shape, 'dtype' : self.dtype}, targets=self.target_ranks)
68 | self.view.execute('%s = np.empty(myshape, dtype=dtype)' % self.name, targets=self.target_ranks)
69 |
70 | def __del__(self):
71 | self.view.execute('del %s' % self.name, targets=self.target_ranks)
72 |
73 | def __repr__(self):
74 | return self.name
75 |
76 | def __setitem__(self, key, value):
77 | # if args is [:] then assign value to the entire ndarray
78 | if key == slice(None):
79 | assert isinstance(value, np.ndarray), "assignment available for numpy-arrays only"
80 |
81 | view = com.getView()
82 | view.push({'np_array' : value}, targets=self.target_ranks)
83 | view.execute("%s = np_array.copy()" % self.name, targets=self.target_ranks)
84 |
85 | return
86 |
87 | if not isinstance(key, list) and not isinstance(key, tuple):
88 | key = (key,)
89 |
90 | assert len(key) == len(self.shape)
91 |
92 | # assign value to sub-array of self
93 | sub_array = self[key]
94 | sub_array[:] = value
95 |
96 | def __getitem__(self, args):
97 | if not isinstance(args, list) and not isinstance(args, tuple):
98 | args = (args,)
99 |
100 | assert len(args) == len(self.shape),\
101 | "number of arguments (%d) does not correspond to the dimension (%d)"\
102 | % (len(args), len(self.shape))
103 |
104 | # shape of the new sliced ndarray
105 | new_shape, clean_view = helper.view_of_shape(self.shape, args)
106 |
107 | result = pyDive.cloned.hollow_engines_like(new_shape, self.dtype, self)
108 |
109 | self.view.push({'args' : args}, targets=self.target_ranks)
110 | self.view.execute('%s = %s[args]' % (result.name, self.name), targets=self.target_ranks)
111 |
112 | return result
113 |
114 | def merge(self, op):
115 | """Merge all local arrays in a pair-wise operation into a single numpy-array.
116 |
117 | :param op: Merging operation. Expects two numpy-arrays and returns one.
118 | :return: merged numpy-array.
119 | """
120 | result = self.view.pull(self.name, targets=self.target_ranks[0])
121 | for target in self.target_ranks[1:]:
122 | result = op(result, self.view.pull(self.name, targets=target))
123 | return result
124 |
125 | def sum(self):
126 | """Add up all local arrays.
127 |
128 | :return: numpy-array.
129 | """
130 | return self.merge(lambda x, y: x+y)
131 |
--------------------------------------------------------------------------------
/pyDive/cloned_ndarray/factories.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2014 Heiko Burau
3 |
4 | This file is part of pyDive.
5 |
6 | pyDive is free software: you can redistribute it and/or modify
7 | it under the terms of of either the GNU General Public License or
8 | the GNU Lesser General Public License as published by
9 | the Free Software Foundation, either version 3 of the License, or
10 | (at your option) any later version.
11 | pyDive is distributed in the hope that it will be useful,
12 | but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | GNU General Public License and the GNU Lesser General Public License
15 | for more details.
16 |
17 | You should have received a copy of the GNU General Public License
18 | and the GNU Lesser General Public License along with pyDive.
19 | If not, see .
20 | """
21 |
22 | __doc__=\
23 | """This module holds high-level functions for instanciating :ref:`pyDive.cloned_ndarrays `."""
24 |
25 | import pyDive.cloned_ndarray.cloned_ndarray as cloned_ndarray
26 | from .. import IPParallelClient as com
27 | import numpy as np
28 |
29 | def hollow_engines_like(shape, dtype, a):
30 | """Return a new :obj:`pyDive.cloned_ndarray` utilizing the same engines *a* does
31 | without allocating a local *numpy-array*.
32 |
33 | :param ints shape: shape of the array
34 | :param numpy-dtype dtype: datatype of a single data value
35 | :param a: :ref:`pyDive.ndarray`
36 | """
37 | return cloned_ndarray.cloned_ndarray(shape, dtype, a.target_ranks, True)
38 |
39 | def empty_engines_like(shape, dtype, a):
40 | """Return a new :obj:`pyDive.cloned_ndarray` utilizing the same engines *a* does
41 | without initializing elements.
42 |
43 | :param ints shape: shape of the array
44 | :param numpy-dtype dtype: datatype of a single data value
45 | :param a: :ref:`pyDive.ndarray`
46 | """
47 | return cloned_ndarray.cloned_ndarray(shape, dtype, a.target_ranks)
48 |
49 | def zeros_engines_like(shape, dtype, a):
50 | """Return a new :ref:`pyDive.cloned_ndarray` utilizing the same engines *a* does
51 | filled with zeros.
52 |
53 | :param ints shape: shape of the array
54 | :param numpy-dtype dtype: datatype of a single data value
55 | :param a: :ref:`pyDive.ndarray`
56 | """
57 | result = cloned_ndarray.cloned_ndarray(shape, dtype, a.target_ranks, True)
58 | view = com.getView()
59 | view.push({'myshape' : shape, 'dtype' : dtype}, targets=result.target_ranks)
60 | view.execute('%s = np.zeros(myshape, dtype)' % repr(result), targets=result.target_ranks)
61 | return result
62 |
63 | def hollow(shape, dtype=np.float):
64 | """Return a new :ref:`pyDive.cloned_ndarray` utilizing all engines without allocating a local
65 | *numpy-array*.
66 |
67 | :param ints shape: shape of the array
68 | :param numpy-dtype dtype: datatype of a single data value
69 | """
70 | result = cloned_ndarray.cloned_ndarray(shape, dtype, target_ranks='all', no_allocation=True)
71 | return result
72 |
73 | def empty(shape, dtype=np.float):
74 | """Return a new :ref:`pyDive.cloned_ndarray` utilizing all engines without initializing elements.
75 |
76 | :param ints shape: shape of the array
77 | :param numpy-dtype dtype: datatype of a single data value
78 | """
79 | result = cloned_ndarray.cloned_ndarray(shape, dtype, target_ranks='all')
80 | return result
81 |
82 | def zeros(shape, dtype=np.float):
83 | """Return a new :ref:`pyDive.cloned_ndarray` utilizing all engines filled with zeros.
84 |
85 | :param ints shape: shape of the array
86 | :param numpy-dtype dtype: datatype of a single data value
87 | """
88 | result = cloned_ndarray.cloned_ndarray(shape, dtype, target_ranks='all', no_allocation=True)
89 | view = com.getView()
90 | view.push({'myshape' : shape, 'dtype' : dtype}, targets=result.target_ranks)
91 | view.execute('%s = np.zeros(myshape, dtype)' % repr(result), targets=result.target_ranks)
92 | return result
93 |
94 | def ones(shape, dtype=np.float):
95 | """Return a new :ref:`pyDive.cloned_ndarray` utilizing all engines filled with ones.
96 |
97 | :param ints shape: shape of the array
98 | :param numpy-dtype dtype: datatype of a single data value
99 | """
100 | result = cloned_ndarray.cloned_ndarray(shape, dtype, target_ranks='all', no_allocation=True)
101 | view = com.getView()
102 | view.push({'myshape' : shape, 'dtype' : dtype}, targets=result.target_ranks)
103 | view.execute('%s = np.ones(myshape, dtype)' % repr(result), targets=result.target_ranks)
104 | return result
--------------------------------------------------------------------------------
/pyDive/distribution/__init__.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
--------------------------------------------------------------------------------
/pyDive/distribution/helper.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2014 Heiko Burau
3 |
4 | This file is part of pyDive.
5 |
6 | pyDive is free software: you can redistribute it and/or modify
7 | it under the terms of of either the GNU General Public License or
8 | the GNU Lesser General Public License as published by
9 | the Free Software Foundation, either version 3 of the License, or
10 | (at your option) any later version.
11 | pyDive is distributed in the hope that it will be useful,
12 | but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | GNU General Public License and the GNU Lesser General Public License
15 | for more details.
16 |
17 | You should have received a copy of the GNU General Public License
18 | and the GNU Lesser General Public License along with pyDive.
19 | If not, see .
20 | """
21 | __doc__ = None
22 |
23 | import numpy as np
24 | from collections import OrderedDict
25 |
26 | def getFirstSliceIdx(slice_obj, begin, end):
27 | if slice_obj.start > begin:
28 | if slice_obj.start >= end: return None
29 | return slice_obj.start
30 | i = (begin-1 - slice_obj.start) / slice_obj.step + 1
31 | idx = slice_obj.start + i * slice_obj.step
32 | if idx >= end or idx >= slice_obj.stop: return None
33 | return idx
34 |
35 | def view_of_shape(shape, window):
36 | new_shape = []
37 | clean_view = []
38 | for s, w in zip(shape, window):
39 | if type(w) is int:
40 | clean_view.append(w)
41 | continue
42 | # create a clean, wrapped slice object
43 | clean_slice = slice(*w.indices(s))
44 | clean_view.append(clean_slice)
45 | # new size of axis i
46 | new_shape.append((clean_slice.stop-1 - clean_slice.start) / clean_slice.step + 1)
47 | return new_shape, clean_view
48 |
49 | def view_of_view(view, window):
50 | result_view = []
51 | window = iter(window)
52 | for v in view:
53 | if type(v) is int:
54 | result_view.append(v)
55 | continue
56 | w = window.next()
57 | if type(w) is int:
58 | result_view.append(v.start + w * v.step)
59 | continue
60 |
61 | result_view.append(slice(v.start + w.start * v.step, v.start + w.stop * v.step, v.step * w.step))
62 |
63 | return result_view
64 |
65 | # create local slice objects for each engine
66 | def createLocalSlices(slices, distaxis, target_offsets, shape):
67 | local_slices = [list(slices) for i in range(len(target_offsets))]
68 | distaxis_slice = slices[distaxis]
69 | for i in range(len(target_offsets)):
70 | begin = target_offsets[i]
71 | end = target_offsets[i+1] if i+1 < len(target_offsets) else shape[distaxis]
72 |
73 | local_slices[i][distaxis] = slice(distaxis_slice.start + distaxis_slice.step * begin,\
74 | distaxis_slice.start + distaxis_slice.step * end,\
75 | distaxis_slice.step)
76 |
77 | return local_slices
78 |
79 | def common_decomposition(axesA, offsetsA, axesB, offsetsB, shape):
80 | axes = list(OrderedDict.fromkeys(axesA + axesB)) # remove double axes while preserving order
81 | offsets = []
82 | ids = []
83 | for axis in axes:
84 | if not (axis in axesA and axis in axesB):
85 | offsets.append(offsetsA[axesA.index(axis)] if axis in axesA else offsetsB[axesB.index(axis)])
86 | ids.append([])
87 | continue
88 |
89 | offsetsA_sa = offsetsA[axesA.index(axis)]
90 | offsetsB_sa = offsetsB[axesB.index(axis)]
91 |
92 | offsets_sa = [] # sa = single axis
93 | ids_sa = []
94 | A_idx = 0 # partition index
95 | B_idx = 0
96 | begin = 0
97 | # loop the common decomposition of A and B.
98 | # the current partition is given by [begin, end)
99 | while not begin == shape[axis]:
100 | end_A = offsetsA_sa[A_idx+1] if A_idx < len(offsetsA_sa)-1 else shape[axis]
101 | end_B = offsetsB_sa[B_idx+1] if B_idx < len(offsetsB_sa)-1 else shape[axis]
102 | end = min(end_A, end_B)
103 |
104 | offsets_sa.append(begin)
105 | ids_sa.append((A_idx, B_idx))
106 |
107 | # go to next common partition
108 | if end == end_A:
109 | A_idx += 1
110 | if end == end_B:
111 | B_idx += 1
112 |
113 | begin = end
114 |
115 | offsets.append(np.array(offsets_sa))
116 | ids.append(np.array(ids_sa))
117 | return axes, offsets, ids
--------------------------------------------------------------------------------
/pyDive/distribution/interengine.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2015 Heiko Burau
3 |
4 | This file is part of pyDive.
5 |
6 | pyDive is free software: you can redistribute it and/or modify
7 | it under the terms of of either the GNU General Public License or
8 | the GNU Lesser General Public License as published by
9 | the Free Software Foundation, either version 3 of the License, or
10 | (at your option) any later version.
11 | pyDive is distributed in the hope that it will be useful,
12 | but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | GNU General Public License and the GNU Lesser General Public License
15 | for more details.
16 |
17 | You should have received a copy of the GNU General Public License
18 | and the GNU Lesser General Public License along with pyDive.
19 | If not, see .
20 | """
21 | __doc__ = None
22 | import numpy as np
23 | from mpi4py import MPI
24 | try:
25 | import pycuda.gpuarray
26 | except ImportError:
27 | pass
28 |
29 | #----------- cpu -> cpu ----------------
30 |
31 | def scatterArrayMPI_async(in_array, commData, target2rank):
32 | tasks = []
33 | for (dest_target, window, tag) in commData:
34 | tasks.append(MPI.COMM_WORLD.Isend(in_array[window].copy(), dest=target2rank[dest_target], tag=tag))
35 |
36 | return tasks
37 |
38 | def gatherArraysMPI_async(out_array, commData, target2rank):
39 | recv_bufs = []
40 | tasks = []
41 | for (src_target, window, tag) in commData:
42 | recv_bufs.append(np.empty_like(out_array[window]))
43 | tasks.append(MPI.COMM_WORLD.Irecv(recv_bufs[-1], source=target2rank[src_target], tag=tag))
44 |
45 | return tasks, recv_bufs
46 |
47 | def finish_MPIcommunication(out_array, commData, recv_bufs):
48 | for (src_target, window, tag), recv_buf in zip(commData, recv_bufs):
49 | out_array[window] = recv_buf
50 |
51 | #----------- gpu -> cpu -> cpu -> gpu ----------------
52 |
53 | def scatterArrayGPU_async(in_array, commData, target2rank):
54 | tasks = []
55 | for (dest_target, window, tag) in commData:
56 | tasks.append(MPI.COMM_WORLD.Isend(in_array[window].to_cpu(), dest=target2rank[dest_target], tag=tag))
57 |
58 | return tasks
59 |
60 | def gatherArraysGPU_async(out_array, commData, target2rank):
61 | recv_bufs = []
62 | tasks = []
63 | for (src_target, window, tag) in commData:
64 | chunk = out_array[window]
65 | recv_bufs.append(np.empty(shape=chunk.shape, dtype=chunk.dtype))
66 | tasks.append(MPI.COMM_WORLD.Irecv(recv_bufs[-1], source=target2rank[src_target], tag=tag))
67 |
68 | return tasks, recv_bufs
69 |
70 | def finish_GPUcommunication(out_array, commData, recv_bufs):
71 | for (src_target, window, tag), recv_buf in zip(commData, recv_bufs):
72 | out_array[window] = pycuda.gpuarray.to_gpu(recv_buf)
73 |
74 | import os
75 | onTarget = os.environ.get("onTarget", 'False')
76 |
77 | # execute this code only if it is not executed on engine
78 | if onTarget == 'False':
79 | import pyDive.IPParallelClient as com
80 |
81 | def MPI_copier(source, dest):
82 | view = com.getView()
83 |
84 | # send
85 | view.execute('{0}_send_tasks = interengine.scatterArrayMPI_async({0}, src_commData[0], target2rank)'\
86 | .format(source.name), targets=source.target_ranks)
87 |
88 | # receive
89 | view.execute("""\
90 | {0}_recv_tasks, {0}_recv_bufs = interengine.gatherArraysMPI_async({1}, dest_commData[0], target2rank)
91 | """.format(source.name, dest.name),\
92 | targets=dest.target_ranks)
93 |
94 | # finish communication
95 | view.execute('''\
96 | if "{0}_send_tasks" in locals():
97 | MPI.Request.Waitall({0}_send_tasks)
98 | del {0}_send_tasks
99 | if "{0}_recv_tasks" in locals():
100 | MPI.Request.Waitall({0}_recv_tasks)
101 | interengine.finish_MPIcommunication({1}, dest_commData[0], {0}_recv_bufs)
102 | del {0}_recv_tasks, {0}_recv_bufs
103 | '''.format(source.name, dest.name),
104 | targets=tuple(set(source.target_ranks + dest.target_ranks)))
105 |
106 | def GPU_copier(source, dest):
107 | view = com.getView()
108 |
109 | # send
110 | view.execute('{0}_send_tasks = interengine.scatterArrayGPU_async({0}, src_commData[0], target2rank)'\
111 | .format(source.name), targets=source.target_ranks)
112 |
113 | # receive
114 | view.execute("""\
115 | {0}_recv_tasks, {0}_recv_bufs = interengine.gatherArraysGPU_async({1}, dest_commData[0], target2rank)
116 | """.format(source.name, dest.name),\
117 | targets=dest.target_ranks)
118 |
119 | # finish communication
120 | view.execute('''\
121 | if "{0}_send_tasks" in locals():
122 | MPI.Request.Waitall({0}_send_tasks)
123 | del {0}_send_tasks
124 | if "{0}_recv_tasks" in locals():
125 | MPI.Request.Waitall({0}_recv_tasks)
126 | interengine.finish_GPUcommunication({1}, dest_commData[0], {0}_recv_bufs)
127 | del {0}_recv_tasks, {0}_recv_bufs
128 | '''.format(source.name, dest.name),
129 | targets=tuple(set(source.target_ranks + dest.target_ranks)))
130 |
--------------------------------------------------------------------------------
/pyDive/distribution/multiple_axes.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | """
3 | Copyright 2015 Heiko Burau
4 |
5 | This file is part of pyDive.
6 |
7 | pyDive is free software: you can redistribute it and/or modify
8 | it under the terms of of either the GNU General Public License or
9 | the GNU Lesser General Public License as published by
10 | the Free Software Foundation, either version 3 of the License, or
11 | (at your option) any later version.
12 | pyDive is distributed in the hope that it will be useful,
13 | but WITHOUT ANY WARRANTY; without even the implied warranty of
14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 | GNU General Public License and the GNU Lesser General Public License
16 | for more details.
17 |
18 | You should have received a copy of the GNU General Public License
19 | and the GNU Lesser General Public License along with pyDive.
20 | If not, see .
21 | """
22 | __doc__ = None
23 |
24 | import numpy as np
25 | import pyDive.IPParallelClient as com
26 | import helper
27 | from collections import defaultdict
28 |
29 | array_id = 0
30 |
31 | class DistributedGenericArray(object):
32 | """
33 | Represents a cluster-wide, multidimensional, homogeneous array of fixed-size elements.
34 | *cluster-wide* means that its elements are distributed across IPython.parallel-engines.
35 | The distribution is done in one or multiply dimensions along user-specified axes.
36 | The user can optionally specify which engine maps to which index range or leave the default
37 | that persuits an uniform distribution across all engines.
38 |
39 | This **{arraytype_name}** - class is auto-generated out of its local counterpart: **{local_arraytype_name}**.
40 |
41 | The implementation is based on IPython.parallel and local {local_arraytype_name} - arrays.
42 | Every special operation {local_arraytype_name} implements ("__add__", "__le__", ...) is also
43 | available for {arraytype_name}.
44 |
45 | Note that array slicing is a cheap operation since no memory is copied.
46 | However this can easily lead to the situation where you end up with two arrays of the same size but of distinct element distribution.
47 | Therefore call dist_like() first before doing any manual stuff on their local arrays.
48 | However every cluster-wide array operation first equalizes the distribution of all involved arrays,
49 | so an explicit call to dist_like() is rather unlikely in most use cases.
50 |
51 | If you try to access an attribute that is only available for the local array, the request
52 | is forwarded to an internal local copy of the whole distributed array (see: :meth:`gather()`).
53 | This internal copy is only created when you want to access it and is held until ``__setitem__`` is called,
54 | i.e. the array's content is manipulated.
55 | """
56 | local_arraytype = None
57 | target_modulename = None
58 | interengine_copier = None
59 | may_allocate = True
60 |
61 | def __init__(self, shape, dtype=np.float, distaxes='all', target_offsets=None, target_ranks=None, no_allocation=False, **kwargs):
62 | """Creates an instance of {arraytype_name}. This is a low-level method of instantiating an array, it should rather be
63 | constructed using factory functions ("empty", "zeros", "open", ...)
64 |
65 | :param ints shape: shape of array
66 | :param dtype: datatype of a single element
67 | :param ints distaxes: distributed axes. Accepts a single integer too. Defaults to 'all' meaning each axis is distributed.
68 | :param target_offsets: For each distributed axis there is a (inner) list in the outer list.
69 | The inner list contains the offsets of the local array.
70 | :type target_offsets: list of lists
71 | :param ints target_ranks: linear list of :term:`engine` ranks holding the local arrays.
72 | The last distributed axis is iterated over first.
73 | :param bool no_allocation: if ``True`` no instance of {local_arraytype_name} will be created on engine. Useful for
74 | manual instantiation of the local array.
75 | :param kwargs: additional keyword arguments are forwarded to the constructor of the local array.
76 | """
77 | #: size of the array on each axis
78 | if type(shape) not in (list, tuple):
79 | shape = (shape,)
80 | elif type(shape) is not tuple:
81 | shape = tuple(shape)
82 | self.shape = shape
83 | ##: datatype of a single data value
84 | self.dtype = dtype
85 | if distaxes == 'all':
86 | distaxes = tuple(range(len(shape)))
87 | elif type(distaxes) not in (list, tuple):
88 | distaxes = (distaxes,)
89 | elif type(distaxes) is not tuple:
90 | distaxes = tuple(distaxes)
91 | #: axes on which memory is distributed across :term:`engines `
92 | self.distaxes = distaxes
93 | #: total bytes consumed by elements of this array.
94 | self.nbytes = np.dtype(dtype).itemsize * np.prod(self.shape)
95 | self.view = com.getView()
96 | self.kwargs = kwargs
97 | self.local_copy_is_dirty = False
98 |
99 | assert len(distaxes) <= len(shape),\
100 | "more distributed axes ({}) than dimensions ({})".format(len(distaxes), len(shape))
101 | for distaxis in distaxes:
102 | assert distaxis >= 0 and distaxis < len(self.shape),\
103 | "distributed axis ({}) has to be within [0,{}]".format(distaxis, len(self.shape)-1)
104 |
105 | if target_offsets is None and target_ranks is None:
106 | # create hypothetical patch with best surface-to-volume ratio
107 | patch_volume = np.prod([self.shape[distaxis] for distaxis in distaxes]) / float(len(self.view.targets))
108 | patch_edge_length = pow(patch_volume, 1.0/len(distaxes))
109 |
110 | def factorize(n):
111 | if n == 1: yield 1; return
112 | for f in range(2,n//2+1) + [n]:
113 | while n%f == 0:
114 | n //= f
115 | yield f
116 | prime_factors = list(factorize(len(self.view.targets)))[::-1] # get prime factors of number of engines in descending order
117 | sorted_distaxes = sorted(distaxes, key=lambda axis: self.shape[axis]) # sort distributed axes in ascending order
118 | # calculate number of available targets (engines) per distributed axis
119 | # This value should be close to array_edge_length / patch_edge_length
120 | num_targets_av = [1] * len(self.shape)
121 |
122 | for distaxis in sorted_distaxes[:-1]:
123 | num_patches = self.shape[distaxis] / patch_edge_length
124 | while float(num_targets_av[distaxis]) < num_patches and prime_factors:
125 | num_targets_av[distaxis] *= prime_factors.pop()
126 | # the largest axis gets the remaining (largest) prime_factors
127 | if prime_factors:
128 | num_targets_av[sorted_distaxes[-1]] *= np.prod(prime_factors)
129 |
130 | # calculate target_offsets
131 | localshape = np.array(self.shape)
132 | for distaxis in distaxes:
133 | localshape[distaxis] = (self.shape[distaxis] - 1) / num_targets_av[distaxis] + 1
134 |
135 | # number of occupied targets for each distributed axis by this ndarray instance
136 | num_targets = [(self.shape[distaxis] - 1) / localshape[distaxis] + 1 for distaxis in distaxes]
137 |
138 | # calculate target_offsets
139 | target_offsets = [np.arange(num_targets[i]) * localshape[distaxes[i]] for i in range(len(distaxes))]
140 |
141 | # generate target_ranks list
142 | target_ranks = tuple(range(np.prod(num_targets)))
143 |
144 | if target_offsets is None:
145 | localshape = np.array(self.shape)
146 | for distaxis in distaxes:
147 | localshape[distaxis] = (self.shape[distaxis] - 1) / len(target_ranks[distaxis]) + 1
148 | target_offsets = [np.arange(num_targets[i]) * localshape[distaxes[i]] for i in range(len(distaxes))]
149 |
150 | if target_ranks is None:
151 | num_targets = [len(target_offsets_axis) for target_offsets_axis in target_offsets]
152 | target_ranks = tuple(range(np.prod(num_targets)))
153 | elif type(target_ranks) is not tuple:
154 | target_ranks = tuple(target_ranks)
155 |
156 | self.target_offsets = target_offsets
157 | self.target_ranks = target_ranks
158 |
159 | # generate a unique variable name used on target representing this instance
160 | global array_id
161 | #: Unique variable name of the local *array* on *engine*.
162 | #: Unless you are doing manual stuff on the *engines* there is no need for dealing with this attribute.
163 | self.name = 'dist_array' + str(array_id)
164 | array_id += 1
165 |
166 | if no_allocation:
167 | self.view.push({self.name : None}, targets=self.target_ranks)
168 | else:
169 | target_shapes = self.target_shapes()
170 |
171 | self.view.scatter('target_shape', target_shapes, targets=self.target_ranks)
172 | self.view.push({'kwargs' : kwargs, 'dtype' : dtype}, targets=self.target_ranks)
173 | self.view.execute('%s = %s(shape=target_shape[0], dtype=dtype, **kwargs)' % \
174 | (self.name, self.__class__.target_modulename + "." + self.__class__.local_arraytype.__name__), targets=self.target_ranks)
175 |
176 | def __del__(self):
177 | self.view.execute('del %s' % self.name, targets=self.target_ranks)
178 |
179 | def target_shapes(self):
180 | """generate a list of the local shape on each target in use"""
181 | targetshapes = []
182 | num_targets = [len(target_offsets_axis) for target_offsets_axis in self.target_offsets]
183 | for rank_idx_vector in np.ndindex(*num_targets): # last dimension is iterated over first
184 | targetshape = list(self.shape)
185 | for distaxis_idx in range(len(self.distaxes)):
186 | distaxis = self.distaxes[distaxis_idx]
187 | i = rank_idx_vector[distaxis_idx]
188 | end = self.target_offsets[distaxis_idx][i+1] if i != num_targets[distaxis_idx] - 1 else self.shape[distaxis]
189 | targetshape[distaxis] = end - self.target_offsets[distaxis_idx][i]
190 | targetshapes.append(targetshape)
191 |
192 | return targetshapes
193 |
194 | def target_offset_vectors(self):
195 | """generate a list of the local offset vectors on each target in use"""
196 | target_offset_vectors = []
197 | num_targets = [len(target_offsets_axis) for target_offsets_axis in self.target_offsets]
198 | for rank_idx_vector in np.ndindex(*num_targets): # last dimension is iterated over first
199 | target_offset_vector = [0] * len(self.shape)
200 | for distaxis_idx in range(len(self.distaxes)):
201 | distaxis = self.distaxes[distaxis_idx]
202 | i = rank_idx_vector[distaxis_idx]
203 | target_offset_vector[distaxis] = self.target_offsets[distaxis_idx][i]
204 | target_offset_vectors.append(target_offset_vector)
205 |
206 | return target_offset_vectors
207 |
208 | def __get_linear_rank_idx(self, rank_idx_vector):
209 | # convert rank_idx_vector to linear rank index
210 | # last dimension is iterated over first
211 | num_targets = [len(target_offsets_axis) for target_offsets_axis in self.target_offsets]
212 | pitch = [int(np.prod(num_targets[i+1:])) for i in range(len(self.distaxes))]
213 | return sum(rank_idx_component * pitch_component for rank_idx_component, pitch_component in zip(rank_idx_vector, pitch))
214 |
215 | def __getitem__(self, args):
216 | # bitmask indexing
217 | if isinstance(args, self.__class__) and args.dtype == bool:
218 | bitmask = args
219 | assert bitmask.shape == self.shape,\
220 | "shape of bitmask (%s) does not correspond to shape of array (%s)"\
221 | % (str(bitmask.shape), str(self.shape))
222 |
223 | bitmask = bitmask.dist_like(self) # equalize distribution if necessary
224 | self.view.execute("tmp = {0}[{1}]; tmp_size = tmp.shape[0]".format(repr(self), repr(bitmask)), targets=self.target_ranks)
225 | sizes = self.view.pull("tmp_size", targets=self.target_ranks)
226 | new_target_ranks = [rank for rank, size in zip(self.target_ranks, sizes) if size > 0]
227 | new_sizes = [size for size in sizes if size > 0]
228 | partial_sum = lambda a, b: a + [a[-1] + b]
229 | new_target_offsets = [ [0] + reduce(partial_sum, new_sizes[1:-1], new_sizes[0:1]) ]
230 | new_shape = [sum(new_sizes)]
231 | # create resulting ndarray
232 | result = self.__class__(new_shape, self.dtype, 0, new_target_offsets, new_target_ranks, no_allocation=True, **self.kwargs)
233 | self.view.execute("{0} = tmp; del tmp".format(result.name), targets=result.target_ranks)
234 | return result
235 |
236 | if args == slice(None):
237 | args = (slice(None),) * len(self.shape)
238 |
239 | if not isinstance(args, list) and not isinstance(args, tuple):
240 | args = (args,)
241 |
242 | assert len(args) == len(self.shape),\
243 | "number of arguments (%d) does not correspond to the dimension (%d)"\
244 | % (len(args), len(self.shape))
245 |
246 | # wrap all integer indices
247 | args = [(arg + s) % s if type(arg) is int else arg for arg, s in zip(args, self.shape)]
248 |
249 | # shape of the new sliced ndarray
250 | new_shape, clean_view = helper.view_of_shape(self.shape, args)
251 |
252 | # if args is a list of indices then return a single data value
253 | if not new_shape:
254 | local_idx = list(args)
255 | rank_idx_vector = []
256 | for distaxis, target_offsets in zip(self.distaxes, self.target_offsets):
257 | dist_idx = args[distaxis]
258 | rank_idx_component = np.searchsorted(target_offsets, dist_idx, side="right") - 1
259 | local_idx[distaxis] = dist_idx - target_offsets[rank_idx_component]
260 | rank_idx_vector.append(rank_idx_component)
261 |
262 | rank_idx = self.__get_linear_rank_idx(rank_idx_vector)
263 | return self.view.pull("%s%s" % (self.name, repr(local_idx)), targets=self.target_ranks[rank_idx])
264 |
265 | if all(type(clean_view[distaxis]) is int for distaxis in self.distaxes):
266 | # return local array because all distributed axes have vanished
267 | rank_idx_vector = []
268 | for distaxis, target_offsets in zip(self.distaxes, self.target_offsets):
269 | dist_idx = clean_view[distaxis]
270 | rank_idx_component = np.searchsorted(target_offsets, dist_idx, side="right") - 1
271 | clean_view[distaxis] = dist_idx - target_offsets[rank_idx_component]
272 | rank_idx_vector.append(rank_idx_component)
273 |
274 | rank_idx = self.__get_linear_rank_idx(rank_idx_vector)
275 | self.view.execute("sliced = %s%s" % (self.name, repr(clean_view)), targets=self.target_ranks[rank_idx])
276 | return self.view.pull("sliced", targets=self.target_ranks[rank_idx])
277 |
278 | # determine properties of the new, sliced ndarray
279 | # keep these in mind when reading the following for loop
280 | local_slices_aa = [] # aa = all (distributed) axes
281 | new_target_offsets = []
282 | new_rank_ids_aa = []
283 | new_distaxes = []
284 |
285 | for distaxis, distaxis_idx, target_offsets \
286 | in zip(self.distaxes, range(len(self.distaxes)), self.target_offsets):
287 |
288 | # slice object in the direction of the distributed axis
289 | distaxis_slice = clean_view[distaxis]
290 |
291 | # if it turns out that distaxis_slice is actually an int, this axis has to vanish.
292 | # That means the local slice object has to be an int too.
293 | if type(distaxis_slice) == int:
294 | dist_idx = distaxis_slice
295 | rank_idx_component = np.searchsorted(target_offsets, dist_idx, side="right") - 1
296 | local_idx = dist_idx - target_offsets[rank_idx_component]
297 | local_slices_aa.append((local_idx,))
298 | new_rank_ids_aa.append((rank_idx_component,))
299 | continue
300 |
301 | # determine properties of the new, sliced ndarray
302 | local_slices_sa = [] # sa = single axis
303 | new_target_offsets_sa = []
304 | new_rank_ids_sa = []
305 | total_ids = 0
306 |
307 | first_rank_idx = np.searchsorted(target_offsets, distaxis_slice.start, side="right") - 1
308 | last_rank_idx = np.searchsorted(target_offsets, distaxis_slice.stop, side="right")
309 |
310 | for i in range(first_rank_idx, last_rank_idx):
311 |
312 | # index range of current target
313 | begin = target_offsets[i]
314 | end = target_offsets[i+1] if i < len(target_offsets)-1 else self.shape[distaxis]
315 | # first slice index within [begin, end)
316 | firstSliceIdx = helper.getFirstSliceIdx(distaxis_slice, begin, end)
317 | if firstSliceIdx is None: continue
318 | # calculate last slice index of distaxis_slice
319 | tmp = (distaxis_slice.stop-1 - distaxis_slice.start) / distaxis_slice.step
320 | lastIdx = distaxis_slice.start + tmp * distaxis_slice.step
321 | # calculate last sub index within [begin,end)
322 | tmp = (end-1 - firstSliceIdx) / distaxis_slice.step
323 | lastSliceIdx = firstSliceIdx + tmp * distaxis_slice.step
324 | lastSliceIdx = min(lastSliceIdx, lastIdx)
325 | # slice object for current target
326 | local_slices_sa.append(slice(firstSliceIdx - begin, lastSliceIdx+1 - begin, distaxis_slice.step))
327 | # number of indices remaining on the current target after slicing
328 | num_ids = (lastSliceIdx - firstSliceIdx) / distaxis_slice.step + 1
329 | # new offset for current target
330 | new_target_offsets_sa.append(total_ids)
331 | total_ids += num_ids
332 | # target rank index
333 | new_rank_ids_sa.append(i)
334 |
335 | new_rank_ids_sa = np.array(new_rank_ids_sa)
336 |
337 | local_slices_aa.append(local_slices_sa)
338 | new_target_offsets.append(new_target_offsets_sa)
339 | new_rank_ids_aa.append(new_rank_ids_sa)
340 |
341 | # shift distaxis by the number of vanished axes in the left of it
342 | new_distaxis = distaxis - sum(1 for arg in args[:distaxis] if type(arg) is int)
343 | new_distaxes.append(new_distaxis)
344 |
345 | # create list of targets which participate slicing, new_target_ranks
346 | num_ranks_aa = [len(new_rank_ids_sa) for new_rank_ids_sa in new_rank_ids_aa]
347 | new_target_ranks = []
348 | for idx in np.ndindex(*num_ranks_aa):
349 | rank_idx_vector = [new_rank_ids_aa[i][idx[i]] for i in range(len(idx))]
350 | rank_idx = self.__get_linear_rank_idx(rank_idx_vector)
351 | new_target_ranks.append(self.target_ranks[rank_idx])
352 |
353 | # create resulting ndarray
354 | result = self.__class__(new_shape, self.dtype, new_distaxes, new_target_offsets, new_target_ranks, no_allocation=True, **self.kwargs)
355 |
356 | # remote slicing
357 | local_args_list = []
358 | for idx in np.ndindex(*num_ranks_aa):
359 | local_slices = [local_slices_aa[i][idx[i]] for i in range(len(idx))]
360 | local_args = list(args)
361 | for distaxis, distaxis_idx in zip(self.distaxes, range(len(self.distaxes))):
362 | local_args[distaxis] = local_slices[distaxis_idx]
363 | local_args_list.append(local_args)
364 |
365 | self.view.scatter('local_args', local_args_list, targets=result.target_ranks)
366 | self.view.execute('%s = %s[local_args[0]]' % (result.name, self.name), targets=result.target_ranks)
367 |
368 | return result
369 |
370 | def __setitem__(self, key, value):
371 | self.local_copy_is_dirty = True
372 | # bitmask indexing
373 | if isinstance(key, self.__class__) and key.dtype == bool:
374 | bitmask = key.dist_like(self)
375 | self.view.execute("%s[%s] = %s" % (repr(self), repr(bitmask), repr(value)), targets=self.target_ranks)
376 | return
377 |
378 | # if args is [:] then assign value to the entire ndarray
379 | if key == slice(None):
380 | # assign local array to self
381 | if isinstance(value, self.__class__.local_arraytype):
382 | subarrays = []
383 | for target_offset_vector, target_shape in zip(self.target_offset_vectors(), self.target_shapes()):
384 | window = [slice(start, start+length) for start, length in zip(target_offset_vector, target_shape)]
385 | subarrays.append(value[window])
386 |
387 | self.view.scatter("subarray", subarrays, targets=self.target_ranks)
388 | self.view.execute("%s[:] = subarray[0]" % self.name, targets=self.target_ranks)
389 | return
390 |
391 | # assign other array or value to self
392 | other = value.dist_like(self) if hasattr(value, "dist_like") else value
393 | self.view.execute("%s[:] = %s" % (repr(self), repr(other)), targets=self.target_ranks)
394 | return
395 |
396 | if not isinstance(key, list) and not isinstance(key, tuple):
397 | key = (key,)
398 |
399 | assert len(key) == len(self.shape),\
400 | "number of arguments (%d) does not correspond to the dimension (%d)"\
401 | % (len(key), len(self.shape))
402 |
403 | # value assignment (key == list of indices)
404 | if all(type(k) is int for k in key):
405 | local_idx = list(key)
406 | local_idx = [(i + s) % s for i, s in zip(local_idx, self.shape)]
407 | rank_idx_vector = []
408 | for distaxis, target_offsets in zip(self.distaxes, self.target_offsets):
409 | dist_idx = key[distaxis]
410 | rank_idx_component = np.searchsorted(target_offsets, dist_idx, side="right") - 1
411 | local_idx[distaxis] = dist_idx - target_offsets[rank_idx_component]
412 | rank_idx_vector.append(rank_idx_component)
413 |
414 | rank_idx = self.__get_linear_rank_idx(rank_idx_vector)
415 | self.view.push({'value' : value}, targets=self.target_ranks[rank_idx])
416 | self.view.execute("%s%s = value" % (self.name, repr(local_idx)), targets=self.target_ranks[rank_idx])
417 | return
418 |
419 | # assign value to sub-array of self
420 | sub_array = self[key]
421 | sub_array[:] = value
422 |
423 | def __str__(self):
424 | return self.gather().__str__()
425 |
426 | def __repr__(self):
427 | return self.name
428 |
429 | @property
430 | def local_copy(self):
431 | if not hasattr(self, "_local_copy") or self.local_copy_is_dirty:
432 | self._local_copy = self.gather()
433 | self.local_copy_is_dirty = False
434 | return self._local_copy
435 |
436 | def __getattr__(self, name):
437 | """If the requested attribute is an attribute of the local array and not of this array then
438 | gather() is called and the request is forwarded to the gathered array. This makes this
439 | distributed array more behaving like a local array."""
440 | if not hasattr(self.__class__.local_arraytype, name):
441 | raise AttributeError(name)
442 |
443 | return getattr(self.local_copy, name)
444 |
445 | def gather(self):
446 | """Gathers local instances of {local_arraytype_name} from *engines*, concatenates them and returns
447 | the result.
448 |
449 | .. note:: You may not call this method explicitly because if you try to access an attribute
450 | of the local array ({local_arraytype_name}), ``gather()`` is called implicitly before the request is forwarded
451 | to that internal gathered array. Just access attributes like you do for the local array.
452 | The internal copy is held until ``__setitem__`` is called, e.g. ``a[1] = 3.0``, setting
453 | a dirty flag to the local copy.
454 |
455 | .. warning:: If another array overlapping this array is manipulating its data there is no chance to set
456 | the dirty flag so you have to keep in mind to call ``gather()`` explicitly in this case!
457 |
458 | :return: instance of {local_arraytype_name}
459 | """
460 | local_arrays = self.view.pull(self.name, targets=self.target_ranks)
461 |
462 | result = self.__class__.local_arraytype(shape=self.shape, dtype=self.dtype, **self.kwargs)
463 |
464 | for target_offset_vector, target_shape, local_array \
465 | in zip(self.target_offset_vectors(), self.target_shapes(), local_arrays):
466 |
467 | window = [slice(start, start+length) for start, length in zip(target_offset_vector, target_shape)]
468 | result[window] = local_array
469 |
470 | return result
471 |
472 | def copy(self):
473 | """Returns a hard copy of this array.
474 | """
475 | assert self.__class__.may_allocate == True, "{0} is not allowed to allocate new memory.".format(self.__class__.__name__)
476 |
477 | result = self.__class__(self.shape, self.dtype, self.distaxes, self.target_offsets, self.target_ranks, no_allocation=True, **self.kwargs)
478 | self.view.execute("%s = %s.copy()" % (result.name, self.name), targets=self.target_ranks)
479 | return result
480 |
481 | def is_distributed_like(self, other):
482 | if self.distaxes == other.distaxes:
483 | if all(np.array_equal(my_target_offsets, other_target_offsets) \
484 | for my_target_offsets, other_target_offsets in zip(self.target_offsets, other.target_offsets)):
485 | if self.target_ranks == other.target_ranks:
486 | return True
487 | return False
488 |
489 | def dist_like(self, other):
490 | """Redistributes a copy of this array (*self*) like *other* and returns the result.
491 | Checks whether redistribution is necessary and returns *self* if not.
492 |
493 | Redistribution involves inter-engine communication.
494 |
495 | :param other: target array
496 | :type other: distributed array
497 | :raises AssertionError: if the shapes of *self* and *other* don't match.
498 | :return: new array with the same content as *self* but distributed like *other*.
499 | If *self* is already distributed like *other* nothing is done and *self* is returned.
500 | """
501 | assert self.shape == other.shape,\
502 | "Shapes do not match: " + str(self.shape) + " <-> " + str(other.shape)
503 |
504 | # if self is already distributed like *other* do nothing
505 | if self.is_distributed_like(other):
506 | return self
507 |
508 | assert self.__class__.may_allocate, "{0} is not allowed to allocate new memory.".format(self.__class__.__name__)
509 |
510 | # todo: optimization -> helper.common_decomposition should be a generator
511 | common_axes, common_offsets, common_idx_pairs = \
512 | helper.common_decomposition(self.distaxes, self.target_offsets, other.distaxes, other.target_offsets, self.shape)
513 |
514 | my_commData = defaultdict(list)
515 | other_commData = defaultdict(list)
516 |
517 | tag = 0
518 | num_offsets = [len(common_offsets_sa) for common_offsets_sa in common_offsets]
519 |
520 | for idx in np.ndindex(*num_offsets):
521 | my_rank_idx_vector = [0] * len(self.distaxes)
522 | other_rank_idx_vector = [0] * len(other.distaxes)
523 | my_window = [slice(None)] * len(self.shape)
524 | other_window = [slice(None)] * len(self.shape)
525 |
526 | for i in range(len(idx)):
527 | common_axis = common_axes[i]
528 | begin = common_offsets[i][idx[i]]
529 | end = common_offsets[i][idx[i]+1] if idx[i] < len(common_offsets[i]) -1 else self.shape[common_axes[i]]
530 |
531 | if common_axis in self.distaxes and common_axis in other.distaxes:
532 | my_rank_idx_comp = common_idx_pairs[i][idx[i],0]
533 | other_rank_idx_comp = common_idx_pairs[i][idx[i],1]
534 | my_rank_idx_vector[self.distaxes.index(common_axis)] = my_rank_idx_comp
535 | other_rank_idx_vector[other.distaxes.index(common_axis)] = other_rank_idx_comp
536 |
537 | my_offset = self.target_offsets[self.distaxes.index(common_axis)][my_rank_idx_comp]
538 | my_window[common_axis] = slice(begin - my_offset, end - my_offset)
539 | other_offset = other.target_offsets[other.distaxes.index(common_axis)][other_rank_idx_comp]
540 | other_window[common_axis] = slice(begin - other_offset, end - other_offset)
541 |
542 | continue
543 |
544 | if common_axis in self.distaxes:
545 | my_rank_idx_vector[self.distaxes.index(common_axis)] = idx[i]
546 | other_window[common_axis] = slice(begin, end)
547 | if common_axis in other.distaxes:
548 | other_rank_idx_vector[other.distaxes.index(common_axis)] = idx[i]
549 | my_window[common_axis] = slice(begin, end)
550 |
551 | my_rank = self.target_ranks[self.__get_linear_rank_idx(my_rank_idx_vector)]
552 | other_rank = other.target_ranks[other.__get_linear_rank_idx(other_rank_idx_vector)]
553 |
554 | my_commData[my_rank].append((other_rank, my_window, tag))
555 | other_commData[other_rank].append((my_rank, other_window, tag))
556 |
557 | tag += 1
558 |
559 | # push communication meta-data to engines
560 | self.view.scatter('src_commData', my_commData.values(), targets=my_commData.keys())
561 | self.view.scatter('dest_commData', other_commData.values(), targets=other_commData.keys())
562 |
563 | # result ndarray
564 | result = self.__class__(self.shape, self.dtype, other.distaxes, other.target_offsets, other.target_ranks, False, **self.kwargs)
565 |
566 | self.__class__.interengine_copier(self, result)
567 |
568 | return result
569 |
570 | def info(self, name):
571 | print name + " info:"
572 | print "{}.name".format(name), self.name
573 | print "{}.target_ranks".format(name), self.target_ranks
574 | print "{}.target_offsets".format(name), self.target_offsets
575 | print "{}.distaxes".format(name), self.distaxes
576 | self.view.execute("dt = str({}.dtype)".format(repr(self)), targets=self.target_ranks)
577 | print "{}.dtypes".format(name), self.view.pull("dt", targets=self.target_ranks)
578 | self.view.execute("t = str(type({}))".format(repr(self)), targets=self.target_ranks)
579 | print "{}.types".format(name), self.view.pull("t", targets=self.target_ranks)
580 |
581 | def __elementwise_op__(self, op, *args):
582 | args = [arg.dist_like(self) if hasattr(arg, "target_ranks") else arg for arg in args]
583 | arg_names = [repr(arg) for arg in args]
584 | arg_string = ",".join(arg_names)
585 |
586 | result = self.__class__(self.shape, self.dtype, self.distaxes, self.target_offsets, self.target_ranks, no_allocation=True, **self.kwargs)
587 |
588 | self.view.execute("{0} = {1}.{2}({3}); dtype={0}.dtype".format(repr(result), repr(self), op, arg_string), targets=self.target_ranks)
589 | result.dtype = self.view.pull("dtype", targets=result.target_ranks[0])
590 | result.nbytes = np.dtype(result.dtype).itemsize * np.prod(result.shape)
591 |
592 | return result
593 |
594 | def __elementwise_iop__(self, op, *args):
595 | args = [arg.dist_like(self) if hasattr(arg, "target_ranks") else arg for arg in args]
596 | arg_names = [repr(arg) for arg in args]
597 | arg_string = ",".join(arg_names)
598 | self.view.execute("%s = %s.%s(%s)" % (repr(self), repr(self), op, arg_string), targets=self.target_ranks)
599 | return self
600 |
601 | #----------------------------------------------------------------
602 |
603 | from multiple_axes_funcs import *
--------------------------------------------------------------------------------
/pyDive/distribution/multiple_axes_funcs.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | import types
4 | from multiple_axes import DistributedGenericArray
5 | import numpy as np
6 | import pyDive.IPParallelClient as com
7 |
8 | def distribute(local_arraytype, newclassname, target_modulename, interengine_copier=None, may_allocate = True):
9 | binary_ops = ["add", "sub", "mul", "floordiv", "div", "mod", "pow", "lshift", "rshift", "and", "xor", "or"]
10 |
11 | binary_iops = ["__i" + op + "__" for op in binary_ops]
12 | binary_rops = ["__r" + op + "__" for op in binary_ops]
13 | binary_ops = ["__" + op + "__" for op in binary_ops]
14 | unary_ops = ["__neg__", "__pos__", "__abs__", "__invert__", "__complex__", "__int__", "__long__", "__float__", "__oct__", "__hex__"]
15 | comp_ops = ["__lt__", "__le__", "__eq__", "__ne__", "__ge__", "__gt__"]
16 |
17 | special_ops_avail = set(name for name in local_arraytype.__dict__.keys() if name.endswith("__"))
18 |
19 | make_special_op = lambda op: lambda self, *args: self.__elementwise_op__(op, *args)
20 | make_special_iop = lambda op: lambda self, *args: self.__elementwise_iop__(op, *args)
21 |
22 | special_ops_dict = {op : make_special_op(op) for op in \
23 | set(binary_ops + binary_rops + unary_ops + comp_ops) & special_ops_avail}
24 | special_iops_dict = {op : make_special_iop(op) for op in set(binary_iops) & special_ops_avail}
25 |
26 | formated_doc_funs = ("__init__", "gather")
27 |
28 | result_dict = dict(DistributedGenericArray.__dict__)
29 |
30 | # docs
31 | result_dict["__doc__"] = result_dict["__doc__"].format(\
32 | local_arraytype_name=local_arraytype.__module__ + "." + local_arraytype.__name__,
33 | arraytype_name=newclassname)
34 | # copy methods which have formated docstrings because their docstrings are going to be modified
35 | copied_methods = {k : types.FunctionType(v.func_code, v.func_globals, name=v.func_name, argdefs=v.func_defaults)\
36 | for k,v in result_dict.items() if k in formated_doc_funs}
37 | result_dict.update(copied_methods)
38 |
39 | result_dict.update(special_ops_dict)
40 | result_dict.update(special_iops_dict)
41 |
42 | result = type(newclassname, (), result_dict)
43 | result.local_arraytype = local_arraytype
44 | result.target_modulename = target_modulename
45 | result.interengine_copier = interengine_copier
46 | result.may_allocate = may_allocate
47 |
48 | # docs
49 | for method in (v for k,v in result.__dict__.items() if k in formated_doc_funs):
50 | method.__doc__ = method.__doc__.format(\
51 | local_arraytype_name=local_arraytype.__module__ + "." + local_arraytype.__name__,
52 | arraytype_name=newclassname)
53 |
54 | return result
55 |
56 | def generate_factories(arraytype, factory_names, dtype_default):
57 |
58 | def factory_wrapper(factory_name, shape, dtype, distaxes, kwargs):
59 | result = arraytype(shape, dtype, distaxes, None, None, True, **kwargs)
60 |
61 | target_shapes = result.target_shapes()
62 |
63 | view = com.getView()
64 | view.scatter('target_shape', target_shapes, targets=result.target_ranks)
65 | view.push({'kwargs' : kwargs, 'dtype' : dtype}, targets=result.target_ranks)
66 |
67 | view.execute("{0} = {1}(shape=target_shape[0], dtype=dtype, **kwargs)".format(result.name, factory_name),\
68 | targets=result.target_ranks)
69 | return result
70 |
71 | make_factory = lambda factory_name: lambda shape, dtype=dtype_default, distaxes='all', **kwargs:\
72 | factory_wrapper(arraytype.target_modulename + "." + factory_name, shape, dtype, distaxes, kwargs)
73 |
74 | factories_dict = {factory_name : make_factory(factory_name) for factory_name in factory_names}
75 |
76 | # add docstrings
77 | for name, factory in factories_dict.items():
78 | factory.__name__ = name
79 | factory.__doc__ = \
80 | """Create a *{0}* instance. This function calls its local counterpart *{1}* on each :term:`engine`.
81 |
82 | :param ints shape: shape of array
83 | :param dtype: datatype of a single element
84 | :param ints distaxes: distributed axes
85 | :param kwargs: keyword arguments are passed to the local function *{1}*
86 | """.format(arraytype.__name__, str(arraytype.local_arraytype.__module__) + "." + name)
87 |
88 | return factories_dict
89 |
90 | def generate_factories_like(arraytype, factory_names):
91 |
92 | def factory_like_wrapper(factory_name, other, kwargs):
93 | result = arraytype(other.shape, other.dtype, other.distaxes, other.target_offsets, other.target_ranks, True, **kwargs)
94 | view = com.getView()
95 | view.push({'kwargs' : kwargs}, targets=result.target_ranks)
96 | view.execute("{0} = {1}({2}, **kwargs)".format(result.name, factory_name, other.name), targets=result.target_ranks)
97 | return result
98 |
99 | make_factory = lambda factory_name: lambda other, **kwargs: \
100 | factory_like_wrapper(arraytype.target_modulename + "." + factory_name, other, kwargs)
101 |
102 | factories_dict = {factory_name : make_factory(factory_name) for factory_name in factory_names}
103 |
104 | # add docstrings
105 | for name, factory in factories_dict.items():
106 | factory.__name__ = name
107 | factory.__doc__ = \
108 | """Create a *{0}* instance with the same shape, dtype and distribution as ``other``.
109 | This function calls its local counterpart *{1}* on each :term:`engine`.
110 |
111 | :param other: other array
112 | :param kwargs: keyword arguments are passed to the local function *{1}*
113 | """.format(arraytype.__name__, str(arraytype.local_arraytype.__module__) + "." + name)
114 |
115 | return factories_dict
116 |
117 | def generate_ufuncs(ufunc_names, target_modulename):
118 |
119 | def ufunc_wrapper(ufunc_name, args, kwargs):
120 | arg0 = args[0]
121 | args = [arg.dist_like(arg0) if hasattr(arg, "target_ranks") else arg for arg in args]
122 | arg_names = [repr(arg) for arg in args]
123 | arg_string = ",".join(arg_names)
124 |
125 | view = com.getView()
126 | result = arg0.__class__(arg0.shape, arg0.dtype, arg0.distaxes, arg0.target_offsets, arg0.target_ranks, no_allocation=True, **arg0.kwargs)
127 |
128 | view.execute("{0} = {1}({2}); dtype={0}.dtype".format(repr(result), ufunc_name, arg_string), targets=arg0.target_ranks)
129 | result.dtype = view.pull("dtype", targets=result.target_ranks[0])
130 | result.nbytes = np.dtype(result.dtype).itemsize * np.prod(result.shape)
131 | return result
132 |
133 | make_ufunc = lambda ufunc_name: lambda *args, **kwargs: ufunc_wrapper(target_modulename + "." + ufunc_name, args, kwargs)
134 |
135 | return {ufunc_name: make_ufunc(ufunc_name) for ufunc_name in ufunc_names}
--------------------------------------------------------------------------------
/pyDive/fragment.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2014 Heiko Burau
3 |
4 | This file is part of pyDive.
5 |
6 | pyDive is free software: you can redistribute it and/or modify
7 | it under the terms of of either the GNU General Public License or
8 | the GNU Lesser General Public License as published by
9 | the Free Software Foundation, either version 3 of the License, or
10 | (at your option) any later version.
11 | pyDive is distributed in the hope that it will be useful,
12 | but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | GNU General Public License and the GNU Lesser General Public License
15 | for more details.
16 |
17 | You should have received a copy of the GNU General Public License
18 | and the GNU Lesser General Public License along with pyDive.
19 | If not, see .
20 | """
21 | __doc__ = None
22 |
23 | import IPParallelClient as com
24 | from IPython.parallel import interactive
25 | try:
26 | from arrays.h5_ndarray import h5_ndarray
27 | except ImportError:
28 | h5_ndarray = None
29 | try:
30 | from arrays.ad_ndarray import ad_ndarray
31 | except ImportError:
32 | ad_ndarray = None
33 | import math
34 |
35 | #: list of array types that store their elements on hard disk
36 | hdd_arraytypes = (h5_ndarray, ad_ndarray)
37 |
38 | def __bestStepSize(arrays, axis, memory_limit):
39 | view = com.getView()
40 |
41 | # minimum amount of memory available and memory needed, both per engine
42 | get_mem_av_node = interactive(lambda: psutil.virtual_memory().available)
43 | tmp_targets = view.targets
44 | view.targets = 'all'
45 | mem_av = min(view.apply(get_mem_av_node)) / com.getPPN()
46 | mem_needed = sum(a.nbytes for a in arrays) / len(view)
47 | view.targets = tmp_targets
48 |
49 | # edge length of the whole array
50 | edge_length = arrays[0].shape[axis]
51 | # maximum edge length on one engine according to the available memory
52 | step_size = memory_limit * edge_length * mem_av / mem_needed
53 |
54 | if step_size >= edge_length:
55 | return edge_length
56 |
57 | # round 'step_size' down to nearest power of two
58 | return pow(2, int(math.log(step_size, 2)))
59 |
60 | def fragment(*arrays, **kwargs):
61 | """Create fragments of *arrays* so that each fragment will fit into the combined
62 | main memory of all engines when calling ``load()``. The fragmentation is done by array slicing
63 | along the longest axis of ``arrays[0]``.
64 | The edge size of the fragments is a power of two except for the last fragment.
65 |
66 | :param array: distributed arrays (e.g. pyDive.ndarray, pyDive.h5_ndarray, ...)
67 | :param kwargs: optional keyword arguments are: ``memory_limit`` and ``offset``.
68 | :param float memory_limit: fraction of the combined main memory of all engines reserved for fragmentation.
69 | Defaults to ``0.25``.
70 | :param bool offset: If ``True`` the returned tuple is extended by the fragments' offset (along the distributed axis).
71 | Defaults to ``False``.
72 | :raises AssertionError: If not all arrays have the same shape.
73 | :raises AssertionError: If not all arrays are distributed along the same axis.
74 | :return: generator object (list) of tuples. Each tuple consists of one fragment for each array in *arrays*.
75 |
76 | Note that *arrays* may contain an arbitrary number of distributed arrays of any type.
77 | While the fragments' size is solely calculated based on the memory consumption of
78 | arrays that store their elements on hard disk (see :obj:`hdd_arraytypes`),
79 | the fragmentation itself is applied on all arrays in the same way.
80 |
81 | Example: ::
82 |
83 | big_h5_array = pyDive.h5.open("monster.h5", "/")
84 | # big_h5_array.load() # crash
85 |
86 | for h5_array, offset in pyDive.fragment(big_h5_array, offset=True):
87 | a = h5_array.load() # no crash
88 | print "This fragment's offset is", offset, "on axis:", a.distaxis
89 | """
90 | # default keyword arguments
91 | kwargs_defaults = {"memory_limit" : 0.25, "offset" : False}
92 | kwargs_defaults.update(kwargs)
93 | kwargs = kwargs_defaults
94 |
95 | memory_limit = kwargs["memory_limit"]
96 | offset = kwargs["offset"]
97 |
98 | if not arrays: return
99 |
100 | assert all(a.shape == arrays[0].shape for a in arrays), \
101 | "all arrays must have the same shape"
102 |
103 | # calculate the best suitable step size (-> fragment's edge size) according to the amount
104 | # of available memory on the engines
105 | hdd_arrays = [a for a in arrays if (hasattr(a, "arraytype") and a.arraytype in hdd_arraytypes) or type(a) in hdd_arraytypes]
106 | longest_axis = max(range(len(arrays[0].shape)), key=lambda axis : arrays[0].shape[axis])
107 | step = __bestStepSize(hdd_arrays, longest_axis, memory_limit)
108 |
109 | shape = arrays[0].shape
110 | # list of slices representing the fragment's shape
111 | fragment_window = [slice(None)] * len(shape)
112 |
113 | for begin in range(0, shape[longest_axis], step):
114 | end = min(begin + step, shape[longest_axis])
115 | fragment_window[longest_axis] = slice(begin, end)
116 |
117 | result = [a[fragment_window] for a in arrays]
118 | if offset:
119 | offset_vector = [0] * len(shape)
120 | offset_vector[longest_axis] = begin
121 | result += [offset_vector]
122 | if len(result) == 1:
123 | yield result[0]
124 | else:
125 | yield result
126 |
127 | return
128 |
--------------------------------------------------------------------------------
/pyDive/mappings.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2014 Heiko Burau
3 |
4 | This file is part of pyDive.
5 |
6 | pyDive is free software: you can redistribute it and/or modify
7 | it under the terms of of either the GNU General Public License or
8 | the GNU Lesser General Public License as published by
9 | the Free Software Foundation, either version 3 of the License, or
10 | (at your option) any later version.
11 | pyDive is distributed in the hope that it will be useful,
12 | but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | GNU General Public License and the GNU Lesser General Public License
15 | for more details.
16 |
17 | You should have received a copy of the GNU General Public License
18 | and the GNU Lesser General Public License along with pyDive.
19 | If not, see .
20 | """
21 |
22 | __doc__=\
23 | """If `numba `_ is installed the particle shape functions will
24 | be compiled which gives an appreciable speedup.
25 | """
26 |
27 | import numpy as np
28 | try:
29 | from numba import vectorize
30 | except ImportError:
31 | vectorize = None
32 |
33 | class NGP:
34 | """Nearest-Grid-Point
35 | """
36 | support = 1.0
37 |
38 | @staticmethod
39 | def __call__(x):
40 | if abs(x) < 0.5:
41 | return 1.0
42 | return 0.0
43 |
44 | class CIC:
45 | """Cloud-in-Cell
46 | """
47 | support = 2.0
48 |
49 | @staticmethod
50 | def __call__(x):
51 | if abs(x) < 1.0:
52 | return 1.0 - abs(x)
53 | return 0.0
54 |
55 | def __apply_MapOp(mesh, particles_pos, shape_function, mapOp):
56 | half_supp = 0.5 * shape_function.support
57 | num_coeffs_per_axis = int(np.ceil(shape_function.support))
58 | dim = len(np.shape(mesh))
59 |
60 | # ndarray which holds the non-zero particle assignment values of one particle
61 | coeffs = np.empty([num_coeffs_per_axis] * dim)
62 |
63 | if vectorize:
64 | # vectorize shape function to a compiled ufunc
65 | v_shape_function = vectorize(['f4(f4)', 'f8(f8)'])(shape_function.__call__)
66 | else:
67 | # numpy fallback
68 | v_shape_function = np.vectorize(shape_function.__call__)
69 |
70 | for idx, pos in enumerate(particles_pos):
71 | # lowest mesh indices that contributes to the mapping
72 | begin = np.ceil(pos - np.ones(dim) * half_supp).astype(int)
73 |
74 | # rearrange mapping area if it overlaps the border
75 | begin = np.maximum(begin, np.zeros(dim, dtype=int))
76 | begin = np.minimum(begin, np.shape(mesh) - np.ones(dim) * half_supp)
77 |
78 | # compute coefficients (particle assignment values)
79 | for coeff_idx in np.ndindex(np.shape(coeffs)):
80 | rel_vec = begin + coeff_idx - pos
81 | coeffs[coeff_idx] = np.prod(v_shape_function(rel_vec))
82 |
83 | # do the actual mapping
84 | window = [slice(begin[i], begin[i] + num_coeffs_per_axis) for i in range(dim)]
85 | mapOp(mesh[window], coeffs, idx)
86 |
87 |
88 | def mesh2particles(mesh, particles_pos, shape_function=CIC):
89 | """
90 | Map mesh values to particles according to a particle shape function.
91 |
92 | :param array-like mesh: n-dimensional array.
93 | Dimension of *mesh* has to be greater or equal to the number of particle position components.
94 | :param particles_pos:
95 | 'd'-dim tuples for 'N' particle positions. The positions can be float32 or float64 and
96 | must be within the shape of *mesh*.
97 | :type particles_pos: (N, d)
98 | :param shape_function:
99 | Callable object returning the particle assignment value for a given param 'x'.
100 | Has to provide a 'support' float attribute which defines the width of the non-zero area.
101 | Defaults to cloud-in-cell.
102 | :type shape_function: callable, optional
103 | :return: Mapped mesh values for each particle.
104 | :type return: ndarray(N, dtype='f')
105 |
106 | Notes:
107 | - The particle shape function is not evaluated outside the mesh.
108 | """
109 | # resulting ndarray
110 | particles = np.empty(len(particles_pos), dtype='f')
111 |
112 | def mapOp(sub_mesh, coeffs, particle_idx):
113 | particles[particle_idx] = np.add.reduce(sub_mesh * coeffs, axis=None)
114 |
115 | __apply_MapOp(mesh, particles_pos, shape_function, mapOp)
116 |
117 | return particles
118 |
119 | def particles2mesh(mesh, particles, particles_pos, shape_function=CIC):
120 | """
121 | Map particle values to mesh according to a particle shape function.
122 | Particle values are added to the mesh.
123 |
124 | :param array-like mesh: n-dimensional array.
125 | Dimension of *mesh* has to be greater or equal to the number of particle position components.
126 | :param particles: particle data.
127 | len(*particles*) has to be the same as len(*particles_pos*)
128 | :type particles: array_like (1 dim)
129 | :param particles_pos:
130 | 'd'-dim tuples for 'N' particle positions. The positions can be float32 or float64 and
131 | must be within the shape of *mesh*.
132 | :type particles_pos: (N, d)
133 | :param shape_function:
134 | Callable object returning the particle assignment value for a given param 'x'.
135 | Has to provide a 'support' float attribute which defines the width of the non-zero area.
136 | Defaults to cloud-in-cell.
137 | :type shape_function: callable, optional
138 | :return: *mesh*
139 |
140 | Notes:
141 | - The particle shape function is not evaluated outside the mesh.
142 | """
143 |
144 | def mapOp(sub_mesh, coeffs, particle_idx):
145 | sub_mesh += particles[particle_idx] * coeffs
146 |
147 | __apply_MapOp(mesh, particles_pos, shape_function, mapOp)
148 |
149 | return mesh
150 |
--------------------------------------------------------------------------------
/pyDive/picongpu.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2014 Heiko Burau
3 |
4 | This file is part of pyDive.
5 |
6 | pyDive is free software: you can redistribute it and/or modify
7 | it under the terms of of either the GNU General Public License or
8 | the GNU Lesser General Public License as published by
9 | the Free Software Foundation, either version 3 of the License, or
10 | (at your option) any later version.
11 | pyDive is distributed in the hope that it will be useful,
12 | but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | GNU General Public License and the GNU Lesser General Public License
15 | for more details.
16 |
17 | You should have received a copy of the GNU General Public License
18 | and the GNU Lesser General Public License along with pyDive.
19 | If not, see .
20 | """
21 |
22 | __doc__=\
23 | """This module holds convenient functions for those who use pyDive together with `picongpu \
24 | `_.
25 | """
26 |
27 | import os
28 | import os.path
29 | import re
30 | import arrays.h5_ndarray as h5
31 | import structured
32 |
33 | def loadSteps(steps, folder_path, data_path, distaxis=0):
34 | """Python generator object looping all hdf5-data found in *folder_path*
35 | from timesteps appearing in *steps*.
36 |
37 | This generator doesn't read or write any data elements from hdf5 but returns dataset-handles
38 | covered by *pyDive.h5_ndarray* objects.
39 |
40 | All datasets within *data_path* must have the same shape.
41 |
42 | :param ints steps: list of timesteps to loop
43 | :param str folder_path: Path to the folder containing the hdf5-files
44 | :param str data_path: Relative path starting from "/data//" within hdf5-file to the dataset or group of datasets
45 | :param int distaxis: axis on which datasets are distributed over when once loaded into memory.
46 | :return: tuple of timestep and a :ref:`pyDive.h5_ndarray `
47 | or a structure of pyDive.h5_ndarrays (:mod:`pyDive.structured`). Ordering is done by timestep.
48 |
49 | Notes:
50 | - If the dataset has a '**sim_unit**' attribute its value is stored in ``h5array.unit``.
51 | """
52 | assert os.path.exists(folder_path), "folder '%s' does not exist" % folder_path
53 |
54 | timestep_and_filename = []
55 | for filename in os.listdir(folder_path):
56 | if not filename.endswith('.h5'): continue
57 |
58 | timestep = int(re.findall("\d+", filename)[-2])
59 | if not timestep in steps: continue
60 | timestep_and_filename.append((timestep, filename))
61 |
62 | # sort by timestep
63 | timestep_and_filename.sort(key=lambda item: item[0])
64 |
65 | for timestep, filename in timestep_and_filename:
66 | full_filename = os.path.join(folder_path, filename)
67 | full_datapath = os.path.join("/data", str(timestep), data_path)
68 |
69 | h5data = h5.open(full_filename, full_datapath, distaxis)
70 |
71 | # add 'sim_unit' as 'unit' attribute
72 | def add_sim_unit(array):
73 | if hasattr(array, "attrs") and 'sim_unit' in array.attrs:
74 | setattr(array, "unit", array.attrs["sim_unit"])
75 | return array
76 | if type(h5data) is h5.h5_ndarray:
77 | h5data = add_sim_unit(h5data)
78 | else:
79 | h5data = structured.structured(\
80 | structured.makeTree_fromTree(h5data.structOfArrays, add_sim_unit))
81 |
82 | yield timestep, h5data
83 |
84 | def getSteps(folder_path):
85 | """Returns a list of all timesteps in *folder_path*.
86 | """
87 | assert os.path.exists(folder_path), "folder '%s' does not exist" % folder_path
88 |
89 | result = []
90 | for filename in os.listdir(folder_path):
91 | if not filename.endswith('.h5'): continue
92 | timestep = int(re.findall("\d+", filename)[-2])
93 | result.append(timestep)
94 | return result
95 |
96 | def loadAllSteps(folder_path, data_path, distaxis=0):
97 | """Python generator object looping hdf5-data of all timesteps found in *folder_path*.
98 |
99 | This generator doesn't read or write any data elements from hdf5 but returns dataset-handles
100 | covered by *pyDive.h5_ndarray* objects.
101 |
102 | All datasets within *data_path* must have the same shape.
103 |
104 | :param str folder_path: Path to the folder containing the hdf5-files
105 | :param str data_path: Relative path starting from "/data//" within hdf5-file to the dataset or group of datasets
106 | :param int distaxis: axis on which datasets are distributed over when once loaded into memory.
107 | :return: tuple of timestep and a :ref:`pyDive.h5_ndarray `
108 | or a structure of pyDive.h5_ndarrays (:mod:`pyDive.structured`). Ordering is done by timestep.
109 |
110 | Notes:
111 | - If the dataset has a '**sim_unit**' attribute its value is stored in ``h5array.unit``.
112 | """
113 | steps = getSteps(folder_path)
114 |
115 | for timestep, data in loadSteps(steps, folder_path, data_path, distaxis):
116 | yield timestep, data
117 |
118 | def loadStep(step, folder_path, data_path, distaxis=0):
119 | """Load hdf5-data from a single timestep found in *folder_path*.
120 |
121 | All datasets within *data_path* must have the same shape.
122 |
123 | :param int step: timestep
124 | :param str folder_path: Path to the folder containing the hdf5-files
125 | :param str data_path: Relative path starting from "/data//" within hdf5-file to the dataset or group of datasets
126 | :param int distaxis: axis on which datasets are distributed over when once loaded into memory.
127 | :return: :ref:`pyDive.h5_ndarray `
128 | or a structure of pyDive.h5_ndarrays (:mod:`pyDive.structured`).
129 |
130 | Notes:
131 | - If the dataset has a '**sim_unit**' attribute its value is stored in ``h5array.unit``.
132 | """
133 | step, field = loadSteps([step], folder_path, data_path, distaxis).next()
134 | return field
--------------------------------------------------------------------------------
/pyDive/pyDive.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2014 Heiko Burau
3 |
4 | This file is part of pyDive.
5 |
6 | pyDive is free software: you can redistribute it and/or modify
7 | it under the terms of of either the GNU General Public License or
8 | the GNU Lesser General Public License as published by
9 | the Free Software Foundation, either version 3 of the License, or
10 | (at your option) any later version.
11 | pyDive is distributed in the hope that it will be useful,
12 | but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | GNU General Public License and the GNU Lesser General Public License
15 | for more details.
16 |
17 | You should have received a copy of the GNU General Public License
18 | and the GNU Lesser General Public License along with pyDive.
19 | If not, see .
20 | """
21 |
22 | __doc__=\
23 | """Make most used functions and modules directly accessable from pyDive."""
24 |
25 | # ndarray
26 | import arrays.ndarray
27 | globals().update(arrays.ndarray.factories)
28 | globals().update(arrays.ndarray.ufuncs)
29 | from arrays.ndarray import ndarray
30 |
31 | # hdf5
32 | try:
33 | import arrays.h5_ndarray as h5
34 | except ImportError:
35 | pass
36 |
37 | # adios
38 | try:
39 | import arrays.ad_ndarray as adios
40 | except ImportError:
41 | pass
42 |
43 | # gpu
44 | try:
45 | import arrays.gpu_ndarray as gpu
46 | except ImportError:
47 | pass
48 |
49 | # cloned_ndarray
50 | import cloned_ndarray.factories
51 | cloned = cloned_ndarray.factories
52 |
53 | # fragmentation
54 | from fragment import fragment
55 |
56 | ## algorithm
57 | import algorithm
58 | map = algorithm.map
59 | reduce = algorithm.reduce
60 | mapReduce = algorithm.mapReduce
61 |
62 | # particle-mesh mappings
63 | import mappings
64 | mesh2particles = mappings.mesh2particles
65 | particles2mesh = mappings.particles2mesh
66 |
67 | # structured
68 | import structured
69 | structured = structured.structured
70 |
71 | # picongpu
72 | import picongpu
73 | picongpu = picongpu
74 |
75 | # init
76 | import IPParallelClient
77 | init = IPParallelClient.init
78 |
79 |
80 | # module doc
81 | items = [item for item in globals().items() if not item[0].startswith("__")]
82 | items.sort(key=lambda item: item[0])
83 |
84 | fun_names = [":obj:`" + item[0] + "<" + item[1].__module__ + "." + item[0] + ">`"\
85 | for item in items if hasattr(item[1], "__module__")]
86 |
87 | import inspect
88 | module_names = [":mod:`" + item[0] + "<" + item[1].__name__ + ">`"\
89 | for item in items if inspect.ismodule(item[1])]
90 |
91 | __doc__ += "\n\n**Functions**:\n\n" + "\n\n".join(fun_names)\
92 | + "\n\n**Modules**:\n\n" + "\n\n".join(module_names)
93 |
--------------------------------------------------------------------------------
/pyDive/structured.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2014 Heiko Burau
3 |
4 | This file is part of pyDive.
5 |
6 | pyDive is free software: you can redistribute it and/or modify
7 | it under the terms of of either the GNU General Public License or
8 | the GNU Lesser General Public License as published by
9 | the Free Software Foundation, either version 3 of the License, or
10 | (at your option) any later version.
11 | pyDive is distributed in the hope that it will be useful,
12 | but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | GNU General Public License and the GNU Lesser General Public License
15 | for more details.
16 |
17 | You should have received a copy of the GNU General Public License
18 | and the GNU Lesser General Public License along with pyDive.
19 | If not, see .
20 | """
21 |
22 | __doc__ =\
23 | """The *structured* module addresses the common problem when dealing with
24 | structured data: While the user likes an array-of-structures layout the machine prefers a structure-of-arrays.
25 | In pyDive the method of choice is a *virtual* *array-of-structures*-object. It holds array-like attributes
26 | such as shape and dtype and allows for slicing but is operating on a structure-of-arrays internally.
27 |
28 | Example: ::
29 |
30 | ...
31 | treeOfArrays = {"FieldE" :
32 | {"x" : fielde_x,
33 | "y" : fielde_y,
34 | "z" : fielde_z},
35 | "FieldB" :
36 | {"x" : fieldb_x,
37 | "y" : fieldb_y,
38 | "z" : fieldb_z}
39 | }
40 |
41 | fields = pyDive.structured(treeOfArrays)
42 |
43 | half = fields[::2]["FieldE/x"]
44 | # equivalent to
45 | half = fields["FieldE/x"][::2]
46 | # equivalent to
47 | half = fields["FieldE"]["x"][::2]
48 | # equivalent to
49 | half = fields["FieldE"][::2]["x"]
50 |
51 | # equivalent to
52 | half = fields.FieldE.x[::2]
53 |
54 | The example shows that in fact *fields* can be treated as an array-of-structures
55 | **or** a structure-of-arrays depending on what is more appropriate.
56 |
57 | The goal is to make the virtual *array-of-structs*-object look like a real array. Therefore
58 | every method call or operation is forwarded to the individual arrays.::
59 |
60 | new_field = fields.FieldE.astype(np.int) + fields.FieldB.astype(np.float)
61 |
62 | Here the forwarded method calls are ``astype`` and ``__add__``.
63 | """
64 |
65 | import sys
66 | import os
67 | # check whether this code is executed on target or not
68 | onTarget = os.environ.get("onTarget", 'False')
69 | if onTarget == 'False':
70 | import IPParallelClient as com
71 | from IPython.parallel import interactive
72 | import numpy as np
73 |
74 | def makeTree_fromTree(tree, expression):
75 | def traverseTree(outTree, inTree):
76 | for key, value in inTree.items():
77 | if type(value) is dict:
78 | outTree[key] = {}
79 | traverseTree(outTree[key], value)
80 | else:
81 | outTree[key] = expression(value)
82 | outTree = {}
83 | traverseTree(outTree, tree)
84 | return outTree
85 |
86 | def makeTree_fromTwoTrees(treeA, treeB, expression):
87 | def traverseTrees(outTree, inTreeA, inTreeB):
88 | for key, valueA in inTreeA.items():
89 | valueB = inTreeB[key]
90 | if type(valueA) is dict:
91 | outTree[key] = {}
92 | traverseTrees(outTree[key], valueA, valueB)
93 | else:
94 | outTree[key] = expression(valueA, valueB)
95 | outTree = {}
96 | traverseTrees(outTree, treeA, treeB)
97 | return outTree
98 |
99 | def visitTwoTrees(treeA, treeB, visitor):
100 | # traverse trees
101 | for key, valueA in treeA.items():
102 | valueB = treeB[key]
103 | if type(valueA) is dict:
104 | visitTwoTrees(valueA, valueB, visitor)
105 | else:
106 | visitor(treeA, treeB, key, valueA, valueB)
107 | return treeA, treeB
108 |
109 | # generator object which iterates the tree's leafs
110 | def treeItems(tree):
111 | items = list(tree.items())
112 | while items:
113 | key, value = items.pop()
114 | if type(value) is dict:
115 | items += list(value.items())
116 | else:
117 | yield key, value
118 |
119 | class ForeachLeafCall(object):
120 | def __init__(self, tree, op):
121 | self.tree = tree
122 | self.op = op
123 |
124 | def __call__(self, *args, **kwargs):
125 | def apply_unary(a):
126 | f = getattr(a, self.op)
127 | return f(*args, **kwargs)
128 | def apply_binary(a, b):
129 | f = getattr(a, self.op)
130 | return f(b, *args[1:], **kwargs)
131 |
132 | if args and type(args[0]).__name__ == "VirtualArrayOfStructs":
133 | structOfArrays = makeTree_fromTwoTrees(self.tree, args[0].structOfArrays, apply_binary)
134 | else:
135 | structOfArrays = makeTree_fromTree(self.tree, apply_unary)
136 | return structured(structOfArrays)
137 |
138 |
139 | arrayOfStructs_id = 0
140 |
141 | class VirtualArrayOfStructs(object):
142 | def __init__(self, structOfArrays):
143 | items = [item for item in treeItems(structOfArrays)]
144 | self.firstArray = items[0][1]
145 | self.arraytype = type(self.firstArray)
146 | assert all(type(a) == self.arraytype for name, a in items),\
147 | "all arrays in 'structOfArrays' must be of the same type: " +\
148 | str({name : type(a) for name, a in items})
149 | assert all(a.shape == self.firstArray.shape for name, a in items),\
150 | "all arrays in 'structOfArrays' must have the same shape: " +\
151 | str({name : a.shape for name, a in items})
152 |
153 | self.shape = self.firstArray.shape
154 | self.dtype = makeTree_fromTree(structOfArrays, lambda a: a.dtype)
155 | self.nbytes = sum(a.nbytes for name, a in items)
156 | self.structOfArrays = structOfArrays
157 | self.has_local_instance = False
158 |
159 | def __del__(self):
160 | if onTarget == 'False' and self.has_local_instance:
161 | # delete remote structured object
162 | self.view.execute('del %s' % self.name, targets=self.target_ranks)
163 |
164 | def __getattr__(self, name):
165 | if hasattr(self.firstArray, name):
166 | assert hasattr(getattr(self.firstArray, name), "__call__"),\
167 | "Unlike method access, attribute access of individual arrays is not supported."
168 | return ForeachLeafCall(self.structOfArrays, name)
169 |
170 | return self[name]
171 |
172 | def __special_operation__(self, op, *args):
173 | return ForeachLeafCall(self.structOfArrays, op)(*args)
174 |
175 | def __repr__(self):
176 | # if arrays are distributed create a local representation of this object on engine
177 | if onTarget == 'False' and not self.has_local_instance and hasattr(self.firstArray, "target_ranks"):
178 | items = [item for item in treeItems(self.structOfArrays)]
179 | assert all(self.firstArray.is_distributed_like(a) for name, a in items),\
180 | "Cannot create a local virtual array-of-structs because not all arrays are distributed equally."
181 |
182 | self.distaxes = self.firstArray.distaxes
183 | self.target_offsets = self.firstArray.target_offsets
184 | self.target_ranks = self.firstArray.target_ranks
185 | view = com.getView()
186 | self.view = view
187 |
188 | # generate a unique variable name used on target representing this instance
189 | global arrayOfStructs_id
190 | self.name = 'arrayOfStructsObj' + str(arrayOfStructs_id)
191 | arrayOfStructs_id += 1
192 |
193 | # create a VirtualArrayOfStructs object containing the local arrays on the targets in use
194 | names_tree = makeTree_fromTree(self.structOfArrays, lambda a: repr(a))
195 |
196 | view.push({'names_tree' : names_tree}, targets=self.target_ranks)
197 |
198 | view.execute('''\
199 | structOfArrays = structured.makeTree_fromTree(names_tree, lambda a_name: globals()[a_name])
200 | %s = structured.structured(structOfArrays)''' % self.name,\
201 | targets=self.target_ranks)
202 |
203 | self.has_local_instance = True
204 |
205 | return self.name
206 |
207 | def __str__(self):
208 | def printTree(tree, indent, result):
209 | for key, value in tree.items():
210 | if type(value) is dict:
211 | result += indent + key + ":\n"
212 | result = printTree(tree[key], indent + " ", result)
213 | else:
214 | result += indent + key + " -> " + str(value.dtype) + "\n"
215 | return result
216 |
217 | result = "VirtualArrayOfStructs:\n"
219 | return printTree(self.structOfArrays, " ", result)
220 |
221 | def __getitem__(self, args):
222 | # component access
223 | # ----------------
224 | if type(args) is str:
225 | node = self.structOfArrays # root node
226 | path = args.split('/')
227 | for node_name in path:
228 | node = node[node_name]
229 | if type(node) is dict:
230 | # node
231 | return structured(node)
232 | else:
233 | # leaf
234 | return node
235 |
236 | # slicing
237 | # -------
238 | result = makeTree_fromTree(self.structOfArrays, lambda a: a[args])
239 |
240 | # if args is a list of indices then return a single data value tree
241 | if type(args) not in (list, tuple):
242 | args = (args,)
243 | if all(type(arg) is int for arg in args):
244 | return result
245 |
246 | return structured(result)
247 |
248 | def __setitem__(self, args, other):
249 | # component access
250 | # ----------------
251 | if type(args) is str:
252 | node = self.structOfArrays # root node
253 | path = args.split('/')
254 | for node_name in path[:-1]:
255 | node = node[node_name]
256 | last_node_name = path[-1]
257 | if type(node[last_node_name]) is dict:
258 | # node
259 | def doArrayAssignment(treeA, treeB, name, arrayA, arrayB):
260 | treeA[name] = arrayB
261 |
262 | visitTwoTrees(node[last_node_name], other.structOfArrays, doArrayAssignment)
263 | else:
264 | # leaf
265 | node[last_node_name] = other
266 | return
267 |
268 | # slicing
269 | # -------
270 | def doArrayAssignmentWithSlice(treeA, treeB, name, arrayA, arrayB):
271 | arrayA[args] = arrayB
272 |
273 | visitTwoTrees(self.structOfArrays, other.structOfArrays, doArrayAssignmentWithSlice)
274 |
275 | # Add special methods like "__add__", "__sub__", ... that call __special_operation__
276 | # forwarding them to the individual arrays.
277 | # All ordinary methods are forwarded by __getattr__
278 |
279 | binary_ops = ["add", "sub", "mul", "floordiv", "div", "mod", "pow", "lshift", "rshift", "and", "xor", "or"]
280 |
281 | binary_iops = ["__i" + op + "__" for op in binary_ops]
282 | binary_rops = ["__r" + op + "__" for op in binary_ops]
283 | binary_ops = ["__" + op + "__" for op in binary_ops]
284 | unary_ops = ["__neg__", "__pos__", "__abs__", "__invert__", "__complex__", "__int__", "__long__", "__float__", "__oct__", "__hex__"]
285 | comp_ops = ["__lt__", "__le__", "__eq__", "__ne__", "__ge__", "__gt__"]
286 |
287 | make_special_op = lambda op: lambda self, *args: self.__special_operation__(op, *args)
288 |
289 | special_ops_dict = {op : make_special_op(op) for op in binary_ops + binary_rops + unary_ops + comp_ops}
290 |
291 | from types import MethodType
292 |
293 | for name, func in special_ops_dict.items():
294 | setattr(VirtualArrayOfStructs, name, MethodType(func, None, VirtualArrayOfStructs))
295 |
296 |
297 | def structured(structOfArrays):
298 | """Convert a *structure-of-arrays* into a virtual *array-of-structures*.
299 |
300 | :param structOfArrays: tree-like (dict-of-dicts) dictionary of arrays.
301 | :raises AssertionError: if the *arrays-types* do not match. Datatypes may differ.
302 | :raises AssertionError: if the shapes do not match.
303 | :return: Custom object representing a virtual array whose elements have the same tree-like structure
304 | as *structOfArrays*.
305 | """
306 |
307 | return VirtualArrayOfStructs(structOfArrays)
308 |
--------------------------------------------------------------------------------
/pyDive/test/__init__.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2014 Heiko Burau
3 |
4 | This file is part of pyDive.
5 |
6 | pyDive is free software: you can redistribute it and/or modify
7 | it under the terms of of either the GNU General Public License or
8 | the GNU Lesser General Public License as published by
9 | the Free Software Foundation, either version 3 of the License, or
10 | (at your option) any later version.
11 | pyDive is distributed in the hope that it will be useful,
12 | but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | GNU General Public License and the GNU Lesser General Public License
15 | for more details.
16 |
17 | You should have received a copy of the GNU General Public License
18 | and the GNU Lesser General Public License along with pyDive.
19 | If not, see .
20 | """
21 | __doc__ = None
22 |
--------------------------------------------------------------------------------
/pyDive/test/conftest.py:
--------------------------------------------------------------------------------
1 | import pyDive
2 | import pytest
3 | import os
4 |
5 | @pytest.fixture(scope="session")
6 | def init_pyDive(request):
7 | pyDive.init(os.environ["IPP_PROFILE_NAME"])
--------------------------------------------------------------------------------
/pyDive/test/m2p_CIC.npy:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ComputationalRadiationPhysics/pyDive/67d36420c2f1d59bef8633b2dea94b4ea4ab8ceb/pyDive/test/m2p_CIC.npy
--------------------------------------------------------------------------------
/pyDive/test/p2m_CIC.npy:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ComputationalRadiationPhysics/pyDive/67d36420c2f1d59bef8633b2dea94b4ea4ab8ceb/pyDive/test/p2m_CIC.npy
--------------------------------------------------------------------------------
/pyDive/test/sample.h5:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ComputationalRadiationPhysics/pyDive/67d36420c2f1d59bef8633b2dea94b4ea4ab8ceb/pyDive/test/sample.h5
--------------------------------------------------------------------------------
/pyDive/test/test_algorithm.py:
--------------------------------------------------------------------------------
1 | import pyDive
2 | import numpy as np
3 | import os
4 |
5 | input_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), "sample.h5")
6 |
7 | def test_map(init_pyDive):
8 | input_array = pyDive.h5.open(input_file, "fields")
9 |
10 | ref_array = input_array["fieldE/x"].load().gather()**2 \
11 | + input_array["fieldE/y"].load().gather()**2 \
12 | + input_array["fieldB/z"].load().gather()**2
13 |
14 | test_array = pyDive.empty(input_array.shape, dtype=input_array.dtype["fieldE"]["x"])
15 |
16 | def energy(out, h5fields):
17 | fields = h5fields.load()
18 | out[:] = fields["fieldE/x"]**2 + fields["fieldE/y"]**2 + fields["fieldB/z"]**2
19 |
20 | pyDive.map(energy, test_array, input_array)
21 |
22 | assert np.array_equal(ref_array, test_array.gather())
23 |
24 | def test_reduce(init_pyDive):
25 | input_array = pyDive.h5.open(input_file, "fields").load()
26 |
27 | energy_array = pyDive.empty(input_array.shape, dtype=input_array.dtype["fieldE"]["x"])
28 |
29 | def energy(out, fields):
30 | out[:] = fields["fieldE/x"]**2 + fields["fieldE/y"]**2 + fields["fieldB/z"]**2
31 |
32 | pyDive.map(energy, energy_array, input_array)
33 |
34 | test_total = pyDive.reduce(energy_array, np.add)
35 | ref_total = np.add.reduce(energy_array.gather(), axis=None)
36 |
37 | diff = abs(ref_total - test_total)
38 | assert diff / ref_total < 1.0e-5
39 |
40 | def test_mapReduce(init_pyDive):
41 | input_array = pyDive.h5.open(input_file, "fields")
42 |
43 | ref_array = input_array["fieldE/x"].load().gather()**2 \
44 | + input_array["fieldE/y"].load().gather()**2 \
45 | + input_array["fieldB/z"].load().gather()**2
46 | ref_total = np.add.reduce(ref_array, axis=None)
47 |
48 | test_total = pyDive.mapReduce(\
49 | lambda fields: fields["fieldE/x"].load()**2 + fields["fieldE/y"].load()**2 + fields["fieldB/z"].load()**2,
50 | np.add, input_array)
51 |
52 | diff = abs(ref_total - test_total)
53 | assert diff / ref_total < 1.0e-5
--------------------------------------------------------------------------------
/pyDive/test/test_cloned_ndarray.py:
--------------------------------------------------------------------------------
1 | import pyDive
2 | import numpy as np
3 | import pytest
4 | import random
5 | from pyDive import IPParallelClient as com
6 |
7 | sizes = ((1,), (5,),
8 | (5, 29), (64, 1), (64, 64),
9 | (1, 1, 1), (12, 37, 50))
10 |
11 | def test_cloned_ndarray(init_pyDive):
12 | view = com.getView()
13 | for size in sizes:
14 | ref_array = np.arange(np.prod(size))
15 |
16 | test_array = pyDive.cloned.empty(size, dtype=np.int)
17 | test_array[:] = ref_array
18 |
19 | assert np.array_equal(ref_array * len(view), test_array.sum())
--------------------------------------------------------------------------------
/pyDive/test/test_gpu.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | import pyDive
4 | import numpy as np
5 | import random
6 | import pytest
7 |
8 | gpu_enabled = pytest.mark.skipif(not hasattr(pyDive, "gpu"), reason="pycuda not installed")
9 |
10 | sizes = ((1,), (5,), (29,), (64,),
11 | (1, 1), (1, 5), (5, 29), (64, 1), (64, 64),
12 | (1, 1, 1), (8, 8, 8), (1, 2, 3), (12, 37, 50), (64,64,64))
13 | dtypes = (np.float,)
14 |
15 | @gpu_enabled
16 | def test_basics(init_pyDive):
17 | for size in sizes:
18 | for dtype in dtypes:
19 | ref = (np.random.rand(*size) * 100.0).astype(dtype)
20 | test_array_cpu = pyDive.array(ref)
21 | test_array = pyDive.gpu.empty(size, dtype, distaxes=range(len(size)))
22 |
23 | test_array[:] = test_array_cpu
24 |
25 | if all(s > 3 for s in size):
26 | slices = [slice(1, -2, None) for i in range(len(size))]
27 |
28 | a = test_array[slices]
29 | a = a + 1
30 | a = a**2
31 | a = a + a
32 | test_array[slices] = a
33 |
34 | b = ref[slices]
35 | b = b + 1
36 | b = b**2
37 | b = b + b
38 | ref[slices] = b
39 |
40 | np.testing.assert_array_almost_equal(test_array.to_cpu().gather(), ref)
41 |
42 | @gpu_enabled
43 | def test_interengine(init_pyDive):
44 | for size in sizes:
45 | for dtype in dtypes:
46 | ref = (np.random.rand(*size) * 100.0).astype(dtype)
47 | test_array_cpu = pyDive.array(ref)
48 | test_array = pyDive.gpu.empty(size, dtype, distaxes=range(len(size)))
49 |
50 | test_array[:] = test_array_cpu
51 |
52 | for i in range(len(size)):
53 | if size[i] < 5: continue
54 |
55 | slicesA = [slice(None)] * len(size)
56 | slicesB = list(slicesA)
57 | slicesA[i] = slice(0, 5)
58 | slicesB[i] = slice(-5, None)
59 |
60 | test_array[slicesA] = test_array[slicesB]
61 | ref[slicesA] = ref[slicesB]
62 |
63 | assert np.array_equal(test_array.to_cpu().gather(), ref)
64 |
65 | slicesA = [s/2 for s in size]
66 | slicesB = list(slicesA)
67 | slicesA[i] = slice(0, 5)
68 | slicesB[i] = slice(-5, None)
69 |
70 | test_array[slicesA] = test_array[slicesB]
71 | ref[slicesA] = ref[slicesB]
72 |
73 | assert np.array_equal(test_array.to_cpu().gather(), ref)
74 |
75 | @gpu_enabled
76 | def test_misc(init_pyDive):
77 | sizes = ((10,20,30), (30,20,10), (16,16,16), (16,32,48), (13,29,37))
78 |
79 | def do_funny_stuff(a, b):
80 | a[1:,1:,1:] = b[:-1,:-1,:-1]
81 | b[1:,1:,1:] = a[1:,:-1,:-1]
82 | a[1:,1:,1:] = b[1:,1:,:-1]
83 | b[1:,1:,1:] = a[1:,:-1,1:]
84 | a[:-1,:-3,:-4] = b[1:,3:,4:]
85 | b[0,:,0] += a[-2,:,-2]
86 | a[4:-3,2:-1,5:-2] = b[4:-3,2:-1,5:-2]
87 | b[1:3,1:4,1:5] *= a[-3:-1,-4:-1,-5:-1]
88 | #a[1,2,3] -= b[3,2,1]
89 | b[:,:,0] = a[:,:,1]
90 |
91 | for size in sizes:
92 | cpu_a = (np.random.rand(*size) * 100.0).astype(np.int)
93 | cpu_b = (np.random.rand(*size) * 100.0).astype(np.int)
94 |
95 | gpu_a = pyDive.gpu.array(cpu_a)
96 | gpu_b = pyDive.gpu.array(cpu_b)
97 |
98 | do_funny_stuff(cpu_a, cpu_b)
99 | do_funny_stuff(gpu_a, gpu_b)
100 |
101 | assert np.array_equal(gpu_a.to_cpu(), cpu_a)
102 | assert np.array_equal(gpu_b.to_cpu(), cpu_b)
103 |
104 |
105 |
106 |
107 |
108 |
109 |
110 |
111 |
112 |
113 |
--------------------------------------------------------------------------------
/pyDive/test/test_h5_ndarray.py:
--------------------------------------------------------------------------------
1 | import pyDive
2 | import numpy as np
3 | import random
4 | import h5py as h5
5 | import os
6 |
7 | input_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), "sample.h5")
8 |
9 | def test_h5_ndarray1(init_pyDive):
10 | dataset = "particles/pos/x"
11 |
12 | test_array = pyDive.h5.open(input_file, dataset)
13 |
14 | ref_array = h5.File(input_file, "r")[dataset][:]
15 |
16 | assert np.array_equal(ref_array, test_array.load().gather())
17 |
18 | def test_h5_ndarray2(init_pyDive):
19 | window = np.s_[31:201:3, 2:233:5]
20 | dataset = "/fields/fieldE/x"
21 |
22 | test_array = pyDive.h5.open(input_file, dataset, distaxes=1)
23 |
24 | ref_array = h5.File(input_file, "r")[dataset][window]
25 |
26 | assert np.array_equal(ref_array, test_array[window].load().gather())
27 |
28 | def test_h5(init_pyDive):
29 | test_array = pyDive.h5.open(input_file, "particles/pos")
30 |
31 | ref_array_x = h5.File(input_file, "r")["particles/pos/x"][:]
32 | ref_array_y = h5.File(input_file, "r")["particles/pos/y"][:]
33 |
34 | assert np.array_equal(ref_array_x, test_array["x"].load().gather())
35 | assert np.array_equal(ref_array_y, test_array["y"].load().gather())
36 | assert np.array_equal(ref_array_x, test_array.load()["x"].gather())
37 | assert np.array_equal(ref_array_y, test_array.load()["y"].gather())
38 |
--------------------------------------------------------------------------------
/pyDive/test/test_mappings.py:
--------------------------------------------------------------------------------
1 | import pyDive
2 | import numpy as np
3 | import os
4 |
5 | dirname = os.path.dirname(os.path.abspath(__file__))
6 | input_file = os.path.join(dirname, "sample.h5")
7 |
8 | def test_particles2mesh(init_pyDive):
9 | shape = [256, 256]
10 | density = pyDive.cloned.zeros(shape)
11 |
12 | particles = pyDive.h5.open(input_file, "/particles")
13 |
14 | def particles2density(particles, density):
15 | particles = particles.load()
16 | total_pos = particles["cellidx"].astype(np.float32) + particles["pos"]
17 |
18 | # convert total_pos to an (N, 2) shaped array
19 | total_pos = np.hstack((total_pos["x"][:,np.newaxis],
20 | total_pos["y"][:,np.newaxis]))
21 |
22 | par_weighting = np.ones(particles.shape)
23 | import pyDive.mappings
24 | pyDive.mappings.particles2mesh(density, par_weighting, total_pos, pyDive.mappings.CIC)
25 |
26 | pyDive.map(particles2density, particles, density)
27 |
28 | test_density = density.sum() # add up all local copies
29 |
30 | ref_density = np.load(os.path.join(dirname, "p2m_CIC.npy"))
31 |
32 | np.testing.assert_array_almost_equal(ref_density, test_density)
33 |
34 | def test_mesh2particles(init_pyDive):
35 | particles = pyDive.h5.open(input_file, "/particles").load()
36 | field = pyDive.h5.open(input_file, "/fields/fieldB/z").load().gather()
37 |
38 | field_strengths = pyDive.empty(particles.shape)
39 |
40 | @pyDive.map
41 | def mesh2particles(field_strengths, particles, field):
42 | total_pos = particles["cellidx"].astype(np.float32) + particles["pos"]
43 |
44 | # convert total_pos to an (N, 2) shaped array
45 | total_pos = np.hstack((total_pos["x"][:,np.newaxis],
46 | total_pos["y"][:,np.newaxis]))
47 |
48 | import pyDive.mappings
49 | field_strengths[:] = pyDive.mappings.mesh2particles(field, total_pos, pyDive.mappings.CIC)
50 |
51 | mesh2particles(field_strengths, particles, field=field)
52 |
53 | ref_field_strengths = np.load(os.path.join(dirname, "m2p_CIC.npy"))
54 |
55 | assert np.array_equal(ref_field_strengths, field_strengths.gather())
56 |
57 |
--------------------------------------------------------------------------------
/pyDive/test/test_ndarray.py:
--------------------------------------------------------------------------------
1 | import pyDive
2 | import numpy as np
3 | import random
4 |
5 | sizes = ((1,), (5,), (29,), (64,),
6 | (1, 1), (1, 5), (5, 29), (64, 1), (64, 64),
7 | (1, 1, 1), (8, 8, 8), (1, 2, 3), (12, 37, 50), (64,64,64))
8 | dtypes = (np.int,)
9 |
10 | def test_slicing(init_pyDive):
11 | for size in sizes:
12 | for dtype in dtypes:
13 | ref = (np.random.rand(*size) * 100.0).astype(dtype)
14 |
15 | for distaxis in range(len(size)):
16 | test_array = pyDive.empty(size, dtype, distaxis)
17 | test_array[:] = ref
18 |
19 | slices = []
20 | for i in range(len(size)):
21 | start = size[i] / 3
22 | stop = size[i] - size[i] / 5
23 | step = 2
24 | slices.append(slice(start, stop, step))
25 |
26 | assert np.array_equal(ref[slices], test_array[slices].gather())
27 |
28 | slices = []
29 | for i in range(len(size)):
30 | slices.append(slice(-5, None, None))
31 |
32 | assert np.array_equal(ref[slices], test_array[slices].gather())
33 |
34 | slices = []
35 | for i in range(len(size)):
36 | slices.append(slice(0, 5, None))
37 |
38 | assert np.array_equal(ref[slices], test_array[slices].gather())
39 |
40 | # bitmask indexing
41 | bitmask = pyDive.array(np.random.rand(*size) > 0.5, distaxes=test_array.distaxes)
42 | assert ref[bitmask.gather()].shape == test_array[bitmask].shape
43 | # ordering can be distinct, thus merely check if sets are equal
44 | assert set(ref[bitmask.gather()]) == set(test_array[bitmask].gather())
45 |
46 | ref2 = ref.copy()
47 | test_array2 = test_array.copy()
48 | ref2[bitmask.gather()] = 1
49 | test_array2[bitmask] = 1
50 | assert np.array_equal(ref2, test_array2.gather())
51 |
52 | def test_multiple_axes(init_pyDive):
53 | for size in sizes:
54 | for dtype in dtypes:
55 | ref = (np.random.rand(*size) * 100.0).astype(dtype)
56 |
57 | for distaxes in [range(i+1) for i in range(len(size))]:
58 | test_array = pyDive.empty(size, dtype, distaxes)
59 | test_array[:] = ref
60 |
61 | slices = []
62 | for i in range(len(size)):
63 | start = size[i] / 3
64 | stop = size[i] - size[i] / 5
65 | step = 2
66 | slices.append(slice(start, stop, step))
67 |
68 | assert np.array_equal(ref[slices], test_array[slices].gather())
69 |
70 | slices = []
71 | for i in range(len(size)):
72 | slices.append(slice(-5, None, None))
73 |
74 | assert np.array_equal(ref[slices], test_array[slices].gather())
75 |
76 | slices = []
77 | for i in range(len(size)):
78 | slices.append(slice(0, 5, None))
79 |
80 | assert np.array_equal(ref[slices], test_array[slices].gather())
81 |
82 | # bitmask indexing
83 | bitmask = pyDive.array(np.random.rand(*size) > 0.5, distaxes=test_array.distaxes)
84 | assert ref[bitmask.gather()].shape == test_array[bitmask].shape
85 | # ordering can be distinct, thus merely check if sets are equal
86 | assert set(ref[bitmask.gather()]) == set(test_array[bitmask].gather())
87 |
88 | ref2 = ref.copy()
89 | test_array2 = test_array.copy()
90 | ref2[bitmask.gather()] = 1
91 | test_array2[bitmask] = 1
92 | assert np.array_equal(ref2, test_array2.gather())
93 |
94 | def test_interengine(init_pyDive):
95 | for size in sizes:
96 | for dtype in dtypes:
97 | ref = (np.random.rand(*size) * 100.0).astype(dtype)
98 |
99 | for distaxis in range(len(size)):
100 | if size[distaxis] < 5: continue
101 |
102 | test_array = pyDive.empty(size, dtype, distaxis)
103 | test_array[:] = ref
104 |
105 | slicesA = [slice(None)] * len(size)
106 | slicesB = list(slicesA)
107 | slicesA[distaxis] = slice(0, 5)
108 | slicesB[distaxis] = slice(-5, None)
109 |
110 | test_array[slicesA] = test_array[slicesB]
111 | ref[slicesA] = ref[slicesB]
112 |
113 | assert np.array_equal(test_array.gather(), ref)
114 |
115 | slicesA = [s/2 for s in size]
116 | slicesB = list(slicesA)
117 | slicesA[distaxis] = slice(0, 5)
118 | slicesB[distaxis] = slice(-5, None)
119 |
120 | test_array[slicesA] = test_array[slicesB]
121 | ref[slicesA] = ref[slicesB]
122 |
123 | assert np.array_equal(test_array.gather(), ref)
124 |
125 | def test_interengine_multiple_axes(init_pyDive):
126 | for size in sizes:
127 | for dtype in dtypes:
128 | ref = (np.random.rand(*size) * 100.0).astype(dtype)
129 |
130 | for distaxesA in [range(i+1) for i in range(len(size))]:
131 | for distaxesB in [range(i,len(size)) for i in range(len(size))]:
132 | test_arrayA = pyDive.empty(size, dtype, distaxesA)
133 | test_arrayB = pyDive.empty(size, dtype, distaxesB)
134 |
135 | test_arrayA[:] = ref
136 | test_arrayB[:] = ref
137 |
138 | for distaxis in range(len(size)):
139 | if size[distaxis] < 5: continue
140 |
141 | slicesA = [slice(None)] * len(size)
142 | slicesB = list(slicesA)
143 | slicesA[distaxis] = slice(0, 5)
144 | slicesB[distaxis] = slice(-5, None)
145 |
146 | ref_sum = ref[slicesA] + ref[slicesB]
147 | test_array_sum = test_arrayA[slicesA] + test_arrayB[slicesB]
148 |
149 | assert np.array_equal(ref_sum, test_array_sum.gather())
150 |
151 | def test_misc(init_pyDive):
152 | sizes = ((10,20,30), (30,20,10), (16,16,16), (16,32,48), (13,29,37))
153 |
154 | def do_funny_stuff(a, b):
155 | a[1:,1:,1:] = b[:-1,:-1,:-1]
156 | b[1:,1:,1:] = a[1:,:-1,:-1]
157 | a[1:,1:,1:] = b[1:,1:,:-1]
158 | b[1:,1:,1:] = a[1:,:-1,1:]
159 | a[:-1,:-3,:-4] = b[1:,3:,4:]
160 | b[0,:,0] += a[-2,:,-2]
161 | a[4:-3,2:-1,5:-2] = b[4:-3,2:-1,5:-2]
162 | b[1:3,1:4,1:5] *= a[-3:-1,-4:-1,-5:-1]
163 | a[1,2,3] -= b[3,2,1]
164 | b[:,:,0] = a[:,:,1]
165 |
166 | for size in sizes:
167 | print "size", size
168 | np_a = (np.random.rand(*size) * 100.0).astype(np.int)
169 | np_b = (np.random.rand(*size) * 100.0).astype(np.int)
170 |
171 | pd_a = pyDive.array(np_a, distaxes=(0,1,2))
172 | pd_b = pyDive.array(np_b, distaxes=(0,1,2))
173 |
174 | do_funny_stuff(np_a, np_b)
175 | do_funny_stuff(pd_a, pd_b)
176 |
177 | assert np.array_equal(pd_a, np_a)
178 | assert np.array_equal(pd_b, np_b)
--------------------------------------------------------------------------------
/pyDive/test/test_structured.py:
--------------------------------------------------------------------------------
1 | import pyDive
2 | import numpy as np
3 |
4 | def test_arrayOfStructs(init_pyDive):
5 | fieldE_x = np.random.rand(100, 100)
6 | fieldE_y = np.random.rand(100, 100)
7 | fieldB_z = np.random.rand(100, 100)
8 |
9 | fields = {"fieldE": {"x" : fieldE_x, "y" : fieldE_y}, "fieldB" : {"z" : fieldB_z}}
10 | fields = pyDive.structured(fields)
11 |
12 | assert np.array_equal(fieldB_z, fields["fieldB/z"])
13 | assert np.array_equal(fieldB_z, fields["fieldB"]["z"])
14 | assert np.array_equal(fieldB_z, fields["fieldB"][:]["z"])
15 | assert np.array_equal(fieldB_z, fields[:]["fieldB/z"])
16 | assert np.array_equal(fieldB_z**2, (fields**2)["fieldB/z"])
17 | assert np.array_equal(fieldB_z**2, (fields**2).fieldB.z)
--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | ipython>=2.0.0
2 | numpy>=1.8.1
3 | mpi4py>=1.3.1
4 | h5py>=2.3.0
5 | psutil>=2.1.1
6 | pyzmq>=14.3.0
7 |
--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2014 Heiko Burau
3 |
4 | This file is part of pyDive.
5 |
6 | pyDive is free software: you can redistribute it and/or modify
7 | it under the terms of of either the GNU General Public License or
8 | the GNU Lesser General Public License as published by
9 | the Free Software Foundation, either version 3 of the License, or
10 | (at your option) any later version.
11 | pyDive is distributed in the hope that it will be useful,
12 | but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | GNU General Public License and the GNU Lesser General Public License
15 | for more details.
16 |
17 | You should have received a copy of the GNU General Public License
18 | and the GNU Lesser General Public License along with pyDive.
19 | If not, see .
20 | """
21 |
22 | from __future__ import print_function
23 | from setuptools import setup, find_packages
24 | from setuptools.command.test import test as TestCommand
25 | import io
26 | import codecs
27 | import os
28 | import sys
29 | import subprocess
30 | import time
31 |
32 | import pyDive
33 |
34 | here = os.path.abspath(os.path.dirname(__file__))
35 |
36 | def read(*filenames, **kwargs):
37 | encoding = kwargs.get('encoding', 'utf-8')
38 | sep = kwargs.get('sep', '\n')
39 | buf = []
40 | for filename in filenames:
41 | with io.open(filename, encoding=encoding) as f:
42 | buf.append(f.read())
43 | return sep.join(buf)
44 |
45 | long_description = read('README.md')#, 'CHANGES.txt')
46 | requirements = [line.rstrip('\n') for line in open('requirements.txt')]
47 |
48 | class PyTest(TestCommand):
49 | def finalize_options(self):
50 | TestCommand.finalize_options(self)
51 | self.test_args = []
52 | self.test_suite = True
53 |
54 | def run_tests(self):
55 | profile_name = raw_input("Name of your IPython-parallel profile you want to run the tests with: ")
56 | n_engines = raw_input("Number of engines: ")
57 |
58 | # start ipcluster
59 | print("Waiting for engines to start...")
60 | subprocess.Popen(("ipcluster", "start", "--n=%s" % n_engines,\
61 | "--profile=%s" % profile_name))
62 | time.sleep(35)
63 |
64 | import pytest
65 | # set profile name as environment variable
66 | os.environ["IPP_PROFILE_NAME"] = profile_name
67 | errcode = pytest.main(self.test_args)
68 |
69 | # stop ipcluster
70 | subprocess.Popen(("ipcluster", "stop", "--profile=%s" % profile_name)).wait()
71 |
72 | sys.exit(errcode)
73 |
74 | setup(
75 | name='pyDive',
76 | version=pyDive.__version__,
77 | url='http://github.com/ComputationalRadiationPhysics/pyDive',
78 | license='GNU Affero General Public License v3',
79 | author='Heiko Burau',
80 | tests_require=['pytest'],
81 | install_requires=requirements,
82 | setup_requires=requirements,
83 | cmdclass={'test': PyTest},
84 | author_email='h.burau@hzdr.de',
85 | description='Distributed Interactive Visualization and Exploration of large datasets',
86 | long_description=long_description,
87 | packages=['pyDive', 'pyDive/distribution', 'pyDive/arrays', 'pyDive/arrays/local', 'pyDive/cloned_ndarray', 'pyDive/test'],
88 | include_package_data=True,
89 | platforms='any',
90 | #test_suite='sandman.test.test_sandman',
91 | classifiers = [
92 | 'Programming Language :: Python',
93 | 'Development Status :: 4 - Beta',
94 | 'Natural Language :: English',
95 | 'Environment :: Console',
96 | 'Environment :: Web Environment',
97 | 'Intended Audience :: Science/Research',
98 | 'License :: OSI Approved :: GNU Affero General Public License v3',
99 | 'Operating System :: OS Independent',
100 | 'Topic :: Scientific/Engineering :: Information Analysis'
101 | ],
102 | extras_require={'testing': ['pytest'],}
103 | )
104 |
--------------------------------------------------------------------------------