├── .github └── workflows │ ├── requirements-middle.txt │ ├── requirements-newest.txt │ ├── requirements-oldest.txt │ └── test.yml ├── .gitignore ├── LICENSE ├── MANIFEST.in ├── README.rst ├── changelog.txt ├── doc ├── Makefile ├── conf.py ├── developers.txt ├── index.txt ├── installation.txt ├── make.bat ├── performance.txt ├── reference.txt └── tutorial.txt ├── lazyarray.py ├── pyproject.toml ├── setup.py └── test ├── __init__.py ├── performance.py ├── test_lazy_arrays_from_Sparse_Matrices.py ├── test_lazyarray.py └── test_ufunc.py /.github/workflows/requirements-middle.txt: -------------------------------------------------------------------------------- 1 | numpy==1.23.5 2 | scipy==1.9.3 3 | -------------------------------------------------------------------------------- /.github/workflows/requirements-newest.txt: -------------------------------------------------------------------------------- 1 | numpy==2.0.1 2 | scipy==1.14.0 3 | -------------------------------------------------------------------------------- /.github/workflows/requirements-oldest.txt: -------------------------------------------------------------------------------- 1 | numpy==1.20.3 2 | scipy==1.7.3 3 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Run all tests 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | test: 11 | name: Test with Python ${{ matrix.python-version }} and ${{ matrix.req-version }} requirements on ${{ matrix.os }} 12 | runs-on: ${{ matrix.os }} 13 | strategy: 14 | fail-fast: true 15 | matrix: 16 | python-version: ["3.8", "3.10", "3.12"] 17 | req-version: ["oldest", "middle", "newest"] 18 | os: ["ubuntu-24.04"] 19 | exclude: 20 | - python-version: "3.8" 21 | req-version: "newest" 22 | - python-version: "3.10" 23 | req-version: "oldest" 24 | - python-version: "3.12" 25 | req-version: "oldest" 26 | - python-version: "3.12" 27 | req-version: "middle" 28 | steps: 29 | - uses: actions/checkout@v4 30 | - name: Set up Python ${{ matrix.python-version }} 31 | uses: actions/setup-python@v5 32 | with: 33 | python-version: ${{ matrix.python-version }} 34 | - name: Install requirements ${{ matrix.req-version }} 35 | run: | 36 | python -m pip install --upgrade pip 37 | pip install -r .github/workflows/requirements-${{ matrix.req-version }}.txt 38 | pip install pytest pytest-cov coveralls flake8 39 | pip install . 40 | - name: Lint with flake8 41 | run: | 42 | # stop the build if there are Python syntax errors or undefined names 43 | flake8 lazyarray.py --count --select=E9,F63,F7,F82 --show-source --statistics 44 | # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide 45 | flake8 lazyarray.py --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics 46 | - name: Run tests 47 | run: | 48 | pytest --cov=lazyarray -v 49 | - name: Upload coverage data 50 | run: | 51 | coveralls --service=github 52 | env: 53 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 54 | COVERALLS_FLAG_NAME: ${{ matrix.test-name }} 55 | COVERALLS_PARALLEL: true 56 | coveralls: 57 | name: Indicate completion to coveralls.io 58 | needs: test 59 | runs-on: ubuntu-latest 60 | container: python:3-slim 61 | steps: 62 | - name: Finished 63 | run: | 64 | pip3 install --upgrade coveralls 65 | coveralls --service=github --finish 66 | env: 67 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .coverage 2 | __pycache__/ 3 | cover/ 4 | lazyarray.egg-info/ 5 | *.pyc 6 | test/__pycache__/ 7 | MANIFEST 8 | dist/ 9 | doc/_build -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012-2024, Andrew P. Davison, Joël Chavas, Elodie Legouée (CNRS) and Ankur Sinha (UCL) 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 5 | 6 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 7 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 8 | Neither the names of the copyright holders nor the names of the contributors may be used to endorse or promote products derived from this software without specific prior written permission. 9 | 10 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 11 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include README.rst 3 | include changelog.txt 4 | recursive-include doc *.txt 5 | prune doc/_build -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | 2 | ========= 3 | lazyarray 4 | ========= 5 | 6 | lazyarray is a Python package that provides a lazily-evaluated numerical array 7 | class, ``larray``, based on and compatible with NumPy arrays. 8 | 9 | Lazy evaluation means that any operations on the array (potentially including 10 | array construction) are not performed immediately, but are delayed until 11 | evaluation is specifically requested. Evaluation of only parts of the array is 12 | also possible. 13 | 14 | Use of an ``larray`` can potentially save considerable computation time 15 | and memory in cases where: 16 | 17 | * arrays are used conditionally (i.e. there are cases in which the array is 18 | never used) 19 | * only parts of an array are used (for example in distributed computation, 20 | in which each MPI node operates on a subset of the elements of the array) 21 | 22 | 23 | .. image:: https://readthedocs.org/projects/lazyarray/badge/?version=latest 24 | :target: http://lazyarray.readthedocs.io/en/latest/ 25 | 26 | .. image:: https://github.com/NeuralEnsemble/lazyarrays/actions/workflows/test.yml/badge.svg 27 | :target: https://github.com/NeuralEnsemble/lazyarray/actions 28 | 29 | .. image:: https://coveralls.io/repos/github/NeuralEnsemble/lazyarray/badge.svg?branch=master 30 | :target: https://coveralls.io/github/NeuralEnsemble/lazyarray?branch=master 31 | -------------------------------------------------------------------------------- /changelog.txt: -------------------------------------------------------------------------------- 1 | ======= 2 | Changes 3 | ======= 4 | 5 | Release 0.2.0 6 | ============= 7 | 8 | * Fixed problems with deepcopying lazy arrays. 9 | * Optimization - uses `x.min()` rather than `min(x)` where possible. 10 | * Some fixes for when using boolean addressing. 11 | * Setting shape on an larray now also sets it on all larrays within the operations list. 12 | * Added `__eq__` method to larray. 13 | * Replaced some assertions with more specific Exceptions. 14 | * Added support for 'vectorized iterables', i.e. objects with a `next(n)` method so that you can return multiple values at once. 15 | * Fixed some bugs when creating a lazy array from an existing lazy array. 16 | * Added `dtype` attribute to `larray` class. 17 | 18 | 19 | Release 0.2.1 20 | ============= 21 | 22 | * Previous release didn't work with Python 3. 23 | 24 | 25 | Release 0.2.2 26 | ============= 27 | 28 | * Fixed behaviour of larray(VectorizedIterable) to match that of numpy array when indexing a single item. 29 | * Pulled out `partial_shape()` and `full_address()` methods of `larray` as standalone functions. 30 | * Better support for lists as masks. 31 | * `larray`s are now callable, provided their `base_value` is callable and the argument is another `larray`. 32 | 33 | 34 | Release 0.2.3 35 | ============= 36 | 37 | * Support `numpy.int64` as indices. 38 | * Better support for boolean indices. 39 | * Handle the case of constant `larray`s of size 1. 40 | 41 | 42 | Release 0.2.4 43 | ============= 44 | 45 | * Fixed bugs related to indexing multiple axes at the same time (`#3`_, `#4`_) 46 | 47 | 48 | Release 0.2.5 49 | ============= 50 | 51 | * Fixed a bug where the base value was homogeneous but one or more operations involved inhomogeneous arrays. 52 | 53 | 54 | Release 0.2.6 55 | ============= 56 | 57 | * Fixed a bug with callable lazyarrays. 58 | 59 | Release 0.2.7 60 | ============= 61 | 62 | * When deepcopying, VectorizedIterable objects as base_value are no longer copied, rather we keep a reference to the original. 63 | 64 | Release 0.2.8 65 | ============= 66 | 67 | * Slices which go past the array limits are now correctly handled. (`#5`_) 68 | 69 | Release 0.2.9 70 | ============= 71 | 72 | * Support base values of type `numpy.float` (which have an empty `shape` attribute) 73 | 74 | Release 0.2.10 75 | ============== 76 | 77 | * We don't create a new instance if the base value is already of the required dtype 78 | 79 | Release 0.3.0 80 | ============= 81 | 82 | * Support SciPy sparse matrices as base values 83 | * Support any object that implements a method `lazily_evaluate` as a base value 84 | * Allow more flexibility in checking 'equality' of types, e.g. accept an array of dtype `numpy.float32` when the specified dtype is `float` 85 | 86 | Release 0.3.1 87 | ============= 88 | 89 | * Fix a packaging issue, update project homepage. 90 | 91 | Release 0.3.2 92 | ============= 93 | 94 | * Ensure SciPy is optional 95 | 96 | Release 0.3.3 97 | ============= 98 | 99 | * Do not raise a "shape mismatch" `ValueError` if the value shape is empty. 100 | 101 | Release 0.3.4 102 | ============= 103 | 104 | * Add support for Brian quantities, and perhaps NumPy scalars in general 105 | * Updated to test with more recent versions of Python, NumPy and SciPy 106 | * Can now compare equality of lazyarrays to numbers and arrays 107 | 108 | Release 0.4.0 109 | ============= 110 | 111 | * Drop support for Python 2.7 112 | * Added a more general way to specify that an array-like object should be treated as a scalar by lazyarray (for arrays of arrays, etc.) 113 | 114 | Release 0.5.0 115 | ============= 116 | 117 | * Add partial support for NumPy ufuncs that require two arguments, e.g. "power". 118 | The second argument must be a scalar, array-valued second args are not yet supported. 119 | 120 | Release 0.5.1 121 | ============= 122 | 123 | * Fix problem where SciPy was required, not optional 124 | 125 | Release 0.5.2 126 | ============= 127 | 128 | * Add support for Python 3.10, remove testing for Python 3.4 and NumPy 1.12 129 | * Switch from nose to pytest for running tests 130 | 131 | Release 0.6.0 132 | ============= 133 | 134 | * Switch from setup.py to pyproject.toml 135 | * Add testing for Python 3.11, 3.12, remove testing for 3.5-3.7. 136 | * Tested with NumPy 2.0, minimum NumPy version tested is now 1.20. 137 | * Switch to GitHub Actions for continuous integration testing. 138 | 139 | .. _`#3`: https://bitbucket.org/apdavison/lazyarray/issue/3/ 140 | .. _`#4`: https://bitbucket.org/apdavison/lazyarray/issue/4/ 141 | .. _`#5`: https://bitbucket.org/apdavison/lazyarray/issue/5/ 142 | -------------------------------------------------------------------------------- /doc/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # Internal variables. 11 | PAPEROPT_a4 = -D latex_paper_size=a4 12 | PAPEROPT_letter = -D latex_paper_size=letter 13 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 14 | 15 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest 16 | 17 | help: 18 | @echo "Please use \`make ' where is one of" 19 | @echo " html to make standalone HTML files" 20 | @echo " dirhtml to make HTML files named index.html in directories" 21 | @echo " singlehtml to make a single large HTML file" 22 | @echo " pickle to make pickle files" 23 | @echo " json to make JSON files" 24 | @echo " htmlhelp to make HTML files and a HTML help project" 25 | @echo " qthelp to make HTML files and a qthelp project" 26 | @echo " devhelp to make HTML files and a Devhelp project" 27 | @echo " epub to make an epub" 28 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 29 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 30 | @echo " text to make text files" 31 | @echo " man to make manual pages" 32 | @echo " changes to make an overview of all changed/added/deprecated items" 33 | @echo " linkcheck to check all external links for integrity" 34 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 35 | 36 | clean: 37 | -rm -rf $(BUILDDIR)/* 38 | 39 | html: 40 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 41 | @echo 42 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 43 | 44 | dirhtml: 45 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 46 | @echo 47 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 48 | 49 | singlehtml: 50 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 51 | @echo 52 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 53 | 54 | pickle: 55 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 56 | @echo 57 | @echo "Build finished; now you can process the pickle files." 58 | 59 | json: 60 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 61 | @echo 62 | @echo "Build finished; now you can process the JSON files." 63 | 64 | htmlhelp: 65 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 66 | @echo 67 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 68 | ".hhp project file in $(BUILDDIR)/htmlhelp." 69 | 70 | qthelp: 71 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 72 | @echo 73 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 74 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 75 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/PyNN.qhcp" 76 | @echo "To view the help file:" 77 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/PyNN.qhc" 78 | 79 | devhelp: 80 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 81 | @echo 82 | @echo "Build finished." 83 | @echo "To view the help file:" 84 | @echo "# mkdir -p $$HOME/.local/share/devhelp/PyNN" 85 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/PyNN" 86 | @echo "# devhelp" 87 | 88 | epub: 89 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 90 | @echo 91 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 92 | 93 | latex: 94 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 95 | @echo 96 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 97 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 98 | "(use \`make latexpdf' here to do that automatically)." 99 | 100 | latexpdf: 101 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 102 | @echo "Running LaTeX files through pdflatex..." 103 | make -C $(BUILDDIR)/latex all-pdf 104 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 105 | 106 | text: 107 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 108 | @echo 109 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 110 | 111 | man: 112 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 113 | @echo 114 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 115 | 116 | changes: 117 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 118 | @echo 119 | @echo "The overview file is in $(BUILDDIR)/changes." 120 | 121 | linkcheck: 122 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 123 | @echo 124 | @echo "Link check complete; look for any errors in the above output " \ 125 | "or in $(BUILDDIR)/linkcheck/output.txt." 126 | 127 | doctest: 128 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 129 | @echo "Testing of doctests in the sources finished, look at the " \ 130 | "results in $(BUILDDIR)/doctest/output.txt." 131 | -------------------------------------------------------------------------------- /doc/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # lazyarray documentation build configuration file 4 | # 5 | # This file is execfile()d with the current directory set to its containing dir. 6 | # 7 | # Note that not all possible configuration values are present in this 8 | # autogenerated file. 9 | # 10 | # All configuration values have a default; values that are commented out 11 | # serve to show the default. 12 | 13 | import sys 14 | import os 15 | 16 | sys.path.append(os.path.abspath('..')) 17 | 18 | # If extensions (or modules to document with autodoc) are in another directory, 19 | # add these directories to sys.path here. If the directory is relative to the 20 | # documentation root, use os.path.abspath to make it absolute, like shown here. 21 | #sys.path.insert(0, os.path.abspath('.')) 22 | 23 | # -- General configuration ----------------------------------------------------- 24 | 25 | # If your documentation needs a minimal Sphinx version, state it here. 26 | #needs_sphinx = '1.0' 27 | 28 | # Add any Sphinx extension module names here, as strings. They can be extensions 29 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 30 | extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.todo', 'sphinx.ext.viewcode'] 31 | 32 | # Add any paths that contain templates here, relative to this directory. 33 | templates_path = ['_templates'] 34 | 35 | # The suffix of source filenames. 36 | source_suffix = '.txt' 37 | 38 | # The encoding of source files. 39 | #source_encoding = 'utf-8-sig' 40 | 41 | # The master toctree document. 42 | master_doc = 'index' 43 | 44 | # General information about the project. 45 | project = u'lazyarray' 46 | contributors = 'Andrew P. Davison, Joël Chavas, Elodie Legouée (CNRS) and Ankur Sinha (UCL)' 47 | copyright = f'2012-2024, {contributors}' 48 | 49 | # The version info for the project you're documenting, acts as replacement for 50 | # |version| and |release|, also used in various other places throughout the 51 | # built documents. 52 | # 53 | # The short X.Y version. 54 | version = '0.5' 55 | # The full version, including alpha/beta/rc tags. 56 | release = '0.6.0' 57 | 58 | # The language for content autogenerated by Sphinx. Refer to documentation 59 | # for a list of supported languages. 60 | #language = None 61 | 62 | # There are two options for replacing |today|: either, you set today to some 63 | # non-false value, then it is used: 64 | #today = '' 65 | # Else, today_fmt is used as the format for a strftime call. 66 | #today_fmt = '%B %d, %Y' 67 | 68 | # List of patterns, relative to source directory, that match files and 69 | # directories to ignore when looking for source files. 70 | exclude_patterns = ['_build'] 71 | 72 | # The reST default role (used for this markup: `text`) to use for all documents. 73 | #default_role = None 74 | 75 | # If true, '()' will be appended to :func: etc. cross-reference text. 76 | add_function_parentheses = True 77 | 78 | # If true, the current module name will be prepended to all description 79 | # unit titles (such as .. function::). 80 | #add_module_names = True 81 | 82 | # If true, sectionauthor and moduleauthor directives will be shown in the 83 | # output. They are ignored by default. 84 | #show_authors = False 85 | 86 | # The name of the Pygments (syntax highlighting) style to use. 87 | pygments_style = 'sphinx' 88 | 89 | # A list of ignored prefixes for module index sorting. 90 | #modindex_common_prefix = [] 91 | 92 | 93 | # -- Options for HTML output --------------------------------------------------- 94 | 95 | # The theme to use for HTML and HTML Help pages. See the documentation for 96 | # a list of builtin themes. 97 | html_theme = 'nature' # 'agogo', 'haiku' 98 | 99 | # Theme options are theme-specific and customize the look and feel of a theme 100 | # further. For a list of options available for each theme, see the 101 | # documentation. 102 | #html_theme_options = {} 103 | 104 | # Add any paths that contain custom themes here, relative to this directory. 105 | #html_theme_path = [] 106 | 107 | # The name for this set of Sphinx documents. If None, it defaults to 108 | # " v documentation". 109 | #html_title = None 110 | 111 | # A shorter title for the navigation bar. Default is the same as html_title. 112 | #html_short_title = None 113 | 114 | # The name of an image file (relative to this directory) to place at the top 115 | # of the sidebar. 116 | #html_logo = 'lazyarray_logo.png' 117 | 118 | # The name of an image file (within the static path) to use as favicon of the 119 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 120 | # pixels large. 121 | #html_favicon = 'lazyarray_icon.ico' 122 | 123 | # Add any paths that contain custom static files (such as style sheets) here, 124 | # relative to this directory. They are copied after the builtin static files, 125 | # so a file named "default.css" will overwrite the builtin "default.css". 126 | html_static_path = ['_static'] 127 | 128 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 129 | # using the given strftime format. 130 | #html_last_updated_fmt = '%b %d, %Y' 131 | 132 | # If true, SmartyPants will be used to convert quotes and dashes to 133 | # typographically correct entities. 134 | #html_use_smartypants = True 135 | 136 | # Custom sidebar templates, maps document names to template names. 137 | #html_sidebars = {} 138 | 139 | # Additional templates that should be rendered to pages, maps page names to 140 | # template names. 141 | #html_additional_pages = {} 142 | 143 | # If false, no module index is generated. 144 | #html_domain_indices = True 145 | 146 | # If false, no index is generated. 147 | #html_use_index = True 148 | 149 | # If true, the index is split into individual pages for each letter. 150 | #html_split_index = False 151 | 152 | # If true, links to the reST sources are added to the pages. 153 | #html_show_sourcelink = True 154 | 155 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 156 | #html_show_sphinx = True 157 | 158 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 159 | #html_show_copyright = True 160 | 161 | # If true, an OpenSearch description file will be output, and all pages will 162 | # contain a tag referring to it. The value of this option must be the 163 | # base URL from which the finished HTML is served. 164 | #html_use_opensearch = '' 165 | 166 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 167 | #html_file_suffix = None 168 | 169 | # Output file base name for HTML help builder. 170 | htmlhelp_basename = 'lazyarraydoc' 171 | 172 | 173 | # -- Options for LaTeX output -------------------------------------------------- 174 | 175 | # The paper size ('letter' or 'a4'). 176 | latex_paper_size = 'a4' 177 | 178 | # The font size ('10pt', '11pt' or '12pt'). 179 | #latex_font_size = '10pt' 180 | 181 | # Grouping the document tree into LaTeX files. List of tuples 182 | # (source start file, target name, title, author, documentclass [howto/manual]). 183 | latex_documents = [ 184 | ('index', 'lazyarray.tex', u'Lazyarray Documentation', 185 | contributors, 'manual'), 186 | ] 187 | 188 | # The name of an image file (relative to this directory) to place at the top of 189 | # the title page. 190 | #latex_logo = None 191 | 192 | # For "manual" documents, if this is true, then toplevel headings are parts, 193 | # not chapters. 194 | #latex_use_parts = False 195 | 196 | # If true, show page references after internal links. 197 | #latex_show_pagerefs = False 198 | 199 | # If true, show URL addresses after external links. 200 | #latex_show_urls = False 201 | 202 | # Additional stuff for the LaTeX preamble. 203 | #latex_preamble = '' 204 | 205 | # Documents to append as an appendix to all manuals. 206 | #latex_appendices = [] 207 | 208 | # If false, no module index is generated. 209 | #latex_domain_indices = True 210 | 211 | 212 | # -- Options for manual page output -------------------------------------------- 213 | 214 | # One entry per manual page. List of tuples 215 | # (source start file, name, description, authors, manual section). 216 | man_pages = [ 217 | ('index', 'lazyarray', u'lazyarray documentation', 218 | [u'Andrew P. Davison'], 1) 219 | ] 220 | 221 | 222 | # -- Options for Epub output --------------------------------------------------- 223 | 224 | # Bibliographic Dublin Core info. 225 | epub_title = u'lazyarray' 226 | epub_author = contributors 227 | epub_publisher = u'Andrew P. Davison' 228 | epub_copyright = copyright 229 | 230 | # The language of the text. It defaults to the language option 231 | # or en if the language is not set. 232 | #epub_language = '' 233 | 234 | # The scheme of the identifier. Typical schemes are ISBN or URL. 235 | #epub_scheme = '' 236 | 237 | # The unique identifier of the text. This can be a ISBN number 238 | # or the project homepage. 239 | #epub_identifier = '' 240 | 241 | # A unique identification for the text. 242 | #epub_uid = '' 243 | 244 | # HTML files that should be inserted before the pages created by sphinx. 245 | # The format is a list of tuples containing the path and title. 246 | #epub_pre_files = [] 247 | 248 | # HTML files shat should be inserted after the pages created by sphinx. 249 | # The format is a list of tuples containing the path and title. 250 | #epub_post_files = [] 251 | 252 | # A list of files that should not be packed into the epub file. 253 | #epub_exclude_files = [] 254 | 255 | # The depth of the table of contents in toc.ncx. 256 | #epub_tocdepth = 3 257 | 258 | # Allow duplicate toc entries. 259 | #epub_tocdup = True 260 | 261 | # -- Options for doctests ---- 262 | 263 | doctest_global_setup = """ 264 | from lazyarray import larray 265 | import numpy as np 266 | """ 267 | -------------------------------------------------------------------------------- /doc/developers.txt: -------------------------------------------------------------------------------- 1 | ================= 2 | Developers' guide 3 | ================= 4 | 5 | TO BE COMPLETED 6 | 7 | Testing 8 | ======= 9 | 10 | In the `test` sub-directory, run:: 11 | 12 | $ pytest 13 | 14 | To see how well the tests cover the code base, run:: 15 | 16 | $ pytest --cov=lazyarray --cov-report html --cov-report term-missing 17 | 18 | 19 | Making a release 20 | ================ 21 | 22 | * Update the version numbers in pyproject.toml, lazyarray.py, doc/conf.py and doc/installation.txt 23 | * Update changelog.txt 24 | * Run all the tests with Python 3 25 | * python -m build 26 | * twine upload dist/lazyarray-* 27 | * Commit the changes, tag with release number, push to Github 28 | * Rebuild the documentation at http://lazyarray.readthedocs.org/ 29 | -------------------------------------------------------------------------------- /doc/index.txt: -------------------------------------------------------------------------------- 1 | ========= 2 | lazyarray 3 | ========= 4 | 5 | lazyarray is a Python package that provides a lazily-evaluated numerical array 6 | class, :class:`larray`, based on and compatible with NumPy arrays. 7 | 8 | Lazy evaluation means that any operations on the array (potentially including 9 | array construction) are not performed immediately, but are delayed until 10 | evaluation is specifically requested. Evaluation of only parts of the array is 11 | also possible. 12 | 13 | Use of an :class:`larray` can potentially save considerable computation time 14 | and memory in cases where: 15 | 16 | * arrays are used conditionally (i.e. there are cases in which the array is 17 | never used); 18 | * only parts of an array are used (for example in distributed computation, 19 | in which each MPI node operates on a subset of the elements of the array). 20 | 21 | It appears that much of this functionality may appear in a future version of 22 | NumPy (see `this discussion`_ on the numpy-discussion mailing list), but at the 23 | time of writing I couldn't find anything equivalent out there. DistNumPy_ might 24 | also be an alternative for some of the use cases of lazyarray. 25 | 26 | Contents 27 | ======== 28 | 29 | .. toctree:: 30 | :maxdepth: 2 31 | 32 | installation 33 | tutorial 34 | performance 35 | reference 36 | developers 37 | 38 | Licence 39 | ======= 40 | 41 | The code is released under the Modified BSD licence. 42 | 43 | 44 | .. _`this discussion`: http://www.mail-archive.com/numpy-discussion@scipy.org/msg29732.html 45 | .. _DistNumPy: http://sites.google.com/site/distnumpy/ -------------------------------------------------------------------------------- /doc/installation.txt: -------------------------------------------------------------------------------- 1 | ============ 2 | Installation 3 | ============ 4 | 5 | Dependencies 6 | ============ 7 | 8 | * Python >= 3.8 9 | * numpy_ >= 1.20 10 | * (optional) scipy_ >= 1.7 11 | 12 | Installing from the Python Package Index 13 | ======================================== 14 | 15 | $ pip install lazyarray 16 | 17 | This will automatically download and install the latest release (you may need 18 | to have administrator privileges on the machine you are installing on). 19 | 20 | Installing from source 21 | ====================== 22 | 23 | To install the latest version of lazyarray from the Git repository:: 24 | 25 | $ git clone https://github.com/NeuralEnsemble/lazyarray 26 | $ cd lazyarray 27 | $ pip install . 28 | 29 | 30 | .. _`numpy`: http://numpy.scipy.org/ 31 | .. _`scipy`: http://scipy.org/ 32 | -------------------------------------------------------------------------------- /doc/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=_build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . 10 | if NOT "%PAPER%" == "" ( 11 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 12 | ) 13 | 14 | if "%1" == "" goto help 15 | 16 | if "%1" == "help" ( 17 | :help 18 | echo.Please use `make ^` where ^ is one of 19 | echo. html to make standalone HTML files 20 | echo. dirhtml to make HTML files named index.html in directories 21 | echo. singlehtml to make a single large HTML file 22 | echo. pickle to make pickle files 23 | echo. json to make JSON files 24 | echo. htmlhelp to make HTML files and a HTML help project 25 | echo. qthelp to make HTML files and a qthelp project 26 | echo. devhelp to make HTML files and a Devhelp project 27 | echo. epub to make an epub 28 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 29 | echo. text to make text files 30 | echo. man to make manual pages 31 | echo. changes to make an overview over all changed/added/deprecated items 32 | echo. linkcheck to check all external links for integrity 33 | echo. doctest to run all doctests embedded in the documentation if enabled 34 | goto end 35 | ) 36 | 37 | if "%1" == "clean" ( 38 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 39 | del /q /s %BUILDDIR%\* 40 | goto end 41 | ) 42 | 43 | if "%1" == "html" ( 44 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 45 | if errorlevel 1 exit /b 1 46 | echo. 47 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 48 | goto end 49 | ) 50 | 51 | if "%1" == "dirhtml" ( 52 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 53 | if errorlevel 1 exit /b 1 54 | echo. 55 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 56 | goto end 57 | ) 58 | 59 | if "%1" == "singlehtml" ( 60 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 61 | if errorlevel 1 exit /b 1 62 | echo. 63 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 64 | goto end 65 | ) 66 | 67 | if "%1" == "pickle" ( 68 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 69 | if errorlevel 1 exit /b 1 70 | echo. 71 | echo.Build finished; now you can process the pickle files. 72 | goto end 73 | ) 74 | 75 | if "%1" == "json" ( 76 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 77 | if errorlevel 1 exit /b 1 78 | echo. 79 | echo.Build finished; now you can process the JSON files. 80 | goto end 81 | ) 82 | 83 | if "%1" == "htmlhelp" ( 84 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 85 | if errorlevel 1 exit /b 1 86 | echo. 87 | echo.Build finished; now you can run HTML Help Workshop with the ^ 88 | .hhp project file in %BUILDDIR%/htmlhelp. 89 | goto end 90 | ) 91 | 92 | if "%1" == "qthelp" ( 93 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 94 | if errorlevel 1 exit /b 1 95 | echo. 96 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 97 | .qhcp project file in %BUILDDIR%/qthelp, like this: 98 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\PyNN.qhcp 99 | echo.To view the help file: 100 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\PyNN.ghc 101 | goto end 102 | ) 103 | 104 | if "%1" == "devhelp" ( 105 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 106 | if errorlevel 1 exit /b 1 107 | echo. 108 | echo.Build finished. 109 | goto end 110 | ) 111 | 112 | if "%1" == "epub" ( 113 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 114 | if errorlevel 1 exit /b 1 115 | echo. 116 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 117 | goto end 118 | ) 119 | 120 | if "%1" == "latex" ( 121 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 122 | if errorlevel 1 exit /b 1 123 | echo. 124 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 125 | goto end 126 | ) 127 | 128 | if "%1" == "text" ( 129 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 130 | if errorlevel 1 exit /b 1 131 | echo. 132 | echo.Build finished. The text files are in %BUILDDIR%/text. 133 | goto end 134 | ) 135 | 136 | if "%1" == "man" ( 137 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 138 | if errorlevel 1 exit /b 1 139 | echo. 140 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 141 | goto end 142 | ) 143 | 144 | if "%1" == "changes" ( 145 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 146 | if errorlevel 1 exit /b 1 147 | echo. 148 | echo.The overview file is in %BUILDDIR%/changes. 149 | goto end 150 | ) 151 | 152 | if "%1" == "linkcheck" ( 153 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 154 | if errorlevel 1 exit /b 1 155 | echo. 156 | echo.Link check complete; look for any errors in the above output ^ 157 | or in %BUILDDIR%/linkcheck/output.txt. 158 | goto end 159 | ) 160 | 161 | if "%1" == "doctest" ( 162 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 163 | if errorlevel 1 exit /b 1 164 | echo. 165 | echo.Testing of doctests in the sources finished, look at the ^ 166 | results in %BUILDDIR%/doctest/output.txt. 167 | goto end 168 | ) 169 | 170 | :end 171 | -------------------------------------------------------------------------------- /doc/performance.txt: -------------------------------------------------------------------------------- 1 | =========== 2 | Performance 3 | =========== 4 | 5 | The main aim of lazyarray is to improve performance (increased speed and 6 | reduced memory use) in two scenarios: 7 | 8 | * arrays are used conditionally (i.e. there are cases in which the array is 9 | never used); 10 | * only parts of an array are used (for example in distributed computation, 11 | in which each MPI node operates on a subset of the elements of the array). 12 | 13 | However, at the same time use of :class:`larray` objects should not be too much 14 | slower than plain NumPy arrays in normal use. 15 | 16 | Here we see that using a lazyarray adds minimal overhead compared to using a 17 | plain array: 18 | 19 | :: 20 | 21 | >>> from timeit import repeat 22 | >>> repeat('np.fromfunction(lambda i,j: i*i + 2*i*j + 3, (5000, 5000))', 23 | ... setup='import numpy as np', number=1, repeat=5) 24 | [1.9397640228271484, 1.92628812789917, 1.8796701431274414, 1.6766629219055176, 1.6844701766967773] 25 | >>> repeat('larray(lambda i,j: i*i + 2*i*j + 3, (5000, 5000)).evaluate()', 26 | ... setup='from lazyarray import larray', number=1, repeat=5) 27 | [1.686661958694458, 1.6836578845977783, 1.6853220462799072, 1.6538069248199463, 1.645576000213623] 28 | 29 | while if we only need to evaluate part of the array (perhaps because the other 30 | parts are being evaluated on other nodes), there is a major gain from using a 31 | lazy array. 32 | 33 | :: 34 | 35 | >>> repeat('np.fromfunction(lambda i,j: i*i + 2*i*j + 3, (5000, 5000))[:, 0:4999:10]', 36 | ... setup='import numpy as np', number=1, repeat=5) 37 | [1.691796064376831, 1.668884038925171, 1.647057056427002, 1.6792259216308594, 1.652547836303711] 38 | >>> repeat('larray(lambda i,j: i*i + 2*i*j + 3, (5000, 5000))[:, 0:4999:10]', 39 | ... setup='from lazyarray import larray', number=1, repeat=5) 40 | [0.23157119750976562, 0.16121792793273926, 0.1594078540802002, 0.16096210479736328, 0.16096997261047363] 41 | 42 | 43 | .. note:: These timings were done on a MacBook Pro 2.8 GHz Intel Core 2 Duo 44 | with 4 GB RAM, Python 2.7 and NumPy 1.6.1. -------------------------------------------------------------------------------- /doc/reference.txt: -------------------------------------------------------------------------------- 1 | ========= 2 | Reference 3 | ========= 4 | 5 | 6 | .. autoclass:: lazyarray.larray 7 | :members: apply, nrows, ncols, is_homogeneous, shape 8 | 9 | .. method:: evaluate(simplify=False) 10 | 11 | Return the lazy array as a real NumPy array. 12 | 13 | If the array is homogeneous and ``simplify`` is ``True``, return a single 14 | numerical value. 15 | -------------------------------------------------------------------------------- /doc/tutorial.txt: -------------------------------------------------------------------------------- 1 | ======== 2 | Tutorial 3 | ======== 4 | 5 | The :mod:`lazyarray` module contains a single class, :class:`larray`. 6 | 7 | .. doctest:: 8 | 9 | >>> from lazyarray import larray 10 | 11 | 12 | Creating a lazy array 13 | ===================== 14 | 15 | Lazy arrays may be created from single numbers, from sequences (lists, NumPy 16 | arrays), from iterators, from generators, or from a certain class of functions. 17 | Here are some examples: 18 | 19 | .. doctest:: 20 | 21 | >>> from_number = larray(20.0) 22 | >>> from_list = larray([0, 1, 1, 2, 3, 5, 8]) 23 | >>> import numpy as np 24 | >>> from_array = larray(np.arange(6).reshape((2, 3))) 25 | >>> from_iter = larray(iter(range(8))) 26 | >>> from_gen = larray((x**2 + 2*x + 3 for x in range(5))) 27 | 28 | To create a lazy array from a function or other callable, the function must 29 | accept one or more integers as arguments (depending on the dimensionality of 30 | the array) and return a single number. 31 | 32 | .. doctest:: 33 | 34 | >>> def f(i, j): 35 | ... return i*np.sin(np.pi*j/100) 36 | >>> from_func = larray(f) 37 | 38 | Specifying array shape 39 | ---------------------- 40 | 41 | Where the :class:`larray` is created from something that does not already have 42 | a known shape (i.e. from something that is not a list or array), it is possible 43 | to specify the shape of the array at the time of construction: 44 | 45 | .. doctest:: 46 | 47 | >>> from_func2 = larray(lambda i: 2*i, shape=(6,)) 48 | >>> print(from_func2.shape) 49 | (6,) 50 | 51 | For sequences, the shape is introspected: 52 | 53 | .. doctest:: 54 | 55 | >>> from_list.shape 56 | (7,) 57 | >>> from_array.shape 58 | (2, 3) 59 | 60 | Otherwise, the :attr:`shape` attribute is set to ``None``, and must be set later 61 | before the array can be evaluated. 62 | 63 | .. doctest:: 64 | 65 | >>> print(from_number.shape) 66 | None 67 | >>> print(from_iter.shape) 68 | None 69 | >>> print(from_gen.shape) 70 | None 71 | >>> print(from_func.shape) 72 | None 73 | 74 | 75 | Evaluating a lazy array 76 | ======================= 77 | 78 | The simplest way to evaluate a lazy array is with the :meth:`evaluate` method, 79 | which returns a NumPy array: 80 | 81 | .. doctest:: 82 | 83 | >>> from_list.evaluate() 84 | array([0, 1, 1, 2, 3, 5, 8]) 85 | >>> from_array.evaluate() 86 | array([[0, 1, 2], 87 | [3, 4, 5]]) 88 | >>> from_number.evaluate() 89 | Traceback (most recent call last): 90 | File "", line 1, in 91 | File "/Users/andrew/dev/lazyarray/lazyarray.py", line 35, in wrapped_meth 92 | raise ValueError("Shape of larray not specified") 93 | ValueError: Shape of larray not specified 94 | >>> from_number.shape = (2, 2) 95 | >>> from_number.evaluate() 96 | array([[ 20., 20.], 97 | [ 20., 20.]]) 98 | 99 | Note that an :class:`larray` can only be evaluated once its shape has been 100 | defined. Note also that a lazy array created from a single number evaluates to 101 | a homogeneous array containing that number. To obtain just the value, use the 102 | ``simplify`` argument: 103 | 104 | .. doctest:: 105 | 106 | >>> from_number.evaluate(simplify=True) 107 | 20.0 108 | 109 | Evaluating a lazy array created from an iterator or generator fills the array 110 | in row-first order. The number of values generated by the iterator must fit 111 | within the array shape: 112 | 113 | .. doctest:: 114 | 115 | >>> from_iter.shape = (2, 4) 116 | >>> from_iter.evaluate() 117 | array([[ 0., 1., 2., 3.], 118 | [ 4., 5., 6., 7.]]) 119 | >>> from_gen.shape = (5,) 120 | >>> from_gen.evaluate() 121 | array([ 3., 6., 11., 18., 27.]) 122 | 123 | If it doesn't, an Exception is raised: 124 | 125 | .. doctest:: 126 | 127 | >>> from_iter.shape = (7,) 128 | >>> from_iter.evaluate() 129 | Traceback (most recent call last): 130 | File "", line 1, in 131 | from_iter.evaluate() 132 | File "/Users/andrew/dev/lazyarray/lazyarray.py", line 36, in wrapped_meth 133 | return meth(self, *args, **kwargs) 134 | File "/Users/andrew/dev/lazyarray/lazyarray.py", line 235, in evaluate 135 | x = x.reshape(self.shape) 136 | ValueError: total size of new array must be unchanged 137 | 138 | When evaluating a lazy array created from a callable, the function is called 139 | with the indices of each element of the array: 140 | 141 | .. doctest:: 142 | 143 | >>> from_func.shape = (3, 4) 144 | >>> from_func.evaluate() 145 | array([[ 0. , 0. , 0. , 0. ], 146 | [ 0. , 0.03141076, 0.06279052, 0.09410831], 147 | [ 0. , 0.06282152, 0.12558104, 0.18821663]]) 148 | 149 | 150 | It is also possible to evaluate only parts of an array. This is explained below. 151 | 152 | 153 | Performing operations on a lazy array 154 | ===================================== 155 | 156 | Just as with a normal NumPy array, it is possible to perform elementwise 157 | arithmetic operations: 158 | 159 | .. doctest:: 160 | 161 | >>> a = from_list + 2 162 | >>> b = 2*a 163 | >>> print(type(b)) 164 | 165 | 166 | However, these operations are not carried out immediately, rather they are 167 | queued up to be carried out later, which can lead to large time and memory 168 | savings if the evaluation step turns out later not to be needed, or if only 169 | part of the array needs to be evaluated. 170 | 171 | .. doctest:: 172 | 173 | >>> b.evaluate() 174 | array([ 4, 6, 6, 8, 10, 14, 20]) 175 | 176 | Some more examples: 177 | 178 | .. doctest:: 179 | 180 | >>> a = 1.0/(from_list + 1) 181 | >>> a.evaluate() 182 | array([ 1. , 0.5 , 0.5 , 0.33333333, 0.25 , 183 | 0.16666667, 0.11111111]) 184 | >>> (from_list < 2).evaluate() 185 | array([ True, True, True, False, False, False, False], dtype=bool) 186 | >>> (from_list**2).evaluate() 187 | array([ 0, 1, 1, 4, 9, 25, 64]) 188 | >>> x = from_list 189 | >>> (x**2 - 2*x + 5).evaluate() 190 | array([ 5, 4, 4, 5, 8, 20, 53]) 191 | 192 | Numpy ufuncs cannot be used directly with lazy arrays, as NumPy does not know 193 | what to do with :class:`larray` objects. The lazyarray module therefore provides 194 | lazy array-compatible versions of a subset of the NumPy ufuncs, e.g.: 195 | 196 | .. doctest:: 197 | 198 | >>> from lazyarray import sqrt 199 | >>> sqrt(from_list).evaluate() 200 | array([ 0. , 1. , 1. , 1.41421356, 1.73205081, 201 | 2.23606798, 2.82842712]) 202 | 203 | For any other function that operates on a NumPy array, it can be applied to a 204 | lazy array using the :meth:`apply()` method: 205 | 206 | .. doctest:: 207 | 208 | >>> def g(x): 209 | ... return x**2 - 2*x + 5 210 | >>> from_list.apply(g) 211 | >>> from_list.evaluate() 212 | array([ 5, 4, 4, 5, 8, 20, 53]) 213 | 214 | 215 | Partial evaluation 216 | ================== 217 | 218 | When accessing a single element of an array, only that element is evaluated, 219 | where possible, not the whole array: 220 | 221 | .. doctest:: 222 | 223 | >>> x = larray(lambda i,j: 2*i + 3*j, shape=(4, 5)) 224 | >>> x[3, 2] 225 | 12 226 | >>> y = larray(lambda i: i*(2-i), shape=(6,)) 227 | >>> y[4] 228 | -8 229 | 230 | The same is true for accessing individual rows or columns: 231 | 232 | .. doctest:: 233 | 234 | >>> x[1] 235 | array([ 2, 5, 8, 11, 14]) 236 | >>> x[:, 4] 237 | array([12, 14, 16, 18]) 238 | >>> x[:, (0, 4)] 239 | array([[ 0, 12], 240 | [ 2, 14], 241 | [ 4, 16], 242 | [ 6, 18]]) 243 | 244 | 245 | Creating lazy arrays from SciPy sparse matrices 246 | =============================================== 247 | 248 | Lazy arrays may also be created from SciPy sparse matrices. There are 7 different sparse matrices. 249 | 250 | - csc_matrix(arg1[, shape, dtype, copy]) Compressed Sparse Column matrix 251 | - csr_matrix(arg1[, shape, dtype, copy]) Compressed Sparse Row matrix 252 | - bsr_matrix(arg1[, shape, dtype, copy, blocksize]) Block Sparse Row matrix 253 | - lil_matrix(arg1[, shape, dtype, copy]) Row-based linked list sparse matrix 254 | - dok_matrix(arg1[, shape, dtype, copy]) Dictionary Of Keys based sparse matrix. 255 | - coo_matrix(arg1[, shape, dtype, copy]) A sparse matrix in COOrdinate format. 256 | - dia_matrix(arg1[, shape, dtype, copy]) Sparse matrix with DIAgonal storage 257 | 258 | Here are some examples to use them. 259 | 260 | 261 | Creating sparse matrices 262 | ------------------------ 263 | 264 | Sparse matrices comes from SciPy package for numerical data. 265 | First to use them it is necessary to import libraries. 266 | 267 | .. doctest:: 268 | 269 | >>> import numpy as np 270 | >>> from lazyarray import larray 271 | >>> from scipy.sparse import bsr_matrix, coo_matrix, csc_matrix, csr_matrix, dia_matrix, dok_matrix, lil_matrix 272 | 273 | Creating a sparse matrix requires filling each row and column with data. For example : 274 | 275 | .. doctest:: 276 | 277 | >>> row = np.array([0, 2, 2, 0, 1, 2]) 278 | >>> col = np.array([0, 0, 1, 2, 2, 2]) 279 | >>> data = np.array([1, 2, 3, 4, 5, 6]) 280 | 281 | The 7 sparse matrices are not defined in the same way. 282 | 283 | The bsr_matrix, coo_matrix, csc_matrix and csr_matrix are defined as follows : 284 | 285 | .. doctest:: 286 | 287 | >>> sparr = bsr_matrix((data, (row, col)), shape=(3, 3)) 288 | >>> sparr = coo_matrix((data, (row, col)), shape=(3, 3)) 289 | >>> sparr = csc_matrix((data, (row, col)), shape=(3, 3)) 290 | >>> sparr = csr_matrix((data, (row, col)), shape=(3, 3)) 291 | 292 | In regards to the dia_matrix : 293 | 294 | .. doctest:: 295 | 296 | >>> data_dia = np.array([[1, 2, 3, 4]]).repeat(3, axis=0) 297 | >>> offsets = np.array([0, -1, 2]) 298 | >>> sparr = dia_matrix((data_dia, offsets), shape=(4, 4)) 299 | 300 | For the dok_matrix : 301 | 302 | .. doctest:: 303 | 304 | >>> sparr = dok_matrix(((row, col)), shape=(3, 3)) 305 | 306 | For the lil_matrix : 307 | 308 | .. doctest:: 309 | 310 | >>> sparr = lil_matrix(data, shape=(3, 3)) 311 | 312 | In the continuation of this tutorial, the sparse matrix used will be called sparr and refers to the csc_matrix. 313 | 314 | It is possible to convert the sparse matrix as a NumPy array, as follows: 315 | 316 | .. doctest:: 317 | 318 | >>> print(sparr.toarray()) 319 | array([[1, 0, 4], 320 | [0, 0, 5], 321 | [2, 3, 6]]) 322 | 323 | 324 | Specifying the shape and the type of a sparse matrix 325 | ---------------------------------------------------- 326 | 327 | To know the shape and the type of the sparse matrices, you can use : 328 | 329 | .. doctest:: 330 | 331 | >>> larr = larray(sparr) 332 | >>> print (larr.shape) 333 | (3, 3) 334 | >>> print (larr.dtype) 335 | dtype('int64') 336 | 337 | 338 | Evaluating a sparse matrix 339 | -------------------------- 340 | 341 | Evaluating a sparse matrix refers to the evaluate() method, which returns a NumPy array : 342 | 343 | .. doctest:: 344 | 345 | >>> print (larr.evaluate()) 346 | array([[1, 0, 4], 347 | [0, 0, 5], 348 | [2, 3, 6]]) 349 | 350 | When creating a sparse matrix, some values ​​may remain empty. 351 | In this case, the evaluate () method has the argument, called empty_val, referring to the special value nan, for Not a Number, defined in NumPy. 352 | This method fills these empty with this nan value. 353 | 354 | .. doctest:: 355 | 356 | >>> print (larr.evaluate(empty_val=np.nan)) 357 | array([[1, nan, 4], 358 | [nan, nan, 5], 359 | [2, 3, 6]]) 360 | 361 | 362 | Accessing individual rows or columns of a sparse matrix 363 | ------------------------------------------------------- 364 | 365 | To access specific elements of the matrix, like individual rows or columns : 366 | 367 | .. doctest:: 368 | 369 | >>> larr[2, :] 370 | 371 | In this case, the third line of the sparse matrix is obtained. 372 | However, this method is different depending on the sparse matrices used : 373 | 374 | For csc_matrix and csr_matrix : 375 | 376 | .. doctest:: 377 | 378 | >>> print (larr[2, :]) 379 | array([2, 3, 6]) 380 | 381 | During execution, the matrices bsr_matrix, coo_matrix and dia_matrix, do not support indexing. 382 | The solution is to convert them to another format. 383 | It is therefore necessary to go through csr_matrix in order to perform the calculation. 384 | 385 | .. doctest:: 386 | 387 | >>> print(sparr.tocsr()[2,:]) 388 | 389 | Depending on the definition given previously to the matrix, for the dok_matrix : 390 | 391 | .. doctest:: 392 | 393 | >>> print (larr[1, :]) 394 | 395 | And for lil_matrix : 396 | 397 | .. doctest:: 398 | 399 | >>> print (larr[0, :]) 400 | 401 | In case we want to access an element of a column, we must proceed in the same way as previously, by changing index. 402 | Here is an example of how to access an item in the third column of the sparse matrix. 403 | 404 | .. doctest:: 405 | 406 | >>> larr[:, 2] 407 | 408 | Finally, to have information on the sparse matrix : 409 | 410 | .. doctest:: 411 | 412 | >>>print (larr.base_value) 413 | <3x3 sparse matrix of type '' 414 | with 6 stored elements in Compressed Sparse Column format> 415 | 416 | -------------------------------------------------------------------------------- /lazyarray.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | """ 3 | lazyarray is a Python package that provides a lazily-evaluated numerical array 4 | class, ``larray``, based on and compatible with NumPy arrays. 5 | 6 | Copyright Andrew P. Davison, Joël Chavas, Elodie Legouée (CNRS) and Ankur Sinha, 2012-2024 7 | """ 8 | 9 | import numbers 10 | import operator 11 | from copy import deepcopy 12 | from functools import wraps, reduce 13 | import logging 14 | 15 | import numpy as np 16 | try: 17 | from scipy import sparse 18 | have_scipy = True 19 | except ImportError: 20 | have_scipy = False 21 | 22 | try: 23 | from collections.abc import Sized 24 | from collections.abc import Mapping 25 | from collections.abc import Iterator 26 | except ImportError: 27 | from collections import Sized 28 | from collections import Mapping 29 | from collections import Iterator 30 | 31 | 32 | __version__ = "0.6.0" 33 | 34 | logger = logging.getLogger("lazyarray") 35 | 36 | 37 | def check_shape(meth): 38 | """ 39 | Decorator for larray magic methods, to ensure that the operand has 40 | the same shape as the array. 41 | """ 42 | @wraps(meth) 43 | def wrapped_meth(self, val): 44 | if isinstance(val, (larray, np.ndarray)) and val.shape: 45 | if val.shape != self._shape: 46 | raise ValueError("shape mismatch: objects cannot be broadcast to a single shape") 47 | return meth(self, val) 48 | return wrapped_meth 49 | 50 | 51 | def requires_shape(meth): 52 | @wraps(meth) 53 | def wrapped_meth(self, *args, **kwargs): 54 | if self._shape is None: 55 | raise ValueError("Shape of larray not specified") 56 | return meth(self, *args, **kwargs) 57 | return wrapped_meth 58 | 59 | 60 | def full_address(addr, full_shape): 61 | if not (isinstance(addr, np.ndarray) and addr.dtype == bool and addr.ndim == len(full_shape)): 62 | if not isinstance(addr, tuple): 63 | addr = (addr,) 64 | if len(addr) < len(full_shape): 65 | full_addr = [slice(None)] * len(full_shape) 66 | for i, val in enumerate(addr): 67 | full_addr[i] = val 68 | addr = full_addr 69 | return addr 70 | 71 | 72 | def partial_shape(addr, full_shape): 73 | """ 74 | Calculate the size of the sub-array represented by `addr` 75 | """ 76 | def size(x, max): 77 | if isinstance(x, (int, np.integer)): 78 | return None 79 | elif isinstance(x, slice): 80 | y = min(max, x.stop or max) # slice limits can go past the bounds 81 | return 1 + (y - (x.start or 0) - 1) // (x.step or 1) 82 | elif isinstance(x, Sized): 83 | if hasattr(x, 'dtype') and x.dtype == bool: 84 | return x.sum() 85 | else: 86 | return len(x) 87 | else: 88 | raise TypeError("Unsupported index type %s" % type(x)) 89 | 90 | addr = full_address(addr, full_shape) 91 | if isinstance(addr, np.ndarray) and addr.dtype == bool: 92 | return (addr.sum(),) 93 | elif all(isinstance(x, Sized) for x in addr): 94 | return (len(addr[0]),) 95 | else: 96 | shape = [size(x, max) for (x, max) in zip(addr, full_shape)] 97 | return tuple([x for x in shape if x is not None]) # remove empty dimensions 98 | 99 | 100 | def reverse(func): 101 | """Given a function f(a, b), returns f(b, a)""" 102 | @wraps(func) 103 | def reversed_func(a, b): 104 | return func(b, a) 105 | reversed_func.__doc__ = "Reversed argument form of %s" % func.__doc__ 106 | reversed_func.__name__ = "reversed %s" % func.__name__ 107 | return reversed_func 108 | # "The hash of a function object is hash(func_code) ^ id(func_globals)" ? 109 | # see http://mail.python.org/pipermail/python-dev/2000-April/003397.html 110 | 111 | 112 | def lazy_operation(name, reversed=False): 113 | def op(self, val): 114 | new_map = deepcopy(self) 115 | f = getattr(operator, name) 116 | if reversed: 117 | f = reverse(f) 118 | new_map.operations.append((f, val)) 119 | return new_map 120 | return check_shape(op) 121 | 122 | 123 | def lazy_inplace_operation(name): 124 | def op(self, val): 125 | self.operations.append((getattr(operator, name), val)) 126 | return self 127 | return check_shape(op) 128 | 129 | 130 | def lazy_unary_operation(name): 131 | def op(self): 132 | new_map = deepcopy(self) 133 | new_map.operations.append((getattr(operator, name), None)) 134 | return new_map 135 | return op 136 | 137 | 138 | def is_array_like(value): 139 | # False for numbers, generators, functions, iterators 140 | if not isinstance(value, Sized): 141 | return False 142 | if have_scipy and sparse.issparse(value): 143 | return True 144 | if isinstance(value, Mapping): 145 | # because we may wish to have lazy arrays in which each 146 | # item is a dict, for example 147 | return False 148 | if getattr(value, "is_lazyarray_scalar", False): 149 | # for user-defined classes that are "Sized" but that should 150 | # be treated as individual elements in a lazy array 151 | # the attribute "is_lazyarray_scalar" can be defined with value 152 | # True. 153 | return False 154 | return True 155 | 156 | 157 | class larray(object): 158 | """ 159 | Optimises storage of and operations on arrays in various ways: 160 | - stores only a single value if all the values in the array are the same; 161 | - if the array is created from a function `f(i)` or `f(i,j)`, then 162 | elements are only evaluated when they are accessed. Any operations 163 | performed on the array are also queued up to be executed on access. 164 | 165 | Two use cases for the latter are: 166 | - to save memory for very large arrays by accessing them one row or 167 | column at a time: the entire array need never be in memory. 168 | - in parallelized code, different rows or columns may be evaluated 169 | on different nodes or in different threads. 170 | """ 171 | 172 | def __init__(self, value, shape=None, dtype=None): 173 | """ 174 | Create a new lazy array. 175 | 176 | `value` : may be an int, float, bool, NumPy array, iterator, 177 | generator or a function, `f(i)` or `f(i,j)`, depending on the 178 | dimensions of the array. 179 | 180 | `f(i,j)` should return a single number when `i` and `j` are integers, 181 | and a 1D array when either `i` or `j` or both is a NumPy array (in the 182 | latter case the two arrays must have equal lengths). 183 | """ 184 | 185 | self.dtype = dtype 186 | self.operations = [] 187 | if isinstance(value, str): 188 | raise TypeError("An larray cannot be created from a string") 189 | elif isinstance(value, larray): 190 | if shape is not None and value.shape is not None: 191 | assert shape == value.shape 192 | self._shape = shape or value.shape 193 | self.base_value = value.base_value 194 | self.dtype = dtype or value.dtype 195 | self.operations = value.operations # should deepcopy? 196 | 197 | elif is_array_like(value): # False for numbers, generators, functions, iterators 198 | if have_scipy and sparse.issparse(value): # For sparse matrices 199 | self.dtype = dtype or value.dtype 200 | elif not isinstance(value, np.ndarray): 201 | value = np.array(value, dtype=dtype) 202 | elif dtype is not None: 203 | assert np.can_cast(value.dtype, dtype, casting='safe') # or could convert value to the provided dtype 204 | if shape and value.shape and value.shape != shape: 205 | raise ValueError("Array has shape %s, value has shape %s" % (shape, value.shape)) 206 | if value.shape: 207 | self._shape = value.shape 208 | else: 209 | self._shape = shape 210 | self.base_value = value 211 | 212 | else: 213 | assert np.isreal(value) # also True for callables, generators, iterators 214 | self._shape = shape 215 | if dtype is None or isinstance(value, dtype): 216 | self.base_value = value 217 | else: 218 | try: 219 | self.base_value = dtype(value) 220 | except TypeError: 221 | self.base_value = value 222 | 223 | def __eq__(self, other): 224 | if isinstance(other, self.__class__): 225 | return self.base_value == other.base_value and self.operations == other.operations and self._shape == other.shape 226 | elif isinstance(other, numbers.Number): 227 | if len(self.operations) == 0: 228 | if isinstance(self.base_value, numbers.Number): 229 | return self.base_value == other 230 | elif isinstance(self.base_value, np.ndarray): 231 | return (self.base_value == other).all() 232 | # todo: we could perform the evaluation ourselves, but that could have a performance hit 233 | raise Exception("You will need to evaluate this lazyarray before checking for equality") 234 | else: 235 | # todo: add support for NumPy arrays 236 | raise TypeError("Cannot at present compare equality of lazyarray and {}".format(type(other))) 237 | 238 | def __deepcopy__(self, memo): 239 | obj = type(self).__new__(type(self)) 240 | if isinstance(self.base_value, VectorizedIterable): # special case, but perhaps need to rethink 241 | obj.base_value = self.base_value # whether deepcopy is appropriate everywhere 242 | else: 243 | try: 244 | obj.base_value = deepcopy(self.base_value) 245 | except TypeError: 246 | # base_value cannot be copied, e.g. is a generator (but see generator_tools from PyPI) 247 | # so here we create a reference rather than deepcopying - could cause problems 248 | obj.base_value = self.base_value 249 | obj._shape = self._shape 250 | obj.dtype = self.dtype 251 | obj.operations = [] 252 | for f, arg in self.operations: 253 | if isinstance(f, np.ufunc): 254 | obj.operations.append((f, deepcopy(arg))) 255 | else: 256 | obj.operations.append((deepcopy(f), deepcopy(arg))) 257 | return obj 258 | 259 | def __repr__(self): 260 | return "" % ( 261 | self.base_value, self.shape, self.dtype, self.operations) 262 | 263 | def _set_shape(self, value): 264 | if (hasattr(self.base_value, "shape") and 265 | self.base_value.shape and # values of type np.float have an empty shape 266 | self.base_value.shape != value): 267 | raise ValueError("Lazy array has fixed shape %s, cannot be changed to %s" % (self.base_value.shape, value)) 268 | self._shape = value 269 | for op in self.operations: 270 | if isinstance(op[1], larray): 271 | op[1].shape = value 272 | shape = property(fget=lambda self: self._shape, 273 | fset=_set_shape, doc="Shape of the array") 274 | 275 | @property 276 | @requires_shape 277 | def nrows(self): 278 | """Size of the first dimension of the array.""" 279 | return self._shape[0] 280 | 281 | @property 282 | @requires_shape 283 | def ncols(self): 284 | """Size of the second dimension (if it exists) of the array.""" 285 | if len(self.shape) > 1: 286 | return self._shape[1] 287 | else: 288 | return 1 289 | 290 | @property 291 | @requires_shape 292 | def size(self): 293 | """Total number of elements in the array.""" 294 | return reduce(operator.mul, self._shape) 295 | 296 | @property 297 | def is_homogeneous(self): 298 | """True if all the elements of the array are the same.""" 299 | hom_base = ( 300 | isinstance(self.base_value, (int, np.integer, float, bool)) 301 | or type(self.base_value) == self.dtype 302 | or (isinstance(self.dtype, type) and isinstance(self.base_value, self.dtype)) 303 | ) 304 | hom_ops = all(obj.is_homogeneous for f, obj in self.operations if isinstance(obj, larray)) 305 | return hom_base and hom_ops 306 | 307 | def _partial_shape(self, addr): 308 | """ 309 | Calculate the size of the sub-array represented by `addr` 310 | """ 311 | return partial_shape(addr, self._shape) 312 | 313 | def _homogeneous_array(self, addr): 314 | self.check_bounds(addr) 315 | shape = self._partial_shape(addr) 316 | return np.ones(shape, type(self.base_value)) 317 | 318 | def _full_address(self, addr): 319 | return full_address(addr, self._shape) 320 | 321 | def _array_indices(self, addr): 322 | self.check_bounds(addr) 323 | 324 | def axis_indices(x, max): 325 | if isinstance(x, (int, np.integer)): 326 | return x 327 | elif isinstance(x, slice): # need to handle negative values in slice 328 | return np.arange((x.start or 0), 329 | (x.stop or max), 330 | (x.step or 1), 331 | dtype=int) 332 | elif isinstance(x, Sized): 333 | if hasattr(x, 'dtype') and x.dtype == bool: 334 | return np.arange(max)[x] 335 | else: 336 | return np.array(x) 337 | else: 338 | raise TypeError("Unsupported index type %s" % type(x)) 339 | addr = self._full_address(addr) 340 | if isinstance(addr, np.ndarray) and addr.dtype == bool: 341 | if addr.ndim == 1: 342 | return (np.arange(self._shape[0])[addr],) 343 | else: 344 | raise NotImplementedError() 345 | elif all(isinstance(x, Sized) for x in addr): 346 | indices = [np.array(x) for x in addr] 347 | return indices 348 | else: 349 | indices = [axis_indices(x, max) for (x, max) in zip(addr, self._shape)] 350 | if len(indices) == 1: 351 | return indices 352 | elif len(indices) == 2: 353 | if isinstance(indices[0], Sized): 354 | if isinstance(indices[1], Sized): 355 | mesh_xy = np.meshgrid(*indices) 356 | return (mesh_xy[0].T, mesh_xy[1].T) # meshgrid works on (x,y), not (i,j) 357 | return indices 358 | else: 359 | raise NotImplementedError("Only 1D and 2D arrays supported") 360 | 361 | @requires_shape 362 | def __getitem__(self, addr): 363 | """ 364 | Return one or more items from the array, as for NumPy arrays. 365 | 366 | `addr` may be a single integer, a slice, a NumPy boolean array or a 367 | NumPy integer array. 368 | """ 369 | return self._partially_evaluate(addr, simplify=False) 370 | 371 | def _partially_evaluate(self, addr, simplify=False): 372 | """ 373 | Return part of the lazy array. 374 | """ 375 | if self.is_homogeneous: 376 | if simplify: 377 | base_val = self.base_value 378 | else: 379 | base_val = self._homogeneous_array(addr) * self.base_value 380 | elif isinstance(self.base_value, (int, np.integer, float, bool)): 381 | base_val = self._homogeneous_array(addr) * self.base_value 382 | elif isinstance(self.base_value, np.ndarray): 383 | base_val = self.base_value[addr] 384 | elif have_scipy and sparse.issparse(self.base_value): # For sparse matrices larr[2, :] 385 | base_val = self.base_value[addr] 386 | elif callable(self.base_value): 387 | indices = self._array_indices(addr) 388 | base_val = self.base_value(*indices) 389 | if isinstance(base_val, np.ndarray) and base_val.shape == (1,): 390 | base_val = base_val[0] 391 | elif hasattr(self.base_value, "lazily_evaluate"): 392 | base_val = self.base_value.lazily_evaluate(addr, shape=self._shape) 393 | elif isinstance(self.base_value, VectorizedIterable): 394 | partial_shape = self._partial_shape(addr) 395 | if partial_shape: 396 | n = reduce(operator.mul, partial_shape) 397 | else: 398 | n = 1 399 | base_val = self.base_value.next(n) # note that the array contents will depend on the order of access to elements 400 | if n == 1: 401 | base_val = base_val[0] 402 | elif partial_shape and base_val.shape != partial_shape: 403 | base_val = base_val.reshape(partial_shape) 404 | elif isinstance(self.base_value, Iterator): 405 | raise NotImplementedError("coming soon...") 406 | else: 407 | raise ValueError("invalid base value for array (%s)" % self.base_value) 408 | return self._apply_operations(base_val, addr, simplify=simplify) 409 | 410 | @requires_shape 411 | def check_bounds(self, addr): 412 | """ 413 | Check whether the given address is within the array bounds. 414 | """ 415 | def is_boolean_array(arr): 416 | return hasattr(arr, 'dtype') and arr.dtype == bool 417 | 418 | def check_axis(x, size): 419 | if isinstance(x, (int, np.integer)): 420 | lower = upper = x 421 | elif isinstance(x, slice): 422 | lower = x.start or 0 423 | upper = min(x.stop or size - 1, size - 1) # slices are allowed to go past the bounds 424 | elif isinstance(x, Sized): 425 | if is_boolean_array(x): 426 | lower = 0 427 | upper = x.size - 1 428 | else: 429 | if len(x) == 0: 430 | raise ValueError("Empty address component (address was %s)" % str(addr)) 431 | if hasattr(x, "min"): 432 | lower = x.min() 433 | else: 434 | lower = min(x) 435 | if hasattr(x, "max"): 436 | upper = x.max() 437 | else: 438 | upper = max(x) 439 | else: 440 | raise TypeError("Invalid array address: %s (element of type %s)" % (str(addr), type(x))) 441 | if (lower < -size) or (upper >= size): 442 | raise IndexError("Index out of bounds") 443 | full_addr = self._full_address(addr) 444 | if isinstance(addr, np.ndarray) and addr.dtype == bool: 445 | if len(addr.shape) > len(self._shape): 446 | raise IndexError("Too many indices for array") 447 | for xmax, size in zip(addr.shape, self._shape): 448 | upper = xmax - 1 449 | if upper >= size: 450 | raise IndexError("Index out of bounds") 451 | else: 452 | for i, size in zip(full_addr, self._shape): 453 | check_axis(i, size) 454 | 455 | def apply(self, f): 456 | """ 457 | Add the function `f(x)` to the list of the operations to be performed, 458 | where `x` will be a scalar or a numpy array. 459 | 460 | >>> m = larray(4, shape=(2,2)) 461 | >>> m.apply(np.sqrt) 462 | >>> m.evaluate() 463 | array([[ 2., 2.], 464 | [ 2., 2.]]) 465 | """ 466 | self.operations.append((f, None)) 467 | 468 | def _apply_operations(self, x, addr=None, simplify=False): 469 | for f, arg in self.operations: 470 | if arg is None: 471 | x = f(x) 472 | elif isinstance(arg, larray): 473 | if addr is None: 474 | x = f(x, arg.evaluate(simplify=simplify)) 475 | else: 476 | x = f(x, arg._partially_evaluate(addr, simplify=simplify)) 477 | 478 | else: 479 | x = f(x, arg) 480 | return x 481 | 482 | @requires_shape 483 | def evaluate(self, simplify=False, empty_val=0): 484 | """ 485 | Return the lazy array as a real NumPy array. 486 | 487 | If the array is homogeneous and ``simplify`` is ``True``, return a 488 | single numerical value. 489 | """ 490 | # need to catch the situation where a generator-based larray is evaluated a second time 491 | if self.is_homogeneous: 492 | if simplify: 493 | x = self.base_value 494 | else: 495 | x = self.base_value * np.ones(self._shape, dtype=self.dtype) 496 | elif isinstance(self.base_value, (int, np.integer, float, bool, np.bool_)): 497 | x = self.base_value * np.ones(self._shape, dtype=self.dtype) 498 | elif isinstance(self.base_value, np.ndarray): 499 | if self.base_value.shape == (1,): 500 | x = self.base_value[0] 501 | else: 502 | x = self.base_value 503 | elif callable(self.base_value): 504 | x = np.array(np.fromfunction(self.base_value, shape=self._shape, dtype=int), dtype=self.dtype) 505 | elif hasattr(self.base_value, "lazily_evaluate"): 506 | x = self.base_value.lazily_evaluate(shape=self._shape) 507 | elif isinstance(self.base_value, VectorizedIterable): 508 | x = self.base_value.next(self.size) 509 | if x.shape != self._shape: 510 | x = x.reshape(self._shape) 511 | elif have_scipy and sparse.issparse(self.base_value): # For sparse matrices 512 | if empty_val != 0: 513 | x = self.base_value.toarray((sparse.csc_matrix)) 514 | x = np.where(x, x, np.nan) 515 | else: 516 | x = self.base_value.toarray((sparse.csc_matrix)) 517 | elif isinstance(self.base_value, Iterator): 518 | x = np.fromiter(self.base_value, dtype=self.dtype or float, count=self.size) 519 | if x.shape != self._shape: 520 | x = x.reshape(self._shape) 521 | else: 522 | raise ValueError("invalid base value for array") 523 | return self._apply_operations(x, simplify=simplify) 524 | 525 | def __call__(self, arg): 526 | if callable(self.base_value): 527 | if isinstance(arg, larray): 528 | new_map = deepcopy(arg) 529 | elif callable(arg): 530 | new_map = larray(arg) 531 | else: 532 | raise Exception("Argument must be either callable or an larray.") 533 | new_map.operations.append((self.base_value, None)) 534 | new_map.operations.extend(self.operations) 535 | return new_map 536 | else: 537 | raise Exception("larray is not callable") 538 | 539 | __iadd__ = lazy_inplace_operation('add') 540 | __isub__ = lazy_inplace_operation('sub') 541 | __imul__ = lazy_inplace_operation('mul') 542 | __idiv__ = lazy_inplace_operation('div') 543 | __ipow__ = lazy_inplace_operation('pow') 544 | 545 | __add__ = lazy_operation('add') 546 | __radd__ = __add__ 547 | __sub__ = lazy_operation('sub') 548 | __rsub__ = lazy_operation('sub', reversed=True) 549 | __mul__ = lazy_operation('mul') 550 | __rmul__ = __mul__ 551 | __div__ = lazy_operation('div') 552 | __rdiv__ = lazy_operation('div', reversed=True) 553 | __truediv__ = lazy_operation('truediv') 554 | __rtruediv__ = lazy_operation('truediv', reversed=True) 555 | __pow__ = lazy_operation('pow') 556 | 557 | __lt__ = lazy_operation('lt') 558 | __gt__ = lazy_operation('gt') 559 | __le__ = lazy_operation('le') 560 | __ge__ = lazy_operation('ge') 561 | 562 | __neg__ = lazy_unary_operation('neg') 563 | __pos__ = lazy_unary_operation('pos') 564 | __abs__ = lazy_unary_operation('abs') 565 | 566 | 567 | class VectorizedIterable(object): 568 | """ 569 | Base class for any class which has a method `next(n)`, i.e., where you 570 | can choose how many values to return rather than just returning one at a 571 | time. 572 | """ 573 | pass 574 | 575 | 576 | def _build_ufunc(func): 577 | """Return a ufunc that works with lazy arrays""" 578 | def larray_compatible_ufunc(x): 579 | if isinstance(x, larray): 580 | y = deepcopy(x) 581 | y.apply(func) 582 | return y 583 | else: 584 | return func(x) 585 | return larray_compatible_ufunc 586 | 587 | 588 | def _build_ufunc_2nd_arg(func): 589 | """Return a ufunc taking a second, non-array argument, that works with lazy arrays""" 590 | def larray_compatible_ufunc2(x1, x2): 591 | if not isinstance(x2, numbers.Number): 592 | raise TypeError("lazyarry ufuncs do not accept an array as the second argument") 593 | if isinstance(x1, larray): 594 | def partial(x): 595 | return func(x, x2) 596 | y = deepcopy(x1) 597 | y.apply(partial) 598 | return y 599 | else: 600 | return func(x1, x2) 601 | return larray_compatible_ufunc2 602 | 603 | 604 | # build lazy-array compatible versions of NumPy ufuncs 605 | namespace = globals() 606 | for name in dir(np): 607 | obj = getattr(np, name) 608 | if isinstance(obj, np.ufunc) and name not in namespace: 609 | if name in ("power", "fmod", "arctan2, hypot, ldexp, maximum, minimum"): 610 | namespace[name] = _build_ufunc_2nd_arg(obj) 611 | else: 612 | namespace[name] = _build_ufunc(obj) 613 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "lazyarray" 3 | version = "0.6.0" 4 | description = "A Python package that provides a lazily-evaluated numerical array class, larray, based on and compatible with NumPy arrays." 5 | readme = "README.rst" 6 | requires-python = ">=3.8" 7 | license = {text = "Modified BSD"} 8 | authors = [ 9 | {name = "Andrew P. Davison", email = "andrew.davison@cnrs.fr"} 10 | ] 11 | maintainers = [ 12 | {name = "Andrew P. Davison", email = "andrew.davison@cnrs.fr"} 13 | ] 14 | keywords = ["lazy evaluation, array"] 15 | classifiers = [ 16 | "Development Status :: 4 - Beta", 17 | "Intended Audience :: Science/Research", 18 | "License :: OSI Approved :: BSD License", 19 | "Natural Language :: English", 20 | "Operating System :: OS Independent", 21 | "Programming Language :: Python :: 3", 22 | "Topic :: Scientific/Engineering" 23 | ] 24 | dependencies = [ 25 | "numpy>=1.20.3" 26 | ] 27 | 28 | [project.optional-dependencies] 29 | sparse = ["scipy>=1.7.3"] 30 | dev = ["sphinx", "pytest", "pytest-cov", "flake8", "wheel"] 31 | 32 | [project.urls] 33 | homepage = "https://github.com/NeuralEnsemble/lazyarray/" 34 | documentation = "https://lazyarray.readthedocs.io/" 35 | repository = "https://github.com/NeuralEnsemble/lazyarray/" 36 | changelog = "https://github.com/NeuralEnsemble/lazyarray/blob/master/changelog.txt" 37 | download = "http://pypi.python.org/pypi/lazyarray" 38 | 39 | [build-system] 40 | requires = ["setuptools"] 41 | build-backend = "setuptools.build_meta" 42 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | if __name__ == "__main__": 4 | setuptools.setup() 5 | -------------------------------------------------------------------------------- /test/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NeuralEnsemble/lazyarray/61d5f9853163a9ce2e5aeca82b37dd5794e22c7b/test/__init__.py -------------------------------------------------------------------------------- /test/performance.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | import numpy as np 4 | from numpy.testing import assert_array_equal 5 | from lazyarray import larray 6 | from timeit import repeat 7 | 8 | 9 | def test_function(i, j): 10 | return i * i + 2 * i * j + 3 11 | 12 | 13 | def array_from_function_full(f, shape): 14 | return np.fromfunction(f, shape) 15 | 16 | 17 | def larray_from_function_full(f, shape): 18 | return larray(f, shape).evaluate() 19 | 20 | 21 | def array_from_function_slice(f, shape): 22 | return np.fromfunction(f, shape)[:, 0:-1:10] 23 | 24 | 25 | def larray_from_function_slice(f, shape): 26 | return larray(f, shape)[:, 0:shape[1] - 1:10] 27 | 28 | 29 | if __name__ == "__main__": 30 | assert_array_equal(array_from_function_full(test_function, (5000, 5000)), 31 | larray_from_function_full(test_function, (5000, 5000))) 32 | 33 | print "Array from function: full array" 34 | print(repeat('array_from_function_full(test_function, (5000, 5000))', 35 | setup='from __main__ import array_from_function_full, test_function', 36 | number=1, repeat=5)) 37 | 38 | print "Lazy array from function: full array" 39 | print(repeat('larray_from_function_full(test_function, (5000, 5000))', 40 | setup='from __main__ import larray_from_function_full, test_function', 41 | number=1, repeat=5)) 42 | 43 | assert_array_equal(array_from_function_slice(test_function, (5000, 5000)), 44 | larray_from_function_slice(test_function, (5000, 5000))) 45 | print "Array from function: slice" 46 | print(repeat('array_from_function_slice(test_function, (5000, 5000))', 47 | setup='from __main__ import array_from_function_slice, test_function', 48 | number=1, repeat=5)) 49 | 50 | print "Lazy array from function: slice" 51 | print(repeat('larray_from_function_slice(test_function, (5000, 5000))', 52 | setup='from __main__ import larray_from_function_slice, test_function', 53 | number=1, repeat=5)) 54 | -------------------------------------------------------------------------------- /test/test_lazy_arrays_from_Sparse_Matrices.py: -------------------------------------------------------------------------------- 1 | # Support creating lazy arrays from SciPy sparse matrices 2 | # 3 | # 1 program for the 7 sparse matrices classes : 4 | # 5 | # csc_matrix(arg1[, shape, dtype, copy]) Compressed Sparse Column matrix 6 | # csr_matrix(arg1[, shape, dtype, copy]) Compressed Sparse Row matrix 7 | # bsr_matrix(arg1[, shape, dtype, copy, blocksize]) Block Sparse Row matrix 8 | # lil_matrix(arg1[, shape, dtype, copy]) Row-based linked list sparse matrix 9 | # dok_matrix(arg1[, shape, dtype, copy]) Dictionary Of Keys based sparse matrix. 10 | # coo_matrix(arg1[, shape, dtype, copy]) A sparse matrix in COOrdinate format. 11 | # dia_matrix(arg1[, shape, dtype, copy]) Sparse matrix with DIAgonal storage 12 | # 13 | 14 | 15 | import numpy as np 16 | from lazyarray import larray 17 | from scipy import sparse 18 | import random 19 | 20 | 21 | ################ 22 | # Random numbers 23 | ################ 24 | i = random.randint(-100, 100) 25 | j = random.randint(-100, 100) 26 | k = random.randint(-100, 100) 27 | l = random.randint(-100, 100) 28 | m = random.randint(-100, 100) 29 | n = random.randint(-100, 100) 30 | p = random.randint(-100, 100) 31 | q = random.randint(-100, 100) 32 | r = random.randint(-100, 100) 33 | 34 | ################ 35 | # An example 36 | ################ 37 | #i = 1 38 | #j = 2 39 | #k = 0 40 | #l = 0 41 | #m = 0 42 | #n = 3 43 | #p = 1 44 | #q = 0 45 | #r = 4 46 | 47 | #print "i =", i 48 | #print "j =", j 49 | #print "k =", k 50 | #print "l =", l 51 | #print "m =", m 52 | #print "n =", n 53 | #print "p =", p 54 | #print "q =", q 55 | #print "r =", r 56 | 57 | 58 | ############################################################## 59 | # Definition of an array 60 | ############################################################## 61 | 62 | def test_function_array_general(): 63 | A = np.array([[i, j, k], [l, m, n], [p, q, r]]) 64 | #print "A =" 65 | #print A 66 | return A 67 | 68 | 69 | ############################################################## 70 | # Definition of 7 sparse matrices 71 | ############################################################## 72 | 73 | def sparse_csc_matrices(): 74 | csc = sparse.csc_matrix([[i, j, k], [l, m, n], [p, q, r]]) 75 | #print "csc matrices =" 76 | #print csc 77 | return csc 78 | 79 | def sparse_csr_matrices(): 80 | csr = sparse.csr_matrix([[i, j, k], [l, m, n], [p, q, r]]) 81 | #print "csr matrices =" 82 | #print csr 83 | return csr 84 | 85 | def sparse_bsr_matrices(): 86 | bsr = sparse.bsr_matrix([[i, j, k], [l, m, n], [p, q, r]]) 87 | #print "bsr matrices =" 88 | #print bsr 89 | return bsr 90 | 91 | def sparse_lil_matrices(): 92 | lil = sparse.lil_matrix([[i, j, k], [l, m, n], [p, q, r]]) 93 | #print "lil matrices =" 94 | #print lil 95 | return lil 96 | 97 | def sparse_dok_matrices(): 98 | dok = sparse.dok_matrix([[i, j, k], [l, m, n], [p, q, r]]) 99 | #print "dok matrices =" 100 | #print dok 101 | return dok 102 | 103 | def sparse_coo_matrices(): 104 | coo = sparse.coo_matrix([[i, j, k], [l, m, n], [p, q, r]]) 105 | #print "coo matrices =" 106 | #print coo 107 | return coo 108 | 109 | def sparse_dia_matrices(): 110 | dia = sparse.dia_matrix([[i, j, k], [l, m, n], [p, q, r]]) 111 | #print "dia matrices =" 112 | #print dia 113 | return dia 114 | 115 | 116 | 117 | if __name__ == "__main__": 118 | 119 | 120 | ############################################################## 121 | # Call test_function_array_general 122 | # Create a sparse matrix from array 123 | # There are 7 sparse matrices 124 | ############################################################## 125 | 126 | #print "Array general =" 127 | test_function_array_general() 128 | #print "Array =" 129 | #print test_function_array_general() 130 | 131 | # print "----" 132 | 133 | # print "Sparse array csc general =" 134 | sA_csc_general = sparse.csc_matrix(test_function_array_general()) 135 | #print ("sparse csc matrices", sparse.csc_matrix(test_function_array_general())) 136 | #print "sparse csc matrices =" 137 | #print sA_csc_general 138 | # print "----" 139 | # print "Sparse array csr general =" 140 | sA_csr = sparse.csr_matrix(test_function_array_general()) 141 | #print ("sparse csr matrices", sparse.csr_matrix(test_function_array_general())) 142 | #print "sparse csr matrices =" 143 | #print sA_csr 144 | # print "----" 145 | # print "Sparse array bsr general =" 146 | sA_bsr = sparse.bsr_matrix(test_function_array_general()) 147 | # print ("sparse bsr matrices", sparse.bsr_matrix(test_function_array_general())) 148 | # print "sparse bsr matrices =" 149 | # print sA_bsr 150 | # print "----" 151 | # print "Sparse array lil general =" 152 | sA_lil = sparse.lil_matrix(test_function_array_general()) 153 | # print ("sparse lil matrices", sparse.lil_matrix(test_function_array_general())) 154 | # print "sparse lil matrices =" 155 | # print sA_lil 156 | # print "----" 157 | # print "Sparse array dok general =" 158 | sA_dok = sparse.dok_matrix(test_function_array_general()) 159 | # print ("sparse dok matrices", sparse.dok_matrix(test_function_array_general())) 160 | # print "sparse dok matrices =" 161 | # print sA_dok 162 | # print "----" 163 | # print "Sparse array coo general =" 164 | sA_coo = sparse.coo_matrix(test_function_array_general()) 165 | # print ("sparse coo matrices", sparse.coo_matrix(test_function_array_general())) 166 | # print "sparse coo matrices =" 167 | # print sA_coo 168 | # print "----" 169 | # print "Sparse array dia general =" 170 | sA_dia = sparse.dia_matrix(test_function_array_general()) 171 | # print ("sparse dia matrices", sparse.dia_matrix(test_function_array_general())) 172 | # print "sparse dia matrices =" 173 | # print sA_dia 174 | 175 | 176 | #print "----------------------------------------------------------------------" 177 | 178 | 179 | ############################################################## 180 | # Call the sparse matrices 181 | # Create a lazy array from sparse matrices 182 | ############################################################## 183 | 184 | 185 | Array_csc_matrices = sparse_csc_matrices().toarray() 186 | #print "Array csc matrices =" 187 | #print Array_csc_matrices 188 | 189 | Array_csr_matrices = sparse_csr_matrices().toarray() 190 | #print "Array csr matrices =" 191 | #print Array_csr_matrices 192 | 193 | Array_bsr_matrices = sparse_bsr_matrices().toarray() 194 | #print "Array bsr matrices =" 195 | #print Array_bsr_matrices 196 | 197 | Array_lil_matrices = sparse_lil_matrices().toarray() 198 | #print "Array lil matrices =" 199 | #print Array_lil_matrices 200 | 201 | Array_dok_matrices = sparse_dok_matrices().toarray() 202 | #print "Array dok matrices =" 203 | #print Array_dok_matrices 204 | 205 | Array_coo_matrices = sparse_coo_matrices().toarray() 206 | #print "Array coo matrices =" 207 | #print Array_coo_matrices 208 | 209 | Array_dia_matrices = sparse_dia_matrices().toarray() 210 | #print "Array dia matrices =" 211 | #print Array_dia_matrices -------------------------------------------------------------------------------- /test/test_lazyarray.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | """ 3 | Unit tests for ``larray`` class 4 | 5 | Copyright Andrew P. Davison, Joël Chavas, Elodie Legouée (CNRS) and Ankur Sinha (UCL), 2012-2024 6 | """ 7 | 8 | from lazyarray import larray, VectorizedIterable, sqrt, partial_shape 9 | import numpy as np 10 | from numpy.testing import assert_array_equal, assert_array_almost_equal 11 | import operator 12 | from copy import deepcopy 13 | import pytest 14 | from scipy.sparse import bsr_matrix, coo_matrix, csc_matrix, csr_matrix, dia_matrix, dok_matrix, lil_matrix 15 | 16 | 17 | 18 | class MockRNG(VectorizedIterable): 19 | 20 | def __init__(self, start, delta): 21 | self.start = start 22 | self.delta = delta 23 | 24 | def next(self, n): 25 | s = self.start 26 | self.start += n * self.delta 27 | return s + self.delta * np.arange(n) 28 | 29 | 30 | # test larray 31 | def test_create_with_int(): 32 | A = larray(3, shape=(5,)) 33 | assert A.shape == (5,) 34 | assert A.evaluate(simplify=True) == 3 35 | 36 | 37 | def test_create_with_int_and_dtype(): 38 | A = larray(3, shape=(5,), dtype=float) 39 | assert A.shape == (5,) 40 | assert A.evaluate(simplify=True) == 3 41 | 42 | 43 | def test_create_with_float(): 44 | A = larray(3.0, shape=(5,)) 45 | assert A.shape == (5,) 46 | assert A.evaluate(simplify=True) == 3.0 47 | 48 | 49 | def test_create_with_list(): 50 | A = larray([1, 2, 3], shape=(3,)) 51 | assert A.shape == (3,) 52 | assert_array_equal(A.evaluate(), np.array([1, 2, 3])) 53 | 54 | 55 | def test_create_with_array(): 56 | A = larray(np.array([1, 2, 3]), shape=(3,)) 57 | assert A.shape == (3,) 58 | assert_array_equal(A.evaluate(), np.array([1, 2, 3])) 59 | 60 | 61 | def test_create_with_array_and_dtype(): 62 | A = larray(np.array([1, 2, 3]), shape=(3,), dtype=int) 63 | assert A.shape == (3,) 64 | assert_array_equal(A.evaluate(), np.array([1, 2, 3])) 65 | 66 | 67 | def test_create_with_generator(): 68 | def plusone(): 69 | i = 0 70 | while True: 71 | yield i 72 | i += 1 73 | A = larray(plusone(), shape=(5, 11)) 74 | assert_array_equal(A.evaluate(), 75 | np.arange(55).reshape((5, 11))) 76 | 77 | 78 | def test_create_with_function1D(): 79 | A = larray(lambda i: 99 - i, shape=(3,)) 80 | assert_array_equal(A.evaluate(), 81 | np.array([99, 98, 97])) 82 | 83 | 84 | def test_create_with_function1D_and_dtype(): 85 | A = larray(lambda i: 99 - i, shape=(3,), dtype=float) 86 | assert_array_equal(A.evaluate(), 87 | np.array([99.0, 98.0, 97.0])) 88 | 89 | 90 | def test_create_with_function2D(): 91 | A = larray(lambda i, j: 3 * j - 2 * i, shape=(2, 3)) 92 | assert_array_equal(A.evaluate(), 93 | np.array([[0, 3, 6], 94 | [-2, 1, 4]])) 95 | 96 | 97 | def test_create_inconsistent(): 98 | pytest.raises(ValueError, larray, [1, 2, 3], shape=4) 99 | 100 | 101 | def test_create_with_string(): 102 | pytest.raises(TypeError, larray, "123", shape=3) 103 | 104 | 105 | def test_create_with_larray(): 106 | A = 3 + larray(lambda i: 99 - i, shape=(3,)) 107 | B = larray(A, shape=(3,), dtype=int) 108 | assert_array_equal(B.evaluate(), 109 | np.array([102, 101, 100])) 110 | 111 | 112 | ## For sparse matrices 113 | def test_create_with_sparse_array(): 114 | row = np.array([0, 2, 2, 0, 1, 2]) 115 | col = np.array([0, 0, 1, 2, 2, 2]) 116 | data = np.array([1, 2, 3, 4, 5, 6]) 117 | bsr = larray(bsr_matrix((data, (row, col)), shape=(3, 3))) # For bsr_matrix 118 | coo = larray(coo_matrix((data, (row, col)), shape=(3, 3))) # For coo_matrix 119 | csc = larray(csc_matrix((data, (row, col)), shape=(3, 3))) # For csc_matrix 120 | csr = larray(csr_matrix((data, (row, col)), shape=(3, 3))) # For csr_matrix 121 | data_dia = np.array([[1, 2, 3, 4]]).repeat(3, axis=0) # For dia_matrix 122 | offsets_dia = np.array([0, -1, 2]) # For dia_matrix 123 | dia = larray(dia_matrix((data_dia, offsets_dia), shape=(4, 4))) # For dia_matrix 124 | dok = larray(dok_matrix(((row, col)), shape=(3, 3))) # For dok_matrix 125 | lil = larray(lil_matrix(data, shape=(3, 3))) # For lil_matrix 126 | assert bsr.shape == (3, 3) 127 | assert coo.shape == (3, 3) 128 | assert csc.shape == (3, 3) 129 | assert csr.shape == (3, 3) 130 | assert dia.shape == (4, 4) 131 | assert dok.shape == (2, 6) 132 | assert lil.shape == (1, 6) 133 | 134 | def test_evaluate_with_sparse_array(): 135 | assert_array_equal(bsr.evaluate(), bsr_matrix((data, (row, col))).toarray()) # For bsr_matrix 136 | assert_array_equal(coo.evaluate(), coo_matrix((data, (row, col))).toarray()) # For coo_matrix 137 | assert_array_equal(csc.evaluate(), csc_matrix((data, (row, col))).toarray()) # For csc_matrix 138 | assert_array_equal(csr.evaluate(), csr_matrix((data, (row, col))).toarray()) # For csr_matrix 139 | assert_array_equal(dia.evaluate(), dia_matrix((data_dia, (row, col))).toarray()) # For dia_matrix 140 | assert_array_equal(dok.evaluate(), dok_matrix((data, (row, col))).toarray()) # For dok_matrix 141 | assert_array_equal(lil.evaluate(), lil_matrix((data, (row, col))).toarray()) # For lil_matrix 142 | 143 | def test_multiple_operations_with_sparse_array(): 144 | # For bsr_matrix 145 | bsr0 = bsr /100.0 146 | bsr1 = 0.2 + bsr0 147 | assert_array_almost_equal(bsr0.evaluate(), np.array([[0.01, 0., 0.04], [0., 0., 0.05], [0.02, 0.03, 0.06]])) 148 | assert_array_almost_equal(bsr0.evaluate(), np.array([[0.21, 0.2, 0.24], [0.2, 0.2, 0.25], [0.22, 0.23, 0.26]])) 149 | # For coo_matrix 150 | coo0 = coo /100.0 151 | coo1 = 0.2 + coo0 152 | assert_array_almost_equal(coo0.evaluate(), np.array([[0.01, 0., 0.04], [0., 0., 0.05], [0.02, 0.03, 0.06]])) 153 | assert_array_almost_equal(coo0.evaluate(), np.array([[0.21, 0.2, 0.24], [0.2, 0.2, 0.25], [0.22, 0.23, 0.26]])) 154 | # For csc_matrix 155 | csc0 = csc /100.0 156 | csc1 = 0.2 + csc0 157 | assert_array_almost_equal(csc0.evaluate(), np.array([[0.01, 0., 0.04], [0., 0., 0.05], [0.02, 0.03, 0.06]])) 158 | assert_array_almost_equal(csc0.evaluate(), np.array([[0.21, 0.2, 0.24], [0.2, 0.2, 0.25], [0.22, 0.23, 0.26]])) 159 | # For csr_matrix 160 | csr0 = csr /100.0 161 | csr1 = 0.2 + csr0 162 | assert_array_almost_equal(csc0.evaluate(), np.array([[0.01, 0., 0.04], [0., 0., 0.05], [0.02, 0.03, 0.06]])) 163 | assert_array_almost_equal(csc0.evaluate(), np.array([[0.21, 0.2, 0.24], [0.2, 0.2, 0.25], [0.22, 0.23, 0.26]])) 164 | # For dia_matrix 165 | dia0 = dia /100.0 166 | dia1 = 0.2 + dia0 167 | assert_array_almost_equal(dia0.evaluate(), np.array([[0.01, 0.02, 0.03, 0.04]])) 168 | assert_array_almost_equal(dia1.evaluate(), np.array([[0.21, 0.22, 0.23, 0.24]])) 169 | # For dok_matrix 170 | dok0 = dok /100.0 171 | dok1 = 0.2 + dok0 172 | assert_array_almost_equal(dok0.evaluate(), np.array([[0., 0.02, 0.02, 0., 0.01, 0.02], [0., 0., 0.01, 0.02, 0.02, 0.02]])) 173 | assert_array_almost_equal(dok1.evaluate(), np.array([[0.2, 0.22, 0.22, 0.2, 0.21, 0.22], [0.2, 0.2, 0.21, 0.22, 0.22, 0.22]])) 174 | # For lil_matrix 175 | lil0 = lil /100.0 176 | lil1 = 0.2 + lil0 177 | assert_array_almost_equal(lil0.evaluate(), np.array([[0.01, 0.02, 0.03, 0.04, 0.05, 0.06]])) 178 | assert_array_almost_equal(lil1.evaluate(), np.array([[0.21, 0.22, 0.23, 0.24, 0.25, 0.26]])) 179 | 180 | 181 | def test_getitem_from_2D_sparse_array(): 182 | pytest.raises(IndexError, bsr.__getitem__, (3, 0)) 183 | pytest.raises(IndexError, coo.__getitem__, (3, 0)) 184 | pytest.raises(IndexError, csc.__getitem__, (3, 0)) 185 | pytest.raises(IndexError, csr.__getitem__, (3, 0)) 186 | pytest.raises(IndexError, dia.__getitem__, (3, 0)) 187 | pytest.raises(IndexError, dok.__getitem__, (3, 0)) 188 | pytest.raises(IndexError, lil.__getitem__, (3, 0)) 189 | 190 | 191 | # def test_columnwise_iteration_with_flat_array(): 192 | # m = larray(5, shape=(4,3)) # 4 rows, 3 columns 193 | # cols = [col for col in m.by_column()] 194 | # assert cols == [5, 5, 5] 195 | # 196 | # def test_columnwise_iteration_with_structured_array(): 197 | # input = np.arange(12).reshape((4,3)) 198 | # m = larray(input, shape=(4,3)) # 4 rows, 3 columns 199 | # cols = [col for col in m.by_column()] 200 | # assert_array_equal(cols[0], input[:,0]) 201 | # assert_array_equal(cols[2], input[:,2]) 202 | # 203 | # def test_columnwise_iteration_with_function(): 204 | # input = lambda i,j: 2*i + j 205 | # m = larray(input, shape=(4,3)) 206 | # cols = [col for col in m.by_column()] 207 | # assert_array_equal(cols[0], np.array([0, 2, 4, 6])) 208 | # assert_array_equal(cols[1], np.array([1, 3, 5, 7])) 209 | # assert_array_equal(cols[2], np.array([2, 4, 6, 8])) 210 | # 211 | # def test_columnwise_iteration_with_flat_array_and_mask(): 212 | # m = larray(5, shape=(4,3)) # 4 rows, 3 columns 213 | # mask = np.array([True, False, True]) 214 | # cols = [col for col in m.by_column(mask=mask)] 215 | # assert cols == [5, 5] 216 | # 217 | # def test_columnwise_iteration_with_structured_array_and_mask(): 218 | # input = np.arange(12).reshape((4,3)) 219 | # m = larray(input, shape=(4,3)) # 4 rows, 3 columns 220 | # mask = np.array([False, True, True]) 221 | # cols = [col for col in m.by_column(mask=mask)] 222 | # assert_array_equal(cols[0], input[:,1]) 223 | # assert_array_equal(cols[1], input[:,2]) 224 | 225 | 226 | def test_size_related_properties(): 227 | m1 = larray(1, shape=(9, 7)) 228 | m2 = larray(1, shape=(13,)) 229 | m3 = larray(1) 230 | assert m1.nrows == 9 231 | assert m1.ncols == 7 232 | assert m1.size == 63 233 | assert m2.nrows == 13 234 | assert m2.ncols == 1 235 | assert m2.size == 13 236 | pytest.raises(ValueError, lambda: m3.nrows) 237 | pytest.raises(ValueError, lambda: m3.ncols) 238 | pytest.raises(ValueError, lambda: m3.size) 239 | 240 | 241 | def test_evaluate_with_flat_array(): 242 | m = larray(5, shape=(4, 3)) 243 | assert_array_equal(m.evaluate(), 5 * np.ones((4, 3))) 244 | 245 | 246 | def test_evaluate_with_structured_array(): 247 | input = np.arange(12).reshape((4, 3)) 248 | m = larray(input, shape=(4, 3)) 249 | assert_array_equal(m.evaluate(), input) 250 | 251 | 252 | def test_evaluate_with_functional_array(): 253 | input = lambda i, j: 2 * i + j 254 | m = larray(input, shape=(4, 3)) 255 | assert_array_equal(m.evaluate(), 256 | np.array([[0, 1, 2], 257 | [2, 3, 4], 258 | [4, 5, 6], 259 | [6, 7, 8]])) 260 | 261 | 262 | def test_evaluate_with_vectorized_iterable(): 263 | input = MockRNG(0, 1) 264 | m = larray(input, shape=(7, 3)) 265 | assert_array_equal(m.evaluate(), 266 | np.arange(21).reshape((7, 3))) 267 | 268 | 269 | def test_evaluate_twice_with_vectorized_iterable(): 270 | input = MockRNG(0, 1) 271 | m1 = larray(input, shape=(7, 3)) + 3 272 | m2 = larray(input, shape=(7, 3)) + 17 273 | assert_array_equal(m1.evaluate(), 274 | np.arange(3, 24).reshape((7, 3))) 275 | assert_array_equal(m2.evaluate(), 276 | np.arange(38, 59).reshape((7, 3))) 277 | 278 | 279 | def test_evaluate_structured_array_size_1_simplify(): 280 | m = larray([5.0], shape=(1,)) 281 | assert m.evaluate(simplify=True) == 5.0 282 | n = larray([2.0], shape=(1,)) 283 | assert (m/n).evaluate(simplify=True) == 2.5 284 | 285 | 286 | def test_iadd_with_flat_array(): 287 | m = larray(5, shape=(4, 3)) 288 | m += 2 289 | assert_array_equal(m.evaluate(), 7 * np.ones((4, 3))) 290 | assert m.base_value == 5 291 | assert m.evaluate(simplify=True) == 7 292 | 293 | 294 | def test_add_with_flat_array(): 295 | m0 = larray(5, shape=(4, 3)) 296 | m1 = m0 + 2 297 | assert m1.evaluate(simplify=True) == 7 298 | assert m0.evaluate(simplify=True) == 5 299 | 300 | 301 | def test_lt_with_flat_array(): 302 | m0 = larray(5, shape=(4, 3)) 303 | m1 = m0 < 10 304 | assert m1.evaluate(simplify=True) is True 305 | assert m0.evaluate(simplify=True) == 5 306 | 307 | 308 | def test_lt_with_structured_array(): 309 | input = np.arange(12).reshape((4, 3)) 310 | m0 = larray(input, shape=(4, 3)) 311 | m1 = m0 < 5 312 | assert_array_equal(m1.evaluate(simplify=True), input < 5) 313 | 314 | 315 | def test_structured_array_lt_array(): 316 | input = np.arange(12).reshape((4, 3)) 317 | m0 = larray(input, shape=(4, 3)) 318 | comparison = 5 * np.ones((4, 3)) 319 | m1 = m0 < comparison 320 | assert_array_equal(m1.evaluate(simplify=True), input < comparison) 321 | 322 | 323 | def test_rsub_with_structured_array(): 324 | m = larray(np.arange(12).reshape((4, 3))) 325 | assert_array_equal((11 - m).evaluate(), 326 | np.arange(11, -1, -1).reshape((4, 3))) 327 | 328 | 329 | def test_inplace_mul_with_structured_array(): 330 | m = larray((3 * x for x in range(4)), shape=(4,)) 331 | m *= 7 332 | assert_array_equal(m.evaluate(), 333 | np.arange(0, 84, 21)) 334 | 335 | 336 | def test_abs_with_structured_array(): 337 | m = larray(lambda i, j: i - j, shape=(3, 4)) 338 | assert_array_equal(abs(m).evaluate(), 339 | np.array([[0, 1, 2, 3], 340 | [1, 0, 1, 2], 341 | [2, 1, 0, 1]])) 342 | 343 | 344 | def test_multiple_operations_with_structured_array(): 345 | input = np.arange(12).reshape((4, 3)) 346 | m0 = larray(input, shape=(4, 3)) 347 | m1 = (m0 + 2) < 5 348 | m2 = (m0 < 5) + 2 349 | assert_array_equal(m1.evaluate(simplify=True), (input + 2) < 5) 350 | assert_array_equal(m2.evaluate(simplify=True), (input < 5) + 2) 351 | assert_array_equal(m0.evaluate(simplify=True), input) 352 | 353 | 354 | def test_multiple_operations_with_functional_array(): 355 | m = larray(lambda i: i, shape=(5,)) 356 | m0 = m / 100.0 357 | m1 = 0.2 + m0 358 | assert_array_almost_equal(m0.evaluate(), np.array([0.0, 0.01, 0.02, 0.03, 0.04]), decimal=12) 359 | assert_array_almost_equal(m1.evaluate(), np.array([0.20, 0.21, 0.22, 0.23, 0.24]), decimal=12) 360 | assert m1[0] == 0.2 361 | 362 | 363 | def test_operations_combining_constant_and_structured_arrays(): 364 | m0 = larray(10, shape=(5,)) 365 | m1 = larray(np.arange(5)) 366 | m2 = m0 + m1 367 | assert_array_almost_equal(m2.evaluate(), np.arange(10, 15)) 368 | 369 | 370 | def test_apply_function_to_constant_array(): 371 | f = lambda m: 2 * m + 3 372 | m0 = larray(5, shape=(4, 3)) 373 | m1 = f(m0) 374 | assert isinstance(m1, larray) 375 | assert m1.evaluate(simplify=True) == 13 376 | # the following tests the internals, not the behaviour 377 | # it is just to check I understand what's going on 378 | assert m1.operations == [(operator.mul, 2), (operator.add, 3)] 379 | 380 | 381 | def test_apply_function_to_structured_array(): 382 | f = lambda m: 2 * m + 3 383 | input = np.arange(12).reshape((4, 3)) 384 | m0 = larray(input, shape=(4, 3)) 385 | m1 = f(m0) 386 | assert isinstance(m1, larray) 387 | assert_array_equal(m1.evaluate(simplify=True), input * 2 + 3) 388 | 389 | 390 | def test_apply_function_to_functional_array(): 391 | input = lambda i, j: 2 * i + j 392 | m0 = larray(input, shape=(4, 3)) 393 | f = lambda m: 2 * m + 3 394 | m1 = f(m0) 395 | assert_array_equal(m1.evaluate(), 396 | np.array([[3, 5, 7], 397 | [7, 9, 11], 398 | [11, 13, 15], 399 | [15, 17, 19]])) 400 | 401 | 402 | def test_add_two_constant_arrays(): 403 | m0 = larray(5, shape=(4, 3)) 404 | m1 = larray(7, shape=(4, 3)) 405 | m2 = m0 + m1 406 | assert m2.evaluate(simplify=True) == 12 407 | # the following tests the internals, not the behaviour 408 | # it is just to check I understand what's going on 409 | assert m2.base_value == m0.base_value 410 | assert m2.operations == [(operator.add, m1)] 411 | 412 | 413 | def test_add_incommensurate_arrays(): 414 | m0 = larray(5, shape=(4, 3)) 415 | m1 = larray(7, shape=(5, 3)) 416 | pytest.raises(ValueError, m0.__add__, m1) 417 | 418 | 419 | def test_getitem_from_2D_constant_array(): 420 | m = larray(3, shape=(4, 3)) 421 | assert m[0, 0] == m[3, 2] == m[-1, 2] == m[-4, 2] == m[2, -3] == 3 422 | pytest.raises(IndexError, m.__getitem__, (4, 0)) 423 | pytest.raises(IndexError, m.__getitem__, (2, -4)) 424 | 425 | 426 | def test_getitem_from_1D_constant_array(): 427 | m = larray(3, shape=(43,)) 428 | assert m[0] == m[42] == 3 429 | 430 | 431 | def test_getitem__with_slice_from_constant_array(): 432 | m = larray(3, shape=(4, 3)) 433 | assert_array_equal(m[:3, 0], 434 | np.array([3, 3, 3])) 435 | 436 | 437 | def test_getitem__with_thinslice_from_constant_array(): 438 | m = larray(3, shape=(4, 3)) 439 | assert m[2:3, 0:1] == 3 440 | 441 | 442 | def test_getitem__with_mask_from_constant_array(): 443 | m = larray(3, shape=(4, 3)) 444 | assert_array_equal(m[1, (0, 2)], 445 | np.array([3, 3])) 446 | 447 | 448 | def test_getitem_with_numpy_integers_from_2D_constant_array(): 449 | if not hasattr(np, "int64"): 450 | pytest.skip("test requires a 64-bit system") 451 | m = larray(3, shape=(4, 3)) 452 | assert m[np.int64(0), np.int32(0)] == 3 453 | 454 | 455 | def test_getslice_from_constant_array(): 456 | m = larray(3, shape=(4, 3)) 457 | assert_array_equal(m[:2], 458 | np.array([[3, 3, 3], 459 | [3, 3, 3]])) 460 | 461 | 462 | def test_getslice_past_bounds_from_constant_array(): 463 | m = larray(3, shape=(5,)) 464 | assert_array_equal(m[2:10], 465 | np.array([3, 3, 3])) 466 | 467 | 468 | def test_getitem_from_structured_array(): 469 | m = larray(3 * np.ones((4, 3)), shape=(4, 3)) 470 | assert m[0, 0] == m[3, 2] == m[-1, 2] == m[-4, 2] == m[2, -3] == 3 471 | pytest.raises(IndexError, m.__getitem__, (4, 0)) 472 | pytest.raises(IndexError, m.__getitem__, (2, -4)) 473 | 474 | 475 | def test_getitem_from_2D_functional_array(): 476 | m = larray(lambda i, j: 2 * i + j, shape=(6, 5)) 477 | assert m[5, 4] == 14 478 | 479 | 480 | def test_getitem_from_1D_functional_array(): 481 | m = larray(lambda i: i ** 3, shape=(6,)) 482 | assert m[5] == 125 483 | 484 | 485 | def test_getitem_from_3D_functional_array(): 486 | m = larray(lambda i, j, k: i + j + k, shape=(2, 3, 4)) 487 | pytest.raises(NotImplementedError, m.__getitem__, (0, 1, 2)) 488 | 489 | 490 | def test_getitem_from_vectorized_iterable(): 491 | input = MockRNG(0, 1) 492 | m = larray(input, shape=(7,)) 493 | m3 = m[3] 494 | assert isinstance(m3, (int, np.integer)) 495 | assert m3 == 0 496 | assert m[0] == 1 497 | 498 | 499 | def test_getitem_with_slice_from_2D_functional_array(): 500 | m = larray(lambda i, j: 2 * i + j, shape=(6, 5)) 501 | assert_array_equal(m[2:5, 3:], 502 | np.array([[7, 8], 503 | [9, 10], 504 | [11, 12]])) 505 | 506 | 507 | def test_getitem_with_slice_from_2D_functional_array_2(): 508 | def test_function(i, j): 509 | return i * i + 2 * i * j + 3 510 | m = larray(test_function, shape=(3, 15)) 511 | assert_array_equal(m[:, 3:14:3], 512 | np.fromfunction(test_function, shape=(3, 15))[:, 3:14:3]) 513 | 514 | 515 | def test_getitem_with_mask_from_2D_functional_array(): 516 | a = np.arange(30).reshape((6, 5)) 517 | m = larray(lambda i, j: 5 * i + j, shape=(6, 5)) 518 | assert_array_equal(a[[2, 3], [3, 4]], 519 | np.array([13, 19])) 520 | assert_array_equal(m[[2, 3], [3, 4]], 521 | np.array([13, 19])) 522 | 523 | 524 | def test_getitem_with_mask_from_1D_functional_array(): 525 | m = larray(lambda i: np.sqrt(i), shape=(10,)) 526 | assert_array_equal(m[[0, 1, 4, 9]], 527 | np.array([0, 1, 2, 3])) 528 | 529 | 530 | def test_getitem_with_boolean_mask_from_1D_functional_array(): 531 | m = larray(lambda i: np.sqrt(i), shape=(10,)) 532 | assert_array_equal(m[np.array([1, 1, 0, 0, 1, 0, 0, 0, 0, 1], dtype=bool)], 533 | np.array([0, 1, 2, 3])) 534 | 535 | 536 | def test_getslice_from_2D_functional_array(): 537 | m = larray(lambda i, j: 2 * i + j, shape=(6, 5)) 538 | assert_array_equal(m[1:3], 539 | np.array([[2, 3, 4, 5, 6], 540 | [4, 5, 6, 7, 8]])) 541 | 542 | 543 | def test_getitem_from_iterator_array(): 544 | m = larray(iter([1, 2, 3]), shape=(3,)) 545 | pytest.raises(NotImplementedError, m.__getitem__, 2) 546 | 547 | 548 | def test_getitem_from_array_with_operations(): 549 | a1 = np.array([[1, 3, 5], [7, 9, 11]]) 550 | m1 = larray(a1) 551 | f = lambda i, j: np.sqrt(i * i + j * j) 552 | a2 = np.fromfunction(f, shape=(2, 3)) 553 | m2 = larray(f, shape=(2, 3)) 554 | a3 = 3 * a1 + a2 555 | m3 = 3 * m1 + m2 556 | assert_array_equal(a3[:, (0, 2)], 557 | m3[:, (0, 2)]) 558 | 559 | 560 | def test_evaluate_with_invalid_base_value(): 561 | m = larray(range(5)) 562 | m.base_value = "foo" 563 | pytest.raises(ValueError, m.evaluate) 564 | 565 | 566 | def test_partially_evaluate_with_invalid_base_value(): 567 | m = larray(range(5)) 568 | m.base_value = "foo" 569 | pytest.raises(ValueError, m._partially_evaluate, 3) 570 | 571 | 572 | def test_check_bounds_with_invalid_address(): 573 | m = larray([[1, 3, 5], [7, 9, 11]]) 574 | pytest.raises(TypeError, m.check_bounds, (object(), 1)) 575 | 576 | 577 | def test_check_bounds_with_invalid_address2(): 578 | m = larray([[1, 3, 5], [7, 9, 11]]) 579 | pytest.raises(ValueError, m.check_bounds, ([], 1)) 580 | 581 | 582 | def test_partially_evaluate_constant_array_with_one_element(): 583 | m = larray(3, shape=(1,)) 584 | a = 3 * np.ones((1,)) 585 | m1 = larray(3, shape=(1, 1)) 586 | a1 = 3 * np.ones((1, 1)) 587 | m2 = larray(3, shape=(1, 1, 1)) 588 | a2 = 3 * np.ones((1, 1, 1)) 589 | assert a[0] == m[0] 590 | assert a.shape == m.shape 591 | assert a[:].shape == m[:].shape 592 | assert a == m.evaluate() 593 | assert a1.shape == m1.shape 594 | assert a1[0, :].shape == m1[0, :].shape 595 | assert a1[:].shape == m1[:].shape 596 | assert a1 == m1.evaluate() 597 | assert a2.shape == m2.shape 598 | assert a2[:, 0, :].shape == m2[:, 0, :].shape 599 | assert a2[:].shape == m2[:].shape 600 | assert a2 == m2.evaluate() 601 | 602 | 603 | def test_partially_evaluate_constant_array_with_boolean_index(): 604 | m = larray(3, shape=(4, 5)) 605 | a = 3 * np.ones((4, 5)) 606 | addr_bool = np.array([True, True, False, False, True]) 607 | addr_int = np.array([0, 1, 4]) 608 | assert a[::2, addr_bool].shape == a[::2, addr_int].shape 609 | assert a[::2, addr_int].shape == m[::2, addr_int].shape 610 | assert a[::2, addr_bool].shape == m[::2, addr_bool].shape 611 | 612 | 613 | def test_partially_evaluate_constant_array_with_all_boolean_indices_false(): 614 | m = larray(3, shape=(3,)) 615 | a = 3 * np.ones((3,)) 616 | addr_bool = np.array([False, False, False]) 617 | assert a[addr_bool].shape == m[addr_bool].shape 618 | 619 | 620 | def test_partially_evaluate_constant_array_with_only_one_boolean_indice_true(): 621 | m = larray(3, shape=(3,)) 622 | a = 3 * np.ones((3,)) 623 | addr_bool = np.array([False, True, False]) 624 | assert a[addr_bool].shape == m[addr_bool].shape 625 | assert m[addr_bool][0] == a[0] 626 | 627 | 628 | def test_partially_evaluate_constant_array_with_boolean_indice_as_random_valid_ndarray(): 629 | m = larray(3, shape=(3,)) 630 | a = 3 * np.ones((3,)) 631 | addr_bool = np.random.rand(3) > 0.5 632 | while not addr_bool.any(): 633 | # random array, but not [False, False, False] 634 | addr_bool = np.random.rand(3) > 0.5 635 | assert a[addr_bool].shape == m[addr_bool].shape 636 | assert m[addr_bool][0] == a[addr_bool][0] 637 | 638 | 639 | def test_partially_evaluate_constant_array_size_one_with_boolean_index_true(): 640 | m = larray(3, shape=(1,)) 641 | a = np.array([3]) 642 | addr_bool = np.array([True]) 643 | m1 = larray(3, shape=(1, 1)) 644 | a1 = 3 * np.ones((1, 1)) 645 | addr_bool1 = np.array([[True]], ndmin=2) 646 | assert m[addr_bool][0] == a[0] 647 | assert m[addr_bool] == a[addr_bool] 648 | assert m[addr_bool].shape == a[addr_bool].shape 649 | assert m1[addr_bool1][0] == a1[addr_bool1][0] 650 | assert m1[addr_bool1].shape == a1[addr_bool1].shape 651 | 652 | 653 | def test_partially_evaluate_constant_array_size_two_with_boolean_index_true(): 654 | m2 = larray(3, shape=(1, 2)) 655 | a2 = 3 * np.ones((1, 2)) 656 | addr_bool2 = np.ones((1, 2), dtype=bool) 657 | assert m2[addr_bool2][0] == a2[addr_bool2][0] 658 | assert m2[addr_bool2].shape == a2[addr_bool2].shape 659 | 660 | 661 | def test_partially_evaluate_constant_array_size_one_with_boolean_index_false(): 662 | m = larray(3, shape=(1,)) 663 | m1 = larray(3, shape=(1, 1)) 664 | a = np.array([3]) 665 | a1 = np.array([[3]], ndmin=2) 666 | addr_bool = np.array([False]) 667 | addr_bool1 = np.array([[False]], ndmin=2) 668 | addr_bool2 = np.array([False]) 669 | assert m[addr_bool].shape == a[addr_bool].shape 670 | assert m1[addr_bool1].shape == a1[addr_bool1].shape 671 | 672 | 673 | def test_partially_evaluate_constant_array_size_with_empty_boolean_index(): 674 | m = larray(3, shape=(1,)) 675 | a = np.array([3]) 676 | addr_bool = np.array([], dtype='bool') 677 | assert m[addr_bool].shape == a[addr_bool].shape 678 | assert m[addr_bool].shape == (0,) 679 | 680 | 681 | def test_partially_evaluate_functional_array_with_boolean_index(): 682 | m = larray(lambda i, j: 5 * i + j, shape=(4, 5)) 683 | a = np.arange(20.0).reshape((4, 5)) 684 | addr_bool = np.array([True, True, False, False, True]) 685 | addr_int = np.array([0, 1, 4]) 686 | assert a[::2, addr_bool].shape == a[::2, addr_int].shape 687 | assert a[::2, addr_int].shape == m[::2, addr_int].shape 688 | assert a[::2, addr_bool].shape == m[::2, addr_bool].shape 689 | 690 | 691 | def test_getslice_with_vectorized_iterable(): 692 | input = MockRNG(0, 1) 693 | m = larray(input, shape=(7, 3)) 694 | assert_array_equal(m[::2, (0, 2)], 695 | np.arange(8).reshape((4, 2))) 696 | 697 | 698 | def test_equality_with_lazyarray(): 699 | m1 = larray(42.0, shape=(4, 5)) / 23.0 + 2.0 700 | m2 = larray(42.0, shape=(4, 5)) / 23.0 + 2.0 701 | assert m1 == m2 702 | 703 | 704 | def test_equality_with_number(): 705 | m1 = larray(42.0, shape=(4, 5)) 706 | m2 = larray([42, 42, 42]) 707 | m3 = larray([42, 42, 43]) 708 | m4 = larray(42.0, shape=(4, 5)) + 2 709 | assert m1 == 42.0 710 | assert m2 == 42 711 | assert m3 != 42 712 | pytest.raises(Exception, m4.__eq__, 44.0) 713 | 714 | 715 | def test_equality_with_array(): 716 | m1 = larray(42.0, shape=(4, 5)) 717 | target = 42.0 * np.ones((4, 5)) 718 | pytest.raises(TypeError, m1.__eq__, target) 719 | 720 | 721 | def test_deepcopy(): 722 | m1 = 3 * larray(lambda i, j: 5 * i + j, shape=(4, 5)) + 2 723 | m2 = deepcopy(m1) 724 | m1.shape = (3, 4) 725 | m3 = deepcopy(m1) 726 | assert m1.shape == m3.shape == (3, 4) 727 | assert m2.shape == (4, 5) 728 | assert_array_equal(m1.evaluate(), m3.evaluate()) 729 | 730 | 731 | def test_deepcopy_with_ufunc(): 732 | m1 = sqrt(larray([x ** 2 for x in range(5)])) 733 | m2 = deepcopy(m1) 734 | m1.base_value[0] = 49 735 | assert_array_equal(m1.evaluate(), np.array([7, 1, 2, 3, 4])) 736 | assert_array_equal(m2.evaluate(), np.array([0, 1, 2, 3, 4])) 737 | 738 | 739 | def test_set_shape(): 740 | m = larray(42) + larray(lambda i: 3 * i) 741 | assert m.shape is None 742 | m.shape = (5,) 743 | assert_array_equal(m.evaluate(), np.array([42, 45, 48, 51, 54])) 744 | 745 | 746 | def test_call(): 747 | A = larray(np.array([1, 2, 3]), shape=(3,)) - 1 748 | B = 0.5 * larray(lambda i: 2 * i, shape=(3,)) 749 | C = B(A) 750 | assert_array_equal(C.evaluate(), np.array([0, 1, 2])) 751 | assert_array_equal(A.evaluate(), np.array([0, 1, 2])) # A should be unchanged 752 | 753 | 754 | def test_call2(): 755 | positions = np.array( 756 | [[0., 2., 4., 6., 8.], 757 | [0., 0., 0., 0., 0.], 758 | [0., 0., 0., 0., 0.]]) 759 | 760 | def position_generator(i): 761 | return positions.T[i] 762 | 763 | def distances(A, B): 764 | d = A - B 765 | d **= 2 766 | d = np.sum(d, axis=-1) 767 | np.sqrt(d, d) 768 | return d 769 | 770 | def distance_generator(f, g): 771 | def distance_map(i, j): 772 | return distances(f(i), g(j)) 773 | return distance_map 774 | distance_map = larray(distance_generator(position_generator, position_generator), 775 | shape=(4, 5)) 776 | f_delay = 1000 * larray(lambda d: 0.1 * (1 + d), shape=(4, 5)) 777 | assert_array_almost_equal( 778 | f_delay(distance_map).evaluate(), 779 | np.array([[100, 300, 500, 700, 900], 780 | [300, 100, 300, 500, 700], 781 | [500, 300, 100, 300, 500], 782 | [700, 500, 300, 100, 300]], dtype=float), 783 | decimal=12) 784 | # repeat, should be idempotent 785 | assert_array_almost_equal( 786 | f_delay(distance_map).evaluate(), 787 | np.array([[100, 300, 500, 700, 900], 788 | [300, 100, 300, 500, 700], 789 | [500, 300, 100, 300, 500], 790 | [700, 500, 300, 100, 300]], dtype=float), 791 | decimal=12) 792 | 793 | 794 | def test__issue4(): 795 | # In order to avoid the errors associated with version changes of numpy, mask1 and mask2 no longer contain boolean values ​​but integer values 796 | a = np.arange(12).reshape((4, 3)) 797 | b = larray(np.arange(12).reshape((4, 3))) 798 | mask1 = (slice(None), int(True)) 799 | mask2 = (slice(None), np.array([int(True)])) 800 | assert b[mask1].shape == partial_shape(mask1, b.shape) == a[mask1].shape 801 | assert b[mask2].shape == partial_shape(mask2, b.shape) == a[mask2].shape 802 | 803 | 804 | def test__issue3(): 805 | a = np.arange(12).reshape((4, 3)) 806 | b = larray(a) 807 | c = larray(lambda i, j: 3*i + j, shape=(4, 3)) 808 | assert_array_equal(a[(1, 3), :][:, (0, 2)], b[(1, 3), :][:, (0, 2)]) 809 | assert_array_equal(b[(1, 3), :][:, (0, 2)], c[(1, 3), :][:, (0, 2)]) 810 | assert_array_equal(a[(1, 3), (0, 2)], b[(1, 3), (0, 2)]) 811 | assert_array_equal(b[(1, 3), (0, 2)], c[(1, 3), (0, 2)]) 812 | 813 | 814 | def test_partial_shape(): 815 | a = np.arange(12).reshape((4, 3)) 816 | test_cases = [ 817 | (slice(None), (4, 3)), 818 | ((slice(None), slice(None)), (4, 3)), 819 | (slice(1, None, 2), (2, 3)), 820 | (1, (3,)), 821 | ((1, slice(None)), (3,)), 822 | ([0, 2, 3], (3, 3)), 823 | (np.array([0, 2, 3]), (3, 3)), 824 | ((np.array([0, 2, 3]), slice(None)), (3, 3)), 825 | (np.array([True, False, True, True]), (3, 3)), 826 | #(np.array([True, False]), (1, 3)), # not valid with NumPy 1.13 827 | (np.array([[True, False, False], [False, False, False], [True, True, False], [False, True, False]]), (4,)), 828 | #(np.array([[True, False, False], [False, False, False], [True, True, False]]), (3,)), # not valid with NumPy 1.13 829 | ((3, 1), tuple()), 830 | ((slice(None), 1), (4,)), 831 | ((slice(None), slice(1, None, 3)), (4, 1)), 832 | ((np.array([0, 3]), 2), (2,)), 833 | ((np.array([0, 3]), np.array([1, 2])), (2,)), 834 | ((slice(None), np.array([2])), (4, 1)), 835 | (((1, 3), (0, 2)), (2,)), 836 | (np.array([], bool), (0, 3)), 837 | ] 838 | for mask, expected_shape in test_cases: 839 | assert partial_shape(mask, a.shape) == a[mask].shape 840 | assert partial_shape(mask, a.shape) == expected_shape 841 | b = np.arange(5) 842 | test_cases = [ 843 | (np.arange(5), (5,)) 844 | ] 845 | for mask, expected_shape in test_cases: 846 | assert partial_shape(mask, b.shape) == b[mask].shape 847 | assert partial_shape(mask, b.shape) == expected_shape 848 | 849 | def test_is_homogeneous(): 850 | m0 = larray(10, shape=(5,)) 851 | m1 = larray(np.arange(1, 6)) 852 | m2 = m0 + m1 853 | m3 = 9 + m0 / m1 854 | assert m0.is_homogeneous 855 | assert not m1.is_homogeneous 856 | assert not m2.is_homogeneous 857 | assert not m3.is_homogeneous 858 | -------------------------------------------------------------------------------- /test/test_ufunc.py: -------------------------------------------------------------------------------- 1 | """ 2 | Unit tests for ``larray``-compatible ufuncs 3 | 4 | Copyright Andrew P. Davison, 2012-2024 5 | """ 6 | 7 | from lazyarray import larray, sqrt, cos, power, fmod 8 | import numpy as np 9 | from numpy.testing import assert_array_equal, assert_array_almost_equal 10 | import pytest 11 | 12 | def test_sqrt_from_array(): 13 | A = larray(np.array([1, 4, 9, 16, 25])) 14 | assert_array_equal(sqrt(A).evaluate(), 15 | np.arange(1, 6)) 16 | 17 | 18 | def test_sqrt_from_iterator(): 19 | A = larray(iter([1, 4, 9, 16, 25]), shape=(5,)) 20 | assert_array_equal(sqrt(A).evaluate(), 21 | np.arange(1, 6)) 22 | 23 | 24 | def test_sqrt_from_func(): 25 | A = larray(lambda x: (x + 1) ** 2, shape=(5,)) 26 | assert_array_equal(sqrt(A).evaluate(), 27 | np.arange(1, 6)) 28 | 29 | 30 | def test_sqrt_normal_array(): 31 | A = np.array([1, 4, 9, 16, 25]) 32 | assert_array_equal(sqrt(A), 33 | np.arange(1, 6)) 34 | 35 | 36 | def test_cos_from_generator(): 37 | def clock(): 38 | for x in np.arange(0, 2 * np.pi, np.pi / 2): 39 | yield x 40 | A = larray(clock(), shape=(2, 2)) 41 | assert_array_almost_equal(cos(A).evaluate(), 42 | np.array([[1.0, 0.0], 43 | [-1.0, 0.0]]), 44 | decimal=15) 45 | 46 | 47 | def test_power_from_array(): 48 | A = larray(np.array([1, 4, 9, 16, 25])) 49 | assert_array_equal(power(A, 0.5).evaluate(), 50 | np.arange(1, 6)) 51 | 52 | 53 | def test_power_with_plain_array(): 54 | A = np.array([1, 4, 9, 16, 25]) 55 | assert_array_equal(power(A, 0.5), np.arange(1, 6)) 56 | 57 | 58 | def test_fmod_with_array_as_2nd_arg(): 59 | A = larray(np.array([1, 4, 9, 16, 25])) 60 | B = larray(np.array([1, 4, 9, 16, 25])) 61 | pytest.raises(TypeError, fmod, A, B) 62 | --------------------------------------------------------------------------------