├── .coveragerc ├── .github └── workflows │ └── test.yml ├── .gitignore ├── .travis.yml ├── AUTHORS.rst ├── CHANGES.rst ├── CONTRIBUTING.rst ├── LICENSE.rst ├── MANIFEST.in ├── Makefile ├── README.rst ├── TODO.rst ├── django_pandas ├── __init__.py ├── io.py ├── managers.py ├── models.py ├── tests │ ├── __init__.py │ ├── models.py │ ├── test_io.py │ ├── test_manager.py │ └── tests.py └── utils.py ├── docs ├── Makefile ├── conf.py ├── index.rst └── make.bat ├── pyproject.toml ├── runtests.py ├── setup.py └── tox.ini /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | source = django_pandas 3 | omit = django_pandas/tests/* 4 | branch = 1 5 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: [push, pull_request, workflow_dispatch] 4 | 5 | env: 6 | FORCE_COLOR: 1 7 | 8 | jobs: 9 | test: 10 | runs-on: ubuntu-latest 11 | strategy: 12 | fail-fast: false 13 | matrix: 14 | python-version: [ "3.7", "3.8", "3.9", "3.10"] 15 | django-version: [ "30", "31", "32","40", "42"] 16 | exclude: 17 | - {python-version: "3.7", django-version: "40"} 18 | - {python-version: "3.7", django-version: "42"} 19 | 20 | steps: 21 | - uses: actions/checkout@v2 22 | 23 | - name: Set up Python ${{ matrix.python-version }} 24 | uses: actions/setup-python@v2 25 | with: 26 | python-version: ${{ matrix.python-version }} 27 | 28 | - name: Install dependencies 29 | run: | 30 | python -m pip install -U tox 31 | 32 | - name: Test 33 | run: tox -e py${{ matrix.python-version }}-django${{ matrix.django-version }} 34 | 35 | all-successful: 36 | # https://github.community/t/is-it-possible-to-require-all-github-actions-tasks-to-pass-without-enumerating-them/117957/4?u=graingert 37 | runs-on: ubuntu-latest 38 | needs: [test] 39 | steps: 40 | - uses: actions/checkout@v2 41 | - name: Set up Python 42 | uses: actions/setup-python@v2 43 | with: 44 | python-version: "3.10" 45 | - name: build package 46 | run: | 47 | python -m pip install -U build 48 | python -m build 49 | - name: Publish package 50 | if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') 51 | uses: pypa/gh-action-pypi-publish@v1.8.14 52 | with: 53 | user: __token__ 54 | password: ${{ secrets.PYPI_API_TOKEN }} 55 | - name: note that all tests succeeded 56 | run: echo "🎉" 57 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.egg* 3 | .DS_Store* 4 | _build* 5 | build* 6 | dist* 7 | *.wpr 8 | *.wpu 9 | *.sw* 10 | .ropeproject* 11 | htmlcov* 12 | .tox* 13 | .coverage 14 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | 3 | python: 4 | - "2.7" 5 | - "3.5" 6 | - "3.6" 7 | - "3.7" 8 | - "3.8" 9 | - "3.9" 10 | 11 | env: 12 | matrix: 13 | - DJANGO=18 14 | - DJANGO=19 15 | - DJANGO=110 16 | - DJANGO=111 17 | - DJANGO=20 18 | - DJANGO=21 19 | - DJANGO=22 20 | - DJANGO=30 21 | - DJANGO=31 22 | - DJANGO=32 23 | 24 | 25 | install: 26 | - pip install --upgrade pip 27 | - pip install --upgrade setuptools tox virtualenv 28 | - pip install coverage coveralls 29 | - pip install numpy>=1.6.1 30 | - pip install pandas>=0.20.1 31 | 32 | script: 33 | - tox -e py${TRAVIS_PYTHON_VERSION}-django${DJANGO} 34 | 35 | matrix: 36 | exclude: 37 | # Django 1.8 supports Python >=2.7, <=3.5 38 | - python: "3.6" 39 | env: DJANGO=18 40 | - python: "3.7" 41 | env: DJANGO=18 42 | - python: "3.8" 43 | env: DJANGO=18 44 | - python: "3.9" 45 | env: DJANGO=18 46 | # Django 1.9 supports Python >=2.7, <=3.5 47 | - python: "3.6" 48 | env: DJANGO=19 49 | - python: "3.7" 50 | env: DJANGO=19 51 | - python: "3.8" 52 | env: DJANGO=19 53 | - python: "3.9" 54 | env: DJANGO=19 55 | # Django 1.10 supports Python >=2.7, <=3.5 56 | - python: "3.6" 57 | env: DJANGO=110 58 | - python: "3.7" 59 | env: DJANGO=110 60 | - python: "3.8" 61 | env: DJANGO=110 62 | - python: "3.9" 63 | env: DJANGO=110 64 | # Django 1.11 supports Python >=2.7, <=3.7 65 | - python: "3.8" 66 | env: DJANGO=111 67 | - python: "3.9" 68 | env: DJANGO=111 69 | # Django 2.0 supports Python >=3.4, <=3.7 70 | - python: "2.7" 71 | env: DJANGO=20 72 | - python: "3.8" 73 | env: DJANGO=20 74 | - python: "3.9" 75 | env: DJANGO=20 76 | # Django 2.1 supports Python >=3.5, <=3.7 77 | - python: "2.7" 78 | env: DJANGO=21 79 | - python: "3.8" 80 | env: DJANGO=21 81 | - python: "3.9" 82 | env: DJANGO=21 83 | # Django 2.2 supports Python >=3.5, <=3.9 84 | - python: "2.7" 85 | env: DJANGO=22 86 | # Django 3.0 supports Python >=3.6, <=3.9 87 | - python: "2.7" 88 | env: DJANGO=30 89 | - python: "3.5" 90 | env: DJANGO=30 91 | # Django 3.1 supports Python >=3.6, <=3.9 92 | - python: "2.7" 93 | env: DJANGO=31 94 | - python: "3.5" 95 | env: DJANGO=31 96 | # Django 3.2 supports Python >=3.6, <=3.9 97 | - python: "2.7" 98 | env: DJANGO=32 99 | - python: "3.5" 100 | env: DJANGO=32 101 | 102 | after_success: coveralls 103 | -------------------------------------------------------------------------------- /AUTHORS.rst: -------------------------------------------------------------------------------- 1 | Django-pandas is written and maintained by Christopher Clarke and 2 | various contributors: 3 | 4 | Development Lead 5 | ```````````````` 6 | 7 | - Christopher Clarke 8 | 9 | Contributions 10 | `````````````` 11 | 12 | - `Christopher Clarke `_ 13 | - `Bertrand Bordage `_ 14 | - `Guillaume Thomas `_ 15 | - `Hélio Meira Lins `_ 16 | - `Parbhat Puri `_ 17 | - `Fredrik Burman (coachHIPPO) `_ 18 | - `Safe Hammad `_ 19 | - `Jeff Sternber `_ 20 | - `@MiddleFork `_ 21 | - `Daniel Andrlik `_ 22 | - `Kevin Abbot `_ 23 | - `Yousuf Jawwad `_ 24 | - `@henhuy `_ 25 | - `Hélio Meira Lins `_ 26 | - `@utpyngo `_ 27 | - `Anthony Monthe `_ 28 | - `Vincent Toupet `_ 29 | - `Anton Ian Sipos `_ 30 | - `Chuan-Jhe Hwong `_ 31 | - `Thomas Grainger `_ 32 | - `Ryan Smith `_ 33 | -------------------------------------------------------------------------------- /CHANGES.rst: -------------------------------------------------------------------------------- 1 | CHANGES 2 | ======== 3 | 0.6.7 (2024-03-27) 4 | 5 | Fix several deprecation warnings in pandas 2.1 which became actual errors in 2.2 6 | as per `#158`_ thanks to @bixbyr 7 | 8 | 0.6.6 (2021-10-27) 9 | ------------------ 10 | The main feature of this is release in the use of a GHA to 11 | automate the publishing of the package to PYPI as per PR `#146`_ 12 | (again much thanks @graingert). Several other minor issues have also 13 | been addressed. 14 | 15 | .. _`#146`: https://github.com/chrisdev/django-pandas/pull/146 16 | 17 | 0.6.5 (2021-10-15) 18 | ------------------ 19 | This version added support for Pandas >=1.3 (thanks to @graingert) 20 | 21 | Other Changes: 22 | 23 | * Migrated from Travis to Github Actions for CI (also @graingert) 24 | 25 | * Avoid the use of deprecated methods `#139`_ and `#142`_ (again much thanks @graingert) 26 | 27 | * Fix for issue `#135`_ (Thanks @Yonimdo) 28 | 29 | * Silence Django 3.2 errors on testing on etc. `#133`_ thanks @whyscream. 30 | 31 | .. _`#139`: https://github.com/chrisdev/django-pandas/issues/135 32 | .. _`#142`: https://github.com/chrisdev/django-pandas/issues/142 33 | .. _`#135`: https://github.com/chrisdev/django-pandas/issues/135 34 | .. _`#133`: https://github.com/chrisdev/django-pandas/issues/133 35 | 36 | 0.6.4 (2021-02-08) 37 | ------------------ 38 | Bumped version number as the previous release was incorrectly uploaded 39 | to pypi 40 | 41 | 0.6.1 (2020-05-26) 42 | ------------------ 43 | Supports the latest release of Pandas 1.0.3 44 | 45 | 0.6.0 (2019-01-11) 46 | ------------------ 47 | Removes compatibility with Django versions < 1.8 48 | 49 | 50 | 0.5.2 (2019-01-3) 51 | ----------------- 52 | **This is the last version that supports Django < 1.8** 53 | 54 | - Improved coerce_float option (thanks @ZuluPro ) 55 | - Ensure compatibility with legacy versions of Django ( < 1.8) 56 | - Test pass with Django 2+ and python 3.7 57 | 58 | 0.5.1 (2018-01-26) 59 | ------------------- 60 | - Address Unicode decode error when installing with pip3 on docker (Thanks @utapyngo) 61 | 62 | 0.5.0 (2018-01-20) 63 | ------------------ 64 | - Django 2.0 compatibility (Thanks @meirains) 65 | 66 | 0.4.5 (2017-10-4) 67 | ----------------- 68 | - A Fix for fieldname deduplication bug thanks to @kgabbott 69 | 70 | 0.4.4 (2017-07-16) 71 | ------------------- 72 | - The `verbose` argument now handles more use cases (Thanks to @henhuy and 73 | Kevin Abbott) 74 | - Corece float argument add to ```to_timeseries()``` method (Thanks Yousuf Jawwad) 75 | 76 | 0.4.3 (2017-06-02) 77 | -------------------- 78 | - Fix doc typos and formatting 79 | - Prevent column duplication in read_frame (Thanks Kevin Abbott) 80 | 81 | 0.4.2 (2017-05-22) 82 | -------------------- 83 | - Compatibility with `pandas 0.20.1` 84 | - Support for Python 2.7 and 3.5 with Django versions 1.8+ 85 | - Suport for Python 3.6 and Django 1.11 86 | - We still support legacy versions (Python 2.7 and Django 1.4) 87 | 88 | 0.4.1 (2016-02-05) 89 | ------------------- 90 | - Address the incompatibility with Django 1.9 due to the removal of 91 | specialized query sets like the 92 | `ValuesQuerySet `_ 93 | - Address the removal of the ``PassThrougManager`` from ``django-model-utils`` 94 | version ``2.4``. We've removed the dependency on django-model-utils and 95 | included the PassThroughManger (which was always a standalone tool 96 | distributed a part of django-model-utils) for compatibility with 97 | earlier versions of Django (<= 1.8). For more recent versions of 98 | Django we're using Django's built in ``QuerySet.as_manager()``. 99 | - Now supports Pandas 0.14.1 and above 100 | - The fall in Coverage in this release largely reflects the integration of 101 | the PassThroughManager into the code base. We'll add the required test 102 | coverage for the PassThroughManager in subsequent releases. 103 | 104 | 0.3.1 (2015-10-25) 105 | ------------------- 106 | - Extends the ability to span a ForeignKey relationship with double underscores 107 | to OneToOneField too thanks to Safe Hammad 108 | - Provide better support for ManyToMany and OneToMany relations thanks to 109 | Jeff Sternberg and @MiddleFork 110 | 111 | 0.3.0 (2015-06-16) 112 | --------------------- 113 | - This version supports Django 1.8 114 | - Support for Pandas 0.16 115 | 116 | 0.2.2 (2015-03-02) 117 | --------------------- 118 | - Added Support for Django 1.7 119 | 120 | 0.2.1 (2015-01-28) 121 | --------------------- 122 | - Added Support for Values QuerySets 123 | - Support for Python 2.6 124 | - Note we still have limited support for Django 1.7 but this will be coming in 125 | the next release 126 | 127 | 0.2.0 (2014-06-15) 128 | -------------------- 129 | 130 | - Added the ``io`` module so that DataFrames can be created from any 131 | queryset so you don't need to to add a ``DataFrame manager`` to your 132 | models. This is good for working with legacy projects. 133 | - added a Boolean ``verbose`` argument to all methods (which defaults to ``True``) 134 | This populate the DataFrame columns with the human readable versions of 135 | foreign key or choice fields. 136 | - Improved the performance DataFrame creation by removing dependency on 137 | ``np.core.records.fromrecords`` 138 | - Loads of bug fixes, more tests and improved coverage and better 139 | documentation 140 | -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | How To Contribute 2 | ================= 3 | 4 | ``django-pandas`` is always open for suggestions and contributions 5 | by generous developers. I’ve collected a few tips to get you started. 6 | 7 | Please: 8 | 9 | - Obey `PEP 8`_ and `PEP 257`_. 10 | - *Always* add tests and docs for your code. 11 | - Add yourself to the AUTHORS.rst_ file in an alphabetical fashion. 12 | - Write `good commit messages`_. 13 | - Ideally, squash_ your commits, i.e. make your pull requests just one commit. 14 | 15 | Thank you for considering to contribute to ``django-pandas``. 16 | 17 | 18 | .. _`squash`: http://gitready.com/advanced/2009/02/10/squashing-commits-with-rebase.html 19 | .. _`PEP 8`: http://www.python.org/dev/peps/pep-0008/ 20 | .. _`PEP 257`: http://www.python.org/dev/peps/pep-0257/ 21 | .. _`good commit messages`: http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html 22 | .. _`AUTHORS.rst`: https://github.com/chrisdev/django-pandas/blob/master/AUTHORS.rst 23 | -------------------------------------------------------------------------------- /LICENSE.rst: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013, Christopher Clarke and contributors 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are 6 | met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | * Redistributions in binary form must reproduce the above 11 | copyright notice, this list of conditions and the following 12 | disclaimer in the documentation and/or other materials provided 13 | with the distribution. 14 | * Neither the name of the author nor the names of other 15 | contributors may be used to endorse or promote products derived 16 | from this software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | # added by check_manifest.py 2 | include *.py 3 | include *.rst 4 | include tox.ini 5 | include MANIFEST.in 6 | recursive-include docs *.bat 7 | recursive-include docs *.py 8 | recursive-include docs *.rst 9 | recursive-include docs Makefile 10 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: clean-pyc clean-build docs clean 2 | 3 | help: 4 | @echo "clean-build - remove build artifacts" 5 | @echo "clean-pyc - remove Python file artifacts" 6 | @echo "lint - check style with flake8" 7 | @echo "test - run tests quickly with the default Python" 8 | @echo "test-all - run tests on every Python version with tox" 9 | @echo "coverage - check code coverage quickly with the default Python" 10 | @echo "docs - generate Sphinx HTML documentation, including API docs" 11 | @echo "release - package and upload a release" 12 | @echo "dist - package" 13 | 14 | clean: clean-build clean-pyc 15 | rm -fr htmlcov/ 16 | 17 | clean-build: 18 | rm -fr build/ 19 | rm -fr dist/ 20 | rm -fr *.egg-info 21 | 22 | clean-pyc: 23 | find . -name '*.pyc' -exec rm -f {} + 24 | find . -name '*.pyo' -exec rm -f {} + 25 | find . -name '*~' -exec rm -f {} + 26 | 27 | develop: 28 | pip install -e .[test] 29 | 30 | lint: 31 | flake8 django_pandas tests 32 | 33 | test: develop 34 | python runtests.py 35 | 36 | test-all: 37 | tox 38 | 39 | coverage: develop 40 | coverage run --source django_pandas runtests.py 41 | coverage report -m 42 | coverage html 43 | open htmlcov/index.html 44 | 45 | docs: 46 | rm -rf docs/django_pandas.rst 47 | rm -f docs/django_pandas.tests.rst 48 | sphinx-apidoc -o docs django_pandas 49 | $(MAKE) -C docs clean 50 | $(MAKE) -C docs html 51 | open docs/_build/html/index.html 52 | 53 | release: clean 54 | python3 -m build 55 | python3 -m twine upload --repository pypi dist/* 56 | 57 | test-release: clean 58 | python3 -m build 59 | python3 -m twine upload --repository testpypi dist/* 60 | 61 | dist: clean 62 | python3 -m build 63 | ls -l dist 64 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ============== 2 | Django Pandas 3 | ============== 4 | 5 | .. image:: https://github.com/chrisdev/django-pandas/actions/workflows/test.yml/badge.svg 6 | :target: https://github.com/chrisdev/django-pandas/actions/workflows/test.yml 7 | 8 | .. image:: https://coveralls.io/repos/chrisdev/django-pandas/badge.png?branch=master 9 | :target: https://coveralls.io/r/chrisdev/django-pandas 10 | 11 | Tools for working with `pandas `_ in your Django 12 | projects 13 | 14 | Contributors 15 | ============ 16 | * `Christopher Clarke `_ 17 | * `Bertrand Bordage `_ 18 | * `Guillaume Thomas `_ 19 | * `Parbhat Puri `_ 20 | * `Fredrik Burman (coachHIPPO) `_ 21 | * `Safe Hammad `_ 22 | * `Jeff Sternber `_ 23 | * `@MiddleFork `_ 24 | * `Daniel Andrlik `_ 25 | * `Kevin Abbot `_ 26 | * `Yousuf Jawwad `_ 27 | * `@henhuy `_ 28 | * `Hélio Meira Lins `_ 29 | * `@utpyngo `_ 30 | * `Anthony Monthe `_ 31 | * `Vincent Toupet `_ 32 | * `Anton Ian Sipos `_ 33 | * `Thomas Grainger `_ 34 | * `Ryan Smith `_ 35 | 36 | What's New 37 | =========== 38 | This is release facilitates running of test with Python 3.10 and automates 39 | the publishing of the package to PYPI as per PR `#146`_ 40 | (again much thanks @graingert). As usual we have attempted support legacy 41 | versions of Python/Django/Pandas and this sometimes results in deperation errors 42 | being displayed in when test are run. To avoid use `python -Werror runtests.py` 43 | 44 | .. _`#146`: https://github.com/chrisdev/django-pandas/pull/146 45 | 46 | Dependencies 47 | ============= 48 | ``django-pandas`` supports `Django`_ (>=1.4.5) or later 49 | and requires `django-model-utils`_ (>= 1.4.0) and `Pandas`_ (>= 0.12.0). 50 | **Note** because of problems with the ``requires`` directive of setuptools 51 | you probably need to install ``numpy`` in your virtualenv before you install 52 | this package or if you want to run the test suite :: 53 | 54 | pip install numpy 55 | pip install -e .[test] 56 | python runtests.py 57 | 58 | Some ``pandas`` functionality requires parts of the Scipy stack. 59 | You may wish to consult http://www.scipy.org/install.html 60 | for more information on installing the ``Scipy`` stack. 61 | 62 | You need to install your preferred version of Django. 63 | as that Django 2 does not support Python 2. 64 | 65 | 66 | .. _Django: http://djangoproject.com/ 67 | .. _django-model-utils: http://pypi.python.org/pypi/django-model-utils 68 | .. _Pandas: http://pandas.pydata.org 69 | 70 | Contributing 71 | ============ 72 | 73 | Please file bugs and send pull requests to the `GitHub repository`_ and `issue 74 | tracker`_. 75 | 76 | .. _GitHub repository: https://github.com/chrisdev/django-pandas/ 77 | .. _issue tracker: https://github.com/chrisdev/django-pandas/issues 78 | 79 | 80 | Installation 81 | ============= 82 | Start by creating a new ``virtualenv`` for your project :: 83 | 84 | mkvirtualenv myproject 85 | 86 | Next install ``numpy`` and ``pandas`` and optionally ``scipy`` :: 87 | 88 | pip install numpy 89 | pip install pandas 90 | 91 | You may want to consult the `scipy documentation`_ for more information 92 | on installing the ``Scipy`` stack. 93 | 94 | .. _scipy documentation: http://www.scipy.org/install.html 95 | 96 | Finally, install ``django-pandas`` using ``pip``:: 97 | 98 | pip install django-pandas 99 | 100 | or install the development version from ``github`` :: 101 | 102 | pip install https://github.com/chrisdev/django-pandas/tarball/master 103 | 104 | Usage 105 | ====== 106 | 107 | 108 | IO Module 109 | ---------- 110 | The ``django-pandas.io`` module provides some convenience methods to 111 | facilitate the creation of DataFrames from Django QuerySets. 112 | 113 | read_frame 114 | ^^^^^^^^^^^ 115 | 116 | **Parameters** 117 | 118 | - qs: A Django QuerySet. 119 | 120 | - fieldnames: A list of model field names to use in creating the ``DataFrame``. 121 | You can span a relationship in the usual Django way 122 | by using double underscores to specify a related field 123 | in another model 124 | 125 | - index_col: Use specify the field name to use for the ``DataFrame`` index. 126 | If the index 127 | field is not in the field list it will be appended 128 | 129 | - coerce_float : Boolean, defaults to True 130 | Attempt to convert values to non-string, 131 | non-numeric objects (like decimal.Decimal) 132 | to floating point. 133 | 134 | - verbose: If this is ``True`` then populate the DataFrame with the 135 | human readable versions of any foreign key or choice fields 136 | else use the actual values set in the model. 137 | 138 | - column_names: If not None, use to override the column names in the 139 | DateFrame 140 | 141 | Examples 142 | ^^^^^^^^^ 143 | Assume that this is your model:: 144 | 145 | class MyModel(models.Model): 146 | 147 | full_name = models.CharField(max_length=25) 148 | age = models.IntegerField() 149 | department = models.CharField(max_length=3) 150 | wage = models.FloatField() 151 | 152 | First create a query set:: 153 | 154 | from django_pandas.io import read_frame 155 | qs = MyModel.objects.all() 156 | 157 | To create a dataframe using all the fields in the underlying model :: 158 | 159 | df = read_frame(qs) 160 | 161 | The `df` will contain human readable column values for foreign key and choice 162 | fields. The `DataFrame` will include all the fields in the underlying 163 | model including the primary key. 164 | To create a DataFrame using specified field names:: 165 | 166 | df = read_frame(qs, fieldnames=['age', 'wage', 'full_name']) 167 | 168 | To set ``full_name`` as the ``DataFrame`` index :: 169 | 170 | qs.to_dataframe(['age', 'wage'], index_col='full_name']) 171 | 172 | You can use filters and excludes :: 173 | 174 | qs.filter(age__gt=20, department='IT').to_dataframe(index_col='full_name') 175 | 176 | 177 | DataFrameManager 178 | ----------------- 179 | ``django-pandas`` provides a custom manager to use with models that 180 | you want to render as Pandas Dataframes. The ``DataFrameManager`` 181 | manager provides the ``to_dataframe`` method that returns 182 | your models queryset as a Pandas DataFrame. To use the DataFrameManager, first 183 | override the default manager (`objects`) in your model's definition 184 | as shown in the example below :: 185 | 186 | #models.py 187 | 188 | from django_pandas.managers import DataFrameManager 189 | 190 | class MyModel(models.Model): 191 | 192 | full_name = models.CharField(max_length=25) 193 | age = models.IntegerField() 194 | department = models.CharField(max_length=3) 195 | wage = models.FloatField() 196 | 197 | objects = DataFrameManager() 198 | 199 | 200 | This will give you access to the following QuerySet methods: 201 | 202 | - ``to_dataframe`` 203 | - ``to_timeseries`` 204 | - ``to_pivot_table`` 205 | 206 | to_dataframe 207 | ^^^^^^^^^^^^^ 208 | 209 | Returns a DataFrame from the QuerySet 210 | 211 | **Parameters** 212 | 213 | - fieldnames: The model field names to utilise in creating the frame. 214 | to span a relationship, use the field name of related 215 | fields across models, separated by double underscores, 216 | 217 | 218 | - index: specify the field to use for the index. If the index 219 | field is not in the field list it will be appended 220 | 221 | - coerce_float: Attempt to convert the numeric non-string data 222 | like object, decimal etc. to float if possible 223 | 224 | - verbose: If this is ``True`` then populate the DataFrame with the 225 | human readable versions of any foreign key or choice fields 226 | else use the actual value set in the model. 227 | 228 | Examples 229 | ^^^^^^^^^ 230 | 231 | Create a dataframe using all the fields in your model as follows :: 232 | 233 | qs = MyModel.objects.all() 234 | 235 | df = qs.to_dataframe() 236 | 237 | This will include your primary key. To create a DataFrame using specified 238 | field names:: 239 | 240 | df = qs.to_dataframe(fieldnames=['age', 'department', 'wage']) 241 | 242 | To set ``full_name`` as the index :: 243 | 244 | qs.to_dataframe(['age', 'department', 'wage'], index='full_name']) 245 | 246 | You can use filters and excludes :: 247 | 248 | qs.filter(age__gt=20, department='IT').to_dataframe(index='full_name') 249 | 250 | to_timeseries 251 | -------------- 252 | 253 | A convenience method for creating a time series i.e the 254 | DataFrame index is instance of a DateTime or PeriodIndex 255 | 256 | **Parameters** 257 | 258 | - fieldnames: The model field names to utilise in creating the frame. 259 | to span a relationship, just use the field name of related 260 | fields across models, separated by double underscores, 261 | 262 | - index: specify the field to use for the index. If the index 263 | field is not in the field list it will be appended. This 264 | is mandatory. 265 | 266 | - storage: Specify if the queryset uses the `wide` or `long` format 267 | for data. 268 | 269 | - pivot_columns: Required once the you specify `long` format 270 | storage. This could either be a list or string identifying 271 | the field name or combination of field. If the pivot_column 272 | is a single column then the unique values in this column become 273 | a new columns in the DataFrame 274 | If the pivot column is a list the values in these columns are 275 | concatenated (using the '-' as a separator) 276 | and these values are used for the new timeseries columns 277 | 278 | - values: Also required if you utilize the `long` storage the 279 | values column name is use for populating new frame values 280 | 281 | - freq: the offset string or object representing a target conversion 282 | 283 | - rs_kwargs: Arguments based on pandas.DataFrame.resample 284 | 285 | - verbose: If this is ``True`` then populate the DataFrame with the 286 | human readable versions of any foreign key or choice fields 287 | else use the actual value set in the model. 288 | 289 | Examples 290 | ^^^^^^^^^ 291 | 292 | Using a *long* storage format :: 293 | 294 | #models.py 295 | 296 | class LongTimeSeries(models.Model): 297 | date_ix = models.DateTimeField() 298 | series_name = models.CharField(max_length=100) 299 | value = models.FloatField() 300 | 301 | objects = DataFrameManager() 302 | 303 | Some sample data::: 304 | 305 | ======== ===== ===== 306 | date_ix series_name value 307 | ======== ===== ====== 308 | 2010-01-01 gdp 204699 309 | 310 | 2010-01-01 inflation 2.0 311 | 312 | 2010-01-01 wages 100.7 313 | 314 | 2010-02-01 gdp 204704 315 | 316 | 2010-02-01 inflation 2.4 317 | 318 | 2010-03-01 wages 100.4 319 | 320 | 2010-02-01 gdp 205966 321 | 322 | 2010-02-01 inflation 2.5 323 | 324 | 2010-03-01 wages 100.5 325 | ========== ========== ====== 326 | 327 | 328 | Create a QuerySet :: 329 | 330 | qs = LongTimeSeries.objects.filter(date_ix__year__gte=2010) 331 | 332 | Create a timeseries dataframe :: 333 | 334 | df = qs.to_timeseries(index='date_ix', 335 | pivot_columns='series_name', 336 | values='value', 337 | storage='long') 338 | df.head() 339 | 340 | date_ix gdp inflation wages 341 | 342 | 2010-01-01 204966 2.0 100.7 343 | 344 | 2010-02-01 204704 2.4 100.4 345 | 346 | 2010-03-01 205966 2.5 100.5 347 | 348 | 349 | Using a *wide* storage format :: 350 | 351 | class WideTimeSeries(models.Model): 352 | date_ix = models.DateTimeField() 353 | col1 = models.FloatField() 354 | col2 = models.FloatField() 355 | col3 = models.FloatField() 356 | col4 = models.FloatField() 357 | 358 | objects = DataFrameManager() 359 | 360 | qs = WideTimeSeries.objects.all() 361 | 362 | rs_kwargs = {'how': 'sum', 'kind': 'period'} 363 | df = qs.to_timeseries(index='date_ix', pivot_columns='series_name', 364 | values='value', storage='long', 365 | freq='M', rs_kwargs=rs_kwargs) 366 | 367 | to_pivot_table 368 | -------------- 369 | A convenience method for creating a pivot table from a QuerySet 370 | 371 | **Parameters** 372 | 373 | - fieldnames: The model field names to utilise in creating the frame. 374 | to span a relationship, just use the field name of related 375 | fields across models, separated by double underscores, 376 | - values : column to aggregate, optional 377 | - rows : list of column names or arrays to group on 378 | Keys to group on the x-axis of the pivot table 379 | - cols : list of column names or arrays to group on 380 | Keys to group on the y-axis of the pivot table 381 | - aggfunc : function, default numpy.mean, or list of functions 382 | If list of functions passed, the resulting pivot table will have 383 | hierarchical columns whose top level are the function names 384 | (inferred from the function objects themselves) 385 | - fill_value : scalar, default None 386 | Value to replace missing values with 387 | - margins : boolean, default False 388 | Add all row / columns (e.g. for subtotal / grand totals) 389 | - dropna : boolean, default True 390 | 391 | **Example** 392 | :: 393 | 394 | # models.py 395 | class PivotData(models.Model): 396 | row_col_a = models.CharField(max_length=15) 397 | row_col_b = models.CharField(max_length=15) 398 | row_col_c = models.CharField(max_length=15) 399 | value_col_d = models.FloatField() 400 | value_col_e = models.FloatField() 401 | value_col_f = models.FloatField() 402 | 403 | objects = DataFrameManager() 404 | 405 | Usage :: 406 | 407 | rows = ['row_col_a', 'row_col_b'] 408 | cols = ['row_col_c'] 409 | 410 | pt = qs.to_pivot_table(values='value_col_d', rows=rows, cols=cols) 411 | 412 | 413 | .. end-here 414 | -------------------------------------------------------------------------------- /TODO.rst: -------------------------------------------------------------------------------- 1 | ==== 2 | TODO 3 | ==== 4 | 5 | - Add coverage and tox and integrate with travis-CL 6 | - Add to pivot_table method 7 | 8 | 2013-07-24 9 | ----------- 10 | - We need to implement the pivot table method 11 | 12 | - Can we bypass the ValuesListQuerySet and numpy_fromrecords and just use the 13 | DatatFrame.from_records with the tuple of sql records 14 | 15 | 16 | 2013-07-19 17 | ----------- 18 | We thinking of implenenting the following API 19 | 20 | - **to\_dataframe** - the core method which returns a dataframe based 21 | on the columns that you specify you can also set the index column 22 | 23 | **Arguments** 24 | 25 | *cols*: the model fields to utilise in creating the frame. to span a 26 | relationship, just use the field name of related fields across 27 | models, separated by double underscores, 28 | 29 | *index*: the model field name to use for the index 30 | 31 | *coerce\_float*: The returned columns (except the index) will be 32 | floats. This may be required if the queryset returns lots of null 33 | values 34 | 35 | - **to\_timeseries** - A convenience method to create a pandas time 36 | series from a queryset 37 | 38 | **Arguments** 39 | 40 | *freq*: A string representing the pandas frequency or date offset 41 | 42 | *storage*: specify if your queryset uses the `wide` or `long` format for 43 | data. 44 | 45 | **wide format** 46 | 47 | ::: 48 | 49 | date gdp inflation wages 50 | 51 | 2010-01-01 204966 2.0 100.7 52 | 53 | 2010-02-02 204704 2.4 100.4 54 | 55 | 2010-03-01 205966 2.5 100.5 56 | 57 | 58 | **long or stacked format** 59 | 60 | :: 61 | 62 | date series_mame value 63 | 64 | 2010-01-01 gdp 204699 65 | 66 | 2010-01-01 inflation 2.0 67 | 68 | 2010-01-01 wages 100.7 69 | 70 | 2010-02-01 gdp 204704 71 | 72 | 2010-02-01 inflation 2.4 73 | 74 | 2010-03-01 wages 100.4 75 | 76 | 2010-02-01 gdp 205966 77 | 78 | 2010-02-01 inflation 2.5 79 | 80 | 2010-03-01 wages 100.5 81 | 82 | *pivot\_column:* This is required once the you specify ``long`` for 83 | the storage\_fmt. This could either be a list or string identifying 84 | the column name or combination of columns that contain the ‘pivot’ 85 | identifying column. If your pivot\_column is a single column then the 86 | unique values in this column become new time series columns. If you 87 | sepecify a list of columns then the values in these columns are 88 | concatenated (using the '-' as a seperator and these values are used 89 | for the new timeseries columns 90 | 91 | *values*: Also required if you utilize the ``long`` storage the 92 | values column name is use for populating new frame’s values 93 | 94 | *fill\_na*: Fill in the missing values using the specifies method 95 | methods {'backfill, 'bill', 'pad', 'ffill'} 96 | 97 | - **to\_pivot\_table** - A convenience method to create a pivot table 98 | from the queryset 99 | 100 | *values*: column to aggregate 101 | 102 | *rows*: the list of column names to group on 103 | 104 | *cols*: list of column names to group on 105 | 106 | *aggfunc*: the function to uses in calculate the group aggregate 107 | 108 | *fill\_value*: the value to replace missing values with 109 | 110 | *margin*: boolean defalut False. Calculate subtotals/grand totals 111 | 112 | *dropna*: Do not include columns whoes entries are all NaN 113 | 114 | 115 | -------------------------------------------------------------------------------- /django_pandas/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = '0.6.7' 2 | -------------------------------------------------------------------------------- /django_pandas/io.py: -------------------------------------------------------------------------------- 1 | import django 2 | import pandas as pd 3 | 4 | from .utils import update_with_verbose, get_related_model 5 | 6 | FieldDoesNotExist = ( 7 | django.db.models.fields.FieldDoesNotExist 8 | if django.VERSION < (1, 8) 9 | else django.core.exceptions.FieldDoesNotExist 10 | ) 11 | 12 | 13 | def to_fields(qs, fieldnames): 14 | for fieldname in fieldnames: 15 | model = qs.model 16 | for fieldname_part in fieldname.split('__'): 17 | try: 18 | field = model._meta.get_field(fieldname_part) 19 | except FieldDoesNotExist: 20 | try: 21 | rels = model._meta.get_all_related_objects_with_model() 22 | except AttributeError: 23 | field = fieldname 24 | else: 25 | for relobj, _ in rels: 26 | if relobj.get_accessor_name() == fieldname_part: 27 | field = relobj.field 28 | model = field.model 29 | break 30 | else: 31 | model = get_related_model(field) 32 | yield field 33 | 34 | 35 | def is_values_queryset(qs): 36 | if django.VERSION < (1, 9): # pragma: no cover 37 | return isinstance(qs, django.db.models.query.ValuesQuerySet) 38 | else: 39 | try: 40 | return qs._iterable_class == django.db.models.query.ValuesIterable 41 | except: 42 | return False 43 | 44 | 45 | def read_frame(qs, fieldnames=(), index_col=None, coerce_float=False, 46 | verbose=True, datetime_index=False, column_names=None): 47 | """ 48 | Returns a dataframe from a QuerySet 49 | 50 | Optionally specify the field names/columns to utilize and 51 | a field as the index 52 | 53 | Parameters 54 | ---------- 55 | 56 | qs: The Django QuerySet. 57 | fieldnames: The model field names to use in creating the frame. 58 | You can span a relationship in the usual Django way 59 | by using double underscores to specify a related field 60 | in another model 61 | You can span a relationship in the usual Django way 62 | by using double underscores to specify a related field 63 | in another model 64 | 65 | index_col: specify the field to use for the index. If the index 66 | field is not in the field list it will be appended 67 | 68 | coerce_float : boolean, default False 69 | Attempt to convert values to non-string, non-numeric data (like 70 | decimal.Decimal) to floating point, useful for SQL result sets 71 | 72 | verbose: boolean If this is ``True`` then populate the DataFrame with the 73 | human readable versions of any foreign key fields else use 74 | the primary keys values. 75 | The human readable version of the foreign key field is 76 | defined in the ``__unicode__`` or ``__str__`` 77 | methods of the related class definition 78 | 79 | datetime_index: specify whether index should be converted to a 80 | DateTimeIndex. 81 | 82 | column_names: If not None, use to override the column names in the 83 | DateFrame 84 | """ 85 | 86 | if fieldnames: 87 | fieldnames = pd.unique(pd.Series(fieldnames)) 88 | if index_col is not None and index_col not in fieldnames: 89 | # Add it to the field names if not already there 90 | fieldnames = tuple(fieldnames) + (index_col,) 91 | if column_names: 92 | column_names = tuple(column_names) + (index_col,) 93 | fields = to_fields(qs, fieldnames) 94 | elif is_values_queryset(qs): 95 | if django.VERSION < (1, 9): # pragma: no cover 96 | annotation_field_names = list(qs.query.annotation_select) 97 | 98 | if annotation_field_names is None: 99 | annotation_field_names = [] 100 | 101 | extra_field_names = qs.extra_names 102 | if extra_field_names is None: 103 | extra_field_names = [] 104 | 105 | select_field_names = qs.field_names 106 | 107 | else: # pragma: no cover 108 | annotation_field_names = list(qs.query.annotation_select) 109 | extra_field_names = list(qs.query.extra_select) 110 | select_field_names = list(qs.query.values_select) 111 | 112 | fieldnames = select_field_names + annotation_field_names + \ 113 | extra_field_names 114 | fields = [None if '__' in f else qs.model._meta.get_field(f) 115 | for f in select_field_names] + \ 116 | [None] * (len(annotation_field_names) + len(extra_field_names)) 117 | 118 | uniq_fields = set() 119 | fieldnames, fields = zip( 120 | *(f for f in zip(fieldnames, fields) 121 | if f[0] not in uniq_fields and not uniq_fields.add(f[0]))) 122 | else: 123 | try: 124 | fields = qs.model._meta.fields 125 | fieldnames = [f.name for f in fields] 126 | fieldnames += list(qs.query.annotation_select.keys()) 127 | except: 128 | pass 129 | 130 | if is_values_queryset(qs): 131 | recs = list(qs) 132 | else: 133 | try: 134 | recs = list(qs.values_list(*fieldnames)) 135 | except: 136 | if fieldnames: 137 | recs = [object_to_dict(q, fieldnames) for q in qs] 138 | else: 139 | recs = [object_to_dict(q) for q in qs] 140 | 141 | df = pd.DataFrame.from_records( 142 | recs, 143 | columns=column_names if column_names else fieldnames, 144 | coerce_float=coerce_float 145 | ) 146 | 147 | if verbose: 148 | update_with_verbose(df, fieldnames, fields) 149 | 150 | if index_col is not None: 151 | df.set_index(index_col, inplace=True) 152 | 153 | if datetime_index: 154 | df.index = pd.to_datetime(df.index) 155 | return df 156 | 157 | 158 | def object_to_dict(obj, fields: list = None): 159 | """ 160 | Convert obj to a dictionary 161 | 162 | Parameters 163 | ---------- 164 | 165 | obj: obj to an item of QuerySet 166 | fieldnames: reserved fields, default to all fields 167 | """ 168 | if not fields: 169 | obj.__dict__.pop('_state') 170 | return obj.__dict__ 171 | return {field: obj.__dict__.get(field) for field in fields} 172 | -------------------------------------------------------------------------------- /django_pandas/managers.py: -------------------------------------------------------------------------------- 1 | from django.db.models.query import QuerySet 2 | from .io import read_frame 3 | import django 4 | from django.db import models 5 | 6 | 7 | class PassThroughManagerMixin(object): 8 | """ 9 | A mixin that enables you to call custom QuerySet methods from your manager. 10 | """ 11 | _deny_methods = ['__getstate__', '__setstate__', '__getinitargs__', 12 | '__getnewargs__', '__copy__', '__deepcopy__', '_db', 13 | '__slots__'] 14 | 15 | def __init__(self, queryset_cls=None): 16 | self._queryset_cls = queryset_cls 17 | super(PassThroughManagerMixin, self).__init__() 18 | 19 | def __getattr__(self, name): # pragma: no cover 20 | if name in self._deny_methods: 21 | raise AttributeError(name) 22 | return getattr(self.get_queryset(), name) 23 | 24 | def __dir__(self): # pragma: no cover 25 | my_values = frozenset(dir(type(self))) 26 | my_values |= frozenset(dir(self.get_query_set())) 27 | return list(my_values) 28 | 29 | def get_queryset(self): # pragma: no cover 30 | try: 31 | qs = super(PassThroughManagerMixin, self).get_queryset() 32 | except AttributeError: 33 | qs = super(PassThroughManagerMixin, self).get_query_set() 34 | if self._queryset_cls is not None: 35 | qs = qs._clone(klass=self._queryset_cls) 36 | return qs 37 | 38 | get_query_set = get_queryset 39 | 40 | @classmethod 41 | def for_queryset_class(cls, queryset_cls): 42 | return create_pass_through_manager_for_queryset_class( 43 | cls, queryset_cls) 44 | 45 | 46 | class PassThroughManager(PassThroughManagerMixin, models.Manager): 47 | """ 48 | Inherit from this Manager to enable you to call any methods from your 49 | custom QuerySet class from your manager. Simply define your QuerySet 50 | class, and return an instance of it from your manager's `get_queryset` 51 | method. 52 | 53 | Alternately, if you don't need any extra methods on your manager that 54 | aren't on your QuerySet, then just pass your QuerySet class to the 55 | ``for_queryset_class`` class method. 56 | 57 | class PostQuerySet(QuerySet): 58 | def enabled(self): 59 | return self.filter(disabled=False) 60 | 61 | class Post(models.Model): 62 | objects = PassThroughManager.for_queryset_class(PostQuerySet)() 63 | 64 | """ 65 | pass 66 | 67 | 68 | def create_pass_through_manager_for_queryset_class(base, queryset_cls): 69 | class _PassThroughManager(base): 70 | def __init__(self, *args, **kwargs): 71 | return super(_PassThroughManager, self).__init__(*args, **kwargs) 72 | 73 | def get_queryset(self): # pragma: no cover 74 | qs = super(_PassThroughManager, self).get_queryset() 75 | return qs._clone(klass=queryset_cls) 76 | 77 | get_query_set = get_queryset 78 | 79 | return _PassThroughManager 80 | 81 | 82 | class DataFrameQuerySet(QuerySet): 83 | 84 | def to_pivot_table(self, fieldnames=(), verbose=True, 85 | values=None, rows=None, cols=None, 86 | aggfunc='mean', fill_value=None, margins=False, 87 | dropna=True, coerce_float=True): 88 | """ 89 | A convenience method for creating a spread sheet style pivot table 90 | as a DataFrame 91 | Parameters 92 | ---------- 93 | fieldnames: The model field names(columns) to utilise in creating 94 | the DataFrame. You can span a relationships in the usual 95 | Django ORM way by using the foreign key field name 96 | separated by double underscores and refer to a field 97 | in a related model. 98 | 99 | values: The field to use to calculate the values to aggregate. 100 | 101 | rows: The list of field names to group on 102 | Keys to group on the x-axis of the pivot table 103 | 104 | cols: The list of column names or arrays to group on 105 | Keys to group on the y-axis of the pivot table 106 | 107 | aggfunc: How to arregate the values. By default this would be 108 | ``numpy.mean``. A list of aggregates functions can be passed 109 | In this case the resulting pivot table will have 110 | hierarchical columns whose top level are the function names 111 | (inferred from the function objects themselves) 112 | 113 | fill_value: A scalar value to replace the missing values with 114 | 115 | margins: Boolean, default False Add all row / columns 116 | (e.g. for subtotal / grand totals) 117 | 118 | dropna: Boolean, default True. 119 | Do not include columns whose entries are all NaN 120 | 121 | verbose: If this is ``True`` then populate the DataFrame with the 122 | human readable versions for foreign key fields else use the 123 | actual values set in the model 124 | 125 | coerce_float: Attempt to convert values to non-string, non-numeric 126 | objects (like decimal.Decimal) to floating point. 127 | """ 128 | df = self.to_dataframe(fieldnames, verbose=verbose, 129 | coerce_float=coerce_float) 130 | 131 | return df.pivot_table(values=values, fill_value=fill_value, index=rows, 132 | columns=cols, aggfunc=aggfunc, margins=margins, 133 | dropna=dropna) 134 | 135 | def to_timeseries(self, fieldnames=(), verbose=True, 136 | index=None, storage='wide', 137 | values=None, pivot_columns=None, freq=None, 138 | coerce_float=True, rs_kwargs=None, agg_args=None, 139 | agg_kwargs=None): 140 | """ 141 | A convenience method for creating a time series DataFrame i.e the 142 | DataFrame index will be an instance of DateTime or PeriodIndex 143 | 144 | Parameters 145 | ---------- 146 | 147 | fieldnames: The model field names(columns) to utilise in creating 148 | the DataFrame. You can span a relationships in the usual 149 | Django ORM way by using the foreign key field name 150 | separated by double underscores and refer to a field 151 | in a related model. 152 | 153 | index: specify the field to use for the index. If the index 154 | field is not in fieldnames it will be appended. This 155 | is mandatory for timeseries. 156 | 157 | storage: Specify if the queryset uses the 158 | ``wide`` format 159 | 160 | date | col1| col2| col3| 161 | -----------|------|-----|-----| 162 | 2001-01-01-| 100.5| 23.3| 2.2| 163 | 2001-02-01-| 106.3| 17.0| 4.6| 164 | 2001-03-01-| 111.7| 11.1| 0.7| 165 | 166 | or the `long` format. 167 | 168 | date |values| names| 169 | -----------|------|------| 170 | 2001-01-01-| 100.5| col1| 171 | 2001-02-01-| 106.3| col1| 172 | 2001-03-01-| 111.7| col1| 173 | 2001-01-01-| 23.3| col2| 174 | 2001-02-01-| 17.0| col2| 175 | 2001-01-01-| 23.3| col2| 176 | 2001-02-01-| 2.2| col3| 177 | 2001-03-01-| 4.6| col3| 178 | 2001-03-01-| 0.7| col3| 179 | 180 | 181 | pivot_columns: Required once the you specify `long` format 182 | storage. This could either be a list or string 183 | identifying the field name or combination of field. 184 | If the pivot_column is a single column then the 185 | unique values in this column become a new columns in 186 | the DataFrame If the pivot column is a list the values 187 | in these columns are concatenated (using the '-' 188 | as a separator) and these values are used for the new 189 | timeseries columns 190 | 191 | values: Also required if you utilize the `long` storage the 192 | values column name is use for populating new frame values 193 | 194 | freq: The offset string or object representing a target conversion 195 | 196 | rs_kwargs: A dictonary of keyword arguments based on the 197 | ``pandas.DataFrame.resample`` method 198 | 199 | agg_kwargs: A dictonary of keyword arguments to send to the 200 | ``pandas.DataFrame.resample().agg()`` method 201 | 202 | agg_args: A list of positional arguments to send to the 203 | ``pandas.DataFrame.resample().agg()`` method 204 | 205 | verbose: If this is ``True`` then populate the DataFrame with the 206 | human readable versions of any foreign key fields else use 207 | the primary keys values else use the actual values set 208 | in the model. 209 | 210 | coerce_float: Attempt to convert values to non-string, non-numeric 211 | objects (like decimal.Decimal) to floating point. 212 | """ 213 | assert index is not None, 'You must supply an index field' 214 | assert storage in ('wide', 'long'), 'storage must be wide or long' 215 | if rs_kwargs is None: 216 | rs_kwargs = {} 217 | 218 | if storage == 'wide': 219 | df = self.to_dataframe(fieldnames, verbose=verbose, index=index, 220 | coerce_float=coerce_float, datetime_index=True) 221 | else: 222 | df = self.to_dataframe(fieldnames, verbose=verbose, 223 | coerce_float=coerce_float, datetime_index=True) 224 | assert values is not None, 'You must specify a values field' 225 | assert pivot_columns is not None, 'You must specify pivot_columns' 226 | 227 | if isinstance(pivot_columns, (tuple, list)): 228 | df['combined_keys'] = '' 229 | for c in pivot_columns: 230 | df['combined_keys'] += df[c].str.upper() + '.' 231 | 232 | df['combined_keys'] += values.lower() 233 | 234 | df = df.pivot(index=index, 235 | columns='combined_keys', 236 | values=values) 237 | else: 238 | df = df.pivot(index=index, 239 | columns=pivot_columns, 240 | values=values) 241 | 242 | if freq is not None: 243 | if agg_kwargs is None: 244 | agg_kwargs = dict() 245 | if agg_args is None: 246 | agg_args = [] 247 | df = df.resample(freq, **rs_kwargs).agg(*agg_args, **agg_kwargs) 248 | 249 | return df 250 | 251 | def to_dataframe(self, fieldnames=(), verbose=True, index=None, 252 | coerce_float=False, datetime_index=False): 253 | """ 254 | Returns a DataFrame from the queryset 255 | 256 | Parameters 257 | ----------- 258 | 259 | fieldnames: The model field names(columns) to utilise in creating 260 | the DataFrame. You can span a relationships in the usual 261 | Django ORM way by using the foreign key field name 262 | separated by double underscores and refer to a field 263 | in a related model. 264 | 265 | 266 | index: specify the field to use for the index. If the index 267 | field is not in fieldnames it will be appended. This 268 | is mandatory for timeseries. 269 | 270 | verbose: If this is ``True`` then populate the DataFrame with the 271 | human readable versions for foreign key fields else 272 | use the actual values set in the model 273 | 274 | coerce_float: Attempt to convert values to non-string, non-numeric 275 | objects (like decimal.Decimal) to floating point. 276 | 277 | datetime_index: specify whether index should be converted to a 278 | DateTimeIndex. 279 | """ 280 | 281 | return read_frame(self, fieldnames=fieldnames, verbose=verbose, 282 | index_col=index, coerce_float=coerce_float, 283 | datetime_index=datetime_index) 284 | 285 | 286 | DataFrameManager = models.Manager.from_queryset(DataFrameQuerySet) 287 | -------------------------------------------------------------------------------- /django_pandas/models.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chrisdev/django-pandas/ed317abcaba924d6205b8f762db65bc5e6fcd81b/django_pandas/models.py -------------------------------------------------------------------------------- /django_pandas/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chrisdev/django-pandas/ed317abcaba924d6205b8f762db65bc5e6fcd81b/django_pandas/tests/__init__.py -------------------------------------------------------------------------------- /django_pandas/tests/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | from six import python_2_unicode_compatible 3 | from django_pandas.managers import DataFrameManager, PassThroughManager 4 | 5 | 6 | @python_2_unicode_compatible 7 | class MyModel(models.Model): 8 | index_col = models.CharField(max_length=1) 9 | col1 = models.IntegerField() 10 | col2 = models.FloatField(null=True) 11 | col3 = models.FloatField(null=True) 12 | col4 = models.IntegerField() 13 | 14 | def __str__(self): 15 | return "{} {} {} {}".format( 16 | self.index_col, 17 | self.col1, 18 | self.col2, 19 | self.col3, 20 | self.col4 21 | ) 22 | 23 | 24 | class MyModelChoice(models.Model): 25 | CHOICES = [ 26 | (1, u'First \U0001f947'), 27 | (2, u'Second \U0001f948'), 28 | (3, u'Third \U0001f949'), 29 | ] 30 | col1 = models.IntegerField(choices=CHOICES) 31 | col2 = models.FloatField(null=True) 32 | objects = DataFrameManager() 33 | 34 | 35 | @python_2_unicode_compatible 36 | class DataFrame(models.Model): 37 | 38 | index = models.CharField(max_length=1) 39 | col1 = models.IntegerField() 40 | col2 = models.FloatField() 41 | col3 = models.FloatField() 42 | col4 = models.IntegerField() 43 | 44 | objects = DataFrameManager() 45 | 46 | def __str__(self): 47 | return "{} {} {} {}".format( 48 | self.index, 49 | self.col1, 50 | self.col2, 51 | self.col3, 52 | self.col4 53 | ) 54 | 55 | 56 | @python_2_unicode_compatible 57 | class LongTimeSeries(models.Model): 58 | date_ix = models.DateTimeField() 59 | series_name = models.CharField(max_length=100) 60 | value = models.FloatField() 61 | 62 | objects = DataFrameManager() 63 | 64 | def __str__(self): 65 | return "{} {} {}".format(self.date_ix, 66 | self.series_name, 67 | self.value) 68 | 69 | 70 | @python_2_unicode_compatible 71 | class WideTimeSeries(models.Model): 72 | date_ix = models.DateTimeField() 73 | col1 = models.FloatField() 74 | col2 = models.FloatField() 75 | col3 = models.FloatField() 76 | col4 = models.FloatField() 77 | 78 | objects = DataFrameManager() 79 | 80 | def __str__(self): 81 | return "{} {} {} {}".format( 82 | self.date_ix, 83 | self.col1, 84 | self.col2, 85 | self.col3, 86 | self.col4 87 | ) 88 | 89 | @python_2_unicode_compatible 90 | class WideTimeSeriesDateField(models.Model): 91 | date_ix = models.DateField() 92 | col1 = models.FloatField() 93 | col2 = models.FloatField() 94 | col3 = models.FloatField() 95 | col4 = models.FloatField() 96 | 97 | objects = DataFrameManager() 98 | 99 | def __str__(self): 100 | return "{} {} {} {}".format( 101 | self.date_ix, 102 | self.col1, 103 | self.col2, 104 | self.col3, 105 | self.col4 106 | ) 107 | 108 | 109 | @python_2_unicode_compatible 110 | class PivotData(models.Model): 111 | row_col_a = models.CharField(max_length=15) 112 | row_col_b = models.CharField(max_length=15) 113 | row_col_c = models.CharField(max_length=15) 114 | value_col_d = models.FloatField() 115 | value_col_e = models.FloatField() 116 | value_col_f = models.FloatField() 117 | 118 | objects = DataFrameManager() 119 | 120 | def __str__(self): 121 | return "{0} {1} {2} {3} {4} {5}".format( 122 | self.row_col_a, self.row_col_b, self.row_col_c, 123 | self.value_col_d, self.value_col_e, self.value_col_f 124 | ) 125 | 126 | 127 | @python_2_unicode_compatible 128 | class Trader(models.Model): 129 | name = models.CharField(max_length=20) 130 | 131 | def __str__(self): 132 | return self.name 133 | 134 | 135 | @python_2_unicode_compatible 136 | class Security(models.Model): 137 | symbol = models.CharField(max_length=20) 138 | isin = models.CharField(max_length=20) 139 | 140 | def __str__(self): 141 | return "{0}-{1}".format(self.isin, self.symbol) 142 | 143 | 144 | @python_2_unicode_compatible 145 | class TradeLogNote(models.Model): 146 | note = models.TextField() 147 | 148 | def __str__(self): 149 | return self.note 150 | 151 | 152 | @python_2_unicode_compatible 153 | class TradeLog(models.Model): 154 | trader = models.ForeignKey(Trader, on_delete=models.CASCADE) 155 | symbol = models.ForeignKey(Security, null=True, on_delete=models.CASCADE) 156 | log_datetime = models.DateTimeField() 157 | price = models.FloatField() 158 | volume = models.IntegerField() 159 | note = models.OneToOneField(TradeLogNote, on_delete=models.CASCADE) 160 | 161 | objects = DataFrameManager() 162 | 163 | def __str__(self): 164 | return "{0}-{1}-{2}-{3}-{4}-{5}".format( 165 | self.trader, 166 | self.symbol, 167 | self.log_datetime, 168 | self.price, 169 | self.volume, 170 | self.note 171 | ) 172 | 173 | @python_2_unicode_compatible 174 | class Portfolio(models.Model): 175 | name = models.CharField(max_length=20) 176 | securities = models.ManyToManyField(Security) 177 | 178 | def __str__(self): 179 | return self.name 180 | 181 | 182 | class DudeQuerySet(models.query.QuerySet): 183 | def abiding(self): 184 | return self.filter(abides=True) 185 | 186 | def rug_positive(self): 187 | return self.filter(has_rug=True) 188 | 189 | def rug_negative(self): 190 | return self.filter(has_rug=False) 191 | 192 | def by_name(self, name): 193 | return self.filter(name__iexact=name) 194 | 195 | 196 | class AbidingManager(PassThroughManager): 197 | def get_queryset(self): 198 | return DudeQuerySet(self.model).abiding() 199 | 200 | get_query_set = get_queryset 201 | 202 | def get_stats(self): 203 | return { 204 | "abiding_count": self.count(), 205 | "rug_count": self.rug_positive().count(), 206 | } 207 | 208 | 209 | class Dude(models.Model): 210 | abides = models.BooleanField(default=True) 211 | name = models.CharField(max_length=20) 212 | has_rug = models.BooleanField(default=False) 213 | 214 | objects = PassThroughManager(DudeQuerySet) 215 | abiders = AbidingManager() 216 | 217 | 218 | class Car(models.Model): 219 | name = models.CharField(max_length=20) 220 | owner = models.ForeignKey(Dude, related_name='cars_owned', on_delete=models.CASCADE) 221 | 222 | objects = PassThroughManager(DudeQuerySet) 223 | 224 | 225 | class SpotManager(PassThroughManager): 226 | def get_queryset(self): 227 | return super(SpotManager, self).get_queryset().filter(secret=False) 228 | 229 | get_query_set = get_queryset 230 | 231 | 232 | class SpotQuerySet(models.query.QuerySet): 233 | def closed(self): 234 | return self.filter(closed=True) 235 | 236 | def secured(self): 237 | return self.filter(secure=True) 238 | 239 | 240 | class Spot(models.Model): 241 | name = models.CharField(max_length=20) 242 | secure = models.BooleanField(default=True) 243 | closed = models.BooleanField(default=False) 244 | secret = models.BooleanField(default=False) 245 | owner = models.ForeignKey(Dude, related_name='spots_owned', on_delete=models.CASCADE) 246 | 247 | objects = SpotManager.for_queryset_class(SpotQuerySet)() 248 | -------------------------------------------------------------------------------- /django_pandas/tests/test_io.py: -------------------------------------------------------------------------------- 1 | from django.core.paginator import Paginator 2 | from django.test import TestCase 3 | import django 4 | from django.db.models import Sum 5 | import pandas as pd 6 | import numpy as np 7 | from .models import (MyModel, Trader, Security, TradeLog, TradeLogNote, 8 | MyModelChoice, Portfolio) 9 | from django_pandas.io import read_frame 10 | 11 | 12 | class IOTest(TestCase): 13 | 14 | def setUp(self): 15 | data = { 16 | 'col1': np.array([1, 2, 3, 5, 6, 5, 5]), 17 | 'col2': np.array([10.0, 2.4, 3.0, 5, 6, np.nan, 5]), 18 | 'col3': np.array([9.5, 2.4, 3.0, 5, 6, 7.5, 2.5]), 19 | 'col4': np.array([9, 2, 3, 5, 6, 7, 2]), 20 | } 21 | index = pd.Index(['a', 'b', 'c', 'd', 'e', 'f', 'h']) 22 | 23 | self.df = pd.DataFrame(index=index, data=data) 24 | 25 | for ix, cols in self.df.iterrows(): 26 | MyModel.objects.create( 27 | index_col=ix, 28 | col1=cols['col1'], 29 | col2=cols['col2'], 30 | col3=cols['col3'], 31 | col4=cols['col4'] 32 | ) 33 | 34 | def test_basic(self): 35 | qs = MyModel.objects.all() 36 | df = read_frame(qs) 37 | n, c = df.shape 38 | self.assertEqual(n, qs.count()) 39 | from itertools import chain 40 | if django.VERSION < (1, 10): 41 | fields = MyModel._meta.get_all_field_names() 42 | else: 43 | fields = list(set(chain.from_iterable((field.name, field.attname) if hasattr(field, 'attname') else (field.name,) 44 | for field in MyModel._meta.get_fields() 45 | if not (field.many_to_one and field.related_model is None) 46 | ))) 47 | self.assertEqual(c, len(fields)) 48 | df1 = read_frame(qs, ['col1', 'col2']) 49 | self.assertEqual(df1.shape, (qs.count(), 2)) 50 | 51 | def test_page(self): 52 | qs = MyModel.objects.all() 53 | qs_list: list = Paginator(qs, 3).page(1).object_list 54 | df = read_frame(qs_list, verbose=False) 55 | self.assertEqual(list(df.columns), 56 | ['id', 'index_col', 'col1', 'col2', 'col3', 'col4']) 57 | 58 | df = read_frame(qs_list, verbose=False, fieldnames=['col1', 'col2']) 59 | self.assertEqual(list(df.columns), 60 | ['col1', 'col2']) 61 | 62 | def test_values(self): 63 | qs = MyModel.objects.all() 64 | qs = qs.extra(select={"ecol1": "col1+1"}) 65 | qs = qs.values("index_col", "ecol1", "col1") 66 | qs = qs.annotate(scol1=Sum("col1")) 67 | df = read_frame(qs) 68 | self.assertEqual(list(df.columns), 69 | ['index_col', 'col1', 'scol1', 'ecol1']) 70 | self.assertEqual(list(df["col1"]), list(df["scol1"])) 71 | 72 | def test_override_column_names(self): 73 | qs = MyModel.objects.all() 74 | df = read_frame( 75 | qs, 76 | index_col='id', 77 | fieldnames=['col1', 'col2', 'col3', 'col4'], 78 | column_names=['a', 'b', 'c', 'd'] 79 | ) 80 | self.assertEqual(list(df.columns), ['a', 'b', 'c', 'd']) 81 | 82 | def test_duplicate_annotation(self): 83 | qs = MyModel.objects.all() 84 | qs = qs.values('index_col') 85 | qs = qs.annotate(col1=Sum('col1')) 86 | qs = qs.values() 87 | df = read_frame(qs) 88 | self.assertEqual(list(df.columns), 89 | ['id', 'index_col', 'col1', 'col2', 'col3', 'col4']) 90 | 91 | def test_choices(self): 92 | 93 | MyModelChoice.objects.create(col1=1, col2=9999.99) 94 | MyModelChoice.objects.create(col1=2, col2=0.99) 95 | MyModelChoice.objects.create(col1=3, col2=45.6) 96 | MyModelChoice.objects.create(col1=2, col2=2.6) 97 | 98 | qs = MyModelChoice.objects.all() 99 | df = read_frame(qs, verbose=True) 100 | self.assertEqual(df.col1[0], u'First \U0001f947') 101 | self.assertEqual(df.col1[1], u'Second \U0001f948') 102 | self.assertEqual(df.col1[2], u'Third \U0001f949') 103 | self.assertEqual(df.col1[3], u'Second \U0001f948') 104 | df = read_frame(qs, verbose=False) 105 | self.assertEqual(df.col1[0], 1) 106 | self.assertEqual(df.col1[1], 2) 107 | self.assertEqual(df.col1[2], 3) 108 | self.assertEqual(df.col1[3], 2) 109 | 110 | def test_index(self): 111 | qs = MyModel.objects.all() 112 | df = read_frame(qs, ['col1', 'col2', 'col3', 'col4'], 113 | index_col='index_col') 114 | self.assertEqual(df.shape, (qs.count(), 4)) 115 | self.assertEqual(set(df.index.tolist()), 116 | set(qs.values_list('index_col', flat=True))) 117 | 118 | 119 | class RelatedFieldsTest(TestCase): 120 | def setUp(self): 121 | bob = Trader.objects.create(name="Jim Brown") 122 | fish = Trader.objects.create(name="Fred Fish") 123 | abc = Security.objects.create(symbol='ABC', isin='999901') 124 | zyz = Security.objects.create(symbol='ZYZ', isin='999907') 125 | TradeLog.objects.create(trader=bob, symbol=None, 126 | log_datetime='2013-01-01T09:30:00', 127 | price=30, volume=300, 128 | note=TradeLogNote.objects.create(note='aaa')) 129 | TradeLog.objects.create(trader=bob, symbol=None, 130 | log_datetime='2013-01-01T10:00:00', 131 | price=30, volume=300, 132 | note=TradeLogNote.objects.create(note='aab')) 133 | TradeLog.objects.create(trader=bob, symbol=abc, 134 | log_datetime='2013-01-01T10:30:00', 135 | price=30, volume=300, 136 | note=TradeLogNote.objects.create(note='aac')) 137 | TradeLog.objects.create(trader=bob, symbol=abc, 138 | log_datetime='2013-01-01T11:00:00', 139 | price=30, volume=300, 140 | note=TradeLogNote.objects.create(note='aad')) 141 | TradeLog.objects.create(trader=fish, symbol=zyz, 142 | log_datetime='2013-01-01T09:30:00', 143 | price=30, volume=300, 144 | note=TradeLogNote.objects.create(note='aae')) 145 | TradeLog.objects.create(trader=fish, symbol=zyz, 146 | log_datetime='2013-01-01T10:00:00', 147 | price=30, volume=300, 148 | note=TradeLogNote.objects.create(note='aaf')) 149 | TradeLog.objects.create(trader=fish, symbol=zyz, 150 | log_datetime='2013-01-01T10:30:00', 151 | price=30, volume=300, 152 | note=TradeLogNote.objects.create(note='aag')) 153 | TradeLog.objects.create(trader=fish, symbol=zyz, 154 | log_datetime='2013-01-01T11:00:00', 155 | price=30, volume=300, 156 | note=TradeLogNote.objects.create(note='aah')) 157 | value = Portfolio.objects.create(name="Fund 1") 158 | value.securities.add(abc) 159 | value.securities.add(zyz) 160 | growth = Portfolio.objects.create(name="Fund 2") 161 | growth.securities.add(abc) 162 | 163 | def test_verbose(self): 164 | qs = TradeLog.objects.all() 165 | df = read_frame(qs, verbose=True) 166 | self.assertListEqual( 167 | list(qs.values_list('trader__name', flat=True)), 168 | df.trader.tolist() 169 | ) 170 | df1 = read_frame(qs, verbose=False) 171 | self.assertListEqual( 172 | list(qs.values_list('trader__pk', flat=True)), 173 | df1.trader.tolist() 174 | ) 175 | 176 | # Testing verbose with annotated column: 177 | if django.VERSION >= (1, 10): 178 | from django.db.models import F, FloatField 179 | from django.db.models.functions import Cast 180 | qs1 = TradeLog.objects.all().annotate( 181 | total_sum=Cast(F('price') * F('volume'), FloatField()), 182 | ) 183 | df2 = read_frame( 184 | qs1, fieldnames=['trader', 'total_sum']) 185 | self.assertListEqual( 186 | list(qs1.values_list('total_sum', flat=True)), 187 | df2.total_sum.tolist() 188 | ) 189 | self.assertListEqual( 190 | list(qs1.values_list('trader__name', flat=True)), 191 | df2.trader.tolist() 192 | ) 193 | 194 | def test_verbose_duplicates_fieldnames(self): 195 | qs = TradeLog.objects.all() 196 | df = read_frame(qs, fieldnames=['trader', 'trader', 'price']) 197 | self.assertListEqual( 198 | list(qs.values_list('price', flat=True)), 199 | df.price.tolist() 200 | ) 201 | 202 | def test_verbose_duplicate_values(self): 203 | qs = TradeLog.objects.all() 204 | qs = qs.values('trader', 'trader', 'price') 205 | df = read_frame(qs) 206 | self.assertListEqual( 207 | list(qs.values_list('price', flat=True)), 208 | df.price.tolist() 209 | ) 210 | 211 | def test_related_selected_field(self): 212 | qs = TradeLog.objects.all().values('trader__name') 213 | df = read_frame(qs) 214 | self.assertEqual(list(df.columns), ['trader__name']) 215 | 216 | def test_related_cols(self): 217 | qs = TradeLog.objects.all() 218 | cols = ['log_datetime', 'symbol', 'symbol__isin', 'trader__name', 219 | 'price', 'volume', 'note__note'] 220 | df = read_frame(qs, cols, verbose=False) 221 | 222 | self.assertEqual(df.shape, (qs.count(), len(cols))) 223 | self.assertListEqual( 224 | list(qs.values_list('symbol__isin', flat=True)), 225 | df.symbol__isin.tolist() 226 | ) 227 | self.assertListEqual( 228 | list(qs.values_list('trader__name', flat=True)), 229 | df.trader__name.tolist() 230 | ) 231 | 232 | def test_many_to_many(self): 233 | qs = Portfolio.objects.all() 234 | cols = ['name', 'securities__symbol', 'securities__tradelog__log_datetime'] 235 | df = read_frame(qs, cols, verbose=True) 236 | 237 | denormalized = Portfolio.objects.all().values_list(*cols) 238 | self.assertEqual(df.shape, (len(denormalized), len(cols))) 239 | for idx, row in enumerate(denormalized): 240 | self.assertListEqual( 241 | df.iloc[idx].tolist(), 242 | list(row) 243 | ) 244 | -------------------------------------------------------------------------------- /django_pandas/tests/test_manager.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | 3 | from django.test import TestCase 4 | import pandas as pd 5 | import numpy as np 6 | import pickle 7 | import django 8 | from pandas.core.indexes.datetimes import bdate_range 9 | 10 | from .models import ( 11 | DataFrame, WideTimeSeries, WideTimeSeriesDateField, 12 | LongTimeSeries, PivotData, Dude, Car, Spot 13 | ) 14 | try: 15 | import pandas._testing as tm 16 | except ImportError: 17 | import pandas.util.testing as tm 18 | 19 | import semver 20 | 21 | PANDAS_VERSIONINFO = semver.VersionInfo.parse(pd.__version__) 22 | 23 | class DataFrameTest(TestCase): 24 | 25 | def setUp(self): 26 | data = { 27 | 'col1': np.array([1, 2, 3, 5, 6, 5, 5]), 28 | 'col2': np.array([10.0, 2.4, 3.0, 5, 6, 5, 5]), 29 | 'col3': np.array([9.5, 2.4, 3.0, 5, 6, 7.5, 2.5]), 30 | 'col4': np.array([9, 2, 3, 5, 6, 7, 2]), 31 | } 32 | index = pd.Index(['a', 'b', 'c', 'd', 'e', 'f', 'h']) 33 | 34 | self.df = pd.DataFrame(index=index, data=data) 35 | 36 | for ix, cols in self.df.iterrows(): 37 | DataFrame.objects.create( 38 | index=ix, 39 | col1=cols['col1'], 40 | col2=cols['col2'], 41 | col3=cols['col3'], 42 | col4=cols['col4'] 43 | ) 44 | 45 | def test_dataframe(self): 46 | qs = DataFrame.objects.all() 47 | df = qs.to_dataframe() 48 | 49 | n, c = df.shape 50 | self.assertEqual(n, qs.count()) 51 | from itertools import chain 52 | if django.VERSION < (1, 10): 53 | flds = DataFrame._meta.get_all_field_names() 54 | else: 55 | flds = list(set(chain.from_iterable((field.name, field.attname) 56 | if hasattr(field, 'attname') else (field.name,) 57 | for field in DataFrame._meta.get_fields() 58 | if not (field.many_to_one and 59 | field.related_model is None)))) 60 | self.assertEqual(c, len(flds)) 61 | qs2 = DataFrame.objects.filter(index__in=['a', 'b', 'c']) 62 | df2 = qs2.to_dataframe(['col1', 'col2', 'col3'], index='index') 63 | n, c = df2.shape 64 | self.assertEqual((n, c), (3, 3)) 65 | 66 | 67 | class TimeSeriesTest(TestCase): 68 | def unpivot(self, frame): 69 | N, K = frame.shape 70 | data = {'value': frame.values.ravel('F'), 71 | 'variable': np.array(frame.columns).repeat(N), 72 | 'date': np.tile(np.array(frame.index), K)} 73 | return pd.DataFrame(data, columns=['date', 'variable', 'value']) 74 | 75 | def _makeTimeDataFrame(self, n_rows: int) -> pd.DataFrame: 76 | # Beginning in 2.2 pandas._testing.makeTimeDataFrame was removed, however all that is required for the tests 77 | # in this module is a dataframe with columns A, B, C, D of random values indexed by a DatetimeIndex. 78 | data = {} 79 | for c in ['A', 'B', 'C', 'D']: 80 | dt = datetime(2000, 1, 1) 81 | dr = bdate_range(dt, periods=n_rows, freq='B', name=c) 82 | pd.DatetimeIndex(dr, name=c) 83 | 84 | data[c] = pd.Series( 85 | np.random.default_rng(2).standard_normal(n_rows), 86 | index=pd.DatetimeIndex(dr, name=c), 87 | name=c, 88 | ) 89 | return pd.DataFrame(data) 90 | 91 | def setUp(self): 92 | if PANDAS_VERSIONINFO >= '2.2.0': 93 | self.ts = self._makeTimeDataFrame(100) 94 | else: 95 | self.ts = tm.makeTimeDataFrame(100) 96 | 97 | self.ts2 = self.unpivot(self.ts).set_index('date') 98 | self.ts.columns = ['col1', 'col2', 'col3', 'col4'] 99 | create_list = [] 100 | for ix, cols in self.ts.iterrows(): 101 | create_list.append(WideTimeSeries(date_ix=ix, col1=cols['col1'], 102 | col2=cols['col2'], 103 | col3=cols['col3'], 104 | col4=cols['col4'])) 105 | WideTimeSeries.objects.bulk_create(create_list) 106 | 107 | for ix, cols in self.ts.iterrows(): 108 | create_list.append(WideTimeSeriesDateField(date_ix=ix, col1=cols['col1'], 109 | col2=cols['col2'], 110 | col3=cols['col3'], 111 | col4=cols['col4'])) 112 | WideTimeSeriesDateField.objects.bulk_create(create_list) 113 | 114 | create_list = [LongTimeSeries(date_ix=timestamp, series_name=s.iloc[0], 115 | value=s.iloc[1]) 116 | for timestamp, s in self.ts2.iterrows()] 117 | 118 | LongTimeSeries.objects.bulk_create(create_list) 119 | 120 | def test_widestorage(self): 121 | 122 | qs = WideTimeSeries.objects.all() 123 | 124 | df = qs.to_timeseries(index='date_ix', storage='wide') 125 | 126 | self.assertEqual(df.shape, (qs.count(), 5)) 127 | self.assertIsInstance(df.index, pd.DatetimeIndex) 128 | self.assertIsNone(df.index.freq) 129 | 130 | def test_widestorage_datefield(self): 131 | 132 | qs = WideTimeSeriesDateField.objects.all() 133 | 134 | df = qs.to_timeseries(index='date_ix', storage='wide') 135 | 136 | self.assertIsInstance(df.index, pd.DatetimeIndex) 137 | 138 | def test_longstorage(self): 139 | qs = LongTimeSeries.objects.all() 140 | df = qs.to_timeseries(index='date_ix', pivot_columns='series_name', 141 | values='value', 142 | storage='long') 143 | self.assertEqual(set(qs.values_list('series_name', flat=True)), 144 | set(df.columns.tolist())) 145 | 146 | self.assertEqual(qs.filter(series_name='A').count(), len(df['A'])) 147 | self.assertIsInstance(df.index, pd.DatetimeIndex) 148 | self.assertIsNone(df.index.freq) 149 | 150 | def test_resampling(self): 151 | qs = LongTimeSeries.objects.all() 152 | agg_args = None 153 | agg_kwargs = None 154 | if PANDAS_VERSIONINFO >= '0.25.0': 155 | agg_kwargs = {'func': 'sum'} 156 | else: 157 | agg_args = ['sum'] 158 | 159 | if PANDAS_VERSIONINFO >= '2.2.0': 160 | freq = 'ME' 161 | else: 162 | freq = 'M' 163 | 164 | df = qs.to_timeseries(index='date_ix', pivot_columns='series_name', 165 | values='value', storage='long', 166 | freq=freq, 167 | agg_args=agg_args, 168 | agg_kwargs=agg_kwargs) 169 | df.index = pd.PeriodIndex(df.index) 170 | 171 | self.assertEqual([d.month for d in qs.dates('date_ix', 'month')], 172 | df.index.month.tolist()) 173 | 174 | self.assertIsInstance(df.index, pd.PeriodIndex) 175 | #try on a wide time seriesd 176 | 177 | qs2 = WideTimeSeries.objects.all() 178 | 179 | df1 = qs2.to_timeseries(index='date_ix', storage='wide', 180 | freq=freq, 181 | agg_args=agg_args, 182 | agg_kwargs=agg_kwargs) 183 | df1.index = pd.PeriodIndex(df1.index) 184 | 185 | self.assertEqual([d.month for d in qs.dates('date_ix', 'month')], 186 | df1.index.month.tolist()) 187 | 188 | self.assertIsInstance(df1.index, pd.PeriodIndex) 189 | 190 | def test_bad_args_wide_ts(self): 191 | qs = WideTimeSeries.objects.all() 192 | rs_kwargs = {'how': 'sum', 'kind': 'period'} 193 | kwargs = { 194 | 'fieldnames': ['col1', 'col2'], 195 | 'freq': 'M', 'rs_kwargs': rs_kwargs 196 | } 197 | self.assertRaises(AssertionError, qs.to_timeseries, **kwargs) 198 | kwargs2 = { 199 | 'index': 'date_ix', 200 | 'fieldnames': ['col1', 'col2'], 201 | 'storage': 'big', 202 | 'freq': 'M', 'rs_kwargs': rs_kwargs 203 | } 204 | self.assertRaises(AssertionError, qs.to_timeseries, **kwargs2) 205 | 206 | def test_bad_args_long_ts(self): 207 | qs = LongTimeSeries.objects.all() 208 | kwargs = { 209 | 'index': 'date_ix', 210 | 'pivot_columns': 'series_name', 211 | 'values': 'value', 212 | 'storage': 'long'} 213 | kwargs.pop('values') 214 | self.assertRaises(AssertionError, qs.to_timeseries, **kwargs) 215 | kwargs['values'] = 'value' 216 | kwargs.pop('pivot_columns') 217 | self.assertRaises(AssertionError, qs.to_timeseries, **kwargs) 218 | # df = qs.to_timeseries(index='date_ix', pivot_columns='series_name', 219 | # values='value', 220 | # storage='long') 221 | 222 | def test_coerce_float(self): 223 | qs = LongTimeSeries.objects.all() 224 | ts = qs.to_timeseries(index='date_ix', 225 | coerce_float=True).resample('D').sum() 226 | self.assertEqual(ts['value'].dtype, np.float64) 227 | 228 | # Testing on Wide Series 229 | 230 | qs = WideTimeSeries.objects.all() 231 | ts = qs.to_timeseries(index='date_ix', 232 | coerce_float=True).resample('D').sum() 233 | self.assertEqual(ts['col1'].dtype, np.float64) 234 | self.assertEqual(ts['col2'].dtype, np.float64) 235 | self.assertEqual(ts['col3'].dtype, np.float64) 236 | self.assertEqual(ts['col4'].dtype, np.float64) 237 | 238 | 239 | class PivotTableTest(TestCase): 240 | 241 | def setUp(self): 242 | self.data = pd.DataFrame({'row_col_a': ['foo', 'foo', 'foo', 'foo', 243 | 'bar', 'bar', 'bar', 'bar', 244 | 'foo', 'foo', 'foo'], 245 | 'row_col_b': ['one', 'one', 'one', 'two', 246 | 'one', 'one', 'one', 'two', 247 | 'two', 'two', 'one'], 248 | 'row_col_c': ['dull', 'dull', 249 | 'shiny', 'dull', 250 | 'dull', 'shiny', 251 | 'shiny', 'dull', 252 | 'shiny', 'shiny', 'shiny'], 253 | 'value_col_d': np.random.randn(11), 254 | 'value_col_e': np.random.randn(11), 255 | 'value_col_f': np.random.randn(11)}) 256 | create_list = [PivotData(row_col_a=r.iloc[0], row_col_b=r.iloc[1], 257 | row_col_c=r.iloc[2], value_col_d=r.iloc[3], 258 | value_col_e=r.iloc[4], value_col_f=r.iloc[5]) 259 | for _, r in self.data.iterrows()] 260 | 261 | PivotData.objects.bulk_create(create_list) 262 | 263 | def test_pivot(self): 264 | qs = PivotData.objects.all() 265 | rows = ['row_col_a', 'row_col_b'] 266 | cols = ['row_col_c'] 267 | 268 | pt = qs.to_pivot_table(values='value_col_d', rows=rows, cols=cols) 269 | self.assertEqual(pt.index.names, rows) 270 | self.assertEqual(pt.columns.names, cols) 271 | 272 | 273 | if django.VERSION < (1, 9): 274 | 275 | class PassThroughManagerTests(TestCase): 276 | 277 | def setUp(self): 278 | Dude.objects.create(name='The Dude', abides=True, has_rug=False) 279 | Dude.objects.create(name='His Dudeness', 280 | abides=False, has_rug=True) 281 | Dude.objects.create(name='Duder', abides=False, has_rug=False) 282 | Dude.objects.create(name='El Duderino', abides=True, has_rug=True) 283 | 284 | def test_chaining(self): 285 | self.assertEqual(Dude.objects.by_name('Duder').count(), 1) 286 | self.assertEqual(Dude.objects.all().by_name('Duder').count(), 1) 287 | self.assertEqual(Dude.abiders.rug_positive().count(), 1) 288 | self.assertEqual(Dude.abiders.all().rug_positive().count(), 1) 289 | 290 | def test_manager_only_methods(self): 291 | stats = Dude.abiders.get_stats() 292 | self.assertEqual(stats['rug_count'], 1) 293 | with self.assertRaises(AttributeError): 294 | Dude.abiders.all().get_stats() 295 | 296 | def test_queryset_pickling(self): 297 | qs = Dude.objects.all() 298 | saltyqs = pickle.dumps(qs) 299 | unqs = pickle.loads(saltyqs) 300 | self.assertEqual(unqs.by_name('The Dude').count(), 1) 301 | 302 | def test_queryset_not_available_on_related_manager(self): 303 | dude = Dude.objects.by_name('Duder').get() 304 | Car.objects.create(name='Ford', owner=dude) 305 | self.assertFalse(hasattr(dude.cars_owned, 'by_name')) 306 | 307 | def test_using_dir(self): 308 | # make sure introspecing via dir() doesn't actually cause queries, 309 | # just as a sanity check. 310 | with self.assertNumQueries(0): 311 | querysets_to_dir = ( 312 | Dude.objects, 313 | Dude.objects.by_name('Duder'), 314 | Dude.objects.all().by_name('Duder'), 315 | Dude.abiders, 316 | Dude.abiders.rug_positive(), 317 | Dude.abiders.all().rug_positive() 318 | ) 319 | for qs in querysets_to_dir: 320 | self.assertTrue('by_name' in dir(qs)) 321 | self.assertTrue('abiding' in dir(qs)) 322 | self.assertTrue('rug_positive' in dir(qs)) 323 | self.assertTrue('rug_negative' in dir(qs)) 324 | # some standard qs methods 325 | self.assertTrue('count' in dir(qs)) 326 | self.assertTrue('order_by' in dir(qs)) 327 | self.assertTrue('select_related' in dir(qs)) 328 | # make sure it's been de-duplicated 329 | self.assertEqual(1, dir(qs).count('distinct')) 330 | 331 | # manager only method. 332 | self.assertTrue('get_stats' in dir(Dude.abiders)) 333 | # manager only method shouldn't appear on the nonAbidingManager 334 | self.assertFalse('get_stats' in dir(Dude.objects)) 335 | # standard manager methods 336 | self.assertTrue('get_query_set' in dir(Dude.abiders)) 337 | self.assertTrue('contribute_to_class' in dir(Dude.abiders)) 338 | 339 | class CreatePassThroughManagerTests(TestCase): 340 | 341 | def setUp(self): 342 | self.dude = Dude.objects.create(name='El Duderino') 343 | self.other_dude = Dude.objects.create(name='Das Dude') 344 | 345 | def test_reverse_manager(self): 346 | Spot.objects.create( 347 | name='The Crib', owner=self.dude, closed=True, secure=True, 348 | secret=False) 349 | self.assertEqual(self.dude.spots_owned.closed().count(), 1) 350 | Spot.objects.create( 351 | name='The Crux', owner=self.other_dude, 352 | closed=True, secure=True, 353 | secret=False 354 | ) 355 | self.assertEqual(self.dude.spots_owned.closed().all().count(), 1) 356 | self.assertEqual(self.dude.spots_owned.closed().count(), 1) 357 | 358 | def test_related_queryset_pickling(self): 359 | Spot.objects.create( 360 | name='The Crib', owner=self.dude, closed=True, secure=True, 361 | secret=False) 362 | qs = self.dude.spots_owned.closed() 363 | pickled_qs = pickle.dumps(qs) 364 | unpickled_qs = pickle.loads(pickled_qs) 365 | self.assertEqual(unpickled_qs.secured().count(), 1) 366 | 367 | def test_related_queryset_superclass_method(self): 368 | Spot.objects.create( 369 | name='The Crib', owner=self.dude, closed=True, secure=True, 370 | secret=False) 371 | Spot.objects.create( 372 | name='The Secret Crib', owner=self.dude, 373 | closed=False, secure=True, 374 | secret=True) 375 | self.assertEqual(self.dude.spots_owned.count(), 1) 376 | 377 | def test_related_manager_create(self): 378 | self.dude.spots_owned.create(name='The Crib', 379 | closed=True, secure=True) 380 | -------------------------------------------------------------------------------- /django_pandas/tests/tests.py: -------------------------------------------------------------------------------- 1 | from .test_manager import * 2 | from .test_io import * 3 | -------------------------------------------------------------------------------- /django_pandas/utils.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | import sys 3 | 4 | from django.core.cache import cache 5 | from django.db.models import Field 6 | 7 | if sys.version_info >= (3, ): 8 | from django.utils.encoding import force_str as force_text 9 | else: 10 | from django.utils.encoding import force_text 11 | 12 | 13 | def get_model_name(model): 14 | """ 15 | Returns the name of the model 16 | """ 17 | return model._meta.model_name 18 | 19 | 20 | def replace_from_choices(choices): 21 | def inner(values): 22 | return [choices.get(v, v) for v in values] 23 | return inner 24 | 25 | 26 | def get_base_cache_key(model): 27 | return 'pandas_%s_%s_%%s_rendering' % ( 28 | model._meta.app_label, get_model_name(model)) 29 | 30 | 31 | def get_cache_key(obj): 32 | return get_base_cache_key(obj._meta.model) % obj.pk 33 | 34 | 35 | def invalidate(obj): 36 | cache.delete(get_cache_key(obj)) 37 | 38 | 39 | def invalidate_signal_handler(sender, **kwargs): 40 | invalidate(kwargs['instance']) 41 | 42 | 43 | def replace_pk(model): 44 | base_cache_key = get_base_cache_key(model) 45 | 46 | def get_cache_key_from_pk(pk): 47 | if pk is None: 48 | return None 49 | else: 50 | try: 51 | return base_cache_key % str(int(pk)) 52 | except: 53 | return base_cache_key % str(pk) 54 | 55 | def inner(pk_series): 56 | pk_series = pk_series.astype(object).where(pk_series.notnull(), None) 57 | cache_keys = pk_series.apply(get_cache_key_from_pk) 58 | unique_cache_keys = list(filter(None, cache_keys.unique())) 59 | 60 | if not unique_cache_keys: 61 | return pk_series 62 | 63 | out_dict = cache.get_many(unique_cache_keys) 64 | 65 | if len(out_dict) < len(unique_cache_keys): 66 | out_dict = dict([(base_cache_key % obj.pk, force_text(obj)) 67 | for obj in model.objects.filter( 68 | pk__in=list(filter(None, pk_series.unique())))]) 69 | cache.set_many(out_dict) 70 | 71 | return list(map(out_dict.get, cache_keys)) 72 | 73 | return inner 74 | 75 | 76 | def build_update_functions(fieldnames, fields): 77 | for fieldname, field in zip(fieldnames, fields): 78 | if not isinstance(field, Field): 79 | yield fieldname, None 80 | else: 81 | if field and field.choices: 82 | choices = dict([(k, force_text(v)) 83 | for k, v in field.flatchoices]) 84 | yield fieldname, replace_from_choices(choices) 85 | 86 | elif field and field.get_internal_type() == 'ForeignKey': 87 | yield fieldname, replace_pk(get_related_model(field)) 88 | 89 | 90 | def update_with_verbose(df, fieldnames, fields): 91 | for fieldname, function in build_update_functions(fieldnames, fields): 92 | if function is not None: 93 | df[fieldname] = function(df[fieldname]) 94 | 95 | 96 | def get_related_model(field): 97 | """Gets the related model from a related field""" 98 | model = None 99 | 100 | if hasattr(field, 'related_model') and field.related_model: # pragma: no cover 101 | model = field.related_model 102 | # Django<1.8 doesn't have the related_model API, so we need to use rel, 103 | # which was removed in Django 2.0 104 | elif hasattr(field, 'rel') and field.rel: # pragma: no cover 105 | model = field.rel.to 106 | 107 | return model 108 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # Internal variables. 11 | PAPEROPT_a4 = -D latex_paper_size=a4 12 | PAPEROPT_letter = -D latex_paper_size=letter 13 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 14 | # the i18n builder cannot share the environment and doctrees with the others 15 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 16 | 17 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 18 | 19 | help: 20 | @echo "Please use \`make ' where is one of" 21 | @echo " html to make standalone HTML files" 22 | @echo " dirhtml to make HTML files named index.html in directories" 23 | @echo " singlehtml to make a single large HTML file" 24 | @echo " pickle to make pickle files" 25 | @echo " json to make JSON files" 26 | @echo " htmlhelp to make HTML files and a HTML help project" 27 | @echo " qthelp to make HTML files and a qthelp project" 28 | @echo " devhelp to make HTML files and a Devhelp project" 29 | @echo " epub to make an epub" 30 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 31 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 32 | @echo " text to make text files" 33 | @echo " man to make manual pages" 34 | @echo " texinfo to make Texinfo files" 35 | @echo " info to make Texinfo files and run them through makeinfo" 36 | @echo " gettext to make PO message catalogs" 37 | @echo " changes to make an overview of all changed/added/deprecated items" 38 | @echo " linkcheck to check all external links for integrity" 39 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 40 | 41 | clean: 42 | -rm -rf $(BUILDDIR)/* 43 | 44 | html: 45 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 46 | @echo 47 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 48 | 49 | dirhtml: 50 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 51 | @echo 52 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 53 | 54 | singlehtml: 55 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 56 | @echo 57 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 58 | 59 | pickle: 60 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 61 | @echo 62 | @echo "Build finished; now you can process the pickle files." 63 | 64 | json: 65 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 66 | @echo 67 | @echo "Build finished; now you can process the JSON files." 68 | 69 | htmlhelp: 70 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 71 | @echo 72 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 73 | ".hhp project file in $(BUILDDIR)/htmlhelp." 74 | 75 | qthelp: 76 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 77 | @echo 78 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 79 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 80 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/django-pandas.qhcp" 81 | @echo "To view the help file:" 82 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/django-pandas.qhc" 83 | 84 | devhelp: 85 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 86 | @echo 87 | @echo "Build finished." 88 | @echo "To view the help file:" 89 | @echo "# mkdir -p $$HOME/.local/share/devhelp/django-pandas" 90 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/django-pandas" 91 | @echo "# devhelp" 92 | 93 | epub: 94 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 95 | @echo 96 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 97 | 98 | latex: 99 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 100 | @echo 101 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 102 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 103 | "(use \`make latexpdf' here to do that automatically)." 104 | 105 | latexpdf: 106 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 107 | @echo "Running LaTeX files through pdflatex..." 108 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 109 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 110 | 111 | text: 112 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 113 | @echo 114 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 115 | 116 | man: 117 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 118 | @echo 119 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 120 | 121 | texinfo: 122 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 123 | @echo 124 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 125 | @echo "Run \`make' in that directory to run these through makeinfo" \ 126 | "(use \`make info' here to do that automatically)." 127 | 128 | info: 129 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 130 | @echo "Running Texinfo files through makeinfo..." 131 | make -C $(BUILDDIR)/texinfo info 132 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 133 | 134 | gettext: 135 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 136 | @echo 137 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 138 | 139 | changes: 140 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 141 | @echo 142 | @echo "The overview file is in $(BUILDDIR)/changes." 143 | 144 | linkcheck: 145 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 146 | @echo 147 | @echo "Link check complete; look for any errors in the above output " \ 148 | "or in $(BUILDDIR)/linkcheck/output.txt." 149 | 150 | doctest: 151 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 152 | @echo "Testing of doctests in the sources finished, look at the " \ 153 | "results in $(BUILDDIR)/doctest/output.txt." 154 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # django-pandas documentation build configuration file, created by 4 | # sphinx-quickstart on Mon Aug 5 09:30:44 2013. 5 | # 6 | # This file is execfile()d with the current directory set to its containing dir. 7 | # 8 | # Note that not all possible configuration values are present in this 9 | # autogenerated file. 10 | # 11 | # All configuration values have a default; values that are commented out 12 | # serve to show the default. 13 | 14 | import sys, os 15 | 16 | # If extensions (or modules to document with autodoc) are in another directory, 17 | # add these directories to sys.path here. If the directory is relative to the 18 | # documentation root, use os.path.abspath to make it absolute, like shown here. 19 | #sys.path.insert(0, os.path.abspath('.')) 20 | 21 | # -- General configuration ----------------------------------------------------- 22 | 23 | # If your documentation needs a minimal Sphinx version, state it here. 24 | #needs_sphinx = '1.0' 25 | 26 | # Add any Sphinx extension module names here, as strings. They can be extensions 27 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 28 | extensions = ['sphinx.ext.intersphinx', 'sphinx.ext.viewcode'] 29 | 30 | # Add any paths that contain templates here, relative to this directory. 31 | templates_path = ['_templates'] 32 | 33 | # The suffix of source filenames. 34 | source_suffix = '.rst' 35 | 36 | # The encoding of source files. 37 | #source_encoding = 'utf-8-sig' 38 | 39 | # The master toctree document. 40 | master_doc = 'index' 41 | 42 | # General information about the project. 43 | project = u'django-pandas' 44 | copyright = u'2013, Christopher Clarke' 45 | 46 | # The version info for the project you're documenting, acts as replacement for 47 | # |version| and |release|, also used in various other places throughout the 48 | # built documents. 49 | # 50 | # The short X.Y version. 51 | version = '0.0.2' 52 | # The full version, including alpha/beta/rc tags. 53 | release = '0.0.2' 54 | 55 | # The language for content autogenerated by Sphinx. Refer to documentation 56 | # for a list of supported languages. 57 | #language = None 58 | 59 | # There are two options for replacing |today|: either, you set today to some 60 | # non-false value, then it is used: 61 | #today = '' 62 | # Else, today_fmt is used as the format for a strftime call. 63 | #today_fmt = '%B %d, %Y' 64 | 65 | # List of patterns, relative to source directory, that match files and 66 | # directories to ignore when looking for source files. 67 | exclude_patterns = ['_build'] 68 | 69 | # The reST default role (used for this markup: `text`) to use for all documents. 70 | #default_role = None 71 | 72 | # If true, '()' will be appended to :func: etc. cross-reference text. 73 | #add_function_parentheses = True 74 | 75 | # If true, the current module name will be prepended to all description 76 | # unit titles (such as .. function::). 77 | #add_module_names = True 78 | 79 | # If true, sectionauthor and moduleauthor directives will be shown in the 80 | # output. They are ignored by default. 81 | #show_authors = False 82 | 83 | # The name of the Pygments (syntax highlighting) style to use. 84 | pygments_style = 'sphinx' 85 | 86 | # A list of ignored prefixes for module index sorting. 87 | #modindex_common_prefix = [] 88 | 89 | 90 | # -- Options for HTML output --------------------------------------------------- 91 | 92 | # The theme to use for HTML and HTML Help pages. See the documentation for 93 | # a list of builtin themes. 94 | html_theme = 'alabaster' 95 | 96 | # Theme options are theme-specific and customize the look and feel of a theme 97 | # further. For a list of options available for each theme, see the 98 | # documentation. 99 | #html_theme_options = {} 100 | 101 | # Add any paths that contain custom themes here, relative to this directory. 102 | #html_theme_path = [] 103 | 104 | # The name for this set of Sphinx documents. If None, it defaults to 105 | # " v documentation". 106 | #html_title = None 107 | 108 | # A shorter title for the navigation bar. Default is the same as html_title. 109 | #html_short_title = None 110 | 111 | # The name of an image file (relative to this directory) to place at the top 112 | # of the sidebar. 113 | #html_logo = None 114 | 115 | # The name of an image file (within the static path) to use as favicon of the 116 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 117 | # pixels large. 118 | #html_favicon = None 119 | 120 | # Add any paths that contain custom static files (such as style sheets) here, 121 | # relative to this directory. They are copied after the builtin static files, 122 | # so a file named "default.css" will overwrite the builtin "default.css". 123 | html_static_path = ['_static'] 124 | 125 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 126 | # using the given strftime format. 127 | #html_last_updated_fmt = '%b %d, %Y' 128 | 129 | # If true, SmartyPants will be used to convert quotes and dashes to 130 | # typographically correct entities. 131 | #html_use_smartypants = True 132 | 133 | # Custom sidebar templates, maps document names to template names. 134 | #html_sidebars = {} 135 | 136 | # Additional templates that should be rendered to pages, maps page names to 137 | # template names. 138 | #html_additional_pages = {} 139 | 140 | # If false, no module index is generated. 141 | #html_domain_indices = True 142 | 143 | # If false, no index is generated. 144 | #html_use_index = True 145 | 146 | # If true, the index is split into individual pages for each letter. 147 | #html_split_index = False 148 | 149 | # If true, links to the reST sources are added to the pages. 150 | #html_show_sourcelink = True 151 | 152 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 153 | #html_show_sphinx = True 154 | 155 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 156 | #html_show_copyright = True 157 | 158 | # If true, an OpenSearch description file will be output, and all pages will 159 | # contain a tag referring to it. The value of this option must be the 160 | # base URL from which the finished HTML is served. 161 | #html_use_opensearch = '' 162 | 163 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 164 | #html_file_suffix = None 165 | 166 | # Output file base name for HTML help builder. 167 | htmlhelp_basename = 'django-pandasdoc' 168 | 169 | 170 | # -- Options for LaTeX output -------------------------------------------------- 171 | 172 | latex_elements = { 173 | # The paper size ('letterpaper' or 'a4paper'). 174 | #'papersize': 'letterpaper', 175 | 176 | # The font size ('10pt', '11pt' or '12pt'). 177 | #'pointsize': '10pt', 178 | 179 | # Additional stuff for the LaTeX preamble. 180 | #'preamble': '', 181 | } 182 | 183 | # Grouping the document tree into LaTeX files. List of tuples 184 | # (source start file, target name, title, author, documentclass [howto/manual]). 185 | latex_documents = [ 186 | ('index', 'django-pandas.tex', u'django-pandas Documentation', 187 | u'Christopher Clarke', 'manual'), 188 | ] 189 | 190 | # The name of an image file (relative to this directory) to place at the top of 191 | # the title page. 192 | #latex_logo = None 193 | 194 | # For "manual" documents, if this is true, then toplevel headings are parts, 195 | # not chapters. 196 | #latex_use_parts = False 197 | 198 | # If true, show page references after internal links. 199 | #latex_show_pagerefs = False 200 | 201 | # If true, show URL addresses after external links. 202 | #latex_show_urls = False 203 | 204 | # Documents to append as an appendix to all manuals. 205 | #latex_appendices = [] 206 | 207 | # If false, no module index is generated. 208 | #latex_domain_indices = True 209 | 210 | 211 | # -- Options for manual page output -------------------------------------------- 212 | 213 | # One entry per manual page. List of tuples 214 | # (source start file, name, description, authors, manual section). 215 | man_pages = [ 216 | ('index', 'django-pandas', u'django-pandas Documentation', 217 | [u'Christopher Clarke'], 1) 218 | ] 219 | 220 | # If true, show URL addresses after external links. 221 | #man_show_urls = False 222 | 223 | 224 | # -- Options for Texinfo output ------------------------------------------------ 225 | 226 | # Grouping the document tree into Texinfo files. List of tuples 227 | # (source start file, target name, title, author, 228 | # dir menu entry, description, category) 229 | texinfo_documents = [ 230 | ('index', 'django-pandas', u'django-pandas Documentation', 231 | u'Christopher Clarke', 'django-pandas', 'One line description of project.', 232 | 'Miscellaneous'), 233 | ] 234 | 235 | # Documents to append as an appendix to all manuals. 236 | #texinfo_appendices = [] 237 | 238 | # If false, no module index is generated. 239 | #texinfo_domain_indices = True 240 | 241 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 242 | #texinfo_show_urls = 'footnote' 243 | 244 | 245 | # Example configuration for intersphinx: refer to the Python standard library. 246 | intersphinx_mapping = {'http://docs.python.org/': None} 247 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../README.rst 2 | :end-before: end-here 3 | 4 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=_build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . 10 | set I18NSPHINXOPTS=%SPHINXOPTS% . 11 | if NOT "%PAPER%" == "" ( 12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 14 | ) 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "help" ( 19 | :help 20 | echo.Please use `make ^` where ^ is one of 21 | echo. html to make standalone HTML files 22 | echo. dirhtml to make HTML files named index.html in directories 23 | echo. singlehtml to make a single large HTML file 24 | echo. pickle to make pickle files 25 | echo. json to make JSON files 26 | echo. htmlhelp to make HTML files and a HTML help project 27 | echo. qthelp to make HTML files and a qthelp project 28 | echo. devhelp to make HTML files and a Devhelp project 29 | echo. epub to make an epub 30 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 31 | echo. text to make text files 32 | echo. man to make manual pages 33 | echo. texinfo to make Texinfo files 34 | echo. gettext to make PO message catalogs 35 | echo. changes to make an overview over all changed/added/deprecated items 36 | echo. linkcheck to check all external links for integrity 37 | echo. doctest to run all doctests embedded in the documentation if enabled 38 | goto end 39 | ) 40 | 41 | if "%1" == "clean" ( 42 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 43 | del /q /s %BUILDDIR%\* 44 | goto end 45 | ) 46 | 47 | if "%1" == "html" ( 48 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 49 | if errorlevel 1 exit /b 1 50 | echo. 51 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 52 | goto end 53 | ) 54 | 55 | if "%1" == "dirhtml" ( 56 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 57 | if errorlevel 1 exit /b 1 58 | echo. 59 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 60 | goto end 61 | ) 62 | 63 | if "%1" == "singlehtml" ( 64 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 65 | if errorlevel 1 exit /b 1 66 | echo. 67 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 68 | goto end 69 | ) 70 | 71 | if "%1" == "pickle" ( 72 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 73 | if errorlevel 1 exit /b 1 74 | echo. 75 | echo.Build finished; now you can process the pickle files. 76 | goto end 77 | ) 78 | 79 | if "%1" == "json" ( 80 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 81 | if errorlevel 1 exit /b 1 82 | echo. 83 | echo.Build finished; now you can process the JSON files. 84 | goto end 85 | ) 86 | 87 | if "%1" == "htmlhelp" ( 88 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 89 | if errorlevel 1 exit /b 1 90 | echo. 91 | echo.Build finished; now you can run HTML Help Workshop with the ^ 92 | .hhp project file in %BUILDDIR%/htmlhelp. 93 | goto end 94 | ) 95 | 96 | if "%1" == "qthelp" ( 97 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 98 | if errorlevel 1 exit /b 1 99 | echo. 100 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 101 | .qhcp project file in %BUILDDIR%/qthelp, like this: 102 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\django-pandas.qhcp 103 | echo.To view the help file: 104 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\django-pandas.ghc 105 | goto end 106 | ) 107 | 108 | if "%1" == "devhelp" ( 109 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 110 | if errorlevel 1 exit /b 1 111 | echo. 112 | echo.Build finished. 113 | goto end 114 | ) 115 | 116 | if "%1" == "epub" ( 117 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 118 | if errorlevel 1 exit /b 1 119 | echo. 120 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 121 | goto end 122 | ) 123 | 124 | if "%1" == "latex" ( 125 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 126 | if errorlevel 1 exit /b 1 127 | echo. 128 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 129 | goto end 130 | ) 131 | 132 | if "%1" == "text" ( 133 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 134 | if errorlevel 1 exit /b 1 135 | echo. 136 | echo.Build finished. The text files are in %BUILDDIR%/text. 137 | goto end 138 | ) 139 | 140 | if "%1" == "man" ( 141 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 142 | if errorlevel 1 exit /b 1 143 | echo. 144 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 145 | goto end 146 | ) 147 | 148 | if "%1" == "texinfo" ( 149 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 150 | if errorlevel 1 exit /b 1 151 | echo. 152 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 153 | goto end 154 | ) 155 | 156 | if "%1" == "gettext" ( 157 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 158 | if errorlevel 1 exit /b 1 159 | echo. 160 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 161 | goto end 162 | ) 163 | 164 | if "%1" == "changes" ( 165 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 166 | if errorlevel 1 exit /b 1 167 | echo. 168 | echo.The overview file is in %BUILDDIR%/changes. 169 | goto end 170 | ) 171 | 172 | if "%1" == "linkcheck" ( 173 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 174 | if errorlevel 1 exit /b 1 175 | echo. 176 | echo.Link check complete; look for any errors in the above output ^ 177 | or in %BUILDDIR%/linkcheck/output.txt. 178 | goto end 179 | ) 180 | 181 | if "%1" == "doctest" ( 182 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 183 | if errorlevel 1 exit /b 1 184 | echo. 185 | echo.Testing of doctests in the sources finished, look at the ^ 186 | results in %BUILDDIR%/doctest/output.txt. 187 | goto end 188 | ) 189 | 190 | :end 191 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chrisdev/django-pandas/ed317abcaba924d6205b8f762db65bc5e6fcd81b/pyproject.toml -------------------------------------------------------------------------------- /runtests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | import sys 5 | import django 6 | 7 | from django.conf import settings 8 | 9 | 10 | if not settings.configured: 11 | settings_dict = dict( 12 | INSTALLED_APPS=( 13 | 'django.contrib.contenttypes', 14 | 'django_pandas', 15 | 'django_pandas.tests', 16 | ), 17 | DATABASES={ 18 | "default": { 19 | "ENGINE": "django.db.backends.sqlite3", 20 | "NAME": ":memory:", 21 | "USER": "", 22 | "PASSWORD": "", 23 | "HOST": "", 24 | "PORT": "", 25 | } 26 | }, 27 | MIDDLEWARE_CLASSES=(), 28 | DEFAULT_AUTO_FIELD='django.db.models.AutoField', 29 | ) 30 | 31 | settings.configure(**settings_dict) 32 | if django.VERSION >= (1, 7): 33 | django.setup() 34 | 35 | 36 | def runtests(*test_args): 37 | if not test_args: 38 | test_args = ['django_pandas'] 39 | 40 | parent = os.path.dirname(os.path.abspath(__file__)) 41 | sys.path.insert(0, parent) 42 | 43 | if django.VERSION < (1, 8): 44 | from django.test.simple import DjangoTestSuiteRunner 45 | failures = DjangoTestSuiteRunner( 46 | verbosity=1, interactive=True, failfast=False).run_tests(['tests']) 47 | sys.exit(failures) 48 | 49 | else: 50 | from django.test.runner import DiscoverRunner 51 | failures = DiscoverRunner( 52 | verbosity=1, interactive=True, failfast=False).run_tests(test_args) 53 | sys.exit(failures) 54 | 55 | 56 | if __name__ == '__main__': 57 | runtests() 58 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import codecs 2 | from setuptools import setup, find_packages 3 | 4 | 5 | long_description = ( 6 | codecs.open('README.rst', 'r', 'utf-8').read() + '\n\n' + 7 | codecs.open('CHANGES.rst', 'r', 'utf-8').read() 8 | ) 9 | MAJOR = 0 10 | MINOR = 6 11 | MICRO = 7 12 | 13 | VERSION = '%d.%d.%d' % (MAJOR, MINOR, MICRO) 14 | 15 | setup( 16 | name='django-pandas', 17 | version=VERSION, 18 | description='Tools for working with pydata.pandas in your Django projects', 19 | long_description=long_description, 20 | long_description_content_type='text/x-rst', 21 | author='Christopher Clarke', 22 | author_email='cclarke@chrisdev.com', 23 | url='https://github.com/chrisdev/django-pandas/', 24 | packages=find_packages(), 25 | install_requires=[ 26 | 'pandas>=0.14.1', 27 | 'six>=1.15.0', 28 | ], 29 | classifiers=[ 30 | 'Development Status :: 3 - Alpha', 31 | 'Environment :: Web Environment', 32 | 'Intended Audience :: Developers', 33 | 'License :: OSI Approved :: BSD License', 34 | 'Operating System :: OS Independent', 35 | 'Programming Language :: Python', 36 | 'Programming Language :: Python :: 2.7', 37 | 'Programming Language :: Python :: 3.3', 38 | 'Programming Language :: Python :: 3.4', 39 | 'Programming Language :: Python :: 3.6', 40 | 'Programming Language :: Python :: 3.7', 41 | 'Programming Language :: Python :: 3.8', 42 | 'Programming Language :: Python :: 3.9', 43 | 'Programming Language :: Python :: 3.10', 44 | 'Framework :: Django', 45 | ], 46 | zip_safe=False, 47 | extras_require={ 48 | "test": [ 49 | "pandas>=0.20.1", 50 | "coverage==5.4", 51 | "semver==2.10.1" 52 | ], 53 | }, 54 | ) 55 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = 3 | py3.7-django{30,31,32,40,41,42} 4 | py3.8-django{30,31,32,40,41,42} 5 | py3.9-django{30,31,32,40,41,42} 6 | py3.10-django{30,31,32,40,41,42} 7 | 8 | [testenv] 9 | basepython = 10 | py3.6: python3.6 11 | py3.7: python3.7 12 | py3.8: python3.8 13 | py3.9: python3.9 14 | py3.10: python3.10 15 | py3.11: python3.11 16 | deps = 17 | django30: Django>=3.0,<3.1 18 | django31: Django>=3.1,<3.2 19 | django32: Django>=3.2,<3.3 20 | django40: Django>=4.0,<4.1 21 | django41: Django>=4.1,<4.2 22 | django42: Django>=4.2,<5.0 23 | extras = test 24 | 25 | commands = coverage run -a runtests.py 26 | coverage report -m --------------------------------------------------------------------------------