├── .coveragerc ├── .github └── workflows │ └── tests.yml ├── .gitignore ├── .readthedocs.yaml ├── .travis_old.yml ├── CONTRIBUTORS ├── LICENSE ├── MANIFEST.in ├── README.rst ├── docs ├── Makefile ├── conf.py ├── contributing.rst ├── index.rst ├── installation.rst ├── ref │ ├── class-documentation.rst │ └── method-documentation.rst ├── release_notes.rst ├── toc.rst └── usage.rst ├── manage.py ├── manager_utils ├── __init__.py ├── apps.py ├── manager_utils.py ├── tests │ ├── __init__.py │ ├── manager_utils_tests.py │ └── models.py ├── upsert2.py └── version.py ├── publish.py ├── requirements ├── docs.txt ├── requirements-testing.txt └── requirements.txt ├── run_tests.py ├── settings.py ├── setup.cfg ├── setup.py └── tox_old.ini /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = True 3 | omit = 4 | manager_utils/version.py 5 | source = manager_utils 6 | [report] 7 | exclude_lines = 8 | # Have to re-enable the standard pragma 9 | pragma: no cover 10 | 11 | # Don't complain if tests don't hit defensive assertion code: 12 | raise NotImplementedError 13 | fail_under = 100 14 | show_missing = 1 15 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | # copied from django-cte 2 | name: manager_utils tests 3 | on: 4 | push: 5 | branches: [master] 6 | pull_request: 7 | branches: [master,develop] 8 | 9 | jobs: 10 | tests: 11 | runs-on: ubuntu-latest 12 | strategy: 13 | fail-fast: false 14 | matrix: 15 | python: ['3.7', '3.8', '3.9'] 16 | # Time to switch to pytest or nose2? 17 | # nosetests is broken on 3.10 18 | # AttributeError: module 'collections' has no attribute 'Callable' 19 | # https://github.com/nose-devs/nose/issues/1099 20 | django: 21 | - 'Django~=3.2.0' 22 | - 'Django~=4.0.0' 23 | - 'Django~=4.1.0' 24 | - 'Django~=4.2.0' 25 | experimental: [false] 26 | exclude: 27 | - python: '3.7' 28 | django: 'Django~=4.0.0' 29 | - python: '3.7' 30 | django: 'Django~=4.1.0' 31 | - python: '3.7' 32 | django: 'Django~=4.2.0' 33 | services: 34 | postgres: 35 | image: postgres:latest 36 | env: 37 | POSTGRES_DB: postgres 38 | POSTGRES_PASSWORD: postgres 39 | POSTGRES_USER: postgres 40 | ports: 41 | - 5432:5432 42 | options: >- 43 | --health-cmd pg_isready 44 | --health-interval 10s 45 | --health-timeout 5s 46 | --health-retries 5 47 | steps: 48 | - uses: actions/checkout@v3 49 | - uses: actions/setup-python@v3 50 | with: 51 | python-version: ${{ matrix.python }} 52 | - name: Setup 53 | run: | 54 | python --version 55 | pip install --upgrade pip wheel 56 | pip install -r requirements/requirements.txt 57 | pip install -r requirements/requirements-testing.txt 58 | pip install "${{ matrix.django }}" 59 | pip freeze 60 | - name: Run tests 61 | env: 62 | DB_SETTINGS: >- 63 | { 64 | "ENGINE":"django.db.backends.postgresql", 65 | "NAME":"manager_utils", 66 | "USER":"postgres", 67 | "PASSWORD":"postgres", 68 | "HOST":"localhost", 69 | "PORT":"5432" 70 | } 71 | run: | 72 | coverage run manage.py test manager_utils 73 | coverage report --fail-under=100 74 | continue-on-error: ${{ matrix.experimental }} 75 | - name: Check style 76 | run: flake8 manager_utils 77 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled python files 2 | *.pyc 3 | 4 | .tox/ 5 | 6 | # Vim files 7 | *.swp 8 | *.swo 9 | 10 | # Coverage files 11 | .coverage 12 | htmlcov/ 13 | 14 | # Setuptools distribution folder. 15 | /dist/ 16 | /build/ 17 | 18 | # Python egg metadata, regenerated from source files by setuptools. 19 | /*.egg-info 20 | /*.egg 21 | .eggs/ 22 | 23 | # Virtualenv 24 | env/ 25 | venv/ 26 | 27 | # OSX 28 | .DS_Store 29 | 30 | # Pycharm 31 | .idea/ 32 | 33 | # Documentation artifacts 34 | docs/_build/ 35 | 36 | # IPython Notebook 37 | .ipynb_checkpoints/ 38 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | build: 4 | os: ubuntu-22.04 5 | tools: 6 | python: "3.8" 7 | 8 | sphinx: 9 | configuration: docs/conf.py 10 | 11 | python: 12 | install: 13 | - requirements: requirements/docs.txt 14 | - requirements: requirements/requirements.txt 15 | -------------------------------------------------------------------------------- /.travis_old.yml: -------------------------------------------------------------------------------- 1 | dist: xenial 2 | language: python 3 | sudo: false 4 | 5 | services: 6 | - postgresql 7 | 8 | python: 9 | - "3.7" 10 | - "3.8" 11 | - "3.9" 12 | 13 | env: 14 | global: 15 | - PGPORT=5433 16 | - PGUSER=travis 17 | matrix: 18 | - DJANGO=2.2 19 | - DJANGO=3.0 20 | - DJANGO=3.1 21 | - DJANGO=3.2 22 | - DJANGO=4.0 23 | - DJANGO=4.1 24 | - DJANGO=master 25 | 26 | addons: 27 | postgresql: '13' 28 | apt: 29 | packages: 30 | - postgresql-13 31 | - postgresql-client-13 32 | 33 | matrix: 34 | include: 35 | - { python: "3.7", env: TOXENV=flake8 } 36 | 37 | allow_failures: 38 | - env: DJANGO=master 39 | 40 | install: 41 | - pip install tox-travis 42 | 43 | before_script: 44 | - psql -c 'CREATE DATABASE manager_utils;' -U travis 45 | 46 | script: 47 | - tox 48 | 49 | after_success: 50 | coveralls 51 | 52 | notifications: 53 | email: false 54 | -------------------------------------------------------------------------------- /CONTRIBUTORS: -------------------------------------------------------------------------------- 1 | Wes Kendall @wesleykendall wes.kendall@ambition.com (primary) 2 | Micah Hausler @micahhausler micah.hausler@ambition.com 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Ambition 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.rst 2 | include LICENSE 3 | recursive-include requirements * 4 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | .. image:: https://travis-ci.org/ambitioninc/django-manager-utils.png 2 | :target: https://travis-ci.org/ambitioninc/django-manager-utils 3 | 4 | .. image:: https://img.shields.io/pypi/v/django-manager-utils.svg 5 | :target: https://pypi.python.org/pypi/django-manager-utils/ 6 | :alt: Latest PyPI version 7 | 8 | .. image:: https://img.shields.io/pypi/dm/django-manager-utils.svg 9 | :target: https://pypi.python.org/pypi/django-manager-utils/ 10 | :alt: Number of PyPI downloads 11 | 12 | django-manager-utils 13 | ==================== 14 | Additional utilities for Django model managers. 15 | 16 | Installation 17 | ------------ 18 | To install the latest release, type:: 19 | 20 | pip install django-manager-utils 21 | 22 | To install the latest code directly from source, type:: 23 | 24 | pip install git+git://github.com/ambitioninc/django-manager-utils.git 25 | 26 | Documentation 27 | ------------- 28 | 29 | Full documentation is available at http://django-manager-utils.readthedocs.org 30 | 31 | License 32 | ------- 33 | MIT License (see LICENSE) 34 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 21 | 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp epub latex latexpdf text man changes linkcheck doctest gettext 23 | 24 | help: 25 | @echo "Please use \`make ' where is one of" 26 | @echo " html to make standalone HTML files" 27 | @echo " dirhtml to make HTML files named index.html in directories" 28 | @echo " singlehtml to make a single large HTML file" 29 | @echo " pickle to make pickle files" 30 | @echo " json to make JSON files" 31 | @echo " htmlhelp to make HTML files and a HTML help project" 32 | @echo " epub to make an epub" 33 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 34 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 35 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 36 | @echo " text to make text files" 37 | @echo " man to make manual pages" 38 | @echo " texinfo to make Texinfo files" 39 | @echo " info to make Texinfo files and run them through makeinfo" 40 | @echo " gettext to make PO message catalogs" 41 | @echo " changes to make an overview of all changed/added/deprecated items" 42 | @echo " xml to make Docutils-native XML files" 43 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 44 | @echo " linkcheck to check all external links for integrity" 45 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 46 | 47 | clean: 48 | rm -rf $(BUILDDIR)/* 49 | 50 | html: 51 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 52 | @echo 53 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 54 | 55 | dirhtml: 56 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 57 | @echo 58 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 59 | 60 | singlehtml: 61 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 62 | @echo 63 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 64 | 65 | pickle: 66 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 67 | @echo 68 | @echo "Build finished; now you can process the pickle files." 69 | 70 | json: 71 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 72 | @echo 73 | @echo "Build finished; now you can process the JSON files." 74 | 75 | htmlhelp: 76 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 77 | @echo 78 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 79 | ".hhp project file in $(BUILDDIR)/htmlhelp." 80 | 81 | epub: 82 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 83 | @echo 84 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 85 | 86 | latex: 87 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 88 | @echo 89 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 90 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 91 | "(use \`make latexpdf' here to do that automatically)." 92 | 93 | latexpdf: 94 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 95 | @echo "Running LaTeX files through pdflatex..." 96 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 97 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 98 | 99 | latexpdfja: 100 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 101 | @echo "Running LaTeX files through platex and dvipdfmx..." 102 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 103 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 104 | 105 | text: 106 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 107 | @echo 108 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 109 | 110 | man: 111 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 112 | @echo 113 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 114 | 115 | texinfo: 116 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 117 | @echo 118 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 119 | @echo "Run \`make' in that directory to run these through makeinfo" \ 120 | "(use \`make info' here to do that automatically)." 121 | 122 | info: 123 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 124 | @echo "Running Texinfo files through makeinfo..." 125 | make -C $(BUILDDIR)/texinfo info 126 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 127 | 128 | gettext: 129 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 130 | @echo 131 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 132 | 133 | changes: 134 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 135 | @echo 136 | @echo "The overview file is in $(BUILDDIR)/changes." 137 | 138 | linkcheck: 139 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 140 | @echo 141 | @echo "Link check complete; look for any errors in the above output " \ 142 | "or in $(BUILDDIR)/linkcheck/output.txt." 143 | 144 | doctest: 145 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 146 | @echo "Testing of doctests in the sources finished, look at the " \ 147 | "results in $(BUILDDIR)/doctest/output.txt." 148 | 149 | xml: 150 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 151 | @echo 152 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 153 | 154 | pseudoxml: 155 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 156 | @echo 157 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 158 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Django manager utils documentation build configuration file 4 | 5 | import os 6 | import re 7 | 8 | 9 | def get_version(): 10 | """ 11 | Extracts the version number from the version.py file. 12 | """ 13 | VERSION_FILE = '../manager_utils/version.py' 14 | mo = re.search(r'^__version__ = [\'"]([^\'"]*)[\'"]', open(VERSION_FILE, 'rt').read(), re.M) 15 | if mo: 16 | return mo.group(1) 17 | else: 18 | raise RuntimeError('Unable to find version string in {0}.'.format(VERSION_FILE)) 19 | 20 | # If extensions (or modules to document with autodoc) are in another directory, 21 | # add these directories to sys.path here. If the directory is relative to the 22 | # documentation root, use os.path.abspath to make it absolute, like shown here. 23 | #sys.path.insert(0, os.path.abspath('.')) 24 | 25 | # -- General configuration ------------------------------------------------ 26 | 27 | extensions = [ 28 | 'sphinx.ext.autodoc', 29 | 'sphinx.ext.intersphinx', 30 | 'sphinx.ext.napoleon' 31 | #'sphinx.ext.viewcode', 32 | ] 33 | 34 | # Add any paths that contain templates here, relative to this directory. 35 | templates_path = ['_templates'] 36 | 37 | # The suffix of source filenames. 38 | source_suffix = '.rst' 39 | 40 | # The master toctree document. 41 | master_doc = 'toc' 42 | 43 | # General information about the project. 44 | project = u'Django manager utils' 45 | copyright = u'2014, Ambition Inc.' 46 | 47 | # The short X.Y version. 48 | version = get_version() 49 | # The full version, including alpha/beta/rc tags. 50 | release = version 51 | 52 | exclude_patterns = ['_build'] 53 | 54 | # The name of the Pygments (syntax highlighting) style to use. 55 | pygments_style = 'sphinx' 56 | 57 | intersphinx_mapping = { 58 | 'python': ('http://python.readthedocs.org/en/v2.7.2/', None), 59 | 'django': ('http://django.readthedocs.org/en/latest/', None), 60 | #'celery': ('http://celery.readthedocs.org/en/latest/', None), 61 | } 62 | 63 | # -- Options for HTML output ---------------------------------------------- 64 | 65 | html_theme = 'default' 66 | #html_theme_path = [] 67 | 68 | on_rtd = os.environ.get('READTHEDOCS', None) == 'True' 69 | if not on_rtd: # only import and set the theme if we're building docs locally 70 | import sphinx_rtd_theme 71 | html_theme = 'sphinx_rtd_theme' 72 | html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] 73 | 74 | # Add any paths that contain custom static files (such as style sheets) here, 75 | # relative to this directory. They are copied after the builtin static files, 76 | # so a file named "default.css" will overwrite the builtin "default.css". 77 | html_static_path = ['_static'] 78 | 79 | # Custom sidebar templates, maps document names to template names. 80 | #html_sidebars = {} 81 | 82 | # Additional templates that should be rendered to pages, maps page names to 83 | # template names. 84 | #html_additional_pages = {} 85 | 86 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 87 | html_show_sphinx = False 88 | 89 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 90 | html_show_copyright = True 91 | 92 | # Output file base name for HTML help builder. 93 | htmlhelp_basename = 'django-manager-utilsdoc' 94 | 95 | 96 | # -- Options for LaTeX output --------------------------------------------- 97 | 98 | latex_elements = { 99 | # The paper size ('letterpaper' or 'a4paper'). 100 | #'papersize': 'letterpaper', 101 | 102 | # The font size ('10pt', '11pt' or '12pt'). 103 | #'pointsize': '10pt', 104 | 105 | # Additional stuff for the LaTeX preamble. 106 | #'preamble': '', 107 | } 108 | 109 | # Grouping the document tree into LaTeX files. List of tuples 110 | # (source start file, target name, title, 111 | # author, documentclass [howto, manual, or own class]). 112 | latex_documents = [ 113 | ('index', 'django-manager-utils.tex', u'django-manager-utils Documentation', 114 | u'Wes Kendall', 'manual'), 115 | ] 116 | 117 | # -- Options for manual page output --------------------------------------- 118 | 119 | # One entry per manual page. List of tuples 120 | # (source start file, name, description, authors, manual section). 121 | man_pages = [ 122 | ('index', 'django-manager-utils', u'django-manager-utils Documentation', 123 | [u'Wes Kendall'], 1) 124 | ] 125 | 126 | # -- Options for Texinfo output ------------------------------------------- 127 | 128 | # Grouping the document tree into Texinfo files. List of tuples 129 | # (source start file, target name, title, author, 130 | # dir menu entry, description, category) 131 | texinfo_documents = [ 132 | ('index', 'django-manager-utils', u'django-manager-utils Documentation', 133 | u'Wes Kendall', 'django-manager-utils', 'A short description', 134 | 'Miscellaneous'), 135 | ] 136 | 137 | # -- Django configuration ------------------------------------------------- 138 | import sys 139 | sys.path.insert(0, os.path.abspath('..')) 140 | from settings import configure_settings 141 | configure_settings() 142 | -------------------------------------------------------------------------------- /docs/contributing.rst: -------------------------------------------------------------------------------- 1 | Contributing 2 | ============ 3 | 4 | Contributions and issues are most welcome! All issues and pull requests are 5 | handled through github on the `ambitioninc repository`_. Also, please check for 6 | any existing issues before filing a new one. If you have a great idea but it 7 | involves big changes, please file a ticket before making a pull request! We 8 | want to make sure you don't spend your time coding something that might not fit 9 | the scope of the project. 10 | 11 | .. _ambitioninc repository: https://github.com/ambitioninc/django-manager-utils/issues 12 | 13 | Running the tests 14 | ----------------- 15 | 16 | To get the source source code and run the unit tests, run:: 17 | 18 | $ git clone git://github.com/ambitioninc/django-manager-utils.git 19 | $ cd django-manager-utils 20 | $ virtualenv env 21 | $ . env/bin/activate 22 | $ python setup.py install 23 | $ coverage run setup.py test 24 | $ coverage report 25 | 26 | While 100% code coverage does not make a library bug-free, it significantly 27 | reduces the number of easily caught bugs! Please make sure coverage is at 100% 28 | before submitting a pull request! 29 | 30 | Code Quality 31 | ------------ 32 | 33 | For code quality, please run flake8:: 34 | 35 | $ pip install flake8 36 | $ flake8 . 37 | 38 | Code Styling 39 | ------------ 40 | Please arrange imports with the following style 41 | 42 | .. code-block:: python 43 | 44 | # Standard library imports 45 | import os 46 | 47 | # Third party package imports 48 | from mock import patch 49 | from django.conf import settings 50 | 51 | # Local package imports 52 | from manager_utils.version import __version__ 53 | 54 | Please follow `Google's python style`_ guide wherever possible. 55 | 56 | .. _Google's python style: http://google-styleguide.googlecode.com/svn/trunk/pyguide.html 57 | 58 | Building the docs 59 | ----------------- 60 | 61 | When in the project directory:: 62 | 63 | $ pip install -r requirements/docs.txt 64 | $ python setup.py build_sphinx 65 | $ open docs/_build/html/index.html 66 | 67 | Release Checklist 68 | ----------------- 69 | 70 | Before a new release, please go through the following checklist: 71 | 72 | * Bump version in manager_utils/version.py 73 | * Add a release note in docs/release_notes.rst 74 | * Git tag the version 75 | * Upload to pypi:: 76 | 77 | pip install wheel 78 | python setup.py sdist bdist_wheel upload 79 | 80 | * Increment the version to ``x.x.(x+1)dev`` 81 | 82 | 83 | Vulnerability Reporting 84 | ----------------------- 85 | 86 | For any security issues, please do NOT file an issue or pull request on github! 87 | Please contact `security@ambition.com`_ with the GPG key provided on `Ambition's 88 | website`_. 89 | 90 | .. _security@ambition.com: mailto:security@ambition.com 91 | .. _Ambition's website: http://ambition.com/security/ 92 | 93 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | Django-manager-utils Documentation 2 | ================================== 3 | Django manager utils allows a user to perform various functions not natively 4 | supported by Django's model managers. 5 | 6 | Continue reading for :doc:`Installation`, :doc:`Usage`, 7 | and :doc:`API documentation ` 8 | 9 | Requirements 10 | ------------ 11 | 12 | * Django >= 1.10 13 | * Python 2.7+ 14 | * Python 3.4, 3.5, 3.6 15 | -------------------------------------------------------------------------------- /docs/installation.rst: -------------------------------------------------------------------------------- 1 | Installation 2 | ============ 3 | 4 | To install the latest release, type:: 5 | 6 | pip install django-manager-utils 7 | 8 | To install the latest code directly from source, type:: 9 | 10 | pip install git+git://github.com/ambitioninc/django-manager-utils.git 11 | -------------------------------------------------------------------------------- /docs/ref/class-documentation.rst: -------------------------------------------------------------------------------- 1 | .. _ref-class-documentation: 2 | 3 | Class Documentation 4 | =================== 5 | 6 | ManagerUtilsMixin 7 | ----------------- 8 | 9 | .. automodule:: manager_utils.manager_utils 10 | .. autoclass:: manager_utils.manager_utils.ManagerUtilsMixin 11 | :members: 12 | :undoc-members: 13 | 14 | ManagerUtilsManager 15 | ------------------- 16 | 17 | .. autoclass:: manager_utils.manager_utils.ManagerUtilsMixin 18 | :members: 19 | :undoc-members: 20 | -------------------------------------------------------------------------------- /docs/ref/method-documentation.rst: -------------------------------------------------------------------------------- 1 | .. _ref-method-documentation: 2 | 3 | Method Documentation 4 | ==================== 5 | 6 | .. automodule:: manager_utils.manager_utils 7 | 8 | single 9 | ------ 10 | 11 | .. autofunction:: manager_utils.manager_utils.single 12 | 13 | get_or_none 14 | ----------- 15 | 16 | .. autofunction:: manager_utils.manager_utils.get_or_none 17 | 18 | upsert 19 | ------ 20 | 21 | .. autofunction:: manager_utils.manager_utils.upsert 22 | 23 | bulk_upsert 24 | ----------- 25 | 26 | .. autofunction:: manager_utils.manager_utils.bulk_upsert 27 | 28 | bulk_upsert2 29 | ------------ 30 | 31 | .. autofunction:: manager_utils.manager_utils.bulk_upsert2 32 | 33 | bulk_update 34 | ----------- 35 | 36 | .. autofunction:: manager_utils.manager_utils.bulk_update 37 | 38 | sync 39 | ---- 40 | 41 | .. autofunction:: manager_utils.manager_utils.sync 42 | 43 | sync2 44 | ----- 45 | 46 | .. autofunction:: manager_utils.manager_utils.sync2 47 | 48 | id_dict 49 | ------- 50 | 51 | .. autofunction:: manager_utils.manager_utils.id_dict 52 | 53 | post_bulk_operation 54 | ------------------- 55 | A signal that is emitted at the end of a bulk operation. The current bulk 56 | operations are Django's update and bulk_create methods and this package's 57 | bulk_update method. The signal provides the model that was updated. 58 | 59 | .. autoattribute:: manager_utils.manager_utils.post_bulk_operation 60 | 61 | .. code-block:: python 62 | 63 | from manager_utils import post_bulk_operation 64 | 65 | def signal_handler(self, *args, **kwargs): 66 | print kwargs['model'] 67 | 68 | post_bulk_operation.connect(signal_handler) 69 | 70 | print(TestModel.objects.all().update(int_field=1)) 71 | 72 | -------------------------------------------------------------------------------- /docs/release_notes.rst: -------------------------------------------------------------------------------- 1 | Release Notes 2 | ============= 3 | 4 | v3.1.5 5 | ------ 6 | * Do not sort records when doing an upsert 7 | 8 | v3.1.4 9 | ------ 10 | * revert psycopg3 code 11 | 12 | v3.1.3 13 | ------ 14 | * Added object ordering to upserts and updates to help reduce the number of deadlocks 15 | 16 | v3.1.2 17 | ------ 18 | * Bump django-query-builder for psycopg3 support 19 | 20 | v3.1.1 21 | ------ 22 | * Read the Docs config file v2 23 | * Github actions for testing psycopg3 24 | 25 | v3.1.0 26 | ------ 27 | * Drop django 2.2 28 | * Unpin time zone field 29 | 30 | v3.0.1 31 | ------ 32 | * Switch to github actions 33 | 34 | v3.0.0 35 | ------ 36 | * Add support for django 3.2, 4.0, 4.1 37 | * Add support for python 3.9 38 | * Drop support for python 3.6 39 | 40 | v2.0.4 41 | ------ 42 | * Only conditionally fetching newly created models to match number of queries prior to 2.0.2 43 | 44 | v2.0.3 45 | ------ 46 | * Fetching newly created models in bulk upsert so that relations are hydrated 47 | 48 | v2.0.2 49 | ------ 50 | * Removed custom operation to fetch newly created models in bulk upsert 51 | 52 | v2.0.1 53 | ------ 54 | * Fixed docstring (alextatarinov) 55 | 56 | v2.0.0 57 | ------ 58 | * Support django 2.2, 3.0, 3.1 only 59 | * Support python 3.6, 3.7, 3.8 only 60 | 61 | v1.4.0 62 | ------ 63 | * Only support django >= 2.0 and python >= 3.6 64 | * Fix bulk upsert sql issue to distinguish fields by table 65 | 66 | v1.3.2 67 | ------ 68 | * Fix bulk_update not properly casting fields 69 | * Add support for returning upserts with multiple unique fields for non native 70 | 71 | v1.3.1 72 | ------ 73 | * BAD RELEASE DO NOT USE 74 | 75 | v1.3.0 76 | ------ 77 | * Updated version 2 interface 78 | * Added optimizations to bulk_upsert and sync to ignore duplicate updates 79 | 80 | v1.2.0 81 | ------ 82 | * Added a parallel version 2 interface for bulk_upsert2 and sync2 83 | 84 | v1.1.1 85 | ------ 86 | * Fix setup version requirements 87 | 88 | v1.1.0 89 | ------ 90 | * Use tox to support more versions 91 | 92 | v1.0.0 93 | ------ 94 | * Drop Django 1.9 95 | * Drop Django 1.10 96 | * Add Django 2.0 97 | * Drop python 2.7 98 | * Drop python 3.4 99 | 100 | v0.13.2 101 | ------- 102 | * Optimize native sync 103 | 104 | v0.13.1 105 | ------- 106 | * Add native support for the sync method, thanks @wesleykendall 107 | 108 | v0.13.0 109 | ------- 110 | * Add python 3.6 support 111 | * Drop Django 1.7 support 112 | * Add Django 1.10 support 113 | * Add Django 1.11 support 114 | 115 | v0.12.0 116 | ------- 117 | * Add python 3.5 support, drop django 1.7 support 118 | 119 | v0.11.1 120 | ------- 121 | * Added bulk_create override for ManagerUtilsQuerySet to emit post bulk operation signal 122 | 123 | v0.11.0 124 | ------- 125 | * Where default return value of bulk_upsert was None, now it is a list of lists, the first being the list of updated models, the second being the created models 126 | 127 | v0.10.0 128 | ------- 129 | * Add native postgres upsert support 130 | 131 | v0.9.1 132 | ------ 133 | * Add Django 1.9 support 134 | 135 | v0.8.4 136 | ------ 137 | * Fixed a bug when doing bulk updates on foreign key ID fields in Django 1.7 138 | 139 | v0.8.3 140 | ------ 141 | * Added support for doing bulk updates on custom django fields 142 | 143 | v0.8.0 144 | ------ 145 | * Dropped Django 1.6 support and added Django 1.8 support 146 | 147 | v0.7.2 148 | ------ 149 | * Added Django 1.7 app config 150 | 151 | v0.7.1 152 | ------ 153 | * Added multiple database support for ``bulk_upsert`` 154 | 155 | v0.6.4 156 | ------ 157 | * Fixed ``.bulk_create()`` argument error 158 | 159 | v0.6.1 160 | ------ 161 | * Added RTD docs 162 | * Added python3 compatibility 163 | -------------------------------------------------------------------------------- /docs/toc.rst: -------------------------------------------------------------------------------- 1 | Table of Contents 2 | ================= 3 | 4 | .. toctree:: 5 | :maxdepth: 2 6 | 7 | index 8 | installation 9 | usage 10 | ref/method-documentation 11 | ref/class-documentation 12 | contributing 13 | release_notes 14 | -------------------------------------------------------------------------------- /docs/usage.rst: -------------------------------------------------------------------------------- 1 | Usage 2 | ===== 3 | 4 | Use as a Model Manager 5 | ---------------------- 6 | Django manager utils allows a user to perform various functions not natively 7 | supported by Django's model managers. To use the manager in your Django 8 | models, do: 9 | 10 | .. code-block:: python 11 | 12 | from manager_utils import ManagerUtilsManager 13 | 14 | class MyModel(Model): 15 | objects = ManagerUtilsManager() 16 | 17 | If you want to extend an existing manager to use the manager utils, include 18 | mixin provided first (since it overrides the get_queryset function) as follows: 19 | 20 | .. code-block:: python 21 | 22 | from django.db import models 23 | from manager_utils import ManagerUtilsMixin 24 | 25 | class MyManager(ManagerUtilsMixin, models.Manager): 26 | pass 27 | 28 | 29 | Calling Manager Utils as Standalone Functions 30 | --------------------------------------------- 31 | All of the main manager utils functions listed can also be called as standalone 32 | functions so that third-party managers can take advantage of them. For example: 33 | 34 | .. code-block:: python 35 | 36 | from manager_utils import bulk_update 37 | 38 | bulk_update(TestModel.objects, [model_obj1, model_obj2], ['int_field', 'float_field']) 39 | 40 | -------------------------------------------------------------------------------- /manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import sys 3 | 4 | # Show warnings about django deprecations - uncomment for version upgrade testing 5 | import warnings 6 | from django.utils.deprecation import RemovedInNextVersionWarning 7 | warnings.filterwarnings('always', category=DeprecationWarning) 8 | warnings.filterwarnings('always', category=PendingDeprecationWarning) 9 | warnings.filterwarnings('always', category=RemovedInNextVersionWarning) 10 | 11 | from settings import configure_settings 12 | 13 | 14 | if __name__ == '__main__': 15 | configure_settings() 16 | 17 | from django.core.management import execute_from_command_line 18 | 19 | execute_from_command_line(sys.argv) 20 | -------------------------------------------------------------------------------- /manager_utils/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | from .version import __version__ 3 | from .manager_utils import ( 4 | ManagerUtilsMixin, ManagerUtilsManager, ManagerUtilsQuerySet, post_bulk_operation, 5 | upsert, bulk_update, single, get_or_none, bulk_upsert, bulk_upsert2, id_dict, sync, 6 | sync2 7 | ) 8 | -------------------------------------------------------------------------------- /manager_utils/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class ManagerUtilsConfig(AppConfig): 5 | name = 'manager_utils' 6 | verbose_name = 'Django Manager Utils' 7 | -------------------------------------------------------------------------------- /manager_utils/manager_utils.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | from typing import List 3 | 4 | from django.db import connection 5 | from django.db.models import Manager, Model 6 | from django.db.models.query import QuerySet 7 | from django.dispatch import Signal 8 | from querybuilder.query import Query 9 | 10 | from . import upsert2 11 | 12 | 13 | # A signal that is emitted when any bulk operation occurs 14 | post_bulk_operation = Signal() 15 | """ 16 | providing_args=['model'] 17 | """ 18 | 19 | 20 | def id_dict(queryset): 21 | """ 22 | Returns a dictionary of all the objects keyed on their ID. 23 | 24 | :rtype: dict 25 | :returns: A dictionary of objects from the queryset or manager that is keyed 26 | on the objects' IDs. 27 | 28 | Examples: 29 | 30 | .. code-block:: python 31 | 32 | TestModel.objects.create(int_field=1) 33 | TestModel.objects.create(int_field=2) 34 | 35 | print(id_dict(TestModel.objects.all())) 36 | 37 | """ 38 | return {obj.pk: obj for obj in queryset} 39 | 40 | 41 | def _get_model_objs_to_update_and_create(model_objs, unique_fields, update_fields, extant_model_objs): 42 | """ 43 | Used by bulk_upsert to gather lists of models that should be updated and created. 44 | """ 45 | 46 | # Find all of the objects to update and all of the objects to create 47 | model_objs_to_update, model_objs_to_create = list(), list() 48 | for model_obj in model_objs: 49 | extant_model_obj = extant_model_objs.get(tuple(getattr(model_obj, field) for field in unique_fields), None) 50 | if extant_model_obj is None: 51 | # If the object needs to be created, make a new instance of it 52 | model_objs_to_create.append(model_obj) 53 | else: 54 | # If the object needs to be updated, update its fields 55 | for field in update_fields: 56 | setattr(extant_model_obj, field, getattr(model_obj, field)) 57 | model_objs_to_update.append(extant_model_obj) 58 | 59 | return model_objs_to_update, model_objs_to_create 60 | 61 | 62 | def _get_prepped_model_field(model_obj, field): 63 | """ 64 | Gets the value of a field of a model obj that is prepared for the db. 65 | """ 66 | 67 | # Get the field 68 | field = model_obj._meta.get_field(field) 69 | 70 | # Get the value 71 | value = field.get_db_prep_save(getattr(model_obj, field.attname), connection) 72 | 73 | # Return the value 74 | return value 75 | 76 | 77 | def _fetch_models_by_pk(queryset: QuerySet, models: List[Model]) -> List[Model]: 78 | """ 79 | If the given list of model objects is not empty, return a list of newly fetched models. 80 | This is important when dependent consumers need relationships hydrated after bulk creating models 81 | """ 82 | if not models: 83 | return models 84 | return list( 85 | queryset.filter(pk__in=[model.pk for model in models]) 86 | ) 87 | 88 | 89 | def bulk_upsert( 90 | queryset, model_objs, unique_fields, update_fields=None, return_upserts=False, return_upserts_distinct=False, 91 | sync=False, native=False 92 | ): 93 | """ 94 | Performs a bulk update or insert on a list of model objects. Matches all objects in the queryset 95 | with the objs provided using the field values in unique_fields. 96 | If an existing object is matched, it is updated with the values from the provided objects. Objects 97 | that don't match anything are bulk created. 98 | A user can provide a list update_fields so that any changed values on those fields will be updated. 99 | However, if update_fields is not provided, this function reduces down to performing a bulk_create 100 | on any non extant objects. 101 | 102 | :type model_objs: list of :class:`Models` 103 | :param model_objs: A list of models to upsert. 104 | 105 | :type unique_fields: list of str 106 | :param unique_fields: A list of fields that are used to determine if an object in objs matches a model 107 | from the queryset. 108 | 109 | :type update_fields: list of str 110 | :param update_fields: A list of fields used from the objects in objs as fields when updating existing 111 | models. If None, this function will only perform a bulk create for model_objs that do not 112 | currently exist in the database. 113 | 114 | :type return_upserts_distinct: bool 115 | :param return_upserts_distinct: A flag specifying whether to return the upserted values as a list of distinct lists, 116 | one containing the updated models and the other containing the new models. If True, this performs an 117 | additional query to fetch any bulk created values. 118 | 119 | :type return_upserts: bool 120 | :param return_upserts: A flag specifying whether to return the upserted values. If True, this performs 121 | an additional query to fetch any bulk created values. 122 | 123 | :type sync: bool 124 | :param sync: A flag specifying whether a sync operation should be applied to the bulk_upsert. If this 125 | is True, all values in the queryset that were not updated will be deleted such that the 126 | entire list of model objects is synced to the queryset. 127 | 128 | :type native: bool 129 | :param native: A flag specifying whether to use postgres insert on conflict (upsert). 130 | 131 | :signals: Emits a post_bulk_operation when a bulk_update or a bulk_create occurs. 132 | 133 | Examples: 134 | 135 | .. code-block:: python 136 | 137 | # Start off with no objects in the database. Call a bulk_upsert on the TestModel, which includes 138 | # a char_field, int_field, and float_field 139 | bulk_upsert(TestModel.objects.all(), [ 140 | TestModel(float_field=1.0, char_field='1', int_field=1), 141 | TestModel(float_field=2.0, char_field='2', int_field=2), 142 | TestModel(float_field=3.0, char_field='3', int_field=3), 143 | ], ['int_field'], ['char_field']) 144 | 145 | # All objects should have been created 146 | print(TestModel.objects.count()) 147 | 3 148 | 149 | # Now perform a bulk upsert on all the char_field values. Since the objects existed previously 150 | # (known by the int_field uniqueness constraint), the char fields should be updated 151 | bulk_upsert(TestModel.objects.all(), [ 152 | TestModel(float_field=1.0, char_field='0', int_field=1), 153 | TestModel(float_field=2.0, char_field='0', int_field=2), 154 | TestModel(float_field=3.0, char_field='0', int_field=3), 155 | ], ['int_field'], ['char_field']) 156 | 157 | # No more new objects should have been created, and every char field should be 0 158 | print(TestModel.objects.count(), TestModel.objects.filter(char_field='-1').count()) 159 | 3, 3 160 | 161 | # Do the exact same operation, but this time add an additional object that is not already 162 | # stored. It will be created. 163 | bulk_upsert(TestModel.objects.all(), [ 164 | TestModel(float_field=1.0, char_field='1', int_field=1), 165 | TestModel(float_field=2.0, char_field='2', int_field=2), 166 | TestModel(float_field=3.0, char_field='3', int_field=3), 167 | TestModel(float_field=4.0, char_field='4', int_field=4), 168 | ], ['int_field'], ['char_field']) 169 | 170 | # There should be one more object 171 | print(TestModel.objects.count()) 172 | 4 173 | 174 | # Note that one can also do the upsert on a queryset. Perform the same data upsert on a 175 | # filter for int_field=1. In this case, only one object has the ability to be updated. 176 | # All of the other objects will be created 177 | bulk_upsert(TestModel.objects.filter(int_field=1), [ 178 | TestModel(float_field=1.0, char_field='1', int_field=1), 179 | TestModel(float_field=2.0, char_field='2', int_field=2), 180 | TestModel(float_field=3.0, char_field='3', int_field=3), 181 | TestModel(float_field=4.0, char_field='4', int_field=4), 182 | ], ['int_field'], ['char_field']) 183 | 184 | # There should be three more objects 185 | print(TestModel.objects.count()) 186 | 7 187 | 188 | """ 189 | if not unique_fields: 190 | raise ValueError('Must provide unique_fields argument') 191 | update_fields = update_fields or [] 192 | 193 | if native: 194 | if return_upserts_distinct: 195 | raise NotImplementedError('return upserts distinct not supported with native postgres upsert') 196 | return_value = Query().from_table(table=queryset.model).upsert( 197 | model_objs, unique_fields, update_fields, return_models=return_upserts or sync 198 | ) or [] 199 | if sync: 200 | orig_ids = frozenset(queryset.values_list('pk', flat=True)) 201 | queryset.filter(pk__in=orig_ids - frozenset([m.pk for m in return_value])).delete() 202 | 203 | post_bulk_operation.send(sender=queryset.model, model=queryset.model) 204 | return return_value 205 | 206 | # Create a look up table for all of the objects in the queryset keyed on the unique_fields 207 | extant_model_objs = { 208 | tuple(getattr(extant_model_obj, field) for field in unique_fields): extant_model_obj 209 | for extant_model_obj in queryset 210 | } 211 | 212 | # Find all of the objects to update and all of the objects to create 213 | model_objs_to_update, model_objs_to_create = _get_model_objs_to_update_and_create( 214 | model_objs, unique_fields, update_fields, extant_model_objs) 215 | 216 | # Find all objects in the queryset that will not be updated. These will be deleted if the sync option is 217 | # True 218 | if sync: 219 | model_objs_to_update_set = frozenset(model_objs_to_update) 220 | model_objs_to_delete = [ 221 | model_obj.pk for model_obj in extant_model_objs.values() if model_obj not in model_objs_to_update_set 222 | ] 223 | if model_objs_to_delete: 224 | queryset.filter(pk__in=model_objs_to_delete).delete() 225 | 226 | # Apply bulk updates and creates 227 | if update_fields: 228 | bulk_update(queryset, model_objs_to_update, update_fields) 229 | created_models = queryset.bulk_create(model_objs_to_create) 230 | 231 | # Optionally return the bulk upserted values 232 | if return_upserts_distinct: 233 | # return a list of lists, the first being the updated models, the second being the newly created objects 234 | return model_objs_to_update, _fetch_models_by_pk(queryset, created_models) 235 | if return_upserts: 236 | # return a union list of created and updated models 237 | return model_objs_to_update + _fetch_models_by_pk(queryset, created_models) 238 | 239 | 240 | def bulk_upsert2( 241 | queryset, model_objs, unique_fields, update_fields=None, returning=False, 242 | ignore_duplicate_updates=True, return_untouched=False 243 | ): 244 | """ 245 | Performs a bulk update or insert on a list of model objects. Matches all objects in the queryset 246 | with the objs provided using the field values in unique_fields. 247 | If an existing object is matched, it is updated with the values from the provided objects. Objects 248 | that don't match anything are bulk created. 249 | A user can provide a list update_fields so that any changed values on those fields will be updated. 250 | However, if update_fields is not provided, this function reduces down to performing a bulk_create 251 | on any non extant objects. 252 | 253 | Args: 254 | queryset (Model|QuerySet): A model or a queryset that defines the collection to sync 255 | model_objs (List[Model]): A list of Django models to sync. All models in this list 256 | will be bulk upserted and any models not in the table (or queryset) will be deleted 257 | if sync=True. 258 | unique_fields (List[str]): A list of fields that define the uniqueness of the model. The 259 | model must have a unique constraint on these fields 260 | update_fields (List[str], default=None): A list of fields to update whenever objects 261 | already exist. If an empty list is provided, it is equivalent to doing a bulk 262 | insert on the objects that don't exist. If ``None``, all fields will be updated. 263 | returning (bool|List[str]): If ``True``, returns all fields. If a list, only returns 264 | fields in the list. Return values are split in a tuple of created and updated models 265 | ignore_duplicate_updates (bool, default=False): Ignore updating a row in the upsert if all of the update fields 266 | are duplicates 267 | return_untouched (bool, default=False): Return values that were not touched by the upsert operation 268 | 269 | Returns: 270 | UpsertResult: A list of results if ``returning`` is not ``False``. created, updated, and untouched, 271 | results can be obtained by accessing the ``created``, ``updated``, and ``untouched`` properties 272 | of the result. 273 | 274 | Examples: 275 | 276 | .. code-block:: python 277 | 278 | # Start off with no objects in the database. Call a bulk_upsert on the TestModel, which includes 279 | # a char_field, int_field, and float_field 280 | bulk_upsert2(TestModel.objects.all(), [ 281 | TestModel(float_field=1.0, char_field='1', int_field=1), 282 | TestModel(float_field=2.0, char_field='2', int_field=2), 283 | TestModel(float_field=3.0, char_field='3', int_field=3), 284 | ], ['int_field'], ['char_field']) 285 | 286 | # All objects should have been created 287 | print(TestModel.objects.count()) 288 | 3 289 | 290 | # Now perform a bulk upsert on all the char_field values. Since the objects existed previously 291 | # (known by the int_field uniqueness constraint), the char fields should be updated 292 | bulk_upsert2(TestModel.objects.all(), [ 293 | TestModel(float_field=1.0, char_field='0', int_field=1), 294 | TestModel(float_field=2.0, char_field='0', int_field=2), 295 | TestModel(float_field=3.0, char_field='0', int_field=3), 296 | ], ['int_field'], ['char_field']) 297 | 298 | # No more new objects should have been created, and every char field should be 0 299 | print(TestModel.objects.count(), TestModel.objects.filter(char_field='-1').count()) 300 | 3, 3 301 | 302 | # Do the exact same operation, but this time add an additional object that is not already 303 | # stored. It will be created. 304 | bulk_upsert2(TestModel.objects.all(), [ 305 | TestModel(float_field=1.0, char_field='1', int_field=1), 306 | TestModel(float_field=2.0, char_field='2', int_field=2), 307 | TestModel(float_field=3.0, char_field='3', int_field=3), 308 | TestModel(float_field=4.0, char_field='4', int_field=4), 309 | ], ['int_field'], ['char_field']) 310 | 311 | # There should be one more object 312 | print(TestModel.objects.count()) 313 | 4 314 | 315 | # Note that one can also do the upsert on a queryset. Perform the same data upsert on a 316 | # filter for int_field=1. In this case, only one object has the ability to be updated. 317 | # All of the other objects will be created 318 | bulk_upsert2(TestModel.objects.filter(int_field=1), [ 319 | TestModel(float_field=1.0, char_field='1', int_field=1), 320 | TestModel(float_field=2.0, char_field='2', int_field=2), 321 | TestModel(float_field=3.0, char_field='3', int_field=3), 322 | TestModel(float_field=4.0, char_field='4', int_field=4), 323 | ], ['int_field'], ['char_field']) 324 | 325 | # There should be three more objects 326 | print(TestModel.objects.count()) 327 | 7 328 | 329 | # Return creates and updates on the same set of models 330 | created, updated = bulk_upsert2(TestModel.objects.filter(int_field=1), [ 331 | TestModel(float_field=1.0, char_field='1', int_field=1), 332 | TestModel(float_field=2.0, char_field='2', int_field=2), 333 | TestModel(float_field=3.0, char_field='3', int_field=3), 334 | TestModel(float_field=4.0, char_field='4', int_field=4), 335 | ], ['int_field'], ['char_field']) 336 | 337 | # All four objects should be updated 338 | print(len(updated)) 339 | 4 340 | """ 341 | results = upsert2.upsert(queryset, model_objs, unique_fields, 342 | update_fields=update_fields, returning=returning, 343 | ignore_duplicate_updates=ignore_duplicate_updates, 344 | return_untouched=return_untouched) 345 | post_bulk_operation.send(sender=queryset.model, model=queryset.model) 346 | return results 347 | 348 | 349 | def sync(queryset, model_objs, unique_fields, update_fields=None, **kwargs): 350 | """ 351 | Performs a sync operation on a queryset, making the contents of the 352 | queryset match the contents of model_objs. 353 | 354 | This function calls bulk_upsert underneath the hood with sync=True. 355 | 356 | :type model_objs: list of :class:`Models` 357 | :param model_objs: The models to sync 358 | 359 | :type update_fields: list of str 360 | :param unique_fields: A list of fields that are used to determine if an 361 | object in objs matches a model from the queryset. 362 | 363 | :type update_fields: list of str 364 | :param update_fields: A list of fields used from the objects in objs as fields when updating existing 365 | models. If None, this function will only perform a bulk create for model_objs that do not 366 | currently exist in the database. 367 | 368 | :type native: bool 369 | :param native: A flag specifying whether to use postgres insert on conflict (upsert) when performing 370 | bulk upsert. 371 | """ 372 | return bulk_upsert(queryset, model_objs, unique_fields, update_fields=update_fields, sync=True, **kwargs) 373 | 374 | 375 | def sync2(queryset, model_objs, unique_fields, update_fields=None, returning=False, ignore_duplicate_updates=True): 376 | """ 377 | Performs a sync operation on a queryset, making the contents of the 378 | queryset match the contents of model_objs. 379 | 380 | Note: The definition of a sync requires that we return untouched rows from the upsert opertion. There is 381 | no way to turn off returning untouched rows in a sync. 382 | 383 | Args: 384 | queryset (Model|QuerySet): A model or a queryset that defines the collection to sync 385 | model_objs (List[Model]): A list of Django models to sync. All models in this list 386 | will be bulk upserted and any models not in the table (or queryset) will be deleted 387 | if sync=True. 388 | unique_fields (List[str]): A list of fields that define the uniqueness of the model. The 389 | model must have a unique constraint on these fields 390 | update_fields (List[str], default=None): A list of fields to update whenever objects 391 | already exist. If an empty list is provided, it is equivalent to doing a bulk 392 | insert on the objects that don't exist. If `None`, all fields will be updated. 393 | returning (bool|List[str]): If True, returns all fields. If a list, only returns 394 | fields in the list. Return values are split in a tuple of created, updated, and 395 | deleted models. 396 | ignore_duplicate_updates (bool, default=False): Ignore updating a row in the upsert if all 397 | of the update fields are duplicates 398 | 399 | Returns: 400 | UpsertResult: A list of results if ``returning`` is not ``False``. created, updated, untouched, 401 | and deleted results can be obtained by accessing the ``created``, ``updated``, ``untouched``, 402 | and ``deleted`` properties of the result. 403 | """ 404 | results = upsert2.upsert(queryset, model_objs, unique_fields, 405 | update_fields=update_fields, returning=returning, sync=True, 406 | ignore_duplicate_updates=ignore_duplicate_updates) 407 | post_bulk_operation.send(sender=queryset.model, model=queryset.model) 408 | return results 409 | 410 | 411 | def get_or_none(queryset, **query_params): 412 | """ 413 | Get an object or return None if it doesn't exist. 414 | 415 | :param query_params: The query parameters used in the lookup. 416 | 417 | :returns: A model object if one exists with the query params, None otherwise. 418 | 419 | Examples: 420 | 421 | .. code-block:: python 422 | 423 | model_obj = get_or_none(TestModel.objects, int_field=1) 424 | print(model_obj) 425 | None 426 | 427 | TestModel.objects.create(int_field=1) 428 | model_obj = get_or_none(TestModel.objects, int_field=1) 429 | print(model_obj.int_field) 430 | 1 431 | 432 | """ 433 | try: 434 | obj = queryset.get(**query_params) 435 | except queryset.model.DoesNotExist: 436 | obj = None 437 | return obj 438 | 439 | 440 | def single(queryset): 441 | """ 442 | Assumes that this model only has one element in the table and returns it. 443 | If the table has more than one or no value, an exception is raised. 444 | 445 | :returns: The only model object in the queryset. 446 | 447 | :raises: :class:`DoesNotExist ` 448 | error when the object does not exist or a 449 | :class:`MultipleObjectsReturned ` 450 | error when thereis more than one object. 451 | 452 | Examples: 453 | 454 | .. code-block:: python 455 | 456 | TestModel.objects.create(int_field=1) 457 | model_obj = single(TestModel.objects) 458 | print(model_obj.int_field) 459 | 1 460 | 461 | """ 462 | return queryset.get() 463 | 464 | 465 | def bulk_update(manager, model_objs, fields_to_update): 466 | """ 467 | Bulk updates a list of model objects that are already saved. 468 | 469 | :type model_objs: list of :class:`Models` 470 | :param model_objs: A list of model objects that have been updated. 471 | fields_to_update: A list of fields to be updated. Only these fields will be updated 472 | 473 | 474 | :signals: Emits a post_bulk_operation signal when completed. 475 | 476 | Examples: 477 | 478 | .. code-block:: python 479 | 480 | # Create a couple test models 481 | model_obj1 = TestModel.objects.create(int_field=1, float_field=2.0, char_field='Hi') 482 | model_obj2 = TestModel.objects.create(int_field=3, float_field=4.0, char_field='Hello') 483 | 484 | # Change their fields and do a bulk update 485 | model_obj1.int_field = 10 486 | model_obj1.float_field = 20.0 487 | model_obj2.int_field = 30 488 | model_obj2.float_field = 40.0 489 | bulk_update(TestModel.objects, [model_obj1, model_obj2], ['int_field', 'float_field']) 490 | 491 | # Reload the models and view their changes 492 | model_obj1 = TestModel.objects.get(id=model_obj1.id) 493 | print(model_obj1.int_field, model_obj1.float_field) 494 | 10, 20.0 495 | 496 | model_obj2 = TestModel.objects.get(id=model_obj2.id) 497 | print(model_obj2.int_field, model_obj2.float_field) 498 | 10, 20.0 499 | 500 | """ 501 | 502 | # Sort the model objects to reduce the likelihood of deadlocks 503 | model_objs = sorted(model_objs, key=lambda obj: obj.pk) 504 | 505 | # Add the pk to the value fields so we can join 506 | value_fields = [manager.model._meta.pk.attname] + fields_to_update 507 | 508 | # Build the row values 509 | row_values = [ 510 | [_get_prepped_model_field(model_obj, field_name) for field_name in value_fields] 511 | for model_obj in model_objs 512 | ] 513 | 514 | # If we do not have any values or fields to update just return 515 | if len(row_values) == 0 or len(fields_to_update) == 0: 516 | return 517 | 518 | # Create a map of db types 519 | db_types = [ 520 | manager.model._meta.get_field(field).db_type(connection) 521 | for field in value_fields 522 | ] 523 | 524 | # Build the value fields sql 525 | value_fields_sql = ', '.join( 526 | '"{field}"'.format(field=manager.model._meta.get_field(field).column) 527 | for field in value_fields 528 | ) 529 | 530 | # Build the set sql 531 | update_fields_sql = ', '.join([ 532 | '"{field}" = "new_values"."{field}"'.format( 533 | field=manager.model._meta.get_field(field).column 534 | ) 535 | for field in fields_to_update 536 | ]) 537 | 538 | # Build the values sql 539 | values_sql = ', '.join([ 540 | '({0})'.format( 541 | ', '.join([ 542 | '%s::{0}'.format( 543 | db_types[i] 544 | ) if not row_number and i else '%s' 545 | for i, _ in enumerate(row) 546 | ]) 547 | ) 548 | for row_number, row in enumerate(row_values) 549 | ]) 550 | 551 | # Start building the query 552 | update_sql = ( 553 | 'UPDATE {table} ' 554 | 'SET {update_fields_sql} ' 555 | 'FROM (VALUES {values_sql}) AS new_values ({value_fields_sql}) ' 556 | 'WHERE "{table}"."{pk_field}" = "new_values"."{pk_field}"' 557 | ).format( 558 | table=manager.model._meta.db_table, 559 | pk_field=manager.model._meta.pk.column, 560 | update_fields_sql=update_fields_sql, 561 | values_sql=values_sql, 562 | value_fields_sql=value_fields_sql 563 | ) 564 | 565 | # Combine all the row values 566 | update_sql_params = list(itertools.chain(*row_values)) 567 | 568 | # Run the update query 569 | with connection.cursor() as cursor: 570 | cursor.execute(update_sql, update_sql_params) 571 | 572 | # call the bulk operation signal 573 | post_bulk_operation.send(sender=manager.model, model=manager.model) 574 | 575 | 576 | def upsert(manager, defaults=None, updates=None, **kwargs): 577 | """ 578 | Performs an update on an object or an insert if the object does not exist. 579 | 580 | :type defaults: dict 581 | :param defaults: These values are set when the object is created, but are irrelevant 582 | when the object already exists. This field should only be used when values only need to 583 | be set during creation. 584 | 585 | :type updates: dict 586 | :param updates: These values are updated when the object is updated. They also override any 587 | values provided in the defaults when inserting the object. 588 | 589 | :param kwargs: These values provide the arguments used when checking for the existence of 590 | the object. They are used in a similar manner to Django's get_or_create function. 591 | 592 | :returns: A tuple of the upserted object and a Boolean that is True if it was created (False otherwise) 593 | 594 | Examples: 595 | 596 | .. code-block:: python 597 | 598 | # Upsert a test model with an int value of 1. Use default values that will be given to it when created 599 | model_obj, created = upsert(TestModel.objects, int_field=1, defaults={'float_field': 2.0}) 600 | print(created) 601 | True 602 | print(model_obj.int_field, model_obj.float_field) 603 | 1, 2.0 604 | 605 | # Do an upsert on that same model with different default fields. Since it already exists, the defaults 606 | # are not used 607 | model_obj, created = upsert(TestModel.objects, int_field=1, defaults={'float_field': 3.0}) 608 | print(created) 609 | False 610 | print(model_obj.int_field, model_obj.float_field) 611 | 1, 2.0 612 | 613 | # In order to update the float field in an existing object, use the updates dictionary 614 | model_obj, created = upsert(TestModel.objects, int_field=1, updates={'float_field': 3.0}) 615 | print(created) 616 | False 617 | print(model_obj.int_field, model_obj.float_field) 618 | 1, 3.0 619 | 620 | # You can use updates on a newly created object that will also be used as initial values. 621 | model_obj, created = upsert(TestModel.objects, int_field=2, updates={'float_field': 4.0}) 622 | print(created) 623 | True 624 | print(model_obj.int_field, model_obj.float_field) 625 | 2, 4.0 626 | 627 | """ 628 | defaults = defaults or {} 629 | # Override any defaults with updates 630 | defaults.update(updates or {}) 631 | 632 | # Do a get or create 633 | obj, created = manager.get_or_create(defaults=defaults, **kwargs) 634 | 635 | # Update any necessary fields 636 | if updates is not None and not created and any(getattr(obj, k) != updates[k] for k in updates): 637 | for k, v in updates.items(): 638 | setattr(obj, k, v) 639 | obj.save(update_fields=updates) 640 | 641 | return obj, created 642 | 643 | 644 | class ManagerUtilsQuerySet(QuerySet): 645 | """ 646 | Defines the methods in the manager utils that can also be applied to querysets. 647 | """ 648 | def id_dict(self): 649 | return id_dict(self) 650 | 651 | def bulk_upsert(self, model_objs, unique_fields, update_fields=None, return_upserts=False, native=False): 652 | return bulk_upsert( 653 | self, model_objs, unique_fields, update_fields=update_fields, return_upserts=return_upserts, native=native 654 | ) 655 | 656 | def bulk_upsert2(self, model_objs, unique_fields, update_fields=None, returning=False, 657 | ignore_duplicate_updates=True, return_untouched=False): 658 | return bulk_upsert2(self, model_objs, unique_fields, 659 | update_fields=update_fields, returning=returning, 660 | ignore_duplicate_updates=ignore_duplicate_updates, 661 | return_untouched=return_untouched) 662 | 663 | def bulk_create(self, *args, **kwargs): 664 | """ 665 | Overrides Django's bulk_create function to emit a post_bulk_operation signal when bulk_create 666 | is finished. 667 | """ 668 | ret_val = super(ManagerUtilsQuerySet, self).bulk_create(*args, **kwargs) 669 | post_bulk_operation.send(sender=self.model, model=self.model) 670 | return ret_val 671 | 672 | def sync(self, model_objs, unique_fields, update_fields=None, native=False): 673 | return sync(self, model_objs, unique_fields, update_fields=update_fields, native=native) 674 | 675 | def sync2(self, model_objs, unique_fields, update_fields=None, returning=False, ignore_duplicate_updates=True): 676 | return sync2(self, model_objs, unique_fields, update_fields=update_fields, returning=returning, 677 | ignore_duplicate_updates=ignore_duplicate_updates) 678 | 679 | def get_or_none(self, **query_params): 680 | return get_or_none(self, **query_params) 681 | 682 | def single(self): 683 | return single(self) 684 | 685 | def update(self, **kwargs): 686 | """ 687 | Overrides Django's update method to emit a post_bulk_operation signal when it completes. 688 | """ 689 | ret_val = super(ManagerUtilsQuerySet, self).update(**kwargs) 690 | post_bulk_operation.send(sender=self.model, model=self.model) 691 | return ret_val 692 | 693 | 694 | class ManagerUtilsMixin(object): 695 | """ 696 | A mixin that can be used by django model managers. It provides additional functionality on top 697 | of the regular Django Manager class. 698 | """ 699 | def get_queryset(self): 700 | return ManagerUtilsQuerySet(self.model) 701 | 702 | def id_dict(self): 703 | return id_dict(self.get_queryset()) 704 | 705 | def bulk_upsert( 706 | self, model_objs, unique_fields, update_fields=None, return_upserts=False, return_upserts_distinct=False, 707 | native=False): 708 | return bulk_upsert( 709 | self.get_queryset(), model_objs, unique_fields, update_fields=update_fields, return_upserts=return_upserts, 710 | return_upserts_distinct=return_upserts_distinct, native=native) 711 | 712 | def bulk_upsert2(self, model_objs, unique_fields, update_fields=None, returning=False, 713 | ignore_duplicate_updates=True, return_untouched=False): 714 | return bulk_upsert2( 715 | self.get_queryset(), model_objs, unique_fields, 716 | update_fields=update_fields, returning=returning, 717 | ignore_duplicate_updates=ignore_duplicate_updates, 718 | return_untouched=return_untouched) 719 | 720 | def sync(self, model_objs, unique_fields, update_fields=None, native=False): 721 | return sync(self.get_queryset(), model_objs, unique_fields, update_fields=update_fields, native=native) 722 | 723 | def sync2(self, model_objs, unique_fields, update_fields=None, returning=False, ignore_duplicate_updates=True): 724 | return sync2( 725 | self.get_queryset(), model_objs, unique_fields, update_fields=update_fields, returning=returning, 726 | ignore_duplicate_updates=ignore_duplicate_updates) 727 | 728 | def bulk_update(self, model_objs, fields_to_update): 729 | return bulk_update(self.get_queryset(), model_objs, fields_to_update) 730 | 731 | def upsert(self, defaults=None, updates=None, **kwargs): 732 | return upsert(self.get_queryset(), defaults=defaults, updates=updates, **kwargs) 733 | 734 | def get_or_none(self, **query_params): 735 | return get_or_none(self.get_queryset(), **query_params) 736 | 737 | def single(self): 738 | return single(self.get_queryset()) 739 | 740 | 741 | class ManagerUtilsManager(ManagerUtilsMixin, Manager): 742 | """ 743 | A class that can be used as a manager. It already inherits the Django Manager class and adds 744 | the mixin. 745 | """ 746 | pass 747 | -------------------------------------------------------------------------------- /manager_utils/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ambitioninc/django-manager-utils/bba62e06ec86198f1bb0452090a3283d7790ad2c/manager_utils/tests/__init__.py -------------------------------------------------------------------------------- /manager_utils/tests/manager_utils_tests.py: -------------------------------------------------------------------------------- 1 | import datetime as dt 2 | 3 | from django.test import TestCase 4 | from django_dynamic_fixture import G 5 | import freezegun 6 | from manager_utils import post_bulk_operation 7 | from manager_utils.manager_utils import _get_prepped_model_field 8 | from unittest.mock import patch 9 | from parameterized import parameterized 10 | from pytz import timezone 11 | 12 | from manager_utils.tests import models 13 | 14 | 15 | class TestGetPreppedModelField(TestCase): 16 | def test_invalid_field(self): 17 | t = models.TestModel() 18 | with self.assertRaises(Exception): 19 | _get_prepped_model_field(t, 'non_extant_field') 20 | 21 | 22 | class SyncTest(TestCase): 23 | """ 24 | Tests the sync function. 25 | """ 26 | @parameterized.expand([(True,), (False,)]) 27 | def test_w_char_pk(self, native): 28 | """ 29 | Tests with a model that has a char pk. 30 | """ 31 | extant_obj1 = G(models.TestPkChar, my_key='1', char_field='1') 32 | extant_obj2 = G(models.TestPkChar, my_key='2', char_field='1') 33 | extant_obj3 = G(models.TestPkChar, my_key='3', char_field='1') 34 | 35 | models.TestPkChar.objects.sync([ 36 | models.TestPkChar(my_key='3', char_field='2'), models.TestPkChar(my_key='4', char_field='2'), 37 | models.TestPkChar(my_key='5', char_field='2') 38 | ], ['my_key'], ['char_field'], native=native) 39 | 40 | self.assertEqual(models.TestPkChar.objects.count(), 3) 41 | self.assertTrue(models.TestPkChar.objects.filter(my_key='3').exists()) 42 | self.assertTrue(models.TestPkChar.objects.filter(my_key='4').exists()) 43 | self.assertTrue(models.TestPkChar.objects.filter(my_key='5').exists()) 44 | 45 | with self.assertRaises(models.TestPkChar.DoesNotExist): 46 | models.TestPkChar.objects.get(pk=extant_obj1.pk) 47 | with self.assertRaises(models.TestPkChar.DoesNotExist): 48 | models.TestPkChar.objects.get(pk=extant_obj2.pk) 49 | test_model = models.TestPkChar.objects.get(pk=extant_obj3.pk) 50 | self.assertEqual(test_model.char_field, '2') 51 | 52 | @parameterized.expand([(True,), (False,)]) 53 | def test_no_existing_objs(self, native): 54 | """ 55 | Tests when there are no existing objects before the sync. 56 | """ 57 | models.TestModel.objects.sync([ 58 | models.TestModel(int_field=1), models.TestModel(int_field=3), 59 | models.TestModel(int_field=4) 60 | ], ['int_field'], ['float_field'], native=native) 61 | self.assertEqual(models.TestModel.objects.count(), 3) 62 | self.assertTrue(models.TestModel.objects.filter(int_field=1).exists()) 63 | self.assertTrue(models.TestModel.objects.filter(int_field=3).exists()) 64 | self.assertTrue(models.TestModel.objects.filter(int_field=4).exists()) 65 | 66 | @parameterized.expand([(True,), (False,)]) 67 | def test_existing_objs_all_deleted(self, native): 68 | """ 69 | Tests when there are existing objects that will all be deleted. 70 | """ 71 | extant_obj1 = G(models.TestModel, int_field=1) 72 | extant_obj2 = G(models.TestModel, int_field=2) 73 | extant_obj3 = G(models.TestModel, int_field=3) 74 | 75 | models.TestModel.objects.sync([ 76 | models.TestModel(int_field=4), models.TestModel(int_field=5), models.TestModel(int_field=6) 77 | ], ['int_field'], ['float_field'], native=native) 78 | 79 | self.assertEqual(models.TestModel.objects.count(), 3) 80 | self.assertTrue(models.TestModel.objects.filter(int_field=4).exists()) 81 | self.assertTrue(models.TestModel.objects.filter(int_field=5).exists()) 82 | self.assertTrue(models.TestModel.objects.filter(int_field=6).exists()) 83 | 84 | with self.assertRaises(models.TestModel.DoesNotExist): 85 | models.TestModel.objects.get(id=extant_obj1.id) 86 | with self.assertRaises(models.TestModel.DoesNotExist): 87 | models.TestModel.objects.get(id=extant_obj2.id) 88 | with self.assertRaises(models.TestModel.DoesNotExist): 89 | models.TestModel.objects.get(id=extant_obj3.id) 90 | 91 | @parameterized.expand([(True,), (False,)]) 92 | def test_existing_objs_all_deleted_empty_sync(self, native): 93 | """ 94 | Tests when there are existing objects deleted because of an emtpy sync. 95 | """ 96 | extant_obj1 = G(models.TestModel, int_field=1) 97 | extant_obj2 = G(models.TestModel, int_field=2) 98 | extant_obj3 = G(models.TestModel, int_field=3) 99 | 100 | models.TestModel.objects.sync([], ['int_field'], ['float_field'], native=native) 101 | 102 | self.assertEqual(models.TestModel.objects.count(), 0) 103 | with self.assertRaises(models.TestModel.DoesNotExist): 104 | models.TestModel.objects.get(id=extant_obj1.id) 105 | with self.assertRaises(models.TestModel.DoesNotExist): 106 | models.TestModel.objects.get(id=extant_obj2.id) 107 | with self.assertRaises(models.TestModel.DoesNotExist): 108 | models.TestModel.objects.get(id=extant_obj3.id) 109 | 110 | @parameterized.expand([(True,), (False,)]) 111 | def test_existing_objs_some_deleted(self, native): 112 | """ 113 | Tests when some existing objects will be deleted. 114 | """ 115 | extant_obj1 = G(models.TestModel, int_field=1, float_field=1) 116 | extant_obj2 = G(models.TestModel, int_field=2, float_field=1) 117 | extant_obj3 = G(models.TestModel, int_field=3, float_field=1) 118 | 119 | models.TestModel.objects.sync([ 120 | models.TestModel(int_field=3, float_field=2), models.TestModel(int_field=4, float_field=2), 121 | models.TestModel(int_field=5, float_field=2) 122 | ], ['int_field'], ['float_field'], native=native) 123 | 124 | self.assertEqual(models.TestModel.objects.count(), 3) 125 | self.assertTrue(models.TestModel.objects.filter(int_field=3).exists()) 126 | self.assertTrue(models.TestModel.objects.filter(int_field=4).exists()) 127 | self.assertTrue(models.TestModel.objects.filter(int_field=5).exists()) 128 | 129 | with self.assertRaises(models.TestModel.DoesNotExist): 130 | models.TestModel.objects.get(id=extant_obj1.id) 131 | with self.assertRaises(models.TestModel.DoesNotExist): 132 | models.TestModel.objects.get(id=extant_obj2.id) 133 | test_model = models.TestModel.objects.get(id=extant_obj3.id) 134 | self.assertEqual(test_model.int_field, 3) 135 | 136 | @parameterized.expand([(True,), (False,)]) 137 | def test_existing_objs_some_deleted_w_queryset(self, native): 138 | """ 139 | Tests when some existing objects will be deleted on a queryset 140 | """ 141 | extant_obj0 = G(models.TestModel, int_field=0, float_field=1) 142 | extant_obj1 = G(models.TestModel, int_field=1, float_field=1) 143 | extant_obj2 = G(models.TestModel, int_field=2, float_field=1) 144 | extant_obj3 = G(models.TestModel, int_field=3, float_field=1) 145 | extant_obj4 = G(models.TestModel, int_field=4, float_field=0) 146 | 147 | models.TestModel.objects.filter(int_field__lt=4).sync([ 148 | models.TestModel(int_field=1, float_field=2), models.TestModel(int_field=2, float_field=2), 149 | models.TestModel(int_field=3, float_field=2) 150 | ], ['int_field'], ['float_field'], native=native) 151 | 152 | self.assertEqual(models.TestModel.objects.count(), 4) 153 | self.assertTrue(models.TestModel.objects.filter(int_field=1).exists()) 154 | self.assertTrue(models.TestModel.objects.filter(int_field=2).exists()) 155 | self.assertTrue(models.TestModel.objects.filter(int_field=3).exists()) 156 | 157 | with self.assertRaises(models.TestModel.DoesNotExist): 158 | models.TestModel.objects.get(id=extant_obj0.id) 159 | 160 | test_model = models.TestModel.objects.get(id=extant_obj1.id) 161 | self.assertEqual(test_model.float_field, 2) 162 | test_model = models.TestModel.objects.get(id=extant_obj2.id) 163 | self.assertEqual(test_model.float_field, 2) 164 | test_model = models.TestModel.objects.get(id=extant_obj3.id) 165 | self.assertEqual(test_model.float_field, 2) 166 | test_model = models.TestModel.objects.get(id=extant_obj4.id) 167 | self.assertEqual(test_model.float_field, 0) 168 | 169 | 170 | class Sync2Test(TestCase): 171 | """ 172 | Tests the sync2 function. 173 | """ 174 | def test_w_char_pk(self): 175 | """ 176 | Tests with a model that has a char pk. 177 | """ 178 | extant_obj1 = G(models.TestPkChar, my_key='1', char_field='1') 179 | extant_obj2 = G(models.TestPkChar, my_key='2', char_field='1') 180 | extant_obj3 = G(models.TestPkChar, my_key='3', char_field='1') 181 | 182 | models.TestPkChar.objects.sync2([ 183 | models.TestPkChar(my_key='3', char_field='2'), models.TestPkChar(my_key='4', char_field='2'), 184 | models.TestPkChar(my_key='5', char_field='2') 185 | ], ['my_key'], ['char_field']) 186 | 187 | self.assertEqual(models.TestPkChar.objects.count(), 3) 188 | self.assertTrue(models.TestPkChar.objects.filter(my_key='3').exists()) 189 | self.assertTrue(models.TestPkChar.objects.filter(my_key='4').exists()) 190 | self.assertTrue(models.TestPkChar.objects.filter(my_key='5').exists()) 191 | 192 | with self.assertRaises(models.TestPkChar.DoesNotExist): 193 | models.TestPkChar.objects.get(pk=extant_obj1.pk) 194 | with self.assertRaises(models.TestPkChar.DoesNotExist): 195 | models.TestPkChar.objects.get(pk=extant_obj2.pk) 196 | test_model = models.TestPkChar.objects.get(pk=extant_obj3.pk) 197 | self.assertEqual(test_model.char_field, '2') 198 | 199 | def test_no_existing_objs(self): 200 | """ 201 | Tests when there are no existing objects before the sync. 202 | """ 203 | models.TestModel.objects.sync([ 204 | models.TestModel(int_field=1), models.TestModel(int_field=3), 205 | models.TestModel(int_field=4) 206 | ], ['int_field'], ['float_field']) 207 | self.assertEqual(models.TestModel.objects.count(), 3) 208 | self.assertTrue(models.TestModel.objects.filter(int_field=1).exists()) 209 | self.assertTrue(models.TestModel.objects.filter(int_field=3).exists()) 210 | self.assertTrue(models.TestModel.objects.filter(int_field=4).exists()) 211 | 212 | def test_existing_objs_all_deleted(self): 213 | """ 214 | Tests when there are existing objects that will all be deleted. 215 | """ 216 | extant_obj1 = G(models.TestModel, int_field=1) 217 | extant_obj2 = G(models.TestModel, int_field=2) 218 | extant_obj3 = G(models.TestModel, int_field=3) 219 | 220 | models.TestModel.objects.sync2([ 221 | models.TestModel(int_field=4), models.TestModel(int_field=5), models.TestModel(int_field=6) 222 | ], ['int_field'], ['float_field']) 223 | 224 | self.assertEqual(models.TestModel.objects.count(), 3) 225 | self.assertTrue(models.TestModel.objects.filter(int_field=4).exists()) 226 | self.assertTrue(models.TestModel.objects.filter(int_field=5).exists()) 227 | self.assertTrue(models.TestModel.objects.filter(int_field=6).exists()) 228 | 229 | with self.assertRaises(models.TestModel.DoesNotExist): 230 | models.TestModel.objects.get(id=extant_obj1.id) 231 | with self.assertRaises(models.TestModel.DoesNotExist): 232 | models.TestModel.objects.get(id=extant_obj2.id) 233 | with self.assertRaises(models.TestModel.DoesNotExist): 234 | models.TestModel.objects.get(id=extant_obj3.id) 235 | 236 | def test_existing_objs_all_deleted_empty_sync(self): 237 | """ 238 | Tests when there are existing objects deleted because of an emtpy sync. 239 | """ 240 | extant_obj1 = G(models.TestModel, int_field=1) 241 | extant_obj2 = G(models.TestModel, int_field=2) 242 | extant_obj3 = G(models.TestModel, int_field=3) 243 | 244 | models.TestModel.objects.sync2([], ['int_field'], ['float_field']) 245 | 246 | self.assertEqual(models.TestModel.objects.count(), 0) 247 | with self.assertRaises(models.TestModel.DoesNotExist): 248 | models.TestModel.objects.get(id=extant_obj1.id) 249 | with self.assertRaises(models.TestModel.DoesNotExist): 250 | models.TestModel.objects.get(id=extant_obj2.id) 251 | with self.assertRaises(models.TestModel.DoesNotExist): 252 | models.TestModel.objects.get(id=extant_obj3.id) 253 | 254 | def test_existing_objs_some_deleted(self): 255 | """ 256 | Tests when some existing objects will be deleted. 257 | """ 258 | extant_obj1 = G(models.TestModel, int_field=1, float_field=1) 259 | extant_obj2 = G(models.TestModel, int_field=2, float_field=1) 260 | extant_obj3 = G(models.TestModel, int_field=3, float_field=1) 261 | 262 | models.TestModel.objects.sync2([ 263 | models.TestModel(int_field=3, float_field=2), models.TestModel(int_field=4, float_field=2), 264 | models.TestModel(int_field=5, float_field=2) 265 | ], ['int_field'], ['float_field']) 266 | 267 | self.assertEqual(models.TestModel.objects.count(), 3) 268 | self.assertTrue(models.TestModel.objects.filter(int_field=3).exists()) 269 | self.assertTrue(models.TestModel.objects.filter(int_field=4).exists()) 270 | self.assertTrue(models.TestModel.objects.filter(int_field=5).exists()) 271 | 272 | with self.assertRaises(models.TestModel.DoesNotExist): 273 | models.TestModel.objects.get(id=extant_obj1.id) 274 | with self.assertRaises(models.TestModel.DoesNotExist): 275 | models.TestModel.objects.get(id=extant_obj2.id) 276 | test_model = models.TestModel.objects.get(id=extant_obj3.id) 277 | self.assertEqual(test_model.int_field, 3) 278 | 279 | def test_existing_objs_some_deleted_w_queryset(self): 280 | """ 281 | Tests when some existing objects will be deleted on a queryset 282 | """ 283 | extant_obj0 = G(models.TestModel, int_field=0, float_field=1) 284 | extant_obj1 = G(models.TestModel, int_field=1, float_field=1) 285 | extant_obj2 = G(models.TestModel, int_field=2, float_field=1) 286 | extant_obj3 = G(models.TestModel, int_field=3, float_field=1) 287 | extant_obj4 = G(models.TestModel, int_field=4, float_field=0) 288 | 289 | models.TestModel.objects.filter(int_field__lt=4).sync2([ 290 | models.TestModel(int_field=1, float_field=2), models.TestModel(int_field=2, float_field=2), 291 | models.TestModel(int_field=3, float_field=2) 292 | ], ['int_field'], ['float_field']) 293 | 294 | self.assertEqual(models.TestModel.objects.count(), 4) 295 | self.assertTrue(models.TestModel.objects.filter(int_field=1).exists()) 296 | self.assertTrue(models.TestModel.objects.filter(int_field=2).exists()) 297 | self.assertTrue(models.TestModel.objects.filter(int_field=3).exists()) 298 | 299 | with self.assertRaises(models.TestModel.DoesNotExist): 300 | models.TestModel.objects.get(id=extant_obj0.id) 301 | 302 | test_model = models.TestModel.objects.get(id=extant_obj1.id) 303 | self.assertEqual(test_model.float_field, 2) 304 | test_model = models.TestModel.objects.get(id=extant_obj2.id) 305 | self.assertEqual(test_model.float_field, 2) 306 | test_model = models.TestModel.objects.get(id=extant_obj3.id) 307 | self.assertEqual(test_model.float_field, 2) 308 | test_model = models.TestModel.objects.get(id=extant_obj4.id) 309 | self.assertEqual(test_model.float_field, 0) 310 | 311 | def test_existing_objs_some_deleted_wo_update(self): 312 | """ 313 | Tests when some existing objects will be deleted on a queryset. Run syncing 314 | with no update fields and verify they are untouched in the sync 315 | """ 316 | objs = [G(models.TestModel, int_field=i, float_field=i) for i in range(5)] 317 | 318 | results = models.TestModel.objects.filter(int_field__lt=4).sync2([ 319 | models.TestModel(int_field=1, float_field=2), models.TestModel(int_field=2, float_field=2), 320 | models.TestModel(int_field=3, float_field=2) 321 | ], ['int_field'], [], returning=True) 322 | 323 | self.assertEqual(len(list(results)), 4) 324 | self.assertEqual(len(list(results.deleted)), 1) 325 | self.assertEqual(len(list(results.untouched)), 3) 326 | self.assertEqual(list(results.deleted)[0].id, objs[0].id) 327 | 328 | def test_existing_objs_some_deleted_some_updated(self): 329 | """ 330 | Tests when some existing objects will be deleted on a queryset. Run syncing 331 | with some update fields. 332 | """ 333 | objs = [G(models.TestModel, int_field=i, float_field=i) for i in range(5)] 334 | 335 | results = models.TestModel.objects.filter(int_field__lt=4).sync2([ 336 | models.TestModel(int_field=1, float_field=2), models.TestModel(int_field=2, float_field=2), 337 | models.TestModel(int_field=3, float_field=2) 338 | ], ['int_field'], ['float_field'], returning=True, ignore_duplicate_updates=True) 339 | 340 | self.assertEqual(len(list(results)), 4) 341 | self.assertEqual(len(list(results.deleted)), 1) 342 | self.assertEqual(len(list(results.updated)), 2) 343 | self.assertEqual(len(list(results.untouched)), 1) 344 | self.assertEqual(list(results.deleted)[0].id, objs[0].id) 345 | 346 | 347 | class BulkUpsertTest(TestCase): 348 | """ 349 | Tests the bulk_upsert function. 350 | """ 351 | def test_return_upserts_none(self): 352 | """ 353 | Tests the return_upserts flag on bulk upserts when there is no data. 354 | """ 355 | return_values = models.TestModel.objects.bulk_upsert([], ['float_field'], ['float_field'], return_upserts=True) 356 | self.assertEqual(return_values, []) 357 | 358 | def test_return_upserts_distinct_none(self): 359 | """ 360 | Tests the return_upserts_distinct flag on bulk upserts when there is no data. 361 | """ 362 | return_values = models.TestModel.objects.bulk_upsert( 363 | [], ['float_field'], ['float_field'], return_upserts_distinct=True) 364 | self.assertEqual(return_values, ([], [])) 365 | 366 | def test_return_upserts_none_native(self): 367 | """ 368 | Tests the return_upserts flag on bulk upserts when there is no data. 369 | """ 370 | return_values = models.TestModel.objects.bulk_upsert( 371 | [], ['float_field'], ['float_field'], return_upserts=True, native=True 372 | ) 373 | self.assertEqual(return_values, []) 374 | 375 | def test_return_upserts_distinct_none_native(self): 376 | """ 377 | verifies that return_upserts_distinct flag with native is not supported 378 | """ 379 | with self.assertRaises(NotImplementedError): 380 | models.TestModel.objects.bulk_upsert( 381 | [], ['float_field'], ['float_field'], return_upserts_distinct=True, native=True) 382 | 383 | def test_return_created_values(self): 384 | """ 385 | Tests that values that are created are returned properly when return_upserts is True. 386 | """ 387 | 388 | return_values = models.TestModel.objects.bulk_upsert( 389 | [ 390 | models.TestModel(int_field=1, char_field='1'), 391 | models.TestModel(int_field=3, char_field='3'), 392 | models.TestModel(int_field=4, char_field='4') 393 | ], 394 | ['int_field', 'char_field'], 395 | ['float_field'], 396 | return_upserts=True 397 | ) 398 | 399 | # Assert that we properly returned the models 400 | self.assertEqual(len(return_values), 3) 401 | for test_model, expected_int in zip(sorted(return_values, key=lambda k: k.int_field), [1, 3, 4]): 402 | self.assertEqual(test_model.int_field, expected_int) 403 | self.assertIsNotNone(test_model.id) 404 | self.assertEqual(models.TestModel.objects.count(), 3) 405 | 406 | # Run additional upserts 407 | return_values = models.TestModel.objects.bulk_upsert( 408 | [ 409 | models.TestModel(int_field=1, char_field='1', float_field=10), 410 | models.TestModel(int_field=3, char_field='3'), 411 | models.TestModel(int_field=4, char_field='4'), 412 | models.TestModel(int_field=5, char_field='5', float_field=50), 413 | ], 414 | ['int_field', 'char_field'], 415 | ['float_field'], 416 | return_upserts=True 417 | ) 418 | self.assertEqual(len(return_values), 4) 419 | self.assertEqual( 420 | [ 421 | [1, '1', 10], 422 | [3, '3', None], 423 | [4, '4', None], 424 | [5, '5', 50], 425 | ], 426 | [ 427 | [test_model.int_field, test_model.char_field, test_model.float_field] 428 | for test_model in return_values 429 | ] 430 | ) 431 | 432 | def test_return_created_values_native(self): 433 | """ 434 | Tests that values that are created are returned properly when return_upserts is True. 435 | """ 436 | return_values = models.TestModel.objects.bulk_upsert( 437 | [ 438 | models.TestModel(int_field=1, char_field='1'), 439 | models.TestModel(int_field=3, char_field='3'), 440 | models.TestModel(int_field=4, char_field='4') 441 | ], 442 | ['int_field', 'char_field'], 443 | ['float_field'], 444 | return_upserts=True, 445 | native=True 446 | ) 447 | 448 | self.assertEqual(len(return_values), 3) 449 | for test_model, expected_int in zip(sorted(return_values, key=lambda k: k.int_field), [1, 3, 4]): 450 | self.assertEqual(test_model.int_field, expected_int) 451 | self.assertIsNotNone(test_model.id) 452 | self.assertEqual(models.TestModel.objects.count(), 3) 453 | 454 | def test_return_created_updated_values(self): 455 | """ 456 | Tests returning values when the items are either updated or created. 457 | """ 458 | # Create an item that will be updated 459 | G(models.TestModel, int_field=2, float_field=1.0) 460 | return_values = models.TestModel.objects.bulk_upsert( 461 | [ 462 | models.TestModel(int_field=1, float_field=3.0), models.TestModel(int_field=2.0, float_field=3.0), 463 | models.TestModel(int_field=3, float_field=3.0), models.TestModel(int_field=4, float_field=3.0) 464 | ], 465 | ['int_field'], ['float_field'], return_upserts=True) 466 | 467 | self.assertEqual(len(return_values), 4) 468 | for test_model, expected_int in zip(sorted(return_values, key=lambda k: k.int_field), [1, 2, 3, 4]): 469 | self.assertEqual(test_model.int_field, expected_int) 470 | self.assertAlmostEqual(test_model.float_field, 3.0) 471 | self.assertIsNotNone(test_model.id) 472 | self.assertEqual(models.TestModel.objects.count(), 4) 473 | 474 | def test_return_created_updated_values_native(self): 475 | """ 476 | Tests returning values when the items are either updated or created. 477 | """ 478 | # Create an item that will be updated 479 | G(models.TestModel, int_field=2, float_field=1.0) 480 | model_objects = [ 481 | models.TestModel(int_field=1, float_field=3.0), 482 | models.TestModel(int_field=2.0, float_field=3.0), 483 | models.TestModel(int_field=3, float_field=3.0), 484 | models.TestModel(int_field=4, float_field=3.0) 485 | ] 486 | return_values = models.TestModel.objects.bulk_upsert( 487 | model_objects, 488 | ['int_field'], 489 | ['float_field'], 490 | return_upserts=True, 491 | native=True 492 | ) 493 | 494 | self.assertEqual(len(return_values), 4) 495 | for test_model, expected_int in zip(sorted(return_values, key=lambda k: k.int_field), [1, 2, 3, 4]): 496 | self.assertEqual(test_model.int_field, expected_int) 497 | self.assertAlmostEqual(test_model.float_field, 3.0) 498 | self.assertIsNotNone(test_model.id) 499 | self.assertEqual(models.TestModel.objects.count(), 4) 500 | 501 | def test_return_created_updated_values_distinct(self): 502 | """ 503 | Tests returning distinct sets of values when the items are either updated or created. 504 | """ 505 | # Create an item that will be updated 506 | G(models.TestModel, int_field=2, float_field=1.0) 507 | model_objects = [ 508 | models.TestModel(int_field=1, float_field=3.0), 509 | models.TestModel(int_field=2.0, float_field=3.0), 510 | models.TestModel(int_field=3, float_field=3.0), 511 | models.TestModel(int_field=4, float_field=3.0) 512 | ] 513 | updated, created = models.TestModel.objects.bulk_upsert( 514 | model_objects, ['int_field'], ['float_field'], return_upserts_distinct=True) 515 | self.assertEqual( 516 | [(2, 3.0)], 517 | [ 518 | (obj.int_field, obj.float_field) 519 | for obj in sorted(updated, key=lambda k: k.int_field) 520 | ] 521 | ) 522 | self.assertEqual( 523 | [(1, 3.0), (3, 3.0), (4, 3.0)], 524 | [ 525 | (obj.int_field, obj.float_field) 526 | for obj in sorted(created, key=lambda k: k.int_field) 527 | ] 528 | ) 529 | 530 | def test_wo_unique_fields(self): 531 | """ 532 | Tests bulk_upsert with no unique fields. A ValueError should be raised since it is required to provide a 533 | list of unique_fields. 534 | """ 535 | with self.assertRaises(ValueError): 536 | models.TestModel.objects.bulk_upsert([], [], ['field']) 537 | 538 | def test_wo_update_fields(self): 539 | """ 540 | Tests bulk_upsert with no update fields. This function in turn should just do a bulk create for any 541 | models that do not already exist. 542 | """ 543 | # Create models that already exist 544 | G(models.TestModel, int_field=1) 545 | G(models.TestModel, int_field=2) 546 | # Perform a bulk_upsert with one new model 547 | models.TestModel.objects.bulk_upsert([ 548 | models.TestModel(int_field=1), models.TestModel(int_field=2), models.TestModel(int_field=3) 549 | ], ['int_field']) 550 | # Three objects should now exist 551 | self.assertEqual(models.TestModel.objects.count(), 3) 552 | for test_model, expected_int_value in zip(models.TestModel.objects.order_by('int_field'), [1, 2, 3]): 553 | self.assertEqual(test_model.int_field, expected_int_value) 554 | 555 | def test_wo_update_fields_native(self): 556 | """ 557 | Tests bulk_upsert with no update fields. This function in turn should just do a bulk create for any 558 | models that do not already exist. 559 | """ 560 | # Create models that already exist 561 | G(models.TestModel, int_field=1) 562 | G(models.TestModel, int_field=2) 563 | # Perform a bulk_upsert with one new model 564 | models.TestModel.objects.bulk_upsert( 565 | [ 566 | models.TestModel(int_field=1), models.TestModel(int_field=2), models.TestModel(int_field=3) 567 | ], 568 | ['int_field'], 569 | native=True 570 | ) 571 | # Three objects should now exist 572 | self.assertEqual(models.TestModel.objects.count(), 3) 573 | for test_model, expected_int_value in zip(models.TestModel.objects.order_by('int_field'), [1, 2, 3]): 574 | self.assertEqual(test_model.int_field, expected_int_value) 575 | 576 | def test_w_blank_arguments(self): 577 | """ 578 | Tests using required arguments and using blank arguments for everything else. 579 | """ 580 | models.TestModel.objects.bulk_upsert([], ['field'], ['field']) 581 | self.assertEqual(models.TestModel.objects.count(), 0) 582 | 583 | # Test native 584 | models.TestModel.objects.bulk_upsert([], ['field'], ['field'], native=True) 585 | self.assertEqual(models.TestModel.objects.count(), 0) 586 | 587 | def test_w_blank_arguments_native(self): 588 | """ 589 | Tests using required arguments and using blank arguments for everything else. 590 | """ 591 | models.TestModel.objects.bulk_upsert([], ['field'], ['field'], native=True) 592 | self.assertEqual(models.TestModel.objects.count(), 0) 593 | 594 | def test_no_updates(self): 595 | """ 596 | Tests the case when no updates were previously stored (i.e objects are only created) 597 | """ 598 | models.TestModel.objects.bulk_upsert([ 599 | models.TestModel(int_field=0, char_field='0', float_field=0), 600 | models.TestModel(int_field=1, char_field='1', float_field=1), 601 | models.TestModel(int_field=2, char_field='2', float_field=2), 602 | ], ['int_field'], ['char_field', 'float_field']) 603 | 604 | for i, model_obj in enumerate(models.TestModel.objects.order_by('int_field')): 605 | self.assertEqual(model_obj.int_field, i) 606 | self.assertEqual(model_obj.char_field, str(i)) 607 | self.assertAlmostEqual(model_obj.float_field, i) 608 | 609 | def test_no_updates_native(self): 610 | """ 611 | Tests the case when no updates were previously stored (i.e objects are only created) 612 | """ 613 | models.TestModel.objects.bulk_upsert([ 614 | models.TestModel(int_field=0, char_field='0', float_field=0), 615 | models.TestModel(int_field=1, char_field='1', float_field=1), 616 | models.TestModel(int_field=2, char_field='2', float_field=2), 617 | ], ['int_field'], ['char_field', 'float_field'], native=True) 618 | 619 | for i, model_obj in enumerate(models.TestModel.objects.order_by('int_field')): 620 | self.assertEqual(model_obj.int_field, i) 621 | self.assertEqual(model_obj.char_field, str(i)) 622 | self.assertAlmostEqual(model_obj.float_field, i) 623 | 624 | def test_all_updates_unique_int_field(self): 625 | """ 626 | Tests the case when all updates were previously stored and the int field is used as a uniqueness 627 | constraint. 628 | """ 629 | # Create previously stored test models with a unique int field and -1 for all other fields 630 | for i in range(3): 631 | G(models.TestModel, int_field=i, char_field='-1', float_field=-1) 632 | 633 | # Update using the int field as a uniqueness constraint 634 | models.TestModel.objects.bulk_upsert([ 635 | models.TestModel(int_field=0, char_field='0', float_field=0), 636 | models.TestModel(int_field=1, char_field='1', float_field=1), 637 | models.TestModel(int_field=2, char_field='2', float_field=2), 638 | ], ['int_field'], ['char_field', 'float_field']) 639 | 640 | # Verify that the fields were updated 641 | self.assertEqual(models.TestModel.objects.count(), 3) 642 | for i, model_obj in enumerate(models.TestModel.objects.order_by('int_field')): 643 | self.assertEqual(model_obj.int_field, i) 644 | self.assertEqual(model_obj.char_field, str(i)) 645 | self.assertAlmostEqual(model_obj.float_field, i) 646 | 647 | def test_all_updates_unique_int_field_native(self): 648 | """ 649 | Tests the case when all updates were previously stored and the int field is used as a uniqueness 650 | constraint. 651 | """ 652 | # Create previously stored test models with a unique int field and -1 for all other fields 653 | for i in range(3): 654 | G(models.TestModel, int_field=i, char_field='-1', float_field=-1) 655 | 656 | # Update using the int field as a uniqueness constraint 657 | models.TestModel.objects.bulk_upsert([ 658 | models.TestModel(int_field=0, char_field='0', float_field=0), 659 | models.TestModel(int_field=1, char_field='1', float_field=1), 660 | models.TestModel(int_field=2, char_field='2', float_field=2), 661 | ], ['int_field'], ['char_field', 'float_field'], native=True) 662 | 663 | # Verify that the fields were updated 664 | self.assertEqual(models.TestModel.objects.count(), 3) 665 | for i, model_obj in enumerate(models.TestModel.objects.order_by('int_field')): 666 | self.assertEqual(model_obj.int_field, i) 667 | self.assertEqual(model_obj.char_field, str(i)) 668 | self.assertAlmostEqual(model_obj.float_field, i) 669 | 670 | def test_all_updates_unique_int_field_update_float_field(self): 671 | """ 672 | Tests the case when all updates were previously stored and the int field is used as a uniqueness 673 | constraint. Only updates the float field 674 | """ 675 | # Create previously stored test models with a unique int field and -1 for all other fields 676 | for i in range(3): 677 | G(models.TestModel, int_field=i, char_field='-1', float_field=-1) 678 | 679 | # Update using the int field as a uniqueness constraint 680 | models.TestModel.objects.bulk_upsert([ 681 | models.TestModel(int_field=0, char_field='0', float_field=0), 682 | models.TestModel(int_field=1, char_field='1', float_field=1), 683 | models.TestModel(int_field=2, char_field='2', float_field=2), 684 | ], ['int_field'], update_fields=['float_field']) 685 | 686 | # Verify that the float field was updated 687 | self.assertEqual(models.TestModel.objects.count(), 3) 688 | for i, model_obj in enumerate(models.TestModel.objects.order_by('int_field')): 689 | self.assertEqual(model_obj.int_field, i) 690 | self.assertEqual(model_obj.char_field, '-1') 691 | self.assertAlmostEqual(model_obj.float_field, i) 692 | 693 | def test_all_updates_unique_int_field_update_float_field_native(self): 694 | """ 695 | Tests the case when all updates were previously stored and the int field is used as a uniqueness 696 | constraint. Only updates the float field 697 | """ 698 | # Create previously stored test models with a unique int field and -1 for all other fields 699 | for i in range(3): 700 | G(models.TestModel, int_field=i, char_field='-1', float_field=-1) 701 | 702 | # Update using the int field as a uniqueness constraint 703 | models.TestModel.objects.bulk_upsert([ 704 | models.TestModel(int_field=0, char_field='0', float_field=0), 705 | models.TestModel(int_field=1, char_field='1', float_field=1), 706 | models.TestModel(int_field=2, char_field='2', float_field=2), 707 | ], ['int_field'], update_fields=['float_field'], native=True) 708 | 709 | # Verify that the float field was updated 710 | self.assertEqual(models.TestModel.objects.count(), 3) 711 | for i, model_obj in enumerate(models.TestModel.objects.order_by('int_field')): 712 | self.assertEqual(model_obj.int_field, i) 713 | self.assertEqual(model_obj.char_field, '-1') 714 | self.assertAlmostEqual(model_obj.float_field, i) 715 | 716 | def test_some_updates_unique_int_field_update_float_field(self): 717 | """ 718 | Tests the case when some updates were previously stored and the int field is used as a uniqueness 719 | constraint. Only updates the float field. 720 | """ 721 | # Create previously stored test models with a unique int field and -1 for all other fields 722 | for i in range(2): 723 | G(models.TestModel, int_field=i, char_field='-1', float_field=-1) 724 | 725 | # Update using the int field as a uniqueness constraint. The first two are updated while the third is created 726 | models.TestModel.objects.bulk_upsert([ 727 | models.TestModel(int_field=0, char_field='0', float_field=0), 728 | models.TestModel(int_field=1, char_field='1', float_field=1), 729 | models.TestModel(int_field=2, char_field='2', float_field=2), 730 | ], ['int_field'], ['float_field']) 731 | 732 | # Verify that the float field was updated for the first two models and the char field was not updated for 733 | # the first two. The char field, however, should be '2' for the third model since it was created 734 | self.assertEqual(models.TestModel.objects.count(), 3) 735 | for i, model_obj in enumerate(models.TestModel.objects.order_by('int_field')): 736 | self.assertEqual(model_obj.int_field, i) 737 | self.assertEqual(model_obj.char_field, '-1' if i < 2 else '2') 738 | self.assertAlmostEqual(model_obj.float_field, i) 739 | 740 | def test_some_updates_unique_int_field_update_float_field_native(self): 741 | """ 742 | Tests the case when some updates were previously stored and the int field is used as a uniqueness 743 | constraint. Only updates the float field. 744 | """ 745 | # Create previously stored test models with a unique int field and -1 for all other fields 746 | for i in range(2): 747 | G(models.TestModel, int_field=i, char_field='-1', float_field=-1) 748 | 749 | # Update using the int field as a uniqueness constraint. The first two are updated while the third is created 750 | models.TestModel.objects.bulk_upsert([ 751 | models.TestModel(int_field=0, char_field='0', float_field=0), 752 | models.TestModel(int_field=1, char_field='1', float_field=1), 753 | models.TestModel(int_field=2, char_field='2', float_field=2), 754 | ], ['int_field'], ['float_field'], native=True) 755 | 756 | # Verify that the float field was updated for the first two models and the char field was not updated for 757 | # the first two. The char field, however, should be '2' for the third model since it was created 758 | self.assertEqual(models.TestModel.objects.count(), 3) 759 | for i, model_obj in enumerate(models.TestModel.objects.order_by('int_field')): 760 | self.assertEqual(model_obj.int_field, i) 761 | self.assertEqual(model_obj.char_field, '-1' if i < 2 else '2') 762 | self.assertAlmostEqual(model_obj.float_field, i) 763 | 764 | def test_some_updates_unique_timezone_field_update_float_field(self): 765 | """ 766 | Tests the case when some updates were previously stored and the timezone field is used as a uniqueness 767 | constraint. Only updates the float field. 768 | """ 769 | # Create previously stored test models with a unique int field and -1 for all other fields 770 | for i in ['US/Eastern', 'US/Central']: 771 | G(models.TestModel, time_zone=i, char_field='-1', float_field=-1) 772 | 773 | # Update using the int field as a uniqueness constraint. The first two are updated while the third is created 774 | models.TestModel.objects.bulk_upsert([ 775 | models.TestModel(time_zone=timezone('US/Eastern'), char_field='0', float_field=0), 776 | models.TestModel(time_zone=timezone('US/Central'), char_field='1', float_field=1), 777 | models.TestModel(time_zone=timezone('UTC'), char_field='2', float_field=2), 778 | ], ['time_zone'], ['float_field']) 779 | 780 | # Verify that the float field was updated for the first two models and the char field was not updated for 781 | # the first two. The char field, however, should be '2' for the third model since it was created 782 | m1 = models.TestModel.objects.get(time_zone=timezone('US/Eastern')) 783 | self.assertEqual(m1.char_field, '-1') 784 | self.assertAlmostEqual(m1.float_field, 0) 785 | 786 | m2 = models.TestModel.objects.get(time_zone=timezone('US/Central')) 787 | self.assertEqual(m2.char_field, '-1') 788 | self.assertAlmostEqual(m2.float_field, 1) 789 | 790 | m3 = models.TestModel.objects.get(time_zone=timezone('UTC')) 791 | self.assertEqual(m3.char_field, '2') 792 | self.assertAlmostEqual(m3.float_field, 2) 793 | 794 | def test_some_updates_unique_int_char_field_update_float_field(self): 795 | """ 796 | Tests the case when some updates were previously stored and the int and char fields are used as a uniqueness 797 | constraint. Only updates the float field. 798 | """ 799 | # Create previously stored test models with a unique int and char field 800 | for i in range(2): 801 | G(models.TestModel, int_field=i, char_field=str(i), float_field=-1) 802 | 803 | # Update using the int field as a uniqueness constraint. The first two are updated while the third is created 804 | models.TestModel.objects.bulk_upsert([ 805 | models.TestModel(int_field=0, char_field='0', float_field=0), 806 | models.TestModel(int_field=1, char_field='1', float_field=1), 807 | models.TestModel(int_field=2, char_field='2', float_field=2), 808 | ], ['int_field', 'char_field'], ['float_field']) 809 | 810 | # Verify that the float field was updated for the first two models and the char field was not updated for 811 | # the first two. The char field, however, should be '2' for the third model since it was created 812 | self.assertEqual(models.TestModel.objects.count(), 3) 813 | for i, model_obj in enumerate(models.TestModel.objects.order_by('int_field')): 814 | self.assertEqual(model_obj.int_field, i) 815 | self.assertEqual(model_obj.char_field, str(i)) 816 | self.assertAlmostEqual(model_obj.float_field, i) 817 | 818 | def test_some_updates_unique_int_char_field_update_float_field_native(self): 819 | """ 820 | Tests the case when some updates were previously stored and the int and char fields are used as a uniqueness 821 | constraint. Only updates the float field. 822 | """ 823 | # Create previously stored test models with a unique int and char field 824 | for i in range(2): 825 | G(models.TestModel, int_field=i, char_field=str(i), float_field=-1) 826 | 827 | # Update using the int field as a uniqueness constraint. The first two are updated while the third is created 828 | models.TestModel.objects.bulk_upsert([ 829 | models.TestModel(int_field=0, char_field='0', float_field=0), 830 | models.TestModel(int_field=1, char_field='1', float_field=1), 831 | models.TestModel(int_field=2, char_field='2', float_field=2), 832 | ], ['int_field', 'char_field'], ['float_field'], native=True) 833 | 834 | # Verify that the float field was updated for the first two models and the char field was not updated for 835 | # the first two. The char field, however, should be '2' for the third model since it was created 836 | self.assertEqual(models.TestModel.objects.count(), 3) 837 | for i, model_obj in enumerate(models.TestModel.objects.order_by('int_field')): 838 | self.assertEqual(model_obj.int_field, i) 839 | self.assertEqual(model_obj.char_field, str(i)) 840 | self.assertAlmostEqual(model_obj.float_field, i) 841 | 842 | def test_no_updates_unique_int_char_field(self): 843 | """ 844 | Tests the case when no updates were previously stored and the int and char fields are used as a uniqueness 845 | constraint. In this case, there is data previously stored, but the uniqueness constraints dont match. 846 | """ 847 | # Create previously stored test models with a unique int field and -1 for all other fields 848 | for i in range(3): 849 | G(models.TestModel, int_field=i, char_field='-1', float_field=-1) 850 | 851 | # Update using the int and char field as a uniqueness constraint. All three objects are created 852 | models.TestModel.objects.bulk_upsert([ 853 | models.TestModel(int_field=3, char_field='0', float_field=0), 854 | models.TestModel(int_field=4, char_field='1', float_field=1), 855 | models.TestModel(int_field=5, char_field='2', float_field=2), 856 | ], ['int_field', 'char_field'], ['float_field']) 857 | 858 | # Verify that no updates occured 859 | self.assertEqual(models.TestModel.objects.count(), 6) 860 | self.assertEqual(models.TestModel.objects.filter(char_field='-1').count(), 3) 861 | for i, model_obj in enumerate(models.TestModel.objects.filter(char_field='-1').order_by('int_field')): 862 | self.assertEqual(model_obj.int_field, i) 863 | self.assertEqual(model_obj.char_field, '-1') 864 | self.assertAlmostEqual(model_obj.float_field, -1) 865 | self.assertEqual(models.TestModel.objects.exclude(char_field='-1').count(), 3) 866 | for i, model_obj in enumerate(models.TestModel.objects.exclude(char_field='-1').order_by('int_field')): 867 | self.assertEqual(model_obj.int_field, i + 3) 868 | self.assertEqual(model_obj.char_field, str(i)) 869 | self.assertAlmostEqual(model_obj.float_field, i) 870 | 871 | def test_no_updates_unique_int_char_field_native(self): 872 | """ 873 | Tests the case when no updates were previously stored and the int and char fields are used as a uniqueness 874 | constraint. In this case, there is data previously stored, but the uniqueness constraints dont match. 875 | """ 876 | # Create previously stored test models with a unique int field and -1 for all other fields 877 | for i in range(3): 878 | G(models.TestModel, int_field=i, char_field='-1', float_field=-1) 879 | 880 | # Update using the int and char field as a uniqueness constraint. All three objects are created 881 | models.TestModel.objects.bulk_upsert([ 882 | models.TestModel(int_field=3, char_field='0', float_field=0), 883 | models.TestModel(int_field=4, char_field='1', float_field=1), 884 | models.TestModel(int_field=5, char_field='2', float_field=2), 885 | ], ['int_field', 'char_field'], ['float_field'], native=True) 886 | 887 | # Verify that no updates occured 888 | self.assertEqual(models.TestModel.objects.count(), 6) 889 | self.assertEqual(models.TestModel.objects.filter(char_field='-1').count(), 3) 890 | for i, model_obj in enumerate(models.TestModel.objects.filter(char_field='-1').order_by('int_field')): 891 | self.assertEqual(model_obj.int_field, i) 892 | self.assertEqual(model_obj.char_field, '-1') 893 | self.assertAlmostEqual(model_obj.float_field, -1) 894 | self.assertEqual(models.TestModel.objects.exclude(char_field='-1').count(), 3) 895 | for i, model_obj in enumerate(models.TestModel.objects.exclude(char_field='-1').order_by('int_field')): 896 | self.assertEqual(model_obj.int_field, i + 3) 897 | self.assertEqual(model_obj.char_field, str(i)) 898 | self.assertAlmostEqual(model_obj.float_field, i) 899 | 900 | def test_some_updates_unique_int_char_field_queryset(self): 901 | """ 902 | Tests the case when some updates were previously stored and a queryset is used on the bulk upsert. 903 | """ 904 | # Create previously stored test models with a unique int field and -1 for all other fields 905 | for i in range(3): 906 | G(models.TestModel, int_field=i, char_field='-1', float_field=-1) 907 | 908 | # Update using the int field as a uniqueness constraint on a queryset. Only one object should be updated. 909 | models.TestModel.objects.filter(int_field=0).bulk_upsert([ 910 | models.TestModel(int_field=0, char_field='0', float_field=0), 911 | models.TestModel(int_field=4, char_field='1', float_field=1), 912 | models.TestModel(int_field=5, char_field='2', float_field=2), 913 | ], ['int_field'], ['float_field']) 914 | 915 | # Verify that two new objecs were created 916 | self.assertEqual(models.TestModel.objects.count(), 5) 917 | self.assertEqual(models.TestModel.objects.filter(char_field='-1').count(), 3) 918 | for i, model_obj in enumerate(models.TestModel.objects.filter(char_field='-1').order_by('int_field')): 919 | self.assertEqual(model_obj.int_field, i) 920 | self.assertEqual(model_obj.char_field, '-1') 921 | 922 | def test_some_updates_unique_int_char_field_queryset_native(self): 923 | """ 924 | Tests the case when some updates were previously stored and a queryset is used on the bulk upsert. 925 | """ 926 | # Create previously stored test models with a unique int field and -1 for all other fields 927 | for i in range(3): 928 | G(models.TestModel, int_field=i, char_field='-1', float_field=-1) 929 | 930 | # Update using the int field as a uniqueness constraint on a queryset. Only one object should be updated. 931 | models.TestModel.objects.filter(int_field=0).bulk_upsert([ 932 | models.TestModel(int_field=0, char_field='0', float_field=0), 933 | models.TestModel(int_field=4, char_field='1', float_field=1), 934 | models.TestModel(int_field=5, char_field='2', float_field=2), 935 | ], ['int_field'], ['float_field'], native=True) 936 | 937 | # Verify that two new objecs were created 938 | self.assertEqual(models.TestModel.objects.count(), 5) 939 | self.assertEqual(models.TestModel.objects.filter(char_field='-1').count(), 3) 940 | for i, model_obj in enumerate(models.TestModel.objects.filter(char_field='-1').order_by('int_field')): 941 | self.assertEqual(model_obj.int_field, i) 942 | self.assertEqual(model_obj.char_field, '-1') 943 | 944 | 945 | class BulkUpsert2Test(TestCase): 946 | """ 947 | Tests the bulk_upsert2 function. 948 | """ 949 | def test_return_upserts_none(self): 950 | """ 951 | Tests the return_upserts flag on bulk upserts when there is no data. 952 | """ 953 | return_values = models.TestModel.objects.bulk_upsert2([], ['float_field'], ['float_field'], returning=True) 954 | self.assertEqual(return_values, []) 955 | 956 | def test_return_multi_unique_fields_not_supported(self): 957 | """ 958 | The new manager utils supports returning bulk upserts when there are multiple unique fields. 959 | """ 960 | return_values = models.TestModel.objects.bulk_upsert2([], ['float_field', 'int_field'], ['float_field'], 961 | returning=True) 962 | self.assertEqual(return_values, []) 963 | 964 | def test_return_created_values(self): 965 | """ 966 | Tests that values that are created are returned properly when returning is True. 967 | """ 968 | results = models.TestModel.objects.bulk_upsert2( 969 | [models.TestModel(int_field=1), models.TestModel(int_field=3), models.TestModel(int_field=4)], 970 | ['int_field'], ['float_field'], returning=True 971 | ) 972 | 973 | self.assertEqual(len(list(results.created)), 3) 974 | for test_model, expected_int in zip(sorted(results.created, key=lambda k: k.int_field), [1, 3, 4]): 975 | self.assertEqual(test_model.int_field, expected_int) 976 | self.assertIsNotNone(test_model.id) 977 | self.assertEqual(models.TestModel.objects.count(), 3) 978 | 979 | def test_return_list_of_values(self): 980 | """ 981 | Tests that values that are created are returned properly when returning is True. 982 | Set returning to a list of fields 983 | """ 984 | results = models.TestModel.objects.bulk_upsert2( 985 | [models.TestModel(int_field=1, float_field=2), 986 | models.TestModel(int_field=3, float_field=4), 987 | models.TestModel(int_field=4, float_field=5)], 988 | ['int_field'], ['float_field'], returning=['float_field'] 989 | ) 990 | 991 | self.assertEqual(len(list(results.created)), 3) 992 | with self.assertRaises(AttributeError): 993 | list(results.created)[0].int_field 994 | self.assertEqual(set([2, 4, 5]), set([m.float_field for m in results.created])) 995 | 996 | def test_return_created_updated_values(self): 997 | """ 998 | Tests returning values when the items are either updated or created. 999 | """ 1000 | # Create an item that will be updated 1001 | G(models.TestModel, int_field=2, float_field=1.0) 1002 | results = models.TestModel.objects.bulk_upsert2( 1003 | [ 1004 | models.TestModel(int_field=1, float_field=3.0), models.TestModel(int_field=2.0, float_field=3.0), 1005 | models.TestModel(int_field=3, float_field=3.0), models.TestModel(int_field=4, float_field=3.0) 1006 | ], 1007 | ['int_field'], ['float_field'], returning=True) 1008 | 1009 | created = list(results.created) 1010 | updated = list(results.updated) 1011 | self.assertEqual(len(created), 3) 1012 | self.assertEqual(len(updated), 1) 1013 | for test_model, expected_int in zip(sorted(created, key=lambda k: k.int_field), [1, 3, 4]): 1014 | self.assertEqual(test_model.int_field, expected_int) 1015 | self.assertAlmostEqual(test_model.float_field, 3.0) 1016 | self.assertIsNotNone(test_model.id) 1017 | 1018 | self.assertEqual(updated[0].int_field, 2) 1019 | self.assertAlmostEqual(updated[0].float_field, 3.0) 1020 | self.assertIsNotNone(updated[0].id) 1021 | self.assertEqual(models.TestModel.objects.count(), 4) 1022 | 1023 | def test_created_updated_auto_datetime_values(self): 1024 | """ 1025 | Tests when the items are either updated or created when auto_now 1026 | and auto_now_add datetime values are used 1027 | """ 1028 | # Create an item that will be updated 1029 | with freezegun.freeze_time('2018-09-01 00:00:00'): 1030 | G(models.TestAutoDateTimeModel, int_field=1) 1031 | 1032 | with freezegun.freeze_time('2018-09-02 00:00:00'): 1033 | results = models.TestAutoDateTimeModel.objects.bulk_upsert2( 1034 | [ 1035 | models.TestAutoDateTimeModel(int_field=1), 1036 | models.TestAutoDateTimeModel(int_field=2), 1037 | models.TestAutoDateTimeModel(int_field=3), 1038 | models.TestAutoDateTimeModel(int_field=4) 1039 | ], 1040 | ['int_field'], returning=True) 1041 | 1042 | self.assertEqual(len(list(results.created)), 3) 1043 | self.assertEqual(len(list(results.updated)), 1) 1044 | 1045 | expected_auto_now = [dt.datetime(2018, 9, 2), dt.datetime(2018, 9, 2), 1046 | dt.datetime(2018, 9, 2), dt.datetime(2018, 9, 2)] 1047 | expected_auto_now_add = [dt.datetime(2018, 9, 1), dt.datetime(2018, 9, 2), 1048 | dt.datetime(2018, 9, 2), dt.datetime(2018, 9, 2)] 1049 | for i, test_model in enumerate(sorted(results, key=lambda k: k.int_field)): 1050 | self.assertEqual(test_model.auto_now_field, expected_auto_now[i]) 1051 | self.assertEqual(test_model.auto_now_add_field, expected_auto_now_add[i]) 1052 | 1053 | def test_wo_update_fields(self): 1054 | """ 1055 | Tests bulk_upsert with no update fields. This function in turn should just do a bulk create for any 1056 | models that do not already exist. 1057 | """ 1058 | # Create models that already exist 1059 | G(models.TestModel, int_field=1, float_field=1) 1060 | G(models.TestModel, int_field=2, float_field=2) 1061 | # Perform a bulk_upsert with one new model 1062 | models.TestModel.objects.bulk_upsert2([ 1063 | models.TestModel(int_field=1, float_field=3), 1064 | models.TestModel(int_field=2, float_field=3), 1065 | models.TestModel(int_field=3, float_field=3) 1066 | ], ['int_field'], update_fields=[]) 1067 | # Three objects should now exist, but no float fields should be updated 1068 | self.assertEqual(models.TestModel.objects.count(), 3) 1069 | for test_model, expected_int_value in zip(models.TestModel.objects.order_by('int_field'), [1, 2, 3]): 1070 | self.assertEqual(test_model.int_field, expected_int_value) 1071 | self.assertEqual(test_model.float_field, expected_int_value) 1072 | 1073 | def test_w_blank_arguments(self): 1074 | """ 1075 | Tests using required arguments and using blank arguments for everything else. 1076 | """ 1077 | models.TestModel.objects.bulk_upsert2([], ['field'], ['field']) 1078 | self.assertEqual(models.TestModel.objects.count(), 0) 1079 | 1080 | def test_no_updates(self): 1081 | """ 1082 | Tests the case when no updates were previously stored (i.e objects are only created) 1083 | """ 1084 | models.TestModel.objects.bulk_upsert([ 1085 | models.TestModel(int_field=0, char_field='0', float_field=0), 1086 | models.TestModel(int_field=1, char_field='1', float_field=1), 1087 | models.TestModel(int_field=2, char_field='2', float_field=2), 1088 | ], ['int_field'], ['char_field', 'float_field']) 1089 | 1090 | for i, model_obj in enumerate(models.TestModel.objects.order_by('int_field')): 1091 | self.assertEqual(model_obj.int_field, i) 1092 | self.assertEqual(model_obj.char_field, str(i)) 1093 | self.assertAlmostEqual(model_obj.float_field, i) 1094 | 1095 | def test_update_fields_returning(self): 1096 | """ 1097 | Tests the case when all updates were previously stored and the int field is used as a uniqueness 1098 | constraint. Assert returned values are expected and that it updates all fields by default 1099 | """ 1100 | # Create previously stored test models with a unique int field and -1 for all other fields 1101 | test_models = [ 1102 | G(models.TestModel, int_field=i, char_field='-1', float_field=-1) 1103 | for i in range(3) 1104 | ] 1105 | 1106 | # Update using the int field as a uniqueness constraint 1107 | results = models.TestModel.objects.bulk_upsert2([ 1108 | models.TestModel(int_field=0, char_field='0', float_field=0), 1109 | models.TestModel(int_field=1, char_field='1', float_field=1), 1110 | models.TestModel(int_field=2, char_field='2', float_field=2), 1111 | ], ['int_field'], returning=True) 1112 | 1113 | self.assertEqual(list(results.created), []) 1114 | self.assertEqual(set([u.id for u in results.updated]), set([t.id for t in test_models])) 1115 | self.assertEqual(set([u.int_field for u in results.updated]), set([0, 1, 2])) 1116 | self.assertEqual(set([u.float_field for u in results.updated]), set([0, 1, 2])) 1117 | self.assertEqual(set([u.char_field for u in results.updated]), set(['0', '1', '2'])) 1118 | 1119 | def test_no_update_fields_returning(self): 1120 | """ 1121 | Tests the case when all updates were previously stored and the int field is used as a uniqueness 1122 | constraint. This test does not update any fields 1123 | """ 1124 | # Create previously stored test models with a unique int field and -1 for all other fields 1125 | for i in range(3): 1126 | G(models.TestModel, int_field=i, char_field='-1', float_field=-1) 1127 | 1128 | # Update using the int field as a uniqueness constraint 1129 | results = models.TestModel.objects.bulk_upsert2([ 1130 | models.TestModel(int_field=0, char_field='0', float_field=0), 1131 | models.TestModel(int_field=1, char_field='1', float_field=1), 1132 | models.TestModel(int_field=2, char_field='2', float_field=2), 1133 | ], ['int_field'], [], returning=True) 1134 | 1135 | self.assertEqual(list(results), []) 1136 | 1137 | def test_update_duplicate_fields_returning_none_updated(self): 1138 | """ 1139 | Tests the case when all updates were previously stored and the upsert tries to update the rows 1140 | with duplicate values. 1141 | """ 1142 | # Create previously stored test models with a unique int field and -1 for all other fields 1143 | for i in range(3): 1144 | G(models.TestModel, int_field=i, char_field='-1', float_field=-1) 1145 | 1146 | # Update using the int field as a uniqueness constraint 1147 | results = models.TestModel.objects.bulk_upsert2([ 1148 | models.TestModel(int_field=0, char_field='-1', float_field=-1), 1149 | models.TestModel(int_field=1, char_field='-1', float_field=-1), 1150 | models.TestModel(int_field=2, char_field='-1', float_field=-1), 1151 | ], ['int_field'], ['char_field', 'float_field'], returning=True, ignore_duplicate_updates=True) 1152 | 1153 | self.assertEqual(list(results), []) 1154 | 1155 | def test_update_duplicate_fields_returning_some_updated(self): 1156 | """ 1157 | Tests the case when all updates were previously stored and the upsert tries to update the rows 1158 | with duplicate values. Test when some aren't duplicates 1159 | """ 1160 | # Create previously stored test models with a unique int field and -1 for all other fields 1161 | for i in range(3): 1162 | G(models.TestModel, int_field=i, char_field='-1', float_field=-1) 1163 | 1164 | # Update using the int field as a uniqueness constraint 1165 | results = models.TestModel.objects.bulk_upsert2([ 1166 | models.TestModel(int_field=0, char_field='-1', float_field=-1), 1167 | models.TestModel(int_field=1, char_field='-1', float_field=-1), 1168 | models.TestModel(int_field=2, char_field='0', float_field=-1), 1169 | ], ['int_field'], ['char_field', 'float_field'], returning=['char_field'], ignore_duplicate_updates=True) 1170 | 1171 | self.assertEqual(list(results.created), []) 1172 | self.assertEqual(len(list(results.updated)), 1) 1173 | self.assertEqual(list(results.updated)[0].char_field, '0') 1174 | 1175 | def test_update_duplicate_fields_returning_some_updated_return_untouched(self): 1176 | """ 1177 | Tests the case when all updates were previously stored and the upsert tries to update the rows 1178 | with duplicate values. Test when some aren't duplicates 1179 | """ 1180 | # Create previously stored test models with a unique int field and -1 for all other fields 1181 | for i in range(3): 1182 | G(models.TestModel, int_field=i, char_field='-1', float_field=-1) 1183 | 1184 | # Update using the int field as a uniqueness constraint 1185 | results = models.TestModel.objects.bulk_upsert2( 1186 | [ 1187 | models.TestModel(int_field=0, char_field='-1', float_field=-1), 1188 | models.TestModel(int_field=1, char_field='-1', float_field=-1), 1189 | models.TestModel(int_field=2, char_field='0', float_field=-1), 1190 | models.TestModel(int_field=3, char_field='3', float_field=3), 1191 | ], 1192 | ['int_field'], ['char_field', 'float_field'], 1193 | returning=['char_field'], ignore_duplicate_updates=True, return_untouched=True) 1194 | 1195 | self.assertEqual(len(list(results.updated)), 1) 1196 | self.assertEqual(len(list(results.untouched)), 2) 1197 | self.assertEqual(len(list(results.created)), 1) 1198 | self.assertEqual(list(results.updated)[0].char_field, '0') 1199 | self.assertEqual(list(results.created)[0].char_field, '3') 1200 | 1201 | def test_update_duplicate_fields_returning_some_updated_return_untouched_ignore_dups(self): 1202 | """ 1203 | Tests the case when all updates were previously stored and the upsert tries to update the rows 1204 | with duplicate values. Test when some aren't duplicates and return untouched results. 1205 | There will be no untouched results in this test since we turn off ignoring duplicate 1206 | updates 1207 | """ 1208 | # Create previously stored test models with a unique int field and -1 for all other fields 1209 | for i in range(3): 1210 | G(models.TestModel, int_field=i, char_field='-1', float_field=-1) 1211 | 1212 | # Update using the int field as a uniqueness constraint 1213 | results = models.TestModel.objects.bulk_upsert2( 1214 | [ 1215 | models.TestModel(int_field=0, char_field='-1', float_field=-1), 1216 | models.TestModel(int_field=1, char_field='-1', float_field=-1), 1217 | models.TestModel(int_field=2, char_field='0', float_field=-1), 1218 | models.TestModel(int_field=3, char_field='3', float_field=3), 1219 | ], 1220 | ['int_field'], ['char_field', 'float_field'], 1221 | returning=['char_field'], ignore_duplicate_updates=False, return_untouched=True) 1222 | 1223 | self.assertEqual(len(list(results.untouched)), 0) 1224 | self.assertEqual(len(list(results.updated)), 3) 1225 | self.assertEqual(len(list(results.created)), 1) 1226 | self.assertEqual(list(results.created)[0].char_field, '3') 1227 | 1228 | def test_all_updates_unique_int_field(self): 1229 | """ 1230 | Tests the case when all updates were previously stored and the int field is used as a uniqueness 1231 | constraint. 1232 | """ 1233 | # Create previously stored test models with a unique int field and -1 for all other fields 1234 | for i in range(3): 1235 | G(models.TestModel, int_field=i, char_field='-1', float_field=-1) 1236 | 1237 | # Update using the int field as a uniqueness constraint 1238 | models.TestModel.objects.bulk_upsert2([ 1239 | models.TestModel(int_field=0, char_field='0', float_field=0), 1240 | models.TestModel(int_field=1, char_field='1', float_field=1), 1241 | models.TestModel(int_field=2, char_field='2', float_field=2), 1242 | ], ['int_field'], ['char_field', 'float_field']) 1243 | 1244 | # Verify that the fields were updated 1245 | self.assertEqual(models.TestModel.objects.count(), 3) 1246 | for i, model_obj in enumerate(models.TestModel.objects.order_by('int_field')): 1247 | self.assertEqual(model_obj.int_field, i) 1248 | self.assertEqual(model_obj.char_field, str(i)) 1249 | self.assertAlmostEqual(model_obj.float_field, i) 1250 | 1251 | def test_all_updates_unique_int_field_update_float_field(self): 1252 | """ 1253 | Tests the case when all updates were previously stored and the int field is used as a uniqueness 1254 | constraint. Only updates the float field 1255 | """ 1256 | # Create previously stored test models with a unique int field and -1 for all other fields 1257 | for i in range(3): 1258 | G(models.TestModel, int_field=i, char_field='-1', float_field=-1) 1259 | 1260 | # Update using the int field as a uniqueness constraint 1261 | models.TestModel.objects.bulk_upsert2([ 1262 | models.TestModel(int_field=0, char_field='0', float_field=0), 1263 | models.TestModel(int_field=1, char_field='1', float_field=1), 1264 | models.TestModel(int_field=2, char_field='2', float_field=2), 1265 | ], ['int_field'], update_fields=['float_field']) 1266 | 1267 | # Verify that the float field was updated 1268 | self.assertEqual(models.TestModel.objects.count(), 3) 1269 | for i, model_obj in enumerate(models.TestModel.objects.order_by('int_field')): 1270 | self.assertEqual(model_obj.int_field, i) 1271 | self.assertEqual(model_obj.char_field, '-1') 1272 | self.assertAlmostEqual(model_obj.float_field, i) 1273 | 1274 | def test_some_updates_unique_int_field_update_float_field(self): 1275 | """ 1276 | Tests the case when some updates were previously stored and the int field is used as a uniqueness 1277 | constraint. Only updates the float field. 1278 | """ 1279 | # Create previously stored test models with a unique int field and -1 for all other fields 1280 | for i in range(2): 1281 | G(models.TestModel, int_field=i, char_field='-1', float_field=-1) 1282 | 1283 | # Update using the int field as a uniqueness constraint. The first two are updated while the third is created 1284 | models.TestModel.objects.bulk_upsert2([ 1285 | models.TestModel(int_field=0, char_field='0', float_field=0), 1286 | models.TestModel(int_field=1, char_field='1', float_field=1), 1287 | models.TestModel(int_field=2, char_field='2', float_field=2), 1288 | ], ['int_field'], ['float_field']) 1289 | 1290 | # Verify that the float field was updated for the first two models and the char field was not updated for 1291 | # the first two. The char field, however, should be '2' for the third model since it was created 1292 | self.assertEqual(models.TestModel.objects.count(), 3) 1293 | for i, model_obj in enumerate(models.TestModel.objects.order_by('int_field')): 1294 | self.assertEqual(model_obj.int_field, i) 1295 | self.assertEqual(model_obj.char_field, '-1' if i < 2 else '2') 1296 | self.assertAlmostEqual(model_obj.float_field, i) 1297 | 1298 | def test_some_updates_unique_timezone_field_update_float_field(self): 1299 | """ 1300 | Tests the case when some updates were previously stored and the timezone field is used as a uniqueness 1301 | constraint. Only updates the float field. 1302 | """ 1303 | # Create previously stored test models with a unique int field and -1 for all other fields 1304 | for i in ['US/Eastern', 'US/Central']: 1305 | G(models.TestUniqueTzModel, time_zone=i, char_field='-1', float_field=-1) 1306 | 1307 | # Update using the int field as a uniqueness constraint. The first two are updated while the third is created 1308 | models.TestUniqueTzModel.objects.bulk_upsert2([ 1309 | models.TestModel(time_zone=timezone('US/Eastern'), char_field='0', float_field=0), 1310 | models.TestModel(time_zone=timezone('US/Central'), char_field='1', float_field=1), 1311 | models.TestModel(time_zone=timezone('UTC'), char_field='2', float_field=2), 1312 | ], ['time_zone'], ['float_field']) 1313 | 1314 | # Verify that the float field was updated for the first two models and the char field was not updated for 1315 | # the first two. The char field, however, should be '2' for the third model since it was created 1316 | m1 = models.TestUniqueTzModel.objects.get(time_zone=timezone('US/Eastern')) 1317 | self.assertEqual(m1.char_field, '-1') 1318 | self.assertAlmostEqual(m1.float_field, 0) 1319 | 1320 | m2 = models.TestUniqueTzModel.objects.get(time_zone=timezone('US/Central')) 1321 | self.assertEqual(m2.char_field, '-1') 1322 | self.assertAlmostEqual(m2.float_field, 1) 1323 | 1324 | m3 = models.TestUniqueTzModel.objects.get(time_zone=timezone('UTC')) 1325 | self.assertEqual(m3.char_field, '2') 1326 | self.assertAlmostEqual(m3.float_field, 2) 1327 | 1328 | def test_some_updates_unique_int_char_field_update_float_field(self): 1329 | """ 1330 | Tests the case when some updates were previously stored and the int and char fields are used as a uniqueness 1331 | constraint. Only updates the float field. 1332 | """ 1333 | # Create previously stored test models with a unique int and char field 1334 | for i in range(2): 1335 | G(models.TestModel, int_field=i, char_field=str(i), float_field=-1) 1336 | 1337 | # Update using the int field as a uniqueness constraint. The first two are updated while the third is created 1338 | models.TestModel.objects.bulk_upsert2([ 1339 | models.TestModel(int_field=0, char_field='0', float_field=0), 1340 | models.TestModel(int_field=1, char_field='1', float_field=1), 1341 | models.TestModel(int_field=2, char_field='2', float_field=2), 1342 | ], ['int_field', 'char_field'], ['float_field']) 1343 | 1344 | # Verify that the float field was updated for the first two models and the char field was not updated for 1345 | # the first two. The char field, however, should be '2' for the third model since it was created 1346 | self.assertEqual(models.TestModel.objects.count(), 3) 1347 | for i, model_obj in enumerate(models.TestModel.objects.order_by('int_field')): 1348 | self.assertEqual(model_obj.int_field, i) 1349 | self.assertEqual(model_obj.char_field, str(i)) 1350 | self.assertAlmostEqual(model_obj.float_field, i) 1351 | 1352 | def test_no_updates_unique_int_char_field(self): 1353 | """ 1354 | Tests the case when no updates were previously stored and the int and char fields are used as a uniqueness 1355 | constraint. In this case, there is data previously stored, but the uniqueness constraints dont match. 1356 | """ 1357 | # Create previously stored test models with a unique int field and -1 for all other fields 1358 | for i in range(3): 1359 | G(models.TestModel, int_field=i, char_field='-1', float_field=-1) 1360 | 1361 | # Update using the int and char field as a uniqueness constraint. All three objects are created 1362 | models.TestModel.objects.bulk_upsert2([ 1363 | models.TestModel(int_field=3, char_field='0', float_field=0), 1364 | models.TestModel(int_field=4, char_field='1', float_field=1), 1365 | models.TestModel(int_field=5, char_field='2', float_field=2), 1366 | ], ['int_field', 'char_field'], ['float_field']) 1367 | 1368 | # Verify that no updates occured 1369 | self.assertEqual(models.TestModel.objects.count(), 6) 1370 | self.assertEqual(models.TestModel.objects.filter(char_field='-1').count(), 3) 1371 | for i, model_obj in enumerate(models.TestModel.objects.filter(char_field='-1').order_by('int_field')): 1372 | self.assertEqual(model_obj.int_field, i) 1373 | self.assertEqual(model_obj.char_field, '-1') 1374 | self.assertAlmostEqual(model_obj.float_field, -1) 1375 | self.assertEqual(models.TestModel.objects.exclude(char_field='-1').count(), 3) 1376 | for i, model_obj in enumerate(models.TestModel.objects.exclude(char_field='-1').order_by('int_field')): 1377 | self.assertEqual(model_obj.int_field, i + 3) 1378 | self.assertEqual(model_obj.char_field, str(i)) 1379 | self.assertAlmostEqual(model_obj.float_field, i) 1380 | 1381 | def test_some_updates_unique_int_char_field_queryset(self): 1382 | """ 1383 | Tests the case when some updates were previously stored and a queryset is used on the bulk upsert. 1384 | """ 1385 | # Create previously stored test models with a unique int field and -1 for all other fields 1386 | for i in range(3): 1387 | G(models.TestModel, int_field=i, char_field='-1', float_field=-1) 1388 | 1389 | # Update using the int field as a uniqueness constraint on a queryset. Only one object should be updated. 1390 | models.TestModel.objects.filter(int_field=0).bulk_upsert2([ 1391 | models.TestModel(int_field=0, char_field='0', float_field=0), 1392 | models.TestModel(int_field=4, char_field='1', float_field=1), 1393 | models.TestModel(int_field=5, char_field='2', float_field=2), 1394 | ], ['int_field'], ['float_field']) 1395 | 1396 | # Verify that two new objecs were created 1397 | self.assertEqual(models.TestModel.objects.count(), 5) 1398 | self.assertEqual(models.TestModel.objects.filter(char_field='-1').count(), 3) 1399 | for i, model_obj in enumerate(models.TestModel.objects.filter(char_field='-1').order_by('int_field')): 1400 | self.assertEqual(model_obj.int_field, i) 1401 | self.assertEqual(model_obj.char_field, '-1') 1402 | 1403 | 1404 | class PostBulkOperationSignalTest(TestCase): 1405 | """ 1406 | Tests that the post_bulk_operation signal is emitted on all functions that emit the signal. 1407 | """ 1408 | def setUp(self): 1409 | """ 1410 | Defines a siangl handler that collects information about fired signals 1411 | """ 1412 | class SignalHandler(object): 1413 | num_times_called = 0 1414 | model = None 1415 | 1416 | def __call__(self, *args, **kwargs): 1417 | self.num_times_called += 1 1418 | self.model = kwargs['model'] 1419 | 1420 | self.signal_handler = SignalHandler() 1421 | post_bulk_operation.connect(self.signal_handler) 1422 | 1423 | def tearDown(self): 1424 | """ 1425 | Disconnect the siangl to make sure it doesn't get connected multiple times. 1426 | """ 1427 | post_bulk_operation.disconnect(self.signal_handler) 1428 | 1429 | def test_custom_field_bulk_update(self): 1430 | model_obj = models.TestModel.objects.create(int_field=2) 1431 | model_obj.time_zone = timezone('US/Eastern') 1432 | models.TestModel.objects.bulk_update([model_obj], ['time_zone']) 1433 | model_obj = models.TestModel.objects.get(id=model_obj.id) 1434 | self.assertEqual(model_obj.time_zone, timezone('US/Eastern')) 1435 | 1436 | def test_post_bulk_operation_queryset_update(self): 1437 | """ 1438 | Tests that the update operation on a queryset emits the post_bulk_operation signal. 1439 | """ 1440 | models.TestModel.objects.all().update(int_field=1) 1441 | 1442 | self.assertEqual(self.signal_handler.model, models.TestModel) 1443 | self.assertEqual(self.signal_handler.num_times_called, 1) 1444 | 1445 | def test_post_bulk_operation_manager_update(self): 1446 | """ 1447 | Tests that the update operation on a manager emits the post_bulk_operation signal. 1448 | """ 1449 | models.TestModel.objects.update(int_field=1) 1450 | 1451 | self.assertEqual(self.signal_handler.model, models.TestModel) 1452 | self.assertEqual(self.signal_handler.num_times_called, 1) 1453 | 1454 | def test_post_bulk_operation_bulk_update(self): 1455 | """ 1456 | Tests that the bulk_update operation emits the post_bulk_operation signal. 1457 | """ 1458 | model_obj = models.TestModel.objects.create(int_field=2) 1459 | models.TestModel.objects.bulk_update([model_obj], ['int_field']) 1460 | 1461 | self.assertEqual(self.signal_handler.model, models.TestModel) 1462 | self.assertEqual(self.signal_handler.num_times_called, 1) 1463 | 1464 | def test_post_bulk_operation_bulk_upsert2(self): 1465 | """ 1466 | Tests that the bulk_upsert2 operation emits the post_bulk_operation signal. 1467 | """ 1468 | model_obj = models.TestModel.objects.create(int_field=2) 1469 | models.TestModel.objects.bulk_upsert2([model_obj], ['int_field']) 1470 | 1471 | self.assertEqual(self.signal_handler.model, models.TestModel) 1472 | self.assertEqual(self.signal_handler.num_times_called, 1) 1473 | 1474 | def test_post_bulk_operation_bulk_create(self): 1475 | """ 1476 | Tests that the bulk_create operation emits the post_bulk_operation signal. 1477 | """ 1478 | models.TestModel.objects.bulk_create([models.TestModel(int_field=2)]) 1479 | 1480 | self.assertEqual(self.signal_handler.model, models.TestModel) 1481 | self.assertEqual(self.signal_handler.num_times_called, 1) 1482 | 1483 | def test_post_bulk_operation_bulk_create_queryset(self): 1484 | """ 1485 | Tests that the bulk_create operation emits the post_bulk_operation signal. 1486 | """ 1487 | models.TestModel.objects.all().bulk_create([models.TestModel(int_field=2)]) 1488 | 1489 | self.assertEqual(self.signal_handler.model, models.TestModel) 1490 | self.assertEqual(self.signal_handler.num_times_called, 1) 1491 | 1492 | def test_save_doesnt_emit_signal(self): 1493 | """ 1494 | Tests that a non-bulk operation doesn't emit the signal. 1495 | """ 1496 | model_obj = models.TestModel.objects.create(int_field=2) 1497 | model_obj.save() 1498 | 1499 | self.assertEqual(self.signal_handler.num_times_called, 0) 1500 | 1501 | 1502 | class IdDictTest(TestCase): 1503 | """ 1504 | Tests the id_dict function. 1505 | """ 1506 | def test_no_objects_manager(self): 1507 | """ 1508 | Tests the output when no objects are present in the manager. 1509 | """ 1510 | self.assertEqual(models.TestModel.objects.id_dict(), {}) 1511 | 1512 | def test_objects_manager(self): 1513 | """ 1514 | Tests retrieving a dict of objects keyed on their ID from the manager. 1515 | """ 1516 | model_obj1 = G(models.TestModel, int_field=1) 1517 | model_obj2 = G(models.TestModel, int_field=2) 1518 | self.assertEqual(models.TestModel.objects.id_dict(), {model_obj1.id: model_obj1, model_obj2.id: model_obj2}) 1519 | 1520 | def test_no_objects_queryset(self): 1521 | """ 1522 | Tests the case when no objects are returned via a queryset. 1523 | """ 1524 | G(models.TestModel, int_field=1) 1525 | G(models.TestModel, int_field=2) 1526 | self.assertEqual(models.TestModel.objects.filter(int_field__gte=3).id_dict(), {}) 1527 | 1528 | def test_objects_queryset(self): 1529 | """ 1530 | Tests the case when objects are returned via a queryset. 1531 | """ 1532 | G(models.TestModel, int_field=1) 1533 | model_obj = G(models.TestModel, int_field=2) 1534 | self.assertEqual(models.TestModel.objects.filter(int_field__gte=2).id_dict(), {model_obj.id: model_obj}) 1535 | 1536 | 1537 | class GetOrNoneTest(TestCase): 1538 | """ 1539 | Tests the get_or_none function in the manager utils 1540 | """ 1541 | def test_existing_using_objects(self): 1542 | """ 1543 | Tests get_or_none on an existing object from Model.objects. 1544 | """ 1545 | # Create an existing model 1546 | model_obj = G(models.TestModel) 1547 | # Verify that get_or_none on objects returns the test model 1548 | self.assertEqual(model_obj, models.TestModel.objects.get_or_none(id=model_obj.id)) 1549 | 1550 | def test_multiple_error_using_objects(self): 1551 | """ 1552 | Tests get_or_none on multiple existing objects from Model.objects. 1553 | """ 1554 | # Create an existing model 1555 | model_obj = G(models.TestModel, char_field='hi') 1556 | model_obj = G(models.TestModel, char_field='hi') 1557 | # Verify that get_or_none on objects returns the test model 1558 | with self.assertRaises(models.TestModel.MultipleObjectsReturned): 1559 | self.assertEqual(model_obj, models.TestModel.objects.get_or_none(char_field='hi')) 1560 | 1561 | def test_existing_using_queryset(self): 1562 | """ 1563 | Tests get_or_none on an existing object from a queryst. 1564 | """ 1565 | # Create an existing model 1566 | model_obj = G(models.TestModel) 1567 | # Verify that get_or_none on objects returns the test model 1568 | self.assertEqual(model_obj, models.TestModel.objects.filter(id=model_obj.id).get_or_none(id=model_obj.id)) 1569 | 1570 | def test_none_using_objects(self): 1571 | """ 1572 | Tests when no object exists when using Model.objects. 1573 | """ 1574 | # Verify that get_or_none on objects returns the test model 1575 | self.assertIsNone(models.TestModel.objects.get_or_none(id=1)) 1576 | 1577 | def test_none_using_queryset(self): 1578 | """ 1579 | Tests when no object exists when using a queryset. 1580 | """ 1581 | # Verify that get_or_none on objects returns the test model 1582 | self.assertIsNone(models.TestModel.objects.filter(id=1).get_or_none(id=1)) 1583 | 1584 | 1585 | class SingleTest(TestCase): 1586 | """ 1587 | Tests the single function in the manager utils. 1588 | """ 1589 | def test_none_using_objects(self): 1590 | """ 1591 | Tests when there are no objects using Model.objects. 1592 | """ 1593 | with self.assertRaises(models.TestModel.DoesNotExist): 1594 | models.TestModel.objects.single() 1595 | 1596 | def test_multiple_using_objects(self): 1597 | """ 1598 | Tests when there are multiple objects using Model.objects. 1599 | """ 1600 | G(models.TestModel) 1601 | G(models.TestModel) 1602 | with self.assertRaises(models.TestModel.MultipleObjectsReturned): 1603 | models.TestModel.objects.single() 1604 | 1605 | def test_none_using_queryset(self): 1606 | """ 1607 | Tests when there are no objects using a queryset. 1608 | """ 1609 | with self.assertRaises(models.TestModel.DoesNotExist): 1610 | models.TestModel.objects.filter(id__gte=0).single() 1611 | 1612 | def test_multiple_using_queryset(self): 1613 | """ 1614 | Tests when there are multiple objects using a queryset. 1615 | """ 1616 | G(models.TestModel) 1617 | G(models.TestModel) 1618 | with self.assertRaises(models.TestModel.MultipleObjectsReturned): 1619 | models.TestModel.objects.filter(id__gte=0).single() 1620 | 1621 | def test_single_using_objects(self): 1622 | """ 1623 | Tests accessing a single object using Model.objects. 1624 | """ 1625 | model_obj = G(models.TestModel) 1626 | self.assertEqual(model_obj, models.TestModel.objects.single()) 1627 | 1628 | def test_single_using_queryset(self): 1629 | """ 1630 | Tests accessing a single object using a queryset. 1631 | """ 1632 | model_obj = G(models.TestModel) 1633 | self.assertEqual(model_obj, models.TestModel.objects.filter(id__gte=0).single()) 1634 | 1635 | def test_mutliple_to_single_using_queryset(self): 1636 | """ 1637 | Tests accessing a single object using a queryset. The queryset is what filters it 1638 | down to a single object. 1639 | """ 1640 | model_obj = G(models.TestModel) 1641 | G(models.TestModel) 1642 | self.assertEqual(model_obj, models.TestModel.objects.filter(id=model_obj.id).single()) 1643 | 1644 | 1645 | class BulkUpdateTest(TestCase): 1646 | """ 1647 | Tests the bulk_update function. 1648 | """ 1649 | def test_update_foreign_key_by_id(self): 1650 | t_model = G(models.TestModel) 1651 | t_fk_model = G(models.TestForeignKeyModel) 1652 | t_fk_model.test_model = t_model 1653 | models.TestForeignKeyModel.objects.bulk_update([t_fk_model], ['test_model_id']) 1654 | self.assertEqual(models.TestForeignKeyModel.objects.get().test_model, t_model) 1655 | 1656 | def test_update_foreign_key_by_name(self): 1657 | t_model = G(models.TestModel) 1658 | t_fk_model = G(models.TestForeignKeyModel) 1659 | t_fk_model.test_model = t_model 1660 | models.TestForeignKeyModel.objects.bulk_update([t_fk_model], ['test_model']) 1661 | self.assertEqual(models.TestForeignKeyModel.objects.get().test_model, t_model) 1662 | 1663 | def test_foreign_key_pk_using_id(self): 1664 | """ 1665 | Tests a bulk update on a model that has a primary key to a foreign key. It uses the id of the pk in the 1666 | update 1667 | """ 1668 | t = G(models.TestPkForeignKey, char_field='hi') 1669 | models.TestPkForeignKey.objects.bulk_update( 1670 | [models.TestPkForeignKey(my_key_id=t.my_key_id, char_field='hello')], ['char_field']) 1671 | self.assertEqual(models.TestPkForeignKey.objects.count(), 1) 1672 | self.assertTrue(models.TestPkForeignKey.objects.filter(char_field='hello', my_key=t.my_key).exists()) 1673 | 1674 | def test_foreign_key_pk(self): 1675 | """ 1676 | Tests a bulk update on a model that has a primary key to a foreign key. It uses the foreign key itself 1677 | in the update 1678 | """ 1679 | t = G(models.TestPkForeignKey, char_field='hi') 1680 | models.TestPkForeignKey.objects.bulk_update( 1681 | [models.TestPkForeignKey(my_key=t.my_key, char_field='hello')], ['char_field']) 1682 | self.assertEqual(models.TestPkForeignKey.objects.count(), 1) 1683 | self.assertTrue(models.TestPkForeignKey.objects.filter(char_field='hello', my_key=t.my_key).exists()) 1684 | 1685 | def test_char_pk(self): 1686 | """ 1687 | Tests a bulk update on a model that has a primary key to a char field. 1688 | """ 1689 | G(models.TestPkChar, char_field='hi', my_key='1') 1690 | models.TestPkChar.objects.bulk_update( 1691 | [models.TestPkChar(my_key='1', char_field='hello')], ['char_field']) 1692 | self.assertEqual(models.TestPkChar.objects.count(), 1) 1693 | self.assertTrue(models.TestPkChar.objects.filter(char_field='hello', my_key='1').exists()) 1694 | 1695 | def test_none(self): 1696 | """ 1697 | Tests when no values are provided to bulk update. 1698 | """ 1699 | models.TestModel.objects.bulk_update([], []) 1700 | 1701 | def test_update_floats_to_null(self): 1702 | """ 1703 | Tests updating a float field to a null field. 1704 | """ 1705 | test_obj_1 = G(models.TestModel, int_field=1, float_field=2) 1706 | test_obj_2 = G(models.TestModel, int_field=2, float_field=3) 1707 | test_obj_1.float_field = None 1708 | test_obj_2.float_field = None 1709 | 1710 | models.TestModel.objects.bulk_update([test_obj_1, test_obj_2], ['float_field']) 1711 | 1712 | test_obj_1 = models.TestModel.objects.get(id=test_obj_1.id) 1713 | test_obj_2 = models.TestModel.objects.get(id=test_obj_2.id) 1714 | self.assertIsNone(test_obj_1.float_field) 1715 | self.assertIsNone(test_obj_2.float_field) 1716 | 1717 | def test_update_ints_to_null(self): 1718 | """ 1719 | Tests updating an int field to a null field. 1720 | """ 1721 | test_obj_1 = G(models.TestModel, int_field=1, float_field=2) 1722 | test_obj_2 = G(models.TestModel, int_field=2, float_field=3) 1723 | test_obj_1.int_field = None 1724 | test_obj_2.int_field = None 1725 | 1726 | models.TestModel.objects.bulk_update([test_obj_1, test_obj_2], ['int_field']) 1727 | 1728 | test_obj_1 = models.TestModel.objects.get(id=test_obj_1.id) 1729 | test_obj_2 = models.TestModel.objects.get(id=test_obj_2.id) 1730 | self.assertIsNone(test_obj_1.int_field) 1731 | self.assertIsNone(test_obj_2.int_field) 1732 | 1733 | def test_update_chars_to_null(self): 1734 | """ 1735 | Tests updating a char field to a null field. 1736 | """ 1737 | test_obj_1 = G(models.TestModel, int_field=1, char_field='2') 1738 | test_obj_2 = G(models.TestModel, int_field=2, char_field='3') 1739 | test_obj_1.char_field = None 1740 | test_obj_2.char_field = None 1741 | 1742 | models.TestModel.objects.bulk_update([test_obj_1, test_obj_2], ['char_field']) 1743 | 1744 | test_obj_1 = models.TestModel.objects.get(id=test_obj_1.id) 1745 | test_obj_2 = models.TestModel.objects.get(id=test_obj_2.id) 1746 | self.assertIsNone(test_obj_1.char_field) 1747 | self.assertIsNone(test_obj_2.char_field) 1748 | 1749 | def test_objs_no_fields_to_update(self): 1750 | """ 1751 | Tests when objects are given to bulk update with no fields to update. Nothing should change in 1752 | the objects. 1753 | """ 1754 | test_obj_1 = G(models.TestModel, int_field=1) 1755 | test_obj_2 = G(models.TestModel, int_field=2) 1756 | # Change the int fields on the models 1757 | test_obj_1.int_field = 3 1758 | test_obj_2.int_field = 4 1759 | # Do a bulk update with no update fields 1760 | models.TestModel.objects.bulk_update([test_obj_1, test_obj_2], []) 1761 | # The test objects int fields should be untouched 1762 | test_obj_1 = models.TestModel.objects.get(id=test_obj_1.id) 1763 | test_obj_2 = models.TestModel.objects.get(id=test_obj_2.id) 1764 | self.assertEqual(test_obj_1.int_field, 1) 1765 | self.assertEqual(test_obj_2.int_field, 2) 1766 | 1767 | def test_objs_one_field_to_update(self): 1768 | """ 1769 | Tests when objects are given to bulk update with one field to update. 1770 | """ 1771 | test_obj_1 = G(models.TestModel, int_field=1) 1772 | test_obj_2 = G(models.TestModel, int_field=2) 1773 | # Change the int fields on the models 1774 | test_obj_1.int_field = 3 1775 | test_obj_2.int_field = 4 1776 | # Do a bulk update with the int fields 1777 | models.TestModel.objects.bulk_update([test_obj_1, test_obj_2], ['int_field']) 1778 | # The test objects int fields should be untouched 1779 | test_obj_1 = models.TestModel.objects.get(id=test_obj_1.id) 1780 | test_obj_2 = models.TestModel.objects.get(id=test_obj_2.id) 1781 | self.assertEqual(test_obj_1.int_field, 3) 1782 | self.assertEqual(test_obj_2.int_field, 4) 1783 | 1784 | def test_objs_one_field_to_update_ignore_other_field(self): 1785 | """ 1786 | Tests when objects are given to bulk update with one field to update. This test changes another field 1787 | not included in the update and verifies it is not updated. 1788 | """ 1789 | test_obj_1 = G(models.TestModel, int_field=1, float_field=1.0) 1790 | test_obj_2 = G(models.TestModel, int_field=2, float_field=2.0) 1791 | # Change the int and float fields on the models 1792 | test_obj_1.int_field = 3 1793 | test_obj_2.int_field = 4 1794 | test_obj_1.float_field = 3.0 1795 | test_obj_2.float_field = 4.0 1796 | # Do a bulk update with the int fields 1797 | models.TestModel.objects.bulk_update([test_obj_1, test_obj_2], ['int_field']) 1798 | # The test objects int fields should be untouched 1799 | test_obj_1 = models.TestModel.objects.get(id=test_obj_1.id) 1800 | test_obj_2 = models.TestModel.objects.get(id=test_obj_2.id) 1801 | self.assertEqual(test_obj_1.int_field, 3) 1802 | self.assertEqual(test_obj_2.int_field, 4) 1803 | # The float fields should not be updated 1804 | self.assertEqual(test_obj_1.float_field, 1.0) 1805 | self.assertEqual(test_obj_2.float_field, 2.0) 1806 | 1807 | def test_objs_two_fields_to_update(self): 1808 | """ 1809 | Tests when objects are given to bulk update with two fields to update. 1810 | """ 1811 | test_obj_1 = G(models.TestModel, int_field=1, float_field=1.0) 1812 | test_obj_2 = G(models.TestModel, int_field=2, float_field=2.0) 1813 | # Change the int and float fields on the models 1814 | test_obj_1.int_field = 3 1815 | test_obj_2.int_field = 4 1816 | test_obj_1.float_field = 3.0 1817 | test_obj_2.float_field = 4.0 1818 | # Do a bulk update with the int fields 1819 | models.TestModel.objects.bulk_update([test_obj_1, test_obj_2], ['int_field', 'float_field']) 1820 | # The test objects int fields should be untouched 1821 | test_obj_1 = models.TestModel.objects.get(id=test_obj_1.id) 1822 | test_obj_2 = models.TestModel.objects.get(id=test_obj_2.id) 1823 | self.assertEqual(test_obj_1.int_field, 3) 1824 | self.assertEqual(test_obj_2.int_field, 4) 1825 | # The float fields should be updated 1826 | self.assertEqual(test_obj_1.float_field, 3.0) 1827 | self.assertEqual(test_obj_2.float_field, 4.0) 1828 | 1829 | def test_updating_objects_with_custom_db_field_types(self): 1830 | """ 1831 | Tests when objects are updated that have custom field types 1832 | """ 1833 | test_obj_1 = G( 1834 | models.TestModel, 1835 | int_field=1, 1836 | float_field=1.0, 1837 | json_field={'test': 'test'}, 1838 | array_field=['one', 'two'] 1839 | ) 1840 | test_obj_2 = G( 1841 | models.TestModel, 1842 | int_field=2, 1843 | float_field=2.0, 1844 | json_field={'test2': 'test2'}, 1845 | array_field=['three', 'four'] 1846 | ) 1847 | 1848 | # Change the fields on the models 1849 | test_obj_1.json_field = {'test': 'updated'} 1850 | test_obj_1.array_field = ['one', 'two', 'updated'] 1851 | 1852 | test_obj_2.json_field = {'test2': 'updated'} 1853 | test_obj_2.array_field = ['three', 'four', 'updated'] 1854 | 1855 | # Do a bulk update with the int fields 1856 | models.TestModel.objects.bulk_update( 1857 | [test_obj_1, test_obj_2], 1858 | ['json_field', 'array_field'] 1859 | ) 1860 | 1861 | # Refetch the objects 1862 | test_obj_1 = models.TestModel.objects.get(id=test_obj_1.id) 1863 | test_obj_2 = models.TestModel.objects.get(id=test_obj_2.id) 1864 | 1865 | # Assert that the json field was updated 1866 | self.assertEqual(test_obj_1.json_field, {'test': 'updated'}) 1867 | self.assertEqual(test_obj_2.json_field, {'test2': 'updated'}) 1868 | 1869 | # Assert that the array field was updated 1870 | self.assertEqual(test_obj_1.array_field, ['one', 'two', 'updated']) 1871 | self.assertEqual(test_obj_2.array_field, ['three', 'four', 'updated']) 1872 | 1873 | 1874 | class UpsertTest(TestCase): 1875 | """ 1876 | Tests the upsert method in the manager utils. 1877 | """ 1878 | @patch.object(models.TestModel, 'save', spec_set=True) 1879 | def test_no_double_save_on_create(self, mock_save): 1880 | """ 1881 | Tests that save isn't called on upsert after the object has been created. 1882 | """ 1883 | model_obj, created = models.TestModel.objects.upsert(int_field=1, updates={'float_field': 1.0}) 1884 | self.assertEqual(mock_save.call_count, 1) 1885 | 1886 | def test_save_on_update(self): 1887 | """ 1888 | Tests that save is called when the model is updated 1889 | """ 1890 | model_obj, created = models.TestModel.objects.upsert(int_field=1, updates={'float_field': 1.0}) 1891 | 1892 | with patch.object(models.TestModel, 'save', spec_set=True) as mock_save: 1893 | models.TestModel.objects.upsert(int_field=1, updates={'float_field': 1.1}) 1894 | self.assertEqual(mock_save.call_count, 1) 1895 | 1896 | def test_no_save_on_no_update(self): 1897 | """ 1898 | Tests that save is not called on upsert if the model is not actually updated. 1899 | """ 1900 | model_obj, created = models.TestModel.objects.upsert(int_field=1, updates={'float_field': 1.0}) 1901 | 1902 | with patch.object(models.TestModel, 'save', spec_set=True) as mock_save: 1903 | models.TestModel.objects.upsert(int_field=1, updates={'float_field': 1.0}) 1904 | self.assertEqual(mock_save.call_count, 0) 1905 | 1906 | def test_upsert_creation_no_defaults(self): 1907 | """ 1908 | Tests an upsert that results in a created object. Don't use defaults 1909 | """ 1910 | model_obj, created = models.TestModel.objects.upsert(int_field=1) 1911 | self.assertTrue(created) 1912 | self.assertEqual(model_obj.int_field, 1) 1913 | self.assertIsNone(model_obj.float_field) 1914 | self.assertIsNone(model_obj.char_field) 1915 | 1916 | def test_upsert_creation_defaults(self): 1917 | """ 1918 | Tests an upsert that results in a created object. Defaults are used. 1919 | """ 1920 | model_obj, created = models.TestModel.objects.upsert(int_field=1, defaults={'float_field': 1.0}) 1921 | self.assertTrue(created) 1922 | self.assertEqual(model_obj.int_field, 1) 1923 | self.assertEqual(model_obj.float_field, 1.0) 1924 | self.assertIsNone(model_obj.char_field) 1925 | 1926 | def test_upsert_creation_updates(self): 1927 | """ 1928 | Tests an upsert that results in a created object. Updates are used. 1929 | """ 1930 | model_obj, created = models.TestModel.objects.upsert(int_field=1, updates={'float_field': 1.0}) 1931 | self.assertTrue(created) 1932 | self.assertEqual(model_obj.int_field, 1) 1933 | self.assertEqual(model_obj.float_field, 1.0) 1934 | self.assertIsNone(model_obj.char_field) 1935 | 1936 | def test_upsert_creation_defaults_updates(self): 1937 | """ 1938 | Tests an upsert that results in a created object. Defaults are used and so are updates. 1939 | """ 1940 | model_obj, created = models.TestModel.objects.upsert( 1941 | int_field=1, defaults={'float_field': 1.0}, updates={'char_field': 'Hello'}) 1942 | self.assertTrue(created) 1943 | self.assertEqual(model_obj.int_field, 1) 1944 | self.assertEqual(model_obj.float_field, 1.0) 1945 | self.assertEqual(model_obj.char_field, 'Hello') 1946 | 1947 | def test_upsert_creation_no_defaults_override(self): 1948 | """ 1949 | Tests an upsert that results in a created object. Defaults are not used and 1950 | the updates values override the defaults on creation. 1951 | """ 1952 | test_model = G(models.TestModel) 1953 | model_obj, created = models.TestForeignKeyModel.objects.upsert(int_field=1, updates={ 1954 | 'test_model': test_model, 1955 | }) 1956 | self.assertTrue(created) 1957 | self.assertEqual(model_obj.int_field, 1) 1958 | self.assertEqual(model_obj.test_model, test_model) 1959 | 1960 | def test_upsert_creation_defaults_updates_override(self): 1961 | """ 1962 | Tests an upsert that results in a created object. Defaults are used and so are updates. Updates 1963 | override the defaults. 1964 | """ 1965 | model_obj, created = models.TestModel.objects.upsert( 1966 | int_field=1, defaults={'float_field': 1.0}, updates={'char_field': 'Hello', 'float_field': 2.0}) 1967 | self.assertTrue(created) 1968 | self.assertEqual(model_obj.int_field, 1) 1969 | self.assertEqual(model_obj.float_field, 2.0) 1970 | self.assertEqual(model_obj.char_field, 'Hello') 1971 | 1972 | def test_upsert_no_creation_no_defaults(self): 1973 | """ 1974 | Tests an upsert that already exists. Don't use defaults 1975 | """ 1976 | G(models.TestModel, int_field=1, float_field=None, char_field=None) 1977 | model_obj, created = models.TestModel.objects.upsert(int_field=1) 1978 | self.assertFalse(created) 1979 | self.assertEqual(model_obj.int_field, 1) 1980 | self.assertIsNone(model_obj.float_field) 1981 | self.assertIsNone(model_obj.char_field) 1982 | 1983 | def test_upsert_no_creation_defaults(self): 1984 | """ 1985 | Tests an upsert that already exists. Defaults are used but don't matter since the object already existed. 1986 | """ 1987 | G(models.TestModel, int_field=1, float_field=None, char_field=None) 1988 | model_obj, created = models.TestModel.objects.upsert(int_field=1, defaults={'float_field': 1.0}) 1989 | self.assertFalse(created) 1990 | self.assertEqual(model_obj.int_field, 1) 1991 | self.assertIsNone(model_obj.float_field) 1992 | self.assertIsNone(model_obj.char_field) 1993 | 1994 | def test_upsert_no_creation_updates(self): 1995 | """ 1996 | Tests an upsert that already exists. Updates are used. 1997 | """ 1998 | G(models.TestModel, int_field=1, float_field=2.0, char_field=None) 1999 | model_obj, created = models.TestModel.objects.upsert(int_field=1, updates={'float_field': 1.0}) 2000 | self.assertFalse(created) 2001 | self.assertEqual(model_obj.int_field, 1) 2002 | self.assertEqual(model_obj.float_field, 1.0) 2003 | self.assertIsNone(model_obj.char_field) 2004 | 2005 | def test_upsert_no_creation_defaults_updates(self): 2006 | """ 2007 | Tests an upsert that already exists. Defaults are used and so are updates. 2008 | """ 2009 | G(models.TestModel, int_field=1, float_field=2.0, char_field='Hi') 2010 | model_obj, created = models.TestModel.objects.upsert( 2011 | int_field=1, defaults={'float_field': 1.0}, updates={'char_field': 'Hello'}) 2012 | self.assertFalse(created) 2013 | self.assertEqual(model_obj.int_field, 1) 2014 | self.assertEqual(model_obj.float_field, 2.0) 2015 | self.assertEqual(model_obj.char_field, 'Hello') 2016 | 2017 | def test_upsert_no_creation_defaults_updates_override(self): 2018 | """ 2019 | Tests an upsert that already exists. Defaults are used and so are updates. Updates override the defaults. 2020 | """ 2021 | G(models.TestModel, int_field=1, float_field=3.0, char_field='Hi') 2022 | model_obj, created = models.TestModel.objects.upsert( 2023 | int_field=1, defaults={'float_field': 1.0}, updates={'char_field': 'Hello', 'float_field': 2.0}) 2024 | self.assertFalse(created) 2025 | self.assertEqual(model_obj.int_field, 1) 2026 | self.assertEqual(model_obj.float_field, 2.0) 2027 | self.assertEqual(model_obj.char_field, 'Hello') 2028 | -------------------------------------------------------------------------------- /manager_utils/tests/models.py: -------------------------------------------------------------------------------- 1 | from django.contrib.postgres.fields import JSONField, ArrayField 2 | from django.db import models 3 | from manager_utils import ManagerUtilsManager 4 | from timezone_field import TimeZoneField 5 | 6 | 7 | class TestModel(models.Model): 8 | """ 9 | A model for testing manager utils. 10 | """ 11 | int_field = models.IntegerField(null=True, unique=True) 12 | char_field = models.CharField(max_length=128, null=True) 13 | float_field = models.FloatField(null=True) 14 | json_field = JSONField(default=dict) 15 | array_field = ArrayField(models.CharField(max_length=128), default=list) 16 | time_zone = TimeZoneField(default='UTC') 17 | 18 | objects = ManagerUtilsManager() 19 | 20 | class Meta: 21 | unique_together = ('int_field', 'char_field') 22 | 23 | 24 | class TestUniqueTzModel(models.Model): 25 | """ 26 | A model for testing manager utils with a timezone field as the uniqueness constraint. 27 | """ 28 | int_field = models.IntegerField(null=True, unique=True) 29 | char_field = models.CharField(max_length=128, null=True) 30 | float_field = models.FloatField(null=True) 31 | time_zone = TimeZoneField(unique=True) 32 | 33 | objects = ManagerUtilsManager() 34 | 35 | class Meta: 36 | unique_together = ('int_field', 'char_field') 37 | 38 | 39 | class TestAutoDateTimeModel(models.Model): 40 | """ 41 | A model to test that upserts work with auto_now and auto_now_add 42 | """ 43 | int_field = models.IntegerField(unique=True) 44 | auto_now_field = models.DateTimeField(auto_now=True) 45 | auto_now_add_field = models.DateTimeField(auto_now_add=True) 46 | 47 | objects = ManagerUtilsManager() 48 | 49 | 50 | class TestForeignKeyModel(models.Model): 51 | """ 52 | A test model that has a foreign key. 53 | """ 54 | int_field = models.IntegerField() 55 | test_model = models.ForeignKey(TestModel, on_delete=models.CASCADE) 56 | 57 | objects = ManagerUtilsManager() 58 | 59 | 60 | class TestPkForeignKey(models.Model): 61 | """ 62 | A test model with a primary key thats a foreign key to another model. 63 | """ 64 | my_key = models.ForeignKey(TestModel, primary_key=True, on_delete=models.CASCADE) 65 | char_field = models.CharField(max_length=128, null=True) 66 | 67 | objects = ManagerUtilsManager() 68 | 69 | 70 | class TestPkChar(models.Model): 71 | """ 72 | A test model with a primary key that is a char field. 73 | """ 74 | my_key = models.CharField(max_length=128, primary_key=True) 75 | char_field = models.CharField(max_length=128, null=True) 76 | 77 | objects = ManagerUtilsManager() 78 | -------------------------------------------------------------------------------- /manager_utils/upsert2.py: -------------------------------------------------------------------------------- 1 | """ 2 | The new interface for manager utils upsert 3 | """ 4 | from collections import namedtuple 5 | 6 | from django.db import connection, models 7 | from django.utils import timezone 8 | 9 | 10 | class UpsertResult(list): 11 | """ 12 | Returned by the upsert operation. 13 | 14 | Wraps a list and provides properties to access created, updated, 15 | untouched, and deleted elements 16 | """ 17 | @property 18 | def created(self): 19 | return (i for i in self if i.status_ == 'c') 20 | 21 | @property 22 | def updated(self): 23 | return (i for i in self if i.status_ == 'u') 24 | 25 | @property 26 | def untouched(self): 27 | return (i for i in self if i.status_ == 'n') 28 | 29 | @property 30 | def deleted(self): 31 | return (i for i in self if i.status_ == 'd') 32 | 33 | 34 | def _quote(field): 35 | return '"{0}"'.format(field) 36 | 37 | 38 | def _get_update_fields(model, uniques, to_update): 39 | """ 40 | Get the fields to be updated in an upsert. 41 | 42 | Always exclude auto_now_add, auto_created fields, and unique fields in an update 43 | """ 44 | fields = { 45 | field.attname: field 46 | for field in model._meta.fields 47 | } 48 | 49 | if to_update is None: 50 | to_update = [ 51 | field.attname for field in model._meta.fields 52 | ] 53 | 54 | to_update = [ 55 | attname for attname in to_update 56 | if (attname not in uniques 57 | and not getattr(fields[attname], 'auto_now_add', False) 58 | and not fields[attname].auto_created) 59 | ] 60 | 61 | return to_update 62 | 63 | 64 | def _fill_auto_fields(model, values): 65 | """ 66 | Given a list of models, fill in auto_now and auto_now_add fields 67 | for upserts. Since django manager utils passes Django's ORM, these values 68 | have to be automatically constructed 69 | """ 70 | auto_field_names = [ 71 | f.attname 72 | for f in model._meta.fields 73 | if getattr(f, 'auto_now', False) or getattr(f, 'auto_now_add', False) 74 | ] 75 | now = timezone.now() 76 | for value in values: 77 | for f in auto_field_names: 78 | setattr(value, f, now) 79 | 80 | return values 81 | 82 | 83 | def _sort_by_unique_fields(model, model_objs, unique_fields): 84 | """ 85 | Sort a list of models by their unique fields. 86 | 87 | Sorting models in an upsert greatly reduces the chances of deadlock 88 | when doing concurrent upserts 89 | """ 90 | unique_fields = [ 91 | field for field in model._meta.fields 92 | if field.attname in unique_fields 93 | ] 94 | 95 | def sort_key(model_obj): 96 | return tuple( 97 | field.get_db_prep_save(getattr(model_obj, field.attname), 98 | connection) 99 | for field in unique_fields 100 | ) 101 | return sorted(model_objs, key=sort_key) 102 | 103 | 104 | def _get_values_for_row(model_obj, all_fields): 105 | return [ 106 | # Convert field value to db value 107 | # Use attname here to support fields with custom db_column names 108 | field.get_db_prep_save(getattr(model_obj, field.attname), connection) 109 | for field in all_fields 110 | ] 111 | 112 | 113 | def _get_values_for_rows(model_objs, all_fields): 114 | row_values = [] 115 | sql_args = [] 116 | 117 | for i, model_obj in enumerate(model_objs): 118 | sql_args.extend(_get_values_for_row(model_obj, all_fields)) 119 | if i == 0: 120 | row_values.append('({0})'.format( 121 | ', '.join(['%s::{0}'.format(f.db_type(connection)) for f in all_fields])) 122 | ) 123 | else: 124 | row_values.append('({0})'.format(', '.join(['%s'] * len(all_fields)))) 125 | 126 | return row_values, sql_args 127 | 128 | 129 | def _get_return_fields_sql(returning, return_status=False, alias=None): 130 | if alias: 131 | return_fields_sql = ', '.join('{0}.{1}'.format(alias, _quote(field)) for field in returning) 132 | else: 133 | return_fields_sql = ', '.join(_quote(field) for field in returning) 134 | 135 | if return_status: 136 | return_fields_sql += ', CASE WHEN xmax = 0 THEN \'c\' ELSE \'u\' END AS status_' 137 | 138 | return return_fields_sql 139 | 140 | 141 | def _get_upsert_sql(queryset, model_objs, unique_fields, update_fields, returning, 142 | ignore_duplicate_updates=True, return_untouched=False): 143 | """ 144 | Generates the postgres specific sql necessary to perform an upsert (ON CONFLICT) 145 | INSERT INTO table_name (field1, field2) 146 | VALUES (1, 'two') 147 | ON CONFLICT (unique_field) DO UPDATE SET field2 = EXCLUDED.field2; 148 | """ 149 | model = queryset.model 150 | 151 | # Use all fields except pk unless the uniqueness constraint is the pk field 152 | all_fields = [ 153 | field for field in model._meta.fields 154 | if field.column != model._meta.pk.name or not field.auto_created 155 | ] 156 | 157 | all_field_names = [field.column for field in all_fields] 158 | returning = returning if returning is not True else [f.column for f in model._meta.fields] 159 | all_field_names_sql = ', '.join([_quote(field) for field in all_field_names]) 160 | 161 | # Convert field names to db column names 162 | unique_fields = [ 163 | model._meta.get_field(unique_field) 164 | for unique_field in unique_fields 165 | ] 166 | update_fields = [ 167 | model._meta.get_field(update_field) 168 | for update_field in update_fields 169 | ] 170 | 171 | unique_field_names_sql = ', '.join([ 172 | _quote(field.column) for field in unique_fields 173 | ]) 174 | update_fields_sql = ', '.join([ 175 | '{0} = EXCLUDED.{0}'.format(_quote(field.column)) 176 | for field in update_fields 177 | ]) 178 | 179 | row_values, sql_args = _get_values_for_rows(model_objs, all_fields) 180 | 181 | return_sql = 'RETURNING ' + _get_return_fields_sql(returning, return_status=True) if returning else '' 182 | ignore_duplicates_sql = '' 183 | if ignore_duplicate_updates: 184 | ignore_duplicates_sql = ( 185 | ' WHERE ({update_fields_sql}) IS DISTINCT FROM ({excluded_update_fields_sql}) ' 186 | ).format( 187 | update_fields_sql=', '.join( 188 | '{0}.{1}'.format(model._meta.db_table, _quote(field.column)) 189 | for field in update_fields 190 | ), 191 | excluded_update_fields_sql=', '.join( 192 | 'EXCLUDED.' + _quote(field.column) 193 | for field in update_fields 194 | ) 195 | ) 196 | 197 | on_conflict = ( 198 | 'DO UPDATE SET {0} {1}'.format(update_fields_sql, ignore_duplicates_sql) if update_fields else 'DO NOTHING' 199 | ) 200 | 201 | if return_untouched: 202 | row_values_sql = ', '.join([ 203 | '(\'{0}\', {1})'.format(i, row_value[1:-1]) 204 | for i, row_value in enumerate(row_values) 205 | ]) 206 | sql = ( 207 | ' WITH input_rows("temp_id_", {all_field_names_sql}) AS (' 208 | ' VALUES {row_values_sql}' 209 | ' ), ins AS ( ' 210 | ' INSERT INTO {table_name} ({all_field_names_sql})' 211 | ' SELECT {all_field_names_sql} FROM input_rows ORDER BY temp_id_' 212 | ' ON CONFLICT ({unique_field_names_sql}) {on_conflict} {return_sql}' 213 | ' )' 214 | ' SELECT DISTINCT ON ({table_pk_name}) * FROM (' 215 | ' SELECT status_, {return_fields_sql}' 216 | ' FROM ins' 217 | ' UNION ALL' 218 | ' SELECT \'n\' AS status_, {aliased_return_fields_sql}' 219 | ' FROM input_rows' 220 | ' JOIN {table_name} c USING ({unique_field_names_sql})' 221 | ' ) as results' 222 | ' ORDER BY results."{table_pk_name}", CASE WHEN(status_ = \'n\') THEN 1 ELSE 0 END;' 223 | ).format( 224 | all_field_names_sql=all_field_names_sql, 225 | row_values_sql=row_values_sql, 226 | table_name=model._meta.db_table, 227 | unique_field_names_sql=unique_field_names_sql, 228 | on_conflict=on_conflict, 229 | return_sql=return_sql, 230 | table_pk_name=model._meta.pk.name, 231 | return_fields_sql=_get_return_fields_sql(returning), 232 | aliased_return_fields_sql=_get_return_fields_sql(returning, alias='c') 233 | ) 234 | else: 235 | row_values_sql = ', '.join(row_values) 236 | sql = ( 237 | ' INSERT INTO {table_name} ({all_field_names_sql})' 238 | ' VALUES {row_values_sql}' 239 | ' ON CONFLICT ({unique_field_names_sql}) {on_conflict} {return_sql}' 240 | ).format( 241 | table_name=model._meta.db_table, 242 | all_field_names_sql=all_field_names_sql, 243 | row_values_sql=row_values_sql, 244 | unique_field_names_sql=unique_field_names_sql, 245 | on_conflict=on_conflict, 246 | return_sql=return_sql 247 | ) 248 | 249 | return sql, sql_args 250 | 251 | 252 | def _fetch( 253 | queryset, model_objs, unique_fields, update_fields, returning, sync, 254 | ignore_duplicate_updates=True, return_untouched=False 255 | ): 256 | """ 257 | Perfom the upsert and do an optional sync operation 258 | """ 259 | model = queryset.model 260 | if (return_untouched or sync) and returning is not True: 261 | returning = set(returning) if returning else set() 262 | returning.add(model._meta.pk.name) 263 | upserted = [] 264 | deleted = [] 265 | # We must return untouched rows when doing a sync operation 266 | return_untouched = True if sync else return_untouched 267 | 268 | if model_objs: 269 | sql, sql_args = _get_upsert_sql(queryset, model_objs, unique_fields, update_fields, returning, 270 | ignore_duplicate_updates=ignore_duplicate_updates, 271 | return_untouched=return_untouched) 272 | 273 | with connection.cursor() as cursor: 274 | cursor.execute(sql, sql_args) 275 | if cursor.description: 276 | nt_result = namedtuple('Result', [col[0] for col in cursor.description]) 277 | upserted = [nt_result(*row) for row in cursor.fetchall()] 278 | 279 | pk_field = model._meta.pk.name 280 | if sync: 281 | orig_ids = queryset.values_list(pk_field, flat=True) 282 | deleted = set(orig_ids) - {getattr(r, pk_field) for r in upserted} 283 | model.objects.filter(pk__in=deleted).delete() 284 | 285 | nt_deleted_result = namedtuple('DeletedResult', [model._meta.pk.name, 'status_']) 286 | return UpsertResult( 287 | upserted + [nt_deleted_result(**{pk_field: d, 'status_': 'd'}) for d in deleted] 288 | ) 289 | 290 | 291 | def upsert( 292 | queryset, model_objs, unique_fields, 293 | update_fields=None, returning=False, sync=False, 294 | ignore_duplicate_updates=True, 295 | return_untouched=False 296 | ): 297 | """ 298 | Perform a bulk upsert on a table, optionally syncing the results. 299 | 300 | Args: 301 | queryset (Model|QuerySet): A model or a queryset that defines the collection to sync 302 | model_objs (List[Model]): A list of Django models to sync. All models in this list 303 | will be bulk upserted and any models not in the table (or queryset) will be deleted 304 | if sync=True. 305 | unique_fields (List[str]): A list of fields that define the uniqueness of the model. The 306 | model must have a unique constraint on these fields 307 | update_fields (List[str], default=None): A list of fields to update whenever objects 308 | already exist. If an empty list is provided, it is equivalent to doing a bulk 309 | insert on the objects that don't exist. If `None`, all fields will be updated. 310 | returning (bool|List[str]): If True, returns all fields. If a list, only returns 311 | fields in the list 312 | sync (bool, default=False): Perform a sync operation on the queryset 313 | ignore_duplicate_updates (bool, default=False): Don't perform an update if the row is 314 | a duplicate. 315 | return_untouched (bool, default=False): Return untouched rows by the operation 316 | """ 317 | queryset = queryset if isinstance(queryset, models.QuerySet) else queryset.objects.all() 318 | model = queryset.model 319 | 320 | # Populate automatically generated fields in the rows like date times 321 | _fill_auto_fields(model, model_objs) 322 | 323 | # Sort the rows to reduce the chances of deadlock during concurrent upserts 324 | model_objs = _sort_by_unique_fields(model, model_objs, unique_fields) 325 | update_fields = _get_update_fields(model, unique_fields, update_fields) 326 | 327 | return _fetch(queryset, model_objs, unique_fields, update_fields, returning, sync, 328 | ignore_duplicate_updates=ignore_duplicate_updates, 329 | return_untouched=return_untouched) 330 | -------------------------------------------------------------------------------- /manager_utils/version.py: -------------------------------------------------------------------------------- 1 | __version__ = '3.1.5' 2 | -------------------------------------------------------------------------------- /publish.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | 3 | subprocess.call(['rm', '-r', 'dist/']) 4 | subprocess.call(['pip', 'install', 'wheel']) 5 | subprocess.call(['pip', 'install', 'twine']) 6 | subprocess.call(['python', 'setup.py', 'clean', '--all']) 7 | subprocess.call(['python', 'setup.py', 'register', 'sdist', 'bdist_wheel']) 8 | subprocess.call(['twine', 'upload', 'dist/*']) 9 | subprocess.call(['rm', '-r', 'dist/']) 10 | subprocess.call(['rm', '-r', 'build/']) 11 | -------------------------------------------------------------------------------- /requirements/docs.txt: -------------------------------------------------------------------------------- 1 | Sphinx>=1.2.2 2 | sphinx_rtd_theme 3 | -------------------------------------------------------------------------------- /requirements/requirements-testing.txt: -------------------------------------------------------------------------------- 1 | coverage 2 | psycopg2 3 | django-nose>=1.3 4 | django-dynamic-fixture 5 | pytz 6 | django-timezone-field 7 | parameterized 8 | freezegun 9 | flake8 10 | -------------------------------------------------------------------------------- /requirements/requirements.txt: -------------------------------------------------------------------------------- 1 | Django>=3.2 2 | django-query-builder>=3.1.0 3 | -------------------------------------------------------------------------------- /run_tests.py: -------------------------------------------------------------------------------- 1 | """ 2 | Provides the ability to run test on a standalone Django app. 3 | """ 4 | import sys 5 | from optparse import OptionParser 6 | from settings import configure_settings 7 | 8 | # Configure the default settings 9 | configure_settings() 10 | 11 | 12 | # Django nose must be imported here since it depends on the settings being configured 13 | from django_nose import NoseTestSuiteRunner 14 | 15 | 16 | def run(*test_args, **kwargs): 17 | if not test_args: 18 | test_args = ['manager_utils'] 19 | 20 | kwargs.setdefault('interactive', False) 21 | 22 | test_runner = NoseTestSuiteRunner(**kwargs) 23 | 24 | failures = test_runner.run_tests(test_args) 25 | sys.exit(failures) 26 | 27 | 28 | if __name__ == '__main__': 29 | parser = OptionParser() 30 | parser.add_option('--verbosity', dest='verbosity', action='store', default=1, type=int) 31 | (options, args) = parser.parse_args() 32 | 33 | run(*args, **options.__dict__) 34 | -------------------------------------------------------------------------------- /settings.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | 4 | from django.conf import settings 5 | 6 | 7 | def configure_settings(): 8 | """ 9 | Configures settings for manage.py and for run_tests.py. 10 | """ 11 | if not settings.configured: 12 | # Determine the database settings depending on if a test_db var is set in CI mode or not 13 | test_db = os.environ.get('DB', None) 14 | if test_db is None: 15 | db_config = { 16 | 'ENGINE': 'django.db.backends.postgresql', 17 | 'NAME': 'ambition_test', 18 | 'USER': 'postgres', 19 | 'PASSWORD': '', 20 | 'HOST': 'db', 21 | } 22 | elif test_db == 'postgres': 23 | db_config = { 24 | 'ENGINE': 'django.db.backends.postgresql', 25 | 'NAME': 'manager_utils', 26 | 'USER': 'travis', 27 | 'PORT': '5433', 28 | } 29 | # db_config = { 30 | # 'ENGINE': 'django.db.backends.postgresql', 31 | # 'NAME': 'manager_utils', 32 | # 'USER': 'postgres', 33 | # 'PASSWORD': '', 34 | # 'HOST': 'db', 35 | # } 36 | elif test_db == 'sqlite': 37 | db_config = { 38 | 'ENGINE': 'django.db.backends.sqlite3', 39 | 'NAME': 'manager_utils', 40 | } 41 | else: 42 | raise RuntimeError('Unsupported test DB {0}'.format(test_db)) 43 | 44 | if os.environ.get('DB_SETTINGS'): 45 | db_config = json.loads(os.environ.get('DB_SETTINGS')) 46 | 47 | installed_apps = [ 48 | 'django.contrib.auth', 49 | 'django.contrib.contenttypes', 50 | 'django.contrib.sessions', 51 | 'django.contrib.admin', 52 | 'manager_utils', 53 | 'manager_utils.tests', 54 | ] 55 | 56 | settings.configure( 57 | DATABASES={ 58 | 'default': db_config, 59 | }, 60 | MIDDLEWARE_CLASSES={}, 61 | INSTALLED_APPS=installed_apps, 62 | ROOT_URLCONF='manager_utils.urls', 63 | DEBUG=False, 64 | NOSE_ARGS=['--nocapture', '--nologcapture', '--verbosity=1'], 65 | TEST_RUNNER='django_nose.NoseTestSuiteRunner', 66 | SECRET_KEY='*', 67 | USE_DEPRECATED_PYTZ=True, 68 | ) 69 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 120 3 | ignore=E123,E133,E226,E241,E242,E402,E731,W504,W503 4 | exclude = build,docs,env,venv,*.egg,migrations 5 | max-complexity = 10 6 | 7 | [build_sphinx] 8 | source-dir = docs/ 9 | build-dir = docs/_build 10 | all_files = 1 11 | 12 | [bdist_wheel] 13 | universal = 1 14 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # import multiprocessing to avoid this bug (http://bugs.python.org/issue15881#msg170215_ 2 | import multiprocessing 3 | assert multiprocessing 4 | import re 5 | from setuptools import setup, find_packages 6 | 7 | 8 | def get_version(): 9 | """ 10 | Extracts the version number from the version.py file. 11 | """ 12 | VERSION_FILE = 'manager_utils/version.py' 13 | mo = re.search(r'^__version__ = [\'"]([^\'"]*)[\'"]', open(VERSION_FILE, 'rt').read(), re.M) 14 | if mo: 15 | return mo.group(1) 16 | else: 17 | raise RuntimeError('Unable to find version string in {0}.'.format(VERSION_FILE)) 18 | 19 | 20 | def get_lines(file_path): 21 | return open(file_path, 'r').read().split('\n') 22 | 23 | 24 | install_requires = get_lines('requirements/requirements.txt') 25 | tests_require = get_lines('requirements/requirements-testing.txt') 26 | 27 | 28 | setup( 29 | name='django-manager-utils', 30 | version=get_version(), 31 | description='Model manager utilities for Django', 32 | long_description=open('README.rst').read(), 33 | url='http://github.com/ambitioninc/django-manager-utils/', 34 | author='Wes Kendall', 35 | author_email='opensource@ambition.com', 36 | project_urls={ 37 | "Bug Tracker": "https://github.com/ambitioninc/django-manager-utils/issues", 38 | "Changes": "https://django-manager-utils.readthedocs.io/en/latest/release_notes.html", 39 | "Documentation": "https://django-manager-utils.readthedocs.io/en/latest/", 40 | "Source Code": "https://github.com/ambitioninc/django-manager-utils", 41 | }, 42 | packages=find_packages(), 43 | classifiers=[ 44 | 'Programming Language :: Python', 45 | 'Programming Language :: Python :: 3.7', 46 | 'Programming Language :: Python :: 3.8', 47 | 'Programming Language :: Python :: 3.9', 48 | 'Intended Audience :: Developers', 49 | 'License :: OSI Approved :: MIT License', 50 | 'Operating System :: OS Independent', 51 | 'Framework :: Django :: 3.2', 52 | 'Framework :: Django :: 4.0', 53 | 'Framework :: Django :: 4.1', 54 | 'Framework :: Django :: 4.2', 55 | ], 56 | install_requires=install_requires, 57 | tests_require=tests_require, 58 | test_suite='run_tests.run', 59 | include_package_data=True, 60 | ) 61 | -------------------------------------------------------------------------------- /tox_old.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = 3 | flake8 4 | coverage 5 | py{37,38}-django22 6 | py{37,38}-django30 7 | py{37,38}-django31 8 | py{37,38}-django32 9 | py{38,39}-django40 10 | py{38,39}-django41 11 | py{38,39}-djangomaster 12 | 13 | [testenv] 14 | setenv = 15 | DB = postgres 16 | deps = 17 | django22: Django~=2.2 18 | django30: Django~=3.0 19 | django31: Django~=3.1 20 | django32: Django~=3.2 21 | django40: Django~=4.0 22 | django41: Django~=4.1 23 | djangomaster: https://github.com/django/django/archive/master.tar.gz 24 | -rrequirements/requirements-testing.txt 25 | 26 | commands = 27 | pip freeze 28 | python --version 29 | coverage run manage.py test manager_utils --failfast 30 | coverage report --fail-under=90 31 | 32 | [testenv:flake8] 33 | deps = flake8 34 | commands = flake8 manager_utils 35 | 36 | [travis:env] 37 | DJANGO = 38 | 2.2: django22 39 | 3.0: django30 40 | 3.1: django31 41 | 3.2: django32 42 | 4.0: django40 43 | 4.1: django41 44 | master: djangomaster 45 | --------------------------------------------------------------------------------