├── .editorconfig ├── .gitignore ├── .travis.yml ├── AUTHORS.rst ├── CONTRIBUTING.rst ├── HISTORY.rst ├── LICENSE ├── MANIFEST.in ├── README.rst ├── docs ├── Makefile ├── authors.rst ├── conf.py ├── contributing.rst ├── examples.rst ├── history.rst ├── index.rst ├── make.bat ├── overview.rst └── quickstart.rst ├── makemigrations.py ├── requirements-test.txt ├── requirements.txt ├── requirements_dev.txt ├── runtests.py ├── setup.cfg ├── setup.py ├── src └── nsync │ ├── __init__.py │ ├── actions.py │ ├── logging.py │ ├── management │ ├── __init__.py │ └── commands │ │ ├── __init__.py │ │ ├── syncfile.py │ │ ├── syncfiles.py │ │ └── utils.py │ ├── migrations │ ├── 0001_initial.py │ └── __init__.py │ ├── models.py │ └── policies.py ├── tests ├── __init__.py ├── models.py ├── test_actions.py ├── test_command_syncfile.py ├── test_command_syncfiles.py ├── test_command_utils.py ├── test_integrations.py ├── test_models.py ├── test_policies.py └── test_utils.py └── tox.ini /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | charset = utf-8 7 | end_of_line = lf 8 | insert_final_newline = true 9 | trim_trailing_whitespace = true 10 | 11 | [*.{py,rst,ini}] 12 | indent_style = space 13 | indent_size = 4 14 | 15 | [*.{html,css,scss,json,yml}] 16 | indent_style = space 17 | indent_size = 2 18 | 19 | [*.md] 20 | trim_trailing_whitespace = false 21 | 22 | [Makefile] 23 | indent_style = tab 24 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | __pycache__ 3 | 4 | # C extensions 5 | *.so 6 | 7 | # Packages 8 | *.egg 9 | *.egg-info 10 | dist 11 | build 12 | eggs 13 | parts 14 | bin 15 | var 16 | sdist 17 | develop-eggs 18 | .installed.cfg 19 | lib 20 | lib64 21 | 22 | # Installer logs 23 | pip-log.txt 24 | 25 | # Unit test / coverage reports 26 | .coverage 27 | .tox 28 | nosetests.xml 29 | htmlcov 30 | 31 | # Translations 32 | *.mo 33 | 34 | # Mr Developer 35 | .mr.developer.cfg 36 | .project 37 | .pydevproject 38 | 39 | # Complexity 40 | output/*.html 41 | output/*/index.html 42 | 43 | # Sphinx 44 | docs/_build 45 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | # Config file for automatic testing at travis-ci.org 2 | 3 | language: python 4 | 5 | matrix: 6 | include: 7 | - python: '3.4' 8 | env: TOXENV=py34-django18 9 | - python: '3.5' 10 | env: TOXENV=py35-django18 11 | 12 | - python: '3.4' 13 | env: TOXENV=py34-django110 14 | - python: '3.5' 15 | env: TOXENV=py35-django110 16 | 17 | - python: '3.4' 18 | env: TOXENV=py34-django20 19 | - python: '3.5' 20 | env: TOXENV=py35-django20 21 | - python: '3.6' 22 | env: TOXENV=py36-django20 23 | 24 | before_install: 25 | - pip install codecov 26 | 27 | # command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors 28 | #install: pip install -r requirements-test.txt 29 | install: pip install --upgrade pip setuptools tox codecov 30 | 31 | # command to run tests using coverage, e.g. python setup.py test 32 | #script: coverage run --source nsync runtests.py 33 | #script: python setup.py test 34 | script: tox 35 | 36 | after_success: 37 | - codecov 38 | -------------------------------------------------------------------------------- /AUTHORS.rst: -------------------------------------------------------------------------------- 1 | ======= 2 | Credits 3 | ======= 4 | 5 | Development Lead 6 | ---------------- 7 | 8 | * Andrew Dodd 9 | 10 | Contributors 11 | ------------ 12 | 13 | * Chris Snell 14 | -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | ============ 2 | Contributing 3 | ============ 4 | 5 | Contributions are welcome, and they are greatly appreciated! Every 6 | little bit helps, and credit will always be given. 7 | 8 | You can contribute in many ways: 9 | 10 | Types of Contributions 11 | ---------------------- 12 | 13 | Report Bugs 14 | ~~~~~~~~~~~ 15 | 16 | Report bugs at https://github.com/andrewdodd/django-nsync/issues. 17 | 18 | If you are reporting a bug, please include: 19 | 20 | * Your operating system name and version. 21 | * Any details about your local setup that might be helpful in troubleshooting. 22 | * Detailed steps to reproduce the bug. 23 | 24 | Fix Bugs 25 | ~~~~~~~~ 26 | 27 | Look through the GitHub issues for bugs. Anything tagged with "bug" 28 | is open to whoever wants to implement it. 29 | 30 | Implement Features 31 | ~~~~~~~~~~~~~~~~~~ 32 | 33 | Look through the GitHub issues for features. Anything tagged with "feature" 34 | is open to whoever wants to implement it. 35 | 36 | Write Documentation 37 | ~~~~~~~~~~~~~~~~~~~ 38 | 39 | django-nsync could always use more documentation, whether as part of the 40 | official django-nsync docs, in docstrings, or even on the web in blog posts, 41 | articles, and such. 42 | 43 | Submit Feedback 44 | ~~~~~~~~~~~~~~~ 45 | 46 | The best way to send feedback is to file an issue at https://github.com/andrewdodd/django-nsync/issues. 47 | 48 | If you are proposing a feature: 49 | 50 | * Explain in detail how it would work. 51 | * Keep the scope as narrow as possible, to make it easier to implement. 52 | * Remember that this is a volunteer-driven project, and that contributions 53 | are welcome :) 54 | 55 | Get Started! 56 | ------------ 57 | 58 | Ready to contribute? Here's how to set up `django-nsync` for local development. 59 | 60 | 1. Fork the `django-nsync` repo on GitHub. 61 | 2. Clone your fork locally:: 62 | 63 | $ git clone git@github.com:your_name_here/django-nsync.git 64 | 65 | 3. Install your local copy into a virtualenv. Assuming you have virtualenvwrapper installed, this is how you set up your fork for local development:: 66 | 67 | $ mkvirtualenv django-nsync 68 | $ cd django-nsync/ 69 | $ python setup.py develop 70 | 71 | 4. Create a branch for local development:: 72 | 73 | $ git checkout -b name-of-your-bugfix-or-feature 74 | 75 | Now you can make your changes locally. 76 | 77 | 5. When you're done making changes, check that your changes pass flake8 and the 78 | tests, including testing other Python versions with tox:: 79 | 80 | $ flake8 src tests 81 | $ python setup.py test 82 | $ tox 83 | 84 | To get flake8 and tox, just pip install them into your virtualenv. NB: Don't 85 | worry about flake8 issues in the migrations files or if URLs are too long. 86 | 87 | 6. Commit your changes and push your branch to GitHub:: 88 | 89 | $ git add . 90 | $ git commit -m "Your detailed description of your changes." 91 | $ git push origin name-of-your-bugfix-or-feature 92 | 93 | 7. Submit a pull request through the GitHub website. 94 | 95 | Pull Request Guidelines 96 | ----------------------- 97 | 98 | Before you submit a pull request, check that it meets these guidelines: 99 | 100 | 1. The pull request should include tests. 101 | 2. If the pull request adds functionality, the docs should be updated. Put 102 | your new functionality into a function with a docstring, and add the 103 | feature to the list in README.rst. 104 | 3. The pull request should work for Python 3.3+, and for PyPy. Check 105 | https://travis-ci.org/andrewdodd/django-nsync/pull_requests 106 | and make sure that the tests pass for all supported Python versions. 107 | 108 | Tips 109 | ---- 110 | 111 | To run a subset of tests:: 112 | 113 | $ python -m unittest tests.test_nsync 114 | -------------------------------------------------------------------------------- /HISTORY.rst: -------------------------------------------------------------------------------- 1 | .. :changelog: 2 | 3 | History 4 | ------- 5 | 6 | 0.1.0 (2015-10-02) 7 | ++++++++++++++++++ 8 | 9 | * First release on PyPI. 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Andrew Dodd 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.rst *.txt LICENSE tox.ini .travis.yml docs/Makefile runtests.py 2 | recursive-include tests *.py 3 | recursive-include docs *.rst 4 | recursive-include docs *.py 5 | 6 | prune docs/_build 7 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ============================= 2 | django-nsync 3 | ============================= 4 | 5 | .. image:: https://badge.fury.io/py/django-nsync.png 6 | :target: https://badge.fury.io/py/django-nsync 7 | 8 | .. image:: https://travis-ci.org/andrewdodd/django-nsync.png?branch=master 9 | :target: https://travis-ci.org/andrewdodd/django-nsync 10 | 11 | .. image:: https://codecov.io/github/andrewdodd/django-nsync/coverage.svg?branch=master 12 | :target: https://codecov.io/github/andrewdodd/django-nsync?branch=master 13 | :alt: Coverage 14 | 15 | Django NSync provides a simple way to keep your Django Model data 'n-sync with N external systems. 16 | 17 | Features 18 | -------- 19 | Includes: 20 | 21 | - Synchronise models with data from external systems 22 | 23 | - Create, update or delete model objects 24 | - Modify relational fields 25 | - Allow multiple systems to modify the same model object 26 | 27 | - CSV file support out of the box 28 | 29 | - Ships with commands to process a single CSV file or multiple CSV files 30 | 31 | - No need for more code 32 | 33 | - Nsync does not require you to inherit from special classes, add 'model mapping' objects or really define anything in Python 34 | 35 | Not-included: 36 | 37 | - Export (to CSV or anything else for that matter) 38 | 39 | - There are other packages that can do this 40 | - Why do you want the data out? Isn't that what your application is for? ;-) 41 | 42 | - Admin integration 43 | 44 | - There isn't much to this package, if you want to add the models to your admin pages it is probably better if you do it (that's what I've done in my use case) 45 | 46 | Not-yet included: 47 | 48 | - Other file formats out of the box 49 | 50 | - Love it or hate it, CSV is ubiquitous and simple (its limitations also force simplicity) 51 | - The CSV handling part is separated from the true NSync part, so feel free to write your own lyrics-from-wav-file importer. 52 | 53 | - Intricate data format handling 54 | 55 | - E.g. parsing date times etc 56 | - This can be side-stepped by creating ``@property`` annotated handlers though (see the examples from more info) 57 | 58 | 59 | Documentation 60 | ------------- 61 | 62 | The full documentation is at https://django-nsync.readthedocs.org. 63 | 64 | 65 | Credits 66 | --------- 67 | 68 | Tools used in rendering this package: 69 | 70 | * Cookiecutter_ Used to create the initial repot 71 | * `cookiecutter-pypackage`_ Used by Cookiecutter_ to create the initial repo 72 | 73 | For helping me make sense of the python pacakging world (and the bad practices codified in some of the tools/blogs out there): 74 | 75 | * `Hynek Schlawack`_ Whose blog posts on packaging Python apps etc were indispensible 76 | * `Ionel Cristian Maries`_ (sorry, too lazy for unicode) Whose blog post on python packaging was also indispensible 77 | 78 | .. _`Hynek Schlawack`: https://hynek.me 79 | .. _`Ionel Cristian Maries`: http://blog.ionelmc.ro/ 80 | .. _Cookiecutter: https://github.com/audreyr/cookiecutter 81 | .. _`cookiecutter-pypackage`: https://github.com/pydanny/cookiecutter-djangopackage 82 | 83 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 21 | 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 23 | 24 | help: 25 | @echo "Please use \`make ' where is one of" 26 | @echo " html to make standalone HTML files" 27 | @echo " dirhtml to make HTML files named index.html in directories" 28 | @echo " singlehtml to make a single large HTML file" 29 | @echo " pickle to make pickle files" 30 | @echo " json to make JSON files" 31 | @echo " htmlhelp to make HTML files and a HTML help project" 32 | @echo " qthelp to make HTML files and a qthelp project" 33 | @echo " devhelp to make HTML files and a Devhelp project" 34 | @echo " epub to make an epub" 35 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 36 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 37 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 38 | @echo " text to make text files" 39 | @echo " man to make manual pages" 40 | @echo " texinfo to make Texinfo files" 41 | @echo " info to make Texinfo files and run them through makeinfo" 42 | @echo " gettext to make PO message catalogs" 43 | @echo " changes to make an overview of all changed/added/deprecated items" 44 | @echo " xml to make Docutils-native XML files" 45 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 46 | @echo " linkcheck to check all external links for integrity" 47 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 48 | 49 | clean: 50 | rm -rf $(BUILDDIR)/* 51 | 52 | html: 53 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 54 | @echo 55 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 56 | 57 | dirhtml: 58 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 59 | @echo 60 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 61 | 62 | singlehtml: 63 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 64 | @echo 65 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 66 | 67 | pickle: 68 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 69 | @echo 70 | @echo "Build finished; now you can process the pickle files." 71 | 72 | json: 73 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 74 | @echo 75 | @echo "Build finished; now you can process the JSON files." 76 | 77 | htmlhelp: 78 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 79 | @echo 80 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 81 | ".hhp project file in $(BUILDDIR)/htmlhelp." 82 | 83 | qthelp: 84 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 85 | @echo 86 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 87 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 88 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/complexity.qhcp" 89 | @echo "To view the help file:" 90 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/complexity.qhc" 91 | 92 | devhelp: 93 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 94 | @echo 95 | @echo "Build finished." 96 | @echo "To view the help file:" 97 | @echo "# mkdir -p $$HOME/.local/share/devhelp/complexity" 98 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/complexity" 99 | @echo "# devhelp" 100 | 101 | epub: 102 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 103 | @echo 104 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 105 | 106 | latex: 107 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 108 | @echo 109 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 110 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 111 | "(use \`make latexpdf' here to do that automatically)." 112 | 113 | latexpdf: 114 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 115 | @echo "Running LaTeX files through pdflatex..." 116 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 117 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 118 | 119 | latexpdfja: 120 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 121 | @echo "Running LaTeX files through platex and dvipdfmx..." 122 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 123 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 124 | 125 | text: 126 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 127 | @echo 128 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 129 | 130 | man: 131 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 132 | @echo 133 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 134 | 135 | texinfo: 136 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 137 | @echo 138 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 139 | @echo "Run \`make' in that directory to run these through makeinfo" \ 140 | "(use \`make info' here to do that automatically)." 141 | 142 | info: 143 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 144 | @echo "Running Texinfo files through makeinfo..." 145 | make -C $(BUILDDIR)/texinfo info 146 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 147 | 148 | gettext: 149 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 150 | @echo 151 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 152 | 153 | changes: 154 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 155 | @echo 156 | @echo "The overview file is in $(BUILDDIR)/changes." 157 | 158 | linkcheck: 159 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 160 | @echo 161 | @echo "Link check complete; look for any errors in the above output " \ 162 | "or in $(BUILDDIR)/linkcheck/output.txt." 163 | 164 | doctest: 165 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 166 | @echo "Testing of doctests in the sources finished, look at the " \ 167 | "results in $(BUILDDIR)/doctest/output.txt." 168 | 169 | xml: 170 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 171 | @echo 172 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 173 | 174 | pseudoxml: 175 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 176 | @echo 177 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 178 | -------------------------------------------------------------------------------- /docs/authors.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../AUTHORS.rst 2 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # complexity documentation build configuration file, created by 4 | # sphinx-quickstart on Tue Jul 9 22:26:36 2013. 5 | # 6 | # This file is execfile()d with the current directory set to its containing dir. 7 | # 8 | # Note that not all possible configuration values are present in this 9 | # autogenerated file. 10 | # 11 | # All configuration values have a default; values that are commented out 12 | # serve to show the default. 13 | 14 | # Hynek lift and shift 15 | import codecs 16 | import datetime 17 | import os 18 | import re 19 | 20 | try: 21 | import sphinx_rtd_theme 22 | except ImportError: 23 | sphinx_rtd_theme = None 24 | 25 | 26 | def read(*parts): 27 | """ 28 | Build an absolute path from *parts* and and return the contents of the 29 | resulting file. Assume UTF-8 encoding. 30 | """ 31 | here = os.path.abspath(os.path.dirname(__file__)) 32 | with codecs.open(os.path.join(here, *parts), "rb", "utf-8") as f: 33 | return f.read() 34 | 35 | 36 | def find_version(*file_paths): 37 | """ 38 | Build a path from *file_paths* and search for a ``__version__`` 39 | string inside. 40 | """ 41 | version_file = read(*file_paths) 42 | version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", 43 | version_file, re.M) 44 | if version_match: 45 | return version_match.group(1) 46 | raise RuntimeError("Unable to find version string.") 47 | # Hynek lift and shift 48 | 49 | # If extensions (or modules to document with autodoc) are in another directory, 50 | # add these directories to sys.path here. If the directory is relative to the 51 | # documentation root, use os.path.abspath to make it absolute, like shown here. 52 | #sys.path.insert(0, os.path.abspath('.')) 53 | 54 | 55 | # -- General configuration ----------------------------------------------------- 56 | 57 | # If your documentation needs a minimal Sphinx version, state it here. 58 | #needs_sphinx = '1.0' 59 | 60 | # Add any Sphinx extension module names here, as strings. They can be extensions 61 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 62 | extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode'] 63 | 64 | # Add any paths that contain templates here, relative to this directory. 65 | templates_path = ['_templates'] 66 | 67 | # The suffix of source filenames. 68 | source_suffix = '.rst' 69 | 70 | # The encoding of source files. 71 | #source_encoding = 'utf-8-sig' 72 | 73 | # The master toctree document. 74 | master_doc = 'index' 75 | 76 | # General information about the project. 77 | project = u'django-nsync' 78 | year = datetime.date.today().year 79 | copyright = u'2015{0}, Andrew Dodd'.format( 80 | u'-{0}'.format(year) if year != 2015 else u'' 81 | ) 82 | 83 | # The version info for the project you're documenting, acts as replacement for 84 | # |version| and |release|, also used in various other places throughout the 85 | # built documents. 86 | # 87 | # The full version, including alpha/beta/rc tags. 88 | release = find_version("..", "src", "nsync", "__init__.py") 89 | # The short X.Y version. 90 | version = release.rsplit(u".", 1)[0] 91 | 92 | # The language for content autogenerated by Sphinx. Refer to documentation 93 | # for a list of supported languages. 94 | #language = None 95 | 96 | # There are two options for replacing |today|: either, you set today to some 97 | # non-false value, then it is used: 98 | #today = '' 99 | # Else, today_fmt is used as the format for a strftime call. 100 | #today_fmt = '%B %d, %Y' 101 | 102 | # List of patterns, relative to source directory, that match files and 103 | # directories to ignore when looking for source files. 104 | exclude_patterns = ['_build'] 105 | 106 | # The reST default role (used for this markup: `text`) to use for all documents. 107 | #default_role = None 108 | 109 | # If true, '()' will be appended to :func: etc. cross-reference text. 110 | #add_function_parentheses = True 111 | 112 | # If true, the current module name will be prepended to all description 113 | # unit titles (such as .. function::). 114 | #add_module_names = True 115 | 116 | # If true, sectionauthor and moduleauthor directives will be shown in the 117 | # output. They are ignored by default. 118 | #show_authors = False 119 | 120 | # The name of the Pygments (syntax highlighting) style to use. 121 | pygments_style = 'sphinx' 122 | 123 | # A list of ignored prefixes for module index sorting. 124 | #modindex_common_prefix = [] 125 | 126 | # If true, keep warnings as "system message" paragraphs in the built documents. 127 | #keep_warnings = False 128 | 129 | 130 | # -- Options for HTML output --------------------------------------------------- 131 | 132 | # The theme to use for HTML and HTML Help pages. See the documentation for 133 | # a list of builtin themes. 134 | if sphinx_rtd_theme: 135 | html_theme = 'sphinx_rtd_theme' 136 | html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] 137 | else: 138 | html_theme = 'default' 139 | 140 | # Theme options are theme-specific and customize the look and feel of a theme 141 | # further. For a list of options available for each theme, see the 142 | # documentation. 143 | #html_theme_options = {} 144 | 145 | # Add any paths that contain custom themes here, relative to this directory. 146 | #html_theme_path = [] 147 | 148 | # The name for this set of Sphinx documents. If None, it defaults to 149 | # " v documentation". 150 | #html_title = None 151 | 152 | # A shorter title for the navigation bar. Default is the same as html_title. 153 | #html_short_title = None 154 | 155 | # The name of an image file (relative to this directory) to place at the top 156 | # of the sidebar. 157 | #html_logo = None 158 | 159 | # The name of an image file (within the static path) to use as favicon of the 160 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 161 | # pixels large. 162 | #html_favicon = None 163 | 164 | # Add any paths that contain custom static files (such as style sheets) here, 165 | # relative to this directory. They are copied after the builtin static files, 166 | # so a file named "default.css" will overwrite the builtin "default.css". 167 | # html_static_path = ['_static'] 168 | 169 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 170 | # using the given strftime format. 171 | #html_last_updated_fmt = '%b %d, %Y' 172 | 173 | # If true, SmartyPants will be used to convert quotes and dashes to 174 | # typographically correct entities. 175 | #html_use_smartypants = True 176 | 177 | # Custom sidebar templates, maps document names to template names. 178 | #html_sidebars = {} 179 | 180 | # Additional templates that should be rendered to pages, maps page names to 181 | # template names. 182 | #html_additional_pages = {} 183 | 184 | # If false, no module index is generated. 185 | #html_domain_indices = True 186 | 187 | # If false, no index is generated. 188 | #html_use_index = True 189 | 190 | # If true, the index is split into individual pages for each letter. 191 | #html_split_index = False 192 | 193 | # If true, links to the reST sources are added to the pages. 194 | #html_show_sourcelink = True 195 | 196 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 197 | #html_show_sphinx = True 198 | 199 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 200 | #html_show_copyright = True 201 | 202 | # If true, an OpenSearch description file will be output, and all pages will 203 | # contain a tag referring to it. The value of this option must be the 204 | # base URL from which the finished HTML is served. 205 | #html_use_opensearch = '' 206 | 207 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 208 | #html_file_suffix = None 209 | 210 | # Output file base name for HTML help builder. 211 | htmlhelp_basename = 'django-nsyncdoc' 212 | 213 | 214 | # -- Options for LaTeX output -------------------------------------------------- 215 | 216 | latex_elements = { 217 | # The paper size ('letterpaper' or 'a4paper'). 218 | #'papersize': 'letterpaper', 219 | 220 | # The font size ('10pt', '11pt' or '12pt'). 221 | #'pointsize': '10pt', 222 | 223 | # Additional stuff for the LaTeX preamble. 224 | #'preamble': '', 225 | } 226 | 227 | # Grouping the document tree into LaTeX files. List of tuples 228 | # (source start file, target name, title, author, documentclass [howto/manual]). 229 | latex_documents = [ 230 | ('index', 'django-nsync.tex', u'django-nsync Documentation', 231 | u'Andrew Dodd', 'manual'), 232 | ] 233 | 234 | # The name of an image file (relative to this directory) to place at the top of 235 | # the title page. 236 | #latex_logo = None 237 | 238 | # For "manual" documents, if this is true, then toplevel headings are parts, 239 | # not chapters. 240 | #latex_use_parts = False 241 | 242 | # If true, show page references after internal links. 243 | #latex_show_pagerefs = False 244 | 245 | # If true, show URL addresses after external links. 246 | #latex_show_urls = False 247 | 248 | # Documents to append as an appendix to all manuals. 249 | #latex_appendices = [] 250 | 251 | # If false, no module index is generated. 252 | #latex_domain_indices = True 253 | 254 | 255 | # -- Options for manual page output -------------------------------------------- 256 | 257 | # One entry per manual page. List of tuples 258 | # (source start file, name, description, authors, manual section). 259 | man_pages = [ 260 | ('index', 'django-nsync', u'django-nsync Documentation', 261 | [u'Andrew Dodd'], 1) 262 | ] 263 | 264 | # If true, show URL addresses after external links. 265 | #man_show_urls = False 266 | 267 | 268 | # -- Options for Texinfo output ------------------------------------------------ 269 | 270 | # Grouping the document tree into Texinfo files. List of tuples 271 | # (source start file, target name, title, author, 272 | # dir menu entry, description, category) 273 | texinfo_documents = [ 274 | ('index', 'django-nsync', u'django-nsync Documentation', 275 | u'Andrew Dodd', 'django-nsync', 'One line description of project.', 276 | 'Miscellaneous'), 277 | ] 278 | 279 | # Documents to append as an appendix to all manuals. 280 | #texinfo_appendices = [] 281 | 282 | # If false, no module index is generated. 283 | #texinfo_domain_indices = True 284 | 285 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 286 | #texinfo_show_urls = 'footnote' 287 | 288 | # If true, do not generate a @detailmenu in the "Top" node's menu. 289 | #texinfo_no_detailmenu = False 290 | -------------------------------------------------------------------------------- /docs/contributing.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../CONTRIBUTING.rst 2 | -------------------------------------------------------------------------------- /docs/examples.rst: -------------------------------------------------------------------------------- 1 | 2 | Examples 3 | ======== 4 | 5 | The following are some examples of using the Nsync functionality. The 6 | following Django models will be used as the target models:: 7 | 8 | class Person(models.Model): 9 | first_name = models.CharField( 10 | blank=False, 11 | max_length=50, 12 | verbose_name='First Name' 13 | ) 14 | last_name = models.CharField( 15 | blank=False, 16 | max_length=50, 17 | verbose_name='Last Name' 18 | ) 19 | age = models.IntegerField(blank=True, null=True) 20 | 21 | hair_colour = models.CharField( 22 | blank=False, 23 | max_length=50, 24 | default="Unknown") 25 | 26 | 27 | class House(models.Model): 28 | address = models.CharField(max_length=100) 29 | country = models.CharField(max_length=100, blank=True) 30 | floors = models.IntegerField(blank=True, null=True) 31 | owner = models.ForeignKey(TestPerson, blank=True, null=True) 32 | 33 | 34 | Example - Basic 35 | --------------- 36 | 37 | Using this file: 38 | 39 | .. csv-table:: persons.csv 40 | :header: "action_flags", "match_on", "first_name", "last_name", "employee_id" 41 | 42 | "cu","employee_id","Andrew","Dodd","EMP1111" 43 | "d*","employee_id","Some","Other-Guy","EMP2222" 44 | "cu","employee_id","Captain","Planet","EMP3333" 45 | "u*","employee_id","C.","Batman","EMP1234" 46 | 47 | And running this command:: 48 | 49 | > python manage.py syncfile TestSystem myapp Person persons.csv 50 | 51 | Would: 52 | - Create and/or update ``myapp.Person`` objects with Employee Ids EMP1111 & EMP3333. However, it would not update the two name fields if the objects already existed with non-blank fields. 53 | - Delete any ``myapp.Person`` objects with Employee Id EMP2222. 54 | - If a person with Employee Id EMP1234 exists, then it will forcibly update the name fields to 'C.' and 'Batman' respectively. 55 | 56 | NB it would also: 57 | - Create an ``nsync.ExternalSystem`` object with the name TestSystem, as the default is to create missing external systems. However, because there are no ``external_key`` values, no ``ExternalKeyMapping`` objects would be created. 58 | 59 | Example - Basic with External Ids 60 | --------------------------------- 61 | 62 | Using this file: 63 | 64 | .. csv-table:: persons.csv 65 | :header: "external_key", "action_flags", "match_on", "first_name", "last_name", "employee_id" 66 | 67 | 12212281,"cu","employee_id","Andrew","Dodd","EMP1111" 68 | 43719289,"d*","employee_id","Some","Other-Guy","EMP2222" 69 | 99999999,"cu","employee_id","Captain","Planet","EMP3333" 70 | 11235813,"u*","employee_id","C.","Batman","EMP1234" 71 | 72 | And running this command:: 73 | 74 | > python manage.py syncfile TestSystem myapp Person persons.csv 75 | 76 | Would: 77 | - Perform all of steps in the 'Plain file' example 78 | - Delete any ``ExternalKeyMapping`` objects that are for the 'TestSystem' and have the external key '43719289' (i.e. the record for Some Other-Guy). 79 | - Create or update ``ExternalKeyMapping`` objects for each of the other three ``myapp.Person`` objects, which contain the ``external_key`` value. 80 | 81 | 82 | Example - Basic with multiple match fields 83 | ------------------------------------------ 84 | 85 | Sometimes you might not have a 'unique' field to find your objects with (like 'Employee Id'). In this instance, you can specify multiple fields for finding your object (separated with a space, ' '). 86 | 87 | For example, using this file: 88 | 89 | .. csv-table:: persons.csv 90 | :header: "action_flags", "match_on", "first_name", "last_name", "age" 91 | 92 | "cu*","first_name last_name","Michael","Martin","30" 93 | "cu*","first_name last_name","Martin","Martin","40" 94 | "cu*","first_name last_name","Michael","Michael","50" 95 | "cu*","first_name last_name","Martin","Michael","60" 96 | 97 | And running this command:: 98 | 99 | > python manage.py syncfile TestSystem myapp Person persons.csv 100 | 101 | Would: 102 | - Create and/or update four persons of various "Michael" and "Martin" name combinations 103 | - Ensure they are updated/created with the correct age! 104 | 105 | Example - Two or more systems 106 | ----------------------------- 107 | This is probably the main purpose of this library: the ability to 108 | synchronise from multiple systems. 109 | 110 | Perhaps we need to synchronise from two data sources on housing information, 111 | one is the 'when built' information and the other is the 'renovations' 112 | information. 113 | 114 | As-built data: 115 | 116 | .. csv-table:: AsBuiltDB_myapp_House.csv 117 | :header: "external_key", "action_flags", "match_on", "address", "country", "floors" 118 | 119 | 111,"cu","address","221B Baker Street","England",1 120 | 222,"cu","address","Wayne Manor","Gotham City",2 121 | 122 | Renovated data: 123 | 124 | .. csv-table:: RenovationsDB_myapp_House.csv 125 | :header: "external_key", "action_flags", "match_on", "address", "floors" 126 | 127 | ABC123,"u*","address","221B Baker Street",2 128 | ABC456,"u*","address","Wayne Manor",4 129 | FOX123,"u*","address","742 Evergreen Terrace",2 130 | 131 | 132 | And running this command:: 133 | 134 | > python manage.py syncfiles AsBuiltDB_myapp_House.csv RenovationsDB_myapp_House.csv 135 | 136 | Would: 137 | - Use the **mutliple file command**, ``syncfiles``, to perform multiple updates in one command 138 | - Create the two houses from the 'AsBuilt' file 139 | - Only update the ``country`` values of the two houses from the 'AsBuilt' file IFF the objects already existed but they did not have a value for ``country`` 140 | - Forcibly set the ``floors`` attribute for the first two houses in the 'Renovations' file. 141 | - Create 4 ``ExternalKeyMapping`` objects: 142 | 143 | +---------------+--------+----------------------+ 144 | | External | Ext. | House Object | 145 | | System | Key | | 146 | +===============+========+======================+ 147 | | AsBuiltDB | 111 | | 148 | +---------------+--------+ 212B Baker Street | 149 | | RenovationsDB | ABC123 | | 150 | +---------------+--------+----------------------+ 151 | | AsBuiltDB | 222 | | 152 | +---------------+--------+ Wayne Manor | 153 | | RenovationsDB | ABC456 | | 154 | +---------------+--------+----------------------+ 155 | - Only update the ``floors`` attribute for "742 Evergreen Terrace" if the house already exists (and would then also create an ``ExternalKeyMapping``) 156 | 157 | 158 | Example - Referential fields 159 | ---------------------------- 160 | You can also manage referential fields with Nsync. For example, if you had the following people: 161 | 162 | .. csv-table:: Examples_myapp_Person.csv 163 | :header: "external_key", "action_flags", "match_on", "first_name", "last_name", "employee_id" 164 | 165 | 1111,"cu*","employee_id","Homer","Simpson","EMP1" 166 | 2222,"cu*","employee_id","Bruce","Wayne","EMP2" 167 | 3333,"cu*","employee_id","John","Wayne","EMP3" 168 | 169 | You could set their houses with a file like this: 170 | 171 | .. csv-table:: Examples_myapp_House.csv 172 | :header: "external_key", "action_flags", "match_on", "address", "owner=>first_name" 173 | 174 | ABC456,"cu*","address","Wayne Manor","Bruce" 175 | FOX123,"cu*","address","742 Evergreen Terrace","Homer" 176 | 177 | The **"=>"** is used by Nsync to follow the the related field on the provided object. 178 | 179 | Example - Referential field gotchas 180 | ----------------------------------- 181 | The referential field update will ONLY be performed if the referred-to-fields target a single object. For example, if you had the following list of people: 182 | 183 | .. csv-table:: Examples_myapp_Person.csv 184 | :header: "external_key", "action_flags", "match_on", "first_name", "last_name", "employee_id" 185 | 186 | 1111,"cu*","employee_id","Homer","Simpson","EMP1" 187 | 2222,"cu*","employee_id","Homer","The Greek","EMP2" 188 | 3333,"cu*","employee_id","Bruce","Wayne","EMP3" 189 | 4444,"cu*","employee_id","Bruce","Lee","EMP4" 190 | 5555,"cu*","employee_id","John","Wayne","EMP5" 191 | 6666,"cu*","employee_id","Marge","Simpson","EMP6" 192 | 193 | The ``owner=>first_name`` from the previous example is insufficient to pick out a single person to link a house to (there are 2 Homers and 2 Bruces). Using just the ``employee_id`` field would work, but that piece of information may not be available in the system for houses. 194 | 195 | Nsync allows you to specify multiple fields to use in order to 'filter' the correct object to create the link with. In this instance, this file would perform correctly: 196 | 197 | .. csv-table:: Examples_myapp_House.csv 198 | :header: "external_key", "action_flags", "match_on", "address", "owner=>first_name", "owner=>last_name" 199 | 200 | ABC456,"cu*","address","Wayne Manor","Bruce","Wayne" 201 | FOX123,"cu*","address","742 Evergreen Terrace","Homer","Simpson" 202 | 203 | 204 | Example - Complex Fields 205 | ------------------------ 206 | If you want a more complex update you can: 207 | - Write an extension to Nsync and submit a Pull Request! OR 208 | - Extend your Django model with a custom setter 209 | 210 | If your Person model has a photo ImageField, then you could add a custom handler to update the photo based on a provided file path:: 211 | 212 | class Person(models.Model): 213 | ... 214 | photo = models.ImageField( 215 | blank = True, 216 | null = True, 217 | max_length = 200, 218 | upload_to = 'person_photos', 219 | ) 220 | ... 221 | 222 | @photo_filename.setter 223 | def photo_filename(self, file_path): 224 | ... 225 | Do the processing of the file to update the model 226 | 227 | And then supply the photos with a file sync file like: 228 | 229 | .. csv-table:: persons.csv 230 | :header: "action_flags", "match_on", "first_name", "last_name", "employee_id", "photo_filename" 231 | 232 | "cu*","employee_id","Andrew","Dodd","EMP1111","/tmp/photos/ugly_headshot.jpg" 233 | 234 | 235 | Example - Update uses external key mapping over matched object 236 | -------------------------------------------------------------- 237 | This is an example that is to do with the changes for `Issue 1`_ 238 | 239 | If Nsync is 'updating' objects but their 'match fields' change, Nsync will still update the 'correct' object. 240 | 241 | A common occurrence of this is if the sync data is being produced from a database and an in-row update occurs which changes the match fields but leaves the 'external key' (i.e. an SQL 'UPDATE ... WHERE ...' statement). 242 | 243 | E.g. A person table might look like this: 244 | 245 | ================== =============== ==== 246 | ID (a DB sequence) Employee Number Name 247 | ================== =============== ==== 248 | 10123 EMP001 Andrew Dodd 249 | ================== =============== ==== 250 | 251 | This could be used to produce an Nsync input CSV like this: 252 | 253 | ============ ============ =============== =============== ==== 254 | external_key action_flags match_on employee_number name 255 | ============ ============ =============== =============== ==== 256 | 10123 cu* employee_number EMP001 Andrew Dodd 257 | ============ ============ =============== =============== ==== 258 | 259 | This would result in an "Andrew Dodd, EMP001" Person object being created and/or updated with an `ExternalKeyMapping` object holding the '10123' id and a link to Andrew. 260 | 261 | If Andrew became a contractor instead of an employee, perhaps the table could be updated to look like this: 262 | 263 | ================== =============== ==== 264 | ID (a DB sequence) Employee Number Name 265 | ================== =============== ==== 266 | 10123 CONT999 Andrew Dodd 267 | ================== =============== ==== 268 | 269 | This would then produce an Nsync input CSV like this: 270 | 271 | ============ ============ =============== =============== ==== 272 | external_key action_flags match_on employee_number name 273 | ============ ============ =============== =============== ==== 274 | 10123 cu* employee_number CONT999 Andrew Dodd 275 | ============ ============ =============== =============== ==== 276 | 277 | Nsync will use the `ExternalKeyMapping` object if it is available instead of relying on the 'match fields'. In this case, the 278 | resulting action will cause the Andrew Dodd object to change its 'employee_number'. This is instead of Nsync using the 279 | 'employee_number' for finding Andrew. 280 | 281 | NB: In this instance, Nsync will also delete any objects that have the 'new' match field but are not pointed to by the external key. 282 | 283 | .. _`Issue 1`: https://github.com/andrewdodd/django-nsync/issues/1 284 | 285 | 286 | Example - Delete tricks 287 | ----------------------- 288 | This is a list of tricky / gotchas to be aware of when deleting objects. 289 | 290 | When syncing from external systems that have external key mappings, it is probably best to use the 'unforced delete'. This ensures that an object is not removed until all of the external systems think it should be removed. 291 | 292 | If using 'forced delete', beware that (depending on which sync policy you use) you may end up with different systems fighting over the existence of an object (i.e. one system creating the object, then another deleting it in the same sync). 293 | 294 | A system without external key mappings cannot delete objects if it uses an 'unforced delete'. The reason for this is that the 'unforced delete' only removes the model object IF AND ONLY IF it is the last remaining external key mapping. Thus, if a system without external key mappings is the source-of-truth for the removal of an object, you must use the 'forced delete' for it to be able to remove the objects. 295 | 296 | 297 | Alternative Sync Policies 298 | ------------------------- 299 | The out-of-the-box sync policies are pretty straightforward and are probably worth a read (see the ``policies.py`` file). The system is made so that it is pretty easy for you to define your own custom policy and write a command (similar to the ones in Nsync) to use it. 300 | 301 | Some examples of alternative policies might be: 302 | - Run deletes before creates and updates 303 | - Search and execute certain actions before all others 304 | 305 | 306 | -------------------------------------------------------------------------------- /docs/history.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../HISTORY.rst 2 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. complexity documentation master file, created by 2 | sphinx-quickstart on Tue Jul 9 22:26:36 2013. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to django-nsync's documentation! 7 | ================================================================= 8 | 9 | .. include:: ../README.rst 10 | 11 | User Guide 12 | ---------- 13 | 14 | .. toctree:: 15 | :maxdepth: 1 16 | 17 | quickstart 18 | overview 19 | examples 20 | 21 | 22 | Project Infomation 23 | ------------------ 24 | .. toctree:: 25 | :maxdepth: 1 26 | 27 | contributing 28 | authors 29 | history 30 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=_build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . 10 | set I18NSPHINXOPTS=%SPHINXOPTS% . 11 | if NOT "%PAPER%" == "" ( 12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 14 | ) 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "help" ( 19 | :help 20 | echo.Please use `make ^` where ^ is one of 21 | echo. html to make standalone HTML files 22 | echo. dirhtml to make HTML files named index.html in directories 23 | echo. singlehtml to make a single large HTML file 24 | echo. pickle to make pickle files 25 | echo. json to make JSON files 26 | echo. htmlhelp to make HTML files and a HTML help project 27 | echo. qthelp to make HTML files and a qthelp project 28 | echo. devhelp to make HTML files and a Devhelp project 29 | echo. epub to make an epub 30 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 31 | echo. text to make text files 32 | echo. man to make manual pages 33 | echo. texinfo to make Texinfo files 34 | echo. gettext to make PO message catalogs 35 | echo. changes to make an overview over all changed/added/deprecated items 36 | echo. xml to make Docutils-native XML files 37 | echo. pseudoxml to make pseudoxml-XML files for display purposes 38 | echo. linkcheck to check all external links for integrity 39 | echo. doctest to run all doctests embedded in the documentation if enabled 40 | goto end 41 | ) 42 | 43 | if "%1" == "clean" ( 44 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 45 | del /q /s %BUILDDIR%\* 46 | goto end 47 | ) 48 | 49 | 50 | %SPHINXBUILD% 2> nul 51 | if errorlevel 9009 ( 52 | echo. 53 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 54 | echo.installed, then set the SPHINXBUILD environment variable to point 55 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 56 | echo.may add the Sphinx directory to PATH. 57 | echo. 58 | echo.If you don't have Sphinx installed, grab it from 59 | echo.http://sphinx-doc.org/ 60 | exit /b 1 61 | ) 62 | 63 | if "%1" == "html" ( 64 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 65 | if errorlevel 1 exit /b 1 66 | echo. 67 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 68 | goto end 69 | ) 70 | 71 | if "%1" == "dirhtml" ( 72 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 73 | if errorlevel 1 exit /b 1 74 | echo. 75 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 76 | goto end 77 | ) 78 | 79 | if "%1" == "singlehtml" ( 80 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 81 | if errorlevel 1 exit /b 1 82 | echo. 83 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 84 | goto end 85 | ) 86 | 87 | if "%1" == "pickle" ( 88 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 89 | if errorlevel 1 exit /b 1 90 | echo. 91 | echo.Build finished; now you can process the pickle files. 92 | goto end 93 | ) 94 | 95 | if "%1" == "json" ( 96 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 97 | if errorlevel 1 exit /b 1 98 | echo. 99 | echo.Build finished; now you can process the JSON files. 100 | goto end 101 | ) 102 | 103 | if "%1" == "htmlhelp" ( 104 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 105 | if errorlevel 1 exit /b 1 106 | echo. 107 | echo.Build finished; now you can run HTML Help Workshop with the ^ 108 | .hhp project file in %BUILDDIR%/htmlhelp. 109 | goto end 110 | ) 111 | 112 | if "%1" == "qthelp" ( 113 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 114 | if errorlevel 1 exit /b 1 115 | echo. 116 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 117 | .qhcp project file in %BUILDDIR%/qthelp, like this: 118 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\complexity.qhcp 119 | echo.To view the help file: 120 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\complexity.ghc 121 | goto end 122 | ) 123 | 124 | if "%1" == "devhelp" ( 125 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 126 | if errorlevel 1 exit /b 1 127 | echo. 128 | echo.Build finished. 129 | goto end 130 | ) 131 | 132 | if "%1" == "epub" ( 133 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 134 | if errorlevel 1 exit /b 1 135 | echo. 136 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 137 | goto end 138 | ) 139 | 140 | if "%1" == "latex" ( 141 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 142 | if errorlevel 1 exit /b 1 143 | echo. 144 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 145 | goto end 146 | ) 147 | 148 | if "%1" == "latexpdf" ( 149 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 150 | cd %BUILDDIR%/latex 151 | make all-pdf 152 | cd %BUILDDIR%/.. 153 | echo. 154 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 155 | goto end 156 | ) 157 | 158 | if "%1" == "latexpdfja" ( 159 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 160 | cd %BUILDDIR%/latex 161 | make all-pdf-ja 162 | cd %BUILDDIR%/.. 163 | echo. 164 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 165 | goto end 166 | ) 167 | 168 | if "%1" == "text" ( 169 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 170 | if errorlevel 1 exit /b 1 171 | echo. 172 | echo.Build finished. The text files are in %BUILDDIR%/text. 173 | goto end 174 | ) 175 | 176 | if "%1" == "man" ( 177 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 178 | if errorlevel 1 exit /b 1 179 | echo. 180 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 181 | goto end 182 | ) 183 | 184 | if "%1" == "texinfo" ( 185 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 186 | if errorlevel 1 exit /b 1 187 | echo. 188 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 189 | goto end 190 | ) 191 | 192 | if "%1" == "gettext" ( 193 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 194 | if errorlevel 1 exit /b 1 195 | echo. 196 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 197 | goto end 198 | ) 199 | 200 | if "%1" == "changes" ( 201 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 202 | if errorlevel 1 exit /b 1 203 | echo. 204 | echo.The overview file is in %BUILDDIR%/changes. 205 | goto end 206 | ) 207 | 208 | if "%1" == "linkcheck" ( 209 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 210 | if errorlevel 1 exit /b 1 211 | echo. 212 | echo.Link check complete; look for any errors in the above output ^ 213 | or in %BUILDDIR%/linkcheck/output.txt. 214 | goto end 215 | ) 216 | 217 | if "%1" == "doctest" ( 218 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 219 | if errorlevel 1 exit /b 1 220 | echo. 221 | echo.Testing of doctests in the sources finished, look at the ^ 222 | results in %BUILDDIR%/doctest/output.txt. 223 | goto end 224 | ) 225 | 226 | if "%1" == "xml" ( 227 | %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml 228 | if errorlevel 1 exit /b 1 229 | echo. 230 | echo.Build finished. The XML files are in %BUILDDIR%/xml. 231 | goto end 232 | ) 233 | 234 | if "%1" == "pseudoxml" ( 235 | %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml 236 | if errorlevel 1 exit /b 1 237 | echo. 238 | echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. 239 | goto end 240 | ) 241 | 242 | :end 243 | -------------------------------------------------------------------------------- /docs/overview.rst: -------------------------------------------------------------------------------- 1 | Overview 2 | ======== 3 | 4 | Yeah, yeah, but whats the point? 5 | -------------------------------- 6 | It is quite common to need to load information from other computer systems into your Django 7 | application. There are also many ways to do this (manually, via a Restful API, through SQL/database 8 | cleverness) and there are a large number of existing tools to do this (see below). 9 | 10 | However, often one must obtain and synchronise information about a model object from **multiple 11 | information sources** (e.g. the HR system, the ERP, that cool new Web API, Dave's spreadsheet), and 12 | **continue to do it** in order to keep one's Django app running properly. This project allows you to do 13 | just that. 14 | 15 | 16 | Similar projects 17 | ---------------- 18 | 19 | There are a number of projects that are similar in nature but are (I believe) unsuitable for the 20 | reasons listed; 21 | 22 | * `django-synchro`_ - Focussed on the synchonisation between databases (e.g. production & 23 | fail-over, production & testing) 24 | 25 | - This is quite a 'full on' project, and it is mainly focussed on synchronising two Django 26 | applications, not disparate systems 27 | 28 | * `django-external-data-sync`_ - This is quite close in purpose (I think, I didn't look at it too 29 | closely yet) to ``django-nsync``, which is periodically synchronising with external data 30 | 31 | - Focusses on the infrastructure surrounding the 'synchronisation' 32 | - Does not provide any synchronisation functions (you must subclass `Synchronizer`) 33 | - Not packaged on PyPI 34 | 35 | * `django-mapped-fields`_ - Provides form fields to map structured data to models 36 | 37 | - Seems ok (I didn't really look too closely) 38 | - Not designed for automated operation (i.e. it is about a Django Form workflow) 39 | 40 | * `django-csvimport`_ - A generic importer tool for uploading CSV files to populate data 41 | 42 | - Extends the Admin Interface functionality, not really automated 43 | 44 | * `django-import-export`_ - Application and library for importing and exporting 45 | 46 | - Looks to be excellent, certainly close in concept to ``django-nsync`` 47 | - Requires the creation of `ModelResource` subtypes to marshall the importing (i.e. requires code 48 | changes) 49 | - Focussed more on the 'Admin interface' interactions 50 | - (NB: Now that I'm writing up these docs and looking at this project it seems that they are 51 | quite similar) 52 | 53 | .. _`django-synchro`: https://github.com/zlorf/django-synchro 54 | .. _`django-external-data-sync`: https://github.com/596acres/django-external-data-sync 55 | .. _`django-mapped-fields`: https://github.com/mypebble/mapped-fields 56 | .. _`django-csvimport`: https://github.com/edcrewe/django-csvimport 57 | .. _`django-import-export`: https://django-import-export.readthedocs.org/en/latest/ 58 | 59 | Concepts 60 | ======== 61 | 62 | Model Actions 63 | ------------- 64 | 65 | There are three key model actions (Create, Update, & Delete), and one action modifier (Force). 66 | 67 | Create Action 68 | ^^^^^^^^^^^^^ 69 | This is used to 'create' a model object with the given information. If a matching object is found 70 | is will **NOT** create another one and it will **NOT** modify the existing object. 71 | 72 | This action is always considered *'forced'*, so that it will override any non-empty/null defaults. 73 | 74 | Update Action 75 | ^^^^^^^^^^^^^ 76 | This action will look for a model object and, if one is found, update it with the given 77 | information. It will **NOT** create an object. 78 | 79 | - If **NOT** forced, the update will only affect fields whose current value is ``None`` or ``''`` 80 | - If **forced**, the update will clobber any exiting value for the field 81 | 82 | Delete Action 83 | ^^^^^^^^^^^^^ 84 | This action will look for a model object and, if one is found, attempt to delete it. 85 | 86 | - If **forced**, the action will remove the object 87 | - If **NOT** forced, the action will only delete the object if: 88 | 89 | - The target has external key mapping information; AND 90 | - The key mapping described exists; AND 91 | - The key mapping points to the corresponding object; AND 92 | - There are no other key mappings pointing to the object 93 | 94 | NB: The *Delete action* will typically also manage the deletion of any corresponding 95 | ``ExternalKeyMapping`` objects associated with the object and the particular ``ExternalSystem`` 96 | that is performing the sync process (more information will be provided later). 97 | 98 | Forced Actions 99 | ^^^^^^^^^^^^^^ 100 | The option to *force* an action is provided to allow optionality to the synchroniser. It allows 101 | systems that 'might' have useful information but not the 'authoritative' answer to provide 102 | 'provisional' information in the sync process. 103 | 104 | As mentioned above, the ``force`` option allows the following modifications of behaviour: 105 | 106 | - ``CREATE`` actions - are *always* forced, to ensure they update non-``None`` and non-``''`` 107 | default values 108 | - For ``UPDATE`` actions - it allows the action to forcibly replace the value in the corresponding 109 | field, rather than only replacing it if the value is ``None`` or ``''`` 110 | - For ``DELETE`` actions - it allows the action to forcibly delete the model object, even if other 111 | systems have synchronised links to it 112 | 113 | CSV Encodings 114 | ^^^^^^^^^^^^^ 115 | For use in CSV files (with the built in ``syncfile`` and ``syncfiles`` commands), the CSV file 116 | should include a column with the header ``action_flags``. The values for the field can be: 117 | 118 | +-------+-------------------+ 119 | | Value | Meaning | 120 | +=======+===================+ 121 | | c + Create only | 122 | +-------+-------------------+ 123 | | u | Update only | 124 | +-------+-------------------+ 125 | | d | Delete only | 126 | +-------+-------------------+ 127 | | cu | Create and update | 128 | +-------+-------------------+ 129 | | u* | Forced update | 130 | +-------+-------------------+ 131 | | d* | Forced delete | 132 | +-------+-------------------+ 133 | 134 | 135 | The following values are pointless / not allowed: 136 | 137 | +-------+---------------------+-------------------------------------------+ 138 | | Value | Meaning | Reason | 139 | +=======+=====================+===========================================+ 140 | | | No action | Pointless, omit the row | 141 | +-------+---------------------+-------------------------------------------+ 142 | | c* + Forced create | Pointless, all creates are already forced | 143 | +-------+---------------------+-------------------------------------------+ 144 | | cd | Create and delete | Illegal, cannot request delete action | 145 | +-------+---------------------+ with either create or update action. | 146 | | ud | Update and delete | | 147 | +-------+---------------------+ | 148 | | cud | Create, update and | | 149 | | | delete | | 150 | +-------+---------------------+-------------------------------------------+ 151 | 152 | 153 | 154 | Which object to act on? 155 | ----------------------- 156 | The action uses the provided information to attempt to find (or guarantee the absence of) the 157 | object it should be acting upon. The *'provided information'* is the set of values used to set the 158 | fields in ``CREATE`` or ``UPDATE`` actions (NB: in all three cases it must contain the information 159 | to find the specific object). 160 | 161 | Rules / Choices in design 162 | ^^^^^^^^^^^^^^^^^^^^^^^^^ 163 | The current choices for how this 'selection' behaves are: 164 | 165 | - Always acts on a single object 166 | - Found by the "``match_on``" value 167 | 168 | - Found by looking for an object that has the same 'values' as those provided for the model fields 169 | specified in the '``match_on``' list of fields. (huh? feeling lost?, it'll make sense in the 170 | examples) 171 | - The '``match_on``' column could be a 'unique' field with which to find your object, OR it could 172 | be a list of fields to use to find your object. 173 | 174 | - Actions that target mulitple obects "could" be possible, but they are hard and probably not worth 175 | the trouble 176 | 177 | - This would be trying to address some very general cases, which would be too hard to get logic 178 | correct for (I feel the risk of doing the wrong thing here accidentally would be too high) 179 | - Computers are good at doing things quickly, get a computer to write the 'same' thing for the 180 | multiple targets and to execuse the request against multiple objects 181 | 182 | Field Options 183 | ------------- 184 | Fields are modified by using the ``setattr()`` built-in Python function. The field to update is 185 | based on the behaviour of this function to set the attribute based on the dictionary of information 186 | provided. **NB:** If the list of values includes 'fields' that are not part of the model object's 187 | definition, they will be ignored (more work to come here) 188 | 189 | Referential Fields 190 | ^^^^^^^^^^^^^^^^^^ 191 | One of the most important features is the ability to update 'referred to' fields, such as 192 | ``Person`` object that is assigned a company ``Car`` object. 193 | 194 | This is specified by including the field and matchfields in the 'key' side of the values, 195 | concatenated with '``=>``' (you can see the CSV heritage creeping in here). For example, if you had 196 | classes like this:: 197 | 198 | class Person(models.Model): 199 | first_name = models.CharField( 200 | blank=False, 201 | max_length=50, 202 | ) 203 | last_name = models.CharField( 204 | blank=False, 205 | max_length=50, 206 | ) 207 | assigned_car = models.ForeignKey(Car, blank=True, null=True) 208 | 209 | class Car(models.Model): 210 | rego_number= models.CharField(max_length=10, unique=True) 211 | name = models.CharField(max_length=50) 212 | 213 | You could load the assignment by synchronising with the following file for ``Person`` model: 214 | 215 | .. csv-table:: persons.csv 216 | :header: "action_flags", "match_on", "first_name", "last_name", "assigned_car=>rego_number" 217 | 218 | "cu","employee_id","Andrew","Dodd","BG29JL" 219 | 220 | 221 | However, you can also supply multiple inputs to a Referential assignment, which is especially handy 222 | for resolving situations where your models do not have a field that can be used to address them 223 | uniquely. For example, if you had classes like this instead (which is far more likely):: 224 | 225 | class Person(models.Model): 226 | first_name = models.CharField( 227 | blank=False, 228 | max_length=50, 229 | ) 230 | last_name = models.CharField( 231 | blank=False, 232 | max_length=50, 233 | ) 234 | 235 | class Car(models.Model): 236 | rego_number= models.CharField(max_length=10, unique=True) 237 | name = models.CharField(max_length=50) 238 | assigned_to = models.ForeignKey(Person, blank=True, null=True) 239 | 240 | You could load the assignment by synchronising with the following file for ``Car`` model: 241 | 242 | .. csv-table:: cars.csv 243 | :header: "action_flags", "match_on", "rego_number", "name", "assigned_to=>first_name", "assigned_to=>last_name" 244 | 245 | "cu","rego_number","BG29JL","Herman the Sherman","Andrew","Dodd" 246 | 247 | 248 | ExternalSystem & ExternalKeyMapping 249 | ----------------------------------- 250 | This library also creates some objects to help keep track of the internal model objects modified by 251 | the external systems. With the purpose being to supply a way for users of the library to peform 252 | their own 'reverse' on which internal objects are being touched by which external systems. This is 253 | not particularly interesting, but it is perhaps worth checking out the ``ExternalSystem`` and 254 | ``ExternalKeyMapping`` classes. 255 | 256 | These classes are also used to decide which 'object' is update if the '``match_on``' fields are 257 | changed (i.e. by an SQL UPDATE) but the 'external system key' remains the same. 258 | 259 | 260 | But how? 261 | -------- 262 | It is probaby easiest to look at the examples page or have a look at the integration tests for the 263 | two out of the box commands. 264 | 265 | -------------------------------------------------------------------------------- /docs/quickstart.rst: -------------------------------------------------------------------------------- 1 | Quickstart 2 | ---------- 3 | 4 | Installation 5 | ^^^^^^^^^^^^ 6 | 7 | .. installation-begin 8 | To get started using ``django-nsync``, install it with ``pip``:: 9 | 10 | $ pip install django-nsync 11 | 12 | Add ``"nsync"`` to your project's ``INSTALLED_APPS`` setting. E.g.:: 13 | 14 | INSTALLED_APPS += ( 15 | 'nsync', 16 | ) 17 | 18 | Run ``python manage.py migrate`` to create the Django-Nsync models. 19 | 20 | .. installation-result 21 | 22 | You will now have in your application: 23 | 24 | - An ``ExternalSystem`` model, used to represent 'where' information is synchronising from 25 | - An ``ExternalKeyMapping`` model, used to record the mappings from an ``ExternalSystem``'s key for a model object, to the actual model object internally 26 | - Two 'built-in' commands for synchronising data with: 27 | 28 | - `syncfile` - Which synchronises a single file, but allows the user to specify the ``ExternalSystem``, the ``Model`` and the ``application`` explicity 29 | - `syncfiles` - Which synchronises multiple files, but uses a Regular Expression to find the required information about the ``ExternalSystem``, the ``Model`` and the ``application``. 30 | 31 | Usage 32 | ^^^^^ 33 | 34 | Create your CSV file(s) with the data you need to synchronise with:: 35 | 36 | first_name,last_name,employee_id,action_flags,match_on 37 | Andrew,Dodd,E1234,cu,employee_id 38 | Some,Other-Guy,E4321,d,employee_id 39 | 40 | 41 | Run one of the built in command (i.e. if you have a "Winner" Django model)s:: 42 | 43 | > python manage.py syncfile 'HRSystem' 'prizes' 'Winner' /tmp/the/file.csv 44 | 45 | Check your application to see that Andrew Dodd is now a Winner and that other guy was deleted. 46 | 47 | **NOTE WELL:** There is no need to write any Python to make this work! 48 | -------------------------------------------------------------------------------- /makemigrations.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | def make_migrations(): 5 | from django.core.management import call_command 6 | call_command('makemigrations', 'nsync') 7 | 8 | 9 | if __name__ == '__main__': 10 | import sys 11 | sys.path.append('./src/') 12 | 13 | try: 14 | from django.conf import settings 15 | 16 | settings.configure( 17 | INSTALLED_APPS=[ 18 | 'django.contrib.contenttypes', 19 | 'nsync', 20 | ], 21 | ) 22 | 23 | import django 24 | django.setup() 25 | 26 | except ImportError: 27 | import traceback 28 | traceback.print_exc() 29 | raise ImportError('To fix this error, sort out the imports') 30 | 31 | make_migrations() 32 | -------------------------------------------------------------------------------- /requirements-test.txt: -------------------------------------------------------------------------------- 1 | django>=1.8.0 2 | coverage 3 | mock>=1.0.1 4 | flake8>=2.1.0 5 | tox>=1.7.0 6 | 7 | # Additional test requirements go here 8 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | django>=1.8.0 2 | # Additional requirements go here 3 | -------------------------------------------------------------------------------- /requirements_dev.txt: -------------------------------------------------------------------------------- 1 | bumpversion==0.5.3 2 | wheel==0.24.0 3 | -------------------------------------------------------------------------------- /runtests.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | import django 4 | from django.conf import settings 5 | from django.test.utils import get_runner 6 | 7 | def setup_env(): 8 | sys.path.append('./src/') 9 | try: 10 | 11 | settings.configure( 12 | DATABASES={ 13 | 'default': { 14 | 'ENGINE': 'django.db.backends.sqlite3', 15 | } 16 | }, 17 | INSTALLED_APPS=[ 18 | 'django.contrib.contenttypes', 19 | 'nsync', 20 | 'tests', 21 | ], 22 | ) 23 | 24 | setup = django.setup() 25 | 26 | except ImportError: 27 | import traceback 28 | traceback.print_exc() 29 | raise ImportError('To fix this error, sort out the imports') 30 | 31 | 32 | def run_tests(*test_args): 33 | if not test_args: 34 | test_args = ['tests'] 35 | 36 | setup_env() 37 | # Run tests 38 | TestRunner = get_runner(settings) 39 | test_runner = TestRunner() 40 | 41 | failures = test_runner.run_tests(test_args) 42 | 43 | sys.exit(failures) 44 | 45 | 46 | if __name__ == '__main__': 47 | run_tests(*sys.argv[1:]) 48 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 0.4.0 3 | commit = True 4 | tag = True 5 | 6 | [bumpversion:file:src/nsync/__init__.py] 7 | 8 | [wheel] 9 | universal = 1 10 | 11 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import os 5 | import re 6 | 7 | 8 | # Start of section from Hynek 9 | # https://github.com/hynek/attrs/blob/master/setup.py 10 | import codecs 11 | from setuptools import setup, find_packages 12 | 13 | ############################################################################### 14 | 15 | PACKAGES = find_packages(where="src") 16 | META_PATH = os.path.join("src", "nsync", "__init__.py") 17 | KEYWORDS = [] 18 | CLASSIFIERS = [ 19 | 'Development Status :: 3 - Alpha', 20 | 'Framework :: Django', 21 | 'Framework :: Django :: 1.7', 22 | 'Framework :: Django :: 1.8', 23 | 'Framework :: Django :: 2.0', 24 | 'Intended Audience :: Developers', 25 | 'License :: OSI Approved :: MIT License', 26 | 'Natural Language :: English', 27 | 'Programming Language :: Python :: 3', 28 | 'Programming Language :: Python :: 3.3', 29 | 'Programming Language :: Python :: 3.4', 30 | 'Programming Language :: Python :: 3.5', 31 | 'Programming Language :: Python :: 3.6', 32 | ] 33 | INSTALL_REQUIRES = ['Django>=1.8'] 34 | TEST_SUITE = 'runtests.run_tests' 35 | TESTS_REQUIRE = ['Django>=1.8'] 36 | 37 | ############################################################################### 38 | 39 | HERE = os.path.abspath(os.path.dirname(__file__)) 40 | 41 | 42 | def read(*parts): 43 | """ 44 | Build an absolute path from *parts* and and return the contents of the 45 | resulting file. Assume UTF-8 encoding. 46 | """ 47 | with codecs.open(os.path.join(HERE, *parts), "rb", "utf-8") as f: 48 | return f.read() 49 | 50 | 51 | META_FILE = read(META_PATH) 52 | 53 | 54 | def find_meta(meta): 55 | """ 56 | Extract __*meta*__ from META_FILE. 57 | """ 58 | meta_match = re.search( 59 | r"^__{meta}__ = ['\"]([^'\"]*)['\"]".format(meta=meta), 60 | META_FILE, re.M 61 | ) 62 | if meta_match: 63 | return meta_match.group(1) 64 | raise RuntimeError("Unable to find __{meta}__ string.".format(meta=meta)) 65 | 66 | # End of section from Hynek 67 | 68 | readme = open('README.rst').read() 69 | history = open('HISTORY.rst').read().replace('.. :changelog:', '') 70 | 71 | setup( 72 | name=find_meta("title"), 73 | description=find_meta("description"), 74 | license=find_meta("license"), 75 | url=find_meta("uri"), 76 | version=find_meta("version"), 77 | author=find_meta("author"), 78 | author_email=find_meta("email"), 79 | maintainer=find_meta("author"), 80 | maintainer_email=find_meta("email"), 81 | keywords=KEYWORDS, 82 | long_description=read("README.rst"), 83 | packages=PACKAGES, 84 | package_dir={"": "src"}, 85 | zip_safe=False, 86 | classifiers=CLASSIFIERS, 87 | install_requires=INSTALL_REQUIRES, 88 | tests_require=TESTS_REQUIRE, 89 | test_suite=TEST_SUITE, 90 | ) 91 | -------------------------------------------------------------------------------- /src/nsync/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | __version__ = '0.4.0' 3 | 4 | __title__ = 'django-nsync' 5 | __description__ = 'Django N Sync provides a simple way to keep your Django ' \ 6 | 'Model data N Sync with N external systems' 7 | __uri__ = 'https://django-nsync.readthedocs.org/' 8 | 9 | __author__ = 'Andrew Dodd' 10 | __email__ = 'andrew.john.dodd@gmail.com' 11 | 12 | __license__ = 'MIT' 13 | __copyright__ = 'Copyright (c) 2015 Andrew Dodd' 14 | 15 | # No imports yet, not sure what exactly to import 16 | -------------------------------------------------------------------------------- /src/nsync/actions.py: -------------------------------------------------------------------------------- 1 | from django import VERSION 2 | from django.core.exceptions import ( 3 | MultipleObjectsReturned, 4 | ObjectDoesNotExist, 5 | FieldDoesNotExist) 6 | from django.contrib.contenttypes.fields import ContentType 7 | from django.db.models.query_utils import Q 8 | from .models import ExternalKeyMapping 9 | from collections import defaultdict 10 | import logging 11 | from .logging import StyleAdapter 12 | 13 | """ 14 | NSync actions for updating Django models 15 | 16 | This module contains the available actions for performing synchronisations. 17 | These include the basic Create / Update / Delete for model object, as well 18 | as the actions for managing the ExternalKeyMapping objects, to record the 19 | identification keys used by external systems for internal objects. 20 | 21 | It is recommended to use the ActionFactory.build() method to create actions 22 | from raw input. 23 | """ 24 | 25 | logger = logging.getLogger(__name__) 26 | logger.addHandler(logging.NullHandler()) # http://pieces.openpolitics.com/2012/04/python-logging-best-practices/ 27 | logger = StyleAdapter(logger) 28 | 29 | def set_value_to_remote(object, attribute, value): 30 | target_attr = getattr(object, attribute) 31 | if object._meta.get_field(attribute).one_to_one: 32 | if VERSION[0] == 1 and VERSION[1] < 9: 33 | target_attr = value 34 | else: 35 | target_attr.set(value) 36 | else: 37 | target_attr.add(value) 38 | 39 | class DissimilarActionTypesError(Exception): 40 | 41 | def __init__(self, action_type1, action_type2, field_name, model_name): 42 | self.action_types = [action_type1, action_type2] 43 | self.field_name = field_name 44 | self.model_name = model_name 45 | 46 | def __str__(self): 47 | return 'Dissimilar action types[{}] for many-to-many field {} on model {}'.format( 48 | ','.join(self.action_types), 49 | self.field_name, 50 | self.model_name) 51 | 52 | class UnknownActionType(Exception): 53 | 54 | def __init__(self, action_type,field_name, model_name): 55 | self.action_type = action_type 56 | self.field_name = field_name 57 | self.model_name = model_name 58 | 59 | def __str__(self): 60 | return 'Unknown action type[{}] for many-to-many field {} on model {}'.format( 61 | self.action_type, 62 | self.field_name, 63 | self.model_name) 64 | 65 | class ObjectSelector: 66 | OPERATORS = set(['|', '&', '~']) 67 | 68 | def __init__(self, match_on, available_fields): 69 | for field_name in match_on: 70 | if field_name in self.OPERATORS: 71 | continue 72 | 73 | if field_name not in available_fields: 74 | raise ValueError( 75 | 'field_name({}) must be in fields({})'.format( 76 | field_name, available_fields)) 77 | 78 | self.match_on = match_on 79 | self.fields = available_fields 80 | 81 | def get_by(self): 82 | def build_selector(match): 83 | return Q(**{match: self.fields[match]}) 84 | 85 | # if no operators present, then just AND all of the match_ons 86 | if len(self.OPERATORS.intersection(self.match_on)) == 0: 87 | match = self.match_on[0] 88 | q = build_selector(match) 89 | for match in self.match_on[1:]: 90 | q = q & build_selector(match) 91 | 92 | return q 93 | 94 | # process post-fix operator string 95 | stack = [] 96 | for match in self.match_on: 97 | if match in self.OPERATORS: 98 | if match is '~': 99 | if len(stack) < 1: 100 | raise ValueError('Insufficient operands for operator:{}', match) 101 | 102 | stack.append(~stack.pop()) 103 | continue 104 | 105 | if len(stack) < 2: 106 | raise ValueError('Insufficient operands for operator:{}', match) 107 | 108 | # remove the operands from the stack in reverse order 109 | # (preserves left-to-right reading) 110 | operand2 = stack.pop() 111 | operand1 = stack.pop() 112 | 113 | if match == '|': 114 | stack.append(operand1 | operand2) 115 | elif match == '&': 116 | stack.append(operand1 & operand2) 117 | else: 118 | pass 119 | else: 120 | stack.append(build_selector(match)) 121 | 122 | if len(stack) != 1: 123 | raise ValueError('Insufficient operators, stack:{}', stack) 124 | 125 | return stack[0] 126 | 127 | 128 | class ModelAction: 129 | """ 130 | The base action, which performs makes no modifications to objects. 131 | 132 | This class consolidates the some of the validity checking and the logic 133 | for finding the target objects. 134 | """ 135 | REFERRED_TO_DELIMITER = '=>' 136 | 137 | def __init__(self, model, match_on, fields={}): 138 | """ 139 | Create a base action. 140 | 141 | :param model: 142 | :param match_on: 143 | :param fields: 144 | :return: 145 | """ 146 | if model is None: 147 | raise ValueError('model cannot be None') 148 | if not match_on: 149 | raise ValueError('match_on({}) must be not "empty"'.format( 150 | match_on)) 151 | 152 | match_on = ObjectSelector(match_on, fields) 153 | 154 | self.model = model 155 | self.match_on = match_on 156 | self.fields = fields 157 | 158 | def __str__(self): 159 | return '{} - Model:{} - MatchFields:{} - Fields:{}'.format( 160 | self.__class__.__name__, 161 | self.model.__name__, 162 | self.match_on.match_on, 163 | self.fields) 164 | 165 | @property 166 | def type(self): 167 | return '' 168 | 169 | def get_object(self): 170 | """Finds the object that matches the provided matching information""" 171 | return self.model.objects.get(self.match_on.get_by()) 172 | 173 | def execute(self): 174 | """Does nothing""" 175 | pass 176 | 177 | def update_from_fields(self, object, force=False): 178 | """ 179 | Update the provided object with the fields. 180 | 181 | This is implemented in a consolidated place, as both Create and 182 | Update style actions require the functionality. 183 | 184 | :param object: the object to update 185 | :param force (bool): (Optional) Whether the update should only 186 | affect 'empty' fields. Default: False 187 | :return: 188 | """ 189 | # we need to support referential attributes, so look for them 190 | # as we iterate and store them for later 191 | 192 | # We store the referential attributes as a dict of dicts, this way 193 | # filtering against many fields is possible 194 | referential_attributes = defaultdict(dict) 195 | for attribute, value in self.fields.items(): 196 | if self.REFERRED_TO_DELIMITER in attribute and value != '': 197 | ref_attr = attribute.split(self.REFERRED_TO_DELIMITER) 198 | referential_attributes[ref_attr[0]][ref_attr[1]] = value 199 | else: 200 | if not force: 201 | current_value = getattr(object, attribute, None) 202 | if not (current_value is None or current_value is ''): 203 | continue 204 | try: 205 | if object._meta.get_field(attribute).null: 206 | value = None if value == '' else value 207 | except FieldDoesNotExist: 208 | pass 209 | setattr(object, attribute, value) 210 | 211 | for attribute, get_by in referential_attributes.items(): 212 | try: 213 | field = object._meta.get_field(attribute) 214 | # For migration advice of the get_field_by_name() call see [1] 215 | # [1]: https://docs.djangoproject.com/en/1.9/ref/models/meta/#migrating-old-meta-api 216 | 217 | if field.related_model: 218 | if field.concrete: 219 | own_attribute = field.name 220 | get_current_value = getattr 221 | set_value = setattr 222 | else: 223 | own_attribute = field.get_accessor_name() 224 | def get_value_from_remote(object, attribute, default): 225 | try: 226 | return getattr(object, attribute).get() 227 | except: 228 | return default 229 | get_current_value = get_value_from_remote 230 | set_value = set_value_to_remote 231 | 232 | if not force: 233 | current_value = get_current_value(object, own_attribute, None) 234 | if current_value is not None: 235 | continue 236 | 237 | try: 238 | if field.many_to_many: 239 | action_type = None 240 | get_by_exact = {} 241 | for k,v in get_by.items(): 242 | if action_type is None: 243 | action_type = k[0] 244 | elif action_type != k[0]: 245 | raise DissimilarActionTypesError( 246 | action_type, k[0], field.verbose_name, 247 | object.__class__.__name__) 248 | get_by_exact[k[1:]] = v 249 | 250 | if action_type not in '+-=': 251 | raise UnknownActionType(action_type, 252 | field.verbose_name, 253 | object.__class__.__name__) 254 | 255 | target = field.related_model.objects.get(**get_by_exact) 256 | 257 | if action_type is '+': 258 | getattr(object, own_attribute).add(target) 259 | elif action_type is '-': 260 | getattr(object, own_attribute).remove(target) 261 | elif action_type is '=': 262 | attr = getattr(object, own_attribute) 263 | # Django 1.9 impl => getattr(object, own_attribute).set([target]) 264 | attr.clear() 265 | for t in set([target]): 266 | attr.add(t) 267 | 268 | else: 269 | target = field.related_model.objects.get(**get_by) 270 | set_value(object, own_attribute, target) 271 | logger.debug(object) 272 | 273 | except ObjectDoesNotExist as e: 274 | logger.warning( 275 | 'Could not find {} with {} for {}[{}].{}', 276 | field.related_model.__name__, 277 | get_by, 278 | object.__class__.__name__, 279 | object, 280 | field.verbose_name) 281 | except MultipleObjectsReturned as e: 282 | logger.warning( 283 | 'Found multiple {} objects with {} for {}[{}].{}', 284 | field.related_model.__name__, 285 | get_by, 286 | object.__class__.__name__, 287 | object, 288 | field.verbose_name) 289 | except FieldDoesNotExist as e: 290 | logger.warning( 'Attibute "{}" does not exist on {}[{}]', 291 | attribute, 292 | object.__class__.__name__, 293 | object) 294 | 295 | except DissimilarActionTypesError as e: 296 | logger.warning('{}', e) 297 | 298 | except UnknownActionType as e: 299 | logger.warning('{}', e) 300 | 301 | 302 | class CreateModelAction(ModelAction): 303 | """ 304 | Action to create a model object if it does not exist. 305 | 306 | Note, this will not create another object if a matching one is 307 | found, nor will it update a matched object. 308 | """ 309 | 310 | def execute(self): 311 | try: 312 | return self.get_object() 313 | except ObjectDoesNotExist as e: 314 | pass 315 | except MultipleObjectsReturned as e: 316 | logger.warning('Mulitple objects found - {} Error:{}', str(self), e) 317 | return None 318 | 319 | 320 | obj=self.model() 321 | # NB: Create uses force to override defaults 322 | self.update_from_fields(obj, True) 323 | obj.save() 324 | return obj 325 | 326 | @property 327 | def type(self): 328 | return 'create' 329 | 330 | 331 | class CreateModelWithReferenceAction(CreateModelAction): 332 | """ 333 | Action to create a model object if it does not exist, and to create or 334 | update an external reference to the object. 335 | """ 336 | 337 | def __init__(self, external_system, model, 338 | external_key, match_on, fields={}): 339 | """ 340 | 341 | :param external_system (model object): The external system to create or 342 | update the reference for. 343 | :param external_key (str): The reference value from the external 344 | system (i.e. the 'id' that the external system uses to refer to the 345 | model object). 346 | :param model (class): See definition on super class 347 | :param match_on (list): See definition on super class 348 | :param fields(dict): See definition on super class 349 | :return: The model object provided by the action 350 | """ 351 | super(CreateModelWithReferenceAction, self).__init__( 352 | model, match_on, fields) 353 | self.external_system=external_system 354 | self.external_key=external_key 355 | 356 | def execute(self): 357 | try: 358 | mapping=ExternalKeyMapping.objects.get( 359 | external_system=self.external_system, 360 | external_key=self.external_key) 361 | except ExternalKeyMapping.DoesNotExist: 362 | mapping=ExternalKeyMapping( 363 | external_system=self.external_system, 364 | external_key=self.external_key) 365 | 366 | model_obj=mapping.content_object 367 | if model_obj is None: 368 | model_obj=super(CreateModelWithReferenceAction, self).execute() 369 | 370 | if model_obj: 371 | mapping.content_type=ContentType.objects.get_for_model( 372 | self.model) 373 | mapping.content_object=model_obj 374 | mapping.object_id=model_obj.id 375 | mapping.save() 376 | return model_obj 377 | 378 | 379 | from django.db import IntegrityError, transaction 380 | class UpdateModelAction(ModelAction): 381 | """ 382 | Action to update the fields of a model object, but not create an 383 | object. 384 | """ 385 | 386 | def __init__(self, model, match_on, fields={}, force_update=False): 387 | """ 388 | Create an Update action to be executed in the future. 389 | 390 | :param model (class): The model to update against 391 | :param match_on (list): A list of names of model attributes/fields 392 | to use to find the object to update. They must be a key in the 393 | provided fields. 394 | :param fields(dict): The set of fields to update, with the values to 395 | update them to. 396 | :param force_update(bool): (Optional) Whether the update should be 397 | forced or only affect 'empty' fields. Default:False 398 | :return: The updated object (if a matching object is found) or None. 399 | """ 400 | super(UpdateModelAction, self).__init__( 401 | model, match_on, fields) 402 | self.force_update=force_update 403 | 404 | @property 405 | def type(self): 406 | return 'update' 407 | 408 | def execute(self): 409 | try: 410 | obj=self.get_object() 411 | self.update_from_fields(obj, self.force_update) 412 | 413 | with transaction.atomic(): 414 | obj.save() 415 | 416 | return obj 417 | except ObjectDoesNotExist: 418 | return None 419 | except MultipleObjectsReturned as e: 420 | logger.warning('Mulitple objects found - {} Error:{}', str(self), e) 421 | return None 422 | except IntegrityError as e: 423 | logger.warning('Integrity issue - {} Error:{}', str(self), e) 424 | return None 425 | 426 | class UpdateModelWithReferenceAction(UpdateModelAction): 427 | """ 428 | Action to create a model object if it does not exist, and to create or 429 | update an external reference to the object. 430 | """ 431 | 432 | def __init__(self, external_system, model, external_key, match_on, 433 | fields={}, force_update=False): 434 | """ 435 | 436 | :param external_system (model object): The external system to create or 437 | update the reference for. 438 | :param external_key (str): The reference value from the external 439 | system (i.e. the 'id' that the external system uses to refer to the 440 | model object). 441 | 442 | :param model (class): See definition on super class 443 | :param match_on (list): See definition on super class 444 | :param fields(dict): See definition on super class 445 | :return: The updated object (if an object is found) or None. 446 | """ 447 | super(UpdateModelWithReferenceAction, self).__init__( 448 | model, match_on, fields, force_update) 449 | self.external_system=external_system 450 | self.external_key=external_key 451 | 452 | def execute(self): 453 | try: 454 | mapping=ExternalKeyMapping.objects.get( 455 | external_system=self.external_system, 456 | external_key=self.external_key) 457 | except ExternalKeyMapping.DoesNotExist: 458 | mapping=ExternalKeyMapping( 459 | external_system=self.external_system, 460 | external_key=self.external_key) 461 | 462 | linked_object=mapping.content_object 463 | 464 | matched_object=None 465 | try: 466 | matched_object=self.get_object() 467 | except ObjectDoesNotExist: 468 | pass 469 | except MultipleObjectsReturned as e: 470 | logger.warning('Mulitple objects found - {} Error:{}', str(self), e) 471 | return None 472 | 473 | # If both matched and linked objects exist but are different, 474 | # get rid of the matched one 475 | if matched_object and linked_object and (matched_object != 476 | linked_object): 477 | matched_object.delete() 478 | 479 | # Choose the most appropriate object to update 480 | if linked_object: 481 | model_obj=linked_object 482 | elif matched_object: 483 | model_obj=matched_object 484 | else: 485 | # No object to update 486 | return None 487 | 488 | if model_obj: 489 | self.update_from_fields(model_obj, self.force_update) 490 | try: 491 | with transaction.atomic(): 492 | model_obj.save() 493 | except IntegrityError as e: 494 | logger.warning('Integrity issue - {} Error:{}', str(self), e) 495 | return None 496 | 497 | if model_obj: 498 | mapping.content_type=ContentType.objects.get_for_model( 499 | self.model) 500 | mapping.content_object=model_obj 501 | mapping.object_id=model_obj.id 502 | mapping.save() 503 | 504 | return model_obj 505 | 506 | 507 | class DeleteIfOnlyReferenceModelAction(ModelAction): 508 | """ 509 | This action only deletes the pointed to object if the key mapping 510 | corresponding to 'this' external key it the only one 511 | 512 | I.e. if there are two references from different external systems to the 513 | same object, then the object will not be deleted. 514 | """ 515 | 516 | def __init__(self, external_system, external_key, delete_action): 517 | self.delete_action=delete_action 518 | self.external_key=external_key 519 | self.external_system=external_system 520 | 521 | @property 522 | def type(self): 523 | return self.delete_action.type 524 | 525 | def execute(self): 526 | try: 527 | obj=self.delete_action.get_object() 528 | 529 | key_mapping=ExternalKeyMapping.objects.get( 530 | object_id=obj.id, 531 | content_type=ContentType.objects.get_for_model( 532 | self.delete_action.model), 533 | external_key=self.external_key) 534 | 535 | if key_mapping.external_system == self.external_system: 536 | self.delete_action.execute() 537 | else: 538 | # The key mapping is not 'this' systems key mapping 539 | pass 540 | except MultipleObjectsReturned: 541 | # There are multiple key mappings or multiple target objects, we shouldn't delete the object 542 | return 543 | except ObjectDoesNotExist: 544 | return 545 | 546 | 547 | class DeleteModelAction(ModelAction): 548 | 549 | @property 550 | def type(self): 551 | return 'delete' 552 | 553 | def execute(self): 554 | """Forcibly delete any objects found by the 555 | ModelAction.get_object() method.""" 556 | try: 557 | self.get_object().delete() 558 | except ObjectDoesNotExist: 559 | pass 560 | except MultipleObjectsReturned as e: 561 | logger.warning('Mulitple objects found - {} Error:{}', str(self), e) 562 | return None 563 | 564 | 565 | class DeleteExternalReferenceAction: 566 | """ 567 | A model action to remove the ExternalKeyMapping object for a model object. 568 | """ 569 | 570 | def __init__(self, external_system, external_key): 571 | self.external_system=external_system 572 | self.external_key=external_key 573 | 574 | @property 575 | def type(self): 576 | return 'delete' 577 | 578 | def execute(self): 579 | """ 580 | Deletes all ExternalKeyMapping objects that match the provided external 581 | system and external key. 582 | :return: Nothing 583 | """ 584 | ExternalKeyMapping.objects.filter( 585 | external_system=self.external_system, 586 | external_key=self.external_key).delete() 587 | 588 | 589 | class ActionFactory: 590 | """ 591 | A factory for producing the most appropriate (set of) ModelAction objects. 592 | 593 | The factory takes care of creating the correct actions in the instances 594 | where it is a little complicated. In particular, when there are unforced 595 | delete actions. 596 | 597 | In the case of unforced delete actions, the builder will create a 598 | DeleteIfOnlyReferenceModelAction. This action will only delete the 599 | underlying model if there is a single link to the object to be deleted 600 | AND it is a link from the same system. 601 | 602 | Example 1: 603 | 604 | 1. Starting State 605 | ----------------- 606 | ExtSys 1 - Mapping 1 (Id: 123) --+ 607 | | 608 | v 609 | Model Object (Person: John) 610 | ^ 611 | | 612 | ExtSys 2 - Mapping 1 (Id: AABB) -+ 613 | 614 | 615 | 2. DeleteIfOnlyReferenceModelAction(ExtSys 2, AABB, DeleteAction(John)) 616 | ----------------------------------------------------------------------- 617 | Although there was a 'delete John' action, it was not performed 618 | because there is another system with a link to John. 619 | 620 | Example 2: 621 | 622 | 1. Starting State 623 | ----------------- 624 | ExtSys 1 - Mapping 1 (Id: 123) --+ 625 | | 626 | v 627 | Model Object (Person: John) 628 | 629 | 2. DeleteIfOnlyReferenceModelAction(ExtSys 2, AABB, DeleteAction(John)) 630 | ----------------------------------------------------------------------- 631 | Although there was only a single reference, it is not for ExtSys 2, 632 | hence the delete is not performed. 633 | 634 | The builder will also include a DeleteExternalReferenceAction if the 635 | provided action is 'externally mappable'. These will always be executed 636 | and will ensure that the reference objects will be removed by their 637 | respective sync systems (and that if they all work correctly the last 638 | one will be able to delete the object). 639 | """ 640 | 641 | def __init__(self, model, external_system=None): 642 | """ 643 | Create an actions factory for a given Django Model. 644 | 645 | :param model: The model to use for the actions 646 | :param external_system: (Optional) The external system object to 647 | create links against 648 | :return: A new actions factory 649 | """ 650 | self.model=model 651 | self.external_system=external_system 652 | 653 | def is_externally_mappable(self, external_key): 654 | """ 655 | Check if the an 'external system mapping' could be created for the 656 | provided key. 657 | :param external_key: 658 | :return: 659 | """ 660 | if self.external_system is None: 661 | return False 662 | 663 | if external_key is None: 664 | return False 665 | 666 | if not isinstance(external_key, str): 667 | return False 668 | 669 | return external_key.strip() is not '' 670 | 671 | def build(self, sync_actions, match_on, external_system_key, 672 | fields): 673 | """ 674 | Builds the list of actions to satisfy the provided information. 675 | 676 | This includes correctly building any actions required to keep the 677 | external system references correctly up to date. 678 | 679 | :param sync_actions: 680 | :param match_on: 681 | :param external_system_key: 682 | :param fields: 683 | :return: 684 | """ 685 | actions=[] 686 | 687 | if sync_actions.is_impotent(): 688 | actions.append(ModelAction(self.model, match_on, fields)) 689 | 690 | if sync_actions.delete: 691 | action=DeleteModelAction(self.model, match_on, fields) 692 | if self.is_externally_mappable(external_system_key): 693 | if not sync_actions.force: 694 | action=DeleteIfOnlyReferenceModelAction( 695 | self.external_system, external_system_key, action) 696 | actions.append(action) 697 | actions.append(DeleteExternalReferenceAction( 698 | self.external_system, external_system_key)) 699 | elif sync_actions.force: 700 | actions.append(action) 701 | 702 | if sync_actions.create: 703 | if self.is_externally_mappable(external_system_key): 704 | action=CreateModelWithReferenceAction(self.external_system, 705 | self.model, 706 | external_system_key, 707 | match_on, 708 | fields) 709 | else: 710 | action=CreateModelAction(self.model, match_on, fields) 711 | actions.append(action) 712 | if sync_actions.update: 713 | if self.is_externally_mappable(external_system_key): 714 | action=UpdateModelWithReferenceAction(self.external_system, 715 | self.model, 716 | external_system_key, 717 | match_on, 718 | fields, 719 | sync_actions.force) 720 | else: 721 | action=UpdateModelAction(self.model, match_on, 722 | fields, sync_actions.force) 723 | 724 | actions.append(action) 725 | 726 | return actions 727 | 728 | 729 | class SyncActions: 730 | """ 731 | A holder object for the actions that can be requested against a model 732 | object concurrently. 733 | """ 734 | 735 | def __init__(self, create=False, update=False, delete=False, force=False): 736 | if delete and create: 737 | raise ValueError("Cannot delete AND create") 738 | if delete and update: 739 | raise ValueError("Cannot delete AND update") 740 | 741 | self.create=create 742 | self.update=update 743 | self.delete=delete 744 | self.force=force 745 | 746 | def __str__(self): 747 | return "SyncActions {}{}{}{}".format( 748 | 'c' if self.create else '', 749 | 'u' if self.update else '', 750 | 'd' if self.delete else '', 751 | '*' if self.force else '') 752 | 753 | def is_impotent(self): 754 | return not (self.create or self.update or self.delete) 755 | 756 | -------------------------------------------------------------------------------- /src/nsync/logging.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | # https://docs.python.org/3/howto/logging-cookbook.html#using-custom-message-objects 4 | class Message(object): 5 | def __init__(self, fmt, args): 6 | self.fmt = fmt 7 | self.args = args 8 | 9 | def __str__(self): 10 | return self.fmt.format(*self.args) 11 | 12 | 13 | class StyleAdapter(logging.LoggerAdapter): 14 | def __init__(self, logger, extra=None): 15 | super(StyleAdapter, self).__init__(logger, extra or {}) 16 | 17 | def log(self, level, msg, *args, **kwargs): 18 | if self.isEnabledFor(level): 19 | msg, kwargs = self.process(msg, kwargs) 20 | self.logger._log(level, Message(msg, args), (), **kwargs) 21 | -------------------------------------------------------------------------------- /src/nsync/management/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andrewdodd/django-nsync/f2c0d21bf54c02a503b14a1cbe3a9723019e495f/src/nsync/management/__init__.py -------------------------------------------------------------------------------- /src/nsync/management/commands/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andrewdodd/django-nsync/f2c0d21bf54c02a503b14a1cbe3a9723019e495f/src/nsync/management/commands/__init__.py -------------------------------------------------------------------------------- /src/nsync/management/commands/syncfile.py: -------------------------------------------------------------------------------- 1 | from django.core.management.base import BaseCommand, CommandError 2 | import os 3 | import csv 4 | 5 | from .utils import ExternalSystemHelper, ModelFinder, CsvActionFactory 6 | from nsync.policies import BasicSyncPolicy, TransactionSyncPolicy 7 | 8 | 9 | class Command(BaseCommand): 10 | help = 'Synchonise model info from one file' 11 | 12 | def add_arguments(self, parser): 13 | # Mandatory 14 | parser.add_argument( 15 | 'ext_system_name', 16 | help='The name of the external system to use for storing ' 17 | 'sync information in relation to') 18 | parser.add_argument( 19 | 'app_label', 20 | default=None, 21 | help='The name of the application the model is part of') 22 | parser.add_argument( 23 | 'model_name', 24 | help='The name of the model to synchronise to') 25 | parser.add_argument( 26 | 'file_name', 27 | help='The file to synchronise from') 28 | 29 | # Optional 30 | parser.add_argument( 31 | '--create_external_system', 32 | type=bool, 33 | default=True, 34 | help='The name of the external system to use for storing ' 35 | 'sync information in relation to') 36 | parser.add_argument( 37 | '--as_transaction', 38 | type=bool, 39 | default=True, 40 | help='Wrap all of the actions in a DB transaction Default:True') 41 | 42 | def handle(self, *args, **options): 43 | external_system = ExternalSystemHelper.find( 44 | options['ext_system_name'], options['create_external_system']) 45 | model = ModelFinder.find(options['app_label'], options['model_name']) 46 | 47 | filename = options['file_name'] 48 | if not os.path.exists(filename): 49 | raise CommandError("Filename '{}' not found".format(filename)) 50 | 51 | with open(filename) as f: 52 | # TODO - Review - This indirection is only due to issues in 53 | # getting the mocks in the tests to work 54 | SyncFileAction.sync(external_system, 55 | model, 56 | f, 57 | options['as_transaction']) 58 | 59 | 60 | class SyncFileAction: 61 | @staticmethod 62 | def sync(external_system, model, file, use_transaction): 63 | reader = csv.DictReader(file) 64 | builder = CsvActionFactory(model, external_system) 65 | actions = [] 66 | for d in reader: 67 | actions.extend(builder.from_dict(d)) 68 | 69 | policy = BasicSyncPolicy(actions) 70 | 71 | if use_transaction: 72 | policy = TransactionSyncPolicy(policy) 73 | 74 | policy.execute() 75 | -------------------------------------------------------------------------------- /src/nsync/management/commands/syncfiles.py: -------------------------------------------------------------------------------- 1 | from django.core.management.base import BaseCommand, CommandError 2 | import os 3 | import csv 4 | import argparse 5 | import re 6 | from .utils import ( 7 | ExternalSystemHelper, 8 | ModelFinder, 9 | SupportedFileChecker, 10 | CsvActionFactory) 11 | from nsync.policies import ( 12 | BasicSyncPolicy, 13 | OrderedSyncPolicy, 14 | TransactionSyncPolicy 15 | ) 16 | 17 | (DEFAULT_FILE_REGEX) = (r'(?P[a-zA-Z0-9]+)_' 18 | r'(?P[a-zA-Z0-9]+)_' 19 | r'(?P[a-zA-Z0-9]+).*\.csv') 20 | 21 | 22 | class Command(BaseCommand): 23 | help = 'Sync info from a list of files' 24 | 25 | def add_arguments(self, parser): 26 | # Mandatory 27 | parser.add_argument('files', type=argparse.FileType('r'), nargs='+') 28 | # Optional 29 | parser.add_argument( 30 | '--file_name_regex', 31 | type=str, 32 | default=DEFAULT_FILE_REGEX, 33 | help='The regular expression to obtain the system name, app name ' 34 | 'and model name from each file') 35 | parser.add_argument( 36 | '--create_external_system', 37 | type=bool, 38 | default=True, 39 | help='If true, the command will create a matching external ' 40 | 'system object if one cannot be found') 41 | parser.add_argument( 42 | '--smart_ordering', 43 | type=bool, 44 | default=True, 45 | help='When this option it true, the command will perform all ' 46 | 'Create actions, then Update actions, and finally Delete ' 47 | 'actions. This ensures that if one file creates an object ' 48 | 'but another deletes it, the order that the files are ' 49 | 'provided to the command is not important. Default: True') 50 | parser.add_argument( 51 | '--as_transaction', 52 | type=bool, 53 | default=True, 54 | help='Wrap all of the actions in a DB transaction Default:True') 55 | 56 | def handle(self, *args, **options): 57 | TestableCommand(**options).execute() 58 | 59 | 60 | class TestableCommand: 61 | def __init__(self, **options): 62 | self.files = options['files'] 63 | self.pattern = re.compile(options['file_name_regex']) 64 | self.create_external_system = options['create_external_system'] 65 | self.ordered = options['smart_ordering'] 66 | self.use_transaction = options['as_transaction'] 67 | 68 | def execute(self): 69 | actions = self.collect_all_actions() 70 | 71 | if self.ordered: 72 | policy = OrderedSyncPolicy(actions) 73 | else: 74 | policy = BasicSyncPolicy(actions) 75 | 76 | if self.use_transaction: 77 | policy = TransactionSyncPolicy(policy) 78 | 79 | policy.execute() 80 | 81 | def collect_all_actions(self): 82 | actions = [] 83 | 84 | for f in self.files: 85 | if not SupportedFileChecker.is_valid(f): 86 | raise CommandError('Unsupported file:{}'.format(f)) 87 | 88 | basename = os.path.basename(f.name) 89 | (system, app, model) = TargetExtractor(self.pattern).extract( 90 | basename) 91 | external_system = ExternalSystemHelper.find( 92 | system, self.create_external_system) 93 | model = ModelFinder.find(app, model) 94 | 95 | reader = csv.DictReader(f) 96 | builder = CsvActionFactory(model, external_system) 97 | for d in reader: 98 | actions.extend(builder.from_dict(d)) 99 | return actions 100 | 101 | 102 | class TargetExtractor: 103 | def __init__(self, pattern): 104 | self.pattern = pattern 105 | 106 | def extract(self, filename): 107 | result = self.pattern.match(filename) 108 | return (result.group('external_system'), 109 | result.group('app_name'), 110 | result.group('model_name')) 111 | -------------------------------------------------------------------------------- /src/nsync/management/commands/utils.py: -------------------------------------------------------------------------------- 1 | from django.core.management.base import CommandError # TODO replace error 2 | from django.apps.registry import apps 3 | import csv 4 | 5 | from nsync.models import ExternalSystem 6 | from nsync.actions import ActionFactory, SyncActions 7 | 8 | 9 | class SupportedFileChecker: 10 | @staticmethod 11 | def is_valid(file): 12 | return file is not None 13 | 14 | 15 | class ModelFinder: 16 | @staticmethod 17 | def find(app_label, model_name): 18 | if not app_label: 19 | raise CommandError('Invalid app label "{}"'.format(app_label)) 20 | 21 | if not model_name: 22 | raise CommandError('Invalid model name "{}"'.format(model_name)) 23 | 24 | return apps.get_model(app_label, model_name) 25 | 26 | 27 | class ExternalSystemHelper: 28 | @staticmethod 29 | def find(name, create=True): 30 | if not name: 31 | raise CommandError('Invalid external system name "{}"'.format( 32 | name)) 33 | 34 | try: 35 | return ExternalSystem.objects.get(name=name) 36 | except ExternalSystem.DoesNotExist: 37 | if create: 38 | return ExternalSystem.objects.create(name=name, 39 | description=name) 40 | else: 41 | raise CommandError('ExternalSystem "{}" not found'.format( 42 | name)) 43 | 44 | 45 | class CsvActionFactory(ActionFactory): 46 | action_flags_label = 'action_flags' 47 | external_key_label = 'external_key' 48 | match_on_label = 'match_on' 49 | match_on_delimiter = ' ' 50 | 51 | def from_dict(self, raw_values): 52 | if not raw_values: 53 | return [] 54 | 55 | action_flags = raw_values.pop(self.action_flags_label) 56 | match_on = raw_values.pop(self.match_on_label) 57 | match_on = match_on.split( 58 | self.match_on_delimiter) 59 | external_system_key = raw_values.pop(self.external_key_label, None) 60 | 61 | sync_actions = CsvSyncActionsDecoder.decode(action_flags) 62 | 63 | return self.build(sync_actions, match_on, 64 | external_system_key, raw_values) 65 | 66 | 67 | class CsvSyncActionsEncoder: 68 | @staticmethod 69 | def encode(sync_actions): 70 | return '{}{}{}{}'.format( 71 | 'c' if sync_actions.create else '', 72 | 'u' if sync_actions.update else '', 73 | 'd' if sync_actions.delete else '', 74 | '*' if sync_actions.force else '') 75 | 76 | 77 | class CsvSyncActionsDecoder: 78 | @staticmethod 79 | def decode(action_flags): 80 | create = False 81 | update = False 82 | delete = False 83 | force = False 84 | 85 | if action_flags: 86 | try: 87 | create = 'C' in action_flags or 'c' in action_flags 88 | update = 'U' in action_flags or 'u' in action_flags 89 | delete = 'D' in action_flags or 'd' in action_flags 90 | force = '*' in action_flags 91 | except TypeError: 92 | # not iterable 93 | pass 94 | 95 | return SyncActions(create, update, delete, force) 96 | -------------------------------------------------------------------------------- /src/nsync/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import migrations, models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('contenttypes', '0002_remove_content_type_name'), 11 | ] 12 | 13 | operations = [ 14 | migrations.CreateModel( 15 | name='ExternalKeyMapping', 16 | fields=[ 17 | ('id', models.AutoField(serialize=False, verbose_name='ID', primary_key=True, auto_created=True)), 18 | ('object_id', models.PositiveIntegerField()), 19 | ('external_key', models.CharField(max_length=80, help_text='The key of the internal object in the external system.')), 20 | ('content_type', models.ForeignKey(to='contenttypes.ContentType', help_text='The type of object that is mapped to the external system.', on_delete=models.CASCADE)), 21 | ], 22 | options={ 23 | 'verbose_name': 'External Key Mapping', 24 | }, 25 | ), 26 | migrations.CreateModel( 27 | name='ExternalSystem', 28 | fields=[ 29 | ('id', models.AutoField(serialize=False, verbose_name='ID', primary_key=True, auto_created=True)), 30 | ('name', models.CharField(unique=True, db_index=True, max_length=30, help_text='A short name, used by software applications\n to locate this particular External System.\n ')), 31 | ('description', models.CharField(blank=True, max_length=80, help_text='A human readable name for this External System.')), 32 | ], 33 | options={ 34 | 'verbose_name': 'External System', 35 | }, 36 | ), 37 | migrations.AddField( 38 | model_name='externalkeymapping', 39 | name='external_system', 40 | field=models.ForeignKey(to='nsync.ExternalSystem', on_delete=models.CASCADE), 41 | ), 42 | migrations.AlterUniqueTogether( 43 | name='externalkeymapping', 44 | unique_together=set([('external_system', 'external_key')]), 45 | ), 46 | migrations.AlterIndexTogether( 47 | name='externalkeymapping', 48 | index_together=set([('external_system', 'external_key')]), 49 | ), 50 | ] 51 | -------------------------------------------------------------------------------- /src/nsync/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andrewdodd/django-nsync/f2c0d21bf54c02a503b14a1cbe3a9723019e495f/src/nsync/migrations/__init__.py -------------------------------------------------------------------------------- /src/nsync/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | from django.contrib.contenttypes.fields import GenericForeignKey 3 | from django.contrib.contenttypes.models import ContentType 4 | 5 | 6 | class ExternalSystem(models.Model): 7 | """ 8 | Computer systems that integration occurs against 9 | """ 10 | name = models.CharField( 11 | blank=False, 12 | db_index=True, 13 | help_text="""A short name, used by software applications 14 | to locate this particular External System. 15 | """, 16 | max_length=30, 17 | unique=True, 18 | ) 19 | description = models.CharField( 20 | blank=True, 21 | help_text='A human readable name for this External System.', 22 | max_length=80, 23 | ) 24 | 25 | class Meta: 26 | verbose_name = 'External System' 27 | 28 | def __str__(self): 29 | return self.description if self.description else self.name 30 | 31 | 32 | class ExternalKeyMapping(models.Model): 33 | """ 34 | Key Mappings for objects in our system to objects in external systems 35 | """ 36 | content_type = models.ForeignKey( 37 | ContentType, 38 | help_text='The type of object that is mapped to the external system.', 39 | on_delete=models.CASCADE) 40 | object_id = models.PositiveIntegerField() 41 | content_object = GenericForeignKey('content_type', 'object_id') 42 | external_system = models.ForeignKey(ExternalSystem, on_delete=models.CASCADE) 43 | external_key = models.CharField( 44 | blank=False, 45 | help_text='The key of the internal object in the external system.', 46 | max_length=80, 47 | ) 48 | 49 | class Meta: 50 | index_together = ('external_system', 'external_key') 51 | unique_together = ('external_system', 'external_key') 52 | verbose_name = 'External Key Mapping' 53 | 54 | def __str__(self): 55 | return '{}:{}-{}:{}'.format( 56 | self.external_system.name, 57 | self.external_key, 58 | self.content_type.model_class().__name__, 59 | self.object_id) 60 | -------------------------------------------------------------------------------- /src/nsync/policies.py: -------------------------------------------------------------------------------- 1 | from django.db import transaction 2 | 3 | 4 | class BasicSyncPolicy: 5 | """A synchronisation policy that simply executes each action in order.""" 6 | def __init__(self, actions): 7 | """ 8 | Create a basic synchronisation policy. 9 | 10 | :param actions: The list of actions to perform 11 | :return: Nothing 12 | """ 13 | self.actions = actions 14 | 15 | def execute(self): 16 | for action in self.actions: 17 | action.execute() 18 | 19 | 20 | class TransactionSyncPolicy: 21 | """ 22 | A synchronisation policy that wraps other sync policies in a database 23 | transaction. 24 | 25 | This allows the changes from all of the actions to occur in an atomic 26 | fashion. The limit to the number of transactions is database dependent 27 | but is usually quite large (i.e. like 2^32). 28 | """ 29 | def __init__(self, policy): 30 | self.policy = policy 31 | 32 | def execute(self): 33 | with transaction.atomic(): 34 | self.policy.execute() 35 | 36 | 37 | class OrderedSyncPolicy: 38 | """ 39 | A synchronisation policy that performs the actions in a controlled order. 40 | 41 | This policy filters the list of actions and executes all of the create 42 | actions, then all of the update actions and finally all of the delete 43 | actions. This is to ensure that the whole list of actions behaves more 44 | predictably. 45 | 46 | For example, if there are create actions and forced delete actions for 47 | the same object in the list, then the net result of the state of the 48 | objects will depend on which action is performed first. If the order is 49 | 'create' then 'delete', the object will be created and then deleted. If 50 | the order is 'delete' then 'create', the delete action will fail and 51 | then the object will be created. This policy avoids this situation by 52 | performing the different types in order. 53 | 54 | This also helps with referential updates, where an update action might be 55 | earlier in the list than the action to create the referred to object. 56 | """ 57 | def __init__(self, actions): 58 | self.actions = actions 59 | 60 | def execute(self): 61 | for filter_by in ['create', 'update', 'delete']: 62 | filtered_actions = filter(lambda a: a.type == filter_by, 63 | self.actions) 64 | for action in filtered_actions: 65 | action.execute() 66 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andrewdodd/django-nsync/f2c0d21bf54c02a503b14a1cbe3a9723019e495f/tests/__init__.py -------------------------------------------------------------------------------- /tests/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | 4 | class TestPerson(models.Model): 5 | first_name = models.CharField( 6 | blank=False, 7 | max_length=50, 8 | verbose_name='First Name' 9 | ) 10 | last_name = models.CharField( 11 | blank=False, 12 | max_length=50, 13 | verbose_name='Last Name' 14 | ) 15 | age = models.IntegerField(blank=True, null=True) 16 | hair_colour = models.CharField( 17 | blank=False, 18 | max_length=50, 19 | default="Unknown") 20 | 21 | def __str__(self): 22 | return '{}:{} {} - {}'.format(self.id, 23 | self.first_name, 24 | self.last_name, 25 | self.age) 26 | 27 | 28 | class TestHouse(models.Model): 29 | address = models.CharField(max_length=100) 30 | country = models.CharField(max_length=100, blank=True) 31 | floors = models.IntegerField(blank=True, null=True) 32 | owner = models.ForeignKey(TestPerson, blank=True, null=True, related_name='houses', 33 | on_delete=models.CASCADE) 34 | built = models.DateField(blank=True, null=True) 35 | 36 | def __str__(self): 37 | return '{} - {}{}{}{}'.format( 38 | self.address, 39 | self.built, 40 | ', {}'.format(self.country) if self.country else '', 41 | ' - {} floors'.format(self.floors) if self.floors else '', 42 | ' - Family:{}'.format(self.owner.last_name) if self.owner else '') 43 | 44 | 45 | class TestBuilder(TestPerson): 46 | company = models.CharField( 47 | blank=False, 48 | max_length=50, 49 | default='Self employed' 50 | ) 51 | buildings = models.ManyToManyField( 52 | TestHouse, 53 | related_name='builders' 54 | ) 55 | 56 | 57 | -------------------------------------------------------------------------------- /tests/test_actions.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import MagicMock, patch, ANY 2 | 3 | from django.contrib.contenttypes.fields import ContentType 4 | from django.core.exceptions import ObjectDoesNotExist 5 | from django.db.models.query_utils import Q 6 | from django.test import TestCase 7 | from nsync.actions import ( 8 | CreateModelAction, 9 | UpdateModelAction, 10 | DeleteModelAction, 11 | CreateModelWithReferenceAction, 12 | UpdateModelWithReferenceAction, 13 | DeleteExternalReferenceAction, 14 | DeleteIfOnlyReferenceModelAction, 15 | SyncActions, 16 | ActionFactory, 17 | ObjectSelector, 18 | ModelAction) 19 | from nsync.models import ExternalSystem, ExternalKeyMapping 20 | 21 | from tests.models import TestPerson, TestHouse, TestBuilder 22 | 23 | 24 | class TestSyncActions(TestCase): 25 | def test_sync_actions_raises_error_if_action_includes_create_and_delete( 26 | self): 27 | with self.assertRaises(ValueError): 28 | SyncActions(create=True, delete=True) 29 | 30 | def test_sync_actions_raises_error_if_action_includes_update_and_delete( 31 | self): 32 | with self.assertRaises(ValueError): 33 | SyncActions(update=True, delete=True) 34 | 35 | def test_string_representations_are_correct(self): 36 | self.assertIn('c', str(SyncActions(create=True))) 37 | self.assertIn('u', str(SyncActions(update=True))) 38 | self.assertIn('d', str(SyncActions(delete=True))) 39 | self.assertIn('cu', str(SyncActions(create=True, update=True))) 40 | self.assertIn('u*', str(SyncActions(update=True, force=True))) 41 | self.assertIn('d*', str(SyncActions(delete=True, force=True))) 42 | self.assertIn('cu*', 43 | str(SyncActions(create=True, update=True, force=True))) 44 | 45 | # http://jamescooke.info/comparing-django-q-objects.html 46 | class QTestMixin(object): 47 | 48 | def assertQEqual(self, left, right): 49 | """ 50 | Assert `Q` objects are equal by ensuring that their 51 | unicode outputs are equal (crappy but good enough) 52 | """ 53 | self.assertIsInstance(left, Q) 54 | self.assertIsInstance(right, Q) 55 | left_u = str(left) 56 | right_u = str(right) 57 | self.assertEqual(left_u, right_u) 58 | 59 | 60 | class TestObjectSelector(TestCase, QTestMixin): 61 | def setUp(self): 62 | self.fields = { 'field' + str(i): 'value' + str(i) for i in range(1,6)} 63 | 64 | def test_it_raises_an_error_if_match_on_field_not_in_available_fields(self): 65 | with self.assertRaises(ValueError): 66 | ObjectSelector(['field'], {'' :'value'}) 67 | 68 | def test_it_does_not_raise_error_for_pipe_character(self): 69 | ObjectSelector(['|'], {'' :'value'}) 70 | 71 | def test_it_does_not_raise_error_for_ampersand_character(self): 72 | ObjectSelector(['&'], {'' :'value'}) 73 | 74 | def test_it_does_not_raise_error_for_tilde_character(self): 75 | ObjectSelector(['~'], {'' :'value'}) 76 | 77 | def test_it_supports_a_single_get_by_field(self): 78 | sut = ObjectSelector(['field1'], self.fields) 79 | result = sut.get_by() 80 | self.assertQEqual(Q(field1='value1'), result) 81 | 82 | def test_get_by_returns_an_AND_filter_by_default(self): 83 | sut = ObjectSelector(['field1', 'field2'], self.fields) 84 | result = sut.get_by() 85 | self.assertQEqual(Q(field1='value1') & Q(field2='value2'), result) 86 | 87 | def test_it_supports_postfix_style_AND_filter_by(self): 88 | sut = ObjectSelector(['field1', 'field2', '&'], self.fields) 89 | result = sut.get_by() 90 | self.assertQEqual(Q(field1='value1') & Q(field2='value2'), result) 91 | 92 | def test_it_supports_postfix_style_OR_filter_by(self): 93 | sut = ObjectSelector(['field1', 'field2', '|'], self.fields) 94 | result = sut.get_by() 95 | self.assertQEqual(Q(field1='value1') | Q(field2='value2'), result) 96 | 97 | def test_it_supports_postfix_style_NOT_filter_by(self): 98 | sut = ObjectSelector(['field1', '~'], self.fields) 99 | result = sut.get_by() 100 | self.assertQEqual(~Q(field1='value1'), result) 101 | 102 | def test_it_supports_postfix_style_filter_by_options_extended(self): 103 | sut = ObjectSelector(['field1', 'field2', '&', 'field3', '~', 'field4', '&', '|'], self.fields) 104 | result = sut.get_by() 105 | self.assertQEqual((Q(field1='value1') & Q(field2='value2')) | 106 | (~Q(field3='value3') & Q(field4='value4')), result) 107 | 108 | def test_it_raises_an_error_if_insufficient_operands_for_AND(self): 109 | with self.assertRaises(ValueError): 110 | sut = ObjectSelector(['field1', '&'], self.fields) 111 | result = sut.get_by() 112 | 113 | def test_it_raises_an_error_if_insufficient_operands_for_OR(self): 114 | with self.assertRaises(ValueError): 115 | sut = ObjectSelector(['field1', '|'], self.fields) 116 | result = sut.get_by() 117 | 118 | def test_it_raises_an_error_if_insufficient_operands_for_NOT(self): 119 | with self.assertRaises(ValueError): 120 | sut = ObjectSelector(['~'], self.fields) 121 | result = sut.get_by() 122 | 123 | def test_it_raises_an_error_if_insufficient_operators(self): 124 | with self.assertRaises(ValueError): 125 | sut = ObjectSelector(['field1', 'field2', 'field3', '&'], self.fields) 126 | result = sut.get_by() 127 | 128 | 129 | class TestModelAction(TestCase): 130 | # http://stackoverflow.com/questions/899067/how-should-i-verify-a-log-message-when-testing-python-code-under-nose/20553331#20553331 131 | @classmethod 132 | def setUpClass(cls): 133 | super(TestModelAction, cls).setUpClass() 134 | # Assuming you follow Python's logging module's documentation's 135 | # recommendation about naming your module's logs after the module's 136 | # __name__,the following getLogger call should fetch the same logger 137 | # you use in the foo module 138 | import logging 139 | import nsync 140 | from tests.test_utils import MockLoggingHandler 141 | logger = logging.getLogger(nsync.actions.__name__) 142 | cls._logger_handler = MockLoggingHandler(level='DEBUG') 143 | logger.addHandler(cls._logger_handler) 144 | cls.logger_messages = cls._logger_handler.messages 145 | 146 | def setUp(self): 147 | super(TestModelAction, self).setUp() 148 | self._logger_handler.reset() # So each test is independent 149 | 150 | def test_it_has_custom_string_format(self): 151 | sut = ModelAction(TestPerson, ['match_field'], {'match_field':'value'}) 152 | result = str(sut) 153 | self.assertIn("ModelAction", result) 154 | self.assertIn("Model:TestPerson", result) 155 | self.assertIn("MatchFields:['match_field']", result) 156 | self.assertIn("Fields:{'match_field': 'value'}", result) 157 | 158 | def test_creating_without_a_model_raises_error(self): 159 | with self.assertRaises(ValueError): 160 | ModelAction(None, None) 161 | 162 | # TODO - Perhaps update this to look for the attribute on the class? 163 | def test_it_raises_an_error_if_match_on_is_empty(self): 164 | """ 165 | Test that an error is raises if an empty match_on value is provided, 166 | even if the fields dict has a matching key 167 | """ 168 | with self.assertRaises(ValueError): 169 | ModelAction(ANY, [], {'': 'value'}) 170 | 171 | def test_it_raises_error_if_any_match_on_not_in_fields(self): 172 | with self.assertRaises(ValueError): 173 | ModelAction(ANY, 174 | ['matchingfield', 'missingfield'], 175 | {'matchingfield': 'value'}) 176 | 177 | def test_it_attempts_to_find_through_the_provided_model_class(self): 178 | model = MagicMock() 179 | found_object = ModelAction(model, ['matchfield'], 180 | {'matchfield': 'value'}).get_object() 181 | self.assertEqual(found_object, model.objects.get.return_value) 182 | 183 | def test_it_attempts_to_find_with_all_matchfields(self): 184 | model = MagicMock() 185 | found_object = ModelAction( 186 | model, 187 | ['matchfield1', 'matchfield2'], 188 | {'matchfield1': 'value1', 'matchfield2': 'value2'}).get_object() 189 | self.assertEqual(str(Q(matchfield1='value1') & Q(matchfield2='value2')), 190 | str(model.objects.get.call_args[0][0])) 191 | self.assertEqual(found_object, model.objects.get.return_value) 192 | 193 | def test_it_builds_OR_Q_object_if_last_token_is_pipe(self): 194 | model = MagicMock() 195 | found_object = ModelAction( 196 | model, 197 | ['matchfield1', 'matchfield2', '|'], 198 | {'matchfield1': 'value1', 'matchfield2': 'value2'}).get_object() 199 | self.assertEqual(str(Q(matchfield1='value1') | Q(matchfield2='value2')), 200 | str(model.objects.get.call_args[0][0])) 201 | self.assertEqual(found_object, model.objects.get.return_value) 202 | 203 | def test_it_finds_an_object_with_alternative_options(self): 204 | model = MagicMock() 205 | found_object = ModelAction( 206 | model, 207 | ['matchfield1', 'matchfield2', '|'], 208 | {'matchfield1': 'value1', 'matchfield2': 'value2'}).get_object() 209 | self.assertEqual(str(Q(matchfield1='value1') | Q(matchfield2='value2')), 210 | str(model.objects.get.call_args[0][0])) 211 | self.assertEqual(found_object, model.objects.get.return_value) 212 | john = TestPerson.objects.create(first_name='John', last_name='Smith') 213 | jill = TestPerson.objects.create(first_name='Jill', last_name='Smyth') 214 | 215 | john = ModelAction(TestPerson, 216 | ['first_name', 'last_name', '|'], 217 | {'first_name': '', 'last_name': 'Smith'}).get_object() 218 | jill = ModelAction(TestPerson, 219 | ['first_name', 'last_name', '|'], 220 | {'first_name': 'Jill', 'last_name': ''}).get_object() 221 | 222 | self.assertEqual('Smith', john.last_name) 223 | self.assertEqual('Smyth', jill.last_name) 224 | 225 | 226 | def test_update_from_fields_changes_values_on_object(self): 227 | john = TestPerson(first_name='John') 228 | ModelAction(TestPerson, ['last_name'], 229 | {'last_name': 'Smith'}).update_from_fields(john) 230 | self.assertEqual('Smith', john.last_name) 231 | 232 | def test_update_from_uses_none_if_field_is_nullable_and_value_is_empty_string(self): 233 | house = TestHouse.objects.create(address='Bottom of the hill') 234 | fields = {'address': 'Bottom of the hill', 'built': ''} 235 | 236 | sut = ModelAction(TestHouse, ['address'], fields) 237 | sut.update_from_fields(house) 238 | house.save() 239 | house.refresh_from_db() 240 | self.assertEqual(house.built, None) 241 | 242 | def test_update_from_fields_updates_related_fields(self): 243 | person = TestPerson.objects.create(first_name="Jill", 244 | last_name="Jones") 245 | house = TestHouse.objects.create(address='Bottom of the hill') 246 | fields = {'address': 'Bottom of the hill', 'owner=>first_name': 'Jill'} 247 | 248 | sut = ModelAction(TestHouse, ['address'], fields) 249 | sut.update_from_fields(house) 250 | self.assertEqual(person, house.owner) 251 | 252 | def test_update_from_fields_updates_related_fields_from_opposite_direction(self): 253 | person = TestPerson.objects.create(first_name="Jill", 254 | last_name="Jones") 255 | house = TestHouse.objects.create(address='Bottom of the hill') 256 | fields = {'first_name': 'Jill', 'houses=>address': 'Bottom of the hill'} 257 | 258 | sut = ModelAction(TestPerson, ['first_name'], fields) 259 | sut.update_from_fields(person) 260 | self.assertEqual(house, person.houses.get()) 261 | 262 | def test_error_is_logged_if_field_not_on_object(self): 263 | house = TestHouse.objects.create(address='Bottom of the hill') 264 | fields = {'address': 'Bottom of the hill', 'buyer=>last_name': 'Jones'} 265 | 266 | sut = ModelAction(TestHouse, ['address'], fields) 267 | sut.update_from_fields(house) 268 | 269 | for msg in self.logger_messages['warning']: 270 | self.assertIn('buyer', msg) 271 | 272 | def test_related_fields_not_touched_if_referred_to_object_does_not_exist( 273 | self): 274 | house = TestHouse.objects.create(address='Bottom of the hill') 275 | fields = {'address': 'Bottom of the hill', 'owner=>last_name': 'Jones'} 276 | 277 | sut = ModelAction(TestHouse, ['address'], fields) 278 | sut.update_from_fields(house) 279 | self.assertEqual(None, house.owner) 280 | 281 | for msg in self.logger_messages['warning']: 282 | self.assertIn('Could not find TestPerson', msg) 283 | self.assertIn(str({'last_name': 'Jones'}), msg) 284 | 285 | def test_related_fields_are_not_touched_if_referred_to_object_ambiguous( 286 | self): 287 | TestPerson.objects.create(first_name="Jill", last_name="Jones") 288 | TestPerson.objects.create(first_name="Jack", last_name="Jones") 289 | house = TestHouse.objects.create(address='Bottom of the hill') 290 | fields = {'address': 'Bottom of the hill', 'owner=>last_name': 'Jones'} 291 | sut = ModelAction(TestHouse, ['address'], fields) 292 | sut.update_from_fields(house) 293 | self.assertEqual(None, house.owner) 294 | 295 | for msg in self.logger_messages['warning']: 296 | self.assertIn('Found multiple TestPerson objects', msg) 297 | self.assertIn(str({'last_name': 'Jones'}), msg) 298 | 299 | 300 | def test_related_fields_update_uses_all_available_filters(self): 301 | person = TestPerson.objects.create(first_name="John", 302 | last_name="Johnson") 303 | TestPerson.objects.create(first_name="Jack", last_name="Johnson") 304 | TestPerson.objects.create(first_name="John", last_name="Jackson") 305 | TestPerson.objects.create(first_name="Jack", last_name="Jackson") 306 | 307 | house = TestHouse.objects.create(address='Bottom of the hill') 308 | fields = { 309 | 'address': 'Bottom of the hill', 310 | 'owner=>first_name': 'John', 311 | 'owner=>last_name': 'Johnson'} 312 | sut = ModelAction(TestHouse, ['address'], fields) 313 | sut.update_from_fields(house) 314 | self.assertEqual(person, house.owner) 315 | 316 | def test_related_fields_update_does_not_use_filters_with_values_as_empty_strings(self): 317 | """ 318 | This effectively prevents over-specification, and allows files 319 | to be constructed with "or" style relations 320 | """ 321 | jill = TestPerson.objects.create(first_name="Jill", last_name="Hill") 322 | jack = TestPerson.objects.create(first_name="Jack", last_name="Shack") 323 | house = TestHouse.objects.create(address='Bottom of the hill') 324 | 325 | fields = { 326 | 'address': 'Bottom of the hill', 327 | 'owner=>first_name': '', 328 | 'owner=>last_name': 'Shack'} 329 | sut = ModelAction(TestHouse, ['address'], fields) 330 | sut.update_from_fields(house, True) 331 | self.assertEqual(jack, house.owner) 332 | 333 | fields = { 334 | 'address': 'Bottom of the hill', 335 | 'owner=>first_name': 'Jill', 336 | 'owner=>last_name': ''} 337 | sut = ModelAction(TestHouse, ['address'], fields) 338 | sut.update_from_fields(house, True) 339 | self.assertEqual(jill, house.owner) 340 | 341 | def test_update_from_fields_does_not_update_values_that_are_not_empty( 342 | self): 343 | john = TestPerson(first_name='John', last_name='Smith') 344 | ModelAction(TestPerson, ['last_name'], 345 | {'last_name': 'Jackson'}).update_from_fields(john) 346 | self.assertEqual('Smith', john.last_name) 347 | 348 | def test_update_from_fields_always_updates_fields_when_forced( 349 | self): 350 | john = TestPerson(first_name='John', last_name='Smith') 351 | ModelAction(TestPerson, ['last_name'], 352 | {'last_name': 'Jackson'}).update_from_fields(john, True) 353 | self.assertEqual('Jackson', john.last_name) 354 | 355 | def test_related_fields_update_does_not_update_if_already_assigned( 356 | self): 357 | jill = TestPerson.objects.create(first_name="Jill", last_name="Jones") 358 | jack = TestPerson.objects.create(first_name="Jack", last_name="Jones") 359 | house = TestHouse.objects.create(address='Bottom of the hill', 360 | owner=jill) 361 | 362 | fields = { 363 | 'address': 'Bottom of the hill', 364 | 'owner=>first_name': 'Jack', 365 | 'owner=>last_name': 'Jones'} 366 | sut = ModelAction(TestHouse, ['address'], fields) 367 | sut.update_from_fields(house) 368 | self.assertEqual(jill, house.owner) 369 | self.assertNotEqual(jack, house.owner) 370 | 371 | def test_related_fields_update_does_update_if_forced(self): 372 | jill = TestPerson.objects.create(first_name="Jill", last_name="Jones") 373 | jack = TestPerson.objects.create(first_name="Jack", last_name="Jones") 374 | house = TestHouse.objects.create(address='Bottom of the hill', 375 | owner=jill) 376 | 377 | fields = { 378 | 'address': 'Bottom of the hill', 379 | 'owner=>first_name': 'Jack', 380 | 'owner=>last_name': 'Jones'} 381 | sut = ModelAction(TestHouse, ['address'], fields) 382 | sut.update_from_fields(house, True) 383 | self.assertEqual(jack, house.owner) 384 | 385 | def test_it_does_not_update_many_to_many_fields_with_simple_referred_to_delimiter(self): 386 | bob = TestBuilder.objects.create(first_name="Bob", last_name="The Builder") 387 | house = TestHouse.objects.create(address='Bottom of the hill') 388 | 389 | fields = { 390 | 'first_name': 'Bob', 391 | 'buildings=>address': 'Bottom of the hill', 392 | } 393 | sut = ModelAction(TestBuilder, ['first_name'], fields) 394 | sut.update_from_fields(house, True) 395 | self.assertNotIn(house, bob.buildings.all()) 396 | 397 | def test_it_adds_elements_to_many_to_many_with_plus_referred_to_delimiter(self): 398 | bob = TestBuilder.objects.create(first_name="Bob", last_name="The Builder") 399 | house = TestHouse.objects.create(address='Bottom of the hill') 400 | 401 | fields = { 402 | 'first_name': 'Bob', 403 | 'buildings=>+address': 'Bottom of the hill', 404 | } 405 | sut = ModelAction(TestBuilder, ['first_name'], fields) 406 | 407 | sut.update_from_fields(bob, True) 408 | self.assertIn(house, bob.buildings.all()) 409 | 410 | def test_it_adds_elements_to_many_to_many_from_opposite_direction(self): 411 | bob = TestBuilder.objects.create(first_name="Bob", last_name="The Builder") 412 | house = TestHouse.objects.create(address='Bottom of the hill') 413 | 414 | fields = { 415 | 'address': 'Bottom of the hill', 416 | 'builders=>+first_name': 'Bob', 417 | } 418 | sut = ModelAction(TestBuilder, ['address'], fields) 419 | 420 | sut.update_from_fields(house, True) 421 | self.assertIn(bob, house.builders.all()) 422 | 423 | 424 | def test_it_removes_elements_from_many_to_many_with_minus_referred_to_delimiter(self): 425 | house = TestHouse.objects.create(address='Bottom of the hill') 426 | bob = TestBuilder.objects.create(first_name="Bob", last_name="The Builder") 427 | bob.buildings.add(house) 428 | bob.save() 429 | 430 | fields = { 431 | 'first_name': 'Bob', 432 | 'buildings=>-address': 'Bottom of the hill', 433 | } 434 | sut = ModelAction(TestBuilder, ['first_name'], fields) 435 | sut.update_from_fields(bob, True) 436 | self.assertNotIn(house, bob.buildings.all()) 437 | 438 | def test_it_replaces_all_elements_in_many_to_many_with_equals_referred_to_delimiter(self): 439 | house1 = TestHouse.objects.create(address='Bottom of the hill') 440 | house2 = TestHouse.objects.create(address='Top of the hill') 441 | bob = TestBuilder.objects.create(first_name="Bob", last_name="The Builder") 442 | bob.buildings.add(house1) 443 | bob.save() 444 | 445 | fields = { 446 | 'first_name': 'Bob', 447 | 'buildings=>=address': 'Top of the hill', 448 | } 449 | sut = ModelAction(TestBuilder, ['first_name'], fields) 450 | sut.update_from_fields(bob, True) 451 | self.assertNotIn(house1, bob.buildings.all()) 452 | self.assertIn(house2, bob.buildings.all()) 453 | 454 | def test_it_uses_all_related_fields_to_find_targets_for_many_to_many_fields(self): 455 | house1 = TestHouse.objects.create(address='Bottom of the hill', country='Australia') 456 | house2 = TestHouse.objects.create(address='Bottom of the hill', country='Belgium') 457 | bob = TestBuilder.objects.create(first_name="Bob", last_name="The Builder") 458 | bob.buildings.add(house1) 459 | bob.save() 460 | 461 | fields = { 462 | 'first_name': 'Bob', 463 | 'buildings=>=address': 'Bottom of the hill', 464 | 'buildings=>=country': 'Belgium', 465 | } 466 | sut = ModelAction(TestBuilder, ['first_name'], fields) 467 | sut.update_from_fields(bob, True) 468 | self.assertNotIn(house1, bob.buildings.all()) 469 | self.assertIn(house2, bob.buildings.all()) 470 | 471 | def test_it_logs_an_error_if_action_type_for_many_to_many_referred_fields_is_unknown(self): 472 | bob = TestBuilder.objects.create(first_name="Bob", last_name="The Builder") 473 | 474 | fields = { 475 | 'first_name': 'Bob', 476 | 'buildings=>*address': 'Bottom of the hill', 477 | } 478 | 479 | sut = ModelAction(TestBuilder, ['first_name'], fields) 480 | sut.update_from_fields(bob, True) 481 | 482 | for msg in self.logger_messages['warning']: 483 | self.assertIn('Unknown action type', msg) 484 | 485 | def test_it_logs_an_error_if_action_type_for_many_to_many_referred_fields_is_dissimilar(self): 486 | bob = TestBuilder.objects.create(first_name="Bob", last_name="The Builder") 487 | 488 | fields = { 489 | 'first_name': 'Bob', 490 | 'buildings=>+address': 'Bottom of the hill', 491 | 'buildings=>=country': 'Belgium', 492 | } 493 | 494 | sut = ModelAction(TestBuilder, ['first_name'], fields) 495 | sut.update_from_fields(bob, True) 496 | 497 | for msg in self.logger_messages['warning']: 498 | self.assertIn('Dissimilar action types', msg) 499 | 500 | 501 | class TestActionFactory(TestCase): 502 | def setUp(self): 503 | self.model = MagicMock() 504 | self.sut = ActionFactory(self.model) 505 | 506 | @patch('nsync.actions.ModelAction') 507 | def test_it_creates_a_base_model_action_if_no_action_flags_are_included( 508 | self, ModelAction): 509 | result = self.sut.build(SyncActions(), ['field'], None, {'field': ''}) 510 | ModelAction.assert_called_with(self.model, ['field'], {'field': ''}) 511 | self.assertIn(ModelAction.return_value, result) 512 | 513 | @patch('nsync.actions.CreateModelAction') 514 | def test_it_calls_create_action_with_correct_parameters(self, 515 | TargetActionClass): 516 | result = self.sut.build(SyncActions(create=True), ['field'], ANY, 517 | {'field': ''}) 518 | TargetActionClass.assert_called_with(self.model, ['field'], 519 | {'field': ''}) 520 | self.assertIn(TargetActionClass.return_value, result) 521 | 522 | @patch('nsync.actions.UpdateModelAction') 523 | def test_it_calls_update_action_with_correct_parameters(self, 524 | TargetActionClass): 525 | for actions in [SyncActions(update=True, force=False), 526 | SyncActions(update=True, force=True)]: 527 | result = self.sut.build(actions, ['field'], ANY, {'field': ''}) 528 | TargetActionClass.assert_called_with(self.model, ['field'], 529 | {'field': ''}, actions.force) 530 | self.assertIn(TargetActionClass.return_value, result) 531 | 532 | @patch('nsync.actions.DeleteModelAction') 533 | def test_it_calls_delete_action_with_correct_parameters(self, 534 | TargetActionClass): 535 | result = self.sut.build(SyncActions(delete=True, force=True), ['field'], 536 | ANY, {'field': ''}) 537 | TargetActionClass.assert_called_with(self.model, ['field'], 538 | {'field': ''}) 539 | self.assertIn(TargetActionClass.return_value, result) 540 | 541 | def test_delete_action_not_built_if_unforced_and_not_externally_mappable( 542 | self): 543 | """ 544 | If there is no external mapping AND the delete is not forced, 545 | then the usual 'DeleteIfOnlyReferenceModelAction' will not actually 546 | do anything anyway, so test that no actions are built. 547 | """ 548 | result = self.sut.build(SyncActions(delete=True), ['field'], ANY, 549 | {'field': ''}) 550 | self.assertEqual([], result) 551 | 552 | def test_it_creates_two_actions_if_create_and_update_flags_are_included( 553 | self): 554 | result = self.sut.build(SyncActions(create=True, update=True), ['field'], 555 | ANY, {'field': ''}) 556 | self.assertEqual(2, len(result)) 557 | 558 | def test_it_considers_nothing_externally_mappable_without_external_system( 559 | self): 560 | self.assertIs(False, self.sut.is_externally_mappable('')) 561 | self.assertIs(False, self.sut.is_externally_mappable("a mappable key")) 562 | 563 | def test_it_considers_non_strings_as_not_externally_mappable(self): 564 | self.assertFalse(ActionFactory(ANY, ANY).is_externally_mappable(None)) 565 | self.assertFalse(ActionFactory(ANY, ANY).is_externally_mappable(0)) 566 | self.assertFalse(ActionFactory(ANY, ANY).is_externally_mappable(1)) 567 | self.assertFalse(ActionFactory(ANY, ANY).is_externally_mappable(ANY)) 568 | 569 | def test_it_considers_non_blank_strings_as_externally_mappable(self): 570 | self.assertTrue( 571 | ActionFactory(ANY, ANY).is_externally_mappable('a mappable key')) 572 | 573 | @patch('nsync.actions.CreateModelWithReferenceAction') 574 | def test_it_builds_a_create_with_external_action_if_externally_mappable( 575 | self, builtAction): 576 | external_system_mock = MagicMock() 577 | model_mock = MagicMock() 578 | sut = ActionFactory(model_mock, external_system_mock) 579 | result = sut.build(SyncActions(create=True), ['field'], 'external_key', 580 | {'field': 'value'}) 581 | builtAction.assert_called_with( 582 | external_system_mock, model_mock, 583 | 'external_key', ['field'], {'field': 'value'}) 584 | self.assertIn(builtAction.return_value, result) 585 | 586 | @patch('nsync.actions.UpdateModelWithReferenceAction') 587 | def test_it_builds_an_update_with_external_action_if_externally_mappable( 588 | self, builtAction): 589 | external_system_mock = MagicMock() 590 | model_mock = MagicMock() 591 | sut = ActionFactory(model_mock, external_system_mock) 592 | result = sut.build(SyncActions(update=True), ['field'], 'external_key', 593 | {'field': 'value'}) 594 | builtAction.assert_called_with( 595 | external_system_mock, model_mock, 596 | 'external_key', ['field'], {'field': 'value'}, False) 597 | self.assertIn(builtAction.return_value, result) 598 | 599 | @patch('nsync.actions.DeleteExternalReferenceAction') 600 | def test_it_creates_delete_external_reference_if_externally_mappable( 601 | self, DeleteExternalReferenceAction): 602 | external_system_mock = MagicMock() 603 | model_mock = MagicMock() 604 | sut = ActionFactory(model_mock, external_system_mock) 605 | result = sut.build(SyncActions(delete=True), ['field'], 'external_key', 606 | {'field': 'value'}) 607 | DeleteExternalReferenceAction.assert_called_with( 608 | external_system_mock, 'external_key') 609 | self.assertIn(DeleteExternalReferenceAction.return_value, result) 610 | 611 | @patch('nsync.actions.DeleteModelAction') 612 | def test_it_creates_delete_action_for_forced_delete_if_externally_mappable( 613 | self, DeleteModelAction): 614 | external_system_mock = MagicMock() 615 | model_mock = MagicMock() 616 | sut = ActionFactory(model_mock, external_system_mock) 617 | result = sut.build(SyncActions(delete=True, force=True), ['field'], 618 | 'external_key', {'field': 'value'}) 619 | DeleteModelAction.assert_called_with(model_mock, ['field'], 620 | {'field': 'value'}) 621 | self.assertIn(DeleteModelAction.return_value, result) 622 | 623 | @patch('nsync.actions.DeleteIfOnlyReferenceModelAction') 624 | @patch('nsync.actions.DeleteModelAction') 625 | def test_it_wraps_delete_action_if_externally_mappable( 626 | self, DeleteModelAction, DeleteIfOnlyReferenceModelAction): 627 | external_system_mock = MagicMock() 628 | model_mock = MagicMock() 629 | sut = ActionFactory(model_mock, external_system_mock) 630 | result = sut.build(SyncActions(delete=True), ['field'], 'external_key', 631 | {'field': 'value'}) 632 | DeleteModelAction.assert_called_with(model_mock, ['field'], 633 | {'field': 'value'}) 634 | DeleteIfOnlyReferenceModelAction.assert_called_with( 635 | external_system_mock, 636 | 'external_key', 637 | DeleteModelAction.return_value) 638 | self.assertIn(DeleteIfOnlyReferenceModelAction.return_value, result) 639 | 640 | 641 | class TestCreateModelAction(TestCase): 642 | def test_it_creates_an_object(self): 643 | sut = CreateModelAction(TestPerson, ['first_name'], 644 | {'first_name': 'John', 'last_name': 'Smith'}) 645 | sut.execute() 646 | self.assertEqual(1, TestPerson.objects.count()) 647 | 648 | def test_it_returns_the_created_object(self): 649 | sut = CreateModelAction(TestPerson, ['first_name'], 650 | {'first_name': 'John', 'last_name': 'Smith'}) 651 | result = sut.execute() 652 | self.assertEqual(TestPerson.objects.first(), result) 653 | 654 | def test_it_does_not_create_if_matching_object_exists(self): 655 | TestPerson.objects.create(first_name='John', last_name='Smith') 656 | sut = CreateModelAction(TestPerson, ['first_name'], 657 | {'first_name': 'John', 'last_name': 'Smith'}) 658 | sut.execute() 659 | self.assertEqual(1, TestPerson.objects.count()) 660 | 661 | def test_it_does_not_modify_existing_object_if_object_already_exists(self): 662 | TestPerson.objects.create(first_name='John', last_name='Jackson') 663 | sut = CreateModelAction(TestPerson, ['first_name'], 664 | {'first_name': 'John', 'last_name': 'Smith'}) 665 | sut.execute() 666 | self.assertEqual(1, TestPerson.objects.count()) 667 | self.assertEquals('Jackson', TestPerson.objects.first().last_name) 668 | 669 | def test_it_returns_the_object_even_if_it_did_not_create_it(self): 670 | john = TestPerson.objects.create(first_name='John', last_name='Smith') 671 | sut = CreateModelAction(TestPerson, ['first_name'], 672 | {'first_name': 'John', 'last_name': 'Smith'}) 673 | result = sut.execute() 674 | self.assertEqual(john, result) 675 | 676 | def test_it_uses_all_included_fields_and_overrides_defaults( 677 | self): 678 | sut = CreateModelAction( 679 | TestPerson, 680 | ['first_name'], 681 | {'first_name': 'John', 'last_name': 'Smith', 'age': 30, 682 | 'hair_colour': 'None, he bald!'}) 683 | result = sut.execute() 684 | self.assertEqual('John', result.first_name) 685 | self.assertEqual('Smith', result.last_name) 686 | self.assertEqual(30, result.age) 687 | self.assertEqual('None, he bald!', result.hair_colour) 688 | 689 | 690 | class TestCreateModelWithReferenceAction(TestCase): 691 | """ 692 | These tests try to cover off the following matrix of behaviour: 693 | +---------------+---------------------+----------------------------------------+ 694 | | Model object | External Link | Behaviour / Outcome desired | 695 | +---------------+---------------------+----------------------------------------+ 696 | | | | The standard case. The model object | 697 | | No | No | should be created, and if successful | 698 | | | | an external linkage object should be | 699 | | | | created to point to it. | 700 | +------------------------------------------------------------------------------+ 701 | | | | Object already exists case. An | 702 | | Exists | No | external linkage object is created | 703 | | | | and pointed at the existing object. | 704 | +------------------------------------------------------------------------------+ 705 | | | | A previously made object was deleted. | 706 | | No | Exists | Create the model object and update | 707 | | | | the linkage object to point at it. | 708 | +------------------------------------------------------------------------------+ 709 | | Exists | Exists, points to | Already pointing to matching / created | 710 | | | matching object | object. Do nothing. NOT TESTED | 711 | +------------------------------------------------------------------------------+ 712 | | | Exists but points | Pointing to a non-matching object. Do | 713 | | Exists | to some 'other' | nothing but potentially log/warn of | 714 | | | object | the discrepancy | 715 | +---------------+---------------------+----------------------------------------+ 716 | """ 717 | def setUp(self): 718 | self.external_system = ExternalSystem.objects.create(name='System') 719 | 720 | def test_it_creates_the_model_object(self): 721 | sut = CreateModelWithReferenceAction( 722 | self.external_system, 723 | TestPerson, 'PersonJohn', ['first_name'], 724 | {'first_name': 'John', 'last_name': 'Smith'}) 725 | sut.execute() 726 | self.assertEqual(1, TestPerson.objects.count()) 727 | 728 | def test_it_creates_the_reference(self): 729 | sut = CreateModelWithReferenceAction( 730 | self.external_system, 731 | TestPerson, 'PersonJohn', ['first_name'], 732 | {'first_name': 'John', 'last_name': 'Smith'}) 733 | sut.execute() 734 | self.assertEqual(1, ExternalKeyMapping.objects.count()) 735 | 736 | def test_it_creates_the_reference_if_the_model_object_already_exists(self): 737 | john = TestPerson.objects.create(first_name='John') 738 | sut = CreateModelWithReferenceAction( 739 | self.external_system, 740 | TestPerson, 'PersonJohn', ['first_name'], 741 | {'first_name': 'John', 'last_name': 'Smith'}) 742 | sut.execute() 743 | self.assertEqual(1, TestPerson.objects.count()) 744 | self.assertEqual(1, ExternalKeyMapping.objects.count()) 745 | self.assertEqual(john, ExternalKeyMapping.objects.first().content_object) 746 | 747 | def test_it_updates_the_reference_if_the_model_does_not_exist(self): 748 | mapping = ExternalKeyMapping.objects.create( 749 | external_system=self.external_system, 750 | external_key='PersonJohn', 751 | content_type=ContentType.objects.get_for_model(TestPerson), 752 | object_id=0) 753 | sut = CreateModelWithReferenceAction( 754 | self.external_system, 755 | TestPerson, 'PersonJohn', ['first_name'], 756 | {'first_name': 'John', 'last_name': 'Smith'}) 757 | sut.execute() 758 | self.assertEqual(1, ExternalKeyMapping.objects.count()) 759 | mapping.refresh_from_db() 760 | self.assertNotEqual(None, mapping.content_object) 761 | 762 | def test_it_does_not_create_model_object_if_reference_is_linked_to_model(self): 763 | """ 764 | A model object should not be created if an external key mapping 765 | exists, even if the mapping and the match fields do not agree. 766 | """ 767 | # create a reference & model objet 768 | CreateModelWithReferenceAction( 769 | self.external_system, 770 | TestPerson, 'PersonJohn', ['first_name'], 771 | {'first_name': 'John', 'last_name': 'Smith'}).execute() 772 | 773 | # Attempt to create another object with different data but same external key 774 | CreateModelWithReferenceAction( 775 | self.external_system, 776 | TestPerson, 'PersonJohn', ['first_name'], 777 | {'first_name': 'David', 'last_name': 'Jones'}).execute() 778 | 779 | linked_person = ExternalKeyMapping.objects.first().content_object 780 | self.assertEqual('John', linked_person.first_name) 781 | self.assertEqual('Smith', linked_person.last_name) 782 | self.assertEqual(1, TestPerson.objects.count()) 783 | self.assertEqual(1, ExternalKeyMapping.objects.count()) 784 | 785 | 786 | class TestUpdateModelAction(TestCase): 787 | def test_it_returns_the_object_even_if_nothing_updated(self): 788 | john = TestPerson.objects.create(first_name='John') 789 | sut = UpdateModelAction(TestPerson, ['first_name'], 790 | {'first_name': 'John'}) 791 | result = sut.execute() 792 | self.assertEquals(john, result) 793 | 794 | def test_it_returns_nothing_if_object_does_not_exist(self): 795 | sut = UpdateModelAction(TestPerson, ['first_name'], 796 | {'first_name': 'John', 'last_name': 'Smith'}) 797 | result = sut.execute() 798 | self.assertIsNone(result) 799 | 800 | def test_it_updates_model_with_new_values(self): 801 | john = TestPerson.objects.create(first_name='John') 802 | self.assertEqual('', john.last_name) 803 | 804 | sut = UpdateModelAction(TestPerson, ['first_name'], 805 | {'first_name': 'John', 'last_name': 'Smith'}) 806 | sut.execute() 807 | john.refresh_from_db() 808 | self.assertEquals('Smith', john.last_name) 809 | 810 | # TODO Review this behaviour? 811 | def test_including_extra_parameters_has_no_effect(self): 812 | john = TestPerson.objects.create(first_name='John') 813 | sut = UpdateModelAction( 814 | TestPerson, 815 | ['first_name'], 816 | {'first_name': 'John', 817 | 'totally_never_going_to_be_a_field': 'Smith'}) 818 | result = sut.execute() 819 | self.assertEquals(john, result) 820 | 821 | def test_it_forces_updates_when_configured(self): 822 | john = TestPerson.objects.create(first_name='John', last_name='Smith') 823 | sut = UpdateModelAction(TestPerson, ['first_name'], 824 | {'first_name': 'John', 'last_name': 'Jackson'}, 825 | True) 826 | sut.execute() 827 | john.refresh_from_db() 828 | self.assertEquals('Jackson', john.last_name) 829 | 830 | 831 | class TestUpdateModelWithReferenceAction(TestCase): 832 | """ 833 | These tests try to cover off the following matrix of behaviour: 834 | +---------------+---------------------+----------------------------------------+ 835 | | Model object | External Link | Behaviour / Outcome desired | 836 | +---------------+---------------------+----------------------------------------+ 837 | | No | No | Do nothing. No effect for update | 838 | +------------------------------------------------------------------------------+ 839 | | | | Do normal update. | 840 | | Exists | No | Create linkage object. | 841 | +------------------------------------------------------------------------------+ 842 | | | | !!! Should probably not happen !!! | 843 | | No | Exists | Do nothing. | 844 | | | | (Possibly remove link?) | 845 | +------------------------------------------------------------------------------+ 846 | | Exists | Exists, points to | Do normal update. | 847 | | | matching object | NOT TESTED | 848 | +------------------------------------------------------------------------------+ 849 | | | Exists but points | This case is tricky. | 850 | | Exists | to some 'other' | Do the update normally, obeying force | 851 | | | object | option. | 852 | | | | IF there is also a matching object ( | 853 | | | | that is not the linked object), not | 854 | | | | sure what to do, for now delete the | 855 | | | | non-linked object. | 856 | +---------------+---------------------+----------------------------------------+ 857 | """ 858 | def setUp(self): 859 | self.external_system = ExternalSystem.objects.create(name='System') 860 | self.update_john = UpdateModelWithReferenceAction( 861 | self.external_system, 862 | TestPerson, 'PersonJohn', ['first_name'], 863 | {'first_name': 'John', 'last_name': 'Smith'}, 864 | True) 865 | 866 | def test_it_does_not_create_a_reference_if_object_does_not_exist(self): 867 | self.update_john.execute() 868 | self.assertEqual(0, ExternalKeyMapping.objects.count()) 869 | 870 | def test_it_updates_the_object(self): 871 | john = TestPerson.objects.create(first_name='John') 872 | self.update_john.execute() 873 | john.refresh_from_db() 874 | self.assertEqual(john.first_name, 'John') 875 | self.assertEqual(john.last_name, 'Smith') 876 | 877 | def test_it_creates_the_reference_if_the_model_object_already_exists(self): 878 | john = TestPerson.objects.create(first_name='John') 879 | self.update_john.execute() 880 | self.assertEqual(1, ExternalKeyMapping.objects.count()) 881 | mapping = ExternalKeyMapping.objects.first() 882 | self.assertEqual(self.external_system, mapping.external_system) 883 | self.assertEqual('PersonJohn', mapping.external_key) 884 | self.assertEqual(john, mapping.content_object) 885 | 886 | def test_it_updates_the_linked_object(self): 887 | """ 888 | Tests that even if the match field does not 'match', the already 889 | pointed to object is updated. 890 | """ 891 | person = TestPerson.objects.create(first_name='Not John') 892 | mapping = ExternalKeyMapping.objects.create( 893 | external_system=self.external_system, 894 | external_key='PersonJohn', 895 | content_type = ContentType.objects.get_for_model( 896 | TestPerson), 897 | content_object = person, 898 | object_id = person.id) 899 | 900 | self.update_john.execute() 901 | person.refresh_from_db() 902 | self.assertEqual(person.first_name, 'John') 903 | self.assertEqual(person.last_name, 'Smith') 904 | 905 | def test_it_removes_the_matched_object_if_there_is_a_linked_object(self): 906 | """ 907 | Tests that if there is a 'linked' object to update, it removes any 908 | 'matched' objects in the process. 909 | In this test, the "John Jackson" person should be deleted and the 910 | "Not John" person should be updated to be "John Smith" 911 | """ 912 | matched_person = TestPerson.objects.create(first_name='John', 913 | last_name='Jackson') 914 | linked_person = TestPerson.objects.create(first_name='Not John') 915 | mapping = ExternalKeyMapping.objects.create( 916 | external_system=self.external_system, 917 | external_key='PersonJohn', 918 | content_type = ContentType.objects.get_for_model( 919 | TestPerson), 920 | content_object = linked_person, 921 | object_id = linked_person.id) 922 | 923 | self.update_john.execute() 924 | self.assertEqual(1, TestPerson.objects.count()) 925 | person = TestPerson.objects.first() 926 | self.assertEqual(person.first_name, 'John') 927 | self.assertEqual(person.last_name, 'Smith') 928 | 929 | def test_it_only_deletes_the_matched_object_if_it_is_not_the_linked_object(self): 930 | """ 931 | Tests that if there are both a linked and matched object (which is the 932 | standard case!!!), the matched_object is only removed if it is actually 933 | a different object. 934 | """ 935 | person = TestPerson.objects.create(first_name='John', 936 | last_name='Jackson') 937 | house = TestHouse.objects.create(address='Bottom of the hill', 938 | owner=person) 939 | mapping = ExternalKeyMapping.objects.create( 940 | external_system=self.external_system, 941 | external_key='PersonJohn', 942 | content_type = ContentType.objects.get_for_model( 943 | TestPerson), 944 | content_object = person, 945 | object_id = person.id) 946 | 947 | with patch.object(self.update_john, 'get_object') as get_object: 948 | get_object.return_value.return_value = person 949 | with patch.object(person, 'delete') as delete: 950 | self.update_john.execute() 951 | delete.assert_not_called() 952 | 953 | 954 | class TestDeleteModelAction(TestCase): 955 | def test_no_objects_are_deleted_if_none_are_matched(self): 956 | john = TestPerson.objects.create(first_name='John') 957 | DeleteModelAction(TestPerson, ['first_name'], 958 | {'first_name': 'A non-matching name'}).execute() 959 | self.assertIn(john, TestPerson.objects.all()) 960 | 961 | def test_only_objects_with_matching_fields_are_deleted(self): 962 | TestPerson.objects.create(first_name='John') 963 | TestPerson.objects.create(first_name='Jack') 964 | TestPerson.objects.create(first_name='Jill') 965 | self.assertEqual(3, TestPerson.objects.count()) 966 | 967 | DeleteModelAction(TestPerson, ['first_name'], 968 | {'first_name': 'Jack'}).execute() 969 | self.assertFalse(TestPerson.objects.filter(first_name='Jack').exists()) 970 | 971 | 972 | class TestDeleteIfOnlyReferenceModelAction(TestCase): 973 | def setUp(self): 974 | self.external_system = ExternalSystem.objects.create(name='System') 975 | 976 | def test_it_does_nothing_if_object_does_not_exist(self): 977 | delete_action = MagicMock() 978 | delete_action.get_object.side_effect = ObjectDoesNotExist 979 | DeleteIfOnlyReferenceModelAction(ANY, ANY, delete_action).execute() 980 | delete_action.get_object.assert_called_with() 981 | self.assertFalse(delete_action.execute.called) 982 | 983 | def test_it_does_nothing_if_no_key_mapping_is_found( 984 | self): 985 | john = TestPerson.objects.create(first_name='John') 986 | delete_action = MagicMock() 987 | delete_action.model = TestPerson 988 | delete_action.get_object.return_value = john 989 | DeleteIfOnlyReferenceModelAction(self.external_system, 'SomeKey', 990 | delete_action).execute() 991 | self.assertFalse(delete_action.execute.called) 992 | 993 | def test_it_calls_delete_action_if_it_is_the_only_key_mapping(self): 994 | john = TestPerson.objects.create(first_name='John') 995 | ExternalKeyMapping.objects.create( 996 | external_system=self.external_system, 997 | external_key='Person123', 998 | content_type=ContentType.objects.get_for_model(TestPerson), 999 | content_object=john, 1000 | object_id=john.id) 1001 | delete_action = MagicMock() 1002 | delete_action.model = TestPerson 1003 | delete_action.get_object.return_value = john 1004 | DeleteIfOnlyReferenceModelAction(self.external_system, 'Person123', 1005 | delete_action).execute() 1006 | delete_action.execute.assert_called_with() 1007 | 1008 | def test_it_does_not_call_the_delete_action_if_it_is_not_the_key_mapping( 1009 | self): 1010 | john = TestPerson.objects.create(first_name='John') 1011 | ExternalKeyMapping.objects.create( 1012 | external_system=ExternalSystem.objects.create( 1013 | name='AlternateSystem'), 1014 | external_key='Person123', 1015 | content_type=ContentType.objects.get_for_model(TestPerson), 1016 | content_object=john, 1017 | object_id=john.id) 1018 | delete_action = MagicMock() 1019 | delete_action.model = TestPerson 1020 | delete_action.get_object.return_value = john 1021 | DeleteIfOnlyReferenceModelAction(self.external_system, 'Person123', 1022 | delete_action).execute() 1023 | self.assertFalse(delete_action.execute.called) 1024 | 1025 | def test_it_does_not_call_the_delete_action_if_there_are_other_mappings( 1026 | self): 1027 | john = TestPerson.objects.create(first_name='John') 1028 | ExternalKeyMapping.objects.create( 1029 | external_system=self.external_system, 1030 | external_key='Person123', 1031 | content_type=ContentType.objects.get_for_model(TestPerson), 1032 | content_object=john, 1033 | object_id=john.id) 1034 | ExternalKeyMapping.objects.create( 1035 | external_system=ExternalSystem.objects.create(name='OtherSystem'), 1036 | external_key='Person123', 1037 | content_type=ContentType.objects.get_for_model(TestPerson), 1038 | content_object=john, 1039 | object_id=john.id) 1040 | delete_action = MagicMock() 1041 | delete_action.model = TestPerson 1042 | delete_action.get_object.return_value = john 1043 | DeleteIfOnlyReferenceModelAction(self.external_system, 'Person123', 1044 | delete_action).execute() 1045 | self.assertFalse(delete_action.execute.called) 1046 | delete_action.execute.assert_not_called() # Works in py3.5 1047 | 1048 | 1049 | class TestDeleteExternalReferenceAction(TestCase): 1050 | def test_it_deletes_the_matching_external_reference(self): 1051 | external_system = ExternalSystem.objects.create(name='System') 1052 | ExternalKeyMapping.objects.create( 1053 | external_system=external_system, 1054 | external_key='Key1', 1055 | content_type=ContentType.objects.get_for_model(TestPerson), 1056 | object_id=1) 1057 | ExternalKeyMapping.objects.create( 1058 | external_system=external_system, 1059 | external_key='Key2', 1060 | content_type=ContentType.objects.get_for_model(TestPerson), 1061 | object_id=1) 1062 | ExternalKeyMapping.objects.create( 1063 | external_system=external_system, 1064 | external_key='Key3', 1065 | content_type=ContentType.objects.get_for_model(TestPerson), 1066 | object_id=1) 1067 | self.assertEqual(3, ExternalKeyMapping.objects.count()) 1068 | DeleteExternalReferenceAction(external_system, 'Key2').execute() 1069 | self.assertEqual(2, ExternalKeyMapping.objects.count()) 1070 | self.assertFalse(ExternalKeyMapping.objects.filter( 1071 | external_system=external_system, 1072 | external_key='Key2').exists()) 1073 | 1074 | 1075 | class TestActionTypes(TestCase): 1076 | def test_model_action_returns_empty_string_for_type(self): 1077 | self.assertEquals('', ModelAction(ANY, ['field'], {'field': ''}).type) 1078 | 1079 | def test_create_model_action_returns_correct_type_string(self): 1080 | self.assertEquals('create', 1081 | CreateModelAction(ANY, ['field'], {'field': ''}).type) 1082 | 1083 | def test_update_model_action_returns_correct_type_string(self): 1084 | self.assertEquals('update', 1085 | UpdateModelAction(ANY, ['field'], {'field': ''}).type) 1086 | 1087 | def test_delete_model_action_returns_correct_type_string(self): 1088 | self.assertEquals('delete', 1089 | DeleteModelAction(ANY, ['field'], {'field': ''}).type) 1090 | 1091 | def test_delete_if_only_reference_model_action_returns_wrapped_action_type( 1092 | self): 1093 | delete_action = MagicMock() 1094 | sut = DeleteIfOnlyReferenceModelAction(ANY, ANY, delete_action) 1095 | self.assertEqual(delete_action.type, sut.type) 1096 | 1097 | def test_delete_external_reference_action_returns_correct_type_string( 1098 | self): 1099 | self.assertEquals('delete', 1100 | DeleteExternalReferenceAction(ANY, ANY).type) 1101 | 1102 | 1103 | -------------------------------------------------------------------------------- /tests/test_command_syncfile.py: -------------------------------------------------------------------------------- 1 | import tempfile 2 | from unittest.mock import MagicMock, patch 3 | 4 | from django.contrib.contenttypes.models import ContentType 5 | from django.core.management import call_command 6 | from django.core.management.base import CommandError 7 | from django.test import TestCase 8 | from nsync.management.commands.syncfile import SyncFileAction 9 | from nsync.models import ExternalKeyMapping, ExternalSystem 10 | 11 | from tests.models import TestHouse 12 | 13 | 14 | class TestSyncFileCommand(TestCase): 15 | def test_command_raises_error_if_file_does_not_exist(self): 16 | with self.assertRaises(CommandError): 17 | call_command('syncfile', 'systemName', 'tests', 'TestPerson', 18 | 'file') 19 | 20 | 21 | class TestSyncFileAction(TestCase): 22 | @patch('nsync.management.commands.syncfile.CsvActionFactory') 23 | @patch('csv.DictReader') 24 | def test_data_flow(self, DictReader, CsvActionFactory): 25 | file = MagicMock() 26 | row = MagicMock() 27 | row_provider = MagicMock() 28 | DictReader.return_value = row_provider 29 | row_provider.__iter__.return_value = [row] 30 | model_mock = MagicMock() 31 | external_system_mock = MagicMock() 32 | action_mock = MagicMock() 33 | CsvActionFactory.return_value.from_dict.return_value = [action_mock] 34 | SyncFileAction.sync(external_system_mock, model_mock, file, False) 35 | DictReader.assert_called_with(file) 36 | CsvActionFactory.assert_called_with(model_mock, external_system_mock) 37 | 38 | CsvActionFactory.return_value.from_dict.assert_called_with(row) 39 | action_mock.execute.assert_called_once_with() 40 | 41 | @patch('nsync.management.commands.syncfile.TransactionSyncPolicy') 42 | @patch('nsync.management.commands.syncfile.BasicSyncPolicy') 43 | @patch('nsync.management.commands.syncfile.CsvActionFactory') 44 | def test_it_wraps_the_basic_policy_in_a_transaction_policy_if_configured( 45 | self, CsvActionFactory, 46 | BasicSyncPolicy, TransactionSyncPolicy): 47 | SyncFileAction.sync(MagicMock(), MagicMock(), MagicMock(), True) 48 | TransactionSyncPolicy.assert_called_with(BasicSyncPolicy.return_value) 49 | TransactionSyncPolicy.return_value.execute.assert_called_once_with() 50 | 51 | 52 | class TestSyncSingleFileIntegrationTests(TestCase): 53 | def test_create_and_update(self): 54 | house1 = TestHouse.objects.create(address='House1') 55 | house2 = TestHouse.objects.create(address='House2') 56 | house3 = TestHouse.objects.create(address='House3', country='Belgium') 57 | house4 = TestHouse.objects.create(address='House4', country='Belgium') 58 | 59 | csv_file_obj = tempfile.NamedTemporaryFile(mode='w') 60 | csv_file_obj.writelines([ 61 | 'action_flags,match_on,address,country\n', 62 | 'c,address,House1,Australia\n', # Should have no effect 63 | 'u,address,House2,Australia\n', # Should update country 64 | 'u,address,House3,Australia\n', # Should have no effect 65 | 'u*,address,House4,Australia\n', # Should update country 66 | 'c,address,House5,Australia\n', # Should create new house 67 | ]) 68 | csv_file_obj.seek(0) 69 | 70 | call_command('syncfile', 'TestSystem', 'tests', 'TestHouse', 71 | csv_file_obj.name) 72 | 73 | for house in [house1, house2, house3, house4]: 74 | house.refresh_from_db() 75 | 76 | self.assertEqual(5, TestHouse.objects.count()) 77 | self.assertEqual('', house1.country) 78 | self.assertEqual('Australia', house2.country) 79 | self.assertEqual('Belgium', house3.country) 80 | self.assertEqual('Australia', house4.country) 81 | house5 = TestHouse.objects.get(address='House5') 82 | self.assertEqual('Australia', house5.country) 83 | 84 | def test_delete(self): 85 | house1 = TestHouse.objects.create(address='House1') 86 | house2 = TestHouse.objects.create(address='House2') 87 | 88 | csv_file_obj = tempfile.NamedTemporaryFile(mode='w') 89 | csv_file_obj.writelines([ 90 | 'action_flags,match_on,address,country\n', 91 | 'd,address,House1,Australia\n', # Should have no effect 92 | 'd*,address,House2,Australia\n', # Should delete 93 | ]) 94 | csv_file_obj.seek(0) 95 | 96 | call_command('syncfile', 'TestSystem', 'tests', 'TestHouse', 97 | csv_file_obj.name) 98 | 99 | house1.refresh_from_db() 100 | self.assertEqual('', house1.country) 101 | 102 | with self.assertRaises(Exception): 103 | house2.refresh_from_db() 104 | 105 | def test_create_and_update_with_external_refs(self): 106 | house1 = TestHouse.objects.create(address='House1') 107 | house2 = TestHouse.objects.create(address='House2') 108 | 109 | external_system = ExternalSystem.objects.create(name='TestSystem') 110 | 111 | house2mapping = ExternalKeyMapping.objects.create( 112 | content_type=ContentType.objects.get_for_model(TestHouse), 113 | external_system=external_system, 114 | external_key='House2Key', 115 | object_id=0) 116 | 117 | csv_file_obj = tempfile.NamedTemporaryFile(mode='w') 118 | csv_file_obj.writelines([ 119 | 'external_key,action_flags,match_on,address\n', 120 | 'House1Key,c,address,House1\n', # Should create a key mapping 121 | 'House2Key,u,address,House2\n', # Should update existing mapping 122 | 'House3Key,c,address,House3\n', 123 | # Should create new house and mapping 124 | ]) 125 | csv_file_obj.seek(0) 126 | 127 | call_command('syncfile', 'TestSystem', 'tests', 'TestHouse', 128 | csv_file_obj.name) 129 | 130 | for object in [house1, house2, house2mapping]: 131 | object.refresh_from_db() 132 | 133 | self.assertEqual(3, ExternalKeyMapping.objects.count()) 134 | self.assertEqual(house2mapping.object_id, house2.id) 135 | 136 | def test_delete_with_external_refs(self): 137 | TestHouse.objects.create(address='House1') 138 | house2 = TestHouse.objects.create(address='House2') 139 | house3 = TestHouse.objects.create(address='House3') 140 | house4 = TestHouse.objects.create(address='House4') 141 | 142 | external_system = ExternalSystem.objects.create( 143 | name='TestSystem', description='TestSystem') 144 | different_external_system = ExternalSystem.objects.create( 145 | name='DifferentSystem', description='DifferentSystem') 146 | 147 | ExternalKeyMapping.objects.create( 148 | content_type=ContentType.objects.get_for_model(TestHouse), 149 | external_system=external_system, 150 | external_key='House2Key', 151 | object_id=house2.id) 152 | 153 | ExternalKeyMapping.objects.create( 154 | content_type=ContentType.objects.get_for_model(TestHouse), 155 | external_system=different_external_system, 156 | external_key='House3Key', 157 | object_id=house3.id) 158 | 159 | ExternalKeyMapping.objects.create( 160 | content_type=ContentType.objects.get_for_model(TestHouse), 161 | external_system=different_external_system, 162 | external_key='House4Key', 163 | object_id=house4.id) 164 | 165 | csv_file_obj = tempfile.NamedTemporaryFile(mode='w') 166 | csv_file_obj.writelines([ 167 | 'external_key,action_flags,match_on,address\n', 168 | 'House1Key,d,address,House1\n', 169 | # Should do nothing, as this does not have the final mapping 170 | 'House2Key,d,address,House2\n', 171 | # Should delete, as this IS the final mapping 172 | 'House3Key,d,address,House3\n', 173 | # Should do nothing, as there is another mapping 174 | 'House4Key,d*,address,House4\n', 175 | # Should delete object but leave mapping, as it is forced 176 | ]) 177 | csv_file_obj.seek(0) 178 | 179 | call_command('syncfile', 'TestSystem', 'tests', 'TestHouse', 180 | csv_file_obj.name) 181 | 182 | self.assertTrue(TestHouse.objects.filter(address='House1').exists()) 183 | self.assertFalse(TestHouse.objects.filter(address='House2').exists()) 184 | self.assertTrue(TestHouse.objects.filter(address='House3').exists()) 185 | self.assertFalse(TestHouse.objects.filter(address='House4').exists()) 186 | 187 | self.assertFalse(ExternalKeyMapping.objects.filter( 188 | external_key='House1Key').exists()) 189 | self.assertFalse(ExternalKeyMapping.objects.filter( 190 | external_key='House2Key').exists()) 191 | self.assertTrue(ExternalKeyMapping.objects.filter( 192 | external_key='House3Key').exists()) 193 | self.assertTrue(ExternalKeyMapping.objects.filter( 194 | external_key='House4Key').exists()) 195 | -------------------------------------------------------------------------------- /tests/test_command_syncfiles.py: -------------------------------------------------------------------------------- 1 | import re 2 | import tempfile 3 | from unittest.mock import MagicMock, patch 4 | 5 | from django.contrib.contenttypes.models import ContentType 6 | from django.core.management import call_command 7 | from django.core.management.base import CommandError 8 | from django.test import TestCase 9 | from nsync.management.commands.syncfiles import TestableCommand, \ 10 | TargetExtractor, DEFAULT_FILE_REGEX 11 | from nsync.models import ExternalKeyMapping, ExternalSystem 12 | 13 | from tests.models import TestHouse 14 | 15 | 16 | class TestSyncFilesCommand(TestCase): 17 | def test_command_raises_error_if_file_list_is_empty(self): 18 | with self.assertRaises(CommandError): 19 | call_command('syncfiles') 20 | 21 | def test_command_raises_error_if_regex_does_not_compile(self): 22 | f = tempfile.NamedTemporaryFile() 23 | import sre_constants 24 | with self.assertRaises(sre_constants.error): 25 | call_command('syncfiles', f.name, file_name_regex='(((') 26 | 27 | def xtest_test(self): 28 | f1 = tempfile.NamedTemporaryFile(mode='w', 29 | prefix='Temp_tests_TestHouse_', 30 | suffix='.csv') 31 | f2 = tempfile.NamedTemporaryFile() 32 | f3 = tempfile.NamedTemporaryFile() 33 | 34 | call_command('syncfiles', f1.name, f2.name, f3.name) 35 | 36 | 37 | class TestTestableCommand(TestCase): 38 | def setUp(self): 39 | self.defaults = { 40 | 'files': '', 41 | 'file_name_regex': '', 42 | 'create_external_system': '', 43 | 'smart_ordering': '', 44 | 'as_transaction': '' 45 | } 46 | 47 | @patch('nsync.management.commands.syncfiles.BasicSyncPolicy') 48 | def test_it_uses_the_basic_policy_if_smart_ordering_is_false(self, Policy): 49 | self.defaults['smart_ordering'] = False 50 | actions_list = MagicMock() 51 | 52 | with patch.object(TestableCommand, 'collect_all_actions', 53 | return_value=actions_list): 54 | sut = TestableCommand(**self.defaults) 55 | sut.execute() 56 | Policy.assert_called_with(actions_list) 57 | Policy.return_value.execute.assert_called_once_with() 58 | 59 | @patch('nsync.management.commands.syncfiles.OrderedSyncPolicy') 60 | def test_it_uses_the_ordered_policy_if_smart_ordering_is_true(self, 61 | Policy): 62 | self.defaults['smart_ordering'] = True 63 | actions_list = MagicMock() 64 | 65 | with patch.object(TestableCommand, 'collect_all_actions', 66 | return_value=actions_list): 67 | sut = TestableCommand(**self.defaults) 68 | sut.execute() 69 | Policy.assert_called_with(actions_list) 70 | Policy.return_value.execute.assert_called_once_with() 71 | 72 | @patch('nsync.management.commands.syncfiles.TransactionSyncPolicy') 73 | @patch('nsync.management.commands.syncfiles.BasicSyncPolicy') 74 | def test_it_wraps_the_basic_policy_in_a_transaction_policy_if_configured( 75 | self, BasicSyncPolicy, TransactionSyncPolicy): 76 | self.defaults['smart_ordering'] = False 77 | self.defaults['as_transaction'] = True 78 | actions_list = MagicMock() 79 | 80 | with patch.object(TestableCommand, 'collect_all_actions', 81 | return_value=actions_list): 82 | sut = TestableCommand(**self.defaults) 83 | sut.execute() 84 | TransactionSyncPolicy.assert_called_with( 85 | BasicSyncPolicy.return_value) 86 | TransactionSyncPolicy.return_value \ 87 | .execute.assert_called_once_with() 88 | 89 | @patch('nsync.management.commands.syncfiles.SupportedFileChecker') 90 | def test_command_raises_error_if_not_CSV_file(self, SupportedFileChecker): 91 | SupportedFileChecker.is_valid.return_value = False 92 | files_mock = MagicMock() 93 | sut = TestableCommand(**{ 94 | 'files': [files_mock], 95 | 'file_name_regex': DEFAULT_FILE_REGEX, 96 | 'create_external_system': MagicMock(), 97 | 'smart_ordering': MagicMock(), 98 | 'as_transaction': MagicMock() 99 | }) 100 | 101 | with self.assertRaises(CommandError): 102 | sut.execute() 103 | SupportedFileChecker.is_valid.assert_called_with(files_mock) 104 | 105 | 106 | class TestTargetExtractor(TestCase): 107 | def setUp(self): 108 | self.sut = TargetExtractor(re.compile(DEFAULT_FILE_REGEX)) 109 | 110 | def test_it_extracts_the_correct_strings_from_the_filename(self): 111 | self.assertEquals(('System', 'App', 'Model'), 112 | self.sut.extract('System_App_Model.csv')) 113 | self.assertEquals(('System', 'App', 'Model'), 114 | self.sut.extract('System_App_Model_1234.csv')) 115 | self.assertEquals(('ABCabc123', 'App', 'Model'), 116 | self.sut.extract('ABCabc123_App_Model.csv')) 117 | 118 | 119 | class TestSyncSingleFileIntegrationTests(TestCase): 120 | def test_create_and_update(self): 121 | house1 = TestHouse.objects.create(address='House1') 122 | house2 = TestHouse.objects.create(address='House2') 123 | house3 = TestHouse.objects.create(address='House3', country='Belgium') 124 | house4 = TestHouse.objects.create(address='House4', country='Belgium') 125 | 126 | csv_file_obj = tempfile.NamedTemporaryFile( 127 | mode='w', prefix='TestSystem_tests_TestHouse_', suffix='.csv') 128 | csv_file_obj.writelines([ 129 | 'action_flags,match_on,address,country\n', 130 | 'c,address,House1,Australia\n', # Should have no effect 131 | 'u,address,House2,Australia\n', # Should update country 132 | 'u,address,House3,Australia\n', # Should have no effect 133 | 'u*,address,House4,Australia\n', # Should update country 134 | 'c,address,House5,Australia\n', # Should create new house 135 | ]) 136 | csv_file_obj.seek(0) 137 | 138 | call_command('syncfiles', csv_file_obj.name) 139 | 140 | for house in [house1, house2, house3, house4]: 141 | house.refresh_from_db() 142 | 143 | self.assertEqual(5, TestHouse.objects.count()) 144 | self.assertEqual('', house1.country) 145 | self.assertEqual('Australia', house2.country) 146 | self.assertEqual('Belgium', house3.country) 147 | self.assertEqual('Australia', house4.country) 148 | house5 = TestHouse.objects.get(address='House5') 149 | self.assertEqual('Australia', house5.country) 150 | 151 | def test_delete(self): 152 | house1 = TestHouse.objects.create(address='House1') 153 | house2 = TestHouse.objects.create(address='House2') 154 | 155 | csv_file_obj = tempfile.NamedTemporaryFile( 156 | mode='w', prefix='TestSystem_tests_TestHouse_', suffix='.csv') 157 | csv_file_obj.writelines([ 158 | 'action_flags,match_on,address,country\n', 159 | 'd,address,House1,Australia\n', # Should have no effect 160 | 'd*,address,House2,Australia\n', # Should delete 161 | ]) 162 | csv_file_obj.seek(0) 163 | 164 | call_command('syncfiles', csv_file_obj.name) 165 | 166 | house1.refresh_from_db() 167 | self.assertEqual('', house1.country) 168 | 169 | with self.assertRaises(Exception): 170 | house2.refresh_from_db() 171 | 172 | def test_create_and_update_with_external_refs(self): 173 | house1 = TestHouse.objects.create(address='House1') 174 | house2 = TestHouse.objects.create(address='House2') 175 | 176 | external_system = ExternalSystem.objects.create(name='TestSystem') 177 | 178 | house2mapping = ExternalKeyMapping.objects.create( 179 | content_type=ContentType.objects.get_for_model(TestHouse), 180 | external_system=external_system, 181 | external_key='House2Key', 182 | object_id=0) 183 | 184 | csv_file_obj = tempfile.NamedTemporaryFile( 185 | mode='w', prefix='TestSystem_tests_TestHouse_', suffix='.csv') 186 | csv_file_obj.writelines([ 187 | 'external_key,action_flags,match_on,address\n', 188 | 'House1Key,c,address,House1\n', # Should create a key mapping 189 | 'House2Key,u,address,House2\n', # Should update existing mapping 190 | 'House3Key,c,address,House3\n', 191 | # Should create new house and mapping 192 | ]) 193 | csv_file_obj.seek(0) 194 | 195 | call_command('syncfiles', csv_file_obj.name) 196 | 197 | for object in [house1, house2, house2mapping]: 198 | object.refresh_from_db() 199 | 200 | self.assertEqual(3, ExternalKeyMapping.objects.count()) 201 | self.assertEqual(house2mapping.object_id, house2.id) 202 | 203 | def test_delete_with_external_refs(self): 204 | TestHouse.objects.create(address='House1') 205 | house2 = TestHouse.objects.create(address='House2') 206 | house3 = TestHouse.objects.create(address='House3') 207 | house4 = TestHouse.objects.create(address='House4') 208 | 209 | external_system = ExternalSystem.objects.create( 210 | name='TestSystem', description='TestSystem') 211 | different_external_system = ExternalSystem.objects.create( 212 | name='DifferentSystem', description='DifferentSystem') 213 | 214 | ExternalKeyMapping.objects.create( 215 | content_type=ContentType.objects.get_for_model(TestHouse), 216 | external_system=external_system, 217 | external_key='House2Key', 218 | object_id=house2.id) 219 | 220 | ExternalKeyMapping.objects.create( 221 | content_type=ContentType.objects.get_for_model(TestHouse), 222 | external_system=different_external_system, 223 | external_key='House3Key', 224 | object_id=house3.id) 225 | 226 | ExternalKeyMapping.objects.create( 227 | content_type=ContentType.objects.get_for_model(TestHouse), 228 | external_system=different_external_system, 229 | external_key='House4Key', 230 | object_id=house4.id) 231 | 232 | csv_file_obj = tempfile.NamedTemporaryFile( 233 | mode='w', prefix='TestSystem_tests_TestHouse_', suffix='.csv') 234 | csv_file_obj.writelines([ 235 | 'external_key,action_flags,match_on,address\n', 236 | 'House1Key,d,address,House1\n', 237 | # Should do nothing, as this does not have the final mapping 238 | 'House2Key,d,address,House2\n', 239 | # Should delete, as this IS the final mapping 240 | 'House3Key,d,address,House3\n', 241 | # Should do nothing, as there is another mapping 242 | 'House4Key,d*,address,House4\n', 243 | # Should delete object but leave mapping, as it is forced 244 | ]) 245 | csv_file_obj.seek(0) 246 | 247 | call_command('syncfiles', csv_file_obj.name) 248 | 249 | self.assertTrue(TestHouse.objects.filter(address='House1').exists()) 250 | self.assertFalse(TestHouse.objects.filter(address='House2').exists()) 251 | self.assertTrue(TestHouse.objects.filter(address='House3').exists()) 252 | self.assertFalse(TestHouse.objects.filter(address='House4').exists()) 253 | 254 | self.assertFalse(ExternalKeyMapping.objects.filter( 255 | external_key='House1Key').exists()) 256 | self.assertFalse(ExternalKeyMapping.objects.filter( 257 | external_key='House2Key').exists()) 258 | self.assertTrue(ExternalKeyMapping.objects.filter( 259 | external_key='House3Key').exists()) 260 | self.assertTrue(ExternalKeyMapping.objects.filter( 261 | external_key='House4Key').exists()) 262 | 263 | def test_it_does_all_creates_before_deletes(self): 264 | 265 | file1 = tempfile.NamedTemporaryFile( 266 | mode='w', prefix='TestSystem1_tests_TestHouse_', suffix='.csv') 267 | file1.writelines([ 268 | 'action_flags,match_on,address,country\n', 269 | 'd*,address,House1,Australia\n', # Should delete 270 | ]) 271 | file1.seek(0) 272 | 273 | file2 = tempfile.NamedTemporaryFile( 274 | mode='w', prefix='TestSystem2_tests_TestHouse_', suffix='.csv') 275 | file2.writelines([ 276 | 'action_flags,match_on,address,country\n', 277 | 'c,address,House1,Australia\n', 278 | # Should attempt to create, but should be undone by delete above 279 | ]) 280 | file2.seek(0) 281 | call_command('syncfiles', file1.name, file2.name) 282 | 283 | self.assertEqual(0, TestHouse.objects.count()) 284 | -------------------------------------------------------------------------------- /tests/test_command_utils.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import MagicMock, patch 2 | 3 | from django.core.management.base import CommandError 4 | from django.test import TestCase 5 | from nsync.actions import SyncActions 6 | from nsync.management.commands.utils import ( 7 | ExternalSystemHelper, 8 | ModelFinder, 9 | SupportedFileChecker, 10 | CsvSyncActionsDecoder, 11 | CsvSyncActionsEncoder, 12 | CsvActionFactory) 13 | 14 | 15 | class TestExternalSystemHelper(TestCase): 16 | def test_find_raises_error_if_external_system_name_is_blank(self): 17 | with self.assertRaises(CommandError): 18 | ExternalSystemHelper.find('') 19 | 20 | def test_find_raises_error_if_external_system_not_found(self): 21 | with self.assertRaises(CommandError): 22 | ExternalSystemHelper.find('systemName', False) 23 | 24 | @patch('nsync.management.commands.utils.ExternalSystem') 25 | def test_find_creates_external_system_if_not_found_and_create_is_true( 26 | self, ExternalSystem): 27 | ExternalSystem.DoesNotExist = Exception 28 | ExternalSystem.objects.get.side_effect = ExternalSystem.DoesNotExist 29 | ExternalSystemHelper.find('systemName', True) 30 | ExternalSystem.objects.create.assert_called_with( 31 | name='systemName', description='systemName') 32 | 33 | 34 | class TestModelFinder(TestCase): 35 | def test_find_raises_error_if_app_label_is_blank(self): 36 | with self.assertRaises(CommandError): 37 | ModelFinder.find('', 'model') 38 | 39 | def test_find_raises_error_if_model_name_is_blank(self): 40 | with self.assertRaises(CommandError): 41 | ModelFinder.find('nsync', '') 42 | 43 | def test_it_raises_an_error_if_the_model_cannot_be_found(self): 44 | with self.assertRaises(LookupError): 45 | ModelFinder.find('fakeApp', 'missingModel') 46 | 47 | def test_it_returns_the_model_if_found(self): 48 | from tests.models import TestPerson 49 | result = ModelFinder.find('tests', 'TestPerson') 50 | self.assertEqual(result, TestPerson) 51 | 52 | 53 | class TestSupportedFileChecker(TestCase): 54 | def test_it_thinks_none_file_is_not_valid(self): 55 | self.assertFalse(SupportedFileChecker.is_valid(None)) 56 | 57 | 58 | class TestCsvSyncActionsEncoder(TestCase): 59 | def test_it_encodes_as_expected(self): 60 | self.assertEqual('c', CsvSyncActionsEncoder.encode( 61 | SyncActions(create=True))) 62 | self.assertEqual('u', CsvSyncActionsEncoder.encode( 63 | SyncActions(update=True))) 64 | self.assertEqual('u*', CsvSyncActionsEncoder.encode( 65 | SyncActions(update=True, force=True))) 66 | self.assertEqual('d', CsvSyncActionsEncoder.encode( 67 | SyncActions(delete=True))) 68 | self.assertEqual('d*', CsvSyncActionsEncoder.encode( 69 | SyncActions(delete=True, force=True))) 70 | 71 | 72 | class TestCsvSyncActionsDecoder(TestCase): 73 | def test_it_is_case_insensitive_when_decoding(self): 74 | self.assertTrue(CsvSyncActionsDecoder.decode('c').create) 75 | self.assertTrue(CsvSyncActionsDecoder.decode('C').create) 76 | self.assertTrue(CsvSyncActionsDecoder.decode('u').update) 77 | self.assertTrue(CsvSyncActionsDecoder.decode('U').update) 78 | self.assertTrue(CsvSyncActionsDecoder.decode('d').delete) 79 | self.assertTrue(CsvSyncActionsDecoder.decode('D').delete) 80 | 81 | def test_it_produces_object_with_no_actions_if_input_invalid(self): 82 | result = CsvSyncActionsDecoder.decode(123) 83 | self.assertFalse(result.create) 84 | self.assertFalse(result.update) 85 | self.assertFalse(result.delete) 86 | self.assertFalse(result.force) 87 | 88 | 89 | class TestCsvActionFactory(TestCase): 90 | def setUp(self): 91 | self.model = MagicMock() 92 | self.sut = CsvActionFactory(self.model) 93 | 94 | @patch('nsync.management.commands.utils.CsvSyncActionsDecoder') 95 | def test_from_dict_maps_to_build_correctly(self, ActionDecoder): 96 | action_flags_mock = MagicMock() 97 | match_on_mock = MagicMock() 98 | external_key_mock = MagicMock() 99 | 100 | with patch.object(self.sut, 'build') as build_method: 101 | result = self.sut.from_dict({ 102 | 'action_flags': action_flags_mock, 103 | 'match_on': match_on_mock, 104 | 'external_key': external_key_mock, 105 | 'other_key': 'value'}) 106 | ActionDecoder.decode.assert_called_with(action_flags_mock) 107 | match_on_mock.split.assert_called_with( 108 | CsvActionFactory.match_on_delimiter) 109 | build_method.assert_called_with( 110 | ActionDecoder.decode.return_value, 111 | match_on_mock.split.return_value, 112 | external_key_mock, 113 | {'other_key': 'value'}) 114 | self.assertEqual(build_method.return_value, result) 115 | 116 | def test_returns_an_empty_list_if_no_actions_in_input(self): 117 | self.assertEqual([], self.sut.from_dict(None)) 118 | 119 | def test_it_raises_an_error_if_the_action_flag_key_is_not_in_values(self): 120 | with self.assertRaises(KeyError): 121 | self.sut.from_dict({'not_matched': 'value'}) 122 | 123 | def test_it_raises_an_error_if_the_match_field_key_is_not_in_values(self): 124 | with self.assertRaises(KeyError): 125 | self.sut.from_dict({'action_flags': ''}) 126 | -------------------------------------------------------------------------------- /tests/test_integrations.py: -------------------------------------------------------------------------------- 1 | import re 2 | import tempfile 3 | from unittest.mock import MagicMock, patch 4 | 5 | from django.contrib.contenttypes.models import ContentType 6 | from django.core.management import call_command 7 | from django.core.management.base import CommandError 8 | from django.test import TestCase 9 | from nsync.management.commands.syncfiles import TestableCommand, \ 10 | TargetExtractor, DEFAULT_FILE_REGEX 11 | from nsync.models import ExternalKeyMapping, ExternalSystem 12 | 13 | from tests.models import TestHouse 14 | 15 | 16 | class TestIntegrations(TestCase): 17 | def setUp(self): 18 | self.external_system = ExternalSystem.objects.create(name='System') 19 | 20 | def test_forced_update_changes_match_fields_if_external_mapping_already_exists(self): 21 | # Setup the objects 22 | csv_file_obj = tempfile.NamedTemporaryFile( 23 | mode='w', prefix='TestSystem_tests_TestHouse_', suffix='.csv') 24 | csv_file_obj.writelines([ 25 | 'external_key,action_flags,match_on,address,country\n', 26 | 'ExternalId1,c,address,House1,Australia\n', # Will create the house 27 | ]) 28 | csv_file_obj.seek(0) 29 | 30 | call_command('syncfiles', csv_file_obj.name) 31 | 32 | # Validate the objects 33 | self.assertEqual(1, TestHouse.objects.count()) 34 | self.assertEqual(1, ExternalKeyMapping.objects.count()) 35 | house = TestHouse.objects.first() 36 | mapping = ExternalKeyMapping.objects.first() 37 | self.assertEqual(house, mapping.content_object) 38 | self.assertEqual(house.address, 'House1') 39 | self.assertEqual(house.country, 'Australia') 40 | 41 | # Build the forced update 42 | csv_file_obj = tempfile.NamedTemporaryFile( 43 | mode='w', prefix='TestSystem_tests_TestHouse_', suffix='.csv') 44 | csv_file_obj.writelines([ 45 | 'external_key,action_flags,match_on,address,country\n', 46 | 'ExternalId1,u*,address,A new address,A different country\n', # Will update the house 47 | ]) 48 | csv_file_obj.seek(0) 49 | 50 | call_command('syncfiles', csv_file_obj.name) 51 | 52 | # Validate the objects 53 | mapping.refresh_from_db() 54 | house.refresh_from_db() 55 | 56 | self.assertEqual(mapping, ExternalKeyMapping.objects.first()) 57 | self.assertEqual(1, TestHouse.objects.count()) 58 | self.assertEqual(1, ExternalKeyMapping.objects.count()) 59 | self.assertEqual(house, mapping.content_object) 60 | self.assertEqual(house.address, 'A new address') 61 | self.assertEqual(house.country, 'A different country') 62 | 63 | def test_multiple_match_fields_select_correct_object(self): 64 | house1 = TestHouse.objects.create(address='BigHouse', country='BigCountry') 65 | house2 = TestHouse.objects.create(address='SmallHouse', country='BigCountry') 66 | house3 = TestHouse.objects.create(address='BigHouse', country='SmallCountry') 67 | house4 = TestHouse.objects.create(address='SmallHouse', country='SmallCountry') 68 | 69 | csv_file_obj = tempfile.NamedTemporaryFile(mode='w') 70 | csv_file_obj.writelines([ 71 | 'action_flags,match_on,address,country,floors\n', 72 | 'cu*,address country,BigHouse,BigCountry,1\n', 73 | 'cu*,address country,SmallHouse,BigCountry,2\n', 74 | 'cu*,address country,BigHouse,SmallCountry,3\n', 75 | 'cu*,address country,SmallHouse,SmallCountry,4\n', 76 | ]) 77 | csv_file_obj.seek(0) 78 | 79 | call_command('syncfile', 'TestSystem', 'tests', 'TestHouse', 80 | csv_file_obj.name) 81 | 82 | for object in [house1, house2, house3, house4]: 83 | object.refresh_from_db() 84 | 85 | self.assertEqual(1, house1.floors) 86 | self.assertEqual(2, house2.floors) 87 | self.assertEqual(3, house3.floors) 88 | self.assertEqual(4, house4.floors) 89 | 90 | def test_ORd_match_fields_select_correct_object(self): 91 | house1 = TestHouse.objects.create(address='OnlyAddress') 92 | house2 = TestHouse.objects.create(country='OnlyCountry') 93 | house3 = TestHouse.objects.create(address='BothAddress', country='BothCountry') 94 | 95 | csv_file_obj = tempfile.NamedTemporaryFile(mode='w') 96 | csv_file_obj.writelines([ 97 | 'action_flags,match_on,address,country,floors\n', 98 | 'cu*,address country |,OnlyAddress,,1\n', 99 | 'cu*,address country |,,OnlyCountry,2\n', 100 | 'cu*,address country |,BothAddress,BothCountry,3\n', 101 | ]) 102 | csv_file_obj.seek(0) 103 | 104 | call_command('syncfile', 'TestSystem', 'tests', 'TestHouse', 105 | csv_file_obj.name) 106 | 107 | for object in [house1, house2, house3]: 108 | object.refresh_from_db() 109 | 110 | self.assertEqual(1, house1.floors) 111 | self.assertEqual(2, house2.floors) 112 | self.assertEqual(3, house3.floors) 113 | 114 | -------------------------------------------------------------------------------- /tests/test_models.py: -------------------------------------------------------------------------------- 1 | from django.contrib.contenttypes.fields import ContentType 2 | from django.test import TestCase 3 | from nsync.models import ExternalSystem, ExternalKeyMapping 4 | 5 | from tests.models import TestPerson 6 | 7 | 8 | class TestExternalSystem(TestCase): 9 | def test_it_uses_name_for_string(self): 10 | self.assertEqual('SystemName', str(ExternalSystem(name='SystemName'))) 11 | 12 | def test_it_returns_the_description_instead_of_name_if_available( 13 | self): 14 | sut = ExternalSystem(name='SystemName', 15 | description='SystemDescription') 16 | self.assertEqual('SystemDescription', str(sut)) 17 | 18 | 19 | class TestExternalKeyMapping(TestCase): 20 | def setUp(self): 21 | self.external_system = ExternalSystem.objects.create( 22 | name='ExternalSystemName') 23 | 24 | def test_it_returns_as_useful_string(self): 25 | john = TestPerson.objects.create(first_name='John') 26 | content_type = ContentType.objects.get_for_model(TestPerson) 27 | sut = ExternalKeyMapping( 28 | external_system=self.external_system, 29 | external_key='Person123', 30 | content_type=content_type, 31 | content_object=john, 32 | object_id=john.id) 33 | 34 | result = str(sut) 35 | self.assertIn('ExternalSystemName', result) 36 | self.assertIn('Person123', result) 37 | self.assertIn(content_type.model_class().__name__, result) 38 | self.assertIn(str(john.id), result) 39 | -------------------------------------------------------------------------------- /tests/test_policies.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import MagicMock, call 2 | 3 | from django.test import TestCase 4 | from nsync.policies import BasicSyncPolicy, OrderedSyncPolicy 5 | 6 | 7 | class TestBasicSyncPolicy(TestCase): 8 | def test_it_calls_execute_for_all_actions(self): 9 | actions = [MagicMock(), MagicMock(), MagicMock()] 10 | BasicSyncPolicy(actions).execute() 11 | for action in actions: 12 | action.execute.assert_called_once_with() 13 | 14 | 15 | class TestOrderedSyncPolicy(TestCase): 16 | def test_it_calls_in_order_create_update_delete(self): 17 | execute_mock = MagicMock() 18 | 19 | def make_mock(type): 20 | mock = MagicMock() 21 | mock.type = type 22 | return mock 23 | 24 | create_action = make_mock('create') 25 | update_action = make_mock('update') 26 | delete_action = make_mock('delete') 27 | execute_mock.create = create_action 28 | execute_mock.update = update_action 29 | execute_mock.delete = delete_action 30 | 31 | OrderedSyncPolicy([ 32 | delete_action, 33 | create_action, 34 | update_action]).execute() 35 | 36 | execute_mock.assert_has_calls([ 37 | call.create.execute(), 38 | call.update.execute(), 39 | call.delete.execute()]) 40 | -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | 4 | # http://stackoverflow.com/questions/899067/how-should-i-verify-a-log-message-when-testing-python-code-under-nose/20553331#20553331 5 | class MockLoggingHandler(logging.Handler): 6 | """Mock logging handler to check for expected logs. 7 | 8 | Messages are available from an instance's ``messages`` dict, in order, indexed by 9 | a lowercase log level string (e.g., 'debug', 'info', etc.). 10 | """ 11 | 12 | def __init__(self, *args, **kwargs): 13 | self.messages = {'debug': [], 'info': [], 'warning': [], 'error': [], 14 | 'critical': []} 15 | super(MockLoggingHandler, self).__init__(*args, **kwargs) 16 | 17 | def emit(self, record): 18 | "Store a message from ``record`` in the instance's ``messages`` dict." 19 | self.acquire() 20 | try: 21 | self.messages[record.levelname.lower()].append(record.getMessage()) 22 | finally: 23 | self.release() 24 | 25 | def reset(self): 26 | self.acquire() 27 | try: 28 | for message_list in self.messages.values(): 29 | message_list.clear() 30 | finally: 31 | self.release() 32 | 33 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = 3 | py33-django{18}, 4 | py{34,35}-django{18,19,110}, 5 | py{34,35,36}-django{20} 6 | 7 | [testenv] 8 | #setenv = 9 | # PYTHONPATH = {toxinidir}:{toxinidir}/nsync 10 | #commands = python runtests.py 11 | #deps = 12 | # -r{toxinidir}/requirements-test.txt 13 | 14 | basepython = 15 | py33: python3.3 16 | py34: python3.4 17 | py35: python3.5 18 | py36: python3.6 19 | 20 | deps = 21 | coverage >= 4.0 22 | django18: Django>=1.8,<1.9 23 | django19: Django>=1.9,<1.10 24 | django110: Django>=1.10,<1.11 25 | django20: Django>=2.0,<2.1 26 | 27 | commands = coverage run -a setup.py test 28 | #commands = coverage run -a runtests.py 29 | --------------------------------------------------------------------------------